-
Notifications
You must be signed in to change notification settings - Fork 14
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
1 parent
4561129
commit 6b89826
Showing
13 changed files
with
261 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,60 @@ | ||
FROM debian:stretch | ||
|
||
COPY entrypoint /entrypoint | ||
RUN chmod 755 /entrypoint | ||
|
||
# LIGHT DEPENDENCIES START | ||
RUN apt-get update | ||
RUN apt-get install -y ftp wget curl unzip telnet openssh-client | ||
# LIGHT DEPENDENCIES END | ||
|
||
# INSTALL JAVA & BEELINE START | ||
# INSTALL JDK8 | ||
RUN apt-get install -y openjdk-8-jdk | ||
ENV JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64 | ||
ENV PATH=$PATH:$JAVA_HOME/bin | ||
|
||
# Hadoop client installation | ||
RUN wget --no-verbose https://archive.cloudera.com/cdh5/cdh/5/hadoop-2.6.0-cdh5.13.0.tar.gz \ | ||
&& tar -xzf hadoop-2.6.0-cdh5.13.0.tar.gz \ | ||
&& rm hadoop-2.6.0-cdh5.13.0.tar.gz \ | ||
&& mv hadoop-2.6.0-cdh5.13.0 /usr/lib/hadoop | ||
|
||
ENV HADOOP_PREFIX=/usr/lib/hadoop | ||
ENV PATH=$PATH:$HADOOP_PREFIX/bin | ||
ENV HADOOP_COMMON_HOME=$HADOOP_PREFIX | ||
ENV HADOOP_CONF_DIR=/etc/hadoop/conf | ||
|
||
# Hive client installation | ||
RUN wget --no-verbose http://archive.cloudera.com/cdh5/cdh/5/hive-1.1.0-cdh5.13.3.tar.gz \ | ||
&& tar -xvzf hive-1.1.0-cdh5.13.3.tar.gz \ | ||
&& rm hive-1.1.0-cdh5.13.3.tar.gz | ||
|
||
ENV HIVE_HOME=/hive-1.1.0-cdh5.13.3 | ||
ENV PATH=$PATH:$HIVE_HOME/bin | ||
ENV HIVE_CONF_DIR=/etc/hive/conf | ||
# INSTALL JAVA & BEELINE END | ||
|
||
# Sqoop client installation | ||
RUN wget --no-verbose http://archive.cloudera.com/cdh5/cdh/5/sqoop-1.4.6-cdh5.13.0.tar.gz \ | ||
&& tar -xvzf sqoop-1.4.6-cdh5.13.0.tar.gz \ | ||
&& rm sqoop-1.4.6-cdh5.13.0.tar.gz \ | ||
&& cp /hive-1.1.0-cdh5.13.3/lib/*.jar /sqoop-1.4.6-cdh5.13.0/lib/ | ||
|
||
ENV SQOOP_HOME=/sqoop-1.4.6-cdh5.13.0 | ||
ENV PATH=$PATH:$SQOOP_HOME/bin | ||
ENV HADOOP_HOME=$HADOOP_PREFIX | ||
|
||
# INSTALL PROCPS (ps) START | ||
RUN apt-get install -y procps | ||
# INSTALL PROCPS (ps) END | ||
|
||
#UPDATE SECURITY PACKAGE BEGIN | ||
RUN apt-get update && apt-get upgrade -y | ||
#UPDATE SECURITY PACKAGE END | ||
|
||
COPY lib/*.jar /sqoop-1.4.6-cdh5.13.0/lib/ | ||
|
||
WORKDIR /sandbox | ||
|
||
ENTRYPOINT ["bash","/entrypoint"] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,22 @@ | ||
/* | ||
* SPDX-License-Identifier: Apache-2.0 | ||
* | ||
* Copyright 2019 Pierre Leresteux. | ||
* | ||
* Licensed under the Apache License, Version 2.0 (the "License"); | ||
* you may not use this file except in compliance with the License. | ||
* You may obtain a copy of the License at | ||
* | ||
* https://www.apache.org/licenses/LICENSE-2.0 | ||
* | ||
* Unless required by applicable law or agreed to in writing, software | ||
* distributed under the License is distributed on an "AS IS" BASIS, | ||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
* See the License for the specific language governing permissions and | ||
* limitations under the License. | ||
*/ | ||
import com.bmuschko.gradle.docker.DockerRemoteApiPlugin | ||
import com.saagie.technologies.SaagieTechnologiesGradlePlugin | ||
|
||
apply<DockerRemoteApiPlugin>() | ||
apply<SaagieTechnologiesGradlePlugin>() |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,8 @@ | ||
#!/bin/bash | ||
|
||
set -euo pipefail | ||
|
||
if test -f main_script; | ||
then ln -s $HIVE_CONF_DIR/hive-site.xml; sh ./main_script; | ||
else exec "$@" | ||
fi; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,125 @@ | ||
schemaVersion: "2.0.0" | ||
|
||
metadataTest: | ||
env: | ||
- key: JAVA_HOME | ||
value: "/usr/lib/jvm/java-8-openjdk-amd64" | ||
- key: "HADOOP_HOME" | ||
value: "/usr/lib/hadoop" | ||
- key: "HIVE_HOME" | ||
value: "/hive-1.1.0-cdh5.13.3" | ||
- key: "HADOOP_CONF_DIR" | ||
value: "/etc/hadoop/conf" | ||
- key: "HIVE_CONF_DIR" | ||
value: "/etc/hive/conf" | ||
- key: "SQOOP_HOME" | ||
value: "/sqoop-1.4.6-cdh5.13.0" | ||
|
||
fileExistenceTests: | ||
- name: "entrypoint" | ||
path: "/entrypoint" | ||
shouldExist: true | ||
permissions: "-rwxr-xr-x" | ||
|
||
fileContentTests: | ||
- name: "entrypoint" | ||
path: "/entrypoint" | ||
expectedContents: ["ln -s \\$HIVE_CONF_DIR/hive-site.xml", "sh ./main_script"] | ||
|
||
commandTests: | ||
- name: "Workdir" | ||
command: "pwd" | ||
expectedOutput: ["/sandbox"] | ||
|
||
- name: "ftp" | ||
args: ["-h"] | ||
command: "ftp" | ||
exitCode: 0 | ||
|
||
- name: "wget" | ||
args: ["--help"] | ||
command: "wget" | ||
exitCode: 0 | ||
|
||
- name: "curl" | ||
args: ["--help"] | ||
command: "curl" | ||
exitCode: 0 | ||
|
||
- name: "apt-get" | ||
args: ["--help"] | ||
command: "apt-get" | ||
exitCode: 0 | ||
|
||
- name: "unzip" | ||
args: ["--help"] | ||
command: "unzip" | ||
exitCode: 0 | ||
|
||
- name: "tar" | ||
args: ["--help"] | ||
command: "tar" | ||
exitCode: 0 | ||
|
||
- name: "awk" | ||
command: "awk" | ||
exitCode: 0 | ||
|
||
- name: "telnet" | ||
command: "which" | ||
args: ["telnet"] | ||
expectedOutput: ["/usr/bin/telnet"] | ||
exitCode: 0 | ||
|
||
- name: "scp" | ||
command: "which" | ||
args: ["scp"] | ||
expectedOutput: ["/usr/bin/scp"] | ||
exitCode: 0 | ||
|
||
- name: "ping" | ||
command: "which" | ||
args: ["ping"] | ||
expectedOutput: ["/bin/ping"] | ||
exitCode: 0 | ||
|
||
- name: "wc" | ||
command: "wc" | ||
args: ["--help"] | ||
exitCode: 0 | ||
|
||
- name: "grep" | ||
args: ["--help"] | ||
command: "grep" | ||
exitCode: 0 | ||
|
||
- name: "sleep" | ||
args: ["--help"] | ||
command: "sleep" | ||
exitCode: 0 | ||
|
||
- name: "java" | ||
args: ["-version"] | ||
command: "java" | ||
exitCode: 0 | ||
|
||
- name: "hadoop" | ||
args: ["--help"] | ||
command: "hadoop" | ||
exitCode: 0 | ||
|
||
- name: "beeline" | ||
args: ["--help"] | ||
command: "beeline" | ||
exitCode: 0 | ||
|
||
- name: "sqoop" | ||
args: ["help"] | ||
command: "sqoop" | ||
exitCode: 0 | ||
|
||
- name: "lib" | ||
args: ["-c", "ls /sqoop-1.4.6-cdh5.13.0/lib/*.jar | wc -l"] | ||
command: "sh" | ||
expectedOutput: ["162"] | ||
exitCode: 0 |
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,18 @@ | ||
/* | ||
* SPDX-License-Identifier: Apache-2.0 | ||
* | ||
* Copyright 2019 Pierre Leresteux. | ||
* | ||
* Licensed under the Apache License, Version 2.0 (the "License"); | ||
* you may not use this file except in compliance with the License. | ||
* You may obtain a copy of the License at | ||
* | ||
* https://www.apache.org/licenses/LICENSE-2.0 | ||
* | ||
* Unless required by applicable law or agreed to in writing, software | ||
* distributed under the License is distributed on an "AS IS" BASIS, | ||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
* See the License for the specific language governing permissions and | ||
* limitations under the License. | ||
*/ | ||
rootProject.name = "sqoop1.4.6" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,20 @@ | ||
version: | ||
label: 1.4.6 | ||
isAvailable: true | ||
defaultResources: | ||
cpu: 0.3 | ||
memory: 512 | ||
disk: 512 | ||
features: | ||
- field: COMMAND_LINE | ||
label: Command line | ||
isMandatory: true | ||
comment: sqoop | ||
defaultValue: "driver=xxx \nhost=\"x.x.x.x\"\nport=xxx \nusername=\"xxx\" \npassword=\"xxxx\" \ndatabase=\"xxxx\" \ntable=\"xxxx\" \nhdfsdest=hdfs:///tmp/sqoop_import \n\nsqoop import --connect jdbc:$driver://$host:$port/$database --username $username --password $password --as-textfile -m 1 --target-dir $hdfsdest --table \"$table\"" | ||
- field: ARTIFACT | ||
label: Package | ||
isMandatory: false | ||
comment: "Any file necessary for the process, can be a zip file containing shell script, jar files, data etc." | ||
- field: SCHEDULER | ||
label: Scheduled | ||
isMandatory: true |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,8 @@ | ||
techno: | ||
id: 66d79135-7915-49d2-bb26-0fdcbdac80d1 | ||
label: SQOOP | ||
available: true | ||
icon: sqoop | ||
recommendedVersion: 1.4.6 | ||
docker: | ||
image: saagie/sqoop |