Merge pull request #6 from saagie/sqoop

(job) add sqoop 1.4.6

certified/job/sqoop/1.4.6/Dockerfile

@@ -0,0 +1,60 @@
+FROM debian:stretch
+
+COPY entrypoint /entrypoint
+RUN chmod 755 /entrypoint
+
+# LIGHT DEPENDENCIES START
+RUN apt-get update
+RUN apt-get install -y ftp wget curl unzip telnet openssh-client
+# LIGHT DEPENDENCIES END
+
+# INSTALL JAVA & BEELINE START
+# INSTALL JDK8
+RUN apt-get install -y openjdk-8-jdk
+ENV JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64
+ENV PATH=$PATH:$JAVA_HOME/bin
+
+# Hadoop client installation
+RUN wget --no-verbose https://archive.cloudera.com/cdh5/cdh/5/hadoop-2.6.0-cdh5.13.0.tar.gz \
+    && tar -xzf hadoop-2.6.0-cdh5.13.0.tar.gz \
+    && rm hadoop-2.6.0-cdh5.13.0.tar.gz \
+    && mv hadoop-2.6.0-cdh5.13.0 /usr/lib/hadoop
+
+ENV HADOOP_PREFIX=/usr/lib/hadoop
+ENV PATH=$PATH:$HADOOP_PREFIX/bin
+ENV HADOOP_COMMON_HOME=$HADOOP_PREFIX
+ENV HADOOP_CONF_DIR=/etc/hadoop/conf
+
+# Hive client installation
+RUN wget --no-verbose http://archive.cloudera.com/cdh5/cdh/5/hive-1.1.0-cdh5.13.3.tar.gz \
+    && tar -xvzf hive-1.1.0-cdh5.13.3.tar.gz \
+    && rm hive-1.1.0-cdh5.13.3.tar.gz
+
+ENV HIVE_HOME=/hive-1.1.0-cdh5.13.3
+ENV PATH=$PATH:$HIVE_HOME/bin
+ENV HIVE_CONF_DIR=/etc/hive/conf
+# INSTALL JAVA & BEELINE END
+
+# Sqoop client installation
+RUN wget --no-verbose http://archive.cloudera.com/cdh5/cdh/5/sqoop-1.4.6-cdh5.13.0.tar.gz \
+    && tar -xvzf sqoop-1.4.6-cdh5.13.0.tar.gz \
+    && rm sqoop-1.4.6-cdh5.13.0.tar.gz \
+    && cp /hive-1.1.0-cdh5.13.3/lib/*.jar /sqoop-1.4.6-cdh5.13.0/lib/
+
+ENV SQOOP_HOME=/sqoop-1.4.6-cdh5.13.0
+ENV PATH=$PATH:$SQOOP_HOME/bin
+ENV HADOOP_HOME=$HADOOP_PREFIX
+
+# INSTALL PROCPS (ps) START
+RUN apt-get install -y procps
+# INSTALL PROCPS (ps) END
+
+#UPDATE SECURITY PACKAGE BEGIN
+RUN apt-get update && apt-get upgrade -y
+#UPDATE SECURITY PACKAGE END
+
+COPY lib/*.jar /sqoop-1.4.6-cdh5.13.0/lib/
+
+WORKDIR /sandbox
+
+ENTRYPOINT  ["bash","/entrypoint"]

certified/job/sqoop/1.4.6/build.gradle.kts

@@ -0,0 +1,22 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Copyright 2019 Pierre Leresteux.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import com.bmuschko.gradle.docker.DockerRemoteApiPlugin
+import com.saagie.technologies.SaagieTechnologiesGradlePlugin
+
+apply<DockerRemoteApiPlugin>()
+apply<SaagieTechnologiesGradlePlugin>()

certified/job/sqoop/1.4.6/entrypoint

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+set -euo pipefail
+
+if test -f main_script;
+then ln -s $HIVE_CONF_DIR/hive-site.xml; sh ./main_script;
+else exec "$@"
+fi;

certified/job/sqoop/1.4.6/image_test.yml

@@ -0,0 +1,125 @@
+schemaVersion: "2.0.0"
+
+metadataTest:
+  env:
+    - key: JAVA_HOME
+      value: "/usr/lib/jvm/java-8-openjdk-amd64"
+    - key: "HADOOP_HOME"
+      value: "/usr/lib/hadoop"
+    - key: "HIVE_HOME"
+      value: "/hive-1.1.0-cdh5.13.3"
+    - key: "HADOOP_CONF_DIR"
+      value: "/etc/hadoop/conf"
+    - key: "HIVE_CONF_DIR"
+      value: "/etc/hive/conf"
+    - key: "SQOOP_HOME"
+      value: "/sqoop-1.4.6-cdh5.13.0"
+
+fileExistenceTests:
+  - name: "entrypoint"
+    path: "/entrypoint"
+    shouldExist: true
+    permissions: "-rwxr-xr-x"
+
+fileContentTests:
+  - name: "entrypoint"
+    path: "/entrypoint"
+    expectedContents: ["ln -s \\$HIVE_CONF_DIR/hive-site.xml", "sh ./main_script"]
+
+commandTests:
+  - name: "Workdir"
+    command: "pwd"
+    expectedOutput: ["/sandbox"]
+
+  - name: "ftp"
+    args: ["-h"]
+    command: "ftp"
+    exitCode: 0
+
+  - name: "wget"
+    args: ["--help"]
+    command: "wget"
+    exitCode: 0
+
+  - name: "curl"
+    args: ["--help"]
+    command: "curl"
+    exitCode: 0
+
+  - name: "apt-get"
+    args: ["--help"]
+    command: "apt-get"
+    exitCode: 0
+
+  - name: "unzip"
+    args: ["--help"]
+    command: "unzip"
+    exitCode: 0
+
+  - name: "tar"
+    args: ["--help"]
+    command: "tar"
+    exitCode: 0
+
+  - name: "awk"
+    command: "awk"
+    exitCode: 0
+
+  - name: "telnet"
+    command: "which"
+    args: ["telnet"]
+    expectedOutput: ["/usr/bin/telnet"]
+    exitCode: 0
+
+  - name: "scp"
+    command: "which"
+    args: ["scp"]
+    expectedOutput: ["/usr/bin/scp"]
+    exitCode: 0
+
+  - name: "ping"
+    command: "which"
+    args: ["ping"]
+    expectedOutput: ["/bin/ping"]
+    exitCode: 0
+
+  - name: "wc"
+    command: "wc"
+    args: ["--help"]
+    exitCode: 0
+
+  - name: "grep"
+    args: ["--help"]
+    command: "grep"
+    exitCode: 0
+
+  - name: "sleep"
+    args: ["--help"]
+    command: "sleep"
+    exitCode: 0
+
+  - name: "java"
+    args: ["-version"]
+    command: "java"
+    exitCode: 0
+
+  - name: "hadoop"
+    args: ["--help"]
+    command: "hadoop"
+    exitCode: 0
+
+  - name: "beeline"
+    args: ["--help"]
+    command: "beeline"
+    exitCode: 0
+
+  - name: "sqoop"
+    args: ["help"]
+    command: "sqoop"
+    exitCode: 0
+
+  - name: "lib"
+    args: ["-c", "ls /sqoop-1.4.6-cdh5.13.0/lib/*.jar | wc -l"]
+    command: "sh"
+    expectedOutput: ["162"]
+    exitCode: 0

certified/job/sqoop/1.4.6/metadata.yml

@@ -0,0 +1,30 @@
+version:
+  label: 1.4.6
+  isAvailable: true
+  defaultResources:
+    cpu: 0.3
+    memory: 512
+    disk: 512
+  features:
+    - field: COMMAND_LINE
+      label: Command line
+      isMandatory: true
+      comment: sqoop
+      defaultValue: "driver=xxx \nhost=\"x.x.x.x\"\nport=xxx \nusername=\"xxx\" \npassword=\"xxxx\" \ndatabase=\"xxxx\" \ntable=\"xxxx\" \nhdfsdest=hdfs:///tmp/sqoop_import \n\nsqoop import --connect jdbc:$driver://$host:$port/$database --username $username --password $password --as-textfile -m 1 --target-dir $hdfsdest --table \"$table\""
+    - field: ARTIFACT
+      label: Package
+      isMandatory: false
+      comment: "Any file necessary for the process, can be a zip file containing shell script, jar files, data etc."
+    - field: SCHEDULER
+      label: Scheduled
+      isMandatory: true
+
+techno:
+  id: 66d79135-7915-49d2-bb26-0fdcbdac80d1
+  label: SQOOP
+  available: true
+  icon: sqoop
+  recommendedVersion: 1.4.6
+  docker:
+    image: saagie/sqoop
+    version: 1.4.6-0.6.0_sqoop

certified/job/sqoop/1.4.6/settings.gradle.kts

@@ -0,0 +1,18 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Copyright 2019 Pierre Leresteux.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+rootProject.name = "sqoop1.4.6"

certified/job/sqoop/1.4.6/version.yml

@@ -0,0 +1,20 @@
+version:
+  label: 1.4.6
+  isAvailable: true
+  defaultResources:
+    cpu: 0.3
+    memory: 512
+    disk: 512
+  features:
+    - field: COMMAND_LINE
+      label: Command line
+      isMandatory: true
+      comment: sqoop
+      defaultValue: "driver=xxx \nhost=\"x.x.x.x\"\nport=xxx \nusername=\"xxx\" \npassword=\"xxxx\" \ndatabase=\"xxxx\" \ntable=\"xxxx\" \nhdfsdest=hdfs:///tmp/sqoop_import \n\nsqoop import --connect jdbc:$driver://$host:$port/$database --username $username --password $password --as-textfile -m 1 --target-dir $hdfsdest --table \"$table\""
+    - field: ARTIFACT
+      label: Package
+      isMandatory: false
+      comment: "Any file necessary for the process, can be a zip file containing shell script, jar files, data etc."
+    - field: SCHEDULER
+      label: Scheduled
+      isMandatory: true

certified/job/sqoop/techno.yml

@@ -0,0 +1,8 @@
+techno:
+  id: 66d79135-7915-49d2-bb26-0fdcbdac80d1
+  label: SQOOP
+  available: true
+  icon: sqoop
+  recommendedVersion: 1.4.6
+  docker:
+    image: saagie/sqoop

version.properties

@@ -1,7 +1,7 @@
-version.buildmeta=
+version.buildmeta=sqoop
 version.buildmeta.prefix=_
 version.major=0
 version.minor=6
 version.patch=0
 version.prerelease=
-version.semver=0.6.0
+version.semver=0.6.0_sqoop