Add java14 and sqoop

* Add Java14 experimental
* Add sqoop 1.4.6

technologies/job/bash/metadata.yml

@@ -93,4 +93,4 @@ contexts:
           comment: All files are accepted.
         - field: SCHEDULER
           label: Scheduled
-          mandatory: true
+          mandatory: true

technologies/job/generic/metadata.yml

@@ -19,4 +19,4 @@ contexts:
           label: Scheduled
           mandatory: true
 
-
+

technologies/job/java/14/Dockerfile

@@ -0,0 +1,14 @@
+FROM openjdk:14-jdk-slim-buster
+
+COPY entrypoint /entrypoint
+RUN chmod 755 /entrypoint
+
+ENV HADOOP_CONF_DIR=/etc/hadoop/conf
+
+#UPDATE SECURITY PACKAGE BEGIN
+RUN apt-get update && apt-get upgrade -y
+#UPDATE SECURITY PACKAGE END
+
+WORKDIR /sandbox
+
+ENTRYPOINT  ["bash","/entrypoint"]

technologies/job/java/14/build.gradle.kts

@@ -0,0 +1,22 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Copyright 2019 Pierre Leresteux.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import com.bmuschko.gradle.docker.DockerRemoteApiPlugin
+import com.saagie.technologies.SaagieTechnologiesGradlePlugin
+
+apply<DockerRemoteApiPlugin>()
+apply<SaagieTechnologiesGradlePlugin>()

technologies/job/java/14/context.yml

@@ -0,0 +1,21 @@
+id: 14
+label: 14
+available: true
+recommended: false
+dockerInfo:
+  image: saagie/java-scala
+trustLevel: "experimental"
+job:
+  features:
+    - field: COMMAND_LINE
+      label: "Command line"
+      mandatory: true
+      comment: "Linux shell command to launch the job."
+      defaultValue: java -jar {file} arg1 arg2
+    - field: ARTIFACT
+      label: "Package"
+      mandatory: true
+      comment: "Compatible upload file : .jar"
+    - field: SCHEDULER
+      label: "Scheduled"
+      mandatory: true

technologies/job/java/14/dockerInfo.yml

@@ -0,0 +1,3 @@
+dockerInfo:
+  image: saagie/java-scala
+  version: 14-1.15.0_add_java14_and_sqoop

technologies/job/java/14/entrypoint

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+set -euo pipefail
+
+
+if test -f main_script;
+then sh ./main_script;
+else exec "$@"
+fi;

technologies/job/java/14/image_test.yml

@@ -0,0 +1,36 @@
+schemaVersion: "2.0.0"
+
+metadataTest:
+  env:
+    - key: LANG
+      value: "C.UTF-8"
+    - key: JAVA_HOME
+      value: "/usr/java/openjdk-14"
+    - key: "HADOOP_CONF_DIR"
+      value: "/etc/hadoop/conf"
+
+fileExistenceTests:
+  - name: "entrypoint"
+    path: "/entrypoint"
+    shouldExist: true
+    permissions: "-rwxr-xr-x"
+
+fileContentTests:
+  - name: "entrypoint"
+    path: "/entrypoint"
+    expectedContents: ["sh ./main_script"]
+
+commandTests:
+  - name: "java installation"
+    command: "which"
+    args: ["java"]
+    expectedOutput: ["/usr/java/openjdk-14/bin/java"]
+
+  - name: "java version"
+    command: "java"
+    args: ["-version"]
+    expectedError: ['openjdk version "14']
+
+  - name: "Workdir"
+    command: "pwd"
+    expectedOutput: ["/sandbox"]

technologies/job/java/14/settings.gradle.kts

@@ -0,0 +1,18 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Copyright 2019 Pierre Leresteux.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+rootProject.name = "java14"

technologies/job/java/metadata.yml

@@ -93,3 +93,25 @@ contexts:
         - field: SCHEDULER
           label: "Scheduled"
           mandatory: true
+  - id: 14
+    label: 14
+    available: true
+    recommended: false
+    dockerInfo:
+      image: saagie/java-scala
+      version: 14-1.15.0_add_java14_and_sqoop
+    trustLevel: "experimental"
+    job:
+      features:
+        - field: COMMAND_LINE
+          label: "Command line"
+          mandatory: true
+          comment: "Linux shell command to launch the job."
+          defaultValue: java -jar {file} arg1 arg2
+        - field: ARTIFACT
+          label: "Package"
+          mandatory: true
+          comment: "Compatible upload file : .jar"
+        - field: SCHEDULER
+          label: "Scheduled"
+          mandatory: true

technologies/job/sqoop/1.4.6/Dockerfile

@@ -0,0 +1,91 @@
+FROM debian:stretch
+
+ARG HADOOP_VERSION="2.6.0"
+ARG HIVE_VERSION="1.1.0"
+ARG SQOOP_VERSION="1.4.6"
+ARG CDH_MAJOR="5"
+ARG CDH_MINOR="13.0"
+ARG CDH_VERSION="$CDH_MAJOR.$CDH_MINOR"
+ARG CDH_URL="https://archive.cloudera.com/cdh$CDH_MAJOR/cdh/$CDH_MAJOR"
+
+ENV DEBIAN_FRONTEND noninteractive
+
+# LIGHT DEPENDENCIES START
+RUN apt update -qq && apt install -qqy --no-install-recommends \
+      ftp wget curl unzip telnet openssh-client \
+    && rm -rf /var/lib/apt/lists/*;
+# LIGHT DEPENDENCIES END
+
+# INSTALL JAVA & BEELINE START
+# INSTALL JDK8
+RUN apt update -qq && apt install -qqy --no-install-recommends \
+      openjdk-8-jdk \
+    && rm -rf /var/lib/apt/lists/*;
+ENV JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64
+ENV PATH=$PATH:$JAVA_HOME/bin
+
+# Hadoop client installation
+ARG HADOOP_NAME=hadoop-$HADOOP_VERSION-cdh$CDH_VERSION
+RUN wget --no-verbose $CDH_URL/$HADOOP_NAME.tar.gz \
+    && tar -xzf $HADOOP_NAME.tar.gz \
+    && rm $HADOOP_NAME.tar.gz \
+    && mv $HADOOP_NAME /usr/lib/hadoop
+
+ENV HADOOP_PREFIX=/usr/lib/hadoop
+ENV PATH=$PATH:$HADOOP_PREFIX/bin
+ENV HADOOP_COMMON_HOME=$HADOOP_PREFIX
+ENV HADOOP_CONF_DIR=/etc/hadoop/conf
+
+# Hive client installation
+ARG HIVE_NAME=hive-$HIVE_VERSION-cdh$CDH_VERSION
+RUN wget --no-verbose $CDH_URL/$HIVE_NAME.tar.gz \
+    && tar -xvzf $HIVE_NAME.tar.gz \
+    && mkdir -p /usr/lib/hive-hcatalog/share/hcatalog/ \
+    && ln -s /$HIVE_NAME/hcatalog/share/hcatalog/hive-hcatalog-core-$HIVE_VERSION-cdh$CDH_VERSION.jar /usr/lib/hive-hcatalog/share/hcatalog/hive-hcatalog-core.jar \
+    && rm $HIVE_NAME.tar.gz
+
+ENV HIVE_HOME=/$HIVE_NAME
+ENV PATH=$PATH:$HIVE_HOME/bin
+ENV HIVE_CONF_DIR=/etc/hive/conf
+# INSTALL JAVA & BEELINE END
+
+# Sqoop client installation
+ARG SQOOP_NAME=sqoop-$SQOOP_VERSION-cdh$CDH_VERSION
+RUN wget --no-verbose $CDH_URL/$SQOOP_NAME.tar.gz \
+    && tar -xvzf $SQOOP_NAME.tar.gz \
+    && rm $SQOOP_NAME.tar.gz \
+    && cp /$HIVE_NAME/lib/*.jar /$SQOOP_NAME/lib/
+
+ENV SQOOP_HOME=/$SQOOP_NAME
+ENV PATH=$PATH:$SQOOP_HOME/bin
+ENV HADOOP_HOME=$HADOOP_PREFIX
+
+# INSTALL PROCPS (ps) START
+RUN apt update -qq && apt install -qqy --no-install-recommends \
+      procps \
+    && rm -rf /var/lib/apt/lists/*;
+# INSTALL PROCPS (ps) END
+
+RUN apt update -qq && apt install -yqq --no-install-recommends \
+      krb5-user && \
+    rm -rf /var/lib/apt/lists/*;
+
+ENV LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:/usr/lib/hadoop/lib/native"
+
+#UPDATE SECURITY PACKAGE BEGIN
+RUN apt-get update -qq && \
+    apt-get upgrade -yqq --no-install-recommends && \
+    apt-get autoremove -yqq && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*;
+#UPDATE SECURITY PACKAGE END
+
+COPY lib/*.jar /$SQOOP_NAME/lib/
+
+# Move scripts and frequently changing directive to the end of the build
+COPY entrypoint /entrypoint
+RUN chmod 755 /entrypoint
+
+WORKDIR /sandbox
+
+ENTRYPOINT  ["bash","/entrypoint"]

technologies/job/sqoop/1.4.6/build.gradle.kts

@@ -0,0 +1,22 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Copyright 2019 Pierre Leresteux.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import com.bmuschko.gradle.docker.DockerRemoteApiPlugin
+import com.saagie.technologies.SaagieTechnologiesGradlePlugin
+
+apply<DockerRemoteApiPlugin>()
+apply<SaagieTechnologiesGradlePlugin>()

technologies/job/sqoop/1.4.6/context.yml

@@ -0,0 +1,21 @@
+id: 1.4.6
+label: 1.4.6
+available: true
+recommended: true
+dockerInfo:
+  image: saagie/sqoop
+trustLevel: "stable"
+job:
+  features:
+    - field: COMMAND_LINE
+      label: "Command line"
+      mandatory: true
+      comment: "sqoop"
+      defaultValue: "driver=xxx \nhost=\"x.x.x.x\"\nport=xxx \nusername=\"xxx\" \npassword=\"xxxx\" \ndatabase=\"xxxx\" \ntable=\"xxxx\" \nhdfsdest=hdfs:///tmp/sqoop_import \n\nsqoop import --connect jdbc:$driver://$host:$port/$database --username $username --password $password --as-textfile -m 1 --target-dir $hdfsdest --table \"$table\""
+    - field: ARTIFACT
+      label: "Package"
+      mandatory: false
+      comment: "Any file necessary for the process, can be a zip file containing shell script, jar files, data etc."
+    - field: SCHEDULER
+      label: "Scheduled"
+      mandatory: true

technologies/job/sqoop/1.4.6/dockerInfo.yml

@@ -0,0 +1,3 @@
+dockerInfo:
+  image: saagie/sqoop
+  version: 1.4.6-1.15.0_add_java14_and_sqoop

technologies/job/sqoop/1.4.6/entrypoint

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+set -euo pipefail
+
+if test -f main_script;
+then ln -s $HIVE_CONF_DIR/hive-site.xml; sh ./main_script;
+else exec "$@"
+fi;