Added s3a support. fixed aux_libs use.

dstreev · Aug 3, 2020 · 4794839 · 4794839
1 parent 79f94e2
commit 4794839
Show file tree

Hide file tree

Showing 9 changed files with 3,718 additions and 17 deletions.
diff --git a/README.md b/README.md
@@ -31,6 +31,15 @@ To use an alternate HADOOP_CONF_DIR:
 
     hadoopcli --config /var/hadoop/dev-cfg
 
+### AUX_LIBS - CLASSPATH Additions
+
+The directory `$HOME/.hadoop-cli/aux_libs` will be scanned for 'jar' files. Each 'jar' will be added the java classpath of the application.  Add any required libaries here.
+
+The application contains all the necesasry `hdfs` classes already.  You will need to add to the `aux_libs` directory the following:
+- AWS S3 Drivers (appropriate versions)
+    - `hadoop-aws.jar`
+    - `aws-java-sdk-bundle.jar`
+
 ### [Release Notes](./release_notes.md)
 
 ### Core Functions

diff --git a/bin/hadoopcli b/bin/hadoopcli
@@ -36,7 +36,7 @@ fi
 # Set the default.
 export HADOOP_CONF_DIR="/etc/hadoop/conf"
 
-AUX_LIBS_DIR=${HADOOP_CLI_AUX_LIBS:-${HOME}/.hadoopcli/aux_libs}
+AUX_LIBS_DIR=${HADOOP_CLI_AUX_LIBS:-${HOME}/.hadoop-cli/aux_libs}
 
 SHOW_CP=
 CMD_DIRECTIVE=

diff --git a/pom.xml b/pom.xml
@@ -25,7 +25,7 @@
     <modelVersion>4.0.0</modelVersion>
     <groupId>com.streever.hadoop</groupId>
     <artifactId>hadoop.cli</artifactId>
-    <version>2.2.1.0-SNAPSHOT</version>
+    <version>2.2.1.1-SNAPSHOT</version>
     <name>Hadoop CLI</name>
     <description>Hadoop Command Line Interface</description>
     <properties>
@@ -39,12 +39,19 @@
         <jackson.parser>2.7.3</jackson.parser>
         <commons-io.version>2.4</commons-io.version>
         <jcabi.manifests.version>1.1</jcabi.manifests.version>
+        <hadoop.version>3.1.1</hadoop.version>
+        <cdp.version>7.1.1.0-565</cdp.version>
         <!-- This needs to be in sync with Hadoop Common when using this cli in api mode  -->
         <!--        <commons-cli.version>1.2</commons-cli.version>-->
 
         <jline.version>2.5</jline.version>
     </properties>
+
     <repositories>
+        <repository>
+            <id>cloudera</id>
+            <url>https://repository.cloudera.com/artifactory/cloudera-repos/</url>
+        </repository>
     </repositories>
 
     <dependencies>
@@ -74,22 +81,28 @@
         <dependency>
             <groupId>org.apache.hadoop</groupId>
             <artifactId>hadoop-client</artifactId>
-            <version>${hadoop.version}</version>
+            <version>${hadoop.version}.${cdp.version}</version>
         </dependency>
         <dependency>
             <groupId>org.apache.hadoop</groupId>
             <artifactId>hadoop-common</artifactId>
-            <version>${hadoop.version}</version>
+            <version>${hadoop.version}.${cdp.version}</version>
         </dependency>
         <dependency>
-            <groupId>org.apache.knox</groupId>
-            <artifactId>gateway-shell</artifactId>
-            <version>${gateway-shell.version}</version>
+            <groupId>org.apache.hadoop</groupId>
+            <artifactId>hadoop-aws</artifactId>
+            <version>${hadoop.version}.${cdp.version}</version>
+            <scope>provided</scope>
         </dependency>
+<!--        <dependency>-->
+<!--            <groupId>org.apache.knox</groupId>-->
+<!--            <artifactId>gateway-shell</artifactId>-->
+<!--            <version>${gateway-shell.version}</version>-->
+<!--        </dependency>-->
         <dependency>
             <groupId>org.apache.hadoop</groupId>
             <artifactId>hadoop-distcp</artifactId>
-            <version>${hadoop.version}</version>
+            <version>${hadoop.version}.${cdp.version}</version>
         </dependency>
         <dependency>
             <groupId>commons-io</groupId>

diff --git a/src/main/java/com/streever/hadoop/hdfs/shell/command/HdfsConnect.java b/src/main/java/com/streever/hadoop/hdfs/shell/command/HdfsConnect.java
@@ -24,6 +24,7 @@
 
 import java.io.File;
 import java.io.IOException;
+import java.net.URI;
 import java.util.Enumeration;
 
 import com.streever.hadoop.shell.command.AbstractCommand;
@@ -55,15 +56,13 @@ public HdfsConnect(String name) {
     public CommandReturn implementation(Environment env, CommandLine cmd, CommandReturn commandReturn) {
         try {
             // Get a value that over rides the default, if nothing then use default.
-// Requires Java 1.8...
-//            String hadoopConfDirProp = System.getenv().getOrDefault(HADOOP_CONF_DIR, "/etc/hadoop/conf");
+            String hadoopConfDirProp = System.getenv().getOrDefault(HADOOP_CONF_DIR, "/etc/hadoop/conf");
 
-            String hadoopConfDirProp = System.getenv().get(HADOOP_CONF_DIR);
             // Set a default
             if (hadoopConfDirProp == null)
                 hadoopConfDirProp = "/etc/hadoop/conf";
 
-            Configuration config = new Configuration(false);
+            Configuration config = new Configuration(true);
 
             File hadoopConfDir = new File(hadoopConfDirProp).getAbsoluteFile();
             for (String file : HADOOP_CONF_FILES) {
@@ -102,16 +101,14 @@ public CommandReturn implementation(Environment env, CommandLine cmd, CommandRet
 //            hdfs.setWorkingDirectory(hdfs.makeQualified(new Path("/")));
             env.setRemoteWorkingDirectory(hdfs.makeQualified(new Path("/")));
 
-            env.setRemoteWorkingDirectory(hdfs.makeQualified(new Path("/")));
-
             FileSystem local = FileSystem.getLocal(new Configuration());
             env.setValue(Constants.LOCAL_FS, local);
             env.getProperties().setProperty(Constants.HDFS_URL, hdfs.getUri().toString());
 
             FSUtil.prompt(env);
 
 //            if (!env.isSilent())
-            logv(env, "Connecting: " + hdfs.getUri());
+            logv(env, "Connecting to default FS: " + hdfs.getUri());
 
             logv(env, "HDFS CWD: " + hdfs.getWorkingDirectory());
             logv(env, "HDFS CWD(env): " + env.getRemoteWorkingDirectory());

diff --git a/src/main/java/com/streever/hadoop/hdfs/shell/command/PathBuilder.java b/src/main/java/com/streever/hadoop/hdfs/shell/command/PathBuilder.java
@@ -85,8 +85,8 @@ public static String resolveFullPath(String current, String input) {
                 adjusted = input;
             }
             if (!adjusted.startsWith("/") && !adjusted.startsWith("hdfs://") && !adjusted.startsWith("s3://") &&
-                    !adjusted.startsWith("s3s://") && !adjusted.startsWith("gs://") && !adjusted.startsWith("adl://")
-                    && !adjusted.startsWith("wasb://") && !adjusted.startsWith("abfs://"))
+                    !adjusted.startsWith("s3a://") && !adjusted.startsWith("gs://") && !adjusted.startsWith("adl://")
+                    && !adjusted.startsWith("s3n://") && !adjusted.startsWith("wasb://") && !adjusted.startsWith("abfs://"))
                 adjusted = current + "/" + adjusted;
         } else {
             adjusted = current;

diff --git a/src/main/resources/META-INF/services/org.apache.hadoop.fs.FileSystem b/src/main/resources/META-INF/services/org.apache.hadoop.fs.FileSystem
@@ -0,0 +1,39 @@
+#
+#  Hadoop CLI
+#
+#  (c) 2016-2019 David W. Streever. All rights reserved.
+#
+# This code is provided to you pursuant to your written agreement with David W. Streever, which may be the terms of the
+# Affero General Public License version 3 (AGPLv3), or pursuant to a written agreement with a third party authorized
+# to distribute this code.  If you do not have a written agreement with David W. Streever or with an authorized and
+# properly licensed third party, you do not have any rights to this code.
+#
+# If this code is provided to you under the terms of the AGPLv3:
+# (A) David W. Streever PROVIDES THIS CODE TO YOU WITHOUT WARRANTIES OF ANY KIND;
+# (B) David W. Streever DISCLAIMS ANY AND ALL EXPRESS AND IMPLIED WARRANTIES WITH RESPECT TO THIS CODE, INCLUDING BUT NOT
+#   LIMITED TO IMPLIED WARRANTIES OF TITLE, NON-INFRINGEMENT, MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE;
+# (C) David W. Streever IS NOT LIABLE TO YOU, AND WILL NOT DEFEND, INDEMNIFY, OR HOLD YOU HARMLESS FOR ANY CLAIMS ARISING
+#    FROM OR RELATED TO THE CODE; AND
+#  (D) WITH RESPECT TO YOUR EXERCISE OF ANY RIGHTS GRANTED TO YOU FOR THE CODE, David W. Streever IS NOT LIABLE FOR ANY
+#    DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, PUNITIVE OR CONSEQUENTIAL DAMAGES INCLUDING, BUT NOT LIMITED TO,
+#   DAMAGES RELATED TO LOST REVENUE, LOST PROFITS, LOSS OF INCOME, LOSS OF BUSINESS ADVANTAGE OR UNAVAILABILITY,
+#     OR LOSS OR CORRUPTION OF DATA.
+#
+#
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+org.apache.hadoop.fs.s3a.S3AFileSystem
diff --git a/src/main/resources/META-INF/services/org.apache.hadoop.security.token.DtFetcher b/src/main/resources/META-INF/services/org.apache.hadoop.security.token.DtFetcher
@@ -0,0 +1,41 @@
+#
+#  Hadoop CLI
+#
+#  (c) 2016-2019 David W. Streever. All rights reserved.
+#
+# This code is provided to you pursuant to your written agreement with David W. Streever, which may be the terms of the
+# Affero General Public License version 3 (AGPLv3), or pursuant to a written agreement with a third party authorized
+# to distribute this code.  If you do not have a written agreement with David W. Streever or with an authorized and
+# properly licensed third party, you do not have any rights to this code.
+#
+# If this code is provided to you under the terms of the AGPLv3:
+# (A) David W. Streever PROVIDES THIS CODE TO YOU WITHOUT WARRANTIES OF ANY KIND;
+# (B) David W. Streever DISCLAIMS ANY AND ALL EXPRESS AND IMPLIED WARRANTIES WITH RESPECT TO THIS CODE, INCLUDING BUT NOT
+#   LIMITED TO IMPLIED WARRANTIES OF TITLE, NON-INFRINGEMENT, MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE;
+# (C) David W. Streever IS NOT LIABLE TO YOU, AND WILL NOT DEFEND, INDEMNIFY, OR HOLD YOU HARMLESS FOR ANY CLAIMS ARISING
+#    FROM OR RELATED TO THE CODE; AND
+#  (D) WITH RESPECT TO YOUR EXERCISE OF ANY RIGHTS GRANTED TO YOU FOR THE CODE, David W. Streever IS NOT LIABLE FOR ANY
+#    DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, PUNITIVE OR CONSEQUENTIAL DAMAGES INCLUDING, BUT NOT LIMITED TO,
+#   DAMAGES RELATED TO LOST REVENUE, LOST PROFITS, LOSS OF INCOME, LOSS OF BUSINESS ADVANTAGE OR UNAVAILABILITY,
+#     OR LOSS OR CORRUPTION OF DATA.
+#
+#
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+org.apache.hadoop.fs.s3a.auth.delegation.S3ADtFetcher
diff --git a/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier b/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier
@@ -0,0 +1,43 @@
+#
+#  Hadoop CLI
+#
+#  (c) 2016-2019 David W. Streever. All rights reserved.
+#
+# This code is provided to you pursuant to your written agreement with David W. Streever, which may be the terms of the
+# Affero General Public License version 3 (AGPLv3), or pursuant to a written agreement with a third party authorized
+# to distribute this code.  If you do not have a written agreement with David W. Streever or with an authorized and
+# properly licensed third party, you do not have any rights to this code.
+#
+# If this code is provided to you under the terms of the AGPLv3:
+# (A) David W. Streever PROVIDES THIS CODE TO YOU WITHOUT WARRANTIES OF ANY KIND;
+# (B) David W. Streever DISCLAIMS ANY AND ALL EXPRESS AND IMPLIED WARRANTIES WITH RESPECT TO THIS CODE, INCLUDING BUT NOT
+#   LIMITED TO IMPLIED WARRANTIES OF TITLE, NON-INFRINGEMENT, MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE;
+# (C) David W. Streever IS NOT LIABLE TO YOU, AND WILL NOT DEFEND, INDEMNIFY, OR HOLD YOU HARMLESS FOR ANY CLAIMS ARISING
+#    FROM OR RELATED TO THE CODE; AND
+#  (D) WITH RESPECT TO YOUR EXERCISE OF ANY RIGHTS GRANTED TO YOU FOR THE CODE, David W. Streever IS NOT LIABLE FOR ANY
+#    DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, PUNITIVE OR CONSEQUENTIAL DAMAGES INCLUDING, BUT NOT LIMITED TO,
+#   DAMAGES RELATED TO LOST REVENUE, LOST PROFITS, LOSS OF INCOME, LOSS OF BUSINESS ADVANTAGE OR UNAVAILABILITY,
+#     OR LOSS OR CORRUPTION OF DATA.
+#
+#
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+org.apache.hadoop.fs.s3a.auth.delegation.FullCredentialsTokenIdentifier
+org.apache.hadoop.fs.s3a.auth.delegation.RoleTokenIdentifier
+org.apache.hadoop.fs.s3a.auth.delegation.SessionTokenIdentifier