Packaging Update.

dstreev · Mar 18, 2021 · a6fdfd4 · a6fdfd4
1 parent b41d17d
commit a6fdfd4
Show file tree

Hide file tree

Showing 6 changed files with 147 additions and 9 deletions.
diff --git a/bin/deploy.yaml b/bin/deploy.yaml
@@ -27,6 +27,12 @@
         dest: /usr/local/hadoop-cli/lib
         mode: 0755
 
+    - name: Copy No Hadoop lib
+      copy:
+        src: ../target/hadoop-cli-shaded-no-hadoop.jar
+        dest: /usr/local/hadoop-cli/lib
+        mode: 0755
+
     - name: Link executable
       file:
         src: /usr/local/hadoop-cli/bin/hadoopcli

diff --git a/bin/hadoopcli b/bin/hadoopcli
@@ -49,6 +49,7 @@ CMD_DIRECTIVE=
 CMD_ARGS=
 VERBOSE=
 ENV_VAR_FILE=
+HADOOP_CLASSPATH=
 APP_JAVA_OPTS=${JAVA_OPTS:-}
 
 if [[ -f ${HOME}/.hadoop-cli/env-var.props || -L ${HOME}/.hadoop-cli/env-var.props ]]; then
@@ -111,6 +112,10 @@ while [[ $# -gt 0 ]]; do
       export AUX_LIBS_DIR=$1
       shift
       ;;
+    --hadoop-classpath)
+      shift
+      HADOOP_CLASSPATH="true"
+      ;;
     --show-cp)
       shift
       SHOW_CP="true"
@@ -145,6 +150,48 @@ if [[ ! -d ${AUX_LIBS_DIR} ]]; then
     mkdir -p ${AUX_LIBS_DIR}
 fi
 
+# Choose to use embedded libs (Hadoop 3+) or get Hadoop Classpath locally
+if [[ "${HADOOP_CLASSPATH}x" != "x" ]]; then
+    if [[ -f ./hadoop-cli-shaded-no-hadoop.jar ]]; then
+        # Look in Current Directory
+        CMD_CP=$APP_DIR/hadoop-cli-shaded-no-hadoop.jar
+        #echo "Using bin from same directory"
+    elif [[ -f $APP_DIR/../target/hadoop-cli-shaded-no-hadoop.jar ]]; then
+        # Look in ../target
+        CMD_CP=$APP_DIR/../target/hadoop-cli-shaded-no-hadoop.jar
+        #echo "Using bin from build directory"
+    else
+        # Look in /usr/local/hadoop-cli/lib
+        if [[ -f /usr/local/hadoop-cli/lib/hadoop-cli-shaded-no-hadoop.jar ]]; then
+            CMD_CP=/usr/local/hadoop-cli/lib/hadoop-cli-shaded-no-hadoop.jar
+            #echo "Using bin from installation directory"
+        else
+            #echo "Couldn't locate Hadoop Cli Library"
+            exit 1
+        fi
+    fi
+    CMD_CP=${CMD_CP}:`hadoop classpath`
+else
+  if [[ -f ./hadoop-cli-shaded.jar ]]; then
+      # Look in Current Directory
+      CMD_CP=$APP_DIR/hadoop-cli-shaded.jar
+      #echo "Using bin from same directory"
+  elif [[ -f $APP_DIR/../target/hadoop-cli-shaded.jar ]]; then
+      # Look in ../target
+      CMD_CP=$APP_DIR/../target/hadoop-cli-shaded.jar
+      #echo "Using bin from build directory"
+  else
+      # Look in /usr/local/hadoop-cli/lib
+      if [[ -f /usr/local/hadoop-cli/lib/hadoop-cli-shaded.jar ]]; then
+          CMD_CP=/usr/local/hadoop-cli/lib/hadoop-cli-shaded.jar
+          #echo "Using bin from installation directory"
+      else
+          #echo "Couldn't locate Hadoop Cli Library"
+          exit 1
+      fi
+  fi
+fi
+
 for jarFile in `ls -d ${AUX_LIBS_DIR}/*.jar 2> /dev/null`; do
     CMD_CP=${CMD_CP}:$jarFile
 done

diff --git a/bin/setup.sh b/bin/setup.sh
@@ -17,10 +17,19 @@ if [ -f ../target/hadoop-cli-shaded.jar ]; then
     cp -f ../target/hadoop-cli-shaded.jar /usr/local/hadoop-cli/lib
 fi
 
+if [ -f ../target/hadoop-cli-shaded-no-hadoop.jar ]; then
+    cp -f ../target/hadoop-cli-shaded-no-hadoop.jar /usr/local/hadoop-cli/lib
+fi
+
 if [ -f hadoop-cli-shaded.jar ]; then
     cp -f hadoop-cli-shaded.jar /usr/local/hadoop-cli/lib
 fi
 
+if [ -f hadoop-cli-shaded-no-hadoop.jar ]; then
+    cp -f hadoop-cli-shaded-no-hadoop.jar /usr/local/hadoop-cli/lib
+fi
+
+
 chmod -R +r /usr/local/hadoop-cli
 chmod +x /usr/local/hadoop-cli/bin/hadoopcli
 chmod +x /usr/local/hadoop-cli/bin/JCECheck

diff --git a/pom.xml b/pom.xml
@@ -25,7 +25,7 @@
     <modelVersion>4.0.0</modelVersion>
     <groupId>com.streever.hadoop</groupId>
     <artifactId>hadoop-cli</artifactId>
-    <version>2.3.0.1-SNAPSHOT</version>
+    <version>2.3.0.2-SNAPSHOT</version>
     <name>Hadoop CLI</name>
     <description>Hadoop Command Line Interface</description>
     <properties>
@@ -96,16 +96,19 @@
             <groupId>org.apache.hadoop</groupId>
             <artifactId>hadoop-hdfs</artifactId>
             <version>${hadoop.version}.${cdp.version}</version>
+<!--            <scope>provided</scope>-->
         </dependency>
         <dependency>
             <groupId>org.apache.hadoop</groupId>
             <artifactId>hadoop-hdfs-client</artifactId>
             <version>${hadoop.version}.${cdp.version}</version>
+<!--            <scope>provided</scope>-->
         </dependency>
         <dependency>
             <groupId>org.apache.hadoop</groupId>
             <artifactId>hadoop-common</artifactId>
             <version>${hadoop.version}.${cdp.version}</version>
+<!--            <scope>provided</scope>-->
         </dependency>
         <dependency>
             <groupId>org.apache.hadoop</groupId>
@@ -118,11 +121,12 @@
         <!--            <artifactId>gateway-shell</artifactId>-->
         <!--            <version>${gateway-shell.version}</version>-->
         <!--        </dependency>-->
-        <dependency>
-            <groupId>org.apache.hadoop</groupId>
-            <artifactId>hadoop-distcp</artifactId>
-            <version>${hadoop.version}.${cdp.version}</version>
-        </dependency>
+<!--        <dependency>-->
+<!--            <groupId>org.apache.hadoop</groupId>-->
+<!--            <artifactId>hadoop-distcp</artifactId>-->
+<!--            <version>${hadoop.version}.${cdp.version}</version>-->
+<!--            <scope>provided</scope>-->
+<!--        </dependency>-->
         <dependency>
             <groupId>commons-io</groupId>
             <artifactId>commons-io</artifactId>
@@ -146,6 +150,7 @@
         </dependency>
     </dependencies>
     <build>
+        <finalName>${project.artifactId}</finalName>
         <pluginManagement>
             <plugins>
                 <!-- clean lifecycle, see https://maven.apache.org/ref/current/maven-core/lifecycles.html#clean_Lifecycle -->
@@ -227,6 +232,7 @@
                 <!--</configuration>-->
                 <executions>
                     <execution>
+                        <id>full-shaded</id>
                         <phase>package</phase>
                         <goals>
                             <goal>shade</goal>
@@ -260,6 +266,50 @@
                             </transformers>
                         </configuration>
                     </execution>
+                    <execution>
+                        <id>shaded-no-hadoop</id>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>shade</goal>
+                        </goals>
+                        <configuration>
+                            <createSourcesJar>false</createSourcesJar>
+                            <finalName>${project.artifactId}-shaded-no-hadoop</finalName>
+                            <shadedArtifactAttached>true</shadedArtifactAttached>
+                            <artifactSet>
+                                <includes>
+                                    <include>com.jcabi:*</include>
+                                    <include>org.sonatype.jline:jline</include>
+                                    <include>commons-io:commons-io</include>
+                                    <include>org.apache.commons:commons-lang3</include>
+                                    <include>com.fasterxml.jackson.core:jackson-core</include>
+                                    <include>org.yaml:snakeyaml</include>
+                                </includes>
+                            </artifactSet>
+                            <filters>
+                                <filter>
+                                    <excludes>
+                                        <exclude>META-INF/*.SF</exclude>
+                                        <exclude>META-INF/*.RSA</exclude>
+                                        <exclude>META-INF/*.INF</exclude>
+                                        <exclude>META-INF/LICENSE</exclude>
+                                        <exclude>META-INF/**/pom.xml</exclude>
+                                        <exclude>META-INF/**/pom.properties</exclude>
+                                    </excludes>
+                                </filter>
+                            </filters>
+                            <transformers>
+                                <transformer
+                                        implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
+                                <transformer
+                                        implementation="org.apache.maven.plugins.shade.resource.ResourceBundleAppendingTransformer"/>
+                                <transformer
+                                        implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
+                                    <mainClass>com.streever.hive.Sre</mainClass>
+                                </transformer>
+                            </transformers>
+                        </configuration>
+                    </execution>
                 </executions>
             </plugin>
             <plugin>
@@ -281,6 +331,34 @@
                     </execution>
                 </executions>
             </plugin>
+<!--            <plugin>-->
+<!--                <groupId>org.apache.maven.plugins</groupId>-->
+<!--                <artifactId>maven-assembly-plugin</artifactId>-->
+<!--&lt;!&ndash;                <version>2.4.1</version>&ndash;&gt;-->
+<!--                <configuration>-->
+<!--                    &lt;!&ndash; get all project dependencies &ndash;&gt;-->
+<!--                    <descriptorRefs>-->
+<!--                        <descriptorRef>jar-with-dependencies</descriptorRef>-->
+<!--                    </descriptorRefs>-->
+<!--                    &lt;!&ndash; MainClass in mainfest make a executable jar &ndash;&gt;-->
+<!--                    <archive>-->
+<!--                        <manifest>-->
+<!--                            <mainClass>com.streever.hadoop.HadoopSession</mainClass>-->
+<!--                        </manifest>-->
+<!--                    </archive>-->
+
+<!--                </configuration>-->
+<!--                <executions>-->
+<!--                    <execution>-->
+<!--                        <id>make-assembly</id>-->
+<!--                        &lt;!&ndash; bind to the packaging phase &ndash;&gt;-->
+<!--                        <phase>package</phase>-->
+<!--                        <goals>-->
+<!--                            <goal>single</goal>-->
+<!--                        </goals>-->
+<!--                    </execution>-->
+<!--                </executions>-->
+<!--            </plugin>-->
         </plugins>
     </build>
 </project>
diff --git a/src/main/assembly/assembly.xml b/src/main/assembly/assembly.xml
@@ -48,7 +48,7 @@
         <fileSet>
             <directory>${project.basedir}/target</directory>
             <includes>
-                <include>*shaded.jar</include>
+                <include>*shaded*.jar</include>
             </includes>
             <outputDirectory>hadoop-cli</outputDirectory>
             <useDefaultExcludes>true</useDefaultExcludes>

diff --git a/src/main/java/com/streever/hadoop/hdfs/util/FileSystemState.java b/src/main/java/com/streever/hadoop/hdfs/util/FileSystemState.java
@@ -7,8 +7,6 @@
 import org.apache.hadoop.hdfs.DistributedFileSystem;
 
 import java.util.Date;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
 
 public class FileSystemState {
     private static String DISTRIBUTED_USER_HOME_BASE = "/user";