Skip to content

Commit 3fbe4fb

Browse files
NihalJainNihalJain
authored
HBASE-28433 Modify the assembly to not include test jars and their transitive dependencies (#6184)
- Before this change, our assembly included test jars and test dependencies, which added unnecessary bloat to the assembly and classpath. This not only increases the distribution size but also potentially introduces CVE-prone JARs into the binary assemblies. - With this change, we modify the assembly to exclude these test jars and their dependencies. This will result in a leaner, more secure assembly with a faster startup time. - To ensure cached_classpath is same as before this change, we retain some jars (including test jars) in assembly pom.xml which we actually remove during tarball creation. Signed-off-by: Istvan Toth <[email protected]>
1 parent 40d9e9d commit 3fbe4fb

File tree

12 files changed

+27
-166
lines changed

12 files changed

+27
-166
lines changed

‎bin/hbase

Lines changed: 2 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -514,8 +514,9 @@ add_opentelemetry_agent() {
514514

515515
#Add the development env class path stuff
516516
if $in_dev_env; then
517+
# The below line makes available all the jars required in dev_env, even if we may not add them
518+
# to assembly. See HBASE-28433 for more info.
517519
add_maven_deps_to_classpath "cached_classpath.txt"
518-
519520
if [[ $jline_needed ]]; then
520521
add_maven_deps_to_classpath "cached_classpath_jline.txt"
521522
elif [[ $jruby_needed ]]; then
@@ -766,13 +767,6 @@ elif [ "$COMMAND" = "copyreppeers" ] ; then
766767
CLASS='org.apache.hadoop.hbase.replication.CopyReplicationPeers'
767768
else
768769
CLASS=$COMMAND
769-
if [[ "$CLASS" =~ .*IntegrationTest.* ]] ; then
770-
for f in ${HBASE_HOME}/lib/test/*.jar; do
771-
if [ -f "${f}" ]; then
772-
CLASSPATH="${CLASSPATH}:${f}"
773-
fi
774-
done
775-
fi
776770
fi
777771

778772
add_jdk17_jvm_flags

‎hbase-assembly/pom.xml

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,13 +48,17 @@
4848
<artifactId>hbase-shaded-mapreduce</artifactId>
4949
</dependency>
5050
<!-- Intra-project dependencies -->
51+
<!-- We don't really add this to assembly tarball, we retain it here just to dump it into
52+
cached_classpath.txt ! See HBASE-28433 for more info. -->
5153
<dependency>
5254
<groupId>org.apache.hbase</groupId>
5355
<artifactId>hbase-it</artifactId>
5456
<type>test-jar</type>
5557
</dependency>
5658
<!-- Hamcrest is required by hbase-it (via junit), but as long as we're grabbing the hbase-it test-jar,
5759
maven dependency resolution won't pick it up for us. -->
60+
<!-- We don't really add this to assembly tarball, we retain it here just to dump it into
61+
cached_classpath.txt ! See HBASE-28433 for more info. -->
5862
<dependency>
5963
<groupId>org.hamcrest</groupId>
6064
<artifactId>hamcrest-core</artifactId>
@@ -73,6 +77,8 @@
7377
<groupId>org.apache.hbase</groupId>
7478
<artifactId>hbase-mapreduce</artifactId>
7579
</dependency>
80+
<!-- We don't really add this to assembly tarball, we retain it here just to dump it into
81+
cached_classpath.txt ! See HBASE-28433 for more info. -->
7682
<dependency>
7783
<groupId>org.apache.hbase</groupId>
7884
<artifactId>hbase-mapreduce</artifactId>
@@ -83,6 +89,8 @@
8389
<groupId>org.apache.hbase</groupId>
8490
<artifactId>hbase-procedure</artifactId>
8591
</dependency>
92+
<!-- We don't really add this to assembly tarball, we retain it here just to dump it into
93+
cached_classpath.txt ! See HBASE-28433 for more info. -->
8694
<dependency>
8795
<groupId>org.apache.hbase</groupId>
8896
<artifactId>hbase-procedure</artifactId>
@@ -115,6 +123,8 @@
115123
<groupId>org.apache.hbase</groupId>
116124
<artifactId>hbase-external-blockcache</artifactId>
117125
</dependency>
126+
<!-- We don't really add this to assembly tarball, we retain it here just to dump it into
127+
cached_classpath.txt ! See HBASE-28433 for more info. -->
118128
<dependency>
119129
<groupId>org.apache.hbase</groupId>
120130
<artifactId>hbase-testing-util</artifactId>
@@ -235,10 +245,14 @@
235245
<groupId>io.opentelemetry.javaagent</groupId>
236246
<artifactId>opentelemetry-javaagent</artifactId>
237247
</dependency>
248+
<!-- We don't really add this to assembly tarball, we retain it here just to dump it into
249+
cached_classpath.txt ! See HBASE-28433 for more info. -->
238250
<dependency>
239251
<groupId>junit</groupId>
240252
<artifactId>junit</artifactId>
241253
</dependency>
254+
<!-- We don't really add this to assembly tarball, we retain it here just to dump it into
255+
cached_classpath.txt ! See HBASE-28433 for more info. -->
242256
<dependency>
243257
<groupId>org.mockito</groupId>
244258
<artifactId>mockito-core</artifactId>

‎hbase-assembly/src/main/assembly/client-components.xml

Lines changed: 0 additions & 55 deletions
Original file line numberDiff line numberDiff line change
@@ -87,60 +87,5 @@
8787
<include>*.dylib</include>
8888
</includes>
8989
</fileSet>
90-
<!-- This is only necessary until maven fixes the intra-project dependency bug
91-
in maven 3.0. Until then, we have to include the test jars for sub-projects. When
92-
fixed, the below dependencySet stuff is sufficient for pulling in the test jars as
93-
well, as long as they are added as dependencies in this project. Right now, we only
94-
have 1 submodule to accumulate, but we can copy/paste as necessary until maven is
95-
fixed. -->
96-
<!-- Used by PE and ltt -->
97-
<fileSet>
98-
<directory>${project.basedir}/../hbase-server/target/</directory>
99-
<outputDirectory>lib</outputDirectory>
100-
<includes>
101-
<include>${server.test.jar}</include>
102-
</includes>
103-
<fileMode>0644</fileMode>
104-
</fileSet>
105-
<fileSet>
106-
<directory>${project.basedir}/../hbase-mapreduce/target/</directory>
107-
<outputDirectory>lib</outputDirectory>
108-
<includes>
109-
<include>${mapreduce.test.jar}</include>
110-
</includes>
111-
<fileMode>0644</fileMode>
112-
</fileSet>
113-
<fileSet>
114-
<directory>${project.basedir}/../hbase-common/target/</directory>
115-
<outputDirectory>lib</outputDirectory>
116-
<includes>
117-
<include>${common.test.jar}</include>
118-
</includes>
119-
<fileMode>0644</fileMode>
120-
</fileSet>
121-
<fileSet>
122-
<directory>${project.basedir}/../hbase-asyncfs/target/</directory>
123-
<outputDirectory>lib</outputDirectory>
124-
<includes>
125-
<include>${asyncfs.test.jar}</include>
126-
</includes>
127-
<fileMode>0644</fileMode>
128-
</fileSet>
129-
<fileSet>
130-
<directory>${project.basedir}/../hbase-zookeeper/target/</directory>
131-
<outputDirectory>lib</outputDirectory>
132-
<includes>
133-
<include>${zookeeper.test.jar}</include>
134-
</includes>
135-
<fileMode>0644</fileMode>
136-
</fileSet>
137-
<fileSet>
138-
<directory>${project.basedir}/../hbase-procedure/target/</directory>
139-
<outputDirectory>lib</outputDirectory>
140-
<includes>
141-
<include>${procedure.test.jar}</include>
142-
</includes>
143-
<fileMode>0644</fileMode>
144-
</fileSet>
14590
</fileSets>
14691
</component>

‎hbase-assembly/src/main/assembly/client.xml

Lines changed: 2 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,8 @@
4545
<dependencySets>
4646
<dependencySet>
4747
<excludes>
48+
<exclude>org.apache.hadoop:*:test-jar</exclude>
49+
<exclude>org.apache.hbase:*:test-jar</exclude>
4850
<!-- Exclude J2EE libraries that get pulled in when building on JDK11 -->
4951
<exclude>com.sun.xml.ws:jaxws-ri</exclude>
5052
<!-- Exclude libraries that we put in their own dirs under lib/ -->
@@ -167,15 +169,6 @@
167169
<include>io.opentelemetry.javaagent:*</include>
168170
</includes>
169171
</dependencySet>
170-
<!-- Adds junit libs to lib/test -->
171-
<dependencySet>
172-
<outputDirectory>lib/test</outputDirectory>
173-
<includes>
174-
<include>junit:junit</include>
175-
<include>org.hamcrest:hamcrest-core</include>
176-
<include>org.mockito:mockito-core</include>
177-
</includes>
178-
</dependencySet>
179172
</dependencySets>
180173

181174
</assembly>

‎hbase-assembly/src/main/assembly/components.xml

Lines changed: 0 additions & 70 deletions
Original file line numberDiff line numberDiff line change
@@ -109,75 +109,5 @@
109109
<include>**/*.h</include>
110110
</includes>
111111
</fileSet>
112-
<!-- This is only necessary until maven fixes the intra-project dependency bug
113-
in maven 3.0. Until then, we have to include the test jars for sub-projects. When
114-
fixed, the below dependencySet stuff is sufficient for pulling in the test jars as
115-
well, as long as they are added as dependencies in this project. Right now, we only
116-
have 1 submodule to accumulate, but we can copy/paste as necessary until maven is
117-
fixed. -->
118-
<fileSet>
119-
<directory>${project.basedir}/../hbase-server/target/</directory>
120-
<outputDirectory>lib</outputDirectory>
121-
<includes>
122-
<include>${server.test.jar}</include>
123-
</includes>
124-
<fileMode>0644</fileMode>
125-
</fileSet>
126-
<fileSet>
127-
<directory>${project.basedir}/../hbase-it/target/</directory>
128-
<outputDirectory>lib</outputDirectory>
129-
<includes>
130-
<include>${it.test.jar}</include>
131-
</includes>
132-
<fileMode>0644</fileMode>
133-
</fileSet>
134-
<fileSet>
135-
<directory>${project.basedir}/../hbase-annotations/target/</directory>
136-
<outputDirectory>lib</outputDirectory>
137-
<includes>
138-
<include>${annotations.test.jar}</include>
139-
</includes>
140-
<fileMode>0644</fileMode>
141-
</fileSet>
142-
<fileSet>
143-
<directory>${project.basedir}/../hbase-common/target/</directory>
144-
<outputDirectory>lib</outputDirectory>
145-
<includes>
146-
<include>${common.test.jar}</include>
147-
</includes>
148-
<fileMode>0644</fileMode>
149-
</fileSet>
150-
<fileSet>
151-
<directory>${project.basedir}/../hbase-mapreduce/target/</directory>
152-
<outputDirectory>lib</outputDirectory>
153-
<includes>
154-
<include>${mapreduce.test.jar}</include>
155-
</includes>
156-
<fileMode>0644</fileMode>
157-
</fileSet>
158-
<fileSet>
159-
<directory>${project.basedir}/../hbase-asyncfs/target/</directory>
160-
<outputDirectory>lib</outputDirectory>
161-
<includes>
162-
<include>${asyncfs.test.jar}</include>
163-
</includes>
164-
<fileMode>0644</fileMode>
165-
</fileSet>
166-
<fileSet>
167-
<directory>${project.basedir}/../hbase-zookeeper/target/</directory>
168-
<outputDirectory>lib</outputDirectory>
169-
<includes>
170-
<include>${zookeeper.test.jar}</include>
171-
</includes>
172-
<fileMode>0644</fileMode>
173-
</fileSet>
174-
<fileSet>
175-
<directory>${project.basedir}/../hbase-procedure/target/</directory>
176-
<outputDirectory>lib</outputDirectory>
177-
<includes>
178-
<include>${procedure.test.jar}</include>
179-
</includes>
180-
<fileMode>0644</fileMode>
181-
</fileSet>
182112
</fileSets>
183113
</component>

‎hbase-assembly/src/main/assembly/hadoop-three-compat.xml

Lines changed: 2 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,6 @@
3333
<useAllReactorProjects>true</useAllReactorProjects>
3434
<includes>
3535
<!-- Keep this list sorted by name -->
36-
<include>org.apache.hbase:hbase-annotations</include>
3736
<include>org.apache.hbase:hbase-asyncfs</include>
3837
<include>org.apache.hbase:hbase-backup</include>
3938
<include>org.apache.hbase:hbase-balancer</include>
@@ -43,7 +42,6 @@
4342
<include>org.apache.hbase:hbase-external-blockcache</include>
4443
<include>org.apache.hbase:hbase-hadoop-compat</include>
4544
<include>org.apache.hbase:hbase-http</include>
46-
<include>org.apache.hbase:hbase-it</include>
4745
<include>org.apache.hbase:hbase-logging</include>
4846
<include>org.apache.hbase:hbase-mapreduce</include>
4947
<include>org.apache.hbase:hbase-diagnostics</include>
@@ -55,7 +53,6 @@
5553
<include>org.apache.hbase:hbase-rest</include>
5654
<include>org.apache.hbase:hbase-server</include>
5755
<include>org.apache.hbase:hbase-shell</include>
58-
<include>org.apache.hbase:hbase-testing-util</include>
5956
<include>org.apache.hbase:hbase-thrift</include>
6057
<include>org.apache.hbase:hbase-zookeeper</include>
6158
</includes>
@@ -66,6 +63,8 @@
6663
<dependencySets>
6764
<dependencySet>
6865
<excludes>
66+
<exclude>org.apache.hadoop:*:test-jar</exclude>
67+
<exclude>org.apache.hbase:*:test-jar</exclude>
6968
<!-- Exclude pom file -->
7069
<exclude>com.sun.xml.ws:jaxws-ri:pom</exclude>
7170
<!-- Exclude libraries that we put in their own dirs under lib/ -->
@@ -206,16 +205,6 @@
206205
<include>io.opentelemetry.javaagent:*</include>
207206
</includes>
208207
</dependencySet>
209-
210-
<!-- Adds junit libs to lib/test -->
211-
<dependencySet>
212-
<outputDirectory>lib/test</outputDirectory>
213-
<includes>
214-
<include>junit:junit</include>
215-
<include>org.hamcrest:hamcrest-core</include>
216-
<include>org.mockito:mockito-core</include>
217-
</includes>
218-
</dependencySet>
219208
</dependencySets>
220209

221210
</assembly>

‎hbase-endpoint/pom.xml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -256,6 +256,7 @@
256256
<dependency>
257257
<groupId>org.apache.hadoop</groupId>
258258
<artifactId>hadoop-minicluster</artifactId>
259+
<scope>test</scope>
259260
<exclusions>
260261
<exclusion>
261262
<groupId>com.google.guava</groupId>

‎hbase-examples/pom.xml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -310,6 +310,7 @@
310310
<dependency>
311311
<groupId>org.apache.hadoop</groupId>
312312
<artifactId>hadoop-minicluster</artifactId>
313+
<scope>test</scope>
313314
<exclusions>
314315
<exclusion>
315316
<groupId>javax.xml.bind.</groupId>

‎hbase-it/pom.xml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -370,6 +370,7 @@
370370
<dependency>
371371
<groupId>org.apache.hadoop</groupId>
372372
<artifactId>hadoop-minicluster</artifactId>
373+
<scope>test</scope>
373374
<exclusions>
374375
<exclusion>
375376
<groupId>javax.xml.bind</groupId>
@@ -384,6 +385,7 @@
384385
<dependency>
385386
<groupId>org.apache.hadoop</groupId>
386387
<artifactId>hadoop-minikdc</artifactId>
388+
<scope>test</scope>
387389
</dependency>
388390
</dependencies>
389391
<build>

‎hbase-shell/pom.xml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -190,6 +190,7 @@
190190
<dependency>
191191
<groupId>org.apache.hadoop</groupId>
192192
<artifactId>hadoop-minicluster</artifactId>
193+
<scope>test</scope>
193194
<exclusions>
194195
<exclusion>
195196
<groupId>com.google.guava</groupId>
@@ -204,6 +205,7 @@
204205
<dependency>
205206
<groupId>org.apache.hadoop</groupId>
206207
<artifactId>hadoop-minikdc</artifactId>
208+
<scope>test</scope>
207209
</dependency>
208210
</dependencies>
209211
<build>

0 commit comments

Comments
 (0)