From 19191f4bb069555d0b9d7d0b051766204c87ca2c Mon Sep 17 00:00:00 2001 From: Martin Wittlinger Date: Tue, 5 Sep 2023 14:15:43 +0200 Subject: [PATCH] feat: add fingerprints for JDK classes (#63) Co-authored-by: Aman Sharma --- terminator-commons/pom.xml | 5 + .../commons/fingerprint/JdkClass.java | 8 ++ .../commons/fingerprint/JdkIndexer.java | 87 ++++++++++++++++++ .../fingerprint/classfile/RuntimeClass.java | 17 ++++ .../commons/fingerprint/provenance/Jdk.java | 10 ++ .../fingerprint/classfile/ClassfileTest.java | 12 +++ .../GeneratedConstructorAccessor15.class | Bin 0 -> 1521 bytes .../classfile/GeneratedMethodAccessor1.class | Bin 0 -> 825 bytes .../java/io/github/algomaster99/Options.java | 61 +++++++++++- .../io/github/algomaster99/Terminator.java | 11 ++- watchdog-agent/src/test/java/AgentTest.java | 7 +- watchdog-agent/src/test/java/OptionsTest.java | 20 ++++ 12 files changed, 227 insertions(+), 11 deletions(-) create mode 100644 terminator-commons/src/main/java/io/github/algomaster99/terminator/commons/fingerprint/JdkClass.java create mode 100644 terminator-commons/src/main/java/io/github/algomaster99/terminator/commons/fingerprint/JdkIndexer.java create mode 100644 terminator-commons/src/main/java/io/github/algomaster99/terminator/commons/fingerprint/provenance/Jdk.java create mode 100644 terminator-commons/src/test/resources/classfile/GeneratedConstructorAccessor15.class create mode 100644 terminator-commons/src/test/resources/classfile/GeneratedMethodAccessor1.class diff --git a/terminator-commons/pom.xml b/terminator-commons/pom.xml index 295b9f33..fd79d21d 100644 --- a/terminator-commons/pom.xml +++ b/terminator-commons/pom.xml @@ -56,6 +56,11 @@ org.ow2.asm asm-util + + io.github.classgraph + classgraph + 4.8.162 + diff --git a/terminator-commons/src/main/java/io/github/algomaster99/terminator/commons/fingerprint/JdkClass.java b/terminator-commons/src/main/java/io/github/algomaster99/terminator/commons/fingerprint/JdkClass.java new file mode 100644 index 00000000..2e96125a --- /dev/null +++ b/terminator-commons/src/main/java/io/github/algomaster99/terminator/commons/fingerprint/JdkClass.java @@ -0,0 +1,8 @@ +package io.github.algomaster99.terminator.commons.fingerprint; + +import java.nio.ByteBuffer; + +/** + * A class that represents a JDK class. It contains the name of the class and the bytes of the class as a {@link ByteBuffer}. + */ +public record JdkClass(String name, ByteBuffer bytes) {} diff --git a/terminator-commons/src/main/java/io/github/algomaster99/terminator/commons/fingerprint/JdkIndexer.java b/terminator-commons/src/main/java/io/github/algomaster99/terminator/commons/fingerprint/JdkIndexer.java new file mode 100644 index 00000000..c6635b42 --- /dev/null +++ b/terminator-commons/src/main/java/io/github/algomaster99/terminator/commons/fingerprint/JdkIndexer.java @@ -0,0 +1,87 @@ +package io.github.algomaster99.terminator.commons.fingerprint; + +import io.github.classgraph.ClassGraph; +import io.github.classgraph.Resource; +import io.github.classgraph.ScanResult; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Enumeration; +import java.util.List; +import java.util.Set; +import java.util.jar.JarEntry; +import java.util.jar.JarFile; +import nonapi.io.github.classgraph.classpath.SystemJarFinder; + +/** + * The JdkIndexer class provides a utility to list all JDK classes by scanning the JDK used for the execution of the application. + */ +public class JdkIndexer { + + /** + * Returns a list of all JDK classes. The list is populated by scanning the JDK used for the execution of this application. + * + * @return a list of all JDK classes, never null. + */ + public static List listJdkClasses() { + List jdkClasses = new ArrayList<>(); + try (ScanResult scanResult = new ClassGraph() + .enableSystemJarsAndModules() + .disableDirScanning() + .disableJarScanning() + .acceptLibOrExtJars() + .acceptModules("jdk.*", "java.*") + .ignoreClassVisibility() + .enableMemoryMapping() + .scan(); ) { + scanResult.getAllClasses().forEach(classInfo -> { + Resource resource = classInfo.getResource(); + if (resource != null) { + byte[] byteBuffer; + try { + byteBuffer = resource.load(); + jdkClasses.add( + new JdkClass(classInfo.getName().replaceAll("\\.", "/"), ByteBuffer.wrap(byteBuffer))); + } catch (IOException e) { + System.err.println("Error loading resource " + resource + ": " + e); + } + } + }); + } + jdkClasses.addAll(indexJrt()); + return jdkClasses; + } + + /** + * Creates an index of the external Jrt jar. This jar provides an API for older jvms to access the modules in the JDK. + * The jvm itself does not need this jar. + * @return the list of external jrt jdk classes + */ + private static List indexJrt() { + List jdkClasses = new ArrayList<>(); + Set jreLibOrExtJars = SystemJarFinder.getJreLibOrExtJars(); + for (String path : jreLibOrExtJars) { + try { + jdkClasses.addAll(readJarFile(path)); + } catch (Exception e) { + System.err.println("Error reading jar file " + path + ": " + e); + } + } + return jdkClasses; + } + + private static List readJarFile(String jarFilePath) throws IOException { + List jdkClasses = new ArrayList<>(); + try (JarFile jarFile = new JarFile(jarFilePath)) { + Enumeration entries = jarFile.entries(); + while (entries.hasMoreElements()) { + JarEntry entry = entries.nextElement(); + if (entry.getName().endsWith(".class")) { + byte[] byteBuffer = jarFile.getInputStream(entry).readAllBytes(); + jdkClasses.add(new JdkClass(entry.getName().replace(".class", ""), ByteBuffer.wrap(byteBuffer))); + } + } + } + return jdkClasses; + } +} diff --git a/terminator-commons/src/main/java/io/github/algomaster99/terminator/commons/fingerprint/classfile/RuntimeClass.java b/terminator-commons/src/main/java/io/github/algomaster99/terminator/commons/fingerprint/classfile/RuntimeClass.java index 8528c1de..77835c77 100644 --- a/terminator-commons/src/main/java/io/github/algomaster99/terminator/commons/fingerprint/classfile/RuntimeClass.java +++ b/terminator-commons/src/main/java/io/github/algomaster99/terminator/commons/fingerprint/classfile/RuntimeClass.java @@ -20,4 +20,21 @@ public static boolean isSynthetic(byte[] classfileBytes) { ClassReader reader = new ClassReader(classfileBytes); return (reader.getAccess() & Opcodes.ACC_SYNTHETIC) != 0; } + + /** + * Skip classes inherited from {@link jdk.internal.reflect.MagicAccessorImpl} because they are generated at runtime using ASM. + */ + public static boolean isGeneratedClassExtendingMagicAccessor(byte[] classfileBytes) { + ClassReader reader = new ClassReader(classfileBytes); + try { + return RuntimeClass.class + .getClassLoader() + .loadClass(reader.getSuperName().replace("/", ".")) + .getSuperclass() + .getName() + .equals("jdk.internal.reflect.MagicAccessorImpl"); + } catch (ClassNotFoundException e) { + return false; + } + } } diff --git a/terminator-commons/src/main/java/io/github/algomaster99/terminator/commons/fingerprint/provenance/Jdk.java b/terminator-commons/src/main/java/io/github/algomaster99/terminator/commons/fingerprint/provenance/Jdk.java new file mode 100644 index 00000000..ad3479cf --- /dev/null +++ b/terminator-commons/src/main/java/io/github/algomaster99/terminator/commons/fingerprint/provenance/Jdk.java @@ -0,0 +1,10 @@ +package io.github.algomaster99.terminator.commons.fingerprint.provenance; + +import io.github.algomaster99.terminator.commons.fingerprint.classfile.ClassFileAttributes; + +public record Jdk(ClassFileAttributes classFileAttributes) implements Provenance { + @Override + public ClassFileAttributes classFileAttributes() { + return classFileAttributes; + } +} diff --git a/terminator-commons/src/test/java/io/github/algomaster99/terminator/commons/fingerprint/classfile/ClassfileTest.java b/terminator-commons/src/test/java/io/github/algomaster99/terminator/commons/fingerprint/classfile/ClassfileTest.java index 2dd3e2ff..ffe45cae 100644 --- a/terminator-commons/src/test/java/io/github/algomaster99/terminator/commons/fingerprint/classfile/ClassfileTest.java +++ b/terminator-commons/src/test/java/io/github/algomaster99/terminator/commons/fingerprint/classfile/ClassfileTest.java @@ -48,6 +48,18 @@ void isSyntheticClass_true() throws IOException { assertThat(RuntimeClass.isSynthetic(modifiedBytes)).isTrue(); } + @Test + void isGeneratedClassExtendingMagicAccessor_true() throws IOException { + Path generatedConstructorAccessor = CLASSFILE.resolve("GeneratedConstructorAccessor15.class"); + assertThat(RuntimeClass.isGeneratedClassExtendingMagicAccessor( + Files.readAllBytes(generatedConstructorAccessor))) + .isTrue(); + + Path generatedMethodAccessor = CLASSFILE.resolve("GeneratedMethodAccessor1.class"); + assertThat(RuntimeClass.isGeneratedClassExtendingMagicAccessor(Files.readAllBytes(generatedMethodAccessor))) + .isTrue(); + } + private static byte[] makeClassfileSynthetic(byte[] classfileBytes) { ClassNode classNode = new ClassNode(); ClassReader classReader = new ClassReader(classfileBytes); diff --git a/terminator-commons/src/test/resources/classfile/GeneratedConstructorAccessor15.class b/terminator-commons/src/test/resources/classfile/GeneratedConstructorAccessor15.class new file mode 100644 index 0000000000000000000000000000000000000000..8d27a8eded17c262b4401ba8c0845e6a46215564 GIT binary patch literal 1521 zcma)+Yf}?37==%`Ei5Y+xd?)QfKo160U0l#BNWgg1B^u!@Rn{vDJ&aicgyA1_!sn3 zXJiKa?2mGMv(#=n^1;ruX>v}UP0pL_uirn;0gU5_VRA3CAJ4gg@LcZ1J+b2mJBYs& zuJCvuG70JWftR-f>CM`<@O|lxPZ?kgV^>bR!k;`ibPQB7w5S<}o;*37h!3uX?YMzz zhMJk2n+u*Z#A3^P{D{XL?q*ACNxDbU=0Q%nueh6Wgf~6BVZlJHi5k=~)Vtz1Nvq_p zEf|Jk+Ztmvy+_-b9{zWTVcQ}IVl~%Dc$=F?XfK# zTa>0P+`|JCaf~xq=}Z5HLNQPhTNWnqP{~sa4I20~psGHia>wS%s5BlTpO|=zrwnyz z8i{yw7H05FX>@?KqmtO=9=GXU8F-*$VCh9@Lq|VWwlDuYbC1!*jBYl>d>MTVn3SrfKPN@=yF8Rq=hfwN=ld2X<52i z0~uAPC7r0oL)?xDfh?uz&`88vvXH}`()P>to!XTi?fXFOJEa3dAtXu@E)HqmzVb%M zlm!pI5(6+)&|OfU3i?lXjebX|*Gw&=wtA$1!S70^ZaplrDS4CLVHMO)ouOuP^qkyZ zQGbT^%>r&+j5Y%kKhaQt`K^Ev8s7R`K;Oj~I@F(m3sln7ZuH@Lv2CSZFUHVB{W1Dh I(Zf1`KbpCRdH?_b literal 0 HcmV?d00001 diff --git a/terminator-commons/src/test/resources/classfile/GeneratedMethodAccessor1.class b/terminator-commons/src/test/resources/classfile/GeneratedMethodAccessor1.class new file mode 100644 index 0000000000000000000000000000000000000000..5aa20ae06f3d7dae6577cd1cd57ba61ec52ea807 GIT binary patch literal 825 zcmZvZTWb?R6vzLkxoq7;Z7z+ityNpKyGbSVMUaRRf=Iwi!62v)GTn^nrjy;UJE@m1 z<5$q9poPXK1wWK{)}+~$x-ffY&iT#%T;|vBZzlkDQ6t>$cMt2Xr)A&?RS)D_B^_OV zBt01jExS*oKJdFu$B|*^2fGG{u>IdF|B~&&P#H)Ow))~o)RpjhbMZY9h=Jsc?nl|} zd>Bdt8A7Eeb<@!!p(0l^=#2)_(;;EavR^+ws}fO-JPa)6ki|Jd*7c73LrJ)6CGcna zeQv0cRG7K!vlO8eC+Zvo{yVX+xV-a(^*GQ{A`Dw1)DPb~a;RP3GjNfx5fe{GO6~c7 zr~D`_N`zX>ojysMyK_X8ts?@SRpP=jH-FFdTz#LAvg{XmT*8Wp zB34b5u}WA@PUN``T(8%dV(Srcg|Lz&=AF-=t0pSAW+EMFEyblx2Weo7u&Dj1E}?8C zCp*iQ$4%TaaRb|gbj$DZGnw-aLJ$^MJ^B`S&(-ta=2L+!vCUM+*!&X3`ItkVu_){y zgB05bwV%ixqxf=+%ahu7WOsg`v~yAbK1KdA``0F4VG=%%VX@=Yn=v*f$EZY`>k}5q Yb8S^D&1{iRHtyg8i&uH2Ic){_1MlJ7W&i*H literal 0 HcmV?d00001 diff --git a/watchdog-agent/src/main/java/io/github/algomaster99/Options.java b/watchdog-agent/src/main/java/io/github/algomaster99/Options.java index fd263312..5c0d56d4 100644 --- a/watchdog-agent/src/main/java/io/github/algomaster99/Options.java +++ b/watchdog-agent/src/main/java/io/github/algomaster99/Options.java @@ -7,12 +7,20 @@ import io.github.algomaster99.terminator.commons.cyclonedx.Bom14Schema; import io.github.algomaster99.terminator.commons.cyclonedx.Component; import io.github.algomaster99.terminator.commons.cyclonedx.CycloneDX; +import io.github.algomaster99.terminator.commons.fingerprint.JdkIndexer; +import io.github.algomaster99.terminator.commons.fingerprint.classfile.ClassFileAttributes; +import io.github.algomaster99.terminator.commons.fingerprint.classfile.ClassfileVersion; +import io.github.algomaster99.terminator.commons.fingerprint.classfile.HashComputer; +import io.github.algomaster99.terminator.commons.fingerprint.provenance.Jdk; import io.github.algomaster99.terminator.commons.fingerprint.provenance.Provenance; import io.github.algomaster99.terminator.commons.jar.JarDownloader; import java.io.File; import java.io.IOException; +import java.nio.ByteBuffer; import java.nio.file.Files; import java.nio.file.Path; +import java.security.NoSuchAlgorithmException; +import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -23,7 +31,7 @@ public class Options { private static final Logger LOGGER = LoggerFactory.getLogger(Options.class); private Map> fingerprints = new HashMap<>(); - + private Map> jdkFingerprints = new HashMap<>(); private boolean skipShutdown = false; private boolean isSbomPassed = false; @@ -91,12 +99,17 @@ public Options(String agentArgs) { } else { LOGGER.info("Taking fingerprint from file: " + fingerprints); } + processJdk(); } public Map> getFingerprints() { return fingerprints; } + public Map> getJdkFingerprints() { + return jdkFingerprints; + } + public boolean shouldSkipShutdown() { return skipShutdown; } @@ -151,4 +164,50 @@ private void processAllComponents(Bom14Schema sbom) { } } } + + private void processJdk() { + JdkIndexer.listJdkClasses().forEach(resource -> { + try { + byte[] classfileBytes = toArray(resource.bytes()); + String classfileVersion = ClassfileVersion.getVersion(classfileBytes); + String hash = HashComputer.computeHash(classfileBytes, algorithm); + jdkFingerprints.computeIfAbsent( + resource.name(), + k -> new ArrayList<>( + List.of((new Jdk(new ClassFileAttributes(classfileVersion, hash, algorithm)))))); + jdkFingerprints.computeIfPresent(resource.name(), (k, v) -> { + v.add(new Jdk(new ClassFileAttributes(classfileVersion, hash, algorithm))); + return v; + }); + fingerprints.computeIfAbsent( + resource.name(), + k -> new ArrayList<>( + List.of((new Jdk(new ClassFileAttributes(classfileVersion, hash, algorithm)))))); + fingerprints.computeIfPresent(resource.name(), (k, v) -> { + v.add(new Jdk(new ClassFileAttributes(classfileVersion, hash, algorithm))); + return v; + }); + } catch (NoSuchAlgorithmException e) { + LOGGER.error("Failed to compute hash with algorithm: " + algorithm, e); + throw new RuntimeException(e); + } catch (Exception e) { + LOGGER.error("Failed to compute hash for: " + resource, e); + } + }); + } + + /** + * Converts a bytebuffer to a byte array. If the buffer has an array, it returns it, otherwise it copies the bytes. This is needed because the buffer is not guaranteed to have an array. + * See {@link java.nio.ByteBuffer#hasArray()} and {@link java.nio.DirectByteBuffer}. + * @param buffer the buffer to convert + * @return the byte array + */ + private byte[] toArray(ByteBuffer buffer) { + if (buffer.hasArray()) { + return buffer.array(); + } + byte[] bytes = new byte[buffer.remaining()]; + buffer.get(bytes); + return bytes; + } } diff --git a/watchdog-agent/src/main/java/io/github/algomaster99/Terminator.java b/watchdog-agent/src/main/java/io/github/algomaster99/Terminator.java index d5935a39..98a508da 100644 --- a/watchdog-agent/src/main/java/io/github/algomaster99/Terminator.java +++ b/watchdog-agent/src/main/java/io/github/algomaster99/Terminator.java @@ -14,9 +14,6 @@ public class Terminator { private static Options options; - private static final List INTERNAL_PACKAGES = - List.of("java/", "javax/", "jdk/", "sun/", "com/sun/", "org/xml/sax", "org/w3c/dom/"); - public static void premain(String agentArgs, Instrumentation inst) { options = new Options(agentArgs); inst.addTransformer(new ClassFileTransformer() { @@ -27,6 +24,7 @@ public byte[] transform( Class classBeingRedefined, ProtectionDomain protectionDomain, byte[] classfileBuffer) { + return isLoadedClassWhitelisted(className, classfileBuffer); } }); @@ -34,10 +32,13 @@ public byte[] transform( private static byte[] isLoadedClassWhitelisted(String className, byte[] classfileBuffer) { Map> fingerprints = options.getFingerprints(); - if (RuntimeClass.isProxyClass(classfileBuffer)) { + if (RuntimeClass.isProxyClass(classfileBuffer) + || RuntimeClass.isGeneratedClassExtendingMagicAccessor(classfileBuffer)) { return classfileBuffer; } - if (INTERNAL_PACKAGES.stream().anyMatch(className::startsWith)) { + if (className.contains("$")) { + // FIXME: we need to check inner classes without loading them. Maybe add the hashes for inner classes in the + // fingerprints? return classfileBuffer; } for (String expectedClassName : fingerprints.keySet()) { diff --git a/watchdog-agent/src/test/java/AgentTest.java b/watchdog-agent/src/test/java/AgentTest.java index f44c92f9..a64a8258 100644 --- a/watchdog-agent/src/test/java/AgentTest.java +++ b/watchdog-agent/src/test/java/AgentTest.java @@ -68,7 +68,8 @@ void shouldDisallowLoadingCustomJDKClass() throws MavenInvocationException, IOEx } @Test - void sorald_0_8_5_shouldExitWith_1() throws IOException, InterruptedException { + void sorald_0_8_5_shouldExitWith_0() throws IOException, InterruptedException { + // contract: sorald 0.8.5 should execute as the SBOM + external jars has every dependency. Path project = Paths.get("src/test/resources/sorald-0.8.5"); Path sbom = project.resolve("bom.json"); @@ -93,7 +94,6 @@ void sorald_0_8_5_shouldExitWith_1() throws IOException, InterruptedException { Process p = pb.start(); int exitCode = p.waitFor(); - assertThat(exitCode).isEqualTo(0); } @@ -124,7 +124,6 @@ void spoon_10_4_0_depscan_4_2_2() throws IOException, InterruptedException { private int runSpoonWithSbom(Path sbom) throws IOException, InterruptedException { Path spoonExecutable = project.resolve("spoon-core-10.4.0-jar-with-dependencies.jar"); Path workload = project.resolve("Main.java").toAbsolutePath(); - String agentArgs = "sbom=" + sbom; String[] cmd = { "java", @@ -206,11 +205,9 @@ private static void deleteContentsOfFile(String file) throws InterruptedExceptio private static String getAgentPath(String agentArgs) throws IOException { String tempDir = System.getProperty("java.io.tmpdir"); Path traceCollector = Path.of(tempDir, "watchdog-agent.jar"); - try (InputStream traceCollectorStream = Terminator.class.getResourceAsStream("/watchdog-agent.jar")) { Files.copy(traceCollectorStream, traceCollector, StandardCopyOption.REPLACE_EXISTING); } - return traceCollector.toAbsolutePath() + "=" + agentArgs; } diff --git a/watchdog-agent/src/test/java/OptionsTest.java b/watchdog-agent/src/test/java/OptionsTest.java index 6fd3afe6..aabcccc1 100644 --- a/watchdog-agent/src/test/java/OptionsTest.java +++ b/watchdog-agent/src/test/java/OptionsTest.java @@ -1,6 +1,7 @@ import static io.github.algomaster99.terminator.commons.fingerprint.ParsingHelper.deserializeFingerprints; import static org.assertj.core.api.Assertions.assertThat; +import io.github.algomaster99.Options; import io.github.algomaster99.terminator.commons.fingerprint.provenance.Jar; import io.github.algomaster99.terminator.commons.fingerprint.provenance.Maven; import io.github.algomaster99.terminator.commons.fingerprint.provenance.Provenance; @@ -8,6 +9,7 @@ import java.nio.file.Path; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; @@ -46,4 +48,22 @@ void maven_jar() throws NoSuchMethodException, InvocationTargetException, Illega .hasAtLeastOneElementOfType(Jar.class); } } + + @Test + void verifyIfJdkIndexerFindsJdkClassesDeterministically() throws Exception { + // generating 2 times the jdk fingerprint should result in the same fingerprint + Options options = new Options("skipShutdown=true"); + Options options2 = new Options("skipShutdown=true"); + assertThat(options.getJdkFingerprints()).isNotEmpty(); + assertThat(options2.getJdkFingerprints()).isNotEmpty(); + assertThat(options.getJdkFingerprints()).isEqualTo(options2.getJdkFingerprints()); + System.out.println(options.getJdkFingerprints().size()); + } + + @Test + void verifyJdkIndexerFindsOrgXmlSax() throws Exception { + Options options = new Options("skipShutdown=true"); + var var = options.getJdkFingerprints().keySet().stream().collect(Collectors.toSet()); + assertThat(var).contains("org/xml/sax/helpers/NamespaceSupport"); + } }