feat(QTDI-693): manage nested jars

component-runtime-manager/src/main/java/org/talend/sdk/component/runtime/manager/ComponentManager.java

@@ -37,9 +37,11 @@
 import static org.talend.sdk.component.runtime.manager.reflect.Constructors.findConstructor;
 import static org.talend.sdk.component.runtime.manager.util.Lazy.lazy;
 
+import java.io.BufferedReader;
 import java.io.File;
 import java.io.IOException;
 import java.io.InputStream;
+import java.io.InputStreamReader;
 import java.io.ObjectStreamException;
 import java.io.Serializable;
 import java.lang.annotation.Annotation;
@@ -1303,16 +1305,42 @@ public void onCreate(final Container container) {
             try {
                 String alreadyScannedClasses = null;
                 Filter filter = KnownClassesFilter.INSTANCE;
-                try (final InputStream containerFilterConfig =
-                        container.getLoader().getResourceAsStream("TALEND-INF/scanning.properties")) {
-                    if (containerFilterConfig != null) {
+                // we need to scan the nested repository
+                if (container.hasNestedRepository()) {
+                    try {
+                        final Enumeration<URL> urls = loader.getResources("TALEND-INF/scanning.properties");
                         final Properties config = new Properties();
-                        config.load(containerFilterConfig);
-                        filter = createScanningFilter(config);
-                        alreadyScannedClasses = config.getProperty("classes.list");
+                        while (urls.hasMoreElements()) {
+                            final URL url = urls.nextElement();
+                            // ensure we scan the correct classes of plugin in the nested repository
+                            if ("nested".equals(url.getProtocol())
+                                    && url.getPath().contains(container.getRootModule())) {
+                                try (final BufferedReader reader =
+                                        new BufferedReader(new InputStreamReader(url.openStream()))) {
+                                    config.load(reader);
+                                    filter = createScanningFilter(config);
+                                    alreadyScannedClasses = config.getProperty("classes.list");
+                                    break;
+                                }
+                            }
+                        }
+                    } catch (IOException e) {
+                        log.info("[onCreate] Can't read nested scanning.properties: {}", e.getMessage());
+                    }
+                }
+                // normal scanning or nested scan failed
+                if (alreadyScannedClasses == null) {
+                    try (final InputStream containerFilterConfig =
+                            container.getLoader().getResourceAsStream("TALEND-INF/scanning.properties")) {
+                        if (containerFilterConfig != null) {
+                            final Properties config = new Properties();
+                            config.load(containerFilterConfig);
+                            filter = createScanningFilter(config);
+                            alreadyScannedClasses = config.getProperty("classes.list");
+                        }
+                    } catch (final IOException e) {
+                        log.debug(e.getMessage(), e);
                     }
-                } catch (final IOException e) {
-                    log.debug(e.getMessage(), e);
                 }
 
                 AnnotationFinder optimizedFinder = null;

component-runtime-manager/src/test/java/org/talend/sdk/component/runtime/manager/ComponentManagerTest.java

@@ -427,10 +427,11 @@ void extendFamilyInNestedRepo(@TempDir final File temporaryFolder) throws Except
                         .of(container.getLoader().getURLs())
                         .map(Files::toFile)
                         .map(File::getName)
-                        .sorted()
+                        .sorted() // !! for asserts
                         .toArray(String[]::new);
-                assertEquals(1, dependencies.length); // ignored transitive deps, enables the new root to control it
-                assertEquals("main.jar", dependencies[0]); // transitive-1.0.0.jar is nested
+                assertEquals(2, dependencies.length); // ignored transitive deps, enables the new root to control it
+                assertEquals("fatjar.jar", dependencies[0]); // transitive-1.0.0.jar is nested
+                assertEquals("main.jar", dependencies[1]); // main.jar containing fatjar.jar
             } finally {
                 if (!transitive.delete()) {
                     transitive.deleteOnExit();

container/container-core/src/main/java/org/talend/sdk/component/classloader/ConfigurableClassLoader.java

@@ -111,6 +111,8 @@ public class ConfigurableClassLoader extends URLClassLoader {
     @Getter
     private final List<String> cacheableClasses;
 
+    private List<URL> nestedURLs = new ArrayList<>();
+
     public ConfigurableClassLoader(final String id, final URL[] urls, final ClassLoader parent,
             final Predicate<String> parentFilter, final Predicate<String> childFirstFilter,
             final String[] nestedDependencies, final String[] jvmPrefixes) {
@@ -155,6 +157,7 @@ private void loadNestedDependencies(final ClassLoader parent, final String[] nes
             if (url == null) {
                 throw new IllegalArgumentException("Didn't find " + resource + " in " + asList(nestedDependencies));
             }
+            nestedURLs.add(url);
             final Map<String, Resource> resources = new HashMap<>();
             final URLConnection urlConnection;
             final Manifest manifest;
@@ -458,6 +461,15 @@ public Enumeration<URL> findResources(final String name) throws IOException {
         return enumeration(aggregated);
     }
 
+    @Override
+    public URL[] getURLs() {
+        final List<URL> urls = new ArrayList<>(Arrays.asList(super.getURLs()));
+        if (!nestedURLs.isEmpty()) {
+            urls.addAll(nestedURLs);
+        }
+        return urls.toArray(new URL[0]);
+    }
+
     private boolean isNestedDependencyResource(final String name) {
         return name != null && name.startsWith(NESTED_MAVEN_REPOSITORY);
     }

container/container-core/src/main/java/org/talend/sdk/component/container/Container.java

@@ -307,6 +307,10 @@ public Date getCreated() {
         return created.get();
     }
 
+    public boolean hasNestedRepository() {
+        return hasNestedRepository;
+    }
+
     public void registerTransformer(final ClassFileTransformer transformer) {
         transformers.add(transformer);
     }

container/container-core/src/main/java/org/talend/sdk/component/container/ContainerManager.java

@@ -29,9 +29,11 @@
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.InputStreamReader;
+import java.net.URL;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.nio.file.Paths;
 import java.security.MessageDigest;
 import java.security.NoSuchAlgorithmException;
 import java.util.ArrayList;
@@ -110,16 +112,23 @@ public ContainerManager(final DependenciesResolutionConfiguration dependenciesRe
         this.logInfoLevelMapping = logInfoLevelMapping;
         this.containerInitializer = containerInitializer;
         this.resolver = dependenciesResolutionConfiguration.getResolver();
-        this.rootRepositoryLocation = ofNullable(dependenciesResolutionConfiguration.getRootRepositoryLocation())
-                .filter(Files::exists)
-                .orElseGet(() -> PathFactory.get(System.getProperty("user.home", "")).resolve(".m2/repository"));
 
-        if (log.isDebugEnabled()) {
-            log.debug("Using root repository: " + this.rootRepositoryLocation.toAbsolutePath());
+        Path rootRepo = ofNullable(dependenciesResolutionConfiguration.getRootRepositoryLocation())
+                .filter(Files::exists)
+                .orElseGet(() -> PathFactory.get(System.getProperty("user.home")).resolve(".m2/repository"));
+        // if we've defaulted to user home m2 (fallback), we want to check if we're in running in a fatjar
+        if (PathFactory.get(System.getProperty("user.home")).resolve(".m2/repository").equals(rootRepo)) {
+            final URL nested = classLoaderConfiguration.getParent().getResource("MAVEN-INF/repository");
+            if (nested != null) {
+                rootRepo = Paths.get(nested.getFile().replace("file:", ""));
+            }
         }
-
+        this.rootRepositoryLocation = rootRepo;
+        info("Using root repository: " + this.rootRepositoryLocation.toAbsolutePath());
+        final String pluginsLocation = System.getProperty("talend.component.manager.plugins.location",
+                "TALEND-INF/plugins.properties");
         final String nestedPluginMappingResource = ofNullable(classLoaderConfiguration.getNestedPluginMappingResource())
-                .orElse("TALEND-INF/plugins.properties");
+                .orElse(pluginsLocation);
         this.classLoaderConfiguration = new ClassLoaderConfiguration(
                 ofNullable(classLoaderConfiguration.getParent()).orElseGet(ContainerManager.class::getClassLoader),
                 ofNullable(classLoaderConfiguration.getClassesFilter()).orElseGet(() -> name -> true),
@@ -477,8 +486,8 @@ public Container create() {
                     ? nestedContainerMapping.getOrDefault(module, module)
                     : module;
             final Path resolved = resolve(moduleLocation);
-            info("Creating module " + moduleLocation + " (from " + module
-                    + (Files.exists(resolved) ? ", location=" + resolved.toAbsolutePath().toString() : "") + ")");
+            info(String.format("Creating module %s (from %s, location=%s)", moduleLocation, module,
+                    resolved.toAbsolutePath()));
             final Stream<Artifact> classpath = Stream
                     .concat(getBuiltInClasspath(moduleLocation),
                             additionalClasspath == null ? Stream.empty() : additionalClasspath.stream());