fix: better categorization of launch directory corruption error (#1522)

src/integrationtests/java/com/aws/greengrass/integrationtests/lifecyclemanager/PluginComponentTest.java

@@ -15,10 +15,10 @@
 import com.aws.greengrass.config.Topics;
 import com.aws.greengrass.dependency.EZPlugins;
 import com.aws.greengrass.dependency.State;
-import com.aws.greengrass.deployment.DeploymentDocumentDownloader;
 import com.aws.greengrass.deployment.DefaultDeploymentTask;
 import com.aws.greengrass.deployment.DeploymentConfigMerger;
 import com.aws.greengrass.deployment.DeploymentDirectoryManager;
+import com.aws.greengrass.deployment.DeploymentDocumentDownloader;
 import com.aws.greengrass.deployment.DeploymentService;
 import com.aws.greengrass.deployment.ThingGroupHelper;
 import com.aws.greengrass.deployment.activator.KernelUpdateActivator;
@@ -90,7 +90,6 @@
 import static org.mockito.ArgumentMatchers.anyInt;
 import static org.mockito.ArgumentMatchers.eq;
 import static org.mockito.Mockito.doNothing;
-import static org.mockito.Mockito.doReturn;
 import static org.mockito.Mockito.spy;
 import static org.mockito.Mockito.verify;
 import static software.amazon.awssdk.services.greengrassv2.model.DeploymentComponentUpdatePolicyAction.NOTIFY_COMPONENTS;
@@ -269,7 +268,7 @@ void GIVEN_plugin_running_WHEN_plugin_removed_THEN_nucleus_bootstraps(ExtensionC
         setupPackageStoreAndConfigWithDigest();
         String deploymentId2 = "deployment2";
         // No need to actually verify directory setup or make directory changes here.
-        doReturn(true).when(kernelAltsSpy).isLaunchDirSetup();
+        doNothing().when(kernelAltsSpy).validateLaunchDirSetupVerbose();
         doNothing().when(kernelAltsSpy).prepareBootstrap(eq(deploymentId2));
 
         doNothing().when(kernelSpy).shutdown(anyInt(), eq(REQUEST_RESTART));
@@ -349,7 +348,7 @@ void GIVEN_kernel_WHEN_deploy_updated_plugin_THEN_request_kernel_restart(Extensi
         setupPackageStoreAndConfigWithDigest();
         String deploymentId2 = "deployment2";
         // No need to actually verify directory setup or make directory changes here.
-        doReturn(true).when(kernelAltsSpy).isLaunchDirSetup();
+        doNothing().when(kernelAltsSpy).validateLaunchDirSetupVerbose();
         doNothing().when(kernelAltsSpy).prepareBootstrap(eq(deploymentId2));
 
         doNothing().when(kernelSpy).shutdown(anyInt(), eq(REQUEST_RESTART));

src/main/java/com/aws/greengrass/deployment/activator/KernelUpdateActivator.java

@@ -6,7 +6,6 @@
 package com.aws.greengrass.deployment.activator;
 
 import com.aws.greengrass.deployment.bootstrap.BootstrapManager;
-import com.aws.greengrass.deployment.errorcode.DeploymentErrorCode;
 import com.aws.greengrass.deployment.errorcode.DeploymentErrorCodeUtils;
 import com.aws.greengrass.deployment.exceptions.DeploymentException;
 import com.aws.greengrass.deployment.exceptions.ServiceUpdateException;
@@ -16,6 +15,7 @@
 import com.aws.greengrass.lifecyclemanager.Kernel;
 import com.aws.greengrass.lifecyclemanager.KernelAlternatives;
 import com.aws.greengrass.lifecyclemanager.KernelLifecycle;
+import com.aws.greengrass.lifecyclemanager.exceptions.DirectoryValidationException;
 import com.aws.greengrass.util.Pair;
 import com.aws.greengrass.util.Utils;
 
@@ -58,13 +58,17 @@ public void activate(Map<String, Object> newConfig, Deployment deployment,
         if (!takeConfigSnapshot(totallyCompleteFuture)) {
             return;
         }
-
-        if (!kernelAlternatives.isLaunchDirSetup()) {
+        try {
+            kernelAlternatives.validateLaunchDirSetupVerbose();
+        } catch (DirectoryValidationException e) {
             totallyCompleteFuture.complete(
                     new DeploymentResult(DeploymentResult.DeploymentStatus.FAILED_NO_STATE_CHANGE,
                             new DeploymentException("Unable to process deployment. Greengrass launch directory"
-                                    + " is not set up or Greengrass is not set up as a system service",
-                                    DeploymentErrorCode.LAUNCH_DIRECTORY_CORRUPTED)));
+                                    + " is not set up or Greengrass is not set up as a system service", e)));
+            return;
+        } catch (DeploymentException e) {
+            totallyCompleteFuture.complete(
+                    new DeploymentResult(DeploymentResult.DeploymentStatus.FAILED_NO_STATE_CHANGE, e));
             return;
         }
 

src/main/java/com/aws/greengrass/lifecyclemanager/KernelAlternatives.java

@@ -7,7 +7,10 @@
 
 import com.aws.greengrass.deployment.DeploymentDirectoryManager;
 import com.aws.greengrass.deployment.bootstrap.BootstrapManager;
+import com.aws.greengrass.deployment.errorcode.DeploymentErrorCode;
+import com.aws.greengrass.deployment.exceptions.DeploymentException;
 import com.aws.greengrass.deployment.model.Deployment;
+import com.aws.greengrass.lifecyclemanager.exceptions.DirectoryValidationException;
 import com.aws.greengrass.logging.api.Logger;
 import com.aws.greengrass.logging.impl.LogManager;
 import com.aws.greengrass.util.CommitableWriter;
@@ -142,6 +145,34 @@ public boolean isLaunchDirSetup() {
         return Files.isSymbolicLink(getCurrentDir()) && validateLaunchDirSetup(getCurrentDir());
     }
 
+    /**
+     * Validate that launch directory is set up.
+     *
+     * @throws DirectoryValidationException when a file is missing
+     * @throws DeploymentException when user is not allowed to change file permission
+     */
+    public void validateLaunchDirSetupVerbose() throws DirectoryValidationException, DeploymentException {
+        Path currentDir = getCurrentDir();
+        if (!Files.isSymbolicLink(currentDir)) {
+            throw new DirectoryValidationException("Missing symlink to current nucleus launch directory");
+        }
+        Path loaderPath = getLoaderPathFromLaunchDir(currentDir);
+        if (Files.exists(loaderPath)) {
+            if (!loaderPath.toFile().canExecute()) {
+                // Ensure that the loader is executable so that we can exec it when restarting Nucleus
+                try {
+                    Platform.getInstance().setPermissions(OWNER_RWX_EVERYONE_RX, loaderPath);
+                } catch (IOException e) {
+                    throw new DeploymentException(
+                            String.format("Unable to set loader script at %s as executable", loaderPath), e)
+                            .withErrorContext(e, DeploymentErrorCode.SET_PERMISSION_ERROR);
+                }
+            }
+        } else {
+            throw new DirectoryValidationException("Missing loader file at " + currentDir.toAbsolutePath());
+        }
+    }
+
     @SuppressWarnings("PMD.ConfusingTernary")
     private boolean validateLaunchDirSetup(Path path) {
         Path loaderPath = getLoaderPathFromLaunchDir(path);

src/main/java/com/aws/greengrass/lifecyclemanager/exceptions/DirectoryValidationException.java

@@ -0,0 +1,18 @@
+/*
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package com.aws.greengrass.lifecyclemanager.exceptions;
+
+import com.aws.greengrass.deployment.errorcode.DeploymentErrorCode;
+import com.aws.greengrass.deployment.exceptions.DeploymentException;
+
+public class DirectoryValidationException extends DeploymentException {
+    static final long serialVersionUID = -3387516993124229948L;
+
+    public DirectoryValidationException(String message) {
+        super(message);
+        super.addErrorCode(DeploymentErrorCode.LAUNCH_DIRECTORY_CORRUPTED);
+    }
+}

src/test/java/com/aws/greengrass/deployment/activator/KernelUpdateActivatorTest.java

@@ -20,7 +20,9 @@
 import com.aws.greengrass.lifecyclemanager.Kernel;
 import com.aws.greengrass.lifecyclemanager.KernelAlternatives;
 import com.aws.greengrass.lifecyclemanager.KernelLifecycle;
+import com.aws.greengrass.lifecyclemanager.exceptions.DirectoryValidationException;
 import com.aws.greengrass.testcommons.testutilities.GGExtension;
+import com.aws.greengrass.testcommons.testutilities.TestUtils;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.ExtendWith;
@@ -33,6 +35,7 @@
 import java.nio.file.Path;
 import java.util.Arrays;
 import java.util.Collections;
+import java.util.List;
 import java.util.Map;
 import java.util.concurrent.CompletableFuture;
 
@@ -81,7 +84,6 @@ class KernelUpdateActivatorTest {
     @BeforeEach
     void beforeEach() {
         doReturn(deploymentDirectoryManager).when(context).get(eq(DeploymentDirectoryManager.class));
-        lenient().doReturn(true).when(kernelAlternatives).isLaunchDirSetup();
         doReturn(kernelAlternatives).when(context).get(eq(KernelAlternatives.class));
         doReturn(context).when(kernel).getContext();
         lenient().doReturn(config).when(kernel).getConfig();
@@ -188,4 +190,24 @@ void GIVEN_deployment_activate_WHEN_bootstrap_requires_reboot_THEN_request_reboo
         verify(kernelAlternatives).prepareBootstrap(eq("testId"));
         verify(kernel).shutdown(eq(30), eq(REQUEST_REBOOT));
     }
+
+    @Test
+    void GIVEN_launch_dir_corrupted_WHEN_deployment_activate_THEN_deployment_fail(ExtensionContext context)
+            throws Exception {
+        ignoreExceptionOfType(context, DirectoryValidationException.class);
+
+        DirectoryValidationException mockException = new DirectoryValidationException("error msg");
+        doThrow(mockException).when(kernelAlternatives).validateLaunchDirSetupVerbose();
+        kernelUpdateActivator.activate(newConfig, deployment, totallyCompleteFuture);
+        ArgumentCaptor<DeploymentResult> captor = ArgumentCaptor.forClass(DeploymentResult.class);
+        verify(totallyCompleteFuture).complete(captor.capture());
+        DeploymentResult result = captor.getValue();
+        assertEquals(result.getDeploymentStatus(), DeploymentResult.DeploymentStatus.FAILED_NO_STATE_CHANGE);
+        assertTrue(result.getFailureCause() instanceof DeploymentException);
+        assertEquals(mockException, result.getFailureCause().getCause());
+
+        List<String> expectedStack = Arrays.asList("DEPLOYMENT_FAILURE", "LAUNCH_DIRECTORY_CORRUPTED");
+        List<String> expectedTypes = Collections.singletonList("NUCLEUS_ERROR");
+        TestUtils.validateGenerateErrorReport(result.getFailureCause(), expectedStack, expectedTypes);
+    }
 }

src/test/java/com/aws/greengrass/deployment/errorcode/DeploymentErrorCodeUtilsTest.java

@@ -17,7 +17,6 @@
 import com.aws.greengrass.lifecyclemanager.GreengrassService;
 import com.aws.greengrass.lifecyclemanager.Kernel;
 import com.aws.greengrass.testcommons.testutilities.GGExtension;
-import com.aws.greengrass.util.Pair;
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.JsonMappingException;
 import com.vdurmont.semver4j.Semver;
@@ -50,8 +49,8 @@
 import static com.aws.greengrass.deployment.errorcode.DeploymentErrorCode.MULTIPLE_NUCLEUS_RESOLVED_ERROR;
 import static com.aws.greengrass.deployment.errorcode.DeploymentErrorCode.S3_HEAD_OBJECT_ACCESS_DENIED;
 import static com.aws.greengrass.testcommons.testutilities.ExceptionLogProtector.ignoreExceptionOfType;
+import static com.aws.greengrass.testcommons.testutilities.TestUtils.validateGenerateErrorReport;
 import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.junit.jupiter.api.Assertions.assertTrue;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
@@ -113,15 +112,15 @@ class DeploymentErrorCodeUtilsTest {
     void GIVEN_internal_exception_WHEN_generate_error_report_THEN_expected_error_stack_and_types_returned() {
         // test an empty exception
         DeploymentException e = new DeploymentException("empty exception");
-        testGenerateErrorReport(e, Collections.singletonList("DEPLOYMENT_FAILURE"), Collections.emptyList());
+        validateGenerateErrorReport(e, Collections.singletonList("DEPLOYMENT_FAILURE"), Collections.emptyList());
 
         // test an exception with inheritance hierarchy and an empty exception
         InvalidImageOrAccessDeniedException e1 = new InvalidImageOrAccessDeniedException("docker access denied", e);
         List<String> expectedStackFromE1 =
                 Arrays.asList("DEPLOYMENT_FAILURE", "ARTIFACT_DOWNLOAD_ERROR", "DOCKER_ERROR",
                         "DOCKER_IMAGE_NOT_VALID");
         List<String> expectedTypesFromE1 = Collections.singletonList("DEPENDENCY_ERROR");
-        testGenerateErrorReport(e1, expectedStackFromE1, expectedTypesFromE1);
+        validateGenerateErrorReport(e1, expectedStackFromE1, expectedTypesFromE1);
 
         // test an arbitrary chain of exception, error stack should order from outside to inside
         List<DeploymentErrorCode> errorCodeList =
@@ -138,70 +137,70 @@ void GIVEN_internal_exception_WHEN_generate_error_report_THEN_expected_error_sta
                         "MULTIPLE_NUCLEUS_RESOLVED_ERROR", "COMPONENT_BROKEN", "COMPONENT_UPDATE_ERROR");
         List<String> expectedTypesFromE2 =
                 Arrays.asList("DEVICE_ERROR", "PERMISSION_ERROR", "REQUEST_ERROR");
-        testGenerateErrorReport(e, expectedStackFromE2, expectedTypesFromE2);
+        validateGenerateErrorReport(e, expectedStackFromE2, expectedTypesFromE2);
 
         // test a combination of inheritance and chain
         List<String> expectedStackFromCombined = Stream.concat(expectedStackFromE1.stream(),
                         expectedStackFromE2.stream().filter(code -> !"DEPLOYMENT_FAILURE".equals(code)))
                 .collect(Collectors.toList());
         List<String> expectedTypesFromCombined =
                 Stream.concat(expectedTypesFromE1.stream(), expectedTypesFromE2.stream()).collect(Collectors.toList());
-        testGenerateErrorReport(e1, expectedStackFromCombined, expectedTypesFromCombined);
+        validateGenerateErrorReport(e1, expectedStackFromCombined, expectedTypesFromCombined);
 
         // test with an additional error context
         IOException ioException = new IOException("some io unzip error");
         rootCause.initCause(ioException);
         e.withErrorContext(ioException, IO_UNZIP_ERROR);
 
         expectedStackFromCombined.addAll(Arrays.asList("IO_ERROR", "IO_UNZIP_ERROR"));
-        testGenerateErrorReport(e1, expectedStackFromCombined, expectedTypesFromCombined);
+        validateGenerateErrorReport(e1, expectedStackFromCombined, expectedTypesFromCombined);
     }
 
     @Test
     void GIVEN_external_exception_WHEN_generate_error_report_THEN_expected_error_stack_and_types_returned() {
         // test s3 exception
         when(s3Exception.statusCode()).thenReturn(502);
-        testGenerateErrorReport(s3Exception, Arrays.asList("DEPLOYMENT_FAILURE", "S3_ERROR", "S3_SERVER_ERROR"),
+        validateGenerateErrorReport(s3Exception, Arrays.asList("DEPLOYMENT_FAILURE", "S3_ERROR", "S3_SERVER_ERROR"),
                 Arrays.asList("DEPENDENCY_ERROR", "SERVER_ERROR"));
         when(s3Exception.statusCode()).thenReturn(404);
-        testGenerateErrorReport(s3Exception, Arrays.asList("DEPLOYMENT_FAILURE", "S3_ERROR", "S3_RESOURCE_NOT_FOUND"),
+        validateGenerateErrorReport(s3Exception, Arrays.asList("DEPLOYMENT_FAILURE", "S3_ERROR", "S3_RESOURCE_NOT_FOUND"),
                 Collections.singletonList("DEPENDENCY_ERROR"));
         when(s3Exception.statusCode()).thenReturn(403);
-        testGenerateErrorReport(s3Exception, Arrays.asList("DEPLOYMENT_FAILURE", "S3_ERROR", "S3_ACCESS_DENIED"),
+        validateGenerateErrorReport(s3Exception, Arrays.asList("DEPLOYMENT_FAILURE", "S3_ERROR", "S3_ACCESS_DENIED"),
                 Arrays.asList("DEPENDENCY_ERROR", "PERMISSION_ERROR"));
         when(s3Exception.statusCode()).thenReturn(429);
-        testGenerateErrorReport(s3Exception, Arrays.asList("DEPLOYMENT_FAILURE", "S3_ERROR", "S3_BAD_REQUEST"),
+        validateGenerateErrorReport(s3Exception, Arrays.asList("DEPLOYMENT_FAILURE", "S3_ERROR", "S3_BAD_REQUEST"),
                 Collections.singletonList("DEPENDENCY_ERROR"));
 
         // test gg v2 data exception
-        testGenerateErrorReport(resourceNotFoundException,
+        validateGenerateErrorReport(resourceNotFoundException,
                 Arrays.asList("DEPLOYMENT_FAILURE", "CLOUD_API_ERROR", "RESOURCE_NOT_FOUND"),
                 Collections.singletonList("REQUEST_ERROR"));
-        testGenerateErrorReport(accessDeniedException,
+        validateGenerateErrorReport(accessDeniedException,
                 Arrays.asList("DEPLOYMENT_FAILURE", "CLOUD_API_ERROR", "ACCESS_DENIED"),
                 Collections.singletonList("PERMISSION_ERROR"));
-        testGenerateErrorReport(validationException,
+        validateGenerateErrorReport(validationException,
                 Arrays.asList("DEPLOYMENT_FAILURE", "CLOUD_API_ERROR", "BAD_REQUEST"),
                 Collections.singletonList("NUCLEUS_ERROR"));
-        testGenerateErrorReport(throttlingException,
+        validateGenerateErrorReport(throttlingException,
                 Arrays.asList("DEPLOYMENT_FAILURE", "CLOUD_API_ERROR", "THROTTLING_ERROR"),
                 Collections.singletonList("REQUEST_ERROR"));
-        testGenerateErrorReport(conflictException,
+        validateGenerateErrorReport(conflictException,
                 Arrays.asList("DEPLOYMENT_FAILURE", "CLOUD_API_ERROR", "CONFLICTED_REQUEST"),
                 Collections.singletonList("REQUEST_ERROR"));
-        testGenerateErrorReport(internalServerException,
+        validateGenerateErrorReport(internalServerException,
                 Arrays.asList("DEPLOYMENT_FAILURE", "CLOUD_API_ERROR", "SERVER_ERROR"),
                 Collections.singletonList("SERVER_ERROR"));
 
         // test io exception
-        testGenerateErrorReport(jsonMappingException,
+        validateGenerateErrorReport(jsonMappingException,
                 Arrays.asList("DEPLOYMENT_FAILURE", "IO_ERROR", "IO_MAPPING_ERROR"), Collections.emptyList());
-        testGenerateErrorReport(jsonProcessingException,
+        validateGenerateErrorReport(jsonProcessingException,
                 Arrays.asList("DEPLOYMENT_FAILURE", "IO_ERROR", "IO_WRITE_ERROR"),
                 Collections.singletonList("DEVICE_ERROR"));
 
         // test network exception
-        testGenerateErrorReport(sdkClientException, Arrays.asList("DEPLOYMENT_FAILURE", "NETWORK_ERROR"),
+        validateGenerateErrorReport(sdkClientException, Arrays.asList("DEPLOYMENT_FAILURE", "NETWORK_ERROR"),
                 Collections.singletonList("NETWORK_ERROR"));
     }
 
@@ -262,24 +261,4 @@ void GIVEN_malformed_arn_WHEN_classify_component_error_THEN_return_generic_type(
         assertEquals(DeploymentErrorCodeUtils.classifyComponentError(service, kernel),
                 DeploymentErrorType.COMPONENT_ERROR);
     }
-
-
-    private static void testGenerateErrorReport(Throwable e, List<String> expectedErrorStack,
-                                                List<String> expectedErrorTypes) {
-        Pair<List<String>, List<String>> errorReport =
-                DeploymentErrorCodeUtils.generateErrorReportFromExceptionStack(e);
-        assertListEquals(errorReport.getLeft(), expectedErrorStack);
-        assertListEqualsWithoutOrder(errorReport.getRight(), expectedErrorTypes);
-    }
-
-    private static void assertListEquals(List<String> first, List<String> second) {
-        assertEquals(first.size(), second.size());
-        for (int i = 0; i < first.size(); i++) {
-            assertEquals(first.get(i), second.get(i));
-        }
-    }
-
-    private static void assertListEqualsWithoutOrder(List<String> first, List<String> second) {
-        assertTrue(first.size() == second.size() && first.containsAll(second) && second.containsAll(first));
-    }
 }