Skip to content

Commit

Permalink
feat(fips): refactor CA downloader, allow downloading multiple CAs (#…
Browse files Browse the repository at this point in the history
  • Loading branch information
yitingb authored Jul 23, 2024
1 parent 48a7153 commit fc4767c
Show file tree
Hide file tree
Showing 3 changed files with 184 additions and 95 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -8,24 +8,18 @@
import com.aws.greengrass.deployment.DeviceConfiguration;
import com.aws.greengrass.deployment.exceptions.DeviceConfigurationException;
import com.aws.greengrass.lifecyclemanager.Kernel;
import com.aws.greengrass.util.EncryptionUtils;
import com.aws.greengrass.util.IamSdkClientFactory;
import com.aws.greengrass.util.IotSdkClientFactory;
import com.aws.greengrass.util.IotSdkClientFactory.EnvironmentStage;
import com.aws.greengrass.util.Permissions;
import com.aws.greengrass.util.ProxyUtils;
import com.aws.greengrass.util.RegionUtils;
import com.aws.greengrass.util.RootCAUtils;
import com.aws.greengrass.util.StsSdkClientFactory;
import com.aws.greengrass.util.Utils;
import com.aws.greengrass.util.exceptions.InvalidEnvironmentStageException;
import lombok.AllArgsConstructor;
import lombok.Getter;
import org.apache.commons.lang3.StringUtils;
import software.amazon.awssdk.http.HttpExecuteRequest;
import software.amazon.awssdk.http.HttpExecuteResponse;
import software.amazon.awssdk.http.SdkHttpClient;
import software.amazon.awssdk.http.SdkHttpFullRequest;
import software.amazon.awssdk.http.SdkHttpMethod;
import software.amazon.awssdk.regions.Region;
import software.amazon.awssdk.services.greengrassv2.GreengrassV2Client;
import software.amazon.awssdk.services.greengrassv2.model.ComponentDeploymentSpecification;
Expand Down Expand Up @@ -70,28 +64,18 @@
import software.amazon.awssdk.services.sts.StsClient;
import software.amazon.awssdk.services.sts.model.GetCallerIdentityRequest;
import software.amazon.awssdk.utils.ImmutableMap;
import software.amazon.awssdk.utils.IoUtils;

import java.io.BufferedWriter;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.PrintStream;
import java.net.HttpURLConnection;
import java.net.URI;
import java.net.URISyntaxException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.StandardOpenOption;
import java.util.Arrays;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.UUID;
import java.util.stream.Collectors;

/**
* Provision a device by registering as an IoT thing, creating roles and template first party components.
Expand All @@ -115,7 +99,6 @@ public class DeviceProvisioningHelper {
+ " }\n"
+ " ]\n"
+ "}";
private static final String ROOT_CA_URL = "https://www.amazontrust.com/repository/AmazonRootCA1.pem";
private static final String IOT_ROLE_POLICY_NAME_PREFIX = "GreengrassTESCertificatePolicy";
private static final String GREENGRASS_CLI_COMPONENT_NAME = "aws.greengrass.Cli";
private static final String INITIAL_DEPLOYMENT_NAME_FORMAT = "Deployment for %s";
Expand Down Expand Up @@ -268,82 +251,6 @@ public void cleanThing(IotClient client, ThingInfo thing, boolean deletePolicies
DeleteCertificateRequest.builder().certificateId(thing.certificateId).forceDelete(true).build());
}

/*
* Download root CA to a local file.
*
* To support HTTPS proxies and other custom truststore configurations, append to the file if it exists.
*/
private void downloadRootCAToFile(File f) {
if (f.exists()) {
outStream.printf("Root CA file found at \"%s\". Contents will be preserved.%n", f);
}
outStream.printf("Downloading Root CA from \"%s\"%n", ROOT_CA_URL);
try {
downloadFileFromURL(ROOT_CA_URL, f);
removeDuplicateCertificates(f);
} catch (IOException e) {
// Do not block as the root CA file may have been manually provisioned
outStream.printf("Failed to download Root CA - %s%n", e);
}
}

private void removeDuplicateCertificates(File f) {
try {
String certificates = new String(Files.readAllBytes(f.toPath()), StandardCharsets.UTF_8);
Set<String> uniqueCertificates =
Arrays.stream(certificates.split(EncryptionUtils.CERTIFICATE_PEM_HEADER))
.map(s -> s.trim())
.collect(Collectors.toSet());

try (BufferedWriter bw = Files.newBufferedWriter(f.toPath(), StandardCharsets.UTF_8)) {
for (String certificate : uniqueCertificates) {
if (certificate.length() > 0) {
bw.write(EncryptionUtils.CERTIFICATE_PEM_HEADER);
bw.write("\n");
bw.write(certificate);
bw.write("\n");
}
}
}
} catch (IOException e) {
outStream.printf("Failed to remove duplicate certificates - %s%n", e);
}
}

/*
* Download content from a URL to a local file.
*/
@SuppressWarnings("PMD.AvoidFileStream")
private void downloadFileFromURL(String url, File f) throws IOException {
SdkHttpFullRequest request = SdkHttpFullRequest.builder()
.uri(URI.create(url))
.method(SdkHttpMethod.GET)
.build();

HttpExecuteRequest executeRequest = HttpExecuteRequest.builder()
.request(request)
.build();

try (SdkHttpClient client = getSdkHttpClient()) {
HttpExecuteResponse executeResponse = client.prepareRequest(executeRequest).call();

int responseCode = executeResponse.httpResponse().statusCode();
if (responseCode != HttpURLConnection.HTTP_OK) {
throw new IOException("Received invalid response code: " + responseCode);
}

try (InputStream inputStream = executeResponse.responseBody().get();
OutputStream outputStream = Files.newOutputStream(f.toPath(), StandardOpenOption.CREATE,
StandardOpenOption.APPEND, StandardOpenOption.SYNC)) {
IoUtils.copy(inputStream, outputStream);
}
}
}

private SdkHttpClient getSdkHttpClient() {
return ProxyUtils.getSdkHttpClientBuilder().build();
}

/**
* Update the kernel config with iot thing info, in specific CA, private Key and cert path.
*
Expand All @@ -366,7 +273,14 @@ public void updateKernelConfigWithIotConfiguration(Kernel kernel, ThingInfo thin
}

Path caFilePath = certPath.resolve("rootCA.pem");
downloadRootCAToFile(caFilePath.toFile());

try {
outStream.printf("Downloading CA from \"%s\"%n", RootCAUtils.AMAZON_ROOT_CA_1_URL);
RootCAUtils.downloadRootCAToFile(caFilePath.toFile(), RootCAUtils.AMAZON_ROOT_CA_1_URL);
} catch (IOException e) {
// Do not block as the root CA file may have been manually provisioned
outStream.printf("Failed to download CA from path - %s%n", e);
}

Path privKeyFilePath = certPath.resolve("privKey.key");
Files.write(privKeyFilePath, thing.keyPair.privateKey().getBytes(StandardCharsets.UTF_8));
Expand Down
145 changes: 145 additions & 0 deletions src/main/java/com/aws/greengrass/util/RootCAUtils.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,145 @@
/*
* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
* SPDX-License-Identifier: Apache-2.0
*/

package com.aws.greengrass.util;

import com.aws.greengrass.logging.api.Logger;
import com.aws.greengrass.logging.impl.LogManager;
import software.amazon.awssdk.http.HttpExecuteRequest;
import software.amazon.awssdk.http.HttpExecuteResponse;
import software.amazon.awssdk.http.SdkHttpClient;
import software.amazon.awssdk.http.SdkHttpFullRequest;
import software.amazon.awssdk.http.SdkHttpMethod;
import software.amazon.awssdk.utils.IoUtils;

import java.io.BufferedWriter;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.HttpURLConnection;
import java.net.URI;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.StandardOpenOption;
import java.util.Arrays;
import java.util.Set;
import java.util.stream.Collectors;

public final class RootCAUtils {
public static final String AMAZON_ROOT_CA_1_URL = "https://www.amazontrust.com/repository/AmazonRootCA1.pem";
public static final String AMAZON_ROOT_CA_2_URL = "https://www.amazontrust.com/repository/AmazonRootCA2.pem";
public static final String AMAZON_ROOT_CA_3_URL = "https://www.amazontrust.com/repository/AmazonRootCA3.pem";
public static final String AMAZON_ROOT_CA_4_URL = "https://www.amazontrust.com/repository/AmazonRootCA4.pem";
private static final Logger logger = LogManager.getLogger(ProxyUtils.class);

private RootCAUtils() {

}

/**
* Download root CA to a local file.
* To support HTTPS proxies and other custom truststore configurations, append to the file if it exists.
* @param f destination file
* @param urls list of URLs needs to be downloaded
* @throws IOException if download failed
*/
public static void downloadRootCAToFile(File f, String... urls) throws IOException {
if (f.exists()) {
logger.atInfo().log("CA file found at {}. Contents will be preserved.", f);
}
try {
for (String url : urls) {
logger.atInfo().log("Downloading CA from {}", url);
downloadFileFromURL(url, f);
}
removeDuplicateCertificates(f);
} catch (IOException e) {
throw new IOException("Failed to download CA from path", e);
}
}

private static void removeDuplicateCertificates(File f) {
try {
String certificates = new String(Files.readAllBytes(f.toPath()), StandardCharsets.UTF_8);
Set<String> uniqueCertificates =
Arrays.stream(certificates.split(EncryptionUtils.CERTIFICATE_PEM_HEADER))
.map(s -> s.trim())
.collect(Collectors.toSet());

try (BufferedWriter bw = Files.newBufferedWriter(f.toPath(), StandardCharsets.UTF_8)) {
for (String certificate : uniqueCertificates) {
if (certificate.length() > 0) {
bw.write(EncryptionUtils.CERTIFICATE_PEM_HEADER);
bw.write("\n");
bw.write(certificate);
bw.write("\n");
}
}
}
} catch (IOException e) {
logger.atDebug().log("Failed to remove duplicate certificates - %s%n", e);
}
}

/**
* Download content from a URL to a local file.
* @param url the URL from which the content needs to be downloaded
* @param f destination local file
* @throws IOException if download failed
*/
@SuppressWarnings("PMD.AvoidFileStream")
public static void downloadFileFromURL(String url, File f) throws IOException {
SdkHttpFullRequest request = SdkHttpFullRequest.builder()
.uri(URI.create(url))
.method(SdkHttpMethod.GET)
.build();

HttpExecuteRequest executeRequest = HttpExecuteRequest.builder()
.request(request)
.build();

try (SdkHttpClient client = ProxyUtils.getSdkHttpClientBuilder().build()) {
HttpExecuteResponse executeResponse = client.prepareRequest(executeRequest).call();

int responseCode = executeResponse.httpResponse().statusCode();
if (responseCode != HttpURLConnection.HTTP_OK) {
throw new IOException("Received invalid response code: " + responseCode);
}

try (InputStream inputStream = executeResponse.responseBody().get();
OutputStream outputStream = Files.newOutputStream(f.toPath(), StandardOpenOption.CREATE,
StandardOpenOption.APPEND, StandardOpenOption.SYNC)) {
IoUtils.copy(inputStream, outputStream);
}
}
}

/**
* Download rootCA 3 to root path.
* @param rootCAPath the root path for CAs
* @param urls the CA url array
* @return if CA downloaded
*/
public static boolean downloadRootCAsWithPath(String rootCAPath, String... urls) {
if (rootCAPath == null || rootCAPath.isEmpty()) {
return false;
}
Path caFilePath = Paths.get(rootCAPath);
if (!Files.exists(caFilePath)) {
return false;
}
try {
downloadRootCAToFile(caFilePath.toFile(), urls);
} catch (IOException e) {
logger.atError().log("Failed to download CA from path - {}", caFilePath.toAbsolutePath(), e);
return false;
}
return true;
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,10 @@

import com.aws.greengrass.lifecyclemanager.Kernel;
import com.aws.greengrass.testcommons.testutilities.GGExtension;
import com.aws.greengrass.util.EncryptionUtils;
import com.aws.greengrass.util.IamSdkClientFactory;
import com.aws.greengrass.util.IotSdkClientFactory;
import com.aws.greengrass.util.RootCAUtils;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
Expand Down Expand Up @@ -59,9 +61,15 @@
import software.amazon.awssdk.services.sts.model.GetCallerIdentityRequest;
import software.amazon.awssdk.services.sts.model.GetCallerIdentityResponse;

import java.io.File;
import java.net.URISyntaxException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.stream.Collectors;

import static com.aws.greengrass.componentmanager.KernelConfigResolver.CONFIGURATION_CONFIG_KEY;
import static com.aws.greengrass.deployment.DeviceConfiguration.DEFAULT_NUCLEUS_COMPONENT_NAME;
Expand Down Expand Up @@ -418,6 +426,28 @@ void GIVEN_test_update_device_config_WHEN_thing_info_provided_THEN_add_config_to
IOT_ROLE_ALIAS_TOPIC).getOnce());
}

@Test
void GIVEN_device_config_WHEN_download_multiple_CAs_THEN_combine_and_save_at_Root_CA_file_location()
throws Exception {
kernel = new Kernel()
.parseArgs("-i", getClass().getResource("blank_config.yaml").toString(), "-r", tempRootDir.toString());

deviceProvisioningHelper.updateKernelConfigWithIotConfiguration(kernel,
new DeviceProvisioningHelper.ThingInfo(getThingArn(), "thingname", "certarn", "certid", "certpem",
KeyPair.builder().privateKey("privateKey").publicKey("publicKey").build(), "xxxxxx-ats.iot.us-east-1.amazonaws.com",
"xxxxxx.credentials.iot.us-east-1.amazonaws.com"), TEST_REGION, "roleAliasName", null);
Path certPath = kernel.getNucleusPaths().rootPath();
Path caFilePath = certPath.resolve("rootCA.pem");
File caFile = caFilePath.toFile();

RootCAUtils.downloadRootCAToFile(caFile, RootCAUtils.AMAZON_ROOT_CA_3_URL);

String certificates = new String(Files.readAllBytes(caFile.toPath()), StandardCharsets.UTF_8);
List<String> certificateArray = Arrays.stream(certificates.split(EncryptionUtils.CERTIFICATE_PEM_HEADER)).filter(s -> !s.isEmpty()).collect(Collectors.toList());

assertEquals(2, certificateArray.size());
}

@Test
void GIVEN_test_clean_thing_WHEN_thing_info_and_cert_and_things_deleted() {
when(iotClient.listAttachedPolicies(any(ListAttachedPoliciesRequest.class)))
Expand Down

0 comments on commit fc4767c

Please sign in to comment.