diff --git a/core/src/main/resources/reference.conf b/core/src/main/resources/reference.conf index fc0960473bf..2128dced028 100644 --- a/core/src/main/resources/reference.conf +++ b/core/src/main/resources/reference.conf @@ -412,6 +412,13 @@ docker { max-retries = 3 // Supported registries (Docker Hub, Google, Quay) can have additional configuration set separately + aws { + throttle { + number-of-requests = 1000 + per = 100 seconds + } + num-threads = 10 + } azure { // Worst case `ReadOps per minute` value from official docs // https://github.com/MicrosoftDocs/azure-docs/blob/main/includes/container-registry-limits.md diff --git a/dockerHashing/src/main/scala/cromwell/docker/DockerImageIdentifier.scala b/dockerHashing/src/main/scala/cromwell/docker/DockerImageIdentifier.scala index 7c20760b644..98acc38af68 100644 --- a/dockerHashing/src/main/scala/cromwell/docker/DockerImageIdentifier.scala +++ b/dockerHashing/src/main/scala/cromwell/docker/DockerImageIdentifier.scala @@ -1,5 +1,6 @@ package cromwell.docker +import cromwell.docker.registryv2.flows.aws.AwsElasticContainerRegistry.isEcr import cromwell.docker.registryv2.flows.azure.AzureContainerRegistry import scala.util.{Failure, Success, Try} @@ -19,7 +20,9 @@ sealed trait DockerImageIdentifier { lazy val nameWithDefaultRepository = // In ACR, the repository is part of the registry domain instead of the path // e.g. `terrabatchdev.azurecr.io` - if (host.exists(_.contains(AzureContainerRegistry.domain))) + // In ECR, an image with no repository is supported + // e.g. 123456790.dkr.ecr.eu-west-2.amazonaws.com/example-tool + if (host.exists(_.contains(AzureContainerRegistry.domain)) || !host.exists(isEcr)) image else repository.getOrElse("library") + s"/$image" diff --git a/dockerHashing/src/main/scala/cromwell/docker/DockerInfoActor.scala b/dockerHashing/src/main/scala/cromwell/docker/DockerInfoActor.scala index 3b3c0e5d7c5..e7bb5e7777b 100644 --- a/dockerHashing/src/main/scala/cromwell/docker/DockerInfoActor.scala +++ b/dockerHashing/src/main/scala/cromwell/docker/DockerInfoActor.scala @@ -14,6 +14,7 @@ import cromwell.core.actor.StreamIntegration.{BackPressure, StreamContext} import cromwell.core.{Dispatcher, DockerConfiguration} import cromwell.docker.DockerInfoActor._ import cromwell.docker.registryv2.DockerRegistryV2Abstract +import cromwell.docker.registryv2.flows.aws.AwsElasticContainerRegistry import cromwell.docker.registryv2.flows.azure.AzureContainerRegistry import cromwell.docker.registryv2.flows.dockerhub.DockerHubRegistry import cromwell.docker.registryv2.flows.google.GoogleRegistry @@ -239,6 +240,7 @@ object DockerInfoActor { // To add a new registry, simply add it to that list List( + ("aws", { c: DockerRegistryConfig => new AwsElasticContainerRegistry(c) }), ("azure", { c: DockerRegistryConfig => new AzureContainerRegistry(c) }), ("dockerhub", { c: DockerRegistryConfig => new DockerHubRegistry(c) }), ("google", { c: DockerRegistryConfig => new GoogleRegistry(c) }), diff --git a/dockerHashing/src/main/scala/cromwell/docker/registryv2/flows/aws/AwsElasticContainerRegistry.scala b/dockerHashing/src/main/scala/cromwell/docker/registryv2/flows/aws/AwsElasticContainerRegistry.scala new file mode 100644 index 00000000000..c2aeda60b33 --- /dev/null +++ b/dockerHashing/src/main/scala/cromwell/docker/registryv2/flows/aws/AwsElasticContainerRegistry.scala @@ -0,0 +1,78 @@ +package cromwell.docker.registryv2.flows.aws + +import cats.effect.IO +import cromwell.docker.{DockerImageIdentifier, DockerInfoActor, DockerRegistryConfig} +import cromwell.docker.registryv2.DockerRegistryV2Abstract +import cromwell.docker.registryv2.flows.aws.AwsElasticContainerRegistry.{isEcr, isPublicEcr} +import org.http4s.{AuthScheme, Header} +import org.http4s.client.Client +import software.amazon.awssdk.regions.Region +import software.amazon.awssdk.services.ecr.EcrClient +import software.amazon.awssdk.services.ecrpublic.EcrPublicClient +import software.amazon.awssdk.services.ecrpublic.model.GetAuthorizationTokenRequest + +import scala.compat.java8.OptionConverters.RichOptionalGeneric + +class AwsElasticContainerRegistry(config: DockerRegistryConfig) extends DockerRegistryV2Abstract(config) { + + private lazy val ecrClient = EcrClient.create() + private lazy val ecrPublicClient = EcrPublicClient.builder().region(Region.US_EAST_1).build() + + override def getAuthorizationScheme(dockerImageIdentifier: DockerImageIdentifier): AuthScheme = + if (isPublicEcr(dockerImageIdentifier.hostAsString)) AuthScheme.Bearer else AuthScheme.Basic + + override def accepts(dockerImageIdentifier: DockerImageIdentifier): Boolean = + isEcr(dockerImageIdentifier.hostAsString) + + /** + * (e.g registry-1.docker.io) + */ + override def registryHostName(dockerImageIdentifier: DockerImageIdentifier): String = + dockerImageIdentifier.host.getOrElse("") + + /** + * (e.g auth.docker.io) + */ + override def authorizationServerHostName(dockerImageIdentifier: DockerImageIdentifier): String = + dockerImageIdentifier.host.getOrElse("") + + override def getToken( + dockerInfoContext: DockerInfoActor.DockerInfoContext + )(implicit client: Client[IO]): IO[Option[String]] = + if (isPublicEcr(dockerInfoContext.dockerImageID.hostAsString)) getPublicEcrToken + else getPrivateEcrToken + + /** + * Builds the list of headers for the token request + */ + override def buildTokenRequestHeaders(dockerInfoContext: DockerInfoActor.DockerInfoContext): List[Header] = + List.empty + + private def getPublicEcrToken: IO[Option[String]] = + IO( + Option( + ecrPublicClient + .getAuthorizationToken(GetAuthorizationTokenRequest.builder().build()) + .authorizationData() + .authorizationToken() + ) + ) + + private def getPrivateEcrToken: IO[Option[String]] = + IO( + ecrClient + .getAuthorizationToken() + .authorizationData() + .stream() + .findFirst() + .asScala + .map(_.authorizationToken()) + ) + +} + +object AwsElasticContainerRegistry { + def isEcr(host: String): Boolean = isPublicEcr(host) || isPrivateEcr(host) + def isPublicEcr(host: String): Boolean = host.contains("public.ecr.aws") + def isPrivateEcr(host: String): Boolean = host.contains("amazonaws.com") +} diff --git a/project/Dependencies.scala b/project/Dependencies.scala index fc9beae1f12..c16856fb3d0 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -400,6 +400,8 @@ object Dependencies { "batch", "core", "cloudwatchlogs", + "ecr", + "ecrpublic", "s3", "sts", ).map(artifactName => "software.amazon.awssdk" % artifactName % awsSdkV)