diff --git a/pulumi/AWSMegatests/Pulumi.dev.yaml b/pulumi/AWSMegatests/Pulumi.dev.yaml index 33c02c1..2239182 100644 --- a/pulumi/AWSMegatests/Pulumi.dev.yaml +++ b/pulumi/AWSMegatests/Pulumi.dev.yaml @@ -1,2 +1,3 @@ config: + aws-native:region: eu-west-1 aws:region: eu-west-1 diff --git a/pulumi/AWSMegatests/Pulumi.yaml b/pulumi/AWSMegatests/Pulumi.yaml index dfc9432..a3f4feb 100644 --- a/pulumi/AWSMegatests/Pulumi.yaml +++ b/pulumi/AWSMegatests/Pulumi.yaml @@ -7,4 +7,4 @@ description: Pulling in the infrastructure that was pre-infra-as-code config: pulumi:tags: value: - pulumi:template: "" + pulumi:template: aws-native-python diff --git a/pulumi/AWSMegatests/README.md b/pulumi/AWSMegatests/README.md new file mode 100644 index 0000000..9de900e --- /dev/null +++ b/pulumi/AWSMegatests/README.md @@ -0,0 +1,6 @@ +```sh +pulumi env run nf-core/AWSMegatests-dev -i pulumi import aws:s3/bucket:Bucket awsmegatests-bucket nf-core-awsmegatests +pulumi env run nf-core/AWSMegatests-dev -i pulumi import aws:s3/bucketAclV2:BucketAclV2 awsmegatests-bucket-acl nf-core-awsmegatests +pulumi env run nf-core/AWSMegatests-dev -i pulumi import aws-native:s3:Bucket awsmegatests-bucket-native nf-core-awsmegatests + +``` diff --git a/pulumi/AWSMegatests/__main__.py b/pulumi/AWSMegatests/__main__.py index 76be25a..8eb35fd 100644 --- a/pulumi/AWSMegatests/__main__.py +++ b/pulumi/AWSMegatests/__main__.py @@ -1,10 +1,217 @@ """An AWS Python Pulumi program""" import pulumi -from pulumi_aws import s3 +import pulumi_aws as aws -# Create an AWS resource (S3 Bucket) -bucket = s3.Bucket("my-bucket") +awsmegatests_bucket = aws.s3.Bucket( + "awsmegatests-bucket", + arn="arn:aws:s3:::nf-core-awsmegatests", + bucket="nf-core-awsmegatests", + cors_rules=[ + aws.s3.BucketCorsRuleArgs( + allowed_headers=["*"], + allowed_methods=[ + "HEAD", + "GET", + ], + allowed_origins=[ + "https://s3.amazonaws.com", + "https://s3-eu-west-1.amazonaws.com", + "https://s3.eu-west-1.amazonaws.com", + "*", + ], + expose_headers=[ + "ETag", + "x-amz-meta-custom-header", + ], + max_age_seconds=0, + ) + ], + hosted_zone_id="Z1BKCTXD74EZPE", + lifecycle_rules=[ + aws.s3.BucketLifecycleRuleArgs( + abort_incomplete_multipart_upload_days=0, + enabled=True, + expiration=aws.s3.BucketLifecycleRuleExpirationArgs( + date="", + days=10, + expired_object_delete_marker=False, + ), + id="Delete_files_older_than_10_days_in_work_directory", + noncurrent_version_transitions=[], + prefix="work", + tags={}, + transitions=[], + ), + aws.s3.BucketLifecycleRuleArgs( + abort_incomplete_multipart_upload_days=0, + enabled=True, + expiration=aws.s3.BucketLifecycleRuleExpirationArgs( + date="", + days=10, + expired_object_delete_marker=False, + ), + id="Delete files older than 10 days in _nextflow folder", + noncurrent_version_transitions=[], + prefix="_nextflow", + tags={}, + transitions=[], + ), + aws.s3.BucketLifecycleRuleArgs( + abort_incomplete_multipart_upload_days=0, + enabled=True, + expiration=aws.s3.BucketLifecycleRuleExpirationArgs( + date="", + days=10, + expired_object_delete_marker=False, + ), + id="Delete files older than 10 days in scratch directory", + noncurrent_version_expiration=aws.s3.BucketLifecycleRuleNoncurrentVersionExpirationArgs( + days=1, + ), + noncurrent_version_transitions=[], + prefix="scratch", + tags={}, + transitions=[], + ), + aws.s3.BucketLifecycleRuleArgs( + abort_incomplete_multipart_upload_days=0, + enabled=True, + id="Move_to_intelligent_tier_after_11_days", + noncurrent_version_transitions=[], + prefix="", + tags={}, + transitions=[ + aws.s3.BucketLifecycleRuleTransitionArgs( + date="", + days=11, + storage_class="INTELLIGENT_TIERING", + ) + ], + ), + ], + request_payer="BucketOwner", + server_side_encryption_configuration=aws.s3.BucketServerSideEncryptionConfigurationArgs( + rule=aws.s3.BucketServerSideEncryptionConfigurationRuleArgs( + apply_server_side_encryption_by_default=aws.s3.BucketServerSideEncryptionConfigurationRuleApplyServerSideEncryptionByDefaultArgs( + sse_algorithm="AES256", + ), + ), + ), + opts=pulumi.ResourceOptions(protect=True), +) -# Export the name of the bucket -pulumi.export("bucket_name", bucket.id) +# awsmegatests_bucket_acl = aws.s3.BucketAclV2( +# "awsmegatests-bucket-acl", +# access_control_policy=aws.s3.BucketAclV2AccessControlPolicyArgs( +# grants=[ +# aws.s3.BucketAclV2AccessControlPolicyGrantArgs( +# grantee=aws.s3.BucketAclV2AccessControlPolicyGrantGranteeArgs( +# display_name="phil.ewels", +# email_address="", +# id="f1ab567ea0ccaf20e2165c9a69391bc9e0ad0517fb77cb733a54b37401b9aa74", +# type="CanonicalUser", +# uri="", +# ), +# permission="FULL_CONTROL", +# ) +# ], +# owner=aws.s3.BucketAclV2AccessControlPolicyOwnerArgs( +# display_name="phil.ewels", +# id="f1ab567ea0ccaf20e2165c9a69391bc9e0ad0517fb77cb733a54b37401b9aa74", +# ), +# ), +# bucket="nf-core-awsmegatests", +# opts=pulumi.ResourceOptions(protect=True), +# ) + +# import pulumi_aws_native as aws_native + +# awsmegatests_bucket_native = aws_native.s3.Bucket( +# "awsmegatests-bucket-native", +# analytics_configurations=[ +# aws_native.s3.BucketAnalyticsConfigurationArgs( +# id="storage_class-0.1", +# storage_class_analysis=aws_native.s3.BucketStorageClassAnalysisArgs(), +# ) +# ], +# bucket_encryption=aws_native.s3.BucketEncryptionArgs( +# server_side_encryption_configuration=[ +# aws_native.s3.BucketServerSideEncryptionRuleArgs( +# bucket_key_enabled=False, +# server_side_encryption_by_default=aws_native.s3.BucketServerSideEncryptionByDefaultArgs( +# sse_algorithm=aws_native.s3.BucketServerSideEncryptionByDefaultSseAlgorithm.AES256, +# ), +# ) +# ], +# ), +# bucket_name="nf-core-awsmegatests", +# cors_configuration=aws_native.s3.BucketCorsConfigurationArgs( +# cors_rules=[ +# aws_native.s3.BucketCorsRuleArgs( +# allowed_headers=["*"], +# allowed_methods=[ +# aws_native.s3.BucketCorsRuleAllowedMethodsItem.HEAD, +# aws_native.s3.BucketCorsRuleAllowedMethodsItem.GET, +# ], +# allowed_origins=[ +# "https://s3.amazonaws.com", +# "https://s3-eu-west-1.amazonaws.com", +# "https://s3.eu-west-1.amazonaws.com", +# "*", +# ], +# exposed_headers=[ +# "ETag", +# "x-amz-meta-custom-header", +# ], +# ) +# ], +# ), +# lifecycle_configuration=aws_native.s3.BucketLifecycleConfigurationArgs( +# rules=[ +# aws_native.s3.BucketRuleArgs( +# expiration_in_days=10, +# id="Delete_files_older_than_10_days_in_work_directory", +# prefix="work", +# status=aws_native.s3.BucketRuleStatus.ENABLED, +# ), +# aws_native.s3.BucketRuleArgs( +# expiration_in_days=10, +# id="Delete files older than 10 days in _nextflow folder", +# prefix="_nextflow", +# status=aws_native.s3.BucketRuleStatus.ENABLED, +# ), +# aws_native.s3.BucketRuleArgs( +# expiration_in_days=10, +# id="Delete files older than 10 days in scratch directory", +# noncurrent_version_expiration=aws_native.s3.BucketNoncurrentVersionExpirationArgs( +# noncurrent_days=1, +# ), +# prefix="scratch", +# status=aws_native.s3.BucketRuleStatus.ENABLED, +# ), +# aws_native.s3.BucketRuleArgs( +# id="Move_to_intelligent_tier_after_11_days", +# status=aws_native.s3.BucketRuleStatus.ENABLED, +# transitions=[ +# aws_native.s3.BucketTransitionArgs( +# storage_class=aws_native.s3.BucketTransitionStorageClass.INTELLIGENT_TIERING, +# transition_in_days=11, +# ) +# ], +# ), +# ], +# ), +# metrics_configurations=[ +# aws_native.s3.BucketMetricsConfigurationArgs( +# id="all-objects", +# ) +# ], +# public_access_block_configuration=aws_native.s3.BucketPublicAccessBlockConfigurationArgs( +# block_public_acls=False, +# block_public_policy=False, +# ignore_public_acls=False, +# restrict_public_buckets=False, +# ), +# opts=pulumi.ResourceOptions(protect=True), +# ) diff --git a/pulumi/AWSMegatests/docs/README.md b/pulumi/AWSMegatests/docs/README.md new file mode 100644 index 0000000..49316e0 --- /dev/null +++ b/pulumi/AWSMegatests/docs/README.md @@ -0,0 +1,5 @@ +# S3 Buckets + +- `s3://nf-core-awsmegatests` Results from nf-core AWS megatests that get pulled in on the website. Also holds the work directory for AWS Megatests +- `s3://nf-core-test-datasets` Hosts a mirror of the `nf-core/test-datasets` repo. NOTE: Currently only the modules branch is there and not actively synced. +- `s3://s3://ngi-igenomes/test-data/` Hosts the full test datasets due to egress costs. Contact @ewels, @maxulysse, or @FriederikeHanssen if you need data uploaded. diff --git a/pulumi/AWSMegatests/requirements.txt b/pulumi/AWSMegatests/requirements.txt index 72aee79..e70610b 100644 --- a/pulumi/AWSMegatests/requirements.txt +++ b/pulumi/AWSMegatests/requirements.txt @@ -1,2 +1,2 @@ pulumi>=3.0.0,<4.0.0 -pulumi-aws>=6.0.2,<7.0.0 +pulumi-aws-native>=0.8.0,<1.0.0