Add package multi-version support provided by reprepro 5.4 (#9)

To keep all versions, pass `package_version_limit=0`.
To keep five versions, pass `package_version_limit=5`.
To keep one version, don't pass `package_version_limit` at all.

After upgrading existing repos, run `ih-s3-reprepro migrate`. See details in ionos-cloud/reprepro#16

Co-authored-by: Oleksandr Kuzminskyi <[email protected]>

README.md

@@ -203,6 +203,7 @@ module "release_infrahouse_com" {
 | [aws_s3_bucket_ownership_controls.repo-logs](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket_ownership_controls) | resource |
 | [aws_s3_bucket_policy.bucket-access](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket_policy) | resource |
 | [aws_s3_bucket_public_access_block.repo](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket_public_access_block) | resource |
+| [aws_s3_bucket_versioning.repo](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket_versioning) | resource |
 | [aws_s3_object.deb-gpg-public-key](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_object) | resource |
 | [aws_s3_object.distributions](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_object) | resource |
 | [aws_s3_object.index-html](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_object) | resource |
@@ -215,6 +216,7 @@ module "release_infrahouse_com" {
 
 | Name | Description | Type | Default | Required |
 |------|-------------|------|---------|:--------:|
+| <a name="input_architectures"></a> [architectures](#input\_architectures) | List of architectures served by the repo | `list(string)` | <pre>[<br>  "amd64"<br>]</pre> | no |
 | <a name="input_bucket_admin_roles"></a> [bucket\_admin\_roles](#input\_bucket\_admin\_roles) | List of AWS IAM role ARN that has permissions to upload to the bucket | `list(string)` | `[]` | no |
 | <a name="input_bucket_force_destroy"></a> [bucket\_force\_destroy](#input\_bucket\_force\_destroy) | If true, the repository bucket will be destroyed even if it contains files. | `bool` | `false` | no |
 | <a name="input_bucket_name"></a> [bucket\_name](#input\_bucket\_name) | S3 bucket name for the repository. | `string` | n/a | yes |
@@ -225,6 +227,7 @@ module "release_infrahouse_com" {
 | <a name="input_http_auth_user"></a> [http\_auth\_user](#input\_http\_auth\_user) | Username for HTTP basic authentication. If not specified, the authentication isn't enabled. | `string` | `null` | no |
 | <a name="input_index_body"></a> [index\_body](#input\_index\_body) | Content of a body tag in index.html. | `string` | `"Stay tuned!"` | no |
 | <a name="input_index_title"></a> [index\_title](#input\_index\_title) | Content of a title tag in index.html. | `string` | `"Debian packages repository"` | no |
+| <a name="input_package_version_limit"></a> [package\_version\_limit](#input\_package\_version\_limit) | Number of versions of a package to keep in the repository. Zero means keep all versions. | `number` | `null` | no |
 | <a name="input_repository_codename"></a> [repository\_codename](#input\_repository\_codename) | Repository codename. Can be focal, jammy, etc. | `string` | n/a | yes |
 | <a name="input_signing_key_readers"></a> [signing\_key\_readers](#input\_signing\_key\_readers) | List of role ARNs that have permission to read GPG signing key and passphrase. | `list(string)` | `null` | no |
 | <a name="input_signing_key_writers"></a> [signing\_key\_writers](#input\_signing\_key\_writers) | List of role ARNs that have permission to write to GPG signing key and passphrase secrets. | `list(string)` | `null` | no |

bucket.tf

@@ -150,3 +150,10 @@ resource "aws_s3_bucket_logging" "server-logs" {
   target_bucket = aws_s3_bucket.repo-logs.bucket
   target_prefix = "server-side/"
 }
+
+resource "aws_s3_bucket_versioning" "repo" {
+  bucket = aws_s3_bucket.repo.id
+  versioning_configuration {
+    status = "Enabled"
+  }
+}

files/distributions

@@ -2,3 +2,4 @@ Codename: ${codename}
 Components: main
 Architectures: ${architectures}
 SignWith: ${signwith}
+%{ if package_version_limit != null }Limit: ${package_version_limit}%{ endif }

repo.tf

@@ -7,6 +7,7 @@ resource "aws_s3_object" "distributions" {
       codename : var.repository_codename
       signwith : var.gpg_sign_with
       architectures : join(" ", var.architectures)
+      package_version_limit : var.package_version_limit
     }
   )
   content_type = "text/plain"

requirements.txt

@@ -1,8 +1,9 @@
-black ~= 23.0
+black ~= 24.0
 boto3 ~= 1.26
 infrahouse-toolkit ~= 2.6
 myst-parser ~= 2.0
-pytest ~= 7.3
-requests~=2.31
+pytest ~= 8.3
+pytest-infrahouse ~= 0.5
+requests ~= 2.32
 Sphinx ~= 6.0
 sphinx-rtd-theme ~= 1.2

test_data/test_module/jumphost.tf

@@ -0,0 +1,22 @@
+data "aws_iam_role" "jumphost" {
+  name = var.jumphost_role_name
+}
+
+data "aws_iam_policy_document" "jumphost-extra" {
+  statement {
+    actions = [
+      "secretsmanager:GetSecretValue",
+      "secretsmanager:PutSecretValue"
+    ]
+    resources = ["*"]
+  }
+}
+
+resource "aws_iam_policy" "jumphost-extra" {
+  policy = data.aws_iam_policy_document.jumphost-extra.json
+}
+
+resource "aws_iam_role_policy_attachment" "jumphost-extra" {
+  policy_arn = aws_iam_policy.jumphost-extra.arn
+  role       = data.aws_iam_role.jumphost.name
+}

test_data/test_module/main.tf

@@ -6,14 +6,17 @@ module "test" {
     aws.ue1 = aws.aws-us-east-1
   }
 
-  source              = "../../"
-  bucket_name         = "infrahouse-${random_pet.bucket_suffix.id}"
-  domain_name         = "debian-repo-test.${data.aws_route53_zone.cicd.name}"
-  gpg_public_key      = file("${path.module}/files/DEB-GPG-KEY-infrahouse-${var.ubuntu_codename}")
-  gpg_sign_with       = "packager-${var.ubuntu_codename}@infrahouse.com"
-  repository_codename = var.ubuntu_codename
-  zone_id             = data.aws_route53_zone.cicd.zone_id
-  http_auth_user      = var.http_user
-  http_auth_password  = var.http_password
-  bucket_admin_roles  = [module.jumphost.jumphost_role_arn]
+  source                = "../../"
+  bucket_name           = "infrahouse-${random_pet.bucket_suffix.id}"
+  domain_name           = "debian-repo-test.${data.aws_route53_zone.cicd.name}"
+  gpg_public_key        = file("${path.module}/files/DEB-GPG-KEY-infrahouse-${var.ubuntu_codename}")
+  gpg_sign_with         = "packager-${var.ubuntu_codename}@infrahouse.com"
+  repository_codename   = var.ubuntu_codename
+  bucket_force_destroy  = true
+  zone_id               = data.aws_route53_zone.cicd.zone_id
+  http_auth_user        = var.http_user
+  http_auth_password    = var.http_password
+  bucket_admin_roles    = [var.jumphost_role_arn]
+  signing_key_writers   = [var.jumphost_role_arn]
+  package_version_limit = 0
 }

test_data/test_module/outputs.tf

@@ -31,11 +31,3 @@ output "packager_key_passphrase_secret_id" {
 output "repository_url" {
   value = module.test.repo_url
 }
-
-output "jumphost" {
-  value = module.jumphost.jumphost_hostname
-}
-
-output "jumphost_role" {
-  value = module.jumphost.jumphost_role_arn
-}

test_data/test_module/providers.tf

@@ -1,7 +1,10 @@
 provider "aws" {
   region = "us-west-1"
-  assume_role {
-    role_arn = var.role_arn
+  dynamic "assume_role" {
+    for_each = var.role_arn != null ? [1] : []
+    content {
+      role_arn = var.role_arn
+    }
   }
   default_tags {
     tags = {

test_data/test_module/variables.tf

@@ -1,4 +1,6 @@
-variable "role_arn" {}
+variable "role_arn" {
+  default = null
+}
 variable "test_zone" {}
 variable "ubuntu_codename" {}
 variable "http_user" {
@@ -7,3 +9,5 @@ variable "http_user" {
 variable "http_password" {
   default = null
 }
+variable "jumphost_role_arn" {}
+variable "jumphost_role_name" {}

tests/conftest.py

@@ -6,55 +6,13 @@
 
 # "303467602807" is our test account
 TEST_ACCOUNT = "303467602807"
-TEST_ROLE_ARN = "arn:aws:iam::303467602807:role/debian-repo-tester"
+# TEST_ROLE_ARN = "arn:aws:iam::303467602807:role/debian-repo-tester"
 DEFAULT_PROGRESS_INTERVAL = 10
 TRACE_TERRAFORM = False
-DESTROY_AFTER = True
 UBUNTU_CODENAME = "jammy"
 
 LOG = logging.getLogger(__name__)
 REGION = "us-east-2"
 TEST_ZONE = "ci-cd.infrahouse.com"
 
 setup_logging(LOG, debug=True)
-
-
-@pytest.fixture(scope="session")
-def aws_iam_role():
-    sts = boto3.client("sts")
-    return sts.assume_role(
-        RoleArn=TEST_ROLE_ARN, RoleSessionName=TEST_ROLE_ARN.split("/")[1]
-    )
-
-
-@pytest.fixture(scope="session")
-def boto3_session(aws_iam_role):
-    return boto3.Session(
-        aws_access_key_id=aws_iam_role["Credentials"]["AccessKeyId"],
-        aws_secret_access_key=aws_iam_role["Credentials"]["SecretAccessKey"],
-        aws_session_token=aws_iam_role["Credentials"]["SessionToken"],
-    )
-
-
-@pytest.fixture(scope="session")
-def ec2_client(boto3_session):
-    assert boto3_session.client("sts").get_caller_identity()["Account"] == TEST_ACCOUNT
-    return boto3_session.client("ec2", region_name=REGION)
-
-
-@pytest.fixture(scope="session")
-def ec2_client_map(ec2_client, boto3_session):
-    regions = [reg["RegionName"] for reg in ec2_client.describe_regions()["Regions"]]
-    ec2_map = {reg: boto3_session.client("ec2", region_name=reg) for reg in regions}
-
-    return ec2_map
-
-
-@pytest.fixture()
-def route53_client(boto3_session):
-    return boto3_session.client("route53", region_name=REGION)
-
-
-@pytest.fixture()
-def elbv2_client(boto3_session):
-    return boto3_session.client("elbv2", region_name=REGION)

tests/test_module.py

@@ -8,9 +8,6 @@
 from tests.conftest import (
     LOG,
     TRACE_TERRAFORM,
-    DESTROY_AFTER,
-    TEST_ROLE_ARN,
-    TEST_ZONE,
     UBUNTU_CODENAME,
 )
 
@@ -30,19 +27,32 @@
         ),
     ],
 )
-def test_module(http_user, http_password):
+def test_module(
+    jumphost, http_user, http_password, keep_after, test_zone_name, test_role_arn
+):
     terraform_dir = "test_data/test_module"
+    jumphost_role_arn = jumphost["jumphost_role_arn"]["value"]
+    jumphost_role_name = jumphost["jumphost_role_name"]["value"]
 
     with open(osp.join(terraform_dir, "terraform.tfvars"), "w") as fp:
         fp.write(
             dedent(
                 f"""
-                role_arn        = "{TEST_ROLE_ARN}"
-                test_zone       = "{TEST_ZONE}"
-                ubuntu_codename = "{UBUNTU_CODENAME}"
+                test_zone          = "{test_zone_name}"
+                ubuntu_codename    = "{UBUNTU_CODENAME}"
+                jumphost_role_arn  = "{jumphost_role_arn}"
+                jumphost_role_name = "{jumphost_role_name}"
                 """
             )
         )
+        if test_role_arn:
+            fp.write(
+                dedent(
+                    f"""
+                    role_arn        = "{test_role_arn}"
+                    """
+                )
+            )
         if http_user:
             fp.write(
                 dedent(
@@ -55,23 +65,23 @@ def test_module(http_user, http_password):
 
     with terraform_apply(
         terraform_dir,
-        destroy_after=DESTROY_AFTER,
+        destroy_after=not keep_after,
         json_output=True,
         enable_trace=TRACE_TERRAFORM,
     ) as tf_output:
         assert tf_output["release_bucket"]["value"]
         if http_user:
-            response = requests.get(f"https://debian-repo-test.{TEST_ZONE}")
+            response = requests.get(f"https://debian-repo-test.{test_zone_name}")
             assert response.status_code == 401
             response = requests.get(
-                f"https://debian-repo-test.{TEST_ZONE}",
+                f"https://debian-repo-test.{test_zone_name}",
                 auth=HTTPBasicAuth(http_user, http_password),
             )
             assert response.status_code == 200
             LOG.info("Response from HTTP server:\n%s", response.text)
             assert "Stay tuned!" in response.text
         else:
-            response = requests.get(f"https://debian-repo-test.{TEST_ZONE}")
+            response = requests.get(f"https://debian-repo-test.{test_zone_name}")
             assert response.status_code == 200
             LOG.info("Response from HTTP server:\n%s", response.text)
             assert "Stay tuned!" in response.text

variables.tf

@@ -77,6 +77,12 @@ variable "signing_key_writers" {
   default     = null
 }
 
+variable "package_version_limit" {
+  description = "Number of versions of a package to keep in the repository. Zero means keep all versions."
+  type        = number
+  default     = null
+}
+
 variable "zone_id" {
   description = "Route53 zone id where the parent domain of var.domain_name is hosted. If var.domain_name is repo.foo.com, then the value should be zone_id of foo.com."
   type        = string