Upgrade GitLab to Amazon Linux 2023 (#6160, 7218)

OPERATOR.rst

@@ -290,11 +290,11 @@ Updating the AMI for GitLab instances
 
 As part of the upgrades isssue, operators must check for updates to the AMI for
 the root volume of the EC2 instance running GitLab. We use a hardened — to the
-requirements of the CIS Amazon Linux 2 benchmark — variant of Amazon's Linux 2
-AMI. The license to use the AMI for an EC2 instance is sold by CIS as a
-subscription on the AWS Marketplace:
+requirements of the CIS Hardened Image Level 1 on Amazon Linux 2023. The license
+to use the AMI for an EC2 instance is sold by CIS as a subscription on the AWS
+Marketplace:
 
-https://aws.amazon.com/marketplace/pp/prodview-wv574yqgjv6jg
+https://aws.amazon.com/marketplace/pp/prodview-fqqp6ebucarnm
 
 The license costs $0.024 per instance/hour. Every AWS account must subscribe
 separately.
@@ -313,7 +313,7 @@ component, say, ``_select dev.gitlab`` and run
 
     aws ec2 describe-images \
             --owners aws-marketplace \
-            --filters="Name=name,Values=*abcfcbaf-134e-4639-a7b4-fd285b9fcf0a*" \
+            --filters="Name=name,Values=*prod-fvm47vekg24oc*" \
         | jq -r '.Images[] | .CreationDate+"\t"+.ImageId+"\t"+.Name' \
         | sort \
         | tail -1

deployments/tempdev/environment.py

@@ -12,12 +12,23 @@
 type SourceSpec = str
 
 
+def bqsrc(google_project: str,
+          snapshot: str,
+          flags: int = 0,
+          ) -> tuple[ProjectName, SourceSpec | None]:
+    assert len(google_project) == 8, google_project
+    project = 'datarepo-dev-' + google_project
+    assert not snapshot.startswith('ANVIL_'), snapshot
+    snapshot = 'ANVIL_' + snapshot
+    return mksrc('bigquery', project, snapshot, flags)
+
+
 def mksrc(source_type: Literal['bigquery', 'parquet'],
           google_project,
           snapshot,
           flags: int = 0,
           ) -> tuple[ProjectName, SourceSpec | None]:
-    _, env, project, _ = snapshot.split('_', 3)
+    project = '_'.join(snapshot.split('_')[1:-3])
     assert flags <= pop
     source = None if flags & pop else ':'.join([
         'tdr',
@@ -52,9 +63,9 @@ def mkdict(previous_catalog: dict[ProjectName, SourceSpec | None],
 
 
 anvil_sources = mkdict({}, 3, mkdelta([
-    mksrc('bigquery', 'datarepo-dev-e53e74aa', 'ANVIL_1000G_2019_Dev_20230609_ANV5_202306121732'),
-    mksrc('bigquery', 'datarepo-dev-42c70e6a', 'ANVIL_CCDG_Sample_1_20230228_ANV5_202302281520'),
-    mksrc('bigquery', 'datarepo-dev-97ad270b', 'ANVIL_CMG_Sample_1_20230225_ANV5_202302281509')
+    bqsrc('e53e74aa', '1000G_2019_Dev_20230609_ANV5_202306121732'),
+    bqsrc('42c70e6a', 'CCDG_Sample_1_20230228_ANV5_202302281520'),
+    bqsrc('97ad270b', 'CMG_Sample_1_20230225_ANV5_202302281509')
 ]))
 
 

scripts/resolve_container_path.py

@@ -1,8 +1,15 @@
+import logging
 import os
 import sys
 
 import docker
 
+from azul.logging import (
+    configure_script_logging,
+)
+
+log = logging.getLogger(__name__)
+
 """
 Convert the given container path to a path that's valid on the host.
 
@@ -21,22 +28,30 @@ def resolve_container_path(container_path):
     container_path = os.path.realpath(container_path)
     try:
         with open('/proc/self/cgroup') as f:
+            log.info('Found /proc/self/cgroup:<dedup>')
             # Entries in /proc/self/cgroup look like this (note the nesting):
             # 11:name=systemd:/docker/82c1bd2…23b5bcf/docker/6547bce…60ca5a7
             prefix, container_id = next(f).strip().split(':')[2].split('/')[-2:]
     except FileNotFoundError:
+        log.info('Did not find /proc/self/cgroup:<dedup>')
         pass
     else:
+        log.info('cgroup prefix is %s', prefix)
         if prefix == 'docker':
+            log.info('cgroup prefix is docker')
             api = docker.client.from_env().api
             for mount in api.inspect_container(container_id)['Mounts']:
                 if container_path.startswith(mount['Destination']):
                     tail = os.path.relpath(container_path, mount['Destination'])
-                    return os.path.normpath(os.path.join(mount['Source'], tail))
+                    host_path = os.path.normpath(os.path.join(mount['Source'], tail))
+                    log.info('Resolving %s to %s', container_path, host_path)
+                    return host_path
+    log.info('Cannot resolve container path: %s', container_path)
     return None
 
 
 if __name__ == '__main__':
+    configure_script_logging(log)
     container_path = sys.argv[1]
     host_path = resolve_container_path(container_path)
     print(container_path if host_path is None else host_path)

terraform/gitlab/gitlab.tf.json.template.py

@@ -245,10 +245,10 @@
 # For instructions on finding the latest CIS-hardened AMI, see "Updating the AMI
 # for GitLab instances" section in OPERATOR.rst.
 #
-# CIS Amazon Linux 2 Kernel 5.10 Benchmark - Level 1 - v10 -abcfcbaf-134e-4639-a7b4-fd285b9fcf0a
+# CIS Amazon Linux 2023 Benchmark - Level 1 - v10 -prod-fvm47vekg24oc
 #
 ami_id = {
-    'us-east-1': 'ami-0ce4ea729be281b05'
+    'us-east-1': 'ami-03c834fe7b8001105'
 }
 
 gitlab_mount = '/mnt/gitlab'
@@ -1768,35 +1768,6 @@ def merge(sets: Iterable[Iterable[str]]) -> Iterable[str]:
                                 'ExecStartPre=-/usr/bin/docker stop gitlab',
                                 'ExecStartPre=-/usr/bin/docker rm gitlab',
                                 'ExecStartPre=/usr/bin/docker pull ' + str(gitlab_image),
-                                # The hardened AMI contains some code that
-                                # creates a default nftables ruleset at boot
-                                # time, in order to satisfy some CIS control, I
-                                # believe. Also at boot time, Docker creates a
-                                # competing iptables ruleset using the
-                                # `iptables` command, which is symlinked to the
-                                # `iptables-legacy` alternative. The result was
-                                # that the GitLab web app provided by this
-                                # container was not reachable from outside the
-                                # host. I tried switching to the `iptables-nft`
-                                # alternative and that prevented the creation of
-                                # a competing iptables ruleset, with only the
-                                # nftables ruleset present, but the webapp
-                                # remained unreachable. I assume this is because
-                                # the nftables ruleset had both the rules from
-                                # the hardened AMI code and those created by
-                                # Docker, still contradicting each other. For
-                                # the time being, we will simply delete the
-                                # default nftables ruleset. This is acceptable
-                                # because even without any firewall rules, the
-                                # the EC2 instance has no public IP and is
-                                # protected by EC2 security groups, which serve
-                                # the same purpose as firewall rules in the
-                                # guest OS.
-                                'ExecStartPre=/sbin/nft flush ruleset',
-                                # FIXME: Re-enable FIPS mode in GitLab container
-                                #        https://github.com/DataBiosphere/azul/issues/7218
-                                f'ExecStartPre=/bin/mkdir -p {gitlab_mount}/etc',
-                                f'ExecStartPre=/bin/sh -c "echo 0 > {gitlab_mount}/etc/fips_enabled"',
                                 jw(
                                     'ExecStart=/usr/bin/docker',
                                     'run',
@@ -1809,9 +1780,6 @@ def merge(sets: Iterable[Iterable[str]]) -> Iterable[str]:
                                     f'--volume {gitlab_mount}/config:/etc/gitlab',
                                     f'--volume {gitlab_mount}/logs:/var/log/gitlab',
                                     f'--volume {gitlab_mount}/data:/var/opt/gitlab',
-                                    # FIXME: Re-enable FIPS mode in GitLab container
-                                    #        https://github.com/DataBiosphere/azul/issues/7218
-                                    f'--volume {gitlab_mount}/etc/fips_enabled:/proc/sys/crypto/fips_enabled',
                                     *vpc_dns_docker_flags,
                                     str(gitlab_image)
                                 ),