diff --git a/.github/workflows/actions.yaml b/.github/workflows/actions.yaml new file mode 100644 index 0000000..888b79d --- /dev/null +++ b/.github/workflows/actions.yaml @@ -0,0 +1,85 @@ +name: static-container-registry + +on: push + +env: + DOCKER_IMAGE: registry.scality.com/static-container-registry-dev/static-container-registry + +jobs: + pylint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Set up Python 3.10 + uses: actions/setup-python@v2 + with: + python-version: "3.10" + #TODO: add annotations + #- uses: cclauss/GitHub-Action-for-pylint@0.7.0 + - name: Install dependencies + run: | + pip install pylint + - name: Analysing the code with pylint + run: | + pylint `ls -R|grep .py$|xargs` + + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + + - name: Set up Docker Buildk + uses: docker/setup-buildx-action@v1 + + - name: Login to Registry + uses: docker/login-action@v1 + with: + registry: registry.scality.com + username: ${{ secrets.REGISTRY_LOGIN }} + password: ${{ secrets.REGISTRY_PASSWORD }} + + - name: Build and push + uses: docker/build-push-action@v2 + with: + context: . + push: true + tags: "${{ env.DOCKER_IMAGE }}:${{ github.sha }}" + cache-from: type=gha + cache-to: type=gha,mode=max + + test: + needs: build + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + + - name: Install dependencies + run: | + OS=xUbuntu_20.04 + CRIO_VERSION=1.21 + KUBIC_REPO=https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable + sudo tee /etc/apt/sources.list.d/kubic-libcontainers.list <<< "deb $KUBIC_REPO/$OS/ /" + sudo tee -a /etc/apt/sources.list.d/kubic-libcontainers.list <<< "deb $KUBIC_REPO:/cri-o:/$CRIO_VERSION/$OS/ /" + curl -L $KUBIC_REPO/$OS/Release.key | sudo apt-key add - + curl -L $KUBIC_REPO:/cri-o:/$CRIO_VERSION/$OS/Release.key | sudo apt-key add - + sudo apt-get update + sudo apt-get -y install cri-o cri-o-runc cri-tools hardlink + + # Fix libpcre2-posix2 package version, use the + sudo apt-get -y install --reinstall --allow-downgrades libpcre2-posix2/$(lsb_release -cs) + + # Also need oras, skopeo ; but they are already installed + + bash <(curl -s https://raw.githubusercontent.com/pgrange/bash_unit/master/install.sh) + + # Start cri-o service + sudo systemctl start crio + # TODO: cache the installed image? + + - name: Unit testing with bash_unit + run: | + export IMAGE="${{ env.DOCKER_IMAGE }}:${{ github.sha }}" + sudo FORCE_COLOR=true ./bash_unit test.sh + # TODO: parse individual tests results (in TAP format) + # - uses: dorny/test-reporter@v1 + # if: success() || failure() diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml new file mode 100644 index 0000000..b2c537e --- /dev/null +++ b/.github/workflows/release.yaml @@ -0,0 +1,43 @@ +name: release + +on: + push: + tags: + - '*.*.*' + +env: + DOCKER_IMAGE: registry.scality.com/static-container-registry/static-container-registry + +jobs: + release: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Set up Docker Buildk + uses: docker/setup-buildx-action@v1 + + - name: Login to Registry + uses: docker/login-action@v1 + with: + registry: registry.scality.com + username: ${{ secrets.REGISTRY_LOGIN }} + password: ${{ secrets.REGISTRY_PASSWORD }} + + - name: Build and push + uses: docker/build-push-action@v2 + with: + context: . + push: true + tags: "${{ env.DOCKER_IMAGE }}:${{ github.ref_name }}" + cache-from: type=gha + cache-to: type=gha,mode=max + + - name: Create Release + uses: softprops/action-gh-release@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + tag_name: ${{ github.ref_name }} + release_name: Release ${{ github.ref_name }} diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index cfbe2af..0000000 --- a/.travis.yml +++ /dev/null @@ -1,22 +0,0 @@ -language: c -dist: xenial - -services: - - docker - -addons: - apt: - sources: - - sourceline: 'ppa:projectatomic/ppa' - - sourceline: 'deb https://pgrange.github.io/bash-unit_deb/debian/ unstable/' - key_url: 'https://pgrange.github.io/bash-unit_deb/keys.asc' - packages: - - bash-unit - - cri-o-1.12 - - hardlink - -before_install: - - ./ci/before-install.sh - -script: - - bash_unit ./test.sh diff --git a/Dockerfile b/Dockerfile index 287f6e5..3a1b0e9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,7 +9,7 @@ RUN apk add --no-cache \ VOLUME /var/lib/images COPY entrypoint.sh /entrypoint.sh -COPY static-container-registry.py /static-container-registry.py +COPY static_container_registry.py /static_container_registry.py ENTRYPOINT ["/entrypoint.sh"] CMD ["nginx", "-g", "daemon off;"] diff --git a/README.md b/README.md index 7a8130f..ac09a29 100644 --- a/README.md +++ b/README.md @@ -55,7 +55,7 @@ Now we're ready to create an Nginx configuration file that can be `include`d in a larger configuration: ``` -$ ./static-container-registry.py ./images > registry.conf +$ ./static_container_registry.py ./images > registry.conf ``` The following options are available: diff --git a/ci/before-install.sh b/ci/before-install.sh deleted file mode 100755 index 7466375..0000000 --- a/ci/before-install.sh +++ /dev/null @@ -1,66 +0,0 @@ -#!/bin/bash - -SKOPEO_VERSION=0.1.38-1~ubuntu16.04~ppa1 -CONTAINERD_VERSION=1.2.6 -CRICTL_VERSION=1.14.0 - -set -xue -o pipefail - -SKOPEO_DEB=skopeo_${SKOPEO_VERSION}_amd64.deb -CONTAINERD_ARCHIVE=containerd-${CONTAINERD_VERSION}.linux-amd64.tar.gz -CRICTL_ARCHIVE=crictl-v${CRICTL_VERSION}-linux-amd64.tar.gz - -curl -LO https://launchpad.net/~projectatomic/+archive/ubuntu/ppa/+files/${SKOPEO_DEB} -dpkg -x ${SKOPEO_DEB} skopeo/ -sudo cp skopeo/usr/bin/skopeo /usr/local/bin/skopeo -rm -r skopeo ${SKOPEO_DEB} - -curl -LO https://github.com/containerd/containerd/releases/download/v${CONTAINERD_VERSION}/${CONTAINERD_ARCHIVE} -tar xvf ${CONTAINERD_ARCHIVE} -sudo mv bin/* /usr/bin/ -rm ${CONTAINERD_ARCHIVE} - -cat << EOF | sudo tee /etc/systemd/system/containerd.service -[Unit] -Description=containerd container runtime -Documentation=https://containerd.io -After=network.target - -[Service] -ExecStartPre=-/sbin/modprobe overlay -ExecStart=/usr/bin/containerd - -Delegate=yes -KillMode=process -# Having non-zero Limit*s causes performance problems due to accounting overhead -# in the kernel. We recommend using cgroups to do container-local accounting. -LimitNPROC=infinity -LimitCORE=infinity -LimitNOFILE=1048576 -# Comment TasksMax if your systemd version does not supports it. -# Only systemd 226 and above support this version. -TasksMax=infinity - -[Install] -WantedBy=multi-user.target -EOF - -sudo systemctl daemon-reload - -sudo mkdir -p /etc/containerd -cat << EOF | sudo tee /etc/containerd/config.toml -[plugins] - [plugins.cri] - [plugins.cri.registry] - [plugins.cri.registry.mirrors] - [plugins.cri.registry.mirrors."127.0.0.1:5000"] - endpoint = ["http://127.0.0.1:5000"] -EOF - -sudo systemctl start containerd - -sudo systemctl start crio - -curl -LO https://github.com/kubernetes-sigs/cri-tools/releases/download/v${CRICTL_VERSION}/${CRICTL_ARCHIVE} -tar xvf ${CRICTL_ARCHIVE} crictl -sudo mv crictl /usr/local/bin/crictl diff --git a/ci/docker/image-provisioner/Dockerfile b/ci/docker/image-provisioner/Dockerfile index 381a812..14831aa 100644 --- a/ci/docker/image-provisioner/Dockerfile +++ b/ci/docker/image-provisioner/Dockerfile @@ -1,16 +1,23 @@ -FROM docker.io/ubuntu:xenial +ARG OS_VERSION=20.04 +FROM docker.io/ubuntu:$OS_VERSION -RUN apt-get update \ +ARG OS_VERSION +ARG REPOSITORY=https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable +RUN export DEBIAN_FRONTEND=noninteractive \ && \ - apt-get install -y software-properties-common\ + apt-get update \ + && \ + apt-get install -y curl software-properties-common \ + && \ + echo "deb ${REPOSITORY}/xUbuntu_${OS_VERSION}/ /" > /etc/apt/sources.list.d/devel:kubic:libcontainers:stable.list \ && \ - add-apt-repository ppa:projectatomic/ppa \ + curl -L ${REPOSITORY}/xUbuntu_${OS_VERSION}/Release.key | apt-key add - \ && \ apt-get update \ && \ apt-get install -y skopeo -RUN apt-get install \ +RUN apt-get install -y \ hardlink COPY provision-images.sh /provision-images.sh diff --git a/ci/docker/sut/Dockerfile b/ci/docker/sut/Dockerfile index c4cffc8..7e9563a 100644 --- a/ci/docker/sut/Dockerfile +++ b/ci/docker/sut/Dockerfile @@ -1,28 +1,29 @@ -FROM docker.io/ubuntu:xenial +ARG OS_VERSION=20.04 +FROM docker.io/ubuntu:$OS_VERSION -RUN apt-get update \ +ARG OS_VERSION +ARG REPOSITORY=https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable +RUN export DEBIAN_FRONTEND=noninteractive \ && \ - apt-get install -y software-properties-common\ + apt-get update \ + && \ + apt-get install -y curl software-properties-common \ + && \ + echo "deb ${REPOSITORY}/xUbuntu_${OS_VERSION}/ /" > /etc/apt/sources.list.d/devel:kubic:libcontainers:stable.list \ && \ - add-apt-repository ppa:projectatomic/ppa \ + curl -L ${REPOSITORY}/xUbuntu_${OS_VERSION}/Release.key | apt-key add - \ && \ apt-get update \ && \ apt-get install -y skopeo -RUN apt-get update \ - && \ - apt-get install -y \ - apt-transport-https \ - curl \ - && \ - curl https://pgrange.github.io/bash-unit_deb/keys.asc | apt-key add - \ - && \ - echo deb https://pgrange.github.io/bash-unit_deb/debian/ unstable/ | \ - tee -a /etc/apt/sources.list.d/bash-unit.list \ - && \ - apt-get update \ - && \ - apt-get install bash-unit +ARG ORAS_VERSION=0.12.0 +RUN curl -#LO https://github.com/oras-project/oras/releases/download/v${ORAS_VERSION}/oras_${ORAS_VERSION}_linux_amd64.tar.gz \ + && \ + tar -zxf oras_${ORAS_VERSION}_*.tar.gz -C /usr/local/bin/ oras \ + && \ + rm oras_${ORAS_VERSION}_*.tar.gz + +RUN [ "bash", "-c", "cd /usr/local/bin && bash <(curl -s https://raw.githubusercontent.com/pgrange/bash_unit/master/install.sh)" ] COPY test.sh / diff --git a/entrypoint.sh b/entrypoint.sh index bb84b52..f0b7ccd 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -2,6 +2,6 @@ set -ue -python3 /static-container-registry.py /var/lib/images > /var/run/static-container-registry.conf +python3 /static_container_registry.py /var/lib/images > /var/run/static-container-registry.conf exec "$@" diff --git a/static-container-registry.py b/static_container_registry.py similarity index 83% rename from static-container-registry.py rename to static_container_registry.py index 254a0de..ef4907c 100755 --- a/static-container-registry.py +++ b/static_container_registry.py @@ -1,5 +1,8 @@ #!/usr/bin/env python +# pylint: disable=consider-using-f-string +""" Script to generate nginx configuration to serve as container registry. """ + import sys import os.path import json @@ -40,6 +43,7 @@ def find_images(root): + """ Find list of images in the specified root folder. """ LOGGER.info('Finding images in %s', root) for name in os.listdir(root): @@ -64,10 +68,10 @@ def find_images(root): LOGGER.info('No manifest file at %s', manifest) continue - with open(manifest, 'r') as fd: + with open(manifest, 'r', encoding='utf-8') as file: LOGGER.info('Attempting to load JSON data from %s', manifest) try: - data = json.load(fd) + data = json.load(file) except json.JSONDecodeError: LOGGER.info('Failed to decode JSON from %s', manifest) data = None @@ -79,19 +83,29 @@ def find_images(root): LOGGER.info('Invalid schemaVersion in %s', manifest) continue - mediaType = data.get('mediaType') - if not mediaType in [ + media_type = data.get('mediaType') + if not media_type in [ 'application/vnd.docker.distribution.manifest.v2+json', 'application/vnd.oci.image.manifest.v1+json']: - LOGGER.info('Invalid mediaType in %s : %s', manifest, mediaType) + LOGGER.info('Invalid mediaType in %s : %s', manifest, media_type) continue LOGGER.info('Found image %s:%s in %s', name, tag, curr) - yield (name, tag, mediaType) + yield (name, tag, media_type) + + +def compute_digest(filename): + """ Compute file digest. """ + digest = hashlib.sha256() + with open(filename, 'rb') as file: + for chunk in iter(lambda: file.read(4096), b''): + digest.update(chunk) + return digest.hexdigest() def create_config(root, server_root, name_prefix, with_constants=True, only_constants=False): + """ Create nginx configuration snippets for the images in folder. """ if with_constants: yield CONSTANTS @@ -99,8 +113,8 @@ def create_config(root, server_root, name_prefix, with_constants=True, return images = {} - for (name, tag, mediaType) in find_images(root): - images.setdefault(name, {})[tag] = mediaType + for (name, tag, media_type) in find_images(root): + images.setdefault(name, {})[tag] = media_type for (name, tags) in sorted(images.items()): tag_list = { @@ -121,21 +135,13 @@ def create_config(root, server_root, name_prefix, with_constants=True, seen_digests = set() - for (tag, mediaType) in sorted(tags.items()): - manifest_file = os.path.join(root, name, tag, MANIFEST_JSON) - - digest = hashlib.sha256() - - with open(manifest_file, 'rb') as fd: - for chunk in iter(lambda: fd.read(4096), b''): - digest.update(chunk) - - hexdigest = digest.hexdigest() + for (tag, media_type) in sorted(tags.items()): + hexdigest = compute_digest(os.path.join(root, name, tag, MANIFEST_JSON)) yield ''' location = "/v2/{name_prefix:s}{name:s}/manifests/{tag:s}" {{ alias {server_root:s}/{name:s}/{tag:s}/; - types {{ }} default_type "{mediaType:s}"; + types {{ }} default_type "{media_type:s}"; add_header 'Docker-Content-Digest' 'sha256:{digest:s}'; try_files manifest.json =404; error_page 404 @404_tag; @@ -143,7 +149,7 @@ def create_config(root, server_root, name_prefix, with_constants=True, '''.format( name=name, tag=tag, - mediaType=mediaType, + media_type=media_type, name_prefix=name_prefix.lstrip('/'), digest=hexdigest, server_root=server_root, @@ -153,7 +159,7 @@ def create_config(root, server_root, name_prefix, with_constants=True, yield ''' location = "/v2/{name_prefix:s}{name:s}/manifests/sha256:{digest:s}" {{ alias {server_root:s}/{name:s}/{tag:s}/; - types {{ }} default_type "{mediaType:s}"; + types {{ }} default_type "{media_type:s}"; add_header 'Docker-Content-Digest' 'sha256:{digest:s}'; try_files manifest.json =404; error_page 404 @404_tag; @@ -161,7 +167,7 @@ def create_config(root, server_root, name_prefix, with_constants=True, '''.format( name=name, tag=tag, - mediaType=mediaType, + media_type=media_type, name_prefix=name_prefix.lstrip('/'), digest=hexdigest, server_root=server_root, @@ -190,6 +196,7 @@ def create_config(root, server_root, name_prefix, with_constants=True, def main(): + """ Main entrypoint. """ logging.basicConfig( level=logging.INFO, )