diff --git a/.github/actions/clean_up_package_registry/action.yml b/.github/actions/clean_up_package_registry/action.yml new file mode 100644 index 0000000000..7b5eb5c563 --- /dev/null +++ b/.github/actions/clean_up_package_registry/action.yml @@ -0,0 +1,52 @@ +# Copyright Helio Chissini de Castro, 2023 +# +# This program and the accompanying materials are made +# available under the terms of the Eclipse Public License 2.0 +# which is available at https://www.eclipse.org/legal/epl-2.0/ +# +# SPDX-License-Identifier: EPL-2.0 + +name: 'Delete old non-release packages from Github package registry' +description: 'Delete older packages set by a minimal level input' +author: 'The ORT Project Authors' + +inputs: + registry: + description: 'Github container registry' + default: 'ghcr.io' + token: + description: 'Github token' + required: true + keep: + description: 'Number of non-release packages to keep' + required: false + default: '3' + packages: + description: 'Name of the packages to be cleaned up' + required: true + dry-run: + description: 'Execute a dry run operation to check the execution is correct' + default: 'false' + +runs: + using: 'composite' + + steps: + - name: Install Python + uses: actions/setup-python@v4 + with: + python-version: '3.10' + cache: 'pip' + + - name: Execute the operation + id: check_image + shell: bash + env: + INPUT_REGISTRY: ${{ inputs.registry }} + INPUT_TOKEN: ${{ inputs.token }} + INPUT_KEEP: ${{ inputs.keep }} + INPUT_PACKAGES: ${{ inputs.packages }} + INPUT_DRY_RUN: ${{ inputs.dry-run}} + run: | + pip install -q -U pip requests rich + python ./.github/actions/clean_up_package_registry/clean_up_package_registry.py diff --git a/.github/actions/clean_up_package_registry/clean_up_package_registry.py b/.github/actions/clean_up_package_registry/clean_up_package_registry.py new file mode 100644 index 0000000000..3438f5374e --- /dev/null +++ b/.github/actions/clean_up_package_registry/clean_up_package_registry.py @@ -0,0 +1,86 @@ +# Copyright Helio Chissini de Castro, 2023 +# +# This program and the accompanying materials are made +# available under the terms of the Eclipse Public License 2.0 +# which is available at https://www.eclipse.org/legal/epl-2.0/ +# +# SPDX-License-Identifier: EPL-2.0 + + +import os + +import requests +from rich import print + +""" Use current Github API to list packages + in registry and remove all but last 3 or custom + set number of packages. + Reference: https://docs.github.com/en/rest/packages/packages?apiVersion=2022-11-28#about-github-packages +""" + +dry_run: bool = True if os.getenv("INPUT_DRY_RUN") == "true" else False +keep = int(os.getenv("INPUT_KEEP")) +org = os.getenv("GITHUB_REPOSITORY_OWNER") +packages = os.getenv("INPUT_PACKAGES").split("\n") +token = os.getenv("INPUT_TOKEN") + +headers = { + "Accept": "application/vnd.github+json", + "Authorization": f"Bearer {token}", + "X-GitHub-Api-Version": "2022-11-28", +} + +# Assembly organization packages url string +pkg_url: str = f"https://api.github.com/orgs/{org}/packages" + + +def delete_packages(): + for package in packages: + print(f":package: {package}") + url = f"{pkg_url}/container/{package.replace('/', '%2F')}/versions?per_page=100" + response = requests.get(url, headers=headers) + + if response.status_code == 404: + print(f":cross_mark: Not found - {url}") + continue + + # Sort all images on id. + images = sorted(response.json(), key=lambda x: x["id"], reverse=True) + + # Slice and remove all + if len(images) > keep: + for image in images[keep + 1 :]: + url = f"{pkg_url}/container/{package.replace('/', '%2F')}/versions/{image['id']}" + + # Never remove latest or non snapshot tagged images + if restrict_delete_tags(image["metadata"]["container"]["tags"]): + print( + f":package: Skip tagged {package} id {image['id']} tags {image['metadata']['container']['tags']}" + ) + continue + + if not dry_run: + response = requests.delete(url, headers=headers) + if response.status_code != 204: + print( + f":cross_mark: Failed to delete package {package} version id {image['id']}." + ) + continue + print( + f":white_heavy_check_mark: Deleted package {package} version id {image['id']}." + ) + + +def restrict_delete_tags(tags: list) -> bool: + if not tags: + return False + for tag in tags: + if tag == "latest": + return True + elif "nightly" in tag: + return True + return False + + +if __name__ == "__main__": + delete_packages() diff --git a/.github/workflows/clean_up_package_registry.yml b/.github/workflows/clean_up_package_registry.yml new file mode 100644 index 0000000000..bf358d19a9 --- /dev/null +++ b/.github/workflows/clean_up_package_registry.yml @@ -0,0 +1,33 @@ +# Copyright Helio Chissini de Castro, 2023 +# +# This program and the accompanying materials are made +# available under the terms of the Eclipse Public License 2.0 +# which is available at https://www.eclipse.org/legal/epl-2.0/ +# +# SPDX-License-Identifier: EPL-2.0 + +name: Clean up packages in Github package registry + +on: + workflow_dispatch: + # Runs always Sunday Midnight + schedule: + - cron: "0 0 * * 0" + +jobs: + clean_all: + name: Cleaning older packages + runs-on: ubuntu-22.04 + steps: + - name: Checkout default branch + uses: actions/checkout@v4 + - name: Clean up package registry + uses: ./.github/actions/clean_up_package_registry + with: + token: ${{ secrets.GITHUB_TOKEN }} + packages: |- + thrift + clucene + base + binaries + sw360 diff --git a/.github/workflows/docker_deploy.yml b/.github/workflows/docker_deploy.yml index 165995ea12..cf25cd3c24 100644 --- a/.github/workflows/docker_deploy.yml +++ b/.github/workflows/docker_deploy.yml @@ -14,13 +14,11 @@ name: Docker Build on: schedule: - - cron: '0 0 * * *' # Midnight + - cron: "0 0 * * *" # Midnight workflow_dispatch: - branches: - - main push: tags: - - 'sw360-*' + - "sw360-*" paths-ignore: - "**.md" @@ -30,9 +28,28 @@ env: permissions: write-all jobs: - docker_push: - if: ${{ github.event.schedule }} == '0 0 * * *' || ${{ github.event.act }} - name: Build Docker Image + sw360_version: + name: SW360 Version + runs-on: ubuntu-22.04 + outputs: + sw360_version: ${{ steps.pom_version.outputs.SW360_VERSION }} + + steps: + - name: Checkout main repository + uses: actions/checkout@v3 + + - name: Set up JDK 11 + uses: actions/setup-java@v3.13.0 + with: + java-version: "11" + distribution: "adopt" + + - name: Get revision from pom.xml + id: pom_version + run: | + echo "SW360_VERSION=$(mvn help:evaluate -Dexpression=revision -q -DforceStdout)" >> "$GITHUB_OUTPUT" + base_image: + name: SW360 Base image runs-on: ubuntu-22.04 permissions: contents: read @@ -45,8 +62,6 @@ jobs: - name: Set environment variables run: | cat .versions >> $GITHUB_ENV - echo "ORG_BASE_NAME=${GITHUB_REPOSITORY}" >> $GITHUB_ENV - echo "GIT_REVISION=$(git describe --abbrev=6 --always --tags --match=[0-9]*)" >> $GITHUB_ENV - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 @@ -58,19 +73,19 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - #------------------------------------------------ - # Base - name: Extract components metadata (tags, labels) for base image id: meta_base uses: docker/metadata-action@v4 with: images: | - ${{ env.REGISTRY }}/${{ env.ORG_BASE_NAME }}/base - - - name: Identify base image - run: echo "Will be tagged as ${{ steps.meta_base.outputs.tags }} and labeled as ${{ steps.meta_base.outputs.labels }}" + ${{ env.REGISTRY }}/${{ github.repository }}/base + tags: | + type=schedule,pattern={{date 'YYYYMMDD'}} + type=schedule,pattern=nightly + type=raw,value=${{ env.JAVA_VERSION }}-jdk-${{ env.UBUNTU_VERSION }} + type=raw,value=latest,enable={{is_default_branch}} - - name: Build sw360 base container + - name: Build image uses: docker/build-push-action@v4 with: context: . @@ -80,27 +95,51 @@ jobs: build-args: | LIFERAY_VERSION=${{ env.LIFERAY_VERSION }} LIFERAY_SOURCE=${{ env.LIFERAY_SOURCE }} - tags: | - ${{ steps.meta_base.outputs.tags }} - ${{ env.REGISTRY }}/${{ env.ORG_BASE_NAME }}/base:${{ env.GIT_REVISION }} - ${{ env.REGISTRY }}/${{ env.ORG_BASE_NAME }}/base:latest + JAVA_VERSION=${{ env.JAVA_VERSION }} + UBUNTU_VERSION=${{ env.UBUNTU_VERSION }} + tags: ${{ steps.meta_base.outputs.tags }} labels: ${{ steps.meta_base.outputs.labels }} cache-from: type=gha,scope=base cache-to: type=gha,scope=base,mode=max - #------------------------------------------------ - # Thrift + thrift_image: + name: SW360 Thrift image + runs-on: ubuntu-22.04 + permissions: + contents: read + packages: write + + steps: + - name: Checkout main repository + uses: actions/checkout@v3 + + - name: Set environment variables + run: | + cat .versions >> $GITHUB_ENV + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Login to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Extract components metadata (tags, labels) for thrift image id: meta_thrift uses: docker/metadata-action@v4 with: images: | - ${{ env.REGISTRY }}/${{ env.ORG_BASE_NAME }}/thrift - - - name: Identify thrift image - run: echo "Will be tagged as ${{ steps.meta_thrift.outputs.tags }} and labeled as ${{ steps.meta_thrift.outputs.labels }}" + ${{ env.REGISTRY }}/${{ github.repository }}/thrift + tags: | + type=schedule,pattern={{date 'YYYYMMDD'}} + type=schedule,pattern=nightly + type=raw,value=${{ env.THRIFT_VERSION }} + type=raw,value=latest,enable={{is_default_branch}} - - name: Build sw360 Thrift container + - name: Build image uses: docker/build-push-action@v4 with: context: . @@ -109,70 +148,111 @@ jobs: load: false build-args: | THRIFT_VERSION=${{ env.THRIFT_VERSION }} - tags: | - ${{ steps.meta_base.outputs.tags }} - ${{ env.REGISTRY }}/${{ env.ORG_BASE_NAME }}/thrift:${{ env.THRIFT_VERSION }} - ${{ env.REGISTRY }}/${{ env.ORG_BASE_NAME }}/thrift:latest + tags: ${{ steps.meta_thrift.outputs.tags }} labels: ${{ steps.meta_thrift.outputs.labels }} cache-from: type=gha,scope=thrift cache-to: type=gha,scope=thrift,mode=max - #------------------------------------------------ - # sw360 compilation + binary_image: + name: SW360 Binary + needs: [sw360_version, base_image, thrift_image] + runs-on: ubuntu-22.04 + permissions: + contents: read + packages: write + + steps: + - name: Checkout main repository + uses: actions/checkout@v3 + + - name: Set environment variables + run: | + cat .versions >> $GITHUB_ENV + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Login to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Extract components metadata (tags, labels) for sw360 id: meta_sw360 uses: docker/metadata-action@v4 with: images: | - ${{ env.REGISTRY }}/${{ env.ORG_BASE_NAME }}/binaries - - - name: Identify sw360 build image - run: echo "Will be tagged as ${{ steps.meta_sw360.outputs.tags }} and labeled as ${{ steps.meta_sw360.outputs.labels }}" + ${{ env.REGISTRY }}/${{ github.repository }}/binaries + tags: | + type=schedule,pattern={{date 'YYYYMMDD'}} + type=schedule,pattern=nightly + type=raw,value=${{ needs.sw360_version.outputs.sw360_version }} + type=sha,enable=true,prefix=sha-,format=short - - name: Build sw360 build container + - name: Build image uses: docker/build-push-action@v4 with: context: . target: sw360 push: true - load: false secret-files: | "sw360=./scripts/docker-config/default_secrets" tags: | ${{ steps.meta_sw360.outputs.tags }} - ${{ env.REGISTRY }}/${{ env.ORG_BASE_NAME }}/binaries:${{ env.GIT_REVISION }} - ${{ env.REGISTRY }}/${{ env.ORG_BASE_NAME }}/binaries:latest labels: ${{ steps.meta_sw360.outputs.labels }} build-contexts: | - sw360thrift=docker-image://${{ env.REGISTRY }}/${{ env.ORG_BASE_NAME }}/thrift:latest - cache-from: type=gha,scope=sw360 - cache-to: type=gha,scope=sw360,mode=max + sw360thrift=docker-image://${{ env.REGISTRY }}/${{ github.repository }}/thrift:${{ env.THRIFT_VERSION }} + + runtime_image: + name: SW360 Runtime image + needs: [sw360_version, base_image, binary_image] + runs-on: ubuntu-22.04 + permissions: + contents: read + packages: write + + steps: + - name: Checkout main repository + uses: actions/checkout@v3 + + - name: Set environment variables + run: | + cat .versions >> $GITHUB_ENV + echo "SHORT_SHA=sha-$(echo ${{ github.sha }} | cut -c 1-7)" >> $GITHUB_ENV + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Login to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} - #------------------------------------------------ - # sw360 runtime container - name: Extract components metadata (tags, labels) runtime image id: meta_runtime uses: docker/metadata-action@v4 with: images: | - ${{ env.REGISTRY }}/${{ env.ORG_BASE_NAME }} - - - name: Identify sw360 runtime image - run: echo "Will be tagged as ${{ steps.meta_runtime.outputs.tags }} and labeled as ${{ steps.meta_runtime.outputs.labels }}" + ${{ env.REGISTRY }}/${{ github.repository }} + tags: | + type=schedule,pattern={{date 'YYYYMMDD'}} + type=schedule,pattern=nightly + type=raw,value=${{ needs.sw360_version.outputs.sw360_version }} + type=sha,enable=true,prefix=sha-,format=short + type=ref,event=tag - - name: Build sw360 build container + - name: Build image uses: docker/build-push-action@v4 with: context: . target: runtime push: true - load: false - tags: | - ${{ steps.meta_runtime.outputs.tags }} + tags: ${{ steps.meta_runtime.outputs.tags }} labels: ${{ steps.meta_runtime.outputs.labels }} build-contexts: | - base=docker-image://${{ env.REGISTRY }}/${{ env.ORG_BASE_NAME }}/base:latest - sw360=docker-image://${{ env.REGISTRY }}/${{ env.ORG_BASE_NAME }}/binaries:latest - cache-from: type=gha,scope=runtime - cache-to: type=gha,scope=runtime,mode=max - + base=docker-image://${{ env.REGISTRY }}/${{ github.repository }}/base:${{ env.JAVA_VERSION }}-jdk-${{ env.UBUNTU_VERSION }} + sw360=docker-image://${{ env.REGISTRY }}/${{ github.repository }}/binaries:${{ env.SHORT_SHA }} diff --git a/.versions b/.versions index 91cf62022e..289c7660af 100755 --- a/.versions +++ b/.versions @@ -1,3 +1,5 @@ THRIFT_VERSION=0.18.1 LIFERAY_VERSION=7.4.3.18-ga18 LIFERAY_SOURCE=liferay-ce-portal-tomcat-7.4.3.18-ga18-20220329092001364.tar.gz +UBUNTU_VERSION=jammy +JAVA_VERSION=17 diff --git a/Dockerfile b/Dockerfile index 25d52df973..6291894a26 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,6 @@ # syntax=docker/dockerfile:1.4 # +# Copyright Helio Chisisni de Castro, 2023. Part of the SW360 Portal Project. # Copyright Siemens AG, 2020. Part of the SW360 Portal Project. # Copyright BMW CarIT GmbH, 2021. # @@ -8,12 +9,15 @@ # which is available at https://www.eclipse.org/legal/epl-2.0/ # # SPDX-License-Identifier: EPL-2.0 -# #----------------------------------------------------------------------------------- # Base image # We need use JDK, JRE is not enough as Liferay do runtime changes and require javac -FROM eclipse-temurin:11-jdk-jammy AS base +ARG JAVA_VERSION=11 +ARG UBUNTU_VERSION=jammy + +# Use OpenJDK Eclipe Temurin Ubuntu LTS +FROM eclipse-temurin:$JAVA_VERSION-jdk-$UBUNTU_VERSION as base ENV LANG=en_US.UTF-8 ENV LANGUAGE=en_US:en