[feature] Multi-repo workflow

README.md

@@ -21,6 +21,35 @@ Build:
   extends: .build
 ```
 
-> Instead of `/main/`, you can specify a specific commit to ensure changes do not affect your CI. 
+> Instead of `/main/`, you can specify a specific commit to ensure changes do not affect your CI.
 
 The [`examples`](examples/) folder contains examples of `.gitlab-ci.yml` that can be assembled from the templates.
+
+## Multi-repository templates
+
+In `templates/multi-repo` the CI workflow differs from `basic` CI (which in `templates`) in the following key aspects:
+
+- In `multi-repo` workflow we can push to `dev` and `prod` registries separately with their own rules (see `jobs/multi-repo` and/or `examples/multi-repo-module.gitlab-ci.yml` for example jobs).
+- All werf's caches and other artifacts (from `build` stage) are stored in Gitlab's module's registry by default. And **only final images** are pushed to the dev/prod registries. So, even in dev-registry there **should be no** "build-time garbage" and/or some "extra" images/layers for each module.
+
+### Detailed differences between `multi-repo` and `basic` workflows
+
+- [General] There is additional stage `lint` before `build` and `cleanup` stage after `deploy`.
+- [General] All `only` sections (like `only: [tags, branches]`) replaced with corresponding `rules` section.
+- [General] Added `Scheduled cleanup` job to cleanup Gitlab's registry by pipeline schedule
+- [General] Added `Auto cleanup` job to cleanup Gitlab's registry BEFORE `build` stage. Can be disabled via `AUTO_CLEANUP="false"` variable.
+- [General] Added `.default_rules` hidden job (see `templates/multi-repo/Setup.gitlab-ci.yml`) for easy modification of this whole workflow.
+- [General] Added `.deploy-prod-rules` hidden job (see `templates/multi-repo/Deploy.gitlab-ci.yml`) for easy modification of `deploy to production` workflow.
+- [General] Added `jobs/multi-repo` jobs files which user can include and use in their own workflow.
+- [General] Added ability to specify which module's `EDITION` (`CE`, `EE`, etc) should be pushed to PRODUCTION registry.
+- [Refactor] Default `before_script` section (see `templates/Setup.gitlab-ci.yml`) moved to `.setup/before_script` job.
+- [Refactor] `dmt lint` job moved to `lint` stage in dedicated `templates/multi-repo/Lint.gitlab-ci.yml` file.
+- [Refactor] All werf's caches and other artifacts (from `build` stage) are stored in Gitlab's registry (`${CI_REGISTRY_IMAGE}/${MODULES_MODULE_NAME}`) by default.
+- [Refactor] Images publishing (via `crane copy`) and module's self-registration processes moved to dedicated hidden job `.publish` (see `templates/multi-repo/Deploy.gitlab-ci.yml`).
+
+## Variables
+
+`$MODULES_REGISTRY` - base URL for the registry, e.g. `registry.example.com`
+`$MODULES_REGISTRY_PATH` - path to modules repository in registry, e.g. `deckhouse/modules`
+`$MODULES_MODULE_NAME` (Optional) - module name, by default it is equal to the project name
+`$RELEASE_CHANNEL` - lowercase release channel name, e.g., `alpha`, `stable`, `early-access`

examples/multi-repo-module.gitlab-ci.yml

@@ -0,0 +1,59 @@
+include:
+  - remote: 'https://raw.githubusercontent.com/deckhouse/modules-gitlab-ci/refs/heads/main/templates/multi-repo/Setup.gitlab-ci.yml'
+  - remote: 'https://raw.githubusercontent.com/deckhouse/modules-gitlab-ci/refs/heads/main/templates/multi-repo/Lint.gitlab-ci.yml'
+  - remote: 'https://raw.githubusercontent.com/deckhouse/modules-gitlab-ci/refs/heads/main/templates/multi-repo/Build.gitlab-ci.yml'
+  - remote: 'https://raw.githubusercontent.com/deckhouse/modules-gitlab-ci/refs/heads/main/templates/multi-repo/Deploy.gitlab-ci.yml'
+  - remote: 'https://raw.githubusercontent.com/deckhouse/modules-gitlab-ci/refs/heads/main/templates/multi-repo/Cleanup.gitlab-ci.yml'
+  # deploy jobs for DEV registry
+  - remote: 'https://raw.githubusercontent.com/deckhouse/modules-gitlab-ci/refs/heads/main/jobs/multi-repo/Deploy_DEV.gitlab-ci.yml'
+  # deploy jobs for PROD registry
+  - remote: 'https://raw.githubusercontent.com/deckhouse/modules-gitlab-ci/refs/heads/main/jobs/multi-repo/Deploy_PROD.gitlab-ci.yml'
+    inputs:
+      # Editions used in your module. Array of following items:
+      # ce - Community edition
+      # ee - Enterprise edition
+      # fe - Flant edition (internal edition for Flant's engineers)
+      # se - Standard edition
+      # se-plus - Standard edition +
+      editions:
+        # All values must be in lowercase and quoted
+        - "ce"
+        - "ee"
+        - "fe"
+        - "se"
+        - "se-plus"
+
+variables:
+  # Do not forget to put these variables to your Gitlab CI secrets:
+  # They are REQUIRED and used for pulling/pushing images to the corresponding registry
+  # - DEV_MODULES_REGISTRY: DEV registry domain (like: registry.example.com)
+  # - DEV_MODULES_REGISTRY_PATH: path to modules repository in DEV registry (like: deckhouse/modules)
+  # - DEV_MODULES_REGISTRY_LOGIN: username to log in to DEV registry
+  # - DEV_MODULES_REGISTRY_PASSWORD: password to log in to DEV registry
+
+  # WARNING: If some of following variables are NOT SET, then there is NO production deployment jobs will be created in pipeline
+  # - PROD_MODULES_REGISTRY: PROD registry domain (like: registry.example.com)
+  # - PROD_MODULES_REGISTRY_PATH: path to modules repository in PROD registry (like: deckhouse/modules)
+  # - PROD_MODULES_REGISTRY_LOGIN: username to log in to PROD registry
+  # - PROD_MODULES_REGISTRY_PASSWORD: password to log in to PROD registry
+  WERF_VERSION: "2 stable"
+  BASE_IMAGES_VERSION: v0.2
+
+default:
+  tags:
+    - my-runner-tag
+
+
+###### LINT STAGE ######
+
+Lint:
+  extends: .lint
+
+###### END OF LINT STAGE ######
+
+###### BUILD STAGE ######
+
+Build:
+  extends: .build
+
+###### END OF BUILD STAGE ######

jobs/multi-repo/Debug.gitlab-ci.yml

@@ -0,0 +1,13 @@
+variables:
+  DEBUG_CI:
+    value: "false"
+    description: "Run debug job(s)"
+
+debug:printenv:
+  stage: build
+  rules:
+    # run if $DEBUG_CI variable is set to true
+    - if: $DEBUG_CI == "true" || $DEBUG_CI == "1"
+  script:
+    - |
+      printenv | sort

jobs/multi-repo/Deploy_DEV.gitlab-ci.yml

@@ -0,0 +1,41 @@
+# emulate same behaviour as in Deckhouse Github registry
+# when opened PRs will pushed to dev registry
+DEV | Publish merge request:
+  extends: .publish
+  variables:
+    MODULES_REGISTRY: ${DEV_MODULES_REGISTRY}
+    MODULES_REGISTRY_PATH: ${DEV_MODULES_REGISTRY_PATH}
+    MODULES_REGISTRY_LOGIN: ${DEV_MODULES_REGISTRY_LOGIN}
+    MODULES_REGISTRY_PASSWORD: ${DEV_MODULES_REGISTRY_PASSWORD}
+    # names as in Github: "pr" + merge request project-level ID instead of branch name
+    MODULES_MODULE_TAG: pr${CI_MERGE_REQUEST_IID}
+  rules:
+    # do not run if some required variables is empty
+    - if: '$DEV_MODULES_REGISTRY == null || $DEV_MODULES_REGISTRY == "" || $DEV_MODULES_REGISTRY_PATH == null || $DEV_MODULES_REGISTRY_PATH == ""'
+      when: never
+    # run only for merge requests
+    - if: $CI_MERGE_REQUEST_IID && $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME != $CI_DEFAULT_BRANCH && $CI_PIPELINE_SOURCE == "merge_request_event"
+      when: on_success
+    # run when new branch is created and there are no opened merge requests for this branch and no commits to this branch yet (completely new branch from master/main)
+    # - if: $CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH && ($CI_MERGE_REQUEST_IID == null || $CI_MERGE_REQUEST_IID == "") && $CI_COMMIT_BEFORE_SHA == "0000000000000000000000000000000000000000"
+    #   when: on_success
+    # do not run in other cases
+    - when: never
+
+DEV | Publish default branch:
+  extends: .publish
+  variables:
+    MODULES_REGISTRY: ${DEV_MODULES_REGISTRY}
+    MODULES_REGISTRY_PATH: ${DEV_MODULES_REGISTRY_PATH}
+    MODULES_REGISTRY_LOGIN: ${DEV_MODULES_REGISTRY_LOGIN}
+    MODULES_REGISTRY_PASSWORD: ${DEV_MODULES_REGISTRY_PASSWORD}
+    MODULES_MODULE_TAG: ${CI_DEFAULT_BRANCH}
+  rules:
+    # do not run if some required variables is empty
+    - if: '$DEV_MODULES_REGISTRY == null || $DEV_MODULES_REGISTRY == "" || $DEV_MODULES_REGISTRY_PATH == null || $DEV_MODULES_REGISTRY_PATH == ""'
+      when: never
+    # run only when push to default (main/master) branch
+    - if: $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
+      when: on_success
+    # do not run in other cases
+    - when: never

jobs/multi-repo/Deploy_PROD.gitlab-ci.yml

@@ -0,0 +1,54 @@
+# https://docs.gitlab.com/ci/inputs/
+spec:
+  inputs:
+    editions:
+      type: array
+      description: List of module editions
+      default:
+        - ee
+        - fe
+        - se
+        - se-plus
+
+---
+
+PROD | Alpha:
+  extends: .deploy_prod
+  variables:
+    RELEASE_CHANNEL: alpha
+  parallel:
+    matrix:
+      - EDITION: $[[ inputs.editions ]]
+
+PROD | Beta:
+  extends: .deploy_prod
+  variables:
+    RELEASE_CHANNEL: beta
+  parallel:
+    matrix:
+      - EDITION: $[[ inputs.editions ]]
+
+PROD | EarlyAccess:
+  extends: .deploy_prod
+  variables:
+    RELEASE_CHANNEL: early-access
+  parallel:
+    matrix:
+      - EDITION: $[[ inputs.editions ]]
+
+PROD | Stable:
+  extends: .deploy_prod
+  variables:
+    RELEASE_CHANNEL: stable
+  parallel:
+    matrix:
+      - EDITION: $[[ inputs.editions ]]
+
+# because uppercased letters are ordered before lowercased, so put rock-solid job last as in stability level, not alphabetical
+PROD | rock-solid:
+  extends: .deploy_prod
+  variables:
+    RELEASE_CHANNEL: rock-solid
+  parallel:
+    matrix:
+      - EDITION: $[[ inputs.editions ]]

templates/Build.gitlab-ci.yml

@@ -43,4 +43,4 @@
         --new_tag "${MODULES_MODULE_SOURCE}:${MODULES_MODULE_NAME}"
   only:
     - tags
-    - branches
+    - branches

templates/Deploy.gitlab-ci.yml

@@ -9,12 +9,12 @@
   script:
   - |
     REPO="${MODULES_MODULE_SOURCE}/${MODULES_MODULE_NAME}/release"
-    
+
     IMAGE_SRC="${REPO}:${MODULES_MODULE_TAG}"
     IMAGE_DST="${REPO}:${RELEASE_CHANNEL}"
-    
+
     echo "✨ Pushing ${IMAGE_SRC} to ${IMAGE_DST}"
     crane copy "${IMAGE_SRC}" "${IMAGE_DST}"
   only:
     - tags
-  when: manual
+  when: manual

templates/Setup.gitlab-ci.yml

@@ -38,4 +38,4 @@ before_script:
   - curl --fail -sSLO https://fox.flant.com/api/v4/projects/deckhouse%2Fbase-images/packages/generic/base_images/${BASE_IMAGES_VERSION}/base_images.yml
 stages:
   - build
-  - deploy
+  - deploy

templates/multi-repo/Build.gitlab-ci.yml

@@ -0,0 +1,15 @@
+.build:
+  stage: build
+  rules:
+    - !reference [.default_rules, rules]
+  before_script:
+    - !reference [.setup, before_script]
+  script:
+    # Build images
+    - |
+      werf build \
+        --save-build-report --build-report-path images_tags_werf.json
+  artifacts:
+    paths:
+      - images_tags_werf.json
+    expire_in: "30 days"

templates/multi-repo/Cleanup.gitlab-ci.yml

@@ -0,0 +1,33 @@
+Scheduled cleanup:
+  stage: cleanup
+  rules:
+    - if: $CI_PIPELINE_SOURCE == "schedule"
+      when: on_success
+    - when: never
+  before_script:
+    - !reference [.setup, before_script]
+  script:
+    - werf managed-images ls
+    - werf cleanup
+
+Auto cleanup:
+  stage: cleanup
+  allow_failure: true
+  timeout: 10 minutes
+  rules:
+    # do not run if this job is explicitly disabled by user
+    - if: $AUTO_CLEANUP == "false" || $AUTO_CLEANUP == "0" || $AUTO_CLEANUP == ""
+      when: never
+    # do not run if there is a tag (release workflow)
+    - if: $CI_COMMIT_TAG
+      when: never
+    - !reference [.default_rules, rules]
+  before_script:
+    - !reference [.setup, before_script]
+  script:
+    - |
+      if (( $(date +%s) % 10 == 0 )); then
+        echo "✨ Run auto cleanup"
+        werf managed-images ls
+        werf cleanup
+      fi

templates/multi-repo/Deploy.gitlab-ci.yml

@@ -0,0 +1,103 @@
+.publish:
+  stage: deploy
+  script:
+    - |
+      # Login to Gitlab (source) registry if target registry is not same Gitlab
+      if [[ "x${MODULES_REGISTRY}" != "x${CI_REGISTRY}" ]]; then
+        echo "Login to Gitlab (source) ${CI_REGISTRY}..."
+        werf cr login -u ${CI_REGISTRY_USER} -p ${CI_REGISTRY_PASSWORD} ${CI_REGISTRY}
+      fi
+
+      # Login to target registry
+      werf cr login -u ${MODULES_REGISTRY_LOGIN} -p ${MODULES_REGISTRY_PASSWORD} ${MODULES_REGISTRY}
+    # generate MODULES_MODULE_SOURCE
+    - |
+      export MODULES_MODULE_SOURCE="${MODULES_REGISTRY}/${MODULES_REGISTRY_PATH}"
+
+    # Module images
+    - |
+      jq -r '.Images | values[] | select (.Final==true) | .WerfImageName + " " + .DockerImageName + " " + .DockerTag + " " + .DockerImageDigest' < images_tags_werf.json | while read line; do
+        image="$(cut -d " " -f 1 <<< ${line})"
+        docker_image="$(cut -d " " -f 2 <<< ${line})"
+        docker_tag="$(cut -d " " -f 3 <<< ${line})"
+        shasum="$(cut -d " " -f 4 <<< ${line})"
+        name="$(tr '[:lower:]' '[:upper:]' <<< ${image} | sed 's|[-/]|_|g')"
+
+        # image: pg-images-17.3-bookworm-standard;
+        # name: PG_IMAGES_17.3_BOOKWORM_STANDARD;
+        # shasum: sha256:cd6daa30c94ec77b352156cbf1c05c8e98b44d1d3a47fa2b0a1083587247f730;
+        # docker_image: registry.flant.com/team/managed-services/managed-psql/managed-psql-d8/managed-postgres:ed0388a743d61926309d1023e02c639c1006f7b7b56d78161f32b0e0-1744971955188;
+        # docker_tag: ed0388a743d61926309d1023e02c639c1006f7b7b56d78161f32b0e0-1744971955188
+
+        IMAGE_SRC="${docker_image}"
+        IMAGE_DST="${MODULES_MODULE_SOURCE}/${MODULES_MODULE_NAME}:${docker_tag}"
+
+        echo "✨ Pushing ${IMAGE_SRC} to ${IMAGE_DST}"
+        crane copy ${IMAGE_SRC} ${IMAGE_DST}
+      done
+
+    # Bundle image
+    - |
+      IMAGE_SRC="$(jq -r '.Images."bundle".DockerImageName' images_tags_werf.json)"
+      IMAGE_DST="${MODULES_MODULE_SOURCE}/${MODULES_MODULE_NAME}:${MODULES_MODULE_TAG}"
+
+      echo "✨ Pushing BUNDLE ${IMAGE_SRC} to ${IMAGE_DST}"
+      crane copy ${IMAGE_SRC} ${IMAGE_DST}
+    # Release-channel image
+    - |
+      IMAGE_SRC="$(jq -r '.Images."release-channel-version".DockerImageName' images_tags_werf.json)"
+      IMAGE_DST="${MODULES_MODULE_SOURCE}/${MODULES_MODULE_NAME}/release:${MODULES_MODULE_TAG}"
+
+      echo "✨ Pushing RELEASE ${IMAGE_SRC} to ${IMAGE_DST}"
+      crane copy ${IMAGE_SRC} ${IMAGE_DST}
+    # Register module
+    - |
+      echo "✨ Register the module ${MODULES_MODULE_NAME}"
+      crane append \
+        --oci-empty-base \
+        --new_layer "" \
+        --new_tag "${MODULES_MODULE_SOURCE}:${MODULES_MODULE_NAME}"
+
+.deploy:
+  stage: deploy
+  script:
+  - |
+    REPO="${MODULES_REGISTRY}/${MODULES_REGISTRY_PATH}/${MODULES_MODULE_NAME}/release"
+
+    IMAGE_SRC="${REPO}:${MODULES_MODULE_TAG}"
+    IMAGE_DST="${REPO}:${RELEASE_CHANNEL}"
+
+    echo "✨ Pushing ${IMAGE_SRC} to ${IMAGE_DST}"
+    crane copy "${IMAGE_SRC}" "${IMAGE_DST}"
+
+.deploy_prod_rules:
+  rules:
+    # add MANUAL deploy job if $FORCE_CI variable is set
+    - if: $FORCE_CI == "true" || $FORCE_CI == "1"
+      when: manual
+    # do not run if some required variables is empty
+    - if: '$PROD_MODULES_REGISTRY == null || $PROD_MODULES_REGISTRY == "" || $EDITION == null || $EDITION == "" || $RELEASE_CHANNEL == null || $RELEASE_CHANNEL == ""'
+      when: never
+    # add MANUAL deploy job only if it is a tag defined (release workflow)
+    - if: '$CI_COMMIT_TAG'
+      when: manual
+
+.deploy_prod:
+  stage: deploy
+  rules:
+    - !reference [.deploy_prod_rules, rules]
+  variables:
+    MODULES_REGISTRY: $PROD_MODULES_REGISTRY
+    MODULES_REGISTRY_LOGIN: $PROD_MODULES_REGISTRY_LOGIN
+    MODULES_REGISTRY_PASSWORD: $PROD_MODULES_REGISTRY_PASSWORD
+    # path in PROD registry must be hardcoded
+    MODULES_REGISTRY_PATH: deckhouse/${EDITION}/modules
+  script:
+    - |
+      if [ "$DEBUG_CI" = "true" -o "$DEBUG_CI" = "1" ]; then
+        printenv | sort
+      fi
+    # publish final images to prod registry and register module with $MODULES_MODULE_TAG
+    - !reference [.publish, script]
+    # make 'symlink' to published module tag for specified release-channel
+    - !reference [.deploy, script]