diff --git a/.github/workflows/build-using-buildscripts.yml b/.github/workflows/build-using-buildscripts.yml index 2641a1199..d245be510 100644 --- a/.github/workflows/build-using-buildscripts.yml +++ b/.github/workflows/build-using-buildscripts.yml @@ -2,6 +2,13 @@ name: Build dependencies and CFEngine hub package on: workflow_call: + inputs: + additional_artifacts: + description: 'Additional files or directories to include in artifacts' + default: '' + required: false + type: string + secrets: GH_ACTIONS_SSH_DEPLOY_KEY_ENTERPRISE_REPO: required: true @@ -14,8 +21,8 @@ on: jobs: build_cfengine_hub_package: - name: Build package and run selenium tests - runs-on: ubuntu-20.04 + name: Build package + runs-on: ubuntu-22.04 steps: - name: Checkout Together Action uses: actions/checkout@v3 @@ -46,9 +53,10 @@ jobs: path: masterfiles ref: ${{steps.together.outputs.masterfiles || github.base_ref}} - - name: Checkout Buildscripts (current project) + - name: Checkout Buildscripts uses: actions/checkout@v3 with: + repository: cfengine/buildscripts path: buildscripts fetch-depth: 20 @@ -88,13 +96,13 @@ jobs: run: echo "DEPS_SHA=$(git log --pretty='format:%h' -1 -- .)" | tee -a ${GITHUB_ENV} working-directory: buildscripts/deps-packaging - - name: restore packages cache + - name: restore artifacts cache uses: actions/cache/restore@v3 with: - path: packages - key: packages-${{ env.PACKAGE_SHA }} + path: artifacts + key: artifacts-${{ env.PACKAGE_SHA }} restore-keys: | - packages-${{ env.PACKAGE_SHA }} + artifacts-${{ env.PACKAGE_SHA }} - name: Restore dependency cache uses: actions/cache/restore@v3 @@ -106,13 +114,22 @@ jobs: deps-master deps + - name: restore configured and built core and nova projects + uses: actions/cache/restore@v3 + with: + path: build + key: build-${{ env.PACKAGE_SHA }} + restore-keys: | + build-${{ env.PACKAGE_SHA }} + + - name: Build package in docker env: GH_ACTIONS_SSH_KEY_BUILD_ARTIFACTS_CACHE: ${{ secrets.GH_ACTIONS_SSH_KEY_BUILD_ARTIFACTS_CACHE }} run: | - if [ ! -f packages/cfe*deb ]; then - buildscripts/ci/docker-build-package.sh - fi + if [ ! -f artifacts/cfe*deb ]; then + buildscripts/ci/docker-build-package.sh ${{ inputs.additional_artifacts }} + fi - name: Save dependency cache uses: actions/cache/save@v3 @@ -120,11 +137,13 @@ jobs: path: cache key: deps-${{ github.base_ref }}-${{ env.DEPS_SHA }} - - name: Save packages cache + - name: Save artifacts cache uses: actions/cache/save@v3 with: - path: packages - key: packages-${{ env.PACKAGE_SHA }} + path: | + artifacts + packages + key: artifacts-${{ env.PACKAGE_SHA }} - name: Save artifacts if: success() || failure() diff --git a/.github/workflows/deployment-tests.yml b/.github/workflows/deployment-tests.yml index ceeabc51b..036d626c0 100644 --- a/.github/workflows/deployment-tests.yml +++ b/.github/workflows/deployment-tests.yml @@ -90,13 +90,13 @@ jobs: run: echo "DEPS_SHA=$(git log --pretty='format:%h' -1 -- .)" | tee -a ${GITHUB_ENV} working-directory: buildscripts/deps-packaging - - name: restore packages cache + - name: restore artifacts cache uses: actions/cache/restore@v3 with: - path: packages - key: packages-${{ env.PACKAGE_SHA }} + path: artifacts + key: artifacts-${{ env.PACKAGE_SHA }} restore-keys: | - packages-${{ env.PACKAGE_SHA }} + artifacts-${{ env.PACKAGE_SHA }} - name: Restore dependency cache uses: actions/cache/restore@v3 @@ -113,9 +113,9 @@ jobs: env: GH_ACTIONS_SSH_KEY_BUILD_ARTIFACTS_CACHE: ${{ secrets.GH_ACTIONS_SSH_KEY_BUILD_ARTIFACTS_CACHE }} run: | - if [ ! -f packages/cfe*deb ]; then - buildscripts/ci/docker-build-package.sh - fi + if [ ! -f artifacts/cfe*deb ]; then + buildscripts/ci/docker-build-package.sh + fi - name: Run deployment tests run: buildscripts/ci/docker-deployment-tests.sh @@ -126,11 +126,11 @@ jobs: path: cache key: deps-${{ github.base_ref }}-${{ env.DEPS_SHA }} - - name: Save packages cache + - name: Save artifacts cache uses: actions/cache/save@v3 with: - path: packages - key: packages-${{ env.PACKAGE_SHA }} + path: artifacts + key: artifacts-${{ env.PACKAGE_SHA }} - name: Save artifacts if: success() || failure() diff --git a/ci/build.sh b/ci/build.sh index b6feafec1..f6dce32d2 100755 --- a/ci/build.sh +++ b/ci/build.sh @@ -1,6 +1,10 @@ #!/usr/bin/env bash # build cfengine hub package +# $@ -- optional space separated paths to copy to artifacts + set -ex + +additional_artifacts="$@" export PROJECT=nova export NO_CONFIGURE=1 export BUILD_TYPE=DEBUG @@ -30,9 +34,15 @@ time sudo rm -rf /var/cfengine time sudo rm -rf /opt/cfengine time ./buildscripts/build-scripts/install-dependencies time ./buildscripts/build-scripts/package -sudo mkdir -p packages -sudo cp cfengine-nova-hub/*.deb packages/ || true -sudo cp cfengine-nova-hub/*.rpm packages/ || true + +sudo mkdir -p artifacts +sudo cp cfengine-nova-hub/*.deb artifacts/ || true +sudo cp cfengine-nova-hub/*.rpm artifacts/ || true + +for artifact_path in $additional_artifacts; do + sudo cp -r "$artifact_path" artifacts/ || true +done + # todo maybe save the cache cp -R ~/.cache buildscripts/ci/cache diff --git a/ci/deployment-tests.sh b/ci/deployment-tests.sh index 8efedc28d..f9e9d5489 100755 --- a/ci/deployment-tests.sh +++ b/ci/deployment-tests.sh @@ -27,8 +27,8 @@ function failure() { } if [ ! -d /var/cfengine ]; then - # ci and local buildscripts should place built packages in $NTECH_ROOT/packages - sudo dpkg -i "$NTECH_ROOT"/packages/cfengine-nova-hub*deb + # ci and local buildscripts should place built packages in $NTECH_ROOT/artifacts + sudo dpkg -i "$NTECH_ROOT"/artifacts/cfengine-nova-hub*deb fi diff --git a/ci/docker-build-package.sh b/ci/docker-build-package.sh index 908b171dd..5c55b9c56 100755 --- a/ci/docker-build-package.sh +++ b/ci/docker-build-package.sh @@ -1,5 +1,7 @@ #!/usr/bin/env bash # run the build in a docker container +# $@ -- additional artifact paths to save +additional_artifacts="$@" set -ex # find the dir two levels up from here, home of all the repositories @@ -19,7 +21,7 @@ set +x # hide secrets if [ -n "$GH_ACTIONS_SSH_KEY_BUILD_ARTIFACTS_CACHE" ]; then export SECRET="$GH_ACTIONS_SSH_KEY_BUILD_ARTIFACTS_CACHE" else - if ! export SECRET="$(pass mystiko/developers/CFEngine/jenkins/sftp-cache.sec)"; then + if ! export SECRET="$(pass mystiko/developers/CFEngine/jenkins/jenkins_sftp_cache@github)"; then echo "The sftp cache ssh secret key must be provided, either with environment variable GH_ACTIONS_SSH_KEY_BUILD_ARTIFACTS_CACHE or access to mystiko path developers/CFEngine/jenkins/sftp-cache.sec" exit 1 fi @@ -29,7 +31,7 @@ set -x # done hiding secrets docker run -d --env SECRET --env JOB_BASE_NAME --privileged -v "${NTECH_ROOT}":/data --name $name $name # copy local caches to docker container -mkdir -p "${NTECH_ROOT}/packages" +mkdir -p "${NTECH_ROOT}/artifacts" mkdir -p "${NTECH_ROOT}/cache" # setup host key trust @@ -49,14 +51,14 @@ docker exec -i $name bash -c "mkdir -p ~/.ssh" docker exec -i $name bash -c "echo $pubkey >> ~/.ssh/known_hosts" docker exec -i $name bash -c 'cd /data; ./buildscripts/ci/setup-projects.sh' -docker exec -i $name bash -c 'cd /data; ./buildscripts/ci/build.sh' +docker exec -i $name bash -c "cd /data; ./buildscripts/ci/build.sh ${additional_artifacts}" -# save back cache and packages to host for handling by CI and such +# save back cache and artifacts to host for handling by CI and such docker cp $name:/root/.cache/. "${NTECH_ROOT}/cache/" -docker cp $name:/data/packages/. "${NTECH_ROOT}/packages/" +docker cp $name:/data/artifacts/. "${NTECH_ROOT}/artifacts/" rc=1 # if we find no packages, fail -for f in packages/*.deb; do +for f in artifacts/*.deb; do [ -f "$f" ] && rc=0 break done diff --git a/ci/docker-deployment-tests.sh b/ci/docker-deployment-tests.sh index 35b9060d3..4274b5293 100755 --- a/ci/docker-deployment-tests.sh +++ b/ci/docker-deployment-tests.sh @@ -21,8 +21,8 @@ if docker ps -a | grep $name; then fi docker run -d --privileged -v "${NTECH_ROOT}":/data --name $name $name || true -if [ ! -d "${NTECH_ROOT}/packages" ]; then - echo "${NTECH_ROOT}/packages directory should exist and have a cfengine-nova-hub package there" +if [ ! -d "${NTECH_ROOT}/artifacts" ]; then + echo "${NTECH_ROOT}/artifacts directory should exist and have a cfengine-nova-hub package there" exit 1 fi docker exec -i $name bash -c 'cd /data; ./buildscripts/ci/deployment-tests.sh'