Skip to content

VyOS nightly build #631

VyOS nightly build

VyOS nightly build #631

name: VyOS rolling nightly build
on:
schedule:
- cron: "0 0 * * *"
workflow_dispatch:
inputs:
BUILD_BY:
description: 'Builder identifier (if empty [email protected] is used)'
default: ''
BUILD_VERSION:
description: 'Version number (if empty 1.5-rolling-$(date -u +%Y%m%d%H%M) is used)'
default: ''
FAKE_BUILDING_PROCESS:
description: 'Skip all Smoketests and fake building process'
required: true
type: boolean
default: false
SKIP_SMOKETEST_RAID1:
description: 'Skip RAID1 Smoketest'
required: true
type: boolean
default: false
SKIP_SMOKETEST_SYSTEM:
description: 'Skip system Smoketest'
required: true
type: boolean
default: false
SKIP_SMOKETEST_CONFIG:
description: 'Skip config Smoketest'
required: true
type: boolean
default: false
SKIP_SNAPSHOT_S3_UPLOAD:
description: 'Skip snapshot upload to S3'
required: true
type: boolean
default: false
SKIP_RELEASE_PUBLISHING:
description: 'Skip release publishing'
required: true
type: boolean
default: false
SKIP_SLACK_NOTIFICATIONS:
description: 'Skip Slack notifications'
required: true
type: boolean
default: false
SKIP_DOWNLOADS_PAGE_UPDATE:
description: 'Skip downloads page update'
required: true
type: boolean
default: false
jobs:
build-iso:
runs-on: ubuntu-latest
permissions:
contents: write
concurrency:
group: main
env:
BUILD_BY: [email protected]
DEBIAN_MIRROR: http://deb.debian.org/debian/
VYOS_MIRROR: https://rolling-packages.vyos.net/current/
DOCKER_CALL: docker run --rm --privileged -v ./vyos-build/:/vyos -w /vyos vyos/vyos-build:current
DOCKER_CALL_ON_KVM_HOST: docker run --rm --privileged -v ~/vyos-build:/vyos -w /vyos vyos/vyos-build:current
DOCKER_CALL_ON_KVM_HOST_WITH_IMAGE_PULL: docker run --rm --privileged --pull=always -v ~/vyos-build:/vyos -w /vyos vyos/vyos-build:current
steps:
### Initialization ###
- uses: actions/checkout@v4
- name: "Initialization: set env variables"
id: set_env_variables
run: |
if [ -n "${{ github.event.inputs.BUILD_BY }}" ]; then
echo "BUILD_BY=${{ github.event.inputs.BUILD_BY }}" >> $GITHUB_ENV
fi
if [ -z "${{ github.event.inputs.BUILD_VERSION }}" ]; then
echo "BUILD_VERSION=1.5-rolling-$(date -u +%Y%m%d%H%M)" >> $GITHUB_ENV
else
echo "BUILD_VERSION=${{ github.event.inputs.BUILD_VERSION }}" >> $GITHUB_ENV
fi
echo "TIMESTAMP=$(date -u +%Y-%m-%dT%H:%M:%SZ)" >> $GITHUB_ENV
echo "PREVIOUS_SUCCESS_BUILD_TIMESTAMP=$(cat version.json | jq -r '.[0].timestamp')" >> $GITHUB_ENV
- name: "Initialization: git clone vyos-build"
id: git_clone_vyos-build
run: git clone -b current --single-branch https://github.com/vyos/vyos-build
- name: "Initialization: git clone vyos-1x"
id: git_clone_vyos-1x
if: ${{ !inputs.SKIP_RELEASE_PUBLISHING }}
run: git clone -b current --single-branch https://github.com/vyos/vyos-1x
### Smoketest ###
- name: "Smoketest: build generic ISO image"
id: build_iso_for_smoketest
if: ${{ !inputs.FAKE_BUILDING_PROCESS && (!inputs.SKIP_SMOKETEST_RAID1 || !inputs.SKIP_SMOKETEST_SYSTEM || !inputs.SKIP_SMOKETEST_CONFIG) }}
run: |
${{ env.DOCKER_CALL }} \
sudo --preserve-env ./build-vyos-image \
--architecture amd64 \
--build-by $BUILD_BY \
--build-type release \
--custom-package vyos-1x-smoketest \
--debian-mirror $DEBIAN_MIRROR \
--version $BUILD_VERSION \
--vyos-mirror $VYOS_MIRROR \
generic
- name: "Smoketest: create generic ISO artifact"
id: upload_iso_artifact_smoketest_image
if: ${{ !inputs.FAKE_BUILDING_PROCESS && (!inputs.SKIP_SMOKETEST_RAID1 || !inputs.SKIP_SMOKETEST_SYSTEM || !inputs.SKIP_SMOKETEST_CONFIG) }}
uses: actions/upload-artifact@v4
with:
name: vyos-${{ env.BUILD_VERSION }}-amd64-smoketest.iso
path: ./vyos-build/build/live-image-amd64.hybrid.iso
retention-days: 30
if-no-files-found: error
- name: "Smoketest: upload generic ISO to KVM host"
if: ${{ !inputs.FAKE_BUILDING_PROCESS && (!inputs.SKIP_SMOKETEST_RAID1 || !inputs.SKIP_SMOKETEST_SYSTEM || !inputs.SKIP_SMOKETEST_CONFIG) }}
id: copy_smoketest_iso_to_kvm_host
uses: cross-the-world/ssh-scp-ssh-pipelines@latest
env:
BUILD_VERSION: ${{ env.BUILD_VERSION }}
with:
host: ${{ secrets.SSH_HOST }}
user: ${{ secrets.SSH_USER }}
port: ${{ secrets.SSH_PORT }}
key: ${{ secrets.SSH_KEY }}
connect_timeout: 600s
first_ssh: |
rm -rf vyos-build || true
git clone -b current --single-branch https://github.com/vyos/vyos-build
mkdir -p ~/vyos-build/build/
scp: |
'./vyos-build/build/live-image-amd64.hybrid.iso' => '~/vyos-build/build/'
timeout-minutes: 300
- name: "Smoketest: RAID"
if: ${{ !inputs.FAKE_BUILDING_PROCESS && !inputs.SKIP_SMOKETEST_RAID1 }}
id: smoketest_raid
uses: appleboy/[email protected]
with:
host: ${{ secrets.SSH_HOST }}
username: ${{ secrets.SSH_USER }}
port: ${{ secrets.SSH_PORT }}
key: ${{ secrets.SSH_KEY }}
timeout: 15m
command_timeout: 180m
script: |
${{ env.DOCKER_CALL_ON_KVM_HOST_WITH_IMAGE_PULL }} /bin/bash -c "set -o pipefail && sudo make testraid | tee smoketest_raid.log"
- name: "Smoketest: system"
if: ${{ !inputs.FAKE_BUILDING_PROCESS && !inputs.SKIP_SMOKETEST_SYSTEM }}
id: smoketest_system
uses: appleboy/[email protected]
with:
host: ${{ secrets.SSH_HOST }}
username: ${{ secrets.SSH_USER }}
port: ${{ secrets.SSH_PORT }}
key: ${{ secrets.SSH_KEY }}
timeout: 15m
command_timeout: 180m
script: |
${{ env.DOCKER_CALL_ON_KVM_HOST }} /bin/bash -c "set -o pipefail && sudo make test | tee smoketest_system.log"
- name: "Smoketest: configuration"
if: ${{ !inputs.FAKE_BUILDING_PROCESS && !inputs.SKIP_SMOKETEST_CONFIG }}
id: smoketest_configuration
uses: appleboy/[email protected]
with:
host: ${{ secrets.SSH_HOST }}
username: ${{ secrets.SSH_USER }}
port: ${{ secrets.SSH_PORT }}
key: ${{ secrets.SSH_KEY }}
timeout: 15m
command_timeout: 180m
script: |
${{ env.DOCKER_CALL_ON_KVM_HOST }} /bin/bash -c "set -o pipefail && sudo make testc | tee smoketest_configuration.log"
- name: "Smoketest: clear files"
if: ${{ !inputs.FAKE_BUILDING_PROCESS && (!inputs.SKIP_SMOKETEST_RAID1 || !inputs.SKIP_SMOKETEST_SYSTEM || !inputs.SKIP_SMOKETEST_CONFIG) }}
id: clear_smoketest_files
uses: appleboy/[email protected]
with:
host: ${{ secrets.SSH_HOST }}
username: ${{ secrets.SSH_USER }}
port: ${{ secrets.SSH_PORT }}
key: ${{ secrets.SSH_KEY }}
timeout: 15m
command_timeout: 45m
script: |
${{ env.DOCKER_CALL_ON_KVM_HOST }} sudo rm -rf ./*
rm -rf ~/vyos-build
### Building ###
- name: "Building: create ISO image"
if: ${{ !inputs.FAKE_BUILDING_PROCESS }}
id: build_iso
run: |
${{ env.DOCKER_CALL }} \
sudo --preserve-env ./build-vyos-image \
--architecture amd64 \
--build-by $BUILD_BY \
--build-type release \
--debian-mirror $DEBIAN_MIRROR \
--version $BUILD_VERSION \
--vyos-mirror $VYOS_MIRROR \
generic
- name: "Building: copy a generic ISO image"
if: ${{ !inputs.FAKE_BUILDING_PROCESS }}
id: copy_iso
run: |
cp ./vyos-build/build/live-image-amd64.hybrid.iso ./vyos-$BUILD_VERSION-amd64.iso
- name: "Building: fake a generic ISO image"
if: ${{ inputs.FAKE_BUILDING_PROCESS }}
id: fake_iso
run: |
echo FAKE > ./vyos-$BUILD_VERSION-amd64.iso
- name: "Building: sign an ISO image with Minisign"
id: sign_iso_minisign
run: |
echo "${{ secrets.minisign_private_key }}" > /tmp/minisign.key
shasum /tmp/minisign.key
echo ${{ secrets.minisign_password }} | $GITHUB_WORKSPACE/bin/minisign -s /tmp/minisign.key -Sm ./vyos-$BUILD_VERSION-amd64.iso
echo "${{ secrets.minisign_public_key }}" > /tmp/minisign.pub
$GITHUB_WORKSPACE/bin/minisign -Vm ./vyos-$BUILD_VERSION-amd64.iso -x ./vyos-$BUILD_VERSION-amd64.iso.minisig -p /tmp/minisign.pub
rm /tmp/minisign.key /tmp/minisign.pub
### Uploading artifacts ###
- name: "Uploading artifacts: ISO image to S3 Glacier"
id: upload_iso_to_s3_glacier
if: ${{ !inputs.SKIP_SNAPSHOT_S3_UPLOAD }}
uses: keithweaver/[email protected]
with:
command: cp
source: ./vyos-${{ env.BUILD_VERSION }}-amd64.iso
destination: s3://${{ secrets.aws_s3_bucket }}/vyos-${{ env.BUILD_VERSION }}-amd64.iso
aws_access_key_id: ${{ secrets.aws_access_key_id }}
aws_secret_access_key: ${{ secrets.aws_secret_access_key }}
aws_region: us-east-1
flags: --storage-class GLACIER
- name: "Uploading artifacts: ISO image's signature to S3 Glacier"
id: upload_iso_signature_to_s3_glacier
if: ${{ !inputs.SKIP_SNAPSHOT_S3_UPLOAD }}
uses: keithweaver/[email protected]
with:
command: cp
source: ./vyos-${{ env.BUILD_VERSION }}-amd64.iso.minisig
destination: s3://${{ secrets.aws_s3_bucket }}/vyos-${{ env.BUILD_VERSION }}-amd64.iso.minisig
aws_access_key_id: ${{ secrets.aws_access_key_id }}
aws_secret_access_key: ${{ secrets.aws_secret_access_key }}
aws_region: us-east-1
flags: --storage-class GLACIER
- name: "Uploading artifacts: ISO image to GitHub"
id: upload_iso_artifact
uses: actions/upload-artifact@v4
with:
name: vyos-${{ env.BUILD_VERSION }}-amd64.iso
path: ./vyos-${{ env.BUILD_VERSION }}-amd64.iso
retention-days: 30
if-no-files-found: error
- name: "Uploading artifacts: ISO image's signature to GitHub"
id: upload_iso_minisign_artifact
uses: actions/upload-artifact@v4
with:
name: vyos-${{ env.BUILD_VERSION }}-amd64.iso.minisig
path: ./vyos-${{ env.BUILD_VERSION }}-amd64.iso.minisig
retention-days: 30
if-no-files-found: error
### Release publishing ###
- name: "Release publishing: retrieve the latest success build for vyos-build and vyos-1x"
id: retrieve_the_latest_success_build_for_vyos-build_vyos-1x
if: ${{ !inputs.SKIP_RELEASE_PUBLISHING }}
run: |
cd ./vyos-build
echo "CHANGELOG_COMMIT_build=$(git log --since "${{ env.PREVIOUS_SUCCESS_BUILD_TIMESTAMP }}" --format="%H" --reverse | head -n1)" >> $GITHUB_ENV
cd ../vyos-1x
echo "CHANGELOG_COMMIT_1x=$(git log --since "${{ env.PREVIOUS_SUCCESS_BUILD_TIMESTAMP }}" --format="%H" --reverse | head -n1)" >> $GITHUB_ENV
env:
GH_TOKEN: ${{ github.token }}
- name: "Release publishing: generate changelog for vyos-1x"
if: ${{ !inputs.SKIP_RELEASE_PUBLISHING && env.CHANGELOG_COMMIT_1x != '' }}
id: generate_changelog_for_vyos-1x
uses: mikepenz/[email protected]
with:
owner: "vyos"
repo: "vyos-1x"
fetchReviewers: false
fromTag: ${{ env.CHANGELOG_COMMIT_1x }}
toTag: HEAD
configurationJson: |
{
"categories": [{"title": "", "labels": []}],
"template": "#{{CHANGELOG}}",
"pr_template": "- #{{TITLE}}\n - PR: vyos/vyos-1x##{{NUMBER}}"
}
- name: "Release publishing: generate changelog for vyos-build"
if: ${{ !inputs.SKIP_RELEASE_PUBLISHING && env.CHANGELOG_COMMIT_build }}
id: generate_changelog_for_vyos-build
uses: mikepenz/[email protected]
with:
owner: "vyos"
repo: "vyos-build"
fetchReviewers: false
fromTag: ${{ env.CHANGELOG_COMMIT_build }}
toTag: HEAD
configurationJson: |
{
"categories": [{"title": "", "labels": []}],
"template": "#{{CHANGELOG}}",
"pr_template": "- #{{TITLE}}\n - PR: vyos/vyos-build##{{NUMBER}}"
}
- name: "Release publishing: generate CHANGELOG.md"
id: generate_changelog_md
if: ${{ !inputs.SKIP_RELEASE_PUBLISHING }}
run: |
cat <<EOF > CHANGELOG.md
## vyos-1x
${{ steps.generate_changelog_for_vyos-1x.outputs.changelog }}
## vyos-build
${{ steps.generate-build-changelog.outputs.changelog }}
EOF
cat CHANGELOG.md
- name: "Release publishing: create version.json"
if: ${{ !inputs.SKIP_RELEASE_PUBLISHING }}
id: create_version_json
uses: jsdaniell/[email protected]
with:
name: "version.json"
json: |
[
{
"url": "https://github.com/vyos/vyos-rolling-nightly-builds/releases/download/${{ env.BUILD_VERSION }}/vyos-${{ env.BUILD_VERSION }}-amd64.iso",
"version": "${{ env.BUILD_VERSION }}",
"timestamp": "${{ env.TIMESTAMP }}"
}
]
- name: "Release publishing: update Minisign public key"
if: ${{ !inputs.SKIP_RELEASE_PUBLISHING }}
id: update_minisign_public_key
run: |
echo "${{ secrets.minisign_public_key }}" > minisign.pub
- name: "Release publishing: check if the repository was modified during runtime to prevent autocommit failure"
id: check_if_the_repository_was_modified_during_runtime
if: ${{ !inputs.SKIP_RELEASE_PUBLISHING }}
run: |
sudo chown -R $(whoami):$(whoami) ./*
git pull --autostash --rebase
- name: "Release publishing: create autocommit and tag"
id: create_autocommit_and_tag
if: ${{ !inputs.SKIP_RELEASE_PUBLISHING }}
uses: stefanzweifel/git-auto-commit-action@v5
with:
tagging_message: ${{ env.BUILD_VERSION }}
commit_message: ${{ env.BUILD_VERSION }}
- name: "Release publishing: publish release"
id: publish_release
if: ${{ !inputs.SKIP_RELEASE_PUBLISHING }}
uses: softprops/action-gh-release@v1
with:
body_path: CHANGELOG.md
tag_name: ${{ env.BUILD_VERSION }}
fail_on_unmatched_files: true
files: |
./vyos-${{ env.BUILD_VERSION }}-amd64.iso
./vyos-${{ env.BUILD_VERSION }}-amd64.iso.minisig
- name: "Release publishing: remove old releases"
id: remove_old_releases
if: ${{ !inputs.SKIP_RELEASE_PUBLISHING }}
uses: dev-drprasad/[email protected]
with:
keep_latest: 30
delete_tags: true
env:
GITHUB_TOKEN: ${{ secrets.CUSTOM_GITHUB_TOKEN }}
### Slack notification ###
- name: "Slack notification: identify a failed step"
id: identify_a_failed_step
if: ${{ failure() && !inputs.SKIP_SLACK_NOTIFICATIONS }}
run: |
echo FAILED_STEP=$((cat <<EOF
${{ toJson(steps) }}
EOF
) | jq -r 'to_entries | map(select(.value.conclusion == "failure")) | .[].key' | head -n1) >> "$GITHUB_OUTPUT"
- name: "Slack notification: download Smoketest failed logs"
id: download_smoketest_failed_logs
if: ${{ failure() && !inputs.SKIP_SLACK_NOTIFICATIONS && (steps.identify_a_failed_step.outputs.FAILED_STEP == 'smoketest_raid' || steps.identify_a_failed_step.outputs.FAILED_STEP == 'smoketest_system' || steps.identify_a_failed_step.outputs.FAILED_STEP == 'smoketest_configuration') }}
uses: nicklasfrahm/scp-action@main
with:
direction: download
host: ${{ secrets.SSH_HOST }}
username: ${{ secrets.SSH_USER }}
port: ${{ secrets.SSH_PORT }}
key: ${{ secrets.SSH_KEY }}
insecure_ignore_fingerprint: true
source: "~/vyos-build/${{ steps.identify_a_failed_step.outputs.FAILED_STEP }}.log"
target: "${{ github.workspace }}/${{ steps.identify_a_failed_step.outputs.FAILED_STEP }}.log"
- name: "Slack notification: filter Smoketest failed logs"
id: filter_smoketest_failed_logs
if: ${{ failure() && !inputs.SKIP_SLACK_NOTIFICATIONS && (steps.identify_a_failed_step.outputs.FAILED_STEP == 'smoketest_raid' || steps.identify_a_failed_step.outputs.FAILED_STEP == 'smoketest_system' || steps.identify_a_failed_step.outputs.FAILED_STEP == 'smoketest_configuration') }}
run: |
cat ${{ github.workspace }}/${{ steps.identify_a_failed_step.outputs.FAILED_STEP }}.log | grep '... FAIL' > ${{ github.workspace }}/smoketest_filtered.log || true
cat ${{ github.workspace }}/${{ steps.identify_a_failed_step.outputs.FAILED_STEP }}.log | grep 'ERROR -' >> ${{ github.workspace }}/smoketest_filtered.log || true
cat ${{ github.workspace }}/${{ steps.identify_a_failed_step.outputs.FAILED_STEP }}.log | grep 'Error:' -A12 >> ${{ github.workspace }}/smoketest_filtered.log || true
- name: "Slack notification: send to channel"
id: send_a_slack_failure_notification
if: ${{ failure() && !inputs.SKIP_SLACK_NOTIFICATIONS }}
uses: slackapi/[email protected]
with:
payload: |
{
"text": "*VyOS nightly build automation didn't pass the step with ID \"${{ steps.identify_a_failed_step.outputs.FAILED_STEP }}\"*",
"attachments": [
{
"color": "FF6600",
"fields": [
{
"title": "Repository",
"short": true,
"value": "${{ github.repository }}"
},
{
"title": "Workflow",
"short": true,
"value": "${{ github.workflow }}"
},
{
"title": "Trigger",
"short": true,
"value": "${{ github.event_name }}"
},
{
"title": "Run number",
"short": true,
"value": "${{ github.run_number }}"
},
{
"title": "URL",
"short": false,
"value": "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
}
]
}
]
}
env:
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK }}
SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK
- name: "Slack notification: send Smoketest log excerpt"
id: send_smoketest_log_excerpt_to_slack
if: ${{ failure() && !inputs.SKIP_SLACK_NOTIFICATIONS }}
uses: MeilCli/slack-upload-file@v4
with:
slack_token: ${{ secrets.SLACK_BOT_TOKEN }}
channel_id: ${{ secrets.SLACK_CHANNEL_ID }}
file_type: 'text'
file_path: ${{ github.workspace }}/smoketest_filtered.log
initial_comment: Smoketest log excerpt
if_no_files_found: ignore
### Update download page ###
update-downloads-page:
if: ${{ !inputs.SKIP_DOWNLOADS_PAGE_UPDATE }}
needs: build-iso
uses: vyos/community.vyos.net/.github/workflows/main.yml@production
with:
branch: production
secrets:
NETLIFY_TOKEN: ${{ secrets.NETLIFY_TOKEN }}
GH_ACCESS_TOKEN: ${{ secrets.GH_ACCESS_TOKEN }}