From d3ef256f2557de192d467ef233e6fadb9155a556 Mon Sep 17 00:00:00 2001 From: smerle33 Date: Fri, 17 May 2024 11:00:17 +0200 Subject: [PATCH 01/18] feat(git): compile git instead of relying on apt ppa repository --- provisioning/ubuntu-provision.sh | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/provisioning/ubuntu-provision.sh b/provisioning/ubuntu-provision.sh index a4d2649ab..d131a4926 100755 --- a/provisioning/ubuntu-provision.sh +++ b/provisioning/ubuntu-provision.sh @@ -281,17 +281,17 @@ Pin-Priority: 1001 apt-get install --yes chromium-browser } -## Install git and git-lfs -function install_git_gitlfs() { - if [ -n "${GIT_LINUX_VERSION}" ] - then - ## a specific git version is required: search it on the official git PPA repositories - add-apt-repository -y ppa:git-core/ppa - install_package_version git "${GIT_LINUX_VERSION}" - else - ## No git version: install the latest git available in the default repos - apt-get install --yes --no-install-recommends git - fi +## Compil git +function compile_git_install_gitlfs() { + apt-get update --quiet + apt-get install --yes --no-install-recommends dh-autoreconf libcurl4-gnutls-dev libexpat1-dev gettext libz-dev libssl-dev install-info make + + git_download_url="https://github.com/git/git/archive/refs/tags/v${GIT_LINUX_VERSION}.tar.gz" + curl --fail --silent --show-error --location "${git_download_url}" | \ + tar --extract --gunzip --directory="/tmp/" + cd /tmp/git-"${GIT_LINUX_VERSION}" + make prefix=/usr/local all + make prefix=/usr/local install ## Install git-lfs (after git) git_lfs_archive="git-lfs-linux-${ARCHITECTURE}-v${GIT_LFS_VERSION}.tar.gz" @@ -302,6 +302,7 @@ function install_git_gitlfs() { tar --extract --directory=/tmp/git-lfs --gzip --verbose --file="/tmp/${git_lfs_archive}" --strip-components=1 #strip the 1st-level directory of the archive as it has a changing name, since git-lfs 3.2.0. bash -x /tmp/git-lfs/install.sh # Execute in debug mode in case something goes wrong rm -rf /tmp/git-lfs* + rm -rf /tmp/git-"${GIT_LINUX_VERSION}" } # Reusable function to perform a Temurin JDK installations @@ -688,7 +689,7 @@ function main() { clean_apt install_common_requirements setuser # Define user Jenkins before all (to allow installing stuff in its home dir) - install_git_gitlfs + compile_git_install_gitlfs # ensure we use the lastest version with arm64 and amd/intel install_ssh_requirements # Ensure that OpenSSH CLI and SSH agent are installed install_asdf # Before all the others but after the jenkins home is created install_goss # needed by the pipeline From c84bc4952c2e879aa3f4cce29e2846b442c19cde Mon Sep 17 00:00:00 2001 From: smerle33 Date: Fri, 17 May 2024 16:05:58 +0200 Subject: [PATCH 02/18] move from erased folder --- provisioning/ubuntu-provision.sh | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/provisioning/ubuntu-provision.sh b/provisioning/ubuntu-provision.sh index d131a4926..0fb9dd95f 100755 --- a/provisioning/ubuntu-provision.sh +++ b/provisioning/ubuntu-provision.sh @@ -300,9 +300,11 @@ function compile_git_install_gitlfs() { curl --fail --silent --location --show-error --output "/tmp/${git_lfs_archive}" "${git_lfs_release_url}" mkdir -p /tmp/git-lfs tar --extract --directory=/tmp/git-lfs --gzip --verbose --file="/tmp/${git_lfs_archive}" --strip-components=1 #strip the 1st-level directory of the archive as it has a changing name, since git-lfs 3.2.0. - bash -x /tmp/git-lfs/install.sh # Execute in debug mode in case something goes wrong - rm -rf /tmp/git-lfs* - rm -rf /tmp/git-"${GIT_LINUX_VERSION}" + bash -x /tmp/git-lfs/install.sh + + cd /tmp # do not stay in a remove folder + rm -rf /tmp/git-lfs* # cleanup + rm -rf /tmp/git-"${GIT_LINUX_VERSION}" # cleanup } # Reusable function to perform a Temurin JDK installations From b2706010bafd378158c19fea244d6df91a2a7124 Mon Sep 17 00:00:00 2001 From: smerle33 Date: Tue, 21 May 2024 11:02:45 +0200 Subject: [PATCH 03/18] add some debug for git package --- provisioning/ubuntu-provision.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/provisioning/ubuntu-provision.sh b/provisioning/ubuntu-provision.sh index 0fb9dd95f..3e6cbe4bd 100755 --- a/provisioning/ubuntu-provision.sh +++ b/provisioning/ubuntu-provision.sh @@ -685,6 +685,9 @@ function cleanup() { } function main() { + echo "============================================================================== BEGIN Installed packages:" + dpkg -l + echo "============================================================================== BEGIN Installed packages" check_commands copy_custom_scripts set_locale # Define the locale @@ -733,8 +736,9 @@ function main() { install_sops install_yamllint - echo "== Installed packages:" + echo "============================================================================== END Installed packages:" dpkg -l + echo "============================================================================== END Installed packages" } main From f8f21b047c7313185e0d6a4de7f9443c19f780cf Mon Sep 17 00:00:00 2001 From: smerle33 Date: Tue, 21 May 2024 11:39:52 +0200 Subject: [PATCH 04/18] remove default installed packages --- provisioning/ubuntu-provision.sh | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/provisioning/ubuntu-provision.sh b/provisioning/ubuntu-provision.sh index 3e6cbe4bd..f5769dfd7 100755 --- a/provisioning/ubuntu-provision.sh +++ b/provisioning/ubuntu-provision.sh @@ -684,10 +684,25 @@ function cleanup() { sync } +## Remove package installed by default +function remove_unusefull_packages() { + for cli in git git-man + do + if command -v $cli >/dev/null 2>&1 + then + echo "removing ${cli} required but not found. Exiting." + dpkg –purge ${cli} + else + echo "cannot remove ${cli} as not installed" + fi + done +} + function main() { echo "============================================================================== BEGIN Installed packages:" dpkg -l echo "============================================================================== BEGIN Installed packages" + remove_unusefull_packages # Remove installed by default package useless or obsolete NEED to be at the start check_commands copy_custom_scripts set_locale # Define the locale From 97d025a1d3cb0b3e496f99ad3610d8b4eb3d6ccb Mon Sep 17 00:00:00 2001 From: smerle33 Date: Tue, 21 May 2024 11:41:20 +0200 Subject: [PATCH 05/18] double - --- provisioning/ubuntu-provision.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/provisioning/ubuntu-provision.sh b/provisioning/ubuntu-provision.sh index f5769dfd7..13955bad0 100755 --- a/provisioning/ubuntu-provision.sh +++ b/provisioning/ubuntu-provision.sh @@ -691,7 +691,7 @@ function remove_unusefull_packages() { if command -v $cli >/dev/null 2>&1 then echo "removing ${cli} required but not found. Exiting." - dpkg –purge ${cli} + dpkg -–purge ${cli} else echo "cannot remove ${cli} as not installed" fi From f2edc055eb3340ad653203f134ad3cd634a13f13 Mon Sep 17 00:00:00 2001 From: smerle33 Date: Tue, 21 May 2024 11:50:10 +0200 Subject: [PATCH 06/18] more debug --- provisioning/ubuntu-provision.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/provisioning/ubuntu-provision.sh b/provisioning/ubuntu-provision.sh index 13955bad0..9fd7deecb 100755 --- a/provisioning/ubuntu-provision.sh +++ b/provisioning/ubuntu-provision.sh @@ -690,8 +690,9 @@ function remove_unusefull_packages() { do if command -v $cli >/dev/null 2>&1 then - echo "removing ${cli} required but not found. Exiting." + echo "removing ${cli} (dpkg --purge ${cli})" dpkg -–purge ${cli} + echo "removed ${cli} (dpkg --purge ${cli})" else echo "cannot remove ${cli} as not installed" fi From 19a2c3aeca06887fa0108e5ee48e938ea78d60fd Mon Sep 17 00:00:00 2001 From: smerle33 Date: Tue, 21 May 2024 12:01:47 +0200 Subject: [PATCH 07/18] add debug --- provisioning/ubuntu-provision.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/provisioning/ubuntu-provision.sh b/provisioning/ubuntu-provision.sh index 9fd7deecb..1938bb54d 100755 --- a/provisioning/ubuntu-provision.sh +++ b/provisioning/ubuntu-provision.sh @@ -691,10 +691,13 @@ function remove_unusefull_packages() { if command -v $cli >/dev/null 2>&1 then echo "removing ${cli} (dpkg --purge ${cli})" - dpkg -–purge ${cli} + dpkg -l | grep "${cli}" + dpkg -–purge "${cli}" + dpkg -l | grep "${cli}" echo "removed ${cli} (dpkg --purge ${cli})" else echo "cannot remove ${cli} as not installed" + dpkg -l | grep "${cli}" fi done } From 9bb7c1bcddf565c805c87cf7a0dffbcdab15e3c4 Mon Sep 17 00:00:00 2001 From: smerle33 Date: Tue, 21 May 2024 14:11:14 +0200 Subject: [PATCH 08/18] use short parameter as long failed --- provisioning/ubuntu-provision.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/provisioning/ubuntu-provision.sh b/provisioning/ubuntu-provision.sh index 1938bb54d..f45e617e5 100755 --- a/provisioning/ubuntu-provision.sh +++ b/provisioning/ubuntu-provision.sh @@ -692,7 +692,7 @@ function remove_unusefull_packages() { then echo "removing ${cli} (dpkg --purge ${cli})" dpkg -l | grep "${cli}" - dpkg -–purge "${cli}" + dpkg -P "${cli}" ## --purge fail so using the short -P dpkg -l | grep "${cli}" echo "removed ${cli} (dpkg --purge ${cli})" else From 87cee8597ebb9bc9b9d8afd318c9ce22932955be Mon Sep 17 00:00:00 2001 From: smerle33 Date: Tue, 21 May 2024 14:25:31 +0200 Subject: [PATCH 09/18] shortcut not used build, limit to arm --- Jenkinsfile_k8s | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/Jenkinsfile_k8s b/Jenkinsfile_k8s index d591160de..9a33ad569 100644 --- a/Jenkinsfile_k8s +++ b/Jenkinsfile_k8s @@ -74,6 +74,7 @@ pipeline { } } } + /* stage('GC on Azure') { environment { PACKER_AZURE = credentials('packer-azure-serviceprincipal-sponsorship') @@ -90,6 +91,7 @@ pipeline { } } } + */ } } stage('Packer Images') { @@ -97,17 +99,19 @@ pipeline { axes { axis { name 'cpu_architecture' - values 'amd64', 'arm64' + // values 'amd64', 'arm64' + values 'arm64' } axis { name 'agent_type' // make sure to port any addition to the list of agent types to the Build Docker Manifest stage if it's docker related - values 'ubuntu-22.04', 'windows-2019', 'windows-2022' + values 'ubuntu-22.04' //, 'windows-2019', 'windows-2022' } axis { name 'compute_type' // "azure-arm" stands for "Azure Resource Manager", unrelated to arm64 CPU - values 'azure-arm', 'docker' + // values 'azure-arm', 'docker' + values 'docker' } } excludes { From fcc6062a8d923afb257f034aead439f3065054d2 Mon Sep 17 00:00:00 2001 From: smerle33 Date: Tue, 21 May 2024 14:25:55 +0200 Subject: [PATCH 10/18] comment and cleanup --- provisioning/ubuntu-provision.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/provisioning/ubuntu-provision.sh b/provisioning/ubuntu-provision.sh index f45e617e5..a97a30b52 100755 --- a/provisioning/ubuntu-provision.sh +++ b/provisioning/ubuntu-provision.sh @@ -690,11 +690,11 @@ function remove_unusefull_packages() { do if command -v $cli >/dev/null 2>&1 then - echo "removing ${cli} (dpkg --purge ${cli})" + echo "removing ${cli} (dpkg -P ${cli})" dpkg -l | grep "${cli}" dpkg -P "${cli}" ## --purge fail so using the short -P dpkg -l | grep "${cli}" - echo "removed ${cli} (dpkg --purge ${cli})" + echo "removed ${cli} (dpkg -P ${cli})" else echo "cannot remove ${cli} as not installed" dpkg -l | grep "${cli}" @@ -735,8 +735,8 @@ function main() { install_gh install_golang install_golangcilint # must come after golang - install_ruby ${RUBY_PUPPET_VERSION} - install_ruby ${RUBY_VERSION} + install_ruby "${RUBY_PUPPET_VERSION}" + install_ruby "${RUBY_VERSION}" install_vagrant install_xq install_yq From 1f8b7d34db322e8fc9ca66a15a4e5edf2c1f347d Mon Sep 17 00:00:00 2001 From: smerle33 Date: Tue, 21 May 2024 14:34:01 +0200 Subject: [PATCH 11/18] remove debug package list as inreadable --- provisioning/ubuntu-provision.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/provisioning/ubuntu-provision.sh b/provisioning/ubuntu-provision.sh index a97a30b52..39e490931 100755 --- a/provisioning/ubuntu-provision.sh +++ b/provisioning/ubuntu-provision.sh @@ -703,9 +703,6 @@ function remove_unusefull_packages() { } function main() { - echo "============================================================================== BEGIN Installed packages:" - dpkg -l - echo "============================================================================== BEGIN Installed packages" remove_unusefull_packages # Remove installed by default package useless or obsolete NEED to be at the start check_commands copy_custom_scripts From 4c0f2bd8ac54b57071e17b2606b4672980616139 Mon Sep 17 00:00:00 2001 From: smerle33 Date: Tue, 21 May 2024 14:42:17 +0200 Subject: [PATCH 12/18] debug errorcode --- provisioning/ubuntu-provision.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/provisioning/ubuntu-provision.sh b/provisioning/ubuntu-provision.sh index 39e490931..c2a3e59d5 100755 --- a/provisioning/ubuntu-provision.sh +++ b/provisioning/ubuntu-provision.sh @@ -694,10 +694,11 @@ function remove_unusefull_packages() { dpkg -l | grep "${cli}" dpkg -P "${cli}" ## --purge fail so using the short -P dpkg -l | grep "${cli}" - echo "removed ${cli} (dpkg -P ${cli})" + echo "removed ${cli} (dpkg -P ${cli}) errorcode $?" else echo "cannot remove ${cli} as not installed" dpkg -l | grep "${cli}" + echo "errorcode $?" fi done } From 4a04ab4dcd7c2f6195792cc1fd28720250a587ee Mon Sep 17 00:00:00 2001 From: smerle33 Date: Tue, 21 May 2024 14:42:45 +0200 Subject: [PATCH 13/18] temporarily remove failfast --- provisioning/ubuntu-provision.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/provisioning/ubuntu-provision.sh b/provisioning/ubuntu-provision.sh index c2a3e59d5..c767bacee 100755 --- a/provisioning/ubuntu-provision.sh +++ b/provisioning/ubuntu-provision.sh @@ -2,7 +2,7 @@ # This script provision the Linux Ubuntu 20 images for Jenkins agents # Architecture supported: amd64, arm64 -set -eux -o pipefail +set -ux -o pipefail ## Check for environment variables or fail miserably (due to set -u enabled) echo "== Provisiong jenkins-infra agent for ubuntu 20" From f20dbd6ece06ca72c431f35f241a55432fdb4f4c Mon Sep 17 00:00:00 2001 From: smerle33 Date: Tue, 21 May 2024 15:02:07 +0200 Subject: [PATCH 14/18] remove errorcode --- provisioning/ubuntu-provision.sh | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/provisioning/ubuntu-provision.sh b/provisioning/ubuntu-provision.sh index c767bacee..f3d8c49f6 100755 --- a/provisioning/ubuntu-provision.sh +++ b/provisioning/ubuntu-provision.sh @@ -691,14 +691,13 @@ function remove_unusefull_packages() { if command -v $cli >/dev/null 2>&1 then echo "removing ${cli} (dpkg -P ${cli})" - dpkg -l | grep "${cli}" + dpkg -l | grep "${cli}" 2>/dev/null dpkg -P "${cli}" ## --purge fail so using the short -P - dpkg -l | grep "${cli}" - echo "removed ${cli} (dpkg -P ${cli}) errorcode $?" + dpkg -l | grep "${cli}" 2>/dev/null + echo "removed ${cli} (dpkg -P ${cli})" else echo "cannot remove ${cli} as not installed" - dpkg -l | grep "${cli}" - echo "errorcode $?" + dpkg -l | grep "${cli}" 2>/dev/null fi done } From db96d5a9ee541f9a8a48a42c53295ab4a473980c Mon Sep 17 00:00:00 2001 From: smerle33 Date: Tue, 21 May 2024 15:17:47 +0200 Subject: [PATCH 15/18] remove exceptions --- Jenkinsfile_k8s | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/Jenkinsfile_k8s b/Jenkinsfile_k8s index 9a33ad569..d591160de 100644 --- a/Jenkinsfile_k8s +++ b/Jenkinsfile_k8s @@ -74,7 +74,6 @@ pipeline { } } } - /* stage('GC on Azure') { environment { PACKER_AZURE = credentials('packer-azure-serviceprincipal-sponsorship') @@ -91,7 +90,6 @@ pipeline { } } } - */ } } stage('Packer Images') { @@ -99,19 +97,17 @@ pipeline { axes { axis { name 'cpu_architecture' - // values 'amd64', 'arm64' - values 'arm64' + values 'amd64', 'arm64' } axis { name 'agent_type' // make sure to port any addition to the list of agent types to the Build Docker Manifest stage if it's docker related - values 'ubuntu-22.04' //, 'windows-2019', 'windows-2022' + values 'ubuntu-22.04', 'windows-2019', 'windows-2022' } axis { name 'compute_type' // "azure-arm" stands for "Azure Resource Manager", unrelated to arm64 CPU - // values 'azure-arm', 'docker' - values 'docker' + values 'azure-arm', 'docker' } } excludes { From 6dc426811c9a2671c0e2627a83b4c5c6d1e6d15e Mon Sep 17 00:00:00 2001 From: smerle33 Date: Tue, 21 May 2024 17:11:26 +0200 Subject: [PATCH 16/18] add back failfast and add a goss check on the git package --- goss/goss-linux.yaml | 3 +++ provisioning/ubuntu-provision.sh | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/goss/goss-linux.yaml b/goss/goss-linux.yaml index aed81270c..664fcfaf1 100644 --- a/goss/goss-linux.yaml +++ b/goss/goss-linux.yaml @@ -143,3 +143,6 @@ file: group: jenkins mode: '0750' owner: jenkins +package: + git: # the git package shouldn't be installed, as it is compiled + installed: false diff --git a/provisioning/ubuntu-provision.sh b/provisioning/ubuntu-provision.sh index f3d8c49f6..6ce5d831f 100755 --- a/provisioning/ubuntu-provision.sh +++ b/provisioning/ubuntu-provision.sh @@ -2,7 +2,7 @@ # This script provision the Linux Ubuntu 20 images for Jenkins agents # Architecture supported: amd64, arm64 -set -ux -o pipefail +set -eux -o pipefail ## Check for environment variables or fail miserably (due to set -u enabled) echo "== Provisiong jenkins-infra agent for ubuntu 20" From 1d26eae7bf57428687f251d88e8f9ef48ba0d82c Mon Sep 17 00:00:00 2001 From: smerle33 Date: Thu, 23 May 2024 09:40:20 +0200 Subject: [PATCH 17/18] cleanup --- provisioning/ubuntu-provision.sh | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/provisioning/ubuntu-provision.sh b/provisioning/ubuntu-provision.sh index 6ce5d831f..7c52eb5e6 100755 --- a/provisioning/ubuntu-provision.sh +++ b/provisioning/ubuntu-provision.sh @@ -445,7 +445,6 @@ function install_jxreleaseversion() { ## Ensure that azure-cli is installed function install_azurecli() { - local az_repo apt-get update --quiet apt-get install --yes --no-install-recommends \ gpg \ @@ -680,7 +679,7 @@ function install_yamllint() { ## Ensure that the VM is cleaned up of provision artifacts function cleanup() { export HISTSIZE=0 - rm -rf /tmp/* /var/log/* ${HOME}/.npm + rm -rf /tmp/* /var/log/* "${HOME}"/.npm sync } @@ -691,13 +690,10 @@ function remove_unusefull_packages() { if command -v $cli >/dev/null 2>&1 then echo "removing ${cli} (dpkg -P ${cli})" - dpkg -l | grep "${cli}" 2>/dev/null dpkg -P "${cli}" ## --purge fail so using the short -P - dpkg -l | grep "${cli}" 2>/dev/null echo "removed ${cli} (dpkg -P ${cli})" else echo "cannot remove ${cli} as not installed" - dpkg -l | grep "${cli}" 2>/dev/null fi done } From b5a1be8315e3c6b6c8714b68030c45e3f4e83cb0 Mon Sep 17 00:00:00 2001 From: smerle33 Date: Thu, 23 May 2024 13:57:12 +0200 Subject: [PATCH 18/18] remove removal of git as it is a strong dependency on ubuntu-server --- goss/goss-linux.yaml | 3 --- provisioning/ubuntu-provision.sh | 16 ---------------- 2 files changed, 19 deletions(-) diff --git a/goss/goss-linux.yaml b/goss/goss-linux.yaml index 664fcfaf1..aed81270c 100644 --- a/goss/goss-linux.yaml +++ b/goss/goss-linux.yaml @@ -143,6 +143,3 @@ file: group: jenkins mode: '0750' owner: jenkins -package: - git: # the git package shouldn't be installed, as it is compiled - installed: false diff --git a/provisioning/ubuntu-provision.sh b/provisioning/ubuntu-provision.sh index 7c52eb5e6..774966214 100755 --- a/provisioning/ubuntu-provision.sh +++ b/provisioning/ubuntu-provision.sh @@ -683,23 +683,7 @@ function cleanup() { sync } -## Remove package installed by default -function remove_unusefull_packages() { - for cli in git git-man - do - if command -v $cli >/dev/null 2>&1 - then - echo "removing ${cli} (dpkg -P ${cli})" - dpkg -P "${cli}" ## --purge fail so using the short -P - echo "removed ${cli} (dpkg -P ${cli})" - else - echo "cannot remove ${cli} as not installed" - fi - done -} - function main() { - remove_unusefull_packages # Remove installed by default package useless or obsolete NEED to be at the start check_commands copy_custom_scripts set_locale # Define the locale