From 29bf037d27a4542f565a08f4319baf28687cc603 Mon Sep 17 00:00:00 2001 From: YZ775 Date: Mon, 11 Nov 2024 07:05:34 +0000 Subject: [PATCH 1/5] add ss2 to cke-template and neco-rebooter --- etc/cke-template.yml | 9 +++++++++ etc/neco-rebooter.yaml | 7 +++++++ 2 files changed, 16 insertions(+) diff --git a/etc/cke-template.yml b/etc/cke-template.yml index 0f11422af..940525714 100644 --- a/etc/cke-template.yml +++ b/etc/cke-template.yml @@ -18,6 +18,15 @@ nodes: - key: cke.cybozu.com/role value: storage effect: NoSchedule +- user: cybozu + control_plane: false + labels: + cke.cybozu.com/role: "ss2" + cke.cybozu.com/weight: "1" + taints: + - key: cke.cybozu.com/role + value: storage + effect: NoSchedule service_subnet: 10.68.0.0/16 dns_service: internet-egress/unbound control_plane_tolerations: ["node.cilium.io/agent-not-ready"] diff --git a/etc/neco-rebooter.yaml b/etc/neco-rebooter.yaml index f78223222..0e4b79880 100644 --- a/etc/neco-rebooter.yaml +++ b/etc/neco-rebooter.yaml @@ -18,5 +18,12 @@ rebootTimes: times: allow: - "* 0-23 * * 1-5" + - name: ss2 + labelSelector: + matchLabels: + cke.cybozu.com/role: ss2 + times: + allow: + - "* 0-23 * * 1-5" timeZone: Asia/Tokyo groupLabelKey: topology.kubernetes.io/zone From 5d888fab952514b6c38bb46a3ed5adf38561061e Mon Sep 17 00:00:00 2001 From: YZ775 Date: Thu, 14 Nov 2024 04:55:44 +0000 Subject: [PATCH 2/5] add ignition for ss2 and change setup-var --- ignition-template/settings.json | 30 +++++++++++++++ ignition-template/site-env/main.libsonnet | 4 +- ignition-template/site-env/ss2/main.libsonnet | 5 +++ ignition-template/utility.libsonnet | 4 ++ ignitions/common/files/opt/sbin/setup-var | 26 +++++++++---- .../roles/ss2/files/etc/udev/crypt-base-path | 37 +++++++++++++++++++ .../ss2/files/etc/udev/rules.d/99-neco.rules | 8 ++++ ignitions/roles/ss2/site-gcp0.yml | 5 +++ ignitions/roles/ss2/site-stage0.yml | 5 +++ ignitions/roles/ss2/site-stage1.yml | 5 +++ ignitions/roles/ss2/site.yml | 5 +++ 11 files changed, 125 insertions(+), 9 deletions(-) create mode 100644 ignition-template/site-env/ss2/main.libsonnet create mode 100755 ignitions/roles/ss2/files/etc/udev/crypt-base-path create mode 100644 ignitions/roles/ss2/files/etc/udev/rules.d/99-neco.rules create mode 100644 ignitions/roles/ss2/site-gcp0.yml create mode 100644 ignitions/roles/ss2/site-stage0.yml create mode 100644 ignitions/roles/ss2/site-stage1.yml create mode 100644 ignitions/roles/ss2/site.yml diff --git a/ignition-template/settings.json b/ignition-template/settings.json index 4a853b0d8..d30773491 100644 --- a/ignition-template/settings.json +++ b/ignition-template/settings.json @@ -109,6 +109,36 @@ ] } }, + "ss2": { + "base": { + "include": "../../common/common.yml", + "files": [ + "/etc/udev/crypt-base-path", + "/etc/udev/rules.d/99-neco.rules" + ] + }, + "gcp0": { + "include": "../../common/common-gcp0.yml", + "files": [ + "/etc/udev/crypt-base-path", + "/etc/udev/rules.d/99-neco.rules" + ] + }, + "stage0": { + "include": "../../common/common-stage0.yml", + "files": [ + "/etc/udev/crypt-base-path", + "/etc/udev/rules.d/99-neco.rules" + ] + }, + "stage1": { + "include": "../../common/common-stage1.yml", + "files": [ + "/etc/udev/crypt-base-path", + "/etc/udev/rules.d/99-neco.rules" + ] + } + }, "common": { "common": { "passwd": "passwd.yml", diff --git a/ignition-template/site-env/main.libsonnet b/ignition-template/site-env/main.libsonnet index a881b4f1b..b9d17b45a 100644 --- a/ignition-template/site-env/main.libsonnet +++ b/ignition-template/site-env/main.libsonnet @@ -1,8 +1,10 @@ local boot_template = import 'boot/main.libsonnet'; local cs_template = import 'cs/main.libsonnet'; local ss_template = import 'ss/main.libsonnet'; +local ss2_template = import 'ss2/main.libsonnet'; local utility = import '../utility.libsonnet'; function(settings) utility.prefix_file_names('boot', boot_template(settings)) + utility.prefix_file_names('cs', cs_template(settings)) + - utility.prefix_file_names('ss', ss_template(settings)) + utility.prefix_file_names('ss', ss_template(settings)) + + utility.prefix_file_names('ss2', ss2_template(settings)) diff --git a/ignition-template/site-env/ss2/main.libsonnet b/ignition-template/site-env/ss2/main.libsonnet new file mode 100644 index 000000000..d91e58ff3 --- /dev/null +++ b/ignition-template/site-env/ss2/main.libsonnet @@ -0,0 +1,5 @@ +local config_template = import '../../config.libsonnet'; +local utility = import '../../utility.libsonnet'; + +function(settings) + utility.union_map(std.map(function(x) { [if x=="base" then 'site.yml' else 'site-'+x +'.yml']: config_template(settings.ss2[x]) }, utility.get_ss2(settings))) diff --git a/ignition-template/utility.libsonnet b/ignition-template/utility.libsonnet index 9845cfffb..7ca630d70 100644 --- a/ignition-template/utility.libsonnet +++ b/ignition-template/utility.libsonnet @@ -49,6 +49,10 @@ // get_ss retrives the array of ss from settings. get_ss(settings):: std.objectFields(settings.ss), + + // get_ss2 retrives the array of ss from settings. + get_ss2(settings):: + std.objectFields(settings.ss2), // get_common retrives the array of common from settings. get_common(settings):: diff --git a/ignitions/common/files/opt/sbin/setup-var b/ignitions/common/files/opt/sbin/setup-var index e08c0ede3..fcc543a48 100755 --- a/ignitions/common/files/opt/sbin/setup-var +++ b/ignitions/common/files/opt/sbin/setup-var @@ -84,15 +84,25 @@ if ls /dev/mapper/crypt-vd* >/dev/null 2>&1; then PVS=$(ls /dev/mapper/crypt-vd[abc]) prepare_lv elif ls /dev/mapper/crypt-nvme* >/dev/null 2>&1; then - # for compute node - PVS=$(ls /dev/mapper/crypt-nvme*) - CONTAINERD_SIZE=1t - DOCKER_SIZE=50g - KUBELET_SIZE=100g - COREDUMP_SIZE=50g - prepare_lv + if [-n "$(find_boss)"]; then + # for ss2 node + PVS=$(find_boss) + CONTAINERD_SIZE=50g + DOCKER_SIZE=50g + KUBELET_SIZE=10g + COREDUMP_SIZE=20g + prepare_lv + else + # for compute node + PVS=$(ls /dev/mapper/crypt-nvme*) + CONTAINERD_SIZE=1t + DOCKER_SIZE=50g + KUBELET_SIZE=100g + COREDUMP_SIZE=50g + prepare_lv + fi else - # for storage node + # for ss1 node PVS=$(find_boss) CONTAINERD_SIZE=50g DOCKER_SIZE=50g diff --git a/ignitions/roles/ss2/files/etc/udev/crypt-base-path b/ignitions/roles/ss2/files/etc/udev/crypt-base-path new file mode 100755 index 000000000..08199f91b --- /dev/null +++ b/ignitions/roles/ss2/files/etc/udev/crypt-base-path @@ -0,0 +1,37 @@ +#!/bin/sh + +SCRIPT_NAME=$(basename $0) +NAME=$1 +UUID=$2 + +output_log() { + logger "$SCRIPT_NAME $NAME $1" +} + +CRYPT_DEV=$(dmsetup deps -u $UUID -o devname | sed -e 's/.*(\(.*\))/\1/') +UDEV_LINKS=$(udevadm info -q symlink -n /dev/$CRYPT_DEV) + +output_log "CRYPT_DEV: $CRYPT_DEV DISK_DEV:$DISK_DEV" +output_log "UDEV_LINKS:$UDEV_LINKS" + +for symlink in $UDEV_LINKS; do + case ${symlink} in + disk/by-path/virtio-*) + ;; + disk/by-path/*-ata-*) + ;; + disk/by-path/*) + output_log "find a symlink:$symlink" + CRYPT_BASE_PATH=${symlink#disk/by-path/} + ;; + esac +done + +if [ -z "$CRYPT_BASE_PATH" ]; then + output_log "could not find symlinks" + exit 1 +fi + +mkdir -p /dev/crypt-disk/by-path/ +echo "CRYPT_BASE_PATH=$CRYPT_BASE_PATH" +exit 0 diff --git a/ignitions/roles/ss2/files/etc/udev/rules.d/99-neco.rules b/ignitions/roles/ss2/files/etc/udev/rules.d/99-neco.rules new file mode 100644 index 000000000..d57019556 --- /dev/null +++ b/ignitions/roles/ss2/files/etc/udev/rules.d/99-neco.rules @@ -0,0 +1,8 @@ +KERNEL!="dm-*", GOTO="neco_end" +ENV{DM_NAME}!="crypt-*", GOTO="neco_end" +# Skip the qemu's system disks +ENV{DM_NAME}=="crypt-vda|crypt-vdb|crypt-vdc", GOTO="neco_end" + +IMPORT{program}="/etc/udev/crypt-base-path $name $env{DM_UUID}", SYMLINK+="crypt-disk/by-path/$env{CRYPT_BASE_PATH}" + +LABEL="neco_end" diff --git a/ignitions/roles/ss2/site-gcp0.yml b/ignitions/roles/ss2/site-gcp0.yml new file mode 100644 index 000000000..4bbb29b05 --- /dev/null +++ b/ignitions/roles/ss2/site-gcp0.yml @@ -0,0 +1,5 @@ +# This file is automatically generated from a template. Please do not edit. +include: ../../common/common-gcp0.yml +files: + - /etc/udev/crypt-base-path + - /etc/udev/rules.d/99-neco.rules diff --git a/ignitions/roles/ss2/site-stage0.yml b/ignitions/roles/ss2/site-stage0.yml new file mode 100644 index 000000000..87524f150 --- /dev/null +++ b/ignitions/roles/ss2/site-stage0.yml @@ -0,0 +1,5 @@ +# This file is automatically generated from a template. Please do not edit. +include: ../../common/common-stage0.yml +files: + - /etc/udev/crypt-base-path + - /etc/udev/rules.d/99-neco.rules diff --git a/ignitions/roles/ss2/site-stage1.yml b/ignitions/roles/ss2/site-stage1.yml new file mode 100644 index 000000000..e61410bc2 --- /dev/null +++ b/ignitions/roles/ss2/site-stage1.yml @@ -0,0 +1,5 @@ +# This file is automatically generated from a template. Please do not edit. +include: ../../common/common-stage1.yml +files: + - /etc/udev/crypt-base-path + - /etc/udev/rules.d/99-neco.rules diff --git a/ignitions/roles/ss2/site.yml b/ignitions/roles/ss2/site.yml new file mode 100644 index 000000000..46aa9ad1b --- /dev/null +++ b/ignitions/roles/ss2/site.yml @@ -0,0 +1,5 @@ +# This file is automatically generated from a template. Please do not edit. +include: ../../common/common.yml +files: + - /etc/udev/crypt-base-path + - /etc/udev/rules.d/99-neco.rules From a2ae45037b2b0e8d58149997ec21efb3268f8b19 Mon Sep 17 00:00:00 2001 From: YZ775 Date: Fri, 8 Nov 2024 08:48:14 +0000 Subject: [PATCH 3/5] add "ss2" sabakan role --- pkg/generate-sabakan-machines/cmd/root.go | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/pkg/generate-sabakan-machines/cmd/root.go b/pkg/generate-sabakan-machines/cmd/root.go index 692d51e7e..caea0755f 100644 --- a/pkg/generate-sabakan-machines/cmd/root.go +++ b/pkg/generate-sabakan-machines/cmd/root.go @@ -19,7 +19,7 @@ var rootCmd = &cobra.Command{ Short: "generate machines.json", Long: `Generate machines.json from the CSV file for use with the 'sabactl create' command. Example: - generate-sabakan-machines input.csv --machine-type-boot=r6525-boot-1 --machine-type-cs=r6525-cs-1 --machine-type-ss=r7525-ss-1 + generate-sabakan-machines input.csv --machine-type-boot=r6525-boot-1 --machine-type-cs=r6525-cs-1 --machine-type-ss=r7525-ss-1 --machine-type-ss2=r7525-ss-2 `, Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { @@ -66,6 +66,8 @@ Example: machineType = *machineTypeCS case "ss": machineType = *machineTypeSS + case "ss2": + machineType = *machineTypeSS2 default: return errors.New("unknown role " + role) } @@ -124,16 +126,19 @@ var ( machineTypeBoot *string machineTypeCS *string machineTypeSS *string + machineTypeSS2 *string bmcType *string ) func init() { - machineTypeBoot = rootCmd.Flags().StringP("machine-type-boot", "b", "", "The machine-type name of Boot") - machineTypeCS = rootCmd.Flags().StringP("machine-type-cs", "c", "", "The machine-type name of CS") - machineTypeSS = rootCmd.Flags().StringP("machine-type-ss", "s", "", "The machine-type name of SS") + machineTypeBoot = rootCmd.Flags().String("machine-type-boot", "", "The machine-type name of Boot") + machineTypeCS = rootCmd.Flags().String("machine-type-cs", "", "The machine-type name of CS") + machineTypeSS = rootCmd.Flags().String("machine-type-ss", "", "The machine-type name of SS") + machineTypeSS2 = rootCmd.Flags().String("machine-type-ss2", "", "The machine-type name of SS2") bmcType = rootCmd.Flags().String("bmc-type", "iDRAC", "The name of BMC") rootCmd.MarkFlagRequired("machine-type-boot") rootCmd.MarkFlagRequired("machine-type-cs") rootCmd.MarkFlagRequired("machine-type-ss") + rootCmd.MarkFlagRequired("machine-type-ss2") } From 5de2b8ba48e04239908095621777aef9eddf6ffc Mon Sep 17 00:00:00 2001 From: YZ775 Date: Fri, 22 Nov 2024 00:29:01 +0000 Subject: [PATCH 4/5] reflect comments --- ignition-template/utility.libsonnet | 2 +- ignitions/common/files/opt/sbin/setup-var | 30 ++++++++--------------- 2 files changed, 11 insertions(+), 21 deletions(-) diff --git a/ignition-template/utility.libsonnet b/ignition-template/utility.libsonnet index 7ca630d70..485126332 100644 --- a/ignition-template/utility.libsonnet +++ b/ignition-template/utility.libsonnet @@ -50,7 +50,7 @@ get_ss(settings):: std.objectFields(settings.ss), - // get_ss2 retrives the array of ss from settings. + // get_ss2 retrives the array of ss2 from settings. get_ss2(settings):: std.objectFields(settings.ss2), diff --git a/ignitions/common/files/opt/sbin/setup-var b/ignitions/common/files/opt/sbin/setup-var index fcc543a48..0465ed9e5 100755 --- a/ignitions/common/files/opt/sbin/setup-var +++ b/ignitions/common/files/opt/sbin/setup-var @@ -83,32 +83,22 @@ if ls /dev/mapper/crypt-vd* >/dev/null 2>&1; then # See https://github.com/cybozu-go/neco/blob/cf1fbc99aeb62d37e37c5c2c69f12e319622b6cd/ignitions/roles/ss/files/etc/udev/rules.d/99-neco.rules#L4 PVS=$(ls /dev/mapper/crypt-vd[abc]) prepare_lv -elif ls /dev/mapper/crypt-nvme* >/dev/null 2>&1; then - if [-n "$(find_boss)"]; then - # for ss2 node - PVS=$(find_boss) - CONTAINERD_SIZE=50g - DOCKER_SIZE=50g - KUBELET_SIZE=10g - COREDUMP_SIZE=20g - prepare_lv - else - # for compute node - PVS=$(ls /dev/mapper/crypt-nvme*) - CONTAINERD_SIZE=1t - DOCKER_SIZE=50g - KUBELET_SIZE=100g - COREDUMP_SIZE=50g - prepare_lv - fi -else - # for ss1 node +elif [ -n "$(find_boss)" ]; then + # for ss, ss2 node PVS=$(find_boss) CONTAINERD_SIZE=50g DOCKER_SIZE=50g KUBELET_SIZE=10g COREDUMP_SIZE=20g prepare_lv +else + # for cs node + PVS=$(ls /dev/mapper/crypt-nvme*) + CONTAINERD_SIZE=1t + DOCKER_SIZE=50g + KUBELET_SIZE=100g + COREDUMP_SIZE=50g + prepare_lv fi for label in $LVLIST; do From 77a6185fec4ab06b5689cd296c52edfeeb780749 Mon Sep 17 00:00:00 2001 From: YZ775 Date: Tue, 26 Nov 2024 07:48:33 +0000 Subject: [PATCH 5/5] downgrade flatcar to 3975.2.2 --- artifacts.go | 6 +++--- artifacts_ignore.yaml | 3 +++ 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/artifacts.go b/artifacts.go index b9fc6f4b7..08841f93a 100644 --- a/artifacts.go +++ b/artifacts.go @@ -9,8 +9,8 @@ var CurrentArtifacts = ArtifactSet{ {Name: "bird", Repository: "ghcr.io/cybozu/bird", Tag: "2.15.1.1", Private: false}, {Name: "chrony", Repository: "ghcr.io/cybozu/chrony", Tag: "4.6.1.1", Private: false}, {Name: "etcd", Repository: "ghcr.io/cybozu/etcd", Tag: "3.5.14.1", Private: false}, - {Name: "promtail", Repository: "ghcr.io/cybozu/promtail", Tag: "3.2.1.1", Private: false}, - {Name: "sabakan", Repository: "ghcr.io/cybozu-go/sabakan", Tag: "3.1.2", Private: false}, + {Name: "promtail", Repository: "ghcr.io/cybozu/promtail", Tag: "3.2.1.2", Private: false}, + {Name: "sabakan", Repository: "ghcr.io/cybozu-go/sabakan", Tag: "3.1.4", Private: false}, {Name: "serf", Repository: "ghcr.io/cybozu/serf", Tag: "0.10.1.7", Private: false}, {Name: "setup-hw", Repository: "ghcr.io/cybozu-go/setup-hw", Tag: "1.17.1", Private: true}, {Name: "squid", Repository: "ghcr.io/cybozu/squid", Tag: "6.10.0.1", Private: false}, @@ -24,5 +24,5 @@ var CurrentArtifacts = ArtifactSet{ Debs: []DebianPackage{ {Name: "etcdpasswd", Owner: "cybozu-go", Repository: "etcdpasswd", Release: "v1.4.9"}, }, - OSImage: OSImage{Channel: "stable", Version: "4081.2.0"}, + OSImage: OSImage{Channel: "stable", Version: "3975.2.2"}, } diff --git a/artifacts_ignore.yaml b/artifacts_ignore.yaml index 7f6aca5ad..3456f5ace 100644 --- a/artifacts_ignore.yaml +++ b/artifacts_ignore.yaml @@ -5,3 +5,6 @@ images: versions: ["6.12.0.1"] - repository: ghcr.io/cybozu/squid-exporter versions: ["1.0.7"] +osImage: +- channel: stable + versions: ["4081.2.0"]