app-containers: switch to upstream docker, containerd, runc, bump docker to v24

.github/workflows/portage-stable-packages-list

@@ -87,6 +87,12 @@ app-arch/zstd
 
 app-cdr/cdrtools
 
+app-containers/cri-tools
+app-containers/docker
+app-containers/docker-cli
+app-containers/docker-proxy
+app-containers/runc
+
 app-crypt/adcli
 app-crypt/libb2
 app-crypt/libmd

build_image

@@ -33,8 +33,8 @@ DEFINE_string base_pkg "coreos-base/coreos" \
   "The base portage package to base the build off of (only applies to prod images)"
 DEFINE_string base_dev_pkg "coreos-base/coreos-dev" \
   "The base portage package to base the build off of (only applies to dev containers)"
-DEFINE_string base_sysexts "containerd-flatcar:app-containers/containerd,docker-flatcar:app-containers/docker" \
-  "Comma-separated list of name:package - build 'package' into sysext 'name', and include with OS image and update payload. Must be in order of dependencies, base sysexts come first."
+DEFINE_string base_sysexts "containerd-flatcar:app-containers/containerd,docker-flatcar:app-containers/docker&app-containers/docker-cli" \
+  "Comma-separated list of name:package[&package[&package]] - build 'package' (a single package or a list of packages separated by '&') into sysext 'name', and include with OS image and update payload. Must be in order of dependencies, base sysexts come first."
 DEFINE_string output_root "${DEFAULT_BUILD_ROOT}/images" \
   "Directory in which to place image result directories (named by version)"
 DEFINE_string disk_layout "" \

build_library/sysext_mangle_containerd-flatcar

@@ -3,5 +3,16 @@
 set -euo pipefail
 rootfs="${1}"
 
+
+# No manpages on Flatcar, no need to ship "stress" tool
+echo ">>> NOTICE: $0: removing 'gen-manpages', 'containerd-stress' from sysext"
+rm -f "${rootfs}/usr/bin/gen-manpages" "${rootfs}/usr/bin/containerd-stress"
+
+script_root="$(cd "$(dirname "$0")/../"; pwd)"
+files_dir="${script_root}/sdk_container/src/third_party/coreos-overlay/coreos/sysext/containerd"
+
+echo ">>> NOTICE $0: installing extra files from '${files_dir}'"
+cp -va "${files_dir}/"* "${rootfs}"
+
 mkdir -p "${rootfs}/usr/lib/systemd/system/multi-user.target.d"
 { echo "[Unit]"; echo "Upholds=containerd.service"; } > "${rootfs}/usr/lib/systemd/system/multi-user.target.d/10-containerd-service.conf"

build_library/sysext_mangle_docker-flatcar

@@ -3,5 +3,15 @@
 set -euo pipefail
 rootfs="${1}"
 
+# Remove debug and contrib
+echo ">>> NOTICE: $0: removing '/usr/lib/debug/', '/usr/share/docker/contrib' from sysext"
+rm -rf "${rootfs}/usr/lib/debug/" "${rootfs}/usr/share/docker/contrib/"
+
+script_root="$(cd "$(dirname "$0")/../"; pwd)"
+files_dir="${script_root}/sdk_container/src/third_party/coreos-overlay/coreos/sysext/docker"
+
+echo ">>> NOTICE $0: installing extra files from '${files_dir}'"
+cp -va "${files_dir}/"* "${rootfs}"
+
 mkdir -p "${rootfs}/usr/lib/systemd/system/sockets.target.d"
 { echo "[Unit]"; echo "Upholds=docker.socket"; } > "${rootfs}/usr/lib/systemd/system/sockets.target.d/10-docker-socket.conf"

build_library/sysext_prod_builder

@@ -30,17 +30,19 @@ create_prod_sysext() {
   local base_sysext="$4"
   local install_root="$5"
   local name="$6"
-  local grp_pkg="$7"
+  local grp_pkgs="$7"
   local pkginfo="${8:-}"
 
   local -a build_sysext_opts=()
 
-  local msg="Installing ${grp_pkg}' in sysext ${name}.raw"
+  local -a grp_pkg
+  mapfile -t grp_pkg <<<"${grp_pkgs//&/$'\n'}"
+  local msg="Installing ${grp_pkg[*]} in sysext ${name}.raw"
 
   # Include previous sysexts' pkginfo if supplied
   if [[ -n "${pkginfo}" ]] ; then
     if [[ ! -f "${output_dir}/${pkginfo}" ]] ; then
-      die "Sysext build '${grp_pkg}': unable to find package info at '${output_dir}/${pkginfo}'."
+      die "Sysext build '${name}': unable to find package info at '${output_dir}/${pkginfo}'."
     fi
     msg="${msg} w/ package info '${pkginfo}'"
     build_sysext_opts+=( "--base_pkginfo=${output_dir}/${pkginfo}" )
@@ -60,7 +62,7 @@ create_prod_sysext() {
             --squashfs_base="${base_sysext}" \
             --generate_pkginfo \
             "${build_sysext_opts[@]}" \
-            "${name}" "${grp_pkg}"
+            "${name}" "${grp_pkg[@]}"
 
   sudo mv "${workdir}/sysext-build/${name}.raw" "${workdir}/sysext-build/${name}_pkginfo.raw" \
                               "${workdir}/sysext-build/${name}"_*.txt "${output_dir}"

build_sysext

@@ -25,6 +25,8 @@ DEFINE_string squashfs_base '' \
   "The path to the squashfs base image. Defaults to the most current image built in '${default_imagedir}/${FLATCAR_PRODUCTION_IMAGE_SYSEXT_BASE}'."
 DEFINE_string image_builddir '' \
   "Custom directory to build the sysext in. Defaults to a 'sysext' sub-directory of the directory the squashfs base image resides in; '${default_imagedir}/sysext' by default."
+DEFINE_boolean strip_binaries "${FLAGS_FALSE}" \
+  "After installation, scan sysext root for unstripped binaries and strip these. WARNING - this can subtly break some packages, e.g. Docker (see https://github.com/moby/moby/blob/master/project/PACKAGERS.md#stripping-binaries)."
 DEFINE_string manglefs_script '' \
   "A path to executable that will customize the rootfs of the sysext image."
 DEFINE_boolean generate_pkginfo "${FLAGS_FALSE}" \
@@ -231,6 +233,22 @@ info "Writing ${SYSEXTNAME}_packages.txt"
 ROOT="${BUILD_DIR}/install-root" PORTAGE_CONFIGROOT="${BUILD_DIR}/install-root" \
       equery --no-color list --format '$cpv::$repo' '*' > "${BUILD_DIR}/${SYSEXTNAME}_packages.txt"
 
+
+if [[ "${FLAGS_strip_binaries}" = "${FLAGS_TRUE}" ]]; then
+    chost="$("portageq-${BOARD}" envvar CHOST)"
+    strip="${chost}-strip"
+
+    info "Stripping all non-stripped binaries in sysext using '${strip}'"
+
+    # Find all non-stripped binaries, remove ':' from filepath, and strip 'em
+    find "${BUILD_DIR}/install-root" -exec file \{\} \; \
+         | awk '/not stripped/ {print substr($1, 1, length($1)-1)}' \
+         | while read bin; do
+                info "     ${strip} ${bin}"
+                "${strip}" "${bin}"
+           done
+fi
+
 if [[ -n "${FLAGS_manglefs_script}" ]]; then
   if [[ ! -x "${FLAGS_manglefs_script}" ]]; then
     die "${FLAGS_manglefs_script} is not executable"

changelog/changes/2023-10-25-docker-gentoo-upstream.md

@@ -0,0 +1,6 @@
+- cri-tools, runc, containerd, docker, and docker-cli are now shipped without debugging symbols and built from Gentoo upstream ebuilds. Docker was updated to Docker 24 (see "updates").
+  - **NOTE** The docker btrfs storage driver has been de-prioritised; BTRFS backed storage will now default to the `overlay2` driver
+    ([changelog](https://docs.docker.com/engine/release-notes/23.0/#bug-fixes-and-enhancements-6), [upstream pr](https://github.com/moby/moby/pull/42661)).
+    Using the btrfs driver can still be enforced by creating a respective [docker config](https://docs.docker.com/storage/storagedriver/btrfs-driver/#configure-docker-to-use-the-btrfs-storage-driver) at `/etc/docker/daemon.json`.
+  - **NOTE** that if you are already using btrfs-backed Docker storage and are upgrading to this new version, Docker will automatically use the `btrfs` storage driver for backwards-compatibility with your deployment.
+    - **Docker will remove the `btrfs` driver entirely in a future version. Please consider migrating your deployments to the `overlay2` driver.**

changelog/updates/2023-10-25-docker-24-critools-1.27.md

@@ -0,0 +1,2 @@
+- docker ([24.0.6](https://docs.docker.com/engine/release-notes/24.0/), includes changes from [23.0](https://docs.docker.com/engine/release-notes/23.0/))
+- cri-tools ([1.27.0](https://github.com/kubernetes-sigs/cri-tools/releases/tag/v1.27.0))

ci-automation/vendor-testing/qemu_update.sh

@@ -99,6 +99,8 @@ run_kola_tests() {
         image="tmp/flatcar_production_image_previous.bin"
     elif [ "${instance_type}" = "first_dual" ]; then
         image="tmp/flatcar_production_image_first_dual.bin"
+        # Test docker 20 -> docker 24 migration btrfs storage driver backwards compatibility
+        tests+=("cl.update.docker-btrfs-compat")
         # Only run this test if the Azure dev payload exists on bincache because the fallback download needs it
         if curl --head -o /dev/null -fsSL --retry-delay 1 --retry 60 --retry-connrefused --retry-max-time 60 --connect-timeout 20 "https://bincache.flatcar-linux.net/images/${CIA_ARCH}/${CIA_VERNUM}/flatcar_test_update-oem-azure.gz"; then
           tests+=("cl.update.oem")