From 83b0a0fc16f0c81d05bb2de66d53df2ad1c88c0e Mon Sep 17 00:00:00 2001 From: Siegfried Weber Date: Mon, 31 Jul 2023 14:34:29 +0200 Subject: [PATCH] Upgrade Vector in all product images to version 0.31.0 (#429) * Upgrade Vector in all product images to version 0.31.0 * Use RPM for Vector * Remove unused AIRFLOW_UID * Use Vector RPM from Stackable Nexus * Assign changelog entries to the corresponding SDP releases * Move entries from vector/CHANGELOG.md to CHANGELOG.md * Update changelog with #429 --- CHANGELOG.md | 65 ++++++++++++++++++++++------- airflow/Dockerfile | 1 - image_tools/conf.py | 62 ++++++++++++++------------- vector/CHANGELOG.md | 20 --------- vector/Dockerfile | 36 ++++++---------- vector/upload_new_vector_version.sh | 31 +++++++++----- 6 files changed, 117 insertions(+), 98 deletions(-) delete mode 100644 vector/CHANGELOG.md diff --git a/CHANGELOG.md b/CHANGELOG.md index baef6082b..7e3be4918 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,35 +4,70 @@ All notable changes to this project will be documented in this file. ## [Unreleased] +### Changed + +- Bump ubi8-rust-builder toolchain to 1.71.0 ([#419]). +- BREAKING: Upgrade Vector in all product images to version 0.31.0. The + integration tests of the operators must be adapted because the metric + `processedEventsTotal` was replaced by `receivedEventsTotal` ([#429]). +- BREAKING: Use RPM instead of tar.gz for Vector. Because of that, the + location of the Vector executable changed, and the operator-rs version + 0.45.0 is required ([#429]). + +### Removed + +- Remove unused environment variable `AIRFLOW_UID` from the Airflow + image ([#429]). + +[#419]: https://github.com/stackabletech/docker-images/pull/419 +[#429]: https://github.com/stackabletech/docker-images/pull/429 + +## [23.7.0] - 2023-07-14 + +## [23.4.0] - 2023-04-17 + ### Added -- Added labels for openshift requirements and licenses for all images ([#95]). -- Added `RELEASE` ARG for all images ([#95]). +- Package inotify-tools added ([#291]). - Added krb5 image ([#338]). ### Changed -- Updated java base image to latest ubi8 tag 8.6-994 ([#249]). -- Updated all java-base images to stackable0.2.2 ([#250]). -- Updated all ubi8 base images to latest (8.6-994) ([#250]). -- Updated all internal images to rebuild their base images on demand ([#321]). +- Updated all internal images to rebuild their base images on demand + ([#321]). - Unpinned testing-tools dependencies ([#326]). -- Bump ubi8-rust-builder toolchain to 1.71.0 ([#419]). ### Removed -- Prometheus image, NodeExporter image, Antora ([#95]). -- Retired Java 1.8.0 support ([#248]). - Tools image ([#325]). -- Replace `build_product_images.py` with the `image_tools` package and add OpenShift preflight checks for images ([#339]) +- Replace `build_product_images.py` with the `image_tools` package and + add OpenShift preflight checks for images ([#339]) -[#95]: https://github.com/stackabletech/docker-images/pull/95 -[#248]: https://github.com/stackabletech/docker-images/pull/248 -[#249]: https://github.com/stackabletech/docker-images/pull/249 -[#250]: https://github.com/stackabletech/docker-images/pull/250 +[#291]: https://github.com/stackabletech/docker-images/pull/291 [#321]: https://github.com/stackabletech/docker-images/pull/321 [#325]: https://github.com/stackabletech/docker-images/pull/325 [#326]: https://github.com/stackabletech/docker-images/pull/326 [#338]: https://github.com/stackabletech/docker-images/pull/338 [#339]: https://github.com/stackabletech/docker-images/pull/339 -[#419]: https://github.com/stackabletech/docker-images/pull/419 + +## [23.1.0] - 2023-01-23 + +### Added + +- Image vector added which installs Vector and is based on the + stackable-base image. ([#268]). + +### Changed + +- Updated java base image to latest ubi8 tag 8.6-994 ([#249]). +- Updated all java-base images to stackable0.2.2 ([#250]). +- Updated all ubi8 base images to latest (8.6-994) ([#250]). + +### Removed + +- Retired Java 1.8.0 support ([#248]). + +[#248]: https://github.com/stackabletech/docker-images/pull/248 +[#249]: https://github.com/stackabletech/docker-images/pull/249 +[#250]: https://github.com/stackabletech/docker-images/pull/250 +[#268]: https://github.com/stackabletech/docker-images/pull/268 diff --git a/airflow/Dockerfile b/airflow/Dockerfile index 10e35b83a..f000b6d47 100644 --- a/airflow/Dockerfile +++ b/airflow/Dockerfile @@ -68,7 +68,6 @@ RUN microdnf install -y yum python${PYTHON} \ ENV HOME=/stackable ENV AIRFLOW_USER_HOME_DIR=/stackable ENV PATH=$PATH:/bin:$HOME/app/bin -ENV AIRFLOW_UID="1000" ENV AIRFLOW_HOME=$HOME/airflow RUN mkdir -pv ${AIRFLOW_HOME} && \ diff --git a/image_tools/conf.py b/image_tools/conf.py index 44f750348..5036369fd 100644 --- a/image_tools/conf.py +++ b/image_tools/conf.py @@ -9,27 +9,27 @@ { "product": "2.2.3", "python": "38", - "vector": "0.26.0", + "vector": "0.31.0", }, { "product": "2.2.4", "python": "39", - "vector": "0.26.0", + "vector": "0.31.0", }, { "product": "2.2.5", "python": "39", - "vector": "0.26.0", + "vector": "0.31.0", }, { "product": "2.4.1", "python": "39", - "vector": "0.26.0", + "vector": "0.31.0", }, { "product": "2.6.1", "python": "39", - "vector": "0.26.0", + "vector": "0.31.0", }, ], }, @@ -138,12 +138,12 @@ "versions": [ { "product": "11", - "vector": "0.26.0", + "vector": "0.31.0", "_security_path": "/usr/lib/jvm/jre-11/conf/security/java.security", }, { "product": "17", - "vector": "0.26.0", + "vector": "0.31.0", "_security_path": "/usr/lib/jvm/jre-17/conf/security/java.security", }, ], @@ -201,7 +201,13 @@ }, { "name": "vector", - "versions": [{"product": "0.26.0", "stackable-base": "1.0.0"}], + "versions": [ + { + "product": "0.31.0", + "rpm_release": "1", + "stackable-base": "1.0.0" + } + ], }, { "name": "nifi", @@ -218,32 +224,32 @@ "versions": [ { "product": "0.27.1", - "vector": "0.26.0", + "vector": "0.31.0", "bundle_builder_version": "1.0.0", }, { "product": "0.28.0", - "vector": "0.26.0", + "vector": "0.31.0", "bundle_builder_version": "1.0.0", }, { "product": "0.37.2", - "vector": "0.26.0", + "vector": "0.31.0", "bundle_builder_version": "1.0.0", }, { "product": "0.41.0", - "vector": "0.26.0", + "vector": "0.31.0", "bundle_builder_version": "1.0.0", }, { "product": "0.45.0", - "vector": "0.26.0", + "vector": "0.31.0", "bundle_builder_version": "1.0.0", }, { "product": "0.51.0", - "vector": "0.26.0", + "vector": "0.31.0", "bundle_builder_version": "1.1.0", }, ], @@ -265,7 +271,7 @@ "jackson_dataformat_xml": "2.12.3", "stax2_api": "4.2.1", "woodstox_core": "6.2.1", - "vector": "0.26.0", + "vector": "0.31.0", }, { "product": "3.3.0", @@ -281,7 +287,7 @@ "jackson_dataformat_xml": "2.13.3", "stax2_api": "4.2.1", "woodstox_core": "6.2.1", - "vector": "0.26.0", + "vector": "0.31.0", }, { "product": "3.3.0-java17", @@ -297,7 +303,7 @@ "jackson_dataformat_xml": "2.13.3", "stax2_api": "4.2.1", "woodstox_core": "6.2.1", - "vector": "0.26.0", + "vector": "0.31.0", }, { "product": "3.4.0", @@ -313,7 +319,7 @@ "jackson_dataformat_xml": "2.14.2", "stax2_api": "4.2.1", "woodstox_core": "6.5.0", - "vector": "0.26.0", + "vector": "0.31.0", }, { "product": "3.4.0-java17", @@ -329,7 +335,7 @@ "jackson_dataformat_xml": "2.14.2", "stax2_api": "4.2.1", "woodstox_core": "6.5.0", - "vector": "0.26.0", + "vector": "0.31.0", }, ], }, @@ -347,7 +353,7 @@ "jackson_dataformat_xml": "2.13.3", "stax2_api": "4.2.1", "woodstox_core": "6.2.1", - "vector": "0.26.0", + "vector": "0.31.0", }, { "product": "3.4.0", @@ -360,7 +366,7 @@ "jackson_dataformat_xml": "2.14.2", # https://mvnrepository.com/artifact/org.apache.spark/spark-core_2.13/3.4.0 "stax2_api": "4.2.1", # https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-xml/2.14.2 "woodstox_core": "6.5.0", # https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-xml/2.14.2 - "vector": "0.26.0", + "vector": "0.31.0", }, ], }, @@ -374,38 +380,38 @@ { "product": "1.3.2", "python": "3.8", - "vector": "0.26.0", + "vector": "0.31.0", }, { "product": "1.4.1", "python": "3.9", - "vector": "0.26.0", + "vector": "0.31.0", }, { "product": "1.4.2", "python": "3.9", - "vector": "0.26.0", + "vector": "0.31.0", }, { "product": "1.5.1", "python": "3.8", - "vector": "0.26.0", + "vector": "0.31.0", }, { "product": "1.5.3", "python": "3.8", - "vector": "0.26.0", + "vector": "0.31.0", }, { 'product': '2.0.1', 'python': '3.9', - 'vector': '0.26.0', + 'vector': '0.31.0', 'authlib': '0.15.4' # https://github.com/dpgaspar/Flask-AppBuilder/blob/4ac9bba008e404b9a1e783cd272c81bb8634de3d/requirements-extra.txt#L10 }, { 'product': '2.1.0', 'python': '3.9', - 'vector': '0.26.0', + 'vector': '0.31.0', 'authlib': '0.15.4' # https://github.com/dpgaspar/Flask-AppBuilder/blob/4554c40e2298d11c4472ca64b9a60236b12c6535/requirements-extra.txt#L10 }, ], diff --git a/vector/CHANGELOG.md b/vector/CHANGELOG.md deleted file mode 100644 index 45c62bc19..000000000 --- a/vector/CHANGELOG.md +++ /dev/null @@ -1,20 +0,0 @@ -# Changelog - -## [Unreleased] - -## [vector-stackable1.1.0] - 2023-02-03 - -### Added - -- Package inotify-tools added ([#291]). - -[#291]: https://github.com/stackabletech/docker-images/pull/291 - -## [vector-stackable1.0.0] - 2022-12-12 - -### Added - -- Image vector added which installs Vector and is based on the stackable-base - image. ([#268]). - -[#268]: https://github.com/stackabletech/docker-images/pull/268 diff --git a/vector/Dockerfile b/vector/Dockerfile index beac048f9..df7a7c5dd 100644 --- a/vector/Dockerfile +++ b/vector/Dockerfile @@ -1,27 +1,9 @@ # syntax=docker/dockerfile:1 -# Build stage -FROM stackable/image/stackable-base AS builder +FROM stackable/image/stackable-base ARG PRODUCT - -# https://github.com/hadolint/hadolint/wiki/DL4006 -SHELL ["/bin/bash", "-o", "pipefail", "-c"] - -RUN microdnf update && \ - microdnf --assumeyes install gzip-1.9 tar-2:1.30 && \ - microdnf clean all && \ - curl "https://repo.stackable.tech/repository/packages/vector/vector-${PRODUCT}-$(arch)-unknown-linux-gnu.tar.gz" | \ - tar x -o -z -C /opt && \ - mkdir --parents /build/licenses && \ - cp "/opt/vector-$(arch)-unknown-linux-gnu/LICENSE" /build/licenses/VECTOR_LICENSE && \ - mkdir --parents /build/stackable/vector/ && \ - cp --recursive "/opt/vector-$(arch)-unknown-linux-gnu/bin/" /build/stackable/vector/ && \ - mkdir --parents /build/stackable/vector/var && \ - chown --recursive stackable:stackable /build/stackable/ - -# Final image -FROM stackable/image/stackable-base +ARG RPM_RELEASE # https://github.com/hadolint/hadolint/wiki/DL4006 SHELL ["/bin/bash", "-o", "pipefail", "-c"] @@ -34,6 +16,14 @@ SHELL ["/bin/bash", "-o", "pipefail", "-c"] RUN rpm --install https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm && \ microdnf update && \ microdnf --assumeyes install inotify-tools && \ - microdnf clean all - -COPY --from=builder /build / + microdnf clean all && \ + rpm --install "https://repo.stackable.tech/repository/packages/vector/vector-${PRODUCT}-${RPM_RELEASE}.$(arch).rpm" && \ + mkdir /licenses && \ + cp /usr/share/licenses/vector-${PRODUCT}/LICENSE /licenses/VECTOR_LICENSE && \ + # Create the directory /stackable/vector/var. + # This directory is set by operator-rs in the parameter `data_dir` + # of the Vector configuration. The directory is used for persisting + # Vector state, such as on-disk buffers, file checkpoints, and more. + # Vector needs write permissions. + mkdir --parents /stackable/vector/var && \ + chown --recursive stackable:stackable /stackable/ diff --git a/vector/upload_new_vector_version.sh b/vector/upload_new_vector_version.sh index 23c7e589c..1ffe2e235 100755 --- a/vector/upload_new_vector_version.sh +++ b/vector/upload_new_vector_version.sh @@ -1,25 +1,34 @@ #!/usr/bin/env bash VERSION=${1:?"Missing version number argument (arg 1)"} -NEXUS_USER=${2:?"Missing Nexus username argument (arg 2)"} +RELEASE=${2:?"Missing release number argument (arg 2)"} +NEXUS_USER=${3:?"Missing Nexus username argument (arg 3)"} + +ARCHITECTURES=( + x86_64 + aarch64 +) read -r -s -p "Nexus Password: " NEXUS_PASSWORD echo "" -# Vector does not currently publish signatures or SBOMs (as of 2023-06-25, latest version at this point 0.30.0) +# Vector does not currently publish signatures or SBOMs (as of +# 2023-07-31, latest version at this point 0.31.0) -echo "Downloading X86 Vector" -curl -LOs "https://packages.timber.io/vector/$VERSION/vector-$VERSION-x86_64-unknown-linux-gnu.tar.gz" +for arch in "${ARCHITECTURES[@]}"; do + file="vector-$VERSION-$RELEASE.$arch.rpm" -echo "Downloading ARM Vector" -curl -LOs "https://packages.timber.io/vector/$VERSION/vector-$VERSION-aarch64-unknown-linux-gnu.tar.gz" + echo "Downloading $file from timber.io" + curl -LOs "https://packages.timber.io/vector/$VERSION/$file" -echo "Uploading both to Nexus" -curl --fail -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "vector-$VERSION-x86_64-unknown-linux-gnu.tar.gz" 'https://repo.stackable.tech/repository/packages/vector/' -curl --fail -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "vector-$VERSION-aarch64-unknown-linux-gnu.tar.gz" 'https://repo.stackable.tech/repository/packages/vector/' + echo "Uploading $file to Nexus" + curl --fail -u "$NEXUS_USER:$NEXUS_PASSWORD" \ + --upload-file "$file" \ + 'https://repo.stackable.tech/repository/packages/vector/' -rm "vector-$VERSION-x86_64-unknown-linux-gnu.tar.gz" -rm "vector-$VERSION-aarch64-unknown-linux-gnu.tar.gz" + echo "Removing downloaded $file" + rm "$file" +done echo "Successfully uploaded new version of Vector ($VERSION) to Nexus" echo "https://repo.stackable.tech/service/rest/repository/browse/packages/vector/"