From c467915967d003a7c7a269098531a5a2a30279c5 Mon Sep 17 00:00:00 2001 From: Siegfried Weber Date: Thu, 27 Jul 2023 14:56:20 +0200 Subject: [PATCH 1/7] Upgrade Vector in all product images to version 0.31.0 --- image_tools/conf.py | 56 ++++++++++++++++++++++----------------------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/image_tools/conf.py b/image_tools/conf.py index 44f750348..5332a1c6b 100644 --- a/image_tools/conf.py +++ b/image_tools/conf.py @@ -9,27 +9,27 @@ { "product": "2.2.3", "python": "38", - "vector": "0.26.0", + "vector": "0.31.0", }, { "product": "2.2.4", "python": "39", - "vector": "0.26.0", + "vector": "0.31.0", }, { "product": "2.2.5", "python": "39", - "vector": "0.26.0", + "vector": "0.31.0", }, { "product": "2.4.1", "python": "39", - "vector": "0.26.0", + "vector": "0.31.0", }, { "product": "2.6.1", "python": "39", - "vector": "0.26.0", + "vector": "0.31.0", }, ], }, @@ -138,12 +138,12 @@ "versions": [ { "product": "11", - "vector": "0.26.0", + "vector": "0.31.0", "_security_path": "/usr/lib/jvm/jre-11/conf/security/java.security", }, { "product": "17", - "vector": "0.26.0", + "vector": "0.31.0", "_security_path": "/usr/lib/jvm/jre-17/conf/security/java.security", }, ], @@ -201,7 +201,7 @@ }, { "name": "vector", - "versions": [{"product": "0.26.0", "stackable-base": "1.0.0"}], + "versions": [{"product": "0.31.0", "stackable-base": "1.0.0"}], }, { "name": "nifi", @@ -218,32 +218,32 @@ "versions": [ { "product": "0.27.1", - "vector": "0.26.0", + "vector": "0.31.0", "bundle_builder_version": "1.0.0", }, { "product": "0.28.0", - "vector": "0.26.0", + "vector": "0.31.0", "bundle_builder_version": "1.0.0", }, { "product": "0.37.2", - "vector": "0.26.0", + "vector": "0.31.0", "bundle_builder_version": "1.0.0", }, { "product": "0.41.0", - "vector": "0.26.0", + "vector": "0.31.0", "bundle_builder_version": "1.0.0", }, { "product": "0.45.0", - "vector": "0.26.0", + "vector": "0.31.0", "bundle_builder_version": "1.0.0", }, { "product": "0.51.0", - "vector": "0.26.0", + "vector": "0.31.0", "bundle_builder_version": "1.1.0", }, ], @@ -265,7 +265,7 @@ "jackson_dataformat_xml": "2.12.3", "stax2_api": "4.2.1", "woodstox_core": "6.2.1", - "vector": "0.26.0", + "vector": "0.31.0", }, { "product": "3.3.0", @@ -281,7 +281,7 @@ "jackson_dataformat_xml": "2.13.3", "stax2_api": "4.2.1", "woodstox_core": "6.2.1", - "vector": "0.26.0", + "vector": "0.31.0", }, { "product": "3.3.0-java17", @@ -297,7 +297,7 @@ "jackson_dataformat_xml": "2.13.3", "stax2_api": "4.2.1", "woodstox_core": "6.2.1", - "vector": "0.26.0", + "vector": "0.31.0", }, { "product": "3.4.0", @@ -313,7 +313,7 @@ "jackson_dataformat_xml": "2.14.2", "stax2_api": "4.2.1", "woodstox_core": "6.5.0", - "vector": "0.26.0", + "vector": "0.31.0", }, { "product": "3.4.0-java17", @@ -329,7 +329,7 @@ "jackson_dataformat_xml": "2.14.2", "stax2_api": "4.2.1", "woodstox_core": "6.5.0", - "vector": "0.26.0", + "vector": "0.31.0", }, ], }, @@ -347,7 +347,7 @@ "jackson_dataformat_xml": "2.13.3", "stax2_api": "4.2.1", "woodstox_core": "6.2.1", - "vector": "0.26.0", + "vector": "0.31.0", }, { "product": "3.4.0", @@ -360,7 +360,7 @@ "jackson_dataformat_xml": "2.14.2", # https://mvnrepository.com/artifact/org.apache.spark/spark-core_2.13/3.4.0 "stax2_api": "4.2.1", # https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-xml/2.14.2 "woodstox_core": "6.5.0", # https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-xml/2.14.2 - "vector": "0.26.0", + "vector": "0.31.0", }, ], }, @@ -374,38 +374,38 @@ { "product": "1.3.2", "python": "3.8", - "vector": "0.26.0", + "vector": "0.31.0", }, { "product": "1.4.1", "python": "3.9", - "vector": "0.26.0", + "vector": "0.31.0", }, { "product": "1.4.2", "python": "3.9", - "vector": "0.26.0", + "vector": "0.31.0", }, { "product": "1.5.1", "python": "3.8", - "vector": "0.26.0", + "vector": "0.31.0", }, { "product": "1.5.3", "python": "3.8", - "vector": "0.26.0", + "vector": "0.31.0", }, { 'product': '2.0.1', 'python': '3.9', - 'vector': '0.26.0', + 'vector': '0.31.0', 'authlib': '0.15.4' # https://github.com/dpgaspar/Flask-AppBuilder/blob/4ac9bba008e404b9a1e783cd272c81bb8634de3d/requirements-extra.txt#L10 }, { 'product': '2.1.0', 'python': '3.9', - 'vector': '0.26.0', + 'vector': '0.31.0', 'authlib': '0.15.4' # https://github.com/dpgaspar/Flask-AppBuilder/blob/4554c40e2298d11c4472ca64b9a60236b12c6535/requirements-extra.txt#L10 }, ], From b786a843e42363f147eb36718eb383ad89369bce Mon Sep 17 00:00:00 2001 From: Siegfried Weber Date: Fri, 28 Jul 2023 11:26:43 +0200 Subject: [PATCH 2/7] Use RPM for Vector --- vector/Dockerfile | 35 ++++++++++++----------------------- 1 file changed, 12 insertions(+), 23 deletions(-) diff --git a/vector/Dockerfile b/vector/Dockerfile index beac048f9..bf7119cdb 100644 --- a/vector/Dockerfile +++ b/vector/Dockerfile @@ -1,31 +1,12 @@ # syntax=docker/dockerfile:1 -# Build stage -FROM stackable/image/stackable-base AS builder +FROM stackable/image/stackable-base ARG PRODUCT # https://github.com/hadolint/hadolint/wiki/DL4006 SHELL ["/bin/bash", "-o", "pipefail", "-c"] -RUN microdnf update && \ - microdnf --assumeyes install gzip-1.9 tar-2:1.30 && \ - microdnf clean all && \ - curl "https://repo.stackable.tech/repository/packages/vector/vector-${PRODUCT}-$(arch)-unknown-linux-gnu.tar.gz" | \ - tar x -o -z -C /opt && \ - mkdir --parents /build/licenses && \ - cp "/opt/vector-$(arch)-unknown-linux-gnu/LICENSE" /build/licenses/VECTOR_LICENSE && \ - mkdir --parents /build/stackable/vector/ && \ - cp --recursive "/opt/vector-$(arch)-unknown-linux-gnu/bin/" /build/stackable/vector/ && \ - mkdir --parents /build/stackable/vector/var && \ - chown --recursive stackable:stackable /build/stackable/ - -# Final image -FROM stackable/image/stackable-base - -# https://github.com/hadolint/hadolint/wiki/DL4006 -SHELL ["/bin/bash", "-o", "pipefail", "-c"] - # Init Jobs/Pods often start a Vector Sidecar Container which collects the logs. # As soon as an Init Container is done it'll need to tell the Vector sidecar that it can now also stop # This happens by writing a "shutdown file" in a shared volume @@ -34,6 +15,14 @@ SHELL ["/bin/bash", "-o", "pipefail", "-c"] RUN rpm --install https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm && \ microdnf update && \ microdnf --assumeyes install inotify-tools && \ - microdnf clean all - -COPY --from=builder /build / + microdnf clean all && \ + rpm --install "https://github.com/vectordotdev/vector/releases/download/v${PRODUCT}/vector-${PRODUCT}-1.$(arch).rpm" && \ + mkdir /licenses && \ + cp /usr/share/licenses/vector-${PRODUCT}/LICENSE /licenses/VECTOR_LICENSE && \ + # Create the directory /stackable/vector/var. + # This directory is set by operator-rs in the parameter `data_dir` + # of the Vector configuration. The directory is used for persisting + # Vector state, such as on-disk buffers, file checkpoints, and more. + # Vector needs write permissions. + mkdir --parents /stackable/vector/var && \ + chown --recursive stackable:stackable /stackable/ From 6ca6f8ae29e8ce0c16639ef56912d089998d9a11 Mon Sep 17 00:00:00 2001 From: Siegfried Weber Date: Fri, 28 Jul 2023 11:27:35 +0200 Subject: [PATCH 3/7] Remove unused AIRFLOW_UID --- airflow/Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/airflow/Dockerfile b/airflow/Dockerfile index 10e35b83a..f000b6d47 100644 --- a/airflow/Dockerfile +++ b/airflow/Dockerfile @@ -68,7 +68,6 @@ RUN microdnf install -y yum python${PYTHON} \ ENV HOME=/stackable ENV AIRFLOW_USER_HOME_DIR=/stackable ENV PATH=$PATH:/bin:$HOME/app/bin -ENV AIRFLOW_UID="1000" ENV AIRFLOW_HOME=$HOME/airflow RUN mkdir -pv ${AIRFLOW_HOME} && \ From bb3caf14f3e9075a3f42369f8f8048dd9ce85987 Mon Sep 17 00:00:00 2001 From: Siegfried Weber Date: Mon, 31 Jul 2023 09:37:22 +0200 Subject: [PATCH 4/7] Use Vector RPM from Stackable Nexus --- image_tools/conf.py | 8 +++++++- vector/Dockerfile | 3 ++- vector/upload_new_vector_version.sh | 31 +++++++++++++++++++---------- 3 files changed, 29 insertions(+), 13 deletions(-) diff --git a/image_tools/conf.py b/image_tools/conf.py index 5332a1c6b..5036369fd 100644 --- a/image_tools/conf.py +++ b/image_tools/conf.py @@ -201,7 +201,13 @@ }, { "name": "vector", - "versions": [{"product": "0.31.0", "stackable-base": "1.0.0"}], + "versions": [ + { + "product": "0.31.0", + "rpm_release": "1", + "stackable-base": "1.0.0" + } + ], }, { "name": "nifi", diff --git a/vector/Dockerfile b/vector/Dockerfile index bf7119cdb..df7a7c5dd 100644 --- a/vector/Dockerfile +++ b/vector/Dockerfile @@ -3,6 +3,7 @@ FROM stackable/image/stackable-base ARG PRODUCT +ARG RPM_RELEASE # https://github.com/hadolint/hadolint/wiki/DL4006 SHELL ["/bin/bash", "-o", "pipefail", "-c"] @@ -16,7 +17,7 @@ RUN rpm --install https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.no microdnf update && \ microdnf --assumeyes install inotify-tools && \ microdnf clean all && \ - rpm --install "https://github.com/vectordotdev/vector/releases/download/v${PRODUCT}/vector-${PRODUCT}-1.$(arch).rpm" && \ + rpm --install "https://repo.stackable.tech/repository/packages/vector/vector-${PRODUCT}-${RPM_RELEASE}.$(arch).rpm" && \ mkdir /licenses && \ cp /usr/share/licenses/vector-${PRODUCT}/LICENSE /licenses/VECTOR_LICENSE && \ # Create the directory /stackable/vector/var. diff --git a/vector/upload_new_vector_version.sh b/vector/upload_new_vector_version.sh index 23c7e589c..1ffe2e235 100755 --- a/vector/upload_new_vector_version.sh +++ b/vector/upload_new_vector_version.sh @@ -1,25 +1,34 @@ #!/usr/bin/env bash VERSION=${1:?"Missing version number argument (arg 1)"} -NEXUS_USER=${2:?"Missing Nexus username argument (arg 2)"} +RELEASE=${2:?"Missing release number argument (arg 2)"} +NEXUS_USER=${3:?"Missing Nexus username argument (arg 3)"} + +ARCHITECTURES=( + x86_64 + aarch64 +) read -r -s -p "Nexus Password: " NEXUS_PASSWORD echo "" -# Vector does not currently publish signatures or SBOMs (as of 2023-06-25, latest version at this point 0.30.0) +# Vector does not currently publish signatures or SBOMs (as of +# 2023-07-31, latest version at this point 0.31.0) -echo "Downloading X86 Vector" -curl -LOs "https://packages.timber.io/vector/$VERSION/vector-$VERSION-x86_64-unknown-linux-gnu.tar.gz" +for arch in "${ARCHITECTURES[@]}"; do + file="vector-$VERSION-$RELEASE.$arch.rpm" -echo "Downloading ARM Vector" -curl -LOs "https://packages.timber.io/vector/$VERSION/vector-$VERSION-aarch64-unknown-linux-gnu.tar.gz" + echo "Downloading $file from timber.io" + curl -LOs "https://packages.timber.io/vector/$VERSION/$file" -echo "Uploading both to Nexus" -curl --fail -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "vector-$VERSION-x86_64-unknown-linux-gnu.tar.gz" 'https://repo.stackable.tech/repository/packages/vector/' -curl --fail -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "vector-$VERSION-aarch64-unknown-linux-gnu.tar.gz" 'https://repo.stackable.tech/repository/packages/vector/' + echo "Uploading $file to Nexus" + curl --fail -u "$NEXUS_USER:$NEXUS_PASSWORD" \ + --upload-file "$file" \ + 'https://repo.stackable.tech/repository/packages/vector/' -rm "vector-$VERSION-x86_64-unknown-linux-gnu.tar.gz" -rm "vector-$VERSION-aarch64-unknown-linux-gnu.tar.gz" + echo "Removing downloaded $file" + rm "$file" +done echo "Successfully uploaded new version of Vector ($VERSION) to Nexus" echo "https://repo.stackable.tech/service/rest/repository/browse/packages/vector/" From b6042ad68db6b779814208fc958a929bbf2ea54f Mon Sep 17 00:00:00 2001 From: Siegfried Weber Date: Mon, 31 Jul 2023 11:12:22 +0200 Subject: [PATCH 5/7] Assign changelog entries to the corresponding SDP releases --- CHANGELOG.md | 45 ++++++++++++++++++++++++++++++--------------- 1 file changed, 30 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index baef6082b..6bb3a5395 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,35 +4,50 @@ All notable changes to this project will be documented in this file. ## [Unreleased] +### Changed + +- Bump ubi8-rust-builder toolchain to 1.71.0 ([#419]). + +[#419]: https://github.com/stackabletech/docker-images/pull/419 + +## [23.7.0] - 2023-07-14 + +## [23.4.0] - 2023-04-17 + ### Added -- Added labels for openshift requirements and licenses for all images ([#95]). -- Added `RELEASE` ARG for all images ([#95]). - Added krb5 image ([#338]). ### Changed -- Updated java base image to latest ubi8 tag 8.6-994 ([#249]). -- Updated all java-base images to stackable0.2.2 ([#250]). -- Updated all ubi8 base images to latest (8.6-994) ([#250]). -- Updated all internal images to rebuild their base images on demand ([#321]). +- Updated all internal images to rebuild their base images on demand + ([#321]). - Unpinned testing-tools dependencies ([#326]). -- Bump ubi8-rust-builder toolchain to 1.71.0 ([#419]). ### Removed -- Prometheus image, NodeExporter image, Antora ([#95]). -- Retired Java 1.8.0 support ([#248]). - Tools image ([#325]). -- Replace `build_product_images.py` with the `image_tools` package and add OpenShift preflight checks for images ([#339]) +- Replace `build_product_images.py` with the `image_tools` package and + add OpenShift preflight checks for images ([#339]) -[#95]: https://github.com/stackabletech/docker-images/pull/95 -[#248]: https://github.com/stackabletech/docker-images/pull/248 -[#249]: https://github.com/stackabletech/docker-images/pull/249 -[#250]: https://github.com/stackabletech/docker-images/pull/250 [#321]: https://github.com/stackabletech/docker-images/pull/321 [#325]: https://github.com/stackabletech/docker-images/pull/325 [#326]: https://github.com/stackabletech/docker-images/pull/326 [#338]: https://github.com/stackabletech/docker-images/pull/338 [#339]: https://github.com/stackabletech/docker-images/pull/339 -[#419]: https://github.com/stackabletech/docker-images/pull/419 + +## [23.1.0] - 2023-01-23 + +### Changed + +- Updated java base image to latest ubi8 tag 8.6-994 ([#249]). +- Updated all java-base images to stackable0.2.2 ([#250]). +- Updated all ubi8 base images to latest (8.6-994) ([#250]). + +### Removed + +- Retired Java 1.8.0 support ([#248]). + +[#248]: https://github.com/stackabletech/docker-images/pull/248 +[#249]: https://github.com/stackabletech/docker-images/pull/249 +[#250]: https://github.com/stackabletech/docker-images/pull/250 From 19ebaf73aa3c15c87dfe62f1d76e5bf12e64c932 Mon Sep 17 00:00:00 2001 From: Siegfried Weber Date: Mon, 31 Jul 2023 11:15:53 +0200 Subject: [PATCH 6/7] Move entries from vector/CHANGELOG.md to CHANGELOG.md --- CHANGELOG.md | 8 ++++++++ vector/CHANGELOG.md | 20 -------------------- 2 files changed, 8 insertions(+), 20 deletions(-) delete mode 100644 vector/CHANGELOG.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 6bb3a5395..7e83182a4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ All notable changes to this project will be documented in this file. ### Added +- Package inotify-tools added ([#291]). - Added krb5 image ([#338]). ### Changed @@ -30,6 +31,7 @@ All notable changes to this project will be documented in this file. - Replace `build_product_images.py` with the `image_tools` package and add OpenShift preflight checks for images ([#339]) +[#291]: https://github.com/stackabletech/docker-images/pull/291 [#321]: https://github.com/stackabletech/docker-images/pull/321 [#325]: https://github.com/stackabletech/docker-images/pull/325 [#326]: https://github.com/stackabletech/docker-images/pull/326 @@ -38,6 +40,11 @@ All notable changes to this project will be documented in this file. ## [23.1.0] - 2023-01-23 +### Added + +- Image vector added which installs Vector and is based on the + stackable-base image. ([#268]). + ### Changed - Updated java base image to latest ubi8 tag 8.6-994 ([#249]). @@ -51,3 +58,4 @@ All notable changes to this project will be documented in this file. [#248]: https://github.com/stackabletech/docker-images/pull/248 [#249]: https://github.com/stackabletech/docker-images/pull/249 [#250]: https://github.com/stackabletech/docker-images/pull/250 +[#268]: https://github.com/stackabletech/docker-images/pull/268 diff --git a/vector/CHANGELOG.md b/vector/CHANGELOG.md deleted file mode 100644 index 45c62bc19..000000000 --- a/vector/CHANGELOG.md +++ /dev/null @@ -1,20 +0,0 @@ -# Changelog - -## [Unreleased] - -## [vector-stackable1.1.0] - 2023-02-03 - -### Added - -- Package inotify-tools added ([#291]). - -[#291]: https://github.com/stackabletech/docker-images/pull/291 - -## [vector-stackable1.0.0] - 2022-12-12 - -### Added - -- Image vector added which installs Vector and is based on the stackable-base - image. ([#268]). - -[#268]: https://github.com/stackabletech/docker-images/pull/268 From d8e728e6cf699b8621803f0dabc2ed874652bd9e Mon Sep 17 00:00:00 2001 From: Siegfried Weber Date: Mon, 31 Jul 2023 11:24:14 +0200 Subject: [PATCH 7/7] Update changelog with #429 --- CHANGELOG.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7e83182a4..7e3be4918 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,8 +7,20 @@ All notable changes to this project will be documented in this file. ### Changed - Bump ubi8-rust-builder toolchain to 1.71.0 ([#419]). +- BREAKING: Upgrade Vector in all product images to version 0.31.0. The + integration tests of the operators must be adapted because the metric + `processedEventsTotal` was replaced by `receivedEventsTotal` ([#429]). +- BREAKING: Use RPM instead of tar.gz for Vector. Because of that, the + location of the Vector executable changed, and the operator-rs version + 0.45.0 is required ([#429]). + +### Removed + +- Remove unused environment variable `AIRFLOW_UID` from the Airflow + image ([#429]). [#419]: https://github.com/stackabletech/docker-images/pull/419 +[#429]: https://github.com/stackabletech/docker-images/pull/429 ## [23.7.0] - 2023-07-14