From ebe3f3a8a49ac769f88deda9a65102ba8a23bc51 Mon Sep 17 00:00:00 2001 From: Swapneil Singh <32959209+swapneils@users.noreply.github.com> Date: Thu, 3 Oct 2024 14:28:58 -0400 Subject: [PATCH] Migrate to temporary fork repository + Linux-only CVE rebuild 2.32.2.20241003 (#858) * Migrate cherry picked commits to temporary aws-owned fork * Linux-only CVE rebuild 2.32.2.20241003 * Use -L1 flag instead of -l when invoking xargs --------- Co-authored-by: Swapneil Singh --- AWS_FLB_CHERRY_PICKS | 145 --------------------------- AWS_FOR_FLUENT_BIT_VERSION | 2 +- CHANGELOG.md | 6 ++ linux.version | 2 +- scripts/dockerfiles/Dockerfile.build | 4 +- 5 files changed, 10 insertions(+), 149 deletions(-) diff --git a/AWS_FLB_CHERRY_PICKS b/AWS_FLB_CHERRY_PICKS index 11c8df271..e69de29bb 100644 --- a/AWS_FLB_CHERRY_PICKS +++ b/AWS_FLB_CHERRY_PICKS @@ -1,145 +0,0 @@ -# Revert datadog fix PR to resolve segfault -https://github.com/matthewfala/fluent-bit.git ecs-datadog-sequential-revert 98313ebf206eec4a4e5375b352fc36849b762323 - -# Support Opensearch Serverless data ingestion -https://github.com/matthewfala/fluent-bit.git aoss-1.9 1633c49aadad55bac483c5e55772de0e6c29704a -https://github.com/matthewfala/fluent-bit.git aoss-1.9 e1301bc52e209b6c6de8602bff12e0a98aefa0c0 -https://github.com/matthewfala/fluent-bit.git aoss-1.9 1917a7760d1bd400ac987620661b48e1bbc001ae - -# Kinesis time format -https://github.com/Claych/fluent-bit.git clay-aws_strftime_precision-1.9 dfeff9de13ba7f1bbebe08fef24ec993dee7e392 -https://github.com/Claych/fluent-bit.git clay-aws_strftime_precision-1.9 66e85a11590f0045294a3d1b104baf73569a1dd0 -https://github.com/Claych/fluent-bit.git clay-aws_strftime_precision-1.9 e225ff7374fbc5e290b0090652158e3d2db5d8a6 - -# S3 log_key warn fix -https://github.com/PettitWesley/fluent-bit.git s3-log-key-warn-1_9 308b73558fb2d3dafb3b80feb0c68dc9b2c18186 - -# ECS Filter -https://github.com/PettitWesley/fluent-bit.git ecs-empty-metadata-fix-one-commit 24934c938e8f63700edf1230a09e4483ac5df6a3 - -# Add back Datadog fixes that do not trigger segfault -https://github.com/matthewfala/fluent-bit.git ecs-datadog-sequential-revert acc01a4bf4ce656023d82943bec2683b5b7755e5 -https://github.com/matthewfala/fluent-bit.git ecs-datadog-sequential-revert ac30b7c876a95d44c33a406b2b58ce08b978587e - -# Datadog Partial Fix Patch -https://github.com/matthewfala/fluent-bit.git datadog-ecs-patch 3c1ad69ada5bb6f2e448c6f39a1a0ea6a6f4ff17 - -# Resolve cloudwatch_logs duplicate tag match SIGSEGV issue -https://github.com/matthewfala/fluent-bit.git sync-scheduler-fix-1.9 2614c46af3a051b2758bef57f01f6b10a8e73b62 - -# resolve user agent wrong type -https://github.com/PettitWesley/fluent-bit.git user-agent-type-fix 0642f42e8097c159d7364f1ff97c0196484815ce -https://github.com/PettitWesley/fluent-bit.git user-agent-type-fix ed21492a94e8ca156897afc32c73bcdc37bc6b0f -https://github.com/PettitWesley/fluent-bit.git user-agent-type-fix 7c16af941bed91da9558e4e6b8cea98d3ef1fa0d - -# Resolve keepalive and priority scheduler issue -https://github.com/PettitWesley/fluent-bit.git sync-io-keepalive-fix 7b4550486b7e8e02b773894d31a08b0cb18154e3 - -https://github.com/PettitWesley/fluent-bit.git mk_event_inject_conn_fix 0f5efc921950feeee99d9251cb330b27d3639863 - -https://github.com/PettitWesley/fluent-bit.git mk_event_add_corruption-one-commit 33651cca41e9f84ce8930a9bca9d3d7319e50fbd - -# Useful debug messages for input events -# input chunk append message with input name context and number of records -https://github.com/PettitWesley/fluent-bit.git chunk-append-context b671ed38e7ae87ab955083e2324ef1972298c5d7 - -# in_tail file name context for inotify events -https://github.com/PettitWesley/fluent-bit.git tail-modify-debug-context-immutable-cherry-pick ab11d1d7438f5254d04eba8dfc2f10b78cc2244d -https://github.com/PettitWesley/fluent-bit.git tail-modify-debug-context-immutable-cherry-pick 1f3bdeec49ac42b04b921303610ea0f53110ab2c - -# S3 tag corruption fix -https://github.com/PettitWesley/fluent-bit.git s3-str-fixes 38303131e049265277881c0d79935ad31fdd3e13 - -# User friend message when storage.total_limit_size causes fs chunk deletion, before there was no indication -https://github.com/fluent/fluent-bit.git master b725d6b8b289fccde4e9b31d3f3ac61f13711ef9 -# use total_chunks_up in max_chunks_up memory overlimit warn message -https://github.com/fluent/fluent-bit.git master 9c72f3ac6510b701277936897cd9701ffce3646e - -# CloudWatch Logs options for connecting to CWL test destinations: tls verify and port -https://github.com/matthewfala/fluent-bit.git immutable-cwl-net-options 5d9692f00b5295728bf0340d332896a7cc450a7e - -# Go exit fix -https://github.com/PettitWesley/fluent-bit.git go-exit-fix-1_9-one-commit ce5739c20b972320dc485587d56c8b6b21f61934 -# fix build warning from original go fix -https://github.com/PettitWesley/fluent-bit.git fix-proxy-go-destroy 79e4e10f31b7468496d4dddb784b502b3ba9e353 - -# sds printf off by 1 fix: https://github.com/fluent/fluent-bit/issues/7143 -# from PR: https://github.com/fluent/fluent-bit/pull/7148/commits -https://github.com/PettitWesley/fluent-bit.git sds-off-by-1-1_9 e7ba91a6c05d884cc6745d8e49faeb1a92909679 -https://github.com/PettitWesley/fluent-bit.git sds-off-by-1-1_9 6c9e49a627931bd1bdbd8d965a64bfd5c325e01d -https://github.com/PettitWesley/fluent-bit.git sds-off-by-1-1_9 f45b3027dfd0ebac20e35df16bed14020718b780 - -# cw mem leak fix (leak is only a few bytes no matter the runtime/throughput) https://github.com/fluent/fluent-bit/pull/7158/commits -https://github.com/PettitWesley/fluent-bit.git cw-stream-free-fix 8e7809ee9f4e7837a5fff75842a47ca5fd42b526 - -# Messagepack Fix https://github.com/fluent/fluent-bit/commit/c0fc0374c54ae5967f12b5ac34ce89a0ca285210 -https://github.com/fluent/fluent-bit.git 1.9 c0fc0374c54ae5967f12b5ac34ce89a0ca285210 - -# STS response parsing improvement/fix -https://github.com/PettitWesley/fluent-bit.git sts-response-parse-fix b1186b92b53466a240b1f16008995dc85afed892 - -# upstream config map fix https://github.com/fluent/fluent-bit/pull/6874 -https://github.com/fluent/fluent-bit.git 1.9 81cdf7eced4e420043277237fba092157b17ffd9 - -# upstream engine retry clean up fix https://github.com/fluent/fluent-bit/pull/6862 -https://github.com/fluent/fluent-bit.git 1.9 712e5fbe10bee44269d5dfed214c4e087ea1ec2a - -# quick fix for S3 key $INDEX bug https://github.com/aws/aws-for-fluent-bit/issues/653 -https://github.com/PettitWesley/fluent-bit.git index_s3_key_format_quick_fix 5e48218670681aef152aeedcf90a4593ac623470 - -# lib upgrades: chunkio to 1.4.0 and monkey to commit 13a4ccd3 -https://github.com/PettitWesley/fluent-bit.git 2_31_12_lib_upgrades d56634674725aee5101fd17845730bbb66318928 -https://github.com/PettitWesley/fluent-bit.git 2_31_12_lib_upgrades e34af51fee5b5b11f0d239c80308a6bb80ab5f13 -# libbacktrace to 8602fda, cfl to v0.2.3, onigmo to 2bfee1eaf526ec2309822243a976cc792d99fbc3, lib: upgrade to 4bd9260 -https://github.com/PettitWesley/fluent-bit.git 2_31_12_lib_upgrades bd7ad8943e79d8515ceed8ef005a7440ef78ecf6 - -# several upstream aws_util memory fixes (see commit message) -https://github.com/PettitWesley/fluent-bit.git 2_31_12_lib_upgrades 97047e018cfcb1f79daecd7d3ccfbff21948e246 - -# upstream 2.x in_exec bug fixes -# resolves https://github.com/aws/aws-for-fluent-bit/issues/661 -https://github.com/fluent/fluent-bit.git master 6ed4aaabd063b8fdf0c034729e45429da87dc142 -https://github.com/fluent/fluent-bit.git master 62431ad0aede70d9748c372ebc7ac9a9917f9c9d - -# Tail memory fix picked from here: https://github.com/fluent/fluent-bit/commit/ed758a5eb85967cc66ca8dff269e7454b2394c3a -https://github.com/PettitWesley/fluent-bit.git upstream-tail-stat-fix 800bb813a00f14a5f457cc69a89f8fb0f715e8c4 - -# in_http: fix memory initialization and enable it on windows https://github.com/fluent/fluent-bit/issues/7008 -https://github.com/fluent/fluent-bit.git master 7a882df735b28002983770f554b365dc63c0be7e - -# record_accessor/rewrite_tag fix to allow single character rules: https://github.com/fluent/fluent-bit/issues/7330 -https://github.com/PettitWesley/fluent-bit.git 2_31_12_lib_upgrades 4c5c8ab56075b7ce63023f8c5c0c963200027a67 - -# output thread, fix memory initialization: https://github.com/fluent/fluent-bit/pull/7303 -https://github.com/fluent/fluent-bit.git master 9a08168a8ab293fc8054180ee04e1176469df88b - -# filter_modify: fix memory clean up: https://github.com/fluent/fluent-bit/issues/7368 -https://github.com/PettitWesley/fluent-bit.git filter-modify-fix-aws-distro 1a72de13ad6cfd5a176e5d8712064a38a7d097f2 - -# AWS core code self-review issue fixes -# upstream version of this: https://github.com/fluent/fluent-bit/pull/7512/files -# use calloc in all credential code to prevent freeing of garbage pointers -https://github.com/PettitWesley/fluent-bit.git aws-distro-crypto-self-review eb48b79b34f91d36d28434390b976e882d553681 -# fix brittle XML parsing -https://github.com/PettitWesley/fluent-bit.git aws-distro-crypto-self-review e6401ad3811b42dee0b7f92aba726cca4bee74ec -# add pthread_mutex + trylock to protect cred providers -https://github.com/PettitWesley/fluent-bit.git aws-distro-crypto-self-review a1d7469da62a4b3ca869b10732f41562d668cfce -# signv4: always use calloc -https://github.com/PettitWesley/fluent-bit.git aws-distro-crypto-self-review 3381c388ee956e7d0e7c0d5fc44683da75095a6a - -# Cloudwatch_logs sequence token deprecation -https://github.com/matthewfala/fluent-bit.git immutable-sequence-token-deprecation 8ee560e388bbbf850069c81bbca06275f330baeb - -# prometheus: sigv4 aws-for-fluent-bit -https://github.com/matthewfala/fluent-bit.git immutable-2.32.0-prometheus-sigv4 ca93bd1d43ebedeb8e81b46b800ea229fde66fa5 - -# multiline: remove incorrect flush -https://github.com/matthewfala/fluent-bit.git immutable-multiline-incorrect-flush 6431a4e584d52170dbe873d93ba532659921740a -https://github.com/matthewfala/fluent-bit.git immutable-multiline-incorrect-flush 35f23875ca356ea30e9aac19854b810cf8ecad8f - -# core: network event drop shutdown fix -https://github.com/matthewfala/fluent-bit.git 2.32.0-premature-connection-destruction 9e2e5d1bffca92bbcc5001fcfc34c1d9ae2716db -https://github.com/matthewfala/fluent-bit.git 2.32.0-premature-connection-destruction b2e8ff1ae738c1db7bf50942ef619609436ffe02 - -# throttle: print_status configuration issue resolution -https://github.com/matthewfala/fluent-bit.git throttle-filter-print-status-fix 7b05b7ebfe55261ed12d5006c8b682572b6abf4c diff --git a/AWS_FOR_FLUENT_BIT_VERSION b/AWS_FOR_FLUENT_BIT_VERSION index 40537db9a..1585b789a 100644 --- a/AWS_FOR_FLUENT_BIT_VERSION +++ b/AWS_FOR_FLUENT_BIT_VERSION @@ -1 +1 @@ -2.32.2.20240820 +2.32.2.20241003 diff --git a/CHANGELOG.md b/CHANGELOG.md index 74b6e05f7..a292480b3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # Changelog +### 2.32.2.20241003 Linux re-build + +*This release has the same Fluent Bit contents as 2.32.2. It is a linux-only re-build to switch to a new change-management system and merge in recent patches in dependencies installed in the image. There are no windows images for this release.* +* Amazon Linux Base: [2.0.20240916.0](https://docs.aws.amazon.com/AL2/latest/relnotes/relnotes-20240916.html) + + ### 2.32.2.20240820 Linux re-build *This release has the same Fluent Bit contents as 2.32.2, and is simply a linux-only re-build for recent patches in dependencies installed in the image. There are no windows images for this release.* diff --git a/linux.version b/linux.version index 4ab2ba940..157df6718 100644 --- a/linux.version +++ b/linux.version @@ -1,6 +1,6 @@ { "linux": { - "version": "2.32.2.20240820", + "version": "2.32.2.20241003", "latest": "true", "build": "1", "fluent-bit": "1.9.10", diff --git a/scripts/dockerfiles/Dockerfile.build b/scripts/dockerfiles/Dockerfile.build index 2d827bd73..113e0814c 100644 --- a/scripts/dockerfiles/Dockerfile.build +++ b/scripts/dockerfiles/Dockerfile.build @@ -77,7 +77,7 @@ FROM builder as compile # Get Fluent Bit source code WORKDIR /tmp/fluent-bit-$FLB_VERSION/ -RUN git clone https://github.com/fluent/fluent-bit.git /tmp/fluent-bit-$FLB_VERSION/ +RUN git clone https://github.com/amazon-contributing/upstream-to-fluent-bit.git /tmp/fluent-bit-$FLB_VERSION/ WORKDIR /tmp/fluent-bit-$FLB_VERSION/build/ RUN git fetch --all --tags && git checkout tags/v${FLB_VERSION} -b v${FLB_VERSION} && git describe --tags @@ -92,7 +92,7 @@ RUN AWS_FLB_CHERRY_PICKS_COUNT=`awk '{print $0 }' /AWS_FLB_CHERRY_PICKS | sed '/ if [ $AWS_FLB_CHERRY_PICKS_COUNT -gt 0 ]; \ then \ cat /AWS_FLB_CHERRY_PICKS | sed '/^#/d' \ - | xargs -l bash -c 'git fetch $0 $1 && git cherry-pick $2 || exit 255' && \ + | xargs -L1 bash -c 'git fetch $0 $1 && git cherry-pick $2 || exit 255' && \ \ (echo "Cherry Pick Patch Summary:"; \ echo -n "Base "; \