diff --git a/.github/workflows/pr-packages-index.yaml b/.github/workflows/pr-packages-index.yaml new file mode 100644 index 00000000000..31c6ee12aaa --- /dev/null +++ b/.github/workflows/pr-packages-index.yaml @@ -0,0 +1,40 @@ +name: Test packages index generator +on: + pull_request: + types: + - opened + - reopened + - synchronize + paths: + - 'packaging/generate-packages-index.sh' + - 'packaging/build-catalog.awk' + - 'packaging/testing/test-packages-index.sh' + - 'packaging/testing/generate-preview-from-s3.sh' + - '.github/workflows/pr-packages-index.yaml' + - '.github/workflows/staging-release.yaml' + + workflow_dispatch: + +jobs: + test-packages-index: + name: Run packages index tests + runs-on: ubuntu-22.04 + permissions: + contents: read + timeout-minutes: 10 + steps: + - name: Checkout Fluent Bit code + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + + - name: Setup runner + run: | + sudo apt-get update + sudo apt-get install -y jq tree + shell: bash + + - name: Run packages index tests + run: | + ./packaging/testing/test-packages-index.sh + shell: bash diff --git a/.github/workflows/staging-release.yaml b/.github/workflows/staging-release.yaml index 85d365e5a9a..21c9ab606b4 100644 --- a/.github/workflows/staging-release.yaml +++ b/.github/workflows/staging-release.yaml @@ -359,6 +359,65 @@ jobs: AWS_RELEASE_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET_RELEASE }} shell: bash + staging-release-packages-index: + name: Update packages index + runs-on: ubuntu-22.04 + environment: release + needs: + - staging-release-apt-packages + - staging-release-yum-packages + - staging-release-update-non-linux-s3 + - staging-release-update-base-s3 + permissions: + contents: read + steps: + - name: Checkout code + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + + - name: Setup runner + run: | + sudo apt-get update + sudo apt-get install -y jq tree + shell: bash + + - name: Generate packages index + run: | + mkdir -p packaging/catalog + ./packaging/generate-packages-index.sh + env: + BASE_PATH: packaging/catalog + BASE_URL: https://packages.fluentbit.io + AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET_RELEASE }} + AWS_S3_REMOTE_DISCOVERY: true + AWS_S3_NO_SIGN_REQUEST: false + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_REGION: us-east-1 + shell: bash + + - name: Upload packages index to S3 + run: | + BUCKET="s3://${{ secrets.AWS_S3_BUCKET_RELEASE }}" + aws s3 cp packaging/catalog/index.html "$BUCKET/index.html" \ + --content-type "text/html; charset=utf-8" \ + --cache-control "public, max-age=300" \ + --no-progress + aws s3 cp packaging/catalog/versions.json "$BUCKET/versions.json" \ + --content-type "application/json" \ + --cache-control "public, max-age=300" \ + --no-progress + aws s3 cp packaging/catalog/latest-version.txt "$BUCKET/latest-version.txt" \ + --content-type "text/plain; charset=utf-8" \ + --cache-control "public, max-age=300" \ + --no-progress + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_REGION: us-east-1 + shell: bash + staging-release-source-s3: name: S3 - update source bucket runs-on: ubuntu-22.04 diff --git a/packaging/build-catalog.awk b/packaging/build-catalog.awk new file mode 100644 index 00000000000..b5a388d5f59 --- /dev/null +++ b/packaging/build-catalog.awk @@ -0,0 +1,277 @@ +BEGIN { + nrepos = split(repo_paths, repos, " ") +} + +function is_version(v) { + return v ~ /^[0-9]+\.[0-9]+(\.[0-9]+)*$/ +} + +function schema_version(name) { + if (name !~ /^fluent-bit-schema-[0-9]+\.[0-9]+(\.[0-9]+)*\.json$/) { + return "" + } + sub(/^fluent-bit-schema-/, "", name) + sub(/\.json$/, "", name) + return name +} + +function extract_version(name, rest, i, c, following) { + if (substr(name, 1, 11) == "fluent-bit-") { + rest = substr(name, 12) + } else if (substr(name, 1, 11) == "fluent-bit_") { + rest = substr(name, 12) + } else { + return "" + } + + for (i = 1; i <= length(rest); i++) { + c = substr(rest, i, 1) + if (c == ".") { + if (i < length(rest)) { + following = substr(rest, i + 1, 1) + if (following !~ /[0-9]/) { + if (i == 1) { + return "" + } + return substr(rest, 1, i - 1) + } + } + continue + } + if (c < "0" || c > "9") { + if (i == 1) { + return "" + } + return substr(rest, 1, i - 1) + } + } + + return rest +} + +function version_includes_linux(v) { + return v ~ /^[45]\./ +} + +function repo_prefix(path, i, p) { + for (i = 1; i <= nrepos; i++) { + p = repos[i] + if (p == "") { + continue + } + if (index(path, p "/") == 1) { + return p + } + } + return "" +} + +function is_deb_repo(repo) { + return index(repo, "debian/") == 1 || index(repo, "ubuntu/") == 1 \ + || index(repo, "raspbian/") == 1 +} + +function is_canonical_linux_path(path, repo) { + if (repo == "") { + return 0 + } + if (is_deb_repo(repo)) { + return index(path, "/pool/") > 0 + } + return 1 +} + +function is_downloadable_linux(name) { + return name ~ /^fluent-bit-[0-9]+(\.[0-9]+)+-[0-9]+\.(x86_64|aarch64|arm64)\.rpm$/ \ + || name ~ /^fluent-bit_[0-9]+(\.[0-9]+)+_(amd64|arm64|aarch64)\.deb$/ +} + +function make_linux_label(path, name, repo, arch, format) { + format = "" + arch = "" + if (name ~ /\.rpm$/) { + format = "rpm" + if (match(name, /\.(x86_64|aarch64|arm64)\.rpm$/)) { + arch = substr(name, RSTART + 1, RLENGTH - 5) + } + } else if (name ~ /\.deb$/) { + format = "deb" + if (match(name, /_(amd64|arm64|aarch64)\.deb$/)) { + arch = substr(name, RSTART + 1, RLENGTH - 5) + } + } + return repo " " arch " " format +} + +function json_str(s) { + gsub(/\\/, "\\\\", s) + gsub(/"/, "\\\"", s) + return "\"" s "\"" +} + +function json_url(path) { + if (path == "") { + return "null" + } + return json_str(base_url "/" path) +} + +function note_linux(v, path, name, repo, key, label, idx) { + if (!version_includes_linux(v)) { + return + } + key = v SUBSEP path + if (linux_seen[key]) { + return + } + linux_seen[key] = 1 + label = make_linux_label(path, name, repo) + linux_count[v]++ + idx = linux_count[v] + linux_path[v, idx] = path + linux_repo[v, idx] = repo + linux_name[v, idx] = name + linux_lbl[v, idx] = label + linux_repo_set[v SUBSEP repo] = 1 +} + +function windows_key(name, v) { + if (name == "fluent-bit-" v "-win32.exe") { + return "win32_exe" + } + if (name == "fluent-bit-" v "-win32.zip") { + return "win32_zip" + } + if (name == "fluent-bit-" v "-win64.exe") { + return "win64_exe" + } + if (name == "fluent-bit-" v "-win64.zip") { + return "win64_zip" + } + if (name == "fluent-bit-" v "-winarm64.exe") { + return "winarm64_exe" + } + if (name == "fluent-bit-" v "-winarm64.zip") { + return "winarm64_zip" + } + return "" +} + +function emit_version(v, out, i, repo_key, repo, repos_out, repos_n) { + out = "{" + out = out "\"version\":" json_str(v) "," + out = out "\"github_release\":" json_str(github_release "/releases/tag/v" v) "," + out = out "\"docs\":" json_str(docs_url) "," + out = out "\"schema\":" json_url(schema[v]) "," + out = out "\"artifacts\":{" + out = out "\"windows\":{" + out = out "\"win32_exe\":" json_url(windows[v, "win32_exe"]) "," + out = out "\"win32_zip\":" json_url(windows[v, "win32_zip"]) "," + out = out "\"win64_exe\":" json_url(windows[v, "win64_exe"]) "," + out = out "\"win64_zip\":" json_url(windows[v, "win64_zip"]) "," + out = out "\"winarm64_exe\":" json_url(windows[v, "winarm64_exe"]) "," + out = out "\"winarm64_zip\":" json_url(windows[v, "winarm64_zip"]) + out = out "}," + out = out "\"macos\":{" + out = out "\"pkg\":" json_url(macos_pkg[v]) "," + out = out "\"dmg\":" json_url(macos_dmg[v]) + out = out "}," + out = out "\"linux\":[" + for (i = 1; i <= linux_count[v]; i++) { + if (i > 1) { + out = out "," + } + out = out "{" + out = out "\"repo\":" json_str(linux_repo[v, i]) "," + out = out "\"name\":" json_str(linux_name[v, i]) "," + out = out "\"label\":" json_str(linux_lbl[v, i]) "," + out = out "\"url\":" json_url(linux_path[v, i]) + out = out "}" + } + out = out "]}," + out = out "\"linux_repos\":[" + repos_n = 0 + for (repo_key in linux_repo_set) { + split(repo_key, parts, SUBSEP) + if (parts[1] != v) { + continue + } + if (repos_n++ > 0) { + out = out "," + } + out = out json_str(base_url "/" parts[2]) + } + out = out "]}" + print v "\t" out +} + +{ + if (NF < 1 || $0 == "") { + next + } + + path = $0 + sub(/\r$/, "", path) + n = split(path, parts, "/") + name = parts[n] + + if (name ~ /^fluent-bit-schema-[0-9]+\.[0-9]+(\.[0-9]+)*\.json$/) { + v = schema_version(name) + if (is_version(v)) { + versions[v] = 1 + schema[v] = path + } + next + } + + if (path ~ /^windows\//) { + v = extract_version(name) + key = windows_key(name, v) + if (key != "") { + versions[v] = 1 + windows[v, key] = path + } + next + } + + if (path ~ /^macos\//) { + v = extract_version(name) + if (v == "") { + next + } + versions[v] = 1 + if (name == "fluent-bit-" v ".pkg") { + macos_pkg[v] = path + } else if (name == "fluent-bit-" v ".dmg") { + macos_dmg[v] = path + } + next + } + + if (is_downloadable_linux(name)) { + repo = repo_prefix(path) + if (!is_canonical_linux_path(path, repo)) { + next + } + v = extract_version(name) + if (!is_version(v)) { + next + } + versions[v] = 1 + note_linux(v, path, name, repo) + } +} + +END { + mode = emit_mode + for (v in versions) { + if (!is_version(v)) { + continue + } + if (mode == "versions") { + emit_version(v) + } else { + print v + } + } +} diff --git a/packaging/generate-packages-index.sh b/packaging/generate-packages-index.sh new file mode 100755 index 00000000000..2029446346a --- /dev/null +++ b/packaging/generate-packages-index.sh @@ -0,0 +1,272 @@ +#!/bin/bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" + +BASE_PATH=${BASE_PATH:-${1:-}} +BASE_URL=${BASE_URL:-https://packages.fluentbit.io} +LATEST_VERSION=${LATEST_VERSION:-} +GITHUB_REPO=${GITHUB_REPO:-https://github.com/fluent/fluent-bit} +DOCS_URL=${DOCS_URL:-https://docs.fluentbit.io/manual/installation} +AWS_S3_BUCKET=${AWS_S3_BUCKET:-} +AWS_S3_REMOTE_DISCOVERY=${AWS_S3_REMOTE_DISCOVERY:-false} +AWS_S3_NO_SIGN_REQUEST=${AWS_S3_NO_SIGN_REQUEST:-true} +AWS_S3_ENDPOINT=${AWS_S3_ENDPOINT:-} + +WORK_DIR="" +OBJECT_LIST="" + +LINUX_REPO_PATHS=( + "amazonlinux/2" + "amazonlinux/2023" + "centos/7" + "centos/8" + "centos/9" + "centos/10" + "rockylinux/8" + "rockylinux/9" + "rockylinux/10" + "almalinux/8" + "almalinux/9" + "almalinux/10" + "debian/bookworm" + "debian/bullseye" + "debian/buster" + "debian/trixie" + "ubuntu/jammy" + "ubuntu/noble" + "ubuntu/resolute" + "raspbian/bookworm" +) + +usage() +{ + cat <&2 + exit 1 +fi + +if ! command -v jq >/dev/null 2>&1; then + echo "ERROR: jq is required" >&2 + exit 1 +fi + +if ! command -v tree >/dev/null 2>&1; then + echo "ERROR: tree is required" >&2 + exit 1 +fi + +BASE_PATH="$(cd "$BASE_PATH" && pwd)" +WORK_DIR="$(mktemp -d)" +OBJECT_LIST="$WORK_DIR/objects.txt" +TREE_LIST="$WORK_DIR/tree.txt" +VERSION_ROWS="$WORK_DIR/version-rows.tsv" +trap cleanup EXIT + +aws_s3_listing_cmd() +{ + local -a cmd=(aws s3 ls "s3://${AWS_S3_BUCKET}/" --recursive) + + if [[ "$AWS_S3_NO_SIGN_REQUEST" == "true" ]]; then + cmd+=(--no-sign-request) + fi + if [[ -n "$AWS_S3_ENDPOINT" ]]; then + cmd+=(--endpoint-url "$AWS_S3_ENDPOINT") + fi + + "${cmd[@]}" +} + +s3_listing_relative_path() +{ + awk 'NF >= 4 { + $1 = "" + $2 = "" + $3 = "" + sub(/^ +/, "") + print + }' +} + +collect_local_objects() +{ + local file base + + while IFS= read -r -d '' file; do + base="$(basename "$file")" + case "$base" in + index.html|versions.json|latest-version.txt) + continue + ;; + esac + echo "${file#"$BASE_PATH"/}" + done < <(find "$BASE_PATH" -type f -print0 2>/dev/null || true) +} + +collect_s3_objects() +{ + local destination="$1" + + if [[ -n "${AWS_S3_LISTING_FILE:-}" ]]; then + if [[ ! -f "$AWS_S3_LISTING_FILE" ]]; then + echo "ERROR: AWS_S3_LISTING_FILE does not exist: $AWS_S3_LISTING_FILE" >&2 + exit 1 + fi + cp "$AWS_S3_LISTING_FILE" "$destination" + return 0 + fi + + if ! command -v aws >/dev/null 2>&1; then + echo "ERROR: aws CLI is required for AWS_S3_REMOTE_DISCOVERY" >&2 + exit 1 + fi + + aws_s3_listing_cmd | s3_listing_relative_path > "$destination" +} + +build_object_list() +{ + : > "$OBJECT_LIST" + collect_local_objects >> "$OBJECT_LIST" + + if [[ "$AWS_S3_REMOTE_DISCOVERY" == "true" ]]; then + if [[ -z "$AWS_S3_BUCKET" && -z "${AWS_S3_LISTING_FILE:-}" ]]; then + echo "ERROR: AWS_S3_BUCKET or AWS_S3_LISTING_FILE is required for remote discovery" >&2 + exit 1 + fi + + collect_s3_objects "$WORK_DIR/s3-listing.txt" + echo "Fetched $(wc -l < "$WORK_DIR/s3-listing.txt" | tr -d ' ') objects from s3://${AWS_S3_BUCKET:-listing}" >&2 + cat "$WORK_DIR/s3-listing.txt" >> "$OBJECT_LIST" + fi + + sort -u -o "$OBJECT_LIST" "$OBJECT_LIST" +} + +render_index_html() +{ + local latest="$1" + local generated_at="$2" + local tree_html="$3" + + sed "s||

Fluent Bit Packages

Latest release: ${latest}
Generated: ${generated_at}
Catalog: versions.json, latest-version.txt, fluentbit.key

|" \ + "$tree_html" > "$BASE_PATH/index.html" +} + +build_object_list + +if [[ ! -s "$OBJECT_LIST" ]]; then + echo "ERROR: no objects found under $BASE_PATH" >&2 + exit 1 +fi + +grep -Ev 'source-|pool|dists' "$OBJECT_LIST" | \ + grep -E '\.(rpm|deb|key|repo|exe|msi|zip|pkg)$' > "$TREE_LIST" || true + +if [[ ! -s "$TREE_LIST" ]]; then + echo "ERROR: no package files found for index.html" >&2 + exit 1 +fi + +tree --noreport --charset utf-8 --fromfile "$TREE_LIST" -H "$BASE_URL" | \ + awk '/
/ { exit } { print } END { print "" }' > "$WORK_DIR/tree.html" + +REPO_PATHS="$(printf '%s ' "${LINUX_REPO_PATHS[@]}")" +awk -v emit_mode=versions \ + -v base_url="$BASE_URL" \ + -v github_release="$GITHUB_REPO" \ + -v docs_url="$DOCS_URL" \ + -v repo_paths="$REPO_PATHS" \ + -f "$SCRIPT_DIR/build-catalog.awk" \ + "$OBJECT_LIST" | sort -t $'\t' -V -k1,1 > "$VERSION_ROWS" + +if [[ ! -s "$VERSION_ROWS" ]]; then + echo "ERROR: no Fluent Bit package versions found" >&2 + exit 1 +fi + +SORTED_VERSIONS=() +while IFS= read -r version; do + SORTED_VERSIONS+=("$version") +done < <(awk -F '\t' '{ print $1 }' "$VERSION_ROWS") + +if [[ -n "$LATEST_VERSION" ]]; then + found=0 + for version in "${SORTED_VERSIONS[@]}"; do + if [[ "$version" == "$LATEST_VERSION" ]]; then + found=1 + break + fi + done + if [[ "$found" -eq 0 ]]; then + echo "ERROR: LATEST_VERSION '$LATEST_VERSION' was not found among discovered versions: ${SORTED_VERSIONS[*]}" >&2 + exit 1 + fi + LATEST="$LATEST_VERSION" +else + LATEST="${SORTED_VERSIONS[${#SORTED_VERSIONS[@]}-1]}" +fi + +GENERATED_AT="$(date -u +"%Y-%m-%dT%H:%M:%SZ")" + +jq -n \ + --arg latest "$LATEST" \ + --arg generated_at "$GENERATED_AT" \ + --arg base_url "$BASE_URL" \ + --argjson versions "$(awk -F '\t' '{ print $2 }' "$VERSION_ROWS" | jq -s 'reverse')" \ + '{ + latest: $latest, + generated_at: $generated_at, + base_url: $base_url, + versions: $versions + }' > "$BASE_PATH/versions.json" + +printf '%s\n' "$LATEST" > "$BASE_PATH/latest-version.txt" +render_index_html "$LATEST" "$GENERATED_AT" "$WORK_DIR/tree.html" + +echo "Generated:" +echo " $BASE_PATH/index.html" +echo " $BASE_PATH/versions.json" +echo " $BASE_PATH/latest-version.txt" +echo "Latest version: $LATEST" +echo "Discovered versions (${#SORTED_VERSIONS[@]}): $(printf '%s ' "${SORTED_VERSIONS[@]}")" diff --git a/packaging/testing/.gitignore b/packaging/testing/.gitignore new file mode 100644 index 00000000000..a60de6218cb --- /dev/null +++ b/packaging/testing/.gitignore @@ -0,0 +1,2 @@ +preview/ +fixtures/ diff --git a/packaging/testing/generate-preview-from-s3.sh b/packaging/testing/generate-preview-from-s3.sh new file mode 100755 index 00000000000..3f2d80625d3 --- /dev/null +++ b/packaging/testing/generate-preview-from-s3.sh @@ -0,0 +1,28 @@ +#!/bin/bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" +PREVIEW_DIR="$SCRIPT_DIR/preview" +GENERATOR="$REPO_ROOT/packaging/generate-packages-index.sh" + +mkdir -p "$PREVIEW_DIR" + +if [[ -n "${AWS_S3_LISTING_FILE:-}" ]]; then + echo "Using cached S3 listing: $AWS_S3_LISTING_FILE" +else + echo "Discovering versions from s3://${AWS_S3_BUCKET:-packages.fluentbit.io} ..." +fi +echo "Writing preview to $PREVIEW_DIR" + +BASE_PATH="$PREVIEW_DIR" \ +AWS_S3_BUCKET="${AWS_S3_BUCKET:-packages.fluentbit.io}" \ +AWS_S3_REMOTE_DISCOVERY=true \ +AWS_S3_LISTING_FILE="${AWS_S3_LISTING_FILE:-}" \ +AWS_S3_NO_SIGN_REQUEST="${AWS_S3_NO_SIGN_REQUEST:-true}" \ +BASE_URL="${BASE_URL:-https://packages.fluentbit.io}" \ +"$GENERATOR" + +echo "" +echo "Open the generated page:" +echo " $PREVIEW_DIR/index.html" diff --git a/packaging/testing/test-packages-index.sh b/packaging/testing/test-packages-index.sh new file mode 100755 index 00000000000..65337e4d201 --- /dev/null +++ b/packaging/testing/test-packages-index.sh @@ -0,0 +1,239 @@ +#!/bin/bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" +GENERATOR="$REPO_ROOT/packaging/generate-packages-index.sh" +OUTPUT_DIR="$(mktemp -d)" +S3_LISTING_FILE="" + +cleanup() +{ + rm -rf "$OUTPUT_DIR" "${OUTPUT_DIR_S3:-}" "${OUTPUT_DIR_OVERRIDE:-}" \ + "${EMPTY_BASE_PATH:-}" "$S3_LISTING_FILE" +} + +trap cleanup EXIT + +if ! command -v jq >/dev/null 2>&1; then + echo "ERROR: jq is required to run this test" >&2 + exit 1 +fi + +if ! command -v tree >/dev/null 2>&1; then + echo "ERROR: tree is required to run this test" >&2 + exit 1 +fi + +setup_local_fixture() +{ + local root="$1" + + mkdir -p \ + "$root/4.2.7" \ + "$root/centos/9" \ + "$root/macos" \ + "$root/windows" \ + "$root/ubuntu/jammy/pool/main/f/fluent-bit" + + touch \ + "$root/4.2.7/fluent-bit-schema-4.2.7.json" \ + "$root/centos/9/fluent-bit-4.2.6-1.x86_64.rpm" \ + "$root/centos/9/fluent-bit-4.2.7-1.x86_64.rpm" \ + "$root/macos/fluent-bit-4.2.7.pkg" \ + "$root/ubuntu/jammy/pool/main/f/fluent-bit/fluent-bit_4.2.6_amd64.deb" \ + "$root/ubuntu/jammy/pool/main/f/fluent-bit/fluent-bit_4.2.7_amd64.deb" \ + "$root/windows/fluent-bit-4.2.6-win64.exe" \ + "$root/windows/fluent-bit-4.2.7-win64.exe" \ + "$root/windows/fluent-bit-4.2.7-win64.zip" +} + +setup_s3_listing_fixture() +{ + S3_LISTING_FILE="$(mktemp)" + cat > "$S3_LISTING_FILE" <<'EOF' +4.2.7/fluent-bit-schema-4.2.7.json +centos/9/fluent-bit-4.2.6-1.x86_64.rpm +centos/9/fluent-bit-4.2.7-1.x86_64.rpm +macos/fluent-bit-4.2.7.pkg +ubuntu/jammy/pool/main/f/fluent-bit/fluent-bit_4.2.6_amd64.deb +ubuntu/jammy/pool/main/f/fluent-bit/fluent-bit_4.2.7_amd64.deb +windows/fluent-bit-4.2.6-win64.exe +windows/fluent-bit-4.2.7-win64.exe +windows/fluent-bit-4.2.7-win64.zip +EOF +} + +assert_contains() +{ + local file="$1" + local needle="$2" + + if ! grep -Fq "$needle" "$file"; then + echo "ERROR: expected to find '$needle' in $file" >&2 + exit 1 + fi +} + +assert_equals() +{ + local file="$1" + local expected="$2" + local actual + + actual="$(<"$file")" + if [[ "$actual" != "$expected" ]]; then + echo "ERROR: expected '$expected' in $file, got '$actual'" >&2 + exit 1 + fi +} + +assert_fails_with() +{ + local expected_msg="$1" + shift + local output="" + + if output=$("$@" 2>&1); then + echo "ERROR: expected command to fail: $*" >&2 + exit 1 + fi + + if [[ -n "$expected_msg" ]] && ! grep -Fq "$expected_msg" <<< "$output"; then + echo "ERROR: expected error message '$expected_msg', got: $output" >&2 + exit 1 + fi +} + +assert_versions_json_match() +{ + local left="$1" + local right="$2" + + if ! diff <(jq -S 'del(.generated_at)' "$left") \ + <(jq -S 'del(.generated_at)' "$right") >/dev/null; then + echo "ERROR: versions.json mismatch between $left and $right" >&2 + diff -u <(jq -S 'del(.generated_at)' "$left") \ + <(jq -S 'del(.generated_at)' "$right") >&2 || true + exit 1 + fi +} + +setup_local_fixture "$OUTPUT_DIR" + +BASE_PATH="$OUTPUT_DIR" \ +AWS_S3_REMOTE_DISCOVERY=false \ +BASE_URL=https://packages.example.test \ +"$GENERATOR" + +assert_equals "$OUTPUT_DIR/latest-version.txt" "4.2.7" +assert_contains "$OUTPUT_DIR/index.html" "Latest release: 4.2.7" +assert_contains "$OUTPUT_DIR/index.html" "4.2.6" +assert_contains "$OUTPUT_DIR/index.html" "fluent-bit-4.2.7-win64.exe" +assert_contains "$OUTPUT_DIR/index.html" "https://packages.example.test/versions.json" + +if ! jq -e '.latest == "4.2.7" and (.versions | length) == 2' "$OUTPUT_DIR/versions.json" >/dev/null; then + echo "ERROR: versions.json did not contain the expected version entries" >&2 + jq . "$OUTPUT_DIR/versions.json" >&2 || true + exit 1 +fi + +if ! jq -e '.versions[] | select(.version == "4.2.7") | .artifacts.linux[] | select(.label == "centos/9 x86_64 rpm")' \ + "$OUTPUT_DIR/versions.json" >/dev/null; then + echo "ERROR: versions.json missing Linux RPM artifact for 4.2.7" >&2 + exit 1 +fi + +if ! jq -e '.versions[] | select(.version == "4.2.7") | .artifacts.linux[] | select(.label == "ubuntu/jammy amd64 deb")' \ + "$OUTPUT_DIR/versions.json" >/dev/null; then + echo "ERROR: versions.json missing Linux DEB artifact for 4.2.7" >&2 + exit 1 +fi + +if ! jq -e '.versions[] | select(.version == "4.2.7") | .artifacts.windows.win64_exe' \ + "$OUTPUT_DIR/versions.json" >/dev/null; then + echo "ERROR: versions.json missing Windows artifact for 4.2.7" >&2 + exit 1 +fi + +if ! jq -e '.versions[] | select(.version == "4.2.7") | .artifacts.macos.pkg' \ + "$OUTPUT_DIR/versions.json" >/dev/null; then + echo "ERROR: versions.json missing macOS pkg artifact for 4.2.7" >&2 + exit 1 +fi + +echo "packages-index generator test passed" + +OUTPUT_DIR_S3="$(mktemp -d)" +setup_s3_listing_fixture + +BASE_PATH="$OUTPUT_DIR_S3" \ +BASE_URL=https://packages.example.test \ +AWS_S3_BUCKET=packages.example.test \ +AWS_S3_REMOTE_DISCOVERY=true \ +AWS_S3_LISTING_FILE="$S3_LISTING_FILE" \ +"$GENERATOR" + +assert_equals "$OUTPUT_DIR_S3/latest-version.txt" "4.2.7" +assert_contains "$OUTPUT_DIR_S3/index.html" "fluent-bit-4.2.7-win64.exe" +assert_versions_json_match "$OUTPUT_DIR/versions.json" "$OUTPUT_DIR_S3/versions.json" + +if ! jq -e '.latest == "4.2.7" and .base_url == "https://packages.example.test" and (.versions | length) == 2' \ + "$OUTPUT_DIR_S3/versions.json" >/dev/null; then + echo "ERROR: S3 listing parse did not produce expected catalog metadata" >&2 + jq . "$OUTPUT_DIR_S3/versions.json" >&2 || true + exit 1 +fi + +echo "packages-index S3 listing parse test passed" + +OUTPUT_DIR_OVERRIDE="$(mktemp -d)" +setup_local_fixture "$OUTPUT_DIR_OVERRIDE" + +BASE_PATH="$OUTPUT_DIR_OVERRIDE" \ +AWS_S3_REMOTE_DISCOVERY=false \ +LATEST_VERSION=4.2.6 \ +BASE_URL=https://packages.example.test \ +"$GENERATOR" + +assert_equals "$OUTPUT_DIR_OVERRIDE/latest-version.txt" "4.2.6" +if ! jq -e '.latest == "4.2.6"' "$OUTPUT_DIR_OVERRIDE/versions.json" >/dev/null; then + echo "ERROR: LATEST_VERSION override did not update latest field" >&2 + exit 1 +fi + +assert_fails_with "LATEST_VERSION '9.9.9' was not found among discovered versions" \ + env BASE_PATH="$OUTPUT_DIR" AWS_S3_REMOTE_DISCOVERY=false LATEST_VERSION=9.9.9 \ + BASE_URL=https://packages.example.test "$GENERATOR" + +EMPTY_BASE_PATH="$(mktemp -d)" +assert_fails_with "ERROR: no objects found under" \ + env BASE_PATH="$EMPTY_BASE_PATH" AWS_S3_REMOTE_DISCOVERY=false \ + BASE_URL=https://packages.example.test "$GENERATOR" + +assert_fails_with "AWS_S3_LISTING_FILE does not exist" \ + env BASE_PATH="$OUTPUT_DIR_S3" AWS_S3_REMOTE_DISCOVERY=true \ + AWS_S3_BUCKET=packages.example.test \ + AWS_S3_LISTING_FILE=/tmp/does-not-exist-packages-index-test \ + BASE_URL=https://packages.example.test "$GENERATOR" + +echo "packages-index validation error handling test passed" + +PREVIEW_SCRIPT="$SCRIPT_DIR/generate-preview-from-s3.sh" +PREVIEW_DIR="$SCRIPT_DIR/preview" + +bash -n "$PREVIEW_SCRIPT" + +AWS_S3_LISTING_FILE="$S3_LISTING_FILE" \ +AWS_S3_BUCKET=packages.example.test \ +"$PREVIEW_SCRIPT" + +test -f "$PREVIEW_DIR/index.html" +assert_equals "$PREVIEW_DIR/latest-version.txt" "4.2.7" +if ! jq -e '.latest == "4.2.7"' "$PREVIEW_DIR/versions.json" >/dev/null; then + echo "ERROR: preview versions.json did not identify 4.2.7 as latest" >&2 + jq . "$PREVIEW_DIR/versions.json" >&2 || true + exit 1 +fi + +echo "packages-index preview script test passed"