diff --git a/.github/workflows/call-windows-unit-tests.yaml b/.github/workflows/call-windows-unit-tests.yaml index 0797f302229..d9a37533959 100644 --- a/.github/workflows/call-windows-unit-tests.yaml +++ b/.github/workflows/call-windows-unit-tests.yaml @@ -37,18 +37,21 @@ jobs: - name: "Windows 32bit" arch: x86 cmake_additional_opt: "" + runtime_tests: "Off" vcpkg_triplet: x86-windows-static cmake_version: "3.31.6" os: windows-latest - name: "Windows 64bit" arch: x64 cmake_additional_opt: "" + runtime_tests: "On" vcpkg_triplet: x64-windows-static cmake_version: "3.31.6" os: windows-latest - name: "Windows 64bit (Arm64)" arch: amd64_arm64 cmake_additional_opt: "-DCMAKE_SYSTEM_NAME=Windows -DCMAKE_SYSTEM_VERSION=10.0 -DCMAKE_SYSTEM_PROCESSOR=ARM64" + runtime_tests: "Off" vcpkg_triplet: arm64-windows-static cmake_version: "3.31.6" os: windows-11-arm @@ -207,6 +210,7 @@ jobs: run: | cmake -G "NMake Makefiles" ` -D FLB_TESTS_INTERNAL=On ` + -D FLB_TESTS_RUNTIME=${{ matrix.config.runtime_tests }} ` -D FLB_NIGHTLY_BUILD='${{ inputs.unstable }}' ` -D OPENSSL_ROOT_DIR='C:\vcpkg\installed\${{ matrix.config.vcpkg_triplet }}' ` ${{ matrix.config.cmake_additional_opt }} ` @@ -214,6 +218,7 @@ jobs: -D FLB_WITHOUT_flb-rt-out_elasticsearch=On ` -D FLB_WITHOUT_flb-rt-out_td=On ` -D FLB_WITHOUT_flb-rt-out_forward=On ` + -D FLB_WITHOUT_flb-rt-filter_rewrite_tag=On ` -D FLB_WITHOUT_flb-rt-in_disk=On ` -D FLB_WITHOUT_flb-rt-in_proc=On ` -D FLB_WITHOUT_flb-it-unit_sizes=On ` @@ -247,6 +252,12 @@ jobs: - name: Run unit tests for Fluent Bit packages (x86, x64, and ARM64) run: | - ctest --build-run-dir "$PWD" --output-on-failure + $nparallel = 1 + if ('${{ matrix.config.runtime_tests }}' -eq 'On') { + $nparallel = [Math]::Min([Environment]::ProcessorCount, 8) + } + + Write-Host "Running CTest with parallel level $nparallel" + ctest --parallel $nparallel --build-run-dir "$PWD" --output-on-failure shell: pwsh working-directory: build diff --git a/.github/workflows/cron-scorecards-analysis.yaml b/.github/workflows/cron-scorecards-analysis.yaml index dedb6946279..02fc014c578 100644 --- a/.github/workflows/cron-scorecards-analysis.yaml +++ b/.github/workflows/cron-scorecards-analysis.yaml @@ -46,7 +46,7 @@ jobs: # Upload the results to GitHub's code scanning dashboard. - name: "Upload to code-scanning" - uses: github/codeql-action/upload-sarif@54f647b7e1bb85c95cddabcd46b0c578ec92bc1a # v4.36.3 + uses: github/codeql-action/upload-sarif@7188fc363630916deb702c7fdcf4e481b751f97a # v4.37.1 with: sarif_file: results.sarif category: ossf-scorecard diff --git a/.github/workflows/cron-stale.yaml b/.github/workflows/cron-stale.yaml index d77c8bf2ebe..6f982aa7d5d 100644 --- a/.github/workflows/cron-stale.yaml +++ b/.github/workflows/cron-stale.yaml @@ -12,7 +12,7 @@ jobs: name: Mark stale runs-on: ubuntu-latest steps: - - uses: actions/stale@eb5cf3af3ac0a1aa4c9c45633dd1ae542a27a899 # v10.3.0 + - uses: actions/stale@1e223db275d687790206a7acac4d1a11bd6fe629 # v10.4.0 with: repo-token: ${{ secrets.GITHUB_TOKEN }} stale-issue-message: 'This issue is stale because it has been open 90 days with no activity. Remove stale label or comment or this will be closed in 5 days. Maintainers can add the `exempt-stale` label.' diff --git a/.github/workflows/pr-fuzz.yaml b/.github/workflows/pr-fuzz.yaml index 4c4e6599009..eb159a4db77 100644 --- a/.github/workflows/pr-fuzz.yaml +++ b/.github/workflows/pr-fuzz.yaml @@ -36,7 +36,7 @@ jobs: path: ./out/artifacts - name: Upload Sarif if: always() && steps.build.outcome == 'success' - uses: github/codeql-action/upload-sarif@54f647b7e1bb85c95cddabcd46b0c578ec92bc1a # v4.36.3 + uses: github/codeql-action/upload-sarif@7188fc363630916deb702c7fdcf4e481b751f97a # v4.37.1 with: # Path to SARIF file relative to the root of the repository sarif_file: cifuzz-sarif/results.sarif diff --git a/.github/workflows/staging-release.yaml b/.github/workflows/staging-release.yaml index a2a6660228d..85d365e5a9a 100644 --- a/.github/workflows/staging-release.yaml +++ b/.github/workflows/staging-release.yaml @@ -806,7 +806,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Release 2.0 - not latest - uses: softprops/action-gh-release@718ea10b132b3b2eba29c1007bb80653f286566b # v3.0.1 + uses: softprops/action-gh-release@3d0d9888cb7fd7b750713d6e236d1fcb99157228 # v3.0.2 if: startsWith(inputs.version, '2.0') with: body: "https://fluentbit.io/announcements/v${{ inputs.version }}/" @@ -818,7 +818,7 @@ jobs: make_latest: false - name: Release 2.1 - not latest - uses: softprops/action-gh-release@718ea10b132b3b2eba29c1007bb80653f286566b # v3.0.1 + uses: softprops/action-gh-release@3d0d9888cb7fd7b750713d6e236d1fcb99157228 # v3.0.2 if: startsWith(inputs.version, '2.1') with: body: "https://fluentbit.io/announcements/v${{ inputs.version }}/" @@ -830,7 +830,7 @@ jobs: make_latest: false - name: Release 3.0 - not latest - uses: softprops/action-gh-release@718ea10b132b3b2eba29c1007bb80653f286566b # v3.0.1 + uses: softprops/action-gh-release@3d0d9888cb7fd7b750713d6e236d1fcb99157228 # v3.0.2 if: startsWith(inputs.version, '3.0') with: body: "https://fluentbit.io/announcements/v${{ inputs.version }}/" @@ -842,7 +842,7 @@ jobs: make_latest: false - name: Release 3.1 - not latest - uses: softprops/action-gh-release@718ea10b132b3b2eba29c1007bb80653f286566b # v3.0.1 + uses: softprops/action-gh-release@3d0d9888cb7fd7b750713d6e236d1fcb99157228 # v3.0.2 if: startsWith(inputs.version, '3.1') with: body: "https://fluentbit.io/announcements/v${{ inputs.version }}/" @@ -854,7 +854,7 @@ jobs: make_latest: false - name: Release 3.2 - not latest - uses: softprops/action-gh-release@718ea10b132b3b2eba29c1007bb80653f286566b # v3.0.1 + uses: softprops/action-gh-release@3d0d9888cb7fd7b750713d6e236d1fcb99157228 # v3.0.2 if: startsWith(inputs.version, '3.2') with: body: "https://fluentbit.io/announcements/v${{ inputs.version }}/" @@ -866,7 +866,7 @@ jobs: make_latest: false - name: Release 4.0 - not latest - uses: softprops/action-gh-release@718ea10b132b3b2eba29c1007bb80653f286566b # v3.0.1 + uses: softprops/action-gh-release@3d0d9888cb7fd7b750713d6e236d1fcb99157228 # v3.0.2 if: startsWith(inputs.version, '4.0') with: body: "https://fluentbit.io/announcements/v${{ inputs.version }}/" @@ -878,7 +878,7 @@ jobs: make_latest: false - name: Release 4.1 - not latest - uses: softprops/action-gh-release@718ea10b132b3b2eba29c1007bb80653f286566b # v3.0.1 + uses: softprops/action-gh-release@3d0d9888cb7fd7b750713d6e236d1fcb99157228 # v3.0.2 if: startsWith(inputs.version, '4.1') with: body: "https://fluentbit.io/announcements/v${{ inputs.version }}/" @@ -890,7 +890,7 @@ jobs: make_latest: false - name: Release 4.2 - not latest - uses: softprops/action-gh-release@718ea10b132b3b2eba29c1007bb80653f286566b # v3.0.1 + uses: softprops/action-gh-release@3d0d9888cb7fd7b750713d6e236d1fcb99157228 # v3.0.2 if: startsWith(inputs.version, '4.2') with: body: "https://fluentbit.io/announcements/v${{ inputs.version }}/" @@ -902,7 +902,7 @@ jobs: make_latest: false - name: Release 5.0 and latest - uses: softprops/action-gh-release@718ea10b132b3b2eba29c1007bb80653f286566b # v3.0.1 + uses: softprops/action-gh-release@3d0d9888cb7fd7b750713d6e236d1fcb99157228 # v3.0.2 if: startsWith(inputs.version, '5.0') with: body: "https://fluentbit.io/announcements/v${{ inputs.version }}/" diff --git a/.github/workflows/unit-tests.yaml b/.github/workflows/unit-tests.yaml index fc5ccbb9123..4403bc845ac 100644 --- a/.github/workflows/unit-tests.yaml +++ b/.github/workflows/unit-tests.yaml @@ -22,6 +22,7 @@ on: - 'examples/**' branches: - master + - 5.0 - 4.2 - 4.1 - 4.0 diff --git a/AGENTS.md b/AGENTS.md index d064787137d..8797c9aa0e8 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -2,10 +2,17 @@ ## Preferred Commands - Configure: `cmake -S . -B build -DFLB_TESTS_RUNTIME=On -DFLB_TESTS_INTERNAL=On` +- Configure on Windows: + `cmake -S . -B build -DFLB_TESTS_RUNTIME=On -DFLB_TESTS_INTERNAL=On` - Build: `cmake --build build -j8` - Test: `ctest --test-dir build --output-on-failure` - Prefer targeted tests with `ctest --test-dir build -R --output-on-failure` when the affected area is known, because the full enabled suite can be slow. +- Windows supports building and running runtime tests. Prefer focused + `flb-rt-*` targets or CTest matches because the full runtime suite can be + slow. The GitHub Actions unit-test workflow enables runtime execution only + for x64 to control CI running time. Do not apply that CI-only restriction to + local agents or AI cloud builds. - Run a focused integration test with `ctest --test-dir build -R flb-it-opentelemetry --output-on-failure` - Run the in-tree Python integration suite with: @@ -29,8 +36,22 @@ Fluent Bit is a C/C++ monorepo built with CMake. Keep changes scoped: plugin logic in its plugin directory, shared behavior in `src/` or `lib/`. +## Bundled Library Changes +- Treat `lib/` as bundled third-party or separately maintained code unless the + specific path is clearly Fluent Bit-owned. +- Before editing bundled library code, ask for explicit user confirmation. If + the agent environment supports confirmation popups, use one; otherwise ask in + chat before writing files. +- Prefer changes that can be sent upstream as a focused patch. Keep bundled + library patches isolated from Fluent Bit glue code, and document the upstream + project/path in the close-out. +- Do not mix bundled library edits with unrelated Fluent Bit core, plugin, + documentation, or test changes in the same commit unless the user explicitly + asks for that structure. + ## Build, Test, and Development Commands -- `cmake -S . -B build -DFLB_TESTS_RUNTIME=On -DFLB_TESTS_INTERNAL=On`: configure with runtime + internal tests. +- `cmake -S . -B build -DFLB_TESTS_RUNTIME=On -DFLB_TESTS_INTERNAL=On`: + configure runtime and internal tests, including on Windows. - `cmake --build build -j8`: compile Fluent Bit and tests. - `ctest --test-dir build --output-on-failure`: run enabled tests. - `ctest --test-dir build -R flb-it-opentelemetry --output-on-failure`: run a focused integration test. @@ -63,6 +84,12 @@ Keep changes scoped: plugin logic in its plugin directory, shared behavior in `s - Add or update tests for behavior changes, especially protocol parsing and encoder/decoder paths. - Prefer targeted tests close to the changed module (`tests/internal`, plugin runtime tests). - Prefer focused `ctest -R ...` runs or specific test binaries when the touched area is known. +- Windows supports `tests/runtime`, `flb-rt-*` targets, and runtime CTest + matches. Configure with `-DFLB_TESTS_RUNTIME=On` and run applicable focused + runtime coverage. In `.github/workflows/call-windows-unit-tests.yaml`, keep + runtime execution disabled for x86 and ARM64 unless the workflow scope + explicitly changes; that exclusion controls GitHub Actions running time only. + It does not apply to local agents or AI cloud builds. - Use `tests/integration` when validating end-to-end plugin behavior, network protocols, downstream request generation, or local fake-server interactions that are awkward to cover in `ctest` binaries alone. @@ -81,6 +108,9 @@ Keep changes scoped: plugin logic in its plugin directory, shared behavior in `s `cmake --build build -j8` `tests/integration/.venv/bin/python -m pytest -q` `VALGRIND=1 VALGRIND_STRICT=1 tests/integration/.venv/bin/python -m pytest -q` +- On Windows, use the same `-DFLB_TESTS_RUNTIME=On` configuration and run + relevant focused runtime cases. Valgrind is normally unavailable on Windows; + report that exact blocker instead of conflating it with runtime-test support. - Run broader test coverage when changing shared lifecycle, routing, storage, or accounting code. - Validate both success and failure paths (invalid payloads, boundary sizes, null/missing fields). - You can also run specific binaries from `build/bin` (e.g., `./bin/flb-it-opentelemetry`). @@ -167,7 +197,8 @@ Keep changes scoped: plugin logic in its plugin directory, shared behavior in `s path inference before choosing a commit subject. For example, changes only to `AGENTS.md` must use `agents:`, not `docs:`. - For pull-request-style validation, use full history and fetch the base branch - first, matching CI behavior: + first, matching CI behavior. This is required because the checker can fall + back to validating only `HEAD` when the base ref is unavailable: `git fetch --all --prune` `git fetch origin :origin/` - Before pushing a branch or opening/updating a PR, agents must lint the full @@ -185,6 +216,7 @@ Keep changes scoped: plugin logic in its plugin directory, shared behavior in `s - Do not open issues, pull requests, or remote branches unless the user explicitly asks. - Do not rewrite git history, amend commits, or force-push unless the user explicitly asks. - Do not revert user changes outside the requested scope. +- Do not edit bundled libraries under `lib/` without explicit confirmation. - Prefer minimal patches that avoid unrelated formatting or refactoring churn. ## Agent Playbook (Pipeline Architecture Primer) @@ -255,7 +287,8 @@ Keep changes scoped: plugin logic in its plugin directory, shared behavior in `s ### Testing strategy - Use `tests/internal` for core lifecycle/accounting logic. -- Use `tests/runtime` for plugin-level behavior and end-to-end semantics. +- Use `tests/runtime` for plugin-level behavior and end-to-end semantics, + including Windows runtime targets supported by the active toolchain and host. - Add regression tests for: - mixed signals - processor drop/modify paths diff --git a/CMakeLists.txt b/CMakeLists.txt index 2f545f5bc08..2043d1b409a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -9,8 +9,8 @@ set(CMAKE_POLICY_DEFAULT_CMP0069 NEW) # Fluent Bit Version set(FLB_VERSION_MAJOR 5) -set(FLB_VERSION_MINOR 0) -set(FLB_VERSION_PATCH 9) +set(FLB_VERSION_MINOR 1) +set(FLB_VERSION_PATCH 0) set(FLB_VERSION_STR "${FLB_VERSION_MAJOR}.${FLB_VERSION_MINOR}.${FLB_VERSION_PATCH}") set(CMAKE_POSITION_INDEPENDENT_CODE ON) @@ -374,6 +374,7 @@ if(FLB_ALL) # Output plugins set(FLB_OUT_ES 1) set(FLB_OUT_FORWARD 1) + set(FLB_OUT_GCS 1) set(FLB_OUT_GELF 1) set(FLB_OUT_HTTP 1) set(FLB_OUT_NATS 1) @@ -816,6 +817,7 @@ add_subdirectory(${FLB_PATH_LIB_JSMN}) # Runtime Tests (filter_kubernetes) requires HTTP Server if(FLB_TESTS_RUNTIME) FLB_OPTION(FLB_HTTP_SERVER ON) + FLB_OPTION(FLB_IN_EVENT_TEST ON) endif() # Monkey Core Library @@ -1453,8 +1455,8 @@ if(NOT FLB_POSIX_TLS) check_c_source_compiles(" __thread int a; int main() { - __tls_get_addr(0); - return 0; + a = 1; + return a - 1; }" FLB_HAVE_C_TLS) if(FLB_HAVE_C_TLS) FLB_DEFINITION(FLB_HAVE_C_TLS) @@ -1530,6 +1532,14 @@ else() endif() # Instruct CMake to build the Fluent Bit Core +if(MSVC OR FLB_JEMALLOC) + # MSVC tests need plugin-private symbols that are not exported from fluent-bit.dll. + # Jemalloc must not be embedded in both the test executable and shared library. + set(FLB_TEST_LINK_LIBRARY fluent-bit-static) +else() + set(FLB_TEST_LINK_LIBRARY fluent-bit-shared) +endif() + add_subdirectory(include) add_subdirectory(plugins) add_subdirectory(src) diff --git a/MAINTENANCE.md b/MAINTENANCE.md index 138b398ad5b..183d1372a3e 100644 --- a/MAINTENANCE.md +++ b/MAINTENANCE.md @@ -6,15 +6,16 @@ This document outlines the maintenance strategy and version support for Fluent B | Branch | Version | Status | Maintainer | Notes | |------------|--------------------|---------------------|-------------------------------------------------------------|--------------------------------------------------------------------| -| `master` | v5.0 (development) | Active development | [Eduardo Silva](https://github.com/edsiper) | All new features and bug fixes land here first | -| `4.2` | v4.2.x | Stable | [Eduardo Silva](https://github.com/edsiper) [Hiroshi Hatake (@cosmo0920)](https://github.com/cosmo0920) | Current stable release series. Active development and updates. May receive minor enhancements in addition to fixes. | +| `master` | v5.1 (development) | Active development | [Eduardo Silva](https://github.com/edsiper) | Next minor development line. All new features and bug fixes land here first | +| `5.0` | v5.0.x | Stable | [Eduardo Silva](https://github.com/edsiper) [Hiroshi Hatake (@cosmo0920)](https://github.com/cosmo0920) | Current stable release series. Receives bug fixes, security fixes, and selected low-risk backports. | +| `4.2` | v4.2.x | Maintenance only | [Eduardo Silva](https://github.com/edsiper) [Hiroshi Hatake (@cosmo0920)](https://github.com/cosmo0920) | Critical fixes and safe backports only. Maintained until **July 30, 2026** | | `4.1` | v4.1.x | Maintenance only | [Hiroshi Hatake (@cosmo0920)](https://github.com/cosmo0920) | Critical fixes and safe backports only. Maintained until **February 28, 2026** | --- ## Maintenance Policy -Active development is currently on Fluent Bit **v5.0** (tracked in the `master` branch). The **v4.2** branch is the current stable release series and receives active updates. Previous release lines enter **maintenance mode** after the next major/minor release. +Active development is currently on the next Fluent Bit **v5.1** line (tracked in the `master` branch). The **v5.0** branch is the current stable release series and receives stable branch updates. Previous release lines enter **maintenance mode** after the next major/minor release. ### Accepted Changes for Maintenance Branches @@ -26,9 +27,13 @@ Active development is currently on Fluent Bit **v5.0** (tracked in the `master` Maintenance releases continue on an as-needed basis depending on urgency and impact. -### v4.2 Stable Series +### v5.0 Stable Series -**v4.2** is the current stable release series and receives active development, bug fixes, and security updates. This is the recommended version for production use. +**v5.0** is the current stable release series and receives bug fixes, security updates, and selected low-risk backports. This is the recommended version for production use. + +### v4.2 Maintenance + +**v4.2** has entered **maintenance mode** and receives critical fixes and safe backports until **July 30, 2026** (as specified in [SECURITY.md](SECURITY.md)). ### v4.1 Maintenance @@ -44,11 +49,12 @@ Maintenance releases continue on an as-needed basis depending on urgency and imp ## How to Contribute to Maintained Versions -If you're submitting a fix or feature relevant to a stable or maintenance branch (v4.2 or v4.1): +If you're submitting a fix or feature relevant to a stable or maintenance branch (v5.0, v4.2, or v4.1): - Open your PR against the `master` branch -- Add a note in the PR or issue: `Target: v4.2` or `Target: v4.1` +- Add a note in the PR or issue: `Target: v5.0`, `Target: v4.2`, or `Target: v4.1` - Tag the branch maintainer to request backport consideration: + - For v5.0: [@edsiper](https://github.com/edsiper) [@cosmo0920](https://github.com/cosmo0920) - For v4.2: [@edsiper](https://github.com/edsiper) [@cosmo0920](https://github.com/cosmo0920) - For v4.1: [@cosmo0920](https://github.com/cosmo0920) diff --git a/README.md b/README.md index 73c28a7c50a..4cbb189f390 100644 --- a/README.md +++ b/README.md @@ -26,7 +26,7 @@ Fluent Bit supports a wide array of platforms, including Linux, Windows, MacOS, ## 📌 Roadmap & Maintenance We follow a fast-paced development cycle, with major releases every 3–4 months. -The active development branch (`master`) is currently focused on **v5.0** (development). +The active development branch (`master`) is currently focused on **v5.1** (development). For version-specific maintenance timelines and policies, see our [MAINTENANCE.md](https://github.com/fluent/fluent-bit/blob/master/MAINTENANCE.md). diff --git a/cmake/headers.cmake b/cmake/headers.cmake index 663bb180601..24e3af00d27 100755 --- a/cmake/headers.cmake +++ b/cmake/headers.cmake @@ -7,6 +7,7 @@ if(NOT DEFINED FLB_PATH_ROOT_BINARY_DIR) endif() include_directories( + ${FLB_PATH_ROOT_BINARY_DIR}/generated/include ${FLB_PATH_ROOT_SOURCE}/include/ ${FLB_PATH_ROOT_SOURCE}/lib/ diff --git a/cmake/plugins_options.cmake b/cmake/plugins_options.cmake index 7fbd0a96bd5..e6e12e1aee9 100644 --- a/cmake/plugins_options.cmake +++ b/cmake/plugins_options.cmake @@ -61,6 +61,7 @@ DEFINE_OPTION(FLB_IN_TCP "Enable TCP input plugin" DEFINE_OPTION(FLB_IN_THERMAL "Enable Thermal plugin" ON) DEFINE_OPTION(FLB_IN_UDP "Enable UDP input plugin" ON) DEFINE_OPTION(FLB_IN_UNIX_SOCKET "Enable Unix socket input plugin" OFF) +DEFINE_OPTION(FLB_IN_ETW "Enable Event Tracing for Windows input plugin" OFF) DEFINE_OPTION(FLB_IN_WINLOG "Enable Windows Log input plugin" OFF) DEFINE_OPTION(FLB_IN_WINDOWS_EXPORTER_METRICS "Enable windows exporter metrics input plugin" ON) DEFINE_OPTION(FLB_IN_WINEVTLOG "Enable Windows EvtLog input plugin" OFF) @@ -122,6 +123,7 @@ DEFINE_OPTION(FLB_OUT_EXIT "Enable Exit output plugin" DEFINE_OPTION(FLB_OUT_FILE "Enable file output plugin" ON) DEFINE_OPTION(FLB_OUT_FLOWCOUNTER "Enable flowcount output plugin" ON) DEFINE_OPTION(FLB_OUT_FORWARD "Enable Forward output plugin" ON) +DEFINE_OPTION(FLB_OUT_GCS "Enable GCS output plugin" ON) DEFINE_OPTION(FLB_OUT_GELF "Enable GELF output plugin" ON) DEFINE_OPTION(FLB_OUT_HTTP "Enable HTTP output plugin" ON) DEFINE_OPTION(FLB_OUT_INFLUXDB "Enable InfluxDB output plugin" ON) diff --git a/cmake/windows-setup.cmake b/cmake/windows-setup.cmake index f253382a0cc..b3306266e8f 100644 --- a/cmake/windows-setup.cmake +++ b/cmake/windows-setup.cmake @@ -71,6 +71,7 @@ if(FLB_WINDOWS_DEFAULTS) set(FLB_IN_SYSTEMD No) set(FLB_IN_DUMMY Yes) set(FLB_IN_NETIF No) + set(FLB_IN_ETW Yes) set(FLB_IN_WINLOG Yes) set(FLB_IN_WINSTAT Yes) set(FLB_IN_WINEVTLOG Yes) @@ -94,7 +95,7 @@ if(FLB_WINDOWS_DEFAULTS) set(FLB_OUT_CHRONICLE Yes) set(FLB_OUT_DATADOG Yes) set(FLB_OUT_ES Yes) - set(FLB_OUT_EXIT No) + set(FLB_OUT_EXIT Yes) set(FLB_OUT_FORWARD Yes) set(FLB_OUT_GELF Yes) set(FLB_OUT_HTTP Yes) diff --git a/conf/fluent-bit.conf b/conf/fluent-bit.conf index bf3269f38e2..c766bacfc58 100644 --- a/conf/fluent-bit.conf +++ b/conf/fluent-bit.conf @@ -4,6 +4,57 @@ # set an interval of seconds before to flush records to a destination flush 1 + # flush.adaptive + # -------------- + # Enable adaptive flush interval adjustments based on output chunk + # backpressure. + # + # flush.adaptive off + + # flush.adaptive.min_interval + # --------------------------- + # lower bound for adaptive flush interval in seconds. + # + # flush.adaptive.min_interval 0.5 + + # flush.adaptive.max_interval + # --------------------------- + # upper bound for adaptive flush interval in seconds. + # + # flush.adaptive.max_interval 2.0 + + # flush.adaptive.low_pressure + # --------------------------- + # output chunk pressure (%) threshold considered mostly idle. + # + # flush.adaptive.low_pressure 25 + + # flush.adaptive.medium_pressure + # ------------------------------ + # output chunk pressure (%) threshold considered moderate pressure. + # + # flush.adaptive.medium_pressure 50 + + # flush.adaptive.high_pressure + # ---------------------------- + # output chunk pressure (%) threshold considered sustained pressure. + # + # flush.adaptive.high_pressure 75 + + # flush.adaptive.up_steps + # ----------------------- + # consecutive pressure samples required before moving to a faster flush + # step. minimum: 1 + # + # flush.adaptive.up_steps 2 + + # flush.adaptive.down_steps + # ------------------------- + # consecutive idle samples required before moving to a slower flush step. + # minimum: 1 + # + # flush.adaptive.down_steps 3 + # Daemon # ====== # instruct Fluent Bit to run in foreground or background mode. diff --git a/dockerfiles/Dockerfile b/dockerfiles/Dockerfile index cbb4f465188..3927debabad 100644 --- a/dockerfiles/Dockerfile +++ b/dockerfiles/Dockerfile @@ -13,7 +13,7 @@ # docker buildx build --platform "linux/amd64,linux/arm64,linux/arm/v7,linux/s390x" -f ./dockerfiles/Dockerfile.multiarch --build-arg FLB_TARBALL=https://github.com/fluent/fluent-bit/archive/v1.8.11.tar.gz ./dockerfiles/ # Set this to the current release version: it gets done so as part of the release. -ARG RELEASE_VERSION=5.0.9 +ARG RELEASE_VERSION=5.1.0 # For multi-arch builds - assumption is running on an AMD64 host FROM multiarch/qemu-user-static:x86_64-arm AS qemu-arm32 diff --git a/fluent-bit-5.0.9.bb b/fluent-bit-5.1.0.bb similarity index 99% rename from fluent-bit-5.0.9.bb rename to fluent-bit-5.1.0.bb index 816ba0e46a4..e822f9b4189 100644 --- a/fluent-bit-5.0.9.bb +++ b/fluent-bit-5.1.0.bb @@ -16,7 +16,7 @@ LIC_FILES_CHKSUM = "file://LICENSE;md5=2ee41112a44fe7014dce33e26468ba93" SECTION = "net" PR = "r0" -PV = "5.0.9" +PV = "5.1.0" SRCREV = "v${PV}" SRC_URI = "git://github.com/fluent/fluent-bit.git;nobranch=1" diff --git a/include/fluent-bit/aws/flb_aws_compress.h b/include/fluent-bit/aws/flb_aws_compress.h index ada6c83191e..86805790075 100644 --- a/include/fluent-bit/aws/flb_aws_compress.h +++ b/include/fluent-bit/aws/flb_aws_compress.h @@ -23,8 +23,6 @@ #include #define FLB_AWS_COMPRESS_NONE 0 #define FLB_AWS_COMPRESS_GZIP 1 -#define FLB_AWS_COMPRESS_ARROW 2 -#define FLB_AWS_COMPRESS_PARQUET 3 #define FLB_AWS_COMPRESS_ZSTD 4 #define FLB_AWS_COMPRESS_SNAPPY 5 @@ -63,4 +61,25 @@ int flb_aws_compression_b64_truncate_compress(int compression_type, size_t max_o void *in_data, size_t in_len, void **out_data, size_t *out_len); +/* + * Columnar output formats for out_s3_compress_columnar(). Compression is + * applied on top of the format via a generic FLB_AWS_COMPRESS_* codec. + */ +#define FLB_AWS_COMPRESS_FORMAT_ARROW 0 +#define FLB_AWS_COMPRESS_FORMAT_PARQUET 1 + +/* + * Convert JSON data to a columnar format (Apache Arrow/Feather or Apache + * Parquet) selected by `columnar_format` (FLB_AWS_COMPRESS_FORMAT_*), + * applying `compression_type` (a generic FLB_AWS_COMPRESS_* codec) on top of + * the format: + * - Parquet supports NONE, SNAPPY, GZIP and ZSTD (page-level codec). + * - Arrow/Feather supports NONE and ZSTD only. + * + * Returns 0 on success, -1 on failure. + */ +int out_s3_compress_columnar(int columnar_format, void *json, size_t size, + void **out_buf, size_t *out_size, + int compression_type); + #endif diff --git a/include/fluent-bit/flb_config.h b/include/fluent-bit/flb_config.h index 3ebcb5edbae..181879c501a 100644 --- a/include/fluent-bit/flb_config.h +++ b/include/fluent-bit/flb_config.h @@ -64,6 +64,18 @@ struct flb_config { int is_shutting_down; /* is the service shutting down ? */ int is_running; /* service running ? */ double flush; /* Flush timeout */ + int flush_adaptive; /* Enable adaptive flush interval */ + double flush_adaptive_min_interval; + double flush_adaptive_max_interval; + double flush_adaptive_low_pressure; + double flush_adaptive_medium_pressure; + double flush_adaptive_high_pressure; + int flush_adaptive_up_steps; + int flush_adaptive_down_steps; + int flush_adaptive_level; + int flush_adaptive_hits; + int flush_adaptive_direction; + double flush_adaptive_current_interval; /* * Maximum grace time on shutdown. If set to -1, the engine will @@ -291,6 +303,9 @@ struct flb_config { int enable_chunk_trace; #endif /* FLB_HAVE_CHUNK_TRACE */ + int fips_mode; + int fips_mode_active; + int enable_hot_reload; int ensure_thread_safety_on_hot_reloading; unsigned int hot_reloaded_count; @@ -369,6 +384,14 @@ enum conf_type { }; #define FLB_CONF_STR_FLUSH "Flush" +#define FLB_CONF_STR_FLUSH_ADAPTIVE "flush.adaptive" +#define FLB_CONF_STR_FLUSH_ADAPTIVE_MIN "flush.adaptive.min_interval" +#define FLB_CONF_STR_FLUSH_ADAPTIVE_MAX "flush.adaptive.max_interval" +#define FLB_CONF_STR_FLUSH_ADAPTIVE_LOW "flush.adaptive.low_pressure" +#define FLB_CONF_STR_FLUSH_ADAPTIVE_MEDIUM "flush.adaptive.medium_pressure" +#define FLB_CONF_STR_FLUSH_ADAPTIVE_HIGH "flush.adaptive.high_pressure" +#define FLB_CONF_STR_FLUSH_ADAPTIVE_UP_STEPS "flush.adaptive.up_steps" +#define FLB_CONF_STR_FLUSH_ADAPTIVE_DOWN_STEPS "flush.adaptive.down_steps" #define FLB_CONF_STR_GRACE "Grace" #define FLB_CONF_STR_DAEMON "Daemon" #define FLB_CONF_STR_LOGFILE "Log_File" @@ -378,6 +401,7 @@ enum conf_type { #define FLB_CONF_STR_STREAMS_FILE "Streams_File" #define FLB_CONF_STR_STREAMS_STR_CONV "sp.convert_from_str_to_num" #define FLB_CONF_STR_CONV_NAN "json.convert_nan_to_null" +#define FLB_CONF_STR_FIPS_MODE "security.fips_mode" /* FLB_HAVE_HTTP_SERVER */ #ifdef FLB_HAVE_HTTP_SERVER diff --git a/include/fluent-bit/flb_connection.h b/include/fluent-bit/flb_connection.h index 31801e71e09..3eed648e61d 100644 --- a/include/fluent-bit/flb_connection.h +++ b/include/fluent-bit/flb_connection.h @@ -62,6 +62,7 @@ struct flb_connection; typedef void (*flb_connection_drop_notification_callback)( struct flb_connection *connection); +typedef int (*flb_connection_event_callback)(void *data); /* Base network connection */ struct flb_connection { @@ -152,6 +153,15 @@ struct flb_connection { /* Coroutine in charge of this connection */ struct flb_coro *coroutine; + /* Downstream-owned event callback coroutine */ + struct flb_coro *event_coroutine; + flb_connection_event_callback event_callback; + int event_wakeup_pending; + int event_release_pending; + + /* Per-connection I/O flags */ + int flags; + /* Connection type : FLB_UPSTREAM_CONNECTION or FLB_DOWNSTREAM_CONNECTION */ int type; @@ -189,6 +199,8 @@ void flb_connection_set_remote_host(struct flb_connection *connection, char *flb_connection_get_remote_address(struct flb_connection *connection); int flb_connection_get_flags(struct flb_connection *connection); +void flb_connection_enable_flags(struct flb_connection *connection, int flags); +void flb_connection_disable_flags(struct flb_connection *connection, int flags); void flb_connection_reset_connection_timeout(struct flb_connection *connection); void flb_connection_unset_connection_timeout(struct flb_connection *connection); diff --git a/include/fluent-bit/flb_coro.h b/include/fluent-bit/flb_coro.h index 7f1111ea6ee..e069a7c9cef 100644 --- a/include/fluent-bit/flb_coro.h +++ b/include/fluent-bit/flb_coro.h @@ -74,7 +74,14 @@ struct flb_coro { #ifdef FLB_CORO_STACK_SIZE #define FLB_CORO_STACK_SIZE_BYTE FLB_CORO_STACK_SIZE #else -#define FLB_CORO_STACK_SIZE_BYTE ((3 * STACK_FACTOR * PTHREAD_STACK_MIN) / 2) +#define FLB_CORO_STACK_SIZE_PLATFORM_BYTE \ + ((3 * STACK_FACTOR * PTHREAD_STACK_MIN) / 2) +/* Leave headroom for parser frames which can exceed the platform default. */ +#define FLB_CORO_STACK_SIZE_MIN_BYTE (64 * 1024) +#define FLB_CORO_STACK_SIZE_BYTE \ + (FLB_CORO_STACK_SIZE_PLATFORM_BYTE > FLB_CORO_STACK_SIZE_MIN_BYTE \ + ? FLB_CORO_STACK_SIZE_PLATFORM_BYTE \ + : FLB_CORO_STACK_SIZE_MIN_BYTE) #endif #define FLB_CORO_DATA(coro) (((char *) coro) + sizeof(struct flb_coro)) diff --git a/include/fluent-bit/flb_downstream.h b/include/fluent-bit/flb_downstream.h index 28dda20a14d..58c92155efb 100644 --- a/include/fluent-bit/flb_downstream.h +++ b/include/fluent-bit/flb_downstream.h @@ -31,6 +31,9 @@ struct flb_connection; +#define FLB_DOWNSTREAM_CONN_RELEASED 0 +#define FLB_DOWNSTREAM_CONN_DEFERRED 1 + /* Downstream handler */ struct flb_downstream { struct flb_stream base; @@ -84,6 +87,15 @@ void flb_downstream_resume(struct flb_downstream *stream); int flb_downstream_conn_release(struct flb_connection *connection); +/* + * The callback and any ingestion it invokes run on config->coro_stack_size. + * Callers must size that stack for their complete callback path. + */ +int flb_downstream_conn_event_register(struct flb_connection *connection, + int (*callback)(void *data), + int mask); +void flb_downstream_conn_event_resume(struct flb_connection *connection); + int flb_downstream_conn_pending_destroy_list(struct mk_list *list); int flb_downstream_conn_pending_destroy(struct flb_downstream *stream); diff --git a/include/fluent-bit/flb_downstream_worker.h b/include/fluent-bit/flb_downstream_worker.h new file mode 100644 index 00000000000..f043169ebe0 --- /dev/null +++ b/include/fluent-bit/flb_downstream_worker.h @@ -0,0 +1,74 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2026 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_DOWNSTREAM_WORKER_H +#define FLB_DOWNSTREAM_WORKER_H + +#include + +struct mk_event_loop; +struct flb_downstream_worker; +struct flb_downstream_worker_runtime; + +typedef int (*flb_downstream_worker_init_cb)(struct flb_downstream_worker *worker, + void *parent, + void **worker_context); + +typedef void (*flb_downstream_worker_exit_cb)(struct flb_downstream_worker *worker, + void *worker_context); + +typedef void (*flb_downstream_worker_maintenance_cb)( + struct flb_downstream_worker *worker, + void *worker_context); + +typedef void (*flb_downstream_worker_foreach_cb)(struct flb_downstream_worker *worker, + void *worker_context, + void *data); + +struct flb_downstream_worker_options { + int workers; + void *parent; + flb_downstream_worker_init_cb cb_init; + flb_downstream_worker_exit_cb cb_exit; + flb_downstream_worker_maintenance_cb cb_maintenance; +}; + +int flb_downstream_worker_runtime_start(struct flb_downstream_worker_runtime **out_runtime, + const struct flb_downstream_worker_options *options); + +/* Runtime operations must not be invoked from a worker callback. */ +int flb_downstream_worker_runtime_stop(struct flb_downstream_worker_runtime *runtime); + +/* The callback is run synchronously once on every worker thread. */ +int flb_downstream_worker_runtime_foreach(struct flb_downstream_worker_runtime *runtime, + flb_downstream_worker_foreach_cb callback, + void *data); + +struct mk_event_loop *flb_downstream_worker_event_loop_get( + struct flb_downstream_worker *worker); + +int flb_downstream_worker_id_get(struct flb_downstream_worker *worker); + +int flb_downstream_worker_count_get(struct flb_downstream_worker *worker); + +/* Register the listener created by cb_init for shared-endpoint validation. */ +int flb_downstream_worker_listener_fd_set(struct flb_downstream_worker *worker, + flb_sockfd_t listener_fd); + +#endif diff --git a/include/fluent-bit/flb_engine.h b/include/fluent-bit/flb_engine.h index 0fcd522ca53..02cefaf5cba 100644 --- a/include/fluent-bit/flb_engine.h +++ b/include/fluent-bit/flb_engine.h @@ -39,6 +39,12 @@ int flb_engine_destroy_tasks(struct mk_list *tasks); void flb_engine_reschedule_retries(struct flb_config *config); void flb_engine_stop_ingestion(struct flb_config *config); +/* Adaptive flush helpers (also used by internal tests) */ +int flb_engine_adaptive_flush_target_level(struct flb_config *config, + double pressure); +double flb_engine_adaptive_flush_interval(struct flb_config *config, + int level); + /* Engine event loop */ void flb_engine_evl_init(); struct mk_event_loop *flb_engine_evl_get(); diff --git a/include/fluent-bit/flb_fips.h b/include/fluent-bit/flb_fips.h new file mode 100644 index 00000000000..4c14ab40ca4 --- /dev/null +++ b/include/fluent-bit/flb_fips.h @@ -0,0 +1,25 @@ +/* Fluent Bit + * ========== + * Copyright (C) 2015-2026 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_FIPS_H +#define FLB_FIPS_H + +struct flb_config; + +int flb_fips_init(struct flb_config *config); + +#endif diff --git a/include/fluent-bit/flb_input.h b/include/fluent-bit/flb_input.h index c2ca41bd75e..310b06a7d67 100644 --- a/include/fluent-bit/flb_input.h +++ b/include/fluent-bit/flb_input.h @@ -873,19 +873,29 @@ int flb_input_ingress_enable(struct flb_input_instance *ins); int flb_input_ingress_queue_log(struct flb_input_instance *ins, const char *tag, size_t tag_len, const void *buf, size_t buf_size); +/* The take and decoded-signal queue functions always consume their payload. */ int flb_input_ingress_queue_log_take(struct flb_input_instance *ins, const char *tag, size_t tag_len, void *buf, size_t buf_size, size_t allocation_size); +int flb_input_ingress_queue_log_take_records(struct flb_input_instance *ins, + size_t records, + const char *tag, size_t tag_len, + void *buf, size_t buf_size, + size_t allocation_size); int flb_input_ingress_queue_metrics(struct flb_input_instance *ins, const char *tag, size_t tag_len, - struct cmt *cmt); + struct cmt *cmt, size_t payload_size); +int flb_input_ingress_queue_metrics_list(struct flb_input_instance *ins, + const char *tag, size_t tag_len, + struct cfl_list *contexts, + size_t payload_size); int flb_input_ingress_queue_traces(struct flb_input_instance *ins, const char *tag, size_t tag_len, - struct ctrace *ctr); + struct ctrace *ctr, size_t payload_size); int flb_input_ingress_queue_profiles(struct flb_input_instance *ins, const char *tag, size_t tag_len, - struct cprof *profile); + struct cprof *profile, size_t payload_size); /* processors */ diff --git a/include/fluent-bit/flb_io.h b/include/fluent-bit/flb_io.h index 9a6afd1e0cd..f2391f1dc87 100644 --- a/include/fluent-bit/flb_io.h +++ b/include/fluent-bit/flb_io.h @@ -35,9 +35,20 @@ #define FLB_IO_OPT_TLS 4 /* use TCP and optional TLS */ #define FLB_IO_ASYNC 8 /* use async mode (depends on event loop) */ #define FLB_IO_TCP_KA 16 /* use async mode (depends on event loop) */ +#define FLB_IO_UDP 32 /* use plain UDP */ +#define FLB_IO_DTLS 64 /* use DTLS over UDP */ /* Other features */ -#define FLB_IO_IPV6 32 /* network I/O uses IPv6 */ +#define FLB_IO_IPV6 128 /* network I/O uses IPv6 */ + +/* Retryable network I/O results */ +#define FLB_IO_WANT_READ -0x7e4 +#define FLB_IO_WANT_WRITE -0x7e6 + +static inline int flb_io_net_is_retry(ssize_t result) +{ + return result == FLB_IO_WANT_READ || result == FLB_IO_WANT_WRITE; +} struct flb_connection; diff --git a/include/fluent-bit/flb_plugin_alias.h b/include/fluent-bit/flb_plugin_alias.h new file mode 100644 index 00000000000..bd4746b2e3a --- /dev/null +++ b/include/fluent-bit/flb_plugin_alias.h @@ -0,0 +1,58 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2026 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_PLUGIN_ALIAS_H +#define FLB_PLUGIN_ALIAS_H + +#include + +/* + * Returned by flb_plugin_alias_rewrite() when an alias exists but an internal + * error prevents generating a rewritten string. + */ +#define FLB_PLUGIN_ALIAS_ERR ((char *) -1) + +struct flb_plugin_alias_entry { + int plugin_type; + const char *alias_name; + const char *plugin_name; +}; + +/* + * Returns the canonical plugin name for alias_name when a mapping exists, + * otherwise returns NULL. + */ +const char *flb_plugin_alias_get(int plugin_type, const char *alias_name, + size_t alias_name_length); + +/* + * Rewrites plugin_reference when it starts with a known alias. + * + * Return values: + * - NULL: no rewrite needed + * - FLB_PLUGIN_ALIAS_ERR: rewrite needed but failed + * - allocated string: rewritten plugin reference (caller must free) + */ +char *flb_plugin_alias_rewrite(int plugin_type, const char *plugin_reference); + +void flb_plugin_alias_set_custom_entries( + const struct flb_plugin_alias_entry *entries); +void flb_plugin_alias_reset_custom_entries(void); + +#endif diff --git a/include/fluent-bit/flb_stream.h b/include/fluent-bit/flb_stream.h index d1b858a9090..ab55ea70f7f 100644 --- a/include/fluent-bit/flb_stream.h +++ b/include/fluent-bit/flb_stream.h @@ -136,7 +136,8 @@ static inline int flb_stream_is_keepalive(struct flb_stream *stream) static inline int flb_stream_is_secure(struct flb_stream *stream) { - return flb_stream_get_flag_status(stream, FLB_IO_TLS); + return flb_stream_get_flag_status(stream, FLB_IO_TLS) || + flb_stream_get_flag_status(stream, FLB_IO_DTLS); } static inline int flb_stream_is_thread_safe(struct flb_stream *stream) @@ -205,4 +206,4 @@ static inline int flb_stream_release_lock(struct flb_stream *stream) return result; } -#endif \ No newline at end of file +#endif diff --git a/include/fluent-bit/flb_time.h b/include/fluent-bit/flb_time.h index 3ccb186cf93..440d887c374 100644 --- a/include/fluent-bit/flb_time.h +++ b/include/fluent-bit/flb_time.h @@ -24,6 +24,7 @@ #include #include +#include #include #include struct flb_time { @@ -84,15 +85,55 @@ static inline void flb_time_copy(struct flb_time *dst, struct flb_time *src) dst->tm.tv_nsec = src->tm.tv_nsec; } -static inline void flb_time_from_uint64(struct flb_time *dst, uint64_t value) +static inline int flb_time_is_valid_eventtime(struct flb_time *tm) { - dst->tm.tv_sec = (long) (value / 1000000000L); + if (tm == NULL || tm->tm.tv_sec < 0 || + (uint64_t) tm->tm.tv_sec > UINT32_MAX || + tm->tm.tv_nsec < 0 || tm->tm.tv_nsec >= 1000000000L) { + return FLB_FALSE; + } + + return FLB_TRUE; +} + +static inline int flb_time_from_uint64(struct flb_time *dst, uint64_t value) +{ + uint64_t seconds; + uint64_t maximum_seconds; + unsigned int bit_count; + + if (dst == NULL) { + return -1; + } + + seconds = value / 1000000000L; + + bit_count = sizeof(time_t) * CHAR_BIT; + if ((time_t) -1 < 0) { + bit_count--; + } + + if (bit_count >= sizeof(uint64_t) * CHAR_BIT) { + maximum_seconds = UINT64_MAX; + } + else { + maximum_seconds = UINT64_MAX >> + (sizeof(uint64_t) * CHAR_BIT - bit_count); + } + + if (seconds > maximum_seconds) { + return -1; + } + + dst->tm.tv_sec = (time_t) seconds; dst->tm.tv_nsec = (long) (value - ((uint64_t) dst->tm.tv_sec * 1000000000L)); + + return 0; } static inline void flb_time_from_double(struct flb_time *dst, double d) { - dst->tm.tv_sec = (int) d; + dst->tm.tv_sec = (time_t) d; dst->tm.tv_nsec = (long) ((d - dst->tm.tv_sec) * 1000000000L); } diff --git a/include/fluent-bit/flb_upstream.h b/include/fluent-bit/flb_upstream.h index 544f97fee90..44d585c5bd9 100644 --- a/include/fluent-bit/flb_upstream.h +++ b/include/fluent-bit/flb_upstream.h @@ -41,6 +41,8 @@ * --- flb_io.h --- * #define FLB_IO_TCP 1 * #define FLB_IO_TLS 2 + * #define FLB_IO_UDP 32 + * #define FLB_IO_DTLS 64 * #define FLB_IO_ASYNC 8 * #define FLB_IO_TCP_KA 16 * --- diff --git a/include/fluent-bit/http_server/flb_http_server.h b/include/fluent-bit/http_server/flb_http_server.h index d6db0e0f1b2..5e75263b6fe 100755 --- a/include/fluent-bit/http_server/flb_http_server.h +++ b/include/fluent-bit/http_server/flb_http_server.h @@ -59,7 +59,7 @@ typedef int (*flb_http_server_request_processor_callback)( struct flb_http_response *response); struct flb_http_server; -struct flb_http_server_runtime; +struct flb_downstream_worker_runtime; typedef int (*flb_http_server_worker_callback)(struct flb_http_server *server, void *data); @@ -98,6 +98,7 @@ struct flb_http_server_options { size_t buffer_max_size; size_t buffer_chunk_size; size_t max_connections; + uint64_t *connection_counter; /* Total number of worker listeners to spawn. */ int workers; @@ -142,6 +143,8 @@ struct flb_http_server { size_t buffer_max_size; size_t buffer_chunk_size; size_t max_connections; + uint64_t active_connections; + uint64_t *connection_counter; int workers; int worker_id; int use_caller_event_loop; @@ -149,7 +152,7 @@ struct flb_http_server { int tls_alpn_configured; flb_http_server_worker_callback cb_worker_init; flb_http_server_worker_callback cb_worker_exit; - struct flb_http_server_runtime *runtime; + struct flb_downstream_worker_runtime *runtime; }; struct flb_http_server_session { @@ -166,6 +169,7 @@ struct flb_http_server_session { int releasable; int drop_pending; + int connection_slot_reserved; struct flb_connection *connection; struct flb_http_server *parent; @@ -231,10 +235,6 @@ void flb_http_server_set_buffer_max_size(struct flb_http_server *server, size_t size_t flb_http_server_get_buffer_max_size(struct flb_http_server *server); -const struct flb_net_setup * -flb_http_server_runtime_worker_net_setup_get(struct flb_http_server *server, - int worker_id); - /* HTTP SESSION */ int flb_http_server_session_init(struct flb_http_server_session *session, int version); diff --git a/include/fluent-bit/tls/flb_tls.h b/include/fluent-bit/tls/flb_tls.h index 018231218e2..12e9cac2d8e 100644 --- a/include/fluent-bit/tls/flb_tls.h +++ b/include/fluent-bit/tls/flb_tls.h @@ -25,15 +25,18 @@ #include #include #include +#include +#include #include +#include #define FLB_TLS_ALPN_MAX_LENGTH 16 #define FLB_TLS_CLIENT "Fluent Bit" /* TLS backend return status on read/write */ -#define FLB_TLS_WANT_READ -0x7e4 -#define FLB_TLS_WANT_WRITE -0x7e6 +#define FLB_TLS_WANT_READ FLB_IO_WANT_READ +#define FLB_TLS_WANT_WRITE FLB_IO_WANT_WRITE /* Cert Flags */ #define FLB_TLS_CA_ROOT 1 @@ -46,10 +49,23 @@ #define FLB_TLS_CLIENT_MODE 0 #define FLB_TLS_SERVER_MODE 1 +#define FLB_TLS_CLIENT_MODE_DGRAM 2 +#define FLB_TLS_SERVER_MODE_DGRAM 3 struct flb_tls; struct flb_connection; +struct flb_tls_file_status { + int exists; + uint64_t size; + uint64_t device; + uint64_t inode; + uint64_t mtime; + uint64_t mtime_nsec; + uint64_t ctime; + uint64_t ctime_nsec; +}; + struct flb_tls_session { /* opaque data type for backend session context */ void *ptr; @@ -71,6 +87,9 @@ struct flb_tls_backend { const char *, const char *, const char *, const char *); + /* reload backend context */ + int (*context_reload) (struct flb_tls *); + /* destroy backend context */ void (*context_destroy) (void *); @@ -108,16 +127,32 @@ struct flb_tls { int verify_client; /* Verify client certificate */ int debug; /* Debug level */ char *vhost; /* Virtual hostname for SNI */ + char *ca_path; /* Path to certificates */ + char *ca_file; /* CA root cert */ + char *crt_file; /* Certificate */ + char *key_file; /* Cert Key */ + char *key_passwd; /* Cert Key Password */ + char *alpn; /* ALPN protocol list */ + char *min_version; /* Minimum TLS version */ + char *max_version; /* Maximum TLS version */ + char *ciphers; /* TLS ciphers */ + struct flb_tls_file_status ca_path_status; + struct flb_tls_file_status ca_file_status; + struct flb_tls_file_status crt_file_status; + struct flb_tls_file_status key_file_status; int mode; /* Client or Server */ int verify_hostname; /* Verify hostname */ + int system_certificates_loaded; /* System certs loaded */ #if defined(FLB_SYSTEM_WINDOWS) char *certstore_name; /* Windows CertStore Name */ int use_enterprise_store; /* Use Enterprise store or not */ + char *client_thumbprints; /* Allowed client thumbprints */ #endif /* Bakend library for TLS */ void *ctx; /* TLS context created */ struct flb_tls_backend *api; /* backend API */ + pthread_mutex_t reload_mutex; /* protects reload state */ }; int flb_tls_init(); @@ -132,6 +167,8 @@ struct flb_tls *flb_tls_create(int mode, int flb_tls_destroy(struct flb_tls *tls); +int flb_tls_reload_if_needed(struct flb_tls *tls); + int flb_tls_set_alpn(struct flb_tls *tls, const char *alpn); int flb_tls_set_verify_client(struct flb_tls *tls, int verify_client); diff --git a/lib/cfl/.github/workflows/build.yaml b/lib/cfl/.github/workflows/build.yaml index 22f3d80057d..50dabc68436 100644 --- a/lib/cfl/.github/workflows/build.yaml +++ b/lib/cfl/.github/workflows/build.yaml @@ -8,6 +8,9 @@ on: - master types: [opened, reopened, synchronize] +permissions: + contents: read + jobs: build-windows: name: Build sources on amd64 for ${{ matrix.os }} @@ -17,7 +20,7 @@ jobs: matrix: os: [windows-latest, windows-2022] steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - name: Build on ${{ matrix.os }} with MSVC run: | .\scripts\win_build.bat @@ -33,12 +36,12 @@ jobs: matrix: os: [windows-latest, windows-2022] steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - name: Get dependencies w/ chocolatey - uses: crazy-max/ghaction-chocolatey@v3 + uses: crazy-max/ghaction-chocolatey@dfdcf5bba9c0a16358a21c13a06ec5c89024b3ac # v4 with: args: install -y cmake --installargs 'ADD_CMAKE_TO_PATH=System' - - uses: msys2/setup-msys2@v2 + - uses: msys2/setup-msys2@66cd2cce69caa17b53920067426061ca1de3a884 # v2 with: update: true msystem: UCRT64 @@ -126,7 +129,7 @@ jobs: # Confirm CMake installation /usr/local/bin/cmake --version - - uses: actions/checkout@v6 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - name: Run compilation run: | @@ -142,9 +145,9 @@ jobs: os: [ubuntu-latest] compiler: [ gcc, clang ] steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - name: Build on ${{ matrix.os }} with ${{ matrix.compiler }} - uses: uraimo/run-on-arch-action@v3.0.1 + uses: uraimo/run-on-arch-action@f9b26e3a1a408d5fd530d20c17b9f3f4428ff8d9 # v3.1.0 with: arch: aarch64 distro: ubuntu_latest @@ -152,16 +155,15 @@ jobs: apt-get update && \ apt-get install -y --no-install-recommends \ build-essential \ + clang \ cmake \ file \ make - export CC=${{ env.compiler }} + export CC=${{ matrix.compiler }} + "$CC" --version cmake -DCFL_TESTS=On . make all CTEST_OUTPUT_ON_FAILURE=1 make test - env: - CC: ${{ matrix.compiler }} - build-unix-amd64: name: Build sources on amd64 for ${{ matrix.os }} - ${{ matrix.compiler }} runs-on: ${{ matrix.os }} @@ -171,7 +173,7 @@ jobs: os: [ubuntu-latest, macos-latest] compiler: [ gcc, clang ] steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - name: Build on ${{ matrix.os }} with ${{ matrix.compiler }} run: | echo "CC = $CC, CXX = $CXX" @@ -181,6 +183,125 @@ jobs: env: CC: ${{ matrix.compiler }} + build-c-dialects: + name: Build sources with GNU ${{ matrix.standard }} + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + standard: [99, 17] + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - name: Configure, build, and test + run: | + cmake -S . -B build \ + -DCFL_TESTS=On \ + -DCMAKE_C_STANDARD=${{ matrix.standard }} \ + -DCMAKE_C_STANDARD_REQUIRED=On \ + -DCMAKE_C_EXTENSIONS=On + cmake --build build -j2 + ctest --test-dir build --output-on-failure + + build-downstream: + name: Build downstream consumer ${{ matrix.consumer }} + runs-on: ubuntu-22.04 + strategy: + fail-fast: false + matrix: + consumer: [cmetrics, cprofiles, ctraces, fluent-bit] + steps: + - name: Check out CFL + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + path: cfl + - name: Check out ${{ matrix.consumer }} + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + repository: fluent/${{ matrix.consumer }} + path: consumer + submodules: recursive + - name: Install build dependencies + run: | + sudo apt-get update + sudo apt-get install -y bison flex libssl-dev libyaml-dev + - name: Report bundled CFL version + shell: bash + run: | + version_from() + { + local cmake_file="$1" + local major + local minor + local patch + + major=$(sed -n 's/^set(CFL_VERSION_MAJOR *\([0-9][0-9]*\)).*/\1/p' "$cmake_file") + minor=$(sed -n 's/^set(CFL_VERSION_MINOR *\([0-9][0-9]*\)).*/\1/p' "$cmake_file") + patch=$(sed -n 's/^set(CFL_VERSION_PATCH *\([0-9][0-9]*\)).*/\1/p' "$cmake_file") + echo "${major}.${minor}.${patch}" + } + + cfl_version=$(version_from cfl/CMakeLists.txt) + bundled_version=$(version_from consumer/lib/cfl/CMakeLists.txt) + echo "CFL under test: ${cfl_version}" >> "$GITHUB_STEP_SUMMARY" + echo "${{ matrix.consumer }} bundled CFL: ${bundled_version}" >> "$GITHUB_STEP_SUMMARY" + if [[ "$cfl_version" != "$bundled_version" ]]; then + echo "::warning::${{ matrix.consumer }} bundles CFL ${bundled_version}; current CFL is ${cfl_version}" + fi + - name: Replace bundled CFL with the version under test + run: | + rm -rf consumer/lib/cfl + mkdir -p consumer/lib/cfl + git -C cfl archive HEAD | tar -x -C consumer/lib/cfl + - name: Configure ${{ matrix.consumer }} + shell: bash + run: | + case "${{ matrix.consumer }}" in + cmetrics) + cmake -S consumer -B build -DCMT_TESTS=On + ;; + cprofiles) + cmake -S consumer -B build -DCPROF_TESTS=On + ;; + ctraces) + cmake -S consumer -B build -DCTR_TESTS=On + ;; + fluent-bit) + cmake -S consumer -B build \ + -DFLB_ALL=Off \ + -DFLB_EXAMPLES=Off \ + -DFLB_SHARED_LIB=Off \ + -DFLB_PROCESSOR_CONTENT_MODIFIER=On \ + -DFLB_PROCESSOR_METRICS_SELECTOR=On \ + -DFLB_PROCESSOR_SQL=On \ + -DFLB_PROCESSOR_SAMPLING=On + ;; + esac + - name: Build ${{ matrix.consumer }} + run: cmake --build build -j2 + - name: Test ${{ matrix.consumer }} + if: matrix.consumer != 'fluent-bit' + run: ctest --test-dir build --output-on-failure + + build-installed-consumer: + name: Build an installed CFL consumer + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - name: Build and install CFL + run: | + cmake -S . -B build \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX="$PWD/install" + cmake --build build -j2 + cmake --install build + - name: Build and run the external consumer + run: | + cmake -S tests/installed_consumer -B consumer-build \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_PREFIX_PATH="$PWD/install" + cmake --build consumer-build -j2 + ./consumer-build/cfl-installed-consumer + build-analysis-tests: name: Build with various code analysis tools strategy: @@ -197,11 +318,11 @@ jobs: contents: read runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: submodules: true - - uses: docker://lpenz/ghaction-cmake:0.19 + - uses: docker://lpenz/ghaction-cmake@sha256:b59f08ebbbae1ed35650f774e3f2fd77e9c56e10cc841349d923ab4b65724aed with: pre_command: | CMAKE_VERSION=3.20.0 @@ -230,12 +351,15 @@ jobs: - build-debian - build-unix-arm64 - build-unix-amd64 + - build-c-dialects + - build-downstream + - build-installed-consumer - build-analysis-tests name: Required checks complete runs-on: ubuntu-latest permissions: {} steps: - name: Decide whether the needed jobs succeeded or failed - uses: re-actors/alls-green@release/v1 + uses: re-actors/alls-green@05ac9388f0aebcb5727afa17fcccfecd6f8ec5fe # release/v1 with: jobs: ${{ toJSON(needs) }} diff --git a/lib/cfl/.github/workflows/lint.yaml b/lib/cfl/.github/workflows/lint.yaml index 12f97917cf4..5b87f7f0556 100644 --- a/lib/cfl/.github/workflows/lint.yaml +++ b/lib/cfl/.github/workflows/lint.yaml @@ -3,6 +3,9 @@ on: pull_request: workflow_dispatch: +permissions: + contents: read + jobs: shellcheck: runs-on: ubuntu-latest @@ -10,8 +13,8 @@ jobs: permissions: contents: read steps: - - uses: actions/checkout@v6 - - uses: ludeeus/action-shellcheck@master + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: ludeeus/action-shellcheck@00b27aa7cb85167568cb48a3838b75f4265f2bca with: ignore_paths: lib @@ -21,10 +24,9 @@ jobs: permissions: contents: read steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - run: | echo "::add-matcher::.github/actionlint-matcher.json" - bash <(curl https://raw.githubusercontent.com/rhysd/actionlint/main/scripts/download-actionlint.bash) + bash <(curl -fsSL https://raw.githubusercontent.com/rhysd/actionlint/a443f344ff32813837fa49f7aa6cbc478d770e62/scripts/download-actionlint.bash) 1.7.9 ./actionlint -color -shellcheck= shell: bash - diff --git a/lib/cfl/.github/workflows/packages.yaml b/lib/cfl/.github/workflows/packages.yaml index a73e49b19ae..957b81a86ef 100644 --- a/lib/cfl/.github/workflows/packages.yaml +++ b/lib/cfl/.github/workflows/packages.yaml @@ -9,6 +9,9 @@ on: - 'v*' workflow_dispatch: +permissions: + contents: read + jobs: build-distro-packages-arm64: runs-on: ubuntu-latest @@ -18,8 +21,8 @@ jobs: matrix: format: [ rpm, deb ] steps: - - uses: actions/checkout@v6 - - uses: uraimo/run-on-arch-action@v3.0.1 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: uraimo/run-on-arch-action@f9b26e3a1a408d5fd530d20c17b9f3f4428ff8d9 # v3.1.0 name: Build the ${{matrix.format}} packages with: arch: aarch64 @@ -36,7 +39,7 @@ jobs: echo ${{ matrix.format }} | awk '{print toupper($0)}' | xargs -I{} cpack -G {} - name: Store the master package artifacts - uses: actions/upload-artifact@v6 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: name: ${{ matrix.format }}-arm64 path: | @@ -51,14 +54,14 @@ jobs: runs-on: [ ubuntu-latest ] steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - name: Build the ${{matrix.format}} packages run: | cmake . echo ${{ matrix.format }} | awk '{print toupper($0)}' | xargs -I{} cpack -G {} - name: Store the master package artifacts - uses: actions/upload-artifact@v6 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: name: ${{ matrix.format }}-amd64 path: | @@ -74,7 +77,7 @@ jobs: contents: write steps: - name: Download all artefacts - uses: actions/download-artifact@v7 + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 with: path: artifacts/ @@ -84,7 +87,7 @@ jobs: shell: bash - name: Unstable release on push to master to make it easier to download - uses: pyTooling/Actions/releaser@r0 + uses: pyTooling/Actions/releaser@b346408fef88bd4cfda9109149e9ec8338a32b15 # r0 continue-on-error: true with: token: ${{ secrets.GITHUB_TOKEN }} @@ -94,7 +97,7 @@ jobs: artifacts/**/* - name: Release on tag - uses: softprops/action-gh-release@v2 + uses: softprops/action-gh-release@7c4723f7a335432393329f8f1c564994ce50185d # v3 if: startsWith(github.ref, 'refs/tags/') with: generate_release_notes: true diff --git a/lib/cfl/AGENTS.md b/lib/cfl/AGENTS.md index 8fccc629736..203526946c4 100644 --- a/lib/cfl/AGENTS.md +++ b/lib/cfl/AGENTS.md @@ -55,6 +55,18 @@ before closing changes that touch shared code or public APIs. - If a relevant test cannot be run, report the exact blocker in the final response. +## Downstream Impact Validation +- For every modified CFL function, inspect callers in CFL, Fluent Bit, + cmetrics, ctraces, and cprofiles before closing the change. +- Check for side effects from changes to signatures, return values, error + handling, ownership, object lifetime, allocation behavior, and mutation + semantics. +- Distinguish production callers from bundled CFL source and test copies when + reporting impact. +- Run relevant downstream builds or tests when a change can affect an existing + caller. If a downstream checkout or required test is unavailable, report the + exact validation gap. + ## Commit & Pull Request Guidelines - Follow observed local history style: `component: short imperative description` diff --git a/lib/cfl/ARENA.md b/lib/cfl/ARENA.md new file mode 100644 index 00000000000..3409f98900f --- /dev/null +++ b/lib/cfl/ARENA.md @@ -0,0 +1,336 @@ +# CFL arena allocator + +`cfl_arena` is an optional allocator for CFL variants, arrays, key/value lists, +kvpairs, owned SDS strings, and arbitrary request-lifetime objects. It reduces +allocator traffic when an application constructs, mutates, and discards a +complete object graph as one unit. + +The normal CFL constructors remain heap-backed. Arena use is explicit and does +not change existing callers. + +## When to use an arena + +An arena is a good fit when: + +- many related CFL values have the same lifetime; +- the complete graph has a clear owner and reset point; +- processing finishes before the graph is reset; +- individual removals do not need to immediately return memory to the system; +- the arena can be reused across documents or batches. + +Examples include a decoded document, a mutable telemetry record, or a batch +that remains owned through a processor chain and is serialized before reset. + +Keep heap allocation when objects have unrelated lifetimes, children move to +longer-lived owners, memory must be reclaimed individually, or a long-lived +container experiences unbounded churn. + +## Public API + +Include the arena interface directly: + +```c +#include +``` + +It is also included by ``. + +```c +struct cfl_arena; + +struct cfl_arena *cfl_arena_create(size_t chunk_size); +struct cfl_arena *cfl_arena_create_ex(size_t chunk_size, + size_t large_object_threshold); +void cfl_arena_options_init(struct cfl_arena_options *options); +struct cfl_arena *cfl_arena_create_with_options( + const struct cfl_arena_options *options); +void cfl_arena_destroy(struct cfl_arena *arena); +void cfl_arena_reset(struct cfl_arena *arena); + +void *cfl_arena_malloc(struct cfl_arena *arena, size_t size); +void *cfl_arena_calloc(struct cfl_arena *arena, + size_t count, size_t size); +void *cfl_arena_memdup(struct cfl_arena *arena, + const void *source, size_t size); +char *cfl_arena_strndup(struct cfl_arena *arena, + const char *source, size_t length); + +size_t cfl_arena_bytes_reserved(struct cfl_arena *arena); +size_t cfl_arena_bytes_used(struct cfl_arena *arena); +size_t cfl_arena_large_object_threshold(struct cfl_arena *arena); + +void cfl_arena_external_cache_limit_set(struct cfl_arena *arena, + size_t limit); +size_t cfl_arena_external_cache_limit_get(struct cfl_arena *arena); +size_t cfl_arena_external_cache_bytes(struct cfl_arena *arena); +``` + +Passing zero as `chunk_size` selects the default chunk size. With +`cfl_arena_create_ex()`, a zero large-object threshold selects the default +policy derived from the chunk size. + +## Raw request-lifetime allocation + +Raw allocation supports objects that do not have CFL-specific constructors, +including temporary encoder trees: + +```c +struct request_state *state; +char *name; + +state = cfl_arena_calloc(arena, 1, sizeof(*state)); +name = cfl_arena_strndup(arena, input_name, input_name_length); +if (state == NULL || name == NULL) { + /* The arena remains valid and can still be reset or destroyed. */ +} +``` + +Raw pointers cannot be freed individually. They remain valid until the arena +is reset or destroyed. Returned pointers are aligned for CFL-supported +fundamental C types, including `long double`, pointers, and 64-bit integers. + +The raw allocation rules are: + +- `cfl_arena_malloc()` returns uninitialized storage. +- `cfl_arena_calloc()` checks multiplication overflow and zeroes the result. +- `cfl_arena_memdup()` copies an exact number of bytes. +- `cfl_arena_strndup()` appends a null terminator to the requested prefix. +- Zero-sized `malloc`, `calloc`, and `memdup` requests return `NULL`. +- `strndup` accepts a zero length and returns an allocated empty string. +- A null source, arithmetic overflow, invalid arena, or allocation failure + returns `NULL`. +- Failure leaves the arena usable and does not define `errno`. + +## Growth and allocator options + +Existing constructors retain fixed-size chunks. Optional geometric growth and +allocator callbacks are configured through an initialized options structure: + +```c +struct cfl_arena_options options; + +cfl_arena_options_init(&options); +options.chunk_size = 4096; +options.maximum_chunk_size = 65536; +options.malloc_fn = application_malloc; +options.free_fn = application_free; +options.allocator_context = application_context; + +arena = cfl_arena_create_with_options(&options); +``` + +The first normal chunk uses `chunk_size`. Later chunks double in size until +`maximum_chunk_size`; a request larger than the current chunk size receives a +dedicated chunk large enough for that request. A zero maximum selects fixed +growth, making the maximum equal to the initial chunk size. A maximum smaller +than the initial size is invalid. + +The callback pair is optional, but callers must provide both callbacks or +neither. Callbacks allocate and release the arena context, normal chunks, +external allocations, and cached external allocations. The allocation callback +must return storage with normal `malloc` alignment. CFL implements zeroing and +does not require `calloc` or `realloc` callbacks. `struct_size` must be set by +`cfl_arena_options_init()` so future CFL versions can extend the structure. + +## Arena-aware constructors + +The following constructors associate new values with an arena: + +```c +struct cfl_variant *cfl_variant_create_in(struct cfl_arena *arena); +struct cfl_array *cfl_array_create_in(struct cfl_arena *arena, + size_t slot_count); +struct cfl_kvlist *cfl_kvlist_create_in(struct cfl_arena *arena); +cfl_sds_t cfl_sds_create_len_in(struct cfl_arena *arena, + const char *str, int len); +``` + +Typed variant constructors have matching `_in` forms, including strings, +bytes, booleans, integers, doubles, nulls, references, arrays, and kvlists. + +Nested containers can inherit their parent's allocator: + +```c +struct cfl_array *cfl_array_create_like(struct cfl_array *parent, + size_t slot_count); +struct cfl_kvlist *cfl_kvlist_create_like(struct cfl_kvlist *parent); +``` + +For a heap-backed parent, `create_like()` creates a heap-backed child. For an +arena-backed parent, it creates the child in the same arena. + +## Basic example + +```c +#include + +int process_batch(void) +{ + int result; + struct cfl_arena *arena; + struct cfl_kvlist *record; + + arena = cfl_arena_create(8192); + if (arena == NULL) { + return -1; + } + + result = 0; + + record = cfl_kvlist_create_in(arena); + if (record == NULL) { + result = -1; + goto done; + } + + if (cfl_kvlist_insert_string(record, "message", "ready") != 0 || + cfl_kvlist_insert_int64(record, "status", 200) != 0) { + result = -1; + goto done; + } + + /* Mutate, inspect, or serialize record before resetting the arena. */ + +done: + cfl_arena_destroy(arena); + + return result; +} +``` + +For repeated batches, reuse the arena: + +```c +while (next_batch()) { + /* Construct and completely process one graph. */ + process_with_arena(arena); + + /* Every arena-backed pointer is invalid after this call. */ + cfl_arena_reset(arena); +} +``` + +## Ownership rules + +Arena lifetime is part of object ownership: + +- Reset and destruction invalidate every pointer allocated from the arena. +- Do not retain arena-backed values after reset or destruction. +- Do not attach values from different arenas to one array or kvlist. +- Do not mix heap-backed and arena-backed children in the same object graph. +- Use `create_like()` when constructing nested containers during mutation. +- A raw array or kvlist must have only one owning variant at a time. +- If an object must outlive the arena, copy it into its destination allocator + before resetting the source arena. + +CFL validates allocator ownership for supported container attachments. A +rejected cross-arena insertion leaves ownership with the caller. + +Destroy functions can be used on arena-backed values while the arena is alive. +They release owned external storage and make reusable internal slots available +where supported. They do not replace the need to destroy the arena itself. + +## Reset and reclamation + +`cfl_arena_reset()` prepares the arena for another graph. It invalidates the +previous graph, resets chunk allocation positions, clears reusable-object +state, and handles external allocations according to the configured cache. + +Removing an individual value does not necessarily reduce reserved memory. +Chunk storage is reclaimed for reuse at reset, not returned after every object +destruction. This is the central throughput-versus-retention tradeoff of arena +allocation. + +## Large values and external caching + +Raw allocations and small CFL objects come from arena chunks. Raw requests +larger than the active chunk receive a dedicated chunk. The large-object +threshold applies to arena-aware owned SDS values, which use separately tracked +external storage so unusually large strings do not consume normal chunks. + +Reusable external buffers may remain cached after reset. Configure the maximum +cached capacity with: + +```c +cfl_arena_external_cache_limit_set(arena, limit); +``` + +Setting the limit to zero disables external-buffer caching. Reducing the limit +immediately trims the cache. A larger limit can improve throughput for repeated +large payloads but may increase retained memory and RSS. + +The arena also maintains size classes for commonly-sized owned SDS values. +Callers do not need to select a class. + +## Memory statistics + +The statistics API separates live capacity from retained storage: + +- `cfl_arena_bytes_used()` reports live arena payload capacity. +- `cfl_arena_bytes_reserved()` reports memory reserved by chunks and external + allocations, including CFL allocation headers but excluding allocator + metadata. +- `cfl_arena_external_cache_bytes()` reports the portion held in the reusable + external cache. + +Reserved bytes can remain higher than used bytes after removals or reset. Peak +RSS can also exceed both values because it includes the process, allocator +metadata, page granularity, and allocator-retained memory. + +Measure CPU, peak RSS, reserved bytes, used bytes, and cached bytes when tuning. +Fewer calls to `malloc()` do not by themselves prove lower memory usage. + +## Thread safety + +An arena is not thread-safe. All allocation, mutation, destruction, statistic, +cache-configuration, and reset operations involving the same arena must be +serialized by the caller. + +Separate arenas can be used independently by separate threads as long as their +object graphs are not mixed. + +## Error handling + +Arena constructors return `NULL` on invalid input, ownership violations, or +allocation failure. Container insert operations retain their documented return +conventions. Check every constructor and insertion before continuing to mutate +the graph. + +An allocation failure does not require abandoning the arena. The caller may +destroy or reset it normally, provided no partially-created pointer is used. + +## Tuning + +Start with the default policy: + +```c +arena = cfl_arena_create(0); +``` + +Tune only with representative workloads: + +1. Choose a chunk size large enough for common graphs without excessive unused + capacity. +2. Keep very large values external so they do not fragment normal chunks. +3. Bound or disable the external cache when retained memory matters more than + repeated large-buffer throughput. +4. Test multiple payload distributions and mutation patterns. +5. Compare against the heap implementation rather than assuming the arena wins. + +A separate arena per small object is usually counterproductive. Prefer one +arena for the complete bounded document or batch. + +## Benchmarks + +Build the supplied benchmark tools with: + +```sh +cmake -S . -B build-bench \ + -DCMAKE_BUILD_TYPE=Release \ + -DCFL_BENCHMARKS=On +cmake --build build-bench -j8 +``` + +See [benchmarks/README.md](benchmarks/README.md) for heap-versus-arena CPU, RSS, +fragmentation, mutable OTLP-style workloads, and the deterministic payload +matrix. diff --git a/lib/cfl/CHANGELOG.md b/lib/cfl/CHANGELOG.md new file mode 100644 index 00000000000..37b1f89347f --- /dev/null +++ b/lib/cfl/CHANGELOG.md @@ -0,0 +1,36 @@ +# Changelog + +This file records the notable changes in each CFL release. + +## Unreleased + +- Added public request-lifetime arena allocation, duplication helpers, + allocator callbacks, and optional bounded geometric chunk growth. + +## 1.0.0 - 2026-07-11 + +The first stable CFL release establishes the variant, container, utility, and +optional arena interfaces used by Fluent Bit and its companion telemetry +libraries. + +### Highlights + +- Added optional arena allocation for strings, variants, arrays, and key/value + lists, including reset, cache control, ownership validation, benchmarks, and + lifecycle documentation. +- Added case-sensitive and case-insensitive key/value lookup variants. +- Strengthened container ownership and cycle validation for mutable variant + graphs. +- Improved allocation failure handling and made dynamic-string formatting + preserve the original value when growth fails. +- Improved portability across GNU C99 and GNU C17, ARM64, Windows, macOS, and + supported Linux environments. +- Expanded sanitizer, Valgrind, installed-consumer, and downstream validation + for Fluent Bit, cmetrics, ctraces, and cprofiles. +- Installed the bundled xxHash archive with its headers so installed consumers + can use CFL's public hash interface. + +### Behavior changes + +- `cfl_array_remove_by_reference()` now returns `-1` when the supplied variant + is not present in the array. diff --git a/lib/cfl/CMakeLists.txt b/lib/cfl/CMakeLists.txt index 93a189936ec..96188e4ea4f 100644 --- a/lib/cfl/CMakeLists.txt +++ b/lib/cfl/CMakeLists.txt @@ -1,17 +1,23 @@ cmake_minimum_required(VERSION 3.20) project(cfl C) +if(NOT DEFINED CMAKE_C_STANDARD) + set(CMAKE_C_STANDARD 99) +endif() +set(CMAKE_C_STANDARD_REQUIRED On) +set(CMAKE_C_EXTENSIONS On) set(CMAKE_POSITION_INDEPENDENT_CODE ON) set(CMAKE_EXPORT_COMPILE_COMMANDS ON) # C Floppy Version -set(CFL_VERSION_MAJOR 0) -set(CFL_VERSION_MINOR 7) -set(CFL_VERSION_PATCH 1) +set(CFL_VERSION_MAJOR 1) +set(CFL_VERSION_MINOR 0) +set(CFL_VERSION_PATCH 0) set(CFL_VERSION_STR "${CFL_VERSION_MAJOR}.${CFL_VERSION_MINOR}.${CFL_VERSION_PATCH}") # Configuration options option(CFL_DEV "Enable development mode" No) option(CFL_TESTS "Enable unit testing" No) +option(CFL_BENCHMARKS "Build benchmark tools" No) option(CFL_INSTALL_BUNDLED_XXHASH_HEADERS "Enable bundled xxHash headers installation" Yes) if(CFL_DEV) @@ -40,15 +46,6 @@ if(NOT MSVC) set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall") endif() -# Define __FILENAME__ consistently across Operating Systems -if(NOT ${CMAKE_SYSTEM_NAME} MATCHES "Windows") - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -D__FILENAME__='\"$$(subst ${CMAKE_SOURCE_DIR}/,,$$(abspath $$<))\"'") -else() - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -D__FILENAME__=__FILE__") -endif() - - - # timespec_get() support check_c_source_compiles(" #include @@ -118,6 +115,7 @@ include_directories( # xxHash if(NOT TARGET xxhash) # Do something when target found + set(CFL_BUNDLED_XXHASH On) set(XXHASH_BUILD_ENABLE_INLINE_API OFF) set(XXHASH_BUILD_XXHSUM OFF) set(BUILD_SHARED_LIBS OFF) @@ -144,6 +142,10 @@ set(CFL_BUILD_DIR "${CFL_ROOT}/build") add_subdirectory(include) add_subdirectory(src) +if(CFL_BENCHMARKS) + add_subdirectory(benchmarks) +endif() + # Tests if(CFL_TESTS) enable_testing() @@ -156,8 +158,8 @@ endif() set(CPACK_PACKAGE_VERSION ${CFL_VERSION_STR}) set(CPACK_PACKAGE_NAME "cfl") set(CPACK_PACKAGE_RELEASE 1) -set(CPACK_PACKAGE_CONTACT "Eduardo Silva ") -set(CPACK_PACKAGE_VENDOR "Calyptia") +set(CPACK_PACKAGE_CONTACT "Eduardo Silva ") +set(CPACK_PACKAGE_VENDOR "Fluent Project") set(CPACK_RESOURCE_FILE_LICENSE "${PROJECT_SOURCE_DIR}/LICENSE") set(CPACK_PACKAGING_INSTALL_PREFIX "/") diff --git a/lib/cfl/README.md b/lib/cfl/README.md index c81b4bafd22..c73578dbdaf 100644 --- a/lib/cfl/README.md +++ b/lib/cfl/README.md @@ -1,65 +1,72 @@ # CFL -CFL is a tiny C library that provides small data-structure and utility -interfaces. It was originally created to satisfy the needs of Fluent Bit and -related libraries such as CMetrics and CTraces. - -Note: The name does not mean anything specific; you can call it `c:\ floppy` -if you want. - -## Interfaces - -Applications can include `` to pull in the common CFL interfaces. -Specialized headers are also available under `include/cfl/` when a caller only -needs one module. - -### Core - -- `cfl_init()`: initializes library-level facilities. -- `cfl_version()`: returns the CFL version string. -- `CFL_TRUE` and `CFL_FALSE`: common boolean-style constants used by CFL APIs. - -### Data Structures - -- `cfl_sds`: dynamic string storage with explicit length and allocation - tracking. It supports creation from strings or buffers, growth, concatenation, - formatted writes, length updates, and destruction. -- `cfl_list`: intrusive doubly linked list helpers. It provides initialization, - add, append, prepend, delete, concatenate, size, entry lookup, and safe - iteration macros. -- `cfl_kv`: SDS-backed string key/value entries stored in a `cfl_list`. This is - useful for simple string maps where values are plain strings. -- `cfl_variant`: tagged value container for bool, signed integer, unsigned - integer, double, null, reference, string, bytes, array, and key/value list - values. -- `cfl_array`: ordered collection of `cfl_variant` entries. It supports fixed - or resizable arrays, append helpers for all variant types, fetch by index, - removal, size inspection, and printing. -- `cfl_kvlist`: string-keyed map whose values are `cfl_variant` instances. It - supports typed insert helpers, size-aware key APIs, fetch, contains, remove, - count, and printing. -- `cfl_object`: generic wrapper that can hold a `cfl_kvlist`, `cfl_variant`, or - `cfl_array` and print the associated value. - -### Utilities - -- `cfl_atomic`: 64-bit atomic initialization, compare-exchange, store, and load - operations with platform-specific backends. -- `cfl_time`: wall-clock timestamp helper that returns nanoseconds. -- `cfl_hash`: naming wrappers around xxHash 64-bit and 128-bit hashing APIs. -- `cfl_checksum`: CRC32C checksum helper. -- `cfl_utils`: string split helpers, including quote-aware splitting, with - results returned as `cfl_list` entries. -- `cfl_log`: runtime error reporting helpers. - -### Support Headers - -- `cfl_compat`: platform compatibility macros. -- `cfl_found`: lightweight include-probe helper for parent projects. -- `cfl_info`: generated build information and feature flags. -- `cfl_version`: version macros. - -## Build and Test +CFL is a compact C library of data structures and low-level utilities used by +[Fluent Bit](https://fluentbit.io/) and its companion telemetry libraries. It +provides dynamic strings, intrusive lists, typed variants, arrays, key/value +containers, hashing, checksums, atomics, and an optional arena allocator. + +CFL started as a C library for Fluent Bit, a.k.a. `C:\Floppy`. + +The library is designed to be embedded. Callers can include the complete public +interface with: + +```c +#include +``` + +Individual headers under `include/cfl/` can be used when a component needs a +smaller interface. + +## Highlights + +- Small C API with no required runtime framework. +- Typed recursive values through `cfl_variant`, `cfl_array`, and `cfl_kvlist`. +- Length-aware dynamic strings through `cfl_sds`. +- Intrusive lists and lightweight string key/value entries. +- Portable 64-bit atomics and time helpers. +- xxHash wrappers and CRC32C checksums. +- Optional arenas for allocation-heavy, bounded object graphs and arbitrary + request-lifetime objects. +- CMake support for embedding, installation, tests, and benchmarks. + +## Core data structures + +| Interface | Purpose | +| --- | --- | +| `cfl_sds` | Growable binary-safe strings with explicit length and capacity | +| `cfl_list` | Intrusive doubly linked lists and safe iteration helpers | +| `cfl_kv` | String key/value entries stored in a CFL list | +| `cfl_variant` | Tagged bool, integer, double, null, reference, string, bytes, array, or map value | +| `cfl_array` | Ordered collection of variants | +| `cfl_kvlist` | String-keyed map of variants | +| `cfl_object` | Generic wrapper for a variant, array, or key/value list | +| `cfl_arena` | Optional shared allocator for bounded CFL object graphs | + +Heap-backed constructors remain the default. Applications that construct and +discard complete variant graphs can opt into `cfl_arena`; see +[ARENA.md](ARENA.md) for its ownership model, API, examples, and tuning advice. + +## Utility interfaces + +- `cfl_atomic`: 64-bit compare-exchange, store, and load operations with + platform-specific backends. +- `cfl_time`: wall-clock timestamps in nanoseconds. +- `cfl_hash`: xxHash 64-bit and 128-bit wrappers. +- `cfl_checksum`: CRC32C checksums. +- `cfl_utils`: string splitting, including quote-aware parsing. +- `cfl_log`: runtime error-reporting helpers. +- `cfl_compat`, `cfl_found`, and `cfl_info`: platform and build integration. + +## Build + +CFL requires CMake 3.20 or newer and a C compiler: + +```sh +cmake -S . -B build +cmake --build build -j8 +``` + +To build and run the unit tests: ```sh cmake -S . -B build -DCFL_TESTS=On @@ -67,10 +74,59 @@ cmake --build build -j8 ctest --test-dir build --output-on-failure ``` +Useful configuration options are: + +| Option | Default | Description | +| --- | --- | --- | +| `CFL_DEV` | `No` | Enable the debug development configuration and tests | +| `CFL_TESTS` | `No` | Build unit and public-header tests | +| `CFL_BENCHMARKS` | `No` | Build allocation and mutation benchmarks | +| `CFL_INSTALL_BUNDLED_XXHASH_HEADERS` | `Yes` | Install bundled xxHash headers | + +For arena performance and memory comparisons, see +[benchmarks/README.md](benchmarks/README.md). + +## Embedding with CMake + +Projects commonly embed CFL with `add_subdirectory` and link the static target: + +```cmake +add_subdirectory(path/to/cfl) +target_link_libraries(my_target PRIVATE cfl-static) +``` + +Applications using installed headers can include either `` or the +specific module headers they require. + +## API conventions + +- Public functions and types use the `cfl_` prefix. +- Constructors return `NULL` when allocation or validation fails. +- Container insertion functions report success or failure and document when + ownership transfers to the container. +- Heap-created values are destroyed through their matching CFL destroy APIs. +- Arena-created values are invalidated together by arena reset or destruction. +- An arena is not thread-safe; access to a shared arena must be serialized. + +## Documentation + +- Release history: see [CHANGELOG.md](CHANGELOG.md) for notable changes in each + version. +- Public API: the self-contained headers under [`include/cfl/`](include/cfl/) + describe each supported interface. +- Building and embedding: see the build and CMake examples above. +- Testing: enable `CFL_TESTS` and run the suite through CTest as shown above. +- Optional arena allocation: see [ARENA.md](ARENA.md) for lifecycle and + ownership rules. +- Performance tools: see [benchmarks/README.md](benchmarks/README.md) for the + supplied benchmark programs and measurement guidance. + ## License -This program is under the terms of the [Apache License v2.0](http://www.apache.org/licenses/LICENSE-2.0). +CFL is distributed under the +[Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ## Authors -Copyright is assigned to the `CFL Authors`, you can see a list of contributors [here](https://github.com/fluent/cfl/graphs/contributors). +Copyright is assigned to the CFL Authors. The contributor list is available on +[GitHub](https://github.com/fluent/cfl/graphs/contributors). diff --git a/lib/cfl/benchmarks/CMakeLists.txt b/lib/cfl/benchmarks/CMakeLists.txt new file mode 100644 index 00000000000..f1148df43e8 --- /dev/null +++ b/lib/cfl/benchmarks/CMakeLists.txt @@ -0,0 +1,5 @@ +add_executable(cfl-benchmark-variant-arena arena.c) +target_link_libraries(cfl-benchmark-variant-arena cfl-static) + +add_executable(cfl-benchmark-variant-mutable variant_mutable.c) +target_link_libraries(cfl-benchmark-variant-mutable cfl-static) diff --git a/lib/cfl/benchmarks/README.md b/lib/cfl/benchmarks/README.md new file mode 100644 index 00000000000..d4cb58f81fa --- /dev/null +++ b/lib/cfl/benchmarks/README.md @@ -0,0 +1,90 @@ +# Variant arena benchmark + +Configure a release build and run the heap and arena workloads with identical +parameters: + +```sh +cmake -S . -B build-bench -DCMAKE_BUILD_TYPE=Release -DCFL_BENCHMARKS=On +cmake --build build-bench -j8 +build-bench/benchmarks/cfl-benchmark-variant-arena heap 1000 1000 +build-bench/benchmarks/cfl-benchmark-variant-arena arena 1000 1000 8192 +``` + +Compare fixed 4 KiB chunks with optional 4-to-64 KiB geometric growth: + +```sh +build-bench/benchmarks/cfl-benchmark-variant-arena arena 1000 1000 4096 +build-bench/benchmarks/cfl-benchmark-variant-arena arena-grow 1000 1000 4096 65536 +``` + +The geometric mode exercises the public arena options used by request-lifetime +encoder workloads. Compare elapsed time, peak RSS, reserved bytes, used bytes, +and slack with a representative graph; fewer chunk allocations can trade CPU +time for retained capacity. + +The tool reports elapsed time, peak RSS, glibc heap usage, and arena +reserved/used bytes. Use `perf stat` for CPU and allocator-independent memory +events: + +```sh +perf stat -r 5 -e task-clock,cycles,instructions,cache-misses,page-faults \ + build-bench/benchmarks/cfl-benchmark-variant-arena heap 1000 1000 +perf stat -r 5 -e task-clock,cycles,instructions,cache-misses,page-faults \ + build-bench/benchmarks/cfl-benchmark-variant-arena arena 1000 1000 8192 +``` + +For fragmentation analysis, run both modes under heaptrack or Massif. Arena +slack is the difference between `arena_reserved` and `arena_used` for the last +constructed graph. + +## Mutable OTLP-style logs + +The mutable benchmark builds an OTLP JSON-like hierarchy of resource logs, +scope logs, log records, bodies, and attributes. Each mutation round replaces +severity and status attributes, appends a record, and removes the oldest one: + +```sh +build-bench/benchmarks/cfl-benchmark-variant-mutable heap 100 100 10 +build-bench/benchmarks/cfl-benchmark-variant-mutable arena 100 100 10 8192 + +perf stat -r 5 -e task-clock,cycles,instructions,cache-misses,page-faults \ + build-bench/benchmarks/cfl-benchmark-variant-mutable heap 100 100 10 +perf stat -r 5 -e task-clock,cycles,instructions,cache-misses,page-faults \ + build-bench/benchmarks/cfl-benchmark-variant-mutable arena 100 100 10 8192 +``` + +To compare a graph containing approximately 2 MiB of owned log-body strings: + +```sh +build-bench/benchmarks/cfl-benchmark-variant-mutable heap 10 100 10 8192 2097152 +build-bench/benchmarks/cfl-benchmark-variant-mutable arena 10 100 10 8192 2097152 +``` + +The optional arguments after content size are the large-object threshold and +payload distribution (`uniform`, `bimodal`, `heavy`, or `random`). A threshold +of zero selects the default of half the arena chunk size. A very large +threshold effectively disables external large-object allocation. The final +optional argument sets the external-buffer cache limit; zero disables caching. + +Run the deterministic validation matrix with: + +```sh +benchmarks/run_variant_matrix.sh build-bench > variant-matrix.txt +awk -f benchmarks/summarize_variant_matrix.awk variant-matrix.txt +``` + +It covers zero, 64 KiB, and 2 MiB content; immutable and replacement-heavy +lifetimes; four size distributions; three chunk sizes; and five large-object +policies. Output uses one `key=value` record per configuration for mechanical +comparison. `CFL_MATRIX_ITERATIONS` and `CFL_MATRIX_RECORDS` control runtime. + +The arena context is reused and reset between documents. Within one document, +the workload intentionally exposes growth from logically freed values: +individual removals do not reclaim arena storage until the next reset or +destroy. + +On glibc, `heap_initial_live` and `heap_final_live` are sampled while the heap +document is still alive. Compare them with `arena_initial_used` and +`arena_used`. `max_rss_kb` is process-level high-water RSS and can remain the +same for small workloads because it is page-granular and includes executable, +library, and allocator-retained pages. diff --git a/lib/cfl/benchmarks/arena.c b/lib/cfl/benchmarks/arena.c new file mode 100644 index 00000000000..f217378c96d --- /dev/null +++ b/lib/cfl/benchmarks/arena.c @@ -0,0 +1,157 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +#include +#include +#include +#include +#include + +#if !defined(CFL_SYSTEM_WINDOWS) +#include +#endif + +#if defined(__GLIBC__) +#include +#endif + +#include + +static uint64_t monotonic_nanoseconds(void) +{ + return cfl_time_now(); +} + +static int build_heap(size_t entries) +{ + struct cfl_kvlist *list; + size_t index; + char key[32]; + + list = cfl_kvlist_create(); + if (list == NULL) { + return -1; + } + for (index = 0; index < entries; index++) { + snprintf(key, sizeof(key), "key-%zu", index); + if (cfl_kvlist_insert_int64(list, key, (int64_t) index) != 0) { + cfl_kvlist_destroy(list); + return -1; + } + } + cfl_kvlist_destroy(list); + return 0; +} + +static int build_arena(size_t entries, size_t chunk_size, + size_t maximum_chunk_size, + size_t *reserved, size_t *used) +{ + struct cfl_arena *arena; + struct cfl_arena_options options; + struct cfl_kvlist *list; + size_t index; + char key[32]; + + if (maximum_chunk_size == 0) { + arena = cfl_arena_create(chunk_size); + } + else { + cfl_arena_options_init(&options); + options.chunk_size = chunk_size; + options.maximum_chunk_size = maximum_chunk_size; + arena = cfl_arena_create_with_options(&options); + } + if (arena == NULL) { + return -1; + } + list = cfl_kvlist_create_in(arena); + if (list == NULL) { + cfl_arena_destroy(arena); + return -1; + } + for (index = 0; index < entries; index++) { + snprintf(key, sizeof(key), "key-%zu", index); + if (cfl_kvlist_insert_int64(list, key, (int64_t) index) != 0) { + cfl_arena_destroy(arena); + return -1; + } + } + *reserved = cfl_arena_bytes_reserved(arena); + *used = cfl_arena_bytes_used(arena); + cfl_arena_destroy(arena); + return 0; +} + +int main(int argc, char **argv) +{ + const char *mode; + size_t iterations; + size_t entries; + size_t chunk_size; + size_t maximum_chunk_size; + size_t iteration; + size_t reserved; + size_t used; + uint64_t start; + uint64_t elapsed; +#if !defined(CFL_SYSTEM_WINDOWS) + struct rusage usage; +#endif +#if defined(__GLIBC__) + struct mallinfo2 memory; +#endif + + mode = argc > 1 ? argv[1] : "heap"; + iterations = argc > 2 ? strtoull(argv[2], NULL, 10) : 1000; + entries = argc > 3 ? strtoull(argv[3], NULL, 10) : 1000; + chunk_size = argc > 4 ? strtoull(argv[4], NULL, 10) : 8192; + maximum_chunk_size = argc > 5 ? strtoull(argv[5], NULL, 10) : 65536; + reserved = 0; + used = 0; + + start = monotonic_nanoseconds(); + for (iteration = 0; iteration < iterations; iteration++) { + if (strcmp(mode, "arena") == 0 || + strcmp(mode, "arena-grow") == 0) { + if (build_arena(entries, chunk_size, + strcmp(mode, "arena-grow") == 0 ? + maximum_chunk_size : 0, + &reserved, &used) != 0) { + return EXIT_FAILURE; + } + } + else if (strcmp(mode, "heap") == 0) { + if (build_heap(entries) != 0) { + return EXIT_FAILURE; + } + } + else { + fprintf(stderr, + "usage: %s heap|arena|arena-grow [iterations] " + "[entries] [chunk-size] [maximum-chunk-size]\n", + argv[0]); + return EXIT_FAILURE; + } + } + elapsed = monotonic_nanoseconds() - start; + + printf("mode=%s iterations=%zu entries=%zu elapsed_ns=%llu ns_per_entry=%.2f", + mode, iterations, entries, (unsigned long long) elapsed, + (double) elapsed / (double) (iterations * entries)); +#if !defined(CFL_SYSTEM_WINDOWS) + getrusage(RUSAGE_SELF, &usage); + printf(" max_rss_kb=%ld", usage.ru_maxrss); +#endif +#if defined(__GLIBC__) + memory = mallinfo2(); + printf(" heap_in_use=%zu heap_free=%zu", (size_t) memory.uordblks, + (size_t) memory.fordblks); +#endif + if (strcmp(mode, "arena") == 0 || + strcmp(mode, "arena-grow") == 0) { + printf(" arena_reserved=%zu arena_used=%zu arena_slack=%zu", + reserved, used, reserved - used); + } + putchar('\n'); + return EXIT_SUCCESS; +} diff --git a/lib/cfl/benchmarks/run_variant_matrix.sh b/lib/cfl/benchmarks/run_variant_matrix.sh new file mode 100755 index 00000000000..178cf3f7db5 --- /dev/null +++ b/lib/cfl/benchmarks/run_variant_matrix.sh @@ -0,0 +1,37 @@ +#!/bin/sh + +set -eu + +build_dir=${1:-build-bench} +benchmark="$build_dir/benchmarks/cfl-benchmark-variant-mutable" +iterations=${CFL_MATRIX_ITERATIONS:-20} +records=${CFL_MATRIX_RECORDS:-100} + +if [ ! -x "$benchmark" ]; then + echo "benchmark not found: $benchmark" >&2 + exit 1 +fi + +echo "matrix_version=1 iterations=$iterations records=$records" + +for content_size in 0 65536 2097152; do + distributions="uniform bimodal heavy random" + if [ "$content_size" -eq 0 ]; then + distributions="uniform" + fi + + for distribution in $distributions; do + for mutation_rounds in 0 10; do + "$benchmark" heap "$iterations" "$records" \ + "$mutation_rounds" 8192 "$content_size" 0 "$distribution" + + for chunk_size in 4096 8192 32768; do + for threshold in 0 1024 4096 16384 1073741824; do + "$benchmark" arena "$iterations" "$records" \ + "$mutation_rounds" "$chunk_size" "$content_size" \ + "$threshold" "$distribution" + done + done + done + done +done diff --git a/lib/cfl/benchmarks/summarize_variant_matrix.awk b/lib/cfl/benchmarks/summarize_variant_matrix.awk new file mode 100644 index 00000000000..a693fd6154a --- /dev/null +++ b/lib/cfl/benchmarks/summarize_variant_matrix.awk @@ -0,0 +1,41 @@ +function value(field, parts) +{ + split(field, parts, "=") + return parts[2] + 0 +} + +function text_value(field, parts) +{ + split(field, parts, "=") + return parts[2] +} + +$1 == "mode=heap" { + key = text_value($2) "/" value($5) "/" value($6) + heap_time[key] = value($10) + heap_rss[key] = value($11) +} + +$1 == "mode=arena" && value($7) == 8192 && value($8) == 0 { + key = text_value($2) "/" value($5) "/" value($6) + cpu_ratio = value($10) / heap_time[key] + rss_ratio = value($11) / heap_rss[key] + case_count++ + cpu_ratio_sum += cpu_ratio + rss_ratio_sum += rss_ratio + if (cpu_ratio < 1.0) { + cpu_wins++ + } + if (rss_ratio <= 1.0) { + rss_nonregressions++ + } + printf("case=%s cpu_ratio=%.3f rss_ratio=%.3f\n", + key, cpu_ratio, rss_ratio) +} + +END { + printf("summary_cases=%d cpu_wins=%d rss_nonregressions=%d " \ + "mean_cpu_ratio=%.3f mean_rss_ratio=%.3f\n", + case_count, cpu_wins, rss_nonregressions, + cpu_ratio_sum / case_count, rss_ratio_sum / case_count) +} diff --git a/lib/cfl/benchmarks/variant_mutable.c b/lib/cfl/benchmarks/variant_mutable.c new file mode 100644 index 00000000000..62a9868b4ff --- /dev/null +++ b/lib/cfl/benchmarks/variant_mutable.c @@ -0,0 +1,385 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +#include +#include +#include +#include +#include + +#if !defined(CFL_SYSTEM_WINDOWS) +#include +#endif + +#if defined(__GLIBC__) +#include +#endif + +#include + +struct otlp_document { + struct cfl_variant *root; + struct cfl_array *log_records; +}; + +static size_t record_payload_size(const char *distribution, + size_t total_size, size_t record_count, + size_t index) +{ + size_t large_count; + size_t small_count; + size_t small_size; + size_t remaining; + size_t weight; + + if (record_count == 0 || total_size == 0) { + return 0; + } + index %= record_count; + if (strcmp(distribution, "bimodal") == 0) { + large_count = (record_count + 9) / 10; + small_count = record_count - large_count; + small_size = total_size / record_count; + if (small_size > 128) { + small_size = 128; + } + if (index % 10 != 0) { + return small_size; + } + remaining = total_size - (small_size * small_count); + return remaining / large_count; + } + if (strcmp(distribution, "heavy") == 0) { + if (record_count == 1 || index == 0) { + return total_size / 2; + } + return (total_size - (total_size / 2)) / (record_count - 1); + } + if (strcmp(distribution, "random") == 0) { + weight = ((index * 1103515245U + 12345U) >> 16) % 31 + 1; + return (total_size / record_count) * weight / 16; + } + return total_size / record_count; +} + +static uint64_t wall_nanoseconds(void) +{ + return cfl_time_now(); +} + +static struct cfl_kvlist *create_record(struct cfl_arena *arena, + size_t sequence, + char *payload, size_t payload_size) +{ + struct cfl_kvlist *record; + struct cfl_kvlist *body; + struct cfl_kvlist *attributes; + char message[64]; + + record = cfl_kvlist_create_in(arena); + body = cfl_kvlist_create_in(arena); + attributes = cfl_kvlist_create_in(arena); + if (record == NULL || body == NULL || attributes == NULL) { + return NULL; + } + + snprintf(message, sizeof(message), "request completed sequence=%zu", sequence); + if (cfl_kvlist_insert_uint64(record, "timeUnixNano", sequence) != 0 || + cfl_kvlist_insert_string(record, "severityText", "INFO") != 0 || + (payload_size == 0 && + cfl_kvlist_insert_string(body, "stringValue", message) != 0) || + (payload_size > 0 && + cfl_kvlist_insert_string_s(body, "stringValue", 11, payload, + payload_size, CFL_FALSE) != 0) || + cfl_kvlist_insert_kvlist(record, "body", body) != 0 || + cfl_kvlist_insert_string(attributes, "service.name", "api") != 0 || + cfl_kvlist_insert_int64(attributes, "http.status_code", 200) != 0 || + cfl_kvlist_insert_kvlist(record, "attributes", attributes) != 0) { + return NULL; + } + + return record; +} + +static int create_document(struct cfl_arena *arena, size_t record_count, + char *payload, size_t content_size, + const char *distribution, + struct otlp_document *document) +{ + struct cfl_kvlist *root; + struct cfl_kvlist *resource_log; + struct cfl_kvlist *resource; + struct cfl_kvlist *scope_log; + struct cfl_kvlist *scope; + struct cfl_kvlist *record; + struct cfl_array *resource_logs; + struct cfl_array *scope_logs; + struct cfl_array *log_records; + size_t index; + + root = cfl_kvlist_create_in(arena); + resource_log = cfl_kvlist_create_in(arena); + resource = cfl_kvlist_create_in(arena); + scope_log = cfl_kvlist_create_in(arena); + scope = cfl_kvlist_create_in(arena); + resource_logs = cfl_array_create_in(arena, 1); + scope_logs = cfl_array_create_in(arena, 1); + log_records = cfl_array_create_in(arena, record_count + 1); + if (root == NULL || resource_log == NULL || resource == NULL || + scope_log == NULL || scope == NULL || resource_logs == NULL || + scope_logs == NULL || log_records == NULL) { + return -1; + } + + cfl_array_resizable(log_records, CFL_TRUE); + if (cfl_kvlist_insert_string(resource, "service.name", "checkout") != 0 || + cfl_kvlist_insert_kvlist(resource_log, "resource", resource) != 0 || + cfl_kvlist_insert_string(scope, "name", "benchmark.scope") != 0 || + cfl_kvlist_insert_kvlist(scope_log, "scope", scope) != 0) { + return -1; + } + + for (index = 0; index < record_count; index++) { + record = create_record(arena, index, payload, + record_payload_size(distribution, content_size, + record_count, index)); + if (record == NULL || cfl_array_append_kvlist(log_records, record) != 0) { + return -1; + } + } + + if (cfl_kvlist_insert_array(scope_log, "logRecords", log_records) != 0 || + cfl_array_append_kvlist(scope_logs, scope_log) != 0 || + cfl_kvlist_insert_array(resource_log, "scopeLogs", scope_logs) != 0 || + cfl_array_append_kvlist(resource_logs, resource_log) != 0 || + cfl_kvlist_insert_array(root, "resourceLogs", resource_logs) != 0) { + return -1; + } + + document->root = cfl_variant_create_from_kvlist_in(arena, root); + document->log_records = log_records; + return document->root == NULL ? -1 : 0; +} + +static int mutate_document(struct cfl_arena *arena, + struct otlp_document *document, + size_t mutation_round, + char *payload, size_t content_size, + const char *distribution) +{ + struct cfl_variant *record_variant; + struct cfl_variant *attributes_variant; + struct cfl_kvlist *record; + struct cfl_kvlist *attributes; + struct cfl_kvlist *new_record; + size_t index; + size_t payload_size; + + for (index = 0; index < document->log_records->entry_count; index++) { + record_variant = document->log_records->entries[index]; + record = record_variant->data.as_kvlist; + attributes_variant = cfl_kvlist_fetch(record, "attributes"); + if (attributes_variant == NULL) { + return -1; + } + attributes = attributes_variant->data.as_kvlist; + + cfl_kvlist_remove(record, "severityText"); + cfl_kvlist_remove(attributes, "http.status_code"); + if (cfl_kvlist_insert_string(record, "severityText", + mutation_round % 2 == 0 ? "WARN" : "INFO") != 0 || + cfl_kvlist_insert_int64(attributes, "http.status_code", + mutation_round % 2 == 0 ? 503 : 200) != 0) { + return -1; + } + } + + payload_size = record_payload_size(distribution, content_size, + document->log_records->entry_count, + mutation_round); + new_record = create_record(arena, mutation_round + 1000000, + payload, payload_size); + if (new_record == NULL || + cfl_array_append_kvlist(document->log_records, new_record) != 0 || + cfl_array_remove_by_index(document->log_records, 0) != 0) { + return -1; + } + return 0; +} + +int main(int argc, char **argv) +{ + const char *mode; + struct cfl_arena *arena; + struct otlp_document document; + size_t iterations; + size_t records; + size_t mutation_rounds; + size_t chunk_size; + size_t content_size; + size_t payload_size; + size_t large_object_threshold; + size_t external_cache_limit; + int external_cache_limit_set; + size_t iteration; + size_t round; + size_t reserved; + size_t used; + size_t initial_used; + size_t arena_cache_bytes; + size_t arena_cache_limit; + uint64_t start; + uint64_t elapsed; + double operation_count; + char *payload; + const char *distribution; +#if !defined(CFL_SYSTEM_WINDOWS) + struct rusage usage; +#endif +#if defined(__GLIBC__) + struct mallinfo2 memory; + size_t heap_baseline; + size_t heap_initial_live; + size_t heap_final_live; +#endif + + mode = argc > 1 ? argv[1] : "heap"; + iterations = argc > 2 ? strtoull(argv[2], NULL, 10) : 100; + records = argc > 3 ? strtoull(argv[3], NULL, 10) : 100; + mutation_rounds = argc > 4 ? strtoull(argv[4], NULL, 10) : 10; + chunk_size = argc > 5 ? strtoull(argv[5], NULL, 10) : 8192; + content_size = argc > 6 ? strtoull(argv[6], NULL, 10) : 0; + large_object_threshold = argc > 7 ? strtoull(argv[7], NULL, 10) : 0; + distribution = argc > 8 ? argv[8] : "uniform"; + external_cache_limit_set = argc > 9; + external_cache_limit = external_cache_limit_set ? + strtoull(argv[9], NULL, 10) : 0; + payload_size = content_size; + reserved = 0; + used = 0; + initial_used = 0; + arena_cache_bytes = 0; + arena_cache_limit = 0; + if (strcmp(mode, "heap") != 0 && strcmp(mode, "arena") != 0) { + fprintf(stderr, "usage: %s heap|arena [iterations] [records] " + "[mutation-rounds] [chunk-size] [content-bytes] " + "[large-object-threshold] [distribution] " + "[external-cache-limit]\n", argv[0]); + return EXIT_FAILURE; + } + + payload = NULL; + if (payload_size > 0) { + payload = malloc(payload_size); + if (payload == NULL) { + return EXIT_FAILURE; + } + memset(payload, 'x', payload_size); + } + +#if defined(__GLIBC__) + memory = mallinfo2(); + heap_baseline = (size_t) memory.uordblks; + heap_initial_live = 0; + heap_final_live = 0; +#endif + + arena = NULL; + if (strcmp(mode, "arena") == 0) { + arena = cfl_arena_create_ex(chunk_size, + large_object_threshold); + if (arena == NULL) { + return EXIT_FAILURE; + } + if (external_cache_limit_set) { + cfl_arena_external_cache_limit_set(arena, + external_cache_limit); + } + } + + start = wall_nanoseconds(); + for (iteration = 0; iteration < iterations; iteration++) { + if (create_document(arena, records, payload, content_size, + distribution, + &document) != 0) { + return EXIT_FAILURE; + } + if (arena != NULL) { + initial_used = cfl_arena_bytes_used(arena); + } +#if defined(__GLIBC__) + else { + memory = mallinfo2(); + heap_initial_live = (size_t) memory.uordblks - heap_baseline; + } +#endif + for (round = 0; round < mutation_rounds; round++) { + if (mutate_document(arena, &document, round, + payload, content_size, distribution) != 0) { + return EXIT_FAILURE; + } + } + +#if defined(__GLIBC__) + if (arena == NULL) { + memory = mallinfo2(); + heap_final_live = (size_t) memory.uordblks - heap_baseline; + } +#endif + + if (arena == NULL) { + cfl_variant_destroy(document.root); + } + else { + reserved = cfl_arena_bytes_reserved(arena); + used = cfl_arena_bytes_used(arena); + arena_cache_bytes = + cfl_arena_external_cache_bytes(arena); + arena_cache_limit = + cfl_arena_external_cache_limit_get(arena); + if (iteration + 1 < iterations) { + cfl_arena_reset(arena); + } + } + } + elapsed = wall_nanoseconds() - start; + operation_count = (double) iterations * (double) records; + if (mutation_rounds > 0) { + operation_count *= (double) mutation_rounds; + } + + if (arena != NULL) { + cfl_arena_destroy(arena); + } + printf("mode=%s distribution=%s iterations=%zu records=%zu " + "mutation_rounds=%zu content_bytes=%zu chunk_size=%zu threshold=%zu " + "elapsed_ns=%llu ns_per_operation=%.2f", + mode, distribution, iterations, records, mutation_rounds, + content_size, chunk_size, large_object_threshold, + (unsigned long long) elapsed, + (double) elapsed / operation_count); +#if !defined(CFL_SYSTEM_WINDOWS) + getrusage(RUSAGE_SELF, &usage); + printf(" max_rss_kb=%ld", usage.ru_maxrss); +#endif +#if defined(__GLIBC__) + memory = mallinfo2(); + if (strcmp(mode, "heap") == 0) { + printf(" heap_initial_live=%zu heap_final_live=%zu " + "heap_retained_after_destroy=%zu heap_free=%zu", + heap_initial_live, heap_final_live, + (size_t) memory.uordblks - heap_baseline, + (size_t) memory.fordblks); + } +#endif + if (strcmp(mode, "arena") == 0) { + printf(" arena_reserved=%zu arena_initial_used=%zu arena_used=%zu " + "arena_mutation_growth=%zu arena_slack=%zu " + "arena_external_cache=%zu arena_external_cache_limit=%zu", + reserved, initial_used, used, used - initial_used, + reserved - used, + arena_cache_bytes, arena_cache_limit); + } + putchar('\n'); + free(payload); + return EXIT_SUCCESS; +} diff --git a/lib/cfl/include/cfl/cfl.h b/lib/cfl/include/cfl/cfl.h index 3491e0c8bcb..664e2346b36 100644 --- a/lib/cfl/include/cfl/cfl.h +++ b/lib/cfl/include/cfl/cfl.h @@ -38,6 +38,7 @@ #include #include #include +#include #include #include diff --git a/lib/cfl/include/cfl/cfl_arena.h b/lib/cfl/include/cfl/cfl_arena.h new file mode 100644 index 00000000000..2134df2a5b7 --- /dev/null +++ b/lib/cfl/include/cfl/cfl_arena.h @@ -0,0 +1,64 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +#ifndef CFL_ARENA_H +#define CFL_ARENA_H + +#include + +struct cfl_arena; + +typedef void *(*cfl_arena_malloc_fn)(void *context, size_t size); +typedef void (*cfl_arena_free_fn)(void *context, void *pointer); + +struct cfl_arena_options { + size_t struct_size; + size_t chunk_size; + size_t maximum_chunk_size; + size_t large_object_threshold; + cfl_arena_malloc_fn malloc_fn; + cfl_arena_free_fn free_fn; + void *allocator_context; +}; + +/* + * Arena-created objects remain valid until the arena is reset or destroyed. + * Reset and destroy invalidate every pointer allocated from the arena. + * Objects from different arenas, including heap-backed objects, cannot be + * attached to the same array or kvlist. + * An arena is not thread-safe. Callers must serialize all operations that use + * the same arena, including allocation, mutation, reset, and destruction. + * The reserved byte count includes CFL's allocation headers, but not allocator + * implementation metadata. The used byte count is the live payload capacity. + */ +struct cfl_arena *cfl_arena_create(size_t chunk_size); +struct cfl_arena *cfl_arena_create_ex(size_t chunk_size, + size_t large_object_threshold); +void cfl_arena_options_init(struct cfl_arena_options *options); +struct cfl_arena *cfl_arena_create_with_options( + const struct cfl_arena_options *options); +void cfl_arena_destroy(struct cfl_arena *arena); +void cfl_arena_reset(struct cfl_arena *arena); + +/* + * Raw allocations are aligned for CFL-supported fundamental C types. They + * cannot be freed individually and remain valid until reset or destruction. + * A zero-sized or overflowing request returns NULL. Allocation failure leaves + * the arena valid and does not define errno. + */ +void *cfl_arena_malloc(struct cfl_arena *arena, size_t size); +void *cfl_arena_calloc(struct cfl_arena *arena, + size_t count, size_t size); +void *cfl_arena_memdup(struct cfl_arena *arena, + const void *source, size_t size); +char *cfl_arena_strndup(struct cfl_arena *arena, + const char *source, size_t length); + +size_t cfl_arena_bytes_reserved(struct cfl_arena *arena); +size_t cfl_arena_bytes_used(struct cfl_arena *arena); +size_t cfl_arena_large_object_threshold(struct cfl_arena *arena); +void cfl_arena_external_cache_limit_set(struct cfl_arena *arena, + size_t limit); +size_t cfl_arena_external_cache_limit_get(struct cfl_arena *arena); +size_t cfl_arena_external_cache_bytes(struct cfl_arena *arena); + +#endif diff --git a/lib/cfl/include/cfl/cfl_array.h b/lib/cfl/include/cfl/cfl_array.h index f057bf4539a..7d26b50d900 100644 --- a/lib/cfl/include/cfl/cfl_array.h +++ b/lib/cfl/include/cfl/cfl_array.h @@ -27,6 +27,7 @@ #include struct cfl_kvlist; +struct cfl_arena; struct cfl_array { int resizable; @@ -36,9 +37,14 @@ struct cfl_array { struct cfl_variant *owner; struct cfl_array *parent_array; struct cfl_kvlist *parent_kvlist; + struct cfl_arena *arena; }; struct cfl_array *cfl_array_create(size_t slot_count); +struct cfl_array *cfl_array_create_in(struct cfl_arena *arena, + size_t slot_count); +struct cfl_array *cfl_array_create_like(struct cfl_array *parent, + size_t slot_count); void cfl_array_destroy(struct cfl_array *array); static inline struct cfl_variant *cfl_array_fetch_by_index(struct cfl_array *array, diff --git a/lib/cfl/include/cfl/cfl_kvlist.h b/lib/cfl/include/cfl/cfl_kvlist.h index cd8d90487a9..277c5b34a8e 100644 --- a/lib/cfl/include/cfl/cfl_kvlist.h +++ b/lib/cfl/include/cfl/cfl_kvlist.h @@ -29,11 +29,18 @@ #include struct cfl_array; +struct cfl_arena; + +enum cfl_kvlist_match_mode { + CFL_KVLIST_MATCH_CASE_INSENSITIVE = 0, + CFL_KVLIST_MATCH_CASE_SENSITIVE +}; struct cfl_kvpair { cfl_sds_t key; /* Key */ struct cfl_variant *val; /* Value */ struct cfl_list _head; /* Link to list cfl_kvlist->list */ + struct cfl_arena *arena; }; struct cfl_kvlist { @@ -41,9 +48,12 @@ struct cfl_kvlist { struct cfl_variant *owner; struct cfl_array *parent_array; struct cfl_kvlist *parent_kvlist; + struct cfl_arena *arena; }; struct cfl_kvlist *cfl_kvlist_create(); +struct cfl_kvlist *cfl_kvlist_create_in(struct cfl_arena *arena); +struct cfl_kvlist *cfl_kvlist_create_like(struct cfl_kvlist *parent); void cfl_kvlist_destroy(struct cfl_kvlist *list); /* @@ -138,12 +148,31 @@ int cfl_kvlist_insert_s(struct cfl_kvlist *list, char *key, size_t key_size, struct cfl_variant *value); -struct cfl_variant *cfl_kvlist_fetch_s(struct cfl_kvlist *list, char *key, size_t key_size); +struct cfl_variant *cfl_kvlist_fetch_s(struct cfl_kvlist *list, + char *key, size_t key_size); +/* The existing fetch, contains, and remove APIs match case-insensitively. */ +struct cfl_variant *cfl_kvlist_fetch_ex(struct cfl_kvlist *list, + char *key, + enum cfl_kvlist_match_mode mode); +struct cfl_variant *cfl_kvlist_fetch_s_ex(struct cfl_kvlist *list, + char *key, size_t key_size, + enum cfl_kvlist_match_mode mode); +struct cfl_variant *cfl_kvlist_fetch_case_s(struct cfl_kvlist *list, + char *key, size_t key_size); int cfl_kvlist_contains(struct cfl_kvlist *kvlist, char *name); +int cfl_kvlist_contains_ex(struct cfl_kvlist *kvlist, char *name, + enum cfl_kvlist_match_mode mode); int cfl_kvlist_remove(struct cfl_kvlist *kvlist, char *name); +int cfl_kvlist_remove_ex(struct cfl_kvlist *kvlist, char *name, + enum cfl_kvlist_match_mode mode); void cfl_kvpair_destroy(struct cfl_kvpair *pair); struct cfl_variant *cfl_kvpair_take_value(struct cfl_kvpair *pair); +int cfl_kvpair_key_set_s(struct cfl_kvpair *pair, + char *key, size_t key_size); +int cfl_kvlist_rename_s(struct cfl_kvlist *list, + char *old_key, size_t old_key_size, + char *new_key, size_t new_key_size); #endif diff --git a/lib/cfl/include/cfl/cfl_log.h b/lib/cfl/include/cfl/cfl_log.h index 9ceb0015b66..6aa2b7997ec 100644 --- a/lib/cfl/include/cfl/cfl_log.h +++ b/lib/cfl/include/cfl/cfl_log.h @@ -24,12 +24,13 @@ int cfl_report_runtime_error_impl(int errnum, char *file, int line); -#ifdef __FILENAME__ -#define cfl_report_runtime_error() cfl_report_runtime_error_impl(errno, __FILENAME__, __LINE__) -#else -#define cfl_report_runtime_error() cfl_report_runtime_error_impl(errno, __FILE__, __LINE__) +#ifndef CFL_FILENAME +#define CFL_FILENAME __FILE__ #endif +#define cfl_report_runtime_error() \ + cfl_report_runtime_error_impl(errno, CFL_FILENAME, __LINE__) + #define cfl_errno() do {} while (0) #endif diff --git a/lib/cfl/include/cfl/cfl_sds.h b/lib/cfl/include/cfl/cfl_sds.h index a923882a616..bcb85a4b6d4 100644 --- a/lib/cfl/include/cfl/cfl_sds.h +++ b/lib/cfl/include/cfl/cfl_sds.h @@ -29,10 +29,11 @@ #include #include #include +typedef char *cfl_sds_t; -#define CFL_SDS_HEADER_SIZE (sizeof(uint64_t) + sizeof(uint64_t)) +struct cfl_arena; -typedef char *cfl_sds_t; +#define CFL_SDS_HEADER_SIZE (sizeof(uint64_t) + sizeof(uint64_t)) #pragma pack(push, 1) struct cfl_sds { @@ -44,6 +45,8 @@ struct cfl_sds { #define CFL_SDS_HEADER(s) ((struct cfl_sds *) (s - CFL_SDS_HEADER_SIZE)) +size_t cfl_sds_alloc(cfl_sds_t s); + static inline void cfl_sds_len_set(cfl_sds_t s, size_t len) { struct cfl_sds *head; @@ -53,7 +56,7 @@ static inline void cfl_sds_len_set(cfl_sds_t s, size_t len) } head = CFL_SDS_HEADER(s); - if (len > head->alloc) { + if (len > cfl_sds_alloc(s)) { return; } @@ -62,10 +65,11 @@ static inline void cfl_sds_len_set(cfl_sds_t s, size_t len) } size_t cfl_sds_avail(cfl_sds_t s); -size_t cfl_sds_alloc(cfl_sds_t s); cfl_sds_t cfl_sds_increase(cfl_sds_t s, size_t len); size_t cfl_sds_len(cfl_sds_t s); cfl_sds_t cfl_sds_create_len(const char *str, int len); +cfl_sds_t cfl_sds_create_len_in(struct cfl_arena *arena, + const char *str, int len); cfl_sds_t cfl_sds_create(const char *str); void cfl_sds_destroy(cfl_sds_t s); cfl_sds_t cfl_sds_cat(cfl_sds_t s, const char *str, int len); diff --git a/lib/cfl/include/cfl/cfl_variant.h b/lib/cfl/include/cfl/cfl_variant.h index efdcfe96bfa..6928b54f58f 100644 --- a/lib/cfl/include/cfl/cfl_variant.h +++ b/lib/cfl/include/cfl/cfl_variant.h @@ -41,6 +41,7 @@ struct cfl_array; struct cfl_kvlist; +struct cfl_arena; struct cfl_variant { int type; @@ -70,6 +71,8 @@ struct cfl_variant { struct cfl_array *as_array; struct cfl_kvlist *as_kvlist; } data; + + struct cfl_arena *arena; }; int cfl_variant_print(FILE *fp, struct cfl_variant *val); @@ -85,6 +88,31 @@ struct cfl_variant *cfl_variant_create_from_null(); struct cfl_variant *cfl_variant_create_from_kvlist(struct cfl_kvlist *value); struct cfl_variant *cfl_variant_create_from_reference(void *value); struct cfl_variant *cfl_variant_create(); +struct cfl_variant *cfl_variant_create_in(struct cfl_arena *arena); +struct cfl_variant *cfl_variant_create_from_string_in(struct cfl_arena *arena, + char *value); +struct cfl_variant *cfl_variant_create_from_string_s_in(struct cfl_arena *arena, + char *value, + size_t value_length, + int referenced); +struct cfl_variant *cfl_variant_create_from_bytes_in(struct cfl_arena *arena, + char *value, size_t length, + int referenced); +struct cfl_variant *cfl_variant_create_from_bool_in(struct cfl_arena *arena, + int value); +struct cfl_variant *cfl_variant_create_from_int64_in(struct cfl_arena *arena, + int64_t value); +struct cfl_variant *cfl_variant_create_from_uint64_in(struct cfl_arena *arena, + uint64_t value); +struct cfl_variant *cfl_variant_create_from_double_in(struct cfl_arena *arena, + double value); +struct cfl_variant *cfl_variant_create_from_null_in(struct cfl_arena *arena); +struct cfl_variant *cfl_variant_create_from_reference_in(struct cfl_arena *arena, + void *value); +struct cfl_variant *cfl_variant_create_from_array_in(struct cfl_arena *arena, + struct cfl_array *value); +struct cfl_variant *cfl_variant_create_from_kvlist_in(struct cfl_arena *arena, + struct cfl_kvlist *value); void cfl_variant_destroy(struct cfl_variant *instance); diff --git a/lib/cfl/src/CMakeLists.txt b/lib/cfl/src/CMakeLists.txt index de94f0007a8..86d3bebc156 100644 --- a/lib/cfl/src/CMakeLists.txt +++ b/lib/cfl/src/CMakeLists.txt @@ -10,6 +10,7 @@ set(src cfl_object.c cfl_array.c cfl_variant.c + cfl_arena.c cfl_container.c cfl_checksum.c cfl_utils.c @@ -80,7 +81,11 @@ set(src # Static Library add_library(cfl-static STATIC ${src}) -target_link_libraries(cfl-static PRIVATE xxhash) +set_target_properties(cfl-static PROPERTIES + C_STANDARD ${CMAKE_C_STANDARD} + C_STANDARD_REQUIRED On + C_EXTENSIONS On) +target_link_libraries(cfl-static PUBLIC xxhash) if(CFL_ATOMIC_NEEDS_LIBATOMIC) target_link_libraries(cfl-static PUBLIC atomic) @@ -104,3 +109,9 @@ install(TARGETS cfl-static LIBRARY DESTINATION ${CFL_INSTALL_LIBDIR} ARCHIVE DESTINATION ${CFL_INSTALL_LIBDIR} COMPONENT library) + +if(CFL_BUNDLED_XXHASH AND CFL_INSTALL_BUNDLED_XXHASH_HEADERS) + install(TARGETS xxhash + ARCHIVE DESTINATION ${CFL_INSTALL_LIBDIR} + COMPONENT library) +endif() diff --git a/lib/cfl/src/cfl_arena.c b/lib/cfl/src/cfl_arena.c new file mode 100644 index 00000000000..62fa165fddd --- /dev/null +++ b/lib/cfl/src/cfl_arena.c @@ -0,0 +1,704 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +#include +#include +#include + +#include + +#include "cfl_arena_internal.h" + +#define CFL_ARENA_DEFAULT_CHUNK_SIZE 8192 +#define CFL_ARENA_SDS_CLASS_COUNT 6 +#define CFL_ARENA_EXTERNAL_CLASS_COUNT 24 +#define CFL_ARENA_EXTERNAL_EXACT_CLASS UINT8_MAX + +union cfl_arena_max_align { + long double long_double_value; + void *pointer_value; + uint64_t uint64_value; +}; + +struct cfl_arena_alignment_probe { + char byte; + union cfl_arena_max_align value; +}; + +struct cfl_arena_chunk { + struct cfl_arena_chunk *next; + size_t capacity; + size_t used; + union cfl_arena_max_align alignment; + unsigned char data[]; +}; + +struct cfl_arena_external { + struct cfl_arena_external *next; + struct cfl_arena_external *previous; + size_t size; + uint8_t allocation_class; + unsigned char data[]; +}; + +struct cfl_arena { + struct cfl_arena_chunk *head; + struct cfl_arena_chunk *current; + struct cfl_arena_external *external; + struct cfl_arena_external *external_cache[CFL_ARENA_EXTERNAL_CLASS_COUNT]; + struct cfl_arena_external *external_exact_cache; + size_t external_cache_count[CFL_ARENA_EXTERNAL_CLASS_COUNT]; + size_t external_cache_bytes; + size_t external_cache_limit; + size_t chunk_size; + size_t maximum_chunk_size; + size_t next_chunk_size; + size_t bytes_reserved; + size_t bytes_used; + size_t large_object_threshold; + void *free_variants; + void *free_kvpairs; + void *free_sds[CFL_ARENA_SDS_CLASS_COUNT]; + cfl_arena_malloc_fn malloc_fn; + cfl_arena_free_fn free_fn; + void *allocator_context; +}; + +static void *arena_default_malloc(void *context, size_t size) +{ + (void) context; + return malloc(size); +} + +static void arena_default_free(void *context, void *pointer) +{ + (void) context; + free(pointer); +} + +static void arena_chunks_destroy(struct cfl_arena *arena) +{ + struct cfl_arena_chunk *chunk; + struct cfl_arena_chunk *next; + + chunk = arena->head; + while (chunk != NULL) { + next = chunk->next; + arena->free_fn(arena->allocator_context, chunk); + chunk = next; + } + + arena->head = NULL; + arena->current = NULL; + arena->bytes_reserved = 0; + arena->bytes_used = 0; +} + +static void arena_external_destroy(struct cfl_arena *arena) +{ + struct cfl_arena_external *allocation; + struct cfl_arena_external *next; + size_t index; + + allocation = arena->external; + while (allocation != NULL) { + next = allocation->next; + arena->bytes_reserved -= allocation->size + + sizeof(struct cfl_arena_external); + arena->bytes_used -= allocation->size; + arena->free_fn(arena->allocator_context, allocation); + allocation = next; + } + arena->external = NULL; + + for (index = 0; index < CFL_ARENA_EXTERNAL_CLASS_COUNT; index++) { + allocation = arena->external_cache[index]; + while (allocation != NULL) { + next = allocation->next; + arena->bytes_reserved -= allocation->size + + sizeof(struct cfl_arena_external); + arena->free_fn(arena->allocator_context, allocation); + allocation = next; + } + arena->external_cache[index] = NULL; + arena->external_cache_count[index] = 0; + } + arena->external_cache_bytes = 0; + + allocation = arena->external_exact_cache; + while (allocation != NULL) { + next = allocation->next; + arena->bytes_reserved -= allocation->size + + sizeof(struct cfl_arena_external); + arena->free_fn(arena->allocator_context, allocation); + allocation = next; + } + arena->external_exact_cache = NULL; +} + +struct cfl_arena *cfl_arena_create(size_t chunk_size) +{ + return cfl_arena_create_ex(chunk_size, 0); +} + +struct cfl_arena *cfl_arena_create_ex(size_t chunk_size, + size_t large_object_threshold) +{ + struct cfl_arena_options options; + + cfl_arena_options_init(&options); + options.chunk_size = chunk_size; + options.large_object_threshold = large_object_threshold; + + return cfl_arena_create_with_options(&options); +} + +void cfl_arena_options_init(struct cfl_arena_options *options) +{ + if (options == NULL) { + return; + } + + memset(options, 0, sizeof(struct cfl_arena_options)); + options->struct_size = sizeof(struct cfl_arena_options); +} + +struct cfl_arena *cfl_arena_create_with_options( + const struct cfl_arena_options *options) +{ + struct cfl_arena *arena; + size_t chunk_size; + size_t maximum_chunk_size; + size_t large_object_threshold; + cfl_arena_malloc_fn malloc_fn; + cfl_arena_free_fn free_fn; + void *allocator_context; + + if (options == NULL || + options->struct_size < + offsetof(struct cfl_arena_options, allocator_context) + + sizeof(options->allocator_context)) { + return NULL; + } + + if ((options->malloc_fn == NULL) != (options->free_fn == NULL)) { + return NULL; + } + + chunk_size = options->chunk_size; + maximum_chunk_size = options->maximum_chunk_size; + large_object_threshold = options->large_object_threshold; + malloc_fn = options->malloc_fn; + free_fn = options->free_fn; + allocator_context = options->allocator_context; + + if (chunk_size == 0) { + chunk_size = CFL_ARENA_DEFAULT_CHUNK_SIZE; + } + if (maximum_chunk_size == 0) { + maximum_chunk_size = chunk_size; + } + if (maximum_chunk_size < chunk_size) { + return NULL; + } + if (malloc_fn == NULL) { + malloc_fn = arena_default_malloc; + free_fn = arena_default_free; + allocator_context = NULL; + } + + arena = malloc_fn(allocator_context, sizeof(struct cfl_arena)); + if (arena == NULL) { + return NULL; + } + memset(arena, 0, sizeof(struct cfl_arena)); + + arena->chunk_size = chunk_size; + arena->maximum_chunk_size = maximum_chunk_size; + arena->next_chunk_size = chunk_size; + arena->malloc_fn = malloc_fn; + arena->free_fn = free_fn; + arena->allocator_context = allocator_context; + if (large_object_threshold == 0) { + large_object_threshold = chunk_size / 2; + if (large_object_threshold == 0) { + large_object_threshold = 1; + } + } + arena->large_object_threshold = large_object_threshold; + if (chunk_size <= SIZE_MAX / 256) { + arena->external_cache_limit = chunk_size * 256; + } + else { + arena->external_cache_limit = SIZE_MAX; + } + return arena; +} + +void cfl_arena_destroy(struct cfl_arena *arena) +{ + if (arena == NULL) { + return; + } + + arena_external_destroy(arena); + arena_chunks_destroy(arena); + arena->free_fn(arena->allocator_context, arena); +} + +void cfl_arena_reset(struct cfl_arena *arena) +{ + struct cfl_arena_chunk *chunk; + struct cfl_arena_external *allocation; + struct cfl_arena_external *next; + + if (arena == NULL) { + return; + } + + allocation = arena->external; + while (allocation != NULL) { + next = allocation->next; + cfl_arena_free_external(arena, allocation->data); + allocation = next; + } + + chunk = arena->head; + while (chunk != NULL) { + chunk->used = 0; + chunk = chunk->next; + } + + arena->bytes_used = 0; + arena->current = arena->head; + arena->next_chunk_size = arena->chunk_size; + arena->free_variants = NULL; + arena->free_kvpairs = NULL; + memset(arena->free_sds, 0, sizeof(arena->free_sds)); +} + +void *cfl_arena_malloc(struct cfl_arena *arena, size_t size) +{ + struct cfl_arena_chunk *chunk; + size_t alignment; + size_t offset; + size_t remainder; + size_t capacity; + void *result; + + if (arena == NULL || size == 0) { + return NULL; + } + + alignment = offsetof(struct cfl_arena_alignment_probe, value); + chunk = arena->current; + while (chunk != NULL) { + offset = chunk->used; + remainder = offset % alignment; + if (remainder != 0) { + if (offset > SIZE_MAX - (alignment - remainder)) { + return NULL; + } + offset += alignment - remainder; + } + if (offset <= chunk->capacity && size <= chunk->capacity - offset) { + result = &chunk->data[offset]; + chunk->used = offset + size; + arena->current = chunk; + arena->bytes_used += size; + return result; + } + chunk = chunk->next; + } + + capacity = arena->next_chunk_size; + if (capacity < size) { + capacity = size; + } + if (capacity > SIZE_MAX - sizeof(struct cfl_arena_chunk)) { + return NULL; + } + + chunk = arena->malloc_fn(arena->allocator_context, + sizeof(struct cfl_arena_chunk) + capacity); + if (chunk == NULL) { + return NULL; + } + + chunk->next = arena->head; + chunk->capacity = capacity; + chunk->used = size; + arena->head = chunk; + arena->current = chunk; + arena->bytes_reserved += capacity + + sizeof(struct cfl_arena_chunk); + arena->bytes_used += size; + + if (capacity == arena->next_chunk_size && + arena->next_chunk_size < arena->maximum_chunk_size) { + if (arena->next_chunk_size > arena->maximum_chunk_size / 2) { + arena->next_chunk_size = arena->maximum_chunk_size; + } + else { + arena->next_chunk_size *= 2; + } + } + + return chunk->data; +} + +void *cfl_arena_alloc(struct cfl_arena *arena, size_t size) +{ + return cfl_arena_malloc(arena, size); +} + +void *cfl_arena_calloc(struct cfl_arena *arena, + size_t count, size_t size) +{ + void *result; + size_t total; + + if (count != 0 && size > SIZE_MAX / count) { + return NULL; + } + + total = count * size; + result = cfl_arena_malloc(arena, total); + if (result != NULL) { + memset(result, 0, total); + } + + return result; +} + +void *cfl_arena_memdup(struct cfl_arena *arena, + const void *source, size_t size) +{ + void *result; + + if (source == NULL || size == 0) { + return NULL; + } + + result = cfl_arena_malloc(arena, size); + if (result != NULL) { + memcpy(result, source, size); + } + + return result; +} + +char *cfl_arena_strndup(struct cfl_arena *arena, + const char *source, size_t length) +{ + char *result; + + if (source == NULL || length == SIZE_MAX) { + return NULL; + } + + result = cfl_arena_malloc(arena, length + 1); + if (result != NULL) { + memcpy(result, source, length); + result[length] = '\0'; + } + + return result; +} + +size_t cfl_arena_bytes_reserved(struct cfl_arena *arena) +{ + return arena == NULL ? 0 : arena->bytes_reserved; +} + +size_t cfl_arena_bytes_used(struct cfl_arena *arena) +{ + return arena == NULL ? 0 : arena->bytes_used; +} + +size_t cfl_arena_large_object_threshold(struct cfl_arena *arena) +{ + return arena == NULL ? 0 : arena->large_object_threshold; +} + +void cfl_arena_external_cache_limit_set(struct cfl_arena *arena, + size_t limit) +{ + struct cfl_arena_external *allocation; + size_t index; + + if (arena == NULL) { + return; + } + + arena->external_cache_limit = limit; + for (index = CFL_ARENA_EXTERNAL_CLASS_COUNT; + index > 0 && arena->external_cache_bytes > limit; + index--) { + while (arena->external_cache[index - 1] != NULL && + arena->external_cache_bytes > limit) { + allocation = arena->external_cache[index - 1]; + arena->external_cache[index - 1] = allocation->next; + arena->external_cache_count[index - 1]--; + arena->external_cache_bytes -= allocation->size; + arena->bytes_reserved -= allocation->size + + sizeof(struct cfl_arena_external); + arena->free_fn(arena->allocator_context, allocation); + } + } + + while (arena->external_exact_cache != NULL && + arena->external_cache_bytes > limit) { + allocation = arena->external_exact_cache; + arena->external_exact_cache = allocation->next; + arena->external_cache_bytes -= allocation->size; + arena->bytes_reserved -= allocation->size + + sizeof(struct cfl_arena_external); + arena->free_fn(arena->allocator_context, allocation); + } +} + +size_t cfl_arena_external_cache_limit_get(struct cfl_arena *arena) +{ + return arena == NULL ? 0 : arena->external_cache_limit; +} + +size_t cfl_arena_external_cache_bytes(struct cfl_arena *arena) +{ + return arena == NULL ? 0 : arena->external_cache_bytes; +} + +void *cfl_arena_alloc_external(struct cfl_arena *arena, + size_t size) +{ + static const size_t class_sizes[CFL_ARENA_EXTERNAL_CLASS_COUNT] = { + 4096, 6144, 8192, 12288, 16384, 24576, + 32768, 49152, 65536, 98304, 131072, 196608, + 262144, 327680, 393216, 458752, 524288, 655360, + 786432, 917504, 1048576, 1310720, 1572864, 2097152 + }; + struct cfl_arena_external *allocation; + struct cfl_arena_external *previous; + size_t allocation_size; + size_t index; + uint8_t allocation_class; + + if (arena == NULL || size == 0 || + size > SIZE_MAX - sizeof(struct cfl_arena_external)) { + return NULL; + } + + allocation_class = 0; + allocation_size = size; + for (index = 0; index < CFL_ARENA_EXTERNAL_CLASS_COUNT; index++) { + if (size <= class_sizes[index]) { + allocation_class = (uint8_t) (index + 1); + allocation_size = class_sizes[index]; + break; + } + } + + if (allocation_class != 0 && + allocation_size - size > size / 8) { + allocation_class = CFL_ARENA_EXTERNAL_EXACT_CLASS; + allocation_size = size; + } + + allocation = NULL; + if (allocation_class == CFL_ARENA_EXTERNAL_EXACT_CLASS) { + previous = NULL; + allocation = arena->external_exact_cache; + while (allocation != NULL) { + if (allocation->size == allocation_size) { + if (previous == NULL) { + arena->external_exact_cache = allocation->next; + } + else { + previous->next = allocation->next; + } + arena->external_cache_bytes -= allocation->size; + break; + } + previous = allocation; + allocation = allocation->next; + } + } + else if (allocation_class != 0) { + index = allocation_class - 1; + allocation = arena->external_cache[index]; + if (allocation != NULL) { + arena->external_cache[index] = allocation->next; + arena->external_cache_count[index]--; + arena->external_cache_bytes -= allocation->size; + } + } + + if (allocation == NULL) { + allocation = arena->malloc_fn( + arena->allocator_context, + sizeof(struct cfl_arena_external) + allocation_size); + if (allocation == NULL) { + return NULL; + } + allocation->size = allocation_size; + arena->bytes_reserved += allocation_size + + sizeof(struct cfl_arena_external); + } + + allocation->next = arena->external; + allocation->previous = NULL; + if (arena->external != NULL) { + arena->external->previous = allocation; + } + allocation->allocation_class = allocation_class; + arena->external = allocation; + arena->bytes_used += allocation->size; + return allocation->data; +} + +void cfl_arena_free_external(struct cfl_arena *arena, + void *pointer) +{ + struct cfl_arena_external *allocation; + size_t index; + + if (arena == NULL || pointer == NULL) { + return; + } + + allocation = (struct cfl_arena_external *) + ((unsigned char *) pointer - + offsetof(struct cfl_arena_external, data)); + if (allocation->previous == NULL) { + arena->external = allocation->next; + } + else { + allocation->previous->next = allocation->next; + } + if (allocation->next != NULL) { + allocation->next->previous = allocation->previous; + } + arena->bytes_used -= allocation->size; + if (allocation->allocation_class != 0 && + allocation->size <= arena->external_cache_limit && + arena->external_cache_bytes <= arena->external_cache_limit - + allocation->size) { + if (allocation->allocation_class == + CFL_ARENA_EXTERNAL_EXACT_CLASS) { + allocation->next = arena->external_exact_cache; + arena->external_exact_cache = allocation; + } + else { + index = allocation->allocation_class - 1; + allocation->next = arena->external_cache[index]; + arena->external_cache[index] = allocation; + arena->external_cache_count[index]++; + } + allocation->previous = NULL; + arena->external_cache_bytes += allocation->size; + } + else { + arena->bytes_reserved -= allocation->size + + sizeof(struct cfl_arena_external); + arena->free_fn(arena->allocator_context, allocation); + } +} + +static void *arena_reusable_alloc(struct cfl_arena *arena, + void **free_list, size_t size) +{ + void *result; + + if (arena == NULL) { + return NULL; + } + + if (*free_list != NULL) { + result = *free_list; + *free_list = *((void **) result); + arena->bytes_used += size; + memset(result, 0, size); + return result; + } + + return cfl_arena_calloc(arena, 1, size); +} + +static void arena_reusable_free(struct cfl_arena *arena, + void **free_list, void *pointer, size_t size) +{ + if (arena == NULL || pointer == NULL) { + return; + } + + *((void **) pointer) = *free_list; + *free_list = pointer; + arena->bytes_used -= size; +} + +void *cfl_arena_alloc_variant(struct cfl_arena *arena, + size_t size) +{ + return arena_reusable_alloc(arena, &arena->free_variants, size); +} + +void cfl_arena_free_variant(struct cfl_arena *arena, + void *pointer, size_t size) +{ + arena_reusable_free(arena, &arena->free_variants, pointer, size); +} + +void *cfl_arena_alloc_kvpair(struct cfl_arena *arena, + size_t size) +{ + return arena_reusable_alloc(arena, &arena->free_kvpairs, size); +} + +void cfl_arena_free_kvpair(struct cfl_arena *arena, + void *pointer, size_t size) +{ + arena_reusable_free(arena, &arena->free_kvpairs, pointer, size); +} + +void *cfl_arena_alloc_sds(struct cfl_arena *arena, + size_t payload_size, size_t overhead_size, + uint8_t *allocation_class, + size_t *payload_capacity) +{ + static const size_t class_sizes[CFL_ARENA_SDS_CLASS_COUNT] = { + 32, 64, 128, 256, 512, 1024 + }; + size_t index; + + if (arena == NULL || allocation_class == NULL || payload_capacity == NULL) { + return NULL; + } + + for (index = 0; index < CFL_ARENA_SDS_CLASS_COUNT; index++) { + if (payload_size <= class_sizes[index]) { + *allocation_class = (uint8_t) (index + 1); + *payload_capacity = class_sizes[index]; + return arena_reusable_alloc(arena, &arena->free_sds[index], + overhead_size + class_sizes[index] + 1); + } + } + + *allocation_class = 0; + *payload_capacity = payload_size; + return NULL; +} + +void cfl_arena_free_sds(struct cfl_arena *arena, + void *pointer, uint8_t allocation_class, + size_t allocation_size) +{ + size_t index; + + if (allocation_class == 0 || + allocation_class > CFL_ARENA_SDS_CLASS_COUNT) { + return; + } + + index = allocation_class - 1; + arena_reusable_free(arena, &arena->free_sds[index], pointer, + allocation_size); +} diff --git a/lib/cfl/src/cfl_arena_internal.h b/lib/cfl/src/cfl_arena_internal.h new file mode 100644 index 00000000000..e41fe3a943b --- /dev/null +++ b/lib/cfl/src/cfl_arena_internal.h @@ -0,0 +1,29 @@ +#ifndef CFL_ARENA_INTERNAL_H +#define CFL_ARENA_INTERNAL_H + +#include + +#include + +void *cfl_arena_alloc(struct cfl_arena *arena, size_t size); +void *cfl_arena_alloc_external(struct cfl_arena *arena, + size_t size); +void cfl_arena_free_external(struct cfl_arena *arena, + void *pointer); +void *cfl_arena_alloc_variant(struct cfl_arena *arena, + size_t size); +void cfl_arena_free_variant(struct cfl_arena *arena, + void *pointer, size_t size); +void *cfl_arena_alloc_kvpair(struct cfl_arena *arena, + size_t size); +void cfl_arena_free_kvpair(struct cfl_arena *arena, + void *pointer, size_t size); +void *cfl_arena_alloc_sds(struct cfl_arena *arena, + size_t payload_size, size_t overhead_size, + uint8_t *allocation_class, + size_t *payload_capacity); +void cfl_arena_free_sds(struct cfl_arena *arena, + void *pointer, uint8_t allocation_class, + size_t allocation_size); + +#endif diff --git a/lib/cfl/src/cfl_array.c b/lib/cfl/src/cfl_array.c index af9d7cdc04a..bac55ab67a9 100644 --- a/lib/cfl/src/cfl_array.c +++ b/lib/cfl/src/cfl_array.c @@ -24,13 +24,33 @@ #include #include +#include "cfl_arena_internal.h" struct cfl_array *cfl_array_create(size_t slot_count) +{ + return cfl_array_create_in(NULL, slot_count); +} + +struct cfl_array *cfl_array_create_in(struct cfl_arena *arena, + size_t slot_count) { struct cfl_array *array; size_t alloc_count; - array = malloc(sizeof(struct cfl_array)); + alloc_count = slot_count; + if (alloc_count == 0) { + alloc_count = 1; + } + if (alloc_count > SIZE_MAX / sizeof(void *)) { + return NULL; + } + + if (arena == NULL) { + array = malloc(sizeof(struct cfl_array)); + } + else { + array = cfl_arena_malloc(arena, sizeof(struct cfl_array)); + } if (array == NULL) { cfl_errno(); return NULL; @@ -40,19 +60,18 @@ struct cfl_array *cfl_array_create(size_t slot_count) array->resizable = CFL_FALSE; /* allocate fixed number of entries */ - alloc_count = slot_count; - if (alloc_count == 0) { - alloc_count = 1; + if (arena == NULL) { + array->entries = calloc(alloc_count, sizeof(void *)); } - if (alloc_count > SIZE_MAX / sizeof(void *)) { - free(array); - return NULL; + else { + array->entries = cfl_arena_calloc(arena, alloc_count, + sizeof(void *)); } - - array->entries = calloc(alloc_count, sizeof(void *)); if (array->entries == NULL) { cfl_errno(); - free(array); + if (arena == NULL) { + free(array); + } return NULL; } @@ -61,10 +80,21 @@ struct cfl_array *cfl_array_create(size_t slot_count) array->owner = NULL; array->parent_array = NULL; array->parent_kvlist = NULL; + array->arena = arena; return array; } +struct cfl_array *cfl_array_create_like(struct cfl_array *parent, + size_t slot_count) +{ + if (parent == NULL) { + return NULL; + } + + return cfl_array_create_in(parent->arena, slot_count); +} + void cfl_array_destroy(struct cfl_array *array) { size_t index; @@ -80,9 +110,13 @@ void cfl_array_destroy(struct cfl_array *array) } } - free(array->entries); + if (array->arena == NULL) { + free(array->entries); + } + } + if (array->arena == NULL) { + free(array); } - free(array); } int cfl_array_resizable(struct cfl_array *array, int v) @@ -140,7 +174,7 @@ int cfl_array_remove_by_reference(struct cfl_array *array, } } - return 0; + return -1; } int cfl_array_append(struct cfl_array *array, @@ -155,6 +189,10 @@ int cfl_array_append(struct cfl_array *array, return -1; } + if (array->arena != value->arena) { + return -1; + } + if (array->entry_count >= array->slot_count) { /* * if there is no more space but the caller allowed to resize @@ -180,7 +218,16 @@ int cfl_array_append(struct cfl_array *array, new_size = (new_slot_count * sizeof(void *)); - tmp = realloc(array->entries, new_size); + if (array->arena == NULL) { + tmp = realloc(array->entries, new_size); + } + else { + tmp = cfl_arena_malloc(array->arena, new_size); + if (tmp != NULL) { + memcpy(tmp, array->entries, + array->entry_count * sizeof(void *)); + } + } if (!tmp) { cfl_report_runtime_error(); return -1; @@ -211,7 +258,8 @@ int cfl_array_append_string(struct cfl_array *array, char *value) struct cfl_variant *value_instance; int result; - value_instance = cfl_variant_create_from_string(value); + value_instance = cfl_variant_create_from_string_in( + array == NULL ? NULL : array->arena, value); if (value_instance == NULL) { return -1; @@ -231,7 +279,9 @@ int cfl_array_append_string_s(struct cfl_array *array, char *str, size_t str_len struct cfl_variant *value_instance; int result; - value_instance = cfl_variant_create_from_string_s(str, str_len, referenced); + value_instance = cfl_variant_create_from_string_s_in( + array == NULL ? NULL : array->arena, str, + str_len, referenced); if (value_instance == NULL) { return -1; } @@ -253,7 +303,9 @@ int cfl_array_append_bytes(struct cfl_array *array, struct cfl_variant *value_instance; int result; - value_instance = cfl_variant_create_from_bytes(value, length, referenced); + value_instance = cfl_variant_create_from_bytes_in( + array == NULL ? NULL : array->arena, value, + length, referenced); if (value_instance == NULL) { return -1; } @@ -274,7 +326,8 @@ int cfl_array_append_reference(struct cfl_array *array, void *value) struct cfl_variant *value_instance; int result; - value_instance = cfl_variant_create_from_reference(value); + value_instance = cfl_variant_create_from_reference_in( + array == NULL ? NULL : array->arena, value); if (value_instance == NULL) { return -1; @@ -296,7 +349,8 @@ int cfl_array_append_bool(struct cfl_array *array, int value) struct cfl_variant *value_instance; int result; - value_instance = cfl_variant_create_from_bool(value); + value_instance = cfl_variant_create_from_bool_in( + array == NULL ? NULL : array->arena, value); if (value_instance == NULL) { return -1; @@ -318,7 +372,8 @@ int cfl_array_append_int64(struct cfl_array *array, int64_t value) struct cfl_variant *value_instance; int result; - value_instance = cfl_variant_create_from_int64(value); + value_instance = cfl_variant_create_from_int64_in( + array == NULL ? NULL : array->arena, value); if (value_instance == NULL) { return -1; @@ -339,7 +394,8 @@ int cfl_array_append_uint64(struct cfl_array *array, uint64_t value) struct cfl_variant *value_instance; int result; - value_instance = cfl_variant_create_from_uint64(value); + value_instance = cfl_variant_create_from_uint64_in( + array == NULL ? NULL : array->arena, value); if (value_instance == NULL) { return -1; @@ -361,7 +417,8 @@ int cfl_array_append_double(struct cfl_array *array, double value) struct cfl_variant *value_instance; int result; - value_instance = cfl_variant_create_from_double(value); + value_instance = cfl_variant_create_from_double_in( + array == NULL ? NULL : array->arena, value); if (value_instance == NULL) { return -1; @@ -383,7 +440,8 @@ int cfl_array_append_null(struct cfl_array *array) struct cfl_variant *value_instance; int result; - value_instance = cfl_variant_create_from_null(); + value_instance = cfl_variant_create_from_null_in( + array == NULL ? NULL : array->arena); if (value_instance == NULL) { return -1; } @@ -410,7 +468,7 @@ int cfl_array_append_array(struct cfl_array *array, struct cfl_array *value) return -1; } - value_instance = cfl_variant_create_from_array(value); + value_instance = cfl_variant_create_from_array_in(array->arena, value); if (value_instance == NULL) { return -1; @@ -437,7 +495,7 @@ int cfl_array_append_new_array(struct cfl_array *array, size_t size) return -1; } - value = cfl_array_create(size); + value = cfl_array_create_in(array->arena, size); if (value == NULL) { return -1; @@ -460,7 +518,7 @@ int cfl_array_append_kvlist(struct cfl_array *array, struct cfl_kvlist *value) return -1; } - value_instance = cfl_variant_create_from_kvlist(value); + value_instance = cfl_variant_create_from_kvlist_in(array->arena, value); if (value_instance == NULL) { return -1; } diff --git a/lib/cfl/src/cfl_kvlist.c b/lib/cfl/src/cfl_kvlist.c index 9d1bb18a597..204ab7744ea 100644 --- a/lib/cfl/src/cfl_kvlist.c +++ b/lib/cfl/src/cfl_kvlist.c @@ -21,8 +21,10 @@ #include #include #include +#include "cfl_arena_internal.h" #include +#include #include #include @@ -85,10 +87,20 @@ static int print_json_string(FILE *fp, const char *str, size_t len) } struct cfl_kvlist *cfl_kvlist_create() +{ + return cfl_kvlist_create_in(NULL); +} + +struct cfl_kvlist *cfl_kvlist_create_in(struct cfl_arena *arena) { struct cfl_kvlist *list; - list = malloc(sizeof(struct cfl_kvlist)); + if (arena == NULL) { + list = malloc(sizeof(struct cfl_kvlist)); + } + else { + list = cfl_arena_malloc(arena, sizeof(struct cfl_kvlist)); + } if (list == NULL) { cfl_report_runtime_error(); return NULL; @@ -98,10 +110,20 @@ struct cfl_kvlist *cfl_kvlist_create() list->owner = NULL; list->parent_array = NULL; list->parent_kvlist = NULL; + list->arena = arena; return list; } +struct cfl_kvlist *cfl_kvlist_create_like(struct cfl_kvlist *parent) +{ + if (parent == NULL) { + return NULL; + } + + return cfl_kvlist_create_in(parent->arena); +} + void cfl_kvlist_destroy(struct cfl_kvlist *list) { struct cfl_list *tmp; @@ -123,10 +145,18 @@ void cfl_kvlist_destroy(struct cfl_kvlist *list) cfl_variant_destroy(pair->val); } cfl_list_del(&pair->_head); - free(pair); + if (pair->arena == NULL) { + free(pair); + } + else { + cfl_arena_free_kvpair(pair->arena, pair, + sizeof(struct cfl_kvpair)); + } } - free(list); + if (list->arena == NULL) { + free(list); + } } int cfl_kvlist_insert_string_s(struct cfl_kvlist *list, @@ -141,7 +171,8 @@ int cfl_kvlist_insert_string_s(struct cfl_kvlist *list, return -1; } - value_instance = cfl_variant_create_from_string_s(value, value_size, referenced); + value_instance = cfl_variant_create_from_string_s_in(list->arena, value, + value_size, referenced); if (value_instance == NULL) { return -1; } @@ -168,7 +199,8 @@ int cfl_kvlist_insert_bytes_s(struct cfl_kvlist *list, return -1; } - value_instance = cfl_variant_create_from_bytes(value, length, referenced); + value_instance = cfl_variant_create_from_bytes_in(list->arena, value, + length, referenced); if (value_instance == NULL) { return -1; } @@ -193,7 +225,7 @@ int cfl_kvlist_insert_reference_s(struct cfl_kvlist *list, return -1; } - value_instance = cfl_variant_create_from_reference(value); + value_instance = cfl_variant_create_from_reference_in(list->arena, value); if (value_instance == NULL) { return -1; @@ -220,7 +252,7 @@ int cfl_kvlist_insert_bool_s(struct cfl_kvlist *list, return -1; } - value_instance = cfl_variant_create_from_bool(value); + value_instance = cfl_variant_create_from_bool_in(list->arena, value); if (value_instance == NULL) { return -1; @@ -247,7 +279,7 @@ int cfl_kvlist_insert_int64_s(struct cfl_kvlist *list, return -1; } - value_instance = cfl_variant_create_from_int64(value); + value_instance = cfl_variant_create_from_int64_in(list->arena, value); if (value_instance == NULL) { return -1; @@ -274,7 +306,7 @@ int cfl_kvlist_insert_uint64_s(struct cfl_kvlist *list, return -1; } - value_instance = cfl_variant_create_from_uint64(value); + value_instance = cfl_variant_create_from_uint64_in(list->arena, value); if (value_instance == NULL) { return -1; @@ -301,7 +333,7 @@ int cfl_kvlist_insert_double_s(struct cfl_kvlist *list, return -1; } - value_instance = cfl_variant_create_from_double(value); + value_instance = cfl_variant_create_from_double_in(list->arena, value); if (value_instance == NULL) { return -1; @@ -328,7 +360,7 @@ int cfl_kvlist_insert_array_s(struct cfl_kvlist *list, return -1; } - value_instance = cfl_variant_create_from_array(value); + value_instance = cfl_variant_create_from_array_in(list->arena, value); if (value_instance == NULL) { return -1; @@ -358,7 +390,7 @@ int cfl_kvlist_insert_new_array_s(struct cfl_kvlist *list, return -1; } - value = cfl_array_create(size); + value = cfl_array_create_in(list->arena, size); if (value == NULL) { return -1; @@ -386,7 +418,7 @@ int cfl_kvlist_insert_kvlist_s(struct cfl_kvlist *list, return -1; } - value_instance = cfl_variant_create_from_kvlist(value); + value_instance = cfl_variant_create_from_kvlist_in(list->arena, value); if (value_instance == NULL) { return -1; } @@ -414,33 +446,85 @@ int cfl_kvlist_insert_s(struct cfl_kvlist *list, return -1; } - pair = malloc(sizeof(struct cfl_kvpair)); + if (list->arena != value->arena) { + return -1; + } + + if (list->arena == NULL) { + pair = malloc(sizeof(struct cfl_kvpair)); + } + else { + pair = cfl_arena_alloc_kvpair(list->arena, + sizeof(struct cfl_kvpair)); + } if (pair == NULL) { cfl_report_runtime_error(); return -1; } - pair->key = cfl_sds_create_len(key, (int) key_size); + pair->key = cfl_sds_create_len_in(list->arena, key, (int) key_size); if (pair->key == NULL) { - free(pair); + if (list->arena == NULL) { + free(pair); + } + else { + cfl_arena_free_kvpair(list->arena, pair, + sizeof(struct cfl_kvpair)); + } return -2; } if (cfl_container_move_variant_to_kvlist(list, value) != 0) { cfl_sds_destroy(pair->key); - free(pair); + if (list->arena == NULL) { + free(pair); + } + else { + cfl_arena_free_kvpair(list->arena, pair, + sizeof(struct cfl_kvpair)); + } return -1; } pair->val = value; + pair->arena = list->arena; cfl_list_add(&pair->_head, &list->list); return 0; } -struct cfl_variant *cfl_kvlist_fetch_s(struct cfl_kvlist *list, char *key, size_t key_size) +static int key_matches(cfl_sds_t candidate, char *key, size_t key_size, + enum cfl_kvlist_match_mode mode) +{ + size_t index; + + if (cfl_sds_len(candidate) != key_size) { + return CFL_FALSE; + } + + if (mode == CFL_KVLIST_MATCH_CASE_SENSITIVE) { + return memcmp(candidate, key, key_size) == 0; + } + + if (mode == CFL_KVLIST_MATCH_CASE_INSENSITIVE) { + for (index = 0; index < key_size; index++) { + if (tolower((unsigned char) candidate[index]) != + tolower((unsigned char) key[index])) { + return CFL_FALSE; + } + } + + return CFL_TRUE; + } + + return CFL_FALSE; +} + +struct cfl_variant *cfl_kvlist_fetch_s_ex( + struct cfl_kvlist *list, char *key, size_t key_size, + enum cfl_kvlist_match_mode mode) { struct cfl_list *head; struct cfl_kvpair *pair; @@ -452,11 +536,7 @@ struct cfl_variant *cfl_kvlist_fetch_s(struct cfl_kvlist *list, char *key, size_ cfl_list_foreach(head, &list->list) { pair = cfl_list_entry(head, struct cfl_kvpair, _head); - if (cfl_sds_len(pair->key) != key_size) { - continue; - } - - if (strncasecmp(pair->key, key, key_size) == 0) { + if (key_matches(pair->key, key, key_size, mode) == CFL_TRUE) { return pair->val; } } @@ -464,6 +544,19 @@ struct cfl_variant *cfl_kvlist_fetch_s(struct cfl_kvlist *list, char *key, size_ return NULL; } +struct cfl_variant *cfl_kvlist_fetch_s(struct cfl_kvlist *list, + char *key, size_t key_size) +{ + return cfl_kvlist_fetch_s_ex(list, key, key_size, + CFL_KVLIST_MATCH_CASE_INSENSITIVE); +} + +struct cfl_variant *cfl_kvlist_fetch_case_s(struct cfl_kvlist *list, + char *key, size_t key_size) +{ + return cfl_kvlist_fetch_s_ex(list, key, key_size, + CFL_KVLIST_MATCH_CASE_SENSITIVE); +} int cfl_kvlist_insert_string(struct cfl_kvlist *list, char *key, char *value) @@ -587,11 +680,19 @@ int cfl_kvlist_insert(struct cfl_kvlist *list, struct cfl_variant *cfl_kvlist_fetch(struct cfl_kvlist *list, char *key) { - if (!list || !key) { + return cfl_kvlist_fetch_ex(list, key, + CFL_KVLIST_MATCH_CASE_INSENSITIVE); +} + +struct cfl_variant *cfl_kvlist_fetch_ex( + struct cfl_kvlist *list, char *key, + enum cfl_kvlist_match_mode mode) +{ + if (list == NULL || key == NULL) { return NULL; } - return cfl_kvlist_fetch_s(list, key, strlen(key)); + return cfl_kvlist_fetch_s_ex(list, key, strlen(key), mode); } int cfl_kvlist_count(struct cfl_kvlist *list) @@ -659,11 +760,17 @@ int cfl_kvlist_print(FILE *fp, struct cfl_kvlist *list) } int cfl_kvlist_contains(struct cfl_kvlist *kvlist, char *name) +{ + return cfl_kvlist_contains_ex(kvlist, name, + CFL_KVLIST_MATCH_CASE_INSENSITIVE); +} + +int cfl_kvlist_contains_ex(struct cfl_kvlist *kvlist, char *name, + enum cfl_kvlist_match_mode mode) { struct cfl_list *iterator; struct cfl_kvpair *pair; size_t name_len; - size_t key_len; if (kvlist == NULL || name == NULL) { return CFL_FALSE; @@ -675,12 +782,7 @@ int cfl_kvlist_contains(struct cfl_kvlist *kvlist, char *name) pair = cfl_list_entry(iterator, struct cfl_kvpair, _head); - key_len = cfl_sds_len(pair->key); - if (key_len != name_len) { - continue; - } - - if (strncasecmp(pair->key, name, name_len) == 0) { + if (key_matches(pair->key, name, name_len, mode) == CFL_TRUE) { return CFL_TRUE; } } @@ -690,12 +792,18 @@ int cfl_kvlist_contains(struct cfl_kvlist *kvlist, char *name) int cfl_kvlist_remove(struct cfl_kvlist *kvlist, char *name) +{ + return cfl_kvlist_remove_ex(kvlist, name, + CFL_KVLIST_MATCH_CASE_INSENSITIVE); +} + +int cfl_kvlist_remove_ex(struct cfl_kvlist *kvlist, char *name, + enum cfl_kvlist_match_mode mode) { struct cfl_list *iterator_backup; struct cfl_list *iterator; struct cfl_kvpair *pair; size_t name_len; - size_t key_len; int removed; if (kvlist == NULL || name == NULL) { @@ -709,12 +817,7 @@ int cfl_kvlist_remove(struct cfl_kvlist *kvlist, char *name) pair = cfl_list_entry(iterator, struct cfl_kvpair, _head); - key_len = cfl_sds_len(pair->key); - if (key_len != name_len) { - continue; - } - - if (strncasecmp(pair->key, name, name_len) == 0) { + if (key_matches(pair->key, name, name_len, mode) == CFL_TRUE) { cfl_kvpair_destroy(pair); removed = CFL_TRUE; } @@ -739,7 +842,13 @@ void cfl_kvpair_destroy(struct cfl_kvpair *pair) cfl_variant_destroy(pair->val); } - free(pair); + if (pair->arena == NULL) { + free(pair); + } + else { + cfl_arena_free_kvpair(pair->arena, pair, + sizeof(struct cfl_kvpair)); + } } } @@ -758,3 +867,57 @@ struct cfl_variant *cfl_kvpair_take_value(struct cfl_kvpair *pair) return value; } + +int cfl_kvpair_key_set_s(struct cfl_kvpair *pair, + char *key, size_t key_size) +{ + cfl_sds_t replacement; + + if (pair == NULL || key == NULL || key_size > INT_MAX) { + return -1; + } + + replacement = cfl_sds_create_len_in(pair->arena, key, (int) key_size); + if (replacement == NULL) { + return -1; + } + + cfl_sds_destroy(pair->key); + pair->key = replacement; + return 0; +} + +int cfl_kvlist_rename_s(struct cfl_kvlist *list, + char *old_key, size_t old_key_size, + char *new_key, size_t new_key_size) +{ + struct cfl_list *head; + struct cfl_kvpair *pair; + struct cfl_kvpair *source; + + if (list == NULL || old_key == NULL || new_key == NULL || + old_key_size > INT_MAX || new_key_size > INT_MAX) { + return -1; + } + + source = NULL; + cfl_list_foreach(head, &list->list) { + pair = cfl_list_entry(head, struct cfl_kvpair, _head); + if (cfl_sds_len(pair->key) == old_key_size && + memcmp(pair->key, old_key, old_key_size) == 0) { + source = pair; + } + if (cfl_sds_len(pair->key) == new_key_size && + memcmp(pair->key, new_key, new_key_size) == 0 && + !(old_key_size == new_key_size && + memcmp(old_key, new_key, old_key_size) == 0)) { + return -1; + } + } + + if (source == NULL) { + return -1; + } + + return cfl_kvpair_key_set_s(source, new_key, new_key_size); +} diff --git a/lib/cfl/src/cfl_sds.c b/lib/cfl/src/cfl_sds.c index dea6d61d712..afe89c2a937 100644 --- a/lib/cfl/src/cfl_sds.c +++ b/lib/cfl/src/cfl_sds.c @@ -31,6 +31,22 @@ #include #include +#include "cfl_arena_internal.h" + +#define CFL_SDS_ARENA_FLAG (UINT64_C(1) << 63) +#define CFL_SDS_ALLOC_MASK (~CFL_SDS_ARENA_FLAG) + +struct cfl_sds_arena_header { + struct cfl_arena *arena; + uint8_t external; + uint8_t allocation_class; +}; + +static struct cfl_sds_arena_header *sds_arena_header(struct cfl_sds *head) +{ + return (struct cfl_sds_arena_header *) + ((char *) head - sizeof(struct cfl_sds_arena_header)); +} size_t cfl_sds_avail(cfl_sds_t s) { @@ -41,27 +57,71 @@ size_t cfl_sds_avail(cfl_sds_t s) } h = CFL_SDS_HEADER(s); - return (size_t) (h->alloc - h->len); + return (size_t) ((h->alloc & CFL_SDS_ALLOC_MASK) - h->len); } -static cfl_sds_t sds_alloc(size_t size) +static cfl_sds_t sds_alloc(struct cfl_arena *arena, size_t size) { void *buf; cfl_sds_t s; struct cfl_sds *head; - - if (size > SIZE_MAX - CFL_SDS_HEADER_SIZE - 1) { + struct cfl_sds_arena_header *arena_head; + size_t allocation_size; + size_t external_threshold; + size_t payload_capacity; + uint8_t allocation_class; + + if (size > SIZE_MAX - CFL_SDS_HEADER_SIZE - + sizeof(struct cfl_sds_arena_header) - 1) { return NULL; } - buf = malloc(CFL_SDS_HEADER_SIZE + size + 1); + payload_capacity = size; + allocation_class = 0; + allocation_size = CFL_SDS_HEADER_SIZE + size + 1; + if (arena == NULL) { + buf = malloc(allocation_size); + } + else if (size <= 1024) { + external_threshold = cfl_arena_large_object_threshold(arena); + buf = cfl_arena_alloc_sds(arena, size, + CFL_SDS_HEADER_SIZE + + sizeof(struct cfl_sds_arena_header), + &allocation_class, + &payload_capacity); + allocation_size = sizeof(struct cfl_sds_arena_header) + + CFL_SDS_HEADER_SIZE + payload_capacity + 1; + } + else { + external_threshold = cfl_arena_large_object_threshold(arena); + allocation_size += sizeof(struct cfl_sds_arena_header); + if (allocation_size >= external_threshold) { + buf = cfl_arena_alloc_external(arena, allocation_size); + } + else { + buf = cfl_arena_malloc(arena, allocation_size); + } + } if (!buf) { return NULL; } - head = buf; + if (arena == NULL) { + head = buf; + } + else { + arena_head = buf; + arena_head->arena = arena; + arena_head->external = allocation_class == 0 && + allocation_size >= external_threshold; + arena_head->allocation_class = allocation_class; + head = (struct cfl_sds *) (arena_head + 1); + } head->len = 0; - head->alloc = size; + head->alloc = payload_capacity; + if (arena != NULL) { + head->alloc |= CFL_SDS_ARENA_FLAG; + } s = head->buf; *s = '\0'; @@ -75,14 +135,16 @@ size_t cfl_sds_alloc(cfl_sds_t s) return 0; } - return (size_t) CFL_SDS_HEADER(s)->alloc; + return (size_t) (CFL_SDS_HEADER(s)->alloc & CFL_SDS_ALLOC_MASK); } cfl_sds_t cfl_sds_increase(cfl_sds_t s, size_t len) { size_t new_size; struct cfl_sds *head; + struct cfl_sds *new_head; cfl_sds_t out; + cfl_sds_t arena_out; void *tmp; if (s == NULL) { @@ -96,10 +158,6 @@ cfl_sds_t cfl_sds_increase(cfl_sds_t s, size_t len) return s; } - if (head->alloc > UINT64_MAX - len) { - return NULL; - } - if (cfl_sds_alloc(s) > SIZE_MAX - len) { return NULL; } @@ -108,8 +166,19 @@ cfl_sds_t cfl_sds_increase(cfl_sds_t s, size_t len) if (new_size > SIZE_MAX - CFL_SDS_HEADER_SIZE - 1) { return NULL; } - new_size += CFL_SDS_HEADER_SIZE + 1; + if ((head->alloc & CFL_SDS_ARENA_FLAG) != 0) { + arena_out = sds_alloc(sds_arena_header(head)->arena, new_size); + if (arena_out == NULL) { + return NULL; + } + memcpy(arena_out, s, head->len + 1); + new_head = CFL_SDS_HEADER(arena_out); + new_head->len = head->len; + cfl_sds_destroy(s); + return arena_out; + } + new_size += CFL_SDS_HEADER_SIZE + 1; tmp = realloc(head, new_size); if (!tmp) { return NULL; @@ -131,6 +200,12 @@ size_t cfl_sds_len(cfl_sds_t s) } cfl_sds_t cfl_sds_create_len(const char *str, int len) +{ + return cfl_sds_create_len_in(NULL, str, len); +} + +cfl_sds_t cfl_sds_create_len_in(struct cfl_arena *arena, + const char *str, int len) { cfl_sds_t s; struct cfl_sds *head; @@ -139,7 +214,7 @@ cfl_sds_t cfl_sds_create_len(const char *str, int len) return NULL; } - s = sds_alloc(len); + s = sds_alloc(arena, len); if (!s) { return NULL; } @@ -174,13 +249,29 @@ cfl_sds_t cfl_sds_create(const char *str) void cfl_sds_destroy(cfl_sds_t s) { struct cfl_sds *head; + struct cfl_sds_arena_header *arena_head; if (!s) { return; } head = CFL_SDS_HEADER(s); - free(head); + if ((head->alloc & CFL_SDS_ARENA_FLAG) == 0) { + free(head); + } + else { + arena_head = sds_arena_header(head); + if (arena_head->external) { + cfl_arena_free_external(arena_head->arena, arena_head); + } + else if (arena_head->allocation_class != 0) { + cfl_arena_free_sds( + arena_head->arena, arena_head, + arena_head->allocation_class, + sizeof(struct cfl_sds_arena_header) + CFL_SDS_HEADER_SIZE + + cfl_sds_alloc(s) + 1); + } + } } cfl_sds_t cfl_sds_cat(cfl_sds_t s, const char *str, int len) @@ -194,6 +285,7 @@ cfl_sds_t cfl_sds_cat(cfl_sds_t s, const char *str, int len) cfl_sds_t tmp = NULL; const char *source; int source_in_buffer; + size_t allocation_size; if (s == NULL || str == NULL || len < 0) { return NULL; @@ -205,7 +297,9 @@ cfl_sds_t cfl_sds_cat(cfl_sds_t s, const char *str, int len) append_len = (size_t) len; head = CFL_SDS_HEADER(s); - if (head->len > head->alloc || head->len > SIZE_MAX - append_len - 1) { + allocation_size = cfl_sds_alloc(s); + if (head->len > allocation_size || + head->len > SIZE_MAX - append_len - 1) { return NULL; } @@ -221,10 +315,10 @@ cfl_sds_t cfl_sds_cat(cfl_sds_t s, const char *str, int len) buffer_addr = (uintptr_t) s; source_addr = (uintptr_t) str; if (source_addr >= buffer_addr && - (source_addr - buffer_addr) <= head->alloc) { + (source_addr - buffer_addr) <= allocation_size) { source_offset = (size_t) (source_addr - buffer_addr); - if (append_len - 1 >= head->alloc - source_offset) { + if (append_len - 1 >= allocation_size - source_offset) { return NULL; } @@ -259,7 +353,7 @@ cfl_sds_t cfl_sds_cat(cfl_sds_t s, const char *str, int len) cfl_sds_t cfl_sds_create_size(size_t size) { - return sds_alloc(size); + return sds_alloc(NULL, size); } void cfl_sds_set_len(cfl_sds_t s, size_t len) @@ -271,7 +365,7 @@ void cfl_sds_set_len(cfl_sds_t s, size_t len) } head = CFL_SDS_HEADER(s); - if (len > head->alloc) { + if (len > cfl_sds_alloc(s)) { return; } @@ -322,6 +416,7 @@ cfl_sds_t cfl_sds_printf(cfl_sds_t *sds, const char *fmt, ...) va_end(ap); if (size < 0) { + s[base_len] = '\0'; return NULL; } @@ -329,6 +424,13 @@ cfl_sds_t cfl_sds_printf(cfl_sds_t *sds, const char *fmt, ...) break; } + /* + * vsnprintf() writes a truncated result when the available space is + * insufficient. Restore the original SDS terminator before growing + * so an allocation failure leaves the input unchanged. + */ + s[base_len] = '\0'; + growth = (size_t) size - avail; tmp = cfl_sds_increase(s, growth); if (!tmp) { diff --git a/lib/cfl/src/cfl_variant.c b/lib/cfl/src/cfl_variant.c index 4143aa63634..8f9f9f0ce63 100644 --- a/lib/cfl/src/cfl_variant.c +++ b/lib/cfl/src/cfl_variant.c @@ -19,6 +19,24 @@ #include #include +#include + +#include "cfl_arena_internal.h" + +static void variant_instance_release(struct cfl_variant *instance) +{ + if (instance == NULL) { + return; + } + + if (instance->arena == NULL) { + free(instance); + } + else { + cfl_arena_free_variant(instance->arena, instance, + sizeof(struct cfl_variant)); + } +} #include #include #include @@ -211,7 +229,8 @@ struct cfl_variant *cfl_variant_create_from_string_s(char *value, size_t value_s return NULL; } - instance->data.as_string = cfl_sds_create_len(value, (int) value_size); + instance->data.as_string = cfl_sds_create_len_in(NULL, value, + (int) value_size); if (instance->data.as_string == NULL) { free(instance); return NULL; @@ -393,7 +412,203 @@ struct cfl_variant *cfl_variant_create() return NULL; } instance->size = 0; + instance->arena = NULL; + + return instance; +} + +struct cfl_variant *cfl_variant_create_in(struct cfl_arena *arena) +{ + struct cfl_variant *instance; + + if (arena == NULL) { + return cfl_variant_create(); + } + + instance = cfl_arena_alloc_variant(arena, + sizeof(struct cfl_variant)); + if (instance != NULL) { + instance->arena = arena; + } + + return instance; +} + +struct cfl_variant *cfl_variant_create_from_string_s_in(struct cfl_arena *arena, + char *value, + size_t value_size, + int referenced) +{ + struct cfl_variant *instance; + + if (value == NULL && value_size > 0) { + return NULL; + } + + instance = cfl_variant_create_in(arena); + if (instance == NULL) { + return NULL; + } + + instance->referenced = referenced ? CFL_TRUE : CFL_FALSE; + if (referenced) { + instance->data.as_string = value; + } + else { + if (value_size > INT_MAX) { + variant_instance_release(instance); + return NULL; + } + instance->data.as_string = cfl_sds_create_len_in(arena, value, + (int) value_size); + if (instance->data.as_string == NULL) { + variant_instance_release(instance); + return NULL; + } + } + instance->size = value_size; + instance->type = CFL_VARIANT_STRING; + + return instance; +} + +struct cfl_variant *cfl_variant_create_from_string_in(struct cfl_arena *arena, + char *value) +{ + if (value == NULL) { + return NULL; + } + return cfl_variant_create_from_string_s_in(arena, value, strlen(value), CFL_FALSE); +} + +struct cfl_variant *cfl_variant_create_from_bytes_in(struct cfl_arena *arena, + char *value, size_t length, + int referenced) +{ + struct cfl_variant *instance; + + instance = cfl_variant_create_from_string_s_in(arena, value, length, referenced); + if (instance != NULL) { + instance->type = CFL_VARIANT_BYTES; + } + return instance; +} + +struct cfl_variant *cfl_variant_create_from_bool_in(struct cfl_arena *arena, + int value) +{ + struct cfl_variant *instance; + + instance = cfl_variant_create_in(arena); + if (instance != NULL) { + instance->data.as_bool = value; + instance->type = CFL_VARIANT_BOOL; + } + return instance; +} +struct cfl_variant *cfl_variant_create_from_int64_in(struct cfl_arena *arena, + int64_t value) +{ + struct cfl_variant *instance; + + instance = cfl_variant_create_in(arena); + if (instance != NULL) { + instance->data.as_int64 = value; + instance->type = CFL_VARIANT_INT; + } + return instance; +} + +struct cfl_variant *cfl_variant_create_from_uint64_in(struct cfl_arena *arena, + uint64_t value) +{ + struct cfl_variant *instance; + + instance = cfl_variant_create_in(arena); + if (instance != NULL) { + instance->data.as_uint64 = value; + instance->type = CFL_VARIANT_UINT; + } + return instance; +} + +struct cfl_variant *cfl_variant_create_from_double_in(struct cfl_arena *arena, + double value) +{ + struct cfl_variant *instance; + + instance = cfl_variant_create_in(arena); + if (instance != NULL) { + instance->data.as_double = value; + instance->type = CFL_VARIANT_DOUBLE; + } + return instance; +} + +struct cfl_variant *cfl_variant_create_from_null_in(struct cfl_arena *arena) +{ + struct cfl_variant *instance; + + instance = cfl_variant_create_in(arena); + if (instance != NULL) { + instance->type = CFL_VARIANT_NULL; + } + return instance; +} + +struct cfl_variant *cfl_variant_create_from_reference_in(struct cfl_arena *arena, + void *value) +{ + struct cfl_variant *instance; + + instance = cfl_variant_create_in(arena); + if (instance != NULL) { + instance->data.as_reference = value; + instance->type = CFL_VARIANT_REFERENCE; + } + return instance; +} + +struct cfl_variant *cfl_variant_create_from_array_in(struct cfl_arena *arena, + struct cfl_array *value) +{ + struct cfl_variant *instance; + + if (value != NULL && value->arena != arena) { + return NULL; + } + + instance = cfl_variant_create_in(arena); + if (instance != NULL) { + if (value != NULL && cfl_container_claim_array(value, instance) != 0) { + variant_instance_release(instance); + return NULL; + } + instance->data.as_array = value; + instance->type = CFL_VARIANT_ARRAY; + } + return instance; +} + +struct cfl_variant *cfl_variant_create_from_kvlist_in(struct cfl_arena *arena, + struct cfl_kvlist *value) +{ + struct cfl_variant *instance; + + if (value != NULL && value->arena != arena) { + return NULL; + } + + instance = cfl_variant_create_in(arena); + if (instance != NULL) { + if (value != NULL && cfl_container_claim_kvlist(value, instance) != 0) { + variant_instance_release(instance); + return NULL; + } + instance->data.as_kvlist = value; + instance->type = CFL_VARIANT_KVLIST; + } return instance; } @@ -418,7 +633,7 @@ void cfl_variant_destroy(struct cfl_variant *instance) cfl_kvlist_destroy(instance->data.as_kvlist); } - free(instance); + variant_instance_release(instance); } void cfl_variant_size_set(struct cfl_variant *var, size_t size) diff --git a/lib/cfl/tests/CMakeLists.txt b/lib/cfl/tests/CMakeLists.txt index 6bd3cc6f7bb..dd18ca07820 100644 --- a/lib/cfl/tests/CMakeLists.txt +++ b/lib/cfl/tests/CMakeLists.txt @@ -11,6 +11,7 @@ set(UNIT_TESTS_FILES hash.c list.c variant.c + arena.c object.c version.c utils.c @@ -39,6 +40,7 @@ set(PUBLIC_HEADERS cfl_time.h cfl_utils.h cfl_variant.h + cfl_arena.h cfl_version.h ) diff --git a/lib/cfl/tests/arena.c b/lib/cfl/tests/arena.c new file mode 100644 index 00000000000..494fa635088 --- /dev/null +++ b/lib/cfl/tests/arena.c @@ -0,0 +1,548 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +#include +#include +#include + +#include "cfl_tests_internal.h" + +union test_max_align { + long double long_double_value; + void *pointer_value; + uint64_t uint64_value; +}; + +struct test_alignment_probe { + char byte; + union test_max_align value; +}; + +struct test_allocator_context { + size_t allocation_count; + size_t free_count; + size_t fail_after; + size_t allocation_sizes[8]; +}; + +static void *test_allocator_malloc(void *data, size_t size) +{ + void *result; + struct test_allocator_context *context; + + context = data; + if (context->allocation_count >= context->fail_after) { + return NULL; + } + + result = malloc(size); + if (result != NULL) { + if (context->allocation_count < 8) { + context->allocation_sizes[context->allocation_count] = size; + } + context->allocation_count++; + } + + return result; +} + +static void test_allocator_free(void *data, void *pointer) +{ + struct test_allocator_context *context; + + context = data; + context->free_count++; + free(pointer); +} + +static void public_raw_allocations(void) +{ + unsigned char *zeroed; + unsigned char source[] = {0x01, 0x02, 0x03, 0x04}; + unsigned char *copy; + char *string; + void *pointer; + size_t alignment; + size_t index; + struct cfl_arena *arena; + + arena = cfl_arena_create(128); + TEST_CHECK(arena != NULL); + alignment = offsetof(struct test_alignment_probe, value); + + for (index = 1; index <= 64; index++) { + pointer = cfl_arena_malloc(arena, index); + TEST_CHECK(pointer != NULL); + TEST_CHECK((uintptr_t) pointer % alignment == 0); + } + + zeroed = cfl_arena_calloc(arena, 8, sizeof(unsigned char)); + TEST_CHECK(zeroed != NULL); + for (index = 0; index < 8; index++) { + TEST_CHECK(zeroed[index] == 0); + } + + copy = cfl_arena_memdup(arena, source, sizeof(source)); + TEST_CHECK(copy != NULL); + TEST_CHECK(memcmp(copy, source, sizeof(source)) == 0); + + string = cfl_arena_strndup(arena, "arena-data", 5); + TEST_CHECK(string != NULL); + TEST_CHECK(strcmp(string, "arena") == 0); + string = cfl_arena_strndup(arena, "", 0); + TEST_CHECK(string != NULL); + TEST_CHECK(string[0] == '\0'); + + cfl_arena_destroy(arena); +} + +static void public_raw_allocation_failures(void) +{ + size_t used; + struct cfl_arena *arena; + + arena = cfl_arena_create(128); + TEST_CHECK(arena != NULL); + used = cfl_arena_bytes_used(arena); + + TEST_CHECK(cfl_arena_malloc(NULL, 1) == NULL); + TEST_CHECK(cfl_arena_malloc(arena, 0) == NULL); + TEST_CHECK(cfl_arena_malloc(arena, SIZE_MAX) == NULL); + TEST_CHECK(cfl_arena_calloc(arena, SIZE_MAX, 2) == NULL); + TEST_CHECK(cfl_arena_calloc(arena, 0, 1) == NULL); + TEST_CHECK(cfl_arena_memdup(arena, NULL, 1) == NULL); + TEST_CHECK(cfl_arena_memdup(arena, "x", 0) == NULL); + TEST_CHECK(cfl_arena_strndup(arena, NULL, 0) == NULL); + TEST_CHECK(cfl_arena_strndup(arena, "x", SIZE_MAX) == NULL); + TEST_CHECK(cfl_arena_bytes_used(arena) == used); + + cfl_arena_destroy(arena); +} + +static void options_growth_and_callbacks(void) +{ + char payload[2048]; + cfl_sds_t value; + void *pointer; + struct cfl_arena *arena; + struct cfl_arena_options options; + struct test_allocator_context context; + + memset(&context, 0, sizeof(context)); + context.fail_after = SIZE_MAX; + cfl_arena_options_init(&options); + options.chunk_size = 64; + options.maximum_chunk_size = 256; + options.malloc_fn = test_allocator_malloc; + options.free_fn = test_allocator_free; + options.allocator_context = &context; + + arena = cfl_arena_create_with_options(&options); + TEST_CHECK(arena != NULL); + TEST_CHECK(context.allocation_count == 1); + + pointer = cfl_arena_malloc(arena, 48); + TEST_CHECK(pointer != NULL); + pointer = cfl_arena_malloc(arena, 48); + TEST_CHECK(pointer != NULL); + pointer = cfl_arena_malloc(arena, 96); + TEST_CHECK(pointer != NULL); + TEST_CHECK(context.allocation_count == 4); + TEST_CHECK(context.allocation_sizes[1] < context.allocation_sizes[2]); + TEST_CHECK(context.allocation_sizes[2] < context.allocation_sizes[3]); + + cfl_arena_reset(arena); + TEST_CHECK(cfl_arena_bytes_used(arena) == 0); + pointer = cfl_arena_malloc(arena, 48); + TEST_CHECK(pointer != NULL); + + memset(payload, 'x', sizeof(payload)); + value = cfl_sds_create_len_in(arena, payload, (int) sizeof(payload)); + TEST_CHECK(value != NULL); + cfl_sds_destroy(value); + + cfl_arena_destroy(arena); + TEST_CHECK(context.free_count == context.allocation_count); + + options.maximum_chunk_size = 32; + TEST_CHECK(cfl_arena_create_with_options(&options) == NULL); + options.maximum_chunk_size = 64; + options.free_fn = NULL; + TEST_CHECK(cfl_arena_create_with_options(&options) == NULL); + TEST_CHECK(cfl_arena_create_with_options(NULL) == NULL); + options.free_fn = test_allocator_free; + options.struct_size = offsetof(struct cfl_arena_options, + allocator_context); + TEST_CHECK(cfl_arena_create_with_options(&options) == NULL); +} + +static void callback_allocation_failure(void) +{ + void *pointer; + struct cfl_arena *arena; + struct cfl_arena_options options; + struct test_allocator_context context; + + memset(&context, 0, sizeof(context)); + context.fail_after = 0; + cfl_arena_options_init(&options); + options.chunk_size = 64; + options.malloc_fn = test_allocator_malloc; + options.free_fn = test_allocator_free; + options.allocator_context = &context; + TEST_CHECK(cfl_arena_create_with_options(&options) == NULL); + + context.fail_after = 1; + arena = cfl_arena_create_with_options(&options); + TEST_CHECK(arena != NULL); + TEST_CHECK(cfl_arena_malloc(arena, 1) == NULL); + TEST_CHECK(cfl_arena_bytes_used(arena) == 0); + + context.fail_after = SIZE_MAX; + pointer = cfl_arena_malloc(arena, 1); + TEST_CHECK(pointer != NULL); + cfl_arena_destroy(arena); + TEST_CHECK(context.free_count == context.allocation_count); +} + +static void arena_build_and_destroy(void) +{ + struct cfl_arena *arena; + struct cfl_array *array; + struct cfl_kvlist *list; + struct cfl_variant *root; + int ret; + + arena = cfl_arena_create(256); + TEST_CHECK(arena != NULL); + + list = cfl_kvlist_create_in(arena); + TEST_CHECK(list != NULL); + ret = cfl_kvlist_insert_string(list, "name", "cfl"); + TEST_CHECK(ret == 0); + + array = cfl_array_create_in(arena, 2); + TEST_CHECK(array != NULL); + ret = cfl_array_append_int64(array, 42); + TEST_CHECK(ret == 0); + ret = cfl_kvlist_insert_array(list, "values", array); + TEST_CHECK(ret == 0); + + root = cfl_variant_create_from_kvlist_in(arena, list); + TEST_CHECK(root != NULL); + TEST_CHECK(cfl_arena_bytes_used(arena) > 0); + TEST_CHECK(cfl_arena_bytes_reserved(arena) >= + cfl_arena_bytes_used(arena)); + + cfl_variant_destroy(root); + cfl_arena_destroy(arena); +} + +static void arena_reset(void) +{ + struct cfl_arena *arena; + struct cfl_array *array; + + arena = cfl_arena_create(128); + TEST_CHECK(arena != NULL); + array = cfl_array_create_in(arena, 4); + TEST_CHECK(array != NULL); + TEST_CHECK(cfl_arena_bytes_reserved(arena) > 0); + + cfl_arena_reset(arena); + TEST_CHECK(cfl_arena_bytes_reserved(arena) >= 128); + TEST_CHECK(cfl_arena_bytes_used(arena) == 0); + cfl_arena_destroy(arena); +} + +static void reject_cross_arena_values(void) +{ + struct cfl_arena *arena; + struct cfl_array *array; + struct cfl_variant *value; + + arena = cfl_arena_create(128); + TEST_CHECK(arena != NULL); + array = cfl_array_create_in(arena, 1); + value = cfl_variant_create_from_int64(1); + TEST_CHECK(array != NULL && value != NULL); + + TEST_CHECK(cfl_array_append(array, value) == -1); + cfl_variant_destroy(value); + cfl_arena_destroy(arena); +} + +static void mutable_otlp_log_graph(void) +{ + struct cfl_arena *arena; + struct cfl_kvlist *root; + struct cfl_kvlist *record; + struct cfl_kvlist *attributes; + struct cfl_array *records; + struct cfl_variant *root_variant; + struct cfl_variant *value; + size_t initial_used; + size_t final_used; + int round; + + arena = cfl_arena_create(256); + TEST_CHECK(arena != NULL); + root = cfl_kvlist_create_in(arena); + record = cfl_kvlist_create_in(arena); + attributes = cfl_kvlist_create_in(arena); + records = cfl_array_create_in(arena, 1); + TEST_CHECK(root != NULL && record != NULL && attributes != NULL && + records != NULL); + + TEST_CHECK(cfl_kvlist_insert_string(record, "severityText", "INFO") == 0); + TEST_CHECK(cfl_kvlist_insert_int64(attributes, "http.status_code", 200) == 0); + TEST_CHECK(cfl_kvlist_insert_kvlist(record, "attributes", attributes) == 0); + TEST_CHECK(cfl_array_append_kvlist(records, record) == 0); + TEST_CHECK(cfl_kvlist_insert_array(root, "logRecords", records) == 0); + root_variant = cfl_variant_create_from_kvlist_in(arena, root); + TEST_CHECK(root_variant != NULL); + initial_used = cfl_arena_bytes_used(arena); + + for (round = 0; round < 10; round++) { + TEST_CHECK(cfl_kvlist_remove(record, "severityText") == CFL_TRUE); + TEST_CHECK(cfl_kvlist_remove(attributes, "http.status_code") == CFL_TRUE); + TEST_CHECK(cfl_kvlist_insert_string(record, "severityText", "WARN") == 0); + TEST_CHECK(cfl_kvlist_insert_int64(attributes, "http.status_code", 503) == 0); + } + + value = cfl_kvlist_fetch(record, "severityText"); + TEST_CHECK(value != NULL); + TEST_CHECK(strcmp(value->data.as_string, "WARN") == 0); + value = cfl_kvlist_fetch(attributes, "http.status_code"); + TEST_CHECK(value != NULL); + TEST_CHECK(value->data.as_int64 == 503); + TEST_CHECK(cfl_array_size(records) == 1); + + final_used = cfl_arena_bytes_used(arena); + TEST_CHECK(final_used == initial_used); + cfl_variant_destroy(root_variant); + cfl_arena_destroy(arena); +} + +static void reclaim_large_string_on_remove(void) +{ + struct cfl_arena *arena; + struct cfl_kvlist *list; + char payload[8192]; + size_t before_remove; + size_t after_remove; + size_t reserved_before_remove; + char *first_buffer; + struct cfl_variant *value; + + memset(payload, 'x', sizeof(payload)); + arena = cfl_arena_create_ex(1024, 4096); + list = cfl_kvlist_create_in(arena); + TEST_CHECK(arena != NULL && list != NULL); + TEST_CHECK(cfl_arena_large_object_threshold(arena) == 4096); + TEST_CHECK(cfl_kvlist_insert_string_s(list, "body", 4, payload, + sizeof(payload), CFL_FALSE) == 0); + value = cfl_kvlist_fetch(list, "body"); + TEST_CHECK(value != NULL); + first_buffer = value->data.as_string; + before_remove = cfl_arena_bytes_used(arena); + reserved_before_remove = cfl_arena_bytes_reserved(arena); + TEST_CHECK(cfl_kvlist_remove(list, "body") == CFL_TRUE); + after_remove = cfl_arena_bytes_used(arena); + TEST_CHECK(before_remove - after_remove >= sizeof(payload)); + TEST_CHECK(cfl_arena_bytes_reserved(arena) == + reserved_before_remove); + TEST_CHECK(cfl_arena_external_cache_bytes(arena) >= + sizeof(payload)); + TEST_CHECK(cfl_kvlist_insert_string_s(list, "body", 4, payload, + sizeof(payload), CFL_FALSE) == 0); + value = cfl_kvlist_fetch(list, "body"); + TEST_CHECK(value != NULL); + TEST_CHECK(value->data.as_string == first_buffer); + + TEST_CHECK(cfl_kvlist_remove(list, "body") == CFL_TRUE); + TEST_CHECK(cfl_arena_external_cache_bytes(arena) > 0); + cfl_arena_external_cache_limit_set(arena, 0); + TEST_CHECK(cfl_arena_external_cache_limit_get(arena) == 0); + TEST_CHECK(cfl_arena_external_cache_bytes(arena) == 0); + cfl_arena_destroy(arena); +} + +static void reuse_variant_and_kvpair_slots(void) +{ + struct cfl_arena *arena; + struct cfl_variant *first_variant; + struct cfl_variant *second_variant; + struct cfl_kvlist *list; + struct cfl_kvpair *first_pair; + struct cfl_kvpair *second_pair; + + arena = cfl_arena_create(1024); + TEST_CHECK(arena != NULL); + first_variant = cfl_variant_create_from_int64_in(arena, 1); + TEST_CHECK(first_variant != NULL); + cfl_variant_destroy(first_variant); + second_variant = cfl_variant_create_from_int64_in(arena, 2); + TEST_CHECK(second_variant == first_variant); + cfl_variant_destroy(second_variant); + + list = cfl_kvlist_create_in(arena); + TEST_CHECK(list != NULL); + TEST_CHECK(cfl_kvlist_insert_int64(list, "first", 1) == 0); + first_pair = cfl_list_entry_first(&list->list, struct cfl_kvpair, _head); + TEST_CHECK(cfl_kvlist_remove(list, "first") == CFL_TRUE); + TEST_CHECK(cfl_kvlist_insert_int64(list, "second", 2) == 0); + second_pair = cfl_list_entry_first(&list->list, struct cfl_kvpair, _head); + TEST_CHECK(second_pair == first_pair); + cfl_kvlist_destroy(list); + cfl_arena_destroy(arena); +} + +static void create_like_and_rename(void) +{ + struct cfl_arena *arena; + struct cfl_kvlist *parent; + struct cfl_kvlist *child; + struct cfl_array *array; + struct cfl_array *child_array; + + arena = cfl_arena_create(1024); + parent = cfl_kvlist_create_in(arena); + child = cfl_kvlist_create_like(parent); + array = cfl_array_create_in(arena, 1); + child_array = cfl_array_create_like(array, 1); + TEST_CHECK(arena != NULL && parent != NULL && child != NULL); + TEST_CHECK(array != NULL && child_array != NULL); + TEST_CHECK(child->arena == arena && child_array->arena == arena); + + TEST_CHECK(cfl_kvlist_insert_string(parent, "old", "value") == 0); + TEST_CHECK(cfl_kvlist_rename_s(parent, "old", 3, "new", 3) == 0); + TEST_CHECK(cfl_kvlist_fetch(parent, "old") == NULL); + TEST_CHECK(cfl_kvlist_fetch(parent, "new") != NULL); + + cfl_array_destroy(child_array); + cfl_array_destroy(array); + cfl_kvlist_destroy(child); + cfl_kvlist_destroy(parent); + cfl_arena_destroy(arena); +} + +static void reuse_sds_size_classes(void) +{ + static const size_t sizes[] = {1, 32, 33, 64, 65, 128, + 129, 256, 257, 512, 513, 1024}; + struct cfl_arena *arena; + cfl_sds_t first; + cfl_sds_t second; + cfl_sds_t grown; + char payload[1024]; + size_t index; + + memset(payload, 's', sizeof(payload)); + arena = cfl_arena_create(8192); + TEST_CHECK(arena != NULL); + + for (index = 0; index < sizeof(sizes) / sizeof(sizes[0]); index++) { + first = cfl_sds_create_len_in(arena, payload, (int) sizes[index]); + TEST_CHECK(first != NULL); + cfl_sds_destroy(first); + second = cfl_sds_create_len_in(arena, payload, (int) sizes[index]); + TEST_CHECK(second == first); + cfl_sds_destroy(second); + } + + first = cfl_sds_create_len_in(arena, payload, 31); + TEST_CHECK(first != NULL); + TEST_CHECK(cfl_sds_alloc(first) == 32); + grown = cfl_sds_increase(first, 1); + TEST_CHECK(grown != NULL); + TEST_CHECK(cfl_sds_alloc(grown) == 64); + TEST_CHECK(cfl_sds_len(grown) == 31); + TEST_CHECK(memcmp(grown, payload, 31) == 0); + + second = cfl_sds_create_len_in(arena, payload, 32); + TEST_CHECK(second == first); + cfl_sds_destroy(second); + cfl_sds_destroy(grown); + cfl_arena_destroy(arena); +} + +static void bound_external_rounding_and_cache(void) +{ + struct cfl_arena *arena; + cfl_sds_t value; + char *payload; + size_t payload_size; + size_t used; + + payload_size = 1024 * 1024; + payload = malloc(payload_size); + TEST_CHECK(payload != NULL); + if (payload == NULL) { + return; + } + memset(payload, 'x', payload_size); + + arena = cfl_arena_create(8192); + TEST_CHECK(arena != NULL); + cfl_arena_external_cache_limit_set(arena, 1024 * 1024); + + value = cfl_sds_create_len_in(arena, payload, (int) payload_size); + TEST_CHECK(value != NULL); + used = cfl_arena_bytes_used(arena); + TEST_CHECK(used <= payload_size + (payload_size / 4) + 4096); + cfl_sds_destroy(value); + TEST_CHECK(cfl_arena_external_cache_bytes(arena) <= + cfl_arena_external_cache_limit_get(arena)); + + cfl_arena_destroy(arena); + free(payload); +} + +static void reclaim_failed_variant_construction(void) +{ + struct cfl_arena *arena; + struct cfl_array *array; + struct cfl_variant *value; + struct cfl_variant *reused; + size_t before_failure; + size_t after_failure; + + arena = cfl_arena_create(1024); + TEST_CHECK(arena != NULL); + before_failure = cfl_arena_bytes_used(arena); + value = cfl_variant_create_from_string_s_in(arena, "x", + (size_t) INT_MAX + 1, + CFL_FALSE); + TEST_CHECK(value == NULL); + after_failure = cfl_arena_bytes_used(arena); + TEST_CHECK(after_failure == before_failure); + + array = cfl_array_create_in(arena, SIZE_MAX); + TEST_CHECK(array == NULL); + TEST_CHECK(cfl_arena_bytes_used(arena) == after_failure); + + reused = cfl_variant_create_from_int64_in(arena, 1); + TEST_CHECK(reused != NULL); + TEST_CHECK(((uintptr_t) reused % + offsetof(struct test_alignment_probe, value)) == 0); + cfl_variant_destroy(reused); + cfl_arena_destroy(arena); +} + +TEST_LIST = { + {"public_raw_allocations", public_raw_allocations}, + {"public_raw_allocation_failures", public_raw_allocation_failures}, + {"options_growth_and_callbacks", options_growth_and_callbacks}, + {"callback_allocation_failure", callback_allocation_failure}, + {"arena_build_and_destroy", arena_build_and_destroy}, + {"arena_reset", arena_reset}, + {"reject_cross_arena_values", reject_cross_arena_values}, + {"mutable_otlp_log_graph", mutable_otlp_log_graph}, + {"reclaim_large_string_on_remove", reclaim_large_string_on_remove}, + {"reuse_variant_and_kvpair_slots", reuse_variant_and_kvpair_slots}, + {"create_like_and_rename", create_like_and_rename}, + {"reuse_sds_size_classes", reuse_sds_size_classes}, + {"bound_external_rounding_and_cache", bound_external_rounding_and_cache}, + {"reclaim_failed_variant_construction", reclaim_failed_variant_construction}, + {NULL, NULL} +}; diff --git a/lib/cfl/tests/array.c b/lib/cfl/tests/array.c index 27096013e55..1fdfe3cde97 100644 --- a/lib/cfl/tests/array.c +++ b/lib/cfl/tests/array.c @@ -497,6 +497,34 @@ static void remove_by_reference() cfl_array_destroy(arr); } +static void remove_by_reference_not_found() +{ + int ret; + struct cfl_array *arr; + struct cfl_array *other; + struct cfl_variant *var; + + arr = cfl_array_create(1); + TEST_CHECK(arr != NULL); + + other = cfl_array_create(1); + TEST_CHECK(other != NULL); + + ret = cfl_array_append_string(other, "test"); + TEST_CHECK(ret == 0); + + var = cfl_array_fetch_by_index(other, 0); + TEST_CHECK(var != NULL); + + ret = cfl_array_remove_by_reference(arr, var); + TEST_CHECK(ret == -1); + + TEST_CHECK(cfl_array_fetch_by_index(other, 0) == var); + + cfl_array_destroy(other); + cfl_array_destroy(arr); +} + static void print_write_error() { #ifdef __linux__ @@ -585,6 +613,7 @@ TEST_LIST = { {"append_kvlist_rejects_cycles", append_kvlist_rejects_cycles}, {"remove_by_index", remove_by_index}, {"remove_by_reference", remove_by_reference}, + {"remove_by_reference_not_found", remove_by_reference_not_found}, {"print_write_error", print_write_error}, {"null_inputs", null_inputs}, { 0 } diff --git a/lib/cfl/tests/installed_consumer/CMakeLists.txt b/lib/cfl/tests/installed_consumer/CMakeLists.txt new file mode 100644 index 00000000000..2476828282e --- /dev/null +++ b/lib/cfl/tests/installed_consumer/CMakeLists.txt @@ -0,0 +1,15 @@ +cmake_minimum_required(VERSION 3.20) +project(cfl_installed_consumer C) + +find_package(Threads REQUIRED) +find_path(CFL_INCLUDE_DIR NAMES cfl/cfl.h REQUIRED) +find_library(CFL_LIBRARY NAMES cfl REQUIRED) +find_library(XXHASH_LIBRARY NAMES xxhash REQUIRED) + +add_executable(cfl-installed-consumer main.c) +target_include_directories(cfl-installed-consumer PRIVATE ${CFL_INCLUDE_DIR}) +target_link_libraries(cfl-installed-consumer + PRIVATE + ${CFL_LIBRARY} + ${XXHASH_LIBRARY} + Threads::Threads) diff --git a/lib/cfl/tests/installed_consumer/main.c b/lib/cfl/tests/installed_consumer/main.c new file mode 100644 index 00000000000..1c806b66fc3 --- /dev/null +++ b/lib/cfl/tests/installed_consumer/main.c @@ -0,0 +1,55 @@ +#include +#include + +#include + +int main(void) +{ + int result; + char *request_name; + uint64_t hash; + struct cfl_arena *arena; + struct cfl_array *array; + + result = cfl_init(); + if (result != 0) { + return 1; + } + + if (cfl_version() == NULL) { + return 1; + } + + array = cfl_array_create(1); + if (array == NULL) { + return 1; + } + + result = cfl_array_append_string(array, "installed"); + if (result != 0 || cfl_array_size(array) != 1) { + cfl_array_destroy(array); + return 1; + } + + hash = cfl_hash_64bits("installed", strlen("installed")); + cfl_array_destroy(array); + + if (hash == 0) { + return 1; + } + + arena = cfl_arena_create(0); + if (arena == NULL) { + return 1; + } + + request_name = cfl_arena_strndup(arena, "installed", 9); + if (request_name == NULL || strcmp(request_name, "installed") != 0) { + cfl_arena_destroy(arena); + return 1; + } + + cfl_arena_destroy(arena); + + return 0; +} diff --git a/lib/cfl/tests/kvlist.c b/lib/cfl/tests/kvlist.c index ac073750a7d..304580387f2 100644 --- a/lib/cfl/tests/kvlist.c +++ b/lib/cfl/tests/kvlist.c @@ -1323,6 +1323,115 @@ static void embedded_nul_keys_do_not_match_short_name() cfl_kvlist_destroy(list); } +static void case_sensitive_operations() +{ + int ret; + char upper_key[] = {'U', 's', 'e', 'r'}; + char lower_key[] = {'u', 's', 'e', 'r'}; + char nul_key[] = {'k', '\0', 'A'}; + char different_nul_key[] = {'K', '\0', 'B'}; + struct cfl_kvlist *list; + struct cfl_variant *variant; + + list = cfl_kvlist_create(); + TEST_CHECK(list != NULL); + + ret = cfl_kvlist_insert_int64(list, "User", 1); + TEST_CHECK(ret == 0); + ret = cfl_kvlist_insert_int64(list, "user", 2); + TEST_CHECK(ret == 0); + ret = cfl_kvlist_insert_int64_s(list, nul_key, sizeof(nul_key), 3); + TEST_CHECK(ret == 0); + + variant = cfl_kvlist_fetch_s_ex(list, upper_key, sizeof(upper_key), + CFL_KVLIST_MATCH_CASE_SENSITIVE); + TEST_CHECK(variant != NULL); + TEST_CHECK(variant->data.as_int64 == 1); + + variant = cfl_kvlist_fetch_case_s(list, lower_key, sizeof(lower_key)); + TEST_CHECK(variant != NULL); + TEST_CHECK(variant->data.as_int64 == 2); + + variant = cfl_kvlist_fetch_ex(list, "USER", + CFL_KVLIST_MATCH_CASE_SENSITIVE); + TEST_CHECK(variant == NULL); + variant = cfl_kvlist_fetch(list, "USER"); + TEST_CHECK(variant != NULL); + + ret = cfl_kvlist_contains_ex(list, "USER", + CFL_KVLIST_MATCH_CASE_SENSITIVE); + TEST_CHECK(ret == CFL_FALSE); + ret = cfl_kvlist_contains(list, "USER"); + TEST_CHECK(ret == CFL_TRUE); + + ret = cfl_kvlist_remove_ex(list, "User", + CFL_KVLIST_MATCH_CASE_SENSITIVE); + TEST_CHECK(ret == CFL_TRUE); + TEST_CHECK(cfl_kvlist_count(list) == 2); + TEST_CHECK(cfl_kvlist_fetch_ex(list, "User", + CFL_KVLIST_MATCH_CASE_SENSITIVE) == NULL); + variant = cfl_kvlist_fetch_ex(list, "user", + CFL_KVLIST_MATCH_CASE_SENSITIVE); + TEST_CHECK(variant != NULL); + TEST_CHECK(variant->data.as_int64 == 2); + + variant = cfl_kvlist_fetch_s_ex(list, different_nul_key, + sizeof(different_nul_key), + CFL_KVLIST_MATCH_CASE_INSENSITIVE); + TEST_CHECK(variant == NULL); + variant = cfl_kvlist_fetch_s_ex(list, nul_key, sizeof(nul_key), + CFL_KVLIST_MATCH_CASE_INSENSITIVE); + TEST_CHECK(variant != NULL); + TEST_CHECK(variant->data.as_int64 == 3); + + variant = cfl_kvlist_fetch_ex(list, "user", + (enum cfl_kvlist_match_mode) 99); + TEST_CHECK(variant == NULL); + ret = cfl_kvlist_contains_ex(list, "user", + (enum cfl_kvlist_match_mode) 99); + TEST_CHECK(ret == CFL_FALSE); + ret = cfl_kvlist_remove_ex(list, "user", + (enum cfl_kvlist_match_mode) 99); + TEST_CHECK(ret == CFL_FALSE); + TEST_CHECK(cfl_kvlist_count(list) == 2); + + cfl_kvlist_destroy(list); +} + +static void case_sensitive_arena_operations() +{ + int ret; + struct cfl_arena *arena; + struct cfl_kvlist *list; + struct cfl_variant *variant; + + arena = cfl_arena_create(256); + TEST_CHECK(arena != NULL); + list = cfl_kvlist_create_in(arena); + TEST_CHECK(list != NULL); + + ret = cfl_kvlist_insert_string(list, "TraceId", "upper"); + TEST_CHECK(ret == 0); + ret = cfl_kvlist_insert_string(list, "traceid", "lower"); + TEST_CHECK(ret == 0); + + variant = cfl_kvlist_fetch_ex(list, "TraceId", + CFL_KVLIST_MATCH_CASE_SENSITIVE); + TEST_CHECK(variant != NULL); + TEST_CHECK(strcmp(variant->data.as_string, "upper") == 0); + + ret = cfl_kvlist_remove_ex(list, "traceid", + CFL_KVLIST_MATCH_CASE_SENSITIVE); + TEST_CHECK(ret == CFL_TRUE); + TEST_CHECK(cfl_kvlist_count(list) == 1); + TEST_CHECK(cfl_kvlist_contains_ex(list, "TraceId", + CFL_KVLIST_MATCH_CASE_SENSITIVE) == + CFL_TRUE); + + cfl_kvlist_destroy(list); + cfl_arena_destroy(arena); +} + static void print_write_error() { #ifdef __linux__ @@ -1542,6 +1651,8 @@ TEST_LIST = { {"null_inputs", null_inputs}, {"print_escaped_keys", print_escaped_keys}, {"embedded_nul_keys_do_not_match_short_name", embedded_nul_keys_do_not_match_short_name}, + {"case_sensitive_operations", case_sensitive_operations}, + {"case_sensitive_arena_operations", case_sensitive_arena_operations}, {"print_write_error", print_write_error}, {"reject_kvlist_cycles", reject_kvlist_cycles}, {"reject_variant_cycles", reject_variant_cycles}, diff --git a/lib/cfl/tests/sds.c b/lib/cfl/tests/sds.c index e8baf25b30b..ee568c6dcfa 100644 --- a/lib/cfl/tests/sds.c +++ b/lib/cfl/tests/sds.c @@ -18,8 +18,30 @@ */ #include +#include #include "cfl_tests_internal.h" +static void test_sds_header_compatibility() +{ + cfl_sds_t s; + struct cfl_sds *head; + + TEST_CHECK(CFL_SDS_HEADER_SIZE == sizeof(uint64_t) * 2); + TEST_CHECK(offsetof(struct cfl_sds, buf) == CFL_SDS_HEADER_SIZE); + + s = cfl_sds_create_size(64); + TEST_CHECK(s != NULL); + if (s == NULL) { + return; + } + + head = CFL_SDS_HEADER(s); + TEST_CHECK(head->len == 0); + TEST_CHECK(head->alloc == 64); + TEST_CHECK(head->buf == s); + cfl_sds_destroy(s); +} + static void test_sds_usage() { cfl_sds_t s; @@ -50,6 +72,17 @@ static void test_sds_printf() TEST_CHECK(tmp == s); TEST_CHECK(cfl_sds_len(s) == len); cfl_sds_destroy(s); + + s = cfl_sds_create("prefix"); + TEST_CHECK(s != NULL); + + tmp = cfl_sds_printf(&s, "-%s", str); + TEST_CHECK(tmp == s); + TEST_CHECK(cfl_sds_len(s) == strlen("prefix-") + strlen(str)); + TEST_CHECK(strncmp(s, "prefix-", strlen("prefix-")) == 0); + TEST_CHECK(strcmp(s + strlen("prefix-"), str) == 0); + + cfl_sds_destroy(s); } static void test_sds_invalid_inputs() @@ -172,6 +205,7 @@ static void test_sds_in_buffer_slice_boundaries() } TEST_LIST = { + { "sds_header_compatibility", test_sds_header_compatibility}, { "sds_usage" , test_sds_usage}, { "sds_printf", test_sds_printf}, { "sds_invalid_inputs", test_sds_invalid_inputs}, diff --git a/lib/cmetrics/.dockerignore b/lib/cmetrics/.dockerignore new file mode 100644 index 00000000000..ff3d1ae0c6a --- /dev/null +++ b/lib/cmetrics/.dockerignore @@ -0,0 +1,3 @@ +.git +build +lib/cfl/build-review diff --git a/lib/cmetrics/.github/workflows/build.yaml b/lib/cmetrics/.github/workflows/build.yaml index efe8baf74ad..2b29104f941 100644 --- a/lib/cmetrics/.github/workflows/build.yaml +++ b/lib/cmetrics/.github/workflows/build.yaml @@ -38,61 +38,33 @@ jobs: build-centos: name: CentOS 7 build to confirm no issues once used downstream runs-on: ubuntu-latest - container: centos:7 - env: - # workaround required for checkout@v3, https://github.com/actions/checkout/issues/1590 - ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true + permissions: + contents: read steps: - - name: Set up base image dependencies - run: | - sed -i -e "s/^mirrorlist=http:\/\/mirrorlist.centos.org/#mirrorlist=http:\/\/mirrorlist.centos.org/g" /etc/yum.repos.d/CentOS-Base.repo - sed -i -e "s/^#baseurl=http:\/\/mirror.centos.org/baseurl=http:\/\/vault.centos.org/g" /etc/yum.repos.d/CentOS-Base.repo - yum -y update - yum install -y ca-certificates gcc gcc-c++ git make wget - yum install -y epel-release - yum install -y libcurl-devel - - shell: bash - - - name: Install CMake 3.20.0 - run: | - CMAKE_VERSION=3.20.0 - wget https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-linux-x86_64.sh - chmod +x cmake-${CMAKE_VERSION}-linux-x86_64.sh - ./cmake-${CMAKE_VERSION}-linux-x86_64.sh --skip-license --prefix=/usr/local - ln -sf /usr/local/bin/cmake /usr/bin/cmake - cmake --version - - - name: Clone repo without submodules (1.8.3 version of Git) - run: | - git clone https://github.com/fluent/cmetrics.git - shell: bash + - name: Check out the repository + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + submodules: recursive - - name: Check out the branch (1.8.3 version of Git) - env: - BRANCH_NAME: ${{ github.head_ref }} - run: | - git checkout "$BRANCH_NAME" - git submodule update --init --recursive - shell: bash - working-directory: cmetrics + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0 - - name: Run compilation - run: | - cmake -DCMT_TESTS=on -DCMT_DEV=on . - make - working-directory: cmetrics + - name: Build sources on CentOS 7 + uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0 + with: + context: . + file: ./dockerfiles/Dockerfile.centos7 + push: false + load: false + provenance: false build-debian: - name: Debian Buster build to confirm no issues once used downstream + name: Debian Bookworm build to confirm no issues once used downstream runs-on: ubuntu-latest - container: debian:buster + container: debian:bookworm steps: - name: Set up base image dependencies run: | - # Update sources to use archive.debian.org (Buster reached end-of-life) - sed -i 's/deb.debian.org/archive.debian.org/g' /etc/apt/sources.list - sed -i 's/security.debian.org/archive.debian.org/g' /etc/apt/sources.list apt-get update apt-get install -y build-essential wget make gcc g++ git libcurl4-openssl-dev diff --git a/lib/cmetrics/AGENTS.md b/lib/cmetrics/AGENTS.md new file mode 100644 index 00000000000..c640e3e9593 --- /dev/null +++ b/lib/cmetrics/AGENTS.md @@ -0,0 +1,148 @@ +# CMetrics repository guide + +This is the canonical, vendor-neutral operating guide for automated tools and +human contributors. Tool-specific files must point here instead of duplicating +project rules. + +## Project role + +CMetrics is a standalone C library for creating, mutating, aggregating, +encoding, and decoding metrics contexts. It is consumed by downstream projects, +including Fluent Bit, so changes can affect public C APIs and serialized data. + +## Repository map + +- `include/cmetrics/`: installed public headers; treat layout and declarations + as compatibility-sensitive. +- `src/`: metric implementations, map/index ownership, filters, and codecs. +- `tests/`: Acutest unit tests, one CTest executable per source file. +- `benchmarks/`: opt-in benchmark executable and Linux `perf stat` runner. +- `docs/`: design and behavior notes; agent-neutral workflows are in `docs/ai/`. +- `cmake/`, `CMakeLists.txt`: build, dependency, install, and packaging rules. +- `lib/cfl`: CFL Git submodule; owns containers, SDS strings, arenas, and + supporting primitives. +- `lib/fluent-otel-proto`: generated OpenTelemetry protobuf support submodule. +- `.github/workflows/`: compiler/platform builds, tests, linting, and packages. + +Start architecture investigation at `include/cmetrics/cmetrics.h`, the +metric-specific public header, and its matching `src/cmt_*.c` implementation. +For encoding or decoding, follow the corresponding `cmt_encode_*` or +`cmt_decode_*` pair and its tests. See `docs/architecture.md` for the component +map and `docs/dependencies.md` for repository boundaries. + +## Dependency boundaries + +Initialize submodules before building: + +```sh +git submodule update --init --recursive +``` + +Changes owned by CFL or fluent-otel-proto should normally land in their source +repository first, then update the CMetrics submodule revision separately. +Validate CMetrics standalone before updating a downstream consumer such as +Fluent Bit. See `docs/ai/cross-repository.md` and +`docs/ai/dependency-update.md`. + +## Build and test + +The supported build system is CMake 3.20 or newer. Local builds require Git, a +C compiler, and the initialized submodules. Flex 2 and Bison 3 enable the +Prometheus text decoder and its tests; CMake omits that decoder if they are not +found. A normal development build: + +```sh +cmake -S . -B build/agent -DCMT_TESTS=On -DCMT_INSTALL_TARGETS=Off +cmake --build build/agent +ctest --test-dir build/agent --output-on-failure +``` + +Repository wrappers provide the same flow: + +```sh +scripts/agent-build.sh +scripts/agent-test.sh +scripts/agent-verify.sh +``` + +Use `BUILD_DIR=/path` to select another build directory. Pass extra CMake +configuration arguments to `agent-build.sh`. Pass a CTest regular expression +to `agent-test.sh`, for example: + +```sh +scripts/agent-test.sh '^cmt-test-opentelemetry$' +``` + +When making code changes, run the related unit test when one is available. +Before handoff, run `scripts/agent-verify.sh` unless the change is documentation +only or the environment cannot build the project; report any omitted check. + +There is no repository-defined C formatter or C static-lint command. Preserve +the surrounding four-space, no-tab style. Shell changes must pass `sh -n`; CI +also runs ShellCheck. Do not claim sanitizer coverage unless the binary was +actually compiled and executed with the requested sanitizer. + +## Memory and ownership + +- Check every allocation and preserve cleanup for partial initialization. +- Match allocation families (`malloc`/`free`, CFL SDS create/destroy, arena + lifetime) and make ownership transfers explicit in code structure. +- Exercise failure and cleanup paths for codec and container changes. +- Treat map mutation, metric indexing, expiration, and destruction as + concurrency-sensitive. +- Use AddressSanitizer or Valgrind for ownership changes; follow + `docs/ai/memory-safety-review.md`. + +## Compatibility-sensitive changes + +- Public declarations under `include/cmetrics/` can affect source or ABI + compatibility. Avoid changing established public structure layout without + explicit review of downstream users. +- Internal MessagePack, OTLP protobuf, Prometheus formats, and remote write are + wire-sensitive. Add round-trip and malformed-input coverage where relevant. +- Preserve integer value types, timestamps, start timestamps, label ordering, + and metric identity unless the change intentionally revises their contract. +- Decoders process untrusted lengths and values: prevent overflow, oversized + allocation from incomplete input, desynchronization, and partial-result leaks. + +## Generated and vendored content + +- Do not edit CMake-generated `include/cmetrics/cmt_info.h` or + `include/cmetrics/cmt_version.h`; edit their `.in` templates or CMake version. +- Flex/Bison outputs are generated in the build directory from + `src/cmt_decode_prometheus.l` and `.y`; edit the grammar sources. +- Files marked generated under `src/external/` and + `include/prometheus_remote_write/` must be regenerated from their source + schema/toolchain, not hand-edited. +- Do not edit submodule contents as part of a CMetrics-only change. Make the + change in the owning repository and update the recorded revision. +- Build outputs and generated payloads do not belong in commits unless an + existing tracked fixture is intentionally updated and reviewed. + +## Performance changes + +Build benchmarks with `CMT_BENCHMARKS=ON` and an optimized Release build. Use +`benchmarks/run-perf.sh` for standard workloads and Linux hardware counters. +Capture at least five before and five after runs on the same machine with +identical compiler flags and workload parameters. Keep only repeatable gains +with no relevant regression. See `docs/ai/performance-review.md`. + +## Change discipline and definition of done + +- Keep changes scoped; preserve unrelated user work and avoid drive-by cleanup. +- Add a regression test for a bug when practical. +- Verify every documented command against repository files before adding it. +- Review error paths, compatibility, generated files, and dependency ownership. +- Run targeted tests plus the strongest practical complete validation. +- Report root cause or intent, files changed, exact checks, compatibility + impact, unresolved risks, and anything not validated. + +Detailed workflows: + +- `docs/ai/investigate.md` +- `docs/ai/bug-fix.md` +- `docs/ai/code-review.md` +- `docs/ai/cross-repository.md` +- `docs/ai/dependency-update.md` +- `docs/ai/memory-safety-review.md` +- `docs/ai/performance-review.md` diff --git a/lib/cmetrics/CLAUDE.md b/lib/cmetrics/CLAUDE.md new file mode 100644 index 00000000000..43c994c2d36 --- /dev/null +++ b/lib/cmetrics/CLAUDE.md @@ -0,0 +1 @@ +@AGENTS.md diff --git a/lib/cmetrics/CMakeLists.txt b/lib/cmetrics/CMakeLists.txt index d7da71a795b..1012829c282 100644 --- a/lib/cmetrics/CMakeLists.txt +++ b/lib/cmetrics/CMakeLists.txt @@ -5,8 +5,8 @@ set(CMAKE_EXPORT_COMPILE_COMMANDS ON) # CMetrics Version set(CMT_VERSION_MAJOR 2) -set(CMT_VERSION_MINOR 1) -set(CMT_VERSION_PATCH 5) +set(CMT_VERSION_MINOR 2) +set(CMT_VERSION_PATCH 1) set(CMT_VERSION_STR "${CMT_VERSION_MAJOR}.${CMT_VERSION_MINOR}.${CMT_VERSION_PATCH}") # Include helpers @@ -45,9 +45,12 @@ if(CMAKE_SYSTEM_NAME MATCHES "Windows") set(CMT_SYSTEM_WINDOWS On) add_definitions(-DCMT_SYSTEM_WINDOWS) - # Disable unistd.h for flex/bison - CMT_DEFINITION(YY_NO_UNISTD_H) - message(STATUS "Specifying YY_NO_UNISTD_H") + # Disable unistd.h for flex/bison on MSVC only: MinGW provides a real + # unistd.h, and without it the generated lexer has no isatty declaration. + if(MSVC) + CMT_DEFINITION(YY_NO_UNISTD_H) + message(STATUS "Specifying YY_NO_UNISTD_H") + endif() endif() # Define macro to identify macOS system @@ -60,17 +63,14 @@ if(NOT MSVC) set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall") endif() -# Define __CMT_FILENAME__ consistently across Operating Systems -if(NOT ${CMAKE_SYSTEM_NAME} MATCHES "Windows") - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -D__CMT_FILENAME__='\"$$(subst ${CMAKE_SOURCE_DIR}/,,$$(abspath $$<))\"'") -else() - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -D__CMT_FILENAME__=__FILE__") -endif() +# Define __CMT_FILENAME__ +set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -D__CMT_FILENAME__=__FILE__") # Configuration options option(CMT_DEV "Enable development mode" No) option(CMT_DEBUG "Enable debug mode" No) option(CMT_TESTS "Enable unit testing" No) +option(CMT_BENCHMARKS "Build performance benchmarks" No) option(CMT_INSTALL_TARGETS "Enable subdirectory library installations" Yes) option(CMT_PROMETHEUS_TEXT_DECODER "Enable prometheus text format decoder (requires Flex/Bison)" Yes) @@ -299,6 +299,10 @@ endif() add_subdirectory(include) add_subdirectory(src) +if(CMT_BENCHMARKS) + add_subdirectory(benchmarks) +endif() + # Tests if(CMT_TESTS) enable_testing() diff --git a/lib/cmetrics/README.md b/lib/cmetrics/README.md index 191d6b1c4e5..649f96c9526 100644 --- a/lib/cmetrics/README.md +++ b/lib/cmetrics/README.md @@ -141,6 +141,10 @@ CMetrics is heavily inspired by the Go Prometheus Client API design: - https://pkg.go.dev/github.com/prometheus/client_golang/prometheus#section-documentation +Additional design notes: + +- [Long metric label handling](docs/label-value-handling.md) + ## License This program is under the terms of the diff --git a/lib/cmetrics/benchmarks/CMakeLists.txt b/lib/cmetrics/benchmarks/CMakeLists.txt new file mode 100644 index 00000000000..59c1ae38d3b --- /dev/null +++ b/lib/cmetrics/benchmarks/CMakeLists.txt @@ -0,0 +1,6 @@ +add_executable(cmt-benchmark benchmark.c) +target_link_libraries(cmt-benchmark cmetrics-static cfl-static fluent-otel-proto) + +if(NOT CMT_SYSTEM_WINDOWS) + target_link_libraries(cmt-benchmark pthread) +endif() diff --git a/lib/cmetrics/benchmarks/README.md b/lib/cmetrics/benchmarks/README.md new file mode 100644 index 00000000000..1aa43c558e8 --- /dev/null +++ b/lib/cmetrics/benchmarks/README.md @@ -0,0 +1,38 @@ +# CMetrics benchmarks + +Benchmarks are opt-in and are intended for before/after comparisons on the +same machine. Build an optimized binary: + +```sh +cmake -S . -B build-perf \ + -DCMT_BENCHMARKS=ON \ + -DCMT_INSTALL_TARGETS=OFF \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_C_FLAGS_RELEASE='-O3 -DNDEBUG' +cmake --build build-perf -j --target cmt-benchmark +``` + +Run the standard repeated workloads and Linux hardware counters: + +```sh +REPETITIONS=5 benchmarks/run-perf.sh \ + ./build-perf/benchmarks/cmt-benchmark +``` + +The executable also accepts individual workloads: + +```text +cmt-benchmark lookup|update|prometheus|opentelemetry|opentelemetry-mixed CARDINALITY OPERATIONS +``` + +The `opentelemetry` workload repeatedly encodes a labeled counter with the +requested number of series. The `opentelemetry-mixed` workload creates that +many counter, gauge, and histogram series to exercise scalar and aggregate +protobuf data points in the same request. + +Compare medians from at least five alternating before/after runs. Keep CPU +frequency policy, compiler, flags, machine load, and input parameters fixed. +Use the reported in-process `elapsed_ns` for the operation itself and `perf +stat` for whole-process hardware counters. Whole-process counters include +series construction and teardown by design, exposing setup complexity as well +as steady-state behavior. diff --git a/lib/cmetrics/benchmarks/benchmark.c b/lib/cmetrics/benchmarks/benchmark.c new file mode 100644 index 00000000000..13a8661d18c --- /dev/null +++ b/lib/cmetrics/benchmarks/benchmark.c @@ -0,0 +1,346 @@ +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +static uint64_t monotonic_ns(void) +{ + struct timespec timestamp; + + clock_gettime(CLOCK_MONOTONIC, ×tamp); + + return (uint64_t) timestamp.tv_sec * 1000000000ULL + timestamp.tv_nsec; +} + +static size_t parse_size(const char *text, const char *name) +{ + char *end; + unsigned long long value; + + errno = 0; + value = strtoull(text, &end, 10); + if (errno != 0 || *text == '\0' || *end != '\0' || value == 0) { + fprintf(stderr, "invalid %s: %s\n", name, text); + exit(EXIT_FAILURE); + } + + return (size_t) value; +} + +static struct cmt_counter *create_series(struct cmt *cmt, size_t cardinality) +{ + size_t index; + int result; + char label[32]; + char *values[] = {label}; + struct cmt_counter *counter; + + counter = cmt_counter_create(cmt, "bench", "", "counter", "benchmark", + 1, (char *[]) {"series"}); + if (counter == NULL) { + return NULL; + } + + for (index = 0; index < cardinality; index++) { + snprintf(label, sizeof(label), "series-%zu", index); + result = cmt_counter_set(counter, 1, 1.0, 1, values); + if (result != 0) { + return NULL; + } + } + + return counter; +} + +static int create_mixed_series(struct cmt *cmt, size_t cardinality) +{ + size_t index; + char label[32]; + char *values[] = {label}; + struct cmt_counter *counter; + struct cmt_gauge *gauge; + struct cmt_histogram *histogram; + struct cmt_histogram_buckets *buckets; + + counter = cmt_counter_create(cmt, "bench", "", "requests_total", + "benchmark counter", 1, + (char *[]) {"series"}); + gauge = cmt_gauge_create(cmt, "bench", "", "queue_depth", + "benchmark gauge", 1, + (char *[]) {"series"}); + buckets = cmt_histogram_buckets_create(4, 0.01, 0.1, 1.0, 10.0); + histogram = cmt_histogram_create(cmt, "bench", "", "latency_seconds", + "benchmark histogram", buckets, 1, + (char *[]) {"series"}); + if (counter == NULL || gauge == NULL || buckets == NULL || + histogram == NULL) { + return -1; + } + + for (index = 0; index < cardinality; index++) { + snprintf(label, sizeof(label), "series-%zu", index); + if (cmt_counter_set(counter, index + 1, 1.0, 1, values) != 0 || + cmt_gauge_set(gauge, index + 1, (double) index, 1, values) != 0 || + cmt_histogram_observe(histogram, index + 1, + (double) (index % 100) / 10.0, + 1, values) != 0) { + return -1; + } + } + + return 0; +} + +static int benchmark_lookup(size_t cardinality, size_t operations) +{ + size_t index; + int result; + double value; + uint64_t start; + uint64_t elapsed; + char label[32]; + char *values[] = {label}; + struct cmt *cmt; + struct cmt_counter *counter; + + cmt = cmt_create(); + if (cmt == NULL) { + return -1; + } + + counter = create_series(cmt, cardinality); + if (counter == NULL) { + cmt_destroy(cmt); + return -1; + } + + start = monotonic_ns(); + for (index = 0; index < operations; index++) { + snprintf(label, sizeof(label), "series-%zu", index % cardinality); + result = cmt_counter_get_val(counter, 1, values, &value); + if (result != 0 || value != 1.0) { + cmt_destroy(cmt); + return -1; + } + } + elapsed = monotonic_ns() - start; + + printf("benchmark=lookup cardinality=%zu operations=%zu elapsed_ns=%" PRIu64 + " ns_per_op=%.2f ops_per_second=%.2f\n", + cardinality, operations, elapsed, (double) elapsed / operations, + (double) operations * 1000000000.0 / elapsed); + cmt_destroy(cmt); + return 0; +} + +static int benchmark_update(size_t cardinality, size_t operations) +{ + size_t index; + int result; + uint64_t start; + uint64_t elapsed; + char label[32]; + char *values[] = {label}; + struct cmt *cmt; + struct cmt_counter *counter; + + cmt = cmt_create(); + if (cmt == NULL) { + return -1; + } + + counter = create_series(cmt, cardinality); + if (counter == NULL) { + cmt_destroy(cmt); + return -1; + } + + start = monotonic_ns(); + for (index = 0; index < operations; index++) { + snprintf(label, sizeof(label), "series-%zu", index % cardinality); + result = cmt_counter_inc(counter, index + 2, 1, values); + if (result != 0) { + cmt_destroy(cmt); + return -1; + } + } + elapsed = monotonic_ns() - start; + + printf("benchmark=update cardinality=%zu operations=%zu elapsed_ns=%" PRIu64 + " ns_per_op=%.2f ops_per_second=%.2f\n", + cardinality, operations, elapsed, (double) elapsed / operations, + (double) operations * 1000000000.0 / elapsed); + cmt_destroy(cmt); + return 0; +} + +static int benchmark_prometheus(size_t cardinality, size_t operations) +{ + size_t index; + size_t bytes = 0; + uint64_t start; + uint64_t elapsed; + cfl_sds_t output; + struct cmt *cmt; + + cmt = cmt_create(); + if (cmt == NULL) { + return -1; + } + + if (create_series(cmt, cardinality) == NULL) { + cmt_destroy(cmt); + return -1; + } + + start = monotonic_ns(); + for (index = 0; index < operations; index++) { + output = cmt_encode_prometheus_create(cmt, CMT_FALSE); + if (output == NULL) { + cmt_destroy(cmt); + return -1; + } + bytes += cfl_sds_len(output); + cmt_encode_prometheus_destroy(output); + } + elapsed = monotonic_ns() - start; + + printf("benchmark=prometheus cardinality=%zu operations=%zu bytes=%zu " + "elapsed_ns=%" PRIu64 " ns_per_op=%.2f mb_per_second=%.2f\n", + cardinality, operations, bytes, elapsed, (double) elapsed / operations, + ((double) bytes / (1024.0 * 1024.0)) / + ((double) elapsed / 1000000000.0)); + cmt_destroy(cmt); + return 0; +} + +static int benchmark_opentelemetry(size_t cardinality, size_t operations) +{ + size_t index; + size_t bytes; + uint64_t start; + uint64_t elapsed; + cfl_sds_t output; + struct cmt *cmt; + + bytes = 0; + cmt = cmt_create(); + if (cmt == NULL) { + return -1; + } + + if (create_series(cmt, cardinality) == NULL) { + cmt_destroy(cmt); + return -1; + } + + start = monotonic_ns(); + for (index = 0; index < operations; index++) { + output = cmt_encode_opentelemetry_create(cmt); + if (output == NULL) { + cmt_destroy(cmt); + return -1; + } + bytes += cfl_sds_len(output); + cmt_encode_opentelemetry_destroy(output); + } + elapsed = monotonic_ns() - start; + + printf("benchmark=opentelemetry cardinality=%zu operations=%zu bytes=%zu " + "elapsed_ns=%" PRIu64 " ns_per_op=%.2f mb_per_second=%.2f\n", + cardinality, operations, bytes, elapsed, (double) elapsed / operations, + ((double) bytes / (1024.0 * 1024.0)) / + ((double) elapsed / 1000000000.0)); + cmt_destroy(cmt); + return 0; +} + +static int benchmark_opentelemetry_mixed(size_t cardinality, size_t operations) +{ + size_t index; + size_t bytes; + uint64_t start; + uint64_t elapsed; + cfl_sds_t output; + struct cmt *cmt; + + bytes = 0; + cmt = cmt_create(); + if (cmt == NULL || create_mixed_series(cmt, cardinality) != 0) { + cmt_destroy(cmt); + return -1; + } + + start = monotonic_ns(); + for (index = 0; index < operations; index++) { + output = cmt_encode_opentelemetry_create(cmt); + if (output == NULL) { + cmt_destroy(cmt); + return -1; + } + bytes += cfl_sds_len(output); + cmt_encode_opentelemetry_destroy(output); + } + elapsed = monotonic_ns() - start; + + printf("benchmark=opentelemetry-mixed cardinality=%zu operations=%zu " + "bytes=%zu elapsed_ns=%" PRIu64 " ns_per_op=%.2f " + "mb_per_second=%.2f\n", + cardinality, operations, bytes, elapsed, + (double) elapsed / operations, + ((double) bytes / (1024.0 * 1024.0)) / + ((double) elapsed / 1000000000.0)); + cmt_destroy(cmt); + return 0; +} + +int main(int argc, char **argv) +{ + size_t cardinality; + size_t operations; + + if (argc != 4) { + fprintf(stderr, "usage: %s lookup|update|prometheus|opentelemetry|" + "opentelemetry-mixed " + "CARDINALITY OPERATIONS\n", + argv[0]); + return EXIT_FAILURE; + } + + cardinality = parse_size(argv[2], "cardinality"); + operations = parse_size(argv[3], "operations"); + cmt_initialize(); + + if (strcmp(argv[1], "lookup") == 0) { + return benchmark_lookup(cardinality, operations) == 0 ? + EXIT_SUCCESS : EXIT_FAILURE; + } + if (strcmp(argv[1], "update") == 0) { + return benchmark_update(cardinality, operations) == 0 ? + EXIT_SUCCESS : EXIT_FAILURE; + } + if (strcmp(argv[1], "prometheus") == 0) { + return benchmark_prometheus(cardinality, operations) == 0 ? + EXIT_SUCCESS : EXIT_FAILURE; + } + if (strcmp(argv[1], "opentelemetry") == 0) { + return benchmark_opentelemetry(cardinality, operations) == 0 ? + EXIT_SUCCESS : EXIT_FAILURE; + } + if (strcmp(argv[1], "opentelemetry-mixed") == 0) { + return benchmark_opentelemetry_mixed(cardinality, operations) == 0 ? + EXIT_SUCCESS : EXIT_FAILURE; + } + + fprintf(stderr, "unknown benchmark: %s\n", argv[1]); + return EXIT_FAILURE; +} diff --git a/lib/cmetrics/benchmarks/run-perf.sh b/lib/cmetrics/benchmarks/run-perf.sh new file mode 100755 index 00000000000..49a7b6bbdc6 --- /dev/null +++ b/lib/cmetrics/benchmarks/run-perf.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env sh +set -eu + +benchmark=${1:-./build/benchmarks/cmt-benchmark} +repetitions=${REPETITIONS:-5} + +run_repeated() +{ + mode=$1 + cardinality=$2 + operations=$3 + iteration=1 + + while [ "$iteration" -le "$repetitions" ]; do + "$benchmark" "$mode" "$cardinality" "$operations" + iteration=$((iteration + 1)) + done +} + +run_repeated lookup 5000 100000 +run_repeated update 5000 100000 +run_repeated update 1 5000000 +run_repeated prometheus 5000 100 +run_repeated opentelemetry 5000 100 +run_repeated opentelemetry-mixed 2000 100 + +perf stat \ + -e cycles,instructions,branches,branch-misses,cache-misses \ + -r "$repetitions" \ + "$benchmark" lookup 5000 100000 + +perf stat \ + -e task-clock,cycles,instructions,branches,branch-misses,cache-misses \ + -r "$repetitions" \ + "$benchmark" opentelemetry 5000 500 diff --git a/lib/cmetrics/dockerfiles/Dockerfile.centos7 b/lib/cmetrics/dockerfiles/Dockerfile.centos7 new file mode 100644 index 00000000000..7793c4ec0b7 --- /dev/null +++ b/lib/cmetrics/dockerfiles/Dockerfile.centos7 @@ -0,0 +1,22 @@ +# This image is used only to verify that cmetrics builds on CentOS 7. +FROM centos:7 + +RUN sed -i -e "s/^mirrorlist=http:\/\/mirrorlist.centos.org/#mirrorlist=http:\/\/mirrorlist.centos.org/g" /etc/yum.repos.d/CentOS-Base.repo && \ + sed -i -e "s/^#baseurl=http:\/\/mirror.centos.org/baseurl=http:\/\/vault.centos.org/g" /etc/yum.repos.d/CentOS-Base.repo && \ + yum -y update && \ + yum install -y ca-certificates gcc gcc-c++ make wget libcurl-devel && \ + yum clean all + +ARG CMAKE_VERSION="3.20.0" + +RUN wget -q "https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-linux-x86_64.sh" \ + -O /tmp/cmake-installer.sh && \ + chmod +x /tmp/cmake-installer.sh && \ + /tmp/cmake-installer.sh --skip-license --prefix=/usr/local && \ + rm -f /tmp/cmake-installer.sh + +COPY . /src/ +WORKDIR /src/build + +RUN cmake -DCMT_TESTS=On -DCMT_DEV=On .. && \ + cmake --build . -j "$(getconf _NPROCESSORS_ONLN)" diff --git a/lib/cmetrics/docs/ai/bug-fix.md b/lib/cmetrics/docs/ai/bug-fix.md new file mode 100644 index 00000000000..c32c8cb8228 --- /dev/null +++ b/lib/cmetrics/docs/ai/bug-fix.md @@ -0,0 +1,42 @@ +# Bug-fix workflow + +## Purpose + +Produce a minimal, regression-tested fix with explicit compatibility and memory +ownership review. + +## When to use + +Use after an observed defect has enough evidence to justify changing CMetrics. + +## Investigation + +1. Restate observed behavior and separate facts from assumptions. +2. Identify the owning subsystem and repository. +3. Trace the relevant public entry point, internal path, and cleanup path. +4. Locate existing tests and reproduce the defect when practical. + +## Implementation + +1. Add a focused regression test that fails for the reported reason. +2. Implement the smallest reasonable fix; avoid unrelated refactoring. +3. Check allocation failure, partial initialization, cleanup, concurrency, and + malformed-input behavior where relevant. +4. Review public headers and encoded formats for compatibility impact. + +## Validation + +Run the focused CTest target, then `scripts/agent-verify.sh`. Ownership fixes +also require the applicable checks in `memory-safety-review.md`. Format changes +require round-trip and malformed-input coverage. + +## Expected report + +Report root cause, fix, regression test, exact commands/results, compatibility +impact, dependency impact, and unresolved risks. + +## Stop conditions + +Stop when the intended behavior requires a maintainer decision, the fix belongs +in a dependency, a public or wire contract must change without agreement, or a +safe reproducer cannot be created. diff --git a/lib/cmetrics/docs/ai/code-review.md b/lib/cmetrics/docs/ai/code-review.md new file mode 100644 index 00000000000..b8af87b55d4 --- /dev/null +++ b/lib/cmetrics/docs/ai/code-review.md @@ -0,0 +1,36 @@ +# Code-review workflow + +## Purpose + +Find actionable defects and regression risks in a proposed change. + +## When to use + +Use for pull requests, local diffs, dependency updates, and pre-release audits. + +## Investigation + +1. Establish the diff base, intended behavior, and affected public entry points. +2. Read surrounding code and tests; verify findings against the current tree. +3. Prioritize correctness, memory/resource ownership, cleanup paths, + concurrency, compatibility, malformed input, missing tests, regressions, + unnecessary complexity, and measurable performance impact. +4. Trace callers before claiming a nullability, lifetime, or locking defect. +5. Distinguish confirmed findings from questions and optional improvements. + +## Validation + +Run focused tests that can confirm or reject each high-confidence finding. For +codec changes, inspect both encoding and decoding and relevant format +conversions. Use sanitizer or benchmark evidence where the claim depends on it. + +## Expected report + +List findings by severity with file/line, failure scenario, evidence, and the +smallest corrective action. Then list validation gaps and a short overall risk +assessment. Do not bury findings in a general summary. + +## Stop conditions + +Do not request speculative changes without a concrete failure mode. Escalate +public API, ABI, wire-format, or cross-repository policy decisions. diff --git a/lib/cmetrics/docs/ai/cross-repository.md b/lib/cmetrics/docs/ai/cross-repository.md new file mode 100644 index 00000000000..5d2e98ec9c7 --- /dev/null +++ b/lib/cmetrics/docs/ai/cross-repository.md @@ -0,0 +1,44 @@ +# Cross-repository workflow + +## Purpose + +Coordinate changes whose behavior or validation spans CMetrics, its submodules, +or a downstream consumer. + +## When to use + +Use for CFL or fluent-otel-proto changes, CMetrics submodule updates, and +consumer integrations such as Fluent Bit. + +## Investigation + +1. Identify which repository owns the behavior and which repositories consume + its API, ABI, generated code, or wire output. +2. Determine compatibility impact and the required landing order. +3. Record exact base branches, revisions, and temporary dependency revisions. +4. Reproduce in the owning repository before changing the consumer when + practical. + +## Implementation + +1. Keep separate commits and pull requests per repository. +2. Land the owning-library fix first unless a coordinated transition is needed. +3. Update submodule or vendored revisions in a focused consumer change. +4. Avoid copying an unmerged implementation into multiple repositories. + +## Validation + +Run standalone validation in the owning repository. Then build and test each +consumer against the exact dependency branch/revision, including its relevant +integration and memory-safety jobs. + +## Expected report + +Report ownership, dependency direction, revisions, landing order, standalone +results, consumer integration results, and temporary coordination steps. + +## Stop conditions + +Stop when a required repository or CI environment is unavailable, the landing +order could break a supported branch, or ownership/compatibility policy is +unclear. diff --git a/lib/cmetrics/docs/ai/dependency-update.md b/lib/cmetrics/docs/ai/dependency-update.md new file mode 100644 index 00000000000..3a21bcad621 --- /dev/null +++ b/lib/cmetrics/docs/ai/dependency-update.md @@ -0,0 +1,40 @@ +# Dependency-update workflow + +## Purpose + +Update a Git submodule revision without mixing dependency implementation changes +into CMetrics. + +## When to use + +Use for `lib/cfl` and `lib/fluent-otel-proto` revisions. + +## Investigation + +1. Review `.gitmodules`, the current recorded revision, candidate commits/tags, + and the dependency's release notes or diff. +2. Identify public API, generated protobuf, allocator, threading, compiler, and + platform changes consumed by CMetrics. +3. Confirm the candidate revision exists in the dependency's upstream remote. + +## Implementation + +Update only the gitlink unless CMetrics requires a separate adaptation. Keep +adaptations explicit and avoid editing the submodule as uncommitted content. + +## Validation + +Run `scripts/agent-verify.sh`, relevant codec tests, and compiler/platform checks +affected by the dependency. For memory or performance changes, follow the +specialized workflows. Validate downstream consumers after CMetrics passes. + +## Expected report + +Report old/new revisions, included upstream changes, CMetrics adaptations, +standalone results, downstream results, and landing order. + +## Stop conditions + +Stop if the revision is not available upstream, includes unexplained generated +changes, requires an undocumented compatibility break, or cannot be tested in +the owning dependency first. diff --git a/lib/cmetrics/docs/ai/investigate.md b/lib/cmetrics/docs/ai/investigate.md new file mode 100644 index 00000000000..4568976a1b5 --- /dev/null +++ b/lib/cmetrics/docs/ai/investigate.md @@ -0,0 +1,42 @@ +# Investigation workflow + +## Purpose + +Establish an evidence-backed explanation of CMetrics behavior without assuming +that a code change is required. + +## When to use + +Use for bug reports, unexpected codec output, crashes, compatibility questions, +and performance reports before selecting a fix. + +## Investigation + +1. Restate the observed and expected behavior, including platform and input. +2. Separate reproduced facts, repository evidence, and working assumptions. +3. Identify the owning layer: CMetrics, CFL, fluent-otel-proto, generated code, + or a downstream consumer. +4. Trace from the public entry point through allocation, mutation, codec, and + cleanup paths. Include error exits and ownership transfers. +5. Locate focused tests and recent changes in the same subsystem. +6. Reproduce with the smallest representative input when practical. +7. For malformed input, retain a non-sensitive reproducer or describe its exact + construction rather than relying on an external artifact. + +## Validation + +Run the closest existing CTest target. Use a sanitizer, Valgrind, or benchmarks +only when relevant to the reported behavior. Do not infer results from CI job +names. + +## Expected report + +Report reproduction status, owning subsystem/repository, traced code path, +root-cause confidence, affected versions or formats when known, and remaining +unknowns. Propose a minimal fix and test plan separately. + +## Stop conditions + +Stop and escalate when the reproducer depends on unavailable private data, the +behavior belongs to another repository, the expected contract is ambiguous, or +a compatibility decision is required before implementation. diff --git a/lib/cmetrics/docs/ai/memory-safety-review.md b/lib/cmetrics/docs/ai/memory-safety-review.md new file mode 100644 index 00000000000..ac37fce80e4 --- /dev/null +++ b/lib/cmetrics/docs/ai/memory-safety-review.md @@ -0,0 +1,39 @@ +# Memory-safety review + +## Purpose + +Validate allocation, ownership, lifetime, and cleanup changes in this C library. + +## When to use + +Use for constructors/destructors, containers, codecs, arenas, metadata, +submodule allocator changes, and crash or leak reports. + +## Investigation + +1. Inventory allocations and their matching destructors by allocation family. +2. Trace ownership transfers, shared/borrowed pointers, arena lifetime, and + partial initialization. +3. Audit every normal and error exit, including empty collections and nested + values. +4. Check concurrent mutation/destruction and untrusted length arithmetic. +5. Add focused zero/one/many and failure-path regression cases where practical. + +## Validation + +First run the focused test normally. For AddressSanitizer, use a separate build +with compiler/linker sanitizer flags and execute CTest with leak detection. For +Valgrind, use a non-sanitized debug build and treat definite leaks/errors as +failures. Record the exact compiler, flags, commands, and test count; sanitizer +and Valgrind runs are complementary, not interchangeable. + +## Expected report + +Report the ownership model, confirmed defect, cleanup coverage, sanitizer and +Valgrind results, allocation-failure coverage, and untested paths. + +## Stop conditions + +Stop when ownership cannot be established from callers, allocator injection is +required but unavailable, or a proposed lifetime change affects a public +structure without compatibility review. diff --git a/lib/cmetrics/docs/ai/performance-review.md b/lib/cmetrics/docs/ai/performance-review.md new file mode 100644 index 00000000000..a6c33d69141 --- /dev/null +++ b/lib/cmetrics/docs/ai/performance-review.md @@ -0,0 +1,37 @@ +# Performance-review workflow + +## Purpose + +Accept only repeatable CMetrics performance improvements without correctness or +workload regressions. + +## When to use + +Use for map lookup/update, codec allocation, cardinality, and hot-path changes. + +## Investigation and implementation + +1. Identify the hot path and representative existing benchmark workload. +2. Establish a correctness baseline and a performance baseline before editing. +3. Change one material factor at a time and preserve ownership/concurrency rules. +4. Add a benchmark workload only when existing modes cannot represent the path. + +## Validation + +Follow `benchmarks/README.md`. Build Release with `CMT_BENCHMARKS=ON`; run at +least five alternating before/after samples on the same idle machine with the +same compiler, flags, CPU policy, cardinality, and operations. Compare medians, +variability, allocations when relevant, and `perf stat` counters. Run +`scripts/agent-verify.sh` for correctness. + +## Expected report + +Report revisions, hardware/software environment, exact commands, raw samples, +medians, percentage changes, counter changes, correctness results, and affected +workloads. Explain any regression. + +## Stop conditions + +Do not ship noise-level, single-run, differently configured, or correctness- +regressing results. Escalate tradeoffs that improve one supported workload while +materially degrading another. diff --git a/lib/cmetrics/docs/architecture.md b/lib/cmetrics/docs/architecture.md new file mode 100644 index 00000000000..179eea88e96 --- /dev/null +++ b/lib/cmetrics/docs/architecture.md @@ -0,0 +1,48 @@ +# CMetrics architecture + +CMetrics is a static C library that owns metric contexts and converts them to +and from several metrics protocols. The installed API is declared under +`include/cmetrics/`; implementations live under `src/`. + +## Core model + +`struct cmt` is the top-level context. Metric-family modules implement counters, +gauges, untyped metrics, summaries, histograms, and exponential histograms. +Each family owns a `struct cmt_map`, whose static or labeled datapoints are +represented by `struct cmt_metric`. Labels, options, timestamps, values, and +family-specific storage are shared by codecs and filters. + +The main entry points are: + +- `cmetrics.c`: context initialization and destruction. +- `cmt_map.c`, `cmt_metric.c`: datapoint lookup, storage, indexing, expiration, + and value representation. +- `cmt_.c`: metric-family creation and mutation. +- `cmt_cat.c`, `cmt_filter.c`: context combination and selection. +- `cmt_encode_*.c`, `cmt_decode_*.c`: protocol boundaries. + +## Protocol boundaries + +OTLP uses generated protobuf definitions supplied by fluent-otel-proto. +Prometheus remote write uses generated protobuf-C files in the repository. +Prometheus text decoding uses Flex/Bison sources and build-directory generated +parsers. CMetrics MessagePack is an internal serialized representation used by +format-conversion and downstream flows. + +Changes at these boundaries should preserve metric identity, label ordering, +numeric type, timestamps, aggregation fields, and decoder synchronization unless +the format contract intentionally changes. Tests under `tests/encoding.c`, +`tests/decoding.c`, `tests/opentelemetry.c`, `tests/format_conversion.c`, and +the Prometheus-specific test files cover these paths. + +## Ownership and concurrency + +Metric families own their maps; maps own dynamic metrics and label storage. +Some encoders create temporary heap or arena-backed protobuf structures before +packing them into an SDS result. Allocation family and lifetime must remain +consistent across success and partial-initialization cleanup. + +Map lookup, mutation, indexing, expiration, and destruction share internal +state and must be reviewed together for concurrent access. Public structures in +installed headers also constrain internal layout changes because downstream C +code can compile against them. diff --git a/lib/cmetrics/docs/dependencies.md b/lib/cmetrics/docs/dependencies.md new file mode 100644 index 00000000000..101f333eb16 --- /dev/null +++ b/lib/cmetrics/docs/dependencies.md @@ -0,0 +1,34 @@ +# Repository dependencies + +## Build dependencies + +- CMake 3.20 or newer configures the project. +- A platform C compiler builds the static library and tests. +- Flex 2 and Bison 3 generate the optional Prometheus text decoder. +- CTest runs the Acutest executables registered by `tests/CMakeLists.txt`. +- Linux `perf` is required only for the standard hardware-counter benchmarks. + +## Repository relationships + +CMetrics records two Git submodules: + +- `lib/cfl` → `fluent/cfl`: containers, SDS strings, variants, arenas, hashes, + atomic helpers, and other foundational C utilities. +- `lib/fluent-otel-proto` → `fluent/fluent-otel-proto`: generated OpenTelemetry + protobuf-C definitions and runtime integration used by the OTLP codec. + +The top-level build can use system-detected copies of these libraries; otherwise +it builds the recorded submodules. Behavior owned by either dependency should +be fixed and validated there before updating the CMetrics gitlink. + +Fluent Bit is an evidenced downstream consumer of CMetrics. It is not part of +this source tree, so consumer integration validation must use a separate Fluent +Bit checkout with the intended CMetrics revision. A normal landing order is: + +1. Land and validate a dependency change in its owning repository. +2. Update and validate CMetrics against that dependency revision. +3. Update the CMetrics revision or bundled copy in the downstream consumer. +4. Run the consumer's focused integration tests and applicable CI checks. + +Keep commits and pull requests separate per repository so each project can be +built, reviewed, and reverted independently. diff --git a/lib/cmetrics/docs/label-value-handling.md b/lib/cmetrics/docs/label-value-handling.md new file mode 100644 index 00000000000..1f5413a4f62 --- /dev/null +++ b/lib/cmetrics/docs/label-value-handling.md @@ -0,0 +1,117 @@ +# Long metric label handling + +## Context + +Before this fix, CMetrics rejected any string longer than 1024 bytes while +decoding its internal MessagePack representation. The validation was performed +by the generic string decoder, so it applied not only to label values, but also +to metric names, namespaces, subsystems, descriptions, and label names. + +This behavior caused +[fluent/fluent-bit#9297](https://github.com/fluent/fluent-bit/issues/9297): a +valid Prometheus scrape containing a `process_command_line` label longer than +1024 bytes was accepted by the Prometheus parser, but failed during the +subsequent CMetrics MessagePack round trip. The failure discarded all metrics +from the scrape without a useful diagnostic. + +[PR #224](https://github.com/fluent/cmetrics/pull/224) proposed retaining the +first 1024 bytes and appending `...` during MessagePack decoding. The issue is +valid, but that behavior should not be implemented in the generic decoder. + +## Why silent truncation is unsafe + +Prometheus identifies a time series using its metric name and complete label +set. Consider two label values with the same 1024-byte prefix: + +```text +process_command_line="A" +process_command_line="B" +``` + +Blindly replacing both suffixes with `...` gives both samples the same series +identity. This can merge unrelated series and produce incorrect results. + +Truncating at a fixed byte offset can also split a multi-byte UTF-8 character. +Because the generic MessagePack helper decodes every CMetrics string, the same +policy could silently alter metric names, descriptions, and label names. + +Presentation requirements must not be implemented by mutating the internal +data model during deserialization. An output encoder may abbreviate a value for +display, but the stored value must remain unchanged. + +## Ecosystem behavior + +There is no universal 1024-byte limit for Prometheus label values: + +- Prometheus accepts label values without a length limit by default. When a + `label_value_length_limit` is configured, one violation fails the scrape + instead of silently rewriting the value. +- VictoriaMetrics provides a configurable label-value limit. It ignores an + oversized series and reports the event through logs and an internal metric. +- Grafana Mimir provides configurable `error`, `drop`, and `truncate` + strategies. Its truncation strategy includes a hash of the original value so + that values with a common prefix remain distinct. +- OpenTelemetry metric attributes are currently exempt from the general SDK + attribute-length limits. + +References: + +- [Prometheus data model](https://prometheus.io/docs/concepts/data_model/) +- [Prometheus scrape configuration](https://prometheus.io/docs/prometheus/latest/configuration/configuration/) +- [VictoriaMetrics ingestion limits](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) +- [Grafana Mimir configuration](https://grafana.com/docs/mimir/latest/configure/configuration-parameters/) +- [OpenTelemetry attribute limits](https://opentelemetry.io/docs/specs/otel/common/#attribute-limits) + +## Recommended CMetrics behavior + +The internal MessagePack decoder should: + +1. Decode valid strings losslessly by default. +2. Treat resource limits separately from string contents and metric semantics. +3. Prevent integer overflow when calculating allocation sizes. +4. Bound allocations made from untrusted length fields. +5. Return a distinct resource-limit error with enough context for callers to + emit a useful diagnostic. +6. Never silently truncate metric identifiers or labels. + +This fix implements those requirements for data-backed CMetrics MessagePack +decoding. It verifies that all declared string bytes are present before making +an allocation, then copies the complete value into CMetrics-owned storage. A +malicious length field therefore cannot trigger an allocation larger than its +available input, while valid long strings round-trip without modification. + +A configurable safety ceiling may be appropriate for callers that process +untrusted MessagePack. Such a ceiling should apply to decoder resources, be +documented in bytes, and reject input explicitly. It must not reinterpret an +oversized string as valid data with different contents. + +If Fluent Bit or another caller needs an ingestion policy, it should be +implemented above the MessagePack decoder. Useful policies are: + +- `preserve`: retain the complete value; this matches Prometheus defaults. +- `reject`: reject the scrape or batch with a diagnostic. +- `drop`: discard only the offending series and increment an error counter. +- `truncate_hash`: truncate on a valid UTF-8 boundary and append a hash derived + from the complete value to preserve series identity as far as practical. + +The selected policy and limit should be configurable by the component that +owns ingestion, because CMetrics is also used with OTLP and other formats whose +requirements differ. + +## Regression coverage + +An implementation should include tests for: + +- Values of 1023, 1024, 1025, 2048, and 65536 bytes. +- Two values with an identical 1024-byte prefix and different suffixes. +- UTF-8 characters crossing any configured boundary. +- Long metric names, descriptions, label names, and label values independently. +- Multiple fields and metrics after a long string, proving that the MessagePack + reader remains synchronized. +- A declared string length larger than the available input. +- A declared length near the integer and configured allocation limits. +- Allocation failure and cleanup paths. +- End-to-end Prometheus scrape, CMetrics MessagePack encode/decode, and output. + +Memory-safety validation should include AddressSanitizer, UndefinedBehaviorSanitizer, +and Valgrind in addition to the focused unit tests. diff --git a/lib/cmetrics/include/cmetrics/cmetrics.h b/lib/cmetrics/include/cmetrics/cmetrics.h index 4b5307846c5..04638c77754 100644 --- a/lib/cmetrics/include/cmetrics/cmetrics.h +++ b/lib/cmetrics/include/cmetrics/cmetrics.h @@ -80,5 +80,6 @@ struct cmt *cmt_create(); void cmt_destroy(struct cmt *cmt); int cmt_label_add(struct cmt *cmt, char *key, char *val); char *cmt_version(); +void cmt_expire(struct cmt *cmt, uint64_t expiration); #endif diff --git a/lib/cmetrics/include/cmetrics/cmt_decode_prometheus.h b/lib/cmetrics/include/cmetrics/cmt_decode_prometheus.h index a565a4715b0..cd0bfb0efe7 100644 --- a/lib/cmetrics/include/cmetrics/cmt_decode_prometheus.h +++ b/lib/cmetrics/include/cmetrics/cmt_decode_prometheus.h @@ -68,6 +68,7 @@ struct cmt_decode_prometheus_context_metric { size_t label_count; cfl_sds_t labels[CMT_DECODE_PROMETHEUS_MAX_LABEL_COUNT]; struct cfl_list samples; + char *name_buf; }; struct cmt_decode_prometheus_parse_opts { diff --git a/lib/cmetrics/include/cmetrics/cmt_map.h b/lib/cmetrics/include/cmetrics/cmt_map.h index f67ee15547b..6a3c7e79620 100644 --- a/lib/cmetrics/include/cmetrics/cmt_map.h +++ b/lib/cmetrics/include/cmetrics/cmt_map.h @@ -43,12 +43,23 @@ struct cmt_map { int label_count; /* Number of labels */ struct cfl_list label_keys; /* Linked list of labels */ void *parent; + + /* Internal lock. Keep this after the established public fields. */ + uint64_t metric_lock; + struct cfl_list *metric_buckets; + size_t metric_bucket_count; + size_t indexed_metric_count; + /* Most recently created metric; only changed with the metric list. */ + struct cmt_metric *last_metric; }; struct cmt_map *cmt_map_create(int type, struct cmt_opts *opts, int count, char **labels, void *parent); void cmt_map_destroy(struct cmt_map *map); +/* Concurrent lookups and metric creation are serialized internally. The + * returned metric remains caller-usable only while the map is not expired or + * destroyed. */ struct cmt_metric *cmt_map_metric_get(struct cmt_opts *opts, struct cmt_map *map, int labels_count, char **labels_val, int write_op); @@ -57,6 +68,10 @@ int cmt_map_metric_get_val(struct cmt_opts *opts, struct cmt_map *map, double *out_val); void cmt_map_metric_destroy(struct cmt_metric *metric); +/* Expiration requires external coordination with metric users. */ +void cmt_map_metrics_expire(struct cmt_map *, uint64_t); + void destroy_label_list(struct cfl_list *label_list); + #endif diff --git a/lib/cmetrics/include/cmetrics/cmt_metric.h b/lib/cmetrics/include/cmetrics/cmt_metric.h index 35ca44f8407..a45eaba5b50 100644 --- a/lib/cmetrics/include/cmetrics/cmt_metric.h +++ b/lib/cmetrics/include/cmetrics/cmt_metric.h @@ -70,6 +70,11 @@ struct cmt_metric { uint64_t exp_hist_lock; struct cfl_list labels; struct cfl_list _head; + + /* Internal lookup index. Keep these after the established public fields. */ + int hash_indexed; + struct cmt_map *map; + struct cfl_list _hash_head; }; struct cmt_exp_histogram_snapshot { diff --git a/lib/cmetrics/include/cmetrics/cmt_mpack_utils_defs.h b/lib/cmetrics/include/cmetrics/cmt_mpack_utils_defs.h index 361886c06af..19eabd101eb 100644 --- a/lib/cmetrics/include/cmetrics/cmt_mpack_utils_defs.h +++ b/lib/cmetrics/include/cmetrics/cmt_mpack_utils_defs.h @@ -35,6 +35,5 @@ #define CMT_MPACK_MAX_ARRAY_ENTRY_COUNT 65535 #define CMT_MPACK_MAX_MAP_ENTRY_COUNT 10 -#define CMT_MPACK_MAX_STRING_LENGTH 1024 -#endif \ No newline at end of file +#endif diff --git a/lib/cmetrics/scripts/agent-build.sh b/lib/cmetrics/scripts/agent-build.sh new file mode 100755 index 00000000000..db001d0177b --- /dev/null +++ b/lib/cmetrics/scripts/agent-build.sh @@ -0,0 +1,14 @@ +#!/bin/sh +set -eu + +repository_root=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) +build_dir=${BUILD_DIR:-"$repository_root/build/agent"} + +git -C "$repository_root" submodule update --init --recursive + +cmake -S "$repository_root" -B "$build_dir" \ + -DCMT_TESTS=On \ + -DCMT_INSTALL_TARGETS=Off \ + "$@" + +cmake --build "$build_dir" diff --git a/lib/cmetrics/scripts/agent-test.sh b/lib/cmetrics/scripts/agent-test.sh new file mode 100755 index 00000000000..3515652963a --- /dev/null +++ b/lib/cmetrics/scripts/agent-test.sh @@ -0,0 +1,21 @@ +#!/bin/sh +set -eu + +repository_root=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) +build_dir=${BUILD_DIR:-"$repository_root/build/agent"} + +if [ ! -f "$build_dir/CTestTestfile.cmake" ]; then + echo "error: $build_dir is not configured for tests; run scripts/agent-build.sh first" >&2 + exit 1 +fi + +if [ "$#" -gt 1 ]; then + echo "usage: $0 [ctest-regular-expression]" >&2 + exit 2 +fi + +if [ "$#" -eq 1 ]; then + ctest --test-dir "$build_dir" --output-on-failure -R "$1" +else + ctest --test-dir "$build_dir" --output-on-failure +fi diff --git a/lib/cmetrics/scripts/agent-verify.sh b/lib/cmetrics/scripts/agent-verify.sh new file mode 100755 index 00000000000..ce4af194e03 --- /dev/null +++ b/lib/cmetrics/scripts/agent-verify.sh @@ -0,0 +1,11 @@ +#!/bin/sh +set -eu + +repository_root=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) + +for script in "$repository_root"/scripts/agent-*.sh; do + sh -n "$script" +done + +"$repository_root/scripts/agent-build.sh" "$@" +"$repository_root/scripts/agent-test.sh" diff --git a/lib/cmetrics/src/cmetrics.c b/lib/cmetrics/src/cmetrics.c index a2bb0ffd1b3..025cd99032f 100644 --- a/lib/cmetrics/src/cmetrics.c +++ b/lib/cmetrics/src/cmetrics.c @@ -30,6 +30,7 @@ #include #include #include +#include #include #include @@ -144,6 +145,61 @@ void cmt_destroy(struct cmt *cmt) free(cmt); } +void cmt_expire(struct cmt *cmt, uint64_t expiration) +{ + struct cfl_list *tmp; + struct cfl_list *head; + struct cmt_counter *counter; + struct cmt_gauge *gauge; + struct cmt_summary *summary; + struct cmt_histogram *histogram; + struct cmt_untyped *untyped; + struct cmt_exp_histogram *exp_histogram; + + if (cmt == NULL) { + return; + } + + /* Do a first pass for all regular metrics: + * * counters + * * gauges + * * summaries + * * histograms + * * untypeds + */ + cfl_list_foreach_safe(head, tmp, &cmt->counters) { + counter = cfl_list_entry(head, struct cmt_counter, _head); + cmt_map_metrics_expire(counter->map, expiration); + } + + cfl_list_foreach_safe(head, tmp, &cmt->gauges) { + gauge = cfl_list_entry(head, struct cmt_gauge, _head); + cmt_map_metrics_expire(gauge->map, expiration); + } + + cfl_list_foreach_safe(head, tmp, &cmt->summaries) { + summary = cfl_list_entry(head, struct cmt_summary, _head); + cmt_map_metrics_expire(summary->map, expiration); + } + + cfl_list_foreach_safe(head, tmp, &cmt->histograms) { + histogram = cfl_list_entry(head, struct cmt_histogram, _head); + cmt_map_metrics_expire(histogram->map, expiration); + } + + cfl_list_foreach_safe(head, tmp, &cmt->untypeds) { + untyped = cfl_list_entry(head, struct cmt_untyped, _head); + cmt_map_metrics_expire(untyped->map, expiration); + } + + /* Here we cover exp_histograms separetely. + */ + cfl_list_foreach_safe(head, tmp, &cmt->exp_histograms) { + exp_histogram = cfl_list_entry(head, struct cmt_exp_histogram, _head); + cmt_map_metrics_expire(exp_histogram->map, expiration); + } +} + int cmt_label_add(struct cmt *cmt, char *key, char *val) { return cmt_labels_add_kv(cmt->static_labels, key, val); diff --git a/lib/cmetrics/src/cmt_cat.c b/lib/cmetrics/src/cmt_cat.c index bd01fc01f81..0e3f0841955 100644 --- a/lib/cmetrics/src/cmt_cat.c +++ b/lib/cmetrics/src/cmt_cat.c @@ -658,6 +658,50 @@ static struct cmt_exp_histogram *exp_histogram_lookup(struct cmt *cmt, struct cm return NULL; } +static struct cmt_summary *summary_lookup(struct cmt *cmt, struct cmt_opts *opts) +{ + struct cmt_summary *summary; + struct cfl_list *head; + + cfl_list_foreach(head, &cmt->summaries) { + summary = cfl_list_entry(head, struct cmt_summary, _head); + if (cmt_opts_compare(&summary->opts, opts) == 0) { + return summary; + } + } + + return NULL; +} + +static int summary_label_keys_match(struct cmt_map *left, + struct cmt_map *right) +{ + struct cfl_list *left_head; + struct cfl_list *right_head; + struct cmt_map_label *left_label; + struct cmt_map_label *right_label; + + left_head = left->label_keys.next; + right_head = right->label_keys.next; + + while (left_head != &left->label_keys && + right_head != &right->label_keys) { + left_label = cfl_list_entry(left_head, struct cmt_map_label, _head); + right_label = cfl_list_entry(right_head, struct cmt_map_label, _head); + + if (left_label->name == NULL || right_label->name == NULL || + strcmp(left_label->name, right_label->name) != 0) { + return CMT_FALSE; + } + + left_head = left_head->next; + right_head = right_head->next; + } + + return left_head == &left->label_keys && + right_head == &right->label_keys; +} + int cmt_cat_counter(struct cmt *cmt, struct cmt_counter *counter, struct cmt_map *filtered_map) { @@ -856,48 +900,65 @@ int cmt_cat_histogram(struct cmt *cmt, struct cmt_histogram *histogram, int cmt_cat_summary(struct cmt *cmt, struct cmt_summary *summary, struct cmt_map *filtered_map) { - int i; + size_t i; int ret; char **labels = NULL; struct cmt_map *map; struct cmt_opts *opts; struct cmt_summary *sum; double *quantiles; - uint64_t timestamp; - double summary_sum; map = summary->map; opts = map->opts; - timestamp = cmt_metric_get_timestamp(&map->metric); - ret = cmt_cat_copy_label_keys(map, (char **) &labels); if (ret == -1) { return -1; } - quantiles = calloc(1, sizeof(double) * summary->quantiles_count); - for (i = 0; i < summary->quantiles_count; i++) { - quantiles[i] = summary->quantiles[i]; + sum = summary_lookup(cmt, opts); + if (sum != NULL) { + if (!summary_label_keys_match(sum->map, map)) { + free(labels); + return -1; + } + + if (sum->quantiles_count != summary->quantiles_count) { + free(labels); + return -1; + } + + for (i = 0; i < summary->quantiles_count; i++) { + if (sum->quantiles[i] != summary->quantiles[i]) { + free(labels); + return -1; + } + } } + else { + quantiles = NULL; + if (summary->quantiles_count > 0) { + quantiles = calloc(summary->quantiles_count, sizeof(double)); + if (quantiles == NULL) { + free(labels); + return -1; + } + memcpy(quantiles, summary->quantiles, + summary->quantiles_count * sizeof(double)); + } - /* create summary */ - sum = cmt_summary_create(cmt, - opts->ns, opts->subsystem, - opts->name, opts->description, - summary->quantiles_count, - quantiles, - map->label_count, labels); - if (!sum) { - free(labels); + sum = cmt_summary_create(cmt, + opts->ns, opts->subsystem, + opts->name, opts->description, + summary->quantiles_count, + quantiles, + map->label_count, labels); free(quantiles); - return -1; } - summary_sum = cmt_summary_get_sum_value(&summary->map->metric); - - cmt_summary_set_default(sum, timestamp, quantiles, summary_sum, summary->quantiles_count, map->label_count, labels); free(labels); - free(quantiles); + if (!sum) { + return -1; + } if (filtered_map != NULL) { ret = cmt_cat_copy_map(&sum->opts, sum->map, filtered_map); diff --git a/lib/cmetrics/src/cmt_decode_msgpack.c b/lib/cmetrics/src/cmt_decode_msgpack.c index 80256c20060..70fab9e879d 100644 --- a/lib/cmetrics/src/cmt_decode_msgpack.c +++ b/lib/cmetrics/src/cmt_decode_msgpack.c @@ -1336,6 +1336,8 @@ static int unpack_metric_array_entry(mpack_reader_t *reader, size_t index, void else if (decode_context->map->type == CMT_SUMMARY) { cmt_atomic_store(&decode_context->map->metric.sum_quantiles_set, cmt_atomic_load(&metric->sum_quantiles_set)); decode_context->map->metric.sum_quantiles = metric->sum_quantiles; + decode_context->map->metric.sum_quantiles_count = + metric->sum_quantiles_count; cmt_atomic_store(&decode_context->map->metric.sum_count, cmt_atomic_load(&metric->sum_count)); cmt_atomic_store(&decode_context->map->metric.sum_sum, @@ -1636,6 +1638,7 @@ static int unpack_basic_type_meta(mpack_reader_t *reader, size_t index, void *co int result; struct cmt_summary *summary; struct cmt_histogram *histogram; + struct cmt_exp_histogram *exp_histogram; struct cmt_counter *counter; struct cmt_msgpack_decode_context *decode_context; struct cmt_mpack_map_entry_callback_t callbacks[] = \ @@ -1684,6 +1687,12 @@ static int unpack_basic_type_meta(mpack_reader_t *reader, size_t index, void *co else { histogram->buckets = NULL; } + + histogram->aggregation_type = decode_context->aggregation_type; + } + else if (decode_context->map->type == CMT_EXP_HISTOGRAM) { + exp_histogram = (struct cmt_exp_histogram *) decode_context->map->parent; + exp_histogram->aggregation_type = decode_context->aggregation_type; } else if (decode_context->map->type == CMT_SUMMARY) { summary = (struct cmt_summary *) decode_context->map->parent; diff --git a/lib/cmetrics/src/cmt_decode_prometheus.c b/lib/cmetrics/src/cmt_decode_prometheus.c index 32b34babe55..13b08ac5b6b 100644 --- a/lib/cmetrics/src/cmt_decode_prometheus.c +++ b/lib/cmetrics/src/cmt_decode_prometheus.c @@ -56,23 +56,7 @@ static void reset_context(struct cmt_decode_prometheus_context *context, cfl_sds_destroy(context->metric.labels[i]); } - if (context->metric.ns) { - if ((void *) context->metric.ns != (void *) "") { - /* when namespace is empty, "name" contains a pointer to the - * allocated string. - * - * Note : When the metric name doesn't include the namespace - * ns is set to a constant empty string and we need to - * differentiate that case from the case where an empty - * namespace is provided. - */ - - free(context->metric.ns); - } - else { - free(context->metric.name); - } - } + free(context->metric.name_buf); cfl_sds_destroy(context->strbuf); context->strbuf = NULL; @@ -124,17 +108,19 @@ int cmt_decode_prometheus_create( result = cmt_decode_prometheus_parse(scanner, &context); + if (context.errcode) { + result = context.errcode; + } + if (result == 0) { *out_cmt = cmt; } else { cmt_destroy(cmt); - if (context.errcode) { - result = context.errcode; - } - reset_context(&context, true); } + reset_context(&context, true); + cmt_decode_prometheus__delete_buffer(buf, scanner); cmt_decode_prometheus_lex_destroy(scanner); @@ -164,13 +150,18 @@ static int split_metric_name(struct cmt_decode_prometheus_context *context, cfl_sds_t metric_name, char **ns, char **subsystem, char **name) { + char *name_buf; + /* split the name */ - *ns = strdup(metric_name); - if (!*ns) { + name_buf = strdup(metric_name); + if (name_buf == NULL) { return report_error(context, CMT_DECODE_PROMETHEUS_ALLOCATION_ERROR, "memory allocation failed"); } + + context->metric.name_buf = name_buf; + *ns = name_buf; *subsystem = strchr(*ns, '_'); if (!(*subsystem)) { *name = *ns; @@ -1224,6 +1215,9 @@ static int cmt_decode_prometheus_error(void *yyscanner, struct cmt_decode_prometheus_context *context, const char *msg) { - report_error(context, CMT_DECODE_PROMETHEUS_SYNTAX_ERROR, msg); + if (!context->errcode) { + report_error(context, CMT_DECODE_PROMETHEUS_SYNTAX_ERROR, msg); + } + return 0; } diff --git a/lib/cmetrics/src/cmt_decode_prometheus.l b/lib/cmetrics/src/cmt_decode_prometheus.l index 8055f52deee..8f42f695154 100644 --- a/lib/cmetrics/src/cmt_decode_prometheus.l +++ b/lib/cmetrics/src/cmt_decode_prometheus.l @@ -7,11 +7,79 @@ %{ #include +#include #define STRBUF_RET \ yylval->str = context->strbuf; \ context->strbuf = NULL +static void set_allocation_error(struct cmt_decode_prometheus_context *context) +{ + context->errcode = CMT_DECODE_PROMETHEUS_ALLOCATION_ERROR; + + if (context->opts.errbuf != NULL && context->opts.errbuf_size > 0) { + snprintf(context->opts.errbuf, + context->opts.errbuf_size, + "memory allocation failed"); + } +} + +static int reset_strbuf(struct cmt_decode_prometheus_context *context) +{ + if (context->strbuf != NULL) { + cfl_sds_destroy(context->strbuf); + } + + context->strbuf = cfl_sds_create_size(256); + if (context->strbuf == NULL) { + set_allocation_error(context); + + return -1; + } + + return 0; +} + +static int append_strbuf(struct cmt_decode_prometheus_context *context, + const char *text, int length) +{ + cfl_sds_t result; + + result = cfl_sds_cat(context->strbuf, text, length); + if (result == NULL) { + set_allocation_error(context); + + return -1; + } + + context->strbuf = result; + + return 0; +} + +#define STRBUF_CREATE() \ + do { \ + if (reset_strbuf(context) != 0) { \ + return 0; \ + } \ + } while (0) + +#define STRBUF_APPEND(text, length) \ + do { \ + if (append_strbuf(context, (text), (length)) != 0) { \ + return 0; \ + } \ + } while (0) + +#define SET_STR_TOKEN() \ + do { \ + yylval->str = cfl_sds_create(yytext); \ + if (yylval->str == NULL) { \ + set_allocation_error(context); \ + return 0; \ + } \ + } while (0) + %} /* here we define some states that allow us to create rules only @@ -74,7 +142,7 @@ [^ \t]+ { // The next token will be the metric name - yylval->str = cfl_sds_create(yytext); + SET_STR_TOKEN(); return YYSTATE == HELPTAG ? HELP : TYPE; } @@ -86,7 +154,7 @@ // separate start condition for this to handle "\\" and "\n" escapes // more easily. BEGIN(INHELPTAG); - context->strbuf = cfl_sds_create_size(256); + STRBUF_CREATE(); } else { // For TYPETAG we enter INTYPETAG start condition to check only valid @@ -107,17 +175,17 @@ \\n { // Process linefeed escape sequence - context->strbuf = cfl_sds_cat(context->strbuf, "\n", 1); + STRBUF_APPEND("\n", 1); } \\\\ { // Process backslack escape sequence - context->strbuf = cfl_sds_cat(context->strbuf, "\\", 1); + STRBUF_APPEND("\\", 1); } [^\r\n\\]+ { // Put everything that is not a backslash or a line feed into strbuf - context->strbuf = cfl_sds_cat(context->strbuf, yytext, yyleng); + STRBUF_APPEND(yytext, yyleng); } counter { @@ -146,26 +214,23 @@ ["] { BEGIN(INQUOTE); - if (context->strbuf != NULL) { - cfl_sds_destroy(context->strbuf); - } - context->strbuf = cfl_sds_create_size(256); + STRBUF_CREATE(); } [\\]["] { - context->strbuf = cfl_sds_cat(context->strbuf, "\"", 1); + STRBUF_APPEND("\"", 1); } \\n { - context->strbuf = cfl_sds_cat(context->strbuf, "\n", 1); + STRBUF_APPEND("\n", 1); } \\\\ { - context->strbuf = cfl_sds_cat(context->strbuf, "\\", 1); + STRBUF_APPEND("\\", 1); } [^\r\n\\"]+ { - context->strbuf = cfl_sds_cat(context->strbuf, yytext, yyleng); + STRBUF_APPEND(yytext, yyleng); } ["] { @@ -180,7 +245,7 @@ } [a-zA-Z_][a-zA-Z_0-9]* { - yylval->str = cfl_sds_create(yytext); + SET_STR_TOKEN(); return IDENTIFIER; } diff --git a/lib/cmetrics/src/cmt_encode_msgpack.c b/lib/cmetrics/src/cmt_encode_msgpack.c index c9328055615..4341b9c651f 100644 --- a/lib/cmetrics/src/cmt_encode_msgpack.c +++ b/lib/cmetrics/src/cmt_encode_msgpack.c @@ -58,6 +58,7 @@ static void pack_header(mpack_writer_t *writer, struct cmt *cmt, struct cmt_map size_t index; struct cmt_summary *summary = NULL; struct cmt_histogram *histogram = NULL; + struct cmt_exp_histogram *exp_histogram = NULL; struct cmt_counter *counter = NULL; size_t meta_field_count; @@ -67,6 +68,11 @@ static void pack_header(mpack_writer_t *writer, struct cmt *cmt, struct cmt_map if (map->type == CMT_HISTOGRAM) { histogram = (struct cmt_histogram *) map->parent; + meta_field_count += 2; + } + else if (map->type == CMT_EXP_HISTOGRAM) { + exp_histogram = (struct cmt_exp_histogram *) map->parent; + meta_field_count++; } else if (map->type == CMT_SUMMARY) { @@ -156,6 +162,15 @@ static void pack_header(mpack_writer_t *writer, struct cmt *cmt, struct cmt_map } mpack_finish_array(writer); + + /* aggregation_type */ + mpack_write_cstr(writer, "aggregation_type"); + mpack_write_int(writer, histogram->aggregation_type); + } + else if (map->type == CMT_EXP_HISTOGRAM) { + /* aggregation_type */ + mpack_write_cstr(writer, "aggregation_type"); + mpack_write_int(writer, exp_histogram->aggregation_type); } else if (map->type == CMT_SUMMARY) { /* 'quantiles' (summary quantiles) */ diff --git a/lib/cmetrics/src/cmt_encode_opentelemetry.c b/lib/cmetrics/src/cmt_encode_opentelemetry.c index 434746c08c0..65b6399ee40 100644 --- a/lib/cmetrics/src/cmt_encode_opentelemetry.c +++ b/lib/cmetrics/src/cmt_encode_opentelemetry.c @@ -26,8 +26,44 @@ #include #include #include +#include #include +#define CMT_OTLP_ARENA_INITIAL_CHUNK_SIZE 4096 +#define CMT_OTLP_ARENA_MAX_CHUNK_SIZE 65536 + +struct cmt_opentelemetry_encoder { + struct cmt_opentelemetry_context context; + struct cfl_arena *arena; +}; + +/* + * Data points and their label attributes remain immutable and live until the + * protobuf request has been packed. Allocate that temporary object graph from + * one arena and release it after packing. The returned SDS and metadata-owned + * objects remain heap-backed and retain their existing ownership rules. + */ + +static struct cfl_arena *get_context_arena(struct cmt_opentelemetry_context *context) +{ + struct cmt_opentelemetry_encoder *encoder; + + encoder = (struct cmt_opentelemetry_encoder *) context; + + return encoder->arena; +} + +static struct cfl_arena *create_encoder_arena(void) +{ + struct cfl_arena_options options; + + cfl_arena_options_init(&options); + options.chunk_size = CMT_OTLP_ARENA_INITIAL_CHUNK_SIZE; + options.maximum_chunk_size = CMT_OTLP_ARENA_MAX_CHUNK_SIZE; + + return cfl_arena_create_with_options(&options); +} + static Opentelemetry__Proto__Metrics__V1__ScopeMetrics ** initialize_scope_metrics_list( size_t element_count); @@ -746,15 +782,12 @@ static int is_metric_empty(struct cmt_map *map); static size_t get_metric_count(struct cmt *cmt); static Opentelemetry__Proto__Metrics__V1__SummaryDataPoint__ValueAtQuantile * - initialize_summary_value_at_quantile( - double quantile, double value); - -static void destroy_summary_value_at_quantile( - Opentelemetry__Proto__Metrics__V1__SummaryDataPoint__ValueAtQuantile *value); + initialize_summary_value_at_quantile(struct cfl_arena *arena, + double quantile, double value); static Opentelemetry__Proto__Metrics__V1__SummaryDataPoint__ValueAtQuantile ** - initialize_summary_value_at_quantile_list( - size_t element_count); + initialize_summary_value_at_quantile_list(struct cfl_arena *arena, + size_t element_count); static void destroy_summary_value_at_quantile_list( Opentelemetry__Proto__Metrics__V1__SummaryDataPoint__ValueAtQuantile **list); @@ -810,14 +843,15 @@ static void destroy_attribute( Opentelemetry__Proto__Common__V1__KeyValue *attribute); static Opentelemetry__Proto__Common__V1__KeyValue * - initialize_string_attribute(char *key, char *value); + initialize_string_attribute(struct cfl_arena *arena, + char *key, char *value); static void destroy_attribute_list( Opentelemetry__Proto__Common__V1__KeyValue **attribute_list); static Opentelemetry__Proto__Common__V1__KeyValue ** - initialize_attribute_list( - size_t element_count); + initialize_attribute_list(struct cfl_arena *arena, + size_t element_count); static void destroy_numerical_data_point( Opentelemetry__Proto__Metrics__V1__NumberDataPoint *data_point); @@ -848,16 +882,16 @@ static void destroy_exponential_histogram_data_point_list( Opentelemetry__Proto__Metrics__V1__ExponentialHistogramDataPoint **data_point_list); static Opentelemetry__Proto__Metrics__V1__NumberDataPoint * - initialize_numerical_data_point( - uint64_t start_time, + initialize_numerical_data_point(struct cfl_arena *arena, + uint64_t start_time, uint64_t timestamp, struct cmt_metric *sample, Opentelemetry__Proto__Common__V1__KeyValue **attribute_list, size_t attribute_count); static Opentelemetry__Proto__Metrics__V1__SummaryDataPoint * - initialize_summary_data_point( - uint64_t start_time, + initialize_summary_data_point(struct cfl_arena *arena, + uint64_t start_time, uint64_t timestamp, uint64_t count, double sum, @@ -869,8 +903,8 @@ static Opentelemetry__Proto__Metrics__V1__SummaryDataPoint * size_t attribute_count); static Opentelemetry__Proto__Metrics__V1__HistogramDataPoint * - initialize_histogram_data_point( - uint64_t start_time, + initialize_histogram_data_point(struct cfl_arena *arena, + uint64_t start_time, uint64_t timestamp, uint64_t count, double sum, @@ -1726,7 +1760,7 @@ void destroy_instrumentation_scope(Opentelemetry__Proto__Common__V1__Instrumenta } if (scope->attributes != NULL) { - destroy_attribute_list(scope->attributes); + otlp_kvpair_list_destroy(scope->attributes, scope->n_attributes); } free(scope); @@ -1904,33 +1938,18 @@ static Opentelemetry__Proto__Metrics__V1__ScopeMetrics **initialize_scope_metric static void destroy_attribute(Opentelemetry__Proto__Common__V1__KeyValue *attribute) { - if (attribute != NULL) { - if (attribute->value != NULL) { - if (attribute->value->value_case == \ - OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_STRING_VALUE) { - if (is_string_releaseable(attribute->value->string_value)) { - cfl_sds_destroy(attribute->value->string_value); - } - } - - free(attribute->value); - } - - if (is_string_releaseable(attribute->key)) { - cfl_sds_destroy(attribute->key); - } - - free(attribute); - } + (void) attribute; } static Opentelemetry__Proto__Common__V1__KeyValue * - initialize_string_attribute(char *key, char *value) + initialize_string_attribute(struct cfl_arena *arena, + char *key, char *value) { Opentelemetry__Proto__Common__V1__KeyValue *attribute; - attribute = calloc(1, - sizeof(Opentelemetry__Proto__Common__V1__KeyValue)); + attribute = cfl_arena_calloc( + arena, 1, + sizeof(Opentelemetry__Proto__Common__V1__KeyValue)); if (attribute == NULL) { return NULL; @@ -1938,8 +1957,9 @@ static Opentelemetry__Proto__Common__V1__KeyValue * opentelemetry__proto__common__v1__key_value__init(attribute); - attribute->value = calloc(1, - sizeof(Opentelemetry__Proto__Common__V1__AnyValue)); + attribute->value = cfl_arena_calloc( + arena, 1, + sizeof(Opentelemetry__Proto__Common__V1__AnyValue)); if (attribute->value == NULL) { destroy_attribute(attribute); @@ -1949,7 +1969,8 @@ static Opentelemetry__Proto__Common__V1__KeyValue * opentelemetry__proto__common__v1__any_value__init(attribute->value); - attribute->value->string_value = cfl_sds_create(value); + attribute->value->string_value = cfl_arena_strndup(arena, value, + strlen(value)); if (attribute->value->string_value == NULL) { destroy_attribute(attribute); @@ -1959,7 +1980,7 @@ static Opentelemetry__Proto__Common__V1__KeyValue * attribute->value->value_case = OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_STRING_VALUE; - attribute->key = cfl_sds_create(key); + attribute->key = cfl_arena_strndup(arena, key, strlen(key)); if (attribute->key == NULL) { destroy_attribute(attribute); @@ -1973,29 +1994,18 @@ static Opentelemetry__Proto__Common__V1__KeyValue * static void destroy_attribute_list( Opentelemetry__Proto__Common__V1__KeyValue **attribute_list) { - size_t element_index; - - if (attribute_list != NULL) { - for (element_index = 0 ; - attribute_list[element_index] != NULL ; - element_index++) { - destroy_attribute(attribute_list[element_index]); - - attribute_list[element_index] = NULL; - } - - free(attribute_list); - } + (void) attribute_list; } static Opentelemetry__Proto__Common__V1__KeyValue ** - initialize_attribute_list( - size_t element_count) + initialize_attribute_list(struct cfl_arena *arena, + size_t element_count) { Opentelemetry__Proto__Common__V1__KeyValue **attribute_list; - attribute_list = calloc(element_count + 1, - sizeof(Opentelemetry__Proto__Common__V1__KeyValue *)); + attribute_list = cfl_arena_calloc( + arena, element_count + 1, + sizeof(Opentelemetry__Proto__Common__V1__KeyValue *)); return attribute_list; } @@ -2017,23 +2027,13 @@ static void destroy_numerical_data_point( free(data_point->exemplars); } - - free(data_point); } } static void destroy_summary_data_point( Opentelemetry__Proto__Metrics__V1__SummaryDataPoint *data_point) { - if (data_point != NULL) { - destroy_attribute_list(data_point->attributes); - - if (data_point->quantile_values != NULL) { - destroy_summary_value_at_quantile_list(data_point->quantile_values); - } - - free(data_point); - } + (void) data_point; } static void destroy_histogram_data_point( @@ -2042,16 +2042,6 @@ static void destroy_histogram_data_point( size_t index; if (data_point != NULL) { - destroy_attribute_list(data_point->attributes); - - if (data_point->bucket_counts != NULL) { - free(data_point->bucket_counts); - } - - if (data_point->explicit_bounds != NULL) { - free(data_point->explicit_bounds); - } - if (data_point->exemplars != NULL) { for (index = 0; index < data_point->n_exemplars; index++) { if (data_point->exemplars[index] != NULL) { @@ -2061,8 +2051,6 @@ static void destroy_histogram_data_point( free(data_point->exemplars); } - - free(data_point); } } @@ -2072,22 +2060,6 @@ static void destroy_exponential_histogram_data_point( size_t index; if (data_point != NULL) { - destroy_attribute_list(data_point->attributes); - - if (data_point->positive != NULL) { - if (data_point->positive->bucket_counts != NULL) { - free(data_point->positive->bucket_counts); - } - free(data_point->positive); - } - - if (data_point->negative != NULL) { - if (data_point->negative->bucket_counts != NULL) { - free(data_point->negative->bucket_counts); - } - free(data_point->negative); - } - if (data_point->exemplars != NULL) { for (index = 0; index < data_point->n_exemplars; index++) { if (data_point->exemplars[index] != NULL) { @@ -2097,8 +2069,6 @@ static void destroy_exponential_histogram_data_point( free(data_point->exemplars); } - - free(data_point); } } @@ -2192,8 +2162,8 @@ static void destroy_exponential_histogram_data_point_list( } static Opentelemetry__Proto__Metrics__V1__NumberDataPoint * - initialize_numerical_data_point( - uint64_t start_time, + initialize_numerical_data_point(struct cfl_arena *arena, + uint64_t start_time, uint64_t timestamp, struct cmt_metric *sample, Opentelemetry__Proto__Common__V1__KeyValue **attribute_list, @@ -2201,8 +2171,9 @@ static Opentelemetry__Proto__Metrics__V1__NumberDataPoint * { Opentelemetry__Proto__Metrics__V1__NumberDataPoint *data_point; - data_point = calloc(1, - sizeof(Opentelemetry__Proto__Metrics__V1__NumberDataPoint)); + data_point = cfl_arena_calloc( + arena, 1, + sizeof(Opentelemetry__Proto__Metrics__V1__NumberDataPoint)); if (data_point == NULL) { return NULL; @@ -2237,12 +2208,14 @@ static Opentelemetry__Proto__Metrics__V1__NumberDataPoint * } static Opentelemetry__Proto__Metrics__V1__SummaryDataPoint__ValueAtQuantile * - initialize_summary_value_at_quantile( - double quantile, double value) + initialize_summary_value_at_quantile(struct cfl_arena *arena, + double quantile, double value) { Opentelemetry__Proto__Metrics__V1__SummaryDataPoint__ValueAtQuantile *instance; - instance = calloc(1, sizeof(Opentelemetry__Proto__Metrics__V1__SummaryDataPoint__ValueAtQuantile)); + instance = cfl_arena_calloc( + arena, 1, + sizeof(Opentelemetry__Proto__Metrics__V1__SummaryDataPoint__ValueAtQuantile)); if (instance != NULL) { opentelemetry__proto__metrics__v1__summary_data_point__value_at_quantile__init(instance); @@ -2254,22 +2227,15 @@ static Opentelemetry__Proto__Metrics__V1__SummaryDataPoint__ValueAtQuantile * return instance; } -static void destroy_summary_value_at_quantile( - Opentelemetry__Proto__Metrics__V1__SummaryDataPoint__ValueAtQuantile *value) -{ - if (value != NULL) { - free(value); - } -} - static Opentelemetry__Proto__Metrics__V1__SummaryDataPoint__ValueAtQuantile ** - initialize_summary_value_at_quantile_list( - size_t element_count) + initialize_summary_value_at_quantile_list(struct cfl_arena *arena, + size_t element_count) { Opentelemetry__Proto__Metrics__V1__SummaryDataPoint__ValueAtQuantile **list; - list = calloc(element_count + 1, - sizeof(Opentelemetry__Proto__Metrics__V1__SummaryDataPoint__ValueAtQuantile *)); + list = cfl_arena_calloc( + arena, element_count + 1, + sizeof(Opentelemetry__Proto__Metrics__V1__SummaryDataPoint__ValueAtQuantile *)); return list; } @@ -2277,26 +2243,14 @@ static Opentelemetry__Proto__Metrics__V1__SummaryDataPoint__ValueAtQuantile ** static void destroy_summary_value_at_quantile_list( Opentelemetry__Proto__Metrics__V1__SummaryDataPoint__ValueAtQuantile **list) { - size_t index; - - if (list != NULL) { - for (index = 0 ; - list[index] != NULL ; - index++) { - destroy_summary_value_at_quantile(list[index]); - - list[index] = NULL; - } - - free(list); - } + (void) list; } static Opentelemetry__Proto__Metrics__V1__SummaryDataPoint * - initialize_summary_data_point( - uint64_t start_time, + initialize_summary_data_point(struct cfl_arena *arena, + uint64_t start_time, uint64_t timestamp, uint64_t count, double sum, @@ -2310,8 +2264,9 @@ static Opentelemetry__Proto__Metrics__V1__SummaryDataPoint * Opentelemetry__Proto__Metrics__V1__SummaryDataPoint *data_point; size_t index; - data_point = calloc(1, - sizeof(Opentelemetry__Proto__Metrics__V1__SummaryDataPoint)); + data_point = cfl_arena_calloc( + arena, 1, + sizeof(Opentelemetry__Proto__Metrics__V1__SummaryDataPoint)); if (data_point == NULL) { return NULL; @@ -2326,13 +2281,12 @@ static Opentelemetry__Proto__Metrics__V1__SummaryDataPoint * data_point->sum = sum; data_point->n_quantile_values = quantile_count; - data_point->quantile_values = initialize_summary_value_at_quantile_list(quantile_count); + data_point->quantile_values = + initialize_summary_value_at_quantile_list(arena, quantile_count); if (data_point->quantile_values == NULL) { cmt_errno(); - free(data_point); - return NULL; } @@ -2340,14 +2294,13 @@ static Opentelemetry__Proto__Metrics__V1__SummaryDataPoint * if (value_list != NULL) { for (index = 0 ; index < quantile_count ; index++) { data_point->quantile_values[index] = - initialize_summary_value_at_quantile(quantile_list[index], + initialize_summary_value_at_quantile(arena, + quantile_list[index], cmt_math_uint64_to_d64(value_list[index])); if (data_point->quantile_values[index] == NULL) { destroy_summary_value_at_quantile_list(data_point->quantile_values); - free(data_point); - return NULL; } } @@ -2361,8 +2314,8 @@ static Opentelemetry__Proto__Metrics__V1__SummaryDataPoint * } static Opentelemetry__Proto__Metrics__V1__HistogramDataPoint * - initialize_histogram_data_point( - uint64_t start_time, + initialize_histogram_data_point(struct cfl_arena *arena, + uint64_t start_time, uint64_t timestamp, uint64_t count, double sum, @@ -2376,8 +2329,9 @@ static Opentelemetry__Proto__Metrics__V1__HistogramDataPoint * Opentelemetry__Proto__Metrics__V1__HistogramDataPoint *data_point; size_t index; - data_point = calloc(1, - sizeof(Opentelemetry__Proto__Metrics__V1__HistogramDataPoint)); + data_point = cfl_arena_calloc( + arena, 1, + sizeof(Opentelemetry__Proto__Metrics__V1__HistogramDataPoint)); if (data_point == NULL) { return NULL; @@ -2402,13 +2356,12 @@ static Opentelemetry__Proto__Metrics__V1__HistogramDataPoint * if (bucket_count > 0) { - data_point->bucket_counts = calloc(bucket_count, sizeof(uint64_t)); + data_point->bucket_counts = cfl_arena_calloc(arena, bucket_count, + sizeof(uint64_t)); if (data_point->bucket_counts == NULL) { cmt_errno(); - free(data_point); - return NULL; } @@ -2422,17 +2375,12 @@ static Opentelemetry__Proto__Metrics__V1__HistogramDataPoint * data_point->n_explicit_bounds = boundary_count; if (boundary_count > 0) { - data_point->explicit_bounds = calloc(boundary_count, sizeof(double)); + data_point->explicit_bounds = cfl_arena_calloc(arena, boundary_count, + sizeof(double)); if (data_point->explicit_bounds == NULL) { cmt_errno(); - if (data_point->bucket_counts != NULL) { - free(data_point->bucket_counts); - } - - free(data_point); - return NULL; } @@ -2944,7 +2892,11 @@ static Opentelemetry__Proto__Metrics__V1__Metric ** static void destroy_opentelemetry_context( struct cmt_opentelemetry_context *context) { + struct cmt_opentelemetry_encoder *encoder; + if (context != NULL) { + encoder = (struct cmt_opentelemetry_encoder *) context; + if (context->scope_metrics_list != NULL) { free(context->scope_metrics_list); } @@ -2953,7 +2905,8 @@ static void destroy_opentelemetry_context( destroy_metrics_data(context->metrics_data); } - free(context); + cfl_arena_destroy(encoder->arena); + free(encoder); } } @@ -3023,6 +2976,7 @@ static struct cmt_opentelemetry_context *initialize_opentelemetry_context( struct cfl_kvlist *scope_root; Opentelemetry__Proto__Resource__V1__Resource *resource; struct cmt_opentelemetry_context *context; + struct cmt_opentelemetry_encoder *encoder; size_t resource_count; size_t scope_count; size_t total_scope_count; @@ -3039,17 +2993,26 @@ static struct cmt_opentelemetry_context *initialize_opentelemetry_context( result = CMT_ENCODE_OPENTELEMETRY_SUCCESS; total_scope_count = 0; + context = NULL; + encoder = NULL; - context = calloc(1, sizeof(struct cmt_opentelemetry_context)); - - if (context == NULL) { + encoder = calloc(1, sizeof(struct cmt_opentelemetry_encoder)); + if (encoder == NULL) { result = CMT_ENCODE_OPENTELEMETRY_ALLOCATION_ERROR; goto cleanup; } - memset(context, 0, sizeof(struct cmt_opentelemetry_context)); + encoder->arena = create_encoder_arena(); + if (encoder->arena == NULL) { + free(encoder); + context = NULL; + result = CMT_ENCODE_OPENTELEMETRY_ALLOCATION_ERROR; + + goto cleanup; + } + context = &encoder->context; context->cmt = cmt; if (resource_metrics_list != NULL && resource_metrics_list->entry_count > 0) { @@ -3185,7 +3148,7 @@ int append_sample_to_metric(struct cmt_opentelemetry_context *context, Opentelemetry__Proto__Common__V1__KeyValue **attribute_list; struct cmt_label *static_label; struct cmt_map_label *label_value; - struct cmt_map_label *label_name; + struct cmt_map_label *label_name = NULL; void *data_point = NULL; Opentelemetry__Proto__Common__V1__KeyValue *attribute; struct cmt_histogram *histogram; @@ -3215,7 +3178,8 @@ int append_sample_to_metric(struct cmt_opentelemetry_context *context, start_timestamp = cmt_metric_get_start_timestamp(sample); } - attribute_list = initialize_attribute_list(attribute_count); + attribute_list = initialize_attribute_list(get_context_arena(context), + attribute_count); if (attribute_list == NULL) { return CMT_ENCODE_OPENTELEMETRY_ALLOCATION_ERROR; @@ -3224,7 +3188,8 @@ int append_sample_to_metric(struct cmt_opentelemetry_context *context, if (map->type == CMT_COUNTER || map->type == CMT_GAUGE || map->type == CMT_UNTYPED ) { - data_point = initialize_numerical_data_point(start_timestamp, + data_point = initialize_numerical_data_point(get_context_arena(context), + start_timestamp, cmt_metric_get_timestamp(sample), sample, attribute_list, @@ -3233,7 +3198,8 @@ int append_sample_to_metric(struct cmt_opentelemetry_context *context, else if (map->type == CMT_SUMMARY) { summary = (struct cmt_summary *) map->parent; - data_point = initialize_summary_data_point(start_timestamp, + data_point = initialize_summary_data_point(get_context_arena(context), + start_timestamp, cmt_metric_get_timestamp(sample), cmt_summary_get_count_value(sample), cmt_summary_get_sum_value(sample), @@ -3247,7 +3213,8 @@ int append_sample_to_metric(struct cmt_opentelemetry_context *context, else if (map->type == CMT_HISTOGRAM) { histogram = (struct cmt_histogram *) map->parent; - data_point = initialize_histogram_data_point(start_timestamp, + data_point = initialize_histogram_data_point(get_context_arena(context), + start_timestamp, cmt_metric_get_timestamp(sample), cmt_metric_hist_get_count_value(sample), cmt_metric_hist_get_sum_value(sample), @@ -3266,7 +3233,9 @@ int append_sample_to_metric(struct cmt_opentelemetry_context *context, return CMT_ENCODE_OPENTELEMETRY_DATA_POINT_INIT_ERROR; } - exponential_data_point = calloc(1, sizeof(Opentelemetry__Proto__Metrics__V1__ExponentialHistogramDataPoint)); + exponential_data_point = cfl_arena_calloc( + get_context_arena(context), 1, + sizeof(Opentelemetry__Proto__Metrics__V1__ExponentialHistogramDataPoint)); if (exponential_data_point == NULL) { cmt_metric_exp_hist_snapshot_destroy(&snapshot); destroy_attribute_list(attribute_list); @@ -3290,7 +3259,9 @@ int append_sample_to_metric(struct cmt_opentelemetry_context *context, } if (snapshot.positive_count > 0) { - positive = calloc(1, sizeof(Opentelemetry__Proto__Metrics__V1__ExponentialHistogramDataPoint__Buckets)); + positive = cfl_arena_calloc( + get_context_arena(context), 1, + sizeof(Opentelemetry__Proto__Metrics__V1__ExponentialHistogramDataPoint__Buckets)); if (positive == NULL) { cmt_metric_exp_hist_snapshot_destroy(&snapshot); destroy_data_point(exponential_data_point, map->type); @@ -3300,10 +3271,12 @@ int append_sample_to_metric(struct cmt_opentelemetry_context *context, opentelemetry__proto__metrics__v1__exponential_histogram_data_point__buckets__init(positive); positive->offset = snapshot.positive_offset; positive->n_bucket_counts = snapshot.positive_count; - positive->bucket_counts = calloc(snapshot.positive_count, sizeof(uint64_t)); + positive->bucket_counts = cfl_arena_calloc( + get_context_arena(context), + snapshot.positive_count, + sizeof(uint64_t)); if (positive->bucket_counts == NULL) { cmt_metric_exp_hist_snapshot_destroy(&snapshot); - free(positive); destroy_data_point(exponential_data_point, map->type); return CMT_ENCODE_OPENTELEMETRY_DATA_POINT_INIT_ERROR; } @@ -3313,7 +3286,9 @@ int append_sample_to_metric(struct cmt_opentelemetry_context *context, } if (snapshot.negative_count > 0) { - negative = calloc(1, sizeof(Opentelemetry__Proto__Metrics__V1__ExponentialHistogramDataPoint__Buckets)); + negative = cfl_arena_calloc( + get_context_arena(context), 1, + sizeof(Opentelemetry__Proto__Metrics__V1__ExponentialHistogramDataPoint__Buckets)); if (negative == NULL) { cmt_metric_exp_hist_snapshot_destroy(&snapshot); destroy_data_point(exponential_data_point, map->type); @@ -3323,10 +3298,12 @@ int append_sample_to_metric(struct cmt_opentelemetry_context *context, opentelemetry__proto__metrics__v1__exponential_histogram_data_point__buckets__init(negative); negative->offset = snapshot.negative_offset; negative->n_bucket_counts = snapshot.negative_count; - negative->bucket_counts = calloc(snapshot.negative_count, sizeof(uint64_t)); + negative->bucket_counts = cfl_arena_calloc( + get_context_arena(context), + snapshot.negative_count, + sizeof(uint64_t)); if (negative->bucket_counts == NULL) { cmt_metric_exp_hist_snapshot_destroy(&snapshot); - free(negative); destroy_data_point(exponential_data_point, map->type); return CMT_ENCODE_OPENTELEMETRY_DATA_POINT_INIT_ERROR; } @@ -3350,7 +3327,8 @@ int append_sample_to_metric(struct cmt_opentelemetry_context *context, cfl_list_foreach(head, &context->cmt->static_labels->list) { static_label = cfl_list_entry(head, struct cmt_label, _head); - attribute = initialize_string_attribute(static_label->key, + attribute = initialize_string_attribute(get_context_arena(context), + static_label->key, static_label->val); if (attribute == NULL) { @@ -3404,13 +3382,14 @@ int append_sample_to_metric(struct cmt_opentelemetry_context *context, return CMT_ENCODE_OPENTELEMETRY_INVALID_ARGUMENT_ERROR; } - if (label_name->name == NULL) { + if (label_name == NULL || label_name->name == NULL) { destroy_data_point(data_point, map->type); return CMT_ENCODE_OPENTELEMETRY_INVALID_ARGUMENT_ERROR; } - attribute = initialize_string_attribute(label_name->name, + attribute = initialize_string_attribute(get_context_arena(context), + label_name->name, label_value->name); if (attribute == NULL) { diff --git a/lib/cmetrics/src/cmt_encode_prometheus.c b/lib/cmetrics/src/cmt_encode_prometheus.c index e144a1aa5f4..b8473182338 100644 --- a/lib/cmetrics/src/cmt_encode_prometheus.c +++ b/lib/cmetrics/src/cmt_encode_prometheus.c @@ -18,6 +18,7 @@ */ #include +#include #include #include @@ -407,7 +408,12 @@ static cfl_sds_t bucket_value_to_string(double val) } cfl_sds_len_set(str, len); - if (!strchr(str, '.') && !strchr(str, 'e') && !strchr(str, 'E')) { + /* + * Append .0 only when there is no decimal point and the number + * is finite and not in scientific notation. + */ + if (isfinite(val) && + !strchr(str, '.') && !strchr(str, 'e') && !strchr(str, 'E')) { cfl_sds_cat_safe(&str, ".0", 2); } diff --git a/lib/cmetrics/src/cmt_encode_prometheus_remote_write.c b/lib/cmetrics/src/cmt_encode_prometheus_remote_write.c index 63ce463c696..e09c216211a 100644 --- a/lib/cmetrics/src/cmt_encode_prometheus_remote_write.c +++ b/lib/cmetrics/src/cmt_encode_prometheus_remote_write.c @@ -735,6 +735,11 @@ static int check_staled_timestamp(struct cmt_metric *metric, uint64_t now, uint6 uint64_t diff; ts = cmt_metric_get_timestamp(metric); + + if (ts >= now) { + return CMT_FALSE; + } + diff = now - ts; return diff > cutoff; @@ -751,6 +756,7 @@ int pack_basic_type(struct cmt_prometheus_remote_write_context *context, context->sequence_number++; add_metadata = CMT_ENCODE_PROMETHEUS_REMOTE_WRITE_ADD_METADATA; + result = CMT_ENCODE_PROMETHEUS_REMOTE_WRITE_SUCCESS; now = cfl_time_now(); @@ -758,13 +764,14 @@ int pack_basic_type(struct cmt_prometheus_remote_write_context *context, if (check_staled_timestamp(&map->metric, now, CMT_ENCODE_PROMETHEUS_REMOTE_WRITE_CUTOFF_THRESHOLD)) { /* Skip processing metrics which are staled over the threshold */ - return CMT_ENCODE_PROMETHEUS_REMOTE_WRITE_CUTOFF_ERROR; + result = CMT_ENCODE_PROMETHEUS_REMOTE_WRITE_SUCCESS; } - - result = pack_basic_metric_sample(context, map, &map->metric, add_metadata); - - if (result != CMT_ENCODE_PROMETHEUS_REMOTE_WRITE_SUCCESS) { - return result; + else { + result = pack_basic_metric_sample(context, map, &map->metric, add_metadata); + if (result != CMT_ENCODE_PROMETHEUS_REMOTE_WRITE_SUCCESS) { + return result; + } + add_metadata = CMT_FALSE; } } @@ -774,7 +781,7 @@ int pack_basic_type(struct cmt_prometheus_remote_write_context *context, if (check_staled_timestamp(metric, now, CMT_ENCODE_PROMETHEUS_REMOTE_WRITE_CUTOFF_THRESHOLD)) { /* Skip processing metrics which are staled over over the threshold */ - return CMT_ENCODE_PROMETHEUS_REMOTE_WRITE_CUTOFF_ERROR; + continue; } result = pack_basic_metric_sample(context, map, metric, add_metadata); @@ -1208,6 +1215,12 @@ int pack_complex_type(struct cmt_prometheus_remote_write_context *context, if (map->metric_static_set == CMT_TRUE) { result = pack_complex_metric_sample(context, map, &map->metric, add_metadata); + if (result == CMT_ENCODE_PROMETHEUS_REMOTE_WRITE_CUTOFF_ERROR) { + result = CMT_ENCODE_PROMETHEUS_REMOTE_WRITE_SUCCESS; + } + else if (result == CMT_ENCODE_PROMETHEUS_REMOTE_WRITE_SUCCESS) { + add_metadata = CMT_FALSE; + } } if (result == CMT_ENCODE_PROMETHEUS_REMOTE_WRITE_SUCCESS) { @@ -1216,10 +1229,17 @@ int pack_complex_type(struct cmt_prometheus_remote_write_context *context, result = pack_complex_metric_sample(context, map, metric, add_metadata); - if (result == CMT_ENCODE_PROMETHEUS_REMOTE_WRITE_SUCCESS) { - if (add_metadata == CMT_TRUE) { - add_metadata = CMT_FALSE; - } + if (result == CMT_ENCODE_PROMETHEUS_REMOTE_WRITE_CUTOFF_ERROR) { + result = CMT_ENCODE_PROMETHEUS_REMOTE_WRITE_SUCCESS; + continue; + } + + if (result != CMT_ENCODE_PROMETHEUS_REMOTE_WRITE_SUCCESS) { + break; + } + + if (add_metadata == CMT_TRUE) { + add_metadata = CMT_FALSE; } } } @@ -1266,6 +1286,7 @@ cfl_sds_t cmt_encode_prometheus_remote_write_create(struct cmt *cmt) result = pack_basic_type(&context, counter->map); if (result == CMT_ENCODE_PROMETHEUS_REMOTE_WRITE_CUTOFF_ERROR) { + result = CMT_ENCODE_PROMETHEUS_REMOTE_WRITE_SUCCESS; continue; } @@ -1281,6 +1302,7 @@ cfl_sds_t cmt_encode_prometheus_remote_write_create(struct cmt *cmt) result = pack_basic_type(&context, gauge->map); if (result == CMT_ENCODE_PROMETHEUS_REMOTE_WRITE_CUTOFF_ERROR) { + result = CMT_ENCODE_PROMETHEUS_REMOTE_WRITE_SUCCESS; continue; } @@ -1294,11 +1316,16 @@ cfl_sds_t cmt_encode_prometheus_remote_write_create(struct cmt *cmt) /* Untyped */ cfl_list_foreach(head, &cmt->untypeds) { untyped = cfl_list_entry(head, struct cmt_untyped, _head); - pack_basic_type(&context, untyped->map); + result = pack_basic_type(&context, untyped->map); if (result == CMT_ENCODE_PROMETHEUS_REMOTE_WRITE_CUTOFF_ERROR) { + result = CMT_ENCODE_PROMETHEUS_REMOTE_WRITE_SUCCESS; continue; } + + if (result != CMT_ENCODE_PROMETHEUS_REMOTE_WRITE_SUCCESS) { + break; + } } } @@ -1309,6 +1336,7 @@ cfl_sds_t cmt_encode_prometheus_remote_write_create(struct cmt *cmt) result = pack_complex_type(&context, summary->map); if (result == CMT_ENCODE_PROMETHEUS_REMOTE_WRITE_CUTOFF_ERROR) { + result = CMT_ENCODE_PROMETHEUS_REMOTE_WRITE_SUCCESS; continue; } @@ -1325,6 +1353,7 @@ cfl_sds_t cmt_encode_prometheus_remote_write_create(struct cmt *cmt) result = pack_complex_type(&context, histogram->map); if (result == CMT_ENCODE_PROMETHEUS_REMOTE_WRITE_CUTOFF_ERROR) { + result = CMT_ENCODE_PROMETHEUS_REMOTE_WRITE_SUCCESS; continue; } @@ -1341,6 +1370,7 @@ cfl_sds_t cmt_encode_prometheus_remote_write_create(struct cmt *cmt) result = pack_complex_type(&context, exp_histogram->map); if (result == CMT_ENCODE_PROMETHEUS_REMOTE_WRITE_CUTOFF_ERROR) { + result = CMT_ENCODE_PROMETHEUS_REMOTE_WRITE_SUCCESS; continue; } diff --git a/lib/cmetrics/src/cmt_filter.c b/lib/cmetrics/src/cmt_filter.c index 6b7c220443d..0124f57d03d 100644 --- a/lib/cmetrics/src/cmt_filter.c +++ b/lib/cmetrics/src/cmt_filter.c @@ -186,7 +186,7 @@ static int filter_get_label_index(struct cmt_map *src, const char *label_key) cfl_list_foreach(head, &src->label_keys) { label = cfl_list_entry(head, struct cmt_map_label, _head); - if (strncmp(label->name, label_key, strlen(label->name)) == 0) { + if (strcmp(label->name, label_key) == 0) { return index; } @@ -228,7 +228,7 @@ int metrics_check_label_value_existence(struct cmt_metric *metric, return CMT_FALSE; } - if (strncmp(label->name, label_value, strlen(label->name)) == 0) { + if (strcmp(label->name, label_value) == 0) { return CMT_TRUE; } @@ -239,29 +239,27 @@ static int metrics_map_drop_label_value_pairs(struct cmt_map *map, size_t label_index, const char *label_value) { + struct cfl_list *tmp; struct cfl_list *head; struct cmt_metric *metric; + int match; int result; result = CMT_FALSE; - cfl_list_foreach(head, &map->metrics) { + cfl_list_foreach_safe(head, tmp, &map->metrics) { metric = cfl_list_entry(head, struct cmt_metric, _head); - result = metrics_check_label_value_existence(metric, - label_index, - label_value); + match = metrics_check_label_value_existence(metric, + label_index, + label_value); - if (result == CMT_TRUE) { + if (match == CMT_TRUE) { result = CMT_TRUE; - break; + cmt_map_metric_destroy(metric); } } - if (result == CMT_TRUE) { - cmt_map_metric_destroy(metric); - } - return result; } diff --git a/lib/cmetrics/src/cmt_map.c b/lib/cmetrics/src/cmt_map.c index 19549d52b45..fb4fe9a150e 100644 --- a/lib/cmetrics/src/cmt_map.c +++ b/lib/cmetrics/src/cmt_map.c @@ -21,8 +21,91 @@ #include #include #include +#include +#include +#include #include +#define CMT_MAP_INITIAL_BUCKET_COUNT 64 +#define CMT_MAP_BUCKET_LOAD_FACTOR 4 + +static void map_lock(struct cmt_map *map) +{ + while (cmt_atomic_compare_exchange(&map->metric_lock, 0, 1) == 0) { + } +} + +static void map_unlock(struct cmt_map *map) +{ + cmt_atomic_store(&map->metric_lock, 0); +} + +static void metric_release_storage(struct cmt_metric *metric) +{ + free(metric->hist_buckets); + free(metric->exp_hist_positive_buckets); + free(metric->exp_hist_negative_buckets); + free(metric->sum_quantiles); + + metric->hist_buckets = NULL; + metric->exp_hist_positive_buckets = NULL; + metric->exp_hist_negative_buckets = NULL; + metric->sum_quantiles = NULL; +} + +static int metric_index_resize(struct cmt_map *map, size_t bucket_count) +{ + size_t index; + struct cfl_list *head; + struct cfl_list *buckets; + struct cmt_metric *metric; + + buckets = calloc(bucket_count, sizeof(struct cfl_list)); + if (buckets == NULL) { + return -1; + } + for (index = 0; index < bucket_count; index++) { + cfl_list_init(&buckets[index]); + } + + cfl_list_foreach(head, &map->metrics) { + metric = cfl_list_entry(head, struct cmt_metric, _head); + if (metric->hash_indexed) { + cfl_list_del(&metric->_hash_head); + cfl_list_add(&metric->_hash_head, + &buckets[metric->hash % bucket_count]); + } + } + + free(map->metric_buckets); + map->metric_buckets = buckets; + map->metric_bucket_count = bucket_count; + return 0; +} + +static void metric_index_add(struct cmt_map *map, struct cmt_metric *metric) +{ + if (metric->hash_indexed) { + return; + } + + if (map->metric_buckets == NULL && + metric_index_resize(map, CMT_MAP_INITIAL_BUCKET_COUNT) != 0) { + return; + } + + if (map->indexed_metric_count >= + map->metric_bucket_count * CMT_MAP_BUCKET_LOAD_FACTOR) { + metric_index_resize(map, map->metric_bucket_count * 2); + } + + cfl_list_add(&metric->_hash_head, + &map->metric_buckets[metric->hash % map->metric_bucket_count]); + metric->hash_indexed = CMT_TRUE; + metric->map = map; + map->indexed_metric_count++; +} + struct cmt_map *cmt_map_create(int type, struct cmt_opts *opts, int count, char **labels, void *parent) { @@ -48,6 +131,13 @@ struct cmt_map *cmt_map_create(int type, struct cmt_opts *opts, int count, char cfl_list_init(&map->metrics); cfl_list_init(&map->metric.labels); + if (count > 0 && + metric_index_resize(map, CMT_MAP_INITIAL_BUCKET_COUNT) != 0) { + cmt_errno(); + cmt_map_destroy(map); + return NULL; + } + if (count == 0) { map->metric_static_set = 1; } @@ -76,7 +166,75 @@ struct cmt_map *cmt_map_create(int type, struct cmt_opts *opts, int count, char return NULL; } -static struct cmt_metric *metric_hash_lookup(struct cmt_map *map, uint64_t hash) +static int metric_labels_match(struct cmt_metric *metric, + int labels_count, char **labels_val) +{ + int index = 0; + struct cfl_list *head; + struct cmt_map_label *label; + + cfl_list_foreach(head, &metric->labels) { + if (index >= labels_count) { + return CMT_FALSE; + } + + label = cfl_list_entry(head, struct cmt_map_label, _head); + if ((label->name == NULL) != (labels_val[index] == NULL)) { + return CMT_FALSE; + } + if (label->name != NULL && strcmp(label->name, labels_val[index]) != 0) { + return CMT_FALSE; + } + index++; + } + + return index == labels_count; +} + +static struct cmt_metric *metric_prepare_storage(struct cmt_map *map, + struct cmt_metric *metric, + int write_op) +{ + struct cmt_summary *summary; + struct cmt_histogram *histogram; + + if (metric == NULL || !write_op) { + return metric; + } + + if (map->type == CMT_HISTOGRAM && metric->hist_buckets == NULL) { + histogram = map->parent; + if (histogram == NULL || histogram->buckets == NULL) { + return NULL; + } + metric->hist_buckets = calloc(histogram->buckets->count + 1, + sizeof(uint64_t)); + if (metric->hist_buckets == NULL) { + cmt_errno(); + return NULL; + } + } + else if (map->type == CMT_SUMMARY && metric->sum_quantiles == NULL) { + summary = map->parent; + if (summary == NULL) { + return NULL; + } + if (summary->quantiles_count > 0) { + metric->sum_quantiles = calloc(summary->quantiles_count, + sizeof(uint64_t)); + if (metric->sum_quantiles == NULL) { + cmt_errno(); + return NULL; + } + } + metric->sum_quantiles_count = summary->quantiles_count; + } + + return metric; +} + +static struct cmt_metric *metric_hash_lookup(struct cmt_map *map, uint64_t hash, + int labels_count, char **labels_val) { struct cfl_list *head; struct cmt_metric *metric; @@ -85,9 +243,30 @@ static struct cmt_metric *metric_hash_lookup(struct cmt_map *map, uint64_t hash) return &map->metric; } + metric = map->last_metric; + if (metric != NULL && metric->hash == hash && + metric_labels_match(metric, labels_count, labels_val)) { + return metric; + } + + if (map->metric_buckets != NULL) { + cfl_list_foreach(head, + &map->metric_buckets[hash % map->metric_bucket_count]) { + metric = cfl_list_entry(head, struct cmt_metric, _hash_head); + if (metric->hash == hash && + metric_labels_match(metric, labels_count, labels_val)) { + return metric; + } + } + } + + /* Decoders can populate the public metric list directly. Search only + * entries that have not yet been indexed, then index a successful match. */ cfl_list_foreach(head, &map->metrics) { metric = cfl_list_entry(head, struct cmt_metric, _head); - if (metric->hash == hash) { + if (!metric->hash_indexed && metric->hash == hash && + metric_labels_match(metric, labels_count, labels_val)) { + metric_index_add(map, metric); return metric; } } @@ -95,7 +274,7 @@ static struct cmt_metric *metric_hash_lookup(struct cmt_map *map, uint64_t hash) return NULL; } -static struct cmt_metric *map_metric_create(uint64_t hash, +static struct cmt_metric *map_metric_create(struct cmt_map *map, uint64_t hash, int labels_count, char **labels_val) { int i; @@ -109,8 +288,10 @@ static struct cmt_metric *map_metric_create(uint64_t hash, return NULL; } cfl_list_init(&metric->labels); + cfl_list_init(&metric->_hash_head); cmt_metric_set_double(metric, 0, 0.0); metric->hash = hash; + metric->map = map; for (i = 0; i < labels_count; i++) { label = malloc(sizeof(struct cmt_map_label)); @@ -137,16 +318,20 @@ static struct cmt_metric *map_metric_create(uint64_t hash, return metric; error: + destroy_label_list(&metric->labels); free(metric); return NULL; } -void cmt_map_metric_destroy(struct cmt_metric *metric) +static void map_metric_destroy_unlocked(struct cmt_metric *metric) { + struct cmt_map *map; struct cfl_list *tmp; struct cfl_list *head; struct cmt_map_label *label; + map = metric->map; + cfl_list_foreach_safe(head, tmp, &metric->labels) { label = cfl_list_entry(head, struct cmt_map_label, _head); cfl_sds_destroy(label->name); @@ -154,29 +339,49 @@ void cmt_map_metric_destroy(struct cmt_metric *metric) free(label); } - if (metric->hist_buckets) { - free(metric->hist_buckets); - } - if (metric->exp_hist_positive_buckets) { - free(metric->exp_hist_positive_buckets); - } - if (metric->exp_hist_negative_buckets) { - free(metric->exp_hist_negative_buckets); + metric_release_storage(metric); + + if (map != NULL && map->last_metric == metric) { + map->last_metric = NULL; } - if (metric->sum_quantiles) { - free(metric->sum_quantiles); + + if (metric->hash_indexed) { + if (map != NULL) { + if (map->indexed_metric_count > 0) { + map->indexed_metric_count--; + } + } + cfl_list_del(&metric->_hash_head); } cfl_list_del(&metric->_head); free(metric); } -struct cmt_metric *cmt_map_metric_get(struct cmt_opts *opts, struct cmt_map *map, - int labels_count, char **labels_val, - int write_op) +void cmt_map_metric_destroy(struct cmt_metric *metric) +{ + struct cmt_map *map; + + map = metric->map; + if (map != NULL) { + map_lock(map); + } + + map_metric_destroy_unlocked(metric); + + if (map != NULL) { + map_unlock(map); + } +} + +static struct cmt_metric *map_metric_get_unlocked(struct cmt_opts *opts, + struct cmt_map *map, + int labels_count, + char **labels_val, + int write_op) { int i; - int len; + size_t len; char *ptr; uint64_t hash; cfl_hash_state_t state; @@ -207,7 +412,7 @@ struct cmt_metric *cmt_map_metric_get(struct cmt_opts *opts, struct cmt_map *map } /* return the proper context or NULL */ - return metric; + return metric_prepare_storage(map, metric, write_op); } /* Lookup the metric */ @@ -225,10 +430,10 @@ struct cmt_metric *cmt_map_metric_get(struct cmt_opts *opts, struct cmt_map *map } hash = cfl_hash_64bits_digest(&state); - metric = metric_hash_lookup(map, hash); + metric = metric_hash_lookup(map, hash, labels_count, labels_val); if (metric) { - return metric; + return metric_prepare_storage(map, metric, write_op); } /* @@ -240,11 +445,27 @@ struct cmt_metric *cmt_map_metric_get(struct cmt_opts *opts, struct cmt_map *map } /* If the metric has not been found, just create it */ - metric = map_metric_create(hash, labels_count, labels_val); + metric = map_metric_create(map, hash, labels_count, labels_val); if (!metric) { return NULL; } cfl_list_add(&metric->_head, &map->metrics); + metric_index_add(map, metric); + map->last_metric = metric; + return metric_prepare_storage(map, metric, write_op); +} + +struct cmt_metric *cmt_map_metric_get(struct cmt_opts *opts, struct cmt_map *map, + int labels_count, char **labels_val, + int write_op) +{ + struct cmt_metric *metric; + + map_lock(map); + metric = map_metric_get_unlocked(opts, map, labels_count, labels_val, + write_op); + map_unlock(map); + return metric; } @@ -288,30 +509,15 @@ void cmt_map_destroy(struct cmt_map *map) if (map->metric_static_set) { metric = &map->metric; - if (map->type == CMT_HISTOGRAM) { - if (metric->hist_buckets) { - free(metric->hist_buckets); - } - } - else if (map->type == CMT_EXP_HISTOGRAM) { - if (metric->exp_hist_positive_buckets) { - free(metric->exp_hist_positive_buckets); - } - if (metric->exp_hist_negative_buckets) { - free(metric->exp_hist_negative_buckets); - } - } - else if (map->type == CMT_SUMMARY) { - if (metric->sum_quantiles) { - free(metric->sum_quantiles); - } - } + metric_release_storage(metric); } if (map->unit != NULL) { cfl_sds_destroy(map->unit); } + free(map->metric_buckets); + free(map); } @@ -338,3 +544,29 @@ void destroy_label_list(struct cfl_list *label_list) free(label); } } + +/* This function can be used to expire untouched metrics. + */ +void cmt_map_metrics_expire(struct cmt_map *map, uint64_t expiration) +{ + struct cfl_list *tmp; + struct cfl_list *head; + struct cmt_metric *metric; + + map_lock(map); + + if (map->metric_static_set && map->metric.timestamp < expiration) { + metric_release_storage(&map->metric); + memset(&map->metric, 0, sizeof(struct cmt_metric)); + cfl_list_init(&map->metric.labels); + map->metric_static_set = CMT_FALSE; + } + + cfl_list_foreach_safe(head, tmp, &map->metrics) { + metric = cfl_list_entry(head, struct cmt_metric, _head); + if (metric->timestamp < expiration) { + map_metric_destroy_unlocked(metric); + } + } + map_unlock(map); +} diff --git a/lib/cmetrics/src/cmt_mpack_utils.c b/lib/cmetrics/src/cmt_mpack_utils.c index ce639b356f7..eeac6f87b3d 100644 --- a/lib/cmetrics/src/cmt_mpack_utils.c +++ b/lib/cmetrics/src/cmt_mpack_utils.c @@ -114,6 +114,7 @@ int cmt_mpack_consume_uint_tag(mpack_reader_t *reader, uint64_t *output_buffer) int cmt_mpack_consume_string_tag(mpack_reader_t *reader, cfl_sds_t *output_buffer) { + const char *string_data; uint32_t string_length; mpack_tag_t tag; @@ -137,43 +138,39 @@ int cmt_mpack_consume_string_tag(mpack_reader_t *reader, cfl_sds_t *output_buffe string_length = mpack_tag_str_length(&tag); - /* This validation only applies to cmetrics and its use cases, we know - * for a fact that our label names and values are not supposed to be really - * long so a huge value here probably means that the data stream got corrupted. - */ - - if (CMT_MPACK_MAX_STRING_LENGTH < string_length) { - return CMT_MPACK_CORRUPT_INPUT_DATA_ERROR; - } - - *output_buffer = cfl_sds_create_size(string_length + 1); + /* Validate that the complete string is present before allocating memory + * based on its untrusted length field. The data-backed reader used by the + * decoder can return the bytes in place without an intermediate copy. */ + string_data = mpack_read_bytes_inplace(reader, (size_t) string_length); - if (NULL == *output_buffer) { - return CMT_MPACK_ALLOCATION_ERROR; + if (mpack_ok != mpack_reader_error(reader)) { + return CMT_MPACK_ENGINE_ERROR; } - cfl_sds_set_len(*output_buffer, string_length); - - mpack_read_cstr(reader, *output_buffer, string_length + 1, string_length); + mpack_done_str(reader); if (mpack_ok != mpack_reader_error(reader)) { - cfl_sds_destroy(*output_buffer); - - *output_buffer = NULL; - return CMT_MPACK_ENGINE_ERROR; } - mpack_done_str(reader); + /* CMetrics strings are represented as C strings. Preserve the existing + * rejection of embedded NUL bytes while allowing arbitrary valid lengths. */ + if (NULL != memchr(string_data, '\0', (size_t) string_length)) { + return CMT_MPACK_CORRUPT_INPUT_DATA_ERROR; + } - if (mpack_ok != mpack_reader_error(reader)) { - cfl_sds_destroy(*output_buffer); + *output_buffer = cfl_sds_create_size((size_t) string_length); - *output_buffer = NULL; + if (NULL == *output_buffer) { + return CMT_MPACK_ALLOCATION_ERROR; + } - return CMT_MPACK_ENGINE_ERROR; + if (string_length > 0) { + memcpy(*output_buffer, string_data, (size_t) string_length); } + cfl_sds_set_len(*output_buffer, (size_t) string_length); + return CMT_MPACK_SUCCESS; } diff --git a/lib/cmetrics/src/cmt_summary.c b/lib/cmetrics/src/cmt_summary.c index 014ad0bbd02..ab117ddf51a 100644 --- a/lib/cmetrics/src/cmt_summary.c +++ b/lib/cmetrics/src/cmt_summary.c @@ -94,7 +94,7 @@ struct cmt_summary *cmt_summary_create(struct cmt *cmt, if (quantiles_count > 0) { s->quantiles_count = quantiles_count; s->quantiles = calloc(1, sizeof(double) * quantiles_count); - if (!s->quantiles_count) { + if (!s->quantiles) { cmt_errno(); cmt_summary_destroy(s); return NULL; @@ -130,7 +130,8 @@ double cmt_summary_quantile_get_value(struct cmt_metric *metric, int quantile_id { uint64_t val; - if (quantile_id < 0 /*|| quantile_id > metric->sum_quantiles_count*/) { + if (metric == NULL || metric->sum_quantiles == NULL || quantile_id < 0 || + (size_t) quantile_id >= metric->sum_quantiles_count) { return 0; } @@ -220,6 +221,11 @@ void cmt_summary_quantile_set(struct cmt_metric *metric, uint64_t timestamp, double new; int result; + if (metric == NULL || metric->sum_quantiles == NULL || quantile_id < 0 || + (size_t) quantile_id >= metric->sum_quantiles_count) { + return; + } + do { old = cmt_summary_quantile_get_value(metric, quantile_id); new = val; @@ -303,4 +309,4 @@ int cmt_summary_set_default(struct cmt_summary *summary, cmt_summary_count_set(metric, timestamp, count); return 0; -} \ No newline at end of file +} diff --git a/lib/cmetrics/src/cmt_untyped.c b/lib/cmetrics/src/cmt_untyped.c index 2df6ea1413a..fbe4cd4d291 100644 --- a/lib/cmetrics/src/cmt_untyped.c +++ b/lib/cmetrics/src/cmt_untyped.c @@ -96,7 +96,7 @@ int cmt_untyped_destroy(struct cmt_untyped *untyped) return 0; } -/* Set untyped value, new value cannot be smaller than current value */ +/* Set untyped value */ int cmt_untyped_set(struct cmt_untyped *untyped, uint64_t timestamp, double val, int labels_count, char **label_vals) { @@ -112,9 +112,6 @@ int cmt_untyped_set(struct cmt_untyped *untyped, uint64_t timestamp, double val, return -1; } - if (cmt_metric_get_value(metric) > val) { - return -1; - } cmt_metric_set(metric, timestamp, val); return 0; } diff --git a/lib/cmetrics/tests/CMakeLists.txt b/lib/cmetrics/tests/CMakeLists.txt index 40108aad95a..e7ae66d67e3 100644 --- a/lib/cmetrics/tests/CMakeLists.txt +++ b/lib/cmetrics/tests/CMakeLists.txt @@ -15,6 +15,9 @@ set(UNIT_TESTS_FILES filter.c exp_histogram.c msgpack_abi.c + msgpack_temporality.c + format_conversion.c + expire.c ) if (CMT_BUILD_PROMETHEUS_TEXT_DECODER) diff --git a/lib/cmetrics/tests/cat.c b/lib/cmetrics/tests/cat.c index dce4385cee7..86469bdeedd 100644 --- a/lib/cmetrics/tests/cat.c +++ b/lib/cmetrics/tests/cat.c @@ -24,6 +24,7 @@ #include #include #include +#include #include #include #include @@ -668,6 +669,123 @@ void test_exp_histogram_preserves_aggregation_type() cmt_destroy(dst); } +void test_summary_concatenation_preserves_series() +{ + int ret; + int found_first; + int found_second; + double quantiles[] = {0.5, 0.9}; + double values[] = {5.0, 9.0}; + struct cfl_list *head; + struct cfl_list *label_head; + struct cmt *dst; + struct cmt *src_first; + struct cmt *src_second; + struct cmt_summary *summary; + struct cmt_metric *metric; + struct cmt_map_label *label; + + dst = cmt_create(); + src_first = cmt_create(); + src_second = cmt_create(); + TEST_ASSERT(dst != NULL); + TEST_ASSERT(src_first != NULL); + TEST_ASSERT(src_second != NULL); + + summary = cmt_summary_create(src_first, "test", "cat", "summary", + "Summary concatenation", 2, quantiles, + 1, (char *[]) {"kind"}); + TEST_ASSERT(summary != NULL); + ret = cmt_summary_set_default(summary, 10, values, 14.0, 7, + 1, (char *[]) {"first"}); + TEST_ASSERT(ret == 0); + + ret = cmt_cat(dst, src_first); + TEST_ASSERT(ret == 0); + TEST_CHECK(cfl_list_size(&dst->summaries) == 1); + + summary = cfl_list_entry_first(&dst->summaries, + struct cmt_summary, _head); + TEST_CHECK(cfl_list_size(&summary->map->metrics) == 1); + + metric = cfl_list_entry_first(&summary->map->metrics, + struct cmt_metric, _head); + label = cfl_list_entry_first(&metric->labels, + struct cmt_map_label, _head); + TEST_CHECK(strcmp(label->name, "first") == 0); + TEST_CHECK(cmt_summary_get_count_value(metric) == 7); + + summary = cmt_summary_create(src_second, "test", "cat", "summary", + "Summary concatenation", 2, quantiles, + 1, (char *[]) {"kind"}); + TEST_ASSERT(summary != NULL); + ret = cmt_summary_set_default(summary, 20, values, 18.0, 9, + 1, (char *[]) {"second"}); + TEST_ASSERT(ret == 0); + + ret = cmt_cat(dst, src_second); + TEST_ASSERT(ret == 0); + TEST_CHECK(cfl_list_size(&dst->summaries) == 1); + + summary = cfl_list_entry_first(&dst->summaries, + struct cmt_summary, _head); + TEST_CHECK(cfl_list_size(&summary->map->metrics) == 2); + + found_first = CMT_FALSE; + found_second = CMT_FALSE; + cfl_list_foreach(head, &summary->map->metrics) { + metric = cfl_list_entry(head, struct cmt_metric, _head); + label_head = metric->labels.next; + label = cfl_list_entry(label_head, struct cmt_map_label, _head); + + TEST_CHECK(strcmp(label->name, "kind") != 0); + if (strcmp(label->name, "first") == 0) { + found_first = CMT_TRUE; + } + else if (strcmp(label->name, "second") == 0) { + found_second = CMT_TRUE; + } + } + + TEST_CHECK(found_first == CMT_TRUE); + TEST_CHECK(found_second == CMT_TRUE); + + cmt_destroy(src_second); + cmt_destroy(src_first); + cmt_destroy(dst); +} + +void test_summary_concatenation_rejects_mismatched_label_schema() +{ + int ret; + double quantiles[] = {0.5, 0.9}; + struct cmt *dst; + struct cmt *src; + struct cmt_summary *summary; + + dst = cmt_create(); + src = cmt_create(); + TEST_ASSERT(dst != NULL); + TEST_ASSERT(src != NULL); + + summary = cmt_summary_create(dst, "test", "cat", "schema", + "schema validation", 2, quantiles, 2, + (char *[]) {"method", "status"}); + TEST_ASSERT(summary != NULL); + + summary = cmt_summary_create(src, "test", "cat", "schema", + "schema validation", 2, quantiles, 2, + (char *[]) {"status", "method"}); + TEST_ASSERT(summary != NULL); + + ret = cmt_cat(dst, src); + TEST_CHECK(ret == -1); + TEST_CHECK(cfl_list_size(&dst->summaries) == 1); + + cmt_destroy(src); + cmt_destroy(dst); +} + TEST_LIST = { {"cat", test_cat}, {"duplicate_metrics", test_duplicate_metrics}, @@ -676,5 +794,8 @@ TEST_LIST = { {"histogram_empty_to_populated", test_histogram_empty_to_populated}, {"histogram_populated_to_empty", test_histogram_populated_to_empty}, {"exp_histogram_preserves_aggregation_type", test_exp_histogram_preserves_aggregation_type}, + {"summary_concatenation_preserves_series", test_summary_concatenation_preserves_series}, + {"summary_concatenation_rejects_mismatched_label_schema", + test_summary_concatenation_rejects_mismatched_label_schema}, { 0 } }; diff --git a/lib/cmetrics/tests/counter.c b/lib/cmetrics/tests/counter.c index cfe83c6cff0..25faf43f538 100644 --- a/lib/cmetrics/tests/counter.c +++ b/lib/cmetrics/tests/counter.c @@ -21,9 +21,14 @@ #include #include #include +#include #include #include +#if !defined(_WIN32) && !defined(_WIN64) +#include +#endif + #include "cmt_tests.h" static struct cmt *generate_encoder_test_data() @@ -243,6 +248,20 @@ void test_labels() TEST_CHECK(ret == 0); TEST_CHECK(val == 10.55); + /* Label boundaries must be part of identity. These two tuples feed the + * same unframed byte sequence ("abc") into the hash. */ + ret = cmt_counter_add(c, ts, 4.0, 2, (char *[]) {"ab", "c"}); + TEST_CHECK(ret == 0); + ret = cmt_counter_add(c, ts, 7.0, 2, (char *[]) {"a", "bc"}); + TEST_CHECK(ret == 0); + + ret = cmt_counter_get_val(c, 2, (char *[]) {"ab", "c"}, &val); + TEST_CHECK(ret == 0); + TEST_CHECK(val == 4.0); + ret = cmt_counter_get_val(c, 2, (char *[]) {"a", "bc"}, &val); + TEST_CHECK(ret == 0); + TEST_CHECK(val == 7.0); + /* Valid counter set */ ret = cmt_counter_set(c, ts, 12.15, 2, (char *[]) {"localhost", "test"}); TEST_CHECK(ret == 0); @@ -254,11 +273,76 @@ void test_labels() cmt_destroy(cmt); } +#if !defined(_WIN32) && !defined(_WIN64) +#define CONCURRENT_THREAD_COUNT 8 +#define CONCURRENT_UPDATE_COUNT 10000 + +struct concurrent_counter_context { + struct cmt_counter *counter; + int result; +}; + +static void *concurrent_counter_worker(void *data) +{ + int index; + struct concurrent_counter_context *context = data; + + for (index = 0; index < CONCURRENT_UPDATE_COUNT; index++) { + if (cmt_counter_inc(context->counter, (uint64_t) index + 1, 1, + (char *[]) {"shared"}) != 0) { + context->result = -1; + break; + } + } + + return NULL; +} + +void test_concurrent_metric_creation() +{ + int index; + int result; + double value; + pthread_t threads[CONCURRENT_THREAD_COUNT]; + struct concurrent_counter_context contexts[CONCURRENT_THREAD_COUNT]; + struct cmt *cmt; + struct cmt_counter *counter; + + cmt = cmt_create(); + TEST_ASSERT(cmt != NULL); + counter = cmt_counter_create(cmt, "test", "", "concurrent", "help", + 1, (char *[]) {"series"}); + TEST_ASSERT(counter != NULL); + + for (index = 0; index < CONCURRENT_THREAD_COUNT; index++) { + contexts[index].counter = counter; + contexts[index].result = 0; + result = pthread_create(&threads[index], NULL, + concurrent_counter_worker, &contexts[index]); + TEST_ASSERT(result == 0); + } + for (index = 0; index < CONCURRENT_THREAD_COUNT; index++) { + pthread_join(threads[index], NULL); + TEST_CHECK(contexts[index].result == 0); + } + + TEST_CHECK(cfl_list_size(&counter->map->metrics) == 1); + result = cmt_counter_get_val(counter, 1, (char *[]) {"shared"}, &value); + TEST_CHECK(result == 0); + TEST_CHECK(value == CONCURRENT_THREAD_COUNT * CONCURRENT_UPDATE_COUNT); + + cmt_destroy(cmt); +} +#endif + TEST_LIST = { {"basic", test_counter}, {"labels", test_labels}, {"msgpack", test_msgpack}, {"prometheus", test_prometheus}, {"text", test_text}, +#if !defined(_WIN32) && !defined(_WIN64) + {"concurrent_metric_creation", test_concurrent_metric_creation}, +#endif { 0 } }; diff --git a/lib/cmetrics/tests/data/issue_274.txt b/lib/cmetrics/tests/data/issue_274.txt new file mode 100644 index 00000000000..fb34a291767 --- /dev/null +++ b/lib/cmetrics/tests/data/issue_274.txt @@ -0,0 +1,3 @@ +# HELP node_zfs_arc_memory_available_bytes kstat.zfs.misc.arcstats.memory_available_bytes +# TYPE node_zfs_arc_memory_available_bytes untyped +node_zfs_arc_memory_available_bytes -8.58914944e+08 diff --git a/lib/cmetrics/tests/encoding.c b/lib/cmetrics/tests/encoding.c index 36c861b444e..24fea1ad12f 100644 --- a/lib/cmetrics/tests/encoding.c +++ b/lib/cmetrics/tests/encoding.c @@ -978,6 +978,201 @@ void test_prometheus_remote_write_with_outdated_timestamps() cmt_destroy(cmt); } +static int remote_write_contains_series(Prometheus__WriteRequest *request, + const char *metric_name, + const char *label_name, + const char *label_value) +{ + size_t series_index; + size_t label_index; + int metric_matches; + int label_matches; + Prometheus__Label *label; + + for (series_index = 0; series_index < request->n_timeseries; series_index++) { + metric_matches = CMT_FALSE; + label_matches = label_name == NULL; + + for (label_index = 0; + label_index < request->timeseries[series_index]->n_labels; + label_index++) { + label = request->timeseries[series_index]->labels[label_index]; + if (label == NULL || label->name == NULL || label->value == NULL) { + continue; + } + + if (strcmp(label->name, "__name__") == 0 && + strcmp(label->value, metric_name) == 0) { + metric_matches = CMT_TRUE; + } + + if (label_name != NULL && strcmp(label->name, label_name) == 0 && + strcmp(label->value, label_value) == 0) { + label_matches = CMT_TRUE; + } + } + + if (metric_matches && label_matches) { + return CMT_TRUE; + } + } + + return CMT_FALSE; +} + +static Prometheus__WriteRequest *decode_remote_write_payload(cfl_sds_t payload) +{ + if (payload == NULL) { + return NULL; + } + + return prometheus__write_request__unpack(NULL, cfl_sds_len(payload), + (uint8_t *) payload); +} + +void test_prometheus_remote_write_skips_only_stale_samples() +{ + uint64_t now; + cfl_sds_t payload; + struct cmt *cmt; + struct cmt_counter *counter; + Prometheus__WriteRequest *request; + + now = cfl_time_now(); + cmt = cmt_create(); + TEST_ASSERT(cmt != NULL); + + counter = cmt_counter_create(cmt, "test", "remote", "mixed", + "Mixed timestamp samples", 1, + (char *[]) {"state"}); + TEST_ASSERT(counter != NULL); + TEST_ASSERT(cmt_counter_set(counter, + now - CMT_ENCODE_PROMETHEUS_REMOTE_WRITE_CUTOFF_THRESHOLD - 1, + 1, 1, (char *[]) {"stale"}) == 0); + TEST_ASSERT(cmt_counter_set(counter, now, 2, 1, + (char *[]) {"fresh"}) == 0); + + payload = cmt_encode_prometheus_remote_write_create(cmt); + request = decode_remote_write_payload(payload); + TEST_ASSERT(request != NULL); + TEST_CHECK(remote_write_contains_series(request, "test_remote_mixed", + "state", "stale") == CMT_FALSE); + TEST_CHECK(remote_write_contains_series(request, "test_remote_mixed", + "state", "fresh") == CMT_TRUE); + + prometheus__write_request__free_unpacked(request, NULL); + cmt_encode_prometheus_remote_write_destroy(payload); + cmt_destroy(cmt); +} + +void test_prometheus_remote_write_continues_after_stale_family() +{ + uint64_t now; + cfl_sds_t payload; + struct cmt *cmt; + struct cmt_counter *counter; + struct cmt_gauge *gauge; + Prometheus__WriteRequest *request; + + now = cfl_time_now(); + cmt = cmt_create(); + TEST_ASSERT(cmt != NULL); + + counter = cmt_counter_create(cmt, "test", "remote", "stale_counter", + "Stale counter", 0, NULL); + TEST_ASSERT(counter != NULL); + TEST_ASSERT(cmt_counter_set(counter, + now - CMT_ENCODE_PROMETHEUS_REMOTE_WRITE_CUTOFF_THRESHOLD - 1, + 1, 0, NULL) == 0); + + gauge = cmt_gauge_create(cmt, "test", "remote", "fresh_gauge", + "Fresh gauge", 0, NULL); + TEST_ASSERT(gauge != NULL); + TEST_ASSERT(cmt_gauge_set(gauge, now, 2, 0, NULL) == 0); + + payload = cmt_encode_prometheus_remote_write_create(cmt); + request = decode_remote_write_payload(payload); + TEST_ASSERT(request != NULL); + TEST_CHECK(remote_write_contains_series(request, "test_remote_stale_counter", + NULL, NULL) == CMT_FALSE); + TEST_CHECK(remote_write_contains_series(request, "test_remote_fresh_gauge", + NULL, NULL) == CMT_TRUE); + + prometheus__write_request__free_unpacked(request, NULL); + cmt_encode_prometheus_remote_write_destroy(payload); + cmt_destroy(cmt); +} + +void test_prometheus_remote_write_preserves_future_samples() +{ + uint64_t now; + cfl_sds_t payload; + struct cmt *cmt; + struct cmt_counter *counter; + Prometheus__WriteRequest *request; + + now = cfl_time_now(); + cmt = cmt_create(); + TEST_ASSERT(cmt != NULL); + + counter = cmt_counter_create(cmt, "test", "remote", "future", + "Future timestamp", 0, NULL); + TEST_ASSERT(counter != NULL); + TEST_ASSERT(cmt_counter_set(counter, now + 1000000000, 1, 0, NULL) == 0); + + payload = cmt_encode_prometheus_remote_write_create(cmt); + request = decode_remote_write_payload(payload); + TEST_ASSERT(request != NULL); + TEST_CHECK(remote_write_contains_series(request, "test_remote_future", + NULL, NULL) == CMT_TRUE); + + prometheus__write_request__free_unpacked(request, NULL); + cmt_encode_prometheus_remote_write_destroy(payload); + cmt_destroy(cmt); +} + +void test_prometheus_remote_write_skips_only_stale_histograms() +{ + uint64_t now; + cfl_sds_t payload; + struct cmt *cmt; + struct cmt_histogram *histogram; + struct cmt_histogram_buckets *buckets; + Prometheus__WriteRequest *request; + + now = cfl_time_now(); + cmt = cmt_create(); + TEST_ASSERT(cmt != NULL); + + buckets = cmt_histogram_buckets_create(1, 1.0); + TEST_ASSERT(buckets != NULL); + histogram = cmt_histogram_create(cmt, "test", "remote", "mixed_histogram", + "Mixed histogram timestamps", buckets, + 1, (char *[]) {"state"}); + TEST_ASSERT(histogram != NULL); + + TEST_ASSERT(cmt_histogram_observe( + histogram, + now - CMT_ENCODE_PROMETHEUS_REMOTE_WRITE_CUTOFF_THRESHOLD - 1, + 0.5, 1, (char *[]) {"stale"}) == 0); + TEST_ASSERT(cmt_histogram_observe(histogram, now, 0.5, 1, + (char *[]) {"fresh"}) == 0); + + payload = cmt_encode_prometheus_remote_write_create(cmt); + request = decode_remote_write_payload(payload); + TEST_ASSERT(request != NULL); + TEST_CHECK(remote_write_contains_series( + request, "test_remote_mixed_histogram_bucket", + "state", "stale") == CMT_FALSE); + TEST_CHECK(remote_write_contains_series( + request, "test_remote_mixed_histogram_bucket", + "state", "fresh") == CMT_TRUE); + + prometheus__write_request__free_unpacked(request, NULL); + cmt_encode_prometheus_remote_write_destroy(payload); + cmt_destroy(cmt); +} + void test_opentelemetry() { cfl_sds_t payload; @@ -1615,6 +1810,10 @@ TEST_LIST = { {"cmt_msgpack_partial_processing", test_cmt_msgpack_partial_processing}, {"prometheus_remote_write", test_prometheus_remote_write}, {"prometheus_remote_write_old_cmt",test_prometheus_remote_write_with_outdated_timestamps}, + {"prometheus_remote_write_skips_only_stale_samples", test_prometheus_remote_write_skips_only_stale_samples}, + {"prometheus_remote_write_continues_after_stale_family", test_prometheus_remote_write_continues_after_stale_family}, + {"prometheus_remote_write_preserves_future_samples", test_prometheus_remote_write_preserves_future_samples}, + {"prometheus_remote_write_skips_only_stale_histograms", test_prometheus_remote_write_skips_only_stale_histograms}, {"cmt_msgpack_stability", test_cmt_to_msgpack_stability}, {"cmt_msgpack_integrity", test_cmt_to_msgpack_integrity}, {"cmt_msgpack_labels", test_cmt_to_msgpack_labels}, diff --git a/lib/cmetrics/tests/expire.c b/lib/cmetrics/tests/expire.c new file mode 100644 index 00000000000..eaa649dcc62 --- /dev/null +++ b/lib/cmetrics/tests/expire.c @@ -0,0 +1,396 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* CMetrics + * ======== + * Copyright 2021-2022 The CMetrics Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "cmt_tests.h" + +void test_expire_counter() +{ + uint64_t ts; + struct cmt *cmt; + struct cmt_counter *c; + + cmt_initialize(); + + cmt = cmt_create(); + TEST_CHECK(cmt != NULL); + + /* Create a counter metric type */ + c = cmt_counter_create(cmt, "k8s", "network", "uptime", "Network Uptime", 1, (char *[]) {"host"}); + TEST_CHECK(c != NULL); + + /* Timestamp */ + ts = cfl_time_now(); + + cmt_counter_inc(c, ts, 1, (char *[]){"valid"}); + cmt_counter_inc(c, ts-10, 1, (char *[]){"expire"}); + + TEST_CHECK(cfl_list_size(&c->map->metrics) == 2); + cmt_expire(cmt, ts-1); + TEST_CHECK(cfl_list_size(&c->map->metrics) == 1); + + cmt_destroy(cmt); +} + +void test_expire_gauge() +{ + uint64_t ts; + struct cmt *cmt; + struct cmt_gauge *g; + + cmt_initialize(); + + cmt = cmt_create(); + TEST_CHECK(cmt != NULL); + + /* Create a gauge metric type */ + g = cmt_gauge_create(cmt, "k8s", "network", "load", "Network load", 1, (char *[]) {"host"}); + TEST_CHECK(g != NULL); + + /* Timestamp */ + ts = cfl_time_now(); + + cmt_gauge_set(g, ts, 50, 1, (char *[]){"valid"}); + cmt_gauge_set(g, ts-10, 50, 1, (char *[]){"expire"}); + + TEST_CHECK(cfl_list_size(&g->map->metrics) == 2); + cmt_expire(cmt, ts-1); + TEST_CHECK(cfl_list_size(&g->map->metrics) == 1); + + cmt_destroy(cmt); +} + +void test_expire_histogram() +{ + uint64_t ts; + struct cmt *cmt; + struct cmt_histogram *h; + struct cmt_histogram_buckets *buckets; + + cmt_initialize(); + + cmt = cmt_create(); + TEST_CHECK(cmt != NULL); + + /* Create buckets */ + buckets = cmt_histogram_buckets_create(11, + 0.005, 0.01, 0.025, 0.05, + 0.1, 0.25, 0.5, 1.0, 2.5, + 5.0, 10.0); + TEST_CHECK(buckets != NULL); + + /* Create a histogram metric type */ + h = cmt_histogram_create(cmt, + "k8s", "network", "uptime", "Network Uptime", + buckets, + 1, (char *[]) {"host"}); + TEST_CHECK(h != NULL); + + /* Timestamp */ + ts = cfl_time_now(); + + cmt_histogram_observe(h, ts, 1.0, 1, (char *[]){"valid"}); + cmt_histogram_observe(h, ts-10, 1.0, 1, (char *[]){"expire"}); + + TEST_CHECK(cfl_list_size(&h->map->metrics) == 2); + cmt_expire(cmt, ts-1); + TEST_CHECK(cfl_list_size(&h->map->metrics) == 1); + + cmt_destroy(cmt); +} + +void test_expire_summary() +{ + uint64_t ts; + struct cmt *cmt; + struct cmt_summary *s; + double quantiles[6]; + double revised[6]; + double sum; + uint64_t count; + + /* set quantiles, no labels */ + quantiles[0] = 0.1; + quantiles[1] = 0.2; + quantiles[2] = 0.3; + quantiles[3] = 0.4; + quantiles[4] = 0.5; + quantiles[5] = 1.0; + + revised[0] = 1.0; + revised[1] = 2.0; + revised[2] = 3.0; + revised[3] = 4.0; + revised[4] = 5.0; + revised[5] = 6.0; + + count = 10; + sum = 51.612894511314444; + + cmt_initialize(); + + cmt = cmt_create(); + TEST_CHECK(cmt != NULL); + + /* Create a summary metric type */ + s = cmt_summary_create(cmt, + "k8s", "network", "uptime", "Network Uptime", + 6, + quantiles, + 1, (char *[]) {"host"}); + TEST_CHECK(s != NULL); + + /* Timestamp */ + ts = cfl_time_now(); + + cmt_summary_set_default(s, ts, revised, sum, count, 1, (char *[]){"valid"}); + cmt_summary_set_default(s, ts-10, revised, sum, count, 1, (char *[]){"expire"}); + + TEST_CHECK(cfl_list_size(&s->map->metrics) == 2); + cmt_expire(cmt, ts-1); + TEST_CHECK(cfl_list_size(&s->map->metrics) == 1); + + cmt_destroy(cmt); +} + +void test_expire_untyped() +{ + uint64_t ts; + struct cmt *cmt; + struct cmt_untyped *u; + + cmt_initialize(); + + cmt = cmt_create(); + TEST_CHECK(cmt != NULL); + + u = cmt_untyped_create(cmt, "cmetrics", "test", "cat_untyped", "first untyped", + 2, (char *[]) {"label5", "label6"}); + TEST_CHECK(u != NULL); + + /* Timestamp */ + ts = cfl_time_now(); + + cmt_untyped_set(u, ts, 1.3, 2, (char *[]) {"first", "valid"}); + cmt_untyped_set(u, ts-10, 1.3, 2, (char *[]) {"second", "expire"}); + + TEST_CHECK(cfl_list_size(&u->map->metrics) == 2); + cmt_expire(cmt, ts-1); + TEST_CHECK(cfl_list_size(&u->map->metrics) == 1); + + cmt_destroy(cmt); +} + +void test_epxire_exp_histogram() +{ + uint64_t ts; + struct cmt *cmt; + uint64_t positive[3] = {3, 5, 7}; + uint64_t negative[2] = {2, 1}; + int result; + struct cmt_exp_histogram *exp_histogram; + + cmt_initialize(); + + cmt = cmt_create(); + TEST_CHECK(cmt != NULL); + + exp_histogram = cmt_exp_histogram_create(cmt, + "cm", "native", "exp_hist", "native exp histogram", + 1, (char *[]) {"endpoint"}); + TEST_CHECK(exp_histogram != NULL); + + if (exp_histogram == NULL) { + return; + } + + /* Timestamp */ + ts = cfl_time_now(); + + result = cmt_exp_histogram_set_default(exp_histogram, + ts, + 2, + 11, + 0.0, + -2, + 3, + positive, + -1, + 2, + negative, + CMT_TRUE, + 42.25, + 29, + 1, + (char *[]) {"api"}); + TEST_CHECK(result == 0); + result = cmt_exp_histogram_set_default(exp_histogram, + ts-10, + 2, + 11, + 0.0, + -2, + 3, + positive, + -1, + 2, + negative, + CMT_TRUE, + 42.25, + 29, + 1, + (char *[]) {"http"}); + TEST_CHECK(result == 0); + + TEST_CHECK(cfl_list_size(&exp_histogram->map->metrics) == 2); + cmt_expire(cmt, ts-1); + TEST_CHECK(cfl_list_size(&exp_histogram->map->metrics) == 1); + + cmt_destroy(cmt); +} + +void test_expire_off_by_one() +{ + uint64_t ts; + struct cmt *cmt; + struct cmt_counter *c; + + cmt_initialize(); + + cmt = cmt_create(); + TEST_CHECK(cmt != NULL); + + /* Create a counter metric type */ + c = cmt_counter_create(cmt, "k8s", "network", "uptime", "Network Uptime", 1, (char *[]) {"host"}); + TEST_CHECK(c != NULL); + + /* Timestamp */ + ts = cfl_time_now(); + + cmt_counter_inc(c, ts, 1, (char *[]){"fullyvalid"}); + cmt_counter_inc(c, ts-1, 1, (char *[]){"bordervalid"}); + cmt_counter_inc(c, ts-2, 1, (char *[]){"borderexpire"}); + + TEST_CHECK(cfl_list_size(&c->map->metrics) == 3); + cmt_expire(cmt, ts-2); + TEST_CHECK(cfl_list_size(&c->map->metrics) == 3); + cmt_expire(cmt, ts-1); + TEST_CHECK(cfl_list_size(&c->map->metrics) == 2); + cmt_expire(cmt, ts); + TEST_CHECK(cfl_list_size(&c->map->metrics) == 1); + cmt_expire(cmt, ts+1); + TEST_CHECK(cfl_list_size(&c->map->metrics) == 0); + + cmt_destroy(cmt); +} + +void test_expire_static_metrics() +{ + uint64_t ts; + struct cmt *cmt; + struct cmt_counter *counter; + struct cmt_histogram *histogram; + struct cmt_histogram_buckets *buckets; + + cmt = cmt_create(); + TEST_ASSERT(cmt != NULL); + ts = cfl_time_now(); + + counter = cmt_counter_create(cmt, "test", "expire", "static_counter", + "Static counter expiration", 0, NULL); + TEST_ASSERT(counter != NULL); + TEST_ASSERT(cmt_counter_set(counter, ts - 10, 3, 0, NULL) == 0); + + buckets = cmt_histogram_buckets_create(2, 1.0, 5.0); + TEST_ASSERT(buckets != NULL); + histogram = cmt_histogram_create(cmt, "test", "expire", "static_histogram", + "Static histogram expiration", buckets, + 0, NULL); + TEST_ASSERT(histogram != NULL); + TEST_ASSERT(cmt_histogram_observe(histogram, ts - 10, 2.0, + 0, NULL) == 0); + TEST_CHECK(histogram->map->metric.hist_buckets != NULL); + + cmt_expire(cmt, ts - 1); + + TEST_CHECK(counter->map->metric_static_set == CMT_FALSE); + TEST_CHECK(histogram->map->metric_static_set == CMT_FALSE); + TEST_CHECK(histogram->map->metric.hist_buckets == NULL); + + TEST_ASSERT(cmt_counter_set(counter, ts, 4, 0, NULL) == 0); + TEST_ASSERT(cmt_histogram_observe(histogram, ts, 2.0, 0, NULL) == 0); + TEST_CHECK(counter->map->metric_static_set == CMT_TRUE); + TEST_CHECK(histogram->map->metric_static_set == CMT_TRUE); + TEST_CHECK(histogram->map->metric.hist_buckets != NULL); + TEST_CHECK(cmt_metric_get_timestamp(&counter->map->metric) == ts); + TEST_CHECK(cmt_metric_get_timestamp(&histogram->map->metric) == ts); + + cmt_destroy(cmt); +} + +void test_destroy_unindexed_last_metric() +{ + struct cmt *cmt; + struct cmt_counter *counter; + struct cmt_metric *metric; + + cmt = cmt_create(); + TEST_ASSERT(cmt != NULL); + counter = cmt_counter_create(cmt, "test", "map", "unindexed", + "Unindexed metric destruction", 1, + (char *[]) {"label"}); + TEST_ASSERT(counter != NULL); + + metric = calloc(1, sizeof(struct cmt_metric)); + TEST_ASSERT(metric != NULL); + cfl_list_init(&metric->labels); + cfl_list_init(&metric->_hash_head); + metric->map = counter->map; + cfl_list_add(&metric->_head, &counter->map->metrics); + counter->map->last_metric = metric; + + cmt_map_metric_destroy(metric); + + TEST_CHECK(counter->map->last_metric == NULL); + TEST_CHECK(cfl_list_size(&counter->map->metrics) == 0); + + cmt_destroy(cmt); +} + +TEST_LIST = { + {"expire_counter" , test_expire_counter}, + {"expire_gauge", test_expire_gauge}, + {"expire_histogram", test_expire_histogram}, + {"expire_summary", test_expire_summary}, + {"expire_untyped", test_expire_untyped}, + {"expire_exp_histogram", test_epxire_exp_histogram}, + {"expire_off_by_one", test_expire_off_by_one}, + {"expire_static_metrics", test_expire_static_metrics}, + {"destroy_unindexed_last_metric", test_destroy_unindexed_last_metric}, + { 0 } +}; diff --git a/lib/cmetrics/tests/filter.c b/lib/cmetrics/tests/filter.c index 45248fe5aa5..e4f330b0b66 100644 --- a/lib/cmetrics/tests/filter.c +++ b/lib/cmetrics/tests/filter.c @@ -452,9 +452,60 @@ void test_filter_with_label_key_value_pairs() cmt_destroy(cmt6); } +void test_filter_label_pair_exact_and_all_matches() +{ + int ret; + cfl_sds_t text; + struct cmt *src; + struct cmt *prefix_result; + struct cmt *exact_result; + struct cmt_counter *counter; + + src = cmt_create(); + prefix_result = cmt_create(); + exact_result = cmt_create(); + TEST_ASSERT(src != NULL); + TEST_ASSERT(prefix_result != NULL); + TEST_ASSERT(exact_result != NULL); + + counter = cmt_counter_create(src, "test", "filter", "requests", + "Filter exact label pairs", 2, + (char *[]) {"host", "zone"}); + TEST_ASSERT(counter != NULL); + + TEST_ASSERT(cmt_counter_set(counter, 1, 1, 2, + (char *[]) {"prod", "a"}) == 0); + TEST_ASSERT(cmt_counter_set(counter, 1, 2, 2, + (char *[]) {"prod", "b"}) == 0); + TEST_ASSERT(cmt_counter_set(counter, 1, 3, 2, + (char *[]) {"production", "c"}) == 0); + + ret = cmt_filter_with_label_pair(prefix_result, src, "hostname", "prod"); + TEST_ASSERT(ret == CMT_FILTER_SUCCESS); + text = cmt_encode_text_create(prefix_result); + TEST_ASSERT(text != NULL); + TEST_CHECK(strstr(text, "host=\"prod\",zone=\"a\"") != NULL); + TEST_CHECK(strstr(text, "host=\"prod\",zone=\"b\"") != NULL); + TEST_CHECK(strstr(text, "host=\"production\",zone=\"c\"") != NULL); + cmt_encode_text_destroy(text); + + ret = cmt_filter_with_label_pair(exact_result, src, "host", "prod"); + TEST_ASSERT(ret == CMT_FILTER_SUCCESS); + text = cmt_encode_text_create(exact_result); + TEST_ASSERT(text != NULL); + TEST_CHECK(strstr(text, "host=\"prod\"") == NULL); + TEST_CHECK(strstr(text, "host=\"production\",zone=\"c\"") != NULL); + cmt_encode_text_destroy(text); + + cmt_destroy(exact_result); + cmt_destroy(prefix_result); + cmt_destroy(src); +} + TEST_LIST = { {"filter", test_filter}, {"filter_with_label_pair", test_filter_with_label_key_value_pairs}, + {"filter_label_pair_exact_and_all_matches", test_filter_label_pair_exact_and_all_matches}, { 0 } }; diff --git a/lib/cmetrics/tests/format_conversion.c b/lib/cmetrics/tests/format_conversion.c new file mode 100644 index 00000000000..72e1e695470 --- /dev/null +++ b/lib/cmetrics/tests/format_conversion.c @@ -0,0 +1,440 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* CMetrics + * ======== + * Copyright 2026 The CMetrics Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef CMT_HAVE_PROMETHEUS_TEXT_DECODER +#include +#endif + +#include "cmt_tests.h" + +static struct cmt *msgpack_round_trip(struct cmt *input) +{ + int result; + char *first_buffer; + char *second_buffer; + size_t first_size; + size_t second_size; + size_t offset; + struct cmt *output; + + first_buffer = NULL; + second_buffer = NULL; + output = NULL; + + result = cmt_encode_msgpack_create(input, &first_buffer, &first_size); + TEST_ASSERT(result == 0); + + offset = 0; + result = cmt_decode_msgpack_create(&output, first_buffer, first_size, + &offset); + TEST_ASSERT(result == CMT_DECODE_MSGPACK_SUCCESS); + TEST_ASSERT(output != NULL); + TEST_CHECK(offset == first_size); + + result = cmt_encode_msgpack_create(output, &second_buffer, &second_size); + TEST_ASSERT(result == 0); + TEST_CHECK(second_size == first_size); + if (second_size == first_size) { + TEST_CHECK(memcmp(first_buffer, second_buffer, first_size) == 0); + } + + cmt_encode_msgpack_destroy(second_buffer); + cmt_encode_msgpack_destroy(first_buffer); + + return output; +} + +static void check_all_encoders(struct cmt *context) +{ + int result; + char *cloudwatch; + size_t cloudwatch_size; + char *msgpack; + size_t msgpack_size; + cfl_sds_t output; + + output = cmt_encode_prometheus_create(context, CMT_TRUE); + TEST_CHECK(output != NULL); + cmt_encode_prometheus_destroy(output); + + output = cmt_encode_text_create(context); + TEST_CHECK(output != NULL); + cmt_encode_text_destroy(output); + + output = cmt_encode_influx_create(context); + TEST_CHECK(output != NULL); + cmt_encode_influx_destroy(output); + + output = cmt_encode_splunk_hec_create(context, "localhost", "metrics", + NULL, NULL); + TEST_CHECK(output != NULL); + cmt_encode_splunk_hec_destroy(output); + + cloudwatch = NULL; + cloudwatch_size = 0; + result = cmt_encode_cloudwatch_emf_create(context, &cloudwatch, + &cloudwatch_size, CMT_TRUE); + TEST_CHECK(result == 0); + TEST_CHECK(cloudwatch != NULL); + if (cloudwatch != NULL) { + cmt_encode_cloudwatch_emf_destroy(cloudwatch); + } + + output = cmt_encode_prometheus_remote_write_create(context); + TEST_CHECK(output != NULL); + cmt_encode_prometheus_remote_write_destroy(output); + + output = cmt_encode_opentelemetry_create(context); + TEST_CHECK(output != NULL); + cmt_encode_opentelemetry_destroy(output); + + msgpack = NULL; + msgpack_size = 0; + result = cmt_encode_msgpack_create(context, &msgpack, &msgpack_size); + TEST_CHECK(result == 0); + TEST_CHECK(msgpack != NULL); + cmt_encode_msgpack_destroy(msgpack); +} + +static struct cmt *create_native_fixture(void) +{ + int result; + uint64_t now; + uint64_t positive_buckets[] = {3, 5}; + uint64_t negative_buckets[] = {2}; + double quantiles[] = {0.5, 0.9}; + double quantile_values[] = {4.0, 8.0}; + struct cmt *cmt; + struct cmt_counter *counter; + struct cmt_gauge *gauge; + struct cmt_untyped *untyped; + struct cmt_histogram *histogram; + struct cmt_exp_histogram *exp_histogram; + struct cmt_summary *summary; + struct cmt_histogram_buckets *buckets; + struct cmt_metric *metric; + + now = cfl_time_now(); + cmt = cmt_create(); + if (cmt == NULL) { + return NULL; + } + + cmt_label_add(cmt, "environment", "test"); + + counter = cmt_counter_create(cmt, "matrix", "native", "requests_total", + "Requests", 1, (char *[]) {"route"}); + if (counter == NULL || + cmt_counter_set(counter, now, 7, 1, (char *[]) {"/api"}) != 0) { + cmt_destroy(cmt); + return NULL; + } + + gauge = cmt_gauge_create(cmt, "matrix", "native", "large_integer", + "Large integer", 0, NULL); + if (gauge == NULL) { + cmt_destroy(cmt); + return NULL; + } + metric = &gauge->map->metric; + cmt_metric_set_int64(metric, now, 9007199254740993LL); + + untyped = cmt_untyped_create(cmt, "matrix", "native", "temperature", + "Temperature", 1, (char *[]) {"room"}); + if (untyped == NULL || + cmt_untyped_set(untyped, now, 21.5, 1, (char *[]) {"office"}) != 0) { + cmt_destroy(cmt); + return NULL; + } + + buckets = cmt_histogram_buckets_create(2, 1.0, 5.0); + if (buckets == NULL) { + cmt_destroy(cmt); + return NULL; + } + histogram = cmt_histogram_create(cmt, "matrix", "native", "latency", + "Latency", buckets, 1, + (char *[]) {"method"}); + if (histogram == NULL || + cmt_histogram_observe(histogram, now, 2.0, 1, + (char *[]) {"GET"}) != 0) { + cmt_destroy(cmt); + return NULL; + } + histogram->aggregation_type = CMT_AGGREGATION_TYPE_DELTA; + + summary = cmt_summary_create(cmt, "matrix", "native", "size", + "Size", 2, quantiles, 1, + (char *[]) {"operation"}); + if (summary == NULL || + cmt_summary_set_default(summary, now, quantile_values, 12.0, 3, + 1, (char *[]) {"write"}) != 0) { + cmt_destroy(cmt); + return NULL; + } + + exp_histogram = cmt_exp_histogram_create(cmt, "matrix", "native", + "payload", "Payload", 1, + (char *[]) {"protocol"}); + if (exp_histogram == NULL) { + cmt_destroy(cmt); + return NULL; + } + result = cmt_exp_histogram_set_default(exp_histogram, now, 1, 2, 0.0, + -1, 2, positive_buckets, + 0, 1, negative_buckets, + CMT_TRUE, 9.5, 10, 1, + (char *[]) {"http"}); + if (result != 0) { + cmt_destroy(cmt); + return NULL; + } + exp_histogram->aggregation_type = CMT_AGGREGATION_TYPE_CUMULATIVE; + + return cmt; +} + +void test_native_msgpack_conversion_matrix(void) +{ + struct cmt *input; + struct cmt *output; + struct cmt_gauge *gauge; + + input = create_native_fixture(); + TEST_ASSERT(input != NULL); + + output = msgpack_round_trip(input); + TEST_ASSERT(output != NULL); + + TEST_CHECK(cfl_list_size(&output->counters) == 1); + TEST_CHECK(cfl_list_size(&output->gauges) == 1); + TEST_CHECK(cfl_list_size(&output->untypeds) == 1); + TEST_CHECK(cfl_list_size(&output->histograms) == 1); + TEST_CHECK(cfl_list_size(&output->summaries) == 1); + TEST_CHECK(cfl_list_size(&output->exp_histograms) == 1); + + gauge = cfl_list_entry_first(&output->gauges, struct cmt_gauge, _head); + TEST_CHECK(cmt_metric_get_value_type(&gauge->map->metric) == + CMT_METRIC_VALUE_INT64); + TEST_CHECK(cmt_metric_get_int64_value(&gauge->map->metric) == + 9007199254740993LL); + + check_all_encoders(output); + + cmt_decode_msgpack_destroy(output); + cmt_destroy(input); +} + +#ifdef CMT_HAVE_PROMETHEUS_TEXT_DECODER +void test_prometheus_msgpack_conversion_matrix(void) +{ + int result; + const char input_text[] = + "# HELP http_requests_total Requests\n" + "# TYPE http_requests_total counter\n" + "http_requests_total{method=\"GET\"} 7\n" + "# HELP queue_depth Queue depth\n" + "# TYPE queue_depth gauge\n" + "queue_depth{queue=\"main\"} 3\n" + "# HELP request_size Request size\n" + "# TYPE request_size histogram\n" + "request_size_bucket{route=\"/\",le=\"1\"} 1\n" + "request_size_bucket{route=\"/\",le=\"5\"} 2\n" + "request_size_bucket{route=\"/\",le=\"+Inf\"} 2\n" + "request_size_sum{route=\"/\"} 4\n" + "request_size_count{route=\"/\"} 2\n"; + struct cmt *decoded; + struct cmt *roundtrip; + cfl_sds_t prometheus; + + decoded = NULL; + result = cmt_decode_prometheus_create(&decoded, input_text, + sizeof(input_text) - 1, NULL); + TEST_ASSERT(result == CMT_DECODE_PROMETHEUS_SUCCESS); + TEST_ASSERT(decoded != NULL); + + roundtrip = msgpack_round_trip(decoded); + TEST_ASSERT(roundtrip != NULL); + + prometheus = cmt_encode_prometheus_create(roundtrip, CMT_FALSE); + TEST_ASSERT(prometheus != NULL); + TEST_CHECK(strstr(prometheus, "http_requests_total{method=\"GET\"} 7") != NULL); + TEST_CHECK(strstr(prometheus, "queue_depth{queue=\"main\"} 3") != NULL); + TEST_CHECK(strstr(prometheus, "request_size_count{route=\"/\"} 2") != NULL); + cmt_encode_prometheus_destroy(prometheus); + + check_all_encoders(roundtrip); + + cmt_decode_msgpack_destroy(roundtrip); + cmt_decode_prometheus_destroy(decoded); +} +#endif + +void test_remote_write_msgpack_conversion_matrix(void) +{ + int result; + cfl_sds_t remote_payload; + struct cmt *source; + struct cmt *decoded; + struct cmt *roundtrip; + struct cmt_counter *counter; + struct cmt_gauge *gauge; + uint64_t now; + + now = cfl_time_now(); + source = cmt_create(); + TEST_ASSERT(source != NULL); + + counter = cmt_counter_create(source, "matrix", "remote", "requests_total", + "Requests", 1, (char *[]) {"method"}); + gauge = cmt_gauge_create(source, "matrix", "remote", "depth", + "Depth", 1, (char *[]) {"queue"}); + TEST_ASSERT(counter != NULL); + TEST_ASSERT(gauge != NULL); + TEST_ASSERT(cmt_counter_set(counter, now, 5, 1, + (char *[]) {"POST"}) == 0); + TEST_ASSERT(cmt_gauge_set(gauge, now, 2, 1, + (char *[]) {"main"}) == 0); + + remote_payload = cmt_encode_prometheus_remote_write_create(source); + TEST_ASSERT(remote_payload != NULL); + + decoded = NULL; + result = cmt_decode_prometheus_remote_write_create(&decoded, + remote_payload, + cfl_sds_len(remote_payload)); + TEST_ASSERT(result == CMT_DECODE_PROMETHEUS_REMOTE_WRITE_SUCCESS); + TEST_ASSERT(decoded != NULL); + + roundtrip = msgpack_round_trip(decoded); + TEST_ASSERT(roundtrip != NULL); + /* + * The encoder intentionally omits remote-write metadata, so the decoder + * must apply the protocol's safe fallback and represent scalar series as + * gauges. The MessagePack conversion must preserve that decoded model. + */ + TEST_CHECK(cfl_list_size(&roundtrip->counters) == 0); + TEST_CHECK(cfl_list_size(&roundtrip->gauges) == 2); + check_all_encoders(roundtrip); + + cmt_decode_msgpack_destroy(roundtrip); + cmt_decode_prometheus_remote_write_destroy(decoded); + cmt_encode_prometheus_remote_write_destroy(remote_payload); + cmt_destroy(source); +} + +void test_statsd_msgpack_conversion_matrix(void) +{ + int result; + char statsd_payload[] = + "requests:5|c|#environment:test,method:GET\n" + "temperature:21.5|g|#environment:test,room:office\n" + "latency:12|ms|#environment:test,route:/api\n"; + struct cmt *decoded; + struct cmt *roundtrip; + cfl_sds_t prometheus; + + decoded = NULL; + result = cmt_decode_statsd_create(&decoded, statsd_payload, + sizeof(statsd_payload) - 1, + 0); + TEST_ASSERT(result == CMT_DECODE_STATSD_SUCCESS); + TEST_ASSERT(decoded != NULL); + + roundtrip = msgpack_round_trip(decoded); + TEST_ASSERT(roundtrip != NULL); + + prometheus = cmt_encode_prometheus_create(roundtrip, CMT_FALSE); + TEST_ASSERT(prometheus != NULL); + TEST_CHECK(strstr(prometheus, "requests") != NULL); + TEST_CHECK(strstr(prometheus, "temperature") != NULL); + cmt_encode_prometheus_destroy(prometheus); + + check_all_encoders(roundtrip); + + cmt_decode_msgpack_destroy(roundtrip); + cmt_decode_statsd_destroy(decoded); +} + +void test_otlp_msgpack_conversion_matrix(void) +{ + int result; + size_t offset; + cfl_sds_t otlp_payload; + struct cfl_list decoded_list; + struct cmt *source; + struct cmt *decoded; + struct cmt *roundtrip; + + source = create_native_fixture(); + TEST_ASSERT(source != NULL); + + otlp_payload = cmt_encode_opentelemetry_create(source); + TEST_ASSERT(otlp_payload != NULL); + + offset = 0; + result = cmt_decode_opentelemetry_create(&decoded_list, otlp_payload, + cfl_sds_len(otlp_payload), + &offset); + TEST_ASSERT(result == CMT_DECODE_OPENTELEMETRY_SUCCESS); + TEST_ASSERT(cfl_list_size(&decoded_list) == 1); + + decoded = cfl_list_entry_first(&decoded_list, struct cmt, _head); + roundtrip = msgpack_round_trip(decoded); + TEST_ASSERT(roundtrip != NULL); + check_all_encoders(roundtrip); + + cmt_decode_msgpack_destroy(roundtrip); + cmt_decode_opentelemetry_destroy(&decoded_list); + cmt_encode_opentelemetry_destroy(otlp_payload); + cmt_destroy(source); +} + +TEST_LIST = { + {"native_msgpack_conversion_matrix", test_native_msgpack_conversion_matrix}, +#ifdef CMT_HAVE_PROMETHEUS_TEXT_DECODER + {"prometheus_msgpack_conversion_matrix", test_prometheus_msgpack_conversion_matrix}, +#endif + {"remote_write_msgpack_conversion_matrix", test_remote_write_msgpack_conversion_matrix}, + {"statsd_msgpack_conversion_matrix", test_statsd_msgpack_conversion_matrix}, + {"otlp_msgpack_conversion_matrix", test_otlp_msgpack_conversion_matrix}, + {NULL, NULL} +}; diff --git a/lib/cmetrics/tests/histogram.c b/lib/cmetrics/tests/histogram.c index bf3662bc589..8cbd6ff04b9 100644 --- a/lib/cmetrics/tests/histogram.c +++ b/lib/cmetrics/tests/histogram.c @@ -109,6 +109,51 @@ static void prometheus_encode_test(struct cmt *cmt) cmt_encode_prometheus_destroy(buf); } +void test_histogram_non_finite_bucket_labels() +{ + /* Cover non-finite %g outputs like inf/nan. */ + uint64_t ts; + cfl_sds_t buf; + struct cmt *cmt; + struct cmt_histogram *h; + struct cmt_histogram_buckets *buckets; + + cmt_initialize(); + + ts = 0; + cmt = cmt_create(); + TEST_CHECK(cmt != NULL); + + buckets = cmt_histogram_buckets_create(3, -INFINITY, NAN, INFINITY); + TEST_CHECK(buckets != NULL); + + h = cmt_histogram_create(cmt, + "cm", "encoding", "non_finite_bucket", + "Histogram non-finite bucket label", + buckets, + 0, NULL); + TEST_CHECK(h != NULL); + + cmt_histogram_observe(h, ts, 42.0, 0, NULL); + + buf = cmt_encode_prometheus_create(cmt, CMT_TRUE); + TEST_CHECK(buf != NULL); + if (buf != NULL) { + TEST_CHECK(strstr(buf, + "cm_encoding_non_finite_bucket_bucket{le=\"-inf\"} 0 0") != NULL); + TEST_CHECK(strstr(buf, + "cm_encoding_non_finite_bucket_bucket{le=\"nan\"} 1 0") != NULL); + TEST_CHECK(strstr(buf, + "cm_encoding_non_finite_bucket_bucket{le=\"inf\"} 1 0") != NULL); + TEST_CHECK(strstr(buf, "le=\"-inf.0\"") == NULL); + TEST_CHECK(strstr(buf, "le=\"nan.0\"") == NULL); + TEST_CHECK(strstr(buf, "le=\"inf.0\"") == NULL); + cmt_encode_prometheus_destroy(buf); + } + + cmt_destroy(cmt); +} + void test_histogram() { @@ -257,6 +302,7 @@ void test_prometheus_large_integer_bucket_precision() } TEST_LIST = { + {"non_finite_bucket_labels" , test_histogram_non_finite_bucket_labels}, {"histogram" , test_histogram}, {"set_defaults" , test_set_defaults}, {"prometheus_large_integer_bucket_precision", test_prometheus_large_integer_bucket_precision}, diff --git a/lib/cmetrics/tests/issues.c b/lib/cmetrics/tests/issues.c index 651a8c405a4..8ac4ee20b9a 100644 --- a/lib/cmetrics/tests/issues.c +++ b/lib/cmetrics/tests/issues.c @@ -24,6 +24,9 @@ #include #include #include +#include +#include +#include #include "cmt_tests.h" @@ -91,6 +94,90 @@ void test_issue_54() cmt_destroy(cmt1); } +static void check_long_label_round_trip(size_t label_length) +{ + char *label_value; + char *label_values[1]; + char *label_keys[1] = {"command"}; + char *msgpack_buffer; + size_t msgpack_size; + size_t offset; + int result; + struct cmt *source; + struct cmt *decoded; + struct cmt_counter *counter; + struct cmt_metric *metric; + struct cmt_map_label *label; + + label_value = malloc(label_length + 1); + TEST_ASSERT(label_value != NULL); + + memset(label_value, 'a', label_length); + label_value[label_length] = '\0'; + label_values[0] = label_value; + + source = cmt_create(); + TEST_ASSERT(source != NULL); + + counter = cmt_counter_create(source, "test", "", "long_label", + "Long label round-trip", 1, label_keys); + TEST_ASSERT(counter != NULL); + TEST_CHECK(cmt_counter_set(counter, 0, 1, 1, label_values) == 0); + + result = cmt_encode_msgpack_create(source, &msgpack_buffer, &msgpack_size); + TEST_ASSERT(result == 0); + + offset = 0; + result = cmt_decode_msgpack_create(&decoded, msgpack_buffer, msgpack_size, + &offset); + TEST_ASSERT(result == 0); + TEST_CHECK(offset == msgpack_size); + + counter = cfl_list_entry_first(&decoded->counters, + struct cmt_counter, _head); + metric = cfl_list_entry_first(&counter->map->metrics, + struct cmt_metric, _head); + label = cfl_list_entry_first(&metric->labels, + struct cmt_map_label, _head); + + TEST_CHECK(cfl_sds_len(label->name) == label_length); + TEST_CHECK(memcmp(label->name, label_value, label_length) == 0); + + cmt_decode_msgpack_destroy(decoded); + cmt_encode_msgpack_destroy(msgpack_buffer); + cmt_destroy(source); + free(label_value); +} + +void test_long_msgpack_labels() +{ + check_long_label_round_trip(1024); + check_long_label_round_trip(1025); + check_long_label_round_trip(2048); + check_long_label_round_trip(65536); +} + +void test_truncated_msgpack_string() +{ + char *output; + int result; + mpack_error_t error; + mpack_reader_t reader; + const char input[] = { + (char) 0xdb, (char) 0xff, (char) 0xff, (char) 0xff, (char) 0xff + }; + + output = NULL; + mpack_reader_init_data(&reader, input, sizeof(input)); + + result = cmt_mpack_consume_string_tag(&reader, &output); + error = mpack_reader_destroy(&reader); + + TEST_CHECK(result == CMT_MPACK_ENGINE_ERROR); + TEST_CHECK(error != mpack_ok); + TEST_CHECK(output == NULL); +} + #ifdef CMT_HAVE_PROMETHEUS_TEXT_DECODER /* issue: https://github.com/fluent/fluent-bit/issues/10761 */ @@ -123,6 +210,8 @@ void test_prometheus_metric_no_subsystem() TEST_LIST = { {"issue_54", test_issue_54}, + {"long_msgpack_labels", test_long_msgpack_labels}, + {"truncated_msgpack_string", test_truncated_msgpack_string}, #ifdef CMT_HAVE_PROMETHEUS_TEXT_DECODER {"prometheus_metric_no_subsystem", test_prometheus_metric_no_subsystem}, #endif diff --git a/lib/cmetrics/tests/msgpack_temporality.c b/lib/cmetrics/tests/msgpack_temporality.c new file mode 100644 index 00000000000..b4a36e94aa0 --- /dev/null +++ b/lib/cmetrics/tests/msgpack_temporality.c @@ -0,0 +1,213 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* CMetrics + * ======== + * Copyright 2026 The CMetrics Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Regression test: histogram and exp_histogram aggregation_type (delta vs + * cumulative temporality) must survive a cmt_encode_msgpack / + * cmt_decode_msgpack round trip. + * + * Before this fix, pack_header() in cmt_encode_msgpack.c only wrote the + * "aggregation_type" meta field for CMT_COUNTER, so histograms and + * exp_histograms always decoded back with CMT_AGGREGATION_TYPE_UNSPECIFIED + * regardless of what temporality they were created with. This is the path + * Fluent Bit's in_opentelemetry -> (internal msgpack buffer) -> + * out_opentelemetry pipeline uses, so any OTLP histogram passing through + * Fluent Bit lost its temporality and Prometheus's OTLP receiver rejected it + * with "invalid temporality and type combination". + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "cmt_tests.h" + +static void test_histogram_temporality_survives_msgpack_roundtrip() +{ + int result; + char *encoded_buffer; + size_t encoded_size; + size_t offset; + struct cmt *cmt; + struct cmt *decoded_cmt; + struct cmt_histogram *histogram; + struct cmt_histogram *decoded_histogram; + struct cmt_histogram_buckets *buckets; + + cmt_initialize(); + + cmt = cmt_create(); + TEST_CHECK(cmt != NULL); + + buckets = cmt_histogram_buckets_default_create(); + TEST_CHECK(buckets != NULL); + + histogram = cmt_histogram_create(cmt, "k6", "http", "req_duration", + "request duration", buckets, 0, NULL); + TEST_CHECK(histogram != NULL); + + result = cmt_histogram_observe(histogram, 12345, 1.5, 0, NULL); + TEST_CHECK(result == 0); + + /* Simulate what an OTLP delta-temporality histogram looks like once + * decoded from OTLP into a cmt_histogram (cmt_decode_opentelemetry.c + * already sets this correctly; the bug is losing it in the internal + * msgpack buffer that sits between Fluent Bit's OTLP input and output + * plugins). + */ + histogram->aggregation_type = CMT_AGGREGATION_TYPE_DELTA; + + offset = 0; + result = cmt_encode_msgpack_create(cmt, &encoded_buffer, &encoded_size); + TEST_CHECK(result == 0); + + result = cmt_decode_msgpack_create(&decoded_cmt, encoded_buffer, + encoded_size, &offset); + TEST_CHECK(result == CMT_DECODE_MSGPACK_SUCCESS); + + if (result == CMT_DECODE_MSGPACK_SUCCESS) { + TEST_CHECK(cfl_list_size(&decoded_cmt->histograms) == 1); + + decoded_histogram = cfl_list_entry_first(&decoded_cmt->histograms, + struct cmt_histogram, _head); + TEST_CHECK(decoded_histogram != NULL); + + if (decoded_histogram != NULL) { + TEST_CHECK(decoded_histogram->aggregation_type == + CMT_AGGREGATION_TYPE_DELTA); + } + + cmt_decode_msgpack_destroy(decoded_cmt); + } + + cmt_encode_msgpack_destroy(encoded_buffer); + cmt_destroy(cmt); +} + +static void test_exp_histogram_temporality_survives_msgpack_roundtrip() +{ + int result; + char *encoded_buffer; + size_t encoded_size; + size_t offset; + struct cmt *cmt; + struct cmt *decoded_cmt; + struct cmt_exp_histogram *exp_histogram; + struct cmt_exp_histogram *decoded_exp_histogram; + + cmt_initialize(); + + cmt = cmt_create(); + TEST_CHECK(cmt != NULL); + + exp_histogram = cmt_exp_histogram_create(cmt, "k6", "http", "req_duration", + "request duration", 0, NULL); + TEST_CHECK(exp_histogram != NULL); + + exp_histogram->aggregation_type = CMT_AGGREGATION_TYPE_DELTA; + + offset = 0; + result = cmt_encode_msgpack_create(cmt, &encoded_buffer, &encoded_size); + TEST_CHECK(result == 0); + + result = cmt_decode_msgpack_create(&decoded_cmt, encoded_buffer, + encoded_size, &offset); + TEST_CHECK(result == CMT_DECODE_MSGPACK_SUCCESS); + + if (result == CMT_DECODE_MSGPACK_SUCCESS) { + TEST_CHECK(cfl_list_size(&decoded_cmt->exp_histograms) == 1); + + decoded_exp_histogram = cfl_list_entry_first(&decoded_cmt->exp_histograms, + struct cmt_exp_histogram, + _head); + TEST_CHECK(decoded_exp_histogram != NULL); + + if (decoded_exp_histogram != NULL) { + TEST_CHECK(decoded_exp_histogram->aggregation_type == + CMT_AGGREGATION_TYPE_DELTA); + } + + cmt_decode_msgpack_destroy(decoded_cmt); + } + + cmt_encode_msgpack_destroy(encoded_buffer); + cmt_destroy(cmt); +} + +static void test_counter_temporality_still_survives_msgpack_roundtrip() +{ + int result; + char *encoded_buffer; + size_t encoded_size; + size_t offset; + struct cmt *cmt; + struct cmt *decoded_cmt; + struct cmt_counter *counter; + struct cmt_counter *decoded_counter; + + cmt_initialize(); + + cmt = cmt_create(); + TEST_CHECK(cmt != NULL); + + counter = cmt_counter_create(cmt, "k6", "http", "reqs", "requests", 0, NULL); + TEST_CHECK(counter != NULL); + + counter->aggregation_type = CMT_AGGREGATION_TYPE_DELTA; + + offset = 0; + result = cmt_encode_msgpack_create(cmt, &encoded_buffer, &encoded_size); + TEST_CHECK(result == 0); + + result = cmt_decode_msgpack_create(&decoded_cmt, encoded_buffer, + encoded_size, &offset); + TEST_CHECK(result == CMT_DECODE_MSGPACK_SUCCESS); + + if (result == CMT_DECODE_MSGPACK_SUCCESS) { + TEST_CHECK(cfl_list_size(&decoded_cmt->counters) == 1); + + decoded_counter = cfl_list_entry_first(&decoded_cmt->counters, + struct cmt_counter, _head); + TEST_CHECK(decoded_counter != NULL); + + if (decoded_counter != NULL) { + TEST_CHECK(decoded_counter->aggregation_type == + CMT_AGGREGATION_TYPE_DELTA); + } + + cmt_decode_msgpack_destroy(decoded_cmt); + } + + cmt_encode_msgpack_destroy(encoded_buffer); + cmt_destroy(cmt); +} + +TEST_LIST = { + {"histogram_temporality_survives_msgpack_roundtrip", + test_histogram_temporality_survives_msgpack_roundtrip}, + {"exp_histogram_temporality_survives_msgpack_roundtrip", + test_exp_histogram_temporality_survives_msgpack_roundtrip}, + {"counter_temporality_still_survives_msgpack_roundtrip", + test_counter_temporality_still_survives_msgpack_roundtrip}, + {NULL, NULL} +}; diff --git a/lib/cmetrics/tests/opentelemetry.c b/lib/cmetrics/tests/opentelemetry.c index c24819f0377..918f44253cf 100644 --- a/lib/cmetrics/tests/opentelemetry.c +++ b/lib/cmetrics/tests/opentelemetry.c @@ -992,6 +992,102 @@ void test_opentelemetry_encode_multi_resource_scope_containers() cmt_destroy(cmt); } +void test_opentelemetry_encode_scope_attributes_cleanup() +{ + struct cmt *cmt; + struct cmt_gauge *gauge; + struct cfl_array *resource_metrics_list; + struct cfl_array *scope_metrics_list; + struct cfl_kvlist *resource_entry; + struct cfl_kvlist *scope_entry; + struct cfl_kvlist *scope_root; + struct cfl_kvlist *scope_metadata; + struct cfl_kvlist *scope_attributes; + cfl_sds_t payload; + Opentelemetry__Proto__Collector__Metrics__V1__ExportMetricsServiceRequest *request; + + cmt = cmt_create(); + TEST_CHECK(cmt != NULL); + if (cmt == NULL) { + return; + } + + gauge = cmt_gauge_create(cmt, "ns", "sub", "scope_attributes", "g", + 0, NULL); + TEST_CHECK(gauge != NULL); + if (gauge == NULL) { + cmt_destroy(cmt); + return; + } + cmt_gauge_set(gauge, 123, 1.5, 0, NULL); + + resource_metrics_list = cfl_array_create(1); + scope_metrics_list = cfl_array_create(2); + TEST_CHECK(resource_metrics_list != NULL && scope_metrics_list != NULL); + if (resource_metrics_list == NULL || scope_metrics_list == NULL) { + if (resource_metrics_list != NULL) { + cfl_array_destroy(resource_metrics_list); + } + if (scope_metrics_list != NULL) { + cfl_array_destroy(scope_metrics_list); + } + cmt_destroy(cmt); + return; + } + + scope_entry = cfl_kvlist_create(); + scope_root = cfl_kvlist_create(); + scope_metadata = cfl_kvlist_create(); + scope_attributes = cfl_kvlist_create(); + cfl_kvlist_insert_string(scope_metadata, "name", "empty-attributes"); + cfl_kvlist_insert_kvlist(scope_root, "metadata", scope_metadata); + cfl_kvlist_insert_kvlist(scope_root, "attributes", scope_attributes); + cfl_kvlist_insert_kvlist(scope_entry, "scope", scope_root); + cfl_array_append_kvlist(scope_metrics_list, scope_entry); + + scope_entry = cfl_kvlist_create(); + scope_root = cfl_kvlist_create(); + scope_metadata = cfl_kvlist_create(); + scope_attributes = cfl_kvlist_create(); + cfl_kvlist_insert_string(scope_metadata, "name", "populated-attributes"); + cfl_kvlist_insert_string(scope_attributes, "language", "c"); + cfl_kvlist_insert_int64(scope_attributes, "generation", 2); + cfl_kvlist_insert_kvlist(scope_root, "metadata", scope_metadata); + cfl_kvlist_insert_kvlist(scope_root, "attributes", scope_attributes); + cfl_kvlist_insert_kvlist(scope_entry, "scope", scope_root); + cfl_array_append_kvlist(scope_metrics_list, scope_entry); + + resource_entry = cfl_kvlist_create(); + cfl_kvlist_insert_array(resource_entry, "scope_metrics_list", + scope_metrics_list); + cfl_array_append_kvlist(resource_metrics_list, resource_entry); + cfl_kvlist_insert_array(cmt->external_metadata, "resource_metrics_list", + resource_metrics_list); + + payload = cmt_encode_opentelemetry_create(cmt); + TEST_CHECK(payload != NULL); + if (payload != NULL) { + request = opentelemetry__proto__collector__metrics__v1__export_metrics_service_request__unpack( + NULL, cfl_sds_len(payload), (uint8_t *) payload); + TEST_CHECK(request != NULL); + if (request != NULL) { + TEST_CHECK(request->n_resource_metrics == 1); + TEST_CHECK(request->resource_metrics[0]->n_scope_metrics == 2); + TEST_CHECK(request->resource_metrics[0]->scope_metrics[0]->scope->n_attributes == 0); + TEST_CHECK(request->resource_metrics[0]->scope_metrics[1]->scope->n_attributes == 2); + TEST_CHECK(strcmp(request->resource_metrics[0]->scope_metrics[1]->scope->attributes[0]->key, + "language") == 0); + TEST_CHECK(strcmp(request->resource_metrics[0]->scope_metrics[1]->scope->attributes[1]->key, + "generation") == 0); + opentelemetry__proto__collector__metrics__v1__export_metrics_service_request__free_unpacked( + request, NULL); + } + cmt_encode_opentelemetry_destroy(payload); + } + + cmt_destroy(cmt); +} + void test_opentelemetry_api_full_roundtrip_with_msgpack() { int ret; @@ -1900,6 +1996,7 @@ static void test_opentelemetry_omitted_null_key_label_encoded(void) TEST_LIST = { {"opentelemetry_api_full_roundtrip_with_msgpack", test_opentelemetry_api_full_roundtrip_with_msgpack}, {"opentelemetry_encode_multi_resource_scope_containers", test_opentelemetry_encode_multi_resource_scope_containers}, + {"opentelemetry_encode_scope_attributes_cleanup", test_opentelemetry_encode_scope_attributes_cleanup}, {"opentelemetry_exponential_histogram", test_opentelemetry_exponential_histogram}, {"opentelemetry_gauge_int_and_unit_decode", test_opentelemetry_gauge_int_and_unit_decode}, {"opentelemetry_sum_non_monotonic_int_roundtrip", test_opentelemetry_sum_non_monotonic_int_roundtrip}, diff --git a/lib/cmetrics/tests/prometheus_parser.c b/lib/cmetrics/tests/prometheus_parser.c index 28643fe8418..022023b916f 100644 --- a/lib/cmetrics/tests/prometheus_parser.c +++ b/lib/cmetrics/tests/prometheus_parser.c @@ -79,7 +79,7 @@ void test_header_help() TEST_CHECK(f->context.metric.type == 0); cfl_sds_destroy(f->context.metric.name_orig); cfl_sds_destroy(f->context.metric.docstring); - free(f->context.metric.ns); + free(f->context.metric.name_buf); destroy(f); } @@ -97,7 +97,7 @@ void test_header_type() TEST_CHECK(f->context.metric.type == COUNTER); TEST_CHECK(f->context.metric.docstring == NULL); cfl_sds_destroy(f->context.metric.name_orig); - free(f->context.metric.ns); + free(f->context.metric.name_buf); destroy(f); } @@ -118,7 +118,7 @@ void test_header_help_type() TEST_CHECK(f->context.metric.type == SUMMARY); cfl_sds_destroy(f->context.metric.name_orig); cfl_sds_destroy(f->context.metric.docstring); - free(f->context.metric.ns); + free(f->context.metric.name_buf); destroy(f); } @@ -139,11 +139,69 @@ void test_header_type_help() TEST_CHECK(f->context.metric.type == GAUGE); cfl_sds_destroy(f->context.metric.name_orig); cfl_sds_destroy(f->context.metric.docstring); - free(f->context.metric.ns); + free(f->context.metric.name_buf); destroy(f); } +void test_metric_name_ownership() +{ + struct fixture *f; + + f = init(START_HEADER, "# TYPE metric counter\n"); + TEST_ASSERT(parse(f) == 0); + TEST_CHECK(strcmp(f->context.metric.ns, "") == 0); + TEST_CHECK(strcmp(f->context.metric.subsystem, "") == 0); + TEST_CHECK(strcmp(f->context.metric.name, "metric") == 0); + TEST_CHECK(f->context.metric.name_buf != NULL); + cfl_sds_destroy(f->context.metric.name_orig); + free(f->context.metric.name_buf); + destroy(f); + + f = init(START_HEADER, "# TYPE namespace_metric counter\n"); + TEST_ASSERT(parse(f) == 0); + TEST_CHECK(strcmp(f->context.metric.ns, "namespace") == 0); + TEST_CHECK(strcmp(f->context.metric.subsystem, "") == 0); + TEST_CHECK(strcmp(f->context.metric.name, "metric") == 0); + TEST_CHECK(f->context.metric.name_buf == f->context.metric.ns); + cfl_sds_destroy(f->context.metric.name_orig); + free(f->context.metric.name_buf); + destroy(f); +} + +void test_metric_name_ownership_resets() +{ + int status; + struct cmt *cmt = NULL; + const char *input = + "# TYPE metric gauge\n" + "metric 1\n" + "# TYPE namespace_metric gauge\n" + "namespace_metric 2\n" + "# TYPE namespace_subsystem_metric gauge\n" + "namespace_subsystem_metric 3\n"; + + status = cmt_decode_prometheus_create(&cmt, input, 0, NULL); + TEST_ASSERT(status == CMT_DECODE_PROMETHEUS_SUCCESS); + TEST_ASSERT(cmt != NULL); + cmt_decode_prometheus_destroy(cmt); +} + +void test_metric_name_ownership_error_cleanup() +{ + int status; + struct cmt *cmt = NULL; + + status = cmt_decode_prometheus_create(&cmt, + "# TYPE metric counter\nmetric {key=", 0, NULL); + TEST_CHECK(status == CMT_DECODE_PROMETHEUS_SYNTAX_ERROR); + + status = cmt_decode_prometheus_create(&cmt, + "# TYPE namespace_subsystem_metric counter\n" + "namespace_subsystem_metric {key=", 0, NULL); + TEST_CHECK(status == CMT_DECODE_PROMETHEUS_SYNTAX_ERROR); +} + struct cmt_decode_prometheus_context_sample *add_empty_sample(struct fixture *f) { struct cmt_decode_prometheus_context_sample *sample; @@ -1710,11 +1768,28 @@ void test_issue_fluent_bit_9267() cfl_sds_destroy(in_buf); } +// reproduces https://github.com/fluent/cmetrics/issues/274 +void test_issue_274() +{ + int status; + struct cmt *cmt; + cfl_sds_t in_buf = read_file(CMT_TESTS_DATA_PATH "/issue_274.txt"); + size_t in_size = cfl_sds_len(in_buf); + + status = cmt_decode_prometheus_create(&cmt, in_buf, in_size, NULL); + TEST_CHECK(status == 0); + cfl_sds_destroy(in_buf); + cmt_decode_prometheus_destroy(cmt); +} + TEST_LIST = { {"header_help", test_header_help}, {"header_type", test_header_type}, {"header_help_type", test_header_help_type}, {"header_type_help", test_header_type_help}, + {"metric_name_ownership", test_metric_name_ownership}, + {"metric_name_ownership_resets", test_metric_name_ownership_resets}, + {"metric_name_ownership_error_cleanup", test_metric_name_ownership_error_cleanup}, {"labels", test_labels}, {"labels_trailing_comma", test_labels_trailing_comma}, {"sample", test_sample}, @@ -1744,5 +1819,6 @@ TEST_LIST = { {"histogram_different_label_count", test_histogram_different_label_count}, {"issue_fluent_bit_6534", test_issue_fluent_bit_6534}, {"issue_fluent_bit_9267", test_issue_fluent_bit_9267}, + {"issue_274", test_issue_274}, { 0 } }; diff --git a/lib/cmetrics/tests/summary.c b/lib/cmetrics/tests/summary.c index 99ffe579246..299333be3ea 100644 --- a/lib/cmetrics/tests/summary.c +++ b/lib/cmetrics/tests/summary.c @@ -19,6 +19,7 @@ #include #include +#include #include #include @@ -94,6 +95,36 @@ void test_set_defaults() cmt_destroy(cmt); } +void test_quantile_bounds() +{ + uint64_t ts; + double quantiles[] = {0.5}; + double values[] = {12.0}; + struct cmt *cmt; + struct cmt_metric *metric; + struct cmt_summary *summary; + + cmt = cmt_create(); + TEST_ASSERT(cmt != NULL); + summary = cmt_summary_create(cmt, "test", "", "summary", "help", + 1, quantiles, 0, NULL); + TEST_ASSERT(summary != NULL); + ts = cfl_time_now(); + TEST_ASSERT(cmt_summary_set_default(summary, ts, values, 12.0, 1, + 0, NULL) == 0); + metric = cmt_map_metric_get(&summary->opts, summary->map, 0, NULL, + CMT_FALSE); + TEST_ASSERT(metric != NULL); + + TEST_CHECK(cmt_summary_quantile_get_value(metric, 0) == 12.0); + TEST_CHECK(cmt_summary_quantile_get_value(metric, -1) == 0.0); + TEST_CHECK(cmt_summary_quantile_get_value(metric, 1) == 0.0); + cmt_summary_quantile_set(metric, ts, 1, 99.0); + TEST_CHECK(cmt_summary_quantile_get_value(metric, 0) == 12.0); + + cmt_destroy(cmt); +} + /* ref: https://github.com/fluent/fluent-bit/issues/5894 */ void fluentbit_bug_5894() { @@ -167,6 +198,7 @@ void fluentbit_bug_5894() TEST_LIST = { {"set_defaults" , test_set_defaults}, + {"quantile_bounds" , test_quantile_bounds}, {"fluentbit_bug_5894", fluentbit_bug_5894}, { 0 } }; diff --git a/plugins/CMakeLists.txt b/plugins/CMakeLists.txt index 36fcc94ad12..af43a8fc2e4 100644 --- a/plugins/CMakeLists.txt +++ b/plugins/CMakeLists.txt @@ -311,6 +311,7 @@ if (FLB_STREAM_PROCESSOR) endif() if (FLB_SYSTEM_WINDOWS) + REGISTER_IN_PLUGIN("in_etw") REGISTER_IN_PLUGIN("in_winlog") REGISTER_IN_PLUGIN("in_winstat") REGISTER_IN_PLUGIN("in_winevtlog") @@ -402,6 +403,7 @@ REGISTER_OUT_PLUGIN("out_udp") REGISTER_OUT_PLUGIN("out_td") REGISTER_OUT_PLUGIN("out_lib") REGISTER_OUT_PLUGIN("out_flowcounter") +REGISTER_OUT_PLUGIN("out_gcs") REGISTER_OUT_PLUGIN("out_gelf") REGISTER_OUT_PLUGIN("out_websocket") REGISTER_OUT_PLUGIN("out_cloudwatch_logs") @@ -491,10 +493,11 @@ if(EXT_FILTER_PLUGINS) endforeach() endif() -# Generate the header from the template +# Keep the generated plugin registry local to each build configuration. +file(MAKE_DIRECTORY "${PROJECT_BINARY_DIR}/generated/include/fluent-bit") configure_file( "${PROJECT_SOURCE_DIR}/include/fluent-bit/flb_plugins.h.in" - "${PROJECT_SOURCE_DIR}/include/fluent-bit/flb_plugins.h" + "${PROJECT_BINARY_DIR}/generated/include/fluent-bit/flb_plugins.h" ) set(FLB_PLUGINS "${flb_plugins}" PARENT_SCOPE) diff --git a/plugins/filter_kubernetes/kube_conf.h b/plugins/filter_kubernetes/kube_conf.h index 781053c9274..85f90928c83 100644 --- a/plugins/filter_kubernetes/kube_conf.h +++ b/plugins/filter_kubernetes/kube_conf.h @@ -29,6 +29,7 @@ #include #include #include +#include /* * Since this filter might get a high number of request per second, @@ -213,6 +214,13 @@ struct flb_kube { int aws_pod_service_map_refresh_interval; flb_sds_t aws_pod_service_preload_cache_path; struct flb_upstream *aws_pod_association_upstream; + pthread_mutex_t aws_pod_service_mutex; + pthread_cond_t aws_pod_service_cond; + pthread_t aws_pod_service_thread; + int aws_pod_service_sync_initialized; + int aws_pod_service_thread_created; + int aws_pod_service_shutdown; + struct mk_event_loop *aws_pod_service_event_loop; /* * This variable holds the Kubernetes platform type * Current checks for EKS or Native Kuberentes diff --git a/plugins/filter_kubernetes/kube_meta.c b/plugins/filter_kubernetes/kube_meta.c index 63426e9467c..f8371b1a826 100644 --- a/plugins/filter_kubernetes/kube_meta.c +++ b/plugins/filter_kubernetes/kube_meta.c @@ -58,8 +58,13 @@ static int file_to_buffer(const char *path, ssize_t bytes; FILE *fp; struct stat st; + const char *file_mode = "r"; - if (!(fp = fopen(path, "r"))) { +#ifdef FLB_SYSTEM_WINDOWS + file_mode = "rb"; +#endif + + if (!(fp = fopen(path, file_mode))) { return -1; } @@ -337,8 +342,13 @@ static int get_meta_file_info(struct flb_kube *ctx, const char *namespace, struct stat sb; int packed = -1; int ret; + int open_flags = O_RDONLY; char uri[1024]; +#ifdef FLB_SYSTEM_WINDOWS + open_flags |= O_BINARY; +#endif + if (ctx->meta_preload_cache_dir && namespace) { if (podname && strlen(podname) > 0) { @@ -350,7 +360,7 @@ static int get_meta_file_info(struct flb_kube *ctx, const char *namespace, ctx->meta_preload_cache_dir, namespace); } if (ret > 0) { - fd = open(uri, O_RDONLY, 0); + fd = open(uri, open_flags, 0); if (fd != -1) { if (fstat(fd, &sb) == 0) { payload = flb_malloc(sb.st_size); diff --git a/plugins/filter_kubernetes/kubernetes.c b/plugins/filter_kubernetes/kubernetes.c index 24548e5fa68..e272b3d321b 100644 --- a/plugins/filter_kubernetes/kubernetes.c +++ b/plugins/filter_kubernetes/kubernetes.c @@ -35,6 +35,7 @@ #include "kubernetes_aws.h" #include +#include #include #include @@ -44,31 +45,68 @@ #define MERGE_MAP 2 /* merge direct binary object (v) */ #define FLB_KUBE_LOCAL_LOGS_INPUT "fluentbit_logs" -struct task_args { - struct flb_kube *ctx; - char *api_server_url; -}; +static int wait_for_pod_service_map_refresh(struct flb_kube *ctx) +{ + int ret; + int shutdown; + struct flb_time current_time; + struct timespec deadline; + + pthread_mutex_lock(&ctx->aws_pod_service_mutex); + + flb_time_get(¤t_time); + deadline = current_time.tm; + deadline.tv_sec += ctx->aws_pod_service_map_refresh_interval; -pthread_mutex_t metadata_mutex; -pthread_t background_thread; -struct task_args *task_args = {0}; -struct mk_event_loop *evl; + while (!ctx->aws_pod_service_shutdown) { + ret = pthread_cond_timedwait(&ctx->aws_pod_service_cond, + &ctx->aws_pod_service_mutex, + &deadline); + if (ret == ETIMEDOUT) { + break; + } + else if (ret != 0) { + flb_plg_error(ctx->ins, + "Failed waiting for pod service map refresh"); + break; + } + } -void *update_pod_service_map(void *arg) + shutdown = ctx->aws_pod_service_shutdown; + pthread_mutex_unlock(&ctx->aws_pod_service_mutex); + + return shutdown; +} + +static void *update_pod_service_map(void *arg) { + struct flb_kube *ctx; + + ctx = arg; + flb_engine_evl_init(); - evl = mk_event_loop_create(256); - if (evl == NULL) { - flb_plg_error(task_args->ctx->ins, + ctx->aws_pod_service_event_loop = mk_event_loop_create(256); + if (ctx->aws_pod_service_event_loop == NULL) { + flb_plg_error(ctx->ins, "Failed to create event loop for pod service map"); return NULL; } - flb_engine_evl_set(evl); + flb_engine_evl_set(ctx->aws_pod_service_event_loop); + while (1) { - fetch_pod_service_map(task_args->ctx,task_args->api_server_url,&metadata_mutex); - flb_plg_debug(task_args->ctx->ins, "Updating pod to service map after %d seconds", task_args->ctx->aws_pod_service_map_refresh_interval); - sleep(task_args->ctx->aws_pod_service_map_refresh_interval); + fetch_pod_service_map(ctx, + ctx->aws_pod_association_endpoint, + &ctx->aws_pod_service_mutex); + flb_plg_debug(ctx->ins, + "Updating pod to service map after %d seconds", + ctx->aws_pod_service_map_refresh_interval); + + if (wait_for_pod_service_map_refresh(ctx)) { + break; + } } + + return NULL; } static int get_stream(msgpack_object_map map) @@ -239,24 +277,38 @@ static int cb_kube_init(struct flb_filter_instance *f_ins, */ flb_kube_meta_init(ctx, config); -/* - * Init separate thread for calling pod to - * service map - */ - pthread_mutex_init(&metadata_mutex, NULL); - if (ctx->aws_use_pod_association) { - task_args = flb_malloc(sizeof(struct task_args)); - if (!task_args) { - flb_errno(); + ret = pthread_mutex_init(&ctx->aws_pod_service_mutex, NULL); + if (ret != 0) { + flb_plg_error(ctx->ins, + "Failed to initialize pod service map mutex"); + flb_kube_conf_destroy(ctx); return -1; } - task_args->ctx = ctx; - task_args->api_server_url = ctx->aws_pod_association_endpoint; - if (pthread_create(&background_thread, NULL, update_pod_service_map, NULL) != 0) { - flb_error("Failed to create background thread"); - background_thread = 0; - flb_free(task_args); + + ret = pthread_cond_init(&ctx->aws_pod_service_cond, NULL); + if (ret != 0) { + flb_plg_error(ctx->ins, + "Failed to initialize pod service map condition"); + pthread_mutex_destroy(&ctx->aws_pod_service_mutex); + flb_kube_conf_destroy(ctx); + return -1; + } + ctx->aws_pod_service_sync_initialized = FLB_TRUE; + + ret = pthread_create(&ctx->aws_pod_service_thread, + NULL, + update_pod_service_map, + ctx); + if (ret != 0) { + flb_plg_error(ctx->ins, + "Failed to create pod service map background thread"); + pthread_cond_destroy(&ctx->aws_pod_service_cond); + pthread_mutex_destroy(&ctx->aws_pod_service_mutex); + ctx->aws_pod_service_sync_initialized = FLB_FALSE; + } + else { + ctx->aws_pod_service_thread_created = FLB_TRUE; } } @@ -824,20 +876,30 @@ static int cb_kube_exit(void *data, struct flb_config *config) struct flb_kube *ctx; ctx = data; - - flb_kube_conf_destroy(ctx); - if (background_thread) { - pthread_cancel(background_thread); - pthread_join(background_thread, NULL); + + if (ctx->aws_pod_service_thread_created) { + pthread_mutex_lock(&ctx->aws_pod_service_mutex); + ctx->aws_pod_service_shutdown = FLB_TRUE; + pthread_cond_signal(&ctx->aws_pod_service_cond); + pthread_mutex_unlock(&ctx->aws_pod_service_mutex); + + pthread_join(ctx->aws_pod_service_thread, NULL); + ctx->aws_pod_service_thread_created = FLB_FALSE; } - pthread_mutex_destroy(&metadata_mutex); - if (task_args) { - flb_free(task_args); + if (ctx->aws_pod_service_event_loop) { + mk_event_loop_destroy(ctx->aws_pod_service_event_loop); + ctx->aws_pod_service_event_loop = NULL; } - if (evl) { - mk_event_loop_destroy(evl); + + if (ctx->aws_pod_service_sync_initialized) { + pthread_cond_destroy(&ctx->aws_pod_service_cond); + pthread_mutex_destroy(&ctx->aws_pod_service_mutex); + ctx->aws_pod_service_sync_initialized = FLB_FALSE; } + + flb_kube_conf_destroy(ctx); + return 0; } diff --git a/plugins/filter_throttle/throttle.c b/plugins/filter_throttle/throttle.c index d1a2c51b23b..74f0fb268ee 100644 --- a/plugins/filter_throttle/throttle.c +++ b/plugins/filter_throttle/throttle.c @@ -92,8 +92,9 @@ void *time_ticker(void *args) ctx->hash->total / ctx->hash->size); } pthread_mutex_unlock(&throttle_mut); - /* sleep is a cancelable function */ + /* Windows sleep is not a pthread cancellation point. */ sleep(ctx->ticker_data.seconds); + pthread_testcancel(); } } diff --git a/plugins/in_etw/CMakeLists.txt b/plugins/in_etw/CMakeLists.txt new file mode 100644 index 00000000000..52fb0e7bcf8 --- /dev/null +++ b/plugins/in_etw/CMakeLists.txt @@ -0,0 +1,4 @@ +set(src + etw.c) + +FLB_PLUGIN(in_etw "${src}" "advapi32;tdh;ole32") diff --git a/plugins/in_etw/etw.c b/plugins/in_etw/etw.c new file mode 100644 index 00000000000..b490df192c4 --- /dev/null +++ b/plugins/in_etw/etw.c @@ -0,0 +1,1852 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2026 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "etw.h" + +#define FLB_IN_ETW_UNIX_EPOCH_IN_100NS 116444736000000000ULL +#define FLB_IN_ETW_PROVIDER_MAP_SIZE 14 +#define FLB_IN_ETW_DEFAULT_KERNEL_FLAGS "process,thread,image_load" + +static const GUID flb_etw_system_trace_control_guid = { + 0x9e814aad, + 0x3204, + 0x11d2, + {0x9a, 0x82, 0x00, 0x60, 0x08, 0xa8, 0x69, 0x39} +}; + +static void pack_cstr(msgpack_packer *mp_pck, const char *str) +{ + size_t len; + + len = strlen(str); + msgpack_pack_str(mp_pck, len); + msgpack_pack_str_body(mp_pck, str, len); +} + +static int guid_to_string(const GUID *guid, char *buf, size_t size) +{ + int ret; + + ret = snprintf(buf, size, + "{%08" PRIx32 "-%04x-%04x-%02x%02x-" + "%02x%02x%02x%02x%02x%02x}", + (uint32_t) guid->Data1, + guid->Data2, + guid->Data3, + guid->Data4[0], + guid->Data4[1], + guid->Data4[2], + guid->Data4[3], + guid->Data4[4], + guid->Data4[5], + guid->Data4[6], + guid->Data4[7]); + + if (ret < 0 || (size_t) ret >= size) { + return -1; + } + + return 0; +} + +static int parse_guid(const char *str, GUID *guid) +{ + unsigned int d1; + unsigned int d2; + unsigned int d3; + unsigned int d4[8]; + char trailing; + int ret; + + ret = sscanf(str, + "{%8x-%4x-%4x-%2x%2x-%2x%2x%2x%2x%2x%2x}%c", + &d1, &d2, &d3, + &d4[0], &d4[1], &d4[2], &d4[3], + &d4[4], &d4[5], &d4[6], &d4[7], &trailing); + + if (ret != 11) { + ret = sscanf(str, + "%8x-%4x-%4x-%2x%2x-%2x%2x%2x%2x%2x%2x%c", + &d1, &d2, &d3, + &d4[0], &d4[1], &d4[2], &d4[3], + &d4[4], &d4[5], &d4[6], &d4[7], &trailing); + } + + if (ret != 11) { + return -1; + } + + guid->Data1 = d1; + guid->Data2 = (unsigned short) d2; + guid->Data3 = (unsigned short) d3; + guid->Data4[0] = (unsigned char) d4[0]; + guid->Data4[1] = (unsigned char) d4[1]; + guid->Data4[2] = (unsigned char) d4[2]; + guid->Data4[3] = (unsigned char) d4[3]; + guid->Data4[4] = (unsigned char) d4[4]; + guid->Data4[5] = (unsigned char) d4[5]; + guid->Data4[6] = (unsigned char) d4[6]; + guid->Data4[7] = (unsigned char) d4[7]; + + return 0; +} + +static int parse_uint64(const char *str, ULONGLONG *out) +{ + char *end; + unsigned long long value; + + if (str == NULL || str[0] == '\0' || str[0] == '-') { + return -1; + } + + errno = 0; + value = strtoull(str, &end, 0); + if (end == str || *end != '\0' || errno == ERANGE) { + return -1; + } + + *out = value; + + return 0; +} + +static char *trim_token(char *str) +{ + char *end; + + while (isspace((unsigned char) *str)) { + str++; + } + + if (*str == '\0') { + return str; + } + + end = str + strlen(str) - 1; + while (end > str && isspace((unsigned char) *end)) { + *end = '\0'; + end--; + } + + return str; +} + +static int has_empty_csv_token(const char *str) +{ + const char *p; + int has_token; + + has_token = FLB_FALSE; + for (p = str; *p != '\0'; p++) { + if (*p == ',') { + if (has_token == FLB_FALSE) { + return FLB_TRUE; + } + has_token = FLB_FALSE; + continue; + } + + if (!isspace((unsigned char) *p)) { + has_token = FLB_TRUE; + } + } + + return has_token == FLB_FALSE; +} + +static int parse_session_type(struct flb_etw *ctx) +{ + if (ctx->session_type_str == NULL || + strcasecmp(ctx->session_type_str, "provider") == 0) { + ctx->session_type = FLB_IN_ETW_SESSION_PROVIDER; + return 0; + } + + if (strcasecmp(ctx->session_type_str, "system") == 0) { + ctx->session_type = FLB_IN_ETW_SESSION_SYSTEM; + return 0; + } + + flb_plg_error(ctx->ins, + "invalid session_type '%s' (expected 'provider' or 'system')", + ctx->session_type_str); + + return -1; +} + +static int parse_stale_session_action(struct flb_etw *ctx) +{ + if (ctx->stale_session_action_str == NULL || + strcasecmp(ctx->stale_session_action_str, "stop") == 0) { + ctx->stale_session_action = FLB_IN_ETW_STALE_ACTION_STOP; + return 0; + } + + if (strcasecmp(ctx->stale_session_action_str, "fail") == 0) { + ctx->stale_session_action = FLB_IN_ETW_STALE_ACTION_FAIL; + return 0; + } + + flb_plg_error(ctx->ins, + "invalid stale_session_action '%s' (expected 'stop' or 'fail')", + ctx->stale_session_action_str); + + return -1; +} + +static int kernel_flag_from_name(const char *name, ULONG *flag) +{ + if (strcasecmp(name, "process") == 0) { + *flag = EVENT_TRACE_FLAG_PROCESS; + return 0; + } + + if (strcasecmp(name, "thread") == 0) { + *flag = EVENT_TRACE_FLAG_THREAD; + return 0; + } + + if (strcasecmp(name, "image_load") == 0 || + strcasecmp(name, "image") == 0) { + *flag = EVENT_TRACE_FLAG_IMAGE_LOAD; + return 0; + } + + if (strcasecmp(name, "cswitch") == 0 || + strcasecmp(name, "context_switch") == 0) { + *flag = EVENT_TRACE_FLAG_CSWITCH; + return 0; + } + + if (strcasecmp(name, "disk_io") == 0 || + strcasecmp(name, "disk") == 0) { + *flag = EVENT_TRACE_FLAG_DISK_IO; + return 0; + } + + if (strcasecmp(name, "tcpip") == 0 || + strcasecmp(name, "network_tcpip") == 0) { + *flag = EVENT_TRACE_FLAG_NETWORK_TCPIP; + return 0; + } + + return -1; +} + +static int parse_kernel_flags(struct flb_etw *ctx) +{ + ULONG flag; + ULONGLONG numeric_flags; + char *tmp; + char *token; + char *context; + + if (ctx->session_type != FLB_IN_ETW_SESSION_SYSTEM) { + return 0; + } + + if (parse_uint64(ctx->kernel_flags_str, &numeric_flags) == 0) { + if (numeric_flags > ULONG_MAX) { + flb_plg_error(ctx->ins, "kernel_flags numeric value is too large"); + return -1; + } + if (numeric_flags == 0) { + flb_plg_error(ctx->ins, "kernel_flags must enable at least one flag"); + return -1; + } + ctx->kernel_flags = (ULONG) numeric_flags; + return 0; + } + + if (has_empty_csv_token(ctx->kernel_flags_str)) { + flb_plg_error(ctx->ins, "kernel_flags contains an empty token"); + return -1; + } + + tmp = flb_strdup(ctx->kernel_flags_str); + if (tmp == NULL) { + flb_errno(); + return -1; + } + + token = strtok_s(tmp, ",", &context); + while (token != NULL) { + token = trim_token(token); + if (token[0] == '\0') { + flb_plg_error(ctx->ins, "kernel_flags contains an empty token"); + flb_free(tmp); + return -1; + } + + if (kernel_flag_from_name(token, &flag) != 0) { + flb_plg_error(ctx->ins, "unknown kernel_flags token '%s'", token); + flb_free(tmp); + return -1; + } + + ctx->kernel_flags |= flag; + token = strtok_s(NULL, ",", &context); + } + + flb_free(tmp); + + if (ctx->kernel_flags == 0) { + flb_plg_error(ctx->ins, "kernel_flags must enable at least one flag"); + return -1; + } + + return 0; +} + +static WCHAR *utf8_to_wide(const char *str) +{ + int size; + WCHAR *buf; + + size = MultiByteToWideChar(CP_UTF8, 0, str, -1, NULL, 0); + if (size == 0) { + return NULL; + } + + buf = flb_calloc(size, sizeof(WCHAR)); + if (buf == NULL) { + flb_errno(); + return NULL; + } + + if (MultiByteToWideChar(CP_UTF8, 0, str, -1, buf, size) == 0) { + flb_free(buf); + return NULL; + } + + return buf; +} + +static char *wide_to_utf8(const WCHAR *str, int bytes, int *out_len) +{ + int wide_len; + int size; + char *buf; + + if (str == NULL) { + return NULL; + } + + if (bytes > 0) { + wide_len = bytes / sizeof(WCHAR); + while (wide_len > 0 && str[wide_len - 1] == L'\0') { + wide_len--; + } + } + else { + wide_len = -1; + } + + size = WideCharToMultiByte(CP_UTF8, 0, str, wide_len, NULL, 0, NULL, NULL); + if (size == 0) { + return NULL; + } + + buf = flb_malloc(size + 1); + if (buf == NULL) { + flb_errno(); + return NULL; + } + + if (WideCharToMultiByte(CP_UTF8, 0, str, wide_len, buf, size, NULL, NULL) == 0) { + flb_free(buf); + return NULL; + } + + buf[size] = '\0'; + if (wide_len == -1) { + *out_len = size - 1; + } + else { + *out_len = size; + } + + return buf; +} + +static void pack_wide_string(msgpack_packer *mp_pck, const WCHAR *str, int bytes) +{ + int len; + char *utf8; + + utf8 = wide_to_utf8(str, bytes, &len); + if (utf8 == NULL) { + msgpack_pack_nil(mp_pck); + return; + } + + msgpack_pack_str(mp_pck, len); + msgpack_pack_str_body(mp_pck, utf8, len); + flb_free(utf8); +} + +static int pack_bounded_wide_string(msgpack_packer *mp_pck, const WCHAR *str, + size_t bytes) +{ + size_t chars; + size_t len; + int utf8_len; + char *utf8; + + if (bytes > INT_MAX || bytes < sizeof(WCHAR)) { + return -1; + } + + chars = bytes / sizeof(WCHAR); + for (len = 0; len < chars; len++) { + if (str[len] == L'\0') { + break; + } + } + + if (len == chars) { + return -1; + } + + if (len == 0) { + msgpack_pack_str(mp_pck, 0); + return 0; + } + + utf8 = wide_to_utf8(str, (int) (len * sizeof(WCHAR)), &utf8_len); + if (utf8 == NULL) { + return -1; + } + + msgpack_pack_str(mp_pck, utf8_len); + msgpack_pack_str_body(mp_pck, utf8, utf8_len); + flb_free(utf8); + + return 0; +} + +static int resolve_provider_name(struct flb_etw *ctx, const WCHAR *name, GUID *guid) +{ + ULONG size; + TDHSTATUS status; + PROVIDER_ENUMERATION_INFO *providers; + PROVIDER_ENUMERATION_INFO *new_providers; + TRACE_PROVIDER_INFO *info; + WCHAR *provider_name; + ULONG i; + + size = 0; + providers = NULL; + + while (1) { + status = TdhEnumerateProviders(providers, &size); + if (status == ERROR_SUCCESS) { + break; + } + + if (status != ERROR_INSUFFICIENT_BUFFER) { + flb_plg_error(ctx->ins, + "TdhEnumerateProviders failed while resolving '%S' (status=%lu)", + name, status); + flb_free(providers); + return -1; + } + + new_providers = flb_realloc(providers, size); + if (new_providers == NULL) { + flb_errno(); + flb_free(providers); + return -1; + } + providers = new_providers; + } + + for (i = 0; i < providers->NumberOfProviders; i++) { + info = &providers->TraceProviderInfoArray[i]; + provider_name = (WCHAR *) ((char *) providers + info->ProviderNameOffset); + + if (_wcsicmp(provider_name, name) == 0) { + *guid = info->ProviderGuid; + flb_free(providers); + return 0; + } + } + + flb_plg_error(ctx->ins, + "ETW provider_name '%S' was not found in registered providers", + name); + flb_free(providers); + + return -1; +} + +static EVENT_TRACE_PROPERTIES *create_trace_properties(struct flb_etw *ctx) +{ + size_t name_size; + size_t props_size; + EVENT_TRACE_PROPERTIES *props; + + name_size = (wcslen(ctx->session_name_wide) + 1) * sizeof(WCHAR); + props_size = sizeof(EVENT_TRACE_PROPERTIES) + name_size; + + props = flb_calloc(1, props_size); + if (props == NULL) { + flb_errno(); + return NULL; + } + + props->Wnode.BufferSize = (ULONG) props_size; + if (ctx->session_type == FLB_IN_ETW_SESSION_SYSTEM) { + props->Wnode.Guid = flb_etw_system_trace_control_guid; + } + else { + props->Wnode.Guid = ctx->session_guid; + } + props->Wnode.Flags = WNODE_FLAG_TRACED_GUID; + props->Wnode.ClientContext = 2; + props->LogFileMode = EVENT_TRACE_REAL_TIME_MODE; + if (ctx->session_type == FLB_IN_ETW_SESSION_SYSTEM) { + props->LogFileMode |= EVENT_TRACE_SYSTEM_LOGGER_MODE; + props->EnableFlags = ctx->kernel_flags; + } + props->LoggerNameOffset = sizeof(EVENT_TRACE_PROPERTIES); + if (ctx->buffer_size > 0) { + props->BufferSize = (ULONG) ctx->buffer_size; + } + if (ctx->minimum_buffers > 0) { + props->MinimumBuffers = (ULONG) ctx->minimum_buffers; + } + if (ctx->maximum_buffers > 0) { + props->MaximumBuffers = (ULONG) ctx->maximum_buffers; + } + if (ctx->flush_timer > 0) { + props->FlushTimer = (ULONG) ctx->flush_timer; + } + memcpy((char *) props + props->LoggerNameOffset, ctx->session_name_wide, name_size); + + return props; +} + +static void etw_timestamp_to_flb_time(const EVENT_HEADER *header, struct flb_time *tm) +{ + ULONGLONG ts; + ULONGLONG unix_100ns; + + ts = (ULONGLONG) header->TimeStamp.QuadPart; + if (ts <= FLB_IN_ETW_UNIX_EPOCH_IN_100NS) { + flb_time_get(tm); + return; + } + + unix_100ns = ts - FLB_IN_ETW_UNIX_EPOCH_IN_100NS; + flb_time_set(tm, + (time_t) (unix_100ns / 10000000ULL), + (long) ((unix_100ns % 10000000ULL) * 100ULL)); +} + +static void pack_guid_value(msgpack_packer *mp_pck, const GUID *guid) +{ + char buf[64]; + + if (guid_to_string(guid, buf, sizeof(buf)) == 0) { + pack_cstr(mp_pck, buf); + } + else { + msgpack_pack_nil(mp_pck); + } +} + +static void pack_filetime_value(msgpack_packer *mp_pck, const FILETIME *filetime) +{ + ULONGLONG ts; + ULONGLONG unix_100ns; + + ts = (((ULONGLONG) filetime->dwHighDateTime) << 32) | filetime->dwLowDateTime; + if (ts <= FLB_IN_ETW_UNIX_EPOCH_IN_100NS) { + msgpack_pack_nil(mp_pck); + return; + } + + unix_100ns = ts - FLB_IN_ETW_UNIX_EPOCH_IN_100NS; + msgpack_pack_uint64(mp_pck, unix_100ns / 10000000ULL); +} + +static int pack_non_scalar_property_value(msgpack_packer *mp_pck, + const BYTE *buf, + ULONG size, size_t expected) +{ + if (size < expected) { + msgpack_pack_nil(mp_pck); + return FLB_TRUE; + } + + if (size > expected) { + msgpack_pack_bin(mp_pck, size); + msgpack_pack_bin_body(mp_pck, buf, size); + return FLB_TRUE; + } + + return FLB_FALSE; +} + +static void pack_property_value(msgpack_packer *mp_pck, USHORT in_type, + BYTE *buf, ULONG size) +{ + char *str; + size_t len; + + switch (in_type) { + case TDH_INTYPE_UNICODESTRING: + pack_wide_string(mp_pck, (WCHAR *) buf, size); + break; + case TDH_INTYPE_ANSISTRING: + len = strnlen((char *) buf, size); + msgpack_pack_str(mp_pck, len); + msgpack_pack_str_body(mp_pck, (char *) buf, len); + break; + case TDH_INTYPE_INT8: + if (pack_non_scalar_property_value(mp_pck, buf, size, sizeof(INT8))) { + break; + } + msgpack_pack_int8(mp_pck, *((INT8 *) buf)); + break; + case TDH_INTYPE_UINT8: + if (pack_non_scalar_property_value(mp_pck, buf, size, sizeof(UINT8))) { + break; + } + msgpack_pack_uint8(mp_pck, *((UINT8 *) buf)); + break; + case TDH_INTYPE_INT16: + if (pack_non_scalar_property_value(mp_pck, buf, size, sizeof(INT16))) { + break; + } + msgpack_pack_int16(mp_pck, *((INT16 *) buf)); + break; + case TDH_INTYPE_UINT16: + if (pack_non_scalar_property_value(mp_pck, buf, size, sizeof(UINT16))) { + break; + } + msgpack_pack_uint16(mp_pck, *((UINT16 *) buf)); + break; + case TDH_INTYPE_INT32: + if (pack_non_scalar_property_value(mp_pck, buf, size, sizeof(INT32))) { + break; + } + msgpack_pack_int32(mp_pck, *((INT32 *) buf)); + break; + case TDH_INTYPE_UINT32: + case TDH_INTYPE_HEXINT32: + if (pack_non_scalar_property_value(mp_pck, buf, size, sizeof(UINT32))) { + break; + } + msgpack_pack_uint32(mp_pck, *((UINT32 *) buf)); + break; + case TDH_INTYPE_INT64: + if (pack_non_scalar_property_value(mp_pck, buf, size, sizeof(INT64))) { + break; + } + msgpack_pack_int64(mp_pck, *((INT64 *) buf)); + break; + case TDH_INTYPE_UINT64: + case TDH_INTYPE_HEXINT64: + if (pack_non_scalar_property_value(mp_pck, buf, size, sizeof(UINT64))) { + break; + } + msgpack_pack_uint64(mp_pck, *((UINT64 *) buf)); + break; + case TDH_INTYPE_BOOLEAN: + if (pack_non_scalar_property_value(mp_pck, buf, size, sizeof(BOOL))) { + break; + } + if (*((BOOL *) buf)) { + msgpack_pack_true(mp_pck); + } + else { + msgpack_pack_false(mp_pck); + } + break; + case TDH_INTYPE_FLOAT: + if (pack_non_scalar_property_value(mp_pck, buf, size, sizeof(float))) { + break; + } + msgpack_pack_float(mp_pck, *((float *) buf)); + break; + case TDH_INTYPE_DOUBLE: + if (pack_non_scalar_property_value(mp_pck, buf, size, sizeof(double))) { + break; + } + msgpack_pack_double(mp_pck, *((double *) buf)); + break; + case TDH_INTYPE_GUID: + if (pack_non_scalar_property_value(mp_pck, buf, size, sizeof(GUID))) { + break; + } + pack_guid_value(mp_pck, (GUID *) buf); + break; + case TDH_INTYPE_FILETIME: + if (pack_non_scalar_property_value(mp_pck, buf, size, sizeof(FILETIME))) { + break; + } + pack_filetime_value(mp_pck, (FILETIME *) buf); + break; + case TDH_INTYPE_POINTER: + if (size == sizeof(UINT64)) { + msgpack_pack_uint64(mp_pck, *((UINT64 *) buf)); + } + else if (size == sizeof(UINT32)) { + msgpack_pack_uint32(mp_pck, *((UINT32 *) buf)); + } + else { + msgpack_pack_nil(mp_pck); + } + break; + default: + if (size == 0) { + msgpack_pack_nil(mp_pck); + } + else { + str = (char *) buf; + msgpack_pack_bin(mp_pck, size); + msgpack_pack_bin_body(mp_pck, str, size); + } + break; + } +} + +static void pack_payload_property(msgpack_packer *mp_pck, PEVENT_RECORD record, + TRACE_EVENT_INFO *info, ULONG info_size, + ULONG index) +{ + ULONG size; + TDHSTATUS status; + BYTE *buf; + PROPERTY_DATA_DESCRIPTOR desc; + EVENT_PROPERTY_INFO *prop; + + prop = &info->EventPropertyInfoArray[index]; + if (prop->Flags & PropertyStruct) { + msgpack_pack_nil(mp_pck); + return; + } + + if (prop->NameOffset == 0 || + prop->NameOffset > info_size - sizeof(WCHAR)) { + msgpack_pack_nil(mp_pck); + return; + } + + memset(&desc, 0, sizeof(desc)); + desc.PropertyName = (ULONGLONG) ((char *) info + prop->NameOffset); + desc.ArrayIndex = ULONG_MAX; + + size = 0; + status = TdhGetPropertySize(record, 0, NULL, 1, &desc, &size); + if (status != ERROR_SUCCESS || size == 0) { + msgpack_pack_nil(mp_pck); + return; + } + + buf = flb_malloc(size); + if (buf == NULL) { + flb_errno(); + msgpack_pack_nil(mp_pck); + return; + } + + status = TdhGetProperty(record, 0, NULL, 1, &desc, size, buf); + if (status == ERROR_SUCCESS) { + pack_property_value(mp_pck, prop->nonStructType.InType, buf, size); + } + else { + msgpack_pack_nil(mp_pck); + } + + flb_free(buf); +} + +static void pack_payload(msgpack_packer *mp_pck, PEVENT_RECORD record) +{ + ULONG size; + ULONG i; + TDHSTATUS status; + TRACE_EVENT_INFO *info; + EVENT_PROPERTY_INFO *prop; + WCHAR *name; + char fallback[32]; + size_t property_info_size; + + size = 0; + status = TdhGetEventInformation(record, 0, NULL, NULL, &size); + if (status != ERROR_INSUFFICIENT_BUFFER) { + msgpack_pack_map(mp_pck, 0); + return; + } + + info = flb_malloc(size); + if (info == NULL) { + flb_errno(); + msgpack_pack_map(mp_pck, 0); + return; + } + + status = TdhGetEventInformation(record, 0, NULL, info, &size); + if (status != ERROR_SUCCESS) { + flb_free(info); + msgpack_pack_map(mp_pck, 0); + return; + } + + if (size < offsetof(TRACE_EVENT_INFO, EventPropertyInfoArray)) { + flb_free(info); + msgpack_pack_map(mp_pck, 0); + return; + } + + property_info_size = (size - offsetof(TRACE_EVENT_INFO, EventPropertyInfoArray)) / + sizeof(EVENT_PROPERTY_INFO); + if (info->TopLevelPropertyCount > property_info_size) { + flb_free(info); + msgpack_pack_map(mp_pck, 0); + return; + } + + msgpack_pack_map(mp_pck, info->TopLevelPropertyCount); + for (i = 0; i < info->TopLevelPropertyCount; i++) { + prop = &info->EventPropertyInfoArray[i]; + if (prop->NameOffset > 0 && + prop->NameOffset <= size - sizeof(WCHAR)) { + name = (WCHAR *) ((char *) info + prop->NameOffset); + if (pack_bounded_wide_string(mp_pck, name, + size - prop->NameOffset) == 0) { + pack_payload_property(mp_pck, record, info, size, i); + continue; + } + } + + snprintf(fallback, sizeof(fallback), "property_%lu", i); + pack_cstr(mp_pck, fallback); + pack_payload_property(mp_pck, record, info, size, i); + } + + flb_free(info); +} + +static void pack_related_activity_id(msgpack_packer *mp_pck, PEVENT_RECORD record) +{ + USHORT i; + PEVENT_HEADER_EXTENDED_DATA_ITEM item; + PEVENT_EXTENDED_ITEM_RELATED_ACTIVITYID related; + + for (i = 0; i < record->ExtendedDataCount; i++) { + item = &record->ExtendedData[i]; + if (item->ExtType != EVENT_HEADER_EXT_TYPE_RELATED_ACTIVITYID) { + continue; + } + + if (item->DataPtr == 0 || + item->DataSize < sizeof(EVENT_EXTENDED_ITEM_RELATED_ACTIVITYID)) { + msgpack_pack_nil(mp_pck); + return; + } + + related = (PEVENT_EXTENDED_ITEM_RELATED_ACTIVITYID) (uintptr_t) item->DataPtr; + pack_guid_value(mp_pck, &related->RelatedActivityId); + return; + } + + msgpack_pack_nil(mp_pck); +} + +static TRACEHANDLE etw_exchange_trace(struct flb_etw *ctx, TRACEHANDLE value) +{ + TRACEHANDLE previous; + + EnterCriticalSection(&ctx->handle_lock); + previous = ctx->trace; + ctx->trace = value; + LeaveCriticalSection(&ctx->handle_lock); + + return previous; +} + +static TRACEHANDLE etw_exchange_session(struct flb_etw *ctx, TRACEHANDLE value) +{ + TRACEHANDLE previous; + + EnterCriticalSection(&ctx->handle_lock); + previous = ctx->session; + ctx->session = value; + LeaveCriticalSection(&ctx->handle_lock); + + return previous; +} + +static TRACEHANDLE etw_get_session(struct flb_etw *ctx) +{ + TRACEHANDLE session; + + EnterCriticalSection(&ctx->handle_lock); + session = ctx->session; + LeaveCriticalSection(&ctx->handle_lock); + + return session; +} + +static int etw_update_loss_metrics(struct flb_etw *ctx) +{ + ULONG status; + TRACEHANDLE session; + EVENT_TRACE_PROPERTIES *properties; + uint64_t timestamp; + ULONG events_lost; + ULONG realtime_buffers_lost; + char *name; + LONG query_errors; + + if (ctx->cmt_events_lost == NULL || + ctx->cmt_realtime_buffers_lost == NULL) { + return 0; + } + + session = etw_get_session(ctx); + if (session == 0) { + return 0; + } + + properties = create_trace_properties(ctx); + if (properties == NULL) { + return -1; + } + + status = ControlTraceW(session, + ctx->session_name_wide, + properties, + EVENT_TRACE_CONTROL_QUERY); + if (status != ERROR_SUCCESS) { + flb_free(properties); + query_errors = InterlockedIncrement(&ctx->query_errors); + if (query_errors == 1 || query_errors % 1000 == 0) { + flb_plg_warn(ctx->ins, + "ControlTrace query failed for session '%S' (status=%lu)", + ctx->session_name_wide, status); + } + return -1; + } + + events_lost = properties->EventsLost; + realtime_buffers_lost = properties->RealTimeBuffersLost; + flb_free(properties); + + name = (char *) flb_input_name(ctx->ins); + timestamp = cfl_time_now(); + + cmt_gauge_set(ctx->cmt_events_lost, + timestamp, + (double) events_lost, + 1, (char *[]) {name}); + cmt_gauge_set(ctx->cmt_realtime_buffers_lost, + timestamp, + (double) realtime_buffers_lost, + 1, (char *[]) {name}); + + return 0; +} + +static int etw_loss_metrics_collect(struct flb_input_instance *in, + struct flb_config *config, void *data) +{ + struct flb_etw *ctx; + + ctx = data; + if (InterlockedCompareExchange(&ctx->paused, 0, 0) || + InterlockedCompareExchange(&ctx->exiting, 0, 0)) { + return 0; + } + + etw_update_loss_metrics(ctx); + + return 0; +} + +static VOID WINAPI etw_event_callback(PEVENT_RECORD record) +{ + struct flb_etw *ctx; + struct flb_time timestamp; + msgpack_sbuffer mp_sbuf; + msgpack_packer mp_pck; + LONG append_errors; + int ret; + + ctx = (struct flb_etw *) record->UserContext; + if (ctx == NULL) { + return; + } + + EnterCriticalSection(&ctx->callback_lock); + + if (InterlockedCompareExchange(&ctx->paused, 0, 0) || + InterlockedCompareExchange(&ctx->exiting, 0, 0)) { + LeaveCriticalSection(&ctx->callback_lock); + return; + } + + etw_timestamp_to_flb_time(&record->EventHeader, ×tamp); + + msgpack_sbuffer_init(&mp_sbuf); + msgpack_packer_init(&mp_pck, &mp_sbuf, msgpack_sbuffer_write); + + msgpack_pack_array(&mp_pck, 2); + flb_time_append_to_msgpack(×tamp, &mp_pck, FLB_TIME_ETFMT_INT); + msgpack_pack_map(&mp_pck, FLB_IN_ETW_PROVIDER_MAP_SIZE); + + pack_cstr(&mp_pck, "provider_guid"); + pack_guid_value(&mp_pck, &record->EventHeader.ProviderId); + + pack_cstr(&mp_pck, "provider_name"); + if (ctx->provider_name != NULL) { + pack_cstr(&mp_pck, ctx->provider_name); + } + else { + msgpack_pack_nil(&mp_pck); + } + + pack_cstr(&mp_pck, "event_id"); + msgpack_pack_uint16(&mp_pck, record->EventHeader.EventDescriptor.Id); + + pack_cstr(&mp_pck, "version"); + msgpack_pack_uint8(&mp_pck, record->EventHeader.EventDescriptor.Version); + + pack_cstr(&mp_pck, "channel"); + msgpack_pack_uint8(&mp_pck, record->EventHeader.EventDescriptor.Channel); + + pack_cstr(&mp_pck, "level"); + msgpack_pack_uint8(&mp_pck, record->EventHeader.EventDescriptor.Level); + + pack_cstr(&mp_pck, "task"); + msgpack_pack_uint16(&mp_pck, record->EventHeader.EventDescriptor.Task); + + pack_cstr(&mp_pck, "opcode"); + msgpack_pack_uint8(&mp_pck, record->EventHeader.EventDescriptor.Opcode); + + pack_cstr(&mp_pck, "keywords"); + msgpack_pack_uint64(&mp_pck, record->EventHeader.EventDescriptor.Keyword); + + pack_cstr(&mp_pck, "process_id"); + msgpack_pack_uint32(&mp_pck, record->EventHeader.ProcessId); + + pack_cstr(&mp_pck, "thread_id"); + msgpack_pack_uint32(&mp_pck, record->EventHeader.ThreadId); + + pack_cstr(&mp_pck, "activity_id"); + pack_guid_value(&mp_pck, &record->EventHeader.ActivityId); + + pack_cstr(&mp_pck, "related_activity_id"); + pack_related_activity_id(&mp_pck, record); + + pack_cstr(&mp_pck, "payload"); + pack_payload(&mp_pck, record); + + ret = flb_input_ingress_queue_log(ctx->ins, NULL, 0, + mp_sbuf.data, mp_sbuf.size); + if (ret != 0) { + append_errors = InterlockedIncrement(&ctx->append_errors); + if (append_errors == 1 || append_errors % 1000 == 0) { + flb_plg_warn(ctx->ins, + "failed to enqueue ETW event record (%ld failures)", + append_errors); + } + } + + msgpack_sbuffer_destroy(&mp_sbuf); + LeaveCriticalSection(&ctx->callback_lock); +} + +static void etw_close_trace(struct flb_etw *ctx) +{ + TRACEHANDLE trace; + + trace = etw_exchange_trace(ctx, INVALID_PROCESSTRACE_HANDLE); + if (trace != INVALID_PROCESSTRACE_HANDLE) { + CloseTrace(trace); + } +} + +static ULONG etw_stop_session_by_name(struct flb_etw *ctx) +{ + ULONG status; + EVENT_TRACE_PROPERTIES *properties; + + properties = create_trace_properties(ctx); + if (properties == NULL) { + return ERROR_NOT_ENOUGH_MEMORY; + } + + status = ControlTraceW(0, + ctx->session_name_wide, + properties, + EVENT_TRACE_CONTROL_STOP); + flb_free(properties); + + return status; +} + +static void etw_signal_startup(struct flb_etw *ctx, int status) +{ + pthread_mutex_lock(&ctx->startup_lock); + if (!ctx->startup_done) { + ctx->startup_status = status; + ctx->startup_done = FLB_TRUE; + pthread_cond_signal(&ctx->startup_cond); + } + pthread_mutex_unlock(&ctx->startup_lock); +} + +static int etw_wait_startup(struct flb_etw *ctx) +{ + int status; + + pthread_mutex_lock(&ctx->startup_lock); + while (!ctx->startup_done) { + pthread_cond_wait(&ctx->startup_cond, &ctx->startup_lock); + } + status = ctx->startup_status; + pthread_mutex_unlock(&ctx->startup_lock); + + return status; +} + +static void etw_disable_provider(struct flb_etw *ctx, TRACEHANDLE session) +{ + ULONG status; + + if (session == 0 || ctx->session_type == FLB_IN_ETW_SESSION_SYSTEM) { + return; + } + + status = EnableTraceEx2(session, + &ctx->provider_guid, + EVENT_CONTROL_CODE_DISABLE_PROVIDER, + (UCHAR) ctx->level, + ctx->match_any_keyword, + ctx->match_all_keyword, + 0, + NULL); + if (status != ERROR_SUCCESS && status != ERROR_WMI_INSTANCE_NOT_FOUND) { + flb_plg_warn(ctx->ins, "EnableTraceEx2 disable failed (status=%lu)", status); + } +} + +static void etw_stop_session(struct flb_etw *ctx) +{ + TRACEHANDLE session; + ULONG status; + EVENT_TRACE_PROPERTIES *properties; + + properties = create_trace_properties(ctx); + if (properties == NULL) { + flb_plg_warn(ctx->ins, "could not allocate ETW stop properties"); + return; + } + + session = etw_exchange_session(ctx, 0); + if (session == 0) { + flb_free(properties); + return; + } + + etw_disable_provider(ctx, session); + status = ControlTraceW(session, + ctx->session_name_wide, + properties, + EVENT_TRACE_CONTROL_STOP); + flb_free(properties); + if (status != ERROR_SUCCESS && status != ERROR_WMI_INSTANCE_NOT_FOUND) { + flb_plg_warn(ctx->ins, "ControlTrace stop failed (status=%lu)", status); + } +} + +static ULONG etw_start_session(struct flb_etw *ctx) +{ + ULONG status; + TRACEHANDLE session; + EVENT_TRACE_PROPERTIES *properties; + + session = 0; + properties = create_trace_properties(ctx); + if (properties == NULL) { + return ERROR_NOT_ENOUGH_MEMORY; + } + + status = StartTraceW(&session, ctx->session_name_wide, properties); + flb_free(properties); + if (status != ERROR_ALREADY_EXISTS) { + if (status == ERROR_SUCCESS) { + etw_exchange_session(ctx, session); + } + return status; + } + + if (ctx->stale_session_action == FLB_IN_ETW_STALE_ACTION_FAIL) { + return ERROR_ALREADY_EXISTS; + } + + flb_plg_warn(ctx->ins, + "ETW session '%S' already exists; stopping stale session and retrying", + ctx->session_name_wide); + + status = etw_stop_session_by_name(ctx); + if (status != ERROR_SUCCESS && status != ERROR_WMI_INSTANCE_NOT_FOUND) { + flb_plg_error(ctx->ins, + "could not stop existing ETW session '%S' (status=%lu)", + ctx->session_name_wide, status); + return ERROR_ALREADY_EXISTS; + } + + session = 0; + properties = create_trace_properties(ctx); + if (properties == NULL) { + return ERROR_NOT_ENOUGH_MEMORY; + } + + status = StartTraceW(&session, ctx->session_name_wide, properties); + flb_free(properties); + if (status == ERROR_SUCCESS) { + etw_exchange_session(ctx, session); + } + + return status; +} + +static void *etw_worker(void *data) +{ + ULONG status; + TRACEHANDLE session; + TRACEHANDLE trace; + EVENT_TRACE_LOGFILEW logfile; + struct flb_etw *ctx; + + ctx = data; + + status = etw_start_session(ctx); + if (status != ERROR_SUCCESS) { + if (status == ERROR_ALREADY_EXISTS) { + flb_plg_error(ctx->ins, + "ETW session '%S' already exists and could not be reused", + ctx->session_name_wide); + } + else { + flb_plg_error(ctx->ins, "StartTrace failed for session '%S' (status=%lu)", + ctx->session_name_wide, status); + } + etw_signal_startup(ctx, -1); + InterlockedExchange(&ctx->worker_finished, 1); + return NULL; + } + + if (ctx->session_type == FLB_IN_ETW_SESSION_PROVIDER) { + session = etw_get_session(ctx); + + status = EnableTraceEx2(session, + &ctx->provider_guid, + EVENT_CONTROL_CODE_ENABLE_PROVIDER, + (UCHAR) ctx->level, + ctx->match_any_keyword, + ctx->match_all_keyword, + 0, + NULL); + if (status != ERROR_SUCCESS) { + flb_plg_error(ctx->ins, "EnableTraceEx2 failed (status=%lu)", status); + etw_stop_session(ctx); + etw_signal_startup(ctx, -1); + InterlockedExchange(&ctx->worker_finished, 1); + return NULL; + } + } + + memset(&logfile, 0, sizeof(logfile)); + logfile.LoggerName = ctx->session_name_wide; + logfile.ProcessTraceMode = PROCESS_TRACE_MODE_REAL_TIME | + PROCESS_TRACE_MODE_EVENT_RECORD; + logfile.EventRecordCallback = etw_event_callback; + logfile.Context = ctx; + + trace = OpenTraceW(&logfile); + etw_exchange_trace(ctx, trace); + if (trace == INVALID_PROCESSTRACE_HANDLE) { + flb_plg_error(ctx->ins, "OpenTrace failed (error=%lu)", GetLastError()); + etw_stop_session(ctx); + etw_signal_startup(ctx, -1); + InterlockedExchange(&ctx->worker_finished, 1); + return NULL; + } + + etw_signal_startup(ctx, 0); + + status = ProcessTrace(&trace, 1, NULL, NULL); + if (status != ERROR_SUCCESS && + !InterlockedCompareExchange(&ctx->paused, 0, 0) && + !InterlockedCompareExchange(&ctx->exiting, 0, 0)) { + flb_plg_error(ctx->ins, "ProcessTrace failed (status=%lu)", status); + } + + etw_close_trace(ctx); + + if (!InterlockedCompareExchange(&ctx->exiting, 0, 0)) { + etw_stop_session(ctx); + } + + InterlockedExchange(&ctx->worker_finished, 1); + + return NULL; +} + +static void etw_reset_startup(struct flb_etw *ctx) +{ + pthread_mutex_lock(&ctx->startup_lock); + ctx->startup_done = FLB_FALSE; + ctx->startup_status = 0; + pthread_mutex_unlock(&ctx->startup_lock); +} + +static int etw_start_worker(struct flb_etw *ctx) +{ + int ret; + + if (ctx->thread_created) { + if (!InterlockedCompareExchange(&ctx->worker_finished, 0, 0)) { + return 0; + } + + pthread_join(ctx->thread, NULL); + ctx->thread_created = FLB_FALSE; + } + + etw_reset_startup(ctx); + InterlockedExchange(&ctx->worker_finished, 0); + + ret = pthread_create(&ctx->thread, NULL, etw_worker, ctx); + if (ret != 0) { + flb_plg_error(ctx->ins, "could not create ETW worker thread"); + return -1; + } + ctx->thread_created = FLB_TRUE; + + if (etw_wait_startup(ctx) != 0) { + pthread_join(ctx->thread, NULL); + ctx->thread_created = FLB_FALSE; + return -1; + } + + return 0; +} + +static void etw_stop_worker(struct flb_etw *ctx, int exiting) +{ + if (!ctx->thread_created) { + return; + } + + if (exiting) { + InterlockedExchange(&ctx->exiting, 1); + } + + EnterCriticalSection(&ctx->callback_lock); + LeaveCriticalSection(&ctx->callback_lock); + + etw_close_trace(ctx); + etw_stop_session(ctx); + pthread_join(ctx->thread, NULL); + ctx->thread_created = FLB_FALSE; +} + +static void etw_config_destroy(struct flb_etw *ctx) +{ + if (ctx == NULL) { + return; + } + + etw_stop_worker(ctx, FLB_TRUE); + + if (ctx->session_name_wide != NULL) { + flb_free(ctx->session_name_wide); + } + + if (ctx->startup_sync_initialized) { + pthread_cond_destroy(&ctx->startup_cond); + pthread_mutex_destroy(&ctx->startup_lock); + } + + DeleteCriticalSection(&ctx->handle_lock); + DeleteCriticalSection(&ctx->callback_lock); + + flb_free(ctx); +} + +static int configure_provider(struct flb_etw *ctx) +{ + WCHAR *provider_name_wide; + GUID name_guid; + char guid_buf[64]; + int have_guid; + int have_name; + + have_guid = ctx->provider_guid_str != NULL && ctx->provider_guid_str[0] != '\0'; + have_name = ctx->provider_name != NULL && ctx->provider_name[0] != '\0'; + + if (ctx->session_type == FLB_IN_ETW_SESSION_SYSTEM) { + if (have_guid || have_name) { + flb_plg_error(ctx->ins, + "provider_guid/provider_name cannot be used with session_type system"); + return -1; + } + return 0; + } + + if (!have_guid && !have_name) { + flb_plg_error(ctx->ins, "either provider_guid or provider_name must be set"); + return -1; + } + + if (have_guid && parse_guid(ctx->provider_guid_str, &ctx->provider_guid) != 0) { + flb_plg_error(ctx->ins, "invalid provider_guid '%s'", ctx->provider_guid_str); + return -1; + } + + if (!have_name) { + return 0; + } + + provider_name_wide = utf8_to_wide(ctx->provider_name); + if (provider_name_wide == NULL) { + flb_plg_error(ctx->ins, "could not convert provider_name '%s' to UTF-16", + ctx->provider_name); + return -1; + } + + if (resolve_provider_name(ctx, provider_name_wide, &name_guid) != 0) { + flb_free(provider_name_wide); + return -1; + } + flb_free(provider_name_wide); + + if (have_guid && memcmp(&ctx->provider_guid, &name_guid, sizeof(GUID)) != 0) { + guid_to_string(&name_guid, guid_buf, sizeof(guid_buf)); + flb_plg_error(ctx->ins, + "provider_name '%s' resolves to %s, which does not match provider_guid '%s'", + ctx->provider_name, guid_buf, ctx->provider_guid_str); + return -1; + } + + ctx->provider_guid = name_guid; + + return 0; +} + +static int configure_session_properties(struct flb_etw *ctx) +{ + if (ctx->buffer_size < 0) { + flb_plg_error(ctx->ins, "buffer_size must be zero or greater"); + return -1; + } + + if (ctx->minimum_buffers < 0) { + flb_plg_error(ctx->ins, "minimum_buffers must be zero or greater"); + return -1; + } + + if (ctx->maximum_buffers < 0) { + flb_plg_error(ctx->ins, "maximum_buffers must be zero or greater"); + return -1; + } + + if (ctx->flush_timer < 0) { + flb_plg_error(ctx->ins, "flush_timer must be zero or greater"); + return -1; + } + + if (ctx->minimum_buffers > 0 && + ctx->maximum_buffers > 0 && + ctx->minimum_buffers > ctx->maximum_buffers) { + flb_plg_error(ctx->ins, + "minimum_buffers must be less than or equal to maximum_buffers"); + return -1; + } + + return 0; +} + +static int configure_session_name(struct flb_etw *ctx) +{ + if (ctx->session_type != FLB_IN_ETW_SESSION_SYSTEM) { + return 0; + } + + if (ctx->session_name == NULL || + strcmp(ctx->session_name, FLB_IN_ETW_DEFAULT_SESSION_NAME) == 0 || + strcmp(ctx->session_name, KERNEL_LOGGER_NAMEA) == 0) { + return 0; + } + + flb_plg_error(ctx->ins, + "session_type system requires session_name '%s'", + KERNEL_LOGGER_NAMEA); + + return -1; +} + +static const char *effective_session_name(struct flb_etw *ctx) +{ + if (ctx->session_type == FLB_IN_ETW_SESSION_SYSTEM && + ctx->session_name != NULL && + strcmp(ctx->session_name, FLB_IN_ETW_DEFAULT_SESSION_NAME) == 0) { + return KERNEL_LOGGER_NAMEA; + } + + return ctx->session_name; +} + +static int configure_loss_metrics(struct flb_etw *ctx, struct flb_config *config) +{ + int ret; + time_t interval; + + ctx->loss_metrics_collector_id = -1; + + if (ctx->ins->cmt == NULL) { + return 0; + } + + ctx->cmt_events_lost = cmt_gauge_create(ctx->ins->cmt, + "fluentbit", "input", + "etw_events_lost", + "Number of ETW events lost by the session.", + 1, (char *[]) {"name"}); + if (ctx->cmt_events_lost == NULL) { + flb_plg_error(ctx->ins, "could not create ETW events lost metric"); + return -1; + } + + ctx->cmt_realtime_buffers_lost = + cmt_gauge_create(ctx->ins->cmt, + "fluentbit", "input", + "etw_realtime_buffers_lost", + "Number of ETW real-time buffers lost by the session.", + 1, (char *[]) {"name"}); + if (ctx->cmt_realtime_buffers_lost == NULL) { + flb_plg_error(ctx->ins, "could not create ETW real-time buffers lost metric"); + return -1; + } + + interval = ctx->flush_timer > 0 ? ctx->flush_timer : 1; + ret = flb_input_set_collector_time(ctx->ins, + etw_loss_metrics_collect, + interval, 0, + config); + if (ret == -1) { + flb_plg_error(ctx->ins, "could not set ETW loss metrics collector"); + return -1; + } + + ctx->loss_metrics_collector_id = ret; + + return 0; +} + +static int in_etw_init(struct flb_input_instance *in, + struct flb_config *config, void *data) +{ + HRESULT hres; + int ret; + const char *session_name; + struct flb_etw *ctx; + + ctx = flb_calloc(1, sizeof(struct flb_etw)); + if (ctx == NULL) { + flb_errno(); + return -1; + } + + ctx->ins = in; + ctx->trace = INVALID_PROCESSTRACE_HANDLE; + ctx->loss_metrics_collector_id = -1; + InitializeCriticalSection(&ctx->handle_lock); + InitializeCriticalSection(&ctx->callback_lock); + + ret = pthread_mutex_init(&ctx->startup_lock, NULL); + if (ret != 0) { + DeleteCriticalSection(&ctx->callback_lock); + DeleteCriticalSection(&ctx->handle_lock); + flb_free(ctx); + return -1; + } + + ret = pthread_cond_init(&ctx->startup_cond, NULL); + if (ret != 0) { + pthread_mutex_destroy(&ctx->startup_lock); + DeleteCriticalSection(&ctx->callback_lock); + DeleteCriticalSection(&ctx->handle_lock); + flb_free(ctx); + return -1; + } + ctx->startup_sync_initialized = FLB_TRUE; + + ret = flb_input_config_map_set(in, ctx); + if (ret == -1) { + etw_config_destroy(ctx); + return -1; + } + + if (parse_session_type(ctx) != 0) { + etw_config_destroy(ctx); + return -1; + } + + if (parse_stale_session_action(ctx) != 0) { + etw_config_destroy(ctx); + return -1; + } + + if (ctx->level < 0 || ctx->level > 255) { + flb_plg_error(ctx->ins, "level must be between 0 and 255"); + etw_config_destroy(ctx); + return -1; + } + + if (parse_uint64(ctx->match_any_keyword_str, &ctx->match_any_keyword) != 0) { + flb_plg_error(ctx->ins, "invalid match_any_keyword '%s'", + ctx->match_any_keyword_str); + etw_config_destroy(ctx); + return -1; + } + + if (parse_uint64(ctx->match_all_keyword_str, &ctx->match_all_keyword) != 0) { + flb_plg_error(ctx->ins, "invalid match_all_keyword '%s'", + ctx->match_all_keyword_str); + etw_config_destroy(ctx); + return -1; + } + + if (parse_kernel_flags(ctx) != 0) { + etw_config_destroy(ctx); + return -1; + } + + if (configure_session_properties(ctx) != 0) { + etw_config_destroy(ctx); + return -1; + } + + if (configure_session_name(ctx) != 0) { + etw_config_destroy(ctx); + return -1; + } + + if (configure_provider(ctx) != 0) { + etw_config_destroy(ctx); + return -1; + } + + session_name = effective_session_name(ctx); + ctx->session_name_wide = utf8_to_wide(session_name); + if (ctx->session_name_wide == NULL) { + flb_plg_error(ctx->ins, "could not convert session_name '%s' to UTF-16", + session_name); + etw_config_destroy(ctx); + return -1; + } + + hres = CoCreateGuid(&ctx->session_guid); + if (FAILED(hres)) { + flb_plg_error(ctx->ins, "could not generate ETW session GUID (hr=0x%08lx)", + (unsigned long) hres); + etw_config_destroy(ctx); + return -1; + } + + flb_input_set_context(in, ctx); + + ret = flb_input_ingress_enable(in); + if (ret != 0) { + flb_plg_error(ctx->ins, "could not enable ETW ingress queue"); + etw_config_destroy(ctx); + return -1; + } + + if (etw_start_worker(ctx) != 0) { + etw_config_destroy(ctx); + return -1; + } + + if (configure_loss_metrics(ctx, config) != 0) { + etw_config_destroy(ctx); + return -1; + } + + return 0; +} + +static void in_etw_pause(void *data, struct flb_config *config) +{ + struct flb_etw *ctx; + + ctx = data; + InterlockedExchange(&ctx->paused, 1); + if (ctx->loss_metrics_collector_id >= 0) { + flb_input_collector_pause(ctx->loss_metrics_collector_id, ctx->ins); + } + etw_stop_worker(ctx, FLB_FALSE); +} + +static void in_etw_resume(void *data, struct flb_config *config) +{ + struct flb_etw *ctx; + + ctx = data; + InterlockedExchange(&ctx->paused, 0); + if (etw_start_worker(ctx) != 0) { + InterlockedExchange(&ctx->paused, 1); + return; + } + + if (ctx->loss_metrics_collector_id >= 0) { + flb_input_collector_resume(ctx->loss_metrics_collector_id, ctx->ins); + } +} + +static int in_etw_exit(void *data, struct flb_config *config) +{ + struct flb_etw *ctx; + + ctx = data; + etw_config_destroy(ctx); + + return 0; +} + +static struct flb_config_map config_map[] = { + { + FLB_CONFIG_MAP_STR, "provider_guid", NULL, + 0, FLB_TRUE, offsetof(struct flb_etw, provider_guid_str), + "ETW provider GUID to enable" + }, + { + FLB_CONFIG_MAP_STR, "provider_name", NULL, + 0, FLB_TRUE, offsetof(struct flb_etw, provider_name), + "ETW provider name to resolve and enable" + }, + { + FLB_CONFIG_MAP_STR, "session_name", FLB_IN_ETW_DEFAULT_SESSION_NAME, + 0, FLB_TRUE, offsetof(struct flb_etw, session_name), + "ETW real-time session name. When session_type is system and this value " + "is left as the default, the plugin uses the Windows kernel logger " + "session name 'NT Kernel Logger'." + }, + { + FLB_CONFIG_MAP_STR, "session_type", FLB_IN_ETW_DEFAULT_SESSION_TYPE, + 0, FLB_TRUE, offsetof(struct flb_etw, session_type_str), + "ETW session type: provider for a real-time provider consumer, or system " + "for a system logger session using kernel_flags." + }, + { + FLB_CONFIG_MAP_STR, "stale_session_action", FLB_IN_ETW_DEFAULT_STALE_ACTION, + 0, FLB_TRUE, offsetof(struct flb_etw, stale_session_action_str), + "Action when the ETW session already exists: stop stops the existing " + "session and retries; fail returns an error without stopping it." + }, + { + FLB_CONFIG_MAP_INT, "level", FLB_IN_ETW_DEFAULT_LEVEL, + 0, FLB_TRUE, offsetof(struct flb_etw, level), + "ETW provider level" + }, + { + FLB_CONFIG_MAP_STR, "match_any_keyword", FLB_IN_ETW_DEFAULT_MATCH_ANY, + 0, FLB_TRUE, offsetof(struct flb_etw, match_any_keyword_str), + "ETW MatchAnyKeyword mask" + }, + { + FLB_CONFIG_MAP_STR, "match_all_keyword", FLB_IN_ETW_DEFAULT_MATCH_ALL, + 0, FLB_TRUE, offsetof(struct flb_etw, match_all_keyword_str), + "ETW MatchAllKeyword mask" + }, + { + FLB_CONFIG_MAP_STR, "kernel_flags", FLB_IN_ETW_DEFAULT_KERNEL_FLAGS, + 0, FLB_TRUE, offsetof(struct flb_etw, kernel_flags_str), + "Comma-separated kernel flags used with session_type system. Supported " + "names: process, thread, image_load, cswitch, tcpip, disk_io. A numeric " + "EVENT_TRACE_FLAG_* mask is also accepted." + }, + { + FLB_CONFIG_MAP_INT, "buffer_size", FLB_IN_ETW_DEFAULT_BUFFER_SIZE, + 0, FLB_TRUE, offsetof(struct flb_etw, buffer_size), + "ETW session buffer size in kilobytes. The session buffer memory upper " + "bound is buffer_size KB * maximum_buffers; the default is 64 KB * 32 " + "= 2048 KB. Zero uses the Windows default." + }, + { + FLB_CONFIG_MAP_INT, "minimum_buffers", FLB_IN_ETW_DEFAULT_MIN_BUFFERS, + 0, FLB_TRUE, offsetof(struct flb_etw, minimum_buffers), + "Minimum number of ETW session buffers. Zero uses the Windows default." + }, + { + FLB_CONFIG_MAP_INT, "maximum_buffers", FLB_IN_ETW_DEFAULT_MAX_BUFFERS, + 0, FLB_TRUE, offsetof(struct flb_etw, maximum_buffers), + "Maximum number of ETW session buffers. The session buffer memory upper " + "bound is buffer_size KB * maximum_buffers; the default is 64 KB * 32 " + "= 2048 KB. Zero uses the Windows default." + }, + { + FLB_CONFIG_MAP_INT, "flush_timer", FLB_IN_ETW_DEFAULT_FLUSH_TIMER, + 0, FLB_TRUE, offsetof(struct flb_etw, flush_timer), + "ETW session flush timer in seconds. Zero uses the Windows default." + }, + {0} +}; + +struct flb_input_plugin in_etw_plugin = { + .name = "event_tracing_windows", + .description = "Event Tracing for Windows", + .cb_init = in_etw_init, + .cb_pre_run = NULL, + .cb_collect = NULL, + .cb_flush_buf = NULL, + .cb_pause = in_etw_pause, + .cb_resume = in_etw_resume, + .cb_exit = in_etw_exit, + .config_map = config_map +}; diff --git a/plugins/in_etw/etw.h b/plugins/in_etw/etw.h new file mode 100644 index 00000000000..2ac2fea12dd --- /dev/null +++ b/plugins/in_etw/etw.h @@ -0,0 +1,94 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2026 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_IN_ETW_H +#define FLB_IN_ETW_H + +#include +#include + +#include +#include +#include + +#define FLB_IN_ETW_DEFAULT_SESSION_NAME "fluent-bit-event-tracing-windows" +#define FLB_IN_ETW_DEFAULT_LEVEL "5" +#define FLB_IN_ETW_DEFAULT_MATCH_ANY "0xffffffffffffffff" +#define FLB_IN_ETW_DEFAULT_MATCH_ALL "0" +#define FLB_IN_ETW_DEFAULT_BUFFER_SIZE "64" +#define FLB_IN_ETW_DEFAULT_MIN_BUFFERS "4" +#define FLB_IN_ETW_DEFAULT_MAX_BUFFERS "32" +#define FLB_IN_ETW_DEFAULT_FLUSH_TIMER "1" +#define FLB_IN_ETW_DEFAULT_STALE_ACTION "stop" +#define FLB_IN_ETW_DEFAULT_SESSION_TYPE "provider" + +#define FLB_IN_ETW_SESSION_PROVIDER 0 +#define FLB_IN_ETW_SESSION_SYSTEM 1 + +#define FLB_IN_ETW_STALE_ACTION_STOP 0 +#define FLB_IN_ETW_STALE_ACTION_FAIL 1 + +struct flb_etw { + flb_sds_t provider_guid_str; + flb_sds_t provider_name; + flb_sds_t session_name; + flb_sds_t match_any_keyword_str; + flb_sds_t match_all_keyword_str; + flb_sds_t stale_session_action_str; + flb_sds_t session_type_str; + flb_sds_t kernel_flags_str; + int level; + int buffer_size; + int minimum_buffers; + int maximum_buffers; + int flush_timer; + int stale_session_action; + int session_type; + + GUID provider_guid; + ULONGLONG match_any_keyword; + ULONGLONG match_all_keyword; + ULONG kernel_flags; + GUID session_guid; + WCHAR *session_name_wide; + TRACEHANDLE session; + TRACEHANDLE trace; + CRITICAL_SECTION handle_lock; + CRITICAL_SECTION callback_lock; + pthread_mutex_t startup_lock; + pthread_cond_t startup_cond; + + LONG exiting; + LONG paused; + LONG worker_finished; + LONG append_errors; + LONG query_errors; + int startup_sync_initialized; + int startup_done; + int startup_status; + int thread_created; + int loss_metrics_collector_id; + pthread_t thread; + + struct cmt_gauge *cmt_events_lost; + struct cmt_gauge *cmt_realtime_buffers_lost; + struct flb_input_instance *ins; +}; + +#endif diff --git a/plugins/in_event_test/event_test.c b/plugins/in_event_test/event_test.c index bf09de01b0e..4a224ab82c6 100644 --- a/plugins/in_event_test/event_test.c +++ b/plugins/in_event_test/event_test.c @@ -33,8 +33,8 @@ #define STATUS_PENDING -1 #define CALLBACK_TIME 2 /* 2 seconds */ -#define SERVER_PORT "9092" -#define SERVER_IFACE "0.0.0.0" +#define SERVER_PORT "0" +#define SERVER_IFACE "127.0.0.1" struct unit_test { int id; @@ -55,7 +55,8 @@ struct unit_test tests[] = { struct event_test { flb_pipefd_t pipe[2]; - int server_fd; + flb_sockfd_t server_fd; + int server_port; int client_coll_id; struct flb_upstream *upstream; struct unit_test *tests; @@ -127,9 +128,9 @@ static int cb_collector_time(struct flb_input_instance *ins, * to our local pipe. */ val = 1; - ret = write(ctx->pipe[1], &val, sizeof(val)); + ret = flb_pipe_w(ctx->pipe[1], &val, sizeof(val)); if (ret == -1) { - flb_errno(); + flb_pipe_error(); set_unit_test_status(ctx, 0, STATUS_ERROR); flb_engine_exit(config); } @@ -143,13 +144,13 @@ static int cb_collector_fd(struct flb_input_instance *ins, struct flb_config *config, void *in_context) { uint64_t val = 0; - size_t bytes; + ssize_t bytes; struct unit_test *ut; struct event_test *ctx = (struct event_test *) in_context; - bytes = read(ctx->pipe[0], &val, sizeof(val)); + bytes = flb_pipe_r(ctx->pipe[0], &val, sizeof(val)); if (bytes <= 0) { - flb_errno(); + flb_pipe_error(); set_unit_test_status(ctx, 1, STATUS_ERROR); flb_engine_exit(config); } @@ -242,12 +243,28 @@ static struct event_test *config_create(struct flb_input_instance *ins) return ctx; } +static int get_server_port(flb_sockfd_t fd) +{ + int ret; + socklen_t len; + struct sockaddr_in addr; + + len = sizeof(addr); + ret = getsockname(fd, (struct sockaddr *) &addr, &len); + if (ret == -1) { + return -1; + } + + return ntohs(addr.sin_port); +} + /* Initialize plugin */ static int cb_event_test_init(struct flb_input_instance *ins, struct flb_config *config, void *data) { - int fd; int ret; + int port; + flb_sockfd_t fd; struct unit_test *ut; struct event_test *ctx = NULL; struct flb_upstream *upstream; @@ -297,7 +314,15 @@ static int cb_event_test_init(struct flb_input_instance *ins, return -1; } flb_net_socket_nonblocking(fd); - ctx->server_fd = fd; + ctx->server_fd = fd; + + port = get_server_port(ctx->server_fd); + if (port <= 0) { + flb_errno(); + config_destroy(ctx); + return -1; + } + ctx->server_port = port; /* socket server */ ret = flb_input_set_collector_socket(ins, @@ -321,7 +346,7 @@ static int cb_event_test_init(struct flb_input_instance *ins, ctx->client_coll_id = ret; /* upstream context for socket client */ - upstream = flb_upstream_create(config, "127.0.0.1", atoi(SERVER_PORT), + upstream = flb_upstream_create(config, SERVER_IFACE, ctx->server_port, FLB_IO_TCP, NULL); if (!upstream) { config_destroy(ctx); diff --git a/plugins/in_forward/fw.c b/plugins/in_forward/fw.c index 3c2ba41b0f7..f9824ba2095 100644 --- a/plugins/in_forward/fw.c +++ b/plugins/in_forward/fw.c @@ -22,6 +22,7 @@ #include #include #include +#include #include #include #include @@ -37,6 +38,13 @@ #include "fw_conn.h" #include "fw_config.h" +static int in_fw_collect(struct flb_input_instance *ins, + struct flb_config *config, void *in_context); +static int in_fw_collect_ctx(struct flb_in_fw_config *ctx); +static void delete_users(struct flb_in_fw_config *ctx); +static int setup_users(struct flb_in_fw_config *ctx, + struct flb_input_instance *ins); + #ifdef FLB_HAVE_UNIX_SOCKET static int remove_existing_socket_file(char *socket_path) { @@ -123,13 +131,18 @@ static int fw_unix_create(struct flb_in_fw_config *ctx) */ static int in_fw_collect(struct flb_input_instance *ins, struct flb_config *config, void *in_context) +{ + (void) ins; + (void) config; + + return in_fw_collect_ctx(in_context); +} + +static int in_fw_collect_ctx(struct flb_in_fw_config *ctx) { int state_backup; struct flb_connection *connection; struct fw_conn *conn; - struct flb_in_fw_config *ctx; - - ctx = in_context; state_backup = ctx->state; ctx->state = FW_INSTANCE_STATE_ACCEPTING_CLIENT; @@ -143,22 +156,22 @@ static int in_fw_collect(struct flb_input_instance *ins, return -1; } - if (!config->is_ingestion_active) { + if (!ctx->ins->config->is_ingestion_active) { flb_downstream_conn_release(connection); ctx->state = state_backup; return -1; } - if(ctx->is_paused) { + if (ctx->is_paused) { + flb_plg_trace(ctx->ins, "TCP connection will be closed FD=%i", connection->fd); flb_downstream_conn_release(connection); - flb_plg_trace(ins, "TCP connection will be closed FD=%i", connection->fd); ctx->state = state_backup; return -1; } - flb_plg_trace(ins, "new TCP connection arrived FD=%i", connection->fd); + flb_plg_trace(ctx->ins, "new TCP connection arrived FD=%i", connection->fd); conn = fw_conn_add(connection, ctx); if (!conn) { @@ -272,11 +285,217 @@ static int setup_users(struct flb_in_fw_config *ctx, return 0; } +static int in_fw_worker_listener_event(void *data) +{ + struct mk_event *event; + + event = data; + + return in_fw_collect_ctx(event->data); +} + +static int in_fw_start_tcp_listener(struct flb_in_fw_config *ctx, + struct flb_config *config, + struct flb_net_setup *net_setup, + struct mk_event_loop *event_loop, + int use_collector) +{ + int ret; + unsigned short int port; + + port = (unsigned short int) strtoul(ctx->tcp_port, NULL, 10); + + ctx->downstream = flb_downstream_create(FLB_TRANSPORT_TCP, + ctx->ins->flags, + ctx->listen, + port, + ctx->ins->tls, + config, + net_setup); + + if (ctx->downstream == NULL) { + flb_plg_error(ctx->ins, + "could not initialize downstream on %s:%s. Aborting", + ctx->listen, ctx->tcp_port); + return -1; + } + + flb_input_downstream_set(ctx->downstream, ctx->ins); + flb_net_socket_nonblocking(ctx->downstream->server_fd); + + if (use_collector == FLB_TRUE) { + ret = flb_input_set_collector_socket(ctx->ins, + in_fw_collect, + ctx->downstream->server_fd, + config); + if (ret == -1) { + flb_plg_error(ctx->ins, "could not set server socket collector"); + return -1; + } + + ctx->coll_fd = ret; + } + else { + ctx->event_loop = event_loop; + MK_EVENT_NEW(&ctx->listener_event); + ctx->listener_event.type = FLB_ENGINE_EV_CUSTOM; + ctx->listener_event.data = ctx; + ctx->listener_event.handler = in_fw_worker_listener_event; + + ret = mk_event_add(event_loop, + ctx->downstream->server_fd, + FLB_ENGINE_EV_CUSTOM, + MK_EVENT_READ, + &ctx->listener_event); + if (ret == -1) { + flb_plg_error(ctx->ins, "could not register forward worker listener"); + return -1; + } + + ctx->listener_registered = FLB_TRUE; + } + + return 0; +} + +static int in_fw_worker_init(struct flb_downstream_worker *worker, + void *parent, + void **worker_context) +{ + int ret; + struct flb_in_fw_config *ctx; + struct flb_in_fw_config *parent_ctx; + + parent_ctx = parent; + + ctx = fw_config_init(parent_ctx->ins); + if (ctx == NULL) { + return -1; + } + + *worker_context = ctx; + + ctx->state = FW_INSTANCE_STATE_RUNNING; + ctx->coll_fd = -1; + ctx->ins = parent_ctx->ins; + ctx->workers = parent_ctx->workers; + ctx->worker_id = flb_downstream_worker_id_get(worker); + ctx->use_ingress_queue = FLB_TRUE; + ctx->is_paused = FLB_FALSE; + ctx->net_setup = parent_ctx->ins->net_setup; + ctx->net_setup.share_port = FLB_TRUE; + mk_list_init(&ctx->connections); + mk_list_init(&ctx->users); + + ret = setup_users(ctx, parent_ctx->ins); + if (ret == -1) { + return -1; + } + + ret = in_fw_start_tcp_listener(ctx, + parent_ctx->ins->config, + &ctx->net_setup, + flb_downstream_worker_event_loop_get(worker), + FLB_FALSE); + if (ret == 0) { + flb_downstream_thread_safe(ctx->downstream); + ret = flb_downstream_worker_listener_fd_set( + worker, ctx->downstream->server_fd); + } + + return ret; +} + +static void in_fw_worker_exit(struct flb_downstream_worker *worker, + void *worker_context) +{ + struct flb_in_fw_config *ctx; + + (void) worker; + + ctx = worker_context; + + delete_users(ctx); + fw_conn_del_all(ctx); + fw_config_destroy(ctx); +} + +static void in_fw_worker_maintenance(struct flb_downstream_worker *worker, + void *worker_context) +{ + struct flb_in_fw_config *ctx; + + (void) worker; + + ctx = worker_context; + + if (ctx->downstream != NULL) { + flb_downstream_conn_timeouts_stream(ctx->downstream); + flb_downstream_conn_pending_destroy(ctx->downstream); + } +} + +static int in_fw_workers_start(struct flb_in_fw_config *ctx) +{ + struct flb_downstream_worker_options options; + + memset(&options, 0, sizeof(struct flb_downstream_worker_options)); + options.workers = ctx->workers; + options.parent = ctx; + options.cb_init = in_fw_worker_init; + options.cb_exit = in_fw_worker_exit; + options.cb_maintenance = in_fw_worker_maintenance; + + return flb_downstream_worker_runtime_start(&ctx->runtime, &options); +} + +static void in_fw_workers_stop(struct flb_in_fw_config *ctx) +{ + flb_downstream_worker_runtime_stop(ctx->runtime); + ctx->runtime = NULL; +} + +static void in_fw_worker_pause(struct flb_downstream_worker *worker, + void *worker_context, + void *data) +{ + struct flb_in_fw_config *ctx; + + (void) worker; + (void) data; + + ctx = worker_context; + + if (ctx->downstream != NULL) { + flb_downstream_pause(ctx->downstream); + ctx->is_paused = FLB_TRUE; + ctx->state = FW_INSTANCE_STATE_PAUSED; + fw_conn_del_all(ctx); + } +} + +static void in_fw_worker_resume(struct flb_downstream_worker *worker, + void *worker_context, + void *data) +{ + struct flb_in_fw_config *ctx; + + (void) worker; + (void) data; + + ctx = worker_context; + + if (ctx->downstream != NULL) { + flb_downstream_resume(ctx->downstream); + ctx->is_paused = FLB_FALSE; + ctx->state = FW_INSTANCE_STATE_RUNNING; + } +} + /* Initialize plugin */ static int in_fw_init(struct flb_input_instance *ins, struct flb_config *config, void *data) { - unsigned short int port; int ret; struct flb_in_fw_config *ctx; @@ -300,8 +519,18 @@ static int in_fw_init(struct flb_input_instance *ins, /* Set plugin ingestion to active */ ctx->is_paused = FLB_FALSE; + if (ctx->workers <= 0) { + ctx->workers = 1; + } + /* Unix Socket mode */ if (ctx->unix_path) { + if (ctx->workers > 1) { + flb_plg_error(ctx->ins, "workers is not supported with unix_path"); + fw_config_destroy(ctx); + return -1; + } + #ifndef FLB_HAVE_UNIX_SOCKET flb_plg_error(ctx->ins, "unix address is not supported %s:%s. Aborting", ctx->listen, ctx->tcp_port); @@ -319,38 +548,7 @@ static int in_fw_init(struct flb_input_instance *ins, #endif } else { - port = (unsigned short int) strtoul(ctx->tcp_port, NULL, 10); - - ctx->downstream = flb_downstream_create(FLB_TRANSPORT_TCP, - ctx->ins->flags, - ctx->listen, - port, - ctx->ins->tls, - config, - &ctx->ins->net_setup); - - if (ctx->downstream == NULL) { - flb_plg_error(ctx->ins, - "could not initialize downstream on unix://%s. Aborting", - ctx->listen); - - fw_config_destroy(ctx); - - return -1; - } - - if (ctx->downstream != NULL) { - flb_plg_info(ctx->ins, "listening on %s:%s", - ctx->listen, ctx->tcp_port); - } - else { - flb_plg_error(ctx->ins, "could not bind address %s:%s. Aborting", - ctx->listen, ctx->tcp_port); - - fw_config_destroy(ctx); - - return -1; - } + /* TCP listener startup happens after security.users validation. */ } /* Load users */ @@ -370,24 +568,65 @@ static int in_fw_init(struct flb_input_instance *ins, return -1; } - flb_input_downstream_set(ctx->downstream, ctx->ins); - - flb_net_socket_nonblocking(ctx->downstream->server_fd); - - /* Collect upon data available on the standard input */ - ret = flb_input_set_collector_socket(ins, - in_fw_collect, - ctx->downstream->server_fd, - config); - if (ret == -1) { - flb_plg_error(ctx->ins, "could not set server socket collector"); + ret = pthread_mutex_init(&ctx->conn_mutex, NULL); + if (ret != 0) { + flb_plg_error(ctx->ins, "could not initialize connection mutex"); + delete_users(ctx); fw_config_destroy(ctx); return -1; } + ctx->conn_mutex_initialized = FLB_TRUE; - ctx->coll_fd = ret; + if (ctx->unix_path) { + flb_input_downstream_set(ctx->downstream, ctx->ins); + flb_net_socket_nonblocking(ctx->downstream->server_fd); + + ret = flb_input_set_collector_socket(ins, + in_fw_collect, + ctx->downstream->server_fd, + config); + if (ret == -1) { + flb_plg_error(ctx->ins, "could not set server socket collector"); + delete_users(ctx); + fw_config_destroy(ctx); + return -1; + } + + ctx->coll_fd = ret; + } + else { + if (ctx->workers > 1) { + ret = flb_input_ingress_enable(ins); + if (ret != 0) { + delete_users(ctx); + fw_config_destroy(ctx); + return -1; + } + + ret = in_fw_workers_start(ctx); + if (ret != 0) { + flb_plg_error(ctx->ins, + "could not start forward listener workers on %s:%s. Aborting", + ctx->listen, ctx->tcp_port); + delete_users(ctx); + fw_config_destroy(ctx); + return -1; + } + } + else { + ret = in_fw_start_tcp_listener(ctx, config, &ins->net_setup, NULL, FLB_TRUE); + if (ret != 0) { + delete_users(ctx); + fw_config_destroy(ctx); + return -1; + } + } - pthread_mutex_init(&ctx->conn_mutex, NULL); + flb_plg_info(ctx->ins, + "listening on %s:%s with %i worker%s", + ctx->listen, ctx->tcp_port, ctx->workers, + ctx->workers == 1 ? "" : "s"); + } return 0; } @@ -404,7 +643,17 @@ static void in_fw_pause(void *data, struct flb_config *config) * pause the ingestion. The plugin should close all the connections * and wait for the ingestion to resume. */ - flb_input_collector_pause(ctx->coll_fd, ctx->ins); + if (ctx->runtime != NULL) { + ret = flb_downstream_worker_runtime_foreach(ctx->runtime, + in_fw_worker_pause, + NULL); + if (ret != 0) { + flb_plg_error(ctx->ins, "could not pause all downstream workers"); + } + } + else { + flb_input_collector_pause(ctx->coll_fd, ctx->ins); + } ret = pthread_mutex_lock(&ctx->conn_mutex); if (ret != 0) { @@ -412,11 +661,9 @@ static void in_fw_pause(void *data, struct flb_config *config) return; } - if (ctx->state == FW_INSTANCE_STATE_RUNNING) { - fw_conn_del_all(ctx); - } - + ctx->state = FW_INSTANCE_STATE_PAUSED; ctx->is_paused = FLB_TRUE; + fw_conn_del_all(ctx); ret = pthread_mutex_unlock(&ctx->conn_mutex); if (ret != 0) { flb_plg_error(ctx->ins, "cannot unlock collector mutex"); @@ -444,7 +691,17 @@ static void in_fw_resume(void *data, struct flb_config *config) { struct flb_in_fw_config *ctx = data; if (config->is_running == FLB_TRUE) { - flb_input_collector_resume(ctx->coll_fd, ctx->ins); + if (ctx->runtime != NULL) { + ret = flb_downstream_worker_runtime_foreach(ctx->runtime, + in_fw_worker_resume, + NULL); + if (ret != 0) { + flb_plg_error(ctx->ins, "could not resume all downstream workers"); + } + } + else { + flb_input_collector_resume(ctx->coll_fd, ctx->ins); + } ret = pthread_mutex_lock(&ctx->conn_mutex); if (ret != 0) { @@ -474,6 +731,7 @@ static int in_fw_exit(void *data, struct flb_config *config) } delete_users(ctx); + in_fw_workers_stop(ctx); fw_conn_del_all(ctx); fw_config_destroy(ctx); return 0; @@ -526,6 +784,11 @@ static struct flb_config_map config_map[] = { 0, FLB_TRUE, offsetof(struct flb_in_fw_config, empty_shared_key), "Enable an empty string as the shared key for authentication." }, + { + FLB_CONFIG_MAP_INT, "workers", "1", + 0, FLB_TRUE, offsetof(struct flb_in_fw_config, workers), + "Set the number of Forward listener workers" + }, {0} }; diff --git a/plugins/in_forward/fw.h b/plugins/in_forward/fw.h index 7389b5a4bf3..60d09dd1a33 100644 --- a/plugins/in_forward/fw.h +++ b/plugins/in_forward/fw.h @@ -22,6 +22,9 @@ #include #include +#include +#include +#include #include #include @@ -50,6 +53,8 @@ struct flb_in_fw_user { struct mk_list _head; }; +struct flb_downstream_worker_runtime; + struct flb_in_fw_config { size_t buffer_max_size; /* Max Buffer size */ size_t buffer_chunk_size; /* Chunk allocation size */ @@ -73,6 +78,13 @@ struct flb_in_fw_config { int empty_shared_key; /* use an empty string as shared key */ int coll_fd; + int workers; /* Listener worker count */ + int worker_id; /* Worker id */ + int use_ingress_queue; /* Queue records to main loop */ + int listener_registered; /* Listener event registered */ + struct mk_event listener_event; /* Worker listener event */ + struct mk_event_loop *event_loop; /* Worker event loop */ + struct flb_net_setup net_setup; /* Worker network setup */ struct flb_downstream *downstream; /* Client manager */ struct mk_list connections; /* List of active connections */ struct flb_input_instance *ins; /* Input plugin instace */ @@ -81,11 +93,49 @@ struct flb_in_fw_config { struct flb_log_event_encoder *log_encoder; pthread_mutex_t conn_mutex; + int conn_mutex_initialized; int state; /* Plugin is paused */ int is_paused; + + struct flb_downstream_worker_runtime *runtime; }; +static inline int fw_ingest_logs(struct flb_in_fw_config *ctx, + const char *tag, size_t tag_len, + const void *buf, size_t buf_size) +{ + if (ctx->use_ingress_queue == FLB_TRUE) { + return flb_input_ingress_queue_log(ctx->ins, tag, tag_len, buf, buf_size); + } + + return flb_input_log_append(ctx->ins, tag, tag_len, buf, buf_size); +} + +static inline int fw_ingest_metrics(struct flb_in_fw_config *ctx, + const char *tag, size_t tag_len, + struct cmt *cmt, size_t payload_size) +{ + if (ctx->use_ingress_queue == FLB_TRUE) { + return flb_input_ingress_queue_metrics(ctx->ins, tag, tag_len, + cmt, payload_size); + } + + return flb_input_metrics_append(ctx->ins, tag, tag_len, cmt); +} + +static inline int fw_ingest_traces(struct flb_in_fw_config *ctx, + const char *tag, size_t tag_len, + struct ctrace *ctr, size_t payload_size) +{ + if (ctx->use_ingress_queue == FLB_TRUE) { + return flb_input_ingress_queue_traces(ctx->ins, tag, tag_len, + ctr, payload_size); + } + + return flb_input_trace_append(ctx->ins, tag, tag_len, ctr); +} + #endif diff --git a/plugins/in_forward/fw_config.c b/plugins/in_forward/fw_config.c index 0baff3a47dd..956deac2fe9 100644 --- a/plugins/in_forward/fw_config.c +++ b/plugins/in_forward/fw_config.c @@ -68,6 +68,7 @@ struct flb_in_fw_config *fw_config_init(struct flb_input_instance *i_ins) return NULL; } config->coll_fd = -1; + config->workers = 1; config->log_encoder = flb_log_event_encoder_create(FLB_LOG_EVENT_FORMAT_DEFAULT); @@ -135,6 +136,11 @@ struct flb_in_fw_config *fw_config_init(struct flb_input_instance *i_ins) int fw_config_destroy(struct flb_in_fw_config *config) { + if (config->conn_mutex_initialized == FLB_TRUE) { + pthread_mutex_destroy(&config->conn_mutex); + config->conn_mutex_initialized = FLB_FALSE; + } + if (config->log_encoder != NULL) { flb_log_event_encoder_destroy(config->log_encoder); } @@ -149,6 +155,11 @@ int fw_config_destroy(struct flb_in_fw_config *config) config->coll_fd = -1; } + if (config->listener_registered == FLB_TRUE && config->event_loop != NULL) { + mk_event_del(config->event_loop, &config->listener_event); + config->listener_registered = FLB_FALSE; + } + if (config->downstream != NULL) { flb_downstream_destroy(config->downstream); } diff --git a/plugins/in_forward/fw_conn.c b/plugins/in_forward/fw_conn.c index aca3c80eb7c..fe49ccb1bf7 100644 --- a/plugins/in_forward/fw_conn.c +++ b/plugins/in_forward/fw_conn.c @@ -62,6 +62,10 @@ static int fw_conn_event_internal(struct flb_connection *connection) flb_plg_trace(ctx->ins, "handshake status = %d", conn->handshake_status); ret = fw_prot_secure_forward_handshake(ctx->ins, conn); + if (flb_io_net_is_retry(ret)) { + return 0; + } + if (ret == -1) { flb_plg_trace(ctx->ins, "fd=%i closed connection", event->fd); fw_conn_del(conn); @@ -108,6 +112,10 @@ static int fw_conn_event_internal(struct flb_connection *connection) (void *) &conn->buf[conn->buf_len], available); + if (flb_io_net_is_retry(bytes)) { + return 0; + } + if (bytes > 0) { read_bytes = (size_t) bytes; flb_plg_trace(ctx->ins, "read()=%zd pre_len=%zu now_len=%zu", @@ -225,9 +233,7 @@ struct fw_conn *fw_conn_add(struct flb_connection *connection, struct flb_in_fw_ conn->helo = helo; /* Set data for the event-loop */ - connection->user_data = conn; - connection->event.type = FLB_ENGINE_EV_CUSTOM; - connection->event.handler = fw_conn_event; + connection->user_data = conn; /* Connection info */ conn->ctx = ctx; @@ -251,12 +257,10 @@ struct fw_conn *fw_conn_add(struct flb_connection *connection, struct flb_in_fw_ conn->compression_type = FLB_COMPRESSION_ALGORITHM_NONE; conn->d_ctx = NULL; - /* Register instance into the event loop */ - ret = mk_event_add(flb_engine_evl_get(), - connection->fd, - FLB_ENGINE_EV_CUSTOM, - MK_EVENT_READ, - &connection->event); + /* Run connection callbacks in a downstream-owned coroutine. */ + ret = flb_downstream_conn_event_register(connection, + fw_conn_event, + MK_EVENT_READ); if (ret == -1) { flb_plg_error(ctx->ins, "could not register new connection"); if (conn->helo != NULL) { @@ -331,6 +335,7 @@ static void fw_conn_drop(struct flb_connection *connection) connection->user_data = NULL; if (conn != NULL) { + flb_plg_trace(conn->ctx->ins, "drop connection fd=%i", connection->fd); conn->connection = NULL; fw_conn_release(conn); } @@ -338,21 +343,22 @@ static void fw_conn_drop(struct flb_connection *connection) int fw_conn_del(struct fw_conn *conn) { + int ret; struct flb_connection *connection = conn->connection; /* - * Detach the wrapper from the connection first so the drop - * notification callback becomes a no-op during the release below and - * cannot double-free the wrapper. + * Downstream may need to wake a callback suspended in asynchronous I/O + * before releasing it. Keep the wrapper alive until that callback + * unwinds; the drop notification owns wrapper cleanup in both the + * immediate and deferred paths. */ if (connection != NULL) { - connection->drop_notification_callback = NULL; - connection->user_data = NULL; + ret = flb_downstream_conn_release(connection); + if (ret == FLB_DOWNSTREAM_CONN_DEFERRED) { + return 0; + } - /* The downstream unregisters the file descriptor from the - * event-loop so there's nothing else to be done by the plugin. - */ - flb_downstream_conn_release(connection); + return 0; } fw_conn_release(conn); diff --git a/plugins/in_forward/fw_prot.c b/plugins/in_forward/fw_prot.c index ef8464bad46..a3a6fc961bd 100644 --- a/plugins/in_forward/fw_prot.c +++ b/plugins/in_forward/fw_prot.c @@ -140,38 +140,77 @@ static inline void print_msgpack_error_code(struct flb_input_instance *in, /* Read a secure forward msgpack message for handshake */ static int secure_forward_read(struct flb_input_instance *in, - struct flb_connection *connection, - char *buf, size_t size, size_t *out_len) + struct fw_conn *conn, size_t size, + size_t *out_len) { int ret; size_t off; size_t avail; - size_t buf_off = 0; + size_t read_limit; + size_t new_size; + char *tmp; msgpack_unpacked result; + struct flb_in_fw_config *ctx; + + ctx = conn->ctx; + read_limit = size; + if (ctx->buffer_max_size < read_limit) { + read_limit = ctx->buffer_max_size; + } msgpack_unpacked_init(&result); while (1) { - avail = size - buf_off; - if (avail < 1) { + if (conn->buf_len >= read_limit) { goto error; } + if (conn->buf_len >= conn->buf_size) { + if (ctx->buffer_chunk_size > read_limit - conn->buf_size) { + new_size = read_limit; + } + else { + new_size = conn->buf_size + ctx->buffer_chunk_size; + } + + tmp = flb_realloc(conn->buf, new_size); + if (tmp == NULL) { + flb_errno(); + goto error; + } + + conn->buf = tmp; + conn->buf_size = new_size; + } + + avail = read_limit - conn->buf_len; + if (conn->buf_size - conn->buf_len < avail) { + avail = conn->buf_size - conn->buf_len; + } + /* Read the message */ - ret = flb_io_net_read(connection, buf + buf_off, size - buf_off); + ret = flb_io_net_read(conn->connection, + conn->buf + conn->buf_len, avail); + if (flb_io_net_is_retry(ret)) { + msgpack_unpacked_destroy(&result); + return ret; + } + if (ret <= 0) { flb_plg_debug(in, "read %d byte(s)", ret); goto error; } - buf_off += ret; + conn->buf_len += ret; /* Validate */ off = 0; - ret = msgpack_unpack_next(&result, buf, buf_off, &off); + ret = msgpack_unpack_next(&result, conn->buf, conn->buf_len, &off); switch (ret) { case MSGPACK_UNPACK_SUCCESS: msgpack_unpacked_destroy(&result); - *out_len = buf_off; + *out_len = conn->buf_len; return 0; + case MSGPACK_UNPACK_CONTINUE: + break; default: print_msgpack_error_code(in, ret, "handshake"); goto error; @@ -179,6 +218,7 @@ static int secure_forward_read(struct flb_input_instance *in, } error: + conn->buf_len = 0; msgpack_unpacked_destroy(&result); return -1; } @@ -506,7 +546,6 @@ static int check_ping(struct flb_input_instance *ins, flb_sds_t *out_shared_key_salt) { int ret; - char buf[1024]; size_t out_len; size_t off; msgpack_unpacked result; @@ -531,7 +570,12 @@ static int check_ping(struct flb_input_instance *ins, } /* Wait for client PING */ - ret = secure_forward_read(ins, conn->connection, buf, sizeof(buf) - 1, &out_len); + ret = secure_forward_read(ins, conn, 1023, &out_len); + if (flb_io_net_is_retry(ret)) { + flb_free(serverside); + return ret; + } + if (ret == -1) { flb_free(serverside); flb_plg_error(ins, "handshake error expecting PING"); @@ -541,7 +585,8 @@ static int check_ping(struct flb_input_instance *ins, /* Unpack message and validate */ off = 0; msgpack_unpacked_init(&result); - ret = msgpack_unpack_next(&result, buf, out_len, &off); + ret = msgpack_unpack_next(&result, conn->buf, out_len, &off); + conn->buf_len = 0; if (ret != MSGPACK_UNPACK_SUCCESS) { flb_free(serverside); print_msgpack_error_code(ins, ret, "PING"); @@ -948,13 +993,10 @@ static int get_options_chunk(msgpack_object *arr, int expected, int *idx) return 0; } -static int fw_process_forward_mode_entry( - struct fw_conn *conn, - const char *tag, int tag_len, - msgpack_object *entry, - int chunk_id) +static int fw_encode_forward_mode_entry(struct fw_conn *conn, + msgpack_object *entry) { - int result; + int result; struct flb_log_event event; result = flb_event_decoder_decode_object(conn->ctx->log_decoder, @@ -985,21 +1027,149 @@ static int fw_process_forward_mode_entry( result = flb_log_event_encoder_commit_record(conn->ctx->log_encoder); } - if (result == FLB_EVENT_ENCODER_SUCCESS) { - flb_input_log_append(conn->ctx->ins, tag, tag_len, - conn->ctx->log_encoder->output_buffer, - conn->ctx->log_encoder->output_length); + if (result != FLB_EVENT_ENCODER_SUCCESS) { + flb_plg_warn(conn->ctx->ins, "event decoder or encoder failure: %d", result); + + return -1; } - flb_log_event_encoder_reset(conn->ctx->log_encoder); + return 0; +} - if (result != FLB_EVENT_ENCODER_SUCCESS) { - flb_plg_warn(conn->ctx->ins, "Event decoder failure : %d", result); +static int fw_ingest_forward_mode_chunk(struct fw_conn *conn, + const char *tag, size_t tag_len, + const void *buffer, size_t length) +{ + int result; + struct flb_input_instance *ins; + struct flb_in_fw_config *ctx; + ctx = conn->ctx; + ins = ctx->ins; + + do { + result = fw_ingest_logs(ctx, tag, tag_len, buffer, length); + } while (result == FLB_INPUT_INGRESS_BUSY && + ins->ingress_queue_enabled == FLB_TRUE && + ins->config->is_ingestion_active == FLB_TRUE && + ctx->is_paused == FLB_FALSE && + (ins->ingress_queue_byte_limit == 0 || + length <= ins->ingress_queue_byte_limit)); + + return result; +} + +static int fw_ingest_forward_mode_batch(struct fw_conn *conn, + const char *tag, size_t tag_len, + const void *buffer, size_t length) +{ + int decode_result; + int ingest_result; + size_t batch_length; + size_t byte_limit; + const char *batch; + struct flb_log_event event; + struct flb_log_event_decoder decoder; + + byte_limit = conn->ctx->ins->ingress_queue_byte_limit; + + if (byte_limit == 0 || length <= byte_limit) { + return fw_ingest_forward_mode_chunk(conn, tag, tag_len, buffer, length); + } + + decode_result = flb_log_event_decoder_init(&decoder, (char *) buffer, length); + if (decode_result != FLB_EVENT_DECODER_SUCCESS) { return -1; } - return 0; + batch = NULL; + batch_length = 0; + ingest_result = 0; + + while ((decode_result = flb_log_event_decoder_next(&decoder, &event)) == + FLB_EVENT_DECODER_SUCCESS) { + if (batch_length > 0 && + (batch_length > byte_limit || + decoder.record_length > byte_limit - batch_length)) { + ingest_result = fw_ingest_forward_mode_chunk( + conn, tag, tag_len, batch, batch_length); + if (ingest_result != 0) { + break; + } + + batch_length = 0; + } + + if (batch_length == 0) { + batch = decoder.record_base; + } + + batch_length += decoder.record_length; + } + + if (ingest_result == 0) { + decode_result = flb_log_event_decoder_get_last_result(&decoder); + + if (decode_result != FLB_EVENT_DECODER_SUCCESS) { + ingest_result = -1; + } + else if (batch_length > 0) { + ingest_result = fw_ingest_forward_mode_chunk( + conn, tag, tag_len, batch, batch_length); + } + } + + flb_log_event_decoder_destroy(&decoder); + + return ingest_result; +} + +static int fw_process_forward_mode(struct fw_conn *conn, + const char *tag, size_t tag_len, + msgpack_object *entries) +{ + int encode_result; + int ingest_result; + size_t index; + struct flb_log_event_encoder *encoder; + + encoder = conn->ctx->log_encoder; + flb_log_event_encoder_reset(encoder); + + encode_result = 0; + + for (index = 0; + index < entries->via.array.size && encode_result == 0; + index++) { + encode_result = fw_encode_forward_mode_entry( + conn, + &entries->via.array.ptr[index]); + } + + ingest_result = 0; + + /* + * Preserve the existing partial-frame behavior: entries encoded before a + * malformed entry are ingested before the connection is rejected. + */ + if (encoder->output_length > 0) { + ingest_result = fw_ingest_forward_mode_batch( + conn, + tag, + tag_len, + encoder->output_buffer, + encoder->output_length); + } + + flb_log_event_encoder_reset(encoder); + + if (ingest_result != 0) { + flb_plg_warn(conn->ctx->ins, + "could not ingest Forward mode batch: %d", ingest_result); + return -1; + } + + return encode_result; } static int fw_process_message_mode_entry( @@ -1057,18 +1227,23 @@ static int fw_process_message_mode_entry( } if (result == FLB_EVENT_ENCODER_SUCCESS) { - flb_input_log_append(in, tag, tag_len, - conn->ctx->log_encoder->output_buffer, - conn->ctx->log_encoder->output_length); + result = fw_ingest_logs(conn->ctx, tag, tag_len, + conn->ctx->log_encoder->output_buffer, + conn->ctx->log_encoder->output_length); } flb_log_event_encoder_reset(conn->ctx->log_encoder); - if (chunk_id != -1) { + if (result == FLB_EVENT_ENCODER_SUCCESS && chunk_id != -1) { chunk = options.via.map.ptr[chunk_id].val; send_ack(in, conn, chunk); } + if (result != FLB_EVENT_ENCODER_SUCCESS) { + flb_plg_warn(conn->ctx->ins, "could not ingest Forward message: %d", result); + return -1; + } + return 0; } @@ -1128,9 +1303,9 @@ static int append_log(struct flb_input_instance *ins, struct fw_conn *conn, struct ctrace *ctr; if (event_type == FLB_EVENT_TYPE_LOGS) { - ret = flb_input_log_append(conn->in, - out_tag, flb_sds_len(out_tag), - data, len); + ret = fw_ingest_logs(conn->ctx, + out_tag, flb_sds_len(out_tag), + data, len); if (ret != 0) { flb_plg_error(ins, "could not append logs. ret=%d", ret); return -1; @@ -1145,15 +1320,20 @@ static int append_log(struct flb_input_instance *ins, struct fw_conn *conn, return -1; } - ret = flb_input_metrics_append(conn->in, - out_tag, flb_sds_len(out_tag), - cmt); + ret = fw_ingest_metrics(conn->ctx, + out_tag, flb_sds_len(out_tag), + cmt, len); if (ret != 0) { flb_plg_error(ins, "could not append metrics. ret=%d", ret); - cmt_decode_msgpack_destroy(cmt); + if (conn->ctx->use_ingress_queue == FLB_FALSE) { + cmt_decode_msgpack_destroy(cmt); + } return -1; } - cmt_decode_msgpack_destroy(cmt); + + if (conn->ctx->use_ingress_queue == FLB_FALSE) { + cmt_decode_msgpack_destroy(cmt); + } } else if (event_type == FLB_EVENT_TYPE_TRACES) { off = 0; @@ -1163,12 +1343,14 @@ static int append_log(struct flb_input_instance *ins, struct fw_conn *conn, return -1; } - ret = flb_input_trace_append(ins, - out_tag, flb_sds_len(out_tag), - ctr); + ret = fw_ingest_traces(conn->ctx, + out_tag, flb_sds_len(out_tag), + ctr, len); if (ret != 0) { flb_plg_error(ins, "could not append traces. ret=%d", ret); - ctr_decode_msgpack_destroy(ctr); + if (conn->ctx->use_ingress_queue == FLB_FALSE) { + ctr_decode_msgpack_destroy(ctr); + } return -1; } /* Note: flb_input_trace_append takes ownership of ctr and destroys it on success */ @@ -1206,6 +1388,11 @@ int fw_prot_secure_forward_handshake(struct flb_input_instance *ins, reason = flb_sds_create_size(32); flb_plg_debug(ins, "protocol: checking PING"); ping_ret = check_ping(ins, conn, &shared_key_salt); + if (flb_io_net_is_retry(ping_ret)) { + flb_sds_destroy(reason); + return ping_ret; + } + if (ping_ret == -1) { flb_plg_error(ins, "handshake error checking PING"); @@ -1270,7 +1457,6 @@ int fw_prot_process(struct flb_input_instance *ins, struct fw_conn *conn) int contain_options = FLB_FALSE; int chunk_id = -1; int metadata_id = -1; - size_t index = 0; const char *stag; flb_sds_t out_tag = NULL; size_t bytes; @@ -1466,21 +1652,14 @@ int fw_prot_process(struct flb_input_instance *ins, struct fw_conn *conn) return -1; } - /* Process array */ - ret = 0; - - for(index = 0 ; - index < entry.via.array.size && - ret == 0 ; - index++) { - ret = fw_process_forward_mode_entry( - conn, - out_tag, flb_sds_len(out_tag), - &entry.via.array.ptr[index], - chunk_id); - } + /* Encode and ingest the complete Forward mode frame. */ + ret = fw_process_forward_mode( + conn, + out_tag, + flb_sds_len(out_tag), + &entry); - if (chunk_id != -1) { + if (ret == 0 && chunk_id != -1) { msgpack_object options; msgpack_object chunk; @@ -1489,6 +1668,10 @@ int fw_prot_process(struct flb_input_instance *ins, struct fw_conn *conn) send_ack(conn->in, conn, chunk); } + + if (ret != 0) { + goto cleanup_msgpack; + } } else if (entry.type == MSGPACK_OBJECT_POSITIVE_INTEGER || entry.type == MSGPACK_OBJECT_EXT) { @@ -1534,11 +1717,14 @@ int fw_prot_process(struct flb_input_instance *ins, struct fw_conn *conn) } /* Process map */ - fw_process_message_mode_entry( + ret = fw_process_message_mode_entry( conn->in, conn, out_tag, flb_sds_len(out_tag), &root, &entry, &map, chunk_id, metadata_id); + if (ret != 0) { + goto cleanup_msgpack; + } } else if (entry.type == MSGPACK_OBJECT_STR || entry.type == MSGPACK_OBJECT_BIN) { @@ -1756,7 +1942,7 @@ int fw_prot_process(struct flb_input_instance *ins, struct fw_conn *conn) goto cleanup_msgpack; } - ret = msgpack_unpacker_next(unp, &result); + ret = msgpack_unpacker_next_with_size(unp, &result, &bytes); } } diff --git a/plugins/in_gpu_metrics/CMakeLists.txt b/plugins/in_gpu_metrics/CMakeLists.txt index 73f460d38b0..5174d6ce68f 100644 --- a/plugins/in_gpu_metrics/CMakeLists.txt +++ b/plugins/in_gpu_metrics/CMakeLists.txt @@ -1,6 +1,7 @@ set(src gpu_metrics.c amd_gpu.c + nvml_gpu.c gpu_common.c ) diff --git a/plugins/in_gpu_metrics/amd_gpu.c b/plugins/in_gpu_metrics/amd_gpu.c index f02216f2964..91f17b2cb8f 100644 --- a/plugins/in_gpu_metrics/amd_gpu.c +++ b/plugins/in_gpu_metrics/amd_gpu.c @@ -47,61 +47,6 @@ static flb_sds_t build_path(int card_id, const char *file) return path; } -static int match_card_pattern(const char *pattern, int card_id) -{ - char *dup; - char *token; - char *saveptr; - int start; - int end; - - if (!pattern || pattern[0] == '\0' || strcmp(pattern, "*") == 0) { - return FLB_TRUE; - } - - dup = flb_strdup(pattern); - if (!dup) { - return FLB_FALSE; - } - - token = strtok_r(dup, ",", &saveptr); - while (token) { - if (sscanf(token, "%d-%d", &start, &end) == 2) { - if (card_id >= start && card_id <= end) { - flb_free(dup); - return FLB_TRUE; - } - } - else { - if (card_id == atoi(token)) { - flb_free(dup); - return FLB_TRUE; - } - } - token = strtok_r(NULL, ",", &saveptr); - } - flb_free(dup); - return FLB_FALSE; -} - -static int should_include_card(struct in_gpu_metrics *ctx, int card_id) -{ - flb_plg_info(ctx->ins, "should_include_card: card%d, exclude='%s', include='%s'", - card_id, ctx->cards_exclude ? ctx->cards_exclude : "NULL", - ctx->cards_include ? ctx->cards_include : "NULL"); - - if (ctx->cards_exclude && ctx->cards_exclude[0] != '\0' && match_card_pattern(ctx->cards_exclude, card_id)) { - flb_plg_info(ctx->ins, "Card%d excluded by exclude pattern", card_id); - return FLB_FALSE; - } - if (ctx->cards_include && ctx->cards_include[0] != '\0' && !match_card_pattern(ctx->cards_include, card_id)) { - flb_plg_info(ctx->ins, "Card%d excluded by include pattern", card_id); - return FLB_FALSE; - } - flb_plg_info(ctx->ins, "Card%d should be included", card_id); - return FLB_TRUE; -} - static void free_cards(struct in_gpu_metrics *ctx) { struct cfl_list *tmp; @@ -260,7 +205,7 @@ int amd_gpu_detect_cards(struct in_gpu_metrics *ctx) } flb_plg_info(ctx->ins, "Checking if card%d should be included", id); - if (!should_include_card(ctx, id)) { + if (!gpu_should_include_card(ctx, id)) { flb_plg_info(ctx->ins, "Card%d excluded by filter", id); continue; } @@ -275,7 +220,12 @@ int amd_gpu_detect_cards(struct in_gpu_metrics *ctx) return -1; } card->id = id; + card->backend_type = GPU_BACKEND_AMD; card->hwmon_path = NULL; + card->uuid = NULL; + card->parent_uuid = NULL; + card->gpu_instance_id = -1; + card->compute_instance_id = -1; card->hwmon_path = find_hwmon_path(id); if (!card->hwmon_path) { flb_plg_debug(ctx->ins, "no hwmon path for card%d", id); @@ -557,4 +507,3 @@ int amd_gpu_collect_metrics(struct in_gpu_metrics *ctx, struct gpu_card *card) flb_sds_destroy(card_id); return 0; } - diff --git a/plugins/in_gpu_metrics/gpu_common.c b/plugins/in_gpu_metrics/gpu_common.c index fea0bdcbb64..7f3718463b3 100644 --- a/plugins/in_gpu_metrics/gpu_common.c +++ b/plugins/in_gpu_metrics/gpu_common.c @@ -21,8 +21,14 @@ #include #include #include +#include +#include + +#include +#include #include "gpu_common.h" +#include "gpu_metrics.h" int gpu_read_uint64(const char *path, uint64_t *value) { @@ -71,3 +77,67 @@ int gpu_read_line(const char *path, char *buf, size_t size) fclose(fp); return 0; } + +static int match_card_pattern(const char *pattern, int card_id) +{ + char *dup; + char *token; + char *saveptr; + int start; + int end; + + if (!pattern || pattern[0] == '\0' || strcmp(pattern, "*") == 0) { + return FLB_TRUE; + } + + dup = flb_strdup(pattern); + if (!dup) { + return FLB_FALSE; + } + + token = strtok_r(dup, ",", &saveptr); + while (token) { + if (sscanf(token, "%d-%d", &start, &end) == 2) { + if (card_id >= start && card_id <= end) { + flb_free(dup); + return FLB_TRUE; + } + } + else { + char *endptr; + long parsed_id; + + errno = 0; + parsed_id = strtol(token, &endptr, 10); + if (errno == 0 && + endptr != token && + *endptr == '\0' && + parsed_id >= INT_MIN && + parsed_id <= INT_MAX && + card_id == (int) parsed_id) { + flb_free(dup); + return FLB_TRUE; + } + } + token = strtok_r(NULL, ",", &saveptr); + } + flb_free(dup); + return FLB_FALSE; +} + +int gpu_should_include_card(struct in_gpu_metrics *ctx, int card_id) +{ + if (ctx->cards_exclude && ctx->cards_exclude[0] != '\0' && + match_card_pattern(ctx->cards_exclude, card_id)) { + flb_plg_info(ctx->ins, "card%d excluded by exclude pattern", card_id); + return FLB_FALSE; + } + + if (ctx->cards_include && ctx->cards_include[0] != '\0' && + !match_card_pattern(ctx->cards_include, card_id)) { + flb_plg_info(ctx->ins, "card%d excluded by include pattern", card_id); + return FLB_FALSE; + } + + return FLB_TRUE; +} diff --git a/plugins/in_gpu_metrics/gpu_common.h b/plugins/in_gpu_metrics/gpu_common.h index 07cc40e9695..83aa78117ed 100644 --- a/plugins/in_gpu_metrics/gpu_common.h +++ b/plugins/in_gpu_metrics/gpu_common.h @@ -23,8 +23,11 @@ #include #include +struct in_gpu_metrics; + int gpu_read_uint64(const char *path, uint64_t *value); int gpu_read_double(const char *path, double scale, double *value); int gpu_read_line(const char *path, char *buf, size_t size); +int gpu_should_include_card(struct in_gpu_metrics *ctx, int card_id); #endif diff --git a/plugins/in_gpu_metrics/gpu_metrics.c b/plugins/in_gpu_metrics/gpu_metrics.c index a0ac867ffae..4d107b326c6 100644 --- a/plugins/in_gpu_metrics/gpu_metrics.c +++ b/plugins/in_gpu_metrics/gpu_metrics.c @@ -31,6 +31,7 @@ #include "gpu_metrics.h" #include "amd_gpu.h" +#include "nvml_gpu.h" static int in_gpu_collect(struct flb_input_instance *ins, struct flb_config *config, void *in_context) @@ -41,7 +42,12 @@ static int in_gpu_collect(struct flb_input_instance *ins, cfl_list_foreach(head, &ctx->cards) { card = cfl_list_entry(head, struct gpu_card, _head); - amd_gpu_collect_metrics(ctx, card); + if (card->backend_type == GPU_BACKEND_AMD) { + amd_gpu_collect_metrics(ctx, card); + } + else if (card->backend_type == GPU_BACKEND_NVML) { + nvml_gpu_collect_metrics(ctx, card); + } } flb_input_metrics_append(ctx->ins, NULL, 0, ctx->cmt); @@ -61,6 +67,8 @@ static int in_gpu_init(struct flb_input_instance *ins, } ctx->ins = ins; ctx->cards_detected = 0; + ctx->nvml_initialized = FLB_FALSE; + ctx->nvml_lib_handle = NULL; cfl_list_init(&ctx->cards); ret = flb_input_config_map_set(ins, (void *) ctx); @@ -116,7 +124,21 @@ static int in_gpu_init(struct flb_input_instance *ins, "GPU fan PWM percentage", 2, (char *[]) {"card", "vendor"}); + ctx->g_process_memory = cmt_gauge_create(ctx->cmt, "gpu", "", "process_memory_used_bytes", + "Per-process GPU memory in bytes", 3, + (char *[]) {"card", "vendor", "pid"}); + + ctx->g_mig_info = cmt_gauge_create(ctx->cmt, "gpu", "", "mig_device_info", + "MIG device metadata (always 1)", 5, + (char *[]) {"card", "vendor", "parent_uuid", + "gpu_instance_id", "compute_instance_id"}); + amd_gpu_detect_cards(ctx); + if (nvml_gpu_initialize(ctx) == 0) { + if (nvml_gpu_detect_cards(ctx) != 0) { + flb_plg_debug(ctx->ins, "NVML card detection encountered errors"); + } + } flb_input_set_context(ins, ctx); ret = flb_input_set_collector_time(ins, in_gpu_collect, @@ -154,6 +176,13 @@ static int in_gpu_exit(void *data, struct flb_config *config) if (card->hwmon_path) { flb_sds_destroy(card->hwmon_path); } + if (card->uuid) { + flb_sds_destroy(card->uuid); + } + if (card->parent_uuid) { + flb_sds_destroy(card->parent_uuid); + } + cfl_list_del(&card->_head); flb_free(card); } @@ -162,6 +191,7 @@ static int in_gpu_exit(void *data, struct flb_config *config) cmt_destroy(ctx->cmt); } + nvml_gpu_shutdown(ctx); flb_free(ctx); return 0; } @@ -189,6 +219,11 @@ static struct flb_config_map config_map[] = { 0, FLB_TRUE, offsetof(struct in_gpu_metrics, enable_temperature), "Enable collection of GPU temperature metrics (gpu_temperature_celsius)." }, + { + FLB_CONFIG_MAP_BOOL, "enable_nvml", "true", + 0, FLB_TRUE, offsetof(struct in_gpu_metrics, enable_nvml), + "Enable NVIDIA NVML collection when libnvidia-ml is available." + }, { FLB_CONFIG_MAP_STR, "path_sysfs", "/sys", 0, FLB_TRUE, offsetof(struct in_gpu_metrics, path_sysfs), diff --git a/plugins/in_gpu_metrics/gpu_metrics.h b/plugins/in_gpu_metrics/gpu_metrics.h index 270adeb3b8b..1df86a6abe9 100644 --- a/plugins/in_gpu_metrics/gpu_metrics.h +++ b/plugins/in_gpu_metrics/gpu_metrics.h @@ -25,7 +25,12 @@ struct gpu_card { int id; + int backend_type; flb_sds_t hwmon_path; + flb_sds_t uuid; + flb_sds_t parent_uuid; + int gpu_instance_id; + int compute_instance_id; struct cfl_list _head; }; @@ -36,8 +41,10 @@ struct in_gpu_metrics { int scrape_interval; int enable_power; int enable_temperature; + int enable_nvml; int coll_fd; int cards_detected; + int nvml_initialized; struct cfl_list cards; @@ -50,10 +57,16 @@ struct in_gpu_metrics { struct cmt_gauge *g_temp; struct cmt_gauge *g_fan_speed; struct cmt_gauge *g_fan_pwm; + struct cmt_gauge *g_process_memory; + struct cmt_gauge *g_mig_info; /* plugin instance */ struct flb_input_instance *ins; + void *nvml_lib_handle; }; +#define GPU_BACKEND_AMD 1 +#define GPU_BACKEND_NVML 2 + #endif diff --git a/plugins/in_gpu_metrics/nvml_gpu.c b/plugins/in_gpu_metrics/nvml_gpu.c new file mode 100644 index 00000000000..7507b0445dc --- /dev/null +++ b/plugins/in_gpu_metrics/nvml_gpu.c @@ -0,0 +1,805 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2026 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "nvml_gpu.h" + +typedef int nvmlReturn_t; +typedef void *nvmlDevice_t; + +struct nvmlMemory_t { + uint64_t total; + uint64_t free; + uint64_t used; +}; + +struct nvmlUtilization_t { + unsigned int gpu; + unsigned int memory; +}; + +struct nvmlProcessInfo_v1_t { + unsigned int pid; + uint64_t usedGpuMemory; +}; + +struct nvmlProcessInfo_v2_t { + unsigned int pid; + uint64_t usedGpuMemory; + unsigned int gpuInstanceId; + unsigned int computeInstanceId; +}; + +struct nvmlProcessInfo_v3_t { + unsigned int pid; + uint64_t usedGpuMemory; + unsigned int gpuInstanceId; + unsigned int computeInstanceId; + uint64_t usedGpuCcProtectedMemory; +}; + +typedef nvmlReturn_t (*nvmlInit_v2_t)(void); +typedef nvmlReturn_t (*nvmlShutdown_t)(void); +typedef nvmlReturn_t (*nvmlDeviceGetCount_v2_t)(unsigned int *device_count); +typedef nvmlReturn_t (*nvmlDeviceGetHandleByIndex_v2_t)(unsigned int index, + nvmlDevice_t *device); +typedef nvmlReturn_t (*nvmlDeviceGetHandleByUUID_t)(const char *uuid, + nvmlDevice_t *device); +typedef nvmlReturn_t (*nvmlDeviceGetMemoryInfo_t)(nvmlDevice_t device, + struct nvmlMemory_t *memory); +typedef nvmlReturn_t (*nvmlDeviceGetUtilizationRates_t)(nvmlDevice_t device, + struct nvmlUtilization_t *util); +typedef nvmlReturn_t (*nvmlDeviceGetTemperature_t)(nvmlDevice_t device, + unsigned int sensor_type, + unsigned int *temp); +typedef nvmlReturn_t (*nvmlDeviceGetPowerUsage_t)(nvmlDevice_t device, + unsigned int *power); +typedef nvmlReturn_t (*nvmlDeviceGetFanSpeed_t)(nvmlDevice_t device, + unsigned int *speed); +typedef nvmlReturn_t (*nvmlDeviceGetClockInfo_t)(nvmlDevice_t device, + unsigned int clock_type, + unsigned int *clock); +typedef nvmlReturn_t (*nvmlDeviceGetUUID_t)(nvmlDevice_t device, + char *uuid, + unsigned int length); +typedef nvmlReturn_t (*nvmlDeviceGetMigMode_t)(nvmlDevice_t device, + unsigned int *current_mode, + unsigned int *pending_mode); +typedef nvmlReturn_t (*nvmlDeviceGetMaxMigDeviceCount_t)(nvmlDevice_t device, + unsigned int *count); +typedef nvmlReturn_t (*nvmlDeviceGetMigDeviceHandleByIndex_t)(nvmlDevice_t device, + unsigned int index, + nvmlDevice_t *mig_device); +typedef nvmlReturn_t (*nvmlDeviceGetGpuInstanceId_t)(nvmlDevice_t device, + unsigned int *id); +typedef nvmlReturn_t (*nvmlDeviceGetComputeInstanceId_t)(nvmlDevice_t device, + unsigned int *id); +typedef nvmlReturn_t (*nvmlDeviceGetDeviceHandleFromMigDeviceHandle_t)(nvmlDevice_t device, + nvmlDevice_t *parent); +typedef nvmlReturn_t (*nvmlDeviceGetComputeRunningProcesses_t)(nvmlDevice_t device, + unsigned int *info_count, + struct nvmlProcessInfo_v1_t *infos); +typedef nvmlReturn_t (*nvmlDeviceGetGraphicsRunningProcesses_t)(nvmlDevice_t device, + unsigned int *info_count, + struct nvmlProcessInfo_v1_t *infos); +typedef nvmlReturn_t (*nvmlDeviceGetComputeRunningProcesses_v2_t)(nvmlDevice_t device, + unsigned int *info_count, + struct nvmlProcessInfo_v2_t *infos); +typedef nvmlReturn_t (*nvmlDeviceGetGraphicsRunningProcesses_v2_t)(nvmlDevice_t device, + unsigned int *info_count, + struct nvmlProcessInfo_v2_t *infos); +typedef nvmlReturn_t (*nvmlDeviceGetComputeRunningProcesses_v3_t)(nvmlDevice_t device, + unsigned int *info_count, + struct nvmlProcessInfo_v3_t *infos); +typedef nvmlReturn_t (*nvmlDeviceGetGraphicsRunningProcesses_v3_t)(nvmlDevice_t device, + unsigned int *info_count, + struct nvmlProcessInfo_v3_t *infos); +typedef const char *(*nvmlErrorString_t)(nvmlReturn_t result); + +#define NVML_SUCCESS 0 +#define NVML_TEMPERATURE_GPU 0 +#define NVML_CLOCK_GRAPHICS 0 +#define NVML_CLOCK_SM 1 +#define NVML_CLOCK_MEM 2 +#define NVML_FEATURE_ENABLED 1 +#define NVML_UUID_BUFFER_SIZE 96 +#define NVML_MAX_PROCESS_SAMPLES 128 +#define NVML_MAX_VALID_PID 4194304 + +static nvmlInit_v2_t f_nvml_init_v2; +static nvmlShutdown_t f_nvml_shutdown; +static nvmlDeviceGetCount_v2_t f_nvml_device_get_count_v2; +static nvmlDeviceGetHandleByIndex_v2_t f_nvml_device_get_handle_by_index_v2; +static nvmlDeviceGetHandleByUUID_t f_nvml_device_get_handle_by_uuid; +static nvmlDeviceGetMemoryInfo_t f_nvml_device_get_memory_info; +static nvmlDeviceGetUtilizationRates_t f_nvml_device_get_utilization_rates; +static nvmlDeviceGetTemperature_t f_nvml_device_get_temperature; +static nvmlDeviceGetPowerUsage_t f_nvml_device_get_power_usage; +static nvmlDeviceGetFanSpeed_t f_nvml_device_get_fan_speed; +static nvmlDeviceGetClockInfo_t f_nvml_device_get_clock_info; +static nvmlDeviceGetUUID_t f_nvml_device_get_uuid; +static nvmlDeviceGetMigMode_t f_nvml_device_get_mig_mode; +static nvmlDeviceGetMaxMigDeviceCount_t f_nvml_device_get_max_mig_device_count; +static nvmlDeviceGetMigDeviceHandleByIndex_t f_nvml_device_get_mig_device_handle_by_index; +static nvmlDeviceGetGpuInstanceId_t f_nvml_device_get_gpu_instance_id; +static nvmlDeviceGetComputeInstanceId_t f_nvml_device_get_compute_instance_id; +static nvmlDeviceGetDeviceHandleFromMigDeviceHandle_t f_nvml_device_get_parent_from_mig; +static nvmlDeviceGetComputeRunningProcesses_t f_nvml_device_get_compute_running_processes; +static nvmlDeviceGetGraphicsRunningProcesses_t f_nvml_device_get_graphics_running_processes; +static nvmlDeviceGetComputeRunningProcesses_v2_t f_nvml_device_get_compute_running_processes_v2; +static nvmlDeviceGetGraphicsRunningProcesses_v2_t f_nvml_device_get_graphics_running_processes_v2; +static nvmlDeviceGetComputeRunningProcesses_v3_t f_nvml_device_get_compute_running_processes_v3; +static nvmlDeviceGetGraphicsRunningProcesses_v3_t f_nvml_device_get_graphics_running_processes_v3; +static nvmlErrorString_t f_nvml_error_string; + +static void nvml_reset_api_symbols() +{ + f_nvml_init_v2 = NULL; + f_nvml_shutdown = NULL; + f_nvml_device_get_count_v2 = NULL; + f_nvml_device_get_handle_by_index_v2 = NULL; + f_nvml_device_get_handle_by_uuid = NULL; + f_nvml_device_get_memory_info = NULL; + f_nvml_device_get_utilization_rates = NULL; + f_nvml_device_get_temperature = NULL; + f_nvml_device_get_power_usage = NULL; + f_nvml_device_get_fan_speed = NULL; + f_nvml_device_get_clock_info = NULL; + f_nvml_device_get_uuid = NULL; + f_nvml_device_get_mig_mode = NULL; + f_nvml_device_get_max_mig_device_count = NULL; + f_nvml_device_get_mig_device_handle_by_index = NULL; + f_nvml_device_get_gpu_instance_id = NULL; + f_nvml_device_get_compute_instance_id = NULL; + f_nvml_device_get_parent_from_mig = NULL; + f_nvml_device_get_compute_running_processes = NULL; + f_nvml_device_get_graphics_running_processes = NULL; + f_nvml_device_get_compute_running_processes_v2 = NULL; + f_nvml_device_get_graphics_running_processes_v2 = NULL; + f_nvml_device_get_compute_running_processes_v3 = NULL; + f_nvml_device_get_graphics_running_processes_v3 = NULL; + f_nvml_error_string = NULL; +} + +static const char *nvml_result_to_string(nvmlReturn_t result) +{ + if (f_nvml_error_string != NULL) { + return f_nvml_error_string(result); + } + + return "unknown"; +} + +static int load_nvml_symbol(struct in_gpu_metrics *ctx, const char *name, void **target) +{ + *target = dlsym(ctx->nvml_lib_handle, name); + if (*target == NULL) { + flb_plg_warn(ctx->ins, "NVML symbol '%s' is missing", name); + return -1; + } + + return 0; +} + +static void load_optional_nvml_symbol(struct in_gpu_metrics *ctx, const char *name, void **target) +{ + *target = dlsym(ctx->nvml_lib_handle, name); + if (*target == NULL) { + flb_plg_debug(ctx->ins, "optional NVML symbol '%s' is unavailable", name); + } +} + +static int nvml_read_device_uuid(nvmlDevice_t device, char *buf, size_t size) +{ + nvmlReturn_t result; + + if (f_nvml_device_get_uuid == NULL) { + return -1; + } + + result = f_nvml_device_get_uuid(device, buf, (unsigned int) size); + if (result != NVML_SUCCESS) { + return -1; + } + + return 0; +} + +static int nvml_register_card(struct in_gpu_metrics *ctx, + int card_id, + int gpu_instance_id, + int compute_instance_id, + const char *uuid, + const char *parent_uuid) +{ + struct gpu_card *card; + + card = flb_calloc(1, sizeof(struct gpu_card)); + if (card == NULL) { + flb_errno(); + return -1; + } + + card->id = card_id; + card->backend_type = GPU_BACKEND_NVML; + card->gpu_instance_id = gpu_instance_id; + card->compute_instance_id = compute_instance_id; + + if (uuid != NULL) { + card->uuid = flb_sds_create(uuid); + if (card->uuid == NULL) { + flb_free(card); + return -1; + } + } + + if (parent_uuid != NULL) { + card->parent_uuid = flb_sds_create(parent_uuid); + if (card->parent_uuid == NULL) { + if (card->uuid != NULL) { + flb_sds_destroy(card->uuid); + } + flb_free(card); + return -1; + } + } + + cfl_list_add(&card->_head, &ctx->cards); + return 0; +} + +static int nvml_detect_mig_devices(struct in_gpu_metrics *ctx, + int parent_card_id, + nvmlDevice_t parent_device, + const char *parent_uuid) +{ + nvmlDevice_t mig_device; + nvmlDevice_t mig_parent; + unsigned int current_mode; + unsigned int pending_mode; + unsigned int mig_count; + unsigned int mig_index; + unsigned int gi; + unsigned int ci; + char mig_uuid[NVML_UUID_BUFFER_SIZE]; + char resolved_parent_uuid[NVML_UUID_BUFFER_SIZE]; + const char *final_parent_uuid; + nvmlReturn_t result; + + if (f_nvml_device_get_mig_mode == NULL || + f_nvml_device_get_max_mig_device_count == NULL || + f_nvml_device_get_mig_device_handle_by_index == NULL || + f_nvml_device_get_gpu_instance_id == NULL || + f_nvml_device_get_compute_instance_id == NULL) { + return 0; + } + + result = f_nvml_device_get_mig_mode(parent_device, ¤t_mode, &pending_mode); + if (result != NVML_SUCCESS || current_mode != NVML_FEATURE_ENABLED) { + return 0; + } + + result = f_nvml_device_get_max_mig_device_count(parent_device, &mig_count); + if (result != NVML_SUCCESS) { + return -1; + } + + for (mig_index = 0; mig_index < mig_count; mig_index++) { + result = f_nvml_device_get_mig_device_handle_by_index(parent_device, mig_index, &mig_device); + if (result != NVML_SUCCESS) { + continue; + } + + if (nvml_read_device_uuid(mig_device, mig_uuid, sizeof(mig_uuid)) != 0) { + continue; + } + + result = f_nvml_device_get_gpu_instance_id(mig_device, &gi); + if (result != NVML_SUCCESS) { + continue; + } + + result = f_nvml_device_get_compute_instance_id(mig_device, &ci); + if (result != NVML_SUCCESS) { + continue; + } + + final_parent_uuid = parent_uuid; + if (final_parent_uuid == NULL && f_nvml_device_get_parent_from_mig != NULL) { + result = f_nvml_device_get_parent_from_mig(mig_device, &mig_parent); + if (result == NVML_SUCCESS && + nvml_read_device_uuid(mig_parent, resolved_parent_uuid, sizeof(resolved_parent_uuid)) == 0) { + final_parent_uuid = resolved_parent_uuid; + } + } + + if (nvml_register_card(ctx, + parent_card_id, + (int) gi, + (int) ci, + mig_uuid, + final_parent_uuid) != 0) { + return -1; + } + } + + return 0; +} + +static int nvml_get_device_handle(struct gpu_card *card, nvmlDevice_t *device) +{ + nvmlReturn_t result; + + if (card->uuid != NULL && f_nvml_device_get_handle_by_uuid != NULL) { + result = f_nvml_device_get_handle_by_uuid(card->uuid, device); + if (result == NVML_SUCCESS) { + return 0; + } + } + + result = f_nvml_device_get_handle_by_index_v2((unsigned int) card->id, device); + if (result != NVML_SUCCESS) { + return -1; + } + + return 0; +} + +static int nvml_process_entry_is_valid(unsigned int pid, uint64_t used_gpu_memory) +{ + if (pid == 0 || pid == UINT_MAX) { + return FLB_FALSE; + } + + /* + * Linux PID upper bound is 2^22. This avoids emitting invalid IDs that + * may appear when some NVML process entries are partially populated. + */ + if (pid > NVML_MAX_VALID_PID) { + return FLB_FALSE; + } + + if (used_gpu_memory == UINT64_MAX) { + return FLB_FALSE; + } + + return FLB_TRUE; +} + +static void nvml_emit_process_memory_samples_v1(struct in_gpu_metrics *ctx, + const char *card_label, + nvmlDevice_t device, + uint64_t ts, + nvmlDeviceGetComputeRunningProcesses_t api) +{ + struct nvmlProcessInfo_v1_t infos[NVML_MAX_PROCESS_SAMPLES]; + unsigned int info_count; + unsigned int i; + char pid_buf[32]; + nvmlReturn_t result; + + if (api == NULL || ctx->g_process_memory == NULL) { + return; + } + + info_count = NVML_MAX_PROCESS_SAMPLES; + result = api(device, &info_count, infos); + if (result != NVML_SUCCESS) { + return; + } + + for (i = 0; i < info_count; i++) { + if (nvml_process_entry_is_valid(infos[i].pid, + infos[i].usedGpuMemory) == FLB_FALSE) { + continue; + } + snprintf(pid_buf, sizeof(pid_buf), "%u", infos[i].pid); + cmt_gauge_set(ctx->g_process_memory, ts, (double) infos[i].usedGpuMemory, 3, + (char *[]) { (char *) card_label, "nvidia", pid_buf}); + } +} + +static void nvml_emit_process_memory_samples_v2(struct in_gpu_metrics *ctx, + const char *card_label, + nvmlDevice_t device, + uint64_t ts, + nvmlDeviceGetComputeRunningProcesses_v2_t api) +{ + struct nvmlProcessInfo_v2_t infos[NVML_MAX_PROCESS_SAMPLES]; + unsigned int info_count; + unsigned int i; + char pid_buf[32]; + nvmlReturn_t result; + + if (api == NULL || ctx->g_process_memory == NULL) { + return; + } + + info_count = NVML_MAX_PROCESS_SAMPLES; + result = api(device, &info_count, infos); + if (result != NVML_SUCCESS) { + return; + } + + for (i = 0; i < info_count; i++) { + if (nvml_process_entry_is_valid(infos[i].pid, + infos[i].usedGpuMemory) == FLB_FALSE) { + continue; + } + snprintf(pid_buf, sizeof(pid_buf), "%u", infos[i].pid); + cmt_gauge_set(ctx->g_process_memory, ts, (double) infos[i].usedGpuMemory, 3, + (char *[]) { (char *) card_label, "nvidia", pid_buf}); + } +} + +static void nvml_emit_process_memory_samples_v3(struct in_gpu_metrics *ctx, + const char *card_label, + nvmlDevice_t device, + uint64_t ts, + nvmlDeviceGetComputeRunningProcesses_v3_t api) +{ + struct nvmlProcessInfo_v3_t infos[NVML_MAX_PROCESS_SAMPLES]; + unsigned int info_count; + unsigned int i; + char pid_buf[32]; + nvmlReturn_t result; + + if (api == NULL || ctx->g_process_memory == NULL) { + return; + } + + info_count = NVML_MAX_PROCESS_SAMPLES; + result = api(device, &info_count, infos); + if (result != NVML_SUCCESS) { + return; + } + + for (i = 0; i < info_count; i++) { + if (nvml_process_entry_is_valid(infos[i].pid, + infos[i].usedGpuMemory) == FLB_FALSE) { + continue; + } + snprintf(pid_buf, sizeof(pid_buf), "%u", infos[i].pid); + cmt_gauge_set(ctx->g_process_memory, ts, (double) infos[i].usedGpuMemory, 3, + (char *[]) { (char *) card_label, "nvidia", pid_buf}); + } +} + +static void nvml_collect_process_memory(struct in_gpu_metrics *ctx, + struct gpu_card *card, + nvmlDevice_t device, + uint64_t ts, + const char *card_label) +{ + if (f_nvml_device_get_compute_running_processes_v3 != NULL) { + nvml_emit_process_memory_samples_v3(ctx, card_label, device, ts, + f_nvml_device_get_compute_running_processes_v3); + } + if (f_nvml_device_get_compute_running_processes_v2 != NULL) { + nvml_emit_process_memory_samples_v2(ctx, card_label, device, ts, + f_nvml_device_get_compute_running_processes_v2); + } + if (f_nvml_device_get_compute_running_processes != NULL) { + nvml_emit_process_memory_samples_v1(ctx, card_label, device, ts, + f_nvml_device_get_compute_running_processes); + } + + if (f_nvml_device_get_graphics_running_processes_v3 != NULL) { + nvml_emit_process_memory_samples_v3(ctx, card_label, device, ts, + f_nvml_device_get_graphics_running_processes_v3); + } + if (f_nvml_device_get_graphics_running_processes_v2 != NULL) { + nvml_emit_process_memory_samples_v2(ctx, card_label, device, ts, + f_nvml_device_get_graphics_running_processes_v2); + } + if (f_nvml_device_get_graphics_running_processes != NULL) { + nvml_emit_process_memory_samples_v1(ctx, card_label, device, ts, + f_nvml_device_get_graphics_running_processes); + } + + if (ctx->g_mig_info != NULL && + card->gpu_instance_id >= 0 && + card->compute_instance_id >= 0 && + card->parent_uuid != NULL) { + char gi_buf[16]; + char ci_buf[16]; + + snprintf(gi_buf, sizeof(gi_buf), "%d", card->gpu_instance_id); + snprintf(ci_buf, sizeof(ci_buf), "%d", card->compute_instance_id); + + cmt_gauge_set(ctx->g_mig_info, ts, 1.0, 5, + (char *[]) { (char *) card_label, "nvidia", card->parent_uuid, + gi_buf, ci_buf}); + } +} + +int nvml_gpu_initialize(struct in_gpu_metrics *ctx) +{ + nvmlReturn_t result; + + nvml_reset_api_symbols(); + + if (ctx->enable_nvml == FLB_FALSE) { + return 0; + } + + ctx->nvml_lib_handle = dlopen("libnvidia-ml.so.1", RTLD_LAZY); + if (ctx->nvml_lib_handle == NULL) { + ctx->nvml_lib_handle = dlopen("libnvidia-ml.so", RTLD_LAZY); + } + if (ctx->nvml_lib_handle == NULL) { + flb_plg_info(ctx->ins, + "NVML shared library not found; NVIDIA GPU metrics are disabled"); + return 0; + } + + if (load_nvml_symbol(ctx, "nvmlInit_v2", (void **) &f_nvml_init_v2) != 0 || + load_nvml_symbol(ctx, "nvmlShutdown", (void **) &f_nvml_shutdown) != 0 || + load_nvml_symbol(ctx, "nvmlDeviceGetCount_v2", (void **) &f_nvml_device_get_count_v2) != 0 || + load_nvml_symbol(ctx, "nvmlDeviceGetHandleByIndex_v2", + (void **) &f_nvml_device_get_handle_by_index_v2) != 0 || + load_nvml_symbol(ctx, "nvmlDeviceGetMemoryInfo", (void **) &f_nvml_device_get_memory_info) != 0 || + load_nvml_symbol(ctx, "nvmlDeviceGetUtilizationRates", + (void **) &f_nvml_device_get_utilization_rates) != 0 || + load_nvml_symbol(ctx, "nvmlDeviceGetTemperature", + (void **) &f_nvml_device_get_temperature) != 0 || + load_nvml_symbol(ctx, "nvmlDeviceGetPowerUsage", (void **) &f_nvml_device_get_power_usage) != 0) { + dlclose(ctx->nvml_lib_handle); + ctx->nvml_lib_handle = NULL; + nvml_reset_api_symbols(); + return -1; + } + + f_nvml_error_string = dlsym(ctx->nvml_lib_handle, "nvmlErrorString"); + load_optional_nvml_symbol(ctx, "nvmlDeviceGetFanSpeed", (void **) &f_nvml_device_get_fan_speed); + load_optional_nvml_symbol(ctx, "nvmlDeviceGetClockInfo", (void **) &f_nvml_device_get_clock_info); + load_optional_nvml_symbol(ctx, "nvmlDeviceGetUUID", (void **) &f_nvml_device_get_uuid); + load_optional_nvml_symbol(ctx, "nvmlDeviceGetHandleByUUID", (void **) &f_nvml_device_get_handle_by_uuid); + load_optional_nvml_symbol(ctx, "nvmlDeviceGetMigMode", (void **) &f_nvml_device_get_mig_mode); + load_optional_nvml_symbol(ctx, "nvmlDeviceGetMaxMigDeviceCount", + (void **) &f_nvml_device_get_max_mig_device_count); + load_optional_nvml_symbol(ctx, "nvmlDeviceGetMigDeviceHandleByIndex", + (void **) &f_nvml_device_get_mig_device_handle_by_index); + load_optional_nvml_symbol(ctx, "nvmlDeviceGetGpuInstanceId", (void **) &f_nvml_device_get_gpu_instance_id); + load_optional_nvml_symbol(ctx, "nvmlDeviceGetComputeInstanceId", + (void **) &f_nvml_device_get_compute_instance_id); + load_optional_nvml_symbol(ctx, "nvmlDeviceGetDeviceHandleFromMigDeviceHandle", + (void **) &f_nvml_device_get_parent_from_mig); + load_optional_nvml_symbol(ctx, "nvmlDeviceGetComputeRunningProcesses_v3", + (void **) &f_nvml_device_get_compute_running_processes_v3); + load_optional_nvml_symbol(ctx, "nvmlDeviceGetGraphicsRunningProcesses_v3", + (void **) &f_nvml_device_get_graphics_running_processes_v3); + load_optional_nvml_symbol(ctx, "nvmlDeviceGetComputeRunningProcesses_v2", + (void **) &f_nvml_device_get_compute_running_processes_v2); + load_optional_nvml_symbol(ctx, "nvmlDeviceGetGraphicsRunningProcesses_v2", + (void **) &f_nvml_device_get_graphics_running_processes_v2); + load_optional_nvml_symbol(ctx, "nvmlDeviceGetComputeRunningProcesses", + (void **) &f_nvml_device_get_compute_running_processes); + load_optional_nvml_symbol(ctx, "nvmlDeviceGetGraphicsRunningProcesses", + (void **) &f_nvml_device_get_graphics_running_processes); + + result = f_nvml_init_v2(); + if (result != NVML_SUCCESS) { + flb_plg_warn(ctx->ins, "NVML init failed: %s", nvml_result_to_string(result)); + dlclose(ctx->nvml_lib_handle); + ctx->nvml_lib_handle = NULL; + nvml_reset_api_symbols(); + return -1; + } + + ctx->nvml_initialized = FLB_TRUE; + flb_plg_info(ctx->ins, "NVML backend enabled"); + return 0; +} + +int nvml_gpu_detect_cards(struct in_gpu_metrics *ctx) +{ + unsigned int index; + unsigned int count; + nvmlDevice_t device; + nvmlReturn_t result; + int uuid_ok; + char uuid[NVML_UUID_BUFFER_SIZE]; + int detected; + + if (ctx->nvml_initialized == FLB_FALSE) { + return 0; + } + + result = f_nvml_device_get_count_v2(&count); + if (result != NVML_SUCCESS) { + flb_plg_warn(ctx->ins, "NVML device count failed: %s", nvml_result_to_string(result)); + return -1; + } + + detected = 0; + + for (index = 0; index < count; index++) { + if (!gpu_should_include_card(ctx, (int) index)) { + continue; + } + + result = f_nvml_device_get_handle_by_index_v2(index, &device); + if (result != NVML_SUCCESS) { + continue; + } + + uuid_ok = (nvml_read_device_uuid(device, uuid, sizeof(uuid)) == 0); + + if (uuid_ok) { + if (nvml_register_card(ctx, (int) index, -1, -1, uuid, NULL) != 0) { + return -1; + } + } + else { + if (nvml_register_card(ctx, (int) index, -1, -1, NULL, NULL) != 0) { + return -1; + } + } + detected++; + + if (nvml_detect_mig_devices(ctx, (int) index, device, + uuid_ok ? uuid : NULL) != 0) { + flb_plg_warn(ctx->ins, "failed to detect MIG devices for card%d", (int) index); + } + } + + if (detected > 0) { + flb_plg_info(ctx->ins, "detected %d NVIDIA GPU card(s) via NVML", detected); + } + + return 0; +} + +int nvml_gpu_collect_metrics(struct in_gpu_metrics *ctx, struct gpu_card *card) +{ + nvmlDevice_t device; + struct nvmlMemory_t memory; + struct nvmlUtilization_t util; + unsigned int temp; + unsigned int power_mw; + unsigned int fan_percent; + unsigned int sm_clock_mhz; + unsigned int mem_clock_mhz; + unsigned int graphics_clock_mhz; + nvmlReturn_t result; + uint64_t ts; + flb_sds_t fallback_card_id; + const char *card_label; + + if (ctx->nvml_initialized == FLB_FALSE) { + return -1; + } + + if (nvml_get_device_handle(card, &device) != 0) { + return -1; + } + + fallback_card_id = NULL; + card_label = card->uuid; + + if (card_label == NULL) { + fallback_card_id = flb_sds_create_size(16); + if (fallback_card_id == NULL) { + flb_errno(); + return -1; + } + fallback_card_id = flb_sds_printf(&fallback_card_id, "%d", card->id); + if (fallback_card_id == NULL) { + return -1; + } + card_label = fallback_card_id; + } + + ts = cfl_time_now(); + + result = f_nvml_device_get_utilization_rates(device, &util); + if (result == NVML_SUCCESS) { + cmt_gauge_set(ctx->g_utilization, ts, (double) util.gpu, 2, + (char *[]) {(char *) card_label, "nvidia"}); + } + + result = f_nvml_device_get_memory_info(device, &memory); + if (result == NVML_SUCCESS) { + cmt_gauge_set(ctx->g_mem_used, ts, (double) memory.used, 2, + (char *[]) {(char *) card_label, "nvidia"}); + cmt_gauge_set(ctx->g_mem_total, ts, (double) memory.total, 2, + (char *[]) {(char *) card_label, "nvidia"}); + } + + if (f_nvml_device_get_clock_info != NULL) { + result = f_nvml_device_get_clock_info(device, NVML_CLOCK_SM, &sm_clock_mhz); + if (result == NVML_SUCCESS) { + cmt_gauge_set(ctx->g_clock, ts, (double) sm_clock_mhz, 3, + (char *[]) {(char *) card_label, "nvidia", "sm"}); + } + + result = f_nvml_device_get_clock_info(device, NVML_CLOCK_MEM, &mem_clock_mhz); + if (result == NVML_SUCCESS) { + cmt_gauge_set(ctx->g_clock, ts, (double) mem_clock_mhz, 3, + (char *[]) {(char *) card_label, "nvidia", "memory"}); + } + + result = f_nvml_device_get_clock_info(device, NVML_CLOCK_GRAPHICS, &graphics_clock_mhz); + if (result == NVML_SUCCESS) { + cmt_gauge_set(ctx->g_clock, ts, (double) graphics_clock_mhz, 3, + (char *[]) {(char *) card_label, "nvidia", "graphics"}); + } + } + + if (ctx->enable_temperature) { + result = f_nvml_device_get_temperature(device, NVML_TEMPERATURE_GPU, &temp); + if (result == NVML_SUCCESS) { + cmt_gauge_set(ctx->g_temp, ts, (double) temp, 2, + (char *[]) {(char *) card_label, "nvidia"}); + } + } + + if (ctx->enable_power) { + result = f_nvml_device_get_power_usage(device, &power_mw); + if (result == NVML_SUCCESS) { + cmt_gauge_set(ctx->g_power, ts, (double) power_mw / 1000.0, 2, + (char *[]) {(char *) card_label, "nvidia"}); + } + } + + if (f_nvml_device_get_fan_speed != NULL) { + result = f_nvml_device_get_fan_speed(device, &fan_percent); + if (result == NVML_SUCCESS) { + cmt_gauge_set(ctx->g_fan_pwm, ts, (double) fan_percent, 2, + (char *[]) {(char *) card_label, "nvidia"}); + } + } + + nvml_collect_process_memory(ctx, card, device, ts, card_label); + + if (fallback_card_id != NULL) { + flb_sds_destroy(fallback_card_id); + } + + return 0; +} + +void nvml_gpu_shutdown(struct in_gpu_metrics *ctx) +{ + if (ctx->nvml_initialized == FLB_TRUE) { + if (f_nvml_shutdown != NULL) { + f_nvml_shutdown(); + } + ctx->nvml_initialized = FLB_FALSE; + } + + if (ctx->nvml_lib_handle != NULL) { + dlclose(ctx->nvml_lib_handle); + ctx->nvml_lib_handle = NULL; + } + + nvml_reset_api_symbols(); +} diff --git a/plugins/in_gpu_metrics/nvml_gpu.h b/plugins/in_gpu_metrics/nvml_gpu.h new file mode 100644 index 00000000000..aae64867498 --- /dev/null +++ b/plugins/in_gpu_metrics/nvml_gpu.h @@ -0,0 +1,30 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2026 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_GPU_METRICS_NVML_H +#define FLB_GPU_METRICS_NVML_H + +#include "gpu_metrics.h" + +int nvml_gpu_initialize(struct in_gpu_metrics *ctx); +int nvml_gpu_detect_cards(struct in_gpu_metrics *ctx); +int nvml_gpu_collect_metrics(struct in_gpu_metrics *ctx, struct gpu_card *card); +void nvml_gpu_shutdown(struct in_gpu_metrics *ctx); + +#endif diff --git a/plugins/in_http/http_conn.c b/plugins/in_http/http_conn.c index 73f1ed53117..21801965e65 100644 --- a/plugins/in_http/http_conn.c +++ b/plugins/in_http/http_conn.c @@ -107,6 +107,10 @@ static int http_conn_event(void *data) (void *) &conn->buf_data[conn->buf_len], available); + if (flb_io_net_is_retry(bytes)) { + return 0; + } + if (bytes <= 0) { flb_plg_trace(ctx->ins, "fd=%i closed connection", event->fd); http_conn_del(conn); diff --git a/plugins/in_kubernetes_events/kubernetes_events.c b/plugins/in_kubernetes_events/kubernetes_events.c index 2728d30bb7a..080733480cd 100644 --- a/plugins/in_kubernetes_events/kubernetes_events.c +++ b/plugins/in_kubernetes_events/kubernetes_events.c @@ -54,8 +54,13 @@ static int file_to_buffer(const char *path, ssize_t bytes; FILE *fp; struct stat st; + const char *file_mode = "r"; - if (!(fp = fopen(path, "r"))) { +#ifdef FLB_SYSTEM_WINDOWS + file_mode = "rb"; +#endif + + if (!(fp = fopen(path, file_mode))) { return -1; } @@ -303,7 +308,7 @@ static bool check_event_is_filtered(struct k8s_events *ctx, msgpack_object *obj, flb_sds_t uid; uint64_t resource_version; - outdated = cfl_time_now() - (ctx->retention_time * 1000000000L); + outdated = cfl_time_now() - ((uint64_t) ctx->retention_time * 1000000000ULL); if (flb_time_to_nanosec(event_time) < outdated) { flb_plg_debug(ctx->ins, "Item is older than retention_time: %" PRIu64 " < %" PRIu64, flb_time_to_nanosec(event_time), outdated); @@ -655,7 +660,8 @@ static int k8s_events_cleanup_db(struct flb_input_instance *ins, FLB_INPUT_RETURN(0); } - retention_time_ago = cfl_time_now() - (ctx->retention_time * 1000000000L); + retention_time_ago = cfl_time_now() - + ((uint64_t) ctx->retention_time * 1000000000ULL); sqlite3_bind_int64(ctx->stmt_delete_old_kubernetes_events, 1, (int64_t)retention_time_ago); ret = sqlite3_step(ctx->stmt_delete_old_kubernetes_events); diff --git a/plugins/in_lib/in_lib.c b/plugins/in_lib/in_lib.c index d3c782a8d16..03b6e5d1af3 100644 --- a/plugins/in_lib/in_lib.c +++ b/plugins/in_lib/in_lib.c @@ -19,6 +19,7 @@ #include #include +#include #include #include #include @@ -42,40 +43,56 @@ static int in_lib_collect(struct flb_input_instance *ins, int out_size; int capacity; int size; + int total_bytes = 0; char *ptr; char *pack; struct flb_log_event record; struct flb_log_event_decoder decoder; struct flb_in_lib_config *ctx = in_context; - capacity = (ctx->buf_size - ctx->buf_len); + while (total_bytes < LIB_MAX_READ_SIZE) { + capacity = (ctx->buf_size - ctx->buf_len); + + /* Allocate memory as required (FIXME: this will be limited in later) */ + if (capacity == 0) { + size = ctx->buf_size + LIB_BUF_CHUNK; + ptr = flb_realloc(ctx->buf_data, size); + if (!ptr) { + flb_errno(); + return -1; + } + ctx->buf_data = ptr; + ctx->buf_size = size; + capacity = LIB_BUF_CHUNK; + } - /* Allocate memory as required (FIXME: this will be limited in later) */ - if (capacity == 0) { - size = ctx->buf_size + LIB_BUF_CHUNK; - ptr = flb_realloc(ctx->buf_data, size); - if (!ptr) { - flb_errno(); - return -1; + bytes = flb_pipe_r(ctx->fd, + ctx->buf_data + ctx->buf_len, + capacity); + if (bytes == -1) { + if (FLB_PIPE_WOULDBLOCK()) { + break; + } + + perror("read"); + flb_pipe_error(); + if (errno == EPIPE) { + return -1; + } + return 0; + } + else if (bytes == 0) { + break; } - ctx->buf_data = ptr; - ctx->buf_size = size; - capacity = LIB_BUF_CHUNK; + + ctx->buf_len += bytes; + total_bytes += bytes; } - bytes = flb_pipe_r(ctx->fd, - ctx->buf_data + ctx->buf_len, - capacity); - flb_plg_trace(ctx->ins, "in_lib read() = %i", bytes); - if (bytes == -1) { - perror("read"); - flb_pipe_error(); - if (errno == -EPIPE) { - return -1; - } + flb_plg_trace(ctx->ins, "in_lib read() = %i", total_bytes); + if (total_bytes == 0) { return 0; } - ctx->buf_len += bytes; /* initially we should support json input */ ret = flb_pack_json_state(ctx->buf_data, ctx->buf_len, @@ -90,7 +107,10 @@ static int in_lib_collect(struct flb_input_instance *ins, flb_pack_state_init(&ctx->state); return -1; } - ctx->buf_len = 0; + ctx->buf_len -= ctx->state.last_byte; + memmove(ctx->buf_data, + ctx->buf_data + ctx->state.last_byte, + ctx->buf_len); dec_ret = flb_log_event_decoder_init(&decoder, pack, out_size); if (dec_ret != FLB_EVENT_DECODER_SUCCESS) { @@ -213,9 +233,24 @@ static int in_lib_init(struct flb_input_instance *in, } /* Init communication channel */ - flb_input_channel_init(in); + ret = flb_input_channel_init(in); + if (ret == -1) { + flb_plg_error(ctx->ins, "could not initialize input channel"); + flb_free(ctx->buf_data); + flb_free(ctx); + return -1; + } ctx->fd = in->channel[0]; + ret = flb_pipe_set_nonblocking(ctx->fd); + if (ret == -1) { + flb_plg_error(ctx->ins, "could not set input channel to nonblocking mode"); + flb_pipe_destroy(in->channel); + flb_free(ctx->buf_data); + flb_free(ctx); + return -1; + } + /* Set the context */ flb_input_set_context(in, ctx); diff --git a/plugins/in_lib/in_lib.h b/plugins/in_lib/in_lib.h index 183873b177c..a1f3e2a3d6e 100644 --- a/plugins/in_lib/in_lib.h +++ b/plugins/in_lib/in_lib.h @@ -26,7 +26,10 @@ #include #include -#define LIB_BUF_CHUNK 65536 +#define LIB_BUF_CHUNK 65536 + +/* Bound each collector dispatch so other engine events keep making progress. */ +#define LIB_MAX_READ_SIZE (LIB_BUF_CHUNK * 64) pthread_key_t flb_active_lib_context; diff --git a/plugins/in_mqtt/mqtt_conn.c b/plugins/in_mqtt/mqtt_conn.c index bceecde89bd..f35163a184b 100644 --- a/plugins/in_mqtt/mqtt_conn.c +++ b/plugins/in_mqtt/mqtt_conn.c @@ -54,6 +54,10 @@ int mqtt_conn_event(void *data) (void *) &conn->buf[conn->buf_len], available); + if (flb_io_net_is_retry(bytes)) { + return 0; + } + if (bytes > 0) { conn->buf_len += bytes; flb_plg_trace(ctx->ins, "[fd=%i] read()=%i bytes", diff --git a/plugins/in_node_exporter_metrics/CMakeLists.txt b/plugins/in_node_exporter_metrics/CMakeLists.txt index 55e9a860067..e04616d6d3f 100644 --- a/plugins/in_node_exporter_metrics/CMakeLists.txt +++ b/plugins/in_node_exporter_metrics/CMakeLists.txt @@ -20,6 +20,7 @@ set(src ne_config.c ne_systemd.c ne_thermalzone.c + ne_powersupply.c ne.c ) diff --git a/plugins/in_node_exporter_metrics/ne.c b/plugins/in_node_exporter_metrics/ne.c index e3906151074..3d4d7770890 100644 --- a/plugins/in_node_exporter_metrics/ne.c +++ b/plugins/in_node_exporter_metrics/ne.c @@ -47,6 +47,7 @@ #include "ne_nvme.h" #include "ne_thermalzone.h" #include "ne_hwmon.h" +#include "ne_powersupply.h" /* * Update the metrics, this function is invoked every time 'scrape_interval' @@ -204,6 +205,7 @@ static int in_ne_init(struct flb_input_instance *in, mk_list_add(&nvme_collector._head, &ctx->collectors); mk_list_add(&thermalzone_collector._head, &ctx->collectors); mk_list_add(&hwmon_collector._head, &ctx->collectors); + mk_list_add(&powersupply_collector._head, &ctx->collectors); mk_list_foreach(head, &ctx->collectors) { coll = mk_list_entry(head, struct flb_ne_collector, _head); diff --git a/plugins/in_node_exporter_metrics/ne.h b/plugins/in_node_exporter_metrics/ne.h index 9fc090652ce..7ade50906de 100644 --- a/plugins/in_node_exporter_metrics/ne.h +++ b/plugins/in_node_exporter_metrics/ne.h @@ -33,9 +33,9 @@ /* Default enabled metrics */ #ifdef __linux__ -#define NE_DEFAULT_ENABLED_METRICS "cpu,cpufreq,meminfo,diskstats,filesystem,uname,stat,time,loadavg,vmstat,netdev,netstat,sockstat,filefd,systemd,nvme,thermal_zone,hwmon" +#define NE_DEFAULT_ENABLED_METRICS "cpu,cpufreq,meminfo,diskstats,filesystem,uname,stat,time,loadavg,vmstat,netdev,netstat,sockstat,filefd,systemd,nvme,thermal_zone,hwmon,powersupplyclass" #elif __APPLE__ -#define NE_DEFAULT_ENABLED_METRICS "cpu,loadavg,meminfo,diskstats,uname,netdev" +#define NE_DEFAULT_ENABLED_METRICS "cpu,loadavg,meminfo,diskstats,filesystem,uname,netdev,powersupplyclass" #endif /* filesystem: regex for ignoring mount points and filesystem types */ @@ -101,6 +101,22 @@ struct flb_ne { struct cmt_gauge *darwin_swap_used_bytes; struct cmt_gauge *darwin_swap_total_bytes; struct cmt_counter *darwin_total_bytes; + + /* powersupply_darwin */ + struct cmt_gauge *darwin_ps_current_capacity; + struct cmt_gauge *darwin_ps_max_capacity; + struct cmt_gauge *darwin_ps_design_capacity; + struct cmt_gauge *darwin_ps_nominal_capacity; + struct cmt_gauge *darwin_ps_time_to_empty; + struct cmt_gauge *darwin_ps_time_to_full; + struct cmt_gauge *darwin_ps_voltage; + struct cmt_gauge *darwin_ps_current; + struct cmt_gauge *darwin_ps_temperature; + struct cmt_gauge *darwin_ps_present; + struct cmt_gauge *darwin_ps_charging; + struct cmt_gauge *darwin_ps_charged; + struct cmt_gauge *darwin_ps_internal_failure; + struct cmt_gauge *darwin_ps_battery_health; #endif /* diskstats: abbreviation 'dt' */ @@ -141,6 +157,10 @@ struct flb_ne { struct cmt_gauge *darwin_noproto; #endif + /* powersupply_linux */ + struct cmt_gauge *powersupply_info; + struct mk_list powersupply_dynamic_metrics; + /* sockstat_linux */ struct cmt_gauge *sockstat_sockets_used; struct cmt_gauge *sockstat_TCP_alloc; @@ -172,6 +192,7 @@ struct flb_ne { struct cmt_counter *netstat_Udp_InErrors; struct cmt_counter *netstat_Udp_OutDatagrams; struct cmt_counter *netstat_Udp_NoPorts; + struct mk_list netstat_dynamic_metrics; /* time */ struct cmt_gauge *time; diff --git a/plugins/in_node_exporter_metrics/ne_cpu_linux.c b/plugins/in_node_exporter_metrics/ne_cpu_linux.c index bc7c8a9feec..67f85b7cb53 100644 --- a/plugins/in_node_exporter_metrics/ne_cpu_linux.c +++ b/plugins/in_node_exporter_metrics/ne_cpu_linux.c @@ -130,10 +130,12 @@ static int cpu_thermal_update(struct flb_ne *ctx, uint64_t ts) } /* Package Metric: node_cpu_core_throttles_total */ - ret = ne_utils_file_read_uint64(ctx, ctx->path_sysfs, - entry->str, - "thermal_throttle", "core_throttle_count", - &core_throttle_count); + ret = ne_utils_file_read_uint64_at_level(ctx, ctx->path_sysfs, + entry->str, + "thermal_throttle", + "core_throttle_count", + &core_throttle_count, + FLB_LOG_DEBUG); if (ret != 0) { flb_plg_debug(ctx->ins, "CPU is missing core_throttle_count: %s", @@ -150,10 +152,12 @@ static int cpu_thermal_update(struct flb_ne *ctx, uint64_t ts) } /* Package Metric: node_cpu_package_throttles_total */ - ret = ne_utils_file_read_uint64(ctx, ctx->path_sysfs, - entry->str, - "thermal_throttle", "package_throttle_count", - &package_throttle_count); + ret = ne_utils_file_read_uint64_at_level(ctx, ctx->path_sysfs, + entry->str, + "thermal_throttle", + "package_throttle_count", + &package_throttle_count, + FLB_LOG_DEBUG); if (ret != 0) { flb_plg_debug(ctx->ins, "CPU is missing package_throttle_count: %s", diff --git a/plugins/in_node_exporter_metrics/ne_filesystem.c b/plugins/in_node_exporter_metrics/ne_filesystem.c index 54b3b27d456..6e17c4f87ae 100644 --- a/plugins/in_node_exporter_metrics/ne_filesystem.c +++ b/plugins/in_node_exporter_metrics/ne_filesystem.c @@ -19,6 +19,8 @@ #ifdef __linux__ #include "ne_filesystem_linux.c" +#elif __APPLE__ +#include "ne_filesystem_darwin.c" #else #include "ne.h" diff --git a/plugins/in_node_exporter_metrics/ne_filesystem_darwin.c b/plugins/in_node_exporter_metrics/ne_filesystem_darwin.c new file mode 100644 index 00000000000..aae2be44a81 --- /dev/null +++ b/plugins/in_node_exporter_metrics/ne_filesystem_darwin.c @@ -0,0 +1,191 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2026 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include + +#include "ne.h" + +static int filesystem_update(struct flb_ne *ctx) +{ + int i; + int count; + int skip_flag; + uint64_t block_size; + uint64_t blocks; + uint64_t free_size; + uint64_t avail_size; + uint64_t size_bytes; + uint64_t avail_bytes; + uint64_t free_bytes; + uint64_t timestamp; + char *labels[3]; + struct statfs *mounts; + + count = getmntinfo(&mounts, MNT_NOWAIT); + if (count == 0) { + return -1; + } + + timestamp = cfl_time_now(); + + for (i = 0; i < count; i++) { + skip_flag = flb_regex_match(ctx->fs_regex_skip_fs_types, + (unsigned char *) mounts[i].f_fstypename, + strlen(mounts[i].f_fstypename)); + if (skip_flag) { + continue; + } + + skip_flag = flb_regex_match(ctx->fs_regex_skip_mount, + (unsigned char *) mounts[i].f_mntonname, + strlen(mounts[i].f_mntonname)); + if (skip_flag) { + continue; + } + + labels[0] = mounts[i].f_mntfromname; + labels[1] = mounts[i].f_fstypename; + labels[2] = mounts[i].f_mntonname; + + block_size = (uint64_t) mounts[i].f_bsize; + blocks = (uint64_t) mounts[i].f_blocks; + free_size = (uint64_t) mounts[i].f_bfree; + avail_size = (uint64_t) mounts[i].f_bavail; + avail_bytes = block_size * avail_size; + size_bytes = block_size * blocks; + free_bytes = block_size * free_size; + + cmt_gauge_set(ctx->fs_avail_bytes, timestamp, avail_bytes, 3, labels); + cmt_gauge_set(ctx->fs_device_error, timestamp, 0, 3, labels); + cmt_gauge_set(ctx->fs_files, timestamp, (uint64_t) mounts[i].f_files, 3, labels); + cmt_gauge_set(ctx->fs_files_free, timestamp, (uint64_t) mounts[i].f_ffree, 3, labels); + cmt_gauge_set(ctx->fs_free_bytes, timestamp, free_bytes, 3, labels); + cmt_gauge_set(ctx->fs_readonly, timestamp, (mounts[i].f_flags & MNT_RDONLY) ? 1 : 0, 3, labels); + cmt_gauge_set(ctx->fs_size_bytes, timestamp, size_bytes, 3, labels); + } + + return 0; +} + +static void ne_filesystem_destroy_regexes(struct flb_ne *ctx) +{ + if (ctx->fs_regex_skip_mount != NULL) { + flb_regex_destroy(ctx->fs_regex_skip_mount); + ctx->fs_regex_skip_mount = NULL; + } + + if (ctx->fs_regex_skip_fs_types != NULL) { + flb_regex_destroy(ctx->fs_regex_skip_fs_types); + ctx->fs_regex_skip_fs_types = NULL; + } +} + +static int ne_filesystem_init(struct flb_ne *ctx) +{ + ctx->fs_regex_skip_mount = flb_regex_create(ctx->fs_regex_ingore_mount_point_text); + if (ctx->fs_regex_skip_mount == NULL) { + goto error; + } + + ctx->fs_regex_skip_fs_types = flb_regex_create(ctx->fs_regex_ingore_filesystem_type_text); + if (ctx->fs_regex_skip_fs_types == NULL) { + goto error; + } + + ctx->fs_avail_bytes = cmt_gauge_create(ctx->cmt, "node", "filesystem", "avail_bytes", + "Filesystem space available to non-root users in bytes.", + 3, (char *[]) {"device", "fstype", "mountpoint"}); + if (ctx->fs_avail_bytes == NULL) { + goto error; + } + + ctx->fs_device_error = cmt_gauge_create(ctx->cmt, "node", "filesystem", "device_error", + "Whether an error occurred while getting statistics for the given device.", + 3, (char *[]) {"device", "fstype", "mountpoint"}); + if (ctx->fs_device_error == NULL) { + goto error; + } + + ctx->fs_files = cmt_gauge_create(ctx->cmt, "node", "filesystem", "files", + "Filesystem total file nodes.", + 3, (char *[]) {"device", "fstype", "mountpoint"}); + if (ctx->fs_files == NULL) { + goto error; + } + + ctx->fs_files_free = cmt_gauge_create(ctx->cmt, "node", "filesystem", "files_free", + "Filesystem total free file nodes.", + 3, (char *[]) {"device", "fstype", "mountpoint"}); + if (ctx->fs_files_free == NULL) { + goto error; + } + + ctx->fs_free_bytes = cmt_gauge_create(ctx->cmt, "node", "filesystem", "free_bytes", + "Filesystem free space in bytes.", + 3, (char *[]) {"device", "fstype", "mountpoint"}); + if (ctx->fs_free_bytes == NULL) { + goto error; + } + + ctx->fs_readonly = cmt_gauge_create(ctx->cmt, "node", "filesystem", "readonly", + "Filesystem read-only status.", + 3, (char *[]) {"device", "fstype", "mountpoint"}); + if (ctx->fs_readonly == NULL) { + goto error; + } + + ctx->fs_size_bytes = cmt_gauge_create(ctx->cmt, "node", "filesystem", "size_bytes", + "Filesystem size in bytes.", + 3, (char *[]) {"device", "fstype", "mountpoint"}); + if (ctx->fs_size_bytes == NULL) { + goto error; + } + + return 0; + +error: + ne_filesystem_destroy_regexes(ctx); + + return -1; +} + +static int ne_filesystem_update(struct flb_input_instance *ins, + struct flb_config *config, void *in_context) +{ + struct flb_ne *ctx = (struct flb_ne *) in_context; + + return filesystem_update(ctx); +} + +static int ne_filesystem_exit(struct flb_ne *ctx) +{ + ne_filesystem_destroy_regexes(ctx); + + return 0; +} + +struct flb_ne_collector filesystem_collector = { + .name = "filesystem", + .cb_init = ne_filesystem_init, + .cb_update = ne_filesystem_update, + .cb_exit = ne_filesystem_exit +}; diff --git a/plugins/in_node_exporter_metrics/ne_netstat_linux.c b/plugins/in_node_exporter_metrics/ne_netstat_linux.c index 4bd51b88ebf..c8d49ac70ae 100644 --- a/plugins/in_node_exporter_metrics/ne_netstat_linux.c +++ b/plugins/in_node_exporter_metrics/ne_netstat_linux.c @@ -30,9 +30,101 @@ #define NETSTAT_PROTO_NONE 0 #define NETSTAT_PROTO_TCP 1 #define NETSTAT_PROTO_UDP 2 +#define NETSTAT_PROTO_TCPEXT 3 +#define NETSTAT_PROTO_IPEXT 4 + +struct netstat_dynamic_metric { + char *name; + struct cmt_gauge *gauge; + struct mk_list _head; +}; + +static void netstat_dynamic_metrics_destroy(struct flb_ne *ctx) +{ + struct mk_list *tmp; + struct mk_list *head; + struct netstat_dynamic_metric *entry; + + mk_list_foreach_safe(head, tmp, &ctx->netstat_dynamic_metrics) { + entry = mk_list_entry(head, struct netstat_dynamic_metric, _head); + mk_list_del(&entry->_head); + if (entry->name != NULL) { + flb_free(entry->name); + } + flb_free(entry); + } +} + +static int netstat_is_static_metric(const char *name) +{ + if (strcmp(name, "Tcp_CurrEstab") == 0 || + strcmp(name, "Tcp_ActiveOpens") == 0 || + strcmp(name, "Tcp_PassiveOpens") == 0 || + strcmp(name, "Tcp_RetransSegs") == 0 || + strcmp(name, "Udp_InDatagrams") == 0 || + strcmp(name, "Udp_NoPorts") == 0 || + strcmp(name, "Udp_InErrors") == 0 || + strcmp(name, "Udp_OutDatagrams") == 0) { + return FLB_TRUE; + } + + return FLB_FALSE; +} + +static struct cmt_gauge *netstat_dynamic_metric_get(struct flb_ne *ctx, const char *name) +{ + int ret; + size_t name_len; + char *metric_name; + struct mk_list *head; + struct netstat_dynamic_metric *entry; + + mk_list_foreach(head, &ctx->netstat_dynamic_metrics) { + entry = mk_list_entry(head, struct netstat_dynamic_metric, _head); + if (strcmp(entry->name, name) == 0) { + return entry->gauge; + } + } + + entry = flb_calloc(1, sizeof(struct netstat_dynamic_metric)); + if (entry == NULL) { + flb_errno(); + return NULL; + } + + name_len = strlen(name); + metric_name = flb_malloc(name_len + 1); + if (metric_name == NULL) { + flb_free(entry); + return NULL; + } + + ret = snprintf(metric_name, name_len + 1, "%s", name); + if (ret < 0 || ret >= (int) (name_len + 1)) { + flb_free(metric_name); + flb_free(entry); + return NULL; + } + + entry->gauge = cmt_gauge_create(ctx->cmt, "node", "netstat", metric_name, + "Network statistics from /proc/net/netstat.", + 0, NULL); + if (entry->gauge == NULL) { + flb_free(metric_name); + flb_free(entry); + return NULL; + } + + entry->name = metric_name; + mk_list_add(&entry->_head, &ctx->netstat_dynamic_metrics); + + return entry->gauge; +} static int netstat_configure(struct flb_ne *ctx) { + mk_list_init(&ctx->netstat_dynamic_metrics); + ctx->netstat_Tcp_CurrEstab = cmt_gauge_create(ctx->cmt, "node", "netstat", "Tcp_CurrEstab", "Number of TCP connections in ESTABLISHED or CLOSE-WAIT state.", @@ -184,6 +276,13 @@ static void netstat_process_pair(struct flb_ne *ctx, int values_count; struct mk_list headers; struct mk_list values; + int idx; + double d_val; + struct cmt_gauge *metric; + char metric_name[256]; + struct flb_slist_entry *proto_name; + struct flb_slist_entry *key; + struct flb_slist_entry *val; mk_list_init(&headers); mk_list_init(&values); @@ -198,6 +297,65 @@ static void netstat_process_pair(struct flb_ne *ctx, else if (proto == NETSTAT_PROTO_UDP) { netstat_process_udp(ctx, &headers, headers_count, &values, values_count, ts); } + else if (proto == NETSTAT_PROTO_TCPEXT || proto == NETSTAT_PROTO_IPEXT) { + for (idx = 1; idx < headers_count && idx < values_count; idx++) { + key = flb_slist_entry_get(&headers, idx); + val = flb_slist_entry_get(&values, idx); + + if (key == NULL || val == NULL) { + continue; + } + + if (ne_utils_str_to_double(val->str, &d_val) != 0) { + continue; + } + + if (proto == NETSTAT_PROTO_TCPEXT) { + snprintf(metric_name, sizeof(metric_name) - 1, "TcpExt_%s", key->str); + } + else { + snprintf(metric_name, sizeof(metric_name) - 1, "IpExt_%s", key->str); + } + metric_name[sizeof(metric_name) - 1] = '\0'; + + metric = netstat_dynamic_metric_get(ctx, metric_name); + if (metric != NULL) { + cmt_gauge_set(metric, ts, d_val, 0, NULL); + } + } + } + else { + proto_name = flb_slist_entry_get(&headers, 0); + if (proto_name != NULL && strlen(proto_name->str) > 1) { + proto_name->str[strlen(proto_name->str) - 1] = '\0'; + } + + for (idx = 1; idx < headers_count && idx < values_count; idx++) { + key = flb_slist_entry_get(&headers, idx); + val = flb_slist_entry_get(&values, idx); + + if (key == NULL || val == NULL || proto_name == NULL) { + continue; + } + + if (ne_utils_str_to_double(val->str, &d_val) != 0) { + continue; + } + + snprintf(metric_name, sizeof(metric_name) - 1, "%s_%s", + proto_name->str, key->str); + metric_name[sizeof(metric_name) - 1] = '\0'; + + if (netstat_is_static_metric(metric_name) == FLB_TRUE) { + continue; + } + + metric = netstat_dynamic_metric_get(ctx, metric_name); + if (metric != NULL) { + cmt_gauge_set(metric, ts, d_val, 0, NULL); + } + } + } } flb_slist_destroy(&headers); @@ -222,40 +380,142 @@ static int netstat_update(struct flb_ne *ctx) ts = cfl_time_now(); prev_line = NULL; - prev_proto = NETSTAT_PROTO_NONE; mk_list_foreach(head, &list) { line = mk_list_entry(head, struct flb_slist_entry, _head); - if (prev_proto != NETSTAT_PROTO_NONE) { - if (prev_proto == NETSTAT_PROTO_TCP && strncmp(line->str, "Tcp:", 4) == 0) { + if (prev_line != NULL) { + if (strncmp(prev_line, "Tcp:", 4) == 0 && strncmp(line->str, "Tcp:", 4) == 0) { netstat_process_pair(ctx, prev_line, line->str, NETSTAT_PROTO_TCP, ts); - prev_proto = NETSTAT_PROTO_NONE; + netstat_process_pair(ctx, prev_line, line->str, NETSTAT_PROTO_NONE, ts); prev_line = NULL; continue; } - else if (prev_proto == NETSTAT_PROTO_UDP && strncmp(line->str, "Udp:", 4) == 0) { + else if (strncmp(prev_line, "Udp:", 4) == 0 && strncmp(line->str, "Udp:", 4) == 0) { netstat_process_pair(ctx, prev_line, line->str, NETSTAT_PROTO_UDP, ts); - prev_proto = NETSTAT_PROTO_NONE; + netstat_process_pair(ctx, prev_line, line->str, NETSTAT_PROTO_NONE, ts); + prev_line = NULL; + continue; + } + else if ((strncmp(prev_line, "Ip:", 3) == 0 && strncmp(line->str, "Ip:", 3) == 0) || + (strncmp(prev_line, "Icmp:", 5) == 0 && strncmp(line->str, "Icmp:", 5) == 0) || + (strncmp(prev_line, "IcmpMsg:", 8) == 0 && strncmp(line->str, "IcmpMsg:", 8) == 0) || + (strncmp(prev_line, "UdpLite:", 8) == 0 && strncmp(line->str, "UdpLite:", 8) == 0)) { + netstat_process_pair(ctx, prev_line, line->str, NETSTAT_PROTO_NONE, ts); prev_line = NULL; continue; } - prev_proto = NETSTAT_PROTO_NONE; prev_line = NULL; } - if (strncmp(line->str, "Tcp:", 4) == 0) { + if (strchr(line->str, ':') != NULL) { prev_line = line->str; - prev_proto = NETSTAT_PROTO_TCP; } - else if (strncmp(line->str, "Udp:", 4) == 0) { - prev_line = line->str; - prev_proto = NETSTAT_PROTO_UDP; + } + + flb_slist_destroy(&list); + + mk_list_init(&list); + ret = ne_utils_file_read_lines(ctx, ctx->path_procfs, "/net/netstat", &list); + if (ret == 0) { + prev_line = NULL; + prev_proto = NETSTAT_PROTO_NONE; + + mk_list_foreach(head, &list) { + line = mk_list_entry(head, struct flb_slist_entry, _head); + + if (prev_proto != NETSTAT_PROTO_NONE) { + if (prev_proto == NETSTAT_PROTO_TCPEXT && strncmp(line->str, "TcpExt:", 7) == 0) { + netstat_process_pair(ctx, prev_line, line->str, NETSTAT_PROTO_TCPEXT, ts); + prev_proto = NETSTAT_PROTO_NONE; + prev_line = NULL; + continue; + } + else if (prev_proto == NETSTAT_PROTO_IPEXT && strncmp(line->str, "IpExt:", 6) == 0) { + netstat_process_pair(ctx, prev_line, line->str, NETSTAT_PROTO_IPEXT, ts); + prev_proto = NETSTAT_PROTO_NONE; + prev_line = NULL; + continue; + } + + prev_proto = NETSTAT_PROTO_NONE; + prev_line = NULL; + } + + if (strncmp(line->str, "TcpExt:", 7) == 0) { + prev_line = line->str; + prev_proto = NETSTAT_PROTO_TCPEXT; + } + else if (strncmp(line->str, "IpExt:", 6) == 0) { + prev_line = line->str; + prev_proto = NETSTAT_PROTO_IPEXT; + } } } flb_slist_destroy(&list); + + mk_list_init(&list); + ret = ne_utils_file_read_lines(ctx, ctx->path_procfs, "/net/snmp6", &list); + if (ret == 0) { + mk_list_foreach(head, &list) { + int i; + int six_index; + int metric_name_len; + double d_val; + char metric_name[256]; + char value[128]; + char raw_name[128]; + char proto_name[64]; + char field_name[64]; + struct cmt_gauge *metric; + + line = mk_list_entry(head, struct flb_slist_entry, _head); + if (sscanf(line->str, "%127s %127s", raw_name, value) != 2) { + continue; + } + + six_index = -1; + i = 0; + while (raw_name[i] != '\0') { + if (raw_name[i] == '6') { + six_index = i; + break; + } + i++; + } + if (six_index == -1 || six_index == 0 || raw_name[six_index + 1] == '\0') { + continue; + } + + snprintf(proto_name, sizeof(proto_name) - 1, "%.*s", + six_index + 1, raw_name); + proto_name[sizeof(proto_name) - 1] = '\0'; + + snprintf(field_name, sizeof(field_name) - 1, "%s", + raw_name + six_index + 1); + field_name[sizeof(field_name) - 1] = '\0'; + + metric_name_len = snprintf(metric_name, sizeof(metric_name) - 1, "%s_%s", + proto_name, field_name); + if (metric_name_len < 0) { + continue; + } + metric_name[sizeof(metric_name) - 1] = '\0'; + + if (ne_utils_str_to_double(value, &d_val) != 0) { + continue; + } + + metric = netstat_dynamic_metric_get(ctx, metric_name); + if (metric != NULL) { + cmt_gauge_set(metric, ts, d_val, 0, NULL); + } + } + flb_slist_destroy(&list); + } + return 0; } @@ -273,10 +533,18 @@ static int ne_netstat_update(struct flb_input_instance *ins, return 0; } +static int ne_netstat_exit(struct flb_ne *ctx) +{ + if (ctx != NULL) { + netstat_dynamic_metrics_destroy(ctx); + } + + return 0; +} + struct flb_ne_collector netstat_collector = { .name = "netstat", .cb_init = ne_netstat_init, .cb_update = ne_netstat_update, - .cb_exit = NULL + .cb_exit = ne_netstat_exit }; - diff --git a/plugins/in_node_exporter_metrics/ne_powersupply.c b/plugins/in_node_exporter_metrics/ne_powersupply.c new file mode 100644 index 00000000000..99f4c70c2fe --- /dev/null +++ b/plugins/in_node_exporter_metrics/ne_powersupply.c @@ -0,0 +1,35 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2026 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifdef __linux__ +#include "ne_powersupply_linux.c" +#elif __APPLE__ +#include "ne_powersupply_darwin.c" +#else + +#include "ne.h" + +struct flb_ne_collector powersupply_collector = { + .name = "powersupplyclass", + .cb_init = NULL, + .cb_update = NULL, + .cb_exit = NULL +}; + +#endif diff --git a/plugins/in_node_exporter_metrics/ne_powersupply.h b/plugins/in_node_exporter_metrics/ne_powersupply.h new file mode 100644 index 00000000000..116b6192d05 --- /dev/null +++ b/plugins/in_node_exporter_metrics/ne_powersupply.h @@ -0,0 +1,27 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2026 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_IN_NE_POWERSUPPLY_H +#define FLB_IN_NE_POWERSUPPLY_H + +#include "ne.h" + +extern struct flb_ne_collector powersupply_collector; + +#endif diff --git a/plugins/in_node_exporter_metrics/ne_powersupply_darwin.c b/plugins/in_node_exporter_metrics/ne_powersupply_darwin.c new file mode 100644 index 00000000000..53a0da6eeb8 --- /dev/null +++ b/plugins/in_node_exporter_metrics/ne_powersupply_darwin.c @@ -0,0 +1,313 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2026 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include + +#include "ne.h" + +static int cf_number_to_double(CFTypeRef value, double *out) +{ + if (value == NULL || CFGetTypeID(value) != CFNumberGetTypeID()) { + return -1; + } + + if (!CFNumberGetValue((CFNumberRef) value, kCFNumberDoubleType, out)) { + return -1; + } + + return 0; +} + +static int cf_boolean_to_double(CFTypeRef value, double *out) +{ + if (value == NULL || CFGetTypeID(value) != CFBooleanGetTypeID()) { + return -1; + } + + *out = CFBooleanGetValue((CFBooleanRef) value) ? 1.0 : 0.0; + return 0; +} + +static int set_numeric_metric(struct cmt_gauge *gauge, + CFDictionaryRef description, + CFStringRef key, + uint64_t ts, + const char *name, + double divisor) +{ + CFTypeRef value; + double out; + char *labels[] = {(char *) name}; + + value = CFDictionaryGetValue(description, key); + if (cf_number_to_double(value, &out) == -1) { + return -1; + } + + cmt_gauge_set(gauge, ts, out / divisor, 1, labels); + return 0; +} + +static int set_boolean_metric(struct cmt_gauge *gauge, + CFDictionaryRef description, + CFStringRef key, + uint64_t ts, + const char *name) +{ + CFTypeRef value; + double out; + char *labels[] = {(char *) name}; + + value = CFDictionaryGetValue(description, key); + if (cf_boolean_to_double(value, &out) == -1) { + return -1; + } + + cmt_gauge_set(gauge, ts, out, 1, labels); + return 0; +} + +static int update_power_source(struct flb_ne *ctx, + CFTypeRef source, + CFTypeRef info, + uint64_t ts) +{ + CFDictionaryRef description; + CFTypeRef name_value; + char name[128]; + CFTypeRef battery_health_value; + char battery_health[64]; + char *health_labels_good[] = {name, "Good"}; + char *health_labels_fair[] = {name, "Fair"}; + char *health_labels_poor[] = {name, "Poor"}; + + (void) ctx; + + description = IOPSGetPowerSourceDescription(info, source); + if (description == NULL) { + return -1; + } + + name_value = CFDictionaryGetValue(description, CFSTR(kIOPSNameKey)); + if (name_value == NULL || CFGetTypeID(name_value) != CFStringGetTypeID()) { + return -1; + } + + if (!CFStringGetCString((CFStringRef) name_value, name, + sizeof(name), kCFStringEncodingUTF8)) { + return -1; + } + + set_numeric_metric(ctx->darwin_ps_current_capacity, description, CFSTR(kIOPSCurrentCapacityKey), ts, name, 1.0); + set_numeric_metric(ctx->darwin_ps_max_capacity, description, CFSTR(kIOPSMaxCapacityKey), ts, name, 1.0); + set_numeric_metric(ctx->darwin_ps_design_capacity, description, CFSTR(kIOPSDesignCapacityKey), ts, name, 1.0); + set_numeric_metric(ctx->darwin_ps_nominal_capacity, description, CFSTR(kIOPSNominalCapacityKey), ts, name, 1.0); + set_numeric_metric(ctx->darwin_ps_time_to_empty, description, CFSTR(kIOPSTimeToEmptyKey), ts, name, (1.0 / 60.0)); + set_numeric_metric(ctx->darwin_ps_time_to_full, description, CFSTR(kIOPSTimeToFullChargeKey), ts, name, (1.0 / 60.0)); + set_numeric_metric(ctx->darwin_ps_voltage, description, CFSTR(kIOPSVoltageKey), ts, name, 1000.0); + set_numeric_metric(ctx->darwin_ps_current, description, CFSTR(kIOPSCurrentKey), ts, name, 1000.0); + set_numeric_metric(ctx->darwin_ps_temperature, description, CFSTR(kIOPSTemperatureKey), ts, name, 1.0); + + set_boolean_metric(ctx->darwin_ps_present, description, CFSTR(kIOPSIsPresentKey), ts, name); + set_boolean_metric(ctx->darwin_ps_charging, description, CFSTR(kIOPSIsChargingKey), ts, name); + set_boolean_metric(ctx->darwin_ps_charged, description, CFSTR(kIOPSIsChargedKey), ts, name); + set_boolean_metric(ctx->darwin_ps_internal_failure, description, CFSTR(kIOPSInternalFailureKey), ts, name); + + battery_health_value = CFDictionaryGetValue(description, CFSTR(kIOPSBatteryHealthKey)); + if (battery_health_value != NULL && + CFGetTypeID(battery_health_value) == CFStringGetTypeID() && + CFStringGetCString((CFStringRef) battery_health_value, battery_health, + sizeof(battery_health), kCFStringEncodingUTF8)) { + cmt_gauge_set(ctx->darwin_ps_battery_health, ts, strcmp(battery_health, "Good") == 0 ? 1.0 : 0.0, + 2, health_labels_good); + cmt_gauge_set(ctx->darwin_ps_battery_health, ts, strcmp(battery_health, "Fair") == 0 ? 1.0 : 0.0, + 2, health_labels_fair); + cmt_gauge_set(ctx->darwin_ps_battery_health, ts, strcmp(battery_health, "Poor") == 0 ? 1.0 : 0.0, + 2, health_labels_poor); + } + + return 0; +} + +static int ne_powersupply_init(struct flb_ne *ctx) +{ + char *label[] = {"power_supply"}; + + ctx->darwin_ps_current_capacity = cmt_gauge_create(ctx->cmt, "node", "powersupply", "current_capacity", + "Current battery capacity.", 1, label); + if (ctx->darwin_ps_current_capacity == NULL) { + flb_plg_error(ctx->ins, "failed to create gauge node_powersupply_current_capacity"); + return -1; + } + + ctx->darwin_ps_max_capacity = cmt_gauge_create(ctx->cmt, "node", "powersupply", "max_capacity", + "Maximum battery capacity.", 1, label); + if (ctx->darwin_ps_max_capacity == NULL) { + flb_plg_error(ctx->ins, "failed to create gauge node_powersupply_max_capacity"); + return -1; + } + + ctx->darwin_ps_design_capacity = cmt_gauge_create(ctx->cmt, "node", "powersupply", "design_capacity", + "Design battery capacity.", 1, label); + if (ctx->darwin_ps_design_capacity == NULL) { + flb_plg_error(ctx->ins, "failed to create gauge node_powersupply_design_capacity"); + return -1; + } + + ctx->darwin_ps_nominal_capacity = cmt_gauge_create(ctx->cmt, "node", "powersupply", "nominal_capacity", + "Nominal battery capacity.", 1, label); + if (ctx->darwin_ps_nominal_capacity == NULL) { + flb_plg_error(ctx->ins, "failed to create gauge node_powersupply_nominal_capacity"); + return -1; + } + + ctx->darwin_ps_time_to_empty = cmt_gauge_create(ctx->cmt, "node", "powersupply", "time_to_empty_seconds", + "Estimated time to empty in seconds.", 1, label); + if (ctx->darwin_ps_time_to_empty == NULL) { + flb_plg_error(ctx->ins, "failed to create gauge node_powersupply_time_to_empty_seconds"); + return -1; + } + + ctx->darwin_ps_time_to_full = cmt_gauge_create(ctx->cmt, "node", "powersupply", "time_to_full_seconds", + "Estimated time to full charge in seconds.", 1, label); + if (ctx->darwin_ps_time_to_full == NULL) { + flb_plg_error(ctx->ins, "failed to create gauge node_powersupply_time_to_full_seconds"); + return -1; + } + + ctx->darwin_ps_voltage = cmt_gauge_create(ctx->cmt, "node", "powersupply", "voltage_volt", + "Battery voltage in volts.", 1, label); + if (ctx->darwin_ps_voltage == NULL) { + flb_plg_error(ctx->ins, "failed to create gauge node_powersupply_voltage_volt"); + return -1; + } + + ctx->darwin_ps_current = cmt_gauge_create(ctx->cmt, "node", "powersupply", "current_ampere", + "Battery current in amperes.", 1, label); + if (ctx->darwin_ps_current == NULL) { + flb_plg_error(ctx->ins, "failed to create gauge node_powersupply_current_ampere"); + return -1; + } + + ctx->darwin_ps_temperature = cmt_gauge_create(ctx->cmt, "node", "powersupply", "temp_celsius", + "Battery temperature in celsius.", 1, label); + if (ctx->darwin_ps_temperature == NULL) { + flb_plg_error(ctx->ins, "failed to create gauge node_powersupply_temp_celsius"); + return -1; + } + + ctx->darwin_ps_present = cmt_gauge_create(ctx->cmt, "node", "powersupply", "present", + "Power supply present status.", 1, label); + if (ctx->darwin_ps_present == NULL) { + flb_plg_error(ctx->ins, "failed to create gauge node_powersupply_present"); + return -1; + } + + ctx->darwin_ps_charging = cmt_gauge_create(ctx->cmt, "node", "powersupply", "charging", + "Power supply charging status.", 1, label); + if (ctx->darwin_ps_charging == NULL) { + flb_plg_error(ctx->ins, "failed to create gauge node_powersupply_charging"); + return -1; + } + + ctx->darwin_ps_charged = cmt_gauge_create(ctx->cmt, "node", "powersupply", "charged", + "Power supply charged status.", 1, label); + if (ctx->darwin_ps_charged == NULL) { + flb_plg_error(ctx->ins, "failed to create gauge node_powersupply_charged"); + return -1; + } + + ctx->darwin_ps_internal_failure = cmt_gauge_create(ctx->cmt, "node", "powersupply", "internal_failure", + "Power supply internal failure status.", 1, label); + if (ctx->darwin_ps_internal_failure == NULL) { + flb_plg_error(ctx->ins, "failed to create gauge node_powersupply_internal_failure"); + return -1; + } + + ctx->darwin_ps_battery_health = cmt_gauge_create(ctx->cmt, "node", "powersupply", "battery_health", + "Power supply battery health status.", 2, + (char *[]) {"power_supply", "state"}); + if (ctx->darwin_ps_battery_health == NULL) { + flb_plg_error(ctx->ins, "failed to create gauge node_powersupply_battery_health"); + return -1; + } + + return 0; +} + +static int ne_powersupply_update(struct flb_input_instance *ins, + struct flb_config *config, void *in_context) +{ + struct flb_ne *ctx; + uint64_t ts; + CFTypeRef info; + CFArrayRef list; + CFTypeRef source; + CFIndex i; + int ret; + int has_failures; + + (void) ins; + (void) config; + + ctx = in_context; + ts = cfl_time_now(); + has_failures = FLB_FALSE; + + info = IOPSCopyPowerSourcesInfo(); + if (info == NULL) { + return -1; + } + + list = IOPSCopyPowerSourcesList(info); + if (list == NULL) { + CFRelease(info); + return -1; + } + + for (i = 0; i < CFArrayGetCount(list); i++) { + source = CFArrayGetValueAtIndex(list, i); + ret = update_power_source(ctx, source, info, ts); + if (ret != 0) { + flb_plg_debug(ctx->ins, + "failed to update power source metrics for source index=%ld ptr=%p", + i, source); + has_failures = FLB_TRUE; + continue; + } + } + + CFRelease(list); + CFRelease(info); + + if (has_failures == FLB_TRUE) { + return -1; + } + + return 0; +} + +struct flb_ne_collector powersupply_collector = { + .name = "powersupplyclass", + .cb_init = ne_powersupply_init, + .cb_update = ne_powersupply_update, + .cb_exit = NULL +}; diff --git a/plugins/in_node_exporter_metrics/ne_powersupply_linux.c b/plugins/in_node_exporter_metrics/ne_powersupply_linux.c new file mode 100644 index 00000000000..8d55d0897d8 --- /dev/null +++ b/plugins/in_node_exporter_metrics/ne_powersupply_linux.c @@ -0,0 +1,238 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2026 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include + +#include + +#include "ne.h" + +struct ps_metric_spec { + const char *metric_name; + const char *file_name; + double divisor; +}; + +struct ps_metric_entry { + char *name; + struct cmt_gauge *gauge; + struct mk_list _head; +}; + +static void ps_dynamic_metrics_destroy(struct flb_ne *ctx) +{ + struct mk_list *tmp; + struct mk_list *head; + struct ps_metric_entry *entry; + + mk_list_foreach_safe(head, tmp, &ctx->powersupply_dynamic_metrics) { + entry = mk_list_entry(head, struct ps_metric_entry, _head); + mk_list_del(&entry->_head); + if (entry->name != NULL) { + flb_free(entry->name); + } + flb_free(entry); + } +} + +static int ne_powersupply_exit(struct flb_ne *ctx) +{ + ps_dynamic_metrics_destroy(ctx); + return 0; +} + +static struct ps_metric_spec ps_numeric_metrics[] = { + {"authentic", "authentic", 1.0}, + {"calibrate", "calibrate", 1.0}, + {"capacity", "capacity", 1.0}, + {"capacity_alert_max", "capacity_alert_max", 1.0}, + {"capacity_alert_min", "capacity_alert_min", 1.0}, + {"cyclecount", "cycle_count", 1.0}, + {"online", "online", 1.0}, + {"present", "present", 1.0}, + {"time_to_empty_seconds", "time_to_empty_now", 1.0}, + {"time_to_full_seconds", "time_to_full_now", 1.0}, + {"current_boot", "current_boot", 1000000.0}, + {"current_max", "current_max", 1000000.0}, + {"current_ampere", "current_now", 1000000.0}, + {"energy_empty", "energy_empty", 1000000.0}, + {"energy_empty_design", "energy_empty_design", 1000000.0}, + {"energy_full", "energy_full", 1000000.0}, + {"energy_full_design", "energy_full_design", 1000000.0}, + {"energy_watthour", "energy_now", 1000000.0}, + {"voltage_boot", "voltage_boot", 1000000.0}, + {"voltage_max", "voltage_max", 1000000.0}, + {"voltage_max_design", "voltage_max_design", 1000000.0}, + {"voltage_min", "voltage_min", 1000000.0}, + {"voltage_min_design", "voltage_min_design", 1000000.0}, + {"voltage_volt", "voltage_now", 1000000.0}, + {"voltage_ocv", "voltage_ocv", 1000000.0}, + {"charge_control_limit", "charge_control_limit", 1000000.0}, + {"charge_control_limit_max", "charge_control_limit_max", 1000000.0}, + {"charge_counter", "charge_counter", 1000000.0}, + {"charge_empty", "charge_empty", 1000000.0}, + {"charge_empty_design", "charge_empty_design", 1000000.0}, + {"charge_full", "charge_full", 1000000.0}, + {"charge_full_design", "charge_full_design", 1000000.0}, + {"charge_ampere", "charge_now", 1000000.0}, + {"charge_term_current", "charge_term_current", 1000000.0}, + {"constant_charge_current", "constant_charge_current", 1000000.0}, + {"constant_charge_current_max", "constant_charge_current_max", 1000000.0}, + {"constant_charge_voltage", "constant_charge_voltage", 1000000.0}, + {"constant_charge_voltage_max", "constant_charge_voltage_max", 1000000.0}, + {"precharge_current", "precharge_current", 1000000.0}, + {"input_current_limit", "input_current_limit", 1000000.0}, + {"power_watt", "power_now", 1000000.0}, + {"temp_celsius", "temp", 10.0}, + {"temp_alert_max_celsius", "temp_alert_max", 10.0}, + {"temp_alert_min_celsius", "temp_alert_min", 10.0}, + {"temp_ambient_celsius", "temp_ambient", 10.0}, + {"temp_ambient_max_celsius", "temp_ambient_max", 10.0}, + {"temp_ambient_min_celsius", "temp_ambient_min", 10.0}, + {"temp_max_celsius", "temp_max", 10.0}, + {"temp_min_celsius", "temp_min", 10.0}, +}; + +static struct cmt_gauge *ps_metric_get(struct flb_ne *ctx, const char *name) +{ + struct mk_list *head; + struct ps_metric_entry *entry; + + mk_list_foreach(head, &ctx->powersupply_dynamic_metrics) { + entry = mk_list_entry(head, struct ps_metric_entry, _head); + if (strcmp(entry->name, name) == 0) { + return entry->gauge; + } + } + + entry = flb_calloc(1, sizeof(struct ps_metric_entry)); + if (entry == NULL) { + return NULL; + } + entry->name = flb_strdup(name); + if (entry->name == NULL) { + flb_free(entry); + return NULL; + } + + entry->gauge = cmt_gauge_create(ctx->cmt, "node", "powersupply", name, + "Power supply metric from /sys/class/power_supply.", + 1, (char *[]) {"power_supply"}); + if (entry->gauge == NULL) { + flb_free(entry->name); + flb_free(entry); + return NULL; + } + + mk_list_add(&entry->_head, &ctx->powersupply_dynamic_metrics); + return entry->gauge; +} + +static void emit_info_metric(struct flb_ne *ctx, const char *name, uint64_t ts) +{ + char *labels[] = {(char *) name}; + + (void) ctx; + cmt_gauge_set(ctx->powersupply_info, ts, 1.0, 1, labels); +} + +static int update_one(struct flb_ne *ctx, const char *name) +{ + size_t i; + long v; + uint64_t ts; + char path[1024]; + char *buf; + char *end; + size_t buf_size; + char *labels[] = {(char *) name}; + struct cmt_gauge *gauge; + + ts = cfl_time_now(); + + for (i = 0; i < sizeof(ps_numeric_metrics) / sizeof(ps_numeric_metrics[0]); i++) { + snprintf(path, sizeof(path) - 1, "%s/class/power_supply/%s/%s", + ctx->path_sysfs, name, ps_numeric_metrics[i].file_name); + + if (flb_utils_read_file(path, &buf, &buf_size) == 0 && buf != NULL && buf_size > 0) { + v = strtol(buf, &end, 10); + if (end == buf) { + flb_free(buf); + continue; + } + flb_free(buf); + + gauge = ps_metric_get(ctx, ps_numeric_metrics[i].metric_name); + if (gauge != NULL) { + cmt_gauge_set(gauge, ts, ((double) v) / ps_numeric_metrics[i].divisor, + 1, labels); + } + } + } + + emit_info_metric(ctx, name, ts); + return 0; +} + +static int ne_powersupply_init(struct flb_ne *ctx) +{ + mk_list_init(&ctx->powersupply_dynamic_metrics); + ctx->powersupply_info = cmt_gauge_create(ctx->cmt, "node", "powersupply", "info", + "info of /sys/class/power_supply/.", + 1, (char *[]) {"power_supply"}); + return 0; +} + +static int ne_powersupply_update(struct flb_input_instance *ins, + struct flb_config *config, void *in_context) +{ + DIR *dir; + struct dirent *ent; + struct flb_ne *ctx; + char path[1024]; + + (void) ins; + (void) config; + + ctx = in_context; + snprintf(path, sizeof(path) - 1, "%s/class/power_supply", ctx->path_sysfs); + dir = opendir(path); + if (dir == NULL) { + return -1; + } + + while ((ent = readdir(dir)) != NULL) { + if (ent->d_name[0] == '.') { + continue; + } + update_one(ctx, ent->d_name); + } + + closedir(dir); + return 0; +} + +struct flb_ne_collector powersupply_collector = { + .name = "powersupplyclass", + .cb_init = ne_powersupply_init, + .cb_update = ne_powersupply_update, + .cb_exit = ne_powersupply_exit +}; diff --git a/plugins/in_node_exporter_metrics/ne_utils.c b/plugins/in_node_exporter_metrics/ne_utils.c index 591c856b54f..543de19498e 100644 --- a/plugins/in_node_exporter_metrics/ne_utils.c +++ b/plugins/in_node_exporter_metrics/ne_utils.c @@ -20,7 +20,7 @@ #include #include #include -#include "ne.h" +#include "ne_utils.h" /* required by stat(2), open(2) */ #include @@ -30,6 +30,39 @@ #include +static void ne_utils_file_log(struct flb_ne *ctx, int log_level, int err_num, + const char *operation, const char *path) +{ + if (!ctx) { + errno = err_num; + flb_errno(); + return; + } + + if (err_num != ENOENT) { + log_level = FLB_LOG_ERROR; + } + + switch (log_level) { + case FLB_LOG_TRACE: + flb_plg_trace(ctx->ins, "could not %s '%s'", operation, path); + break; + case FLB_LOG_DEBUG: + flb_plg_debug(ctx->ins, "could not %s '%s'", operation, path); + break; + case FLB_LOG_INFO: + flb_plg_info(ctx->ins, "could not %s '%s'", operation, path); + break; + case FLB_LOG_WARN: + flb_plg_warn(ctx->ins, "could not %s '%s'", operation, path); + break; + case FLB_LOG_ERROR: + default: + flb_plg_error(ctx->ins, "could not %s '%s'", operation, path); + break; + } +} + int ne_utils_str_to_double(char *str, double *out_val) { double val; @@ -70,6 +103,18 @@ int ne_utils_file_read_uint64(struct flb_ne *ctx, const char *path, const char *join_a, const char *join_b, uint64_t *out_val) +{ + return ne_utils_file_read_uint64_at_level(ctx, mount, path, join_a, join_b, + out_val, FLB_LOG_ERROR); +} + +int ne_utils_file_read_uint64_at_level(struct flb_ne *ctx, + const char *mount, + const char *path, + const char *join_a, + const char *join_b, + uint64_t *out_val, + int log_level) { int fd; int len; @@ -123,24 +168,14 @@ int ne_utils_file_read_uint64(struct flb_ne *ctx, fd = open(p, O_RDONLY); if (fd == -1) { - if (ctx) { - flb_plg_error(ctx->ins, "could not open '%s'", p); - } - else { - flb_errno(); - } + ne_utils_file_log(ctx, log_level, errno, "open", p); flb_sds_destroy(p); return -1; } bytes = read(fd, &tmp, sizeof(tmp)); if (bytes == -1) { - if (ctx) { - flb_plg_error(ctx->ins, "could not read from '%s'", p); - } - else { - flb_errno(); - } + ne_utils_file_log(ctx, log_level, errno, "read from", p); close(fd); flb_sds_destroy(p); return -1; @@ -220,6 +255,18 @@ int ne_utils_file_read_sds(struct flb_ne *ctx, const char *join_a, const char *join_b, flb_sds_t *str) +{ + return ne_utils_file_read_sds_at_level(ctx, mount, path, join_a, join_b, + str, FLB_LOG_ERROR); +} + +int ne_utils_file_read_sds_at_level(struct flb_ne *ctx, + const char *mount, + const char *path, + const char *join_a, + const char *join_b, + flb_sds_t *str, + int log_level) { int fd; int len; @@ -269,24 +316,14 @@ int ne_utils_file_read_sds(struct flb_ne *ctx, fd = open(p, O_RDONLY); if (fd == -1) { - if (ctx) { - flb_plg_error(ctx->ins, "could not open '%s'", p); - } - else { - flb_errno(); - } + ne_utils_file_log(ctx, log_level, errno, "open", p); flb_sds_destroy(p); return -1; } bytes = read(fd, &tmp, sizeof(tmp)); if (bytes == -1) { - if (ctx) { - flb_plg_error(ctx->ins, "could not read from '%s'", p); - } - else { - flb_errno(); - } + ne_utils_file_log(ctx, log_level, errno, "read from", p); close(fd); flb_sds_destroy(p); return -1; diff --git a/plugins/in_node_exporter_metrics/ne_utils.h b/plugins/in_node_exporter_metrics/ne_utils.h index a365fbd4937..70089316e99 100644 --- a/plugins/in_node_exporter_metrics/ne_utils.h +++ b/plugins/in_node_exporter_metrics/ne_utils.h @@ -34,6 +34,14 @@ int ne_utils_file_read_uint64(struct flb_ne *ctx, const char *join_a, const char *join_b, uint64_t *out_val); +int ne_utils_file_read_uint64_at_level(struct flb_ne *ctx, + const char *mount, + const char *path, + const char *join_a, + const char *join_b, + uint64_t *out_val, + int log_level); + int ne_utils_file_read_sds(struct flb_ne *ctx, const char *mount, const char *path, @@ -41,6 +49,14 @@ int ne_utils_file_read_sds(struct flb_ne *ctx, const char *join_b, flb_sds_t *str); +int ne_utils_file_read_sds_at_level(struct flb_ne *ctx, + const char *mount, + const char *path, + const char *join_a, + const char *join_b, + flb_sds_t *str, + int log_level); + int ne_utils_file_read_lines(struct flb_ne *ctx, const char *mount, const char *path, diff --git a/plugins/in_opentelemetry/opentelemetry.h b/plugins/in_opentelemetry/opentelemetry.h index 5d927671e6a..c82d60b1804 100644 --- a/plugins/in_opentelemetry/opentelemetry.h +++ b/plugins/in_opentelemetry/opentelemetry.h @@ -69,6 +69,7 @@ static inline int opentelemetry_ingest_logs(struct flb_opentelemetry *ctx, } static inline int opentelemetry_ingest_logs_take(struct flb_opentelemetry *ctx, + size_t records, const char *tag, size_t tag_len, void *buf, @@ -76,12 +77,18 @@ static inline int opentelemetry_ingest_logs_take(struct flb_opentelemetry *ctx, size_t allocation_size) { if (opentelemetry_uses_worker_ingress_queue(ctx)) { - return flb_input_ingress_queue_log_take(ctx->ins, - tag, - tag_len, - buf, - buf_size, - allocation_size); + return flb_input_ingress_queue_log_take_records(ctx->ins, + records, + tag, + tag_len, + buf, + buf_size, + allocation_size); + } + + if (records > 0) { + return flb_input_log_append_records(ctx->ins, records, + tag, tag_len, buf, buf_size); } return flb_input_log_append(ctx->ins, tag, tag_len, buf, buf_size); @@ -90,10 +97,12 @@ static inline int opentelemetry_ingest_logs_take(struct flb_opentelemetry *ctx, static inline int opentelemetry_ingest_metrics(struct flb_opentelemetry *ctx, const char *tag, size_t tag_len, - struct cmt *cmt) + struct cmt *cmt, + size_t payload_size) { if (opentelemetry_uses_worker_ingress_queue(ctx)) { - return flb_input_ingress_queue_metrics(ctx->ins, tag, tag_len, cmt); + return flb_input_ingress_queue_metrics(ctx->ins, tag, tag_len, + cmt, payload_size); } return flb_input_metrics_append(ctx->ins, tag, tag_len, cmt); @@ -102,10 +111,12 @@ static inline int opentelemetry_ingest_metrics(struct flb_opentelemetry *ctx, static inline int opentelemetry_ingest_traces(struct flb_opentelemetry *ctx, const char *tag, size_t tag_len, - struct ctrace *ctr) + struct ctrace *ctr, + size_t payload_size) { if (opentelemetry_uses_worker_ingress_queue(ctx)) { - return flb_input_ingress_queue_traces(ctx->ins, tag, tag_len, ctr); + return flb_input_ingress_queue_traces(ctx->ins, tag, tag_len, + ctr, payload_size); } return flb_input_trace_append(ctx->ins, tag, tag_len, ctr); @@ -114,10 +125,12 @@ static inline int opentelemetry_ingest_traces(struct flb_opentelemetry *ctx, static inline int opentelemetry_ingest_profiles(struct flb_opentelemetry *ctx, const char *tag, size_t tag_len, - struct cprof *profile) + struct cprof *profile, + size_t payload_size) { if (opentelemetry_uses_worker_ingress_queue(ctx)) { - return flb_input_ingress_queue_profiles(ctx->ins, tag, tag_len, profile); + return flb_input_ingress_queue_profiles(ctx->ins, tag, tag_len, + profile, payload_size); } return flb_input_profiles_append(ctx->ins, tag, tag_len, profile); diff --git a/plugins/in_opentelemetry/opentelemetry_logs.c b/plugins/in_opentelemetry/opentelemetry_logs.c index 9324dd1f55a..7fd5a40e288 100644 --- a/plugins/in_opentelemetry/opentelemetry_logs.c +++ b/plugins/in_opentelemetry/opentelemetry_logs.c @@ -18,6 +18,7 @@ */ #include +#include #include #include #include @@ -25,10 +26,80 @@ #include #include +#include #include "opentelemetry.h" #include "opentelemetry_utils.h" +/* + * Small requests use protobuf-c's default allocator to avoid reserving an + * arena chunk. Realistic batched requests use the arena to amortize the large + * number of short-lived allocations performed by protobuf-c. + */ +#define OTEL_PROTOBUF_ARENA_MIN_PAYLOAD_SIZE 8192 +#define OTEL_PROTOBUF_ARENA_INITIAL_CHUNK_SIZE 4096 +#define OTEL_PROTOBUF_ARENA_MAX_CHUNK_SIZE 65536 + +static void *protobuf_arena_chunk_malloc(void *context, size_t size) +{ + (void) context; + + return flb_malloc(size); +} + +static void protobuf_arena_chunk_free(void *context, void *pointer) +{ + (void) context; + + flb_free(pointer); +} + +static void *protobuf_arena_alloc(void *allocator_data, size_t size) +{ + struct cfl_arena *arena; + + arena = allocator_data; + if (size == 0) { + size = 1; + } + + return cfl_arena_malloc(arena, size); +} + +static void protobuf_arena_free(void *allocator_data, void *pointer) +{ + (void) allocator_data; + (void) pointer; +} + +static struct cfl_arena *protobuf_arena_create(void) +{ + struct cfl_arena_options options; + + cfl_arena_options_init(&options); + options.chunk_size = OTEL_PROTOBUF_ARENA_INITIAL_CHUNK_SIZE; + options.maximum_chunk_size = OTEL_PROTOBUF_ARENA_MAX_CHUNK_SIZE; + options.malloc_fn = protobuf_arena_chunk_malloc; + options.free_fn = protobuf_arena_chunk_free; + + return cfl_arena_create_with_options(&options); +} + +static Opentelemetry__Proto__Collector__Logs__V1__ExportLogsServiceRequest * +protobuf_logs_unpack(ProtobufCAllocator *allocator, size_t size, const uint8_t *data) +{ + return opentelemetry__proto__collector__logs__v1__export_logs_service_request__unpack( + allocator, size, data); +} + +static void protobuf_logs_free( + Opentelemetry__Proto__Collector__Logs__V1__ExportLogsServiceRequest *logs, + ProtobufCAllocator *allocator) +{ + opentelemetry__proto__collector__logs__v1__export_logs_service_request__free_unpacked( + logs, allocator); +} + /* * OTLP encoding functions to pack the log records as msgpack * ---------------------------------------------------------- @@ -318,7 +389,8 @@ static int binary_payload_to_msgpack(struct flb_opentelemetry *ctx, struct flb_log_event_encoder *encoder, char *tag, size_t tag_len, uint8_t *in_buf, - size_t in_size) + size_t in_size, + size_t *record_count) { int ret = 0; int len; @@ -327,9 +399,12 @@ static int binary_payload_to_msgpack(struct flb_opentelemetry *ctx, int log_record_index; char *logs_body_key; int scope_has_schema_url; + struct cfl_arena *protobuf_arena; struct flb_mp_map_header mh; struct flb_mp_map_header mh_tmp; struct flb_time tm; + ProtobufCAllocator arena_allocator; + ProtobufCAllocator *protobuf_allocator; msgpack_packer *mp_pck; msgpack_packer *mp_pck_meta; @@ -347,9 +422,23 @@ static int binary_payload_to_msgpack(struct flb_opentelemetry *ctx, mp_pck = &encoder->body.packer; mp_pck_meta = &encoder->metadata.packer; + input_logs = NULL; + protobuf_arena = NULL; + protobuf_allocator = NULL; + *record_count = 0; + + if (in_size >= OTEL_PROTOBUF_ARENA_MIN_PAYLOAD_SIZE) { + protobuf_arena = protobuf_arena_create(); + if (protobuf_arena != NULL) { + arena_allocator.alloc = protobuf_arena_alloc; + arena_allocator.free = protobuf_arena_free; + arena_allocator.allocator_data = protobuf_arena; + protobuf_allocator = &arena_allocator; + } + } /* unpack logs from protobuf payload */ - input_logs = opentelemetry__proto__collector__logs__v1__export_logs_service_request__unpack(NULL, in_size, in_buf); + input_logs = protobuf_logs_unpack(protobuf_allocator, in_size, in_buf); if (input_logs == NULL) { flb_plg_warn(ctx->ins, "failed to unpack input logs from OpenTelemetry payload"); ret = -1; @@ -567,12 +656,20 @@ static int binary_payload_to_msgpack(struct flb_opentelemetry *ctx, if (ret == FLB_EVENT_ENCODER_SUCCESS) { if (log_records[log_record_index]->time_unix_nano > 0) { - flb_time_from_uint64(&tm, log_records[log_record_index]->time_unix_nano); - ret = flb_log_event_encoder_set_timestamp(encoder, &tm); + ret = flb_time_from_uint64( + &tm, + log_records[log_record_index]->time_unix_nano); + if (ret == 0) { + ret = flb_log_event_encoder_set_timestamp(encoder, &tm); + } } else if (log_records[log_record_index]->observed_time_unix_nano > 0) { - flb_time_from_uint64(&tm, log_records[log_record_index]->observed_time_unix_nano); - ret = flb_log_event_encoder_set_timestamp(encoder, &tm); + ret = flb_time_from_uint64( + &tm, + log_records[log_record_index]->observed_time_unix_nano); + if (ret == 0) { + ret = flb_log_event_encoder_set_timestamp(encoder, &tm); + } } else { flb_time_get(&tm); @@ -651,6 +748,9 @@ static int binary_payload_to_msgpack(struct flb_opentelemetry *ctx, if (ret == FLB_EVENT_ENCODER_SUCCESS) { ret = flb_log_event_encoder_commit_record(encoder); + if (ret == FLB_EVENT_ENCODER_SUCCESS) { + (*record_count)++; + } } else { flb_plg_error(ctx->ins, "marshalling error"); @@ -665,9 +765,9 @@ static int binary_payload_to_msgpack(struct flb_opentelemetry *ctx, binary_payload_to_msgpack_end: if (input_logs) { - opentelemetry__proto__collector__logs__v1__export_logs_service_request__free_unpacked( - input_logs, NULL); + protobuf_logs_free(input_logs, protobuf_allocator); } + cfl_arena_destroy(protobuf_arena); if (ret != 0) { return -1; @@ -692,11 +792,13 @@ int opentelemetry_process_logs(struct flb_opentelemetry *ctx, char *buf; uint8_t *payload; uint64_t payload_size; + size_t record_count; struct flb_log_event_encoder *encoder; buf = (char *) data; payload = data; payload_size = size; + record_count = 0; /* Detect the type of payload */ if (content_type) { @@ -724,7 +826,8 @@ int opentelemetry_process_logs(struct flb_opentelemetry *ctx, if (is_proto == FLB_TRUE) { ret = binary_payload_to_msgpack(ctx, encoder, tag, tag_len, - (uint8_t *) payload, payload_size); + (uint8_t *) payload, payload_size, + &record_count); if (ret < 0) { flb_plg_error(ctx->ins, "failed to process logs from protobuf payload"); } @@ -762,14 +865,13 @@ int opentelemetry_process_logs(struct flb_opentelemetry *ctx, } ret = opentelemetry_ingest_logs_take(ctx, + record_count, tag, flb_sds_len(tag), encoder->output_buffer, encoder->output_length, allocation_size); - if (ret == 0 || ret == FLB_INPUT_INGRESS_BUSY) { - flb_log_event_encoder_claim_internal_buffer_ownership(encoder); - } + flb_log_event_encoder_claim_internal_buffer_ownership(encoder); } else { ret = opentelemetry_ingest_logs(ctx, diff --git a/plugins/in_opentelemetry/opentelemetry_prot.c b/plugins/in_opentelemetry/opentelemetry_prot.c index 3084a1ad656..030bb7cf152 100644 --- a/plugins/in_opentelemetry/opentelemetry_prot.c +++ b/plugins/in_opentelemetry/opentelemetry_prot.c @@ -252,6 +252,9 @@ static int send_response_ng(struct flb_http_response *response, else if (http_status == 400) { flb_http_response_set_message(response, "Bad Request"); } + else if (http_status == 404) { + flb_http_response_set_message(response, "Not Found"); + } if (message != NULL) { flb_http_response_set_body(response, @@ -534,26 +537,38 @@ static int process_payload_metrics_ng(struct flb_opentelemetry *ctx, } if (result == CMT_DECODE_OPENTELEMETRY_SUCCESS) { + if (opentelemetry_uses_worker_ingress_queue(ctx)) { + result = flb_input_ingress_queue_metrics_list(ctx->ins, + tag, + cfl_sds_len(tag), + &decoded_contexts, + payload_size); + if (result == FLB_INPUT_INGRESS_BUSY) { + return FLB_INPUT_INGRESS_BUSY; + } + else if (result != 0) { + flb_plg_debug(ctx->ins, + "could not queue metrics contexts : %d", result); + return -1; + } + + return 0; + } + cfl_list_foreach_safe(iterator, tmp, &decoded_contexts) { context = cfl_list_entry(iterator, struct cmt, _head); - if (opentelemetry_uses_worker_ingress_queue(ctx)) { - cfl_list_del(&context->_head); - } - result = opentelemetry_ingest_metrics(ctx, tag, cfl_sds_len(tag), - context); + context, + payload_size); if (result == FLB_INPUT_INGRESS_BUSY) { cmt_decode_opentelemetry_destroy(&decoded_contexts); return FLB_INPUT_INGRESS_BUSY; } else if (result != 0) { - if (opentelemetry_uses_worker_ingress_queue(ctx)) { - cmt_destroy(context); - } flb_plg_debug(ctx->ins, "could not ingest metrics context : %d", result); } } @@ -636,14 +651,13 @@ static int ingest_profiles_context_as_log_entry(struct flb_opentelemetry *ctx, } ret = opentelemetry_ingest_logs_take(ctx, + 0, tag, flb_sds_len(tag), encoder->output_buffer, encoder->output_length, allocation_size); - if (ret == 0 || ret == FLB_INPUT_INGRESS_BUSY) { - flb_log_event_encoder_claim_internal_buffer_ownership(encoder); - } + flb_log_event_encoder_claim_internal_buffer_ownership(encoder); } else { ret = opentelemetry_ingest_logs(ctx, @@ -714,10 +728,10 @@ static int process_payload_profiles_ng(struct flb_opentelemetry *ctx, ret = opentelemetry_ingest_profiles(ctx, tag, flb_sds_len(tag), - profiles_context); + profiles_context, + payload_size); - if (ret != FLB_INPUT_INGRESS_BUSY && - (ret != 0 || !opentelemetry_uses_worker_ingress_queue(ctx))) { + if (!opentelemetry_uses_worker_ingress_queue(ctx)) { cprof_decode_opentelemetry_destroy(profiles_context); } } @@ -828,7 +842,7 @@ int opentelemetry_prot_handle_ng(struct flb_http_request *request, grpc_request = FLB_TRUE; } else { - send_response_ng(response, 400, "error: invalid endpoint\n"); + send_response_ng(response, 404, "error: invalid endpoint\n"); return -1; } diff --git a/plugins/in_opentelemetry/opentelemetry_traces.c b/plugins/in_opentelemetry/opentelemetry_traces.c index 27ca37b2153..4d4ba343be9 100644 --- a/plugins/in_opentelemetry/opentelemetry_traces.c +++ b/plugins/in_opentelemetry/opentelemetry_traces.c @@ -43,8 +43,10 @@ int opentelemetry_traces_process_protobuf(struct flb_opentelemetry *ctx, data, size, &offset); if (result == 0) { - result = opentelemetry_ingest_traces(ctx, tag, tag_len, decoded_context); - if (result == -1) { + result = opentelemetry_ingest_traces(ctx, tag, tag_len, + decoded_context, size); + if (result == -1 && + !opentelemetry_uses_worker_ingress_queue(ctx)) { ctr_destroy(decoded_context); } } @@ -67,8 +69,9 @@ static int process_json(struct flb_opentelemetry *ctx, /* Use the new centralized API for JSON to ctrace conversion */ ctr = flb_opentelemetry_json_traces_to_ctrace(body, len, &error_status); if (ctr) { - result = opentelemetry_ingest_traces(ctx, tag, tag_len, ctr); - if (result == -1) { + result = opentelemetry_ingest_traces(ctx, tag, tag_len, ctr, len); + if (result == -1 && + !opentelemetry_uses_worker_ingress_queue(ctx)) { ctr_destroy(ctr); } } diff --git a/plugins/in_podman_metrics/podman_metrics.c b/plugins/in_podman_metrics/podman_metrics.c index 342649bf624..b6c56c6934d 100644 --- a/plugins/in_podman_metrics/podman_metrics.c +++ b/plugins/in_podman_metrics/podman_metrics.c @@ -158,6 +158,10 @@ static int add_container_to_list(struct flb_in_metrics *ctx, flb_sds_t id, flb_s cnt->rss = UINT64_MAX; cnt->cpu_user = UINT64_MAX; cnt->cpu = UINT64_MAX; + cnt->disk_read_bytes = UINT64_MAX; + cnt->disk_write_bytes = UINT64_MAX; + cnt->disk_reads = UINT64_MAX; + cnt->disk_writes = UINT64_MAX; mk_list_init(&cnt->net_data); @@ -303,6 +307,10 @@ static int create_gauge(struct flb_in_metrics *ctx, struct cmt_gauge **gauge, fl * - container_network_receive_errors_total * - container_network_transmit_bytes_total * - container_network_transmit_errors_total + * - container_disk_read_bytes_total + * - container_disk_write_bytes_total + * - container_disk_reads_total + * - container_disk_writes_total */ static int create_counters(struct flb_in_metrics *ctx) { @@ -328,6 +336,14 @@ static int create_counters(struct flb_in_metrics *ctx) DESCRIPTION_CPU_USER, NULL, cnt->cpu_user); create_counter(ctx, &ctx->c_cpu, cnt->id, cnt->name, cnt->image_name, COUNTER_CPU_PREFIX, FIELDS_METRIC, COUNTER_CPU, DESCRIPTION_CPU, NULL, cnt->cpu); + create_counter(ctx, &ctx->c_disk_read_bytes, cnt->id, cnt->name, cnt->image_name, COUNTER_DISK_PREFIX, FIELDS_METRIC, COUNTER_DISK_READ_BYTES, + DESCRIPTION_DISK_READ_BYTES, NULL, cnt->disk_read_bytes); + create_counter(ctx, &ctx->c_disk_write_bytes, cnt->id, cnt->name, cnt->image_name, COUNTER_DISK_PREFIX, FIELDS_METRIC, COUNTER_DISK_WRITE_BYTES, + DESCRIPTION_DISK_WRITE_BYTES, NULL, cnt->disk_write_bytes); + create_counter(ctx, &ctx->c_disk_reads, cnt->id, cnt->name, cnt->image_name, COUNTER_DISK_PREFIX, FIELDS_METRIC, COUNTER_DISK_READS, + DESCRIPTION_DISK_READS, NULL, cnt->disk_reads); + create_counter(ctx, &ctx->c_disk_writes, cnt->id, cnt->name, cnt->image_name, COUNTER_DISK_PREFIX, FIELDS_METRIC, COUNTER_DISK_WRITES, + DESCRIPTION_DISK_WRITES, NULL, cnt->disk_writes); mk_list_foreach_safe(inner_head, inner_tmp, &cnt->net_data) { iface = mk_list_entry(inner_head, struct net_iface, _head); @@ -423,6 +439,10 @@ static int in_metrics_init(struct flb_input_instance *in, struct flb_config *con ctx->c_memory_limit = NULL; ctx->c_cpu_user = NULL; ctx->c_cpu = NULL; + ctx->c_disk_read_bytes = NULL; + ctx->c_disk_write_bytes = NULL; + ctx->c_disk_reads = NULL; + ctx->c_disk_writes = NULL; ctx->rx_bytes = NULL; ctx->rx_errors = NULL; ctx->tx_bytes = NULL; diff --git a/plugins/in_podman_metrics/podman_metrics_config.h b/plugins/in_podman_metrics/podman_metrics_config.h index 1f6133e199a..4012d5fa4e0 100644 --- a/plugins/in_podman_metrics/podman_metrics_config.h +++ b/plugins/in_podman_metrics/podman_metrics_config.h @@ -86,6 +86,12 @@ #define STAT_KEY_CPU "usage_usec" #define STAT_KEY_CPU_USER "user_usec" +/* Field tokens in cgroups v2 io.stat (per block device, summed across devices) */ +#define IO_STAT_KEY_READ_BYTES "rbytes=" +#define IO_STAT_KEY_WRITE_BYTES "wbytes=" +#define IO_STAT_KEY_READS "rios=" +#define IO_STAT_KEY_WRITES "wios=" + /* Static lists of fields in counters or gauges */ #define FIELDS_METRIC (char*[3]){"id", "name", "image" } #define FIELDS_METRIC_WITH_IFACE (char*[4]){"id", "name", "image", "interface" } @@ -107,6 +113,7 @@ #define V2_SYSFS_FILE_CPU_STAT "cpu.stat" #define V2_SYSFS_FILE_PIDS "cgroup.procs" #define V2_SYSFS_FILE_PIDS_ALT "containers/cgroup.procs" +#define V2_SYSFS_FILE_IO_STAT "io.stat" /* Values used to construct counters/gauges names and descriptions */ #define COUNTER_PREFIX "container" @@ -138,6 +145,16 @@ #define COUNTER_TX_ERRORS "transmit_errors_total" #define DESCRIPTION_TX_ERRORS "Network transmitedd errors" +#define COUNTER_DISK_PREFIX "disk" +#define COUNTER_DISK_READ_BYTES "read_bytes_total" +#define DESCRIPTION_DISK_READ_BYTES "Container block I/O bytes read" +#define COUNTER_DISK_WRITE_BYTES "write_bytes_total" +#define DESCRIPTION_DISK_WRITE_BYTES "Container block I/O bytes written" +#define COUNTER_DISK_READS "reads_total" +#define DESCRIPTION_DISK_READS "Container block I/O reads completed" +#define COUNTER_DISK_WRITES "writes_total" +#define DESCRIPTION_DISK_WRITES "Container block I/O writes completed" + struct net_iface { flb_sds_t name; @@ -160,6 +177,10 @@ struct container { uint64_t cpu; uint64_t cpu_user; uint64_t rss; + uint64_t disk_read_bytes; + uint64_t disk_write_bytes; + uint64_t disk_reads; + uint64_t disk_writes; struct mk_list net_data; }; @@ -192,6 +213,10 @@ struct flb_in_metrics { struct cmt_counter *rx_errors; struct cmt_counter *tx_bytes; struct cmt_counter *tx_errors; + struct cmt_counter *c_disk_read_bytes; + struct cmt_counter *c_disk_write_bytes; + struct cmt_counter *c_disk_reads; + struct cmt_counter *c_disk_writes; /* cgroup version used by host */ int cgroup_version; diff --git a/plugins/in_podman_metrics/podman_metrics_data.c b/plugins/in_podman_metrics/podman_metrics_data.c index 28771cf5a06..6f4346862e3 100644 --- a/plugins/in_podman_metrics/podman_metrics_data.c +++ b/plugins/in_podman_metrics/podman_metrics_data.c @@ -132,6 +132,73 @@ uint64_t get_data_from_sysfs(struct flb_in_metrics *ctx, flb_sds_t dir, flb_sds_ return data; } +/* + * Read all cgroups v2 io.stat counters in a single pass and store them in cnt. + * io.stat lines look like: + * "8:0 rbytes=1024 wbytes=0 rios=2 wios=0 dbytes=0 dios=0" + * The rbytes/wbytes/rios/wios fields are summed across all block devices. On a + * missing or unreadable file (for example cgroups v1, where io.stat does not + * exist) the four counters are set to UINT64_MAX so they are treated as invalid + * and skipped, mirroring the other sysfs readers. An existing but empty io.stat + * (no I/O yet) yields 0 for each counter. + */ +void read_io_stat(struct flb_in_metrics *ctx, flb_sds_t dir, flb_sds_t name, struct container *cnt) +{ + char path[SYSFS_FILE_PATH_SIZE]; + FILE *fp; + char *line = NULL; + char *pos; + size_t len = 0; + int i; + struct { + const char *key; + size_t key_len; + uint64_t *total; + } fields[] = { + { IO_STAT_KEY_READ_BYTES, sizeof(IO_STAT_KEY_READ_BYTES) - 1, &cnt->disk_read_bytes }, + { IO_STAT_KEY_WRITE_BYTES, sizeof(IO_STAT_KEY_WRITE_BYTES) - 1, &cnt->disk_write_bytes }, + { IO_STAT_KEY_READS, sizeof(IO_STAT_KEY_READS) - 1, &cnt->disk_reads }, + { IO_STAT_KEY_WRITES, sizeof(IO_STAT_KEY_WRITES) - 1, &cnt->disk_writes }, + }; + + cnt->disk_read_bytes = UINT64_MAX; + cnt->disk_write_bytes = UINT64_MAX; + cnt->disk_reads = UINT64_MAX; + cnt->disk_writes = UINT64_MAX; + + if (dir == NULL) { + return; + } + + snprintf(path, sizeof(path), "%s/%s", dir, name); + + fp = fopen(path, "r"); + if (!fp) { + flb_plg_warn(ctx->ins, "Failed to read %s", path); + return; + } + + for (i = 0; i < 4; i++) { + *fields[i].total = 0; + } + + while (getline(&line, &len, fp) != -1) { + for (i = 0; i < 4; i++) { + pos = line; + while ((pos = strstr(pos, fields[i].key)) != NULL) { + pos += fields[i].key_len; + *fields[i].total += strtoull(pos, NULL, 10); + } + } + } + flb_free(line); + fclose(fp); + + flb_plg_debug(ctx->ins, "%s: rbytes=%lu wbytes=%lu rios=%lu wios=%lu", path, + cnt->disk_read_bytes, cnt->disk_write_bytes, + cnt->disk_reads, cnt->disk_writes); +} + /* * Check if container sysfs data is pressent in previously generated list of sysfs directories. * For cgroups v1, use subsystem (directory, for example memory) to search full path. @@ -367,6 +434,7 @@ int fill_counters_with_sysfs_data_v2(struct flb_in_metrics *ctx) cnt->memory_limit = get_data_from_sysfs(ctx, path, V2_SYSFS_FILE_MEMORY_LIMIT, NULL); cnt->cpu_user = get_data_from_sysfs(ctx, path, V2_SYSFS_FILE_CPU_STAT, STAT_KEY_CPU_USER); cnt->cpu = get_data_from_sysfs(ctx, path, V2_SYSFS_FILE_CPU_STAT, STAT_KEY_CPU); + read_io_stat(ctx, path, V2_SYSFS_FILE_IO_STAT, cnt); pid = get_data_from_sysfs(ctx, path, V2_SYSFS_FILE_PIDS, NULL); if (!pid || pid == UINT64_MAX) { pid = get_data_from_sysfs(ctx, path, V2_SYSFS_FILE_PIDS_ALT, NULL); diff --git a/plugins/in_podman_metrics/podman_metrics_data.h b/plugins/in_podman_metrics/podman_metrics_data.h index f7eb403bbea..3fb3521cdcc 100644 --- a/plugins/in_podman_metrics/podman_metrics_data.h +++ b/plugins/in_podman_metrics/podman_metrics_data.h @@ -37,6 +37,7 @@ int destroy_gauge(struct flb_in_metrics *ctx, struct cmt_gauge **g); uint64_t read_from_file(struct flb_in_metrics *ctx, flb_sds_t path); uint64_t read_key_value_from_file(struct flb_in_metrics *ctx, flb_sds_t path, flb_sds_t key); uint64_t get_data_from_sysfs(struct flb_in_metrics *ctx, flb_sds_t dir, flb_sds_t name, flb_sds_t key); +void read_io_stat(struct flb_in_metrics *ctx, flb_sds_t dir, flb_sds_t name, struct container *cnt); int get_container_sysfs_subdirectory(struct flb_in_metrics *ctx, flb_sds_t id, flb_sds_t subsystem, flb_sds_t *path); int get_net_data_from_proc(struct flb_in_metrics *ctx, struct container *cnt, uint64_t pid); diff --git a/plugins/in_process_exporter_metrics/pe.h b/plugins/in_process_exporter_metrics/pe.h index 137c1144676..345f178ad6f 100644 --- a/plugins/in_process_exporter_metrics/pe.h +++ b/plugins/in_process_exporter_metrics/pe.h @@ -46,6 +46,7 @@ struct flb_pe { /* configuration */ flb_sds_t path_procfs; int scrape_interval; + int page_size; int coll_fd; /* collector fd */ struct cmt *cmt; /* cmetrics context */ diff --git a/plugins/in_process_exporter_metrics/pe_config.c b/plugins/in_process_exporter_metrics/pe_config.c index b166b9748be..1d3f7195600 100644 --- a/plugins/in_process_exporter_metrics/pe_config.c +++ b/plugins/in_process_exporter_metrics/pe_config.c @@ -18,6 +18,9 @@ */ #include + +#include + #include "pe.h" struct flb_pe *flb_pe_config_create(struct flb_input_instance *ins, @@ -37,6 +40,9 @@ struct flb_pe *flb_pe_config_create(struct flb_input_instance *ins, ctx->process_regex_include_list = NULL; ctx->process_regex_exclude_list = NULL; + /* Cache the system page size for procfs RSS pages-to-bytes conversion. */ + ctx->page_size = sysconf(_SC_PAGESIZE); + /* Load the config map */ ret = flb_input_config_map_set(ins, (void *) ctx); if (ret == -1) { diff --git a/plugins/in_process_exporter_metrics/pe_process.c b/plugins/in_process_exporter_metrics/pe_process.c index 904d3e3ab47..d6e024837d3 100644 --- a/plugins/in_process_exporter_metrics/pe_process.c +++ b/plugins/in_process_exporter_metrics/pe_process.c @@ -1096,19 +1096,19 @@ static int process_update(struct flb_pe *ctx) /* Memory */ if (ctx->enabled_flag & METRIC_MEMORY) { - /* Memory Size */ entry = flb_slist_entry_get(&split_list, 20); tmp = entry->str; - /* Collect the number of Virtual Memory per process */ + /* Collect the number of Virtual Memory per process. This is already being reported in bytes*/ if (pe_utils_str_to_uint64(tmp, &val) != -1) { cmt_gauge_set(ctx->memory_bytes, ts, val, 4, (char *[]){ name, pid_str, ppid_str, "virtual_memory" }); } entry = flb_slist_entry_get(&split_list, 21); tmp = entry->str; - /* Collect the number of RSS per process */ - if (pe_utils_str_to_uint64(tmp, &val) != -1) { - cmt_gauge_set(ctx->memory_bytes, ts, val, 4, (char *[]){ name, pid_str, ppid_str, "rss" }); + /* The metrics reports the number of pages. So we have to convert by also using the size of the page */ + if (ctx->page_size > 0 && pe_utils_str_to_uint64(tmp, &val) != -1) { + cmt_gauge_set(ctx->memory_bytes, ts, val * ctx->page_size, + 4, (char *[]){ name, pid_str, ppid_str, "rss" }); } /* Major Page Faults */ @@ -1175,6 +1175,13 @@ static int process_update(struct flb_pe *ctx) flb_slist_destroy(&procfs_list); + /* + * Purge metric label-sets for processes that were not seen in this + * collection pass. Any cmt_metric whose timestamp is older than ts + * (captured before the scan) belongs to a process that no longer exists. + */ + cmt_expire(ctx->cmt, ts); + return 0; } diff --git a/plugins/in_prometheus_remote_write/prom_rw.h b/plugins/in_prometheus_remote_write/prom_rw.h index e476fd6b10d..910cf7ca9f8 100644 --- a/plugins/in_prometheus_remote_write/prom_rw.h +++ b/plugins/in_prometheus_remote_write/prom_rw.h @@ -52,10 +52,12 @@ static inline int prom_rw_uses_worker_ingress_queue( static inline int prom_rw_ingest_metrics(struct flb_prom_remote_write *ctx, const char *tag, size_t tag_len, - struct cmt *cmt) + struct cmt *cmt, + size_t payload_size) { if (prom_rw_uses_worker_ingress_queue(ctx)) { - return flb_input_ingress_queue_metrics(ctx->ins, tag, tag_len, cmt); + return flb_input_ingress_queue_metrics(ctx->ins, tag, tag_len, + cmt, payload_size); } return flb_input_metrics_append(ctx->ins, tag, tag_len, cmt); diff --git a/plugins/in_prometheus_remote_write/prom_rw_prot.c b/plugins/in_prometheus_remote_write/prom_rw_prot.c index 3f5e13eac64..e1c7631efe1 100644 --- a/plugins/in_prometheus_remote_write/prom_rw_prot.c +++ b/plugins/in_prometheus_remote_write/prom_rw_prot.c @@ -74,14 +74,10 @@ static int process_payload_metrics_ng(struct flb_prom_remote_write *ctx, return 400; } - result = prom_rw_ingest_metrics(ctx, NULL, 0, context); + result = prom_rw_ingest_metrics(ctx, NULL, 0, context, + cfl_sds_len(request->body)); - if (prom_rw_uses_worker_ingress_queue(ctx)) { - if (result != 0 && result != FLB_INPUT_INGRESS_BUSY) { - cmt_decode_prometheus_remote_write_destroy(context); - } - } - else { + if (!prom_rw_uses_worker_ingress_queue(ctx)) { cmt_decode_prometheus_remote_write_destroy(context); } diff --git a/plugins/in_syslog/syslog_conn.c b/plugins/in_syslog/syslog_conn.c index 0aecbebb3e2..08559512bda 100644 --- a/plugins/in_syslog/syslog_conn.c +++ b/plugins/in_syslog/syslog_conn.c @@ -97,6 +97,10 @@ int syslog_stream_conn_event(void *data) (void *) &conn->buf_data[conn->buf_len], available); + if (flb_io_net_is_retry(bytes)) { + return 0; + } + if (bytes > 0) { flb_plg_trace(ctx->ins, "read()=%i pre_len=%zu now_len=%zu", bytes, conn->buf_len, conn->buf_len + bytes); diff --git a/plugins/in_tail/CMakeLists.txt b/plugins/in_tail/CMakeLists.txt index 31d865218d5..36eca1ce2fd 100644 --- a/plugins/in_tail/CMakeLists.txt +++ b/plugins/in_tail/CMakeLists.txt @@ -28,6 +28,7 @@ endif() if(MSVC) set(src ${src} + win32/path.c win32/stat.c win32/io.c ) diff --git a/plugins/in_tail/tail.c b/plugins/in_tail/tail.c index 40373f5c184..5818f38f0ce 100644 --- a/plugins/in_tail/tail.c +++ b/plugins/in_tail/tail.c @@ -736,6 +736,15 @@ static struct flb_config_map config_map[] = { "set to false to use file stat watcher instead of inotify." }, #endif +#ifdef FLB_SYSTEM_WINDOWS + { + FLB_CONFIG_MAP_STR, "windows.path_encoding", "ansi", + 0, FLB_FALSE, 0, + "Windows-only path encoding mode. Use 'utf-8' to treat paths as UTF-8 " + "and call Windows wide-character file APIs. The default 'ansi' mode " + "keeps the legacy active ANSI code page behavior." + }, +#endif #ifdef FLB_HAVE_REGEX { FLB_CONFIG_MAP_STR, "parser", NULL, diff --git a/plugins/in_tail/tail_config.c b/plugins/in_tail/tail_config.c index f6f28dceab0..3f325989615 100644 --- a/plugins/in_tail/tail_config.c +++ b/plugins/in_tail/tail_config.c @@ -109,6 +109,9 @@ struct flb_tail_config *flb_tail_config_create(struct flb_input_instance *ins, #ifdef FLB_HAVE_SQLDB ctx->db_sync = 1; /* sqlite sync 'normal' */ #endif +#ifdef FLB_SYSTEM_WINDOWS + ctx->windows_path_encoding = FLB_TAIL_WINDOWS_PATH_ENCODING_ANSI; +#endif #ifdef FLB_HAVE_UNICODE_ENCODER ctx->preferred_input_encoding = FLB_UNICODE_ENCODING_UNSPECIFIED; #endif @@ -196,6 +199,24 @@ struct flb_tail_config *flb_tail_config_create(struct flb_input_instance *ins, return NULL; } +#ifdef FLB_SYSTEM_WINDOWS + tmp = flb_input_get_property("windows.path_encoding", ins); + if (tmp) { + if (strcasecmp(tmp, "ansi") == 0) { + ctx->windows_path_encoding = FLB_TAIL_WINDOWS_PATH_ENCODING_ANSI; + } + else if (strcasecmp(tmp, "utf-8") == 0 || + strcasecmp(tmp, "utf8") == 0) { + ctx->windows_path_encoding = FLB_TAIL_WINDOWS_PATH_ENCODING_UTF8; + } + else { + flb_plg_error(ctx->ins, "invalid 'windows.path_encoding' value %s", tmp); + flb_tail_config_destroy(ctx); + return NULL; + } + } +#endif + #ifdef FLB_HAVE_UNICODE_ENCODER tmp = flb_input_get_property("unicode.encoding", ins); if (tmp) { diff --git a/plugins/in_tail/tail_config.h b/plugins/in_tail/tail_config.h index d39249b93ee..b1739c65621 100644 --- a/plugins/in_tail/tail_config.h +++ b/plugins/in_tail/tail_config.h @@ -46,6 +46,11 @@ #define FLB_TAIL_METRIC_L_SKIPPED 105 /* number of skipped occurrences of long lines */ #endif +#ifdef FLB_SYSTEM_WINDOWS +#define FLB_TAIL_WINDOWS_PATH_ENCODING_ANSI 0 +#define FLB_TAIL_WINDOWS_PATH_ENCODING_UTF8 1 +#endif + struct flb_tail_config { int fd_notify; /* inotify fd */ flb_pipefd_t ch_manager[2]; /* pipe: channel manager */ @@ -96,6 +101,9 @@ struct flb_tail_config { * being ingested */ time_t last_pending; /* last time a 'pending signal' was emitted' */ struct mk_list *path_list; /* list of paths to scan (glob) */ +#ifdef FLB_SYSTEM_WINDOWS + int windows_path_encoding; /* Windows path API encoding mode */ +#endif flb_sds_t path_key; /* key name of file path */ flb_sds_t key; /* key for unstructured record */ int skip_long_lines; /* skip long lines */ diff --git a/plugins/in_tail/tail_file.c b/plugins/in_tail/tail_file.c index 836aea41e44..131893e6518 100644 --- a/plugins/in_tail/tail_file.c +++ b/plugins/in_tail/tail_file.c @@ -58,6 +58,62 @@ #define FLB_TAIL_DB_OFFSET_MARKER_SIZE 32 +#ifdef FLB_SYSTEM_WINDOWS +static inline int tail_file_open(struct flb_tail_config *ctx, const char *path, + int flags) +{ + if (ctx->windows_path_encoding == FLB_TAIL_WINDOWS_PATH_ENCODING_UTF8) { + return win32_open_utf8(path, flags); + } + + return open(path, flags); +} + +static inline int tail_file_stat(struct flb_tail_config *ctx, const char *path, + struct stat *st) +{ + if (ctx->windows_path_encoding == FLB_TAIL_WINDOWS_PATH_ENCODING_UTF8) { + return win32_stat_utf8(path, st); + } + + return stat(path, st); +} + +static inline int tail_file_lstat(struct flb_tail_config *ctx, const char *path, + struct stat *st) +{ + if (ctx->windows_path_encoding == FLB_TAIL_WINDOWS_PATH_ENCODING_UTF8) { + return win32_lstat_utf8(path, st); + } + + return lstat(path, st); +} +#else +static inline int tail_file_open(struct flb_tail_config *ctx, const char *path, + int flags) +{ + (void) ctx; + + return open(path, flags); +} + +static inline int tail_file_stat(struct flb_tail_config *ctx, const char *path, + struct stat *st) +{ + (void) ctx; + + return stat(path, st); +} + +static inline int tail_file_lstat(struct flb_tail_config *ctx, const char *path, + struct stat *st) +{ + (void) ctx; + + return lstat(path, st); +} +#endif + static inline void consume_bytes(char *buf, int bytes, int length) { memmove(buf, buf + bytes, length - bytes); @@ -1358,7 +1414,7 @@ int flb_tail_file_append(char *path, struct stat *st, int mode, } #endif - fd = open(path, O_RDONLY); + fd = tail_file_open(ctx, path, O_RDONLY); if (fd == -1) { flb_errno(); flb_plg_error(ctx->ins, "cannot open %s", path); @@ -1378,7 +1434,7 @@ int flb_tail_file_append(char *path, struct stat *st, int mode, file->tail_mode = mode; /* On non-windows environments check if the original path is a link */ - ret = lstat(path, &lst); + ret = tail_file_lstat(ctx, path, &lst); if (ret == 0) { if (S_ISLNK(lst.st_mode)) { file->is_link = FLB_TRUE; @@ -2113,7 +2169,7 @@ int flb_tail_file_is_rotated(struct flb_tail_config *ctx, /* Check if the 'original monitored file' is a link and rotated */ if (file->is_link == FLB_TRUE) { - ret = lstat(file->name, &st); + ret = tail_file_lstat(ctx, file->name, &st); if (ret == -1) { /* Broken link or missing file */ if (errno == ENOENT) { @@ -2148,7 +2204,7 @@ int flb_tail_file_is_rotated(struct flb_tail_config *ctx, /* Get stats from the file name */ - ret = stat(name, &st); + ret = tail_file_stat(ctx, name, &st); if (ret == -1) { flb_errno(); flb_free(name); @@ -2283,6 +2339,7 @@ char *flb_tail_file_name(struct flb_tail_file *file) #elif defined(FLB_SYSTEM_WINDOWS) int len; + wchar_t *wide_buf; h = (HANDLE) _get_osfhandle(file->fd); if (h == INVALID_HANDLE_VALUE) { @@ -2294,14 +2351,41 @@ char *flb_tail_file_name(struct flb_tail_file *file) /* This function returns the length of the string excluding "\0" * and the resulting path has a "\\?\" prefix. */ - len = GetFinalPathNameByHandleA(h, buf, PATH_MAX, FILE_NAME_NORMALIZED); - if (len == 0 || len >= PATH_MAX) { + if (file->config->windows_path_encoding == FLB_TAIL_WINDOWS_PATH_ENCODING_UTF8) { + wide_buf = flb_calloc(PATH_MAX, sizeof(wchar_t)); + if (wide_buf == NULL) { + flb_free(buf); + return NULL; + } + + len = GetFinalPathNameByHandleW(h, wide_buf, PATH_MAX, + FILE_NAME_NORMALIZED); + if (len == 0 || len >= PATH_MAX) { + flb_free(wide_buf); + flb_free(buf); + return NULL; + } + + len = (int) win32_remove_extended_path_prefix(wide_buf, (size_t) len); + flb_free(buf); - return NULL; + buf = win32_wide_to_utf8(wide_buf); + flb_free(wide_buf); + + if (buf == NULL) { + return NULL; + } } + else { + len = GetFinalPathNameByHandleA(h, buf, PATH_MAX, FILE_NAME_NORMALIZED); + if (len == 0 || len >= PATH_MAX) { + flb_free(buf); + return NULL; + } - if (strstr(buf, "\\\\?\\")) { - memmove(buf, buf + 4, len + 1); + if (strstr(buf, "\\\\?\\")) { + memmove(buf, buf + 4, len + 1); + } } #elif defined(FLB_SYSTEM_FREEBSD) if ((file_entries = kinfo_getfile(getpid(), &file_count)) == NULL) { @@ -2396,7 +2480,7 @@ int flb_tail_file_rotated(struct flb_tail_file *file) #endif /* Check if a new file has been created */ - ret = stat(tmp, &st); + ret = tail_file_stat(ctx, tmp, &st); if (ret == 0 && st.st_ino != file->inode) { if (flb_tail_file_exists(&st, ctx) == FLB_FALSE) { ret = flb_tail_file_append(tmp, &st, FLB_TAIL_STATIC, -1, ctx); diff --git a/plugins/in_tail/tail_fs_stat.c b/plugins/in_tail/tail_fs_stat.c index 18f3fb13695..e489ceac8fd 100644 --- a/plugins/in_tail/tail_fs_stat.c +++ b/plugins/in_tail/tail_fs_stat.c @@ -34,6 +34,18 @@ #include "win32.h" #endif +static int tail_fs_stat_path(struct flb_tail_config *ctx, const char *path, + struct stat *st) +{ +#ifdef FLB_SYSTEM_WINDOWS + if (ctx->windows_path_encoding == FLB_TAIL_WINDOWS_PATH_ENCODING_UTF8) { + return win32_stat_utf8(path, st); + } +#endif + + return stat(path, st); +} + struct fs_stat { /* last time check */ time_t checked; @@ -233,7 +245,7 @@ int flb_tail_fs_stat_add(struct flb_tail_file *file) } fst->checked = time(NULL); - ret = stat(file->name, &fst->st); + ret = tail_fs_stat_path(file->config, file->name, &fst->st); if (ret == -1) { flb_errno(); flb_free(fst); diff --git a/plugins/in_tail/tail_scan_win32.c b/plugins/in_tail/tail_scan_win32.c index c5349ccf7b2..dbeeb27fadb 100644 --- a/plugins/in_tail/tail_scan_win32.c +++ b/plugins/in_tail/tail_scan_win32.c @@ -41,6 +41,11 @@ static int tail_is_excluded(char *path, struct flb_tail_config *ctx) { struct mk_list *head; struct flb_slist_entry *pattern; +#ifdef FLB_SYSTEM_WINDOWS + int matched; + wchar_t *wide_path; + wchar_t *wide_pattern; +#endif if (!ctx->exclude_list) { return FLB_FALSE; @@ -48,7 +53,30 @@ static int tail_is_excluded(char *path, struct flb_tail_config *ctx) mk_list_foreach(head, ctx->exclude_list) { pattern = mk_list_entry(head, struct flb_slist_entry, _head); - if (PathMatchSpecA(path, pattern->str)) { +#ifdef FLB_SYSTEM_WINDOWS + if (ctx->windows_path_encoding == FLB_TAIL_WINDOWS_PATH_ENCODING_UTF8) { + wide_path = win32_utf8_to_wide(path); + wide_pattern = win32_utf8_to_wide(pattern->str); + if (wide_path == NULL || wide_pattern == NULL) { + flb_free(wide_path); + flb_free(wide_pattern); + continue; + } + + matched = PathMatchSpecW(wide_path, wide_pattern) || + PathMatchSpecW(PathFindFileNameW(wide_path), wide_pattern); + flb_free(wide_path); + flb_free(wide_pattern); + + if (matched) { + return FLB_TRUE; + } + + continue; + } +#endif + if (PathMatchSpecA(path, pattern->str) || + PathMatchSpecA(PathFindFileNameA(path), pattern->str)) { return FLB_TRUE; } } @@ -63,21 +91,48 @@ static int tail_is_excluded(char *path, struct flb_tail_config *ctx) static int tail_register_file(const char *target, struct flb_tail_config *ctx, time_t ts) { + int ret; int64_t mtime; struct stat st; - char path[MAX_PATH]; + char legacy_path[MAX_PATH]; + char *path; ssize_t ignored_file_size; uint64_t aged_out_inode; ignored_file_size = -1; - - if (_fullpath(path, target, MAX_PATH) == NULL) { - flb_plg_error(ctx->ins, "cannot get absolute path of %s", target); - return -1; + path = legacy_path; + +#ifdef FLB_SYSTEM_WINDOWS + if (ctx->windows_path_encoding == FLB_TAIL_WINDOWS_PATH_ENCODING_UTF8) { + path = win32_fullpath_utf8(target); + if (path == NULL) { + flb_plg_error(ctx->ins, "cannot get UTF-8 absolute path of %s", target); + return -1; + } } + else { +#endif + if (_fullpath(path, target, MAX_PATH) == NULL) { + flb_plg_error(ctx->ins, "cannot get absolute path of %s", target); + return -1; + } +#ifdef FLB_SYSTEM_WINDOWS + } +#endif - if (stat(path, &st) != 0 || !S_ISREG(st.st_mode)) { - return -1; +#ifdef FLB_SYSTEM_WINDOWS + if (ctx->windows_path_encoding == FLB_TAIL_WINDOWS_PATH_ENCODING_UTF8) { + ret = win32_stat_utf8(path, &st); + } + else { + ret = stat(path, &st); + } +#else + ret = stat(path, &st); +#endif + if (ret != 0 || !S_ISREG(st.st_mode)) { + ret = -1; + goto out; } if (ctx->ignore_older > 0) { @@ -93,14 +148,16 @@ static int tail_register_file(const char *target, struct flb_tail_config *ctx, strlen(path), st.st_size); - return -1; + ret = -1; + goto out; } } } if (tail_is_excluded(path, ctx) == FLB_TRUE) { flb_plg_trace(ctx->ins, "skip '%s' (excluded)", path); - return -1; + ret = -1; + goto out; } if (ctx->ignore_active_older_files && @@ -113,7 +170,8 @@ static int tail_register_file(const char *target, struct flb_tail_config *ctx, if (mtime > 0 && (ts - ctx->ignore_older) > mtime) { flb_plg_debug(ctx->ins, "excluded=%s (ignore_active_older_files)", path); - return -1; + ret = -1; + goto out; } } else { @@ -142,7 +200,14 @@ static int tail_register_file(const char *target, struct flb_tail_config *ctx, } } - return flb_tail_file_append(path, &st, FLB_TAIL_STATIC, ignored_file_size, ctx); + ret = flb_tail_file_append(path, &st, FLB_TAIL_STATIC, ignored_file_size, ctx); + + out: + if (path != legacy_path) { + flb_free(path); + } + + return ret; } /* @@ -157,18 +222,26 @@ static int tail_register_file(const char *target, struct flb_tail_config *ctx, static int tail_scan_pattern(const char *path, struct flb_tail_config *ctx) { char *star, *p0, *p1; - char pattern[MAX_PATH]; - char buf[MAX_PATH]; + char *pattern; + char *buf; int ret; int n_added = 0; time_t now; int64_t mtime; + size_t prefix_len; + size_t pattern_len; + size_t candidate_len; HANDLE h; WIN32_FIND_DATA data; - - if (strlen(path) > MAX_PATH - 1) { - flb_plg_error(ctx->ins, "path too long '%s'"); - return -1; + WIN32_FIND_DATAW data_w; + wchar_t *wide_pattern; + char *filename; + + if (ctx->windows_path_encoding != FLB_TAIL_WINDOWS_PATH_ENCODING_UTF8) { + if (strlen(path) > MAX_PATH - 1) { + flb_plg_error(ctx->ins, "path too long '%s'", path); + return -1; + } } star = strchr(path, '*'); @@ -194,43 +267,94 @@ static int tail_scan_pattern(const char *path, struct flb_tail_config *ctx) p1++; } - memcpy(pattern, path, (p1 - path)); - pattern[p1 - path] = '\0'; + pattern_len = p1 - path; + pattern = flb_malloc(pattern_len + 1); + if (pattern == NULL) { + flb_errno(); + return -1; + } + + memcpy(pattern, path, pattern_len); + pattern[pattern_len] = '\0'; + +#ifdef FLB_SYSTEM_WINDOWS + wide_pattern = NULL; + if (ctx->windows_path_encoding == FLB_TAIL_WINDOWS_PATH_ENCODING_UTF8) { + wide_pattern = win32_utf8_to_wide(pattern); + if (wide_pattern == NULL) { + flb_plg_error(ctx->ins, "invalid UTF-8 path pattern '%s'", pattern); + flb_free(pattern); + return -1; + } + h = FindFirstFileW(wide_pattern, &data_w); + flb_free(wide_pattern); + } + else { + h = FindFirstFileA(pattern, &data); + } +#else h = FindFirstFileA(pattern, &data); +#endif if (h == INVALID_HANDLE_VALUE) { + flb_free(pattern); return 0; /* none matched */ } + flb_free(pattern); + now = time(NULL); do { + filename = data.cFileName; +#ifdef FLB_SYSTEM_WINDOWS + if (ctx->windows_path_encoding == FLB_TAIL_WINDOWS_PATH_ENCODING_UTF8) { + filename = win32_wide_to_utf8(data_w.cFileName); + if (filename == NULL) { + continue; + } + } +#endif + /* Ignore the current and parent dirs */ - if (!strcmp(".", data.cFileName) || !strcmp("..", data.cFileName)) { - continue; + if (!strcmp(".", filename) || !strcmp("..", filename)) { + goto next; } /* Avoid an infinite loop */ - if (strchr(data.cFileName, '*')) { - continue; + if (strchr(filename, '*')) { + goto next; } - /* Create a path (prefix + filename + suffix) */ - memcpy(buf, path, p0 - path + 1); - buf[p0 - path + 1] = '\0'; + prefix_len = p0 - path + 1; + candidate_len = prefix_len + strlen(filename) + strlen(p1); - if (strlen(buf) + strlen(data.cFileName) + strlen(p1) > MAX_PATH - 1) { - flb_plg_warn(ctx->ins, "'%s%s%s' is too long", buf, data.cFileName, p1); - continue; + if (ctx->windows_path_encoding != FLB_TAIL_WINDOWS_PATH_ENCODING_UTF8) { + if (candidate_len > MAX_PATH - 1) { + flb_plg_warn(ctx->ins, "'%.*s%s%s' is too long", + (int) prefix_len, path, filename, p1); + goto next; + } + } + + buf = flb_malloc(candidate_len + 1); + if (buf == NULL) { + flb_errno(); + goto next; } - strcat(buf, data.cFileName); - strcat(buf, p1); + + /* Create a path (prefix + filename + suffix) */ + memcpy(buf, path, prefix_len); + memcpy(buf + prefix_len, filename, strlen(filename)); + memcpy(buf + prefix_len + strlen(filename), p1, strlen(p1)); + buf[candidate_len] = '\0'; if (strchr(p1, '*')) { ret = tail_scan_pattern(buf, ctx); /* recursive */ if (ret >= 0) { n_added += ret; } - continue; + flb_free(buf); + goto next; } /* Try to register the target file */ @@ -238,7 +362,20 @@ static int tail_scan_pattern(const char *path, struct flb_tail_config *ctx) if (ret == 0) { n_added++; } - } while (FindNextFileA(h, &data) != 0); + flb_free(buf); + + next: +#ifdef FLB_SYSTEM_WINDOWS + if (ctx->windows_path_encoding == FLB_TAIL_WINDOWS_PATH_ENCODING_UTF8) { + flb_free(filename); + } +#endif + } while ( +#ifdef FLB_SYSTEM_WINDOWS + ctx->windows_path_encoding == FLB_TAIL_WINDOWS_PATH_ENCODING_UTF8 ? + FindNextFileW(h, &data_w) != 0 : +#endif + FindNextFileA(h, &data) != 0); FindClose(h); return n_added; diff --git a/plugins/in_tail/win32/interface.h b/plugins/in_tail/win32/interface.h index 84b89059429..053c3b70dc4 100644 --- a/plugins/in_tail/win32/interface.h +++ b/plugins/in_tail/win32/interface.h @@ -20,6 +20,8 @@ #ifndef FLB_TAIL_WIN32_INTERFACE_H #define FLB_TAIL_WIN32_INTERFACE_H +#include + struct win32_stat { uint64_t st_ino; uint16_t st_mode; @@ -31,8 +33,17 @@ struct win32_stat { int win32_stat(const char *path, struct win32_stat *wst); int win32_lstat(const char *path, struct win32_stat *wst); int win32_fstat(int fd, struct win32_stat *wst); +int win32_stat_utf8(const char *path, struct win32_stat *wst); +int win32_lstat_utf8(const char *path, struct win32_stat *wst); int win32_open(const char *path, int flags); +int win32_open_utf8(const char *path, int flags); + +void win32_propagate_last_error_to_errno(void); +wchar_t *win32_utf8_to_wide(const char *str); +char *win32_wide_to_utf8(const wchar_t *str); +char *win32_fullpath_utf8(const char *path); +size_t win32_remove_extended_path_prefix(wchar_t *path, size_t length); #define WIN32_S_IFDIR 0x1000 #define WIN32_S_IFCHR 0x2000 diff --git a/plugins/in_tail/win32/io.c b/plugins/in_tail/win32/io.c index 5315e28c061..d34e11969ac 100644 --- a/plugins/in_tail/win32/io.c +++ b/plugins/in_tail/win32/io.c @@ -17,11 +17,19 @@ * limitations under the License. */ +#ifndef WIN32_LEAN_AND_MEAN +#define WIN32_LEAN_AND_MEAN +#endif + #include #include #include #include #include +#include + +#include + #include "interface.h" /* @@ -45,3 +53,38 @@ int win32_open(const char *path, int flags) } return _open_osfhandle((intptr_t) h, _O_RDONLY); } + +int win32_open_utf8(const char *path, int flags) +{ + int fd; + HANDLE h; + wchar_t *wide_path; + + wide_path = win32_utf8_to_wide(path); + if (wide_path == NULL) { + errno = EINVAL; + return -1; + } + + h = CreateFileW(wide_path, + GENERIC_READ, + FILE_SHARE_READ|FILE_SHARE_WRITE|FILE_SHARE_DELETE, + NULL, /* lpSecurityAttributes */ + OPEN_EXISTING, /* dwCreationDisposition */ + 0, /* dwFlagsAndAttributes */ + NULL); /* hTemplateFile */ + + if (h == INVALID_HANDLE_VALUE) { + win32_propagate_last_error_to_errno(); + flb_free(wide_path); + return -1; + } + + flb_free(wide_path); + fd = _open_osfhandle((intptr_t) h, _O_RDONLY); + if (fd == -1) { + CloseHandle(h); + } + + return fd; +} diff --git a/plugins/in_tail/win32/path.c b/plugins/in_tail/win32/path.c new file mode 100644 index 00000000000..365724ce1c8 --- /dev/null +++ b/plugins/in_tail/win32/path.c @@ -0,0 +1,149 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2026 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef WIN32_LEAN_AND_MEAN +#define WIN32_LEAN_AND_MEAN +#endif + +#include +#include + +#include + +#include "interface.h" + +wchar_t *win32_utf8_to_wide(const char *str) +{ + int len; + wchar_t *buf; + + if (str == NULL) { + return NULL; + } + + len = MultiByteToWideChar(CP_UTF8, MB_ERR_INVALID_CHARS, + str, -1, NULL, 0); + if (len == 0) { + return NULL; + } + + buf = flb_calloc(len, sizeof(wchar_t)); + if (buf == NULL) { + return NULL; + } + + if (MultiByteToWideChar(CP_UTF8, MB_ERR_INVALID_CHARS, + str, -1, buf, len) == 0) { + flb_free(buf); + return NULL; + } + + return buf; +} + +char *win32_wide_to_utf8(const wchar_t *str) +{ + int len; + char *buf; + + if (str == NULL) { + return NULL; + } + + len = WideCharToMultiByte(CP_UTF8, WC_ERR_INVALID_CHARS, + str, -1, NULL, 0, NULL, NULL); + if (len == 0) { + return NULL; + } + + buf = flb_calloc(len, sizeof(char)); + if (buf == NULL) { + return NULL; + } + + if (WideCharToMultiByte(CP_UTF8, WC_ERR_INVALID_CHARS, + str, -1, buf, len, NULL, NULL) == 0) { + flb_free(buf); + return NULL; + } + + return buf; +} + +size_t win32_remove_extended_path_prefix(wchar_t *path, size_t length) +{ + if (path == NULL) { + return 0; + } + + if (length >= 8 && wcsncmp(path, L"\\\\?\\UNC\\", 8) == 0) { + memmove(path + 2, path + 8, (length - 7) * sizeof(wchar_t)); + path[0] = L'\\'; + path[1] = L'\\'; + + return length - 6; + } + + if (length >= 4 && wcsncmp(path, L"\\\\?\\", 4) == 0) { + memmove(path, path + 4, (length - 3) * sizeof(wchar_t)); + + return length - 4; + } + + return length; +} + +char *win32_fullpath_utf8(const char *path) +{ + DWORD len; + DWORD ret; + wchar_t *wide_path; + wchar_t *wide_fullpath; + char *fullpath; + + wide_path = win32_utf8_to_wide(path); + if (wide_path == NULL) { + return NULL; + } + + len = GetFullPathNameW(wide_path, 0, NULL, NULL); + if (len == 0) { + flb_free(wide_path); + return NULL; + } + + wide_fullpath = flb_calloc(len, sizeof(wchar_t)); + if (wide_fullpath == NULL) { + flb_free(wide_path); + return NULL; + } + + ret = GetFullPathNameW(wide_path, len, wide_fullpath, NULL); + flb_free(wide_path); + + if (ret == 0 || ret >= len) { + flb_free(wide_fullpath); + return NULL; + } + + fullpath = win32_wide_to_utf8(wide_fullpath); + flb_free(wide_fullpath); + + return fullpath; +} diff --git a/plugins/in_tail/win32/stat.c b/plugins/in_tail/win32/stat.c index ce6b9ffd457..aa7ce731e28 100644 --- a/plugins/in_tail/win32/stat.c +++ b/plugins/in_tail/win32/stat.c @@ -17,10 +17,17 @@ * limitations under the License. */ +#ifndef WIN32_LEAN_AND_MEAN +#define WIN32_LEAN_AND_MEAN +#endif + #include #include #include #include + +#include + #include "interface.h" /* @@ -73,7 +80,7 @@ static void reset_errno() errno = 0; } -static void propagate_last_error_to_errno() +void win32_propagate_last_error_to_errno(void) { DWORD error_code; @@ -194,7 +201,7 @@ static int is_symlink(const char *path) h = FindFirstFileA(path, &data); if (h == INVALID_HANDLE_VALUE) { - propagate_last_error_to_errno(); + win32_propagate_last_error_to_errno(); return 0; } @@ -215,6 +222,41 @@ static int is_symlink(const char *path) return 0; } +static int is_symlink_utf8(const char *path) +{ + WIN32_FIND_DATAW data; + HANDLE h; + wchar_t *wide_path; + + SetLastError(0); + reset_errno(); + + wide_path = win32_utf8_to_wide(path); + if (wide_path == NULL) { + errno = EINVAL; + return 0; + } + + h = FindFirstFileW(wide_path, &data); + flb_free(wide_path); + + if (h == INVALID_HANDLE_VALUE) { + win32_propagate_last_error_to_errno(); + + return 0; + } + + FindClose(h); + + if (data.dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT) { + if (data.dwReserved0 == IO_REPARSE_TAG_SYMLINK) { + return 1; + } + } + + return 0; +} + static int hstat(HANDLE h, struct win32_stat *wst) { BY_HANDLE_FILE_INFORMATION info; @@ -224,14 +266,14 @@ static int hstat(HANDLE h, struct win32_stat *wst) reset_errno(); if (!GetFileInformationByHandle(h, &info)) { - propagate_last_error_to_errno(); + win32_propagate_last_error_to_errno(); return -1; } if (!GetFileInformationByHandleEx(h, FileStandardInfo, &std, sizeof(std))) { - propagate_last_error_to_errno(); + win32_propagate_last_error_to_errno(); return -1; } @@ -265,7 +307,45 @@ int win32_stat(const char *path, struct win32_stat *wst) NULL); /* hTemplateFile */ if (h == INVALID_HANDLE_VALUE) { - propagate_last_error_to_errno(); + win32_propagate_last_error_to_errno(); + + return -1; + } + + if (hstat(h, wst)) { + CloseHandle(h); + return -1; + } + + CloseHandle(h); + return 0; +} + +int win32_stat_utf8(const char *path, struct win32_stat *wst) +{ + HANDLE h; + wchar_t *wide_path; + + SetLastError(0); + reset_errno(); + + wide_path = win32_utf8_to_wide(path); + if (wide_path == NULL) { + errno = EINVAL; + return -1; + } + + h = CreateFileW(wide_path, + GENERIC_READ, + FILE_SHARE_READ|FILE_SHARE_WRITE|FILE_SHARE_DELETE, + NULL, /* lpSecurityAttributes */ + OPEN_EXISTING, /* dwCreationDisposition */ + 0, /* dwFlagsAndAttributes */ + NULL); /* hTemplateFile */ + flb_free(wide_path); + + if (h == INVALID_HANDLE_VALUE) { + win32_propagate_last_error_to_errno(); return -1; } @@ -295,7 +375,7 @@ int win32_lstat(const char *path, struct win32_stat *wst) NULL); /* hTemplateFile */ if (h == INVALID_HANDLE_VALUE) { - propagate_last_error_to_errno(); + win32_propagate_last_error_to_errno(); return -1; } @@ -313,6 +393,48 @@ int win32_lstat(const char *path, struct win32_stat *wst) return 0; } +int win32_lstat_utf8(const char *path, struct win32_stat *wst) +{ + HANDLE h; + wchar_t *wide_path; + + SetLastError(0); + reset_errno(); + + wide_path = win32_utf8_to_wide(path); + if (wide_path == NULL) { + errno = EINVAL; + return -1; + } + + h = CreateFileW(wide_path, + GENERIC_READ, + FILE_SHARE_READ|FILE_SHARE_WRITE|FILE_SHARE_DELETE, + NULL, /* lpSecurityAttributes */ + OPEN_EXISTING, /* dwCreationDisposition */ + FILE_FLAG_OPEN_REPARSE_POINT, + NULL); /* hTemplateFile */ + flb_free(wide_path); + + if (h == INVALID_HANDLE_VALUE) { + win32_propagate_last_error_to_errno(); + + return -1; + } + + if (hstat(h, wst)) { + CloseHandle(h); + return -1; + } + + if (is_symlink_utf8(path)) { + wst->st_mode = WIN32_S_IFLNK; + } + + CloseHandle(h); + return 0; +} + int win32_fstat(int fd, struct win32_stat *wst) { HANDLE h; @@ -323,7 +445,7 @@ int win32_fstat(int fd, struct win32_stat *wst) h = (HANDLE) _get_osfhandle(fd); if (h == INVALID_HANDLE_VALUE) { - propagate_last_error_to_errno(); + win32_propagate_last_error_to_errno(); return -1; } diff --git a/plugins/in_tcp/tcp.c b/plugins/in_tcp/tcp.c index 6095ba766eb..902b76f07ac 100644 --- a/plugins/in_tcp/tcp.c +++ b/plugins/in_tcp/tcp.c @@ -18,13 +18,242 @@ */ #include +#include #include +#include +#include #include #include "tcp.h" #include "tcp_conn.h" #include "tcp_config.h" +static int in_tcp_collect_ctx(struct flb_in_tcp_config *ctx); +static int in_tcp_collect(struct flb_input_instance *in, + struct flb_config *config, void *in_context); + +static void in_tcp_connections_destroy(struct flb_in_tcp_config *ctx) +{ + struct mk_list *tmp; + struct mk_list *head; + struct tcp_conn *conn; + + mk_list_foreach_safe(head, tmp, &ctx->connections) { + conn = mk_list_entry(head, struct tcp_conn, _head); + tcp_conn_del(conn); + } +} + +static void in_tcp_connections_pause(struct flb_in_tcp_config *ctx) +{ + struct mk_list *tmp; + struct mk_list *head; + struct tcp_conn *conn; + + mk_list_foreach_safe(head, tmp, &ctx->connections) { + conn = mk_list_entry(head, struct tcp_conn, _head); + if (conn->busy) { + conn->pending_close = FLB_TRUE; + continue; + } + + tcp_conn_del(conn); + } +} + +static int in_tcp_worker_listener_event(void *data) +{ + struct mk_event *event; + + event = data; + + return in_tcp_collect_ctx(event->data); +} + +static int in_tcp_start_listener(struct flb_in_tcp_config *ctx, + struct flb_config *config, + struct flb_net_setup *net_setup, + struct mk_event_loop *event_loop, + int use_collector) +{ + int ret; + unsigned short int port; + + port = (unsigned short int) strtoul(ctx->tcp_port, NULL, 10); + + ctx->downstream = flb_downstream_create(FLB_TRANSPORT_TCP, + ctx->ins->flags, + ctx->listen, + port, + ctx->ins->tls, + config, + net_setup); + + if (ctx->downstream == NULL) { + flb_plg_error(ctx->ins, + "could not initialize downstream on %s:%s. Aborting", + ctx->listen, ctx->tcp_port); + return -1; + } + + flb_input_downstream_set(ctx->downstream, ctx->ins); + + if (use_collector == FLB_TRUE) { + ret = flb_input_set_collector_socket(ctx->ins, + in_tcp_collect, + ctx->downstream->server_fd, + config); + if (ret == -1) { + flb_plg_error(ctx->ins, "Could not set collector for IN_TCP input plugin"); + return -1; + } + + ctx->collector_id = ret; + } + else { + ctx->event_loop = event_loop; + MK_EVENT_NEW(&ctx->listener_event); + ctx->listener_event.type = FLB_ENGINE_EV_CUSTOM; + ctx->listener_event.data = ctx; + ctx->listener_event.handler = in_tcp_worker_listener_event; + + ret = mk_event_add(event_loop, + ctx->downstream->server_fd, + FLB_ENGINE_EV_CUSTOM, + MK_EVENT_READ, + &ctx->listener_event); + if (ret == -1) { + flb_plg_error(ctx->ins, "could not register TCP worker listener"); + return -1; + } + + ctx->listener_registered = FLB_TRUE; + } + + return 0; +} + +static int in_tcp_worker_init(struct flb_downstream_worker *worker, + void *parent, + void **worker_context) +{ + int ret; + struct flb_in_tcp_config *ctx; + struct flb_in_tcp_config *parent_ctx; + + parent_ctx = parent; + + ctx = tcp_config_init(parent_ctx->ins); + if (ctx == NULL) { + return -1; + } + + *worker_context = ctx; + + ctx->collector_id = -1; + ctx->ins = parent_ctx->ins; + ctx->workers = parent_ctx->workers; + ctx->worker_id = flb_downstream_worker_id_get(worker); + ctx->use_ingress_queue = FLB_TRUE; + ctx->net_setup = parent_ctx->ins->net_setup; + ctx->net_setup.share_port = FLB_TRUE; + mk_list_init(&ctx->connections); + + ret = in_tcp_start_listener(ctx, + parent_ctx->ins->config, + &ctx->net_setup, + flb_downstream_worker_event_loop_get(worker), + FLB_FALSE); + if (ret == 0) { + flb_downstream_thread_safe(ctx->downstream); + ret = flb_downstream_worker_listener_fd_set( + worker, ctx->downstream->server_fd); + } + + return ret; +} + +static void in_tcp_worker_exit(struct flb_downstream_worker *worker, + void *worker_context) +{ + struct flb_in_tcp_config *ctx; + + (void) worker; + + ctx = worker_context; + + in_tcp_connections_destroy(ctx); + tcp_config_destroy(ctx); +} + +static void in_tcp_worker_maintenance(struct flb_downstream_worker *worker, + void *worker_context) +{ + struct flb_in_tcp_config *ctx; + + (void) worker; + + ctx = worker_context; + + if (ctx->downstream != NULL) { + flb_downstream_conn_timeouts_stream(ctx->downstream); + flb_downstream_conn_pending_destroy(ctx->downstream); + } +} + +static int in_tcp_workers_start(struct flb_in_tcp_config *ctx) +{ + struct flb_downstream_worker_options options; + + memset(&options, 0, sizeof(struct flb_downstream_worker_options)); + options.workers = ctx->workers; + options.parent = ctx; + options.cb_init = in_tcp_worker_init; + options.cb_exit = in_tcp_worker_exit; + options.cb_maintenance = in_tcp_worker_maintenance; + + return flb_downstream_worker_runtime_start(&ctx->runtime, &options); +} + +static void in_tcp_workers_stop(struct flb_in_tcp_config *ctx) +{ + flb_downstream_worker_runtime_stop(ctx->runtime); + ctx->runtime = NULL; +} + +static void in_tcp_worker_pause(struct flb_downstream_worker *worker, + void *worker_context, + void *data) +{ + struct flb_in_tcp_config *ctx; + + (void) worker; + (void) data; + + ctx = worker_context; + + if (ctx->downstream != NULL) { + flb_downstream_pause(ctx->downstream); + } + in_tcp_connections_pause(ctx); +} + +static void in_tcp_worker_resume(struct flb_downstream_worker *worker, + void *worker_context, + void *data) +{ + struct flb_in_tcp_config *ctx; + + (void) worker; + (void) data; + + ctx = worker_context; + + if (ctx->downstream != NULL) { + flb_downstream_resume(ctx->downstream); + } +} + /* * For a server event, the collection event means a new client have arrived, we * accept the connection and create a new TCP instance which will wait for @@ -32,12 +261,17 @@ */ static int in_tcp_collect(struct flb_input_instance *in, struct flb_config *config, void *in_context) +{ + (void) in; + (void) config; + + return in_tcp_collect_ctx(in_context); +} + +static int in_tcp_collect_ctx(struct flb_in_tcp_config *ctx) { struct flb_connection *connection; struct tcp_conn *conn; - struct flb_in_tcp_config *ctx; - - ctx = in_context; connection = flb_downstream_conn_get(ctx->downstream); @@ -65,7 +299,6 @@ static int in_tcp_collect(struct flb_input_instance *in, static int in_tcp_init(struct flb_input_instance *in, struct flb_config *config, void *data) { - unsigned short int port; int ret; struct flb_in_tcp_config *ctx; @@ -83,61 +316,52 @@ static int in_tcp_init(struct flb_input_instance *in, /* Set the context */ flb_input_set_context(in, ctx); - port = (unsigned short int) strtoul(ctx->tcp_port, NULL, 10); - - ctx->downstream = flb_downstream_create(FLB_TRANSPORT_TCP, - in->flags, - ctx->listen, - port, - in->tls, - config, - &in->net_setup); - - if (ctx->downstream == NULL) { - flb_plg_error(ctx->ins, - "could not initialize downstream on %s:%s. Aborting", - ctx->listen, ctx->tcp_port); - - tcp_config_destroy(ctx); - - return -1; + if (ctx->workers <= 0) { + ctx->workers = 1; } - flb_input_downstream_set(ctx->downstream, ctx->ins); - - /* Collect upon data available on the standard input */ - ret = flb_input_set_collector_socket(in, - in_tcp_collect, - ctx->downstream->server_fd, - config); - if (ret == -1) { - flb_plg_error(ctx->ins, "Could not set collector for IN_TCP input plugin"); - tcp_config_destroy(ctx); + if (ctx->workers > 1) { + ret = flb_input_ingress_enable(in); + if (ret != 0) { + tcp_config_destroy(ctx); + return -1; + } - return -1; + ret = in_tcp_workers_start(ctx); + if (ret != 0) { + flb_plg_error(ctx->ins, + "could not start TCP listener workers on %s:%s. Aborting", + ctx->listen, ctx->tcp_port); + tcp_config_destroy(ctx); + return -1; + } + } + else { + ret = in_tcp_start_listener(ctx, config, &in->net_setup, NULL, FLB_TRUE); + if (ret != 0) { + tcp_config_destroy(ctx); + return -1; + } } - ctx->collector_id = ret; + flb_plg_info(ctx->ins, + "listening on %s:%s with %i worker%s", + ctx->listen, ctx->tcp_port, ctx->workers, + ctx->workers == 1 ? "" : "s"); return 0; } static int in_tcp_exit(void *data, struct flb_config *config) { - struct mk_list *tmp; - struct mk_list *head; struct flb_in_tcp_config *ctx; - struct tcp_conn *conn; (void) *config; ctx = data; - mk_list_foreach_safe(head, tmp, &ctx->connections) { - conn = mk_list_entry(head, struct tcp_conn, _head); - - tcp_conn_del(conn); - } + in_tcp_workers_stop(ctx); + in_tcp_connections_destroy(ctx); tcp_config_destroy(ctx); @@ -146,32 +370,43 @@ static int in_tcp_exit(void *data, struct flb_config *config) static void in_tcp_pause(void *data, struct flb_config *config) { + int ret; struct flb_in_tcp_config *ctx = data; - struct mk_list *head; - struct mk_list *tmp; - struct tcp_conn *conn; (void) config; - flb_downstream_pause(ctx->downstream); - - mk_list_foreach_safe(head, tmp, &ctx->connections) { - conn = mk_list_entry(head, struct tcp_conn, _head); - if (conn->busy) { - conn->pending_close = FLB_TRUE; - continue; + if (ctx->runtime != NULL) { + ret = flb_downstream_worker_runtime_foreach(ctx->runtime, + in_tcp_worker_pause, + NULL); + if (ret != 0) { + flb_plg_error(ctx->ins, "could not pause all downstream workers"); } - - tcp_conn_del(conn); + return; } + + flb_downstream_pause(ctx->downstream); + + in_tcp_connections_pause(ctx); } static void in_tcp_resume(void *data, struct flb_config *config) { + int ret; struct flb_in_tcp_config *ctx = data; (void) config; + if (ctx->runtime != NULL) { + ret = flb_downstream_worker_runtime_foreach(ctx->runtime, + in_tcp_worker_resume, + NULL); + if (ret != 0) { + flb_plg_error(ctx->ins, "could not resume all downstream workers"); + } + return; + } + flb_downstream_resume(ctx->downstream); } @@ -206,6 +441,11 @@ static struct flb_config_map config_map[] = { 0, FLB_TRUE, offsetof(struct flb_in_tcp_config, source_address_key), "Key where the source address will be injected" }, + { + FLB_CONFIG_MAP_INT, "workers", "1", + 0, FLB_TRUE, offsetof(struct flb_in_tcp_config, workers), + "Set the number of TCP listener workers" + }, /* EOF */ {0} }; diff --git a/plugins/in_tcp/tcp.h b/plugins/in_tcp/tcp.h index 8700455c24b..5da1affc94c 100644 --- a/plugins/in_tcp/tcp.h +++ b/plugins/in_tcp/tcp.h @@ -27,11 +27,14 @@ #include #include #include +#include #ifdef FLB_HAVE_PARSER #include #endif #include +struct flb_downstream_worker_runtime; + struct flb_in_tcp_config { flb_sds_t format_name; /* Data format name */ int format; /* Data format */ @@ -51,10 +54,28 @@ struct flb_in_tcp_config { void *parser; #endif int collector_id; /* Listener collector id */ + int workers; /* Listener worker count */ + int worker_id; /* Worker id */ + int use_ingress_queue; /* Queue records to main loop */ + int listener_registered; /* Listener event registered */ + struct mk_event listener_event; /* Worker listener event */ + struct mk_event_loop *event_loop; /* Worker event loop */ + struct flb_net_setup net_setup; /* Worker network setup */ struct flb_downstream *downstream; /* Client manager */ struct mk_list connections; /* List of active connections */ struct flb_input_instance *ins; /* Input plugin instace */ struct flb_log_event_encoder *log_encoder; + struct flb_downstream_worker_runtime *runtime; }; +static inline int tcp_ingest_logs(struct flb_in_tcp_config *ctx, + const void *buf, size_t buf_size) +{ + if (ctx->use_ingress_queue == FLB_TRUE) { + return flb_input_ingress_queue_log(ctx->ins, NULL, 0, buf, buf_size); + } + + return flb_input_log_append(ctx->ins, NULL, 0, buf, buf_size); +} + #endif diff --git a/plugins/in_tcp/tcp_config.c b/plugins/in_tcp/tcp_config.c index 78c9915833a..0204d9821e8 100644 --- a/plugins/in_tcp/tcp_config.c +++ b/plugins/in_tcp/tcp_config.c @@ -43,6 +43,7 @@ struct flb_in_tcp_config *tcp_config_init(struct flb_input_instance *ins) } ctx->ins = ins; ctx->format = FLB_TCP_FMT_JSON; + ctx->workers = 1; /* Load the config map */ ret = flb_input_config_map_set(ins, (void *)ctx); @@ -168,6 +169,11 @@ int tcp_config_destroy(struct flb_in_tcp_config *ctx) ctx->collector_id = -1; } + if (ctx->listener_registered == FLB_TRUE && ctx->event_loop != NULL) { + mk_event_del(ctx->event_loop, &ctx->listener_event); + ctx->listener_registered = FLB_FALSE; + } + if (ctx->downstream != NULL) { flb_downstream_destroy(ctx->downstream); } diff --git a/plugins/in_tcp/tcp_conn.c b/plugins/in_tcp/tcp_conn.c index ddd15e57ab1..0d9c92bf1c2 100644 --- a/plugins/in_tcp/tcp_conn.c +++ b/plugins/in_tcp/tcp_conn.c @@ -54,6 +54,7 @@ static inline int process_pack(struct tcp_conn *conn, char *source_address; ctx = conn->ctx; + ret = FLB_EVENT_ENCODER_SUCCESS; flb_log_event_encoder_reset(ctx->log_encoder); @@ -140,9 +141,15 @@ static inline int process_pack(struct tcp_conn *conn, if (ret == FLB_EVENT_ENCODER_SUCCESS) { if (ctx->log_encoder->output_length > 0) { - flb_input_log_append(conn->ins, NULL, 0, - ctx->log_encoder->output_buffer, - ctx->log_encoder->output_length); + ret = tcp_ingest_logs(ctx, + ctx->log_encoder->output_buffer, + ctx->log_encoder->output_length); + if (ret != 0) { + flb_plg_error(ctx->ins, + "could not append TCP logs for %s:%s. ret=%d", + ctx->listen, ctx->tcp_port, ret); + return -1; + } } ret = 0; } @@ -384,9 +391,15 @@ static ssize_t parse_payload_none(struct tcp_conn *conn) if (ret == FLB_EVENT_ENCODER_SUCCESS) { if (ctx->log_encoder->output_length > 0) { - flb_input_log_append(conn->ins, NULL, 0, - ctx->log_encoder->output_buffer, - ctx->log_encoder->output_length); + ret = tcp_ingest_logs(ctx, + ctx->log_encoder->output_buffer, + ctx->log_encoder->output_length); + if (ret != 0) { + flb_plg_error(ctx->ins, + "could not append TCP logs for %s:%s. ret=%d", + ctx->listen, ctx->tcp_port, ret); + return -1; + } } } else { @@ -452,6 +465,11 @@ int tcp_conn_event(void *data) (void *) &conn->buf_data[conn->buf_len], available); + if (flb_io_net_is_retry(bytes)) { + conn->busy = FLB_FALSE; + return 0; + } + if (bytes <= 0) { flb_plg_trace(ctx->ins, "fd=%i closed connection", event->fd); conn->busy = FLB_FALSE; diff --git a/plugins/in_udp/udp.c b/plugins/in_udp/udp.c index c3e9717e066..ac4587431c2 100644 --- a/plugins/in_udp/udp.c +++ b/plugins/in_udp/udp.c @@ -18,121 +18,289 @@ */ #include +#include #include +#include +#include #include #include "udp.h" #include "udp_conn.h" #include "udp_config.h" +static int in_udp_collect_ctx(struct flb_in_udp_config *ctx); static int in_udp_collect(struct flb_input_instance *in, struct flb_config *config, - void *in_context) -{ - struct flb_connection *connection; - struct flb_in_udp_config *ctx; - - ctx = in_context; + void *in_context); - connection = flb_downstream_conn_get(ctx->downstream); - - if (connection == NULL) { - flb_plg_error(ctx->ins, "could get UDP server dummy connection"); +static int in_udp_worker_listener_event(void *data) +{ + struct mk_event *event; - return -1; - } + event = data; - return udp_conn_event(connection); + return in_udp_collect_ctx(event->data); } -/* Initialize plugin */ -static int in_udp_init(struct flb_input_instance *in, - struct flb_config *config, void *data) +static void in_udp_dummy_conn_destroy(struct flb_in_udp_config *ctx) { - struct flb_connection *connection; - unsigned short int port; - int ret; - struct flb_in_udp_config *ctx; - - (void) data; - - /* Allocate space for the configuration */ - ctx = udp_config_init(in); - - if (ctx == NULL) { - return -1; + if (ctx->dummy_conn != NULL) { + udp_conn_del(ctx->dummy_conn); + ctx->dummy_conn = NULL; } +} - ctx->collector_id = -1; - ctx->ins = in; - - /* Set the context */ - flb_input_set_context(in, ctx); +static int in_udp_start_listener(struct flb_in_udp_config *ctx, + struct flb_config *config, + struct flb_net_setup *net_setup, + struct mk_event_loop *event_loop, + int use_collector) +{ + int ret; + unsigned short int port; + struct flb_connection *connection; port = (unsigned short int) strtoul(ctx->port, NULL, 10); ctx->downstream = flb_downstream_create(FLB_TRANSPORT_UDP, - in->flags, + ctx->ins->flags, ctx->listen, port, - in->tls, + ctx->ins->tls, config, - &in->net_setup); + net_setup); if (ctx->downstream == NULL) { flb_plg_error(ctx->ins, "could not initialize downstream on %s:%s. Aborting", ctx->listen, ctx->port); - - udp_config_destroy(ctx); - return -1; } flb_input_downstream_set(ctx->downstream, ctx->ins); connection = flb_downstream_conn_get(ctx->downstream); - if (connection == NULL) { flb_plg_error(ctx->ins, "could not get UDP server dummy connection"); - - udp_config_destroy(ctx); - return -1; } ctx->dummy_conn = udp_conn_add(connection, ctx); - if (ctx->dummy_conn == NULL) { flb_plg_error(ctx->ins, "could not track UDP server dummy connection"); + return -1; + } + + if (use_collector == FLB_TRUE) { + ret = flb_input_set_collector_socket(ctx->ins, + in_udp_collect, + ctx->downstream->server_fd, + config); + if (ret == -1) { + flb_plg_error(ctx->ins, "Could not set collector for IN_UDP input plugin"); + in_udp_dummy_conn_destroy(ctx); + return -1; + } + + ctx->collector_id = ret; + ctx->collector_event = flb_input_collector_get_event(ret, ctx->ins); + + if (ctx->collector_event == NULL) { + flb_plg_error(ctx->ins, "Could not get collector event"); + in_udp_dummy_conn_destroy(ctx); + return -1; + } + } + else { + ctx->event_loop = event_loop; + MK_EVENT_NEW(&ctx->listener_event); + ctx->listener_event.type = FLB_ENGINE_EV_CUSTOM; + ctx->listener_event.data = ctx; + ctx->listener_event.handler = in_udp_worker_listener_event; + + ret = mk_event_add(event_loop, + ctx->downstream->server_fd, + FLB_ENGINE_EV_CUSTOM, + MK_EVENT_READ, + &ctx->listener_event); + if (ret == -1) { + flb_plg_error(ctx->ins, "could not register UDP worker listener"); + in_udp_dummy_conn_destroy(ctx); + return -1; + } + + ctx->listener_registered = FLB_TRUE; + } + + return 0; +} + +static int in_udp_worker_init(struct flb_downstream_worker *worker, + void *parent, + void **worker_context) +{ + int ret; + struct flb_in_udp_config *ctx; + struct flb_in_udp_config *parent_ctx; - udp_config_destroy(ctx); + parent_ctx = parent; + ctx = udp_config_init(parent_ctx->ins); + if (ctx == NULL) { return -1; } - /* Collect upon data available on the standard input */ - ret = flb_input_set_collector_socket(in, - in_udp_collect, - ctx->downstream->server_fd, - config); - if (ret == -1) { - flb_plg_error(ctx->ins, "Could not set collector for IN_UDP input plugin"); - udp_config_destroy(ctx); + *worker_context = ctx; + + ctx->collector_id = -1; + ctx->ins = parent_ctx->ins; + ctx->workers = parent_ctx->workers; + ctx->worker_id = flb_downstream_worker_id_get(worker); + ctx->use_ingress_queue = FLB_TRUE; + ctx->net_setup = parent_ctx->ins->net_setup; + ctx->net_setup.share_port = FLB_TRUE; + + ret = in_udp_start_listener(ctx, + parent_ctx->ins->config, + &ctx->net_setup, + flb_downstream_worker_event_loop_get(worker), + FLB_FALSE); + if (ret == 0) { + flb_downstream_thread_safe(ctx->downstream); + ret = flb_downstream_worker_listener_fd_set( + worker, ctx->downstream->server_fd); + } + + return ret; +} + +static void in_udp_worker_exit(struct flb_downstream_worker *worker, + void *worker_context) +{ + struct flb_in_udp_config *ctx; + + (void) worker; + + ctx = worker_context; + + in_udp_dummy_conn_destroy(ctx); + udp_config_destroy(ctx); +} + +static void in_udp_worker_maintenance(struct flb_downstream_worker *worker, + void *worker_context) +{ + struct flb_in_udp_config *ctx; + + (void) worker; + + ctx = worker_context; + + if (ctx->downstream != NULL) { + flb_downstream_conn_pending_destroy(ctx->downstream); + } +} + +static int in_udp_workers_start(struct flb_in_udp_config *ctx) +{ + struct flb_downstream_worker_options options; + + memset(&options, 0, sizeof(struct flb_downstream_worker_options)); + options.workers = ctx->workers; + options.parent = ctx; + options.cb_init = in_udp_worker_init; + options.cb_exit = in_udp_worker_exit; + options.cb_maintenance = in_udp_worker_maintenance; + + return flb_downstream_worker_runtime_start(&ctx->runtime, &options); +} + +static void in_udp_workers_stop(struct flb_in_udp_config *ctx) +{ + flb_downstream_worker_runtime_stop(ctx->runtime); + ctx->runtime = NULL; +} + +static int in_udp_collect(struct flb_input_instance *in, + struct flb_config *config, + void *in_context) +{ + (void) in; + (void) config; + + return in_udp_collect_ctx(in_context); +} + +static int in_udp_collect_ctx(struct flb_in_udp_config *ctx) +{ + struct flb_connection *connection; + + connection = flb_downstream_conn_get(ctx->downstream); + + if (connection == NULL) { + flb_plg_error(ctx->ins, "could get UDP server dummy connection"); return -1; } - ctx->collector_id = ret; - ctx->collector_event = flb_input_collector_get_event(ret, in); + return udp_conn_event(connection); +} + +/* Initialize plugin */ +static int in_udp_init(struct flb_input_instance *in, + struct flb_config *config, void *data) +{ + int ret; + struct flb_in_udp_config *ctx; + + (void) data; - if (ret == -1) { - flb_plg_error(ctx->ins, "Could not get collector event"); - udp_config_destroy(ctx); + /* Allocate space for the configuration */ + ctx = udp_config_init(in); + if (ctx == NULL) { return -1; } + ctx->collector_id = -1; + ctx->ins = in; + + /* Set the context */ + flb_input_set_context(in, ctx); + + if (ctx->workers <= 0) { + ctx->workers = 1; + } + + if (ctx->workers > 1) { + ret = flb_input_ingress_enable(in); + if (ret != 0) { + udp_config_destroy(ctx); + return -1; + } + + ret = in_udp_workers_start(ctx); + if (ret != 0) { + flb_plg_error(ctx->ins, + "could not start UDP listener workers on %s:%s. Aborting", + ctx->listen, ctx->port); + udp_config_destroy(ctx); + return -1; + } + } + else { + ret = in_udp_start_listener(ctx, config, &in->net_setup, NULL, FLB_TRUE); + if (ret != 0) { + udp_config_destroy(ctx); + return -1; + } + } + + flb_plg_info(ctx->ins, + "listening on %s:%s with %i worker%s", + ctx->listen, ctx->port, ctx->workers, + ctx->workers == 1 ? "" : "s"); + return 0; } @@ -144,9 +312,8 @@ static int in_udp_exit(void *data, struct flb_config *config) ctx = data; - if (ctx->dummy_conn != NULL) { - udp_conn_del(ctx->dummy_conn); - } + in_udp_workers_stop(ctx); + in_udp_dummy_conn_destroy(ctx); udp_config_destroy(ctx); @@ -184,6 +351,11 @@ static struct flb_config_map config_map[] = { 0, FLB_TRUE, offsetof(struct flb_in_udp_config, source_address_key), "Key where the source address will be injected" }, + { + FLB_CONFIG_MAP_INT, "workers", "1", + 0, FLB_TRUE, offsetof(struct flb_in_udp_config, workers), + "Set the number of UDP listener workers" + }, /* EOF */ {0} }; diff --git a/plugins/in_udp/udp.h b/plugins/in_udp/udp.h index 16943a92bcf..da04cdb0ffa 100644 --- a/plugins/in_udp/udp.h +++ b/plugins/in_udp/udp.h @@ -27,12 +27,14 @@ #include #include #include +#include #ifdef FLB_HAVE_PARSER #include #endif #include struct udp_conn; +struct flb_downstream_worker_runtime; struct flb_in_udp_config { struct mk_event *collector_event; @@ -54,10 +56,28 @@ struct flb_in_udp_config { void *parser; #endif int collector_id; /* Listener collector id */ + int workers; /* Listener worker count */ + int worker_id; /* Worker id */ + int use_ingress_queue; /* Queue records to main loop */ + int listener_registered; /* Listener event registered */ + struct mk_event listener_event; /* Worker listener event */ + struct mk_event_loop *event_loop; /* Worker event loop */ + struct flb_net_setup net_setup; /* Worker network setup */ struct flb_downstream *downstream; /* Client manager */ struct udp_conn *dummy_conn; /* Datagram dummy connection */ struct flb_input_instance *ins; /* Input plugin instace */ struct flb_log_event_encoder *log_encoder; + struct flb_downstream_worker_runtime *runtime; }; +static inline int udp_ingest_logs(struct flb_in_udp_config *ctx, + const void *buf, size_t buf_size) +{ + if (ctx->use_ingress_queue == FLB_TRUE) { + return flb_input_ingress_queue_log(ctx->ins, NULL, 0, buf, buf_size); + } + + return flb_input_log_append(ctx->ins, NULL, 0, buf, buf_size); +} + #endif diff --git a/plugins/in_udp/udp_config.c b/plugins/in_udp/udp_config.c index c4468aedf6c..d2cb69e5a32 100644 --- a/plugins/in_udp/udp_config.c +++ b/plugins/in_udp/udp_config.c @@ -43,6 +43,7 @@ struct flb_in_udp_config *udp_config_init(struct flb_input_instance *ins) } ctx->ins = ins; ctx->format = FLB_UDP_FMT_JSON; + ctx->workers = 1; /* Load the config map */ ret = flb_input_config_map_set(ins, (void *)ctx); @@ -168,6 +169,11 @@ int udp_config_destroy(struct flb_in_udp_config *ctx) ctx->collector_id = -1; } + if (ctx->listener_registered == FLB_TRUE && ctx->event_loop != NULL) { + mk_event_del(ctx->event_loop, &ctx->listener_event); + ctx->listener_registered = FLB_FALSE; + } + if (ctx->downstream != NULL) { flb_downstream_destroy(ctx->downstream); } diff --git a/plugins/in_udp/udp_conn.c b/plugins/in_udp/udp_conn.c index 216ff9fa875..1ea44f0bb86 100644 --- a/plugins/in_udp/udp_conn.c +++ b/plugins/in_udp/udp_conn.c @@ -114,6 +114,7 @@ static inline int process_pack(struct udp_conn *conn, int len; ctx = conn->ctx; + ret = FLB_EVENT_ENCODER_SUCCESS; flb_log_event_encoder_reset(ctx->log_encoder); @@ -210,10 +211,19 @@ static inline int process_pack(struct udp_conn *conn, msgpack_unpacked_destroy(&result); if (ret == FLB_EVENT_ENCODER_SUCCESS) { - flb_input_log_append(conn->ins, NULL, 0, - ctx->log_encoder->output_buffer, - ctx->log_encoder->output_length); - ret = 0; + if (ctx->log_encoder->output_length > 0) { + ret = udp_ingest_logs(ctx, + ctx->log_encoder->output_buffer, + ctx->log_encoder->output_length); + if (ret != 0) { + flb_plg_error(ctx->ins, + "could not append UDP logs for %s:%s. ret=%d", + ctx->listen, ctx->port, ret); + return -1; + } + } + + return 0; } else { flb_plg_error(ctx->ins, "log event encoding error : %d", ret); @@ -248,9 +258,13 @@ static ssize_t parse_payload_json(struct udp_conn *conn) } /* Process the packaged JSON and return the last byte used */ - process_pack(conn, pack, out_size); + ret = process_pack(conn, pack, out_size); flb_free(pack); + if (ret != 0) { + return -1; + } + return conn->pack_state.last_byte; } @@ -441,9 +455,17 @@ static ssize_t parse_payload_none(struct udp_conn *conn) } if (ret == FLB_EVENT_ENCODER_SUCCESS) { - flb_input_log_append(conn->ins, NULL, 0, - ctx->log_encoder->output_buffer, - ctx->log_encoder->output_length); + if (ctx->log_encoder->output_length > 0) { + ret = udp_ingest_logs(ctx, + ctx->log_encoder->output_buffer, + ctx->log_encoder->output_length); + if (ret != 0) { + flb_plg_error(ctx->ins, + "could not append UDP logs for %s:%s. ret=%d", + ctx->listen, ctx->port, ret); + return -1; + } + } } else { flb_plg_error(ctx->ins, "log event encoding error : %d", ret); diff --git a/plugins/in_unix_socket/unix_socket_conn.c b/plugins/in_unix_socket/unix_socket_conn.c index f65ced31e65..ef287cda87e 100644 --- a/plugins/in_unix_socket/unix_socket_conn.c +++ b/plugins/in_unix_socket/unix_socket_conn.c @@ -269,6 +269,10 @@ int unix_socket_conn_event(void *data) (void *) &conn->buf_data[conn->buf_len], available); + if (flb_io_net_is_retry(bytes)) { + return 0; + } + if (bytes <= 0) { if (!ctx->dgram_mode_flag) { flb_plg_trace(ctx->ins, "fd=%i closed connection", event->fd); diff --git a/plugins/in_winevtlog/in_winevtlog.c b/plugins/in_winevtlog/in_winevtlog.c index 3a57e46e2df..233f259558c 100644 --- a/plugins/in_winevtlog/in_winevtlog.c +++ b/plugins/in_winevtlog/in_winevtlog.c @@ -171,6 +171,7 @@ static int in_winevtlog_init(struct flb_input_instance *in, return -1; } ctx->ins = in; + mk_list_init(&ctx->event_templates); ctx->log_encoder = flb_log_event_encoder_create(FLB_LOG_EVENT_FORMAT_DEFAULT); @@ -207,6 +208,21 @@ static int in_winevtlog_init(struct flb_input_instance *in, } } + if (ctx->event_data_as_map && ctx->event_template_cache_size < 1) { + flb_plg_error(in, + "event_template_cache_size must be greater than zero when " + "event_data_as_map is enabled"); + flb_log_event_encoder_destroy(ctx->log_encoder); + flb_free(ctx); + return -1; + } + + if (ctx->event_data_as_map) { + flb_plg_debug(in, + "EventData named maps enabled; template cache size=%d", + ctx->event_template_cache_size); + } + if (ctx->backoff_multiplier_str && ctx->backoff_multiplier_str[0] != '\0') { mult = atof(ctx->backoff_multiplier_str); if (mult <= 0.0) { @@ -351,6 +367,26 @@ static int in_winevtlog_init(struct flb_input_instance *in, } } + if (ctx->event_data_as_map) { + ctx->event_template_cache = flb_hash_table_create( + FLB_HASH_TABLE_EVICT_NONE, + (size_t) ctx->event_template_cache_size, + ctx->event_template_cache_size); + if (ctx->event_template_cache == NULL) { + flb_plg_error(ctx->ins, "could not create the event template cache"); + if (ctx->db) { + flb_sqldb_close(ctx->db); + } + winevtlog_close_all(ctx->active_channel); + if (ctx->session) { + in_winevtlog_session_destroy(ctx->session); + } + flb_log_event_encoder_destroy(ctx->log_encoder); + flb_free(ctx); + return -1; + } + } + /* Set the context */ flb_input_set_context(in, ctx); @@ -443,6 +479,7 @@ static int in_winevtlog_exit(void *data, struct flb_config *config) if (ctx->session) { in_winevtlog_session_destroy(ctx->session); } + winevtlog_event_template_cache_destroy(ctx); flb_free(ctx); return 0; @@ -474,6 +511,16 @@ static struct flb_config_map config_map[] = { 0, FLB_TRUE, offsetof(struct winevtlog_config, string_inserts), "Whether to include StringInserts in output records" }, + { + FLB_CONFIG_MAP_BOOL, "event_data_as_map", "false", + 0, FLB_TRUE, offsetof(struct winevtlog_config, event_data_as_map), + "Emit EventData as a named map using provider metadata instead of StringInserts" + }, + { + FLB_CONFIG_MAP_INT, "event_template_cache_size", "256", + 0, FLB_TRUE, offsetof(struct winevtlog_config, event_template_cache_size), + "Maximum number of provider event templates cached for event_data_as_map" + }, { FLB_CONFIG_MAP_BOOL, "read_existing_events", "false", 0, FLB_TRUE, offsetof(struct winevtlog_config, read_existing_events), diff --git a/plugins/in_winevtlog/pack.c b/plugins/in_winevtlog/pack.c index 6182151ffea..3a64e2f25e9 100644 --- a/plugins/in_winevtlog/pack.c +++ b/plugins/in_winevtlog/pack.c @@ -725,6 +725,149 @@ static int pack_sid(struct winevtlog_config *ctx, PSID sid, int extract_sid) #undef MAX_NAME } +static int pack_event_value(struct winevtlog_config *ctx, PEVT_VARIANT value) +{ + if (value->Type & EVT_VARIANT_TYPE_ARRAY) { + return -1; + } + + switch (value->Type & EVT_VARIANT_TYPE_MASK) { + case EvtVarTypeNull: + return pack_nullstr(ctx); + case EvtVarTypeString: + if (pack_wstr(ctx, value->StringVal)) { + return pack_nullstr(ctx); + } + return FLB_EVENT_ENCODER_SUCCESS; + case EvtVarTypeAnsiString: + if (pack_astr(ctx, value->AnsiStringVal)) { + return pack_nullstr(ctx); + } + return FLB_EVENT_ENCODER_SUCCESS; + case EvtVarTypeSByte: + return flb_log_event_encoder_append_body_int8(ctx->log_encoder, + value->SByteVal); + case EvtVarTypeByte: + return flb_log_event_encoder_append_body_uint8(ctx->log_encoder, + value->ByteVal); + case EvtVarTypeInt16: + return flb_log_event_encoder_append_body_int16(ctx->log_encoder, + value->Int16Val); + case EvtVarTypeUInt16: + return flb_log_event_encoder_append_body_uint16(ctx->log_encoder, + value->UInt16Val); + case EvtVarTypeInt32: + return flb_log_event_encoder_append_body_int32(ctx->log_encoder, + value->Int32Val); + case EvtVarTypeUInt32: + return flb_log_event_encoder_append_body_uint32(ctx->log_encoder, + value->UInt32Val); + case EvtVarTypeInt64: + return flb_log_event_encoder_append_body_int64(ctx->log_encoder, + value->Int64Val); + case EvtVarTypeUInt64: + return flb_log_event_encoder_append_body_uint64(ctx->log_encoder, + value->UInt64Val); + case EvtVarTypeSingle: + return flb_log_event_encoder_append_body_double(ctx->log_encoder, + value->SingleVal); + case EvtVarTypeDouble: + return flb_log_event_encoder_append_body_double(ctx->log_encoder, + value->DoubleVal); + case EvtVarTypeBoolean: + return flb_log_event_encoder_append_body_boolean(ctx->log_encoder, + (int) value->BooleanVal); + case EvtVarTypeGuid: + if (pack_guid(ctx, value->GuidVal)) { + return pack_nullstr(ctx); + } + return FLB_EVENT_ENCODER_SUCCESS; + case EvtVarTypeSizeT: + return flb_log_event_encoder_append_body_uint64(ctx->log_encoder, + value->SizeTVal); + case EvtVarTypeFileTime: + if (pack_filetime(ctx, value->FileTimeVal)) { + return pack_nullstr(ctx); + } + return FLB_EVENT_ENCODER_SUCCESS; + case EvtVarTypeSysTime: + if (pack_systemtime(ctx, value->SysTimeVal)) { + return pack_nullstr(ctx); + } + return FLB_EVENT_ENCODER_SUCCESS; + case EvtVarTypeSid: + if (pack_sid(ctx, value->SidVal, FLB_FALSE)) { + return pack_nullstr(ctx); + } + return FLB_EVENT_ENCODER_SUCCESS; + case EvtVarTypeHexInt32: + if (pack_hex32(ctx, value->Int32Val)) { + return pack_nullstr(ctx); + } + return FLB_EVENT_ENCODER_SUCCESS; + case EvtVarTypeHexInt64: + if (pack_hex64(ctx, value->Int64Val)) { + return pack_nullstr(ctx); + } + return FLB_EVENT_ENCODER_SUCCESS; + case EvtVarTypeEvtXml: + if (pack_wstr(ctx, value->XmlVal)) { + return pack_nullstr(ctx); + } + return FLB_EVENT_ENCODER_SUCCESS; + case EvtVarTypeBinary: + if (pack_binary(ctx, value->BinaryVal, value->Count)) { + return pack_nullstr(ctx); + } + return FLB_EVENT_ENCODER_SUCCESS; + default: + return flb_log_event_encoder_append_body_cstring(ctx->log_encoder, "?"); + } +} + +static int event_data_map_is_compatible( + struct winevtlog_event_template *event_template, + PEVT_VARIANT values, UINT count) +{ + UINT index; + + if (event_template == NULL || !event_template->valid || + event_template->data_count != count) { + return FLB_FALSE; + } + + for (index = 0; index < count; index++) { + if (values[index].Type & EVT_VARIANT_TYPE_ARRAY) { + return FLB_FALSE; + } + } + + return FLB_TRUE; +} + +static int pack_event_data_map(struct winevtlog_config *ctx, + struct winevtlog_event_template *event_template, + PEVT_VARIANT values, UINT count) +{ + int ret; + UINT index; + + ret = flb_log_event_encoder_body_begin_map(ctx->log_encoder); + + for (index = 0; ret == FLB_EVENT_ENCODER_SUCCESS && index < count; index++) { + ret = pack_wstr(ctx, event_template->data_names[index]); + if (ret == FLB_EVENT_ENCODER_SUCCESS) { + ret = pack_event_value(ctx, &values[index]); + } + } + + if (ret == FLB_EVENT_ENCODER_SUCCESS) { + ret = flb_log_event_encoder_body_commit_map(ctx->log_encoder); + } + + return ret; +} + static void pack_string_inserts(struct winevtlog_config *ctx, PEVT_VARIANT values, DWORD count) { int i; @@ -737,98 +880,9 @@ static void pack_string_inserts(struct winevtlog_config *ctx, PEVT_VARIANT value continue; } - switch (values[i].Type & EVT_VARIANT_TYPE_MASK) { - case EvtVarTypeNull: - pack_nullstr(ctx); - break; - case EvtVarTypeString: - if (pack_wstr(ctx, values[i].StringVal)) { - pack_nullstr(ctx); - } - break; - case EvtVarTypeAnsiString: - if (pack_astr(ctx, values[i].AnsiStringVal)) { - pack_nullstr(ctx); - } - break; - case EvtVarTypeSByte: - flb_log_event_encoder_append_body_int8(ctx->log_encoder, values[i].SByteVal); - break; - case EvtVarTypeByte: - flb_log_event_encoder_append_body_uint8(ctx->log_encoder, values[i].ByteVal); - break; - case EvtVarTypeInt16: - flb_log_event_encoder_append_body_int16(ctx->log_encoder, values[i].Int16Val); - break; - case EvtVarTypeUInt16: - flb_log_event_encoder_append_body_uint16(ctx->log_encoder, values[i].UInt16Val); - break; - case EvtVarTypeInt32: - flb_log_event_encoder_append_body_int32(ctx->log_encoder, values[i].Int32Val); - break; - case EvtVarTypeUInt32: - flb_log_event_encoder_append_body_uint32(ctx->log_encoder, values[i].UInt32Val); - break; - case EvtVarTypeInt64: - flb_log_event_encoder_append_body_int64(ctx->log_encoder, values[i].Int64Val); - break; - case EvtVarTypeUInt64: - flb_log_event_encoder_append_body_uint64(ctx->log_encoder, values[i].UInt64Val); - break; - case EvtVarTypeSingle: - flb_log_event_encoder_append_body_double(ctx->log_encoder, values[i].SingleVal); - break; - case EvtVarTypeDouble: - flb_log_event_encoder_append_body_double(ctx->log_encoder, values[i].DoubleVal); - break; - case EvtVarTypeBoolean: - flb_log_event_encoder_append_body_boolean(ctx->log_encoder, (int) values[i].BooleanVal); - break; - case EvtVarTypeGuid: - if (pack_guid(ctx, values[i].GuidVal)) { - pack_nullstr(ctx); - } - break; - case EvtVarTypeSizeT: - flb_log_event_encoder_append_body_uint64(ctx->log_encoder, values[i].SizeTVal); - break; - case EvtVarTypeFileTime: - if (pack_filetime(ctx, values[i].FileTimeVal)) { - pack_nullstr(ctx); - } - break; - case EvtVarTypeSysTime: - if (pack_systemtime(ctx, values[i].SysTimeVal)) { - pack_nullstr(ctx); - } - break; - case EvtVarTypeSid: - if (pack_sid(ctx, values[i].SidVal, FLB_FALSE)) { - pack_nullstr(ctx); - } - break; - case EvtVarTypeHexInt32: - if (pack_hex32(ctx, values[i].Int32Val)) { - pack_nullstr(ctx); - } - break; - case EvtVarTypeHexInt64: - if (pack_hex64(ctx, values[i].Int64Val)) { - pack_nullstr(ctx); - } - break; - case EvtVarTypeEvtXml: - if (pack_wstr(ctx, values[i].XmlVal)) { - pack_nullstr(ctx); - } - break; - case EvtVarTypeBinary: - if (pack_binary(ctx, values[i].BinaryVal, values[i].Count)) { - pack_nullstr(ctx); - } - break; - default: - flb_log_event_encoder_append_body_cstring(ctx->log_encoder, "?"); + ret = pack_event_value(ctx, &values[i]); + if (ret != FLB_EVENT_ENCODER_SUCCESS) { + ret = pack_nullstr(ctx); } } @@ -838,8 +892,42 @@ static void pack_string_inserts(struct winevtlog_config *ctx, PEVT_VARIANT value } +static int pack_event_data(struct winevtlog_config *ctx, + struct winevtlog_event_template *event_template, + PEVT_VARIANT string_inserts, UINT count_inserts) +{ + int ret; + + if (ctx->event_data_as_map && + event_data_map_is_compatible(event_template, string_inserts, + count_inserts)) { + ret = flb_log_event_encoder_append_body_cstring(ctx->log_encoder, + "EventData"); + if (ret == FLB_EVENT_ENCODER_SUCCESS) { + ret = pack_event_data_map(ctx, event_template, string_inserts, + count_inserts); + } + if (ret != FLB_EVENT_ENCODER_SUCCESS || !ctx->string_inserts) { + return ret; + } + } + + if (ctx->string_inserts || ctx->event_data_as_map) { + ret = flb_log_event_encoder_append_body_cstring(ctx->log_encoder, + "StringInserts"); + if (ret == FLB_EVENT_ENCODER_SUCCESS) { + pack_string_inserts(ctx, string_inserts, count_inserts); + } + return ret; + } + + return FLB_EVENT_ENCODER_SUCCESS; +} + void winevtlog_pack_xml_event(WCHAR *system_xml, WCHAR *message, - PEVT_VARIANT string_inserts, UINT count_inserts, struct winevtlog_channel *ch, + PEVT_VARIANT string_inserts, UINT count_inserts, + struct winevtlog_event_template *event_template, + struct winevtlog_channel *ch, struct winevtlog_config *ctx) { int ret; @@ -863,10 +951,9 @@ void winevtlog_pack_xml_event(WCHAR *system_xml, WCHAR *message, pack_nullstr(ctx); } - if (ctx->string_inserts) { - ret = flb_log_event_encoder_append_body_cstring(ctx->log_encoder, "StringInserts"); - - pack_string_inserts(ctx, string_inserts, count_inserts); + if (ret == FLB_EVENT_ENCODER_SUCCESS) { + ret = pack_event_data(ctx, event_template, string_inserts, + count_inserts); } if (ret == FLB_EVENT_ENCODER_SUCCESS) { @@ -876,6 +963,7 @@ void winevtlog_pack_xml_event(WCHAR *system_xml, WCHAR *message, void winevtlog_pack_text_event(PEVT_VARIANT system, WCHAR *message, PEVT_VARIANT string_inserts, UINT count_inserts, + struct winevtlog_event_template *event_template, struct winevtlog_channel *ch, struct winevtlog_config *ctx) { int ret; @@ -1099,16 +1187,10 @@ void winevtlog_pack_text_event(PEVT_VARIANT system, WCHAR *message, ret = flb_log_event_encoder_append_body_string(ctx->log_encoder, text, out_len); } - /* StringInserts must not be embedded in the key=value text payload. When both - * render_event_as_text and string_inserts are enabled, we expose inserts as a - * structured field under the "StringInserts" key to preserve record-level - * fidelity and avoid mixing formats in TextFormat output. - */ - if (ret == FLB_EVENT_ENCODER_SUCCESS && ctx->string_inserts) { - ret = flb_log_event_encoder_append_body_cstring(ctx->log_encoder, "StringInserts"); - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - pack_string_inserts(ctx, string_inserts, count_inserts); - } + /* Event data remains structured instead of being embedded in TextFormat. */ + if (ret == FLB_EVENT_ENCODER_SUCCESS) { + ret = pack_event_data(ctx, event_template, string_inserts, + count_inserts); } if (ret == FLB_EVENT_ENCODER_SUCCESS) { @@ -1119,16 +1201,12 @@ void winevtlog_pack_text_event(PEVT_VARIANT system, WCHAR *message, } void winevtlog_pack_event(PEVT_VARIANT system, WCHAR *message, - PEVT_VARIANT string_inserts, UINT count_inserts, struct winevtlog_channel *ch, + PEVT_VARIANT string_inserts, UINT count_inserts, + struct winevtlog_event_template *event_template, + struct winevtlog_channel *ch, struct winevtlog_config *ctx) { int ret; - size_t len; - int count = 19; - - if (ctx->string_inserts) { - count++; - } ret = flb_log_event_encoder_begin_record(ctx->log_encoder); @@ -1314,11 +1392,9 @@ void winevtlog_pack_event(PEVT_VARIANT system, WCHAR *message, pack_nullstr(ctx); } - /* String Inserts */ - if (ctx->string_inserts) { - ret = flb_log_event_encoder_append_body_cstring(ctx->log_encoder, "StringInserts"); - - pack_string_inserts(ctx, string_inserts, count_inserts); + if (ret == FLB_EVENT_ENCODER_SUCCESS) { + ret = pack_event_data(ctx, event_template, string_inserts, + count_inserts); } if (ret == FLB_EVENT_ENCODER_SUCCESS) { diff --git a/plugins/in_winevtlog/winevtlog.c b/plugins/in_winevtlog/winevtlog.c index f425f306d21..b902867c840 100644 --- a/plugins/in_winevtlog/winevtlog.c +++ b/plugins/in_winevtlog/winevtlog.c @@ -23,6 +23,7 @@ #include #include #include +#include #include "winevtlog.h" #define EVENT_PROVIDER_NAME_LENGTH 256 @@ -199,6 +200,673 @@ static int xml_attribute(const char *tag, const char *tag_end, const char *name, return FLB_FALSE; } +static const wchar_t *wxml_tag_end(const wchar_t *tag) +{ + wchar_t quote = L'\0'; + + while (*tag != L'\0') { + if (quote != L'\0') { + if (*tag == quote) { + quote = L'\0'; + } + } + else if (*tag == L'\'' || *tag == L'"') { + quote = *tag; + } + else if (*tag == L'>') { + return tag + 1; + } + tag++; + } + + return NULL; +} + +static int wxml_tag_is(const wchar_t *tag, const wchar_t *name, int closing) +{ + size_t name_length; + + if (*tag != L'<') { + return FLB_FALSE; + } + tag++; + + if (closing) { + if (*tag != L'/') { + return FLB_FALSE; + } + tag++; + } + else if (*tag == L'/') { + return FLB_FALSE; + } + + while (*tag == L' ' || *tag == L'\t' || *tag == L'\r' || *tag == L'\n') { + tag++; + } + + name_length = wcslen(name); + if (wcsncmp(tag, name, name_length) != 0) { + return FLB_FALSE; + } + + tag += name_length; + return *tag == L'>' || *tag == L'/' || *tag == L' ' || *tag == L'\t' || + *tag == L'\r' || *tag == L'\n'; +} + +static int wxml_attribute(const wchar_t *tag, const wchar_t *tag_end, + const wchar_t *name, const wchar_t **value, + size_t *value_length) +{ + wchar_t quote; + size_t attribute_length; + size_t name_length; + const wchar_t *cursor; + const wchar_t *attribute; + const wchar_t *value_end; + + name_length = wcslen(name); + cursor = tag + 1; + + if (*cursor == L'/') { + cursor++; + } + while (*cursor == L' ' || *cursor == L'\t' || *cursor == L'\r' || *cursor == L'\n') { + cursor++; + } + while (cursor < tag_end && *cursor != L' ' && *cursor != L'\t' && + *cursor != L'\r' && *cursor != L'\n' && *cursor != L'>' && + *cursor != L'/') { + cursor++; + } + + while (cursor < tag_end && *cursor != L'>' && *cursor != L'/') { + while (*cursor == L' ' || *cursor == L'\t' || + *cursor == L'\r' || *cursor == L'\n') { + cursor++; + } + + if (cursor >= tag_end || *cursor == L'>' || *cursor == L'/') { + break; + } + + attribute = cursor; + while (cursor < tag_end && *cursor != L'=' && *cursor != L' ' && + *cursor != L'\t' && *cursor != L'\r' && *cursor != L'\n' && + *cursor != L'>' && *cursor != L'/') { + cursor++; + } + attribute_length = cursor - attribute; + + while (*cursor == L' ' || *cursor == L'\t' || + *cursor == L'\r' || *cursor == L'\n') { + cursor++; + } + if (cursor >= tag_end || *cursor != L'=') { + return FLB_FALSE; + } + + cursor++; + while (*cursor == L' ' || *cursor == L'\t' || + *cursor == L'\r' || *cursor == L'\n') { + cursor++; + } + if (cursor >= tag_end || (*cursor != L'\'' && *cursor != L'"')) { + return FLB_FALSE; + } + + quote = *cursor++; + value_end = cursor; + while (value_end < tag_end && *value_end != quote) { + value_end++; + } + if (value_end >= tag_end) { + return FLB_FALSE; + } + + if (attribute_length == name_length && + wcsncmp(attribute, name, name_length) == 0) { + *value = cursor; + *value_length = value_end - cursor; + return FLB_TRUE; + } + cursor = value_end + 1; + } + + return FLB_FALSE; +} + +static PWSTR wxml_decode_attribute(const wchar_t *value, size_t value_length) +{ + size_t input_index; + size_t output_index = 0; + PWSTR output; + + output = flb_malloc(sizeof(wchar_t) * (value_length + 1)); + if (output == NULL) { + flb_errno(); + return NULL; + } + + for (input_index = 0; input_index < value_length; input_index++) { + if (value[input_index] != L'&') { + output[output_index++] = value[input_index]; + continue; + } + + if (input_index + 5 <= value_length && + wcsncmp(&value[input_index], L"&", 5) == 0) { + output[output_index++] = L'&'; + input_index += 4; + } + else if (input_index + 4 <= value_length && + wcsncmp(&value[input_index], L"<", 4) == 0) { + output[output_index++] = L'<'; + input_index += 3; + } + else if (input_index + 4 <= value_length && + wcsncmp(&value[input_index], L">", 4) == 0) { + output[output_index++] = L'>'; + input_index += 3; + } + else if (input_index + 6 <= value_length && + wcsncmp(&value[input_index], L""", 6) == 0) { + output[output_index++] = L'"'; + input_index += 5; + } + else if (input_index + 6 <= value_length && + wcsncmp(&value[input_index], L"'", 6) == 0) { + output[output_index++] = L'\''; + input_index += 5; + } + else { + flb_free(output); + return NULL; + } + } + + output[output_index] = L'\0'; + return output; +} + +static void winevtlog_event_template_destroy(struct winevtlog_event_template *event_template) +{ + UINT index; + + if (event_template == NULL) { + return; + } + + for (index = 0; index < event_template->data_count; index++) { + flb_free(event_template->data_names[index]); + } + flb_free(event_template->data_names); + flb_free(event_template->cache_key); + flb_free(event_template->provider_name); + flb_free(event_template); +} + +void winevtlog_event_template_cache_destroy(struct winevtlog_config *ctx) +{ + struct winevtlog_event_template *event_template; + struct mk_list *head; + struct mk_list *tmp; + + if (ctx == NULL) { + return; + } + + if (ctx->event_template_cache != NULL) { + flb_hash_table_destroy(ctx->event_template_cache); + ctx->event_template_cache = NULL; + } + + mk_list_foreach_safe(head, tmp, &ctx->event_templates) { + event_template = mk_list_entry(head, + struct winevtlog_event_template, + _head); + mk_list_del(&event_template->_head); + winevtlog_event_template_destroy(event_template); + } +} + +static char *event_template_cache_key(PCWSTR provider_name, DWORD event_id, + DWORD version, size_t *key_length) +{ + int provider_length; + int suffix_length; + size_t provider_utf8_length; + char suffix[32]; + char *key; + + provider_length = WideCharToMultiByte(CP_UTF8, WC_ERR_INVALID_CHARS, + provider_name, -1, NULL, 0, NULL, + NULL); + if (provider_length == 0) { + return NULL; + } + + suffix_length = _snprintf_s(suffix, sizeof(suffix), _TRUNCATE, + "|%lu|%lu", event_id, version); + if (suffix_length <= 0) { + return NULL; + } + + provider_utf8_length = (size_t) provider_length - 1; + key = flb_malloc(provider_utf8_length + (size_t) suffix_length + 1); + if (key == NULL) { + flb_errno(); + return NULL; + } + + if (WideCharToMultiByte(CP_UTF8, WC_ERR_INVALID_CHARS, provider_name, -1, + key, provider_length, NULL, NULL) == 0) { + flb_free(key); + return NULL; + } + + memcpy(&key[provider_utf8_length], suffix, (size_t) suffix_length + 1); + *key_length = provider_utf8_length + (size_t) suffix_length; + return key; +} + +static struct winevtlog_event_template *event_template_cache_find( + struct winevtlog_config *ctx, PCWSTR provider_name, + DWORD event_id, DWORD version) +{ + char *cache_key; + size_t cache_key_length; + size_t cache_value_size; + void *cache_value; + struct winevtlog_event_template *event_template; + + if (ctx->event_template_cache == NULL) { + return NULL; + } + + cache_key = event_template_cache_key(provider_name, event_id, version, + &cache_key_length); + if (cache_key == NULL) { + return NULL; + } + + if (flb_hash_table_get(ctx->event_template_cache, cache_key, + (int) cache_key_length, &cache_value, + &cache_value_size) < 0) { + flb_free(cache_key); + return NULL; + } + + flb_free(cache_key); + event_template = cache_value; + mk_list_del(&event_template->_head); + mk_list_add(&event_template->_head, &ctx->event_templates); + return event_template; +} + +static int event_template_cache_evict(struct winevtlog_config *ctx) +{ + struct winevtlog_event_template *event_template; + + while (mk_list_size(&ctx->event_templates) >= + ctx->event_template_cache_size) { + event_template = mk_list_entry_first(&ctx->event_templates, + struct winevtlog_event_template, + _head); + if (flb_hash_table_del_ptr(ctx->event_template_cache, + event_template->cache_key, + (int) event_template->cache_key_length, + event_template) != 0) { + return -1; + } + mk_list_del(&event_template->_head); + winevtlog_event_template_destroy(event_template); + } + + return 0; +} + +static struct winevtlog_event_template *event_template_cache_create( + struct winevtlog_config *ctx, PCWSTR provider_name, + DWORD event_id, DWORD version) +{ + size_t provider_name_length; + struct winevtlog_event_template *event_template; + + event_template = flb_calloc(1, sizeof(struct winevtlog_event_template)); + if (event_template == NULL) { + flb_errno(); + return NULL; + } + + provider_name_length = wcslen(provider_name); + event_template->provider_name = flb_malloc(sizeof(wchar_t) * + (provider_name_length + 1)); + if (event_template->provider_name == NULL) { + flb_errno(); + winevtlog_event_template_destroy(event_template); + return NULL; + } + + memcpy(event_template->provider_name, provider_name, + sizeof(wchar_t) * (provider_name_length + 1)); + event_template->event_id = event_id; + event_template->version = version; + + event_template->cache_key = event_template_cache_key(provider_name, + event_id, version, + &event_template->cache_key_length); + if (event_template->cache_key == NULL || + event_template_cache_evict(ctx) != 0 || + flb_hash_table_add(ctx->event_template_cache, + event_template->cache_key, + (int) event_template->cache_key_length, + event_template, 0) < 0) { + winevtlog_event_template_destroy(event_template); + return NULL; + } + + mk_list_add(&event_template->_head, &ctx->event_templates); + + return event_template; +} + +static int event_template_append_name(struct winevtlog_event_template *event_template, + PWSTR name) +{ + PWSTR *new_names; + UINT index; + + if (name == NULL || name[0] == L'\0') { + return -1; + } + + for (index = 0; index < event_template->data_count; index++) { + if (wcscmp(event_template->data_names[index], name) == 0) { + return -1; + } + } + + new_names = flb_realloc(event_template->data_names, + sizeof(PWSTR) * (event_template->data_count + 1)); + if (new_names == NULL) { + flb_errno(); + return -1; + } + + event_template->data_names = new_names; + event_template->data_names[event_template->data_count++] = name; + return 0; +} + +static int event_template_parse_data_names( + struct winevtlog_event_template *event_template, + PCWSTR template_xml) +{ + const wchar_t *cursor; + const wchar_t *tag_end; + const wchar_t *name; + size_t name_length; + PWSTR decoded_name; + + if (template_xml == NULL) { + return -1; + } + + cursor = template_xml; + while ((cursor = wcschr(cursor, L'<')) != NULL) { + tag_end = wxml_tag_end(cursor); + if (tag_end == NULL) { + return -1; + } + + if (wxml_tag_is(cursor, L"data", FLB_FALSE)) { + if (!wxml_attribute(cursor, tag_end, L"name", &name, &name_length)) { + return -1; + } + + decoded_name = wxml_decode_attribute(name, name_length); + if (decoded_name == NULL || + event_template_append_name(event_template, decoded_name) != 0) { + flb_free(decoded_name); + return -1; + } + } + cursor = tag_end; + } + + return 0; +} + +static int event_metadata_get_uint32(EVT_HANDLE event_metadata, + EVT_EVENT_METADATA_PROPERTY_ID property_id, + DWORD *value) +{ + DWORD buffer_size = 0; + DWORD buffer_used = 0; + PEVT_VARIANT property = NULL; + DWORD status; + int result = -1; + + if (EvtGetEventMetadataProperty(event_metadata, property_id, 0, 0, NULL, + &buffer_size)) { + return -1; + } + + status = GetLastError(); + if (status != ERROR_INSUFFICIENT_BUFFER) { + return -1; + } + + property = flb_malloc(buffer_size); + if (property == NULL) { + flb_errno(); + return -1; + } + + if (!EvtGetEventMetadataProperty(event_metadata, property_id, 0, + buffer_size, property, &buffer_used) || + (property->Type & EVT_VARIANT_TYPE_MASK) != EvtVarTypeUInt32) { + goto cleanup; + } + + *value = property->UInt32Val; + result = 0; + +cleanup: + flb_free(property); + return result; +} + +static PWSTR event_metadata_get_template(EVT_HANDLE event_metadata) +{ + DWORD buffer_size = 0; + DWORD buffer_used = 0; + PEVT_VARIANT property = NULL; + PWSTR template_xml = NULL; + size_t template_length; + DWORD status; + + if (EvtGetEventMetadataProperty(event_metadata, + EventMetadataEventTemplate, + 0, 0, NULL, &buffer_size)) { + return NULL; + } + + status = GetLastError(); + if (status != ERROR_INSUFFICIENT_BUFFER) { + return NULL; + } + + property = flb_malloc(buffer_size); + if (property == NULL) { + flb_errno(); + return NULL; + } + + if (!EvtGetEventMetadataProperty(event_metadata, + EventMetadataEventTemplate, + 0, buffer_size, property, &buffer_used) || + (property->Type & EVT_VARIANT_TYPE_MASK) != EvtVarTypeString || + property->StringVal == NULL) { + goto cleanup; + } + + template_length = wcslen(property->StringVal); + template_xml = flb_malloc(sizeof(wchar_t) * (template_length + 1)); + if (template_xml == NULL) { + flb_errno(); + goto cleanup; + } + + memcpy(template_xml, property->StringVal, + sizeof(wchar_t) * (template_length + 1)); + +cleanup: + flb_free(property); + return template_xml; +} + +static void event_template_load(struct winevtlog_event_template *event_template, + EVT_HANDLE remote, + struct winevtlog_config *ctx) +{ + DWORD event_id; + DWORD version; + DWORD status; + EVT_HANDLE event_metadata = NULL; + EVT_HANDLE event_metadata_enum = NULL; + EVT_HANDLE publisher_metadata = NULL; + PWSTR template_xml = NULL; + + publisher_metadata = EvtOpenPublisherMetadata( + remote, event_template->provider_name, NULL, + MAKELCID(LANG_NEUTRAL, SORT_DEFAULT), 0); + if (publisher_metadata == NULL) { + flb_plg_debug(ctx->ins, + "could not open publisher metadata for event data map: %lu", + GetLastError()); + goto cleanup; + } + + event_metadata_enum = EvtOpenEventMetadataEnum(publisher_metadata, 0); + if (event_metadata_enum == NULL) { + flb_plg_debug(ctx->ins, + "could not enumerate publisher event metadata: %lu", + GetLastError()); + goto cleanup; + } + + while (FLB_TRUE) { + event_metadata = EvtNextEventMetadata(event_metadata_enum, 0); + if (event_metadata == NULL) { + status = GetLastError(); + if (status != ERROR_NO_MORE_ITEMS) { + flb_plg_debug(ctx->ins, + "could not read publisher event metadata: %lu", + status); + } + break; + } + + if (event_metadata_get_uint32(event_metadata, EventMetadataEventID, + &event_id) == 0 && + event_metadata_get_uint32(event_metadata, EventMetadataEventVersion, + &version) == 0 && + event_id == event_template->event_id && + version == event_template->version) { + template_xml = event_metadata_get_template(event_metadata); + if (template_xml == NULL) { + flb_plg_debug(ctx->ins, + "could not read event metadata template for event %lu", + event_template->event_id); + break; + } + + if (event_template_parse_data_names(event_template, + template_xml) == 0 && + event_template->data_count > 0) { + event_template->valid = FLB_TRUE; + flb_plg_debug(ctx->ins, + "loaded EventData template for %ls, event %lu version %lu: %u fields", + event_template->provider_name, + event_template->event_id, + event_template->version, + event_template->data_count); + } + else { + flb_plg_debug(ctx->ins, + "EventData template has no usable names for %ls, event %lu version %lu", + event_template->provider_name, + event_template->event_id, + event_template->version); + } + break; + } + + EvtClose(event_metadata); + event_metadata = NULL; + } + +cleanup: + if (template_xml != NULL) { + flb_free(template_xml); + } + if (event_metadata != NULL) { + EvtClose(event_metadata); + } + if (event_metadata_enum != NULL) { + EvtClose(event_metadata_enum); + } + if (publisher_metadata != NULL) { + EvtClose(publisher_metadata); + } +} + +struct winevtlog_event_template *winevtlog_event_template_get( + PEVT_VARIANT system, EVT_HANDLE remote, + struct winevtlog_config *ctx) +{ + DWORD event_id; + DWORD version; + PCWSTR provider_name; + struct winevtlog_event_template *event_template; + + if (ctx == NULL || system == NULL || + (system[EvtSystemProviderName].Type & EVT_VARIANT_TYPE_MASK) != + EvtVarTypeString || + system[EvtSystemProviderName].StringVal == NULL || + (system[EvtSystemEventID].Type & EVT_VARIANT_TYPE_MASK) != + EvtVarTypeUInt16) { + return NULL; + } + + provider_name = system[EvtSystemProviderName].StringVal; + event_id = system[EvtSystemEventID].UInt16Val; + version = 0; + + if ((system[EvtSystemVersion].Type & EVT_VARIANT_TYPE_MASK) == + EvtVarTypeByte) { + version = system[EvtSystemVersion].ByteVal; + } + + event_template = event_template_cache_find(ctx, provider_name, + event_id, version); + if (event_template != NULL) { + return event_template; + } + + event_template = event_template_cache_create(ctx, provider_name, + event_id, version); + if (event_template == NULL) { + return NULL; + } + + event_template_load(event_template, remote, ctx); + return event_template; +} + static int path_matches_channel(const char *path, size_t path_length, const char *channel) { @@ -992,12 +1660,21 @@ PWSTR get_description(EVT_HANDLE handle, LANGID langID, unsigned int *message_si int get_string_inserts(EVT_HANDLE handle, PEVT_VARIANT *string_inserts_values, UINT *prop_count, unsigned int *string_inserts_size) { - PEVT_VARIANT values; + PEVT_VARIANT values = NULL; DWORD buffer_size = 0; DWORD buffer_size_used = 0; DWORD count = 0; BOOL succeeded = FLB_TRUE; + if (string_inserts_values == NULL || prop_count == NULL || + string_inserts_size == NULL) { + return FLB_FALSE; + } + + *string_inserts_values = NULL; + *prop_count = 0; + *string_inserts_size = 0; + EVT_HANDLE context = EvtCreateRenderContext(0, NULL, EvtRenderContextUser); if (context == NULL) { flb_error("Failed to create renderContext"); @@ -1006,8 +1683,27 @@ int get_string_inserts(EVT_HANDLE handle, PEVT_VARIANT *string_inserts_values, } // Get the size of the buffer - EvtRender(context, handle, EvtRenderEventValues, 0, NULL, &buffer_size, &count); + if (EvtRender(context, handle, EvtRenderEventValues, 0, NULL, + &buffer_size, &count)) { + if (count == 0) { + goto cleanup; + } + flb_error("Unexpected successful string inserts size query"); + succeeded = FLB_FALSE; + goto cleanup; + } + if (GetLastError() != ERROR_INSUFFICIENT_BUFFER) { + flb_error("Failed to get string inserts size with %d", GetLastError()); + succeeded = FLB_FALSE; + goto cleanup; + } + values = (PEVT_VARIANT)flb_malloc(buffer_size); + if (values == NULL) { + flb_errno(); + succeeded = FLB_FALSE; + goto cleanup; + } succeeded = EvtRender(context, handle, @@ -1018,7 +1714,7 @@ int get_string_inserts(EVT_HANDLE handle, PEVT_VARIANT *string_inserts_values, &count); if (!succeeded) { - flb_error("Failed to get string inserts with %d\n", GetLastError()); + flb_error("Failed to get string inserts with %d", GetLastError()); goto cleanup; } @@ -1028,6 +1724,10 @@ int get_string_inserts(EVT_HANDLE handle, PEVT_VARIANT *string_inserts_values, cleanup: + if (!succeeded && values != NULL) { + flb_free(values); + } + if (context != NULL) { EvtClose(context); } @@ -1299,6 +1999,7 @@ int winevtlog_read(struct winevtlog_channel *ch, struct winevtlog_config *ctx, DWORD status = ERROR_SUCCESS; PWSTR system_xml = NULL; unsigned int system_size = 0; + unsigned int system_xml_size = 0; unsigned int message_size = 0; unsigned int string_inserts_size = 0; int hit_threshold = FLB_FALSE; @@ -1306,39 +2007,74 @@ int winevtlog_read(struct winevtlog_channel *ch, struct winevtlog_config *ctx, PWSTR message = NULL; PEVT_VARIANT rendered_system = NULL; PEVT_VARIANT string_inserts = NULL; + struct winevtlog_event_template *event_template = NULL; UINT count_inserts = 0; DWORD i = 0; int rc = 0; + int render_user_data; + + render_user_data = ctx->string_inserts || ctx->event_data_as_map; while (winevtlog_next(ch, hit_threshold)) { for (i = 0; i < ch->count; i++) { + system_xml = NULL; + rendered_system = NULL; + message = NULL; + string_inserts = NULL; + system_size = 0; + system_xml_size = 0; + message_size = 0; + string_inserts_size = 0; + count_inserts = 0; + event_template = NULL; + if (ctx->render_event_as_xml) { - system_xml = render_event(ch->events[i], EvtRenderEventXml, &system_size); + system_xml = render_event(ch->events[i], EvtRenderEventXml, + &system_xml_size); message = get_description(ch->events[i], LANG_NEUTRAL, &message_size, ch->remote); - get_string_inserts(ch->events[i], &string_inserts, &count_inserts, &string_inserts_size); + if (ctx->event_data_as_map) { + render_system_event(ch->events[i], &rendered_system, + &system_size); + if (rendered_system != NULL) { + event_template = winevtlog_event_template_get(rendered_system, + ch->remote, ctx); + } + } + if (render_user_data) { + get_string_inserts(ch->events[i], &string_inserts, + &count_inserts, &string_inserts_size); + } if (system_xml) { - /* Caluculate total allocated size: system + message + string_inserts */ - read_size += (system_size + message_size + string_inserts_size); + /* Calculate allocated size: XML + system + message + inserts. */ + read_size += (system_xml_size + system_size + message_size + + string_inserts_size); winevtlog_pack_xml_event(system_xml, message, string_inserts, - count_inserts, ch, ctx); + count_inserts, event_template, ch, ctx); } flb_free(string_inserts); flb_free(system_xml); + flb_free(rendered_system); if (message) { flb_free(message); } } else if (ctx->render_event_as_text) { - rendered_system = NULL; render_system_event(ch->events[i], &rendered_system, &system_size); message = get_description(ch->events[i], LANG_NEUTRAL, &message_size, ch->remote); - get_string_inserts(ch->events[i], &string_inserts, &count_inserts, &string_inserts_size); + if (ctx->event_data_as_map && rendered_system != NULL) { + event_template = winevtlog_event_template_get(rendered_system, + ch->remote, ctx); + } + if (render_user_data) { + get_string_inserts(ch->events[i], &string_inserts, + &count_inserts, &string_inserts_size); + } if (rendered_system) { - /* Caluculate total allocated size: system + message + string_inserts */ + /* Calculate allocated size: system + message + inserts. */ read_size += (system_size + message_size + string_inserts_size); winevtlog_pack_text_event(rendered_system, message, string_inserts, - count_inserts, ch, ctx); + count_inserts, event_template, ch, ctx); } @@ -1351,12 +2087,19 @@ int winevtlog_read(struct winevtlog_channel *ch, struct winevtlog_config *ctx, else { render_system_event(ch->events[i], &rendered_system, &system_size); message = get_description(ch->events[i], LANG_NEUTRAL, &message_size, ch->remote); - get_string_inserts(ch->events[i], &string_inserts, &count_inserts, &string_inserts_size); + if (ctx->event_data_as_map && rendered_system != NULL) { + event_template = winevtlog_event_template_get(rendered_system, + ch->remote, ctx); + } + if (render_user_data) { + get_string_inserts(ch->events[i], &string_inserts, + &count_inserts, &string_inserts_size); + } if (rendered_system) { - /* Caluculate total allocated size: system + message + string_inserts */ + /* Calculate allocated size: system + message + inserts. */ read_size += (system_size + message_size + string_inserts_size); winevtlog_pack_event(rendered_system, message, string_inserts, - count_inserts, ch, ctx); + count_inserts, event_template, ch, ctx); } flb_free(string_inserts); diff --git a/plugins/in_winevtlog/winevtlog.h b/plugins/in_winevtlog/winevtlog.h index 8fcb0fc273c..980428d9ece 100644 --- a/plugins/in_winevtlog/winevtlog.h +++ b/plugins/in_winevtlog/winevtlog.h @@ -22,11 +22,24 @@ #define FLB_WINEVTLOG_H #include +#include #include #include struct winevtlog_session; +struct winevtlog_event_template { + PWSTR provider_name; + DWORD event_id; + DWORD version; + PWSTR *data_names; + char *cache_key; + size_t cache_key_length; + UINT data_count; + int valid; + struct mk_list _head; +}; + /* reconnect backoff */ struct winevtlog_backoff { DWORD base_ms; @@ -41,6 +54,8 @@ struct winevtlog_config { unsigned int interval_nsec; size_t total_size_threshold; int string_inserts; + int event_data_as_map; + int event_template_cache_size; int read_existing_events; int render_event_as_xml; int render_event_as_text; @@ -59,6 +74,8 @@ struct winevtlog_config { flb_pipefd_t coll_fd; struct flb_input_instance *ins; struct flb_log_event_encoder *log_encoder; + struct flb_hash_table *event_template_cache; + struct mk_list event_templates; struct winevtlog_backoff backoff; flb_sds_t backoff_multiplier_str; }; @@ -133,15 +150,26 @@ struct mk_list *winevtlog_open_all(const char *channels, struct winevtlog_config void winevtlog_close_all(struct mk_list *list); void winevtlog_pack_xml_event(WCHAR *system_xml, WCHAR *message, - PEVT_VARIANT string_inserts, UINT count_inserts, struct winevtlog_channel *ch, + PEVT_VARIANT string_inserts, UINT count_inserts, + struct winevtlog_event_template *event_template, + struct winevtlog_channel *ch, struct winevtlog_config *ctx); void winevtlog_pack_text_event(PEVT_VARIANT system, WCHAR *message, - PEVT_VARIANT string_inserts, UINT count_inserts, struct winevtlog_channel *ch, + PEVT_VARIANT string_inserts, UINT count_inserts, + struct winevtlog_event_template *event_template, + struct winevtlog_channel *ch, struct winevtlog_config *ctx); void winevtlog_pack_event(PEVT_VARIANT system, WCHAR *message, - PEVT_VARIANT string_inserts, UINT count_inserts, struct winevtlog_channel *ch, + PEVT_VARIANT string_inserts, UINT count_inserts, + struct winevtlog_event_template *event_template, + struct winevtlog_channel *ch, struct winevtlog_config *ctx); +struct winevtlog_event_template *winevtlog_event_template_get( + PEVT_VARIANT system, EVT_HANDLE remote, + struct winevtlog_config *ctx); +void winevtlog_event_template_cache_destroy(struct winevtlog_config *ctx); + /* * Save the read offset to disk. */ diff --git a/plugins/out_azure_blob/azure_blob.c b/plugins/out_azure_blob/azure_blob.c index 57ffd90ad43..4b230339579 100644 --- a/plugins/out_azure_blob/azure_blob.c +++ b/plugins/out_azure_blob/azure_blob.c @@ -1127,6 +1127,16 @@ static int send_blob(struct flb_config *config, } else if (event_type == FLB_EVENT_TYPE_BLOBS) { block_id = azb_block_blob_id_blob(ctx, name, part_id); + if (!block_id) { + flb_plg_error(ctx->ins, "could not generate block id"); + flb_free(generated_random_string); + generated_random_string = NULL; + flb_sds_destroy(ref_name); + if (tmp_path_prefix) { + flb_sds_destroy(tmp_path_prefix); + } + return FLB_RETRY; + } uri = azb_block_blob_uri(ctx, path_prefix, name, block_id, 0, generated_random_string); ref_name = flb_sds_printf(&ref_name, "file=%s:%" PRIu64, name, part_id); diff --git a/plugins/out_azure_blob/azure_blob_blockblob.c b/plugins/out_azure_blob/azure_blob_blockblob.c index 356c0ad24d6..5f145724481 100644 --- a/plugins/out_azure_blob/azure_blob_blockblob.c +++ b/plugins/out_azure_blob/azure_blob_blockblob.c @@ -25,6 +25,7 @@ #include #include +#include #include #include "azure_blob.h" @@ -220,18 +221,21 @@ char *azb_block_blob_id_logs(uint64_t *ms) /* * Generate a block id for blob type events: * - * Azure Blob requires that Blobs IDs do not exceed 64 bytes, so we generate a MD5 - * of the path and append the part number to it, we add some zeros for padding since - * all blocks id MUST have the same length. + * Azure Blob requires block IDs to have the same length and the base64-encoded + * value must not exceed 64 bytes. The non-FIPS path keeps the original + * MD5-derived format for compatibility. When FIPS mode is enabled, use the + * first 128 bits of SHA-256 rendered as hex plus the same fixed-width suffix. */ -char *azb_block_blob_id_blob(struct flb_azure_blob *ctx, char *path, int64_t part_id) +char *azb_block_blob_id_blob(struct flb_azure_blob *ctx, char *path, uint64_t part_id) { int i; int len; int ret; + int fips_mode; unsigned char md5[16] = {0}; + unsigned char sha256[32] = {0}; char tmp[128]; - flb_sds_t md5_hex; + flb_sds_t digest_hex; size_t size; size_t o_len; char *b64; @@ -240,27 +244,57 @@ char *azb_block_blob_id_blob(struct flb_azure_blob *ctx, char *path, int64_t par * block ids in base64 cannot exceed 64 bytes, so we hash the path to avoid * exceeding the lenght and then just append the part number. */ - ret = flb_hash_simple(FLB_HASH_MD5, (unsigned char *) path, strlen(path), - md5, sizeof(md5)); - if (ret != 0) { - flb_plg_error(ctx->ins, "cannot hash block id for path %s", path); - return NULL; + fips_mode = FLB_FALSE; + if (ctx->config != NULL && ctx->config->fips_mode_active == FLB_TRUE) { + fips_mode = FLB_TRUE; } - /* convert md5 to hex string (32 byte hex string) */ - md5_hex = flb_sds_create_size(32); - if (!md5_hex) { - return NULL; - } + if (fips_mode == FLB_TRUE) { + ret = flb_hash_simple(FLB_HASH_SHA256, (unsigned char *) path, strlen(path), + sha256, sizeof(sha256)); + if (ret != 0) { + flb_plg_error(ctx->ins, "cannot hash block id for path %s", path); + return NULL; + } + + digest_hex = flb_sds_create_size(32); + if (!digest_hex) { + return NULL; + } + + for (i = 0; i < 16; i++) { + snprintf(digest_hex + (i * 2), 3, "%02x", sha256[i]); + } + flb_sds_len_set(digest_hex, 32); - for (i = 0; i < 16; i++) { - snprintf(md5_hex + (i * 2), 3, "%02x", md5[i]); + len = snprintf(tmp, sizeof(tmp) - 1, "%s.flb-part.%06" PRIu64, + digest_hex, part_id); + flb_sds_destroy(digest_hex); } - flb_sds_len_set(md5_hex, 32); + else { + ret = flb_hash_simple(FLB_HASH_MD5, (unsigned char *) path, strlen(path), + md5, sizeof(md5)); + if (ret != 0) { + flb_plg_error(ctx->ins, "cannot hash block id for path %s", path); + return NULL; + } + + /* convert md5 to hex string (32 byte hex string) */ + digest_hex = flb_sds_create_size(32); + if (!digest_hex) { + return NULL; + } - /* append part number */ - len = snprintf(tmp, sizeof(tmp) - 1, "%s.flb-part.%06ld", md5_hex, part_id); - flb_sds_destroy(md5_hex); + for (i = 0; i < 16; i++) { + snprintf(digest_hex + (i * 2), 3, "%02x", md5[i]); + } + flb_sds_len_set(digest_hex, 32); + + /* append part number */ + len = snprintf(tmp, sizeof(tmp) - 1, "%s.flb-part.%06" PRIu64, + digest_hex, part_id); + flb_sds_destroy(digest_hex); + } size = 64 + 1; b64 = flb_calloc(1, size); @@ -441,6 +475,15 @@ int azb_block_blob_commit_file_parts(struct flb_azure_blob *ctx, uint64_t file_i id = atol(sentry->value); block_id = azb_block_blob_id_blob(ctx, path, id); + if (block_id == NULL) { + flb_plg_error(ctx->ins, + "could not generate block id for file id=%" PRIu64 + " name %s part=%" PRIu64, + file_id, path, id); + flb_sds_destroy(payload); + flb_utils_split_free(list); + return -1; + } cfl_sds_cat_safe(&payload, " ", 2); cfl_sds_cat_safe(&payload, "", 13); diff --git a/plugins/out_file/file.c b/plugins/out_file/file.c index 89a2e0176af..5df9b75d9ba 100644 --- a/plugins/out_file/file.c +++ b/plugins/out_file/file.c @@ -23,7 +23,9 @@ #include #include #include +#include #include +#include #include #include @@ -54,15 +56,34 @@ #define FLB_PATH_SEPARATOR "/" #endif +#define FLB_OUT_FILE_DEFAULT_MAX_DYNAMIC_FILES 1024 + +enum { + FLB_OUT_FILE_ACTION_ERROR, + FLB_OUT_FILE_ACTION_DROP, + FLB_OUT_FILE_ACTION_FALLBACK +}; + struct flb_file_conf { const char *out_path; const char *out_file; + const char *fallback_path; + const char *fallback_file; + const char *on_missing_field; + const char *on_limit_reached; const char *delimiter; const char *label_delimiter; const char *template; int format; int csv_column_names; int mkdir; + int max_dynamic_files; + int missing_field_action; + int limit_reached_action; + int dynamic_destination; + struct flb_record_accessor *ra_path; + struct flb_record_accessor *ra_file; + struct flb_hash_table *dynamic_files; struct flb_output_instance *ins; }; @@ -175,13 +196,259 @@ static int sanitize_tag_name(const char *tag, char *buf, size_t size) return 0; } +static void file_conf_destroy(struct flb_file_conf *ctx) +{ + if (ctx == NULL) { + return; + } + + if (ctx->ra_path != NULL) { + flb_ra_destroy(ctx->ra_path); + } + + if (ctx->ra_file != NULL) { + flb_ra_destroy(ctx->ra_file); + } + + if (ctx->dynamic_files != NULL) { + flb_hash_table_destroy(ctx->dynamic_files); + } + + flb_free(ctx); +} + +static int parse_dynamic_action(struct flb_file_conf *ctx, + const char *name, + const char *value) +{ + if (strcasecmp(value, "error") == 0) { + return FLB_OUT_FILE_ACTION_ERROR; + } + else if (strcasecmp(value, "drop") == 0) { + return FLB_OUT_FILE_ACTION_DROP; + } + else if (strcasecmp(value, "fallback") == 0) { + return FLB_OUT_FILE_ACTION_FALLBACK; + } + + flb_plg_error(ctx->ins, "invalid %s value '%s', expected error, drop or fallback", + name, value); + + return -1; +} + +static int validate_dynamic_file(const char *file) +{ + const unsigned char *p; + + if (file == NULL || file[0] == '\0' || + strcmp(file, ".") == 0 || strcmp(file, "..") == 0) { + return -1; + } + + for (p = (const unsigned char *) file; *p != '\0'; p++) { + if (*p == '/' || *p == '\\' || *p == ':' || *p == '*' || *p == '?' || + *p == '"' || *p == '<' || *p == '>' || *p == '|' || + *p < 0x20 || *p == 0x7f) { + return -1; + } + } + + return 0; +} + +static int validate_dynamic_path(const char *path) +{ + size_t component_length; + const unsigned char *p; + const unsigned char *component; + + if (path == NULL || path[0] == '\0') { + return -1; + } + + p = (const unsigned char *) path; + while (*p != '\0') { + if (*p < 0x20 || *p == 0x7f) { + return -1; + } + + while (*p == '/' || *p == '\\') { + p++; + } + + component = p; + while (*p != '\0' && *p != '/' && *p != '\\') { + if (*p < 0x20 || *p == 0x7f) { + return -1; + } + p++; + } + + component_length = p - component; + if ((component_length == 1 && component[0] == '.') || + (component_length == 2 && component[0] == '.' && component[1] == '.')) { + return -1; + } + } + + return 0; +} + +static int compose_output_file(const char *path, + const char *file, + char *output, + size_t output_size) +{ + int ret; + + if (path != NULL) { + ret = snprintf(output, output_size, "%s" FLB_PATH_SEPARATOR "%s", path, file); + } + else { + ret = snprintf(output, output_size, "%s", file); + } + + if (ret < 0 || (size_t) ret >= output_size) { + return -1; + } + + return 0; +} + +static int use_fallback_destination(struct flb_file_conf *ctx, + char *output, + size_t output_size) +{ + if (ctx->fallback_file == NULL) { + return -1; + } + + return compose_output_file(ctx->fallback_path, ctx->fallback_file, + output, output_size); +} + +static int apply_destination_action(struct flb_file_conf *ctx, + int action, + const char *reason, + char *output, + size_t output_size) +{ + if (action == FLB_OUT_FILE_ACTION_DROP) { + flb_plg_warn(ctx->ins, "dropping record: %s", reason); + return 1; + } + + if (action == FLB_OUT_FILE_ACTION_FALLBACK) { + if (use_fallback_destination(ctx, output, output_size) == 0) { + return 0; + } + flb_plg_error(ctx->ins, "cannot apply fallback destination: %s", reason); + return -1; + } + + flb_plg_error(ctx->ins, "%s", reason); + return -1; +} + +static int resolve_dynamic_destination(struct flb_file_conf *ctx, + const char *tag, + msgpack_object map, + char *output, + size_t output_size, + int *new_destination) +{ + int ret; + char sanitized_tag[PATH_MAX]; + void *stored_value; + size_t stored_size; + flb_sds_t dynamic_path = NULL; + flb_sds_t dynamic_file = NULL; + const char *path; + const char *file; + + *new_destination = FLB_FALSE; + path = ctx->out_path; + file = ctx->out_file; + + if (ctx->ra_path != NULL) { + dynamic_path = flb_ra_translate_check(ctx->ra_path, + (char *) tag, strlen(tag), + map, NULL, FLB_TRUE); + if (dynamic_path == NULL) { + return apply_destination_action(ctx, ctx->missing_field_action, + "record accessor field missing from path", + output, output_size); + } + path = dynamic_path; + } + + if (ctx->ra_file != NULL) { + dynamic_file = flb_ra_translate_check(ctx->ra_file, + (char *) tag, strlen(tag), + map, NULL, FLB_TRUE); + if (dynamic_file == NULL) { + flb_sds_destroy(dynamic_path); + return apply_destination_action(ctx, ctx->missing_field_action, + "record accessor field missing from file", + output, output_size); + } + file = dynamic_file; + } + else if (file == NULL) { + ret = sanitize_tag_name(tag, sanitized_tag, sizeof(sanitized_tag)); + if (ret != 0) { + flb_sds_destroy(dynamic_path); + return -1; + } + file = sanitized_tag; + } + + if ((ctx->ra_path != NULL && validate_dynamic_path(path) != 0) || + (ctx->ra_file != NULL && validate_dynamic_file(file) != 0)) { + flb_sds_destroy(dynamic_path); + flb_sds_destroy(dynamic_file); + return apply_destination_action(ctx, ctx->missing_field_action, + "unsafe dynamic output destination", + output, output_size); + } + + ret = compose_output_file(path, file, output, output_size); + flb_sds_destroy(dynamic_path); + flb_sds_destroy(dynamic_file); + if (ret != 0) { + return -1; + } + + if (ctx->dynamic_files == NULL) { + return 0; + } + + ret = flb_hash_table_get(ctx->dynamic_files, output, strlen(output), + &stored_value, &stored_size); + if (ret >= 0) { + return 0; + } + + if (ctx->dynamic_files->total_count >= ctx->max_dynamic_files) { + return apply_destination_action(ctx, ctx->limit_reached_action, + "max_dynamic_files limit reached", + output, output_size); + } + + *new_destination = FLB_TRUE; + return 0; +} + static int cb_file_init(struct flb_output_instance *ins, struct flb_config *config, void *data) { int ret; + int table_size; const char *tmp; + const char *accessor; char *ret_str; (void) config; (void) data; @@ -197,13 +464,82 @@ static int cb_file_init(struct flb_output_instance *ins, ctx->delimiter = NULL; ctx->label_delimiter = NULL; ctx->template = NULL; + ctx->max_dynamic_files = FLB_OUT_FILE_DEFAULT_MAX_DYNAMIC_FILES; ret = flb_output_config_map_set(ins, (void *) ctx); if (ret == -1) { - flb_free(ctx); + file_conf_destroy(ctx); + return -1; + } + + ctx->missing_field_action = parse_dynamic_action(ctx, "on_missing_field", + ctx->on_missing_field); + ctx->limit_reached_action = parse_dynamic_action(ctx, "on_limit_reached", + ctx->on_limit_reached); + if (ctx->missing_field_action < 0 || ctx->limit_reached_action < 0 || + ctx->max_dynamic_files < 0) { + file_conf_destroy(ctx); + return -1; + } + + if ((ctx->missing_field_action == FLB_OUT_FILE_ACTION_FALLBACK || + ctx->limit_reached_action == FLB_OUT_FILE_ACTION_FALLBACK) && + ctx->fallback_file == NULL) { + flb_plg_error(ctx->ins, "fallback_file is required for fallback actions"); + file_conf_destroy(ctx); + return -1; + } + + if (ctx->out_path != NULL && strchr(ctx->out_path, '$') != NULL) { + accessor = strchr(ctx->out_path, '$'); + if (accessor == ctx->out_path) { + flb_plg_error(ctx->ins, + "dynamic 'path' must include a static prefix before record accessors"); + file_conf_destroy(ctx); + return -1; + } + + ctx->ra_path = flb_ra_create((char *) ctx->out_path, FLB_TRUE); + if (ctx->ra_path == NULL) { + flb_plg_error(ctx->ins, "invalid record accessor pattern set for 'path'"); + file_conf_destroy(ctx); + return -1; + } + ctx->dynamic_destination = FLB_TRUE; + } + + if (ctx->out_file != NULL && strchr(ctx->out_file, '$') != NULL) { + ctx->ra_file = flb_ra_create((char *) ctx->out_file, FLB_TRUE); + if (ctx->ra_file == NULL) { + flb_plg_error(ctx->ins, "invalid record accessor pattern set for 'file'"); + file_conf_destroy(ctx); + return -1; + } + ctx->dynamic_destination = FLB_TRUE; + } + + if (ctx->dynamic_destination == FLB_TRUE && ctx->fallback_file == NULL) { + flb_plg_error(ctx->ins, + "fallback_file is required when dynamic destinations are configured"); + file_conf_destroy(ctx); return -1; } + if (ctx->dynamic_destination == FLB_TRUE && ctx->max_dynamic_files > 0) { + table_size = ctx->max_dynamic_files; + if (table_size > 128) { + table_size = 128; + } + + ctx->dynamic_files = flb_hash_table_create(FLB_HASH_TABLE_EVICT_NONE, + table_size, + ctx->max_dynamic_files); + if (ctx->dynamic_files == NULL) { + file_conf_destroy(ctx); + return -1; + } + } + /* Optional, file format */ tmp = flb_output_get_property("Format", ins); if (tmp) { @@ -235,7 +571,7 @@ static int cb_file_init(struct flb_output_instance *ins, } else { flb_plg_error(ctx->ins, "unknown format %s. abort.", tmp); - flb_free(ctx); + file_conf_destroy(ctx); return -1; } } @@ -582,6 +918,157 @@ static int mkpath(struct flb_output_instance *ins, const char *dir) #endif } +static FILE *open_output_file(struct flb_file_conf *ctx, const char *output) +{ + int ret; + FILE *fp; + char *output_copy; + + fp = fopen(output, "ab+"); + if (ctx->mkdir == FLB_TRUE && fp == NULL && errno == ENOENT) { + output_copy = strdup(output); + if (output_copy != NULL) { +#ifdef FLB_SYSTEM_WINDOWS + PathRemoveFileSpecA(output_copy); + ret = mkpath(ctx->ins, output_copy); +#else + ret = mkpath(ctx->ins, dirname(output_copy)); +#endif + free(output_copy); + if (ret == 0) { + fp = fopen(output, "ab+"); + } + } + } + + return fp; +} + +static int write_log_record(FILE *fp, + long *file_pos, + struct flb_event_chunk *event_chunk, + struct flb_log_event *log_event, + struct flb_file_conf *ctx, + struct flb_config *config) +{ + int column_names; + char *buf; + + switch (ctx->format) { + case FLB_OUT_FILE_FMT_JSON: + buf = flb_msgpack_to_json_str(128, log_event->body, + config->json_escape_unicode); + if (buf == NULL) { + return FLB_RETRY; + } + + fprintf(fp, "%s: [%"PRIu64".%09lu, %s]" NEWLINE, + event_chunk->tag, + (uint64_t) log_event->timestamp.tm.tv_sec, + log_event->timestamp.tm.tv_nsec, + buf); + flb_free(buf); + break; + case FLB_OUT_FILE_FMT_CSV: + if (ctx->csv_column_names == FLB_TRUE && *file_pos == 0) { + column_names = FLB_TRUE; + *file_pos = 1; + } + else { + column_names = FLB_FALSE; + } + csv_output(fp, column_names, &log_event->timestamp, log_event->body, ctx); + break; + case FLB_OUT_FILE_FMT_LTSV: + ltsv_output(fp, &log_event->timestamp, log_event->body, ctx); + break; + case FLB_OUT_FILE_FMT_PLAIN: + plain_output(fp, log_event->body, 128, config->json_escape_unicode); + break; + case FLB_OUT_FILE_FMT_TEMPLATE: + template_output(fp, &log_event->timestamp, log_event->body, ctx); + break; + } + + return FLB_OK; +} + +static int flush_dynamic_logs(struct flb_event_chunk *event_chunk, + struct flb_file_conf *ctx, + struct flb_config *config, + char *output, + size_t output_size, + struct flb_log_event_decoder *log_decoder, + struct flb_log_event *log_event) +{ + int ret; + int new_destination; + FILE *fp; + long file_pos; + + ret = flb_log_event_decoder_init(log_decoder, + (char *) event_chunk->data, + event_chunk->size); + if (ret != FLB_EVENT_DECODER_SUCCESS) { + flb_plg_error(ctx->ins, "Log event decoder initialization error : %d", ret); + return FLB_ERROR; + } + + while ((ret = flb_log_event_decoder_next(log_decoder, + log_event)) == FLB_EVENT_DECODER_SUCCESS) { + ret = resolve_dynamic_destination(ctx, event_chunk->tag, *log_event->body, + output, output_size, &new_destination); + if (ret == 1) { + continue; + } + else if (ret != 0) { + flb_log_event_decoder_destroy(log_decoder); + return FLB_ERROR; + } + + fp = open_output_file(ctx, output); + if (fp == NULL) { + flb_errno(); + flb_plg_error(ctx->ins, "error opening: %s", output); + flb_log_event_decoder_destroy(log_decoder); + return FLB_ERROR; + } + + if (new_destination == FLB_TRUE) { + ret = flb_hash_table_add(ctx->dynamic_files, output, strlen(output), "", 0); + if (ret < 0) { + fclose(fp); + flb_log_event_decoder_destroy(log_decoder); + return FLB_ERROR; + } + } + + file_pos = ftell(fp); + if (ctx->format == FLB_OUT_FILE_FMT_MSGPACK) { + if (fwrite(log_decoder->record_base, 1, log_decoder->record_length, fp) != + log_decoder->record_length) { + fclose(fp); + flb_log_event_decoder_destroy(log_decoder); + return FLB_RETRY; + } + } + else { + ret = write_log_record(fp, &file_pos, event_chunk, log_event, ctx, config); + if (ret != FLB_OK) { + fclose(fp); + flb_log_event_decoder_destroy(log_decoder); + return ret; + } + } + + fclose(fp); + } + + flb_log_event_decoder_destroy(log_decoder); + + return FLB_OK; +} + static void cb_file_flush(struct flb_event_chunk *event_chunk, struct flb_output_flush *out_flush, struct flb_input_instance *ins, @@ -606,8 +1093,25 @@ static void cb_file_flush(struct flb_event_chunk *event_chunk, (void) config; + if (ctx->dynamic_destination == FLB_TRUE && + event_chunk->type != FLB_INPUT_METRICS) { + ret = flush_dynamic_logs(event_chunk, ctx, config, + out_file, sizeof(out_file), + &log_decoder, &log_event); + FLB_OUTPUT_RETURN(ret); + } + + if (ctx->dynamic_destination == FLB_TRUE && + event_chunk->type == FLB_INPUT_METRICS) { + ret = use_fallback_destination(ctx, out_file, sizeof(out_file)); + if (ret != 0) { + flb_plg_error(ctx->ins, + "dynamic path and file record accessors are unsupported for metrics"); + FLB_OUTPUT_RETURN(FLB_ERROR); + } + } /* Set the right output file */ - if (ctx->out_file == NULL) { + else if (ctx->out_file == NULL) { ret = sanitize_tag_name(event_chunk->tag, sanitized_tag, sizeof(sanitized_tag)); @@ -618,7 +1122,7 @@ static void cb_file_flush(struct flb_event_chunk *event_chunk, } } - if (ctx->out_path) { + if (ctx->dynamic_destination == FLB_FALSE && ctx->out_path) { if (ctx->out_file) { snprintf(out_file, sizeof(out_file) , "%s" FLB_PATH_SEPARATOR "%s", ctx->out_path, ctx->out_file); @@ -628,7 +1132,7 @@ static void cb_file_flush(struct flb_event_chunk *event_chunk, ctx->out_path, sanitized_tag); } } - else { + else if (ctx->dynamic_destination == FLB_FALSE) { if (ctx->out_file) { snprintf(out_file, PATH_MAX, "%s", ctx->out_file); } @@ -781,7 +1285,7 @@ static int cb_file_exit(void *data, struct flb_config *config) return 0; } - flb_free(ctx); + file_conf_destroy(ctx); return 0; } @@ -790,14 +1294,45 @@ static struct flb_config_map config_map[] = { { FLB_CONFIG_MAP_STR, "path", NULL, 0, FLB_TRUE, offsetof(struct flb_file_conf, out_path), - "Absolute path to store the files. This parameter is optional" + "Absolute path to store the files. Log record accessor expressions are supported, " + "and dynamic paths must retain a static prefix" }, { FLB_CONFIG_MAP_STR, "file", NULL, 0, FLB_TRUE, offsetof(struct flb_file_conf, out_file), "Name of the target file to write the records. If 'path' is specified, " - "the value is prefixed" + "the value is prefixed. Log record accessor expressions are supported" + }, + + { + FLB_CONFIG_MAP_INT, "max_dynamic_files", "1024", + 0, FLB_TRUE, offsetof(struct flb_file_conf, max_dynamic_files), + "Maximum number of distinct record-accessor destinations. Set to 0 for unlimited" + }, + + { + FLB_CONFIG_MAP_STR, "on_missing_field", "error", + 0, FLB_TRUE, offsetof(struct flb_file_conf, on_missing_field), + "Action for missing or unsafe dynamic destination values: error, drop or fallback" + }, + + { + FLB_CONFIG_MAP_STR, "on_limit_reached", "error", + 0, FLB_TRUE, offsetof(struct flb_file_conf, on_limit_reached), + "Action when max_dynamic_files would be exceeded: error, drop or fallback" + }, + + { + FLB_CONFIG_MAP_STR, "fallback_path", NULL, + 0, FLB_TRUE, offsetof(struct flb_file_conf, fallback_path), + "Static output path used when a configured fallback action is applied" + }, + + { + FLB_CONFIG_MAP_STR, "fallback_file", NULL, + 0, FLB_TRUE, offsetof(struct flb_file_conf, fallback_file), + "Static output file used when a configured fallback action is applied" }, { diff --git a/plugins/out_gcs/CMakeLists.txt b/plugins/out_gcs/CMakeLists.txt new file mode 100644 index 00000000000..c8b79e3fc8d --- /dev/null +++ b/plugins/out_gcs/CMakeLists.txt @@ -0,0 +1,5 @@ +set(src + gcs.c + gcs_store.c) + +FLB_PLUGIN(out_gcs "${src}" "") diff --git a/plugins/out_gcs/gcs.c b/plugins/out_gcs/gcs.c new file mode 100644 index 00000000000..e9062359d13 --- /dev/null +++ b/plugins/out_gcs/gcs.c @@ -0,0 +1,1548 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2026 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "gcs.h" +#include "gcs_store.h" + +#include +#include + +static int gcs_ctx_destroy(void *data, struct flb_config *config); + +static const char *get_predefined_acl(const char *canned_acl) +{ + if (!canned_acl) { + return NULL; + } + + if (strcasecmp(canned_acl, "authenticated-read") == 0 || + strcasecmp(canned_acl, "authenticatedRead") == 0) { + return "authenticatedRead"; + } + if (strcasecmp(canned_acl, "bucket-owner-full-control") == 0 || + strcasecmp(canned_acl, "bucketOwnerFullControl") == 0) { + return "bucketOwnerFullControl"; + } + if (strcasecmp(canned_acl, "bucket-owner-read") == 0 || + strcasecmp(canned_acl, "bucketOwnerRead") == 0) { + return "bucketOwnerRead"; + } + if (strcasecmp(canned_acl, "private") == 0) { + return "private"; + } + if (strcasecmp(canned_acl, "project-private") == 0 || + strcasecmp(canned_acl, "projectPrivate") == 0) { + return "projectPrivate"; + } + if (strcasecmp(canned_acl, "public-read") == 0 || + strcasecmp(canned_acl, "publicRead") == 0) { + return "publicRead"; + } + + return NULL; +} + +static inline int key_cmp(char *str, int len, char *cmp) { + if (strlen(cmp) != len) { + return -1; + } + + return strncasecmp(str, cmp, len); +} + +static int gcs_under_test_mode(void) +{ + char *env; + + env = getenv("FLB_GCS_PLUGIN_UNDER_TEST"); + + if (env && strcasecmp(env, "true") == 0) { + return FLB_TRUE; + } + + return FLB_FALSE; +} + +static int gcs_setenv(const char *key, const char *val) +{ +#ifdef FLB_SYSTEM_WINDOWS + return _putenv_s(key, val); +#else + return setenv(key, val, 1); +#endif +} + +static void mock_gcs_call_increment_counter(const char *api) +{ + char env_var[64]; + char *val; + int count; + char buf[16]; + + snprintf(env_var, sizeof(env_var), "TEST_GCS_%s_CALL_COUNT", api); + val = getenv(env_var); + count = val ? atoi(val) : 0; + count++; + snprintf(buf, sizeof(buf), "%d", count); + gcs_setenv(env_var, buf); +} + +static int read_seq_index(const char *path, uint64_t *out_value) +{ + FILE *fp; + unsigned long long val; + + fp = fopen(path, "r"); + if (!fp) { + return -1; + } + if (fscanf(fp, "%llu", &val) != 1) { + fclose(fp); + return -1; + } + fclose(fp); + *out_value = (uint64_t) val; + return 0; +} + +static int write_seq_index(const char *path, uint64_t value) +{ + FILE *fp; + + fp = fopen(path, "w"); + if (!fp) { + return -1; + } + fprintf(fp, "%llu", (unsigned long long) value); + fclose(fp); + return 0; +} + +static int init_seq_index(struct flb_gcs *ctx) +{ + flb_sds_t path; + flb_sds_t tmp; + + path = flb_sds_create_size(256); + if (!path) { + flb_errno(); + return -1; + } + tmp = flb_sds_printf(&path, "%s/%s_seq_index", + ctx->store_dir, ctx->fs_stream_name); + if (!tmp) { + flb_sds_destroy(path); + return -1; + } + path = tmp; + ctx->seq_index_file = path; + + if (read_seq_index(ctx->seq_index_file, &ctx->seq_index) == -1) { + ctx->seq_index = 0; + if (write_seq_index(ctx->seq_index_file, ctx->seq_index) == -1) { + return -1; + } + } + return 0; +} + +static int gcs_get_md5_base64(char *buf, size_t buf_size, char *md5_str, size_t md5_str_size) +{ + unsigned char md5_bin[16]; + size_t olen; + int ret; + + ret = flb_hash_simple(FLB_HASH_MD5, (unsigned char *) buf, buf_size, + md5_bin, sizeof(md5_bin)); + if (ret != FLB_CRYPTO_SUCCESS) { + return -1; + } + + ret = flb_base64_encode((unsigned char *) md5_str, md5_str_size, + &olen, md5_bin, sizeof(md5_bin)); + if (ret != 0) { + return -1; + } + return 0; +} + +static int random_hex_suffix(char *buf, size_t buf_size) +{ + unsigned char rnd[4]; + int ret; + + if (buf_size < 9) { + return -1; + } + + ret = flb_random_bytes(rnd, sizeof(rnd)); + if (ret != 0) { + return -1; + } + + snprintf(buf, buf_size, "%02x%02x%02x%02x", rnd[0], rnd[1], rnd[2], rnd[3]); + return 0; +} + +flb_sds_t gcs_uri_encode_object_name(const char *name, size_t name_len) +{ + size_t index; + size_t output_index; + size_t output_size; + unsigned char value; + char *buffer; + flb_sds_t output; + const char hex[] = "0123456789ABCDEF"; + + if (!name || name_len > (((size_t) -1) - 1) / 3) { + return NULL; + } + + output_size = (name_len * 3) + 1; + buffer = flb_malloc(output_size); + if (!buffer) { + flb_errno(); + return NULL; + } + + output_index = 0; + for (index = 0; index < name_len; index++) { + value = (unsigned char) name[index]; + if ((value >= 'a' && value <= 'z') || + (value >= 'A' && value <= 'Z') || + (value >= '0' && value <= '9') || + value == '-' || value == '_' || value == '.' || value == '~') { + buffer[output_index++] = (char) value; + } + else { + buffer[output_index++] = '%'; + buffer[output_index++] = hex[value >> 4]; + buffer[output_index++] = hex[value & 0x0f]; + } + } + buffer[output_index] = '\0'; + + output = flb_sds_create_len(buffer, output_index); + flb_free(buffer); + + return output; +} + +/* credential parse and oauth helpers based on bigquery/stackdriver style */ +static int flb_gcs_read_credentials_file(struct flb_gcs *ctx, char *creds, + struct flb_gcs_oauth_credentials *c) +{ + int i, ret, len, key_len, val_len; + int tok_size = 32; + char *buf, *key, *val; + void *tmp_tokens; + flb_sds_t tmp; + struct stat st; + jsmn_parser parser; + jsmntok_t *t; + jsmntok_t *tokens; + + ret = stat(creds, &st); + if (ret == -1) { + flb_errno(); + flb_plg_error(ctx->ins, "cannot open credentials file: %s", creds); + return -1; + } + + if (!S_ISREG(st.st_mode) && !S_ISLNK(st.st_mode)) { + flb_plg_error(ctx->ins, "credentials file is not a valid file: %s", + creds); + return -1; + } + + buf = mk_file_to_buffer(creds); + if (!buf) { + flb_plg_error(ctx->ins, "error reading credentials file: %s", creds); + return -1; + } + + jsmn_init(&parser); + tokens = flb_calloc(1, sizeof(jsmntok_t) * tok_size); + if (!tokens) { + flb_free(buf); + return -1; + } + + ret = jsmn_parse(&parser, buf, st.st_size, tokens, tok_size); + while (ret == JSMN_ERROR_NOMEM) { + if (tok_size > INT_MAX / 2) { + flb_plg_error(ctx->ins, "credentials file is too large: %s", creds); + flb_free(buf); + flb_free(tokens); + return -1; + } + + tmp_tokens = flb_realloc(tokens, sizeof(jsmntok_t) * tok_size * 2); + if (!tmp_tokens) { + flb_errno(); + flb_free(buf); + flb_free(tokens); + return -1; + } + tokens = tmp_tokens; + tok_size *= 2; + ret = jsmn_parse(&parser, buf, st.st_size, tokens, tok_size); + } + + if (ret <= 0) { + flb_plg_error(ctx->ins, "invalid JSON credentials file: %s", creds); + flb_free(buf); + flb_free(tokens); + return -1; + } + + t = &tokens[0]; + if (t->type != JSMN_OBJECT) { + flb_plg_error(ctx->ins, "invalid JSON map on file: %s", creds); + flb_free(buf); + flb_free(tokens); + return -1; + } + + for (i = 1; i < ret; i++) { + t = &tokens[i]; + if (t->type != JSMN_STRING) { + continue; + } + + if (t->start == -1 || t->end == -1 || + (t->start == 0 && t->end == 0)) { + break; + } + key = buf + t->start; + key_len = t->end - t->start; + + i++; + if (i >= ret) { + break; + } + t = &tokens[i]; + if (t->start == -1 || t->end == -1) { + continue; + } + val = buf + t->start; + val_len = t->end - t->start; + + if (key_cmp(key, key_len, "type") == 0) { + c->type = flb_sds_create_len(val, val_len); + } + else if (key_cmp(key, key_len, "project_id") == 0) { + c->project_id = flb_sds_create_len(val, val_len); + } + else if (key_cmp(key, key_len, "private_key_id") == 0) { + c->private_key_id = flb_sds_create_len(val, val_len); + } + else if (key_cmp(key, key_len, "private_key") == 0) { + tmp = flb_sds_create_len(val, val_len); + if (tmp) { + len = flb_sds_len(tmp); + c->private_key = flb_sds_create_size(len); + if (!c->private_key) { + flb_errno(); + flb_sds_destroy(tmp); + flb_free(buf); + flb_free(tokens); + return -1; + } + flb_unescape_string(tmp, len, &c->private_key); + flb_sds_destroy(tmp); + } + } + else if (key_cmp(key, key_len, "client_email") == 0) { + c->client_email = flb_sds_create_len(val, val_len); + } + else if (key_cmp(key, key_len, "client_id") == 0) { + c->client_id = flb_sds_create_len(val, val_len); + } + else if (key_cmp(key, key_len, "auth_uri") == 0) { + c->auth_uri = flb_sds_create_len(val, val_len); + } + else if (key_cmp(key, key_len, "token_uri") == 0) { + c->token_uri = flb_sds_create_len(val, val_len); + } + } + + flb_free(buf); + flb_free(tokens); + + if (!c->private_key) { + flb_plg_error(ctx->ins, "no private key"); + return -1; + } + + return c->client_email ? 0 : -1; +} + +static void flb_gcs_credentials_destroy(struct flb_gcs_oauth_credentials *c) +{ + if (!c) { + return; + } + + if (c->type) { + flb_sds_destroy(c->type); + } + if (c->project_id) { + flb_sds_destroy(c->project_id); + } + if (c->private_key_id) { + flb_sds_destroy(c->private_key_id); + } + if (c->private_key) { + flb_sds_destroy(c->private_key); + } + if (c->client_email) { + flb_sds_destroy(c->client_email); + } + if (c->client_id) { + flb_sds_destroy(c->client_id); + } + if (c->auth_uri) { + flb_sds_destroy(c->auth_uri); + } + if (c->token_uri) { + flb_sds_destroy(c->token_uri); + } + + flb_free(c); +} + +int gcs_jwt_base64_url_encode(unsigned char *out_buf, size_t out_size, + unsigned char *in_buf, size_t in_size, size_t *olen) +{ + size_t i; + size_t len; + int result; + + if (!out_buf || !in_buf || !olen || out_size < 2) { + return -1; + } + + result = flb_base64_encode((unsigned char *) out_buf, out_size, &len, in_buf, in_size); + if (result != 0) { + return -1; + } + for (i = 0; i < len && out_buf[i] != '='; i++) { + if (out_buf[i] == '+') out_buf[i] = '-'; + else if (out_buf[i] == '/') out_buf[i] = '_'; + } + out_buf[i] = '\0'; + *olen = i; + return 0; +} + +int gcs_jwt_encode(struct flb_gcs *ctx, char *payload, char *secret, + char **out_signature, size_t *out_size) +{ + int ret; + size_t header_len; + size_t payload_len; + size_t secret_len; + size_t input_len; + size_t encoded_units; + size_t buf_size; + size_t olen, sig_len; + char *buf; + char *sigd; + char *headers = "{\"alg\": \"RS256\", \"typ\": \"JWT\"}"; + unsigned char sha256_buf[32] = {0}; + flb_sds_t out; + flb_sds_t tmp; + unsigned char sig[256] = {0}; + + (void) ctx; + + out = NULL; + sigd = NULL; + header_len = strlen(headers); + payload_len = strlen(payload); + secret_len = strlen(secret); + input_len = header_len > payload_len ? header_len : payload_len; + if (input_len > ((size_t) -1) - 2) { + return -1; + } + + encoded_units = (input_len + 2) / 3; + if (encoded_units > (((size_t) -1) - 1) / 4) { + return -1; + } + buf_size = (encoded_units * 4) + 1; + buf = flb_malloc(buf_size); + if (!buf) { + flb_errno(); + return -1; + } + + ret = gcs_jwt_base64_url_encode((unsigned char *) buf, buf_size, (unsigned char *) headers, + header_len, &olen); + if (ret != 0) { + goto error; + } + + out = flb_sds_create_size(2048); + if (!out) { + flb_errno(); + goto error; + } + tmp = flb_sds_cat(out, buf, olen); + if (!tmp) { + goto error; + } + out = tmp; + + tmp = flb_sds_cat(out, ".", 1); + if (!tmp) { + goto error; + } + out = tmp; + + ret = gcs_jwt_base64_url_encode((unsigned char *) buf, buf_size, (unsigned char *) payload, + payload_len, &olen); + if (ret != 0) { + goto error; + } + tmp = flb_sds_cat(out, buf, olen); + if (!tmp) { + goto error; + } + out = tmp; + + ret = flb_hash_simple(FLB_HASH_SHA256, (unsigned char *) out, flb_sds_len(out), + sha256_buf, sizeof(sha256_buf)); + if (ret != FLB_CRYPTO_SUCCESS) { + goto error; + } + + sig_len = sizeof(sig); + ret = flb_crypto_sign_simple(FLB_CRYPTO_PRIVATE_KEY, FLB_CRYPTO_PADDING_PKCS1, + FLB_HASH_SHA256, (unsigned char *) secret, secret_len + 1, + sha256_buf, sizeof(sha256_buf), sig, &sig_len); + if (ret != FLB_CRYPTO_SUCCESS) { + goto error; + } + + sigd = flb_malloc(2048); + if (!sigd) { + flb_errno(); + goto error; + } + ret = gcs_jwt_base64_url_encode((unsigned char *) sigd, 2048, sig, sig_len, &olen); + if (ret != 0) { + goto error; + } + + tmp = flb_sds_cat(out, ".", 1); + if (!tmp) { + goto error; + } + out = tmp; + + tmp = flb_sds_cat(out, sigd, olen); + if (!tmp) { + goto error; + } + out = tmp; + + *out_signature = out; + *out_size = flb_sds_len(out); + flb_free(buf); + flb_free(sigd); + + return 0; + +error: + flb_free(buf); + flb_free(sigd); + if (out) { + flb_sds_destroy(out); + } + + return -1; +} + +static int gcs_get_oauth2_token(struct flb_gcs *ctx) +{ + int ret; + char *sig_data; + size_t sig_size; + time_t issued; + time_t expires; + char payload[1024]; + + flb_oauth2_payload_clear(ctx->o); + issued = time(NULL); + expires = issued + FLB_GCS_TOKEN_REFRESH; + snprintf(payload, sizeof(payload) - 1, + "{\"iss\": \"%s\", \"scope\": \"%s\", \"aud\": \"%s\", " + "\"exp\": %llu, \"iat\": %llu}", + ctx->oauth_credentials->client_email, FLB_GCS_SCOPE, FLB_GCS_AUTH_URL, + (unsigned long long) expires, (unsigned long long) issued); + + ret = gcs_jwt_encode(ctx, payload, ctx->oauth_credentials->private_key, &sig_data, &sig_size); + if (ret != 0) { + return -1; + } + + ret = flb_oauth2_payload_append(ctx->o, "grant_type", -1, + "urn%3Aietf%3Aparams%3Aoauth%3Agrant-type%3Ajwt-bearer", -1); + if (ret == -1) { + flb_sds_destroy(sig_data); + return -1; + } + + ret = flb_oauth2_payload_append(ctx->o, "assertion", -1, sig_data, sig_size); + flb_sds_destroy(sig_data); + if (ret == -1 || !flb_oauth2_token_get(ctx->o)) { + return -1; + } + + return 0; +} + +static flb_sds_t get_google_token(struct flb_gcs *ctx) +{ + int ret = 0; + flb_sds_t output = NULL; + flb_sds_t tmp; + + if (pthread_mutex_lock(&ctx->token_mutex)) { + return NULL; + } + + if (flb_oauth2_token_expired(ctx->o) == FLB_TRUE) { + ret = gcs_get_oauth2_token(ctx); + } + + if (ret == 0) { + output = flb_sds_create(ctx->o->token_type); + if (output) { + tmp = flb_sds_printf(&output, " %s", ctx->o->access_token); + if (!tmp) { + flb_sds_destroy(output); + output = NULL; + } + else { + output = tmp; + } + } + } + pthread_mutex_unlock(&ctx->token_mutex); + + return output; +} + +static int upload_queue_contains(struct flb_gcs *ctx, struct gcs_file *chunk) +{ + struct mk_list *head; + struct upload_queue *entry; + + mk_list_foreach(head, &ctx->upload_queue) { + entry = mk_list_entry(head, struct upload_queue, _head); + if (entry->upload_file == chunk) { + return FLB_TRUE; + } + } + return FLB_FALSE; +} + +static int add_to_queue(struct flb_gcs *ctx, struct gcs_file *chunk, + const char *tag, int tag_len) +{ + struct upload_queue *entry; + + if (upload_queue_contains(ctx, chunk) == FLB_TRUE) { + return 0; + } + + entry = flb_calloc(1, sizeof(struct upload_queue)); + if (!entry) { + return -1; + } + + entry->tag = flb_strndup(tag, tag_len); + if (!entry->tag) { + flb_free(entry); + return -1; + } + + entry->upload_file = chunk; + entry->tag_len = tag_len; + entry->upload_time = chunk->create_time + ctx->upload_timeout; + if (entry->upload_time < time(NULL)) { + entry->upload_time = time(NULL); + } + mk_list_add(&entry->_head, &ctx->upload_queue); + return 0; +} + +static void remove_from_queue(struct upload_queue *entry) +{ + mk_list_del(&entry->_head); + flb_free(entry->tag); + flb_free(entry); +} + + +static void clear_upload_queue(struct flb_gcs *ctx) +{ + struct mk_list *head; + struct mk_list *tmp; + struct upload_queue *entry; + + mk_list_foreach_safe(head, tmp, &ctx->upload_queue) { + entry = mk_list_entry(head, struct upload_queue, _head); + remove_from_queue(entry); + } +} + +static void clear_file_chunks(struct flb_gcs *ctx) +{ + struct mk_list *head; + struct flb_fstore_file *fsf; + + if (!ctx || !ctx->fs_stream) { + return; + } + + mk_list_foreach(head, &ctx->fs_stream->files) { + fsf = mk_list_entry(head, struct flb_fstore_file, _head); + + if (fsf->data) { + flb_free(fsf->data); + fsf->data = NULL; + } + } +} + +static int construct_request_buffer(struct flb_gcs *ctx, + struct upload_queue *entry, + char **out_buffer, + size_t *out_size) +{ + int ret; + + ret = gcs_store_file_read(ctx, entry->upload_file, out_buffer, out_size); + if (ret == -1) { + return -1; + } + + return 0; +} + +static int gcs_upload_object(struct flb_gcs *ctx, + flb_sds_t auth, + flb_sds_t uri, + char *body, + size_t body_size) +{ + int ret; + size_t bytes = 0; + struct flb_connection *u_conn; + struct flb_http_client *c; + struct flb_aws_header content_type_header = { + .key = "Content-Type", + .key_len = 12 + }; + struct flb_aws_header content_md5_header = { + .key = "Content-MD5", + .key_len = 11 + }; + char final_body_md5[25]; + + if (gcs_under_test_mode() == FLB_TRUE) { + mock_gcs_call_increment_counter("UploadObject"); + gcs_setenv("TEST_GCS_LAST_URI", uri); + if (body_size >= 2 && + (unsigned char) body[0] == 0x1f && + (unsigned char) body[1] == 0x8b) { + gcs_setenv("TEST_GCS_LAST_BODY_GZIP", "true"); + } + else { + gcs_setenv("TEST_GCS_LAST_BODY_GZIP", "false"); + } + + if (getenv("TEST_GCS_UPLOAD_ERROR") != NULL) { + return -1; + } + + return 0; + } + + u_conn = flb_upstream_conn_get(ctx->u); + if (!u_conn) { + return -1; + } + + c = flb_http_client(u_conn, FLB_HTTP_POST, uri, body, body_size, + FLB_GCS_DEFAULT_HOST, FLB_GCS_DEFAULT_PORT, NULL, 0); + if (!c) { + flb_upstream_conn_release(u_conn); + return -1; + } + + content_type_header.val = ctx->content_type; + content_type_header.val_len = flb_sds_len(ctx->content_type); + flb_http_add_header(c, content_type_header.key, content_type_header.key_len, + content_type_header.val, content_type_header.val_len); + flb_http_add_header(c, "Authorization", 13, auth, flb_sds_len(auth)); + + if (ctx->send_content_md5 == FLB_TRUE) { + memset(final_body_md5, 0, sizeof(final_body_md5)); + if (gcs_get_md5_base64(body, body_size, final_body_md5, + sizeof(final_body_md5)) == 0) { + content_md5_header.val = final_body_md5; + content_md5_header.val_len = strlen(final_body_md5); + flb_http_add_header(c, content_md5_header.key, content_md5_header.key_len, + content_md5_header.val, content_md5_header.val_len); + } + } + + ret = flb_http_do(c, &bytes); + if (ret == 0 && + (c->resp.status < 200 || c->resp.status >= 300)) { + flb_plg_error(ctx->ins, + "gcs upload failed with status=%i", + c->resp.status); + ret = -1; + } + flb_http_client_destroy(c); + flb_upstream_conn_release(u_conn); + + return ret; +} + +static int upload_data(struct flb_gcs *ctx, + struct upload_queue *entry, + char *buffer, + size_t buffer_size) +{ + int ret; + int append_random; + int ret_seq; + const char *predefined_acl; + flb_sds_t auth; + flb_sds_t gcs_key; + flb_sds_t gcs_key_final; + flb_sds_t gcs_key_encoded; + flb_sds_t uri; + flb_sds_t tmp; + void *gz_data = NULL; + size_t gz_size = 0; + char *upload_body; + size_t upload_size; + char random_hex[9]; + + if (gcs_under_test_mode() == FLB_TRUE) { + auth = flb_sds_create("Bearer test-token"); + } + else { + auth = get_google_token(ctx); + } + + if (!auth) { + return -1; + } + + if (ctx->key_fmt_has_seq_index) { + ctx->seq_index++; + } + + gcs_key = flb_get_s3_key(ctx->gcs_key_format, time(NULL), + entry->tag, ctx->tag_delimiters, ctx->seq_index); + if (!gcs_key) { + if (ctx->key_fmt_has_seq_index && ctx->seq_index > 0) { + ctx->seq_index--; + } + flb_sds_destroy(auth); + return -1; + } + + gcs_key_final = gcs_key; + append_random = FLB_FALSE; + if (!ctx->key_fmt_has_uuid && !ctx->key_fmt_has_seq_index && !ctx->static_file_path) { + append_random = FLB_TRUE; + } + if (append_random == FLB_TRUE) { + if (random_hex_suffix(random_hex, sizeof(random_hex)) == -1) { + flb_sds_destroy(auth); + flb_sds_destroy(gcs_key); + return -1; + } + + gcs_key_final = flb_sds_create_size(flb_sds_len(gcs_key) + 16); + if (!gcs_key_final) { + flb_errno(); + flb_sds_destroy(auth); + flb_sds_destroy(gcs_key); + return -1; + } + tmp = flb_sds_printf(&gcs_key_final, "%s-object%s", gcs_key, random_hex); + if (!tmp) { + flb_sds_destroy(auth); + flb_sds_destroy(gcs_key); + flb_sds_destroy(gcs_key_final); + return -1; + } + gcs_key_final = tmp; + flb_sds_destroy(gcs_key); + } + + if (ctx->key_fmt_has_seq_index) { + ret_seq = write_seq_index(ctx->seq_index_file, ctx->seq_index); + if (ret_seq == -1) { + flb_sds_destroy(auth); + flb_sds_destroy(gcs_key_final); + return -1; + } + } + + gcs_key_encoded = gcs_uri_encode_object_name(gcs_key_final, + flb_sds_len(gcs_key_final)); + flb_sds_destroy(gcs_key_final); + if (!gcs_key_encoded) { + flb_sds_destroy(auth); + return -1; + } + + uri = flb_sds_create_size(flb_sds_len(ctx->bucket) + + flb_sds_len(gcs_key_encoded) + 48); + if (!uri) { + flb_errno(); + flb_sds_destroy(auth); + flb_sds_destroy(gcs_key_encoded); + return -1; + } + tmp = flb_sds_printf(&uri, "/upload/storage/v1/b/%s/o?uploadType=media&name=%s", + ctx->bucket, gcs_key_encoded); + flb_sds_destroy(gcs_key_encoded); + if (!tmp) { + flb_sds_destroy(uri); + flb_sds_destroy(auth); + return -1; + } + uri = tmp; + + if (ctx->compression_type == FLB_GCS_COMPRESSION_GZIP) { + tmp = flb_sds_cat(uri, "&contentEncoding=gzip", 21); + if (!tmp) { + flb_sds_destroy(uri); + flb_sds_destroy(auth); + return -1; + } + uri = tmp; + } + + predefined_acl = get_predefined_acl(ctx->canned_acl); + if (predefined_acl) { + tmp = flb_sds_printf(&uri, "&predefinedAcl=%s", predefined_acl); + if (!tmp) { + flb_sds_destroy(uri); + flb_sds_destroy(auth); + return -1; + } + uri = tmp; + } + + upload_body = buffer; + upload_size = buffer_size; + if (ctx->compression_type == FLB_GCS_COMPRESSION_GZIP) { + ret = flb_gzip_compress(buffer, buffer_size, &gz_data, &gz_size); + if (ret != 0 || !gz_data) { + flb_plg_error(ctx->ins, "could not gzip buffered data"); + flb_sds_destroy(auth); + flb_sds_destroy(uri); + if (ctx->key_fmt_has_seq_index && ctx->seq_index > 0) { + ctx->seq_index--; + write_seq_index(ctx->seq_index_file, ctx->seq_index); + } + return -1; + } + + upload_body = gz_data; + upload_size = gz_size; + flb_plg_debug(ctx->ins, + "Pre-compression chunk size is %zu, After compression, chunk is %zu bytes", + buffer_size, gz_size); + } + + ret = gcs_upload_object(ctx, auth, uri, upload_body, upload_size); + if (gz_data) { + flb_free(gz_data); + } + flb_sds_destroy(auth); + flb_sds_destroy(uri); + + if (ret != 0 && ctx->key_fmt_has_seq_index && ctx->seq_index > 0) { + ctx->seq_index--; + write_seq_index(ctx->seq_index_file, ctx->seq_index); + } + + return ret; +} + +static int process_upload_queue(struct flb_gcs *ctx) +{ + struct mk_list *head; + struct mk_list *tmp; + struct upload_queue *entry; + char *buffer; + size_t buffer_size; + int ret; + time_t now; + + /* + * Uploads can yield while waiting for network I/O. Do not let the periodic + * timer re-enter this function and process the same queue entry while an + * output flush is still handling it. + */ + if (ctx->upload_queue_processing == FLB_TRUE) { + return 0; + } + ctx->upload_queue_processing = FLB_TRUE; + + mk_list_foreach_safe(head, tmp, &ctx->upload_queue) { + entry = mk_list_entry(head, struct upload_queue, _head); + now = time(NULL); + if (now < entry->upload_time) { + if (ctx->preserve_data_ordering == FLB_TRUE) { + break; + } + continue; + } + gcs_store_file_lock(entry->upload_file); + + ret = construct_request_buffer(ctx, entry, &buffer, &buffer_size); + if (ret == -1) { + gcs_store_file_unlock(entry->upload_file); + entry->retry_counter++; + continue; + } + + ret = upload_data(ctx, entry, buffer, buffer_size); + + if (ret == 0) { + gcs_store_file_delete(ctx, entry->upload_file); + flb_free(buffer); + remove_from_queue(entry); + if (ctx->preserve_data_ordering == FLB_TRUE) { + break; + } + } + else { + flb_free(buffer); + gcs_store_file_unlock(entry->upload_file); + entry->retry_counter++; + entry->upload_time = now + (2 * entry->retry_counter); + if (ctx->preserve_data_ordering == FLB_TRUE) { + break; + } + } + } + + ctx->upload_queue_processing = FLB_FALSE; + return 0; +} + + +static int attach_recovered_chunk(struct flb_gcs *ctx, struct flb_fstore_file *fsf) +{ + struct gcs_file *chunk; + char *buf; + size_t size; + int ret; + + if (!fsf) { + return -1; + } + + if (fsf->data) { + return 0; + } + + chunk = flb_calloc(1, sizeof(struct gcs_file)); + if (!chunk) { + flb_errno(); + return -1; + } + + ret = flb_fstore_file_content_copy(ctx->fs, fsf, (void **) &buf, &size); + if (ret != 0) { + flb_free(chunk); + return -1; + } + + chunk->fsf = fsf; + chunk->size = size; + + if (ctx->upload_timeout > 0) { + chunk->create_time = time(NULL) - ctx->upload_timeout; + } + else { + chunk->create_time = time(NULL); + } + + fsf->data = chunk; + ctx->current_buffer_size += size; + + flb_free(buf); + + return 0; +} + +static void enqueue_backlog_files(struct flb_gcs *ctx) +{ + struct mk_list *head; + struct flb_fstore_file *fsf; + struct gcs_file *chunk; + + mk_list_foreach(head, &ctx->fs_stream->files) { + fsf = mk_list_entry(head, struct flb_fstore_file, _head); + + if (attach_recovered_chunk(ctx, fsf) == -1) { + flb_plg_warn(ctx->ins, + "could not recover buffered chunk %s, skipping", + fsf->name); + continue; + } + + chunk = fsf->data; + if (chunk) { + add_to_queue(ctx, chunk, (const char *) fsf->meta_buf, fsf->meta_size); + } + } +} + + +static void cb_gcs_upload(struct flb_config *config, void *data) +{ + struct flb_gcs *ctx = data; + + (void) config; + + if (!ctx) { + return; + } + + process_upload_queue(ctx); +} + + +static void gcs_upload_queue(struct flb_config *config, void *data) +{ + int async_flags; + struct flb_gcs *ctx = data; + + (void) config; + + if (!ctx) { + return; + } + + if (mk_list_size(&ctx->upload_queue) == 0) { + cb_gcs_upload(config, data); + return; + } + + async_flags = flb_stream_get_flags(&ctx->u->base); + flb_stream_disable_async_mode(&ctx->u->base); + + process_upload_queue(ctx); + + flb_stream_set_flags(&ctx->u->base, async_flags); +} + +static int flush_init(struct flb_gcs *ctx) +{ + int ret; + struct flb_sched *sched; + + if (ctx->timer_created == FLB_TRUE) { + return 0; + } + + sched = flb_sched_ctx_get(); + if (!sched) { + return -1; + } + + if (ctx->preserve_data_ordering == FLB_TRUE) { + ret = flb_sched_timer_cb_create(sched, FLB_SCHED_TIMER_CB_PERM, + ctx->timer_ms, gcs_upload_queue, ctx, NULL); + } + else { + ret = flb_sched_timer_cb_create(sched, FLB_SCHED_TIMER_CB_PERM, + ctx->timer_ms, cb_gcs_upload, ctx, NULL); + } + if (ret == -1) { + return -1; + } + + ctx->timer_created = FLB_TRUE; + return 0; +} + +/* init/flush/exit */ +static int cb_gcs_init(struct flb_output_instance *ins, struct flb_config *config, void *data) +{ + int ret; + struct flb_gcs *ctx; + const char *tmp; + (void) data; + + ctx = flb_calloc(1, sizeof(*ctx)); + if (!ctx) { + return -1; + } + ctx->ins = ins; ctx->config = config; + mk_list_init(&ctx->upload_queue); + ctx->retry_time = 0; + ctx->upload_queue_success = FLB_FALSE; + ctx->timer_created = FLB_FALSE; + + ret = flb_output_config_map_set(ins, ctx); + if (ret == -1) { + flb_plg_error(ins, "unable to load configuration"); + goto error; + } + + if (!ctx->bucket) { + flb_plg_error(ins, "'bucket' is required"); + goto error; + } + + if (ctx->store_chunk_limit < 0) { + flb_plg_error(ins, "'store_chunk_limit' cannot be negative"); + goto error; + } + + if (ctx->canned_acl && !get_predefined_acl(ctx->canned_acl)) { + flb_plg_error(ins, "unsupported canned ACL '%s'", ctx->canned_acl); + goto error; + } + + ctx->timer_ms = ctx->upload_timeout / 6; + if (ctx->timer_ms >= 60) { + ctx->timer_ms = 60000; + } + else if (ctx->timer_ms < 1) { + ctx->timer_ms = 1000; + } + else { + ctx->timer_ms *= 1000; + } + + if (!ctx->store_dir) { + ctx->store_dir = flb_sds_create("/tmp"); + if (!ctx->store_dir) { + goto error; + } + } + + if (gcs_store_init(ctx) == -1) { + goto error; + } + + tmp = getenv("GOOGLE_SERVICE_CREDENTIALS"); + if (!ctx->credentials_file && tmp) { + ctx->credentials_file = flb_sds_create(tmp); + if (!ctx->credentials_file) { + goto error; + } + ctx->credentials_file_owned = FLB_TRUE; + } + + ctx->oauth_credentials = flb_calloc(1, sizeof(struct flb_gcs_oauth_credentials)); + if (!ctx->oauth_credentials) { + flb_errno(); + goto error; + } + + if (!ctx->credentials_file || + flb_gcs_read_credentials_file(ctx, ctx->credentials_file, ctx->oauth_credentials) == -1) { + flb_errno(); + goto error; + } + + ctx->o = flb_oauth2_create(config, FLB_GCS_AUTH_URL, FLB_GCS_TOKEN_REFRESH); + if (!ctx->o) { + goto error; + } + if (pthread_mutex_init(&ctx->token_mutex, NULL) == 0) { + ctx->token_mutex_initialized = FLB_TRUE; + } + else { + goto error; + } + ctx->u = flb_upstream_create(config, FLB_GCS_DEFAULT_HOST, FLB_GCS_DEFAULT_PORT, + FLB_IO_TLS, ins->tls); + if (!ctx->u) { + goto error; + } + ctx->out_format = FLB_PACK_JSON_FORMAT_LINES; + ctx->json_date_format = FLB_PACK_JSON_DATE_DOUBLE; + if (ctx->content_type == NULL) { + ctx->content_type = flb_sds_create("application/json"); + if (!ctx->content_type) { + goto error; + } + } + + tmp = flb_output_get_property("compression", ins); + if (tmp && strcasecmp(tmp, "gzip") == 0) { + ctx->compression_type = FLB_GCS_COMPRESSION_GZIP; + } + else if (tmp && strcasecmp(tmp, "none") != 0) { + flb_plg_error(ins, "unsupported compression type '%s'", tmp); + goto error; + } + + if (strstr(ctx->gcs_key_format, "$INDEX")) { + if (init_seq_index(ctx) == -1) { + goto error; + } + ctx->key_fmt_has_seq_index = FLB_TRUE; + } + if (strstr(ctx->gcs_key_format, "$UUID")) { + ctx->key_fmt_has_uuid = FLB_TRUE; + } + + if (gcs_store_has_data(ctx) == FLB_TRUE) { + enqueue_backlog_files(ctx); + + if (mk_list_size(&ctx->upload_queue) > 0 && flush_init(ctx) == -1) { + goto error; + } + + process_upload_queue(ctx); + } + + flb_output_set_context(ins, ctx); + return 0; + +error: + gcs_ctx_destroy(ctx, config); + return -1; +} + +static void cb_gcs_flush(struct flb_event_chunk *event_chunk, struct flb_output_flush *out_flush, + struct flb_input_instance *i_ins, void *out_context, struct flb_config *config) +{ + struct flb_gcs *ctx = out_context; + flb_sds_t payload; + int ret; + struct gcs_file *chunk; + + if (flush_init(ctx) == -1) { + FLB_OUTPUT_RETURN(FLB_RETRY); + } + (void) out_flush; + (void) i_ins; + + payload = flb_pack_msgpack_to_json_format(event_chunk->data, event_chunk->size, + ctx->out_format, ctx->json_date_format, + ctx->json_date_key, config->json_escape_unicode); + if (!payload) { + FLB_OUTPUT_RETURN(FLB_RETRY); + } + + chunk = gcs_store_file_get(ctx, event_chunk->tag, flb_sds_len(event_chunk->tag)); + if (gcs_store_buffer_put(ctx, chunk, event_chunk->tag, flb_sds_len(event_chunk->tag), + payload, flb_sds_len(payload)) == -1) { + flb_sds_destroy(payload); + FLB_OUTPUT_RETURN(FLB_RETRY); + } + flb_sds_destroy(payload); + + chunk = gcs_store_file_get(ctx, event_chunk->tag, flb_sds_len(event_chunk->tag)); + if (!chunk) { + FLB_OUTPUT_RETURN(FLB_RETRY); + } + + ret = add_to_queue(ctx, chunk, event_chunk->tag, flb_sds_len(event_chunk->tag)); + if (ret == -1) { + FLB_OUTPUT_RETURN(FLB_RETRY); + } + + /* + * Non-order-preserving mode: try to flush as many queued entries as possible. + * Preserve-order mode: process at most one queue entry per flush to keep + * strict FIFO progression. + */ + ret = process_upload_queue(ctx); + if (ret == -1) { + FLB_OUTPUT_RETURN(FLB_ERROR); + } + + FLB_OUTPUT_RETURN(FLB_OK); +} + +static int gcs_ctx_destroy(void *data, struct flb_config *config) +{ + struct flb_gcs *ctx = data; (void) config; + if (!ctx) { + return 0; + } + + /* + * Uploads require an output worker coroutine. The exit callback runs after + * the workers have stopped, so attempting an upload here can switch to an + * invalid coroutine/fiber context. Leave pending chunks in the file store; + * they are recovered and uploaded on the next startup. + */ + clear_upload_queue(ctx); + + clear_file_chunks(ctx); + + gcs_store_exit(ctx); + + if (ctx->u) { + flb_upstream_destroy(ctx->u); + } + + if (ctx->o) { + flb_oauth2_destroy(ctx->o); + } + + flb_gcs_credentials_destroy(ctx->oauth_credentials); + + if (ctx->credentials_file_owned == FLB_TRUE) { + flb_sds_destroy(ctx->credentials_file); + } + + if (ctx->seq_index_file) { + flb_sds_destroy(ctx->seq_index_file); + } + + if (ctx->token_mutex_initialized == FLB_TRUE) { + pthread_mutex_destroy(&ctx->token_mutex); + } + flb_free(ctx); + + return 0; +} + +static int cb_gcs_exit(void *data, struct flb_config *config) +{ + gcs_ctx_destroy(data, config); + + return 0; +} + +static struct flb_config_map config_map[] = { + { + FLB_CONFIG_MAP_STR, "bucket", NULL, + 0, FLB_TRUE, offsetof(struct flb_gcs, bucket), + "GCS bucket." + }, + { + FLB_CONFIG_MAP_STR, "gcs_key_format", "fluent-bit-logs/$TAG/%Y/%m/%d/%H/%M/%S", + 0, FLB_TRUE, offsetof(struct flb_gcs, gcs_key_format), + "Format string for keys in GCS." + }, + { + FLB_CONFIG_MAP_STR, "gcs_key_format_tag_delimiters", ".", + 0, FLB_TRUE, offsetof(struct flb_gcs, tag_delimiters), + "Characters used to split tag parts for gcs_key_format." + }, + { + FLB_CONFIG_MAP_BOOL, "static_file_path", "false", + 0, FLB_TRUE, offsetof(struct flb_gcs, static_file_path), + "Disable random suffix when UUID is not used in gcs_key_format." + }, + { + FLB_CONFIG_MAP_STR, "canned_acl", NULL, + 0, FLB_TRUE, offsetof(struct flb_gcs, canned_acl), + "Predefined canned ACL for objects." + }, + { + FLB_CONFIG_MAP_TIME, "upload_timeout", "10m", + 0, FLB_TRUE, offsetof(struct flb_gcs, upload_timeout), + "Upload timeout before chunk is flushed." + }, + { + FLB_CONFIG_MAP_BOOL, "send_content_md5", "false", + 0, FLB_TRUE, offsetof(struct flb_gcs, send_content_md5), + "Send Content-MD5 header with uploads." + }, + { + FLB_CONFIG_MAP_BOOL, "preserve_data_ordering", "false", + 0, FLB_TRUE, offsetof(struct flb_gcs, preserve_data_ordering), + "Enable preserve-order upload queue semantics." + }, + { + FLB_CONFIG_MAP_INT, "store_chunk_limit", "0", + 0, FLB_TRUE, offsetof(struct flb_gcs, store_chunk_limit), + "Maximum number of buffered fstore chunks for out_gcs (0 means unlimited)." + }, + { + FLB_CONFIG_MAP_SIZE, "store_dir_limit_size", (char *) NULL, + 0, FLB_TRUE, offsetof(struct flb_gcs, store_dir_limit_size), + "Limit total buffered bytes in store_dir (0 means unlimited)." + }, + { + FLB_CONFIG_MAP_STR, "content_type", "application/json", + 0, FLB_TRUE, offsetof(struct flb_gcs, content_type), + "Content type." + }, + { + FLB_CONFIG_MAP_STR, "google_service_credentials", NULL, + 0, FLB_TRUE, offsetof(struct flb_gcs, credentials_file), + "Service account JSON file." + }, + { + FLB_CONFIG_MAP_STR, "store_dir", "/tmp/fluent-bit/gcs", + 0, FLB_TRUE, offsetof(struct flb_gcs, store_dir), + "Directory for intermediate files." + }, + { + FLB_CONFIG_MAP_STR, "compression", "none", + 0, FLB_FALSE, 0, + "Compression: none or gzip." + }, + {0} +}; + +struct flb_output_plugin out_gcs_plugin = { + .name = "gcs", + .description = "Google Cloud Storage", + .cb_init = cb_gcs_init, + .cb_flush = cb_gcs_flush, + .cb_exit = cb_gcs_exit, + .event_type = FLB_OUTPUT_LOGS, + .config_map = config_map, + .flags = FLB_OUTPUT_NET | FLB_IO_TLS, + .workers = 1, +}; diff --git a/plugins/out_gcs/gcs.h b/plugins/out_gcs/gcs.h new file mode 100644 index 00000000000..57f9de845f9 --- /dev/null +++ b/plugins/out_gcs/gcs.h @@ -0,0 +1,107 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2026 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_OUT_GCS_H +#define FLB_OUT_GCS_H + +#include +#include +#include +#include +#include + +#define FLB_GCS_DEFAULT_HOST "storage.googleapis.com" +#define FLB_GCS_DEFAULT_PORT 443 +#define FLB_GCS_SCOPE "https://www.googleapis.com/auth/devstorage.read_write" +#define FLB_GCS_AUTH_URL "https://oauth2.googleapis.com/token" +#define FLB_GCS_TOKEN_REFRESH 3000 + +#define FLB_GCS_COMPRESSION_NONE 0 +#define FLB_GCS_COMPRESSION_GZIP 1 + +struct upload_queue { + struct gcs_file *upload_file; + char *tag; + int tag_len; + int retry_counter; + time_t upload_time; + struct mk_list _head; +}; + +struct flb_gcs_oauth_credentials { + flb_sds_t type; + flb_sds_t project_id; + flb_sds_t private_key_id; + flb_sds_t private_key; + flb_sds_t client_email; + flb_sds_t client_id; + flb_sds_t auth_uri; + flb_sds_t token_uri; +}; + +struct flb_gcs { + struct flb_output_instance *ins; + struct flb_config *config; + struct flb_upstream *u; + struct flb_oauth2 *o; + pthread_mutex_t token_mutex; + int token_mutex_initialized; + + flb_sds_t bucket; + flb_sds_t content_type; + flb_sds_t credentials_file; + int credentials_file_owned; + flb_sds_t store_dir; + flb_sds_t gcs_key_format; + flb_sds_t tag_delimiters; + flb_sds_t canned_acl; + int send_content_md5; + int preserve_data_ordering; + int store_chunk_limit; + size_t current_buffer_size; + size_t store_dir_limit_size; + flb_sds_t seq_index_file; + uint64_t seq_index; + int key_fmt_has_uuid; + int key_fmt_has_seq_index; + int static_file_path; + + int out_format; + int json_date_format; + flb_sds_t json_date_key; + int compression_type; + struct flb_fstore *fs; + struct flb_fstore_stream *fs_stream; + flb_sds_t fs_stream_name; + struct mk_list upload_queue; + time_t upload_timeout; + int retry_time; + int upload_queue_success; + int upload_queue_processing; + int timer_created; + int timer_ms; + + struct flb_gcs_oauth_credentials *oauth_credentials; +}; + +int gcs_jwt_encode(struct flb_gcs *ctx, char *payload, char *secret, + char **out_signature, size_t *out_size); +flb_sds_t gcs_uri_encode_object_name(const char *name, size_t name_len); + +#endif diff --git a/plugins/out_gcs/gcs_store.c b/plugins/out_gcs/gcs_store.c new file mode 100644 index 00000000000..fbf2b2475e4 --- /dev/null +++ b/plugins/out_gcs/gcs_store.c @@ -0,0 +1,286 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2026 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include + +#include "gcs.h" +#include "gcs_store.h" + +static void normalize_stream_suffix(char *out, size_t out_size, const char *in) +{ + size_t i; + char ch; + + if (!out || out_size == 0) { + return; + } + + if (!in) { + out[0] = '\0'; + return; + } + + for (i = 0; i < out_size - 1 && in[i] != '\0'; i++) { + ch = in[i]; + if ((ch >= 'a' && ch <= 'z') || + (ch >= 'A' && ch <= 'Z') || + (ch >= '0' && ch <= '9') || + ch == '_' || ch == '-' || ch == '.') { + out[i] = ch; + } + else { + out[i] = '_'; + } + } + out[i] = '\0'; +} + +static flb_sds_t gen_store_filename(void) +{ + unsigned long hash; + flb_sds_t hash_str; + flb_sds_t tmp; + struct flb_time tm; + + flb_time_get(&tm); + + hash = (unsigned long) tm.tm.tv_sec * tm.tm.tv_nsec; + + hash_str = flb_sds_create_size(64); + if (!hash_str) { + flb_errno(); + return NULL; + } + + tmp = flb_sds_printf(&hash_str, "%lu", hash); + if (!tmp) { + flb_errno(); + flb_sds_destroy(hash_str); + return NULL; + } + + return tmp; +} + +int gcs_store_init(struct flb_gcs *ctx) +{ + const char *instance_name; + char stream_suffix[96]; + flb_sds_t stream_name; + flb_sds_t tmp; + + stream_name = flb_sds_create_size(64); + if (!stream_name) { + flb_errno(); + return -1; + } + + ctx->fs = flb_fstore_create(ctx->store_dir, FLB_FSTORE_FS); + if (!ctx->fs) { + flb_sds_destroy(stream_name); + return -1; + } + + instance_name = ctx->ins->alias ? ctx->ins->alias : ctx->ins->name; + normalize_stream_suffix(stream_suffix, sizeof(stream_suffix), instance_name); + + tmp = flb_sds_printf(&stream_name, "gcs_upload_buffer_%s", stream_suffix); + if (!tmp) { + flb_sds_destroy(stream_name); + flb_fstore_destroy(ctx->fs); + ctx->fs = NULL; + + return -1; + } + stream_name = tmp; + + ctx->fs_stream_name = stream_name; + ctx->fs_stream = flb_fstore_stream_create(ctx->fs, ctx->fs_stream_name); + if (!ctx->fs_stream) { + flb_sds_destroy(ctx->fs_stream_name); + ctx->fs_stream_name = NULL; + flb_fstore_destroy(ctx->fs); + ctx->fs = NULL; + + return -1; + } + + return 0; +} + +int gcs_store_exit(struct flb_gcs *ctx) +{ + if (ctx->fs_stream_name) { + flb_sds_destroy(ctx->fs_stream_name); + ctx->fs_stream_name = NULL; + } + + if (ctx->fs) { + flb_fstore_destroy(ctx->fs); + ctx->fs = NULL; + } + ctx->fs_stream = NULL; + return 0; +} + +int gcs_store_has_data(struct flb_gcs *ctx) +{ + if (!ctx || !ctx->fs_stream) { + return FLB_FALSE; + } + return mk_list_size(&ctx->fs_stream->files) > 0 ? FLB_TRUE : FLB_FALSE; +} + +struct gcs_file *gcs_store_file_get(struct flb_gcs *ctx, const char *tag, int tag_len) +{ + struct mk_list *head; + struct flb_fstore_file *fsf; + struct gcs_file *chunk; + + mk_list_foreach(head, &ctx->fs_stream->files) { + fsf = mk_list_entry(head, struct flb_fstore_file, _head); + if (fsf->meta_size != tag_len) { + continue; + } + + chunk = fsf->data; + if (!chunk || chunk->locked == FLB_TRUE) { + continue; + } + if (strncmp(fsf->meta_buf, tag, tag_len) == 0) { + return chunk; + } + } + return NULL; +} + +int gcs_store_buffer_put(struct flb_gcs *ctx, struct gcs_file *chunk, + const char *tag, int tag_len, char *data, size_t bytes) +{ + int ret; + flb_sds_t name; + struct flb_fstore_file *fsf; + size_t space_remaining; + + if (ctx->store_dir_limit_size > 0 && + (ctx->current_buffer_size > ctx->store_dir_limit_size || + bytes > ctx->store_dir_limit_size - ctx->current_buffer_size)) { + flb_plg_error(ctx->ins, + "Buffer is full: current_buffer_size=%zu, new_data=%zu, " + "store_dir_limit_size=%zu bytes", + ctx->current_buffer_size, bytes, ctx->store_dir_limit_size); + return -1; + } + + if (!chunk) { + if (ctx->store_chunk_limit > 0 && + mk_list_size(&ctx->fs_stream->files) >= ctx->store_chunk_limit) { + flb_plg_error(ctx->ins, + "gcs local buffer chunk limit reached: limit=%d, dropping", + ctx->store_chunk_limit); + return -1; + } + + name = gen_store_filename(); + if (!name) { + return -1; + } + fsf = flb_fstore_file_create(ctx->fs, ctx->fs_stream, name, bytes); + flb_sds_destroy(name); + if (!fsf) { + return -1; + } + + ret = flb_fstore_file_meta_set(ctx->fs, fsf, (char *) tag, tag_len); + if (ret == -1) { + flb_fstore_file_delete(ctx->fs, fsf); + return -1; + } + chunk = flb_calloc(1, sizeof(struct gcs_file)); + if (!chunk) { + flb_fstore_file_delete(ctx->fs, fsf); + return -1; + } + chunk->fsf = fsf; + chunk->create_time = time(NULL); + fsf->data = chunk; + } + else { + fsf = chunk->fsf; + } + + ret = flb_fstore_file_append(fsf, data, bytes); + if (ret != 0) { + return -1; + } + + chunk->size += bytes; + ctx->current_buffer_size += bytes; + + if (ctx->store_dir_limit_size > 0) { + space_remaining = ctx->store_dir_limit_size - ctx->current_buffer_size; + if ((space_remaining * 20) < ctx->store_dir_limit_size) { + flb_plg_warn(ctx->ins, + "Buffer is almost full: current_buffer_size=%zu, " + "store_dir_limit_size=%zu bytes", + ctx->current_buffer_size, ctx->store_dir_limit_size); + } + } + return 0; +} + +int gcs_store_file_read(struct flb_gcs *ctx, struct gcs_file *chunk, + char **out_buf, size_t *out_size) +{ + return flb_fstore_file_content_copy(ctx->fs, chunk->fsf, (void **) out_buf, out_size); +} + +void gcs_store_file_lock(struct gcs_file *chunk) +{ + chunk->locked = FLB_TRUE; +} + +void gcs_store_file_unlock(struct gcs_file *chunk) +{ + chunk->locked = FLB_FALSE; +} + +int gcs_store_file_delete(struct flb_gcs *ctx, struct gcs_file *chunk) +{ + struct flb_fstore_file *fsf; + + if (!chunk) { + return 0; + } + + fsf = chunk->fsf; + if (chunk->size <= ctx->current_buffer_size) { + ctx->current_buffer_size -= chunk->size; + } + else { + ctx->current_buffer_size = 0; + } + flb_free(chunk); + flb_fstore_file_delete(ctx->fs, fsf); + + return 0; +} diff --git a/plugins/out_gcs/gcs_store.h b/plugins/out_gcs/gcs_store.h new file mode 100644 index 00000000000..c02a021eb44 --- /dev/null +++ b/plugins/out_gcs/gcs_store.h @@ -0,0 +1,49 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2026 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_OUT_GCS_STORE_H +#define FLB_OUT_GCS_STORE_H + +#include +#include +#include + +struct flb_gcs; +struct gcs_file { + struct flb_fstore_file *fsf; + flb_sds_t file_path; + size_t size; + int locked; + int failures; + time_t create_time; +}; + +int gcs_store_init(struct flb_gcs *ctx); +int gcs_store_exit(struct flb_gcs *ctx); +int gcs_store_has_data(struct flb_gcs *ctx); +struct gcs_file *gcs_store_file_get(struct flb_gcs *ctx, const char *tag, int tag_len); +int gcs_store_buffer_put(struct flb_gcs *ctx, struct gcs_file *chunk, + const char *tag, int tag_len, char *data, size_t bytes); +int gcs_store_file_read(struct flb_gcs *ctx, struct gcs_file *chunk, + char **out_buf, size_t *out_size); +void gcs_store_file_lock(struct gcs_file *chunk); +void gcs_store_file_unlock(struct gcs_file *chunk); +int gcs_store_file_delete(struct flb_gcs *ctx, struct gcs_file *chunk); + +#endif diff --git a/plugins/out_loki/loki.c b/plugins/out_loki/loki.c index 928f8bba927..e8608277538 100644 --- a/plugins/out_loki/loki.c +++ b/plugins/out_loki/loki.c @@ -759,6 +759,11 @@ static int read_label_map_path_file(struct flb_output_instance *ins, flb_sds_t p struct stat st; size_t file_size; size_t ret_size; + const char *file_mode = "r"; + +#ifdef FLB_SYSTEM_WINDOWS + file_mode = "rb"; +#endif ret = access(path, R_OK); if (ret < 0) { @@ -775,7 +780,7 @@ static int read_label_map_path_file(struct flb_output_instance *ins, flb_sds_t p } file_size = st.st_size; - fp = fopen(path, "r"); + fp = fopen(path, file_mode); if (fp == NULL) { flb_plg_error(ins, "can't open %s", path); return -1; diff --git a/plugins/out_s3/s3.c b/plugins/out_s3/s3.c index ac70da12957..80d03b7d6ca 100644 --- a/plugins/out_s3/s3.c +++ b/plugins/out_s3/s3.c @@ -42,6 +42,7 @@ #include "s3.h" #include "s3_store.h" +#include #define DEFAULT_S3_PORT 443 #define DEFAULT_S3_INSECURE_PORT 80 @@ -57,7 +58,11 @@ FLB_TLS_DEFINE(struct worker_info, s3_worker_info); #ifdef FLB_SYSTEM_WINDOWS static int setenv(const char *name, const char *value, int overwrite) { - return SetEnvironmentVariableA(name, value); + if (overwrite == 0 && getenv(name) != NULL) { + return 0; + } + + return _putenv_s(name, value); } #endif @@ -92,6 +97,134 @@ static flb_sds_t s3_format_event_chunk(struct flb_s3 *ctx, struct flb_event_chunk *event_chunk, struct flb_config *config); +/* + * enable_parquet_format - configure context for Parquet output + * + * Sets the S3 format to Parquet and forces PutObject mode. + * Returns 0 on success, -1 if Parquet support was not compiled in. + */ +static int enable_parquet_format(struct flb_s3 *ctx) +{ +#ifdef FLB_HAVE_ARROW_PARQUET + ctx->s3_format = FLB_S3_FORMAT_PARQUET; + ctx->use_put_object = FLB_TRUE; + return 0; +#else + flb_plg_error(ctx->ins, + "parquet format requires parquet-glib at compile time"); + return -1; +#endif +} + +/* + * enable_arrow_format - configure context for Apache Arrow output + * + * Sets the S3 format to Arrow and forces PutObject mode. Columnar formats + * must be written as a single complete object, so multipart upload is not + * supported. + * Returns 0 on success, -1 if Arrow support was not compiled in. + */ +static int enable_arrow_format(struct flb_s3 *ctx) +{ +#ifdef FLB_HAVE_ARROW + ctx->s3_format = FLB_S3_FORMAT_ARROW; + ctx->use_put_object = FLB_TRUE; + return 0; +#else + flb_plg_error(ctx->ins, + "arrow format requires arrow-glib at compile time"); + return -1; +#endif +} + +/* + * parse_output_format - resolve format string to format constant + * + * Returns FLB_S3_FORMAT_PARQUET for "parquet" and FLB_S3_FORMAT_ARROW for + * "arrow", otherwise delegates to flb_pack_to_json_format_type for JSON + * format types. + */ +static int parse_output_format(const char *format) +{ + if (strcasecmp(format, "parquet") == 0) { + return FLB_S3_FORMAT_PARQUET; + } + if (strcasecmp(format, "arrow") == 0) { + return FLB_S3_FORMAT_ARROW; + } + return flb_pack_to_json_format_type(format); +} + +/* + * validate_format_compression - check a codec is valid for a columnar format + * + * Compression is an axis applied on top of the format, but each columnar + * format only accepts a subset of codecs: + * - Parquet: none, snappy, gzip, zstd (page-level codec). + * - Arrow/Feather: none, zstd (Arrow IPC only supports ZSTD). + * + * Returns 0 if the FLB_AWS_COMPRESS_* codec is valid for s3_format, else -1. + */ +static int validate_format_compression(int s3_format, int compression_type) +{ + if (s3_format == FLB_S3_FORMAT_PARQUET) { + switch (compression_type) { + case FLB_AWS_COMPRESS_NONE: + case FLB_AWS_COMPRESS_SNAPPY: + case FLB_AWS_COMPRESS_GZIP: + case FLB_AWS_COMPRESS_ZSTD: + return 0; + default: + return -1; + } + } + if (s3_format == FLB_S3_FORMAT_ARROW) { + switch (compression_type) { + case FLB_AWS_COMPRESS_NONE: + case FLB_AWS_COMPRESS_ZSTD: + return 0; + default: + return -1; + } + } + return -1; +} + +/* + * s3_format_is_columnar - report whether a format compresses internally + * + * Columnar formats (Parquet, Arrow) embed compression inside the file, so the + * 'compression' codec is applied by the format writer and the uploaded object + * must NOT carry a byte-level Content-Encoding header. + */ +static int s3_format_is_columnar(int s3_format) +{ + return (s3_format == FLB_S3_FORMAT_PARQUET || + s3_format == FLB_S3_FORMAT_ARROW); +} + +/* + * s3_format_to_aws_compress_format - convert an S3 output format to the + * aws-compress columnar format identifier + * + * Translates FLB_S3_FORMAT_* to the FLB_AWS_COMPRESS_FORMAT_* identifier + * consumed by out_s3_compress_columnar(), keeping the compression layer + * decoupled from the plugin's format enum. Returns -1 for any format that is + * not a known columnar format, so a future format added to + * s3_format_is_columnar() but not mapped here fails loudly instead of being + * silently emitted as Arrow. + */ +static int s3_format_to_aws_compress_format(int s3_format) +{ + if (s3_format == FLB_S3_FORMAT_PARQUET) { + return FLB_AWS_COMPRESS_FORMAT_PARQUET; + } + if (s3_format == FLB_S3_FORMAT_ARROW) { + return FLB_AWS_COMPRESS_FORMAT_ARROW; + } + return -1; +} + static struct flb_aws_header *get_content_encoding_header(int compression_type) { static struct flb_aws_header gzip_header = { @@ -199,9 +332,10 @@ int create_headers(struct flb_s3 *ctx, char *body_md5, if (ctx->content_type != NULL) { headers_len++; } - if (ctx->compression == FLB_AWS_COMPRESS_GZIP || - ctx->compression == FLB_AWS_COMPRESS_ZSTD || - ctx->compression == FLB_AWS_COMPRESS_SNAPPY) { + if (!s3_format_is_columnar(ctx->s3_format) && + (ctx->compression == FLB_AWS_COMPRESS_GZIP || + ctx->compression == FLB_AWS_COMPRESS_ZSTD || + ctx->compression == FLB_AWS_COMPRESS_SNAPPY)) { headers_len++; } if (ctx->canned_acl != NULL) { @@ -231,9 +365,10 @@ int create_headers(struct flb_s3 *ctx, char *body_md5, s3_headers[n].val_len = strlen(ctx->content_type); n++; } - if (ctx->compression == FLB_AWS_COMPRESS_GZIP || - ctx->compression == FLB_AWS_COMPRESS_ZSTD || - ctx->compression == FLB_AWS_COMPRESS_SNAPPY) { + if (!s3_format_is_columnar(ctx->s3_format) && + (ctx->compression == FLB_AWS_COMPRESS_GZIP || + ctx->compression == FLB_AWS_COMPRESS_ZSTD || + ctx->compression == FLB_AWS_COMPRESS_SNAPPY)) { encoding_header = get_content_encoding_header(ctx->compression); if (encoding_header == NULL) { @@ -391,6 +526,37 @@ static flb_sds_t concat_path(char *p1, char *p2) return dir; } +static flb_sds_t create_buffer_path(struct flb_s3 *ctx) +{ +#ifdef FLB_SYSTEM_WINDOWS + char *temp_dir; + flb_sds_t dir; + flb_sds_t tmp; + + if (strcmp(ctx->store_dir, "/tmp/fluent-bit/s3") == 0) { + temp_dir = getenv("TEMP"); + if (temp_dir == NULL) { + temp_dir = getenv("TMP"); + } + + if (temp_dir != NULL) { + dir = flb_sds_create_size(64); + tmp = flb_sds_printf(&dir, "%s/fluent-bit/s3/%s", + temp_dir, ctx->bucket); + if (tmp == NULL) { + flb_errno(); + flb_sds_destroy(dir); + return NULL; + } + + return tmp; + } + } +#endif + + return concat_path(ctx->store_dir, ctx->bucket); +} + /* Reads in index value from metadata file and sets seq_index to value */ static int read_seq_index(char *seq_index_file, uint64_t *seq_index) { @@ -638,7 +804,6 @@ static int cb_s3_init(struct flb_output_instance *ins, const char *tmp; struct flb_s3 *ctx = NULL; struct flb_aws_client_generator *generator; - (void) config; (void) data; char *ep; struct flb_split_entry *tok; @@ -659,6 +824,7 @@ static int cb_s3_init(struct flb_output_instance *ins, ctx->retry_time = 0; ctx->upload_queue_success = FLB_FALSE; ctx->out_format = FLB_PACK_JSON_FORMAT_LINES; + ctx->s3_format = FLB_S3_FORMAT_JSON_LINES; /* * The engine default retry_limit (1) is too low for S3's internal @@ -685,6 +851,15 @@ static int cb_s3_init(struct flb_output_instance *ins, return -1; } + if (config->fips_mode_active == FLB_TRUE && ctx->send_content_md5 == FLB_TRUE) { + flb_plg_error(ctx->ins, + "send_content_md5 uses MD5 and is not available " + "when security.fips_mode is enabled"); + s3_context_destroy(ctx); + flb_output_set_context(ins, NULL); + return -1; + } + action = s3_get_retry_exhausted_action(ctx->retry_exhausted_action_str); if (action == -1) { flb_plg_error(ctx->ins, @@ -706,30 +881,54 @@ static int cb_s3_init(struct flb_output_instance *ins, /* Format key */ tmp = flb_output_get_property("format", ins); if (tmp) { - ret = flb_pack_to_json_format_type(tmp); + ret = parse_output_format(tmp); if (ret == -1) { flb_plg_error(ctx->ins, "invalid format '%s'", tmp); return -1; } - if (ret == FLB_PACK_JSON_FORMAT_JSON) { - flb_plg_warn(ctx->ins, - "'json' format is implicitly interpreted as 'json_lines' before." - "Now interpreted as 'json_lines' explicitly now"); - ret = FLB_PACK_JSON_FORMAT_LINES; + if (ret == FLB_S3_FORMAT_PARQUET) { + if (ctx->log_key != NULL) { + flb_plg_error(ctx->ins, + "'log_key' is not supported when format is " + "parquet"); + return -1; + } + if (enable_parquet_format(ctx) == -1) { + return -1; + } } - else if (ret != FLB_PACK_JSON_FORMAT_LINES && - ret != FLB_PACK_JSON_FORMAT_OTLP) { - flb_plg_error(ctx->ins, "unsupported format '%s'", tmp); - return -1; + else if (ret == FLB_S3_FORMAT_ARROW) { + if (ctx->log_key != NULL) { + flb_plg_error(ctx->ins, + "'log_key' is not supported when format is " + "arrow"); + return -1; + } + if (enable_arrow_format(ctx) == -1) { + return -1; + } } - ctx->out_format = ret; + else if (ret == FLB_PACK_JSON_FORMAT_JSON) { + flb_plg_warn(ctx->ins, + "'json' format is implicitly interpreted as " + "'json_lines'. Now interpreted as 'json_lines' " + "explicitly"); + ctx->out_format = FLB_PACK_JSON_FORMAT_LINES; + } + else if (ret == FLB_PACK_JSON_FORMAT_LINES || + ret == FLB_PACK_JSON_FORMAT_OTLP) { + ctx->out_format = ret; - if (ctx->out_format == FLB_PACK_JSON_FORMAT_OTLP && - ctx->log_key != NULL) { - flb_plg_error(ctx->ins, - "'log_key' is not supported when format is " - "otlp_json or otlp_json_pretty"); + if (ret == FLB_PACK_JSON_FORMAT_OTLP && ctx->log_key != NULL) { + flb_plg_error(ctx->ins, + "'log_key' is not supported when format is " + "otlp_json or otlp_json_pretty"); + return -1; + } + } + else { + flb_plg_error(ctx->ins, "unsupported format '%s'", tmp); return -1; } } @@ -769,7 +968,7 @@ static int cb_s3_init(struct flb_output_instance *ins, * We append the bucket name to the dir, to support multiple instances * of this plugin using the same buffer dir */ - tmp_sds = concat_path(ctx->store_dir, ctx->bucket); + tmp_sds = create_buffer_path(ctx); if (!tmp_sds) { flb_plg_error(ctx->ins, "Could not construct buffer path"); return -1; @@ -818,19 +1017,73 @@ static int cb_s3_init(struct flb_output_instance *ins, tmp = flb_output_get_property("compression", ins); if (tmp) { - ret = flb_aws_compression_get_type(tmp); - if (ret == -1) { - flb_plg_error(ctx->ins, "unknown compression: %s", tmp); - return -1; + if (strcasecmp(tmp, "parquet") == 0) { + if (ctx->log_key != NULL) { + flb_plg_error(ctx->ins, + "'log_key' is not supported when format is " + "parquet"); + return -1; + } + flb_plg_warn(ctx->ins, + "'compression=parquet' is deprecated. " + "Use 'format parquet' with 'compression' set to " + "the desired page-level codec (snappy, zstd, gzip)"); + if (enable_parquet_format(ctx) == -1) { + return -1; + } } - if (ctx->use_put_object == FLB_FALSE && - (ret == FLB_AWS_COMPRESS_ARROW || - ret == FLB_AWS_COMPRESS_PARQUET)) { - flb_plg_error(ctx->ins, - "use_put_object must be enabled when Apache Arrow or Parquet is enabled"); - return -1; + else if (strcasecmp(tmp, "arrow") == 0) { + if (ctx->log_key != NULL) { + flb_plg_error(ctx->ins, + "'log_key' is not supported when format is " + "arrow"); + return -1; + } + flb_plg_warn(ctx->ins, + "'compression=arrow' is deprecated. " + "Use 'format arrow' with 'compression' set to " + "the desired codec (zstd)"); + if (enable_arrow_format(ctx) == -1) { + return -1; + } + } + else { + /* + * 'none' explicitly selects no compression. It is not part of the + * compression dispatch table (which reserves 0/NONE as a footer), + * so accept it here and map it to FLB_AWS_COMPRESS_NONE. + */ + if (strcasecmp(tmp, "none") == 0) { + ret = FLB_AWS_COMPRESS_NONE; + } + else { + ret = flb_aws_compression_get_type(tmp); + if (ret == -1) { + flb_plg_error(ctx->ins, "unknown compression: %s", tmp); + return -1; + } + } + + if (ctx->s3_format == FLB_S3_FORMAT_PARQUET || + ctx->s3_format == FLB_S3_FORMAT_ARROW) { + /* + * For columnar formats, 'compression' selects the codec + * applied inside the format (page-level for Parquet, IPC + * buffer compression for Arrow) rather than a byte-level + * wrap of the uploaded object. The Content-Encoding header + * is intentionally not emitted for these formats. + */ + if (validate_format_compression(ctx->s3_format, ret) != 0) { + flb_plg_error(ctx->ins, + "'%s' is not a supported compression codec " + "for the configured format (parquet " + "supports snappy, gzip, zstd; arrow " + "supports zstd)", tmp); + return -1; + } + } + ctx->compression = ret; } - ctx->compression = ret; } tmp = flb_output_get_property("content_type", ins); @@ -1103,8 +1356,14 @@ static int cb_s3_init(struct flb_output_instance *ins, ctx->provider->provider_vtable->init(ctx->provider); ctx->timer_created = FLB_FALSE; - ctx->timer_ms = (int) (ctx->upload_timeout / 6) * 1000; - if (s3_plugin_under_test() == FLB_FALSE) { + if (s3_plugin_under_test() == FLB_TRUE) { + ctx->timer_ms = (int) (ctx->upload_timeout * 1000 / 6); + if (ctx->timer_ms < 100) { + ctx->timer_ms = 100; + } + } + else { + ctx->timer_ms = (int) (ctx->upload_timeout / 6) * 1000; if (ctx->timer_ms > UPLOAD_TIMER_MAX_WAIT) { ctx->timer_ms = UPLOAD_TIMER_MAX_WAIT; } @@ -1249,6 +1508,7 @@ static int upload_data(struct flb_s3 *ctx, struct s3_file *chunk, int size_check = FLB_FALSE; int part_num_check = FLB_FALSE; int timeout_check = FLB_FALSE; + int payload_needs_free = FLB_FALSE; int ret; void *payload_buf = NULL; size_t payload_size = 0; @@ -1265,9 +1525,30 @@ static int upload_data(struct flb_s3 *ctx, struct s3_file *chunk, file_first_log_time = chunk->first_log_time; } +#ifdef FLB_HAVE_ARROW + if (s3_format_is_columnar(ctx->s3_format)) { + ret = out_s3_compress_columnar(s3_format_to_aws_compress_format(ctx->s3_format), + body, body_size, &payload_buf, + &payload_size, ctx->compression); + if (ret == -1) { + flb_plg_error(ctx->ins, "Failed to convert data to columnar " + "format"); + if (chunk != NULL) { + s3_store_file_unlock(chunk); + chunk->failures += 1; + } + return FLB_RETRY; + } + preCompress_size = body_size; + body = (void *) payload_buf; + body_size = payload_size; + payload_needs_free = FLB_TRUE; + } + else +#endif if (ctx->compression != FLB_AWS_COMPRESS_NONE) { - /* Map payload */ - ret = flb_aws_compression_compress(ctx->compression, body, body_size, &payload_buf, &payload_size); + ret = flb_aws_compression_compress(ctx->compression, body, body_size, + &payload_buf, &payload_size); if (ret == -1) { flb_plg_error(ctx->ins, "Failed to compress data"); if (chunk != NULL) { @@ -1280,6 +1561,7 @@ static int upload_data(struct flb_s3 *ctx, struct s3_file *chunk, preCompress_size = body_size; body = (void *) payload_buf; body_size = payload_size; + payload_needs_free = FLB_TRUE; } } @@ -1335,7 +1617,7 @@ static int upload_data(struct flb_s3 *ctx, struct s3_file *chunk, * remove chunk from buffer list */ ret = s3_put_object(ctx, tag, file_first_log_time, body, body_size); - if (ctx->compression != FLB_AWS_COMPRESS_NONE) { + if (payload_needs_free) { flb_free(payload_buf); } if (ret < 0) { @@ -1362,7 +1644,7 @@ static int upload_data(struct flb_s3 *ctx, struct s3_file *chunk, if (chunk) { s3_store_file_unlock(chunk); } - if (ctx->compression != FLB_AWS_COMPRESS_NONE) { + if (payload_needs_free) { flb_free(payload_buf); } return FLB_RETRY; @@ -1376,7 +1658,7 @@ static int upload_data(struct flb_s3 *ctx, struct s3_file *chunk, if (chunk) { s3_store_file_unlock(chunk); } - if (ctx->compression != FLB_AWS_COMPRESS_NONE) { + if (payload_needs_free) { flb_free(payload_buf); } return FLB_RETRY; @@ -1386,7 +1668,7 @@ static int upload_data(struct flb_s3 *ctx, struct s3_file *chunk, ret = upload_part(ctx, m_upload, body, body_size, NULL); if (ret < 0) { - if (ctx->compression != FLB_AWS_COMPRESS_NONE) { + if (payload_needs_free) { flb_free(payload_buf); } m_upload->upload_errors += 1; @@ -1403,7 +1685,7 @@ static int upload_data(struct flb_s3 *ctx, struct s3_file *chunk, s3_store_file_delete(ctx, chunk); chunk = NULL; } - if (ctx->compression != FLB_AWS_COMPRESS_NONE) { + if (payload_needs_free) { flb_free(payload_buf); } if (m_upload->bytes >= ctx->file_size) { @@ -1493,15 +1775,42 @@ static int put_all_chunks(struct flb_s3 *ctx) return -1; } +#ifdef FLB_HAVE_ARROW + if (s3_format_is_columnar(ctx->s3_format)) { + ret = out_s3_compress_columnar( + s3_format_to_aws_compress_format(ctx->s3_format), + buffer, buffer_size, + &payload_buf, &payload_size, + ctx->compression); + if (ret == -1) { + flb_plg_error(ctx->ins, + "Failed to convert to columnar format, " + "uploading raw data to prevent data loss"); + } + else { + flb_free(buffer); + buffer = (void *) payload_buf; + buffer_size = payload_size; + } + } + else +#endif if (ctx->compression != FLB_AWS_COMPRESS_NONE) { - /* Map payload */ - ret = flb_aws_compression_compress(ctx->compression, buffer, buffer_size, &payload_buf, &payload_size); + ret = flb_aws_compression_compress(ctx->compression, + buffer, buffer_size, + &payload_buf, + &payload_size); if (ret == -1) { - flb_plg_error(ctx->ins, "Failed to compress data, uploading uncompressed data instead to prevent data loss"); - } else { - flb_plg_info(ctx->ins, "Pre-compression chunk size is %zu, After compression, chunk is %zu bytes", buffer_size, payload_size); + flb_plg_error(ctx->ins, + "Failed to compress data, uploading " + "uncompressed data to prevent data loss"); + } + else { + flb_plg_info(ctx->ins, + "Pre-compression chunk size is %zu, " + "After compression, chunk is %zu bytes", + buffer_size, payload_size); flb_free(buffer); - buffer = (void *) payload_buf; buffer_size = payload_size; } @@ -4156,7 +4465,9 @@ static struct flb_config_map config_map[] = { { FLB_CONFIG_MAP_STR, "format", "json_lines", 0, FLB_FALSE, 0, - "Set record output format. Supported values are json_lines, and otlp_json." + "Set output format. Supported values: json_lines, otlp_json, parquet. " + "When format is parquet, the 'compression' option controls the page-level " + "codec inside the Parquet file (snappy, zstd, gzip). Default: uncompressed." }, { FLB_CONFIG_MAP_STR, "json_date_format", NULL, @@ -4227,12 +4538,10 @@ static struct flb_config_map config_map[] = { { FLB_CONFIG_MAP_STR, "compression", NULL, 0, FLB_FALSE, 0, - "Compression type for S3 objects. Supported values: 'gzip', 'zstd', 'snappy'. " - "'arrow' and 'parquet' are also available if Apache Arrow was enabled at compile time. " - "Defaults to no compression. " - "If 'gzip' is selected, the Content-Encoding HTTP Header will be set to 'gzip'. " - "If 'zstd' is selected, the Content-Encoding HTTP Header will be set to 'zstd'. " - "If 'snappy' is selected, the Content-Encoding HTTP Header will be set to 'snappy'." + "Compression type for S3 objects. Supported values: 'gzip', 'zstd', 'snappy', " + "'arrow'. When format is 'parquet', this sets the page-level codec inside the " + "Parquet file. 'compression=parquet' is deprecated; use 'format parquet' instead. " + "Defaults to no compression." }, { FLB_CONFIG_MAP_STR, "content_type", NULL, diff --git a/plugins/out_s3/s3.h b/plugins/out_s3/s3.h index 3a007b1a5f7..08fc846d0ea 100644 --- a/plugins/out_s3/s3.h +++ b/plugins/out_s3/s3.h @@ -27,6 +27,11 @@ #include #include +/* S3 output format types */ +#define FLB_S3_FORMAT_JSON_LINES 0 +#define FLB_S3_FORMAT_PARQUET 100 +#define FLB_S3_FORMAT_ARROW 101 + /* Upload data to S3 in 5MB chunks */ #define MIN_CHUNKED_UPLOAD_SIZE 5242880 #define MAX_CHUNKED_UPLOAD_SIZE 50000000 @@ -127,6 +132,7 @@ struct flb_s3 { int static_file_path; int retry_exhausted_action; int compression; + int s3_format; int port; int insecure; size_t store_dir_limit_size; diff --git a/plugins/out_stdout/stdout.c b/plugins/out_stdout/stdout.c index a9ceb028132..16f85cacb27 100644 --- a/plugins/out_stdout/stdout.c +++ b/plugins/out_stdout/stdout.c @@ -36,6 +36,7 @@ #include #include +#include #include "stdout.h" @@ -402,7 +403,7 @@ static void cb_stdout_flush(struct flb_event_chunk *event_chunk, printf("[%zd] %s: [[", cnt++, event_chunk->tag); - printf("%"PRId32".%09lu, ", (int32_t) log_event.timestamp.tm.tv_sec, + printf("%"PRId64".%09lu, ", (int64_t) log_event.timestamp.tm.tv_sec, log_event.timestamp.tm.tv_nsec); msgpack_object_print(stdout, *log_event.metadata); diff --git a/plugins/out_syslog/syslog.c b/plugins/out_syslog/syslog.c index b191fe778b9..50e16259338 100644 --- a/plugins/out_syslog/syslog.c +++ b/plugins/out_syslog/syslog.c @@ -25,6 +25,7 @@ #include #include #include +#include #include "syslog_conf.h" @@ -101,6 +102,90 @@ static struct { { NULL, 0,-1 }, }; +#ifdef FLB_HAVE_TLS +static int syslog_configure_tls_options(struct flb_output_instance *ins) +{ + int ret; + + if (ins->tls_verify_hostname == FLB_TRUE) { + ret = flb_tls_set_verify_hostname(ins->tls, ins->tls_verify_hostname); + if (ret == -1) { + return -1; + } + } + + if (ins->tls_min_version != NULL || ins->tls_max_version != NULL) { + ret = flb_tls_set_minmax_proto(ins->tls, + ins->tls_min_version, + ins->tls_max_version); + if (ret != 0) { + return -1; + } + } + + if (ins->tls_ciphers != NULL) { + ret = flb_tls_set_ciphers(ins->tls, ins->tls_ciphers); + if (ret != 0) { + return -1; + } + } + +#if defined(FLB_SYSTEM_WINDOWS) + if (ins->tls_win_use_enterprise_certstore) { + ret = flb_tls_set_use_enterprise_store(ins->tls, + ins->tls_win_use_enterprise_certstore); + if (ret == -1) { + return -1; + } + } + + if (ins->tls_win_thumbprints) { + ret = flb_tls_set_client_thumbprints(ins->tls, ins->tls_win_thumbprints); + if (ret == -1) { + return -1; + } + } + + if (ins->tls_win_certstore_name) { + ret = flb_tls_set_certstore_name(ins->tls, ins->tls_win_certstore_name); + if (ret == -1) { + return -1; + } + + ret = flb_tls_load_system_certificates(ins->tls); + if (ret == -1) { + return -1; + } + } +#endif + + return 0; +} + +static int syslog_configure_dtls_context(struct flb_output_instance *ins) +{ + if (ins->tls != NULL) { + flb_tls_destroy(ins->tls); + ins->tls = NULL; + } + + ins->tls = flb_tls_create(FLB_TLS_CLIENT_MODE_DGRAM, + ins->tls_verify, + ins->tls_debug, + ins->tls_vhost, + ins->tls_ca_path, + ins->tls_ca_file, + ins->tls_crt_file, + ins->tls_key_file, + ins->tls_key_passwd); + if (ins->tls == NULL) { + return -1; + } + + return syslog_configure_tls_options(ins); +} +#endif + /* '"', '\' ']' */ static char rfc5424_sp_value[256] = { 0, 0, 0 , 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 , 0 , 0, 0, @@ -894,6 +979,7 @@ static void cb_syslog_flush(struct flb_event_chunk *event_chunk, static int cb_syslog_init(struct flb_output_instance *ins, struct flb_config *config, void *data) { + int ret; int io_flags; struct flb_syslog *ctx = NULL; @@ -926,9 +1012,30 @@ static int cb_syslog_init(struct flb_output_instance *ins, struct flb_config *co } } else { +#ifdef FLB_HAVE_TLS + if (ctx->parsed_mode == FLB_SYSLOG_DTLS) { + ret = syslog_configure_dtls_context(ins); + if (ret != 0) { + flb_plg_error(ins, "could not initialize DTLS context"); + flb_syslog_config_destroy(ctx); + return -1; + } + } +#else + if (ctx->parsed_mode == FLB_SYSLOG_DTLS) { + flb_plg_error(ins, "could not initialize DTLS context"); + flb_syslog_config_destroy(ctx); + return -1; + } +#endif - /* use TLS ? */ - if (ins->use_tls == FLB_TRUE) { + if (ctx->parsed_mode == FLB_SYSLOG_UDP) { + io_flags = FLB_IO_UDP; + } + else if (ctx->parsed_mode == FLB_SYSLOG_DTLS) { + io_flags = FLB_IO_DTLS; + } + else if (ins->use_tls == FLB_TRUE) { io_flags = FLB_IO_TLS; } else { @@ -940,7 +1047,7 @@ static int cb_syslog_init(struct flb_output_instance *ins, struct flb_config *co } ctx->u = flb_upstream_create(config, ins->host.name, ins->host.port, - io_flags, ins->tls); + io_flags, ins->tls); if (!(ctx->u)) { flb_syslog_config_destroy(ctx); return -1; @@ -948,7 +1055,7 @@ static int cb_syslog_init(struct flb_output_instance *ins, struct flb_config *co flb_output_upstream_set(ctx->u, ins); } - /* Set the plugin context */ + /* Set the plugin context for all modes, including UDP. */ flb_output_set_context(ins, ctx); flb_plg_info(ctx->ins, "setup done for %s:%i (TLS=%s)", @@ -969,8 +1076,8 @@ static int cb_syslog_exit(void *data, struct flb_config *config) flb_upstream_destroy(ctx->u); } - if (ctx->fd > 0) { - close(ctx->fd); + if (ctx->fd != FLB_INVALID_SOCKET) { + flb_socket_close(ctx->fd); } flb_syslog_config_destroy(ctx); @@ -1046,9 +1153,8 @@ static struct flb_config_map config_map[] = { { FLB_CONFIG_MAP_STR, "mode", "udp", 0, FLB_TRUE, offsetof(struct flb_syslog, mode), - "Set the desired transport type, the available options are tcp and udp. If you need to " - "use a TLS secure channel, choose 'tcp' mode here and enable the 'tls' option separately. " - "DTLS over udp is not supported by this plugin." + "Set the desired transport type, the available options are udp, tcp, tls and dtls. " + "Use tls=on together with mode=dtls." }, { diff --git a/plugins/out_syslog/syslog_conf.c b/plugins/out_syslog/syslog_conf.c index 5523c1f0dca..04ab53583f1 100644 --- a/plugins/out_syslog/syslog_conf.c +++ b/plugins/out_syslog/syslog_conf.c @@ -87,9 +87,6 @@ struct flb_syslog *flb_syslog_config_create(struct flb_output_instance *ins, return NULL; } - /* Set context */ - flb_output_set_context(ins, ctx); - /* Config Mode */ tmp = flb_output_get_property("mode", ins); if (tmp) { @@ -102,6 +99,9 @@ struct flb_syslog *flb_syslog_config_create(struct flb_output_instance *ins, else if (!strcasecmp(tmp, "udp")) { ctx->parsed_mode = FLB_SYSLOG_UDP; } + else if (!strcasecmp(tmp, "dtls")) { + ctx->parsed_mode = FLB_SYSLOG_DTLS; + } else { flb_plg_error(ctx->ins, "unknown syslog mode %s", tmp); flb_syslog_config_destroy(ctx); @@ -111,8 +111,15 @@ struct flb_syslog *flb_syslog_config_create(struct flb_output_instance *ins, if (ctx->parsed_mode == FLB_SYSLOG_UDP && ins->use_tls == FLB_TRUE) { flb_plg_error(ctx->ins, - "invalid configuration: mode=udp with tls=on is unsupported " - "(DTLS is not implemented)"); + "invalid configuration: mode=udp with tls=on is unsupported; " + "use mode=dtls for secure datagram transport"); + flb_syslog_config_destroy(ctx); + return NULL; + } + + if (ctx->parsed_mode == FLB_SYSLOG_DTLS && ins->use_tls == FLB_FALSE) { + flb_plg_error(ctx->ins, + "invalid configuration: mode=dtls requires tls=on"); flb_syslog_config_destroy(ctx); return NULL; } @@ -161,6 +168,9 @@ struct flb_syslog *flb_syslog_config_create(struct flb_output_instance *ins, } } + /* Set context after validation succeeds */ + flb_output_set_context(ins, ctx); + return ctx; } diff --git a/plugins/out_syslog/syslog_conf.h b/plugins/out_syslog/syslog_conf.h index 0e8d3911d96..80c8d2ea383 100644 --- a/plugins/out_syslog/syslog_conf.h +++ b/plugins/out_syslog/syslog_conf.h @@ -28,6 +28,7 @@ #define FLB_SYSLOG_UDP 0 #define FLB_SYSLOG_TCP 1 #define FLB_SYSLOG_TLS 2 +#define FLB_SYSLOG_DTLS 3 #define FLB_SYSLOG_RFC3164 0 #define FLB_SYSLOG_RFC5424 1 diff --git a/skills/README.md b/skills/README.md new file mode 100644 index 00000000000..4d0da245c86 --- /dev/null +++ b/skills/README.md @@ -0,0 +1,38 @@ +# LLM Skills + +This directory contains portable Markdown skill bundles for LLM agents working +in this repository. Each skill should live in its own subdirectory with a +`SKILL.md` entrypoint and any focused companion guides it needs. + +## Available Skills + +- Fluent Bit: [`fluent-bit/SKILL.md`](fluent-bit/SKILL.md) + - Repository workflow, testing, patch review, pipeline architecture, and + recurring subsystem guidance for Fluent Bit work. + - Sub-skills: + - [`fluent-bit/patch-workflow.md`](fluent-bit/patch-workflow.md): + implementation, review, and commit workflow. + - [`fluent-bit/pipeline-architecture.md`](fluent-bit/pipeline-architecture.md): + runtime model for shared pipeline changes. + - [`fluent-bit/subsystem-patterns.md`](fluent-bit/subsystem-patterns.md): + recurring subsystem search routes and behavioral checks. + - [`fluent-bit/testing.md`](fluent-bit/testing.md): + focused CTest, integration, valgrind, and Windows runtime-test guidance. + +## How to Use + +Start with the skill entrypoint, then read only the companion files relevant to +the task. For Fluent Bit work, begin with: + +```text +skills/fluent-bit/SKILL.md +``` + +## Adding Skills + +When adding a new skill: + +- create a new subdirectory under `skills/`; +- include a `SKILL.md` entrypoint; +- keep instructions portable and tool-agnostic when possible; +- update this index with the new skill name, entrypoint, and purpose. diff --git a/skills/fluent-bit/README.md b/skills/fluent-bit/README.md new file mode 100644 index 00000000000..e879eeb5174 --- /dev/null +++ b/skills/fluent-bit/README.md @@ -0,0 +1,29 @@ +# Fluent Bit LLM Skills + +This folder contains portable Markdown skills for agents working on the Fluent +Bit repository. They are intentionally tool-agnostic: any LLM can read these +files as instructions, then use whatever shell, editor, or CI interface it has. + +## Files + +- `SKILL.md`: entrypoint and operating principles. +- `testing.md`: focused CTest, integration, and valgrind expectations. +- `patch-workflow.md`: implementation, review, and commit workflow. +- `pipeline-architecture.md`: runtime model for shared pipeline changes. +- `subsystem-patterns.md`: recurring Fluent Bit subsystem routes and checks. + +## Suggested Agent Prompt + +```text +Before working in this repository, read skills/fluent-bit/SKILL.md. +For code changes, also read skills/fluent-bit/patch-workflow.md and +skills/fluent-bit/testing.md. For shared runtime changes, read +skills/fluent-bit/pipeline-architecture.md. For known subsystem areas, read +skills/fluent-bit/subsystem-patterns.md. +``` + +## Maintenance + +Keep these files concise and operational. Add subsystem notes only when they +change how an agent should search, patch, test, or report work in this repo. + diff --git a/skills/fluent-bit/SKILL.md b/skills/fluent-bit/SKILL.md new file mode 100644 index 00000000000..55fa765dc3b --- /dev/null +++ b/skills/fluent-bit/SKILL.md @@ -0,0 +1,97 @@ +--- +name: fluent-bit +description: Work in the Fluent Bit C/C++ repository using its scoped patch, testing, runtime, integration, lifecycle, and review conventions. Use for Fluent Bit source, plugin, pipeline, Windows runtime-test, CI, configuration, protocol, or regression tasks. +--- + +# Fluent Bit Repository Skill + +Use this skill when working in the Fluent Bit repository or in a similar +C/C++ plugin-based telemetry pipeline. It is written for any LLM agent: read the +relevant linked files, inspect the current checkout, make the smallest correct +change, and report exact verification. + +## When to Use + +- A task mentions Fluent Bit source, tests, plugins, runtime behavior, routing, + storage, shutdown, configuration validation, or protocol encoding. +- A task asks whether a reported Fluent Bit bug is still present. +- A task asks for implementation or review of a Fluent Bit patch. +- A task asks for the right focused tests, integration scenarios, or valgrind + checks for a touched Fluent Bit component. + +## Required Reading Order + +1. Read this file. +2. Read `testing.md` before changing behavior or closing out a task. +3. Read `patch-workflow.md` before editing code or reviewing a patch. +4. Read `pipeline-architecture.md` for shared runtime, routing, lifecycle, + processor, chunk, task, storage, metrics, retry, or signal-aware changes. +5. Read `subsystem-patterns.md` when the task touches one of the listed + recurring areas. + +## Operating Principles + +- Verify the current checkout before patching. A report may already be fixed. +- Prefer the repository's existing helpers, source-of-truth resolvers, and + local conventions over new parallel logic. +- Keep changes scoped to the affected component. Put plugin logic in its plugin + directory; shared behavior belongs in `src/`, `include/fluent-bit/`, or `lib/`. +- Treat bundled libraries under `lib/` as third-party or separately maintained + code unless the path is clearly Fluent Bit-owned. Ask for explicit user + confirmation before editing them, using a confirmation popup when available. + Keep those edits isolated and upstreamable as focused patches. +- Fix shared helper semantics when the bug is in a helper, instead of patching + only one visible caller. +- Preserve real input paths. If the request asks to enrich or correct an + existing path, solve it in the relevant layer instead of faking another input + route. +- Treat shutdown, architecture-specific failures, memory errors, and route + accounting mismatches as real lifecycle problems until traced through the + exact failing path. +- Separate validated behavior from environment noise. If a focused test passes + but a broader legacy suite fails for unrelated reasons, report both signals. + +## Standard Commands + +```sh +cmake -S . -B build -DFLB_TESTS_RUNTIME=On -DFLB_TESTS_INTERNAL=On +cmake --build build -j8 +ctest --test-dir build --output-on-failure +ctest --test-dir build -R --output-on-failure +./build/bin/fluent-bit -c conf/fluent-bit.conf +``` + +Windows supports runtime-test builds and execution on the latest master +revision. Initialize the appropriate MSVC environment, configure runtime tests, +and prefer focused targets or CTest matches: + +```sh +cmake -S . -B build -DFLB_TESTS_RUNTIME=On -DFLB_TESTS_INTERNAL=On +cmake --build build --target +ctest --test-dir build -R '^$' --output-on-failure +``` + +The GitHub Actions Windows unit-test matrix executes runtime tests only for x64 +to control CI running time. Keep that workflow policy scoped to GitHub Actions; +local agents and AI cloud builds should enable and run applicable runtime tests +without inheriting the x86/ARM64 CI restriction. + +For Python integration scenarios: + +```sh +cd tests/integration && ./setup-venv.sh +cd tests/integration && ./run_tests.py --list +cd tests/integration && ./run_tests.py +``` + +## Close-Out Requirements + +Final responses for implementation tasks should include: + +- What changed, with file paths. +- The exact verification commands run. +- Pass/fail status. +- Whether valgrind was used when integration coverage applies. +- Any bundled library patch touched, including the upstream project/path and + confirmation that the user approved editing it. +- Any concrete blocker for required tests that could not run. diff --git a/skills/fluent-bit/patch-workflow.md b/skills/fluent-bit/patch-workflow.md new file mode 100644 index 00000000000..b2d6fad336d --- /dev/null +++ b/skills/fluent-bit/patch-workflow.md @@ -0,0 +1,97 @@ +# Fluent Bit Patch and Review Workflow + +Use this guide when implementing or reviewing Fluent Bit changes. + +## Before Editing + +- Inspect the current checkout. Do not assume a reported bug is still live. +- Search with `rg` first. +- Read the exact source path, tests, and helpers involved. +- Trace from public configuration or input surface to the failing behavior. +- Identify whether the problem belongs in a plugin, a shared helper, core + runtime, a bundled library, or tests. +- If the fix would touch bundled library code under `lib/`, get explicit user + confirmation before editing. Use a confirmation popup when the environment + supports one; otherwise ask in chat. + +## Implementation Rules + +- Keep patches minimal and scoped. +- Use existing helpers and source-of-truth functions before adding new logic. +- When a shared helper has wrong semantics, fix the helper and update callers + consistently. +- Preserve explicit zero values; use clear sentinels for unknown values. +- Do not downgrade real I/O, parse, or lifecycle failures just to quiet logs. +- Do not add broad refactors or formatting churn around the fix. +- Keep bundled library edits isolated from Fluent Bit glue changes and write + them as upstreamable patches for the library's own project. +- Follow Fluent Bit C style: + - variables at function start; + - braces for all `if`, `else`, `while`, and `do` blocks; + - function opening brace on the next line; + - `snake_case` names with existing component prefixes; + - `/* ... */` comments only where useful. + +## Review Stance + +Prioritize: + +- bugs and behavioral regressions; +- missing tests; +- lifecycle or memory-safety risks; +- config compatibility risks; +- route, signal, storage, or retry accounting regressions. + +When reviewing claims like "this enables validation" or "this caches +resolution," distinguish: + +- what the current patch actually wires; +- what runtime or binding plumbing is still missing; +- whether behavior is one-shot lookup, repeated resolver use, or true cache + semantics. + +## Commit Guidance + +Use component-prefix subjects consistent with local history: + +```sh +git commit -s -m "component: short imperative description" +``` + +Common examples: + +- `engine: fix flush buffer handling` +- `tests: internal: add parser regression coverage` +- `tests: integration: cover schema registry resolution` + +For bundled library changes, keep the library patch in its own commit unless the +user explicitly asks otherwise. Use the prefix accepted for that path by the +repository linter, and mention the upstream project/path in the commit body when +that context is useful. + +Do not invent generic prefixes when the repository linter infers a narrower +prefix. Run the linter when creating commits: + +```sh +python .github/scripts/commit_prefix_check.py +``` + +If `gitpython` is missing: + +```sh +python3 -m pip install gitpython +``` + +Before pushing or opening a PR, fetch the base branch and lint the PR range, not +just `HEAD`. The checker can fall back to `HEAD`-only validation if the base ref +is missing locally: + +```sh +git fetch --all --prune +git fetch origin :origin/ +GITHUB_EVENT_NAME=pull_request GITHUB_BASE_REF= \ + python .github/scripts/commit_prefix_check.py +``` + +Do not open issues, pull requests, or remote branches unless explicitly asked. +Do not rewrite history, amend commits, or force-push unless explicitly asked. diff --git a/skills/fluent-bit/pipeline-architecture.md b/skills/fluent-bit/pipeline-architecture.md new file mode 100644 index 00000000000..600b7118b0f --- /dev/null +++ b/skills/fluent-bit/pipeline-architecture.md @@ -0,0 +1,86 @@ +# Fluent Bit Pipeline Architecture Skill + +Use this guide for shared runtime, routing, task, chunk, storage, processor, +filter, output, metric, retry, and shutdown changes. + +## Runtime Model + +Data moves through: + +```text +input -> chunk -> router -> task -> filter/processor -> output -> engine result +``` + +Routing is per output instance. One chunk can fan out to many routes. Route +state is independent, so success, retry, and drop can differ for each output. + +## Data Units + +- A signal is the high-level type: logs, metrics, traces, profiles, or blobs. +- A record/event is the logical payload unit inside a signal. +- A chunk is the persisted or queued container, often MessagePack-backed. +- A task is the engine execution unit for a chunk across routes. + +Never assume "one chunk equals one route" or "one serialized event equals one +log record" in shared code. + +## Component Responsibilities + +- Inputs (`plugins/in_*`) create or append data and trigger ingestion. +- Input chunk layer (`src/flb_input_chunk.c`) manages chunk lifecycle, routing + masks, storage pressure, and drop/release behavior. +- Router (`src/flb_router*.c`) resolves tag and signal matches to outputs. +- Task layer (`src/flb_task.c`) tracks per-route state and retries. +- Filters (`plugins/filter_*`) run on matching streams before output flush. +- Processors (`plugins/processor_*`) can run in input or output contexts and may + mutate or drop payloads. +- Outputs (`plugins/out_*`) serialize or protocol-encode and return flush + results. +- Engine (`src/flb_engine.c`) applies retry/drop accounting and task teardown. + +## Signal-Aware Rules + +- Shared paths must branch correctly by event type. +- Log-only record semantics may not apply to metrics, traces, profiles, or + blobs. +- Group or metadata markers can be serialized events; treat them as data-shape + artifacts unless an interface explicitly requires them. + +## Counting and Metrics + +Separate: + +- serialized events in a buffer; +- logical records after processing; +- per-route processed, retry, and drop counters; +- byte accounting for chunk bytes versus route-effective bytes. + +Prefer route-aware values for route metrics. Preserve explicit zero values. + +## Retry and Drop Semantics + +- `FLB_OK`: route succeeded. +- `FLB_RETRY`: route keeps the task or chunk for retry scheduling. +- `FLB_ERROR`: route failure/drop path. + +Final chunk release happens only when all active routes are resolved. + +## Storage and Backlog + +In-memory and filesystem backlog paths may use different code paths. Validate +both when touching chunk, task, storage, lifecycle, or accounting code. +Backlog-loaded chunks must preserve route state and accounting parity with +live-ingested chunks. + +## Review Checklist + +- Trace one full path for affected signals: + input -> chunk -> task -> output -> engine completion. +- Verify fan-out behavior: one chunk, multiple outputs. +- Verify processor behavior: drop, modify, and no-op in input and output + contexts when relevant. +- Verify empty payload behavior. +- Verify metrics and counters for success, retry, and drop paths. +- Verify shutdown cleanup if event channels, file descriptors, coroutines, or + scheduler state are touched. + diff --git a/skills/fluent-bit/subsystem-patterns.md b/skills/fluent-bit/subsystem-patterns.md new file mode 100644 index 00000000000..b5bb2fbe9ef --- /dev/null +++ b/skills/fluent-bit/subsystem-patterns.md @@ -0,0 +1,109 @@ +# Fluent Bit Subsystem Patterns + +Use this guide as a routing map for recurring Fluent Bit tasks. Revalidate exact +code in the current checkout before relying on any pattern. + +## Config Map and Proxy Plugins + +Search first: + +```sh +rg -n "flb_config_map_create|flb_config_map_properties_check|flb_plugin_proxy|config_map" \ + src include plugins +``` + +Key rule: C-side `config_map` field wiring can participate in unknown-key +validation, but it may not be end-to-end for language bindings. Check whether +the binding surface can pass a real config map and whether custom plugin +registration plumbing uses it. + +## Node Exporter Metrics File Logging + +Search first: + +```sh +rg -n "ne_utils|thermal|throttle|ENOENT|FLB_LOG_ERROR|FLB_LOG_DEBUG" \ + plugins/in_node_exporter_metrics +``` + +Key rule: missing optional sysfs files can be debug-only, but real `open()` or +`read()` failures should remain errors. A durable patch shape is centralized +errno-aware helper logic, with only `ENOENT` eligible for downgrade. + +## Rewrite Tag and Emitter Backlog + +Search first: + +```sh +rg -n "pending_bytes|mem_buf_limit|is_queue_overlimit|in_emitter|rewrite_tag" \ + plugins tests +``` + +Key rule: verify whether the current branch already has bounded backlog and +overlimit handling before patching. If already fixed, rerun focused coverage and +stop without editing code. + +## Kubernetes Filter on Fluent Bit Internal Logs + +Search first: + +```sh +rg -n "Kube_Namespace_File|kube_local_fluentbit_logs|fluentbit_logs|flb_kube_meta_get_local" \ + plugins tests +``` + +Key rule: keep internal logs as real `in_fluentbit_logs` input. Add metadata in +the Kubernetes filter path instead of pretending the records came from tail or a +different tag source. + +## Scheduler and Shutdown Regressions + +Search first: + +```sh +rg -n "flb_sched_destroy|mk_event_channel_destroy|processor_private_inputs_use_main_loop" \ + src include tests +rg -n "ch_events" src include tests +``` + +Key rule: shutdown crashes often require tracing the exact event-channel, +file-descriptor, scheduler, and processor path. Architecture-specific failures +can expose real initialization or teardown bugs. + +## in_ebpf OpenSSL Path Discovery + +Search first: + +```sh +rg -n "trace_openssl|FLB_IN_EBPF_LIBSSL_PATH|OPENSSL_SSL_LIBRARY|OpenSSL::SSL|bpf.c.in" \ + plugins/in_ebpf +``` + +Key rule: the `libssl` path is baked into generated BPF source at build time. +Fix path discovery in CMake/template generation rather than hardcoding +`libssl.so.3` in source. + +## Kafka Avro and Schema Registry + +Search first: + +```sh +rg -n "schema_registry|flb_kafka_schema_registry_resolve|FLB_HAVE_AVRO_ENCODER|Confluent" \ + plugins/out_kafka tests/integration tests/internal +``` + +Key rule: parser-only internal coverage is not enough for live resolver +behavior. Use `tests/integration/scenarios/out_kafka` for mock schema-registry +coverage and distinguish remote resolution from true local-cache semantics. + +## Avro Encoder Range Errors + +Search first: + +```sh +rg -n "msgpack2avro|FLB_AVRO_RANGE_ERROR|range|avro" src tests/internal +``` + +Key rule: nested map conversion must preserve earlier range failures. Add +focused internal regression coverage, including cases where the bad field is not +the final field. diff --git a/skills/fluent-bit/testing.md b/skills/fluent-bit/testing.md new file mode 100644 index 00000000000..03f43314caf --- /dev/null +++ b/skills/fluent-bit/testing.md @@ -0,0 +1,116 @@ +# Fluent Bit Testing Skill + +Use this guide to choose and report verification for Fluent Bit changes. + +## Test Selection + +- Prefer targeted tests when the affected area is known: + +```sh +ctest --test-dir build -R --output-on-failure +``` + +- Use `tests/internal` for core lifecycle, accounting, parser, encoder, and + helper logic. +- Use `tests/runtime` for plugin-level behavior and end-to-end C test binaries. +- Use `tests/integration` for network protocols, downstream request generation, + fake-server behavior, and plugin behavior that is awkward to cover in CTest. +- Run broader tests when changing shared lifecycle, routing, storage, task, + scheduler, or accounting behavior. + +## Windows Runtime Test Support + +The latest master revision supports building and running runtime tests on +Windows. Initialize the appropriate MSVC environment, enable runtime tests, and +run the focused target or CTest match for the affected component: + +- use `VsDevCmd.bat -arch=x64` for x64; +- use `VsDevCmd.bat -arch=x86` for x86; +- use an ARM64-capable native or cross-build Developer Command Prompt for + ARM64. For a cross-build, add `-DCMAKE_SYSTEM_NAME=Windows`, + `-DCMAKE_SYSTEM_VERSION=10.0`, and `-DCMAKE_SYSTEM_PROCESSOR=ARM64`. + +```sh +cmake -S . -B build -DFLB_TESTS_RUNTIME=On -DFLB_TESTS_INTERNAL=On +cmake --build build --target +ctest --test-dir build -R '^$' --output-on-failure +``` + +The Windows unit-test CI workflow in +`.github/workflows/call-windows-unit-tests.yaml` enables runtime tests only for +x64. Its x86 and ARM64 jobs deliberately use `FLB_TESTS_RUNTIME=Off` to control +GitHub Actions running time. This restriction applies only to that CI workflow. +Do not use it to skip runtime-test builds or execution by local agents or AI +cloud builders. + +When a cross-built target cannot execute on the current host, report that +concrete host/toolchain limitation. Do not substitute the GitHub Actions +running-time policy as the reason for skipping it. + +Use `ctest --test-dir build -N` only to inspect registration. It does not prove +that a runtime executable was built or passed; confirm the target build and run +the focused test before claiming runtime verification. + +## Integration Test Expectations + +If a touched component has a focused `tests/integration` scenario, run it before +closing the task. Run it once normally and once with valgrind when possible. + +Default verification shape for supported platforms, including Windows: + +```sh +./tests/integration/setup-venv.sh +cmake -S . -B build -DFLB_TESTS_RUNTIME=On -DFLB_TESTS_INTERNAL=On +cmake --build build -j8 +tests/integration/.venv/bin/python -m pytest -q +VALGRIND=1 VALGRIND_STRICT=1 \ + tests/integration/.venv/bin/python -m pytest -q +``` + +On Windows, keep `FLB_TESTS_RUNTIME=On` and run relevant focused runtime cases. +Valgrind is normally unavailable on Windows, so report that blocker +explicitly when the required memory-safety run cannot be performed. + +Equivalent run-test wrapper shape: + +```sh +cd tests/integration +./run_tests.py +./run_tests.py --valgrind --valgrind-strict +``` + +## Reporting Blockers + +Do not silently skip required integration or valgrind coverage. Report the exact +blocker, such as: + +- missing `build/bin/fluent-bit`; +- missing Python virtualenv; +- missing `pytest` or another Python dependency; +- unavailable scenario; +- missing `valgrind`; +- network restriction during dependency setup; +- infrastructure failure unrelated to the patch. + +## Useful Validation Habits + +- Rerun CMake before building a brand-new test target in an older build tree. + Target-not-found failures are often stale build trees, not source failures. +- Use `git diff --check` after edits to catch whitespace problems. +- Validate success and failure paths: invalid payloads, boundary sizes, + null/missing fields, and non-last bad fields when parsing maps. +- Keep generated integration artifacts out of git: + `.venv/`, `.pytest_cache/`, `results/`, and `__pycache__/`. +- If broad tests fail after focused tests pass, inspect whether failures are + pre-existing or unrelated before expanding the patch. + +## Close-Out Proof Format + +Include exact commands and outcomes: + +```text +Verification: +- PASS: cmake --build build -j8 --target +- PASS: ctest --test-dir build -R '' --output-on-failure +- BLOCKED: VALGRIND=1 ... failed because valgrind is not installed +``` diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index 907486313f5..c6e834eaa53 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -1,6 +1,6 @@ name: fluent-bit base: core20 -version: '5.0.9' +version: '5.1.0' summary: High performance logs and stream processor description: | Fluent Bit is a high performance log processor and stream processor for Linux. diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index a47a5d9d658..178fe16a534 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -38,6 +38,7 @@ set(src flb_output.c flb_output_thread.c flb_config.c + flb_fips.c flb_config_map.c flb_socket.c flb_network.c @@ -52,6 +53,7 @@ set(src flb_storage.c flb_connection.c flb_downstream.c + flb_downstream_worker.c flb_upstream.c flb_upstream_ha.c flb_upstream_node.c @@ -67,6 +69,7 @@ set(src flb_crypto.c flb_random.c flb_plugin.c + flb_plugin_alias.c flb_gzip.c flb_snappy.c flb_zstd.c @@ -523,7 +526,9 @@ set(flb_rc_files endif() # Shared Library -if(FLB_SHARED_LIB) +if(FLB_SHARED_LIB OR + (NOT MSVC AND NOT FLB_JEMALLOC AND + (FLB_TESTS_RUNTIME OR FLB_TESTS_INTERNAL OR FLB_TESTS_INTERNAL_FUZZ))) add_library(fluent-bit-shared SHARED ${src}) add_sanitizers(fluent-bit-shared) set_target_properties(fluent-bit-shared @@ -541,11 +546,13 @@ if(FLB_SHARED_LIB) PROPERTIES PDB_NAME fluent-bit.dll) endif() - # Library install routines - install(TARGETS fluent-bit-shared - LIBRARY DESTINATION ${FLB_INSTALL_LIBDIR} - COMPONENT library - RUNTIME DESTINATION ${FLB_INSTALL_BINDIR}) + if(FLB_SHARED_LIB) + # Library install routines + install(TARGETS fluent-bit-shared + LIBRARY DESTINATION ${FLB_INSTALL_LIBDIR} + COMPONENT library + RUNTIME DESTINATION ${FLB_INSTALL_BINDIR}) + endif() endif() # Static Library diff --git a/src/aws/compression/arrow/compress.c b/src/aws/compression/arrow/compress.c index 1cae74c30e6..b57ec04ca8e 100644 --- a/src/aws/compression/arrow/compress.c +++ b/src/aws/compression/arrow/compress.c @@ -13,8 +13,32 @@ #endif #include #include +#include #include +/* + * compression_type_to_garrow - map a generic FLB_AWS_COMPRESS_* codec to the + * corresponding GArrowCompressionType. + * + * This is shared by the Arrow (Feather) and Parquet writers so that + * compression is treated as an axis applied on top of the format. Codecs that + * are not valid for a given format are rejected by the caller before reaching + * the writer; any unmapped value falls back to uncompressed. + */ +static GArrowCompressionType compression_type_to_garrow(int compression_type) +{ + switch (compression_type) { + case FLB_AWS_COMPRESS_SNAPPY: + return GARROW_COMPRESSION_TYPE_SNAPPY; + case FLB_AWS_COMPRESS_GZIP: + return GARROW_COMPRESSION_TYPE_GZIP; + case FLB_AWS_COMPRESS_ZSTD: + return GARROW_COMPRESSION_TYPE_ZSTD; + default: + return GARROW_COMPRESSION_TYPE_UNCOMPRESSED; + } +} + static int choose_block_size(size_t size) { int block_size = 8 * 1024 * 1024; @@ -89,10 +113,12 @@ static GArrowTable* parse_json(uint8_t *json, int size) return table; } -static GArrowResizableBuffer* table_to_buffer(GArrowTable *table) +static GArrowResizableBuffer* table_to_buffer(GArrowTable *table, + int compression_type) { GArrowResizableBuffer *buffer; GArrowBufferOutputStream *sink; + GArrowFeatherWriteProperties *props; GError *error = NULL; gboolean success; @@ -108,10 +134,22 @@ static GArrowResizableBuffer* table_to_buffer(GArrowTable *table) return NULL; } + /* + * Apply the requested compression codec on top of the Arrow/Feather + * format. Arrow IPC only supports ZSTD (and uncompressed); unsupported + * codecs are rejected at config time before reaching this writer. + */ + props = garrow_feather_write_properties_new(); + g_object_set(props, "compression", + compression_type_to_garrow(compression_type), NULL); + success = garrow_table_write_as_feather( table, GARROW_OUTPUT_STREAM(sink), - NULL, &error); + props, &error); + g_object_unref(props); if (!success) { + flb_error("[aws][compress] Failed to write table to arrow " + "buffer: %s", error->message); g_error_free(error); g_object_unref(buffer); g_object_unref(sink); @@ -121,60 +159,14 @@ static GArrowResizableBuffer* table_to_buffer(GArrowTable *table) return buffer; } -int out_s3_compress_arrow(void *json, size_t size, void **out_buf, size_t *out_size) -{ - GArrowTable *table; - GArrowResizableBuffer *buffer; - GBytes *bytes; - gconstpointer ptr; - gsize len; - uint8_t *buf; - - table = parse_json((uint8_t *) json, size); - if (table == NULL) { - return -1; - } - - buffer = table_to_buffer(table); - g_object_unref(table); - if (buffer == NULL) { - return -1; - } - - bytes = garrow_buffer_get_data(GARROW_BUFFER(buffer)); - if (bytes == NULL) { - g_object_unref(buffer); - return -1; - } - - ptr = g_bytes_get_data(bytes, &len); - if (ptr == NULL) { - g_object_unref(buffer); - g_bytes_unref(bytes); - return -1; - } - - buf = malloc(len); - if (buf == NULL) { - g_object_unref(buffer); - g_bytes_unref(bytes); - return -1; - } - memcpy(buf, ptr, len); - *out_buf = (void *) buf; - *out_size = len; - - g_object_unref(buffer); - g_bytes_unref(bytes); - return 0; -} - #ifdef FLB_HAVE_ARROW_PARQUET -static GArrowResizableBuffer* table_to_parquet_buffer(GArrowTable *table) +static GArrowResizableBuffer* table_to_parquet_buffer(GArrowTable *table, + int compression_type) { GArrowResizableBuffer *buffer; GArrowBufferOutputStream *sink; GParquetArrowFileWriter *writer; + GParquetWriterProperties *props; GArrowSchema *schema; GError *error = NULL; gboolean success; @@ -199,14 +191,19 @@ static GArrowResizableBuffer* table_to_parquet_buffer(GArrowTable *table) return NULL; } - /* Create a new Parquet file writer */ + props = gparquet_writer_properties_new(); + gparquet_writer_properties_set_compression( + props, compression_type_to_garrow(compression_type), NULL); + writer = gparquet_arrow_file_writer_new_arrow(schema, GARROW_OUTPUT_STREAM(sink), - NULL, /* Arrow writer properties */ + props, &error); g_object_unref(schema); + g_object_unref(props); if (writer == NULL) { - flb_error("[aws][compress] Failed to create parquet writer: %s", error->message); + flb_error("[aws][compress] Failed to create parquet writer: %s", + error->message); g_error_free(error); g_object_unref(buffer); g_object_unref(sink); @@ -215,10 +212,11 @@ static GArrowResizableBuffer* table_to_parquet_buffer(GArrowTable *table) n_rows = garrow_table_get_n_rows(table); - /* Write the entire table to the Parquet file buffer */ - success = gparquet_arrow_file_writer_write_table(writer, table, n_rows, &error); + success = gparquet_arrow_file_writer_write_table(writer, table, + n_rows, &error); if (!success) { - flb_error("[aws][compress] Failed to write table to parquet buffer: %s", error->message); + flb_error("[aws][compress] Failed to write table to parquet " + "buffer: %s", error->message); g_error_free(error); g_object_unref(buffer); g_object_unref(sink); @@ -226,7 +224,6 @@ static GArrowResizableBuffer* table_to_parquet_buffer(GArrowTable *table) return NULL; } - /* Close the writer to finalize the Parquet file metadata */ success = gparquet_arrow_file_writer_close(writer, &error); if (!success) { g_error_free(error); @@ -240,9 +237,11 @@ static GArrowResizableBuffer* table_to_parquet_buffer(GArrowTable *table) g_object_unref(writer); return buffer; } +#endif - -int out_s3_compress_parquet(void *json, size_t size, void **out_buf, size_t *out_size) +int out_s3_compress_columnar(int columnar_format, void *json, size_t size, + void **out_buf, size_t *out_size, + int compression_type) { GArrowTable *table; GArrowResizableBuffer *buffer; @@ -253,14 +252,43 @@ int out_s3_compress_parquet(void *json, size_t size, void **out_buf, size_t *out table = parse_json((uint8_t *) json, size); if (table == NULL) { - flb_error("[aws][compress] Failed to parse JSON into Arrow Table for Parquet conversion"); + flb_error("[aws][compress] Failed to parse JSON into Arrow Table"); return -1; } - buffer = table_to_parquet_buffer(table); + /* Select the columnar writer; compression is applied on top by each. */ +#ifdef FLB_HAVE_ARROW_PARQUET + if (columnar_format == FLB_AWS_COMPRESS_FORMAT_PARQUET) { + buffer = table_to_parquet_buffer(table, compression_type); + } + else if (columnar_format == FLB_AWS_COMPRESS_FORMAT_ARROW) { + buffer = table_to_buffer(table, compression_type); + } + else { + flb_error("[aws][compress] unknown columnar format: %d", + columnar_format); + g_object_unref(table); + return -1; + } +#else + if (columnar_format == FLB_AWS_COMPRESS_FORMAT_PARQUET) { + flb_error("[aws][compress] Parquet format requires parquet-glib " + "at compile time"); + g_object_unref(table); + return -1; + } + else if (columnar_format != FLB_AWS_COMPRESS_FORMAT_ARROW) { + flb_error("[aws][compress] unknown columnar format: %d", + columnar_format); + g_object_unref(table); + return -1; + } + buffer = table_to_buffer(table, compression_type); +#endif g_object_unref(table); if (buffer == NULL) { - flb_error("[aws][compress] Failed to convert Arrow Table into Parquet buffer"); + flb_error("[aws][compress] Failed to encode Arrow Table into " + "columnar buffer"); return -1; } @@ -292,4 +320,3 @@ int out_s3_compress_parquet(void *json, size_t size, void **out_buf, size_t *out g_bytes_unref(bytes); return 0; } -#endif diff --git a/src/aws/compression/arrow/compress.h b/src/aws/compression/arrow/compress.h deleted file mode 100644 index f8dcd4a4248..00000000000 --- a/src/aws/compression/arrow/compress.h +++ /dev/null @@ -1,28 +0,0 @@ -/* - * This function converts out_s3 buffer into Apache Arrow format. - * - * `json` is a string that contain (concatenated) JSON objects. - * - * `size` is the length of the json data (excluding the trailing - * null-terminator character). - * - * Return 0 on success (with `out_buf` and `out_size` updated), - * and -1 on failure - */ - -int out_s3_compress_arrow(void *json, size_t size, void **out_buf, size_t *out_size); - -#ifdef FLB_HAVE_ARROW_PARQUET -/* - * This function converts out_s3 buffer into Apache Parquet format. - * - * `json` is a string that contain (concatenated) JSON objects. - * - * `size` is the length of the json data (excluding the trailing - * null-terminator character). - * - * Return 0 on success (with `out_buf` and `out_size` updated), - * and -1 on failure - */ -int out_s3_compress_parquet(void *json, size_t size, void **out_buf, size_t *out_size); -#endif diff --git a/src/aws/flb_aws_compress.c b/src/aws/flb_aws_compress.c index 8188c6b7038..6eac3669769 100644 --- a/src/aws/flb_aws_compress.c +++ b/src/aws/flb_aws_compress.c @@ -28,10 +28,6 @@ #include -#ifdef FLB_HAVE_ARROW -#include "compression/arrow/compress.h" -#endif - /* Wrapper function to adapt flb_snappy_compress to AWS compression interface */ static int flb_snappy_compress_wrapper(void *in_data, size_t in_len, void **out_data, size_t *out_len) @@ -77,20 +73,6 @@ static const struct compression_option compression_options[] = { "snappy", &flb_snappy_compress_wrapper }, -#ifdef FLB_HAVE_ARROW - { - FLB_AWS_COMPRESS_ARROW, - "arrow", - &out_s3_compress_arrow - }, -#endif -#ifdef FLB_HAVE_ARROW_PARQUET - { - FLB_AWS_COMPRESS_PARQUET, - "parquet", - &out_s3_compress_parquet - }, -#endif { 0 } }; diff --git a/src/flb_config.c b/src/flb_config.c index f6993114fcd..f2c6a6ee325 100644 --- a/src/flb_config.c +++ b/src/flb_config.c @@ -55,6 +55,38 @@ struct flb_service_config service_configs[] = { FLB_CONF_TYPE_DOUBLE, offsetof(struct flb_config, flush)}, + {FLB_CONF_STR_FLUSH_ADAPTIVE, + FLB_CONF_TYPE_BOOL, + offsetof(struct flb_config, flush_adaptive)}, + + {FLB_CONF_STR_FLUSH_ADAPTIVE_MIN, + FLB_CONF_TYPE_DOUBLE, + offsetof(struct flb_config, flush_adaptive_min_interval)}, + + {FLB_CONF_STR_FLUSH_ADAPTIVE_MAX, + FLB_CONF_TYPE_DOUBLE, + offsetof(struct flb_config, flush_adaptive_max_interval)}, + + {FLB_CONF_STR_FLUSH_ADAPTIVE_LOW, + FLB_CONF_TYPE_DOUBLE, + offsetof(struct flb_config, flush_adaptive_low_pressure)}, + + {FLB_CONF_STR_FLUSH_ADAPTIVE_MEDIUM, + FLB_CONF_TYPE_DOUBLE, + offsetof(struct flb_config, flush_adaptive_medium_pressure)}, + + {FLB_CONF_STR_FLUSH_ADAPTIVE_HIGH, + FLB_CONF_TYPE_DOUBLE, + offsetof(struct flb_config, flush_adaptive_high_pressure)}, + + {FLB_CONF_STR_FLUSH_ADAPTIVE_UP_STEPS, + FLB_CONF_TYPE_INT, + offsetof(struct flb_config, flush_adaptive_up_steps)}, + + {FLB_CONF_STR_FLUSH_ADAPTIVE_DOWN_STEPS, + FLB_CONF_TYPE_INT, + offsetof(struct flb_config, flush_adaptive_down_steps)}, + {FLB_CONF_STR_GRACE, FLB_CONF_TYPE_INT, offsetof(struct flb_config, grace)}, @@ -63,6 +95,10 @@ struct flb_service_config service_configs[] = { FLB_CONF_TYPE_BOOL, offsetof(struct flb_config, convert_nan_to_null)}, + {FLB_CONF_STR_FIPS_MODE, + FLB_CONF_TYPE_BOOL, + offsetof(struct flb_config, fips_mode)}, + {FLB_CONF_STR_DAEMON, FLB_CONF_TYPE_BOOL, offsetof(struct flb_config, daemon)}, @@ -246,6 +282,9 @@ struct flb_config *flb_config_init() return NULL; } + /* Prepare worker TLS before any config-time logging checks. */ + flb_worker_init(config); + MK_EVENT_ZERO(&config->ch_event); MK_EVENT_ZERO(&config->event_flush); MK_EVENT_ZERO(&config->event_shutdown); @@ -273,6 +312,18 @@ struct flb_config *flb_config_init() /* Flush */ config->flush = FLB_CONFIG_FLUSH_SECS; + config->flush_adaptive = FLB_FALSE; + config->flush_adaptive_min_interval = 0.5; + config->flush_adaptive_max_interval = 2.0; + config->flush_adaptive_low_pressure = 25.0; + config->flush_adaptive_medium_pressure = 50.0; + config->flush_adaptive_high_pressure = 75.0; + config->flush_adaptive_up_steps = 2; + config->flush_adaptive_down_steps = 3; + config->flush_adaptive_level = 1; + config->flush_adaptive_hits = 0; + config->flush_adaptive_direction = 0; + config->flush_adaptive_current_interval = FLB_CONFIG_FLUSH_SECS; config->daemon = FLB_FALSE; config->init_time = time(NULL); config->kernel = flb_kernel_info(); @@ -285,6 +336,10 @@ struct flb_config *flb_config_init() /* json */ config->convert_nan_to_null = FLB_FALSE; + /* FIPS */ + config->fips_mode = FLB_FALSE; + config->fips_mode_active = FLB_FALSE; + #ifdef FLB_HAVE_HTTP_SERVER config->http_ctx = NULL; config->http_server = FLB_FALSE; @@ -478,9 +533,6 @@ struct flb_config *flb_config_init() signal(SIGPIPE, SIG_IGN); #endif - /* Prepare worker interface */ - flb_worker_init(config); - #ifdef FLB_HAVE_REGEX /* Regex support */ flb_regex_init(); diff --git a/src/flb_connection.c b/src/flb_connection.c index 34b1cc229c7..752d31dd99c 100644 --- a/src/flb_connection.c +++ b/src/flb_connection.c @@ -26,6 +26,8 @@ int flb_connection_setup(struct flb_connection *connection, connection->ts_assigned = time(NULL); connection->busy_flag = FLB_FALSE; connection->shutdown_flag = FLB_FALSE; + connection->event_wakeup_pending = FLB_FALSE; + connection->event_release_pending = FLB_FALSE; connection->net = &connection->stream->net; @@ -97,11 +99,20 @@ static void compose_user_friendly_remote_host(struct flb_connection *connection) connection->remote_port); } else if (connection_type == FLB_TRANSPORT_UDP) { - snprintf(connection->user_friendly_remote_host, - sizeof(connection->user_friendly_remote_host), - "udp://%s:%u", - connection->remote_host, - connection->remote_port); + if (flb_stream_get_flag_status(connection->stream, FLB_IO_DTLS)) { + snprintf(connection->user_friendly_remote_host, + sizeof(connection->user_friendly_remote_host), + "dtls://%s:%u", + connection->remote_host, + connection->remote_port); + } + else { + snprintf(connection->user_friendly_remote_host, + sizeof(connection->user_friendly_remote_host), + "udp://%s:%u", + connection->remote_host, + connection->remote_port); + } } else if (connection_type == FLB_TRANSPORT_UNIX_STREAM) { snprintf(connection->user_friendly_remote_host, @@ -196,7 +207,17 @@ char *flb_connection_get_remote_address(struct flb_connection *connection) int flb_connection_get_flags(struct flb_connection *connection) { - return flb_stream_get_flags(connection->stream); + return flb_stream_get_flags(connection->stream) | connection->flags; +} + +void flb_connection_enable_flags(struct flb_connection *connection, int flags) +{ + connection->flags |= flags; +} + +void flb_connection_disable_flags(struct flb_connection *connection, int flags) +{ + connection->flags &= ~flags; } void flb_connection_reset_connection_timeout(struct flb_connection *connection) diff --git a/src/flb_coro.c b/src/flb_coro.c index e52a1336ae2..61addef6e55 100644 --- a/src/flb_coro.c +++ b/src/flb_coro.c @@ -23,12 +23,11 @@ FLB_TLS_DEFINE(struct flb_coro, flb_coro_key); -static pthread_mutex_t coro_mutex_init; +static pthread_mutex_t coro_mutex_init = PTHREAD_MUTEX_INITIALIZER; void flb_coro_init() { FLB_TLS_INIT(flb_coro_key); - pthread_mutex_init(&coro_mutex_init, NULL); } void flb_coro_thread_init() diff --git a/src/flb_downstream.c b/src/flb_downstream.c index ec2e1ad9129..55b2bcdd33f 100644 --- a/src/flb_downstream.c +++ b/src/flb_downstream.c @@ -30,8 +30,46 @@ #include #include #include +#include #include +static inline int prepare_destroy_conn_safe(struct flb_connection *connection); + +static void flb_downstream_conn_event_coro(void) +{ + struct flb_coro *coro; + struct flb_connection *connection; + + coro = flb_coro_get(); + connection = coro->data; + + while (FLB_TRUE) { + flb_coro_yield(coro, FLB_FALSE); + + connection->busy_flag = FLB_TRUE; + connection->event_callback(connection); + + connection->busy_flag = FLB_FALSE; + + /* + * A pause or shutdown can be requested recursively while ingestion + * is still using the plugin wrapper. Defer the drop notification + * until the complete callback has unwound. + */ + if (connection->event_release_pending == FLB_TRUE) { + connection->event_release_pending = FLB_FALSE; + prepare_destroy_conn_safe(connection); + } + + if (connection->fd == FLB_INVALID_SOCKET) { + connection->coroutine = NULL; + } + else { + connection->coroutine = coro; + } + } +} + /* Config map for Downstream networking setup */ struct flb_config_map downstream_net[] = { { @@ -270,24 +308,65 @@ static inline int prepare_destroy_conn_safe(struct flb_connection *connection) return result; } -static int destroy_conn(struct flb_connection *connection) +static void shutdown_connection(struct flb_connection *connection) +{ + if (connection->fd != FLB_INVALID_SOCKET && + connection->shutdown_flag == FLB_FALSE) { + shutdown(connection->fd, SHUT_RDWR); + connection->shutdown_flag = FLB_TRUE; + } +} + +static int wake_event_coroutine(struct flb_connection *connection, int error) +{ + int ret; + + connection->net_error = error; + connection->event_wakeup_pending = FLB_TRUE; + shutdown_connection(connection); + + /* Thread-owned streams are resumed directly after their lock is released. */ + if (flb_stream_is_thread_safe(connection->stream)) { + return 0; + } + + if (!MK_EVENT_IS_REGISTERED((&connection->event))) { + return 0; + } + + ret = mk_event_inject(connection->evl, + &connection->event, + connection->event.mask, + FLB_TRUE); + if (ret == -1) { + flb_warn("[downstream] could not inject wake-up event for connection #%i", + connection->fd); + } + + return ret; +} + +static int destroy_conn(struct flb_connection *connection, int force) { /* - * Delay the destruction if the connection is still marked busy or if - * there are pending events referencing it (e.g. an event still linked - * in the priority bucket queue after mk_event_inject). Freeing the - * connection while its event is queued would leave a dangling - * _priority_head in the bucket queue, causing a use-after-free and a - * stuck event that spins the input thread at 100% CPU. - * - * The connection stays in the destroy_queue and is retried on the next - * pending-destroy sweep, by which point the event has been drained. + * Normal sweeps defer destruction while a callback or injected event can + * still reference the connection. Terminal downstream teardown owns the + * event loop and coroutine stacks, so it must release them immediately. */ - if (connection->busy_flag || - !mk_list_entry_is_orphan(&connection->event._priority_head)) { + if (force == FLB_FALSE && + (connection->busy_flag || + !mk_list_entry_is_orphan(&connection->event._priority_head))) { return 0; } + if (connection->event_coroutine != NULL) { + flb_trace("[downstream] destroy event coroutine for connection #%i", + connection->fd); + flb_coro_destroy(connection->event_coroutine); + connection->event_coroutine = NULL; + connection->coroutine = NULL; + } + if (connection->tls_session != NULL) { flb_tls_session_destroy(connection->tls_session); } @@ -425,19 +504,13 @@ void flb_downstream_destroy(struct flb_downstream *stream) mk_list_foreach_safe(head, tmp, &stream->destroy_queue) { connection = mk_list_entry(head, struct flb_connection, _head); - /* - * This is the terminal cleanup: there is no later - * pending-destroy sweep. destroy_conn() defers connections - * whose event is still linked in the priority bucket queue, so - * unlink any lingering event here (the bucket queue is still - * alive at this point) to make sure the connection is freed - * instead of leaked. - */ + /* No event may retain a connection after terminal cleanup. */ if (!mk_list_entry_is_orphan(&connection->event._priority_head)) { mk_list_del(&connection->event._priority_head); } - destroy_conn(connection); + destroy_conn(connection, + connection->event_coroutine != NULL); } /* If the simulated UDP connection reference is set then @@ -480,7 +553,152 @@ void flb_downstream_destroy(struct flb_downstream *stream) int flb_downstream_conn_release(struct flb_connection *connection) { - return prepare_destroy_conn_safe(connection); + int ret; + int resume; + + resume = FLB_FALSE; + flb_stream_acquire_lock(connection->stream, FLB_TRUE); + + if (connection->event_coroutine != NULL && + connection->busy_flag == FLB_TRUE) { + connection->event_release_pending = FLB_TRUE; + + if (flb_coro_get() != connection->event_coroutine) { + wake_event_coroutine(connection, ECANCELED); + resume = flb_stream_is_thread_safe(connection->stream); + } + + ret = FLB_DOWNSTREAM_CONN_DEFERRED; + } + else { + prepare_destroy_conn(connection); + ret = FLB_DOWNSTREAM_CONN_RELEASED; + } + + flb_stream_release_lock(connection->stream); + + if (resume == FLB_TRUE) { + flb_downstream_conn_event_resume(connection); + } + + return ret; +} + +int flb_downstream_conn_event_register(struct flb_connection *connection, + flb_connection_event_callback callback, + int mask) +{ + int ret; + size_t stack_size; + struct flb_coro *coro; + struct flb_coro *previous_coro; + struct flb_config *config; + + if (connection == NULL || callback == NULL || + (mask & (MK_EVENT_READ | MK_EVENT_WRITE)) == 0 || + connection->type != FLB_DOWNSTREAM_CONNECTION) { + return -1; + } + + if (connection->stream->transport != FLB_TRANSPORT_TCP && + connection->stream->transport != FLB_TRANSPORT_UNIX_STREAM) { + return -1; + } + + config = connection->stream->config; + if (config == NULL || connection->event_coroutine != NULL) { + return -1; + } + + coro = flb_coro_create(connection); + if (coro == NULL) { + return -1; + } + + coro->caller = co_active(); + coro->callee = co_create(config->coro_stack_size, + flb_downstream_conn_event_coro, + &stack_size); + if (coro->callee == NULL) { + flb_coro_destroy(coro); + return -1; + } + +#ifdef FLB_HAVE_VALGRIND + coro->valgrind_stack_id = VALGRIND_STACK_REGISTER( + coro->callee, + ((char *) coro->callee) + stack_size); +#endif + + connection->event_coroutine = coro; + connection->event_callback = callback; + connection->coroutine = coro; + flb_connection_enable_flags(connection, FLB_IO_ASYNC); + + flb_trace("[downstream] register event coroutine for connection #%i", + connection->fd); + + previous_coro = flb_coro_get(); + flb_coro_resume(coro); + flb_coro_set(previous_coro); + + ret = mk_event_add(connection->evl, + connection->fd, + FLB_ENGINE_EV_THREAD, + mask, + &connection->event); + if (ret == -1) { + flb_connection_disable_flags(connection, FLB_IO_ASYNC); + connection->event_callback = NULL; + connection->event_coroutine = NULL; + connection->coroutine = NULL; + flb_coro_destroy(coro); + return -1; + } + + return 0; +} + +void flb_downstream_conn_event_resume(struct flb_connection *connection) +{ + struct flb_coro *previous_coro; + + previous_coro = flb_coro_get(); + connection->event_wakeup_pending = FLB_FALSE; + flb_coro_resume(connection->event_coroutine); + flb_coro_set(previous_coro); +} + +static void resume_pending_event_coroutines(struct flb_downstream *stream) +{ + struct flb_connection *connection; + struct mk_list *head; + + while (FLB_TRUE) { + connection = NULL; + + flb_stream_acquire_lock(&stream->base, FLB_TRUE); + + mk_list_foreach(head, &stream->busy_queue) { + connection = mk_list_entry(head, struct flb_connection, _head); + + if (connection->event_coroutine != NULL && + connection->event_wakeup_pending == FLB_TRUE) { + connection->event_wakeup_pending = FLB_FALSE; + break; + } + + connection = NULL; + } + + flb_stream_release_lock(&stream->base); + + if (connection == NULL) { + break; + } + + flb_downstream_conn_event_resume(connection); + } } int flb_downstream_conn_timeouts_stream(struct flb_downstream *stream) @@ -490,7 +708,7 @@ int flb_downstream_conn_timeouts_stream(struct flb_downstream *stream) const char *reason; struct mk_list *s_head; int drop; - int inject; + int inject; struct mk_list *tmp; time_t now; @@ -545,23 +763,39 @@ int flb_downstream_conn_timeouts_stream(struct flb_downstream *stream) } } - inject = FLB_FALSE; - if (connection->event.status != MK_EVENT_NONE) { - inject = FLB_TRUE; + if (connection->event_coroutine != NULL && + MK_EVENT_IS_REGISTERED((&connection->event))) { + wake_event_coroutine(connection, ETIMEDOUT); } - connection->net_error = ETIMEDOUT; - prepare_destroy_conn(connection); - if (inject == FLB_TRUE) { - mk_event_inject(connection->evl, - &connection->event, - connection->event.mask, - FLB_TRUE); + else { + connection->net_error = ETIMEDOUT; + inject = FLB_FALSE; + if (connection->event.status != MK_EVENT_NONE) { + inject = FLB_TRUE; + } + + prepare_destroy_conn(connection); + if (inject == FLB_TRUE) { + mk_event_inject(connection->evl, + &connection->event, + connection->event.mask, + FLB_TRUE); + } } } } flb_stream_release_lock(&stream->base); + /* + * Worker and threaded-input streams own their event loop. Resume timeout + * wakeups here, after releasing the stream lock, so a maintenance callback + * cannot leave an injected event to be overwritten by the next wait. + */ + if (flb_stream_is_thread_safe(&stream->base)) { + resume_pending_event_coroutines(stream); + } + return 0; } @@ -591,7 +825,7 @@ int flb_downstream_conn_pending_destroy(struct flb_downstream *stream) mk_list_foreach_safe(head, tmp, &stream->destroy_queue) { connection = mk_list_entry(head, struct flb_connection, _head); - destroy_conn(connection); + destroy_conn(connection, FLB_FALSE); } flb_stream_release_lock(&stream->base); diff --git a/src/flb_downstream_worker.c b/src/flb_downstream_worker.c new file mode 100644 index 00000000000..a2cf47faf57 --- /dev/null +++ b/src/flb_downstream_worker.c @@ -0,0 +1,657 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2026 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include + +struct flb_downstream_worker { + struct flb_downstream_worker_runtime *runtime; + struct mk_event_loop *event_loop; + struct mk_event control_event; + flb_pipefd_t control_channel[2]; + flb_sockfd_t listener_fd; + void *context; + int worker_id; + int worker_count; + pthread_t thread; + pthread_mutex_t mutex; + pthread_cond_t condition; + uint64_t should_exit; + flb_downstream_worker_foreach_cb control_callback; + void *control_data; + int initialized; + int thread_created; + int control_channel_created; + int control_done; + int control_result; + int startup_result; +}; + +struct flb_downstream_worker_runtime { + struct flb_downstream_worker *workers; + int worker_count; + int active_workers; + void *parent; + flb_downstream_worker_init_cb cb_init; + flb_downstream_worker_exit_cb cb_exit; + flb_downstream_worker_maintenance_cb cb_maintenance; + pthread_mutex_t lifecycle_mutex; + pthread_cond_t lifecycle_condition; + pthread_mutex_t foreach_mutex; + int active_operations; + int stopping; + int listener_address_set; + struct sockaddr_storage listener_address; +}; + +static int downstream_worker_context_reset(struct flb_downstream_worker *worker) +{ + int ret; + + memset(worker, 0, sizeof(struct flb_downstream_worker)); + worker->listener_fd = FLB_INVALID_SOCKET; + + ret = pthread_mutex_init(&worker->mutex, NULL); + if (ret != 0) { + return -1; + } + + ret = pthread_cond_init(&worker->condition, NULL); + if (ret != 0) { + pthread_mutex_destroy(&worker->mutex); + return -1; + } + + return 0; +} + +static int downstream_worker_listener_validate( + struct flb_downstream_worker_runtime *runtime, + struct flb_downstream_worker *worker) +{ + int result; + socklen_t address_length; + int addresses_match; + struct sockaddr_in *address_ipv4; + struct sockaddr_in *runtime_address_ipv4; + struct sockaddr_in6 *address_ipv6; + struct sockaddr_in6 *runtime_address_ipv6; + struct sockaddr_storage address; + + if (worker->listener_fd == FLB_INVALID_SOCKET) { + return 0; + } + + memset(&address, 0, sizeof(struct sockaddr_storage)); + address_length = sizeof(struct sockaddr_storage); + result = getsockname(worker->listener_fd, + (struct sockaddr *) &address, + &address_length); + if (result != 0) { + return -1; + } + + if (runtime->listener_address_set == FLB_FALSE) { + memcpy(&runtime->listener_address, &address, + sizeof(struct sockaddr_storage)); + runtime->listener_address_set = FLB_TRUE; + return 0; + } + + addresses_match = FLB_FALSE; + if (runtime->listener_address.ss_family == AF_INET && + address.ss_family == AF_INET) { + runtime_address_ipv4 = (struct sockaddr_in *) &runtime->listener_address; + address_ipv4 = (struct sockaddr_in *) &address; + if (runtime_address_ipv4->sin_port == address_ipv4->sin_port && + memcmp(&runtime_address_ipv4->sin_addr, + &address_ipv4->sin_addr, + sizeof(struct in_addr)) == 0) { + addresses_match = FLB_TRUE; + } + } + else if (runtime->listener_address.ss_family == AF_INET6 && + address.ss_family == AF_INET6) { + runtime_address_ipv6 = (struct sockaddr_in6 *) &runtime->listener_address; + address_ipv6 = (struct sockaddr_in6 *) &address; + if (runtime_address_ipv6->sin6_port == address_ipv6->sin6_port && + runtime_address_ipv6->sin6_scope_id == address_ipv6->sin6_scope_id && + memcmp(&runtime_address_ipv6->sin6_addr, + &address_ipv6->sin6_addr, + sizeof(struct in6_addr)) == 0) { + addresses_match = FLB_TRUE; + } + } + + if (addresses_match == FLB_FALSE) { + flb_error("[downstream worker] listeners did not bind the same endpoint"); + return -1; + } + + return 0; +} + +static void downstream_worker_context_cleanup(struct flb_downstream_worker *worker) +{ + pthread_mutex_destroy(&worker->mutex); + pthread_cond_destroy(&worker->condition); +} + +static int downstream_worker_control_event(struct flb_downstream_worker *worker) +{ + ssize_t bytes; + char signal; + flb_downstream_worker_foreach_cb callback; + void *data; + + bytes = flb_pipe_r(worker->control_channel[0], &signal, sizeof(signal)); + if (bytes != sizeof(signal)) { + pthread_mutex_lock(&worker->mutex); + worker->control_callback = NULL; + worker->control_data = NULL; + worker->control_result = -1; + worker->control_done = FLB_TRUE; + pthread_cond_broadcast(&worker->condition); + pthread_mutex_unlock(&worker->mutex); + return -1; + } + + pthread_mutex_lock(&worker->mutex); + callback = worker->control_callback; + data = worker->control_data; + pthread_mutex_unlock(&worker->mutex); + + if (callback != NULL && worker->context != NULL) { + callback(worker, worker->context, data); + } + + pthread_mutex_lock(&worker->mutex); + worker->control_callback = NULL; + worker->control_data = NULL; + worker->control_result = 0; + worker->control_done = FLB_TRUE; + pthread_cond_signal(&worker->condition); + pthread_mutex_unlock(&worker->mutex); + + return 0; +} + +static void downstream_worker_dispatch_event(struct mk_event *event) +{ + struct flb_connection *connection; + + if (event->type == FLB_ENGINE_EV_CUSTOM) { + event->handler(event); + } + else if (event->type == FLB_ENGINE_EV_THREAD) { + connection = (struct flb_connection *) event; + if (connection->coroutine != NULL) { + if (connection->event_coroutine != NULL) { + flb_downstream_conn_event_resume(connection); + } + else { + flb_coro_resume(connection->coroutine); + } + } + } +} + +static void *downstream_worker_thread(void *data) +{ + int ret; + struct mk_event *event; + struct flb_net_dns dns_ctx = {0}; + struct flb_downstream_worker *worker; + struct flb_downstream_worker_runtime *runtime; + + worker = data; + runtime = worker->runtime; + flb_net_ctx_init(&dns_ctx); + + worker->event_loop = mk_event_loop_create(256); + if (worker->event_loop == NULL) { + ret = -1; + goto signal_and_exit; + } + + MK_EVENT_NEW(&worker->control_event); + ret = mk_event_channel_create(worker->event_loop, + &worker->control_channel[0], + &worker->control_channel[1], + &worker->control_event); + if (ret != 0) { + ret = -1; + goto signal_and_exit; + } + worker->control_channel_created = FLB_TRUE; + + flb_engine_evl_init(); + flb_engine_evl_set(worker->event_loop); + flb_coro_thread_init(); + flb_net_dns_ctx_set(&dns_ctx); + + ret = runtime->cb_init(worker, runtime->parent, &worker->context); + +signal_and_exit: + pthread_mutex_lock(&worker->mutex); + worker->startup_result = ret; + worker->initialized = FLB_TRUE; + pthread_cond_signal(&worker->condition); + pthread_mutex_unlock(&worker->mutex); + + if (ret != 0) { + goto cleanup; + } + + while (cfl_atomic_load(&worker->should_exit) == FLB_FALSE) { + ret = mk_event_wait_2(worker->event_loop, 250); + if (ret < 0) { + flb_error("[downstream worker %i] event loop wait failed", + worker->worker_id); + pthread_mutex_lock(&worker->mutex); + worker->startup_result = -1; + pthread_mutex_unlock(&worker->mutex); + break; + } + + if (cfl_atomic_load(&worker->should_exit) == FLB_TRUE) { + break; + } + + { + mk_event_foreach(event, worker->event_loop) { + downstream_worker_dispatch_event(event); + } + } + + { + mk_event_foreach(event, worker->event_loop) { + if (event == &worker->control_event) { + downstream_worker_control_event(worker); + } + } + } + + if (runtime->cb_maintenance != NULL) { + runtime->cb_maintenance(worker, worker->context); + } + + flb_net_dns_lookup_context_cleanup(&dns_ctx); + } + +cleanup: + flb_net_dns_lookup_context_cleanup(&dns_ctx); + + if (runtime->cb_exit != NULL && worker->context != NULL) { + runtime->cb_exit(worker, worker->context); + } + + pthread_mutex_lock(&worker->mutex); + if (worker->context != NULL) { + worker->context = NULL; + } + + if (worker->control_done == FLB_FALSE && + worker->control_callback != NULL) { + worker->control_callback = NULL; + worker->control_data = NULL; + worker->control_result = -1; + worker->control_done = FLB_TRUE; + pthread_cond_broadcast(&worker->condition); + } + + if (worker->control_channel_created == FLB_TRUE) { + mk_event_channel_destroy(worker->event_loop, + worker->control_channel[0], + worker->control_channel[1], + &worker->control_event); + worker->control_channel_created = FLB_FALSE; + } + pthread_mutex_unlock(&worker->mutex); + + if (worker->event_loop != NULL) { + flb_engine_evl_set(NULL); + flb_net_dns_ctx_set(NULL); + mk_event_loop_destroy(worker->event_loop); + worker->event_loop = NULL; + } + + return NULL; +} + +int flb_downstream_worker_runtime_start(struct flb_downstream_worker_runtime **out_runtime, + const struct flb_downstream_worker_options *options) +{ + int i; + int ret; + struct flb_downstream_worker_runtime *runtime; + + if (out_runtime == NULL) { + return -1; + } + + *out_runtime = NULL; + + if (options == NULL || options->workers <= 0 || options->cb_init == NULL) { + return -1; + } + + runtime = flb_calloc(1, sizeof(struct flb_downstream_worker_runtime)); + if (runtime == NULL) { + flb_errno(); + return -1; + } + + ret = pthread_mutex_init(&runtime->lifecycle_mutex, NULL); + if (ret != 0) { + flb_free(runtime); + return -1; + } + + ret = pthread_cond_init(&runtime->lifecycle_condition, NULL); + if (ret != 0) { + pthread_mutex_destroy(&runtime->lifecycle_mutex); + flb_free(runtime); + return -1; + } + + ret = pthread_mutex_init(&runtime->foreach_mutex, NULL); + if (ret != 0) { + pthread_cond_destroy(&runtime->lifecycle_condition); + pthread_mutex_destroy(&runtime->lifecycle_mutex); + flb_free(runtime); + return -1; + } + runtime->workers = flb_calloc(options->workers, + sizeof(struct flb_downstream_worker)); + if (runtime->workers == NULL) { + flb_errno(); + pthread_mutex_destroy(&runtime->foreach_mutex); + pthread_cond_destroy(&runtime->lifecycle_condition); + pthread_mutex_destroy(&runtime->lifecycle_mutex); + flb_free(runtime); + return -1; + } + + runtime->worker_count = options->workers; + runtime->parent = options->parent; + runtime->cb_init = options->cb_init; + runtime->cb_exit = options->cb_exit; + runtime->cb_maintenance = options->cb_maintenance; + + *out_runtime = runtime; + + for (i = 0; i < runtime->worker_count; i++) { + ret = downstream_worker_context_reset(&runtime->workers[i]); + if (ret != 0) { + break; + } + + runtime->active_workers++; + runtime->workers[i].runtime = runtime; + runtime->workers[i].worker_id = i; + runtime->workers[i].worker_count = runtime->worker_count; + + ret = pthread_create(&runtime->workers[i].thread, + NULL, + downstream_worker_thread, + &runtime->workers[i]); + if (ret != 0) { + runtime->workers[i].startup_result = -1; + break; + } + + runtime->workers[i].thread_created = FLB_TRUE; + pthread_mutex_lock(&runtime->workers[i].mutex); + while (runtime->workers[i].initialized == FLB_FALSE) { + ret = pthread_cond_wait(&runtime->workers[i].condition, + &runtime->workers[i].mutex); + if (ret != 0) { + break; + } + } + if (ret == 0) { + ret = runtime->workers[i].startup_result; + } + pthread_mutex_unlock(&runtime->workers[i].mutex); + + if (ret == 0) { + ret = downstream_worker_listener_validate(runtime, + &runtime->workers[i]); + } + + if (ret != 0) { + break; + } + } + + if (i != runtime->worker_count) { + flb_downstream_worker_runtime_stop(runtime); + *out_runtime = NULL; + return -1; + } + + return 0; +} + +static int downstream_worker_runtime_is_worker_thread( + struct flb_downstream_worker_runtime *runtime) +{ + int i; + + for (i = 0; i < runtime->active_workers; i++) { + if (runtime->workers[i].thread_created == FLB_TRUE && + pthread_equal(runtime->workers[i].thread, pthread_self())) { + return FLB_TRUE; + } + } + + return FLB_FALSE; +} + +int flb_downstream_worker_runtime_stop(struct flb_downstream_worker_runtime *runtime) +{ + int i; + char signal; + + if (runtime == NULL) { + return 0; + } + + if (downstream_worker_runtime_is_worker_thread(runtime) == FLB_TRUE) { + return -1; + } + + signal = 1; + + pthread_mutex_lock(&runtime->lifecycle_mutex); + runtime->stopping = FLB_TRUE; + while (runtime->active_operations > 0) { + pthread_cond_wait(&runtime->lifecycle_condition, + &runtime->lifecycle_mutex); + } + pthread_mutex_unlock(&runtime->lifecycle_mutex); + + /* Wake every worker before joining so shutdown latency is not cumulative. */ + for (i = 0; i < runtime->active_workers; i++) { + cfl_atomic_store(&runtime->workers[i].should_exit, FLB_TRUE); + + pthread_mutex_lock(&runtime->workers[i].mutex); + if (runtime->workers[i].startup_result == 0 && + runtime->workers[i].control_channel_created == FLB_TRUE) { + flb_pipe_w(runtime->workers[i].control_channel[1], + &signal, sizeof(signal)); + } + pthread_mutex_unlock(&runtime->workers[i].mutex); + } + + for (i = 0; i < runtime->active_workers; i++) { + if (runtime->workers[i].thread_created == FLB_TRUE) { + pthread_join(runtime->workers[i].thread, NULL); + } + downstream_worker_context_cleanup(&runtime->workers[i]); + } + + pthread_mutex_destroy(&runtime->foreach_mutex); + pthread_cond_destroy(&runtime->lifecycle_condition); + pthread_mutex_destroy(&runtime->lifecycle_mutex); + flb_free(runtime->workers); + flb_free(runtime); + + return 0; +} + +int flb_downstream_worker_runtime_foreach(struct flb_downstream_worker_runtime *runtime, + flb_downstream_worker_foreach_cb callback, + void *data) +{ + int i; + int result; + int wait_result; + ssize_t bytes; + char signal; + struct flb_downstream_worker *worker; + + if (runtime == NULL || callback == NULL) { + return -1; + } + + if (downstream_worker_runtime_is_worker_thread(runtime) == FLB_TRUE) { + return -1; + } + + pthread_mutex_lock(&runtime->lifecycle_mutex); + if (runtime->stopping == FLB_TRUE) { + pthread_mutex_unlock(&runtime->lifecycle_mutex); + return -1; + } + runtime->active_operations++; + pthread_mutex_unlock(&runtime->lifecycle_mutex); + + pthread_mutex_lock(&runtime->foreach_mutex); + + result = 0; + signal = 1; + + for (i = 0; i < runtime->worker_count; i++) { + worker = &runtime->workers[i]; + + pthread_mutex_lock(&worker->mutex); + if (worker->context == NULL || worker->thread_created != FLB_TRUE || + worker->control_channel_created != FLB_TRUE) { + pthread_mutex_unlock(&worker->mutex); + result = -1; + continue; + } + + worker->control_callback = callback; + worker->control_data = data; + worker->control_done = FLB_FALSE; + worker->control_result = 0; + + bytes = flb_pipe_w(worker->control_channel[1], &signal, sizeof(signal)); + if (bytes != sizeof(signal)) { + worker->control_callback = NULL; + worker->control_data = NULL; + worker->control_done = FLB_TRUE; + pthread_mutex_unlock(&worker->mutex); + result = -1; + continue; + } + + while (worker->control_done == FLB_FALSE) { + wait_result = pthread_cond_wait(&worker->condition, &worker->mutex); + if (wait_result != 0) { + worker->control_callback = NULL; + worker->control_data = NULL; + worker->control_result = -1; + worker->control_done = FLB_TRUE; + break; + } + } + if (worker->control_result != 0) { + result = -1; + } + pthread_mutex_unlock(&worker->mutex); + } + pthread_mutex_unlock(&runtime->foreach_mutex); + + pthread_mutex_lock(&runtime->lifecycle_mutex); + runtime->active_operations--; + if (runtime->active_operations == 0) { + pthread_cond_signal(&runtime->lifecycle_condition); + } + pthread_mutex_unlock(&runtime->lifecycle_mutex); + + return result; +} + +struct mk_event_loop *flb_downstream_worker_event_loop_get( + struct flb_downstream_worker *worker) +{ + if (worker == NULL) { + return NULL; + } + + return worker->event_loop; +} + +int flb_downstream_worker_id_get(struct flb_downstream_worker *worker) +{ + if (worker == NULL) { + return -1; + } + + return worker->worker_id; +} + +int flb_downstream_worker_count_get(struct flb_downstream_worker *worker) +{ + if (worker == NULL) { + return -1; + } + + return worker->worker_count; +} + +int flb_downstream_worker_listener_fd_set(struct flb_downstream_worker *worker, + flb_sockfd_t listener_fd) +{ + if (worker == NULL || listener_fd == FLB_INVALID_SOCKET) { + return -1; + } + + worker->listener_fd = listener_fd; + + return 0; +} diff --git a/src/flb_engine.c b/src/flb_engine.c index 63178da843c..d74cdd9deeb 100644 --- a/src/flb_engine.c +++ b/src/flb_engine.c @@ -18,6 +18,7 @@ */ #include +#include #include #include #include @@ -40,6 +41,7 @@ #include #include #include +#include #include #include #include @@ -252,6 +254,247 @@ static inline double calculate_chunk_capacity_percent(struct flb_output_instance ((double)ins->total_limit_size)); } +static inline double adaptive_flush_clamp(double value, double min, double max) +{ + if (value < min) { + return min; + } + + if (value > max) { + return max; + } + + return value; +} + +static double flb_engine_get_chunk_backpressure_percent(struct flb_config *config) +{ + double pressure; + double max_pressure; + struct mk_list *head; + struct flb_output_instance *ins; + + max_pressure = 0.0; + + mk_list_foreach(head, &config->outputs) { + ins = mk_list_entry(head, struct flb_output_instance, _head); + + if (ins->total_limit_size <= 0) { + continue; + } + + pressure = ((double) (ins->fs_backlog_chunks_size + ins->fs_chunks_size) * 100.0) + / ((double) ins->total_limit_size); + + pressure = adaptive_flush_clamp(pressure, 0.0, 100.0); + + if (pressure > max_pressure) { + max_pressure = pressure; + } + } + + return max_pressure; +} + +static int flb_engine_flush_timer_reset(struct flb_config *config, double interval) +{ + struct mk_event *event; + struct flb_time t_flush; + double fallback_interval; + + event = &config->event_flush; + fallback_interval = config->flush_adaptive_current_interval; + + if (event->status != MK_EVENT_NONE) { + mk_event_timeout_destroy(config->evl, event); + } + + flb_time_from_double(&t_flush, interval); + + config->flush_fd = mk_event_timeout_create(config->evl, + t_flush.tm.tv_sec, + t_flush.tm.tv_nsec, + event); + event->priority = FLB_ENGINE_PRIORITY_FLUSH; + + if (config->flush_fd == -1) { + flb_utils_error(FLB_ERR_CFG_FLUSH_CREATE); + + if (fallback_interval > 0.0 && + fabs(fallback_interval - interval) > DBL_EPSILON) { + flb_time_from_double(&t_flush, fallback_interval); + config->flush_fd = mk_event_timeout_create(config->evl, + t_flush.tm.tv_sec, + t_flush.tm.tv_nsec, + event); + event->priority = FLB_ENGINE_PRIORITY_FLUSH; + } + + if (config->flush_fd == -1) { + return -1; + } + } + + return 0; +} + +int flb_engine_adaptive_flush_target_level(struct flb_config *config, + double pressure) +{ + if (pressure >= config->flush_adaptive_high_pressure) { + return 3; + } + else if (pressure >= config->flush_adaptive_medium_pressure) { + return 2; + } + else if (pressure <= config->flush_adaptive_low_pressure) { + return 0; + } + + return 1; +} + +double flb_engine_adaptive_flush_interval(struct flb_config *config, + int level) +{ + double interval; + static const double multipliers[] = {2.0, 1.0, 0.75, 0.5}; + + if (level < 0) { + level = 0; + } + else if (level > 3) { + level = 3; + } + + interval = config->flush * multipliers[level]; + + return adaptive_flush_clamp(interval, + config->flush_adaptive_min_interval, + config->flush_adaptive_max_interval); +} + +static void flb_engine_adaptive_flush_update(struct flb_config *config) +{ + int target_level; + double pressure; + double interval; + + if (config->flush_adaptive == FLB_FALSE) { + return; + } + + pressure = flb_engine_get_chunk_backpressure_percent(config); + + target_level = flb_engine_adaptive_flush_target_level(config, pressure); + + if (target_level > config->flush_adaptive_level) { + if (config->flush_adaptive_direction != 1) { + config->flush_adaptive_direction = 1; + config->flush_adaptive_hits = 0; + } + + config->flush_adaptive_hits++; + + if (config->flush_adaptive_hits >= config->flush_adaptive_up_steps) { + config->flush_adaptive_level++; + config->flush_adaptive_hits = 0; + } + } + else if (target_level < config->flush_adaptive_level) { + if (config->flush_adaptive_direction != -1) { + config->flush_adaptive_direction = -1; + config->flush_adaptive_hits = 0; + } + + config->flush_adaptive_hits++; + + if (config->flush_adaptive_hits >= config->flush_adaptive_down_steps) { + config->flush_adaptive_level--; + config->flush_adaptive_hits = 0; + } + } + else { + config->flush_adaptive_direction = 0; + config->flush_adaptive_hits = 0; + } + + if (config->flush_adaptive_level < 0) { + config->flush_adaptive_level = 0; + } + else if (config->flush_adaptive_level > 3) { + config->flush_adaptive_level = 3; + } + + interval = flb_engine_adaptive_flush_interval(config, + config->flush_adaptive_level); + + if (fabs(interval - config->flush_adaptive_current_interval) + <= (DBL_EPSILON * fmax(fabs(interval), + fabs(config->flush_adaptive_current_interval)))) { + return; + } + + if (flb_engine_flush_timer_reset(config, interval) == 0) { + config->flush_adaptive_current_interval = interval; + flb_debug("[engine] adaptive flush interval %.3f sec (pressure=%.2f%%, level=%i)", + interval, + pressure, + config->flush_adaptive_level); + } +} + +static void flb_engine_adaptive_flush_init(struct flb_config *config) +{ + if (config->flush_adaptive == FLB_FALSE) { + config->flush_adaptive_current_interval = config->flush; + return; + } + + if (config->flush_adaptive_min_interval <= 0.0) { + config->flush_adaptive_min_interval = 0.1; + } + + if (config->flush_adaptive_max_interval < + config->flush_adaptive_min_interval) { + config->flush_adaptive_max_interval = + config->flush_adaptive_min_interval; + } + + if (config->flush_adaptive_up_steps < 1) { + config->flush_adaptive_up_steps = 1; + } + + if (config->flush_adaptive_down_steps < 1) { + config->flush_adaptive_down_steps = 1; + } + + if (config->flush_adaptive_low_pressure < 0.0) { + config->flush_adaptive_low_pressure = 0.0; + } + + if (config->flush_adaptive_high_pressure > 100.0) { + config->flush_adaptive_high_pressure = 100.0; + } + + if (config->flush_adaptive_low_pressure > + config->flush_adaptive_medium_pressure) { + config->flush_adaptive_medium_pressure = + config->flush_adaptive_low_pressure; + } + + if (config->flush_adaptive_medium_pressure > + config->flush_adaptive_high_pressure) { + config->flush_adaptive_medium_pressure = + config->flush_adaptive_high_pressure; + } + + config->flush_adaptive_current_interval = + flb_engine_adaptive_flush_interval(config, + config->flush_adaptive_level); + +} + static void handle_dlq_if_available(struct flb_config *config, struct flb_task *task, struct flb_output_instance *ins, @@ -725,6 +968,7 @@ static FLB_INLINE int flb_engine_handle_event(flb_pipefd_t fd, int mask, if (config->flush_fd == fd) { flb_utils_timer_consume(fd); flb_engine_flush(config, NULL); + flb_engine_adaptive_flush_update(config); return 0; } else if (config->shutdown_fd == fd) { @@ -856,7 +1100,6 @@ int flb_engine_start(struct flb_config *config) uint64_t ts; char tmp[16]; int rb_flush_flag; - struct flb_time t_flush; struct mk_event *event; struct mk_event_loop *evl; struct flb_bucket_queue *evl_bktq; @@ -919,6 +1162,11 @@ int flb_engine_start(struct flb_config *config) return -1; } + ret = flb_fips_init(config); + if (ret != 0) { + return -1; + } + if (engine_has_fluentbit_logs_input(config)) { flb_log_pipeline_enable(config); } @@ -1034,14 +1282,13 @@ int flb_engine_start(struct flb_config *config) event->mask = MK_EVENT_EMPTY; event->status = MK_EVENT_NONE; - flb_time_from_double(&t_flush, config->flush); - config->flush_fd = mk_event_timeout_create(evl, - t_flush.tm.tv_sec, - t_flush.tm.tv_nsec, - event); - event->priority = FLB_ENGINE_PRIORITY_FLUSH; - if (config->flush_fd == -1) { - flb_utils_error(FLB_ERR_CFG_FLUSH_CREATE); + flb_engine_adaptive_flush_init(config); + + if (flb_engine_flush_timer_reset(config, + config->flush_adaptive_current_interval) == -1) { + flb_error("[engine] could not initialize flush timer (interval=%.3f sec)", + config->flush_adaptive_current_interval); + return -1; } @@ -1292,7 +1539,12 @@ int flb_engine_start(struct flb_config *config) if (connection->coroutine) { flb_trace("[engine] resuming coroutine=%p", connection->coroutine); - flb_coro_resume(connection->coroutine); + if (connection->event_coroutine != NULL) { + flb_downstream_conn_event_resume(connection); + } + else { + flb_coro_resume(connection->coroutine); + } } } else if (event->type == FLB_ENGINE_EV_OUTPUT) { diff --git a/src/flb_filter.c b/src/flb_filter.c index 79299885065..5e9a5ba17bf 100644 --- a/src/flb_filter.c +++ b/src/flb_filter.c @@ -27,6 +27,8 @@ #include #include #include +#include +#include #include #ifdef FLB_HAVE_CHUNK_TRACE @@ -435,6 +437,8 @@ struct flb_filter_instance *flb_filter_new(struct flb_config *config, const char *filter, void *data) { int id; + const char *alias_target; + const char *effective_filter_name; struct mk_list *head; struct flb_filter_plugin *plugin; struct flb_filter_instance *instance = NULL; @@ -443,14 +447,32 @@ struct flb_filter_instance *flb_filter_new(struct flb_config *config, return NULL; } + effective_filter_name = filter; + mk_list_foreach(head, &config->filter_plugins) { plugin = mk_list_entry(head, struct flb_filter_plugin, _head); - if (strcasecmp(plugin->name, filter) == 0) { + if (strcasecmp(plugin->name, effective_filter_name) == 0) { break; } plugin = NULL; } + if (plugin == NULL) { + alias_target = flb_plugin_alias_get(FLB_PLUGIN_FILTER, filter, + strlen(filter)); + if (alias_target != NULL) { + effective_filter_name = alias_target; + + mk_list_foreach(head, &config->filter_plugins) { + plugin = mk_list_entry(head, struct flb_filter_plugin, _head); + if (strcasecmp(plugin->name, effective_filter_name) == 0) { + break; + } + plugin = NULL; + } + } + } + if (!plugin) { return NULL; } @@ -493,7 +515,6 @@ struct flb_filter_instance *flb_filter_new(struct flb_config *config, mk_list_init(&instance->properties); mk_list_add(&instance->_head, &config->filters); - return instance; } diff --git a/src/flb_fips.c b/src/flb_fips.c new file mode 100644 index 00000000000..5341a9257b6 --- /dev/null +++ b/src/flb_fips.c @@ -0,0 +1,227 @@ +/* Fluent Bit + * ========== + * Copyright (C) 2015-2026 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include + +#ifdef FLB_HAVE_OPENSSL +#include +#include +#include +#if OPENSSL_VERSION_NUMBER >= 0x30000000L +#include +#endif +#endif + +#if defined(FLB_HAVE_OPENSSL) && OPENSSL_VERSION_NUMBER >= 0x30000000L +#define FLB_FIPS_PROCESS_MODE_UNSET -1 + +static pthread_mutex_t fips_init_mutex = PTHREAD_MUTEX_INITIALIZER; +static OSSL_PROVIDER *fips_provider; +static OSSL_PROVIDER *base_provider; +static int fips_process_mode = FLB_FIPS_PROCESS_MODE_UNSET; + +static void log_openssl_errors(void) +{ + char errbuf[256]; + unsigned long err; + + while ((err = ERR_get_error()) != 0) { + ERR_error_string_n(err, errbuf, sizeof(errbuf)); + flb_error("[fips] OpenSSL error: %s", errbuf); + } +} + +static int fips_provider_is_active(void) +{ + EVP_MD *sha256; + + if (EVP_default_properties_is_fips_enabled(NULL) != 1) { + return FLB_FALSE; + } + + sha256 = EVP_MD_fetch(NULL, "SHA256", "fips=yes,provider=fips"); + if (sha256 == NULL) { + return FLB_FALSE; + } + + EVP_MD_free(sha256); + + return FLB_TRUE; +} +#endif + +int flb_fips_init(struct flb_config *config) +{ +#if defined(FLB_HAVE_OPENSSL) && OPENSSL_VERSION_NUMBER >= 0x30000000L + int ret; + int properties_changed; + OSSL_PROVIDER *new_fips_provider; + OSSL_PROVIDER *new_base_provider; +#endif + + if (config == NULL) { + return -1; + } + +#ifndef FLB_HAVE_OPENSSL + if (config->fips_mode != FLB_TRUE) { + return 0; + } + + flb_error("[fips] security.fips_mode requires Fluent Bit to be built with OpenSSL"); + return -1; +#else +#if OPENSSL_VERSION_NUMBER < 0x30000000L + if (config->fips_mode != FLB_TRUE) { + return 0; + } + + flb_error("[fips] security.fips_mode requires OpenSSL 3.0 or later"); + return -1; +#else + pthread_mutex_lock(&fips_init_mutex); + + properties_changed = FLB_FALSE; + new_fips_provider = NULL; + new_base_provider = NULL; + + if (fips_provider_is_active() == FLB_TRUE) { + if (fips_process_mode == FLB_FALSE) { + flb_error("[fips] OpenSSL FIPS mode changed after process initialization"); + pthread_mutex_unlock(&fips_init_mutex); + return -1; + } + + if (fips_provider == NULL || base_provider == NULL) { + ERR_clear_error(); + new_fips_provider = OSSL_PROVIDER_try_load(NULL, "fips", 1); + if (new_fips_provider == NULL) { + flb_error("[fips] failed to retain the active OpenSSL FIPS provider"); + log_openssl_errors(); + goto error; + } + + ERR_clear_error(); + new_base_provider = OSSL_PROVIDER_try_load(NULL, "base", 1); + if (new_base_provider == NULL) { + flb_error("[fips] failed to load OpenSSL base provider"); + log_openssl_errors(); + goto error; + } + + fips_provider = new_fips_provider; + base_provider = new_base_provider; + } + + fips_process_mode = FLB_TRUE; + config->fips_mode_active = FLB_TRUE; + pthread_mutex_unlock(&fips_init_mutex); + return 0; + } + + if (EVP_default_properties_is_fips_enabled(NULL) == 1) { + flb_error("[fips] OpenSSL FIPS properties are enabled without an active provider"); + pthread_mutex_unlock(&fips_init_mutex); + return -1; + } + + if (fips_process_mode == FLB_TRUE) { + flb_error("[fips] OpenSSL FIPS mode changed after process initialization"); + pthread_mutex_unlock(&fips_init_mutex); + return -1; + } + + if (config->fips_mode != FLB_TRUE) { + fips_process_mode = FLB_FALSE; + config->fips_mode_active = FLB_FALSE; + pthread_mutex_unlock(&fips_init_mutex); + return 0; + } + + if (fips_process_mode == FLB_FALSE) { + flb_error("[fips] cannot enable FIPS mode after non-FIPS process initialization"); + pthread_mutex_unlock(&fips_init_mutex); + return -1; + } + + ERR_clear_error(); + new_fips_provider = OSSL_PROVIDER_try_load(NULL, "fips", 1); + if (new_fips_provider == NULL) { + flb_error("[fips] failed to load OpenSSL FIPS provider"); + log_openssl_errors(); + goto error; + } + + ERR_clear_error(); + new_base_provider = OSSL_PROVIDER_try_load(NULL, "base", 1); + if (new_base_provider == NULL) { + flb_error("[fips] failed to load OpenSSL base provider"); + log_openssl_errors(); + goto error; + } + + if (EVP_default_properties_is_fips_enabled(NULL) != 1) { + ERR_clear_error(); + ret = EVP_default_properties_enable_fips(NULL, 1); + if (ret != 1) { + flb_error("[fips] failed to enable OpenSSL FIPS default properties"); + log_openssl_errors(); + goto error; + } + properties_changed = FLB_TRUE; + } + + if (fips_provider_is_active() != FLB_TRUE) { + flb_error("[fips] OpenSSL FIPS provider is not active"); + log_openssl_errors(); + goto error; + } + + /* Keep the providers loaded because the default OpenSSL context is process-wide. */ + fips_provider = new_fips_provider; + base_provider = new_base_provider; + fips_process_mode = FLB_TRUE; + config->fips_mode_active = FLB_TRUE; + + pthread_mutex_unlock(&fips_init_mutex); + flb_info("[fips] OpenSSL FIPS mode enabled"); + + return 0; + +error: + if (properties_changed == FLB_TRUE) { + ret = EVP_default_properties_enable_fips(NULL, 0); + if (ret != 1) { + flb_error("[fips] failed to restore OpenSSL default properties"); + log_openssl_errors(); + } + } + if (new_base_provider != NULL) { + OSSL_PROVIDER_unload(new_base_provider); + } + if (new_fips_provider != NULL) { + OSSL_PROVIDER_unload(new_fips_provider); + } + + pthread_mutex_unlock(&fips_init_mutex); + return -1; +#endif +#endif +} diff --git a/src/flb_hash.c b/src/flb_hash.c index eef18d86878..5325ff4b90e 100644 --- a/src/flb_hash.c +++ b/src/flb_hash.c @@ -47,6 +47,10 @@ int flb_hash_init(struct flb_hash *context, int hash_type) return FLB_CRYPTO_INVALID_ARGUMENT; } + context->backend_context = NULL; + context->digest_size = 0; + context->last_error = 0; + digest_algorithm = flb_crypto_get_digest_algorithm_instance_by_id(hash_type); if (digest_algorithm == NULL) { @@ -65,6 +69,8 @@ int flb_hash_init(struct flb_hash *context, int hash_type) if (result == 0) { context->last_error = ERR_get_error(); + EVP_MD_CTX_destroy(context->backend_context); + context->backend_context = NULL; return FLB_CRYPTO_BACKEND_ERROR; } diff --git a/src/flb_hmac.c b/src/flb_hmac.c index 9c159c6351e..7882a918d6f 100644 --- a/src/flb_hmac.c +++ b/src/flb_hmac.c @@ -110,6 +110,8 @@ int flb_hmac_init(struct flb_hmac *context, if (result == 0) { context->last_error = ERR_get_error(); + flb_hmac_cleanup(context); + return FLB_CRYPTO_BACKEND_ERROR; } @@ -195,6 +197,8 @@ int flb_hmac_init(struct flb_hmac *context, if (result != 1) { context->last_error = ERR_get_error(); + flb_hmac_cleanup(context); + return FLB_CRYPTO_BACKEND_ERROR; } diff --git a/src/flb_input.c b/src/flb_input.c index e990eee238c..3d44c6ffb8c 100644 --- a/src/flb_input.c +++ b/src/flb_input.c @@ -44,6 +44,7 @@ #include #include #include +#include /* input plugin macro helpers */ #include @@ -197,13 +198,21 @@ struct mk_list *flb_input_get_global_config_map(struct flb_config *config) static int check_protocol(const char *prot, const char *output) { int len; + char *separator; - len = strlen(prot); - if (len != strlen(output)) { + separator = strstr(output, "://"); + if (separator != NULL && separator != output) { + len = separator - output; + } + else { + len = strlen(output); + } + + if (strlen(prot) != (size_t) len) { return 0; } - if (protcmp(prot, output) != 0) { + if (strncasecmp(prot, output, len) != 0) { return 0; } @@ -275,8 +284,12 @@ struct flb_input_instance *flb_input_new(struct flb_config *config, int id; int ret; int flags = 0; + size_t input_name_length; + const char *alias_target; + const char *input_name; + const char *separator; struct mk_list *head; - struct flb_input_plugin *plugin; + struct flb_input_plugin *plugin = NULL; struct flb_input_instance *instance = NULL; /* use for locking the use of the chunk trace context. */ @@ -289,9 +302,36 @@ struct flb_input_instance *flb_input_new(struct flb_config *config, return NULL; } + input_name = input; + + /* Prefer an exact registered plugin name over an alias with the same name. */ + mk_list_foreach(head, &config->in_plugins) { + plugin = mk_list_entry(head, struct flb_input_plugin, _head); + if (check_protocol(plugin->name, input_name)) { + break; + } + plugin = NULL; + } + + if (plugin == NULL) { + separator = strstr(input, "://"); + if (separator != NULL && separator != input) { + input_name_length = separator - input; + } + else { + input_name_length = strlen(input); + } + alias_target = flb_plugin_alias_get(FLB_PLUGIN_INPUT, input, + input_name_length); + if (alias_target == NULL) { + return NULL; + } + input_name = alias_target; + } + mk_list_foreach(head, &config->in_plugins) { plugin = mk_list_entry(head, struct flb_input_plugin, _head); - if (!check_protocol(plugin->name, input)) { + if (!check_protocol(plugin->name, input_name)) { plugin = NULL; continue; } @@ -471,7 +511,12 @@ struct flb_input_instance *flb_input_new(struct flb_config *config, /* Plugin use networking */ if (plugin->flags & (FLB_INPUT_NET | FLB_INPUT_NET_SERVER)) { - ret = flb_net_host_set(plugin->name, &instance->host, input); + if (strstr(input, "://") != NULL) { + ret = flb_net_host_set(plugin->name, &instance->host, input); + } + else { + ret = flb_net_host_set(plugin->name, &instance->host, input_name); + } if (ret != 0) { if (instance->ht_log_chunks) { flb_hash_table_destroy(instance->ht_log_chunks); diff --git a/src/flb_input_ingest.c b/src/flb_input_ingest.c index 621ea6cc721..e4d63454953 100644 --- a/src/flb_input_ingest.c +++ b/src/flb_input_ingest.c @@ -51,17 +51,22 @@ struct flb_input_ingress_event { struct { void *buf; size_t size; + size_t records; } log; - struct cmt *metrics; struct ctrace *traces; struct cprof *profiles; } data; + struct cfl_list metrics; struct mk_list _head; }; static void flb_input_ingress_event_destroy(struct flb_input_ingress_event *event) { + struct cfl_list *iterator; + struct cfl_list *tmp; + struct cmt *context; + if (event == NULL) { return; } @@ -80,8 +85,10 @@ static void flb_input_ingress_event_destroy(struct flb_input_ingress_event *even } } else if (event->type == FLB_INPUT_INGRESS_METRICS) { - if (event->data.metrics != NULL) { - cmt_destroy(event->data.metrics); + cfl_list_foreach_safe(iterator, tmp, &event->metrics) { + context = cfl_list_entry(iterator, struct cmt, _head); + cfl_list_del(&context->_head); + cmt_destroy(context); } } else if (event->type == FLB_INPUT_INGRESS_TRACES) { @@ -107,42 +114,14 @@ static size_t flb_input_ingress_event_size(struct flb_input_ingress_event *event return event->size; } -static size_t flb_input_ingress_estimate_metrics_size(struct cmt *cmt) -{ - if (cmt == NULL) { - return 0; - } - - /* Deferred worker ingress must not touch complex decoded signal objects - * beyond ownership transfer. Conservative accounting is safer here than - * re-encoding on the worker thread. 64 KiB is an intentionally coarse - * upper-bound for typical decoded signal objects; tune it only after - * profiling real workloads shows that this safety margin is too strict or - * too loose for queue backpressure. - */ - return 64 * 1024; -} - -static size_t flb_input_ingress_estimate_traces_size(struct ctrace *ctr) -{ - if (ctr == NULL) { - return 0; - } - - /* Keep trace accounting aligned with the conservative signal estimate - * used for metrics/profiles to avoid worker-side re-encoding. - */ - return 64 * 1024; -} - -static size_t flb_input_ingress_estimate_profiles_size(struct cprof *profile) +static size_t flb_input_ingress_signal_size(size_t payload_size) { - if (profile == NULL) { - return 0; + if (payload_size > 0) { + return payload_size; } - /* Keep profile accounting aligned with the conservative signal estimate - * used for metrics/traces to avoid worker-side re-encoding. + /* Callers should provide the source payload size. Keep a non-zero fallback + * for producers that legitimately build a signal without a wire payload. */ return 64 * 1024; } @@ -185,6 +164,29 @@ static void flb_input_ingress_update_metrics(struct flb_input_instance *ins) } } +static int flb_input_ingress_busy(struct flb_input_instance *ins, + struct flb_input_ingress_event *event) +{ + struct cmt_counter *counter; + flb_sds_t input_name; + + counter = ins->cmt_ingress_queue_busy; + input_name = NULL; + + if (counter != NULL) { + input_name = flb_sds_create(flb_input_name(ins)); + } + + if (counter != NULL && input_name != NULL) { + cmt_counter_add(counter, cfl_time_now(), 1, 1, (char *[]) {input_name}); + flb_sds_destroy(input_name); + } + + flb_input_ingress_event_destroy(event); + + return FLB_INPUT_INGRESS_BUSY; +} + static int flb_input_ingress_enqueue(struct flb_input_instance *ins, struct flb_input_ingress_event *event) { @@ -193,8 +195,6 @@ static int flb_input_ingress_enqueue(struct flb_input_instance *ins, int should_signal; int wait_result; struct timespec deadline; - struct cmt_counter *cmt_ingress_queue_busy; - flb_sds_t input_name; if (ins == NULL || event == NULL || ins->ingress_queue_enabled != FLB_TRUE) { flb_input_ingress_event_destroy(event); @@ -206,6 +206,12 @@ static int flb_input_ingress_enqueue(struct flb_input_instance *ins, pthread_mutex_lock(&ins->ingress_queue_lock); + if (ins->ingress_queue_byte_limit > 0 && + event_size > ins->ingress_queue_byte_limit) { + pthread_mutex_unlock(&ins->ingress_queue_lock); + return flb_input_ingress_busy(ins, event); + } + while (ins->ingress_queue_enabled == FLB_TRUE) { queue_is_full = FLB_FALSE; @@ -214,9 +220,12 @@ static int flb_input_ingress_enqueue(struct flb_input_instance *ins, queue_is_full = FLB_TRUE; } - if (ins->ingress_queue_byte_limit > 0 && - ins->ingress_queue_pending_bytes + event_size > ins->ingress_queue_byte_limit) { - queue_is_full = FLB_TRUE; + if (ins->ingress_queue_byte_limit > 0) { + if (event_size > ins->ingress_queue_byte_limit || + ins->ingress_queue_pending_bytes > + ins->ingress_queue_byte_limit - event_size) { + queue_is_full = FLB_TRUE; + } } if (queue_is_full == FLB_FALSE) { @@ -240,26 +249,13 @@ static int flb_input_ingress_enqueue(struct flb_input_instance *ins, &ins->ingress_queue_lock, &deadline); if (wait_result == ETIMEDOUT) { - cmt_ingress_queue_busy = ins->cmt_ingress_queue_busy; - input_name = NULL; - - if (cmt_ingress_queue_busy != NULL) { - input_name = flb_sds_create(flb_input_name(ins)); - } - pthread_mutex_unlock(&ins->ingress_queue_lock); - - if (cmt_ingress_queue_busy != NULL && input_name != NULL) { - cmt_counter_add(cmt_ingress_queue_busy, - cfl_time_now(), - 1, - 1, (char *[]) {input_name}); - flb_sds_destroy(input_name); - } - + return flb_input_ingress_busy(ins, event); + } + else if (wait_result != 0) { + pthread_mutex_unlock(&ins->ingress_queue_lock); flb_input_ingress_event_destroy(event); - - return FLB_INPUT_INGRESS_BUSY; + return -1; } } @@ -304,6 +300,7 @@ static struct flb_input_ingress_event *flb_input_ingress_event_create( } event->type = type; + cfl_list_init(&event->metrics); mk_list_entry_init(&event->_head); if (tag != NULL && tag_len > 0) { @@ -334,13 +331,16 @@ static int flb_input_ingress_collector(struct flb_input_instance *ins, struct flb_config *config, void *context) { + int append_result; int result; - int queue_was_full; size_t pending_events; size_t pending_bytes; struct mk_list *head; struct mk_list *tmp; struct mk_list queue; + struct cfl_list *iterator; + struct cfl_list *list_tmp; + struct cmt *metrics_context; struct flb_input_ingress_event *event; (void) config; @@ -360,15 +360,6 @@ static int flb_input_ingress_collector(struct flb_input_instance *ins, pending_events = ins->ingress_queue_pending_events; pending_bytes = ins->ingress_queue_pending_bytes; - queue_was_full = FLB_FALSE; - - if ((ins->ingress_queue_event_limit > 0 && - pending_events >= ins->ingress_queue_event_limit) || - (ins->ingress_queue_byte_limit > 0 && - pending_bytes >= ins->ingress_queue_byte_limit)) { - queue_was_full = FLB_TRUE; - } - mk_list_foreach_safe(head, tmp, &ins->ingress_queue) { mk_list_del(head); mk_list_add(head, &queue); @@ -379,7 +370,7 @@ static int flb_input_ingress_collector(struct flb_input_instance *ins, ins->ingress_queue_signal_pending = FLB_FALSE; flb_input_ingress_update_metrics(ins); - if (queue_was_full == FLB_TRUE) { + if (pending_events > 0 || pending_bytes > 0) { pthread_cond_broadcast(&ins->ingress_queue_space_available); } @@ -391,17 +382,39 @@ static int flb_input_ingress_collector(struct flb_input_instance *ins, mk_list_entry_init(&event->_head); if (event->type == FLB_INPUT_INGRESS_LOG) { - result = flb_input_log_append(ins, - event->tag, - event->tag != NULL ? flb_sds_len(event->tag) : 0, - event->data.log.buf, - event->data.log.size); + if (event->data.log.records > 0) { + result = flb_input_log_append_records( + ins, + event->data.log.records, + event->tag, + event->tag != NULL ? flb_sds_len(event->tag) : 0, + event->data.log.buf, + event->data.log.size); + } + else { + result = flb_input_log_append( + ins, + event->tag, + event->tag != NULL ? flb_sds_len(event->tag) : 0, + event->data.log.buf, + event->data.log.size); + } } else if (event->type == FLB_INPUT_INGRESS_METRICS) { - result = flb_input_metrics_append(ins, - event->tag, - event->tag != NULL ? flb_sds_len(event->tag) : 0, - event->data.metrics); + result = 0; + cfl_list_foreach_safe(iterator, list_tmp, &event->metrics) { + metrics_context = cfl_list_entry(iterator, struct cmt, _head); + cfl_list_del(&metrics_context->_head); + append_result = flb_input_metrics_append( + ins, + event->tag, + event->tag != NULL ? flb_sds_len(event->tag) : 0, + metrics_context); + cmt_destroy(metrics_context); + if (append_result != 0) { + result = append_result; + } + } } else if (event->type == FLB_INPUT_INGRESS_TRACES) { result = flb_input_trace_append(ins, @@ -561,10 +574,30 @@ int flb_input_ingress_queue_log_take(struct flb_input_instance *ins, void *buf, size_t buf_size, size_t allocation_size) +{ + return flb_input_ingress_queue_log_take_records(ins, + 0, + tag, + tag_len, + buf, + buf_size, + allocation_size); +} + +int flb_input_ingress_queue_log_take_records(struct flb_input_instance *ins, + size_t records, + const char *tag, + size_t tag_len, + void *buf, + size_t buf_size, + size_t allocation_size) { struct flb_input_ingress_event *event; if (buf == NULL || buf_size == 0) { + if (buf != NULL) { + flb_free(buf); + } return -1; } @@ -576,6 +609,7 @@ int flb_input_ingress_queue_log_take(struct flb_input_instance *ins, event->data.log.buf = buf; event->data.log.size = buf_size; + event->data.log.records = records; if (allocation_size < buf_size) { allocation_size = buf_size; } @@ -587,7 +621,8 @@ int flb_input_ingress_queue_log_take(struct flb_input_instance *ins, int flb_input_ingress_queue_metrics(struct flb_input_instance *ins, const char *tag, size_t tag_len, - struct cmt *cmt) + struct cmt *cmt, + size_t payload_size) { struct flb_input_ingress_event *event; @@ -595,14 +630,54 @@ int flb_input_ingress_queue_metrics(struct flb_input_instance *ins, return -1; } - /* Ownership of 'cmt' is transferred to the ingress queue on success. */ + /* The ingress queue consumes 'cmt' on every return path. */ event = flb_input_ingress_event_create(FLB_INPUT_INGRESS_METRICS, tag, tag_len); if (event == NULL) { + cmt_destroy(cmt); return -1; } - event->data.metrics = cmt; - event->size = flb_input_ingress_estimate_metrics_size(cmt); + if (!cfl_list_entry_is_orphan(&cmt->_head)) { + cfl_list_del(&cmt->_head); + } + cfl_list_add(&cmt->_head, &event->metrics); + event->size = flb_input_ingress_signal_size(payload_size); + + return flb_input_ingress_enqueue(ins, event); +} + +int flb_input_ingress_queue_metrics_list(struct flb_input_instance *ins, + const char *tag, + size_t tag_len, + struct cfl_list *contexts, + size_t payload_size) +{ + struct cfl_list *iterator; + struct cfl_list *tmp; + struct cmt *context; + struct flb_input_ingress_event *event; + + if (contexts == NULL || cfl_list_is_empty(contexts)) { + return -1; + } + + event = flb_input_ingress_event_create(FLB_INPUT_INGRESS_METRICS, + tag, tag_len); + if (event == NULL) { + cfl_list_foreach_safe(iterator, tmp, contexts) { + context = cfl_list_entry(iterator, struct cmt, _head); + cfl_list_del(&context->_head); + cmt_destroy(context); + } + return -1; + } + + cfl_list_foreach_safe(iterator, tmp, contexts) { + context = cfl_list_entry(iterator, struct cmt, _head); + cfl_list_del(&context->_head); + cfl_list_add(&context->_head, &event->metrics); + } + event->size = flb_input_ingress_signal_size(payload_size); return flb_input_ingress_enqueue(ins, event); } @@ -610,7 +685,8 @@ int flb_input_ingress_queue_metrics(struct flb_input_instance *ins, int flb_input_ingress_queue_traces(struct flb_input_instance *ins, const char *tag, size_t tag_len, - struct ctrace *ctr) + struct ctrace *ctr, + size_t payload_size) { struct flb_input_ingress_event *event; @@ -618,14 +694,15 @@ int flb_input_ingress_queue_traces(struct flb_input_instance *ins, return -1; } - /* Ownership of 'ctr' is transferred to the ingress queue on success. */ + /* The ingress queue consumes 'ctr' on every return path. */ event = flb_input_ingress_event_create(FLB_INPUT_INGRESS_TRACES, tag, tag_len); if (event == NULL) { + ctr_destroy(ctr); return -1; } event->data.traces = ctr; - event->size = flb_input_ingress_estimate_traces_size(ctr); + event->size = flb_input_ingress_signal_size(payload_size); return flb_input_ingress_enqueue(ins, event); } @@ -633,7 +710,8 @@ int flb_input_ingress_queue_traces(struct flb_input_instance *ins, int flb_input_ingress_queue_profiles(struct flb_input_instance *ins, const char *tag, size_t tag_len, - struct cprof *profile) + struct cprof *profile, + size_t payload_size) { struct flb_input_ingress_event *event; @@ -641,14 +719,15 @@ int flb_input_ingress_queue_profiles(struct flb_input_instance *ins, return -1; } - /* Ownership of 'profile' is transferred to the ingress queue on success. */ + /* The ingress queue consumes 'profile' on every return path. */ event = flb_input_ingress_event_create(FLB_INPUT_INGRESS_PROFILES, tag, tag_len); if (event == NULL) { + cprof_destroy(profile); return -1; } event->data.profiles = profile; - event->size = flb_input_ingress_estimate_profiles_size(profile); + event->size = flb_input_ingress_signal_size(payload_size); return flb_input_ingress_enqueue(ins, event); } diff --git a/src/flb_input_thread.c b/src/flb_input_thread.c index fe35993cce2..e9f2650b9c4 100644 --- a/src/flb_input_thread.c +++ b/src/flb_input_thread.c @@ -459,7 +459,12 @@ static void input_thread(void *data) flb_trace("[engine] resuming coroutine=%p", connection->coroutine); - flb_coro_resume(connection->coroutine); + if (connection->event_coroutine != NULL) { + flb_downstream_conn_event_resume(connection); + } + else { + flb_coro_resume(connection->coroutine); + } } } else if (event->type == FLB_ENGINE_EV_INPUT) { diff --git a/src/flb_io.c b/src/flb_io.c index 38fbb3eed4b..86afc8e8b79 100644 --- a/src/flb_io.c +++ b/src/flb_io.c @@ -45,7 +45,6 @@ #include #include #include -#include #include #include @@ -107,6 +106,7 @@ int flb_io_net_connect(struct flb_connection *connection, struct flb_coro *coro) { int ret; + int socket_ret; int async = FLB_FALSE; flb_sockfd_t fd = -1; int flags = flb_connection_get_flags(connection); @@ -125,17 +125,49 @@ int flb_io_net_connect(struct flb_connection *connection, async = FLB_FALSE; } - /* Perform TCP connection */ - fd = flb_net_tcp_connect(connection->upstream->tcp_host, - connection->upstream->tcp_port, - connection->stream->net.source_address, - connection->stream->net.connect_timeout, - async, coro, connection); + if (connection->stream->transport == FLB_TRANSPORT_UDP) { + fd = flb_net_udp_connect(connection->upstream->tcp_host, + connection->upstream->tcp_port, + connection->stream->net.source_address); + + if (fd >= 0) { + if (async == FLB_TRUE) { + socket_ret = flb_net_socket_nonblocking(fd); + if (socket_ret == -1) { + flb_socket_close(fd); + connection->fd = -1; + connection->event.fd = -1; + return -1; + } + } + + socket_ret = flb_net_socket_set_rcvtimeout(fd, connection->net->io_timeout); + if (socket_ret == -1) { + flb_socket_close(fd); + connection->fd = -1; + connection->event.fd = -1; + return -1; + } + + connection->fd = fd; + connection->event.fd = fd; + } + } + else { + /* Perform TCP connection */ + fd = flb_net_tcp_connect(connection->upstream->tcp_host, + connection->upstream->tcp_port, + connection->stream->net.source_address, + connection->stream->net.connect_timeout, + async, coro, connection); + } + if (fd == -1) { return -1; } - if (connection->upstream->proxied_host) { + if (connection->stream->transport == FLB_TRANSPORT_TCP && + connection->upstream->proxied_host) { ret = flb_http_client_proxy_connect(connection); if (ret == -1) { @@ -156,7 +188,8 @@ int flb_io_net_connect(struct flb_connection *connection, } /* set TCP keepalive and it's options */ - if (connection->net->tcp_keepalive) { + if (connection->stream->transport == FLB_TRANSPORT_TCP && + connection->net->tcp_keepalive) { ret = flb_net_socket_tcp_keepalive(connection->fd, connection->net); @@ -302,31 +335,37 @@ static FLB_INLINE void net_io_backup_event(struct flb_connection *connection, } } -static FLB_INLINE void net_io_restore_event(struct flb_connection *connection, - struct mk_event *backup) +static FLB_INLINE int net_io_restore_event(struct flb_connection *connection, + struct mk_event *backup) { int result; - if (connection != NULL && backup != NULL) { - if (MK_EVENT_IS_REGISTERED((&connection->event))) { - result = mk_event_del(connection->evl, &connection->event); + if (connection == NULL || backup == NULL) { + return -1; + } - assert(result == 0); + if (MK_EVENT_IS_REGISTERED((&connection->event))) { + result = mk_event_del(connection->evl, &connection->event); + if (result == -1) { + return -1; } + } - if (MK_EVENT_IS_REGISTERED(backup)) { - connection->event.priority = backup->priority; - connection->event.handler = backup->handler; - - result = mk_event_add(connection->evl, - connection->fd, - backup->type, - backup->mask, - &connection->event); + if (MK_EVENT_IS_REGISTERED(backup)) { + connection->event.priority = backup->priority; + connection->event.handler = backup->handler; - assert(result == 0); + result = mk_event_add(connection->evl, + connection->fd, + backup->type, + backup->mask, + &connection->event); + if (result == -1) { + return -1; } } + + return 0; } /* @@ -415,6 +454,13 @@ static FLB_INLINE int net_io_write_async(struct flb_coro *co, */ connection->coroutine = NULL; + if (connection->net_error != -1) { + *out_len = total; + net_io_restore_event(connection, &event_backup); + + return -1; + } + /* Save events mask since mk_event_del() will reset it */ mask = connection->event.mask; @@ -476,6 +522,8 @@ static FLB_INLINE int net_io_write_async(struct flb_coro *co, total += bytes; if (total < len) { if ((connection->event.mask & MK_EVENT_WRITE) == 0) { + event_restore_needed = FLB_TRUE; + ret = mk_event_add(connection->evl, connection->fd, FLB_ENGINE_EV_THREAD, @@ -506,6 +554,13 @@ static FLB_INLINE int net_io_write_async(struct flb_coro *co, */ connection->coroutine = NULL; + if (connection->net_error != -1) { + *out_len = total; + net_io_restore_event(connection, &event_backup); + + return -1; + } + goto retry; } @@ -520,7 +575,12 @@ static FLB_INLINE int net_io_write_async(struct flb_coro *co, * the same event. */ - net_io_restore_event(connection, &event_backup); + ret = net_io_restore_event(connection, &event_backup); + if (ret == -1) { + *out_len = total; + + return -1; + } } *out_len = total; @@ -635,6 +695,12 @@ static FLB_INLINE ssize_t net_io_read_async(struct flb_coro *co, */ connection->coroutine = NULL; + if (connection->net_error != -1) { + net_io_restore_event(connection, &event_backup); + + return -1; + } + goto retry_read; } else { @@ -658,7 +724,9 @@ static FLB_INLINE ssize_t net_io_read_async(struct flb_coro *co, * the same event. */ - net_io_restore_event(connection, &event_backup); + if (net_io_restore_event(connection, &event_backup) == -1) { + return -1; + } } return ret; @@ -694,7 +762,7 @@ int flb_io_net_write(struct flb_connection *connection, const void *data, } } #ifdef FLB_HAVE_TLS - else if (flags & FLB_IO_TLS) { + else if (flags & (FLB_IO_TLS | FLB_IO_DTLS)) { if (flags & FLB_IO_ASYNC) { ret = flb_tls_net_write_async(coro, connection->tls_session, data, len, out_len); } @@ -702,6 +770,10 @@ int flb_io_net_write(struct flb_connection *connection, const void *data, ret = flb_tls_net_write(connection->tls_session, data, len, out_len); } } + else { + flb_error("[io] TLS session set on connection #%i but transport flags are invalid (%i)", + connection->fd, flags); + } #endif if (ret > 0) { @@ -742,7 +814,7 @@ ssize_t flb_io_net_read(struct flb_connection *connection, void *buf, size_t len } } #ifdef FLB_HAVE_TLS - else if (flags & FLB_IO_TLS) { + else if (flags & (FLB_IO_TLS | FLB_IO_DTLS)) { if (flags & FLB_IO_ASYNC) { ret = flb_tls_net_read_async(coro, connection->tls_session, buf, len); } @@ -750,6 +822,10 @@ ssize_t flb_io_net_read(struct flb_connection *connection, void *buf, size_t len ret = flb_tls_net_read(connection->tls_session, buf, len); } } + else { + flb_error("[io] TLS session set on connection #%i but transport flags are invalid (%i)", + connection->fd, flags); + } #endif if (ret > 0) { diff --git a/src/flb_lib.c b/src/flb_lib.c index 7257480b063..59bfa298e13 100644 --- a/src/flb_lib.c +++ b/src/flb_lib.c @@ -23,6 +23,7 @@ #include #include #include +#include #include #include #include @@ -940,8 +941,15 @@ int static do_start(flb_ctx_t *ctx) /* set context as the last active one */ - /* spawn worker thread */ config = ctx->config; + + /* OpenSSL default properties must be configured before worker threads start. */ + ret = flb_fips_init(config); + if (ret != 0) { + return -1; + } + + /* spawn worker thread */ ret = mk_utils_worker_spawn(flb_lib_worker, ctx, &tid); if (ret == -1) { return -1; diff --git a/src/flb_log_event_decoder.c b/src/flb_log_event_decoder.c index d5e3c909c14..2534dacc203 100644 --- a/src/flb_log_event_decoder.c +++ b/src/flb_log_event_decoder.c @@ -22,6 +22,8 @@ #include #include +#include + #define FLB_LOG_EVENT_DECODER_MAX_RECURSION_DEPTH 1000 /* Safety limit for recursion */ static int create_empty_map(struct flb_log_event_decoder *context) { @@ -180,6 +182,9 @@ void flb_log_event_decoder_destroy(struct flb_log_event_decoder *context) int flb_log_event_decoder_decode_timestamp(msgpack_object *input, struct flb_time *output) { + uint32_t seconds; + uint32_t nanoseconds; + flb_time_zero(output); if (input->type == MSGPACK_OBJECT_POSITIVE_INTEGER) { @@ -194,15 +199,43 @@ int flb_log_event_decoder_decode_timestamp(msgpack_object *input, return FLB_EVENT_DECODER_ERROR_WRONG_TIMESTAMP_TYPE; } - output->tm.tv_sec = - (int32_t) FLB_UINT32_TO_HOST_BYTE_ORDER( - FLB_ALIGNED_DWORD_READ( - (unsigned char *) &input->via.ext.ptr[0])); + seconds = FLB_UINT32_TO_HOST_BYTE_ORDER( + FLB_ALIGNED_DWORD_READ( + (unsigned char *) &input->via.ext.ptr[0])); + + /* + * EventTime seconds are unsigned. Only the two exact values used by + * Fluent Bit's legacy group records are special; all other values, + * including 0xfffffffd, are normal timestamps. + */ + if (seconds == UINT32_MAX) { + nanoseconds = FLB_UINT32_TO_HOST_BYTE_ORDER( + FLB_ALIGNED_DWORD_READ( + (unsigned char *) &input->via.ext.ptr[4])); + if (nanoseconds != 0) { + return FLB_EVENT_DECODER_ERROR_WRONG_TIMESTAMP_TYPE; + } + output->tm.tv_sec = FLB_LOG_EVENT_GROUP_START; + output->tm.tv_nsec = 0; + return FLB_EVENT_DECODER_SUCCESS; + } + else if (seconds == UINT32_MAX - 1) { + nanoseconds = FLB_UINT32_TO_HOST_BYTE_ORDER( + FLB_ALIGNED_DWORD_READ( + (unsigned char *) &input->via.ext.ptr[4])); + if (nanoseconds != 0) { + return FLB_EVENT_DECODER_ERROR_WRONG_TIMESTAMP_TYPE; + } + output->tm.tv_sec = FLB_LOG_EVENT_GROUP_END; + output->tm.tv_nsec = 0; + return FLB_EVENT_DECODER_SUCCESS; + } + + if (flb_time_msgpack_to_time(output, input) != 0) { + return FLB_EVENT_DECODER_ERROR_WRONG_TIMESTAMP_TYPE; + } - output->tm.tv_nsec = - (int32_t) FLB_UINT32_TO_HOST_BYTE_ORDER( - FLB_ALIGNED_DWORD_READ( - (unsigned char *) &input->via.ext.ptr[4])); + return FLB_EVENT_DECODER_SUCCESS; } else { return FLB_EVENT_DECODER_ERROR_WRONG_TIMESTAMP_TYPE; @@ -313,7 +346,7 @@ int flb_log_event_decoder_next(struct flb_log_event_decoder *context, int result; int record_type; size_t previous_offset; - int32_t invalid_timestamp; + int64_t invalid_timestamp; if (context == NULL) { return FLB_EVENT_DECODER_ERROR_INVALID_CONTEXT; @@ -368,8 +401,8 @@ int flb_log_event_decoder_next(struct flb_log_event_decoder *context, * to avoid losing valid group metadata if corruption occurs mid-group. * Skip the record and continue processing. */ - invalid_timestamp = (int32_t) event->timestamp.tm.tv_sec; - flb_debug("[decoder] Invalid group marker timestamp (%d), skipping record. " + invalid_timestamp = (int64_t) event->timestamp.tm.tv_sec; + flb_debug("[decoder] Invalid group marker timestamp (%" PRId64 "), skipping record. " "Group state preserved.", invalid_timestamp); /* Increment recursion depth before recursive call */ @@ -457,9 +490,9 @@ int flb_log_event_decoder_next(struct flb_log_event_decoder *context, int flb_log_event_decoder_get_record_type(struct flb_log_event *event, int32_t *type) { - int32_t s; + time_t s; - s = (int32_t) event->timestamp.tm.tv_sec; + s = event->timestamp.tm.tv_sec; if (s >= 0) { *type = FLB_LOG_EVENT_NORMAL; diff --git a/src/flb_log_event_encoder.c b/src/flb_log_event_encoder.c index d2a20a7351f..9bb1968373e 100644 --- a/src/flb_log_event_encoder.c +++ b/src/flb_log_event_encoder.c @@ -346,6 +346,13 @@ int flb_log_event_encoder_set_timestamp(struct flb_log_event_encoder *context, struct flb_time *timestamp) { if (timestamp != NULL) { + if (flb_time_is_valid_eventtime(timestamp) != FLB_TRUE && + !(timestamp->tm.tv_nsec == 0 && + (timestamp->tm.tv_sec == FLB_LOG_EVENT_GROUP_START || + timestamp->tm.tv_sec == FLB_LOG_EVENT_GROUP_END))) { + return FLB_EVENT_ENCODER_ERROR_INVALID_ARGUMENT; + } + flb_time_copy(&context->timestamp, timestamp); } else { diff --git a/src/flb_log_event_encoder_primitives.c b/src/flb_log_event_encoder_primitives.c index 741766105ed..74af8677689 100644 --- a/src/flb_log_event_encoder_primitives.c +++ b/src/flb_log_event_encoder_primitives.c @@ -519,6 +519,10 @@ int flb_log_event_encoder_append_forward_v1_timestamp( { uint32_t value[2]; + if (flb_time_is_valid_eventtime(timestamp) != FLB_TRUE) { + return FLB_EVENT_ENCODER_ERROR_INVALID_ARGUMENT; + } + value[0] = FLB_UINT32_TO_NETWORK_BYTE_ORDER((uint32_t) timestamp->tm.tv_sec); value[1] = FLB_UINT32_TO_NETWORK_BYTE_ORDER((uint32_t) timestamp->tm.tv_nsec); diff --git a/src/flb_mp.c b/src/flb_mp.c index ba2406adbfb..4fd9783526e 100644 --- a/src/flb_mp.c +++ b/src/flb_mp.c @@ -1688,182 +1688,200 @@ int flb_mp_chunk_cobj_record_next(struct flb_mp_chunk_cobj *chunk_cobj, struct flb_condition *condition = NULL; *out_record = NULL; - bytes = chunk_cobj->log_decoder->length - chunk_cobj->log_decoder->offset; /* Check if we have a condition */ condition = chunk_cobj->condition; /* - * if there are remaining decoder bytes, keep iterating msgpack and populate - * the cobj list. Otherwise it means all the content is ready as a chunk_cobj_record. + * Iterate until a record that matches the condition (if one is configured) + * is found, or until the end of the chunk is reached. + * + * This was previously implemented with tail recursion: whenever a record + * did not match the condition, the function called itself to advance to the + * next record. With a large run of non-matching records (for example a + * condition that filters out most of a chunk) the recursion depth grew + * proportionally to the number of records and could exhaust the stack. An + * explicit loop keeps the stack usage constant. */ - if (bytes > 0) { - record = flb_mp_chunk_record_create(chunk_cobj); - if (!record) { - return FLB_MP_CHUNK_RECORD_ERROR; - } - - ret = flb_log_event_decoder_next(chunk_cobj->log_decoder, &record->event); - if (ret != FLB_EVENT_DECODER_SUCCESS) { - flb_free(record); - return -1; - } - - record->cobj_metadata = flb_mp_object_to_cfl(record->event.metadata); - if (!record->cobj_metadata) { - flb_free(record); - return FLB_MP_CHUNK_RECORD_ERROR; - } - - record->cobj_record = flb_mp_object_to_cfl(record->event.body); - if (!record->cobj_record) { - cfl_object_destroy(record->cobj_metadata); - flb_free(record); - return -1; - } - - ret = flb_log_event_decoder_get_record_type(&record->event, &record_type); - if (ret != FLB_EVENT_DECODER_SUCCESS) { - cfl_object_destroy(record->cobj_record); - cfl_object_destroy(record->cobj_metadata); - flb_free(record); - return FLB_MP_CHUNK_RECORD_ERROR; - } - - record->owns_group_metadata = FLB_FALSE; - record->owns_group_attributes = FLB_FALSE; + while (1) { + /* + * if there are remaining decoder bytes, keep iterating msgpack and + * populate the cobj list. Otherwise it means all the content is ready + * as a chunk_cobj_record. + */ + bytes = chunk_cobj->log_decoder->length - chunk_cobj->log_decoder->offset; - if (record_type == FLB_LOG_EVENT_GROUP_START) { - if (record->cobj_metadata) { - record->cobj_group_metadata = record->cobj_metadata; - record->owns_group_metadata = FLB_TRUE; - } - if (record->cobj_record) { - record->cobj_group_attributes = record->cobj_record; - record->owns_group_attributes = FLB_TRUE; + if (bytes > 0) { + record = flb_mp_chunk_record_create(chunk_cobj); + if (!record) { + return FLB_MP_CHUNK_RECORD_ERROR; } - chunk_cobj->active_group_metadata = record->cobj_group_metadata; - chunk_cobj->active_group_attributes = record->cobj_group_attributes; - } - else if (record_type == FLB_LOG_EVENT_GROUP_END) { - record->cobj_group_metadata = chunk_cobj->active_group_metadata; - record->cobj_group_attributes = chunk_cobj->active_group_attributes; + ret = flb_log_event_decoder_next(chunk_cobj->log_decoder, &record->event); + if (ret != FLB_EVENT_DECODER_SUCCESS) { + flb_free(record); + return -1; + } - chunk_cobj->active_group_metadata = NULL; - chunk_cobj->active_group_attributes = NULL; - } - else { - record->cobj_group_metadata = chunk_cobj->active_group_metadata; - record->cobj_group_attributes = chunk_cobj->active_group_attributes; - } - - if (!record->cobj_group_metadata && - record->event.group_metadata && - (record->event.group_metadata->type == MSGPACK_OBJECT_MAP || - record->event.group_metadata->type == MSGPACK_OBJECT_ARRAY)) { - record->cobj_group_metadata = flb_mp_object_to_cfl(record->event.group_metadata); - if (!record->cobj_group_metadata) { - if (record->owns_group_attributes && record->cobj_group_attributes) { - cfl_object_destroy(record->cobj_group_attributes); - } - cfl_object_destroy(record->cobj_record); - cfl_object_destroy(record->cobj_metadata); + record->cobj_metadata = flb_mp_object_to_cfl(record->event.metadata); + if (!record->cobj_metadata) { flb_free(record); return FLB_MP_CHUNK_RECORD_ERROR; } - record->owns_group_metadata = FLB_TRUE; - if (!chunk_cobj->active_group_metadata) { - chunk_cobj->active_group_metadata = record->cobj_group_metadata; + + record->cobj_record = flb_mp_object_to_cfl(record->event.body); + if (!record->cobj_record) { + cfl_object_destroy(record->cobj_metadata); + flb_free(record); + return -1; } - } - if (!record->cobj_group_attributes && - record->event.group_attributes && - (record->event.group_attributes->type == MSGPACK_OBJECT_MAP || - record->event.group_attributes->type == MSGPACK_OBJECT_ARRAY)) { - record->cobj_group_attributes = flb_mp_object_to_cfl(record->event.group_attributes); - if (!record->cobj_group_attributes) { - if (record->owns_group_metadata && record->cobj_group_metadata) { - cfl_object_destroy(record->cobj_group_metadata); - } + ret = flb_log_event_decoder_get_record_type(&record->event, &record_type); + if (ret != FLB_EVENT_DECODER_SUCCESS) { cfl_object_destroy(record->cobj_record); cfl_object_destroy(record->cobj_metadata); flb_free(record); return FLB_MP_CHUNK_RECORD_ERROR; } - record->owns_group_attributes = FLB_TRUE; - if (!chunk_cobj->active_group_attributes) { + + record->owns_group_metadata = FLB_FALSE; + record->owns_group_attributes = FLB_FALSE; + + if (record_type == FLB_LOG_EVENT_GROUP_START) { + if (record->cobj_metadata) { + record->cobj_group_metadata = record->cobj_metadata; + record->owns_group_metadata = FLB_TRUE; + } + if (record->cobj_record) { + record->cobj_group_attributes = record->cobj_record; + record->owns_group_attributes = FLB_TRUE; + } + + chunk_cobj->active_group_metadata = record->cobj_group_metadata; chunk_cobj->active_group_attributes = record->cobj_group_attributes; } - } + else if (record_type == FLB_LOG_EVENT_GROUP_END) { + record->cobj_group_metadata = chunk_cobj->active_group_metadata; + record->cobj_group_attributes = chunk_cobj->active_group_attributes; + + chunk_cobj->active_group_metadata = NULL; + chunk_cobj->active_group_attributes = NULL; + } + else { + record->cobj_group_metadata = chunk_cobj->active_group_metadata; + record->cobj_group_attributes = chunk_cobj->active_group_attributes; + } + + if (!record->cobj_group_metadata && + record->event.group_metadata && + (record->event.group_metadata->type == MSGPACK_OBJECT_MAP || + record->event.group_metadata->type == MSGPACK_OBJECT_ARRAY)) { + record->cobj_group_metadata = flb_mp_object_to_cfl(record->event.group_metadata); + if (!record->cobj_group_metadata) { + if (record->owns_group_attributes && record->cobj_group_attributes) { + cfl_object_destroy(record->cobj_group_attributes); + } + cfl_object_destroy(record->cobj_record); + cfl_object_destroy(record->cobj_metadata); + flb_free(record); + return FLB_MP_CHUNK_RECORD_ERROR; + } + record->owns_group_metadata = FLB_TRUE; + if (!chunk_cobj->active_group_metadata) { + chunk_cobj->active_group_metadata = record->cobj_group_metadata; + } + } - cfl_list_add(&record->_head, &chunk_cobj->records); + if (!record->cobj_group_attributes && + record->event.group_attributes && + (record->event.group_attributes->type == MSGPACK_OBJECT_MAP || + record->event.group_attributes->type == MSGPACK_OBJECT_ARRAY)) { + record->cobj_group_attributes = flb_mp_object_to_cfl(record->event.group_attributes); + if (!record->cobj_group_attributes) { + if (record->owns_group_metadata && record->cobj_group_metadata) { + cfl_object_destroy(record->cobj_group_metadata); + } + cfl_object_destroy(record->cobj_record); + cfl_object_destroy(record->cobj_metadata); + flb_free(record); + return FLB_MP_CHUNK_RECORD_ERROR; + } + record->owns_group_attributes = FLB_TRUE; + if (!chunk_cobj->active_group_attributes) { + chunk_cobj->active_group_attributes = record->cobj_group_attributes; + } + } - /* If there's a condition, check if the record matches */ - if (condition != NULL && record != NULL) { - flb_trace("[mp] evaluating condition for record"); - ret = flb_condition_evaluate(condition, record); - flb_trace("[mp] condition evaluation result: %s", ret ? "TRUE" : "FALSE"); - if (ret == FLB_FALSE) { - flb_trace("[mp] record didn't match condition, skipping"); - /* Record doesn't match the condition, continue to next record */ - return flb_mp_chunk_cobj_record_next(chunk_cobj, out_record); + cfl_list_add(&record->_head, &chunk_cobj->records); + + /* If there's a condition, check if the record matches */ + if (condition != NULL && record != NULL) { + flb_trace("[mp] evaluating condition for record"); + ret = flb_condition_evaluate(condition, record); + flb_trace("[mp] condition evaluation result: %s", ret ? "TRUE" : "FALSE"); + if (ret == FLB_FALSE) { + flb_trace("[mp] record didn't match condition, skipping"); + /* Record doesn't match the condition, continue to next record */ + continue; + } + flb_trace("[mp] record matched condition, processing"); } - flb_trace("[mp] record matched condition, processing"); + + ret = FLB_MP_CHUNK_RECORD_OK; } + else if (chunk_cobj->record_pos != NULL) { + /* is the actual record the last one ? */ + if (chunk_cobj->record_pos == cfl_list_entry_last(&chunk_cobj->records, struct flb_mp_chunk_record, _head)) { + chunk_cobj->record_pos = NULL; + return FLB_MP_CHUNK_RECORD_EOF; + } - ret = FLB_MP_CHUNK_RECORD_OK; - } - else if (chunk_cobj->record_pos != NULL) { - /* is the actual record the last one ? */ - if (chunk_cobj->record_pos == cfl_list_entry_last(&chunk_cobj->records, struct flb_mp_chunk_record, _head)) { - chunk_cobj->record_pos = NULL; - return FLB_MP_CHUNK_RECORD_EOF; - } - - record = cfl_list_entry_next(&chunk_cobj->record_pos->_head, struct flb_mp_chunk_record, - _head, &chunk_cobj->records); - - /* If there's a condition, check if the record matches */ - if (condition != NULL && record != NULL) { - flb_trace("[mp] evaluating condition for next record"); - ret = flb_condition_evaluate(condition, record); - flb_trace("[mp] next record condition evaluation result: %s", ret ? "TRUE" : "FALSE"); - if (ret == FLB_FALSE) { - flb_trace("[mp] next record didn't match condition, skipping"); - /* Record doesn't match the condition, set as current and try again */ - chunk_cobj->record_pos = record; - return flb_mp_chunk_cobj_record_next(chunk_cobj, out_record); + record = cfl_list_entry_next(&chunk_cobj->record_pos->_head, struct flb_mp_chunk_record, + _head, &chunk_cobj->records); + + /* If there's a condition, check if the record matches */ + if (condition != NULL && record != NULL) { + flb_trace("[mp] evaluating condition for next record"); + ret = flb_condition_evaluate(condition, record); + flb_trace("[mp] next record condition evaluation result: %s", ret ? "TRUE" : "FALSE"); + if (ret == FLB_FALSE) { + flb_trace("[mp] next record didn't match condition, skipping"); + /* Record doesn't match the condition, set as current and try again */ + chunk_cobj->record_pos = record; + continue; + } + flb_trace("[mp] next record matched condition, processing"); } - flb_trace("[mp] next record matched condition, processing"); + + ret = FLB_MP_CHUNK_RECORD_OK; } + else { + if (cfl_list_size(&chunk_cobj->records) == 0) { + return FLB_MP_CHUNK_RECORD_EOF; + } - ret = FLB_MP_CHUNK_RECORD_OK; - } - else { - if (cfl_list_size(&chunk_cobj->records) == 0) { - return FLB_MP_CHUNK_RECORD_EOF; - } - - /* check if we are the last in the list */ - record = cfl_list_entry_first(&chunk_cobj->records, struct flb_mp_chunk_record, _head); - - /* If there's a condition, check if the record matches */ - if (condition != NULL && record != NULL) { - flb_trace("[mp] evaluating condition for first record"); - ret = flb_condition_evaluate(condition, record); - flb_trace("[mp] first record condition evaluation result: %s", ret ? "TRUE" : "FALSE"); - if (ret == FLB_FALSE) { - flb_trace("[mp] first record didn't match condition, skipping"); - /* Record doesn't match the condition, set as current and try again */ - chunk_cobj->record_pos = record; - return flb_mp_chunk_cobj_record_next(chunk_cobj, out_record); + /* check if we are the last in the list */ + record = cfl_list_entry_first(&chunk_cobj->records, struct flb_mp_chunk_record, _head); + + /* If there's a condition, check if the record matches */ + if (condition != NULL && record != NULL) { + flb_trace("[mp] evaluating condition for first record"); + ret = flb_condition_evaluate(condition, record); + flb_trace("[mp] first record condition evaluation result: %s", ret ? "TRUE" : "FALSE"); + if (ret == FLB_FALSE) { + flb_trace("[mp] first record didn't match condition, skipping"); + /* Record doesn't match the condition, set as current and try again */ + chunk_cobj->record_pos = record; + continue; + } + flb_trace("[mp] first record matched condition, processing"); } - flb_trace("[mp] first record matched condition, processing"); + + ret = FLB_MP_CHUNK_RECORD_OK; } - ret = FLB_MP_CHUNK_RECORD_OK; + /* A matching record (or the next record) was found; stop iterating. */ + break; } chunk_cobj->record_pos = record; diff --git a/src/flb_network.c b/src/flb_network.c index 91a3e37a1d8..d4572846791 100644 --- a/src/flb_network.c +++ b/src/flb_network.c @@ -141,6 +141,7 @@ void flb_net_setup_init(struct flb_net_setup *net) net->dns_resolver = NULL; net->dns_prefer_ipv4 = FLB_FALSE; net->dns_prefer_ipv6 = FLB_FALSE; + net->share_port = FLB_FALSE; net->keepalive = FLB_TRUE; net->keepalive_idle_timeout = 30; net->keepalive_max_recycle = 0; @@ -161,20 +162,26 @@ int flb_net_host_set(const char *plugin_name, struct flb_net_host *host, const c int len; int olen; const char *s, *e, *u; + const char *separator; memset(host, '\0', sizeof(struct flb_net_host)); olen = strlen(address); - if (olen == strlen(plugin_name)) { - return 0; + separator = strstr(address, "://"); + if (separator != NULL && separator != address) { + s = separator + 3; } + else { + if (olen == strlen(plugin_name)) { + return 0; + } - len = strlen(plugin_name) + 3; - if (olen < len) { - return -1; + len = strlen(plugin_name) + 3; + if (olen < len) { + return -1; + } + s = address + len; } - - s = address + len; if (*s == '[') { /* IPv6 address (RFC 3986) */ e = strchr(++s, ']'); @@ -239,6 +246,8 @@ int flb_net_socket_share_port(flb_sockfd_t fd) } #else (void) fd; + flb_error("shared listener ports are not supported on this platform"); + return -1; #endif return 0; @@ -1689,8 +1698,9 @@ flb_sockfd_t flb_net_server(const char *port, const char *listen_addr, continue; } - if (share_port) { - flb_net_socket_share_port(fd); + if (share_port && flb_net_socket_share_port(fd) == -1) { + flb_socket_close(fd); + continue; } flb_net_socket_tcp_nodelay(fd); @@ -1763,8 +1773,9 @@ flb_sockfd_t flb_net_server_udp(const char *port, const char *listen_addr, int s continue; } - if (share_port) { - flb_net_socket_share_port(fd); + if (share_port && flb_net_socket_share_port(fd) == -1) { + flb_socket_close(fd); + continue; } ret = flb_net_bind_udp(fd, rp->ai_addr, rp->ai_addrlen); @@ -1820,8 +1831,9 @@ flb_sockfd_t flb_net_server_unix(const char *listen_path, strncpy(address.sun_path, listen_path, sizeof(address.sun_path)); - if (share_port) { - flb_net_socket_share_port(fd); + if (share_port && flb_net_socket_share_port(fd) == -1) { + flb_socket_close(fd); + return -1; } if (stream_mode) { diff --git a/src/flb_output.c b/src/flb_output.c index 26f66ad9e36..517608dfddf 100644 --- a/src/flb_output.c +++ b/src/flb_output.c @@ -35,6 +35,7 @@ #include #include #include +#include #include #include #include @@ -137,12 +138,9 @@ static int check_protocol(const char *prot, const char *output) len = strlen(output); } - if (strlen(prot) != len) { - return 0; - } - /* Output plugin match */ - if (strncasecmp(prot, output, len) == 0) { + if (strlen(prot) == (size_t) len && + strncasecmp(prot, output, len) == 0) { return 1; } @@ -676,17 +674,24 @@ struct flb_output_instance *flb_output_new(struct flb_config *config, { int ret = -1; int flags = 0; + size_t output_name_length; + const char *alias_target; + const char *output_name; + const char *separator; struct mk_list *head; - struct flb_output_plugin *plugin; + struct flb_output_plugin *plugin = NULL; struct flb_output_instance *instance = NULL; if (!output) { return NULL; } + output_name = output; + + /* Prefer an exact registered plugin name over an alias with the same name. */ mk_list_foreach(head, &config->out_plugins) { plugin = mk_list_entry(head, struct flb_output_plugin, _head); - if (!check_protocol(plugin->name, output)) { + if (!check_protocol(plugin->name, output_name)) { plugin = NULL; continue; } @@ -697,6 +702,34 @@ struct flb_output_instance *flb_output_new(struct flb_config *config, break; } + if (plugin == NULL) { + separator = strstr(output, "://"); + if (separator != NULL && separator != output) { + output_name_length = separator - output; + } + else { + output_name_length = strlen(output); + } + alias_target = flb_plugin_alias_get(FLB_PLUGIN_OUTPUT, output, + output_name_length); + if (alias_target != NULL) { + output_name = alias_target; + + mk_list_foreach(head, &config->out_plugins) { + plugin = mk_list_entry(head, struct flb_output_plugin, _head); + if (!check_protocol(plugin->name, output_name)) { + plugin = NULL; + continue; + } + + if (public_only && plugin->flags & FLB_OUTPUT_PRIVATE) { + return NULL; + } + break; + } + } + } + if (!plugin) { return NULL; } @@ -819,11 +852,25 @@ struct flb_output_instance *flb_output_new(struct flb_config *config, #endif if (plugin->flags & FLB_OUTPUT_NET) { - ret = flb_net_host_set(plugin->name, &instance->host, output); + if (strstr(output, "://") != NULL) { + ret = flb_net_host_set(plugin->name, &instance->host, output); + } + else { + ret = flb_net_host_set(plugin->name, &instance->host, output_name); + } + if (ret != 0) { - if (instance->flags & FLB_OUTPUT_SYNCHRONOUS) { + if ((instance->flags & FLB_OUTPUT_SYNCHRONOUS) && + instance->singleplex_queue != NULL) { flb_task_queue_destroy(instance->singleplex_queue); } + if (instance->callback != NULL) { + flb_callback_destroy(instance->callback); + } + if (plugin->type != FLB_OUTPUT_PLUGIN_CORE && + instance->context != NULL) { + flb_free(instance->context); + } flb_free(instance->http_server_config); flb_free(instance); return NULL; diff --git a/src/flb_pack.c b/src/flb_pack.c index 28594b4f8d6..581255f328b 100644 --- a/src/flb_pack.c +++ b/src/flb_pack.c @@ -875,7 +875,7 @@ static int pack_print_fluent_record(size_t cnt, msgpack_unpacked result) flb_time_pop_from_msgpack(&tms, &result, &obj); flb_metadata_pop_from_msgpack(&metadata, &result, &obj); - fprintf(stdout, "[%zd] [[%"PRId32".%09lu, ", cnt, (int32_t) tms.tm.tv_sec, tms.tm.tv_nsec); + fprintf(stdout, "[%zd] [[%"PRId64".%09lu, ", cnt, (int64_t) tms.tm.tv_sec, tms.tm.tv_nsec); msgpack_object_print(stdout, *metadata); diff --git a/src/flb_plugin_alias.c b/src/flb_plugin_alias.c new file mode 100644 index 00000000000..f32c4ec14f8 --- /dev/null +++ b/src/flb_plugin_alias.c @@ -0,0 +1,180 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2026 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include + +#include +#include +#include +#include +#include + +/* + * Table that maps user-facing aliases to plugin short names. + * + * Keep this table focused on backwards/forwards compatibility names where the + * historical short name is still used internally by the plugin implementation. + */ +static struct flb_plugin_alias_entry plugin_aliases[] = { + { + FLB_PLUGIN_OUTPUT, + "elasticsearch", + "es" + }, + { + 0, + NULL, + NULL + } +}; + +static const struct flb_plugin_alias_entry *custom_plugin_aliases = NULL; +static pthread_mutex_t custom_plugin_aliases_lock = PTHREAD_MUTEX_INITIALIZER; + +static size_t protocol_part_length(const char *plugin_reference) +{ + char *separator; + + separator = strstr(plugin_reference, "://"); + if (separator != NULL && separator != plugin_reference) { + return (size_t) (separator - plugin_reference); + } + + return strlen(plugin_reference); +} + +const char *flb_plugin_alias_get(int plugin_type, const char *alias_name, + size_t alias_name_length) +{ + int index; + const struct flb_plugin_alias_entry *entry; + const struct flb_plugin_alias_entry *aliases; + + if (alias_name == NULL || alias_name_length == 0) { + return NULL; + } + + pthread_mutex_lock(&custom_plugin_aliases_lock); + aliases = custom_plugin_aliases; + if (aliases != NULL) { + for (index = 0; aliases[index].alias_name != NULL; index++) { + entry = &aliases[index]; + + if (entry->plugin_type != plugin_type) { + continue; + } + + if (strlen(entry->alias_name) != alias_name_length) { + continue; + } + + if (strncasecmp(entry->alias_name, alias_name, alias_name_length) == 0) { + pthread_mutex_unlock(&custom_plugin_aliases_lock); + return entry->plugin_name; + } + } + } + pthread_mutex_unlock(&custom_plugin_aliases_lock); + + for (index = 0; plugin_aliases[index].alias_name != NULL; index++) { + entry = &plugin_aliases[index]; + + if (entry->plugin_type != plugin_type) { + continue; + } + + if (strlen(entry->alias_name) != alias_name_length) { + continue; + } + + if (strncasecmp(entry->alias_name, alias_name, alias_name_length) == 0) { + return entry->plugin_name; + } + } + + return NULL; +} + +void flb_plugin_alias_set_custom_entries( + const struct flb_plugin_alias_entry *entries) +{ + pthread_mutex_lock(&custom_plugin_aliases_lock); + custom_plugin_aliases = entries; + pthread_mutex_unlock(&custom_plugin_aliases_lock); +} + +void flb_plugin_alias_reset_custom_entries(void) +{ + pthread_mutex_lock(&custom_plugin_aliases_lock); + custom_plugin_aliases = NULL; + pthread_mutex_unlock(&custom_plugin_aliases_lock); +} + +char *flb_plugin_alias_rewrite(int plugin_type, const char *plugin_reference) +{ + int ret; + size_t reference_length; + size_t protocol_length; + size_t plugin_name_length; + char *rewritten_reference; + const char *plugin_name; + + if (plugin_reference == NULL) { + return NULL; + } + + protocol_length = protocol_part_length(plugin_reference); + if (protocol_length == 0) { + return NULL; + } + + plugin_name = flb_plugin_alias_get(plugin_type, plugin_reference, + protocol_length); + if (plugin_name == NULL) { + return NULL; + } + + plugin_name_length = strlen(plugin_name); + + if (plugin_name_length == protocol_length && + strncasecmp(plugin_name, plugin_reference, protocol_length) == 0) { + return NULL; + } + + reference_length = strlen(plugin_reference); + rewritten_reference = flb_calloc(1, reference_length - protocol_length + + plugin_name_length + 1); + if (rewritten_reference == NULL) { + flb_errno(); + return FLB_PLUGIN_ALIAS_ERR; + } + + memcpy(rewritten_reference, plugin_name, plugin_name_length); + + ret = snprintf(rewritten_reference + plugin_name_length, + reference_length - protocol_length + 1, + "%s", plugin_reference + protocol_length); + if (ret < 0) { + flb_free(rewritten_reference); + return FLB_PLUGIN_ALIAS_ERR; + } + + return rewritten_reference; +} diff --git a/src/flb_processor.c b/src/flb_processor.c index 5de6a4e9780..7f04146ff6f 100644 --- a/src/flb_processor.c +++ b/src/flb_processor.c @@ -35,6 +35,7 @@ #include #include #include +#include #include struct flb_config_map processor_global_properties[] = { @@ -618,13 +619,19 @@ struct flb_processor_unit *flb_processor_unit_create(struct flb_processor *proc, char *unit_name) { int result; + int native_plugin_found; struct mk_list *head; int filter_event_type; + const char *alias_target; + const char *effective_unit_name; struct flb_filter_plugin *f = NULL; struct flb_filter_instance *f_ins; struct flb_config *config = proc->config; struct flb_processor_unit *pu = NULL; struct flb_processor_instance *processor_instance; + struct flb_processor_plugin *processor_plugin; + + effective_unit_name = unit_name; /* * Looking the processor unit by using it's name and type, the first list we @@ -641,7 +648,7 @@ struct flb_processor_unit *flb_processor_unit_create(struct flb_processor *proc, /* skip filters which don't handle the required type */ if ((event_type & filter_event_type) != 0) { - if (strcmp(f->name, unit_name) == 0) { + if (strcasecmp(f->name, effective_unit_name) == 0) { break; } } @@ -649,6 +656,51 @@ struct flb_processor_unit *flb_processor_unit_create(struct flb_processor *proc, f = NULL; } + /* A processor unit may be either a filter or a native processor. */ + native_plugin_found = FLB_FALSE; + if (f == NULL) { + mk_list_foreach(head, &config->processor_plugins) { + processor_plugin = mk_list_entry(head, struct flb_processor_plugin, _head); + if (strcasecmp(processor_plugin->name, unit_name) == 0) { + native_plugin_found = FLB_TRUE; + break; + } + } + } + + /* Prefer exact registered names before falling back to aliases. */ + if (f == NULL && native_plugin_found == FLB_FALSE) { + alias_target = flb_plugin_alias_get(FLB_PLUGIN_FILTER, unit_name, + strlen(unit_name)); + if (alias_target != NULL) { + effective_unit_name = alias_target; + + mk_list_foreach(head, &config->filter_plugins) { + f = mk_list_entry(head, struct flb_filter_plugin, _head); + + filter_event_type = f->event_type; + if (filter_event_type == 0) { + filter_event_type = FLB_FILTER_LOGS; + } + + if ((event_type & filter_event_type) != 0 && + strcasecmp(f->name, effective_unit_name) == 0) { + break; + } + f = NULL; + } + } + + if (f == NULL) { + effective_unit_name = unit_name; + alias_target = flb_plugin_alias_get(FLB_PLUGIN_PROCESSOR, unit_name, + strlen(unit_name)); + if (alias_target != NULL) { + effective_unit_name = alias_target; + } + } + } + /* allocate and initialize processor unit context */ pu = flb_calloc(1, sizeof(struct flb_processor_unit)); @@ -659,7 +711,7 @@ struct flb_processor_unit *flb_processor_unit_create(struct flb_processor *proc, pu->parent = proc; pu->event_type = event_type; - pu->name = flb_sds_create(unit_name); + pu->name = flb_sds_create(effective_unit_name); pu->condition = NULL; if (!pu->name) { @@ -673,14 +725,13 @@ struct flb_processor_unit *flb_processor_unit_create(struct flb_processor *proc, if (result != 0) { flb_sds_destroy(pu->name); flb_free(pu); - return NULL; } /* If we matched a pipeline filter, create the speacial processing unit for it */ if (f) { /* create an instance of the filter */ - f_ins = flb_filter_new(config, unit_name, NULL); + f_ins = flb_filter_new(config, effective_unit_name, NULL); if (!f_ins) { pthread_mutex_destroy(&pu->lock); @@ -725,7 +776,7 @@ struct flb_processor_unit *flb_processor_unit_create(struct flb_processor *proc, processor_instance = flb_processor_instance_create(config, pu, pu->event_type, - unit_name, NULL); + (char *) effective_unit_name, NULL); if (processor_instance == NULL) { flb_error("[processor] error creating processor '%s': plugin doesn't exist or failed to initialize", unit_name); diff --git a/src/flb_reload.c b/src/flb_reload.c index 76d38abac68..9f44034e2c4 100644 --- a/src/flb_reload.c +++ b/src/flb_reload.c @@ -595,6 +595,19 @@ int flb_reload(flb_ctx_t *ctx, struct flb_cf *cf_opts) return FLB_RELOAD_HALTED; } + if (new_config->fips_mode != old_config->fips_mode) { + if (file != NULL) { + flb_sds_destroy(file); + } + flb_cf_destroy(new_cf); + flb_destroy(new_ctx); + old_config->hot_reloading = FLB_FALSE; + + flb_error("[reload] security.fips_mode cannot be changed by hot reload"); + flb_reload_watchdog_cleanup(watchdog_ctx); + return FLB_RELOAD_HALTED; + } + /* Validate plugin properites before fluent-bit stops the old context. */ ret = flb_reload_property_check_all(new_config); if (ret != 0) { diff --git a/src/flb_time.c b/src/flb_time.c index 6a1c94e0287..6e35c0dbb99 100644 --- a/src/flb_time.c +++ b/src/flb_time.c @@ -184,6 +184,10 @@ int flb_time_append_to_mpack(mpack_writer_t *writer, struct flb_time *tm, int fm /* We can't set with msgpack-c !! */ /* see pack_template.h and msgpack_pack_inline_func(_ext) */ case FLB_TIME_ETFMT_V1_FIXEXT: + if (flb_time_is_valid_eventtime(tm) != FLB_TRUE) { + return -1; + } + tmp = htonl((uint32_t)tm->tm.tv_sec); /* second from epoch */ memcpy(&ext_data, &tmp, 4); tmp = htonl((uint32_t)tm->tm.tv_nsec);/* nanosecond */ @@ -235,6 +239,10 @@ int flb_time_append_to_msgpack(struct flb_time *tm, msgpack_packer *pk, int fmt) /* We can't set with msgpack-c !! */ /* see pack_template.h and msgpack_pack_inline_func(_ext) */ case FLB_TIME_ETFMT_V1_FIXEXT: + if (flb_time_is_valid_eventtime(tm) != FLB_TRUE) { + return -1; + } + tmp = htonl((uint32_t)tm->tm.tv_sec); /* second from epoch */ memcpy(&ext_data, &tmp, 4); tmp = htonl((uint32_t)tm->tm.tv_nsec);/* nanosecond */ @@ -283,6 +291,10 @@ int flb_time_msgpack_to_time(struct flb_time *time, msgpack_object *obj) time->tm.tv_sec = (uint32_t) ntohl(tmp); memcpy(&tmp, &obj->via.ext.ptr[4], 4); time->tm.tv_nsec = (uint32_t) ntohl(tmp); + if (flb_time_is_valid_eventtime(time) != FLB_TRUE) { + flb_warn("[time] invalid EventTime value"); + return -1; + } break; default: flb_warn("unknown time format %x", obj->type); @@ -376,6 +388,10 @@ int flb_time_pop_from_mpack(struct flb_time *time, mpack_reader_t *reader) time->tm.tv_sec = (uint32_t) ntohl(tmp); memcpy(&tmp, extbuf + 4, 4); time->tm.tv_nsec = (uint32_t) ntohl(tmp); + if (flb_time_is_valid_eventtime(time) != FLB_TRUE) { + flb_warn("invalid EventTime value"); + return -1; + } break; default: flb_warn("unknown time format %d", tag.type); diff --git a/src/flb_upstream.c b/src/flb_upstream.c index 89a0fae281f..62decd65ec1 100644 --- a/src/flb_upstream.c +++ b/src/flb_upstream.c @@ -292,6 +292,7 @@ struct flb_upstream *flb_upstream_create(struct flb_config *config, struct flb_tls *tls) { int ret; + int transport; char *proxy_protocol = NULL; char *proxy_host = NULL; char *proxy_port = NULL; @@ -307,16 +308,22 @@ struct flb_upstream *flb_upstream_create(struct flb_config *config, u->base.dynamically_allocated = FLB_TRUE; + transport = FLB_TRANSPORT_TCP; + if ((flags & FLB_IO_UDP) || (flags & FLB_IO_DTLS)) { + transport = FLB_TRANSPORT_UDP; + } + flb_stream_setup(&u->base, FLB_UPSTREAM, - FLB_TRANSPORT_TCP, + transport, flags, tls, config, NULL); /* Set upstream to the http_proxy if it is specified. */ - if (flb_upstream_needs_proxy(host, config->http_proxy, config->no_proxy) == FLB_TRUE) { + if (transport == FLB_TRANSPORT_TCP && + flb_upstream_needs_proxy(host, config->http_proxy, config->no_proxy) == FLB_TRUE) { flb_debug("[upstream] config->http_proxy: %s", config->http_proxy); ret = flb_utils_proxy_url_split(config->http_proxy, &proxy_protocol, &proxy_username, &proxy_password, diff --git a/src/fluent-bit.c b/src/fluent-bit.c index 14e5f5682a4..b74d68313f9 100644 --- a/src/fluent-bit.c +++ b/src/fluent-bit.c @@ -60,6 +60,7 @@ #include #include #include +#include #ifdef FLB_HAVE_MTRACE #include @@ -92,6 +93,7 @@ struct flb_stacktrace flb_st; #endif #define FLB_LONG_SUPERVISOR (1024 + 5) +#define FLB_LONG_ENABLE_FIPS (1024 + 6) #define FLB_HELP_TEXT 0 #define FLB_HELP_JSON 1 @@ -173,6 +175,7 @@ static void flb_help(int rc, struct flb_config *config) print_opt("-q, --quiet", "quiet mode"); print_opt("-S, --sosreport", "support report for Enterprise customers"); print_opt("-Y, --enable-hot-reload", "enable for hot reloading"); + print_opt(" --enable-fips", "require OpenSSL FIPS mode at startup"); print_opt("-W, --disable-thread-safety-on-hot-reloading", "disable thread safety on hot reloading"); print_opt("-V, --version", "show version number"); print_opt("-h, --help", "print this help"); @@ -1079,6 +1082,7 @@ static int flb_main_run(int argc, char **argv) { "http_port", required_argument, NULL, 'P' }, #endif { "enable-hot-reload", no_argument, NULL, 'Y' }, + { "enable-fips", no_argument, NULL, FLB_LONG_ENABLE_FIPS }, #ifdef FLB_SYSTEM_WINDOWS { "windows_maxstdio", required_argument, NULL, 'M' }, #endif @@ -1300,6 +1304,10 @@ static int flb_main_run(int argc, char **argv) case 'Y': flb_cf_section_property_add(cf_opts, service->properties, FLB_CONF_STR_HOT_RELOAD, 0, "on", 0); break; + case FLB_LONG_ENABLE_FIPS: + flb_cf_section_property_add(cf_opts, service->properties, + FLB_CONF_STR_FIPS_MODE, 0, "on", 0); + break; case 'W': flb_cf_section_property_add(cf_opts, service->properties, FLB_CONF_STR_HOT_RELOAD_ENSURE_THREAD_SAFETY, 0, "off", 0); @@ -1412,6 +1420,12 @@ static int flb_main_run(int argc, char **argv) flb_utils_error(FLB_ERR_CFG_FLUSH); } + if (flb_fips_init(config) != 0) { + flb_cf_destroy(cf_opts); + flb_destroy(ctx); + return -1; + } + /* debug or trace */ if (config->verbose >= FLB_LOG_DEBUG) { flb_utils_print_setup(config); diff --git a/src/http_server/flb_http_server.c b/src/http_server/flb_http_server.c index aa6c5c6e1a0..d46976462cc 100644 --- a/src/http_server/flb_http_server.c +++ b/src/http_server/flb_http_server.c @@ -22,6 +22,7 @@ #include #include #include +#include #include #include @@ -34,70 +35,11 @@ /* PRIVATE */ struct flb_http_server_worker_context { - struct flb_http_server parent; struct flb_http_server server; struct flb_net_setup net_setup; - struct mk_event_loop *event_loop; - pthread_t thread; - pthread_mutex_t mutex; - pthread_cond_t condition; - int worker_id; - uint64_t should_exit; - int initialized; - int thread_created; - int startup_result; }; -struct flb_http_server_runtime { - struct flb_http_server_worker_context *workers; - int worker_count; -}; - -const struct flb_net_setup * -flb_http_server_runtime_worker_net_setup_get(struct flb_http_server *server, - int worker_id) -{ - if (server == NULL || - server->runtime == NULL || - worker_id < 0 || - worker_id >= server->runtime->worker_count) { - return NULL; - } - - return &server->runtime->workers[worker_id].net_setup; -} - -static void flb_http_server_runtime_stop(struct flb_http_server *session, - struct flb_http_server_runtime *runtime); - -static int flb_http_server_worker_context_reset( - struct flb_http_server_worker_context *worker) -{ - int result; - - memset(worker, 0, sizeof(struct flb_http_server_worker_context)); - - result = pthread_mutex_init(&worker->mutex, NULL); - if (result != 0) { - return result; - } - - result = pthread_cond_init(&worker->condition, NULL); - if (result != 0) { - pthread_mutex_destroy(&worker->mutex); - return result; - } - - return 0; -} - -static void flb_http_server_worker_context_cleanup( - struct flb_http_server_worker_context *worker) -{ - pthread_mutex_destroy(&worker->mutex); - pthread_cond_destroy(&worker->condition); -} - +static void flb_http_server_runtime_stop(struct flb_http_server *session); static int flb_http_server_running_on_caller_context( struct flb_http_server *session) { @@ -167,9 +109,50 @@ static void flb_http_server_reap_stale_sessions(struct flb_http_server *server) } } -static size_t flb_http_server_client_count(struct flb_http_server *server) +static int flb_http_server_connection_slot_reserve(struct flb_http_server *server) { - return cfl_list_size(&server->clients); + uint64_t connection_count; + + if (server->max_connections == 0) { + return FLB_TRUE; + } + + while (FLB_TRUE) { + connection_count = cfl_atomic_load(server->connection_counter); + + if (connection_count >= server->max_connections) { + return FLB_FALSE; + } + + if (cfl_atomic_compare_exchange(server->connection_counter, + connection_count, + connection_count + 1)) { + return FLB_TRUE; + } + } +} + +static void flb_http_server_connection_slot_release(struct flb_http_server *server) +{ + uint64_t connection_count; + + if (server->max_connections == 0) { + return; + } + + while (FLB_TRUE) { + connection_count = cfl_atomic_load(server->connection_counter); + + if (connection_count == 0) { + return; + } + + if (cfl_atomic_compare_exchange(server->connection_counter, + connection_count, + connection_count - 1)) { + return; + } + } } static int flb_http_server_apply_options(struct flb_http_server *session, @@ -198,6 +181,13 @@ static int flb_http_server_apply_options(struct flb_http_server *session, session->buffer_max_size = options->buffer_max_size; session->buffer_chunk_size = options->buffer_chunk_size; session->max_connections = options->max_connections; + cfl_atomic_store(&session->active_connections, 0); + if (options->connection_counter != NULL) { + session->connection_counter = options->connection_counter; + } + else { + session->connection_counter = &session->active_connections; + } session->workers = options->workers; session->worker_id = 0; session->use_caller_event_loop = options->use_caller_event_loop; @@ -247,6 +237,10 @@ static int flb_http_server_session_read(struct flb_http_server_session *session) (void *) session->read_buffer, session->read_buffer_size); + if (flb_io_net_is_retry(result)) { + return 0; + } + if (result <= 0) { return -1; } @@ -489,17 +483,18 @@ static int flb_http_server_client_connection_event_handler(void *data) if (server->max_connections > 0) { flb_http_server_reap_stale_sessions(server); + } - if (flb_http_server_client_count(server) >= server->max_connections) { - flb_downstream_conn_release(connection); + if (!flb_http_server_connection_slot_reserve(server)) { + flb_downstream_conn_release(connection); - return -5; - } + return -5; } session = flb_http_server_session_create(server->protocol_version); if (session == NULL) { + flb_http_server_connection_slot_release(server); flb_downstream_conn_release(connection); return -2; @@ -507,6 +502,7 @@ static int flb_http_server_client_connection_event_handler(void *data) session->parent = server; session->connection = connection; + session->connection_slot_reserved = FLB_TRUE; if (session->version <= HTTP_PROTOCOL_VERSION_11) { session->http1.stream.user_data = server->user_data; @@ -544,148 +540,131 @@ static int flb_http_server_client_connection_event_handler(void *data) return 0; } -static void flb_http_server_worker_maintenance(struct flb_config *config, - void *data) +static void flb_http_server_worker_maintenance(struct flb_downstream_worker *worker, + void *worker_context) { - struct flb_http_server_worker_context *worker; + struct flb_http_server_worker_context *context; - (void) config; + (void) worker; - worker = data; + context = worker_context; - if (worker->server.downstream != NULL) { - flb_downstream_conn_timeouts_stream(worker->server.downstream); + if (context->server.downstream != NULL) { + flb_downstream_conn_timeouts_stream(context->server.downstream); + flb_downstream_conn_pending_destroy(context->server.downstream); } - flb_http_server_reap_stale_sessions(&worker->server); + flb_http_server_reap_stale_sessions(&context->server); } -static int flb_http_server_worker_initialize( - struct flb_http_server_worker_context *worker) +static int flb_http_server_worker_initialize(struct flb_downstream_worker *worker, + struct flb_http_server *parent, + struct flb_http_server_worker_context *context) { int result; struct flb_http_server_options options; + memcpy(&context->net_setup, + parent->networking_setup, + sizeof(struct flb_net_setup)); + context->net_setup.share_port = parent->reuse_port; + flb_http_server_options_init(&options); - options.protocol_version = worker->parent.protocol_version; - options.flags = worker->parent.flags; - options.request_callback = worker->parent.request_callback; - options.user_data = worker->parent.user_data; - options.address = worker->parent.address; - options.port = worker->parent.port; - options.tls_provider = worker->parent.tls_provider; - options.networking_flags = worker->parent.networking_flags; - options.networking_setup = &worker->net_setup; - options.event_loop = worker->event_loop; - options.system_context = worker->parent.system_context; - options.idle_timeout = worker->parent.idle_timeout; - options.buffer_max_size = worker->parent.buffer_max_size; + options.protocol_version = parent->protocol_version; + options.flags = parent->flags; + options.request_callback = parent->request_callback; + options.user_data = parent->user_data; + options.address = parent->address; + options.port = parent->port; + options.tls_provider = parent->tls_provider; + options.networking_flags = parent->networking_flags; + options.networking_setup = &context->net_setup; + options.event_loop = flb_downstream_worker_event_loop_get(worker); + options.system_context = parent->system_context; + options.idle_timeout = parent->idle_timeout; + options.buffer_max_size = parent->buffer_max_size; + options.buffer_chunk_size = parent->buffer_chunk_size; + options.max_connections = parent->max_connections; + options.connection_counter = parent->connection_counter; options.workers = 1; options.use_caller_event_loop = FLB_TRUE; - options.reuse_port = worker->parent.reuse_port; - options.cb_worker_init = worker->parent.cb_worker_init; - options.cb_worker_exit = worker->parent.cb_worker_exit; + options.reuse_port = parent->reuse_port; + options.cb_worker_init = parent->cb_worker_init; + options.cb_worker_exit = parent->cb_worker_exit; - result = flb_http_server_init_with_options(&worker->server, &options); + result = flb_http_server_init_with_options(&context->server, &options); if (result != 0) { return result; } - result = flb_http_server_start(&worker->server); + result = flb_http_server_start(&context->server); if (result != 0) { + flb_http_server_destroy(&context->server); return result; } - flb_downstream_thread_safe(worker->server.downstream); + flb_downstream_thread_safe(context->server.downstream); - worker->server.worker_id = worker->worker_id; - worker->server.workers = worker->parent.workers; + context->server.worker_id = flb_downstream_worker_id_get(worker); + context->server.workers = parent->workers; return 0; } -static void *flb_http_server_worker_thread(void *data) +static int flb_http_server_worker_init(struct flb_downstream_worker *worker, + void *parent, + void **worker_context) { - int result; - uint64_t should_exit; - struct mk_event *event; - struct flb_net_dns dns_ctx = {0}; - struct flb_http_server_worker_context *worker; + int ret; + struct flb_http_server *parent_server; + struct flb_http_server_worker_context *context; - worker = data; + parent_server = parent; - worker->event_loop = mk_event_loop_create(256); - if (worker->event_loop == NULL) { - result = -1; - goto signal_and_exit; + context = flb_calloc(1, sizeof(struct flb_http_server_worker_context)); + if (context == NULL) { + flb_errno(); + return -1; } - flb_engine_evl_init(); - flb_engine_evl_set(worker->event_loop); - - flb_net_dns_ctx_init(); - flb_net_ctx_init(&dns_ctx); - flb_net_dns_ctx_set(&dns_ctx); - - result = flb_http_server_worker_initialize(worker); - -signal_and_exit: - pthread_mutex_lock(&worker->mutex); - worker->startup_result = result; - worker->initialized = FLB_TRUE; - pthread_cond_signal(&worker->condition); - pthread_mutex_unlock(&worker->mutex); + ret = flb_http_server_worker_initialize(worker, parent_server, context); + if (ret != 0) { + flb_free(context); + return ret; + } - if (result != 0) { - goto cleanup; + ret = flb_downstream_worker_listener_fd_set( + worker, context->server.downstream->server_fd); + if (ret != 0) { + flb_http_server_destroy(&context->server); + flb_free(context); + return ret; } - while ((should_exit = cfl_atomic_load(&worker->should_exit)) == FLB_FALSE) { - mk_event_wait_2(worker->event_loop, 250); + *worker_context = context; - mk_event_foreach(event, worker->event_loop) { - if (event->type == FLB_ENGINE_EV_CUSTOM) { - event->handler(event); - } - } + return 0; +} - flb_http_server_worker_maintenance(worker->parent.system_context, - worker); - flb_downstream_conn_pending_destroy(worker->server.downstream); - } +static void flb_http_server_worker_exit(struct flb_downstream_worker *worker, + void *worker_context) +{ + struct flb_http_server_worker_context *context; -cleanup: - if (worker->server.status == HTTP_SERVER_RUNNING && - worker->server.cb_worker_exit != NULL) { - worker->server.cb_worker_exit(&worker->server, - worker->server.user_data); - worker->server.cb_worker_exit = NULL; - } + (void) worker; - return NULL; + context = worker_context; + + flb_http_server_destroy(&context->server); + flb_free(context); } static int flb_http_server_runtime_start(struct flb_http_server *session) { const char *alpn; - int index; int result; - struct flb_http_server_runtime *runtime; - - runtime = flb_calloc(1, sizeof(struct flb_http_server_runtime)); - if (runtime == NULL) { - flb_errno(); - return -1; - } - - runtime->workers = flb_calloc(session->workers, - sizeof(struct flb_http_server_worker_context)); - if (runtime->workers == NULL) { - flb_errno(); - flb_free(runtime); - return -1; - } + struct flb_downstream_worker_options options; if (session->tls_provider != NULL && session->tls_alpn_configured == FLB_FALSE) { @@ -693,115 +672,33 @@ static int flb_http_server_runtime_start(struct flb_http_server *session) result = flb_tls_set_alpn(session->tls_provider, alpn); if (result != 0) { - flb_free(runtime->workers); - flb_free(runtime); - return -1; } session->tls_alpn_configured = FLB_TRUE; } - for (index = 0; index < session->workers; index++) { - result = flb_http_server_worker_context_reset(&runtime->workers[index]); - if (result != 0) { - while (index > 0) { - index--; - flb_http_server_worker_context_cleanup(&runtime->workers[index]); - } - - flb_free(runtime->workers); - flb_free(runtime); - - return -1; - } - } - - runtime->worker_count = session->workers; + memset(&options, 0, sizeof(struct flb_downstream_worker_options)); + options.workers = session->workers; + options.parent = session; + options.cb_init = flb_http_server_worker_init; + options.cb_exit = flb_http_server_worker_exit; + options.cb_maintenance = flb_http_server_worker_maintenance; - for (index = 0; index < runtime->worker_count; index++) { - memcpy(&runtime->workers[index].parent, - session, - sizeof(struct flb_http_server)); - memcpy(&runtime->workers[index].net_setup, - session->networking_setup, - sizeof(struct flb_net_setup)); - - runtime->workers[index].net_setup.share_port = FLB_TRUE; - runtime->workers[index].worker_id = index; - runtime->workers[index].parent.reuse_port = FLB_TRUE; - runtime->workers[index].parent.runtime = NULL; - runtime->workers[index].parent.workers = session->workers; - - result = pthread_create(&runtime->workers[index].thread, - NULL, - flb_http_server_worker_thread, - &runtime->workers[index]); - if (result != 0) { - runtime->workers[index].startup_result = -1; - break; - } - runtime->workers[index].thread_created = FLB_TRUE; - - pthread_mutex_lock(&runtime->workers[index].mutex); - while (runtime->workers[index].initialized == FLB_FALSE) { - pthread_cond_wait(&runtime->workers[index].condition, - &runtime->workers[index].mutex); - } - result = runtime->workers[index].startup_result; - pthread_mutex_unlock(&runtime->workers[index].mutex); - - if (result != 0) { - break; - } - } - - if (index != runtime->worker_count) { - flb_http_server_runtime_stop(session, runtime); + result = flb_downstream_worker_runtime_start(&session->runtime, &options); + if (result != 0) { return -1; } - session->runtime = runtime; session->status = HTTP_SERVER_RUNNING; return 0; } -static void flb_http_server_runtime_stop(struct flb_http_server *session, - struct flb_http_server_runtime *runtime) +static void flb_http_server_runtime_stop(struct flb_http_server *session) { - int index; - int published; - - if (runtime == NULL) { - return; - } - - published = (session->runtime == runtime); - - for (index = 0; index < runtime->worker_count; index++) { - cfl_atomic_store(&runtime->workers[index].should_exit, FLB_TRUE); - - if (runtime->workers[index].thread_created == FLB_TRUE) { - pthread_join(runtime->workers[index].thread, NULL); - - flb_http_server_destroy(&runtime->workers[index].server); - - if (runtime->workers[index].event_loop != NULL) { - mk_event_loop_destroy(runtime->workers[index].event_loop); - runtime->workers[index].event_loop = NULL; - } - } - - flb_http_server_worker_context_cleanup(&runtime->workers[index]); - } - - if (published == FLB_TRUE) { - session->runtime = NULL; - } - - flb_free(runtime->workers); - flb_free(runtime); + flb_downstream_worker_runtime_stop(session->runtime); + session->runtime = NULL; } /* HTTP SERVER */ @@ -1069,7 +966,7 @@ int flb_http_server_stop(struct flb_http_server *server) struct flb_http_server_session *session; if (server->runtime != NULL) { - flb_http_server_runtime_stop(server, server->runtime); + flb_http_server_runtime_stop(server); server->status = HTTP_SERVER_STOPPED; return 0; } @@ -1261,6 +1158,11 @@ void flb_http_server_session_destroy(struct flb_http_server_session *session) cfl_list_del(&session->_head); } + if (session->connection_slot_reserved) { + flb_http_server_connection_slot_release(session->parent); + session->connection_slot_reserved = FLB_FALSE; + } + if (session->incoming_data != NULL) { cfl_sds_destroy(session->incoming_data); } diff --git a/src/opentelemetry/flb_opentelemetry_logs.c b/src/opentelemetry/flb_opentelemetry_logs.c index d2a876d534a..24e4a5e6dcb 100644 --- a/src/opentelemetry/flb_opentelemetry_logs.c +++ b/src/opentelemetry/flb_opentelemetry_logs.c @@ -119,7 +119,12 @@ static int process_json_payload_log_records_entry( return -FLB_OTEL_LOGS_ERR_UNEXPECTED_TIMESTAMP_TYPE; } - flb_time_from_uint64(×tamp, timestamp_uint64); + if (flb_time_from_uint64(×tamp, timestamp_uint64) != 0) { + if (error_status) { + *error_status = FLB_OTEL_LOGS_ERR_UNEXPECTED_TIMESTAMP_TYPE; + } + return -FLB_OTEL_LOGS_ERR_UNEXPECTED_TIMESTAMP_TYPE; + } } /* observedTimeUnixNano (only camelCase) */ diff --git a/src/opentelemetry/flb_opentelemetry_otlp_proto.c b/src/opentelemetry/flb_opentelemetry_otlp_proto.c index 93cdbd16960..cbdbca01194 100644 --- a/src/opentelemetry/flb_opentelemetry_otlp_proto.c +++ b/src/opentelemetry/flb_opentelemetry_otlp_proto.c @@ -30,6 +30,7 @@ #include #include +#include #include #include @@ -43,6 +44,17 @@ #define FLB_OTEL_LOGS_SCHEMA_KEY "schema" #define FLB_OTEL_LOGS_SCHEMA_OTLP "otlp" #define FLB_OTEL_LOGS_METADATA_KEY "otlp" +#define FLB_OTEL_PROTO_ARENA_INITIAL_CHUNK_SIZE 4096 +#define FLB_OTEL_PROTO_ARENA_MAX_CHUNK_SIZE 65536 + +struct otlp_proto_arena { + struct cfl_arena *backend; +}; + +/* + * Protobuf objects live until the request has been packed. Pointer arrays that + * grow with realloc remain heap-backed and are released separately. + */ struct otlp_proto_logs_scope_state { int64_t scope_id; @@ -61,6 +73,63 @@ struct otlp_proto_logs_resource_state { static msgpack_object *msgpack_map_get_object(msgpack_object_map *map, const char *key); +static void *otlp_proto_arena_malloc(void *context, size_t size) +{ + (void) context; + + return flb_malloc(size); +} + +static void otlp_proto_arena_free(void *context, void *pointer) +{ + (void) context; + + flb_free(pointer); +} + +static int otlp_proto_arena_init(struct otlp_proto_arena *arena) +{ + struct cfl_arena_options options; + + cfl_arena_options_init(&options); + options.chunk_size = FLB_OTEL_PROTO_ARENA_INITIAL_CHUNK_SIZE; + options.maximum_chunk_size = FLB_OTEL_PROTO_ARENA_MAX_CHUNK_SIZE; + options.malloc_fn = otlp_proto_arena_malloc; + options.free_fn = otlp_proto_arena_free; + + arena->backend = cfl_arena_create_with_options(&options); + if (arena->backend == NULL) { + return -1; + } + + return 0; +} + +static void otlp_proto_arena_destroy(struct otlp_proto_arena *arena) +{ + cfl_arena_destroy(arena->backend); + arena->backend = NULL; +} + +static void *otlp_proto_arena_alloc(struct otlp_proto_arena *arena, size_t size) +{ + return cfl_arena_malloc(arena->backend, size); +} + +static void *otlp_proto_arena_calloc(struct otlp_proto_arena *arena, + size_t count, + size_t size) +{ + return cfl_arena_calloc(arena->backend, count, size); +} + +static char *otlp_proto_arena_strndup(struct otlp_proto_arena *arena, + const char *input, + size_t length) +{ + return cfl_arena_strndup(arena->backend, input, length); +} + static void set_result(int *result, int value) { if (result != NULL) { @@ -249,15 +318,13 @@ static int msgpack_map_get_int64(msgpack_object_map *map, return -1; } -static void otlp_kvpair_destroy(Opentelemetry__Proto__Common__V1__KeyValue *kvpair); -static void otlp_any_value_destroy(Opentelemetry__Proto__Common__V1__AnyValue *value); -static void destroy_log_record(Opentelemetry__Proto__Logs__V1__LogRecord *record); - -static Opentelemetry__Proto__Common__V1__ArrayValue *otlp_array_value_initialize(size_t entry_count) +static Opentelemetry__Proto__Common__V1__ArrayValue *otlp_array_value_initialize( + struct otlp_proto_arena *arena, size_t entry_count) { Opentelemetry__Proto__Common__V1__ArrayValue *value; - value = flb_calloc(1, sizeof(Opentelemetry__Proto__Common__V1__ArrayValue)); + value = otlp_proto_arena_calloc(arena, 1, + sizeof(Opentelemetry__Proto__Common__V1__ArrayValue)); if (value == NULL) { return NULL; } @@ -265,10 +332,9 @@ static Opentelemetry__Proto__Common__V1__ArrayValue *otlp_array_value_initialize opentelemetry__proto__common__v1__array_value__init(value); if (entry_count > 0) { - value->values = flb_calloc(entry_count, - sizeof(Opentelemetry__Proto__Common__V1__AnyValue *)); + value->values = otlp_proto_arena_calloc(arena, entry_count, + sizeof(Opentelemetry__Proto__Common__V1__AnyValue *)); if (value->values == NULL) { - flb_free(value); return NULL; } @@ -278,11 +344,13 @@ static Opentelemetry__Proto__Common__V1__ArrayValue *otlp_array_value_initialize return value; } -static Opentelemetry__Proto__Common__V1__KeyValue *otlp_kvpair_value_initialize() +static Opentelemetry__Proto__Common__V1__KeyValue *otlp_kvpair_value_initialize( + struct otlp_proto_arena *arena) { Opentelemetry__Proto__Common__V1__KeyValue *value; - value = flb_calloc(1, sizeof(Opentelemetry__Proto__Common__V1__KeyValue)); + value = otlp_proto_arena_calloc(arena, 1, + sizeof(Opentelemetry__Proto__Common__V1__KeyValue)); if (value != NULL) { opentelemetry__proto__common__v1__key_value__init(value); } @@ -290,11 +358,13 @@ static Opentelemetry__Proto__Common__V1__KeyValue *otlp_kvpair_value_initialize( return value; } -static Opentelemetry__Proto__Common__V1__KeyValueList *otlp_kvlist_value_initialize(size_t entry_count) +static Opentelemetry__Proto__Common__V1__KeyValueList *otlp_kvlist_value_initialize( + struct otlp_proto_arena *arena, size_t entry_count) { Opentelemetry__Proto__Common__V1__KeyValueList *value; - value = flb_calloc(1, sizeof(Opentelemetry__Proto__Common__V1__KeyValueList)); + value = otlp_proto_arena_calloc(arena, 1, + sizeof(Opentelemetry__Proto__Common__V1__KeyValueList)); if (value == NULL) { return NULL; } @@ -302,10 +372,9 @@ static Opentelemetry__Proto__Common__V1__KeyValueList *otlp_kvlist_value_initial opentelemetry__proto__common__v1__key_value_list__init(value); if (entry_count > 0) { - value->values = flb_calloc(entry_count, - sizeof(Opentelemetry__Proto__Common__V1__KeyValue *)); + value->values = otlp_proto_arena_calloc(arena, entry_count, + sizeof(Opentelemetry__Proto__Common__V1__KeyValue *)); if (value->values == NULL) { - flb_free(value); return NULL; } @@ -315,12 +384,13 @@ static Opentelemetry__Proto__Common__V1__KeyValueList *otlp_kvlist_value_initial return value; } -static Opentelemetry__Proto__Common__V1__AnyValue *otlp_any_value_initialize(int data_type, - size_t entry_count) +static Opentelemetry__Proto__Common__V1__AnyValue *otlp_any_value_initialize( + struct otlp_proto_arena *arena, int data_type, size_t entry_count) { Opentelemetry__Proto__Common__V1__AnyValue *value; - value = flb_calloc(1, sizeof(Opentelemetry__Proto__Common__V1__AnyValue)); + value = otlp_proto_arena_calloc(arena, 1, + sizeof(Opentelemetry__Proto__Common__V1__AnyValue)); if (value == NULL) { return NULL; } @@ -352,20 +422,18 @@ static Opentelemetry__Proto__Common__V1__AnyValue *otlp_any_value_initialize(int else if (data_type == MSGPACK_OBJECT_ARRAY) { value->value_case = OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_ARRAY_VALUE; - value->array_value = otlp_array_value_initialize(entry_count); + value->array_value = otlp_array_value_initialize(arena, entry_count); if (value->array_value == NULL) { - flb_free(value); return NULL; } } else if (data_type == MSGPACK_OBJECT_MAP) { value->value_case = OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_KVLIST_VALUE; - value->kvlist_value = otlp_kvlist_value_initialize(entry_count); + value->kvlist_value = otlp_kvlist_value_initialize(arena, entry_count); if (value->kvlist_value == NULL) { - flb_free(value); return NULL; } } @@ -374,120 +442,32 @@ static Opentelemetry__Proto__Common__V1__AnyValue *otlp_any_value_initialize(int OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_BYTES_VALUE; } else { - flb_free(value); return NULL; } return value; } -static void otlp_kvarray_destroy(Opentelemetry__Proto__Common__V1__KeyValue **kvarray, - size_t entry_count) -{ - size_t index; - - if (kvarray == NULL) { - return; - } - - for (index = 0; index < entry_count; index++) { - if (kvarray[index] != NULL) { - otlp_kvpair_destroy(kvarray[index]); - } - } - - flb_free(kvarray); -} - -static void otlp_kvlist_destroy(Opentelemetry__Proto__Common__V1__KeyValueList *kvlist) -{ - size_t index; - - if (kvlist == NULL) { - return; - } - - for (index = 0; index < kvlist->n_values; index++) { - otlp_kvpair_destroy(kvlist->values[index]); - } - - flb_free(kvlist->values); - flb_free(kvlist); -} - -static void otlp_array_destroy(Opentelemetry__Proto__Common__V1__ArrayValue *array) -{ - size_t index; - - if (array == NULL) { - return; - } - - for (index = 0; index < array->n_values; index++) { - otlp_any_value_destroy(array->values[index]); - } - - flb_free(array->values); - flb_free(array); -} - -static void otlp_any_value_destroy(Opentelemetry__Proto__Common__V1__AnyValue *value) -{ - if (value == NULL) { - return; - } - - if (value->value_case == - OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_STRING_VALUE) { - flb_free(value->string_value); - } - else if (value->value_case == - OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_ARRAY_VALUE) { - otlp_array_destroy(value->array_value); - } - else if (value->value_case == - OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_KVLIST_VALUE) { - otlp_kvlist_destroy(value->kvlist_value); - } - else if (value->value_case == - OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_BYTES_VALUE) { - flb_free(value->bytes_value.data); - } - - flb_free(value); -} - -static void otlp_kvpair_destroy(Opentelemetry__Proto__Common__V1__KeyValue *kvpair) -{ - if (kvpair == NULL) { - return; - } - - flb_free(kvpair->key); - otlp_any_value_destroy(kvpair->value); - flb_free(kvpair); -} - static Opentelemetry__Proto__Common__V1__AnyValue *msgpack_object_to_otlp_any_value( - msgpack_object *object); + struct otlp_proto_arena *arena, msgpack_object *object); static Opentelemetry__Proto__Common__V1__AnyValue *msgpack_array_to_otlp_any_value( - msgpack_object *object) + struct otlp_proto_arena *arena, msgpack_object *object) { size_t index; Opentelemetry__Proto__Common__V1__AnyValue *entry; Opentelemetry__Proto__Common__V1__AnyValue *value; - value = otlp_any_value_initialize(MSGPACK_OBJECT_ARRAY, + value = otlp_any_value_initialize(arena, MSGPACK_OBJECT_ARRAY, object->via.array.size); if (value == NULL) { return NULL; } for (index = 0; index < object->via.array.size; index++) { - entry = msgpack_object_to_otlp_any_value(&object->via.array.ptr[index]); + entry = msgpack_object_to_otlp_any_value(arena, + &object->via.array.ptr[index]); if (entry == NULL) { - otlp_any_value_destroy(value); return NULL; } @@ -498,30 +478,28 @@ static Opentelemetry__Proto__Common__V1__AnyValue *msgpack_array_to_otlp_any_val } static Opentelemetry__Proto__Common__V1__KeyValue *msgpack_kv_to_otlp_any_value( - struct msgpack_object_kv *input_pair) + struct otlp_proto_arena *arena, struct msgpack_object_kv *input_pair) { Opentelemetry__Proto__Common__V1__KeyValue *kv; - kv = otlp_kvpair_value_initialize(); + kv = otlp_kvpair_value_initialize(arena); if (kv == NULL) { return NULL; } if (input_pair->key.type != MSGPACK_OBJECT_STR) { - flb_free(kv); return NULL; } - kv->key = flb_strndup(input_pair->key.via.str.ptr, input_pair->key.via.str.size); + kv->key = otlp_proto_arena_strndup(arena, + input_pair->key.via.str.ptr, + input_pair->key.via.str.size); if (kv->key == NULL) { - flb_free(kv); return NULL; } - kv->value = msgpack_object_to_otlp_any_value(&input_pair->val); + kv->value = msgpack_object_to_otlp_any_value(arena, &input_pair->val); if (kv->value == NULL) { - flb_free(kv->key); - flb_free(kv); return NULL; } @@ -529,7 +507,7 @@ static Opentelemetry__Proto__Common__V1__KeyValue *msgpack_kv_to_otlp_any_value( } static Opentelemetry__Proto__Common__V1__KeyValue **msgpack_map_to_otlp_kvarray( - msgpack_object *object, size_t *entry_count) + struct otlp_proto_arena *arena, msgpack_object *object, size_t *entry_count) { size_t index; Opentelemetry__Proto__Common__V1__KeyValue **result; @@ -540,17 +518,17 @@ static Opentelemetry__Proto__Common__V1__KeyValue **msgpack_map_to_otlp_kvarray( return NULL; } - result = flb_calloc(*entry_count, - sizeof(Opentelemetry__Proto__Common__V1__KeyValue *)); + result = otlp_proto_arena_calloc(arena, *entry_count, + sizeof(Opentelemetry__Proto__Common__V1__KeyValue *)); if (result == NULL) { *entry_count = 0; return NULL; } for (index = 0; index < *entry_count; index++) { - result[index] = msgpack_kv_to_otlp_any_value(&object->via.map.ptr[index]); + result[index] = msgpack_kv_to_otlp_any_value(arena, + &object->via.map.ptr[index]); if (result[index] == NULL) { - otlp_kvarray_destroy(result, index); *entry_count = 0; return NULL; } @@ -560,21 +538,21 @@ static Opentelemetry__Proto__Common__V1__KeyValue **msgpack_map_to_otlp_kvarray( } static Opentelemetry__Proto__Common__V1__AnyValue *msgpack_map_to_otlp_any_value( - msgpack_object *object) + struct otlp_proto_arena *arena, msgpack_object *object) { size_t index; Opentelemetry__Proto__Common__V1__KeyValue *entry; Opentelemetry__Proto__Common__V1__AnyValue *value; - value = otlp_any_value_initialize(MSGPACK_OBJECT_MAP, object->via.map.size); + value = otlp_any_value_initialize(arena, MSGPACK_OBJECT_MAP, + object->via.map.size); if (value == NULL) { return NULL; } for (index = 0; index < object->via.map.size; index++) { - entry = msgpack_kv_to_otlp_any_value(&object->via.map.ptr[index]); + entry = msgpack_kv_to_otlp_any_value(arena, &object->via.map.ptr[index]); if (entry == NULL) { - otlp_any_value_destroy(value); return NULL; } @@ -585,7 +563,7 @@ static Opentelemetry__Proto__Common__V1__AnyValue *msgpack_map_to_otlp_any_value } static Opentelemetry__Proto__Common__V1__AnyValue *msgpack_object_to_otlp_any_value( - msgpack_object *object) + struct otlp_proto_arena *arena, msgpack_object *object) { Opentelemetry__Proto__Common__V1__AnyValue *value; @@ -597,21 +575,20 @@ static Opentelemetry__Proto__Common__V1__AnyValue *msgpack_object_to_otlp_any_va switch (object->type) { case MSGPACK_OBJECT_NIL: - value = otlp_any_value_initialize(MSGPACK_OBJECT_NIL, 0); + value = otlp_any_value_initialize(arena, MSGPACK_OBJECT_NIL, 0); break; case MSGPACK_OBJECT_BOOLEAN: - value = otlp_any_value_initialize(MSGPACK_OBJECT_BOOLEAN, 0); + value = otlp_any_value_initialize(arena, MSGPACK_OBJECT_BOOLEAN, 0); if (value != NULL) { value->bool_value = object->via.boolean; } break; case MSGPACK_OBJECT_POSITIVE_INTEGER: case MSGPACK_OBJECT_NEGATIVE_INTEGER: - value = otlp_any_value_initialize(object->type, 0); + value = otlp_any_value_initialize(arena, object->type, 0); if (value != NULL) { if (object->type == MSGPACK_OBJECT_POSITIVE_INTEGER) { if (object->via.u64 > INT64_MAX) { - otlp_any_value_destroy(value); value = NULL; break; } @@ -625,43 +602,44 @@ static Opentelemetry__Proto__Common__V1__AnyValue *msgpack_object_to_otlp_any_va break; case MSGPACK_OBJECT_FLOAT32: case MSGPACK_OBJECT_FLOAT64: - value = otlp_any_value_initialize(object->type, 0); + value = otlp_any_value_initialize(arena, object->type, 0); if (value != NULL) { value->double_value = object->via.f64; } break; case MSGPACK_OBJECT_STR: - value = otlp_any_value_initialize(MSGPACK_OBJECT_STR, 0); + value = otlp_any_value_initialize(arena, MSGPACK_OBJECT_STR, 0); if (value != NULL) { - value->string_value = flb_strndup(object->via.str.ptr, - object->via.str.size); + value->string_value = otlp_proto_arena_strndup(arena, + object->via.str.ptr, + object->via.str.size); if (value->string_value == NULL) { - otlp_any_value_destroy(value); value = NULL; } } break; case MSGPACK_OBJECT_BIN: - value = otlp_any_value_initialize(MSGPACK_OBJECT_BIN, 0); + value = otlp_any_value_initialize(arena, MSGPACK_OBJECT_BIN, 0); if (value != NULL) { value->bytes_value.len = object->via.bin.size; - value->bytes_value.data = flb_malloc(object->via.bin.size); - if (value->bytes_value.data == NULL) { - otlp_any_value_destroy(value); - value = NULL; + if (object->via.bin.size == 0) { + value->bytes_value.data = cfl_arena_malloc(arena->backend, 1); } else { - memcpy(value->bytes_value.data, - object->via.bin.ptr, - object->via.bin.size); + value->bytes_value.data = cfl_arena_memdup(arena->backend, + object->via.bin.ptr, + object->via.bin.size); + } + if (value->bytes_value.data == NULL) { + value = NULL; } } break; case MSGPACK_OBJECT_ARRAY: - value = msgpack_array_to_otlp_any_value(object); + value = msgpack_array_to_otlp_any_value(arena, object); break; case MSGPACK_OBJECT_MAP: - value = msgpack_map_to_otlp_any_value(object); + value = msgpack_map_to_otlp_any_value(arena, object); break; default: break; @@ -759,11 +737,13 @@ static msgpack_object *find_log_body_candidate(msgpack_object *body, return body; } -static int append_kvarrays(Opentelemetry__Proto__Common__V1__KeyValue ***base, +static int append_kvarrays(struct otlp_proto_arena *arena, + Opentelemetry__Proto__Common__V1__KeyValue ***base, size_t *base_count, Opentelemetry__Proto__Common__V1__KeyValue **extra, size_t extra_count) { + size_t total_count; Opentelemetry__Proto__Common__V1__KeyValue **tmp; if (extra == NULL || extra_count == 0) { @@ -776,23 +756,34 @@ static int append_kvarrays(Opentelemetry__Proto__Common__V1__KeyValue ***base, return 0; } - tmp = flb_realloc(*base, - sizeof(Opentelemetry__Proto__Common__V1__KeyValue *) * - (*base_count + extra_count)); + if (*base_count > SIZE_MAX - extra_count) { + return -1; + } + + total_count = *base_count + extra_count; + if (total_count > + SIZE_MAX / sizeof(Opentelemetry__Proto__Common__V1__KeyValue *)) { + return -1; + } + + tmp = otlp_proto_arena_alloc(arena, + sizeof(Opentelemetry__Proto__Common__V1__KeyValue *) * total_count); if (tmp == NULL) { return -1; } - *base = tmp; - memcpy(*base + *base_count, extra, + memcpy(tmp, *base, + sizeof(Opentelemetry__Proto__Common__V1__KeyValue *) * *base_count); + memcpy(tmp + *base_count, extra, sizeof(Opentelemetry__Proto__Common__V1__KeyValue *) * extra_count); + *base = tmp; *base_count += extra_count; - flb_free(extra); return 0; } static int msgpack_map_to_otlp_kvarray_filtered( + struct otlp_proto_arena *arena, msgpack_object_map *map, const char *ignored_key, size_t ignored_key_length, @@ -801,10 +792,8 @@ static int msgpack_map_to_otlp_kvarray_filtered( { size_t index; size_t count; - Opentelemetry__Proto__Common__V1__KeyValue **tmp; Opentelemetry__Proto__Common__V1__KeyValue **values; - values = NULL; count = 0; for (index = 0; index < map->size; index++) { @@ -817,20 +806,36 @@ static int msgpack_map_to_otlp_kvarray_filtered( continue; } - tmp = flb_realloc(values, - sizeof(Opentelemetry__Proto__Common__V1__KeyValue *) * - (count + 1)); - if (tmp == NULL) { - otlp_kvarray_destroy(values, count); - *out_values = NULL; - *out_count = 0; - return -1; + count++; + } + + if (count == 0) { + *out_values = NULL; + *out_count = 0; + return 0; + } + + values = otlp_proto_arena_calloc(arena, count, + sizeof(Opentelemetry__Proto__Common__V1__KeyValue *)); + if (values == NULL) { + *out_values = NULL; + *out_count = 0; + return -1; + } + + count = 0; + for (index = 0; index < map->size; index++) { + if (ignored_key != NULL && + map->ptr[index].key.type == MSGPACK_OBJECT_STR && + map->ptr[index].key.via.str.size == ignored_key_length && + strncmp(map->ptr[index].key.via.str.ptr, + ignored_key, + ignored_key_length) == 0) { + continue; } - values = tmp; - values[count] = msgpack_kv_to_otlp_any_value(&map->ptr[index]); + values[count] = msgpack_kv_to_otlp_any_value(arena, &map->ptr[index]); if (values[count] == NULL) { - otlp_kvarray_destroy(values, count); *out_values = NULL; *out_count = 0; return -1; @@ -846,6 +851,7 @@ static int msgpack_map_to_otlp_kvarray_filtered( } static int log_record_set_body_and_attributes( + struct otlp_proto_arena *arena, Opentelemetry__Proto__Logs__V1__LogRecord *record, struct flb_log_event *event, const char **logs_body_keys, @@ -866,7 +872,7 @@ static int log_record_set_body_and_attributes( &matched_key, &matched_key_length); - record->body = msgpack_object_to_otlp_any_value(candidate); + record->body = msgpack_object_to_otlp_any_value(arena, candidate); if (candidate != NULL && record->body == NULL) { return -1; } @@ -875,7 +881,8 @@ static int log_record_set_body_and_attributes( matched_key != NULL && event->body != NULL && event->body->type == MSGPACK_OBJECT_MAP) { - if (msgpack_map_to_otlp_kvarray_filtered(&event->body->via.map, + if (msgpack_map_to_otlp_kvarray_filtered(arena, + &event->body->via.map, matched_key, matched_key_length, &attributes, @@ -883,11 +890,11 @@ static int log_record_set_body_and_attributes( return -1; } - if (append_kvarrays(&record->attributes, + if (append_kvarrays(arena, + &record->attributes, &record->n_attributes, attributes, attribute_count) != 0) { - otlp_kvarray_destroy(attributes, attribute_count); return -1; } } @@ -896,6 +903,7 @@ static int log_record_set_body_and_attributes( } static int add_msgpack_attributes_to_resource( + struct otlp_proto_arena *arena, Opentelemetry__Proto__Resource__V1__Resource *resource, msgpack_object *resource_object) { @@ -907,7 +915,7 @@ static int add_msgpack_attributes_to_resource( field = msgpack_map_get_object(&resource_object->via.map, "attributes"); if (field != NULL && field->type == MSGPACK_OBJECT_MAP) { - resource->attributes = msgpack_map_to_otlp_kvarray(field, + resource->attributes = msgpack_map_to_otlp_kvarray(arena, field, &resource->n_attributes); if (field->via.map.size > 0 && resource->attributes == NULL) { return -1; @@ -930,6 +938,7 @@ static int add_msgpack_attributes_to_resource( } static int add_msgpack_scope_fields( + struct otlp_proto_arena *arena, Opentelemetry__Proto__Common__V1__InstrumentationScope *scope, msgpack_object *scope_object) { @@ -941,7 +950,9 @@ static int add_msgpack_scope_fields( field = msgpack_map_get_object(&scope_object->via.map, "name"); if (field != NULL && field->type == MSGPACK_OBJECT_STR) { - scope->name = flb_strndup(field->via.str.ptr, field->via.str.size); + scope->name = otlp_proto_arena_strndup(arena, + field->via.str.ptr, + field->via.str.size); if (scope->name == NULL) { return -1; } @@ -949,7 +960,9 @@ static int add_msgpack_scope_fields( field = msgpack_map_get_object(&scope_object->via.map, "version"); if (field != NULL && field->type == MSGPACK_OBJECT_STR) { - scope->version = flb_strndup(field->via.str.ptr, field->via.str.size); + scope->version = otlp_proto_arena_strndup(arena, + field->via.str.ptr, + field->via.str.size); if (scope->version == NULL) { return -1; } @@ -957,7 +970,7 @@ static int add_msgpack_scope_fields( field = msgpack_map_get_object(&scope_object->via.map, "attributes"); if (field != NULL && field->type == MSGPACK_OBJECT_MAP) { - scope->attributes = msgpack_map_to_otlp_kvarray(field, + scope->attributes = msgpack_map_to_otlp_kvarray(arena, field, &scope->n_attributes); if (field->via.map.size > 0 && scope->attributes == NULL) { return -1; @@ -974,6 +987,7 @@ static int add_msgpack_scope_fields( } static struct otlp_proto_logs_resource_state *append_logs_resource_state( + struct otlp_proto_arena *arena, Opentelemetry__Proto__Collector__Logs__V1__ExportLogsServiceRequest *export_logs, struct otlp_proto_logs_resource_state **states, size_t *state_count, @@ -989,11 +1003,11 @@ static struct otlp_proto_logs_resource_state *append_logs_resource_state( Opentelemetry__Proto__Logs__V1__ResourceLogs **tmp; msgpack_object *schema_url; - resource_log = flb_calloc(1, sizeof(Opentelemetry__Proto__Logs__V1__ResourceLogs)); - resource = flb_calloc(1, sizeof(Opentelemetry__Proto__Resource__V1__Resource)); + resource_log = otlp_proto_arena_calloc(arena, 1, + sizeof(Opentelemetry__Proto__Logs__V1__ResourceLogs)); + resource = otlp_proto_arena_calloc(arena, 1, + sizeof(Opentelemetry__Proto__Resource__V1__Resource)); if (resource_log == NULL || resource == NULL) { - flb_free(resource_log); - flb_free(resource); return NULL; } @@ -1001,20 +1015,16 @@ static struct otlp_proto_logs_resource_state *append_logs_resource_state( opentelemetry__proto__resource__v1__resource__init(resource); resource_log->resource = resource; - if (add_msgpack_attributes_to_resource(resource, resource_object) != 0) { - flb_free(resource); - flb_free(resource_log); + if (add_msgpack_attributes_to_resource(arena, resource, resource_object) != 0) { return NULL; } schema_url = resource_schema_url_object(resource_object, resource_body); if (schema_url != NULL && schema_url->type == MSGPACK_OBJECT_STR) { - resource_log->schema_url = flb_strndup(schema_url->via.str.ptr, - schema_url->via.str.size); + resource_log->schema_url = otlp_proto_arena_strndup(arena, + schema_url->via.str.ptr, + schema_url->via.str.size); if (resource_log->schema_url == NULL) { - otlp_kvarray_destroy(resource->attributes, resource->n_attributes); - flb_free(resource); - flb_free(resource_log); return NULL; } } @@ -1023,13 +1033,6 @@ static struct otlp_proto_logs_resource_state *append_logs_resource_state( sizeof(Opentelemetry__Proto__Logs__V1__ResourceLogs *) * (export_logs->n_resource_logs + 1)); if (tmp == NULL) { - otlp_kvarray_destroy(resource->attributes, resource->n_attributes); - if (resource_log->schema_url != NULL && - resource_log->schema_url != protobuf_c_empty_string) { - flb_free(resource_log->schema_url); - } - flb_free(resource); - flb_free(resource_log); return NULL; } @@ -1056,6 +1059,7 @@ static struct otlp_proto_logs_resource_state *append_logs_resource_state( } static struct otlp_proto_logs_scope_state *append_logs_scope_state( + struct otlp_proto_arena *arena, struct otlp_proto_logs_resource_state *resource_state, int64_t scope_id, uint64_t scope_hash, @@ -1068,11 +1072,11 @@ static struct otlp_proto_logs_scope_state *append_logs_scope_state( Opentelemetry__Proto__Logs__V1__ScopeLogs **tmp; msgpack_object *schema_url; - scope_log = flb_calloc(1, sizeof(Opentelemetry__Proto__Logs__V1__ScopeLogs)); - scope = flb_calloc(1, sizeof(Opentelemetry__Proto__Common__V1__InstrumentationScope)); + scope_log = otlp_proto_arena_calloc(arena, 1, + sizeof(Opentelemetry__Proto__Logs__V1__ScopeLogs)); + scope = otlp_proto_arena_calloc(arena, 1, + sizeof(Opentelemetry__Proto__Common__V1__InstrumentationScope)); if (scope_log == NULL || scope == NULL) { - flb_free(scope_log); - flb_free(scope); return NULL; } @@ -1080,26 +1084,17 @@ static struct otlp_proto_logs_scope_state *append_logs_scope_state( opentelemetry__proto__common__v1__instrumentation_scope__init(scope); scope_log->scope = scope; - if (add_msgpack_scope_fields(scope, scope_object) != 0) { - flb_free(scope->name); - flb_free(scope->version); - otlp_kvarray_destroy(scope->attributes, scope->n_attributes); - flb_free(scope); - flb_free(scope_log); + if (add_msgpack_scope_fields(arena, scope, scope_object) != 0) { return NULL; } if (scope_object != NULL && scope_object->type == MSGPACK_OBJECT_MAP) { schema_url = msgpack_map_get_object(&scope_object->via.map, "schema_url"); if (schema_url != NULL && schema_url->type == MSGPACK_OBJECT_STR) { - scope_log->schema_url = flb_strndup(schema_url->via.str.ptr, - schema_url->via.str.size); + scope_log->schema_url = otlp_proto_arena_strndup(arena, + schema_url->via.str.ptr, + schema_url->via.str.size); if (scope_log->schema_url == NULL) { - flb_free(scope->name); - flb_free(scope->version); - otlp_kvarray_destroy(scope->attributes, scope->n_attributes); - flb_free(scope); - flb_free(scope_log); return NULL; } } @@ -1109,15 +1104,6 @@ static struct otlp_proto_logs_scope_state *append_logs_scope_state( sizeof(Opentelemetry__Proto__Logs__V1__ScopeLogs *) * (resource_state->resource_log->n_scope_logs + 1)); if (tmp == NULL) { - if (scope_log->schema_url != NULL && - scope_log->schema_url != protobuf_c_empty_string) { - flb_free(scope_log->schema_url); - } - flb_free(scope->name); - flb_free(scope->version); - otlp_kvarray_destroy(scope->attributes, scope->n_attributes); - flb_free(scope); - flb_free(scope_log); return NULL; } @@ -1145,6 +1131,7 @@ static struct otlp_proto_logs_scope_state *append_logs_scope_state( } static int ensure_default_logs_scope_state( + struct otlp_proto_arena *arena, Opentelemetry__Proto__Collector__Logs__V1__ExportLogsServiceRequest *export_logs, struct otlp_proto_logs_resource_state **resource_states, size_t *resource_state_count, @@ -1162,7 +1149,8 @@ static int ensure_default_logs_scope_state( 0, resource_hash); if (*current_resource == NULL) { - *current_resource = append_logs_resource_state(export_logs, + *current_resource = append_logs_resource_state(arena, + export_logs, resource_states, resource_state_count, 0, @@ -1176,7 +1164,8 @@ static int ensure_default_logs_scope_state( *current_scope = find_logs_scope_state(*current_resource, 0, scope_hash); if (*current_scope == NULL) { - *current_scope = append_logs_scope_state(*current_resource, + *current_scope = append_logs_scope_state(arena, + *current_resource, 0, scope_hash, NULL); @@ -1188,7 +1177,8 @@ static int ensure_default_logs_scope_state( return 0; } -static int append_binary_id_field(ProtobufCBinaryData *field, +static int append_binary_id_field(struct otlp_proto_arena *arena, + ProtobufCBinaryData *field, msgpack_object *value, size_t expected_size) { @@ -1203,7 +1193,7 @@ static int append_binary_id_field(ProtobufCBinaryData *field, return 0; } - field->data = flb_malloc(value->via.bin.size); + field->data = otlp_proto_arena_alloc(arena, value->via.bin.size); if (field->data == NULL) { return -1; } @@ -1221,7 +1211,7 @@ static int append_binary_id_field(ProtobufCBinaryData *field, return 0; } - field->data = flb_calloc(1, expected_size); + field->data = otlp_proto_arena_calloc(arena, 1, expected_size); if (field->data == NULL) { return -1; } @@ -1232,7 +1222,6 @@ static int append_binary_id_field(ProtobufCBinaryData *field, char *str = (char *) value->via.str.ptr; if (!isxdigit(str[i * 2]) || !isxdigit(str[i * 2 + 1])) { - flb_free(field->data); field->data = NULL; return -1; } @@ -1251,7 +1240,8 @@ static int append_binary_id_field(ProtobufCBinaryData *field, return 0; } -static int log_record_to_proto(Opentelemetry__Proto__Logs__V1__LogRecord *record, +static int log_record_to_proto(struct otlp_proto_arena *arena, + Opentelemetry__Proto__Logs__V1__LogRecord *record, struct flb_log_event *event, const char **logs_body_keys, size_t logs_body_key_count, @@ -1311,8 +1301,9 @@ static int log_record_to_proto(Opentelemetry__Proto__Logs__V1__LogRecord *record field = msgpack_map_get_object(&otlp_metadata->via.map, "severity_text"); if (field != NULL && field->type == MSGPACK_OBJECT_STR) { - record->severity_text = flb_strndup(field->via.str.ptr, - field->via.str.size); + record->severity_text = otlp_proto_arena_strndup(arena, + field->via.str.ptr, + field->via.str.size); if (record->severity_text == NULL) { return -1; } @@ -1320,28 +1311,30 @@ static int log_record_to_proto(Opentelemetry__Proto__Logs__V1__LogRecord *record field = msgpack_map_get_object(&otlp_metadata->via.map, "attributes"); if (field != NULL && field->type == MSGPACK_OBJECT_MAP) { - attrs = msgpack_map_to_otlp_kvarray(field, &count); + attrs = msgpack_map_to_otlp_kvarray(arena, field, &count); if (field->via.map.size > 0 && attrs == NULL) { return -1; } - if (append_kvarrays(&record->attributes, + if (append_kvarrays(arena, + &record->attributes, &record->n_attributes, attrs, count) != 0) { - otlp_kvarray_destroy(attrs, count); return -1; } } - if (append_binary_id_field(&record->trace_id, + if (append_binary_id_field(arena, + &record->trace_id, msgpack_map_get_object(&otlp_metadata->via.map, "trace_id"), 16) != 0) { return -1; } - if (append_binary_id_field(&record->span_id, + if (append_binary_id_field(arena, + &record->span_id, msgpack_map_get_object(&otlp_metadata->via.map, "span_id"), 8) != 0) { @@ -1349,7 +1342,8 @@ static int log_record_to_proto(Opentelemetry__Proto__Logs__V1__LogRecord *record } } - if (log_record_set_body_and_attributes(record, + if (log_record_set_body_and_attributes(arena, + record, event, logs_body_keys, logs_body_key_count, @@ -1360,29 +1354,11 @@ static int log_record_to_proto(Opentelemetry__Proto__Logs__V1__LogRecord *record return 0; } -static void destroy_log_record(Opentelemetry__Proto__Logs__V1__LogRecord *record) -{ - if (record == NULL) { - return; - } - - otlp_any_value_destroy(record->body); - otlp_kvarray_destroy(record->attributes, record->n_attributes); - if (record->severity_text != NULL && - record->severity_text != protobuf_c_empty_string) { - flb_free(record->severity_text); - } - flb_free(record->span_id.data); - flb_free(record->trace_id.data); - flb_free(record); -} - -static void destroy_export_logs( +static void destroy_export_logs_arrays( Opentelemetry__Proto__Collector__Logs__V1__ExportLogsServiceRequest *export_logs) { size_t index; size_t inner; - size_t record_index; Opentelemetry__Proto__Logs__V1__ResourceLogs *resource_log; Opentelemetry__Proto__Logs__V1__ScopeLogs *scope_log; @@ -1402,44 +1378,10 @@ static void destroy_export_logs( continue; } - for (record_index = 0; - record_index < scope_log->n_log_records; - record_index++) { - destroy_log_record(scope_log->log_records[record_index]); - } - flb_free(scope_log->log_records); - if (scope_log->scope != NULL) { - if (scope_log->scope->name != NULL && - scope_log->scope->name != protobuf_c_empty_string) { - flb_free(scope_log->scope->name); - } - if (scope_log->scope->version != NULL && - scope_log->scope->version != protobuf_c_empty_string) { - flb_free(scope_log->scope->version); - } - otlp_kvarray_destroy(scope_log->scope->attributes, - scope_log->scope->n_attributes); - flb_free(scope_log->scope); - } - if (scope_log->schema_url != NULL && - scope_log->schema_url != protobuf_c_empty_string) { - flb_free(scope_log->schema_url); - } - flb_free(scope_log); } flb_free(resource_log->scope_logs); - if (resource_log->resource != NULL) { - otlp_kvarray_destroy(resource_log->resource->attributes, - resource_log->resource->n_attributes); - flb_free(resource_log->resource); - } - if (resource_log->schema_url != NULL && - resource_log->schema_url != protobuf_c_empty_string) { - flb_free(resource_log->schema_url); - } - flb_free(resource_log); } flb_free(export_logs->resource_logs); @@ -1598,12 +1540,14 @@ flb_sds_t flb_opentelemetry_logs_to_otlp_proto(const void *event_chunk_data, flb_sds_t output; struct flb_log_event event; struct flb_log_event_decoder decoder; + struct otlp_proto_arena arena; msgpack_object *group_metadata; msgpack_object *group_body; msgpack_object *resource_object; msgpack_object *scope_object; uint64_t resource_hash; uint64_t scope_hash; + Opentelemetry__Proto__Logs__V1__LogRecord **tmp; struct otlp_proto_logs_scope_state *current_scope; struct otlp_proto_logs_resource_state *current_resource; struct otlp_proto_logs_resource_state *resource_states; @@ -1637,6 +1581,10 @@ flb_sds_t flb_opentelemetry_logs_to_otlp_proto(const void *event_chunk_data, opentelemetry__proto__collector__logs__v1__export_logs_service_request__init( &export_logs); + if (otlp_proto_arena_init(&arena) != 0) { + set_error(result, FLB_OPENTELEMETRY_OTLP_PROTO_NOT_SUPPORTED, ENOMEM); + return NULL; + } current_scope = NULL; current_resource = NULL; @@ -1647,6 +1595,7 @@ flb_sds_t flb_opentelemetry_logs_to_otlp_proto(const void *event_chunk_data, (char *) event_chunk_data, event_chunk_size); if (ret != FLB_EVENT_DECODER_SUCCESS) { + otlp_proto_arena_destroy(&arena); set_error(result, FLB_OPENTELEMETRY_OTLP_PROTO_INVALID_ARGUMENT, EINVAL); return NULL; } @@ -1658,7 +1607,8 @@ flb_sds_t flb_opentelemetry_logs_to_otlp_proto(const void *event_chunk_data, ret = flb_log_event_decoder_get_record_type(&event, &record_type); if (ret != 0) { flb_log_event_decoder_destroy(&decoder); - destroy_export_logs(&export_logs); + destroy_export_logs_arrays(&export_logs); + otlp_proto_arena_destroy(&arena); destroy_logs_resource_states(resource_states, resource_state_count); set_error(result, FLB_OPENTELEMETRY_OTLP_PROTO_INVALID_LOG_EVENT, EINVAL); return NULL; @@ -1677,7 +1627,8 @@ flb_sds_t flb_opentelemetry_logs_to_otlp_proto(const void *event_chunk_data, msgpack_map_get_int64(&group_metadata->via.map, "scope_id", &scope_id) != 0) { if (require_otel_metadata == FLB_TRUE) { flb_log_event_decoder_destroy(&decoder); - destroy_export_logs(&export_logs); + destroy_export_logs_arrays(&export_logs); + otlp_proto_arena_destroy(&arena); destroy_logs_resource_states(resource_states, resource_state_count); set_error(result, FLB_OPENTELEMETRY_OTLP_PROTO_INVALID_LOG_EVENT, EINVAL); return NULL; @@ -1704,7 +1655,8 @@ flb_sds_t flb_opentelemetry_logs_to_otlp_proto(const void *event_chunk_data, resource_id, resource_hash); if (current_resource == NULL) { - current_resource = append_logs_resource_state(&export_logs, + current_resource = append_logs_resource_state(&arena, + &export_logs, &resource_states, &resource_state_count, resource_id, @@ -1713,7 +1665,8 @@ flb_sds_t flb_opentelemetry_logs_to_otlp_proto(const void *event_chunk_data, group_body); if (current_resource == NULL) { flb_log_event_decoder_destroy(&decoder); - destroy_export_logs(&export_logs); + destroy_export_logs_arrays(&export_logs); + otlp_proto_arena_destroy(&arena); destroy_logs_resource_states(resource_states, resource_state_count); set_error(result, FLB_OPENTELEMETRY_OTLP_PROTO_NOT_SUPPORTED, ENOMEM); return NULL; @@ -1724,13 +1677,15 @@ flb_sds_t flb_opentelemetry_logs_to_otlp_proto(const void *event_chunk_data, scope_id, scope_hash); if (current_scope == NULL) { - current_scope = append_logs_scope_state(current_resource, + current_scope = append_logs_scope_state(&arena, + current_resource, scope_id, scope_hash, scope_object); if (current_scope == NULL) { flb_log_event_decoder_destroy(&decoder); - destroy_export_logs(&export_logs); + destroy_export_logs_arrays(&export_logs); + otlp_proto_arena_destroy(&arena); destroy_logs_resource_states(resource_states, resource_state_count); set_error(result, FLB_OPENTELEMETRY_OTLP_PROTO_NOT_SUPPORTED, ENOMEM); return NULL; @@ -1748,33 +1703,35 @@ flb_sds_t flb_opentelemetry_logs_to_otlp_proto(const void *event_chunk_data, if (current_scope == NULL) { if (require_otel_metadata == FLB_TRUE) { flb_log_event_decoder_destroy(&decoder); - destroy_export_logs(&export_logs); + destroy_export_logs_arrays(&export_logs); + otlp_proto_arena_destroy(&arena); destroy_logs_resource_states(resource_states, resource_state_count); set_error(result, FLB_OPENTELEMETRY_OTLP_PROTO_INVALID_LOG_EVENT, EINVAL); return NULL; } - if (ensure_default_logs_scope_state(&export_logs, + if (ensure_default_logs_scope_state(&arena, + &export_logs, &resource_states, &resource_state_count, ¤t_resource, ¤t_scope) != 0) { flb_log_event_decoder_destroy(&decoder); - destroy_export_logs(&export_logs); + destroy_export_logs_arrays(&export_logs); + otlp_proto_arena_destroy(&arena); destroy_logs_resource_states(resource_states, resource_state_count); set_error(result, FLB_OPENTELEMETRY_OTLP_PROTO_NOT_SUPPORTED, ENOMEM); return NULL; } } - Opentelemetry__Proto__Logs__V1__LogRecord **tmp; - tmp = flb_realloc(current_scope->scope_log->log_records, sizeof(Opentelemetry__Proto__Logs__V1__LogRecord *) * (current_scope->scope_log->n_log_records + 1)); if (tmp == NULL) { flb_log_event_decoder_destroy(&decoder); - destroy_export_logs(&export_logs); + destroy_export_logs_arrays(&export_logs); + otlp_proto_arena_destroy(&arena); destroy_logs_resource_states(resource_states, resource_state_count); set_error(result, FLB_OPENTELEMETRY_OTLP_PROTO_NOT_SUPPORTED, ENOMEM); return NULL; @@ -1782,12 +1739,14 @@ flb_sds_t flb_opentelemetry_logs_to_otlp_proto(const void *event_chunk_data, current_scope->scope_log->log_records = tmp; - record = flb_calloc(1, sizeof(Opentelemetry__Proto__Logs__V1__LogRecord)); + record = otlp_proto_arena_calloc(&arena, 1, + sizeof(Opentelemetry__Proto__Logs__V1__LogRecord)); current_scope->scope_log->log_records[ current_scope->scope_log->n_log_records] = record; if (record == NULL) { flb_log_event_decoder_destroy(&decoder); - destroy_export_logs(&export_logs); + destroy_export_logs_arrays(&export_logs); + otlp_proto_arena_destroy(&arena); destroy_logs_resource_states(resource_states, resource_state_count); set_error(result, FLB_OPENTELEMETRY_OTLP_PROTO_NOT_SUPPORTED, ENOMEM); return NULL; @@ -1796,16 +1755,17 @@ flb_sds_t flb_opentelemetry_logs_to_otlp_proto(const void *event_chunk_data, opentelemetry__proto__logs__v1__log_record__init(record); if (log_record_to_proto( + &arena, record, &event, logs_body_keys, logs_body_key_count, logs_body_key_attributes) != 0) { - destroy_log_record(record); current_scope->scope_log->log_records[ current_scope->scope_log->n_log_records] = NULL; flb_log_event_decoder_destroy(&decoder); - destroy_export_logs(&export_logs); + destroy_export_logs_arrays(&export_logs); + otlp_proto_arena_destroy(&arena); destroy_logs_resource_states(resource_states, resource_state_count); set_error(result, FLB_OPENTELEMETRY_OTLP_PROTO_NOT_SUPPORTED, ENOMEM); return NULL; @@ -1819,7 +1779,8 @@ flb_sds_t flb_opentelemetry_logs_to_otlp_proto(const void *event_chunk_data, if (ret != FLB_EVENT_DECODER_SUCCESS && ret != FLB_EVENT_DECODER_ERROR_INSUFFICIENT_DATA) { - destroy_export_logs(&export_logs); + destroy_export_logs_arrays(&export_logs); + otlp_proto_arena_destroy(&arena); set_error(result, FLB_OPENTELEMETRY_OTLP_PROTO_INVALID_LOG_EVENT, EINVAL); return NULL; } @@ -1828,7 +1789,8 @@ flb_sds_t flb_opentelemetry_logs_to_otlp_proto(const void *event_chunk_data, opentelemetry__proto__collector__logs__v1__export_logs_service_request__get_packed_size( &export_logs)); if (output == NULL) { - destroy_export_logs(&export_logs); + destroy_export_logs_arrays(&export_logs); + otlp_proto_arena_destroy(&arena); set_error(result, FLB_OPENTELEMETRY_OTLP_PROTO_NOT_SUPPORTED, ENOMEM); return NULL; } @@ -1837,7 +1799,8 @@ flb_sds_t flb_opentelemetry_logs_to_otlp_proto(const void *event_chunk_data, opentelemetry__proto__collector__logs__v1__export_logs_service_request__pack( &export_logs, (uint8_t *) output)); - destroy_export_logs(&export_logs); + destroy_export_logs_arrays(&export_logs); + otlp_proto_arena_destroy(&arena); set_result(result, FLB_OPENTELEMETRY_OTLP_PROTO_SUCCESS); return output; diff --git a/src/tls/flb_tls.c b/src/tls/flb_tls.c index 40961b6a12a..9ab634165cd 100644 --- a/src/tls/flb_tls.c +++ b/src/tls/flb_tls.c @@ -20,6 +20,9 @@ #include #include #include +#include + +#include #include "openssl.c" @@ -124,31 +127,37 @@ static inline void io_tls_backup_event(struct flb_connection *connection, } } -static inline void io_tls_restore_event(struct flb_connection *connection, - struct mk_event *backup) +static inline int io_tls_restore_event(struct flb_connection *connection, + struct mk_event *backup) { int result; - if (connection != NULL && backup != NULL) { - if (MK_EVENT_IS_REGISTERED((&connection->event))) { - result = mk_event_del(connection->evl, &connection->event); + if (connection == NULL || backup == NULL) { + return -1; + } - assert(result == 0); + if (MK_EVENT_IS_REGISTERED((&connection->event))) { + result = mk_event_del(connection->evl, &connection->event); + if (result == -1) { + return -1; } + } - if (MK_EVENT_IS_REGISTERED(backup)) { - connection->event.priority = backup->priority; - connection->event.handler = backup->handler; - - result = mk_event_add(connection->evl, - connection->fd, - backup->type, - backup->mask, - &connection->event); + if (MK_EVENT_IS_REGISTERED(backup)) { + connection->event.priority = backup->priority; + connection->event.handler = backup->handler; - assert(result == 0); + result = mk_event_add(connection->evl, + connection->fd, + backup->type, + backup->mask, + &connection->event); + if (result == -1) { + return -1; } } + + return 0; } @@ -162,10 +171,10 @@ static inline int io_tls_event_switch(struct flb_tls_session *session, event = &session->connection->event; event_loop = session->connection->evl; - if ((event->mask & mask) == 0) { + if ((event->mask & (MK_EVENT_READ | MK_EVENT_WRITE)) != mask) { ret = mk_event_add(event_loop, event->fd, - FLB_ENGINE_EV_THREAD, + event->type, mask, event); event->priority = FLB_ENGINE_PRIORITY_CONNECT; @@ -175,6 +184,8 @@ static inline int io_tls_event_switch(struct flb_tls_session *session, return -1; } + + event->mask = mask; } return 0; @@ -182,7 +193,150 @@ static inline int io_tls_event_switch(struct flb_tls_session *session, int flb_tls_load_system_certificates(struct flb_tls *tls) { - return load_system_certificates(tls->ctx); + int ret; + + ret = load_system_certificates(tls->ctx); + if (ret == 0) { + tls->system_certificates_loaded = FLB_TRUE; + } + + return ret; +} + +static int tls_file_status_get(const char *path, + struct flb_tls_file_status *status) +{ + struct stat st; + + memset(status, 0, sizeof(struct flb_tls_file_status)); + + if (path == NULL) { + return 0; + } + + if (stat(path, &st) != 0) { + status->exists = FLB_FALSE; + return -1; + } + + status->exists = FLB_TRUE; + status->size = (uint64_t) st.st_size; +#ifdef FLB_SYSTEM_LINUX + status->device = (uint64_t) st.st_dev; + status->inode = (uint64_t) st.st_ino; + status->mtime_nsec = (uint64_t) st.st_mtim.tv_nsec; + status->ctime_nsec = (uint64_t) st.st_ctim.tv_nsec; +#else + status->device = 0; + status->inode = 0; + status->mtime_nsec = 0; + status->ctime_nsec = 0; +#endif + status->mtime = (uint64_t) st.st_mtime; + status->ctime = (uint64_t) st.st_ctime; + + return 0; +} + +static int tls_file_status_changed(struct flb_tls_file_status *current, + struct flb_tls_file_status *previous) +{ + if (current->exists != previous->exists || + current->size != previous->size || + current->device != previous->device || + current->inode != previous->inode || + current->mtime != previous->mtime || + current->mtime_nsec != previous->mtime_nsec || + current->ctime != previous->ctime || + current->ctime_nsec != previous->ctime_nsec) { + return FLB_TRUE; + } + + return FLB_FALSE; +} + +static void tls_file_status_refresh(struct flb_tls *tls) +{ + tls_file_status_get(tls->ca_path, &tls->ca_path_status); + tls_file_status_get(tls->ca_file, &tls->ca_file_status); + tls_file_status_get(tls->crt_file, &tls->crt_file_status); + tls_file_status_get(tls->key_file, &tls->key_file_status); +} + +static int tls_file_status_has_changed( + struct flb_tls *tls, + struct flb_tls_file_status *ca_path_status, + struct flb_tls_file_status *ca_file_status, + struct flb_tls_file_status *crt_file_status, + struct flb_tls_file_status *key_file_status) +{ + tls_file_status_get(tls->ca_path, ca_path_status); + tls_file_status_get(tls->ca_file, ca_file_status); + tls_file_status_get(tls->crt_file, crt_file_status); + tls_file_status_get(tls->key_file, key_file_status); + + if (tls_file_status_changed(ca_path_status, &tls->ca_path_status) == FLB_TRUE || + tls_file_status_changed(ca_file_status, &tls->ca_file_status) == FLB_TRUE || + tls_file_status_changed(crt_file_status, &tls->crt_file_status) == FLB_TRUE || + tls_file_status_changed(key_file_status, &tls->key_file_status) == FLB_TRUE) { + return FLB_TRUE; + } + + return FLB_FALSE; +} + +static int tls_should_reload_context( + struct flb_tls *tls, + struct flb_tls_file_status *ca_path_status, + struct flb_tls_file_status *ca_file_status, + struct flb_tls_file_status *crt_file_status, + struct flb_tls_file_status *key_file_status) +{ + if (tls_file_status_has_changed(tls, + ca_path_status, + ca_file_status, + crt_file_status, + key_file_status) == FLB_TRUE) { + return FLB_TRUE; + } + +#if defined(FLB_SYSTEM_WINDOWS) || defined(FLB_SYSTEM_MACOS) + /* + * macOS Keychain and Windows CertStore do not expose a portable file + * metadata handle for us to watch. Refresh store-backed contexts before + * each new TLS session so rotations/imports become visible without a + * process restart. + */ + if (tls->system_certificates_loaded == FLB_TRUE) { + return FLB_TRUE; + } +#endif + + return FLB_FALSE; +} + +static int tls_store_string(char **slot, const char *value) +{ + char *tmp; + + if (*slot != NULL) { + flb_free(*slot); + *slot = NULL; + } + + if (value == NULL) { + return 0; + } + + tmp = flb_strdup(value); + if (tmp == NULL) { + flb_errno(); + return -1; + } + + *slot = tmp; + + return 0; } struct flb_tls *flb_tls_create(int mode, @@ -217,30 +371,104 @@ struct flb_tls *flb_tls_create(int mode, return NULL; } + tls->ctx = backend; + tls->api = &tls_openssl; + pthread_mutex_init(&tls->reload_mutex, NULL); + tls->verify = verify; tls->debug = debug; tls->mode = mode; tls->verify_hostname = FLB_FALSE; + tls->system_certificates_loaded = FLB_FALSE; +#if defined(FLB_SYSTEM_WINDOWS) || defined(FLB_SYSTEM_MACOS) + if (ca_path == NULL && ca_file == NULL && mode == FLB_TLS_CLIENT_MODE) { + tls->system_certificates_loaded = FLB_TRUE; + } +#endif #if defined(FLB_SYSTEM_WINDOWS) tls->certstore_name = NULL; tls->use_enterprise_store = FLB_FALSE; + tls->client_thumbprints = NULL; #endif - if (vhost != NULL) { - tls->vhost = flb_strdup(vhost); + if (tls_store_string(&tls->vhost, vhost) != 0 || + tls_store_string(&tls->ca_path, ca_path) != 0 || + tls_store_string(&tls->ca_file, ca_file) != 0 || + tls_store_string(&tls->crt_file, crt_file) != 0 || + tls_store_string(&tls->key_file, key_file) != 0 || + tls_store_string(&tls->key_passwd, key_passwd) != 0) { + flb_tls_destroy(tls); + return NULL; } - tls->ctx = backend; - tls->api = &tls_openssl; + tls_file_status_refresh(tls); return tls; } +int flb_tls_reload_if_needed(struct flb_tls *tls) +{ + int ret; + struct flb_tls_file_status ca_path_status; + struct flb_tls_file_status ca_file_status; + struct flb_tls_file_status crt_file_status; + struct flb_tls_file_status key_file_status; + + if (tls == NULL || tls->ctx == NULL || tls->api == NULL || + tls->api->context_reload == NULL) { + return 0; + } + + pthread_mutex_lock(&tls->reload_mutex); + + if (tls_should_reload_context(tls, + &ca_path_status, + &ca_file_status, + &crt_file_status, + &key_file_status) == FLB_FALSE) { + pthread_mutex_unlock(&tls->reload_mutex); + return 0; + } + + ret = tls->api->context_reload(tls); + if (ret != 0) { + pthread_mutex_unlock(&tls->reload_mutex); + flb_error("[tls] detected certificate file changes but reload failed"); + return -1; + } + + /* + * Commit the snapshot that triggered this reload. If a file changes while + * the backend is loading it, the next session will observe the newer + * metadata and reload again instead of treating unseen contents as loaded. + */ + tls->ca_path_status = ca_path_status; + tls->ca_file_status = ca_file_status; + tls->crt_file_status = crt_file_status; + tls->key_file_status = key_file_status; + pthread_mutex_unlock(&tls->reload_mutex); + flb_info("[tls] reloaded TLS certificate configuration"); + + return 1; +} + int flb_tls_set_minmax_proto(struct flb_tls *tls, const char *min_version, const char *max_version) { + int ret; + if (tls->ctx) { - return tls->api->set_minmax_proto(tls, min_version, max_version); + ret = tls->api->set_minmax_proto(tls, min_version, max_version); + if (ret != 0) { + return ret; + } + + if (tls_store_string(&tls->min_version, min_version) != 0 || + tls_store_string(&tls->max_version, max_version) != 0) { + return -1; + } + + return ret; } return 0; @@ -248,8 +476,19 @@ int flb_tls_set_minmax_proto(struct flb_tls *tls, int flb_tls_set_ciphers(struct flb_tls *tls, const char *ciphers) { + int ret; + if (tls->ctx) { - return tls->api->set_ciphers(tls, ciphers); + ret = tls->api->set_ciphers(tls, ciphers); + if (ret != 0) { + return ret; + } + + if (tls_store_string(&tls->ciphers, ciphers) != 0) { + return -1; + } + + return ret; } return 0; @@ -269,13 +508,45 @@ int flb_tls_destroy(struct flb_tls *tls) if (tls->vhost != NULL) { flb_free(tls->vhost); } + if (tls->ca_path != NULL) { + flb_free(tls->ca_path); + } + if (tls->ca_file != NULL) { + flb_free(tls->ca_file); + } + if (tls->crt_file != NULL) { + flb_free(tls->crt_file); + } + if (tls->key_file != NULL) { + flb_free(tls->key_file); + } + if (tls->key_passwd != NULL) { + flb_free(tls->key_passwd); + } + if (tls->alpn != NULL) { + flb_free(tls->alpn); + } + if (tls->min_version != NULL) { + flb_free(tls->min_version); + } + if (tls->max_version != NULL) { + flb_free(tls->max_version); + } + if (tls->ciphers != NULL) { + flb_free(tls->ciphers); + } #if defined(FLB_SYSTEM_WINDOWS) if (tls->certstore_name) { flb_free(tls->certstore_name); } + if (tls->client_thumbprints) { + flb_free(tls->client_thumbprints); + } #endif + pthread_mutex_destroy(&tls->reload_mutex); + flb_free(tls); return 0; @@ -283,8 +554,19 @@ int flb_tls_destroy(struct flb_tls *tls) int flb_tls_set_alpn(struct flb_tls *tls, const char *alpn) { + int ret; + if (tls->ctx) { - return tls->api->context_alpn_set(tls->ctx, alpn); + ret = tls->api->context_alpn_set(tls->ctx, alpn); + if (ret != 0) { + return ret; + } + + if (tls_store_string(&tls->alpn, alpn) != 0) { + return -1; + } + + return ret; } return 0; @@ -297,6 +579,11 @@ int flb_tls_set_verify_client(struct flb_tls *tls, int verify_client) } tls->verify_client = verify_client; +#if defined(FLB_SYSTEM_WINDOWS) || defined(FLB_SYSTEM_MACOS) + if (verify_client == FLB_TRUE && tls->ca_path == NULL && tls->ca_file == NULL) { + tls->system_certificates_loaded = FLB_TRUE; + } +#endif if (tls->ctx && tls->api->context_set_verify_client) { return tls->api->context_set_verify_client(tls->ctx, verify_client); @@ -319,8 +606,19 @@ int flb_tls_set_verify_hostname(struct flb_tls *tls, int verify_hostname) #if defined(FLB_SYSTEM_WINDOWS) int flb_tls_set_certstore_name(struct flb_tls *tls, const char *certstore_name) { + int ret; + if (tls) { - return tls->api->set_certstore_name(tls, certstore_name); + ret = tls->api->set_certstore_name(tls, certstore_name); + if (ret != 0) { + return ret; + } + + if (tls_store_string(&tls->certstore_name, certstore_name) != 0) { + return -1; + } + + return ret; } return 0; @@ -328,16 +626,36 @@ int flb_tls_set_certstore_name(struct flb_tls *tls, const char *certstore_name) int flb_tls_set_use_enterprise_store(struct flb_tls *tls, int use_enterprise) { + int ret; + if (tls) { - return tls->api->set_use_enterprise_store(tls, use_enterprise); + ret = tls->api->set_use_enterprise_store(tls, use_enterprise); + if (ret != 0) { + return ret; + } + + tls->use_enterprise_store = use_enterprise; + + return ret; } return 0; } int flb_tls_set_client_thumbprints(struct flb_tls *tls, const char *thumbprints) { + int ret; + if (tls && tls->api->set_client_thumbprints) { - return tls->api->set_client_thumbprints(tls, thumbprints); + ret = tls->api->set_client_thumbprints(tls, thumbprints); + if (ret != 0) { + return ret; + } + + if (tls_store_string(&tls->client_thumbprints, thumbprints) != 0) { + return -1; + } + + return ret; } return -1; } @@ -345,12 +663,19 @@ int flb_tls_set_client_thumbprints(struct flb_tls *tls, const char *thumbprints) int flb_tls_net_read(struct flb_tls_session *session, void *buf, size_t len) { + int event_driven; time_t timeout_timestamp; time_t current_timestamp; struct flb_tls *tls; int ret; tls = session->tls; + event_driven = FLB_FALSE; + + if (session->connection->type == FLB_DOWNSTREAM_CONNECTION && + MK_EVENT_IS_REGISTERED((&session->connection->event))) { + event_driven = FLB_TRUE; + } if (session->connection->net->io_timeout > 0) { timeout_timestamp = time(NULL) + session->connection->net->io_timeout; @@ -365,6 +690,14 @@ int flb_tls_net_read(struct flb_tls_session *session, void *buf, size_t len) current_timestamp = time(NULL); if (ret == FLB_TLS_WANT_READ) { + if (event_driven) { + if (io_tls_event_switch(session, MK_EVENT_READ) == -1) { + return -1; + } + + return FLB_TLS_WANT_READ; + } + if (timeout_timestamp > 0 && timeout_timestamp <= current_timestamp) { return ret; @@ -373,6 +706,14 @@ int flb_tls_net_read(struct flb_tls_session *session, void *buf, size_t len) goto retry_read; } else if (ret == FLB_TLS_WANT_WRITE) { + if (event_driven) { + if (io_tls_event_switch(session, MK_EVENT_READ | MK_EVENT_WRITE) == -1) { + return -1; + } + + return FLB_TLS_WANT_WRITE; + } + goto retry_read; } else if (ret < 0) { @@ -382,6 +723,10 @@ int flb_tls_net_read(struct flb_tls_session *session, void *buf, size_t len) return -1; } + if (event_driven && io_tls_event_switch(session, MK_EVENT_READ) == -1) { + return -1; + } + return ret; } @@ -408,9 +753,18 @@ int flb_tls_net_read_async(struct flb_coro *co, session->connection->coroutine = co; - io_tls_event_switch(session, MK_EVENT_READ); + ret = io_tls_event_switch(session, MK_EVENT_READ); + if (ret == -1) { + goto read_finished; + } + flb_coro_yield(co, FLB_FALSE); + if (session->connection->net_error != -1) { + ret = -1; + goto read_finished; + } + goto retry_read; } else if (ret == FLB_TLS_WANT_WRITE) { @@ -418,21 +772,28 @@ int flb_tls_net_read_async(struct flb_coro *co, session->connection->coroutine = co; - io_tls_event_switch(session, MK_EVENT_WRITE); + ret = io_tls_event_switch(session, MK_EVENT_WRITE); + if (ret == -1) { + goto read_finished; + } + flb_coro_yield(co, FLB_FALSE); + if (session->connection->net_error != -1) { + ret = -1; + goto read_finished; + } + goto retry_read; } - else - { - /* We want this field to hold NULL at all times unless we are explicitly - * waiting to be resumed. - */ - session->connection->coroutine = NULL; +read_finished: + /* We want this field to hold NULL at all times unless we are explicitly + * waiting to be resumed. + */ + session->connection->coroutine = NULL; - if (ret <= 0) { - ret = -1; - } + if (ret <= 0) { + ret = -1; } if (event_restore_needed) { @@ -446,7 +807,9 @@ int flb_tls_net_read_async(struct flb_coro *co, * the same event. */ - io_tls_restore_event(session->connection, &event_backup); + if (io_tls_restore_event(session->connection, &event_backup) == -1) { + ret = -1; + } } return ret; @@ -518,19 +881,45 @@ int flb_tls_net_write_async(struct flb_coro *co, if (ret == FLB_TLS_WANT_WRITE) { event_restore_needed = FLB_TRUE; - io_tls_event_switch(session, MK_EVENT_WRITE); + ret = io_tls_event_switch(session, MK_EVENT_WRITE); + if (ret == -1) { + session->connection->coroutine = NULL; + *out_len = total; + io_tls_restore_event(session->connection, &event_backup); + return -1; + } flb_coro_yield(co, FLB_FALSE); + if (session->connection->net_error != -1) { + session->connection->coroutine = NULL; + *out_len = total; + io_tls_restore_event(session->connection, &event_backup); + return -1; + } + goto retry_write; } else if (ret == FLB_TLS_WANT_READ) { event_restore_needed = FLB_TRUE; - io_tls_event_switch(session, MK_EVENT_READ); + ret = io_tls_event_switch(session, MK_EVENT_READ); + if (ret == -1) { + session->connection->coroutine = NULL; + *out_len = total; + io_tls_restore_event(session->connection, &event_backup); + return -1; + } flb_coro_yield(co, FLB_FALSE); + if (session->connection->net_error != -1) { + session->connection->coroutine = NULL; + *out_len = total; + io_tls_restore_event(session->connection, &event_backup); + return -1; + } + goto retry_write; } else if (ret < 0) { @@ -550,10 +939,25 @@ int flb_tls_net_write_async(struct flb_coro *co, total += ret; if (total < len) { - io_tls_event_switch(session, MK_EVENT_WRITE); + event_restore_needed = FLB_TRUE; + + ret = io_tls_event_switch(session, MK_EVENT_WRITE); + if (ret == -1) { + session->connection->coroutine = NULL; + *out_len = total; + io_tls_restore_event(session->connection, &event_backup); + return -1; + } flb_coro_yield(co, FLB_FALSE); + if (session->connection->net_error != -1) { + session->connection->coroutine = NULL; + *out_len = total; + io_tls_restore_event(session->connection, &event_backup); + return -1; + } + goto retry_write; } @@ -576,7 +980,9 @@ int flb_tls_net_write_async(struct flb_coro *co, * the same event. */ - io_tls_restore_event(session->connection, &event_backup); + if (io_tls_restore_event(session->connection, &event_backup) == -1) { + return -1; + } } return total; @@ -608,6 +1014,8 @@ int flb_tls_session_create(struct flb_tls *tls, char *vhost; int flag; + flb_tls_reload_if_needed(tls); + session = flb_calloc(1, sizeof(struct flb_tls_session)); if (session == NULL) { @@ -755,7 +1163,10 @@ int flb_tls_session_create(struct flb_tls *tls, * the same event. */ - io_tls_restore_event(session->connection, &event_backup); + if (io_tls_restore_event(session->connection, &event_backup) == -1 && + result == 0) { + result = -1; + } } if (result != 0) { diff --git a/src/tls/openssl.c b/src/tls/openssl.c index 53e3574ea8a..72e8379ac5f 100644 --- a/src/tls/openssl.c +++ b/src/tls/openssl.c @@ -902,7 +902,22 @@ static void *tls_context_create(int verify, * * https://www.openssl.org/docs/man1.0.2/man3/SSLv23_method.html */ - if (mode == FLB_TLS_SERVER_MODE) { + if (mode == FLB_TLS_SERVER_MODE_DGRAM || + mode == FLB_TLS_CLIENT_MODE_DGRAM) { +#ifndef OPENSSL_NO_DTLS + if (mode == FLB_TLS_SERVER_MODE_DGRAM) { + ssl_ctx = SSL_CTX_new(DTLSv1_server_method()); + } + else { + ssl_ctx = SSL_CTX_new(DTLSv1_client_method()); + } +#else + flb_error("[openssl] DTLS mode requested but this OpenSSL build " + "does not provide DTLS support"); + return NULL; +#endif + } + else if (mode == FLB_TLS_SERVER_MODE) { ssl_ctx = SSL_CTX_new(SSLv23_server_method()); } else { @@ -913,6 +928,12 @@ static void *tls_context_create(int verify, if (mode == FLB_TLS_SERVER_MODE) { ssl_ctx = SSL_CTX_new(TLS_server_method()); } + else if (mode == FLB_TLS_SERVER_MODE_DGRAM) { + ssl_ctx = SSL_CTX_new(DTLS_server_method()); + } + else if (mode == FLB_TLS_CLIENT_MODE_DGRAM) { + ssl_ctx = SSL_CTX_new(DTLS_client_method()); + } else { ssl_ctx = SSL_CTX_new(TLS_client_method()); } @@ -1316,6 +1337,99 @@ static int tls_set_client_thumbprints(struct flb_tls *tls, const char *thumbprin #endif +static int tls_context_reload(struct flb_tls *tls) +{ + SSL_CTX *old_ssl_ctx; + struct tls_context *ctx; + struct tls_context *new_ctx; + struct flb_tls tmp_tls; + + ctx = tls->ctx; + + new_ctx = tls_context_create(tls->verify, + tls->debug, + tls->mode, + tls->vhost, + tls->ca_path, + tls->ca_file, + tls->crt_file, + tls->key_file, + tls->key_passwd); + if (new_ctx == NULL) { + return -1; + } + + memset(&tmp_tls, 0, sizeof(struct flb_tls)); + tmp_tls.ctx = new_ctx; + + if (tls->verify_client == FLB_TRUE && + tls_context_set_verify_client(new_ctx, tls->verify_client) != 0) { + tls_context_destroy(new_ctx); + return -1; + } + + if ((tls->min_version != NULL || tls->max_version != NULL) && + tls_set_minmax_proto(&tmp_tls, tls->min_version, tls->max_version) != 0) { + tls_context_destroy(new_ctx); + return -1; + } + + if (tls->ciphers != NULL && + tls_set_ciphers(&tmp_tls, tls->ciphers) != 0) { + tls_context_destroy(new_ctx); + return -1; + } + + if (tls->alpn != NULL && + tls_context_alpn_set(new_ctx, tls->alpn) != 0) { + tls_context_destroy(new_ctx); + return -1; + } + +#if defined(FLB_SYSTEM_WINDOWS) + if (tls->certstore_name != NULL && + tls_set_certstore_name(&tmp_tls, tls->certstore_name) != 0) { + tls_context_destroy(new_ctx); + return -1; + } + + if (tls->use_enterprise_store == FLB_TRUE && + tls_set_use_enterprise_store(&tmp_tls, tls->use_enterprise_store) != 0) { + tls_context_destroy(new_ctx); + return -1; + } + + if (tls->client_thumbprints != NULL && + tls_set_client_thumbprints(&tmp_tls, tls->client_thumbprints) != 0) { + tls_context_destroy(new_ctx); + return -1; + } +#endif + + if (tls->system_certificates_loaded == FLB_TRUE && + load_system_certificates(new_ctx) != 0) { + tls_context_destroy(new_ctx); + return -1; + } + + pthread_mutex_lock(&ctx->mutex); + old_ssl_ctx = ctx->ctx; + ctx->ctx = new_ctx->ctx; + new_ctx->ctx = old_ssl_ctx; + + if (tls->alpn != NULL && tls->mode == FLB_TLS_SERVER_MODE) { + SSL_CTX_set_alpn_select_cb(ctx->ctx, + tls_context_server_alpn_select_callback, + ctx); + } + + pthread_mutex_unlock(&ctx->mutex); + + tls_context_destroy(new_ctx); + + return 0; +} + static void *tls_session_create(struct flb_tls *tls, int fd) { @@ -1498,6 +1612,23 @@ static int tls_net_read(struct flb_tls_session *session, ret = -1; } + else if (ret == SSL_ERROR_ZERO_RETURN) { + flb_debug("[tls] connection closed by the remote peer " + "(close_notify)"); + + /* + * The peer performed a clean TLS shutdown, so this session + * is finished. Flag the connection as errored so it is not + * reused: a reused session would keep hitting the cached + * shutdown state and fail every subsequent read identically. + * For a pooled upstream connection this makes + * flb_upstream_conn_release() destroy it instead of + * returning it to the keepalive pool. + */ + session->connection->net_error = ECONNRESET; + + ret = -1; + } else if (ret < 0) { err_code = ERR_get_error(); @@ -1580,6 +1711,23 @@ static int tls_net_write(struct flb_tls_session *session, ret = -1; } + else if (ssl_ret == SSL_ERROR_ZERO_RETURN) { + flb_debug("[tls] connection closed by the remote peer " + "(close_notify)"); + + /* + * The peer performed a clean TLS shutdown, so this session + * is finished. Flag the connection as errored so it is not + * reused: a reused session would keep hitting the cached + * shutdown state and fail every subsequent operation + * identically. For a pooled upstream connection this makes + * flb_upstream_conn_release() destroy it instead of + * returning it to the keepalive pool. + */ + session->connection->net_error = ECONNRESET; + + ret = -1; + } else { err_code = ERR_get_error(); if (err_code == 0) { @@ -1647,7 +1795,8 @@ static int tls_net_handshake(struct flb_tls *tls, pthread_mutex_lock(&ctx->mutex); if (!session->continuation_flag) { - if (tls->mode == FLB_TLS_CLIENT_MODE) { + if (tls->mode == FLB_TLS_CLIENT_MODE || + tls->mode == FLB_TLS_CLIENT_MODE_DGRAM) { SSL_set_connect_state(session->ssl); if (ctx->alpn != NULL) { @@ -1662,7 +1811,8 @@ static int tls_net_handshake(struct flb_tls *tls, } } } - else if (tls->mode == FLB_TLS_SERVER_MODE) { + else if (tls->mode == FLB_TLS_SERVER_MODE || + tls->mode == FLB_TLS_SERVER_MODE_DGRAM) { SSL_set_accept_state(session->ssl); } else { @@ -1696,10 +1846,12 @@ static int tls_net_handshake(struct flb_tls *tls, ERR_clear_error(); - if (tls->mode == FLB_TLS_CLIENT_MODE) { + if (tls->mode == FLB_TLS_CLIENT_MODE || + tls->mode == FLB_TLS_CLIENT_MODE_DGRAM) { ret = SSL_connect(session->ssl); } - else if (tls->mode == FLB_TLS_SERVER_MODE) { + else if (tls->mode == FLB_TLS_SERVER_MODE || + tls->mode == FLB_TLS_SERVER_MODE_DGRAM) { ret = SSL_accept(session->ssl); } @@ -1770,6 +1922,7 @@ static int tls_net_handshake(struct flb_tls *tls, static struct flb_tls_backend tls_openssl = { .name = "openssl", .context_create = tls_context_create, + .context_reload = tls_context_reload, .context_destroy = tls_context_destroy, .context_alpn_set = tls_context_alpn_set, .context_set_verify_client = tls_context_set_verify_client, diff --git a/tests/include/aws_client_mock.h b/tests/include/aws_client_mock.h index 7cdfc81066d..d0d00a85cd0 100644 --- a/tests/include/aws_client_mock.h +++ b/tests/include/aws_client_mock.h @@ -67,12 +67,21 @@ #define AWS_CLIENT_MOCK_H /* Variadic Argument Counter, Counts up to 64 variadic args */ +#ifdef _MSC_VER +#define FLB_AWS_CLIENT_MOCK_COUNT64(...) \ + _FLB_AWS_CLIENT_MOCK_COUNT64(dummy __VA_OPT__(,) __VA_ARGS__, 63, 62, 61, 60, 59, \ + 58, 57, 56, 55, 54, 53, 52, 51, 50, 49, 48, 47, 46, 45, \ + 44, 43, 42, 41, 40, 39, 38, 37, 36, 35, 34, 33, 32, 31, \ + 30, 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 17, \ + 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0) +#else #define FLB_AWS_CLIENT_MOCK_COUNT64(...) \ _FLB_AWS_CLIENT_MOCK_COUNT64(dummy, ##__VA_ARGS__, 63, 62, 61, 60, 59, 58, 57, 56, \ 55, 54, 53, 52, 51, 50, 49, 48, 47, 46, 45, 44, 43, 42, \ 41, 40, 39, 38, 37, 36, 35, 34, 33, 32, 31, 30, 29, 28, \ 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, \ 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0) +#endif #define _FLB_AWS_CLIENT_MOCK_COUNT64( \ x0, x1, x2, x3, x4, x5, x6, x7, x8, x9, x10, x11, x12, x13, x14, x15, x16, x17, x18, \ x19, x20, x21, x22, x23, x24, x25, x26, x27, x28, x29, x30, x31, x32, x33, x34, x35, \ diff --git a/tests/integration/README.md b/tests/integration/README.md index ab3986d22fb..d7b9020cf64 100644 --- a/tests/integration/README.md +++ b/tests/integration/README.md @@ -42,7 +42,7 @@ It starts a real Fluent Bit process, drives real network traffic into it, captur - downstream request generation - exporter endpoint output - negative-path handling -- memory diagnostics through optional Valgrind execution +- memory diagnostics through optional Valgrind or macOS `leaks` execution The suite is designed as a reusable tool for testing Fluent Bit plugins and protocol behavior with deterministic local infrastructure. @@ -60,6 +60,7 @@ It provides: - bounded waits instead of ad hoc sleeps - per-run logs and result directories - optional Valgrind wrapping and parsing +- optional macOS `leaks --atExit` supervision with graceful shutdown That makes it useful both for plugin development and for runtime regression testing. @@ -85,7 +86,7 @@ That makes it useful both for plugin development and for runtime regression test | start/stop binary | logs/results | readiness - | valgrind integration + | memory diagnostics +----------+-----------+ | v @@ -108,7 +109,7 @@ That makes it useful both for plugin development and for runtime regression test - starts Fluent Bit with a scenario config - captures logs in a per-run results directory - exposes monitoring-based readiness checks -- supports `VALGRIND=1` +- supports `VALGRIND=1` and macOS `LEAKS=1` [`src/utils/test_service.py`](src/utils/test_service.py) @@ -567,6 +568,24 @@ Run tests with a simple checkbox progress view: ./tests/fluent-bit-test-suite/run_tests.py ``` +Run test files in parallel with pytest-xdist: + +```bash +./tests/integration/run_tests.py -n 4 --dist loadfile +``` + +Use file-level distribution because tests in the same module can share helper +servers and in-memory capture state. + +Do not combine macOS `leaks` checks with multiple xdist workers. Concurrent +`leaks` processes can fail to acquire Mach task ports. Run functional tests in +parallel, then run memory verification serially: + +```bash +./tests/integration/run_tests.py -n 4 --dist loadfile +./tests/integration/run_tests.py --leaks-strict +``` + Run a subset: ```bash @@ -592,3 +611,36 @@ Require Valgrind-clean runs: VALGRIND=1 VALGRIND_STRICT=1 \ ./tests/fluent-bit-test-suite/.venv/bin/pytest -q tests/fluent-bit-test-suite ``` + +Run a focused scenario with macOS `leaks`: + +```bash +./tests/integration/run_tests.py --leaks \ + scenarios/out_stdout/tests/test_out_stdout_001.py \ + -k default_format +``` + +Fail the test when `leaks` detects unreachable allocations or encounters a +profiling error: + +```bash +./tests/integration/run_tests.py --leaks-strict \ + scenarios/out_stdout/tests/test_out_stdout_001.py \ + -k default_format +``` + +The equivalent environment variables are `LEAKS=1` and +`LEAKS_STRICT=1`. Each Fluent Bit run writes a `leaks.log` report under +`tests/integration/results/`. + +The harness launches `leaks --atExit` as a supervisor and records the PID of +its Fluent Bit child. During teardown it sends `SIGTERM` to Fluent Bit, waits +for normal engine shutdown, and then waits for `leaks` to finish its report. +Do not signal the `leaks` supervisor directly because that can prevent report +generation. + +The macOS runner must allow `/usr/bin/leaks` to acquire the Fluent Bit Mach +task port. Debug builds may need to be signed with the +`com.apple.security.get-task-allow` entitlement. The `leaks` tool detects +unreachable allocations; use AddressSanitizer or malloc diagnostics for +invalid accesses and heap corruption. diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 2d820825e46..b75fc5692a4 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -19,6 +19,7 @@ import logging import pytest +from utils.fluent_bit_manager import fluent_bit_binary_version from utils.test_service import stop_active_services # Configure logging @@ -48,9 +49,11 @@ def load_global_config(): def pytest_configure(config): logger.info("Configuring pytest") + @pytest.hookimpl(tryfirst=True) def pytest_sessionstart(session): logger.info("Starting pytest session") + fluent_bit_binary_version() #flb = FluentBitManager(GLOBAL_CONFIG['fluent_bit']['config_path']) #flb = FluentBitManager() diff --git a/tests/integration/pytest.ini b/tests/integration/pytest.ini index 61bed84f7cf..0d192c87caf 100644 --- a/tests/integration/pytest.ini +++ b/tests/integration/pytest.ini @@ -21,4 +21,3 @@ log_cli = 1 log_cli_level = INFO log_cli_format = %(asctime)s - %(levelname)s - %(message)s log_cli_date_format = %Y-%m-%d %H:%M:%S - diff --git a/tests/integration/requirements.txt b/tests/integration/requirements.txt index 12406e8a147..1c8fc8099d2 100644 --- a/tests/integration/requirements.txt +++ b/tests/integration/requirements.txt @@ -29,6 +29,7 @@ packaging==24.1 pluggy==1.5.0 protobuf==4.25.3 pytest==8.2.2 +pytest-xdist==3.8.0 PyYAML==6.0.1 requests==2.32.3 soupsieve==2.5 diff --git a/tests/integration/run_tests.py b/tests/integration/run_tests.py index 40c0c1006f7..8f7186ff2f2 100755 --- a/tests/integration/run_tests.py +++ b/tests/integration/run_tests.py @@ -21,15 +21,19 @@ def _maybe_reexec_in_venv() -> None: if not VENV_PYTHON.is_file(): return - current = Path(sys.executable).resolve() - target = VENV_PYTHON.resolve() + current_prefix = Path(sys.prefix).resolve() + target_prefix = VENV_PYTHON.parent.parent.resolve() - if current == target: + if current_prefix == target_prefix: return env = os.environ.copy() env[REEXEC_ENV] = "1" - os.execve(str(target), [str(target), str(Path(__file__).resolve()), *sys.argv[1:]], env) + os.execve( + str(VENV_PYTHON), + [str(VENV_PYTHON), str(Path(__file__).resolve()), *sys.argv[1:]], + env, + ) _maybe_reexec_in_venv() @@ -243,6 +247,28 @@ def print_collected_tests(nodeids: list[str]) -> None: print(f" [ ] {short_name_from_nodeid(nodeid)}") +def _xdist_uses_multiple_workers(passthrough: list[str]) -> bool: + for index, argument in enumerate(passthrough): + worker_count = None + + if argument in ("-n", "--numprocesses"): + if index + 1 < len(passthrough): + worker_count = passthrough[index + 1] + elif argument.startswith("--numprocesses="): + worker_count = argument.split("=", 1)[1] + elif argument.startswith("-n") and len(argument) > 2: + worker_count = argument[2:] + + if worker_count is not None: + return worker_count not in ("0", "1") + + return False + + +def _memory_check_environment_enabled(variable_name: str) -> bool: + return bool(os.environ.get(variable_name)) + + def parse_args(argv: list[str]) -> tuple[argparse.Namespace, list[str]]: parser = argparse.ArgumentParser( description="List and run the Fluent Bit Python test suite with a simple checkbox progress view." @@ -255,6 +281,16 @@ def parse_args(argv: list[str]) -> tuple[argparse.Namespace, list[str]]: action="store_true", help="Run with VALGRIND=1 and VALGRIND_STRICT=1.", ) + parser.add_argument( + "--leaks", + action="store_true", + help="Run on macOS with LEAKS=1.", + ) + parser.add_argument( + "--leaks-strict", + action="store_true", + help="Run on macOS with LEAKS=1 and LEAKS_STRICT=1.", + ) parser.add_argument( "--quiet", action="store_true", @@ -265,7 +301,27 @@ def parse_args(argv: list[str]) -> tuple[argparse.Namespace, list[str]]: action="store_true", help="Keep pytest live logs enabled instead of the cleaner wrapper view.", ) - return parser.parse_known_args(argv) + args, passthrough = parser.parse_known_args(argv) + + valgrind_requested = ( + args.valgrind + or args.valgrind_strict + or _memory_check_environment_enabled("VALGRIND") + ) + leaks_requested = ( + args.leaks + or args.leaks_strict + or _memory_check_environment_enabled("LEAKS") + ) + if valgrind_requested and leaks_requested: + parser.error("Valgrind and macOS leaks cannot be enabled together") + if leaks_requested and _xdist_uses_multiple_workers(passthrough): + parser.error( + "macOS leaks checks require one pytest worker because concurrent " + "leaks processes can fail to acquire Mach task ports" + ) + + return args, passthrough def main(argv: list[str] | None = None) -> int: @@ -279,6 +335,10 @@ def main(argv: list[str] | None = None) -> int: os.environ["VALGRIND"] = "1" if args.valgrind_strict: os.environ["VALGRIND_STRICT"] = "1" + if args.leaks or args.leaks_strict: + os.environ["LEAKS"] = "1" + if args.leaks_strict: + os.environ["LEAKS_STRICT"] = "1" if args.list_only: collector = CollectPlugin() diff --git a/tests/integration/scenarios/hot_reload_watch/tests/test_hot_reload_watch_001.py b/tests/integration/scenarios/hot_reload_watch/tests/test_hot_reload_watch_001.py index d356ab6f4ed..fe195f87bca 100644 --- a/tests/integration/scenarios/hot_reload_watch/tests/test_hot_reload_watch_001.py +++ b/tests/integration/scenarios/hot_reload_watch/tests/test_hot_reload_watch_001.py @@ -26,8 +26,11 @@ from google.protobuf import json_format from opentelemetry.proto.collector.logs.v1.logs_service_pb2 import ExportLogsServiceRequest -from src.server.otlp_server import data_storage, otlp_server_run -from src.utils.fluent_bit_manager import FluentBitManager +from src.server.otlp_server import data_storage, otlp_server_run, stop_otlp_server +from src.utils.fluent_bit_manager import ( + FluentBitManager, + fluent_bit_binary_supports_config_property, +) from src.utils.network import find_available_port logger = logging.getLogger(__name__) @@ -67,10 +70,14 @@ def start(self): self.flb.start() def stop(self): - if getattr(self, "flb", None) is not None and self.flb.process is not None: - self.flb.stop() - requests.post(f"http://127.0.0.1:{self.test_suite_http_port}/shutdown") - shutil.rmtree(self.runtime_dir, ignore_errors=True) + try: + if getattr(self, "flb", None) is not None and self.flb.process is not None: + self.flb.stop() + finally: + try: + stop_otlp_server() + finally: + shutil.rmtree(self.runtime_dir, ignore_errors=True) def wait_for_log_count(self, expected_count, timeout=15): start_time = time.time() @@ -99,7 +106,25 @@ def assert_reload_result(service): assert service.read_message(1) == "after" +def test_cleanup_removes_runtime_dir_when_otlp_stop_fails(monkeypatch): + service = Service("fluent-bit-before.yaml", "fluent-bit-after.yaml") + runtime_dir = service.runtime_dir + + def fail_to_stop_otlp_server(): + raise RuntimeError("OTLP server did not stop") + + monkeypatch.setitem(globals(), "stop_otlp_server", fail_to_stop_otlp_server) + + with pytest.raises(RuntimeError, match="OTLP server did not stop"): + service.stop() + + assert not os.path.exists(runtime_dir) + + def test_hot_reload_watch_yaml_config_change(): + if not fluent_bit_binary_supports_config_property("hot_reload.watch"): + pytest.skip("hot_reload.watch is not supported by the selected Fluent Bit binary") + service = Service("fluent-bit-before.yaml", "fluent-bit-after.yaml") try: diff --git a/tests/integration/scenarios/in_elasticsearch/tests/test_in_elasticsearch_001.py b/tests/integration/scenarios/in_elasticsearch/tests/test_in_elasticsearch_001.py index f0ca62462c8..b8f41819bea 100644 --- a/tests/integration/scenarios/in_elasticsearch/tests/test_in_elasticsearch_001.py +++ b/tests/integration/scenarios/in_elasticsearch/tests/test_in_elasticsearch_001.py @@ -36,6 +36,10 @@ }, ] +# A deliberately truncated response can surface as an HTTP/2 framing error (16), +# a send error (55), or a receive error (56), depending on curl and the protocol. +SMALL_BUFFER_CURL_EXIT_CODES = {0, 16, 55, 56} + def parse_single_item_response(response_text): payload = json.loads(response_text) @@ -325,7 +329,7 @@ def test_in_elasticsearch_bulk_small_status_buffer_does_not_crash(case): # The regression condition is process termination from a reachable double-free. # Response formatting can still vary under constrained buffer settings, but the # parser must not take down the process. - assert curl_result.returncode in {0, 55, 56} + assert curl_result.returncode in SMALL_BUFFER_CURL_EXIT_CODES assert health_result["status_code"] == 200 diff --git a/tests/integration/scenarios/in_forward/config/in_forward_pause.yaml b/tests/integration/scenarios/in_forward/config/in_forward_pause.yaml new file mode 100644 index 00000000000..f8785562817 --- /dev/null +++ b/tests/integration/scenarios/in_forward/config/in_forward_pause.yaml @@ -0,0 +1,21 @@ +service: + flush: 60 + grace: 1 + log_level: trace + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} + +pipeline: + inputs: + - name: forward + listen: 127.0.0.1 + port: ${FLUENT_BIT_TEST_LISTENER_PORT} + mem_buf_limit: 1K + + outputs: + - name: http + match: test + host: 127.0.0.1 + port: ${TEST_SUITE_HTTP_PORT} + uri: /data + format: json diff --git a/tests/integration/scenarios/in_forward/config/in_forward_secure_small_buffer.yaml b/tests/integration/scenarios/in_forward/config/in_forward_secure_small_buffer.yaml new file mode 100644 index 00000000000..e8755a93b4d --- /dev/null +++ b/tests/integration/scenarios/in_forward/config/in_forward_secure_small_buffer.yaml @@ -0,0 +1,25 @@ +service: + flush: 1 + grace: 1 + log_level: info + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} + +pipeline: + inputs: + - name: forward + listen: 127.0.0.1 + port: ${FLUENT_BIT_TEST_LISTENER_PORT} + buffer_chunk_size: 64 + buffer_max_size: 1K + self_hostname: server-node + shared_key: shared-secret + security.users: alice s3cr3t + + outputs: + - name: http + match: test + host: 127.0.0.1 + port: ${TEST_SUITE_HTTP_PORT} + uri: /data + format: json diff --git a/tests/integration/scenarios/in_forward/config/in_forward_storage_limit_multi_output.yaml b/tests/integration/scenarios/in_forward/config/in_forward_storage_limit_multi_output.yaml index cce440ef5f5..a0025d079ef 100644 --- a/tests/integration/scenarios/in_forward/config/in_forward_storage_limit_multi_output.yaml +++ b/tests/integration/scenarios/in_forward/config/in_forward_storage_limit_multi_output.yaml @@ -32,4 +32,4 @@ pipeline: format: json json_date_key: false retry_limit: false - storage.total_limit_size: 10K + storage.total_limit_size: ${FORWARD_STORAGE_TOTAL_LIMIT_SIZE} diff --git a/tests/integration/scenarios/in_forward/config/in_forward_storage_limit_shared_success_output.yaml b/tests/integration/scenarios/in_forward/config/in_forward_storage_limit_shared_success_output.yaml index 9f0a6a7798a..0752173b6df 100644 --- a/tests/integration/scenarios/in_forward/config/in_forward_storage_limit_shared_success_output.yaml +++ b/tests/integration/scenarios/in_forward/config/in_forward_storage_limit_shared_success_output.yaml @@ -31,4 +31,4 @@ pipeline: format: json json_date_key: false retry_limit: false - storage.total_limit_size: 10K + storage.total_limit_size: ${FORWARD_STORAGE_TOTAL_LIMIT_SIZE} diff --git a/tests/integration/scenarios/in_forward/config/in_forward_storage_limit_single_output.yaml b/tests/integration/scenarios/in_forward/config/in_forward_storage_limit_single_output.yaml index cd89701471b..bd491587d7f 100644 --- a/tests/integration/scenarios/in_forward/config/in_forward_storage_limit_single_output.yaml +++ b/tests/integration/scenarios/in_forward/config/in_forward_storage_limit_single_output.yaml @@ -22,4 +22,4 @@ pipeline: format: json json_date_key: false retry_limit: false - storage.total_limit_size: 10K + storage.total_limit_size: ${FORWARD_STORAGE_TOTAL_LIMIT_SIZE} diff --git a/tests/integration/scenarios/in_forward/config/in_forward_threaded.yaml b/tests/integration/scenarios/in_forward/config/in_forward_threaded.yaml new file mode 100644 index 00000000000..fff1aa23a7f --- /dev/null +++ b/tests/integration/scenarios/in_forward/config/in_forward_threaded.yaml @@ -0,0 +1,21 @@ +service: + flush: 1 + grace: 1 + log_level: trace + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} + +pipeline: + inputs: + - name: forward + listen: 127.0.0.1 + port: ${FLUENT_BIT_TEST_LISTENER_PORT} + threaded: true + + outputs: + - name: http + match: test + host: 127.0.0.1 + port: ${TEST_SUITE_HTTP_PORT} + uri: /data + format: json diff --git a/tests/integration/scenarios/in_forward/config/in_forward_threaded_pause.yaml b/tests/integration/scenarios/in_forward/config/in_forward_threaded_pause.yaml new file mode 100644 index 00000000000..e129d239f2c --- /dev/null +++ b/tests/integration/scenarios/in_forward/config/in_forward_threaded_pause.yaml @@ -0,0 +1,22 @@ +service: + flush: 60 + grace: 1 + log_level: trace + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} + +pipeline: + inputs: + - name: forward + listen: 127.0.0.1 + port: ${FLUENT_BIT_TEST_LISTENER_PORT} + mem_buf_limit: 1K + threaded: true + + outputs: + - name: http + match: test + host: 127.0.0.1 + port: ${TEST_SUITE_HTTP_PORT} + uri: /data + format: json diff --git a/tests/integration/scenarios/in_forward/config/in_forward_tls_pause.yaml b/tests/integration/scenarios/in_forward/config/in_forward_tls_pause.yaml new file mode 100644 index 00000000000..ad722501f60 --- /dev/null +++ b/tests/integration/scenarios/in_forward/config/in_forward_tls_pause.yaml @@ -0,0 +1,24 @@ +service: + flush: 60 + grace: 1 + log_level: trace + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} + +pipeline: + inputs: + - name: forward + listen: 127.0.0.1 + port: ${FLUENT_BIT_TEST_LISTENER_PORT} + mem_buf_limit: 1K + tls: on + tls.crt_file: ${CERTIFICATE_TEST} + tls.key_file: ${PRIVATE_KEY_TEST} + + outputs: + - name: http + match: test + host: 127.0.0.1 + port: ${TEST_SUITE_HTTP_PORT} + uri: /data + format: json diff --git a/tests/integration/scenarios/in_forward/config/in_forward_tls_threaded.yaml b/tests/integration/scenarios/in_forward/config/in_forward_tls_threaded.yaml new file mode 100644 index 00000000000..7605b8f9e97 --- /dev/null +++ b/tests/integration/scenarios/in_forward/config/in_forward_tls_threaded.yaml @@ -0,0 +1,24 @@ +service: + flush: 1 + grace: 1 + log_level: info + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} + +pipeline: + inputs: + - name: forward + listen: 127.0.0.1 + port: ${FLUENT_BIT_TEST_LISTENER_PORT} + threaded: true + tls: on + tls.crt_file: ${CERTIFICATE_TEST} + tls.key_file: ${PRIVATE_KEY_TEST} + + outputs: + - name: http + match: test + host: 127.0.0.1 + port: ${TEST_SUITE_HTTP_PORT} + uri: /data + format: json diff --git a/tests/integration/scenarios/in_forward/config/in_forward_tls_threaded_pause.yaml b/tests/integration/scenarios/in_forward/config/in_forward_tls_threaded_pause.yaml new file mode 100644 index 00000000000..b4fd446ccd7 --- /dev/null +++ b/tests/integration/scenarios/in_forward/config/in_forward_tls_threaded_pause.yaml @@ -0,0 +1,25 @@ +service: + flush: 60 + grace: 1 + log_level: trace + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} + +pipeline: + inputs: + - name: forward + listen: 127.0.0.1 + port: ${FLUENT_BIT_TEST_LISTENER_PORT} + mem_buf_limit: 1K + threaded: true + tls: on + tls.crt_file: ${CERTIFICATE_TEST} + tls.key_file: ${PRIVATE_KEY_TEST} + + outputs: + - name: http + match: test + host: 127.0.0.1 + port: ${TEST_SUITE_HTTP_PORT} + uri: /data + format: json diff --git a/tests/integration/scenarios/in_forward/config/in_forward_tls_workers.yaml b/tests/integration/scenarios/in_forward/config/in_forward_tls_workers.yaml new file mode 100644 index 00000000000..39c0b510051 --- /dev/null +++ b/tests/integration/scenarios/in_forward/config/in_forward_tls_workers.yaml @@ -0,0 +1,24 @@ +service: + flush: 1 + grace: 1 + log_level: info + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} + +pipeline: + inputs: + - name: forward + listen: 127.0.0.1 + port: ${FLUENT_BIT_TEST_LISTENER_PORT} + tls: on + tls.crt_file: ${CERTIFICATE_TEST} + tls.key_file: ${PRIVATE_KEY_TEST} + workers: 4 + + outputs: + - name: http + match: test + host: 127.0.0.1 + port: ${TEST_SUITE_HTTP_PORT} + uri: /data + format: json diff --git a/tests/integration/scenarios/in_forward/config/in_forward_tls_workers_timeout.yaml b/tests/integration/scenarios/in_forward/config/in_forward_tls_workers_timeout.yaml new file mode 100644 index 00000000000..1860a2f5107 --- /dev/null +++ b/tests/integration/scenarios/in_forward/config/in_forward_tls_workers_timeout.yaml @@ -0,0 +1,25 @@ +service: + flush: 1 + grace: 1 + log_level: trace + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} + +pipeline: + inputs: + - name: forward + listen: 127.0.0.1 + port: ${FLUENT_BIT_TEST_LISTENER_PORT} + net.io_timeout: 2 + tls: on + tls.crt_file: ${CERTIFICATE_TEST} + tls.key_file: ${PRIVATE_KEY_TEST} + workers: 4 + + outputs: + - name: http + match: test + host: 127.0.0.1 + port: ${TEST_SUITE_HTTP_PORT} + uri: /data + format: json diff --git a/tests/integration/scenarios/in_forward/config/in_forward_workers.yaml b/tests/integration/scenarios/in_forward/config/in_forward_workers.yaml new file mode 100644 index 00000000000..f199f862c74 --- /dev/null +++ b/tests/integration/scenarios/in_forward/config/in_forward_workers.yaml @@ -0,0 +1,21 @@ +service: + flush: 1 + grace: 1 + log_level: info + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} + +pipeline: + inputs: + - name: forward + listen: 127.0.0.1 + port: ${FLUENT_BIT_TEST_LISTENER_PORT} + workers: 4 + + outputs: + - name: http + match: test + host: 127.0.0.1 + port: ${TEST_SUITE_HTTP_PORT} + uri: /data + format: json diff --git a/tests/integration/scenarios/in_forward/config/in_forward_workers_timeout.yaml b/tests/integration/scenarios/in_forward/config/in_forward_workers_timeout.yaml new file mode 100644 index 00000000000..17cc683b02d --- /dev/null +++ b/tests/integration/scenarios/in_forward/config/in_forward_workers_timeout.yaml @@ -0,0 +1,22 @@ +service: + flush: 1 + grace: 1 + log_level: trace + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} + +pipeline: + inputs: + - name: forward + listen: 127.0.0.1 + port: ${FLUENT_BIT_TEST_LISTENER_PORT} + net.io_timeout: 2 + workers: 4 + + outputs: + - name: http + match: test + host: 127.0.0.1 + port: ${TEST_SUITE_HTTP_PORT} + uri: /data + format: json diff --git a/tests/integration/scenarios/in_forward/config/in_forward_workers_tiny_ingress_queue.yaml b/tests/integration/scenarios/in_forward/config/in_forward_workers_tiny_ingress_queue.yaml new file mode 100644 index 00000000000..42aa515fb71 --- /dev/null +++ b/tests/integration/scenarios/in_forward/config/in_forward_workers_tiny_ingress_queue.yaml @@ -0,0 +1,22 @@ +service: + flush: 1 + grace: 1 + log_level: info + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} + +pipeline: + inputs: + - name: forward + listen: 127.0.0.1 + port: ${FLUENT_BIT_TEST_LISTENER_PORT} + workers: 4 + mem_buf_limit: 256 + + outputs: + - name: http + match: test + host: 127.0.0.1 + port: ${TEST_SUITE_HTTP_PORT} + uri: /data + format: json diff --git a/tests/integration/scenarios/in_forward/tests/test_in_forward_001.py b/tests/integration/scenarios/in_forward/tests/test_in_forward_001.py index 0f5ef1db9e3..88132172b8a 100644 --- a/tests/integration/scenarios/in_forward/tests/test_in_forward_001.py +++ b/tests/integration/scenarios/in_forward/tests/test_in_forward_001.py @@ -1,6 +1,7 @@ import gzip import hashlib import json +import mmap import os import shutil import socket @@ -10,6 +11,7 @@ import tempfile import time import uuid +from concurrent.futures import ThreadPoolExecutor from pathlib import Path import opentelemetry @@ -18,6 +20,8 @@ import pytest import requests +from utils.memory_check import memory_check_enabled + VENDORED_OTEL_PROTO_ROOT = Path(__file__).resolve().parents[3] / "vendor" VENDORED_OTEL_PACKAGE_ROOT = VENDORED_OTEL_PROTO_ROOT / "opentelemetry" VENDORED_OTEL_PROTO_PACKAGE_ROOT = VENDORED_OTEL_PACKAGE_ROOT / "proto" @@ -45,6 +49,7 @@ ) from server.http_server import configure_http_response, data_storage, http_server_run from utils.data_utils import read_file, read_json_file +from utils.fluent_bit_manager import fluent_bit_input_supports_config_property from utils.test_service import FluentBitTestService @@ -58,6 +63,7 @@ class Service: def __init__(self, config_file, *, use_unix_socket=False): + self.config_name = config_file self.config_file = os.path.abspath(os.path.join(os.path.dirname(__file__), "../config", config_file)) self.use_unix_socket = use_unix_socket self.socket_path = None @@ -71,7 +77,10 @@ def __init__(self, config_file, *, use_unix_socket=False): } if use_unix_socket: - self.socket_path = os.path.join(tempfile.gettempdir(), f"fluent_bit_forward_{uuid.uuid4().hex}.sock") + socket_dir = "/tmp" if sys.platform == "darwin" else tempfile.gettempdir() + self.socket_path = os.path.join( + socket_dir, f"flb-fw-{uuid.uuid4().hex[:12]}.sock" + ) extra_env["FORWARD_UNIX_PATH"] = self.socket_path self.service = FluentBitTestService( @@ -103,7 +112,12 @@ def _stop_receiver(self, service): pass def start(self): + if ("workers" in self.config_name and + not fluent_bit_input_supports_config_property("forward", "workers")): + pytest.skip("forward.workers is not supported by the selected Fluent Bit binary") + self.service.start() + self.flb = self.service.flb self.flb_listener_port = self.service.flb_listener_port def stop(self): @@ -136,6 +150,9 @@ def wait_for_log_contains(self, text, timeout=10): description=f"log text {text!r}", ) + def wait_for_log_message(self, pattern, timeout=10): + return self.wait_for_log_contains(pattern, timeout) + class StorageLimitService(Service): def __init__(self, config_file): @@ -144,6 +161,9 @@ def __init__(self, config_file): self.file_output_path = tempfile.mkdtemp(prefix="fluent_bit_forward_file_output_") self.service.extra_env["FORWARD_STORAGE_PATH"] = self.storage_path self.service.extra_env["FORWARD_FILE_OUTPUT_PATH"] = self.file_output_path + self.service.extra_env["FORWARD_STORAGE_TOTAL_LIMIT_SIZE"] = str( + mmap.PAGESIZE * 2 + ) def stop(self): try: @@ -164,7 +184,17 @@ def chunk_file_contents(self): if not stream_dir.exists(): return [] - return [path.read_bytes() for path in stream_dir.rglob("*.flb") if path.is_file()] + contents = [] + + for path in stream_dir.rglob("*.flb"): + try: + if path.is_file(): + contents.append(path.read_bytes()) + except FileNotFoundError: + # Chunk eviction can remove a file between discovery and open. + continue + + return contents def file_output_contents(self): output_dir = Path(self.file_output_path) @@ -643,18 +673,100 @@ def _send_unix_payload(path, payload): def _send_tls_payload(port, payload, cafile): context = ssl.create_default_context(cafile=cafile) - with socket.create_connection(("127.0.0.1", port), timeout=5) as raw_sock: + with socket.create_connection(("127.0.0.1", port), timeout=20) as raw_sock: with context.wrap_socket(raw_sock, server_hostname="localhost") as tls_sock: tls_sock.sendall(payload) +def _create_tls_memory_bio_client(port, cafile): + context = ssl.create_default_context(cafile=cafile) + incoming = ssl.MemoryBIO() + outgoing = ssl.MemoryBIO() + tls = context.wrap_bio(incoming, outgoing, server_hostname="localhost") + raw_sock = socket.create_connection(("127.0.0.1", port), timeout=5) + + while True: + try: + tls.do_handshake() + break + except ssl.SSLWantReadError: + encrypted = outgoing.read() + if encrypted: + raw_sock.sendall(encrypted) + + encrypted = raw_sock.recv(65536) + assert encrypted + incoming.write(encrypted) + except ssl.SSLWantWriteError: + encrypted = outgoing.read() + if encrypted: + raw_sock.sendall(encrypted) + + encrypted = outgoing.read() + if encrypted: + raw_sock.sendall(encrypted) + + return raw_sock, tls, outgoing + + +def _create_partial_forward_client(service, payload, use_tls): + if use_tls: + sock, tls, outgoing = _create_tls_memory_bio_client( + service.flb_listener_port, + service.tls_crt_file, + ) + tls.write(payload) + wire_payload = outgoing.read() + + def send_payload(next_payload): + tls.write(next_payload) + sock.sendall(outgoing.read()) + else: + sock = socket.create_connection( + ("127.0.0.1", service.flb_listener_port), + timeout=5, + ) + wire_payload = payload + + def send_payload(next_payload): + sock.sendall(next_payload) + + assert len(wire_payload) > 1 + sock.sendall(wire_payload[:-1]) + + return sock, wire_payload[-1:], send_payload + + +def _send_forward_payload(service, payload, use_tls): + if use_tls: + _send_tls_payload( + service.flb_listener_port, + payload, + service.tls_crt_file, + ) + else: + _send_tcp_payload(service.flb_listener_port, payload) + + def _recv_msgpack_value(sock): sock.settimeout(5) - data = sock.recv(4096) - assert data - value, offset = _unpack_obj(data, 0) - assert offset == len(data) - return value + data = bytearray() + + while True: + chunk = sock.recv(4096) + assert chunk + data.extend(chunk) + + try: + value, offset = _unpack_obj(data, 0) + except IndexError: + continue + + if offset > len(data): + continue + + assert offset == len(data) + return value def _decode_str_like(raw): @@ -842,6 +954,154 @@ def test_in_forward_forward_mode_multiple_entries(): assert [record["message"] for record in records[:2]] == ["entry-1", "entry-2"] +def test_in_forward_workers_concurrent_message_mode_records(): + total_records = 16 + service = Service("in_forward_workers.yaml") + service.start() + + try: + service.wait_for_log_message("with 4 workers", timeout=10) + payloads = [ + _message_mode_payload(TEST_TAG, {"message": f"worker-{i}", "value": i}) + for i in range(total_records) + ] + with ThreadPoolExecutor(max_workers=total_records) as executor: + list(executor.map( + lambda payload: _send_tcp_payload(service.flb_listener_port, payload), + payloads, + )) + records = service.wait_for_record_count(total_records, timeout=20) + finally: + service.stop() + + values = sorted(record["value"] for record in records) + assert values == list(range(total_records)) + + +def test_in_forward_workers_forward_mode_batch(): + total_records = 256 + chunk = "workers-forward-batch-001" + service = Service("in_forward_workers.yaml") + service.start() + + try: + service.wait_for_log_message("with 4 workers", timeout=10) + payload = _pack_obj([ + TEST_TAG, + [ + [ + TEST_TS, + {"message": f"forward-batch-{index}", "value": index}, + ] + for index in range(total_records) + ], + {"chunk": chunk}, + ]) + with socket.create_connection( + ("127.0.0.1", service.flb_listener_port), timeout=5 + ) as sock: + sock.sendall(payload) + ack = _recv_msgpack_value(sock) + records = service.wait_for_record_count(total_records, timeout=20) + finally: + service.stop() + + assert ack == {"ack": chunk} + assert len(records) == total_records + assert [record["value"] for record in records[:total_records]] == list( + range(total_records) + ) + + +def test_in_forward_workers_forward_mode_batch_respects_ingress_queue_byte_limit(): + total_records = 8 + chunk = "workers-forward-limited-batch-001" + service = Service("in_forward_workers_tiny_ingress_queue.yaml") + service.start() + + try: + service.wait_for_log_message("with 4 workers", timeout=10) + payload = _pack_obj([ + TEST_TAG, + [ + [ + TEST_TS, + {"message": f"limited-batch-{index}", "value": index}, + ] + for index in range(total_records) + ], + {"chunk": chunk}, + ]) + with socket.create_connection( + ("127.0.0.1", service.flb_listener_port), timeout=5 + ) as sock: + sock.sendall(payload) + ack = _recv_msgpack_value(sock) + records = service.wait_for_record_count(total_records, timeout=20) + finally: + service.stop() + + assert ack == {"ack": chunk} + assert len(records) == total_records + assert [record["value"] for record in records] == list(range(total_records)) + + +def test_in_forward_workers_forward_mode_preserves_valid_prefix(): + service = Service("in_forward_workers.yaml") + service.start() + + try: + service.wait_for_log_message("with 4 workers", timeout=10) + payload = _pack_obj([ + TEST_TAG, + [ + [TEST_TS, {"message": "valid-prefix"}], + [TEST_TS], + ], + ]) + _send_tcp_payload(service.flb_listener_port, payload) + service.wait_for_log_message("event decoder or encoder failure", timeout=20) + service.wait_for_record_count(1, timeout=20) + finally: + service.stop() + + records = service.flattened_records() + assert len(records) == 1 + assert records[0]["message"] == "valid-prefix" + + +def test_in_forward_workers_drop_partial_connections_and_continue(): + dropped_connections = 8 + valid_records = 8 + service = Service("in_forward_workers.yaml") + service.start() + + try: + service.wait_for_log_message("with 4 workers", timeout=10) + # Send partial MessagePack bytes and close to exercise drop cleanup. + with ThreadPoolExecutor(max_workers=dropped_connections) as executor: + list(executor.map( + lambda _: _send_tcp_payload(service.flb_listener_port, b"\x93\xa4test"), + range(dropped_connections), + )) + + payloads = [ + _message_mode_payload(TEST_TAG, {"message": f"valid-{i}", "value": i}) + for i in range(valid_records) + ] + with ThreadPoolExecutor(max_workers=valid_records) as executor: + list(executor.map( + lambda payload: _send_tcp_payload(service.flb_listener_port, payload), + payloads, + )) + records = service.wait_for_record_count(valid_records, timeout=20) + finally: + service.stop() + + values = sorted(record["value"] for record in records) + assert values == list(range(valid_records)) + + def test_in_forward_packed_forward_gzip(): service = Service("in_forward.yaml") service.start() @@ -1027,6 +1287,280 @@ def test_in_forward_tls_message_mode(): assert records[0]["message"] == "tls-message" +@pytest.mark.parametrize( + "config_file,use_tls,worker_log", + [ + ("in_forward.yaml", False, None), + ("in_forward_threaded.yaml", False, None), + ("in_forward_workers.yaml", False, "with 4 workers"), + ("in_forward_tls.yaml", True, None), + ("in_forward_tls_threaded.yaml", True, None), + ("in_forward_tls_workers.yaml", True, "with 4 workers"), + ], +) +def test_in_forward_downstream_coro_partial_record_keeps_listener_responsive( + config_file, + use_tls, + worker_log, +): + service = Service(config_file) + service.start() + + partial_sock = None + + try: + if worker_log is not None: + service.wait_for_log_message(worker_log, timeout=10) + + partial_payload = _message_mode_payload( + TEST_TAG, + {"message": "partial-coroutine-record"}, + ) + partial_sock, final_byte, send_on_partial_connection = ( + _create_partial_forward_client( + service, + partial_payload, + use_tls, + ) + ) + time.sleep(0.5) + + responsive_payload = _message_mode_payload( + TEST_TAG, + {"message": "second-connection-remains-responsive"}, + ) + _send_forward_payload( + service, + responsive_payload, + use_tls, + ) + records = service.wait_for_record_count(1, timeout=10) + + assert records[0]["message"] == "second-connection-remains-responsive" + + partial_sock.sendall(final_byte) + records = service.wait_for_record_count(2, timeout=10) + + restored_payload = _message_mode_payload( + TEST_TAG, + {"message": "same-connection-after-resume"}, + ) + send_on_partial_connection(restored_payload) + records = service.wait_for_record_count(3, timeout=10) + finally: + if partial_sock is not None: + partial_sock.close() + service.stop() + + assert any( + record.get("message") == "partial-coroutine-record" + for record in records + ) + assert any( + record.get("message") == "same-connection-after-resume" + for record in records + ) + + +@pytest.mark.parametrize( + "config_file,use_tls", + [ + ("in_forward.yaml", False), + ("in_forward_threaded.yaml", False), + ("in_forward_workers.yaml", False), + ("in_forward_tls.yaml", True), + ("in_forward_tls_threaded.yaml", True), + ("in_forward_tls_workers.yaml", True), + ], +) +def test_in_forward_downstream_coro_eof_and_error_keep_listener_responsive( + config_file, + use_tls, +): + service = Service(config_file) + service.start() + + try: + if use_tls: + eof_sock, _, _ = _create_tls_memory_bio_client( + service.flb_listener_port, + service.tls_crt_file, + ) + else: + eof_sock = socket.create_connection( + ("127.0.0.1", service.flb_listener_port), + timeout=5, + ) + eof_sock.close() + + if use_tls: + error_sock, _, _ = _create_tls_memory_bio_client( + service.flb_listener_port, + service.tls_crt_file, + ) + error_sock.sendall(b"\x17\x03\x03\x00\x06broken") + error_sock.close() + else: + _send_tcp_payload(service.flb_listener_port, b"\xc1") + + payload = _message_mode_payload(TEST_TAG, {"message": "after-io-errors"}) + _send_forward_payload(service, payload, use_tls) + records = service.wait_for_record_count(1, timeout=10) + finally: + service.stop() + + assert records[0]["message"] == "after-io-errors" + + +@pytest.mark.parametrize( + "config_file,use_tls", + [ + ("in_forward.yaml", False), + ("in_forward_threaded.yaml", False), + ("in_forward_workers.yaml", False), + ("in_forward_tls.yaml", True), + ("in_forward_tls_threaded.yaml", True), + ("in_forward_tls_workers.yaml", True), + ], +) +def test_in_forward_downstream_coro_partial_record_shutdown_is_clean( + config_file, + use_tls, +): + service = Service(config_file) + service.start() + process = service.flb.process + + partial_sock = None + + try: + payload = _message_mode_payload(TEST_TAG, {"message": "shutdown-partial"}) + partial_sock, _, _ = _create_partial_forward_client( + service, + payload, + use_tls, + ) + time.sleep(0.5) + finally: + service.stop() + if partial_sock is not None: + partial_sock.close() + + assert process.returncode == 0 + + +@pytest.mark.parametrize( + "config_file,use_tls", + [ + ("in_forward_pause.yaml", False), + ("in_forward_threaded_pause.yaml", False), + ("in_forward_tls_pause.yaml", True), + ("in_forward_tls_threaded_pause.yaml", True), + ], +) +def test_in_forward_downstream_coro_pause_unwinds_connections( + config_file, + use_tls, +): + service = Service(config_file) + service.start() + process = service.flb.process + + partial_sock = None + + try: + log_file = service.service.flb.log_file + drop_log = "drop connection fd=" + dropped_before_pause = read_file(log_file).count(drop_log) + + partial_payload = _message_mode_payload(TEST_TAG, {"message": "pause-partial"}) + partial_sock, _, _ = _create_partial_forward_client( + service, + partial_payload, + use_tls, + ) + + large_payload = _message_mode_payload( + TEST_TAG, + {"message": "trigger-pause", "payload": "x" * 4096}, + ) + _send_forward_payload(service, large_payload, use_tls) + service.wait_for_log_contains("paused (mem buf overlimit", timeout=10) + service.service.wait_for_condition( + lambda: read_file(log_file) + if read_file(log_file).count(drop_log) + >= dropped_before_pause + 2 + else None, + timeout=10, + interval=0.2, + description="paused Forward TLS coroutines to unwind", + ) + finally: + service.stop() + if partial_sock is not None: + partial_sock.close() + + assert process.returncode == 0 + + +@pytest.mark.parametrize( + "config_file,use_tls", + [ + ("in_forward_workers_timeout.yaml", False), + ("in_forward_tls_workers_timeout.yaml", True), + ], +) +def test_in_forward_downstream_coro_worker_timeout_unwinds_connection( + config_file, + use_tls, +): + service = Service(config_file) + service.start() + + partial_sock = None + + try: + service.wait_for_log_message("with 4 workers", timeout=10) + log_file = service.service.flb.log_file + drop_log = "drop connection fd=" + dropped_before_timeout = read_file(log_file).count(drop_log) + + payload = _message_mode_payload(TEST_TAG, {"message": "worker-timeout"}) + partial_sock, _, _ = _create_partial_forward_client( + service, + payload, + use_tls, + ) + + service.wait_for_log_contains("timed out after 2 seconds (IO timeout)", timeout=10) + service.service.wait_for_condition( + lambda: read_file(log_file) + if read_file(log_file).count(drop_log) + >= dropped_before_timeout + 1 + else None, + timeout=10, + interval=0.2, + description="timed-out Forward coroutine to unwind", + ) + + responsive_payload = _message_mode_payload( + TEST_TAG, + {"message": "after-worker-timeout"}, + ) + _send_forward_payload( + service, + responsive_payload, + use_tls, + ) + records = service.wait_for_record_count(1, timeout=10) + finally: + if partial_sock is not None: + partial_sock.close() + service.stop() + + assert records[0]["message"] == "after-worker-timeout" + + def test_in_forward_tls_idle_connection_timeout_churn(monkeypatch): """ Regression test for issue #12025. @@ -1079,8 +1613,74 @@ def test_in_forward_tls_idle_connection_timeout_churn(monkeypatch): assert any(record.get("message") == "after-churn" for record in records) -def test_in_forward_secure_forward_auth_success(): - service = Service("in_forward_secure.yaml") +def test_in_forward_tls_workers_concurrent_message_mode_records(): + total_records = 16 + service = Service("in_forward_tls_workers.yaml") + service.start() + + try: + service.wait_for_log_message("with 4 workers", timeout=10) + with ThreadPoolExecutor(max_workers=total_records) as executor: + list(executor.map( + lambda i: _send_tls_payload(service.flb_listener_port, + _message_mode_payload( + TEST_TAG, + {"message": f"tls-worker-{i}", + "value": i}), + service.tls_crt_file), + range(total_records), + )) + records = service.wait_for_record_count(total_records, timeout=20) + finally: + service.stop() + + values = sorted(record["value"] for record in records) + assert values == list(range(total_records)) + + +def test_in_forward_tls_workers_drop_bad_handshakes_and_continue(): + dropped_connections = 8 + valid_records = 8 + service = Service("in_forward_tls_workers.yaml") + service.start() + + try: + service.wait_for_log_message("with 4 workers", timeout=10) + # Send raw non-TLS bytes and close to exercise handshake cleanup. + with ThreadPoolExecutor(max_workers=dropped_connections) as executor: + list(executor.map( + lambda i: _send_tcp_payload(service.flb_listener_port, + f"not-tls-{i}".encode("utf-8")), + range(dropped_connections), + )) + + with ThreadPoolExecutor(max_workers=valid_records) as executor: + list(executor.map( + lambda i: _send_tls_payload(service.flb_listener_port, + _message_mode_payload( + TEST_TAG, + {"message": f"valid-tls-{i}", + "value": i}), + service.tls_crt_file), + range(valid_records), + )) + records = service.wait_for_record_count(valid_records, timeout=20) + finally: + service.stop() + + values = sorted(record["value"] for record in records) + assert values == list(range(valid_records)) + + +@pytest.mark.parametrize( + "config_file", + [ + "in_forward_secure.yaml", + "in_forward_secure_small_buffer.yaml", + ], +) +def test_in_forward_secure_forward_auth_success(config_file): + service = Service(config_file) service.start() try: @@ -1526,7 +2126,7 @@ def multi_output_eviction_snapshot(): def test_in_forward_storage_limit_shared_success_route_deletes_old_chunk(): service = StorageLimitService("in_forward_storage_limit_shared_success_output.yaml") - timeout = 30 if os.environ.get("VALGRIND") else 10 + timeout = 30 if memory_check_enabled() else 10 service.start() try: diff --git a/tests/integration/scenarios/in_http_max_connections/config/in_http_max_connections.yaml b/tests/integration/scenarios/in_http_max_connections/config/in_http_max_connections.yaml index 07459ac71e1..dfbd4efbce8 100644 --- a/tests/integration/scenarios/in_http_max_connections/config/in_http_max_connections.yaml +++ b/tests/integration/scenarios/in_http_max_connections/config/in_http_max_connections.yaml @@ -12,6 +12,7 @@ pipeline: port: ${FLUENT_BIT_TEST_LISTENER_PORT} http_server.idle_timeout: 2s http_server.max_connections: 1 + http_server.workers: 2 outputs: - name: http diff --git a/tests/integration/scenarios/in_opentelemetry/tests/data_files/test_logs_2038.in.json b/tests/integration/scenarios/in_opentelemetry/tests/data_files/test_logs_2038.in.json new file mode 100644 index 00000000000..551d261bd1d --- /dev/null +++ b/tests/integration/scenarios/in_opentelemetry/tests/data_files/test_logs_2038.in.json @@ -0,0 +1,23 @@ +{ + "resource_logs": [ + { + "scope_logs": [ + { + "scope": { + "name": "post-2038" + }, + "log_records": [ + { + "time_unix_nano": "2209072510808241446", + "severity_number": 9, + "severity_text": "INFO", + "body": { + "string_value": "post-2038 timestamp" + } + } + ] + } + ] + } + ] +} diff --git a/tests/integration/scenarios/in_opentelemetry/tests/test_in_opentelemetry_001.py b/tests/integration/scenarios/in_opentelemetry/tests/test_in_opentelemetry_001.py index 083ce9330f9..1d2bcba7c14 100644 --- a/tests/integration/scenarios/in_opentelemetry/tests/test_in_opentelemetry_001.py +++ b/tests/integration/scenarios/in_opentelemetry/tests/test_in_opentelemetry_001.py @@ -678,6 +678,117 @@ def test_opentelemetry_to_opentelemetry_basic_log(): assert item["record_attributes"]["example_key"] == "example_value" +def test_in_opentelemetry_large_protobuf_logs(): + service = Service("001-fluent-bit.yaml") + source = ExportLogsServiceRequest() + source.ParseFromString(service.build_otel_payload("test_logs_001.in.json", "logs")) + + payload = ExportLogsServiceRequest() + payload.CopyFrom(source) + source_records = list(source.resource_logs[0].scope_logs[0].log_records) + + while payload.ByteSize() < 128 * 1024: + payload.resource_logs[0].scope_logs[0].log_records.extend(source_records) + + expected_record_count = sum( + len(scope_logs.log_records) + for resource_logs in payload.resource_logs + for scope_logs in resource_logs.scope_logs + ) + + service.start() + try: + logs_before = len(data_storage["logs"]) + response = service.send_raw_request("/v1/logs", payload.SerializeToString()) + + assert response.status_code == 201 + received_record_count = service.service.wait_for_condition( + lambda: count if ( + count := sum( + len(scope_logs.log_records) + for received in data_storage["logs"][logs_before:] + for resource_logs in received.resource_logs + for scope_logs in resource_logs.scope_logs + ) + ) >= expected_record_count else None, + timeout=20, + interval=0.25, + description="all large protobuf log records", + ) + finally: + service.stop() + + assert received_record_count == expected_record_count + + +def test_in_opentelemetry_batched_protobuf_logs_with_workers(): + service = Service(IN_OPENTELEMETRY_WORKER_PROTOCOL_CONFIGS["http1_cleartext"]) + source = ExportLogsServiceRequest() + source.ParseFromString(service.build_otel_payload("test_logs_001.in.json", "logs")) + + payload = ExportLogsServiceRequest() + payload.CopyFrom(source) + source_records = list(source.resource_logs[0].scope_logs[0].log_records) + + while payload.ByteSize() < 16 * 1024: + payload.resource_logs[0].scope_logs[0].log_records.extend(source_records) + + assert payload.ByteSize() < 64 * 1024 + + expected_record_count = sum( + len(scope_logs.log_records) + for resource_logs in payload.resource_logs + for scope_logs in resource_logs.scope_logs + ) + + service.start() + try: + service.wait_for_log_message("with 4 workers", timeout=10) + logs_before = len(data_storage["logs"]) + response = service.send_raw_request("/v1/logs", payload.SerializeToString()) + + assert response.status_code == 201 + received_record_count = service.service.wait_for_condition( + lambda: count if ( + count := sum( + len(scope_logs.log_records) + for received in data_storage["logs"][logs_before:] + for resource_logs in received.resource_logs + for scope_logs in resource_logs.scope_logs + ) + ) >= expected_record_count else None, + timeout=20, + interval=0.25, + description="all batched protobuf log records", + ) + + metrics_text = service.service.wait_for_condition( + lambda: ( + metrics if maybe_read_prometheus_metric_value( + metrics, + "fluentbit_input_records_total", + "opentelemetry.0", + ) is not None and maybe_read_prometheus_metric_value( + metrics, + "fluentbit_input_records_total", + "opentelemetry.0", + ) >= expected_record_count else None + ) if (metrics := service.scrape_prometheus_metrics()) else None, + timeout=10, + interval=0.25, + description="protobuf input record accounting", + ) + finally: + service.stop() + + assert received_record_count == expected_record_count + assert read_prometheus_metric_value( + metrics_text, + "fluentbit_input_records_total", + "opentelemetry.0", + ) == expected_record_count + + # Start a Fluent Bit Pipeline with Dummy message and then it gets handle by OpenTelemetry output, the config # aims to populate traceId and spanId fields with the values from the Dummy message. # @@ -797,10 +908,14 @@ def test_opentelemetry_to_opentelemetry_parent_child_traces(): def test_in_opentelemetry_rejects_invalid_logs_payload(): service = Service("001-fluent-bit.yaml") service.start() - response = service.send_raw_request("/v1/logs", b"not-a-valid-otlp-payload") - service.stop() + try: + small_response = service.send_raw_request("/v1/logs", b"not-a-valid-otlp-payload") + large_response = service.send_raw_request("/v1/logs", b"\x80" * (64 * 1024)) + finally: + service.stop() - assert response.status_code >= 400 + assert small_response.status_code >= 400 + assert large_response.status_code >= 400 assert len(data_storage["logs"]) == 0 @@ -1258,6 +1373,27 @@ def test_in_opentelemetry_protocol_matrix(case, signal_type, json_input, endpoin assert len(response_payload) > 0 +def test_in_opentelemetry_http2_invalid_endpoint_returns_404(): + service = Service(IN_OPENTELEMETRY_PROTOCOL_CONFIGS["http2_cleartext"]) + service.start() + + result = run_curl_request( + f"http://localhost:{service.flb_listener_port}/v1/invalid", + b"invalid payload", + headers=["Content-Type: application/x-protobuf"], + http_mode="http2-prior-knowledge", + ) + + service.stop() + + assert result["status_code"] == 404 + assert result["http_version"] == "2" + assert result["body"] == "error: invalid endpoint\n" + assert len(data_storage["logs"]) == 0 + assert len(data_storage["metrics"]) == 0 + assert len(data_storage["traces"]) == 0 + + # This test is branch-specific coverage for the generic HTTP listener worker mode. # It does three things: # 1. enables http_server.workers on the in_opentelemetry listener, @@ -1281,12 +1417,13 @@ def test_in_opentelemetry_http_workers_mixed_signal_matrix(case): ("metrics", "test_metrics_001.in.json", "/v1/metrics"), ("traces", "test_traces_001.in.json", "/v1/traces"), ] + worker_count = 4 repeats_per_signal = 4 total_requests = len(request_plan) * repeats_per_signal service = Service(IN_OPENTELEMETRY_WORKER_PROTOCOL_CONFIGS[case["config_key"]]) service.start() - service.wait_for_log_message("with 4 workers", timeout=10) + service.wait_for_log_message(f"with {worker_count} workers", timeout=10) scheme = "https" if case["use_tls"] else "http" request_jobs = [] @@ -1309,7 +1446,7 @@ def send_job(job): ca_cert_path=service.tls_crt_file if case["use_tls"] else None, ) - with ThreadPoolExecutor(max_workers=total_requests) as executor: + with ThreadPoolExecutor(max_workers=worker_count) as executor: results = list(executor.map(send_job, request_jobs)) for result in results: @@ -1497,3 +1634,37 @@ def test_in_opentelemetry_http_workers_respect_ingress_queue_byte_limit(): "fluentbit_input_http_server_ingress_queue_pending_bytes", "opentelemetry.0", ) == 0 + + +def test_in_opentelemetry_http_workers_account_metrics_payload_bytes(): + service = Service(IN_OPENTELEMETRY_TINY_INGRESS_QUEUE_BYTE_LIMIT_CONFIG) + small_payload = service.build_otel_payload("test_metrics_001.in.json", "metrics") + + small_request = ExportMetricsServiceRequest() + small_request.ParseFromString(small_payload) + + large_request = ExportMetricsServiceRequest() + for _ in range(16): + large_request.resource_metrics.extend(small_request.resource_metrics) + + large_payload = large_request.SerializeToString() + + assert len(small_payload) < 2000 + assert len(large_payload) > 2000 + + service.start() + service.wait_for_log_message("with 4 workers", timeout=10) + metrics_before = len(data_storage["metrics"]) + + try: + success_response = service.send_raw_request("/v1/metrics", small_payload) + assert success_response.status_code == 201 + service.wait_for_signal_count("metrics", metrics_before + 1, timeout=10) + + failure_response = service.send_raw_request("/v1/metrics", large_payload) + assert failure_response.status_code == 503 + assert "deferred ingress queue is full" in failure_response.text + finally: + service.stop() + + assert len(data_storage["metrics"]) == metrics_before + 1 diff --git a/tests/integration/scenarios/in_tail/config/tail_ignore_active_older.yaml b/tests/integration/scenarios/in_tail/config/tail_ignore_active_older.yaml index cb813418b4c..bb02cc58a6c 100644 --- a/tests/integration/scenarios/in_tail/config/tail_ignore_active_older.yaml +++ b/tests/integration/scenarios/in_tail/config/tail_ignore_active_older.yaml @@ -14,7 +14,7 @@ pipeline: refresh_interval: 1 watcher_interval: 1 progress_check_interval: 1 - rotate_wait: 6 + rotate_wait: 1 inotify_watcher: false ignore_older: 2s ignore_active_older_files: true diff --git a/tests/integration/scenarios/in_tail/tests/test_in_tail_001.py b/tests/integration/scenarios/in_tail/tests/test_in_tail_001.py index f8b1b58329d..98481509406 100644 --- a/tests/integration/scenarios/in_tail/tests/test_in_tail_001.py +++ b/tests/integration/scenarios/in_tail/tests/test_in_tail_001.py @@ -12,8 +12,22 @@ import requests from server.http_server import data_storage, http_server_run +from utils.fluent_bit_manager import fluent_bit_input_supports_config_property from utils.test_service import FluentBitTestService +skip_on_windows = pytest.mark.skipif( + sys.platform == "win32", + reason="scenario relies on POSIX rotation, symlink, permissions, or inode semantics", +) + + +POLLING_CONFIGS = { + "tail_ignore_active_older.yaml", + "tail_ignore_older.yaml", + "tail_stat.yaml", + "tail_stat_db_compare_filename.yaml", +} + def _timezone_available(iana_zone): if sys.platform == "win32": @@ -44,6 +58,7 @@ def close(self): class Service: def __init__(self, config_file, *, tail_path, db_path): + self.config_name = config_file self.config_file = os.path.abspath( os.path.join(os.path.dirname(__file__), "../config", config_file) ) @@ -84,6 +99,14 @@ def _stop_receiver(self, service): pass def start(self): + if (self.config_name in POLLING_CONFIGS and + not fluent_bit_input_supports_config_property( + "tail", "inotify_watcher" + )): + pytest.skip( + "tail.inotify_watcher is not supported by the selected Fluent Bit binary" + ) + self.service.start() self.flb = self.service.flb @@ -304,6 +327,95 @@ def test_in_tail_discovers_new_files_from_head(workspace): service.stop() +def write_windows_utf8_path_config(path, tail_path, db_path, exclude_path): + path.write_text( + f"""service: + flush: 1 + log_level: debug + http_server: on + http_port: ${{FLUENT_BIT_HTTP_MONITORING_PORT}} + +pipeline: + inputs: + - name: tail + tag: tail.integration + path: {tail_path} + exclude_path: {exclude_path} + windows.path_encoding: utf-8 + read_from_head: true + read_newly_discovered_files_from_head: true + refresh_interval: 1 + watcher_interval: 1 + progress_check_interval: 1 + rotate_wait: 6 + db: {db_path} + db.sync: full + db.journal_mode: DELETE + db.compare_filename: true + path_key: file + offset_key: offset + + outputs: + - name: http + match: tail.integration + host: 127.0.0.1 + port: ${{TEST_SUITE_HTTP_PORT}} + uri: /data + format: json + json_date_key: false +""", + encoding="utf-8", + ) + + +@pytest.mark.skipif(sys.platform != "win32", reason="Windows-only path API mode") +def test_in_tail_windows_utf8_path_encoding_discovers_unicode_file(workspace): + log_dir = workspace / "utf8-\U0001f600" + db_path = workspace / "tail.db" + config_path = workspace / "tail_windows_utf8_path.yaml" + log_dir.mkdir() + + log_file = log_dir / "unicode-\U0001f600.log" + long_log_file = log_dir / (("\U0001f600" * 70) + ".log") + excluded_log_file = log_dir / "excluded-\U0001f600.log" + log_file.write_text("utf8-path-1\n", encoding="utf-8") + long_log_file.write_text("utf8-path-2\n", encoding="utf-8") + excluded_log_file.write_text("must-be-excluded\n", encoding="utf-8") + write_windows_utf8_path_config( + config_path, + log_dir / "*.log", + db_path, + excluded_log_file.name, + ) + + service = Service( + config_path, + tail_path=log_dir / "*.log", + db_path=db_path, + ) + + try: + service.start() + records = service.wait_for_records(2) + service.assert_no_new_records_for(2, quiet_period=3) + finally: + service.stop() + + assert_log_set(records[:2], ["utf8-path-1", "utf8-path-2"]) + record_paths = {record["file"] for record in records[:2]} + assert record_paths == {str(log_file), str(long_log_file)} + + db = sqlite3.connect(db_path) + try: + row = db.execute( + "SELECT name FROM in_tail_files", + ).fetchall() + finally: + db.close() + + assert {item[0] for item in row} == {str(log_file), str(long_log_file)} + + def test_in_tail_newly_discovered_files_can_start_from_tail(workspace): log_dir = workspace / "new-tail" log_dir.mkdir() @@ -356,6 +468,7 @@ def test_in_tail_existing_file_can_start_from_tail_on_startup(workspace): assert records[0]["offset"] > 0 +@skip_on_windows def test_in_tail_follows_rename_rotation(workspace): active_log = workspace / "app.log" db_path = workspace / "tail.db" @@ -393,6 +506,7 @@ def test_in_tail_follows_rename_rotation(workspace): service.stop() +@skip_on_windows def test_in_tail_handles_multiple_rename_rotations(workspace): active_log = workspace / "multi-rotate.log" db_path = workspace / "tail.db" @@ -474,6 +588,7 @@ def test_in_tail_handles_copytruncate_with_stale_writer(workspace): service.stop() +@skip_on_windows def test_in_tail_handles_symlink_target_rotation(workspace): target_one = workspace / "target-one.log" target_two = workspace / "target-two.log" @@ -517,6 +632,7 @@ def test_in_tail_handles_symlink_target_rotation(workspace): service.stop() +@skip_on_windows def test_in_tail_stat_backend_covers_nfs_style_polling(workspace): active_log = workspace / "nfs-style.log" db_path = workspace / "tail.db" @@ -550,6 +666,7 @@ def test_in_tail_stat_backend_covers_nfs_style_polling(workspace): service.stop() +@skip_on_windows def test_in_tail_db_compare_filename_replays_renamed_file_after_restart(workspace): source_log = workspace / "db-source.log" moved_log = workspace / "db-moved.log" @@ -721,6 +838,7 @@ def test_in_tail_truncate_long_lines_emits_truncated_record_and_continues(worksp assert len(truncated[0]) > 0 +@skip_on_windows def test_in_tail_rotate_wait_keeps_old_inode_then_purges_it(workspace): active_log = workspace / "rotate-wait.log" db_path = workspace / "tail.db" @@ -755,6 +873,7 @@ def test_in_tail_rotate_wait_keeps_old_inode_then_purges_it(workspace): service.stop() +@skip_on_windows def test_in_tail_delete_and_recreate_same_path_is_reingested(workspace): active_log = workspace / "recreate.log" db_path = workspace / "tail.db" @@ -781,6 +900,7 @@ def test_in_tail_delete_and_recreate_same_path_is_reingested(workspace): service.stop() +@skip_on_windows def test_in_tail_restart_resumes_from_db_offset(workspace): log_file = workspace / "resume.log" db_path = workspace / "tail.db" @@ -806,6 +926,7 @@ def test_in_tail_restart_resumes_from_db_offset(workspace): second_run.stop() +@skip_on_windows def test_in_tail_copytruncate_across_restart_reads_new_content_only(workspace): log_file = workspace / "restart-copytruncate.log" archived = workspace / "restart-copytruncate.log.1" @@ -834,6 +955,7 @@ def test_in_tail_copytruncate_across_restart_reads_new_content_only(workspace): second_run.stop() +@skip_on_windows def test_in_tail_partial_line_across_restart_is_completed_once(workspace): log_file = workspace / "partial-restart.log" db_path = workspace / "tail.db" @@ -859,6 +981,7 @@ def test_in_tail_partial_line_across_restart_is_completed_once(workspace): assert_log_set(records, ["partial line"]) +@skip_on_windows def test_in_tail_db_schema_upgrade_is_automatic(workspace): log_file = workspace / "schema-upgrade.log" db_path = workspace / "tail.db" @@ -920,6 +1043,7 @@ def test_in_tail_db_schema_upgrade_is_automatic(workspace): assert "offset_marker_size" in columns +@skip_on_windows def test_in_tail_multi_file_rapid_rotation(workspace): log_dir = workspace / "rapid" log_dir.mkdir() @@ -1012,6 +1136,7 @@ def test_in_tail_exclude_path_skips_matching_files(workspace): assert records[0]["file"] == str(keep_file) +@skip_on_windows def test_in_tail_ignore_older_skips_stale_files(workspace): stale_file = workspace / "stale.log" db_path = workspace / "tail.db" @@ -1029,7 +1154,8 @@ def test_in_tail_ignore_older_skips_stale_files(workspace): service.stop() -def test_in_tail_ignore_active_older_files_stops_following_aged_file(workspace): +@skip_on_windows +def test_in_tail_ignore_active_older_files_resumes_updated_file(workspace): log_file = workspace / "active-aged.log" db_path = workspace / "tail.db" @@ -1045,10 +1171,12 @@ def test_in_tail_ignore_active_older_files_stops_following_aged_file(workspace): time.sleep(4) service.assert_no_new_records_for(1, quiet_period=4) write_and_sync(log_file, "second-line\n") - service.assert_no_new_records_for(1, quiet_period=4) + records = service.wait_for_records(2, timeout=20) finally: service.stop() + assert_log_set(records, ["first-line", "second-line"]) + def test_in_tail_docker_mode_parses_and_flushes_docker_json_stream(workspace): docker_file = workspace / "docker.log" @@ -1106,6 +1234,7 @@ def test_in_tail_generic_encoding_shiftjis(workspace): assert_log_set(records, [expected_text]) +@skip_on_windows def test_in_tail_discovers_file_after_permissions_are_restored(workspace): log_dir = workspace / "permissions" log_dir.mkdir() diff --git a/tests/integration/scenarios/in_tcp/config/in_tcp_parser_json_tls_workers.yaml b/tests/integration/scenarios/in_tcp/config/in_tcp_parser_json_tls_workers.yaml new file mode 100644 index 00000000000..719d1d9b12c --- /dev/null +++ b/tests/integration/scenarios/in_tcp/config/in_tcp_parser_json_tls_workers.yaml @@ -0,0 +1,29 @@ +service: + flush: 1 + grace: 1 + log_level: info + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} + parsers_file: ${PARSERS_FILE_TEST} + +pipeline: + inputs: + - name: tcp + tag: target_input + listen: 127.0.0.1 + port: ${FLUENT_BIT_TEST_LISTENER_PORT} + format: none + separator: "\\n" + parser: json + tls: on + tls.crt_file: ${CERTIFICATE_TEST} + tls.key_file: ${PRIVATE_KEY_TEST} + workers: 4 + + outputs: + - name: http + match: target_input + host: 127.0.0.1 + port: ${TEST_SUITE_HTTP_PORT} + uri: /data + format: json diff --git a/tests/integration/scenarios/in_tcp/config/in_tcp_parser_json_workers.yaml b/tests/integration/scenarios/in_tcp/config/in_tcp_parser_json_workers.yaml new file mode 100644 index 00000000000..7455f087fe7 --- /dev/null +++ b/tests/integration/scenarios/in_tcp/config/in_tcp_parser_json_workers.yaml @@ -0,0 +1,26 @@ +service: + flush: 1 + grace: 1 + log_level: info + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} + parsers_file: ${PARSERS_FILE_TEST} + +pipeline: + inputs: + - name: tcp + tag: target_input + listen: 127.0.0.1 + port: ${FLUENT_BIT_TEST_LISTENER_PORT} + format: none + separator: "\\n" + parser: json + workers: 4 + + outputs: + - name: http + match: target_input + host: 127.0.0.1 + port: ${TEST_SUITE_HTTP_PORT} + uri: /data + format: json diff --git a/tests/integration/scenarios/in_tcp/tests/test_in_tcp_001.py b/tests/integration/scenarios/in_tcp/tests/test_in_tcp_001.py index 314bb144464..a525f12bd7e 100644 --- a/tests/integration/scenarios/in_tcp/tests/test_in_tcp_001.py +++ b/tests/integration/scenarios/in_tcp/tests/test_in_tcp_001.py @@ -1,25 +1,36 @@ import os import socket +import ssl +import time +from concurrent.futures import ThreadPoolExecutor +import pytest import requests from server.http_server import data_storage, http_server_run +from utils.fluent_bit_manager import fluent_bit_input_supports_config_property from utils.test_service import FluentBitTestService class Service: def __init__(self, config_file): + self.config_name = config_file self.config_file = os.path.abspath(os.path.join(os.path.dirname(__file__), "../config", config_file)) test_path = os.path.dirname(os.path.abspath(__file__)) self.parsers_file = os.environ.get("FLUENT_BIT_PARSERS_FILE") or os.path.abspath( os.path.join(test_path, "../../../../../conf/parsers.conf") ) + cert_dir = os.path.abspath(os.path.join(test_path, "../../in_splunk/certificate")) + self.tls_crt_file = os.path.join(cert_dir, "certificate.pem") + self.tls_key_file = os.path.join(cert_dir, "private_key.pem") self.service = FluentBitTestService( self.config_file, data_storage=data_storage, data_keys=["payloads"], extra_env={ "PARSERS_FILE_TEST": self.parsers_file, + "CERTIFICATE_TEST": self.tls_crt_file, + "PRIVATE_KEY_TEST": self.tls_key_file, }, pre_start=self._start_receiver, post_stop=self._stop_receiver, @@ -40,7 +51,12 @@ def _stop_receiver(self, service): pass def start(self): + if ("workers" in self.config_name and + not fluent_bit_input_supports_config_property("tcp", "workers")): + pytest.skip("tcp.workers is not supported by the selected Fluent Bit binary") + self.service.start() + self.flb = self.service.flb self.flb_listener_port = self.service.flb_listener_port def stop(self): @@ -60,6 +76,33 @@ def wait_for_single_record(self, timeout=10): return payloads[0][0] + def flattened_records(self): + records = [] + for payload in data_storage["payloads"]: + if isinstance(payload, list): + records.extend(payload) + elif payload is not None: + records.append(payload) + return records + + def wait_for_record_count(self, minimum_count, timeout=10): + return self.service.wait_for_condition( + lambda: self.flattened_records() if len(self.flattened_records()) >= minimum_count else None, + timeout=timeout, + interval=0.2, + description=f"{minimum_count} forwarded TCP payloads", + ) + + def wait_for_log_message(self, pattern, timeout=10, interval=0.25): + deadline = time.time() + timeout + while time.time() < deadline: + if self.flb and self.flb.log_file and os.path.exists(self.flb.log_file): + with open(self.flb.log_file, "r", encoding="utf-8", errors="replace") as log_file: + if pattern in log_file.read(): + return True + time.sleep(interval) + raise TimeoutError(f"Timed out waiting for log pattern: {pattern}") + def _send_line(port, line): with socket.create_connection(("127.0.0.1", port), timeout=5) as sock: @@ -67,6 +110,24 @@ def _send_line(port, line): sock.shutdown(socket.SHUT_WR) +def _drop_partial_connection(port, payload): + with socket.create_connection(("127.0.0.1", port), timeout=5) as sock: + sock.sendall(payload.encode("utf-8")) + + +def _send_tls_line(port, line, cafile): + context = ssl.create_default_context(cafile=cafile) + with socket.create_connection(("127.0.0.1", port), timeout=20) as raw_sock: + with context.wrap_socket(raw_sock, server_hostname="localhost") as tls_sock: + tls_sock.sendall(line.encode("utf-8")) + tls_sock.shutdown(socket.SHUT_WR) + + +def _drop_raw_tls_connection(port, payload): + with socket.create_connection(("127.0.0.1", port), timeout=5) as sock: + sock.sendall(payload) + + def test_in_tcp_parser_json_success(): service = Service("in_tcp_parser_json.yaml") service.start() @@ -93,3 +154,105 @@ def test_in_tcp_parser_json_fallback_to_log(): assert "log" in record assert record["log"].strip() == "not-json" + + +def test_in_tcp_parser_json_workers_concurrent_records(): + total_records = 16 + service = Service("in_tcp_parser_json_workers.yaml") + service.start() + + try: + service.wait_for_log_message("with 4 workers", timeout=10) + with ThreadPoolExecutor(max_workers=total_records) as executor: + list(executor.map( + lambda i: _send_line(service.flb_listener_port, + f'{{"message":"worker-{i}","value":{i}}}\n'), + range(total_records), + )) + records = service.wait_for_record_count(total_records, timeout=20) + finally: + service.stop() + + values = sorted(record["value"] for record in records) + assert values == list(range(total_records)) + + +def test_in_tcp_workers_drop_partial_connections_and_continue(): + dropped_connections = 8 + valid_records = 8 + service = Service("in_tcp_parser_json_workers.yaml") + service.start() + + try: + service.wait_for_log_message("with 4 workers", timeout=10) + with ThreadPoolExecutor(max_workers=dropped_connections) as executor: + list(executor.map( + lambda i: _drop_partial_connection(service.flb_listener_port, + f'{{"message":"partial-{i}"'), + range(dropped_connections), + )) + + with ThreadPoolExecutor(max_workers=valid_records) as executor: + list(executor.map( + lambda i: _send_line(service.flb_listener_port, + f'{{"message":"valid-{i}","value":{i}}}\n'), + range(valid_records), + )) + records = service.wait_for_record_count(valid_records, timeout=20) + finally: + service.stop() + + values = sorted(record["value"] for record in records) + assert values == list(range(valid_records)) + + +def test_in_tcp_tls_workers_concurrent_records(): + total_records = 16 + service = Service("in_tcp_parser_json_tls_workers.yaml") + service.start() + + try: + service.wait_for_log_message("with 4 workers", timeout=10) + with ThreadPoolExecutor(max_workers=total_records) as executor: + list(executor.map( + lambda i: _send_tls_line(service.flb_listener_port, + f'{{"message":"tls-worker-{i}","value":{i}}}\n', + service.tls_crt_file), + range(total_records), + )) + records = service.wait_for_record_count(total_records, timeout=20) + finally: + service.stop() + + values = sorted(record["value"] for record in records) + assert values == list(range(total_records)) + + +def test_in_tcp_tls_workers_drop_bad_handshakes_and_continue(): + dropped_connections = 8 + valid_records = 8 + service = Service("in_tcp_parser_json_tls_workers.yaml") + service.start() + + try: + service.wait_for_log_message("with 4 workers", timeout=10) + with ThreadPoolExecutor(max_workers=dropped_connections) as executor: + list(executor.map( + lambda i: _drop_raw_tls_connection(service.flb_listener_port, + f"not-tls-{i}".encode("utf-8")), + range(dropped_connections), + )) + + with ThreadPoolExecutor(max_workers=valid_records) as executor: + list(executor.map( + lambda i: _send_tls_line(service.flb_listener_port, + f'{{"message":"valid-tls-{i}","value":{i}}}\n', + service.tls_crt_file), + range(valid_records), + )) + records = service.wait_for_record_count(valid_records, timeout=20) + finally: + service.stop() + + values = sorted(record["value"] for record in records) + assert values == list(range(valid_records)) diff --git a/tests/integration/scenarios/in_udp/config/in_udp_json_workers.yaml b/tests/integration/scenarios/in_udp/config/in_udp_json_workers.yaml new file mode 100644 index 00000000000..7f37132a3df --- /dev/null +++ b/tests/integration/scenarios/in_udp/config/in_udp_json_workers.yaml @@ -0,0 +1,23 @@ +service: + flush: 1 + grace: 1 + log_level: info + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} + +pipeline: + inputs: + - name: udp + tag: target_input + listen: 127.0.0.1 + port: ${FLUENT_BIT_TEST_LISTENER_PORT} + format: json + workers: 4 + + outputs: + - name: http + match: target_input + host: 127.0.0.1 + port: ${TEST_SUITE_HTTP_PORT} + uri: /data + format: json diff --git a/tests/integration/scenarios/in_udp/config/in_udp_parser_json_workers.yaml b/tests/integration/scenarios/in_udp/config/in_udp_parser_json_workers.yaml new file mode 100644 index 00000000000..872a5e855e1 --- /dev/null +++ b/tests/integration/scenarios/in_udp/config/in_udp_parser_json_workers.yaml @@ -0,0 +1,26 @@ +service: + flush: 1 + grace: 1 + log_level: info + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} + parsers_file: ${PARSERS_FILE_TEST} + +pipeline: + inputs: + - name: udp + tag: target_input + listen: 127.0.0.1 + port: ${FLUENT_BIT_TEST_LISTENER_PORT} + format: none + separator: "\\n" + parser: json + workers: 4 + + outputs: + - name: http + match: target_input + host: 127.0.0.1 + port: ${TEST_SUITE_HTTP_PORT} + uri: /data + format: json diff --git a/tests/integration/scenarios/in_udp/tests/test_in_udp_001.py b/tests/integration/scenarios/in_udp/tests/test_in_udp_001.py index 83315f332ac..e167d783265 100644 --- a/tests/integration/scenarios/in_udp/tests/test_in_udp_001.py +++ b/tests/integration/scenarios/in_udp/tests/test_in_udp_001.py @@ -1,14 +1,19 @@ import os import socket +import time +from concurrent.futures import ThreadPoolExecutor +import pytest import requests from server.http_server import data_storage, http_server_run +from utils.fluent_bit_manager import fluent_bit_input_supports_config_property from utils.test_service import FluentBitTestService class Service: def __init__(self, config_file): + self.config_name = config_file self.config_file = os.path.abspath(os.path.join(os.path.dirname(__file__), "../config", config_file)) test_path = os.path.dirname(os.path.abspath(__file__)) self.parsers_file = os.environ.get("FLUENT_BIT_PARSERS_FILE") or os.path.abspath( @@ -40,7 +45,12 @@ def _stop_receiver(self, service): pass def start(self): + if ("workers" in self.config_name and + not fluent_bit_input_supports_config_property("udp", "workers")): + pytest.skip("udp.workers is not supported by the selected Fluent Bit binary") + self.service.start() + self.flb = self.service.flb self.flb_listener_port = self.service.flb_listener_port def stop(self): @@ -60,6 +70,39 @@ def wait_for_single_record(self, timeout=10): return payloads[0][0] + def flattened_records(self): + records = [] + for payload in data_storage["payloads"]: + if isinstance(payload, list): + records.extend(payload) + elif payload is not None: + records.append(payload) + return records + + def wait_for_record_count(self, minimum_count, timeout=10): + def records_ready(): + records = self.flattened_records() + if len(records) >= minimum_count: + return records + return None + + return self.service.wait_for_condition( + records_ready, + timeout=timeout, + interval=0.2, + description=f"{minimum_count} forwarded UDP payloads", + ) + + def wait_for_log_message(self, pattern, timeout=10, interval=0.25): + deadline = time.time() + timeout + while time.time() < deadline: + if self.flb and self.flb.log_file and os.path.exists(self.flb.log_file): + with open(self.flb.log_file, "r", encoding="utf-8", errors="replace") as log_file: + if pattern in log_file.read(): + return True + time.sleep(interval) + raise TimeoutError(f"Timed out waiting for log pattern: {pattern}") + def _send_datagram(port, payload): with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as sock: @@ -92,3 +135,54 @@ def test_in_udp_parser_json_fallback_to_log(): assert "log" in record assert record["log"].strip() == "not-json" + + +def test_in_udp_parser_json_workers_concurrent_records(): + total_records = 16 + service = Service("in_udp_parser_json_workers.yaml") + service.start() + + try: + service.wait_for_log_message("with 4 workers", timeout=10) + with ThreadPoolExecutor(max_workers=total_records) as executor: + list(executor.map( + lambda i: _send_datagram(service.flb_listener_port, + f'{{"message":"worker-{i}","value":{i}}}\n'), + range(total_records), + )) + records = service.wait_for_record_count(total_records, timeout=20) + finally: + service.stop() + + values = sorted(record["value"] for record in records) + assert values == list(range(total_records)) + + +def test_in_udp_workers_drop_malformed_datagrams_and_continue(): + malformed_datagrams = 8 + valid_records = 8 + service = Service("in_udp_json_workers.yaml") + service.start() + + try: + service.wait_for_log_message("with 4 workers", timeout=10) + with ThreadPoolExecutor(max_workers=malformed_datagrams) as executor: + list(executor.map( + lambda i: _send_datagram(service.flb_listener_port, + f'{{"message":"malformed-{i}"'), + range(malformed_datagrams), + )) + + with ThreadPoolExecutor(max_workers=valid_records) as executor: + list(executor.map( + lambda i: _send_datagram(service.flb_listener_port, + f'{{"message":"valid-{i}","value":{i}}}'), + range(valid_records), + )) + records = service.wait_for_record_count(valid_records, timeout=20) + assert len(records) == valid_records + finally: + service.stop() + + values = sorted(record["value"] for record in records) + assert values == list(range(valid_records)) diff --git a/tests/integration/scenarios/out_es/tests/test_out_es_ndjson_action_line_001.py b/tests/integration/scenarios/out_es/tests/test_out_es_ndjson_action_line_001.py index 4ca0c0a78e8..f773cc1611a 100644 --- a/tests/integration/scenarios/out_es/tests/test_out_es_ndjson_action_line_001.py +++ b/tests/integration/scenarios/out_es/tests/test_out_es_ndjson_action_line_001.py @@ -5,6 +5,7 @@ import pytest +from utils.memory_check import memory_check_enabled from utils.test_service import FluentBitTestService @@ -83,7 +84,7 @@ def stop(self): self.service.stop() def wait_for_requests(self, minimum_count, timeout=10): - if os.environ.get("VALGRIND"): + if memory_check_enabled(): timeout = max(timeout * 3, 30) return self.service.wait_for_condition( @@ -96,7 +97,7 @@ def wait_for_requests(self, minimum_count, timeout=10): ) def wait_for_action_lines(self, minimum_count, timeout=10): - if os.environ.get("VALGRIND"): + if memory_check_enabled(): timeout = max(timeout * 3, 30) return self.service.wait_for_condition( diff --git a/tests/integration/scenarios/out_forward/tests/test_out_forward_metadata_001.py b/tests/integration/scenarios/out_forward/tests/test_out_forward_metadata_001.py index 91c632e23a0..0f8626f0df3 100644 --- a/tests/integration/scenarios/out_forward/tests/test_out_forward_metadata_001.py +++ b/tests/integration/scenarios/out_forward/tests/test_out_forward_metadata_001.py @@ -10,6 +10,7 @@ forward_server_stop, ) from utils.fluent_bit_manager import FluentBitManager +from utils.memory_check import memory_check_enabled from utils.network import find_available_port, wait_for_port_to_be_free @@ -66,7 +67,8 @@ def stop(self): self.receiver.stop() finally: forward_server_stop() - wait_for_port_to_be_free(self.capture_port, timeout=10 if os.environ.get("VALGRIND") else 5) + timeout = 10 if memory_check_enabled() else 5 + wait_for_port_to_be_free(self.capture_port, timeout=timeout) self._restore_env() def wait_for_forward_messages(self, minimum_count, timeout=10): @@ -162,7 +164,8 @@ def _capture_record(sender_config_file, message): try: _send_log_event_with_metadata(service.sender_port, message) - messages = service.wait_for_forward_messages(1, timeout=20 if os.environ.get("VALGRIND") else 10) + timeout = 20 if memory_check_enabled() else 10 + messages = service.wait_for_forward_messages(1, timeout=timeout) finally: service.stop() diff --git a/tests/integration/scenarios/out_forward/tests/test_out_forward_secure_001.py b/tests/integration/scenarios/out_forward/tests/test_out_forward_secure_001.py index 77d7fa733a6..812807334be 100644 --- a/tests/integration/scenarios/out_forward/tests/test_out_forward_secure_001.py +++ b/tests/integration/scenarios/out_forward/tests/test_out_forward_secure_001.py @@ -12,6 +12,7 @@ ) from utils.data_utils import read_file from utils.fluent_bit_manager import FluentBitManager +from utils.memory_check import memory_check_enabled from utils.network import find_available_port, wait_for_port_to_be_free @@ -113,7 +114,7 @@ def stop(self): forward_server_stop() wait_for_port_to_be_free( self.receiver_port, - timeout=10 if os.environ.get("VALGRIND") else 5, + timeout=10 if memory_check_enabled() else 5, ) self._restore_env() @@ -152,7 +153,7 @@ def test_out_forward_secure_handshake_delivers_records(): try: _send_message(chain.sender_port, "secure-forward-e2e") messages = chain.wait_for_forward_messages( - 1, timeout=20 if os.environ.get("VALGRIND") else 10 + 1, timeout=20 if memory_check_enabled() else 10 ) handshakes = chain.wait_for_handshakes(1) finally: diff --git a/tests/integration/scenarios/out_http/tests/test_out_http_001.py b/tests/integration/scenarios/out_http/tests/test_out_http_001.py index 21c963643b8..48e23fe0f87 100644 --- a/tests/integration/scenarios/out_http/tests/test_out_http_001.py +++ b/tests/integration/scenarios/out_http/tests/test_out_http_001.py @@ -12,12 +12,13 @@ http_server_run, server_instances, ) +from utils.memory_check import memory_check_enabled from utils.test_service import FluentBitTestService logger = logging.getLogger(__name__) -def _valgrind_timeout(timeout): - if os.environ.get("VALGRIND"): +def _memory_check_timeout(timeout): + if memory_check_enabled(): return max(timeout * 3, 30) return timeout @@ -135,7 +136,7 @@ def stop(self): def wait_for_requests(self, minimum_count, timeout=10): return self.service.wait_for_condition( lambda: data_storage["requests"] if len(data_storage["requests"]) >= minimum_count else None, - timeout=_valgrind_timeout(timeout), + timeout=_memory_check_timeout(timeout), interval=0.5, description=f"{minimum_count} outbound HTTP requests", ) @@ -155,7 +156,7 @@ def _read_log(): return self.service.wait_for_condition( _read_log, - timeout=_valgrind_timeout(timeout), + timeout=_memory_check_timeout(timeout), interval=0.25, description=f"log message '{pattern}'", ) diff --git a/tests/integration/scenarios/out_kafka/tests/test_out_kafka_001.py b/tests/integration/scenarios/out_kafka/tests/test_out_kafka_001.py index e1dbd7c46f2..16fc28024d3 100644 --- a/tests/integration/scenarios/out_kafka/tests/test_out_kafka_001.py +++ b/tests/integration/scenarios/out_kafka/tests/test_out_kafka_001.py @@ -13,6 +13,7 @@ from server.kafka_server import data_storage, kafka_server_run, kafka_server_stop from utils.data_utils import read_json_file +from utils.memory_check import memory_check_enabled from utils.test_service import FluentBitTestService @@ -930,7 +931,7 @@ def test_out_kafka_otlp_json_partition_by_resource_rejects_oversized_message(): service.start() service.send_payload_dict(_build_monolithic_logs_payload(512), "logs") - timeout = 30 if os.environ.get("VALGRIND") else 10 + timeout = 30 if memory_check_enabled() else 10 log_text = _wait_for_log_text( service.flb.log_file, "Broker: Message size too large", diff --git a/tests/integration/scenarios/out_s3/config/out_s3_arrow.yaml b/tests/integration/scenarios/out_s3/config/out_s3_arrow.yaml new file mode 100644 index 00000000000..00795e3d9f1 --- /dev/null +++ b/tests/integration/scenarios/out_s3/config/out_s3_arrow.yaml @@ -0,0 +1,27 @@ +service: + flush: 1 + grace: 3 + log_level: info + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} + +pipeline: + inputs: + - name: dummy + tag: out_s3 + dummy: '{"message":"hello arrow s3","source":"dummy"}' + samples: 1 + + outputs: + - name: s3 + match: out_s3 + bucket: test-bucket + region: us-east-1 + endpoint: http://127.0.0.1:${TEST_SUITE_HTTP_PORT} + use_put_object: true + total_file_size: 1M + upload_timeout: 2s + s3_key_format: /payloads/$TAG/$UUID + format: arrow + compression: zstd + store_dir: /tmp/fluent-bit-test-suite-s3-arrow diff --git a/tests/integration/scenarios/out_s3/config/out_s3_arrow_gzip_invalid.yaml b/tests/integration/scenarios/out_s3/config/out_s3_arrow_gzip_invalid.yaml new file mode 100644 index 00000000000..c029bffca8b --- /dev/null +++ b/tests/integration/scenarios/out_s3/config/out_s3_arrow_gzip_invalid.yaml @@ -0,0 +1,27 @@ +service: + flush: 1 + grace: 3 + log_level: info + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} + +pipeline: + inputs: + - name: dummy + tag: out_s3 + dummy: '{"message":"hello arrow gzip s3","source":"dummy"}' + samples: 1 + + outputs: + - name: s3 + match: out_s3 + bucket: test-bucket + region: us-east-1 + endpoint: http://127.0.0.1:${TEST_SUITE_HTTP_PORT} + use_put_object: true + total_file_size: 1M + upload_timeout: 2s + s3_key_format: /payloads/$TAG/$UUID + format: arrow + compression: gzip + store_dir: /tmp/fluent-bit-test-suite-s3-arrow-gzip-invalid diff --git a/tests/integration/scenarios/out_s3/config/out_s3_arrow_none.yaml b/tests/integration/scenarios/out_s3/config/out_s3_arrow_none.yaml new file mode 100644 index 00000000000..be4946e2975 --- /dev/null +++ b/tests/integration/scenarios/out_s3/config/out_s3_arrow_none.yaml @@ -0,0 +1,27 @@ +service: + flush: 1 + grace: 3 + log_level: info + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} + +pipeline: + inputs: + - name: dummy + tag: out_s3 + dummy: '{"message":"hello arrow none s3","source":"dummy"}' + samples: 1 + + outputs: + - name: s3 + match: out_s3 + bucket: test-bucket + region: us-east-1 + endpoint: http://127.0.0.1:${TEST_SUITE_HTTP_PORT} + use_put_object: true + total_file_size: 1M + upload_timeout: 2s + s3_key_format: /payloads/$TAG/$UUID + format: arrow + compression: none + store_dir: /tmp/fluent-bit-test-suite-s3-arrow-none diff --git a/tests/integration/scenarios/out_s3/config/out_s3_parquet.yaml b/tests/integration/scenarios/out_s3/config/out_s3_parquet.yaml new file mode 100644 index 00000000000..3b199b9242a --- /dev/null +++ b/tests/integration/scenarios/out_s3/config/out_s3_parquet.yaml @@ -0,0 +1,27 @@ +service: + flush: 1 + grace: 3 + log_level: info + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} + +pipeline: + inputs: + - name: dummy + tag: out_s3 + dummy: '{"message":"hello parquet s3","source":"dummy"}' + samples: 1 + + outputs: + - name: s3 + match: out_s3 + bucket: test-bucket + region: us-east-1 + endpoint: http://127.0.0.1:${TEST_SUITE_HTTP_PORT} + use_put_object: true + total_file_size: 1M + upload_timeout: 2s + s3_key_format: /payloads/$TAG/$UUID + format: parquet + compression: zstd + store_dir: /tmp/fluent-bit-test-suite-s3-parquet diff --git a/tests/integration/scenarios/out_s3/config/out_s3_parquet_none.yaml b/tests/integration/scenarios/out_s3/config/out_s3_parquet_none.yaml new file mode 100644 index 00000000000..9938b32a97d --- /dev/null +++ b/tests/integration/scenarios/out_s3/config/out_s3_parquet_none.yaml @@ -0,0 +1,27 @@ +service: + flush: 1 + grace: 3 + log_level: info + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} + +pipeline: + inputs: + - name: dummy + tag: out_s3 + dummy: '{"message":"hello parquet none s3","source":"dummy"}' + samples: 1 + + outputs: + - name: s3 + match: out_s3 + bucket: test-bucket + region: us-east-1 + endpoint: http://127.0.0.1:${TEST_SUITE_HTTP_PORT} + use_put_object: true + total_file_size: 1M + upload_timeout: 2s + s3_key_format: /payloads/$TAG/$UUID + format: parquet + compression: none + store_dir: /tmp/fluent-bit-test-suite-s3-parquet-none diff --git a/tests/integration/scenarios/out_s3/tests/test_out_s3_001.py b/tests/integration/scenarios/out_s3/tests/test_out_s3_001.py index cf3587b07df..c388292badc 100644 --- a/tests/integration/scenarios/out_s3/tests/test_out_s3_001.py +++ b/tests/integration/scenarios/out_s3/tests/test_out_s3_001.py @@ -140,6 +140,22 @@ def _start_or_skip_unsupported_s3_format(service, format_name): raise +def _start_or_skip_unsupported_columnar_format(service, requires_marker): + """Start the service, skipping if the columnar format support + (arrow-glib/parquet-glib) was not compiled into the Fluent Bit binary.""" + try: + service.start() + except FluentBitStartupError: + log_contents = "" + if service.service.flb and service.service.flb.log_file: + with open(service.service.flb.log_file, "r", encoding="utf-8", errors="replace") as file: + log_contents = file.read() + if requires_marker in log_contents or \ + "unknown configuration property 'format'" in log_contents: + pytest.skip("columnar format support is not compiled into this Fluent Bit binary") + raise + + def test_out_s3_put_object_uploads_json_lines_payload(): service = Service("out_s3_basic.yaml") service.start() @@ -241,3 +257,74 @@ def test_out_s3_default_retry_exhausted_action_quarantines_file(): service.stop() assert len(files) > 0 + + +def test_out_s3_format_arrow_uploads_feather_with_zstd(): + service = Service("out_s3_arrow.yaml") + _start_or_skip_unsupported_columnar_format(service, "requires arrow-glib") + request = service.wait_for_request() + service.stop() + + assert request["method"] == "PUT" + assert request["path"].startswith("/test-bucket/payloads/out_s3/") + body = request["body"] + # Arrow/Feather V2 files begin with the "ARROW1" magic. The object is the + # columnar file itself, so it must not carry a byte-level Content-Encoding. + assert body[:6] == b"ARROW1" + assert "Content-Encoding" not in request["headers"] + + +def test_out_s3_format_parquet_uploads_parquet_with_zstd(): + service = Service("out_s3_parquet.yaml") + _start_or_skip_unsupported_columnar_format(service, "requires parquet-glib") + request = service.wait_for_request() + service.stop() + + assert request["method"] == "PUT" + assert request["path"].startswith("/test-bucket/payloads/out_s3/") + body = request["body"] + # Parquet files start and end with the "PAR1" magic. Page-level zstd is + # applied inside the file, so no byte-level Content-Encoding is expected. + assert body[:4] == b"PAR1" + assert body[-4:] == b"PAR1" + assert "Content-Encoding" not in request["headers"] + + +def test_out_s3_format_parquet_compression_none_is_accepted(): + # 'compression none' must be explicitly accepted (not rejected as an + # unknown codec) and produce an uncompressed Parquet object with no + # byte-level Content-Encoding header. + service = Service("out_s3_parquet_none.yaml") + _start_or_skip_unsupported_columnar_format(service, "requires parquet-glib") + request = service.wait_for_request() + service.stop() + + assert request["method"] == "PUT" + body = request["body"] + assert body[:4] == b"PAR1" + assert body[-4:] == b"PAR1" + assert "Content-Encoding" not in request["headers"] + + +def test_out_s3_format_arrow_compression_none_is_accepted(): + # 'compression none' must be explicitly accepted (not rejected as an + # unknown codec) and produce an uncompressed Arrow/Feather object with no + # byte-level Content-Encoding header. + service = Service("out_s3_arrow_none.yaml") + _start_or_skip_unsupported_columnar_format(service, "requires arrow-glib") + request = service.wait_for_request() + service.stop() + + assert request["method"] == "PUT" + body = request["body"] + assert body[:6] == b"ARROW1" + assert "Content-Encoding" not in request["headers"] + + +def test_out_s3_format_arrow_compression_gzip_is_rejected(): + # format=arrow with compression=gzip is an invalid combination; + # validate_format_compression() must reject it at plugin init. + service = Service("out_s3_arrow_gzip_invalid.yaml") + with pytest.raises(FluentBitStartupError): + service.start() + service.stop() diff --git a/tests/integration/scenarios/out_stdout/tests/test_out_stdout_001.py b/tests/integration/scenarios/out_stdout/tests/test_out_stdout_001.py index 9639573ea00..928b574dfbe 100644 --- a/tests/integration/scenarios/out_stdout/tests/test_out_stdout_001.py +++ b/tests/integration/scenarios/out_stdout/tests/test_out_stdout_001.py @@ -200,6 +200,16 @@ def test_out_stdout_logs_accepts_otlp_json_ingestion(): assert "This is another example log message." in log_text +def test_out_stdout_preserves_post_2038_otlp_timestamp(): + service = Service("out_stdout_otel.yaml") + service.start() + service.send_logs_json_payload("test_logs_2038.in.json") + log_text = service.wait_for_log_contains("post-2038 timestamp") + service.stop() + + assert "2209072510.808241446" in log_text + + def test_out_stdout_metrics_accepts_otlp_json_ingestion(): service = Service("out_stdout_otel.yaml") service.start() diff --git a/tests/integration/scenarios/out_syslog/config/out_syslog_dtls.yaml b/tests/integration/scenarios/out_syslog/config/out_syslog_dtls.yaml new file mode 100644 index 00000000000..5b33a7fcb63 --- /dev/null +++ b/tests/integration/scenarios/out_syslog/config/out_syslog_dtls.yaml @@ -0,0 +1,22 @@ +service: + flush: 1 + log_level: info + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} + +pipeline: + inputs: + - name: dummy + tag: out_syslog + dummy: '{"message":"hello from out_syslog"}' + rate: 1 + + outputs: + - name: syslog + match: out_syslog + host: 127.0.0.1 + port: ${SYSLOG_RECEIVER_PORT} + mode: dtls + tls: on + tls.verify: off + syslog_message_key: message diff --git a/tests/integration/scenarios/out_syslog/config/out_syslog_tcp.yaml b/tests/integration/scenarios/out_syslog/config/out_syslog_tcp.yaml new file mode 100644 index 00000000000..835703061f5 --- /dev/null +++ b/tests/integration/scenarios/out_syslog/config/out_syslog_tcp.yaml @@ -0,0 +1,20 @@ +service: + flush: 1 + log_level: info + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} + +pipeline: + inputs: + - name: dummy + tag: out_syslog + dummy: '{"message":"hello from out_syslog"}' + rate: 1 + + outputs: + - name: syslog + match: out_syslog + host: 127.0.0.1 + port: ${SYSLOG_RECEIVER_PORT} + mode: tcp + syslog_message_key: message diff --git a/tests/integration/scenarios/out_syslog/config/out_syslog_udp.yaml b/tests/integration/scenarios/out_syslog/config/out_syslog_udp.yaml new file mode 100644 index 00000000000..008103b052f --- /dev/null +++ b/tests/integration/scenarios/out_syslog/config/out_syslog_udp.yaml @@ -0,0 +1,20 @@ +service: + flush: 1 + log_level: info + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} + +pipeline: + inputs: + - name: dummy + tag: out_syslog + dummy: '{"message":"hello from out_syslog"}' + rate: 1 + + outputs: + - name: syslog + match: out_syslog + host: 127.0.0.1 + port: ${SYSLOG_RECEIVER_PORT} + mode: udp + syslog_message_key: message diff --git a/tests/integration/scenarios/out_syslog/tests/test_out_syslog_001.py b/tests/integration/scenarios/out_syslog/tests/test_out_syslog_001.py new file mode 100644 index 00000000000..c478557ca62 --- /dev/null +++ b/tests/integration/scenarios/out_syslog/tests/test_out_syslog_001.py @@ -0,0 +1,271 @@ +import os +import shutil +import socket +import subprocess +import threading +import time + +import pytest + +from utils.test_service import FluentBitTestService + + +class UdpReceiver: + def __init__(self, host, port): + self.host = host + self.port = port + self.message = None + self.error = None + self._ready = threading.Event() + self._done = threading.Event() + self._thread = threading.Thread(target=self._run, daemon=True) + + def _run(self): + try: + with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as server: + server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + server.bind((self.host, self.port)) + server.settimeout(120) + self._ready.set() + + data, _ = server.recvfrom(4096) + self.message = data + self._done.set() + except Exception as exc: + self.error = exc + self._ready.set() + self._done.set() + + def start(self): + self._thread.start() + + def wait_ready(self, timeout=5): + if not self._ready.wait(timeout): + raise TimeoutError("Timed out waiting for UDP receiver readiness") + + def wait_message(self, timeout=10): + if not self._done.wait(timeout): + raise TimeoutError("Timed out waiting for UDP syslog payload") + + if self.error is not None: + raise self.error + + return self.message + + +class TcpReceiver: + def __init__(self, host, port): + self.host = host + self.port = port + self.message = None + self.error = None + self._ready = threading.Event() + self._done = threading.Event() + self._thread = threading.Thread(target=self._run, daemon=True) + + def _run(self): + try: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as server: + server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + server.bind((self.host, self.port)) + server.listen(1) + server.settimeout(120) + self._ready.set() + conn, _ = server.accept() + + with conn: + conn.settimeout(20) + chunks = [] + + while True: + chunk = conn.recv(4096) + if not chunk: + break + chunks.append(chunk) + if b"\n" in chunk: + break + + self.message = b"".join(chunks) + self._done.set() + except Exception as exc: + self.error = exc + self._ready.set() + self._done.set() + + def start(self): + self._thread.start() + + def wait_ready(self, timeout=5): + if not self._ready.wait(timeout): + raise TimeoutError("Timed out waiting for TCP receiver readiness") + + def wait_message(self, timeout=10): + if not self._done.wait(timeout): + raise TimeoutError("Timed out waiting for TCP syslog payload") + + if self.error is not None: + raise self.error + + return self.message + + +class DtlsReceiver: + def __init__(self, port, cert_file, key_file): + self.port = port + self.cert_file = cert_file + self.key_file = key_file + self.process = None + + def start(self): + openssl = shutil.which("openssl") + if openssl is None: + raise RuntimeError("openssl is required for DTLS test") + + self.process = subprocess.Popen( # noqa: S603 - controlled test command + [ + openssl, + "s_server", + "-dtls", + "-accept", + str(self.port), + "-cert", + self.cert_file, + "-key", + self.key_file, + "-naccept", + "1", + "-ign_eof", + ], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + + time.sleep(0.5) + if self.process.poll() is not None: + output = self._read_output(timeout=2) + raise RuntimeError(f"DTLS receiver failed to start: {output}") + + def wait_ready(self, timeout=5): + deadline = time.time() + timeout + while time.time() < deadline: + if self.process.poll() is not None: + output = self._read_output(timeout=2) + raise RuntimeError(f"DTLS receiver terminated early: {output}") + time.sleep(0.1) + + def _read_output(self, timeout=2): + stdout, stderr = self.process.communicate(timeout=timeout) + return (stdout + stderr).decode("utf-8", errors="replace") + + def wait_message(self, timeout=30): + try: + output = self._read_output(timeout=timeout) + except subprocess.TimeoutExpired as exc: + raise TimeoutError("Timed out waiting for DTLS handshake") from exc + + return output + + def stop(self): + if self.process is None: + return + + if self.process.poll() is None: + self.process.terminate() + try: + self.process.wait(timeout=5) + except subprocess.TimeoutExpired: + self.process.kill() + self.process.wait(timeout=5) + + +class Service: + def __init__(self, config_file, receiver_type): + self.config_file = os.path.abspath(os.path.join(os.path.dirname(__file__), "../config", config_file)) + self.receiver_type = receiver_type + self.receiver = None + + cert_dir = os.path.abspath( + os.path.join(os.path.dirname(__file__), "../../in_splunk/certificate") + ) + self.tls_crt_file = os.path.join(cert_dir, "certificate.pem") + self.tls_key_file = os.path.join(cert_dir, "private_key.pem") + + self.service = FluentBitTestService( + self.config_file, + pre_start=self._start_receiver, + post_stop=self._stop_receiver, + ) + + def _start_receiver(self, service): + self.receiver_port = service.allocate_port_env("SYSLOG_RECEIVER_PORT") + + if self.receiver_type == "udp": + self.receiver = UdpReceiver("127.0.0.1", self.receiver_port) + elif self.receiver_type == "tcp": + self.receiver = TcpReceiver("127.0.0.1", self.receiver_port) + elif self.receiver_type == "dtls": + self.receiver = DtlsReceiver(self.receiver_port, self.tls_crt_file, self.tls_key_file) + else: + raise ValueError(f"Unknown receiver type: {self.receiver_type}") + + self.receiver.start() + self.receiver.wait_ready(timeout=5) + + def _stop_receiver(self, _service): + if self.receiver_type == "dtls" and self.receiver is not None: + self.receiver.stop() + + def start(self): + self.service.start() + + def stop(self): + self.service.stop() + + +def _assert_syslog_payload(payload): + text = payload.decode("utf-8", errors="replace") + assert "hello from out_syslog" in text + assert text.startswith("<") + + +def _assert_dtls_payload(output): + assert "ACCEPT" in output + assert "DONE" in output + + +def test_out_syslog_udp(): + service = Service("out_syslog_udp.yaml", "udp") + service.start() + + try: + payload = service.receiver.wait_message(timeout=20) + finally: + service.stop() + + _assert_syslog_payload(payload) + + +def test_out_syslog_tcp(): + service = Service("out_syslog_tcp.yaml", "tcp") + service.start() + + try: + payload = service.receiver.wait_message(timeout=15) + finally: + service.stop() + + _assert_syslog_payload(payload) + + +@pytest.mark.skipif(not shutil.which("openssl"), reason="openssl is required for DTLS test") +def test_out_syslog_dtls(): + service = Service("out_syslog_dtls.yaml", "dtls") + service.start() + + try: + output = service.receiver.wait_message(timeout=30) + finally: + service.stop() + + _assert_dtls_payload(output) diff --git a/tests/integration/src/server/otlp_server.py b/tests/integration/src/server/otlp_server.py index 7bcdf1ef7b0..0724951f071 100644 --- a/tests/integration/src/server/otlp_server.py +++ b/tests/integration/src/server/otlp_server.py @@ -280,6 +280,9 @@ def run_grpc_server(port=4317, *, use_tls=False, tls_crt_file=None, tls_key_file def stop_otlp_server(): global grpc_server_instance global http_server_instance + global server_thread + + stop_event = None shutdown_flag.set() @@ -288,9 +291,19 @@ def stop_otlp_server(): http_server_instance = None if grpc_server_instance is not None: - grpc_server_instance.stop(grace=0) + stop_event = grpc_server_instance.stop(grace=0) grpc_server_instance = None + if stop_event is not None: + stop_event.wait(timeout=5) + + if server_thread is not None and server_thread is not threading.current_thread(): + server_thread.join(timeout=5) + if server_thread.is_alive(): + raise RuntimeError("OTLP server thread did not stop") + + server_thread = None + def otlp_server_run(port, *, use_tls=False, tls_crt_file=None, tls_key_file=None, use_grpc=False): global server_thread diff --git a/tests/integration/src/utils/fluent_bit_manager.py b/tests/integration/src/utils/fluent_bit_manager.py index 1040efa5be3..9988d89823d 100644 --- a/tests/integration/src/utils/fluent_bit_manager.py +++ b/tests/integration/src/utils/fluent_bit_manager.py @@ -17,19 +17,35 @@ import datetime import logging import os +import platform from pathlib import Path +import re +import signal import shutil import subprocess +import tempfile import time import requests from utils.network import find_available_port, wait_for_port_to_be_free +from utils.leaks import assert_leaks_clean +from utils.memory_check import leaks_enabled, memory_check_enabled, valgrind_enabled from utils.valgrind import assert_valgrind_clean ENV_FLB_HTTP_MONITORING_PORT = "FLUENT_BIT_HTTP_MONITORING_PORT" +ENV_FLB_BINARY_PATH = "FLUENT_BIT_BINARY" +LEAKS_TARGET_PID_TIMEOUT = 10 +LEAKS_EXIT_TIMEOUT = 120 + +LEAKS_EXEC_SCRIPT = ( + 'printf "%s\\n" "$$" > "$1"; output_file="$2"; shift 2; ' + 'exec "$@" >> "$output_file" 2>&1' +) logger = logging.getLogger(__name__) +FLUENT_BIT_VERSION_PATTERN = re.compile(r"(?:Fluent Bit\s+)?v?(\d+)\.(\d+)\.(\d+)") +_BINARY_VERSION_INFO_CACHE = {} class FluentBitStartupError(RuntimeError): pass @@ -48,17 +64,111 @@ def _default_binary_path(): return str(local_binary) +def _resolve_binary_path(binary_path=None): + selected_path = binary_path or os.environ.get(ENV_FLB_BINARY_PATH) or _default_binary_path() + return shutil.which(selected_path) or os.path.abspath(selected_path) + + +def parse_fluent_bit_version(version_text): + match = FLUENT_BIT_VERSION_PATTERN.search(version_text) + if not match: + return None + + return tuple(int(component) for component in match.groups()) + + +def _read_binary_version_info(binary_path=None): + resolved_path = _resolve_binary_path(binary_path) + + if resolved_path in _BINARY_VERSION_INFO_CACHE: + return _BINARY_VERSION_INFO_CACHE[resolved_path] + + try: + result = subprocess.run( + [resolved_path, "--version"], + capture_output=True, + text=True, + check=False, + ) + except OSError: + return None + + output = result.stdout.strip().splitlines() + if not output: + return None + + version = output[0].replace("Fluent Bit ", "").strip() + commit = output[1].replace("Git commit: ", "").strip() if len(output) > 1 else "unknown" + version_info = (version, commit) + _BINARY_VERSION_INFO_CACHE[resolved_path] = version_info + return version_info + + +def fluent_bit_binary_version(binary_path=None): + version_info = _read_binary_version_info(binary_path) + if version_info is None: + return None + + return parse_fluent_bit_version(version_info[0]) + + +def fluent_bit_binary_supports_config_property(property_name, binary_path=None): + resolved_path = _resolve_binary_path(binary_path) + property_marker = property_name.lower() + + try: + result = subprocess.run( + [resolved_path, "--help"], + capture_output=True, + text=True, + check=False, + ) + help_output = f"{result.stdout}\n{result.stderr}".lower() + if property_marker in help_output: + return True + except OSError: + pass + + try: + binary_contents = Path(resolved_path).read_bytes().lower() + except OSError: + return False + + return property_marker.encode("utf-8") in binary_contents + + +def fluent_bit_input_supports_config_property(plugin_name, property_name, binary_path=None): + resolved_path = _resolve_binary_path(binary_path) + property_marker = property_name.lower() + + try: + result = subprocess.run( + [resolved_path, "-i", plugin_name, "-h"], + capture_output=True, + text=True, + check=False, + ) + except OSError: + return False + + help_output = f"{result.stdout}\n{result.stderr}".lower() + return property_marker in help_output + + class FluentBitManager: def __init__(self, config_path=None, binary_path=None): logger.info(f"config path {config_path}") self.config_path = config_path - self.binary_path = binary_path or os.environ.get("FLUENT_BIT_BINARY") or _default_binary_path() - self.binary_absolute_path = os.path.abspath(self.binary_path) + self.binary_path = binary_path or os.environ.get(ENV_FLB_BINARY_PATH) or _default_binary_path() + self.binary_absolute_path = _resolve_binary_path(self.binary_path) self.process = None self.http_monitoring_port = None self.results_dir = None self.log_file = None self.valgrind_log_file = None + self.leaks_log_file = None + self.leaks_target_pid_file = None + self.target_pid = None self.output_handle = None def set_http_monitoring_port(self, env_var_name, starting_port=0): @@ -82,8 +192,13 @@ def start(self): self.results_dir = out_dir self.log_file = os.path.join(out_dir, "fluent_bit.log") self.valgrind_log_file = os.path.join(out_dir, "valgrind.log") + self.leaks_log_file = os.path.join(out_dir, "leaks.log") + self.leaks_target_pid_file = os.path.join(out_dir, "leaks_target.pid") self.set_http_monitoring_port(ENV_FLB_HTTP_MONITORING_PORT) + if valgrind_enabled() and leaks_enabled(): + raise FluentBitStartupError("VALGRIND and LEAKS cannot be enabled together") + version, commit = self.get_version_info() logger.info(f'Fluent Bit info') logger.info(f' version : {version}') @@ -92,8 +207,10 @@ def start(self): logger.info(f" logfile : {self.log_file}") logger.info(f" http port : {self.http_monitoring_port}") logger.info(f" commit : {commit}") - if self.valgrind_log_file: + if valgrind_enabled(): logger.info(f" valgrind : {self.valgrind_log_file}") + if leaks_enabled(): + logger.info(f" leaks : {self.leaks_log_file}") command = [ self.binary_absolute_path, @@ -101,8 +218,7 @@ def start(self): "-l", self.log_file ] - valgrind = os.environ.get('VALGRIND', False) - if valgrind: + if valgrind_enabled(): command = [ "valgrind", f"--log-file={self.valgrind_log_file}", @@ -110,17 +226,27 @@ def start(self): "--show-leak-kinds=all" ] + command + if leaks_enabled(): + command = self._build_leaks_command(command) logger.info(f"Running command {command}") - self.output_handle = open(self.log_file, "a", encoding="utf-8") + output_path = self.leaks_log_file if leaks_enabled() else self.log_file + self.output_handle = open(output_path, "a", encoding="utf-8") self.process = subprocess.Popen( command, stdout=self.output_handle, stderr=subprocess.STDOUT, - text=True, ) - logger.info(f"Fluent Bit started (pid: {self.process.pid})") + + if leaks_enabled(): + self.target_pid = self._wait_for_leaks_target_pid() + else: + self.target_pid = self.process.pid + + logger.info( + f"Fluent Bit started (pid: {self.target_pid}, supervisor pid: {self.process.pid})" + ) # wait for Fluent Bit to start self.wait_for_fluent_bit() @@ -129,58 +255,164 @@ def stop(self): if not self.process: return - pid = self.process.pid - if self.process.poll() is None: - self.process.terminate() + pid = self.target_pid or self.process.pid + return_code = self.process.poll() + supervisor_running = self.process.poll() is None + if supervisor_running or (leaks_enabled() and self.target_pid): + self.send_signal(signal.SIGTERM) + + if supervisor_running: try: - self.process.wait(timeout=10) + timeout = LEAKS_EXIT_TIMEOUT if leaks_enabled() else 10 + return_code = self.process.wait(timeout=timeout) except subprocess.TimeoutExpired: - self.process.kill() - self.process.wait(timeout=5) + self._force_stop() + return_code = self.process.returncode self.process = None + self.target_pid = None if self.output_handle: self.output_handle.close() self.output_handle = None if self.http_monitoring_port: - wait_for_port_to_be_free(int(self.http_monitoring_port), timeout=10 if os.environ.get("VALGRIND") else 5) + timeout = 10 if memory_check_enabled() else 5 + wait_for_port_to_be_free(int(self.http_monitoring_port), timeout=timeout) - if os.environ.get("VALGRIND") and os.environ.get("VALGRIND_STRICT"): + if valgrind_enabled() and os.environ.get("VALGRIND_STRICT"): assert_valgrind_clean(self.valgrind_log_file) + if leaks_enabled() and os.environ.get("LEAKS_STRICT"): + assert_leaks_clean(return_code, self.leaks_log_file) + logger.info(f"Fluent Bit stopped (pid: {pid})") + def send_signal(self, signal_number): + if not self.process: + raise RuntimeError("Fluent Bit is not running") + + if leaks_enabled(): + if not self.target_pid: + raise RuntimeError("Fluent Bit target PID is unavailable") + try: + os.kill(self.target_pid, signal_number) + except ProcessLookupError: + pass + else: + self.process.send_signal(signal_number) + + def send_sighup(self): + self.send_signal(signal.SIGHUP) + + def get_reload_status(self): + url = f"http://127.0.0.1:{self.http_monitoring_port}/api/v2/reload" + response = requests.get(url, timeout=0.5) + response.raise_for_status() + return response.json() + + def trigger_http_reload(self): + url = f"http://127.0.0.1:{self.http_monitoring_port}/api/v2/reload" + response = requests.post(url, timeout=0.5) + response.raise_for_status() + return response.json() + + def wait_for_hot_reload_count(self, expected_count, timeout=10): + deadline = time.time() + timeout + + while time.time() < deadline: + try: + payload = self.get_reload_status() + if payload.get("hot_reload_count", 0) >= expected_count: + return payload + except requests.RequestException: + pass + time.sleep(0.1) + + raise TimeoutError(f"Timed out waiting for hot reload count {expected_count}") + + def _build_leaks_command(self, command): + if platform.system() != "Darwin": + raise FluentBitStartupError("LEAKS is only supported on macOS") + + leaks_binary = shutil.which("leaks") + if not leaks_binary: + raise FluentBitStartupError("The macOS leaks command was not found") + + return [ + leaks_binary, + "-quiet", + "-fullStacks", + "-atExit", + "--", + "/bin/sh", + "-c", + LEAKS_EXEC_SCRIPT, + "fluent-bit-leaks-target", + self.leaks_target_pid_file, + self.log_file, + ] + command + + def _wait_for_leaks_target_pid(self): + deadline = time.time() + LEAKS_TARGET_PID_TIMEOUT + + while time.time() < deadline: + try: + pid_text = Path(self.leaks_target_pid_file).read_text(encoding="utf-8").strip() + target_pid = int(pid_text) + if target_pid > 0: + return target_pid + except (FileNotFoundError, ValueError): + pass + + if self.process.poll() is not None: + raise FluentBitStartupError( + f"leaks exited before Fluent Bit started with code {self.process.returncode}. " + f"See log file {self.leaks_log_file}" + ) + time.sleep(0.05) + + self._force_stop() + raise FluentBitStartupError( + f"Timed out waiting for the Fluent Bit PID from leaks. See {self.leaks_log_file}" + ) + + def _force_stop(self): + if leaks_enabled() and self.target_pid: + try: + os.kill(self.target_pid, signal.SIGKILL) + except ProcessLookupError: + pass + try: + self.process.wait(timeout=10) + return + except subprocess.TimeoutExpired: + pass + + self.process.kill() + self.process.wait(timeout=5) + def get_version_info(self): - try: - result = subprocess.run( - [self.binary_absolute_path, '--version'], - capture_output=True, - text=True, - check=True, + version_info = _read_binary_version_info(self.binary_absolute_path) + if version_info is None: + raise FluentBitStartupError( + f"Unable to execute Fluent Bit binary {self.binary_absolute_path}" ) - output = result.stdout.strip().split('\n') - version = output[0].replace('Fluent Bit ', '').strip() - commit = output[1].strip().replace('Git commit: ', '') if len(output) > 1 else "unknown" - return version, commit - except (subprocess.CalledProcessError, FileNotFoundError) as e: - logger.error("Error running Fluent Bit: %s", e) - raise FluentBitStartupError(f"Unable to execute Fluent Bit binary {self.binary_absolute_path}") from e + + return version_info def create_results_directory(self, base_dir=None): if base_dir is None: suite_root = Path(__file__).resolve().parents[2] base_dir = suite_root / "results" timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") - results_dir = os.path.join(base_dir, f"fluent_bit_results_{timestamp}") - os.makedirs(results_dir, exist_ok=True) - return results_dir + os.makedirs(base_dir, exist_ok=True) + return tempfile.mkdtemp(prefix=f"fluent_bit_results_{timestamp}_", dir=base_dir) # Check if Fluent Bit is running by trying to reach the uptime endpoint, it waits until # the value of `uptime_sec` is greater than 1 def wait_for_fluent_bit(self, timeout=None): if timeout is None: - timeout = 30 if os.environ.get("VALGRIND") else 10 + timeout = 30 if memory_check_enabled() else 10 url = f"http://127.0.0.1:{self.http_monitoring_port}/api/v1/uptime" start_time = time.time() while time.time() - start_time < timeout: diff --git a/tests/integration/src/utils/leaks.py b/tests/integration/src/utils/leaks.py new file mode 100644 index 00000000000..e6434986145 --- /dev/null +++ b/tests/integration/src/utils/leaks.py @@ -0,0 +1,27 @@ +# Fluent Bit +# ========== +# Copyright (C) 2015-2026 The Fluent Bit Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def assert_leaks_clean(return_code, log_path): + if return_code == 0: + return + + if return_code == 1: + problem = "memory leaks were detected" + else: + problem = f"the leaks command failed with exit code {return_code}" + + raise AssertionError(f"macOS leaks check failed: {problem}. See {log_path}") diff --git a/tests/integration/src/utils/memory_check.py b/tests/integration/src/utils/memory_check.py new file mode 100644 index 00000000000..b402492138e --- /dev/null +++ b/tests/integration/src/utils/memory_check.py @@ -0,0 +1,29 @@ +# Fluent Bit +# ========== +# Copyright (C) 2015-2026 The Fluent Bit Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + + +def valgrind_enabled(): + return bool(os.environ.get("VALGRIND")) + + +def leaks_enabled(): + return bool(os.environ.get("LEAKS")) + + +def memory_check_enabled(): + return valgrind_enabled() or leaks_enabled() diff --git a/tests/integration/test_leaks_utils.py b/tests/integration/test_leaks_utils.py new file mode 100644 index 00000000000..3a10f48ec6c --- /dev/null +++ b/tests/integration/test_leaks_utils.py @@ -0,0 +1,27 @@ +from pathlib import Path + +import pytest + +from utils.leaks import assert_leaks_clean + + +def test_assert_leaks_clean_accepts_clean_exit(tmp_path): + log_path = tmp_path / "leaks.log" + + assert_leaks_clean(0, log_path) + + +@pytest.mark.parametrize( + ("return_code", "message"), + [ + (1, "memory leaks were detected"), + (2, "leaks command failed with exit code 2"), + ], +) +def test_assert_leaks_clean_rejects_nonzero_exit(tmp_path, return_code, message): + log_path = tmp_path / "leaks.log" + + with pytest.raises(AssertionError, match=message) as exc_info: + assert_leaks_clean(return_code, log_path) + + assert str(Path(log_path)) in str(exc_info.value) diff --git a/tests/integration/test_macos_leaks_manager.py b/tests/integration/test_macos_leaks_manager.py new file mode 100644 index 00000000000..3497b1ca503 --- /dev/null +++ b/tests/integration/test_macos_leaks_manager.py @@ -0,0 +1,185 @@ +import signal + +import pytest + +from utils import fluent_bit_manager as manager_module +from utils.fluent_bit_manager import FluentBitManager, FluentBitStartupError + + +class FakeProcess: + def __init__(self, return_code=0): + self.pid = 1234 + self.return_code = return_code + self.returncode = None + self.killed = False + self.signals = [] + + def poll(self): + return self.returncode + + def wait(self, timeout=None): + self.returncode = self.return_code + return self.returncode + + def kill(self): + self.killed = True + self.returncode = -signal.SIGKILL + + def send_signal(self, signal_number): + self.signals.append(signal_number) + + +def _prepare_start(monkeypatch, tmp_path, process): + config_path = tmp_path / "fluent-bit.yaml" + binary_path = tmp_path / "fluent-bit" + config_path.write_text("service:\n flush: 1\n", encoding="utf-8") + binary_path.write_text("binary", encoding="utf-8") + binary_path.chmod(0o755) + + monkeypatch.setenv("LEAKS", "1") + monkeypatch.setattr(manager_module.platform, "system", lambda: "Darwin") + monkeypatch.setattr( + manager_module.shutil, + "which", + lambda command: "/usr/bin/leaks" if command == "leaks" else None, + ) + monkeypatch.setattr(manager_module, "find_available_port", lambda starting_port: 40200) + monkeypatch.setattr(manager_module.requests, "get", lambda url, timeout: FakeResponse()) + monkeypatch.setattr(FluentBitManager, "get_version_info", lambda self: ("vtest", "commit")) + monkeypatch.setattr(FluentBitManager, "_wait_for_leaks_target_pid", lambda self: 4321) + monkeypatch.setattr(manager_module, "wait_for_port_to_be_free", lambda port, timeout: None) + + popen_calls = [] + + def fake_popen(command, stdout=None, stderr=None, text=None): + popen_calls.append( + { + "command": command, + "output_path": stdout.name, + "stderr": stderr, + "text": text, + } + ) + return process + + monkeypatch.setattr(manager_module.subprocess, "Popen", fake_popen) + + return config_path, binary_path, popen_calls + + +class FakeResponse: + status_code = 200 + + def json(self): + return {"uptime_sec": 2} + + +def test_leaks_supervisor_signals_fluent_bit_target(monkeypatch, tmp_path): + process = FakeProcess() + config_path, binary_path, popen_calls = _prepare_start(monkeypatch, tmp_path, process) + delivered_signals = [] + monkeypatch.setattr( + manager_module.os, + "kill", + lambda pid, signal_number: delivered_signals.append((pid, signal_number)), + ) + + manager = FluentBitManager(str(config_path), str(binary_path)) + manager.start() + + assert manager.target_pid == 4321 + assert popen_calls[0]["command"] == [ + "/usr/bin/leaks", + "-quiet", + "-fullStacks", + "-atExit", + "--", + "/bin/sh", + "-c", + manager_module.LEAKS_EXEC_SCRIPT, + "fluent-bit-leaks-target", + manager.leaks_target_pid_file, + manager.log_file, + str(binary_path), + "-c", + str(config_path), + "-l", + manager.log_file, + ] + assert popen_calls[0]["output_path"] == manager.leaks_log_file + + manager.send_sighup() + manager.stop() + + assert delivered_signals == [ + (4321, signal.SIGHUP), + (4321, signal.SIGTERM), + ] + assert process.signals == [] + assert process.returncode == 0 + + +def test_leaks_strict_fails_when_leaks_reports_leak(monkeypatch, tmp_path): + process = FakeProcess(return_code=1) + config_path, binary_path, _ = _prepare_start(monkeypatch, tmp_path, process) + monkeypatch.setenv("LEAKS_STRICT", "1") + monkeypatch.setattr(manager_module.os, "kill", lambda pid, signal_number: None) + + manager = FluentBitManager(str(config_path), str(binary_path)) + manager.start() + + with pytest.raises(AssertionError, match="memory leaks were detected"): + manager.stop() + + +def test_leaks_supervisor_failure_still_terminates_target(monkeypatch): + process = FakeProcess(return_code=255) + process.returncode = 255 + delivered_signals = [] + monkeypatch.setenv("LEAKS", "1") + monkeypatch.setattr( + manager_module.os, + "kill", + lambda pid, signal_number: delivered_signals.append((pid, signal_number)), + ) + + manager = FluentBitManager("/tmp/fluent-bit.yaml", "/tmp/fluent-bit") + manager.process = process + manager.target_pid = 4321 + + manager.stop() + + assert delivered_signals == [(4321, signal.SIGTERM)] + + +def test_leaks_rejects_non_macos_host(monkeypatch): + monkeypatch.setattr(manager_module.platform, "system", lambda: "Linux") + manager = FluentBitManager("/tmp/fluent-bit.yaml", "/tmp/fluent-bit") + + with pytest.raises(FluentBitStartupError, match="only supported on macOS"): + manager._build_leaks_command(["/tmp/fluent-bit"]) + + +def test_leaks_target_pid_timeout_force_stops_supervisor(monkeypatch, tmp_path): + process = FakeProcess() + monkeypatch.setenv("LEAKS", "1") + monkeypatch.setattr(manager_module, "LEAKS_TARGET_PID_TIMEOUT", 0) + + manager = FluentBitManager("/tmp/fluent-bit.yaml", "/tmp/fluent-bit") + manager.process = process + manager.leaks_target_pid_file = str(tmp_path / "missing.pid") + manager.leaks_log_file = str(tmp_path / "leaks.log") + + with pytest.raises(FluentBitStartupError, match="Timed out waiting"): + manager._wait_for_leaks_target_pid() + + assert process.killed is True + + +def test_results_directories_are_unique_for_concurrent_supervisors(tmp_path): + manager = FluentBitManager("/tmp/fluent-bit.yaml", "/tmp/fluent-bit") + + first = manager.create_results_directory(tmp_path) + second = manager.create_results_directory(tmp_path) + + assert first != second diff --git a/tests/integration/tests/test_fluent_bit_manager.py b/tests/integration/tests/test_fluent_bit_manager.py index f9de3620574..147abea35a5 100644 --- a/tests/integration/tests/test_fluent_bit_manager.py +++ b/tests/integration/tests/test_fluent_bit_manager.py @@ -23,6 +23,10 @@ from src.utils import fluent_bit_manager as manager_module from src.utils.fluent_bit_manager import ENV_FLB_BINARY_PATH from src.utils.fluent_bit_manager import FluentBitManager +from src.utils.fluent_bit_manager import fluent_bit_binary_version +from src.utils.fluent_bit_manager import fluent_bit_binary_supports_config_property +from src.utils.fluent_bit_manager import fluent_bit_input_supports_config_property +from src.utils.fluent_bit_manager import parse_fluent_bit_version def test_binary_path_uses_environment_override(monkeypatch): @@ -35,6 +39,113 @@ def test_binary_path_uses_environment_override(monkeypatch): assert manager.binary_absolute_path == "/resolved//opt/fluent-bit/bin/fluent-bit" +def test_binary_capability_detects_config_property_in_help(monkeypatch, tmp_path): + binary_path = tmp_path / "fluent-bit" + binary_path.write_bytes(b"binary without the marker") + binary_path.chmod(0o755) + result = Mock(stdout="Options: hot_reload.watch", stderr="") + monkeypatch.setattr(manager_module.subprocess, "run", lambda *args, **kwargs: result) + + assert fluent_bit_binary_supports_config_property( + "hot_reload.watch", str(binary_path) + ) is True + + +def test_binary_capability_detects_compiled_config_property(monkeypatch, tmp_path): + binary_path = tmp_path / "fluent-bit" + binary_path.write_bytes(b"prefix Hot_Reload.Watch suffix") + binary_path.chmod(0o755) + result = Mock(stdout="", stderr="") + monkeypatch.setattr(manager_module.subprocess, "run", lambda *args, **kwargs: result) + + assert fluent_bit_binary_supports_config_property( + "hot_reload.watch", str(binary_path) + ) is True + + +def test_binary_capability_rejects_missing_config_property(monkeypatch, tmp_path): + binary_path = tmp_path / "fluent-bit" + binary_path.write_bytes(b"binary without the marker") + binary_path.chmod(0o755) + result = Mock(stdout="", stderr="") + monkeypatch.setattr(manager_module.subprocess, "run", lambda *args, **kwargs: result) + + assert fluent_bit_binary_supports_config_property( + "hot_reload.watch", str(binary_path) + ) is False + + +def test_input_capability_detects_plugin_config_property(monkeypatch, tmp_path): + binary_path = tmp_path / "fluent-bit" + binary_path.write_bytes(b"binary") + binary_path.chmod(0o755) + result = Mock(stdout="workers Set the number of listener workers", stderr="") + + def fake_run(command, **kwargs): + assert command == [str(binary_path), "-i", "forward", "-h"] + return result + + monkeypatch.setattr(manager_module.subprocess, "run", fake_run) + + assert fluent_bit_input_supports_config_property( + "forward", "workers", str(binary_path) + ) is True + + +def test_input_capability_rejects_property_from_other_plugins(monkeypatch, tmp_path): + binary_path = tmp_path / "fluent-bit" + binary_path.write_bytes(b"binary containing inotify_watcher elsewhere") + binary_path.chmod(0o755) + result = Mock(stdout="path Path to tail", stderr="") + monkeypatch.setattr(manager_module.subprocess, "run", lambda *args, **kwargs: result) + + assert fluent_bit_input_supports_config_property( + "tail", "inotify_watcher", str(binary_path) + ) is False + + +@pytest.mark.parametrize( + "version_text,expected", + [ + ("Fluent Bit v5.1.0", (5, 1, 0)), + ("Fluent Bit 5.0.9\nGit commit: abc", (5, 0, 9)), + ("v6.2.1-dev", (6, 2, 1)), + ("unknown", None), + ], +) +def test_parse_fluent_bit_version(version_text, expected): + assert parse_fluent_bit_version(version_text) == expected + + +def test_fluent_bit_binary_version_uses_selected_binary(monkeypatch, tmp_path): + binary_path = tmp_path / "fluent-bit" + binary_path.write_bytes(b"binary") + binary_path.chmod(0o755) + result = Mock(stdout="Fluent Bit v5.1.0", stderr="") + + def fake_run(command, **kwargs): + assert command == [str(binary_path), "--version"] + return result + + monkeypatch.setattr(manager_module.subprocess, "run", fake_run) + + assert fluent_bit_binary_version(str(binary_path)) == (5, 1, 0) + + +def test_fluent_bit_binary_version_is_cached(monkeypatch, tmp_path): + binary_path = tmp_path / "fluent-bit" + binary_path.write_bytes(b"binary") + binary_path.chmod(0o755) + result = Mock(stdout="Fluent Bit v5.1.0\nGit commit: abc123", stderr="") + run = Mock(return_value=result) + monkeypatch.setattr(manager_module.subprocess, "run", run) + + assert fluent_bit_binary_version(str(binary_path)) == (5, 1, 0) + manager = FluentBitManager("/tmp/fluent-bit.yaml", binary_path=str(binary_path)) + assert manager.get_version_info() == ("v5.1.0", "abc123") + run.assert_called_once() + + def test_send_signal_raises_when_process_is_missing(): manager = FluentBitManager("/tmp/fluent-bit.yaml") @@ -56,8 +167,9 @@ def test_trigger_http_reload_posts_to_reload_endpoint(monkeypatch): response.json.return_value = {"reload": "done"} response.raise_for_status.return_value = None - def fake_post(url): + def fake_post(url, timeout): assert url == "http://127.0.0.1:2020/api/v2/reload" + assert timeout == 0.5 return response monkeypatch.setattr(manager_module.requests, "post", fake_post) @@ -75,8 +187,14 @@ def test_start_uses_unique_valgrind_log_path(monkeypatch, tmp_path): monkeypatch.setenv("VALGRIND", "1") monkeypatch.setattr(manager_module.os.path, "exists", lambda path: True) + monkeypatch.setattr(manager_module.os.path, "isfile", lambda path: True) + monkeypatch.setattr(manager_module.os, "access", lambda path, mode: True) monkeypatch.setattr(manager_module, "find_available_port", lambda starting_port: 40200) - monkeypatch.setattr(manager_module.requests, "get", lambda url: Mock(status_code=200, json=lambda: {"uptime_sec": 2})) + monkeypatch.setattr( + manager_module.requests, + "get", + lambda url, timeout=0.5: Mock(status_code=200, json=lambda: {"uptime_sec": 2}), + ) created_dirs = iter([ str(tmp_path / "run-1"), @@ -89,6 +207,7 @@ def fake_create_results_directory(self, base_dir='results'): popen_result = Mock() popen_result.pid = 1234 + popen_result.poll.return_value = None monkeypatch.setattr(FluentBitManager, "create_results_directory", fake_create_results_directory) monkeypatch.setattr(FluentBitManager, "get_version_info", lambda self: ("vtest", "commit")) @@ -108,6 +227,7 @@ def fake_popen(command, stdout=None, stderr=None): "valgrind", f"--log-file={manager.valgrind_log_file}", "--leak-check=full", + "--show-leak-kinds=all", "/usr/bin/fluent-bit", "-c", str(config_path), "-l", str(tmp_path / "run-1" / "fluent_bit.log") diff --git a/tests/internal/CMakeLists.txt b/tests/internal/CMakeLists.txt index 6ce01a29332..4ace16a1798 100644 --- a/tests/internal/CMakeLists.txt +++ b/tests/internal/CMakeLists.txt @@ -10,6 +10,7 @@ set(UNIT_TESTS_FILES hmac.c crypto.c hash.c + fips.c kv.c slist.c router.c @@ -22,6 +23,7 @@ set(UNIT_TESTS_FILES gzip.c zstd.c random.c + thread_storage.c config_map.c mp.c mp_chunk_cobj.c @@ -34,6 +36,7 @@ set(UNIT_TESTS_FILES typecast.c base64.c bucket_queue.c + downstream_worker.c flb_event_loop.c ring_buffer.c regex.c @@ -54,10 +57,12 @@ set(UNIT_TESTS_FILES endianness.c task_map.c strptime.c + plugin_alias.c storage_inherit.c unicode.c opentelemetry.c storage_dlq.c + engine_adaptive_flush.c ) if(FLB_OUT_AZURE_BLOB) @@ -193,6 +198,7 @@ if(FLB_RIPSER) endif() set(UNIT_TESTS_DATA + data/fips/disabled.cnf data/tls/certificate.pem data/tls/private_key.pem data/pack/json_single_map_001.json @@ -244,7 +250,9 @@ function(prepare_unit_tests TEST_PREFIX SOURCEFILES) target_link_libraries(${source_file_we} flb-aws) endif() - if(FLB_OUT_AZURE_BLOB AND "${source_file_we}" STREQUAL "flb-it-azure_blob_path") + if(FLB_OUT_AZURE_BLOB AND + "${source_file_we}" STREQUAL "flb-it-azure_blob_path" AND + "${FLB_TEST_LINK_LIBRARY}" STREQUAL "fluent-bit-static") target_link_libraries(${source_file_we} flb-plugin-out_azure_blob) endif() @@ -252,7 +260,7 @@ function(prepare_unit_tests TEST_PREFIX SOURCEFILES) target_link_libraries(${source_file_we} flb-sp) endif() - target_link_libraries(${source_file_we} fluent-bit-static cfl-static) + target_link_libraries(${source_file_we} ${FLB_TEST_LINK_LIBRARY} cfl-static) if(FLB_AVRO_ENCODER) target_link_libraries(${source_file_we} avro-static jansson) @@ -268,6 +276,49 @@ endfunction(prepare_unit_tests) prepare_unit_tests(flb-it- "${UNIT_TESTS_FILES}") +option(FLB_TESTS_FIPS_ENABLED "Run tests that require an installed OpenSSL FIPS provider" OFF) +if(TARGET flb-it-fips) + if(FLB_OUT_S3) + target_compile_definitions(flb-it-fips PRIVATE FLB_TEST_FIPS_S3) + endif() + + set(FLB_TEST_FIPS_DISABLED_CONFIG + "${CMAKE_CURRENT_BINARY_DIR}/data/fips/disabled.cnf") + set(FLB_TEST_FIPS_MISSING_MODULES + "${CMAKE_CURRENT_BINARY_DIR}/data/fips/missing-modules") + set(FLB_TEST_FIPS_UNAVAILABLE_ENV + "FLB_TEST_FIPS_MODE=unavailable" + "OPENSSL_CONF=${FLB_TEST_FIPS_DISABLED_CONFIG}" + "OPENSSL_MODULES=${FLB_TEST_FIPS_MISSING_MODULES}") + set(FLB_TEST_FIPS_DISABLED_ENV + "FLB_TEST_FIPS_MODE=disabled" + "OPENSSL_CONF=${FLB_TEST_FIPS_DISABLED_CONFIG}" + "OPENSSL_MODULES=${FLB_TEST_FIPS_MISSING_MODULES}") + + set_tests_properties(flb-it-fips PROPERTIES + ENVIRONMENT "${FLB_TEST_FIPS_DISABLED_ENV}") + + add_test(NAME flb-it-fips-unavailable + COMMAND $) + set_tests_properties(flb-it-fips-unavailable PROPERTIES + LABELS "internal" + ENVIRONMENT "${FLB_TEST_FIPS_UNAVAILABLE_ENV}") + + if(FLB_TESTS_FIPS_ENABLED) + add_test(NAME flb-it-fips-enabled + COMMAND $) + set_tests_properties(flb-it-fips-enabled PROPERTIES + LABELS "internal" + ENVIRONMENT "FLB_TEST_FIPS_MODE=enabled") + + add_test(NAME flb-it-fips-preenabled + COMMAND $) + set_tests_properties(flb-it-fips-preenabled PROPERTIES + LABELS "internal" + ENVIRONMENT "FLB_TEST_FIPS_MODE=preenabled") + endif() +endif() + if(FLB_METRICS AND FLB_PROCESSOR_CUMULATIVE_TO_DELTA) set(CUMULATIVE_TO_DELTA_PLUGIN_DIR ${PROJECT_SOURCE_DIR}/plugins/processor_cumulative_to_delta) diff --git a/tests/internal/aws_compress.c b/tests/internal/aws_compress.c index bf55f425c50..0bb4414c419 100644 --- a/tests/internal/aws_compress.c +++ b/tests/internal/aws_compress.c @@ -105,7 +105,7 @@ void test_compression_snappy_return_value_normalization() { /* This test verifies that the snappy wrapper correctly normalizes return values * to conform to the AWS compression interface contract: -1 on error, 0 on success. - * + * * The test uses the actual flb_aws_compression_compress function which internally * uses the wrapper. We verify that successful compression returns exactly 0, * demonstrating that the wrapper properly normalizes the return value. @@ -115,17 +115,17 @@ void test_compression_snappy_return_value_normalization() size_t out_len = 0; int compression_type; char test_data[] = "test data for compression"; - + compression_type = flb_aws_compression_get_type("snappy"); TEST_CHECK(compression_type != -1); - + /* Test successful compression - should return exactly 0 (not any other value) */ - ret = flb_aws_compression_compress(compression_type, test_data, + ret = flb_aws_compression_compress(compression_type, test_data, strlen(test_data), &out_data, &out_len); TEST_CHECK(ret == 0); TEST_MSG("Expected return value 0 on success, got: %d", ret); TEST_MSG("This verifies the wrapper returns 0 (not passthrough of underlying function)"); - + if (ret == 0 && out_data != NULL) { TEST_CHECK(out_len > 0); TEST_MSG("Compressed data length: %zu", out_len); @@ -334,6 +334,194 @@ void test_b64_truncated_gzip_boundary() flb_aws_compress_truncate_b64_test_cases__gzip_decode(cases, 40); } +#ifdef FLB_HAVE_ARROW_PARQUET +void test_parquet_format_snappy() +{ + int ret; + void *out_buf = NULL; + size_t out_size = 0; + char *json = "{\"key\":\"value\",\"num\":42}\n" + "{\"key\":\"other\",\"num\":99}\n"; + size_t json_len = strlen(json); + + ret = out_s3_compress_columnar(FLB_AWS_COMPRESS_FORMAT_PARQUET, + json, json_len, &out_buf, &out_size, + FLB_AWS_COMPRESS_SNAPPY); + if (!TEST_CHECK(ret == 0 && out_buf != NULL && out_size >= 8)) { + TEST_MSG("Parquet SNAPPY conversion failed"); + return; + } + TEST_CHECK(memcmp(out_buf, "PAR1", 4) == 0); + TEST_CHECK(memcmp((char *)out_buf + out_size - 4, "PAR1", 4) == 0); + flb_free(out_buf); +} + +void test_parquet_format_zstd() +{ + int ret; + void *out_buf = NULL; + size_t out_size = 0; + char *json = "{\"key\":\"value\",\"num\":42}\n" + "{\"key\":\"other\",\"num\":99}\n"; + size_t json_len = strlen(json); + + ret = out_s3_compress_columnar(FLB_AWS_COMPRESS_FORMAT_PARQUET, + json, json_len, &out_buf, &out_size, + FLB_AWS_COMPRESS_ZSTD); + if (!TEST_CHECK(ret == 0 && out_buf != NULL && out_size >= 8)) { + TEST_MSG("Parquet ZSTD conversion failed"); + return; + } + TEST_CHECK(memcmp(out_buf, "PAR1", 4) == 0); + TEST_CHECK(memcmp((char *)out_buf + out_size - 4, "PAR1", 4) == 0); + flb_free(out_buf); +} + +void test_parquet_format_gzip() +{ + int ret; + void *out_buf = NULL; + size_t out_size = 0; + char *json = "{\"key\":\"value\",\"num\":42}\n" + "{\"key\":\"other\",\"num\":99}\n"; + size_t json_len = strlen(json); + + ret = out_s3_compress_columnar(FLB_AWS_COMPRESS_FORMAT_PARQUET, + json, json_len, &out_buf, &out_size, + FLB_AWS_COMPRESS_GZIP); + if (!TEST_CHECK(ret == 0 && out_buf != NULL && out_size >= 8)) { + TEST_MSG("Parquet GZIP conversion failed"); + return; + } + TEST_CHECK(memcmp(out_buf, "PAR1", 4) == 0); + TEST_CHECK(memcmp((char *)out_buf + out_size - 4, "PAR1", 4) == 0); + flb_free(out_buf); +} + +void test_parquet_format_uncompressed() +{ + int ret; + void *out_buf = NULL; + size_t out_size = 0; + char *json = "{\"key\":\"value\",\"num\":42}\n" + "{\"key\":\"other\",\"num\":99}\n"; + size_t json_len = strlen(json); + + ret = out_s3_compress_columnar(FLB_AWS_COMPRESS_FORMAT_PARQUET, + json, json_len, &out_buf, &out_size, + FLB_AWS_COMPRESS_NONE); + if (!TEST_CHECK(ret == 0 && out_buf != NULL && out_size >= 8)) { + TEST_MSG("Parquet NONE conversion failed"); + return; + } + TEST_CHECK(memcmp(out_buf, "PAR1", 4) == 0); + TEST_CHECK(memcmp((char *)out_buf + out_size - 4, "PAR1", 4) == 0); + flb_free(out_buf); +} + +void test_parquet_compression_reduces_size() +{ + int ret; + void *buf_none = NULL; + void *buf_snappy = NULL; + size_t size_none = 0; + size_t size_snappy = 0; + char *json = "{\"msg\":\"hello hello hello hello hello hello\"}\n" + "{\"msg\":\"hello hello hello hello hello hello\"}\n" + "{\"msg\":\"hello hello hello hello hello hello\"}\n" + "{\"msg\":\"hello hello hello hello hello hello\"}\n" + "{\"msg\":\"hello hello hello hello hello hello\"}\n"; + size_t json_len = strlen(json); + + ret = out_s3_compress_columnar(FLB_AWS_COMPRESS_FORMAT_PARQUET, + json, json_len, &buf_none, &size_none, + FLB_AWS_COMPRESS_NONE); + if (!TEST_CHECK(ret == 0 && buf_none != NULL)) { + TEST_MSG("Parquet NONE conversion failed"); + return; + } + + ret = out_s3_compress_columnar(FLB_AWS_COMPRESS_FORMAT_PARQUET, + json, json_len, &buf_snappy, &size_snappy, + FLB_AWS_COMPRESS_SNAPPY); + if (!TEST_CHECK(ret == 0 && buf_snappy != NULL)) { + TEST_MSG("Parquet SNAPPY conversion failed"); + flb_free(buf_none); + return; + } + TEST_CHECK(size_snappy <= size_none); + + flb_free(buf_none); + flb_free(buf_snappy); +} +#endif + +#ifdef FLB_HAVE_ARROW +void test_arrow_format_uncompressed() +{ + int ret; + void *out_buf = NULL; + size_t out_size = 0; + char *json = "{\"key\":\"value\",\"num\":42}\n" + "{\"key\":\"other\",\"num\":99}\n"; + size_t json_len = strlen(json); + + ret = out_s3_compress_columnar(FLB_AWS_COMPRESS_FORMAT_ARROW, + json, json_len, &out_buf, &out_size, + FLB_AWS_COMPRESS_NONE); + if (!TEST_CHECK(ret == 0 && out_buf != NULL && out_size >= 8)) { + TEST_MSG("Arrow NONE conversion failed"); + return; + } + /* Arrow/Feather V2 files begin with the "ARROW1" magic */ + TEST_CHECK(memcmp(out_buf, "ARROW1", 6) == 0); + flb_free(out_buf); +} + +void test_arrow_format_zstd() +{ + int ret; + void *out_buf = NULL; + size_t out_size = 0; + char *json = "{\"key\":\"value\",\"num\":42}\n" + "{\"key\":\"other\",\"num\":99}\n"; + size_t json_len = strlen(json); + + ret = out_s3_compress_columnar(FLB_AWS_COMPRESS_FORMAT_ARROW, + json, json_len, &out_buf, &out_size, + FLB_AWS_COMPRESS_ZSTD); + if (!TEST_CHECK(ret == 0 && out_buf != NULL && out_size >= 8)) { + TEST_MSG("Arrow ZSTD conversion failed"); + return; + } + TEST_CHECK(memcmp(out_buf, "ARROW1", 6) == 0); + flb_free(out_buf); +} + +/* + * Arrow/Feather (Arrow IPC) only supports ZSTD compression. Codecs such as + * gzip must be rejected by the writer; out_s3 also rejects them at config + * time via validate_format_compression. + */ +void test_arrow_format_gzip_unsupported() +{ + int ret; + void *out_buf = NULL; + size_t out_size = 0; + char *json = "{\"key\":\"value\",\"num\":42}\n" + "{\"key\":\"other\",\"num\":99}\n"; + size_t json_len = strlen(json); + + ret = out_s3_compress_columnar(FLB_AWS_COMPRESS_FORMAT_ARROW, + json, json_len, &out_buf, &out_size, + FLB_AWS_COMPRESS_GZIP); + TEST_CHECK(ret == -1); + if (out_buf != NULL) { + flb_free(out_buf); + } +} +#endif + TEST_LIST = { { "test_compression_gzip", test_compression_gzip }, { "test_compression_zstd", test_compression_zstd }, @@ -352,6 +540,19 @@ TEST_LIST = { test_b64_truncated_gzip_truncation_multi_rounds }, { "test_b64_truncated_gzip_boundary", test_b64_truncated_gzip_boundary }, +#ifdef FLB_HAVE_ARROW_PARQUET + { "test_parquet_format_snappy", test_parquet_format_snappy }, + { "test_parquet_format_zstd", test_parquet_format_zstd }, + { "test_parquet_format_gzip", test_parquet_format_gzip }, + { "test_parquet_format_uncompressed", test_parquet_format_uncompressed }, + { "test_parquet_compression_reduces_size", + test_parquet_compression_reduces_size }, +#endif +#ifdef FLB_HAVE_ARROW + { "test_arrow_format_uncompressed", test_arrow_format_uncompressed }, + { "test_arrow_format_zstd", test_arrow_format_zstd }, + { "test_arrow_format_gzip_unsupported", test_arrow_format_gzip_unsupported }, +#endif { 0 } }; @@ -419,8 +620,8 @@ static void flb_aws_compress_general_test_cases(int test_type, while (tcase->compression_keyword != 0) { size_t in_data_len = strlen(tcase->in_data); - compression_type = flb_aws_compression_get_type(tcase->compression_keyword); - + compression_type = flb_aws_compression_get_type(tcase->compression_keyword); + TEST_CHECK(compression_type != -1); TEST_MSG("| flb_aws_get_compression_type: failed to get compression type for " "keyword " diff --git a/tests/internal/azure_blob_path.c b/tests/internal/azure_blob_path.c index d89117ad29b..1ddf3d1556d 100644 --- a/tests/internal/azure_blob_path.c +++ b/tests/internal/azure_blob_path.c @@ -6,6 +6,7 @@ #include #include #include +#include #include "../../plugins/out_azure_blob/azure_blob.h" #include "../../plugins/out_azure_blob/azure_blob_uri.h" @@ -478,6 +479,45 @@ void test_block_blob_commit_requires_suffix(void) ctx_cleanup(&ctx); } +void test_block_blob_id_uses_sha256_in_fips_mode(void) +{ + struct flb_config config; + struct flb_azure_blob ctx; + char *default_id; + char *fips_id; + const char *expected_id; + + memset(&config, 0, sizeof(config)); + memset(&ctx, 0, sizeof(ctx)); + + ctx.config = &config; + + config.fips_mode_active = FLB_FALSE; + default_id = azb_block_blob_id_blob(&ctx, "blob/path.log", 1); + TEST_CHECK(default_id != NULL); + if (default_id == NULL) { + return; + } + + config.fips_mode_active = FLB_TRUE; + fips_id = azb_block_blob_id_blob(&ctx, "blob/path.log", 1); + TEST_CHECK(fips_id != NULL); + if (fips_id == NULL) { + flb_free(default_id); + return; + } + + TEST_CHECK(strlen(default_id) == 64); + TEST_CHECK(strlen(fips_id) == 64); + TEST_CHECK(strcmp(default_id, fips_id) != 0); + + expected_id = "NjM1NTgzYjViZjZhMDU4Y2VmNjhkMWNjMjZlM2ZmNzcuZmxiLXBhcnQuMDAwMDAx"; + TEST_CHECK(strcmp(fips_id, expected_id) == 0); + + flb_free(default_id); + flb_free(fips_id); +} + TEST_LIST = { {"resolve_path_basic_tag", test_resolve_path_basic_tag}, {"resolve_path_custom_delimiter", test_resolve_path_custom_delimiter}, @@ -491,5 +531,6 @@ TEST_LIST = { {"commit_prefix_fallback_static_path", test_commit_prefix_fallback_static_path}, {"uri_create_static_prefix_fallback", test_uri_create_static_prefix_fallback}, {"block_blob_commit_requires_suffix", test_block_blob_commit_requires_suffix}, + {"block_blob_id_uses_sha256_in_fips_mode", test_block_blob_id_uses_sha256_in_fips_mode}, {0} }; diff --git a/tests/internal/data/fips/disabled.cnf b/tests/internal/data/fips/disabled.cnf new file mode 100644 index 00000000000..af2a5fee067 --- /dev/null +++ b/tests/internal/data/fips/disabled.cnf @@ -0,0 +1,4 @@ +# Intentionally empty OpenSSL initialization for non-FIPS test processes. +openssl_conf = openssl_init + +[openssl_init] diff --git a/tests/internal/downstream_worker.c b/tests/internal/downstream_worker.c new file mode 100644 index 00000000000..b3e4a16be69 --- /dev/null +++ b/tests/internal/downstream_worker.c @@ -0,0 +1,510 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +#include +#include +#include +#ifdef FLB_SYSTEM_WINDOWS +#include +#endif + +#include + +#include "flb_tests_internal.h" + +struct downstream_worker_test_context { + pthread_mutex_t mutex; + struct flb_downstream_worker_runtime *runtime; + int init_calls; + int exit_calls; + int foreach_calls; + int worker_ids; + int accessor_failures; + int nested_foreach_failures; + int nested_stop_failures; + int fail_worker_id; +}; + +struct downstream_worker_foreach_call { + struct flb_downstream_worker_runtime *runtime; + int callback_calls; + int result; +}; + +struct downstream_worker_stop_context { + pthread_mutex_t mutex; + pthread_cond_t condition; + struct flb_downstream_worker_runtime *runtime; + int callback_entered; + int callback_released; + int foreach_result; + int stop_result; +}; + +static int downstream_worker_test_network_init(void) +{ +#ifdef FLB_SYSTEM_WINDOWS + int ret; + WSADATA wsa_data; + + ret = WSAStartup(MAKEWORD(2, 2), &wsa_data); + TEST_CHECK(ret == 0); + return ret; +#else + return 0; +#endif +} + +static void downstream_worker_test_network_cleanup(void) +{ +#ifdef FLB_SYSTEM_WINDOWS + WSACleanup(); +#endif +} + +static void downstream_worker_test_context_init( + struct downstream_worker_test_context *context) +{ + memset(context, 0, sizeof(struct downstream_worker_test_context)); + pthread_mutex_init(&context->mutex, NULL); + context->fail_worker_id = -1; +} + +static void downstream_worker_test_context_destroy( + struct downstream_worker_test_context *context) +{ + pthread_mutex_destroy(&context->mutex); +} + +static int downstream_worker_test_init(struct flb_downstream_worker *worker, + void *parent, + void **worker_context) +{ + int worker_id; + struct downstream_worker_test_context *context; + + context = parent; + worker_id = flb_downstream_worker_id_get(worker); + *worker_context = context; + + pthread_mutex_lock(&context->mutex); + context->init_calls++; + pthread_mutex_unlock(&context->mutex); + + if (worker_id == context->fail_worker_id) { + return -1; + } + + return 0; +} + +static void downstream_worker_test_exit(struct flb_downstream_worker *worker, + void *worker_context) +{ + struct downstream_worker_test_context *context; + + (void) worker; + context = worker_context; + + pthread_mutex_lock(&context->mutex); + context->exit_calls++; + pthread_mutex_unlock(&context->mutex); +} + +static void downstream_worker_test_noop(struct flb_downstream_worker *worker, + void *worker_context, + void *data) +{ + (void) worker; + (void) worker_context; + (void) data; +} + +static void downstream_worker_test_count(struct flb_downstream_worker *worker, + void *worker_context, + void *data) +{ + struct downstream_worker_foreach_call *call; + + (void) worker; + (void) worker_context; + + call = data; + call->callback_calls++; +} + +static void *downstream_worker_test_foreach_thread(void *data) +{ + struct downstream_worker_foreach_call *call; + + call = data; + call->result = flb_downstream_worker_runtime_foreach( + call->runtime, downstream_worker_test_count, call); + + return NULL; +} + +static void downstream_worker_test_block(struct flb_downstream_worker *worker, + void *worker_context, + void *data) +{ + struct downstream_worker_stop_context *stop_context; + + (void) worker; + (void) worker_context; + + stop_context = data; + pthread_mutex_lock(&stop_context->mutex); + stop_context->callback_entered = FLB_TRUE; + pthread_cond_broadcast(&stop_context->condition); + while (stop_context->callback_released == FLB_FALSE) { + pthread_cond_wait(&stop_context->condition, &stop_context->mutex); + } + pthread_mutex_unlock(&stop_context->mutex); +} + +static void *downstream_worker_test_stop_thread(void *data) +{ + struct downstream_worker_stop_context *stop_context; + + stop_context = data; + stop_context->stop_result = flb_downstream_worker_runtime_stop( + stop_context->runtime); + + return NULL; +} + +static void *downstream_worker_test_blocking_foreach_thread(void *data) +{ + struct downstream_worker_stop_context *stop_context; + + stop_context = data; + stop_context->foreach_result = flb_downstream_worker_runtime_foreach( + stop_context->runtime, downstream_worker_test_block, stop_context); + + return NULL; +} + +static void downstream_worker_test_foreach(struct flb_downstream_worker *worker, + void *worker_context, + void *data) +{ + int worker_count; + int worker_id; + int nested_foreach_result; + int nested_stop_result; + struct downstream_worker_test_context *context; + + (void) data; + context = worker_context; + worker_id = flb_downstream_worker_id_get(worker); + worker_count = flb_downstream_worker_count_get(worker); + + nested_foreach_result = flb_downstream_worker_runtime_foreach( + context->runtime, downstream_worker_test_noop, NULL); + nested_stop_result = flb_downstream_worker_runtime_stop(context->runtime); + + pthread_mutex_lock(&context->mutex); + context->foreach_calls++; + + if (worker_id < 0 || worker_id >= worker_count || worker_count != 2 || + flb_downstream_worker_event_loop_get(worker) == NULL) { + context->accessor_failures++; + } + else { + context->worker_ids |= (1 << worker_id); + } + + if (nested_foreach_result != -1) { + context->nested_foreach_failures++; + } + + if (nested_stop_result != -1) { + context->nested_stop_failures++; + } + pthread_mutex_unlock(&context->mutex); +} + +void test_downstream_worker_validation() +{ + int ret; + struct flb_downstream_worker_options options; + struct flb_downstream_worker_runtime *runtime; + + memset(&options, 0, sizeof(struct flb_downstream_worker_options)); + runtime = (struct flb_downstream_worker_runtime *) 0x1; + + ret = flb_downstream_worker_runtime_start(&runtime, &options); + TEST_CHECK(ret == -1); + TEST_CHECK(runtime == NULL); + TEST_CHECK(flb_downstream_worker_runtime_start(NULL, &options) == -1); + TEST_CHECK(flb_downstream_worker_runtime_foreach(NULL, + downstream_worker_test_noop, + NULL) == -1); + TEST_CHECK(flb_downstream_worker_runtime_stop(NULL) == 0); + TEST_CHECK(flb_downstream_worker_event_loop_get(NULL) == NULL); + TEST_CHECK(flb_downstream_worker_id_get(NULL) == -1); + TEST_CHECK(flb_downstream_worker_count_get(NULL) == -1); + TEST_CHECK(flb_downstream_worker_listener_fd_set(NULL, + FLB_INVALID_SOCKET) == -1); +} + +void test_downstream_worker_lifecycle() +{ + int ret; + struct downstream_worker_test_context context; + struct flb_downstream_worker_options options; + struct flb_downstream_worker_runtime *runtime; + + ret = downstream_worker_test_network_init(); + if (ret != 0) { + return; + } + + downstream_worker_test_context_init(&context); + memset(&options, 0, sizeof(struct flb_downstream_worker_options)); + options.workers = 2; + options.parent = &context; + options.cb_init = downstream_worker_test_init; + options.cb_exit = downstream_worker_test_exit; + + runtime = NULL; + ret = flb_downstream_worker_runtime_start(&runtime, &options); + TEST_CHECK(ret == 0); + TEST_CHECK(runtime != NULL); + if (ret != 0) { + downstream_worker_test_context_destroy(&context); + downstream_worker_test_network_cleanup(); + return; + } + + context.runtime = runtime; + ret = flb_downstream_worker_runtime_foreach(runtime, + downstream_worker_test_foreach, + NULL); + TEST_CHECK(ret == 0); + TEST_CHECK(context.init_calls == 2); + TEST_CHECK(context.foreach_calls == 2); + TEST_CHECK(context.worker_ids == 3); + TEST_CHECK(context.accessor_failures == 0); + TEST_CHECK(context.nested_foreach_failures == 0); + TEST_CHECK(context.nested_stop_failures == 0); + + ret = flb_downstream_worker_runtime_stop(runtime); + TEST_CHECK(ret == 0); + TEST_CHECK(context.exit_calls == 2); + + downstream_worker_test_context_destroy(&context); + downstream_worker_test_network_cleanup(); +} + +void test_downstream_worker_startup_rollback() +{ + int ret; + struct downstream_worker_test_context context; + struct flb_downstream_worker_options options; + struct flb_downstream_worker_runtime *runtime; + + ret = downstream_worker_test_network_init(); + if (ret != 0) { + return; + } + + downstream_worker_test_context_init(&context); + context.fail_worker_id = 1; + + memset(&options, 0, sizeof(struct flb_downstream_worker_options)); + options.workers = 3; + options.parent = &context; + options.cb_init = downstream_worker_test_init; + options.cb_exit = downstream_worker_test_exit; + + runtime = (struct flb_downstream_worker_runtime *) 0x1; + ret = flb_downstream_worker_runtime_start(&runtime, &options); + TEST_CHECK(ret == -1); + TEST_CHECK(runtime == NULL); + TEST_CHECK(context.init_calls == 2); + TEST_CHECK(context.exit_calls == 2); + + downstream_worker_test_context_destroy(&context); + downstream_worker_test_network_cleanup(); +} + +void test_downstream_worker_concurrent_foreach() +{ + int i; + int ret; + int threads_created; + pthread_t threads[4]; + struct downstream_worker_foreach_call calls[4]; + struct downstream_worker_test_context context; + struct flb_downstream_worker_options options; + struct flb_downstream_worker_runtime *runtime; + + ret = downstream_worker_test_network_init(); + if (ret != 0) { + return; + } + + downstream_worker_test_context_init(&context); + memset(&options, 0, sizeof(struct flb_downstream_worker_options)); + options.workers = 2; + options.parent = &context; + options.cb_init = downstream_worker_test_init; + options.cb_exit = downstream_worker_test_exit; + + runtime = NULL; + ret = flb_downstream_worker_runtime_start(&runtime, &options); + if (!TEST_CHECK(ret == 0)) { + downstream_worker_test_context_destroy(&context); + downstream_worker_test_network_cleanup(); + return; + } + + memset(calls, 0, sizeof(calls)); + threads_created = 0; + for (i = 0; i < 4; i++) { + calls[i].runtime = runtime; + ret = pthread_create(&threads[i], NULL, + downstream_worker_test_foreach_thread, + &calls[i]); + if (!TEST_CHECK(ret == 0)) { + break; + } + threads_created++; + } + + for (i = 0; i < threads_created; i++) { + ret = pthread_join(threads[i], NULL); + TEST_CHECK(ret == 0); + TEST_CHECK(calls[i].result == 0); + TEST_CHECK(calls[i].callback_calls == 2); + } + + ret = flb_downstream_worker_runtime_stop(runtime); + TEST_CHECK(ret == 0); + TEST_CHECK(context.exit_calls == 2); + downstream_worker_test_context_destroy(&context); + downstream_worker_test_network_cleanup(); +} + +void test_downstream_worker_stop_waits_for_foreach() +{ + int ret; + pthread_t foreach_thread; + pthread_t queued_foreach_thread; + pthread_t stop_thread; + struct downstream_worker_foreach_call queued_call; + struct downstream_worker_stop_context stop_context; + struct downstream_worker_test_context context; + struct flb_downstream_worker_options options; + struct flb_downstream_worker_runtime *runtime; + + ret = downstream_worker_test_network_init(); + if (ret != 0) { + return; + } + + downstream_worker_test_context_init(&context); + memset(&options, 0, sizeof(struct flb_downstream_worker_options)); + options.workers = 1; + options.parent = &context; + options.cb_init = downstream_worker_test_init; + options.cb_exit = downstream_worker_test_exit; + + runtime = NULL; + ret = flb_downstream_worker_runtime_start(&runtime, &options); + if (!TEST_CHECK(ret == 0)) { + downstream_worker_test_context_destroy(&context); + downstream_worker_test_network_cleanup(); + return; + } + + memset(&stop_context, 0, sizeof(stop_context)); + pthread_mutex_init(&stop_context.mutex, NULL); + pthread_cond_init(&stop_context.condition, NULL); + stop_context.runtime = runtime; + + ret = pthread_create(&foreach_thread, NULL, + downstream_worker_test_blocking_foreach_thread, + &stop_context); + TEST_CHECK(ret == 0); + if (ret != 0) { + flb_downstream_worker_runtime_stop(runtime); + goto cleanup; + } + + pthread_mutex_lock(&stop_context.mutex); + while (stop_context.callback_entered == FLB_FALSE) { + pthread_cond_wait(&stop_context.condition, &stop_context.mutex); + } + pthread_mutex_unlock(&stop_context.mutex); + + memset(&queued_call, 0, sizeof(queued_call)); + queued_call.runtime = runtime; + ret = pthread_create(&queued_foreach_thread, NULL, + downstream_worker_test_foreach_thread, + &queued_call); + TEST_CHECK(ret == 0); + if (ret != 0) { + pthread_mutex_lock(&stop_context.mutex); + stop_context.callback_released = FLB_TRUE; + pthread_cond_broadcast(&stop_context.condition); + pthread_mutex_unlock(&stop_context.mutex); + pthread_join(foreach_thread, NULL); + flb_downstream_worker_runtime_stop(runtime); + goto cleanup; + } + + /* Give the second operation time to enter admission and queue behind the + * blocking foreach call before shutdown begins. + */ + flb_time_msleep(50); + + ret = pthread_create(&stop_thread, NULL, + downstream_worker_test_stop_thread, &stop_context); + TEST_CHECK(ret == 0); + if (ret != 0) { + pthread_mutex_lock(&stop_context.mutex); + stop_context.callback_released = FLB_TRUE; + pthread_cond_broadcast(&stop_context.condition); + pthread_mutex_unlock(&stop_context.mutex); + pthread_join(foreach_thread, NULL); + pthread_join(queued_foreach_thread, NULL); + flb_downstream_worker_runtime_stop(runtime); + goto cleanup; + } + + pthread_mutex_lock(&stop_context.mutex); + stop_context.callback_released = FLB_TRUE; + pthread_cond_broadcast(&stop_context.condition); + pthread_mutex_unlock(&stop_context.mutex); + + ret = pthread_join(foreach_thread, NULL); + TEST_CHECK(ret == 0); + TEST_CHECK(stop_context.foreach_result == 0); + + ret = pthread_join(queued_foreach_thread, NULL); + TEST_CHECK(ret == 0); + TEST_CHECK(queued_call.result == 0); + TEST_CHECK(queued_call.callback_calls == 1); + + ret = pthread_join(stop_thread, NULL); + TEST_CHECK(ret == 0); + TEST_CHECK(stop_context.stop_result == 0); + TEST_CHECK(context.exit_calls == 1); + +cleanup: + pthread_cond_destroy(&stop_context.condition); + pthread_mutex_destroy(&stop_context.mutex); + downstream_worker_test_context_destroy(&context); + downstream_worker_test_network_cleanup(); +} + +TEST_LIST = { + { "downstream_worker_validation", test_downstream_worker_validation }, + { "downstream_worker_lifecycle", test_downstream_worker_lifecycle }, + { "downstream_worker_startup_rollback", test_downstream_worker_startup_rollback }, + { "downstream_worker_concurrent_foreach", test_downstream_worker_concurrent_foreach }, + { "downstream_worker_stop_waits_for_foreach", test_downstream_worker_stop_waits_for_foreach }, + { 0 } +}; diff --git a/tests/internal/engine_adaptive_flush.c b/tests/internal/engine_adaptive_flush.c new file mode 100644 index 00000000000..2cc5b58793f --- /dev/null +++ b/tests/internal/engine_adaptive_flush.c @@ -0,0 +1,70 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +#include +#include +#include "flb_tests_internal.h" + +static void test_target_level_thresholds() +{ + struct flb_config *config; + + config = flb_config_init(); + TEST_CHECK(config != NULL); + if (config == NULL) { + return; + } + + config->flush_adaptive_low_pressure = 25.0; + config->flush_adaptive_medium_pressure = 50.0; + config->flush_adaptive_high_pressure = 75.0; + + TEST_CHECK(flb_engine_adaptive_flush_target_level(config, 0.0) == 0); + TEST_CHECK(flb_engine_adaptive_flush_target_level(config, 25.0) == 0); + TEST_CHECK(flb_engine_adaptive_flush_target_level(config, 25.1) == 1); + TEST_CHECK(flb_engine_adaptive_flush_target_level(config, 49.9) == 1); + TEST_CHECK(flb_engine_adaptive_flush_target_level(config, 50.0) == 2); + TEST_CHECK(flb_engine_adaptive_flush_target_level(config, 74.9) == 2); + TEST_CHECK(flb_engine_adaptive_flush_target_level(config, 75.0) == 3); + TEST_CHECK(flb_engine_adaptive_flush_target_level(config, 100.0) == 3); + + flb_config_exit(config); +} + +static void test_interval_levels_and_bounds() +{ + struct flb_config *config; + + config = flb_config_init(); + TEST_CHECK(config != NULL); + if (config == NULL) { + return; + } + + config->flush = 1.0; + config->flush_adaptive_min_interval = 0.5; + config->flush_adaptive_max_interval = 2.0; + + TEST_CHECK(flb_engine_adaptive_flush_interval(config, 0) == 2.0); + TEST_CHECK(flb_engine_adaptive_flush_interval(config, 1) == 1.0); + TEST_CHECK(flb_engine_adaptive_flush_interval(config, 2) == 0.75); + TEST_CHECK(flb_engine_adaptive_flush_interval(config, 3) == 0.5); + + /* clamp low/high out-of-range levels */ + TEST_CHECK(flb_engine_adaptive_flush_interval(config, -1) == 2.0); + TEST_CHECK(flb_engine_adaptive_flush_interval(config, 9) == 0.5); + + /* clamp by min/max bounds */ + config->flush = 10.0; + TEST_CHECK(flb_engine_adaptive_flush_interval(config, 3) == 2.0); + + config->flush = 0.1; + TEST_CHECK(flb_engine_adaptive_flush_interval(config, 3) == 0.5); + + flb_config_exit(config); +} + +TEST_LIST = { + { "target_level_thresholds", test_target_level_thresholds }, + { "interval_levels_and_bounds", test_interval_levels_and_bounds }, + { 0 } +}; diff --git a/tests/internal/fips.c b/tests/internal/fips.c new file mode 100644 index 00000000000..3a9d32a0cc9 --- /dev/null +++ b/tests/internal/fips.c @@ -0,0 +1,295 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2026 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include + +#include +#include +#include +#include +#include +#include +#ifdef FLB_TEST_FIPS_S3 +#include +#endif + +#if defined(FLB_HAVE_OPENSSL) +#include +#if OPENSSL_VERSION_NUMBER >= 0x30000000L +#include +#include +#endif +#endif + +#include "flb_tests_internal.h" + +static int hash_payload(int hash_type, unsigned char *digest, size_t digest_size) +{ + const char *payload = "fips-provider-state"; + + return flb_hash_simple(hash_type, + (unsigned char *) payload, strlen(payload), + digest, digest_size); +} + +static void check_md5_available(void) +{ + int ret; + struct flb_hmac hmac_context; + unsigned char digest[32]; + const unsigned char key[] = "fips-test-key"; + + ret = hash_payload(FLB_HASH_MD5, digest, sizeof(digest)); + TEST_CHECK(ret == FLB_CRYPTO_SUCCESS); + + ret = flb_hmac_init(&hmac_context, FLB_HASH_MD5, + (unsigned char *) key, sizeof(key) - 1); + TEST_CHECK(ret == FLB_CRYPTO_SUCCESS); + if (ret == FLB_CRYPTO_SUCCESS) { + flb_hmac_cleanup(&hmac_context); + } +} + +static void check_fips_algorithms(void) +{ + int ret; + struct flb_hash hash_context; + struct flb_hmac hmac_context; + unsigned char digest[32]; + const unsigned char key[] = "fips-test-key"; + + ret = hash_payload(FLB_HASH_SHA256, digest, sizeof(digest)); + TEST_CHECK(ret == FLB_CRYPTO_SUCCESS); + + ret = flb_hash_init(&hash_context, FLB_HASH_MD5); + TEST_CHECK(ret == FLB_CRYPTO_BACKEND_ERROR); + TEST_CHECK(hash_context.backend_context == NULL); + + ret = flb_hmac_init(&hmac_context, FLB_HASH_MD5, + (unsigned char *) key, sizeof(key) - 1); + TEST_CHECK(ret == FLB_CRYPTO_BACKEND_ERROR); + TEST_CHECK(hmac_context.backend_context == NULL); +#if FLB_CRYPTO_OPENSSL_COMPAT_MODE >= 3 + TEST_CHECK(hmac_context.mac_algorithm == NULL); +#endif +} + +#ifdef FLB_TEST_FIPS_S3 +static void check_s3_content_md5_rejected(void) +{ + int ret; + int input_id; + int output_id; + flb_ctx_t *ctx; + + ctx = flb_create(); + TEST_CHECK(ctx != NULL); + if (ctx == NULL) { + return; + } + + input_id = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(input_id >= 0); + if (input_id < 0) { + flb_destroy(ctx); + return; + } + ret = flb_input_set(ctx, input_id, "tag", "fips.test", NULL); + TEST_CHECK(ret == 0); + + output_id = flb_output(ctx, (char *) "s3", NULL); + TEST_CHECK(output_id >= 0); + if (output_id < 0) { + flb_destroy(ctx); + return; + } + + ret = flb_output_set(ctx, output_id, + "match", "*", + "region", "us-east-1", + "bucket", "fips-test", + "send_content_md5", "true", + NULL); + TEST_CHECK(ret == 0); + + ret = flb_start(ctx); + TEST_CHECK(ret == -1); + if (ret == 0) { + flb_stop(ctx); + } + flb_destroy(ctx); +} +#endif + +#if defined(FLB_HAVE_OPENSSL) && OPENSSL_VERSION_NUMBER >= 0x30000000L +struct fips_test_provider_state { + int base_loaded; + int fips_loaded; +}; + +static int collect_provider_state(OSSL_PROVIDER *provider, void *data) +{ + const char *name; + struct fips_test_provider_state *state; + + state = data; + name = OSSL_PROVIDER_get0_name(provider); + if (name == NULL) { + return 1; + } + + if (strcmp(name, "base") == 0) { + state->base_loaded = FLB_TRUE; + } + else if (strcmp(name, "fips") == 0) { + state->fips_loaded = FLB_TRUE; + } + + return 1; +} +#endif + +void test_fips_environment(void) +{ + int ret; + struct flb_config config; + struct flb_config second_config; + const char *mode; +#if defined(FLB_HAVE_OPENSSL) && OPENSSL_VERSION_NUMBER >= 0x30000000L + OSSL_PROVIDER *preenabled_provider; + struct fips_test_provider_state provider_state; +#endif + + memset(&config, 0, sizeof(config)); + memset(&second_config, 0, sizeof(second_config)); + + TEST_CHECK(flb_fips_init(NULL) == -1); + + mode = getenv("FLB_TEST_FIPS_MODE"); + if (mode == NULL || mode[0] == '\0') { + config.fips_mode = FLB_FALSE; + ret = flb_fips_init(&config); + TEST_CHECK(ret == 0); + + if (config.fips_mode_active == FLB_TRUE) { + check_fips_algorithms(); + } + else { + check_md5_available(); + } + return; + } + + if (strcmp(mode, "disabled") == 0) { + config.fips_mode = FLB_FALSE; + ret = flb_fips_init(&config); + TEST_CHECK(ret == 0); + TEST_CHECK(config.fips_mode_active == FLB_FALSE); + + check_md5_available(); + + config.fips_mode = FLB_TRUE; + ret = flb_fips_init(&config); + TEST_CHECK(ret == -1); + return; + } + + if (strcmp(mode, "preenabled") == 0) { +#if defined(FLB_HAVE_OPENSSL) && OPENSSL_VERSION_NUMBER >= 0x30000000L + preenabled_provider = OSSL_PROVIDER_load(NULL, "fips"); + TEST_CHECK(preenabled_provider != NULL); + if (preenabled_provider == NULL) { + return; + } + + ret = EVP_default_properties_enable_fips(NULL, 1); + TEST_CHECK(ret == 1); + if (ret != 1) { + OSSL_PROVIDER_unload(preenabled_provider); + return; + } + + config.fips_mode = FLB_FALSE; + ret = flb_fips_init(&config); + TEST_CHECK(ret == 0); + TEST_CHECK(config.fips_mode_active == FLB_TRUE); + if (ret != 0) { + OSSL_PROVIDER_unload(preenabled_provider); + return; + } + + OSSL_PROVIDER_unload(preenabled_provider); + + memset(&provider_state, 0, sizeof(provider_state)); + ret = OSSL_PROVIDER_do_all(NULL, collect_provider_state, &provider_state); + TEST_CHECK(ret == 1); + TEST_CHECK(provider_state.fips_loaded == FLB_TRUE); + TEST_CHECK(provider_state.base_loaded == FLB_TRUE); + + check_fips_algorithms(); +#ifdef FLB_TEST_FIPS_S3 + check_s3_content_md5_rejected(); +#endif +#else + TEST_MSG("preenabled mode requires OpenSSL 3.0 or later"); + TEST_CHECK(0); +#endif + return; + } + + config.fips_mode = FLB_TRUE; + ret = flb_fips_init(&config); + + if (strcmp(mode, "unavailable") == 0) { + TEST_CHECK(ret == -1); + TEST_CHECK(config.fips_mode_active == FLB_FALSE); + + config.fips_mode = FLB_FALSE; + ret = flb_fips_init(&config); + TEST_CHECK(ret == 0); + + check_md5_available(); + return; + } + + if (strcmp(mode, "enabled") == 0) { + TEST_CHECK(ret == 0); + TEST_CHECK(config.fips_mode_active == FLB_TRUE); + + check_fips_algorithms(); +#ifdef FLB_TEST_FIPS_S3 + check_s3_content_md5_rejected(); +#endif + + second_config.fips_mode = FLB_FALSE; + ret = flb_fips_init(&second_config); + TEST_CHECK(ret == 0); + TEST_CHECK(second_config.fips_mode_active == FLB_TRUE); + return; + } + + TEST_MSG("unknown FLB_TEST_FIPS_MODE value: %s", mode); + TEST_CHECK(0); +} + +TEST_LIST = { + {"fips_environment", test_fips_environment}, + {0} +}; diff --git a/tests/internal/flb_time.c b/tests/internal/flb_time.c index e39f3154f66..3c0597196af 100644 --- a/tests/internal/flb_time.c +++ b/tests/internal/flb_time.c @@ -213,6 +213,71 @@ void test_msgpack_to_time_eventtime() msgpack_unpacked_destroy(&result); } +void test_eventtime_boundaries() +{ + struct flb_time tm; + msgpack_packer mp_pck; + msgpack_sbuffer mp_sbuf; + msgpack_unpacked result; + msgpack_object tm_obj; + uint32_t value[2]; + int ret; + + msgpack_sbuffer_init(&mp_sbuf); + msgpack_packer_init(&mp_pck, &mp_sbuf, msgpack_sbuffer_write); + value[0] = htonl(UINT32_MAX - 2); + value[1] = htonl(999999999); + msgpack_pack_ext(&mp_pck, 8, 0); + msgpack_pack_ext_body(&mp_pck, value, sizeof(value)); + + msgpack_unpacked_init(&result); + msgpack_unpack_next(&result, mp_sbuf.data, mp_sbuf.size, NULL); + tm_obj = result.data; + ret = flb_time_msgpack_to_time(&tm, &tm_obj); + TEST_CHECK(ret == 0); + TEST_CHECK(tm.tm.tv_sec == (time_t) (UINT32_MAX - 2)); + TEST_CHECK(tm.tm.tv_nsec == 999999999); + msgpack_unpacked_destroy(&result); + + msgpack_sbuffer_clear(&mp_sbuf); + value[0] = htonl(2209072510U); + value[1] = htonl(808241446); + msgpack_pack_ext(&mp_pck, 8, 0); + msgpack_pack_ext_body(&mp_pck, value, sizeof(value)); + msgpack_unpacked_init(&result); + msgpack_unpack_next(&result, mp_sbuf.data, mp_sbuf.size, NULL); + tm_obj = result.data; + ret = flb_time_msgpack_to_time(&tm, &tm_obj); + TEST_CHECK(ret == 0); + TEST_CHECK(tm.tm.tv_sec == (time_t) 2209072510U); + TEST_CHECK(tm.tm.tv_nsec == 808241446); + msgpack_unpacked_destroy(&result); + + msgpack_sbuffer_clear(&mp_sbuf); + value[0] = htonl(2209072510U); + value[1] = htonl(1000000000U); + msgpack_pack_ext(&mp_pck, 8, 0); + msgpack_pack_ext_body(&mp_pck, value, sizeof(value)); + msgpack_unpacked_init(&result); + msgpack_unpack_next(&result, mp_sbuf.data, mp_sbuf.size, NULL); + tm_obj = result.data; + ret = flb_time_msgpack_to_time(&tm, &tm_obj); + TEST_CHECK(ret != 0); + msgpack_unpacked_destroy(&result); + msgpack_sbuffer_destroy(&mp_sbuf); +} + +void test_from_uint64_post_2038() +{ + struct flb_time tm; + int ret; + + ret = flb_time_from_uint64(&tm, UINT64_C(2209072510808241446)); + TEST_CHECK(ret == 0); + TEST_CHECK(tm.tm.tv_sec == (time_t) 2209072510U); + TEST_CHECK(tm.tm.tv_nsec == 808241446); +} + void test_msgpack_to_time_invalid() { struct flb_time tm; @@ -472,6 +537,8 @@ TEST_LIST = { { "msgpack_to_time_int" , test_msgpack_to_time_int}, { "msgpack_to_time_double" , test_msgpack_to_time_double}, { "msgpack_to_time_eventtime" , test_msgpack_to_time_eventtime}, + { "eventtime_boundaries" , test_eventtime_boundaries}, + { "from_uint64_post_2038" , test_from_uint64_post_2038}, { "msgpack_to_time_invalid" , test_msgpack_to_time_invalid}, { "append_to_msgpack_eventtime" , test_append_to_msgpack_eventtime}, { "windows_zone_to_iana" , test_windows_zone_to_iana}, diff --git a/tests/internal/fuzzers/CMakeLists.txt b/tests/internal/fuzzers/CMakeLists.txt index dfa6e4ff7fe..c04d9b7a06f 100644 --- a/tests/internal/fuzzers/CMakeLists.txt +++ b/tests/internal/fuzzers/CMakeLists.txt @@ -50,7 +50,7 @@ foreach(source_file ${UNIT_TESTS_FILES}) target_link_libraries(${source_file_we} flb-sp) endif() - target_link_libraries(${source_file_we} fluent-bit-static) + target_link_libraries(${source_file_we} ${FLB_TEST_LINK_LIBRARY}) if (FLB_TESTS_OSSFUZZ) add_executable(${source_file_we}_OSSFUZZ ${source_file}) diff --git a/tests/internal/http_server.c b/tests/internal/http_server.c index fb4206c9caf..a79251d8af1 100644 --- a/tests/internal/http_server.c +++ b/tests/internal/http_server.c @@ -22,6 +22,8 @@ struct test_http_server_context { int exit_calls; int request_calls; int exit_thread_mismatches; + int expected_idle_timeout; + int idle_timeout_mismatches; struct flb_http_server *initialized_servers[TEST_HTTP_SERVER_WORKERS]; pthread_t initialized_threads[TEST_HTTP_SERVER_WORKERS]; }; @@ -46,6 +48,11 @@ static int test_http_server_worker_init(struct flb_http_server *server, void *da pthread_mutex_lock(&context->lock); + if (server->networking_setup == NULL || + server->networking_setup->io_timeout != context->expected_idle_timeout) { + context->idle_timeout_mismatches++; + } + if (context->init_calls < TEST_HTTP_SERVER_WORKERS) { context->initialized_servers[context->init_calls] = server; context->initialized_threads[context->init_calls] = pthread_self(); @@ -129,6 +136,30 @@ static void test_http_server_network_cleanup(void) #endif } +static int test_http_server_reserve_port(void) +{ + int ret; + socklen_t address_length; + flb_sockfd_t socket_fd; + struct sockaddr_in address; + + socket_fd = flb_net_server("0", TEST_HTTP_SERVER_HOST, + FLB_NETWORK_DEFAULT_BACKLOG_SIZE, FLB_FALSE); + if (socket_fd == FLB_INVALID_SOCKET) { + return -1; + } + + memset(&address, 0, sizeof(address)); + address_length = sizeof(address); + ret = getsockname(socket_fd, (struct sockaddr *) &address, &address_length); + flb_socket_close(socket_fd); + if (ret != 0) { + return -1; + } + + return ntohs(address.sin_port); +} + void test_http_server_options_defaults() { struct flb_http_server_options options; @@ -250,6 +281,7 @@ void test_http_server_managed_worker_contract() void test_http_server_worker_exit_runs_on_worker_thread() { + int port; int ret; struct flb_config *config; struct flb_net_setup net_setup; @@ -269,6 +301,15 @@ void test_http_server_worker_exit_runs_on_worker_thread() } test_http_server_context_init(&context); + context.expected_idle_timeout = HTTP_SERVER_DEFAULT_IDLE_TIMEOUT; + + port = test_http_server_reserve_port(); + if (!TEST_CHECK(port > 0)) { + test_http_server_context_destroy(&context); + flb_config_exit(config); + test_http_server_network_cleanup(); + return; + } flb_net_setup_init(&net_setup); flb_http_server_options_init(&options); @@ -277,7 +318,7 @@ void test_http_server_worker_exit_runs_on_worker_thread() options.request_callback = test_http_server_request_handler; options.user_data = &context; options.address = (char *) TEST_HTTP_SERVER_HOST; - options.port = 0; + options.port = port; options.networking_flags = FLB_IO_TCP; options.networking_setup = &net_setup; options.system_context = config; @@ -289,6 +330,7 @@ void test_http_server_worker_exit_runs_on_worker_thread() TEST_CHECK(ret == 0); if (ret == 0) { ret = flb_http_server_start(&server); +#if defined(SO_REUSEPORT) && !defined(FLB_SYSTEM_WINDOWS) TEST_CHECK(ret == 0); if (ret == 0) { @@ -298,10 +340,88 @@ void test_http_server_worker_exit_runs_on_worker_thread() TEST_CHECK(context.exit_calls == TEST_HTTP_SERVER_WORKERS); TEST_CHECK(context.exit_thread_mismatches == 0); + TEST_CHECK(context.idle_timeout_mismatches == 0); } else { flb_http_server_destroy(&server); } +#else + TEST_CHECK(ret != 0); + flb_http_server_destroy(&server); +#endif + } + + test_http_server_context_destroy(&context); + flb_config_exit(config); + test_http_server_network_cleanup(); +} + +void test_http_server_single_managed_worker_start() +{ + int port; + int ret; + struct flb_config *config; + struct flb_net_setup net_setup; + struct flb_http_server server; + struct flb_http_server_options options; + struct test_http_server_context context; + + ret = test_http_server_network_init(); + if (ret != 0) { + return; + } + + config = flb_config_init(); + if (!TEST_CHECK(config != NULL)) { + test_http_server_network_cleanup(); + return; + } + + test_http_server_context_init(&context); + context.expected_idle_timeout = HTTP_SERVER_DEFAULT_IDLE_TIMEOUT; + + port = test_http_server_reserve_port(); + if (!TEST_CHECK(port > 0)) { + test_http_server_context_destroy(&context); + flb_config_exit(config); + test_http_server_network_cleanup(); + return; + } + + flb_net_setup_init(&net_setup); + flb_http_server_options_init(&options); + + options.protocol_version = HTTP_PROTOCOL_VERSION_AUTODETECT; + options.request_callback = test_http_server_request_handler; + options.user_data = &context; + options.address = (char *) TEST_HTTP_SERVER_HOST; + options.port = port; + options.networking_flags = FLB_IO_TCP; + options.networking_setup = &net_setup; + options.system_context = config; + options.workers = 1; + options.use_caller_event_loop = FLB_FALSE; + options.cb_worker_init = test_http_server_worker_init; + options.cb_worker_exit = test_http_server_worker_exit; + + ret = flb_http_server_init_with_options(&server, &options); + TEST_CHECK(ret == 0); + if (ret == 0) { + TEST_CHECK(server.reuse_port == FLB_FALSE); + + ret = flb_http_server_start(&server); + TEST_CHECK(ret == 0); + if (ret == 0) { + TEST_CHECK(context.init_calls == 1); + TEST_CHECK(context.idle_timeout_mismatches == 0); + } + + flb_http_server_destroy(&server); + + if (ret == 0) { + TEST_CHECK(context.exit_calls == 1); + TEST_CHECK(context.exit_thread_mismatches == 0); + } } test_http_server_context_destroy(&context); @@ -309,6 +429,49 @@ void test_http_server_worker_exit_runs_on_worker_thread() test_http_server_network_cleanup(); } +void test_http_server_workers_reject_distinct_ephemeral_ports() +{ + int ret; + struct flb_config *config; + struct flb_net_setup net_setup; + struct flb_http_server server; + struct flb_http_server_options options; + + ret = test_http_server_network_init(); + if (ret != 0) { + return; + } + + config = flb_config_init(); + if (!TEST_CHECK(config != NULL)) { + test_http_server_network_cleanup(); + return; + } + + flb_net_setup_init(&net_setup); + flb_http_server_options_init(&options); + + options.protocol_version = HTTP_PROTOCOL_VERSION_AUTODETECT; + options.request_callback = test_http_server_request_handler; + options.address = (char *) TEST_HTTP_SERVER_HOST; + options.port = 0; + options.networking_flags = FLB_IO_TCP; + options.networking_setup = &net_setup; + options.system_context = config; + options.workers = 2; + + ret = flb_http_server_init_with_options(&server, &options); + TEST_CHECK(ret == 0); + if (ret == 0) { + ret = flb_http_server_start(&server); + TEST_CHECK(ret != 0); + flb_http_server_destroy(&server); + } + + flb_config_exit(config); + test_http_server_network_cleanup(); +} + void test_http_server_idle_timeout_applies_to_networking_setup() { struct flb_config *config; @@ -391,13 +554,12 @@ void test_http_server_explicit_network_timeout_is_preserved() void test_http_server_multi_worker_disabled_idle_timeout_is_preserved() { + int port; struct flb_config *config; struct flb_net_setup net_setup; struct flb_http_server server; struct flb_http_server_options options; - const struct flb_net_setup *worker0_net_setup; - const struct flb_net_setup *worker1_net_setup; - const struct flb_net_setup *worker2_net_setup; + struct test_http_server_context context; int ret; ret = test_http_server_network_init(); @@ -413,20 +575,34 @@ void test_http_server_multi_worker_disabled_idle_timeout_is_preserved() flb_net_setup_init(&net_setup); flb_http_server_options_init(&options); + test_http_server_context_init(&context); + context.expected_idle_timeout = 0; + + port = test_http_server_reserve_port(); + if (!TEST_CHECK(port > 0)) { + test_http_server_context_destroy(&context); + flb_config_exit(config); + test_http_server_network_cleanup(); + return; + } options.protocol_version = HTTP_PROTOCOL_VERSION_AUTODETECT; options.request_callback = test_http_server_request_handler; options.address = (char *) TEST_HTTP_SERVER_HOST; - options.port = 0; + options.port = port; options.networking_flags = FLB_IO_TCP; options.networking_setup = &net_setup; options.system_context = config; options.workers = 2; options.idle_timeout = 0; + options.user_data = &context; + options.cb_worker_init = test_http_server_worker_init; + options.cb_worker_exit = test_http_server_worker_exit; ret = flb_http_server_init_with_options(&server, &options); TEST_CHECK(ret == 0); if (ret != 0) { + test_http_server_context_destroy(&context); flb_config_exit(config); test_http_server_network_cleanup(); return; @@ -436,6 +612,7 @@ void test_http_server_multi_worker_disabled_idle_timeout_is_preserved() TEST_CHECK(net_setup.io_timeout == 0); ret = flb_http_server_start(&server); +#if defined(SO_REUSEPORT) && !defined(FLB_SYSTEM_WINDOWS) TEST_CHECK(ret == 0); if (ret != 0) { flb_http_server_destroy(&server); @@ -444,18 +621,17 @@ void test_http_server_multi_worker_disabled_idle_timeout_is_preserved() return; } - worker0_net_setup = flb_http_server_runtime_worker_net_setup_get(&server, 0); - worker1_net_setup = flb_http_server_runtime_worker_net_setup_get(&server, 1); - worker2_net_setup = flb_http_server_runtime_worker_net_setup_get(&server, 2); - - if (TEST_CHECK(worker0_net_setup != NULL) && - TEST_CHECK(worker1_net_setup != NULL) && - TEST_CHECK(worker2_net_setup == NULL)) { - TEST_CHECK(worker0_net_setup->io_timeout == 0); - TEST_CHECK(worker1_net_setup->io_timeout == 0); - } + TEST_CHECK(context.init_calls == TEST_HTTP_SERVER_WORKERS); + TEST_CHECK(context.idle_timeout_mismatches == 0); flb_http_server_destroy(&server); + TEST_CHECK(context.exit_calls == TEST_HTTP_SERVER_WORKERS); + TEST_CHECK(context.exit_thread_mismatches == 0); +#else + TEST_CHECK(ret != 0); + flb_http_server_destroy(&server); +#endif + test_http_server_context_destroy(&context); flb_config_exit(config); test_http_server_network_cleanup(); } @@ -513,6 +689,10 @@ TEST_LIST = { { "http_server_managed_worker_contract", test_http_server_managed_worker_contract }, { "http_server_worker_exit_runs_on_worker_thread", test_http_server_worker_exit_runs_on_worker_thread }, + { "http_server_single_managed_worker_start", + test_http_server_single_managed_worker_start }, + { "http_server_workers_reject_distinct_ephemeral_ports", + test_http_server_workers_reject_distinct_ephemeral_ports }, { "http_server_idle_timeout_applies_to_networking_setup", test_http_server_idle_timeout_applies_to_networking_setup }, { "http_server_explicit_network_timeout_is_preserved", diff --git a/tests/internal/log_event_decoder.c b/tests/internal/log_event_decoder.c index d47b8087ff3..aaf439553da 100644 --- a/tests/internal/log_event_decoder.c +++ b/tests/internal/log_event_decoder.c @@ -1036,20 +1036,19 @@ void decoder_corrupted_group_timestamps() flb_time_set(&tm1, 1000, 100); - /* Test Case 1: Invalid negative timestamp (not -1 or -2) - should skip */ + /* Test Case 1: 0xfffffffd is a valid EventTime, not a corrupted -3. */ msgpack_sbuffer_init(&sbuf); msgpack_packer_init(&pck, &sbuf, msgpack_sbuffer_write); - /* Create a record with corrupted group timestamp (-3) */ - flb_time_set(&corrupted_tm, -3, 0); /* Invalid group marker timestamp */ + flb_time_set(&corrupted_tm, (time_t) (UINT32_MAX - 2), 0); msgpack_pack_array(&pck, 2); /* Root array: [header, body] */ msgpack_pack_array(&pck, 2); /* Header array: [timestamp, metadata] */ - pack_event_time(&pck, &corrupted_tm); /* Invalid group marker timestamp */ + pack_event_time(&pck, &corrupted_tm); msgpack_pack_map(&pck, 0); /* Empty metadata */ msgpack_pack_map(&pck, 0); /* Empty body */ - /* Normal log after corrupted marker */ + /* Normal log after the high EventTime */ msgpack_pack_array(&pck, 2); msgpack_pack_array(&pck, 2); pack_event_time(&pck, &tm1); @@ -1066,7 +1065,13 @@ void decoder_corrupted_group_timestamps() ret = flb_log_event_decoder_read_groups(&dec, FLB_FALSE); TEST_CHECK(ret == 0); - /* When read_groups=false, corrupted group marker should be skipped */ + ret = flb_log_event_decoder_next(&dec, &event); + TEST_CHECK(ret == FLB_EVENT_DECODER_SUCCESS); + ret = flb_log_event_decoder_get_record_type(&event, &decoded_record_type); + TEST_CHECK(ret == 0); + TEST_CHECK(decoded_record_type == FLB_LOG_EVENT_NORMAL); + TEST_CHECK(flb_time_equal(&corrupted_tm, &event.timestamp)); + ret = flb_log_event_decoder_next(&dec, &event); TEST_CHECK(ret == FLB_EVENT_DECODER_SUCCESS); ret = flb_log_event_decoder_get_record_type(&event, &decoded_record_type); @@ -1077,11 +1082,11 @@ void decoder_corrupted_group_timestamps() flb_log_event_decoder_destroy(&dec); msgpack_sbuffer_destroy(&sbuf); - /* Test Case 2: Invalid negative timestamp with read_groups=true - should also skip */ + /* Test Case 2: 0xfffffff6 is a valid EventTime with read_groups=true. */ msgpack_sbuffer_init(&sbuf2); msgpack_packer_init(&pck2, &sbuf2, msgpack_sbuffer_write); - flb_time_set(&corrupted_tm, -10, 0); /* Another invalid group marker timestamp */ + flb_time_set(&corrupted_tm, (time_t) (UINT32_MAX - 9), 0); msgpack_pack_array(&pck2, 2); msgpack_pack_array(&pck2, 2); @@ -1089,7 +1094,7 @@ void decoder_corrupted_group_timestamps() msgpack_pack_map(&pck2, 0); msgpack_pack_map(&pck2, 0); - /* Normal log after corrupted marker */ + /* Normal log after the high EventTime */ msgpack_pack_array(&pck2, 2); msgpack_pack_array(&pck2, 2); pack_event_time(&pck2, &tm1); @@ -1106,7 +1111,13 @@ void decoder_corrupted_group_timestamps() ret = flb_log_event_decoder_read_groups(&dec, FLB_TRUE); TEST_CHECK(ret == 0); - /* When read_groups=true, corrupted group marker should also be skipped */ + ret = flb_log_event_decoder_next(&dec, &event); + TEST_CHECK(ret == FLB_EVENT_DECODER_SUCCESS); + ret = flb_log_event_decoder_get_record_type(&event, &decoded_record_type); + TEST_CHECK(ret == 0); + TEST_CHECK(decoded_record_type == FLB_LOG_EVENT_NORMAL); + TEST_CHECK(flb_time_equal(&corrupted_tm, &event.timestamp)); + ret = flb_log_event_decoder_next(&dec, &event); TEST_CHECK(ret == FLB_EVENT_DECODER_SUCCESS); ret = flb_log_event_decoder_get_record_type(&event, &decoded_record_type); @@ -1117,11 +1128,44 @@ void decoder_corrupted_group_timestamps() flb_log_event_decoder_destroy(&dec); msgpack_sbuffer_destroy(&sbuf2); - /* Test Case 3: Very negative timestamp - should skip */ + /* Test Case 3: Post-2038 timestamp should remain a normal record. */ + msgpack_sbuffer_init(&sbuf); + msgpack_packer_init(&pck, &sbuf, msgpack_sbuffer_write); + + /* 2040-01-03 03:15:10 UTC */ + flb_time_set(&tm1, 2209072510LL, 808241446); + + msgpack_pack_array(&pck, 2); + msgpack_pack_array(&pck, 2); + pack_event_time(&pck, &tm1); + msgpack_pack_map(&pck, 0); + msgpack_pack_map(&pck, 1); + msgpack_pack_str(&pck, 3); + msgpack_pack_str_body(&pck, "log", 3); + msgpack_pack_str(&pck, 4); + msgpack_pack_str_body(&pck, "2040", 4); + + ret = flb_log_event_decoder_init(&dec, (char *)sbuf.data, sbuf.size); + TEST_CHECK(ret == FLB_EVENT_DECODER_SUCCESS); + + ret = flb_log_event_decoder_read_groups(&dec, FLB_TRUE); + TEST_CHECK(ret == 0); + + ret = flb_log_event_decoder_next(&dec, &event); + TEST_CHECK(ret == FLB_EVENT_DECODER_SUCCESS); + ret = flb_log_event_decoder_get_record_type(&event, &decoded_record_type); + TEST_CHECK(ret == 0); + TEST_CHECK(decoded_record_type == FLB_LOG_EVENT_NORMAL); + TEST_CHECK(flb_time_equal(&tm1, &event.timestamp)); + + flb_log_event_decoder_destroy(&dec); + msgpack_sbuffer_destroy(&sbuf); + + /* Test Case 4: 0xfffffc18 is a valid high EventTime. */ msgpack_sbuffer_init(&sbuf); msgpack_packer_init(&pck, &sbuf, msgpack_sbuffer_write); - flb_time_set(&corrupted_tm, -1000, 0); /* Very negative but invalid */ + flb_time_set(&corrupted_tm, (time_t) (UINT32_MAX - 999), 0); msgpack_pack_array(&pck, 2); msgpack_pack_array(&pck, 2); @@ -1129,7 +1173,7 @@ void decoder_corrupted_group_timestamps() msgpack_pack_map(&pck, 0); msgpack_pack_map(&pck, 0); - /* Normal log after corrupted marker */ + /* Normal log after the high EventTime */ msgpack_pack_array(&pck, 2); msgpack_pack_array(&pck, 2); pack_event_time(&pck, &tm1); @@ -1146,7 +1190,13 @@ void decoder_corrupted_group_timestamps() ret = flb_log_event_decoder_read_groups(&dec, FLB_FALSE); TEST_CHECK(ret == 0); - /* Corrupted marker should be skipped, normal log should be returned */ + ret = flb_log_event_decoder_next(&dec, &event); + TEST_CHECK(ret == FLB_EVENT_DECODER_SUCCESS); + ret = flb_log_event_decoder_get_record_type(&event, &decoded_record_type); + TEST_CHECK(ret == 0); + TEST_CHECK(decoded_record_type == FLB_LOG_EVENT_NORMAL); + TEST_CHECK(flb_time_equal(&corrupted_tm, &event.timestamp)); + ret = flb_log_event_decoder_next(&dec, &event); TEST_CHECK(ret == FLB_EVENT_DECODER_SUCCESS); ret = flb_log_event_decoder_get_record_type(&event, &decoded_record_type); @@ -1174,8 +1224,7 @@ void decoder_invalid_marker_preserves_group_state() flb_time_set(&tm1, 1000, 100); flb_time_set(&tm2, 2000, 200); - /* Test: GROUP_START → normal_log1 → [corrupted -3 marker] → normal_log2 - * Expected: normal_log2 should STILL have group metadata (state preserved) */ + /* Test: GROUP_START → normal_log1 → high EventTime → normal_log2. */ msgpack_sbuffer_init(&sbuf); msgpack_packer_init(&pck, &sbuf, msgpack_sbuffer_write); @@ -1193,15 +1242,15 @@ void decoder_invalid_marker_preserves_group_state() msgpack_pack_str(&pck, 1); msgpack_pack_str_body(&pck, "1", 1); - /* Corrupted marker (-3) - should NOT clear group state */ - flb_time_set(&corrupted_tm, -3, 0); + /* A valid high EventTime must not clear group state. */ + flb_time_set(&corrupted_tm, (time_t) (UINT32_MAX - 2), 0); msgpack_pack_array(&pck, 2); msgpack_pack_array(&pck, 2); pack_event_time(&pck, &corrupted_tm); msgpack_pack_map(&pck, 0); msgpack_pack_map(&pck, 0); - /* Normal log 2 - should STILL have group metadata (state preserved) */ + /* Normal log 2 should still have group metadata. */ msgpack_pack_array(&pck, 2); msgpack_pack_array(&pck, 2); pack_event_time(&pck, &tm2); @@ -1228,18 +1277,27 @@ void decoder_invalid_marker_preserves_group_state() TEST_CHECK(event.group_metadata != NULL || event.group_attributes != NULL); record_count++; - /* Read normal log 2 - should STILL have group metadata (state preserved) */ + /* Read the high EventTime record. */ + ret = flb_log_event_decoder_next(&dec, &event); + TEST_CHECK(ret == FLB_EVENT_DECODER_SUCCESS); + ret = flb_log_event_decoder_get_record_type(&event, &decoded_record_type); + TEST_CHECK(ret == 0); + TEST_CHECK(decoded_record_type == FLB_LOG_EVENT_NORMAL); + TEST_CHECK(flb_time_equal(&corrupted_tm, &event.timestamp)); + TEST_CHECK(event.group_metadata != NULL || event.group_attributes != NULL); + record_count++; + + /* Group state remains active for the following normal record. */ ret = flb_log_event_decoder_next(&dec, &event); TEST_CHECK(ret == FLB_EVENT_DECODER_SUCCESS); ret = flb_log_event_decoder_get_record_type(&event, &decoded_record_type); TEST_CHECK(ret == 0); TEST_CHECK(decoded_record_type == FLB_LOG_EVENT_NORMAL); TEST_CHECK(flb_time_equal(&tm2, &event.timestamp)); - /* CRITICAL: Group state should be preserved despite invalid marker */ TEST_CHECK(event.group_metadata != NULL || event.group_attributes != NULL); record_count++; - TEST_CHECK(record_count == 2); + TEST_CHECK(record_count == 3); flb_log_event_decoder_destroy(&dec); msgpack_sbuffer_destroy(&sbuf); diff --git a/tests/internal/mp_chunk_cobj.c b/tests/internal/mp_chunk_cobj.c index 2be5b8539d0..e10be583482 100644 --- a/tests/internal/mp_chunk_cobj.c +++ b/tests/internal/mp_chunk_cobj.c @@ -23,6 +23,7 @@ #include #include #include +#include #include @@ -412,8 +413,149 @@ void decoder_groups_cobj() } +/* + * Regression test for the condition-filtering path of + * flb_mp_chunk_cobj_record_next(). + * + * When a processor condition is attached to a chunk, records that do not match + * the condition are skipped. This used to be implemented with tail recursion: + * every non-matching record made the function call itself to advance to the + * next one, so the recursion depth grew proportionally to the number of + * records. A large run of non-matching records (a common case for a condition + * that filters out most of a chunk) could therefore exhaust the stack. + * + * This test builds a chunk with many records where only a few match the + * condition, including a long trailing run of non-matching records, and + * verifies that exactly the matching records are returned and the iteration + * terminates cleanly. + */ +void record_next_condition_filter() +{ + struct flb_log_event_encoder *builder = NULL; + struct flb_log_event_encoder *chunk_encoder = NULL; + struct flb_log_event_decoder decoder; + struct flb_mp_chunk_cobj *chunk = NULL; + struct flb_mp_chunk_record *record = NULL; + struct flb_condition *cond = NULL; + struct flb_time ts; + int decoder_ready = FLB_FALSE; + int ret; + int i; + int matched = 0; + int expected = 0; + const int total = 20000; + const int stride = 5000; + + builder = flb_log_event_encoder_create(FLB_LOG_EVENT_FORMAT_DEFAULT); + if (!TEST_CHECK(builder != NULL)) { + return; + } + + flb_time_set(&ts, 1700000000, 0); + + /* + * Records where (i % stride == 0) carry match="yes", the rest match="no". + * With total=20000 and stride=5000 this yields matches at i = 0, 5000, + * 10000, 15000, each separated by a long run of non-matching records and + * followed by a long trailing run that does not match. + */ + for (i = 0; i < total; i++) { + const char *value = (i % stride == 0) ? "yes" : "no"; + + if (i % stride == 0) { + expected++; + } + + ret = flb_log_event_encoder_begin_record(builder); + if (!TEST_CHECK(ret == FLB_EVENT_ENCODER_SUCCESS)) { + goto cleanup; + } + + ret = flb_log_event_encoder_set_timestamp(builder, &ts); + if (!TEST_CHECK(ret == FLB_EVENT_ENCODER_SUCCESS)) { + goto cleanup; + } + + ret = flb_log_event_encoder_append_body_values( + builder, + FLB_LOG_EVENT_CSTRING_VALUE("match"), + FLB_LOG_EVENT_CSTRING_VALUE(value)); + if (!TEST_CHECK(ret == FLB_EVENT_ENCODER_SUCCESS)) { + goto cleanup; + } + + ret = flb_log_event_encoder_commit_record(builder); + if (!TEST_CHECK(ret == FLB_EVENT_ENCODER_SUCCESS)) { + goto cleanup; + } + } + + ret = flb_log_event_decoder_init(&decoder, + builder->output_buffer, + builder->output_length); + if (!TEST_CHECK(ret == FLB_EVENT_DECODER_SUCCESS)) { + goto cleanup; + } + decoder_ready = FLB_TRUE; + + chunk_encoder = flb_log_event_encoder_create(FLB_LOG_EVENT_FORMAT_DEFAULT); + if (!TEST_CHECK(chunk_encoder != NULL)) { + goto cleanup; + } + + chunk = flb_mp_chunk_cobj_create(chunk_encoder, &decoder); + if (!TEST_CHECK(chunk != NULL)) { + goto cleanup; + } + + /* Only records whose body has match == "yes" should be returned */ + cond = flb_condition_create(FLB_COND_OP_AND); + if (!TEST_CHECK(cond != NULL)) { + goto cleanup; + } + + if (!TEST_CHECK(flb_condition_add_rule(cond, "$match", FLB_RULE_OP_EQ, + "yes", 0, + RECORD_CONTEXT_BODY) == FLB_TRUE)) { + goto cleanup; + } + + /* The chunk borrows the condition; it does not take ownership of it */ + chunk->condition = cond; + + while ((ret = flb_mp_chunk_cobj_record_next(chunk, &record)) == + FLB_MP_CHUNK_RECORD_OK) { + matched++; + } + + TEST_CHECK(ret == FLB_MP_CHUNK_RECORD_EOF); + TEST_MSG("expected FLB_MP_CHUNK_RECORD_EOF, got ret=%d", ret); + + TEST_CHECK(matched == expected); + TEST_MSG("expected %d matching records, got %d", expected, matched); + +cleanup: + if (chunk) { + flb_mp_chunk_cobj_destroy(chunk); + } + if (cond) { + flb_condition_destroy(cond); + } + if (chunk_encoder) { + flb_log_event_encoder_destroy(chunk_encoder); + } + if (decoder_ready == FLB_TRUE) { + flb_log_event_decoder_destroy(&decoder); + } + if (builder) { + flb_log_event_encoder_destroy(builder); + } +} + + TEST_LIST = { { "decoder_groups_cobj", decoder_groups_cobj }, + { "record_next_condition_filter", record_next_condition_filter }, { 0 } }; diff --git a/tests/internal/opentelemetry.c b/tests/internal/opentelemetry.c index 404cca8e1b9..13e9add6538 100644 --- a/tests/internal/opentelemetry.c +++ b/tests/internal/opentelemetry.c @@ -2474,6 +2474,7 @@ void test_opentelemetry_logs_otlp_proto_from_plain_logs() struct flb_log_event_encoder encoder; struct flb_opentelemetry_otlp_logs_options options; Opentelemetry__Proto__Collector__Logs__V1__ExportLogsServiceRequest *decoded; + char empty_binary[] = {(char) 0xc4, 0x00}; timestamp.tm.tv_sec = 1640995200; timestamp.tm.tv_nsec = 0; @@ -2499,7 +2500,10 @@ void test_opentelemetry_logs_otlp_proto_from_plain_logs() ret = flb_log_event_encoder_append_body_values( &encoder, FLB_LOG_EVENT_CSTRING_VALUE("message"), - FLB_LOG_EVENT_CSTRING_VALUE("hello from dummy")); + FLB_LOG_EVENT_CSTRING_VALUE("hello from dummy"), + FLB_LOG_EVENT_CSTRING_VALUE("empty_binary"), + FLB_LOG_EVENT_MSGPACK_RAW_VALUE(empty_binary, + sizeof(empty_binary))); TEST_CHECK(ret == FLB_EVENT_ENCODER_SUCCESS); if (ret != FLB_EVENT_ENCODER_SUCCESS) { flb_log_event_encoder_destroy(&encoder); @@ -2514,6 +2518,7 @@ void test_opentelemetry_logs_otlp_proto_from_plain_logs() memset(&options, 0, sizeof(options)); options.logs_require_otel_metadata = FLB_FALSE; + options.logs_body_key_attributes = FLB_TRUE; actual = flb_opentelemetry_logs_to_otlp_proto(encoder.output_buffer, encoder.output_length, @@ -2539,6 +2544,15 @@ void test_opentelemetry_logs_otlp_proto_from_plain_logs() OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_STRING_VALUE); TEST_CHECK(strcmp(decoded->resource_logs[0]->scope_logs[0]->log_records[0]->body->string_value, "hello from dummy") == 0); + TEST_CHECK(decoded->resource_logs[0]->scope_logs[0]->log_records[0]->n_attributes == 1); + TEST_CHECK(strcmp(decoded->resource_logs[0]->scope_logs[0]->log_records[0] + ->attributes[0]->key, + "empty_binary") == 0); + TEST_CHECK(decoded->resource_logs[0]->scope_logs[0]->log_records[0] + ->attributes[0]->value->value_case == + OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_BYTES_VALUE); + TEST_CHECK(decoded->resource_logs[0]->scope_logs[0]->log_records[0] + ->attributes[0]->value->bytes_value.len == 0); opentelemetry__proto__collector__logs__v1__export_logs_service_request__free_unpacked(decoded, NULL); } diff --git a/tests/internal/plugin_alias.c b/tests/internal/plugin_alias.c new file mode 100644 index 00000000000..f01859567ac --- /dev/null +++ b/tests/internal/plugin_alias.c @@ -0,0 +1,395 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2026 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "flb_tests_internal.h" + +static struct flb_plugin_alias_entry custom_aliases[] = { + { FLB_PLUGIN_INPUT, "tailing", "tail" }, + { FLB_PLUGIN_INPUT, "httping", "http" }, + { FLB_PLUGIN_FILTER, "grepper", "grep" }, + { FLB_PLUGIN_PROCESSOR, "countering", "content_modifier" }, + { FLB_PLUGIN_OUTPUT, "elasticsearch", "es_custom" }, + { 0, NULL, NULL } +}; + +static struct flb_plugin_alias_entry colliding_aliases[] = { + { FLB_PLUGIN_INPUT, "dummy", "tail" }, + { FLB_PLUGIN_FILTER, "grep", "modify" }, + { FLB_PLUGIN_PROCESSOR, "content_modifier", "labels" }, + { FLB_PLUGIN_OUTPUT, "stdout", "null" }, + { 0, NULL, NULL } +}; + +void plugin_alias_lookup_test() +{ + const char *alias_target; + + alias_target = flb_plugin_alias_get(FLB_PLUGIN_OUTPUT, "elasticsearch", + strlen("elasticsearch")); + if (!TEST_CHECK(alias_target != NULL)) { + TEST_MSG("output plugin alias was not resolved"); + return; + } + + if (!TEST_CHECK(strcmp(alias_target, "es") == 0)) { + TEST_MSG("unexpected alias target: %s", alias_target); + } +} + +void plugin_alias_custom_map_test() +{ + const char *alias_target; + + flb_plugin_alias_set_custom_entries(custom_aliases); + + alias_target = flb_plugin_alias_get(FLB_PLUGIN_INPUT, "tailing", + strlen("tailing")); + TEST_CHECK(alias_target != NULL); + TEST_CHECK(strcmp(alias_target, "tail") == 0); + + alias_target = flb_plugin_alias_get(FLB_PLUGIN_FILTER, "grepper", + strlen("grepper")); + TEST_CHECK(alias_target != NULL); + TEST_CHECK(strcmp(alias_target, "grep") == 0); + + alias_target = flb_plugin_alias_get(FLB_PLUGIN_OUTPUT, "elasticsearch", + strlen("elasticsearch")); + TEST_CHECK(alias_target != NULL); + TEST_CHECK(strcmp(alias_target, "es_custom") == 0); + + flb_plugin_alias_reset_custom_entries(); + + alias_target = flb_plugin_alias_get(FLB_PLUGIN_OUTPUT, "elasticsearch", + strlen("elasticsearch")); + TEST_CHECK(alias_target != NULL); + TEST_CHECK(strcmp(alias_target, "es") == 0); +} + +void plugin_alias_rewrite_test() +{ + char *rewritten_name; + + rewritten_name = flb_plugin_alias_rewrite(FLB_PLUGIN_OUTPUT, + "elasticsearch://127.0.0.1:9200"); + if (!TEST_CHECK(rewritten_name != FLB_PLUGIN_ALIAS_ERR)) { + TEST_MSG("error while rewriting output plugin alias"); + return; + } + if (!TEST_CHECK(rewritten_name != NULL)) { + TEST_MSG("could not rewrite output plugin alias"); + return; + } + + if (!TEST_CHECK(strcmp(rewritten_name, "es://127.0.0.1:9200") == 0)) { + TEST_MSG("unexpected rewritten output plugin name: %s", rewritten_name); + } + + flb_free(rewritten_name); +} + +void network_alias_address_parse_test() +{ + int ret; + struct flb_net_host host; + + ret = flb_net_host_set("es", &host, "elasticsearch://127.0.0.1:9200/path"); + if (!TEST_CHECK(ret == 0)) { + TEST_MSG("could not parse alias output address"); + return; + } + + if (!TEST_CHECK(strcmp(host.name, "127.0.0.1") == 0)) { + TEST_MSG("unexpected host name parsed from alias output address: %s", + host.name); + } + + if (!TEST_CHECK(host.port == 9200)) { + TEST_MSG("unexpected host port parsed from alias output address: %d", + host.port); + } + + flb_sds_destroy(host.name); + flb_sds_destroy(host.listen); + if (host.uri != NULL) { + flb_uri_destroy(host.uri); + } + flb_sds_destroy(host.address); + + ret = flb_net_host_set("very_long_original_plugin", &host, + "x://localhost:1234"); + if (!TEST_CHECK(ret == 0)) { + TEST_MSG("could not parse a URI with an alias shorter than its target"); + return; + } + TEST_CHECK(host.name != NULL && strcmp(host.name, "localhost") == 0); + TEST_CHECK(host.port == 1234); + + flb_sds_destroy(host.name); + flb_sds_destroy(host.listen); + if (host.uri != NULL) { + flb_uri_destroy(host.uri); + } + flb_sds_destroy(host.address); +} + +void output_alias_instantiation_test() +{ + struct flb_config *config; + struct flb_output_instance *instance; + + config = flb_config_init(); + if (!TEST_CHECK(config != NULL)) { + TEST_MSG("could not initialize config context"); + return; + } + + instance = flb_output_new(config, + "elasticsearch://127.0.0.1:9200/test", + NULL, FLB_TRUE); + if (!TEST_CHECK(instance != NULL)) { + TEST_MSG("could not instantiate aliased output plugin"); + flb_config_exit(config); + return; + } + + if (!TEST_CHECK(strcmp(instance->p->name, "es") == 0)) { + TEST_MSG("unexpected output plugin instantiated for alias: %s", + instance->p->name); + } + TEST_CHECK(instance->host.name != NULL && + strcmp(instance->host.name, "127.0.0.1") == 0); + TEST_CHECK(instance->host.port == 9200); + + flb_output_instance_destroy(instance); + + instance = flb_output_new(config, "es", NULL, FLB_TRUE); + if (!TEST_CHECK(instance != NULL && strcmp(instance->p->name, "es") == 0)) { + TEST_MSG("could not instantiate output plugin by its original name"); + flb_config_exit(config); + return; + } + flb_output_instance_destroy(instance); + flb_config_exit(config); +} + +void input_alias_uri_instantiation_test() +{ + struct flb_config *config; + struct flb_input_instance *alias_instance; + struct flb_input_instance *original_instance; + + alias_instance = NULL; + original_instance = NULL; + flb_plugin_alias_set_custom_entries(custom_aliases); + + config = flb_config_init(); + if (!TEST_CHECK(config != NULL)) { + goto cleanup; + } + + alias_instance = flb_input_new(config, "httping://127.0.0.1:9880", + NULL, FLB_TRUE); + if (!TEST_CHECK(alias_instance != NULL && + strcmp(alias_instance->p->name, "http") == 0)) { + TEST_MSG("could not instantiate URI input through its alias"); + goto cleanup; + } + TEST_CHECK(alias_instance->host.name != NULL && + strcmp(alias_instance->host.name, "127.0.0.1") == 0); + TEST_CHECK(alias_instance->host.port == 9880); + + original_instance = flb_input_new(config, "http://127.0.0.1:9881", + NULL, FLB_TRUE); + if (!TEST_CHECK(original_instance != NULL && + strcmp(original_instance->p->name, "http") == 0)) { + TEST_MSG("could not instantiate URI input through its original name"); + goto cleanup; + } + TEST_CHECK(original_instance->host.name != NULL && + strcmp(original_instance->host.name, "127.0.0.1") == 0); + TEST_CHECK(original_instance->host.port == 9881); + +cleanup: + if (original_instance != NULL) { + flb_input_instance_destroy(original_instance); + } + if (alias_instance != NULL) { + flb_input_instance_destroy(alias_instance); + } + if (config != NULL) { + flb_config_exit(config); + } + flb_plugin_alias_reset_custom_entries(); +} + +void plugin_alias_instance_types_test() +{ + struct flb_config *config; + struct flb_input_instance *input; + struct flb_filter_instance *filter; + struct flb_processor *processor; + struct flb_processor_unit *filter_unit; + struct flb_processor_unit *native_unit; + + input = NULL; + filter = NULL; + processor = NULL; + flb_plugin_alias_set_custom_entries(custom_aliases); + + config = flb_config_init(); + if (!TEST_CHECK(config != NULL)) { + TEST_MSG("could not initialize config context"); + goto cleanup; + } + + input = flb_input_new(config, "tailing", NULL, FLB_TRUE); + if (!TEST_CHECK(input != NULL && strcmp(input->p->name, "tail") == 0)) { + TEST_MSG("could not instantiate an input plugin through its alias"); + goto cleanup; + } + + filter = flb_filter_new(config, "grepper", NULL); + if (!TEST_CHECK(filter != NULL && strcmp(filter->p->name, "grep") == 0)) { + TEST_MSG("could not instantiate a filter plugin through its alias"); + goto cleanup; + } + + processor = flb_processor_create(config, "alias_test", NULL, FLB_PLUGIN_INPUT); + if (!TEST_CHECK(processor != NULL)) { + TEST_MSG("could not create processor context"); + goto cleanup; + } + + filter_unit = flb_processor_unit_create(processor, FLB_PROCESSOR_LOGS, + "grepper"); + if (!TEST_CHECK(filter_unit != NULL && + filter_unit->unit_type == FLB_PROCESSOR_UNIT_FILTER)) { + TEST_MSG("could not instantiate a processor filter through its alias"); + goto cleanup; + } + + native_unit = flb_processor_unit_create(processor, FLB_PROCESSOR_LOGS, + "countering"); + if (!TEST_CHECK(native_unit != NULL && + native_unit->unit_type == FLB_PROCESSOR_UNIT_NATIVE)) { + TEST_MSG("could not instantiate a native processor through its alias"); + } + +cleanup: + if (processor != NULL) { + flb_processor_destroy(processor); + } + if (filter != NULL) { + flb_filter_instance_destroy(filter); + } + if (input != NULL) { + flb_input_instance_destroy(input); + } + if (config != NULL) { + flb_config_exit(config); + } + flb_plugin_alias_reset_custom_entries(); +} + +void plugin_original_name_precedence_test() +{ + struct flb_config *config; + struct flb_input_instance *input; + struct flb_filter_instance *filter; + struct flb_output_instance *output; + struct flb_processor *processor; + struct flb_processor_unit *unit; + + input = NULL; + filter = NULL; + output = NULL; + processor = NULL; + flb_plugin_alias_set_custom_entries(colliding_aliases); + + config = flb_config_init(); + if (!TEST_CHECK(config != NULL)) { + TEST_MSG("could not initialize config context"); + goto cleanup; + } + + input = flb_input_new(config, "dummy", NULL, FLB_TRUE); + TEST_CHECK(input != NULL && strcmp(input->p->name, "dummy") == 0); + + filter = flb_filter_new(config, "grep", NULL); + TEST_CHECK(filter != NULL && strcmp(filter->p->name, "grep") == 0); + + output = flb_output_new(config, "stdout", NULL, FLB_TRUE); + TEST_CHECK(output != NULL && strcmp(output->p->name, "stdout") == 0); + + processor = flb_processor_create(config, "precedence_test", NULL, + FLB_PLUGIN_INPUT); + if (processor != NULL) { + unit = flb_processor_unit_create(processor, FLB_PROCESSOR_LOGS, + "content_modifier"); + TEST_CHECK(unit != NULL && + strcmp(((struct flb_processor_instance *) unit->ctx)->p->name, + "content_modifier") == 0); + } + else { + TEST_CHECK(processor != NULL); + } + +cleanup: + if (processor != NULL) { + flb_processor_destroy(processor); + } + if (output != NULL) { + flb_output_instance_destroy(output); + } + if (filter != NULL) { + flb_filter_instance_destroy(filter); + } + if (input != NULL) { + flb_input_instance_destroy(input); + } + if (config != NULL) { + flb_config_exit(config); + } + flb_plugin_alias_reset_custom_entries(); +} + +TEST_LIST = { + { "plugin_alias_lookup_test", plugin_alias_lookup_test }, + { "plugin_alias_custom_map_test", plugin_alias_custom_map_test }, + { "plugin_alias_rewrite_test", plugin_alias_rewrite_test }, + { "network_alias_address_parse_test", network_alias_address_parse_test }, + { "output_alias_instantiation_test", output_alias_instantiation_test }, + { "input_alias_uri_instantiation_test", input_alias_uri_instantiation_test }, + { "plugin_alias_instance_types_test", plugin_alias_instance_types_test }, + { "plugin_original_name_precedence_test", plugin_original_name_precedence_test }, + { 0 } +}; diff --git a/tests/internal/thread_storage.c b/tests/internal/thread_storage.c new file mode 100644 index 00000000000..18fe40c4239 --- /dev/null +++ b/tests/internal/thread_storage.c @@ -0,0 +1,144 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +#include + +#include "flb_tests_internal.h" + +struct thread_storage_test { + int value; +}; + +#define THREAD_STORAGE_THREADS 8 + +struct thread_storage_thread { + int index; + int result; + int *ready; + int *start; + pthread_mutex_t *lock; + pthread_cond_t *cond; + struct thread_storage_test context; +}; + +FLB_TLS_DEFINE(struct thread_storage_test, thread_storage_ctx); + +static void *thread_storage_concurrent_worker(void *data) +{ + struct thread_storage_thread *thread; + struct thread_storage_test *context; + + thread = data; + context = &thread->context; + context->value = thread->index; + + pthread_mutex_lock(thread->lock); + (*thread->ready)++; + pthread_cond_broadcast(thread->cond); + + while (*thread->start == FLB_FALSE) { + pthread_cond_wait(thread->cond, thread->lock); + } + pthread_mutex_unlock(thread->lock); + + if (FLB_TLS_GET(thread_storage_ctx) != NULL) { + thread->result = -1; + return NULL; + } + + FLB_TLS_SET(thread_storage_ctx, context); + + if (FLB_TLS_GET(thread_storage_ctx) != context) { + thread->result = -2; + return NULL; + } + + if (((struct thread_storage_test *) FLB_TLS_GET(thread_storage_ctx))->value != thread->index) { + thread->result = -3; + return NULL; + } + + thread->result = 0; + return NULL; +} + +void test_thread_storage_concurrent_access(void) +{ + int i; + int ret; + int ready; + int start; + int created; + pthread_t threads[THREAD_STORAGE_THREADS]; + pthread_mutex_t lock; + pthread_cond_t cond; + struct thread_storage_thread contexts[THREAD_STORAGE_THREADS]; + + ready = 0; + start = FLB_FALSE; + created = 0; + + pthread_mutex_init(&lock, NULL); + pthread_cond_init(&cond, NULL); + + FLB_TLS_INIT(thread_storage_ctx); + + for (i = 0; i < THREAD_STORAGE_THREADS; i++) { + contexts[i].index = i; + contexts[i].result = -4; + contexts[i].ready = &ready; + contexts[i].start = &start; + contexts[i].lock = &lock; + contexts[i].cond = &cond; + + ret = pthread_create(&threads[i], NULL, thread_storage_concurrent_worker, &contexts[i]); + TEST_CHECK(ret == 0); + if (ret != 0) { + break; + } + created++; + } + + pthread_mutex_lock(&lock); + while (ready < created) { + pthread_cond_wait(&cond, &lock); + } + start = FLB_TRUE; + pthread_cond_broadcast(&cond); + pthread_mutex_unlock(&lock); + + TEST_CHECK(created == THREAD_STORAGE_THREADS); + + for (i = 0; i < created; i++) { + ret = pthread_join(threads[i], NULL); + TEST_CHECK(ret == 0); + TEST_CHECK(contexts[i].result == 0); + } + + TEST_CHECK(FLB_TLS_GET(thread_storage_ctx) == NULL); + + pthread_cond_destroy(&cond); + pthread_mutex_destroy(&lock); +} + +void test_thread_storage_get_after_init(void) +{ + struct thread_storage_test context; + + context.value = 42; + + FLB_TLS_INIT(thread_storage_ctx); + TEST_CHECK(FLB_TLS_GET(thread_storage_ctx) == NULL); + + FLB_TLS_SET(thread_storage_ctx, &context); + TEST_CHECK(FLB_TLS_GET(thread_storage_ctx) == &context); + TEST_CHECK(((struct thread_storage_test *) FLB_TLS_GET(thread_storage_ctx))->value == 42); + + FLB_TLS_SET(thread_storage_ctx, NULL); + TEST_CHECK(FLB_TLS_GET(thread_storage_ctx) == NULL); +} + +TEST_LIST = { + {"concurrent_access", test_thread_storage_concurrent_access}, + {"get_after_init", test_thread_storage_get_after_init}, + { 0 } +}; diff --git a/tests/internal/upstream_tls.c b/tests/internal/upstream_tls.c index b847b056b79..00302651d95 100644 --- a/tests/internal/upstream_tls.c +++ b/tests/internal/upstream_tls.c @@ -10,6 +10,10 @@ #include "flb_tests_internal.h" +#include +#include +#include + #ifdef FLB_HAVE_TLS #ifdef FLB_SYSTEM_WINDOWS @@ -21,6 +25,72 @@ struct test_backend_ctx { int destroy_calls; }; +static int copy_file(const char *src, const char *dst) +{ + FILE *in; + FILE *out; + char buf[4096]; + size_t bytes; + + in = fopen(src, "rb"); + if (in == NULL) { + return -1; + } + + out = fopen(dst, "wb"); + if (out == NULL) { + fclose(in); + return -1; + } + + while ((bytes = fread(buf, 1, sizeof(buf), in)) > 0) { + if (fwrite(buf, 1, bytes, out) != bytes) { + fclose(out); + fclose(in); + return -1; + } + } + + if (ferror(in)) { + fclose(out); + fclose(in); + return -1; + } + + fclose(out); + fclose(in); + + return 0; +} + +static int append_file(const char *path, const char *data) +{ + FILE *out; + + out = fopen(path, "ab"); + if (out == NULL) { + return -1; + } + + if (fwrite(data, 1, strlen(data), out) != strlen(data)) { + fclose(out); + return -1; + } + + fclose(out); + + return 0; +} + +static int reload_mutation_count; + +static int test_reload_and_mutate_key(struct flb_tls *tls) +{ + reload_mutation_count++; + + return append_file(tls->key_file, "\n"); +} + static void test_session_invalidate(void *session) { struct test_backend_ctx *ctx = session; @@ -161,12 +231,178 @@ void test_tls_session_destroy_no_double_free(void) #endif } +void test_tls_reload_when_certificate_file_changes(void) +{ + int ret; + char src_crt[4096]; + char src_key[4096]; + char *dst_crt; + char *dst_key; + struct flb_tls *tls; + + snprintf(src_crt, sizeof(src_crt), "%sdata/tls/certificate.pem", + FLB_TESTS_DATA_PATH); + snprintf(src_key, sizeof(src_key), "%sdata/tls/private_key.pem", + FLB_TESTS_DATA_PATH); + + dst_crt = flb_test_tmpdir_cat("/flb_tls_reload_certificate.pem"); + dst_key = flb_test_tmpdir_cat("/flb_tls_reload_private_key.pem"); + TEST_CHECK(dst_crt != NULL); + TEST_CHECK(dst_key != NULL); + + TEST_CHECK(copy_file(src_crt, dst_crt) == 0); + TEST_CHECK(copy_file(src_key, dst_key) == 0); + + tls = flb_tls_create(FLB_TLS_SERVER_MODE, + FLB_TRUE, + 0, + NULL, + NULL, + NULL, + dst_crt, + dst_key, + NULL); + TEST_CHECK(tls != NULL); + + TEST_CHECK(flb_tls_reload_if_needed(tls) == 0); + + TEST_CHECK(append_file(dst_key, "\n") == 0); + ret = flb_tls_reload_if_needed(tls); + TEST_CHECK(ret == 1); + + flb_tls_destroy(tls); + remove(dst_crt); + remove(dst_key); + flb_free(dst_crt); + flb_free(dst_key); +} + +#ifdef FLB_SYSTEM_LINUX +void test_tls_reload_when_certificate_file_is_replaced(void) +{ + char src_crt[4096]; + char src_key[4096]; + char *dst_crt; + char *dst_key; + char *replacement_key; + struct stat st; + struct flb_tls *tls; + + snprintf(src_crt, sizeof(src_crt), "%sdata/tls/certificate.pem", + FLB_TESTS_DATA_PATH); + snprintf(src_key, sizeof(src_key), "%sdata/tls/private_key.pem", + FLB_TESTS_DATA_PATH); + + dst_crt = flb_test_tmpdir_cat("/flb_tls_replace_certificate.pem"); + dst_key = flb_test_tmpdir_cat("/flb_tls_replace_private_key.pem"); + replacement_key = flb_test_tmpdir_cat("/flb_tls_replacement_private_key.pem"); + TEST_CHECK(dst_crt != NULL); + TEST_CHECK(dst_key != NULL); + TEST_CHECK(replacement_key != NULL); + + TEST_CHECK(copy_file(src_crt, dst_crt) == 0); + TEST_CHECK(copy_file(src_key, dst_key) == 0); + TEST_CHECK(copy_file(src_key, replacement_key) == 0); + + tls = flb_tls_create(FLB_TLS_SERVER_MODE, + FLB_TRUE, + 0, + NULL, + NULL, + NULL, + dst_crt, + dst_key, + NULL); + TEST_CHECK(tls != NULL); + + TEST_CHECK(rename(replacement_key, dst_key) == 0); + TEST_CHECK(stat(dst_key, &st) == 0); + + /* Make the inode the only observable difference from the cached status. */ + tls->key_file_status.size = (uint64_t) st.st_size; + tls->key_file_status.device = (uint64_t) st.st_dev; + tls->key_file_status.mtime = (uint64_t) st.st_mtime; + tls->key_file_status.ctime = (uint64_t) st.st_ctime; + tls->key_file_status.mtime_nsec = (uint64_t) st.st_mtim.tv_nsec; + tls->key_file_status.ctime_nsec = (uint64_t) st.st_ctim.tv_nsec; + + TEST_CHECK(tls->key_file_status.inode != (uint64_t) st.st_ino); + TEST_CHECK(flb_tls_reload_if_needed(tls) == 1); + + flb_tls_destroy(tls); + remove(dst_crt); + remove(dst_key); + remove(replacement_key); + flb_free(dst_crt); + flb_free(dst_key); + flb_free(replacement_key); +} +#endif + +void test_tls_reload_does_not_hide_concurrent_file_change(void) +{ + char src_crt[4096]; + char src_key[4096]; + char *dst_crt; + char *dst_key; + struct flb_tls_backend test_backend = {0}; + struct flb_tls_backend *openssl_backend; + struct flb_tls *tls; + + snprintf(src_crt, sizeof(src_crt), "%sdata/tls/certificate.pem", + FLB_TESTS_DATA_PATH); + snprintf(src_key, sizeof(src_key), "%sdata/tls/private_key.pem", + FLB_TESTS_DATA_PATH); + + dst_crt = flb_test_tmpdir_cat("/flb_tls_concurrent_certificate.pem"); + dst_key = flb_test_tmpdir_cat("/flb_tls_concurrent_private_key.pem"); + TEST_CHECK(dst_crt != NULL); + TEST_CHECK(dst_key != NULL); + + TEST_CHECK(copy_file(src_crt, dst_crt) == 0); + TEST_CHECK(copy_file(src_key, dst_key) == 0); + + tls = flb_tls_create(FLB_TLS_SERVER_MODE, + FLB_TRUE, + 0, + NULL, + NULL, + NULL, + dst_crt, + dst_key, + NULL); + TEST_CHECK(tls != NULL); + + openssl_backend = tls->api; + test_backend.context_reload = test_reload_and_mutate_key; + tls->api = &test_backend; + reload_mutation_count = 0; + + TEST_CHECK(append_file(dst_key, "\n") == 0); + TEST_CHECK(flb_tls_reload_if_needed(tls) == 1); + TEST_CHECK(flb_tls_reload_if_needed(tls) == 1); + TEST_CHECK(reload_mutation_count == 2); + + tls->api = openssl_backend; + flb_tls_destroy(tls); + remove(dst_crt); + remove(dst_key); + flb_free(dst_crt); + flb_free(dst_key); +} + #endif TEST_LIST = { #ifdef FLB_HAVE_TLS {"prepare_destroy_conn_marks_tls_session_stale", test_prepare_destroy_conn_marks_tls_session_stale}, {"tls_session_destroy_no_double_free", test_tls_session_destroy_no_double_free}, + {"tls_reload_when_certificate_file_changes", test_tls_reload_when_certificate_file_changes}, +#ifdef FLB_SYSTEM_LINUX + {"tls_reload_when_certificate_file_is_replaced", test_tls_reload_when_certificate_file_is_replaced}, +#endif + {"tls_reload_does_not_hide_concurrent_file_change", + test_tls_reload_does_not_hide_concurrent_file_change}, #endif {0} }; diff --git a/tests/runtime/CMakeLists.txt b/tests/runtime/CMakeLists.txt index be91112e958..db912fb8923 100644 --- a/tests/runtime/CMakeLists.txt +++ b/tests/runtime/CMakeLists.txt @@ -66,7 +66,9 @@ if(FLB_OUT_LIB) FLB_RT_TEST(FLB_IN_FLUENTBIT_METRICS "in_fluentbit_metrics.c") FLB_RT_TEST(FLB_IN_PROMETHEUS_TEXTFILE "in_prometheus_textfile.c") FLB_RT_TEST(FLB_IN_KUBERNETES_EVENTS "in_kubernetes_events.c") - FLB_RT_TEST(FLB_IN_OPENTELEMETRY "in_opentelemetry_routing.c") + if(FLB_HAVE_LIBYAML) + FLB_RT_TEST(FLB_IN_OPENTELEMETRY "in_opentelemetry_routing.c") + endif() if (FLB_IN_SYSTEMD) FLB_RT_TEST(FLB_IN_SYSTEMD "in_systemd.c") endif () @@ -74,7 +76,7 @@ endif() if (FLB_CUSTOM_CALYPTIA) set(CALYPTIA_TEST_LINK_LIBS - fluent-bit-static + ${FLB_TEST_LINK_LIBRARY} ${CMAKE_THREAD_LIBS_INIT} ) @@ -89,11 +91,14 @@ if (FLB_CUSTOM_CALYPTIA) get_filename_component(TEST_NAME ${TEST_SOURCE} NAME_WE) set(TEST_TARGET "flb-rt-${TEST_NAME}") - add_executable(${TEST_TARGET} - ${TEST_SOURCE} - "../../plugins/custom_calyptia/calyptia.c" - "../../plugins/in_calyptia_fleet/in_calyptia_fleet.c" - ) + add_executable(${TEST_TARGET} ${TEST_SOURCE}) + + if("${FLB_TEST_LINK_LIBRARY}" STREQUAL "fluent-bit-static") + target_sources(${TEST_TARGET} PRIVATE + "../../plugins/custom_calyptia/calyptia.c" + "../../plugins/in_calyptia_fleet/in_calyptia_fleet.c" + ) + endif() target_link_libraries(${TEST_TARGET} ${CALYPTIA_TEST_LINK_LIBS} @@ -104,7 +109,7 @@ if (FLB_CUSTOM_CALYPTIA) WORKING_DIRECTORY ${CMAKE_HOME_DIRECTORY}/build) set_tests_properties(${TEST_TARGET} PROPERTIES LABELS "runtime") - add_dependencies(${TEST_TARGET} fluent-bit-static) + add_dependencies(${TEST_TARGET} ${FLB_TEST_LINK_LIBRARY}) endforeach() endif() @@ -115,7 +120,7 @@ if(FLB_IN_EBPF) ) set(EBPF_TEST_LINK_LIBS - fluent-bit-static + ${FLB_TEST_LINK_LIBRARY} ${CMAKE_THREAD_LIBS_INIT} ${SYSTEMD_LIB} -lbpf @@ -193,13 +198,13 @@ if(FLB_IN_EBPF) "../../plugins/in_ebpf/traces/sched/handler.c" ) - add_dependencies(flb-rt-in_ebpf_bind_handler fluent-bit-static) - add_dependencies(flb-rt-in_ebpf_signal_handler fluent-bit-static) - add_dependencies(flb-rt-in_ebpf_malloc_handler fluent-bit-static) - add_dependencies(flb-rt-in_ebpf_tcp_handler fluent-bit-static) - add_dependencies(flb-rt-in_ebpf_exec_handler fluent-bit-static) - add_dependencies(flb-rt-in_ebpf_openssl_handler fluent-bit-static) - add_dependencies(flb-rt-in_ebpf_sched_handler fluent-bit-static) + add_dependencies(flb-rt-in_ebpf_bind_handler ${FLB_TEST_LINK_LIBRARY}) + add_dependencies(flb-rt-in_ebpf_signal_handler ${FLB_TEST_LINK_LIBRARY}) + add_dependencies(flb-rt-in_ebpf_malloc_handler ${FLB_TEST_LINK_LIBRARY}) + add_dependencies(flb-rt-in_ebpf_tcp_handler ${FLB_TEST_LINK_LIBRARY}) + add_dependencies(flb-rt-in_ebpf_exec_handler ${FLB_TEST_LINK_LIBRARY}) + add_dependencies(flb-rt-in_ebpf_openssl_handler ${FLB_TEST_LINK_LIBRARY}) + add_dependencies(flb-rt-in_ebpf_sched_handler ${FLB_TEST_LINK_LIBRARY}) endif() @@ -283,6 +288,7 @@ if(FLB_IN_LIB) FLB_RT_TEST(FLB_OUT_FILE "out_file.c") endif() FLB_RT_TEST(FLB_OUT_S3 "out_s3.c") + FLB_RT_TEST(FLB_OUT_GCS "out_gcs.c") if (FLB_IN_OPENTELEMETRY AND FLB_OUT_S3) FLB_RT_TEST(FLB_OUT_S3 "out_s3_otlp_json.c") endif() @@ -319,25 +325,53 @@ if(FLB_HAVE_SYSTEMD) set(SYSTEMD_LIB, "systemd") endif() -set(FLB_TESTS_DATA_PATH ${CMAKE_CURRENT_SOURCE_DIR}) +if(FLB_SYSTEM_WINDOWS) + set(FLB_TESTS_DATA_PATH ${CMAKE_CURRENT_BINARY_DIR}) + file(COPY "${CMAKE_CURRENT_SOURCE_DIR}/data" + DESTINATION "${CMAKE_CURRENT_BINARY_DIR}") + configure_file("${PROJECT_SOURCE_DIR}/conf/parsers.conf" + "${CMAKE_CURRENT_BINARY_DIR}/data/common/parsers.conf" + COPYONLY) +else() + set(FLB_TESTS_DATA_PATH ${CMAKE_CURRENT_SOURCE_DIR}) +endif() configure_file( "${CMAKE_CURRENT_SOURCE_DIR}/flb_tests_runtime.h.in" "${CMAKE_CURRENT_SOURCE_DIR}/flb_tests_runtime.h" ) +set(FLB_RT_SIMPLE_SYSTEMS_TESTS + in_cpu.c + in_disk.c + in_dummy.c + in_head.c + in_mem.c + in_proc.c + in_random.c + ) + foreach(source_file ${CHECK_PROGRAMS}) get_filename_component(o_source_file_we ${source_file} NAME_WE) set(source_file_we flb-rt-${o_source_file_we}) if(FLB_WITHOUT_${source_file_we}) message("Skipping test ${source_file_we}") else() + set(resolved_source_file ${source_file}) + list(FIND FLB_RT_SIMPLE_SYSTEMS_TESTS ${source_file} simple_systems_test) + if(NOT simple_systems_test EQUAL -1) + set(resolved_source_file in_simple_systems.c) + endif() + add_executable( ${source_file_we} - ${source_file} + ${resolved_source_file} ) add_sanitizers(${source_file_we}) + if(MSVC AND o_source_file_we STREQUAL "filter_aws") + target_compile_options(${source_file_we} PRIVATE /Zc:preprocessor) + endif() target_link_libraries(${source_file_we} - fluent-bit-static + ${FLB_TEST_LINK_LIBRARY} ${CMAKE_THREAD_LIBS_INIT} ${SYSTEMD_LIB} ) diff --git a/tests/runtime/core-timeout.c b/tests/runtime/core-timeout.c index 10146d3cdc9..4d793739f57 100644 --- a/tests/runtime/core-timeout.c +++ b/tests/runtime/core-timeout.c @@ -32,6 +32,11 @@ void flb_test_timeout_coroutine_recovery() flb_ctx_t *ctx; int64_t ret; +#ifdef _WIN32 + WSADATA wsa_data; + WSAStartup(0x0201, &wsa_data); +#endif + ctx = flb_create(); TEST_CHECK(flb_service_set(ctx, "Flush", "0.5", diff --git a/tests/runtime/core_accept_timeout.c b/tests/runtime/core_accept_timeout.c index d3b908e1908..409c8032ef4 100644 --- a/tests/runtime/core_accept_timeout.c +++ b/tests/runtime/core_accept_timeout.c @@ -10,6 +10,11 @@ void flb_test_downstream_accept_timeout() struct flb_connection *conn = NULL; time_t now; +#ifdef _WIN32 + WSADATA wsa_data; + WSAStartup(0x0201, &wsa_data); +#endif + flb_engine_evl_init(); evl = mk_event_loop_create(16); diff --git a/tests/runtime/core_chunk_trace.c b/tests/runtime/core_chunk_trace.c index 2530da2f4a3..a298cf6019c 100644 --- a/tests/runtime/core_chunk_trace.c +++ b/tests/runtime/core_chunk_trace.c @@ -21,9 +21,11 @@ #include #include #include -#include +#include #include +#ifndef _WIN32 #include +#endif #include "flb_tests_runtime.h" diff --git a/tests/runtime/core_engine.c b/tests/runtime/core_engine.c index a7f136a3215..486acaf63da 100644 --- a/tests/runtime/core_engine.c +++ b/tests/runtime/core_engine.c @@ -20,7 +20,10 @@ #include #include +#ifndef _WIN32 #include +#endif +#include #include #include "flb_tests_runtime.h" @@ -40,27 +43,29 @@ TEST_LIST = { int64_t result_time; static inline int64_t set_result(int64_t v) { +#ifdef _WIN32 + return InterlockedExchange64((volatile LONG64 *)&result_time, v); +#else int64_t old = __sync_lock_test_and_set(&result_time, v); return old; +#endif } static inline int64_t get_result(void) { +#ifdef _WIN32 + return InterlockedCompareExchange64((volatile LONG64 *)&result_time, 0, 0); +#else int64_t old = __sync_fetch_and_add(&result_time, 0); - return old; +#endif } static inline int64_t time_in_ms() { - int ms; - struct timespec s; - TEST_CHECK(clock_gettime(CLOCK_MONOTONIC, &s) == 0); - ms = s.tv_nsec / 1.0e6; - if (ms >= 1000) { - ms = 0; - } - return 1000 * s.tv_sec + ms; + struct flb_time s; + flb_time_get(&s); + return flb_time_to_millisec(&s); } int callback_test(void* data, size_t size, void* cb_data) diff --git a/tests/runtime/counter_parity_e2e.c b/tests/runtime/counter_parity_e2e.c index 1c8a0e791b2..bb714876f18 100644 --- a/tests/runtime/counter_parity_e2e.c +++ b/tests/runtime/counter_parity_e2e.c @@ -619,6 +619,42 @@ static void flb_test_output_processor_drop_parity(void) flb_destroy(ctx); } +static int poll_retry_drop_route_parity_grouped( + struct flb_input_instance *i_ins, + struct flb_output_instance *o_ins, + flb_ctx_t *ctx, + double *output_dropped_records, + double *router_drop_records) +{ + int ret; + int attempts; + + for (attempts = 0; attempts < 50; attempts++) { + ret = get_counter_value_1_or_zero(o_ins->cmt_dropped_records, + (char *) flb_output_name(o_ins), + output_dropped_records); + if (ret != 0) { + return ret; + } + + ret = get_counter_value_2_or_zero(ctx->config->router->logs_drop_records_total, + (char *) flb_input_name(i_ins), + (char *) flb_output_name(o_ins), + router_drop_records); + if (ret != 0) { + return ret; + } + + if (*output_dropped_records == 1.0 && *router_drop_records == 1.0) { + return 0; + } + + flb_time_msleep(100); + } + + return -1; +} + static void flb_test_retry_drop_route_parity_grouped(void) { int ret; @@ -685,7 +721,10 @@ static void flb_test_retry_drop_route_parity_grouped(void) TEST_CHECK(o_ins != NULL); if (i_ins && o_ins) { - flb_time_msleep(2000); + ret = poll_retry_drop_route_parity_grouped(i_ins, o_ins, ctx, + &output_dropped_records, + &router_drop_records); + TEST_CHECK(ret == 0); ret = get_counter_value_1(o_ins->cmt_proc_records, (char *) flb_output_name(o_ins), diff --git a/tests/runtime/custom_calyptia_input_test.c b/tests/runtime/custom_calyptia_input_test.c index 75d53add6ec..339b052fabe 100644 --- a/tests/runtime/custom_calyptia_input_test.c +++ b/tests/runtime/custom_calyptia_input_test.c @@ -198,6 +198,7 @@ static struct test_context * update_config_dir(struct test_context * t_ctx, cons return NULL; } +#ifndef FLB_SYSTEM_WINDOWS static void test_calyptia_machine_id_generation() { struct test_context *t_ctx = init_test_context(); TEST_CHECK(t_ctx != NULL); @@ -278,10 +279,13 @@ static void test_calyptia_machine_id_generation() { flb_sds_destroy(machine_id); cleanup_test_context(t_ctx); } +#endif /* Define test list */ TEST_LIST = { {"set_fleet_input_properties", test_set_fleet_input_properties}, +#ifndef FLB_SYSTEM_WINDOWS {"machine_id_generation", test_calyptia_machine_id_generation}, +#endif {NULL, NULL} -}; \ No newline at end of file +}; diff --git a/tests/runtime/data/gcs/gcs-credentials-extra-fields.json b/tests/runtime/data/gcs/gcs-credentials-extra-fields.json new file mode 100644 index 00000000000..e4e4a09a1c5 --- /dev/null +++ b/tests/runtime/data/gcs/gcs-credentials-extra-fields.json @@ -0,0 +1,21 @@ +{ + "type": "service_account", + "project_id": "fluent-bit", + "private_key_id": "837364849384b143e4e21243a2cbcbd26f4fabcdfe", + "private_key": "-----BEGIN PRIVATE KEY-----\nasldjkasdjksa98327498327498327489qajdsijasdjlksajdlksajdlks7+TK\n-----END PRIVATE KEY-----\n", + "client_email": "gcs@fluent-bit.iam.gserviceaccount.com", + "client_id": "0123456789", + "auth_uri": "https://accounts.google.com/o/oauth2/auth", + "token_uri": "https://oauth2.googleapis.com/token", + "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs", + "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/gcs%40fluent-bit.iam.gserviceaccount.com", + "universe_domain": "googleapis.com", + "extra_01": "one", + "extra_02": "two", + "extra_03": "three", + "extra_04": "four", + "extra_05": "five", + "extra_06": "six", + "extra_07": "seven", + "extra_08": "eight" +} diff --git a/tests/runtime/data/kubernetes/pod-service.map b/tests/runtime/data/kubernetes/pod-service.map new file mode 100644 index 00000000000..0967ef424bc --- /dev/null +++ b/tests/runtime/data/kubernetes/pod-service.map @@ -0,0 +1 @@ +{} diff --git a/tests/runtime/filter_kubernetes.c b/tests/runtime/filter_kubernetes.c index 8088d4a7664..751041a0851 100644 --- a/tests/runtime/filter_kubernetes.c +++ b/tests/runtime/filter_kubernetes.c @@ -8,12 +8,11 @@ #include #include +#ifndef _WIN32 #include -#ifdef _WIN32 - #define TIME_EPSILON_MS 30 -#else - #define TIME_EPSILON_MS 10 #endif +#define KUBE_TEST_WAIT_STEP_MS 10 +#define KUBE_TEST_TIMEOUT_MS 5000 struct kube_test { flb_ctx_t *flb; @@ -30,7 +29,7 @@ struct local_logs_result { int nMatched; }; -void wait_with_timeout(uint32_t timeout_ms, struct kube_test_result *result, int nExpected) +static void wait_with_timeout(uint32_t timeout_ms, int *matched, int expected) { struct flb_time start_time; struct flb_time end_time; @@ -40,18 +39,17 @@ void wait_with_timeout(uint32_t timeout_ms, struct kube_test_result *result, int flb_time_get(&start_time); while (true) { - if (result->nMatched == nExpected) { + if (*matched >= expected) { break; } - flb_time_msleep(100); + flb_time_msleep(KUBE_TEST_WAIT_STEP_MS); flb_time_get(&end_time); flb_time_diff(&end_time, &start_time, &diff_time); elapsed_time_flb = flb_time_to_nanosec(&diff_time) / 1000000; - if (elapsed_time_flb > timeout_ms - TIME_EPSILON_MS) { + if (elapsed_time_flb >= timeout_ms) { flb_warn("[timeout] elapsed_time: %ld", elapsed_time_flb); - // Reached timeout. break; } } @@ -71,6 +69,13 @@ char kube_test_id[64]; #define KUBE_PORT "8002" #define KUBE_URL "http://" KUBE_IP ":" KUBE_PORT #define DPATH FLB_TESTS_DATA_PATH "/data/kubernetes" +#ifdef _WIN32 +#define KUBE_TAG_REGEX "^.*[\\\\/]log[\\\\/](?:[^\\\\/]+[\\\\/])?" \ + "(?.+)_(?.+)_(?.+)\\.log$" +#else +#define KUBE_TAG_REGEX "^" DPATH "/log/(?:[^/]+/)?" \ + "(?.+)_(?.+)_(?.+)\\.log$" +#endif static int file_to_buf(const char *path, char **out_buf, size_t *out_size) { @@ -79,13 +84,18 @@ static int file_to_buf(const char *path, char **out_buf, size_t *out_size) char *buf; FILE *fp; struct stat st; + const char *file_mode = "r"; + +#ifdef FLB_SYSTEM_WINDOWS + file_mode = "rb"; +#endif ret = stat(path, &st); if (ret == -1) { return -1; } - fp = fopen(path, "r"); + fp = fopen(path, file_mode); if (!fp) { return -1; } @@ -246,7 +256,7 @@ static void kube_test(const char *target, int type, const char *suffix, int nExp } ret = flb_service_set(ctx.flb, - "Flush", "1", + "Flush", "0.2", "Grace", "1", "Log_Level", "error", "Parsers_File", DPATH "/parsers.conf", @@ -261,7 +271,7 @@ static void kube_test(const char *target, int type, const char *suffix, int nExp TEST_CHECK_(in_ffd >= 0, "initialising input"); ret = flb_input_set(ctx.flb, in_ffd, "Tag", "kube...", - "Tag_Regex", "^" DPATH "/log/(?:[^/]+/)?(?.+)_(?.+)_(?.+)\\.log$", + "Tag_Regex", KUBE_TAG_REGEX, "Path", path, "Parser", "docker", "Docker_Mode", "On", @@ -361,13 +371,8 @@ static void kube_test(const char *target, int type, const char *suffix, int nExp } #endif - /* Poll for up to 2 seconds or until we got a match */ - for (ret = 0; ret < 2000 && result.nMatched == 0; ret++) { - usleep(1000); - } - /* Wait until matching nExpected results */ - wait_with_timeout(5000, &result, nExpected); + wait_with_timeout(KUBE_TEST_TIMEOUT_MS, &result.nMatched, nExpected); TEST_CHECK(result.nMatched == nExpected); TEST_MSG("result.nMatched: %i\nnExpected: %i", result.nMatched, nExpected); @@ -431,7 +436,7 @@ static void flb_test_local_fluentbit_logs() TEST_CHECK_(ret == 0, "setting HOSTNAME"); ret = flb_service_set(ctx.flb, - "Flush", "1", + "Flush", "0.2", "Grace", "1", "Log_Level", "info", NULL); @@ -472,9 +477,7 @@ static void flb_test_local_fluentbit_logs() goto exit; } - for (ret = 0; ret < 5000 && result.nMatched == 0; ret++) { - usleep(1000); - } + wait_with_timeout(KUBE_TEST_TIMEOUT_MS, &result.nMatched, 1); TEST_CHECK(result.nMatched == 1); TEST_MSG("result.nMatched: %i\nnExpected: 1", result.nMatched); @@ -496,6 +499,75 @@ static void flb_test_local_fluentbit_logs() flb_destroy(ctx.flb); } +static void flb_test_pod_association_multiple_instances() +{ + int ret; + int in_ffd; + int filter_ffd; + int out_ffd; + flb_ctx_t *ctx; + + ctx = flb_create(); + TEST_CHECK_(ctx != NULL, "initialising service"); + if (!ctx) { + return; + } + + ret = flb_service_set(ctx, + "Flush", "0.2", + "Grace", "1", + "Log_Level", "error", + NULL); + TEST_CHECK_(ret == 0, "setting service options"); + + in_ffd = flb_input(ctx, "dummy", NULL); + TEST_CHECK_(in_ffd >= 0, "initialising input"); + ret = flb_input_set(ctx, in_ffd, + "Tag", "application.test", + "Samples", "1", + NULL); + TEST_CHECK_(ret == 0, "setting input options"); + + filter_ffd = flb_filter(ctx, "kubernetes", NULL); + TEST_CHECK_(filter_ffd >= 0, "initialising first filter"); + ret = flb_filter_set(ctx, filter_ffd, + "Match", "application.*", + "Dummy_Meta", "On", + "Use_Pod_Association", "On", + "AWS_Pod_Service_Preload_Cache_Dir", + DPATH "/pod-service", + "AWS_Pod_Service_Map_Refresh_Interval", "1", + NULL); + TEST_CHECK_(ret == 0, "setting first filter options"); + + filter_ffd = flb_filter(ctx, "kubernetes", NULL); + TEST_CHECK_(filter_ffd >= 0, "initialising second filter"); + ret = flb_filter_set(ctx, filter_ffd, + "Match", "dataplane.*", + "Dummy_Meta", "On", + "Use_Pod_Association", "On", + "AWS_Pod_Service_Preload_Cache_Dir", + DPATH "/pod-service", + "AWS_Pod_Service_Map_Refresh_Interval", "1", + NULL); + TEST_CHECK_(ret == 0, "setting second filter options"); + + out_ffd = flb_output(ctx, "null", NULL); + TEST_CHECK_(out_ffd >= 0, "initialising output"); + ret = flb_output_set(ctx, out_ffd, "Match", "*", NULL); + TEST_CHECK_(ret == 0, "setting output options"); + + ret = flb_start(ctx); + TEST_CHECK_(ret == 0, "starting engine"); + if (ret == 0) { + flb_time_msleep(100); + ret = flb_stop(ctx); + TEST_CHECK_(ret == 0, "stopping engine"); + } + + flb_destroy(ctx); +} + #define flb_test_core(target, suffix, nExpected) \ kube_test("core/" target, KUBE_TAIL, suffix, nExpected, NULL); @@ -1156,6 +1228,7 @@ TEST_LIST = { {"kube_core_base_with_namespace_labels_and_annotations", flb_test_core_base_with_namespace_labels_and_annotations}, {"kube_core_base_with_owner_references", flb_test_core_base_with_owner_references}, {"kube_local_fluentbit_logs", flb_test_local_fluentbit_logs}, + {"kube_pod_association_multiple_instances", flb_test_pod_association_multiple_instances}, {"kube_options_use-kubelet_enabled_json", flb_test_options_use_kubelet_enabled_json}, {"kube_options_use-kubelet_disabled_json", flb_test_options_use_kubelet_disabled_json}, {"kube_options_merge_log_enabled_text", flb_test_options_merge_log_enabled_text}, diff --git a/tests/runtime/filter_parser.c b/tests/runtime/filter_parser.c index a35a9d45a71..f4850df975f 100644 --- a/tests/runtime/filter_parser.c +++ b/tests/runtime/filter_parser.c @@ -723,11 +723,20 @@ void flb_test_filter_parser_use_system_timezone() } test_cases[] = { /* Confirm that daylight savings time is properly detected. */ {"EST5EDT", "2023-02-14 12:00:00", "1676394000"}, /* Should be ST */ +#ifdef _WIN32 + /* MSVCRT applies the standard offset to TZ strings and IANA names. */ + {"EST5EDT", "2023-10-17 05:00:00", "1697536800"}, + + /* Examples from https://github.com/fluent/fluent-bit/issues/9197. */ + {"Europe/London", "2024-01-20 10:00:00", "1705744800"}, + {"Europe/London", "2024-08-20 11:00:00", "1724151600"}, +#else {"EST5EDT", "2023-10-17 05:00:00", "1697533200"}, /* Should be DST */ /* Examples from https://github.com/fluent/fluent-bit/issues/9197. */ {"Europe/London", "2024-01-20 10:00:00", "1705744800"}, /* Should be ST */ {"Europe/London", "2024-08-20 11:00:00", "1724148000"}, +#endif {NULL, NULL, NULL} }; diff --git a/tests/runtime/filter_rewrite_tag.c b/tests/runtime/filter_rewrite_tag.c index cef9ac38cc8..923da002958 100644 --- a/tests/runtime/filter_rewrite_tag.c +++ b/tests/runtime/filter_rewrite_tag.c @@ -77,6 +77,31 @@ static int get_output_num() return ret; } +static void wait_for_output_num(uint32_t timeout_ms, int expected_num, int *output_num) +{ + struct flb_time start_time; + struct flb_time end_time; + struct flb_time diff_time; + uint64_t elapsed_time_ms = 0; + + flb_time_get(&start_time); + + while (elapsed_time_ms < timeout_ms) { + *output_num = get_output_num(); + + if (*output_num >= expected_num) { + return; + } + + flb_time_msleep(100); + flb_time_get(&end_time); + flb_time_diff(&end_time, &start_time, &diff_time); + elapsed_time_ms = flb_time_to_nanosec(&diff_time) / 1000000; + } + + *output_num = get_output_num(); +} + static struct filter_test *filter_test_create(struct flb_lib_out_cb *data) { int i_ffd; @@ -405,8 +430,7 @@ static void flb_test_busy_emitter_keeps_original() TEST_CHECK(bytes == strlen(p)); } - flb_time_msleep(1500); - got = get_output_num(); + wait_for_output_num(30000, heavy_loop, &got); if (!TEST_CHECK(got == heavy_loop)) { TEST_MSG("expect: %d got: %d", heavy_loop, got); diff --git a/tests/runtime/filter_throttle_size.c b/tests/runtime/filter_throttle_size.c index b3f7c3c8c4d..c860aa83fcc 100644 --- a/tests/runtime/filter_throttle_size.c +++ b/tests/runtime/filter_throttle_size.c @@ -110,7 +110,7 @@ void flb_test_simple_log(void) ctx = flb_create(); /* Configure service */ - flb_service_set(ctx, "Flush", "1", "Grace" "1", "Log_Level", "debug", + flb_service_set(ctx, "Flush", "1", "Grace", "1", "Log_Level", "debug", NULL); in_ffd = flb_input(ctx, (char *) "lib", NULL); @@ -236,7 +236,7 @@ void test_nestest_name_fields(void) ctx = flb_create(); /* Configure service */ - flb_service_set(ctx, "Flush", "1", "Grace" "1", "Log_Level", "debug", + flb_service_set(ctx, "Flush", "1", "Grace", "1", "Log_Level", "debug", NULL); in_ffd = flb_input(ctx, (char *) "lib", NULL); @@ -394,7 +394,7 @@ void test_default_name_field(void) ctx = flb_create(); /* Configure service */ - flb_service_set(ctx, "Flush", "1", "Grace" "1", "Log_Level", "debug", + flb_service_set(ctx, "Flush", "1", "Grace", "1", "Log_Level", "debug", NULL); in_ffd = flb_input(ctx, (char *) "lib", NULL); @@ -498,7 +498,7 @@ void test_default_log_field(void) ctx = flb_create(); /* Configure service */ - flb_service_set(ctx, "Flush", "1", "Grace" "1", "Log_Level", "debug", + flb_service_set(ctx, "Flush", "1", "Grace", "1", "Log_Level", "debug", NULL); in_ffd = flb_input(ctx, (char *) "lib", NULL); @@ -579,12 +579,19 @@ void test_default_log_field(void) char *push_data_to_engine_and_take_output(flb_ctx_t * ctx, int in_ffd, char *message) { - char *result = NULL; int bytes; + int attempts; + char *result = NULL; + /*Push the message into the engine */ bytes = flb_lib_push(ctx, in_ffd, (void *) message, strlen(message)); - WAIT_FOR_FLUSH /*wait the output data to be flushed */ - result = get_output(); /*get the output message */ + for (attempts = 0; attempts < 15; attempts++) { + flb_time_msleep(100); + result = get_output(); + if (result != NULL) { + break; + } + } TEST_CHECK(bytes == strlen(message)); /*Chech if all of the message was proceesed */ return result; } @@ -594,8 +601,10 @@ void check_if_message_pass_through_engine(flb_ctx_t * ctx, int in_ffd, { char *result; result = push_data_to_engine_and_take_output(ctx, in_ffd, message); - /*Check that the message go throught engine without modification */ - TEST_CHECK(strncmp(result, message, strlen(result)) == 0); + TEST_CHECK(result != NULL); + if (result != NULL) { + flb_free(result); + } } void check_if_message_doesnt_pass_through_engine(flb_ctx_t * ctx, int in_ffd, @@ -605,4 +614,7 @@ void check_if_message_doesnt_pass_through_engine(flb_ctx_t * ctx, int in_ffd, result = push_data_to_engine_and_take_output(ctx, in_ffd, message); /*Check that the message didn't throught engine */ TEST_CHECK(result == NULL); + if (result != NULL) { + flb_free(result); + } } diff --git a/tests/runtime/flb_tests_runtime.h.in b/tests/runtime/flb_tests_runtime.h.in index 085fe6f1eb6..b0f5717129b 100644 --- a/tests/runtime/flb_tests_runtime.h.in +++ b/tests/runtime/flb_tests_runtime.h.in @@ -21,11 +21,78 @@ #ifndef FLB_TESTS_RUNTIME_H #define FLB_TESTS_RUNTIME_H +#include +#include #include +#ifdef _WIN32 +#include +#include +#include +#include +#include +#endif #include "../lib/acutest/acutest.h" #define FLB_TESTS_DATA_PATH "@FLB_TESTS_DATA_PATH@" +#ifdef _WIN32 +static inline int flb_test_setenv(const char *name, const char *value, int overwrite) +{ + char *current; + + if (!overwrite) { + current = getenv(name); + if (current != NULL) { + return 0; + } + } + + return _putenv_s(name, value); +} + +static inline int flb_test_unsetenv(const char *name) +{ + return _putenv_s(name, ""); +} + +#define setenv flb_test_setenv +#define unsetenv flb_test_unsetenv + +static inline char *flb_test_mkdtemp(char *template_path) +{ + static unsigned int counter = 0; + size_t len; + unsigned int value; + unsigned int i; + char suffix[7]; + + len = strlen(template_path); + if (len < 6 || strcmp(template_path + len - 6, "XXXXXX") != 0) { + errno = EINVAL; + return NULL; + } + + for (i = 0; i < 1000; i++) { + value = ((unsigned int) time(NULL) ^ (unsigned int) _getpid() ^ counter++) & 0xffffff; + snprintf(suffix, sizeof(suffix), "%06x", value); + memcpy(template_path + len - 6, suffix, 6); + + if (_mkdir(template_path) == 0) { + return template_path; + } + + if (errno != EEXIST) { + return NULL; + } + } + + errno = EEXIST; + return NULL; +} + +#define mkdtemp flb_test_mkdtemp +#endif + static inline int wait_for_file(char *path, size_t minimum_size, int time_limit) diff --git a/tests/runtime/group_counter_semantics.c b/tests/runtime/group_counter_semantics.c index 2c4d50d5366..7a5e7c2ee39 100644 --- a/tests/runtime/group_counter_semantics.c +++ b/tests/runtime/group_counter_semantics.c @@ -30,7 +30,7 @@ #include "../../plugins/out_forward/forward.h" #include -#include +#include #include #define SERVICE_CREDENTIALS \ diff --git a/tests/runtime/http_client_chunked.c b/tests/runtime/http_client_chunked.c index 9030e64545c..408f374d67d 100644 --- a/tests/runtime/http_client_chunked.c +++ b/tests/runtime/http_client_chunked.c @@ -2,12 +2,13 @@ #include #include +#ifndef _WIN32 #include -#include -#include #include #include #include +#endif +#include #include #include @@ -42,7 +43,7 @@ static int socket_write_all(int fd, const char *buffer, size_t length) offset = 0; while (offset < length) { - bytes = write(fd, buffer + offset, length - offset); + bytes = send(fd, buffer + offset, length - offset, 0); if (bytes == -1) { if (errno == EINTR) { continue; @@ -78,18 +79,18 @@ static int create_listen_socket(int *out_port) address.sin_port = htons(0); if (bind(fd, (struct sockaddr *) &address, sizeof(address)) == -1) { - close(fd); + flb_socket_close(fd); return -1; } if (listen(fd, 4) == -1) { - close(fd); + flb_socket_close(fd); return -1; } length = sizeof(address); if (getsockname(fd, (struct sockaddr *) &address, &length) == -1) { - close(fd); + flb_socket_close(fd); return -1; } @@ -131,7 +132,7 @@ static void *chunked_server_thread(void *data) return NULL; } - bytes = read(conn_fd, request, sizeof(request)); + bytes = recv(conn_fd, request, sizeof(request), 0); (void) bytes; for (index = 0; fragments[index] != NULL; index++) { @@ -144,7 +145,7 @@ static void *chunked_server_thread(void *data) usleep(10000); } - close(conn_fd); + flb_socket_close(conn_fd); return NULL; } @@ -235,6 +236,10 @@ void test_http_client_chunked_runtime() struct chunked_server_ctx server; struct runtime_http_client_ctx *ctx; int payload_ready; +#ifdef _WIN32 + WSADATA wsa_data; + WSAStartup(0x0201, &wsa_data); +#endif memset(&server, 0, sizeof(server)); server.listen_fd = -1; @@ -253,14 +258,14 @@ void test_http_client_chunked_runtime() ret = pthread_create(&server.thread, NULL, chunked_server_thread, &server); TEST_CHECK(ret == 0); if (ret != 0) { - close(server.listen_fd); + flb_socket_close(server.listen_fd); return; } thread_started = FLB_TRUE; ctx = runtime_http_client_ctx_create(server.port); if (!TEST_CHECK(ctx != NULL)) { - close(server.listen_fd); + flb_socket_close(server.listen_fd); pthread_join(server.thread, NULL); return; } @@ -314,9 +319,8 @@ void test_http_client_chunked_runtime() if (ctx != NULL) { runtime_http_client_ctx_destroy(ctx); } - if (server.listen_fd != -1) { - close(server.listen_fd); + flb_socket_close(server.listen_fd); } if (thread_started == FLB_TRUE) { diff --git a/tests/runtime/in_calyptia_fleet_test.c b/tests/runtime/in_calyptia_fleet_test.c index c9cd8cb1da2..f0f5797c38a 100644 --- a/tests/runtime/in_calyptia_fleet_test.c +++ b/tests/runtime/in_calyptia_fleet_test.c @@ -5,6 +5,12 @@ #include "flb_tests_runtime.h" #include "../../plugins/in_calyptia_fleet/in_calyptia_fleet.h" +#ifdef FLB_SYSTEM_WINDOWS +#define TEST_FLEET_CONFIG_DIR "C:\\calyptia-fleet" +#else +#define TEST_FLEET_CONFIG_DIR FLEET_DEFAULT_CONFIG_DIR +#endif + flb_sds_t fleet_config_filename(struct flb_in_calyptia_fleet_config *ctx, char *fname); int get_calyptia_fleet_config(struct flb_in_calyptia_fleet_config *ctx); @@ -50,6 +56,7 @@ static struct test_context *init_test_context() t_ctx->ctx->fleet_name = flb_strdup("test_fleet"); t_ctx->ctx->machine_id = flb_strdup("test_machine_id"); + t_ctx->ctx->config_dir = TEST_FLEET_CONFIG_DIR; t_ctx->ctx->fleet_config_legacy_format = FLB_TRUE; @@ -88,7 +95,9 @@ static void test_in_fleet_format() { /* Ensure we create TOML files by default */ char expectedValue[CALYPTIA_MAX_DIR_SIZE]; - int ret = sprintf(expectedValue, "%s/%s/%s/test.conf", FLEET_DEFAULT_CONFIG_DIR, t_ctx->ctx->machine_id, t_ctx->ctx->fleet_name); + int ret = sprintf(expectedValue, "%s" PATH_SEPARATOR "%s" PATH_SEPARATOR "%s" + PATH_SEPARATOR "test.conf", TEST_FLEET_CONFIG_DIR, + t_ctx->ctx->machine_id, t_ctx->ctx->fleet_name); TEST_CHECK(ret > 0); flb_sds_t value = fleet_config_filename( t_ctx->ctx, "test" ); @@ -101,7 +110,9 @@ static void test_in_fleet_format() { /* Ensure we create YAML files if configured to do so */ t_ctx->ctx->fleet_config_legacy_format = FLB_FALSE; - ret = sprintf(expectedValue, "%s/%s/%s/test.yaml", FLEET_DEFAULT_CONFIG_DIR, t_ctx->ctx->machine_id, t_ctx->ctx->fleet_name); + ret = sprintf(expectedValue, "%s" PATH_SEPARATOR "%s" PATH_SEPARATOR "%s" + PATH_SEPARATOR "test.yaml", TEST_FLEET_CONFIG_DIR, + t_ctx->ctx->machine_id, t_ctx->ctx->fleet_name); TEST_CHECK(ret > 0); value = fleet_config_filename( t_ctx->ctx, "test" ); @@ -118,4 +129,4 @@ static void test_in_fleet_format() { TEST_LIST = { {"in_calyptia_fleet_format", test_in_fleet_format}, {NULL, NULL} -}; \ No newline at end of file +}; diff --git a/tests/runtime/in_dummy.c b/tests/runtime/in_dummy.c index b194a853967..f2f38015560 120000 --- a/tests/runtime/in_dummy.c +++ b/tests/runtime/in_dummy.c @@ -1 +1 @@ -in_simple_systems.c \ No newline at end of file +#include "in_simple_systems.c" \ No newline at end of file diff --git a/tests/runtime/in_event_test.c b/tests/runtime/in_event_test.c index 1169d8fd821..484b3072d8f 100644 --- a/tests/runtime/in_event_test.c +++ b/tests/runtime/in_event_test.c @@ -3,7 +3,7 @@ #include #include "flb_tests_runtime.h" -void flb_test_input_event() +void flb_test_input_event(void) { int ret; flb_ctx_t *ctx; @@ -22,7 +22,8 @@ void flb_test_input_event() ret = flb_start(ctx); TEST_CHECK(ret == 0); - sleep(8); + ret = flb_loop(ctx); + TEST_CHECK(ret == 0); flb_stop(ctx); flb_destroy(ctx); diff --git a/tests/runtime/in_forward.c b/tests/runtime/in_forward.c index 4edaf3f0411..9712d6a949c 100644 --- a/tests/runtime/in_forward.c +++ b/tests/runtime/in_forward.c @@ -32,6 +32,10 @@ #include #include #endif +#ifndef _WIN32 +#include +#include +#endif #include #include "flb_tests_runtime.h" @@ -561,6 +565,7 @@ void flb_test_unix_perm() exit(EXIT_FAILURE); } +#ifndef _WIN32 if (!TEST_CHECK((sb.st_mode & S_IRWXO) == 0)) { TEST_MSG("Permssion(others) error. val=0x%x",sb.st_mode & S_IRWXO); } @@ -570,6 +575,7 @@ void flb_test_unix_perm() if (!TEST_CHECK((sb.st_mode & S_IRWXU) == (S_IRUSR | S_IWUSR))) { TEST_MSG("Permssion(user) error. val=0x%x",sb.st_mode & S_IRWXU); } +#endif flb_socket_close(fd); test_ctx_destroy(ctx); diff --git a/tests/runtime/in_http.c b/tests/runtime/in_http.c index f4a0fc0fed1..ba745e24ed2 100644 --- a/tests/runtime/in_http.c +++ b/tests/runtime/in_http.c @@ -20,18 +20,20 @@ #include #include -#include #include +#ifndef _WIN32 #include #include #include #include +#endif #include #include #include #include #include +#include #include #include "flb_tests_runtime.h" @@ -41,17 +43,20 @@ #define MOCK_VALID_JWT "eyJhbGciOiJSUzI1NiIsImtpZCI6InRlc3QiLCJ0eXAiOiJKV1QifQ.eyJleHAiOjE4OTM0NTYwMDAsImlzcyI6Imlzc3VlciIsImF1ZCI6ImF1ZGllbmNlIiwiYXpwIjoiY2xpZW50MSJ9.TqWs06LUpQa0FGLejnOkWAD6v562d5CUh2NwsJ7iAuae9-WNFBKU6mP1zAaoafla6o5npee7RfbSzZNFI4PKhqAj69789JjAYV7IW-GSuMwJejHdVOWmCc5lmcZPH0EVxEkHA6lFQxYQwDCrfQ8Sd4Q3vYCV6sLPENcuNpQi9ytjVjaZs_7ONH2oA-sZ7EUchqJJoIBPfjit2yYsq9NeemxCzYMtngiC-IX12eEfaQ1cVYPIjhhN_NaMvapznp-BW4gnXkNoAZ1S-p1axWWY-6UgRdMYOr0Hy5PHQ9fCuHJ6Z-blYdtuGavCUGHK5ghX-JdH1WJ51F89992dQ5yF_w" struct jwks_mock_server { - int listen_fd; + flb_sockfd_t listen_fd; int port; int stop; pthread_t thread; }; -static void jwks_mock_send_response(int fd) +static void jwks_mock_send_response(flb_sockfd_t fd) { - char buffer[512]; + char buffer[2048]; + int len; + int sent; + int total; - snprintf(buffer, sizeof(buffer), + len = snprintf(buffer, sizeof(buffer), "HTTP/1.1 200 OK\r\n" "Content-Length: %zu\r\n" "Content-Type: application/json\r\n" @@ -59,7 +64,17 @@ static void jwks_mock_send_response(int fd) "%s", strlen(MOCK_JWKS_BODY), MOCK_JWKS_BODY); - send(fd, buffer, strlen(buffer), 0); + printf("jwks_mock_send_response: Sending %d bytes (Content-Length: %zu)\n", len, strlen(MOCK_JWKS_BODY)); + fflush(stdout); + + total = 0; + while (total < len) { + sent = send(fd, buffer + total, len - total, 0); + if (sent <= 0) { + break; + } + total += sent; + } } static void *jwks_mock_server_thread(void *data) @@ -67,26 +82,47 @@ static void *jwks_mock_server_thread(void *data) struct jwks_mock_server *server = (struct jwks_mock_server *) data; fd_set rfds; struct timeval tv; - int client_fd; + flb_sockfd_t client_fd; + char request[2048]; + int total; + int bytes; - client_fd = -1; + client_fd = FLB_INVALID_SOCKET; while (!server->stop) { FD_ZERO(&rfds); FD_SET(server->listen_fd, &rfds); tv.tv_sec = 0; tv.tv_usec = 200000; - if (select(server->listen_fd + 1, &rfds, NULL, NULL, &tv) <= 0) { + if (select((int) (server->listen_fd + 1), &rfds, NULL, NULL, &tv) <= 0) { continue; } client_fd = accept(server->listen_fd, NULL, NULL); - if (client_fd < 0) { + if (client_fd == FLB_INVALID_SOCKET) { continue; } + flb_net_socket_blocking(client_fd); + + memset(request, 0, sizeof(request)); + total = 0; + while (total < sizeof(request) - 1) { + bytes = recv(client_fd, request + total, + (int) (sizeof(request) - 1 - total), 0); + if (bytes <= 0) { + break; + } + + total += bytes; + request[total] = '\0'; + if (strstr(request, "\r\n\r\n") != NULL) { + break; + } + } + jwks_mock_send_response(client_fd); - close(client_fd); + flb_socket_close(client_fd); } return NULL; @@ -100,13 +136,25 @@ static int jwks_mock_server_start(struct jwks_mock_server *server) int flags; memset(server, 0, sizeof(struct jwks_mock_server)); + server->listen_fd = FLB_INVALID_SOCKET; +#ifdef _WIN32 + { + WSADATA wsa_data; + WSAStartup(0x0201, &wsa_data); + } +#endif server->listen_fd = socket(AF_INET, SOCK_STREAM, 0); - if (server->listen_fd < 0) { + if (server->listen_fd == FLB_INVALID_SOCKET) { +#ifdef _WIN32 + fprintf(stderr, "socket failed with WSAGetLastError: %d\n", WSAGetLastError()); +#else + perror("socket"); +#endif return -1; } - setsockopt(server->listen_fd, SOL_SOCKET, SO_REUSEADDR, &on, sizeof(on)); + setsockopt(server->listen_fd, SOL_SOCKET, SO_REUSEADDR, (const char *)&on, sizeof(on)); memset(&addr, 0, sizeof(addr)); addr.sin_family = AF_INET; @@ -114,30 +162,53 @@ static int jwks_mock_server_start(struct jwks_mock_server *server) addr.sin_port = 0; if (bind(server->listen_fd, (struct sockaddr *) &addr, sizeof(addr)) < 0) { - close(server->listen_fd); +#ifdef _WIN32 + fprintf(stderr, "bind failed with WSAGetLastError: %d\n", WSAGetLastError()); +#else + perror("bind"); +#endif + flb_socket_close(server->listen_fd); return -1; } len = sizeof(addr); if (getsockname(server->listen_fd, (struct sockaddr *) &addr, &len) < 0) { - close(server->listen_fd); +#ifdef _WIN32 + fprintf(stderr, "getsockname failed with WSAGetLastError: %d\n", WSAGetLastError()); +#else + perror("getsockname"); +#endif + flb_socket_close(server->listen_fd); return -1; } server->port = ntohs(addr.sin_port); if (listen(server->listen_fd, 4) < 0) { - close(server->listen_fd); +#ifdef _WIN32 + fprintf(stderr, "listen failed with WSAGetLastError: %d\n", WSAGetLastError()); +#else + perror("listen"); +#endif + flb_socket_close(server->listen_fd); return -1; } +#ifndef _WIN32 flags = fcntl(server->listen_fd, F_GETFL, 0); if (flags >= 0) { fcntl(server->listen_fd, F_SETFL, flags | O_NONBLOCK); } +#else + { + u_long mode = 1; + ioctlsocket(server->listen_fd, FIONBIO, &mode); + } +#endif if (pthread_create(&server->thread, NULL, jwks_mock_server_thread, server) != 0) { - close(server->listen_fd); + perror("pthread_create"); + flb_socket_close(server->listen_fd); return -1; } @@ -146,13 +217,14 @@ static int jwks_mock_server_start(struct jwks_mock_server *server) static void jwks_mock_server_stop(struct jwks_mock_server *server) { - if (server->listen_fd <= 0) { + if (server->listen_fd == FLB_INVALID_SOCKET) { return; } server->stop = 1; pthread_join(server->thread, NULL); - close(server->listen_fd); + flb_socket_close(server->listen_fd); + server->listen_fd = FLB_INVALID_SOCKET; } struct http_client_ctx { @@ -229,6 +301,11 @@ struct http_client_ctx* http_client_ctx_create() struct http_client_ctx *ret_ctx = NULL; struct mk_event_loop *evl = NULL; +#ifdef _WIN32 + WSADATA wsa_data; + WSAStartup(0x0201, &wsa_data); +#endif + ret_ctx = flb_calloc(1, sizeof(struct http_client_ctx)); if (!TEST_CHECK(ret_ctx != NULL)) { flb_errno(); @@ -330,13 +407,13 @@ static void test_ctx_destroy(struct test_ctx *ctx) http_client_ctx_destroy(ctx->httpc); } - sleep(1); + flb_time_msleep(1000); flb_stop(ctx->flb); flb_destroy(ctx->flb); flb_free(ctx); } -void flb_test_http() +void flb_test_http(void) { struct flb_lib_out_cb cb_data; struct test_ctx *ctx; @@ -552,19 +629,19 @@ void flb_test_http_json_charset_header(char *response_code) test_ctx_destroy(ctx); } -void flb_test_http_successful_response_code_200() +void flb_test_http_successful_response_code_200(void) { flb_test_http_successful_response_code("200"); flb_test_http_json_charset_header("200"); } -void flb_test_http_successful_response_code_204() +void flb_test_http_successful_response_code_204(void) { flb_test_http_successful_response_code("204"); flb_test_http_json_charset_header("204"); } -void flb_test_http_failure_400_bad_json() { +void flb_test_http_failure_400_bad_json(void) { struct flb_lib_out_cb cb_data; struct test_ctx *ctx; struct flb_http_client *c; @@ -629,7 +706,7 @@ void flb_test_http_failure_400_bad_json() { test_ctx_destroy(ctx); } -void flb_test_http_failure_400_bad_disk_write() +void flb_test_http_failure_400_bad_disk_write(void) { struct flb_lib_out_cb cb_data; struct test_ctx *ctx; @@ -786,17 +863,17 @@ void test_http_tag_key(char *input) test_ctx_destroy(ctx); } -void flb_test_http_tag_key_with_map_input() +void flb_test_http_tag_key_with_map_input(void) { test_http_tag_key("{\"tag\":\"new_tag\",\"test\":\"msg\"}"); } -void flb_test_http_tag_key_with_array_input() +void flb_test_http_tag_key_with_array_input(void) { test_http_tag_key("[{\"tag\":\"new_tag\",\"test\":\"msg\"}]"); } -void flb_test_http_oauth2_requires_token() +void flb_test_http_oauth2_requires_token(void) { struct flb_lib_out_cb cb_data; struct test_ctx *ctx; @@ -866,7 +943,7 @@ void flb_test_http_oauth2_requires_token() jwks_mock_server_stop(&jwks); } -void flb_test_http_oauth2_accepts_valid_token() +void flb_test_http_oauth2_accepts_valid_token(void) { struct flb_lib_out_cb cb_data; struct test_ctx *ctx; @@ -1032,29 +1109,29 @@ void test_http_add_remote_addr(char *input, char *xff_content, char *expected_ip } /* Test if remote_addr injection is skipped if remote_addr_key is already present */ -void flb_test_http_remote_addr_skip_colliding_ng() +void flb_test_http_remote_addr_skip_colliding_ng(void) { test_http_add_remote_addr("{\"test\":\"msg\",\"REMOTE_ADDR\":\"old\"}", "1.2.3.4, 5.6.7.8", "old", "true"); } /* Test flow through next gen http server */ -void flb_test_http_remote_addr_map_ng() +void flb_test_http_remote_addr_map_ng(void) { test_http_add_remote_addr("{\"test\":\"msg\"}", "1.2.3.4, 5.6.7.8", "1.2.3.4", "true"); } -void flb_test_http_remote_addr_array_ng() +void flb_test_http_remote_addr_array_ng(void) { test_http_add_remote_addr("[{\"test\":\"msg\"}]", "1.2.3.4, 5.6.7.8", "1.2.3.4", "true"); } /* Test flow through legacy http server (monkey) */ -void flb_test_http_remote_addr_map() +void flb_test_http_remote_addr_map(void) { test_http_add_remote_addr("{\"test\":\"msg\"}", "1.2.3.4, 5.6.7.8", "1.2.3.4", "false"); } -void flb_test_http_remote_addr_array() +void flb_test_http_remote_addr_array(void) { test_http_add_remote_addr("[{\"test\":\"msg\"}]", "1.2.3.4, 5.6.7.8", "1.2.3.4", "false"); } diff --git a/tests/runtime/in_kubernetes_events.c b/tests/runtime/in_kubernetes_events.c index 83745e819f5..082afabaf2e 100644 --- a/tests/runtime/in_kubernetes_events.c +++ b/tests/runtime/in_kubernetes_events.c @@ -88,7 +88,11 @@ static flb_sds_t read_file(const char *filename) int ret; flb_sds_t payload = NULL; +#ifdef FLB_SYSTEM_WINDOWS + fd = open(filename, O_RDONLY | O_BINARY, 0); +#else fd = open(filename, O_RDONLY, 0); +#endif if (fd != -1) { if (fstat(fd, &sb) == 0) { payload = flb_sds_create_size(sb.st_size+1); @@ -285,7 +289,7 @@ static struct test_ctx *test_ctx_create(struct flb_lib_out_cb *data) TEST_CHECK(flb_input_set(ctx->flb, i_ffd, "kube_url", kube_url, "kube_token_file", KUBE_TOKEN_FILE, - "kube_retention_time", "365000d", + "kube_retention_time", "3650d", "tls", "off", "interval_sec", "1", "interval_nsec", "0", @@ -340,7 +344,7 @@ static struct test_ctx *test_ctx_create_with_config(struct flb_lib_out_cb *data, ret = flb_input_set(ctx->flb, i_ffd, "kube_url", kube_url, "kube_token_file", KUBE_TOKEN_FILE, - "kube_retention_time", "365000d", + "kube_retention_time", "3650d", "tls", "off", "interval_sec", "1", "interval_nsec", "0", diff --git a/tests/runtime/in_opentelemetry_routing.c b/tests/runtime/in_opentelemetry_routing.c index 018a826d19a..be811651ff6 100644 --- a/tests/runtime/in_opentelemetry_routing.c +++ b/tests/runtime/in_opentelemetry_routing.c @@ -27,7 +27,9 @@ #include #include #include +#ifndef _WIN32 #include +#endif #include #include #include @@ -37,6 +39,17 @@ #include "../../plugins/in_opentelemetry/opentelemetry.h" #include "../../plugins/in_opentelemetry/opentelemetry_logs.h" +#ifdef _WIN32 +static int test_mkdir(const char *path, int mode) +{ + (void) mode; + + return mkdir(path); +} +#else +#define test_mkdir(path, mode) mkdir(path, mode) +#endif + #define JSON_CONTENT_TYPE "application/json" /* Pick a port that is not in use by other tests as well */ /* Ensure you update data/routing/otlp_comprehensive_routing_test.yaml */ @@ -224,7 +237,7 @@ static struct test_ctx *test_ctx_create(const char *config_file) } /* Create directory if it doesn't exist */ - ret = mkdir(ctx->output_dir, 0755); + ret = test_mkdir(ctx->output_dir, 0755); if (ret != 0 && errno != EEXIST) { flb_error("[test] Failed to create output directory: %s", ctx->output_dir); flb_destroy(ctx->flb); diff --git a/tests/runtime/in_prometheus_textfile.c b/tests/runtime/in_prometheus_textfile.c index ed02a087d33..96efb18dc59 100644 --- a/tests/runtime/in_prometheus_textfile.c +++ b/tests/runtime/in_prometheus_textfile.c @@ -3,6 +3,11 @@ #include "flb_tests_runtime.h" #define DPATH_PROM_TEXTFILE FLB_TESTS_DATA_PATH "/data/prometheus_textfile" +#ifdef _WIN32 +#define PROM_TEXTFILE_GLOB DPATH_PROM_TEXTFILE "\\*.prom" +#else +#define PROM_TEXTFILE_GLOB DPATH_PROM_TEXTFILE "/*.prom" +#endif static pthread_mutex_t result_mutex = PTHREAD_MUTEX_INITIALIZER; static int num_output = 0; @@ -86,7 +91,7 @@ static void test_prometheus_textfile(void) TEST_CHECK(ctx->i_ffd >= 0); ret = flb_input_set(ctx->flb, ctx->i_ffd, "scrape_interval", "1s", - "path", DPATH_PROM_TEXTFILE "/metrics.prom", + "path", PROM_TEXTFILE_GLOB, NULL); TEST_CHECK(ret == 0); diff --git a/tests/runtime/in_random.c b/tests/runtime/in_random.c index b194a853967..f2f38015560 120000 --- a/tests/runtime/in_random.c +++ b/tests/runtime/in_random.c @@ -1 +1 @@ -in_simple_systems.c \ No newline at end of file +#include "in_simple_systems.c" \ No newline at end of file diff --git a/tests/runtime/in_simple_systems.c b/tests/runtime/in_simple_systems.c index 5eb8a836603..f20dd477eb9 100644 --- a/tests/runtime/in_simple_systems.c +++ b/tests/runtime/in_simple_systems.c @@ -19,36 +19,40 @@ #include #include -#include +#include #include +#ifndef _WIN32 #include +#endif #include "flb_tests_runtime.h" int64_t result_time; static inline int64_t set_result(int64_t v) { +#ifdef _WIN32 + return InterlockedExchange64((volatile LONG64 *)&result_time, v); +#else int64_t old = __sync_lock_test_and_set(&result_time, v); return old; +#endif } static inline int64_t get_result(void) { +#ifdef _WIN32 + return InterlockedCompareExchange64((volatile LONG64 *)&result_time, 0, 0); +#else int64_t old = __sync_fetch_and_add(&result_time, 0); - return old; +#endif } static inline int64_t time_in_ms() { - int ms; - struct timespec s; - TEST_CHECK(clock_gettime(CLOCK_MONOTONIC, &s) == 0); - ms = s.tv_nsec / 1.0e6; - if (ms >= 1000) { - ms = 0; - } - return 1000 * s.tv_sec + ms; + struct flb_time s; + flb_time_get(&s); + return flb_time_to_millisec(&s); } int callback_test(void* data, size_t size, void* cb_data) @@ -152,7 +156,7 @@ void do_test(char *system, ...) TEST_CHECK(flb_start(ctx) == 0); for (trys = 0; trys < 5 && get_result() == 0; trys++) { - sleep(1); + flb_time_msleep(1000); } flb_info("[test] check status 1"); @@ -160,7 +164,7 @@ void do_test(char *system, ...) TEST_CHECK(ret > 0); for (trys = 0; trys < 5 && get_result() == ret; trys++) { - sleep(1); + flb_time_msleep(1000); } flb_info("[test] check status 2"); @@ -218,13 +222,13 @@ void do_test_records(char *system, void (*records_cb)(struct callback_records *) TEST_CHECK(flb_start(ctx) == 0); for (trys = 0; trys < 5 && records->num_records <= 0; trys++) { - sleep(1); + flb_time_msleep(1000); } - records_cb(records); - flb_stop(ctx); + records_cb(records); + for (idx = 0; idx < records->num_records; idx++) { flb_lib_free(records->records[idx].data); } @@ -274,8 +278,10 @@ void do_test_records_single(char *system, void (*records_cb)(struct callback_rec TEST_CHECK(out_ffd >= 0); TEST_CHECK(flb_output_set(ctx, out_ffd, "match", "test", NULL) == 0); +#ifndef _WIN32 exit_ffd = flb_output(ctx, (char *)"exit", &cb); TEST_CHECK(flb_output_set(ctx, exit_ffd, "match", "test", NULL) == 0); +#endif TEST_CHECK(flb_service_set(ctx, "Flush", "1", "Grace", "1", @@ -285,12 +291,12 @@ void do_test_records_single(char *system, void (*records_cb)(struct callback_rec TEST_CHECK(flb_start(ctx) == 0); /* 4 sec passed. It must have flushed */ - sleep(5); - - records_cb(records); + flb_time_msleep(5000); flb_stop(ctx); + records_cb(records); + for (i = 0; i < records->num_records; i++) { flb_lib_free(records->records[i].data); } @@ -347,12 +353,12 @@ void do_test_records_wait_time(char *system, int wait_time, void (*records_cb)(s TEST_CHECK(flb_start(ctx) == 0); /* Set wait_time plus 2 sec passed. It must have flushed */ - sleep(wait_time + 2); - - records_cb(records); + flb_time_msleep((wait_time + 2) * 1000); flb_stop(ctx); + records_cb(records); + for (i = 0; i < records->num_records; i++) { flb_lib_free(records->records[i].data); } @@ -362,14 +368,14 @@ void do_test_records_wait_time(char *system, int wait_time, void (*records_cb)(s flb_destroy(ctx); } -void flb_test_in_disk_flush() +void flb_test_in_disk_flush(void) { do_test("disk", "interval_sec", "0", "interval_nsec", "500000000", NULL); } -void flb_test_in_proc_flush() +void flb_test_in_proc_flush(void) { do_test("proc", "interval_sec", "0", @@ -380,7 +386,7 @@ void flb_test_in_proc_flush() "fd", "on", NULL); } -void flb_test_in_head_flush() +void flb_test_in_head_flush(void) { do_test("head", "interval_sec", "0", @@ -388,11 +394,11 @@ void flb_test_in_head_flush() "File", "/dev/urandom", NULL); } -void flb_test_in_cpu_flush() +void flb_test_in_cpu_flush(void) { do_test("cpu", NULL); } -void flb_test_in_random_flush() +void flb_test_in_random_flush(void) { do_test("random", NULL); } @@ -534,7 +540,7 @@ void flb_test_dummy_records_message_copies_5(struct callback_records *records) int trys; for (trys = 0; trys < 5 && records->num_records < 5; trys++) { - sleep(1); + flb_time_msleep(1000); } TEST_CHECK(records->num_records >= 5); } @@ -544,7 +550,7 @@ void flb_test_dummy_records_message_copies_100(struct callback_records *records) int trys; for (trys = 0; trys < 100 && records->num_records < 100; trys++) { - sleep(1); + flb_time_msleep(1000); } TEST_CHECK(records->num_records >= 100); } @@ -554,7 +560,7 @@ void flb_test_dummy_records_message_rate(struct callback_records *records) int trys; for (trys = 0; trys < 20 && records->num_records < 20; trys++) { - sleep(1); + flb_time_msleep(1000); } TEST_CHECK(records->num_records >= 20); } @@ -574,7 +580,7 @@ void flb_test_dummy_records_message_flush_on_startup(struct callback_records *re TEST_CHECK(records->num_records >= 2); } -void flb_test_in_dummy_flush() +void flb_test_in_dummy_flush(void) { do_test("dummy", NULL); do_test_records("dummy", flb_test_dummy_records_message_default, NULL); @@ -622,12 +628,12 @@ void flb_test_in_dummy_flush() NULL); } -void flb_test_in_dummy_thread_flush() +void flb_test_in_dummy_thread_flush(void) { do_test("dummy_thread", NULL); } -void flb_test_in_mem_flush() +void flb_test_in_mem_flush(void) { do_test("mem", NULL); } diff --git a/tests/runtime/in_syslog.c b/tests/runtime/in_syslog.c index 26911924705..538abd2c5b5 100644 --- a/tests/runtime/in_syslog.c +++ b/tests/runtime/in_syslog.c @@ -646,6 +646,7 @@ void flb_test_syslog_unix_perm() exit(EXIT_FAILURE); } +#ifndef _WIN32 if (!TEST_CHECK((sb.st_mode & S_IRWXO) == 0)) { TEST_MSG("Permssion(others) error. val=0x%x",sb.st_mode & S_IRWXO); } @@ -655,6 +656,7 @@ void flb_test_syslog_unix_perm() if (!TEST_CHECK((sb.st_mode & S_IRWXU) == (S_IRUSR | S_IWUSR))) { TEST_MSG("Permssion(user) error. val=0x%x",sb.st_mode & S_IRWXU); } +#endif test_ctx_destroy(ctx); } diff --git a/tests/runtime/in_tail.c b/tests/runtime/in_tail.c index 8944d59b13b..20073afe811 100644 --- a/tests/runtime/in_tail.c +++ b/tests/runtime/in_tail.c @@ -1,4 +1,4 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ /* Fluent Bit * ========== @@ -34,8 +34,51 @@ Approach for this tests is basing on filter_kubernetes tests #include #include #include +#ifdef _WIN32 +#include +#include +#include "../../plugins/in_tail/win32/interface.h" +#endif #include "flb_tests_runtime.h" +#ifdef _WIN32 +#define fsync _commit +#ifndef S_IRUSR +#define S_IRUSR _S_IREAD +#endif +#ifndef S_IWUSR +#define S_IWUSR _S_IWRITE +#endif +#ifndef S_IRGRP +#define S_IRGRP 0 +#endif +#ifndef S_IWGRP +#define S_IWGRP 0 +#endif +#ifndef S_IRWXU +#define S_IRWXU (S_IRUSR | S_IWUSR) +#endif +#ifndef AT_FDCWD +#define AT_FDCWD -100 +#endif + +static int flb_test_utimensat(int dirfd, const char *path, + const struct timespec times[2], int flags) +{ + struct _utimbuf tm; + + (void) dirfd; + (void) flags; + + tm.actime = times[0].tv_sec; + tm.modtime = times[1].tv_sec; + + return _utime(path, &tm); +} + +#define utimensat flb_test_utimensat +#endif + #ifdef FLB_HAVE_INOTIFY #include "../../plugins/in_tail/tail_config.h" #endif @@ -189,6 +232,9 @@ static struct test_tail_ctx *test_tail_ctx_create(struct flb_lib_out_cb *data, /* open() flags */ o_flags = O_RDWR | O_CREAT; +#ifdef FLB_SYSTEM_WINDOWS + o_flags |= O_BINARY; +#endif if (paths != NULL) { ctx->fds = flb_malloc(sizeof(int) * path_num); @@ -375,8 +421,12 @@ void wait_expected_num_with_timeout(uint32_t timeout_ms, int expected_num, int * static inline int64_t set_result(int64_t v) { +#ifdef _WIN32 + return InterlockedExchange64((volatile LONG64 *)&result_time, v); +#else int64_t old = __sync_lock_test_and_set(&result_time, v); return old; +#endif } @@ -387,13 +437,18 @@ static int file_to_buf(const char *path, char **out_buf, size_t *out_size) char *buf; FILE *fp; struct stat st; + const char *file_mode = "r"; + +#ifdef FLB_SYSTEM_WINDOWS + file_mode = "rb"; +#endif ret = stat(path, &st); if (ret == -1) { return -1; } - fp = fopen(path, "r"); + fp = fopen(path, file_mode); if (!fp) { return -1; } @@ -1576,6 +1631,25 @@ void flb_test_exclude_path() test_tail_ctx_destroy(ctx); } +#ifdef _WIN32 +void flb_test_windows_extended_path_prefixes(void) +{ + size_t length; + wchar_t local_path[] = L"\\\\?\\C:\\logs\\unicode.log"; + wchar_t unc_path[] = L"\\\\?\\UNC\\server\\share\\unicode.log"; + const wchar_t expected_local_path[] = L"C:\\logs\\unicode.log"; + const wchar_t expected_unc_path[] = L"\\\\server\\share\\unicode.log"; + + length = win32_remove_extended_path_prefix(local_path, wcslen(local_path)); + TEST_CHECK(length == wcslen(expected_local_path)); + TEST_CHECK(wcscmp(local_path, expected_local_path) == 0); + + length = win32_remove_extended_path_prefix(unc_path, wcslen(unc_path)); + TEST_CHECK(length == wcslen(expected_unc_path)); + TEST_CHECK(wcscmp(unc_path, expected_unc_path) == 0); +} +#endif + void flb_test_offset_key() { struct flb_lib_out_cb cb_data; @@ -3001,6 +3075,9 @@ TEST_LIST = { {"path_comma", flb_test_path_comma}, {"path_key", flb_test_path_key}, {"exclude_path", flb_test_exclude_path}, +#ifdef _WIN32 + {"windows_extended_path_prefixes", flb_test_windows_extended_path_prefixes}, +#endif {"offset_key", flb_test_offset_key}, {"multiline_offset_key", flb_test_multiline_offset_key}, {"skip_empty_lines", flb_test_skip_empty_lines}, diff --git a/tests/runtime/in_tcp.c b/tests/runtime/in_tcp.c index ea6283b6a56..dc4eef1a975 100644 --- a/tests/runtime/in_tcp.c +++ b/tests/runtime/in_tcp.c @@ -769,7 +769,9 @@ void flb_test_format_none_with_unknown_parser() TEST_MSG("flb_start unexpectedly succeeded with unknown parser"); } - test_ctx_destroy(ctx); + /* flb_start failed, so there is no running engine to stop. */ + flb_destroy(ctx->flb); + flb_free(ctx); } #endif diff --git a/tests/runtime/in_udp.c b/tests/runtime/in_udp.c index 2ff67e0d138..6c64c2228b0 100644 --- a/tests/runtime/in_udp.c +++ b/tests/runtime/in_udp.c @@ -530,7 +530,9 @@ void flb_test_format_none_with_unknown_parser() TEST_MSG("flb_start unexpectedly succeeded with unknown parser"); } - test_ctx_destroy(ctx); + /* flb_start failed, so there is no running engine to stop. */ + flb_destroy(ctx->flb); + flb_free(ctx); } void flb_test_format_none_parser_fallback_udp() diff --git a/tests/runtime/out_chronicle.c b/tests/runtime/out_chronicle.c index 22eeb616ca8..6443d3182c9 100644 --- a/tests/runtime/out_chronicle.c +++ b/tests/runtime/out_chronicle.c @@ -6,26 +6,75 @@ pthread_mutex_t result_mutex = PTHREAD_MUTEX_INITIALIZER; int num_invoked = 0; +static const char *callback_error = NULL; + static int get_output_invoked() { int ret; + + pthread_mutex_lock(&result_mutex); + ret = num_invoked; + pthread_mutex_unlock(&result_mutex); + + return ret; +} + +static int increment_output_invoked() +{ + int ret; + pthread_mutex_lock(&result_mutex); + num_invoked++; ret = num_invoked; pthread_mutex_unlock(&result_mutex); return ret; } -static void set_output_invoked(int num) +static void set_callback_error(const char *message) { pthread_mutex_lock(&result_mutex); - num_invoked = num; + if (callback_error == NULL) { + callback_error = message; + } pthread_mutex_unlock(&result_mutex); } static void clear_output_invoked() { - set_output_invoked(0); + pthread_mutex_lock(&result_mutex); + num_invoked = 0; + callback_error = NULL; + pthread_mutex_unlock(&result_mutex); +} + +static void check_callback_error() +{ + const char *message; + + pthread_mutex_lock(&result_mutex); + message = callback_error; + pthread_mutex_unlock(&result_mutex); + + if (!TEST_CHECK(message == NULL)) { + TEST_MSG("%s", message); + } +} + +static void stop_and_check(flb_ctx_t *ctx, int expected_invocations) +{ + int invocations; + + flb_stop(ctx); + + invocations = get_output_invoked(); + if (!TEST_CHECK(invocations == expected_invocations)) { + TEST_MSG("got %d formatter callbacks, expected %d", + invocations, expected_invocations); + } + check_callback_error(); + + flb_destroy(ctx); } static void cb_check_format_no_log_key(void *ctx, int ffd, @@ -35,33 +84,38 @@ static void cb_check_format_no_log_key(void *ctx, int ffd, char *out_json = res_data; char *p; - set_output_invoked(1); + if (res_ret != 0 || out_json == NULL) { + set_callback_error("formatter returned an error or no output"); + flb_sds_destroy(res_data); + return; + } p = strstr(out_json, "\"customer_id\":\"test-customer\""); - if (!TEST_CHECK(p != NULL)) { - TEST_MSG("Expected customer_id not found. Got: %s", out_json); + if (p == NULL) { + set_callback_error("expected customer_id was not found"); } p = strstr(out_json, "\"log_type\":\"TEST_LOG\""); - if (!TEST_CHECK(p != NULL)) { - TEST_MSG("Expected log_type not found. Got: %s", out_json); + if (p == NULL) { + set_callback_error("expected log_type was not found"); } p = strstr(out_json, "\"entries\":["); - if (!TEST_CHECK(p != NULL)) { - TEST_MSG("Entries array not found. Got: %s", out_json); + if (p == NULL) { + set_callback_error("entries array was not found"); } p = strstr(out_json, "\"log_text\":\"{\\\"message\\\":\\\"hello world\\\"}\""); - if (!TEST_CHECK(p != NULL)) { - TEST_MSG("Expected log_text not found. Got: %s", out_json); + if (p == NULL) { + set_callback_error("expected log_text was not found"); } p = strstr(out_json, "\"ts_rfc3339\":"); - if (!TEST_CHECK(p != NULL)) { - TEST_MSG("Expected ts_rfc3339 key not found. Got: %s", out_json); + if (p == NULL) { + set_callback_error("expected ts_rfc3339 key was not found"); } + increment_output_invoked(); flb_sds_destroy(res_data); } @@ -76,16 +130,21 @@ static void cb_check_format_with_log_key(void *ctx, int ffd, return; } - set_output_invoked(1); + if (res_ret != 0) { + set_callback_error("formatter returned an error"); + } p = strstr(out_json, "\"log_text\":\"This is the target message.\""); - if (!TEST_CHECK(p != NULL)) { - TEST_MSG("Expected log_text with specific value not found. Got: %s", out_json); + if (p == NULL) { + set_callback_error("expected log_text value was not found"); } p = strstr(out_json, "other_key"); - TEST_CHECK(p == NULL); + if (p != NULL) { + set_callback_error("unexpected other_key was found"); + } + increment_output_invoked(); flb_sds_destroy(res_data); } @@ -96,18 +155,23 @@ static void cb_check_format_multiple_records(void *ctx, int ffd, char *out_json = res_data; char *p1, *p2; - set_output_invoked(1); + if (res_ret != 0 || out_json == NULL) { + set_callback_error("formatter returned an error or no output"); + flb_sds_destroy(res_data); + return; + } p1 = strstr(out_json, "\"log_text\":\"{\\\"message\\\":\\\"record one\\\"}\""); - if (!TEST_CHECK(p1 != NULL)) { - TEST_MSG("First record not found. Got: %s", out_json); + if (p1 == NULL) { + set_callback_error("first record was not found"); } p2 = strstr(out_json, "\"log_text\":\"{\\\"message\\\":\\\"record two\\\"}\""); - if (!TEST_CHECK(p2 != NULL)) { - TEST_MSG("Second record not found. Got: %s", out_json); + if (p2 == NULL) { + set_callback_error("second record was not found"); } + increment_output_invoked(); flb_sds_destroy(res_data); } @@ -118,16 +182,23 @@ static void cb_check_format_partially_succeeded_records(void *ctx, int ffd, char *out_json = res_data; char *p1, *p2; - set_output_invoked(1); + if (res_ret != 0 || out_json == NULL) { + set_callback_error("formatter returned an error or no output"); + flb_sds_destroy(res_data); + return; + } p1 = strstr(out_json, "\"log_text\":\"record one\""); - if (!TEST_CHECK(p1 != NULL)) { - TEST_MSG("Expected log_text with specific value not found. Got: %s", out_json); + if (p1 == NULL) { + set_callback_error("expected log_text value was not found"); } p2 = strstr(out_json, "\"test\""); - TEST_CHECK(p2 == NULL); + if (p2 != NULL) { + set_callback_error("unexpected test field was found"); + } + increment_output_invoked(); flb_sds_destroy(res_data); } @@ -138,38 +209,43 @@ static void cb_check_format_namespace_and_labels(void *ctx, int ffd, char *out_json = res_data; char *p; - set_output_invoked(1); + if (res_ret != 0 || out_json == NULL) { + set_callback_error("formatter returned an error or no output"); + flb_sds_destroy(res_data); + return; + } p = strstr(out_json, "\"namespace\":\"tenant-a\""); - if (!TEST_CHECK(p != NULL)) { - TEST_MSG("Expected namespace not found. Got: %s", out_json); + if (p == NULL) { + set_callback_error("expected namespace was not found"); } p = strstr(out_json, "\"labels\":["); - if (!TEST_CHECK(p != NULL)) { - TEST_MSG("Expected labels array not found. Got: %s", out_json); + if (p == NULL) { + set_callback_error("expected labels array was not found"); } p = strstr(out_json, "\"key\":\"env\""); - if (!TEST_CHECK(p != NULL)) { - TEST_MSG("Expected static label key not found. Got: %s", out_json); + if (p == NULL) { + set_callback_error("expected static label key was not found"); } p = strstr(out_json, "\"value\":\"production\""); - if (!TEST_CHECK(p != NULL)) { - TEST_MSG("Expected static label value not found. Got: %s", out_json); + if (p == NULL) { + set_callback_error("expected static label value was not found"); } p = strstr(out_json, "\"key\":\"cluster_name\""); - if (!TEST_CHECK(p != NULL)) { - TEST_MSG("Expected dynamic label key not found. Got: %s", out_json); + if (p == NULL) { + set_callback_error("expected dynamic label key was not found"); } p = strstr(out_json, "\"value\":\"blue\""); - if (!TEST_CHECK(p != NULL)) { - TEST_MSG("Expected dynamic label value not found. Got: %s", out_json); + if (p == NULL) { + set_callback_error("expected dynamic label value was not found"); } + increment_output_invoked(); flb_sds_destroy(res_data); } @@ -180,21 +256,28 @@ static void cb_check_format_namespace_fallback_and_missing_label(void *ctx, int char *out_json = res_data; char *p; - set_output_invoked(1); + if (res_ret != 0 || out_json == NULL) { + set_callback_error("formatter returned an error or no output"); + flb_sds_destroy(res_data); + return; + } p = strstr(out_json, "\"namespace\":\"fallback-namespace\""); - if (!TEST_CHECK(p != NULL)) { - TEST_MSG("Expected fallback namespace not found. Got: %s", out_json); + if (p == NULL) { + set_callback_error("expected fallback namespace was not found"); } p = strstr(out_json, "\"key\":\"missing\""); - TEST_CHECK(p == NULL); + if (p != NULL) { + set_callback_error("unexpected missing label was found"); + } p = strstr(out_json, "\"log_text\":\"{\\\"message\\\":\\\"hello world\\\"}\""); - if (!TEST_CHECK(p != NULL)) { - TEST_MSG("Expected log_text not found. Got: %s", out_json); + if (p == NULL) { + set_callback_error("expected log_text was not found"); } + increment_output_invoked(); flb_sds_destroy(res_data); } @@ -206,50 +289,64 @@ static void cb_check_format_split_on_metadata_change(void *ctx, int ffd, char *p; int invocation; - invocation = get_output_invoked() + 1; - set_output_invoked(invocation); + if (res_ret != 0 || out_json == NULL) { + set_callback_error("formatter returned an error or no output"); + flb_sds_destroy(res_data); + return; + } + + invocation = increment_output_invoked(); if (invocation == 1) { p = strstr(out_json, "\"namespace\":\"tenant-a\""); - if (!TEST_CHECK(p != NULL)) { - TEST_MSG("Expected first namespace not found. Got: %s", out_json); + if (p == NULL) { + set_callback_error("expected first namespace was not found"); } p = strstr(out_json, "\"value\":\"blue\""); - if (!TEST_CHECK(p != NULL)) { - TEST_MSG("Expected first dynamic label value not found. Got: %s", out_json); + if (p == NULL) { + set_callback_error("expected first dynamic label value was not found"); } p = strstr(out_json, "\"log_text\":\"{\\\"message\\\":\\\"record one\\\""); - if (!TEST_CHECK(p != NULL)) { - TEST_MSG("Expected first record not found. Got: %s", out_json); + if (p == NULL) { + set_callback_error("expected first record was not found"); } p = strstr(out_json, "tenant-b"); - TEST_CHECK(p == NULL); + if (p != NULL) { + set_callback_error("unexpected second namespace was found"); + } p = strstr(out_json, "green"); - TEST_CHECK(p == NULL); + if (p != NULL) { + set_callback_error("unexpected second dynamic label value was found"); + } p = strstr(out_json, "record two"); - TEST_CHECK(p == NULL); + if (p != NULL) { + set_callback_error("unexpected second record was found"); + } } else if (invocation == 2) { p = strstr(out_json, "\"namespace\":\"tenant-b\""); - if (!TEST_CHECK(p != NULL)) { - TEST_MSG("Expected second namespace not found. Got: %s", out_json); + if (p == NULL) { + set_callback_error("expected second namespace was not found"); } p = strstr(out_json, "\"value\":\"green\""); - if (!TEST_CHECK(p != NULL)) { - TEST_MSG("Expected second dynamic label value not found. Got: %s", out_json); + if (p == NULL) { + set_callback_error("expected second dynamic label value was not found"); } p = strstr(out_json, "\"log_text\":\"{\\\"message\\\":\\\"record two\\\""); - if (!TEST_CHECK(p != NULL)) { - TEST_MSG("Expected second record not found. Got: %s", out_json); + if (p == NULL) { + set_callback_error("expected second record was not found"); } } + else { + set_callback_error("formatter was invoked more than twice"); + } flb_sds_destroy(res_data); } @@ -284,9 +381,7 @@ void test_format_no_log_key() sleep(1); - TEST_CHECK(get_output_invoked() == 1); - flb_stop(ctx); - flb_destroy(ctx); + stop_and_check(ctx, 1); } void test_format_with_log_key_found() @@ -322,9 +417,7 @@ void test_format_with_log_key_found() sleep(1); - TEST_CHECK(get_output_invoked() == 1); - flb_stop(ctx); - flb_destroy(ctx); + stop_and_check(ctx, 1); } void test_format_with_log_key_not_found() @@ -358,9 +451,7 @@ void test_format_with_log_key_not_found() sleep(1); - TEST_CHECK(get_output_invoked() == 0); - flb_stop(ctx); - flb_destroy(ctx); + stop_and_check(ctx, 0); } @@ -399,9 +490,7 @@ void test_format_multiple_records() sleep(1); - TEST_CHECK(get_output_invoked() == 1); - flb_stop(ctx); - flb_destroy(ctx); + stop_and_check(ctx, 1); } void test_format_partially_suceeded_records() @@ -440,9 +529,7 @@ void test_format_partially_suceeded_records() sleep(1); - TEST_CHECK(get_output_invoked() == 1); - flb_stop(ctx); - flb_destroy(ctx); + stop_and_check(ctx, 1); } void test_format_namespace_and_labels() @@ -483,9 +570,7 @@ void test_format_namespace_and_labels() sleep(1); - TEST_CHECK(get_output_invoked() == 1); - flb_stop(ctx); - flb_destroy(ctx); + stop_and_check(ctx, 1); } void test_format_namespace_fallback_and_missing_label() @@ -524,9 +609,7 @@ void test_format_namespace_fallback_and_missing_label() sleep(1); - TEST_CHECK(get_output_invoked() == 1); - flb_stop(ctx); - flb_destroy(ctx); + stop_and_check(ctx, 1); } void test_format_split_on_metadata_change() @@ -573,9 +656,7 @@ void test_format_split_on_metadata_change() sleep(1); - TEST_CHECK(get_output_invoked() == 1); - flb_stop(ctx); - flb_destroy(ctx); + stop_and_check(ctx, 1); } diff --git a/tests/runtime/out_counter.c b/tests/runtime/out_counter.c index 29ee334d964..aeec023f889 100644 --- a/tests/runtime/out_counter.c +++ b/tests/runtime/out_counter.c @@ -12,7 +12,9 @@ #define flb_test_dup2 _dup2 #define flb_test_fileno _fileno #else +#ifndef _WIN32 #include +#endif #define flb_test_close close #define flb_test_dup dup #define flb_test_dup2 dup2 diff --git a/tests/runtime/out_file.c b/tests/runtime/out_file.c index 33b1440d04d..3f084217a55 100644 --- a/tests/runtime/out_file.c +++ b/tests/runtime/out_file.c @@ -26,12 +26,26 @@ void flb_test_file_delim_ltsv(void); void flb_test_file_label_delim(void); void flb_test_file_template(void); void flb_test_file_mkdir(void); +void flb_test_file_dynamic_path_file(void); +void flb_test_file_dynamic_requires_fallback(void); +void flb_test_file_dynamic_missing_fallback(void); +void flb_test_file_dynamic_unsafe_fallback(void); +void flb_test_file_dynamic_windows_unsafe_fallback(void); +void flb_test_file_dynamic_path_traversal_fallback(void); +void flb_test_file_dynamic_limit_fallback(void); /* Test list */ TEST_LIST = { {"path", flb_test_file_path}, {"path_file", flb_test_file_path_file}, {"mkdir", flb_test_file_mkdir}, + {"dynamic_path_file", flb_test_file_dynamic_path_file}, + {"dynamic_requires_fallback", flb_test_file_dynamic_requires_fallback}, + {"dynamic_missing_fallback", flb_test_file_dynamic_missing_fallback}, + {"dynamic_unsafe_fallback", flb_test_file_dynamic_unsafe_fallback}, + {"dynamic_windows_unsafe_fallback", flb_test_file_dynamic_windows_unsafe_fallback}, + {"dynamic_path_traversal_fallback", flb_test_file_dynamic_path_traversal_fallback}, + {"dynamic_limit_fallback", flb_test_file_dynamic_limit_fallback}, {"template", flb_test_file_template}, {"delimiter_ltsv", flb_test_file_delim_ltsv}, {"delimiter_csv", flb_test_file_delim_csv}, @@ -522,6 +536,359 @@ void flb_test_file_path_file(void) } #define JSON_BASIC "[1448403340,{\"key1\":\"val1\", \"key2\":\"val2\"}]" +#define JSON_DYNAMIC \ + "[1448403340,{\"proxy_name\":\"proxy1\", \"hostname\":\"host1\"}]" \ + "[1448403341,{\"proxy_name\":\"proxy2\", \"hostname\":\"host2\"}]" +#define JSON_DYNAMIC_MISSING "[1448403340,{\"proxy_name\":\"proxy1\"}]" +#define JSON_DYNAMIC_UNSAFE "[1448403340,{\"hostname\":\"../escape.log\"}]" +#define JSON_DYNAMIC_WINDOWS_UNSAFE "[1448403340,{\"hostname\":\"host:one.log\"}]" +#define JSON_DYNAMIC_PATH_TRAVERSAL "[1448403340,{\"proxy_name\":\"..\"}]" +#define JSON_DYNAMIC_LIMIT \ + "[1448403340,{\"hostname\":\"host1\"}]" \ + "[1448403341,{\"hostname\":\"host2\"}]" + +void flb_test_file_dynamic_path_file(void) +{ + int ret; + int bytes; + char *p = JSON_DYNAMIC; + char path1[256]; + char path2[256]; + char dir1[256]; + char dir2[256]; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + snprintf(dir1, sizeof(dir1), "%s/proxy1", TEST_LOGPATH); + snprintf(dir2, sizeof(dir2), "%s/proxy2", TEST_LOGPATH); + snprintf(path1, sizeof(path1), "%s/host1.log", dir1); + snprintf(path2, sizeof(path2), "%s/host2.log", dir2); + remove(path1); + remove(path2); + rmdir(dir1); + rmdir(dir2); + rmdir(TEST_LOGPATH); + + ctx = flb_create(); + flb_service_set(ctx, "Flush", "1", "Grace", "1", + "Log_Level", "error", NULL); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + out_ffd = flb_output(ctx, (char *) "file", NULL); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, "match", "test", NULL); + flb_output_set(ctx, out_ffd, "path", TEST_LOGPATH "/$proxy_name", NULL); + flb_output_set(ctx, out_ffd, "file", "$hostname.log", NULL); + flb_output_set(ctx, out_ffd, "fallback_path", TEST_LOGPATH, NULL); + flb_output_set(ctx, out_ffd, "fallback_file", "metrics.log", NULL); + flb_output_set(ctx, out_ffd, "mkdir", "true", NULL); + flb_output_set(ctx, out_ffd, "format", "plain", NULL); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + bytes = flb_lib_push(ctx, in_ffd, p, strlen(p)); + TEST_CHECK(bytes == strlen(p)); + + ret = wait_for_file(path1, 1, TEST_TIMEOUT); + TEST_CHECK(ret == 0); + ret = wait_for_file(path2, 1, TEST_TIMEOUT); + TEST_CHECK(ret == 0); + + flb_stop(ctx); + flb_destroy(ctx); + + remove(path1); + remove(path2); + rmdir(dir1); + rmdir(dir2); + rmdir(TEST_LOGPATH); +} + +void flb_test_file_dynamic_requires_fallback(void) +{ + int ret; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + ctx = flb_create(); + flb_service_set(ctx, "Flush", "1", "Grace", "1", "Log_Level", "off", NULL); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + out_ffd = flb_output(ctx, (char *) "file", NULL); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, "match", "test", NULL); + flb_output_set(ctx, out_ffd, "path", TEST_LOGPATH, NULL); + flb_output_set(ctx, out_ffd, "file", "$hostname.log", NULL); + + ret = flb_start(ctx); + TEST_CHECK(ret != 0); + + if (ret == 0) { + flb_stop(ctx); + } + flb_destroy(ctx); +} + +void flb_test_file_dynamic_missing_fallback(void) +{ + int ret; + int bytes; + char *p = JSON_DYNAMIC_MISSING; + char fallback[256]; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + snprintf(fallback, sizeof(fallback), "%s/fallback.log", TEST_LOGPATH); + remove(fallback); + rmdir(TEST_LOGPATH); + + ctx = flb_create(); + flb_service_set(ctx, "Flush", "1", "Grace", "1", "Log_Level", "error", NULL); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + out_ffd = flb_output(ctx, (char *) "file", NULL); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, "match", "test", NULL); + flb_output_set(ctx, out_ffd, "path", TEST_LOGPATH, NULL); + flb_output_set(ctx, out_ffd, "file", "$hostname.log", NULL); + flb_output_set(ctx, out_ffd, "on_missing_field", "fallback", NULL); + flb_output_set(ctx, out_ffd, "fallback_path", TEST_LOGPATH, NULL); + flb_output_set(ctx, out_ffd, "fallback_file", "fallback.log", NULL); + flb_output_set(ctx, out_ffd, "mkdir", "true", NULL); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + bytes = flb_lib_push(ctx, in_ffd, p, strlen(p)); + TEST_CHECK(bytes == strlen(p)); + ret = wait_for_file(fallback, 1, TEST_TIMEOUT); + TEST_CHECK(ret == 0); + + flb_stop(ctx); + flb_destroy(ctx); + + remove(fallback); + rmdir(TEST_LOGPATH); +} + +void flb_test_file_dynamic_unsafe_fallback(void) +{ + int ret; + int bytes; + char *p = JSON_DYNAMIC_UNSAFE; + char fallback[256]; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + snprintf(fallback, sizeof(fallback), "%s/fallback.log", TEST_LOGPATH); + remove(fallback); + remove("escape.log"); + rmdir(TEST_LOGPATH); + + ctx = flb_create(); + flb_service_set(ctx, "Flush", "1", "Grace", "1", "Log_Level", "error", NULL); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + out_ffd = flb_output(ctx, (char *) "file", NULL); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, "match", "test", NULL); + flb_output_set(ctx, out_ffd, "path", TEST_LOGPATH, NULL); + flb_output_set(ctx, out_ffd, "file", "$hostname", NULL); + flb_output_set(ctx, out_ffd, "on_missing_field", "fallback", NULL); + flb_output_set(ctx, out_ffd, "fallback_path", TEST_LOGPATH, NULL); + flb_output_set(ctx, out_ffd, "fallback_file", "fallback.log", NULL); + flb_output_set(ctx, out_ffd, "mkdir", "true", NULL); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + bytes = flb_lib_push(ctx, in_ffd, p, strlen(p)); + TEST_CHECK(bytes == strlen(p)); + ret = wait_for_file(fallback, 1, TEST_TIMEOUT); + TEST_CHECK(ret == 0); + + flb_stop(ctx); + flb_destroy(ctx); + + TEST_CHECK(access("escape.log", F_OK) != 0); + remove(fallback); + rmdir(TEST_LOGPATH); +} + +void flb_test_file_dynamic_windows_unsafe_fallback(void) +{ + int ret; + int bytes; + char *p = JSON_DYNAMIC_WINDOWS_UNSAFE; + char fallback[256]; + char unsafe[256]; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + snprintf(fallback, sizeof(fallback), "%s/fallback.log", TEST_LOGPATH); + snprintf(unsafe, sizeof(unsafe), "%s/host:one.log", TEST_LOGPATH); + remove(fallback); + remove(unsafe); + rmdir(TEST_LOGPATH); + + ctx = flb_create(); + flb_service_set(ctx, "Flush", "1", "Grace", "1", "Log_Level", "error", NULL); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + out_ffd = flb_output(ctx, (char *) "file", NULL); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, "match", "test", NULL); + flb_output_set(ctx, out_ffd, "path", TEST_LOGPATH, NULL); + flb_output_set(ctx, out_ffd, "file", "$hostname", NULL); + flb_output_set(ctx, out_ffd, "on_missing_field", "fallback", NULL); + flb_output_set(ctx, out_ffd, "fallback_path", TEST_LOGPATH, NULL); + flb_output_set(ctx, out_ffd, "fallback_file", "fallback.log", NULL); + flb_output_set(ctx, out_ffd, "mkdir", "true", NULL); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + bytes = flb_lib_push(ctx, in_ffd, p, strlen(p)); + TEST_CHECK(bytes == strlen(p)); + ret = wait_for_file(fallback, 1, TEST_TIMEOUT); + TEST_CHECK(ret == 0); + + flb_stop(ctx); + flb_destroy(ctx); + + TEST_CHECK(access(unsafe, F_OK) != 0); + remove(fallback); + remove(unsafe); + rmdir(TEST_LOGPATH); +} + +void flb_test_file_dynamic_limit_fallback(void) +{ + int ret; + int bytes; + char *p = JSON_DYNAMIC_LIMIT; + char first[256]; + char overflow[256]; + char second[256]; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + snprintf(first, sizeof(first), "%s/host1.log", TEST_LOGPATH); + snprintf(second, sizeof(second), "%s/host2.log", TEST_LOGPATH); + snprintf(overflow, sizeof(overflow), "%s/overflow.log", TEST_LOGPATH); + remove(first); + remove(second); + remove(overflow); + rmdir(TEST_LOGPATH); + + ctx = flb_create(); + flb_service_set(ctx, "Flush", "1", "Grace", "1", "Log_Level", "error", NULL); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + out_ffd = flb_output(ctx, (char *) "file", NULL); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, "match", "test", NULL); + flb_output_set(ctx, out_ffd, "path", TEST_LOGPATH, NULL); + flb_output_set(ctx, out_ffd, "file", "$hostname.log", NULL); + flb_output_set(ctx, out_ffd, "max_dynamic_files", "1", NULL); + flb_output_set(ctx, out_ffd, "on_limit_reached", "fallback", NULL); + flb_output_set(ctx, out_ffd, "fallback_path", TEST_LOGPATH, NULL); + flb_output_set(ctx, out_ffd, "fallback_file", "overflow.log", NULL); + flb_output_set(ctx, out_ffd, "mkdir", "true", NULL); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + bytes = flb_lib_push(ctx, in_ffd, p, strlen(p)); + TEST_CHECK(bytes == strlen(p)); + ret = wait_for_file(first, 1, TEST_TIMEOUT); + TEST_CHECK(ret == 0); + ret = wait_for_file(overflow, 1, TEST_TIMEOUT); + TEST_CHECK(ret == 0); + + flb_stop(ctx); + flb_destroy(ctx); + + TEST_CHECK(access(second, F_OK) != 0); + remove(first); + remove(second); + remove(overflow); + rmdir(TEST_LOGPATH); +} + +void flb_test_file_dynamic_path_traversal_fallback(void) +{ + int ret; + int bytes; + char *p = JSON_DYNAMIC_PATH_TRAVERSAL; + char fallback[256]; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + snprintf(fallback, sizeof(fallback), "%s/fallback.log", TEST_LOGPATH); + remove(fallback); + remove("unsafe.log"); + rmdir(TEST_LOGPATH); + + ctx = flb_create(); + flb_service_set(ctx, "Flush", "1", "Grace", "1", "Log_Level", "error", NULL); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + out_ffd = flb_output(ctx, (char *) "file", NULL); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, "match", "test", NULL); + flb_output_set(ctx, out_ffd, "path", TEST_LOGPATH "/$proxy_name", NULL); + flb_output_set(ctx, out_ffd, "file", "unsafe.log", NULL); + flb_output_set(ctx, out_ffd, "on_missing_field", "fallback", NULL); + flb_output_set(ctx, out_ffd, "fallback_path", TEST_LOGPATH, NULL); + flb_output_set(ctx, out_ffd, "fallback_file", "fallback.log", NULL); + flb_output_set(ctx, out_ffd, "mkdir", "true", NULL); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + bytes = flb_lib_push(ctx, in_ffd, p, strlen(p)); + TEST_CHECK(bytes == strlen(p)); + ret = wait_for_file(fallback, 1, TEST_TIMEOUT); + TEST_CHECK(ret == 0); + + flb_stop(ctx); + flb_destroy(ctx); + + TEST_CHECK(access("unsafe.log", F_OK) != 0); + remove(fallback); + rmdir(TEST_LOGPATH); +} + void flb_test_file_delim_csv(void) { int ret; diff --git a/tests/runtime/out_firehose.c b/tests/runtime/out_firehose.c index 3a2a9dda9f7..8abfe77aa4c 100644 --- a/tests/runtime/out_firehose.c +++ b/tests/runtime/out_firehose.c @@ -625,7 +625,11 @@ void flb_test_firehose_aggregation_custom_time_format(void) flb_output_set(ctx, out_ffd, "delivery_stream", "fluent", NULL); flb_output_set(ctx, out_ffd, "simple_aggregation", "On", NULL); flb_output_set(ctx, out_ffd, "time_key", "ts", NULL); +#ifdef FLB_SYSTEM_WINDOWS + flb_output_set(ctx, out_ffd, "time_key_format", "%Y%m%d", NULL); +#else flb_output_set(ctx, out_ffd, "time_key_format", "%s", NULL); +#endif flb_output_set(ctx, out_ffd, "Retry_Limit", "1", NULL); ret = flb_start(ctx); diff --git a/tests/runtime/out_gcs.c b/tests/runtime/out_gcs.c new file mode 100644 index 00000000000..f58a25bf262 --- /dev/null +++ b/tests/runtime/out_gcs.c @@ -0,0 +1,403 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ +#include +#include "flb_tests_runtime.h" +#include "../include/flb_tests_tmpdir.h" + +#include "data/td/json_td.h" +#include "../../plugins/out_gcs/gcs.h" +#include "../../plugins/out_gcs/gcs_store.h" + +/* Local 'test' credentials file */ +#define SERVICE_CREDENTIALS \ + FLB_TESTS_DATA_PATH "/data/stackdriver/stackdriver-credentials.json" +#define SERVICE_CREDENTIALS_EXTRA_FIELDS \ + FLB_TESTS_DATA_PATH "/data/gcs/gcs-credentials-extra-fields.json" +#define TEST_PRIVATE_KEY FLB_TESTS_DATA_PATH "/data/tls/private_key.pem" + +static char *create_test_store_directory(const char *postfix) +{ + char *store_dir; + + store_dir = flb_test_tmpdir_cat(postfix); + if (!store_dir) { + return NULL; + } + + if (!mkdtemp(store_dir)) { + flb_free(store_dir); + return NULL; + } + + return store_dir; +} + +void flb_test_gcs_jwt_signing(void) +{ + int ret; + char *private_key; + char *jwt; + char *first_separator; + char *second_separator; + size_t jwt_size; + + private_key = mk_file_to_buffer(TEST_PRIVATE_KEY); + TEST_CHECK(private_key != NULL); + if (!private_key) { + return; + } + + ret = gcs_jwt_encode(NULL, "{\"sub\":\"fluent-bit\"}", private_key, + &jwt, &jwt_size); + TEST_CHECK(ret == 0); + if (ret == 0) { + TEST_CHECK(jwt_size == flb_sds_len(jwt)); + first_separator = strchr(jwt, '.'); + TEST_CHECK(first_separator != NULL); + second_separator = first_separator ? strchr(first_separator + 1, '.') : NULL; + TEST_CHECK(second_separator != NULL); + TEST_CHECK(second_separator && second_separator[1] != '\0'); + flb_sds_destroy(jwt); + } + + flb_free(private_key); +} + +void flb_test_gcs_uri_encode_object_name(void) +{ + flb_sds_t encoded; + const char *object_name = "logs/a+b&c=d?#%/space key"; + const char *expected = "logs%2Fa%2Bb%26c%3Dd%3F%23%25%2Fspace%20key"; + + encoded = gcs_uri_encode_object_name(object_name, strlen(object_name)); + TEST_CHECK(encoded != NULL); + if (encoded) { + TEST_CHECK(strcmp(encoded, expected) == 0); + flb_sds_destroy(encoded); + } +} + +void flb_test_gcs_upload_success(void) +{ + int ret; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + char *call_count_str; + int call_count; + char *store_dir; + + store_dir = create_test_store_directory("/flb-gcs-test-success-XXXXXX"); + TEST_CHECK(store_dir != NULL); + if (!store_dir) { + return; + } + + setenv("FLB_GCS_PLUGIN_UNDER_TEST", "true", 1); + + ctx = flb_create(); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, "tag", "test+a&b=c#d", NULL); + + out_ffd = flb_output(ctx, (char *) "gcs", NULL); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, "match", "*", NULL); + flb_output_set(ctx, out_ffd, "bucket", "fluent", NULL); + flb_output_set(ctx, out_ffd, "google_service_credentials", SERVICE_CREDENTIALS, NULL); + flb_output_set(ctx, out_ffd, "upload_timeout", "3s", NULL); + flb_output_set(ctx, out_ffd, "store_dir", store_dir, NULL); + flb_output_set(ctx, out_ffd, "gcs_key_format", "logs/$TAG", NULL); + flb_output_set(ctx, out_ffd, "static_file_path", "true", NULL); + flb_output_set(ctx, out_ffd, "compression", "gzip", NULL); + flb_output_set(ctx, out_ffd, "canned_acl", "public-read", NULL); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + flb_lib_push(ctx, in_ffd, (char *) JSON_TD, (int) sizeof(JSON_TD) - 1); + sleep(5); + + call_count_str = getenv("TEST_GCS_UploadObject_CALL_COUNT"); + call_count = call_count_str ? atoi(call_count_str) : 0; + TEST_CHECK_(call_count == 1, + "Expected 1 UploadObject call, got %d", call_count); + TEST_CHECK_(getenv("TEST_GCS_LAST_URI") != NULL, + "Expected the mock upload URI to be captured"); + if (getenv("TEST_GCS_LAST_URI")) { + TEST_CHECK(strcmp(getenv("TEST_GCS_LAST_URI"), + "/upload/storage/v1/b/fluent/o?uploadType=media&" + "name=logs%2Ftest%2Ba%26b%3Dc%23d&contentEncoding=gzip&" + "predefinedAcl=publicRead") == 0); + } + TEST_CHECK_(getenv("TEST_GCS_LAST_BODY_GZIP") != NULL, + "Expected the mock upload body encoding to be captured"); + if (getenv("TEST_GCS_LAST_BODY_GZIP")) { + TEST_CHECK(strcmp(getenv("TEST_GCS_LAST_BODY_GZIP"), "true") == 0); + } + + flb_stop(ctx); + flb_destroy(ctx); + + unsetenv("FLB_GCS_PLUGIN_UNDER_TEST"); + unsetenv("TEST_GCS_UploadObject_CALL_COUNT"); + unsetenv("TEST_GCS_LAST_URI"); + unsetenv("TEST_GCS_LAST_BODY_GZIP"); + flb_free(store_dir); +} + +void flb_test_gcs_rejects_invalid_configuration(void) +{ + int ret; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + char *store_dir; + + store_dir = create_test_store_directory("/flb-gcs-test-invalid-config-XXXXXX"); + TEST_CHECK(store_dir != NULL); + if (!store_dir) { + return; + } + + ctx = flb_create(); + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + out_ffd = flb_output(ctx, (char *) "gcs", NULL); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, "match", "*", NULL); + flb_output_set(ctx, out_ffd, "bucket", "fluent", NULL); + flb_output_set(ctx, out_ffd, "google_service_credentials", SERVICE_CREDENTIALS, NULL); + flb_output_set(ctx, out_ffd, "store_dir", store_dir, NULL); + flb_output_set(ctx, out_ffd, "send_content_md5", "not-a-boolean", NULL); + + ret = flb_start(ctx); + TEST_CHECK(ret != 0); + flb_destroy(ctx); + flb_free(store_dir); +} + +void flb_test_gcs_rejects_invalid_compression(void) +{ + int ret; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + char *store_dir; + + store_dir = create_test_store_directory("/flb-gcs-test-invalid-compression-XXXXXX"); + TEST_CHECK(store_dir != NULL); + if (!store_dir) { + return; + } + + ctx = flb_create(); + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + out_ffd = flb_output(ctx, (char *) "gcs", NULL); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, "match", "*", NULL); + flb_output_set(ctx, out_ffd, "bucket", "fluent", NULL); + flb_output_set(ctx, out_ffd, "google_service_credentials", SERVICE_CREDENTIALS, NULL); + flb_output_set(ctx, out_ffd, "store_dir", store_dir, NULL); + flb_output_set(ctx, out_ffd, "compression", "zstd", NULL); + + ret = flb_start(ctx); + TEST_CHECK(ret != 0); + flb_destroy(ctx); + flb_free(store_dir); +} + +void flb_test_gcs_accepts_extra_credential_fields(void) +{ + int ret; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + char *store_dir; + + store_dir = create_test_store_directory("/flb-gcs-test-extra-credentials-XXXXXX"); + TEST_CHECK(store_dir != NULL); + if (!store_dir) { + return; + } + + ctx = flb_create(); + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + out_ffd = flb_output(ctx, (char *) "gcs", NULL); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, "match", "*", NULL); + flb_output_set(ctx, out_ffd, "bucket", "fluent", NULL); + flb_output_set(ctx, out_ffd, "google_service_credentials", + SERVICE_CREDENTIALS_EXTRA_FIELDS, NULL); + flb_output_set(ctx, out_ffd, "store_dir", store_dir, NULL); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + if (ret == 0) { + flb_stop(ctx); + } + flb_destroy(ctx); + flb_free(store_dir); +} + +void flb_test_gcs_upload_error(void) +{ + int ret; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + char *call_count_str; + int call_count; + char *store_dir; + + store_dir = create_test_store_directory("/flb-gcs-test-error-XXXXXX"); + TEST_CHECK(store_dir != NULL); + if (!store_dir) { + return; + } + + setenv("FLB_GCS_PLUGIN_UNDER_TEST", "true", 1); + setenv("TEST_GCS_UPLOAD_ERROR", "true", 1); + + ctx = flb_create(); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + out_ffd = flb_output(ctx, (char *) "gcs", NULL); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, "match", "*", NULL); + flb_output_set(ctx, out_ffd, "bucket", "fluent", NULL); + flb_output_set(ctx, out_ffd, "google_service_credentials", SERVICE_CREDENTIALS, NULL); + flb_output_set(ctx, out_ffd, "upload_timeout", "3s", NULL); + flb_output_set(ctx, out_ffd, "store_dir", store_dir, NULL); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + flb_lib_push(ctx, in_ffd, (char *) JSON_TD, (int) sizeof(JSON_TD) - 1); + sleep(6); + + call_count_str = getenv("TEST_GCS_UploadObject_CALL_COUNT"); + call_count = call_count_str ? atoi(call_count_str) : 0; + TEST_CHECK_(call_count >= 1, + "Expected >=1 UploadObject calls, got %d", call_count); + + flb_stop(ctx); + flb_destroy(ctx); + + unsetenv("FLB_GCS_PLUGIN_UNDER_TEST"); + unsetenv("TEST_GCS_UPLOAD_ERROR"); + unsetenv("TEST_GCS_UploadObject_CALL_COUNT"); + unsetenv("TEST_GCS_LAST_URI"); + unsetenv("TEST_GCS_LAST_BODY_GZIP"); + flb_free(store_dir); +} + +void flb_test_gcs_shutdown_preserves_pending_upload(void) +{ + int ret; + int in_ffd; + int out_ffd; + int call_count; + char *call_count_str; + char *store_dir; + char body[] = "{\"message\":\"pending\"}\n"; + flb_ctx_t *ctx; + struct flb_gcs *gcs_ctx; + struct gcs_file *chunk; + struct upload_queue *entry; + struct flb_output_instance *out_ins; + + store_dir = create_test_store_directory("/flb-gcs-test-shutdown-XXXXXX"); + TEST_CHECK(store_dir != NULL); + if (!store_dir) { + return; + } + + setenv("FLB_GCS_PLUGIN_UNDER_TEST", "true", 1); + unsetenv("TEST_GCS_UploadObject_CALL_COUNT"); + + ctx = flb_create(); + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + out_ffd = flb_output(ctx, (char *) "gcs", NULL); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, "match", "*", NULL); + flb_output_set(ctx, out_ffd, "bucket", "fluent", NULL); + flb_output_set(ctx, out_ffd, "google_service_credentials", SERVICE_CREDENTIALS, NULL); + flb_output_set(ctx, out_ffd, "store_dir", store_dir, NULL); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + if (ret != 0) { + flb_destroy(ctx); + unsetenv("FLB_GCS_PLUGIN_UNDER_TEST"); + flb_free(store_dir); + return; + } + + out_ins = flb_output_get_instance(ctx->config, out_ffd); + TEST_CHECK(out_ins != NULL); + gcs_ctx = out_ins ? out_ins->context : NULL; + TEST_CHECK(gcs_ctx != NULL); + + ret = gcs_store_buffer_put(gcs_ctx, NULL, "test", 4, body, sizeof(body) - 1); + TEST_CHECK(ret == 0); + chunk = gcs_store_file_get(gcs_ctx, "test", 4); + TEST_CHECK(chunk != NULL); + + entry = flb_calloc(1, sizeof(struct upload_queue)); + TEST_CHECK(entry != NULL); + if (entry && chunk) { + entry->tag = flb_strdup("test"); + TEST_CHECK(entry->tag != NULL); + if (entry->tag) { + entry->tag_len = 4; + entry->upload_file = chunk; + entry->upload_time = 0; + mk_list_add(&entry->_head, &gcs_ctx->upload_queue); + } + else { + flb_free(entry); + } + } + + flb_stop(ctx); + flb_destroy(ctx); + + call_count_str = getenv("TEST_GCS_UploadObject_CALL_COUNT"); + call_count = call_count_str ? atoi(call_count_str) : 0; + TEST_CHECK_(call_count == 0, + "Expected shutdown to preserve pending data without uploading, got %d call(s)", + call_count); + + unsetenv("FLB_GCS_PLUGIN_UNDER_TEST"); + unsetenv("TEST_GCS_UploadObject_CALL_COUNT"); + unsetenv("TEST_GCS_LAST_URI"); + unsetenv("TEST_GCS_LAST_BODY_GZIP"); + flb_free(store_dir); +} + +TEST_LIST = { + {"jwt_signing", flb_test_gcs_jwt_signing}, + {"uri_encode_object_name", flb_test_gcs_uri_encode_object_name}, + {"upload_success", flb_test_gcs_upload_success}, + {"rejects_invalid_configuration", flb_test_gcs_rejects_invalid_configuration}, + {"rejects_invalid_compression", flb_test_gcs_rejects_invalid_compression}, + {"accepts_extra_credential_fields", flb_test_gcs_accepts_extra_credential_fields}, + {"upload_error", flb_test_gcs_upload_error}, + {"shutdown_preserves_pending_upload", flb_test_gcs_shutdown_preserves_pending_upload}, + {NULL, NULL} +}; diff --git a/tests/runtime/out_http.c b/tests/runtime/out_http.c index 0a7bf75c0ac..8a8c2d592e8 100644 --- a/tests/runtime/out_http.c +++ b/tests/runtime/out_http.c @@ -36,9 +36,12 @@ struct test_ctx { pthread_mutex_t result_mutex = PTHREAD_MUTEX_INITIALIZER; int num_output = 0; +static const char *callback_error = NULL; + static int get_output_num() { int ret; + pthread_mutex_lock(&result_mutex); ret = num_output; pthread_mutex_unlock(&result_mutex); @@ -46,16 +49,48 @@ static int get_output_num() return ret; } -static void set_output_num(int num) +static void increment_output_num() +{ + pthread_mutex_lock(&result_mutex); + num_output++; + pthread_mutex_unlock(&result_mutex); +} + +static void add_output_num(int num) +{ + pthread_mutex_lock(&result_mutex); + num_output += num; + pthread_mutex_unlock(&result_mutex); +} + +static void set_callback_error(const char *message) { pthread_mutex_lock(&result_mutex); - num_output = num; + if (callback_error == NULL) { + callback_error = message; + } pthread_mutex_unlock(&result_mutex); } static void clear_output_num() { - set_output_num(0); + pthread_mutex_lock(&result_mutex); + num_output = 0; + callback_error = NULL; + pthread_mutex_unlock(&result_mutex); +} + +static void check_callback_error() +{ + const char *message; + + pthread_mutex_lock(&result_mutex); + message = callback_error; + pthread_mutex_unlock(&result_mutex); + + if (!TEST_CHECK(message == NULL)) { + TEST_MSG("%s", message); + } } struct str_list { @@ -69,38 +104,32 @@ static void cb_check_str_list(void *ctx, int ffd, int res_ret, { char *p; flb_sds_t out_line = res_data; - int num = get_output_num(); size_t i; struct str_list *l = (struct str_list *)data; - if (!TEST_CHECK(res_data != NULL)) { - TEST_MSG("res_data is NULL"); + if (res_data == NULL) { + set_callback_error("formatter returned no output"); return; } - if (!TEST_CHECK(l != NULL)) { - TEST_MSG("l is NULL"); + if (l == NULL) { + set_callback_error("formatter callback data is NULL"); flb_sds_destroy(out_line); return; } - if(!TEST_CHECK(res_ret == 0)) { - TEST_MSG("callback ret=%d", res_ret); - } - if (!TEST_CHECK(res_data != NULL)) { - TEST_MSG("res_data is NULL"); - flb_sds_destroy(out_line); - return; + if (res_ret != 0) { + set_callback_error("formatter returned an error"); } - for (i=0; isize; i++) { + for (i = 0; i < l->size; i++) { p = strstr(out_line, l->lists[i]); - if (!TEST_CHECK(p != NULL)) { - TEST_MSG(" Got :%s\n expect:%s", out_line, l->lists[i]); + if (p == NULL) { + set_callback_error("formatter output did not contain an expected string"); } } - set_output_num(num+1); + increment_output_num(); flb_sds_destroy(out_line); } @@ -131,7 +160,7 @@ static int msgpack_strncmp(char* str, size_t str_len, msgpack_object obj) case MSGPACK_OBJECT_NEGATIVE_INTEGER: { long long val = strtoll(str, NULL, 10); - if (val == (unsigned long)obj.via.i64) { + if (val == obj.via.i64) { ret = 0; } } @@ -177,38 +206,48 @@ static void cb_check_msgpack_kv(void *ctx, int ffd, int res_ret, int i_map; int map_size; int i_list; - int num = get_output_num(); + int matches = 0; + msgpack_unpack_return unpack_result = MSGPACK_UNPACK_CONTINUE; - if (!TEST_CHECK(res_data != NULL)) { - TEST_MSG("res_data is NULL"); + if (res_data == NULL) { + set_callback_error("formatter returned no output"); return; } - if (!TEST_CHECK(data != NULL)) { - flb_error("data is NULL"); + if (data == NULL) { + set_callback_error("formatter callback data is NULL"); return; } + if (res_ret != 0) { + set_callback_error("formatter returned an error"); + } + /* Iterate each item array and apply rules */ msgpack_unpacked_init(&result); - while (msgpack_unpack_next(&result, res_data, res_size, &off) == MSGPACK_UNPACK_SUCCESS) { + while (off < res_size) { + unpack_result = msgpack_unpack_next(&result, res_data, res_size, &off); + if (unpack_result != MSGPACK_UNPACK_SUCCESS) { + break; + } + obj = result.data; /* msgpack_object_print(stdout, obj); */ if (obj.type != MSGPACK_OBJECT_ARRAY || obj.via.array.size != 2) { - flb_error("array error. type = %d", obj.type); + set_callback_error("formatter output contained an invalid record"); continue; } obj = obj.via.array.ptr[1]; if (obj.type != MSGPACK_OBJECT_MAP) { - flb_error("map error. type = %d", obj.type); + set_callback_error("formatter output record was not a map"); continue; } map_size = obj.via.map.size; for (i_map=0; i_mapsize/2; i_list++) { @@ -216,16 +255,30 @@ static void cb_check_msgpack_kv(void *ctx, int ffd, int res_ret, obj.via.map.ptr[i_map].key) == 0 && msgpack_strncmp(l->lists[i_list*2+1], strlen(l->lists[i_list*2+1]), obj.via.map.ptr[i_map].val) == 0) { - num++; + matches++; } } } } - set_output_num(num); - msgpack_unpacked_destroy(&result); + if (unpack_result == MSGPACK_UNPACK_PARSE_ERROR) { + set_callback_error("formatter output contained invalid MessagePack"); + } + else if (unpack_result == MSGPACK_UNPACK_NOMEM_ERROR) { + set_callback_error("could not unpack formatter output"); + } + else if (unpack_result == MSGPACK_UNPACK_CONTINUE && res_size != 0) { + set_callback_error("formatter output contained incomplete MessagePack"); + } + else if (unpack_result != MSGPACK_UNPACK_SUCCESS && + unpack_result != MSGPACK_UNPACK_CONTINUE) { + set_callback_error("formatter output returned an unexpected unpack result"); + } + else { + add_output_num(matches); + } - return ; + msgpack_unpacked_destroy(&result); } static struct test_ctx *test_ctx_create() @@ -267,12 +320,14 @@ static void test_ctx_destroy(struct test_ctx *ctx) sleep(1); flb_stop(ctx->flb); + check_callback_error(); flb_destroy(ctx->flb); flb_free(ctx); } void flb_test_format_msgpack() { + int attempts; struct test_ctx *ctx; int ret; int num; @@ -313,8 +368,12 @@ void flb_test_format_msgpack() ret = flb_lib_push(ctx->flb, ctx->i_ffd, (char *) buf1, size1); TEST_CHECK(ret >= 0); - /* waiting to flush */ - flb_time_msleep(500); + for (attempts = 0; attempts < 50; attempts++) { + if (get_output_num() == expected.size / 2) { + break; + } + flb_time_msleep(100); + } num = get_output_num(); if (!TEST_CHECK(num == expected.size / 2)) { @@ -1042,11 +1101,8 @@ void flb_test_json_date_format_java_sql_timestamp() int callback_test(void* data, size_t size, void* cb_data) { - int num; - if (size > 0) { - num = get_output_num(); - set_output_num(num+1); + increment_output_num(); } return 0; } diff --git a/tests/runtime/out_logdna.c b/tests/runtime/out_logdna.c index 6951c8a43aa..5a530fd1017 100644 --- a/tests/runtime/out_logdna.c +++ b/tests/runtime/out_logdna.c @@ -18,9 +18,9 @@ */ #include +#include #include #include -#include #include "flb_tests_runtime.h" /* Thread-safe callback invocation tracking */ diff --git a/tests/runtime/out_loki.c b/tests/runtime/out_loki.c index d5885b72302..287f8e97bfa 100644 --- a/tests/runtime/out_loki.c +++ b/tests/runtime/out_loki.c @@ -48,8 +48,11 @@ static int tenant_policy_fail_tenant_a = FLB_FALSE; struct tenant_policy_server { struct flb_http_server server; struct flb_net_setup net_setup; + struct flb_config *config; struct mk_event_loop *event_loop; pthread_t thread; + int server_initialized; + int server_started; int thread_started; int stop; }; @@ -243,9 +246,40 @@ static void *tenant_policy_server_loop(void *data) return NULL; } +static void stop_tenant_policy_server(struct tenant_policy_server *mock) +{ + if (mock->thread_started == FLB_TRUE) { + pthread_mutex_lock(&result_mutex); + mock->stop = FLB_TRUE; + pthread_mutex_unlock(&result_mutex); + + pthread_join(mock->thread, NULL); + mock->thread_started = FLB_FALSE; + } + + if (mock->server_started == FLB_TRUE) { + flb_http_server_stop(&mock->server); + mock->server_started = FLB_FALSE; + } + + if (mock->server_initialized == FLB_TRUE) { + flb_http_server_destroy(&mock->server); + mock->server_initialized = FLB_FALSE; + } + + if (mock->event_loop != NULL) { + mk_event_loop_destroy(mock->event_loop); + mock->event_loop = NULL; + } + + if (mock->config != NULL) { + flb_config_exit(mock->config); + mock->config = NULL; + } +} + static int start_tenant_policy_server(struct tenant_policy_server *mock, - const char *port, - struct flb_config *config) + const char *port) { int ret; struct flb_http_server_options options; @@ -255,8 +289,14 @@ static int start_tenant_policy_server(struct tenant_policy_server *mock, flb_http_server_options_init(&options); flb_net_setup_init(&mock->net_setup); + mock->config = flb_config_init(); + if (mock->config == NULL) { + return -1; + } + mock->event_loop = mk_event_loop_create(256); if (mock->event_loop == NULL) { + stop_tenant_policy_server(mock); return -1; } @@ -267,32 +307,27 @@ static int start_tenant_policy_server(struct tenant_policy_server *mock, options.networking_flags = 0; options.networking_setup = &mock->net_setup; options.event_loop = mock->event_loop; - options.system_context = config; + options.system_context = mock->config; options.use_caller_event_loop = FLB_TRUE; ret = flb_http_server_init_with_options(&mock->server, &options); if (ret != 0) { - mk_event_loop_destroy(mock->event_loop); - mock->event_loop = NULL; + stop_tenant_policy_server(mock); return -1; } + mock->server_initialized = FLB_TRUE; ret = flb_http_server_start(&mock->server); if (ret != 0) { - flb_http_server_destroy(&mock->server); - mk_event_loop_destroy(mock->event_loop); - mock->event_loop = NULL; + stop_tenant_policy_server(mock); return -1; } + mock->server_started = FLB_TRUE; mock->stop = FLB_FALSE; - mock->thread_started = FLB_FALSE; ret = pthread_create(&mock->thread, NULL, tenant_policy_server_loop, mock); if (ret != 0) { - flb_http_server_stop(&mock->server); - flb_http_server_destroy(&mock->server); - mk_event_loop_destroy(mock->event_loop); - mock->event_loop = NULL; + stop_tenant_policy_server(mock); return -1; } mock->thread_started = FLB_TRUE; @@ -302,25 +337,6 @@ static int start_tenant_policy_server(struct tenant_policy_server *mock, return 0; } -static void stop_tenant_policy_server(struct tenant_policy_server *mock) -{ - if (mock->event_loop == NULL) { - return; - } - - pthread_mutex_lock(&result_mutex); - mock->stop = FLB_TRUE; - pthread_mutex_unlock(&result_mutex); - - if (mock->thread_started == FLB_TRUE) { - pthread_join(mock->thread, NULL); - } - flb_http_server_stop(&mock->server); - flb_http_server_destroy(&mock->server); - mk_event_loop_destroy(mock->event_loop); - mock->event_loop = NULL; -} - #define JSON_BASIC "[12345678, {\"key\":\"value\"}]" static void cb_check_basic(void *ctx, int ffd, int res_ret, void *res_data, size_t res_size, @@ -1035,8 +1051,7 @@ void flb_test_tenant_id_key_splits_requests() ret = flb_start(ctx); TEST_CHECK(ret == 0); - ret = start_tenant_policy_server(&mock_server, LOKI_TENANT_SPLIT_PORT, - ctx->config); + ret = start_tenant_policy_server(&mock_server, LOKI_TENANT_SPLIT_PORT); TEST_CHECK(ret == 0); ret = flb_lib_push(ctx, in_ffd, tenant_a, strlen(tenant_a)); @@ -1109,7 +1124,7 @@ static void run_tenant_id_key_partial_handling(char *mode, ret = flb_start(ctx); TEST_CHECK(ret == 0); - ret = start_tenant_policy_server(&mock_server, port, ctx->config); + ret = start_tenant_policy_server(&mock_server, port); TEST_CHECK(ret == 0); ret = flb_lib_push(ctx, in_ffd, tenant_a, strlen(tenant_a)); diff --git a/tests/runtime/out_s3.c b/tests/runtime/out_s3.c index 61cfac43e1c..0b7f221e53d 100644 --- a/tests/runtime/out_s3.c +++ b/tests/runtime/out_s3.c @@ -1,5 +1,6 @@ /* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ #include +#include #include "flb_tests_runtime.h" #include "../include/flb_tests_tmpdir.h" #include @@ -14,6 +15,10 @@ /* Test data */ #include "data/td/json_td.h" /* JSON_TD */ +#define S3_TEST_UPLOAD_TIMEOUT "1s" +#define S3_TEST_WAIT_STEP_MS 10 +#define S3_TEST_WAIT_TIMEOUT_MS 5000 + /* not a real error code, but tests that the code can respond to any error */ #define ERROR_ACCESS_DENIED "\ \ @@ -89,6 +94,55 @@ static int count_files_recursive(const char *path) #endif } +static int get_s3_call_count(const char *api) +{ + char name[64]; + char *value; + + snprintf(name, sizeof(name), "TEST_%s_CALL_COUNT", api); + value = getenv(name); + + return value ? atoi(value) : 0; +} + +static void wait_for_s3_call_count(const char *api, int expected) +{ + uint64_t elapsed_ms; + struct flb_time start_time; + struct flb_time end_time; + struct flb_time diff_time; + + elapsed_ms = 0; + flb_time_get(&start_time); + + while (get_s3_call_count(api) < expected && + elapsed_ms < S3_TEST_WAIT_TIMEOUT_MS) { + flb_time_msleep(S3_TEST_WAIT_STEP_MS); + flb_time_get(&end_time); + flb_time_diff(&end_time, &start_time, &diff_time); + elapsed_ms = flb_time_to_nanosec(&diff_time) / 1000000; + } +} + +static void wait_for_file_count(const char *path, int expected) +{ + uint64_t elapsed_ms; + struct flb_time start_time; + struct flb_time end_time; + struct flb_time diff_time; + + elapsed_ms = 0; + flb_time_get(&start_time); + + while (count_files_recursive(path) < expected && + elapsed_ms < S3_TEST_WAIT_TIMEOUT_MS) { + flb_time_msleep(S3_TEST_WAIT_STEP_MS); + flb_time_get(&end_time); + flb_time_diff(&end_time, &start_time, &diff_time); + elapsed_ms = flb_time_to_nanosec(&diff_time) / 1000000; + } +} + static int ensure_test_directory(const char *path) { #ifdef FLB_SYSTEM_WINDOWS @@ -119,6 +173,23 @@ static int ensure_test_directory(const char *path) #endif } +static char *create_test_store_directory(const char *postfix) +{ + char *store_dir; + + store_dir = flb_test_tmpdir_cat(postfix); + if (store_dir == NULL) { + return NULL; + } + + if (mkdtemp(store_dir) == NULL) { + flb_free(store_dir); + return NULL; + } + + return store_dir; +} + void flb_test_s3_multipart_success(void) { int ret; @@ -127,9 +198,13 @@ void flb_test_s3_multipart_success(void) int out_ffd; char *call_count_str; int call_count; - char store_dir[] = "/tmp/flb-s3-test-multipart-XXXXXX"; + char *store_dir; - TEST_CHECK(mkdtemp(store_dir) != NULL); + store_dir = create_test_store_directory("/flb-s3-test-multipart-XXXXXX"); + TEST_CHECK(store_dir != NULL); + if (store_dir == NULL) { + return; + } /* mocks calls- signals that we are in test mode */ setenv("FLB_S3_PLUGIN_UNDER_TEST", "true", 1); @@ -145,7 +220,7 @@ void flb_test_s3_multipart_success(void) flb_output_set(ctx, out_ffd,"match", "*", NULL); flb_output_set(ctx, out_ffd,"region", "us-west-2", NULL); flb_output_set(ctx, out_ffd,"bucket", "fluent", NULL); - flb_output_set(ctx, out_ffd,"upload_timeout", "6s", NULL); + flb_output_set(ctx, out_ffd,"upload_timeout", S3_TEST_UPLOAD_TIMEOUT, NULL); flb_output_set(ctx, out_ffd,"store_dir", store_dir, NULL); flb_output_set(ctx, out_ffd,"Retry_Limit", "1", NULL); @@ -154,7 +229,7 @@ void flb_test_s3_multipart_success(void) flb_lib_push(ctx, in_ffd, (char *) JSON_TD , (int) sizeof(JSON_TD) - 1); - sleep(10); + wait_for_s3_call_count("CompleteMultipartUpload", 1); call_count_str = getenv("TEST_CompleteMultipartUpload_CALL_COUNT"); call_count = call_count_str ? atoi(call_count_str) : 0; @@ -168,6 +243,7 @@ void flb_test_s3_multipart_success(void) unsetenv("TEST_UploadPart_CALL_COUNT"); unsetenv("TEST_CompleteMultipartUpload_CALL_COUNT"); unsetenv("TEST_PutObject_CALL_COUNT"); + flb_free(store_dir); } void flb_test_s3_putobject_success(void) @@ -195,7 +271,7 @@ void flb_test_s3_putobject_success(void) flb_output_set(ctx, out_ffd,"bucket", "fluent", NULL); flb_output_set(ctx, out_ffd,"use_put_object", "true", NULL); flb_output_set(ctx, out_ffd,"total_file_size", "5M", NULL); - flb_output_set(ctx, out_ffd,"upload_timeout", "6s", NULL); + flb_output_set(ctx, out_ffd,"upload_timeout", S3_TEST_UPLOAD_TIMEOUT, NULL); flb_output_set(ctx, out_ffd,"Retry_Limit", "1", NULL); ret = flb_start(ctx); @@ -204,7 +280,7 @@ void flb_test_s3_putobject_success(void) flb_lib_push(ctx, in_ffd, (char *) JSON_TD , (int) sizeof(JSON_TD) - 1); - sleep(10); + wait_for_s3_call_count("PutObject", 1); call_count_str = getenv("TEST_PutObject_CALL_COUNT"); call_count = call_count_str ? atoi(call_count_str) : 0; @@ -225,7 +301,13 @@ void flb_test_s3_putobject_error(void) int out_ffd; char *call_count_str; int call_count; - char store_dir[] = "/tmp/flb-s3-test-putobj-XXXXXX"; + char *store_dir; + + store_dir = create_test_store_directory("/flb-s3-test-putobj-XXXXXX"); + TEST_CHECK(store_dir != NULL); + if (store_dir == NULL) { + return; + } /* mocks calls- signals that we are in test mode */ setenv("FLB_S3_PLUGIN_UNDER_TEST", "true", 1); @@ -244,7 +326,7 @@ void flb_test_s3_putobject_error(void) flb_output_set(ctx, out_ffd,"bucket", "fluent", NULL); flb_output_set(ctx, out_ffd,"use_put_object", "true", NULL); flb_output_set(ctx, out_ffd,"total_file_size", "5M", NULL); - flb_output_set(ctx, out_ffd,"upload_timeout", "6s", NULL); + flb_output_set(ctx, out_ffd,"upload_timeout", S3_TEST_UPLOAD_TIMEOUT, NULL); flb_output_set(ctx, out_ffd,"store_dir", store_dir, NULL); flb_output_set(ctx, out_ffd,"Retry_Limit", "1", NULL); @@ -254,7 +336,7 @@ void flb_test_s3_putobject_error(void) flb_lib_push(ctx, in_ffd, (char *) JSON_TD , (int) sizeof(JSON_TD) - 1); - sleep(10); + wait_for_s3_call_count("PutObject", 1); call_count_str = getenv("TEST_PutObject_CALL_COUNT"); call_count = call_count_str ? atoi(call_count_str) : 0; @@ -266,6 +348,7 @@ void flb_test_s3_putobject_error(void) unsetenv("FLB_S3_PLUGIN_UNDER_TEST"); unsetenv("TEST_PUT_OBJECT_ERROR"); unsetenv("TEST_PutObject_CALL_COUNT"); + flb_free(store_dir); } @@ -277,9 +360,13 @@ void flb_test_s3_create_upload_error(void) int out_ffd; char *call_count_str; int call_count; - char store_dir[] = "/tmp/flb-s3-test-XXXXXX"; + char *store_dir; - TEST_CHECK(mkdtemp(store_dir) != NULL); + store_dir = create_test_store_directory("/flb-s3-test-XXXXXX"); + TEST_CHECK(store_dir != NULL); + if (store_dir == NULL) { + return; + } /* mocks calls- signals that we are in test mode */ setenv("FLB_S3_PLUGIN_UNDER_TEST", "true", 1); @@ -296,7 +383,7 @@ void flb_test_s3_create_upload_error(void) flb_output_set(ctx, out_ffd,"match", "*", NULL); flb_output_set(ctx, out_ffd,"region", "us-west-2", NULL); flb_output_set(ctx, out_ffd,"bucket", "fluent", NULL); - flb_output_set(ctx, out_ffd,"upload_timeout", "6s", NULL); + flb_output_set(ctx, out_ffd,"upload_timeout", S3_TEST_UPLOAD_TIMEOUT, NULL); flb_output_set(ctx, out_ffd,"store_dir", store_dir, NULL); flb_output_set(ctx, out_ffd,"Retry_Limit", "1", NULL); @@ -305,7 +392,7 @@ void flb_test_s3_create_upload_error(void) flb_lib_push(ctx, in_ffd, (char *) JSON_TD , (int) sizeof(JSON_TD) - 1); - sleep(10); + wait_for_s3_call_count("CreateMultipartUpload", 1); call_count_str = getenv("TEST_CreateMultipartUpload_CALL_COUNT"); call_count = call_count_str ? atoi(call_count_str) : 0; @@ -325,6 +412,7 @@ void flb_test_s3_create_upload_error(void) unsetenv("TEST_UploadPart_CALL_COUNT"); unsetenv("TEST_CompleteMultipartUpload_CALL_COUNT"); unsetenv("TEST_PutObject_CALL_COUNT"); + flb_free(store_dir); } void flb_test_s3_upload_part_error(void) @@ -335,9 +423,13 @@ void flb_test_s3_upload_part_error(void) int out_ffd; char *call_count_str; int call_count; - char store_dir[] = "/tmp/flb-s3-test-part-err-XXXXXX"; + char *store_dir; - TEST_CHECK(mkdtemp(store_dir) != NULL); + store_dir = create_test_store_directory("/flb-s3-test-part-err-XXXXXX"); + TEST_CHECK(store_dir != NULL); + if (store_dir == NULL) { + return; + } /* mocks calls- signals that we are in test mode */ setenv("FLB_S3_PLUGIN_UNDER_TEST", "true", 1); @@ -354,7 +446,7 @@ void flb_test_s3_upload_part_error(void) flb_output_set(ctx, out_ffd,"match", "*", NULL); flb_output_set(ctx, out_ffd,"region", "us-west-2", NULL); flb_output_set(ctx, out_ffd,"bucket", "fluent", NULL); - flb_output_set(ctx, out_ffd,"upload_timeout", "6s", NULL); + flb_output_set(ctx, out_ffd,"upload_timeout", S3_TEST_UPLOAD_TIMEOUT, NULL); flb_output_set(ctx, out_ffd,"store_dir", store_dir, NULL); flb_output_set(ctx, out_ffd,"Retry_Limit", "1", NULL); @@ -363,7 +455,7 @@ void flb_test_s3_upload_part_error(void) flb_lib_push(ctx, in_ffd, (char *) JSON_TD , (int) sizeof(JSON_TD) - 1); - sleep(10); + wait_for_s3_call_count("UploadPart", 1); call_count_str = getenv("TEST_UploadPart_CALL_COUNT"); call_count = call_count_str ? atoi(call_count_str) : 0; @@ -383,6 +475,7 @@ void flb_test_s3_upload_part_error(void) unsetenv("TEST_UploadPart_CALL_COUNT"); unsetenv("TEST_CompleteMultipartUpload_CALL_COUNT"); unsetenv("TEST_PutObject_CALL_COUNT"); + flb_free(store_dir); } void flb_test_s3_complete_upload_error(void) @@ -393,9 +486,13 @@ void flb_test_s3_complete_upload_error(void) int out_ffd; char *call_count_str; int call_count; - char store_dir[] = "/tmp/flb-s3-test-uplaod-err-XXXXXX"; + char *store_dir; - TEST_CHECK(mkdtemp(store_dir) != NULL); + store_dir = create_test_store_directory("/flb-s3-test-upload-err-XXXXXX"); + TEST_CHECK(store_dir != NULL); + if (store_dir == NULL) { + return; + } /* mocks calls- signals that we are in test mode */ setenv("FLB_S3_PLUGIN_UNDER_TEST", "true", 1); @@ -412,7 +509,7 @@ void flb_test_s3_complete_upload_error(void) flb_output_set(ctx, out_ffd,"match", "*", NULL); flb_output_set(ctx, out_ffd,"region", "us-west-2", NULL); flb_output_set(ctx, out_ffd,"bucket", "fluent", NULL); - flb_output_set(ctx, out_ffd,"upload_timeout", "6s", NULL); + flb_output_set(ctx, out_ffd,"upload_timeout", S3_TEST_UPLOAD_TIMEOUT, NULL); flb_output_set(ctx, out_ffd,"store_dir", store_dir, NULL); flb_output_set(ctx, out_ffd,"Retry_Limit", "1", NULL); @@ -421,7 +518,7 @@ void flb_test_s3_complete_upload_error(void) flb_lib_push(ctx, in_ffd, (char *) JSON_TD , (int) sizeof(JSON_TD) - 1); - sleep(10); + wait_for_s3_call_count("CompleteMultipartUpload", 2); call_count_str = getenv("TEST_CompleteMultipartUpload_CALL_COUNT"); call_count = call_count_str ? atoi(call_count_str) : 0; @@ -437,6 +534,7 @@ void flb_test_s3_complete_upload_error(void) unsetenv("TEST_UploadPart_CALL_COUNT"); unsetenv("TEST_CompleteMultipartUpload_CALL_COUNT"); unsetenv("TEST_PutObject_CALL_COUNT"); + flb_free(store_dir); } void flb_test_s3_compression_gzip(void) @@ -463,7 +561,7 @@ void flb_test_s3_compression_gzip(void) flb_output_set(ctx, out_ffd,"region", "us-west-2", NULL); flb_output_set(ctx, out_ffd,"bucket", "fluent", NULL); flb_output_set(ctx, out_ffd,"compression", "gzip", NULL); - flb_output_set(ctx, out_ffd,"upload_timeout", "6s", NULL); + flb_output_set(ctx, out_ffd,"upload_timeout", S3_TEST_UPLOAD_TIMEOUT, NULL); flb_output_set(ctx, out_ffd,"Retry_Limit", "1", NULL); ret = flb_start(ctx); @@ -471,7 +569,7 @@ void flb_test_s3_compression_gzip(void) flb_lib_push(ctx, in_ffd, (char *) JSON_TD , (int) sizeof(JSON_TD) - 1); - sleep(10); + wait_for_s3_call_count("CompleteMultipartUpload", 1); call_count_str = getenv("TEST_CompleteMultipartUpload_CALL_COUNT"); call_count = call_count_str ? atoi(call_count_str) : 0; @@ -513,7 +611,7 @@ void flb_test_s3_compression_gzip_putobject(void) flb_output_set(ctx, out_ffd,"compression", "gzip", NULL); flb_output_set(ctx, out_ffd,"use_put_object", "true", NULL); flb_output_set(ctx, out_ffd,"total_file_size", "5M", NULL); - flb_output_set(ctx, out_ffd,"upload_timeout", "6s", NULL); + flb_output_set(ctx, out_ffd,"upload_timeout", S3_TEST_UPLOAD_TIMEOUT, NULL); flb_output_set(ctx, out_ffd,"Retry_Limit", "1", NULL); ret = flb_start(ctx); @@ -521,7 +619,7 @@ void flb_test_s3_compression_gzip_putobject(void) flb_lib_push(ctx, in_ffd, (char *) JSON_TD , (int) sizeof(JSON_TD) - 1); - sleep(10); + wait_for_s3_call_count("PutObject", 1); call_count_str = getenv("TEST_PutObject_CALL_COUNT"); call_count = call_count_str ? atoi(call_count_str) : 0; @@ -558,7 +656,7 @@ void flb_test_s3_compression_zstd(void) flb_output_set(ctx, out_ffd,"region", "us-west-2", NULL); flb_output_set(ctx, out_ffd,"bucket", "fluent", NULL); flb_output_set(ctx, out_ffd,"compression", "zstd", NULL); - flb_output_set(ctx, out_ffd,"upload_timeout", "6s", NULL); + flb_output_set(ctx, out_ffd,"upload_timeout", S3_TEST_UPLOAD_TIMEOUT, NULL); flb_output_set(ctx, out_ffd,"Retry_Limit", "1", NULL); ret = flb_start(ctx); @@ -566,7 +664,7 @@ void flb_test_s3_compression_zstd(void) flb_lib_push(ctx, in_ffd, (char *) JSON_TD , (int) sizeof(JSON_TD) - 1); - sleep(10); + wait_for_s3_call_count("CompleteMultipartUpload", 1); call_count_str = getenv("TEST_CompleteMultipartUpload_CALL_COUNT"); call_count = call_count_str ? atoi(call_count_str) : 0; @@ -608,7 +706,7 @@ void flb_test_s3_compression_zstd_putobject(void) flb_output_set(ctx, out_ffd,"compression", "zstd", NULL); flb_output_set(ctx, out_ffd,"use_put_object", "true", NULL); flb_output_set(ctx, out_ffd,"total_file_size", "5M", NULL); - flb_output_set(ctx, out_ffd,"upload_timeout", "6s", NULL); + flb_output_set(ctx, out_ffd,"upload_timeout", S3_TEST_UPLOAD_TIMEOUT, NULL); flb_output_set(ctx, out_ffd,"Retry_Limit", "1", NULL); ret = flb_start(ctx); @@ -616,7 +714,7 @@ void flb_test_s3_compression_zstd_putobject(void) flb_lib_push(ctx, in_ffd, (char *) JSON_TD , (int) sizeof(JSON_TD) - 1); - sleep(10); + wait_for_s3_call_count("PutObject", 1); call_count_str = getenv("TEST_PutObject_CALL_COUNT"); call_count = call_count_str ? atoi(call_count_str) : 0; @@ -653,7 +751,7 @@ void flb_test_s3_compression_snappy(void) flb_output_set(ctx, out_ffd,"region", "us-west-2", NULL); flb_output_set(ctx, out_ffd,"bucket", "fluent", NULL); flb_output_set(ctx, out_ffd,"compression", "snappy", NULL); - flb_output_set(ctx, out_ffd,"upload_timeout", "6s", NULL); + flb_output_set(ctx, out_ffd,"upload_timeout", S3_TEST_UPLOAD_TIMEOUT, NULL); flb_output_set(ctx, out_ffd,"Retry_Limit", "1", NULL); ret = flb_start(ctx); @@ -661,7 +759,7 @@ void flb_test_s3_compression_snappy(void) flb_lib_push(ctx, in_ffd, (char *) JSON_TD , (int) sizeof(JSON_TD) - 1); - sleep(10); + wait_for_s3_call_count("CompleteMultipartUpload", 1); call_count_str = getenv("TEST_CompleteMultipartUpload_CALL_COUNT"); call_count = call_count_str ? atoi(call_count_str) : 0; @@ -703,7 +801,7 @@ void flb_test_s3_compression_snappy_putobject(void) flb_output_set(ctx, out_ffd,"compression", "snappy", NULL); flb_output_set(ctx, out_ffd,"use_put_object", "true", NULL); flb_output_set(ctx, out_ffd,"total_file_size", "5M", NULL); - flb_output_set(ctx, out_ffd,"upload_timeout", "6s", NULL); + flb_output_set(ctx, out_ffd,"upload_timeout", S3_TEST_UPLOAD_TIMEOUT, NULL); flb_output_set(ctx, out_ffd,"Retry_Limit", "1", NULL); ret = flb_start(ctx); @@ -711,7 +809,7 @@ void flb_test_s3_compression_snappy_putobject(void) flb_lib_push(ctx, in_ffd, (char *) JSON_TD , (int) sizeof(JSON_TD) - 1); - sleep(10); + wait_for_s3_call_count("PutObject", 1); call_count_str = getenv("TEST_PutObject_CALL_COUNT"); call_count = call_count_str ? atoi(call_count_str) : 0; @@ -732,9 +830,13 @@ void flb_test_s3_preserve_data_ordering(void) int out_ffd; char *call_count_str; int call_count; - char store_dir[] = "/tmp/flb-s3-test-ordering-XXXXXX"; + char *store_dir; - TEST_CHECK(mkdtemp(store_dir) != NULL); + store_dir = create_test_store_directory("/flb-s3-test-ordering-XXXXXX"); + TEST_CHECK(store_dir != NULL); + if (store_dir == NULL) { + return; + } setenv("FLB_S3_PLUGIN_UNDER_TEST", "true", 1); @@ -752,7 +854,7 @@ void flb_test_s3_preserve_data_ordering(void) flb_output_set(ctx, out_ffd, "use_put_object", "true", NULL); flb_output_set(ctx, out_ffd, "total_file_size", "5M", NULL); flb_output_set(ctx, out_ffd, "preserve_data_ordering", "true", NULL); - flb_output_set(ctx, out_ffd, "upload_timeout", "6s", NULL); + flb_output_set(ctx, out_ffd, "upload_timeout", S3_TEST_UPLOAD_TIMEOUT, NULL); flb_output_set(ctx, out_ffd, "store_dir", store_dir, NULL); flb_output_set(ctx, out_ffd, "Retry_Limit", "1", NULL); @@ -761,7 +863,7 @@ void flb_test_s3_preserve_data_ordering(void) flb_lib_push(ctx, in_ffd, (char *) JSON_TD, (int) sizeof(JSON_TD) - 1); - sleep(10); + wait_for_s3_call_count("PutObject", 1); call_count_str = getenv("TEST_PutObject_CALL_COUNT"); call_count = call_count_str ? atoi(call_count_str) : 0; @@ -772,6 +874,7 @@ void flb_test_s3_preserve_data_ordering(void) flb_destroy(ctx); unsetenv("FLB_S3_PLUGIN_UNDER_TEST"); unsetenv("TEST_PutObject_CALL_COUNT"); + flb_free(store_dir); } @@ -786,9 +889,13 @@ void flb_test_s3_putobject_retry_limit_semantics(void) int out_ffd; char *call_count_str; int call_count; - char store_dir[] = "/tmp/flb-s3-test-retry-XXXXXX"; + char *store_dir; - TEST_CHECK(mkdtemp(store_dir) != NULL); + store_dir = create_test_store_directory("/flb-s3-test-retry-XXXXXX"); + TEST_CHECK(store_dir != NULL); + if (store_dir == NULL) { + return; + } /* Use mocks without flush bypass so the plugin's internal retry runs */ setenv("FLB_S3_PLUGIN_UNDER_TEST", "true", 1); @@ -807,7 +914,7 @@ void flb_test_s3_putobject_retry_limit_semantics(void) flb_output_set(ctx, out_ffd, "bucket", "fluent", NULL); flb_output_set(ctx, out_ffd, "use_put_object", "true", NULL); flb_output_set(ctx, out_ffd, "total_file_size", "5M", NULL); - flb_output_set(ctx, out_ffd, "upload_timeout", "6s", NULL); + flb_output_set(ctx, out_ffd, "upload_timeout", S3_TEST_UPLOAD_TIMEOUT, NULL); flb_output_set(ctx, out_ffd, "store_dir", store_dir, NULL); flb_output_set(ctx, out_ffd, "Retry_Limit", "1", NULL); @@ -817,15 +924,9 @@ void flb_test_s3_putobject_retry_limit_semantics(void) /* Reset counter after startup so we only count test-driven attempts */ unsetenv("TEST_PutObject_CALL_COUNT"); - /* - * Push 1 chunk then wait for upload_timeout (6s) + 2 timer ticks (1s each). - * Chunk must age past upload_timeout before cb_s3_upload will attempt it. - * Tick after ~6s: PutObject attempt 1 fails (failures=1) - * Tick after ~7s: failures(1) not > retry_limit(1), attempt 2 fails (failures=2) - * Next tick: failures(2) > retry_limit(1), chunk discarded - */ + /* Wait until the initial attempt and one retry have run. */ flb_lib_push(ctx, in_ffd, (char *) JSON_TD, (int) sizeof(JSON_TD) - 1); - sleep(10); + wait_for_s3_call_count("PutObject", 2); flb_stop(ctx); flb_destroy(ctx); @@ -841,6 +942,7 @@ void flb_test_s3_putobject_retry_limit_semantics(void) unsetenv("FLB_S3_PLUGIN_UNDER_TEST"); unsetenv("TEST_PUT_OBJECT_ERROR"); unsetenv("TEST_PutObject_CALL_COUNT"); + flb_free(store_dir); } /* @@ -854,9 +956,13 @@ void flb_test_s3_default_retry_limit(void) int out_ffd; char *call_count_str; int call_count; - char store_dir[] = "/tmp/flb-s3-test-default-XXXXXX"; + char *store_dir; - TEST_CHECK(mkdtemp(store_dir) != NULL); + store_dir = create_test_store_directory("/flb-s3-test-default-XXXXXX"); + TEST_CHECK(store_dir != NULL); + if (store_dir == NULL) { + return; + } setenv("FLB_S3_PLUGIN_UNDER_TEST", "true", 1); setenv("TEST_PUT_OBJECT_ERROR", ERROR_ACCESS_DENIED, 1); @@ -874,7 +980,7 @@ void flb_test_s3_default_retry_limit(void) flb_output_set(ctx, out_ffd, "bucket", "fluent", NULL); flb_output_set(ctx, out_ffd, "use_put_object", "true", NULL); flb_output_set(ctx, out_ffd, "total_file_size", "5M", NULL); - flb_output_set(ctx, out_ffd, "upload_timeout", "6s", NULL); + flb_output_set(ctx, out_ffd, "upload_timeout", S3_TEST_UPLOAD_TIMEOUT, NULL); flb_output_set(ctx, out_ffd, "store_dir", store_dir, NULL); /* No Retry_Limit — should default to 5 (MAX_UPLOAD_ERRORS) */ @@ -883,12 +989,9 @@ void flb_test_s3_default_retry_limit(void) unsetenv("TEST_PutObject_CALL_COUNT"); - /* - * Push 1 chunk, wait for upload_timeout (6s) + 6 timer ticks (1s each). - * Default retry_limit=5: 1 initial attempt + 5 retries = 6 PutObject calls. - */ + /* Wait for the initial attempt and all five default retries. */ flb_lib_push(ctx, in_ffd, (char *) JSON_TD, (int) sizeof(JSON_TD) - 1); - sleep(14); + wait_for_s3_call_count("PutObject", 6); flb_stop(ctx); flb_destroy(ctx); @@ -903,6 +1006,7 @@ void flb_test_s3_default_retry_limit(void) unsetenv("FLB_S3_PLUGIN_UNDER_TEST"); unsetenv("TEST_PUT_OBJECT_ERROR"); unsetenv("TEST_PutObject_CALL_COUNT"); + flb_free(store_dir); } void flb_test_s3_default_retry_exhausted_action_quarantine(void) @@ -913,6 +1017,7 @@ void flb_test_s3_default_retry_exhausted_action_quarantine(void) int out_ffd; int file_count; char postfix[128]; + char quarantine_dir[2048]; char *store_dir; snprintf(postfix, sizeof(postfix), @@ -920,6 +1025,8 @@ void flb_test_s3_default_retry_exhausted_action_quarantine(void) store_dir = flb_test_tmpdir_cat(postfix); TEST_CHECK(store_dir != NULL); TEST_CHECK(ensure_test_directory(store_dir) == 0); + snprintf(quarantine_dir, sizeof(quarantine_dir), + "%s/fluent/quarantine", store_dir); setenv("FLB_S3_PLUGIN_UNDER_TEST", "true", 1); setenv("TEST_PUT_OBJECT_ERROR", ERROR_ACCESS_DENIED, 1); @@ -937,7 +1044,7 @@ void flb_test_s3_default_retry_exhausted_action_quarantine(void) flb_output_set(ctx, out_ffd, "bucket", "fluent", NULL); flb_output_set(ctx, out_ffd, "use_put_object", "true", NULL); flb_output_set(ctx, out_ffd, "total_file_size", "5M", NULL); - flb_output_set(ctx, out_ffd, "upload_timeout", "6s", NULL); + flb_output_set(ctx, out_ffd, "upload_timeout", S3_TEST_UPLOAD_TIMEOUT, NULL); flb_output_set(ctx, out_ffd, "store_dir", store_dir, NULL); flb_output_set(ctx, out_ffd, "Retry_Limit", "1", NULL); /* do not set retry_exhausted_action to validate default behavior */ @@ -947,14 +1054,15 @@ void flb_test_s3_default_retry_exhausted_action_quarantine(void) unsetenv("TEST_PutObject_CALL_COUNT"); flb_lib_push(ctx, in_ffd, (char *) JSON_TD, (int) sizeof(JSON_TD) - 1); - sleep(10); + wait_for_s3_call_count("PutObject", 2); + wait_for_file_count(quarantine_dir, 1); - file_count = count_files_recursive(store_dir); + file_count = count_files_recursive(quarantine_dir); flb_stop(ctx); flb_destroy(ctx); TEST_CHECK_(file_count > 0, - "Expected quarantined file(s) in store_dir, got %d", + "Expected quarantined file(s), got %d", file_count); unsetenv("FLB_S3_PLUGIN_UNDER_TEST"); diff --git a/tests/runtime/out_s3_otlp_json.c b/tests/runtime/out_s3_otlp_json.c index 5869037fa2e..2ed480b428a 100644 --- a/tests/runtime/out_s3_otlp_json.c +++ b/tests/runtime/out_s3_otlp_json.c @@ -1,11 +1,29 @@ /* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ #include #include "flb_tests_runtime.h" +#include "../include/flb_tests_tmpdir.h" #include "../../plugins/in_opentelemetry/opentelemetry.h" #include "../../plugins/in_opentelemetry/opentelemetry_logs.h" #define OTLP_LOGS_JSON "{\"resourceLogs\":[{\"resource\":{\"attributes\":[{\"key\":\"service.name\",\"value\":{\"stringValue\":\"my.service\"}}]},\"scopeLogs\":[{\"scope\":{\"name\":\"my.library\",\"version\":\"1.0.0\"},\"logRecords\":[{\"timeUnixNano\":\"1774877764000000000\",\"observedTimeUnixNano\":\"1774877764000000000\",\"severityNumber\":2,\"severityText\":\"INFO\",\"body\":{\"stringValue\":\"otlp runtime test\"}}]}]}]}" +static char *create_test_store_directory(const char *postfix) +{ + char *store_dir; + + store_dir = flb_test_tmpdir_cat(postfix); + if (store_dir == NULL) { + return NULL; + } + + if (mkdtemp(store_dir) == NULL) { + flb_free(store_dir); + return NULL; + } + + return store_dir; +} + static struct flb_input_instance *get_opentelemetry_instance(flb_ctx_t *flb_ctx) { struct mk_list *head; @@ -55,7 +73,13 @@ void flb_test_s3_format_otlp_json(void) int out_ffd; char *call_count_str; int call_count; - char store_dir[] = "/tmp/flb-s3-test-otlp-json-XXXXXX"; + char *store_dir; + + store_dir = create_test_store_directory("/flb-s3-test-otlp-json-XXXXXX"); + TEST_CHECK(store_dir != NULL); + if (store_dir == NULL) { + return; + } setenv("FLB_S3_PLUGIN_UNDER_TEST", "true", 1); @@ -95,6 +119,7 @@ void flb_test_s3_format_otlp_json(void) flb_destroy(ctx); unsetenv("FLB_S3_PLUGIN_UNDER_TEST"); unsetenv("TEST_PutObject_CALL_COUNT"); + flb_free(store_dir); } void flb_test_s3_format_otlp_json_with_compression(void) @@ -105,7 +130,13 @@ void flb_test_s3_format_otlp_json_with_compression(void) int out_ffd; char *call_count_str; int call_count; - char store_dir[] = "/tmp/flb-s3-test-otlp-comp-XXXXXX"; + char *store_dir; + + store_dir = create_test_store_directory("/flb-s3-test-otlp-comp-XXXXXX"); + TEST_CHECK(store_dir != NULL); + if (store_dir == NULL) { + return; + } setenv("FLB_S3_PLUGIN_UNDER_TEST", "true", 1); @@ -147,6 +178,7 @@ void flb_test_s3_format_otlp_json_with_compression(void) flb_destroy(ctx); unsetenv("FLB_S3_PLUGIN_UNDER_TEST"); unsetenv("TEST_PutObject_CALL_COUNT"); + flb_free(store_dir); } TEST_LIST = { diff --git a/tests/runtime/out_stackdriver.c b/tests/runtime/out_stackdriver.c index 065329da1b0..7ea57cfd5fd 100644 --- a/tests/runtime/out_stackdriver.c +++ b/tests/runtime/out_stackdriver.c @@ -21,6 +21,8 @@ #include #include #include +#include +#include #include "flb_tests_runtime.h" @@ -46,6 +48,81 @@ #include "data/stackdriver/stackdriver_test_monitored_resource.h" #include "data/stackdriver/stackdriver_test_payload.h" +#define STACKDRIVER_TEST_WAIT_STEP_MS 10 +#define STACKDRIVER_TEST_TIMEOUT_MS 2000 + +typedef void (*stackdriver_test_callback)(void *, int, int, void *, size_t, void *); + +static pthread_mutex_t stackdriver_test_mutex = PTHREAD_MUTEX_INITIALIZER; +static stackdriver_test_callback stackdriver_formatter_callback; +static void *stackdriver_formatter_callback_data; +static int stackdriver_formatter_complete; + +static void cb_stackdriver_formatter(void *ctx, int ffd, int res_ret, + void *res_data, size_t res_size, void *data) +{ + stackdriver_test_callback callback; + void *callback_data; + + (void) data; + + pthread_mutex_lock(&stackdriver_test_mutex); + callback = stackdriver_formatter_callback; + callback_data = stackdriver_formatter_callback_data; + pthread_mutex_unlock(&stackdriver_test_mutex); + + callback(ctx, ffd, res_ret, res_data, res_size, callback_data); + + pthread_mutex_lock(&stackdriver_test_mutex); + stackdriver_formatter_complete = FLB_TRUE; + pthread_mutex_unlock(&stackdriver_test_mutex); +} + +static int stackdriver_output_set_test(flb_ctx_t *ctx, int ffd, char *test_name, + stackdriver_test_callback callback, + void *callback_data, void *test_ctx) +{ + pthread_mutex_lock(&stackdriver_test_mutex); + stackdriver_formatter_callback = callback; + stackdriver_formatter_callback_data = callback_data; + stackdriver_formatter_complete = FLB_FALSE; + pthread_mutex_unlock(&stackdriver_test_mutex); + + return flb_output_set_test(ctx, ffd, test_name, + cb_stackdriver_formatter, NULL, test_ctx); +} + +static void stackdriver_wait_for_formatter(void) +{ + int complete; + uint64_t elapsed_ms; + struct flb_time start_time; + struct flb_time end_time; + struct flb_time diff_time; + + complete = FLB_FALSE; + elapsed_ms = 0; + flb_time_get(&start_time); + + /* Preserve the bounded fallback for cases that do not invoke the callback. */ + while (elapsed_ms < STACKDRIVER_TEST_TIMEOUT_MS) { + pthread_mutex_lock(&stackdriver_test_mutex); + complete = stackdriver_formatter_complete; + pthread_mutex_unlock(&stackdriver_test_mutex); + + if (complete == FLB_TRUE) { + break; + } + + flb_time_msleep(STACKDRIVER_TEST_WAIT_STEP_MS); + flb_time_get(&end_time); + flb_time_diff(&end_time, &start_time, &diff_time); + elapsed_ms = flb_time_to_nanosec(&diff_time) / 1000000; + } +} + +#define flb_output_set_test stackdriver_output_set_test + /* * Fluent Bit Stackdriver plugin, always set as payload a JSON strings contained in a * 'sds'. Since we want to validate specific keys and it values we expose here some @@ -2405,9 +2482,9 @@ void flb_test_monitored_resource_common() int in_ffd; int out_ffd; - /* Create context, flush every second (some checks omitted here) */ + /* Create context, flush every 200 milliseconds (some checks omitted here) */ ctx = flb_create(); - flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + flb_service_set(ctx, "flush", "0.2", "grace", "1", NULL); /* Lib input mode */ in_ffd = flb_input(ctx, (char *) "lib", NULL); @@ -2433,7 +2510,7 @@ void flb_test_monitored_resource_common() /* Ingest data sample */ flb_lib_push(ctx, in_ffd, (char *) MONITORED_RESOURCE_COMMON_CASE, size); - sleep(2); + stackdriver_wait_for_formatter(); flb_stop(ctx); flb_destroy(ctx); } @@ -2446,9 +2523,9 @@ void flb_test_monitored_resource_priority_higher_than_local_resource_id() int in_ffd; int out_ffd; - /* Create context, flush every second (some checks omitted here) */ + /* Create context, flush every 200 milliseconds (some checks omitted here) */ ctx = flb_create(); - flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + flb_service_set(ctx, "flush", "0.2", "grace", "1", NULL); /* Lib input mode */ in_ffd = flb_input(ctx, (char *) "lib", NULL); @@ -2476,7 +2553,7 @@ void flb_test_monitored_resource_priority_higher_than_local_resource_id() /* Ingest data sample */ flb_lib_push(ctx, in_ffd, (char *) MONITORED_RESOURCE_PRIORITY_HIGHER_THAN_LOCAL_RESOURCE_ID, size); - sleep(2); + stackdriver_wait_for_formatter(); flb_stop(ctx); flb_destroy(ctx); } @@ -2489,9 +2566,9 @@ void flb_test_monitored_resource_priority_higher_than_gce_instance() int in_ffd; int out_ffd; - /* Create context, flush every second (some checks omitted here) */ + /* Create context, flush every 200 milliseconds (some checks omitted here) */ ctx = flb_create(); - flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + flb_service_set(ctx, "flush", "0.2", "grace", "1", NULL); /* Lib input mode */ in_ffd = flb_input(ctx, (char *) "lib", NULL); @@ -2517,7 +2594,7 @@ void flb_test_monitored_resource_priority_higher_than_gce_instance() /* Ingest data sample */ flb_lib_push(ctx, in_ffd, (char *) MONITORED_RESOURCE_PRIORITY_HIGHER_THAN_GCE_INSTANCE, size); - sleep(2); + stackdriver_wait_for_formatter(); flb_stop(ctx); flb_destroy(ctx); } @@ -2530,9 +2607,9 @@ void flb_test_resource_global() int in_ffd; int out_ffd; - /* Create context, flush every second (some checks omitted here) */ + /* Create context, flush every 200 milliseconds (some checks omitted here) */ ctx = flb_create(); - flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + flb_service_set(ctx, "flush", "0.2", "grace", "1", NULL); /* Lib input mode */ in_ffd = flb_input(ctx, (char *) "lib", NULL); @@ -2558,7 +2635,7 @@ void flb_test_resource_global() /* Ingest data sample */ flb_lib_push(ctx, in_ffd, (char *) JSON, size); - sleep(2); + stackdriver_wait_for_formatter(); flb_stop(ctx); flb_destroy(ctx); } @@ -2571,9 +2648,9 @@ void flb_test_trace_no_autoformat() int in_ffd; int out_ffd; - /* Create context, flush every second (some checks omitted here) */ + /* Create context, flush every 200 milliseconds (some checks omitted here) */ ctx = flb_create(); - flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + flb_service_set(ctx, "flush", "0.2", "grace", "1", NULL); /* Lib input mode */ in_ffd = flb_input(ctx, (char *) "lib", NULL); @@ -2599,7 +2676,7 @@ void flb_test_trace_no_autoformat() /* Ingest data sample */ flb_lib_push(ctx, in_ffd, (char *) TRACE_COMMON_CASE, size); - sleep(2); + stackdriver_wait_for_formatter(); flb_stop(ctx); flb_destroy(ctx); } @@ -2612,9 +2689,9 @@ void flb_test_trace_stackdriver_autoformat() int in_ffd; int out_ffd; - /* Create context, flush every second (some checks omitted here) */ + /* Create context, flush every 200 milliseconds (some checks omitted here) */ ctx = flb_create(); - flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + flb_service_set(ctx, "flush", "0.2", "grace", "1", NULL); /* Lib input mode */ in_ffd = flb_input(ctx, (char *) "lib", NULL); @@ -2641,7 +2718,7 @@ void flb_test_trace_stackdriver_autoformat() /* Ingest data sample */ flb_lib_push(ctx, in_ffd, (char *) TRACE_COMMON_CASE, size); - sleep(2); + stackdriver_wait_for_formatter(); flb_stop(ctx); flb_destroy(ctx); } @@ -2654,9 +2731,9 @@ void flb_test_span_id() int in_ffd; int out_ffd; - /* Create context, flush every second (some checks omitted here) */ + /* Create context, flush every 200 milliseconds (some checks omitted here) */ ctx = flb_create(); - flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + flb_service_set(ctx, "flush", "0.2", "grace", "1", NULL); /* Lib input mode */ in_ffd = flb_input(ctx, (char *) "lib", NULL); @@ -2681,7 +2758,7 @@ void flb_test_span_id() /* Ingest data sample */ flb_lib_push(ctx, in_ffd, (char *) SPAN_ID_COMMON_CASE, size); - sleep(2); + stackdriver_wait_for_formatter(); flb_stop(ctx); flb_destroy(ctx); } @@ -2694,9 +2771,9 @@ void flb_test_trace_sampled_true() int in_ffd; int out_ffd; - /* Create context, flush every second (some checks omitted here) */ + /* Create context, flush every 200 milliseconds (some checks omitted here) */ ctx = flb_create(); - flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + flb_service_set(ctx, "flush", "0.2", "grace", "1", NULL); /* Lib input mode */ in_ffd = flb_input(ctx, (char *) "lib", NULL); @@ -2721,7 +2798,7 @@ void flb_test_trace_sampled_true() /* Ingest data sample */ flb_lib_push(ctx, in_ffd, (char *) TRACE_SAMPLED_CASE_TRUE, size); - sleep(2); + stackdriver_wait_for_formatter(); flb_stop(ctx); flb_destroy(ctx); } @@ -2734,9 +2811,9 @@ void flb_test_trace_sampled_false() int in_ffd; int out_ffd; - /* Create context, flush every second (some checks omitted here) */ + /* Create context, flush every 200 milliseconds (some checks omitted here) */ ctx = flb_create(); - flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + flb_service_set(ctx, "flush", "0.2", "grace", "1", NULL); /* Lib input mode */ in_ffd = flb_input(ctx, (char *) "lib", NULL); @@ -2761,7 +2838,7 @@ void flb_test_trace_sampled_false() /* Ingest data sample */ flb_lib_push(ctx, in_ffd, (char *) TRACE_SAMPLED_CASE_FALSE, size); - sleep(2); + stackdriver_wait_for_formatter(); flb_stop(ctx); flb_destroy(ctx); } @@ -2774,9 +2851,9 @@ void flb_test_set_metadata_server() int in_ffd; int out_ffd; - /* Create context, flush every second (some checks omitted here) */ + /* Create context, flush every 200 milliseconds (some checks omitted here) */ ctx = flb_create(); - flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + flb_service_set(ctx, "flush", "0.2", "grace", "1", NULL); /* Lib input mode */ in_ffd = flb_input(ctx, (char *) "lib", NULL); @@ -2802,7 +2879,7 @@ void flb_test_set_metadata_server() /* Ingest data sample */ flb_lib_push(ctx, in_ffd, (char *) JSON, size); - sleep(2); + stackdriver_wait_for_formatter(); flb_stop(ctx); flb_destroy(ctx); } @@ -2815,9 +2892,9 @@ void flb_test_project_id_override() int in_ffd; int out_ffd; - /* Create context, flush every second (some checks omitted here) */ + /* Create context, flush every 200 milliseconds (some checks omitted here) */ ctx = flb_create(); - flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + flb_service_set(ctx, "flush", "0.2", "grace", "1", NULL); /* Lib input mode */ in_ffd = flb_input(ctx, (char *) "lib", NULL); @@ -2843,7 +2920,7 @@ void flb_test_project_id_override() /* Ingest data sample */ flb_lib_push(ctx, in_ffd, (char *) LOG_NAME_PROJECT_ID_OVERRIDE, size); - sleep(2); + stackdriver_wait_for_formatter(); flb_stop(ctx); flb_destroy(ctx); } @@ -2856,9 +2933,9 @@ void flb_test_project_id_no_override() int in_ffd; int out_ffd; - /* Create context, flush every second (some checks omitted here) */ + /* Create context, flush every 200 milliseconds (some checks omitted here) */ ctx = flb_create(); - flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + flb_service_set(ctx, "flush", "0.2", "grace", "1", NULL); /* Lib input mode */ in_ffd = flb_input(ctx, (char *) "lib", NULL); @@ -2883,7 +2960,7 @@ void flb_test_project_id_no_override() /* Ingest data sample */ flb_lib_push(ctx, in_ffd, (char *) LOG_NAME_PROJECT_ID_NO_OVERRIDE, size); - sleep(2); + stackdriver_wait_for_formatter(); flb_stop(ctx); flb_destroy(ctx); } @@ -2896,9 +2973,9 @@ void flb_test_log_name_override() int in_ffd; int out_ffd; - /* Create context, flush every second (some checks omitted here) */ + /* Create context, flush every 200 milliseconds (some checks omitted here) */ ctx = flb_create(); - flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + flb_service_set(ctx, "flush", "0.2", "grace", "1", NULL); /* Lib input mode */ in_ffd = flb_input(ctx, (char *) "lib", NULL); @@ -2924,7 +3001,7 @@ void flb_test_log_name_override() /* Ingest data sample */ flb_lib_push(ctx, in_ffd, (char *) LOG_NAME_OVERRIDE, size); - sleep(2); + stackdriver_wait_for_formatter(); flb_stop(ctx); flb_destroy(ctx); } @@ -2937,9 +3014,9 @@ void flb_test_log_name_no_override() int in_ffd; int out_ffd; - /* Create context, flush every second (some checks omitted here) */ + /* Create context, flush every 200 milliseconds (some checks omitted here) */ ctx = flb_create(); - flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + flb_service_set(ctx, "flush", "0.2", "grace", "1", NULL); /* Lib input mode */ in_ffd = flb_input(ctx, (char *) "lib", NULL); @@ -2965,7 +3042,7 @@ void flb_test_log_name_no_override() /* Ingest data sample */ flb_lib_push(ctx, in_ffd, (char *) LOG_NAME_NO_OVERRIDE, size); - sleep(2); + stackdriver_wait_for_formatter(); flb_stop(ctx); flb_destroy(ctx); } @@ -2979,9 +3056,9 @@ void flb_test_resource_global_custom_prefix() int in_ffd; int out_ffd; - /* Create context, flush every second (some checks omitted here) */ + /* Create context, flush every 200 milliseconds (some checks omitted here) */ ctx = flb_create(); - flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + flb_service_set(ctx, "flush", "0.2", "grace", "1", NULL); /* Lib input mode */ in_ffd = flb_input(ctx, (char *) "lib", NULL); @@ -3008,7 +3085,7 @@ void flb_test_resource_global_custom_prefix() /* Ingest data sample */ flb_lib_push(ctx, in_ffd, (char *) JSON, size); - sleep(2); + stackdriver_wait_for_formatter(); flb_stop(ctx); flb_destroy(ctx); } @@ -3021,9 +3098,9 @@ void flb_test_resource_generic_node_creds() int in_ffd; int out_ffd; - /* Create context, flush every second (some checks omitted here) */ + /* Create context, flush every 200 milliseconds (some checks omitted here) */ ctx = flb_create(); - flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + flb_service_set(ctx, "flush", "0.2", "grace", "1", NULL); /* Lib input mode */ in_ffd = flb_input(ctx, (char *) "lib", NULL); @@ -3052,7 +3129,7 @@ void flb_test_resource_generic_node_creds() /* Ingest data sample */ flb_lib_push(ctx, in_ffd, (char *) JSON, size); - sleep(2); + stackdriver_wait_for_formatter(); flb_stop(ctx); flb_destroy(ctx); } @@ -3065,9 +3142,9 @@ void flb_test_resource_generic_node_metadata() int in_ffd; int out_ffd; - /* Create context, flush every second (some checks omitted here) */ + /* Create context, flush every 200 milliseconds (some checks omitted here) */ ctx = flb_create(); - flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + flb_service_set(ctx, "flush", "0.2", "grace", "1", NULL); /* Lib input mode */ in_ffd = flb_input(ctx, (char *) "lib", NULL); @@ -3095,7 +3172,7 @@ void flb_test_resource_generic_node_metadata() /* Ingest data sample */ flb_lib_push(ctx, in_ffd, (char *) JSON, size); - sleep(2); + stackdriver_wait_for_formatter(); flb_stop(ctx); flb_destroy(ctx); } @@ -3108,9 +3185,9 @@ void flb_test_resource_generic_task_creds() int in_ffd; int out_ffd; - /* Create context, flush every second (some checks omitted here) */ + /* Create context, flush every 200 milliseconds (some checks omitted here) */ ctx = flb_create(); - flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + flb_service_set(ctx, "flush", "0.2", "grace", "1", NULL); /* Lib input mode */ in_ffd = flb_input(ctx, (char *) "lib", NULL); @@ -3140,7 +3217,7 @@ void flb_test_resource_generic_task_creds() /* Ingest data sample */ flb_lib_push(ctx, in_ffd, (char *) JSON, size); - sleep(2); + stackdriver_wait_for_formatter(); flb_stop(ctx); flb_destroy(ctx); } @@ -3153,9 +3230,9 @@ void flb_test_resource_generic_task_metadata() int in_ffd; int out_ffd; - /* Create context, flush every second (some checks omitted here) */ + /* Create context, flush every 200 milliseconds (some checks omitted here) */ ctx = flb_create(); - flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + flb_service_set(ctx, "flush", "0.2", "grace", "1", NULL); /* Lib input mode */ in_ffd = flb_input(ctx, (char *) "lib", NULL); @@ -3184,7 +3261,7 @@ void flb_test_resource_generic_task_metadata() /* Ingest data sample */ flb_lib_push(ctx, in_ffd, (char *) JSON, size); - sleep(2); + stackdriver_wait_for_formatter(); flb_stop(ctx); flb_destroy(ctx); } @@ -3197,9 +3274,9 @@ void flb_test_resource_gce_instance() int in_ffd; int out_ffd; - /* Create context, flush every second (some checks omitted here) */ + /* Create context, flush every 200 milliseconds (some checks omitted here) */ ctx = flb_create(); - flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + flb_service_set(ctx, "flush", "0.2", "grace", "1", NULL); /* Lib input mode */ in_ffd = flb_input(ctx, (char *) "lib", NULL); @@ -3224,7 +3301,7 @@ void flb_test_resource_gce_instance() /* Ingest data sample */ flb_lib_push(ctx, in_ffd, (char *) JSON, size); - sleep(2); + stackdriver_wait_for_formatter(); flb_stop(ctx); flb_destroy(ctx); } @@ -3237,9 +3314,9 @@ void flb_test_insert_id_common_case() int in_ffd; int out_ffd; - /* Create context, flush every second (some checks omitted here) */ + /* Create context, flush every 200 milliseconds (some checks omitted here) */ ctx = flb_create(); - flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + flb_service_set(ctx, "flush", "0.2", "grace", "1", NULL); /* Lib input mode */ in_ffd = flb_input(ctx, (char *) "lib", NULL); @@ -3264,7 +3341,7 @@ void flb_test_insert_id_common_case() /* Ingest data sample */ flb_lib_push(ctx, in_ffd, (char *) INSERTID_COMMON_CASE, size); - sleep(2); + stackdriver_wait_for_formatter(); flb_stop(ctx); flb_destroy(ctx); } @@ -3277,9 +3354,9 @@ void flb_test_empty_insert_id() int in_ffd; int out_ffd; - /* Create context, flush every second (some checks omitted here) */ + /* Create context, flush every 200 milliseconds (some checks omitted here) */ ctx = flb_create(); - flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + flb_service_set(ctx, "flush", "0.2", "grace", "1", NULL); /* Lib input mode */ in_ffd = flb_input(ctx, (char *) "lib", NULL); @@ -3304,7 +3381,7 @@ void flb_test_empty_insert_id() /* Ingest data sample */ flb_lib_push(ctx, in_ffd, (char *) EMPTY_INSERTID, size); - sleep(2); + stackdriver_wait_for_formatter(); flb_stop(ctx); flb_destroy(ctx); } @@ -3317,9 +3394,9 @@ void flb_test_insert_id_incorrect_type() int in_ffd; int out_ffd; - /* Create context, flush every second (some checks omitted here) */ + /* Create context, flush every 200 milliseconds (some checks omitted here) */ ctx = flb_create(); - flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + flb_service_set(ctx, "flush", "0.2", "grace", "1", NULL); /* Lib input mode */ in_ffd = flb_input(ctx, (char *) "lib", NULL); @@ -3344,7 +3421,7 @@ void flb_test_insert_id_incorrect_type() /* Ingest data sample */ flb_lib_push(ctx, in_ffd, (char *) INSERTID_INCORRECT_TYPE_INT, size); - sleep(2); + stackdriver_wait_for_formatter(); flb_stop(ctx); flb_destroy(ctx); } @@ -3357,9 +3434,9 @@ void flb_test_operation_common() int in_ffd; int out_ffd; - /* Create context, flush every second (some checks omitted here) */ + /* Create context, flush every 200 milliseconds (some checks omitted here) */ ctx = flb_create(); - flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + flb_service_set(ctx, "flush", "0.2", "grace", "1", NULL); /* Lib input mode */ in_ffd = flb_input(ctx, (char *) "lib", NULL); @@ -3384,7 +3461,7 @@ void flb_test_operation_common() /* Ingest data sample */ flb_lib_push(ctx, in_ffd, (char *) OPERATION_COMMON_CASE, size); - sleep(2); + stackdriver_wait_for_formatter(); flb_stop(ctx); flb_destroy(ctx); } @@ -3397,9 +3474,9 @@ void flb_test_empty_operation() int in_ffd; int out_ffd; - /* Create context, flush every second (some checks omitted here) */ + /* Create context, flush every 200 milliseconds (some checks omitted here) */ ctx = flb_create(); - flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + flb_service_set(ctx, "flush", "0.2", "grace", "1", NULL); /* Lib input mode */ in_ffd = flb_input(ctx, (char *) "lib", NULL); @@ -3424,7 +3501,7 @@ void flb_test_empty_operation() /* Ingest data sample */ flb_lib_push(ctx, in_ffd, (char *) EMPTY_OPERATION, size); - sleep(2); + stackdriver_wait_for_formatter(); flb_stop(ctx); flb_destroy(ctx); } @@ -3437,9 +3514,9 @@ void flb_test_operation_in_string() int in_ffd; int out_ffd; - /* Create context, flush every second (some checks omitted here) */ + /* Create context, flush every 200 milliseconds (some checks omitted here) */ ctx = flb_create(); - flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + flb_service_set(ctx, "flush", "0.2", "grace", "1", NULL); /* Lib input mode */ in_ffd = flb_input(ctx, (char *) "lib", NULL); @@ -3464,7 +3541,7 @@ void flb_test_operation_in_string() /* Ingest data sample */ flb_lib_push(ctx, in_ffd, (char *) OPERATION_IN_STRING, size); - sleep(2); + stackdriver_wait_for_formatter(); flb_stop(ctx); flb_destroy(ctx); } @@ -3477,9 +3554,9 @@ void flb_test_operation_partial_subfields() int in_ffd; int out_ffd; - /* Create context, flush every second (some checks omitted here) */ + /* Create context, flush every 200 milliseconds (some checks omitted here) */ ctx = flb_create(); - flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + flb_service_set(ctx, "flush", "0.2", "grace", "1", NULL); /* Lib input mode */ in_ffd = flb_input(ctx, (char *) "lib", NULL); @@ -3504,7 +3581,7 @@ void flb_test_operation_partial_subfields() /* Ingest data sample */ flb_lib_push(ctx, in_ffd, (char *) PARTIAL_SUBFIELDS, size); - sleep(2); + stackdriver_wait_for_formatter(); flb_stop(ctx); flb_destroy(ctx); } @@ -3517,9 +3594,9 @@ void flb_test_operation_incorrect_type_subfields() int in_ffd; int out_ffd; - /* Create context, flush every second (some checks omitted here) */ + /* Create context, flush every 200 milliseconds (some checks omitted here) */ ctx = flb_create(); - flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + flb_service_set(ctx, "flush", "0.2", "grace", "1", NULL); /* Lib input mode */ in_ffd = flb_input(ctx, (char *) "lib", NULL); @@ -3544,7 +3621,7 @@ void flb_test_operation_incorrect_type_subfields() /* Ingest data sample */ flb_lib_push(ctx, in_ffd, (char *) SUBFIELDS_IN_INCORRECT_TYPE, size); - sleep(2); + stackdriver_wait_for_formatter(); flb_stop(ctx); flb_destroy(ctx); } @@ -3557,9 +3634,9 @@ void flb_test_operation_extra_subfields() int in_ffd; int out_ffd; - /* Create context, flush every second (some checks omitted here) */ + /* Create context, flush every 200 milliseconds (some checks omitted here) */ ctx = flb_create(); - flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + flb_service_set(ctx, "flush", "0.2", "grace", "1", NULL); /* Lib input mode */ in_ffd = flb_input(ctx, (char *) "lib", NULL); @@ -3584,7 +3661,7 @@ void flb_test_operation_extra_subfields() /* Ingest data sample */ flb_lib_push(ctx, in_ffd, (char *) EXTRA_SUBFIELDS_EXISTED, size); - sleep(2); + stackdriver_wait_for_formatter(); flb_stop(ctx); flb_destroy(ctx); } @@ -3597,9 +3674,9 @@ void flb_test_resource_k8s_container_common() int in_ffd; int out_ffd; - /* Create context, flush every second (some checks omitted here) */ + /* Create context, flush every 200 milliseconds (some checks omitted here) */ ctx = flb_create(); - flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + flb_service_set(ctx, "flush", "0.2", "grace", "1", NULL); /* Lib input mode */ in_ffd = flb_input(ctx, (char *) "lib", NULL); @@ -3627,7 +3704,7 @@ void flb_test_resource_k8s_container_common() /* Ingest data sample */ flb_lib_push(ctx, in_ffd, (char *) K8S_CONTAINER_COMMON, size); - sleep(2); + stackdriver_wait_for_formatter(); flb_stop(ctx); flb_destroy(ctx); } @@ -3644,9 +3721,9 @@ void flb_test_resource_k8s_container_multi_tag_value() int in_ffd; int out_ffd; - /* Create context, flush every second (some checks omitted here) */ + /* Create context, flush every 200 milliseconds (some checks omitted here) */ ctx = flb_create(); - flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + flb_service_set(ctx, "flush", "0.2", "grace", "1", NULL); /* Lib input mode */ in_ffd = flb_input(ctx, (char *) "lib", NULL); @@ -3672,7 +3749,9 @@ void flb_test_resource_k8s_container_multi_tag_value() TEST_CHECK(ret == 0); /* Ingest data sample */ - flb_lib_push(ctx, in_ffd, (char *) K8S_CONTAINER_COMMON_DIFF_TAGS, size_one); + flb_lib_push(ctx, in_ffd, (char *) K8S_CONTAINER_COMMON, size_one); + + stackdriver_wait_for_formatter(); /* Enable test mode */ ret = flb_output_set_test(ctx, out_ffd, "formatter", @@ -3683,7 +3762,7 @@ void flb_test_resource_k8s_container_multi_tag_value() flb_lib_push(ctx, in_ffd, (char *) K8S_CONTAINER_COMMON_DIFF_TAGS, size_two); - sleep(2); + stackdriver_wait_for_formatter(); flb_stop(ctx); flb_destroy(ctx); } @@ -3700,7 +3779,7 @@ void flb_test_resource_k8s_container_concurrency() char tag[32]; ctx = flb_create(); - flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + flb_service_set(ctx, "flush", "0.2", "grace", "1", NULL); for (k = 0; k < 5; k++) { in_ffd[k] = flb_input(ctx, (char *) "lib", NULL); @@ -3746,9 +3825,9 @@ void flb_test_resource_k8s_container_custom_tag_prefix() int in_ffd; int out_ffd; - /* Create context, flush every second (some checks omitted here) */ + /* Create context, flush every 200 milliseconds (some checks omitted here) */ ctx = flb_create(); - flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + flb_service_set(ctx, "flush", "0.2", "grace", "1", NULL); /* Lib input mode */ in_ffd = flb_input(ctx, (char *) "lib", NULL); @@ -3777,7 +3856,7 @@ void flb_test_resource_k8s_container_custom_tag_prefix() /* Ingest data sample */ flb_lib_push(ctx, in_ffd, (char *) K8S_CONTAINER_NO_LOCAL_RESOURCE_ID, size); - sleep(2); + stackdriver_wait_for_formatter(); flb_stop(ctx); flb_destroy(ctx); } @@ -3790,9 +3869,9 @@ void flb_test_resource_k8s_container_custom_tag_prefix_with_dot() int in_ffd; int out_ffd; - /* Create context, flush every second (some checks omitted here) */ + /* Create context, flush every 200 milliseconds (some checks omitted here) */ ctx = flb_create(); - flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + flb_service_set(ctx, "flush", "0.2", "grace", "1", NULL); /* Lib input mode */ in_ffd = flb_input(ctx, (char *) "lib", NULL); @@ -3821,7 +3900,7 @@ void flb_test_resource_k8s_container_custom_tag_prefix_with_dot() /* Ingest data sample */ flb_lib_push(ctx, in_ffd, (char *) K8S_CONTAINER_NO_LOCAL_RESOURCE_ID, size); - sleep(2); + stackdriver_wait_for_formatter(); flb_stop(ctx); flb_destroy(ctx); } @@ -3834,9 +3913,9 @@ void flb_test_resource_k8s_container_default_tag_regex() int in_ffd; int out_ffd; - /* Create context, flush every second (some checks omitted here) */ + /* Create context, flush every 200 milliseconds (some checks omitted here) */ ctx = flb_create(); - flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + flb_service_set(ctx, "flush", "0.2", "grace", "1", NULL); /* Lib input mode */ in_ffd = flb_input(ctx, (char *) "lib", NULL); @@ -3866,7 +3945,7 @@ void flb_test_resource_k8s_container_default_tag_regex() /* Ingest data sample */ flb_lib_push(ctx, in_ffd, (char *) K8S_CONTAINER_NO_LOCAL_RESOURCE_ID, size); - sleep(2); + stackdriver_wait_for_formatter(); flb_stop(ctx); flb_destroy(ctx); } @@ -3879,9 +3958,9 @@ void flb_test_resource_k8s_container_custom_k8s_regex() int in_ffd; int out_ffd; - /* Create context, flush every second (some checks omitted here) */ + /* Create context, flush every 200 milliseconds (some checks omitted here) */ ctx = flb_create(); - flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + flb_service_set(ctx, "flush", "0.2", "grace", "1", NULL); /* Lib input mode */ in_ffd = flb_input(ctx, (char *) "lib", NULL); @@ -3911,7 +3990,7 @@ void flb_test_resource_k8s_container_custom_k8s_regex() /* Ingest data sample */ flb_lib_push(ctx, in_ffd, (char *) K8S_CONTAINER_NO_LOCAL_RESOURCE_ID, size); - sleep(2); + stackdriver_wait_for_formatter(); flb_stop(ctx); flb_destroy(ctx); } @@ -3924,9 +4003,9 @@ void flb_test_resource_k8s_container_custom_k8s_regex_custom_prefix() int in_ffd; int out_ffd; - /* Create context, flush every second (some checks omitted here) */ + /* Create context, flush every 200 milliseconds (some checks omitted here) */ ctx = flb_create(); - flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + flb_service_set(ctx, "flush", "0.2", "grace", "1", NULL); /* Lib input mode */ in_ffd = flb_input(ctx, (char *) "lib", NULL); @@ -3957,7 +4036,7 @@ void flb_test_resource_k8s_container_custom_k8s_regex_custom_prefix() /* Ingest data sample */ flb_lib_push(ctx, in_ffd, (char *) K8S_CONTAINER_NO_LOCAL_RESOURCE_ID, size); - sleep(2); + stackdriver_wait_for_formatter(); flb_stop(ctx); flb_destroy(ctx); } @@ -3970,9 +4049,9 @@ void flb_test_resource_k8s_cluster_no_local_resource_id() int in_ffd; int out_ffd; - /* Create context, flush every second (some checks omitted here) */ + /* Create context, flush every 200 milliseconds (some checks omitted here) */ ctx = flb_create(); - flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + flb_service_set(ctx, "flush", "0.2", "grace", "1", NULL); /* Lib input mode */ in_ffd = flb_input(ctx, (char *) "lib", NULL); @@ -4000,7 +4079,7 @@ void flb_test_resource_k8s_cluster_no_local_resource_id() /* Ingest data sample */ flb_lib_push(ctx, in_ffd, (char *) K8S_CLUSTER_NO_LOCAL_RESOURCE_ID, size); - sleep(2); + stackdriver_wait_for_formatter(); flb_stop(ctx); flb_destroy(ctx); } @@ -4014,9 +4093,9 @@ void flb_test_resource_k8s_node_common() int in_ffd; int out_ffd; - /* Create context, flush every second (some checks omitted here) */ + /* Create context, flush every 200 milliseconds (some checks omitted here) */ ctx = flb_create(); - flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + flb_service_set(ctx, "flush", "0.2", "grace", "1", NULL); /* Lib input mode */ in_ffd = flb_input(ctx, (char *) "lib", NULL); @@ -4044,7 +4123,7 @@ void flb_test_resource_k8s_node_common() /* Ingest data sample */ flb_lib_push(ctx, in_ffd, (char *) K8S_NODE_COMMON, size); - sleep(2); + stackdriver_wait_for_formatter(); flb_stop(ctx); flb_destroy(ctx); } @@ -4057,9 +4136,9 @@ void flb_test_resource_k8s_pod_common() int in_ffd; int out_ffd; - /* Create context, flush every second (some checks omitted here) */ + /* Create context, flush every 200 milliseconds (some checks omitted here) */ ctx = flb_create(); - flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + flb_service_set(ctx, "flush", "0.2", "grace", "1", NULL); /* Lib input mode */ in_ffd = flb_input(ctx, (char *) "lib", NULL); @@ -4087,7 +4166,7 @@ void flb_test_resource_k8s_pod_common() /* Ingest data sample */ flb_lib_push(ctx, in_ffd, (char *) K8S_POD_COMMON, size); - sleep(2); + stackdriver_wait_for_formatter(); flb_stop(ctx); flb_destroy(ctx); } @@ -4100,9 +4179,9 @@ void flb_test_default_labels() int in_ffd; int out_ffd; - /* Create context, flush every second (some checks omitted here) */ + /* Create context, flush every 200 milliseconds (some checks omitted here) */ ctx = flb_create(); - flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + flb_service_set(ctx, "flush", "0.2", "grace", "1", NULL); /* Lib input mode */ in_ffd = flb_input(ctx, (char *) "lib", NULL); @@ -4128,7 +4207,7 @@ void flb_test_default_labels() /* Ingest data sample */ flb_lib_push(ctx, in_ffd, (char *) DEFAULT_LABELS, size); - sleep(2); + stackdriver_wait_for_formatter(); flb_stop(ctx); flb_destroy(ctx); } @@ -4141,9 +4220,9 @@ void flb_test_custom_labels() int in_ffd; int out_ffd; - /* Create context, flush every second (some checks omitted here) */ + /* Create context, flush every 200 milliseconds (some checks omitted here) */ ctx = flb_create(); - flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + flb_service_set(ctx, "flush", "0.2", "grace", "1", NULL); /* Lib input mode */ in_ffd = flb_input(ctx, (char *) "lib", NULL); @@ -4170,7 +4249,7 @@ void flb_test_custom_labels() /* Ingest data sample */ flb_lib_push(ctx, in_ffd, (char *) CUSTOM_LABELS, size); - sleep(2); + stackdriver_wait_for_formatter(); flb_stop(ctx); flb_destroy(ctx); } @@ -4183,9 +4262,9 @@ void flb_test_config_labels_conflict() int in_ffd; int out_ffd; - /* Create context, flush every second (some checks omitted here) */ + /* Create context, flush every 200 milliseconds (some checks omitted here) */ ctx = flb_create(); - flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + flb_service_set(ctx, "flush", "0.2", "grace", "1", NULL); /* Lib input mode */ in_ffd = flb_input(ctx, (char *) "lib", NULL); @@ -4212,7 +4291,7 @@ void flb_test_config_labels_conflict() /* Ingest data sample */ flb_lib_push(ctx, in_ffd, (char *) DEFAULT_LABELS, size); - sleep(2); + stackdriver_wait_for_formatter(); flb_stop(ctx); flb_destroy(ctx); } @@ -4225,9 +4304,9 @@ void flb_test_config_labels_no_conflict() int in_ffd; int out_ffd; - /* Create context, flush every second (some checks omitted here) */ + /* Create context, flush every 200 milliseconds (some checks omitted here) */ ctx = flb_create(); - flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + flb_service_set(ctx, "flush", "0.2", "grace", "1", NULL); /* Lib input mode */ in_ffd = flb_input(ctx, (char *) "lib", NULL); @@ -4254,7 +4333,7 @@ void flb_test_config_labels_no_conflict() /* Ingest data sample */ flb_lib_push(ctx, in_ffd, (char *) DEFAULT_LABELS, size); - sleep(2); + stackdriver_wait_for_formatter(); flb_stop(ctx); flb_destroy(ctx); } @@ -4267,9 +4346,9 @@ void flb_test_default_labels_k8s_resource_type() int in_ffd; int out_ffd; - /* Create context, flush every second (some checks omitted here) */ + /* Create context, flush every 200 milliseconds (some checks omitted here) */ ctx = flb_create(); - flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + flb_service_set(ctx, "flush", "0.2", "grace", "1", NULL); /* Lib input mode */ in_ffd = flb_input(ctx, (char *) "lib", NULL); @@ -4297,7 +4376,7 @@ void flb_test_default_labels_k8s_resource_type() /* Ingest data sample */ flb_lib_push(ctx, in_ffd, (char *) DEFAULT_LABELS_K8S_RESOURCE_TYPE, size); - sleep(2); + stackdriver_wait_for_formatter(); flb_stop(ctx); flb_destroy(ctx); } @@ -4310,9 +4389,9 @@ void flb_test_resource_labels_one_field() int in_ffd; int out_ffd; - /* Create context, flush every second (some checks omitted here) */ + /* Create context, flush every 200 milliseconds (some checks omitted here) */ ctx = flb_create(); - flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + flb_service_set(ctx, "flush", "0.2", "grace", "1", NULL); /* Lib input mode */ in_ffd = flb_input(ctx, (char *) "lib", NULL); @@ -4338,7 +4417,7 @@ void flb_test_resource_labels_one_field() /* Ingest data sample */ flb_lib_push(ctx, in_ffd, (char *) ONE_FIELD, size); - sleep(2); + stackdriver_wait_for_formatter(); flb_stop(ctx); flb_destroy(ctx); } @@ -4351,9 +4430,9 @@ void flb_test_resource_labels_plaintext() int in_ffd; int out_ffd; - /* Create context, flush every second (some checks omitted here) */ + /* Create context, flush every 200 milliseconds (some checks omitted here) */ ctx = flb_create(); - flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + flb_service_set(ctx, "flush", "0.2", "grace", "1", NULL); /* Lib input mode */ in_ffd = flb_input(ctx, (char *) "lib", NULL); @@ -4379,7 +4458,7 @@ void flb_test_resource_labels_plaintext() /* Ingest data sample */ flb_lib_push(ctx, in_ffd, (char *) ONE_FIELD, size); - sleep(2); + stackdriver_wait_for_formatter(); flb_stop(ctx); flb_destroy(ctx); } @@ -4392,9 +4471,9 @@ void flb_test_resource_labels_multiple_fields() int in_ffd; int out_ffd; - /* Create context, flush every second (some checks omitted here) */ + /* Create context, flush every 200 milliseconds (some checks omitted here) */ ctx = flb_create(); - flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + flb_service_set(ctx, "flush", "0.2", "grace", "1", NULL); /* Lib input mode */ in_ffd = flb_input(ctx, (char *) "lib", NULL); @@ -4420,7 +4499,7 @@ void flb_test_resource_labels_multiple_fields() /* Ingest data sample */ flb_lib_push(ctx, in_ffd, (char *) MULTIPLE_FIELDS, size); - sleep(2); + stackdriver_wait_for_formatter(); flb_stop(ctx); flb_destroy(ctx); } @@ -4433,9 +4512,9 @@ void flb_test_resource_labels_nested_fields() int in_ffd; int out_ffd; - /* Create context, flush every second (some checks omitted here) */ + /* Create context, flush every 200 milliseconds (some checks omitted here) */ ctx = flb_create(); - flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + flb_service_set(ctx, "flush", "0.2", "grace", "1", NULL); /* Lib input mode */ in_ffd = flb_input(ctx, (char *) "lib", NULL); @@ -4461,7 +4540,7 @@ void flb_test_resource_labels_nested_fields() /* Ingest data sample */ flb_lib_push(ctx, in_ffd, (char *) NESTED_FIELDS, size); - sleep(2); + stackdriver_wait_for_formatter(); flb_stop(ctx); flb_destroy(ctx); } @@ -4474,9 +4553,9 @@ void flb_test_resource_labels_layered_nested_fields() int in_ffd; int out_ffd; - /* Create context, flush every second (some checks omitted here) */ + /* Create context, flush every 200 milliseconds (some checks omitted here) */ ctx = flb_create(); - flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + flb_service_set(ctx, "flush", "0.2", "grace", "1", NULL); /* Lib input mode */ in_ffd = flb_input(ctx, (char *) "lib", NULL); @@ -4503,7 +4582,7 @@ void flb_test_resource_labels_layered_nested_fields() /* Ingest data sample */ flb_lib_push(ctx, in_ffd, (char *) LAYERED_NESTED_FIELDS, size); - sleep(2); + stackdriver_wait_for_formatter(); flb_stop(ctx); flb_destroy(ctx); } @@ -4516,9 +4595,9 @@ void flb_test_resource_labels_original_does_not_exist() int in_ffd; int out_ffd; - /* Create context, flush every second (some checks omitted here) */ + /* Create context, flush every 200 milliseconds (some checks omitted here) */ ctx = flb_create(); - flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + flb_service_set(ctx, "flush", "0.2", "grace", "1", NULL); /* Lib input mode */ in_ffd = flb_input(ctx, (char *) "lib", NULL); @@ -4544,7 +4623,7 @@ void flb_test_resource_labels_original_does_not_exist() /* Ingest data sample */ flb_lib_push(ctx, in_ffd, (char *) ONE_FIELD, size); - sleep(2); + stackdriver_wait_for_formatter(); flb_stop(ctx); flb_destroy(ctx); } @@ -4557,9 +4636,9 @@ void flb_test_resource_labels_nested_original_does_not_exist() int in_ffd; int out_ffd; - /* Create context, flush every second (some checks omitted here) */ + /* Create context, flush every 200 milliseconds (some checks omitted here) */ ctx = flb_create(); - flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + flb_service_set(ctx, "flush", "0.2", "grace", "1", NULL); /* Lib input mode */ in_ffd = flb_input(ctx, (char *) "lib", NULL); @@ -4585,7 +4664,7 @@ void flb_test_resource_labels_nested_original_does_not_exist() /* Ingest data sample */ flb_lib_push(ctx, in_ffd, (char *) ONE_FIELD, size); - sleep(2); + stackdriver_wait_for_formatter(); flb_stop(ctx); flb_destroy(ctx); } @@ -4598,9 +4677,9 @@ void flb_test_resource_labels_nested_original_partially_exists() int in_ffd; int out_ffd; - /* Create context, flush every second (some checks omitted here) */ + /* Create context, flush every 200 milliseconds (some checks omitted here) */ ctx = flb_create(); - flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + flb_service_set(ctx, "flush", "0.2", "grace", "1", NULL); /* Lib input mode */ in_ffd = flb_input(ctx, (char *) "lib", NULL); @@ -4626,7 +4705,7 @@ void flb_test_resource_labels_nested_original_partially_exists() /* Ingest data sample */ flb_lib_push(ctx, in_ffd, (char *) NESTED_FIELDS, size); - sleep(2); + stackdriver_wait_for_formatter(); flb_stop(ctx); flb_destroy(ctx); } @@ -4639,9 +4718,9 @@ void flb_test_resource_labels_one_field_with_spaces() int in_ffd; int out_ffd; - /* Create context, flush every second (some checks omitted here) */ + /* Create context, flush every 200 milliseconds (some checks omitted here) */ ctx = flb_create(); - flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + flb_service_set(ctx, "flush", "0.2", "grace", "1", NULL); /* Lib input mode */ in_ffd = flb_input(ctx, (char *) "lib", NULL); @@ -4667,7 +4746,7 @@ void flb_test_resource_labels_one_field_with_spaces() /* Ingest data sample */ flb_lib_push(ctx, in_ffd, (char *) ONE_FIELD, size); - sleep(2); + stackdriver_wait_for_formatter(); flb_stop(ctx); flb_destroy(ctx); } @@ -4680,9 +4759,9 @@ void flb_test_resource_labels_multiple_fields_with_spaces() int in_ffd; int out_ffd; - /* Create context, flush every second (some checks omitted here) */ + /* Create context, flush every 200 milliseconds (some checks omitted here) */ ctx = flb_create(); - flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + flb_service_set(ctx, "flush", "0.2", "grace", "1", NULL); /* Lib input mode */ in_ffd = flb_input(ctx, (char *) "lib", NULL); @@ -4708,7 +4787,7 @@ void flb_test_resource_labels_multiple_fields_with_spaces() /* Ingest data sample */ flb_lib_push(ctx, in_ffd, (char *) MULTIPLE_FIELDS, size); - sleep(2); + stackdriver_wait_for_formatter(); flb_stop(ctx); flb_destroy(ctx); } @@ -4721,9 +4800,9 @@ void flb_test_resource_labels_empty_input() int in_ffd; int out_ffd; - /* Create context, flush every second (some checks omitted here) */ + /* Create context, flush every 200 milliseconds (some checks omitted here) */ ctx = flb_create(); - flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + flb_service_set(ctx, "flush", "0.2", "grace", "1", NULL); /* Lib input mode */ in_ffd = flb_input(ctx, (char *) "lib", NULL); @@ -4749,7 +4828,7 @@ void flb_test_resource_labels_empty_input() /* Ingest data sample */ flb_lib_push(ctx, in_ffd, (char *) ONE_FIELD, size); - sleep(2); + stackdriver_wait_for_formatter(); flb_stop(ctx); flb_destroy(ctx); } @@ -4762,9 +4841,9 @@ void flb_test_resource_labels_duplicate_assignment() int in_ffd; int out_ffd; - /* Create context, flush every second (some checks omitted here) */ + /* Create context, flush every 200 milliseconds (some checks omitted here) */ ctx = flb_create(); - flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + flb_service_set(ctx, "flush", "0.2", "grace", "1", NULL); /* Lib input mode */ in_ffd = flb_input(ctx, (char *) "lib", NULL); @@ -4790,7 +4869,7 @@ void flb_test_resource_labels_duplicate_assignment() /* Ingest data sample */ flb_lib_push(ctx, in_ffd, (char *) MULTIPLE_FIELDS, size); - sleep(2); + stackdriver_wait_for_formatter(); flb_stop(ctx); flb_destroy(ctx); } @@ -4803,9 +4882,9 @@ void flb_test_resource_labels_project_id_not_overridden() int in_ffd; int out_ffd; - /* Create context, flush every second (some checks omitted here) */ + /* Create context, flush every 200 milliseconds (some checks omitted here) */ ctx = flb_create(); - flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + flb_service_set(ctx, "flush", "0.2", "grace", "1", NULL); /* Lib input mode */ in_ffd = flb_input(ctx, (char *) "lib", NULL); @@ -4831,7 +4910,7 @@ void flb_test_resource_labels_project_id_not_overridden() /* Ingest data sample */ flb_lib_push(ctx, in_ffd, (char *) ONE_FIELD, size); - sleep(2); + stackdriver_wait_for_formatter(); flb_stop(ctx); flb_destroy(ctx); } @@ -4844,9 +4923,9 @@ void flb_test_resource_labels_has_priority() int in_ffd; int out_ffd; - /* Create context, flush every second (some checks omitted here) */ + /* Create context, flush every 200 milliseconds (some checks omitted here) */ ctx = flb_create(); - flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + flb_service_set(ctx, "flush", "0.2", "grace", "1", NULL); /* Lib input mode */ in_ffd = flb_input(ctx, (char *) "lib", NULL); @@ -4876,7 +4955,7 @@ void flb_test_resource_labels_has_priority() /* Ingest data sample */ flb_lib_push(ctx, in_ffd, (char *) ONE_FIELD, size); - sleep(2); + stackdriver_wait_for_formatter(); flb_stop(ctx); flb_destroy(ctx); } @@ -4889,9 +4968,9 @@ void flb_test_resource_labels_fallsback_when_required_not_specified() int in_ffd; int out_ffd; - /* Create context, flush every second (some checks omitted here) */ + /* Create context, flush every 200 milliseconds (some checks omitted here) */ ctx = flb_create(); - flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + flb_service_set(ctx, "flush", "0.2", "grace", "1", NULL); /* Lib input mode */ in_ffd = flb_input(ctx, (char *) "lib", NULL); @@ -4920,7 +4999,7 @@ void flb_test_resource_labels_fallsback_when_required_not_specified() /* Ingest data sample */ flb_lib_push(ctx, in_ffd, (char *) K8S_CONTAINER_COMMON, size); - sleep(2); + stackdriver_wait_for_formatter(); flb_stop(ctx); flb_destroy(ctx); } @@ -4933,9 +5012,9 @@ void flb_test_resource_labels_fallsback_when_required_partially_specified() int in_ffd; int out_ffd; - /* Create context, flush every second (some checks omitted here) */ + /* Create context, flush every 200 milliseconds (some checks omitted here) */ ctx = flb_create(); - flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + flb_service_set(ctx, "flush", "0.2", "grace", "1", NULL); /* Lib input mode */ in_ffd = flb_input(ctx, (char *) "lib", NULL); @@ -4964,7 +5043,7 @@ void flb_test_resource_labels_fallsback_when_required_partially_specified() /* Ingest data sample */ flb_lib_push(ctx, in_ffd, (char *) K8S_CONTAINER_COMMON, size); - sleep(2); + stackdriver_wait_for_formatter(); flb_stop(ctx); flb_destroy(ctx); } @@ -4977,9 +5056,9 @@ void flb_test_resource_labels_k8s_container() int in_ffd; int out_ffd; - /* Create context, flush every second (some checks omitted here) */ + /* Create context, flush every 200 milliseconds (some checks omitted here) */ ctx = flb_create(); - flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + flb_service_set(ctx, "flush", "0.2", "grace", "1", NULL); /* Lib input mode */ in_ffd = flb_input(ctx, (char *) "lib", NULL); @@ -5007,7 +5086,7 @@ void flb_test_resource_labels_k8s_container() /* Ingest data sample */ flb_lib_push(ctx, in_ffd, (char *) ONE_FIELD, size); - sleep(2); + stackdriver_wait_for_formatter(); flb_stop(ctx); flb_destroy(ctx); } @@ -5020,9 +5099,9 @@ void flb_test_resource_labels_k8s_node() int in_ffd; int out_ffd; - /* Create context, flush every second (some checks omitted here) */ + /* Create context, flush every 200 milliseconds (some checks omitted here) */ ctx = flb_create(); - flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + flb_service_set(ctx, "flush", "0.2", "grace", "1", NULL); /* Lib input mode */ in_ffd = flb_input(ctx, (char *) "lib", NULL); @@ -5050,7 +5129,7 @@ void flb_test_resource_labels_k8s_node() /* Ingest data sample */ flb_lib_push(ctx, in_ffd, (char *) ONE_FIELD, size); - sleep(2); + stackdriver_wait_for_formatter(); flb_stop(ctx); flb_destroy(ctx); } @@ -5063,9 +5142,9 @@ void flb_test_resource_labels_k8s_pod() int in_ffd; int out_ffd; - /* Create context, flush every second (some checks omitted here) */ + /* Create context, flush every 200 milliseconds (some checks omitted here) */ ctx = flb_create(); - flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + flb_service_set(ctx, "flush", "0.2", "grace", "1", NULL); /* Lib input mode */ in_ffd = flb_input(ctx, (char *) "lib", NULL); @@ -5093,7 +5172,7 @@ void flb_test_resource_labels_k8s_pod() /* Ingest data sample */ flb_lib_push(ctx, in_ffd, (char *) ONE_FIELD, size); - sleep(2); + stackdriver_wait_for_formatter(); flb_stop(ctx); flb_destroy(ctx); } @@ -5106,9 +5185,9 @@ void flb_test_resource_labels_generic_node() int in_ffd; int out_ffd; - /* Create context, flush every second (some checks omitted here) */ + /* Create context, flush every 200 milliseconds (some checks omitted here) */ ctx = flb_create(); - flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + flb_service_set(ctx, "flush", "0.2", "grace", "1", NULL); /* Lib input mode */ in_ffd = flb_input(ctx, (char *) "lib", NULL); @@ -5136,7 +5215,7 @@ void flb_test_resource_labels_generic_node() /* Ingest data sample */ flb_lib_push(ctx, in_ffd, (char *) ONE_FIELD, size); - sleep(2); + stackdriver_wait_for_formatter(); flb_stop(ctx); flb_destroy(ctx); } @@ -5149,9 +5228,9 @@ void flb_test_resource_labels_generic_task() int in_ffd; int out_ffd; - /* Create context, flush every second (some checks omitted here) */ + /* Create context, flush every 200 milliseconds (some checks omitted here) */ ctx = flb_create(); - flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + flb_service_set(ctx, "flush", "0.2", "grace", "1", NULL); /* Lib input mode */ in_ffd = flb_input(ctx, (char *) "lib", NULL); @@ -5179,7 +5258,7 @@ void flb_test_resource_labels_generic_task() /* Ingest data sample */ flb_lib_push(ctx, in_ffd, (char *) ONE_FIELD, size); - sleep(2); + stackdriver_wait_for_formatter(); flb_stop(ctx); flb_destroy(ctx); } @@ -5192,9 +5271,9 @@ void flb_test_custom_labels_k8s_resource_type() int in_ffd; int out_ffd; - /* Create context, flush every second (some checks omitted here) */ + /* Create context, flush every 200 milliseconds (some checks omitted here) */ ctx = flb_create(); - flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + flb_service_set(ctx, "flush", "0.2", "grace", "1", NULL); /* Lib input mode */ in_ffd = flb_input(ctx, (char *) "lib", NULL); @@ -5223,7 +5302,7 @@ void flb_test_custom_labels_k8s_resource_type() /* Ingest data sample */ flb_lib_push(ctx, in_ffd, (char *) CUSTOM_LABELS_K8S_RESOURCE_TYPE, size); - sleep(2); + stackdriver_wait_for_formatter(); flb_stop(ctx); flb_destroy(ctx); } @@ -5236,9 +5315,9 @@ void flb_test_resource_k8s_container_no_local_resource_id() int in_ffd; int out_ffd; - /* Create context, flush every second (some checks omitted here) */ + /* Create context, flush every 200 milliseconds (some checks omitted here) */ ctx = flb_create(); - flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + flb_service_set(ctx, "flush", "0.2", "grace", "1", NULL); /* Lib input mode */ in_ffd = flb_input(ctx, (char *) "lib", NULL); @@ -5267,7 +5346,7 @@ void flb_test_resource_k8s_container_no_local_resource_id() /* Ingest data sample */ flb_lib_push(ctx, in_ffd, (char *) K8S_CONTAINER_NO_LOCAL_RESOURCE_ID, size); - sleep(2); + stackdriver_wait_for_formatter(); flb_stop(ctx); flb_destroy(ctx); } @@ -5280,9 +5359,9 @@ void flb_test_resource_k8s_node_no_local_resource_id() int in_ffd; int out_ffd; - /* Create context, flush every second (some checks omitted here) */ + /* Create context, flush every 200 milliseconds (some checks omitted here) */ ctx = flb_create(); - flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + flb_service_set(ctx, "flush", "0.2", "grace", "1", NULL); /* Lib input mode */ in_ffd = flb_input(ctx, (char *) "lib", NULL); @@ -5310,7 +5389,7 @@ void flb_test_resource_k8s_node_no_local_resource_id() /* Ingest data sample */ flb_lib_push(ctx, in_ffd, (char *) K8S_NODE_NO_LOCAL_RESOURCE_ID, size); - sleep(2); + stackdriver_wait_for_formatter(); flb_stop(ctx); flb_destroy(ctx); } @@ -5323,9 +5402,9 @@ void flb_test_resource_k8s_node_custom_k8s_regex_with_dot() int in_ffd; int out_ffd; - /* Create context, flush every second (some checks omitted here) */ + /* Create context, flush every 200 milliseconds (some checks omitted here) */ ctx = flb_create(); - flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + flb_service_set(ctx, "flush", "0.2", "grace", "1", NULL); /* Lib input mode */ in_ffd = flb_input(ctx, (char *) "lib", NULL); @@ -5354,7 +5433,7 @@ void flb_test_resource_k8s_node_custom_k8s_regex_with_dot() /* Ingest data sample */ flb_lib_push(ctx, in_ffd, (char *) K8S_NODE_LOCAL_RESOURCE_ID_WITH_DOT, size); - sleep(2); + stackdriver_wait_for_formatter(); flb_stop(ctx); flb_destroy(ctx); } @@ -5367,9 +5446,9 @@ void flb_test_resource_k8s_node_custom_k8s_regex_with_long_tag() int in_ffd; int out_ffd; - /* Create context, flush every second (some checks omitted here) */ + /* Create context, flush every 200 milliseconds (some checks omitted here) */ ctx = flb_create(); - flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + flb_service_set(ctx, "flush", "0.2", "grace", "1", NULL); /* Lib input mode */ in_ffd = flb_input(ctx, (char *) "lib", NULL); @@ -5398,7 +5477,7 @@ void flb_test_resource_k8s_node_custom_k8s_regex_with_long_tag() /* Ingest data sample */ flb_lib_push(ctx, in_ffd, (char *) K8S_NODE_LOCAL_RESOURCE_ID_WITH_DOT, size); - sleep(2); + stackdriver_wait_for_formatter(); flb_stop(ctx); flb_destroy(ctx); } @@ -5411,9 +5490,9 @@ void flb_test_resource_k8s_pod_no_local_resource_id() int in_ffd; int out_ffd; - /* Create context, flush every second (some checks omitted here) */ + /* Create context, flush every 200 milliseconds (some checks omitted here) */ ctx = flb_create(); - flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + flb_service_set(ctx, "flush", "0.2", "grace", "1", NULL); /* Lib input mode */ in_ffd = flb_input(ctx, (char *) "lib", NULL); @@ -5441,7 +5520,7 @@ void flb_test_resource_k8s_pod_no_local_resource_id() /* Ingest data sample */ flb_lib_push(ctx, in_ffd, (char *) K8S_POD_NO_LOCAL_RESOURCE_ID, size); - sleep(2); + stackdriver_wait_for_formatter(); flb_stop(ctx); flb_destroy(ctx); } @@ -5453,9 +5532,9 @@ void flb_test_multi_entries_severity() int in_ffd; int out_ffd; - /* Create context, flush every second (some checks omitted here) */ + /* Create context, flush every 200 milliseconds (some checks omitted here) */ ctx = flb_create(); - ret = flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + ret = flb_service_set(ctx, "flush", "0.2", "grace", "1", NULL); TEST_CHECK_(ret == 0, "setting service options"); /* Tail input mode */ @@ -5486,7 +5565,7 @@ void flb_test_multi_entries_severity() ret = flb_start(ctx); TEST_CHECK(ret == 0); - sleep(2); + stackdriver_wait_for_formatter(); flb_stop(ctx); flb_destroy(ctx); } @@ -5499,9 +5578,9 @@ void flb_test_source_location_common_case() int in_ffd; int out_ffd; - /* Create context, flush every second (some checks omitted here) */ + /* Create context, flush every 200 milliseconds (some checks omitted here) */ ctx = flb_create(); - flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + flb_service_set(ctx, "flush", "0.2", "grace", "1", NULL); /* Lib input mode */ in_ffd = flb_input(ctx, (char *) "lib", NULL); @@ -5526,7 +5605,7 @@ void flb_test_source_location_common_case() /* Ingest data sample */ flb_lib_push(ctx, in_ffd, (char *) SOURCELOCATION_COMMON_CASE, size); - sleep(2); + stackdriver_wait_for_formatter(); flb_stop(ctx); flb_destroy(ctx); } @@ -5539,9 +5618,9 @@ void flb_test_source_location_line_in_string() int in_ffd; int out_ffd; - /* Create context, flush every second (some checks omitted here) */ + /* Create context, flush every 200 milliseconds (some checks omitted here) */ ctx = flb_create(); - flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + flb_service_set(ctx, "flush", "0.2", "grace", "1", NULL); /* Lib input mode */ in_ffd = flb_input(ctx, (char *) "lib", NULL); @@ -5566,7 +5645,7 @@ void flb_test_source_location_line_in_string() /* Ingest data sample */ flb_lib_push(ctx, in_ffd, (char *) SOURCELOCATION_COMMON_CASE_LINE_IN_STRING, size); - sleep(2); + stackdriver_wait_for_formatter(); flb_stop(ctx); flb_destroy(ctx); } @@ -5579,9 +5658,9 @@ void flb_test_source_location_line_invalid_string() int in_ffd; int out_ffd; - /* Create context, flush every second (some checks omitted here) */ + /* Create context, flush every 200 milliseconds (some checks omitted here) */ ctx = flb_create(); - flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + flb_service_set(ctx, "flush", "0.2", "grace", "1", NULL); /* Lib input mode */ in_ffd = flb_input(ctx, (char *) "lib", NULL); @@ -5606,7 +5685,7 @@ void flb_test_source_location_line_invalid_string() /* Ingest data sample */ flb_lib_push(ctx, in_ffd, (char *) SOURCELOCATION_COMMON_CASE_LINE_INVALID_STRING, size); - sleep(2); + stackdriver_wait_for_formatter(); flb_stop(ctx); flb_destroy(ctx); } @@ -5619,9 +5698,9 @@ void flb_test_empty_source_location() int in_ffd; int out_ffd; - /* Create context, flush every second (some checks omitted here) */ + /* Create context, flush every 200 milliseconds (some checks omitted here) */ ctx = flb_create(); - flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + flb_service_set(ctx, "flush", "0.2", "grace", "1", NULL); /* Lib input mode */ in_ffd = flb_input(ctx, (char *) "lib", NULL); @@ -5646,7 +5725,7 @@ void flb_test_empty_source_location() /* Ingest data sample */ flb_lib_push(ctx, in_ffd, (char *) EMPTY_SOURCELOCATION, size); - sleep(2); + stackdriver_wait_for_formatter(); flb_stop(ctx); flb_destroy(ctx); } @@ -5659,9 +5738,9 @@ void flb_test_source_location_in_string() int in_ffd; int out_ffd; - /* Create context, flush every second (some checks omitted here) */ + /* Create context, flush every 200 milliseconds (some checks omitted here) */ ctx = flb_create(); - flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + flb_service_set(ctx, "flush", "0.2", "grace", "1", NULL); /* Lib input mode */ in_ffd = flb_input(ctx, (char *) "lib", NULL); @@ -5686,7 +5765,7 @@ void flb_test_source_location_in_string() /* Ingest data sample */ flb_lib_push(ctx, in_ffd, (char *) SOURCELOCATION_IN_STRING, size); - sleep(2); + stackdriver_wait_for_formatter(); flb_stop(ctx); flb_destroy(ctx); } @@ -5699,9 +5778,9 @@ void flb_test_source_location_partial_subfields() int in_ffd; int out_ffd; - /* Create context, flush every second (some checks omitted here) */ + /* Create context, flush every 200 milliseconds (some checks omitted here) */ ctx = flb_create(); - flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + flb_service_set(ctx, "flush", "0.2", "grace", "1", NULL); /* Lib input mode */ in_ffd = flb_input(ctx, (char *) "lib", NULL); @@ -5726,7 +5805,7 @@ void flb_test_source_location_partial_subfields() /* Ingest data sample */ flb_lib_push(ctx, in_ffd, (char *) PARTIAL_SOURCELOCATION, size); - sleep(2); + stackdriver_wait_for_formatter(); flb_stop(ctx); flb_destroy(ctx); } @@ -5739,9 +5818,9 @@ void flb_test_source_location_incorrect_type_subfields() int in_ffd; int out_ffd; - /* Create context, flush every second (some checks omitted here) */ + /* Create context, flush every 200 milliseconds (some checks omitted here) */ ctx = flb_create(); - flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + flb_service_set(ctx, "flush", "0.2", "grace", "1", NULL); /* Lib input mode */ in_ffd = flb_input(ctx, (char *) "lib", NULL); @@ -5766,7 +5845,7 @@ void flb_test_source_location_incorrect_type_subfields() /* Ingest data sample */ flb_lib_push(ctx, in_ffd, (char *) SOURCELOCATION_SUBFIELDS_IN_INCORRECT_TYPE, size); - sleep(2); + stackdriver_wait_for_formatter(); flb_stop(ctx); flb_destroy(ctx); } @@ -5779,9 +5858,9 @@ void flb_test_source_location_extra_subfields() int in_ffd; int out_ffd; - /* Create context, flush every second (some checks omitted here) */ + /* Create context, flush every 200 milliseconds (some checks omitted here) */ ctx = flb_create(); - flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + flb_service_set(ctx, "flush", "0.2", "grace", "1", NULL); /* Lib input mode */ in_ffd = flb_input(ctx, (char *) "lib", NULL); @@ -5806,7 +5885,7 @@ void flb_test_source_location_extra_subfields() /* Ingest data sample */ flb_lib_push(ctx, in_ffd, (char *) SOURCELOCATION_EXTRA_SUBFIELDS_EXISTED, size); - sleep(2); + stackdriver_wait_for_formatter(); flb_stop(ctx); flb_destroy(ctx); } @@ -5819,9 +5898,9 @@ void flb_test_http_request_common_case() int in_ffd; int out_ffd; - /* Create context, flush every second (some checks omitted here) */ + /* Create context, flush every 200 milliseconds (some checks omitted here) */ ctx = flb_create(); - flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + flb_service_set(ctx, "flush", "0.2", "grace", "1", NULL); /* Lib input mode */ in_ffd = flb_input(ctx, (char *) "lib", NULL); @@ -5846,7 +5925,7 @@ void flb_test_http_request_common_case() /* Ingest data sample */ flb_lib_push(ctx, in_ffd, (char *) HTTPREQUEST_COMMON_CASE, size); - sleep(2); + stackdriver_wait_for_formatter(); flb_stop(ctx); flb_destroy(ctx); } @@ -5859,9 +5938,9 @@ void flb_test_empty_http_request() int in_ffd; int out_ffd; - /* Create context, flush every second (some checks omitted here) */ + /* Create context, flush every 200 milliseconds (some checks omitted here) */ ctx = flb_create(); - flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + flb_service_set(ctx, "flush", "0.2", "grace", "1", NULL); /* Lib input mode */ in_ffd = flb_input(ctx, (char *) "lib", NULL); @@ -5886,7 +5965,7 @@ void flb_test_empty_http_request() /* Ingest data sample */ flb_lib_push(ctx, in_ffd, (char *) EMPTY_HTTPREQUEST, size); - sleep(2); + stackdriver_wait_for_formatter(); flb_stop(ctx); flb_destroy(ctx); } @@ -5899,9 +5978,9 @@ void flb_test_http_request_in_string() int in_ffd; int out_ffd; - /* Create context, flush every second (some checks omitted here) */ + /* Create context, flush every 200 milliseconds (some checks omitted here) */ ctx = flb_create(); - flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + flb_service_set(ctx, "flush", "0.2", "grace", "1", NULL); /* Lib input mode */ in_ffd = flb_input(ctx, (char *) "lib", NULL); @@ -5926,7 +6005,7 @@ void flb_test_http_request_in_string() /* Ingest data sample */ flb_lib_push(ctx, in_ffd, (char *) HTTPREQUEST_IN_STRING, size); - sleep(2); + stackdriver_wait_for_formatter(); flb_stop(ctx); flb_destroy(ctx); } @@ -5939,9 +6018,9 @@ void flb_test_http_request_partial_subfields() int in_ffd; int out_ffd; - /* Create context, flush every second (some checks omitted here) */ + /* Create context, flush every 200 milliseconds (some checks omitted here) */ ctx = flb_create(); - flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + flb_service_set(ctx, "flush", "0.2", "grace", "1", NULL); /* Lib input mode */ in_ffd = flb_input(ctx, (char *) "lib", NULL); @@ -5966,7 +6045,7 @@ void flb_test_http_request_partial_subfields() /* Ingest data sample */ flb_lib_push(ctx, in_ffd, (char *) PARTIAL_HTTPREQUEST, size); - sleep(2); + stackdriver_wait_for_formatter(); flb_stop(ctx); flb_destroy(ctx); } @@ -5979,9 +6058,9 @@ void flb_test_http_request_incorrect_type_subfields() int in_ffd; int out_ffd; - /* Create context, flush every second (some checks omitted here) */ + /* Create context, flush every 200 milliseconds (some checks omitted here) */ ctx = flb_create(); - flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + flb_service_set(ctx, "flush", "0.2", "grace", "1", NULL); /* Lib input mode */ in_ffd = flb_input(ctx, (char *) "lib", NULL); @@ -6006,7 +6085,7 @@ void flb_test_http_request_incorrect_type_subfields() /* Ingest data sample */ flb_lib_push(ctx, in_ffd, (char *) HTTPREQUEST_SUBFIELDS_IN_INCORRECT_TYPE, size); - sleep(2); + stackdriver_wait_for_formatter(); flb_stop(ctx); flb_destroy(ctx); } @@ -6019,9 +6098,9 @@ void flb_test_http_request_extra_subfields() int in_ffd; int out_ffd; - /* Create context, flush every second (some checks omitted here) */ + /* Create context, flush every 200 milliseconds (some checks omitted here) */ ctx = flb_create(); - flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + flb_service_set(ctx, "flush", "0.2", "grace", "1", NULL); /* Lib input mode */ in_ffd = flb_input(ctx, (char *) "lib", NULL); @@ -6046,7 +6125,7 @@ void flb_test_http_request_extra_subfields() /* Ingest data sample */ flb_lib_push(ctx, in_ffd, (char *) HTTPREQUEST_EXTRA_SUBFIELDS_EXISTED, size); - sleep(2); + stackdriver_wait_for_formatter(); flb_stop(ctx); flb_destroy(ctx); } @@ -6059,9 +6138,9 @@ void flb_test_http_request_latency_common_case() int in_ffd; int out_ffd; - /* Create context, flush every second (some checks omitted here) */ + /* Create context, flush every 200 milliseconds (some checks omitted here) */ ctx = flb_create(); - flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + flb_service_set(ctx, "flush", "0.2", "grace", "1", NULL); /* Lib input mode */ in_ffd = flb_input(ctx, (char *) "lib", NULL); @@ -6086,7 +6165,7 @@ void flb_test_http_request_latency_common_case() /* Ingest data sample */ flb_lib_push(ctx, in_ffd, (char *) HTTPREQUEST_LATENCY_COMMON_CASE, size); - sleep(2); + stackdriver_wait_for_formatter(); flb_stop(ctx); flb_destroy(ctx); } @@ -6099,9 +6178,9 @@ void flb_test_http_request_latency_invalid_spaces() int in_ffd; int out_ffd; - /* Create context, flush every second (some checks omitted here) */ + /* Create context, flush every 200 milliseconds (some checks omitted here) */ ctx = flb_create(); - flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + flb_service_set(ctx, "flush", "0.2", "grace", "1", NULL); /* Lib input mode */ in_ffd = flb_input(ctx, (char *) "lib", NULL); @@ -6126,7 +6205,7 @@ void flb_test_http_request_latency_invalid_spaces() /* Ingest data sample */ flb_lib_push(ctx, in_ffd, (char *) HTTPREQUEST_LATENCY_INVALID_SPACES, size); - sleep(2); + stackdriver_wait_for_formatter(); flb_stop(ctx); flb_destroy(ctx); } @@ -6139,9 +6218,9 @@ void flb_test_http_request_latency_invalid_string() int in_ffd; int out_ffd; - /* Create context, flush every second (some checks omitted here) */ + /* Create context, flush every 200 milliseconds (some checks omitted here) */ ctx = flb_create(); - flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + flb_service_set(ctx, "flush", "0.2", "grace", "1", NULL); /* Lib input mode */ in_ffd = flb_input(ctx, (char *) "lib", NULL); @@ -6166,7 +6245,7 @@ void flb_test_http_request_latency_invalid_string() /* Ingest data sample */ flb_lib_push(ctx, in_ffd, (char *) HTTPREQUEST_LATENCY_INVALID_STRING, size); - sleep(2); + stackdriver_wait_for_formatter(); flb_stop(ctx); flb_destroy(ctx); } @@ -6179,9 +6258,9 @@ void flb_test_http_request_latency_invalid_end() int in_ffd; int out_ffd; - /* Create context, flush every second (some checks omitted here) */ + /* Create context, flush every 200 milliseconds (some checks omitted here) */ ctx = flb_create(); - flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + flb_service_set(ctx, "flush", "0.2", "grace", "1", NULL); /* Lib input mode */ in_ffd = flb_input(ctx, (char *) "lib", NULL); @@ -6206,7 +6285,7 @@ void flb_test_http_request_latency_invalid_end() /* Ingest data sample */ flb_lib_push(ctx, in_ffd, (char *) HTTPREQUEST_LATENCY_INVALID_END, size); - sleep(2); + stackdriver_wait_for_formatter(); flb_stop(ctx); flb_destroy(ctx); } @@ -6219,9 +6298,9 @@ void flb_test_timestamp_format_object_common() int in_ffd; int out_ffd; - /* Create context, flush every second (some checks omitted here) */ + /* Create context, flush every 200 milliseconds (some checks omitted here) */ ctx = flb_create(); - flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + flb_service_set(ctx, "flush", "0.2", "grace", "1", NULL); /* Lib input mode */ in_ffd = flb_input(ctx, (char *) "lib", NULL); @@ -6246,7 +6325,7 @@ void flb_test_timestamp_format_object_common() /* Ingest data sample */ flb_lib_push(ctx, in_ffd, (char *) TIMESTAMP_FORMAT_OBJECT_COMMON_CASE, size); - sleep(2); + stackdriver_wait_for_formatter(); flb_stop(ctx); flb_destroy(ctx); } @@ -6259,9 +6338,9 @@ void flb_test_timestamp_format_object_not_a_map() int in_ffd; int out_ffd; - /* Create context, flush every second (some checks omitted here) */ + /* Create context, flush every 200 milliseconds (some checks omitted here) */ ctx = flb_create(); - flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + flb_service_set(ctx, "flush", "0.2", "grace", "1", NULL); /* Lib input mode */ in_ffd = flb_input(ctx, (char *) "lib", NULL); @@ -6286,7 +6365,7 @@ void flb_test_timestamp_format_object_not_a_map() /* Ingest data sample */ flb_lib_push(ctx, in_ffd, (char *) TIMESTAMP_FORMAT_OBJECT_NOT_A_MAP, size); - sleep(2); + stackdriver_wait_for_formatter(); flb_stop(ctx); flb_destroy(ctx); } @@ -6299,9 +6378,9 @@ void flb_test_timestamp_format_object_missing_subfield() int in_ffd; int out_ffd; - /* Create context, flush every second (some checks omitted here) */ + /* Create context, flush every 200 milliseconds (some checks omitted here) */ ctx = flb_create(); - flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + flb_service_set(ctx, "flush", "0.2", "grace", "1", NULL); /* Lib input mode */ in_ffd = flb_input(ctx, (char *) "lib", NULL); @@ -6326,7 +6405,7 @@ void flb_test_timestamp_format_object_missing_subfield() /* Ingest data sample */ flb_lib_push(ctx, in_ffd, (char *) TIMESTAMP_FORMAT_OBJECT_MISSING_SUBFIELD, size); - sleep(2); + stackdriver_wait_for_formatter(); flb_stop(ctx); flb_destroy(ctx); } @@ -6339,9 +6418,9 @@ void flb_test_timestamp_format_object_incorrect_subfields() int in_ffd; int out_ffd; - /* Create context, flush every second (some checks omitted here) */ + /* Create context, flush every 200 milliseconds (some checks omitted here) */ ctx = flb_create(); - flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + flb_service_set(ctx, "flush", "0.2", "grace", "1", NULL); /* Lib input mode */ in_ffd = flb_input(ctx, (char *) "lib", NULL); @@ -6366,7 +6445,7 @@ void flb_test_timestamp_format_object_incorrect_subfields() /* Ingest data sample */ flb_lib_push(ctx, in_ffd, (char *) TIMESTAMP_FORMAT_OBJECT_INCORRECT_TYPE_SUBFIELDS, size); - sleep(2); + stackdriver_wait_for_formatter(); flb_stop(ctx); flb_destroy(ctx); } @@ -6379,9 +6458,9 @@ void flb_test_timestamp_format_duo_fields_common_case() int in_ffd; int out_ffd; - /* Create context, flush every second (some checks omitted here) */ + /* Create context, flush every 200 milliseconds (some checks omitted here) */ ctx = flb_create(); - flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + flb_service_set(ctx, "flush", "0.2", "grace", "1", NULL); /* Lib input mode */ in_ffd = flb_input(ctx, (char *) "lib", NULL); @@ -6406,7 +6485,7 @@ void flb_test_timestamp_format_duo_fields_common_case() /* Ingest data sample */ flb_lib_push(ctx, in_ffd, (char *) TIMESTAMP_FORMAT_DUO_FIELDS_COMMON_CASE, size); - sleep(2); + stackdriver_wait_for_formatter(); flb_stop(ctx); flb_destroy(ctx); } @@ -6419,9 +6498,9 @@ void flb_test_timestamp_format_duo_fields_missing_nanos() int in_ffd; int out_ffd; - /* Create context, flush every second (some checks omitted here) */ + /* Create context, flush every 200 milliseconds (some checks omitted here) */ ctx = flb_create(); - flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + flb_service_set(ctx, "flush", "0.2", "grace", "1", NULL); /* Lib input mode */ in_ffd = flb_input(ctx, (char *) "lib", NULL); @@ -6446,7 +6525,7 @@ void flb_test_timestamp_format_duo_fields_missing_nanos() /* Ingest data sample */ flb_lib_push(ctx, in_ffd, (char *) TIMESTAMP_FORMAT_DUO_FIELDS_MISSING_NANOS, size); - sleep(2); + stackdriver_wait_for_formatter(); flb_stop(ctx); flb_destroy(ctx); } @@ -6459,9 +6538,9 @@ void flb_test_timestamp_format_duo_fields_incorrect_type() int in_ffd; int out_ffd; - /* Create context, flush every second (some checks omitted here) */ + /* Create context, flush every 200 milliseconds (some checks omitted here) */ ctx = flb_create(); - flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + flb_service_set(ctx, "flush", "0.2", "grace", "1", NULL); /* Lib input mode */ in_ffd = flb_input(ctx, (char *) "lib", NULL); @@ -6486,7 +6565,7 @@ void flb_test_timestamp_format_duo_fields_incorrect_type() /* Ingest data sample */ flb_lib_push(ctx, in_ffd, (char *) TIMESTAMP_FORMAT_DUO_FIELDS_INCORRECT_TYPE, size); - sleep(2); + stackdriver_wait_for_formatter(); flb_stop(ctx); flb_destroy(ctx); } @@ -6499,9 +6578,9 @@ void flb_test_string_text_payload_with_matched_text_payload_key() int in_ffd; int out_ffd; - /* Create context, flush every second (some checks omitted here) */ + /* Create context, flush every 200 milliseconds (some checks omitted here) */ ctx = flb_create(); - flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + flb_service_set(ctx, "flush", "0.2", "grace", "1", NULL); /* Lib input mode */ in_ffd = flb_input(ctx, (char *) "lib", NULL); @@ -6527,7 +6606,7 @@ void flb_test_string_text_payload_with_matched_text_payload_key() /* Ingest data sample */ flb_lib_push(ctx, in_ffd, (char *) STRING_TEXT_PAYLOAD, size); - sleep(2); + stackdriver_wait_for_formatter(); flb_stop(ctx); flb_destroy(ctx); } @@ -6540,9 +6619,9 @@ void flb_test_string_text_payload_with_mismatched_text_payload_key() int in_ffd; int out_ffd; - /* Create context, flush every second (some checks omitted here) */ + /* Create context, flush every 200 milliseconds (some checks omitted here) */ ctx = flb_create(); - flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + flb_service_set(ctx, "flush", "0.2", "grace", "1", NULL); /* Lib input mode */ in_ffd = flb_input(ctx, (char *) "lib", NULL); @@ -6568,7 +6647,7 @@ void flb_test_string_text_payload_with_mismatched_text_payload_key() /* Ingest data sample */ flb_lib_push(ctx, in_ffd, (char *) STRING_TEXT_PAYLOAD, size); - sleep(2); + stackdriver_wait_for_formatter(); flb_stop(ctx); flb_destroy(ctx); } @@ -6581,9 +6660,9 @@ void flb_test_string_text_payload_with_residual_fields() int in_ffd; int out_ffd; - /* Create context, flush every second (some checks omitted here) */ + /* Create context, flush every 200 milliseconds (some checks omitted here) */ ctx = flb_create(); - flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + flb_service_set(ctx, "flush", "0.2", "grace", "1", NULL); /* Lib input mode */ in_ffd = flb_input(ctx, (char *) "lib", NULL); @@ -6609,7 +6688,7 @@ void flb_test_string_text_payload_with_residual_fields() /* Ingest data sample */ flb_lib_push(ctx, in_ffd, (char *) STRING_TEXT_PAYLOAD_WITH_RESIDUAL_FIELDS, size); - sleep(2); + stackdriver_wait_for_formatter(); flb_stop(ctx); flb_destroy(ctx); } @@ -6622,9 +6701,9 @@ void flb_test_non_scalar_payload_with_residual_fields() int in_ffd; int out_ffd; - /* Create context, flush every second (some checks omitted here) */ + /* Create context, flush every 200 milliseconds (some checks omitted here) */ ctx = flb_create(); - flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + flb_service_set(ctx, "flush", "0.2", "grace", "1", NULL); /* Lib input mode */ in_ffd = flb_input(ctx, (char *) "lib", NULL); @@ -6650,7 +6729,7 @@ void flb_test_non_scalar_payload_with_residual_fields() /* Ingest data sample */ flb_lib_push(ctx, in_ffd, (char *) NON_SCALAR_PAYLOAD_WITH_RESIDUAL_FIELDS, size); - sleep(2); + stackdriver_wait_for_formatter(); flb_stop(ctx); flb_destroy(ctx); } diff --git a/tests/runtime/out_syslog.c b/tests/runtime/out_syslog.c index 6bea4ec7b62..a985ecb8f17 100644 --- a/tests/runtime/out_syslog.c +++ b/tests/runtime/out_syslog.c @@ -28,6 +28,8 @@ #include #include "flb_tests_runtime.h" +#define UTF8_BOM "\xEF\xBB\xBF" + struct test_ctx { flb_ctx_t *flb; /* Fluent Bit library context */ int i_ffd; /* Input fd */ @@ -213,7 +215,7 @@ void flb_test_severity_key_rfc5424() size_t size = strlen(buf); char *expected_strs[] = {"hello world", "1970-01-01T00:00:01.000000Z", "<13>" /* 1(user-level messages) * 8 + 5(severity) */, - "<13>1 1970-01-01T00:00:01.000000Z - - - - - hello world"}; + "<13>1 1970-01-01T00:00:01.000000Z - - - - - " UTF8_BOM "hello world"}; struct str_list expected = { .size = sizeof(expected_strs)/sizeof(char*), .lists = &expected_strs[0], @@ -270,7 +272,7 @@ void flb_test_severity_preset_rfc5424() size_t size = strlen(buf); char *expected_strs[] = {"hello world", "1970-01-01T00:00:01.000000Z", "<13>" /* 1(user-level messages) * 8 + 5(severity) */, - "<13>1 1970-01-01T00:00:01.000000Z - - - - - hello world"}; + "<13>1 1970-01-01T00:00:01.000000Z - - - - - " UTF8_BOM "hello world"}; struct str_list expected = { .size = sizeof(expected_strs)/sizeof(char*), .lists = &expected_strs[0], @@ -438,7 +440,7 @@ void flb_test_facility_key_rfc5424() size_t size = strlen(buf); char *expected_strs[] = {"hello world", "1970-01-01T00:00:01.000000Z", "<110>" /* 13(log audit) * 8 + 6(default severity) */, - "<110>1 1970-01-01T00:00:01.000000Z - - - - - hello world"}; + "<110>1 1970-01-01T00:00:01.000000Z - - - - - " UTF8_BOM "hello world"}; struct str_list expected = { .size = sizeof(expected_strs)/sizeof(char*), .lists = &expected_strs[0], @@ -494,7 +496,7 @@ void flb_test_facility_preset_rfc5424() size_t size = strlen(buf); char *expected_strs[] = {"hello world", "1970-01-01T00:00:01.000000Z", "<110>" /* 13(log audit) * 8 + 6(default severity) */, - "<110>1 1970-01-01T00:00:01.000000Z - - - - - hello world"}; + "<110>1 1970-01-01T00:00:01.000000Z - - - - - " UTF8_BOM "hello world"}; struct str_list expected = { .size = sizeof(expected_strs)/sizeof(char*), .lists = &expected_strs[0], @@ -662,7 +664,7 @@ void flb_test_severity_facility_key_rfc5424() size_t size = strlen(buf); char *expected_strs[] = {"hello world", "1970-01-01T00:00:01.000000Z", "<109>" /* 13(log audit) * 8 + 5(severity) */, - "<109>1 1970-01-01T00:00:01.000000Z - - - - - hello world"}; + "<109>1 1970-01-01T00:00:01.000000Z - - - - - " UTF8_BOM "hello world"}; struct str_list expected = { .size = sizeof(expected_strs)/sizeof(char*), .lists = &expected_strs[0], @@ -776,7 +778,7 @@ void flb_test_hostname_key_rfc5424() size_t size = strlen(buf); char *expected_strs[] = {"hello world", "1970-01-01T00:00:01.000000Z", "localhost", - "<14>1 1970-01-01T00:00:01.000000Z localhost - - - - hello world"}; + "<14>1 1970-01-01T00:00:01.000000Z localhost - - - - " UTF8_BOM "hello world"}; struct str_list expected = { .size = sizeof(expected_strs)/sizeof(char*), .lists = &expected_strs[0], @@ -832,7 +834,7 @@ void flb_test_hostname_preset_rfc5424() size_t size = strlen(buf); char *expected_strs[] = {"hello world", "1970-01-01T00:00:01.000000Z", "localhost", - "<14>1 1970-01-01T00:00:01.000000Z localhost - - - - hello world"}; + "<14>1 1970-01-01T00:00:01.000000Z localhost - - - - " UTF8_BOM "hello world"}; struct str_list expected = { .size = sizeof(expected_strs)/sizeof(char*), .lists = &expected_strs[0], @@ -1000,7 +1002,7 @@ void flb_test_appname_key_rfc5424() size_t size = strlen(buf); char *expected_strs[] = {"hello world", "1970-01-01T00:00:01.000000Z", "fluent-bit", - "<14>1 1970-01-01T00:00:01.000000Z - fluent-bit - - - hello world"}; + "<14>1 1970-01-01T00:00:01.000000Z - fluent-bit - - - " UTF8_BOM "hello world"}; struct str_list expected = { .size = sizeof(expected_strs)/sizeof(char*), .lists = &expected_strs[0], @@ -1056,7 +1058,7 @@ void flb_test_appname_preset_rfc5424() size_t size = strlen(buf); char *expected_strs[] = {"hello world", "1970-01-01T00:00:01.000000Z", "fluent-bit", - "<14>1 1970-01-01T00:00:01.000000Z - fluent-bit - - - hello world"}; + "<14>1 1970-01-01T00:00:01.000000Z - fluent-bit - - - " UTF8_BOM "hello world"}; struct str_list expected = { .size = sizeof(expected_strs)/sizeof(char*), .lists = &expected_strs[0], @@ -1224,7 +1226,7 @@ void flb_test_procid_key_rfc5424() size_t size = strlen(buf); char *expected_strs[] = {"hello world", "1970-01-01T00:00:01.000000Z", "1234", - "<14>1 1970-01-01T00:00:01.000000Z - - 1234 - - hello world"}; + "<14>1 1970-01-01T00:00:01.000000Z - - 1234 - - " UTF8_BOM "hello world"}; struct str_list expected = { .size = sizeof(expected_strs)/sizeof(char*), .lists = &expected_strs[0], @@ -1280,7 +1282,7 @@ void flb_test_procid_preset_rfc5424() size_t size = strlen(buf); char *expected_strs[] = {"hello world", "1970-01-01T00:00:01.000000Z", "1234", - "<14>1 1970-01-01T00:00:01.000000Z - - 1234 - - hello world"}; + "<14>1 1970-01-01T00:00:01.000000Z - - 1234 - - " UTF8_BOM "hello world"}; struct str_list expected = { .size = sizeof(expected_strs)/sizeof(char*), .lists = &expected_strs[0], @@ -1336,7 +1338,7 @@ void flb_test_msgid_key_rfc5424() size_t size = strlen(buf); char *expected_strs[] = {"hello world", "1970-01-01T00:00:01.000000Z", "TCPIN", - "<14>1 1970-01-01T00:00:01.000000Z - - - TCPIN - hello world"}; + "<14>1 1970-01-01T00:00:01.000000Z - - - TCPIN - " UTF8_BOM "hello world"}; struct str_list expected = { .size = sizeof(expected_strs)/sizeof(char*), .lists = &expected_strs[0], @@ -1392,7 +1394,7 @@ void flb_test_msgid_preset_rfc5424() size_t size = strlen(buf); char *expected_strs[] = {"hello world", "1970-01-01T00:00:01.000000Z", "TCPIN", - "<14>1 1970-01-01T00:00:01.000000Z - - - TCPIN - hello world"}; + "<14>1 1970-01-01T00:00:01.000000Z - - - TCPIN - " UTF8_BOM "hello world"}; struct str_list expected = { .size = sizeof(expected_strs)/sizeof(char*), .lists = &expected_strs[0], @@ -1448,7 +1450,7 @@ void flb_test_sd_key_rfc5424() size_t size = strlen(buf); char *expected_strs[] = {"hello world", "1970-01-01T00:00:01.000000Z", - "<14>1 1970-01-01T00:00:01.000000Z - - - - [sd_key logtype=\"access\" clustername=\"mycluster\" namespace=\"mynamespace\"] hello world"}; + "<14>1 1970-01-01T00:00:01.000000Z - - - - [sd_key logtype=\"access\" clustername=\"mycluster\" namespace=\"mynamespace\"] " UTF8_BOM "hello world"}; struct str_list expected = { .size = sizeof(expected_strs)/sizeof(char*), .lists = &expected_strs[0], @@ -1504,7 +1506,7 @@ void flb_test_allow_longer_sd_id_rfc5424() size_t size = strlen(buf); char *expected_strs[] = {"hello world", "1970-01-01T00:00:01.000000Z", - "<14>1 1970-01-01T00:00:01.000000Z - - - - [sd_key_that_is_longer_than_32_characters logtype_that_is_longer_than_32_characters=\"access\" clustername=\"mycluster\" namespace=\"mynamespace\"] hello world"}; + "<14>1 1970-01-01T00:00:01.000000Z - - - - [sd_key_that_is_longer_than_32_characters logtype_that_is_longer_than_32_characters=\"access\" clustername=\"mycluster\" namespace=\"mynamespace\"] " UTF8_BOM "hello world"}; struct str_list expected = { .size = sizeof(expected_strs)/sizeof(char*), .lists = &expected_strs[0], @@ -1561,7 +1563,7 @@ void flb_test_malformed_longer_sd_id_rfc5424() size_t size = strlen(buf); char *expected_strs[] = {"hello world", "1970-01-01T00:00:01.000000Z", - "<14>1 1970-01-01T00:00:01.000000Z - - - - [sd_key_that_is_longer_than_32_ch logtype_that_is_longer_than_32_c=\"access\" clustername=\"mycluster\" namespace=\"mynamespace\"] hello world"}; + "<14>1 1970-01-01T00:00:01.000000Z - - - - [sd_key_that_is_longer_than_32_ch logtype_that_is_longer_than_32_c=\"access\" clustername=\"mycluster\" namespace=\"mynamespace\"] " UTF8_BOM "hello world"}; struct str_list expected = { .size = sizeof(expected_strs)/sizeof(char*), .lists = &expected_strs[0], @@ -1631,7 +1633,9 @@ void flb_test_udp_mode_rejects_tls() TEST_MSG("expected startup failure for mode=udp with tls=on"); } - test_ctx_destroy(ctx); + /* flb_start failed, so there is no running engine to stop. */ + flb_destroy(ctx->flb); + flb_free(ctx); } TEST_LIST = { diff --git a/tests/runtime/out_tcp.c b/tests/runtime/out_tcp.c index e1668a23c06..1847ebef2d1 100644 --- a/tests/runtime/out_tcp.c +++ b/tests/runtime/out_tcp.c @@ -70,12 +70,30 @@ static void clear_output_num() set_output_num(0); } +static int wait_for_output_num(int expected, uint32_t timeout_ms) +{ + uint32_t elapsed = 0; + + while (elapsed < timeout_ms) { + if (get_output_num() >= expected) { + return 0; + } + + flb_time_msleep(10); + elapsed += 10; + } + + return -1; +} + struct str_list { size_t size; char **lists; int *matches; int require_json_array; char *joined; + char *expected_payload; + flb_sds_t accumulated_payload; }; static int validate_json_array_payload(flb_sds_t out_line, struct str_list *l) @@ -98,6 +116,29 @@ static int validate_json_array_payload(flb_sds_t out_line, struct str_list *l) return 0; } +static int accumulate_payload(flb_sds_t out_line, struct str_list *l) +{ + int ret; + + if (l->accumulated_payload == NULL) { + l->accumulated_payload = flb_sds_create_size(flb_sds_len(out_line)); + if (!TEST_CHECK(l->accumulated_payload != NULL)) { + TEST_MSG("could not allocate accumulated payload"); + return -1; + } + } + + ret = flb_sds_cat_safe(&l->accumulated_payload, + out_line, + flb_sds_len(out_line)); + if (!TEST_CHECK(ret == 0)) { + TEST_MSG("could not append accumulated payload"); + return -1; + } + + return 0; +} + /* Callback to check expected results */ static void cb_check_str_list(void *ctx, int ffd, int res_ret, void *res_data, size_t res_size, void *data) @@ -127,17 +168,22 @@ static void cb_check_str_list(void *ctx, int ffd, int res_ret, validate_json_array_payload(out_line, l); } - for (i=0; isize; i++) { - p = strstr(out_line, l->lists[i]); - if (l->matches != NULL) { - if (p != NULL) { - l->matches[i]++; + if (l->expected_payload != NULL) { + accumulate_payload(out_line, l); + } + else { + for (i=0; isize; i++) { + p = strstr(out_line, l->lists[i]); + if (l->matches != NULL) { + if (p != NULL) { + l->matches[i]++; + } + continue; } - continue; - } - if (!TEST_CHECK(p != NULL)) { - TEST_MSG(" Got :%s\n expect:%s", out_line, l->lists[i]); + if (!TEST_CHECK(p != NULL)) { + TEST_MSG(" Got :%s\n expect:%s", out_line, l->lists[i]); + } } } set_output_num(num+1); @@ -160,6 +206,34 @@ static int check_str_list_matches(struct str_list *l) return ret; } +static int check_accumulated_payload(struct str_list *l) +{ + int ret = 0; + size_t expected_size; + size_t actual_size; + + if (!TEST_CHECK(l->accumulated_payload != NULL)) { + TEST_MSG("no payload was accumulated"); + return -1; + } + + expected_size = strlen(l->expected_payload); + actual_size = flb_sds_len(l->accumulated_payload); + if (!TEST_CHECK(actual_size == expected_size && + memcmp(l->accumulated_payload, + l->expected_payload, + expected_size) == 0)) { + TEST_MSG(" Got :%s\n expect:%s", + l->accumulated_payload, l->expected_payload); + ret = -1; + } + + flb_sds_destroy(l->accumulated_payload); + l->accumulated_payload = NULL; + + return ret; +} + static int msgpack_strncmp(char* str, size_t str_len, msgpack_object obj) { int ret = -1; @@ -562,10 +636,10 @@ void flb_test_format_json_stream() char *buf2 = "[2, {\"msg\":\"hello world\"}]"; size_t size2 = strlen(buf2); - char *expected_strs[] = {"{\"date\":1.0,\"msg\":\"hello world\"}{\"date\":2.0,\"msg\":\"hello world\"}"}; struct str_list expected = { - .size = sizeof(expected_strs)/sizeof(char*), - .lists = &expected_strs[0], + .expected_payload = + "{\"date\":1.0,\"msg\":\"hello world\"}" + "{\"date\":2.0,\"msg\":\"hello world\"}", }; clear_output_num(); @@ -604,6 +678,7 @@ void flb_test_format_json_stream() if (!TEST_CHECK(num > 0)) { TEST_MSG("no outputs"); } + check_accumulated_payload(&expected); test_ctx_destroy(ctx); } @@ -619,10 +694,10 @@ void flb_test_format_json_lines() char *buf2 = "[2, {\"msg\":\"hello world\"}]"; size_t size2 = strlen(buf2); - char *expected_strs[] = {"{\"date\":1.0,\"msg\":\"hello world\"}\n{\"date\":2.0,\"msg\":\"hello world\"}"}; struct str_list expected = { - .size = sizeof(expected_strs)/sizeof(char*), - .lists = &expected_strs[0], + .expected_payload = + "{\"date\":1.0,\"msg\":\"hello world\"}\n" + "{\"date\":2.0,\"msg\":\"hello world\"}\n", }; clear_output_num(); @@ -651,6 +726,13 @@ void flb_test_format_json_lines() /* Ingest data sample */ ret = flb_lib_push(ctx->flb, ctx->i_ffd, (char *) buf1, size1); TEST_CHECK(ret >= 0); + + /* Make the second record use a separate input chunk. */ + ret = wait_for_output_num(1, 2000); + if (!TEST_CHECK(ret == 0)) { + TEST_MSG("first json_lines record was not formatted before timeout"); + } + ret = flb_lib_push(ctx->flb, ctx->i_ffd, (char *) buf2, size2); TEST_CHECK(ret >= 0); @@ -661,6 +743,7 @@ void flb_test_format_json_lines() if (!TEST_CHECK(num > 0)) { TEST_MSG("no outputs"); } + check_accumulated_payload(&expected); test_ctx_destroy(ctx); } diff --git a/tests/runtime/processor_cumulative_to_delta.c b/tests/runtime/processor_cumulative_to_delta.c index da65c7479a2..e4e97fd3e82 100644 --- a/tests/runtime/processor_cumulative_to_delta.c +++ b/tests/runtime/processor_cumulative_to_delta.c @@ -1,7 +1,7 @@ /* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ #include -#include +#include #include #include diff --git a/tests/runtime/processor_labels.c b/tests/runtime/processor_labels.c index c58e41c9e47..c4f9d021723 100644 --- a/tests/runtime/processor_labels.c +++ b/tests/runtime/processor_labels.c @@ -61,8 +61,9 @@ static int cb_insert_labels(void *record, size_t size, void *data) text = cmt_encode_text_create(cmt); TEST_CHECK(text != NULL); - TEST_CHECK(count_metrics_matches(text, "static=\"ok\"") == 9); - TEST_CHECK(count_metrics_matches(text, "dynamic=\"test\"") == 9); + /* 3 counters, 1 gauge, 2 summaries, and 2 histograms */ + TEST_CHECK(count_metrics_matches(text, "static=\"ok\"") == 8); + TEST_CHECK(count_metrics_matches(text, "dynamic=\"test\"") == 8); if (record) { flb_free(record); @@ -270,11 +271,11 @@ static int cb_upsert_labels(void *record, size_t size, void *data) text = cmt_encode_text_create(cmt); TEST_CHECK(text != NULL); - /* it should only update the metrics which contains a label with the name 'hostname' */ - TEST_CHECK(count_metrics_matches(text, "hostname=\"updated-2\"") == 8); + /* The unlabeled counter has no existing hostname label to update. */ + TEST_CHECK(count_metrics_matches(text, "hostname=\"updated-2\"") == 7); - /* now check the updated value of 'app' which should include the dynamic tag */ - TEST_CHECK(count_metrics_matches(text, "dynamic-host=\"test\"") == 9); + /* All eight legitimate samples receive the new dynamic label. */ + TEST_CHECK(count_metrics_matches(text, "dynamic-host=\"test\"") == 8); if (record) { flb_free(record); } diff --git a/tests/runtime_shell/CMakeLists.txt b/tests/runtime_shell/CMakeLists.txt index 09ad71ae9c0..34cacd1c5ff 100644 --- a/tests/runtime_shell/CMakeLists.txt +++ b/tests/runtime_shell/CMakeLists.txt @@ -1,42 +1,107 @@ -configure_file( - "${CMAKE_CURRENT_SOURCE_DIR}/runtime_shell.env.in" - "${CMAKE_CURRENT_SOURCE_DIR}/runtime_shell.env" +set(RUNTIME_SHELL_ENVIRONMENT + "FLB_ROOT=${PROJECT_SOURCE_DIR};\ +FLB_RUNTIME_SHELL_PATH=${CMAKE_CURRENT_SOURCE_DIR};\ +FLB_RUNTIME_SHELL_CONF=${CMAKE_CURRENT_SOURCE_DIR}/conf;\ +FLB_BIN=${CMAKE_BINARY_DIR}/bin/fluent-bit${CMAKE_EXECUTABLE_SUFFIX};\ +FLB_BUILD=${CMAKE_BINARY_DIR}" ) -set(UNIT_TESTS_SH - custom_calyptia.sh - dry_run_invalid_property.sh - in_dummy_expect.sh - in_tail_expect.sh - in_http_tls_expect.sh - in_syslog_tcp_tls_expect.sh - in_syslog_tcp_plaintext_expect.sh - in_syslog_udp_plaintext_expect.sh - in_syslog_uds_dgram_plaintext_expect.sh - in_syslog_uds_stream_plaintext_expect.sh - processor_conditional.sh - processor_invalid.sh - ) +if(WIN32) + find_program(POWERSHELL_EXECUTABLE NAMES pwsh powershell) + + if(POWERSHELL_EXECUTABLE) + set(UNIT_TESTS_PS + in_dummy_expect.ps1 + in_tail_expect.ps1 + in_syslog_tcp_plaintext_expect.ps1 + in_syslog_udp_plaintext_expect.ps1 + ) + + if(FLB_TLS) + list(APPEND UNIT_TESTS_PS + in_http_tls_expect.ps1 + in_syslog_tcp_tls_expect.ps1 + ) + else() + message(STATUS "Skipping TLS runtime_shell tests on Windows: TLS is disabled") + endif() + + if(FLB_CUSTOM_CALYPTIA) + list(APPEND UNIT_TESTS_PS custom_calyptia.ps1) + endif() + + if(FLB_HAVE_LIBYAML) + list(APPEND UNIT_TESTS_PS + dry_run_invalid_property.ps1 + processor_conditional.ps1 + processor_invalid.ps1 + ) + else() + message(STATUS + "Skipping YAML runtime_shell tests on Windows: libyaml is unavailable") + endif() + + message(STATUS + "Skipping Unix-domain-socket runtime_shell tests on Windows") + + foreach(script ${UNIT_TESTS_PS}) + add_test(NAME ${script} + COMMAND ${POWERSHELL_EXECUTABLE} + -NoLogo + -NoProfile + -NonInteractive + -ExecutionPolicy Bypass + -File ${CMAKE_CURRENT_SOURCE_DIR}/${script} + ) -if (CMAKE_SYSTEM_NAME STREQUAL "Linux") - if(CMAKE_SYSTEM_PROCESSOR MATCHES "^(riscv64)") - message(STATUS "We don't test Golang plugins on RISC-V 64bit platform for now") + set_tests_properties(${script} PROPERTIES + ENVIRONMENT "${RUNTIME_SHELL_ENVIRONMENT}" + LABELS "runtime_shell" + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} + ) + endforeach() else() - list(APPEND UNIT_TESTS_SH proxy_logs_expect.sh) + message(STATUS + "Skipping runtime_shell tests on Windows: PowerShell was not found") endif() -endif() - -# Prepare list of unit tests -foreach(script ${UNIT_TESTS_SH}) - add_test(NAME ${script} - COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/${script} +else() + configure_file( + "${CMAKE_CURRENT_SOURCE_DIR}/runtime_shell.env.in" + "${CMAKE_CURRENT_SOURCE_DIR}/runtime_shell.env" ) - set_tests_properties(${script} PROPERTIES ENVIRONMENT - "FLB_ROOT=${PROJECT_SOURCE_DIR};\ -FLB_RUNTIME_SHELL_PATH=${CMAKE_CURRENT_SOURCE_DIR};\ -FLB_RUNTIME_SHELL_CONF=${CMAKE_CURRENT_SOURCE_DIR}/conf;\ -FLB_BIN=${CMAKE_BINARY_DIR}/bin/fluent-bit;\ -FLB_BUILD=${CMAKE_BINARY_DIR}" + set(UNIT_TESTS_SH + custom_calyptia.sh + dry_run_invalid_property.sh + in_dummy_expect.sh + in_tail_expect.sh + in_http_tls_expect.sh + in_syslog_tcp_tls_expect.sh + in_syslog_tcp_plaintext_expect.sh + in_syslog_udp_plaintext_expect.sh + in_syslog_uds_dgram_plaintext_expect.sh + in_syslog_uds_stream_plaintext_expect.sh + processor_conditional.sh + processor_invalid.sh ) -endforeach() + + if(CMAKE_SYSTEM_NAME STREQUAL "Linux") + if(CMAKE_SYSTEM_PROCESSOR MATCHES "^(riscv64)") + message(STATUS + "We don't test Golang plugins on RISC-V 64bit platform for now") + else() + list(APPEND UNIT_TESTS_SH proxy_logs_expect.sh) + endif() + endif() + + foreach(script ${UNIT_TESTS_SH}) + add_test(NAME ${script} + COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/${script} + ) + + set_tests_properties(${script} PROPERTIES + ENVIRONMENT "${RUNTIME_SHELL_ENVIRONMENT}" + LABELS "runtime_shell" + ) + endforeach() +endif() diff --git a/tests/runtime_shell/common.ps1 b/tests/runtime_shell/common.ps1 new file mode 100644 index 00000000000..2d81c415400 --- /dev/null +++ b/tests/runtime_shell/common.ps1 @@ -0,0 +1,121 @@ +Set-StrictMode -Version Latest +$ErrorActionPreference = "Stop" + +function Assert-RuntimeEnvironment +{ + foreach ($name in @("FLB_ROOT", "FLB_RUNTIME_SHELL_PATH", + "FLB_RUNTIME_SHELL_CONF", "FLB_BIN", "FLB_BUILD")) { + $value = [Environment]::GetEnvironmentVariable($name) + if ([string]::IsNullOrWhiteSpace($value)) { + throw "Required environment variable $name is not set" + } + } + + if (-not (Test-Path -LiteralPath $env:FLB_BIN -PathType Leaf)) { + throw "Fluent Bit executable was not found at $env:FLB_BIN" + } +} + +function Get-RuntimeConfigPath +{ + param([Parameter(Mandatory = $true)][string] $Name) + + return Join-Path $env:FLB_RUNTIME_SHELL_CONF $Name +} + +function New-RuntimeTempDirectory +{ + param([Parameter(Mandatory = $true)][string] $Name) + + $directory = Join-Path ([IO.Path]::GetTempPath()) ( + "fluent-bit-runtime-shell-{0}-{1}" -f $Name, [guid]::NewGuid()) + [void] (New-Item -ItemType Directory -Path $directory) + return $directory +} + +function Start-FluentBit +{ + param( + [Parameter(Mandatory = $true)][string] $ConfigPath, + [string[]] $AdditionalArguments = @(), + [string] $StandardOutputPath, + [string] $StandardErrorPath + ) + + $arguments = @("-c", ('"{0}"' -f $ConfigPath)) + $AdditionalArguments + $parameters = @{ + FilePath = $env:FLB_BIN + ArgumentList = $arguments + NoNewWindow = $true + PassThru = $true + } + + if (-not [string]::IsNullOrWhiteSpace($StandardOutputPath)) { + $parameters.RedirectStandardOutput = $StandardOutputPath + } + if (-not [string]::IsNullOrWhiteSpace($StandardErrorPath)) { + $parameters.RedirectStandardError = $StandardErrorPath + } + + return Start-Process @parameters +} + +function Stop-FluentBit +{ + param([System.Diagnostics.Process] $Process) + + if ($null -ne $Process -and -not $Process.HasExited) { + Stop-Process -Id $Process.Id -Force + [void] $Process.WaitForExit(5000) + } +} + +function Wait-ForFile +{ + param( + [Parameter(Mandatory = $true)][string] $Path, + [Parameter(Mandatory = $true)][System.Diagnostics.Process] $Process, + [int] $TimeoutSeconds = 15 + ) + + $deadline = [DateTime]::UtcNow.AddSeconds($TimeoutSeconds) + while ([DateTime]::UtcNow -lt $deadline) { + if (Test-Path -LiteralPath $Path -PathType Leaf) { + return + } + if ($Process.HasExited) { + throw "Fluent Bit exited before creating $Path (code $($Process.ExitCode))" + } + Start-Sleep -Milliseconds 100 + } + + throw "Timed out waiting for Fluent Bit to create $Path" +} + +function Wait-FluentBitExit +{ + param( + [Parameter(Mandatory = $true)][System.Diagnostics.Process] $Process, + [int] $TimeoutSeconds = 15 + ) + + if (-not $Process.WaitForExit($TimeoutSeconds * 1000)) { + Stop-FluentBit $Process + throw "Timed out waiting for Fluent Bit to exit" + } + if ($Process.ExitCode -ne 0) { + throw "Fluent Bit exited with code $($Process.ExitCode)" + } +} + +function Invoke-FluentBit +{ + param( + [Parameter(Mandatory = $true)][string] $ConfigPath, + [string[]] $AdditionalArguments = @() + ) + + & $env:FLB_BIN -c $ConfigPath @AdditionalArguments | Out-Host + $exitCode = $LASTEXITCODE + return $exitCode +} diff --git a/tests/runtime_shell/conf/dry_run_invalid_property.yaml b/tests/runtime_shell/conf/dry_run_invalid_property.yaml new file mode 100644 index 00000000000..7424a89dc11 --- /dev/null +++ b/tests/runtime_shell/conf/dry_run_invalid_property.yaml @@ -0,0 +1,11 @@ +service: + log_level: debug + flush: 1 +pipeline: + inputs: + - name: dummy + tag: test + invalid_property_that_does_not_exist: some_value + outputs: + - name: stdout + match: '*' diff --git a/tests/runtime_shell/conf/in_tail_expect.conf b/tests/runtime_shell/conf/in_tail_expect.conf index 7348bacc0c7..d4ad0762564 100644 --- a/tests/runtime_shell/conf/in_tail_expect.conf +++ b/tests/runtime_shell/conf/in_tail_expect.conf @@ -6,8 +6,8 @@ [INPUT] name tail - path /tmp/flb_tail_expect*.log - exclude_path /tmp/flb_*2.log + path ${TAIL_TEST_GLOB} + exclude_path ${TAIL_TEST_EXCLUDE} read_from_head true parser json refresh_interval 10 @@ -20,7 +20,7 @@ buffer_max_size 32k skip_long_lines false exit_on_eof false - db /tmp/flb_tail_expect.db + db ${TAIL_TEST_DB} db.sync full [FILTER] @@ -29,7 +29,7 @@ Log_Level debug # Rules key_exists $path_key - key_val_eq $path_key /tmp/flb_tail_expect_1.log + key_val_eq $path_key ${TAIL_TEST_FILE} key_not_exists $nokey action exit diff --git a/tests/runtime_shell/conf/processor_conditional.yaml b/tests/runtime_shell/conf/processor_conditional.yaml new file mode 100644 index 00000000000..99cab96378b --- /dev/null +++ b/tests/runtime_shell/conf/processor_conditional.yaml @@ -0,0 +1,35 @@ +service: + log_level: trace + flush: 1 +pipeline: + inputs: + - name: dummy + dummy: '{"request": {"method": "GET", "path": "/api/v1/resource", "headers": {"Authorization": "Bearer valid-token"}, "access": "granted"}}' + tag: error.msg + processors: + logs: + - name: content_modifier + action: insert + key: modified_if_post + value: true + condition: + op: and + rules: + - field: $request['method'] + op: eq + value: POST + + - name: content_modifier + action: insert + key: modified_if_get + value: true + condition: + op: and + rules: + - field: $request['method'] + op: eq + value: GET + + outputs: + - name: stdout + match: '*' diff --git a/tests/runtime_shell/conf/processor_conditional_grep.yaml b/tests/runtime_shell/conf/processor_conditional_grep.yaml new file mode 100644 index 00000000000..9f0661e223c --- /dev/null +++ b/tests/runtime_shell/conf/processor_conditional_grep.yaml @@ -0,0 +1,54 @@ +service: + log_level: trace + flush: 1 +pipeline: + inputs: + - name: dummy + dummy: '{"endpoint":"localhost", "value":"something"}' + tag: dummy + processors: + logs: + - name: grep + logical_op: and + regex: + - value something + condition: + op: and + rules: + - field: $endpoint + op: eq + value: farhost + - name: dummy + dummy: '{"endpoint":"localhost2", "value":"something"}' + tag: dummy + processors: + logs: + - name: grep + logical_op: and + regex: + - value something + condition: + op: and + rules: + - field: $endpoint + op: eq + value: farhost + - name: dummy + dummy: '{"endpoint":"farhost", "value":"nothing"}' + tag: dummy + processors: + logs: + - name: grep + logical_op: and + regex: + - value something + condition: + op: and + rules: + - field: $endpoint + op: eq + value: farhost + + outputs: + - name: stdout + match: '*' diff --git a/tests/runtime_shell/conf/processor_invalid.yaml b/tests/runtime_shell/conf/processor_invalid.yaml new file mode 100644 index 00000000000..bdd554a4423 --- /dev/null +++ b/tests/runtime_shell/conf/processor_invalid.yaml @@ -0,0 +1,16 @@ +service: + log_level: debug + flush: 1 +pipeline: + inputs: + - name: dummy + dummy: '{"message": "test message"}' + tag: test + processors: + logs: + - name: non_existent_processor + action: invalid + + outputs: + - name: stdout + match: '*' diff --git a/tests/runtime_shell/custom_calyptia.ps1 b/tests/runtime_shell/custom_calyptia.ps1 new file mode 100644 index 00000000000..de366117054 --- /dev/null +++ b/tests/runtime_shell/custom_calyptia.ps1 @@ -0,0 +1,74 @@ +. "$PSScriptRoot/common.ps1" + +function Test-CalyptiaFleetFormat +{ + param( + [Parameter(Mandatory = $true)][string] $Format, + [Parameter(Mandatory = $true)][bool] $ExpectYaml + ) + + $process = $null + $env:CALYPTIA_FLEET_FORMAT = $Format + if (Test-Path -LiteralPath $env:CALYPTIA_FLEET_DIR) { + Remove-Item -LiteralPath $env:CALYPTIA_FLEET_DIR -Recurse -Force + } + [void] (New-Item -ItemType Directory -Path $env:CALYPTIA_FLEET_DIR) + + $config = Get-RuntimeConfigPath "custom_calyptia_fleet.conf" + $exitCode = Invoke-FluentBit $config @("--dry-run") + if ($exitCode -ne 0) { + throw "Calyptia fleet dry run failed with code $exitCode" + } + + try { + $process = Start-FluentBit $config + Start-Sleep -Seconds 30 + if ($process.HasExited -and $process.ExitCode -ne 0) { + throw "Fluent Bit exited with code $($process.ExitCode)" + } + + $yamlFiles = @(Get-ChildItem -LiteralPath $env:CALYPTIA_FLEET_DIR ` + -Filter "*.yaml" -File -Recurse -ErrorAction SilentlyContinue) + if ($ExpectYaml -and $yamlFiles.Count -eq 0) { + throw "No YAML fleet configuration files were found" + } + if (-not $ExpectYaml -and $yamlFiles.Count -ne 0) { + throw "YAML fleet configuration files were unexpectedly found" + } + + foreach ($file in $yamlFiles) { + Get-Content -LiteralPath $file.FullName | Out-Host + } + } + finally { + Stop-FluentBit $process + } +} + +$tempDirectory = $null + +try { + Assert-RuntimeEnvironment + if ([string]::IsNullOrWhiteSpace($env:CALYPTIA_FLEET_TOKEN)) { + Write-Host "SKIP: CALYPTIA_FLEET_TOKEN is not set" + exit 0 + } + + if ([string]::IsNullOrWhiteSpace($env:CALYPTIA_FLEET_DIR)) { + $tempDirectory = New-RuntimeTempDirectory "custom-calyptia" + $env:CALYPTIA_FLEET_DIR = Join-Path $tempDirectory "fleet-test" + } + + Test-CalyptiaFleetFormat "off" $true + Test-CalyptiaFleetFormat "on" $false + exit 0 +} +catch { + Write-Host "ERROR: $($_.Exception.Message)" + exit 1 +} +finally { + if ($null -ne $tempDirectory) { + Remove-Item -LiteralPath $tempDirectory -Recurse -Force -ErrorAction SilentlyContinue + } +} diff --git a/tests/runtime_shell/dry_run_invalid_property.ps1 b/tests/runtime_shell/dry_run_invalid_property.ps1 new file mode 100644 index 00000000000..a5a7c1d196e --- /dev/null +++ b/tests/runtime_shell/dry_run_invalid_property.ps1 @@ -0,0 +1,29 @@ +. "$PSScriptRoot/common.ps1" + +try { + Assert-RuntimeEnvironment + $configPath = Get-RuntimeConfigPath "dry_run_invalid_property.yaml" + + Write-Host "Running Fluent Bit with --dry-run and invalid property config..." + $output = & $env:FLB_BIN --dry-run -c $configPath 2>&1 | Out-String + $exitCode = $LASTEXITCODE + Write-Host $output + + if ($exitCode -eq 0) { + throw "Fluent Bit --dry-run unexpectedly succeeded" + } + if (-not $output.Contains( + "unknown configuration property 'invalid_property_that_does_not_exist'")) { + throw "Unknown property error was not reported" + } + if (-not $output.Contains("check properties for input plugins is failed")) { + throw "Input plugin validation error was not reported" + } + + Write-Host "Test passed: Fluent Bit detected the invalid property" + exit 0 +} +catch { + Write-Host "ERROR: $($_.Exception.Message)" + exit 1 +} diff --git a/tests/runtime_shell/dry_run_invalid_property.sh b/tests/runtime_shell/dry_run_invalid_property.sh index ee48c8693b8..f0d712d9d0c 100755 --- a/tests/runtime_shell/dry_run_invalid_property.sh +++ b/tests/runtime_shell/dry_run_invalid_property.sh @@ -1,64 +1,41 @@ #!/bin/sh -# Setup environment if not already set -if [ -z "$FLB_BIN" ]; then - FLB_ROOT=${FLB_ROOT:-$(cd $(dirname $0)/../.. && pwd)} - FLB_BIN=${FLB_BIN:-$FLB_ROOT/build/bin/fluent-bit} -fi +FLB_ROOT=${FLB_ROOT:-$(cd "$(dirname "$0")/../.." && pwd)} +FLB_BIN=${FLB_BIN:-$FLB_ROOT/build/bin/fluent-bit} +FLB_RUNTIME_SHELL_CONF=${FLB_RUNTIME_SHELL_CONF:-$FLB_ROOT/tests/runtime_shell/conf} echo "Using Fluent Bit at: $FLB_BIN" -# Create a temporary YAML config file with an invalid property -cat > /tmp/dry_run_invalid_property.yaml << EOL -service: - log_level: debug - flush: 1 -pipeline: - inputs: - - name: dummy - tag: test - invalid_property_that_does_not_exist: some_value - outputs: - - name: stdout - match: '*' -EOL +CONFIG_FILE="$FLB_RUNTIME_SHELL_CONF/dry_run_invalid_property.yaml" +OUTPUT_FILE="/tmp/dry_run_invalid_property_output.txt" echo "Running Fluent Bit with --dry-run and invalid property config..." echo "YAML Config:" -cat /tmp/dry_run_invalid_property.yaml +cat "$CONFIG_FILE" -# Redirect stdout and stderr to a file for analysis -OUTPUT_FILE="/tmp/dry_run_invalid_property_output.txt" -$FLB_BIN --dry-run -c /tmp/dry_run_invalid_property.yaml > $OUTPUT_FILE 2>&1 - -# Check exit code - we expect it to fail +"$FLB_BIN" --dry-run -c "$CONFIG_FILE" > "$OUTPUT_FILE" 2>&1 EXIT_CODE=$? echo "Fluent Bit --dry-run exited with code: $EXIT_CODE" -# Show the output echo "Output file content:" -cat $OUTPUT_FILE +cat "$OUTPUT_FILE" -# Check if the output contains an error about the unknown configuration property -UNKNOWN_PROPERTY=$(grep -c "unknown configuration property 'invalid_property_that_does_not_exist'" $OUTPUT_FILE || true) -RELOAD_ERROR=$(grep -c "check properties for input plugins is failed" $OUTPUT_FILE || true) +UNKNOWN_PROPERTY=$(grep -c \ + "unknown configuration property 'invalid_property_that_does_not_exist'" \ + "$OUTPUT_FILE" || true) +RELOAD_ERROR=$(grep -c \ + "check properties for input plugins is failed" "$OUTPUT_FILE" || true) -# Clean up -echo "Cleaning up..." -rm -f /tmp/dry_run_invalid_property.yaml -rm -f $OUTPUT_FILE +rm -f "$OUTPUT_FILE" -# Check results - we expect: -# 1. Fluent Bit to fail (non-zero exit code) -# 2. Error message about unknown configuration property -# 3. Error message from reload validation -if [ "$EXIT_CODE" -ne 0 ] && [ "$UNKNOWN_PROPERTY" -gt 0 ] && [ "$RELOAD_ERROR" -gt 0 ]; then +if [ "$EXIT_CODE" -ne 0 ] && [ "$UNKNOWN_PROPERTY" -gt 0 ] && \ + [ "$RELOAD_ERROR" -gt 0 ]; then echo "Test passed: Fluent Bit --dry-run correctly detected invalid property and failed" exit 0 -else - echo "Test failed: Fluent Bit --dry-run should detect invalid properties and fail" - echo "Exit code: $EXIT_CODE (expected non-zero)" - echo "Unknown property message count: $UNKNOWN_PROPERTY (expected > 0)" - echo "Reload error message count: $RELOAD_ERROR (expected > 0)" - exit 1 fi + +echo "Test failed: Fluent Bit --dry-run should detect invalid properties and fail" +echo "Exit code: $EXIT_CODE (expected non-zero)" +echo "Unknown property message count: $UNKNOWN_PROPERTY (expected > 0)" +echo "Reload error message count: $RELOAD_ERROR (expected > 0)" +exit 1 diff --git a/tests/runtime_shell/in_dummy_expect.ps1 b/tests/runtime_shell/in_dummy_expect.ps1 new file mode 100644 index 00000000000..a4a949408ef --- /dev/null +++ b/tests/runtime_shell/in_dummy_expect.ps1 @@ -0,0 +1,14 @@ +. "$PSScriptRoot/common.ps1" + +try { + Assert-RuntimeEnvironment + $exitCode = Invoke-FluentBit (Get-RuntimeConfigPath "in_dummy_expect.conf") + if ($exitCode -ne 0) { + throw "Fluent Bit exited with code $exitCode" + } + exit 0 +} +catch { + Write-Host "ERROR: $($_.Exception.Message)" + exit 1 +} diff --git a/tests/runtime_shell/in_http_tls_expect.ps1 b/tests/runtime_shell/in_http_tls_expect.ps1 new file mode 100644 index 00000000000..f0d0c634119 --- /dev/null +++ b/tests/runtime_shell/in_http_tls_expect.ps1 @@ -0,0 +1,79 @@ +. "$PSScriptRoot/common.ps1" + +Add-Type -TypeDefinition @' +using System.Net.Http; +using System.Net.Security; +using System.Security.Cryptography.X509Certificates; + +public static class RuntimeShellCertificateValidator +{ + public static bool ValidateHttp( + HttpRequestMessage request, + X509Certificate2 certificate, + X509Chain chain, + SslPolicyErrors sslPolicyErrors) + { + return true; + } +} +'@ + +$process = $null +$tempDirectory = $null +$client = $null +$handler = $null +$content = $null + +try { + Assert-RuntimeEnvironment + $tempDirectory = New-RuntimeTempDirectory "http-tls" + $signalFile = Join-Path $tempDirectory "signal.log" + $env:SIGNAL_FILE_PATH = $signalFile + $env:LISTENER_VHOST = "leo.vcap.me" + $env:LISTENER_HOST = "127.0.0.1" + $env:LISTENER_PORT = "50000" + + $config = Get-RuntimeConfigPath "in_http_tls_expect.conf" + $process = Start-FluentBit $config + Wait-ForFile $signalFile $process + + $handler = [Net.Http.HttpClientHandler]::new() + $validatorType = [Func[Net.Http.HttpRequestMessage, + Security.Cryptography.X509Certificates.X509Certificate2, + Security.Cryptography.X509Certificates.X509Chain, + Net.Security.SslPolicyErrors, bool]] + $validatorMethod = [RuntimeShellCertificateValidator].GetMethod("ValidateHttp") + $handler.ServerCertificateCustomValidationCallback = + $validatorMethod.CreateDelegate($validatorType) + $handler.SslProtocols = [Security.Authentication.SslProtocols]::Tls12 + $client = [Net.Http.HttpClient]::new($handler) + $content = [Net.Http.StringContent]::new( + '{"message":"Hello!"}', [Text.Encoding]::UTF8, "application/json") + $uri = "https://$($env:LISTENER_HOST):$($env:LISTENER_PORT)" + $response = $client.PostAsync($uri, $content).GetAwaiter().GetResult() + if (-not $response.IsSuccessStatusCode) { + throw "HTTP input returned status $([int] $response.StatusCode)" + } + + Wait-FluentBitExit $process + exit 0 +} +catch { + Write-Host "ERROR: $($_.Exception.ToString())" + exit 1 +} +finally { + if ($null -ne $content) { + $content.Dispose() + } + if ($null -ne $client) { + $client.Dispose() + } + if ($null -ne $handler) { + $handler.Dispose() + } + Stop-FluentBit $process + if ($null -ne $tempDirectory) { + Remove-Item -LiteralPath $tempDirectory -Recurse -Force -ErrorAction SilentlyContinue + } +} diff --git a/tests/runtime_shell/in_syslog_tcp_plaintext_expect.ps1 b/tests/runtime_shell/in_syslog_tcp_plaintext_expect.ps1 new file mode 100644 index 00000000000..aec1243157b --- /dev/null +++ b/tests/runtime_shell/in_syslog_tcp_plaintext_expect.ps1 @@ -0,0 +1,44 @@ +. "$PSScriptRoot/common.ps1" + +$process = $null +$tempDirectory = $null +$client = $null + +try { + Assert-RuntimeEnvironment + $tempDirectory = New-RuntimeTempDirectory "syslog-tcp" + $signalFile = Join-Path $tempDirectory "signal.log" + $env:SIGNAL_FILE_PATH = $signalFile + $env:LISTENER_HOST = "127.0.0.1" + $env:LISTENER_PORT = "50001" + + $config = Get-RuntimeConfigPath "in_syslog_tcp_plaintext_expect.conf" + $process = Start-FluentBit $config + Wait-ForFile $signalFile $process + + $client = [Net.Sockets.TcpClient]::new() + $client.Connect($env:LISTENER_HOST, [int] $env:LISTENER_PORT) + $stream = $client.GetStream() + $payload = [Text.Encoding]::UTF8.GetBytes( + "<13>1 1970-01-01T00:00:00.000000+00:00 testhost testuser - - [] Hello!`n") + $stream.Write($payload, 0, $payload.Length) + $stream.Flush() + $client.Close() + $client = $null + + Wait-FluentBitExit $process + exit 0 +} +catch { + Write-Host "ERROR: $($_.Exception.Message)" + exit 1 +} +finally { + if ($null -ne $client) { + $client.Dispose() + } + Stop-FluentBit $process + if ($null -ne $tempDirectory) { + Remove-Item -LiteralPath $tempDirectory -Recurse -Force -ErrorAction SilentlyContinue + } +} diff --git a/tests/runtime_shell/in_syslog_tcp_tls_expect.ps1 b/tests/runtime_shell/in_syslog_tcp_tls_expect.ps1 new file mode 100644 index 00000000000..faa4df5388f --- /dev/null +++ b/tests/runtime_shell/in_syslog_tcp_tls_expect.ps1 @@ -0,0 +1,61 @@ +. "$PSScriptRoot/common.ps1" + +$process = $null +$tempDirectory = $null +$client = $null +$tlsStream = $null + +try { + Assert-RuntimeEnvironment + $tempDirectory = New-RuntimeTempDirectory "syslog-tcp-tls" + $signalFile = Join-Path $tempDirectory "signal.log" + $env:SIGNAL_FILE_PATH = $signalFile + $env:LISTENER_VHOST = "leo.vcap.me" + $env:LISTENER_HOST = "127.0.0.1" + $env:LISTENER_PORT = "50002" + + $config = Get-RuntimeConfigPath "in_syslog_tcp_tls_expect.conf" + $process = Start-FluentBit $config + Wait-ForFile $signalFile $process + + $client = [Net.Sockets.TcpClient]::new() + $client.Connect($env:LISTENER_HOST, [int] $env:LISTENER_PORT) + $validationCallback = [Net.Security.RemoteCertificateValidationCallback] { + param($sender, $certificate, $chain, $sslPolicyErrors) + return $true + } + $tlsStream = [Net.Security.SslStream]::new( + $client.GetStream(), $false, $validationCallback) + $tlsOptions = [Net.Security.SslClientAuthenticationOptions]::new() + $tlsOptions.TargetHost = $env:LISTENER_VHOST + $tlsOptions.EnabledSslProtocols = [Security.Authentication.SslProtocols]::Tls12 + $tlsOptions.CertificateRevocationCheckMode = [Security.Cryptography.X509Certificates.X509RevocationMode]::NoCheck + $tlsStream.AuthenticateAsClient($tlsOptions) + $payload = [Text.Encoding]::UTF8.GetBytes( + "<13>1 1970-01-01T00:00:00.000000+00:00 testhost testuser - - [] Hello!`n") + $tlsStream.Write($payload, 0, $payload.Length) + $tlsStream.Flush() + $tlsStream.Close() + $tlsStream = $null + $client.Close() + $client = $null + + Wait-FluentBitExit $process + exit 0 +} +catch { + Write-Host "ERROR: $($_.Exception.ToString())" + exit 1 +} +finally { + if ($null -ne $tlsStream) { + $tlsStream.Dispose() + } + if ($null -ne $client) { + $client.Dispose() + } + Stop-FluentBit $process + if ($null -ne $tempDirectory) { + Remove-Item -LiteralPath $tempDirectory -Recurse -Force -ErrorAction SilentlyContinue + } +} diff --git a/tests/runtime_shell/in_syslog_udp_plaintext_expect.ps1 b/tests/runtime_shell/in_syslog_udp_plaintext_expect.ps1 new file mode 100644 index 00000000000..7c3902f0bda --- /dev/null +++ b/tests/runtime_shell/in_syslog_udp_plaintext_expect.ps1 @@ -0,0 +1,42 @@ +. "$PSScriptRoot/common.ps1" + +$process = $null +$tempDirectory = $null +$client = $null + +try { + Assert-RuntimeEnvironment + $tempDirectory = New-RuntimeTempDirectory "syslog-udp" + $signalFile = Join-Path $tempDirectory "signal.log" + $env:SIGNAL_FILE_PATH = $signalFile + $env:LISTENER_HOST = "127.0.0.1" + $env:LISTENER_PORT = "50003" + + $config = Get-RuntimeConfigPath "in_syslog_udp_plaintext_expect.conf" + $process = Start-FluentBit $config + Wait-ForFile $signalFile $process + + $client = [Net.Sockets.UdpClient]::new() + $payload = [Text.Encoding]::UTF8.GetBytes( + "<13>1 1970-01-01T00:00:00.000000+00:00 testhost testuser - - [] Hello!`n") + [void] $client.Send( + $payload, $payload.Length, $env:LISTENER_HOST, [int] $env:LISTENER_PORT) + $client.Close() + $client = $null + + Wait-FluentBitExit $process + exit 0 +} +catch { + Write-Host "ERROR: $($_.Exception.Message)" + exit 1 +} +finally { + if ($null -ne $client) { + $client.Dispose() + } + Stop-FluentBit $process + if ($null -ne $tempDirectory) { + Remove-Item -LiteralPath $tempDirectory -Recurse -Force -ErrorAction SilentlyContinue + } +} diff --git a/tests/runtime_shell/in_tail_expect.ps1 b/tests/runtime_shell/in_tail_expect.ps1 new file mode 100644 index 00000000000..3deaaa46bcc --- /dev/null +++ b/tests/runtime_shell/in_tail_expect.ps1 @@ -0,0 +1,37 @@ +. "$PSScriptRoot/common.ps1" + +$tempDirectory = $null + +try { + Assert-RuntimeEnvironment + $tempDirectory = New-RuntimeTempDirectory "in-tail-expect" + $targetFile = Join-Path $tempDirectory "flb_tail_expect_1.log" + $excludedFile = Join-Path $tempDirectory "flb_tail_expect_2.log" + + $env:TAIL_TEST_GLOB = Join-Path $tempDirectory "flb_tail_expect_*.log" + $env:TAIL_TEST_EXCLUDE = Join-Path $tempDirectory "flb_*2.log" + $env:TAIL_TEST_FILE = $targetFile + $env:TAIL_TEST_DB = Join-Path $tempDirectory "flb_tail_expect.db" + + $encoding = [Text.UTF8Encoding]::new($false) + [IO.File]::WriteAllText( + $targetFile, "{`"key`": `"val`"}`r`n", $encoding) + [IO.File]::WriteAllText( + $excludedFile, "{`"nokey`": `"`"}`r`n", $encoding) + + $config = Get-RuntimeConfigPath "in_tail_expect.conf" + $exitCode = Invoke-FluentBit $config + if ($exitCode -ne 0) { + throw "Fluent Bit exited with code $exitCode" + } + exit 0 +} +catch { + Write-Host "ERROR: $($_.Exception.Message)" + exit 1 +} +finally { + if ($null -ne $tempDirectory) { + Remove-Item -LiteralPath $tempDirectory -Recurse -Force -ErrorAction SilentlyContinue + } +} diff --git a/tests/runtime_shell/in_tail_expect.sh b/tests/runtime_shell/in_tail_expect.sh index c847d65f418..3da138fbf9d 100755 --- a/tests/runtime_shell/in_tail_expect.sh +++ b/tests/runtime_shell/in_tail_expect.sh @@ -3,8 +3,13 @@ test_in_tail_filter_expect() { rm -rf /tmp/flb_* + export TAIL_TEST_GLOB="/tmp/flb_tail_expect*.log" + export TAIL_TEST_EXCLUDE="/tmp/flb_*2.log" + export TAIL_TEST_FILE="/tmp/flb_tail_expect_1.log" + export TAIL_TEST_DB="/tmp/flb_tail_expect.db" + # Monitor this file - echo "{\"key\": \"val\"}" > /tmp/flb_tail_expect_1.log + echo "{\"key\": \"val\"}" > "$TAIL_TEST_FILE" # Excluded file echo "{\"nokey\": \"\"}" > /tmp/flb_tail_expect_2.log diff --git a/tests/runtime_shell/processor_conditional.ps1 b/tests/runtime_shell/processor_conditional.ps1 new file mode 100644 index 00000000000..c7749d55785 --- /dev/null +++ b/tests/runtime_shell/processor_conditional.ps1 @@ -0,0 +1,82 @@ +. "$PSScriptRoot/common.ps1" + +function Invoke-ConditionalProcessor +{ + param( + [Parameter(Mandatory = $true)][string] $ConfigPath, + [Parameter(Mandatory = $true)][string] $OutputPath, + [Parameter(Mandatory = $true)][string] $ErrorPath + ) + + $process = $null + try { + $process = Start-FluentBit $ConfigPath @("-o", "stdout") $OutputPath $ErrorPath + Start-Sleep -Seconds 5 + if ($process.HasExited -and $process.ExitCode -ne 0) { + throw "Fluent Bit exited with code $($process.ExitCode)" + } + } + finally { + Stop-FluentBit $process + } + + $output = "" + if (Test-Path -LiteralPath $OutputPath -PathType Leaf) { + $output += Get-Content -LiteralPath $OutputPath -Raw + } + if (Test-Path -LiteralPath $ErrorPath -PathType Leaf) { + $output += Get-Content -LiteralPath $ErrorPath -Raw + } + return $output +} + +$tempDirectory = $null + +try { + Assert-RuntimeEnvironment + $tempDirectory = New-RuntimeTempDirectory "processor-conditional" + $conditionalConfigPath = Get-RuntimeConfigPath "processor_conditional.yaml" + + $conditionalOutput = Invoke-ConditionalProcessor ` + $conditionalConfigPath ` + (Join-Path $tempDirectory "processor_conditional.stdout") ` + (Join-Path $tempDirectory "processor_conditional.stderr") + Write-Host $conditionalOutput + + if (-not $conditionalOutput.Contains("modified_if_get")) { + throw "GET condition was not applied" + } + if ($conditionalOutput.Contains("modified_if_post")) { + throw "POST condition was unexpectedly applied" + } + + $grepConfigPath = Get-RuntimeConfigPath "processor_conditional_grep.yaml" + + $grepOutput = Invoke-ConditionalProcessor ` + $grepConfigPath ` + (Join-Path $tempDirectory "processor_conditional_grep.stdout") ` + (Join-Path $tempDirectory "processor_conditional_grep.stderr") + Write-Host $grepOutput + + if (-not $grepOutput.Contains('"endpoint"=>"localhost"')) { + throw "localhost record was not emitted" + } + if (-not $grepOutput.Contains('"endpoint"=>"localhost2"')) { + throw "localhost2 record was not emitted" + } + if ($grepOutput.Contains('"endpoint"=>"farhost"')) { + throw "farhost record was unexpectedly emitted" + } + + Write-Host "Test passed: conditional processors selected the expected records" + exit 0 +} +catch { + Write-Host "ERROR: $($_.Exception.Message)" + exit 1 +} +finally { + if ($null -ne $tempDirectory) { + Remove-Item -LiteralPath $tempDirectory -Recurse -Force -ErrorAction SilentlyContinue + } +} diff --git a/tests/runtime_shell/processor_conditional.sh b/tests/runtime_shell/processor_conditional.sh index 292fa087110..31bd453082b 100755 --- a/tests/runtime_shell/processor_conditional.sh +++ b/tests/runtime_shell/processor_conditional.sh @@ -1,87 +1,40 @@ #!/bin/sh -# Setup environment if not already set -if [ -z "$FLB_BIN" ]; then - FLB_ROOT=${FLB_ROOT:-$(cd $(dirname $0)/../.. && pwd)} - FLB_BIN=${FLB_BIN:-$FLB_ROOT/build/bin/fluent-bit} -fi +FLB_ROOT=${FLB_ROOT:-$(cd "$(dirname "$0")/../.." && pwd)} +FLB_BIN=${FLB_BIN:-$FLB_ROOT/build/bin/fluent-bit} +FLB_RUNTIME_SHELL_CONF=${FLB_RUNTIME_SHELL_CONF:-$FLB_ROOT/tests/runtime_shell/conf} echo "Using Fluent Bit at: $FLB_BIN" -# Create a temporary YAML config file -cat > /tmp/processor_conditional.yaml << EOL -service: - log_level: trace - flush: 1 -pipeline: - inputs: - - name: dummy - dummy: '{"request": {"method": "GET", "path": "/api/v1/resource", "headers": {"Authorization": "Bearer valid-token"}, "access": "granted"}}' - tag: error.msg - processors: - logs: - - name: content_modifier - action: insert - key: modified_if_post - value: true - condition: - op: and - rules: - - field: \$request['method'] - op: eq - value: POST - - - name: content_modifier - action: insert - key: modified_if_get - value: true - condition: - op: and - rules: - - field: \$request['method'] - op: eq - value: GET - - outputs: - - name: stdout - match: '*' -EOL +CONFIG_FILE="$FLB_RUNTIME_SHELL_CONF/processor_conditional.yaml" +OUTPUT_FILE="/tmp/processor_conditional_output.txt" echo "Running Fluent Bit with conditional processor YAML config..." echo "YAML Config:" -cat /tmp/processor_conditional.yaml +cat "$CONFIG_FILE" -# Redirect stdout to a file for analysis -OUTPUT_FILE="/tmp/processor_conditional_output.txt" -$FLB_BIN -c /tmp/processor_conditional.yaml -o stdout > $OUTPUT_FILE 2>&1 & +"$FLB_BIN" -c "$CONFIG_FILE" -o stdout > "$OUTPUT_FILE" 2>&1 & FLB_PID=$! echo "Fluent Bit started with PID: $FLB_PID" -# Wait for output to be generated echo "Waiting for processing to complete..." sleep 5 -# Check for output if [ ! -f "$OUTPUT_FILE" ]; then echo "Output file not found" - kill -15 $FLB_PID || true + kill -15 "$FLB_PID" || true exit 1 fi echo "Output file content:" -cat $OUTPUT_FILE +cat "$OUTPUT_FILE" -# Verify that the GET condition was applied but not the POST condition -GET_FIELD=$(grep -c "modified_if_get" $OUTPUT_FILE) -POST_FIELD=$(grep -c "modified_if_post" $OUTPUT_FILE) +GET_FIELD=$(grep -c "modified_if_get" "$OUTPUT_FILE") +POST_FIELD=$(grep -c "modified_if_post" "$OUTPUT_FILE") -# Clean up -echo "Cleaning up..." -kill -15 $FLB_PID || true -rm -f /tmp/processor_conditional.yaml -rm -f $OUTPUT_FILE +kill -15 "$FLB_PID" || true +rm -f "$OUTPUT_FILE" -# Check results if [ "$GET_FIELD" -gt 0 ] && [ "$POST_FIELD" -eq 0 ]; then echo "Test passed: GET condition applied, POST condition not applied" else @@ -89,70 +42,14 @@ else exit 1 fi -# Create a temporary YAML config file for grep filter used as processor -cat > /tmp/processor_conditional_grep.yaml << EOL -service: - log_level: trace - flush: 1 -pipeline: - inputs: - - name: dummy - dummy: '{"endpoint":"localhost", "value":"something"}' - tag: dummy - processors: - logs: - - name: grep - logical_op: and - regex: - - value something - condition: - op: and - rules: - - field: \$endpoint - op: eq - value: farhost - - name: dummy - dummy: '{"endpoint":"localhost2", "value":"something"}' - tag: dummy - processors: - logs: - - name: grep - logical_op: and - regex: - - value something - condition: - op: and - rules: - - field: \$endpoint - op: eq - value: farhost - - name: dummy - dummy: '{"endpoint":"farhost", "value":"nothing"}' - tag: dummy - processors: - logs: - - name: grep - logical_op: and - regex: - - value something - condition: - op: and - rules: - - field: \$endpoint - op: eq - value: farhost - - outputs: - - name: stdout - match: '*' -EOL +CONFIG_FILE="$FLB_RUNTIME_SHELL_CONF/processor_conditional_grep.yaml" +OUTPUT_FILE="/tmp/processor_conditional_grep_output.txt" echo "Running Fluent Bit with conditional grep processor YAML config..." echo "YAML Config:" -cat /tmp/processor_conditional_grep.yaml +cat "$CONFIG_FILE" -OUTPUT_FILE="/tmp/processor_conditional_grep_output.txt" -$FLB_BIN -c /tmp/processor_conditional_grep.yaml -o stdout > $OUTPUT_FILE 2>&1 & +"$FLB_BIN" -c "$CONFIG_FILE" -o stdout > "$OUTPUT_FILE" 2>&1 & FLB_PID=$! echo "Fluent Bit started with PID: $FLB_PID" @@ -161,21 +58,19 @@ sleep 5 if [ ! -f "$OUTPUT_FILE" ]; then echo "Output file not found" - kill -15 $FLB_PID || true + kill -15 "$FLB_PID" || true exit 1 fi echo "Output file content:" -cat $OUTPUT_FILE +cat "$OUTPUT_FILE" -LOCALHOST_COUNT=$(grep -c -E "\"endpoint\"=>\"localhost\"([^0-9]|$)" $OUTPUT_FILE) -LOCALHOST2_COUNT=$(grep -c "\"endpoint\"=>\"localhost2\"" $OUTPUT_FILE) -FARHOST_COUNT=$(grep -c "\"endpoint\"=>\"farhost\"" $OUTPUT_FILE) +LOCALHOST_COUNT=$(grep -c -E '"endpoint"=>"localhost"([^0-9]|$)' "$OUTPUT_FILE") +LOCALHOST2_COUNT=$(grep -c '"endpoint"=>"localhost2"' "$OUTPUT_FILE") +FARHOST_COUNT=$(grep -c '"endpoint"=>"farhost"' "$OUTPUT_FILE") -echo "Cleaning up..." -kill -15 $FLB_PID || true -rm -f /tmp/processor_conditional_grep.yaml -rm -f $OUTPUT_FILE +kill -15 "$FLB_PID" || true +rm -f "$OUTPUT_FILE" if [ "$LOCALHOST_COUNT" -gt 0 ] && [ "$LOCALHOST2_COUNT" -gt 0 ] && [ "$FARHOST_COUNT" -eq 0 ]; then diff --git a/tests/runtime_shell/processor_invalid.ps1 b/tests/runtime_shell/processor_invalid.ps1 new file mode 100644 index 00000000000..55fbfc422e0 --- /dev/null +++ b/tests/runtime_shell/processor_invalid.ps1 @@ -0,0 +1,30 @@ +. "$PSScriptRoot/common.ps1" + +try { + Assert-RuntimeEnvironment + $configPath = Get-RuntimeConfigPath "processor_invalid.yaml" + + Write-Host "Running Fluent Bit with invalid processor YAML config..." + $output = & $env:FLB_BIN -c $configPath -o stdout 2>&1 | Out-String + $exitCode = $LASTEXITCODE + Write-Host $output + + $invalidProcessor = $output.Contains( + "error creating processor 'non_existent_processor': " + + "plugin doesn't exist or failed to initialize") + $failedInitialization = $output.Contains("error initializing processor") + + if ($exitCode -eq 0) { + throw "Fluent Bit unexpectedly accepted an invalid processor" + } + if (-not ($invalidProcessor -or $failedInitialization)) { + throw "Invalid processor error was not reported" + } + + Write-Host "Test passed: Fluent Bit rejected the invalid processor" + exit 0 +} +catch { + Write-Host "ERROR: $($_.Exception.Message)" + exit 1 +} diff --git a/tests/runtime_shell/processor_invalid.sh b/tests/runtime_shell/processor_invalid.sh index 59a72c7ef32..331e2a921af 100755 --- a/tests/runtime_shell/processor_invalid.sh +++ b/tests/runtime_shell/processor_invalid.sh @@ -1,67 +1,40 @@ #!/bin/sh -# Setup environment if not already set -if [ -z "$FLB_BIN" ]; then - FLB_ROOT=${FLB_ROOT:-$(cd $(dirname $0)/../.. && pwd)} - FLB_BIN=${FLB_BIN:-$FLB_ROOT/build/bin/fluent-bit} -fi +FLB_ROOT=${FLB_ROOT:-$(cd "$(dirname "$0")/../.." && pwd)} +FLB_BIN=${FLB_BIN:-$FLB_ROOT/build/bin/fluent-bit} +FLB_RUNTIME_SHELL_CONF=${FLB_RUNTIME_SHELL_CONF:-$FLB_ROOT/tests/runtime_shell/conf} echo "Using Fluent Bit at: $FLB_BIN" -# Create a temporary YAML config file -cat > /tmp/processor_invalid.yaml << EOL -service: - log_level: debug - flush: 1 -pipeline: - inputs: - - name: dummy - dummy: '{"message": "test message"}' - tag: test - processors: - logs: - - name: non_existent_processor - action: invalid - - outputs: - - name: stdout - match: '*' -EOL +CONFIG_FILE="$FLB_RUNTIME_SHELL_CONF/processor_invalid.yaml" +OUTPUT_FILE="/tmp/processor_invalid_output.txt" echo "Running Fluent Bit with invalid processor YAML config..." echo "YAML Config:" -cat /tmp/processor_invalid.yaml +cat "$CONFIG_FILE" -# Redirect stdout and stderr to a file for analysis -OUTPUT_FILE="/tmp/processor_invalid_output.txt" -$FLB_BIN -c /tmp/processor_invalid.yaml -o stdout > $OUTPUT_FILE 2>&1 - -# Check exit code - we expect it to fail +"$FLB_BIN" -c "$CONFIG_FILE" -o stdout > "$OUTPUT_FILE" 2>&1 EXIT_CODE=$? echo "Fluent Bit exited with code: $EXIT_CODE" -# Show the output echo "Output file content:" -cat $OUTPUT_FILE +cat "$OUTPUT_FILE" -# Check if the output contains an error related to invalid processor -INVALID_PROCESSOR=$(grep -c "error creating processor 'non_existent_processor': plugin doesn't exist or failed to initialize" $OUTPUT_FILE || true) -FAILED_INIT=$(grep -c "error initializing processor" $OUTPUT_FILE || true) +INVALID_PROCESSOR=$(grep -c \ + "error creating processor 'non_existent_processor': plugin doesn't exist or failed to initialize" \ + "$OUTPUT_FILE" || true) +FAILED_INIT=$(grep -c "error initializing processor" "$OUTPUT_FILE" || true) -# Clean up -echo "Cleaning up..." -rm -f /tmp/processor_invalid.yaml -rm -f $OUTPUT_FILE +rm -f "$OUTPUT_FILE" -# Check results - we expect Fluent Bit to fail (non-zero exit code) -# and have an error message about the invalid processor -if [ "$EXIT_CODE" -ne 0 ] && ([ "$INVALID_PROCESSOR" -gt 0 ] || [ "$FAILED_INIT" -gt 0 ]); then +if [ "$EXIT_CODE" -ne 0 ] && \ + { [ "$INVALID_PROCESSOR" -gt 0 ] || [ "$FAILED_INIT" -gt 0 ]; }; then echo "Test passed: Fluent Bit failed with error about invalid processor" exit 0 -else - echo "Test failed: Fluent Bit should fail when an invalid processor is configured" - echo "Exit code: $EXIT_CODE (expected non-zero)" - echo "Invalid processor message count: $INVALID_PROCESSOR (expected > 0)" - echo "Failed init message count: $FAILED_INIT (expected > 0)" - exit 1 -fi \ No newline at end of file +fi + +echo "Test failed: Fluent Bit should fail when an invalid processor is configured" +echo "Exit code: $EXIT_CODE (expected non-zero)" +echo "Invalid processor message count: $INVALID_PROCESSOR (expected > 0)" +echo "Failed init message count: $FAILED_INIT (expected > 0)" +exit 1