s3_output: add parquet format support

.github/workflows/unit-tests.yaml

@@ -53,7 +53,7 @@ jobs:
           - "-DFLB_SANITIZE_THREAD=On"
           - "-DFLB_SIMD=On"
           - "-DFLB_SIMD=Off"
-          - "-DFLB_ARROW=On"
+          - "-DFLB_PARQUET_ENCODER=On"
           - "-DFLB_COMPILER_STRICT_POINTER_TYPES=On"
         cmake_version:
           - "3.31.6"
@@ -69,7 +69,7 @@ jobs:
             compiler:
               cc: clang
               cxx: clang++
-          - flb_option: "-DFLB_ARROW=On"
+          - flb_option: "-DFLB_PARQUET_ENCODER=On"
             compiler:
               cc: clang
               cxx: clang++
@@ -114,15 +114,15 @@ jobs:
         with:
           repository: calyptia/fluent-bit-ci
           path: ci
-      - name: Setup Apache Arrow libraries for parquet (-DFLB_ARROW=On Only)
-        if: matrix.flb_option == '-DFLB_ARROW=On'
+      - name: Setup Apache Arrow libraries for Parquet encoder (-DFLB_PARQUET_ENCODER=On Only)
+        if: matrix.flb_option == '-DFLB_PARQUET_ENCODER=On'
         run: |
           sudo apt-get update
           sudo apt-get install -y -V ca-certificates lsb-release wget
-          wget https://packages.apache.org/artifactory/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb
+          wget https://packages.apache.org/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb
           sudo apt-get install -y -V ./apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb
           sudo apt-get update
-          sudo apt-get install -y -V libarrow-glib-dev libparquet-glib-dev
+          sudo apt-get install -y -V libarrow-dev libparquet-dev
 
       - name: ${{ matrix.compiler.cc }} & ${{ matrix.compiler.cxx }} - ${{ matrix.flb_option }}
         run: |

CMakeLists.txt

@@ -199,7 +199,7 @@ option(FLB_SIMD                "Enable SIMD support"                         No)
 option(FLB_CORO_STACK_SIZE     "Set coroutine stack size")
 option(FLB_AVRO_ENCODER        "Build with Avro encoding support"            No)
 option(FLB_AWS_ERROR_REPORTER  "Build with aws error reporting support"      No)
-option(FLB_ARROW               "Build with Apache Arrow support"             No)
+option(FLB_PARQUET_ENCODER     "Build with Parquet encoding support"         No)
 option(FLB_WINDOWS_DEFAULTS    "Build with predefined Windows settings"     Yes)
 option(FLB_WASM                "Build with WASM runtime support"            Yes)
 option(FLB_WAMRC               "Build with WASM AOT compiler executable"    No)
@@ -281,6 +281,11 @@ if(FLB_ALL)
   set(FLB_DEBUG           1)
   set(FLB_TLS             1)
 
+  # Encoders
+  set(FLB_AVRO_ENCODER    1)
+  # Note: FLB_PARQUET_ENCODER is not enabled by FLB_ALL due to external dependencies
+  # Use -DFLB_PARQUET_ENCODER=On explicitly to enable it
+
   # Input plugins
   set(FLB_IN_CPU          1)
   set(FLB_IN_MEM          1)
@@ -1069,11 +1074,12 @@ if(FLB_CONFIG_YAML)
     # For non-standard libyaml installation paths such as homebrew bottled libyaml.
     include_directories(${LIBYAML_INCLUDEDIR})
     link_directories(${LIBYAML_LIBRARY_DIRS})
+    message(STATUS "libyaml found via pkg-config: ${LIBYAML_VERSION}")
   else()
     if (FLB_LIBYAML_DIR)
       set(LIBYAML_LIBRARY_DIRS "${FLB_LIBYAML_DIR}/lib")
       set(LIBYAML_INCLUDEDIR "${FLB_LIBYAML_DIR}/include")
-      message(STATUS "specified libyaml dir: ${FLB_LIBYAML_DIR}")
+      message(STATUS "Using specified libyaml dir: ${FLB_LIBYAML_DIR}")
       if (MSVC)
         FLB_DEFINITION(YAML_DECLARE_STATIC)
       endif ()
@@ -1082,21 +1088,45 @@ if(FLB_CONFIG_YAML)
       include_directories(${LIBYAML_INCLUDEDIR})
       link_directories(${LIBYAML_LIBRARY_DIRS})
     else ()
-      # Requires libyaml support
-      check_c_source_compiles("
-      #include <yaml.h>
-      int main() {
-        yaml_parser_t parser;
-        return 0;
-      }" FLB_HAVE_LIBYAML)
-
-      if(NOT FLB_HAVE_LIBYAML)
-        message(FATAL_ERROR
-          "YAML development dependencies required for YAML configuration format handling.\n"
-          "This is a build time dependency, you can either install the "
-          "dependencies or disable the feature setting the CMake option "
-          "-DFLB_CONFIG_YAML=Off ."
-        )
+      # Try to auto-detect libyaml on macOS via Homebrew
+      if (FLB_SYSTEM_MACOS)
+        find_program(BREW_EXECUTABLE brew)
+        if (BREW_EXECUTABLE)
+          execute_process(
+            COMMAND ${BREW_EXECUTABLE} --prefix libyaml
+            OUTPUT_VARIABLE BREW_LIBYAML_PREFIX
+            OUTPUT_STRIP_TRAILING_WHITESPACE
+            ERROR_QUIET
+          )
+          if (BREW_LIBYAML_PREFIX AND EXISTS "${BREW_LIBYAML_PREFIX}")
+            set(LIBYAML_LIBRARY_DIRS "${BREW_LIBYAML_PREFIX}/lib")
+            set(LIBYAML_INCLUDEDIR "${BREW_LIBYAML_PREFIX}/include")
+            set(FLB_HAVE_LIBYAML 1)
+            FLB_DEFINITION(FLB_HAVE_LIBYAML)
+            include_directories(${LIBYAML_INCLUDEDIR})
+            link_directories(${LIBYAML_LIBRARY_DIRS})
+            message(STATUS "libyaml found via Homebrew: ${BREW_LIBYAML_PREFIX}")
+          endif()
+        endif()
+      endif()
+
+      # If still not found, try compile test
+      if (NOT FLB_HAVE_LIBYAML)
+        check_c_source_compiles("
+        #include <yaml.h>
+        int main() {
+          yaml_parser_t parser;
+          return 0;
+        }" FLB_HAVE_LIBYAML)
+
+        if(NOT FLB_HAVE_LIBYAML)
+          message(FATAL_ERROR
+            "YAML development dependencies required for YAML configuration format handling.\n"
+            "This is a build time dependency, you can either install the "
+            "dependencies or disable the feature setting the CMake option "
+            "-DFLB_CONFIG_YAML=Off ."
+          )
+        endif()
       endif()
     endif ()
 
@@ -1304,22 +1334,24 @@ if(FLB_OUT_PGSQL AND (NOT PostgreSQL_FOUND))
    FLB_OPTION(FLB_OUT_PGSQL OFF)
 endif()
 
-# Arrow GLib
-# ==========
-find_package(PkgConfig)
-pkg_check_modules(ARROW_GLIB QUIET arrow-glib)
-if(FLB_ARROW AND ARROW_GLIB_FOUND)
-  FLB_DEFINITION(FLB_HAVE_ARROW)
-else()
-  set(FLB_ARROW OFF)
-endif()
+# Parquet
+if(FLB_PARQUET_ENCODER)
+  # Enable C++ for Parquet support
+  enable_language(CXX)
+  set(CMAKE_CXX_STANDARD 17)
 
-# Additional prerequisites for Apache Parquet
-pkg_check_modules(ARROW_GLIB_PARQUET QUIET parquet-glib)
-if(FLB_ARROW AND ARROW_GLIB_PARQUET_FOUND)
-  FLB_DEFINITION(FLB_HAVE_ARROW_PARQUET)
-else()
-  message(STATUS "Arrow GLib Parquet not found. Disabling parquet compression")
+  # Detect Arrow and Parquet libraries
+  include(cmake/parquet.cmake)
+
+  if(ARROW_FOUND AND PARQUET_FOUND)
+    FLB_DEFINITION(FLB_HAVE_PARQUET_ENCODER)
+    include_directories(${ARROW_INCLUDE_DIRS})
+    include_directories(${PARQUET_INCLUDE_DIRS})
+  else()
+    message(WARNING "FLB_PARQUET_ENCODER is enabled but Arrow/Parquet libraries not found.")
+    message(WARNING "Disabling Parquet encoder support.")
+    set(FLB_PARQUET_ENCODER OFF)
+  endif()
 endif()
 
 # EBPF Support

cmake/kafka.cmake

@@ -1,44 +1,53 @@
 # Kafka CMake Configuration
-# kafka.cmake - Clean version without internal AWS check
 FLB_OPTION(RDKAFKA_BUILD_STATIC    On)
 FLB_OPTION(RDKAFKA_BUILD_EXAMPLES Off)
 FLB_OPTION(RDKAFKA_BUILD_TESTS    Off)
 FLB_OPTION(ENABLE_LZ4_EXT         Off)
 
 include(FindPkgConfig)
 
-# Check for libsasl2 (required for SASL authentication)
-set(FLB_SASL_ENABLED OFF)
+# librdkafka has built-in support for:
+# - SASL/PLAIN (built-in, no external deps)
+# - SASL/SCRAM (built-in, no external deps)
+# - SASL/OAUTHBEARER (built-in, no external deps)
+# Only SASL/GSSAPI (Kerberos) requires cyrus-sasl library
+
+# Check for cyrus-sasl (optional, only needed for GSSAPI/Kerberos)
+set(FLB_SASL_CYRUS_ENABLED OFF)
 if(PkgConfig_FOUND)
   pkg_check_modules(SASL libsasl2)
   if(SASL_FOUND)
-    message(STATUS "Found libsasl2: ${SASL_VERSION}")
-    set(FLB_SASL_ENABLED ON)
+    message(STATUS "Found cyrus-sasl: ${SASL_VERSION}")
+    set(FLB_SASL_CYRUS_ENABLED ON)
   else()
-    message(WARNING "libsasl2 not found - SASL authentication will be disabled")
+    message(STATUS "cyrus-sasl not found - SASL/GSSAPI (Kerberos) will be disabled")
   endif()
 else()
-  message(WARNING "pkg-config not available - trying fallback SASL detection")
-  # Fallback detection
+  message(STATUS "pkg-config not available - trying fallback cyrus-sasl detection")
   find_library(SASL2_LIB NAMES sasl2)
   find_path(SASL2_INCLUDE NAMES sasl/sasl.h)
   if(SASL2_LIB AND SASL2_INCLUDE)
-    set(FLB_SASL_ENABLED ON)
-    message(STATUS "Found libsasl2 via fallback: ${SASL2_LIB}")
+    set(FLB_SASL_CYRUS_ENABLED ON)
+    set(SASL_LIBRARIES ${SASL2_LIB})
+    set(SASL_INCLUDE_DIRS ${SASL2_INCLUDE})
+    message(STATUS "Found cyrus-sasl via fallback: ${SASL2_LIB}")
   endif()
 endif()
 
-# OAuth Bearer is built into librdkafka when SASL is available
-set(FLB_SASL_OAUTHBEARER_ENABLED ${FLB_SASL_ENABLED})
+# SASL is always enabled (built-in PLAIN/SCRAM/OAUTHBEARER support)
+set(FLB_SASL_ENABLED ON)
+
+# OAuth Bearer is built into librdkafka (no external deps needed)
+set(FLB_SASL_OAUTHBEARER_ENABLED ON)
 
-# MSK IAM requires OAuth Bearer support
-set(FLB_KAFKA_MSK_IAM_ENABLED ${FLB_SASL_OAUTHBEARER_ENABLED})
+# MSK IAM requires OAuth Bearer support (which is always available now)
+set(FLB_KAFKA_MSK_IAM_ENABLED ON)
 
 # Configure librdkafka options
-FLB_OPTION(WITH_SASL ${FLB_SASL_ENABLED})
-FLB_OPTION(WITH_SSL On)
-FLB_OPTION(WITH_SASL_OAUTHBEARER ${FLB_SASL_OAUTHBEARER_ENABLED})
-FLB_OPTION(WITH_SASL_CYRUS ${FLB_SASL_ENABLED})
+FLB_OPTION(WITH_SASL ON)                                        # Always ON (built-in PLAIN/SCRAM)
+FLB_OPTION(WITH_SSL On)                                        # SSL support
+FLB_OPTION(WITH_SASL_OAUTHBEARER ON)                          # Always ON (built-in)
+FLB_OPTION(WITH_SASL_CYRUS ${FLB_SASL_CYRUS_ENABLED})         # Only if cyrus-sasl found
 
 # Export compile-time definitions using FLB_DEFINITION macro
 if(FLB_SASL_ENABLED)
@@ -66,6 +75,12 @@ add_subdirectory(${FLB_PATH_LIB_RDKAFKA} EXCLUDE_FROM_ALL)
 
 set(KAFKA_LIBRARIES "rdkafka")
 
+# Add SASL libraries if cyrus-sasl is enabled
+if(FLB_SASL_CYRUS_ENABLED AND SASL_LIBRARIES)
+  list(APPEND KAFKA_LIBRARIES ${SASL_LIBRARIES})
+  message(STATUS "Added SASL libraries to Kafka: ${SASL_LIBRARIES}")
+endif()
+
 # Summary of what's enabled
 message(STATUS "=== Kafka Feature Summary ===")
 message(STATUS "SASL Auth:     ${FLB_SASL_ENABLED}")