Add ZSTD Compressor

BUILD.bazel

@@ -213,6 +213,7 @@ cc_library(
         "src/lib/OpenEXRCore/internal_win32_file_impl.h",
         "src/lib/OpenEXRCore/internal_xdr.h",
         "src/lib/OpenEXRCore/internal_zip.c",
+        "src/lib/OpenEXRCore/internal_zstd.c",
         "src/lib/OpenEXRCore/memory.c",
         "src/lib/OpenEXRCore/opaque.c",
         "src/lib/OpenEXRCore/openexr_version.h",
@@ -264,12 +265,14 @@ cc_library(
             ":windows": [],
             "//conditions:default": [
                 "-pthread",
+                "-ldl",
             ],
         }),
     visibility = ["//visibility:public"],
     deps = [
         "@imath",
         "@openjph",
+        "@c-blosc2",
         "@libdeflate//:deflate",
     ],
 )
@@ -373,6 +376,7 @@ cc_library(
         "src/lib/OpenEXR/ImfWav.cpp",
         "src/lib/OpenEXR/ImfZip.cpp",
         "src/lib/OpenEXR/ImfZipCompressor.cpp",
+        "src/lib/OpenEXR/ImfZstdCompressor.cpp",
     ],
     hdrs = [
         "src/lib/Iex/IexConfig.h",
@@ -493,6 +497,7 @@ cc_library(
         "src/lib/OpenEXR/ImfXdr.h",
         "src/lib/OpenEXR/ImfZip.h",
         "src/lib/OpenEXR/ImfZipCompressor.h",
+        "src/lib/OpenEXR/ImfZstdCompressor.h",
         "src/lib/OpenEXR/OpenEXRConfig.h",
         "src/lib/OpenEXR/OpenEXRConfigInternal.h",
     ],
@@ -509,6 +514,7 @@ cc_library(
             ":windows": [],
             "//conditions:default": [
                 "-pthread",
+                "-ldl",
             ],
         }),
     visibility = ["//visibility:public"],
@@ -517,6 +523,7 @@ cc_library(
         ":OpenEXRCore",
         "@imath",
         "@openjph"
+        "@c-blosc2",
     ],
 )
 

MODULE.bazel

@@ -10,5 +10,6 @@ bazel_dep(name = "bazel_skylib", version = "1.8.2")
 bazel_dep(name = "imath", version = "3.2.1")
 bazel_dep(name = "libdeflate", version = "1.24")
 bazel_dep(name = "openjph", version = "0.24.1")
+bazel_dep(name = "c-blosc2", version = "2.12.0.bcr.2")
 bazel_dep(name = "platforms", version = "1.0.0")
 bazel_dep(name = "rules_cc", version = "0.2.9")

cmake/CMakeLists.txt

@@ -192,6 +192,7 @@ if(OPENEXR_INSTALL_PKG_CONFIG)
   function(openexr_pkg_config_help pcinfile)
     string(TOUPPER "${CMAKE_BUILD_TYPE}" uppercase_CMAKE_BUILD_TYPE)
     set(LIB_SUFFIX_DASH ${OPENEXR_LIB_SUFFIX}${CMAKE_${uppercase_CMAKE_BUILD_TYPE}_POSTFIX})
+    set(LIB_BUILD_SUFFIX ${CMAKE_${uppercase_CMAKE_BUILD_TYPE}_POSTFIX})
     if(OPENEXR_ENABLE_THREADING AND TARGET Threads::Threads)
       # hrm, can't use properties as they end up as generator expressions
       # which don't seem to evaluate

cmake/OpenEXR.pc.in

@@ -9,12 +9,13 @@ libdir=@PKG_CONFIG_INSTALL_LIBDIR@
 includedir=@PKG_CONFIG_INSTALL_INCLUDEDIR@
 OpenEXR_includedir=${includedir}/OpenEXR
 libsuffix=@LIB_SUFFIX_DASH@
+libbuildsuffix=@LIB_BUILD_SUFFIX@
 
 Name: OpenEXR
 Description: OpenEXR image library
 Version: @OPENEXR_VERSION@
 
-Libs: @exr_pthread_libs@ -L${libdir} -lOpenEXR${libsuffix} -lOpenEXRUtil${libsuffix} -lOpenEXRCore${libsuffix} -lIex${libsuffix} -lIlmThread${libsuffix}
+Libs: @exr_pthread_libs@ -L${libdir} -lOpenEXR${libsuffix} -lOpenEXRUtil${libsuffix} -lOpenEXRCore${libsuffix} -lIex${libsuffix} -lIlmThread${libsuffix} -lblosc2${libbuildsuffix} -ldl
 Cflags: -I${includedir} -I${OpenEXR_includedir} @exr_pthread_cflags@
 Requires: Imath
 Requires.private: @EXR_DEFLATE_PKGCONFIG_REQUIRES@ @EXR_OPENJPH_PKGCONFIG_REQUIRES@

cmake/OpenEXRConfig.cmake.in

@@ -5,12 +5,10 @@
 
 include(CMakeFindDependencyMacro)
 
-set(openexr_needthreads @OPENEXR_ENABLE_THREADING@)
-if (openexr_needthreads)
-  set(THREADS_PREFER_PTHREAD_FLAG ON)
-  find_dependency(Threads)
-endif()
-unset(openexr_needthreads)
+# blosc2 needs threads, so we set it irrespective of OPENEXR_ENABLE_THREADING 
+# which enables threaded processing of requests.
+set(THREADS_PREFER_PTHREAD_FLAG ON)
+find_dependency(Threads)
 
 find_dependency(Imath)
 
@@ -22,5 +20,9 @@ if (@openjph_FOUND@)
   find_dependency(openjph)
 endif()
 
+if (@c-blosc2_FOUND@)
+  find_dependency(c-blosc2)
+endif()
+
 include("${CMAKE_CURRENT_LIST_DIR}/@[email protected]")
 check_required_components("@PROJECT_NAME@")

cmake/OpenEXRSetup.cmake

@@ -1,6 +1,20 @@
 # SPDX-License-Identifier: BSD-3-Clause
 # Copyright (c) Contributors to the OpenEXR Project.
 
+function(_error_if_not_found prop var fallback)
+  message(STATUS "Blosc2: ${prop}  ${var}  '${fallback}'")
+  string(FIND "${var}" "-NOTFOUND" pos)
+  if(NOT pos EQUAL -1)
+    if(fallback STREQUAL "")
+      message(FATAL_ERROR "Blosc2: Property ${prop} not found: ${var}")
+    else()
+      string(SUBSTRING "${var}" 0 ${pos} var_name)
+      message(STATUS "Blosc2: Property ${prop} not found: ${var_name} falling back to '${fallback}'")
+      set(${var_name} "${fallback}" PARENT_SCOPE) 
+    endif()
+  endif()
+endfunction(_error_if_not_found)
+
 include(GNUInstallDirs)
 
 if(NOT "${CMAKE_PROJECT_NAME}" STREQUAL "${PROJECT_NAME}")
@@ -251,6 +265,83 @@ else()
 endif()
 
 
+#######################################
+# Find or install Blosc2
+#######################################
+
+set(MINIMUM_BLOSC2_VERSION 2.11.0)
+option(OPENEXR_FORCE_INTERNAL_BLOSC2 [=[Force using installed Blosc2.]=] OFF)
+
+set(BLOSC2_REPO "https://github.com/Blosc/c-blosc2.git" CACHE STRING "Repo path for blosc2 source")
+set(BLOSC2_TAG "v${MINIMUM_BLOSC2_VERSION}" CACHE STRING "Tag to use for blosc2 source repo")
+
+# Try to find a local bloc2 install if allowed to.
+if(NOT OPENEXR_FORCE_INTERNAL_BLOSC2)
+  message(STATUS "Blosc2: Looking for local install...")
+  set(CMAKE_IGNORE_PATH "${CMAKE_CURRENT_BINARY_DIR}/_deps/blosc2-src/config;${CMAKE_CURRENT_BINARY_DIR}/_deps/blosc2-build/config")
+  find_package(Blosc2 ${MINIMUM_BLOSC2_VERSION})
+  set(CMAKE_IGNORE_PATH)
+endif()
+
+if(NOT TARGET Blosc2::blosc2_static AND NOT Blosc2_FOUND)
+  # we didn't find a local install: let's get it from its repository.
+  if(OPENEXR_FORCE_INTERNAL_BLOSC2)
+    message(STATUS "Blosc2: forced internal, installing from ${BLOSC2_REPO} (${BLOSC2_TAG})")
+  else()
+    message(STATUS "Blosc2: no local blosc2 found, installing from ${BLOSC2_REPO} (${BLOSC2_TAG})")
+  endif()
+
+  # configure the blosc2 build
+  set(BUILD_BENCHMARKS OFF CACHE INTERNAL "no benchmarks")
+  set(BUILD_EXAMPLES OFF CACHE INTERNAL "no examples")
+  set(BUILD_FUZZERS OFF CACHE INTERNAL "no fuzzer")
+  set(BUILD_SHARED OFF CACHE INTERNAL "no shared library")
+  set(BUILD_TESTS OFF CACHE INTERNAL "no tests")
+
+  include(FetchContent)
+  FetchContent_Declare(Blosc2
+    GIT_REPOSITORY "${BLOSC2_REPO}"
+    GIT_TAG "${BLOSC2_TAG}"
+    GIT_SHALLOW ON
+    GIT_PROGRESS ON)
+
+  FetchContent_GetProperties(Blosc2)
+  if(NOT Blosc2_POPULATED)
+    message(STATUS "Blosc2: Downloading ${BLOSC2_TAG} from ${BLOSC2_REPO}...")
+    FetchContent_Populate(Blosc2)
+    add_subdirectory(${blosc2_SOURCE_DIR} ${blosc2_BINARY_DIR})
+  else()
+    message(STATUS "Blosc2: repo code has already been downloaded.")
+  endif()
+
+  # the install creates this but if we're using the library locally we
+  # haven't installed the header files yet, so need to extract those
+  # and make a variable for header only usage
+  if(TARGET Blosc2::blosc2_static)
+    message(STATUS "Blosc2: Setting up blosc directories")
+
+    get_target_property(blosc2inc Blosc2::blosc2_static INCLUDE_DIRECTORIES)
+    set(BLOSC2_INCLUDE_DIRS ${blosc2inc})
+
+    get_target_property(blosc2libdir Blosc2::blosc2_static BINARY_DIR)
+    set(BLOSC2_LIB_DIR ${blosc2libdir})
+  endif()
+else()
+  message(STATUS "Blosc2: Using installed Blosc2 ${Blosc2_VERSION} from ${Blosc2_DIR}")
+  # local build
+  if(TARGET Blosc2::blosc2_static)
+    message(STATUS "Blosc2: Setting up installed blosc directories")
+
+    get_target_property(blosc2inc Blosc2::blosc2_static INTERFACE_INCLUDE_DIRECTORIES)
+    _error_if_not_found("INTERFACE_INCLUDE_DIRECTORIES" ${blosc2inc} "")
+    set(BLOSC2_INCLUDE_DIRS ${blosc2inc})
+
+    get_target_property(blosc2libdir Blosc2::blosc2_static BINARY_DIR)
+    _error_if_not_found("BINARY_DIR" ${blosc2libdir} "")
+    set(BLOSC2_LIB_DIR ${blosc2libdir})
+  endif()
+endif()
+
 #######################################
 # Find or download OpenJPH
 #######################################

src/bin/exrmetrics/exrmetrics.cpp

@@ -1022,9 +1022,10 @@ exrmetrics (
                     outHeaders[p].zipCompressionLevel () = level;
                     compressionSet                       = true;
                     break;
-                    //            case ZSTD_COMPRESSION :
-                    //                outHeader.zstdCompressionLevel()=level;
-                    //                break;
+                case ZSTD_COMPRESSION :
+                    outHeader[p].zstdCompressionLevel () = level;
+                    compressionSet                       = true;
+                    break;
                 default: break;
             }
         }

src/bin/exrmetrics/main.cpp

@@ -64,7 +64,7 @@ usageMessage (ostream& stream, const char* program_name, bool verbose = false)
                "  -t n                        Use a pool of n worker threads for processing files.\n"
                "                              Default is single threaded (no thread pool)\n"
                "\n"
-               "  -l level                    set DWA or ZIP compression level\n"
+               "  -l level                    set DWA, ZIP or ZSTD compression level\n"
                "\n"
                "  -z,--compression list       list of compression methods to test\n"
                "                              ("

src/lib/OpenEXR/CMakeLists.txt

@@ -125,6 +125,8 @@ openexr_define_library(OpenEXR
     ImfZip.h
     ImfZipCompressor.cpp
     ImfZipCompressor.h
+    ImfZstdCompressor.cpp
+    ImfZstdCompressor.h
   HEADERS
     ImfAcesFile.h
     ImfArray.h
@@ -222,6 +224,7 @@ openexr_define_library(OpenEXR
     ImfXdr.h
   DEPENDENCIES
     Imath::Imath
+    Blosc2::blosc2_static
     OpenEXR::Config
     OpenEXR::Iex
     OpenEXR::IlmThread

src/lib/OpenEXR/ImfCRgbaFile.h

@@ -82,7 +82,8 @@ typedef struct ImfRgba ImfRgba;
 #define IMF_DWAB_COMPRESSION 9
 #define IMF_HTJ2K256_COMPRESSION 10
 #define IMF_HTJ2K32_COMPRESSION 11
-#define IMF_NUM_COMPRESSION_METHODS 12
+#define IMF_ZSTD_COMPRESSION 12
+#define IMF_NUM_COMPRESSION_METHODS 13
 
 /*
 ** Channels; values must be the same as in Imf::RgbaChannels.

src/lib/OpenEXR/ImfCompression.cpp

@@ -188,6 +188,12 @@ static const CompressionDesc IdToDesc[] = {
         32,
         true,
         false),
+    CompressionDesc (
+        "zstd",
+        "blosc zstd lossless compression, one scan line at a time.",
+        1,
+        false,
+        true),
 };
 // clang-format on
 
@@ -206,6 +212,7 @@ static const std::map<std::string, Compression> CompressionNameToId = {
     {"dwab", Compression::DWAB_COMPRESSION},
     {"htj2k256", Compression::HTJ2K256_COMPRESSION},
     {"htj2k32", Compression::HTJ2K32_COMPRESSION},
+    {"zstd", Compression::ZSTD_COMPRESSION},
 };
 
 #define UNKNOWN_COMPRESSION_ID_MSG "INVALID COMPRESSION ID"

src/lib/OpenEXR/ImfCompression.h

@@ -55,6 +55,9 @@ enum IMF_EXPORT_ENUM Compression
 
     HTJ2K32_COMPRESSION = 11,    // High-Throughput JPEG2000 (HTJ2K), 32 scanlines
 
+    ZSTD_COMPRESSION = 12, // blosc zstd lossless compression, one scan line
+                           // at a time.
+
     NUM_COMPRESSION_METHODS // number of different compression methods
 };
 
@@ -92,6 +95,9 @@ IMF_EXPORT void setDefaultZipCompressionLevel (int level);
 /// Controls the default quality level for the DWA lossy compression
 IMF_EXPORT void setDefaultDwaCompressionLevel (float level);
 
+/// Controls the default zstd compression level used. Zstd is used for
+IMF_EXPORT void setDefaultZstdCompressionLevel (int level);
+
 OPENEXR_IMF_INTERNAL_NAMESPACE_HEADER_EXIT
 
 #endif

src/lib/OpenEXR/ImfCompressor.cpp

@@ -17,6 +17,7 @@
 #include "ImfPizCompressor.h"
 #include "ImfPxr24Compressor.h"
 #include "ImfRleCompressor.h"
+#include "ImfZstdCompressor.h"
 #include "ImfZipCompressor.h"
 #include "ImfZip.h"
 
@@ -54,6 +55,7 @@ Compressor::Compressor (
 
     exr_set_zip_compression_level (_ctxt, 0, hdr.zipCompressionLevel ());
     exr_set_dwa_compression_level (_ctxt, 0, hdr.dwaCompressionLevel ());
+    exr_set_zstd_compression_level (_ctxt, 0, hdr.zstdCompressionLevel ());
 
     exr_compression_t hdrcomp;
     if (EXR_ERR_SUCCESS != exr_get_compression (_ctxt, 0, &hdrcomp))

src/lib/OpenEXR/ImfContextInit.h

@@ -105,6 +105,12 @@ class IMF_EXPORT_TYPE ContextInitializer
         return *this;
     }
 
+    ContextInitializer& setZstdLevel (int zl) noexcept
+    {
+        _initializer.zstd_level = zl;
+        return *this;
+    }
+
     ContextInitializer& strictHeaderValidation (bool onoff) noexcept
     {
         setFlag (EXR_CONTEXT_FLAG_STRICT_HEADER, onoff);

src/lib/OpenEXR/ImfHeader.cpp

@@ -71,9 +71,11 @@ struct CompressionRecord
     {
         exr_get_default_zip_compression_level (&zip_level);
         exr_get_default_dwa_compression_quality (&dwa_level);
+        exr_get_default_zstd_compression_level (&zstd_level);
     }
     int   zip_level;
     float dwa_level;
+    int   zstd_level;
 };
 // NB: This is extra complicated than one would normally write to
 // handle scenario that seems to happen on MacOS/Windows (probably
@@ -693,6 +695,18 @@ Header::zipCompressionLevel () const
     return retrieveCompressionRecord (this).zip_level;
 }
 
+int&
+Header::zstdCompressionLevel ()
+{
+    return retrieveCompressionRecord (this).zstd_level;
+}
+
+int
+Header::zstdCompressionLevel () const
+{
+    return retrieveCompressionRecord (this).zstd_level;
+}
+
 float&
 Header::dwaCompressionLevel ()
 {

src/lib/OpenEXR/ImfHeader.h

@@ -285,6 +285,10 @@ class IMF_EXPORT_TYPE Header
     float& dwaCompressionLevel ();
     IMF_EXPORT
     float dwaCompressionLevel () const;
+    IMF_EXPORT
+    int& zstdCompressionLevel ();
+    IMF_EXPORT
+    int zstdCompressionLevel () const;
 
     //-----------------------------------------------------
     // Access to required attributes for multipart files

src/lib/OpenEXR/ImfZstdCompressor.cpp

@@ -0,0 +1,17 @@
+//
+// SPDX-License-Identifier: BSD-3-Clause
+// Copyright (c) Contributors to the OpenEXR Project.
+//
+
+#include "ImfZstdCompressor.h"
+
+OPENEXR_IMF_INTERNAL_NAMESPACE_SOURCE_ENTER
+
+ZstdCompressor::ZstdCompressor (const Header& hdr, size_t maxScanLineSize, int scanLines)
+    : Compressor (hdr, EXR_COMPRESSION_ZSTD, maxScanLineSize, scanLines)
+{}
+
+ZstdCompressor::~ZstdCompressor ()
+{}
+
+OPENEXR_IMF_INTERNAL_NAMESPACE_SOURCE_EXIT