Add support for brotli compression (#37)

.gitmodules

@@ -7,3 +7,6 @@
 [submodule "zstd"]
 	path = zstd
 	url = https://github.com/facebook/zstd.git
+[submodule "brotli"]
+	path = brotli
+	url = https://github.com/google/brotli.git

ChangeLog

@@ -1,3 +1,6 @@
+2024-06-23 Konstantin Kushnir <[email protected]>
+	* Add support for brotli compression
+
 2024-06-21 Konstantin Kushnir <[email protected]>
 	* Refactor compression functions
 

Makefile.in

@@ -314,13 +314,18 @@ $(PKG_STUB_LIB_FILE): $(PKG_STUB_OBJECTS)
 # As necessary, add $(srcdir):$(srcdir)/compat:....
 #========================================================================
 
-VPATH = $(srcdir):$(srcdir)/generic:$(srcdir)/unix:$(srcdir)/win:$(srcdir)/bzip2:$(srcdir)/7zip/C:$(srcdir)/zstd/lib:$(srcdir)/zstd/lib/common:$(srcdir)/zstd/lib/compress:$(srcdir)/zstd/lib/decompress
+VPATH = $(srcdir):$(srcdir)/generic:$(srcdir)/unix:$(srcdir)/win:$(srcdir)/bzip2:$(srcdir)/7zip/C:$(srcdir)/zstd/lib:$(srcdir)/zstd/lib/common:$(srcdir)/zstd/lib/compress:$(srcdir)/zstd/lib/decompress:$(srcdir)/brotli/c/common:$(srcdir)/brotli/c/enc:$(srcdir)/brotli/c/dec
 
 override TESTFLAGS += -verbose "body error start"
 
 .c.@OBJEXT@:
 	$(COMPILE) -c `@CYGPATH@ $<` -o $@
 
+# This is a hack to build huffman.o because of a conflict between
+# the huffman.c files from the bzip2 and brotli sources.
+brotli_huffman.@OBJEXT@: brotli/c/dec/huffman.c
+	$(COMPILE) -c `@CYGPATH@ $<` -o $@
+
 %.tcl: $(srcdir)/scripts/%.tcl
 	cp -f "$<" "$@"
 

README.md

@@ -67,3 +67,8 @@ This package contains Zstandard source code, which is copyright Meta Platforms,
 
 License for Zstandard is contained in file "zstd/LICENSE" and applies to all
 files under zstd/ subdirectory.
+
+This package contains brotli source code, which is copyright the Brotli Authors.
+
+License for brotli is contained in file "brotli/LICENSE" and applies to all
+files under brotli/ subdirectory.

configure

@@ -662,6 +662,7 @@ COOKFS_PKGCONFIG_USECWRITER
 COOKFS_PKGCONFIG_USECVFS
 COOKFS_PKGCONFIG_FEATURE_METADATA
 COOKFS_PKGCONFIG_USECFSINDEX
+COOKFS_PKGCONFIG_USEBROTLI
 COOKFS_PKGCONFIG_USEZSTD
 COOKFS_PKGCONFIG_USELZMA
 COOKFS_PKGCONFIG_USEBZ2
@@ -790,6 +791,7 @@ enable_internal_debug
 enable_bz2
 enable_lzma
 enable_zstd
+enable_brotli
 enable_tcl_fallback
 enable_c_pages
 enable_c_fsindex
@@ -1439,6 +1441,7 @@ Optional Features:
   --enable-bz2              Use bz2 compression. Defaults to false
   --disable-lzma            Use lzma compression. Defaults to true
   --enable-zstd             Use zstd compression. Defaults to false
+  --enable-brotli           Use brotli compression. Defaults to false
   --enable-tcl-fallback     Install Tcl scripts if load fails. Defaults to false
   --disable-c-pages         Use pages handling written in C. Defaults to true
   --disable-c-fsindex       Use index handling written in C. Defaults to true
@@ -8612,6 +8615,14 @@ else $as_nop
   USEZSTD=no
 fi
 
+# Check whether --enable-brotli was given.
+if test ${enable_brotli+y}
+then :
+  enableval=$enable_brotli; USEBROTLI=${enableval}
+else $as_nop
+  USEBROTLI=no
+fi
+
 # Check whether --enable-tcl-fallback was given.
 if test ${enable_tcl_fallback+y}
 then :
@@ -9177,11 +9188,203 @@ rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext
         COOKFS_PKGCONFIG_USEZSTD=0
     fi
 
+    if test ${USEBROTLI} = yes; then
+
+    vars="-I\"`${CYGPATH} ${srcdir}/brotli/c/include`\""
+    for i in $vars; do
+	PKG_INCLUDES="$PKG_INCLUDES $i"
+    done
+
+
+
+    vars="pagesComprBrotli.c"
+    for i in $vars; do
+	case $i in
+	    \$*)
+		# allow $-var names
+		PKG_SOURCES="$PKG_SOURCES $i"
+		PKG_OBJECTS="$PKG_OBJECTS $i"
+		;;
+	    *)
+		# check for existence - allows for generic/win/unix VPATH
+		# To add more dirs here (like 'src'), you have to update VPATH
+		# in Makefile.in as well
+		if test ! -f "${srcdir}/$i" -a ! -f "${srcdir}/generic/$i" \
+		    -a ! -f "${srcdir}/win/$i" -a ! -f "${srcdir}/unix/$i" \
+		    -a ! -f "${srcdir}/macosx/$i" \
+		    ; then
+		    as_fn_error $? "could not find source file '$i'" "$LINENO" 5
+		fi
+		PKG_SOURCES="$PKG_SOURCES $i"
+		# this assumes it is in a VPATH dir
+		i=`basename $i`
+		# handle user calling this before or after TEA_SETUP_COMPILER
+		if test x"${OBJEXT}" != x ; then
+		    j="`echo $i | sed -e 's/\.[^.]*$//'`.${OBJEXT}"
+		else
+		    j="`echo $i | sed -e 's/\.[^.]*$//'`.\${OBJEXT}"
+		fi
+		PKG_OBJECTS="$PKG_OBJECTS $j"
+		;;
+	esac
+    done
+
+
+
+
+    vars="
+            brotli/c/common/constants.c
+            brotli/c/common/platform.c
+            brotli/c/common/context.c
+            brotli/c/common/transform.c
+            brotli/c/common/shared_dictionary.c
+            brotli/c/common/dictionary.c
+        "
+    for i in $vars; do
+	case $i in
+	    \$*)
+		# allow $-var names
+		PKG_SOURCES="$PKG_SOURCES $i"
+		PKG_OBJECTS="$PKG_OBJECTS $i"
+		;;
+	    *)
+		# check for existence - allows for generic/win/unix VPATH
+		# To add more dirs here (like 'src'), you have to update VPATH
+		# in Makefile.in as well
+		if test ! -f "${srcdir}/$i" -a ! -f "${srcdir}/generic/$i" \
+		    -a ! -f "${srcdir}/win/$i" -a ! -f "${srcdir}/unix/$i" \
+		    -a ! -f "${srcdir}/macosx/$i" \
+		    ; then
+		    as_fn_error $? "could not find source file '$i'" "$LINENO" 5
+		fi
+		PKG_SOURCES="$PKG_SOURCES $i"
+		# this assumes it is in a VPATH dir
+		i=`basename $i`
+		# handle user calling this before or after TEA_SETUP_COMPILER
+		if test x"${OBJEXT}" != x ; then
+		    j="`echo $i | sed -e 's/\.[^.]*$//'`.${OBJEXT}"
+		else
+		    j="`echo $i | sed -e 's/\.[^.]*$//'`.\${OBJEXT}"
+		fi
+		PKG_OBJECTS="$PKG_OBJECTS $j"
+		;;
+	esac
+    done
+
+
+
+
+    vars="
+            brotli/c/enc/encode.c
+            brotli/c/enc/memory.c
+            brotli/c/enc/fast_log.c
+            brotli/c/enc/utf8_util.c
+            brotli/c/enc/brotli_bit_stream.c
+            brotli/c/enc/block_splitter.c
+            brotli/c/enc/metablock.c
+            brotli/c/enc/encoder_dict.c
+            brotli/c/enc/compress_fragment.c
+            brotli/c/enc/compress_fragment_two_pass.c
+            brotli/c/enc/backward_references_hq.c
+            brotli/c/enc/backward_references.c
+            brotli/c/enc/entropy_encode.c
+            brotli/c/enc/command.c
+            brotli/c/enc/static_dict.c
+            brotli/c/enc/literal_cost.c
+            brotli/c/enc/bit_cost.c
+            brotli/c/enc/histogram.c
+            brotli/c/enc/compound_dictionary.c
+            brotli/c/enc/dictionary_hash.c
+            brotli/c/enc/cluster.c
+        "
+    for i in $vars; do
+	case $i in
+	    \$*)
+		# allow $-var names
+		PKG_SOURCES="$PKG_SOURCES $i"
+		PKG_OBJECTS="$PKG_OBJECTS $i"
+		;;
+	    *)
+		# check for existence - allows for generic/win/unix VPATH
+		# To add more dirs here (like 'src'), you have to update VPATH
+		# in Makefile.in as well
+		if test ! -f "${srcdir}/$i" -a ! -f "${srcdir}/generic/$i" \
+		    -a ! -f "${srcdir}/win/$i" -a ! -f "${srcdir}/unix/$i" \
+		    -a ! -f "${srcdir}/macosx/$i" \
+		    ; then
+		    as_fn_error $? "could not find source file '$i'" "$LINENO" 5
+		fi
+		PKG_SOURCES="$PKG_SOURCES $i"
+		# this assumes it is in a VPATH dir
+		i=`basename $i`
+		# handle user calling this before or after TEA_SETUP_COMPILER
+		if test x"${OBJEXT}" != x ; then
+		    j="`echo $i | sed -e 's/\.[^.]*$//'`.${OBJEXT}"
+		else
+		    j="`echo $i | sed -e 's/\.[^.]*$//'`.\${OBJEXT}"
+		fi
+		PKG_OBJECTS="$PKG_OBJECTS $j"
+		;;
+	esac
+    done
+
+
+
+        # This is a hack to build huffman.o because of a conflict between
+        # the huffman.c files from the bzip2 and brotli sources.
+        PKG_OBJECTS="$PKG_OBJECTS brotli_huffman.${OBJEXT}"
+
+    vars="
+            brotli/c/dec/decode.c
+            brotli/c/dec/bit_reader.c
+            brotli/c/dec/state.c
+        "
+    for i in $vars; do
+	case $i in
+	    \$*)
+		# allow $-var names
+		PKG_SOURCES="$PKG_SOURCES $i"
+		PKG_OBJECTS="$PKG_OBJECTS $i"
+		;;
+	    *)
+		# check for existence - allows for generic/win/unix VPATH
+		# To add more dirs here (like 'src'), you have to update VPATH
+		# in Makefile.in as well
+		if test ! -f "${srcdir}/$i" -a ! -f "${srcdir}/generic/$i" \
+		    -a ! -f "${srcdir}/win/$i" -a ! -f "${srcdir}/unix/$i" \
+		    -a ! -f "${srcdir}/macosx/$i" \
+		    ; then
+		    as_fn_error $? "could not find source file '$i'" "$LINENO" 5
+		fi
+		PKG_SOURCES="$PKG_SOURCES $i"
+		# this assumes it is in a VPATH dir
+		i=`basename $i`
+		# handle user calling this before or after TEA_SETUP_COMPILER
+		if test x"${OBJEXT}" != x ; then
+		    j="`echo $i | sed -e 's/\.[^.]*$//'`.${OBJEXT}"
+		else
+		    j="`echo $i | sed -e 's/\.[^.]*$//'`.\${OBJEXT}"
+		fi
+		PKG_OBJECTS="$PKG_OBJECTS $j"
+		;;
+	esac
+    done
+
+
+
+        printf "%s\n" "#define COOKFS_USEBROTLI 1" >>confdefs.h
+
+        COOKFS_PKGCONFIG_USEBROTLI=1
+    else
+        COOKFS_PKGCONFIG_USEBROTLI=0
+    fi
+
 else
     COOKFS_PKGCONFIG_USECPAGES=0
     COOKFS_PKGCONFIG_FEATURE_ASIDE=0
     COOKFS_PKGCONFIG_USELZMA=0
     COOKFS_PKGCONFIG_USEZSTD=0
+    COOKFS_PKGCONFIG_USEBROTLI=0
 
     # Tcl implementation offers bzip2 compression only with Trf package
     COOKFS_PKGCONFIG_USEBZ2=0
@@ -9452,6 +9655,7 @@ fi
 
 
 
+
 if test ${USECPKGCONFIG} = yes; then
     COOKFS_PKGCONFIG_USECPKGCONFIG=1
     printf "%s\n" "#define COOKFS_PKGCONFIG_FEATURE_ASIDE $COOKFS_PKGCONFIG_FEATURE_ASIDE" >>confdefs.h
@@ -9470,6 +9674,8 @@ if test ${USECPKGCONFIG} = yes; then
 
     printf "%s\n" "#define COOKFS_PKGCONFIG_USEZSTD $COOKFS_PKGCONFIG_USEZSTD" >>confdefs.h
 
+    printf "%s\n" "#define COOKFS_PKGCONFIG_USEBROTLI $COOKFS_PKGCONFIG_USEBROTLI" >>confdefs.h
+
     printf "%s\n" "#define COOKFS_PKGCONFIG_USECFSINDEX $COOKFS_PKGCONFIG_USECFSINDEX" >>confdefs.h
 
     printf "%s\n" "#define COOKFS_PKGCONFIG_FEATURE_METADATA $COOKFS_PKGCONFIG_FEATURE_METADATA" >>confdefs.h

configure.in

@@ -118,6 +118,7 @@ AC_ARG_ENABLE(internal-debug,  [  --enable-internal-debug   Enable internal debu
 AC_ARG_ENABLE(bz2,             [  --enable-bz2              Use bz2 compression. Defaults to false],                USEBZ2=${enableval},           USEBZ2=no)
 AC_ARG_ENABLE(lzma,            [  --disable-lzma            Use lzma compression. Defaults to true],                USELZMA=${enableval},          USELZMA=yes)
 AC_ARG_ENABLE(zstd,            [  --enable-zstd             Use zstd compression. Defaults to false],               USEZSTD=${enableval},          USEZSTD=no)
+AC_ARG_ENABLE(brotli,          [  --enable-brotli           Use brotli compression. Defaults to false],             USEBROTLI=${enableval},        USEBROTLI=no)
 AC_ARG_ENABLE(tcl-fallback,    [  --enable-tcl-fallback     Install Tcl scripts if load fails. Defaults to false],  USEFALLBACK=${enableval},      USEFALLBACK=no)
 AC_ARG_ENABLE(c-pages,         [  --disable-c-pages         Use pages handling written in C. Defaults to true],     USECPAGES=${enableval},        USECPAGES=yes)
 AC_ARG_ENABLE(c-fsindex,       [  --disable-c-fsindex       Use index handling written in C. Defaults to true],     USECFSINDEX=${enableval},      USECFSINDEX=yes)
@@ -230,11 +231,60 @@ if test ${USECPAGES} = yes; then
         COOKFS_PKGCONFIG_USEZSTD=0
     fi
 
+    if test ${USEBROTLI} = yes; then
+        TEA_ADD_INCLUDES([-I\"`${CYGPATH} ${srcdir}/brotli/c/include`\"])
+        TEA_ADD_SOURCES([pagesComprBrotli.c])
+        TEA_ADD_SOURCES([
+            brotli/c/common/constants.c
+            brotli/c/common/platform.c
+            brotli/c/common/context.c
+            brotli/c/common/transform.c
+            brotli/c/common/shared_dictionary.c
+            brotli/c/common/dictionary.c
+        ])
+        TEA_ADD_SOURCES([
+            brotli/c/enc/encode.c
+            brotli/c/enc/memory.c
+            brotli/c/enc/fast_log.c
+            brotli/c/enc/utf8_util.c
+            brotli/c/enc/brotli_bit_stream.c
+            brotli/c/enc/block_splitter.c
+            brotli/c/enc/metablock.c
+            brotli/c/enc/encoder_dict.c
+            brotli/c/enc/compress_fragment.c
+            brotli/c/enc/compress_fragment_two_pass.c
+            brotli/c/enc/backward_references_hq.c
+            brotli/c/enc/backward_references.c
+            brotli/c/enc/entropy_encode.c
+            brotli/c/enc/command.c
+            brotli/c/enc/static_dict.c
+            brotli/c/enc/literal_cost.c
+            brotli/c/enc/bit_cost.c
+            brotli/c/enc/histogram.c
+            brotli/c/enc/compound_dictionary.c
+            brotli/c/enc/dictionary_hash.c
+            brotli/c/enc/cluster.c
+        ])
+        # This is a hack to build huffman.o because of a conflict between
+        # the huffman.c files from the bzip2 and brotli sources.
+        PKG_OBJECTS="$PKG_OBJECTS brotli_huffman.${OBJEXT}"
+        TEA_ADD_SOURCES([
+            brotli/c/dec/decode.c
+            brotli/c/dec/bit_reader.c
+            brotli/c/dec/state.c
+        ])
+        AC_DEFINE(COOKFS_USEBROTLI)
+        COOKFS_PKGCONFIG_USEBROTLI=1
+    else
+        COOKFS_PKGCONFIG_USEBROTLI=0
+    fi
+
 else
     COOKFS_PKGCONFIG_USECPAGES=0
     COOKFS_PKGCONFIG_FEATURE_ASIDE=0
     COOKFS_PKGCONFIG_USELZMA=0
     COOKFS_PKGCONFIG_USEZSTD=0
+    COOKFS_PKGCONFIG_USEBROTLI=0
 
     # Tcl implementation offers bzip2 compression only with Trf package
     COOKFS_PKGCONFIG_USEBZ2=0
@@ -323,6 +373,7 @@ AC_SUBST(COOKFS_PKGCONFIG_USECWRITERCHAN)
 AC_SUBST(COOKFS_PKGCONFIG_USEBZ2)
 AC_SUBST(COOKFS_PKGCONFIG_USELZMA)
 AC_SUBST(COOKFS_PKGCONFIG_USEZSTD)
+AC_SUBST(COOKFS_PKGCONFIG_USEBROTLI)
 AC_SUBST(COOKFS_PKGCONFIG_USECFSINDEX)
 AC_SUBST(COOKFS_PKGCONFIG_FEATURE_METADATA)
 AC_SUBST(COOKFS_PKGCONFIG_USECVFS)
@@ -339,6 +390,7 @@ if test ${USECPKGCONFIG} = yes; then
     AC_DEFINE_UNQUOTED(COOKFS_PKGCONFIG_USEBZ2, $COOKFS_PKGCONFIG_USEBZ2)
     AC_DEFINE_UNQUOTED(COOKFS_PKGCONFIG_USELZMA, $COOKFS_PKGCONFIG_USELZMA)
     AC_DEFINE_UNQUOTED(COOKFS_PKGCONFIG_USEZSTD, $COOKFS_PKGCONFIG_USEZSTD)
+    AC_DEFINE_UNQUOTED(COOKFS_PKGCONFIG_USEBROTLI, $COOKFS_PKGCONFIG_USEBROTLI)
     AC_DEFINE_UNQUOTED(COOKFS_PKGCONFIG_USECFSINDEX, $COOKFS_PKGCONFIG_USECFSINDEX)
     AC_DEFINE_UNQUOTED(COOKFS_PKGCONFIG_FEATURE_METADATA, $COOKFS_PKGCONFIG_FEATURE_METADATA)
     AC_DEFINE_UNQUOTED(COOKFS_PKGCONFIG_USECVFS, $COOKFS_PKGCONFIG_USECVFS)