Merge pull request #2435 from facebook/dev

v1.4.8 hotfix

.circleci/config.yml

@@ -143,8 +143,9 @@ workflows:
           filters:
             branches:
               only:
-                - master
+                - release
                 - dev
+                - master
     jobs:
       # Run daily long regression tests
       - regression-test

.github/workflows/generic-dev.yml

@@ -2,14 +2,13 @@ name: generic-dev
 
 on:
   pull_request:
-    branches: [ dev, master, actionsTest ]
+    branches: [ dev, release, actionsTest ]
 
 jobs:
 
 # Dev PR jobs that still have to be migrated from travis
 #
-# icc (need self-hosted)
-# versionTag
+# versionTag (only on release tags)
 # valgrindTest (keeps failing for some reason. need investigation)
 # staticAnalyze (need trusty so need self-hosted)
 # pcc-fuzz: (need trusty so need self-hosted)
@@ -19,7 +18,7 @@ jobs:
 # I need admins permissions to the repo for that it looks like
 # So I'm tabling that for now
 #
-# The master branch exclusive jobs will be in a separate
+# The release branch exclusive jobs will be in a separate
 # workflow file (the osx tests and meson build that is)
 
   benchmarking:
@@ -36,6 +35,15 @@ jobs:
     - name: make test
       run: make test
 
+  check-32bit: # designed to catch https://github.com/facebook/zstd/issues/2428
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: make check on 32-bit
+      run: |
+        make libc6install
+        CFLAGS="-m32 -O1 -fstack-protector" make check V=1
+
   gcc-6-7-libzstd:
     runs-on: ubuntu-latest
     steps:
@@ -50,6 +58,10 @@ jobs:
         LDFLAGS=-Wl,--no-undefined make -C lib libzstd-mt
         make -C tests zbufftest-dll
 
+    # candidate test (to check) : underlink test
+    # LDFLAGS=-Wl,--no-undefined : will make the linker fail if dll is underlinked
+    # zbufftest-dll : test that a user program can link to multi-threaded libzstd without specifying -pthread
+
   gcc-8-asan-ubsan-testzstd:
     runs-on: ubuntu-16.04 # fails on 18.04
     steps:
@@ -84,6 +96,10 @@ jobs:
         sudo apt-get install clang-3.8
         CC=clang-3.8 make clean msan-test-zstd HAVE_ZLIB=0 HAVE_LZ4=0 HAVE_LZMA=0
 
+    # Note : external libraries must be turned off when using MSAN tests,
+    # because they are not msan-instrumented,
+    # so any data coming from these libraries is always considered "uninitialized"
+
   cmake-build-and-test-check:
     runs-on: ubuntu-latest
     steps:
@@ -147,7 +163,7 @@ jobs:
         make clean
         make c99build
         make clean
-        make travis-install
+        make travis-install   # just ensures `make install` works
 
   mingw-cross-compilation:
     runs-on: ubuntu-latest

.github/workflows/generic-release.yml

@@ -2,10 +2,10 @@ name: generic-release
 
 on:
   pull_request:
-    # This will eventually only be for pushes to master
+    # This will eventually only be for pushes to release
     # but for dogfooding purposes, I'm running it even
     # on dev pushes
-    branches: [ dev, master, actionsTest ]
+    branches: [ dev, release, actionsTest ]
 
 jobs:
   # missing jobs
@@ -44,6 +44,7 @@ jobs:
         sudo apt-get install clang-3.8
         CC=clang-3.8 make tsan-test-zstream
         CC=clang-3.8 make tsan-fuzztest
+
   zlib-wrapper:
     runs-on: ubuntu-16.04
     steps:

.github/workflows/linux-kernel.yml

@@ -2,7 +2,7 @@ name: linux-kernel
 
 on:
   pull_request:
-    branches: [ dev, master, actionsTest ]
+    branches: [ dev, release, actionsTest ]
 
 jobs:
   test:

.travis.yml

@@ -8,6 +8,7 @@ git:
 branches:
   only:
   - dev
+  - release
   - master
   - travisTest
 
@@ -31,49 +32,11 @@ matrix:
       script:
         - make check
 
-    - name: make benchmarking
-      script:
-        - make benchmarking
-
     - name: make test (complete)
       script:
         # DEVNULLRIGHTS : will request sudo rights to test permissions on /dev/null
         - DEVNULLRIGHTS=test make test
 
-    - name: gcc-6 + gcc-7 + libzstdmt compilation   # ~ 6mn
-      script:
-        - make gcc6install gcc7install
-        - CC=gcc-6 CFLAGS=-Werror make -j all
-        - make clean
-        - CC=gcc-7 CFLAGS=-Werror make -j all
-        - make clean
-        - LDFLAGS=-Wl,--no-undefined make -C lib libzstd-mt
-        - make -C tests zbufftest-dll
-        # LDFLAGS=-Wl,--no-undefined : will make the linker fail if dll is underlinked
-        # zbufftest-dll : test that a user program can link to multi-threaded libzstd without specifying -pthread
-
-    - name: gcc-8 + ASan + UBSan + Test Zstd   # ~6.5mn
-      script:
-        - make gcc8install
-        - CC=gcc-8 CFLAGS="-Werror" make -j all
-        - make clean
-        - CC=gcc-8 make -j uasan-test-zstd </dev/null   # test when stdin is not a tty
-
-    - name: gcc-6 + ASan + UBSan + Test Zstd, 32bit mode    # ~4mn
-      script:
-        - make gcc6install libc6install
-        - CC=gcc-6 CFLAGS="-Werror -m32" make -j all32
-        - make clean
-        - CC=gcc-6 make -j uasan-test-zstd32   # note : can complain about pointer overflow
-
-    - name: clang-3.8 + MSan + Test Zstd    # ~3.5mn
-      script:
-        - make clang38install
-        # External libraries must be turned off when using MSAN tests,
-        # because they are not msan-instrumented,
-        # so any data coming from these libraries is always considered "uninitialized"
-        - CC=clang-3.8 make clean msan-test-zstd HAVE_ZLIB=0 HAVE_LZ4=0 HAVE_LZMA=0
-
     - name: Minimal Decompressor Macros    # ~5mn
       script:
         - make clean && make -j all ZSTD_LIB_MINIFY=1 MOREFLAGS="-Werror"
@@ -85,51 +48,11 @@ matrix:
         - make clean && make -j all MOREFLAGS="-Werror -DZSTD_NO_INLINE -DZSTD_STRIP_ERROR_STRINGS"
         - make clean && make check MOREFLAGS="-Werror -DZSTD_NO_INLINE -DZSTD_STRIP_ERROR_STRINGS"
 
-    - name: cmake build and test check    # ~6mn
-      script:
-        - make cmakebuild
-
     - name: static analyzer scanbuild    # ~26mn
       dist: trusty  # note : it's important to pin down a version of static analyzer, since different versions report different false positives
       script:
         - make staticAnalyze
 
-    - name: gcc-8 + ASan + UBSan + Fuzz Test    # ~19mn
-      script:
-        - make gcc8install
-        - CC=gcc-8 make clean uasan-fuzztest
-
-    - name: gcc-6 + ASan + UBSan + Fuzz Test 32bit    # ~15.5mn
-      script:
-        - make gcc6install libc6install
-        - CC=gcc-6 CFLAGS="-O2 -m32" make uasan-fuzztest   # can complain about pointer overflow
-
-    - name: clang-3.8 + MSan + Fuzz Test     # ~14.5mn
-      script:
-        - make clang38install
-        - CC=clang-3.8 make clean msan-fuzztest
-
-    - name: ASan + UBSan + MSan + Regression Test    # ~ 4.5mn
-      script:
-        - make -j uasanregressiontest
-        - make clean
-        - make -j msanregressiontest
-
-    - name: C++, gnu90 and c99 compatibility   # ~3mn
-      script:
-        - make cxxtest
-        - make clean
-        - make gnu90build
-        - make clean
-        - make c99build
-        - make clean
-        - make travis-install    # just ensures `make install` works
-
-    - name: mingw cross-compilation
-      script :
-        - sudo update-alternatives --set x86_64-w64-mingw32-g++ /usr/bin/x86_64-w64-mingw32-g++-posix;
-        - CC=x86_64-w64-mingw32-gcc CXX=x86_64-w64-mingw32-g++ CFLAGS="-Werror -O1" make zstd
-
     - name: Valgrind + Fuzz Test Stack Mode    # ~ 7mn
       script:
         - make valgrindinstall
@@ -162,74 +85,51 @@ matrix:
         - make -C tests checkTag
         - tests/checkTag "$TRAVIS_BRANCH"
 
-    # tests for master branch and cron job only
+    # tests for release branch and cron job only
     - name: OS-X    # ~13mn
-      if: branch = master
+      if: branch = release
       os: osx
       script:
         - make test
         - make -C lib all
 
-    - name: zbuff test
-      if: branch = master
-      script:
-        - make -C tests test-zbuff
-
     - name: Versions Compatibility Test   # 11.5mn
-      if: branch = master
+      if: branch = release
       script:
         - make -C tests versionsTest
 
     - name: thread sanitizer   # ~29mn
-      if: branch = master
+      if: branch = release
       script:
         - make clang38install
         - CC=clang-3.8 make tsan-test-zstream
         - CC=clang-3.8 make tsan-fuzztest
 
     - name: PPC64LE + Fuzz test  # ~13mn
-      if: branch = master
+      if: branch = release
       arch: ppc64le
       script:
         - cat /proc/cpuinfo
         - make test
 
     - name: Qemu PPC64 + Fuzz test  # ~13mn, presumed Big-Endian (?)
       dist: trusty  # note : PPC64 cross-compilation for Qemu tests seems broken on Xenial
-      if: branch = master
+      if: branch = release
       script:
         - make ppcinstall
         - make ppc64fuzz
 
     # note : we already have aarch64 tests on hardware
     - name: Qemu aarch64 + Fuzz Test (on Xenial)    # ~14mn
-      if: branch = master
+      if: branch = release
       dist: xenial
       script:
         - make arminstall
         - make aarch64fuzz
 
-    - name: zlib wrapper test    # ~7.5mn
-      if: branch = master
-      script:
-        - make gpp6install valgrindinstall
-        - make -C zlibWrapper test
-        - make -C zlibWrapper valgrindTest
-
-    - name: LZ4, thread pool, and partial libs tests    # ~4mn
-      if: branch = master
-      script:
-        - make lz4install
-        - make -C tests test-lz4
-        - make check < /dev/null | tee    # mess with lz4 console detection
-        - make clean
-        - make -C tests test-pool
-        - make clean
-        - bash tests/libzstd_partial_builds.sh
-
     # meson dedicated test
     - name: Xenial (Meson + clang)    # ~15mn
-      if: branch = master
+      if: branch = release
       dist: xenial
       language: cpp
       compiler: clang

CHANGELOG

@@ -1,4 +1,7 @@
-v1.4.7
+v1.4.8  (Dec 18, 2020)
+hotfix: wrong alignment of an internal buffer
+
+v1.4.7  (Dec 16, 2020)
 perf: stronger --long mode at high compression levels, by @senhuang42
 perf: stronger --patch-from at high compression levels, thanks to --long improvements
 perf: faster dictionary compression at medium compression levels, by @felixhandte

CONTRIBUTING.md

@@ -5,7 +5,7 @@ possible.
 ## Our Development Process
 New versions are being developed in the "dev" branch,
 or in their own feature branch.
-When they are deemed ready for a release, they are merged into "master".
+When they are deemed ready for a release, they are merged into "release".
 
 As a consequences, all contributions must stage first through "dev"
 or their own feature branch.
@@ -383,7 +383,7 @@ CI tests run every time a pull request (PR) is created or updated. The exact tes
 that get run will depend on the destination branch you specify. Some tests take
 longer to run than others. Currently, our CI is set up to run a short
 series of tests when creating a PR to the dev branch and a longer series of tests
-when creating a PR to the master branch. You can look in the configuration files
+when creating a PR to the release branch. You can look in the configuration files
 of the respective CI platform for more information on what gets run when.
 
 Most people will just want to create a PR with the destination set to their local dev

README.md

@@ -193,7 +193,7 @@ Zstandard is dual-licensed under [BSD](LICENSE) and [GPLv2](COPYING).
 
 ## Contributing
 
-The "dev" branch is the one where all contributions are merged before reaching "master".
-If you plan to propose a patch, please commit into the "dev" branch, or its own feature branch.
-Direct commit to "master" are not permitted.
+The `dev` branch is the one where all contributions are merged before reaching `release`.
+If you plan to propose a patch, please commit into the `dev` branch, or its own feature branch.
+Direct commit to `release` are not permitted.
 For more information, please read [CONTRIBUTING](CONTRIBUTING.md).

TESTING.md

@@ -27,7 +27,7 @@ They consist of the following tests:
 
 Long Tests
 ----------
-Long tests run on all commits to `master` branch,
+Long tests run on all commits to `release` branch,
 and once a day on the current version of `dev` branch,
 on TravisCI.
 They consist of the following tests:

appveyor.yml

@@ -1,20 +1,20 @@
-# Following tests are run _only_ on master branch
-# To reproduce these tests, it's possible to push into a branch `appveyorTest`
-# or a branch `visual*`, they will intentionnally trigger `master` tests
+# Following tests are run _only_ on `release` branch
+# and on selected feature branch named `appveyorTest` or `visual*`
 
 -
   version: 1.0.{build}
   branches:
     only:
+    - release
     - master
-    - appveyorTest
+    - /appveyor*/
     - /visual*/
   environment:
     matrix:
     - COMPILER: "gcc"
       HOST:     "mingw"
       PLATFORM: "x64"
-      SCRIPT:   "make allzstd MOREFLAGS=-static && make -C tests fullbench-lib"
+      SCRIPT:   "make allzstd MOREFLAGS=-static"
       ARTIFACT: "true"
       BUILD:    "true"
     - COMPILER: "gcc"
@@ -92,9 +92,9 @@
           cd programs\ && 7z a -tzip -mx9 zstd-win-binary-%PLATFORM%.zip zstd.exe &&
           appveyor PushArtifact zstd-win-binary-%PLATFORM%.zip &&
           cp zstd.exe ..\bin\zstd.exe &&
-          git clone --depth 1 --branch master https://github.com/facebook/zstd &&
+          git clone --depth 1 --branch release https://github.com/facebook/zstd &&
           cd zstd &&
-          git archive --format=tar master -o zstd-src.tar &&
+          git archive --format=tar release -o zstd-src.tar &&
           ..\zstd -19 zstd-src.tar &&
           appveyor PushArtifact zstd-src.tar.zst &&
           certUtil -hashfile zstd-src.tar.zst SHA256 > zstd-src.tar.zst.sha256.sig &&