Skip to content

Commit

Permalink
v1.4.1: Merge pull request #1691 from facebook/dev
Browse files Browse the repository at this point in the history
  • Loading branch information
felixhandte authored Jul 19, 2019
2 parents 83b51e9 + d636cd1 commit 52181f8
Show file tree
Hide file tree
Showing 78 changed files with 2,665 additions and 1,603 deletions.
30 changes: 4 additions & 26 deletions .circleci/config.yml
Original file line number Diff line number Diff line change
@@ -1,33 +1,14 @@
version: 2

references:
# Install the dependencies required for tests.
# Add the step "- *install-dependencies" to the beginning of your job to run
# this command.
install-dependencies: &install-dependencies
run:
name: Install dependencies
# TODO: We can split these dependencies up by job to reduce installation
# time.
command: |
sudo dpkg --add-architecture i386
sudo apt-get -y -qq update
sudo apt-get -y install \
gcc-multilib-powerpc-linux-gnu gcc-arm-linux-gnueabi \
libc6-dev-armel-cross gcc-aarch64-linux-gnu libc6-dev-arm64-cross \
libc6-dev-ppc64-powerpc-cross zstd gzip coreutils \
libcurl4-openssl-dev
jobs:
# the first half of the jobs are in this test
short-tests-0:
# TODO: Create a small custom docker image with all the dependencies we need
# preinstalled to reduce installation time.
docker:
- image: circleci/buildpack-deps:bionic
- image: fbopensource/zstd-circleci-primary:0.0.1
steps:
- checkout
- *install-dependencies
- run:
name: Test
command: |
Expand All @@ -41,10 +22,9 @@ jobs:
# the second half of the jobs are in this test
short-tests-1:
docker:
- image: circleci/buildpack-deps:bionic
- image: fbopensource/zstd-circleci-primary:0.0.1
steps:
- checkout
- *install-dependencies
- run:
name: Test
command: |
Expand All @@ -61,12 +41,11 @@ jobs:
# tagged release.
publish-github-release:
docker:
- image: circleci/buildpack-deps:bionic
- image: fbopensource/zstd-circleci-primary:0.0.1
environment:
CIRCLE_ARTIFACTS: /tmp/circleci-artifacts
steps:
- checkout
- *install-dependencies
- run:
name: Publish
command: |
Expand All @@ -86,12 +65,11 @@ jobs:
# This step should only be run in a cron job
regression-test:
docker:
- image: circleci/buildpack-deps:bionic
- image: fbopensource/zstd-circleci-primary:0.0.1
environment:
CIRCLE_ARTIFACTS: /tmp/circleci-artifacts
steps:
- checkout
- *install-dependencies
# Restore the cached resources.
- restore_cache:
# We try our best to bust the cache when the data changes by hashing
Expand Down
9 changes: 9 additions & 0 deletions .circleci/images/primary/Dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
FROM circleci/buildpack-deps:bionic

RUN sudo dpkg --add-architecture i386
RUN sudo apt-get -y -qq update
RUN sudo apt-get -y install \
gcc-multilib-powerpc-linux-gnu gcc-arm-linux-gnueabi \
libc6-dev-armel-cross gcc-aarch64-linux-gnu libc6-dev-arm64-cross \
libc6-dev-ppc64-powerpc-cross zstd gzip coreutils \
libcurl4-openssl-dev
5 changes: 3 additions & 2 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -14,15 +14,15 @@
*.dylib

# Executables
zstd
/zstd
zstdmt
*.exe
*.out
*.app

# Test artefacts
tmp*
dictionary*
dictionary.
NUL

# Build artefacts
Expand All @@ -41,3 +41,4 @@ _zstdbench/
.DS_Store
googletest/
*.d
*.vscode
42 changes: 29 additions & 13 deletions .travis.yml
Original file line number Diff line number Diff line change
Expand Up @@ -153,6 +153,12 @@ matrix:
- make clean
- make travis-install # just ensures `make install` works

- name: c99 compatibility
if: branch = master
script:
- make c99build
- make -C tests test-zstd

- name: PPC64
if: branch = master
script:
Expand Down Expand Up @@ -184,20 +190,30 @@ matrix:
compiler: clang
install:
- sudo apt-get install -qq liblz4-dev valgrind tree
- travis_retry curl -o ~/ninja.zip -L 'https://github.com/ninja-build/ninja/releases/download/v1.8.2/ninja-linux.zip'
&& unzip ~/ninja.zip -d ~/.local/bin
- travis_retry curl -o ~/get-pip.py -L 'https://bootstrap.pypa.io/get-pip.py'
&& python3 ~/get-pip.py --user
&& pip3 install --user meson
script:
- meson --buildtype=debug
-Db_lundef=false
-Dauto_features=enabled
-Dbuild_{programs,tests,contrib}=true
-Ddefault_library=both
build/meson builddir
- cd builddir
- |
travis_retry curl -o ~/ninja.zip -L 'https://github.com/ninja-build/ninja/releases/download/v1.9.0/ninja-linux.zip' &&
unzip ~/ninja.zip -d ~/.local/bin
- |
travis_retry curl -o ~/get-pip.py -L 'https://bootstrap.pypa.io/get-pip.py' &&
python3 ~/get-pip.py --user &&
pip3 install --user meson
script:
- |
meson setup \
--buildtype=debugoptimized \
-Db_lundef=false \
-Dauto_features=enabled \
-Dbin_programs=true \
-Dbin_tests=true \
-Dbin_contrib=true \
-Ddefault_library=both \
build/meson builddir
- pushd builddir
- ninja
- meson test --verbose --no-rebuild
- DESTDIR=./staging ninja install
- tree ./staging
after_failure:
- cat "$TRAVIS_BUILD_DIR"/builddir/meson-logs/testlog.txt
allow_failures:
- env: ALLOW_FAILURES=true
30 changes: 30 additions & 0 deletions CHANGELOG
Original file line number Diff line number Diff line change
@@ -1,3 +1,33 @@
v1.4.1
bug: Fix data corruption in niche use cases by @terrelln (#1659)
bug: Fuzz legacy modes, fix uncovered bugs by @terrelln (#1593, #1594, #1595)
bug: Fix out of bounds read by @terrelln (#1590)
perf: Improve decode speed by ~7% @mgrice (#1668)
perf: Slightly improved compression ratio of level 3 and 4 (ZSTD_dfast) by @cyan4973 (#1681)
perf: Slightly faster compression speed when re-using a context by @cyan4973 (#1658)
perf: Improve compression ratio for small windowLog by @cyan4973 (#1624)
perf: Faster compression speed in high compression mode for repetitive data by @terrelln (#1635)
api: Add parameter to generate smaller dictionaries by @tyler-tran (#1656)
cli: Recognize symlinks when built in C99 mode by @felixhandte (#1640)
cli: Expose cpu load indicator for each file on -vv mode by @ephiepark (#1631)
cli: Restrict read permissions on destination files by @chungy (#1644)
cli: zstdgrep: handle -f flag by @felixhandte (#1618)
cli: zstdcat: follow symlinks by @vejnar (#1604)
doc: Remove extra size limit on compressed blocks by @felixhandte (#1689)
doc: Fix typo by @yk-tanigawa (#1633)
doc: Improve documentation on streaming buffer sizes by @cyan4973 (#1629)
build: CMake: support building with LZ4 @leeyoung624 (#1626)
build: CMake: install zstdless and zstdgrep by @leeyoung624 (#1647)
build: CMake: respect existing uninstall target by @j301scott (#1619)
build: Make: skip multithread tests when built without support by @michaelforney (#1620)
build: Make: Fix examples/ test target by @sjnam (#1603)
build: Meson: rename options out of deprecated namespace by @lzutao (#1665)
build: Meson: fix build by @lzutao (#1602)
build: Visual Studio: don't export symbols in static lib by @scharan (#1650)
build: Visual Studio: fix linking by @absotively (#1639)
build: Fix MinGW-W64 build by @myzhang1029 (#1600)
misc: Expand decodecorpus coverage by @ephiepark (#1664)

v1.4.0
perf: Improve level 1 compression speed in most scenarios by 6% by @gbtucker and @terrelln
api: Move the advanced API, including all functions in the staging section, to the stable section
Expand Down
2 changes: 2 additions & 0 deletions build/.gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -29,3 +29,5 @@ compile_commands.json
CTestTestfile.cmake
build
lib
!cmake/lib
!meson/lib
2 changes: 0 additions & 2 deletions build/VS2010/libzstd-dll/libzstd-dll.vcxproj
Original file line number Diff line number Diff line change
Expand Up @@ -209,7 +209,6 @@
<EnablePREfast>false</EnablePREfast>
<RuntimeLibrary>MultiThreaded</RuntimeLibrary>
<DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
<AssemblerOutput>All</AssemblerOutput>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
Expand All @@ -234,7 +233,6 @@
<DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
<WholeProgramOptimization>true</WholeProgramOptimization>
<OmitFramePointers>true</OmitFramePointers>
<AssemblerOutput>All</AssemblerOutput>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
Expand Down
30 changes: 12 additions & 18 deletions build/VS2010/libzstd/libzstd.vcxproj
Original file line number Diff line number Diff line change
Expand Up @@ -90,31 +90,26 @@
<ProjectGuid>{8BFD8150-94D5-4BF9-8A50-7BD9929A0850}</ProjectGuid>
<Keyword>Win32Proj</Keyword>
<RootNamespace>libzstd</RootNamespace>
<TargetName>libzstd_static</TargetName>
<CharacterSet>MultiByte</CharacterSet>
<ConfigurationType>StaticLibrary</ConfigurationType>
<OutDir>$(SolutionDir)bin\$(Platform)_$(Configuration)\</OutDir>
<IntDir>$(SolutionDir)bin\obj\$(RootNamespace)_$(Platform)_$(Configuration)\</IntDir>
</PropertyGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration">
<ConfigurationType>StaticLibrary</ConfigurationType>
<UseDebugLibraries>true</UseDebugLibraries>
<CharacterSet>MultiByte</CharacterSet>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
<ConfigurationType>StaticLibrary</ConfigurationType>
<UseDebugLibraries>true</UseDebugLibraries>
<CharacterSet>MultiByte</CharacterSet>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
<ConfigurationType>StaticLibrary</ConfigurationType>
<UseDebugLibraries>false</UseDebugLibraries>
<WholeProgramOptimization>true</WholeProgramOptimization>
<CharacterSet>MultiByte</CharacterSet>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
<ConfigurationType>StaticLibrary</ConfigurationType>
<UseDebugLibraries>false</UseDebugLibraries>
<WholeProgramOptimization>true</WholeProgramOptimization>
<CharacterSet>MultiByte</CharacterSet>
</PropertyGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
<ImportGroup Label="ExtensionSettings">
Expand All @@ -134,35 +129,36 @@
<PropertyGroup Label="UserMacros" />
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
<LinkIncremental>true</LinkIncremental>
<TargetName>libzstd_static</TargetName>
<IncludePath>$(IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath);</IncludePath>
<RunCodeAnalysis>false</RunCodeAnalysis>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
<LinkIncremental>true</LinkIncremental>
<TargetName>libzstd_static</TargetName>
<IncludePath>$(IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath);</IncludePath>
<RunCodeAnalysis>false</RunCodeAnalysis>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
<LinkIncremental>false</LinkIncremental>
<TargetName>libzstd_static</TargetName>
<IncludePath>$(IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath);</IncludePath>
<RunCodeAnalysis>false</RunCodeAnalysis>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
<LinkIncremental>false</LinkIncremental>
<TargetName>libzstd_static</TargetName>
<IncludePath>$(IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath);</IncludePath>
<RunCodeAnalysis>false</RunCodeAnalysis>
</PropertyGroup>
<ItemDefinitionGroup>
<ClCompile>
<ProgramDataBaseFileName>$(OutDir)$(TargetName).pdb</ProgramDataBaseFileName>
</ClCompile>
</ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
<ClCompile>
<PrecompiledHeader>
</PrecompiledHeader>
<WarningLevel>Level4</WarningLevel>
<Optimization>Disabled</Optimization>
<PreprocessorDefinitions>ZSTD_DLL_EXPORT=1;ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<PreprocessorDefinitions>ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<MinimalRebuild>true</MinimalRebuild>
<BasicRuntimeChecks>EnableFastChecks</BasicRuntimeChecks>
<RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
Expand All @@ -182,7 +178,7 @@
</PrecompiledHeader>
<WarningLevel>Level4</WarningLevel>
<Optimization>Disabled</Optimization>
<PreprocessorDefinitions>ZSTD_DLL_EXPORT=1;ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<PreprocessorDefinitions>ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<TreatWarningAsError>true</TreatWarningAsError>
<BasicRuntimeChecks>EnableFastChecks</BasicRuntimeChecks>
<RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
Expand All @@ -202,11 +198,10 @@
<Optimization>MaxSpeed</Optimization>
<FunctionLevelLinking>true</FunctionLevelLinking>
<IntrinsicFunctions>true</IntrinsicFunctions>
<PreprocessorDefinitions>ZSTD_DLL_EXPORT=1;ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<PreprocessorDefinitions>ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<EnablePREfast>false</EnablePREfast>
<RuntimeLibrary>MultiThreaded</RuntimeLibrary>
<DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
<AssemblerOutput>All</AssemblerOutput>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
Expand All @@ -224,14 +219,13 @@
<Optimization>MaxSpeed</Optimization>
<FunctionLevelLinking>true</FunctionLevelLinking>
<IntrinsicFunctions>true</IntrinsicFunctions>
<PreprocessorDefinitions>ZSTD_DLL_EXPORT=1;ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<PreprocessorDefinitions>ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<TreatWarningAsError>false</TreatWarningAsError>
<EnablePREfast>false</EnablePREfast>
<RuntimeLibrary>MultiThreaded</RuntimeLibrary>
<DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
<WholeProgramOptimization>true</WholeProgramOptimization>
<OmitFramePointers>true</OmitFramePointers>
<AssemblerOutput>All</AssemblerOutput>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
Expand Down
49 changes: 49 additions & 0 deletions build/cmake/CMakeModules/FindLibLZ4.cmake
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
# Find LibLZ4
#
# Find LibLZ4 headers and library
#
# Result Variables
#
# LIBLZ4_FOUND - True if lz4 is found
# LIBLZ4_INCLUDE_DIRS - lz4 headers directories
# LIBLZ4_LIBRARIES - lz4 libraries
# LIBLZ4_VERSION_MAJOR - The major version of lz4
# LIBLZ4_VERSION_MINOR - The minor version of lz4
# LIBLZ4_VERSION_RELEASE - The release version of lz4
# LIBLZ4_VERSION_STRING - version number string (e.g. 1.8.3)
#
# Hints
#
# Set ``LZ4_ROOT_DIR`` to the directory of lz4.h and lz4 library

set(_LIBLZ4_ROOT_HINTS
ENV LZ4_ROOT_DIR)

find_path( LIBLZ4_INCLUDE_DIR lz4.h
HINTS ${_LIBLZ4_ROOT_HINTS})
find_library( LIBLZ4_LIBRARY NAMES lz4 liblz4 liblz4_static
HINTS ${_LIBLZ4_ROOT_HINTS})

if(LIBLZ4_INCLUDE_DIR)
file(STRINGS "${LIBLZ4_INCLUDE_DIR}/lz4.h" LIBLZ4_HEADER_CONTENT REGEX "#define LZ4_VERSION_[A-Z]+ +[0-9]+")

string(REGEX REPLACE ".*#define LZ4_VERSION_MAJOR +([0-9]+).*" "\\1" LIBLZ4_VERSION_MAJOR "${LIBLZ4_HEADER_CONTENT}")
string(REGEX REPLACE ".*#define LZ4_VERSION_MINOR +([0-9]+).*" "\\1" LIBLZ4_VERSION_MINOR "${LIBLZ4_HEADER_CONTENT}")
string(REGEX REPLACE ".*#define LZ4_VERSION_RELEASE +([0-9]+).*" "\\1" LIBLZ4_VERSION_RELEASE "${LIBLZ4_HEADER_CONTENT}")

set(LIBLZ4_VERSION_STRING "${LIBLZ4_VERSION_MAJOR}.${LIBLZ4_VERSION_MINOR}.${LIBLZ4_VERSION_RELEASE}")
unset(LIBLZ4_HEADER_CONTENT)
endif()

include(FindPackageHandleStandardArgs)
FIND_PACKAGE_HANDLE_STANDARD_ARGS(LibLZ4 REQUIRED_VARS LIBLZ4_INCLUDE_DIR
LIBLZ4_LIBRARY
VERSION_VAR LIBLZ4_VERSION_STRING
FAIL_MESSAGE "Could NOT find LZ4, try to set the paths to lz4.h and lz4 library in environment variable LZ4_ROOT_DIR")

if (LIBLZ4_FOUND)
set(LIBLZ4_LIBRARIES ${LIBLZ4_LIBRARY})
set(LIBLZ4_INCLUDE_DIRS ${LIBLZ4_INCLUDE_DIR})
endif ()

mark_as_advanced( LIBLZ4_INCLUDE_DIR LIBLZ4_LIBRARY )
Loading

0 comments on commit 52181f8

Please sign in to comment.