Merge pull request #43 from Cyan4973/dev

Dev

.travis.yml

@@ -1,6 +1,6 @@
 language: c
 compiler: gcc
-script: make test-travis
+script: make $ZSTD_TRAVIS_CI_ENV
 before_install:
   - sudo apt-get update  -qq
   - sudo apt-get install -qq gcc-arm-linux-gnueabi
@@ -11,11 +11,13 @@ before_install:
 
 env:
   - ZSTD_TRAVIS_CI_ENV=travis-install
-  - ZSTD_TRAVIS_CI_ENV=test-all  
   - ZSTD_TRAVIS_CI_ENV=clangtest  
   - ZSTD_TRAVIS_CI_ENV=gpptest  
   - ZSTD_TRAVIS_CI_ENV=armtest  
+  - ZSTD_TRAVIS_CI_ENV=test  
+  - ZSTD_TRAVIS_CI_ENV="-C programs test32"  
   - ZSTD_TRAVIS_CI_ENV=sanitize
+  - ZSTD_TRAVIS_CI_ENV="-C programs memtest"  
 
 matrix:
   fast_finish: true

Makefile

@@ -32,40 +32,25 @@
 # ################################################################
 
 # Version number
-export VERSION=0.1.0
-export RELEASE=r$(VERSION)
+export VERSION=0.1.1
 
-DESTDIR?=
-PREFIX ?= /usr
-
-LIBDIR ?= $(PREFIX)/lib
-INCLUDEDIR=$(PREFIX)/include
 PRGDIR  = programs
 ZSTDDIR = lib
 
-# Select test target for Travis CI's Build Matrix
-ifneq (,$(filter test-%,$(ZSTD_TRAVIS_CI_ENV)))
-TRAVIS_TARGET=prg-travis
-else
-TRAVIS_TARGET=$(ZSTD_TRAVIS_CI_ENV)
-endif
-
-
 .PHONY: clean
 
-default: zstdprograms
+default: zstdprogram
 
 all: 
-	@cd $(ZSTDDIR); $(MAKE) -e all
-	@cd $(PRGDIR); $(MAKE) -e all
+	$(MAKE) -C $(ZSTDDIR) $@
+	$(MAKE) -C $(PRGDIR) $@
 
-zstdprograms:
-	@cd $(PRGDIR); $(MAKE) -e
+zstdprogram:
+	$(MAKE) -C $(PRGDIR)
 
 clean:
-	@cd $(PRGDIR); $(MAKE) clean
-	@cd $(ZSTDDIR); $(MAKE) clean
-#	@cd examples; $(MAKE) clean
+	$(MAKE) -C $(ZSTDDIR) $@
+	$(MAKE) -C $(PRGDIR) $@
 	@echo Cleaning completed
 
 
@@ -74,23 +59,18 @@ clean:
 ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU))
 
 install:
-	@cd $(ZSTDDIR); $(MAKE) -e install
-	@cd $(PRGDIR); $(MAKE) -e install
+	$(MAKE) -C $(ZSTDDIR) $@
+	$(MAKE) -C $(PRGDIR) $@
 
 uninstall:
-	@cd $(ZSTDDIR); $(MAKE) uninstall
-	@cd $(PRGDIR); $(MAKE) uninstall
+	$(MAKE) -C $(ZSTDDIR) $@
+	$(MAKE) -C $(PRGDIR) $@
 
 travis-install:
 	sudo $(MAKE) install
 
 test:
-	@cd $(PRGDIR); $(MAKE) -e test
-
-test-travis: $(TRAVIS_TARGET)
-
-prg-travis:
-	@cd $(PRGDIR); $(MAKE) -e $(ZSTD_TRAVIS_CI_ENV)
+	$(MAKE) -C $(PRGDIR) $@
 
 clangtest: clean
 	clang -v
@@ -100,8 +80,8 @@ gpptest: clean
 	$(MAKE) all CC=g++ CFLAGS="-O3 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror"
 
 armtest: clean
-	cd $(ZSTDDIR); $(MAKE) -e all CC=arm-linux-gnueabi-gcc MOREFLAGS="-Werror"
-	cd $(PRGDIR); $(MAKE) -e CC=arm-linux-gnueabi-gcc MOREFLAGS="-Werror"
+	$(MAKE) -C $(ZSTDDIR) -e all CC=arm-linux-gnueabi-gcc MOREFLAGS="-Werror"
+	$(MAKE) -C $(PRGDIR) -e CC=arm-linux-gnueabi-gcc MOREFLAGS="-Werror"
 
 sanitize: clean
 	$(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=undefined"

lib/Makefile

@@ -32,14 +32,14 @@
 # ################################################################
 
 # Version numbers
-VERSION?= 0
+VERSION?= 0.1.1
 LIBVER_MAJOR=`sed -n '/define ZSTD_VERSION_MAJOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < zstd.h`
 LIBVER_MINOR=`sed -n '/define ZSTD_VERSION_MINOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < zstd.h`
 LIBVER_PATCH=`sed -n '/define ZSTD_VERSION_RELEASE/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < zstd.h`
 LIBVER  = $(LIBVER_MAJOR).$(LIBVER_MINOR).$(LIBVER_PATCH)
 
 DESTDIR?=
-PREFIX ?= /usr
+PREFIX ?= /usr/local
 CFLAGS ?= -O3
 CFLAGS += -std=c99 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Wstrict-prototypes
 LDFLAGS = -I.

lib/fse.c

@@ -135,7 +135,7 @@ typedef   signed long long  S64;
  * Method 1 : `__packed` statement. It depends on compiler extension (ie, not portable).
  *            This method is safe if your compiler supports it, and *generally* as fast or faster than `memcpy`.
  * Method 2 : direct access. This method is portable but violate C standard.
- *            It can generate buggy code on targets which generate assembly depending on alignment.
+ *            It can generate buggy code on targets generating assembly depending on alignment.
  *            But in some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6)
  * See http://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details.
  * Prefer these methods in priority order (0 > 1 > 2)
@@ -1398,6 +1398,7 @@ size_t FSE_compress2 (void* dst, size_t dstSize, const void* src, size_t srcSize
     errorCode = FSE_count (count, &maxSymbolValue, ip, srcSize);
     if (FSE_isError(errorCode)) return errorCode;
     if (errorCode == srcSize) return 1;
+    if (errorCode == 1) return 0;   /* each symbol only present once */
     if (errorCode < (srcSize >> 7)) return 0;   /* Heuristic : not compressible enough */
 
     tableLog = FSE_optimalTableLog(tableLog, srcSize, maxSymbolValue);
@@ -1529,12 +1530,14 @@ size_t FSE_initDStream(FSE_DStream_t* bitD, const void* srcBuffer, size_t srcSiz
  */
 static size_t FSE_lookBits(FSE_DStream_t* bitD, U32 nbBits)
 {
-    return ((bitD->bitContainer << (bitD->bitsConsumed & ((sizeof(bitD->bitContainer)*8)-1))) >> 1) >> (((sizeof(bitD->bitContainer)*8)-1)-nbBits);
+    const U32 bitMask = sizeof(bitD->bitContainer)*8 - 1;
+    return ((bitD->bitContainer << (bitD->bitsConsumed & bitMask)) >> 1) >> ((bitMask-nbBits) & bitMask);
 }
 
 static size_t FSE_lookBitsFast(FSE_DStream_t* bitD, U32 nbBits)   /* only if nbBits >= 1 !! */
 {
-    return (bitD->bitContainer << bitD->bitsConsumed) >> ((sizeof(bitD->bitContainer)*8)-nbBits);
+    const U32 bitMask = sizeof(bitD->bitContainer)*8 - 1;
+    return (bitD->bitContainer << (bitD->bitsConsumed & bitMask)) >> (((bitMask+1)-nbBits) & bitMask);
 }
 
 static void FSE_skipBits(FSE_DStream_t* bitD, U32 nbBits)
@@ -1904,11 +1907,17 @@ static U32 HUF_setMaxHeight(nodeElt* huffNode, U32 lastNonNull, U32 maxNbBits)
                 if (rankLast[nBitsToDecrease-1] == noOne)
                     rankLast[nBitsToDecrease-1] = rankLast[nBitsToDecrease];   // now there is one elt
                 huffNode[rankLast[nBitsToDecrease]].nbBits ++;
-                rankLast[nBitsToDecrease]--;
-				if (huffNode[rankLast[nBitsToDecrease]].nbBits != maxNbBits-nBitsToDecrease)
-                    rankLast[nBitsToDecrease] = noOne;   // rank list emptied
+                if (rankLast[nBitsToDecrease] == 0)
+                    rankLast[nBitsToDecrease] = noOne;
+                else
+                {
+                    rankLast[nBitsToDecrease]--;
+                    if (huffNode[rankLast[nBitsToDecrease]].nbBits != maxNbBits-nBitsToDecrease)
+                        rankLast[nBitsToDecrease] = noOne;   // rank list emptied
+                }
             }
-			while (totalCost < 0)   // Sometimes, cost correction overshoot
+
+			while (totalCost < 0)   /* Sometimes, cost correction overshoot */
 			{
 				if (rankLast[1] == noOne)   /* special case, no weight 1, let's find it back at n */
 				{

lib/zstd.c

@@ -1389,7 +1389,7 @@ static void ZSTD_decodeSequence(seq_t* seq, seqState_t* seqState)
         if (ZSTD_32bits()) FSE_reloadDStream(&(seqState->DStream));
         nbBits = offsetCode - 1;
         if (offsetCode==0) nbBits = 0;   /* cmove */
-        offset = ((size_t)1 << nbBits) + FSE_readBits(&(seqState->DStream), nbBits);
+        offset = ((size_t)1 << (nbBits & ((sizeof(offset)*8)-1))) + FSE_readBits(&(seqState->DStream), nbBits);
         if (ZSTD_32bits()) FSE_reloadDStream(&(seqState->DStream));
         if (offsetCode==0) offset = prevOffset;
     }

programs/Makefile

@@ -30,21 +30,18 @@
 # fullbench32: Same as fullbench, but forced to compile in 32-bits mode
 # ##########################################################################
 
-RELEASE?= v0.1.0
+VERSION?= v0.1.1
 
 DESTDIR?=
-PREFIX ?= /usr
+PREFIX ?= /usr/local
 CFLAGS ?= -O3
-CFLAGS += -std=c99 -Wall -Wextra -Wundef -Wshadow -Wcast-qual -Wcast-align -Wstrict-prototypes -DZSTD_VERSION=\"$(RELEASE)\"
+CFLAGS += -std=c99 -Wall -Wextra -Wundef -Wshadow -Wcast-qual -Wcast-align -Wstrict-prototypes -DZSTD_VERSION=\"$(VERSION)\"
 LDFLAGS = -I../lib
 FLAGS   = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) $(MOREFLAGS)
 
-BINDIR=$(PREFIX)/bin
-MANDIR=$(PREFIX)/share/man/man1
-ZSTDDIR=../lib
-
-TEST_FILES = COPYING
-TEST_TARGETS=test-native
+BINDIR  = $(PREFIX)/bin
+MANDIR  = $(PREFIX)/share/man/man1
+ZSTDDIR = ../lib
 
 
 # Define *.exe as extension for Windows systems
@@ -120,9 +117,12 @@ test: test-zstd test-fullbench test-fuzzer
 
 test32: test-zstd32 test-fullbench32 test-fuzzer32
 
-test-all: test test32 test-mem
+test-all: test test32 memtest
 
 test-zstd: zstd datagen
+	@echo "*** zstd cli write error test ***"
+	echo foo | ./zstd > /dev/full; if [ $$? -eq 0 ] ; then echo "write error not detected!"; false; fi
+	@echo "*** zstd round-trip tests *** "
 	./datagen          | ./zstd -v    | ./zstd -d > $(VOID)
 	./datagen -g256MB  | ./zstd -v    | ./zstd -d > $(VOID)
 	./datagen -g6GB -P99 | ./zstd -vq | ./zstd -d > $(VOID)
@@ -146,15 +146,15 @@ test-fuzzer: fuzzer
 test-fuzzer32: fuzzer32
 	./fuzzer32
 
-test-mem: zstd datagen fuzzer fullbench
+memtest: zstd datagen fuzzer fullbench
 	@echo "\n ---- valgrind tests : memory analyzer ----"
-	valgrind --leak-check=yes --error-exitcode=1 ./datagen -g50M > /dev/null
+	valgrind --leak-check=yes --error-exitcode=1 ./datagen -g50M > $(VOID)
 	./datagen -g16KB > tmp
-	valgrind --leak-check=yes --error-exitcode=1 ./zstd -vf tmp /dev/null
+	valgrind --leak-check=yes --error-exitcode=1 ./zstd -vf tmp $(VOID)
 	./datagen -g64MB > tmp
-	valgrind --leak-check=yes --error-exitcode=1 ./zstd -vf tmp /dev/null
+	valgrind --leak-check=yes --error-exitcode=1 ./zstd -vf tmp $(VOID)
 	@rm tmp
-	valgrind --leak-check=yes --error-exitcode=1 ./fuzzer -i128 -t1
+	valgrind --leak-check=yes --error-exitcode=1 ./fuzzer -i1000 -t1
 	valgrind --leak-check=yes --error-exitcode=1 ./fullbench -i1
 
 endif

programs/fileio.c

@@ -52,6 +52,7 @@
 #include <stdlib.h>   /* malloc, free */
 #include <string.h>   /* strcmp, strlen */
 #include <time.h>     /* clock */
+#include <errno.h>    /* errno */
 #include "fileio.h"
 #include "zstd_static.h"
 
@@ -254,7 +255,7 @@ unsigned long long FIO_compressFilename(const char* output_filename, const char*
     if (ZSTD_isError(cSize)) EXM_THROW(22, "Compression error : cannot create frame header");
 
     sizeCheck = fwrite(outBuff, 1, cSize, foutput);
-    if (sizeCheck!=cSize) EXM_THROW(23, "Write error : cannot write header");
+    if (sizeCheck!=cSize) EXM_THROW(23, "Write error : cannot write header into %s", output_filename);
     compressedfilesize += cSize;
 
     /* Main compression loop */
@@ -276,7 +277,7 @@ unsigned long long FIO_compressFilename(const char* output_filename, const char*
 
         /* Write cBlock */
         sizeCheck = fwrite(outBuff, 1, cSize, foutput);
-        if (sizeCheck!=cSize) EXM_THROW(25, "Write error : cannot write compressed block");
+        if (sizeCheck!=cSize) EXM_THROW(25, "Write error : cannot write compressed block into %s", output_filename);
         compressedfilesize += cSize;
         inSlot += inSize;
 
@@ -288,7 +289,7 @@ unsigned long long FIO_compressFilename(const char* output_filename, const char*
     if (ZSTD_isError(cSize)) EXM_THROW(26, "Compression error : cannot create frame end");
 
     sizeCheck = fwrite(outBuff, 1, cSize, foutput);
-    if (sizeCheck!=cSize) EXM_THROW(27, "Write error : cannot write frame end");
+    if (sizeCheck!=cSize) EXM_THROW(27, "Write error : cannot write frame end into %s", output_filename);
     compressedfilesize += cSize;
 
     /* Status */
@@ -299,9 +300,9 @@ unsigned long long FIO_compressFilename(const char* output_filename, const char*
     /* clean */
     free(inBuff);
     free(outBuff);
-    fclose(finput);
-    fclose(foutput);
     ZSTD_freeCCtx(ctx);
+    fclose(finput);
+    if (fclose(foutput)) EXM_THROW(28, "Write error : cannot properly close %s", output_filename);
 
     return compressedfilesize;
 }

programs/fullbench.c

@@ -616,7 +616,7 @@ int main(int argc, char** argv)
 
                     /* Modify Nb Iterations */
                 case 'i':
-                    if ((argument[1] >='1') && (argument[1] <='9'))
+                    if ((argument[1] >='0') && (argument[1] <='9'))
                     {
                         int iters = argument[1] - '0';
                         BMK_SetNbIterations(iters);

programs/unzstd.1

@@ -0,0 +1 @@
+zstd.1