From b754a4c91ef581d7cfeeba0505c477ce754d0c6b Mon Sep 17 00:00:00 2001 From: fhanau Date: Sun, 17 Jul 2016 19:53:33 +0200 Subject: [PATCH 1/9] Use non-recursive BoundaryPM --- src/zopfli/katajainen.c | 131 ++++++++++++++++++++-------------------- 1 file changed, 66 insertions(+), 65 deletions(-) mode change 100755 => 100644 src/zopfli/katajainen.c diff --git a/src/zopfli/katajainen.c b/src/zopfli/katajainen.c old mode 100755 new mode 100644 index 14590175..391fd3b6 --- a/src/zopfli/katajainen.c +++ b/src/zopfli/katajainen.c @@ -39,13 +39,6 @@ struct Node { int count; /* Leaf symbol index, or number of leaves before this chain. */ }; -/* -Memory pool for nodes. -*/ -typedef struct NodePool { - Node* next; /* Pointer to a free node in the pool. */ -} NodePool; - /* Initializes a chain node with the given values and marks it as in use. */ @@ -55,64 +48,18 @@ static void InitNode(size_t weight, int count, Node* tail, Node* node) { node->tail = tail; } -/* -Performs a Boundary Package-Merge step. Puts a new chain in the given list. The -new chain is, depending on the weights, a leaf or a combination of two chains -from the previous list. -lists: The lists of chains. -maxbits: Number of lists. -leaves: The leaves, one per symbol. -numsymbols: Number of leaves. -pool: the node memory pool. -index: The index of the list in which a new chain or leaf is required. -*/ -static void BoundaryPM(Node* (*lists)[2], Node* leaves, int numsymbols, - NodePool* pool, int index) { - Node* newchain; - Node* oldchain; - int lastcount = lists[index][1]->count; /* Count of last chain of list. */ - - if (index == 0 && lastcount >= numsymbols) return; - - newchain = pool->next++; - oldchain = lists[index][1]; - - /* These are set up before the recursive calls below, so that there is a list - pointing to the new node, to let the garbage collection know it's in use. */ - lists[index][0] = oldchain; - lists[index][1] = newchain; - - if (index == 0) { - /* New leaf node in list 0. */ - InitNode(leaves[lastcount].weight, lastcount + 1, 0, newchain); - } else { - size_t sum = lists[index - 1][0]->weight + lists[index - 1][1]->weight; - if (lastcount < numsymbols && sum > leaves[lastcount].weight) { - /* New leaf inserted in list, so count is incremented. */ - InitNode(leaves[lastcount].weight, lastcount + 1, oldchain->tail, - newchain); - } else { - InitNode(sum, lastcount, lists[index - 1][1], newchain); - /* Two lookahead chains of previous list used up, create new ones. */ - BoundaryPM(lists, leaves, numsymbols, pool, index - 1); - BoundaryPM(lists, leaves, numsymbols, pool, index - 1); - } - } -} - static void BoundaryPMFinal(Node* (*lists)[2], - Node* leaves, int numsymbols, NodePool* pool, int index) { + Node* leaves, int numsymbols, Node* pool, int index) { int lastcount = lists[index][1]->count; /* Count of last chain of list. */ size_t sum = lists[index - 1][0]->weight + lists[index - 1][1]->weight; if (lastcount < numsymbols && sum > leaves[lastcount].weight) { - Node* newchain = pool->next; Node* oldchain = lists[index][1]->tail; - lists[index][1] = newchain; - newchain->count = lastcount + 1; - newchain->tail = oldchain; + lists[index][1] = pool; + pool->count = lastcount + 1; + pool->tail = oldchain; } else { lists[index][1]->tail = lists[index - 1][1]; } @@ -123,10 +70,10 @@ Initializes each list with as lookahead chains the two leaves with lowest weights. */ static void InitLists( - NodePool* pool, const Node* leaves, int maxbits, Node* (*lists)[2]) { + Node* pool, const Node* leaves, int maxbits, Node* (*lists)[2]) { int i; - Node* node0 = pool->next++; - Node* node1 = pool->next++; + Node* node0 = pool; + Node* node1 = pool + 1; InitNode(leaves[0].weight, 1, 0, node0); InitNode(leaves[1].weight, 2, 0, node1); for (i = 0; i < maxbits; i++) { @@ -171,11 +118,12 @@ static int LeafComparator(const void* a, const void* b) { int ZopfliLengthLimitedCodeLengths( const size_t* frequencies, int n, int maxbits, unsigned* bitlengths) { - NodePool pool; + Node* pool; int i; int numsymbols = 0; /* Amount of symbols with frequency > 0. */ int numBoundaryPMRuns; Node* nodes; + unsigned char stack[16]; /* Array of lists of chains. Each list requires only two lookahead chains at a time, so each list is a array of two Node*'s. */ @@ -240,18 +188,71 @@ int ZopfliLengthLimitedCodeLengths( /* Initialize node memory pool. */ nodes = (Node*)malloc(maxbits * 2 * numsymbols * sizeof(Node)); - pool.next = nodes; + pool = nodes; lists = (Node* (*)[2])malloc(maxbits * sizeof(*lists)); - InitLists(&pool, leaves, maxbits, lists); + InitLists(pool, leaves, maxbits, lists); + pool += 2; /* In the last list, 2 * numsymbols - 2 active chains need to be created. Two are already created in the initialization. Each BoundaryPM run creates one. */ numBoundaryPMRuns = 2 * numsymbols - 4; for (i = 0; i < numBoundaryPMRuns - 1; i++) { - BoundaryPM(lists, leaves, numsymbols, &pool, maxbits - 1); + /* + Performs a Boundary Package-Merge step. Puts a new chain in the given list. The + new chain is, depending on the weights, a leaf or a combination of two chains + from the previous list. + */ + unsigned stackpos; + stack[0] = maxbits - 1; + + for (stackpos = 0; ;) { + unsigned char index = stack[stackpos]; + + int lastcount = lists[index][1]->count; /* Count of last chain of list. */ + + Node* newchain = pool++; + Node* oldchain = lists[index][1]; + size_t sum; + + /* These are set up before the recursive calls below, so that there is a list + pointing to the new node, to let the garbage collection know it's in use. */ + lists[index][0] = oldchain; + lists[index][1] = newchain; + + sum = lists[index - 1][0]->weight + lists[index - 1][1]->weight; + + if (lastcount < numsymbols && sum > leaves[lastcount].weight) { + /* New leaf inserted in list, so count is incremented. */ + InitNode(leaves[lastcount].weight, lastcount + 1, oldchain->tail, newchain); + } else { + InitNode(sum, lastcount, lists[index - 1][1], newchain); + /* Two lookahead chains of previous list used up, create new ones. */ + if (index == 1) { + if (lists[0][1]->count < numsymbols) { + lastcount = lists[0][1]->count; + lists[0][0] = lists[0][1]; + lists[0][1] = pool++; + InitNode(leaves[lastcount].weight, lastcount + 1, 0, lists[0][1]); + lastcount++; + if(lastcount < numsymbols){ + lists[0][0] = lists[0][1]; + lists[0][1] = pool++; + InitNode(leaves[lastcount].weight, lastcount + 1, 0, lists[0][1]); + } + } + } + else { + stack[stackpos++] = index - 1; + stack[stackpos++] = index - 1; + } + } + if (!stackpos--) { + break; + } + } } - BoundaryPMFinal(lists, leaves, numsymbols, &pool, maxbits - 1); + BoundaryPMFinal(lists, leaves, numsymbols, pool, maxbits - 1); ExtractBitLengths(lists[maxbits - 1][1], leaves, bitlengths); From a20ade78ae55f8f96d8e7ba8debecb762d9e8e83 Mon Sep 17 00:00:00 2001 From: fhanau Date: Mon, 18 Jul 2016 13:59:36 +0200 Subject: [PATCH 2/9] Better lazy/greedy matching --- src/zopfli/lz77.c | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/zopfli/lz77.c b/src/zopfli/lz77.c index 9df899dd..a5e529f1 100644 --- a/src/zopfli/lz77.c +++ b/src/zopfli/lz77.c @@ -263,11 +263,8 @@ Indirectly, this affects: to the optimal output */ static int GetLengthScore(int length, int distance) { - /* - At 1024, the distance uses 9+ extra bits and this seems to be the sweet spot - on tested files. - */ - return distance > 1024 ? length - 1 : length; + return (length == 3 && distance > 1024) || (length == 4 && distance > 2048) || + (length == 5 && distance > 4096) ? length - 1 : length; } void ZopfliVerifyLenDist(const unsigned char* data, size_t datasize, size_t pos, From d6804289491449342e76c5a9e079b1a0950c90d0 Mon Sep 17 00:00:00 2001 From: fhanau Date: Sat, 27 Aug 2016 20:25:23 +0200 Subject: [PATCH 3/9] Faster blocksplitting and greedy only done once --- src/zopfli/blocksplitter.c | 38 ++++++++++++++++++++++++++++++++++---- src/zopfli/blocksplitter.h | 3 ++- src/zopfli/deflate.c | 11 +++++++++-- src/zopfli/squeeze.c | 29 ++++++++++------------------- src/zopfli/squeeze.h | 20 +++++++++++++++++++- 5 files changed, 74 insertions(+), 27 deletions(-) diff --git a/src/zopfli/blocksplitter.c b/src/zopfli/blocksplitter.c index 161783d8..144cd6ad 100644 --- a/src/zopfli/blocksplitter.c +++ b/src/zopfli/blocksplitter.c @@ -24,7 +24,6 @@ Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala) #include #include "deflate.h" -#include "squeeze.h" #include "tree.h" #include "util.h" @@ -62,7 +61,7 @@ static size_t FindMinimum(FindMinimumFun f, void* context, size_t p[NUM]; double vp[NUM]; size_t besti; - double best; + double best = ZOPFLI_LARGE_FLOAT; double lastbest = ZOPFLI_LARGE_FLOAT; size_t pos = start; @@ -71,6 +70,10 @@ static size_t FindMinimum(FindMinimumFun f, void* context, for (i = 0; i < NUM; i++) { p[i] = start + (i + 1) * ((end - start) / (NUM + 1)); + if(pos == p[i]){ + vp[i] = best; + continue; + } vp[i] = f(p[i], context); } besti = 0; @@ -274,7 +277,7 @@ void ZopfliBlockSplitLZ77(const ZopfliOptions* options, void ZopfliBlockSplit(const ZopfliOptions* options, const unsigned char* in, size_t instart, size_t inend, - size_t maxblocks, size_t** splitpoints, size_t* npoints) { + size_t maxblocks, size_t** splitpoints, size_t* npoints, SymbolStats** stats) { size_t pos = 0; size_t i; ZopfliBlockState s; @@ -298,19 +301,46 @@ void ZopfliBlockSplit(const ZopfliOptions* options, ZopfliBlockSplitLZ77(options, &store, maxblocks, &lz77splitpoints, &nlz77points); + (*stats) = (SymbolStats*)realloc(*stats, (nlz77points + 1) * sizeof(SymbolStats)); /* Convert LZ77 positions to positions in the uncompressed input. */ pos = instart; if (nlz77points > 0) { for (i = 0; i < store.size; i++) { size_t length = store.dists[i] == 0 ? 1 : store.litlens[i]; - if (lz77splitpoints[*npoints] == i) { + if (lz77splitpoints[(*npoints)] == i) { + size_t temp = store.size; + size_t shift = (*npoints) ? lz77splitpoints[*npoints - 1] : 0; + store.size = i - shift; + store.dists += shift; + store.litlens += shift; + + InitStats(&((*stats)[*npoints])); + GetStatistics(&store, &((*stats)[*npoints])); + store.size = temp; + store.dists -= shift; + store.litlens -= shift; ZOPFLI_APPEND_DATA(pos, splitpoints, npoints); if (*npoints == nlz77points) break; } pos += length; } + size_t shift = lz77splitpoints[*npoints - 1]; + store.size -= shift; + store.dists += shift; + store.litlens += shift; + + InitStats(&((*stats)[*npoints])); + GetStatistics(&store, &((*stats)[*npoints])); + store.size += shift; + store.dists -= shift; + store.litlens -= shift; } + else{ + InitStats(*stats); + GetStatistics(&store, *stats); + } + assert(*npoints == nlz77points); free(lz77splitpoints); diff --git a/src/zopfli/blocksplitter.h b/src/zopfli/blocksplitter.h index d1d622f1..e4b769f8 100644 --- a/src/zopfli/blocksplitter.h +++ b/src/zopfli/blocksplitter.h @@ -31,6 +31,7 @@ ones that enhance it. #include #include "lz77.h" +#include "squeeze.h" #include "zopfli.h" @@ -59,7 +60,7 @@ npoints: pointer to amount of splitpoints, for the dynamic array. The amount of */ void ZopfliBlockSplit(const ZopfliOptions* options, const unsigned char* in, size_t instart, size_t inend, - size_t maxblocks, size_t** splitpoints, size_t* npoints); + size_t maxblocks, size_t** splitpoints, size_t* npoints, SymbolStats** stats); /* Divides the input into equal blocks, does not even take LZ77 lengths into diff --git a/src/zopfli/deflate.c b/src/zopfli/deflate.c index abe73602..132f4b63 100644 --- a/src/zopfli/deflate.c +++ b/src/zopfli/deflate.c @@ -813,6 +813,8 @@ void ZopfliDeflatePart(const ZopfliOptions* options, int btype, int final, unsigned char* bp, unsigned char** out, size_t* outsize) { size_t i; + SymbolStats* stats; + /* byte coordinates rather than lz77 index */ size_t* splitpoints_uncompressed = 0; size_t npoints = 0; @@ -820,6 +822,7 @@ void ZopfliDeflatePart(const ZopfliOptions* options, int btype, int final, double totalcost = 0; ZopfliLZ77Store lz77; + stats = 0; /* If btype=2 is specified, it tries all block types. If a lesser btype is given, then however it forces that one. Neither of the lesser types needs block splitting as they have no dynamic huffman trees. */ @@ -845,7 +848,7 @@ void ZopfliDeflatePart(const ZopfliOptions* options, int btype, int final, if (options->blocksplitting) { ZopfliBlockSplit(options, in, instart, inend, options->blocksplittingmax, - &splitpoints_uncompressed, &npoints); + &splitpoints_uncompressed, &npoints, &stats); splitpoints = (size_t*)malloc(sizeof(*splitpoints) * npoints); } @@ -858,7 +861,7 @@ void ZopfliDeflatePart(const ZopfliOptions* options, int btype, int final, ZopfliLZ77Store store; ZopfliInitLZ77Store(in, &store); ZopfliInitBlockState(options, start, end, 1, &s); - ZopfliLZ77Optimal(&s, in, start, end, options->numiterations, &store); + ZopfliLZ77Optimal(&s, in, start, end, options->numiterations, &store, stats ? &stats[i] : 0); totalcost += ZopfliCalculateBlockSizeAutoType(&store, 0, store.size); ZopfliAppendLZ77Store(&store, &lz77); @@ -868,6 +871,10 @@ void ZopfliDeflatePart(const ZopfliOptions* options, int btype, int final, ZopfliCleanLZ77Store(&store); } + if(stats){ + free(stats); + } + /* Second block splitting attempt */ if (options->blocksplitting && npoints > 1) { size_t* splitpoints2 = 0; diff --git a/src/zopfli/squeeze.c b/src/zopfli/squeeze.c index a695c18c..c677adac 100644 --- a/src/zopfli/squeeze.c +++ b/src/zopfli/squeeze.c @@ -29,20 +29,7 @@ Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala) #include "tree.h" #include "util.h" -typedef struct SymbolStats { - /* The literal and length symbols. */ - size_t litlens[ZOPFLI_NUM_LL]; - /* The 32 unique dist symbols, not the 32768 possible dists. */ - size_t dists[ZOPFLI_NUM_D]; - - /* Length of each lit/len symbol in bits. */ - double ll_symbols[ZOPFLI_NUM_LL]; - /* Length of each dist symbol in bits. */ - double d_symbols[ZOPFLI_NUM_D]; -} SymbolStats; - -/* Sets everything to 0. */ -static void InitStats(SymbolStats* stats) { +void InitStats(SymbolStats* stats) { memset(stats->litlens, 0, ZOPFLI_NUM_LL * sizeof(stats->litlens[0])); memset(stats->dists, 0, ZOPFLI_NUM_D * sizeof(stats->dists[0])); @@ -394,8 +381,7 @@ static void CalculateStatistics(SymbolStats* stats) { ZopfliCalculateEntropy(stats->dists, ZOPFLI_NUM_D, stats->d_symbols); } -/* Appends the symbol statistics from the store. */ -static void GetStatistics(const ZopfliLZ77Store* store, SymbolStats* stats) { +void GetStatistics(const ZopfliLZ77Store* store, SymbolStats* stats) { size_t i; for (i = 0; i < store->size; i++) { if (store->dists[i] == 0) { @@ -446,7 +432,7 @@ static double LZ77OptimalRun(ZopfliBlockState* s, void ZopfliLZ77Optimal(ZopfliBlockState *s, const unsigned char* in, size_t instart, size_t inend, int numiterations, - ZopfliLZ77Store* store) { + ZopfliLZ77Store* store, SymbolStats* instats) { /* Dist to get to here with smallest cost. */ size_t blocksize = inend - instart; unsigned short* length_array = @@ -478,8 +464,13 @@ void ZopfliLZ77Optimal(ZopfliBlockState *s, the statistics of the previous run. */ /* Initial run. */ - ZopfliLZ77Greedy(s, in, instart, inend, ¤tstore, h); - GetStatistics(¤tstore, &stats); + if(!instats){ + ZopfliLZ77Greedy(s, in, instart, inend, ¤tstore, h); + GetStatistics(¤tstore, &stats); + } + else{ + CopyStats(instats, &stats); + } /* Repeat statistics with each time the cost model from the previous stat run. */ diff --git a/src/zopfli/squeeze.h b/src/zopfli/squeeze.h index 48bb7753..68db4378 100644 --- a/src/zopfli/squeeze.h +++ b/src/zopfli/squeeze.h @@ -33,6 +33,24 @@ solution. #include "lz77.h" +typedef struct SymbolStats { + /* The literal and length symbols. */ + size_t litlens[ZOPFLI_NUM_LL]; + /* The 32 unique dist symbols, not the 32768 possible dists. */ + size_t dists[ZOPFLI_NUM_D]; + + /* Length of each lit/len symbol in bits. */ + double ll_symbols[ZOPFLI_NUM_LL]; + /* Length of each dist symbol in bits. */ + double d_symbols[ZOPFLI_NUM_D]; +} SymbolStats; + +/* Sets everything to 0. */ +void InitStats(SymbolStats* stats); + +/* Appends the symbol statistics from the store. */ +void GetStatistics(const ZopfliLZ77Store* store, SymbolStats* stats); + /* Calculates lit/len and dist pairs for given data. If instart is larger than 0, it uses values before instart as starting @@ -41,7 +59,7 @@ dictionary. void ZopfliLZ77Optimal(ZopfliBlockState *s, const unsigned char* in, size_t instart, size_t inend, int numiterations, - ZopfliLZ77Store* store); + ZopfliLZ77Store* store, SymbolStats* instats); /* Does the same as ZopfliLZ77Optimal, but optimized for the fixed tree of the From 484a1f6e174ba50ed7ba76ff1419887748eab03c Mon Sep 17 00:00:00 2001 From: fhanau Date: Tue, 18 Oct 2016 18:05:13 +0200 Subject: [PATCH 4/9] Faster cost model --- src/zopfli/squeeze.c | 92 +++++++++++++++++++++----------------------- 1 file changed, 43 insertions(+), 49 deletions(-) diff --git a/src/zopfli/squeeze.c b/src/zopfli/squeeze.c index c677adac..ede42fc2 100644 --- a/src/zopfli/squeeze.c +++ b/src/zopfli/squeeze.c @@ -99,36 +99,8 @@ static void ClearStatFreqs(SymbolStats* stats) { for (i = 0; i < ZOPFLI_NUM_D; i++) stats->dists[i] = 0; } -/* -Function that calculates a cost based on a model for the given LZ77 symbol. -litlen: means literal symbol if dist is 0, length otherwise. -*/ -typedef double CostModelFun(unsigned litlen, unsigned dist, void* context); - -/* -Cost model which should exactly match fixed tree. -type: CostModelFun -*/ -static double GetCostFixed(unsigned litlen, unsigned dist, void* unused) { - (void)unused; - if (dist == 0) { - if (litlen <= 143) return 8; - else return 9; - } else { - int dbits = ZopfliGetDistExtraBits(dist); - int lbits = ZopfliGetLengthExtraBits(litlen); - int lsym = ZopfliGetLengthSymbol(litlen); - int cost = 0; - if (lsym <= 279) cost += 7; - else cost += 8; - cost += 5; /* Every dist symbol has length 5. */ - return cost + dbits + lbits; - } -} - /* Cost model based on symbol statistics. -type: CostModelFun */ static double GetCostStat(unsigned litlen, unsigned dist, void* context) { SymbolStats* stats = (SymbolStats*)context; @@ -147,7 +119,7 @@ static double GetCostStat(unsigned litlen, unsigned dist, void* context) { Finds the minimum possible cost this cost model can return for valid length and distance symbols. */ -static double GetCostModelMinCost(CostModelFun* costmodel, void* costcontext) { +static double GetCostModelMinCost(void* costcontext) { double mincost; int bestlength = 0; /* length that has lowest cost in the cost model */ int bestdist = 0; /* distance that has lowest cost in the cost model */ @@ -165,7 +137,7 @@ static double GetCostModelMinCost(CostModelFun* costmodel, void* costcontext) { mincost = ZOPFLI_LARGE_FLOAT; for (i = 3; i < 259; i++) { - double c = costmodel(i, 1, costcontext); + double c = GetCostStat(i, 1, costcontext); if (c < mincost) { bestlength = i; mincost = c; @@ -174,14 +146,14 @@ static double GetCostModelMinCost(CostModelFun* costmodel, void* costcontext) { mincost = ZOPFLI_LARGE_FLOAT; for (i = 0; i < 30; i++) { - double c = costmodel(3, dsymbols[i], costcontext); + double c = GetCostStat(3, dsymbols[i], costcontext); if (c < mincost) { bestdist = dsymbols[i]; mincost = c; } } - return costmodel(bestlength, bestdist, costcontext); + return GetCostStat(bestlength, bestdist, costcontext); } static size_t zopfli_min(size_t a, size_t b) { @@ -195,8 +167,7 @@ s: the ZopfliBlockState in: the input data array instart: where to start inend: where to stop (not inclusive) -costmodel: function to calculate the cost of some lit/len/dist pair. -costcontext: abstract context for the costmodel function +costcontext: abstract context for the costmodel length_array: output array of size (inend - instart) which will receive the best length to reach this byte from a previous byte. returns the cost that was, according to the costmodel, needed to get to the end. @@ -204,7 +175,7 @@ returns the cost that was, according to the costmodel, needed to get to the end. static double GetBestLengths(ZopfliBlockState *s, const unsigned char* in, size_t instart, size_t inend, - CostModelFun* costmodel, void* costcontext, + SymbolStats* costcontext, unsigned short* length_array, ZopfliHash* h, float* costs) { /* Best cost to get here so far. */ @@ -216,8 +187,33 @@ static double GetBestLengths(ZopfliBlockState *s, size_t windowstart = instart > ZOPFLI_WINDOW_SIZE ? instart - ZOPFLI_WINDOW_SIZE : 0; double result; - double mincost = GetCostModelMinCost(costmodel, costcontext); + double mincost = costcontext ? GetCostModelMinCost(costcontext) : 12; double mincostaddcostj; + double* literals; /*Cost of a literal*/ + double litlentable[259]; /*Cost of the length bits of a match*/ + double disttable[30]; /*Cost of the distance bits of a match*/ + if(costcontext){ + literals = costcontext->ll_symbols; + for (i = 3; i < 259; i++){ + litlentable[i] = costcontext->ll_symbols[ZopfliGetLengthSymbol(i)] + ZopfliGetLengthExtraBits(i); + } + for (i = 0; i < 30; i++){ + disttable[i] = costcontext->d_symbols[i] + (i < 4 ? 0 : (i - 2) / 2); + } + } + else{ + double litstack[256]; + literals = litstack; + for (i = 0; i < 256; i++){ + literals[i] = 8 + (i > 143); + } + for (i = 3; i < 259; i++){ + litlentable[i] = 12 + (i > 114) + ZopfliGetLengthExtraBits(i); + } + for (i = 0; i < 30; i++){ + disttable[i] = i < 4 ? 0 : (i - 2) / 2; + } + } if (instart == inend) return 0; @@ -243,7 +239,7 @@ static double GetBestLengths(ZopfliBlockState *s, && i + ZOPFLI_MAX_MATCH * 2 + 1 < inend && h->same[(i - ZOPFLI_MAX_MATCH) & ZOPFLI_WINDOW_MASK] > ZOPFLI_MAX_MATCH) { - double symbolcost = costmodel(ZOPFLI_MAX_MATCH, 1, costcontext); + double symbolcost = disttable[0] + litlentable[ZOPFLI_MAX_MATCH]; /* Set the length to reach each one to ZOPFLI_MAX_MATCH, and the cost to the cost corresponding to that length. Doing this, we skip ZOPFLI_MAX_MATCH values to avoid calling ZopfliFindLongestMatch. */ @@ -262,7 +258,7 @@ static double GetBestLengths(ZopfliBlockState *s, /* Literal. */ if (i + 1 <= inend) { - double newCost = costmodel(in[i], 0, costcontext) + costs[j]; + double newCost = literals[in[i]] + costs[j]; assert(newCost >= 0); if (newCost < costs[j + 1]) { costs[j + 1] = newCost; @@ -275,11 +271,10 @@ static double GetBestLengths(ZopfliBlockState *s, for (k = 3; k <= kend; k++) { double newCost; - /* Calling the cost model is expensive, avoid this if we are already at - the minimum possible cost that it can return. */ - if (costs[j + k] <= mincostaddcostj) continue; - - newCost = costmodel(k, sublen[k], costcontext) + costs[j]; + /* Calculating the cost is expensive, avoid this if we are already at + the minimum possible cost that it can be. */ + if (costs[j + k] <= mincostaddcostj) continue; + newCost = disttable[ZopfliGetDistSymbol(sublen[k])] + litlentable[k] + costs[j]; assert(newCost >= 0); if (newCost < costs[j + k]) { assert(k <= ZOPFLI_MAX_MATCH); @@ -406,8 +401,7 @@ inend: where to stop (not inclusive) path: pointer to dynamically allocated memory to store the path pathsize: pointer to the size of the dynamic path array length_array: array of size (inend - instart) used to store lengths -costmodel: function to use as the cost model for this squeeze run -costcontext: abstract context for the costmodel function +costcontext: abstract context for the costmodel store: place to output the LZ77 data returns the cost that was, according to the costmodel, needed to get to the end. This is not the actual cost. @@ -415,10 +409,10 @@ returns the cost that was, according to the costmodel, needed to get to the end. static double LZ77OptimalRun(ZopfliBlockState* s, const unsigned char* in, size_t instart, size_t inend, unsigned short** path, size_t* pathsize, - unsigned short* length_array, CostModelFun* costmodel, + unsigned short* length_array, void* costcontext, ZopfliLZ77Store* store, ZopfliHash* h, float* costs) { - double cost = GetBestLengths(s, in, instart, inend, costmodel, + double cost = GetBestLengths(s, in, instart, inend, costcontext, length_array, h, costs); free(*path); *path = 0; @@ -478,7 +472,7 @@ void ZopfliLZ77Optimal(ZopfliBlockState *s, ZopfliCleanLZ77Store(¤tstore); ZopfliInitLZ77Store(in, ¤tstore); LZ77OptimalRun(s, in, instart, inend, &path, &pathsize, - length_array, GetCostStat, (void*)&stats, + length_array, (void*)&stats, ¤tstore, h, costs); cost = ZopfliCalculateBlockSize(¤tstore, 0, currentstore.size, 2); if (s->options->verbose_more || (s->options->verbose && cost < bestcost)) { @@ -542,7 +536,7 @@ void ZopfliLZ77OptimalFixed(ZopfliBlockState *s, /* Shortest path for fixed tree This one should give the shortest possible result for fixed tree, no repeated runs are needed since the tree is known. */ LZ77OptimalRun(s, in, instart, inend, &path, &pathsize, - length_array, GetCostFixed, 0, store, h, costs); + length_array, 0, store, h, costs); free(length_array); free(path); From 33963e71ebf2d1e6e263b9a5c896099d265273a3 Mon Sep 17 00:00:00 2001 From: fhanau Date: Tue, 18 Oct 2016 18:26:19 +0200 Subject: [PATCH 5/9] Speed up quicksort when katajainen.c is compiled as C++ --- src/zopfli/katajainen.c | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/src/zopfli/katajainen.c b/src/zopfli/katajainen.c index 391fd3b6..0eeb6086 100644 --- a/src/zopfli/katajainen.c +++ b/src/zopfli/katajainen.c @@ -23,11 +23,20 @@ Bounded package merge algorithm, based on the paper Jyrki Katajainen, Alistair Moffat, Andrew Turpin". */ +#ifdef __cplusplus +#include +extern "C" { +#endif + #include "katajainen.h" #include #include #include +#ifdef __cplusplus +} +#endif + typedef struct Node Node; /* @@ -109,13 +118,18 @@ static void ExtractBitLengths(Node* chain, Node* leaves, unsigned* bitlengths) { } } +#ifndef __cplusplus /* Comparator for sorting the leaves. Has the function signature for qsort. */ static int LeafComparator(const void* a, const void* b) { return ((const Node*)a)->weight - ((const Node*)b)->weight; } +#endif +#ifdef __cplusplus +extern "C" +#endif int ZopfliLengthLimitedCodeLengths( const size_t* frequencies, int n, int maxbits, unsigned* bitlengths) { Node* pool; @@ -177,7 +191,17 @@ int ZopfliLengthLimitedCodeLengths( } leaves[i].weight = (leaves[i].weight << 9) | leaves[i].count; } +#ifdef __cplusplus + struct { + bool operator()(const Node a, const Node b) { + return (a.weight < b.weight); + } + } cmp; + std::sort(leaves, leaves + numsymbols, cmp); + +#else qsort(leaves, numsymbols, sizeof(Node), LeafComparator); +#endif for (i = 0; i < numsymbols; i++) { leaves[i].weight >>= 9; } From b48be4c20534596926fbcddb515a411af7e7361b Mon Sep 17 00:00:00 2001 From: fhanau Date: Tue, 18 Oct 2016 21:57:26 +0200 Subject: [PATCH 6/9] Fix warnings --- src/zopfli/katajainen.c | 12 ++++++------ src/zopfli/katajainen.h | 3 +++ 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/src/zopfli/katajainen.c b/src/zopfli/katajainen.c index 0eeb6086..7cb10102 100644 --- a/src/zopfli/katajainen.c +++ b/src/zopfli/katajainen.c @@ -125,6 +125,12 @@ Comparator for sorting the leaves. Has the function signature for qsort. static int LeafComparator(const void* a, const void* b) { return ((const Node*)a)->weight - ((const Node*)b)->weight; } +#else +struct { + bool operator()(const Node a, const Node b) { + return (a.weight < b.weight); + } +} cmp; #endif #ifdef __cplusplus @@ -192,13 +198,7 @@ int ZopfliLengthLimitedCodeLengths( leaves[i].weight = (leaves[i].weight << 9) | leaves[i].count; } #ifdef __cplusplus - struct { - bool operator()(const Node a, const Node b) { - return (a.weight < b.weight); - } - } cmp; std::sort(leaves, leaves + numsymbols, cmp); - #else qsort(leaves, numsymbols, sizeof(Node), LeafComparator); #endif diff --git a/src/zopfli/katajainen.h b/src/zopfli/katajainen.h index 5927350d..f5232d80 100644 --- a/src/zopfli/katajainen.h +++ b/src/zopfli/katajainen.h @@ -36,6 +36,9 @@ maxbits: Maximum bit length, inclusive. bitlengths: Output, the bitlengths for the symbol prefix codes. return: 0 for OK, non-0 for error. */ +#ifdef __cplusplus +extern "C" +#endif int ZopfliLengthLimitedCodeLengths( const size_t* frequencies, int n, int maxbits, unsigned* bitlengths); From 537ca9bf78066f5267b5aec9813638ed1d1e8071 Mon Sep 17 00:00:00 2001 From: Mike Kuta Date: Wed, 15 Mar 2017 11:56:11 -0400 Subject: [PATCH 7/9] Check for greyscale when small image uses palette When a small file is stored with a palette, a test is made to see if removing the palette, and the overhead that goes with it, results in an even smaller file. That test was always attempting RGB or RGBA mode even when the image in question was greyscale. I added a check for greyscale. --- src/zopflipng/zopflipng_lib.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/zopflipng/zopflipng_lib.cc b/src/zopflipng/zopflipng_lib.cc index b93bb18b..89312eca 100644 --- a/src/zopflipng/zopflipng_lib.cc +++ b/src/zopflipng/zopflipng_lib.cc @@ -239,7 +239,7 @@ unsigned TryOptimize( // Too small for tRNS chunk overhead. if (w * h <= 16 && profile.key) profile.alpha = 1; state.encoder.auto_convert = 0; - state.info_png.color.colortype = (profile.alpha ? LCT_RGBA : LCT_RGB); + state.info_png.color.colortype = (profile.alpha ? (profile.colored ? LCT_RGBA : LCT_GREY_ALPHA) : (profile.colored ? LCT_RGB : LCT_GREY)); state.info_png.color.bitdepth = 8; state.info_png.color.key_defined = (profile.key && !profile.alpha); if (state.info_png.color.key_defined) { From 38b6ffb5860fb4da0656f32e554805cb9a8571ce Mon Sep 17 00:00:00 2001 From: Mike Kuta Date: Thu, 16 Mar 2017 10:58:05 -0400 Subject: [PATCH 8/9] Use bit depth of greyscale when trying to beat palette When checking if an image is smaller without a palette, use the minimum bit depth for greyscale images instead of defaulting to 8 bits. --- src/zopflipng/zopflipng_lib.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/zopflipng/zopflipng_lib.cc b/src/zopflipng/zopflipng_lib.cc index 89312eca..72e5cf69 100644 --- a/src/zopflipng/zopflipng_lib.cc +++ b/src/zopflipng/zopflipng_lib.cc @@ -240,7 +240,7 @@ unsigned TryOptimize( if (w * h <= 16 && profile.key) profile.alpha = 1; state.encoder.auto_convert = 0; state.info_png.color.colortype = (profile.alpha ? (profile.colored ? LCT_RGBA : LCT_GREY_ALPHA) : (profile.colored ? LCT_RGB : LCT_GREY)); - state.info_png.color.bitdepth = 8; + state.info_png.color.bitdepth = (profile.alpha || profile.colored) ? 8 : profile.bits; state.info_png.color.key_defined = (profile.key && !profile.alpha); if (state.info_png.color.key_defined) { state.info_png.color.key_defined = 1; From 3315c43be52e189d947a645a5ecc790e320c35e3 Mon Sep 17 00:00:00 2001 From: fhanau Date: Tue, 28 Mar 2017 17:47:20 +0200 Subject: [PATCH 9/9] Fix zlib header --- src/zopflipng/lodepng/lodepng.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/zopflipng/lodepng/lodepng.cpp b/src/zopflipng/lodepng/lodepng.cpp index 59e3af94..72d51502 100644 --- a/src/zopflipng/lodepng/lodepng.cpp +++ b/src/zopflipng/lodepng/lodepng.cpp @@ -2181,7 +2181,7 @@ unsigned lodepng_zlib_compress(unsigned char** out, size_t* outsize, const unsig /*zlib data: 1 byte CMF (CM+CINFO), 1 byte FLG, deflate data, 4 byte ADLER32 checksum of the Decompressed data*/ unsigned CMF = 120; /*0b01111000: CM 8, CINFO 7. With CINFO 7, any window size up to 32768 can be used.*/ - unsigned FLEVEL = 0; + unsigned FLEVEL = settings->custom_deflate ? 3 : 0; unsigned FDICT = 0; unsigned CMFFLG = 256 * CMF + FDICT * 32 + FLEVEL * 64; unsigned FCHECK = 31 - CMFFLG % 31;