From b82f05caa81cb654138d4a57416167f6e9bd2591 Mon Sep 17 00:00:00 2001 From: inikep Date: Mon, 1 Aug 2016 15:15:11 +0200 Subject: [PATCH 01/27] introduce ZSTD_REP_MOVE_OPT --- lib/common/zstd_internal.h | 7 ++--- lib/compress/zstd_opt.h | 52 +++++++++++++++++++++----------------- 2 files changed, 33 insertions(+), 26 deletions(-) diff --git a/lib/common/zstd_internal.h b/lib/common/zstd_internal.h index 0a1935a9..9f1ec2ec 100644 --- a/lib/common/zstd_internal.h +++ b/lib/common/zstd_internal.h @@ -67,9 +67,10 @@ #define ZSTD_OPT_NUM (1<<12) #define ZSTD_DICT_MAGIC 0xEC30A437 /* v0.7+ */ -#define ZSTD_REP_NUM 3 /* number of repcodes */ -#define ZSTD_REP_CHECK (ZSTD_REP_NUM-0) /* number of repcodes to check by the optimal parser */ -#define ZSTD_REP_MOVE (ZSTD_REP_NUM-1) +#define ZSTD_REP_NUM 3 /* number of repcodes */ +#define ZSTD_REP_CHECK (ZSTD_REP_NUM) /* number of repcodes to check by the optimal parser */ +#define ZSTD_REP_MOVE (ZSTD_REP_NUM-1) +#define ZSTD_REP_MOVE_OPT (ZSTD_REP_NUM-1) static const U32 repStartValue[ZSTD_REP_NUM] = { 1, 4, 8 }; #define KB *(1 <<10) diff --git a/lib/compress/zstd_opt.h b/lib/compress/zstd_opt.h index 3a1e9e19..b32d173c 100644 --- a/lib/compress/zstd_opt.h +++ b/lib/compress/zstd_opt.h @@ -276,7 +276,7 @@ static U32 ZSTD_insertBtAndGetAllMatches ( /* save best solution */ if (currentMl > bestLength) { bestLength = currentMl; - matches[mnum].off = ZSTD_REP_MOVE + current - matchIndex3; + matches[mnum].off = ZSTD_REP_MOVE_OPT + current - matchIndex3; matches[mnum].len = (U32)currentMl; mnum++; if (currentMl > ZSTD_OPT_NUM) goto update; @@ -321,7 +321,7 @@ static U32 ZSTD_insertBtAndGetAllMatches ( if (matchLength > bestLength) { if (matchLength > matchEndIdx - matchIndex) matchEndIdx = matchIndex + (U32)matchLength; bestLength = matchLength; - matches[mnum].off = ZSTD_REP_MOVE + current - matchIndex; + matches[mnum].off = ZSTD_REP_MOVE_OPT + current - matchIndex; matches[mnum].len = (U32)matchLength; mnum++; if (matchLength > ZSTD_OPT_NUM) break; @@ -798,12 +798,12 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx, for (u = 0; u < match_num; u++) { mlen = (u>0) ? matches[u-1].len+1 : best_mlen; best_mlen = matches[u].len; - ZSTD_LOG_PARSER("%d: start Found mlen=%d off=%d best_mlen=%d last_pos=%d\n", (int)(ip-base), matches[u].len, matches[u].off, (int)best_mlen, (int)last_pos); + ZSTD_LOG_PARSER("%d: start Found mlen=%d off=%d best_mlen=%d last_pos=%d\n", (int)(ip-base), matches[u].len, matches[u].off-1, (int)best_mlen, (int)last_pos); litlen = opt[0].litlen; while (mlen <= best_mlen) { - price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[u].off, mlen - MINMATCH); + price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[u].off-1, mlen - MINMATCH); if (mlen > last_pos || price < opt[mlen].price) - SET_PRICE(mlen, mlen, matches[u].off, litlen, price); + SET_PRICE(mlen, mlen, matches[u].off-1, litlen, price); mlen++; } } @@ -836,14 +836,15 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx, continue; mlen = opt[cur].mlen; - if (opt[cur].off >= ZSTD_REP_NUM) { + if (opt[cur].off > ZSTD_REP_MOVE_OPT) { opt[cur].rep[2] = opt[cur-mlen].rep[1]; opt[cur].rep[1] = opt[cur-mlen].rep[0]; - opt[cur].rep[0] = opt[cur].off - ZSTD_REP_MOVE; + opt[cur].rep[0] = opt[cur].off - ZSTD_REP_MOVE_OPT; ZSTD_LOG_ENCODE("%d: COPYREP_OFF cur=%d mlen=%d rep[0]=%d rep[1]=%d\n", (int)(inr-base), cur, mlen, opt[cur].rep[0], opt[cur].rep[1]); } else { opt[cur].rep[2] = (opt[cur].off > 1) ? opt[cur-mlen].rep[1] : opt[cur-mlen].rep[2]; opt[cur].rep[1] = (opt[cur].off > 0) ? opt[cur-mlen].rep[0] : opt[cur-mlen].rep[1]; +// opt[cur].rep[0] = ((mlen != 1) && (opt[cur].off==3)) ? (opt[cur-mlen].rep[0] - 1) : (opt[cur-mlen].rep[opt[cur].off]); opt[cur].rep[0] = opt[cur-mlen].rep[opt[cur].off]; ZSTD_LOG_ENCODE("%d: COPYREP_NOR cur=%d mlen=%d rep[0]=%d rep[1]=%d\n", (int)(inr-base), cur, mlen, opt[cur].rep[0], opt[cur].rep[1]); } @@ -851,12 +852,14 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx, ZSTD_LOG_PARSER("%d: CURRENT_Ext price[%d/%d]=%d off=%d mlen=%d litlen=%d rep[0]=%d rep[1]=%d\n", (int)(inr-base), cur, last_pos, opt[cur].price, opt[cur].off, opt[cur].mlen, opt[cur].litlen, opt[cur].rep[0], opt[cur].rep[1]); best_mlen = 0; - { U32 i; - for (i = (opt[cur].mlen != 1); i 0 && repCur <= current+cur) && (((U32)((dictLimit-1) - repIndex) >= 3) & (repIndex>lowestIndex)) /* intentional overflow */ && (MEM_readMINMATCH(inr, minMatch) == MEM_readMINMATCH(repMatch, minMatch)) ) { /* repcode detected */ @@ -909,22 +912,22 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx, mlen = (u>0) ? matches[u-1].len+1 : best_mlen; best_mlen = (cur + matches[u].len < ZSTD_OPT_NUM) ? matches[u].len : ZSTD_OPT_NUM - cur; - // ZSTD_LOG_PARSER("%d: Found1 cur=%d mlen=%d off=%d best_mlen=%d last_pos=%d\n", (int)(inr-base), cur, matches[u].len, matches[u].off, best_mlen, last_pos); + // ZSTD_LOG_PARSER("%d: Found1 cur=%d mlen=%d off=%d best_mlen=%d last_pos=%d\n", (int)(inr-base), cur, matches[u].len, matches[u].off-1, best_mlen, last_pos); while (mlen <= best_mlen) { if (opt[cur].mlen == 1) { litlen = opt[cur].litlen; if (cur > litlen) - price = opt[cur - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, ip+cur-litlen, matches[u].off, mlen - MINMATCH); + price = opt[cur - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, ip+cur-litlen, matches[u].off-1, mlen - MINMATCH); else - price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[u].off, mlen - MINMATCH); + price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[u].off-1, mlen - MINMATCH); } else { litlen = 0; - price = opt[cur].price + ZSTD_getPrice(seqStorePtr, 0, NULL, matches[u].off, mlen - MINMATCH); + price = opt[cur].price + ZSTD_getPrice(seqStorePtr, 0, NULL, matches[u].off-1, mlen - MINMATCH); } - // ZSTD_LOG_PARSER("%d: Found2 mlen=%d best_mlen=%d off=%d price=%d litlen=%d\n", (int)(inr-base), mlen, best_mlen, matches[u].off, price, litlen); + // ZSTD_LOG_PARSER("%d: Found2 mlen=%d best_mlen=%d off=%d price=%d litlen=%d\n", (int)(inr-base), mlen, best_mlen, matches[u].off-1, price, litlen); if (cur + mlen > last_pos || (price < opt[cur + mlen].price)) - SET_PRICE(cur + mlen, mlen, matches[u].off, litlen, price); + SET_PRICE(cur + mlen, mlen, matches[u].off-1, litlen, price); mlen++; } } } /* for (cur = 1; cur <= last_pos; cur++) */ @@ -966,27 +969,30 @@ _storeSequence: /* cur, last_pos, best_mlen, best_off have to be set */ litLength = (U32)(ip - anchor); // ZSTD_LOG_ENCODE("%d/%d: ENCODE1 literals=%d mlen=%d off=%d rep[0]=%d rep[1]=%d\n", (int)(ip-base), (int)(iend-base), (int)(litLength), (int)mlen, (int)(offset), (int)rep[0], (int)rep[1]); - if (offset >= ZSTD_REP_NUM) { + if (offset > ZSTD_REP_MOVE_OPT) { rep[2] = rep[1]; rep[1] = rep[0]; - rep[0] = offset - ZSTD_REP_MOVE; + rep[0] = offset - ZSTD_REP_MOVE_OPT; + // offset--; } else { if (offset != 0) { best_off = rep[offset]; +// best_off = ((litLength==0) && (offset==3)) ? (rep[0] - 1) : (rep[offset]); if (offset != 1) rep[2] = rep[1]; rep[1] = rep[0]; rep[0] = best_off; } - if ((litLength==0) & (offset==0)) offset = rep[1]; /* protection, but should never happen */ - if ((litLength==0) & (offset<=2)) offset --; + + if ((litLength==0) & (offset==0)) { ZSTD_LOG_ENCODE("ERROR (litLength==0) & (offset==0)\n"); }; + if (litLength==0) offset--; } ZSTD_LOG_ENCODE("%d/%d: ENCODE literals=%d mlen=%d off=%d rep[0]=%d rep[1]=%d\n", (int)(ip-base), (int)(iend-base), (int)(litLength), (int)mlen, (int)(offset), (int)rep[0], (int)rep[1]); #if ZSTD_OPT_DEBUG >= 5 U32 ml2; - if (offset >= ZSTD_REP_NUM) { - best_off = offset - ZSTD_REP_MOVE; + if (offset > ZSTD_REP_MOVE_OPT) { + best_off = offset - ZSTD_REP_MOVE_OPT; if (best_off > (size_t)(ip - prefixStart)) { const BYTE* match = dictEnd - (best_off - (ip - prefixStart)); ml2 = ZSTD_count_2segments(ip, match, iend, dictEnd, prefixStart); From bd32aff9571e95ab8c5c14569998c6bcd4608b3f Mon Sep 17 00:00:00 2001 From: inikep Date: Mon, 1 Aug 2016 15:29:03 +0200 Subject: [PATCH 02/27] #define ZSTD_REP_MOVE_OPT (ZSTD_REP_NUM) --- lib/common/zstd_internal.h | 2 +- lib/compress/zstd_opt.h | 21 +++++++++------------ 2 files changed, 10 insertions(+), 13 deletions(-) diff --git a/lib/common/zstd_internal.h b/lib/common/zstd_internal.h index 9f1ec2ec..60891bad 100644 --- a/lib/common/zstd_internal.h +++ b/lib/common/zstd_internal.h @@ -70,7 +70,7 @@ #define ZSTD_REP_NUM 3 /* number of repcodes */ #define ZSTD_REP_CHECK (ZSTD_REP_NUM) /* number of repcodes to check by the optimal parser */ #define ZSTD_REP_MOVE (ZSTD_REP_NUM-1) -#define ZSTD_REP_MOVE_OPT (ZSTD_REP_NUM-1) +#define ZSTD_REP_MOVE_OPT (ZSTD_REP_NUM) static const U32 repStartValue[ZSTD_REP_NUM] = { 1, 4, 8 }; #define KB *(1 <<10) diff --git a/lib/compress/zstd_opt.h b/lib/compress/zstd_opt.h index b32d173c..1ff8dbdb 100644 --- a/lib/compress/zstd_opt.h +++ b/lib/compress/zstd_opt.h @@ -798,12 +798,12 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx, for (u = 0; u < match_num; u++) { mlen = (u>0) ? matches[u-1].len+1 : best_mlen; best_mlen = matches[u].len; - ZSTD_LOG_PARSER("%d: start Found mlen=%d off=%d best_mlen=%d last_pos=%d\n", (int)(ip-base), matches[u].len, matches[u].off-1, (int)best_mlen, (int)last_pos); + ZSTD_LOG_PARSER("%d: start Found mlen=%d off=%d best_mlen=%d last_pos=%d\n", (int)(ip-base), matches[u].len, matches[u].off, (int)best_mlen, (int)last_pos); litlen = opt[0].litlen; while (mlen <= best_mlen) { price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[u].off-1, mlen - MINMATCH); if (mlen > last_pos || price < opt[mlen].price) - SET_PRICE(mlen, mlen, matches[u].off-1, litlen, price); + SET_PRICE(mlen, mlen, matches[u].off, litlen, price); mlen++; } } @@ -844,8 +844,7 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx, } else { opt[cur].rep[2] = (opt[cur].off > 1) ? opt[cur-mlen].rep[1] : opt[cur-mlen].rep[2]; opt[cur].rep[1] = (opt[cur].off > 0) ? opt[cur-mlen].rep[0] : opt[cur-mlen].rep[1]; -// opt[cur].rep[0] = ((mlen != 1) && (opt[cur].off==3)) ? (opt[cur-mlen].rep[0] - 1) : (opt[cur-mlen].rep[opt[cur].off]); - opt[cur].rep[0] = opt[cur-mlen].rep[opt[cur].off]; + opt[cur].rep[0] = ((opt[cur].off==3) && (mlen != 1)) ? (opt[cur-mlen].rep[0] - 1) : (opt[cur-mlen].rep[opt[cur].off]); ZSTD_LOG_ENCODE("%d: COPYREP_NOR cur=%d mlen=%d rep[0]=%d rep[1]=%d\n", (int)(inr-base), cur, mlen, opt[cur].rep[0], opt[cur].rep[1]); } @@ -854,8 +853,7 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx, { U32 i, last_i = ZSTD_REP_CHECK; //+(mlen != 1); for (i = (mlen != 1); i0) ? matches[u-1].len+1 : best_mlen; best_mlen = (cur + matches[u].len < ZSTD_OPT_NUM) ? matches[u].len : ZSTD_OPT_NUM - cur; - // ZSTD_LOG_PARSER("%d: Found1 cur=%d mlen=%d off=%d best_mlen=%d last_pos=%d\n", (int)(inr-base), cur, matches[u].len, matches[u].off-1, best_mlen, last_pos); + // ZSTD_LOG_PARSER("%d: Found1 cur=%d mlen=%d off=%d best_mlen=%d last_pos=%d\n", (int)(inr-base), cur, matches[u].len, matches[u].off, best_mlen, last_pos); while (mlen <= best_mlen) { if (opt[cur].mlen == 1) { litlen = opt[cur].litlen; @@ -925,9 +923,9 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx, price = opt[cur].price + ZSTD_getPrice(seqStorePtr, 0, NULL, matches[u].off-1, mlen - MINMATCH); } - // ZSTD_LOG_PARSER("%d: Found2 mlen=%d best_mlen=%d off=%d price=%d litlen=%d\n", (int)(inr-base), mlen, best_mlen, matches[u].off-1, price, litlen); + // ZSTD_LOG_PARSER("%d: Found2 mlen=%d best_mlen=%d off=%d price=%d litlen=%d\n", (int)(inr-base), mlen, best_mlen, matches[u].off, price, litlen); if (cur + mlen > last_pos || (price < opt[cur + mlen].price)) - SET_PRICE(cur + mlen, mlen, matches[u].off-1, litlen, price); + SET_PRICE(cur + mlen, mlen, matches[u].off, litlen, price); mlen++; } } } /* for (cur = 1; cur <= last_pos; cur++) */ @@ -973,11 +971,10 @@ _storeSequence: /* cur, last_pos, best_mlen, best_off have to be set */ rep[2] = rep[1]; rep[1] = rep[0]; rep[0] = offset - ZSTD_REP_MOVE_OPT; - // offset--; + offset--; } else { if (offset != 0) { - best_off = rep[offset]; -// best_off = ((litLength==0) && (offset==3)) ? (rep[0] - 1) : (rep[offset]); + best_off = ((offset==3) && (litLength==0)) ? (rep[0] - 1) : (rep[offset]); if (offset != 1) rep[2] = rep[1]; rep[1] = rep[0]; rep[0] = best_off; From 050909de29a60df0669d23d9cf48e4e5036937c6 Mon Sep 17 00:00:00 2001 From: inikep Date: Mon, 1 Aug 2016 16:00:24 +0200 Subject: [PATCH 03/27] when ll=0 offset 3 becomes rep[0]-1 : in ZSTD_compressBlock_opt_extDict_generic --- lib/compress/zstd_opt.h | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/lib/compress/zstd_opt.h b/lib/compress/zstd_opt.h index 1ff8dbdb..c86e9f56 100644 --- a/lib/compress/zstd_opt.h +++ b/lib/compress/zstd_opt.h @@ -748,12 +748,13 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx, opt[0].litlen = (U32)(ip - anchor); /* check repCode */ - { U32 i; - for (i = (ip==anchor); i 0 && repCur <= (S32)current) && (((U32)((dictLimit-1) - repIndex) >= 3) & (repIndex>lowestIndex)) /* intentional overflow */ && (MEM_readMINMATCH(ip, minMatch) == MEM_readMINMATCH(repMatch, minMatch)) ) { /* repcode detected we should take it */ @@ -844,20 +845,20 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx, } else { opt[cur].rep[2] = (opt[cur].off > 1) ? opt[cur-mlen].rep[1] : opt[cur-mlen].rep[2]; opt[cur].rep[1] = (opt[cur].off > 0) ? opt[cur-mlen].rep[0] : opt[cur-mlen].rep[1]; - opt[cur].rep[0] = ((opt[cur].off==3) && (mlen != 1)) ? (opt[cur-mlen].rep[0] - 1) : (opt[cur-mlen].rep[opt[cur].off]); + opt[cur].rep[0] = ((opt[cur].off==ZSTD_REP_MOVE_OPT) && (mlen != 1)) ? (opt[cur-mlen].rep[0] - 1) : (opt[cur-mlen].rep[opt[cur].off]); ZSTD_LOG_ENCODE("%d: COPYREP_NOR cur=%d mlen=%d rep[0]=%d rep[1]=%d\n", (int)(inr-base), cur, mlen, opt[cur].rep[0], opt[cur].rep[1]); } ZSTD_LOG_PARSER("%d: CURRENT_Ext price[%d/%d]=%d off=%d mlen=%d litlen=%d rep[0]=%d rep[1]=%d\n", (int)(inr-base), cur, last_pos, opt[cur].price, opt[cur].off, opt[cur].mlen, opt[cur].litlen, opt[cur].rep[0], opt[cur].rep[1]); best_mlen = 0; - { U32 i, last_i = ZSTD_REP_CHECK; //+(mlen != 1); + { U32 i, last_i = ZSTD_REP_CHECK + (mlen != 1); for (i = (mlen != 1); i 0 && repCur <= current+cur) + if ( (repCur > 0 && repCur <= (S32)(current+cur)) && (((U32)((dictLimit-1) - repIndex) >= 3) & (repIndex>lowestIndex)) /* intentional overflow */ && (MEM_readMINMATCH(inr, minMatch) == MEM_readMINMATCH(repMatch, minMatch)) ) { /* repcode detected */ @@ -974,7 +975,7 @@ _storeSequence: /* cur, last_pos, best_mlen, best_off have to be set */ offset--; } else { if (offset != 0) { - best_off = ((offset==3) && (litLength==0)) ? (rep[0] - 1) : (rep[offset]); + best_off = ((offset==ZSTD_REP_MOVE_OPT) && (litLength==0)) ? (rep[0] - 1) : (rep[offset]); if (offset != 1) rep[2] = rep[1]; rep[1] = rep[0]; rep[0] = best_off; From a95f328578d7ccac5c3d1351ad5c33d3c615d802 Mon Sep 17 00:00:00 2001 From: inikep Date: Tue, 2 Aug 2016 12:00:54 +0200 Subject: [PATCH 04/27] when ll=0 offset 3 becomes rep[0]-1 : in ZSTD_compressBlock_opt_generic --- lib/compress/zstd_opt.h | 51 ++++++++++++++++++++++------------------- 1 file changed, 27 insertions(+), 24 deletions(-) diff --git a/lib/compress/zstd_opt.h b/lib/compress/zstd_opt.h index c86e9f56..d9f6cfdc 100644 --- a/lib/compress/zstd_opt.h +++ b/lib/compress/zstd_opt.h @@ -452,11 +452,12 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx, litlen = (U32)(ip - anchor); /* check repCode */ - { U32 i; - for (i=(ip == anchor); i 0) && (repCur < (S32)(ip-prefixStart)) + && (MEM_readMINMATCH(ip, minMatch) == MEM_readMINMATCH(ip - repCur, minMatch))) { + mlen = (U32)ZSTD_count(ip+minMatch, ip+minMatch-repCur, iend) + minMatch; ZSTD_LOG_PARSER("%d: start try REP rep[%d]=%d mlen=%d\n", (int)(ip-base), i, (int)rep[i], (int)mlen); if (mlen > sufficient_len || mlen >= ZSTD_OPT_NUM) { best_mlen = mlen; best_off = i; cur = 0; last_pos = 1; @@ -491,7 +492,7 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx, best_mlen = matches[u].len; ZSTD_LOG_PARSER("%d: start Found mlen=%d off=%d best_mlen=%d last_pos=%d\n", (int)(ip-base), matches[u].len, matches[u].off, (int)best_mlen, (int)last_pos); while (mlen <= best_mlen) { - price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[u].off, mlen - MINMATCH); + price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[u].off-1, mlen - MINMATCH); if (mlen > last_pos || price < opt[mlen].price) SET_PRICE(mlen, mlen, matches[u].off, litlen, price); /* note : macro modifies last_pos */ mlen++; @@ -528,26 +529,27 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx, continue; mlen = opt[cur].mlen; - if (opt[cur].off >= ZSTD_REP_NUM) { + if (opt[cur].off > ZSTD_REP_MOVE_OPT) { opt[cur].rep[2] = opt[cur-mlen].rep[1]; opt[cur].rep[1] = opt[cur-mlen].rep[0]; - opt[cur].rep[0] = opt[cur].off - ZSTD_REP_MOVE; + opt[cur].rep[0] = opt[cur].off - ZSTD_REP_MOVE_OPT; ZSTD_LOG_ENCODE("%d: COPYREP_OFF cur=%d mlen=%d rep[0]=%d rep[1]=%d\n", (int)(inr-base), cur, mlen, opt[cur].rep[0], opt[cur].rep[1]); } else { opt[cur].rep[2] = (opt[cur].off > 1) ? opt[cur-mlen].rep[1] : opt[cur-mlen].rep[2]; opt[cur].rep[1] = (opt[cur].off > 0) ? opt[cur-mlen].rep[0] : opt[cur-mlen].rep[1]; - opt[cur].rep[0] = opt[cur-mlen].rep[opt[cur].off]; + opt[cur].rep[0] = ((opt[cur].off==ZSTD_REP_MOVE_OPT) && (mlen != 1)) ? (opt[cur-mlen].rep[0] - 1) : (opt[cur-mlen].rep[opt[cur].off]); ZSTD_LOG_ENCODE("%d: COPYREP_NOR cur=%d mlen=%d rep[0]=%d rep[1]=%d\n", (int)(inr-base), cur, mlen, opt[cur].rep[0], opt[cur].rep[1]); } ZSTD_LOG_PARSER("%d: CURRENT_NoExt price[%d/%d]=%d off=%d mlen=%d litlen=%d rep[0]=%d rep[1]=%d\n", (int)(inr-base), cur, last_pos, opt[cur].price, opt[cur].off, opt[cur].mlen, opt[cur].litlen, opt[cur].rep[0], opt[cur].rep[1]); best_mlen = minMatch; - { U32 i; - for (i=(opt[cur].mlen != 1); i 0) && (repCur < (S32)(inr-prefixStart)) + && (MEM_readMINMATCH(inr, minMatch) == MEM_readMINMATCH(inr - repCur, minMatch))) { + mlen = (U32)ZSTD_count(inr+minMatch, inr+minMatch - repCur, iend) + minMatch; ZSTD_LOG_PARSER("%d: Found REP %d/%d mlen=%d off=%d rep=%d opt[%d].off=%d\n", (int)(inr-base), i, ZSTD_REP_NUM, mlen, i, opt[cur].rep[i], cur, opt[cur].off); if (mlen > sufficient_len || cur + mlen >= ZSTD_OPT_NUM) { @@ -600,12 +602,12 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx, if (opt[cur].mlen == 1) { litlen = opt[cur].litlen; if (cur > litlen) - price = opt[cur - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, ip+cur-litlen, matches[u].off, mlen - MINMATCH); + price = opt[cur - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, ip+cur-litlen, matches[u].off-1, mlen - MINMATCH); else - price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[u].off, mlen - MINMATCH); + price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[u].off-1, mlen - MINMATCH); } else { litlen = 0; - price = opt[cur].price + ZSTD_getPrice(seqStorePtr, 0, NULL, matches[u].off, mlen - MINMATCH); + price = opt[cur].price + ZSTD_getPrice(seqStorePtr, 0, NULL, matches[u].off-1, mlen - MINMATCH); } // ZSTD_LOG_PARSER("%d: Found2 mlen=%d best_mlen=%d off=%d price=%d litlen=%d\n", (int)(inr-base), mlen, best_mlen, matches[u].off, price, litlen); @@ -652,27 +654,28 @@ _storeSequence: /* cur, last_pos, best_mlen, best_off have to be set */ litLength = (U32)(ip - anchor); // ZSTD_LOG_ENCODE("%d/%d: ENCODE literals=%d mlen=%d off=%d rep[0]=%d rep[1]=%d\n", (int)(ip-base), (int)(iend-base), (int)(litLength), (int)mlen, (int)(offset), (int)rep[0], (int)rep[1]); - if (offset >= ZSTD_REP_NUM) { + if (offset > ZSTD_REP_MOVE_OPT) { rep[2] = rep[1]; rep[1] = rep[0]; - rep[0] = offset - ZSTD_REP_MOVE; + rep[0] = offset - ZSTD_REP_MOVE_OPT; + offset--; } else { if (offset != 0) { - best_off = rep[offset]; + best_off = ((offset==ZSTD_REP_MOVE_OPT) && (litLength==0)) ? (rep[0] - 1) : (rep[offset]); if (offset != 1) rep[2] = rep[1]; rep[1] = rep[0]; rep[0] = best_off; } - if ((litLength == 0) & (offset==0)) offset = rep[1]; /* protection, but should never happen */ - if ((litLength == 0) & (offset<=2)) offset--; + if ((litLength==0) & (offset==0)) { ZSTD_LOG_ENCODE("ERROR (litLength==0) & (offset==0)\n"); }; + if (litLength==0) offset--; } ZSTD_LOG_ENCODE("%d/%d: ENCODE literals=%d mlen=%d off=%d rep[0]=%d rep[1]=%d\n", (int)(ip-base), (int)(iend-base), (int)(litLength), (int)mlen, (int)(offset), (int)rep[0], (int)rep[1]); #if ZSTD_OPT_DEBUG >= 5 U32 ml2; - if (offset >= ZSTD_REP_NUM) - ml2 = (U32)ZSTD_count(ip, ip-(offset-ZSTD_REP_MOVE), iend); + if (offset > ZSTD_REP_MOVE_OPT) + ml2 = (U32)ZSTD_count(ip, ip-(offset-ZSTD_REP_MOVE_OPT), iend); else ml2 = (U32)ZSTD_count(ip, ip-rep[0], iend); if ((offset >= 8) && (ml2 < mlen || ml2 < minMatch)) { From 387800f367d8fe6d880dcbae032cbf2843f9b6d2 Mon Sep 17 00:00:00 2001 From: inikep Date: Wed, 3 Aug 2016 11:08:16 +0200 Subject: [PATCH 05/27] bench.c: removed benchResult_t --- programs/bench.c | 34 +++++++++++----------------------- 1 file changed, 11 insertions(+), 23 deletions(-) diff --git a/programs/bench.c b/programs/bench.c index f4bff883..c0bfd0da 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -125,14 +125,6 @@ typedef struct size_t resSize; } blockParam_t; -typedef struct -{ - double ratio; - size_t cSize; - double cSpeed; - double dSpeed; -} benchResult_t; - #define MIN(a,b) ((a)<(b) ? (a) : (b)) #define MAX(a,b) ((a)>(b) ? (a) : (b)) @@ -140,7 +132,7 @@ typedef struct static int BMK_benchMem(const void* srcBuffer, size_t srcSize, const char* displayName, int cLevel, const size_t* fileSizes, U32 nbFiles, - const void* dictBuffer, size_t dictBufferSize, benchResult_t *result) + const void* dictBuffer, size_t dictBufferSize) { size_t const blockSize = (g_blockSize>=32 ? g_blockSize : srcSize) + (!srcSize) /* avoid div by 0 */ ; U32 const maxNbBlocks = (U32) ((srcSize + (blockSize-1)) / blockSize) + nbFiles; @@ -312,10 +304,14 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, #endif } /* for (testNb = 1; testNb <= (g_nbIterations + !g_nbIterations); testNb++) */ - result->ratio = ratio; - result->cSize = cSize; - result->cSpeed = (double)srcSize / fastestC; - result->dSpeed = (double)srcSize / fastestD; + if (g_displayLevel == 1) { + double cSpeed = (double)srcSize / fastestC; + double dSpeed = (double)srcSize / fastestD; + if (g_additionalParam) + DISPLAY("-%-3i%11i (%5.3f) %6.2f MB/s %6.1f MB/s %s (param=%d)\n", cLevel, (int)cSize, ratio, cSpeed, dSpeed, displayName, g_additionalParam); + else + DISPLAY("-%-3i%11i (%5.3f) %6.2f MB/s %6.1f MB/s %s\n", cLevel, (int)cSize, ratio, cSpeed, dSpeed, displayName); + } DISPLAYLEVEL(2, "%2i#\n", cLevel); } /* Bench */ @@ -352,7 +348,6 @@ static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize, const size_t* fileSizes, unsigned nbFiles, const void* dictBuffer, size_t dictBufferSize) { - benchResult_t result; int l; const char* pch = strrchr(displayName, '\\'); /* Windows */ @@ -361,8 +356,6 @@ static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize, SET_HIGH_PRIORITY; - memset(&result, 0, sizeof(result)); - if (g_displayLevel == 1 && !g_additionalParam) DISPLAY("bench %s %s: input %u bytes, %i iterations, %u KB blocks\n", ZSTD_VERSION_STRING, ZSTD_GIT_COMMIT_STRING, (U32)benchedSize, g_nbIterations, (U32)(g_blockSize>>10)); @@ -372,13 +365,8 @@ static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize, BMK_benchMem(srcBuffer, benchedSize, displayName, l, fileSizes, nbFiles, - dictBuffer, dictBufferSize, &result); - if (g_displayLevel == 1) { - if (g_additionalParam) - DISPLAY("%-3i%11i (%5.3f) %6.2f MB/s %6.1f MB/s %s (param=%d)\n", -l, (int)result.cSize, result.ratio, result.cSpeed, result.dSpeed, displayName, g_additionalParam); - else - DISPLAY("%-3i%11i (%5.3f) %6.2f MB/s %6.1f MB/s %s\n", -l, (int)result.cSize, result.ratio, result.cSpeed, result.dSpeed, displayName); - } } + dictBuffer, dictBufferSize); + } } From fa3fbb4537468e24a3d73a03a1d14a50e64386e6 Mon Sep 17 00:00:00 2001 From: inikep Date: Wed, 3 Aug 2016 13:10:47 +0200 Subject: [PATCH 06/27] test-zstd-speed.py: added tests for clang --- tests/test-zstd-speed.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/tests/test-zstd-speed.py b/tests/test-zstd-speed.py index c517097a..d8d9543e 100755 --- a/tests/test-zstd-speed.py +++ b/tests/test-zstd-speed.py @@ -176,7 +176,10 @@ def test_commit(branch, commit, last_commit, args, testFilePaths, have_mutt, hav local_branch = string.split(branch, '/')[1] version = local_branch.rpartition('-')[2] + '_' + commit if not args.dry_run: - execute('make -C programs clean zstd MOREFLAGS="-DZSTD_GIT_COMMIT=%s" && make -B -C programs zstd32 MOREFLAGS="-DZSTD_GIT_COMMIT=%s"' % (version, version)) + execute('make -C programs clean zstd CC=clang MOREFLAGS="-Werror -Wconversion -Wno-sign-conversion -DZSTD_GIT_COMMIT=%s" && ' % version + + 'mv programs/zstd programs/zstd_clang && ' + + 'make -C programs clean zstd MOREFLAGS="-DZSTD_GIT_COMMIT=%s" && ' % version + + 'make -B -C programs zstd32 MOREFLAGS="-DZSTD_GIT_COMMIT=%s"' % version) logFileName = working_path + "/log_" + branch.replace("/", "_") + ".txt" text_to_send = [] results_files = "" @@ -192,6 +195,11 @@ def test_commit(branch, commit, last_commit, args, testFilePaths, have_mutt, hav if text: text_to_send.append(text) results_files += resultsFileName + " " + resultsFileName = working_path + "/resultsClang_" + branch.replace("/", "_") + "_" + fileName.replace(".", "_") + ".txt" + text = double_check(branch, commit, args, 'zstd_clang', resultsFileName, filePath, fileName) + if text: + text_to_send.append(text) + results_files += resultsFileName + " " if text_to_send: send_email_with_attachments(branch, commit, last_commit, args, text_to_send, results_files, logFileName, have_mutt, have_mail) From ded65ec8ceaf96ee739b125f5d76e82a0d462904 Mon Sep 17 00:00:00 2001 From: inikep Date: Wed, 3 Aug 2016 13:17:21 +0200 Subject: [PATCH 07/27] test-zstd-speed.py: script versioning --- tests/test-zstd-speed.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test-zstd-speed.py b/tests/test-zstd-speed.py index d8d9543e..216fb5ea 100755 --- a/tests/test-zstd-speed.py +++ b/tests/test-zstd-speed.py @@ -7,7 +7,7 @@ import subprocess import time import traceback - +script_version = 'v0.8.0 [2016-08-03]' default_repo_url = 'https://github.com/Cyan4973/zstd.git' working_dir_name = 'speedTest' working_path = os.getcwd() + '/' + working_dir_name # /path/to/zstd/tests/speedTest @@ -271,7 +271,7 @@ if __name__ == '__main__': log("ERROR: %s already exists, exiting" % pidfile) exit(1) - send_email(args.emails, email_header + ':%s test-zstd-speed.py has been started' % pid, args.message, have_mutt, have_mail) + send_email(args.emails, email_header + ':%s test-zstd-speed.py %s has been started' % (pid, script_version), args.message, have_mutt, have_mail) file(pidfile, 'w').write(pid) while True: @@ -302,5 +302,5 @@ if __name__ == '__main__': time.sleep(args.sleepTime) except KeyboardInterrupt: os.unlink(pidfile) - send_email(args.emails, email_header + ':%s test-zstd-speed.py has been stopped' % pid, args.message, have_mutt, have_mail) + send_email(args.emails, email_header + ':%s test-zstd-speed.py %s has been stopped' % (pid, script_version), args.message, have_mutt, have_mail) exit(0) From 1b5a8fafec776cefb5193a9fa40fcfaac5ffca2d Mon Sep 17 00:00:00 2001 From: inikep Date: Wed, 3 Aug 2016 14:05:49 +0200 Subject: [PATCH 08/27] test-zstd-speed.py: md5 sum of executables --- tests/test-zstd-speed.py | 41 ++++++++++++++++++++++++++-------------- 1 file changed, 27 insertions(+), 14 deletions(-) diff --git a/tests/test-zstd-speed.py b/tests/test-zstd-speed.py index 216fb5ea..a0e1a281 100755 --- a/tests/test-zstd-speed.py +++ b/tests/test-zstd-speed.py @@ -6,17 +6,26 @@ import string import subprocess import time import traceback +import hashlib -script_version = 'v0.8.0 [2016-08-03]' +script_version = 'v0.8.0 (2016-08-03)' default_repo_url = 'https://github.com/Cyan4973/zstd.git' working_dir_name = 'speedTest' working_path = os.getcwd() + '/' + working_dir_name # /path/to/zstd/tests/speedTest clone_path = working_path + '/' + 'zstd' # /path/to/zstd/tests/speedTest/zstd -email_header = '[ZSTD_speedTest]' +email_header = 'ZSTD_speedTest' pid = str(os.getpid()) verbose = False + +def hashfile(hasher, fname, blocksize=65536): + with open(fname, "rb") as f: + for chunk in iter(lambda: f.read(blocksize), b""): + hasher.update(chunk) + return hasher.hexdigest() + + def log(text): print(time.strftime("%Y/%m/%d %H:%M:%S") + ' - ' + text) @@ -64,7 +73,7 @@ def send_email_with_attachments(branch, commit, last_commit, args, text, results with open(logFileName, "w") as myfile: myfile.writelines(text) myfile.close() - email_topic = '%s:%s Warning for %s:%s last_commit=%s speed<%s ratio<%s' \ + email_topic = '[%s:%s] Warning for %s:%s last_commit=%s speed<%s ratio<%s' \ % (email_header, pid, branch, commit, last_commit, args.lowerLimit, args.ratioLimit) if have_mutt: @@ -117,7 +126,7 @@ def get_last_results(resultsFileName): return commit, csize, cspeed, dspeed -def benchmark_and_compare(branch, commit, last_commit, args, executableName, resultsFileName, +def benchmark_and_compare(branch, commit, last_commit, args, executableName, md5sum, resultsFileName, testFilePath, fileName, last_csize, last_cspeed, last_dspeed): sleepTime = 30 while os.getloadavg()[0] > args.maxLoadAvg: @@ -149,7 +158,7 @@ def benchmark_and_compare(branch, commit, last_commit, args, executableName, res if (float(last_csize[i])/csize[i] < args.ratioLimit): text += "WARNING: %s -%d cSize=%d last_cSize=%d diff=%.4f %s\n" % (executableName, i+1, csize[i], last_csize[i], float(last_csize[i])/csize[i], fileName) if text: - text = args.message + ("\nmaxLoadAvg=%s load average at start=%s end=%s last_commit=%s\n" % (args.maxLoadAvg, start_load, end_load, last_commit)) + text + text = args.message + ("\nmaxLoadAvg=%s load average at start=%s end=%s last_commit=%s md5=%s\n" % (args.maxLoadAvg, start_load, end_load, last_commit, md5sum)) + text return text @@ -162,13 +171,13 @@ def update_config_file(branch, commit): return last_commit -def double_check(branch, commit, args, executableName, resultsFileName, filePath, fileName): +def double_check(branch, commit, args, executableName, md5sum, resultsFileName, filePath, fileName): last_commit, csize, cspeed, dspeed = get_last_results(resultsFileName) if not args.dry_run: - text = benchmark_and_compare(branch, commit, last_commit, args, executableName, resultsFileName, filePath, fileName, csize, cspeed, dspeed) + text = benchmark_and_compare(branch, commit, last_commit, args, executableName, md5sum, resultsFileName, filePath, fileName, csize, cspeed, dspeed) if text: log("WARNING: redoing tests for branch %s: commit %s" % (branch, commit)) - text = benchmark_and_compare(branch, commit, last_commit, args, executableName, resultsFileName, filePath, fileName, csize, cspeed, dspeed) + text = benchmark_and_compare(branch, commit, last_commit, args, executableName, md5sum, resultsFileName, filePath, fileName, csize, cspeed, dspeed) return text @@ -180,23 +189,27 @@ def test_commit(branch, commit, last_commit, args, testFilePaths, have_mutt, hav 'mv programs/zstd programs/zstd_clang && ' + 'make -C programs clean zstd MOREFLAGS="-DZSTD_GIT_COMMIT=%s" && ' % version + 'make -B -C programs zstd32 MOREFLAGS="-DZSTD_GIT_COMMIT=%s"' % version) + md5_zstd = hashfile(hashlib.md5(), clone_path + '/programs/zstd') + md5_zstd32 = hashfile(hashlib.md5(), clone_path + '/programs/zstd32') + md5_zstd_clang = hashfile(hashlib.md5(), clone_path + '/programs/zstd_clang') + print("md5(zstd)=%s\nmd5(zstd32)=%s\nmd5(zstd32_clang)=%s" % (md5_zstd, md5_zstd32, md5_zstd_clang)) logFileName = working_path + "/log_" + branch.replace("/", "_") + ".txt" text_to_send = [] results_files = "" for filePath in testFilePaths: fileName = filePath.rpartition('/')[2] resultsFileName = working_path + "/results_" + branch.replace("/", "_") + "_" + fileName.replace(".", "_") + ".txt" - text = double_check(branch, commit, args, 'zstd', resultsFileName, filePath, fileName) + text = double_check(branch, commit, args, 'zstd', md5_zstd, resultsFileName, filePath, fileName) if text: text_to_send.append(text) results_files += resultsFileName + " " resultsFileName = working_path + "/results32_" + branch.replace("/", "_") + "_" + fileName.replace(".", "_") + ".txt" - text = double_check(branch, commit, args, 'zstd32', resultsFileName, filePath, fileName) + text = double_check(branch, commit, args, 'zstd32', md5_zstd32, resultsFileName, filePath, fileName) if text: text_to_send.append(text) results_files += resultsFileName + " " resultsFileName = working_path + "/resultsClang_" + branch.replace("/", "_") + "_" + fileName.replace(".", "_") + ".txt" - text = double_check(branch, commit, args, 'zstd_clang', resultsFileName, filePath, fileName) + text = double_check(branch, commit, args, 'zstd_clang', md5_zstd_clang, resultsFileName, filePath, fileName) if text: text_to_send.append(text) results_files += resultsFileName + " " @@ -271,7 +284,7 @@ if __name__ == '__main__': log("ERROR: %s already exists, exiting" % pidfile) exit(1) - send_email(args.emails, email_header + ':%s test-zstd-speed.py %s has been started' % (pid, script_version), args.message, have_mutt, have_mail) + send_email(args.emails, '[%s:%s] test-zstd-speed.py %s has been started' % (email_header, pid, script_version), args.message, have_mutt, have_mail) file(pidfile, 'w').write(pid) while True: @@ -296,11 +309,11 @@ if __name__ == '__main__': time.sleep(args.sleepTime) except Exception as e: stack = traceback.format_exc() - email_topic = '%s:%s ERROR in %s:%s' % (email_header, pid, branch, commit) + email_topic = '[%s:%s] ERROR in %s:%s' % (email_header, pid, branch, commit) send_email(args.emails, email_topic, stack, have_mutt, have_mail) print(stack) time.sleep(args.sleepTime) except KeyboardInterrupt: os.unlink(pidfile) - send_email(args.emails, email_header + ':%s test-zstd-speed.py %s has been stopped' % (pid, script_version), args.message, have_mutt, have_mail) + send_email(args.emails, '[%s:%s] test-zstd-speed.py %s has been stopped' % (email_header, pid, script_version), args.message, have_mutt, have_mail) exit(0) From 586a0550f142691e38c3d2e2f7d57fb5a70cae44 Mon Sep 17 00:00:00 2001 From: inikep Date: Wed, 3 Aug 2016 16:16:38 +0200 Subject: [PATCH 09/27] huffman -> Huffman --- zstd_compression_format.md | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/zstd_compression_format.md b/zstd_compression_format.md index da5c94af..b6ace361 100644 --- a/zstd_compression_format.md +++ b/zstd_compression_format.md @@ -452,11 +452,11 @@ This field uses 2 lowest bits of first byte, describing 4 different block types - Raw literals block - Literals are stored uncompressed. - RLE literals block - Literals consist of a single byte value repeated N times. -- Compressed literals block - This is a standard huffman-compressed block, - starting with a huffman tree description. +- Compressed literals block - This is a standard Huffman-compressed block, + starting with a Huffman tree description. See details below. -- Repeat Stats literals block - This is a huffman-compressed block, - using huffman tree _from previous huffman-compressed literals block_. +- Repeat Stats literals block - This is a Huffman-compressed block, + using Huffman tree _from previous Huffman-compressed literals block_. Huffman tree description will be skipped. __Sizes format__ : @@ -533,7 +533,7 @@ by completing to the nearest power of 2. This power of 2 gives `maxBits`, the depth of the current tree. __Example__ : -Let's presume the following huffman tree must be described : +Let's presume the following Huffman tree must be described : | literal | 0 | 1 | 2 | 3 | 4 | 5 | | ------- | --- | --- | --- | --- | --- | --- | @@ -575,7 +575,7 @@ which tells how to decode the list of weights. the serie of weights is compressed by FSE. The length of the FSE-compressed serie is `headerByte` (0-127). -##### FSE (Finite State Entropy) compression of huffman weights +##### FSE (Finite State Entropy) compression of Huffman weights The serie of weights is compressed using FSE compression. It's a single bitstream with 2 interleaved states, @@ -590,7 +590,7 @@ and last symbol value is not represented. An FSE bitstream starts by a header, describing probabilities distribution. It will create a Decoding Table. Table must be pre-allocated, which requires to support a maximum accuracy. -For a list of huffman weights, maximum accuracy is 7 bits. +For a list of Huffman weights, maximum accuracy is 7 bits. FSE header is [described in relevant chapter](#fse-distribution-table--condensed-format), and so is [FSE bitstream](#bitstream). @@ -602,7 +602,7 @@ by tracking bitStream overflow condition. When both states have overflowed the bitstream, end is reached. -##### Conversion from weights to huffman prefix codes +##### Conversion from weights to Huffman prefix codes All present symbols shall now have a `weight` value. It is possible to transform weights into nbBits, using this formula : @@ -634,7 +634,7 @@ it gives the following distribution : ##### Bitstreams sizes As seen in a previous paragraph, -there are 2 flavors of huffman-compressed literals : +there are 2 flavors of Huffman-compressed literals : single stream, and 4-streams. 4-streams is useful for CPU with multiple execution units and OoO operations. @@ -1144,6 +1144,6 @@ __Content__ : Where the actual dictionary content is. Version changes --------------- - 0.2.0 : numerous format adjustments for zstd v0.8 -- 0.1.2 : limit huffman tree depth to 11 bits +- 0.1.2 : limit Huffman tree depth to 11 bits - 0.1.1 : reserved dictID ranges - 0.1.0 : initial release From f896c1d7175c03bab533bd7e391aa152a4b09c22 Mon Sep 17 00:00:00 2001 From: inikep Date: Wed, 3 Aug 2016 16:37:42 +0200 Subject: [PATCH 10/27] zstd_compression_format.md: improved formatting --- zstd_compression_format.md | 65 +++++++++++++++++++------------------- 1 file changed, 33 insertions(+), 32 deletions(-) diff --git a/zstd_compression_format.md b/zstd_compression_format.md index b6ace361..7357e56f 100644 --- a/zstd_compression_format.md +++ b/zstd_compression_format.md @@ -403,8 +403,8 @@ in order to properly allocate destination buffer. See [`Data_Block`](#the-structure-of-data_block) for more details. A compressed block consists of 2 sections : -- [Literals section](#literals-section) -- [Sequences section](#sequences-section) +- [Literals_Section](#literals_section) +- [Sequences_Section](#sequences_section) ### Prerequisites To decode a compressed block, the following elements are necessary : @@ -415,91 +415,92 @@ To decode a compressed block, the following elements are necessary : (literals, litLength, matchLength, offset). -### Literals section +### `Literals_Section` During sequence phase, literals will be entangled with match copy operations. All literals are regrouped in the first part of the block. They can be decoded first, and then copied during sequence operations, or they can be decoded on the flow, as needed by sequence commands. -| Literals section header | [Huffman Tree Description] | Stream1 | [Stream2] | [Stream3] | [Stream4] | -| ----------------------- | -------------------------- | ------- | --------- | --------- | --------- | +| `Literals_Section_Header` | [Huffman Tree Description] | Stream1 | [Stream2] | [Stream3] | [Stream4] | +| ------------------------- | -------------------------- | ------- | --------- | --------- | --------- | Literals can be stored uncompressed or compressed using Huffman prefix codes. When compressed, an optional tree description can be present, followed by 1 or 4 streams. -#### Literals section header +#### `Literals_Section_Header` Header is in charge of describing how literals are packed. It's a byte-aligned variable-size bitfield, ranging from 1 to 5 bytes, using little-endian convention. -| Literals Block Type | sizes format | regenerated size | [compressed size] | -| ------------------- | ------------ | ---------------- | ----------------- | -| 2 bits | 1 - 2 bits | 5 - 20 bits | 0 - 18 bits | +| `Literals_Block_Type` | `Size_Format` | `Regenerated_Size` | [`Compressed_Size`] | +| --------------------- | ------------- | ------------------ | ----------------- | +| 2 bits | 1 - 2 bits | 5 - 20 bits | 0 - 18 bits | In this representation, bits on the left are smallest bits. -__Literals Block Type__ : +__`Literals_Block_Type`__ : This field uses 2 lowest bits of first byte, describing 4 different block types : -| Value | 0 | 1 | 2 | 3 | -| ------------------- | --- | --- | ---------- | ----------- | -| Literals Block Type | Raw | RLE | Compressed | RepeatStats | +| Value | 0 | 1 | 2 | 3 | +| --------------------- | --- | --- | ---------- | ----------- | +| `Literals_Block_Type` | Raw | RLE | Compressed | RepeatStats | -- Raw literals block - Literals are stored uncompressed. -- RLE literals block - Literals consist of a single byte value repeated N times. -- Compressed literals block - This is a standard Huffman-compressed block, +- `Raw_Literals_Block` - Literals are stored uncompressed. +- `RLE_Literals_Block` - Literals consist of a single byte value repeated N times. +- `Compressed_Literals_Block` - This is a standard Huffman-compressed block, starting with a Huffman tree description. See details below. -- Repeat Stats literals block - This is a Huffman-compressed block, +- `Repeat_Stats_Literals_Block` - This is a Huffman-compressed block, using Huffman tree _from previous Huffman-compressed literals block_. Huffman tree description will be skipped. -__Sizes format__ : +__`Size_Format`__ : -Sizes format are divided into 2 families : +`Size_Format` is divided into 2 families : -- For compressed block, it requires to decode both the compressed size - and the decompressed size. It will also decode the number of streams. -- For Raw or RLE blocks, it's enough to decode the size to regenerate. +- For `Compressed_Block`, it requires to decode both `Compressed_Size` + and `Regenerated_Size` (the decompressed size). It will also decode the number of streams. +- For `Raw_Block` and `RLE_Block` it's enough to decode `Regenerated_Size`. For values spanning several bytes, convention is Little-endian. -__Sizes format for Raw and RLE literals block__ : +__Sizes format for `Raw_Literals_Block` and `RLE_Literals_Block`__ : -- Value : x0 : Regenerated size uses 5 bits (0-31). +- Value : x0 : `Regenerated_Size` uses 5 bits (0-31). Total literal header size is 1 byte. `size = h[0]>>3;` -- Value : 01 : Regenerated size uses 12 bits (0-4095). +- Value : 01 : `Regenerated_Size` uses 12 bits (0-4095). Total literal header size is 2 bytes. `size = (h[0]>>4) + (h[1]<<4);` -- Value : 11 : Regenerated size uses 20 bits (0-1048575). +- Value : 11 : `Regenerated_Size` uses 20 bits (0-1048575). Total literal header size is 3 bytes. `size = (h[0]>>4) + (h[1]<<4) + (h[2]<<12);` Note : it's allowed to represent a short value (ex : `13`) using a long format, accepting the reduced compacity. -__Sizes format for Compressed literals block and Repeat Stats literals block__ : +__Sizes format for `Compressed_Literals_Block` and `Repeat_Stats_Literals_Block`__ : - Value : 00 : _Single stream_. - Compressed and regenerated sizes use 10 bits (0-1023). + `Compressed_Size` and `Regenerated_Size` use 10 bits (0-1023). Total literal header size is 3 bytes. - Value : 01 : 4 streams. - Compressed and regenerated sizes use 10 bits (0-1023). + `Compressed_Size` and `Regenerated_Size` use 10 bits (0-1023). Total literal header size is 3 bytes. - Value : 10 : 4 streams. - Compressed and regenerated sizes use 14 bits (0-16383). + `Compressed_Size` and `Regenerated_Size` use 14 bits (0-16383). Total literal header size is 4 bytes. - Value : 11 : 4 streams. - Compressed and regenerated sizes use 18 bits (0-262143). + `Compressed_Size` and `Regenerated_Size` use 18 bits (0-262143). Total literal header size is 5 bytes. -Compressed and regenerated size fields follow little-endian convention. +`Compressed_Size` and `Regenerated_Size` size fields follow little-endian convention. + #### Huffman Tree description From 9d003c1d4c13e2acea70fa018f136933fb0b66db Mon Sep 17 00:00:00 2001 From: inikep Date: Thu, 4 Aug 2016 10:41:49 +0200 Subject: [PATCH 11/27] .travis.yml: joined 7 smaller tests into 2 bigger --- .travis.yml | 17 ++--------------- zstd_compression_format.md | 14 +++++++------- 2 files changed, 9 insertions(+), 22 deletions(-) diff --git a/.travis.yml b/.travis.yml index 6cd3f5f6..d160e36a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -31,16 +31,7 @@ matrix: env: PLATFORM="Ubuntu 12.04" CMD="make libc6install clangtest" - os: linux sudo: required - env: PLATFORM="Ubuntu 12.04" CMD="make gppinstall gpptest" - - os: linux - sudo: required - env: PLATFORM="Ubuntu 12.04" CMD="make libc6install gnu90test" - - os: linux - sudo: required - env: PLATFORM="Ubuntu 12.04" CMD="make libc6install c99test" - - os: linux - sudo: required - env: PLATFORM="Ubuntu 12.04" CMD="make libc6install gnu99test" + env: PLATFORM="Ubuntu 12.04" CMD="make gppinstall gpptest && make clean && make libc6install gnu90test && make clean && make libc6install c99test && make clean && make libc6install gnu99test" - os: linux sudo: required env: PLATFORM="Ubuntu 12.04" CMD="make arminstall armtest" @@ -75,11 +66,7 @@ matrix: env: PLATFORM="Ubuntu 14.04" CMD="make zlibwrapper" # OS X Mavericks - os: osx - env: PLATFORM="OS X Mavericks" CMD="make travis-install" - - os: osx - env: PLATFORM="OS X Mavericks" CMD="make gnu90test" - - os: osx - env: PLATFORM="OS X Mavericks" CMD="make test" + env: PLATFORM="OS X Mavericks" CMD="make gnu90test && make clean && make gnu90test && make clean && make travis-install" exclude: - compiler: gcc diff --git a/zstd_compression_format.md b/zstd_compression_format.md index 7357e56f..f35e8711 100644 --- a/zstd_compression_format.md +++ b/zstd_compression_format.md @@ -442,13 +442,13 @@ using little-endian convention. In this representation, bits on the left are smallest bits. -__`Literals_Block_Type`__ : +__`Literals_Block_Type`__ This field uses 2 lowest bits of first byte, describing 4 different block types : -| Value | 0 | 1 | 2 | 3 | -| --------------------- | --- | --- | ---------- | ----------- | -| `Literals_Block_Type` | Raw | RLE | Compressed | RepeatStats | +| Value | 0 | 1 | 2 | 3 | +| --------------------- | -------------------- | -------------------- | --------------------------- | ----------------------------- | +| `Literals_Block_Type` | `Raw_Literals_Block` | `RLE_Literals_Block` | `Compressed_Literals_Block` | `Repeat_Stats_Literals_Block` | - `Raw_Literals_Block` - Literals are stored uncompressed. - `RLE_Literals_Block` - Literals consist of a single byte value repeated N times. @@ -459,7 +459,7 @@ This field uses 2 lowest bits of first byte, describing 4 different block types using Huffman tree _from previous Huffman-compressed literals block_. Huffman tree description will be skipped. -__`Size_Format`__ : +__`Size_Format`__ `Size_Format` is divided into 2 families : @@ -469,7 +469,7 @@ __`Size_Format`__ : For values spanning several bytes, convention is Little-endian. -__Sizes format for `Raw_Literals_Block` and `RLE_Literals_Block`__ : +__`Size_Format` for `Raw_Literals_Block` and `RLE_Literals_Block`__ : - Value : x0 : `Regenerated_Size` uses 5 bits (0-31). Total literal header size is 1 byte. @@ -484,7 +484,7 @@ __Sizes format for `Raw_Literals_Block` and `RLE_Literals_Block`__ : Note : it's allowed to represent a short value (ex : `13`) using a long format, accepting the reduced compacity. -__Sizes format for `Compressed_Literals_Block` and `Repeat_Stats_Literals_Block`__ : +__`Size_Format` for `Compressed_Literals_Block` and `Repeat_Stats_Literals_Block`__ : - Value : 00 : _Single stream_. `Compressed_Size` and `Regenerated_Size` use 10 bits (0-1023). From 4f270ac851c8385643e491aa45049fd478fabbfa Mon Sep 17 00:00:00 2001 From: inikep Date: Thu, 4 Aug 2016 11:25:52 +0200 Subject: [PATCH 12/27] .travis.yml: clang test joined into bigger test --- .travis.yml | 7 ++----- zlibWrapper/zstd_zlibwrapper.c | 5 +++-- zstd_compression_format.md | 12 ++++++------ 3 files changed, 11 insertions(+), 13 deletions(-) diff --git a/.travis.yml b/.travis.yml index d160e36a..2e4611ca 100644 --- a/.travis.yml +++ b/.travis.yml @@ -28,10 +28,7 @@ matrix: # Standard Ubuntu 12.04 LTS Server Edition 64 bit - os: linux sudo: required - env: PLATFORM="Ubuntu 12.04" CMD="make libc6install clangtest" - - os: linux - sudo: required - env: PLATFORM="Ubuntu 12.04" CMD="make gppinstall gpptest && make clean && make libc6install gnu90test && make clean && make libc6install c99test && make clean && make libc6install gnu99test" + env: PLATFORM="Ubuntu 12.04" CMD="make gppinstall gpptest && make clean && make libc6install gnu90test && make clean && make c99test && make clean && make gnu99test && make clean && make clangtest" - os: linux sudo: required env: PLATFORM="Ubuntu 12.04" CMD="make arminstall armtest" @@ -66,7 +63,7 @@ matrix: env: PLATFORM="Ubuntu 14.04" CMD="make zlibwrapper" # OS X Mavericks - os: osx - env: PLATFORM="OS X Mavericks" CMD="make gnu90test && make clean && make gnu90test && make clean && make travis-install" + env: PLATFORM="OS X Mavericks" CMD="make gnu90test && make clean && make test && make clean && make travis-install" exclude: - compiler: gcc diff --git a/zlibWrapper/zstd_zlibwrapper.c b/zlibWrapper/zstd_zlibwrapper.c index 06667b61..34de73a6 100644 --- a/zlibWrapper/zstd_zlibwrapper.c +++ b/zlibWrapper/zstd_zlibwrapper.c @@ -29,14 +29,15 @@ - zstd source repository : https://github.com/Cyan4973/zstd */ -#include /* va_list, for z_gzprintf */ +#include /* vsprintf */ +#include /* va_list, for z_gzprintf */ #include #include "zstd_zlibwrapper.h" #define ZSTD_STATIC_LINKING_ONLY /* ZSTD_MAGICNUMBER */ #include "zstd.h" #define ZBUFF_STATIC_LINKING_ONLY /* ZBUFF_createCCtx_advanced */ #include "zbuff.h" -#include "zstd_internal.h" /* defaultCustomMem */ +#include "zstd_internal.h" /* defaultCustomMem */ #define Z_INFLATE_SYNC 8 diff --git a/zstd_compression_format.md b/zstd_compression_format.md index f35e8711..1c5908fa 100644 --- a/zstd_compression_format.md +++ b/zstd_compression_format.md @@ -422,8 +422,8 @@ All literals are regrouped in the first part of the block. They can be decoded first, and then copied during sequence operations, or they can be decoded on the flow, as needed by sequence commands. -| `Literals_Section_Header` | [Huffman Tree Description] | Stream1 | [Stream2] | [Stream3] | [Stream4] | -| ------------------------- | -------------------------- | ------- | --------- | --------- | --------- | +| `Literals_Section_Header` | [`Huffman_Tree_Description`] | Stream1 | [Stream2] | [Stream3] | [Stream4] | +| ------------------------- | ---------------------------- | ------- | --------- | --------- | --------- | Literals can be stored uncompressed or compressed using Huffman prefix codes. When compressed, an optional tree description can be present, @@ -499,12 +499,12 @@ __`Size_Format` for `Compressed_Literals_Block` and `Repeat_Stats_Literals_Block `Compressed_Size` and `Regenerated_Size` use 18 bits (0-262143). Total literal header size is 5 bytes. -`Compressed_Size` and `Regenerated_Size` size fields follow little-endian convention. +`Compressed_Size` and `Regenerated_Size` fields follow little-endian convention. -#### Huffman Tree description +#### `Huffman_Tree_Description` -This section is only present when literals block type is `Compressed` (`0`). +This section is only present when literals block type is `Compressed_Block` (`2`). Prefix coding represents symbols from an a priori known alphabet by bit sequences (codewords), one codeword for each symbol, @@ -686,7 +686,7 @@ hence reaching exactly its beginning position with _all_ bits consumed, the decoding process is considered faulty. -### Sequences section +### `Sequences_Section` A compressed block is a succession of _sequences_ . A sequence is a literal copy command, followed by a match copy command. From 0079425297fce6e25d674615c25596daabf590e7 Mon Sep 17 00:00:00 2001 From: inikep Date: Thu, 4 Aug 2016 14:43:21 +0200 Subject: [PATCH 13/27] .travis.yml: tests merged together --- .travis.yml | 27 +++------------------------ zstd_compression_format.md | 6 +++--- 2 files changed, 6 insertions(+), 27 deletions(-) diff --git a/.travis.yml b/.travis.yml index 2e4611ca..22271cde 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,25 +6,16 @@ matrix: # Container-based Ubuntu 12.04 LTS Server Edition 64 bit (doesn't support 32-bit includes) - os: linux sudo: false - env: PLATFORM="Ubuntu 12.04 container" CMD="make travis-install" + env: PLATFORM="Ubuntu 12.04 container" CMD="make test && make clean && make travis-install" - os: linux sudo: false - env: PLATFORM="Ubuntu 12.04 container" CMD="make cmaketest" - - os: linux - sudo: false - env: PLATFORM="Ubuntu 12.04 container" CMD="make test" - - os: linux - sudo: false - env: PLATFORM="Ubuntu 12.04 container" CMD="make -C programs test-zstd_nolegacy" + env: PLATFORM="Ubuntu 12.04 container" CMD="make -C programs test-zstd_nolegacy && make clean && make zlibwrapper && make clean && make cmaketest" - os: linux sudo: false env: PLATFORM="Ubuntu 12.04 container" CMD="make usan" - os: linux sudo: false env: PLATFORM="Ubuntu 12.04 container" CMD="make asan" - - os: linux - sudo: false - env: PLATFORM="Ubuntu 12.04 container" CMD="make zlibwrapper" # Standard Ubuntu 12.04 LTS Server Edition 64 bit - os: linux sudo: required @@ -48,19 +39,7 @@ matrix: - os: linux dist: trusty sudo: required - env: PLATFORM="Ubuntu 14.04" CMD="make gcc5install gcc5test" - - os: linux - dist: trusty - sudo: required - env: PLATFORM="Ubuntu 14.04" CMD="make gcc6install gcc6test" - - os: linux - dist: trusty - sudo: required - env: PLATFORM="Ubuntu 14.04" CMD="make ppcinstall ppctest" - - os: linux - dist: trusty - sudo: required - env: PLATFORM="Ubuntu 14.04" CMD="make zlibwrapper" + env: PLATFORM="Ubuntu 14.04" CMD="make zlibwrapper && make clean && make gcc5install gcc5test && make clean && make gcc6install gcc6test && make clean && make ppcinstall ppctest" # OS X Mavericks - os: osx env: PLATFORM="OS X Mavericks" CMD="make gnu90test && make clean && make test && make clean && make travis-install" diff --git a/zstd_compression_format.md b/zstd_compression_format.md index 1c5908fa..a04730be 100644 --- a/zstd_compression_format.md +++ b/zstd_compression_format.md @@ -403,8 +403,8 @@ in order to properly allocate destination buffer. See [`Data_Block`](#the-structure-of-data_block) for more details. A compressed block consists of 2 sections : -- [Literals_Section](#literals_section) -- [Sequences_Section](#sequences_section) +- [`Literals_Section`](#literals_section) +- [`Sequences_Section`](#sequences_section) ### Prerequisites To decode a compressed block, the following elements are necessary : @@ -504,7 +504,7 @@ __`Size_Format` for `Compressed_Literals_Block` and `Repeat_Stats_Literals_Block #### `Huffman_Tree_Description` -This section is only present when literals block type is `Compressed_Block` (`2`). +This section is only present when `Literals_Block_Type` type is `Compressed_Block` (`2`). Prefix coding represents symbols from an a priori known alphabet by bit sequences (codewords), one codeword for each symbol, From 35605fa54696f5a7ca59ff7ccbef95226d21247f Mon Sep 17 00:00:00 2001 From: inikep Date: Thu, 4 Aug 2016 16:58:13 +0200 Subject: [PATCH 14/27] appveyor.yml: test C++ compilation with Visual Studio --- appveyor.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/appveyor.yml b/appveyor.yml index 4f938120..eb6e709e 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -61,26 +61,31 @@ build_script: ECHO *** && ECHO *** Building Visual Studio 2008 %PLATFORM%\%CONFIGURATION% && ECHO *** && + msbuild "projects\VS2008\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v90 /p:CompileAs=CompileAsCpp /t:Clean,Build /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && msbuild "projects\VS2008\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v90 /t:Clean,Build /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && COPY projects\VS2008\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe projects\fuzzer_VS2008_%PLATFORM%_%CONFIGURATION%.exe && ECHO *** && ECHO *** Building Visual Studio 2010 %PLATFORM%\%CONFIGURATION% && ECHO *** && + msbuild "projects\VS2010\zstd.sln" %ADDITIONALPARAM% /m /verbosity:minimal /property:PlatformToolset=v100 /p:CompileAs=CompileAsCpp /t:Clean,Build /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && msbuild "projects\VS2010\zstd.sln" %ADDITIONALPARAM% /m /verbosity:minimal /property:PlatformToolset=v100 /t:Clean,Build /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe projects\fuzzer_VS2010_%PLATFORM%_%CONFIGURATION%.exe && ECHO *** && ECHO *** Building Visual Studio 2012 %PLATFORM%\%CONFIGURATION% && ECHO *** && + msbuild "projects\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v110 /p:CompileAs=CompileAsCpp /t:Clean,Build /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && msbuild "projects\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v110 /t:Clean,Build /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe projects\fuzzer_VS2012_%PLATFORM%_%CONFIGURATION%.exe && ECHO *** && ECHO *** Building Visual Studio 2013 %PLATFORM%\%CONFIGURATION% && ECHO *** && + msbuild "projects\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v120 /p:CompileAs=CompileAsCpp /t:Clean,Build /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && msbuild "projects\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v120 /t:Clean,Build /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe projects\fuzzer_VS2013_%PLATFORM%_%CONFIGURATION%.exe && ECHO *** && ECHO *** Building Visual Studio 2015 %PLATFORM%\%CONFIGURATION% && ECHO *** && + msbuild "projects\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v140 /p:CompileAs=CompileAsCpp /t:Clean,Build /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && msbuild "projects\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v140 /t:Clean,Build /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe projects\fuzzer_VS2015_%PLATFORM%_%CONFIGURATION%.exe && COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\*.exe programs\ From 468191ce4afa353213edb7d0811c115b8daa2e36 Mon Sep 17 00:00:00 2001 From: inikep Date: Thu, 4 Aug 2016 17:30:10 +0200 Subject: [PATCH 15/27] appveyor.yml: list generated executables --- appveyor.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/appveyor.yml b/appveyor.yml index eb6e709e..edc8bd33 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -62,13 +62,18 @@ build_script: ECHO *** Building Visual Studio 2008 %PLATFORM%\%CONFIGURATION% && ECHO *** && msbuild "projects\VS2008\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v90 /p:CompileAs=CompileAsCpp /t:Clean,Build /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && + DIR projects\VS2008\bin\%PLATFORM%\%CONFIGURATION%\*.exe && msbuild "projects\VS2008\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v90 /t:Clean,Build /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && + DIR projects\VS2008\bin\%PLATFORM%\%CONFIGURATION%\*.exe && COPY projects\VS2008\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe projects\fuzzer_VS2008_%PLATFORM%_%CONFIGURATION%.exe && ECHO *** && ECHO *** Building Visual Studio 2010 %PLATFORM%\%CONFIGURATION% && ECHO *** && msbuild "projects\VS2010\zstd.sln" %ADDITIONALPARAM% /m /verbosity:minimal /property:PlatformToolset=v100 /p:CompileAs=CompileAsCpp /t:Clean,Build /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && + DIR projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\*.exe && msbuild "projects\VS2010\zstd.sln" %ADDITIONALPARAM% /m /verbosity:minimal /property:PlatformToolset=v100 /t:Clean,Build /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && + DIR projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\*.exe && + MD5sum projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\*.exe && COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe projects\fuzzer_VS2010_%PLATFORM%_%CONFIGURATION%.exe && ECHO *** && ECHO *** Building Visual Studio 2012 %PLATFORM%\%CONFIGURATION% && From b918a6e23939354ade874bf9b43ecf16b39c76e5 Mon Sep 17 00:00:00 2001 From: inikep Date: Thu, 4 Aug 2016 18:19:16 +0200 Subject: [PATCH 16/27] appveyor.yml: list generated executables (2) --- appveyor.yml | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index edc8bd33..4136ef56 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -63,35 +63,50 @@ build_script: ECHO *** && msbuild "projects\VS2008\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v90 /p:CompileAs=CompileAsCpp /t:Clean,Build /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && DIR projects\VS2008\bin\%PLATFORM%\%CONFIGURATION%\*.exe && + MD5sum projects/VS2008/bin/%PLATFORM%/%CONFIGURATION%/*.exe && msbuild "projects\VS2008\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v90 /t:Clean,Build /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && DIR projects\VS2008\bin\%PLATFORM%\%CONFIGURATION%\*.exe && + MD5sum projects/VS2008/bin/%PLATFORM%/%CONFIGURATION%/*.exe && COPY projects\VS2008\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe projects\fuzzer_VS2008_%PLATFORM%_%CONFIGURATION%.exe && ECHO *** && ECHO *** Building Visual Studio 2010 %PLATFORM%\%CONFIGURATION% && ECHO *** && msbuild "projects\VS2010\zstd.sln" %ADDITIONALPARAM% /m /verbosity:minimal /property:PlatformToolset=v100 /p:CompileAs=CompileAsCpp /t:Clean,Build /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && DIR projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\*.exe && + MD5sum projects/VS2010/bin/%PLATFORM%/%CONFIGURATION%/*.exe && msbuild "projects\VS2010\zstd.sln" %ADDITIONALPARAM% /m /verbosity:minimal /property:PlatformToolset=v100 /t:Clean,Build /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && DIR projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\*.exe && - MD5sum projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\*.exe && + MD5sum projects/VS2010/bin/%PLATFORM%/%CONFIGURATION%/*.exe && COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe projects\fuzzer_VS2010_%PLATFORM%_%CONFIGURATION%.exe && ECHO *** && ECHO *** Building Visual Studio 2012 %PLATFORM%\%CONFIGURATION% && ECHO *** && msbuild "projects\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v110 /p:CompileAs=CompileAsCpp /t:Clean,Build /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && + DIR projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\*.exe && + MD5sum projects/VS2010/bin/%PLATFORM%/%CONFIGURATION%/*.exe && msbuild "projects\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v110 /t:Clean,Build /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && + DIR projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\*.exe && + MD5sum projects/VS2010/bin/%PLATFORM%/%CONFIGURATION%/*.exe && COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe projects\fuzzer_VS2012_%PLATFORM%_%CONFIGURATION%.exe && ECHO *** && ECHO *** Building Visual Studio 2013 %PLATFORM%\%CONFIGURATION% && ECHO *** && msbuild "projects\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v120 /p:CompileAs=CompileAsCpp /t:Clean,Build /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && + DIR projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\*.exe && + MD5sum projects/VS2010/bin/%PLATFORM%/%CONFIGURATION%/*.exe && msbuild "projects\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v120 /t:Clean,Build /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && + DIR projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\*.exe && + MD5sum projects/VS2010/bin/%PLATFORM%/%CONFIGURATION%/*.exe && COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe projects\fuzzer_VS2013_%PLATFORM%_%CONFIGURATION%.exe && ECHO *** && ECHO *** Building Visual Studio 2015 %PLATFORM%\%CONFIGURATION% && ECHO *** && msbuild "projects\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v140 /p:CompileAs=CompileAsCpp /t:Clean,Build /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && + DIR projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\*.exe && + MD5sum projects/VS2010/bin/%PLATFORM%/%CONFIGURATION%/*.exe && msbuild "projects\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v140 /t:Clean,Build /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && + DIR projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\*.exe && + MD5sum projects/VS2010/bin/%PLATFORM%/%CONFIGURATION%/*.exe && COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe projects\fuzzer_VS2015_%PLATFORM%_%CONFIGURATION%.exe && COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\*.exe programs\ ) From 4f5f301a4ddb70f49140e0f536e3a284c4c644ab Mon Sep 17 00:00:00 2001 From: inikep Date: Thu, 4 Aug 2016 18:34:15 +0200 Subject: [PATCH 17/27] appveyor.yml: fixed missing /p:Platform and /p:Configuration --- appveyor.yml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 4136ef56..75040f40 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -61,50 +61,50 @@ build_script: ECHO *** && ECHO *** Building Visual Studio 2008 %PLATFORM%\%CONFIGURATION% && ECHO *** && - msbuild "projects\VS2008\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v90 /p:CompileAs=CompileAsCpp /t:Clean,Build /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && + msbuild "projects\VS2008\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v90 /p:CompileAs=CompileAsCpp /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && DIR projects\VS2008\bin\%PLATFORM%\%CONFIGURATION%\*.exe && MD5sum projects/VS2008/bin/%PLATFORM%/%CONFIGURATION%/*.exe && - msbuild "projects\VS2008\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v90 /t:Clean,Build /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && + msbuild "projects\VS2008\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v90 /p:CompileAs=CompileAsC /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && DIR projects\VS2008\bin\%PLATFORM%\%CONFIGURATION%\*.exe && MD5sum projects/VS2008/bin/%PLATFORM%/%CONFIGURATION%/*.exe && COPY projects\VS2008\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe projects\fuzzer_VS2008_%PLATFORM%_%CONFIGURATION%.exe && ECHO *** && ECHO *** Building Visual Studio 2010 %PLATFORM%\%CONFIGURATION% && ECHO *** && - msbuild "projects\VS2010\zstd.sln" %ADDITIONALPARAM% /m /verbosity:minimal /property:PlatformToolset=v100 /p:CompileAs=CompileAsCpp /t:Clean,Build /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && + msbuild "projects\VS2010\zstd.sln" %ADDITIONALPARAM% /m /verbosity:minimal /property:PlatformToolset=v100 /p:CompileAs=CompileAsCpp /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && DIR projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\*.exe && MD5sum projects/VS2010/bin/%PLATFORM%/%CONFIGURATION%/*.exe && - msbuild "projects\VS2010\zstd.sln" %ADDITIONALPARAM% /m /verbosity:minimal /property:PlatformToolset=v100 /t:Clean,Build /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && + msbuild "projects\VS2010\zstd.sln" %ADDITIONALPARAM% /m /verbosity:minimal /property:PlatformToolset=v100 /p:CompileAs=CompileAsC /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && DIR projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\*.exe && MD5sum projects/VS2010/bin/%PLATFORM%/%CONFIGURATION%/*.exe && COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe projects\fuzzer_VS2010_%PLATFORM%_%CONFIGURATION%.exe && ECHO *** && ECHO *** Building Visual Studio 2012 %PLATFORM%\%CONFIGURATION% && ECHO *** && - msbuild "projects\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v110 /p:CompileAs=CompileAsCpp /t:Clean,Build /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && + msbuild "projects\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v110 /p:CompileAs=CompileAsCpp /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && DIR projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\*.exe && MD5sum projects/VS2010/bin/%PLATFORM%/%CONFIGURATION%/*.exe && - msbuild "projects\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v110 /t:Clean,Build /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && + msbuild "projects\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v110 /p:CompileAs=CompileAsC /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && DIR projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\*.exe && MD5sum projects/VS2010/bin/%PLATFORM%/%CONFIGURATION%/*.exe && COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe projects\fuzzer_VS2012_%PLATFORM%_%CONFIGURATION%.exe && ECHO *** && ECHO *** Building Visual Studio 2013 %PLATFORM%\%CONFIGURATION% && ECHO *** && - msbuild "projects\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v120 /p:CompileAs=CompileAsCpp /t:Clean,Build /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && + msbuild "projects\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v120 /p:CompileAs=CompileAsCpp /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && DIR projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\*.exe && MD5sum projects/VS2010/bin/%PLATFORM%/%CONFIGURATION%/*.exe && - msbuild "projects\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v120 /t:Clean,Build /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && + msbuild "projects\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v120 /p:CompileAs=CompileAsC /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && DIR projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\*.exe && MD5sum projects/VS2010/bin/%PLATFORM%/%CONFIGURATION%/*.exe && COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe projects\fuzzer_VS2013_%PLATFORM%_%CONFIGURATION%.exe && ECHO *** && ECHO *** Building Visual Studio 2015 %PLATFORM%\%CONFIGURATION% && ECHO *** && - msbuild "projects\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v140 /p:CompileAs=CompileAsCpp /t:Clean,Build /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && + msbuild "projects\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v140 /p:CompileAs=CompileAsCpp /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && DIR projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\*.exe && MD5sum projects/VS2010/bin/%PLATFORM%/%CONFIGURATION%/*.exe && - msbuild "projects\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v140 /t:Clean,Build /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && + msbuild "projects\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v140 /p:CompileAs=CompileAsC /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && DIR projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\*.exe && MD5sum projects/VS2010/bin/%PLATFORM%/%CONFIGURATION%/*.exe && COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe projects\fuzzer_VS2015_%PLATFORM%_%CONFIGURATION%.exe && From 28f85a6b20efcd9ec05e44d180c84d8b1c671afa Mon Sep 17 00:00:00 2001 From: inikep Date: Fri, 5 Aug 2016 10:12:32 +0200 Subject: [PATCH 18/27] added CompileAsCpp.props --- appveyor.yml | 14 ++++++++------ projects/VS2010/CompileAsCpp.props | 8 ++++++++ 2 files changed, 16 insertions(+), 6 deletions(-) create mode 100644 projects/VS2010/CompileAsCpp.props diff --git a/appveyor.yml b/appveyor.yml index 75040f40..af8a1ad5 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -61,10 +61,12 @@ build_script: ECHO *** && ECHO *** Building Visual Studio 2008 %PLATFORM%\%CONFIGURATION% && ECHO *** && - msbuild "projects\VS2008\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v90 /p:CompileAs=CompileAsCpp /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && + ECHO %APPVEYOR_BUILD_FOLDER% + CD + msbuild "projects\VS2008\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v90 /p:ForceImportBeforeCppTargets=%APPVEYOR_BUILD_FOLDER%\projects\VS2010\CompileAsCpp.props /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && DIR projects\VS2008\bin\%PLATFORM%\%CONFIGURATION%\*.exe && MD5sum projects/VS2008/bin/%PLATFORM%/%CONFIGURATION%/*.exe && - msbuild "projects\VS2008\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v90 /p:CompileAs=CompileAsC /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && + msbuild "projects\VS2008\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v90 /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && DIR projects\VS2008\bin\%PLATFORM%\%CONFIGURATION%\*.exe && MD5sum projects/VS2008/bin/%PLATFORM%/%CONFIGURATION%/*.exe && COPY projects\VS2008\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe projects\fuzzer_VS2008_%PLATFORM%_%CONFIGURATION%.exe && @@ -74,7 +76,7 @@ build_script: msbuild "projects\VS2010\zstd.sln" %ADDITIONALPARAM% /m /verbosity:minimal /property:PlatformToolset=v100 /p:CompileAs=CompileAsCpp /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && DIR projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\*.exe && MD5sum projects/VS2010/bin/%PLATFORM%/%CONFIGURATION%/*.exe && - msbuild "projects\VS2010\zstd.sln" %ADDITIONALPARAM% /m /verbosity:minimal /property:PlatformToolset=v100 /p:CompileAs=CompileAsC /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && + msbuild "projects\VS2010\zstd.sln" %ADDITIONALPARAM% /m /verbosity:minimal /property:PlatformToolset=v100 /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && DIR projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\*.exe && MD5sum projects/VS2010/bin/%PLATFORM%/%CONFIGURATION%/*.exe && COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe projects\fuzzer_VS2010_%PLATFORM%_%CONFIGURATION%.exe && @@ -84,7 +86,7 @@ build_script: msbuild "projects\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v110 /p:CompileAs=CompileAsCpp /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && DIR projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\*.exe && MD5sum projects/VS2010/bin/%PLATFORM%/%CONFIGURATION%/*.exe && - msbuild "projects\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v110 /p:CompileAs=CompileAsC /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && + msbuild "projects\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v110 /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && DIR projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\*.exe && MD5sum projects/VS2010/bin/%PLATFORM%/%CONFIGURATION%/*.exe && COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe projects\fuzzer_VS2012_%PLATFORM%_%CONFIGURATION%.exe && @@ -94,7 +96,7 @@ build_script: msbuild "projects\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v120 /p:CompileAs=CompileAsCpp /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && DIR projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\*.exe && MD5sum projects/VS2010/bin/%PLATFORM%/%CONFIGURATION%/*.exe && - msbuild "projects\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v120 /p:CompileAs=CompileAsC /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && + msbuild "projects\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v120 /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && DIR projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\*.exe && MD5sum projects/VS2010/bin/%PLATFORM%/%CONFIGURATION%/*.exe && COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe projects\fuzzer_VS2013_%PLATFORM%_%CONFIGURATION%.exe && @@ -104,7 +106,7 @@ build_script: msbuild "projects\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v140 /p:CompileAs=CompileAsCpp /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && DIR projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\*.exe && MD5sum projects/VS2010/bin/%PLATFORM%/%CONFIGURATION%/*.exe && - msbuild "projects\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v140 /p:CompileAs=CompileAsC /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && + msbuild "projects\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v140 /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && DIR projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\*.exe && MD5sum projects/VS2010/bin/%PLATFORM%/%CONFIGURATION%/*.exe && COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe projects\fuzzer_VS2015_%PLATFORM%_%CONFIGURATION%.exe && diff --git a/projects/VS2010/CompileAsCpp.props b/projects/VS2010/CompileAsCpp.props new file mode 100644 index 00000000..372a94b1 --- /dev/null +++ b/projects/VS2010/CompileAsCpp.props @@ -0,0 +1,8 @@ + + + + + CompileAsCpp + + + From 1805d3a5057efe5af709bcc6eb87f9f9f547026a Mon Sep 17 00:00:00 2001 From: inikep Date: Fri, 5 Aug 2016 11:03:49 +0200 Subject: [PATCH 19/27] appveyor.yml: test C++ compilation with Visual Studio --- appveyor.yml | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index af8a1ad5..1ee8cf8f 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -59,11 +59,9 @@ build_script: ) - if [%COMPILER%]==[visual] ( ECHO *** && - ECHO *** Building Visual Studio 2008 %PLATFORM%\%CONFIGURATION% && + ECHO *** Building Visual Studio 2008 %PLATFORM%\%CONFIGURATION% in %APPVEYOR_BUILD_FOLDER% && ECHO *** && - ECHO %APPVEYOR_BUILD_FOLDER% - CD - msbuild "projects\VS2008\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v90 /p:ForceImportBeforeCppTargets=%APPVEYOR_BUILD_FOLDER%\projects\VS2010\CompileAsCpp.props /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && + msbuild "projects\VS2008\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v90 /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && DIR projects\VS2008\bin\%PLATFORM%\%CONFIGURATION%\*.exe && MD5sum projects/VS2008/bin/%PLATFORM%/%CONFIGURATION%/*.exe && msbuild "projects\VS2008\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v90 /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && @@ -73,7 +71,7 @@ build_script: ECHO *** && ECHO *** Building Visual Studio 2010 %PLATFORM%\%CONFIGURATION% && ECHO *** && - msbuild "projects\VS2010\zstd.sln" %ADDITIONALPARAM% /m /verbosity:minimal /property:PlatformToolset=v100 /p:CompileAs=CompileAsCpp /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && + msbuild "projects\VS2010\zstd.sln" %ADDITIONALPARAM% /m /verbosity:minimal /property:PlatformToolset=v100 /p:ForceImportBeforeCppTargets=%APPVEYOR_BUILD_FOLDER%\projects\VS2010\CompileAsCpp.props /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && DIR projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\*.exe && MD5sum projects/VS2010/bin/%PLATFORM%/%CONFIGURATION%/*.exe && msbuild "projects\VS2010\zstd.sln" %ADDITIONALPARAM% /m /verbosity:minimal /property:PlatformToolset=v100 /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && @@ -83,7 +81,7 @@ build_script: ECHO *** && ECHO *** Building Visual Studio 2012 %PLATFORM%\%CONFIGURATION% && ECHO *** && - msbuild "projects\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v110 /p:CompileAs=CompileAsCpp /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && + msbuild "projects\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v110 /p:ForceImportBeforeCppTargets=%APPVEYOR_BUILD_FOLDER%\projects\VS2010\CompileAsCpp.props /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && DIR projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\*.exe && MD5sum projects/VS2010/bin/%PLATFORM%/%CONFIGURATION%/*.exe && msbuild "projects\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v110 /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && @@ -93,7 +91,7 @@ build_script: ECHO *** && ECHO *** Building Visual Studio 2013 %PLATFORM%\%CONFIGURATION% && ECHO *** && - msbuild "projects\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v120 /p:CompileAs=CompileAsCpp /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && + msbuild "projects\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v120 /p:ForceImportBeforeCppTargets=%APPVEYOR_BUILD_FOLDER%\projects\VS2010\CompileAsCpp.props /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && DIR projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\*.exe && MD5sum projects/VS2010/bin/%PLATFORM%/%CONFIGURATION%/*.exe && msbuild "projects\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v120 /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && @@ -103,7 +101,7 @@ build_script: ECHO *** && ECHO *** Building Visual Studio 2015 %PLATFORM%\%CONFIGURATION% && ECHO *** && - msbuild "projects\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v140 /p:CompileAs=CompileAsCpp /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && + msbuild "projects\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v140 /p:ForceImportBeforeCppTargets=%APPVEYOR_BUILD_FOLDER%\projects\VS2010\CompileAsCpp.props /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && DIR projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\*.exe && MD5sum projects/VS2010/bin/%PLATFORM%/%CONFIGURATION%/*.exe && msbuild "projects\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v140 /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && From 2cde2d971bf017d0b1a04707ed01b87fe800fcb6 Mon Sep 17 00:00:00 2001 From: inikep Date: Fri, 5 Aug 2016 11:04:32 +0200 Subject: [PATCH 20/27] fix warnings during C++ compilation with Visual Studio --- lib/decompress/zstd_decompress.c | 1 + lib/legacy/zstd_v02.c | 4 ++++ lib/legacy/zstd_v03.c | 4 ++++ lib/legacy/zstd_v04.c | 4 ++++ lib/legacy/zstd_v06.c | 4 ++++ lib/legacy/zstd_v07.c | 1 + 6 files changed, 18 insertions(+) diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index 958d6369..052ccd60 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -77,6 +77,7 @@ # include /* For Visual 2005 */ # pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ # pragma warning(disable : 4324) /* disable: C4324: padded structure */ +# pragma warning(disable : 4100) /* disable: C4100: unreferenced formal parameter */ #else # ifdef __GNUC__ # define FORCE_INLINE static inline __attribute__((always_inline)) diff --git a/lib/legacy/zstd_v02.c b/lib/legacy/zstd_v02.c index 2d4cfa59..be847671 100644 --- a/lib/legacy/zstd_v02.c +++ b/lib/legacy/zstd_v02.c @@ -44,6 +44,10 @@ extern "C" { /****************************************** * Compiler-specific ******************************************/ +#if defined(_MSC_VER) /* Visual Studio */ +# include /* _byteswap_ulong */ +# include /* _byteswap_* */ +#endif #if defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) # define ERR_STATIC static inline #elif defined(_MSC_VER) diff --git a/lib/legacy/zstd_v03.c b/lib/legacy/zstd_v03.c index f06b47fd..86b8138e 100644 --- a/lib/legacy/zstd_v03.c +++ b/lib/legacy/zstd_v03.c @@ -44,6 +44,10 @@ extern "C" { /****************************************** * Compiler-specific ******************************************/ +#if defined(_MSC_VER) /* Visual Studio */ +# include /* _byteswap_ulong */ +# include /* _byteswap_* */ +#endif #if defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) # define ERR_STATIC static inline #elif defined(_MSC_VER) diff --git a/lib/legacy/zstd_v04.c b/lib/legacy/zstd_v04.c index 66a47e7a..9fb07caa 100644 --- a/lib/legacy/zstd_v04.c +++ b/lib/legacy/zstd_v04.c @@ -87,6 +87,10 @@ extern "C" { /****************************************** * Compiler-specific ******************************************/ +#if defined(_MSC_VER) /* Visual Studio */ +# include /* _byteswap_ulong */ +# include /* _byteswap_* */ +#endif #if defined(__GNUC__) # define MEM_STATIC static __attribute__((unused)) #elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) diff --git a/lib/legacy/zstd_v06.c b/lib/legacy/zstd_v06.c index ce6967eb..aad3d13d 100644 --- a/lib/legacy/zstd_v06.c +++ b/lib/legacy/zstd_v06.c @@ -85,6 +85,10 @@ extern "C" { /*-**************************************** * Compiler specifics ******************************************/ +#if defined(_MSC_VER) /* Visual Studio */ +# include /* _byteswap_ulong */ +# include /* _byteswap_* */ +#endif #if defined(__GNUC__) # define MEM_STATIC static __attribute__((unused)) #elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) diff --git a/lib/legacy/zstd_v07.c b/lib/legacy/zstd_v07.c index d95fd438..fb4d65b8 100644 --- a/lib/legacy/zstd_v07.c +++ b/lib/legacy/zstd_v07.c @@ -3373,6 +3373,7 @@ static const ZSTDv07_customMem defaultCustomMem = { ZSTDv07_defaultAllocFunction # include /* For Visual 2005 */ # pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ # pragma warning(disable : 4324) /* disable: C4324: padded structure */ +# pragma warning(disable : 4100) /* disable: C4100: unreferenced formal parameter */ #else # ifdef __GNUC__ # define FORCE_INLINE static inline __attribute__((always_inline)) From 3809c3b50c62ea87ced187ae515c446fc2358819 Mon Sep 17 00:00:00 2001 From: inikep Date: Fri, 5 Aug 2016 13:11:32 +0200 Subject: [PATCH 21/27] added Visual Studio 2005 project --- .gitignore | 11 - appveyor.yml | 3 - programs/datagen.c | 6 +- projects/.gitignore | 12 +- projects/VS2005/fullbench/fullbench.vcproj | 448 +++++++++++++++++ projects/VS2005/fuzzer/fuzzer.vcproj | 460 +++++++++++++++++ projects/VS2005/zstd.sln | 55 ++ projects/VS2005/zstd/zstd.vcproj | 552 +++++++++++++++++++++ projects/VS2005/zstdlib/zstdlib.vcproj | 494 ++++++++++++++++++ 9 files changed, 2024 insertions(+), 17 deletions(-) create mode 100644 projects/VS2005/fullbench/fullbench.vcproj create mode 100644 projects/VS2005/fuzzer/fuzzer.vcproj create mode 100644 projects/VS2005/zstd.sln create mode 100644 projects/VS2005/zstd/zstd.vcproj create mode 100644 projects/VS2005/zstdlib/zstdlib.vcproj diff --git a/.gitignore b/.gitignore index 0c458153..01a4dd91 100644 --- a/.gitignore +++ b/.gitignore @@ -18,14 +18,6 @@ zstd *.out *.app -# Visual C++ -projects/VS2008 -projects/VS2010 -projects/VS2012 -projects/VS2013 -projects/VS2015 -build/bin - # IDEA solution files *.idea @@ -35,9 +27,6 @@ _codelite/ _zstdbench/ .clang_complete -# CMake -projects/cmake/ - # Test artefacts tmp* dictionary diff --git a/appveyor.yml b/appveyor.yml index 1ee8cf8f..03fb7930 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -64,9 +64,6 @@ build_script: msbuild "projects\VS2008\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v90 /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && DIR projects\VS2008\bin\%PLATFORM%\%CONFIGURATION%\*.exe && MD5sum projects/VS2008/bin/%PLATFORM%/%CONFIGURATION%/*.exe && - msbuild "projects\VS2008\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v90 /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && - DIR projects\VS2008\bin\%PLATFORM%\%CONFIGURATION%\*.exe && - MD5sum projects/VS2008/bin/%PLATFORM%/%CONFIGURATION%/*.exe && COPY projects\VS2008\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe projects\fuzzer_VS2008_%PLATFORM%_%CONFIGURATION%.exe && ECHO *** && ECHO *** Building Visual Studio 2010 %PLATFORM%\%CONFIGURATION% && diff --git a/programs/datagen.c b/programs/datagen.c index 6cb5111f..d1fb858a 100644 --- a/programs/datagen.c +++ b/programs/datagen.c @@ -26,8 +26,10 @@ /* ************************************* * Compiler Options ***************************************/ -#define _CRT_SECURE_NO_WARNINGS /* removes Visual warning on strerror() */ - +#if defined(_MSC_VER) +# define _CRT_SECURE_NO_WARNINGS /* removes Visual warning on strerror() */ +# define _CRT_SECURE_NO_DEPRECATE /* removes VS2005 warning on strerror() */ +#endif /*-************************************ * Dependencies diff --git a/projects/.gitignore b/projects/.gitignore index dc203515..c84b2f31 100644 --- a/projects/.gitignore +++ b/projects/.gitignore @@ -1,2 +1,12 @@ -build *Copy + +# Visual C++ +build/ +VS2008/ +VS2010/ +VS2012/ +VS2013/ +VS2015/ + +# CMake +cmake/ diff --git a/projects/VS2005/fullbench/fullbench.vcproj b/projects/VS2005/fullbench/fullbench.vcproj new file mode 100644 index 00000000..6379cf26 --- /dev/null +++ b/projects/VS2005/fullbench/fullbench.vcproj @@ -0,0 +1,448 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/projects/VS2005/fuzzer/fuzzer.vcproj b/projects/VS2005/fuzzer/fuzzer.vcproj new file mode 100644 index 00000000..e58ca70c --- /dev/null +++ b/projects/VS2005/fuzzer/fuzzer.vcproj @@ -0,0 +1,460 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/projects/VS2005/zstd.sln b/projects/VS2005/zstd.sln new file mode 100644 index 00000000..0c31ae12 --- /dev/null +++ b/projects/VS2005/zstd.sln @@ -0,0 +1,55 @@ +Microsoft Visual Studio Solution File, Format Version 9.00 +# Visual C++ Express 2005 +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "zstd", "zstd\zstd.vcproj", "{1A2AB08E-5CE7-4C5B-BE55-458157C14051}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "fuzzer", "fuzzer\fuzzer.vcproj", "{A62E89D2-9DDE-42BA-8F9B-9DA74889A6B0}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "fullbench", "fullbench\fullbench.vcproj", "{CC8F1D1B-BA2F-43E3-A71F-FA415D81AAD3}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "zstdlib", "zstdlib\zstdlib.vcproj", "{99DE2A79-7298-4004-A0ED-030D7A3796CA}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Win32 = Debug|Win32 + Debug|x64 = Debug|x64 + Release|Win32 = Release|Win32 + Release|x64 = Release|x64 + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {1A2AB08E-5CE7-4C5B-BE55-458157C14051}.Debug|Win32.ActiveCfg = Debug|Win32 + {1A2AB08E-5CE7-4C5B-BE55-458157C14051}.Debug|Win32.Build.0 = Debug|Win32 + {1A2AB08E-5CE7-4C5B-BE55-458157C14051}.Debug|x64.ActiveCfg = Debug|x64 + {1A2AB08E-5CE7-4C5B-BE55-458157C14051}.Debug|x64.Build.0 = Debug|x64 + {1A2AB08E-5CE7-4C5B-BE55-458157C14051}.Release|Win32.ActiveCfg = Release|Win32 + {1A2AB08E-5CE7-4C5B-BE55-458157C14051}.Release|Win32.Build.0 = Release|Win32 + {1A2AB08E-5CE7-4C5B-BE55-458157C14051}.Release|x64.ActiveCfg = Release|x64 + {1A2AB08E-5CE7-4C5B-BE55-458157C14051}.Release|x64.Build.0 = Release|x64 + {A62E89D2-9DDE-42BA-8F9B-9DA74889A6B0}.Debug|Win32.ActiveCfg = Debug|Win32 + {A62E89D2-9DDE-42BA-8F9B-9DA74889A6B0}.Debug|Win32.Build.0 = Debug|Win32 + {A62E89D2-9DDE-42BA-8F9B-9DA74889A6B0}.Debug|x64.ActiveCfg = Debug|x64 + {A62E89D2-9DDE-42BA-8F9B-9DA74889A6B0}.Debug|x64.Build.0 = Debug|x64 + {A62E89D2-9DDE-42BA-8F9B-9DA74889A6B0}.Release|Win32.ActiveCfg = Release|Win32 + {A62E89D2-9DDE-42BA-8F9B-9DA74889A6B0}.Release|Win32.Build.0 = Release|Win32 + {A62E89D2-9DDE-42BA-8F9B-9DA74889A6B0}.Release|x64.ActiveCfg = Release|x64 + {A62E89D2-9DDE-42BA-8F9B-9DA74889A6B0}.Release|x64.Build.0 = Release|x64 + {CC8F1D1B-BA2F-43E3-A71F-FA415D81AAD3}.Debug|Win32.ActiveCfg = Debug|Win32 + {CC8F1D1B-BA2F-43E3-A71F-FA415D81AAD3}.Debug|Win32.Build.0 = Debug|Win32 + {CC8F1D1B-BA2F-43E3-A71F-FA415D81AAD3}.Debug|x64.ActiveCfg = Debug|x64 + {CC8F1D1B-BA2F-43E3-A71F-FA415D81AAD3}.Debug|x64.Build.0 = Debug|x64 + {CC8F1D1B-BA2F-43E3-A71F-FA415D81AAD3}.Release|Win32.ActiveCfg = Release|Win32 + {CC8F1D1B-BA2F-43E3-A71F-FA415D81AAD3}.Release|Win32.Build.0 = Release|Win32 + {CC8F1D1B-BA2F-43E3-A71F-FA415D81AAD3}.Release|x64.ActiveCfg = Release|x64 + {CC8F1D1B-BA2F-43E3-A71F-FA415D81AAD3}.Release|x64.Build.0 = Release|x64 + {99DE2A79-7298-4004-A0ED-030D7A3796CA}.Debug|Win32.ActiveCfg = Debug|Win32 + {99DE2A79-7298-4004-A0ED-030D7A3796CA}.Debug|Win32.Build.0 = Debug|Win32 + {99DE2A79-7298-4004-A0ED-030D7A3796CA}.Debug|x64.ActiveCfg = Debug|x64 + {99DE2A79-7298-4004-A0ED-030D7A3796CA}.Debug|x64.Build.0 = Debug|x64 + {99DE2A79-7298-4004-A0ED-030D7A3796CA}.Release|Win32.ActiveCfg = Release|Win32 + {99DE2A79-7298-4004-A0ED-030D7A3796CA}.Release|Win32.Build.0 = Release|Win32 + {99DE2A79-7298-4004-A0ED-030D7A3796CA}.Release|x64.ActiveCfg = Release|x64 + {99DE2A79-7298-4004-A0ED-030D7A3796CA}.Release|x64.Build.0 = Release|x64 + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection +EndGlobal diff --git a/projects/VS2005/zstd/zstd.vcproj b/projects/VS2005/zstd/zstd.vcproj new file mode 100644 index 00000000..c9b7442b --- /dev/null +++ b/projects/VS2005/zstd/zstd.vcproj @@ -0,0 +1,552 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/projects/VS2005/zstdlib/zstdlib.vcproj b/projects/VS2005/zstdlib/zstdlib.vcproj new file mode 100644 index 00000000..2313c87a --- /dev/null +++ b/projects/VS2005/zstdlib/zstdlib.vcproj @@ -0,0 +1,494 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + From 32c02fa665ed5ac45bf8b2520b91fd1894d2ee99 Mon Sep 17 00:00:00 2001 From: inikep Date: Mon, 8 Aug 2016 10:41:46 +0200 Subject: [PATCH 22/27] Fix for Intel Compiler with Windows --- lib/common/mem.h | 2 +- lib/common/xxhash.c | 2 +- lib/legacy/zstd_v01.c | 2 +- lib/legacy/zstd_v02.c | 2 +- lib/legacy/zstd_v03.c | 2 +- lib/legacy/zstd_v04.c | 2 +- lib/legacy/zstd_v05.c | 2 +- lib/legacy/zstd_v06.c | 2 +- lib/legacy/zstd_v07.c | 2 +- projects/.gitignore | 1 + 10 files changed, 10 insertions(+), 9 deletions(-) diff --git a/lib/common/mem.h b/lib/common/mem.h index fc7b103e..da4f5925 100644 --- a/lib/common/mem.h +++ b/lib/common/mem.h @@ -110,7 +110,7 @@ MEM_STATIC void MEM_check(void) { MEM_STATIC_ASSERT((sizeof(size_t)==4) || (size #ifndef MEM_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */ # if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) ) # define MEM_FORCE_MEMORY_ACCESS 2 -# elif defined(__INTEL_COMPILER) || \ +# elif (defined(__INTEL_COMPILER) && !defined(WIN32)) || \ (defined(__GNUC__) && ( defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) || defined(__ARM_ARCH_7R__) || defined(__ARM_ARCH_7M__) || defined(__ARM_ARCH_7S__) )) # define MEM_FORCE_MEMORY_ACCESS 1 # endif diff --git a/lib/common/xxhash.c b/lib/common/xxhash.c index e175ae96..40f00d4c 100644 --- a/lib/common/xxhash.c +++ b/lib/common/xxhash.c @@ -52,7 +52,7 @@ #ifndef XXH_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */ # if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) ) # define XXH_FORCE_MEMORY_ACCESS 2 -# elif defined(__INTEL_COMPILER) || \ +# elif (defined(__INTEL_COMPILER) && !defined(WIN32)) || \ (defined(__GNUC__) && ( defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) || defined(__ARM_ARCH_7R__) || defined(__ARM_ARCH_7M__) || defined(__ARM_ARCH_7S__) )) # define XXH_FORCE_MEMORY_ACCESS 1 # endif diff --git a/lib/legacy/zstd_v01.c b/lib/legacy/zstd_v01.c index d62367df..5540de17 100644 --- a/lib/legacy/zstd_v01.c +++ b/lib/legacy/zstd_v01.c @@ -225,7 +225,7 @@ typedef signed long long S64; #ifndef FSE_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */ # if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) ) # define FSE_FORCE_MEMORY_ACCESS 2 -# elif defined(__INTEL_COMPILER) || \ +# elif (defined(__INTEL_COMPILER) && !defined(WIN32)) || \ (defined(__GNUC__) && ( defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) || defined(__ARM_ARCH_7R__) || defined(__ARM_ARCH_7M__) || defined(__ARM_ARCH_7S__) )) # define FSE_FORCE_MEMORY_ACCESS 1 # endif diff --git a/lib/legacy/zstd_v02.c b/lib/legacy/zstd_v02.c index be847671..0b292dba 100644 --- a/lib/legacy/zstd_v02.c +++ b/lib/legacy/zstd_v02.c @@ -194,7 +194,7 @@ extern "C" { #ifndef MEM_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */ # if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) ) # define MEM_FORCE_MEMORY_ACCESS 2 -# elif defined(__INTEL_COMPILER) || \ +# elif (defined(__INTEL_COMPILER) && !defined(WIN32)) || \ (defined(__GNUC__) && ( defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) || defined(__ARM_ARCH_7R__) || defined(__ARM_ARCH_7M__) || defined(__ARM_ARCH_7S__) )) # define MEM_FORCE_MEMORY_ACCESS 1 # endif diff --git a/lib/legacy/zstd_v03.c b/lib/legacy/zstd_v03.c index 86b8138e..d3cb1db3 100644 --- a/lib/legacy/zstd_v03.c +++ b/lib/legacy/zstd_v03.c @@ -194,7 +194,7 @@ extern "C" { #ifndef MEM_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */ # if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) ) # define MEM_FORCE_MEMORY_ACCESS 2 -# elif defined(__INTEL_COMPILER) || \ +# elif (defined(__INTEL_COMPILER) && !defined(WIN32)) || \ (defined(__GNUC__) && ( defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) || defined(__ARM_ARCH_7R__) || defined(__ARM_ARCH_7M__) || defined(__ARM_ARCH_7S__) )) # define MEM_FORCE_MEMORY_ACCESS 1 # endif diff --git a/lib/legacy/zstd_v04.c b/lib/legacy/zstd_v04.c index 9fb07caa..a803e171 100644 --- a/lib/legacy/zstd_v04.c +++ b/lib/legacy/zstd_v04.c @@ -144,7 +144,7 @@ extern "C" { #ifndef MEM_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */ # if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) ) # define MEM_FORCE_MEMORY_ACCESS 2 -# elif defined(__INTEL_COMPILER) || \ +# elif (defined(__INTEL_COMPILER) && !defined(WIN32)) || \ (defined(__GNUC__) && ( defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) || defined(__ARM_ARCH_7R__) || defined(__ARM_ARCH_7M__) || defined(__ARM_ARCH_7S__) )) # define MEM_FORCE_MEMORY_ACCESS 1 # endif diff --git a/lib/legacy/zstd_v05.c b/lib/legacy/zstd_v05.c index f3c720fd..26ea32bd 100644 --- a/lib/legacy/zstd_v05.c +++ b/lib/legacy/zstd_v05.c @@ -140,7 +140,7 @@ extern "C" { #ifndef MEM_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */ # if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) ) # define MEM_FORCE_MEMORY_ACCESS 2 -# elif defined(__INTEL_COMPILER) || \ +# elif (defined(__INTEL_COMPILER) && !defined(WIN32)) || \ (defined(__GNUC__) && ( defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) || defined(__ARM_ARCH_7R__) || defined(__ARM_ARCH_7M__) || defined(__ARM_ARCH_7S__) )) # define MEM_FORCE_MEMORY_ACCESS 1 # endif diff --git a/lib/legacy/zstd_v06.c b/lib/legacy/zstd_v06.c index aad3d13d..8c621b2e 100644 --- a/lib/legacy/zstd_v06.c +++ b/lib/legacy/zstd_v06.c @@ -142,7 +142,7 @@ extern "C" { #ifndef MEM_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */ # if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) ) # define MEM_FORCE_MEMORY_ACCESS 2 -# elif defined(__INTEL_COMPILER) || \ +# elif (defined(__INTEL_COMPILER) && !defined(WIN32)) || \ (defined(__GNUC__) && ( defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) || defined(__ARM_ARCH_7R__) || defined(__ARM_ARCH_7M__) || defined(__ARM_ARCH_7S__) )) # define MEM_FORCE_MEMORY_ACCESS 1 # endif diff --git a/lib/legacy/zstd_v07.c b/lib/legacy/zstd_v07.c index fb4d65b8..16a6fcdf 100644 --- a/lib/legacy/zstd_v07.c +++ b/lib/legacy/zstd_v07.c @@ -303,7 +303,7 @@ MEM_STATIC void MEM_check(void) { MEM_STATIC_ASSERT((sizeof(size_t)==4) || (size #ifndef MEM_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */ # if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) ) # define MEM_FORCE_MEMORY_ACCESS 2 -# elif defined(__INTEL_COMPILER) || \ +# elif (defined(__INTEL_COMPILER) && !defined(WIN32)) || \ (defined(__GNUC__) && ( defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) || defined(__ARM_ARCH_7R__) || defined(__ARM_ARCH_7M__) || defined(__ARM_ARCH_7S__) )) # define MEM_FORCE_MEMORY_ACCESS 1 # endif diff --git a/projects/.gitignore b/projects/.gitignore index c84b2f31..7ceb958e 100644 --- a/projects/.gitignore +++ b/projects/.gitignore @@ -2,6 +2,7 @@ # Visual C++ build/ +VS2005/ VS2008/ VS2010/ VS2012/ From ad9f70e4405d06640ca044cbda0fc34e30063e46 Mon Sep 17 00:00:00 2001 From: inikep Date: Mon, 8 Aug 2016 11:05:38 +0200 Subject: [PATCH 23/27] Improved speed of Intel Compiler for Windows --- lib/common/mem.h | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/lib/common/mem.h b/lib/common/mem.h index da4f5925..6f31aac6 100644 --- a/lib/common/mem.h +++ b/lib/common/mem.h @@ -110,7 +110,7 @@ MEM_STATIC void MEM_check(void) { MEM_STATIC_ASSERT((sizeof(size_t)==4) || (size #ifndef MEM_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */ # if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) ) # define MEM_FORCE_MEMORY_ACCESS 2 -# elif (defined(__INTEL_COMPILER) && !defined(WIN32)) || \ +# elif defined(__INTEL_COMPILER) /*|| defined(_MSC_VER)*/ || \ (defined(__GNUC__) && ( defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) || defined(__ARM_ARCH_7R__) || defined(__ARM_ARCH_7M__) || defined(__ARM_ARCH_7S__) )) # define MEM_FORCE_MEMORY_ACCESS 1 # endif @@ -142,7 +142,13 @@ MEM_STATIC void MEM_write64(void* memPtr, U64 value) { *(U64*)memPtr = value; } /* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */ /* currently only defined for gcc and icc */ -typedef union { U16 u16; U32 u32; U64 u64; size_t st; } __attribute__((packed)) unalign; +#if defined(_MSC_VER) || (defined(__INTEL_COMPILER) && defined(WIN32)) + __pragma( pack(push, 1) ) + typedef union { U16 u16; U32 u32; U64 u64; size_t st; } unalign; + __pragma( pack(pop) ) +#else + typedef union { U16 u16; U32 u32; U64 u64; size_t st; } __attribute__((packed)) unalign; +#endif MEM_STATIC U16 MEM_read16(const void* ptr) { return ((const unalign*)ptr)->u16; } MEM_STATIC U32 MEM_read32(const void* ptr) { return ((const unalign*)ptr)->u32; } From ac1e7e8fc19a7e701b30323df798f871e20ae21b Mon Sep 17 00:00:00 2001 From: inikep Date: Tue, 9 Aug 2016 11:03:15 +0200 Subject: [PATCH 24/27] fixed: tree not updated after finding very long rep matches --- lib/compress/zstd_compress.c | 5 +++++ lib/compress/zstd_opt.h | 8 ++++---- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 56c63601..a496317c 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -2205,8 +2205,13 @@ static ZSTD_blockCompressor ZSTD_selectBlockCompressor(ZSTD_strategy strat, int static size_t ZSTD_compressBlock_internal(ZSTD_CCtx* zc, void* dst, size_t dstCapacity, const void* src, size_t srcSize) { ZSTD_blockCompressor const blockCompressor = ZSTD_selectBlockCompressor(zc->params.cParams.strategy, zc->lowLimit < zc->dictLimit); + const BYTE* const base = zc->base; + const BYTE* const istart = (const BYTE*)src; + const U32 current = (U32)(istart-base); if (srcSize < MIN_CBLOCK_SIZE+ZSTD_blockHeaderSize+1) return 0; /* don't even attempt compression below a certain srcSize */ ZSTD_resetSeqStore(&(zc->seqStore)); + if (current > zc->nextToUpdate + 384) + zc->nextToUpdate = current - MIN(192, (U32)(current - zc->nextToUpdate - 384)); /* update tree not updated after finding very long rep matches */ blockCompressor(zc, src, srcSize); return ZSTD_compressSequences(zc, dst, dstCapacity, srcSize); } diff --git a/lib/compress/zstd_opt.h b/lib/compress/zstd_opt.h index d9f6cfdc..f30cdbfd 100644 --- a/lib/compress/zstd_opt.h +++ b/lib/compress/zstd_opt.h @@ -674,8 +674,8 @@ _storeSequence: /* cur, last_pos, best_mlen, best_off have to be set */ #if ZSTD_OPT_DEBUG >= 5 U32 ml2; - if (offset > ZSTD_REP_MOVE_OPT) - ml2 = (U32)ZSTD_count(ip, ip-(offset-ZSTD_REP_MOVE_OPT), iend); + if (offset+1 > ZSTD_REP_MOVE_OPT) + ml2 = (U32)ZSTD_count(ip, ip-(offset+1-ZSTD_REP_MOVE_OPT), iend); else ml2 = (U32)ZSTD_count(ip, ip-rep[0], iend); if ((offset >= 8) && (ml2 < mlen || ml2 < minMatch)) { @@ -992,8 +992,8 @@ _storeSequence: /* cur, last_pos, best_mlen, best_off have to be set */ #if ZSTD_OPT_DEBUG >= 5 U32 ml2; - if (offset > ZSTD_REP_MOVE_OPT) { - best_off = offset - ZSTD_REP_MOVE_OPT; + if (offset+1 > ZSTD_REP_MOVE_OPT) { + best_off = offset+1 - ZSTD_REP_MOVE_OPT; if (best_off > (size_t)(ip - prefixStart)) { const BYTE* match = dictEnd - (best_off - (ip - prefixStart)); ml2 = ZSTD_count_2segments(ip, match, iend, dictEnd, prefixStart); From 7132fb15ba301ce37feeea659c7d6116752c9b8e Mon Sep 17 00:00:00 2001 From: inikep Date: Wed, 10 Aug 2016 14:59:18 +0200 Subject: [PATCH 25/27] bench.c: removed benchResult_t --- programs/bench.c | 34 +++++++++++----------------------- 1 file changed, 11 insertions(+), 23 deletions(-) diff --git a/programs/bench.c b/programs/bench.c index 2364348d..c85ab2ed 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -133,14 +133,6 @@ typedef struct size_t resSize; } blockParam_t; -typedef struct -{ - double ratio; - size_t cSize; - double cSpeed; - double dSpeed; -} benchResult_t; - #define MIN(a,b) ((a)<(b) ? (a) : (b)) #define MAX(a,b) ((a)>(b) ? (a) : (b)) @@ -148,7 +140,7 @@ typedef struct static int BMK_benchMem(const void* srcBuffer, size_t srcSize, const char* displayName, int cLevel, const size_t* fileSizes, U32 nbFiles, - const void* dictBuffer, size_t dictBufferSize, benchResult_t *result) + const void* dictBuffer, size_t dictBufferSize) { size_t const blockSize = (g_blockSize>=32 ? g_blockSize : srcSize) + (!srcSize) /* avoid div by 0 */ ; size_t const avgSize = MIN(g_blockSize, (srcSize / nbFiles)); @@ -332,10 +324,14 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, #endif } /* for (testNb = 1; testNb <= (g_nbIterations + !g_nbIterations); testNb++) */ - result->ratio = ratio; - result->cSize = cSize; - result->cSpeed = (double)srcSize / fastestC; - result->dSpeed = (double)srcSize / fastestD; + if (g_displayLevel == 1) { + double cSpeed = (double)srcSize / fastestC; + double dSpeed = (double)srcSize / fastestD; + if (g_additionalParam) + DISPLAY("-%-3i%11i (%5.3f) %6.2f MB/s %6.1f MB/s %s (param=%d)\n", cLevel, (int)cSize, ratio, cSpeed, dSpeed, displayName, g_additionalParam); + else + DISPLAY("-%-3i%11i (%5.3f) %6.2f MB/s %6.1f MB/s %s\n", cLevel, (int)cSize, ratio, cSpeed, dSpeed, displayName); + } DISPLAYLEVEL(2, "%2i#\n", cLevel); } /* Bench */ @@ -372,7 +368,6 @@ static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize, const size_t* fileSizes, unsigned nbFiles, const void* dictBuffer, size_t dictBufferSize) { - benchResult_t result; int l; const char* pch = strrchr(displayName, '\\'); /* Windows */ @@ -381,8 +376,6 @@ static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize, SET_HIGH_PRIORITY; - memset(&result, 0, sizeof(result)); - if (g_displayLevel == 1 && !g_additionalParam) DISPLAY("bench %s %s: input %u bytes, %i iterations, %u KB blocks\n", ZSTD_VERSION_STRING, ZSTD_GIT_COMMIT_STRING, (U32)benchedSize, g_nbIterations, (U32)(g_blockSize>>10)); @@ -392,13 +385,8 @@ static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize, BMK_benchMem(srcBuffer, benchedSize, displayName, l, fileSizes, nbFiles, - dictBuffer, dictBufferSize, &result); - if (g_displayLevel == 1) { - if (g_additionalParam) - DISPLAY("%-3i%11i (%5.3f) %6.2f MB/s %6.1f MB/s %s (param=%d)\n", -l, (int)result.cSize, result.ratio, result.cSpeed, result.dSpeed, displayName, g_additionalParam); - else - DISPLAY("%-3i%11i (%5.3f) %6.2f MB/s %6.1f MB/s %s\n", -l, (int)result.cSize, result.ratio, result.cSpeed, result.dSpeed, displayName); - } } + dictBuffer, dictBufferSize); + } } From 98e08cbe344880a68382939371515e2af2c0fae2 Mon Sep 17 00:00:00 2001 From: inikep Date: Wed, 10 Aug 2016 15:00:30 +0200 Subject: [PATCH 26/27] fixed: tree not updated after finding very long rep matches --- lib/compress/zstd_compress.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 0f79fc85..e218b637 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -2205,8 +2205,13 @@ static ZSTD_blockCompressor ZSTD_selectBlockCompressor(ZSTD_strategy strat, int static size_t ZSTD_compressBlock_internal(ZSTD_CCtx* zc, void* dst, size_t dstCapacity, const void* src, size_t srcSize) { ZSTD_blockCompressor const blockCompressor = ZSTD_selectBlockCompressor(zc->params.cParams.strategy, zc->lowLimit < zc->dictLimit); + const BYTE* const base = zc->base; + const BYTE* const istart = (const BYTE*)src; + const U32 current = (U32)(istart-base); if (srcSize < MIN_CBLOCK_SIZE+ZSTD_blockHeaderSize+1) return 0; /* don't even attempt compression below a certain srcSize */ ZSTD_resetSeqStore(&(zc->seqStore)); + if (current > zc->nextToUpdate + 384) + zc->nextToUpdate = current - MIN(192, (U32)(current - zc->nextToUpdate - 384)); /* update tree not updated after finding very long rep matches */ blockCompressor(zc, src, srcSize); return ZSTD_compressSequences(zc, dst, dstCapacity, srcSize); } From 5f49eba5123259d584d7e81a090fb95c91f55f60 Mon Sep 17 00:00:00 2001 From: inikep Date: Wed, 10 Aug 2016 15:01:53 +0200 Subject: [PATCH 27/27] added usage of rep[0]-1 for the optimal parser --- lib/common/zstd_internal.h | 7 +-- lib/compress/zstd_opt.h | 103 ++++++++++++++++++++----------------- 2 files changed, 59 insertions(+), 51 deletions(-) diff --git a/lib/common/zstd_internal.h b/lib/common/zstd_internal.h index cc54d772..7dc843e3 100644 --- a/lib/common/zstd_internal.h +++ b/lib/common/zstd_internal.h @@ -67,9 +67,10 @@ #define ZSTD_OPT_NUM (1<<12) #define ZSTD_DICT_MAGIC 0xEC30A437 /* v0.7+ */ -#define ZSTD_REP_NUM 3 /* number of repcodes */ -#define ZSTD_REP_CHECK (ZSTD_REP_NUM-0) /* number of repcodes to check by the optimal parser */ -#define ZSTD_REP_MOVE (ZSTD_REP_NUM-1) +#define ZSTD_REP_NUM 3 /* number of repcodes */ +#define ZSTD_REP_CHECK (ZSTD_REP_NUM) /* number of repcodes to check by the optimal parser */ +#define ZSTD_REP_MOVE (ZSTD_REP_NUM-1) +#define ZSTD_REP_MOVE_OPT (ZSTD_REP_NUM) static const U32 repStartValue[ZSTD_REP_NUM] = { 1, 4, 8 }; #define KB *(1 <<10) diff --git a/lib/compress/zstd_opt.h b/lib/compress/zstd_opt.h index 3a1e9e19..f30cdbfd 100644 --- a/lib/compress/zstd_opt.h +++ b/lib/compress/zstd_opt.h @@ -276,7 +276,7 @@ static U32 ZSTD_insertBtAndGetAllMatches ( /* save best solution */ if (currentMl > bestLength) { bestLength = currentMl; - matches[mnum].off = ZSTD_REP_MOVE + current - matchIndex3; + matches[mnum].off = ZSTD_REP_MOVE_OPT + current - matchIndex3; matches[mnum].len = (U32)currentMl; mnum++; if (currentMl > ZSTD_OPT_NUM) goto update; @@ -321,7 +321,7 @@ static U32 ZSTD_insertBtAndGetAllMatches ( if (matchLength > bestLength) { if (matchLength > matchEndIdx - matchIndex) matchEndIdx = matchIndex + (U32)matchLength; bestLength = matchLength; - matches[mnum].off = ZSTD_REP_MOVE + current - matchIndex; + matches[mnum].off = ZSTD_REP_MOVE_OPT + current - matchIndex; matches[mnum].len = (U32)matchLength; mnum++; if (matchLength > ZSTD_OPT_NUM) break; @@ -452,11 +452,12 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx, litlen = (U32)(ip - anchor); /* check repCode */ - { U32 i; - for (i=(ip == anchor); i 0) && (repCur < (S32)(ip-prefixStart)) + && (MEM_readMINMATCH(ip, minMatch) == MEM_readMINMATCH(ip - repCur, minMatch))) { + mlen = (U32)ZSTD_count(ip+minMatch, ip+minMatch-repCur, iend) + minMatch; ZSTD_LOG_PARSER("%d: start try REP rep[%d]=%d mlen=%d\n", (int)(ip-base), i, (int)rep[i], (int)mlen); if (mlen > sufficient_len || mlen >= ZSTD_OPT_NUM) { best_mlen = mlen; best_off = i; cur = 0; last_pos = 1; @@ -491,7 +492,7 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx, best_mlen = matches[u].len; ZSTD_LOG_PARSER("%d: start Found mlen=%d off=%d best_mlen=%d last_pos=%d\n", (int)(ip-base), matches[u].len, matches[u].off, (int)best_mlen, (int)last_pos); while (mlen <= best_mlen) { - price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[u].off, mlen - MINMATCH); + price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[u].off-1, mlen - MINMATCH); if (mlen > last_pos || price < opt[mlen].price) SET_PRICE(mlen, mlen, matches[u].off, litlen, price); /* note : macro modifies last_pos */ mlen++; @@ -528,26 +529,27 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx, continue; mlen = opt[cur].mlen; - if (opt[cur].off >= ZSTD_REP_NUM) { + if (opt[cur].off > ZSTD_REP_MOVE_OPT) { opt[cur].rep[2] = opt[cur-mlen].rep[1]; opt[cur].rep[1] = opt[cur-mlen].rep[0]; - opt[cur].rep[0] = opt[cur].off - ZSTD_REP_MOVE; + opt[cur].rep[0] = opt[cur].off - ZSTD_REP_MOVE_OPT; ZSTD_LOG_ENCODE("%d: COPYREP_OFF cur=%d mlen=%d rep[0]=%d rep[1]=%d\n", (int)(inr-base), cur, mlen, opt[cur].rep[0], opt[cur].rep[1]); } else { opt[cur].rep[2] = (opt[cur].off > 1) ? opt[cur-mlen].rep[1] : opt[cur-mlen].rep[2]; opt[cur].rep[1] = (opt[cur].off > 0) ? opt[cur-mlen].rep[0] : opt[cur-mlen].rep[1]; - opt[cur].rep[0] = opt[cur-mlen].rep[opt[cur].off]; + opt[cur].rep[0] = ((opt[cur].off==ZSTD_REP_MOVE_OPT) && (mlen != 1)) ? (opt[cur-mlen].rep[0] - 1) : (opt[cur-mlen].rep[opt[cur].off]); ZSTD_LOG_ENCODE("%d: COPYREP_NOR cur=%d mlen=%d rep[0]=%d rep[1]=%d\n", (int)(inr-base), cur, mlen, opt[cur].rep[0], opt[cur].rep[1]); } ZSTD_LOG_PARSER("%d: CURRENT_NoExt price[%d/%d]=%d off=%d mlen=%d litlen=%d rep[0]=%d rep[1]=%d\n", (int)(inr-base), cur, last_pos, opt[cur].price, opt[cur].off, opt[cur].mlen, opt[cur].litlen, opt[cur].rep[0], opt[cur].rep[1]); best_mlen = minMatch; - { U32 i; - for (i=(opt[cur].mlen != 1); i 0) && (repCur < (S32)(inr-prefixStart)) + && (MEM_readMINMATCH(inr, minMatch) == MEM_readMINMATCH(inr - repCur, minMatch))) { + mlen = (U32)ZSTD_count(inr+minMatch, inr+minMatch - repCur, iend) + minMatch; ZSTD_LOG_PARSER("%d: Found REP %d/%d mlen=%d off=%d rep=%d opt[%d].off=%d\n", (int)(inr-base), i, ZSTD_REP_NUM, mlen, i, opt[cur].rep[i], cur, opt[cur].off); if (mlen > sufficient_len || cur + mlen >= ZSTD_OPT_NUM) { @@ -600,12 +602,12 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx, if (opt[cur].mlen == 1) { litlen = opt[cur].litlen; if (cur > litlen) - price = opt[cur - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, ip+cur-litlen, matches[u].off, mlen - MINMATCH); + price = opt[cur - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, ip+cur-litlen, matches[u].off-1, mlen - MINMATCH); else - price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[u].off, mlen - MINMATCH); + price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[u].off-1, mlen - MINMATCH); } else { litlen = 0; - price = opt[cur].price + ZSTD_getPrice(seqStorePtr, 0, NULL, matches[u].off, mlen - MINMATCH); + price = opt[cur].price + ZSTD_getPrice(seqStorePtr, 0, NULL, matches[u].off-1, mlen - MINMATCH); } // ZSTD_LOG_PARSER("%d: Found2 mlen=%d best_mlen=%d off=%d price=%d litlen=%d\n", (int)(inr-base), mlen, best_mlen, matches[u].off, price, litlen); @@ -652,27 +654,28 @@ _storeSequence: /* cur, last_pos, best_mlen, best_off have to be set */ litLength = (U32)(ip - anchor); // ZSTD_LOG_ENCODE("%d/%d: ENCODE literals=%d mlen=%d off=%d rep[0]=%d rep[1]=%d\n", (int)(ip-base), (int)(iend-base), (int)(litLength), (int)mlen, (int)(offset), (int)rep[0], (int)rep[1]); - if (offset >= ZSTD_REP_NUM) { + if (offset > ZSTD_REP_MOVE_OPT) { rep[2] = rep[1]; rep[1] = rep[0]; - rep[0] = offset - ZSTD_REP_MOVE; + rep[0] = offset - ZSTD_REP_MOVE_OPT; + offset--; } else { if (offset != 0) { - best_off = rep[offset]; + best_off = ((offset==ZSTD_REP_MOVE_OPT) && (litLength==0)) ? (rep[0] - 1) : (rep[offset]); if (offset != 1) rep[2] = rep[1]; rep[1] = rep[0]; rep[0] = best_off; } - if ((litLength == 0) & (offset==0)) offset = rep[1]; /* protection, but should never happen */ - if ((litLength == 0) & (offset<=2)) offset--; + if ((litLength==0) & (offset==0)) { ZSTD_LOG_ENCODE("ERROR (litLength==0) & (offset==0)\n"); }; + if (litLength==0) offset--; } ZSTD_LOG_ENCODE("%d/%d: ENCODE literals=%d mlen=%d off=%d rep[0]=%d rep[1]=%d\n", (int)(ip-base), (int)(iend-base), (int)(litLength), (int)mlen, (int)(offset), (int)rep[0], (int)rep[1]); #if ZSTD_OPT_DEBUG >= 5 U32 ml2; - if (offset >= ZSTD_REP_NUM) - ml2 = (U32)ZSTD_count(ip, ip-(offset-ZSTD_REP_MOVE), iend); + if (offset+1 > ZSTD_REP_MOVE_OPT) + ml2 = (U32)ZSTD_count(ip, ip-(offset+1-ZSTD_REP_MOVE_OPT), iend); else ml2 = (U32)ZSTD_count(ip, ip-rep[0], iend); if ((offset >= 8) && (ml2 < mlen || ml2 < minMatch)) { @@ -748,12 +751,13 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx, opt[0].litlen = (U32)(ip - anchor); /* check repCode */ - { U32 i; - for (i = (ip==anchor); i 0 && repCur <= (S32)current) && (((U32)((dictLimit-1) - repIndex) >= 3) & (repIndex>lowestIndex)) /* intentional overflow */ && (MEM_readMINMATCH(ip, minMatch) == MEM_readMINMATCH(repMatch, minMatch)) ) { /* repcode detected we should take it */ @@ -801,7 +805,7 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx, ZSTD_LOG_PARSER("%d: start Found mlen=%d off=%d best_mlen=%d last_pos=%d\n", (int)(ip-base), matches[u].len, matches[u].off, (int)best_mlen, (int)last_pos); litlen = opt[0].litlen; while (mlen <= best_mlen) { - price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[u].off, mlen - MINMATCH); + price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[u].off-1, mlen - MINMATCH); if (mlen > last_pos || price < opt[mlen].price) SET_PRICE(mlen, mlen, matches[u].off, litlen, price); mlen++; @@ -836,27 +840,28 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx, continue; mlen = opt[cur].mlen; - if (opt[cur].off >= ZSTD_REP_NUM) { + if (opt[cur].off > ZSTD_REP_MOVE_OPT) { opt[cur].rep[2] = opt[cur-mlen].rep[1]; opt[cur].rep[1] = opt[cur-mlen].rep[0]; - opt[cur].rep[0] = opt[cur].off - ZSTD_REP_MOVE; + opt[cur].rep[0] = opt[cur].off - ZSTD_REP_MOVE_OPT; ZSTD_LOG_ENCODE("%d: COPYREP_OFF cur=%d mlen=%d rep[0]=%d rep[1]=%d\n", (int)(inr-base), cur, mlen, opt[cur].rep[0], opt[cur].rep[1]); } else { opt[cur].rep[2] = (opt[cur].off > 1) ? opt[cur-mlen].rep[1] : opt[cur-mlen].rep[2]; opt[cur].rep[1] = (opt[cur].off > 0) ? opt[cur-mlen].rep[0] : opt[cur-mlen].rep[1]; - opt[cur].rep[0] = opt[cur-mlen].rep[opt[cur].off]; + opt[cur].rep[0] = ((opt[cur].off==ZSTD_REP_MOVE_OPT) && (mlen != 1)) ? (opt[cur-mlen].rep[0] - 1) : (opt[cur-mlen].rep[opt[cur].off]); ZSTD_LOG_ENCODE("%d: COPYREP_NOR cur=%d mlen=%d rep[0]=%d rep[1]=%d\n", (int)(inr-base), cur, mlen, opt[cur].rep[0], opt[cur].rep[1]); } ZSTD_LOG_PARSER("%d: CURRENT_Ext price[%d/%d]=%d off=%d mlen=%d litlen=%d rep[0]=%d rep[1]=%d\n", (int)(inr-base), cur, last_pos, opt[cur].price, opt[cur].off, opt[cur].mlen, opt[cur].litlen, opt[cur].rep[0], opt[cur].rep[1]); best_mlen = 0; - { U32 i; - for (i = (opt[cur].mlen != 1); i 0 && repCur <= (S32)(current+cur)) && (((U32)((dictLimit-1) - repIndex) >= 3) & (repIndex>lowestIndex)) /* intentional overflow */ && (MEM_readMINMATCH(inr, minMatch) == MEM_readMINMATCH(repMatch, minMatch)) ) { /* repcode detected */ @@ -914,12 +919,12 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx, if (opt[cur].mlen == 1) { litlen = opt[cur].litlen; if (cur > litlen) - price = opt[cur - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, ip+cur-litlen, matches[u].off, mlen - MINMATCH); + price = opt[cur - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, ip+cur-litlen, matches[u].off-1, mlen - MINMATCH); else - price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[u].off, mlen - MINMATCH); + price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[u].off-1, mlen - MINMATCH); } else { litlen = 0; - price = opt[cur].price + ZSTD_getPrice(seqStorePtr, 0, NULL, matches[u].off, mlen - MINMATCH); + price = opt[cur].price + ZSTD_getPrice(seqStorePtr, 0, NULL, matches[u].off-1, mlen - MINMATCH); } // ZSTD_LOG_PARSER("%d: Found2 mlen=%d best_mlen=%d off=%d price=%d litlen=%d\n", (int)(inr-base), mlen, best_mlen, matches[u].off, price, litlen); @@ -966,27 +971,29 @@ _storeSequence: /* cur, last_pos, best_mlen, best_off have to be set */ litLength = (U32)(ip - anchor); // ZSTD_LOG_ENCODE("%d/%d: ENCODE1 literals=%d mlen=%d off=%d rep[0]=%d rep[1]=%d\n", (int)(ip-base), (int)(iend-base), (int)(litLength), (int)mlen, (int)(offset), (int)rep[0], (int)rep[1]); - if (offset >= ZSTD_REP_NUM) { + if (offset > ZSTD_REP_MOVE_OPT) { rep[2] = rep[1]; rep[1] = rep[0]; - rep[0] = offset - ZSTD_REP_MOVE; + rep[0] = offset - ZSTD_REP_MOVE_OPT; + offset--; } else { if (offset != 0) { - best_off = rep[offset]; + best_off = ((offset==ZSTD_REP_MOVE_OPT) && (litLength==0)) ? (rep[0] - 1) : (rep[offset]); if (offset != 1) rep[2] = rep[1]; rep[1] = rep[0]; rep[0] = best_off; } - if ((litLength==0) & (offset==0)) offset = rep[1]; /* protection, but should never happen */ - if ((litLength==0) & (offset<=2)) offset --; + + if ((litLength==0) & (offset==0)) { ZSTD_LOG_ENCODE("ERROR (litLength==0) & (offset==0)\n"); }; + if (litLength==0) offset--; } ZSTD_LOG_ENCODE("%d/%d: ENCODE literals=%d mlen=%d off=%d rep[0]=%d rep[1]=%d\n", (int)(ip-base), (int)(iend-base), (int)(litLength), (int)mlen, (int)(offset), (int)rep[0], (int)rep[1]); #if ZSTD_OPT_DEBUG >= 5 U32 ml2; - if (offset >= ZSTD_REP_NUM) { - best_off = offset - ZSTD_REP_MOVE; + if (offset+1 > ZSTD_REP_MOVE_OPT) { + best_off = offset+1 - ZSTD_REP_MOVE_OPT; if (best_off > (size_t)(ip - prefixStart)) { const BYTE* match = dictEnd - (best_off - (ip - prefixStart)); ml2 = ZSTD_count_2segments(ip, match, iend, dictEnd, prefixStart);