facebook
/
zstd
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

introduced ZSTD_REP_MOVE and ZSTD_REP_INIT

dev
inikep 2016-03-18 11:03:43 +01:00
parent c56e32e782
commit eca2ec0b3b
4 changed files with 54 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
lib/zstd_compress.c
View File

@ -993,7 +993,7 @@ void ZSTD_compressBlock_fast_generic(ZSTD_CCtx* zc,
offset_2 = offset_1; offset_2 = offset_1;
offset_1 = offset; offset_1 = offset;
ZSTD_storeSeq(seqStorePtr, ip-anchor, anchor, offset + ZSTD_REP_NUM - 1, mlCode); ZSTD_storeSeq(seqStorePtr, ip-anchor, anchor, offset + ZSTD_REP_MOVE, mlCode);
} }
/* match found */ /* match found */
@ -1105,7 +1105,7 @@ static void ZSTD_compressBlock_fast_extDict_generic(ZSTD_CCtx* ctx,
offset = current - matchIndex; offset = current - matchIndex;
offset_2 = offset_1; offset_2 = offset_1;
offset_1 = offset; offset_1 = offset;
ZSTD_storeSeq(seqStorePtr, ip-anchor, anchor, offset + ZSTD_REP_NUM - 1, mlCode); ZSTD_storeSeq(seqStorePtr, ip-anchor, anchor, offset + ZSTD_REP_MOVE, mlCode);
} } } }
/* found a match : store it */ /* found a match : store it */
@ -1563,8 +1563,8 @@ void ZSTD_compressBlock_lazy_generic(ZSTD_CCtx* ctx,
searchMax_f searchMax = searchMethod ? ZSTD_BtFindBestMatch_selectMLS : ZSTD_HcFindBestMatch_selectMLS; searchMax_f searchMax = searchMethod ? ZSTD_BtFindBestMatch_selectMLS : ZSTD_HcFindBestMatch_selectMLS;
/* init */ /* init */
U32 rep[ZSTD_REP_NUM+1]; U32 rep[ZSTD_REP_INIT];
for (int i=0; i<ZSTD_REP_NUM+1; i++) for (int i=0; i<ZSTD_REP_INIT; i++)
rep[i]=REPCODE_STARTVALUE; rep[i]=REPCODE_STARTVALUE;
ZSTD_resetSeqStore(seqStorePtr); ZSTD_resetSeqStore(seqStorePtr);
@ -1598,7 +1598,7 @@ void ZSTD_compressBlock_lazy_generic(ZSTD_CCtx* ctx,
size_t offsetFound = 99999999; size_t offsetFound = 99999999;
size_t ml2 = searchMax(ctx, ip, iend, &offsetFound, maxSearches, mls); size_t ml2 = searchMax(ctx, ip, iend, &offsetFound, maxSearches, mls);
if (ml2 > matchLength) if (ml2 > matchLength)
matchLength = ml2, start = ip, offset=offsetFound + (ZSTD_REP_NUM - 1); matchLength = ml2, start = ip, offset=offsetFound + ZSTD_REP_MOVE;
} }
if (matchLength < MINMATCH) { if (matchLength < MINMATCH) {
@ -1624,7 +1624,7 @@ void ZSTD_compressBlock_lazy_generic(ZSTD_CCtx* ctx,
int gain2 = (int)(ml2*4 - ZSTD_highbit((U32)offset2+1)); /* raw approx */ int gain2 = (int)(ml2*4 - ZSTD_highbit((U32)offset2+1)); /* raw approx */
int gain1 = (int)(matchLength*4 - ZSTD_highbit((U32)offset+1) + 4); int gain1 = (int)(matchLength*4 - ZSTD_highbit((U32)offset+1) + 4);
if ((ml2 >= MINMATCH) && (gain2 > gain1)) { if ((ml2 >= MINMATCH) && (gain2 > gain1)) {
matchLength = ml2, offset = offset2 + (ZSTD_REP_NUM - 1), start = ip; matchLength = ml2, offset = offset2 + ZSTD_REP_MOVE, start = ip;
continue; /* search a better one */ continue; /* search a better one */
} } } }
@ -1645,7 +1645,7 @@ void ZSTD_compressBlock_lazy_generic(ZSTD_CCtx* ctx,
int gain2 = (int)(ml2*4 - ZSTD_highbit((U32)offset2+1)); /* raw approx */ int gain2 = (int)(ml2*4 - ZSTD_highbit((U32)offset2+1)); /* raw approx */
int gain1 = (int)(matchLength*4 - ZSTD_highbit((U32)offset+1) + 7); int gain1 = (int)(matchLength*4 - ZSTD_highbit((U32)offset+1) + 7);
if ((ml2 >= MINMATCH) && (gain2 > gain1)) { if ((ml2 >= MINMATCH) && (gain2 > gain1)) {
matchLength = ml2, offset = offset2 + (ZSTD_REP_NUM - 1), start = ip; matchLength = ml2, offset = offset2 + ZSTD_REP_MOVE, start = ip;
continue; continue;
} } } } } }
break; /* nothing found : store previous solution */ break; /* nothing found : store previous solution */
@ -1653,7 +1653,7 @@ void ZSTD_compressBlock_lazy_generic(ZSTD_CCtx* ctx,
/* catch up */ /* catch up */
if (offset >= ZSTD_REP_NUM) { if (offset >= ZSTD_REP_NUM) {
while ((start>anchor) && (start>base+offset-(ZSTD_REP_NUM-1)) && (start[-1] == start[-1-offset+(ZSTD_REP_NUM-1)])) /* only search for offset within prefix */ while ((start>anchor) && (start>base+offset-ZSTD_REP_MOVE) && (start[-1] == start[-1-offset+ZSTD_REP_MOVE])) /* only search for offset within prefix */
{ start--; matchLength++; } { start--; matchLength++; }
} }
@ -1666,7 +1666,7 @@ _storeSequence:
rep[3] = rep[2]; rep[3] = rep[2];
rep[2] = rep[1]; rep[2] = rep[1];
rep[1] = rep[0]; rep[1] = rep[0];
rep[0] = offset - (ZSTD_REP_NUM - 1); rep[0] = offset - ZSTD_REP_MOVE;
#else #else
if (kSlotNew < 3) if (kSlotNew < 3)
rep[3] = rep[2]; rep[3] = rep[2];
@ -1674,7 +1674,7 @@ _storeSequence:
rep[2] = rep[1]; rep[2] = rep[1];
if (kSlotNew < 1) if (kSlotNew < 1)
rep[1] = rep[0]; rep[1] = rep[0];
rep[kSlotNew] = offset - (ZSTD_REP_NUM - 1); rep[kSlotNew] = offset - ZSTD_REP_MOVE;
#endif #endif
} else { } else {
if (offset != 0) { if (offset != 0) {
@ -1691,7 +1691,7 @@ _storeSequence:
} }
#else #else
if (offset >= ZSTD_REP_NUM) { if (offset >= ZSTD_REP_NUM) {
rep[1] = rep[0]; rep[0] = offset - (ZSTD_REP_NUM - 1); rep[1] = rep[0]; rep[0] = offset - ZSTD_REP_MOVE;
} }
#endif #endif
size_t litLength = start - anchor; size_t litLength = start - anchor;
@ -1764,8 +1764,8 @@ void ZSTD_compressBlock_lazy_extDict_generic(ZSTD_CCtx* ctx,
printf("ZSTD_compressBlock_lazy_extDict_generic reps not implemented!\n"); exit(1); printf("ZSTD_compressBlock_lazy_extDict_generic reps not implemented!\n"); exit(1);
/* init */ /* init */
U32 rep[ZSTD_REP_NUM]; U32 rep[ZSTD_REP_INIT];
for (int i=0; i<ZSTD_REP_NUM; i++) for (int i=0; i<ZSTD_REP_INIT; i++)
rep[i]=REPCODE_STARTVALUE; rep[i]=REPCODE_STARTVALUE;
ZSTD_resetSeqStore(seqStorePtr); ZSTD_resetSeqStore(seqStorePtr);
if ((ip - prefixStart) < REPCODE_STARTVALUE) ip += REPCODE_STARTVALUE; if ((ip - prefixStart) < REPCODE_STARTVALUE) ip += REPCODE_STARTVALUE;
@ -1795,7 +1795,7 @@ void ZSTD_compressBlock_lazy_extDict_generic(ZSTD_CCtx* ctx,
size_t offsetFound = 99999999; size_t offsetFound = 99999999;
size_t ml2 = searchMax(ctx, ip, iend, &offsetFound, maxSearches, mls); size_t ml2 = searchMax(ctx, ip, iend, &offsetFound, maxSearches, mls);
if (ml2 > matchLength) if (ml2 > matchLength)
matchLength = ml2, start = ip, offset=offsetFound + (ZSTD_REP_NUM - 1); matchLength = ml2, start = ip, offset=offsetFound + ZSTD_REP_MOVE;
} }
if (matchLength < MINMATCH) { if (matchLength < MINMATCH) {
@ -1831,7 +1831,7 @@ void ZSTD_compressBlock_lazy_extDict_generic(ZSTD_CCtx* ctx,
int gain2 = (int)(ml2*4 - ZSTD_highbit((U32)offset2+1)); /* raw approx */ int gain2 = (int)(ml2*4 - ZSTD_highbit((U32)offset2+1)); /* raw approx */
int gain1 = (int)(matchLength*4 - ZSTD_highbit((U32)offset+1) + 4); int gain1 = (int)(matchLength*4 - ZSTD_highbit((U32)offset+1) + 4);
if ((ml2 >= MINMATCH) && (gain2 > gain1)) { if ((ml2 >= MINMATCH) && (gain2 > gain1)) {
matchLength = ml2, offset = offset2 + (ZSTD_REP_NUM - 1), start = ip; matchLength = ml2, offset = offset2 + ZSTD_REP_MOVE, start = ip;
continue; /* search a better one */ continue; /* search a better one */
} } } }
@ -1862,7 +1862,7 @@ void ZSTD_compressBlock_lazy_extDict_generic(ZSTD_CCtx* ctx,
int gain2 = (int)(ml2*4 - ZSTD_highbit((U32)offset2+1)); /* raw approx */ int gain2 = (int)(ml2*4 - ZSTD_highbit((U32)offset2+1)); /* raw approx */
int gain1 = (int)(matchLength*4 - ZSTD_highbit((U32)offset+1) + 7); int gain1 = (int)(matchLength*4 - ZSTD_highbit((U32)offset+1) + 7);
if ((ml2 >= MINMATCH) && (gain2 > gain1)) { if ((ml2 >= MINMATCH) && (gain2 > gain1)) {
matchLength = ml2, offset = offset2 + (ZSTD_REP_NUM - 1), start = ip; matchLength = ml2, offset = offset2 + ZSTD_REP_MOVE, start = ip;
continue; continue;
} } } } } }
break; /* nothing found : store previous solution */ break; /* nothing found : store previous solution */
@ -1870,11 +1870,11 @@ void ZSTD_compressBlock_lazy_extDict_generic(ZSTD_CCtx* ctx,
/* catch up */ /* catch up */
if (offset >= ZSTD_REP_NUM) { if (offset >= ZSTD_REP_NUM) {
U32 matchIndex = (U32)((start-base) - (offset - (ZSTD_REP_NUM - 1))); U32 matchIndex = (U32)((start-base) - (offset - ZSTD_REP_MOVE));
const BYTE* match = (matchIndex < dictLimit) ? dictBase + matchIndex : base + matchIndex; const BYTE* match = (matchIndex < dictLimit) ? dictBase + matchIndex : base + matchIndex;
const BYTE* const mStart = (matchIndex < dictLimit) ? dictStart : prefixStart; const BYTE* const mStart = (matchIndex < dictLimit) ? dictStart : prefixStart;
while ((start>anchor) && (match>mStart) && (start[-1] == match[-1])) { start--; match--; matchLength++; } /* catch up */ while ((start>anchor) && (match>mStart) && (start[-1] == match[-1])) { start--; match--; matchLength++; } /* catch up */
rep[1] = rep[0]; rep[0] = offset - (ZSTD_REP_NUM - 1); rep[1] = rep[0]; rep[0] = offset - ZSTD_REP_MOVE;
} }
/* store sequence */ /* store sequence */

28
lib/zstd_decompress.c
View File

@ -615,7 +615,7 @@ typedef struct {
FSE_DState_t stateLL; FSE_DState_t stateLL;
FSE_DState_t stateOffb; FSE_DState_t stateOffb;
FSE_DState_t stateML; FSE_DState_t stateML;
size_t prevOffset[ZSTD_REP_NUM]; size_t prevOffset[ZSTD_REP_INIT];
const BYTE* dumps; const BYTE* dumps;
const BYTE* dumpsEnd; const BYTE* dumpsEnd;
} seqState_t; } seqState_t;
@ -672,13 +672,13 @@ static void ZSTD_decodeSequence(seq_t* seq, seqState_t* seqState, const U32 mls)
else else
offset = seqState->prevOffset[0]; offset = seqState->prevOffset[0];
} else { } else {
offset -= ZSTD_REP_NUM - 1; offset -= ZSTD_REP_MOVE;
#if 0 #if 0
seqState->prevOffset[3] = seqState->prevOffset[2]; seqState->prevOffset[3] = seqState->prevOffset[2];
seqState->prevOffset[2] = seqState->prevOffset[1]; seqState->prevOffset[2] = seqState->prevOffset[1];
seqState->prevOffset[1] = seqState->prevOffset[0]; seqState->prevOffset[1] = seqState->prevOffset[0];
seqState->prevOffset[0] = offset; seqState->prevOffset[0] = offset;
#else #else
if (kSlotNew < 3) if (kSlotNew < 3)
seqState->prevOffset[3] = seqState->prevOffset[2]; seqState->prevOffset[3] = seqState->prevOffset[2];
if (kSlotNew < 2) if (kSlotNew < 2)
@ -686,11 +686,25 @@ static void ZSTD_decodeSequence(seq_t* seq, seqState_t* seqState, const U32 mls)
if (kSlotNew < 1) if (kSlotNew < 1)
seqState->prevOffset[1] = seqState->prevOffset[0]; seqState->prevOffset[1] = seqState->prevOffset[0];
seqState->prevOffset[kSlotNew] = offset; seqState->prevOffset[kSlotNew] = offset;
#endif #endif
} }
#else #else // ZSTD_REP_NUM == 1
#if 0
if (offsetCode==0) offset = litLength ? seq->offset : seqState->prevOffset[0]; /* repcode, cmove */ if (offsetCode==0) offset = litLength ? seq->offset : seqState->prevOffset[0]; /* repcode, cmove */
else offset -= ZSTD_REP_MOVE;
if (offsetCode | !litLength) seqState->prevOffset[0] = seq->offset; /* cmove */ if (offsetCode | !litLength) seqState->prevOffset[0] = seq->offset; /* cmove */
#else
if (offsetCode==0) {
if (!litLength) {
offset = seqState->prevOffset[0]; /* repcode, cmove */
seqState->prevOffset[0] = seq->offset; /* cmove */
} else
offset = seq->offset; /* repcode, cmove */
} else {
seqState->prevOffset[0] = seq->offset; /* cmove */
offset -= ZSTD_REP_MOVE;
}
#endif
#endif #endif
FSE_decodeSymbol(&(seqState->stateOffb), &(seqState->DStream)); /* update */ FSE_decodeSymbol(&(seqState->stateOffb), &(seqState->DStream)); /* update */
// printf("offsetCode=%d nbBits=%d offset=%d\n", offsetCode, nbBits, (int)offset); fflush(stdout); // printf("offsetCode=%d nbBits=%d offset=%d\n", offsetCode, nbBits, (int)offset); fflush(stdout);
@ -845,7 +859,7 @@ static size_t ZSTD_decompressSequences(
sequence.offset = REPCODE_STARTVALUE; sequence.offset = REPCODE_STARTVALUE;
seqState.dumps = dumps; seqState.dumps = dumps;
seqState.dumpsEnd = dumps + dumpsLength; seqState.dumpsEnd = dumps + dumpsLength;
for (int i=0; i<ZSTD_REP_NUM; i++) for (int i=0; i<ZSTD_REP_INIT; i++)
seqState.prevOffset[i] = REPCODE_STARTVALUE; seqState.prevOffset[i] = REPCODE_STARTVALUE;
errorCode = BIT_initDStream(&(seqState.DStream), ip, iend-ip); errorCode = BIT_initDStream(&(seqState.DStream), ip, iend-ip);
if (ERR_isError(errorCode)) return ERROR(corruption_detected); if (ERR_isError(errorCode)) return ERROR(corruption_detected);

10
lib/zstd_internal.h
View File

@ -64,7 +64,15 @@
#define ZSTD_OPT_NUM (1<<12) #define ZSTD_OPT_NUM (1<<12)
#define ZSTD_DICT_MAGIC 0xEC30A435 #define ZSTD_DICT_MAGIC 0xEC30A435
#define ZSTD_REP_NUM 4 #if 1
#define ZSTD_REP_NUM 4
#define ZSTD_REP_INIT 4
#define ZSTD_REP_MOVE (ZSTD_REP_NUM-1)
#else
#define ZSTD_REP_NUM 1
#define ZSTD_REP_INIT 2
#define ZSTD_REP_MOVE (ZSTD_REP_NUM-1)
#endif
#define KB *(1 <<10) #define KB *(1 <<10)
#define MB *(1 <<20) #define MB *(1 <<20)

12
lib/zstd_opt.h
View File

@ -420,8 +420,8 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx,
U32 cur, match_num, last_pos, litlen, price; U32 cur, match_num, last_pos, litlen, price;
/* init */ /* init */
U32 rep[ZSTD_REP_NUM+1]; U32 rep[ZSTD_REP_INIT];
for (int i=0; i<ZSTD_REP_NUM+1; i++) for (int i=0; i<ZSTD_REP_INIT; i++)
rep[i]=REPCODE_STARTVALUE; rep[i]=REPCODE_STARTVALUE;
ctx->nextToUpdate3 = ctx->nextToUpdate; ctx->nextToUpdate3 = ctx->nextToUpdate;
@ -688,7 +688,7 @@ _storeSequence: /* cur, last_pos, best_mlen, best_off have to be set */
#endif #endif
ZSTD_updatePrice(seqStorePtr, litLength, anchor, offset, mlen-minMatch); ZSTD_updatePrice(seqStorePtr, litLength, anchor, offset, mlen-minMatch);
ZSTD_storeSeq(seqStorePtr, litLength, anchor, offset ? offset + ZSTD_REP_NUM - 1 : 0, mlen-minMatch); ZSTD_storeSeq(seqStorePtr, litLength, anchor, offset ? offset + ZSTD_REP_MOVE : 0, mlen-minMatch);
anchor = ip = ip + mlen; anchor = ip = ip + mlen;
} /* for (cur=0; cur < last_pos; ) */ } /* for (cur=0; cur < last_pos; ) */
@ -748,8 +748,8 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx,
U32 cur, match_num, last_pos, litlen, price; U32 cur, match_num, last_pos, litlen, price;
/* init */ /* init */
U32 rep[ZSTD_REP_NUM+1]; U32 rep[ZSTD_REP_INIT];
for (int i=0; i<ZSTD_REP_NUM+1; i++) for (int i=0; i<ZSTD_REP_INIT; i++)
rep[i]=REPCODE_STARTVALUE; rep[i]=REPCODE_STARTVALUE;
ctx->nextToUpdate3 = ctx->nextToUpdate; ctx->nextToUpdate3 = ctx->nextToUpdate;
@ -1036,7 +1036,7 @@ _storeSequence: // cur, last_pos, best_mlen, best_off have to be set
#endif #endif
ZSTD_updatePrice(seqStorePtr, litLength, anchor, offset, mlen-minMatch); ZSTD_updatePrice(seqStorePtr, litLength, anchor, offset, mlen-minMatch);
ZSTD_storeSeq(seqStorePtr, litLength, anchor, offset ? offset + ZSTD_REP_NUM - 1 : 0, mlen-minMatch); ZSTD_storeSeq(seqStorePtr, litLength, anchor, offset ? offset + ZSTD_REP_MOVE : 0, mlen-minMatch);
anchor = ip = ip + mlen; anchor = ip = ip + mlen;
} }