diff --git a/lib/common/zstd_internal.h b/lib/common/zstd_internal.h index 5d2900eb..49f19a01 100644 --- a/lib/common/zstd_internal.h +++ b/lib/common/zstd_internal.h @@ -228,8 +228,6 @@ typedef struct { BYTE* ofCode; U32 longLengthID; /* 0 == no longLength; 1 == Lit.longLength; 2 == Match.longLength; */ U32 longLengthPos; - U32 rep[ZSTD_REP_NUM]; - U32 repToConfirm[ZSTD_REP_NUM]; } seqStore_t; const seqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx); /* compress & dictBuilder */ diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 101c689a..eba0baad 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -81,11 +81,15 @@ ZSTD_CCtx* ZSTD_initStaticCCtx(void *workspace, size_t workspaceSize) cctx->workSpace = (void*)(cctx+1); cctx->workSpaceSize = workspaceSize - sizeof(ZSTD_CCtx); - /* entropy space (never moves) */ - if (cctx->workSpaceSize < sizeof(ZSTD_entropyCTables_t)) return NULL; + /* statically sized space. entropyWorkspace never moves (but prev/next block swap places) */ + if (cctx->workSpaceSize < HUF_WORKSPACE_SIZE + 2 * sizeof(ZSTD_blockState_t)) return NULL; assert(((size_t)cctx->workSpace & (sizeof(void*)-1)) == 0); /* ensure correct alignment */ - cctx->entropy = (ZSTD_entropyCTables_t*)cctx->workSpace; - + cctx->prevBlock = (ZSTD_blockState_t*)cctx->workSpace; + cctx->nextBlock = cctx->prevBlock + 1; + { + void* const ptr = cctx->nextBlock + 1; + cctx->entropyWorkspace = (U32*)ptr; + } return cctx; } @@ -660,7 +664,8 @@ size_t ZSTD_estimateCCtxSize_usingCCtxParams(const ZSTD_CCtx_params* params) U32 const hashLog3 = (cParams.searchLength>3) ? 0 : MIN(ZSTD_HASHLOG3_MAX, cParams.windowLog); size_t const h3Size = ((size_t)1) << hashLog3; - size_t const entropySpace = sizeof(ZSTD_entropyCTables_t); + size_t const entropySpace = HUF_WORKSPACE_SIZE; + size_t const blockStateSpace = 2 * sizeof(ZSTD_blockState_t); size_t const tableSpace = (chainSize + hSize + h3Size) * sizeof(U32); size_t const optBudget = @@ -672,7 +677,7 @@ size_t ZSTD_estimateCCtxSize_usingCCtxParams(const ZSTD_CCtx_params* params) ZSTD_ldm_getTableSize(params->ldmParams.hashLog, params->ldmParams.bucketSizeLog) : 0; - size_t const neededSpace = entropySpace + tableSpace + tokenSpace + + size_t const neededSpace = entropySpace + blockStateSpace + tableSpace + tokenSpace + optSpace + ldmSpace; DEBUGLOG(5, "sizeof(ZSTD_CCtx) : %u", (U32)sizeof(ZSTD_CCtx)); @@ -794,16 +799,41 @@ static U32 ZSTD_equivalentParams(ZSTD_CCtx_params params1, ZSTD_sufficientBuff(buffSize1, blockSize1, buffPol2, params2.cParams, pledgedSrcSize); } +static void ZSTD_resetBlockState(ZSTD_blockState_t* bs) +{ + int i; + for (i = 0; i < ZSTD_REP_NUM; ++i) + bs->rep[i] = repStartValue[i]; + bs->entropy.hufCTable_repeatMode = HUF_repeat_none; + bs->entropy.offcode_repeatMode = FSE_repeat_none; + bs->entropy.matchlength_repeatMode = FSE_repeat_none; + bs->entropy.litlength_repeatMode = FSE_repeat_none; +} + +/*! ZSTD_invalidateMatchState() + * Invalidate all the matches in the match finder tables. + * Requires nextSrc and base to be set (can be NULL). + */ +static void ZSTD_invalidateMatchState(ZSTD_matchState_t* ms) +{ + size_t const endT = (size_t)(ms->nextSrc - ms->base); + U32 const end = (U32)endT; + assert(endT < (3U<<30)); + + ms->lowLimit = end; + ms->dictLimit = end; + ms->nextToUpdate = end + 1; + ms->loadedDictEnd = 0; + ms->opt.litLengthSum = 0; /* force reset of btopt stats */ +} + /*! ZSTD_continueCCtx() : * reuse CCtx without reset (note : requires no dictionary) */ static size_t ZSTD_continueCCtx(ZSTD_CCtx* cctx, ZSTD_CCtx_params params, U64 pledgedSrcSize) { - size_t const endT = (size_t)(cctx->nextSrc - cctx->base); - U32 const end = (U32)endT; size_t const windowSize = MAX(1, (size_t)MIN(((U64)1 << params.cParams.windowLog), pledgedSrcSize)); size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, windowSize); DEBUGLOG(4, "ZSTD_continueCCtx: re-use context in place"); - assert(endT < (3U<<30)); cctx->blockSize = blockSize; /* previous block size could be different even for same windowLog, due to pledgedSrcSize */ cctx->appliedParams = params; @@ -813,14 +843,10 @@ static size_t ZSTD_continueCCtx(ZSTD_CCtx* cctx, ZSTD_CCtx_params params, U64 pl cctx->appliedParams.fParams.contentSizeFlag = 0; DEBUGLOG(4, "pledged content size : %u ; flag : %u", (U32)pledgedSrcSize, cctx->appliedParams.fParams.contentSizeFlag); - cctx->lowLimit = end; - cctx->dictLimit = end; - cctx->nextToUpdate = end+1; cctx->stage = ZSTDcs_init; cctx->dictID = 0; - cctx->loadedDictEnd = 0; - { int i; for (i=0; iseqStore.rep[i] = repStartValue[i]; } - cctx->optState.litLengthSum = 0; /* force reset of btopt stats */ + ZSTD_invalidateMatchState(&cctx->matchState); + ZSTD_resetBlockState(cctx->prevBlock); XXH64_reset(&cctx->xxhState, 0); return 0; } @@ -846,10 +872,6 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, zc->appliedParams.cParams.windowLog, (U32)zc->blockSize); assert(!(params.ldmParams.enableLdm && params.ldmParams.hashEveryLog == ZSTD_LDM_HASHEVERYLOG_NOTSET)); - zc->entropy->hufCTable_repeatMode = HUF_repeat_none; - zc->entropy->offcode_repeatMode = FSE_repeat_none; - zc->entropy->matchlength_repeatMode = FSE_repeat_none; - zc->entropy->litlength_repeatMode = FSE_repeat_none; return ZSTD_continueCCtx(zc, params, pledgedSrcSize); } } DEBUGLOG(4, "ZSTD_equivalentParams()==0 -> reset CCtx"); @@ -880,7 +902,8 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, void* ptr; /* Check if workSpace is large enough, alloc a new one if needed */ - { size_t const entropySpace = sizeof(ZSTD_entropyCTables_t); + { size_t const entropySpace = HUF_WORKSPACE_SIZE; + size_t const blockStateSpace = 2 * sizeof(ZSTD_blockState_t); size_t const optPotentialSpace = ((MaxML+1) + (MaxLL+1) + (MaxOff+1) + (1<>10), (U32)(tableSpace>>10), (U32)(bufferSpace>>10)); @@ -911,10 +934,13 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, zc->workSpaceSize = neededSpace; ptr = zc->workSpace; - /* entropy space */ + /* Statically sized space. entropyWorkspace never moves (but prev/next block swap places) */ assert(((size_t)zc->workSpace & 3) == 0); /* ensure correct alignment */ - assert(zc->workSpaceSize >= sizeof(ZSTD_entropyCTables_t)); - zc->entropy = (ZSTD_entropyCTables_t*)zc->workSpace; + assert(zc->workSpaceSize >= 2 * sizeof(ZSTD_blockState_t)); + zc->prevBlock = (ZSTD_blockState_t*)zc->workSpace; + zc->nextBlock = zc->prevBlock + 1; + ptr = zc->nextBlock + 1; + zc->entropyWorkspace = (U32*)ptr; } } /* init params */ @@ -930,36 +956,31 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, XXH64_reset(&zc->xxhState, 0); zc->stage = ZSTDcs_init; zc->dictID = 0; - zc->loadedDictEnd = 0; - zc->entropy->hufCTable_repeatMode = HUF_repeat_none; - zc->entropy->offcode_repeatMode = FSE_repeat_none; - zc->entropy->matchlength_repeatMode = FSE_repeat_none; - zc->entropy->litlength_repeatMode = FSE_repeat_none; - zc->base = NULL; - zc->dictBase = NULL; - zc->lowLimit = 0; - zc->dictLimit = 0; - zc->nextToUpdate = 1; - zc->nextSrc = NULL; - { int i; for (i=0; iseqStore.rep[i] = repStartValue[i]; } - zc->hashLog3 = hashLog3; - zc->optState.litLengthSum = 0; - ptr = zc->entropy + 1; + zc->matchState.nextSrc = NULL; + zc->matchState.base = NULL; + zc->matchState.dictBase = NULL; + zc->matchState.hashLog3 = hashLog3; + ZSTD_invalidateMatchState(&zc->matchState); + + ZSTD_resetBlockState(zc->prevBlock); + + ptr = zc->entropyWorkspace + HUF_WORKSPACE_SIZE_U32; /* opt parser space */ if ((params.cParams.strategy == ZSTD_btopt) | (params.cParams.strategy == ZSTD_btultra)) { + optState_t* opt = &zc->matchState.opt; DEBUGLOG(4, "reserving optimal parser space"); assert(((size_t)ptr & 3) == 0); /* ensure ptr is properly aligned */ - zc->optState.litFreq = (U32*)ptr; - zc->optState.litLengthFreq = zc->optState.litFreq + (1<optState.matchLengthFreq = zc->optState.litLengthFreq + (MaxLL+1); - zc->optState.offCodeFreq = zc->optState.matchLengthFreq + (MaxML+1); - ptr = zc->optState.offCodeFreq + (MaxOff+1); - zc->optState.matchTable = (ZSTD_match_t*)ptr; - ptr = zc->optState.matchTable + ZSTD_OPT_NUM+1; - zc->optState.priceTable = (ZSTD_optimal_t*)ptr; - ptr = zc->optState.priceTable + ZSTD_OPT_NUM+1; + opt->litFreq = (U32*)ptr; + opt->litLengthFreq = opt->litFreq + (1<matchLengthFreq = opt->litLengthFreq + (MaxLL+1); + opt->offCodeFreq = opt->matchLengthFreq + (MaxML+1); + ptr = opt->offCodeFreq + (MaxOff+1); + opt->matchTable = (ZSTD_match_t*)ptr; + ptr = opt->matchTable + ZSTD_OPT_NUM+1; + opt->priceTable = (ZSTD_optimal_t*)ptr; + ptr = opt->priceTable + ZSTD_OPT_NUM+1; } /* ldm hash table */ @@ -976,10 +997,10 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, DEBUGLOG(4, "reset table : %u", crp!=ZSTDcrp_noMemset); if (crp!=ZSTDcrp_noMemset) memset(ptr, 0, tableSpace); /* reset tables only */ assert(((size_t)ptr & 3) == 0); /* ensure ptr is properly aligned */ - zc->hashTable = (U32*)(ptr); - zc->chainTable = zc->hashTable + hSize; - zc->hashTable3 = zc->chainTable + chainSize; - ptr = zc->hashTable3 + h3Size; + zc->matchState.hashTable = (U32*)(ptr); + zc->matchState.chainTable = zc->matchState.hashTable + hSize; + zc->matchState.hashTable3 = zc->matchState.chainTable + chainSize; + ptr = zc->matchState.hashTable3 + h3Size; /* sequences storage */ zc->seqStore.sequencesStart = (seqDef*)ptr; @@ -1016,7 +1037,8 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, * do not use with extDict variant ! */ void ZSTD_invalidateRepCodes(ZSTD_CCtx* cctx) { int i; - for (i=0; iseqStore.rep[i] = 0; + for (i=0; iprevBlock->rep[i] = 0; + assert(/* !extDict */ cctx->matchState.lowLimit == cctx->matchState.dictLimit); } @@ -1050,32 +1072,30 @@ static size_t ZSTD_copyCCtx_internal(ZSTD_CCtx* dstCCtx, /* copy tables */ { size_t const chainSize = (srcCCtx->appliedParams.cParams.strategy == ZSTD_fast) ? 0 : ((size_t)1 << srcCCtx->appliedParams.cParams.chainLog); size_t const hSize = (size_t)1 << srcCCtx->appliedParams.cParams.hashLog; - size_t const h3Size = (size_t)1 << srcCCtx->hashLog3; + size_t const h3Size = (size_t)1 << srcCCtx->matchState.hashLog3; size_t const tableSpace = (chainSize + hSize + h3Size) * sizeof(U32); - assert((U32*)dstCCtx->chainTable == (U32*)dstCCtx->hashTable + hSize); /* chainTable must follow hashTable */ - assert((U32*)dstCCtx->hashTable3 == (U32*)dstCCtx->chainTable + chainSize); - memcpy(dstCCtx->hashTable, srcCCtx->hashTable, tableSpace); /* presumes all tables follow each other */ + assert((U32*)dstCCtx->matchState.chainTable == (U32*)dstCCtx->matchState.hashTable + hSize); /* chainTable must follow hashTable */ + assert((U32*)dstCCtx->matchState.hashTable3 == (U32*)dstCCtx->matchState.chainTable + chainSize); + memcpy(dstCCtx->matchState.hashTable, srcCCtx->matchState.hashTable, tableSpace); /* presumes all tables follow each other */ } /* copy dictionary offsets */ - dstCCtx->nextToUpdate = srcCCtx->nextToUpdate; - dstCCtx->nextToUpdate3= srcCCtx->nextToUpdate3; - dstCCtx->nextSrc = srcCCtx->nextSrc; - dstCCtx->base = srcCCtx->base; - dstCCtx->dictBase = srcCCtx->dictBase; - dstCCtx->dictLimit = srcCCtx->dictLimit; - dstCCtx->lowLimit = srcCCtx->lowLimit; - dstCCtx->loadedDictEnd= srcCCtx->loadedDictEnd; - dstCCtx->dictID = srcCCtx->dictID; - - /* copy entropy tables */ - memcpy(dstCCtx->entropy, srcCCtx->entropy, sizeof(ZSTD_entropyCTables_t)); - /* copy repcodes */ { - int i; - for (i = 0; i < ZSTD_REP_NUM; ++i) - dstCCtx->seqStore.rep[i] = srcCCtx->seqStore.rep[i]; + ZSTD_matchState_t const* srcMatchState = &srcCCtx->matchState; + ZSTD_matchState_t* dstMatchState = &dstCCtx->matchState; + dstMatchState->nextToUpdate = srcMatchState->nextToUpdate; + dstMatchState->nextToUpdate3= srcMatchState->nextToUpdate3; + dstMatchState->nextSrc = srcMatchState->nextSrc; + dstMatchState->base = srcMatchState->base; + dstMatchState->dictBase = srcMatchState->dictBase; + dstMatchState->dictLimit = srcMatchState->dictLimit; + dstMatchState->lowLimit = srcMatchState->lowLimit; + dstMatchState->loadedDictEnd= srcMatchState->loadedDictEnd; } + dstCCtx->dictID = srcCCtx->dictID; + + /* copy block state */ + memcpy(dstCCtx->prevBlock, srcCCtx->prevBlock, sizeof(*srcCCtx->prevBlock)); return 0; } @@ -1142,20 +1162,21 @@ static void ZSTD_ldm_reduceTable(ldmEntry_t* const table, U32 const size, * rescale all indexes to avoid future overflow (indexes are U32) */ static void ZSTD_reduceIndex (ZSTD_CCtx* zc, const U32 reducerValue) { + ZSTD_matchState_t* const ms = &zc->matchState; { U32 const hSize = (U32)1 << zc->appliedParams.cParams.hashLog; - ZSTD_reduceTable(zc->hashTable, hSize, reducerValue); + ZSTD_reduceTable(ms->hashTable, hSize, reducerValue); } if (zc->appliedParams.cParams.strategy != ZSTD_fast) { U32 const chainSize = (U32)1 << zc->appliedParams.cParams.chainLog; if (zc->appliedParams.cParams.strategy == ZSTD_btlazy2) - ZSTD_preserveUnsortedMark(zc->chainTable, chainSize, reducerValue); - ZSTD_reduceTable(zc->chainTable, chainSize, reducerValue); + ZSTD_preserveUnsortedMark(ms->chainTable, chainSize, reducerValue); + ZSTD_reduceTable(ms->chainTable, chainSize, reducerValue); } - if (zc->hashLog3) { - U32 const h3Size = (U32)1 << zc->hashLog3; - ZSTD_reduceTable(zc->hashTable3, h3Size, reducerValue); + if (ms->hashLog3) { + U32 const h3Size = (U32)1 << ms->hashLog3; + ZSTD_reduceTable(ms->hashTable3, h3Size, reducerValue); } if (zc->appliedParams.ldmParams.enableLdm) { @@ -1235,10 +1256,12 @@ static size_t ZSTD_compressRleLiteralsBlock (void* dst, size_t dstCapacity, cons static size_t ZSTD_minGain(size_t srcSize) { return (srcSize >> 6) + 2; } -static size_t ZSTD_compressLiterals (ZSTD_entropyCTables_t * entropy, +static size_t ZSTD_compressLiterals (ZSTD_entropyCTables_t const* prevEntropy, + ZSTD_entropyCTables_t* nextEntropy, ZSTD_strategy strategy, void* dst, size_t dstCapacity, - const void* src, size_t srcSize) + const void* src, size_t srcSize, + U32* workspace) { size_t const minGain = ZSTD_minGain(srcSize); size_t const lhSize = 3 + (srcSize >= 1 KB) + (srcSize >= 16 KB); @@ -1247,34 +1270,45 @@ static size_t ZSTD_compressLiterals (ZSTD_entropyCTables_t * entropy, symbolEncodingType_e hType = set_compressed; size_t cLitSize; + /* Prepare nextEntropy assuming reusing the existing table */ + nextEntropy->hufCTable_repeatMode = prevEntropy->hufCTable_repeatMode; + memcpy(nextEntropy->hufCTable, prevEntropy->hufCTable, + sizeof(prevEntropy->hufCTable)); /* small ? don't even attempt compression (speed opt) */ # define LITERAL_NOENTROPY 63 - { size_t const minLitSize = entropy->hufCTable_repeatMode == HUF_repeat_valid ? 6 : LITERAL_NOENTROPY; + { size_t const minLitSize = prevEntropy->hufCTable_repeatMode == HUF_repeat_valid ? 6 : LITERAL_NOENTROPY; if (srcSize <= minLitSize) return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize); } if (dstCapacity < lhSize+1) return ERROR(dstSize_tooSmall); /* not enough space for compression */ - { HUF_repeat repeat = entropy->hufCTable_repeatMode; + { HUF_repeat repeat = prevEntropy->hufCTable_repeatMode; int const preferRepeat = strategy < ZSTD_lazy ? srcSize <= 1024 : 0; if (repeat == HUF_repeat_valid && lhSize == 3) singleStream = 1; cLitSize = singleStream ? HUF_compress1X_repeat(ostart+lhSize, dstCapacity-lhSize, src, srcSize, 255, 11, - entropy->workspace, sizeof(entropy->workspace), (HUF_CElt*)entropy->hufCTable, &repeat, preferRepeat) + workspace, HUF_WORKSPACE_SIZE, (HUF_CElt*)nextEntropy->hufCTable, &repeat, preferRepeat) : HUF_compress4X_repeat(ostart+lhSize, dstCapacity-lhSize, src, srcSize, 255, 11, - entropy->workspace, sizeof(entropy->workspace), (HUF_CElt*)entropy->hufCTable, &repeat, preferRepeat); - if (repeat != HUF_repeat_none) { hType = set_repeat; } /* reused the existing table */ - else { entropy->hufCTable_repeatMode = HUF_repeat_check; } /* now have a table to reuse */ + workspace, HUF_WORKSPACE_SIZE, (HUF_CElt*)nextEntropy->hufCTable, &repeat, preferRepeat); + if (repeat != HUF_repeat_none) { + /* reused the existing table */ + hType = set_repeat; + } } if ((cLitSize==0) | (cLitSize >= srcSize - minGain) | ERR_isError(cLitSize)) { - entropy->hufCTable_repeatMode = HUF_repeat_none; + memcpy(nextEntropy->hufCTable, prevEntropy->hufCTable, sizeof(prevEntropy->hufCTable)); return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize); } if (cLitSize==1) { - entropy->hufCTable_repeatMode = HUF_repeat_none; + memcpy(nextEntropy->hufCTable, prevEntropy->hufCTable, sizeof(prevEntropy->hufCTable)); return ZSTD_compressRleLiteralsBlock(dst, dstCapacity, src, srcSize); } + if (hType == set_compressed) { + /* using a newly constructed table */ + nextEntropy->hufCTable_repeatMode = HUF_repeat_check; + } + /* Build header */ switch(lhSize) { @@ -1368,10 +1402,11 @@ symbolEncodingType_e ZSTD_selectEncodingType( MEM_STATIC size_t ZSTD_buildCTable(void* dst, size_t dstCapacity, - FSE_CTable* CTable, U32 FSELog, symbolEncodingType_e type, + FSE_CTable* nextCTable, U32 FSELog, symbolEncodingType_e type, U32* count, U32 max, BYTE const* codeTable, size_t nbSeq, S16 const* defaultNorm, U32 defaultNormLog, U32 defaultMax, + FSE_CTable const* prevCTable, size_t prevCTableSize, void* workspace, size_t workspaceSize) { BYTE* op = (BYTE*)dst; @@ -1380,12 +1415,13 @@ size_t ZSTD_buildCTable(void* dst, size_t dstCapacity, switch (type) { case set_rle: *op = codeTable[0]; - CHECK_F(FSE_buildCTable_rle(CTable, (BYTE)max)); + CHECK_F(FSE_buildCTable_rle(nextCTable, (BYTE)max)); return 1; case set_repeat: + memcpy(nextCTable, prevCTable, prevCTableSize); return 0; case set_basic: - CHECK_F(FSE_buildCTable_wksp(CTable, defaultNorm, defaultMax, defaultNormLog, workspace, workspaceSize)); /* note : could be pre-calculated */ + CHECK_F(FSE_buildCTable_wksp(nextCTable, defaultNorm, defaultMax, defaultNormLog, workspace, workspaceSize)); /* note : could be pre-calculated */ return 0; case set_compressed: { S16 norm[MaxSeq + 1]; @@ -1399,7 +1435,7 @@ size_t ZSTD_buildCTable(void* dst, size_t dstCapacity, CHECK_F(FSE_normalizeCount(norm, tableLog, count, nbSeq_1, max)); { size_t const NCountSize = FSE_writeNCount(op, oend - op, norm, max, tableLog); /* overflow protected */ if (FSE_isError(NCountSize)) return NCountSize; - CHECK_F(FSE_buildCTable_wksp(CTable, norm, max, tableLog, workspace, workspaceSize)); + CHECK_F(FSE_buildCTable_wksp(nextCTable, norm, max, tableLog, workspace, workspaceSize)); return NCountSize; } } @@ -1492,15 +1528,16 @@ size_t ZSTD_encodeSequences( } MEM_STATIC size_t ZSTD_compressSequences_internal(seqStore_t* seqStorePtr, - ZSTD_entropyCTables_t* entropy, + ZSTD_entropyCTables_t const* prevEntropy, + ZSTD_entropyCTables_t* nextEntropy, ZSTD_compressionParameters const* cParams, - void* dst, size_t dstCapacity) + void* dst, size_t dstCapacity, U32* workspace) { const int longOffsets = cParams->windowLog > STREAM_ACCUMULATOR_MIN; U32 count[MaxSeq+1]; - FSE_CTable* CTable_LitLength = entropy->litlengthCTable; - FSE_CTable* CTable_OffsetBits = entropy->offcodeCTable; - FSE_CTable* CTable_MatchLength = entropy->matchlengthCTable; + FSE_CTable* CTable_LitLength = nextEntropy->litlengthCTable; + FSE_CTable* CTable_OffsetBits = nextEntropy->offcodeCTable; + FSE_CTable* CTable_MatchLength = nextEntropy->matchlengthCTable; U32 LLtype, Offtype, MLtype; /* compressed, raw or rle */ const seqDef* const sequences = seqStorePtr->sequencesStart; const BYTE* const ofCodeTable = seqStorePtr->ofCode; @@ -1512,13 +1549,14 @@ MEM_STATIC size_t ZSTD_compressSequences_internal(seqStore_t* seqStorePtr, size_t const nbSeq = seqStorePtr->sequences - seqStorePtr->sequencesStart; BYTE* seqHead; - ZSTD_STATIC_ASSERT(sizeof(entropy->workspace) >= (1<= (1<litStart; size_t const litSize = seqStorePtr->lit - literals; - size_t const cSize = ZSTD_compressLiterals( - entropy, cParams->strategy, op, dstCapacity, literals, litSize); + size_t const cSize = ZSTD_compressLiterals(prevEntropy, nextEntropy, + cParams->strategy, op, dstCapacity, literals, litSize, + workspace); if (ZSTD_isError(cSize)) return cSize; assert(cSize <= dstCapacity); @@ -1533,7 +1571,15 @@ MEM_STATIC size_t ZSTD_compressSequences_internal(seqStore_t* seqStorePtr, op[0] = (BYTE)((nbSeq>>8) + 0x80), op[1] = (BYTE)nbSeq, op+=2; else op[0]=0xFF, MEM_writeLE16(op+1, (U16)(nbSeq - LONGNBSEQ)), op+=3; - if (nbSeq==0) return op - ostart; + if (nbSeq==0) { + memcpy(nextEntropy->litlengthCTable, prevEntropy->litlengthCTable, sizeof(prevEntropy->litlengthCTable)); + nextEntropy->litlength_repeatMode = prevEntropy->litlength_repeatMode; + memcpy(nextEntropy->offcodeCTable, prevEntropy->offcodeCTable, sizeof(prevEntropy->offcodeCTable)); + nextEntropy->offcode_repeatMode = prevEntropy->offcode_repeatMode; + memcpy(nextEntropy->matchlengthCTable, prevEntropy->matchlengthCTable, sizeof(prevEntropy->matchlengthCTable)); + nextEntropy->matchlength_repeatMode = prevEntropy->matchlength_repeatMode; + return op - ostart; + } /* seqHead : flags for FSE encoding type */ seqHead = op++; @@ -1542,36 +1588,42 @@ MEM_STATIC size_t ZSTD_compressSequences_internal(seqStore_t* seqStorePtr, ZSTD_seqToCodes(seqStorePtr); /* build CTable for Literal Lengths */ { U32 max = MaxLL; - size_t const mostFrequent = FSE_countFast_wksp(count, &max, llCodeTable, nbSeq, entropy->workspace); + size_t const mostFrequent = FSE_countFast_wksp(count, &max, llCodeTable, nbSeq, workspace); DEBUGLOG(5, "Building LL table"); - LLtype = ZSTD_selectEncodingType(&entropy->litlength_repeatMode, mostFrequent, nbSeq, LL_defaultNormLog, ZSTD_defaultAllowed); + nextEntropy->litlength_repeatMode = prevEntropy->litlength_repeatMode; + LLtype = ZSTD_selectEncodingType(&nextEntropy->litlength_repeatMode, mostFrequent, nbSeq, LL_defaultNormLog, ZSTD_defaultAllowed); { size_t const countSize = ZSTD_buildCTable(op, oend - op, CTable_LitLength, LLFSELog, (symbolEncodingType_e)LLtype, count, max, llCodeTable, nbSeq, LL_defaultNorm, LL_defaultNormLog, MaxLL, - entropy->workspace, sizeof(entropy->workspace)); + prevEntropy->litlengthCTable, sizeof(prevEntropy->litlengthCTable), + workspace, HUF_WORKSPACE_SIZE); if (ZSTD_isError(countSize)) return countSize; op += countSize; } } /* build CTable for Offsets */ { U32 max = MaxOff; - size_t const mostFrequent = FSE_countFast_wksp(count, &max, ofCodeTable, nbSeq, entropy->workspace); + size_t const mostFrequent = FSE_countFast_wksp(count, &max, ofCodeTable, nbSeq, workspace); /* We can only use the basic table if max <= DefaultMaxOff, otherwise the offsets are too large */ ZSTD_defaultPolicy_e const defaultPolicy = (max <= DefaultMaxOff) ? ZSTD_defaultAllowed : ZSTD_defaultDisallowed; DEBUGLOG(5, "Building OF table"); - Offtype = ZSTD_selectEncodingType(&entropy->offcode_repeatMode, mostFrequent, nbSeq, OF_defaultNormLog, defaultPolicy); + nextEntropy->offcode_repeatMode = prevEntropy->offcode_repeatMode; + Offtype = ZSTD_selectEncodingType(&nextEntropy->offcode_repeatMode, mostFrequent, nbSeq, OF_defaultNormLog, defaultPolicy); { size_t const countSize = ZSTD_buildCTable(op, oend - op, CTable_OffsetBits, OffFSELog, (symbolEncodingType_e)Offtype, count, max, ofCodeTable, nbSeq, OF_defaultNorm, OF_defaultNormLog, DefaultMaxOff, - entropy->workspace, sizeof(entropy->workspace)); + prevEntropy->offcodeCTable, sizeof(prevEntropy->offcodeCTable), + workspace, HUF_WORKSPACE_SIZE); if (ZSTD_isError(countSize)) return countSize; op += countSize; } } /* build CTable for MatchLengths */ { U32 max = MaxML; - size_t const mostFrequent = FSE_countFast_wksp(count, &max, mlCodeTable, nbSeq, entropy->workspace); + size_t const mostFrequent = FSE_countFast_wksp(count, &max, mlCodeTable, nbSeq, workspace); DEBUGLOG(5, "Building ML table"); - MLtype = ZSTD_selectEncodingType(&entropy->matchlength_repeatMode, mostFrequent, nbSeq, ML_defaultNormLog, ZSTD_defaultAllowed); + nextEntropy->matchlength_repeatMode = prevEntropy->matchlength_repeatMode; + MLtype = ZSTD_selectEncodingType(&nextEntropy->matchlength_repeatMode, mostFrequent, nbSeq, ML_defaultNormLog, ZSTD_defaultAllowed); { size_t const countSize = ZSTD_buildCTable(op, oend - op, CTable_MatchLength, MLFSELog, (symbolEncodingType_e)MLtype, count, max, mlCodeTable, nbSeq, ML_defaultNorm, ML_defaultNormLog, MaxML, - entropy->workspace, sizeof(entropy->workspace)); + prevEntropy->matchlengthCTable, sizeof(prevEntropy->matchlengthCTable), + workspace, HUF_WORKSPACE_SIZE); if (ZSTD_isError(countSize)) return countSize; op += countSize; } } @@ -1593,13 +1645,15 @@ MEM_STATIC size_t ZSTD_compressSequences_internal(seqStore_t* seqStorePtr, } MEM_STATIC size_t ZSTD_compressSequences(seqStore_t* seqStorePtr, - ZSTD_entropyCTables_t* entropy, + ZSTD_entropyCTables_t const* prevEntropy, + ZSTD_entropyCTables_t* nextEntropy, ZSTD_compressionParameters const* cParams, void* dst, size_t dstCapacity, - size_t srcSize) + size_t srcSize, U32* workspace) { - size_t const cSize = ZSTD_compressSequences_internal(seqStorePtr, entropy, cParams, - dst, dstCapacity); + size_t const cSize = ZSTD_compressSequences_internal( + seqStorePtr, prevEntropy, nextEntropy, cParams, dst, dstCapacity, + workspace); /* If the srcSize <= dstCapacity, then there is enough space to write a * raw uncompressed block. Since we ran out of space, the block must not * be compressible, so fall back to a raw uncompressed block. @@ -1607,34 +1661,28 @@ MEM_STATIC size_t ZSTD_compressSequences(seqStore_t* seqStorePtr, int const uncompressibleError = (cSize == ERROR(dstSize_tooSmall)) && (srcSize <= dstCapacity); if (ZSTD_isError(cSize) && !uncompressibleError) return cSize; - /* We check that dictionaries have offset codes available for the first - * block. After the first block, the offcode table might not have large - * enough codes to represent the offsets in the data. - */ - if (entropy->offcode_repeatMode == FSE_repeat_valid) - entropy->offcode_repeatMode = FSE_repeat_check; /* Check compressibility */ { size_t const minGain = ZSTD_minGain(srcSize); /* note : fixed formula, maybe should depend on compression level, or strategy */ size_t const maxCSize = srcSize - minGain; if (cSize >= maxCSize || uncompressibleError) { - entropy->hufCTable_repeatMode = HUF_repeat_none; - entropy->offcode_repeatMode = FSE_repeat_none; - entropy->matchlength_repeatMode = FSE_repeat_none; - entropy->litlength_repeatMode = FSE_repeat_none; return 0; /* block not compressed */ } } assert(!ZSTD_isError(cSize)); - /* block is compressed => confirm repcodes in history */ - { int i; for (i=0; irep[i] = seqStorePtr->repToConfirm[i]; } + /* We check that dictionaries have offset codes available for the first + * block. After the first block, the offcode table might not have large + * enough codes to represent the offsets in the data. + */ + if (nextEntropy->offcode_repeatMode == FSE_repeat_valid) + nextEntropy->offcode_repeatMode = FSE_repeat_check; + return cSize; } /* ZSTD_selectBlockCompressor() : * Not static, but internal use only (used by long distance matcher) * assumption : strat is a valid strategy */ -typedef size_t (*ZSTD_blockCompressor) (ZSTD_CCtx* ctx, const void* src, size_t srcSize); ZSTD_blockCompressor ZSTD_selectBlockCompressor(ZSTD_strategy strat, int extDict) { static const ZSTD_blockCompressor blockCompressor[2][(unsigned)ZSTD_btultra+1] = { @@ -1673,30 +1721,54 @@ static size_t ZSTD_compressBlock_internal(ZSTD_CCtx* zc, const void* src, size_t srcSize) { DEBUGLOG(5, "ZSTD_compressBlock_internal (dstCapacity=%u) (dictLimit=%u, nextToUpdate=%u)", - (U32)dstCapacity, zc->dictLimit, zc->nextToUpdate); + (U32)dstCapacity, zc->matchState.dictLimit, zc->matchState.nextToUpdate); if (srcSize < MIN_CBLOCK_SIZE+ZSTD_blockHeaderSize+1) return 0; /* don't even attempt compression below a certain srcSize */ ZSTD_resetSeqStore(&(zc->seqStore)); /* limited update after a very long match */ - { const BYTE* const base = zc->base; + { const BYTE* const base = zc->matchState.base; const BYTE* const istart = (const BYTE*)src; const U32 current = (U32)(istart-base); - if (current > zc->nextToUpdate + 384) - zc->nextToUpdate = current - MIN(192, (U32)(current - zc->nextToUpdate - 384)); + if (current > zc->matchState.nextToUpdate + 384) + zc->matchState.nextToUpdate = current - MIN(192, (U32)(current - zc->matchState.nextToUpdate - 384)); } /* find and store sequences */ - { U32 const extDict = zc->lowLimit < zc->dictLimit; - const ZSTD_blockCompressor blockCompressor = - zc->appliedParams.ldmParams.enableLdm - ? (extDict ? ZSTD_compressBlock_ldm_extDict : ZSTD_compressBlock_ldm) - : ZSTD_selectBlockCompressor(zc->appliedParams.cParams.strategy, extDict); - size_t const lastLLSize = blockCompressor(zc, src, srcSize); - const BYTE* const anchor = (const BYTE*)src + srcSize - lastLLSize; - ZSTD_storeLastLiterals(&zc->seqStore, anchor, lastLLSize); + { + U32 const extDict = zc->matchState.lowLimit < zc->matchState.dictLimit; + size_t lastLLSize; + { int i; for (i = 0; i < ZSTD_REP_NUM; ++i) zc->nextBlock->rep[i] = zc->prevBlock->rep[i]; } + if (zc->appliedParams.ldmParams.enableLdm) { + typedef size_t (*ZSTD_ldmBlockCompressor)( + ldmState_t* ldms, ZSTD_matchState_t* ms, seqStore_t* seqStore, + U32 rep[ZSTD_REP_NUM], ZSTD_CCtx_params const* params, + void const* src, size_t srcSize); + ZSTD_ldmBlockCompressor const ldmBlockCompressor = extDict ? ZSTD_compressBlock_ldm_extDict : ZSTD_compressBlock_ldm; + + lastLLSize = ldmBlockCompressor(&zc->ldmState, &zc->matchState, &zc->seqStore, zc->nextBlock->rep, &zc->appliedParams, src, srcSize); + } else { + ZSTD_blockCompressor const blockCompressor = ZSTD_selectBlockCompressor(zc->appliedParams.cParams.strategy, extDict); + + lastLLSize = blockCompressor(&zc->matchState, &zc->seqStore, zc->nextBlock->rep, &zc->appliedParams.cParams, src, srcSize); + } + { + const BYTE* const anchor = (const BYTE*)src + srcSize - lastLLSize; + ZSTD_storeLastLiterals(&zc->seqStore, anchor, lastLLSize); + } } /* encode */ - return ZSTD_compressSequences(&zc->seqStore, zc->entropy, &zc->appliedParams.cParams, dst, dstCapacity, srcSize); + { + size_t const cSize = ZSTD_compressSequences(&zc->seqStore, &zc->prevBlock->entropy, &zc->nextBlock->entropy, &zc->appliedParams.cParams, dst, dstCapacity, srcSize, zc->entropyWorkspace); + if (ZSTD_isError(cSize) || cSize == 0) + return cSize; + /* confirm repcodes and entropy tables */ + { + ZSTD_blockState_t* const tmp = zc->prevBlock; + zc->prevBlock = zc->nextBlock; + zc->nextBlock = tmp; + } + return cSize; + } } @@ -1725,6 +1797,7 @@ static size_t ZSTD_compress_frameChunk (ZSTD_CCtx* cctx, XXH64_update(&cctx->xxhState, src, srcSize); while (remaining) { + ZSTD_matchState_t* const ms = &cctx->matchState; U32 const lastBlock = lastFrameChunk & (blockSize >= remaining); if (dstCapacity < ZSTD_blockHeaderSize + MIN_CBLOCK_SIZE) @@ -1748,9 +1821,9 @@ static size_t ZSTD_compress_frameChunk (ZSTD_CCtx* cctx, * 3. cctx->lowLimit < 1<<32: * windowLog <= 31 ==> 3<<29 + 1<lowLimit > (3U<<29)) { + if (ms->lowLimit > (3U<<29)) { U32 const cycleMask = ((U32)1 << ZSTD_cycleLog(cctx->appliedParams.cParams.chainLog, cctx->appliedParams.cParams.strategy)) - 1; - U32 const current = (U32)(ip - cctx->base); + U32 const current = (U32)(ip - cctx->matchState.base); U32 const newCurrent = (current & cycleMask) + ((U32)1 << cctx->appliedParams.cParams.windowLog); U32 const correction = current - newCurrent; ZSTD_STATIC_ASSERT(ZSTD_CHAINLOG_MAX <= 30); @@ -1759,23 +1832,23 @@ static size_t ZSTD_compress_frameChunk (ZSTD_CCtx* cctx, assert(current > newCurrent); assert(correction > 1<<28); /* Loose bound, should be about 1<<29 */ ZSTD_reduceIndex(cctx, correction); - cctx->base += correction; - cctx->dictBase += correction; - cctx->lowLimit -= correction; - cctx->dictLimit -= correction; - if (cctx->nextToUpdate < correction) cctx->nextToUpdate = 0; - else cctx->nextToUpdate -= correction; - DEBUGLOG(4, "Correction of 0x%x bytes to lowLimit=0x%x", correction, cctx->lowLimit); + ms->base += correction; + ms->dictBase += correction; + ms->lowLimit -= correction; + ms->dictLimit -= correction; + if (ms->nextToUpdate < correction) ms->nextToUpdate = 0; + else ms->nextToUpdate -= correction; + DEBUGLOG(4, "Correction of 0x%x bytes to lowLimit=0x%x", correction, ms->lowLimit); } /* enforce maxDist */ - if ((U32)(ip+blockSize - cctx->base) > cctx->loadedDictEnd + maxDist) { - U32 const newLowLimit = (U32)(ip+blockSize - cctx->base) - maxDist; - if (cctx->lowLimit < newLowLimit) cctx->lowLimit = newLowLimit; - if (cctx->dictLimit < cctx->lowLimit) + if ((U32)(ip+blockSize - ms->base) > ms->loadedDictEnd + maxDist) { + U32 const newLowLimit = (U32)(ip+blockSize - ms->base) - maxDist; + if (ms->lowLimit < newLowLimit) ms->lowLimit = newLowLimit; + if (ms->dictLimit < ms->lowLimit) DEBUGLOG(5, "ZSTD_compress_frameChunk : update dictLimit from %u to %u ", - cctx->dictLimit, cctx->lowLimit); - if (cctx->dictLimit < cctx->lowLimit) cctx->dictLimit = cctx->lowLimit; - if (cctx->nextToUpdate < cctx->lowLimit) cctx->nextToUpdate = cctx->lowLimit; + ms->dictLimit, ms->lowLimit); + if (ms->dictLimit < ms->lowLimit) ms->dictLimit = ms->lowLimit; + if (ms->nextToUpdate < ms->lowLimit) ms->nextToUpdate = ms->lowLimit; } { size_t cSize = ZSTD_compressBlock_internal(cctx, @@ -1854,12 +1927,38 @@ static size_t ZSTD_writeFrameHeader(void* dst, size_t dstCapacity, } +static void ZSTD_manageWindowContinuity(ZSTD_matchState_t* ms, void const* src, size_t srcSize) +{ + const BYTE* const ip = (const BYTE*) src; + + /* Check if blocks follow each other */ + if (src != ms->nextSrc) { + /* not contiguous */ + size_t const distanceFromBase = (size_t)(ms->nextSrc - ms->base); + DEBUGLOG(5, "ZSTD_manageWindowContinuity: non contiguous blocks, new segment starts at %u", ms->dictLimit); + ms->lowLimit = ms->dictLimit; + assert(distanceFromBase == (size_t)(U32)distanceFromBase); /* should never overflow */ + ms->dictLimit = (U32)distanceFromBase; + ms->dictBase = ms->base; + ms->base = ip - distanceFromBase; + ms->nextToUpdate = ms->dictLimit; + if (ms->dictLimit - ms->lowLimit < HASH_READ_SIZE) ms->lowLimit = ms->dictLimit; /* too small extDict */ + } + ms->nextSrc = ip + srcSize; + /* if input and dictionary overlap : reduce dictionary (area presumed modified by input) */ + if ((ip+srcSize > ms->dictBase + ms->lowLimit) & (ip < ms->dictBase + ms->dictLimit)) { + ptrdiff_t const highInputIdx = (ip + srcSize) - ms->dictBase; + U32 const lowLimitMax = (highInputIdx > (ptrdiff_t)ms->dictLimit) ? ms->dictLimit : (U32)highInputIdx; + ms->lowLimit = lowLimitMax; + } +} + + static size_t ZSTD_compressContinue_internal (ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, U32 frame, U32 lastFrameChunk) { - const BYTE* const ip = (const BYTE*) src; size_t fhSize = 0; DEBUGLOG(5, "ZSTD_compressContinue_internal, stage: %u", cctx->stage); @@ -1876,27 +1975,7 @@ static size_t ZSTD_compressContinue_internal (ZSTD_CCtx* cctx, if (!srcSize) return fhSize; /* do not generate an empty block if no input */ - /* Check if blocks follow each other */ - if (src != cctx->nextSrc) { - size_t const distanceFromBase = (size_t)(cctx->nextSrc - cctx->base); - DEBUGLOG(5, "ZSTD_compressContinue_internal: non contiguous blocks, new segment starts at %u", - cctx->dictLimit); - cctx->lowLimit = cctx->dictLimit; - assert(distanceFromBase == (size_t)(U32)distanceFromBase); /* should never overflow */ - cctx->dictLimit = (U32)distanceFromBase; - cctx->dictBase = cctx->base; - cctx->base = ip - distanceFromBase; - cctx->nextToUpdate = cctx->dictLimit; - if (cctx->dictLimit - cctx->lowLimit < HASH_READ_SIZE) cctx->lowLimit = cctx->dictLimit; /* too small extDict */ - } - cctx->nextSrc = ip + srcSize; - - /* if input and dictionary overlap : reduce dictionary (area presumed modified by input) */ - if ((ip+srcSize > cctx->dictBase + cctx->lowLimit) & (ip < cctx->dictBase + cctx->dictLimit)) { - ptrdiff_t const highInputIdx = (ip + srcSize) - cctx->dictBase; - U32 const lowLimitMax = (highInputIdx > (ptrdiff_t)cctx->dictLimit) ? cctx->dictLimit : (U32)highInputIdx; - cctx->lowLimit = lowLimitMax; - } + ZSTD_manageWindowContinuity(&cctx->matchState, src, srcSize); DEBUGLOG(5, "ZSTD_compressContinue_internal (blockSize=%u)", (U32)cctx->blockSize); { size_t const cSize = frame ? @@ -1934,50 +2013,51 @@ size_t ZSTD_compressBlock(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const /*! ZSTD_loadDictionaryContent() : * @return : 0, or an error code */ -static size_t ZSTD_loadDictionaryContent(ZSTD_CCtx* zc, const void* src, size_t srcSize) +static size_t ZSTD_loadDictionaryContent(ZSTD_matchState_t* ms, ZSTD_CCtx_params const* params, const void* src, size_t srcSize) { const BYTE* const ip = (const BYTE*) src; const BYTE* const iend = ip + srcSize; + ZSTD_compressionParameters const* cParams = ¶ms->cParams; /* input becomes current prefix */ - zc->lowLimit = zc->dictLimit; - zc->dictLimit = (U32)(zc->nextSrc - zc->base); - zc->dictBase = zc->base; - zc->base = ip - zc->dictLimit; - zc->nextToUpdate = zc->dictLimit; - zc->loadedDictEnd = zc->appliedParams.forceWindow ? 0 : (U32)(iend - zc->base); + ms->lowLimit = ms->dictLimit; + ms->dictLimit = (U32)(ms->nextSrc - ms->base); + ms->dictBase = ms->base; + ms->base = ip - ms->dictLimit; + ms->nextToUpdate = ms->dictLimit; + ms->loadedDictEnd = params->forceWindow ? 0 : (U32)(iend - ms->base); - zc->nextSrc = iend; + ms->nextSrc = iend; if (srcSize <= HASH_READ_SIZE) return 0; - switch(zc->appliedParams.cParams.strategy) + switch(params->cParams.strategy) { case ZSTD_fast: - ZSTD_fillHashTable (zc, iend, zc->appliedParams.cParams.searchLength); + ZSTD_fillHashTable(ms, cParams, iend); break; case ZSTD_dfast: - ZSTD_fillDoubleHashTable (zc, iend, zc->appliedParams.cParams.searchLength); + ZSTD_fillDoubleHashTable(ms, cParams, iend); break; case ZSTD_greedy: case ZSTD_lazy: case ZSTD_lazy2: if (srcSize >= HASH_READ_SIZE) - ZSTD_insertAndFindFirstIndex(zc, iend-HASH_READ_SIZE, zc->appliedParams.cParams.searchLength); + ZSTD_insertAndFindFirstIndex(ms, cParams, iend-HASH_READ_SIZE); break; case ZSTD_btlazy2: /* we want the dictionary table fully sorted */ case ZSTD_btopt: case ZSTD_btultra: if (srcSize >= HASH_READ_SIZE) - ZSTD_updateTree(zc, iend-HASH_READ_SIZE, iend, (U32)1 << zc->appliedParams.cParams.searchLog, zc->appliedParams.cParams.searchLength); + ZSTD_updateTree(ms, cParams, iend-HASH_READ_SIZE, iend); break; default: assert(0); /* not possible : not a valid strategy id */ } - zc->nextToUpdate = (U32)(iend - zc->base); + ms->nextToUpdate = (U32)(iend - ms->base); return 0; } @@ -2001,25 +2081,26 @@ static size_t ZSTD_checkDictNCount(short* normalizedCounter, unsigned dictMaxSym * https://github.com/facebook/zstd/blob/master/doc/zstd_compression_format.md#dictionary-format */ /*! ZSTD_loadZstdDictionary() : - * @return : 0, or an error code + * @return : dictID, or an error code * assumptions : magic number supposed already checked * dictSize supposed > 8 */ -static size_t ZSTD_loadZstdDictionary(ZSTD_CCtx* cctx, const void* dict, size_t dictSize) +static size_t ZSTD_loadZstdDictionary(ZSTD_blockState_t* bs, ZSTD_matchState_t* ms, ZSTD_CCtx_params const* params, const void* dict, size_t dictSize, void* workspace) { const BYTE* dictPtr = (const BYTE*)dict; const BYTE* const dictEnd = dictPtr + dictSize; short offcodeNCount[MaxOff+1]; unsigned offcodeMaxValue = MaxOff; + size_t dictID; - ZSTD_STATIC_ASSERT(sizeof(cctx->entropy->workspace) >= (1<= (1<dictID = cctx->appliedParams.fParams.noDictIDFlag ? 0 : MEM_readLE32(dictPtr); + dictID = params->fParams.noDictIDFlag ? 0 : MEM_readLE32(dictPtr); dictPtr += 4; { unsigned maxSymbolValue = 255; - size_t const hufHeaderSize = HUF_readCTable((HUF_CElt*)cctx->entropy->hufCTable, &maxSymbolValue, dictPtr, dictEnd-dictPtr); + size_t const hufHeaderSize = HUF_readCTable((HUF_CElt*)bs->entropy.hufCTable, &maxSymbolValue, dictPtr, dictEnd-dictPtr); if (HUF_isError(hufHeaderSize)) return ERROR(dictionary_corrupted); if (maxSymbolValue < 255) return ERROR(dictionary_corrupted); dictPtr += hufHeaderSize; @@ -2030,7 +2111,7 @@ static size_t ZSTD_loadZstdDictionary(ZSTD_CCtx* cctx, const void* dict, size_t if (FSE_isError(offcodeHeaderSize)) return ERROR(dictionary_corrupted); if (offcodeLog > OffFSELog) return ERROR(dictionary_corrupted); /* Defer checking offcodeMaxValue because we need to know the size of the dictionary content */ - CHECK_E( FSE_buildCTable_wksp(cctx->entropy->offcodeCTable, offcodeNCount, offcodeMaxValue, offcodeLog, cctx->entropy->workspace, sizeof(cctx->entropy->workspace)), + CHECK_E( FSE_buildCTable_wksp(bs->entropy.offcodeCTable, offcodeNCount, offcodeMaxValue, offcodeLog, workspace, HUF_WORKSPACE_SIZE), dictionary_corrupted); dictPtr += offcodeHeaderSize; } @@ -2042,7 +2123,7 @@ static size_t ZSTD_loadZstdDictionary(ZSTD_CCtx* cctx, const void* dict, size_t if (matchlengthLog > MLFSELog) return ERROR(dictionary_corrupted); /* Every match length code must have non-zero probability */ CHECK_F( ZSTD_checkDictNCount(matchlengthNCount, matchlengthMaxValue, MaxML)); - CHECK_E( FSE_buildCTable_wksp(cctx->entropy->matchlengthCTable, matchlengthNCount, matchlengthMaxValue, matchlengthLog, cctx->entropy->workspace, sizeof(cctx->entropy->workspace)), + CHECK_E( FSE_buildCTable_wksp(bs->entropy.matchlengthCTable, matchlengthNCount, matchlengthMaxValue, matchlengthLog, workspace, HUF_WORKSPACE_SIZE), dictionary_corrupted); dictPtr += matchlengthHeaderSize; } @@ -2054,15 +2135,15 @@ static size_t ZSTD_loadZstdDictionary(ZSTD_CCtx* cctx, const void* dict, size_t if (litlengthLog > LLFSELog) return ERROR(dictionary_corrupted); /* Every literal length code must have non-zero probability */ CHECK_F( ZSTD_checkDictNCount(litlengthNCount, litlengthMaxValue, MaxLL)); - CHECK_E( FSE_buildCTable_wksp(cctx->entropy->litlengthCTable, litlengthNCount, litlengthMaxValue, litlengthLog, cctx->entropy->workspace, sizeof(cctx->entropy->workspace)), + CHECK_E( FSE_buildCTable_wksp(bs->entropy.litlengthCTable, litlengthNCount, litlengthMaxValue, litlengthLog, workspace, HUF_WORKSPACE_SIZE), dictionary_corrupted); dictPtr += litlengthHeaderSize; } if (dictPtr+12 > dictEnd) return ERROR(dictionary_corrupted); - cctx->seqStore.rep[0] = MEM_readLE32(dictPtr+0); - cctx->seqStore.rep[1] = MEM_readLE32(dictPtr+4); - cctx->seqStore.rep[2] = MEM_readLE32(dictPtr+8); + bs->rep[0] = MEM_readLE32(dictPtr+0); + bs->rep[1] = MEM_readLE32(dictPtr+4); + bs->rep[2] = MEM_readLE32(dictPtr+8); dictPtr += 12; { size_t const dictContentSize = (size_t)(dictEnd - dictPtr); @@ -2076,15 +2157,16 @@ static size_t ZSTD_loadZstdDictionary(ZSTD_CCtx* cctx, const void* dict, size_t /* All repCodes must be <= dictContentSize and != 0*/ { U32 u; for (u=0; u<3; u++) { - if (cctx->seqStore.rep[u] == 0) return ERROR(dictionary_corrupted); - if (cctx->seqStore.rep[u] > dictContentSize) return ERROR(dictionary_corrupted); + if (bs->rep[u] == 0) return ERROR(dictionary_corrupted); + if (bs->rep[u] > dictContentSize) return ERROR(dictionary_corrupted); } } - cctx->entropy->hufCTable_repeatMode = HUF_repeat_valid; - cctx->entropy->offcode_repeatMode = FSE_repeat_valid; - cctx->entropy->matchlength_repeatMode = FSE_repeat_valid; - cctx->entropy->litlength_repeatMode = FSE_repeat_valid; - return ZSTD_loadDictionaryContent(cctx, dictPtr, dictContentSize); + bs->entropy.hufCTable_repeatMode = HUF_repeat_valid; + bs->entropy.offcode_repeatMode = FSE_repeat_valid; + bs->entropy.matchlength_repeatMode = FSE_repeat_valid; + bs->entropy.litlength_repeatMode = FSE_repeat_valid; + CHECK_F(ZSTD_loadDictionaryContent(ms, params, dictPtr, dictContentSize)); + return dictID; } } @@ -2099,12 +2181,12 @@ static size_t ZSTD_compress_insertDictionary(ZSTD_CCtx* cctx, /* dict restricted modes */ if (dictMode==ZSTD_dm_rawContent) - return ZSTD_loadDictionaryContent(cctx, dict, dictSize); + return ZSTD_loadDictionaryContent(&cctx->matchState, &cctx->appliedParams, dict, dictSize); if (MEM_readLE32(dict) != ZSTD_MAGIC_DICTIONARY) { if (dictMode == ZSTD_dm_auto) { DEBUGLOG(4, "raw content dictionary detected"); - return ZSTD_loadDictionaryContent(cctx, dict, dictSize); + return ZSTD_loadDictionaryContent(&cctx->matchState, &cctx->appliedParams, dict, dictSize); } if (dictMode == ZSTD_dm_fullDict) return ERROR(dictionary_wrong); @@ -2112,7 +2194,12 @@ static size_t ZSTD_compress_insertDictionary(ZSTD_CCtx* cctx, } /* dict as full zstd dictionary */ - return ZSTD_loadZstdDictionary(cctx, dict, dictSize); + { + size_t const dictID = ZSTD_loadZstdDictionary(cctx->prevBlock, &cctx->matchState, &cctx->appliedParams, dict, dictSize, cctx->entropyWorkspace); + if (ZSTD_isError(dictID)) return dictID; + cctx->dictID = (U32)dictID; + } + return 0; } /*! ZSTD_compressBegin_internal() : diff --git a/lib/compress/zstd_compress_internal.h b/lib/compress/zstd_compress_internal.h index f104fe98..7a7b1db1 100644 --- a/lib/compress/zstd_compress_internal.h +++ b/lib/compress/zstd_compress_internal.h @@ -51,13 +51,17 @@ typedef struct { FSE_CTable offcodeCTable[FSE_CTABLE_SIZE_U32(OffFSELog, MaxOff)]; FSE_CTable matchlengthCTable[FSE_CTABLE_SIZE_U32(MLFSELog, MaxML)]; FSE_CTable litlengthCTable[FSE_CTABLE_SIZE_U32(LLFSELog, MaxLL)]; - U32 workspace[HUF_WORKSPACE_SIZE_U32]; HUF_repeat hufCTable_repeatMode; FSE_repeat offcode_repeatMode; FSE_repeat matchlength_repeatMode; FSE_repeat litlength_repeatMode; } ZSTD_entropyCTables_t; +typedef struct { + ZSTD_entropyCTables_t entropy; + U32 rep[ZSTD_REP_NUM]; +} ZSTD_blockState_t; + typedef struct { U32 off; U32 len; @@ -93,6 +97,22 @@ typedef struct { U32 staticPrices; /* prices follow a pre-defined cost structure, statistics are irrelevant */ } optState_t; +typedef struct { + BYTE const* nextSrc; /* next block here to continue on current prefix */ /* TODO: Does this belong here? */ + BYTE const* base; /* All regular indexes relative to this position */ + BYTE const* dictBase; /* extDict indexes relative to this position */ + U32 dictLimit; /* below that point, need extDict */ + U32 lowLimit; /* below that point, no more data */ + U32 nextToUpdate; /* index from which to continue table update */ + U32 nextToUpdate3; /* index from which to continue table update */ + U32 hashLog3; /* dispatch table : larger == faster, more memory */ + U32 loadedDictEnd; /* index of end of dictionary */ /* TODO: Does this belong here? */ + U32* hashTable; + U32* hashTable3; + U32* chainTable; + optState_t opt; /* optimal parser state */ +} ZSTD_matchState_t; + typedef struct { U32 offset; U32 checksum; @@ -136,15 +156,6 @@ struct ZSTD_CCtx_params_s { }; /* typedef'd to ZSTD_CCtx_params within "zstd.h" */ struct ZSTD_CCtx_s { - const BYTE* nextSrc; /* next block here to continue on current prefix */ - const BYTE* base; /* All regular indexes relative to this position */ - const BYTE* dictBase; /* extDict indexes relative to this position */ - U32 dictLimit; /* below that point, need extDict */ - U32 lowLimit; /* below that point, no more data */ - U32 nextToUpdate; /* index from which to continue dictionary update */ - U32 nextToUpdate3; /* index from which to continue dictionary update */ - U32 hashLog3; /* dispatch table : larger == faster, more memory */ - U32 loadedDictEnd; /* index of end of dictionary */ ZSTD_compressionStage_e stage; U32 dictID; ZSTD_CCtx_params requestedParams; @@ -159,12 +170,11 @@ struct ZSTD_CCtx_s { size_t staticSize; seqStore_t seqStore; /* sequences storage ptrs */ - optState_t optState; ldmState_t ldmState; /* long distance matching state */ - U32* hashTable; - U32* hashTable3; - U32* chainTable; - ZSTD_entropyCTables_t* entropy; + ZSTD_matchState_t matchState; + ZSTD_blockState_t* prevBlock; + ZSTD_blockState_t* nextBlock; + U32* entropyWorkspace; /* entropy workspace of HUF_WORKSPACE_SIZE bytes */ /* streaming */ char* inBuff; @@ -191,6 +201,12 @@ struct ZSTD_CCtx_s { }; +typedef size_t (*ZSTD_blockCompressor) ( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize); +ZSTD_blockCompressor ZSTD_selectBlockCompressor(ZSTD_strategy strat, int extDict); + + MEM_STATIC U32 ZSTD_LLcode(U32 litLength) { static const BYTE LL_Code[64] = { 0, 1, 2, 3, 4, 5, 6, 7, diff --git a/lib/compress/zstd_double_fast.c b/lib/compress/zstd_double_fast.c index a6d8ad88..834a64b0 100644 --- a/lib/compress/zstd_double_fast.c +++ b/lib/compress/zstd_double_fast.c @@ -12,14 +12,17 @@ #include "zstd_double_fast.h" -void ZSTD_fillDoubleHashTable(ZSTD_CCtx* cctx, const void* end, const U32 mls) +void ZSTD_fillDoubleHashTable(ZSTD_matchState_t* ms, + ZSTD_compressionParameters const* cParams, + void const* end) { - U32* const hashLarge = cctx->hashTable; - U32 const hBitsL = cctx->appliedParams.cParams.hashLog; - U32* const hashSmall = cctx->chainTable; - U32 const hBitsS = cctx->appliedParams.cParams.chainLog; - const BYTE* const base = cctx->base; - const BYTE* ip = base + cctx->nextToUpdate; + U32* const hashLarge = ms->hashTable; + U32 const hBitsL = cParams->hashLog; + U32 const mls = cParams->searchLength; + U32* const hashSmall = ms->chainTable; + U32 const hBitsS = cParams->chainLog; + const BYTE* const base = ms->base; + const BYTE* ip = base + ms->nextToUpdate; const BYTE* const iend = ((const BYTE*)end) - HASH_READ_SIZE; const U32 fastHashFillStep = 3; @@ -43,24 +46,24 @@ void ZSTD_fillDoubleHashTable(ZSTD_CCtx* cctx, const void* end, const U32 mls) FORCE_INLINE_TEMPLATE -size_t ZSTD_compressBlock_doubleFast_generic(ZSTD_CCtx* cctx, - const void* src, size_t srcSize, - const U32 mls) +size_t ZSTD_compressBlock_doubleFast_generic( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize, + U32 const mls /* template */) { - U32* const hashLong = cctx->hashTable; - const U32 hBitsL = cctx->appliedParams.cParams.hashLog; - U32* const hashSmall = cctx->chainTable; - const U32 hBitsS = cctx->appliedParams.cParams.chainLog; - seqStore_t* seqStorePtr = &(cctx->seqStore); - const BYTE* const base = cctx->base; + U32* const hashLong = ms->hashTable; + const U32 hBitsL = cParams->hashLog; + U32* const hashSmall = ms->chainTable; + const U32 hBitsS = cParams->chainLog; + const BYTE* const base = ms->base; const BYTE* const istart = (const BYTE*)src; const BYTE* ip = istart; const BYTE* anchor = istart; - const U32 lowestIndex = cctx->dictLimit; + const U32 lowestIndex = ms->dictLimit; const BYTE* const lowest = base + lowestIndex; const BYTE* const iend = istart + srcSize; const BYTE* const ilimit = iend - HASH_READ_SIZE; - U32 offset_1=seqStorePtr->rep[0], offset_2=seqStorePtr->rep[1]; + U32 offset_1=rep[0], offset_2=rep[1]; U32 offsetSaved = 0; /* init */ @@ -87,7 +90,7 @@ size_t ZSTD_compressBlock_doubleFast_generic(ZSTD_CCtx* cctx, /* favor repcode */ mLength = ZSTD_count(ip+1+4, ip+1+4-offset_1, iend) + 4; ip++; - ZSTD_storeSeq(seqStorePtr, ip-anchor, anchor, 0, mLength-MINMATCH); + ZSTD_storeSeq(seqStore, ip-anchor, anchor, 0, mLength-MINMATCH); } else { U32 offset; if ( (matchIndexL > lowestIndex) && (MEM_read64(matchLong) == MEM_read64(ip)) ) { @@ -117,7 +120,7 @@ size_t ZSTD_compressBlock_doubleFast_generic(ZSTD_CCtx* cctx, offset_2 = offset_1; offset_1 = offset; - ZSTD_storeSeq(seqStorePtr, ip-anchor, anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH); + ZSTD_storeSeq(seqStore, ip-anchor, anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH); } /* match found */ @@ -140,61 +143,63 @@ size_t ZSTD_compressBlock_doubleFast_generic(ZSTD_CCtx* cctx, { U32 const tmpOff = offset_2; offset_2 = offset_1; offset_1 = tmpOff; } /* swap offset_2 <=> offset_1 */ hashSmall[ZSTD_hashPtr(ip, hBitsS, mls)] = (U32)(ip-base); hashLong[ZSTD_hashPtr(ip, hBitsL, 8)] = (U32)(ip-base); - ZSTD_storeSeq(seqStorePtr, 0, anchor, 0, rLength-MINMATCH); + ZSTD_storeSeq(seqStore, 0, anchor, 0, rLength-MINMATCH); ip += rLength; anchor = ip; continue; /* faster when present ... (?) */ } } } /* save reps for next block */ - seqStorePtr->repToConfirm[0] = offset_1 ? offset_1 : offsetSaved; - seqStorePtr->repToConfirm[1] = offset_2 ? offset_2 : offsetSaved; + rep[0] = offset_1 ? offset_1 : offsetSaved; + rep[1] = offset_2 ? offset_2 : offsetSaved; /* Return the last literals size */ return iend - anchor; } -size_t ZSTD_compressBlock_doubleFast(ZSTD_CCtx* ctx, const void* src, size_t srcSize) +size_t ZSTD_compressBlock_doubleFast( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize) { - const U32 mls = ctx->appliedParams.cParams.searchLength; + const U32 mls = cParams->searchLength; switch(mls) { default: /* includes case 3 */ case 4 : - return ZSTD_compressBlock_doubleFast_generic(ctx, src, srcSize, 4); + return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, cParams, src, srcSize, 4); case 5 : - return ZSTD_compressBlock_doubleFast_generic(ctx, src, srcSize, 5); + return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, cParams, src, srcSize, 5); case 6 : - return ZSTD_compressBlock_doubleFast_generic(ctx, src, srcSize, 6); + return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, cParams, src, srcSize, 6); case 7 : - return ZSTD_compressBlock_doubleFast_generic(ctx, src, srcSize, 7); + return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, cParams, src, srcSize, 7); } } -static size_t ZSTD_compressBlock_doubleFast_extDict_generic(ZSTD_CCtx* ctx, - const void* src, size_t srcSize, - const U32 mls) +static size_t ZSTD_compressBlock_doubleFast_extDict_generic( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize, + U32 const mls /* template */) { - U32* const hashLong = ctx->hashTable; - U32 const hBitsL = ctx->appliedParams.cParams.hashLog; - U32* const hashSmall = ctx->chainTable; - U32 const hBitsS = ctx->appliedParams.cParams.chainLog; - seqStore_t* seqStorePtr = &(ctx->seqStore); - const BYTE* const base = ctx->base; - const BYTE* const dictBase = ctx->dictBase; + U32* const hashLong = ms->hashTable; + U32 const hBitsL = cParams->hashLog; + U32* const hashSmall = ms->chainTable; + U32 const hBitsS = cParams->chainLog; + const BYTE* const base = ms->base; + const BYTE* const dictBase = ms->dictBase; const BYTE* const istart = (const BYTE*)src; const BYTE* ip = istart; const BYTE* anchor = istart; - const U32 lowestIndex = ctx->lowLimit; + const U32 lowestIndex = ms->lowLimit; const BYTE* const dictStart = dictBase + lowestIndex; - const U32 dictLimit = ctx->dictLimit; + const U32 dictLimit = ms->dictLimit; const BYTE* const lowPrefixPtr = base + dictLimit; const BYTE* const dictEnd = dictBase + dictLimit; const BYTE* const iend = istart + srcSize; const BYTE* const ilimit = iend - 8; - U32 offset_1=seqStorePtr->rep[0], offset_2=seqStorePtr->rep[1]; + U32 offset_1=rep[0], offset_2=rep[1]; /* Search Loop */ while (ip < ilimit) { /* < instead of <=, because (ip+1) */ @@ -220,7 +225,7 @@ static size_t ZSTD_compressBlock_doubleFast_extDict_generic(ZSTD_CCtx* ctx, const BYTE* repMatchEnd = repIndex < dictLimit ? dictEnd : iend; mLength = ZSTD_count_2segments(ip+1+4, repMatch+4, iend, repMatchEnd, lowPrefixPtr) + 4; ip++; - ZSTD_storeSeq(seqStorePtr, ip-anchor, anchor, 0, mLength-MINMATCH); + ZSTD_storeSeq(seqStore, ip-anchor, anchor, 0, mLength-MINMATCH); } else { if ((matchLongIndex > lowestIndex) && (MEM_read64(matchLong) == MEM_read64(ip))) { const BYTE* matchEnd = matchLongIndex < dictLimit ? dictEnd : iend; @@ -231,7 +236,7 @@ static size_t ZSTD_compressBlock_doubleFast_extDict_generic(ZSTD_CCtx* ctx, while (((ip>anchor) & (matchLong>lowMatchPtr)) && (ip[-1] == matchLong[-1])) { ip--; matchLong--; mLength++; } /* catch up */ offset_2 = offset_1; offset_1 = offset; - ZSTD_storeSeq(seqStorePtr, ip-anchor, anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH); + ZSTD_storeSeq(seqStore, ip-anchor, anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH); } else if ((matchIndex > lowestIndex) && (MEM_read32(match) == MEM_read32(ip))) { size_t const h3 = ZSTD_hashPtr(ip+1, hBitsL, 8); @@ -256,7 +261,7 @@ static size_t ZSTD_compressBlock_doubleFast_extDict_generic(ZSTD_CCtx* ctx, } offset_2 = offset_1; offset_1 = offset; - ZSTD_storeSeq(seqStorePtr, ip-anchor, anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH); + ZSTD_storeSeq(seqStore, ip-anchor, anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH); } else { ip += ((ip-anchor) >> g_searchStrength) + 1; @@ -283,7 +288,7 @@ static size_t ZSTD_compressBlock_doubleFast_extDict_generic(ZSTD_CCtx* ctx, const BYTE* const repEnd2 = repIndex2 < dictLimit ? dictEnd : iend; size_t const repLength2 = ZSTD_count_2segments(ip+4, repMatch2+4, iend, repEnd2, lowPrefixPtr) + 4; U32 tmpOffset = offset_2; offset_2 = offset_1; offset_1 = tmpOffset; /* swap offset_2 <=> offset_1 */ - ZSTD_storeSeq(seqStorePtr, 0, anchor, 0, repLength2-MINMATCH); + ZSTD_storeSeq(seqStore, 0, anchor, 0, repLength2-MINMATCH); hashSmall[ZSTD_hashPtr(ip, hBitsS, mls)] = current2; hashLong[ZSTD_hashPtr(ip, hBitsL, 8)] = current2; ip += repLength2; @@ -294,27 +299,29 @@ static size_t ZSTD_compressBlock_doubleFast_extDict_generic(ZSTD_CCtx* ctx, } } } /* save reps for next block */ - seqStorePtr->repToConfirm[0] = offset_1; seqStorePtr->repToConfirm[1] = offset_2; + rep[0] = offset_1; + rep[1] = offset_2; /* Return the last literals size */ return iend - anchor; } -size_t ZSTD_compressBlock_doubleFast_extDict(ZSTD_CCtx* ctx, - const void* src, size_t srcSize) +size_t ZSTD_compressBlock_doubleFast_extDict( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize) { - U32 const mls = ctx->appliedParams.cParams.searchLength; + U32 const mls = cParams->searchLength; switch(mls) { default: /* includes case 3 */ case 4 : - return ZSTD_compressBlock_doubleFast_extDict_generic(ctx, src, srcSize, 4); + return ZSTD_compressBlock_doubleFast_extDict_generic(ms, seqStore, rep, cParams, src, srcSize, 4); case 5 : - return ZSTD_compressBlock_doubleFast_extDict_generic(ctx, src, srcSize, 5); + return ZSTD_compressBlock_doubleFast_extDict_generic(ms, seqStore, rep, cParams, src, srcSize, 5); case 6 : - return ZSTD_compressBlock_doubleFast_extDict_generic(ctx, src, srcSize, 6); + return ZSTD_compressBlock_doubleFast_extDict_generic(ms, seqStore, rep, cParams, src, srcSize, 6); case 7 : - return ZSTD_compressBlock_doubleFast_extDict_generic(ctx, src, srcSize, 7); + return ZSTD_compressBlock_doubleFast_extDict_generic(ms, seqStore, rep, cParams, src, srcSize, 7); } } diff --git a/lib/compress/zstd_double_fast.h b/lib/compress/zstd_double_fast.h index 75e04158..6d80b277 100644 --- a/lib/compress/zstd_double_fast.h +++ b/lib/compress/zstd_double_fast.h @@ -16,11 +16,18 @@ extern "C" { #endif #include "mem.h" /* U32 */ -#include "zstd.h" /* ZSTD_CCtx, size_t */ +#include "zstd_compress_internal.h" /* ZSTD_CCtx, size_t */ + +void ZSTD_fillDoubleHashTable(ZSTD_matchState_t* ms, + ZSTD_compressionParameters const* cParams, + void const* end); +size_t ZSTD_compressBlock_doubleFast( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize); +size_t ZSTD_compressBlock_doubleFast_extDict( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize); -void ZSTD_fillDoubleHashTable(ZSTD_CCtx* cctx, const void* end, const U32 mls); -size_t ZSTD_compressBlock_doubleFast(ZSTD_CCtx* ctx, const void* src, size_t srcSize); -size_t ZSTD_compressBlock_doubleFast_extDict(ZSTD_CCtx* ctx, const void* src, size_t srcSize); #if defined (__cplusplus) } diff --git a/lib/compress/zstd_fast.c b/lib/compress/zstd_fast.c index 6497a2d8..8941326e 100644 --- a/lib/compress/zstd_fast.c +++ b/lib/compress/zstd_fast.c @@ -12,12 +12,15 @@ #include "zstd_fast.h" -void ZSTD_fillHashTable (ZSTD_CCtx* zc, const void* end, const U32 mls) +void ZSTD_fillHashTable(ZSTD_matchState_t* ms, + ZSTD_compressionParameters const* cParams, + void const* end) { - U32* const hashTable = zc->hashTable; - U32 const hBits = zc->appliedParams.cParams.hashLog; - const BYTE* const base = zc->base; - const BYTE* ip = base + zc->nextToUpdate; + U32* const hashTable = ms->hashTable; + U32 const hBits = cParams->hashLog; + U32 const mls = cParams->searchLength; + const BYTE* const base = ms->base; + const BYTE* ip = base + ms->nextToUpdate; const BYTE* const iend = ((const BYTE*)end) - HASH_READ_SIZE; const U32 fastHashFillStep = 3; @@ -35,24 +38,22 @@ void ZSTD_fillHashTable (ZSTD_CCtx* zc, const void* end, const U32 mls) } } - FORCE_INLINE_TEMPLATE -size_t ZSTD_compressBlock_fast_generic(ZSTD_CCtx* cctx, - const void* src, size_t srcSize, - const U32 mls) +size_t ZSTD_compressBlock_fast_generic( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize, + U32 const hlog, U32 const mls) { - U32* const hashTable = cctx->hashTable; - U32 const hBits = cctx->appliedParams.cParams.hashLog; - seqStore_t* seqStorePtr = &(cctx->seqStore); - const BYTE* const base = cctx->base; + U32* const hashTable = ms->hashTable; + const BYTE* const base = ms->base; const BYTE* const istart = (const BYTE*)src; const BYTE* ip = istart; const BYTE* anchor = istart; - const U32 lowestIndex = cctx->dictLimit; + const U32 lowestIndex = ms->dictLimit; const BYTE* const lowest = base + lowestIndex; const BYTE* const iend = istart + srcSize; const BYTE* const ilimit = iend - HASH_READ_SIZE; - U32 offset_1=seqStorePtr->rep[0], offset_2=seqStorePtr->rep[1]; + U32 offset_1=rep[0], offset_2=rep[1]; U32 offsetSaved = 0; /* init */ @@ -65,7 +66,7 @@ size_t ZSTD_compressBlock_fast_generic(ZSTD_CCtx* cctx, /* Main Search Loop */ while (ip < ilimit) { /* < instead of <=, because repcode check at (ip+1) */ size_t mLength; - size_t const h = ZSTD_hashPtr(ip, hBits, mls); + size_t const h = ZSTD_hashPtr(ip, hlog, mls); U32 const current = (U32)(ip-base); U32 const matchIndex = hashTable[h]; const BYTE* match = base + matchIndex; @@ -74,7 +75,7 @@ size_t ZSTD_compressBlock_fast_generic(ZSTD_CCtx* cctx, if ((offset_1 > 0) & (MEM_read32(ip+1-offset_1) == MEM_read32(ip+1))) { mLength = ZSTD_count(ip+1+4, ip+1+4-offset_1, iend) + 4; ip++; - ZSTD_storeSeq(seqStorePtr, ip-anchor, anchor, 0, mLength-MINMATCH); + ZSTD_storeSeq(seqStore, ip-anchor, anchor, 0, mLength-MINMATCH); } else { U32 offset; if ( (matchIndex <= lowestIndex) || (MEM_read32(match) != MEM_read32(ip)) ) { @@ -87,7 +88,7 @@ size_t ZSTD_compressBlock_fast_generic(ZSTD_CCtx* cctx, offset_2 = offset_1; offset_1 = offset; - ZSTD_storeSeq(seqStorePtr, ip-anchor, anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH); + ZSTD_storeSeq(seqStore, ip-anchor, anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH); } /* match found */ @@ -96,8 +97,8 @@ size_t ZSTD_compressBlock_fast_generic(ZSTD_CCtx* cctx, if (ip <= ilimit) { /* Fill Table */ - hashTable[ZSTD_hashPtr(base+current+2, hBits, mls)] = current+2; /* here because current+2 could be > iend-8 */ - hashTable[ZSTD_hashPtr(ip-2, hBits, mls)] = (U32)(ip-2-base); + hashTable[ZSTD_hashPtr(base+current+2, hlog, mls)] = current+2; /* here because current+2 could be > iend-8 */ + hashTable[ZSTD_hashPtr(ip-2, hlog, mls)] = (U32)(ip-2-base); /* check immediate repcode */ while ( (ip <= ilimit) && ( (offset_2>0) @@ -105,65 +106,66 @@ size_t ZSTD_compressBlock_fast_generic(ZSTD_CCtx* cctx, /* store sequence */ size_t const rLength = ZSTD_count(ip+4, ip+4-offset_2, iend) + 4; { U32 const tmpOff = offset_2; offset_2 = offset_1; offset_1 = tmpOff; } /* swap offset_2 <=> offset_1 */ - hashTable[ZSTD_hashPtr(ip, hBits, mls)] = (U32)(ip-base); - ZSTD_storeSeq(seqStorePtr, 0, anchor, 0, rLength-MINMATCH); + hashTable[ZSTD_hashPtr(ip, hlog, mls)] = (U32)(ip-base); + ZSTD_storeSeq(seqStore, 0, anchor, 0, rLength-MINMATCH); ip += rLength; anchor = ip; continue; /* faster when present ... (?) */ } } } /* save reps for next block */ - seqStorePtr->repToConfirm[0] = offset_1 ? offset_1 : offsetSaved; - seqStorePtr->repToConfirm[1] = offset_2 ? offset_2 : offsetSaved; + rep[0] = offset_1 ? offset_1 : offsetSaved; + rep[1] = offset_2 ? offset_2 : offsetSaved; /* Return the last literals size */ return iend - anchor; } -size_t ZSTD_compressBlock_fast(ZSTD_CCtx* ctx, - const void* src, size_t srcSize) +size_t ZSTD_compressBlock_fast( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize) { - const U32 mls = ctx->appliedParams.cParams.searchLength; + U32 const hlog = cParams->hashLog; + U32 const mls = cParams->searchLength; switch(mls) { default: /* includes case 3 */ case 4 : - return ZSTD_compressBlock_fast_generic(ctx, src, srcSize, 4); + return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, hlog, 4); case 5 : - return ZSTD_compressBlock_fast_generic(ctx, src, srcSize, 5); + return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, hlog, 5); case 6 : - return ZSTD_compressBlock_fast_generic(ctx, src, srcSize, 6); + return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, hlog, 6); case 7 : - return ZSTD_compressBlock_fast_generic(ctx, src, srcSize, 7); + return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, hlog, 7); } } -static size_t ZSTD_compressBlock_fast_extDict_generic(ZSTD_CCtx* ctx, - const void* src, size_t srcSize, - const U32 mls) +static size_t ZSTD_compressBlock_fast_extDict_generic( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize, + U32 const hlog, U32 const mls) { - U32* hashTable = ctx->hashTable; - const U32 hBits = ctx->appliedParams.cParams.hashLog; - seqStore_t* seqStorePtr = &(ctx->seqStore); - const BYTE* const base = ctx->base; - const BYTE* const dictBase = ctx->dictBase; + U32* hashTable = ms->hashTable; + const BYTE* const base = ms->base; + const BYTE* const dictBase = ms->dictBase; const BYTE* const istart = (const BYTE*)src; const BYTE* ip = istart; const BYTE* anchor = istart; - const U32 lowestIndex = ctx->lowLimit; + const U32 lowestIndex = ms->lowLimit; const BYTE* const dictStart = dictBase + lowestIndex; - const U32 dictLimit = ctx->dictLimit; + const U32 dictLimit = ms->dictLimit; const BYTE* const lowPrefixPtr = base + dictLimit; const BYTE* const dictEnd = dictBase + dictLimit; const BYTE* const iend = istart + srcSize; const BYTE* const ilimit = iend - 8; - U32 offset_1=seqStorePtr->rep[0], offset_2=seqStorePtr->rep[1]; + U32 offset_1=rep[0], offset_2=rep[1]; /* Search Loop */ while (ip < ilimit) { /* < instead of <=, because (ip+1) */ - const size_t h = ZSTD_hashPtr(ip, hBits, mls); + const size_t h = ZSTD_hashPtr(ip, hlog, mls); const U32 matchIndex = hashTable[h]; const BYTE* matchBase = matchIndex < dictLimit ? dictBase : base; const BYTE* match = matchBase + matchIndex; @@ -179,7 +181,7 @@ static size_t ZSTD_compressBlock_fast_extDict_generic(ZSTD_CCtx* ctx, const BYTE* repMatchEnd = repIndex < dictLimit ? dictEnd : iend; mLength = ZSTD_count_2segments(ip+1+4, repMatch+4, iend, repMatchEnd, lowPrefixPtr) + 4; ip++; - ZSTD_storeSeq(seqStorePtr, ip-anchor, anchor, 0, mLength-MINMATCH); + ZSTD_storeSeq(seqStore, ip-anchor, anchor, 0, mLength-MINMATCH); } else { if ( (matchIndex < lowestIndex) || (MEM_read32(match) != MEM_read32(ip)) ) { @@ -194,7 +196,7 @@ static size_t ZSTD_compressBlock_fast_extDict_generic(ZSTD_CCtx* ctx, offset = current - matchIndex; offset_2 = offset_1; offset_1 = offset; - ZSTD_storeSeq(seqStorePtr, ip-anchor, anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH); + ZSTD_storeSeq(seqStore, ip-anchor, anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH); } } /* found a match : store it */ @@ -203,8 +205,8 @@ static size_t ZSTD_compressBlock_fast_extDict_generic(ZSTD_CCtx* ctx, if (ip <= ilimit) { /* Fill Table */ - hashTable[ZSTD_hashPtr(base+current+2, hBits, mls)] = current+2; - hashTable[ZSTD_hashPtr(ip-2, hBits, mls)] = (U32)(ip-2-base); + hashTable[ZSTD_hashPtr(base+current+2, hlog, mls)] = current+2; + hashTable[ZSTD_hashPtr(ip-2, hlog, mls)] = (U32)(ip-2-base); /* check immediate repcode */ while (ip <= ilimit) { U32 const current2 = (U32)(ip-base); @@ -215,8 +217,8 @@ static size_t ZSTD_compressBlock_fast_extDict_generic(ZSTD_CCtx* ctx, const BYTE* const repEnd2 = repIndex2 < dictLimit ? dictEnd : iend; size_t const repLength2 = ZSTD_count_2segments(ip+4, repMatch2+4, iend, repEnd2, lowPrefixPtr) + 4; U32 tmpOffset = offset_2; offset_2 = offset_1; offset_1 = tmpOffset; /* swap offset_2 <=> offset_1 */ - ZSTD_storeSeq(seqStorePtr, 0, anchor, 0, repLength2-MINMATCH); - hashTable[ZSTD_hashPtr(ip, hBits, mls)] = current2; + ZSTD_storeSeq(seqStore, 0, anchor, 0, repLength2-MINMATCH); + hashTable[ZSTD_hashPtr(ip, hlog, mls)] = current2; ip += repLength2; anchor = ip; continue; @@ -225,27 +227,30 @@ static size_t ZSTD_compressBlock_fast_extDict_generic(ZSTD_CCtx* ctx, } } } /* save reps for next block */ - seqStorePtr->repToConfirm[0] = offset_1; seqStorePtr->repToConfirm[1] = offset_2; + rep[0] = offset_1; + rep[1] = offset_2; /* Return the last literals size */ return iend - anchor; } -size_t ZSTD_compressBlock_fast_extDict(ZSTD_CCtx* ctx, - const void* src, size_t srcSize) +size_t ZSTD_compressBlock_fast_extDict( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize) { - U32 const mls = ctx->appliedParams.cParams.searchLength; + U32 const hlog = cParams->hashLog; + U32 const mls = cParams->searchLength; switch(mls) { default: /* includes case 3 */ case 4 : - return ZSTD_compressBlock_fast_extDict_generic(ctx, src, srcSize, 4); + return ZSTD_compressBlock_fast_extDict_generic(ms, seqStore, rep, src, srcSize, hlog, 4); case 5 : - return ZSTD_compressBlock_fast_extDict_generic(ctx, src, srcSize, 5); + return ZSTD_compressBlock_fast_extDict_generic(ms, seqStore, rep, src, srcSize, hlog, 5); case 6 : - return ZSTD_compressBlock_fast_extDict_generic(ctx, src, srcSize, 6); + return ZSTD_compressBlock_fast_extDict_generic(ms, seqStore, rep, src, srcSize, hlog, 6); case 7 : - return ZSTD_compressBlock_fast_extDict_generic(ctx, src, srcSize, 7); + return ZSTD_compressBlock_fast_extDict_generic(ms, seqStore, rep, src, srcSize, hlog, 7); } } diff --git a/lib/compress/zstd_fast.h b/lib/compress/zstd_fast.h index d8b77719..f0438ad5 100644 --- a/lib/compress/zstd_fast.h +++ b/lib/compress/zstd_fast.h @@ -16,13 +16,17 @@ extern "C" { #endif #include "mem.h" /* U32 */ -#include "zstd.h" /* ZSTD_CCtx, size_t */ +#include "zstd_compress_internal.h" -void ZSTD_fillHashTable(ZSTD_CCtx* zc, const void* end, const U32 mls); -size_t ZSTD_compressBlock_fast(ZSTD_CCtx* ctx, - const void* src, size_t srcSize); -size_t ZSTD_compressBlock_fast_extDict(ZSTD_CCtx* ctx, - const void* src, size_t srcSize); +void ZSTD_fillHashTable(ZSTD_matchState_t* ms, + ZSTD_compressionParameters const* cParams, + void const* end); +size_t ZSTD_compressBlock_fast( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize); +size_t ZSTD_compressBlock_fast_extDict( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize); #if defined (__cplusplus) } diff --git a/lib/compress/zstd_lazy.c b/lib/compress/zstd_lazy.c index 1d575804..7f59eb34 100644 --- a/lib/compress/zstd_lazy.c +++ b/lib/compress/zstd_lazy.c @@ -45,28 +45,29 @@ void ZSTD_preserveUnsortedMark (U32* const table, U32 const size, U32 const redu table[u] = ZSTD_DUBT_UNSORTED_MARK + reducerValue; } -void ZSTD_updateDUBT(ZSTD_CCtx* zc, +void ZSTD_updateDUBT( + ZSTD_matchState_t* ms, ZSTD_compressionParameters const* cParams, const BYTE* ip, const BYTE* iend, U32 mls) { - U32* const hashTable = zc->hashTable; - U32 const hashLog = zc->appliedParams.cParams.hashLog; + U32* const hashTable = ms->hashTable; + U32 const hashLog = cParams->hashLog; - U32* const bt = zc->chainTable; - U32 const btLog = zc->appliedParams.cParams.chainLog - 1; + U32* const bt = ms->chainTable; + U32 const btLog = cParams->chainLog - 1; U32 const btMask = (1 << btLog) - 1; - const BYTE* const base = zc->base; + const BYTE* const base = ms->base; U32 const target = (U32)(ip - base); - U32 idx = zc->nextToUpdate; + U32 idx = ms->nextToUpdate; if (idx != target) DEBUGLOG(7, "ZSTD_updateDUBT, from %u to %u (dictLimit:%u)", - idx, target, zc->dictLimit); + idx, target, ms->dictLimit); assert(ip + 8 <= iend); /* condition for ZSTD_hashPtr */ (void)iend; - assert(idx >= zc->dictLimit); /* condition for valid base+idx */ + assert(idx >= ms->dictLimit); /* condition for valid base+idx */ for ( ; idx < target ; idx++) { size_t const h = ZSTD_hashPtr(base + idx, hashLog, mls); /* assumption : ip + 8 <= iend */ U32 const matchIndex = hashTable[h]; @@ -79,7 +80,7 @@ void ZSTD_updateDUBT(ZSTD_CCtx* zc, *nextCandidatePtr = matchIndex; /* update BT like a chain */ *sortMarkPtr = ZSTD_DUBT_UNSORTED_MARK; } - zc->nextToUpdate = target; + ms->nextToUpdate = target; } @@ -87,17 +88,18 @@ void ZSTD_updateDUBT(ZSTD_CCtx* zc, * sort one already inserted but unsorted position * assumption : current >= btlow == (current - btmask) * doesn't fail */ -static void ZSTD_insertDUBT1(ZSTD_CCtx* zc, +static void ZSTD_insertDUBT1( + ZSTD_matchState_t* ms, ZSTD_compressionParameters const* cParams, U32 current, const BYTE* inputEnd, U32 nbCompares, U32 btLow, int extDict) { - U32* const bt = zc->chainTable; - U32 const btLog = zc->appliedParams.cParams.chainLog - 1; + U32* const bt = ms->chainTable; + U32 const btLog = cParams->chainLog - 1; U32 const btMask = (1 << btLog) - 1; size_t commonLengthSmaller=0, commonLengthLarger=0; - const BYTE* const base = zc->base; - const BYTE* const dictBase = zc->dictBase; - const U32 dictLimit = zc->dictLimit; + const BYTE* const base = ms->base; + const BYTE* const dictBase = ms->dictBase; + const U32 dictLimit = ms->dictLimit; const BYTE* const ip = (current>=dictLimit) ? base + current : dictBase + current; const BYTE* const iend = (current>=dictLimit) ? inputEnd : dictBase + dictLimit; const BYTE* const dictEnd = dictBase + dictLimit; @@ -107,7 +109,7 @@ static void ZSTD_insertDUBT1(ZSTD_CCtx* zc, U32* largerPtr = smallerPtr + 1; U32 matchIndex = *smallerPtr; U32 dummy32; /* to be nullified at the end */ - U32 const windowLow = zc->lowLimit; + U32 const windowLow = ms->lowLimit; DEBUGLOG(8, "ZSTD_insertDUBT1(%u) (dictLimit=%u, lowLimit=%u)", current, dictLimit, windowLow); @@ -166,29 +168,30 @@ static void ZSTD_insertDUBT1(ZSTD_CCtx* zc, static size_t ZSTD_DUBT_findBestMatch ( - ZSTD_CCtx* zc, + ZSTD_matchState_t* ms, ZSTD_compressionParameters const* cParams, const BYTE* const ip, const BYTE* const iend, size_t* offsetPtr, - U32 nbCompares, const U32 mls, - U32 extDict) + U32 const mls, + U32 const extDict) { - U32* const hashTable = zc->hashTable; - U32 const hashLog = zc->appliedParams.cParams.hashLog; + U32* const hashTable = ms->hashTable; + U32 const hashLog = cParams->hashLog; size_t const h = ZSTD_hashPtr(ip, hashLog, mls); U32 matchIndex = hashTable[h]; - const BYTE* const base = zc->base; + const BYTE* const base = ms->base; U32 const current = (U32)(ip-base); - U32 const windowLow = zc->lowLimit; + U32 const windowLow = ms->lowLimit; - U32* const bt = zc->chainTable; - U32 const btLog = zc->appliedParams.cParams.chainLog - 1; + U32* const bt = ms->chainTable; + U32 const btLog = cParams->chainLog - 1; U32 const btMask = (1 << btLog) - 1; U32 const btLow = (btMask >= current) ? 0 : current - btMask; U32 const unsortLimit = MAX(btLow, windowLow); U32* nextCandidate = bt + 2*(matchIndex&btMask); U32* unsortedMark = bt + 2*(matchIndex&btMask) + 1; + U32 nbCompares = 1U << cParams->searchLog; U32 nbCandidates = nbCompares; U32 previousCandidate = 0; @@ -221,7 +224,7 @@ static size_t ZSTD_DUBT_findBestMatch ( while (matchIndex) { /* will end on matchIndex == 0 */ U32* const nextCandidateIdxPtr = bt + 2*(matchIndex&btMask) + 1; U32 const nextCandidateIdx = *nextCandidateIdxPtr; - ZSTD_insertDUBT1(zc, matchIndex, iend, + ZSTD_insertDUBT1(ms, cParams, matchIndex, iend, nbCandidates, unsortLimit, extDict); matchIndex = nextCandidateIdx; nbCandidates++; @@ -229,8 +232,8 @@ static size_t ZSTD_DUBT_findBestMatch ( /* find longest match */ { size_t commonLengthSmaller=0, commonLengthLarger=0; - const BYTE* const dictBase = zc->dictBase; - const U32 dictLimit = zc->dictLimit; + const BYTE* const dictBase = ms->dictBase; + const U32 dictLimit = ms->dictLimit; const BYTE* const dictEnd = dictBase + dictLimit; const BYTE* const prefixStart = base + dictLimit; U32* smallerPtr = bt + 2*(current&btMask); @@ -286,7 +289,7 @@ static size_t ZSTD_DUBT_findBestMatch ( *smallerPtr = *largerPtr = 0; assert(matchEndIdx > current+8); /* ensure nextToUpdate is increased */ - zc->nextToUpdate = matchEndIdx - 8; /* skip repetitive patterns */ + ms->nextToUpdate = matchEndIdx - 8; /* skip repetitive patterns */ if (bestLength >= MINMATCH) { U32 const mIndex = current - ((U32)*offsetPtr - ZSTD_REP_MOVE); (void)mIndex; DEBUGLOG(8, "ZSTD_DUBT_findBestMatch(%u) : found match of length %u and offsetCode %u (pos %u)", @@ -299,62 +302,60 @@ static size_t ZSTD_DUBT_findBestMatch ( /** ZSTD_BtFindBestMatch() : Tree updater, providing best match */ static size_t ZSTD_BtFindBestMatch ( - ZSTD_CCtx* zc, + ZSTD_matchState_t* ms, ZSTD_compressionParameters const* cParams, const BYTE* const ip, const BYTE* const iLimit, size_t* offsetPtr, - const U32 maxNbAttempts, const U32 mls) + const U32 mls /* template */) { DEBUGLOG(7, "ZSTD_BtFindBestMatch"); - if (ip < zc->base + zc->nextToUpdate) return 0; /* skipped area */ - ZSTD_updateDUBT(zc, ip, iLimit, mls); - return ZSTD_DUBT_findBestMatch(zc, ip, iLimit, offsetPtr, maxNbAttempts, mls, 0); + if (ip < ms->base + ms->nextToUpdate) return 0; /* skipped area */ + ZSTD_updateDUBT(ms, cParams, ip, iLimit, mls); + return ZSTD_DUBT_findBestMatch(ms, cParams, ip, iLimit, offsetPtr, mls, 0); } static size_t ZSTD_BtFindBestMatch_selectMLS ( - ZSTD_CCtx* zc, /* Index table will be updated */ + ZSTD_matchState_t* ms, ZSTD_compressionParameters const* cParams, const BYTE* ip, const BYTE* const iLimit, - size_t* offsetPtr, - const U32 maxNbAttempts, const U32 matchLengthSearch) + size_t* offsetPtr) { - switch(matchLengthSearch) + switch(cParams->searchLength) { default : /* includes case 3 */ - case 4 : return ZSTD_BtFindBestMatch(zc, ip, iLimit, offsetPtr, maxNbAttempts, 4); - case 5 : return ZSTD_BtFindBestMatch(zc, ip, iLimit, offsetPtr, maxNbAttempts, 5); + case 4 : return ZSTD_BtFindBestMatch(ms, cParams, ip, iLimit, offsetPtr, 4); + case 5 : return ZSTD_BtFindBestMatch(ms, cParams, ip, iLimit, offsetPtr, 5); case 7 : - case 6 : return ZSTD_BtFindBestMatch(zc, ip, iLimit, offsetPtr, maxNbAttempts, 6); + case 6 : return ZSTD_BtFindBestMatch(ms, cParams, ip, iLimit, offsetPtr, 6); } } /** Tree updater, providing best match */ static size_t ZSTD_BtFindBestMatch_extDict ( - ZSTD_CCtx* zc, + ZSTD_matchState_t* ms, ZSTD_compressionParameters const* cParams, const BYTE* const ip, const BYTE* const iLimit, size_t* offsetPtr, - const U32 maxNbAttempts, const U32 mls) + const U32 mls) { DEBUGLOG(7, "ZSTD_BtFindBestMatch_extDict"); - if (ip < zc->base + zc->nextToUpdate) return 0; /* skipped area */ - ZSTD_updateDUBT(zc, ip, iLimit, mls); - return ZSTD_DUBT_findBestMatch(zc, ip, iLimit, offsetPtr, maxNbAttempts, mls, 1); + if (ip < ms->base + ms->nextToUpdate) return 0; /* skipped area */ + ZSTD_updateDUBT(ms, cParams, ip, iLimit, mls); + return ZSTD_DUBT_findBestMatch(ms, cParams, ip, iLimit, offsetPtr, mls, 1); } static size_t ZSTD_BtFindBestMatch_selectMLS_extDict ( - ZSTD_CCtx* zc, /* Index table will be updated */ + ZSTD_matchState_t* ms, ZSTD_compressionParameters const* cParams, const BYTE* ip, const BYTE* const iLimit, - size_t* offsetPtr, - const U32 maxNbAttempts, const U32 matchLengthSearch) + size_t* offsetPtr) { - switch(matchLengthSearch) + switch(cParams->searchLength) { default : /* includes case 3 */ - case 4 : return ZSTD_BtFindBestMatch_extDict(zc, ip, iLimit, offsetPtr, maxNbAttempts, 4); - case 5 : return ZSTD_BtFindBestMatch_extDict(zc, ip, iLimit, offsetPtr, maxNbAttempts, 5); + case 4 : return ZSTD_BtFindBestMatch_extDict(ms, cParams, ip, iLimit, offsetPtr, 4); + case 5 : return ZSTD_BtFindBestMatch_extDict(ms, cParams, ip, iLimit, offsetPtr, 5); case 7 : - case 6 : return ZSTD_BtFindBestMatch_extDict(zc, ip, iLimit, offsetPtr, maxNbAttempts, 6); + case 6 : return ZSTD_BtFindBestMatch_extDict(ms, cParams, ip, iLimit, offsetPtr, 6); } } @@ -367,15 +368,17 @@ static size_t ZSTD_BtFindBestMatch_selectMLS_extDict ( /* Update chains up to ip (excluded) Assumption : always within prefix (i.e. not within extDict) */ -U32 ZSTD_insertAndFindFirstIndex (ZSTD_CCtx* zc, const BYTE* ip, U32 mls) +static U32 ZSTD_insertAndFindFirstIndex_internal( + ZSTD_matchState_t* ms, ZSTD_compressionParameters const* cParams, + const BYTE* ip, U32 const mls) { - U32* const hashTable = zc->hashTable; - const U32 hashLog = zc->appliedParams.cParams.hashLog; - U32* const chainTable = zc->chainTable; - const U32 chainMask = (1 << zc->appliedParams.cParams.chainLog) - 1; - const BYTE* const base = zc->base; + U32* const hashTable = ms->hashTable; + const U32 hashLog = cParams->hashLog; + U32* const chainTable = ms->chainTable; + const U32 chainMask = (1 << cParams->chainLog) - 1; + const BYTE* const base = ms->base; const U32 target = (U32)(ip - base); - U32 idx = zc->nextToUpdate; + U32 idx = ms->nextToUpdate; while(idx < target) { /* catch up */ size_t const h = ZSTD_hashPtr(base+idx, hashLog, mls); @@ -384,35 +387,42 @@ U32 ZSTD_insertAndFindFirstIndex (ZSTD_CCtx* zc, const BYTE* ip, U32 mls) idx++; } - zc->nextToUpdate = target; + ms->nextToUpdate = target; return hashTable[ZSTD_hashPtr(ip, hashLog, mls)]; } +U32 ZSTD_insertAndFindFirstIndex( + ZSTD_matchState_t* ms, ZSTD_compressionParameters const* cParams, + const BYTE* ip) +{ + return ZSTD_insertAndFindFirstIndex_internal(ms, cParams, ip, cParams->searchLength); +} + /* inlining is important to hardwire a hot branch (template emulation) */ FORCE_INLINE_TEMPLATE size_t ZSTD_HcFindBestMatch_generic ( - ZSTD_CCtx* zc, /* Index table will be updated */ + ZSTD_matchState_t* ms, ZSTD_compressionParameters const* cParams, const BYTE* const ip, const BYTE* const iLimit, size_t* offsetPtr, - const U32 maxNbAttempts, const U32 mls, const U32 extDict) + const U32 mls, const U32 extDict) { - U32* const chainTable = zc->chainTable; - const U32 chainSize = (1 << zc->appliedParams.cParams.chainLog); + U32* const chainTable = ms->chainTable; + const U32 chainSize = (1 << cParams->chainLog); const U32 chainMask = chainSize-1; - const BYTE* const base = zc->base; - const BYTE* const dictBase = zc->dictBase; - const U32 dictLimit = zc->dictLimit; + const BYTE* const base = ms->base; + const BYTE* const dictBase = ms->dictBase; + const U32 dictLimit = ms->dictLimit; const BYTE* const prefixStart = base + dictLimit; const BYTE* const dictEnd = dictBase + dictLimit; - const U32 lowLimit = zc->lowLimit; + const U32 lowLimit = ms->lowLimit; const U32 current = (U32)(ip-base); const U32 minChain = current > chainSize ? current - chainSize : 0; - int nbAttempts=maxNbAttempts; + U32 nbAttempts = 1U << cParams->searchLog; size_t ml=4-1; /* HC4 match finder */ - U32 matchIndex = ZSTD_insertAndFindFirstIndex (zc, ip, mls); + U32 matchIndex = ZSTD_insertAndFindFirstIndex_internal(ms, cParams, ip, mls); for ( ; (matchIndex>lowLimit) & (nbAttempts>0) ; nbAttempts--) { size_t currentMl=0; @@ -443,35 +453,33 @@ size_t ZSTD_HcFindBestMatch_generic ( FORCE_INLINE_TEMPLATE size_t ZSTD_HcFindBestMatch_selectMLS ( - ZSTD_CCtx* zc, + ZSTD_matchState_t* ms, ZSTD_compressionParameters const* cParams, const BYTE* ip, const BYTE* const iLimit, - size_t* offsetPtr, - const U32 maxNbAttempts, const U32 matchLengthSearch) + size_t* offsetPtr) { - switch(matchLengthSearch) + switch(cParams->searchLength) { default : /* includes case 3 */ - case 4 : return ZSTD_HcFindBestMatch_generic(zc, ip, iLimit, offsetPtr, maxNbAttempts, 4, 0); - case 5 : return ZSTD_HcFindBestMatch_generic(zc, ip, iLimit, offsetPtr, maxNbAttempts, 5, 0); + case 4 : return ZSTD_HcFindBestMatch_generic(ms, cParams, ip, iLimit, offsetPtr, 4, 0); + case 5 : return ZSTD_HcFindBestMatch_generic(ms, cParams, ip, iLimit, offsetPtr, 5, 0); case 7 : - case 6 : return ZSTD_HcFindBestMatch_generic(zc, ip, iLimit, offsetPtr, maxNbAttempts, 6, 0); + case 6 : return ZSTD_HcFindBestMatch_generic(ms, cParams, ip, iLimit, offsetPtr, 6, 0); } } FORCE_INLINE_TEMPLATE size_t ZSTD_HcFindBestMatch_extDict_selectMLS ( - ZSTD_CCtx* const zc, + ZSTD_matchState_t* ms, ZSTD_compressionParameters const* cParams, const BYTE* ip, const BYTE* const iLimit, - size_t* const offsetPtr, - U32 const maxNbAttempts, U32 const matchLengthSearch) + size_t* const offsetPtr) { - switch(matchLengthSearch) + switch(cParams->searchLength) { default : /* includes case 3 */ - case 4 : return ZSTD_HcFindBestMatch_generic(zc, ip, iLimit, offsetPtr, maxNbAttempts, 4, 1); - case 5 : return ZSTD_HcFindBestMatch_generic(zc, ip, iLimit, offsetPtr, maxNbAttempts, 5, 1); + case 4 : return ZSTD_HcFindBestMatch_generic(ms, cParams, ip, iLimit, offsetPtr, 4, 1); + case 5 : return ZSTD_HcFindBestMatch_generic(ms, cParams, ip, iLimit, offsetPtr, 5, 1); case 7 : - case 6 : return ZSTD_HcFindBestMatch_generic(zc, ip, iLimit, offsetPtr, maxNbAttempts, 6, 1); + case 6 : return ZSTD_HcFindBestMatch_generic(ms, cParams, ip, iLimit, offsetPtr, 6, 1); } } @@ -480,30 +488,29 @@ FORCE_INLINE_TEMPLATE size_t ZSTD_HcFindBestMatch_extDict_selectMLS ( * Common parser - lazy strategy *********************************/ FORCE_INLINE_TEMPLATE -size_t ZSTD_compressBlock_lazy_generic(ZSTD_CCtx* ctx, - const void* src, size_t srcSize, - const U32 searchMethod, const U32 depth) +size_t ZSTD_compressBlock_lazy_generic( + ZSTD_matchState_t* ms, seqStore_t* seqStore, + U32 rep[ZSTD_REP_NUM], + ZSTD_compressionParameters const* cParams, + const void* src, size_t srcSize, + const U32 searchMethod, const U32 depth) { - seqStore_t* seqStorePtr = &(ctx->seqStore); const BYTE* const istart = (const BYTE*)src; const BYTE* ip = istart; const BYTE* anchor = istart; const BYTE* const iend = istart + srcSize; const BYTE* const ilimit = iend - 8; - const BYTE* const base = ctx->base + ctx->dictLimit; + const BYTE* const base = ms->base + ms->dictLimit; - U32 const maxSearches = 1 << ctx->appliedParams.cParams.searchLog; - U32 const mls = ctx->appliedParams.cParams.searchLength; - - typedef size_t (*searchMax_f)(ZSTD_CCtx* zc, const BYTE* ip, const BYTE* iLimit, - size_t* offsetPtr, - U32 maxNbAttempts, U32 matchLengthSearch); + typedef size_t (*searchMax_f)( + ZSTD_matchState_t* ms, ZSTD_compressionParameters const* cParams, + const BYTE* ip, const BYTE* iLimit, size_t* offsetPtr); searchMax_f const searchMax = searchMethod ? ZSTD_BtFindBestMatch_selectMLS : ZSTD_HcFindBestMatch_selectMLS; - U32 offset_1 = seqStorePtr->rep[0], offset_2 = seqStorePtr->rep[1], savedOffset=0; + U32 offset_1 = rep[0], offset_2 = rep[1], savedOffset=0; /* init */ ip += (ip==base); - ctx->nextToUpdate3 = ctx->nextToUpdate; + ms->nextToUpdate3 = ms->nextToUpdate; { U32 const maxRep = (U32)(ip-base); if (offset_2 > maxRep) savedOffset = offset_2, offset_2 = 0; if (offset_1 > maxRep) savedOffset = offset_1, offset_1 = 0; @@ -524,7 +531,7 @@ size_t ZSTD_compressBlock_lazy_generic(ZSTD_CCtx* ctx, /* first search (depth 0) */ { size_t offsetFound = 99999999; - size_t const ml2 = searchMax(ctx, ip, iend, &offsetFound, maxSearches, mls); + size_t const ml2 = searchMax(ms, cParams, ip, iend, &offsetFound); if (ml2 > matchLength) matchLength = ml2, start = ip, offset=offsetFound; } @@ -546,7 +553,7 @@ size_t ZSTD_compressBlock_lazy_generic(ZSTD_CCtx* ctx, matchLength = mlRep, offset = 0, start = ip; } { size_t offset2=99999999; - size_t const ml2 = searchMax(ctx, ip, iend, &offset2, maxSearches, mls); + size_t const ml2 = searchMax(ms, cParams, ip, iend, &offset2); int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)offset2+1)); /* raw approx */ int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offset+1) + 4); if ((ml2 >= 4) && (gain2 > gain1)) { @@ -565,7 +572,7 @@ size_t ZSTD_compressBlock_lazy_generic(ZSTD_CCtx* ctx, matchLength = ml2, offset = 0, start = ip; } { size_t offset2=99999999; - size_t const ml2 = searchMax(ctx, ip, iend, &offset2, maxSearches, mls); + size_t const ml2 = searchMax(ms, cParams, ip, iend, &offset2); int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)offset2+1)); /* raw approx */ int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offset+1) + 7); if ((ml2 >= 4) && (gain2 > gain1)) { @@ -590,7 +597,7 @@ size_t ZSTD_compressBlock_lazy_generic(ZSTD_CCtx* ctx, /* store sequence */ _storeSequence: { size_t const litLength = start - anchor; - ZSTD_storeSeq(seqStorePtr, litLength, anchor, (U32)offset, matchLength-MINMATCH); + ZSTD_storeSeq(seqStore, litLength, anchor, (U32)offset, matchLength-MINMATCH); anchor = ip = start + matchLength; } @@ -600,73 +607,80 @@ _storeSequence: /* store sequence */ matchLength = ZSTD_count(ip+4, ip+4-offset_2, iend) + 4; offset = offset_2; offset_2 = offset_1; offset_1 = (U32)offset; /* swap repcodes */ - ZSTD_storeSeq(seqStorePtr, 0, anchor, 0, matchLength-MINMATCH); + ZSTD_storeSeq(seqStore, 0, anchor, 0, matchLength-MINMATCH); ip += matchLength; anchor = ip; continue; /* faster when present ... (?) */ } } /* Save reps for next block */ - seqStorePtr->repToConfirm[0] = offset_1 ? offset_1 : savedOffset; - seqStorePtr->repToConfirm[1] = offset_2 ? offset_2 : savedOffset; + rep[0] = offset_1 ? offset_1 : savedOffset; + rep[1] = offset_2 ? offset_2 : savedOffset; /* Return the last literals size */ return iend - anchor; } -size_t ZSTD_compressBlock_btlazy2(ZSTD_CCtx* ctx, const void* src, size_t srcSize) +size_t ZSTD_compressBlock_btlazy2( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize) { - return ZSTD_compressBlock_lazy_generic(ctx, src, srcSize, 1, 2); + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, cParams, src, srcSize, 1, 2); } -size_t ZSTD_compressBlock_lazy2(ZSTD_CCtx* ctx, const void* src, size_t srcSize) +size_t ZSTD_compressBlock_lazy2( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize) { - return ZSTD_compressBlock_lazy_generic(ctx, src, srcSize, 0, 2); + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, cParams, src, srcSize, 0, 2); } -size_t ZSTD_compressBlock_lazy(ZSTD_CCtx* ctx, const void* src, size_t srcSize) +size_t ZSTD_compressBlock_lazy( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize) { - return ZSTD_compressBlock_lazy_generic(ctx, src, srcSize, 0, 1); + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, cParams, src, srcSize, 0, 1); } -size_t ZSTD_compressBlock_greedy(ZSTD_CCtx* ctx, const void* src, size_t srcSize) +size_t ZSTD_compressBlock_greedy( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize) { - return ZSTD_compressBlock_lazy_generic(ctx, src, srcSize, 0, 0); + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, cParams, src, srcSize, 0, 0); } FORCE_INLINE_TEMPLATE -size_t ZSTD_compressBlock_lazy_extDict_generic(ZSTD_CCtx* ctx, - const void* src, size_t srcSize, - const U32 searchMethod, const U32 depth) +size_t ZSTD_compressBlock_lazy_extDict_generic( + ZSTD_matchState_t* ms, seqStore_t* seqStore, + U32 rep[ZSTD_REP_NUM], + ZSTD_compressionParameters const* cParams, + const void* src, size_t srcSize, + const U32 searchMethod, const U32 depth) { - seqStore_t* seqStorePtr = &(ctx->seqStore); const BYTE* const istart = (const BYTE*)src; const BYTE* ip = istart; const BYTE* anchor = istart; const BYTE* const iend = istart + srcSize; const BYTE* const ilimit = iend - 8; - const BYTE* const base = ctx->base; - const U32 dictLimit = ctx->dictLimit; - const U32 lowestIndex = ctx->lowLimit; + const BYTE* const base = ms->base; + const U32 dictLimit = ms->dictLimit; + const U32 lowestIndex = ms->lowLimit; const BYTE* const prefixStart = base + dictLimit; - const BYTE* const dictBase = ctx->dictBase; + const BYTE* const dictBase = ms->dictBase; const BYTE* const dictEnd = dictBase + dictLimit; - const BYTE* const dictStart = dictBase + ctx->lowLimit; + const BYTE* const dictStart = dictBase + ms->lowLimit; - const U32 maxSearches = 1 << ctx->appliedParams.cParams.searchLog; - const U32 mls = ctx->appliedParams.cParams.searchLength; - - typedef size_t (*searchMax_f)(ZSTD_CCtx* zc, const BYTE* ip, const BYTE* iLimit, - size_t* offsetPtr, - U32 maxNbAttempts, U32 matchLengthSearch); + typedef size_t (*searchMax_f)( + ZSTD_matchState_t* ms, ZSTD_compressionParameters const* cParams, + const BYTE* ip, const BYTE* iLimit, size_t* offsetPtr); searchMax_f searchMax = searchMethod ? ZSTD_BtFindBestMatch_selectMLS_extDict : ZSTD_HcFindBestMatch_extDict_selectMLS; - U32 offset_1 = seqStorePtr->rep[0], offset_2 = seqStorePtr->rep[1]; + U32 offset_1 = rep[0], offset_2 = rep[1]; /* init */ - ctx->nextToUpdate3 = ctx->nextToUpdate; + ms->nextToUpdate3 = ms->nextToUpdate; ip += (ip == prefixStart); /* Match Loop */ @@ -690,7 +704,7 @@ size_t ZSTD_compressBlock_lazy_extDict_generic(ZSTD_CCtx* ctx, /* first search (depth 0) */ { size_t offsetFound = 99999999; - size_t const ml2 = searchMax(ctx, ip, iend, &offsetFound, maxSearches, mls); + size_t const ml2 = searchMax(ms, cParams, ip, iend, &offsetFound); if (ml2 > matchLength) matchLength = ml2, start = ip, offset=offsetFound; } @@ -723,7 +737,7 @@ size_t ZSTD_compressBlock_lazy_extDict_generic(ZSTD_CCtx* ctx, /* search match, depth 1 */ { size_t offset2=99999999; - size_t const ml2 = searchMax(ctx, ip, iend, &offset2, maxSearches, mls); + size_t const ml2 = searchMax(ms, cParams, ip, iend, &offset2); int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)offset2+1)); /* raw approx */ int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offset+1) + 4); if ((ml2 >= 4) && (gain2 > gain1)) { @@ -753,7 +767,7 @@ size_t ZSTD_compressBlock_lazy_extDict_generic(ZSTD_CCtx* ctx, /* search match, depth 2 */ { size_t offset2=99999999; - size_t const ml2 = searchMax(ctx, ip, iend, &offset2, maxSearches, mls); + size_t const ml2 = searchMax(ms, cParams, ip, iend, &offset2); int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)offset2+1)); /* raw approx */ int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offset+1) + 7); if ((ml2 >= 4) && (gain2 > gain1)) { @@ -775,7 +789,7 @@ size_t ZSTD_compressBlock_lazy_extDict_generic(ZSTD_CCtx* ctx, /* store sequence */ _storeSequence: { size_t const litLength = start - anchor; - ZSTD_storeSeq(seqStorePtr, litLength, anchor, (U32)offset, matchLength-MINMATCH); + ZSTD_storeSeq(seqStore, litLength, anchor, (U32)offset, matchLength-MINMATCH); anchor = ip = start + matchLength; } @@ -790,7 +804,7 @@ _storeSequence: const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend; matchLength = ZSTD_count_2segments(ip+4, repMatch+4, iend, repEnd, prefixStart) + 4; offset = offset_2; offset_2 = offset_1; offset_1 = (U32)offset; /* swap offset history */ - ZSTD_storeSeq(seqStorePtr, 0, anchor, 0, matchLength-MINMATCH); + ZSTD_storeSeq(seqStore, 0, anchor, 0, matchLength-MINMATCH); ip += matchLength; anchor = ip; continue; /* faster when present ... (?) */ @@ -799,29 +813,41 @@ _storeSequence: } } /* Save reps for next block */ - seqStorePtr->repToConfirm[0] = offset_1; seqStorePtr->repToConfirm[1] = offset_2; + rep[0] = offset_1; + rep[1] = offset_2; /* Return the last literals size */ return iend - anchor; } -size_t ZSTD_compressBlock_greedy_extDict(ZSTD_CCtx* ctx, const void* src, size_t srcSize) +size_t ZSTD_compressBlock_greedy_extDict( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize) { - return ZSTD_compressBlock_lazy_extDict_generic(ctx, src, srcSize, 0, 0); + return ZSTD_compressBlock_lazy_extDict_generic(ms, seqStore, rep, cParams, src, srcSize, 0, 0); } -size_t ZSTD_compressBlock_lazy_extDict(ZSTD_CCtx* ctx, const void* src, size_t srcSize) +size_t ZSTD_compressBlock_lazy_extDict( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize) + { - return ZSTD_compressBlock_lazy_extDict_generic(ctx, src, srcSize, 0, 1); + return ZSTD_compressBlock_lazy_extDict_generic(ms, seqStore, rep, cParams, src, srcSize, 0, 1); } -size_t ZSTD_compressBlock_lazy2_extDict(ZSTD_CCtx* ctx, const void* src, size_t srcSize) +size_t ZSTD_compressBlock_lazy2_extDict( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize) + { - return ZSTD_compressBlock_lazy_extDict_generic(ctx, src, srcSize, 0, 2); + return ZSTD_compressBlock_lazy_extDict_generic(ms, seqStore, rep, cParams, src, srcSize, 0, 2); } -size_t ZSTD_compressBlock_btlazy2_extDict(ZSTD_CCtx* ctx, const void* src, size_t srcSize) +size_t ZSTD_compressBlock_btlazy2_extDict( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize) + { - return ZSTD_compressBlock_lazy_extDict_generic(ctx, src, srcSize, 1, 2); + return ZSTD_compressBlock_lazy_extDict_generic(ms, seqStore, rep, cParams, src, srcSize, 1, 2); } diff --git a/lib/compress/zstd_lazy.h b/lib/compress/zstd_lazy.h index 5c02f166..bda064f1 100644 --- a/lib/compress/zstd_lazy.h +++ b/lib/compress/zstd_lazy.h @@ -15,22 +15,39 @@ extern "C" { #endif -#include "mem.h" /* U32 */ -#include "zstd.h" /* ZSTD_CCtx, size_t */ +#include "zstd_compress_internal.h" + +U32 ZSTD_insertAndFindFirstIndex( + ZSTD_matchState_t* ms, ZSTD_compressionParameters const* cParams, + const BYTE* ip); -U32 ZSTD_insertAndFindFirstIndex (ZSTD_CCtx* zc, const BYTE* ip, U32 mls); /* used in ZSTD_loadDictionaryContent() */ -void ZSTD_updateDUBT(ZSTD_CCtx* zc, const BYTE* ip, const BYTE* iend, U32 mls); /* used in ZSTD_loadDictionaryContent() */ void ZSTD_preserveUnsortedMark (U32* const table, U32 const size, U32 const reducerValue); /*! used in ZSTD_reduceIndex(). pre-emptively increase value of ZSTD_DUBT_UNSORTED_MARK */ -size_t ZSTD_compressBlock_btlazy2(ZSTD_CCtx* ctx, const void* src, size_t srcSize); -size_t ZSTD_compressBlock_lazy2(ZSTD_CCtx* ctx, const void* src, size_t srcSize); -size_t ZSTD_compressBlock_lazy(ZSTD_CCtx* ctx, const void* src, size_t srcSize); -size_t ZSTD_compressBlock_greedy(ZSTD_CCtx* ctx, const void* src, size_t srcSize); +size_t ZSTD_compressBlock_btlazy2( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize); +size_t ZSTD_compressBlock_lazy2( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize); +size_t ZSTD_compressBlock_lazy( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize); +size_t ZSTD_compressBlock_greedy( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize); -size_t ZSTD_compressBlock_greedy_extDict(ZSTD_CCtx* ctx, const void* src, size_t srcSize); -size_t ZSTD_compressBlock_lazy_extDict(ZSTD_CCtx* ctx, const void* src, size_t srcSize); -size_t ZSTD_compressBlock_lazy2_extDict(ZSTD_CCtx* ctx, const void* src, size_t srcSize); -size_t ZSTD_compressBlock_btlazy2_extDict(ZSTD_CCtx* ctx, const void* src, size_t srcSize); +size_t ZSTD_compressBlock_greedy_extDict( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize); +size_t ZSTD_compressBlock_lazy_extDict( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize); +size_t ZSTD_compressBlock_lazy2_extDict( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize); +size_t ZSTD_compressBlock_btlazy2_extDict( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize); #if defined (__cplusplus) } diff --git a/lib/compress/zstd_ldm.c b/lib/compress/zstd_ldm.c index be50872c..cc71de2e 100644 --- a/lib/compress/zstd_ldm.c +++ b/lib/compress/zstd_ldm.c @@ -205,21 +205,22 @@ static size_t ZSTD_ldm_countBackwardsMatch( * * The tables for the other strategies are filled within their * block compressors. */ -static size_t ZSTD_ldm_fillFastTables(ZSTD_CCtx* zc, const void* end) +static size_t ZSTD_ldm_fillFastTables(ZSTD_matchState_t* ms, + ZSTD_compressionParameters const* cParams, + void const* end) { const BYTE* const iend = (const BYTE*)end; - const U32 mls = zc->appliedParams.cParams.searchLength; - switch(zc->appliedParams.cParams.strategy) + switch(cParams->strategy) { case ZSTD_fast: - ZSTD_fillHashTable(zc, iend, mls); - zc->nextToUpdate = (U32)(iend - zc->base); + ZSTD_fillHashTable(ms, cParams, iend); + ms->nextToUpdate = (U32)(iend - ms->base); break; case ZSTD_dfast: - ZSTD_fillDoubleHashTable(zc, iend, mls); - zc->nextToUpdate = (U32)(iend - zc->base); + ZSTD_fillDoubleHashTable(ms, cParams, iend); + ms->nextToUpdate = (U32)(iend - ms->base); break; case ZSTD_greedy: @@ -268,51 +269,41 @@ static U64 ZSTD_ldm_fillLdmHashTable(ldmState_t* state, * Sets cctx->nextToUpdate to a position corresponding closer to anchor * if it is far way * (after a long match, only update tables a limited amount). */ -static void ZSTD_ldm_limitTableUpdate(ZSTD_CCtx* cctx, const BYTE* anchor) +static void ZSTD_ldm_limitTableUpdate(ZSTD_matchState_t* ms, const BYTE* anchor) { - U32 const current = (U32)(anchor - cctx->base); - if (current > cctx->nextToUpdate + 1024) { - cctx->nextToUpdate = - current - MIN(512, current - cctx->nextToUpdate - 1024); + U32 const current = (U32)(anchor - ms->base); + if (current > ms->nextToUpdate + 1024) { + ms->nextToUpdate = + current - MIN(512, current - ms->nextToUpdate - 1024); } } -typedef size_t (*ZSTD_blockCompressor) (ZSTD_CCtx* ctx, const void* src, size_t srcSize); -/* defined in zstd_compress.c */ -ZSTD_blockCompressor ZSTD_selectBlockCompressor(ZSTD_strategy strat, int extDict); +size_t ZSTD_compressBlock_ldm( + ldmState_t* ldmState, ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + ZSTD_CCtx_params const* params, void const* src, size_t srcSize) -FORCE_INLINE_TEMPLATE -size_t ZSTD_compressBlock_ldm_generic(ZSTD_CCtx* cctx, - const void* src, size_t srcSize) { - ldmState_t* const ldmState = &(cctx->ldmState); - const ldmParams_t ldmParams = cctx->appliedParams.ldmParams; + ZSTD_compressionParameters const* cParams = ¶ms->cParams; + const ldmParams_t ldmParams = params->ldmParams; const U64 hashPower = ldmState->hashPower; const U32 hBits = ldmParams.hashLog - ldmParams.bucketSizeLog; const U32 ldmBucketSize = ((U32)1 << ldmParams.bucketSizeLog); const U32 ldmTagMask = ((U32)1 << ldmParams.hashEveryLog) - 1; - seqStore_t* const seqStorePtr = &(cctx->seqStore); - const BYTE* const base = cctx->base; + const BYTE* const base = ms->base; const BYTE* const istart = (const BYTE*)src; const BYTE* ip = istart; const BYTE* anchor = istart; - const U32 lowestIndex = cctx->dictLimit; + const U32 lowestIndex = ms->dictLimit; const BYTE* const lowest = base + lowestIndex; const BYTE* const iend = istart + srcSize; const BYTE* const ilimit = iend - MAX(ldmParams.minMatchLength, HASH_READ_SIZE); const ZSTD_blockCompressor blockCompressor = - ZSTD_selectBlockCompressor(cctx->appliedParams.cParams.strategy, 0); - U32* const repToConfirm = seqStorePtr->repToConfirm; - U32 savedRep[ZSTD_REP_NUM]; + ZSTD_selectBlockCompressor(cParams->strategy, 0); U64 rollingHash = 0; const BYTE* lastHashed = NULL; size_t i, lastLiterals; - /* Save seqStorePtr->rep and copy repToConfirm */ - for (i = 0; i < ZSTD_REP_NUM; i++) - savedRep[i] = repToConfirm[i] = seqStorePtr->rep[i]; - /* Main Search Loop */ while (ip < ilimit) { /* < instead of <=, because repcode check at (ip+1) */ size_t mLength; @@ -390,25 +381,21 @@ size_t ZSTD_compressBlock_ldm_generic(ZSTD_CCtx* cctx, const BYTE* const match = base + matchIndex - backwardMatchLength; U32 const offset = (U32)(ip - match); - /* Overwrite rep codes */ - for (i = 0; i < ZSTD_REP_NUM; i++) - seqStorePtr->rep[i] = repToConfirm[i]; - /* Fill tables for block compressor */ - ZSTD_ldm_limitTableUpdate(cctx, anchor); - ZSTD_ldm_fillFastTables(cctx, anchor); + ZSTD_ldm_limitTableUpdate(ms, anchor); + ZSTD_ldm_fillFastTables(ms, cParams, anchor); /* Call block compressor and get remaining literals */ - lastLiterals = blockCompressor(cctx, anchor, ip - anchor); - cctx->nextToUpdate = (U32)(ip - base); + lastLiterals = blockCompressor(ms, seqStore, rep, cParams, anchor, ip - anchor); + ms->nextToUpdate = (U32)(ip - base); /* Update repToConfirm with the new offset */ for (i = ZSTD_REP_NUM - 1; i > 0; i--) - repToConfirm[i] = repToConfirm[i-1]; - repToConfirm[0] = offset; + rep[i] = rep[i-1]; + rep[0] = offset; /* Store the sequence with the leftover literals */ - ZSTD_storeSeq(seqStorePtr, lastLiterals, ip - lastLiterals, + ZSTD_storeSeq(seqStore, lastLiterals, ip - lastLiterals, offset + ZSTD_REP_MOVE, mLength - MINMATCH); } @@ -431,19 +418,19 @@ size_t ZSTD_compressBlock_ldm_generic(ZSTD_CCtx* cctx, anchor = ip; /* Check immediate repcode */ while ( (ip < ilimit) - && ( (repToConfirm[1] > 0) && (repToConfirm[1] <= (U32)(ip-lowest)) - && (MEM_read32(ip) == MEM_read32(ip - repToConfirm[1])) )) { + && ( (rep[1] > 0) && (rep[1] <= (U32)(ip-lowest)) + && (MEM_read32(ip) == MEM_read32(ip - rep[1])) )) { - size_t const rLength = ZSTD_count(ip+4, ip+4-repToConfirm[1], + size_t const rLength = ZSTD_count(ip+4, ip+4-rep[1], iend) + 4; /* Swap repToConfirm[1] <=> repToConfirm[0] */ { - U32 const tmpOff = repToConfirm[1]; - repToConfirm[1] = repToConfirm[0]; - repToConfirm[0] = tmpOff; + U32 const tmpOff = rep[1]; + rep[1] = rep[0]; + rep[0] = tmpOff; } - ZSTD_storeSeq(seqStorePtr, 0, anchor, 0, rLength-MINMATCH); + ZSTD_storeSeq(seqStore, 0, anchor, 0, rLength-MINMATCH); /* Fill the hash table from lastHashed+1 to ip+rLength*/ if (ip + rLength < ilimit) { @@ -457,67 +444,45 @@ size_t ZSTD_compressBlock_ldm_generic(ZSTD_CCtx* cctx, } } - /* Overwrite rep */ - for (i = 0; i < ZSTD_REP_NUM; i++) - seqStorePtr->rep[i] = repToConfirm[i]; + ZSTD_ldm_limitTableUpdate(ms, anchor); + ZSTD_ldm_fillFastTables(ms, cParams, anchor); - ZSTD_ldm_limitTableUpdate(cctx, anchor); - ZSTD_ldm_fillFastTables(cctx, anchor); - - lastLiterals = blockCompressor(cctx, anchor, iend - anchor); - cctx->nextToUpdate = (U32)(iend - base); - - /* Restore seqStorePtr->rep */ - for (i = 0; i < ZSTD_REP_NUM; i++) - seqStorePtr->rep[i] = savedRep[i]; + lastLiterals = blockCompressor(ms, seqStore, rep, cParams, anchor, iend - anchor); + ms->nextToUpdate = (U32)(iend - base); /* Return the last literals size */ return lastLiterals; } -size_t ZSTD_compressBlock_ldm(ZSTD_CCtx* ctx, - const void* src, size_t srcSize) +size_t ZSTD_compressBlock_ldm_extDict( + ldmState_t* ldmState, ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + ZSTD_CCtx_params const* params, void const* src, size_t srcSize) { - return ZSTD_compressBlock_ldm_generic(ctx, src, srcSize); -} - -static size_t ZSTD_compressBlock_ldm_extDict_generic( - ZSTD_CCtx* ctx, - const void* src, size_t srcSize) -{ - ldmState_t* const ldmState = &(ctx->ldmState); - const ldmParams_t ldmParams = ctx->appliedParams.ldmParams; + const ldmParams_t ldmParams = params->ldmParams; + ZSTD_compressionParameters const* cParams = ¶ms->cParams; const U64 hashPower = ldmState->hashPower; const U32 hBits = ldmParams.hashLog - ldmParams.bucketSizeLog; const U32 ldmBucketSize = ((U32)1 << ldmParams.bucketSizeLog); const U32 ldmTagMask = ((U32)1 << ldmParams.hashEveryLog) - 1; - seqStore_t* const seqStorePtr = &(ctx->seqStore); - const BYTE* const base = ctx->base; - const BYTE* const dictBase = ctx->dictBase; + const BYTE* const base = ms->base; + const BYTE* const dictBase = ms->dictBase; const BYTE* const istart = (const BYTE*)src; const BYTE* ip = istart; const BYTE* anchor = istart; - const U32 lowestIndex = ctx->lowLimit; + const U32 lowestIndex = ms->lowLimit; const BYTE* const dictStart = dictBase + lowestIndex; - const U32 dictLimit = ctx->dictLimit; + const U32 dictLimit = ms->dictLimit; const BYTE* const lowPrefixPtr = base + dictLimit; const BYTE* const dictEnd = dictBase + dictLimit; const BYTE* const iend = istart + srcSize; const BYTE* const ilimit = iend - MAX(ldmParams.minMatchLength, HASH_READ_SIZE); const ZSTD_blockCompressor blockCompressor = - ZSTD_selectBlockCompressor(ctx->appliedParams.cParams.strategy, 1); - U32* const repToConfirm = seqStorePtr->repToConfirm; - U32 savedRep[ZSTD_REP_NUM]; + ZSTD_selectBlockCompressor(cParams->strategy, 1); U64 rollingHash = 0; const BYTE* lastHashed = NULL; size_t i, lastLiterals; - /* Save seqStorePtr->rep and copy repToConfirm */ - for (i = 0; i < ZSTD_REP_NUM; i++) { - savedRep[i] = repToConfirm[i] = seqStorePtr->rep[i]; - } - /* Search Loop */ while (ip < ilimit) { /* < instead of <=, because (ip+1) */ size_t mLength; @@ -605,25 +570,21 @@ static size_t ZSTD_compressBlock_ldm_extDict_generic( U32 const matchIndex = bestEntry->offset; U32 const offset = current - matchIndex; - /* Overwrite rep codes */ - for (i = 0; i < ZSTD_REP_NUM; i++) - seqStorePtr->rep[i] = repToConfirm[i]; - /* Fill the hash table for the block compressor */ - ZSTD_ldm_limitTableUpdate(ctx, anchor); - ZSTD_ldm_fillFastTables(ctx, anchor); + ZSTD_ldm_limitTableUpdate(ms, anchor); + ZSTD_ldm_fillFastTables(ms, cParams, anchor); /* Call block compressor and get remaining literals */ - lastLiterals = blockCompressor(ctx, anchor, ip - anchor); - ctx->nextToUpdate = (U32)(ip - base); + lastLiterals = blockCompressor(ms, seqStore, rep, cParams, anchor, ip - anchor); + ms->nextToUpdate = (U32)(ip - base); /* Update repToConfirm with the new offset */ for (i = ZSTD_REP_NUM - 1; i > 0; i--) - repToConfirm[i] = repToConfirm[i-1]; - repToConfirm[0] = offset; + rep[i] = rep[i-1]; + rep[0] = offset; /* Store the sequence with the leftover literals */ - ZSTD_storeSeq(seqStorePtr, lastLiterals, ip - lastLiterals, + ZSTD_storeSeq(seqStore, lastLiterals, ip - lastLiterals, offset + ZSTD_REP_MOVE, mLength - MINMATCH); } @@ -647,7 +608,7 @@ static size_t ZSTD_compressBlock_ldm_extDict_generic( /* check immediate repcode */ while (ip < ilimit) { U32 const current2 = (U32)(ip-base); - U32 const repIndex2 = current2 - repToConfirm[1]; + U32 const repIndex2 = current2 - rep[1]; const BYTE* repMatch2 = repIndex2 < dictLimit ? dictBase + repIndex2 : base + repIndex2; if ( (((U32)((dictLimit-1) - repIndex2) >= 3) & @@ -659,11 +620,11 @@ static size_t ZSTD_compressBlock_ldm_extDict_generic( ZSTD_count_2segments(ip+4, repMatch2+4, iend, repEnd2, lowPrefixPtr) + 4; - U32 tmpOffset = repToConfirm[1]; - repToConfirm[1] = repToConfirm[0]; - repToConfirm[0] = tmpOffset; + U32 tmpOffset = rep[1]; + rep[1] = rep[0]; + rep[0] = tmpOffset; - ZSTD_storeSeq(seqStorePtr, 0, anchor, 0, repLength2-MINMATCH); + ZSTD_storeSeq(seqStore, 0, anchor, 0, repLength2-MINMATCH); /* Fill the hash table from lastHashed+1 to ip+repLength2*/ if (ip + repLength2 < ilimit) { @@ -681,27 +642,13 @@ static size_t ZSTD_compressBlock_ldm_extDict_generic( } } - /* Overwrite rep */ - for (i = 0; i < ZSTD_REP_NUM; i++) - seqStorePtr->rep[i] = repToConfirm[i]; - - ZSTD_ldm_limitTableUpdate(ctx, anchor); - ZSTD_ldm_fillFastTables(ctx, anchor); + ZSTD_ldm_limitTableUpdate(ms, anchor); + ZSTD_ldm_fillFastTables(ms, cParams, anchor); /* Call the block compressor one last time on the last literals */ - lastLiterals = blockCompressor(ctx, anchor, iend - anchor); - ctx->nextToUpdate = (U32)(iend - base); - - /* Restore seqStorePtr->rep */ - for (i = 0; i < ZSTD_REP_NUM; i++) - seqStorePtr->rep[i] = savedRep[i]; + lastLiterals = blockCompressor(ms, seqStore, rep, cParams, anchor, iend - anchor); + ms->nextToUpdate = (U32)(iend - base); /* Return the last literals size */ return lastLiterals; } - -size_t ZSTD_compressBlock_ldm_extDict(ZSTD_CCtx* ctx, - const void* src, size_t srcSize) -{ - return ZSTD_compressBlock_ldm_extDict_generic(ctx, src, srcSize); -} diff --git a/lib/compress/zstd_ldm.h b/lib/compress/zstd_ldm.h index 8f12c677..7041c65a 100644 --- a/lib/compress/zstd_ldm.h +++ b/lib/compress/zstd_ldm.h @@ -37,9 +37,13 @@ extern "C" { * parameters), which stores the matched sequences. The "long distance" * match is then stored with the remaining literals from the * ZSTD_blockCompressor. */ -size_t ZSTD_compressBlock_ldm(ZSTD_CCtx* cctx, const void* src, size_t srcSize); -size_t ZSTD_compressBlock_ldm_extDict(ZSTD_CCtx* ctx, - const void* src, size_t srcSize); +size_t ZSTD_compressBlock_ldm( + ldmState_t* ldms, ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + ZSTD_CCtx_params const* params, void const* src, size_t srcSize); + +size_t ZSTD_compressBlock_ldm_extDict( + ldmState_t* ldms, ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + ZSTD_CCtx_params const* params, void const* src, size_t srcSize); /** ZSTD_ldm_initializeParameters() : * Initialize the long distance matching parameters to their default values. */ diff --git a/lib/compress/zstd_opt.c b/lib/compress/zstd_opt.c index 04824ccf..1c56ff32 100644 --- a/lib/compress/zstd_opt.c +++ b/lib/compress/zstd_opt.c @@ -243,13 +243,13 @@ MEM_STATIC U32 ZSTD_readMINMATCH(const void* memPtr, U32 length) /* Update hashTable3 up to ip (excluded) Assumption : always within prefix (i.e. not within extDict) */ -static U32 ZSTD_insertAndFindFirstIndexHash3 (ZSTD_CCtx* const cctx, const BYTE* const ip) +static U32 ZSTD_insertAndFindFirstIndexHash3 (ZSTD_matchState_t* ms, const BYTE* const ip) { - U32* const hashTable3 = cctx->hashTable3; - U32 const hashLog3 = cctx->hashLog3; - const BYTE* const base = cctx->base; - U32 idx = cctx->nextToUpdate3; - U32 const target = cctx->nextToUpdate3 = (U32)(ip - base); + U32* const hashTable3 = ms->hashTable3; + U32 const hashLog3 = ms->hashLog3; + const BYTE* const base = ms->base; + U32 idx = ms->nextToUpdate3; + U32 const target = ms->nextToUpdate3 = (U32)(ip - base); size_t const hash3 = ZSTD_hash3Ptr(ip, hashLog3); while(idx < target) { @@ -267,21 +267,22 @@ static U32 ZSTD_insertAndFindFirstIndexHash3 (ZSTD_CCtx* const cctx, const BYTE* /** ZSTD_insertBt1() : add one or multiple positions to tree. * ip : assumed <= iend-8 . * @return : nb of positions added */ -static U32 ZSTD_insertBt1(ZSTD_CCtx* zc, +static U32 ZSTD_insertBt1( + ZSTD_matchState_t* ms, ZSTD_compressionParameters const* cParams, const BYTE* const ip, const BYTE* const iend, - U32 nbCompares, U32 const mls, U32 const extDict) + U32 const mls, U32 const extDict) { - U32* const hashTable = zc->hashTable; - U32 const hashLog = zc->appliedParams.cParams.hashLog; + U32* const hashTable = ms->hashTable; + U32 const hashLog = cParams->hashLog; size_t const h = ZSTD_hashPtr(ip, hashLog, mls); - U32* const bt = zc->chainTable; - U32 const btLog = zc->appliedParams.cParams.chainLog - 1; + U32* const bt = ms->chainTable; + U32 const btLog = cParams->chainLog - 1; U32 const btMask = (1 << btLog) - 1; U32 matchIndex = hashTable[h]; size_t commonLengthSmaller=0, commonLengthLarger=0; - const BYTE* const base = zc->base; - const BYTE* const dictBase = zc->dictBase; - const U32 dictLimit = zc->dictLimit; + const BYTE* const base = ms->base; + const BYTE* const dictBase = ms->dictBase; + const U32 dictLimit = ms->dictLimit; const BYTE* const dictEnd = dictBase + dictLimit; const BYTE* const prefixStart = base + dictLimit; const BYTE* match; @@ -290,9 +291,10 @@ static U32 ZSTD_insertBt1(ZSTD_CCtx* zc, U32* smallerPtr = bt + 2*(current&btMask); U32* largerPtr = smallerPtr + 1; U32 dummy32; /* to be nullified at the end */ - U32 const windowLow = zc->lowLimit; + U32 const windowLow = ms->lowLimit; U32 matchEndIdx = current+8+1; size_t bestLength = 8; + U32 nbCompares = 1U << cParams->searchLog; #ifdef ZSTD_C_PREDICT U32 predictedSmall = *(bt + 2*((current-1)&btMask) + 0); U32 predictedLarge = *(bt + 2*((current-1)&btMask) + 1); @@ -375,66 +377,60 @@ static U32 ZSTD_insertBt1(ZSTD_CCtx* zc, } FORCE_INLINE_TEMPLATE -void ZSTD_updateTree_internal(ZSTD_CCtx* zc, +void ZSTD_updateTree_internal( + ZSTD_matchState_t* ms, ZSTD_compressionParameters const* cParams, const BYTE* const ip, const BYTE* const iend, - const U32 nbCompares, const U32 mls, const U32 extDict) + const U32 mls, const U32 extDict) { - const BYTE* const base = zc->base; + const BYTE* const base = ms->base; U32 const target = (U32)(ip - base); - U32 idx = zc->nextToUpdate; + U32 idx = ms->nextToUpdate; DEBUGLOG(7, "ZSTD_updateTree_internal, from %u to %u (extDict:%u)", idx, target, extDict); while(idx < target) - idx += ZSTD_insertBt1(zc, base+idx, iend, nbCompares, mls, extDict); - zc->nextToUpdate = target; + idx += ZSTD_insertBt1(ms, cParams, base+idx, iend, mls, extDict); + ms->nextToUpdate = target; } -void ZSTD_updateTree(ZSTD_CCtx* zc, - const BYTE* ip, const BYTE* iend, - U32 nbCompares, U32 mls) +void ZSTD_updateTree( + ZSTD_matchState_t* ms, ZSTD_compressionParameters const* cParams, + const BYTE* ip, const BYTE* iend) { - ZSTD_updateTree_internal(zc, ip, iend, nbCompares, mls, 0 /*extDict*/); + ZSTD_updateTree_internal(ms, cParams, ip, iend, cParams->searchLength, 0 /*extDict*/); } -static void ZSTD_updateTree_extDict(ZSTD_CCtx* zc, - const BYTE* const ip, const BYTE* const iend, - const U32 nbCompares, const U32 mls) -{ - ZSTD_updateTree_internal(zc, ip, iend, nbCompares, mls, 1 /*extDict*/); -} - - FORCE_INLINE_TEMPLATE U32 ZSTD_insertBtAndGetAllMatches ( - ZSTD_CCtx* zc, + ZSTD_matchState_t* ms, ZSTD_compressionParameters const* cParams, const BYTE* const ip, const BYTE* const iLimit, int const extDict, - U32 nbCompares, U32 const mls, U32 const sufficient_len, U32 rep[ZSTD_REP_NUM], U32 const ll0, - ZSTD_match_t* matches, const U32 lengthToBeat) + ZSTD_match_t* matches, const U32 lengthToBeat, U32 const mls /* template */) { - const BYTE* const base = zc->base; + U32 const sufficient_len = MIN(cParams->targetLength, ZSTD_OPT_NUM -1); + const BYTE* const base = ms->base; U32 const current = (U32)(ip-base); - U32 const hashLog = zc->appliedParams.cParams.hashLog; + U32 const hashLog = cParams->hashLog; U32 const minMatch = (mls==3) ? 3 : 4; - U32* const hashTable = zc->hashTable; + U32* const hashTable = ms->hashTable; size_t const h = ZSTD_hashPtr(ip, hashLog, mls); U32 matchIndex = hashTable[h]; - U32* const bt = zc->chainTable; - U32 const btLog = zc->appliedParams.cParams.chainLog - 1; + U32* const bt = ms->chainTable; + U32 const btLog = cParams->chainLog - 1; U32 const btMask= (1U << btLog) - 1; size_t commonLengthSmaller=0, commonLengthLarger=0; - const BYTE* const dictBase = zc->dictBase; - U32 const dictLimit = zc->dictLimit; + const BYTE* const dictBase = ms->dictBase; + U32 const dictLimit = ms->dictLimit; const BYTE* const dictEnd = dictBase + dictLimit; const BYTE* const prefixStart = base + dictLimit; U32 const btLow = btMask >= current ? 0 : current - btMask; - U32 const windowLow = zc->lowLimit; + U32 const windowLow = ms->lowLimit; U32* smallerPtr = bt + 2*(current&btMask); U32* largerPtr = bt + 2*(current&btMask) + 1; U32 matchEndIdx = current+8+1; /* farthest referenced position of any match => detects repetitive patterns */ U32 dummy32; /* to be nullified at the end */ U32 mnum = 0; + U32 nbCompares = 1U << cParams->searchLog; size_t bestLength = lengthToBeat-1; DEBUGLOG(7, "ZSTD_insertBtAndGetAllMatches"); @@ -475,7 +471,7 @@ U32 ZSTD_insertBtAndGetAllMatches ( /* HC3 match finder */ if ((mls == 3) /*static*/ && (bestLength < mls)) { - U32 const matchIndex3 = ZSTD_insertAndFindFirstIndexHash3 (zc, ip); + U32 const matchIndex3 = ZSTD_insertAndFindFirstIndexHash3(ms, ip); if ((matchIndex3 > windowLow) & (current - matchIndex3 < (1<<18)) /*heuristic : longer distance likely too expensive*/ ) { size_t mlen; @@ -499,7 +495,7 @@ U32 ZSTD_insertBtAndGetAllMatches ( mnum = 1; if ( (mlen > sufficient_len) | (ip+mlen == iLimit) ) { /* best possible length */ - zc->nextToUpdate = current+1; /* skip insertion */ + ms->nextToUpdate = current+1; /* skip insertion */ return 1; } } } } @@ -556,30 +552,29 @@ U32 ZSTD_insertBtAndGetAllMatches ( *smallerPtr = *largerPtr = 0; assert(matchEndIdx > current+8); - zc->nextToUpdate = matchEndIdx - 8; /* skip repetitive patterns */ + ms->nextToUpdate = matchEndIdx - 8; /* skip repetitive patterns */ return mnum; } FORCE_INLINE_TEMPLATE U32 ZSTD_BtGetAllMatches ( - ZSTD_CCtx* zc, /* Index table will be updated */ + ZSTD_matchState_t* ms, ZSTD_compressionParameters const* cParams, const BYTE* ip, const BYTE* const iHighLimit, int const extDict, - U32 const maxNbAttempts, U32 const matchLengthSearch, U32 const sufficient_len, U32 rep[ZSTD_REP_NUM], U32 const ll0, ZSTD_match_t* matches, U32 const lengthToBeat) { + U32 const matchLengthSearch = cParams->searchLength; DEBUGLOG(7, "ZSTD_BtGetAllMatches"); - if (ip < zc->base + zc->nextToUpdate) return 0; /* skipped area */ - if (extDict) ZSTD_updateTree_extDict(zc, ip, iHighLimit, maxNbAttempts, matchLengthSearch); - else ZSTD_updateTree(zc, ip, iHighLimit, maxNbAttempts, matchLengthSearch); + if (ip < ms->base + ms->nextToUpdate) return 0; /* skipped area */ + ZSTD_updateTree_internal(ms, cParams, ip, iHighLimit, matchLengthSearch, extDict); switch(matchLengthSearch) { - case 3 : return ZSTD_insertBtAndGetAllMatches(zc, ip, iHighLimit, extDict, maxNbAttempts, 3, sufficient_len, rep, ll0, matches, lengthToBeat); + case 3 : return ZSTD_insertBtAndGetAllMatches(ms, cParams, ip, iHighLimit, extDict, rep, ll0, matches, lengthToBeat, 3); default : - case 4 : return ZSTD_insertBtAndGetAllMatches(zc, ip, iHighLimit, extDict, maxNbAttempts, 4, sufficient_len, rep, ll0, matches, lengthToBeat); - case 5 : return ZSTD_insertBtAndGetAllMatches(zc, ip, iHighLimit, extDict, maxNbAttempts, 5, sufficient_len, rep, ll0, matches, lengthToBeat); + case 4 : return ZSTD_insertBtAndGetAllMatches(ms, cParams, ip, iHighLimit, extDict, rep, ll0, matches, lengthToBeat, 4); + case 5 : return ZSTD_insertBtAndGetAllMatches(ms, cParams, ip, iHighLimit, extDict, rep, ll0, matches, lengthToBeat, 5); case 7 : - case 6 : return ZSTD_insertBtAndGetAllMatches(zc, ip, iHighLimit, extDict, maxNbAttempts, 6, sufficient_len, rep, ll0, matches, lengthToBeat); + case 6 : return ZSTD_insertBtAndGetAllMatches(ms, cParams, ip, iHighLimit, extDict, rep, ll0, matches, lengthToBeat, 6); } } @@ -667,36 +662,33 @@ static int ZSTD_literalsContribution_cached( } FORCE_INLINE_TEMPLATE -size_t ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx, +size_t ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms,seqStore_t* seqStore, + U32 rep[ZSTD_REP_NUM], + ZSTD_compressionParameters const* cParams, const void* src, size_t srcSize, const int optLevel, const int extDict) { - seqStore_t* const seqStorePtr = &(ctx->seqStore); - optState_t* const optStatePtr = &(ctx->optState); + optState_t* const optStatePtr = &ms->opt; const BYTE* const istart = (const BYTE*)src; const BYTE* ip = istart; const BYTE* anchor = istart; const BYTE* const iend = istart + srcSize; const BYTE* const ilimit = iend - 8; - const BYTE* const base = ctx->base; - const BYTE* const prefixStart = base + ctx->dictLimit; + const BYTE* const base = ms->base; + const BYTE* const prefixStart = base + ms->dictLimit; - U32 const maxSearches = 1U << ctx->appliedParams.cParams.searchLog; - U32 const sufficient_len = MIN(ctx->appliedParams.cParams.targetLength, ZSTD_OPT_NUM -1); - U32 const mls = ctx->appliedParams.cParams.searchLength; - U32 const minMatch = (ctx->appliedParams.cParams.searchLength == 3) ? 3 : 4; + U32 const sufficient_len = MIN(cParams->targetLength, ZSTD_OPT_NUM -1); + U32 const minMatch = (cParams->searchLength == 3) ? 3 : 4; ZSTD_optimal_t* const opt = optStatePtr->priceTable; ZSTD_match_t* const matches = optStatePtr->matchTable; cachedLiteralPrice_t cachedLitPrice; - U32 rep[ZSTD_REP_NUM]; /* init */ DEBUGLOG(5, "ZSTD_compressBlock_opt_generic"); - ctx->nextToUpdate3 = ctx->nextToUpdate; + ms->nextToUpdate3 = ms->nextToUpdate; ZSTD_rescaleFreqs(optStatePtr, (const BYTE*)src, srcSize); ip += (ip==prefixStart); - { int i; for (i=0; irep[i]; } memset(&cachedLitPrice, 0, sizeof(cachedLitPrice)); /* Match Loop */ @@ -707,7 +699,7 @@ size_t ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx, /* find first match */ { U32 const litlen = (U32)(ip - anchor); U32 const ll0 = !litlen; - U32 const nbMatches = ZSTD_BtGetAllMatches(ctx, ip, iend, extDict, maxSearches, mls, sufficient_len, rep, ll0, matches, minMatch); + U32 const nbMatches = ZSTD_BtGetAllMatches(ms, cParams, ip, iend, extDict, rep, ll0, matches, minMatch); if (!nbMatches) { ip++; continue; } /* initialize opt[0] */ @@ -793,7 +785,7 @@ size_t ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx, U32 const litlen = (opt[cur].mlen == 1) ? opt[cur].litlen : 0; U32 const previousPrice = (cur > litlen) ? opt[cur-litlen].price : 0; U32 const basePrice = previousPrice + ZSTD_fullLiteralsCost(inr-litlen, litlen, optStatePtr); - U32 const nbMatches = ZSTD_BtGetAllMatches(ctx, inr, iend, extDict, maxSearches, mls, sufficient_len, opt[cur].rep, ll0, matches, minMatch); + U32 const nbMatches = ZSTD_BtGetAllMatches(ms, cParams, inr, iend, extDict, opt[cur].rep, ll0, matches, minMatch); U32 matchNb; if (!nbMatches) continue; @@ -889,37 +881,42 @@ _shortestPath: /* cur, last_pos, best_mlen, best_off have to be set */ } ZSTD_updateStats(optStatePtr, llen, anchor, offset, mlen); - ZSTD_storeSeq(seqStorePtr, llen, anchor, offset, mlen-MINMATCH); + ZSTD_storeSeq(seqStore, llen, anchor, offset, mlen-MINMATCH); anchor = ip; } } ZSTD_setLog2Prices(optStatePtr); } /* while (ip < ilimit) */ - /* Save reps for next block */ - { int i; for (i=0; irepToConfirm[i] = rep[i]; } - /* Return the last literals size */ return iend - anchor; } -size_t ZSTD_compressBlock_btopt(ZSTD_CCtx* ctx, const void* src, size_t srcSize) +size_t ZSTD_compressBlock_btopt( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize) { DEBUGLOG(5, "ZSTD_compressBlock_btopt"); - return ZSTD_compressBlock_opt_generic(ctx, src, srcSize, 0 /*optLevel*/, 0 /*extDict*/); + return ZSTD_compressBlock_opt_generic(ms, seqStore, rep, cParams, src, srcSize, 0 /*optLevel*/, 0 /*extDict*/); } -size_t ZSTD_compressBlock_btultra(ZSTD_CCtx* ctx, const void* src, size_t srcSize) +size_t ZSTD_compressBlock_btultra( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize) { - return ZSTD_compressBlock_opt_generic(ctx, src, srcSize, 2 /*optLevel*/, 0 /*extDict*/); + return ZSTD_compressBlock_opt_generic(ms, seqStore, rep, cParams, src, srcSize, 2 /*optLevel*/, 0 /*extDict*/); } -size_t ZSTD_compressBlock_btopt_extDict(ZSTD_CCtx* ctx, const void* src, size_t srcSize) +size_t ZSTD_compressBlock_btopt_extDict( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize) { - return ZSTD_compressBlock_opt_generic(ctx, src, srcSize, 0 /*optLevel*/, 1 /*extDict*/); + return ZSTD_compressBlock_opt_generic(ms, seqStore, rep, cParams, src, srcSize, 0 /*optLevel*/, 1 /*extDict*/); } -size_t ZSTD_compressBlock_btultra_extDict(ZSTD_CCtx* ctx, const void* src, size_t srcSize) +size_t ZSTD_compressBlock_btultra_extDict( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize) { - return ZSTD_compressBlock_opt_generic(ctx, src, srcSize, 2 /*optLevel*/, 1 /*extDict*/); + return ZSTD_compressBlock_opt_generic(ms, seqStore, rep, cParams, src, srcSize, 2 /*optLevel*/, 1 /*extDict*/); } diff --git a/lib/compress/zstd_opt.h b/lib/compress/zstd_opt.h index b803a0f2..b8dc389f 100644 --- a/lib/compress/zstd_opt.h +++ b/lib/compress/zstd_opt.h @@ -15,16 +15,25 @@ extern "C" { #endif -#include "mem.h" /* U32 */ -#include "zstd.h" /* ZSTD_CCtx, size_t */ +#include "zstd_compress_internal.h" -void ZSTD_updateTree(ZSTD_CCtx* ctx, const BYTE* ip, const BYTE* iend, U32 nbCompares, U32 mls); /* used in ZSTD_loadDictionaryContent() */ +void ZSTD_updateTree( + ZSTD_matchState_t* ms, ZSTD_compressionParameters const* cParams, + const BYTE* ip, const BYTE* iend); /* used in ZSTD_loadDictionaryContent() */ -size_t ZSTD_compressBlock_btopt(ZSTD_CCtx* ctx, const void* src, size_t srcSize); -size_t ZSTD_compressBlock_btultra(ZSTD_CCtx* ctx, const void* src, size_t srcSize); +size_t ZSTD_compressBlock_btopt( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize); +size_t ZSTD_compressBlock_btultra( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize); -size_t ZSTD_compressBlock_btopt_extDict(ZSTD_CCtx* ctx, const void* src, size_t srcSize); -size_t ZSTD_compressBlock_btultra_extDict(ZSTD_CCtx* ctx, const void* src, size_t srcSize); +size_t ZSTD_compressBlock_btopt_extDict( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize); +size_t ZSTD_compressBlock_btultra_extDict( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize); #if defined (__cplusplus) }