facebook/zstd
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Split ZSTD_CCtx into smaller sub-structures

Browse Source
This commit is contained in:
Nick Terrell 2017-12-12 16:51:00 -08:00
parent a187486e3c
commit 887cd4e35e
13 changed files with 844 additions and 720 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
lib/common/zstd_internal.h
View File

@ -228,8 +228,6 @@ typedef struct {
BYTE* ofCode;
U32 longLengthID; /* 0 == no longLength; 1 == Lit.longLength; 2 == Match.longLength; */
U32 longLengthPos;
U32 rep[ZSTD_REP_NUM];
U32 repToConfirm[ZSTD_REP_NUM];
} seqStore_t;
const seqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx); /* compress & dictBuilder */

511
lib/compress/zstd_compress.c
View File

@ -81,11 +81,15 @@ ZSTD_CCtx* ZSTD_initStaticCCtx(void *workspace, size_t workspaceSize)
cctx->workSpace = (void*)(cctx+1);
cctx->workSpaceSize = workspaceSize - sizeof(ZSTD_CCtx);
/* entropy space (never moves) */
if (cctx->workSpaceSize < sizeof(ZSTD_entropyCTables_t)) return NULL;
/* statically sized space. entropyWorkspace never moves (but prev/next block swap places) */
if (cctx->workSpaceSize < HUF_WORKSPACE_SIZE + 2 * sizeof(ZSTD_blockState_t)) return NULL;
assert(((size_t)cctx->workSpace & (sizeof(void*)-1)) == 0); /* ensure correct alignment */
cctx->entropy = (ZSTD_entropyCTables_t*)cctx->workSpace;
cctx->prevBlock = (ZSTD_blockState_t*)cctx->workSpace;
cctx->nextBlock = cctx->prevBlock + 1;
{
void* const ptr = cctx->nextBlock + 1;
cctx->entropyWorkspace = (U32*)ptr;
}
return cctx;
}
@ -660,7 +664,8 @@ size_t ZSTD_estimateCCtxSize_usingCCtxParams(const ZSTD_CCtx_params* params)
U32 const hashLog3 = (cParams.searchLength>3) ?
0 : MIN(ZSTD_HASHLOG3_MAX, cParams.windowLog);
size_t const h3Size = ((size_t)1) << hashLog3;
size_t const entropySpace = sizeof(ZSTD_entropyCTables_t);
size_t const entropySpace = HUF_WORKSPACE_SIZE;
size_t const blockStateSpace = 2 * sizeof(ZSTD_blockState_t);
size_t const tableSpace = (chainSize + hSize + h3Size) * sizeof(U32);
size_t const optBudget =
@ -672,7 +677,7 @@ size_t ZSTD_estimateCCtxSize_usingCCtxParams(const ZSTD_CCtx_params* params)
ZSTD_ldm_getTableSize(params->ldmParams.hashLog,
params->ldmParams.bucketSizeLog) : 0;
size_t const neededSpace = entropySpace + tableSpace + tokenSpace +
size_t const neededSpace = entropySpace + blockStateSpace + tableSpace + tokenSpace +
optSpace + ldmSpace;
DEBUGLOG(5, "sizeof(ZSTD_CCtx) : %u", (U32)sizeof(ZSTD_CCtx));
@ -794,16 +799,41 @@ static U32 ZSTD_equivalentParams(ZSTD_CCtx_params params1,
ZSTD_sufficientBuff(buffSize1, blockSize1, buffPol2, params2.cParams, pledgedSrcSize);
}
static void ZSTD_resetBlockState(ZSTD_blockState_t* bs)
{
int i;
for (i = 0; i < ZSTD_REP_NUM; ++i)
bs->rep[i] = repStartValue[i];
bs->entropy.hufCTable_repeatMode = HUF_repeat_none;
bs->entropy.offcode_repeatMode = FSE_repeat_none;
bs->entropy.matchlength_repeatMode = FSE_repeat_none;
bs->entropy.litlength_repeatMode = FSE_repeat_none;
}
/*! ZSTD_invalidateMatchState()
* Invalidate all the matches in the match finder tables.
* Requires nextSrc and base to be set (can be NULL).
*/
static void ZSTD_invalidateMatchState(ZSTD_matchState_t* ms)
{
size_t const endT = (size_t)(ms->nextSrc - ms->base);
U32 const end = (U32)endT;
assert(endT < (3U<<30));
ms->lowLimit = end;
ms->dictLimit = end;
ms->nextToUpdate = end + 1;
ms->loadedDictEnd = 0;
ms->opt.litLengthSum = 0; /* force reset of btopt stats */
}
/*! ZSTD_continueCCtx() :
* reuse CCtx without reset (note : requires no dictionary) */
static size_t ZSTD_continueCCtx(ZSTD_CCtx* cctx, ZSTD_CCtx_params params, U64 pledgedSrcSize)
{
size_t const endT = (size_t)(cctx->nextSrc - cctx->base);
U32 const end = (U32)endT;
size_t const windowSize = MAX(1, (size_t)MIN(((U64)1 << params.cParams.windowLog), pledgedSrcSize));
size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, windowSize);
DEBUGLOG(4, "ZSTD_continueCCtx: re-use context in place");
assert(endT < (3U<<30));
cctx->blockSize = blockSize; /* previous block size could be different even for same windowLog, due to pledgedSrcSize */
cctx->appliedParams = params;
@ -813,14 +843,10 @@ static size_t ZSTD_continueCCtx(ZSTD_CCtx* cctx, ZSTD_CCtx_params params, U64 pl
cctx->appliedParams.fParams.contentSizeFlag = 0;
DEBUGLOG(4, "pledged content size : %u ; flag : %u",
(U32)pledgedSrcSize, cctx->appliedParams.fParams.contentSizeFlag);
cctx->lowLimit = end;
cctx->dictLimit = end;
cctx->nextToUpdate = end+1;
cctx->stage = ZSTDcs_init;
cctx->dictID = 0;
cctx->loadedDictEnd = 0;
{ int i; for (i=0; i<ZSTD_REP_NUM; i++) cctx->seqStore.rep[i] = repStartValue[i]; }
cctx->optState.litLengthSum = 0; /* force reset of btopt stats */
ZSTD_invalidateMatchState(&cctx->matchState);
ZSTD_resetBlockState(cctx->prevBlock);
XXH64_reset(&cctx->xxhState, 0);
return 0;
}
@ -846,10 +872,6 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
zc->appliedParams.cParams.windowLog, (U32)zc->blockSize);
assert(!(params.ldmParams.enableLdm &&
params.ldmParams.hashEveryLog == ZSTD_LDM_HASHEVERYLOG_NOTSET));
zc->entropy->hufCTable_repeatMode = HUF_repeat_none;
zc->entropy->offcode_repeatMode = FSE_repeat_none;
zc->entropy->matchlength_repeatMode = FSE_repeat_none;
zc->entropy->litlength_repeatMode = FSE_repeat_none;
return ZSTD_continueCCtx(zc, params, pledgedSrcSize);
} }
DEBUGLOG(4, "ZSTD_equivalentParams()==0 -> reset CCtx");
@ -880,7 +902,8 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
void* ptr;
/* Check if workSpace is large enough, alloc a new one if needed */
{ size_t const entropySpace = sizeof(ZSTD_entropyCTables_t);
{ size_t const entropySpace = HUF_WORKSPACE_SIZE;
size_t const blockStateSpace = 2 * sizeof(ZSTD_blockState_t);
size_t const optPotentialSpace = ((MaxML+1) + (MaxLL+1) + (MaxOff+1) + (1<<Litbits)) * sizeof(U32)
+ (ZSTD_OPT_NUM+1) * (sizeof(ZSTD_match_t)+sizeof(ZSTD_optimal_t));
size_t const optSpace = ( (params.cParams.strategy == ZSTD_btopt)
@ -890,7 +913,7 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
size_t const ldmSpace = params.ldmParams.enableLdm
? ZSTD_ldm_getTableSize(params.ldmParams.hashLog, params.ldmParams.bucketSizeLog)
: 0;
size_t const neededSpace = entropySpace + optSpace + ldmSpace +
size_t const neededSpace = entropySpace + blockStateSpace + optSpace + ldmSpace +
tableSpace + tokenSpace + bufferSpace;
DEBUGLOG(4, "Need %uKB workspace, including %uKB for tables, and %uKB for buffers",
(U32)(neededSpace>>10), (U32)(tableSpace>>10), (U32)(bufferSpace>>10));
@ -911,10 +934,13 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
zc->workSpaceSize = neededSpace;
ptr = zc->workSpace;
/* entropy space */
/* Statically sized space. entropyWorkspace never moves (but prev/next block swap places) */
assert(((size_t)zc->workSpace & 3) == 0); /* ensure correct alignment */
assert(zc->workSpaceSize >= sizeof(ZSTD_entropyCTables_t));
zc->entropy = (ZSTD_entropyCTables_t*)zc->workSpace;
assert(zc->workSpaceSize >= 2 * sizeof(ZSTD_blockState_t));
zc->prevBlock = (ZSTD_blockState_t*)zc->workSpace;
zc->nextBlock = zc->prevBlock + 1;
ptr = zc->nextBlock + 1;
zc->entropyWorkspace = (U32*)ptr;
} }
/* init params */
@ -930,36 +956,31 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
XXH64_reset(&zc->xxhState, 0);
zc->stage = ZSTDcs_init;
zc->dictID = 0;
zc->loadedDictEnd = 0;
zc->entropy->hufCTable_repeatMode = HUF_repeat_none;
zc->entropy->offcode_repeatMode = FSE_repeat_none;
zc->entropy->matchlength_repeatMode = FSE_repeat_none;
zc->entropy->litlength_repeatMode = FSE_repeat_none;
zc->base = NULL;
zc->dictBase = NULL;
zc->lowLimit = 0;
zc->dictLimit = 0;
zc->nextToUpdate = 1;
zc->nextSrc = NULL;
{ int i; for (i=0; i<ZSTD_REP_NUM; i++) zc->seqStore.rep[i] = repStartValue[i]; }
zc->hashLog3 = hashLog3;
zc->optState.litLengthSum = 0;
ptr = zc->entropy + 1;
zc->matchState.nextSrc = NULL;
zc->matchState.base = NULL;
zc->matchState.dictBase = NULL;
zc->matchState.hashLog3 = hashLog3;
ZSTD_invalidateMatchState(&zc->matchState);
ZSTD_resetBlockState(zc->prevBlock);
ptr = zc->entropyWorkspace + HUF_WORKSPACE_SIZE_U32;
/* opt parser space */
if ((params.cParams.strategy == ZSTD_btopt) | (params.cParams.strategy == ZSTD_btultra)) {
optState_t* opt = &zc->matchState.opt;
DEBUGLOG(4, "reserving optimal parser space");
assert(((size_t)ptr & 3) == 0); /* ensure ptr is properly aligned */
zc->optState.litFreq = (U32*)ptr;
zc->optState.litLengthFreq = zc->optState.litFreq + (1<<Litbits);
zc->optState.matchLengthFreq = zc->optState.litLengthFreq + (MaxLL+1);
zc->optState.offCodeFreq = zc->optState.matchLengthFreq + (MaxML+1);
ptr = zc->optState.offCodeFreq + (MaxOff+1);
zc->optState.matchTable = (ZSTD_match_t*)ptr;
ptr = zc->optState.matchTable + ZSTD_OPT_NUM+1;
zc->optState.priceTable = (ZSTD_optimal_t*)ptr;
ptr = zc->optState.priceTable + ZSTD_OPT_NUM+1;
opt->litFreq = (U32*)ptr;
opt->litLengthFreq = opt->litFreq + (1<<Litbits);
opt->matchLengthFreq = opt->litLengthFreq + (MaxLL+1);
opt->offCodeFreq = opt->matchLengthFreq + (MaxML+1);
ptr = opt->offCodeFreq + (MaxOff+1);
opt->matchTable = (ZSTD_match_t*)ptr;
ptr = opt->matchTable + ZSTD_OPT_NUM+1;
opt->priceTable = (ZSTD_optimal_t*)ptr;
ptr = opt->priceTable + ZSTD_OPT_NUM+1;
}
/* ldm hash table */
@ -976,10 +997,10 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
DEBUGLOG(4, "reset table : %u", crp!=ZSTDcrp_noMemset);
if (crp!=ZSTDcrp_noMemset) memset(ptr, 0, tableSpace); /* reset tables only */
assert(((size_t)ptr & 3) == 0); /* ensure ptr is properly aligned */
zc->hashTable = (U32*)(ptr);
zc->chainTable = zc->hashTable + hSize;
zc->hashTable3 = zc->chainTable + chainSize;
ptr = zc->hashTable3 + h3Size;
zc->matchState.hashTable = (U32*)(ptr);
zc->matchState.chainTable = zc->matchState.hashTable + hSize;
zc->matchState.hashTable3 = zc->matchState.chainTable + chainSize;
ptr = zc->matchState.hashTable3 + h3Size;
/* sequences storage */
zc->seqStore.sequencesStart = (seqDef*)ptr;
@ -1016,7 +1037,8 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
* do not use with extDict variant ! */
void ZSTD_invalidateRepCodes(ZSTD_CCtx* cctx) {
int i;
for (i=0; i<ZSTD_REP_NUM; i++) cctx->seqStore.rep[i] = 0;
for (i=0; i<ZSTD_REP_NUM; i++) cctx->prevBlock->rep[i] = 0;
assert(/* !extDict */ cctx->matchState.lowLimit == cctx->matchState.dictLimit);
}
@ -1050,32 +1072,30 @@ static size_t ZSTD_copyCCtx_internal(ZSTD_CCtx* dstCCtx,
/* copy tables */
{ size_t const chainSize = (srcCCtx->appliedParams.cParams.strategy == ZSTD_fast) ? 0 : ((size_t)1 << srcCCtx->appliedParams.cParams.chainLog);
size_t const hSize = (size_t)1 << srcCCtx->appliedParams.cParams.hashLog;
size_t const h3Size = (size_t)1 << srcCCtx->hashLog3;
size_t const h3Size = (size_t)1 << srcCCtx->matchState.hashLog3;
size_t const tableSpace = (chainSize + hSize + h3Size) * sizeof(U32);
assert((U32*)dstCCtx->chainTable == (U32*)dstCCtx->hashTable + hSize); /* chainTable must follow hashTable */
assert((U32*)dstCCtx->hashTable3 == (U32*)dstCCtx->chainTable + chainSize);
memcpy(dstCCtx->hashTable, srcCCtx->hashTable, tableSpace); /* presumes all tables follow each other */
assert((U32*)dstCCtx->matchState.chainTable == (U32*)dstCCtx->matchState.hashTable + hSize); /* chainTable must follow hashTable */
assert((U32*)dstCCtx->matchState.hashTable3 == (U32*)dstCCtx->matchState.chainTable + chainSize);
memcpy(dstCCtx->matchState.hashTable, srcCCtx->matchState.hashTable, tableSpace); /* presumes all tables follow each other */
}
/* copy dictionary offsets */
dstCCtx->nextToUpdate = srcCCtx->nextToUpdate;
dstCCtx->nextToUpdate3= srcCCtx->nextToUpdate3;
dstCCtx->nextSrc = srcCCtx->nextSrc;
dstCCtx->base = srcCCtx->base;
dstCCtx->dictBase = srcCCtx->dictBase;
dstCCtx->dictLimit = srcCCtx->dictLimit;
dstCCtx->lowLimit = srcCCtx->lowLimit;
dstCCtx->loadedDictEnd= srcCCtx->loadedDictEnd;
{
ZSTD_matchState_t const* srcMatchState = &srcCCtx->matchState;
ZSTD_matchState_t* dstMatchState = &dstCCtx->matchState;
dstMatchState->nextToUpdate = srcMatchState->nextToUpdate;
dstMatchState->nextToUpdate3= srcMatchState->nextToUpdate3;
dstMatchState->nextSrc = srcMatchState->nextSrc;
dstMatchState->base = srcMatchState->base;
dstMatchState->dictBase = srcMatchState->dictBase;
dstMatchState->dictLimit = srcMatchState->dictLimit;
dstMatchState->lowLimit = srcMatchState->lowLimit;
dstMatchState->loadedDictEnd= srcMatchState->loadedDictEnd;
}
dstCCtx->dictID = srcCCtx->dictID;
/* copy entropy tables */
memcpy(dstCCtx->entropy, srcCCtx->entropy, sizeof(ZSTD_entropyCTables_t));
/* copy repcodes */
{
int i;
for (i = 0; i < ZSTD_REP_NUM; ++i)
dstCCtx->seqStore.rep[i] = srcCCtx->seqStore.rep[i];
}
/* copy block state */
memcpy(dstCCtx->prevBlock, srcCCtx->prevBlock, sizeof(*srcCCtx->prevBlock));
return 0;
}
@ -1142,20 +1162,21 @@ static void ZSTD_ldm_reduceTable(ldmEntry_t* const table, U32 const size,
* rescale all indexes to avoid future overflow (indexes are U32) */
static void ZSTD_reduceIndex (ZSTD_CCtx* zc, const U32 reducerValue)
{
ZSTD_matchState_t* const ms = &zc->matchState;
{ U32 const hSize = (U32)1 << zc->appliedParams.cParams.hashLog;
ZSTD_reduceTable(zc->hashTable, hSize, reducerValue);
ZSTD_reduceTable(ms->hashTable, hSize, reducerValue);
}
if (zc->appliedParams.cParams.strategy != ZSTD_fast) {
U32 const chainSize = (U32)1 << zc->appliedParams.cParams.chainLog;
if (zc->appliedParams.cParams.strategy == ZSTD_btlazy2)
ZSTD_preserveUnsortedMark(zc->chainTable, chainSize, reducerValue);
ZSTD_reduceTable(zc->chainTable, chainSize, reducerValue);
ZSTD_preserveUnsortedMark(ms->chainTable, chainSize, reducerValue);
ZSTD_reduceTable(ms->chainTable, chainSize, reducerValue);
}
if (zc->hashLog3) {
U32 const h3Size = (U32)1 << zc->hashLog3;
ZSTD_reduceTable(zc->hashTable3, h3Size, reducerValue);
if (ms->hashLog3) {
U32 const h3Size = (U32)1 << ms->hashLog3;
ZSTD_reduceTable(ms->hashTable3, h3Size, reducerValue);
}
if (zc->appliedParams.ldmParams.enableLdm) {
@ -1235,10 +1256,12 @@ static size_t ZSTD_compressRleLiteralsBlock (void* dst, size_t dstCapacity, cons
static size_t ZSTD_minGain(size_t srcSize) { return (srcSize >> 6) + 2; }
static size_t ZSTD_compressLiterals (ZSTD_entropyCTables_t * entropy,
static size_t ZSTD_compressLiterals (ZSTD_entropyCTables_t const* prevEntropy,
ZSTD_entropyCTables_t* nextEntropy,
ZSTD_strategy strategy,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize)
const void* src, size_t srcSize,
U32* workspace)
{
size_t const minGain = ZSTD_minGain(srcSize);
size_t const lhSize = 3 + (srcSize >= 1 KB) + (srcSize >= 16 KB);
@ -1247,34 +1270,45 @@ static size_t ZSTD_compressLiterals (ZSTD_entropyCTables_t * entropy,
symbolEncodingType_e hType = set_compressed;
size_t cLitSize;
/* Prepare nextEntropy assuming reusing the existing table */
nextEntropy->hufCTable_repeatMode = prevEntropy->hufCTable_repeatMode;
memcpy(nextEntropy->hufCTable, prevEntropy->hufCTable,
sizeof(prevEntropy->hufCTable));
/* small ? don't even attempt compression (speed opt) */
# define LITERAL_NOENTROPY 63
{ size_t const minLitSize = entropy->hufCTable_repeatMode == HUF_repeat_valid ? 6 : LITERAL_NOENTROPY;
{ size_t const minLitSize = prevEntropy->hufCTable_repeatMode == HUF_repeat_valid ? 6 : LITERAL_NOENTROPY;
if (srcSize <= minLitSize) return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize);
}
if (dstCapacity < lhSize+1) return ERROR(dstSize_tooSmall); /* not enough space for compression */
{ HUF_repeat repeat = entropy->hufCTable_repeatMode;
{ HUF_repeat repeat = prevEntropy->hufCTable_repeatMode;
int const preferRepeat = strategy < ZSTD_lazy ? srcSize <= 1024 : 0;
if (repeat == HUF_repeat_valid && lhSize == 3) singleStream = 1;
cLitSize = singleStream ? HUF_compress1X_repeat(ostart+lhSize, dstCapacity-lhSize, src, srcSize, 255, 11,
entropy->workspace, sizeof(entropy->workspace), (HUF_CElt*)entropy->hufCTable, &repeat, preferRepeat)
workspace, HUF_WORKSPACE_SIZE, (HUF_CElt*)nextEntropy->hufCTable, &repeat, preferRepeat)
: HUF_compress4X_repeat(ostart+lhSize, dstCapacity-lhSize, src, srcSize, 255, 11,
entropy->workspace, sizeof(entropy->workspace), (HUF_CElt*)entropy->hufCTable, &repeat, preferRepeat);
if (repeat != HUF_repeat_none) { hType = set_repeat; } /* reused the existing table */
else { entropy->hufCTable_repeatMode = HUF_repeat_check; } /* now have a table to reuse */
workspace, HUF_WORKSPACE_SIZE, (HUF_CElt*)nextEntropy->hufCTable, &repeat, preferRepeat);
if (repeat != HUF_repeat_none) {
/* reused the existing table */
hType = set_repeat;
}
}
if ((cLitSize==0) | (cLitSize >= srcSize - minGain) | ERR_isError(cLitSize)) {
entropy->hufCTable_repeatMode = HUF_repeat_none;
memcpy(nextEntropy->hufCTable, prevEntropy->hufCTable, sizeof(prevEntropy->hufCTable));
return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize);
}
if (cLitSize==1) {
entropy->hufCTable_repeatMode = HUF_repeat_none;
memcpy(nextEntropy->hufCTable, prevEntropy->hufCTable, sizeof(prevEntropy->hufCTable));
return ZSTD_compressRleLiteralsBlock(dst, dstCapacity, src, srcSize);
}
if (hType == set_compressed) {
/* using a newly constructed table */
nextEntropy->hufCTable_repeatMode = HUF_repeat_check;
}
/* Build header */
switch(lhSize)
{
@ -1368,10 +1402,11 @@ symbolEncodingType_e ZSTD_selectEncodingType(
MEM_STATIC
size_t ZSTD_buildCTable(void* dst, size_t dstCapacity,
FSE_CTable* CTable, U32 FSELog, symbolEncodingType_e type,
FSE_CTable* nextCTable, U32 FSELog, symbolEncodingType_e type,
U32* count, U32 max,
BYTE const* codeTable, size_t nbSeq,
S16 const* defaultNorm, U32 defaultNormLog, U32 defaultMax,
FSE_CTable const* prevCTable, size_t prevCTableSize,
void* workspace, size_t workspaceSize)
{
BYTE* op = (BYTE*)dst;
@ -1380,12 +1415,13 @@ size_t ZSTD_buildCTable(void* dst, size_t dstCapacity,
switch (type) {
case set_rle:
*op = codeTable[0];
CHECK_F(FSE_buildCTable_rle(CTable, (BYTE)max));
CHECK_F(FSE_buildCTable_rle(nextCTable, (BYTE)max));
return 1;
case set_repeat:
memcpy(nextCTable, prevCTable, prevCTableSize);
return 0;
case set_basic:
CHECK_F(FSE_buildCTable_wksp(CTable, defaultNorm, defaultMax, defaultNormLog, workspace, workspaceSize)); /* note : could be pre-calculated */
CHECK_F(FSE_buildCTable_wksp(nextCTable, defaultNorm, defaultMax, defaultNormLog, workspace, workspaceSize)); /* note : could be pre-calculated */
return 0;
case set_compressed: {
S16 norm[MaxSeq + 1];
@ -1399,7 +1435,7 @@ size_t ZSTD_buildCTable(void* dst, size_t dstCapacity,
CHECK_F(FSE_normalizeCount(norm, tableLog, count, nbSeq_1, max));
{ size_t const NCountSize = FSE_writeNCount(op, oend - op, norm, max, tableLog); /* overflow protected */
if (FSE_isError(NCountSize)) return NCountSize;
CHECK_F(FSE_buildCTable_wksp(CTable, norm, max, tableLog, workspace, workspaceSize));
CHECK_F(FSE_buildCTable_wksp(nextCTable, norm, max, tableLog, workspace, workspaceSize));
return NCountSize;
}
}
@ -1492,15 +1528,16 @@ size_t ZSTD_encodeSequences(
}
MEM_STATIC size_t ZSTD_compressSequences_internal(seqStore_t* seqStorePtr,
ZSTD_entropyCTables_t* entropy,
ZSTD_entropyCTables_t const* prevEntropy,
ZSTD_entropyCTables_t* nextEntropy,
ZSTD_compressionParameters const* cParams,
void* dst, size_t dstCapacity)
void* dst, size_t dstCapacity, U32* workspace)
{
const int longOffsets = cParams->windowLog > STREAM_ACCUMULATOR_MIN;
U32 count[MaxSeq+1];
FSE_CTable* CTable_LitLength = entropy->litlengthCTable;
FSE_CTable* CTable_OffsetBits = entropy->offcodeCTable;
FSE_CTable* CTable_MatchLength = entropy->matchlengthCTable;
FSE_CTable* CTable_LitLength = nextEntropy->litlengthCTable;
FSE_CTable* CTable_OffsetBits = nextEntropy->offcodeCTable;
FSE_CTable* CTable_MatchLength = nextEntropy->matchlengthCTable;
U32 LLtype, Offtype, MLtype; /* compressed, raw or rle */
const seqDef* const sequences = seqStorePtr->sequencesStart;
const BYTE* const ofCodeTable = seqStorePtr->ofCode;
@ -1512,13 +1549,14 @@ MEM_STATIC size_t ZSTD_compressSequences_internal(seqStore_t* seqStorePtr,
size_t const nbSeq = seqStorePtr->sequences - seqStorePtr->sequencesStart;
BYTE* seqHead;
ZSTD_STATIC_ASSERT(sizeof(entropy->workspace) >= (1<<MAX(MLFSELog,LLFSELog)));
ZSTD_STATIC_ASSERT(HUF_WORKSPACE_SIZE >= (1<<MAX(MLFSELog,LLFSELog)));
/* Compress literals */
{ const BYTE* const literals = seqStorePtr->litStart;
size_t const litSize = seqStorePtr->lit - literals;
size_t const cSize = ZSTD_compressLiterals(
entropy, cParams->strategy, op, dstCapacity, literals, litSize);
size_t const cSize = ZSTD_compressLiterals(prevEntropy, nextEntropy,
cParams->strategy, op, dstCapacity, literals, litSize,
workspace);
if (ZSTD_isError(cSize))
return cSize;
assert(cSize <= dstCapacity);
@ -1533,7 +1571,15 @@ MEM_STATIC size_t ZSTD_compressSequences_internal(seqStore_t* seqStorePtr,
op[0] = (BYTE)((nbSeq>>8) + 0x80), op[1] = (BYTE)nbSeq, op+=2;
else
op[0]=0xFF, MEM_writeLE16(op+1, (U16)(nbSeq - LONGNBSEQ)), op+=3;
if (nbSeq==0) return op - ostart;
if (nbSeq==0) {
memcpy(nextEntropy->litlengthCTable, prevEntropy->litlengthCTable, sizeof(prevEntropy->litlengthCTable));
nextEntropy->litlength_repeatMode = prevEntropy->litlength_repeatMode;
memcpy(nextEntropy->offcodeCTable, prevEntropy->offcodeCTable, sizeof(prevEntropy->offcodeCTable));
nextEntropy->offcode_repeatMode = prevEntropy->offcode_repeatMode;
memcpy(nextEntropy->matchlengthCTable, prevEntropy->matchlengthCTable, sizeof(prevEntropy->matchlengthCTable));
nextEntropy->matchlength_repeatMode = prevEntropy->matchlength_repeatMode;
return op - ostart;
}
/* seqHead : flags for FSE encoding type */
seqHead = op++;
@ -1542,36 +1588,42 @@ MEM_STATIC size_t ZSTD_compressSequences_internal(seqStore_t* seqStorePtr,
ZSTD_seqToCodes(seqStorePtr);
/* build CTable for Literal Lengths */
{ U32 max = MaxLL;
size_t const mostFrequent = FSE_countFast_wksp(count, &max, llCodeTable, nbSeq, entropy->workspace);
size_t const mostFrequent = FSE_countFast_wksp(count, &max, llCodeTable, nbSeq, workspace);
DEBUGLOG(5, "Building LL table");
LLtype = ZSTD_selectEncodingType(&entropy->litlength_repeatMode, mostFrequent, nbSeq, LL_defaultNormLog, ZSTD_defaultAllowed);
nextEntropy->litlength_repeatMode = prevEntropy->litlength_repeatMode;
LLtype = ZSTD_selectEncodingType(&nextEntropy->litlength_repeatMode, mostFrequent, nbSeq, LL_defaultNormLog, ZSTD_defaultAllowed);
{ size_t const countSize = ZSTD_buildCTable(op, oend - op, CTable_LitLength, LLFSELog, (symbolEncodingType_e)LLtype,
count, max, llCodeTable, nbSeq, LL_defaultNorm, LL_defaultNormLog, MaxLL,
entropy->workspace, sizeof(entropy->workspace));
prevEntropy->litlengthCTable, sizeof(prevEntropy->litlengthCTable),
workspace, HUF_WORKSPACE_SIZE);
if (ZSTD_isError(countSize)) return countSize;
op += countSize;
} }
/* build CTable for Offsets */
{ U32 max = MaxOff;
size_t const mostFrequent = FSE_countFast_wksp(count, &max, ofCodeTable, nbSeq, entropy->workspace);
size_t const mostFrequent = FSE_countFast_wksp(count, &max, ofCodeTable, nbSeq, workspace);
/* We can only use the basic table if max <= DefaultMaxOff, otherwise the offsets are too large */
ZSTD_defaultPolicy_e const defaultPolicy = (max <= DefaultMaxOff) ? ZSTD_defaultAllowed : ZSTD_defaultDisallowed;
DEBUGLOG(5, "Building OF table");
Offtype = ZSTD_selectEncodingType(&entropy->offcode_repeatMode, mostFrequent, nbSeq, OF_defaultNormLog, defaultPolicy);
nextEntropy->offcode_repeatMode = prevEntropy->offcode_repeatMode;
Offtype = ZSTD_selectEncodingType(&nextEntropy->offcode_repeatMode, mostFrequent, nbSeq, OF_defaultNormLog, defaultPolicy);
{ size_t const countSize = ZSTD_buildCTable(op, oend - op, CTable_OffsetBits, OffFSELog, (symbolEncodingType_e)Offtype,
count, max, ofCodeTable, nbSeq, OF_defaultNorm, OF_defaultNormLog, DefaultMaxOff,
entropy->workspace, sizeof(entropy->workspace));
prevEntropy->offcodeCTable, sizeof(prevEntropy->offcodeCTable),
workspace, HUF_WORKSPACE_SIZE);
if (ZSTD_isError(countSize)) return countSize;
op += countSize;
} }
/* build CTable for MatchLengths */
{ U32 max = MaxML;
size_t const mostFrequent = FSE_countFast_wksp(count, &max, mlCodeTable, nbSeq, entropy->workspace);
size_t const mostFrequent = FSE_countFast_wksp(count, &max, mlCodeTable, nbSeq, workspace);
DEBUGLOG(5, "Building ML table");
MLtype = ZSTD_selectEncodingType(&entropy->matchlength_repeatMode, mostFrequent, nbSeq, ML_defaultNormLog, ZSTD_defaultAllowed);
nextEntropy->matchlength_repeatMode = prevEntropy->matchlength_repeatMode;
MLtype = ZSTD_selectEncodingType(&nextEntropy->matchlength_repeatMode, mostFrequent, nbSeq, ML_defaultNormLog, ZSTD_defaultAllowed);
{ size_t const countSize = ZSTD_buildCTable(op, oend - op, CTable_MatchLength, MLFSELog, (symbolEncodingType_e)MLtype,
count, max, mlCodeTable, nbSeq, ML_defaultNorm, ML_defaultNormLog, MaxML,
entropy->workspace, sizeof(entropy->workspace));
prevEntropy->matchlengthCTable, sizeof(prevEntropy->matchlengthCTable),
workspace, HUF_WORKSPACE_SIZE);
if (ZSTD_isError(countSize)) return countSize;
op += countSize;
} }
@ -1593,13 +1645,15 @@ MEM_STATIC size_t ZSTD_compressSequences_internal(seqStore_t* seqStorePtr,
}
MEM_STATIC size_t ZSTD_compressSequences(seqStore_t* seqStorePtr,
ZSTD_entropyCTables_t* entropy,
ZSTD_entropyCTables_t const* prevEntropy,
ZSTD_entropyCTables_t* nextEntropy,
ZSTD_compressionParameters const* cParams,
void* dst, size_t dstCapacity,
size_t srcSize)
size_t srcSize, U32* workspace)
{
size_t const cSize = ZSTD_compressSequences_internal(seqStorePtr, entropy, cParams,
dst, dstCapacity);
size_t const cSize = ZSTD_compressSequences_internal(
seqStorePtr, prevEntropy, nextEntropy, cParams, dst, dstCapacity,
workspace);
/* If the srcSize <= dstCapacity, then there is enough space to write a
* raw uncompressed block. Since we ran out of space, the block must not
* be compressible, so fall back to a raw uncompressed block.
@ -1607,34 +1661,28 @@ MEM_STATIC size_t ZSTD_compressSequences(seqStore_t* seqStorePtr,
int const uncompressibleError = (cSize == ERROR(dstSize_tooSmall)) && (srcSize <= dstCapacity);
if (ZSTD_isError(cSize) && !uncompressibleError)
return cSize;
/* We check that dictionaries have offset codes available for the first
* block. After the first block, the offcode table might not have large
* enough codes to represent the offsets in the data.
*/
if (entropy->offcode_repeatMode == FSE_repeat_valid)
entropy->offcode_repeatMode = FSE_repeat_check;
/* Check compressibility */
{ size_t const minGain = ZSTD_minGain(srcSize); /* note : fixed formula, maybe should depend on compression level, or strategy */
size_t const maxCSize = srcSize - minGain;
if (cSize >= maxCSize || uncompressibleError) {
entropy->hufCTable_repeatMode = HUF_repeat_none;
entropy->offcode_repeatMode = FSE_repeat_none;
entropy->matchlength_repeatMode = FSE_repeat_none;
entropy->litlength_repeatMode = FSE_repeat_none;
return 0; /* block not compressed */
} }
assert(!ZSTD_isError(cSize));
/* block is compressed => confirm repcodes in history */
{ int i; for (i=0; i<ZSTD_REP_NUM; i++) seqStorePtr->rep[i] = seqStorePtr->repToConfirm[i]; }
/* We check that dictionaries have offset codes available for the first
* block. After the first block, the offcode table might not have large
* enough codes to represent the offsets in the data.
*/
if (nextEntropy->offcode_repeatMode == FSE_repeat_valid)
nextEntropy->offcode_repeatMode = FSE_repeat_check;
return cSize;
}
/* ZSTD_selectBlockCompressor() :
* Not static, but internal use only (used by long distance matcher)
* assumption : strat is a valid strategy */
typedef size_t (*ZSTD_blockCompressor) (ZSTD_CCtx* ctx, const void* src, size_t srcSize);
ZSTD_blockCompressor ZSTD_selectBlockCompressor(ZSTD_strategy strat, int extDict)
{
static const ZSTD_blockCompressor blockCompressor[2][(unsigned)ZSTD_btultra+1] = {
@ -1673,30 +1721,54 @@ static size_t ZSTD_compressBlock_internal(ZSTD_CCtx* zc,
const void* src, size_t srcSize)
{
DEBUGLOG(5, "ZSTD_compressBlock_internal (dstCapacity=%u) (dictLimit=%u, nextToUpdate=%u)",
(U32)dstCapacity, zc->dictLimit, zc->nextToUpdate);
(U32)dstCapacity, zc->matchState.dictLimit, zc->matchState.nextToUpdate);
if (srcSize < MIN_CBLOCK_SIZE+ZSTD_blockHeaderSize+1)
return 0; /* don't even attempt compression below a certain srcSize */
ZSTD_resetSeqStore(&(zc->seqStore));
/* limited update after a very long match */
{ const BYTE* const base = zc->base;
{ const BYTE* const base = zc->matchState.base;
const BYTE* const istart = (const BYTE*)src;
const U32 current = (U32)(istart-base);
if (current > zc->nextToUpdate + 384)
zc->nextToUpdate = current - MIN(192, (U32)(current - zc->nextToUpdate - 384));
if (current > zc->matchState.nextToUpdate + 384)
zc->matchState.nextToUpdate = current - MIN(192, (U32)(current - zc->matchState.nextToUpdate - 384));
}
/* find and store sequences */
{ U32 const extDict = zc->lowLimit < zc->dictLimit;
const ZSTD_blockCompressor blockCompressor =
zc->appliedParams.ldmParams.enableLdm
? (extDict ? ZSTD_compressBlock_ldm_extDict : ZSTD_compressBlock_ldm)
: ZSTD_selectBlockCompressor(zc->appliedParams.cParams.strategy, extDict);
size_t const lastLLSize = blockCompressor(zc, src, srcSize);
{
U32 const extDict = zc->matchState.lowLimit < zc->matchState.dictLimit;
size_t lastLLSize;
{ int i; for (i = 0; i < ZSTD_REP_NUM; ++i) zc->nextBlock->rep[i] = zc->prevBlock->rep[i]; }
if (zc->appliedParams.ldmParams.enableLdm) {
typedef size_t (*ZSTD_ldmBlockCompressor)(
ldmState_t* ldms, ZSTD_matchState_t* ms, seqStore_t* seqStore,
U32 rep[ZSTD_REP_NUM], ZSTD_CCtx_params const* params,
void const* src, size_t srcSize);
ZSTD_ldmBlockCompressor const ldmBlockCompressor = extDict ? ZSTD_compressBlock_ldm_extDict : ZSTD_compressBlock_ldm;
lastLLSize = ldmBlockCompressor(&zc->ldmState, &zc->matchState, &zc->seqStore, zc->nextBlock->rep, &zc->appliedParams, src, srcSize);
} else {
ZSTD_blockCompressor const blockCompressor = ZSTD_selectBlockCompressor(zc->appliedParams.cParams.strategy, extDict);
lastLLSize = blockCompressor(&zc->matchState, &zc->seqStore, zc->nextBlock->rep, &zc->appliedParams.cParams, src, srcSize);
}
{
const BYTE* const anchor = (const BYTE*)src + srcSize - lastLLSize;
ZSTD_storeLastLiterals(&zc->seqStore, anchor, lastLLSize);
}
}
/* encode */
return ZSTD_compressSequences(&zc->seqStore, zc->entropy, &zc->appliedParams.cParams, dst, dstCapacity, srcSize);
{
size_t const cSize = ZSTD_compressSequences(&zc->seqStore, &zc->prevBlock->entropy, &zc->nextBlock->entropy, &zc->appliedParams.cParams, dst, dstCapacity, srcSize, zc->entropyWorkspace);
if (ZSTD_isError(cSize) || cSize == 0)
return cSize;
/* confirm repcodes and entropy tables */
{
ZSTD_blockState_t* const tmp = zc->prevBlock;
zc->prevBlock = zc->nextBlock;
zc->nextBlock = tmp;
}
return cSize;
}
}
@ -1725,6 +1797,7 @@ static size_t ZSTD_compress_frameChunk (ZSTD_CCtx* cctx,
XXH64_update(&cctx->xxhState, src, srcSize);
while (remaining) {
ZSTD_matchState_t* const ms = &cctx->matchState;
U32 const lastBlock = lastFrameChunk & (blockSize >= remaining);
if (dstCapacity < ZSTD_blockHeaderSize + MIN_CBLOCK_SIZE)
@ -1748,9 +1821,9 @@ static size_t ZSTD_compress_frameChunk (ZSTD_CCtx* cctx,
* 3. cctx->lowLimit < 1<<32:
* windowLog <= 31 ==> 3<<29 + 1<<windowLog < 7<<29 < 1<<32.
*/
if (cctx->lowLimit > (3U<<29)) {
if (ms->lowLimit > (3U<<29)) {
U32 const cycleMask = ((U32)1 << ZSTD_cycleLog(cctx->appliedParams.cParams.chainLog, cctx->appliedParams.cParams.strategy)) - 1;
U32 const current = (U32)(ip - cctx->base);
U32 const current = (U32)(ip - cctx->matchState.base);
U32 const newCurrent = (current & cycleMask) + ((U32)1 << cctx->appliedParams.cParams.windowLog);
U32 const correction = current - newCurrent;
ZSTD_STATIC_ASSERT(ZSTD_CHAINLOG_MAX <= 30);
@ -1759,23 +1832,23 @@ static size_t ZSTD_compress_frameChunk (ZSTD_CCtx* cctx,
assert(current > newCurrent);
assert(correction > 1<<28); /* Loose bound, should be about 1<<29 */
ZSTD_reduceIndex(cctx, correction);
cctx->base += correction;
cctx->dictBase += correction;
cctx->lowLimit -= correction;
cctx->dictLimit -= correction;
if (cctx->nextToUpdate < correction) cctx->nextToUpdate = 0;
else cctx->nextToUpdate -= correction;
DEBUGLOG(4, "Correction of 0x%x bytes to lowLimit=0x%x", correction, cctx->lowLimit);
ms->base += correction;
ms->dictBase += correction;
ms->lowLimit -= correction;
ms->dictLimit -= correction;
if (ms->nextToUpdate < correction) ms->nextToUpdate = 0;
else ms->nextToUpdate -= correction;
DEBUGLOG(4, "Correction of 0x%x bytes to lowLimit=0x%x", correction, ms->lowLimit);
}
/* enforce maxDist */
if ((U32)(ip+blockSize - cctx->base) > cctx->loadedDictEnd + maxDist) {
U32 const newLowLimit = (U32)(ip+blockSize - cctx->base) - maxDist;
if (cctx->lowLimit < newLowLimit) cctx->lowLimit = newLowLimit;
if (cctx->dictLimit < cctx->lowLimit)
if ((U32)(ip+blockSize - ms->base) > ms->loadedDictEnd + maxDist) {
U32 const newLowLimit = (U32)(ip+blockSize - ms->base) - maxDist;
if (ms->lowLimit < newLowLimit) ms->lowLimit = newLowLimit;
if (ms->dictLimit < ms->lowLimit)
DEBUGLOG(5, "ZSTD_compress_frameChunk : update dictLimit from %u to %u ",
cctx->dictLimit, cctx->lowLimit);
if (cctx->dictLimit < cctx->lowLimit) cctx->dictLimit = cctx->lowLimit;
if (cctx->nextToUpdate < cctx->lowLimit) cctx->nextToUpdate = cctx->lowLimit;
ms->dictLimit, ms->lowLimit);
if (ms->dictLimit < ms->lowLimit) ms->dictLimit = ms->lowLimit;
if (ms->nextToUpdate < ms->lowLimit) ms->nextToUpdate = ms->lowLimit;
}
{ size_t cSize = ZSTD_compressBlock_internal(cctx,
@ -1854,12 +1927,38 @@ static size_t ZSTD_writeFrameHeader(void* dst, size_t dstCapacity,
}
static void ZSTD_manageWindowContinuity(ZSTD_matchState_t* ms, void const* src, size_t srcSize)
{
const BYTE* const ip = (const BYTE*) src;
/* Check if blocks follow each other */
if (src != ms->nextSrc) {
/* not contiguous */
size_t const distanceFromBase = (size_t)(ms->nextSrc - ms->base);
DEBUGLOG(5, "ZSTD_manageWindowContinuity: non contiguous blocks, new segment starts at %u", ms->dictLimit);
ms->lowLimit = ms->dictLimit;
assert(distanceFromBase == (size_t)(U32)distanceFromBase); /* should never overflow */
ms->dictLimit = (U32)distanceFromBase;
ms->dictBase = ms->base;
ms->base = ip - distanceFromBase;
ms->nextToUpdate = ms->dictLimit;
if (ms->dictLimit - ms->lowLimit < HASH_READ_SIZE) ms->lowLimit = ms->dictLimit; /* too small extDict */
}
ms->nextSrc = ip + srcSize;
/* if input and dictionary overlap : reduce dictionary (area presumed modified by input) */
if ((ip+srcSize > ms->dictBase + ms->lowLimit) & (ip < ms->dictBase + ms->dictLimit)) {
ptrdiff_t const highInputIdx = (ip + srcSize) - ms->dictBase;
U32 const lowLimitMax = (highInputIdx > (ptrdiff_t)ms->dictLimit) ? ms->dictLimit : (U32)highInputIdx;
ms->lowLimit = lowLimitMax;
}
}
static size_t ZSTD_compressContinue_internal (ZSTD_CCtx* cctx,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize,
U32 frame, U32 lastFrameChunk)
{
const BYTE* const ip = (const BYTE*) src;
size_t fhSize = 0;
DEBUGLOG(5, "ZSTD_compressContinue_internal, stage: %u", cctx->stage);
@ -1876,27 +1975,7 @@ static size_t ZSTD_compressContinue_internal (ZSTD_CCtx* cctx,
if (!srcSize) return fhSize; /* do not generate an empty block if no input */
/* Check if blocks follow each other */
if (src != cctx->nextSrc) {
size_t const distanceFromBase = (size_t)(cctx->nextSrc - cctx->base);
DEBUGLOG(5, "ZSTD_compressContinue_internal: non contiguous blocks, new segment starts at %u",
cctx->dictLimit);
cctx->lowLimit = cctx->dictLimit;
assert(distanceFromBase == (size_t)(U32)distanceFromBase); /* should never overflow */
cctx->dictLimit = (U32)distanceFromBase;
cctx->dictBase = cctx->base;
cctx->base = ip - distanceFromBase;
cctx->nextToUpdate = cctx->dictLimit;
if (cctx->dictLimit - cctx->lowLimit < HASH_READ_SIZE) cctx->lowLimit = cctx->dictLimit; /* too small extDict */
}
cctx->nextSrc = ip + srcSize;
/* if input and dictionary overlap : reduce dictionary (area presumed modified by input) */
if ((ip+srcSize > cctx->dictBase + cctx->lowLimit) & (ip < cctx->dictBase + cctx->dictLimit)) {
ptrdiff_t const highInputIdx = (ip + srcSize) - cctx->dictBase;
U32 const lowLimitMax = (highInputIdx > (ptrdiff_t)cctx->dictLimit) ? cctx->dictLimit : (U32)highInputIdx;
cctx->lowLimit = lowLimitMax;
}
ZSTD_manageWindowContinuity(&cctx->matchState, src, srcSize);
DEBUGLOG(5, "ZSTD_compressContinue_internal (blockSize=%u)", (U32)cctx->blockSize);
{ size_t const cSize = frame ?
@ -1934,50 +2013,51 @@ size_t ZSTD_compressBlock(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const
/*! ZSTD_loadDictionaryContent() :
* @return : 0, or an error code
*/
static size_t ZSTD_loadDictionaryContent(ZSTD_CCtx* zc, const void* src, size_t srcSize)
static size_t ZSTD_loadDictionaryContent(ZSTD_matchState_t* ms, ZSTD_CCtx_params const* params, const void* src, size_t srcSize)
{
const BYTE* const ip = (const BYTE*) src;
const BYTE* const iend = ip + srcSize;
ZSTD_compressionParameters const* cParams = &params->cParams;
/* input becomes current prefix */
zc->lowLimit = zc->dictLimit;
zc->dictLimit = (U32)(zc->nextSrc - zc->base);
zc->dictBase = zc->base;
zc->base = ip - zc->dictLimit;
zc->nextToUpdate = zc->dictLimit;
zc->loadedDictEnd = zc->appliedParams.forceWindow ? 0 : (U32)(iend - zc->base);
ms->lowLimit = ms->dictLimit;
ms->dictLimit = (U32)(ms->nextSrc - ms->base);
ms->dictBase = ms->base;
ms->base = ip - ms->dictLimit;
ms->nextToUpdate = ms->dictLimit;
ms->loadedDictEnd = params->forceWindow ? 0 : (U32)(iend - ms->base);
zc->nextSrc = iend;
ms->nextSrc = iend;
if (srcSize <= HASH_READ_SIZE) return 0;
switch(zc->appliedParams.cParams.strategy)
switch(params->cParams.strategy)
{
case ZSTD_fast:
ZSTD_fillHashTable (zc, iend, zc->appliedParams.cParams.searchLength);
ZSTD_fillHashTable(ms, cParams, iend);
break;
case ZSTD_dfast:
ZSTD_fillDoubleHashTable (zc, iend, zc->appliedParams.cParams.searchLength);
ZSTD_fillDoubleHashTable(ms, cParams, iend);
break;
case ZSTD_greedy:
case ZSTD_lazy:
case ZSTD_lazy2:
if (srcSize >= HASH_READ_SIZE)
ZSTD_insertAndFindFirstIndex(zc, iend-HASH_READ_SIZE, zc->appliedParams.cParams.searchLength);
ZSTD_insertAndFindFirstIndex(ms, cParams, iend-HASH_READ_SIZE);
break;
case ZSTD_btlazy2: /* we want the dictionary table fully sorted */
case ZSTD_btopt:
case ZSTD_btultra:
if (srcSize >= HASH_READ_SIZE)
ZSTD_updateTree(zc, iend-HASH_READ_SIZE, iend, (U32)1 << zc->appliedParams.cParams.searchLog, zc->appliedParams.cParams.searchLength);
ZSTD_updateTree(ms, cParams, iend-HASH_READ_SIZE, iend);
break;
default:
assert(0); /* not possible : not a valid strategy id */
}
zc->nextToUpdate = (U32)(iend - zc->base);
ms->nextToUpdate = (U32)(iend - ms->base);
return 0;
}
@ -2001,25 +2081,26 @@ static size_t ZSTD_checkDictNCount(short* normalizedCounter, unsigned dictMaxSym
* https://github.com/facebook/zstd/blob/master/doc/zstd_compression_format.md#dictionary-format
*/
/*! ZSTD_loadZstdDictionary() :
* @return : 0, or an error code
* @return : dictID, or an error code
* assumptions : magic number supposed already checked
* dictSize supposed > 8
*/
static size_t ZSTD_loadZstdDictionary(ZSTD_CCtx* cctx, const void* dict, size_t dictSize)
static size_t ZSTD_loadZstdDictionary(ZSTD_blockState_t* bs, ZSTD_matchState_t* ms, ZSTD_CCtx_params const* params, const void* dict, size_t dictSize, void* workspace)
{
const BYTE* dictPtr = (const BYTE*)dict;
const BYTE* const dictEnd = dictPtr + dictSize;
short offcodeNCount[MaxOff+1];
unsigned offcodeMaxValue = MaxOff;
size_t dictID;
ZSTD_STATIC_ASSERT(sizeof(cctx->entropy->workspace) >= (1<<MAX(MLFSELog,LLFSELog)));
ZSTD_STATIC_ASSERT(HUF_WORKSPACE_SIZE >= (1<<MAX(MLFSELog,LLFSELog)));
dictPtr += 4; /* skip magic number */
cctx->dictID = cctx->appliedParams.fParams.noDictIDFlag ? 0 : MEM_readLE32(dictPtr);
dictID = params->fParams.noDictIDFlag ? 0 : MEM_readLE32(dictPtr);
dictPtr += 4;
{ unsigned maxSymbolValue = 255;
size_t const hufHeaderSize = HUF_readCTable((HUF_CElt*)cctx->entropy->hufCTable, &maxSymbolValue, dictPtr, dictEnd-dictPtr);
size_t const hufHeaderSize = HUF_readCTable((HUF_CElt*)bs->entropy.hufCTable, &maxSymbolValue, dictPtr, dictEnd-dictPtr);
if (HUF_isError(hufHeaderSize)) return ERROR(dictionary_corrupted);
if (maxSymbolValue < 255) return ERROR(dictionary_corrupted);
dictPtr += hufHeaderSize;
@ -2030,7 +2111,7 @@ static size_t ZSTD_loadZstdDictionary(ZSTD_CCtx* cctx, const void* dict, size_t
if (FSE_isError(offcodeHeaderSize)) return ERROR(dictionary_corrupted);
if (offcodeLog > OffFSELog) return ERROR(dictionary_corrupted);
/* Defer checking offcodeMaxValue because we need to know the size of the dictionary content */
CHECK_E( FSE_buildCTable_wksp(cctx->entropy->offcodeCTable, offcodeNCount, offcodeMaxValue, offcodeLog, cctx->entropy->workspace, sizeof(cctx->entropy->workspace)),
CHECK_E( FSE_buildCTable_wksp(bs->entropy.offcodeCTable, offcodeNCount, offcodeMaxValue, offcodeLog, workspace, HUF_WORKSPACE_SIZE),
dictionary_corrupted);
dictPtr += offcodeHeaderSize;
}
@ -2042,7 +2123,7 @@ static size_t ZSTD_loadZstdDictionary(ZSTD_CCtx* cctx, const void* dict, size_t
if (matchlengthLog > MLFSELog) return ERROR(dictionary_corrupted);
/* Every match length code must have non-zero probability */
CHECK_F( ZSTD_checkDictNCount(matchlengthNCount, matchlengthMaxValue, MaxML));
CHECK_E( FSE_buildCTable_wksp(cctx->entropy->matchlengthCTable, matchlengthNCount, matchlengthMaxValue, matchlengthLog, cctx->entropy->workspace, sizeof(cctx->entropy->workspace)),
CHECK_E( FSE_buildCTable_wksp(bs->entropy.matchlengthCTable, matchlengthNCount, matchlengthMaxValue, matchlengthLog, workspace, HUF_WORKSPACE_SIZE),
dictionary_corrupted);
dictPtr += matchlengthHeaderSize;
}
@ -2054,15 +2135,15 @@ static size_t ZSTD_loadZstdDictionary(ZSTD_CCtx* cctx, const void* dict, size_t
if (litlengthLog > LLFSELog) return ERROR(dictionary_corrupted);
/* Every literal length code must have non-zero probability */
CHECK_F( ZSTD_checkDictNCount(litlengthNCount, litlengthMaxValue, MaxLL));
CHECK_E( FSE_buildCTable_wksp(cctx->entropy->litlengthCTable, litlengthNCount, litlengthMaxValue, litlengthLog, cctx->entropy->workspace, sizeof(cctx->entropy->workspace)),
CHECK_E( FSE_buildCTable_wksp(bs->entropy.litlengthCTable, litlengthNCount, litlengthMaxValue, litlengthLog, workspace, HUF_WORKSPACE_SIZE),
dictionary_corrupted);
dictPtr += litlengthHeaderSize;
}
if (dictPtr+12 > dictEnd) return ERROR(dictionary_corrupted);
cctx->seqStore.rep[0] = MEM_readLE32(dictPtr+0);
cctx->seqStore.rep[1] = MEM_readLE32(dictPtr+4);
cctx->seqStore.rep[2] = MEM_readLE32(dictPtr+8);
bs->rep[0] = MEM_readLE32(dictPtr+0);
bs->rep[1] = MEM_readLE32(dictPtr+4);
bs->rep[2] = MEM_readLE32(dictPtr+8);
dictPtr += 12;
{ size_t const dictContentSize = (size_t)(dictEnd - dictPtr);
@ -2076,15 +2157,16 @@ static size_t ZSTD_loadZstdDictionary(ZSTD_CCtx* cctx, const void* dict, size_t
/* All repCodes must be <= dictContentSize and != 0*/
{ U32 u;
for (u=0; u<3; u++) {
if (cctx->seqStore.rep[u] == 0) return ERROR(dictionary_corrupted);
if (cctx->seqStore.rep[u] > dictContentSize) return ERROR(dictionary_corrupted);
if (bs->rep[u] == 0) return ERROR(dictionary_corrupted);
if (bs->rep[u] > dictContentSize) return ERROR(dictionary_corrupted);
} }
cctx->entropy->hufCTable_repeatMode = HUF_repeat_valid;
cctx->entropy->offcode_repeatMode = FSE_repeat_valid;
cctx->entropy->matchlength_repeatMode = FSE_repeat_valid;
cctx->entropy->litlength_repeatMode = FSE_repeat_valid;
return ZSTD_loadDictionaryContent(cctx, dictPtr, dictContentSize);
bs->entropy.hufCTable_repeatMode = HUF_repeat_valid;
bs->entropy.offcode_repeatMode = FSE_repeat_valid;
bs->entropy.matchlength_repeatMode = FSE_repeat_valid;
bs->entropy.litlength_repeatMode = FSE_repeat_valid;
CHECK_F(ZSTD_loadDictionaryContent(ms, params, dictPtr, dictContentSize));
return dictID;
}
}
@ -2099,12 +2181,12 @@ static size_t ZSTD_compress_insertDictionary(ZSTD_CCtx* cctx,
/* dict restricted modes */
if (dictMode==ZSTD_dm_rawContent)
return ZSTD_loadDictionaryContent(cctx, dict, dictSize);
return ZSTD_loadDictionaryContent(&cctx->matchState, &cctx->appliedParams, dict, dictSize);
if (MEM_readLE32(dict) != ZSTD_MAGIC_DICTIONARY) {
if (dictMode == ZSTD_dm_auto) {
DEBUGLOG(4, "raw content dictionary detected");
return ZSTD_loadDictionaryContent(cctx, dict, dictSize);
return ZSTD_loadDictionaryContent(&cctx->matchState, &cctx->appliedParams, dict, dictSize);
}
if (dictMode == ZSTD_dm_fullDict)
return ERROR(dictionary_wrong);
@ -2112,7 +2194,12 @@ static size_t ZSTD_compress_insertDictionary(ZSTD_CCtx* cctx,
}
/* dict as full zstd dictionary */
return ZSTD_loadZstdDictionary(cctx, dict, dictSize);
{
size_t const dictID = ZSTD_loadZstdDictionary(cctx->prevBlock, &cctx->matchState, &cctx->appliedParams, dict, dictSize, cctx->entropyWorkspace);
if (ZSTD_isError(dictID)) return dictID;
cctx->dictID = (U32)dictID;
}
return 0;
}
/*! ZSTD_compressBegin_internal() :

46
lib/compress/zstd_compress_internal.h
View File

@ -51,13 +51,17 @@ typedef struct {
FSE_CTable offcodeCTable[FSE_CTABLE_SIZE_U32(OffFSELog, MaxOff)];
FSE_CTable matchlengthCTable[FSE_CTABLE_SIZE_U32(MLFSELog, MaxML)];
FSE_CTable litlengthCTable[FSE_CTABLE_SIZE_U32(LLFSELog, MaxLL)];
U32 workspace[HUF_WORKSPACE_SIZE_U32];
HUF_repeat hufCTable_repeatMode;
FSE_repeat offcode_repeatMode;
FSE_repeat matchlength_repeatMode;
FSE_repeat litlength_repeatMode;
} ZSTD_entropyCTables_t;
typedef struct {
ZSTD_entropyCTables_t entropy;
U32 rep[ZSTD_REP_NUM];
} ZSTD_blockState_t;
typedef struct {
U32 off;
U32 len;
@ -93,6 +97,22 @@ typedef struct {
U32 staticPrices; /* prices follow a pre-defined cost structure, statistics are irrelevant */
} optState_t;
typedef struct {
BYTE const* nextSrc; /* next block here to continue on current prefix */ /* TODO: Does this belong here? */
BYTE const* base; /* All regular indexes relative to this position */
BYTE const* dictBase; /* extDict indexes relative to this position */
U32 dictLimit; /* below that point, need extDict */
U32 lowLimit; /* below that point, no more data */
U32 nextToUpdate; /* index from which to continue table update */
U32 nextToUpdate3; /* index from which to continue table update */
U32 hashLog3; /* dispatch table : larger == faster, more memory */
U32 loadedDictEnd; /* index of end of dictionary */ /* TODO: Does this belong here? */
U32* hashTable;
U32* hashTable3;
U32* chainTable;
optState_t opt; /* optimal parser state */
} ZSTD_matchState_t;
typedef struct {
U32 offset;
U32 checksum;
@ -136,15 +156,6 @@ struct ZSTD_CCtx_params_s {
}; /* typedef'd to ZSTD_CCtx_params within "zstd.h" */
struct ZSTD_CCtx_s {
const BYTE* nextSrc; /* next block here to continue on current prefix */
const BYTE* base; /* All regular indexes relative to this position */
const BYTE* dictBase; /* extDict indexes relative to this position */
U32 dictLimit; /* below that point, need extDict */
U32 lowLimit; /* below that point, no more data */
U32 nextToUpdate; /* index from which to continue dictionary update */
U32 nextToUpdate3; /* index from which to continue dictionary update */
U32 hashLog3; /* dispatch table : larger == faster, more memory */
U32 loadedDictEnd; /* index of end of dictionary */
ZSTD_compressionStage_e stage;
U32 dictID;
ZSTD_CCtx_params requestedParams;
@ -159,12 +170,11 @@ struct ZSTD_CCtx_s {
size_t staticSize;
seqStore_t seqStore; /* sequences storage ptrs */
optState_t optState;
ldmState_t ldmState; /* long distance matching state */
U32* hashTable;
U32* hashTable3;
U32* chainTable;
ZSTD_entropyCTables_t* entropy;
ZSTD_matchState_t matchState;
ZSTD_blockState_t* prevBlock;
ZSTD_blockState_t* nextBlock;
U32* entropyWorkspace; /* entropy workspace of HUF_WORKSPACE_SIZE bytes */
/* streaming */
char* inBuff;
@ -191,6 +201,12 @@ struct ZSTD_CCtx_s {
};
typedef size_t (*ZSTD_blockCompressor) (
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize);
ZSTD_blockCompressor ZSTD_selectBlockCompressor(ZSTD_strategy strat, int extDict);
MEM_STATIC U32 ZSTD_LLcode(U32 litLength)
{
static const BYTE LL_Code[64] = { 0, 1, 2, 3, 4, 5, 6, 7,

115
lib/compress/zstd_double_fast.c
View File

@ -12,14 +12,17 @@
#include "zstd_double_fast.h"
void ZSTD_fillDoubleHashTable(ZSTD_CCtx* cctx, const void* end, const U32 mls)
void ZSTD_fillDoubleHashTable(ZSTD_matchState_t* ms,
ZSTD_compressionParameters const* cParams,
void const* end)
{
U32* const hashLarge = cctx->hashTable;
U32 const hBitsL = cctx->appliedParams.cParams.hashLog;
U32* const hashSmall = cctx->chainTable;
U32 const hBitsS = cctx->appliedParams.cParams.chainLog;
const BYTE* const base = cctx->base;
const BYTE* ip = base + cctx->nextToUpdate;
U32* const hashLarge = ms->hashTable;
U32 const hBitsL = cParams->hashLog;
U32 const mls = cParams->searchLength;
U32* const hashSmall = ms->chainTable;
U32 const hBitsS = cParams->chainLog;
const BYTE* const base = ms->base;
const BYTE* ip = base + ms->nextToUpdate;
const BYTE* const iend = ((const BYTE*)end) - HASH_READ_SIZE;
const U32 fastHashFillStep = 3;
@ -43,24 +46,24 @@ void ZSTD_fillDoubleHashTable(ZSTD_CCtx* cctx, const void* end, const U32 mls)
FORCE_INLINE_TEMPLATE
size_t ZSTD_compressBlock_doubleFast_generic(ZSTD_CCtx* cctx,
const void* src, size_t srcSize,
const U32 mls)
size_t ZSTD_compressBlock_doubleFast_generic(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize,
U32 const mls /* template */)
{
U32* const hashLong = cctx->hashTable;
const U32 hBitsL = cctx->appliedParams.cParams.hashLog;
U32* const hashSmall = cctx->chainTable;
const U32 hBitsS = cctx->appliedParams.cParams.chainLog;
seqStore_t* seqStorePtr = &(cctx->seqStore);
const BYTE* const base = cctx->base;
U32* const hashLong = ms->hashTable;
const U32 hBitsL = cParams->hashLog;
U32* const hashSmall = ms->chainTable;
const U32 hBitsS = cParams->chainLog;
const BYTE* const base = ms->base;
const BYTE* const istart = (const BYTE*)src;
const BYTE* ip = istart;
const BYTE* anchor = istart;
const U32 lowestIndex = cctx->dictLimit;
const U32 lowestIndex = ms->dictLimit;
const BYTE* const lowest = base + lowestIndex;
const BYTE* const iend = istart + srcSize;
const BYTE* const ilimit = iend - HASH_READ_SIZE;
U32 offset_1=seqStorePtr->rep[0], offset_2=seqStorePtr->rep[1];
U32 offset_1=rep[0], offset_2=rep[1];
U32 offsetSaved = 0;
/* init */
@ -87,7 +90,7 @@ size_t ZSTD_compressBlock_doubleFast_generic(ZSTD_CCtx* cctx,
/* favor repcode */
mLength = ZSTD_count(ip+1+4, ip+1+4-offset_1, iend) + 4;
ip++;
ZSTD_storeSeq(seqStorePtr, ip-anchor, anchor, 0, mLength-MINMATCH);
ZSTD_storeSeq(seqStore, ip-anchor, anchor, 0, mLength-MINMATCH);
} else {
U32 offset;
if ( (matchIndexL > lowestIndex) && (MEM_read64(matchLong) == MEM_read64(ip)) ) {
@ -117,7 +120,7 @@ size_t ZSTD_compressBlock_doubleFast_generic(ZSTD_CCtx* cctx,
offset_2 = offset_1;
offset_1 = offset;
ZSTD_storeSeq(seqStorePtr, ip-anchor, anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH);
ZSTD_storeSeq(seqStore, ip-anchor, anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH);
}
/* match found */
@ -140,61 +143,63 @@ size_t ZSTD_compressBlock_doubleFast_generic(ZSTD_CCtx* cctx,
{ U32 const tmpOff = offset_2; offset_2 = offset_1; offset_1 = tmpOff; } /* swap offset_2 <=> offset_1 */
hashSmall[ZSTD_hashPtr(ip, hBitsS, mls)] = (U32)(ip-base);
hashLong[ZSTD_hashPtr(ip, hBitsL, 8)] = (U32)(ip-base);
ZSTD_storeSeq(seqStorePtr, 0, anchor, 0, rLength-MINMATCH);
ZSTD_storeSeq(seqStore, 0, anchor, 0, rLength-MINMATCH);
ip += rLength;
anchor = ip;
continue; /* faster when present ... (?) */
} } }
/* save reps for next block */
seqStorePtr->repToConfirm[0] = offset_1 ? offset_1 : offsetSaved;
seqStorePtr->repToConfirm[1] = offset_2 ? offset_2 : offsetSaved;
rep[0] = offset_1 ? offset_1 : offsetSaved;
rep[1] = offset_2 ? offset_2 : offsetSaved;
/* Return the last literals size */
return iend - anchor;
}
size_t ZSTD_compressBlock_doubleFast(ZSTD_CCtx* ctx, const void* src, size_t srcSize)
size_t ZSTD_compressBlock_doubleFast(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize)
{
const U32 mls = ctx->appliedParams.cParams.searchLength;
const U32 mls = cParams->searchLength;
switch(mls)
{
default: /* includes case 3 */
case 4 :
return ZSTD_compressBlock_doubleFast_generic(ctx, src, srcSize, 4);
return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, cParams, src, srcSize, 4);
case 5 :
return ZSTD_compressBlock_doubleFast_generic(ctx, src, srcSize, 5);
return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, cParams, src, srcSize, 5);
case 6 :
return ZSTD_compressBlock_doubleFast_generic(ctx, src, srcSize, 6);
return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, cParams, src, srcSize, 6);
case 7 :
return ZSTD_compressBlock_doubleFast_generic(ctx, src, srcSize, 7);
return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, cParams, src, srcSize, 7);
}
}
static size_t ZSTD_compressBlock_doubleFast_extDict_generic(ZSTD_CCtx* ctx,
const void* src, size_t srcSize,
const U32 mls)
static size_t ZSTD_compressBlock_doubleFast_extDict_generic(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize,
U32 const mls /* template */)
{
U32* const hashLong = ctx->hashTable;
U32 const hBitsL = ctx->appliedParams.cParams.hashLog;
U32* const hashSmall = ctx->chainTable;
U32 const hBitsS = ctx->appliedParams.cParams.chainLog;
seqStore_t* seqStorePtr = &(ctx->seqStore);
const BYTE* const base = ctx->base;
const BYTE* const dictBase = ctx->dictBase;
U32* const hashLong = ms->hashTable;
U32 const hBitsL = cParams->hashLog;
U32* const hashSmall = ms->chainTable;
U32 const hBitsS = cParams->chainLog;
const BYTE* const base = ms->base;
const BYTE* const dictBase = ms->dictBase;
const BYTE* const istart = (const BYTE*)src;
const BYTE* ip = istart;
const BYTE* anchor = istart;
const U32 lowestIndex = ctx->lowLimit;
const U32 lowestIndex = ms->lowLimit;
const BYTE* const dictStart = dictBase + lowestIndex;
const U32 dictLimit = ctx->dictLimit;
const U32 dictLimit = ms->dictLimit;
const BYTE* const lowPrefixPtr = base + dictLimit;
const BYTE* const dictEnd = dictBase + dictLimit;
const BYTE* const iend = istart + srcSize;
const BYTE* const ilimit = iend - 8;
U32 offset_1=seqStorePtr->rep[0], offset_2=seqStorePtr->rep[1];
U32 offset_1=rep[0], offset_2=rep[1];
/* Search Loop */
while (ip < ilimit) { /* < instead of <=, because (ip+1) */
@ -220,7 +225,7 @@ static size_t ZSTD_compressBlock_doubleFast_extDict_generic(ZSTD_CCtx* ctx,
const BYTE* repMatchEnd = repIndex < dictLimit ? dictEnd : iend;
mLength = ZSTD_count_2segments(ip+1+4, repMatch+4, iend, repMatchEnd, lowPrefixPtr) + 4;
ip++;
ZSTD_storeSeq(seqStorePtr, ip-anchor, anchor, 0, mLength-MINMATCH);
ZSTD_storeSeq(seqStore, ip-anchor, anchor, 0, mLength-MINMATCH);
} else {
if ((matchLongIndex > lowestIndex) && (MEM_read64(matchLong) == MEM_read64(ip))) {
const BYTE* matchEnd = matchLongIndex < dictLimit ? dictEnd : iend;
@ -231,7 +236,7 @@ static size_t ZSTD_compressBlock_doubleFast_extDict_generic(ZSTD_CCtx* ctx,
while (((ip>anchor) & (matchLong>lowMatchPtr)) && (ip[-1] == matchLong[-1])) { ip--; matchLong--; mLength++; } /* catch up */
offset_2 = offset_1;
offset_1 = offset;
ZSTD_storeSeq(seqStorePtr, ip-anchor, anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH);
ZSTD_storeSeq(seqStore, ip-anchor, anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH);
} else if ((matchIndex > lowestIndex) && (MEM_read32(match) == MEM_read32(ip))) {
size_t const h3 = ZSTD_hashPtr(ip+1, hBitsL, 8);
@ -256,7 +261,7 @@ static size_t ZSTD_compressBlock_doubleFast_extDict_generic(ZSTD_CCtx* ctx,
}
offset_2 = offset_1;
offset_1 = offset;
ZSTD_storeSeq(seqStorePtr, ip-anchor, anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH);
ZSTD_storeSeq(seqStore, ip-anchor, anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH);
} else {
ip += ((ip-anchor) >> g_searchStrength) + 1;
@ -283,7 +288,7 @@ static size_t ZSTD_compressBlock_doubleFast_extDict_generic(ZSTD_CCtx* ctx,
const BYTE* const repEnd2 = repIndex2 < dictLimit ? dictEnd : iend;
size_t const repLength2 = ZSTD_count_2segments(ip+4, repMatch2+4, iend, repEnd2, lowPrefixPtr) + 4;
U32 tmpOffset = offset_2; offset_2 = offset_1; offset_1 = tmpOffset; /* swap offset_2 <=> offset_1 */
ZSTD_storeSeq(seqStorePtr, 0, anchor, 0, repLength2-MINMATCH);
ZSTD_storeSeq(seqStore, 0, anchor, 0, repLength2-MINMATCH);
hashSmall[ZSTD_hashPtr(ip, hBitsS, mls)] = current2;
hashLong[ZSTD_hashPtr(ip, hBitsL, 8)] = current2;
ip += repLength2;
@ -294,27 +299,29 @@ static size_t ZSTD_compressBlock_doubleFast_extDict_generic(ZSTD_CCtx* ctx,
} } }
/* save reps for next block */
seqStorePtr->repToConfirm[0] = offset_1; seqStorePtr->repToConfirm[1] = offset_2;
rep[0] = offset_1;
rep[1] = offset_2;
/* Return the last literals size */
return iend - anchor;
}
size_t ZSTD_compressBlock_doubleFast_extDict(ZSTD_CCtx* ctx,
const void* src, size_t srcSize)
size_t ZSTD_compressBlock_doubleFast_extDict(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize)
{
U32 const mls = ctx->appliedParams.cParams.searchLength;
U32 const mls = cParams->searchLength;
switch(mls)
{
default: /* includes case 3 */
case 4 :
return ZSTD_compressBlock_doubleFast_extDict_generic(ctx, src, srcSize, 4);
return ZSTD_compressBlock_doubleFast_extDict_generic(ms, seqStore, rep, cParams, src, srcSize, 4);
case 5 :
return ZSTD_compressBlock_doubleFast_extDict_generic(ctx, src, srcSize, 5);
return ZSTD_compressBlock_doubleFast_extDict_generic(ms, seqStore, rep, cParams, src, srcSize, 5);
case 6 :
return ZSTD_compressBlock_doubleFast_extDict_generic(ctx, src, srcSize, 6);
return ZSTD_compressBlock_doubleFast_extDict_generic(ms, seqStore, rep, cParams, src, srcSize, 6);
case 7 :
return ZSTD_compressBlock_doubleFast_extDict_generic(ctx, src, srcSize, 7);
return ZSTD_compressBlock_doubleFast_extDict_generic(ms, seqStore, rep, cParams, src, srcSize, 7);
}
}

15
lib/compress/zstd_double_fast.h
View File

@ -16,11 +16,18 @@ extern "C" {
#endif
#include "mem.h" /* U32 */
#include "zstd.h" /* ZSTD_CCtx, size_t */
#include "zstd_compress_internal.h" /* ZSTD_CCtx, size_t */
void ZSTD_fillDoubleHashTable(ZSTD_matchState_t* ms,
ZSTD_compressionParameters const* cParams,
void const* end);
size_t ZSTD_compressBlock_doubleFast(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize);
size_t ZSTD_compressBlock_doubleFast_extDict(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize);
void ZSTD_fillDoubleHashTable(ZSTD_CCtx* cctx, const void* end, const U32 mls);
size_t ZSTD_compressBlock_doubleFast(ZSTD_CCtx* ctx, const void* src, size_t srcSize);
size_t ZSTD_compressBlock_doubleFast_extDict(ZSTD_CCtx* ctx, const void* src, size_t srcSize);
#if defined (__cplusplus)
}

119
lib/compress/zstd_fast.c
View File

@ -12,12 +12,15 @@
#include "zstd_fast.h"
void ZSTD_fillHashTable (ZSTD_CCtx* zc, const void* end, const U32 mls)
void ZSTD_fillHashTable(ZSTD_matchState_t* ms,
ZSTD_compressionParameters const* cParams,
void const* end)
{
U32* const hashTable = zc->hashTable;
U32 const hBits = zc->appliedParams.cParams.hashLog;
const BYTE* const base = zc->base;
const BYTE* ip = base + zc->nextToUpdate;
U32* const hashTable = ms->hashTable;
U32 const hBits = cParams->hashLog;
U32 const mls = cParams->searchLength;
const BYTE* const base = ms->base;
const BYTE* ip = base + ms->nextToUpdate;
const BYTE* const iend = ((const BYTE*)end) - HASH_READ_SIZE;
const U32 fastHashFillStep = 3;
@ -35,24 +38,22 @@ void ZSTD_fillHashTable (ZSTD_CCtx* zc, const void* end, const U32 mls)
}
}
FORCE_INLINE_TEMPLATE
size_t ZSTD_compressBlock_fast_generic(ZSTD_CCtx* cctx,
const void* src, size_t srcSize,
const U32 mls)
size_t ZSTD_compressBlock_fast_generic(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize,
U32 const hlog, U32 const mls)
{
U32* const hashTable = cctx->hashTable;
U32 const hBits = cctx->appliedParams.cParams.hashLog;
seqStore_t* seqStorePtr = &(cctx->seqStore);
const BYTE* const base = cctx->base;
U32* const hashTable = ms->hashTable;
const BYTE* const base = ms->base;
const BYTE* const istart = (const BYTE*)src;
const BYTE* ip = istart;
const BYTE* anchor = istart;
const U32 lowestIndex = cctx->dictLimit;
const U32 lowestIndex = ms->dictLimit;
const BYTE* const lowest = base + lowestIndex;
const BYTE* const iend = istart + srcSize;
const BYTE* const ilimit = iend - HASH_READ_SIZE;
U32 offset_1=seqStorePtr->rep[0], offset_2=seqStorePtr->rep[1];
U32 offset_1=rep[0], offset_2=rep[1];
U32 offsetSaved = 0;
/* init */
@ -65,7 +66,7 @@ size_t ZSTD_compressBlock_fast_generic(ZSTD_CCtx* cctx,
/* Main Search Loop */
while (ip < ilimit) { /* < instead of <=, because repcode check at (ip+1) */
size_t mLength;
size_t const h = ZSTD_hashPtr(ip, hBits, mls);
size_t const h = ZSTD_hashPtr(ip, hlog, mls);
U32 const current = (U32)(ip-base);
U32 const matchIndex = hashTable[h];
const BYTE* match = base + matchIndex;
@ -74,7 +75,7 @@ size_t ZSTD_compressBlock_fast_generic(ZSTD_CCtx* cctx,
if ((offset_1 > 0) & (MEM_read32(ip+1-offset_1) == MEM_read32(ip+1))) {
mLength = ZSTD_count(ip+1+4, ip+1+4-offset_1, iend) + 4;
ip++;
ZSTD_storeSeq(seqStorePtr, ip-anchor, anchor, 0, mLength-MINMATCH);
ZSTD_storeSeq(seqStore, ip-anchor, anchor, 0, mLength-MINMATCH);
} else {
U32 offset;
if ( (matchIndex <= lowestIndex) || (MEM_read32(match) != MEM_read32(ip)) ) {
@ -87,7 +88,7 @@ size_t ZSTD_compressBlock_fast_generic(ZSTD_CCtx* cctx,
offset_2 = offset_1;
offset_1 = offset;
ZSTD_storeSeq(seqStorePtr, ip-anchor, anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH);
ZSTD_storeSeq(seqStore, ip-anchor, anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH);
}
/* match found */
@ -96,8 +97,8 @@ size_t ZSTD_compressBlock_fast_generic(ZSTD_CCtx* cctx,
if (ip <= ilimit) {
/* Fill Table */
hashTable[ZSTD_hashPtr(base+current+2, hBits, mls)] = current+2; /* here because current+2 could be > iend-8 */
hashTable[ZSTD_hashPtr(ip-2, hBits, mls)] = (U32)(ip-2-base);
hashTable[ZSTD_hashPtr(base+current+2, hlog, mls)] = current+2; /* here because current+2 could be > iend-8 */
hashTable[ZSTD_hashPtr(ip-2, hlog, mls)] = (U32)(ip-2-base);
/* check immediate repcode */
while ( (ip <= ilimit)
&& ( (offset_2>0)
@ -105,65 +106,66 @@ size_t ZSTD_compressBlock_fast_generic(ZSTD_CCtx* cctx,
/* store sequence */
size_t const rLength = ZSTD_count(ip+4, ip+4-offset_2, iend) + 4;
{ U32 const tmpOff = offset_2; offset_2 = offset_1; offset_1 = tmpOff; } /* swap offset_2 <=> offset_1 */
hashTable[ZSTD_hashPtr(ip, hBits, mls)] = (U32)(ip-base);
ZSTD_storeSeq(seqStorePtr, 0, anchor, 0, rLength-MINMATCH);
hashTable[ZSTD_hashPtr(ip, hlog, mls)] = (U32)(ip-base);
ZSTD_storeSeq(seqStore, 0, anchor, 0, rLength-MINMATCH);
ip += rLength;
anchor = ip;
continue; /* faster when present ... (?) */
} } }
/* save reps for next block */
seqStorePtr->repToConfirm[0] = offset_1 ? offset_1 : offsetSaved;
seqStorePtr->repToConfirm[1] = offset_2 ? offset_2 : offsetSaved;
rep[0] = offset_1 ? offset_1 : offsetSaved;
rep[1] = offset_2 ? offset_2 : offsetSaved;
/* Return the last literals size */
return iend - anchor;
}
size_t ZSTD_compressBlock_fast(ZSTD_CCtx* ctx,
const void* src, size_t srcSize)
size_t ZSTD_compressBlock_fast(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize)
{
const U32 mls = ctx->appliedParams.cParams.searchLength;
U32 const hlog = cParams->hashLog;
U32 const mls = cParams->searchLength;
switch(mls)
{
default: /* includes case 3 */
case 4 :
return ZSTD_compressBlock_fast_generic(ctx, src, srcSize, 4);
return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, hlog, 4);
case 5 :
return ZSTD_compressBlock_fast_generic(ctx, src, srcSize, 5);
return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, hlog, 5);
case 6 :
return ZSTD_compressBlock_fast_generic(ctx, src, srcSize, 6);
return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, hlog, 6);
case 7 :
return ZSTD_compressBlock_fast_generic(ctx, src, srcSize, 7);
return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, hlog, 7);
}
}
static size_t ZSTD_compressBlock_fast_extDict_generic(ZSTD_CCtx* ctx,
const void* src, size_t srcSize,
const U32 mls)
static size_t ZSTD_compressBlock_fast_extDict_generic(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize,
U32 const hlog, U32 const mls)
{
U32* hashTable = ctx->hashTable;
const U32 hBits = ctx->appliedParams.cParams.hashLog;
seqStore_t* seqStorePtr = &(ctx->seqStore);
const BYTE* const base = ctx->base;
const BYTE* const dictBase = ctx->dictBase;
U32* hashTable = ms->hashTable;
const BYTE* const base = ms->base;
const BYTE* const dictBase = ms->dictBase;
const BYTE* const istart = (const BYTE*)src;
const BYTE* ip = istart;
const BYTE* anchor = istart;
const U32 lowestIndex = ctx->lowLimit;
const U32 lowestIndex = ms->lowLimit;
const BYTE* const dictStart = dictBase + lowestIndex;
const U32 dictLimit = ctx->dictLimit;
const U32 dictLimit = ms->dictLimit;
const BYTE* const lowPrefixPtr = base + dictLimit;
const BYTE* const dictEnd = dictBase + dictLimit;
const BYTE* const iend = istart + srcSize;
const BYTE* const ilimit = iend - 8;
U32 offset_1=seqStorePtr->rep[0], offset_2=seqStorePtr->rep[1];
U32 offset_1=rep[0], offset_2=rep[1];
/* Search Loop */
while (ip < ilimit) { /* < instead of <=, because (ip+1) */
const size_t h = ZSTD_hashPtr(ip, hBits, mls);
const size_t h = ZSTD_hashPtr(ip, hlog, mls);
const U32 matchIndex = hashTable[h];
const BYTE* matchBase = matchIndex < dictLimit ? dictBase : base;
const BYTE* match = matchBase + matchIndex;
@ -179,7 +181,7 @@ static size_t ZSTD_compressBlock_fast_extDict_generic(ZSTD_CCtx* ctx,
const BYTE* repMatchEnd = repIndex < dictLimit ? dictEnd : iend;
mLength = ZSTD_count_2segments(ip+1+4, repMatch+4, iend, repMatchEnd, lowPrefixPtr) + 4;
ip++;
ZSTD_storeSeq(seqStorePtr, ip-anchor, anchor, 0, mLength-MINMATCH);
ZSTD_storeSeq(seqStore, ip-anchor, anchor, 0, mLength-MINMATCH);
} else {
if ( (matchIndex < lowestIndex) ||
(MEM_read32(match) != MEM_read32(ip)) ) {
@ -194,7 +196,7 @@ static size_t ZSTD_compressBlock_fast_extDict_generic(ZSTD_CCtx* ctx,
offset = current - matchIndex;
offset_2 = offset_1;
offset_1 = offset;
ZSTD_storeSeq(seqStorePtr, ip-anchor, anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH);
ZSTD_storeSeq(seqStore, ip-anchor, anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH);
} }
/* found a match : store it */
@ -203,8 +205,8 @@ static size_t ZSTD_compressBlock_fast_extDict_generic(ZSTD_CCtx* ctx,
if (ip <= ilimit) {
/* Fill Table */
hashTable[ZSTD_hashPtr(base+current+2, hBits, mls)] = current+2;
hashTable[ZSTD_hashPtr(ip-2, hBits, mls)] = (U32)(ip-2-base);
hashTable[ZSTD_hashPtr(base+current+2, hlog, mls)] = current+2;
hashTable[ZSTD_hashPtr(ip-2, hlog, mls)] = (U32)(ip-2-base);
/* check immediate repcode */
while (ip <= ilimit) {
U32 const current2 = (U32)(ip-base);
@ -215,8 +217,8 @@ static size_t ZSTD_compressBlock_fast_extDict_generic(ZSTD_CCtx* ctx,
const BYTE* const repEnd2 = repIndex2 < dictLimit ? dictEnd : iend;
size_t const repLength2 = ZSTD_count_2segments(ip+4, repMatch2+4, iend, repEnd2, lowPrefixPtr) + 4;
U32 tmpOffset = offset_2; offset_2 = offset_1; offset_1 = tmpOffset; /* swap offset_2 <=> offset_1 */
ZSTD_storeSeq(seqStorePtr, 0, anchor, 0, repLength2-MINMATCH);
hashTable[ZSTD_hashPtr(ip, hBits, mls)] = current2;
ZSTD_storeSeq(seqStore, 0, anchor, 0, repLength2-MINMATCH);
hashTable[ZSTD_hashPtr(ip, hlog, mls)] = current2;
ip += repLength2;
anchor = ip;
continue;
@ -225,27 +227,30 @@ static size_t ZSTD_compressBlock_fast_extDict_generic(ZSTD_CCtx* ctx,
} } }
/* save reps for next block */
seqStorePtr->repToConfirm[0] = offset_1; seqStorePtr->repToConfirm[1] = offset_2;
rep[0] = offset_1;
rep[1] = offset_2;
/* Return the last literals size */
return iend - anchor;
}
size_t ZSTD_compressBlock_fast_extDict(ZSTD_CCtx* ctx,
const void* src, size_t srcSize)
size_t ZSTD_compressBlock_fast_extDict(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize)
{
U32 const mls = ctx->appliedParams.cParams.searchLength;
U32 const hlog = cParams->hashLog;
U32 const mls = cParams->searchLength;
switch(mls)
{
default: /* includes case 3 */
case 4 :
return ZSTD_compressBlock_fast_extDict_generic(ctx, src, srcSize, 4);
return ZSTD_compressBlock_fast_extDict_generic(ms, seqStore, rep, src, srcSize, hlog, 4);
case 5 :
return ZSTD_compressBlock_fast_extDict_generic(ctx, src, srcSize, 5);
return ZSTD_compressBlock_fast_extDict_generic(ms, seqStore, rep, src, srcSize, hlog, 5);
case 6 :
return ZSTD_compressBlock_fast_extDict_generic(ctx, src, srcSize, 6);
return ZSTD_compressBlock_fast_extDict_generic(ms, seqStore, rep, src, srcSize, hlog, 6);
case 7 :
return ZSTD_compressBlock_fast_extDict_generic(ctx, src, srcSize, 7);
return ZSTD_compressBlock_fast_extDict_generic(ms, seqStore, rep, src, srcSize, hlog, 7);
}
}

16
lib/compress/zstd_fast.h
View File

@ -16,13 +16,17 @@ extern "C" {
#endif
#include "mem.h" /* U32 */
#include "zstd.h" /* ZSTD_CCtx, size_t */
#include "zstd_compress_internal.h"
void ZSTD_fillHashTable(ZSTD_CCtx* zc, const void* end, const U32 mls);
size_t ZSTD_compressBlock_fast(ZSTD_CCtx* ctx,
const void* src, size_t srcSize);
size_t ZSTD_compressBlock_fast_extDict(ZSTD_CCtx* ctx,
const void* src, size_t srcSize);
void ZSTD_fillHashTable(ZSTD_matchState_t* ms,
ZSTD_compressionParameters const* cParams,
void const* end);
size_t ZSTD_compressBlock_fast(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize);
size_t ZSTD_compressBlock_fast_extDict(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize);
#if defined (__cplusplus)
}

308
lib/compress/zstd_lazy.c
View File

@ -45,28 +45,29 @@ void ZSTD_preserveUnsortedMark (U32* const table, U32 const size, U32 const redu
table[u] = ZSTD_DUBT_UNSORTED_MARK + reducerValue;
}
void ZSTD_updateDUBT(ZSTD_CCtx* zc,
void ZSTD_updateDUBT(
ZSTD_matchState_t* ms, ZSTD_compressionParameters const* cParams,
const BYTE* ip, const BYTE* iend,
U32 mls)
{
U32* const hashTable = zc->hashTable;
U32 const hashLog = zc->appliedParams.cParams.hashLog;
U32* const hashTable = ms->hashTable;
U32 const hashLog = cParams->hashLog;
U32* const bt = zc->chainTable;
U32 const btLog = zc->appliedParams.cParams.chainLog - 1;
U32* const bt = ms->chainTable;
U32 const btLog = cParams->chainLog - 1;
U32 const btMask = (1 << btLog) - 1;
const BYTE* const base = zc->base;
const BYTE* const base = ms->base;
U32 const target = (U32)(ip - base);
U32 idx = zc->nextToUpdate;
U32 idx = ms->nextToUpdate;
if (idx != target)
DEBUGLOG(7, "ZSTD_updateDUBT, from %u to %u (dictLimit:%u)",
idx, target, zc->dictLimit);
idx, target, ms->dictLimit);
assert(ip + 8 <= iend); /* condition for ZSTD_hashPtr */
(void)iend;
assert(idx >= zc->dictLimit); /* condition for valid base+idx */
assert(idx >= ms->dictLimit); /* condition for valid base+idx */
for ( ; idx < target ; idx++) {
size_t const h = ZSTD_hashPtr(base + idx, hashLog, mls); /* assumption : ip + 8 <= iend */
U32 const matchIndex = hashTable[h];
@ -79,7 +80,7 @@ void ZSTD_updateDUBT(ZSTD_CCtx* zc,
*nextCandidatePtr = matchIndex; /* update BT like a chain */
*sortMarkPtr = ZSTD_DUBT_UNSORTED_MARK;
}
zc->nextToUpdate = target;
ms->nextToUpdate = target;
}
@ -87,17 +88,18 @@ void ZSTD_updateDUBT(ZSTD_CCtx* zc,
* sort one already inserted but unsorted position
* assumption : current >= btlow == (current - btmask)
* doesn't fail */
static void ZSTD_insertDUBT1(ZSTD_CCtx* zc,
static void ZSTD_insertDUBT1(
ZSTD_matchState_t* ms, ZSTD_compressionParameters const* cParams,
U32 current, const BYTE* inputEnd,
U32 nbCompares, U32 btLow, int extDict)
{
U32* const bt = zc->chainTable;
U32 const btLog = zc->appliedParams.cParams.chainLog - 1;
U32* const bt = ms->chainTable;
U32 const btLog = cParams->chainLog - 1;
U32 const btMask = (1 << btLog) - 1;
size_t commonLengthSmaller=0, commonLengthLarger=0;
const BYTE* const base = zc->base;
const BYTE* const dictBase = zc->dictBase;
const U32 dictLimit = zc->dictLimit;
const BYTE* const base = ms->base;
const BYTE* const dictBase = ms->dictBase;
const U32 dictLimit = ms->dictLimit;
const BYTE* const ip = (current>=dictLimit) ? base + current : dictBase + current;
const BYTE* const iend = (current>=dictLimit) ? inputEnd : dictBase + dictLimit;
const BYTE* const dictEnd = dictBase + dictLimit;
@ -107,7 +109,7 @@ static void ZSTD_insertDUBT1(ZSTD_CCtx* zc,
U32* largerPtr = smallerPtr + 1;
U32 matchIndex = *smallerPtr;
U32 dummy32; /* to be nullified at the end */
U32 const windowLow = zc->lowLimit;
U32 const windowLow = ms->lowLimit;
DEBUGLOG(8, "ZSTD_insertDUBT1(%u) (dictLimit=%u, lowLimit=%u)",
current, dictLimit, windowLow);
@ -166,29 +168,30 @@ static void ZSTD_insertDUBT1(ZSTD_CCtx* zc,
static size_t ZSTD_DUBT_findBestMatch (
ZSTD_CCtx* zc,
ZSTD_matchState_t* ms, ZSTD_compressionParameters const* cParams,
const BYTE* const ip, const BYTE* const iend,
size_t* offsetPtr,
U32 nbCompares, const U32 mls,
U32 extDict)
U32 const mls,
U32 const extDict)
{
U32* const hashTable = zc->hashTable;
U32 const hashLog = zc->appliedParams.cParams.hashLog;
U32* const hashTable = ms->hashTable;
U32 const hashLog = cParams->hashLog;
size_t const h = ZSTD_hashPtr(ip, hashLog, mls);
U32 matchIndex = hashTable[h];
const BYTE* const base = zc->base;
const BYTE* const base = ms->base;
U32 const current = (U32)(ip-base);
U32 const windowLow = zc->lowLimit;
U32 const windowLow = ms->lowLimit;
U32* const bt = zc->chainTable;
U32 const btLog = zc->appliedParams.cParams.chainLog - 1;
U32* const bt = ms->chainTable;
U32 const btLog = cParams->chainLog - 1;
U32 const btMask = (1 << btLog) - 1;
U32 const btLow = (btMask >= current) ? 0 : current - btMask;
U32 const unsortLimit = MAX(btLow, windowLow);
U32* nextCandidate = bt + 2*(matchIndex&btMask);
U32* unsortedMark = bt + 2*(matchIndex&btMask) + 1;
U32 nbCompares = 1U << cParams->searchLog;
U32 nbCandidates = nbCompares;
U32 previousCandidate = 0;
@ -221,7 +224,7 @@ static size_t ZSTD_DUBT_findBestMatch (
while (matchIndex) { /* will end on matchIndex == 0 */
U32* const nextCandidateIdxPtr = bt + 2*(matchIndex&btMask) + 1;
U32 const nextCandidateIdx = *nextCandidateIdxPtr;
ZSTD_insertDUBT1(zc, matchIndex, iend,
ZSTD_insertDUBT1(ms, cParams, matchIndex, iend,
nbCandidates, unsortLimit, extDict);
matchIndex = nextCandidateIdx;
nbCandidates++;
@ -229,8 +232,8 @@ static size_t ZSTD_DUBT_findBestMatch (
/* find longest match */
{ size_t commonLengthSmaller=0, commonLengthLarger=0;
const BYTE* const dictBase = zc->dictBase;
const U32 dictLimit = zc->dictLimit;
const BYTE* const dictBase = ms->dictBase;
const U32 dictLimit = ms->dictLimit;
const BYTE* const dictEnd = dictBase + dictLimit;
const BYTE* const prefixStart = base + dictLimit;
U32* smallerPtr = bt + 2*(current&btMask);
@ -286,7 +289,7 @@ static size_t ZSTD_DUBT_findBestMatch (
*smallerPtr = *largerPtr = 0;
assert(matchEndIdx > current+8); /* ensure nextToUpdate is increased */
zc->nextToUpdate = matchEndIdx - 8; /* skip repetitive patterns */
ms->nextToUpdate = matchEndIdx - 8; /* skip repetitive patterns */
if (bestLength >= MINMATCH) {
U32 const mIndex = current - ((U32)*offsetPtr - ZSTD_REP_MOVE); (void)mIndex;
DEBUGLOG(8, "ZSTD_DUBT_findBestMatch(%u) : found match of length %u and offsetCode %u (pos %u)",
@ -299,62 +302,60 @@ static size_t ZSTD_DUBT_findBestMatch (
/** ZSTD_BtFindBestMatch() : Tree updater, providing best match */
static size_t ZSTD_BtFindBestMatch (
ZSTD_CCtx* zc,
ZSTD_matchState_t* ms, ZSTD_compressionParameters const* cParams,
const BYTE* const ip, const BYTE* const iLimit,
size_t* offsetPtr,
const U32 maxNbAttempts, const U32 mls)
const U32 mls /* template */)
{
DEBUGLOG(7, "ZSTD_BtFindBestMatch");
if (ip < zc->base + zc->nextToUpdate) return 0; /* skipped area */
ZSTD_updateDUBT(zc, ip, iLimit, mls);
return ZSTD_DUBT_findBestMatch(zc, ip, iLimit, offsetPtr, maxNbAttempts, mls, 0);
if (ip < ms->base + ms->nextToUpdate) return 0; /* skipped area */
ZSTD_updateDUBT(ms, cParams, ip, iLimit, mls);
return ZSTD_DUBT_findBestMatch(ms, cParams, ip, iLimit, offsetPtr, mls, 0);
}
static size_t ZSTD_BtFindBestMatch_selectMLS (
ZSTD_CCtx* zc, /* Index table will be updated */
ZSTD_matchState_t* ms, ZSTD_compressionParameters const* cParams,
const BYTE* ip, const BYTE* const iLimit,
size_t* offsetPtr,
const U32 maxNbAttempts, const U32 matchLengthSearch)
size_t* offsetPtr)
{
switch(matchLengthSearch)
switch(cParams->searchLength)
{
default : /* includes case 3 */
case 4 : return ZSTD_BtFindBestMatch(zc, ip, iLimit, offsetPtr, maxNbAttempts, 4);
case 5 : return ZSTD_BtFindBestMatch(zc, ip, iLimit, offsetPtr, maxNbAttempts, 5);
case 4 : return ZSTD_BtFindBestMatch(ms, cParams, ip, iLimit, offsetPtr, 4);
case 5 : return ZSTD_BtFindBestMatch(ms, cParams, ip, iLimit, offsetPtr, 5);
case 7 :
case 6 : return ZSTD_BtFindBestMatch(zc, ip, iLimit, offsetPtr, maxNbAttempts, 6);
case 6 : return ZSTD_BtFindBestMatch(ms, cParams, ip, iLimit, offsetPtr, 6);
}
}
/** Tree updater, providing best match */
static size_t ZSTD_BtFindBestMatch_extDict (
ZSTD_CCtx* zc,
ZSTD_matchState_t* ms, ZSTD_compressionParameters const* cParams,
const BYTE* const ip, const BYTE* const iLimit,
size_t* offsetPtr,
const U32 maxNbAttempts, const U32 mls)
const U32 mls)
{
DEBUGLOG(7, "ZSTD_BtFindBestMatch_extDict");
if (ip < zc->base + zc->nextToUpdate) return 0; /* skipped area */
ZSTD_updateDUBT(zc, ip, iLimit, mls);
return ZSTD_DUBT_findBestMatch(zc, ip, iLimit, offsetPtr, maxNbAttempts, mls, 1);
if (ip < ms->base + ms->nextToUpdate) return 0; /* skipped area */
ZSTD_updateDUBT(ms, cParams, ip, iLimit, mls);
return ZSTD_DUBT_findBestMatch(ms, cParams, ip, iLimit, offsetPtr, mls, 1);
}
static size_t ZSTD_BtFindBestMatch_selectMLS_extDict (
ZSTD_CCtx* zc, /* Index table will be updated */
ZSTD_matchState_t* ms, ZSTD_compressionParameters const* cParams,
const BYTE* ip, const BYTE* const iLimit,
size_t* offsetPtr,
const U32 maxNbAttempts, const U32 matchLengthSearch)
size_t* offsetPtr)
{
switch(matchLengthSearch)
switch(cParams->searchLength)
{
default : /* includes case 3 */
case 4 : return ZSTD_BtFindBestMatch_extDict(zc, ip, iLimit, offsetPtr, maxNbAttempts, 4);
case 5 : return ZSTD_BtFindBestMatch_extDict(zc, ip, iLimit, offsetPtr, maxNbAttempts, 5);
case 4 : return ZSTD_BtFindBestMatch_extDict(ms, cParams, ip, iLimit, offsetPtr, 4);
case 5 : return ZSTD_BtFindBestMatch_extDict(ms, cParams, ip, iLimit, offsetPtr, 5);
case 7 :
case 6 : return ZSTD_BtFindBestMatch_extDict(zc, ip, iLimit, offsetPtr, maxNbAttempts, 6);
case 6 : return ZSTD_BtFindBestMatch_extDict(ms, cParams, ip, iLimit, offsetPtr, 6);
}
}
@ -367,15 +368,17 @@ static size_t ZSTD_BtFindBestMatch_selectMLS_extDict (
/* Update chains up to ip (excluded)
Assumption : always within prefix (i.e. not within extDict) */
U32 ZSTD_insertAndFindFirstIndex (ZSTD_CCtx* zc, const BYTE* ip, U32 mls)
static U32 ZSTD_insertAndFindFirstIndex_internal(
ZSTD_matchState_t* ms, ZSTD_compressionParameters const* cParams,
const BYTE* ip, U32 const mls)
{
U32* const hashTable = zc->hashTable;
const U32 hashLog = zc->appliedParams.cParams.hashLog;
U32* const chainTable = zc->chainTable;
const U32 chainMask = (1 << zc->appliedParams.cParams.chainLog) - 1;
const BYTE* const base = zc->base;
U32* const hashTable = ms->hashTable;
const U32 hashLog = cParams->hashLog;
U32* const chainTable = ms->chainTable;
const U32 chainMask = (1 << cParams->chainLog) - 1;
const BYTE* const base = ms->base;
const U32 target = (U32)(ip - base);
U32 idx = zc->nextToUpdate;
U32 idx = ms->nextToUpdate;
while(idx < target) { /* catch up */
size_t const h = ZSTD_hashPtr(base+idx, hashLog, mls);
@ -384,35 +387,42 @@ U32 ZSTD_insertAndFindFirstIndex (ZSTD_CCtx* zc, const BYTE* ip, U32 mls)
idx++;
}
zc->nextToUpdate = target;
ms->nextToUpdate = target;
return hashTable[ZSTD_hashPtr(ip, hashLog, mls)];
}
U32 ZSTD_insertAndFindFirstIndex(
ZSTD_matchState_t* ms, ZSTD_compressionParameters const* cParams,
const BYTE* ip)
{
return ZSTD_insertAndFindFirstIndex_internal(ms, cParams, ip, cParams->searchLength);
}
/* inlining is important to hardwire a hot branch (template emulation) */
FORCE_INLINE_TEMPLATE
size_t ZSTD_HcFindBestMatch_generic (
ZSTD_CCtx* zc, /* Index table will be updated */
ZSTD_matchState_t* ms, ZSTD_compressionParameters const* cParams,
const BYTE* const ip, const BYTE* const iLimit,
size_t* offsetPtr,
const U32 maxNbAttempts, const U32 mls, const U32 extDict)
const U32 mls, const U32 extDict)
{
U32* const chainTable = zc->chainTable;
const U32 chainSize = (1 << zc->appliedParams.cParams.chainLog);
U32* const chainTable = ms->chainTable;
const U32 chainSize = (1 << cParams->chainLog);
const U32 chainMask = chainSize-1;
const BYTE* const base = zc->base;
const BYTE* const dictBase = zc->dictBase;
const U32 dictLimit = zc->dictLimit;
const BYTE* const base = ms->base;
const BYTE* const dictBase = ms->dictBase;
const U32 dictLimit = ms->dictLimit;
const BYTE* const prefixStart = base + dictLimit;
const BYTE* const dictEnd = dictBase + dictLimit;
const U32 lowLimit = zc->lowLimit;
const U32 lowLimit = ms->lowLimit;
const U32 current = (U32)(ip-base);
const U32 minChain = current > chainSize ? current - chainSize : 0;
int nbAttempts=maxNbAttempts;
U32 nbAttempts = 1U << cParams->searchLog;
size_t ml=4-1;
/* HC4 match finder */
U32 matchIndex = ZSTD_insertAndFindFirstIndex (zc, ip, mls);
U32 matchIndex = ZSTD_insertAndFindFirstIndex_internal(ms, cParams, ip, mls);
for ( ; (matchIndex>lowLimit) & (nbAttempts>0) ; nbAttempts--) {
size_t currentMl=0;
@ -443,35 +453,33 @@ size_t ZSTD_HcFindBestMatch_generic (
FORCE_INLINE_TEMPLATE size_t ZSTD_HcFindBestMatch_selectMLS (
ZSTD_CCtx* zc,
ZSTD_matchState_t* ms, ZSTD_compressionParameters const* cParams,
const BYTE* ip, const BYTE* const iLimit,
size_t* offsetPtr,
const U32 maxNbAttempts, const U32 matchLengthSearch)
size_t* offsetPtr)
{
switch(matchLengthSearch)
switch(cParams->searchLength)
{
default : /* includes case 3 */
case 4 : return ZSTD_HcFindBestMatch_generic(zc, ip, iLimit, offsetPtr, maxNbAttempts, 4, 0);
case 5 : return ZSTD_HcFindBestMatch_generic(zc, ip, iLimit, offsetPtr, maxNbAttempts, 5, 0);
case 4 : return ZSTD_HcFindBestMatch_generic(ms, cParams, ip, iLimit, offsetPtr, 4, 0);
case 5 : return ZSTD_HcFindBestMatch_generic(ms, cParams, ip, iLimit, offsetPtr, 5, 0);
case 7 :
case 6 : return ZSTD_HcFindBestMatch_generic(zc, ip, iLimit, offsetPtr, maxNbAttempts, 6, 0);
case 6 : return ZSTD_HcFindBestMatch_generic(ms, cParams, ip, iLimit, offsetPtr, 6, 0);
}
}
FORCE_INLINE_TEMPLATE size_t ZSTD_HcFindBestMatch_extDict_selectMLS (
ZSTD_CCtx* const zc,
ZSTD_matchState_t* ms, ZSTD_compressionParameters const* cParams,
const BYTE* ip, const BYTE* const iLimit,
size_t* const offsetPtr,
U32 const maxNbAttempts, U32 const matchLengthSearch)
size_t* const offsetPtr)
{
switch(matchLengthSearch)
switch(cParams->searchLength)
{
default : /* includes case 3 */
case 4 : return ZSTD_HcFindBestMatch_generic(zc, ip, iLimit, offsetPtr, maxNbAttempts, 4, 1);
case 5 : return ZSTD_HcFindBestMatch_generic(zc, ip, iLimit, offsetPtr, maxNbAttempts, 5, 1);
case 4 : return ZSTD_HcFindBestMatch_generic(ms, cParams, ip, iLimit, offsetPtr, 4, 1);
case 5 : return ZSTD_HcFindBestMatch_generic(ms, cParams, ip, iLimit, offsetPtr, 5, 1);
case 7 :
case 6 : return ZSTD_HcFindBestMatch_generic(zc, ip, iLimit, offsetPtr, maxNbAttempts, 6, 1);
case 6 : return ZSTD_HcFindBestMatch_generic(ms, cParams, ip, iLimit, offsetPtr, 6, 1);
}
}
@ -480,30 +488,29 @@ FORCE_INLINE_TEMPLATE size_t ZSTD_HcFindBestMatch_extDict_selectMLS (
* Common parser - lazy strategy
*********************************/
FORCE_INLINE_TEMPLATE
size_t ZSTD_compressBlock_lazy_generic(ZSTD_CCtx* ctx,
size_t ZSTD_compressBlock_lazy_generic(
ZSTD_matchState_t* ms, seqStore_t* seqStore,
U32 rep[ZSTD_REP_NUM],
ZSTD_compressionParameters const* cParams,
const void* src, size_t srcSize,
const U32 searchMethod, const U32 depth)
{
seqStore_t* seqStorePtr = &(ctx->seqStore);
const BYTE* const istart = (const BYTE*)src;
const BYTE* ip = istart;
const BYTE* anchor = istart;
const BYTE* const iend = istart + srcSize;
const BYTE* const ilimit = iend - 8;
const BYTE* const base = ctx->base + ctx->dictLimit;
const BYTE* const base = ms->base + ms->dictLimit;
U32 const maxSearches = 1 << ctx->appliedParams.cParams.searchLog;
U32 const mls = ctx->appliedParams.cParams.searchLength;
typedef size_t (*searchMax_f)(ZSTD_CCtx* zc, const BYTE* ip, const BYTE* iLimit,
size_t* offsetPtr,
U32 maxNbAttempts, U32 matchLengthSearch);
typedef size_t (*searchMax_f)(
ZSTD_matchState_t* ms, ZSTD_compressionParameters const* cParams,
const BYTE* ip, const BYTE* iLimit, size_t* offsetPtr);
searchMax_f const searchMax = searchMethod ? ZSTD_BtFindBestMatch_selectMLS : ZSTD_HcFindBestMatch_selectMLS;
U32 offset_1 = seqStorePtr->rep[0], offset_2 = seqStorePtr->rep[1], savedOffset=0;
U32 offset_1 = rep[0], offset_2 = rep[1], savedOffset=0;
/* init */
ip += (ip==base);
ctx->nextToUpdate3 = ctx->nextToUpdate;
ms->nextToUpdate3 = ms->nextToUpdate;
{ U32 const maxRep = (U32)(ip-base);
if (offset_2 > maxRep) savedOffset = offset_2, offset_2 = 0;
if (offset_1 > maxRep) savedOffset = offset_1, offset_1 = 0;
@ -524,7 +531,7 @@ size_t ZSTD_compressBlock_lazy_generic(ZSTD_CCtx* ctx,
/* first search (depth 0) */
{ size_t offsetFound = 99999999;
size_t const ml2 = searchMax(ctx, ip, iend, &offsetFound, maxSearches, mls);
size_t const ml2 = searchMax(ms, cParams, ip, iend, &offsetFound);
if (ml2 > matchLength)
matchLength = ml2, start = ip, offset=offsetFound;
}
@ -546,7 +553,7 @@ size_t ZSTD_compressBlock_lazy_generic(ZSTD_CCtx* ctx,
matchLength = mlRep, offset = 0, start = ip;
}
{ size_t offset2=99999999;
size_t const ml2 = searchMax(ctx, ip, iend, &offset2, maxSearches, mls);
size_t const ml2 = searchMax(ms, cParams, ip, iend, &offset2);
int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)offset2+1)); /* raw approx */
int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offset+1) + 4);
if ((ml2 >= 4) && (gain2 > gain1)) {
@ -565,7 +572,7 @@ size_t ZSTD_compressBlock_lazy_generic(ZSTD_CCtx* ctx,
matchLength = ml2, offset = 0, start = ip;
}
{ size_t offset2=99999999;
size_t const ml2 = searchMax(ctx, ip, iend, &offset2, maxSearches, mls);
size_t const ml2 = searchMax(ms, cParams, ip, iend, &offset2);
int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)offset2+1)); /* raw approx */
int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offset+1) + 7);
if ((ml2 >= 4) && (gain2 > gain1)) {
@ -590,7 +597,7 @@ size_t ZSTD_compressBlock_lazy_generic(ZSTD_CCtx* ctx,
/* store sequence */
_storeSequence:
{ size_t const litLength = start - anchor;
ZSTD_storeSeq(seqStorePtr, litLength, anchor, (U32)offset, matchLength-MINMATCH);
ZSTD_storeSeq(seqStore, litLength, anchor, (U32)offset, matchLength-MINMATCH);
anchor = ip = start + matchLength;
}
@ -600,73 +607,80 @@ _storeSequence:
/* store sequence */
matchLength = ZSTD_count(ip+4, ip+4-offset_2, iend) + 4;
offset = offset_2; offset_2 = offset_1; offset_1 = (U32)offset; /* swap repcodes */
ZSTD_storeSeq(seqStorePtr, 0, anchor, 0, matchLength-MINMATCH);
ZSTD_storeSeq(seqStore, 0, anchor, 0, matchLength-MINMATCH);
ip += matchLength;
anchor = ip;
continue; /* faster when present ... (?) */
} }
/* Save reps for next block */
seqStorePtr->repToConfirm[0] = offset_1 ? offset_1 : savedOffset;
seqStorePtr->repToConfirm[1] = offset_2 ? offset_2 : savedOffset;
rep[0] = offset_1 ? offset_1 : savedOffset;
rep[1] = offset_2 ? offset_2 : savedOffset;
/* Return the last literals size */
return iend - anchor;
}
size_t ZSTD_compressBlock_btlazy2(ZSTD_CCtx* ctx, const void* src, size_t srcSize)
size_t ZSTD_compressBlock_btlazy2(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize)
{
return ZSTD_compressBlock_lazy_generic(ctx, src, srcSize, 1, 2);
return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, cParams, src, srcSize, 1, 2);
}
size_t ZSTD_compressBlock_lazy2(ZSTD_CCtx* ctx, const void* src, size_t srcSize)
size_t ZSTD_compressBlock_lazy2(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize)
{
return ZSTD_compressBlock_lazy_generic(ctx, src, srcSize, 0, 2);
return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, cParams, src, srcSize, 0, 2);
}
size_t ZSTD_compressBlock_lazy(ZSTD_CCtx* ctx, const void* src, size_t srcSize)
size_t ZSTD_compressBlock_lazy(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize)
{
return ZSTD_compressBlock_lazy_generic(ctx, src, srcSize, 0, 1);
return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, cParams, src, srcSize, 0, 1);
}
size_t ZSTD_compressBlock_greedy(ZSTD_CCtx* ctx, const void* src, size_t srcSize)
size_t ZSTD_compressBlock_greedy(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize)
{
return ZSTD_compressBlock_lazy_generic(ctx, src, srcSize, 0, 0);
return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, cParams, src, srcSize, 0, 0);
}
FORCE_INLINE_TEMPLATE
size_t ZSTD_compressBlock_lazy_extDict_generic(ZSTD_CCtx* ctx,
size_t ZSTD_compressBlock_lazy_extDict_generic(
ZSTD_matchState_t* ms, seqStore_t* seqStore,
U32 rep[ZSTD_REP_NUM],
ZSTD_compressionParameters const* cParams,
const void* src, size_t srcSize,
const U32 searchMethod, const U32 depth)
{
seqStore_t* seqStorePtr = &(ctx->seqStore);
const BYTE* const istart = (const BYTE*)src;
const BYTE* ip = istart;
const BYTE* anchor = istart;
const BYTE* const iend = istart + srcSize;
const BYTE* const ilimit = iend - 8;
const BYTE* const base = ctx->base;
const U32 dictLimit = ctx->dictLimit;
const U32 lowestIndex = ctx->lowLimit;
const BYTE* const base = ms->base;
const U32 dictLimit = ms->dictLimit;
const U32 lowestIndex = ms->lowLimit;
const BYTE* const prefixStart = base + dictLimit;
const BYTE* const dictBase = ctx->dictBase;
const BYTE* const dictBase = ms->dictBase;
const BYTE* const dictEnd = dictBase + dictLimit;
const BYTE* const dictStart = dictBase + ctx->lowLimit;
const BYTE* const dictStart = dictBase + ms->lowLimit;
const U32 maxSearches = 1 << ctx->appliedParams.cParams.searchLog;
const U32 mls = ctx->appliedParams.cParams.searchLength;
typedef size_t (*searchMax_f)(ZSTD_CCtx* zc, const BYTE* ip, const BYTE* iLimit,
size_t* offsetPtr,
U32 maxNbAttempts, U32 matchLengthSearch);
typedef size_t (*searchMax_f)(
ZSTD_matchState_t* ms, ZSTD_compressionParameters const* cParams,
const BYTE* ip, const BYTE* iLimit, size_t* offsetPtr);
searchMax_f searchMax = searchMethod ? ZSTD_BtFindBestMatch_selectMLS_extDict : ZSTD_HcFindBestMatch_extDict_selectMLS;
U32 offset_1 = seqStorePtr->rep[0], offset_2 = seqStorePtr->rep[1];
U32 offset_1 = rep[0], offset_2 = rep[1];
/* init */
ctx->nextToUpdate3 = ctx->nextToUpdate;
ms->nextToUpdate3 = ms->nextToUpdate;
ip += (ip == prefixStart);
/* Match Loop */
@ -690,7 +704,7 @@ size_t ZSTD_compressBlock_lazy_extDict_generic(ZSTD_CCtx* ctx,
/* first search (depth 0) */
{ size_t offsetFound = 99999999;
size_t const ml2 = searchMax(ctx, ip, iend, &offsetFound, maxSearches, mls);
size_t const ml2 = searchMax(ms, cParams, ip, iend, &offsetFound);
if (ml2 > matchLength)
matchLength = ml2, start = ip, offset=offsetFound;
}
@ -723,7 +737,7 @@ size_t ZSTD_compressBlock_lazy_extDict_generic(ZSTD_CCtx* ctx,
/* search match, depth 1 */
{ size_t offset2=99999999;
size_t const ml2 = searchMax(ctx, ip, iend, &offset2, maxSearches, mls);
size_t const ml2 = searchMax(ms, cParams, ip, iend, &offset2);
int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)offset2+1)); /* raw approx */
int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offset+1) + 4);
if ((ml2 >= 4) && (gain2 > gain1)) {
@ -753,7 +767,7 @@ size_t ZSTD_compressBlock_lazy_extDict_generic(ZSTD_CCtx* ctx,
/* search match, depth 2 */
{ size_t offset2=99999999;
size_t const ml2 = searchMax(ctx, ip, iend, &offset2, maxSearches, mls);
size_t const ml2 = searchMax(ms, cParams, ip, iend, &offset2);
int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)offset2+1)); /* raw approx */
int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offset+1) + 7);
if ((ml2 >= 4) && (gain2 > gain1)) {
@ -775,7 +789,7 @@ size_t ZSTD_compressBlock_lazy_extDict_generic(ZSTD_CCtx* ctx,
/* store sequence */
_storeSequence:
{ size_t const litLength = start - anchor;
ZSTD_storeSeq(seqStorePtr, litLength, anchor, (U32)offset, matchLength-MINMATCH);
ZSTD_storeSeq(seqStore, litLength, anchor, (U32)offset, matchLength-MINMATCH);
anchor = ip = start + matchLength;
}
@ -790,7 +804,7 @@ _storeSequence:
const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend;
matchLength = ZSTD_count_2segments(ip+4, repMatch+4, iend, repEnd, prefixStart) + 4;
offset = offset_2; offset_2 = offset_1; offset_1 = (U32)offset; /* swap offset history */
ZSTD_storeSeq(seqStorePtr, 0, anchor, 0, matchLength-MINMATCH);
ZSTD_storeSeq(seqStore, 0, anchor, 0, matchLength-MINMATCH);
ip += matchLength;
anchor = ip;
continue; /* faster when present ... (?) */
@ -799,29 +813,41 @@ _storeSequence:
} }
/* Save reps for next block */
seqStorePtr->repToConfirm[0] = offset_1; seqStorePtr->repToConfirm[1] = offset_2;
rep[0] = offset_1;
rep[1] = offset_2;
/* Return the last literals size */
return iend - anchor;
}
size_t ZSTD_compressBlock_greedy_extDict(ZSTD_CCtx* ctx, const void* src, size_t srcSize)
size_t ZSTD_compressBlock_greedy_extDict(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize)
{
return ZSTD_compressBlock_lazy_extDict_generic(ctx, src, srcSize, 0, 0);
return ZSTD_compressBlock_lazy_extDict_generic(ms, seqStore, rep, cParams, src, srcSize, 0, 0);
}
size_t ZSTD_compressBlock_lazy_extDict(ZSTD_CCtx* ctx, const void* src, size_t srcSize)
size_t ZSTD_compressBlock_lazy_extDict(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize)
{
return ZSTD_compressBlock_lazy_extDict_generic(ctx, src, srcSize, 0, 1);
return ZSTD_compressBlock_lazy_extDict_generic(ms, seqStore, rep, cParams, src, srcSize, 0, 1);
}
size_t ZSTD_compressBlock_lazy2_extDict(ZSTD_CCtx* ctx, const void* src, size_t srcSize)
size_t ZSTD_compressBlock_lazy2_extDict(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize)
{
return ZSTD_compressBlock_lazy_extDict_generic(ctx, src, srcSize, 0, 2);
return ZSTD_compressBlock_lazy_extDict_generic(ms, seqStore, rep, cParams, src, srcSize, 0, 2);
}
size_t ZSTD_compressBlock_btlazy2_extDict(ZSTD_CCtx* ctx, const void* src, size_t srcSize)
size_t ZSTD_compressBlock_btlazy2_extDict(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize)
{
return ZSTD_compressBlock_lazy_extDict_generic(ctx, src, srcSize, 1, 2);
return ZSTD_compressBlock_lazy_extDict_generic(ms, seqStore, rep, cParams, src, srcSize, 1, 2);
}

41
lib/compress/zstd_lazy.h
View File

@ -15,22 +15,39 @@
extern "C" {
#endif
#include "mem.h" /* U32 */
#include "zstd.h" /* ZSTD_CCtx, size_t */
#include "zstd_compress_internal.h"
U32 ZSTD_insertAndFindFirstIndex(
ZSTD_matchState_t* ms, ZSTD_compressionParameters const* cParams,
const BYTE* ip);
U32 ZSTD_insertAndFindFirstIndex (ZSTD_CCtx* zc, const BYTE* ip, U32 mls); /* used in ZSTD_loadDictionaryContent() */
void ZSTD_updateDUBT(ZSTD_CCtx* zc, const BYTE* ip, const BYTE* iend, U32 mls); /* used in ZSTD_loadDictionaryContent() */
void ZSTD_preserveUnsortedMark (U32* const table, U32 const size, U32 const reducerValue); /*! used in ZSTD_reduceIndex(). pre-emptively increase value of ZSTD_DUBT_UNSORTED_MARK */
size_t ZSTD_compressBlock_btlazy2(ZSTD_CCtx* ctx, const void* src, size_t srcSize);
size_t ZSTD_compressBlock_lazy2(ZSTD_CCtx* ctx, const void* src, size_t srcSize);
size_t ZSTD_compressBlock_lazy(ZSTD_CCtx* ctx, const void* src, size_t srcSize);
size_t ZSTD_compressBlock_greedy(ZSTD_CCtx* ctx, const void* src, size_t srcSize);
size_t ZSTD_compressBlock_btlazy2(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize);
size_t ZSTD_compressBlock_lazy2(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize);
size_t ZSTD_compressBlock_lazy(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize);
size_t ZSTD_compressBlock_greedy(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize);
size_t ZSTD_compressBlock_greedy_extDict(ZSTD_CCtx* ctx, const void* src, size_t srcSize);
size_t ZSTD_compressBlock_lazy_extDict(ZSTD_CCtx* ctx, const void* src, size_t srcSize);
size_t ZSTD_compressBlock_lazy2_extDict(ZSTD_CCtx* ctx, const void* src, size_t srcSize);
size_t ZSTD_compressBlock_btlazy2_extDict(ZSTD_CCtx* ctx, const void* src, size_t srcSize);
size_t ZSTD_compressBlock_greedy_extDict(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize);
size_t ZSTD_compressBlock_lazy_extDict(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize);
size_t ZSTD_compressBlock_lazy2_extDict(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize);
size_t ZSTD_compressBlock_btlazy2_extDict(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize);
#if defined (__cplusplus)
}

183
lib/compress/zstd_ldm.c
View File

@ -205,21 +205,22 @@ static size_t ZSTD_ldm_countBackwardsMatch(
*
* The tables for the other strategies are filled within their
* block compressors. */
static size_t ZSTD_ldm_fillFastTables(ZSTD_CCtx* zc, const void* end)
static size_t ZSTD_ldm_fillFastTables(ZSTD_matchState_t* ms,
ZSTD_compressionParameters const* cParams,
void const* end)
{
const BYTE* const iend = (const BYTE*)end;
const U32 mls = zc->appliedParams.cParams.searchLength;
switch(zc->appliedParams.cParams.strategy)
switch(cParams->strategy)
{
case ZSTD_fast:
ZSTD_fillHashTable(zc, iend, mls);
zc->nextToUpdate = (U32)(iend - zc->base);
ZSTD_fillHashTable(ms, cParams, iend);
ms->nextToUpdate = (U32)(iend - ms->base);
break;
case ZSTD_dfast:
ZSTD_fillDoubleHashTable(zc, iend, mls);
zc->nextToUpdate = (U32)(iend - zc->base);
ZSTD_fillDoubleHashTable(ms, cParams, iend);
ms->nextToUpdate = (U32)(iend - ms->base);
break;
case ZSTD_greedy:
@ -268,51 +269,41 @@ static U64 ZSTD_ldm_fillLdmHashTable(ldmState_t* state,
* Sets cctx->nextToUpdate to a position corresponding closer to anchor
* if it is far way
* (after a long match, only update tables a limited amount). */
static void ZSTD_ldm_limitTableUpdate(ZSTD_CCtx* cctx, const BYTE* anchor)
static void ZSTD_ldm_limitTableUpdate(ZSTD_matchState_t* ms, const BYTE* anchor)
{
U32 const current = (U32)(anchor - cctx->base);
if (current > cctx->nextToUpdate + 1024) {
cctx->nextToUpdate =
current - MIN(512, current - cctx->nextToUpdate - 1024);
U32 const current = (U32)(anchor - ms->base);
if (current > ms->nextToUpdate + 1024) {
ms->nextToUpdate =
current - MIN(512, current - ms->nextToUpdate - 1024);
}
}
typedef size_t (*ZSTD_blockCompressor) (ZSTD_CCtx* ctx, const void* src, size_t srcSize);
/* defined in zstd_compress.c */
ZSTD_blockCompressor ZSTD_selectBlockCompressor(ZSTD_strategy strat, int extDict);
size_t ZSTD_compressBlock_ldm(
ldmState_t* ldmState, ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_CCtx_params const* params, void const* src, size_t srcSize)
FORCE_INLINE_TEMPLATE
size_t ZSTD_compressBlock_ldm_generic(ZSTD_CCtx* cctx,
const void* src, size_t srcSize)
{
ldmState_t* const ldmState = &(cctx->ldmState);
const ldmParams_t ldmParams = cctx->appliedParams.ldmParams;
ZSTD_compressionParameters const* cParams = &params->cParams;
const ldmParams_t ldmParams = params->ldmParams;
const U64 hashPower = ldmState->hashPower;
const U32 hBits = ldmParams.hashLog - ldmParams.bucketSizeLog;
const U32 ldmBucketSize = ((U32)1 << ldmParams.bucketSizeLog);
const U32 ldmTagMask = ((U32)1 << ldmParams.hashEveryLog) - 1;
seqStore_t* const seqStorePtr = &(cctx->seqStore);
const BYTE* const base = cctx->base;
const BYTE* const base = ms->base;
const BYTE* const istart = (const BYTE*)src;
const BYTE* ip = istart;
const BYTE* anchor = istart;
const U32 lowestIndex = cctx->dictLimit;
const U32 lowestIndex = ms->dictLimit;
const BYTE* const lowest = base + lowestIndex;
const BYTE* const iend = istart + srcSize;
const BYTE* const ilimit = iend - MAX(ldmParams.minMatchLength, HASH_READ_SIZE);
const ZSTD_blockCompressor blockCompressor =
ZSTD_selectBlockCompressor(cctx->appliedParams.cParams.strategy, 0);
U32* const repToConfirm = seqStorePtr->repToConfirm;
U32 savedRep[ZSTD_REP_NUM];
ZSTD_selectBlockCompressor(cParams->strategy, 0);
U64 rollingHash = 0;
const BYTE* lastHashed = NULL;
size_t i, lastLiterals;
/* Save seqStorePtr->rep and copy repToConfirm */
for (i = 0; i < ZSTD_REP_NUM; i++)
savedRep[i] = repToConfirm[i] = seqStorePtr->rep[i];
/* Main Search Loop */
while (ip < ilimit) { /* < instead of <=, because repcode check at (ip+1) */
size_t mLength;
@ -390,25 +381,21 @@ size_t ZSTD_compressBlock_ldm_generic(ZSTD_CCtx* cctx,
const BYTE* const match = base + matchIndex - backwardMatchLength;
U32 const offset = (U32)(ip - match);
/* Overwrite rep codes */
for (i = 0; i < ZSTD_REP_NUM; i++)
seqStorePtr->rep[i] = repToConfirm[i];
/* Fill tables for block compressor */
ZSTD_ldm_limitTableUpdate(cctx, anchor);
ZSTD_ldm_fillFastTables(cctx, anchor);
ZSTD_ldm_limitTableUpdate(ms, anchor);
ZSTD_ldm_fillFastTables(ms, cParams, anchor);
/* Call block compressor and get remaining literals */
lastLiterals = blockCompressor(cctx, anchor, ip - anchor);
cctx->nextToUpdate = (U32)(ip - base);
lastLiterals = blockCompressor(ms, seqStore, rep, cParams, anchor, ip - anchor);
ms->nextToUpdate = (U32)(ip - base);
/* Update repToConfirm with the new offset */
for (i = ZSTD_REP_NUM - 1; i > 0; i--)
repToConfirm[i] = repToConfirm[i-1];
repToConfirm[0] = offset;
rep[i] = rep[i-1];
rep[0] = offset;
/* Store the sequence with the leftover literals */
ZSTD_storeSeq(seqStorePtr, lastLiterals, ip - lastLiterals,
ZSTD_storeSeq(seqStore, lastLiterals, ip - lastLiterals,
offset + ZSTD_REP_MOVE, mLength - MINMATCH);
}
@ -431,19 +418,19 @@ size_t ZSTD_compressBlock_ldm_generic(ZSTD_CCtx* cctx,
anchor = ip;
/* Check immediate repcode */
while ( (ip < ilimit)
&& ( (repToConfirm[1] > 0) && (repToConfirm[1] <= (U32)(ip-lowest))
&& (MEM_read32(ip) == MEM_read32(ip - repToConfirm[1])) )) {
&& ( (rep[1] > 0) && (rep[1] <= (U32)(ip-lowest))
&& (MEM_read32(ip) == MEM_read32(ip - rep[1])) )) {
size_t const rLength = ZSTD_count(ip+4, ip+4-repToConfirm[1],
size_t const rLength = ZSTD_count(ip+4, ip+4-rep[1],
iend) + 4;
/* Swap repToConfirm[1] <=> repToConfirm[0] */
{
U32 const tmpOff = repToConfirm[1];
repToConfirm[1] = repToConfirm[0];
repToConfirm[0] = tmpOff;
U32 const tmpOff = rep[1];
rep[1] = rep[0];
rep[0] = tmpOff;
}
ZSTD_storeSeq(seqStorePtr, 0, anchor, 0, rLength-MINMATCH);
ZSTD_storeSeq(seqStore, 0, anchor, 0, rLength-MINMATCH);
/* Fill the hash table from lastHashed+1 to ip+rLength*/
if (ip + rLength < ilimit) {
@ -457,67 +444,45 @@ size_t ZSTD_compressBlock_ldm_generic(ZSTD_CCtx* cctx,
}
}
/* Overwrite rep */
for (i = 0; i < ZSTD_REP_NUM; i++)
seqStorePtr->rep[i] = repToConfirm[i];
ZSTD_ldm_limitTableUpdate(ms, anchor);
ZSTD_ldm_fillFastTables(ms, cParams, anchor);
ZSTD_ldm_limitTableUpdate(cctx, anchor);
ZSTD_ldm_fillFastTables(cctx, anchor);
lastLiterals = blockCompressor(cctx, anchor, iend - anchor);
cctx->nextToUpdate = (U32)(iend - base);
/* Restore seqStorePtr->rep */
for (i = 0; i < ZSTD_REP_NUM; i++)
seqStorePtr->rep[i] = savedRep[i];
lastLiterals = blockCompressor(ms, seqStore, rep, cParams, anchor, iend - anchor);
ms->nextToUpdate = (U32)(iend - base);
/* Return the last literals size */
return lastLiterals;
}
size_t ZSTD_compressBlock_ldm(ZSTD_CCtx* ctx,
const void* src, size_t srcSize)
size_t ZSTD_compressBlock_ldm_extDict(
ldmState_t* ldmState, ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_CCtx_params const* params, void const* src, size_t srcSize)
{
return ZSTD_compressBlock_ldm_generic(ctx, src, srcSize);
}
static size_t ZSTD_compressBlock_ldm_extDict_generic(
ZSTD_CCtx* ctx,
const void* src, size_t srcSize)
{
ldmState_t* const ldmState = &(ctx->ldmState);
const ldmParams_t ldmParams = ctx->appliedParams.ldmParams;
const ldmParams_t ldmParams = params->ldmParams;
ZSTD_compressionParameters const* cParams = &params->cParams;
const U64 hashPower = ldmState->hashPower;
const U32 hBits = ldmParams.hashLog - ldmParams.bucketSizeLog;
const U32 ldmBucketSize = ((U32)1 << ldmParams.bucketSizeLog);
const U32 ldmTagMask = ((U32)1 << ldmParams.hashEveryLog) - 1;
seqStore_t* const seqStorePtr = &(ctx->seqStore);
const BYTE* const base = ctx->base;
const BYTE* const dictBase = ctx->dictBase;
const BYTE* const base = ms->base;
const BYTE* const dictBase = ms->dictBase;
const BYTE* const istart = (const BYTE*)src;
const BYTE* ip = istart;
const BYTE* anchor = istart;
const U32 lowestIndex = ctx->lowLimit;
const U32 lowestIndex = ms->lowLimit;
const BYTE* const dictStart = dictBase + lowestIndex;
const U32 dictLimit = ctx->dictLimit;
const U32 dictLimit = ms->dictLimit;
const BYTE* const lowPrefixPtr = base + dictLimit;
const BYTE* const dictEnd = dictBase + dictLimit;
const BYTE* const iend = istart + srcSize;
const BYTE* const ilimit = iend - MAX(ldmParams.minMatchLength, HASH_READ_SIZE);
const ZSTD_blockCompressor blockCompressor =
ZSTD_selectBlockCompressor(ctx->appliedParams.cParams.strategy, 1);
U32* const repToConfirm = seqStorePtr->repToConfirm;
U32 savedRep[ZSTD_REP_NUM];
ZSTD_selectBlockCompressor(cParams->strategy, 1);
U64 rollingHash = 0;
const BYTE* lastHashed = NULL;
size_t i, lastLiterals;
/* Save seqStorePtr->rep and copy repToConfirm */
for (i = 0; i < ZSTD_REP_NUM; i++) {
savedRep[i] = repToConfirm[i] = seqStorePtr->rep[i];
}
/* Search Loop */
while (ip < ilimit) { /* < instead of <=, because (ip+1) */
size_t mLength;
@ -605,25 +570,21 @@ static size_t ZSTD_compressBlock_ldm_extDict_generic(
U32 const matchIndex = bestEntry->offset;
U32 const offset = current - matchIndex;
/* Overwrite rep codes */
for (i = 0; i < ZSTD_REP_NUM; i++)
seqStorePtr->rep[i] = repToConfirm[i];
/* Fill the hash table for the block compressor */
ZSTD_ldm_limitTableUpdate(ctx, anchor);
ZSTD_ldm_fillFastTables(ctx, anchor);
ZSTD_ldm_limitTableUpdate(ms, anchor);
ZSTD_ldm_fillFastTables(ms, cParams, anchor);
/* Call block compressor and get remaining literals */
lastLiterals = blockCompressor(ctx, anchor, ip - anchor);
ctx->nextToUpdate = (U32)(ip - base);
lastLiterals = blockCompressor(ms, seqStore, rep, cParams, anchor, ip - anchor);
ms->nextToUpdate = (U32)(ip - base);
/* Update repToConfirm with the new offset */
for (i = ZSTD_REP_NUM - 1; i > 0; i--)
repToConfirm[i] = repToConfirm[i-1];
repToConfirm[0] = offset;
rep[i] = rep[i-1];
rep[0] = offset;
/* Store the sequence with the leftover literals */
ZSTD_storeSeq(seqStorePtr, lastLiterals, ip - lastLiterals,
ZSTD_storeSeq(seqStore, lastLiterals, ip - lastLiterals,
offset + ZSTD_REP_MOVE, mLength - MINMATCH);
}
@ -647,7 +608,7 @@ static size_t ZSTD_compressBlock_ldm_extDict_generic(
/* check immediate repcode */
while (ip < ilimit) {
U32 const current2 = (U32)(ip-base);
U32 const repIndex2 = current2 - repToConfirm[1];
U32 const repIndex2 = current2 - rep[1];
const BYTE* repMatch2 = repIndex2 < dictLimit ?
dictBase + repIndex2 : base + repIndex2;
if ( (((U32)((dictLimit-1) - repIndex2) >= 3) &
@ -659,11 +620,11 @@ static size_t ZSTD_compressBlock_ldm_extDict_generic(
ZSTD_count_2segments(ip+4, repMatch2+4, iend,
repEnd2, lowPrefixPtr) + 4;
U32 tmpOffset = repToConfirm[1];
repToConfirm[1] = repToConfirm[0];
repToConfirm[0] = tmpOffset;
U32 tmpOffset = rep[1];
rep[1] = rep[0];
rep[0] = tmpOffset;
ZSTD_storeSeq(seqStorePtr, 0, anchor, 0, repLength2-MINMATCH);
ZSTD_storeSeq(seqStore, 0, anchor, 0, repLength2-MINMATCH);
/* Fill the hash table from lastHashed+1 to ip+repLength2*/
if (ip + repLength2 < ilimit) {
@ -681,27 +642,13 @@ static size_t ZSTD_compressBlock_ldm_extDict_generic(
}
}
/* Overwrite rep */
for (i = 0; i < ZSTD_REP_NUM; i++)
seqStorePtr->rep[i] = repToConfirm[i];
ZSTD_ldm_limitTableUpdate(ctx, anchor);
ZSTD_ldm_fillFastTables(ctx, anchor);
ZSTD_ldm_limitTableUpdate(ms, anchor);
ZSTD_ldm_fillFastTables(ms, cParams, anchor);
/* Call the block compressor one last time on the last literals */
lastLiterals = blockCompressor(ctx, anchor, iend - anchor);
ctx->nextToUpdate = (U32)(iend - base);
/* Restore seqStorePtr->rep */
for (i = 0; i < ZSTD_REP_NUM; i++)
seqStorePtr->rep[i] = savedRep[i];
lastLiterals = blockCompressor(ms, seqStore, rep, cParams, anchor, iend - anchor);
ms->nextToUpdate = (U32)(iend - base);
/* Return the last literals size */
return lastLiterals;
}
size_t ZSTD_compressBlock_ldm_extDict(ZSTD_CCtx* ctx,
const void* src, size_t srcSize)
{
return ZSTD_compressBlock_ldm_extDict_generic(ctx, src, srcSize);
}

10
lib/compress/zstd_ldm.h
View File

@ -37,9 +37,13 @@ extern "C" {
* parameters), which stores the matched sequences. The "long distance"
* match is then stored with the remaining literals from the
* ZSTD_blockCompressor. */
size_t ZSTD_compressBlock_ldm(ZSTD_CCtx* cctx, const void* src, size_t srcSize);
size_t ZSTD_compressBlock_ldm_extDict(ZSTD_CCtx* ctx,
const void* src, size_t srcSize);
size_t ZSTD_compressBlock_ldm(
ldmState_t* ldms, ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_CCtx_params const* params, void const* src, size_t srcSize);
size_t ZSTD_compressBlock_ldm_extDict(
ldmState_t* ldms, ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_CCtx_params const* params, void const* src, size_t srcSize);
/** ZSTD_ldm_initializeParameters() :
* Initialize the long distance matching parameters to their default values. */

163
lib/compress/zstd_opt.c
View File

@ -243,13 +243,13 @@ MEM_STATIC U32 ZSTD_readMINMATCH(const void* memPtr, U32 length)
/* Update hashTable3 up to ip (excluded)
Assumption : always within prefix (i.e. not within extDict) */
static U32 ZSTD_insertAndFindFirstIndexHash3 (ZSTD_CCtx* const cctx, const BYTE* const ip)
static U32 ZSTD_insertAndFindFirstIndexHash3 (ZSTD_matchState_t* ms, const BYTE* const ip)
{
U32* const hashTable3 = cctx->hashTable3;
U32 const hashLog3 = cctx->hashLog3;
const BYTE* const base = cctx->base;
U32 idx = cctx->nextToUpdate3;
U32 const target = cctx->nextToUpdate3 = (U32)(ip - base);
U32* const hashTable3 = ms->hashTable3;
U32 const hashLog3 = ms->hashLog3;
const BYTE* const base = ms->base;
U32 idx = ms->nextToUpdate3;
U32 const target = ms->nextToUpdate3 = (U32)(ip - base);
size_t const hash3 = ZSTD_hash3Ptr(ip, hashLog3);
while(idx < target) {
@ -267,21 +267,22 @@ static U32 ZSTD_insertAndFindFirstIndexHash3 (ZSTD_CCtx* const cctx, const BYTE*
/** ZSTD_insertBt1() : add one or multiple positions to tree.
* ip : assumed <= iend-8 .
* @return : nb of positions added */
static U32 ZSTD_insertBt1(ZSTD_CCtx* zc,
static U32 ZSTD_insertBt1(
ZSTD_matchState_t* ms, ZSTD_compressionParameters const* cParams,
const BYTE* const ip, const BYTE* const iend,
U32 nbCompares, U32 const mls, U32 const extDict)
U32 const mls, U32 const extDict)
{
U32* const hashTable = zc->hashTable;
U32 const hashLog = zc->appliedParams.cParams.hashLog;
U32* const hashTable = ms->hashTable;
U32 const hashLog = cParams->hashLog;
size_t const h = ZSTD_hashPtr(ip, hashLog, mls);
U32* const bt = zc->chainTable;
U32 const btLog = zc->appliedParams.cParams.chainLog - 1;
U32* const bt = ms->chainTable;
U32 const btLog = cParams->chainLog - 1;
U32 const btMask = (1 << btLog) - 1;
U32 matchIndex = hashTable[h];
size_t commonLengthSmaller=0, commonLengthLarger=0;
const BYTE* const base = zc->base;
const BYTE* const dictBase = zc->dictBase;
const U32 dictLimit = zc->dictLimit;
const BYTE* const base = ms->base;
const BYTE* const dictBase = ms->dictBase;
const U32 dictLimit = ms->dictLimit;
const BYTE* const dictEnd = dictBase + dictLimit;
const BYTE* const prefixStart = base + dictLimit;
const BYTE* match;
@ -290,9 +291,10 @@ static U32 ZSTD_insertBt1(ZSTD_CCtx* zc,
U32* smallerPtr = bt + 2*(current&btMask);
U32* largerPtr = smallerPtr + 1;
U32 dummy32; /* to be nullified at the end */
U32 const windowLow = zc->lowLimit;
U32 const windowLow = ms->lowLimit;
U32 matchEndIdx = current+8+1;
size_t bestLength = 8;
U32 nbCompares = 1U << cParams->searchLog;
#ifdef ZSTD_C_PREDICT
U32 predictedSmall = *(bt + 2*((current-1)&btMask) + 0);
U32 predictedLarge = *(bt + 2*((current-1)&btMask) + 1);
@ -375,66 +377,60 @@ static U32 ZSTD_insertBt1(ZSTD_CCtx* zc,
}
FORCE_INLINE_TEMPLATE
void ZSTD_updateTree_internal(ZSTD_CCtx* zc,
void ZSTD_updateTree_internal(
ZSTD_matchState_t* ms, ZSTD_compressionParameters const* cParams,
const BYTE* const ip, const BYTE* const iend,
const U32 nbCompares, const U32 mls, const U32 extDict)
const U32 mls, const U32 extDict)
{
const BYTE* const base = zc->base;
const BYTE* const base = ms->base;
U32 const target = (U32)(ip - base);
U32 idx = zc->nextToUpdate;
U32 idx = ms->nextToUpdate;
DEBUGLOG(7, "ZSTD_updateTree_internal, from %u to %u (extDict:%u)",
idx, target, extDict);
while(idx < target)
idx += ZSTD_insertBt1(zc, base+idx, iend, nbCompares, mls, extDict);
zc->nextToUpdate = target;
idx += ZSTD_insertBt1(ms, cParams, base+idx, iend, mls, extDict);
ms->nextToUpdate = target;
}
void ZSTD_updateTree(ZSTD_CCtx* zc,
const BYTE* ip, const BYTE* iend,
U32 nbCompares, U32 mls)
void ZSTD_updateTree(
ZSTD_matchState_t* ms, ZSTD_compressionParameters const* cParams,
const BYTE* ip, const BYTE* iend)
{
ZSTD_updateTree_internal(zc, ip, iend, nbCompares, mls, 0 /*extDict*/);
ZSTD_updateTree_internal(ms, cParams, ip, iend, cParams->searchLength, 0 /*extDict*/);
}
static void ZSTD_updateTree_extDict(ZSTD_CCtx* zc,
const BYTE* const ip, const BYTE* const iend,
const U32 nbCompares, const U32 mls)
{
ZSTD_updateTree_internal(zc, ip, iend, nbCompares, mls, 1 /*extDict*/);
}
FORCE_INLINE_TEMPLATE
U32 ZSTD_insertBtAndGetAllMatches (
ZSTD_CCtx* zc,
ZSTD_matchState_t* ms, ZSTD_compressionParameters const* cParams,
const BYTE* const ip, const BYTE* const iLimit, int const extDict,
U32 nbCompares, U32 const mls, U32 const sufficient_len,
U32 rep[ZSTD_REP_NUM], U32 const ll0,
ZSTD_match_t* matches, const U32 lengthToBeat)
ZSTD_match_t* matches, const U32 lengthToBeat, U32 const mls /* template */)
{
const BYTE* const base = zc->base;
U32 const sufficient_len = MIN(cParams->targetLength, ZSTD_OPT_NUM -1);
const BYTE* const base = ms->base;
U32 const current = (U32)(ip-base);
U32 const hashLog = zc->appliedParams.cParams.hashLog;
U32 const hashLog = cParams->hashLog;
U32 const minMatch = (mls==3) ? 3 : 4;
U32* const hashTable = zc->hashTable;
U32* const hashTable = ms->hashTable;
size_t const h = ZSTD_hashPtr(ip, hashLog, mls);
U32 matchIndex = hashTable[h];
U32* const bt = zc->chainTable;
U32 const btLog = zc->appliedParams.cParams.chainLog - 1;
U32* const bt = ms->chainTable;
U32 const btLog = cParams->chainLog - 1;
U32 const btMask= (1U << btLog) - 1;
size_t commonLengthSmaller=0, commonLengthLarger=0;
const BYTE* const dictBase = zc->dictBase;
U32 const dictLimit = zc->dictLimit;
const BYTE* const dictBase = ms->dictBase;
U32 const dictLimit = ms->dictLimit;
const BYTE* const dictEnd = dictBase + dictLimit;
const BYTE* const prefixStart = base + dictLimit;
U32 const btLow = btMask >= current ? 0 : current - btMask;
U32 const windowLow = zc->lowLimit;
U32 const windowLow = ms->lowLimit;
U32* smallerPtr = bt + 2*(current&btMask);
U32* largerPtr = bt + 2*(current&btMask) + 1;
U32 matchEndIdx = current+8+1; /* farthest referenced position of any match => detects repetitive patterns */
U32 dummy32; /* to be nullified at the end */
U32 mnum = 0;
U32 nbCompares = 1U << cParams->searchLog;
size_t bestLength = lengthToBeat-1;
DEBUGLOG(7, "ZSTD_insertBtAndGetAllMatches");
@ -475,7 +471,7 @@ U32 ZSTD_insertBtAndGetAllMatches (
/* HC3 match finder */
if ((mls == 3) /*static*/ && (bestLength < mls)) {
U32 const matchIndex3 = ZSTD_insertAndFindFirstIndexHash3 (zc, ip);
U32 const matchIndex3 = ZSTD_insertAndFindFirstIndexHash3(ms, ip);
if ((matchIndex3 > windowLow)
& (current - matchIndex3 < (1<<18)) /*heuristic : longer distance likely too expensive*/ ) {
size_t mlen;
@ -499,7 +495,7 @@ U32 ZSTD_insertBtAndGetAllMatches (
mnum = 1;
if ( (mlen > sufficient_len) |
(ip+mlen == iLimit) ) { /* best possible length */
zc->nextToUpdate = current+1; /* skip insertion */
ms->nextToUpdate = current+1; /* skip insertion */
return 1;
} } } }
@ -556,30 +552,29 @@ U32 ZSTD_insertBtAndGetAllMatches (
*smallerPtr = *largerPtr = 0;
assert(matchEndIdx > current+8);
zc->nextToUpdate = matchEndIdx - 8; /* skip repetitive patterns */
ms->nextToUpdate = matchEndIdx - 8; /* skip repetitive patterns */
return mnum;
}
FORCE_INLINE_TEMPLATE U32 ZSTD_BtGetAllMatches (
ZSTD_CCtx* zc, /* Index table will be updated */
ZSTD_matchState_t* ms, ZSTD_compressionParameters const* cParams,
const BYTE* ip, const BYTE* const iHighLimit, int const extDict,
U32 const maxNbAttempts, U32 const matchLengthSearch, U32 const sufficient_len,
U32 rep[ZSTD_REP_NUM], U32 const ll0,
ZSTD_match_t* matches, U32 const lengthToBeat)
{
U32 const matchLengthSearch = cParams->searchLength;
DEBUGLOG(7, "ZSTD_BtGetAllMatches");
if (ip < zc->base + zc->nextToUpdate) return 0; /* skipped area */
if (extDict) ZSTD_updateTree_extDict(zc, ip, iHighLimit, maxNbAttempts, matchLengthSearch);
else ZSTD_updateTree(zc, ip, iHighLimit, maxNbAttempts, matchLengthSearch);
if (ip < ms->base + ms->nextToUpdate) return 0; /* skipped area */
ZSTD_updateTree_internal(ms, cParams, ip, iHighLimit, matchLengthSearch, extDict);
switch(matchLengthSearch)
{
case 3 : return ZSTD_insertBtAndGetAllMatches(zc, ip, iHighLimit, extDict, maxNbAttempts, 3, sufficient_len, rep, ll0, matches, lengthToBeat);
case 3 : return ZSTD_insertBtAndGetAllMatches(ms, cParams, ip, iHighLimit, extDict, rep, ll0, matches, lengthToBeat, 3);
default :
case 4 : return ZSTD_insertBtAndGetAllMatches(zc, ip, iHighLimit, extDict, maxNbAttempts, 4, sufficient_len, rep, ll0, matches, lengthToBeat);
case 5 : return ZSTD_insertBtAndGetAllMatches(zc, ip, iHighLimit, extDict, maxNbAttempts, 5, sufficient_len, rep, ll0, matches, lengthToBeat);
case 4 : return ZSTD_insertBtAndGetAllMatches(ms, cParams, ip, iHighLimit, extDict, rep, ll0, matches, lengthToBeat, 4);
case 5 : return ZSTD_insertBtAndGetAllMatches(ms, cParams, ip, iHighLimit, extDict, rep, ll0, matches, lengthToBeat, 5);
case 7 :
case 6 : return ZSTD_insertBtAndGetAllMatches(zc, ip, iHighLimit, extDict, maxNbAttempts, 6, sufficient_len, rep, ll0, matches, lengthToBeat);
case 6 : return ZSTD_insertBtAndGetAllMatches(ms, cParams, ip, iHighLimit, extDict, rep, ll0, matches, lengthToBeat, 6);
}
}
@ -667,36 +662,33 @@ static int ZSTD_literalsContribution_cached(
}
FORCE_INLINE_TEMPLATE
size_t ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx,
size_t ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms,seqStore_t* seqStore,
U32 rep[ZSTD_REP_NUM],
ZSTD_compressionParameters const* cParams,
const void* src, size_t srcSize,
const int optLevel, const int extDict)
{
seqStore_t* const seqStorePtr = &(ctx->seqStore);
optState_t* const optStatePtr = &(ctx->optState);
optState_t* const optStatePtr = &ms->opt;
const BYTE* const istart = (const BYTE*)src;
const BYTE* ip = istart;
const BYTE* anchor = istart;
const BYTE* const iend = istart + srcSize;
const BYTE* const ilimit = iend - 8;
const BYTE* const base = ctx->base;
const BYTE* const prefixStart = base + ctx->dictLimit;
const BYTE* const base = ms->base;
const BYTE* const prefixStart = base + ms->dictLimit;
U32 const maxSearches = 1U << ctx->appliedParams.cParams.searchLog;
U32 const sufficient_len = MIN(ctx->appliedParams.cParams.targetLength, ZSTD_OPT_NUM -1);
U32 const mls = ctx->appliedParams.cParams.searchLength;
U32 const minMatch = (ctx->appliedParams.cParams.searchLength == 3) ? 3 : 4;
U32 const sufficient_len = MIN(cParams->targetLength, ZSTD_OPT_NUM -1);
U32 const minMatch = (cParams->searchLength == 3) ? 3 : 4;
ZSTD_optimal_t* const opt = optStatePtr->priceTable;
ZSTD_match_t* const matches = optStatePtr->matchTable;
cachedLiteralPrice_t cachedLitPrice;
U32 rep[ZSTD_REP_NUM];
/* init */
DEBUGLOG(5, "ZSTD_compressBlock_opt_generic");
ctx->nextToUpdate3 = ctx->nextToUpdate;
ms->nextToUpdate3 = ms->nextToUpdate;
ZSTD_rescaleFreqs(optStatePtr, (const BYTE*)src, srcSize);
ip += (ip==prefixStart);
{ int i; for (i=0; i<ZSTD_REP_NUM; i++) rep[i]=seqStorePtr->rep[i]; }
memset(&cachedLitPrice, 0, sizeof(cachedLitPrice));
/* Match Loop */
@ -707,7 +699,7 @@ size_t ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx,
/* find first match */
{ U32 const litlen = (U32)(ip - anchor);
U32 const ll0 = !litlen;
U32 const nbMatches = ZSTD_BtGetAllMatches(ctx, ip, iend, extDict, maxSearches, mls, sufficient_len, rep, ll0, matches, minMatch);
U32 const nbMatches = ZSTD_BtGetAllMatches(ms, cParams, ip, iend, extDict, rep, ll0, matches, minMatch);
if (!nbMatches) { ip++; continue; }
/* initialize opt[0] */
@ -793,7 +785,7 @@ size_t ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx,
U32 const litlen = (opt[cur].mlen == 1) ? opt[cur].litlen : 0;
U32 const previousPrice = (cur > litlen) ? opt[cur-litlen].price : 0;
U32 const basePrice = previousPrice + ZSTD_fullLiteralsCost(inr-litlen, litlen, optStatePtr);
U32 const nbMatches = ZSTD_BtGetAllMatches(ctx, inr, iend, extDict, maxSearches, mls, sufficient_len, opt[cur].rep, ll0, matches, minMatch);
U32 const nbMatches = ZSTD_BtGetAllMatches(ms, cParams, inr, iend, extDict, opt[cur].rep, ll0, matches, minMatch);
U32 matchNb;
if (!nbMatches) continue;
@ -889,37 +881,42 @@ _shortestPath: /* cur, last_pos, best_mlen, best_off have to be set */
}
ZSTD_updateStats(optStatePtr, llen, anchor, offset, mlen);
ZSTD_storeSeq(seqStorePtr, llen, anchor, offset, mlen-MINMATCH);
ZSTD_storeSeq(seqStore, llen, anchor, offset, mlen-MINMATCH);
anchor = ip;
} }
ZSTD_setLog2Prices(optStatePtr);
} /* while (ip < ilimit) */
/* Save reps for next block */
{ int i; for (i=0; i<ZSTD_REP_NUM; i++) seqStorePtr->repToConfirm[i] = rep[i]; }
/* Return the last literals size */
return iend - anchor;
}
size_t ZSTD_compressBlock_btopt(ZSTD_CCtx* ctx, const void* src, size_t srcSize)
size_t ZSTD_compressBlock_btopt(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize)
{
DEBUGLOG(5, "ZSTD_compressBlock_btopt");
return ZSTD_compressBlock_opt_generic(ctx, src, srcSize, 0 /*optLevel*/, 0 /*extDict*/);
return ZSTD_compressBlock_opt_generic(ms, seqStore, rep, cParams, src, srcSize, 0 /*optLevel*/, 0 /*extDict*/);
}
size_t ZSTD_compressBlock_btultra(ZSTD_CCtx* ctx, const void* src, size_t srcSize)
size_t ZSTD_compressBlock_btultra(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize)
{
return ZSTD_compressBlock_opt_generic(ctx, src, srcSize, 2 /*optLevel*/, 0 /*extDict*/);
return ZSTD_compressBlock_opt_generic(ms, seqStore, rep, cParams, src, srcSize, 2 /*optLevel*/, 0 /*extDict*/);
}
size_t ZSTD_compressBlock_btopt_extDict(ZSTD_CCtx* ctx, const void* src, size_t srcSize)
size_t ZSTD_compressBlock_btopt_extDict(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize)
{
return ZSTD_compressBlock_opt_generic(ctx, src, srcSize, 0 /*optLevel*/, 1 /*extDict*/);
return ZSTD_compressBlock_opt_generic(ms, seqStore, rep, cParams, src, srcSize, 0 /*optLevel*/, 1 /*extDict*/);
}
size_t ZSTD_compressBlock_btultra_extDict(ZSTD_CCtx* ctx, const void* src, size_t srcSize)
size_t ZSTD_compressBlock_btultra_extDict(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize)
{
return ZSTD_compressBlock_opt_generic(ctx, src, srcSize, 2 /*optLevel*/, 1 /*extDict*/);
return ZSTD_compressBlock_opt_generic(ms, seqStore, rep, cParams, src, srcSize, 2 /*optLevel*/, 1 /*extDict*/);
}

23
lib/compress/zstd_opt.h
View File

@ -15,16 +15,25 @@
extern "C" {
#endif
#include "mem.h" /* U32 */
#include "zstd.h" /* ZSTD_CCtx, size_t */
#include "zstd_compress_internal.h"
void ZSTD_updateTree(ZSTD_CCtx* ctx, const BYTE* ip, const BYTE* iend, U32 nbCompares, U32 mls); /* used in ZSTD_loadDictionaryContent() */
void ZSTD_updateTree(
ZSTD_matchState_t* ms, ZSTD_compressionParameters const* cParams,
const BYTE* ip, const BYTE* iend); /* used in ZSTD_loadDictionaryContent() */
size_t ZSTD_compressBlock_btopt(ZSTD_CCtx* ctx, const void* src, size_t srcSize);
size_t ZSTD_compressBlock_btultra(ZSTD_CCtx* ctx, const void* src, size_t srcSize);
size_t ZSTD_compressBlock_btopt(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize);
size_t ZSTD_compressBlock_btultra(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize);
size_t ZSTD_compressBlock_btopt_extDict(ZSTD_CCtx* ctx, const void* src, size_t srcSize);
size_t ZSTD_compressBlock_btultra_extDict(ZSTD_CCtx* ctx, const void* src, size_t srcSize);
size_t ZSTD_compressBlock_btopt_extDict(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize);
size_t ZSTD_compressBlock_btultra_extDict(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize);
#if defined (__cplusplus)
}