facebook
/
zstd
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Disambiguate 'workspace' and 'entropyWorkspace'

dev
W. Felix Handte 2019-07-15 17:14:51 -04:00
parent ccaac852e8
commit c25283cf00
5 changed files with 53 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

59
lib/compress/zstd_compress.c
View File

@ -1960,7 +1960,7 @@ ZSTD_compressSequences_internal(seqStore_t* seqStorePtr,
ZSTD_entropyCTables_t* nextEntropy, ZSTD_entropyCTables_t* nextEntropy,
const ZSTD_CCtx_params* cctxParams, const ZSTD_CCtx_params* cctxParams,
void* dst, size_t dstCapacity, void* dst, size_t dstCapacity,
void* workspace, size_t wkspSize, void* entropyWorkspace, size_t entropyWkspSize,
const int bmi2) const int bmi2)
{ {
const int longOffsets = cctxParams->cParams.windowLog > STREAM_ACCUMULATOR_MIN; const int longOffsets = cctxParams->cParams.windowLog > STREAM_ACCUMULATOR_MIN;
@ -1993,7 +1993,7 @@ ZSTD_compressSequences_internal(seqStore_t* seqStorePtr,
ZSTD_disableLiteralsCompression(cctxParams), ZSTD_disableLiteralsCompression(cctxParams),
op, dstCapacity, op, dstCapacity,
literals, litSize, literals, litSize,
workspace, wkspSize, entropyWorkspace, entropyWkspSize,
bmi2); bmi2);
FORWARD_IF_ERROR(cSize); FORWARD_IF_ERROR(cSize);
assert(cSize <= dstCapacity); assert(cSize <= dstCapacity);
@ -2029,7 +2029,7 @@ ZSTD_compressSequences_internal(seqStore_t* seqStorePtr,
ZSTD_seqToCodes(seqStorePtr); ZSTD_seqToCodes(seqStorePtr);
/* build CTable for Literal Lengths */ /* build CTable for Literal Lengths */
{ unsigned max = MaxLL; { unsigned max = MaxLL;
size_t const mostFrequent = HIST_countFast_wksp(count, &max, llCodeTable, nbSeq, workspace, wkspSize); /* can't fail */ size_t const mostFrequent = HIST_countFast_wksp(count, &max, llCodeTable, nbSeq, entropyWorkspace, entropyWkspSize); /* can't fail */
DEBUGLOG(5, "Building LL table"); DEBUGLOG(5, "Building LL table");
nextEntropy->fse.litlength_repeatMode = prevEntropy->fse.litlength_repeatMode; nextEntropy->fse.litlength_repeatMode = prevEntropy->fse.litlength_repeatMode;
LLtype = ZSTD_selectEncodingType(&nextEntropy->fse.litlength_repeatMode, LLtype = ZSTD_selectEncodingType(&nextEntropy->fse.litlength_repeatMode,
@ -2039,10 +2039,14 @@ ZSTD_compressSequences_internal(seqStore_t* seqStorePtr,
ZSTD_defaultAllowed, strategy); ZSTD_defaultAllowed, strategy);
assert(set_basic < set_compressed && set_rle < set_compressed); assert(set_basic < set_compressed && set_rle < set_compressed);
assert(!(LLtype < set_compressed && nextEntropy->fse.litlength_repeatMode != FSE_repeat_none)); /* We don't copy tables */ assert(!(LLtype < set_compressed && nextEntropy->fse.litlength_repeatMode != FSE_repeat_none)); /* We don't copy tables */
{ size_t const countSize = ZSTD_buildCTable(op, (size_t)(oend - op), CTable_LitLength, LLFSELog, (symbolEncodingType_e)LLtype, { size_t const countSize = ZSTD_buildCTable(
count, max, llCodeTable, nbSeq, LL_defaultNorm, LL_defaultNormLog, MaxLL, op, (size_t)(oend - op),
prevEntropy->fse.litlengthCTable, sizeof(prevEntropy->fse.litlengthCTable), CTable_LitLength, LLFSELog, (symbolEncodingType_e)LLtype,
workspace, wkspSize); count, max, llCodeTable, nbSeq,
LL_defaultNorm, LL_defaultNormLog, MaxLL,
prevEntropy->fse.litlengthCTable,
sizeof(prevEntropy->fse.litlengthCTable),
entropyWorkspace, entropyWkspSize);
FORWARD_IF_ERROR(countSize); FORWARD_IF_ERROR(countSize);
if (LLtype == set_compressed) if (LLtype == set_compressed)
lastNCount = op; lastNCount = op;
@ -2051,7 +2055,8 @@ ZSTD_compressSequences_internal(seqStore_t* seqStorePtr,
} } } }
/* build CTable for Offsets */ /* build CTable for Offsets */
{ unsigned max = MaxOff; { unsigned max = MaxOff;
size_t const mostFrequent = HIST_countFast_wksp(count, &max, ofCodeTable, nbSeq, workspace, wkspSize); /* can't fail */ size_t const mostFrequent = HIST_countFast_wksp(
count, &max, ofCodeTable, nbSeq, entropyWorkspace, entropyWkspSize); /* can't fail */
/* We can only use the basic table if max <= DefaultMaxOff, otherwise the offsets are too large */ /* We can only use the basic table if max <= DefaultMaxOff, otherwise the offsets are too large */
ZSTD_defaultPolicy_e const defaultPolicy = (max <= DefaultMaxOff) ? ZSTD_defaultAllowed : ZSTD_defaultDisallowed; ZSTD_defaultPolicy_e const defaultPolicy = (max <= DefaultMaxOff) ? ZSTD_defaultAllowed : ZSTD_defaultDisallowed;
DEBUGLOG(5, "Building OF table"); DEBUGLOG(5, "Building OF table");
@ -2062,10 +2067,14 @@ ZSTD_compressSequences_internal(seqStore_t* seqStorePtr,
OF_defaultNorm, OF_defaultNormLog, OF_defaultNorm, OF_defaultNormLog,
defaultPolicy, strategy); defaultPolicy, strategy);
assert(!(Offtype < set_compressed && nextEntropy->fse.offcode_repeatMode != FSE_repeat_none)); /* We don't copy tables */ assert(!(Offtype < set_compressed && nextEntropy->fse.offcode_repeatMode != FSE_repeat_none)); /* We don't copy tables */
{ size_t const countSize = ZSTD_buildCTable(op, (size_t)(oend - op), CTable_OffsetBits, OffFSELog, (symbolEncodingType_e)Offtype, { size_t const countSize = ZSTD_buildCTable(
count, max, ofCodeTable, nbSeq, OF_defaultNorm, OF_defaultNormLog, DefaultMaxOff, op, (size_t)(oend - op),
prevEntropy->fse.offcodeCTable, sizeof(prevEntropy->fse.offcodeCTable), CTable_OffsetBits, OffFSELog, (symbolEncodingType_e)Offtype,
workspace, wkspSize); count, max, ofCodeTable, nbSeq,
OF_defaultNorm, OF_defaultNormLog, DefaultMaxOff,
prevEntropy->fse.offcodeCTable,
sizeof(prevEntropy->fse.offcodeCTable),
entropyWorkspace, entropyWkspSize);
FORWARD_IF_ERROR(countSize); FORWARD_IF_ERROR(countSize);
if (Offtype == set_compressed) if (Offtype == set_compressed)
lastNCount = op; lastNCount = op;
@ -2074,7 +2083,8 @@ ZSTD_compressSequences_internal(seqStore_t* seqStorePtr,
} } } }
/* build CTable for MatchLengths */ /* build CTable for MatchLengths */
{ unsigned max = MaxML; { unsigned max = MaxML;
size_t const mostFrequent = HIST_countFast_wksp(count, &max, mlCodeTable, nbSeq, workspace, wkspSize); /* can't fail */ size_t const mostFrequent = HIST_countFast_wksp(
count, &max, mlCodeTable, nbSeq, entropyWorkspace, entropyWkspSize); /* can't fail */
DEBUGLOG(5, "Building ML table (remaining space : %i)", (int)(oend-op)); DEBUGLOG(5, "Building ML table (remaining space : %i)", (int)(oend-op));
nextEntropy->fse.matchlength_repeatMode = prevEntropy->fse.matchlength_repeatMode; nextEntropy->fse.matchlength_repeatMode = prevEntropy->fse.matchlength_repeatMode;
MLtype = ZSTD_selectEncodingType(&nextEntropy->fse.matchlength_repeatMode, MLtype = ZSTD_selectEncodingType(&nextEntropy->fse.matchlength_repeatMode,
@ -2083,10 +2093,14 @@ ZSTD_compressSequences_internal(seqStore_t* seqStorePtr,
ML_defaultNorm, ML_defaultNormLog, ML_defaultNorm, ML_defaultNormLog,
ZSTD_defaultAllowed, strategy); ZSTD_defaultAllowed, strategy);
assert(!(MLtype < set_compressed && nextEntropy->fse.matchlength_repeatMode != FSE_repeat_none)); /* We don't copy tables */ assert(!(MLtype < set_compressed && nextEntropy->fse.matchlength_repeatMode != FSE_repeat_none)); /* We don't copy tables */
{ size_t const countSize = ZSTD_buildCTable(op, (size_t)(oend - op), CTable_MatchLength, MLFSELog, (symbolEncodingType_e)MLtype, { size_t const countSize = ZSTD_buildCTable(
count, max, mlCodeTable, nbSeq, ML_defaultNorm, ML_defaultNormLog, MaxML, op, (size_t)(oend - op),
prevEntropy->fse.matchlengthCTable, sizeof(prevEntropy->fse.matchlengthCTable), CTable_MatchLength, MLFSELog, (symbolEncodingType_e)MLtype,
workspace, wkspSize); count, max, mlCodeTable, nbSeq,
ML_defaultNorm, ML_defaultNormLog, MaxML,
prevEntropy->fse.matchlengthCTable,
sizeof(prevEntropy->fse.matchlengthCTable),
entropyWorkspace, entropyWkspSize);
FORWARD_IF_ERROR(countSize); FORWARD_IF_ERROR(countSize);
if (MLtype == set_compressed) if (MLtype == set_compressed)
lastNCount = op; lastNCount = op;
@ -2134,13 +2148,13 @@ ZSTD_compressSequences(seqStore_t* seqStorePtr,
const ZSTD_CCtx_params* cctxParams, const ZSTD_CCtx_params* cctxParams,
void* dst, size_t dstCapacity, void* dst, size_t dstCapacity,
size_t srcSize, size_t srcSize,
void* workspace, size_t wkspSize, void* entropyWorkspace, size_t entropyWkspSize,
int bmi2) int bmi2)
{ {
size_t const cSize = ZSTD_compressSequences_internal( size_t const cSize = ZSTD_compressSequences_internal(
seqStorePtr, prevEntropy, nextEntropy, cctxParams, seqStorePtr, prevEntropy, nextEntropy, cctxParams,
dst, dstCapacity, dst, dstCapacity,
workspace, wkspSize, bmi2); entropyWorkspace, entropyWkspSize, bmi2);
if (cSize == 0) return 0; if (cSize == 0) return 0;
/* When srcSize <= dstCapacity, there is enough space to write a raw uncompressed block. /* When srcSize <= dstCapacity, there is enough space to write a raw uncompressed block.
* Since we ran out of space, block must be not compressible, so fall back to raw uncompressed block. * Since we ran out of space, block must be not compressible, so fall back to raw uncompressed block.
@ -2380,6 +2394,7 @@ static void ZSTD_overflowCorrectIfNeeded(ZSTD_matchState_t* ms, ZSTD_CCtx_params
} }
} }
/*! ZSTD_compress_frameChunk() : /*! ZSTD_compress_frameChunk() :
* Compress a chunk of data into one or multiple blocks. * Compress a chunk of data into one or multiple blocks.
* All blocks will be terminated, all input will be consumed. * All blocks will be terminated, all input will be consumed.
@ -2848,7 +2863,8 @@ static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* cctx,
ZSTDcrp_continue, zbuff) ); ZSTDcrp_continue, zbuff) );
{ size_t const dictID = ZSTD_compress_insertDictionary( { size_t const dictID = ZSTD_compress_insertDictionary(
cctx->blockState.prevCBlock, &cctx->blockState.matchState, cctx->blockState.prevCBlock, &cctx->blockState.matchState,
params, dict, dictSize, dictContentType, dtlm, cctx->entropyWorkspace); params, dict, dictSize, dictContentType, dtlm,
cctx->entropyWorkspace);
FORWARD_IF_ERROR(dictID); FORWARD_IF_ERROR(dictID);
assert(dictID <= UINT_MAX); assert(dictID <= UINT_MAX);
cctx->dictID = (U32)dictID; cctx->dictID = (U32)dictID;
@ -3393,7 +3409,8 @@ size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pss)
* Assumption 2 : either dict, or cdict, is defined, not both */ * Assumption 2 : either dict, or cdict, is defined, not both */
size_t ZSTD_initCStream_internal(ZSTD_CStream* zcs, size_t ZSTD_initCStream_internal(ZSTD_CStream* zcs,
const void* dict, size_t dictSize, const ZSTD_CDict* cdict, const void* dict, size_t dictSize, const ZSTD_CDict* cdict,
const ZSTD_CCtx_params* params, unsigned long long pledgedSrcSize) const ZSTD_CCtx_params* params,
unsigned long long pledgedSrcSize)
{ {
DEBUGLOG(4, "ZSTD_initCStream_internal"); DEBUGLOG(4, "ZSTD_initCStream_internal");
FORWARD_IF_ERROR( ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only) ); FORWARD_IF_ERROR( ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only) );

15
lib/compress/zstd_compress_literals.c
View File

@ -70,7 +70,7 @@ size_t ZSTD_compressLiterals (ZSTD_hufCTables_t const* prevHuf,
ZSTD_strategy strategy, int disableLiteralCompression, ZSTD_strategy strategy, int disableLiteralCompression,
void* dst, size_t dstCapacity, void* dst, size_t dstCapacity,
const void* src, size_t srcSize, const void* src, size_t srcSize,
void* workspace, size_t wkspSize, void* entropyWorkspace, size_t entropyWorkspaceSize,
const int bmi2) const int bmi2)
{ {
size_t const minGain = ZSTD_minGain(srcSize, strategy); size_t const minGain = ZSTD_minGain(srcSize, strategy);
@ -99,10 +99,15 @@ size_t ZSTD_compressLiterals (ZSTD_hufCTables_t const* prevHuf,
{ HUF_repeat repeat = prevHuf->repeatMode; { HUF_repeat repeat = prevHuf->repeatMode;
int const preferRepeat = strategy < ZSTD_lazy ? srcSize <= 1024 : 0; int const preferRepeat = strategy < ZSTD_lazy ? srcSize <= 1024 : 0;
if (repeat == HUF_repeat_valid && lhSize == 3) singleStream = 1; if (repeat == HUF_repeat_valid && lhSize == 3) singleStream = 1;
cLitSize = singleStream ? HUF_compress1X_repeat(ostart+lhSize, dstCapacity-lhSize, src, srcSize, 255, 11, cLitSize = singleStream ?
workspace, wkspSize, (HUF_CElt*)nextHuf->CTable, &repeat, preferRepeat, bmi2) HUF_compress1X_repeat(
: HUF_compress4X_repeat(ostart+lhSize, dstCapacity-lhSize, src, srcSize, 255, 11, ostart+lhSize, dstCapacity-lhSize, src, srcSize,
workspace, wkspSize, (HUF_CElt*)nextHuf->CTable, &repeat, preferRepeat, bmi2); 255, 11, entropyWorkspace, entropyWorkspaceSize,
(HUF_CElt*)nextHuf->CTable, &repeat, preferRepeat, bmi2) :
HUF_compress4X_repeat(
ostart+lhSize, dstCapacity-lhSize, src, srcSize,
255, 11, entropyWorkspace, entropyWorkspaceSize,
(HUF_CElt*)nextHuf->CTable, &repeat, preferRepeat, bmi2);
if (repeat != HUF_repeat_none) { if (repeat != HUF_repeat_none) {
/* reused the existing table */ /* reused the existing table */
hType = set_repeat; hType = set_repeat;

2
lib/compress/zstd_compress_literals.h
View File

@ -23,7 +23,7 @@ size_t ZSTD_compressLiterals (ZSTD_hufCTables_t const* prevHuf,
ZSTD_strategy strategy, int disableLiteralCompression, ZSTD_strategy strategy, int disableLiteralCompression,
void* dst, size_t dstCapacity, void* dst, size_t dstCapacity,
const void* src, size_t srcSize, const void* src, size_t srcSize,
void* workspace, size_t wkspSize, void* entropyWorkspace, size_t entropyWorkspaceSize,
const int bmi2); const int bmi2);
#endif /* ZSTD_COMPRESS_LITERALS_H */ #endif /* ZSTD_COMPRESS_LITERALS_H */

6
lib/compress/zstd_compress_sequences.c
View File

@ -222,7 +222,7 @@ ZSTD_buildCTable(void* dst, size_t dstCapacity,
const BYTE* codeTable, size_t nbSeq, const BYTE* codeTable, size_t nbSeq,
const S16* defaultNorm, U32 defaultNormLog, U32 defaultMax, const S16* defaultNorm, U32 defaultNormLog, U32 defaultMax,
const FSE_CTable* prevCTable, size_t prevCTableSize, const FSE_CTable* prevCTable, size_t prevCTableSize,
void* workspace, size_t workspaceSize) void* entropyWorkspace, size_t entropyWorkspaceSize)
{ {
BYTE* op = (BYTE*)dst; BYTE* op = (BYTE*)dst;
const BYTE* const oend = op + dstCapacity; const BYTE* const oend = op + dstCapacity;
@ -238,7 +238,7 @@ ZSTD_buildCTable(void* dst, size_t dstCapacity,
memcpy(nextCTable, prevCTable, prevCTableSize); memcpy(nextCTable, prevCTable, prevCTableSize);
return 0; return 0;
case set_basic: case set_basic:
FORWARD_IF_ERROR(FSE_buildCTable_wksp(nextCTable, defaultNorm, defaultMax, defaultNormLog, workspace, workspaceSize)); /* note : could be pre-calculated */ FORWARD_IF_ERROR(FSE_buildCTable_wksp(nextCTable, defaultNorm, defaultMax, defaultNormLog, entropyWorkspace, entropyWorkspaceSize)); /* note : could be pre-calculated */
return 0; return 0;
case set_compressed: { case set_compressed: {
S16 norm[MaxSeq + 1]; S16 norm[MaxSeq + 1];
@ -252,7 +252,7 @@ ZSTD_buildCTable(void* dst, size_t dstCapacity,
FORWARD_IF_ERROR(FSE_normalizeCount(norm, tableLog, count, nbSeq_1, max)); FORWARD_IF_ERROR(FSE_normalizeCount(norm, tableLog, count, nbSeq_1, max));
{ size_t const NCountSize = FSE_writeNCount(op, oend - op, norm, max, tableLog); /* overflow protected */ { size_t const NCountSize = FSE_writeNCount(op, oend - op, norm, max, tableLog); /* overflow protected */
FORWARD_IF_ERROR(NCountSize); FORWARD_IF_ERROR(NCountSize);
FORWARD_IF_ERROR(FSE_buildCTable_wksp(nextCTable, norm, max, tableLog, workspace, workspaceSize)); FORWARD_IF_ERROR(FSE_buildCTable_wksp(nextCTable, norm, max, tableLog, entropyWorkspace, entropyWorkspaceSize));
return NCountSize; return NCountSize;
} }
} }

2
lib/compress/zstd_compress_sequences.h
View File

@ -35,7 +35,7 @@ ZSTD_buildCTable(void* dst, size_t dstCapacity,
const BYTE* codeTable, size_t nbSeq, const BYTE* codeTable, size_t nbSeq,
const S16* defaultNorm, U32 defaultNormLog, U32 defaultMax, const S16* defaultNorm, U32 defaultNormLog, U32 defaultMax,
const FSE_CTable* prevCTable, size_t prevCTableSize, const FSE_CTable* prevCTable, size_t prevCTableSize,
void* workspace, size_t workspaceSize); void* entropyWorkspace, size_t entropyWorkspaceSize);
size_t ZSTD_encodeSequences( size_t ZSTD_encodeSequences(
void* dst, size_t dstCapacity, void* dst, size_t dstCapacity,