From ccaac852e83b311881076d24048cada5944bdb22 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Mon, 15 Jul 2019 15:10:33 -0400 Subject: [PATCH 01/33] Normalize Case 'workSpace' -> 'workspace' --- lib/compress/zstd_compress.c | 56 +++++++++++++-------------- lib/compress/zstd_compress_internal.h | 6 +-- 2 files changed, 31 insertions(+), 31 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 43cfedc9..61dbebac 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -89,13 +89,13 @@ ZSTD_CCtx* ZSTD_initStaticCCtx(void *workspace, size_t workspaceSize) if ((size_t)workspace & 7) return NULL; /* must be 8-aligned */ memset(workspace, 0, workspaceSize); /* may be a bit generous, could memset be smaller ? */ cctx->staticSize = workspaceSize; - cctx->workSpace = (void*)(cctx+1); - cctx->workSpaceSize = workspaceSize - sizeof(ZSTD_CCtx); + cctx->workspace = (void*)(cctx+1); + cctx->workspaceSize = workspaceSize - sizeof(ZSTD_CCtx); /* statically sized space. entropyWorkspace never moves (but prev/next block swap places) */ - if (cctx->workSpaceSize < HUF_WORKSPACE_SIZE + 2 * sizeof(ZSTD_compressedBlockState_t)) return NULL; - assert(((size_t)cctx->workSpace & (sizeof(void*)-1)) == 0); /* ensure correct alignment */ - cctx->blockState.prevCBlock = (ZSTD_compressedBlockState_t*)cctx->workSpace; + if (cctx->workspaceSize < HUF_WORKSPACE_SIZE + 2 * sizeof(ZSTD_compressedBlockState_t)) return NULL; + assert(((size_t)cctx->workspace & (sizeof(void*)-1)) == 0); /* ensure correct alignment */ + cctx->blockState.prevCBlock = (ZSTD_compressedBlockState_t*)cctx->workspace; cctx->blockState.nextCBlock = cctx->blockState.prevCBlock + 1; { void* const ptr = cctx->blockState.nextCBlock + 1; @@ -128,7 +128,7 @@ static void ZSTD_freeCCtxContent(ZSTD_CCtx* cctx) { assert(cctx != NULL); assert(cctx->staticSize == 0); - ZSTD_free(cctx->workSpace, cctx->customMem); cctx->workSpace = NULL; + ZSTD_free(cctx->workspace, cctx->customMem); cctx->workspace = NULL; ZSTD_clearAllDicts(cctx); #ifdef ZSTD_MULTITHREAD ZSTDMT_freeCCtx(cctx->mtctx); cctx->mtctx = NULL; @@ -160,7 +160,7 @@ static size_t ZSTD_sizeof_mtctx(const ZSTD_CCtx* cctx) size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx) { if (cctx==NULL) return 0; /* support sizeof on NULL */ - return sizeof(*cctx) + cctx->workSpaceSize + return sizeof(*cctx) + cctx->workspaceSize + ZSTD_sizeof_localDict(cctx->localDict) + ZSTD_sizeof_mtctx(cctx); } @@ -1101,7 +1101,7 @@ size_t ZSTD_estimateCCtxSize_usingCCtxParams(const ZSTD_CCtx_params* params) matchStateSize + ldmSpace + ldmSeqSpace; DEBUGLOG(5, "sizeof(ZSTD_CCtx) : %u", (U32)sizeof(ZSTD_CCtx)); - DEBUGLOG(5, "estimate workSpace : %u", (U32)neededSpace); + DEBUGLOG(5, "estimate workspace : %u", (U32)neededSpace); return sizeof(ZSTD_CCtx) + neededSpace; } } @@ -1444,8 +1444,8 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, zc->seqStore.maxNbSeq, zc->seqStore.maxNbLit, zbuff, pledgedSrcSize) ) { DEBUGLOG(4, "ZSTD_equivalentParams()==1 -> consider continue mode"); - zc->workSpaceOversizedDuration += (zc->workSpaceOversizedDuration > 0); /* if it was too large, it still is */ - if (zc->workSpaceOversizedDuration <= ZSTD_WORKSPACETOOLARGE_MAXDURATION) { + zc->workspaceOversizedDuration += (zc->workspaceOversizedDuration > 0); /* if it was too large, it still is */ + if (zc->workspaceOversizedDuration <= ZSTD_WORKSPACETOOLARGE_MAXDURATION) { DEBUGLOG(4, "continue mode confirmed (wLog1=%u, blockSize1=%zu)", zc->appliedParams.cParams.windowLog, zc->blockSize); if (ZSTD_indexTooCloseToMax(zc->blockState.matchState.window)) { @@ -1476,9 +1476,9 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, size_t const buffInSize = (zbuff==ZSTDb_buffered) ? windowSize + blockSize : 0; size_t const matchStateSize = ZSTD_sizeof_matchState(¶ms.cParams, /* forCCtx */ 1); size_t const maxNbLdmSeq = ZSTD_ldm_getMaxNbSeq(params.ldmParams, blockSize); - void* ptr; /* used to partition workSpace */ + void* ptr; /* used to partition workspace */ - /* Check if workSpace is large enough, alloc a new one if needed */ + /* Check if workspace is large enough, alloc a new one if needed */ { size_t const entropySpace = HUF_WORKSPACE_SIZE; size_t const blockStateSpace = 2 * sizeof(ZSTD_compressedBlockState_t); size_t const bufferSpace = buffInSize + buffOutSize; @@ -1489,35 +1489,35 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, ldmSeqSpace + matchStateSize + tokenSpace + bufferSpace; - int const workSpaceTooSmall = zc->workSpaceSize < neededSpace; - int const workSpaceTooLarge = zc->workSpaceSize > ZSTD_WORKSPACETOOLARGE_FACTOR * neededSpace; - int const workSpaceWasteful = workSpaceTooLarge && (zc->workSpaceOversizedDuration > ZSTD_WORKSPACETOOLARGE_MAXDURATION); - zc->workSpaceOversizedDuration = workSpaceTooLarge ? zc->workSpaceOversizedDuration+1 : 0; + int const workspaceTooSmall = zc->workspaceSize < neededSpace; + int const workspaceTooLarge = zc->workspaceSize > ZSTD_WORKSPACETOOLARGE_FACTOR * neededSpace; + int const workspaceWasteful = workspaceTooLarge && (zc->workspaceOversizedDuration > ZSTD_WORKSPACETOOLARGE_MAXDURATION); + zc->workspaceOversizedDuration = workspaceTooLarge ? zc->workspaceOversizedDuration+1 : 0; DEBUGLOG(4, "Need %zuKB workspace, including %zuKB for match state, and %zuKB for buffers", neededSpace>>10, matchStateSize>>10, bufferSpace>>10); DEBUGLOG(4, "windowSize: %zu - blockSize: %zu", windowSize, blockSize); - if (workSpaceTooSmall || workSpaceWasteful) { - DEBUGLOG(4, "Resize workSpaceSize from %zuKB to %zuKB", - zc->workSpaceSize >> 10, + if (workspaceTooSmall || workspaceWasteful) { + DEBUGLOG(4, "Resize workspaceSize from %zuKB to %zuKB", + zc->workspaceSize >> 10, neededSpace >> 10); RETURN_ERROR_IF(zc->staticSize, memory_allocation, "static cctx : no resize"); - zc->workSpaceSize = 0; - ZSTD_free(zc->workSpace, zc->customMem); - zc->workSpace = ZSTD_malloc(neededSpace, zc->customMem); - RETURN_ERROR_IF(zc->workSpace == NULL, memory_allocation); - zc->workSpaceSize = neededSpace; - zc->workSpaceOversizedDuration = 0; + zc->workspaceSize = 0; + ZSTD_free(zc->workspace, zc->customMem); + zc->workspace = ZSTD_malloc(neededSpace, zc->customMem); + RETURN_ERROR_IF(zc->workspace == NULL, memory_allocation); + zc->workspaceSize = neededSpace; + zc->workspaceOversizedDuration = 0; /* Statically sized space. * entropyWorkspace never moves, * though prev/next block swap places */ - assert(((size_t)zc->workSpace & 3) == 0); /* ensure correct alignment */ - assert(zc->workSpaceSize >= 2 * sizeof(ZSTD_compressedBlockState_t)); - zc->blockState.prevCBlock = (ZSTD_compressedBlockState_t*)zc->workSpace; + assert(((size_t)zc->workspace & 3) == 0); /* ensure correct alignment */ + assert(zc->workspaceSize >= 2 * sizeof(ZSTD_compressedBlockState_t)); + zc->blockState.prevCBlock = (ZSTD_compressedBlockState_t*)zc->workspace; zc->blockState.nextCBlock = zc->blockState.prevCBlock + 1; ptr = zc->blockState.nextCBlock + 1; zc->entropyWorkspace = (U32*)ptr; diff --git a/lib/compress/zstd_compress_internal.h b/lib/compress/zstd_compress_internal.h index 262a6309..52d6f878 100644 --- a/lib/compress/zstd_compress_internal.h +++ b/lib/compress/zstd_compress_internal.h @@ -231,9 +231,9 @@ struct ZSTD_CCtx_s { ZSTD_CCtx_params appliedParams; U32 dictID; - int workSpaceOversizedDuration; - void* workSpace; - size_t workSpaceSize; + int workspaceOversizedDuration; + void* workspace; + size_t workspaceSize; size_t blockSize; unsigned long long pledgedSrcSizePlusOne; /* this way, 0 (default) == unknown */ unsigned long long consumedSrcSize; From c25283cf00283dbaa45aca49bef2f5bda60650f6 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Mon, 15 Jul 2019 17:14:51 -0400 Subject: [PATCH 02/33] Disambiguate 'workspace' and 'entropyWorkspace' --- lib/compress/zstd_compress.c | 59 +++++++++++++++++--------- lib/compress/zstd_compress_literals.c | 15 ++++--- lib/compress/zstd_compress_literals.h | 2 +- lib/compress/zstd_compress_sequences.c | 6 +-- lib/compress/zstd_compress_sequences.h | 2 +- 5 files changed, 53 insertions(+), 31 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 61dbebac..c4a659c3 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -1960,7 +1960,7 @@ ZSTD_compressSequences_internal(seqStore_t* seqStorePtr, ZSTD_entropyCTables_t* nextEntropy, const ZSTD_CCtx_params* cctxParams, void* dst, size_t dstCapacity, - void* workspace, size_t wkspSize, + void* entropyWorkspace, size_t entropyWkspSize, const int bmi2) { const int longOffsets = cctxParams->cParams.windowLog > STREAM_ACCUMULATOR_MIN; @@ -1993,7 +1993,7 @@ ZSTD_compressSequences_internal(seqStore_t* seqStorePtr, ZSTD_disableLiteralsCompression(cctxParams), op, dstCapacity, literals, litSize, - workspace, wkspSize, + entropyWorkspace, entropyWkspSize, bmi2); FORWARD_IF_ERROR(cSize); assert(cSize <= dstCapacity); @@ -2029,7 +2029,7 @@ ZSTD_compressSequences_internal(seqStore_t* seqStorePtr, ZSTD_seqToCodes(seqStorePtr); /* build CTable for Literal Lengths */ { unsigned max = MaxLL; - size_t const mostFrequent = HIST_countFast_wksp(count, &max, llCodeTable, nbSeq, workspace, wkspSize); /* can't fail */ + size_t const mostFrequent = HIST_countFast_wksp(count, &max, llCodeTable, nbSeq, entropyWorkspace, entropyWkspSize); /* can't fail */ DEBUGLOG(5, "Building LL table"); nextEntropy->fse.litlength_repeatMode = prevEntropy->fse.litlength_repeatMode; LLtype = ZSTD_selectEncodingType(&nextEntropy->fse.litlength_repeatMode, @@ -2039,10 +2039,14 @@ ZSTD_compressSequences_internal(seqStore_t* seqStorePtr, ZSTD_defaultAllowed, strategy); assert(set_basic < set_compressed && set_rle < set_compressed); assert(!(LLtype < set_compressed && nextEntropy->fse.litlength_repeatMode != FSE_repeat_none)); /* We don't copy tables */ - { size_t const countSize = ZSTD_buildCTable(op, (size_t)(oend - op), CTable_LitLength, LLFSELog, (symbolEncodingType_e)LLtype, - count, max, llCodeTable, nbSeq, LL_defaultNorm, LL_defaultNormLog, MaxLL, - prevEntropy->fse.litlengthCTable, sizeof(prevEntropy->fse.litlengthCTable), - workspace, wkspSize); + { size_t const countSize = ZSTD_buildCTable( + op, (size_t)(oend - op), + CTable_LitLength, LLFSELog, (symbolEncodingType_e)LLtype, + count, max, llCodeTable, nbSeq, + LL_defaultNorm, LL_defaultNormLog, MaxLL, + prevEntropy->fse.litlengthCTable, + sizeof(prevEntropy->fse.litlengthCTable), + entropyWorkspace, entropyWkspSize); FORWARD_IF_ERROR(countSize); if (LLtype == set_compressed) lastNCount = op; @@ -2051,7 +2055,8 @@ ZSTD_compressSequences_internal(seqStore_t* seqStorePtr, } } /* build CTable for Offsets */ { unsigned max = MaxOff; - size_t const mostFrequent = HIST_countFast_wksp(count, &max, ofCodeTable, nbSeq, workspace, wkspSize); /* can't fail */ + size_t const mostFrequent = HIST_countFast_wksp( + count, &max, ofCodeTable, nbSeq, entropyWorkspace, entropyWkspSize); /* can't fail */ /* We can only use the basic table if max <= DefaultMaxOff, otherwise the offsets are too large */ ZSTD_defaultPolicy_e const defaultPolicy = (max <= DefaultMaxOff) ? ZSTD_defaultAllowed : ZSTD_defaultDisallowed; DEBUGLOG(5, "Building OF table"); @@ -2062,10 +2067,14 @@ ZSTD_compressSequences_internal(seqStore_t* seqStorePtr, OF_defaultNorm, OF_defaultNormLog, defaultPolicy, strategy); assert(!(Offtype < set_compressed && nextEntropy->fse.offcode_repeatMode != FSE_repeat_none)); /* We don't copy tables */ - { size_t const countSize = ZSTD_buildCTable(op, (size_t)(oend - op), CTable_OffsetBits, OffFSELog, (symbolEncodingType_e)Offtype, - count, max, ofCodeTable, nbSeq, OF_defaultNorm, OF_defaultNormLog, DefaultMaxOff, - prevEntropy->fse.offcodeCTable, sizeof(prevEntropy->fse.offcodeCTable), - workspace, wkspSize); + { size_t const countSize = ZSTD_buildCTable( + op, (size_t)(oend - op), + CTable_OffsetBits, OffFSELog, (symbolEncodingType_e)Offtype, + count, max, ofCodeTable, nbSeq, + OF_defaultNorm, OF_defaultNormLog, DefaultMaxOff, + prevEntropy->fse.offcodeCTable, + sizeof(prevEntropy->fse.offcodeCTable), + entropyWorkspace, entropyWkspSize); FORWARD_IF_ERROR(countSize); if (Offtype == set_compressed) lastNCount = op; @@ -2074,7 +2083,8 @@ ZSTD_compressSequences_internal(seqStore_t* seqStorePtr, } } /* build CTable for MatchLengths */ { unsigned max = MaxML; - size_t const mostFrequent = HIST_countFast_wksp(count, &max, mlCodeTable, nbSeq, workspace, wkspSize); /* can't fail */ + size_t const mostFrequent = HIST_countFast_wksp( + count, &max, mlCodeTable, nbSeq, entropyWorkspace, entropyWkspSize); /* can't fail */ DEBUGLOG(5, "Building ML table (remaining space : %i)", (int)(oend-op)); nextEntropy->fse.matchlength_repeatMode = prevEntropy->fse.matchlength_repeatMode; MLtype = ZSTD_selectEncodingType(&nextEntropy->fse.matchlength_repeatMode, @@ -2083,10 +2093,14 @@ ZSTD_compressSequences_internal(seqStore_t* seqStorePtr, ML_defaultNorm, ML_defaultNormLog, ZSTD_defaultAllowed, strategy); assert(!(MLtype < set_compressed && nextEntropy->fse.matchlength_repeatMode != FSE_repeat_none)); /* We don't copy tables */ - { size_t const countSize = ZSTD_buildCTable(op, (size_t)(oend - op), CTable_MatchLength, MLFSELog, (symbolEncodingType_e)MLtype, - count, max, mlCodeTable, nbSeq, ML_defaultNorm, ML_defaultNormLog, MaxML, - prevEntropy->fse.matchlengthCTable, sizeof(prevEntropy->fse.matchlengthCTable), - workspace, wkspSize); + { size_t const countSize = ZSTD_buildCTable( + op, (size_t)(oend - op), + CTable_MatchLength, MLFSELog, (symbolEncodingType_e)MLtype, + count, max, mlCodeTable, nbSeq, + ML_defaultNorm, ML_defaultNormLog, MaxML, + prevEntropy->fse.matchlengthCTable, + sizeof(prevEntropy->fse.matchlengthCTable), + entropyWorkspace, entropyWkspSize); FORWARD_IF_ERROR(countSize); if (MLtype == set_compressed) lastNCount = op; @@ -2134,13 +2148,13 @@ ZSTD_compressSequences(seqStore_t* seqStorePtr, const ZSTD_CCtx_params* cctxParams, void* dst, size_t dstCapacity, size_t srcSize, - void* workspace, size_t wkspSize, + void* entropyWorkspace, size_t entropyWkspSize, int bmi2) { size_t const cSize = ZSTD_compressSequences_internal( seqStorePtr, prevEntropy, nextEntropy, cctxParams, dst, dstCapacity, - workspace, wkspSize, bmi2); + entropyWorkspace, entropyWkspSize, bmi2); if (cSize == 0) return 0; /* When srcSize <= dstCapacity, there is enough space to write a raw uncompressed block. * Since we ran out of space, block must be not compressible, so fall back to raw uncompressed block. @@ -2380,6 +2394,7 @@ static void ZSTD_overflowCorrectIfNeeded(ZSTD_matchState_t* ms, ZSTD_CCtx_params } } + /*! ZSTD_compress_frameChunk() : * Compress a chunk of data into one or multiple blocks. * All blocks will be terminated, all input will be consumed. @@ -2848,7 +2863,8 @@ static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* cctx, ZSTDcrp_continue, zbuff) ); { size_t const dictID = ZSTD_compress_insertDictionary( cctx->blockState.prevCBlock, &cctx->blockState.matchState, - params, dict, dictSize, dictContentType, dtlm, cctx->entropyWorkspace); + params, dict, dictSize, dictContentType, dtlm, + cctx->entropyWorkspace); FORWARD_IF_ERROR(dictID); assert(dictID <= UINT_MAX); cctx->dictID = (U32)dictID; @@ -3393,7 +3409,8 @@ size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pss) * Assumption 2 : either dict, or cdict, is defined, not both */ size_t ZSTD_initCStream_internal(ZSTD_CStream* zcs, const void* dict, size_t dictSize, const ZSTD_CDict* cdict, - const ZSTD_CCtx_params* params, unsigned long long pledgedSrcSize) + const ZSTD_CCtx_params* params, + unsigned long long pledgedSrcSize) { DEBUGLOG(4, "ZSTD_initCStream_internal"); FORWARD_IF_ERROR( ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only) ); diff --git a/lib/compress/zstd_compress_literals.c b/lib/compress/zstd_compress_literals.c index eb3e5a44..6c133311 100644 --- a/lib/compress/zstd_compress_literals.c +++ b/lib/compress/zstd_compress_literals.c @@ -70,7 +70,7 @@ size_t ZSTD_compressLiterals (ZSTD_hufCTables_t const* prevHuf, ZSTD_strategy strategy, int disableLiteralCompression, void* dst, size_t dstCapacity, const void* src, size_t srcSize, - void* workspace, size_t wkspSize, + void* entropyWorkspace, size_t entropyWorkspaceSize, const int bmi2) { size_t const minGain = ZSTD_minGain(srcSize, strategy); @@ -99,10 +99,15 @@ size_t ZSTD_compressLiterals (ZSTD_hufCTables_t const* prevHuf, { HUF_repeat repeat = prevHuf->repeatMode; int const preferRepeat = strategy < ZSTD_lazy ? srcSize <= 1024 : 0; if (repeat == HUF_repeat_valid && lhSize == 3) singleStream = 1; - cLitSize = singleStream ? HUF_compress1X_repeat(ostart+lhSize, dstCapacity-lhSize, src, srcSize, 255, 11, - workspace, wkspSize, (HUF_CElt*)nextHuf->CTable, &repeat, preferRepeat, bmi2) - : HUF_compress4X_repeat(ostart+lhSize, dstCapacity-lhSize, src, srcSize, 255, 11, - workspace, wkspSize, (HUF_CElt*)nextHuf->CTable, &repeat, preferRepeat, bmi2); + cLitSize = singleStream ? + HUF_compress1X_repeat( + ostart+lhSize, dstCapacity-lhSize, src, srcSize, + 255, 11, entropyWorkspace, entropyWorkspaceSize, + (HUF_CElt*)nextHuf->CTable, &repeat, preferRepeat, bmi2) : + HUF_compress4X_repeat( + ostart+lhSize, dstCapacity-lhSize, src, srcSize, + 255, 11, entropyWorkspace, entropyWorkspaceSize, + (HUF_CElt*)nextHuf->CTable, &repeat, preferRepeat, bmi2); if (repeat != HUF_repeat_none) { /* reused the existing table */ hType = set_repeat; diff --git a/lib/compress/zstd_compress_literals.h b/lib/compress/zstd_compress_literals.h index 7adbecc0..97273d7c 100644 --- a/lib/compress/zstd_compress_literals.h +++ b/lib/compress/zstd_compress_literals.h @@ -23,7 +23,7 @@ size_t ZSTD_compressLiterals (ZSTD_hufCTables_t const* prevHuf, ZSTD_strategy strategy, int disableLiteralCompression, void* dst, size_t dstCapacity, const void* src, size_t srcSize, - void* workspace, size_t wkspSize, + void* entropyWorkspace, size_t entropyWorkspaceSize, const int bmi2); #endif /* ZSTD_COMPRESS_LITERALS_H */ diff --git a/lib/compress/zstd_compress_sequences.c b/lib/compress/zstd_compress_sequences.c index 3c3deae0..0ff7a268 100644 --- a/lib/compress/zstd_compress_sequences.c +++ b/lib/compress/zstd_compress_sequences.c @@ -222,7 +222,7 @@ ZSTD_buildCTable(void* dst, size_t dstCapacity, const BYTE* codeTable, size_t nbSeq, const S16* defaultNorm, U32 defaultNormLog, U32 defaultMax, const FSE_CTable* prevCTable, size_t prevCTableSize, - void* workspace, size_t workspaceSize) + void* entropyWorkspace, size_t entropyWorkspaceSize) { BYTE* op = (BYTE*)dst; const BYTE* const oend = op + dstCapacity; @@ -238,7 +238,7 @@ ZSTD_buildCTable(void* dst, size_t dstCapacity, memcpy(nextCTable, prevCTable, prevCTableSize); return 0; case set_basic: - FORWARD_IF_ERROR(FSE_buildCTable_wksp(nextCTable, defaultNorm, defaultMax, defaultNormLog, workspace, workspaceSize)); /* note : could be pre-calculated */ + FORWARD_IF_ERROR(FSE_buildCTable_wksp(nextCTable, defaultNorm, defaultMax, defaultNormLog, entropyWorkspace, entropyWorkspaceSize)); /* note : could be pre-calculated */ return 0; case set_compressed: { S16 norm[MaxSeq + 1]; @@ -252,7 +252,7 @@ ZSTD_buildCTable(void* dst, size_t dstCapacity, FORWARD_IF_ERROR(FSE_normalizeCount(norm, tableLog, count, nbSeq_1, max)); { size_t const NCountSize = FSE_writeNCount(op, oend - op, norm, max, tableLog); /* overflow protected */ FORWARD_IF_ERROR(NCountSize); - FORWARD_IF_ERROR(FSE_buildCTable_wksp(nextCTable, norm, max, tableLog, workspace, workspaceSize)); + FORWARD_IF_ERROR(FSE_buildCTable_wksp(nextCTable, norm, max, tableLog, entropyWorkspace, entropyWorkspaceSize)); return NCountSize; } } diff --git a/lib/compress/zstd_compress_sequences.h b/lib/compress/zstd_compress_sequences.h index f5234d94..57e8e367 100644 --- a/lib/compress/zstd_compress_sequences.h +++ b/lib/compress/zstd_compress_sequences.h @@ -35,7 +35,7 @@ ZSTD_buildCTable(void* dst, size_t dstCapacity, const BYTE* codeTable, size_t nbSeq, const S16* defaultNorm, U32 defaultNormLog, U32 defaultMax, const FSE_CTable* prevCTable, size_t prevCTableSize, - void* workspace, size_t workspaceSize); + void* entropyWorkspace, size_t entropyWorkspaceSize); size_t ZSTD_encodeSequences( void* dst, size_t dstCapacity, From 786f2266bb77076054498241755b4ef6f42ab382 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Mon, 15 Jul 2019 15:08:49 -0400 Subject: [PATCH 03/33] TMP --- lib/compress/zstd_compress.c | 402 ++++++++++++++++++-------- lib/compress/zstd_compress_internal.h | 78 ++++- 2 files changed, 352 insertions(+), 128 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index c4a659c3..b0e57e4d 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -38,6 +38,159 @@ size_t ZSTD_compressBound(size_t srcSize) { } +/*-************************************* +* Workspace memory management +***************************************/ +#define ZSTD_WORKSPACETOOLARGE_FACTOR 3 /* define "workspace is too large" as this number of times larger than needed */ +#define ZSTD_WORKSPACETOOLARGE_MAXDURATION 128 /* when workspace is continuously too large + * during at least this number of times, + * context's memory usage is considered wasteful, + * because it's sized to handle a worst case scenario which rarely happens. + * In which case, resize it down to free some memory */ + +static size_t ZSTD_workspace_round_size(size_t size) { + // return size + sizeof(void*) - 1 - ((size - 1) & (sizeof(void*) - 1)); + return size + 3 - ((size - 1) & 3); +} + +/** + * Align must be a power of 2. + */ +static size_t ZSTD_workspace_align(size_t size, size_t align) { + return size + align - 1 - ((size - 1) & (align - 1)); + // return size + 3 - ((size - 1) & 3); +} + +static void* ZSTD_workspace_reserve(ZSTD_CCtx_workspace* ws, size_t bytes) { + /* TODO(felixh): alignment */ + void* alloc = ws->allocEnd; + void* newAllocEnd = (BYTE *)ws->allocEnd + bytes; + DEBUGLOG(3, "wksp: reserving %zd bytes, %zd bytes remaining", bytes, (BYTE *)ws->workspaceEnd - (BYTE *)newAllocEnd); + assert(newAllocEnd <= ws->workspaceEnd); + if (newAllocEnd > ws->workspaceEnd) { + ws->allocFailed = 1; + return NULL; + } + ws->allocEnd = newAllocEnd; + ws->staticAllocDone = 1; + return alloc; +} + +/** + * Aligned on sizeof(void*). + */ +static void* ZSTD_workspace_reserve_object(ZSTD_CCtx_workspace* ws, size_t bytes) { + size_t roundedBytes = ZSTD_workspace_align(bytes, sizeof(void*)); + void* start = ws->objectEnd; + void* end = (BYTE*)start + roundedBytes; + assert(ws->allocEnd == ws->objectEnd); + DEBUGLOG(3, "wksp: reserving %zd bytes object (rounded to %zd), %zd bytes remaining", bytes, roundedBytes, (BYTE *)ws->workspaceEnd - (BYTE *)end); + assert((bytes & (sizeof(void*)-1)) == 0); // TODO ??? + if (ws->staticAllocDone || end > ws->workspaceEnd) { + DEBUGLOG(3, "wksp: object alloc failed!"); + ws->allocFailed = 1; + return NULL; + } + ws->objectEnd = end; + ws->allocEnd = end; + return start; +} + +/** + * Aligned on sizeof(unsigned). These buffers have the special property that + * their values remain constrained, allowing us to re-use them without + * memset()-ing them. + */ +static void* ZSTD_workspace_reserve_table(ZSTD_CCtx_workspace* ws, size_t bytes) { + assert((bytes & (sizeof(U32)-1)) == 0); // TODO ??? + ws->staticAllocDone = 1; + return ZSTD_workspace_reserve(ws, ZSTD_workspace_align(bytes, sizeof(unsigned))); +} + +/** + * Aligned on sizeof(unsigned). + */ +static void* ZSTD_workspace_reserve_aligned(ZSTD_CCtx_workspace* ws, size_t bytes) { + assert((bytes & (sizeof(U32)-1)) == 0); // TODO ??? + ws->staticAllocDone = 1; + return ZSTD_workspace_reserve(ws, ZSTD_workspace_align(bytes, sizeof(unsigned))); +} + +/** + * Unaligned. + */ +static void* ZSTD_workspace_reserve_buffer(ZSTD_CCtx_workspace* ws, size_t bytes) { + ws->staticAllocDone = 1; + return ZSTD_workspace_reserve(ws, bytes); +} + +// TODO +static int ZSTD_workspace_bump_oversized_duration(ZSTD_CCtx_workspace* ws) { + // if (((BYTE*)ws->allocEnd - (BYTE*)ws->workspace) * ZSTD_WORKSPACETOOLARGE_FACTOR < (BYTE*)ws->workspaceEnd - (BYTE*)ws->workspace) { + // ws->workspaceOversizedDuration++; + // } else { + // ws->workspaceOversizedDuration = 0; + // } + // return ws->workspaceOversizedDuration; + return 0; +} + +static void ZSTD_workspace_clear(ZSTD_CCtx_workspace* ws) { + DEBUGLOG(3, "wksp: clearing!"); + ZSTD_workspace_bump_oversized_duration(ws); + ws->allocEnd = ws->objectEnd; + ws->allocFailed = 0; + + // ws->table = NULL; + // ws->tableEnd = NULL; + + // ws->bufferBegin = ws->workspaceEnd; +} + +static void ZSTD_workspace_init(ZSTD_CCtx_workspace* ws, void* start, size_t size) { + DEBUGLOG(3, "wksp: init'ing with %zd bytes", size); + assert(((size_t)start & (sizeof(void*)-1)) == 0); /* ensure correct alignment */ + ws->workspace = start; + ws->workspaceEnd = (BYTE*)start + size; + ws->objectEnd = ws->workspace; + ZSTD_workspace_clear(ws); + ws->workspaceOversizedDuration = 0; + ws->staticAllocDone = 0; +} + +static size_t ZSTD_workspace_create(ZSTD_CCtx_workspace* ws, size_t size, ZSTD_customMem customMem) { + void* workspace = ZSTD_malloc(size, customMem); + DEBUGLOG(3, "wksp: creating with %zd bytes", size); + RETURN_ERROR_IF(workspace == NULL, memory_allocation); + ZSTD_workspace_init(ws, workspace, size); + return 0; +} + +static void ZSTD_workspace_free(ZSTD_CCtx_workspace* ws, ZSTD_customMem customMem) { + DEBUGLOG(3, "wksp: freeing"); + ZSTD_free(ws->workspace, customMem); + ws->workspace = NULL; + ws->workspaceEnd = NULL; + ZSTD_workspace_clear(ws); +} + +static int ZSTD_workspace_check_available(ZSTD_CCtx_workspace* ws, size_t minFree) { + return (size_t)((BYTE*)ws->workspaceEnd - (BYTE*)ws->allocEnd) >= minFree; +} + +static int ZSTD_workspace_check_wasteful(ZSTD_CCtx_workspace* ws, size_t minFree) { + return ZSTD_workspace_check_available(ws, minFree * ZSTD_WORKSPACETOOLARGE_FACTOR) && ws->workspaceOversizedDuration > ZSTD_WORKSPACETOOLARGE_MAXDURATION; +} + +static size_t ZSTD_workspace_sizeof(const ZSTD_CCtx_workspace* ws) { + return (BYTE*)ws->workspaceEnd - (BYTE*)ws->workspace; +} + +static int ZSTD_workspace_reserve_failed(const ZSTD_CCtx_workspace* ws) { + return ws->allocFailed; +} + + /*-************************************* * Context memory management ***************************************/ @@ -45,8 +198,8 @@ struct ZSTD_CDict_s { void* dictBuffer; const void* dictContent; size_t dictContentSize; - void* workspace; - size_t workspaceSize; + U32* entropyWorkspace; /* entropy workspace of HUF_WORKSPACE_SIZE bytes */ + ZSTD_CCtx_workspace workspace; ZSTD_matchState_t matchState; ZSTD_compressedBlockState_t cBlockState; ZSTD_customMem customMem; @@ -87,20 +240,16 @@ ZSTD_CCtx* ZSTD_initStaticCCtx(void *workspace, size_t workspaceSize) ZSTD_CCtx* const cctx = (ZSTD_CCtx*) workspace; if (workspaceSize <= sizeof(ZSTD_CCtx)) return NULL; /* minimum size */ if ((size_t)workspace & 7) return NULL; /* must be 8-aligned */ - memset(workspace, 0, workspaceSize); /* may be a bit generous, could memset be smaller ? */ + memset(workspace, 0, workspaceSize); /* may be a bit generous, could memset be smaller ? */ /* TODO(felixh): investigate */ cctx->staticSize = workspaceSize; - cctx->workspace = (void*)(cctx+1); - cctx->workspaceSize = workspaceSize - sizeof(ZSTD_CCtx); + ZSTD_workspace_init(&cctx->workspace, (void*)(cctx+1), workspaceSize - sizeof(ZSTD_CCtx)); /* statically sized space. entropyWorkspace never moves (but prev/next block swap places) */ - if (cctx->workspaceSize < HUF_WORKSPACE_SIZE + 2 * sizeof(ZSTD_compressedBlockState_t)) return NULL; - assert(((size_t)cctx->workspace & (sizeof(void*)-1)) == 0); /* ensure correct alignment */ - cctx->blockState.prevCBlock = (ZSTD_compressedBlockState_t*)cctx->workspace; - cctx->blockState.nextCBlock = cctx->blockState.prevCBlock + 1; - { - void* const ptr = cctx->blockState.nextCBlock + 1; - cctx->entropyWorkspace = (U32*)ptr; - } + if (!ZSTD_workspace_check_available(&cctx->workspace, HUF_WORKSPACE_SIZE + 2 * sizeof(ZSTD_compressedBlockState_t))) return NULL; + cctx->blockState.prevCBlock = (ZSTD_compressedBlockState_t*)ZSTD_workspace_reserve_object(&cctx->workspace, sizeof(ZSTD_compressedBlockState_t)); + cctx->blockState.nextCBlock = (ZSTD_compressedBlockState_t*)ZSTD_workspace_reserve_object(&cctx->workspace, sizeof(ZSTD_compressedBlockState_t)); + cctx->entropyWorkspace = (U32*)ZSTD_workspace_reserve_object( + &cctx->workspace, HUF_WORKSPACE_SIZE); cctx->bmi2 = ZSTD_cpuid_bmi2(ZSTD_cpuid()); return cctx; } @@ -128,7 +277,7 @@ static void ZSTD_freeCCtxContent(ZSTD_CCtx* cctx) { assert(cctx != NULL); assert(cctx->staticSize == 0); - ZSTD_free(cctx->workspace, cctx->customMem); cctx->workspace = NULL; + ZSTD_workspace_free(&cctx->workspace, cctx->customMem); ZSTD_clearAllDicts(cctx); #ifdef ZSTD_MULTITHREAD ZSTDMT_freeCCtx(cctx->mtctx); cctx->mtctx = NULL; @@ -160,7 +309,7 @@ static size_t ZSTD_sizeof_mtctx(const ZSTD_CCtx* cctx) size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx) { if (cctx==NULL) return 0; /* support sizeof on NULL */ - return sizeof(*cctx) + cctx->workspaceSize + return sizeof(*cctx) + ZSTD_workspace_sizeof(&cctx->workspace) + ZSTD_sizeof_localDict(cctx->localDict) + ZSTD_sizeof_mtctx(cctx); } @@ -1355,9 +1504,9 @@ typedef enum { ZSTDcrp_continue, ZSTDcrp_noMemset } ZSTD_compResetPolicy_e; typedef enum { ZSTD_resetTarget_CDict, ZSTD_resetTarget_CCtx } ZSTD_resetTarget_e; -static void* +static size_t ZSTD_reset_matchState(ZSTD_matchState_t* ms, - void* ptr, + ZSTD_CCtx_workspace* ws, const ZSTD_compressionParameters* cParams, ZSTD_compResetPolicy_e const crp, ZSTD_resetTarget_e const forWho) { @@ -1365,9 +1514,7 @@ ZSTD_reset_matchState(ZSTD_matchState_t* ms, size_t const hSize = ((size_t)1) << cParams->hashLog; U32 const hashLog3 = ((forWho == ZSTD_resetTarget_CCtx) && cParams->minMatch==3) ? MIN(ZSTD_HASHLOG3_MAX, cParams->windowLog) : 0; size_t const h3Size = ((size_t)1) << hashLog3; - size_t const tableSpace = (chainSize + hSize + h3Size) * sizeof(U32); - assert(((size_t)ptr & 3) == 0); ms->hashLog3 = hashLog3; memset(&ms->window, 0, sizeof(ms->window)); @@ -1376,33 +1523,40 @@ ZSTD_reset_matchState(ZSTD_matchState_t* ms, ms->window.nextSrc = ms->window.base + 1; /* see issue #1241 */ ZSTD_invalidateMatchState(ms); + assert(!ZSTD_workspace_reserve_failed(ws)); /* check that allocation hasn't already failed */ + + DEBUGLOG(5, "reserving table space"); + /* table Space */ + ms->hashTable = (U32*)ZSTD_workspace_reserve_table(ws, hSize * sizeof(U32)); + ms->chainTable = (U32*)ZSTD_workspace_reserve_table(ws, chainSize * sizeof(U32)); + ms->hashTable3 = (U32*)ZSTD_workspace_reserve_table(ws, h3Size * sizeof(U32)); + RETURN_ERROR_IF(ZSTD_workspace_reserve_failed(ws), memory_allocation, + "failed a workspace allocation in ZSTD_reset_matchState"); + DEBUGLOG(4, "reset table : %u", crp!=ZSTDcrp_noMemset); + if (crp!=ZSTDcrp_noMemset) { + /* reset tables only */ + memset(ms->hashTable, 0, hSize * sizeof(U32)); + memset(ms->chainTable, 0, chainSize * sizeof(U32)); + memset(ms->hashTable3, 0, h3Size * sizeof(U32)); + } + /* opt parser space */ if ((forWho == ZSTD_resetTarget_CCtx) && (cParams->strategy >= ZSTD_btopt)) { DEBUGLOG(4, "reserving optimal parser space"); - ms->opt.litFreq = (unsigned*)ptr; - ms->opt.litLengthFreq = ms->opt.litFreq + (1<opt.matchLengthFreq = ms->opt.litLengthFreq + (MaxLL+1); - ms->opt.offCodeFreq = ms->opt.matchLengthFreq + (MaxML+1); - ptr = ms->opt.offCodeFreq + (MaxOff+1); - ms->opt.matchTable = (ZSTD_match_t*)ptr; - ptr = ms->opt.matchTable + ZSTD_OPT_NUM+1; - ms->opt.priceTable = (ZSTD_optimal_t*)ptr; - ptr = ms->opt.priceTable + ZSTD_OPT_NUM+1; + ms->opt.litFreq = (unsigned*)ZSTD_workspace_reserve_aligned(ws, (1<opt.litLengthFreq = (unsigned*)ZSTD_workspace_reserve_aligned(ws, (MaxLL+1) * sizeof(unsigned)); + ms->opt.matchLengthFreq = (unsigned*)ZSTD_workspace_reserve_aligned(ws, (MaxML+1) * sizeof(unsigned)); + ms->opt.offCodeFreq = (unsigned*)ZSTD_workspace_reserve_aligned(ws, (MaxOff+1) * sizeof(unsigned)); + ms->opt.matchTable = (ZSTD_match_t*)ZSTD_workspace_reserve_aligned(ws, (ZSTD_OPT_NUM+1) * sizeof(ZSTD_match_t)); + ms->opt.priceTable = (ZSTD_optimal_t*)ZSTD_workspace_reserve_aligned(ws, (ZSTD_OPT_NUM+1) * sizeof(ZSTD_optimal_t)); } - /* table Space */ - DEBUGLOG(4, "reset table : %u", crp!=ZSTDcrp_noMemset); - assert(((size_t)ptr & 3) == 0); /* ensure ptr is properly aligned */ - if (crp!=ZSTDcrp_noMemset) memset(ptr, 0, tableSpace); /* reset tables only */ - ms->hashTable = (U32*)(ptr); - ms->chainTable = ms->hashTable + hSize; - ms->hashTable3 = ms->chainTable + chainSize; - ptr = ms->hashTable3 + h3Size; - ms->cParams = *cParams; - assert(((size_t)ptr & 3) == 0); - return ptr; + RETURN_ERROR_IF(ZSTD_workspace_reserve_failed(ws), memory_allocation, + "failed a workspace allocation in ZSTD_reset_matchState"); + + return 0; } /* ZSTD_indexTooCloseToMax() : @@ -1418,13 +1572,6 @@ static int ZSTD_indexTooCloseToMax(ZSTD_window_t w) return (size_t)(w.nextSrc - w.base) > (ZSTD_CURRENT_MAX - ZSTD_INDEXOVERFLOW_MARGIN); } -#define ZSTD_WORKSPACETOOLARGE_FACTOR 3 /* define "workspace is too large" as this number of times larger than needed */ -#define ZSTD_WORKSPACETOOLARGE_MAXDURATION 128 /* when workspace is continuously too large - * during at least this number of times, - * context's memory usage is considered wasteful, - * because it's sized to handle a worst case scenario which rarely happens. - * In which case, resize it down to free some memory */ - /*! ZSTD_resetCCtx_internal() : note : `params` are assumed fully validated at this stage */ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, @@ -1444,16 +1591,16 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, zc->seqStore.maxNbSeq, zc->seqStore.maxNbLit, zbuff, pledgedSrcSize) ) { DEBUGLOG(4, "ZSTD_equivalentParams()==1 -> consider continue mode"); - zc->workspaceOversizedDuration += (zc->workspaceOversizedDuration > 0); /* if it was too large, it still is */ - if (zc->workspaceOversizedDuration <= ZSTD_WORKSPACETOOLARGE_MAXDURATION) { + if (ZSTD_workspace_bump_oversized_duration(&zc->workspace) <= ZSTD_WORKSPACETOOLARGE_MAXDURATION) { DEBUGLOG(4, "continue mode confirmed (wLog1=%u, blockSize1=%zu)", zc->appliedParams.cParams.windowLog, zc->blockSize); if (ZSTD_indexTooCloseToMax(zc->blockState.matchState.window)) { /* prefer a reset, faster than a rescale */ - ZSTD_reset_matchState(&zc->blockState.matchState, - zc->entropyWorkspace + HUF_WORKSPACE_SIZE_U32, - ¶ms.cParams, - crp, ZSTD_resetTarget_CCtx); + FORWARD_IF_ERROR(ZSTD_reset_matchState( + &zc->blockState.matchState, + &zc->workspace, + ¶ms.cParams, + crp, ZSTD_resetTarget_CCtx)); } return ZSTD_continueCCtx(zc, ¶ms, pledgedSrcSize); } } } @@ -1476,7 +1623,6 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, size_t const buffInSize = (zbuff==ZSTDb_buffered) ? windowSize + blockSize : 0; size_t const matchStateSize = ZSTD_sizeof_matchState(¶ms.cParams, /* forCCtx */ 1); size_t const maxNbLdmSeq = ZSTD_ldm_getMaxNbSeq(params.ldmParams, blockSize); - void* ptr; /* used to partition workspace */ /* Check if workspace is large enough, alloc a new one if needed */ { size_t const entropySpace = HUF_WORKSPACE_SIZE; @@ -1485,14 +1631,17 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, size_t const ldmSpace = ZSTD_ldm_getTableSize(params.ldmParams); size_t const ldmSeqSpace = maxNbLdmSeq * sizeof(rawSeq); - size_t const neededSpace = entropySpace + blockStateSpace + ldmSpace + - ldmSeqSpace + matchStateSize + tokenSpace + - bufferSpace; + size_t const neededSpace = + entropySpace + + blockStateSpace + + ldmSpace + + ldmSeqSpace + + matchStateSize + + tokenSpace + + bufferSpace; - int const workspaceTooSmall = zc->workspaceSize < neededSpace; - int const workspaceTooLarge = zc->workspaceSize > ZSTD_WORKSPACETOOLARGE_FACTOR * neededSpace; - int const workspaceWasteful = workspaceTooLarge && (zc->workspaceOversizedDuration > ZSTD_WORKSPACETOOLARGE_MAXDURATION); - zc->workspaceOversizedDuration = workspaceTooLarge ? zc->workspaceOversizedDuration+1 : 0; + int const workspaceTooSmall = !ZSTD_workspace_check_available(&zc->workspace, neededSpace); + int const workspaceWasteful = ZSTD_workspace_check_wasteful(&zc->workspace, neededSpace); DEBUGLOG(4, "Need %zuKB workspace, including %zuKB for match state, and %zuKB for buffers", neededSpace>>10, matchStateSize>>10, bufferSpace>>10); @@ -1500,29 +1649,30 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, if (workspaceTooSmall || workspaceWasteful) { DEBUGLOG(4, "Resize workspaceSize from %zuKB to %zuKB", - zc->workspaceSize >> 10, + ZSTD_workspace_sizeof(&zc->workspace) >> 10, neededSpace >> 10); RETURN_ERROR_IF(zc->staticSize, memory_allocation, "static cctx : no resize"); - zc->workspaceSize = 0; - ZSTD_free(zc->workspace, zc->customMem); - zc->workspace = ZSTD_malloc(neededSpace, zc->customMem); - RETURN_ERROR_IF(zc->workspace == NULL, memory_allocation); - zc->workspaceSize = neededSpace; - zc->workspaceOversizedDuration = 0; + ZSTD_workspace_free(&zc->workspace, zc->customMem); + FORWARD_IF_ERROR(ZSTD_workspace_create(&zc->workspace, neededSpace, zc->customMem)); + DEBUGLOG(5, "reserving object space"); /* Statically sized space. * entropyWorkspace never moves, * though prev/next block swap places */ - assert(((size_t)zc->workspace & 3) == 0); /* ensure correct alignment */ - assert(zc->workspaceSize >= 2 * sizeof(ZSTD_compressedBlockState_t)); - zc->blockState.prevCBlock = (ZSTD_compressedBlockState_t*)zc->workspace; - zc->blockState.nextCBlock = zc->blockState.prevCBlock + 1; - ptr = zc->blockState.nextCBlock + 1; - zc->entropyWorkspace = (U32*)ptr; + /* assert(((size_t)zc->workspace.workspace & 3) == 0); */ /* ensure correct alignment */ /* TODO(felixh): check elsewhere */ + assert(ZSTD_workspace_check_available(&zc->workspace, 2 * sizeof(ZSTD_compressedBlockState_t))); + zc->blockState.prevCBlock = (ZSTD_compressedBlockState_t*) ZSTD_workspace_reserve_object(&zc->workspace, sizeof(ZSTD_compressedBlockState_t)); + RETURN_ERROR_IF(zc->blockState.prevCBlock == NULL, memory_allocation, "couldn't allocate prevCBlock"); + zc->blockState.nextCBlock = (ZSTD_compressedBlockState_t*) ZSTD_workspace_reserve_object(&zc->workspace, sizeof(ZSTD_compressedBlockState_t)); + RETURN_ERROR_IF(zc->blockState.nextCBlock == NULL, memory_allocation, "couldn't allocate nextCBlock"); + zc->entropyWorkspace = (U32*) ZSTD_workspace_reserve_object(&zc->workspace, HUF_WORKSPACE_SIZE); + RETURN_ERROR_IF(zc->blockState.nextCBlock == NULL, memory_allocation, "couldn't allocate entropyWorkspace"); } } + ZSTD_workspace_clear(&zc->workspace); + /* init params */ zc->appliedParams = params; zc->blockState.matchState.cParams = params.cParams; @@ -1541,58 +1691,55 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, ZSTD_reset_compressedBlockState(zc->blockState.prevCBlock); - ptr = ZSTD_reset_matchState(&zc->blockState.matchState, - zc->entropyWorkspace + HUF_WORKSPACE_SIZE_U32, - ¶ms.cParams, - crp, ZSTD_resetTarget_CCtx); + FORWARD_IF_ERROR(ZSTD_reset_matchState( + &zc->blockState.matchState, + &zc->workspace, + ¶ms.cParams, + crp, ZSTD_resetTarget_CCtx)); + + DEBUGLOG(3, "Done allocating match state"); /* ldm hash table */ /* initialize bucketOffsets table later for pointer alignment */ if (params.ldmParams.enableLdm) { size_t const ldmHSize = ((size_t)1) << params.ldmParams.hashLog; - memset(ptr, 0, ldmHSize * sizeof(ldmEntry_t)); - assert(((size_t)ptr & 3) == 0); /* ensure ptr is properly aligned */ - zc->ldmState.hashTable = (ldmEntry_t*)ptr; - ptr = zc->ldmState.hashTable + ldmHSize; - zc->ldmSequences = (rawSeq*)ptr; - ptr = zc->ldmSequences + maxNbLdmSeq; + zc->ldmState.hashTable = (ldmEntry_t*)ZSTD_workspace_reserve_aligned(&zc->workspace, ldmHSize * sizeof(ldmEntry_t)); + memset(zc->ldmState.hashTable, 0, ldmHSize * sizeof(ldmEntry_t)); + zc->ldmSequences = (rawSeq*)ZSTD_workspace_reserve_aligned(&zc->workspace, maxNbLdmSeq * sizeof(rawSeq)); zc->maxNbLdmSequences = maxNbLdmSeq; memset(&zc->ldmState.window, 0, sizeof(zc->ldmState.window)); } - assert(((size_t)ptr & 3) == 0); /* ensure ptr is properly aligned */ - - /* sequences storage */ - zc->seqStore.maxNbSeq = maxNbSeq; - zc->seqStore.sequencesStart = (seqDef*)ptr; - ptr = zc->seqStore.sequencesStart + maxNbSeq; - zc->seqStore.llCode = (BYTE*) ptr; - zc->seqStore.mlCode = zc->seqStore.llCode + maxNbSeq; - zc->seqStore.ofCode = zc->seqStore.mlCode + maxNbSeq; - zc->seqStore.litStart = zc->seqStore.ofCode + maxNbSeq; - /* ZSTD_wildcopy() is used to copy into the literals buffer, - * so we have to oversize the buffer by WILDCOPY_OVERLENGTH bytes. - */ - zc->seqStore.maxNbLit = blockSize; - ptr = zc->seqStore.litStart + blockSize + WILDCOPY_OVERLENGTH; /* ldm bucketOffsets table */ if (params.ldmParams.enableLdm) { size_t const ldmBucketSize = ((size_t)1) << (params.ldmParams.hashLog - params.ldmParams.bucketSizeLog); - memset(ptr, 0, ldmBucketSize); - zc->ldmState.bucketOffsets = (BYTE*)ptr; - ptr = zc->ldmState.bucketOffsets + ldmBucketSize; + zc->ldmState.bucketOffsets = (BYTE*)ZSTD_workspace_reserve_aligned(&zc->workspace, ldmBucketSize); + memset(zc->ldmState.bucketOffsets, 0, ldmBucketSize); ZSTD_window_clear(&zc->ldmState.window); } ZSTD_referenceExternalSequences(zc, NULL, 0); + /* sequences storage */ + zc->seqStore.maxNbSeq = maxNbSeq; + zc->seqStore.sequencesStart = (seqDef*)ZSTD_workspace_reserve_aligned(&zc->workspace, maxNbSeq * sizeof(seqDef)); + zc->seqStore.llCode = (BYTE*)ZSTD_workspace_reserve_buffer(&zc->workspace, maxNbSeq * sizeof(BYTE)); + zc->seqStore.mlCode = (BYTE*)ZSTD_workspace_reserve_buffer(&zc->workspace, maxNbSeq * sizeof(BYTE)); + zc->seqStore.ofCode = (BYTE*)ZSTD_workspace_reserve_buffer(&zc->workspace, maxNbSeq * sizeof(BYTE)); + + /* ZSTD_wildcopy() is used to copy into the literals buffer, + * so we have to oversize the buffer by WILDCOPY_OVERLENGTH bytes. + */ + zc->seqStore.litStart = (BYTE*)ZSTD_workspace_reserve_buffer(&zc->workspace, blockSize + WILDCOPY_OVERLENGTH); + zc->seqStore.maxNbLit = blockSize; + /* buffers */ zc->inBuffSize = buffInSize; - zc->inBuff = (char*)ptr; + zc->inBuff = (char*)ZSTD_workspace_reserve_buffer(&zc->workspace, buffInSize); zc->outBuffSize = buffOutSize; - zc->outBuff = zc->inBuff + buffInSize; + zc->outBuff = (char*)ZSTD_workspace_reserve_buffer(&zc->workspace, buffOutSize); return 0; } @@ -3092,7 +3239,7 @@ size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict) { if (cdict==NULL) return 0; /* support sizeof on NULL */ DEBUGLOG(5, "sizeof(*cdict) : %u", (unsigned)sizeof(*cdict)); - return cdict->workspaceSize + (cdict->dictBuffer ? cdict->dictContentSize : 0) + sizeof(*cdict); + return ZSTD_workspace_sizeof(&cdict->workspace) + (cdict->dictBuffer ? cdict->dictContentSize : 0) + sizeof(*cdict); } static size_t ZSTD_initCDict_internal( @@ -3117,15 +3264,16 @@ static size_t ZSTD_initCDict_internal( } cdict->dictContentSize = dictSize; + cdict->entropyWorkspace = (U32*)ZSTD_workspace_reserve_object(&cdict->workspace, HUF_WORKSPACE_SIZE); + + /* Reset the state to no dictionary */ ZSTD_reset_compressedBlockState(&cdict->cBlockState); - { void* const end = ZSTD_reset_matchState(&cdict->matchState, - (U32*)cdict->workspace + HUF_WORKSPACE_SIZE_U32, - &cParams, - ZSTDcrp_continue, ZSTD_resetTarget_CDict); - assert(end == (char*)cdict->workspace + cdict->workspaceSize); - (void)end; - } + FORWARD_IF_ERROR(ZSTD_reset_matchState( + &cdict->matchState, + &cdict->workspace, + &cParams, + ZSTDcrp_continue, ZSTD_resetTarget_CDict)); /* (Maybe) load the dictionary * Skips loading the dictionary if it is <= 8 bytes. */ @@ -3137,7 +3285,7 @@ static size_t ZSTD_initCDict_internal( { size_t const dictID = ZSTD_compress_insertDictionary( &cdict->cBlockState, &cdict->matchState, ¶ms, cdict->dictContent, cdict->dictContentSize, - dictContentType, ZSTD_dtlm_full, cdict->workspace); + dictContentType, ZSTD_dtlm_full, cdict->entropyWorkspace); FORWARD_IF_ERROR(dictID); assert(dictID <= (size_t)(U32)-1); cdict->dictID = (U32)dictID; @@ -3165,8 +3313,7 @@ ZSTD_CDict* ZSTD_createCDict_advanced(const void* dictBuffer, size_t dictSize, return NULL; } cdict->customMem = customMem; - cdict->workspace = workspace; - cdict->workspaceSize = workspaceSize; + ZSTD_workspace_init(&cdict->workspace, workspace, workspaceSize); if (ZSTD_isError( ZSTD_initCDict_internal(cdict, dictBuffer, dictSize, dictLoadMethod, dictContentType, @@ -3199,7 +3346,7 @@ size_t ZSTD_freeCDict(ZSTD_CDict* cdict) { if (cdict==NULL) return 0; /* support free on NULL */ { ZSTD_customMem const cMem = cdict->customMem; - ZSTD_free(cdict->workspace, cMem); + ZSTD_workspace_free(&cdict->workspace, cMem); ZSTD_free(cdict->dictBuffer, cMem); ZSTD_free(cdict, cMem); return 0; @@ -3227,24 +3374,29 @@ const ZSTD_CDict* ZSTD_initStaticCDict( ZSTD_compressionParameters cParams) { size_t const matchStateSize = ZSTD_sizeof_matchState(&cParams, /* forCCtx */ 0); - size_t const neededSize = sizeof(ZSTD_CDict) + (dictLoadMethod == ZSTD_dlm_byRef ? 0 : dictSize) + size_t const neededSize = sizeof(ZSTD_CDict) + (dictLoadMethod == ZSTD_dlm_byRef ? 0 : ZSTD_workspace_align(dictSize, sizeof(void*))) + HUF_WORKSPACE_SIZE + matchStateSize; - ZSTD_CDict* const cdict = (ZSTD_CDict*) workspace; - void* ptr; + ZSTD_CDict* cdict; + if ((size_t)workspace & 7) return NULL; /* 8-aligned */ + + { + ZSTD_CCtx_workspace ws; + ZSTD_workspace_init(&ws, workspace, workspaceSize); + cdict = (ZSTD_CDict*)ZSTD_workspace_reserve_object(&ws, sizeof(ZSTD_CDict)); + if (cdict == NULL) return NULL; + cdict->workspace = ws; + } + DEBUGLOG(4, "(workspaceSize < neededSize) : (%u < %u) => %u", (unsigned)workspaceSize, (unsigned)neededSize, (unsigned)(workspaceSize < neededSize)); if (workspaceSize < neededSize) return NULL; if (dictLoadMethod == ZSTD_dlm_byCopy) { - memcpy(cdict+1, dict, dictSize); - dict = cdict+1; - ptr = (char*)workspace + sizeof(ZSTD_CDict) + dictSize; - } else { - ptr = cdict+1; + void *dictCopy = ZSTD_workspace_reserve_object(&cdict->workspace, ZSTD_workspace_align(dictSize, sizeof(void*))); + memcpy(dictCopy, dict, dictSize); + dict = dictCopy; } - cdict->workspace = ptr; - cdict->workspaceSize = HUF_WORKSPACE_SIZE + matchStateSize; if (ZSTD_isError( ZSTD_initCDict_internal(cdict, dict, dictSize, diff --git a/lib/compress/zstd_compress_internal.h b/lib/compress/zstd_compress_internal.h index 52d6f878..287be309 100644 --- a/lib/compress/zstd_compress_internal.h +++ b/lib/compress/zstd_compress_internal.h @@ -223,6 +223,80 @@ struct ZSTD_CCtx_params_s { ZSTD_customMem customMem; }; /* typedef'd to ZSTD_CCtx_params within "zstd.h" */ +/** + * Zstd fits all its internal datastructures into a single continuous buffer, + * so that it only needs to perform a single OS allocation (or so that a buffer + * can be provided to it and it can perform no allocations at all). This buffer + * is called the workspace. + * + * Several optimizations complicate that process of allocating memory ranges + * from this workspace for each datastructure: + * + * - These different internal datastructures have different setup requirements. + * Some (e.g., the window buffer) don't care, and are happy to accept + * uninitialized memory. Others (e.g., the matchstate tables) can accept + * memory filled with unknown but bounded values (i.e., a memory area whose + * values are known to be constrained between 0 and some upper bound). If + * that constraint isn't known to be satisfied, the area has to be cleared. + * + * - We would like to reuse the objects in the workspace for multiple + * compressions without having to perform any expensive reallocation or + * reinitialization work. + * + * - We would like to be able to efficiently reuse the workspace across + * multiple compressions **even when the compression parameters change** and + * we need to resize some of the objects (where possible). + * + * Workspace Layout: + * + * In order to accomplish this, the various objects that live in the workspace + * are divided into the following categories: + * + * - Static objects: this is optionally the enclosing ZSTD_CCtx or ZSTD_CDict, + * so that literally everything fits in a single buffer. + * + * - Fixed size objects: these are fixed-size, fixed-count objects that are + * nonetheless "dynamically" allocated in the workspace so that we can + * control how they're initialized separately from the broader ZSTD_CCtx. + * Examples: + * - Entropy Workspace + * - 2 x ZSTD_compressedBlockState_t + * + * - Tables: these are any of several different datastructures (hash tables, + * chain tables, binary trees) that all respect a common format: they are + * uint32_t arrays, all of whose values are between 0 and (nextSrc - base). + * Their sizes depend on the cparams. + * + * - Uninitialized memory: these buffers are used for various purposes that + * don't require any initialization before they're used. This means they can + * be moved around at no cost for a new compression. + * - I/O Buffers + * + * [workspace, workspace + workspaceSize) + * [] + */ +typedef struct { + void* workspace; + void* workspaceEnd; + + void* objectEnd; + + // // void* tableZoneStart; + // void* tableAllocStart; + // void* tableAllocEnd; + // // void* tableZoneEnd; + + // void* seqEnd; + + // void* bufferBegin; + + void* allocEnd; + int allocFailed; + + int workspaceOversizedDuration; + int staticAllocDone; +} ZSTD_CCtx_workspace; + struct ZSTD_CCtx_s { ZSTD_compressionStage_e stage; int cParamsChanged; /* == 1 if cParams(except wlog) or compression level are changed in requestedParams. Triggers transmission of new params to ZSTDMT (if available) then reset to 0. */ @@ -231,9 +305,7 @@ struct ZSTD_CCtx_s { ZSTD_CCtx_params appliedParams; U32 dictID; - int workspaceOversizedDuration; - void* workspace; - size_t workspaceSize; + ZSTD_CCtx_workspace workspace; /* manages buffer for dynamic allocations */ size_t blockSize; unsigned long long pledgedSrcSizePlusOne; /* this way, 0 (default) == unknown */ unsigned long long consumedSrcSize; From 6177354b362e56afb22220b67a25f32ddf6a9d81 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Mon, 12 Aug 2019 14:22:54 -0400 Subject: [PATCH 04/33] Begin Introducing Phases --- lib/compress/zstd_compress.c | 129 +++++++++++++------------- lib/compress/zstd_compress_internal.h | 8 +- 2 files changed, 73 insertions(+), 64 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index b0e57e4d..8d7d4439 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -48,11 +48,6 @@ size_t ZSTD_compressBound(size_t srcSize) { * because it's sized to handle a worst case scenario which rarely happens. * In which case, resize it down to free some memory */ -static size_t ZSTD_workspace_round_size(size_t size) { - // return size + sizeof(void*) - 1 - ((size - 1) & (sizeof(void*) - 1)); - return size + 3 - ((size - 1) & 3); -} - /** * Align must be a power of 2. */ @@ -61,18 +56,24 @@ static size_t ZSTD_workspace_align(size_t size, size_t align) { // return size + 3 - ((size - 1) & 3); } -static void* ZSTD_workspace_reserve(ZSTD_CCtx_workspace* ws, size_t bytes) { +static void* ZSTD_workspace_reserve(ZSTD_CCtx_workspace* ws, size_t bytes, ZSTD_workspace_alloc_phase_e phase) { /* TODO(felixh): alignment */ void* alloc = ws->allocEnd; void* newAllocEnd = (BYTE *)ws->allocEnd + bytes; DEBUGLOG(3, "wksp: reserving %zd bytes, %zd bytes remaining", bytes, (BYTE *)ws->workspaceEnd - (BYTE *)newAllocEnd); + assert(phase >= ws->phase); + if (phase > ws->phase) { + if (ws->phase <= ZSTD_workspace_alloc_buffers) { + + } + ws->phase = phase; + } assert(newAllocEnd <= ws->workspaceEnd); if (newAllocEnd > ws->workspaceEnd) { ws->allocFailed = 1; return NULL; } ws->allocEnd = newAllocEnd; - ws->staticAllocDone = 1; return alloc; } @@ -86,7 +87,7 @@ static void* ZSTD_workspace_reserve_object(ZSTD_CCtx_workspace* ws, size_t bytes assert(ws->allocEnd == ws->objectEnd); DEBUGLOG(3, "wksp: reserving %zd bytes object (rounded to %zd), %zd bytes remaining", bytes, roundedBytes, (BYTE *)ws->workspaceEnd - (BYTE *)end); assert((bytes & (sizeof(void*)-1)) == 0); // TODO ??? - if (ws->staticAllocDone || end > ws->workspaceEnd) { + if (ws->phase != ZSTD_workspace_alloc_objects || end > ws->workspaceEnd) { DEBUGLOG(3, "wksp: object alloc failed!"); ws->allocFailed = 1; return NULL; @@ -103,8 +104,7 @@ static void* ZSTD_workspace_reserve_object(ZSTD_CCtx_workspace* ws, size_t bytes */ static void* ZSTD_workspace_reserve_table(ZSTD_CCtx_workspace* ws, size_t bytes) { assert((bytes & (sizeof(U32)-1)) == 0); // TODO ??? - ws->staticAllocDone = 1; - return ZSTD_workspace_reserve(ws, ZSTD_workspace_align(bytes, sizeof(unsigned))); + return ZSTD_workspace_reserve(ws, ZSTD_workspace_align(bytes, sizeof(unsigned)), ZSTD_workspace_alloc_aligned); } /** @@ -112,20 +112,19 @@ static void* ZSTD_workspace_reserve_table(ZSTD_CCtx_workspace* ws, size_t bytes) */ static void* ZSTD_workspace_reserve_aligned(ZSTD_CCtx_workspace* ws, size_t bytes) { assert((bytes & (sizeof(U32)-1)) == 0); // TODO ??? - ws->staticAllocDone = 1; - return ZSTD_workspace_reserve(ws, ZSTD_workspace_align(bytes, sizeof(unsigned))); + return ZSTD_workspace_reserve(ws, ZSTD_workspace_align(bytes, sizeof(unsigned)), ZSTD_workspace_alloc_aligned); } /** * Unaligned. */ -static void* ZSTD_workspace_reserve_buffer(ZSTD_CCtx_workspace* ws, size_t bytes) { - ws->staticAllocDone = 1; - return ZSTD_workspace_reserve(ws, bytes); +static BYTE* ZSTD_workspace_reserve_buffer(ZSTD_CCtx_workspace* ws, size_t bytes) { + return (BYTE*)ZSTD_workspace_reserve(ws, bytes, ZSTD_workspace_alloc_buffers); } // TODO static int ZSTD_workspace_bump_oversized_duration(ZSTD_CCtx_workspace* ws) { + (void)ws; // if (((BYTE*)ws->allocEnd - (BYTE*)ws->workspace) * ZSTD_WORKSPACETOOLARGE_FACTOR < (BYTE*)ws->workspaceEnd - (BYTE*)ws->workspace) { // ws->workspaceOversizedDuration++; // } else { @@ -140,6 +139,9 @@ static void ZSTD_workspace_clear(ZSTD_CCtx_workspace* ws) { ZSTD_workspace_bump_oversized_duration(ws); ws->allocEnd = ws->objectEnd; ws->allocFailed = 0; + if (ws->phase > ZSTD_workspace_alloc_buffers) { + ws->phase = ZSTD_workspace_alloc_buffers; + } // ws->table = NULL; // ws->tableEnd = NULL; @@ -153,9 +155,9 @@ static void ZSTD_workspace_init(ZSTD_CCtx_workspace* ws, void* start, size_t siz ws->workspace = start; ws->workspaceEnd = (BYTE*)start + size; ws->objectEnd = ws->workspace; + ws->phase = ZSTD_workspace_alloc_objects; ZSTD_workspace_clear(ws); ws->workspaceOversizedDuration = 0; - ws->staticAllocDone = 0; } static size_t ZSTD_workspace_create(ZSTD_CCtx_workspace* ws, size_t size, ZSTD_customMem customMem) { @@ -1500,7 +1502,7 @@ static size_t ZSTD_continueCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx_params* params, return 0; } -typedef enum { ZSTDcrp_continue, ZSTDcrp_noMemset } ZSTD_compResetPolicy_e; +typedef enum { ZSTDcrp_continue, ZSTDcrp_noMemset, ZSTDcrp_noRealloc } ZSTD_compResetPolicy_e; typedef enum { ZSTD_resetTarget_CDict, ZSTD_resetTarget_CCtx } ZSTD_resetTarget_e; @@ -1525,19 +1527,22 @@ ZSTD_reset_matchState(ZSTD_matchState_t* ms, assert(!ZSTD_workspace_reserve_failed(ws)); /* check that allocation hasn't already failed */ - DEBUGLOG(5, "reserving table space"); - /* table Space */ - ms->hashTable = (U32*)ZSTD_workspace_reserve_table(ws, hSize * sizeof(U32)); - ms->chainTable = (U32*)ZSTD_workspace_reserve_table(ws, chainSize * sizeof(U32)); - ms->hashTable3 = (U32*)ZSTD_workspace_reserve_table(ws, h3Size * sizeof(U32)); - RETURN_ERROR_IF(ZSTD_workspace_reserve_failed(ws), memory_allocation, - "failed a workspace allocation in ZSTD_reset_matchState"); - DEBUGLOG(4, "reset table : %u", crp!=ZSTDcrp_noMemset); - if (crp!=ZSTDcrp_noMemset) { - /* reset tables only */ - memset(ms->hashTable, 0, hSize * sizeof(U32)); - memset(ms->chainTable, 0, chainSize * sizeof(U32)); - memset(ms->hashTable3, 0, h3Size * sizeof(U32)); + if (crp!=ZSTDcrp_noRealloc) { + DEBUGLOG(5, "reserving table space"); + /* table Space */ + ms->hashTable = (U32*)ZSTD_workspace_reserve_table(ws, hSize * sizeof(U32)); + ms->chainTable = (U32*)ZSTD_workspace_reserve_table(ws, chainSize * sizeof(U32)); + ms->hashTable3 = (U32*)ZSTD_workspace_reserve_table(ws, h3Size * sizeof(U32)); + RETURN_ERROR_IF(ZSTD_workspace_reserve_failed(ws), memory_allocation, + "failed a workspace allocation in ZSTD_reset_matchState"); + + DEBUGLOG(4, "reset table : %u", crp!=ZSTDcrp_noMemset); + if (crp!=ZSTDcrp_noMemset) { + /* reset tables only */ + memset(ms->hashTable, 0, hSize * sizeof(U32)); + memset(ms->chainTable, 0, chainSize * sizeof(U32)); + memset(ms->hashTable3, 0, h3Size * sizeof(U32)); + } } /* opt parser space */ @@ -1600,7 +1605,7 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, &zc->blockState.matchState, &zc->workspace, ¶ms.cParams, - crp, ZSTD_resetTarget_CCtx)); + ZSTDcrp_noRealloc, ZSTD_resetTarget_CCtx)); } return ZSTD_continueCCtx(zc, ¶ms, pledgedSrcSize); } } } @@ -1691,16 +1696,43 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, ZSTD_reset_compressedBlockState(zc->blockState.prevCBlock); + /* ZSTD_wildcopy() is used to copy into the literals buffer, + * so we have to oversize the buffer by WILDCOPY_OVERLENGTH bytes. + */ + zc->seqStore.litStart = ZSTD_workspace_reserve_buffer(&zc->workspace, blockSize + WILDCOPY_OVERLENGTH); + zc->seqStore.maxNbLit = blockSize; + + /* buffers */ + zc->inBuffSize = buffInSize; + zc->inBuff = (char*)ZSTD_workspace_reserve_buffer(&zc->workspace, buffInSize); + zc->outBuffSize = buffOutSize; + zc->outBuff = (char*)ZSTD_workspace_reserve_buffer(&zc->workspace, buffOutSize); + + /* ldm bucketOffsets table */ + if (params.ldmParams.enableLdm) { + size_t const ldmBucketSize = + ((size_t)1) << (params.ldmParams.hashLog - + params.ldmParams.bucketSizeLog); + zc->ldmState.bucketOffsets = ZSTD_workspace_reserve_buffer(&zc->workspace, ldmBucketSize); + memset(zc->ldmState.bucketOffsets, 0, ldmBucketSize); + } + + /* sequences storage */ + ZSTD_referenceExternalSequences(zc, NULL, 0); + zc->seqStore.maxNbSeq = maxNbSeq; + zc->seqStore.llCode = ZSTD_workspace_reserve_buffer(&zc->workspace, maxNbSeq * sizeof(BYTE)); + zc->seqStore.mlCode = ZSTD_workspace_reserve_buffer(&zc->workspace, maxNbSeq * sizeof(BYTE)); + zc->seqStore.ofCode = ZSTD_workspace_reserve_buffer(&zc->workspace, maxNbSeq * sizeof(BYTE)); + zc->seqStore.sequencesStart = (seqDef*)ZSTD_workspace_reserve_aligned(&zc->workspace, maxNbSeq * sizeof(seqDef)); + FORWARD_IF_ERROR(ZSTD_reset_matchState( &zc->blockState.matchState, &zc->workspace, ¶ms.cParams, crp, ZSTD_resetTarget_CCtx)); - DEBUGLOG(3, "Done allocating match state"); - /* ldm hash table */ - /* initialize bucketOffsets table later for pointer alignment */ + /* initialize bucketOffsets table separately for pointer alignment */ if (params.ldmParams.enableLdm) { size_t const ldmHSize = ((size_t)1) << params.ldmParams.hashLog; zc->ldmState.hashTable = (ldmEntry_t*)ZSTD_workspace_reserve_aligned(&zc->workspace, ldmHSize * sizeof(ldmEntry_t)); @@ -1709,37 +1741,8 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, zc->maxNbLdmSequences = maxNbLdmSeq; memset(&zc->ldmState.window, 0, sizeof(zc->ldmState.window)); - } - - /* ldm bucketOffsets table */ - if (params.ldmParams.enableLdm) { - size_t const ldmBucketSize = - ((size_t)1) << (params.ldmParams.hashLog - - params.ldmParams.bucketSizeLog); - zc->ldmState.bucketOffsets = (BYTE*)ZSTD_workspace_reserve_aligned(&zc->workspace, ldmBucketSize); - memset(zc->ldmState.bucketOffsets, 0, ldmBucketSize); ZSTD_window_clear(&zc->ldmState.window); } - ZSTD_referenceExternalSequences(zc, NULL, 0); - - /* sequences storage */ - zc->seqStore.maxNbSeq = maxNbSeq; - zc->seqStore.sequencesStart = (seqDef*)ZSTD_workspace_reserve_aligned(&zc->workspace, maxNbSeq * sizeof(seqDef)); - zc->seqStore.llCode = (BYTE*)ZSTD_workspace_reserve_buffer(&zc->workspace, maxNbSeq * sizeof(BYTE)); - zc->seqStore.mlCode = (BYTE*)ZSTD_workspace_reserve_buffer(&zc->workspace, maxNbSeq * sizeof(BYTE)); - zc->seqStore.ofCode = (BYTE*)ZSTD_workspace_reserve_buffer(&zc->workspace, maxNbSeq * sizeof(BYTE)); - - /* ZSTD_wildcopy() is used to copy into the literals buffer, - * so we have to oversize the buffer by WILDCOPY_OVERLENGTH bytes. - */ - zc->seqStore.litStart = (BYTE*)ZSTD_workspace_reserve_buffer(&zc->workspace, blockSize + WILDCOPY_OVERLENGTH); - zc->seqStore.maxNbLit = blockSize; - - /* buffers */ - zc->inBuffSize = buffInSize; - zc->inBuff = (char*)ZSTD_workspace_reserve_buffer(&zc->workspace, buffInSize); - zc->outBuffSize = buffOutSize; - zc->outBuff = (char*)ZSTD_workspace_reserve_buffer(&zc->workspace, buffOutSize); return 0; } diff --git a/lib/compress/zstd_compress_internal.h b/lib/compress/zstd_compress_internal.h index 287be309..60c00799 100644 --- a/lib/compress/zstd_compress_internal.h +++ b/lib/compress/zstd_compress_internal.h @@ -223,6 +223,12 @@ struct ZSTD_CCtx_params_s { ZSTD_customMem customMem; }; /* typedef'd to ZSTD_CCtx_params within "zstd.h" */ +typedef enum { + ZSTD_workspace_alloc_objects, + ZSTD_workspace_alloc_buffers, + ZSTD_workspace_alloc_aligned +} ZSTD_workspace_alloc_phase_e; + /** * Zstd fits all its internal datastructures into a single continuous buffer, * so that it only needs to perform a single OS allocation (or so that a buffer @@ -294,7 +300,7 @@ typedef struct { int allocFailed; int workspaceOversizedDuration; - int staticAllocDone; + ZSTD_workspace_alloc_phase_e phase; } ZSTD_CCtx_workspace; struct ZSTD_CCtx_s { From e69b67e33ad26b0250e039da089762c8a19a44d6 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Mon, 12 Aug 2019 18:36:48 -0400 Subject: [PATCH 05/33] Alloc Tables Separately --- lib/compress/zstd_compress.c | 99 +++++++++++++++++---------- lib/compress/zstd_compress_internal.h | 14 ++-- 2 files changed, 68 insertions(+), 45 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 8d7d4439..012ea0cb 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -58,9 +58,10 @@ static size_t ZSTD_workspace_align(size_t size, size_t align) { static void* ZSTD_workspace_reserve(ZSTD_CCtx_workspace* ws, size_t bytes, ZSTD_workspace_alloc_phase_e phase) { /* TODO(felixh): alignment */ - void* alloc = ws->allocEnd; - void* newAllocEnd = (BYTE *)ws->allocEnd + bytes; - DEBUGLOG(3, "wksp: reserving %zd bytes, %zd bytes remaining", bytes, (BYTE *)ws->workspaceEnd - (BYTE *)newAllocEnd); + void* alloc = (BYTE *)ws->allocStart - bytes; + void* bottom = ws->tableEnd; + DEBUGLOG(3, "wksp: reserving align %zd bytes, %zd bytes remaining", + bytes, (BYTE *)alloc - (BYTE *)bottom); assert(phase >= ws->phase); if (phase > ws->phase) { if (ws->phase <= ZSTD_workspace_alloc_buffers) { @@ -68,12 +69,42 @@ static void* ZSTD_workspace_reserve(ZSTD_CCtx_workspace* ws, size_t bytes, ZSTD_ } ws->phase = phase; } - assert(newAllocEnd <= ws->workspaceEnd); - if (newAllocEnd > ws->workspaceEnd) { + assert(alloc >= bottom); + if (alloc < bottom) { ws->allocFailed = 1; return NULL; } - ws->allocEnd = newAllocEnd; + ws->allocStart = alloc; + return alloc; +} + +/** + * Aligned on sizeof(unsigned). These buffers have the special property that + * their values remain constrained, allowing us to re-use them without + * memset()-ing them. + */ +static void* ZSTD_workspace_reserve_table(ZSTD_CCtx_workspace* ws, size_t bytes) { + /* TODO(felixh): alignment */ + const ZSTD_workspace_alloc_phase_e phase = ZSTD_workspace_alloc_aligned; + void* alloc = ws->tableEnd; + void* end = (BYTE *)alloc + bytes; + void* top = ws->allocStart; + DEBUGLOG(3, "wksp: reserving table %zd bytes, %zd bytes remaining", + bytes, (BYTE *)top - (BYTE *)end); + assert((bytes & (sizeof(U32)-1)) == 0); // TODO ??? + assert(phase >= ws->phase); + if (phase > ws->phase) { + if (ws->phase <= ZSTD_workspace_alloc_buffers) { + + } + ws->phase = phase; + } + assert(end <= top); + if (end > top) { + ws->allocFailed = 1; + return NULL; + } + ws->tableEnd = end; return alloc; } @@ -84,7 +115,6 @@ static void* ZSTD_workspace_reserve_object(ZSTD_CCtx_workspace* ws, size_t bytes size_t roundedBytes = ZSTD_workspace_align(bytes, sizeof(void*)); void* start = ws->objectEnd; void* end = (BYTE*)start + roundedBytes; - assert(ws->allocEnd == ws->objectEnd); DEBUGLOG(3, "wksp: reserving %zd bytes object (rounded to %zd), %zd bytes remaining", bytes, roundedBytes, (BYTE *)ws->workspaceEnd - (BYTE *)end); assert((bytes & (sizeof(void*)-1)) == 0); // TODO ??? if (ws->phase != ZSTD_workspace_alloc_objects || end > ws->workspaceEnd) { @@ -93,20 +123,10 @@ static void* ZSTD_workspace_reserve_object(ZSTD_CCtx_workspace* ws, size_t bytes return NULL; } ws->objectEnd = end; - ws->allocEnd = end; + ws->tableEnd = end; return start; } -/** - * Aligned on sizeof(unsigned). These buffers have the special property that - * their values remain constrained, allowing us to re-use them without - * memset()-ing them. - */ -static void* ZSTD_workspace_reserve_table(ZSTD_CCtx_workspace* ws, size_t bytes) { - assert((bytes & (sizeof(U32)-1)) == 0); // TODO ??? - return ZSTD_workspace_reserve(ws, ZSTD_workspace_align(bytes, sizeof(unsigned)), ZSTD_workspace_alloc_aligned); -} - /** * Aligned on sizeof(unsigned). */ @@ -134,10 +154,15 @@ static int ZSTD_workspace_bump_oversized_duration(ZSTD_CCtx_workspace* ws) { return 0; } +static void ZSTD_workspace_clear_tables(ZSTD_CCtx_workspace* ws) { + ws->tableEnd = ws->objectEnd; +} + static void ZSTD_workspace_clear(ZSTD_CCtx_workspace* ws) { DEBUGLOG(3, "wksp: clearing!"); ZSTD_workspace_bump_oversized_duration(ws); - ws->allocEnd = ws->objectEnd; + ws->tableEnd = ws->objectEnd; + ws->allocStart = ws->workspaceEnd; ws->allocFailed = 0; if (ws->phase > ZSTD_workspace_alloc_buffers) { ws->phase = ZSTD_workspace_alloc_buffers; @@ -177,7 +202,7 @@ static void ZSTD_workspace_free(ZSTD_CCtx_workspace* ws, ZSTD_customMem customMe } static int ZSTD_workspace_check_available(ZSTD_CCtx_workspace* ws, size_t minFree) { - return (size_t)((BYTE*)ws->workspaceEnd - (BYTE*)ws->allocEnd) >= minFree; + return (size_t)((BYTE*)ws->allocStart - (BYTE*)ws->tableEnd) >= minFree; } static int ZSTD_workspace_check_wasteful(ZSTD_CCtx_workspace* ws, size_t minFree) { @@ -1527,22 +1552,22 @@ ZSTD_reset_matchState(ZSTD_matchState_t* ms, assert(!ZSTD_workspace_reserve_failed(ws)); /* check that allocation hasn't already failed */ - if (crp!=ZSTDcrp_noRealloc) { - DEBUGLOG(5, "reserving table space"); - /* table Space */ - ms->hashTable = (U32*)ZSTD_workspace_reserve_table(ws, hSize * sizeof(U32)); - ms->chainTable = (U32*)ZSTD_workspace_reserve_table(ws, chainSize * sizeof(U32)); - ms->hashTable3 = (U32*)ZSTD_workspace_reserve_table(ws, h3Size * sizeof(U32)); - RETURN_ERROR_IF(ZSTD_workspace_reserve_failed(ws), memory_allocation, - "failed a workspace allocation in ZSTD_reset_matchState"); - - DEBUGLOG(4, "reset table : %u", crp!=ZSTDcrp_noMemset); - if (crp!=ZSTDcrp_noMemset) { - /* reset tables only */ - memset(ms->hashTable, 0, hSize * sizeof(U32)); - memset(ms->chainTable, 0, chainSize * sizeof(U32)); - memset(ms->hashTable3, 0, h3Size * sizeof(U32)); - } + ZSTD_workspace_clear_tables(ws); + + DEBUGLOG(5, "reserving table space"); + /* table Space */ + ms->hashTable = (U32*)ZSTD_workspace_reserve_table(ws, hSize * sizeof(U32)); + ms->chainTable = (U32*)ZSTD_workspace_reserve_table(ws, chainSize * sizeof(U32)); + ms->hashTable3 = (U32*)ZSTD_workspace_reserve_table(ws, h3Size * sizeof(U32)); + RETURN_ERROR_IF(ZSTD_workspace_reserve_failed(ws), memory_allocation, + "failed a workspace allocation in ZSTD_reset_matchState"); + + DEBUGLOG(4, "reset table : %u", crp!=ZSTDcrp_noMemset); + if (crp!=ZSTDcrp_noMemset) { + /* reset tables only */ + memset(ms->hashTable, 0, hSize * sizeof(U32)); + memset(ms->chainTable, 0, chainSize * sizeof(U32)); + memset(ms->hashTable3, 0, h3Size * sizeof(U32)); } /* opt parser space */ @@ -1744,6 +1769,8 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, ZSTD_window_clear(&zc->ldmState.window); } + assert(!ZSTD_workspace_check_available(&zc->workspace, 1)); + return 0; } } diff --git a/lib/compress/zstd_compress_internal.h b/lib/compress/zstd_compress_internal.h index 60c00799..d067c095 100644 --- a/lib/compress/zstd_compress_internal.h +++ b/lib/compress/zstd_compress_internal.h @@ -255,6 +255,9 @@ typedef enum { * * Workspace Layout: * + * [ ... workspace ... ] + * [objects][tables ... ->] free space [<- ... aligned][<- ... buffers] + * * In order to accomplish this, the various objects that live in the workspace * are divided into the following categories: * @@ -287,18 +290,11 @@ typedef struct { void* objectEnd; - // // void* tableZoneStart; - // void* tableAllocStart; - // void* tableAllocEnd; - // // void* tableZoneEnd; + void* tableEnd; - // void* seqEnd; + void* allocStart; - // void* bufferBegin; - - void* allocEnd; int allocFailed; - int workspaceOversizedDuration; ZSTD_workspace_alloc_phase_e phase; } ZSTD_CCtx_workspace; From 75d574368b4ca876a4bfca884e13cb22d97b075a Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Mon, 12 Aug 2019 19:24:24 -0400 Subject: [PATCH 06/33] When Loading Dict By Copy, Always Put it in the Workspace --- lib/compress/zstd_compress.c | 24 +++++++----------------- lib/compress/zstd_compress_internal.h | 1 + 2 files changed, 8 insertions(+), 17 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 012ea0cb..b975efda 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -132,7 +132,7 @@ static void* ZSTD_workspace_reserve_object(ZSTD_CCtx_workspace* ws, size_t bytes */ static void* ZSTD_workspace_reserve_aligned(ZSTD_CCtx_workspace* ws, size_t bytes) { assert((bytes & (sizeof(U32)-1)) == 0); // TODO ??? - return ZSTD_workspace_reserve(ws, ZSTD_workspace_align(bytes, sizeof(unsigned)), ZSTD_workspace_alloc_aligned); + return ZSTD_workspace_reserve(ws, ZSTD_workspace_align(bytes, sizeof(U32)), ZSTD_workspace_alloc_aligned); } /** @@ -222,7 +222,6 @@ static int ZSTD_workspace_reserve_failed(const ZSTD_CCtx_workspace* ws) { * Context memory management ***************************************/ struct ZSTD_CDict_s { - void* dictBuffer; const void* dictContent; size_t dictContentSize; U32* entropyWorkspace; /* entropy workspace of HUF_WORKSPACE_SIZE bytes */ @@ -3256,7 +3255,7 @@ size_t ZSTD_estimateCDictSize_advanced( { DEBUGLOG(5, "sizeof(ZSTD_CDict) : %u", (unsigned)sizeof(ZSTD_CDict)); return sizeof(ZSTD_CDict) + HUF_WORKSPACE_SIZE + ZSTD_sizeof_matchState(&cParams, /* forCCtx */ 0) - + (dictLoadMethod == ZSTD_dlm_byRef ? 0 : dictSize); + + (dictLoadMethod == ZSTD_dlm_byRef ? 0 : ZSTD_workspace_align(dictSize, sizeof(void *))); } size_t ZSTD_estimateCDictSize(size_t dictSize, int compressionLevel) @@ -3269,7 +3268,7 @@ size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict) { if (cdict==NULL) return 0; /* support sizeof on NULL */ DEBUGLOG(5, "sizeof(*cdict) : %u", (unsigned)sizeof(*cdict)); - return ZSTD_workspace_sizeof(&cdict->workspace) + (cdict->dictBuffer ? cdict->dictContentSize : 0) + sizeof(*cdict); + return ZSTD_workspace_sizeof(&cdict->workspace) + sizeof(*cdict); } static size_t ZSTD_initCDict_internal( @@ -3283,13 +3282,11 @@ static size_t ZSTD_initCDict_internal( assert(!ZSTD_checkCParams(cParams)); cdict->matchState.cParams = cParams; if ((dictLoadMethod == ZSTD_dlm_byRef) || (!dictBuffer) || (!dictSize)) { - cdict->dictBuffer = NULL; cdict->dictContent = dictBuffer; } else { - void* const internalBuffer = ZSTD_malloc(dictSize, cdict->customMem); - cdict->dictBuffer = internalBuffer; - cdict->dictContent = internalBuffer; + void *internalBuffer = ZSTD_workspace_reserve_object(&cdict->workspace, ZSTD_workspace_align(dictSize, sizeof(void*))); RETURN_ERROR_IF(!internalBuffer, memory_allocation); + cdict->dictContent = internalBuffer; memcpy(internalBuffer, dictBuffer, dictSize); } cdict->dictContentSize = dictSize; @@ -3334,7 +3331,7 @@ ZSTD_CDict* ZSTD_createCDict_advanced(const void* dictBuffer, size_t dictSize, if (!customMem.customAlloc ^ !customMem.customFree) return NULL; { ZSTD_CDict* const cdict = (ZSTD_CDict*)ZSTD_malloc(sizeof(ZSTD_CDict), customMem); - size_t const workspaceSize = HUF_WORKSPACE_SIZE + ZSTD_sizeof_matchState(&cParams, /* forCCtx */ 0); + size_t const workspaceSize = HUF_WORKSPACE_SIZE + ZSTD_sizeof_matchState(&cParams, /* forCCtx */ 0) + (dictLoadMethod == ZSTD_dlm_byRef ? 0 : ZSTD_workspace_align(dictSize, sizeof(void*))); void* const workspace = ZSTD_malloc(workspaceSize, customMem); if (!cdict || !workspace) { @@ -3377,7 +3374,6 @@ size_t ZSTD_freeCDict(ZSTD_CDict* cdict) if (cdict==NULL) return 0; /* support free on NULL */ { ZSTD_customMem const cMem = cdict->customMem; ZSTD_workspace_free(&cdict->workspace, cMem); - ZSTD_free(cdict->dictBuffer, cMem); ZSTD_free(cdict, cMem); return 0; } @@ -3422,15 +3418,9 @@ const ZSTD_CDict* ZSTD_initStaticCDict( (unsigned)workspaceSize, (unsigned)neededSize, (unsigned)(workspaceSize < neededSize)); if (workspaceSize < neededSize) return NULL; - if (dictLoadMethod == ZSTD_dlm_byCopy) { - void *dictCopy = ZSTD_workspace_reserve_object(&cdict->workspace, ZSTD_workspace_align(dictSize, sizeof(void*))); - memcpy(dictCopy, dict, dictSize); - dict = dictCopy; - } - if (ZSTD_isError( ZSTD_initCDict_internal(cdict, dict, dictSize, - ZSTD_dlm_byRef, dictContentType, + dictLoadMethod, dictContentType, cParams) )) return NULL; diff --git a/lib/compress/zstd_compress_internal.h b/lib/compress/zstd_compress_internal.h index d067c095..71226757 100644 --- a/lib/compress/zstd_compress_internal.h +++ b/lib/compress/zstd_compress_internal.h @@ -270,6 +270,7 @@ typedef enum { * Examples: * - Entropy Workspace * - 2 x ZSTD_compressedBlockState_t + * - CDict dictionary contents sometimes??? // TODO * * - Tables: these are any of several different datastructures (hash tables, * chain tables, binary trees) that all respect a common format: they are From e936b73889395cdadd6a5855dc001520f91dddfc Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Mon, 12 Aug 2019 20:01:57 -0400 Subject: [PATCH 07/33] Remove Overly-Restrictive Assert --- lib/compress/zstd_compress.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index b975efda..f081f838 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -201,8 +201,12 @@ static void ZSTD_workspace_free(ZSTD_CCtx_workspace* ws, ZSTD_customMem customMe ZSTD_workspace_clear(ws); } +static size_t ZSTD_workspace_available_space(ZSTD_CCtx_workspace* ws) { + return (size_t)((BYTE*)ws->allocStart - (BYTE*)ws->tableEnd); +} + static int ZSTD_workspace_check_available(ZSTD_CCtx_workspace* ws, size_t minFree) { - return (size_t)((BYTE*)ws->allocStart - (BYTE*)ws->tableEnd) >= minFree; + return ZSTD_workspace_available_space(ws) >= minFree; } static int ZSTD_workspace_check_wasteful(ZSTD_CCtx_workspace* ws, size_t minFree) { @@ -1768,7 +1772,7 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, ZSTD_window_clear(&zc->ldmState.window); } - assert(!ZSTD_workspace_check_available(&zc->workspace, 1)); + DEBUGLOG(3, "wksp: finished allocating, %zd bytes remain available", ZSTD_workspace_available_space(&zc->workspace)); return 0; } From 88c2fcd0ee13d566fd34886bd3a8ff921eb246db Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Tue, 13 Aug 2019 11:57:20 -0400 Subject: [PATCH 08/33] Align Alloc Pointer When Transitioning from Buffers to Aligned Allocs --- lib/compress/zstd_compress.c | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index f081f838..78a57945 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -64,8 +64,19 @@ static void* ZSTD_workspace_reserve(ZSTD_CCtx_workspace* ws, size_t bytes, ZSTD_ bytes, (BYTE *)alloc - (BYTE *)bottom); assert(phase >= ws->phase); if (phase > ws->phase) { - if (ws->phase <= ZSTD_workspace_alloc_buffers) { - + if (ws->phase < ZSTD_workspace_alloc_buffers && + phase >= ZSTD_workspace_alloc_buffers) { + } + if (ws->phase < ZSTD_workspace_alloc_aligned && + phase >= ZSTD_workspace_alloc_aligned) { + /* If unaligned allocations down from a too-large top have left us + * unaligned, we need to realign our alloc ptr. Technically, this + * can consume space that is unaccounted for in the neededSpace + * calculation. However, I believe this can only happen when the + * workspace is too large, and specifically when it is too large + * by a larger margin than the space that will be consumed. */ + /* TODO: cleaner, compiler warning friendly way to do this??? */ + alloc = (BYTE*)alloc - ((size_t)alloc & (sizeof(U32)-1)); } ws->phase = phase; } From 58b69ab15c9088a7c8afbc01431f3e33add54ed5 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Wed, 14 Aug 2019 11:35:53 -0400 Subject: [PATCH 09/33] Only the CCtx Itself Needs to be Cleared during Static CCtx Init --- lib/compress/zstd_compress.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 78a57945..a8bf0d92 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -281,7 +281,7 @@ ZSTD_CCtx* ZSTD_initStaticCCtx(void *workspace, size_t workspaceSize) ZSTD_CCtx* const cctx = (ZSTD_CCtx*) workspace; if (workspaceSize <= sizeof(ZSTD_CCtx)) return NULL; /* minimum size */ if ((size_t)workspace & 7) return NULL; /* must be 8-aligned */ - memset(workspace, 0, workspaceSize); /* may be a bit generous, could memset be smaller ? */ /* TODO(felixh): investigate */ + memset(cctx, 0, sizeof(ZSTD_CCtx)); cctx->staticSize = workspaceSize; ZSTD_workspace_init(&cctx->workspace, (void*)(cctx+1), workspaceSize - sizeof(ZSTD_CCtx)); From 65057cf009a749e66b045b0a709f5e95c6c8abd1 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Wed, 14 Aug 2019 14:44:17 -0400 Subject: [PATCH 10/33] Rewrite ZSTD_initStaticCCtx to Alloc CCtx in Workspace --- lib/compress/zstd_compress.c | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index a8bf0d92..f27698c0 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -278,12 +278,19 @@ ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem) ZSTD_CCtx* ZSTD_initStaticCCtx(void *workspace, size_t workspaceSize) { - ZSTD_CCtx* const cctx = (ZSTD_CCtx*) workspace; + ZSTD_CCtx_workspace tmpWorkspace; + ZSTD_CCtx* cctx; if (workspaceSize <= sizeof(ZSTD_CCtx)) return NULL; /* minimum size */ if ((size_t)workspace & 7) return NULL; /* must be 8-aligned */ + ZSTD_workspace_init(&tmpWorkspace, workspace, workspaceSize); + + cctx = (ZSTD_CCtx*)ZSTD_workspace_reserve_object(&tmpWorkspace, sizeof(ZSTD_CCtx)); + if (cctx == NULL) { + return NULL; + } memset(cctx, 0, sizeof(ZSTD_CCtx)); + cctx->workspace = tmpWorkspace; cctx->staticSize = workspaceSize; - ZSTD_workspace_init(&cctx->workspace, (void*)(cctx+1), workspaceSize - sizeof(ZSTD_CCtx)); /* statically sized space. entropyWorkspace never moves (but prev/next block swap places) */ if (!ZSTD_workspace_check_available(&cctx->workspace, HUF_WORKSPACE_SIZE + 2 * sizeof(ZSTD_compressedBlockState_t))) return NULL; From 7a2416a863803956cd64979f04f46858d369cfbc Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Wed, 14 Aug 2019 16:48:01 -0400 Subject: [PATCH 11/33] Allocate CDict in Workspace (Rather than in Separate Allocation) --- lib/compress/zstd_compress.c | 32 +++++++++++++++++++++++++------- 1 file changed, 25 insertions(+), 7 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index f27698c0..4e0b76b1 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -325,7 +325,11 @@ static void ZSTD_freeCCtxContent(ZSTD_CCtx* cctx) { assert(cctx != NULL); assert(cctx->staticSize == 0); - ZSTD_workspace_free(&cctx->workspace, cctx->customMem); + /* Only free workspace if cctx not in workspace, otherwise the workspace + * will be freed when the cctx itself is freed. */ + if ((void*)cctx->workspace.workspace != (void*)cctx) { + ZSTD_workspace_free(&cctx->workspace, cctx->customMem); + } ZSTD_clearAllDicts(cctx); #ifdef ZSTD_MULTITHREAD ZSTDMT_freeCCtx(cctx->mtctx); cctx->mtctx = NULL; @@ -3352,17 +3356,27 @@ ZSTD_CDict* ZSTD_createCDict_advanced(const void* dictBuffer, size_t dictSize, DEBUGLOG(3, "ZSTD_createCDict_advanced, mode %u", (unsigned)dictContentType); if (!customMem.customAlloc ^ !customMem.customFree) return NULL; - { ZSTD_CDict* const cdict = (ZSTD_CDict*)ZSTD_malloc(sizeof(ZSTD_CDict), customMem); - size_t const workspaceSize = HUF_WORKSPACE_SIZE + ZSTD_sizeof_matchState(&cParams, /* forCCtx */ 0) + (dictLoadMethod == ZSTD_dlm_byRef ? 0 : ZSTD_workspace_align(dictSize, sizeof(void*))); + { size_t const workspaceSize = + sizeof(ZSTD_CDict) + + HUF_WORKSPACE_SIZE + + ZSTD_sizeof_matchState(&cParams, /* forCCtx */ 0) + + (dictLoadMethod == ZSTD_dlm_byRef ? 0 + : ZSTD_workspace_align(dictSize, sizeof(void*))); void* const workspace = ZSTD_malloc(workspaceSize, customMem); + ZSTD_CCtx_workspace tmpWorkspace; + ZSTD_CDict* cdict; - if (!cdict || !workspace) { - ZSTD_free(cdict, customMem); + if (!workspace) { ZSTD_free(workspace, customMem); return NULL; } + + ZSTD_workspace_init(&tmpWorkspace, workspace, workspaceSize); + + cdict = (ZSTD_CDict*)ZSTD_workspace_reserve_object(&tmpWorkspace, sizeof(ZSTD_CDict)); + assert(cdict != NULL); + cdict->workspace = tmpWorkspace; cdict->customMem = customMem; - ZSTD_workspace_init(&cdict->workspace, workspace, workspaceSize); if (ZSTD_isError( ZSTD_initCDict_internal(cdict, dictBuffer, dictSize, dictLoadMethod, dictContentType, @@ -3395,7 +3409,11 @@ size_t ZSTD_freeCDict(ZSTD_CDict* cdict) { if (cdict==NULL) return 0; /* support free on NULL */ { ZSTD_customMem const cMem = cdict->customMem; - ZSTD_workspace_free(&cdict->workspace, cMem); + /* Only free workspace if cdict not in workspace, otherwise the + * workspace will be freed when the cdict itself is freed. */ + if ((void*)cdict->workspace.workspace != (void*)cdict) { + ZSTD_workspace_free(&cdict->workspace, cMem); + } ZSTD_free(cdict, cMem); return 0; } From 2abe0145b18ec5aba2367e35ebacd12c5150eac5 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Wed, 14 Aug 2019 16:55:12 -0400 Subject: [PATCH 12/33] Improve Comments a Bit --- lib/compress/zstd_compress.c | 9 ++++++++- lib/compress/zstd_compress_internal.h | 20 ++++++++++++++++---- 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 4e0b76b1..262efead 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -53,7 +53,6 @@ size_t ZSTD_compressBound(size_t srcSize) { */ static size_t ZSTD_workspace_align(size_t size, size_t align) { return size + align - 1 - ((size - 1) & (align - 1)); - // return size + 3 - ((size - 1) & 3); } static void* ZSTD_workspace_reserve(ZSTD_CCtx_workspace* ws, size_t bytes, ZSTD_workspace_alloc_phase_e phase) { @@ -165,10 +164,18 @@ static int ZSTD_workspace_bump_oversized_duration(ZSTD_CCtx_workspace* ws) { return 0; } +/** + * Invalidates table allocations. + * All other allocations remain valid. + */ static void ZSTD_workspace_clear_tables(ZSTD_CCtx_workspace* ws) { ws->tableEnd = ws->objectEnd; } +/** + * Invalidates all buffer, aligned, and table allocations. + * Object allocations remain valid. + */ static void ZSTD_workspace_clear(ZSTD_CCtx_workspace* ws) { DEBUGLOG(3, "wksp: clearing!"); ZSTD_workspace_bump_oversized_duration(ws); diff --git a/lib/compress/zstd_compress_internal.h b/lib/compress/zstd_compress_internal.h index 71226757..52d15544 100644 --- a/lib/compress/zstd_compress_internal.h +++ b/lib/compress/zstd_compress_internal.h @@ -262,7 +262,10 @@ typedef enum { * are divided into the following categories: * * - Static objects: this is optionally the enclosing ZSTD_CCtx or ZSTD_CDict, - * so that literally everything fits in a single buffer. + * so that literally everything fits in a single buffer. Note: if present, + * this must be the first object in the workspace, since ZSTD_free{CCtx, + * CDict}() rely on a pointer comparison to see whether one or two frees are + * required. * * - Fixed size objects: these are fixed-size, fixed-count objects that are * nonetheless "dynamically" allocated in the workspace so that we can @@ -277,13 +280,22 @@ typedef enum { * uint32_t arrays, all of whose values are between 0 and (nextSrc - base). * Their sizes depend on the cparams. * + * - Aligned: these buffers are used for various purposes that don't require + * any initialization before they're used. + * * - Uninitialized memory: these buffers are used for various purposes that * don't require any initialization before they're used. This means they can * be moved around at no cost for a new compression. - * - I/O Buffers * - * [workspace, workspace + workspaceSize) - * [] + * Allocating Memory: + * + * The various types of objects must be allocated in order, so they can be + * correctly packed into the workspace buffer. That order is: + * + * 1. Objects + * 2. Buffers + * 3. Aligned + * 4. Tables */ typedef struct { void* workspace; From ebd162194fb7624162154206dea9bc3824e33ffe Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Wed, 14 Aug 2019 17:11:16 -0400 Subject: [PATCH 13/33] Clean Up TODOs and Comments --- lib/compress/zstd_compress.c | 52 +++++++++++++-------------- lib/compress/zstd_compress_internal.h | 8 +++-- 2 files changed, 31 insertions(+), 29 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 262efead..7002e0d4 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -51,11 +51,16 @@ size_t ZSTD_compressBound(size_t srcSize) { /** * Align must be a power of 2. */ -static size_t ZSTD_workspace_align(size_t size, size_t align) { - return size + align - 1 - ((size - 1) & (align - 1)); +static size_t ZSTD_workspace_align(size_t size, size_t const align) { + size_t const mask = align - 1; + assert((align & mask) == 0); + return (size + mask) & ~mask; } -static void* ZSTD_workspace_reserve(ZSTD_CCtx_workspace* ws, size_t bytes, ZSTD_workspace_alloc_phase_e phase) { +/** + * Internal function, use wrappers instead. + */ +static void* ZSTD_workspace_reserve_internal(ZSTD_CCtx_workspace* ws, size_t bytes, ZSTD_workspace_alloc_phase_e phase) { /* TODO(felixh): alignment */ void* alloc = (BYTE *)ws->allocStart - bytes; void* bottom = ws->tableEnd; @@ -88,6 +93,21 @@ static void* ZSTD_workspace_reserve(ZSTD_CCtx_workspace* ws, size_t bytes, ZSTD_ return alloc; } +/** + * Unaligned. + */ +static BYTE* ZSTD_workspace_reserve_buffer(ZSTD_CCtx_workspace* ws, size_t bytes) { + return (BYTE*)ZSTD_workspace_reserve_internal(ws, bytes, ZSTD_workspace_alloc_buffers); +} + +/** + * Aligned on sizeof(unsigned). + */ +static void* ZSTD_workspace_reserve_aligned(ZSTD_CCtx_workspace* ws, size_t bytes) { + assert((bytes & (sizeof(U32)-1)) == 0); // TODO ??? + return ZSTD_workspace_reserve_internal(ws, ZSTD_workspace_align(bytes, sizeof(U32)), ZSTD_workspace_alloc_aligned); +} + /** * Aligned on sizeof(unsigned). These buffers have the special property that * their values remain constrained, allowing us to re-use them without @@ -126,7 +146,8 @@ static void* ZSTD_workspace_reserve_object(ZSTD_CCtx_workspace* ws, size_t bytes void* start = ws->objectEnd; void* end = (BYTE*)start + roundedBytes; DEBUGLOG(3, "wksp: reserving %zd bytes object (rounded to %zd), %zd bytes remaining", bytes, roundedBytes, (BYTE *)ws->workspaceEnd - (BYTE *)end); - assert((bytes & (sizeof(void*)-1)) == 0); // TODO ??? + assert(((size_t)start & (sizeof(void*)-1)) == 0); + assert((bytes & (sizeof(void*)-1)) == 0); if (ws->phase != ZSTD_workspace_alloc_objects || end > ws->workspaceEnd) { DEBUGLOG(3, "wksp: object alloc failed!"); ws->allocFailed = 1; @@ -137,21 +158,6 @@ static void* ZSTD_workspace_reserve_object(ZSTD_CCtx_workspace* ws, size_t bytes return start; } -/** - * Aligned on sizeof(unsigned). - */ -static void* ZSTD_workspace_reserve_aligned(ZSTD_CCtx_workspace* ws, size_t bytes) { - assert((bytes & (sizeof(U32)-1)) == 0); // TODO ??? - return ZSTD_workspace_reserve(ws, ZSTD_workspace_align(bytes, sizeof(U32)), ZSTD_workspace_alloc_aligned); -} - -/** - * Unaligned. - */ -static BYTE* ZSTD_workspace_reserve_buffer(ZSTD_CCtx_workspace* ws, size_t bytes) { - return (BYTE*)ZSTD_workspace_reserve(ws, bytes, ZSTD_workspace_alloc_buffers); -} - // TODO static int ZSTD_workspace_bump_oversized_duration(ZSTD_CCtx_workspace* ws) { (void)ws; @@ -185,16 +191,11 @@ static void ZSTD_workspace_clear(ZSTD_CCtx_workspace* ws) { if (ws->phase > ZSTD_workspace_alloc_buffers) { ws->phase = ZSTD_workspace_alloc_buffers; } - - // ws->table = NULL; - // ws->tableEnd = NULL; - - // ws->bufferBegin = ws->workspaceEnd; } static void ZSTD_workspace_init(ZSTD_CCtx_workspace* ws, void* start, size_t size) { DEBUGLOG(3, "wksp: init'ing with %zd bytes", size); - assert(((size_t)start & (sizeof(void*)-1)) == 0); /* ensure correct alignment */ + assert(((size_t)start & (sizeof(void*)-1)) == 0); /* ensure correct alignment */ ws->workspace = start; ws->workspaceEnd = (BYTE*)start + size; ws->objectEnd = ws->workspace; @@ -1723,7 +1724,6 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, /* Statically sized space. * entropyWorkspace never moves, * though prev/next block swap places */ - /* assert(((size_t)zc->workspace.workspace & 3) == 0); */ /* ensure correct alignment */ /* TODO(felixh): check elsewhere */ assert(ZSTD_workspace_check_available(&zc->workspace, 2 * sizeof(ZSTD_compressedBlockState_t))); zc->blockState.prevCBlock = (ZSTD_compressedBlockState_t*) ZSTD_workspace_reserve_object(&zc->workspace, sizeof(ZSTD_compressedBlockState_t)); RETURN_ERROR_IF(zc->blockState.prevCBlock == NULL, memory_allocation, "couldn't allocate prevCBlock"); diff --git a/lib/compress/zstd_compress_internal.h b/lib/compress/zstd_compress_internal.h index 52d15544..fd3d030d 100644 --- a/lib/compress/zstd_compress_internal.h +++ b/lib/compress/zstd_compress_internal.h @@ -273,7 +273,7 @@ typedef enum { * Examples: * - Entropy Workspace * - 2 x ZSTD_compressedBlockState_t - * - CDict dictionary contents sometimes??? // TODO + * - CDict dictionary contents * * - Tables: these are any of several different datastructures (hash tables, * chain tables, binary trees) that all respect a common format: they are @@ -296,15 +296,17 @@ typedef enum { * 2. Buffers * 3. Aligned * 4. Tables + * + * Reusing Table Space: + * + * TODO(felixh): ... */ typedef struct { void* workspace; void* workspaceEnd; void* objectEnd; - void* tableEnd; - void* allocStart; int allocFailed; From 077a2d7dc92a3009beccb128cc809105b43f6e40 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Thu, 15 Aug 2019 12:51:24 -0400 Subject: [PATCH 14/33] Rename --- lib/compress/zstd_compress.c | 194 +++++++++++++------------- lib/compress/zstd_compress_internal.h | 14 +- 2 files changed, 104 insertions(+), 104 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 7002e0d4..45720e1d 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -51,7 +51,7 @@ size_t ZSTD_compressBound(size_t srcSize) { /** * Align must be a power of 2. */ -static size_t ZSTD_workspace_align(size_t size, size_t const align) { +static size_t ZSTD_cwksp_align(size_t size, size_t const align) { size_t const mask = align - 1; assert((align & mask) == 0); return (size + mask) & ~mask; @@ -60,7 +60,7 @@ static size_t ZSTD_workspace_align(size_t size, size_t const align) { /** * Internal function, use wrappers instead. */ -static void* ZSTD_workspace_reserve_internal(ZSTD_CCtx_workspace* ws, size_t bytes, ZSTD_workspace_alloc_phase_e phase) { +static void* ZSTD_cwksp_reserve_internal(ZSTD_cwksp* ws, size_t bytes, ZSTD_cwksp_alloc_phase_e phase) { /* TODO(felixh): alignment */ void* alloc = (BYTE *)ws->allocStart - bytes; void* bottom = ws->tableEnd; @@ -68,11 +68,11 @@ static void* ZSTD_workspace_reserve_internal(ZSTD_CCtx_workspace* ws, size_t byt bytes, (BYTE *)alloc - (BYTE *)bottom); assert(phase >= ws->phase); if (phase > ws->phase) { - if (ws->phase < ZSTD_workspace_alloc_buffers && - phase >= ZSTD_workspace_alloc_buffers) { + if (ws->phase < ZSTD_cwksp_alloc_buffers && + phase >= ZSTD_cwksp_alloc_buffers) { } - if (ws->phase < ZSTD_workspace_alloc_aligned && - phase >= ZSTD_workspace_alloc_aligned) { + if (ws->phase < ZSTD_cwksp_alloc_aligned && + phase >= ZSTD_cwksp_alloc_aligned) { /* If unaligned allocations down from a too-large top have left us * unaligned, we need to realign our alloc ptr. Technically, this * can consume space that is unaccounted for in the neededSpace @@ -96,16 +96,16 @@ static void* ZSTD_workspace_reserve_internal(ZSTD_CCtx_workspace* ws, size_t byt /** * Unaligned. */ -static BYTE* ZSTD_workspace_reserve_buffer(ZSTD_CCtx_workspace* ws, size_t bytes) { - return (BYTE*)ZSTD_workspace_reserve_internal(ws, bytes, ZSTD_workspace_alloc_buffers); +static BYTE* ZSTD_cwksp_reserve_buffer(ZSTD_cwksp* ws, size_t bytes) { + return (BYTE*)ZSTD_cwksp_reserve_internal(ws, bytes, ZSTD_cwksp_alloc_buffers); } /** * Aligned on sizeof(unsigned). */ -static void* ZSTD_workspace_reserve_aligned(ZSTD_CCtx_workspace* ws, size_t bytes) { +static void* ZSTD_cwksp_reserve_aligned(ZSTD_cwksp* ws, size_t bytes) { assert((bytes & (sizeof(U32)-1)) == 0); // TODO ??? - return ZSTD_workspace_reserve_internal(ws, ZSTD_workspace_align(bytes, sizeof(U32)), ZSTD_workspace_alloc_aligned); + return ZSTD_cwksp_reserve_internal(ws, ZSTD_cwksp_align(bytes, sizeof(U32)), ZSTD_cwksp_alloc_aligned); } /** @@ -113,9 +113,9 @@ static void* ZSTD_workspace_reserve_aligned(ZSTD_CCtx_workspace* ws, size_t byte * their values remain constrained, allowing us to re-use them without * memset()-ing them. */ -static void* ZSTD_workspace_reserve_table(ZSTD_CCtx_workspace* ws, size_t bytes) { +static void* ZSTD_cwksp_reserve_table(ZSTD_cwksp* ws, size_t bytes) { /* TODO(felixh): alignment */ - const ZSTD_workspace_alloc_phase_e phase = ZSTD_workspace_alloc_aligned; + const ZSTD_cwksp_alloc_phase_e phase = ZSTD_cwksp_alloc_aligned; void* alloc = ws->tableEnd; void* end = (BYTE *)alloc + bytes; void* top = ws->allocStart; @@ -124,7 +124,7 @@ static void* ZSTD_workspace_reserve_table(ZSTD_CCtx_workspace* ws, size_t bytes) assert((bytes & (sizeof(U32)-1)) == 0); // TODO ??? assert(phase >= ws->phase); if (phase > ws->phase) { - if (ws->phase <= ZSTD_workspace_alloc_buffers) { + if (ws->phase <= ZSTD_cwksp_alloc_buffers) { } ws->phase = phase; @@ -141,14 +141,14 @@ static void* ZSTD_workspace_reserve_table(ZSTD_CCtx_workspace* ws, size_t bytes) /** * Aligned on sizeof(void*). */ -static void* ZSTD_workspace_reserve_object(ZSTD_CCtx_workspace* ws, size_t bytes) { - size_t roundedBytes = ZSTD_workspace_align(bytes, sizeof(void*)); +static void* ZSTD_cwksp_reserve_object(ZSTD_cwksp* ws, size_t bytes) { + size_t roundedBytes = ZSTD_cwksp_align(bytes, sizeof(void*)); void* start = ws->objectEnd; void* end = (BYTE*)start + roundedBytes; DEBUGLOG(3, "wksp: reserving %zd bytes object (rounded to %zd), %zd bytes remaining", bytes, roundedBytes, (BYTE *)ws->workspaceEnd - (BYTE *)end); assert(((size_t)start & (sizeof(void*)-1)) == 0); assert((bytes & (sizeof(void*)-1)) == 0); - if (ws->phase != ZSTD_workspace_alloc_objects || end > ws->workspaceEnd) { + if (ws->phase != ZSTD_cwksp_alloc_objects || end > ws->workspaceEnd) { DEBUGLOG(3, "wksp: object alloc failed!"); ws->allocFailed = 1; return NULL; @@ -159,7 +159,7 @@ static void* ZSTD_workspace_reserve_object(ZSTD_CCtx_workspace* ws, size_t bytes } // TODO -static int ZSTD_workspace_bump_oversized_duration(ZSTD_CCtx_workspace* ws) { +static int ZSTD_cwksp_bump_oversized_duration(ZSTD_cwksp* ws) { (void)ws; // if (((BYTE*)ws->allocEnd - (BYTE*)ws->workspace) * ZSTD_WORKSPACETOOLARGE_FACTOR < (BYTE*)ws->workspaceEnd - (BYTE*)ws->workspace) { // ws->workspaceOversizedDuration++; @@ -174,7 +174,7 @@ static int ZSTD_workspace_bump_oversized_duration(ZSTD_CCtx_workspace* ws) { * Invalidates table allocations. * All other allocations remain valid. */ -static void ZSTD_workspace_clear_tables(ZSTD_CCtx_workspace* ws) { +static void ZSTD_cwksp_clear_tables(ZSTD_cwksp* ws) { ws->tableEnd = ws->objectEnd; } @@ -182,61 +182,61 @@ static void ZSTD_workspace_clear_tables(ZSTD_CCtx_workspace* ws) { * Invalidates all buffer, aligned, and table allocations. * Object allocations remain valid. */ -static void ZSTD_workspace_clear(ZSTD_CCtx_workspace* ws) { +static void ZSTD_cwksp_clear(ZSTD_cwksp* ws) { DEBUGLOG(3, "wksp: clearing!"); - ZSTD_workspace_bump_oversized_duration(ws); + ZSTD_cwksp_bump_oversized_duration(ws); ws->tableEnd = ws->objectEnd; ws->allocStart = ws->workspaceEnd; ws->allocFailed = 0; - if (ws->phase > ZSTD_workspace_alloc_buffers) { - ws->phase = ZSTD_workspace_alloc_buffers; + if (ws->phase > ZSTD_cwksp_alloc_buffers) { + ws->phase = ZSTD_cwksp_alloc_buffers; } } -static void ZSTD_workspace_init(ZSTD_CCtx_workspace* ws, void* start, size_t size) { +static void ZSTD_cwksp_init(ZSTD_cwksp* ws, void* start, size_t size) { DEBUGLOG(3, "wksp: init'ing with %zd bytes", size); assert(((size_t)start & (sizeof(void*)-1)) == 0); /* ensure correct alignment */ ws->workspace = start; ws->workspaceEnd = (BYTE*)start + size; ws->objectEnd = ws->workspace; - ws->phase = ZSTD_workspace_alloc_objects; - ZSTD_workspace_clear(ws); + ws->phase = ZSTD_cwksp_alloc_objects; + ZSTD_cwksp_clear(ws); ws->workspaceOversizedDuration = 0; } -static size_t ZSTD_workspace_create(ZSTD_CCtx_workspace* ws, size_t size, ZSTD_customMem customMem) { +static size_t ZSTD_cwksp_create(ZSTD_cwksp* ws, size_t size, ZSTD_customMem customMem) { void* workspace = ZSTD_malloc(size, customMem); DEBUGLOG(3, "wksp: creating with %zd bytes", size); RETURN_ERROR_IF(workspace == NULL, memory_allocation); - ZSTD_workspace_init(ws, workspace, size); + ZSTD_cwksp_init(ws, workspace, size); return 0; } -static void ZSTD_workspace_free(ZSTD_CCtx_workspace* ws, ZSTD_customMem customMem) { +static void ZSTD_cwksp_free(ZSTD_cwksp* ws, ZSTD_customMem customMem) { DEBUGLOG(3, "wksp: freeing"); ZSTD_free(ws->workspace, customMem); ws->workspace = NULL; ws->workspaceEnd = NULL; - ZSTD_workspace_clear(ws); + ZSTD_cwksp_clear(ws); } -static size_t ZSTD_workspace_available_space(ZSTD_CCtx_workspace* ws) { +static size_t ZSTD_cwksp_available_space(ZSTD_cwksp* ws) { return (size_t)((BYTE*)ws->allocStart - (BYTE*)ws->tableEnd); } -static int ZSTD_workspace_check_available(ZSTD_CCtx_workspace* ws, size_t minFree) { - return ZSTD_workspace_available_space(ws) >= minFree; +static int ZSTD_cwksp_check_available(ZSTD_cwksp* ws, size_t minFree) { + return ZSTD_cwksp_available_space(ws) >= minFree; } -static int ZSTD_workspace_check_wasteful(ZSTD_CCtx_workspace* ws, size_t minFree) { - return ZSTD_workspace_check_available(ws, minFree * ZSTD_WORKSPACETOOLARGE_FACTOR) && ws->workspaceOversizedDuration > ZSTD_WORKSPACETOOLARGE_MAXDURATION; +static int ZSTD_cwksp_check_wasteful(ZSTD_cwksp* ws, size_t minFree) { + return ZSTD_cwksp_check_available(ws, minFree * ZSTD_WORKSPACETOOLARGE_FACTOR) && ws->workspaceOversizedDuration > ZSTD_WORKSPACETOOLARGE_MAXDURATION; } -static size_t ZSTD_workspace_sizeof(const ZSTD_CCtx_workspace* ws) { +static size_t ZSTD_cwksp_sizeof(const ZSTD_cwksp* ws) { return (BYTE*)ws->workspaceEnd - (BYTE*)ws->workspace; } -static int ZSTD_workspace_reserve_failed(const ZSTD_CCtx_workspace* ws) { +static int ZSTD_cwksp_reserve_failed(const ZSTD_cwksp* ws) { return ws->allocFailed; } @@ -248,7 +248,7 @@ struct ZSTD_CDict_s { const void* dictContent; size_t dictContentSize; U32* entropyWorkspace; /* entropy workspace of HUF_WORKSPACE_SIZE bytes */ - ZSTD_CCtx_workspace workspace; + ZSTD_cwksp workspace; ZSTD_matchState_t matchState; ZSTD_compressedBlockState_t cBlockState; ZSTD_customMem customMem; @@ -286,25 +286,25 @@ ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem) ZSTD_CCtx* ZSTD_initStaticCCtx(void *workspace, size_t workspaceSize) { - ZSTD_CCtx_workspace tmpWorkspace; + ZSTD_cwksp ws; ZSTD_CCtx* cctx; if (workspaceSize <= sizeof(ZSTD_CCtx)) return NULL; /* minimum size */ if ((size_t)workspace & 7) return NULL; /* must be 8-aligned */ - ZSTD_workspace_init(&tmpWorkspace, workspace, workspaceSize); + ZSTD_cwksp_init(&ws, workspace, workspaceSize); - cctx = (ZSTD_CCtx*)ZSTD_workspace_reserve_object(&tmpWorkspace, sizeof(ZSTD_CCtx)); + cctx = (ZSTD_CCtx*)ZSTD_cwksp_reserve_object(&ws, sizeof(ZSTD_CCtx)); if (cctx == NULL) { return NULL; } memset(cctx, 0, sizeof(ZSTD_CCtx)); - cctx->workspace = tmpWorkspace; + cctx->workspace = ws; cctx->staticSize = workspaceSize; /* statically sized space. entropyWorkspace never moves (but prev/next block swap places) */ - if (!ZSTD_workspace_check_available(&cctx->workspace, HUF_WORKSPACE_SIZE + 2 * sizeof(ZSTD_compressedBlockState_t))) return NULL; - cctx->blockState.prevCBlock = (ZSTD_compressedBlockState_t*)ZSTD_workspace_reserve_object(&cctx->workspace, sizeof(ZSTD_compressedBlockState_t)); - cctx->blockState.nextCBlock = (ZSTD_compressedBlockState_t*)ZSTD_workspace_reserve_object(&cctx->workspace, sizeof(ZSTD_compressedBlockState_t)); - cctx->entropyWorkspace = (U32*)ZSTD_workspace_reserve_object( + if (!ZSTD_cwksp_check_available(&cctx->workspace, HUF_WORKSPACE_SIZE + 2 * sizeof(ZSTD_compressedBlockState_t))) return NULL; + cctx->blockState.prevCBlock = (ZSTD_compressedBlockState_t*)ZSTD_cwksp_reserve_object(&cctx->workspace, sizeof(ZSTD_compressedBlockState_t)); + cctx->blockState.nextCBlock = (ZSTD_compressedBlockState_t*)ZSTD_cwksp_reserve_object(&cctx->workspace, sizeof(ZSTD_compressedBlockState_t)); + cctx->entropyWorkspace = (U32*)ZSTD_cwksp_reserve_object( &cctx->workspace, HUF_WORKSPACE_SIZE); cctx->bmi2 = ZSTD_cpuid_bmi2(ZSTD_cpuid()); return cctx; @@ -336,7 +336,7 @@ static void ZSTD_freeCCtxContent(ZSTD_CCtx* cctx) /* Only free workspace if cctx not in workspace, otherwise the workspace * will be freed when the cctx itself is freed. */ if ((void*)cctx->workspace.workspace != (void*)cctx) { - ZSTD_workspace_free(&cctx->workspace, cctx->customMem); + ZSTD_cwksp_free(&cctx->workspace, cctx->customMem); } ZSTD_clearAllDicts(cctx); #ifdef ZSTD_MULTITHREAD @@ -369,7 +369,7 @@ static size_t ZSTD_sizeof_mtctx(const ZSTD_CCtx* cctx) size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx) { if (cctx==NULL) return 0; /* support sizeof on NULL */ - return sizeof(*cctx) + ZSTD_workspace_sizeof(&cctx->workspace) + return sizeof(*cctx) + ZSTD_cwksp_sizeof(&cctx->workspace) + ZSTD_sizeof_localDict(cctx->localDict) + ZSTD_sizeof_mtctx(cctx); } @@ -1566,7 +1566,7 @@ typedef enum { ZSTD_resetTarget_CDict, ZSTD_resetTarget_CCtx } ZSTD_resetTarget_ static size_t ZSTD_reset_matchState(ZSTD_matchState_t* ms, - ZSTD_CCtx_workspace* ws, + ZSTD_cwksp* ws, const ZSTD_compressionParameters* cParams, ZSTD_compResetPolicy_e const crp, ZSTD_resetTarget_e const forWho) { @@ -1583,16 +1583,16 @@ ZSTD_reset_matchState(ZSTD_matchState_t* ms, ms->window.nextSrc = ms->window.base + 1; /* see issue #1241 */ ZSTD_invalidateMatchState(ms); - assert(!ZSTD_workspace_reserve_failed(ws)); /* check that allocation hasn't already failed */ + assert(!ZSTD_cwksp_reserve_failed(ws)); /* check that allocation hasn't already failed */ - ZSTD_workspace_clear_tables(ws); + ZSTD_cwksp_clear_tables(ws); DEBUGLOG(5, "reserving table space"); /* table Space */ - ms->hashTable = (U32*)ZSTD_workspace_reserve_table(ws, hSize * sizeof(U32)); - ms->chainTable = (U32*)ZSTD_workspace_reserve_table(ws, chainSize * sizeof(U32)); - ms->hashTable3 = (U32*)ZSTD_workspace_reserve_table(ws, h3Size * sizeof(U32)); - RETURN_ERROR_IF(ZSTD_workspace_reserve_failed(ws), memory_allocation, + ms->hashTable = (U32*)ZSTD_cwksp_reserve_table(ws, hSize * sizeof(U32)); + ms->chainTable = (U32*)ZSTD_cwksp_reserve_table(ws, chainSize * sizeof(U32)); + ms->hashTable3 = (U32*)ZSTD_cwksp_reserve_table(ws, h3Size * sizeof(U32)); + RETURN_ERROR_IF(ZSTD_cwksp_reserve_failed(ws), memory_allocation, "failed a workspace allocation in ZSTD_reset_matchState"); DEBUGLOG(4, "reset table : %u", crp!=ZSTDcrp_noMemset); @@ -1606,17 +1606,17 @@ ZSTD_reset_matchState(ZSTD_matchState_t* ms, /* opt parser space */ if ((forWho == ZSTD_resetTarget_CCtx) && (cParams->strategy >= ZSTD_btopt)) { DEBUGLOG(4, "reserving optimal parser space"); - ms->opt.litFreq = (unsigned*)ZSTD_workspace_reserve_aligned(ws, (1<opt.litLengthFreq = (unsigned*)ZSTD_workspace_reserve_aligned(ws, (MaxLL+1) * sizeof(unsigned)); - ms->opt.matchLengthFreq = (unsigned*)ZSTD_workspace_reserve_aligned(ws, (MaxML+1) * sizeof(unsigned)); - ms->opt.offCodeFreq = (unsigned*)ZSTD_workspace_reserve_aligned(ws, (MaxOff+1) * sizeof(unsigned)); - ms->opt.matchTable = (ZSTD_match_t*)ZSTD_workspace_reserve_aligned(ws, (ZSTD_OPT_NUM+1) * sizeof(ZSTD_match_t)); - ms->opt.priceTable = (ZSTD_optimal_t*)ZSTD_workspace_reserve_aligned(ws, (ZSTD_OPT_NUM+1) * sizeof(ZSTD_optimal_t)); + ms->opt.litFreq = (unsigned*)ZSTD_cwksp_reserve_aligned(ws, (1<opt.litLengthFreq = (unsigned*)ZSTD_cwksp_reserve_aligned(ws, (MaxLL+1) * sizeof(unsigned)); + ms->opt.matchLengthFreq = (unsigned*)ZSTD_cwksp_reserve_aligned(ws, (MaxML+1) * sizeof(unsigned)); + ms->opt.offCodeFreq = (unsigned*)ZSTD_cwksp_reserve_aligned(ws, (MaxOff+1) * sizeof(unsigned)); + ms->opt.matchTable = (ZSTD_match_t*)ZSTD_cwksp_reserve_aligned(ws, (ZSTD_OPT_NUM+1) * sizeof(ZSTD_match_t)); + ms->opt.priceTable = (ZSTD_optimal_t*)ZSTD_cwksp_reserve_aligned(ws, (ZSTD_OPT_NUM+1) * sizeof(ZSTD_optimal_t)); } ms->cParams = *cParams; - RETURN_ERROR_IF(ZSTD_workspace_reserve_failed(ws), memory_allocation, + RETURN_ERROR_IF(ZSTD_cwksp_reserve_failed(ws), memory_allocation, "failed a workspace allocation in ZSTD_reset_matchState"); return 0; @@ -1654,7 +1654,7 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, zc->seqStore.maxNbSeq, zc->seqStore.maxNbLit, zbuff, pledgedSrcSize) ) { DEBUGLOG(4, "ZSTD_equivalentParams()==1 -> consider continue mode"); - if (ZSTD_workspace_bump_oversized_duration(&zc->workspace) <= ZSTD_WORKSPACETOOLARGE_MAXDURATION) { + if (ZSTD_cwksp_bump_oversized_duration(&zc->workspace) <= ZSTD_WORKSPACETOOLARGE_MAXDURATION) { DEBUGLOG(4, "continue mode confirmed (wLog1=%u, blockSize1=%zu)", zc->appliedParams.cParams.windowLog, zc->blockSize); if (ZSTD_indexTooCloseToMax(zc->blockState.matchState.window)) { @@ -1703,8 +1703,8 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, tokenSpace + bufferSpace; - int const workspaceTooSmall = !ZSTD_workspace_check_available(&zc->workspace, neededSpace); - int const workspaceWasteful = ZSTD_workspace_check_wasteful(&zc->workspace, neededSpace); + int const workspaceTooSmall = !ZSTD_cwksp_check_available(&zc->workspace, neededSpace); + int const workspaceWasteful = ZSTD_cwksp_check_wasteful(&zc->workspace, neededSpace); DEBUGLOG(4, "Need %zuKB workspace, including %zuKB for match state, and %zuKB for buffers", neededSpace>>10, matchStateSize>>10, bufferSpace>>10); @@ -1712,28 +1712,28 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, if (workspaceTooSmall || workspaceWasteful) { DEBUGLOG(4, "Resize workspaceSize from %zuKB to %zuKB", - ZSTD_workspace_sizeof(&zc->workspace) >> 10, + ZSTD_cwksp_sizeof(&zc->workspace) >> 10, neededSpace >> 10); RETURN_ERROR_IF(zc->staticSize, memory_allocation, "static cctx : no resize"); - ZSTD_workspace_free(&zc->workspace, zc->customMem); - FORWARD_IF_ERROR(ZSTD_workspace_create(&zc->workspace, neededSpace, zc->customMem)); + ZSTD_cwksp_free(&zc->workspace, zc->customMem); + FORWARD_IF_ERROR(ZSTD_cwksp_create(&zc->workspace, neededSpace, zc->customMem)); DEBUGLOG(5, "reserving object space"); /* Statically sized space. * entropyWorkspace never moves, * though prev/next block swap places */ - assert(ZSTD_workspace_check_available(&zc->workspace, 2 * sizeof(ZSTD_compressedBlockState_t))); - zc->blockState.prevCBlock = (ZSTD_compressedBlockState_t*) ZSTD_workspace_reserve_object(&zc->workspace, sizeof(ZSTD_compressedBlockState_t)); + assert(ZSTD_cwksp_check_available(&zc->workspace, 2 * sizeof(ZSTD_compressedBlockState_t))); + zc->blockState.prevCBlock = (ZSTD_compressedBlockState_t*) ZSTD_cwksp_reserve_object(&zc->workspace, sizeof(ZSTD_compressedBlockState_t)); RETURN_ERROR_IF(zc->blockState.prevCBlock == NULL, memory_allocation, "couldn't allocate prevCBlock"); - zc->blockState.nextCBlock = (ZSTD_compressedBlockState_t*) ZSTD_workspace_reserve_object(&zc->workspace, sizeof(ZSTD_compressedBlockState_t)); + zc->blockState.nextCBlock = (ZSTD_compressedBlockState_t*) ZSTD_cwksp_reserve_object(&zc->workspace, sizeof(ZSTD_compressedBlockState_t)); RETURN_ERROR_IF(zc->blockState.nextCBlock == NULL, memory_allocation, "couldn't allocate nextCBlock"); - zc->entropyWorkspace = (U32*) ZSTD_workspace_reserve_object(&zc->workspace, HUF_WORKSPACE_SIZE); + zc->entropyWorkspace = (U32*) ZSTD_cwksp_reserve_object(&zc->workspace, HUF_WORKSPACE_SIZE); RETURN_ERROR_IF(zc->blockState.nextCBlock == NULL, memory_allocation, "couldn't allocate entropyWorkspace"); } } - ZSTD_workspace_clear(&zc->workspace); + ZSTD_cwksp_clear(&zc->workspace); /* init params */ zc->appliedParams = params; @@ -1756,31 +1756,31 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, /* ZSTD_wildcopy() is used to copy into the literals buffer, * so we have to oversize the buffer by WILDCOPY_OVERLENGTH bytes. */ - zc->seqStore.litStart = ZSTD_workspace_reserve_buffer(&zc->workspace, blockSize + WILDCOPY_OVERLENGTH); + zc->seqStore.litStart = ZSTD_cwksp_reserve_buffer(&zc->workspace, blockSize + WILDCOPY_OVERLENGTH); zc->seqStore.maxNbLit = blockSize; /* buffers */ zc->inBuffSize = buffInSize; - zc->inBuff = (char*)ZSTD_workspace_reserve_buffer(&zc->workspace, buffInSize); + zc->inBuff = (char*)ZSTD_cwksp_reserve_buffer(&zc->workspace, buffInSize); zc->outBuffSize = buffOutSize; - zc->outBuff = (char*)ZSTD_workspace_reserve_buffer(&zc->workspace, buffOutSize); + zc->outBuff = (char*)ZSTD_cwksp_reserve_buffer(&zc->workspace, buffOutSize); /* ldm bucketOffsets table */ if (params.ldmParams.enableLdm) { size_t const ldmBucketSize = ((size_t)1) << (params.ldmParams.hashLog - params.ldmParams.bucketSizeLog); - zc->ldmState.bucketOffsets = ZSTD_workspace_reserve_buffer(&zc->workspace, ldmBucketSize); + zc->ldmState.bucketOffsets = ZSTD_cwksp_reserve_buffer(&zc->workspace, ldmBucketSize); memset(zc->ldmState.bucketOffsets, 0, ldmBucketSize); } /* sequences storage */ ZSTD_referenceExternalSequences(zc, NULL, 0); zc->seqStore.maxNbSeq = maxNbSeq; - zc->seqStore.llCode = ZSTD_workspace_reserve_buffer(&zc->workspace, maxNbSeq * sizeof(BYTE)); - zc->seqStore.mlCode = ZSTD_workspace_reserve_buffer(&zc->workspace, maxNbSeq * sizeof(BYTE)); - zc->seqStore.ofCode = ZSTD_workspace_reserve_buffer(&zc->workspace, maxNbSeq * sizeof(BYTE)); - zc->seqStore.sequencesStart = (seqDef*)ZSTD_workspace_reserve_aligned(&zc->workspace, maxNbSeq * sizeof(seqDef)); + zc->seqStore.llCode = ZSTD_cwksp_reserve_buffer(&zc->workspace, maxNbSeq * sizeof(BYTE)); + zc->seqStore.mlCode = ZSTD_cwksp_reserve_buffer(&zc->workspace, maxNbSeq * sizeof(BYTE)); + zc->seqStore.ofCode = ZSTD_cwksp_reserve_buffer(&zc->workspace, maxNbSeq * sizeof(BYTE)); + zc->seqStore.sequencesStart = (seqDef*)ZSTD_cwksp_reserve_aligned(&zc->workspace, maxNbSeq * sizeof(seqDef)); FORWARD_IF_ERROR(ZSTD_reset_matchState( &zc->blockState.matchState, @@ -1792,16 +1792,16 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, /* initialize bucketOffsets table separately for pointer alignment */ if (params.ldmParams.enableLdm) { size_t const ldmHSize = ((size_t)1) << params.ldmParams.hashLog; - zc->ldmState.hashTable = (ldmEntry_t*)ZSTD_workspace_reserve_aligned(&zc->workspace, ldmHSize * sizeof(ldmEntry_t)); + zc->ldmState.hashTable = (ldmEntry_t*)ZSTD_cwksp_reserve_aligned(&zc->workspace, ldmHSize * sizeof(ldmEntry_t)); memset(zc->ldmState.hashTable, 0, ldmHSize * sizeof(ldmEntry_t)); - zc->ldmSequences = (rawSeq*)ZSTD_workspace_reserve_aligned(&zc->workspace, maxNbLdmSeq * sizeof(rawSeq)); + zc->ldmSequences = (rawSeq*)ZSTD_cwksp_reserve_aligned(&zc->workspace, maxNbLdmSeq * sizeof(rawSeq)); zc->maxNbLdmSequences = maxNbLdmSeq; memset(&zc->ldmState.window, 0, sizeof(zc->ldmState.window)); ZSTD_window_clear(&zc->ldmState.window); } - DEBUGLOG(3, "wksp: finished allocating, %zd bytes remain available", ZSTD_workspace_available_space(&zc->workspace)); + DEBUGLOG(3, "wksp: finished allocating, %zd bytes remain available", ZSTD_cwksp_available_space(&zc->workspace)); return 0; } @@ -3288,7 +3288,7 @@ size_t ZSTD_estimateCDictSize_advanced( { DEBUGLOG(5, "sizeof(ZSTD_CDict) : %u", (unsigned)sizeof(ZSTD_CDict)); return sizeof(ZSTD_CDict) + HUF_WORKSPACE_SIZE + ZSTD_sizeof_matchState(&cParams, /* forCCtx */ 0) - + (dictLoadMethod == ZSTD_dlm_byRef ? 0 : ZSTD_workspace_align(dictSize, sizeof(void *))); + + (dictLoadMethod == ZSTD_dlm_byRef ? 0 : ZSTD_cwksp_align(dictSize, sizeof(void *))); } size_t ZSTD_estimateCDictSize(size_t dictSize, int compressionLevel) @@ -3301,7 +3301,7 @@ size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict) { if (cdict==NULL) return 0; /* support sizeof on NULL */ DEBUGLOG(5, "sizeof(*cdict) : %u", (unsigned)sizeof(*cdict)); - return ZSTD_workspace_sizeof(&cdict->workspace) + sizeof(*cdict); + return ZSTD_cwksp_sizeof(&cdict->workspace) + sizeof(*cdict); } static size_t ZSTD_initCDict_internal( @@ -3317,14 +3317,14 @@ static size_t ZSTD_initCDict_internal( if ((dictLoadMethod == ZSTD_dlm_byRef) || (!dictBuffer) || (!dictSize)) { cdict->dictContent = dictBuffer; } else { - void *internalBuffer = ZSTD_workspace_reserve_object(&cdict->workspace, ZSTD_workspace_align(dictSize, sizeof(void*))); + void *internalBuffer = ZSTD_cwksp_reserve_object(&cdict->workspace, ZSTD_cwksp_align(dictSize, sizeof(void*))); RETURN_ERROR_IF(!internalBuffer, memory_allocation); cdict->dictContent = internalBuffer; memcpy(internalBuffer, dictBuffer, dictSize); } cdict->dictContentSize = dictSize; - cdict->entropyWorkspace = (U32*)ZSTD_workspace_reserve_object(&cdict->workspace, HUF_WORKSPACE_SIZE); + cdict->entropyWorkspace = (U32*)ZSTD_cwksp_reserve_object(&cdict->workspace, HUF_WORKSPACE_SIZE); /* Reset the state to no dictionary */ @@ -3368,9 +3368,9 @@ ZSTD_CDict* ZSTD_createCDict_advanced(const void* dictBuffer, size_t dictSize, HUF_WORKSPACE_SIZE + ZSTD_sizeof_matchState(&cParams, /* forCCtx */ 0) + (dictLoadMethod == ZSTD_dlm_byRef ? 0 - : ZSTD_workspace_align(dictSize, sizeof(void*))); + : ZSTD_cwksp_align(dictSize, sizeof(void*))); void* const workspace = ZSTD_malloc(workspaceSize, customMem); - ZSTD_CCtx_workspace tmpWorkspace; + ZSTD_cwksp ws; ZSTD_CDict* cdict; if (!workspace) { @@ -3378,11 +3378,11 @@ ZSTD_CDict* ZSTD_createCDict_advanced(const void* dictBuffer, size_t dictSize, return NULL; } - ZSTD_workspace_init(&tmpWorkspace, workspace, workspaceSize); + ZSTD_cwksp_init(&ws, workspace, workspaceSize); - cdict = (ZSTD_CDict*)ZSTD_workspace_reserve_object(&tmpWorkspace, sizeof(ZSTD_CDict)); + cdict = (ZSTD_CDict*)ZSTD_cwksp_reserve_object(&ws, sizeof(ZSTD_CDict)); assert(cdict != NULL); - cdict->workspace = tmpWorkspace; + cdict->workspace = ws; cdict->customMem = customMem; if (ZSTD_isError( ZSTD_initCDict_internal(cdict, dictBuffer, dictSize, @@ -3419,7 +3419,7 @@ size_t ZSTD_freeCDict(ZSTD_CDict* cdict) /* Only free workspace if cdict not in workspace, otherwise the * workspace will be freed when the cdict itself is freed. */ if ((void*)cdict->workspace.workspace != (void*)cdict) { - ZSTD_workspace_free(&cdict->workspace, cMem); + ZSTD_cwksp_free(&cdict->workspace, cMem); } ZSTD_free(cdict, cMem); return 0; @@ -3447,16 +3447,16 @@ const ZSTD_CDict* ZSTD_initStaticCDict( ZSTD_compressionParameters cParams) { size_t const matchStateSize = ZSTD_sizeof_matchState(&cParams, /* forCCtx */ 0); - size_t const neededSize = sizeof(ZSTD_CDict) + (dictLoadMethod == ZSTD_dlm_byRef ? 0 : ZSTD_workspace_align(dictSize, sizeof(void*))) + size_t const neededSize = sizeof(ZSTD_CDict) + (dictLoadMethod == ZSTD_dlm_byRef ? 0 : ZSTD_cwksp_align(dictSize, sizeof(void*))) + HUF_WORKSPACE_SIZE + matchStateSize; ZSTD_CDict* cdict; if ((size_t)workspace & 7) return NULL; /* 8-aligned */ { - ZSTD_CCtx_workspace ws; - ZSTD_workspace_init(&ws, workspace, workspaceSize); - cdict = (ZSTD_CDict*)ZSTD_workspace_reserve_object(&ws, sizeof(ZSTD_CDict)); + ZSTD_cwksp ws; + ZSTD_cwksp_init(&ws, workspace, workspaceSize); + cdict = (ZSTD_CDict*)ZSTD_cwksp_reserve_object(&ws, sizeof(ZSTD_CDict)); if (cdict == NULL) return NULL; cdict->workspace = ws; } diff --git a/lib/compress/zstd_compress_internal.h b/lib/compress/zstd_compress_internal.h index fd3d030d..806ce849 100644 --- a/lib/compress/zstd_compress_internal.h +++ b/lib/compress/zstd_compress_internal.h @@ -224,10 +224,10 @@ struct ZSTD_CCtx_params_s { }; /* typedef'd to ZSTD_CCtx_params within "zstd.h" */ typedef enum { - ZSTD_workspace_alloc_objects, - ZSTD_workspace_alloc_buffers, - ZSTD_workspace_alloc_aligned -} ZSTD_workspace_alloc_phase_e; + ZSTD_cwksp_alloc_objects, + ZSTD_cwksp_alloc_buffers, + ZSTD_cwksp_alloc_aligned +} ZSTD_cwksp_alloc_phase_e; /** * Zstd fits all its internal datastructures into a single continuous buffer, @@ -311,8 +311,8 @@ typedef struct { int allocFailed; int workspaceOversizedDuration; - ZSTD_workspace_alloc_phase_e phase; -} ZSTD_CCtx_workspace; + ZSTD_cwksp_alloc_phase_e phase; +} ZSTD_cwksp; struct ZSTD_CCtx_s { ZSTD_compressionStage_e stage; @@ -322,7 +322,7 @@ struct ZSTD_CCtx_s { ZSTD_CCtx_params appliedParams; U32 dictID; - ZSTD_CCtx_workspace workspace; /* manages buffer for dynamic allocations */ + ZSTD_cwksp workspace; /* manages buffer for dynamic allocations */ size_t blockSize; unsigned long long pledgedSrcSizePlusOne; /* this way, 0 (default) == unknown */ unsigned long long consumedSrcSize; From b511a84adc733fb505f7bbec6f7cc063dee8ba36 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Tue, 20 Aug 2019 14:02:50 -0400 Subject: [PATCH 15/33] Move Workspace Functions to Their Own File --- lib/compress/zstd_compress.c | 203 -------------------------- lib/compress/zstd_compress_internal.h | 92 +----------- lib/compress/zstd_cwksp.c | 203 ++++++++++++++++++++++++++ lib/compress/zstd_cwksp.h | 175 ++++++++++++++++++++++ 4 files changed, 379 insertions(+), 294 deletions(-) create mode 100644 lib/compress/zstd_cwksp.c create mode 100644 lib/compress/zstd_cwksp.h diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 45720e1d..f88f3b7c 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -38,209 +38,6 @@ size_t ZSTD_compressBound(size_t srcSize) { } -/*-************************************* -* Workspace memory management -***************************************/ -#define ZSTD_WORKSPACETOOLARGE_FACTOR 3 /* define "workspace is too large" as this number of times larger than needed */ -#define ZSTD_WORKSPACETOOLARGE_MAXDURATION 128 /* when workspace is continuously too large - * during at least this number of times, - * context's memory usage is considered wasteful, - * because it's sized to handle a worst case scenario which rarely happens. - * In which case, resize it down to free some memory */ - -/** - * Align must be a power of 2. - */ -static size_t ZSTD_cwksp_align(size_t size, size_t const align) { - size_t const mask = align - 1; - assert((align & mask) == 0); - return (size + mask) & ~mask; -} - -/** - * Internal function, use wrappers instead. - */ -static void* ZSTD_cwksp_reserve_internal(ZSTD_cwksp* ws, size_t bytes, ZSTD_cwksp_alloc_phase_e phase) { - /* TODO(felixh): alignment */ - void* alloc = (BYTE *)ws->allocStart - bytes; - void* bottom = ws->tableEnd; - DEBUGLOG(3, "wksp: reserving align %zd bytes, %zd bytes remaining", - bytes, (BYTE *)alloc - (BYTE *)bottom); - assert(phase >= ws->phase); - if (phase > ws->phase) { - if (ws->phase < ZSTD_cwksp_alloc_buffers && - phase >= ZSTD_cwksp_alloc_buffers) { - } - if (ws->phase < ZSTD_cwksp_alloc_aligned && - phase >= ZSTD_cwksp_alloc_aligned) { - /* If unaligned allocations down from a too-large top have left us - * unaligned, we need to realign our alloc ptr. Technically, this - * can consume space that is unaccounted for in the neededSpace - * calculation. However, I believe this can only happen when the - * workspace is too large, and specifically when it is too large - * by a larger margin than the space that will be consumed. */ - /* TODO: cleaner, compiler warning friendly way to do this??? */ - alloc = (BYTE*)alloc - ((size_t)alloc & (sizeof(U32)-1)); - } - ws->phase = phase; - } - assert(alloc >= bottom); - if (alloc < bottom) { - ws->allocFailed = 1; - return NULL; - } - ws->allocStart = alloc; - return alloc; -} - -/** - * Unaligned. - */ -static BYTE* ZSTD_cwksp_reserve_buffer(ZSTD_cwksp* ws, size_t bytes) { - return (BYTE*)ZSTD_cwksp_reserve_internal(ws, bytes, ZSTD_cwksp_alloc_buffers); -} - -/** - * Aligned on sizeof(unsigned). - */ -static void* ZSTD_cwksp_reserve_aligned(ZSTD_cwksp* ws, size_t bytes) { - assert((bytes & (sizeof(U32)-1)) == 0); // TODO ??? - return ZSTD_cwksp_reserve_internal(ws, ZSTD_cwksp_align(bytes, sizeof(U32)), ZSTD_cwksp_alloc_aligned); -} - -/** - * Aligned on sizeof(unsigned). These buffers have the special property that - * their values remain constrained, allowing us to re-use them without - * memset()-ing them. - */ -static void* ZSTD_cwksp_reserve_table(ZSTD_cwksp* ws, size_t bytes) { - /* TODO(felixh): alignment */ - const ZSTD_cwksp_alloc_phase_e phase = ZSTD_cwksp_alloc_aligned; - void* alloc = ws->tableEnd; - void* end = (BYTE *)alloc + bytes; - void* top = ws->allocStart; - DEBUGLOG(3, "wksp: reserving table %zd bytes, %zd bytes remaining", - bytes, (BYTE *)top - (BYTE *)end); - assert((bytes & (sizeof(U32)-1)) == 0); // TODO ??? - assert(phase >= ws->phase); - if (phase > ws->phase) { - if (ws->phase <= ZSTD_cwksp_alloc_buffers) { - - } - ws->phase = phase; - } - assert(end <= top); - if (end > top) { - ws->allocFailed = 1; - return NULL; - } - ws->tableEnd = end; - return alloc; -} - -/** - * Aligned on sizeof(void*). - */ -static void* ZSTD_cwksp_reserve_object(ZSTD_cwksp* ws, size_t bytes) { - size_t roundedBytes = ZSTD_cwksp_align(bytes, sizeof(void*)); - void* start = ws->objectEnd; - void* end = (BYTE*)start + roundedBytes; - DEBUGLOG(3, "wksp: reserving %zd bytes object (rounded to %zd), %zd bytes remaining", bytes, roundedBytes, (BYTE *)ws->workspaceEnd - (BYTE *)end); - assert(((size_t)start & (sizeof(void*)-1)) == 0); - assert((bytes & (sizeof(void*)-1)) == 0); - if (ws->phase != ZSTD_cwksp_alloc_objects || end > ws->workspaceEnd) { - DEBUGLOG(3, "wksp: object alloc failed!"); - ws->allocFailed = 1; - return NULL; - } - ws->objectEnd = end; - ws->tableEnd = end; - return start; -} - -// TODO -static int ZSTD_cwksp_bump_oversized_duration(ZSTD_cwksp* ws) { - (void)ws; - // if (((BYTE*)ws->allocEnd - (BYTE*)ws->workspace) * ZSTD_WORKSPACETOOLARGE_FACTOR < (BYTE*)ws->workspaceEnd - (BYTE*)ws->workspace) { - // ws->workspaceOversizedDuration++; - // } else { - // ws->workspaceOversizedDuration = 0; - // } - // return ws->workspaceOversizedDuration; - return 0; -} - -/** - * Invalidates table allocations. - * All other allocations remain valid. - */ -static void ZSTD_cwksp_clear_tables(ZSTD_cwksp* ws) { - ws->tableEnd = ws->objectEnd; -} - -/** - * Invalidates all buffer, aligned, and table allocations. - * Object allocations remain valid. - */ -static void ZSTD_cwksp_clear(ZSTD_cwksp* ws) { - DEBUGLOG(3, "wksp: clearing!"); - ZSTD_cwksp_bump_oversized_duration(ws); - ws->tableEnd = ws->objectEnd; - ws->allocStart = ws->workspaceEnd; - ws->allocFailed = 0; - if (ws->phase > ZSTD_cwksp_alloc_buffers) { - ws->phase = ZSTD_cwksp_alloc_buffers; - } -} - -static void ZSTD_cwksp_init(ZSTD_cwksp* ws, void* start, size_t size) { - DEBUGLOG(3, "wksp: init'ing with %zd bytes", size); - assert(((size_t)start & (sizeof(void*)-1)) == 0); /* ensure correct alignment */ - ws->workspace = start; - ws->workspaceEnd = (BYTE*)start + size; - ws->objectEnd = ws->workspace; - ws->phase = ZSTD_cwksp_alloc_objects; - ZSTD_cwksp_clear(ws); - ws->workspaceOversizedDuration = 0; -} - -static size_t ZSTD_cwksp_create(ZSTD_cwksp* ws, size_t size, ZSTD_customMem customMem) { - void* workspace = ZSTD_malloc(size, customMem); - DEBUGLOG(3, "wksp: creating with %zd bytes", size); - RETURN_ERROR_IF(workspace == NULL, memory_allocation); - ZSTD_cwksp_init(ws, workspace, size); - return 0; -} - -static void ZSTD_cwksp_free(ZSTD_cwksp* ws, ZSTD_customMem customMem) { - DEBUGLOG(3, "wksp: freeing"); - ZSTD_free(ws->workspace, customMem); - ws->workspace = NULL; - ws->workspaceEnd = NULL; - ZSTD_cwksp_clear(ws); -} - -static size_t ZSTD_cwksp_available_space(ZSTD_cwksp* ws) { - return (size_t)((BYTE*)ws->allocStart - (BYTE*)ws->tableEnd); -} - -static int ZSTD_cwksp_check_available(ZSTD_cwksp* ws, size_t minFree) { - return ZSTD_cwksp_available_space(ws) >= minFree; -} - -static int ZSTD_cwksp_check_wasteful(ZSTD_cwksp* ws, size_t minFree) { - return ZSTD_cwksp_check_available(ws, minFree * ZSTD_WORKSPACETOOLARGE_FACTOR) && ws->workspaceOversizedDuration > ZSTD_WORKSPACETOOLARGE_MAXDURATION; -} - -static size_t ZSTD_cwksp_sizeof(const ZSTD_cwksp* ws) { - return (BYTE*)ws->workspaceEnd - (BYTE*)ws->workspace; -} - -static int ZSTD_cwksp_reserve_failed(const ZSTD_cwksp* ws) { - return ws->allocFailed; -} - - /*-************************************* * Context memory management ***************************************/ diff --git a/lib/compress/zstd_compress_internal.h b/lib/compress/zstd_compress_internal.h index 806ce849..ae106e02 100644 --- a/lib/compress/zstd_compress_internal.h +++ b/lib/compress/zstd_compress_internal.h @@ -19,6 +19,7 @@ * Dependencies ***************************************/ #include "zstd_internal.h" +#include "zstd_cwksp.h" #ifdef ZSTD_MULTITHREAD # include "zstdmt_compress.h" #endif @@ -223,97 +224,6 @@ struct ZSTD_CCtx_params_s { ZSTD_customMem customMem; }; /* typedef'd to ZSTD_CCtx_params within "zstd.h" */ -typedef enum { - ZSTD_cwksp_alloc_objects, - ZSTD_cwksp_alloc_buffers, - ZSTD_cwksp_alloc_aligned -} ZSTD_cwksp_alloc_phase_e; - -/** - * Zstd fits all its internal datastructures into a single continuous buffer, - * so that it only needs to perform a single OS allocation (or so that a buffer - * can be provided to it and it can perform no allocations at all). This buffer - * is called the workspace. - * - * Several optimizations complicate that process of allocating memory ranges - * from this workspace for each datastructure: - * - * - These different internal datastructures have different setup requirements. - * Some (e.g., the window buffer) don't care, and are happy to accept - * uninitialized memory. Others (e.g., the matchstate tables) can accept - * memory filled with unknown but bounded values (i.e., a memory area whose - * values are known to be constrained between 0 and some upper bound). If - * that constraint isn't known to be satisfied, the area has to be cleared. - * - * - We would like to reuse the objects in the workspace for multiple - * compressions without having to perform any expensive reallocation or - * reinitialization work. - * - * - We would like to be able to efficiently reuse the workspace across - * multiple compressions **even when the compression parameters change** and - * we need to resize some of the objects (where possible). - * - * Workspace Layout: - * - * [ ... workspace ... ] - * [objects][tables ... ->] free space [<- ... aligned][<- ... buffers] - * - * In order to accomplish this, the various objects that live in the workspace - * are divided into the following categories: - * - * - Static objects: this is optionally the enclosing ZSTD_CCtx or ZSTD_CDict, - * so that literally everything fits in a single buffer. Note: if present, - * this must be the first object in the workspace, since ZSTD_free{CCtx, - * CDict}() rely on a pointer comparison to see whether one or two frees are - * required. - * - * - Fixed size objects: these are fixed-size, fixed-count objects that are - * nonetheless "dynamically" allocated in the workspace so that we can - * control how they're initialized separately from the broader ZSTD_CCtx. - * Examples: - * - Entropy Workspace - * - 2 x ZSTD_compressedBlockState_t - * - CDict dictionary contents - * - * - Tables: these are any of several different datastructures (hash tables, - * chain tables, binary trees) that all respect a common format: they are - * uint32_t arrays, all of whose values are between 0 and (nextSrc - base). - * Their sizes depend on the cparams. - * - * - Aligned: these buffers are used for various purposes that don't require - * any initialization before they're used. - * - * - Uninitialized memory: these buffers are used for various purposes that - * don't require any initialization before they're used. This means they can - * be moved around at no cost for a new compression. - * - * Allocating Memory: - * - * The various types of objects must be allocated in order, so they can be - * correctly packed into the workspace buffer. That order is: - * - * 1. Objects - * 2. Buffers - * 3. Aligned - * 4. Tables - * - * Reusing Table Space: - * - * TODO(felixh): ... - */ -typedef struct { - void* workspace; - void* workspaceEnd; - - void* objectEnd; - void* tableEnd; - void* allocStart; - - int allocFailed; - int workspaceOversizedDuration; - ZSTD_cwksp_alloc_phase_e phase; -} ZSTD_cwksp; - struct ZSTD_CCtx_s { ZSTD_compressionStage_e stage; int cParamsChanged; /* == 1 if cParams(except wlog) or compression level are changed in requestedParams. Triggers transmission of new params to ZSTDMT (if available) then reset to 0. */ diff --git a/lib/compress/zstd_cwksp.c b/lib/compress/zstd_cwksp.c new file mode 100644 index 00000000..3b7341eb --- /dev/null +++ b/lib/compress/zstd_cwksp.c @@ -0,0 +1,203 @@ +/* + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#include "zstd_cwksp.h" + +/** + * Align must be a power of 2. + */ +size_t ZSTD_cwksp_align(size_t size, size_t const align) { + size_t const mask = align - 1; + assert((align & mask) == 0); + return (size + mask) & ~mask; +} + +/** + * Internal function, use wrappers instead. + */ +void* ZSTD_cwksp_reserve_internal(ZSTD_cwksp* ws, size_t bytes, ZSTD_cwksp_alloc_phase_e phase) { + /* TODO(felixh): alignment */ + void* alloc = (BYTE *)ws->allocStart - bytes; + void* bottom = ws->tableEnd; + DEBUGLOG(3, "wksp: reserving align %zd bytes, %zd bytes remaining", + bytes, (BYTE *)alloc - (BYTE *)bottom); + assert(phase >= ws->phase); + if (phase > ws->phase) { + if (ws->phase < ZSTD_cwksp_alloc_buffers && + phase >= ZSTD_cwksp_alloc_buffers) { + } + if (ws->phase < ZSTD_cwksp_alloc_aligned && + phase >= ZSTD_cwksp_alloc_aligned) { + /* If unaligned allocations down from a too-large top have left us + * unaligned, we need to realign our alloc ptr. Technically, this + * can consume space that is unaccounted for in the neededSpace + * calculation. However, I believe this can only happen when the + * workspace is too large, and specifically when it is too large + * by a larger margin than the space that will be consumed. */ + /* TODO: cleaner, compiler warning friendly way to do this??? */ + alloc = (BYTE*)alloc - ((size_t)alloc & (sizeof(U32)-1)); + } + ws->phase = phase; + } + assert(alloc >= bottom); + if (alloc < bottom) { + ws->allocFailed = 1; + return NULL; + } + ws->allocStart = alloc; + return alloc; +} + +/** + * Unaligned. + */ +BYTE* ZSTD_cwksp_reserve_buffer(ZSTD_cwksp* ws, size_t bytes) { + return (BYTE*)ZSTD_cwksp_reserve_internal(ws, bytes, ZSTD_cwksp_alloc_buffers); +} + +/** + * Aligned on sizeof(unsigned). + */ +void* ZSTD_cwksp_reserve_aligned(ZSTD_cwksp* ws, size_t bytes) { + assert((bytes & (sizeof(U32)-1)) == 0); // TODO ??? + return ZSTD_cwksp_reserve_internal(ws, ZSTD_cwksp_align(bytes, sizeof(U32)), ZSTD_cwksp_alloc_aligned); +} + +/** + * Aligned on sizeof(unsigned). These buffers have the special property that + * their values remain constrained, allowing us to re-use them without + * memset()-ing them. + */ +void* ZSTD_cwksp_reserve_table(ZSTD_cwksp* ws, size_t bytes) { + /* TODO(felixh): alignment */ + const ZSTD_cwksp_alloc_phase_e phase = ZSTD_cwksp_alloc_aligned; + void* alloc = ws->tableEnd; + void* end = (BYTE *)alloc + bytes; + void* top = ws->allocStart; + DEBUGLOG(3, "wksp: reserving table %zd bytes, %zd bytes remaining", + bytes, (BYTE *)top - (BYTE *)end); + assert((bytes & (sizeof(U32)-1)) == 0); // TODO ??? + assert(phase >= ws->phase); + if (phase > ws->phase) { + if (ws->phase <= ZSTD_cwksp_alloc_buffers) { + + } + ws->phase = phase; + } + assert(end <= top); + if (end > top) { + ws->allocFailed = 1; + return NULL; + } + ws->tableEnd = end; + return alloc; +} + +/** + * Aligned on sizeof(void*). + */ +void* ZSTD_cwksp_reserve_object(ZSTD_cwksp* ws, size_t bytes) { + size_t roundedBytes = ZSTD_cwksp_align(bytes, sizeof(void*)); + void* start = ws->objectEnd; + void* end = (BYTE*)start + roundedBytes; + DEBUGLOG(3, "wksp: reserving %zd bytes object (rounded to %zd), %zd bytes remaining", bytes, roundedBytes, (BYTE *)ws->workspaceEnd - (BYTE *)end); + assert(((size_t)start & (sizeof(void*)-1)) == 0); + assert((bytes & (sizeof(void*)-1)) == 0); + if (ws->phase != ZSTD_cwksp_alloc_objects || end > ws->workspaceEnd) { + DEBUGLOG(3, "wksp: object alloc failed!"); + ws->allocFailed = 1; + return NULL; + } + ws->objectEnd = end; + ws->tableEnd = end; + return start; +} + +// TODO +int ZSTD_cwksp_bump_oversized_duration(ZSTD_cwksp* ws) { + (void)ws; + // if (((BYTE*)ws->allocEnd - (BYTE*)ws->workspace) * ZSTD_WORKSPACETOOLARGE_FACTOR < (BYTE*)ws->workspaceEnd - (BYTE*)ws->workspace) { + // ws->workspaceOversizedDuration++; + // } else { + // ws->workspaceOversizedDuration = 0; + // } + // return ws->workspaceOversizedDuration; + return 0; +} + +/** + * Invalidates table allocations. + * All other allocations remain valid. + */ +void ZSTD_cwksp_clear_tables(ZSTD_cwksp* ws) { + ws->tableEnd = ws->objectEnd; +} + +/** + * Invalidates all buffer, aligned, and table allocations. + * Object allocations remain valid. + */ +void ZSTD_cwksp_clear(ZSTD_cwksp* ws) { + DEBUGLOG(3, "wksp: clearing!"); + ZSTD_cwksp_bump_oversized_duration(ws); + ws->tableEnd = ws->objectEnd; + ws->allocStart = ws->workspaceEnd; + ws->allocFailed = 0; + if (ws->phase > ZSTD_cwksp_alloc_buffers) { + ws->phase = ZSTD_cwksp_alloc_buffers; + } +} + +void ZSTD_cwksp_init(ZSTD_cwksp* ws, void* start, size_t size) { + DEBUGLOG(3, "wksp: init'ing with %zd bytes", size); + assert(((size_t)start & (sizeof(void*)-1)) == 0); /* ensure correct alignment */ + ws->workspace = start; + ws->workspaceEnd = (BYTE*)start + size; + ws->objectEnd = ws->workspace; + ws->phase = ZSTD_cwksp_alloc_objects; + ZSTD_cwksp_clear(ws); + ws->workspaceOversizedDuration = 0; +} + +size_t ZSTD_cwksp_create(ZSTD_cwksp* ws, size_t size, ZSTD_customMem customMem) { + void* workspace = ZSTD_malloc(size, customMem); + DEBUGLOG(3, "wksp: creating with %zd bytes", size); + RETURN_ERROR_IF(workspace == NULL, memory_allocation); + ZSTD_cwksp_init(ws, workspace, size); + return 0; +} + +void ZSTD_cwksp_free(ZSTD_cwksp* ws, ZSTD_customMem customMem) { + DEBUGLOG(3, "wksp: freeing"); + ZSTD_free(ws->workspace, customMem); + ws->workspace = NULL; + ws->workspaceEnd = NULL; + ZSTD_cwksp_clear(ws); +} + +size_t ZSTD_cwksp_available_space(ZSTD_cwksp* ws) { + return (size_t)((BYTE*)ws->allocStart - (BYTE*)ws->tableEnd); +} + +int ZSTD_cwksp_check_available(ZSTD_cwksp* ws, size_t minFree) { + return ZSTD_cwksp_available_space(ws) >= minFree; +} + +int ZSTD_cwksp_check_wasteful(ZSTD_cwksp* ws, size_t minFree) { + return ZSTD_cwksp_check_available(ws, minFree * ZSTD_WORKSPACETOOLARGE_FACTOR) && ws->workspaceOversizedDuration > ZSTD_WORKSPACETOOLARGE_MAXDURATION; +} + +size_t ZSTD_cwksp_sizeof(const ZSTD_cwksp* ws) { + return (BYTE*)ws->workspaceEnd - (BYTE*)ws->workspace; +} + +int ZSTD_cwksp_reserve_failed(const ZSTD_cwksp* ws) { + return ws->allocFailed; +} \ No newline at end of file diff --git a/lib/compress/zstd_cwksp.h b/lib/compress/zstd_cwksp.h new file mode 100644 index 00000000..b5b35849 --- /dev/null +++ b/lib/compress/zstd_cwksp.h @@ -0,0 +1,175 @@ +/* + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_CWKSP_H +#define ZSTD_CWKSP_H + +#include "zstd_internal.h" + +#define ZSTD_WORKSPACETOOLARGE_FACTOR 3 /* define "workspace is too large" as this number of times larger than needed */ +#define ZSTD_WORKSPACETOOLARGE_MAXDURATION 128 /* when workspace is continuously too large + * during at least this number of times, + * context's memory usage is considered wasteful, + * because it's sized to handle a worst case scenario which rarely happens. + * In which case, resize it down to free some memory */ +typedef enum { + ZSTD_cwksp_alloc_objects, + ZSTD_cwksp_alloc_buffers, + ZSTD_cwksp_alloc_aligned +} ZSTD_cwksp_alloc_phase_e; + +/** + * Zstd fits all its internal datastructures into a single continuous buffer, + * so that it only needs to perform a single OS allocation (or so that a buffer + * can be provided to it and it can perform no allocations at all). This buffer + * is called the workspace. + * + * Several optimizations complicate that process of allocating memory ranges + * from this workspace for each datastructure: + * + * - These different internal datastructures have different setup requirements. + * Some (e.g., the window buffer) don't care, and are happy to accept + * uninitialized memory. Others (e.g., the matchstate tables) can accept + * memory filled with unknown but bounded values (i.e., a memory area whose + * values are known to be constrained between 0 and some upper bound). If + * that constraint isn't known to be satisfied, the area has to be cleared. + * + * - We would like to reuse the objects in the workspace for multiple + * compressions without having to perform any expensive reallocation or + * reinitialization work. + * + * - We would like to be able to efficiently reuse the workspace across + * multiple compressions **even when the compression parameters change** and + * we need to resize some of the objects (where possible). + * + * Workspace Layout: + * + * [ ... workspace ... ] + * [objects][tables ... ->] free space [<- ... aligned][<- ... buffers] + * + * In order to accomplish this, the various objects that live in the workspace + * are divided into the following categories: + * + * - Static objects: this is optionally the enclosing ZSTD_CCtx or ZSTD_CDict, + * so that literally everything fits in a single buffer. Note: if present, + * this must be the first object in the workspace, since ZSTD_free{CCtx, + * CDict}() rely on a pointer comparison to see whether one or two frees are + * required. + * + * - Fixed size objects: these are fixed-size, fixed-count objects that are + * nonetheless "dynamically" allocated in the workspace so that we can + * control how they're initialized separately from the broader ZSTD_CCtx. + * Examples: + * - Entropy Workspace + * - 2 x ZSTD_compressedBlockState_t + * - CDict dictionary contents + * + * - Tables: these are any of several different datastructures (hash tables, + * chain tables, binary trees) that all respect a common format: they are + * uint32_t arrays, all of whose values are between 0 and (nextSrc - base). + * Their sizes depend on the cparams. + * + * - Aligned: these buffers are used for various purposes that don't require + * any initialization before they're used. + * + * - Uninitialized memory: these buffers are used for various purposes that + * don't require any initialization before they're used. This means they can + * be moved around at no cost for a new compression. + * + * Allocating Memory: + * + * The various types of objects must be allocated in order, so they can be + * correctly packed into the workspace buffer. That order is: + * + * 1. Objects + * 2. Buffers + * 3. Aligned + * 4. Tables + * + * Reusing Table Space: + * + * TODO(felixh): ... + */ +typedef struct { + void* workspace; + void* workspaceEnd; + + void* objectEnd; + void* tableEnd; + void* allocStart; + + int allocFailed; + int workspaceOversizedDuration; + ZSTD_cwksp_alloc_phase_e phase; +} ZSTD_cwksp; + +/** + * Align must be a power of 2. + */ +size_t ZSTD_cwksp_align(size_t size, size_t const align); + +/** + * Internal function, use wrappers instead. + */ +void* ZSTD_cwksp_reserve_internal(ZSTD_cwksp* ws, size_t bytes, ZSTD_cwksp_alloc_phase_e phase); + +/** + * Unaligned. + */ +BYTE* ZSTD_cwksp_reserve_buffer(ZSTD_cwksp* ws, size_t bytes); + +/** + * Aligned on sizeof(unsigned). + */ +void* ZSTD_cwksp_reserve_aligned(ZSTD_cwksp* ws, size_t bytes); + +/** + * Aligned on sizeof(unsigned). These buffers have the special property that + * their values remain constrained, allowing us to re-use them without + * memset()-ing them. + */ +void* ZSTD_cwksp_reserve_table(ZSTD_cwksp* ws, size_t bytes); + +/** + * Aligned on sizeof(void*). + */ +void* ZSTD_cwksp_reserve_object(ZSTD_cwksp* ws, size_t bytes); + +int ZSTD_cwksp_bump_oversized_duration(ZSTD_cwksp* ws); + +/** + * Invalidates table allocations. + * All other allocations remain valid. + */ +void ZSTD_cwksp_clear_tables(ZSTD_cwksp* ws); + +/** + * Invalidates all buffer, aligned, and table allocations. + * Object allocations remain valid. + */ +void ZSTD_cwksp_clear(ZSTD_cwksp* ws); + +void ZSTD_cwksp_init(ZSTD_cwksp* ws, void* start, size_t size); + +size_t ZSTD_cwksp_create(ZSTD_cwksp* ws, size_t size, ZSTD_customMem customMem); + +void ZSTD_cwksp_free(ZSTD_cwksp* ws, ZSTD_customMem customMem); + +size_t ZSTD_cwksp_available_space(ZSTD_cwksp* ws); + +int ZSTD_cwksp_check_available(ZSTD_cwksp* ws, size_t minFree); + +int ZSTD_cwksp_check_wasteful(ZSTD_cwksp* ws, size_t minFree); + +size_t ZSTD_cwksp_sizeof(const ZSTD_cwksp* ws); + +int ZSTD_cwksp_reserve_failed(const ZSTD_cwksp* ws); + +#endif /* ZSTD_CWKSP_H */ From 881bcd80cacaecdc457654ba47ff13f8b27752fe Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Tue, 20 Aug 2019 14:15:06 -0400 Subject: [PATCH 16/33] Cleanup from Move --- lib/compress/zstd_cwksp.c | 12 +++++++++-- lib/compress/zstd_cwksp.h | 43 +++++++++++++++++++++++++++++---------- 2 files changed, 42 insertions(+), 13 deletions(-) diff --git a/lib/compress/zstd_cwksp.c b/lib/compress/zstd_cwksp.c index 3b7341eb..a117ad09 100644 --- a/lib/compress/zstd_cwksp.c +++ b/lib/compress/zstd_cwksp.c @@ -10,6 +10,10 @@ #include "zstd_cwksp.h" +#if defined (__cplusplus) +extern "C" { +#endif + /** * Align must be a power of 2. */ @@ -22,7 +26,7 @@ size_t ZSTD_cwksp_align(size_t size, size_t const align) { /** * Internal function, use wrappers instead. */ -void* ZSTD_cwksp_reserve_internal(ZSTD_cwksp* ws, size_t bytes, ZSTD_cwksp_alloc_phase_e phase) { +static void* ZSTD_cwksp_reserve_internal(ZSTD_cwksp* ws, size_t bytes, ZSTD_cwksp_alloc_phase_e phase) { /* TODO(felixh): alignment */ void* alloc = (BYTE *)ws->allocStart - bytes; void* bottom = ws->tableEnd; @@ -200,4 +204,8 @@ size_t ZSTD_cwksp_sizeof(const ZSTD_cwksp* ws) { int ZSTD_cwksp_reserve_failed(const ZSTD_cwksp* ws) { return ws->allocFailed; -} \ No newline at end of file +} + +#if defined (__cplusplus) +} +#endif diff --git a/lib/compress/zstd_cwksp.h b/lib/compress/zstd_cwksp.h index b5b35849..965ca6b0 100644 --- a/lib/compress/zstd_cwksp.h +++ b/lib/compress/zstd_cwksp.h @@ -11,14 +11,32 @@ #ifndef ZSTD_CWKSP_H #define ZSTD_CWKSP_H +/*-************************************* +* Dependencies +***************************************/ #include "zstd_internal.h" -#define ZSTD_WORKSPACETOOLARGE_FACTOR 3 /* define "workspace is too large" as this number of times larger than needed */ -#define ZSTD_WORKSPACETOOLARGE_MAXDURATION 128 /* when workspace is continuously too large - * during at least this number of times, - * context's memory usage is considered wasteful, - * because it's sized to handle a worst case scenario which rarely happens. - * In which case, resize it down to free some memory */ +#if defined (__cplusplus) +extern "C" { +#endif + +/*-************************************* +* Constants +***************************************/ + +/* define "workspace is too large" as this number of times larger than needed */ +#define ZSTD_WORKSPACETOOLARGE_FACTOR 3 + +/* when workspace is continuously too large + * during at least this number of times, + * context's memory usage is considered wasteful, + * because it's sized to handle a worst case scenario which rarely happens. + * In which case, resize it down to free some memory */ +#define ZSTD_WORKSPACETOOLARGE_MAXDURATION 128 + +/*-************************************* +* Structures +***************************************/ typedef enum { ZSTD_cwksp_alloc_objects, ZSTD_cwksp_alloc_buffers, @@ -110,16 +128,15 @@ typedef struct { ZSTD_cwksp_alloc_phase_e phase; } ZSTD_cwksp; +/*-************************************* +* Functions +***************************************/ + /** * Align must be a power of 2. */ size_t ZSTD_cwksp_align(size_t size, size_t const align); -/** - * Internal function, use wrappers instead. - */ -void* ZSTD_cwksp_reserve_internal(ZSTD_cwksp* ws, size_t bytes, ZSTD_cwksp_alloc_phase_e phase); - /** * Unaligned. */ @@ -172,4 +189,8 @@ size_t ZSTD_cwksp_sizeof(const ZSTD_cwksp* ws); int ZSTD_cwksp_reserve_failed(const ZSTD_cwksp* ws); +#if defined (__cplusplus) +} +#endif + #endif /* ZSTD_CWKSP_H */ From 4e9b1341cfa573f369239c02402a71fe14ac1e52 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Tue, 20 Aug 2019 15:14:12 -0400 Subject: [PATCH 17/33] Add New File to Meson Build --- build/meson/lib/meson.build | 1 + 1 file changed, 1 insertion(+) diff --git a/build/meson/lib/meson.build b/build/meson/lib/meson.build index ef669327..48f1e7cc 100644 --- a/build/meson/lib/meson.build +++ b/build/meson/lib/meson.build @@ -30,6 +30,7 @@ libzstd_sources = [join_paths(zstd_rootdir, 'lib/common/entropy_common.c'), join_paths(zstd_rootdir, 'lib/compress/zstd_compress.c'), join_paths(zstd_rootdir, 'lib/compress/zstd_compress_literals.c'), join_paths(zstd_rootdir, 'lib/compress/zstd_compress_sequences.c'), + join_paths(zstd_rootdir, 'lib/compress/zstd_cwksp.c'), join_paths(zstd_rootdir, 'lib/compress/zstdmt_compress.c'), join_paths(zstd_rootdir, 'lib/compress/zstd_fast.c'), join_paths(zstd_rootdir, 'lib/compress/zstd_double_fast.c'), From e8cc1374770f2631bbea2e4f9b7c8cd0d8128005 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Tue, 20 Aug 2019 15:19:58 -0400 Subject: [PATCH 18/33] Add New File to Visual Studio Projects --- build/VS2008/fullbench/fullbench.vcproj | 8 ++++++++ build/VS2008/fuzzer/fuzzer.vcproj | 8 ++++++++ build/VS2008/zstd/zstd.vcproj | 8 ++++++++ build/VS2008/zstdlib/zstdlib.vcproj | 8 ++++++++ build/VS2010/fullbench/fullbench.vcxproj | 2 ++ build/VS2010/fuzzer/fuzzer.vcxproj | 2 ++ build/VS2010/libzstd-dll/libzstd-dll.vcxproj | 2 ++ build/VS2010/libzstd/libzstd.vcxproj | 2 ++ build/VS2010/zstd/zstd.vcxproj | 2 ++ 9 files changed, 42 insertions(+) diff --git a/build/VS2008/fullbench/fullbench.vcproj b/build/VS2008/fullbench/fullbench.vcproj index cd9391ec..ffb0a015 100644 --- a/build/VS2008/fullbench/fullbench.vcproj +++ b/build/VS2008/fullbench/fullbench.vcproj @@ -372,6 +372,10 @@ RelativePath="..\..\..\lib\compress\zstd_compress_sequences.c" > + + @@ -510,6 +514,10 @@ RelativePath="..\..\..\lib\compress\zstd_compress_sequences.h" > + + diff --git a/build/VS2008/fuzzer/fuzzer.vcproj b/build/VS2008/fuzzer/fuzzer.vcproj index 5f10bbe7..c3875ec7 100644 --- a/build/VS2008/fuzzer/fuzzer.vcproj +++ b/build/VS2008/fuzzer/fuzzer.vcproj @@ -420,6 +420,10 @@ RelativePath="..\..\..\lib\compress\zstd_compress_sequences.c" > + + @@ -546,6 +550,10 @@ RelativePath="..\..\..\lib\compress\zstd_compress_sequences.h" > + + diff --git a/build/VS2008/zstd/zstd.vcproj b/build/VS2008/zstd/zstd.vcproj index 34ccb6f4..5b57dde5 100644 --- a/build/VS2008/zstd/zstd.vcproj +++ b/build/VS2008/zstd/zstd.vcproj @@ -432,6 +432,10 @@ RelativePath="..\..\..\lib\compress\zstd_compress_sequences.c" > + + @@ -626,6 +630,10 @@ RelativePath="..\..\..\lib\compress\zstd_compress_sequences.h" > + + diff --git a/build/VS2008/zstdlib/zstdlib.vcproj b/build/VS2008/zstdlib/zstdlib.vcproj index be46d351..e864b5fb 100644 --- a/build/VS2008/zstdlib/zstdlib.vcproj +++ b/build/VS2008/zstdlib/zstdlib.vcproj @@ -404,6 +404,10 @@ RelativePath="..\..\..\lib\compress\zstd_compress_sequences.c" > + + @@ -558,6 +562,10 @@ RelativePath="..\..\..\lib\compress\zstd_compress_sequences.h" > + + diff --git a/build/VS2010/fullbench/fullbench.vcxproj b/build/VS2010/fullbench/fullbench.vcxproj index 105c692b..65e7ff78 100644 --- a/build/VS2010/fullbench/fullbench.vcxproj +++ b/build/VS2010/fullbench/fullbench.vcxproj @@ -169,6 +169,7 @@ + @@ -197,6 +198,7 @@ + diff --git a/build/VS2010/fuzzer/fuzzer.vcxproj b/build/VS2010/fuzzer/fuzzer.vcxproj index 7f7e404d..e9b88a8a 100644 --- a/build/VS2010/fuzzer/fuzzer.vcxproj +++ b/build/VS2010/fuzzer/fuzzer.vcxproj @@ -169,6 +169,7 @@ + @@ -200,6 +201,7 @@ + diff --git a/build/VS2010/libzstd-dll/libzstd-dll.vcxproj b/build/VS2010/libzstd-dll/libzstd-dll.vcxproj index 840d1ec3..4140879d 100644 --- a/build/VS2010/libzstd-dll/libzstd-dll.vcxproj +++ b/build/VS2010/libzstd-dll/libzstd-dll.vcxproj @@ -33,6 +33,7 @@ + @@ -82,6 +83,7 @@ + diff --git a/build/VS2010/libzstd/libzstd.vcxproj b/build/VS2010/libzstd/libzstd.vcxproj index b84d45b9..02690cef 100644 --- a/build/VS2010/libzstd/libzstd.vcxproj +++ b/build/VS2010/libzstd/libzstd.vcxproj @@ -33,6 +33,7 @@ + @@ -82,6 +83,7 @@ + diff --git a/build/VS2010/zstd/zstd.vcxproj b/build/VS2010/zstd/zstd.vcxproj index d65f55b3..2a21ba4c 100644 --- a/build/VS2010/zstd/zstd.vcxproj +++ b/build/VS2010/zstd/zstd.vcxproj @@ -34,6 +34,7 @@ + @@ -79,6 +80,7 @@ + From 901bba4ca6b4486aaa3e1bc827e13c804fbb2260 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Thu, 22 Aug 2019 16:09:00 -0400 Subject: [PATCH 19/33] Re-Implement Workspace Shrinking when Oversized --- lib/compress/zstd_compress.c | 3 ++- lib/compress/zstd_cwksp.c | 52 +++++++++++++++++++----------------- lib/compress/zstd_cwksp.h | 22 +++++++++------ 3 files changed, 43 insertions(+), 34 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index f88f3b7c..8abb7a48 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -1451,7 +1451,8 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, zc->seqStore.maxNbSeq, zc->seqStore.maxNbLit, zbuff, pledgedSrcSize) ) { DEBUGLOG(4, "ZSTD_equivalentParams()==1 -> consider continue mode"); - if (ZSTD_cwksp_bump_oversized_duration(&zc->workspace) <= ZSTD_WORKSPACETOOLARGE_MAXDURATION) { + ZSTD_cwksp_bump_oversized_duration(&zc->workspace, 0); + if (!ZSTD_cwksp_check_wasteful(&zc->workspace, 0)) { DEBUGLOG(4, "continue mode confirmed (wLog1=%u, blockSize1=%zu)", zc->appliedParams.cParams.windowLog, zc->blockSize); if (ZSTD_indexTooCloseToMax(zc->blockState.matchState.window)) { diff --git a/lib/compress/zstd_cwksp.c b/lib/compress/zstd_cwksp.c index a117ad09..51df4ee3 100644 --- a/lib/compress/zstd_cwksp.c +++ b/lib/compress/zstd_cwksp.c @@ -124,18 +124,6 @@ void* ZSTD_cwksp_reserve_object(ZSTD_cwksp* ws, size_t bytes) { return start; } -// TODO -int ZSTD_cwksp_bump_oversized_duration(ZSTD_cwksp* ws) { - (void)ws; - // if (((BYTE*)ws->allocEnd - (BYTE*)ws->workspace) * ZSTD_WORKSPACETOOLARGE_FACTOR < (BYTE*)ws->workspaceEnd - (BYTE*)ws->workspace) { - // ws->workspaceOversizedDuration++; - // } else { - // ws->workspaceOversizedDuration = 0; - // } - // return ws->workspaceOversizedDuration; - return 0; -} - /** * Invalidates table allocations. * All other allocations remain valid. @@ -150,7 +138,6 @@ void ZSTD_cwksp_clear_tables(ZSTD_cwksp* ws) { */ void ZSTD_cwksp_clear(ZSTD_cwksp* ws) { DEBUGLOG(3, "wksp: clearing!"); - ZSTD_cwksp_bump_oversized_duration(ws); ws->tableEnd = ws->objectEnd; ws->allocStart = ws->workspaceEnd; ws->allocFailed = 0; @@ -186,18 +173,6 @@ void ZSTD_cwksp_free(ZSTD_cwksp* ws, ZSTD_customMem customMem) { ZSTD_cwksp_clear(ws); } -size_t ZSTD_cwksp_available_space(ZSTD_cwksp* ws) { - return (size_t)((BYTE*)ws->allocStart - (BYTE*)ws->tableEnd); -} - -int ZSTD_cwksp_check_available(ZSTD_cwksp* ws, size_t minFree) { - return ZSTD_cwksp_available_space(ws) >= minFree; -} - -int ZSTD_cwksp_check_wasteful(ZSTD_cwksp* ws, size_t minFree) { - return ZSTD_cwksp_check_available(ws, minFree * ZSTD_WORKSPACETOOLARGE_FACTOR) && ws->workspaceOversizedDuration > ZSTD_WORKSPACETOOLARGE_MAXDURATION; -} - size_t ZSTD_cwksp_sizeof(const ZSTD_cwksp* ws) { return (BYTE*)ws->workspaceEnd - (BYTE*)ws->workspace; } @@ -206,6 +181,33 @@ int ZSTD_cwksp_reserve_failed(const ZSTD_cwksp* ws) { return ws->allocFailed; } +size_t ZSTD_cwksp_available_space(ZSTD_cwksp* ws) { + return (size_t)((BYTE*)ws->allocStart - (BYTE*)ws->tableEnd); +} + +void ZSTD_cwksp_bump_oversized_duration( + ZSTD_cwksp* ws, size_t additionalNeededSpace) { + if (ZSTD_cwksp_check_too_large(ws, additionalNeededSpace)) { + ws->workspaceOversizedDuration++; + } else { + ws->workspaceOversizedDuration = 0; + } +} + +int ZSTD_cwksp_check_available(ZSTD_cwksp* ws, size_t additionalNeededSpace) { + return ZSTD_cwksp_available_space(ws) >= additionalNeededSpace; +} + +int ZSTD_cwksp_check_too_large(ZSTD_cwksp* ws, size_t additionalNeededSpace) { + return ZSTD_cwksp_check_available( + ws, additionalNeededSpace * ZSTD_WORKSPACETOOLARGE_FACTOR); +} + +int ZSTD_cwksp_check_wasteful(ZSTD_cwksp* ws, size_t additionalNeededSpace) { + return ZSTD_cwksp_check_too_large(ws, additionalNeededSpace) + && ws->workspaceOversizedDuration > ZSTD_WORKSPACETOOLARGE_MAXDURATION; +} + #if defined (__cplusplus) } #endif diff --git a/lib/compress/zstd_cwksp.h b/lib/compress/zstd_cwksp.h index 965ca6b0..5abc97be 100644 --- a/lib/compress/zstd_cwksp.h +++ b/lib/compress/zstd_cwksp.h @@ -159,8 +159,6 @@ void* ZSTD_cwksp_reserve_table(ZSTD_cwksp* ws, size_t bytes); */ void* ZSTD_cwksp_reserve_object(ZSTD_cwksp* ws, size_t bytes); -int ZSTD_cwksp_bump_oversized_duration(ZSTD_cwksp* ws); - /** * Invalidates table allocations. * All other allocations remain valid. @@ -179,16 +177,24 @@ size_t ZSTD_cwksp_create(ZSTD_cwksp* ws, size_t size, ZSTD_customMem customMem); void ZSTD_cwksp_free(ZSTD_cwksp* ws, ZSTD_customMem customMem); -size_t ZSTD_cwksp_available_space(ZSTD_cwksp* ws); - -int ZSTD_cwksp_check_available(ZSTD_cwksp* ws, size_t minFree); - -int ZSTD_cwksp_check_wasteful(ZSTD_cwksp* ws, size_t minFree); - size_t ZSTD_cwksp_sizeof(const ZSTD_cwksp* ws); int ZSTD_cwksp_reserve_failed(const ZSTD_cwksp* ws); +/*-************************************* +* Functions Checking Free Space +***************************************/ + +size_t ZSTD_cwksp_available_space(ZSTD_cwksp* ws); + +int ZSTD_cwksp_check_available(ZSTD_cwksp* ws, size_t additionalNeededSpace); + +int ZSTD_cwksp_check_too_large(ZSTD_cwksp* ws, size_t additionalNeededSpace); + +int ZSTD_cwksp_check_wasteful(ZSTD_cwksp* ws, size_t additionalNeededSpace); + +void ZSTD_cwksp_bump_oversized_duration(ZSTD_cwksp* ws, size_t additionalNeededSpace); + #if defined (__cplusplus) } #endif From 7321e4c9f3c79f88b0e60eaf23615e03a751719b Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Tue, 27 Aug 2019 15:21:59 -0400 Subject: [PATCH 20/33] Remove Unused noRealloc CRP Value --- lib/compress/zstd_compress.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 8abb7a48..663bfeac 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -1357,7 +1357,7 @@ static size_t ZSTD_continueCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx_params* params, return 0; } -typedef enum { ZSTDcrp_continue, ZSTDcrp_noMemset, ZSTDcrp_noRealloc } ZSTD_compResetPolicy_e; +typedef enum { ZSTDcrp_continue, ZSTDcrp_noMemset } ZSTD_compResetPolicy_e; typedef enum { ZSTD_resetTarget_CDict, ZSTD_resetTarget_CCtx } ZSTD_resetTarget_e; @@ -1461,7 +1461,7 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, &zc->blockState.matchState, &zc->workspace, ¶ms.cParams, - ZSTDcrp_noRealloc, ZSTD_resetTarget_CCtx)); + ZSTDcrp_noMemset, ZSTD_resetTarget_CCtx)); } return ZSTD_continueCCtx(zc, ¶ms, pledgedSrcSize); } } } From 7100d24221b0cc4235bfdc67a8783924903ed055 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Thu, 29 Aug 2019 16:33:45 -0400 Subject: [PATCH 21/33] Fix Rescale Continue Special Case --- lib/compress/zstd_compress.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 663bfeac..d2ff96b5 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -1461,7 +1461,8 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, &zc->blockState.matchState, &zc->workspace, ¶ms.cParams, - ZSTDcrp_noMemset, ZSTD_resetTarget_CCtx)); + crp, + ZSTD_resetTarget_CCtx)); } return ZSTD_continueCCtx(zc, ¶ms, pledgedSrcSize); } } } From 2405c03bcd048caf30e2e6549320840b289de3de Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Tue, 3 Sep 2019 12:41:35 -0400 Subject: [PATCH 22/33] Fix DEBUGLOG Statement Levels --- lib/compress/zstd_cwksp.c | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/lib/compress/zstd_cwksp.c b/lib/compress/zstd_cwksp.c index 51df4ee3..f8503975 100644 --- a/lib/compress/zstd_cwksp.c +++ b/lib/compress/zstd_cwksp.c @@ -30,7 +30,7 @@ static void* ZSTD_cwksp_reserve_internal(ZSTD_cwksp* ws, size_t bytes, ZSTD_cwks /* TODO(felixh): alignment */ void* alloc = (BYTE *)ws->allocStart - bytes; void* bottom = ws->tableEnd; - DEBUGLOG(3, "wksp: reserving align %zd bytes, %zd bytes remaining", + DEBUGLOG(4, "cwksp: reserving align %zd bytes, %zd bytes remaining", bytes, (BYTE *)alloc - (BYTE *)bottom); assert(phase >= ws->phase); if (phase > ws->phase) { @@ -80,14 +80,13 @@ void* ZSTD_cwksp_reserve_aligned(ZSTD_cwksp* ws, size_t bytes) { * memset()-ing them. */ void* ZSTD_cwksp_reserve_table(ZSTD_cwksp* ws, size_t bytes) { - /* TODO(felixh): alignment */ const ZSTD_cwksp_alloc_phase_e phase = ZSTD_cwksp_alloc_aligned; void* alloc = ws->tableEnd; void* end = (BYTE *)alloc + bytes; void* top = ws->allocStart; - DEBUGLOG(3, "wksp: reserving table %zd bytes, %zd bytes remaining", - bytes, (BYTE *)top - (BYTE *)end); - assert((bytes & (sizeof(U32)-1)) == 0); // TODO ??? + DEBUGLOG(4, "cwksp: reserving table %zd bytes, %zd bytes remaining", + bytes, ZSTD_cwksp_available_space(ws) - bytes); + assert((bytes & (sizeof(U32)-1)) == 0); assert(phase >= ws->phase); if (phase > ws->phase) { if (ws->phase <= ZSTD_cwksp_alloc_buffers) { @@ -97,6 +96,7 @@ void* ZSTD_cwksp_reserve_table(ZSTD_cwksp* ws, size_t bytes) { } assert(end <= top); if (end > top) { + DEBUGLOG(4, "cwksp: object alloc failed!"); ws->allocFailed = 1; return NULL; } @@ -111,11 +111,13 @@ void* ZSTD_cwksp_reserve_object(ZSTD_cwksp* ws, size_t bytes) { size_t roundedBytes = ZSTD_cwksp_align(bytes, sizeof(void*)); void* start = ws->objectEnd; void* end = (BYTE*)start + roundedBytes; - DEBUGLOG(3, "wksp: reserving %zd bytes object (rounded to %zd), %zd bytes remaining", bytes, roundedBytes, (BYTE *)ws->workspaceEnd - (BYTE *)end); + DEBUGLOG(4, + "cwksp: reserving %zd bytes object (rounded to %zd), %zd bytes remaining", + bytes, roundedBytes, ZSTD_cwksp_available_space(ws) - roundedBytes); assert(((size_t)start & (sizeof(void*)-1)) == 0); assert((bytes & (sizeof(void*)-1)) == 0); if (ws->phase != ZSTD_cwksp_alloc_objects || end > ws->workspaceEnd) { - DEBUGLOG(3, "wksp: object alloc failed!"); + DEBUGLOG(4, "cwksp: object alloc failed!"); ws->allocFailed = 1; return NULL; } @@ -129,6 +131,7 @@ void* ZSTD_cwksp_reserve_object(ZSTD_cwksp* ws, size_t bytes) { * All other allocations remain valid. */ void ZSTD_cwksp_clear_tables(ZSTD_cwksp* ws) { + DEBUGLOG(4, "cwksp: clearing tables!"); ws->tableEnd = ws->objectEnd; } @@ -137,7 +140,7 @@ void ZSTD_cwksp_clear_tables(ZSTD_cwksp* ws) { * Object allocations remain valid. */ void ZSTD_cwksp_clear(ZSTD_cwksp* ws) { - DEBUGLOG(3, "wksp: clearing!"); + DEBUGLOG(4, "cwksp: clearing!"); ws->tableEnd = ws->objectEnd; ws->allocStart = ws->workspaceEnd; ws->allocFailed = 0; @@ -147,7 +150,7 @@ void ZSTD_cwksp_clear(ZSTD_cwksp* ws) { } void ZSTD_cwksp_init(ZSTD_cwksp* ws, void* start, size_t size) { - DEBUGLOG(3, "wksp: init'ing with %zd bytes", size); + DEBUGLOG(4, "cwksp: init'ing workspace with %zd bytes", size); assert(((size_t)start & (sizeof(void*)-1)) == 0); /* ensure correct alignment */ ws->workspace = start; ws->workspaceEnd = (BYTE*)start + size; @@ -159,14 +162,14 @@ void ZSTD_cwksp_init(ZSTD_cwksp* ws, void* start, size_t size) { size_t ZSTD_cwksp_create(ZSTD_cwksp* ws, size_t size, ZSTD_customMem customMem) { void* workspace = ZSTD_malloc(size, customMem); - DEBUGLOG(3, "wksp: creating with %zd bytes", size); + DEBUGLOG(4, "cwksp: creating new workspace with %zd bytes", size); RETURN_ERROR_IF(workspace == NULL, memory_allocation); ZSTD_cwksp_init(ws, workspace, size); return 0; } void ZSTD_cwksp_free(ZSTD_cwksp* ws, ZSTD_customMem customMem) { - DEBUGLOG(3, "wksp: freeing"); + DEBUGLOG(4, "cwksp: freeing workspace"); ZSTD_free(ws->workspace, customMem); ws->workspace = NULL; ws->workspaceEnd = NULL; From 8549ae9f1da6e01c90cbac674b5495a49467f7d9 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Tue, 3 Sep 2019 12:48:45 -0400 Subject: [PATCH 23/33] Hide Workspace Movement Behind Helper Function --- lib/compress/zstd_compress.c | 6 +++--- lib/compress/zstd_cwksp.c | 5 +++++ lib/compress/zstd_cwksp.h | 6 ++++++ 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index d2ff96b5..99c8a606 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -94,7 +94,7 @@ ZSTD_CCtx* ZSTD_initStaticCCtx(void *workspace, size_t workspaceSize) return NULL; } memset(cctx, 0, sizeof(ZSTD_CCtx)); - cctx->workspace = ws; + ZSTD_cwksp_move(&cctx->workspace, &ws); cctx->staticSize = workspaceSize; /* statically sized space. entropyWorkspace never moves (but prev/next block swap places) */ @@ -3181,7 +3181,7 @@ ZSTD_CDict* ZSTD_createCDict_advanced(const void* dictBuffer, size_t dictSize, cdict = (ZSTD_CDict*)ZSTD_cwksp_reserve_object(&ws, sizeof(ZSTD_CDict)); assert(cdict != NULL); - cdict->workspace = ws; + ZSTD_cwksp_move(&cdict->workspace, &ws); cdict->customMem = customMem; if (ZSTD_isError( ZSTD_initCDict_internal(cdict, dictBuffer, dictSize, @@ -3257,7 +3257,7 @@ const ZSTD_CDict* ZSTD_initStaticCDict( ZSTD_cwksp_init(&ws, workspace, workspaceSize); cdict = (ZSTD_CDict*)ZSTD_cwksp_reserve_object(&ws, sizeof(ZSTD_CDict)); if (cdict == NULL) return NULL; - cdict->workspace = ws; + ZSTD_cwksp_move(&cdict->workspace, &ws); } DEBUGLOG(4, "(workspaceSize < neededSize) : (%u < %u) => %u", diff --git a/lib/compress/zstd_cwksp.c b/lib/compress/zstd_cwksp.c index f8503975..0ff6bec0 100644 --- a/lib/compress/zstd_cwksp.c +++ b/lib/compress/zstd_cwksp.c @@ -176,6 +176,11 @@ void ZSTD_cwksp_free(ZSTD_cwksp* ws, ZSTD_customMem customMem) { ZSTD_cwksp_clear(ws); } +void ZSTD_cwksp_move(ZSTD_cwksp* dst, ZSTD_cwksp* src) { + *dst = *src; + memset(src, 0, sizeof(ZSTD_cwksp)); +} + size_t ZSTD_cwksp_sizeof(const ZSTD_cwksp* ws) { return (BYTE*)ws->workspaceEnd - (BYTE*)ws->workspace; } diff --git a/lib/compress/zstd_cwksp.h b/lib/compress/zstd_cwksp.h index 5abc97be..630ad3ba 100644 --- a/lib/compress/zstd_cwksp.h +++ b/lib/compress/zstd_cwksp.h @@ -177,6 +177,12 @@ size_t ZSTD_cwksp_create(ZSTD_cwksp* ws, size_t size, ZSTD_customMem customMem); void ZSTD_cwksp_free(ZSTD_cwksp* ws, ZSTD_customMem customMem); +/** + * Moves the management of a workspace from one cwksp to another. The src cwksp + * is left in an invalid state (src must be re-init()'ed before its used again). + */ +void ZSTD_cwksp_move(ZSTD_cwksp* dst, ZSTD_cwksp* src); + size_t ZSTD_cwksp_sizeof(const ZSTD_cwksp* ws); int ZSTD_cwksp_reserve_failed(const ZSTD_cwksp* ws); From 7d7b665c9041d6ee53e0aab73a487e7f61001064 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Tue, 3 Sep 2019 12:59:10 -0400 Subject: [PATCH 24/33] Pull Phase Advance Logic Out into Internal Function --- lib/compress/zstd_cwksp.c | 35 ++++++++++++++++++----------------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/lib/compress/zstd_cwksp.c b/lib/compress/zstd_cwksp.c index 0ff6bec0..36ba4bbe 100644 --- a/lib/compress/zstd_cwksp.c +++ b/lib/compress/zstd_cwksp.c @@ -23,15 +23,8 @@ size_t ZSTD_cwksp_align(size_t size, size_t const align) { return (size + mask) & ~mask; } -/** - * Internal function, use wrappers instead. - */ -static void* ZSTD_cwksp_reserve_internal(ZSTD_cwksp* ws, size_t bytes, ZSTD_cwksp_alloc_phase_e phase) { - /* TODO(felixh): alignment */ - void* alloc = (BYTE *)ws->allocStart - bytes; - void* bottom = ws->tableEnd; - DEBUGLOG(4, "cwksp: reserving align %zd bytes, %zd bytes remaining", - bytes, (BYTE *)alloc - (BYTE *)bottom); +static void ZSTD_cwksp_internal_advance_phase( + ZSTD_cwksp* ws, ZSTD_cwksp_alloc_phase_e phase) { assert(phase >= ws->phase); if (phase > ws->phase) { if (ws->phase < ZSTD_cwksp_alloc_buffers && @@ -46,10 +39,23 @@ static void* ZSTD_cwksp_reserve_internal(ZSTD_cwksp* ws, size_t bytes, ZSTD_cwks * workspace is too large, and specifically when it is too large * by a larger margin than the space that will be consumed. */ /* TODO: cleaner, compiler warning friendly way to do this??? */ - alloc = (BYTE*)alloc - ((size_t)alloc & (sizeof(U32)-1)); + ws->allocStart = (BYTE*)ws->allocStart - ((size_t)ws->allocStart & (sizeof(U32)-1)); } ws->phase = phase; } +} + +/** + * Internal function, use wrappers instead. + */ +static void* ZSTD_cwksp_reserve_internal(ZSTD_cwksp* ws, size_t bytes, ZSTD_cwksp_alloc_phase_e phase) { + /* TODO(felixh): alignment */ + void* alloc; + void* bottom = ws->tableEnd; + ZSTD_cwksp_internal_advance_phase(ws, phase); + alloc = (BYTE *)ws->allocStart - bytes; + DEBUGLOG(4, "cwksp: reserving align %zd bytes, %zd bytes remaining", + bytes, ZSTD_cwksp_available_space(ws) - bytes); assert(alloc >= bottom); if (alloc < bottom) { ws->allocFailed = 1; @@ -87,13 +93,7 @@ void* ZSTD_cwksp_reserve_table(ZSTD_cwksp* ws, size_t bytes) { DEBUGLOG(4, "cwksp: reserving table %zd bytes, %zd bytes remaining", bytes, ZSTD_cwksp_available_space(ws) - bytes); assert((bytes & (sizeof(U32)-1)) == 0); - assert(phase >= ws->phase); - if (phase > ws->phase) { - if (ws->phase <= ZSTD_cwksp_alloc_buffers) { - - } - ws->phase = phase; - } + ZSTD_cwksp_internal_advance_phase(ws, phase); assert(end <= top); if (end > top) { DEBUGLOG(4, "cwksp: object alloc failed!"); @@ -116,6 +116,7 @@ void* ZSTD_cwksp_reserve_object(ZSTD_cwksp* ws, size_t bytes) { bytes, roundedBytes, ZSTD_cwksp_available_space(ws) - roundedBytes); assert(((size_t)start & (sizeof(void*)-1)) == 0); assert((bytes & (sizeof(void*)-1)) == 0); + /* we must be in the first phase, no advance is possible */ if (ws->phase != ZSTD_cwksp_alloc_objects || end > ws->workspaceEnd) { DEBUGLOG(4, "cwksp: object alloc failed!"); ws->allocFailed = 1; From c60e1c3be5d4dd73040f5b16c0672bf758186503 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Tue, 3 Sep 2019 13:00:59 -0400 Subject: [PATCH 25/33] Nit --- lib/compress/zstd_cwksp.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/compress/zstd_cwksp.c b/lib/compress/zstd_cwksp.c index 36ba4bbe..a9471639 100644 --- a/lib/compress/zstd_cwksp.c +++ b/lib/compress/zstd_cwksp.c @@ -112,7 +112,7 @@ void* ZSTD_cwksp_reserve_object(ZSTD_cwksp* ws, size_t bytes) { void* start = ws->objectEnd; void* end = (BYTE*)start + roundedBytes; DEBUGLOG(4, - "cwksp: reserving %zd bytes object (rounded to %zd), %zd bytes remaining", + "cwksp: reserving object %zd bytes (rounded to %zd), %zd bytes remaining", bytes, roundedBytes, ZSTD_cwksp_available_space(ws) - roundedBytes); assert(((size_t)start & (sizeof(void*)-1)) == 0); assert((bytes & (sizeof(void*)-1)) == 0); From 1120e4d9624a6ccc6ae68530011a4dbb0548bda1 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Mon, 9 Sep 2019 14:04:39 -0400 Subject: [PATCH 26/33] Clean Up TODOs and Comments pt. II --- lib/compress/zstd_cwksp.c | 11 ++++----- lib/compress/zstd_cwksp.h | 49 ++++++++++++++++++++++++++------------- 2 files changed, 37 insertions(+), 23 deletions(-) diff --git a/lib/compress/zstd_cwksp.c b/lib/compress/zstd_cwksp.c index a9471639..08765e40 100644 --- a/lib/compress/zstd_cwksp.c +++ b/lib/compress/zstd_cwksp.c @@ -45,16 +45,13 @@ static void ZSTD_cwksp_internal_advance_phase( } } -/** - * Internal function, use wrappers instead. - */ -static void* ZSTD_cwksp_reserve_internal(ZSTD_cwksp* ws, size_t bytes, ZSTD_cwksp_alloc_phase_e phase) { - /* TODO(felixh): alignment */ +static void* ZSTD_cwksp_reserve_internal( + ZSTD_cwksp* ws, size_t bytes, ZSTD_cwksp_alloc_phase_e phase) { void* alloc; void* bottom = ws->tableEnd; ZSTD_cwksp_internal_advance_phase(ws, phase); alloc = (BYTE *)ws->allocStart - bytes; - DEBUGLOG(4, "cwksp: reserving align %zd bytes, %zd bytes remaining", + DEBUGLOG(4, "cwksp: reserving %zd bytes, %zd bytes remaining", bytes, ZSTD_cwksp_available_space(ws) - bytes); assert(alloc >= bottom); if (alloc < bottom) { @@ -76,7 +73,7 @@ BYTE* ZSTD_cwksp_reserve_buffer(ZSTD_cwksp* ws, size_t bytes) { * Aligned on sizeof(unsigned). */ void* ZSTD_cwksp_reserve_aligned(ZSTD_cwksp* ws, size_t bytes) { - assert((bytes & (sizeof(U32)-1)) == 0); // TODO ??? + assert((bytes & (sizeof(U32)-1)) == 0); return ZSTD_cwksp_reserve_internal(ws, ZSTD_cwksp_align(bytes, sizeof(U32)), ZSTD_cwksp_alloc_aligned); } diff --git a/lib/compress/zstd_cwksp.h b/lib/compress/zstd_cwksp.h index 630ad3ba..80d174f0 100644 --- a/lib/compress/zstd_cwksp.h +++ b/lib/compress/zstd_cwksp.h @@ -50,14 +50,25 @@ typedef enum { * is called the workspace. * * Several optimizations complicate that process of allocating memory ranges - * from this workspace for each datastructure: + * from this workspace for each internal datastructure: * - * - These different internal datastructures have different setup requirements. - * Some (e.g., the window buffer) don't care, and are happy to accept - * uninitialized memory. Others (e.g., the matchstate tables) can accept - * memory filled with unknown but bounded values (i.e., a memory area whose - * values are known to be constrained between 0 and some upper bound). If - * that constraint isn't known to be satisfied, the area has to be cleared. + * - These different internal datastructures have different setup requirements: + * + * - The static objects need to be cleared once and can then be trivially + * reused for each compression. + * + * - Various buffers don't need to be initialized at all--they are always + * written into before they're read. + * + * - The matchstate tables have a unique requirement that they don't need + * their memory to be totally cleared, but they do need the memory to have + * some bound, i.e., a guarantee that all values in the memory they've been + * allocated is less than some maximum value (which is the starting value + * for the indices that they will then use for compression). When this + * guarantee is provided to them, they can use the memory without any setup + * work. When it can't, they have to clear the area. + * + * - These buffers also have different alignment requirements. * * - We would like to reuse the objects in the workspace for multiple * compressions without having to perform any expensive reallocation or @@ -67,13 +78,16 @@ typedef enum { * multiple compressions **even when the compression parameters change** and * we need to resize some of the objects (where possible). * + * To attempt to manage this buffer, given these constraints, the ZSTD_cwksp + * abstraction was created. It works as follows: + * * Workspace Layout: * * [ ... workspace ... ] * [objects][tables ... ->] free space [<- ... aligned][<- ... buffers] * - * In order to accomplish this, the various objects that live in the workspace - * are divided into the following categories: + * The various objects that live in the workspace are divided into the + * following categories, and are allocated separately: * * - Static objects: this is optionally the enclosing ZSTD_CCtx or ZSTD_CDict, * so that literally everything fits in a single buffer. Note: if present, @@ -94,11 +108,11 @@ typedef enum { * uint32_t arrays, all of whose values are between 0 and (nextSrc - base). * Their sizes depend on the cparams. * - * - Aligned: these buffers are used for various purposes that don't require - * any initialization before they're used. + * - Aligned: these buffers are used for various purposes that require 4 byte + * alignment, but don't require any initialization before they're used. * - * - Uninitialized memory: these buffers are used for various purposes that - * don't require any initialization before they're used. This means they can + * - Buffers: these buffers are used for various purposes that don't require + * any alignment or initialization before they're used. This means they can * be moved around at no cost for a new compression. * * Allocating Memory: @@ -111,9 +125,7 @@ typedef enum { * 3. Aligned * 4. Tables * - * Reusing Table Space: - * - * TODO(felixh): ... + * Attempts to reserve objects of different types out of order will fail. */ typedef struct { void* workspace; @@ -171,6 +183,11 @@ void ZSTD_cwksp_clear_tables(ZSTD_cwksp* ws); */ void ZSTD_cwksp_clear(ZSTD_cwksp* ws); +/** + * The provided workspace takes ownership of the buffer [start, start+size). + * Any existing values in the workspace are ignored (the previously managed + * buffer, if present, must be separately freed). + */ void ZSTD_cwksp_init(ZSTD_cwksp* ws, void* start, size_t size); size_t ZSTD_cwksp_create(ZSTD_cwksp* ws, size_t size, ZSTD_customMem customMem); From 0a65a67901f6cc3d12f1b42f111680d0419a5c5c Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Mon, 9 Sep 2019 14:59:09 -0400 Subject: [PATCH 27/33] Shorten `&zc->workspace` -> `ws` in `ZSTD_resetCCtx_internal()` --- lib/compress/zstd_compress.c | 51 ++++++++++++++++++------------------ 1 file changed, 26 insertions(+), 25 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 99c8a606..cbda17b8 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -1440,6 +1440,7 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, ZSTD_compResetPolicy_e const crp, ZSTD_buffered_policy_e const zbuff) { + ZSTD_cwksp* const ws = &zc->workspace; DEBUGLOG(4, "ZSTD_resetCCtx_internal: pledgedSrcSize=%u, wlog=%u", (U32)pledgedSrcSize, params.cParams.windowLog); assert(!ZSTD_isError(ZSTD_checkCParams(params.cParams))); @@ -1451,15 +1452,15 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, zc->seqStore.maxNbSeq, zc->seqStore.maxNbLit, zbuff, pledgedSrcSize) ) { DEBUGLOG(4, "ZSTD_equivalentParams()==1 -> consider continue mode"); - ZSTD_cwksp_bump_oversized_duration(&zc->workspace, 0); - if (!ZSTD_cwksp_check_wasteful(&zc->workspace, 0)) { + ZSTD_cwksp_bump_oversized_duration(ws, 0); + if (!ZSTD_cwksp_check_wasteful(ws, 0)) { DEBUGLOG(4, "continue mode confirmed (wLog1=%u, blockSize1=%zu)", zc->appliedParams.cParams.windowLog, zc->blockSize); if (ZSTD_indexTooCloseToMax(zc->blockState.matchState.window)) { /* prefer a reset, faster than a rescale */ FORWARD_IF_ERROR(ZSTD_reset_matchState( &zc->blockState.matchState, - &zc->workspace, + ws, ¶ms.cParams, crp, ZSTD_resetTarget_CCtx)); @@ -1502,8 +1503,8 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, tokenSpace + bufferSpace; - int const workspaceTooSmall = !ZSTD_cwksp_check_available(&zc->workspace, neededSpace); - int const workspaceWasteful = ZSTD_cwksp_check_wasteful(&zc->workspace, neededSpace); + int const workspaceTooSmall = !ZSTD_cwksp_check_available(ws, neededSpace); + int const workspaceWasteful = ZSTD_cwksp_check_wasteful(ws, neededSpace); DEBUGLOG(4, "Need %zuKB workspace, including %zuKB for match state, and %zuKB for buffers", neededSpace>>10, matchStateSize>>10, bufferSpace>>10); @@ -1511,28 +1512,28 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, if (workspaceTooSmall || workspaceWasteful) { DEBUGLOG(4, "Resize workspaceSize from %zuKB to %zuKB", - ZSTD_cwksp_sizeof(&zc->workspace) >> 10, + ZSTD_cwksp_sizeof(ws) >> 10, neededSpace >> 10); RETURN_ERROR_IF(zc->staticSize, memory_allocation, "static cctx : no resize"); - ZSTD_cwksp_free(&zc->workspace, zc->customMem); - FORWARD_IF_ERROR(ZSTD_cwksp_create(&zc->workspace, neededSpace, zc->customMem)); + ZSTD_cwksp_free(ws, zc->customMem); + FORWARD_IF_ERROR(ZSTD_cwksp_create(ws, neededSpace, zc->customMem)); DEBUGLOG(5, "reserving object space"); /* Statically sized space. * entropyWorkspace never moves, * though prev/next block swap places */ - assert(ZSTD_cwksp_check_available(&zc->workspace, 2 * sizeof(ZSTD_compressedBlockState_t))); - zc->blockState.prevCBlock = (ZSTD_compressedBlockState_t*) ZSTD_cwksp_reserve_object(&zc->workspace, sizeof(ZSTD_compressedBlockState_t)); + assert(ZSTD_cwksp_check_available(ws, 2 * sizeof(ZSTD_compressedBlockState_t))); + zc->blockState.prevCBlock = (ZSTD_compressedBlockState_t*) ZSTD_cwksp_reserve_object(ws, sizeof(ZSTD_compressedBlockState_t)); RETURN_ERROR_IF(zc->blockState.prevCBlock == NULL, memory_allocation, "couldn't allocate prevCBlock"); - zc->blockState.nextCBlock = (ZSTD_compressedBlockState_t*) ZSTD_cwksp_reserve_object(&zc->workspace, sizeof(ZSTD_compressedBlockState_t)); + zc->blockState.nextCBlock = (ZSTD_compressedBlockState_t*) ZSTD_cwksp_reserve_object(ws, sizeof(ZSTD_compressedBlockState_t)); RETURN_ERROR_IF(zc->blockState.nextCBlock == NULL, memory_allocation, "couldn't allocate nextCBlock"); - zc->entropyWorkspace = (U32*) ZSTD_cwksp_reserve_object(&zc->workspace, HUF_WORKSPACE_SIZE); + zc->entropyWorkspace = (U32*) ZSTD_cwksp_reserve_object(ws, HUF_WORKSPACE_SIZE); RETURN_ERROR_IF(zc->blockState.nextCBlock == NULL, memory_allocation, "couldn't allocate entropyWorkspace"); } } - ZSTD_cwksp_clear(&zc->workspace); + ZSTD_cwksp_clear(ws); /* init params */ zc->appliedParams = params; @@ -1555,35 +1556,35 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, /* ZSTD_wildcopy() is used to copy into the literals buffer, * so we have to oversize the buffer by WILDCOPY_OVERLENGTH bytes. */ - zc->seqStore.litStart = ZSTD_cwksp_reserve_buffer(&zc->workspace, blockSize + WILDCOPY_OVERLENGTH); + zc->seqStore.litStart = ZSTD_cwksp_reserve_buffer(ws, blockSize + WILDCOPY_OVERLENGTH); zc->seqStore.maxNbLit = blockSize; /* buffers */ zc->inBuffSize = buffInSize; - zc->inBuff = (char*)ZSTD_cwksp_reserve_buffer(&zc->workspace, buffInSize); + zc->inBuff = (char*)ZSTD_cwksp_reserve_buffer(ws, buffInSize); zc->outBuffSize = buffOutSize; - zc->outBuff = (char*)ZSTD_cwksp_reserve_buffer(&zc->workspace, buffOutSize); + zc->outBuff = (char*)ZSTD_cwksp_reserve_buffer(ws, buffOutSize); /* ldm bucketOffsets table */ if (params.ldmParams.enableLdm) { size_t const ldmBucketSize = ((size_t)1) << (params.ldmParams.hashLog - params.ldmParams.bucketSizeLog); - zc->ldmState.bucketOffsets = ZSTD_cwksp_reserve_buffer(&zc->workspace, ldmBucketSize); + zc->ldmState.bucketOffsets = ZSTD_cwksp_reserve_buffer(ws, ldmBucketSize); memset(zc->ldmState.bucketOffsets, 0, ldmBucketSize); } /* sequences storage */ ZSTD_referenceExternalSequences(zc, NULL, 0); zc->seqStore.maxNbSeq = maxNbSeq; - zc->seqStore.llCode = ZSTD_cwksp_reserve_buffer(&zc->workspace, maxNbSeq * sizeof(BYTE)); - zc->seqStore.mlCode = ZSTD_cwksp_reserve_buffer(&zc->workspace, maxNbSeq * sizeof(BYTE)); - zc->seqStore.ofCode = ZSTD_cwksp_reserve_buffer(&zc->workspace, maxNbSeq * sizeof(BYTE)); - zc->seqStore.sequencesStart = (seqDef*)ZSTD_cwksp_reserve_aligned(&zc->workspace, maxNbSeq * sizeof(seqDef)); + zc->seqStore.llCode = ZSTD_cwksp_reserve_buffer(ws, maxNbSeq * sizeof(BYTE)); + zc->seqStore.mlCode = ZSTD_cwksp_reserve_buffer(ws, maxNbSeq * sizeof(BYTE)); + zc->seqStore.ofCode = ZSTD_cwksp_reserve_buffer(ws, maxNbSeq * sizeof(BYTE)); + zc->seqStore.sequencesStart = (seqDef*)ZSTD_cwksp_reserve_aligned(ws, maxNbSeq * sizeof(seqDef)); FORWARD_IF_ERROR(ZSTD_reset_matchState( &zc->blockState.matchState, - &zc->workspace, + ws, ¶ms.cParams, crp, ZSTD_resetTarget_CCtx)); @@ -1591,16 +1592,16 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, /* initialize bucketOffsets table separately for pointer alignment */ if (params.ldmParams.enableLdm) { size_t const ldmHSize = ((size_t)1) << params.ldmParams.hashLog; - zc->ldmState.hashTable = (ldmEntry_t*)ZSTD_cwksp_reserve_aligned(&zc->workspace, ldmHSize * sizeof(ldmEntry_t)); + zc->ldmState.hashTable = (ldmEntry_t*)ZSTD_cwksp_reserve_aligned(ws, ldmHSize * sizeof(ldmEntry_t)); memset(zc->ldmState.hashTable, 0, ldmHSize * sizeof(ldmEntry_t)); - zc->ldmSequences = (rawSeq*)ZSTD_cwksp_reserve_aligned(&zc->workspace, maxNbLdmSeq * sizeof(rawSeq)); + zc->ldmSequences = (rawSeq*)ZSTD_cwksp_reserve_aligned(ws, maxNbLdmSeq * sizeof(rawSeq)); zc->maxNbLdmSequences = maxNbLdmSeq; memset(&zc->ldmState.window, 0, sizeof(zc->ldmState.window)); ZSTD_window_clear(&zc->ldmState.window); } - DEBUGLOG(3, "wksp: finished allocating, %zd bytes remain available", ZSTD_cwksp_available_space(&zc->workspace)); + DEBUGLOG(3, "wksp: finished allocating, %zd bytes remain available", ZSTD_cwksp_available_space(ws)); return 0; } From e3703825a843b04be0aff3ff38336ee5188ea84c Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Mon, 9 Sep 2019 15:12:14 -0400 Subject: [PATCH 28/33] Fix workspaceTooSmall Calculation --- lib/compress/zstd_compress.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index cbda17b8..7adcb7c1 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -1488,13 +1488,15 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, size_t const maxNbLdmSeq = ZSTD_ldm_getMaxNbSeq(params.ldmParams, blockSize); /* Check if workspace is large enough, alloc a new one if needed */ - { size_t const entropySpace = HUF_WORKSPACE_SIZE; + { size_t const cctxSpace = zc->staticSize ? sizeof(ZSTD_CCtx) : 0; + size_t const entropySpace = HUF_WORKSPACE_SIZE; size_t const blockStateSpace = 2 * sizeof(ZSTD_compressedBlockState_t); size_t const bufferSpace = buffInSize + buffOutSize; size_t const ldmSpace = ZSTD_ldm_getTableSize(params.ldmParams); size_t const ldmSeqSpace = maxNbLdmSeq * sizeof(rawSeq); size_t const neededSpace = + cctxSpace + entropySpace + blockStateSpace + ldmSpace + @@ -1503,7 +1505,7 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, tokenSpace + bufferSpace; - int const workspaceTooSmall = !ZSTD_cwksp_check_available(ws, neededSpace); + int const workspaceTooSmall = ZSTD_cwksp_sizeof(ws) < neededSpace; int const workspaceWasteful = ZSTD_cwksp_check_wasteful(ws, neededSpace); DEBUGLOG(4, "Need %zuKB workspace, including %zuKB for match state, and %zuKB for buffers", From eb6f69d97838add9d713140999900acdcb8b4c9b Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Mon, 9 Sep 2019 16:45:17 -0400 Subject: [PATCH 29/33] Fix sizeof_CCtx and sizeof_CDict Calculations for Statically Init'ed Objects --- lib/compress/zstd_compress.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 7adcb7c1..c8b031f8 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -166,7 +166,9 @@ static size_t ZSTD_sizeof_mtctx(const ZSTD_CCtx* cctx) size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx) { if (cctx==NULL) return 0; /* support sizeof on NULL */ - return sizeof(*cctx) + ZSTD_cwksp_sizeof(&cctx->workspace) + /* cctx may be in the workspace */ + return (cctx->workspace.workspace == cctx ? 0 : sizeof(*cctx)) + + ZSTD_cwksp_sizeof(&cctx->workspace) + ZSTD_sizeof_localDict(cctx->localDict) + ZSTD_sizeof_mtctx(cctx); } @@ -3103,7 +3105,9 @@ size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict) { if (cdict==NULL) return 0; /* support sizeof on NULL */ DEBUGLOG(5, "sizeof(*cdict) : %u", (unsigned)sizeof(*cdict)); - return ZSTD_cwksp_sizeof(&cdict->workspace) + sizeof(*cdict); + /* cdict may be in the workspace */ + return (cdict->workspace.workspace == cdict ? 0 : sizeof(*cdict)) + + ZSTD_cwksp_sizeof(&cdict->workspace); } static size_t ZSTD_initCDict_internal( From 0db3ffe7eea3836d3d54c876281a8e9533d5ea22 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Tue, 3 Sep 2019 13:13:16 -0400 Subject: [PATCH 30/33] Forward resetCCtx Errors when Using CDict --- lib/compress/zstd_compress.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index c8b031f8..2cf508f3 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -1665,8 +1665,8 @@ ZSTD_resetCCtx_byAttachingCDict(ZSTD_CCtx* cctx, * has its own tables. */ params.cParams = ZSTD_adjustCParams_internal(*cdict_cParams, pledgedSrcSize, 0); params.cParams.windowLog = windowLog; - ZSTD_resetCCtx_internal(cctx, params, pledgedSrcSize, - ZSTDcrp_continue, zbuff); + FORWARD_IF_ERROR(ZSTD_resetCCtx_internal(cctx, params, pledgedSrcSize, + ZSTDcrp_continue, zbuff)); assert(cctx->appliedParams.cParams.strategy == cdict_cParams->strategy); } @@ -1714,8 +1714,8 @@ static size_t ZSTD_resetCCtx_byCopyingCDict(ZSTD_CCtx* cctx, /* Copy only compression parameters related to tables. */ params.cParams = *cdict_cParams; params.cParams.windowLog = windowLog; - ZSTD_resetCCtx_internal(cctx, params, pledgedSrcSize, - ZSTDcrp_noMemset, zbuff); + FORWARD_IF_ERROR(ZSTD_resetCCtx_internal(cctx, params, pledgedSrcSize, + ZSTDcrp_noMemset, zbuff)); assert(cctx->appliedParams.cParams.strategy == cdict_cParams->strategy); assert(cctx->appliedParams.cParams.hashLog == cdict_cParams->hashLog); assert(cctx->appliedParams.cParams.chainLog == cdict_cParams->chainLog); From 91bf1babd1f3be9607c5bb917fe2442c20a0c2ba Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Mon, 9 Sep 2019 18:30:53 -0400 Subject: [PATCH 31/33] Inline Workspace Functions --- lib/compress/zstd_cwksp.c | 200 +------------------------------------- lib/compress/zstd_cwksp.h | 182 ++++++++++++++++++++++++++++++---- 2 files changed, 163 insertions(+), 219 deletions(-) diff --git a/lib/compress/zstd_cwksp.c b/lib/compress/zstd_cwksp.c index 08765e40..a757910d 100644 --- a/lib/compress/zstd_cwksp.c +++ b/lib/compress/zstd_cwksp.c @@ -14,205 +14,7 @@ extern "C" { #endif -/** - * Align must be a power of 2. - */ -size_t ZSTD_cwksp_align(size_t size, size_t const align) { - size_t const mask = align - 1; - assert((align & mask) == 0); - return (size + mask) & ~mask; -} - -static void ZSTD_cwksp_internal_advance_phase( - ZSTD_cwksp* ws, ZSTD_cwksp_alloc_phase_e phase) { - assert(phase >= ws->phase); - if (phase > ws->phase) { - if (ws->phase < ZSTD_cwksp_alloc_buffers && - phase >= ZSTD_cwksp_alloc_buffers) { - } - if (ws->phase < ZSTD_cwksp_alloc_aligned && - phase >= ZSTD_cwksp_alloc_aligned) { - /* If unaligned allocations down from a too-large top have left us - * unaligned, we need to realign our alloc ptr. Technically, this - * can consume space that is unaccounted for in the neededSpace - * calculation. However, I believe this can only happen when the - * workspace is too large, and specifically when it is too large - * by a larger margin than the space that will be consumed. */ - /* TODO: cleaner, compiler warning friendly way to do this??? */ - ws->allocStart = (BYTE*)ws->allocStart - ((size_t)ws->allocStart & (sizeof(U32)-1)); - } - ws->phase = phase; - } -} - -static void* ZSTD_cwksp_reserve_internal( - ZSTD_cwksp* ws, size_t bytes, ZSTD_cwksp_alloc_phase_e phase) { - void* alloc; - void* bottom = ws->tableEnd; - ZSTD_cwksp_internal_advance_phase(ws, phase); - alloc = (BYTE *)ws->allocStart - bytes; - DEBUGLOG(4, "cwksp: reserving %zd bytes, %zd bytes remaining", - bytes, ZSTD_cwksp_available_space(ws) - bytes); - assert(alloc >= bottom); - if (alloc < bottom) { - ws->allocFailed = 1; - return NULL; - } - ws->allocStart = alloc; - return alloc; -} - -/** - * Unaligned. - */ -BYTE* ZSTD_cwksp_reserve_buffer(ZSTD_cwksp* ws, size_t bytes) { - return (BYTE*)ZSTD_cwksp_reserve_internal(ws, bytes, ZSTD_cwksp_alloc_buffers); -} - -/** - * Aligned on sizeof(unsigned). - */ -void* ZSTD_cwksp_reserve_aligned(ZSTD_cwksp* ws, size_t bytes) { - assert((bytes & (sizeof(U32)-1)) == 0); - return ZSTD_cwksp_reserve_internal(ws, ZSTD_cwksp_align(bytes, sizeof(U32)), ZSTD_cwksp_alloc_aligned); -} - -/** - * Aligned on sizeof(unsigned). These buffers have the special property that - * their values remain constrained, allowing us to re-use them without - * memset()-ing them. - */ -void* ZSTD_cwksp_reserve_table(ZSTD_cwksp* ws, size_t bytes) { - const ZSTD_cwksp_alloc_phase_e phase = ZSTD_cwksp_alloc_aligned; - void* alloc = ws->tableEnd; - void* end = (BYTE *)alloc + bytes; - void* top = ws->allocStart; - DEBUGLOG(4, "cwksp: reserving table %zd bytes, %zd bytes remaining", - bytes, ZSTD_cwksp_available_space(ws) - bytes); - assert((bytes & (sizeof(U32)-1)) == 0); - ZSTD_cwksp_internal_advance_phase(ws, phase); - assert(end <= top); - if (end > top) { - DEBUGLOG(4, "cwksp: object alloc failed!"); - ws->allocFailed = 1; - return NULL; - } - ws->tableEnd = end; - return alloc; -} - -/** - * Aligned on sizeof(void*). - */ -void* ZSTD_cwksp_reserve_object(ZSTD_cwksp* ws, size_t bytes) { - size_t roundedBytes = ZSTD_cwksp_align(bytes, sizeof(void*)); - void* start = ws->objectEnd; - void* end = (BYTE*)start + roundedBytes; - DEBUGLOG(4, - "cwksp: reserving object %zd bytes (rounded to %zd), %zd bytes remaining", - bytes, roundedBytes, ZSTD_cwksp_available_space(ws) - roundedBytes); - assert(((size_t)start & (sizeof(void*)-1)) == 0); - assert((bytes & (sizeof(void*)-1)) == 0); - /* we must be in the first phase, no advance is possible */ - if (ws->phase != ZSTD_cwksp_alloc_objects || end > ws->workspaceEnd) { - DEBUGLOG(4, "cwksp: object alloc failed!"); - ws->allocFailed = 1; - return NULL; - } - ws->objectEnd = end; - ws->tableEnd = end; - return start; -} - -/** - * Invalidates table allocations. - * All other allocations remain valid. - */ -void ZSTD_cwksp_clear_tables(ZSTD_cwksp* ws) { - DEBUGLOG(4, "cwksp: clearing tables!"); - ws->tableEnd = ws->objectEnd; -} - -/** - * Invalidates all buffer, aligned, and table allocations. - * Object allocations remain valid. - */ -void ZSTD_cwksp_clear(ZSTD_cwksp* ws) { - DEBUGLOG(4, "cwksp: clearing!"); - ws->tableEnd = ws->objectEnd; - ws->allocStart = ws->workspaceEnd; - ws->allocFailed = 0; - if (ws->phase > ZSTD_cwksp_alloc_buffers) { - ws->phase = ZSTD_cwksp_alloc_buffers; - } -} - -void ZSTD_cwksp_init(ZSTD_cwksp* ws, void* start, size_t size) { - DEBUGLOG(4, "cwksp: init'ing workspace with %zd bytes", size); - assert(((size_t)start & (sizeof(void*)-1)) == 0); /* ensure correct alignment */ - ws->workspace = start; - ws->workspaceEnd = (BYTE*)start + size; - ws->objectEnd = ws->workspace; - ws->phase = ZSTD_cwksp_alloc_objects; - ZSTD_cwksp_clear(ws); - ws->workspaceOversizedDuration = 0; -} - -size_t ZSTD_cwksp_create(ZSTD_cwksp* ws, size_t size, ZSTD_customMem customMem) { - void* workspace = ZSTD_malloc(size, customMem); - DEBUGLOG(4, "cwksp: creating new workspace with %zd bytes", size); - RETURN_ERROR_IF(workspace == NULL, memory_allocation); - ZSTD_cwksp_init(ws, workspace, size); - return 0; -} - -void ZSTD_cwksp_free(ZSTD_cwksp* ws, ZSTD_customMem customMem) { - DEBUGLOG(4, "cwksp: freeing workspace"); - ZSTD_free(ws->workspace, customMem); - ws->workspace = NULL; - ws->workspaceEnd = NULL; - ZSTD_cwksp_clear(ws); -} - -void ZSTD_cwksp_move(ZSTD_cwksp* dst, ZSTD_cwksp* src) { - *dst = *src; - memset(src, 0, sizeof(ZSTD_cwksp)); -} - -size_t ZSTD_cwksp_sizeof(const ZSTD_cwksp* ws) { - return (BYTE*)ws->workspaceEnd - (BYTE*)ws->workspace; -} - -int ZSTD_cwksp_reserve_failed(const ZSTD_cwksp* ws) { - return ws->allocFailed; -} - -size_t ZSTD_cwksp_available_space(ZSTD_cwksp* ws) { - return (size_t)((BYTE*)ws->allocStart - (BYTE*)ws->tableEnd); -} - -void ZSTD_cwksp_bump_oversized_duration( - ZSTD_cwksp* ws, size_t additionalNeededSpace) { - if (ZSTD_cwksp_check_too_large(ws, additionalNeededSpace)) { - ws->workspaceOversizedDuration++; - } else { - ws->workspaceOversizedDuration = 0; - } -} - -int ZSTD_cwksp_check_available(ZSTD_cwksp* ws, size_t additionalNeededSpace) { - return ZSTD_cwksp_available_space(ws) >= additionalNeededSpace; -} - -int ZSTD_cwksp_check_too_large(ZSTD_cwksp* ws, size_t additionalNeededSpace) { - return ZSTD_cwksp_check_available( - ws, additionalNeededSpace * ZSTD_WORKSPACETOOLARGE_FACTOR); -} - -int ZSTD_cwksp_check_wasteful(ZSTD_cwksp* ws, size_t additionalNeededSpace) { - return ZSTD_cwksp_check_too_large(ws, additionalNeededSpace) - && ws->workspaceOversizedDuration > ZSTD_WORKSPACETOOLARGE_MAXDURATION; -} +/* all functions have been inlined... */ #if defined (__cplusplus) } diff --git a/lib/compress/zstd_cwksp.h b/lib/compress/zstd_cwksp.h index 80d174f0..cd44d8ad 100644 --- a/lib/compress/zstd_cwksp.h +++ b/lib/compress/zstd_cwksp.h @@ -147,76 +147,218 @@ typedef struct { /** * Align must be a power of 2. */ -size_t ZSTD_cwksp_align(size_t size, size_t const align); +MEM_STATIC size_t ZSTD_cwksp_align(size_t size, size_t const align) { + size_t const mask = align - 1; + assert((align & mask) == 0); + return (size + mask) & ~mask; +} + +MEM_STATIC void ZSTD_cwksp_internal_advance_phase( + ZSTD_cwksp* ws, ZSTD_cwksp_alloc_phase_e phase) { + assert(phase >= ws->phase); + if (phase > ws->phase) { + if (ws->phase < ZSTD_cwksp_alloc_buffers && + phase >= ZSTD_cwksp_alloc_buffers) { + } + if (ws->phase < ZSTD_cwksp_alloc_aligned && + phase >= ZSTD_cwksp_alloc_aligned) { + /* If unaligned allocations down from a too-large top have left us + * unaligned, we need to realign our alloc ptr. Technically, this + * can consume space that is unaccounted for in the neededSpace + * calculation. However, I believe this can only happen when the + * workspace is too large, and specifically when it is too large + * by a larger margin than the space that will be consumed. */ + /* TODO: cleaner, compiler warning friendly way to do this??? */ + ws->allocStart = (BYTE*)ws->allocStart - ((size_t)ws->allocStart & (sizeof(U32)-1)); + } + ws->phase = phase; + } +} /** - * Unaligned. + * Internal function. Do not use directly. */ -BYTE* ZSTD_cwksp_reserve_buffer(ZSTD_cwksp* ws, size_t bytes); +MEM_STATIC void* ZSTD_cwksp_reserve_internal( + ZSTD_cwksp* ws, size_t bytes, ZSTD_cwksp_alloc_phase_e phase) { + void* alloc; + void* bottom = ws->tableEnd; + ZSTD_cwksp_internal_advance_phase(ws, phase); + alloc = (BYTE *)ws->allocStart - bytes; + DEBUGLOG(4, "cwksp: reserving %zd bytes, %zd bytes remaining", + bytes, ZSTD_cwksp_available_space(ws) - bytes); + assert(alloc >= bottom); + if (alloc < bottom) { + ws->allocFailed = 1; + return NULL; + } + ws->allocStart = alloc; + return alloc; +} /** - * Aligned on sizeof(unsigned). + * Reserves and returns unaligned memory. */ -void* ZSTD_cwksp_reserve_aligned(ZSTD_cwksp* ws, size_t bytes); +MEM_STATIC BYTE* ZSTD_cwksp_reserve_buffer(ZSTD_cwksp* ws, size_t bytes) { + return (BYTE*)ZSTD_cwksp_reserve_internal(ws, bytes, ZSTD_cwksp_alloc_buffers); +} + +/** + * Reserves and returns memory sized on and aligned on sizeof(unsigned). + */ +MEM_STATIC void* ZSTD_cwksp_reserve_aligned(ZSTD_cwksp* ws, size_t bytes) { + assert((bytes & (sizeof(U32)-1)) == 0); + return ZSTD_cwksp_reserve_internal(ws, ZSTD_cwksp_align(bytes, sizeof(U32)), ZSTD_cwksp_alloc_aligned); +} /** * Aligned on sizeof(unsigned). These buffers have the special property that * their values remain constrained, allowing us to re-use them without * memset()-ing them. */ -void* ZSTD_cwksp_reserve_table(ZSTD_cwksp* ws, size_t bytes); +MEM_STATIC void* ZSTD_cwksp_reserve_table(ZSTD_cwksp* ws, size_t bytes) { + const ZSTD_cwksp_alloc_phase_e phase = ZSTD_cwksp_alloc_aligned; + void* alloc = ws->tableEnd; + void* end = (BYTE *)alloc + bytes; + void* top = ws->allocStart; + DEBUGLOG(4, "cwksp: reserving table %zd bytes, %zd bytes remaining", + bytes, ZSTD_cwksp_available_space(ws) - bytes); + assert((bytes & (sizeof(U32)-1)) == 0); + ZSTD_cwksp_internal_advance_phase(ws, phase); + assert(end <= top); + if (end > top) { + DEBUGLOG(4, "cwksp: object alloc failed!"); + ws->allocFailed = 1; + return NULL; + } + ws->tableEnd = end; + return alloc; +} /** * Aligned on sizeof(void*). */ -void* ZSTD_cwksp_reserve_object(ZSTD_cwksp* ws, size_t bytes); +MEM_STATIC void* ZSTD_cwksp_reserve_object(ZSTD_cwksp* ws, size_t bytes) { + size_t roundedBytes = ZSTD_cwksp_align(bytes, sizeof(void*)); + void* start = ws->objectEnd; + void* end = (BYTE*)start + roundedBytes; + DEBUGLOG(4, + "cwksp: reserving object %zd bytes (rounded to %zd), %zd bytes remaining", + bytes, roundedBytes, ZSTD_cwksp_available_space(ws) - roundedBytes); + assert(((size_t)start & (sizeof(void*)-1)) == 0); + assert((bytes & (sizeof(void*)-1)) == 0); + /* we must be in the first phase, no advance is possible */ + if (ws->phase != ZSTD_cwksp_alloc_objects || end > ws->workspaceEnd) { + DEBUGLOG(4, "cwksp: object alloc failed!"); + ws->allocFailed = 1; + return NULL; + } + ws->objectEnd = end; + ws->tableEnd = end; + return start; +} /** * Invalidates table allocations. * All other allocations remain valid. */ -void ZSTD_cwksp_clear_tables(ZSTD_cwksp* ws); +MEM_STATIC void ZSTD_cwksp_clear_tables(ZSTD_cwksp* ws) { + DEBUGLOG(4, "cwksp: clearing tables!"); + ws->tableEnd = ws->objectEnd; +} /** * Invalidates all buffer, aligned, and table allocations. * Object allocations remain valid. */ -void ZSTD_cwksp_clear(ZSTD_cwksp* ws); +MEM_STATIC void ZSTD_cwksp_clear(ZSTD_cwksp* ws) { + DEBUGLOG(4, "cwksp: clearing!"); + ws->tableEnd = ws->objectEnd; + ws->allocStart = ws->workspaceEnd; + ws->allocFailed = 0; + if (ws->phase > ZSTD_cwksp_alloc_buffers) { + ws->phase = ZSTD_cwksp_alloc_buffers; + } +} /** * The provided workspace takes ownership of the buffer [start, start+size). * Any existing values in the workspace are ignored (the previously managed * buffer, if present, must be separately freed). */ -void ZSTD_cwksp_init(ZSTD_cwksp* ws, void* start, size_t size); +MEM_STATIC void ZSTD_cwksp_init(ZSTD_cwksp* ws, void* start, size_t size) { + DEBUGLOG(4, "cwksp: init'ing workspace with %zd bytes", size); + assert(((size_t)start & (sizeof(void*)-1)) == 0); /* ensure correct alignment */ + ws->workspace = start; + ws->workspaceEnd = (BYTE*)start + size; + ws->objectEnd = ws->workspace; + ws->phase = ZSTD_cwksp_alloc_objects; + ZSTD_cwksp_clear(ws); + ws->workspaceOversizedDuration = 0; +} -size_t ZSTD_cwksp_create(ZSTD_cwksp* ws, size_t size, ZSTD_customMem customMem); +MEM_STATIC size_t ZSTD_cwksp_create(ZSTD_cwksp* ws, size_t size, ZSTD_customMem customMem) { + void* workspace = ZSTD_malloc(size, customMem); + DEBUGLOG(4, "cwksp: creating new workspace with %zd bytes", size); + RETURN_ERROR_IF(workspace == NULL, memory_allocation); + ZSTD_cwksp_init(ws, workspace, size); + return 0; +} -void ZSTD_cwksp_free(ZSTD_cwksp* ws, ZSTD_customMem customMem); +MEM_STATIC void ZSTD_cwksp_free(ZSTD_cwksp* ws, ZSTD_customMem customMem) { + DEBUGLOG(4, "cwksp: freeing workspace"); + ZSTD_free(ws->workspace, customMem); + ws->workspace = NULL; + ws->workspaceEnd = NULL; + ZSTD_cwksp_clear(ws); +} /** * Moves the management of a workspace from one cwksp to another. The src cwksp * is left in an invalid state (src must be re-init()'ed before its used again). */ -void ZSTD_cwksp_move(ZSTD_cwksp* dst, ZSTD_cwksp* src); +MEM_STATIC void ZSTD_cwksp_move(ZSTD_cwksp* dst, ZSTD_cwksp* src) { + *dst = *src; + memset(src, 0, sizeof(ZSTD_cwksp)); +} -size_t ZSTD_cwksp_sizeof(const ZSTD_cwksp* ws); +MEM_STATIC size_t ZSTD_cwksp_sizeof(const ZSTD_cwksp* ws) { + return (BYTE*)ws->workspaceEnd - (BYTE*)ws->workspace; +} -int ZSTD_cwksp_reserve_failed(const ZSTD_cwksp* ws); +MEM_STATIC int ZSTD_cwksp_reserve_failed(const ZSTD_cwksp* ws) { + return ws->allocFailed; +} /*-************************************* * Functions Checking Free Space ***************************************/ -size_t ZSTD_cwksp_available_space(ZSTD_cwksp* ws); +MEM_STATIC size_t ZSTD_cwksp_available_space(ZSTD_cwksp* ws) { + return (size_t)((BYTE*)ws->allocStart - (BYTE*)ws->tableEnd); +} -int ZSTD_cwksp_check_available(ZSTD_cwksp* ws, size_t additionalNeededSpace); +MEM_STATIC int ZSTD_cwksp_check_available(ZSTD_cwksp* ws, size_t additionalNeededSpace) { + return ZSTD_cwksp_available_space(ws) >= additionalNeededSpace; +} -int ZSTD_cwksp_check_too_large(ZSTD_cwksp* ws, size_t additionalNeededSpace); +MEM_STATIC int ZSTD_cwksp_check_too_large(ZSTD_cwksp* ws, size_t additionalNeededSpace) { + return ZSTD_cwksp_check_available( + ws, additionalNeededSpace * ZSTD_WORKSPACETOOLARGE_FACTOR); +} -int ZSTD_cwksp_check_wasteful(ZSTD_cwksp* ws, size_t additionalNeededSpace); +MEM_STATIC int ZSTD_cwksp_check_wasteful(ZSTD_cwksp* ws, size_t additionalNeededSpace) { + return ZSTD_cwksp_check_too_large(ws, additionalNeededSpace) + && ws->workspaceOversizedDuration > ZSTD_WORKSPACETOOLARGE_MAXDURATION; +} -void ZSTD_cwksp_bump_oversized_duration(ZSTD_cwksp* ws, size_t additionalNeededSpace); +MEM_STATIC void ZSTD_cwksp_bump_oversized_duration( + ZSTD_cwksp* ws, size_t additionalNeededSpace) { + if (ZSTD_cwksp_check_too_large(ws, additionalNeededSpace)) { + ws->workspaceOversizedDuration++; + } else { + ws->workspaceOversizedDuration = 0; + } +} #if defined (__cplusplus) } From 81208fd7c2bdf7ef266ce95a7ba20e5475e69aa5 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Mon, 9 Sep 2019 19:10:09 -0400 Subject: [PATCH 32/33] Forward Declare `ZSTD_cwksp_available_space` to Fix Build --- lib/compress/zstd_cwksp.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/compress/zstd_cwksp.h b/lib/compress/zstd_cwksp.h index cd44d8ad..76ace25e 100644 --- a/lib/compress/zstd_cwksp.h +++ b/lib/compress/zstd_cwksp.h @@ -144,6 +144,8 @@ typedef struct { * Functions ***************************************/ +MEM_STATIC size_t ZSTD_cwksp_available_space(ZSTD_cwksp* ws); + /** * Align must be a power of 2. */ From a9d373f0938a53f814a812032357f7b9756ed8f0 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Tue, 10 Sep 2019 16:03:13 -0400 Subject: [PATCH 33/33] Remove Empty lib/compress/zstd_cwksp.c --- build/VS2008/fullbench/fullbench.vcproj | 4 ---- build/VS2008/fuzzer/fuzzer.vcproj | 4 ---- build/VS2008/zstd/zstd.vcproj | 4 ---- build/VS2008/zstdlib/zstdlib.vcproj | 4 ---- build/VS2010/fullbench/fullbench.vcxproj | 1 - build/VS2010/fuzzer/fuzzer.vcxproj | 1 - build/VS2010/libzstd-dll/libzstd-dll.vcxproj | 1 - build/VS2010/libzstd/libzstd.vcxproj | 1 - build/VS2010/zstd/zstd.vcxproj | 1 - build/meson/lib/meson.build | 1 - lib/compress/zstd_cwksp.c | 21 -------------------- 11 files changed, 43 deletions(-) delete mode 100644 lib/compress/zstd_cwksp.c diff --git a/build/VS2008/fullbench/fullbench.vcproj b/build/VS2008/fullbench/fullbench.vcproj index ffb0a015..66ac8223 100644 --- a/build/VS2008/fullbench/fullbench.vcproj +++ b/build/VS2008/fullbench/fullbench.vcproj @@ -372,10 +372,6 @@ RelativePath="..\..\..\lib\compress\zstd_compress_sequences.c" > - - diff --git a/build/VS2008/fuzzer/fuzzer.vcproj b/build/VS2008/fuzzer/fuzzer.vcproj index c3875ec7..a9008a67 100644 --- a/build/VS2008/fuzzer/fuzzer.vcproj +++ b/build/VS2008/fuzzer/fuzzer.vcproj @@ -420,10 +420,6 @@ RelativePath="..\..\..\lib\compress\zstd_compress_sequences.c" > - - diff --git a/build/VS2008/zstd/zstd.vcproj b/build/VS2008/zstd/zstd.vcproj index 5b57dde5..4d75a9d3 100644 --- a/build/VS2008/zstd/zstd.vcproj +++ b/build/VS2008/zstd/zstd.vcproj @@ -432,10 +432,6 @@ RelativePath="..\..\..\lib\compress\zstd_compress_sequences.c" > - - diff --git a/build/VS2008/zstdlib/zstdlib.vcproj b/build/VS2008/zstdlib/zstdlib.vcproj index e864b5fb..100a501a 100644 --- a/build/VS2008/zstdlib/zstdlib.vcproj +++ b/build/VS2008/zstdlib/zstdlib.vcproj @@ -404,10 +404,6 @@ RelativePath="..\..\..\lib\compress\zstd_compress_sequences.c" > - - diff --git a/build/VS2010/fullbench/fullbench.vcxproj b/build/VS2010/fullbench/fullbench.vcxproj index 65e7ff78..4597239b 100644 --- a/build/VS2010/fullbench/fullbench.vcxproj +++ b/build/VS2010/fullbench/fullbench.vcxproj @@ -169,7 +169,6 @@ - diff --git a/build/VS2010/fuzzer/fuzzer.vcxproj b/build/VS2010/fuzzer/fuzzer.vcxproj index e9b88a8a..a6f136ee 100644 --- a/build/VS2010/fuzzer/fuzzer.vcxproj +++ b/build/VS2010/fuzzer/fuzzer.vcxproj @@ -169,7 +169,6 @@ - diff --git a/build/VS2010/libzstd-dll/libzstd-dll.vcxproj b/build/VS2010/libzstd-dll/libzstd-dll.vcxproj index 4140879d..50c4e817 100644 --- a/build/VS2010/libzstd-dll/libzstd-dll.vcxproj +++ b/build/VS2010/libzstd-dll/libzstd-dll.vcxproj @@ -33,7 +33,6 @@ - diff --git a/build/VS2010/libzstd/libzstd.vcxproj b/build/VS2010/libzstd/libzstd.vcxproj index 02690cef..c4e82877 100644 --- a/build/VS2010/libzstd/libzstd.vcxproj +++ b/build/VS2010/libzstd/libzstd.vcxproj @@ -33,7 +33,6 @@ - diff --git a/build/VS2010/zstd/zstd.vcxproj b/build/VS2010/zstd/zstd.vcxproj index 2a21ba4c..1058d229 100644 --- a/build/VS2010/zstd/zstd.vcxproj +++ b/build/VS2010/zstd/zstd.vcxproj @@ -34,7 +34,6 @@ - diff --git a/build/meson/lib/meson.build b/build/meson/lib/meson.build index 48f1e7cc..ef669327 100644 --- a/build/meson/lib/meson.build +++ b/build/meson/lib/meson.build @@ -30,7 +30,6 @@ libzstd_sources = [join_paths(zstd_rootdir, 'lib/common/entropy_common.c'), join_paths(zstd_rootdir, 'lib/compress/zstd_compress.c'), join_paths(zstd_rootdir, 'lib/compress/zstd_compress_literals.c'), join_paths(zstd_rootdir, 'lib/compress/zstd_compress_sequences.c'), - join_paths(zstd_rootdir, 'lib/compress/zstd_cwksp.c'), join_paths(zstd_rootdir, 'lib/compress/zstdmt_compress.c'), join_paths(zstd_rootdir, 'lib/compress/zstd_fast.c'), join_paths(zstd_rootdir, 'lib/compress/zstd_double_fast.c'), diff --git a/lib/compress/zstd_cwksp.c b/lib/compress/zstd_cwksp.c deleted file mode 100644 index a757910d..00000000 --- a/lib/compress/zstd_cwksp.c +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - * You may select, at your option, one of the above-listed licenses. - */ - -#include "zstd_cwksp.h" - -#if defined (__cplusplus) -extern "C" { -#endif - -/* all functions have been inlined... */ - -#if defined (__cplusplus) -} -#endif