From c0ce4f1211d72b6ea2247be3a1e82bccf71ea301 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sat, 30 Jul 2016 00:55:13 +0200 Subject: [PATCH] slightly improved compression speed --- lib/common/zstd_internal.h | 2 +- lib/compress/zstd_compress.c | 27 +++++++++++++-------------- lib/dictBuilder/zdict.c | 2 +- lib/zstd.h | 15 +++++++++------ 4 files changed, 24 insertions(+), 22 deletions(-) diff --git a/lib/common/zstd_internal.h b/lib/common/zstd_internal.h index 438045b7..d0391871 100644 --- a/lib/common/zstd_internal.h +++ b/lib/common/zstd_internal.h @@ -192,13 +192,13 @@ typedef struct seqDef_s { typedef struct { + seqDef* sequencesStart; seqDef* sequences; BYTE* litStart; BYTE* lit; BYTE* llCode; BYTE* mlCode; BYTE* ofCode; - U32 nbSeq; U32 longLengthID; /* 0 == no longLength; 1 == Lit.longLength; 2 == Match.longLength; */ U32 longLengthPos; /* opt */ diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 49b8b0af..07f23b5a 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -82,7 +82,7 @@ size_t ZSTD_compressBound(size_t srcSize) { return FSE_compressBound(srcSize) + static void ZSTD_resetSeqStore(seqStore_t* ssPtr) { ssPtr->lit = ssPtr->litStart; - ssPtr->nbSeq = 0; + ssPtr->sequences = ssPtr->sequencesStart; ssPtr->longLengthID = 0; } @@ -312,8 +312,8 @@ static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc, ptr = zc->seqStore.priceTable + ZSTD_OPT_NUM+1; zc->seqStore.litLengthSum = 0; } - zc->seqStore.sequences = (seqDef*)ptr; - ptr = zc->seqStore.sequences + maxNbSeq; + zc->seqStore.sequencesStart = (seqDef*)ptr; + ptr = zc->seqStore.sequencesStart + maxNbSeq; zc->seqStore.llCode = (BYTE*) ptr; zc->seqStore.mlCode = zc->seqStore.llCode + maxNbSeq; zc->seqStore.ofCode = zc->seqStore.mlCode + maxNbSeq; @@ -545,11 +545,11 @@ void ZSTD_seqToCodes(const seqStore_t* seqStorePtr) { BYTE const LL_deltaCode = 19; BYTE const ML_deltaCode = 36; - const seqDef* const sequences = seqStorePtr->sequences; + const seqDef* const sequences = seqStorePtr->sequencesStart; BYTE* const llCodeTable = seqStorePtr->llCode; BYTE* const ofCodeTable = seqStorePtr->ofCode; BYTE* const mlCodeTable = seqStorePtr->mlCode; - U32 const nbSeq = seqStorePtr->nbSeq; + U32 const nbSeq = (U32)(seqStorePtr->sequences - seqStorePtr->sequencesStart); U32 u; for (u=0; uoffcodeCTable; FSE_CTable* CTable_MatchLength = zc->matchlengthCTable; U32 LLtype, Offtype, MLtype; /* compressed, raw or rle */ - const seqDef* const sequences = seqStorePtr->sequences; + const seqDef* const sequences = seqStorePtr->sequencesStart; const BYTE* const ofCodeTable = seqStorePtr->ofCode; const BYTE* const llCodeTable = seqStorePtr->llCode; const BYTE* const mlCodeTable = seqStorePtr->mlCode; BYTE* const ostart = (BYTE*)dst; BYTE* const oend = ostart + dstCapacity; BYTE* op = ostart; - size_t const nbSeq = seqStorePtr->nbSeq; + size_t const nbSeq = seqStorePtr->sequences - seqStorePtr->sequencesStart; BYTE* seqHead; /* Compress literals */ @@ -765,7 +765,6 @@ MEM_STATIC void ZSTD_storeSeq(seqStore_t* seqStorePtr, size_t litLength, const v printf("Cpos %6u :%5u literals & match %3u bytes at distance %6u \n", pos, (U32)litLength, (U32)matchCode+MINMATCH, (U32)offsetCode); #endif - U32 const nbSeq = seqStorePtr->nbSeq; ZSTD_statsUpdatePrices(&seqStorePtr->stats, litLength, (const BYTE*)literals, offsetCode, matchCode); /* debug only */ /* copy Literals */ @@ -773,17 +772,17 @@ MEM_STATIC void ZSTD_storeSeq(seqStore_t* seqStorePtr, size_t litLength, const v seqStorePtr->lit += litLength; /* literal Length */ - if (litLength>0xFFFF) { seqStorePtr->longLengthID = 1; seqStorePtr->longLengthPos = nbSeq; } - seqStorePtr->sequences[nbSeq].litLength = (U16)litLength; + if (litLength>0xFFFF) { seqStorePtr->longLengthID = 1; seqStorePtr->longLengthPos = (U32)(seqStorePtr->sequences - seqStorePtr->sequencesStart); } + seqStorePtr->sequences[0].litLength = (U16)litLength; /* match offset */ - seqStorePtr->sequences[nbSeq].offset = offsetCode + 1; + seqStorePtr->sequences[0].offset = offsetCode + 1; /* match Length */ - if (matchCode>0xFFFF) { seqStorePtr->longLengthID = 2; seqStorePtr->longLengthPos = nbSeq; } - seqStorePtr->sequences[nbSeq].matchLength = (U16)matchCode; + if (matchCode>0xFFFF) { seqStorePtr->longLengthID = 2; seqStorePtr->longLengthPos = (U32)(seqStorePtr->sequences - seqStorePtr->sequencesStart); } + seqStorePtr->sequences[0].matchLength = (U16)matchCode; - seqStorePtr->nbSeq++; + seqStorePtr->sequences++; } diff --git a/lib/dictBuilder/zdict.c b/lib/dictBuilder/zdict.c index d16e1efc..84272d19 100644 --- a/lib/dictBuilder/zdict.c +++ b/lib/dictBuilder/zdict.c @@ -594,7 +594,7 @@ static void ZDICT_countEStats(EStats_ress_t esr, ZSTD_parameters params, } /* seqStats */ - { U32 const nbSeq = seqStorePtr->nbSeq; + { U32 const nbSeq = (U32)(seqStorePtr->sequences - seqStorePtr->sequencesStart); ZSTD_seqToCodes(seqStorePtr); { const BYTE* codePtr = seqStorePtr->ofCode; diff --git a/lib/zstd.h b/lib/zstd.h index 1dded9b4..6b9ed463 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -80,16 +80,19 @@ ZSTDLIB_API size_t ZSTD_compress( void* dst, size_t dstCapacity, /*! ZSTD_getDecompressedSize() : * @return : decompressed size if known, 0 otherwise. - note 1 : decompressed size could be wrong or intentionally modified ! - Always ensure result fits within application's authorized limits ! - Each application can set its own limit, depending on local limitations. - For extended interoperability, it is recommended to support at least 8 MB. - note 2 : when `0`, if precise failure cause is needed, use ZSTD_getFrameParams() to know more. */ +* note 1 : decompressed size could be wrong or intentionally modified ! +* Always ensure result fits within application's authorized limits ! +* Each application can set its own limit, depending on local restrictions. +* For extended interoperability, it is recommended to support at least 8 MB. +* note 2 : when `0`, if precise failure cause is needed, use ZSTD_getFrameParams() to know more. +* note 3 : when `0`, and if no external guarantee about maximum possible decompressed size, +* it's necessary to use "streaming mode" to decompress data. */ unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize); /*! ZSTD_decompress() : `compressedSize` : must be the _exact_ size of compressed input, otherwise decompression will fail. - `dstCapacity` must be equal or larger than originalSize. + `dstCapacity` must be equal or larger than originalSize (see ZSTD_getDecompressedSize() ). + If maximum possible content size is unknown, use streaming mode to decompress data. @return : the number of bytes decompressed into `dst` (<= `dstCapacity`), or an errorCode if it fails (which can be tested using ZSTD_isError()) */ ZSTDLIB_API size_t ZSTD_decompress( void* dst, size_t dstCapacity,