facebook/zstd
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

added ZSTD_createDDict_byReference()

Browse Source
This commit is contained in:
Yann Collet 2016-12-21 16:44:35 +01:00
parent 8333106b8a
commit 4e5eea61a8
4 changed files with 35 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
NEWS
View File

@ -1,6 +1,7 @@
v1.1.3 v1.1.3
cli : new : commands for advanced parameters, by Przemyslaw Skibinski cli : new : commands for advanced parameters, by Przemyslaw Skibinski
API : fix : all symbols properly exposed in libzstd, by Nick Terrell API : fix : all symbols properly exposed in libzstd, by Nick Terrell
API : new : ZSTD_create?Dict_byReference(), requested by Bartosz Taudul
v1.1.2 v1.1.2
API : streaming : decompression : changed : automatic implicit reset when chain-decoding new frames without init API : streaming : decompression : changed : automatic implicit reset when chain-decoding new frames without init

7
lib/compress/zstd_compress.c
View File

@ -2736,7 +2736,6 @@ struct ZSTD_CDict_s {
void* dictBuffer; void* dictBuffer;
const void* dictContent; const void* dictContent;
size_t dictContentSize; size_t dictContentSize;
unsigned byReference;
ZSTD_CCtx* refContext; ZSTD_CCtx* refContext;
}; /* typedef'd tp ZSTD_CDict within "zstd.h" */ }; /* typedef'd tp ZSTD_CDict within "zstd.h" */
@ -2764,14 +2763,12 @@ ZSTD_CDict* ZSTD_createCDict_advanced(const void* dictBuffer, size_t dictSize, u
if ((byReference) || (!dictBuffer) || (!dictSize)) { if ((byReference) || (!dictBuffer) || (!dictSize)) {
cdict->dictBuffer = NULL; cdict->dictBuffer = NULL;
cdict->dictContent = dictBuffer; cdict->dictContent = dictBuffer;
cdict->byReference = 1;
} else { } else {
void* const internalBuffer = ZSTD_malloc(dictSize, customMem); void* const internalBuffer = ZSTD_malloc(dictSize, customMem);
if (!internalBuffer) return NULL; if (!internalBuffer) { ZSTD_free(cctx, customMem); ZSTD_free(cdict, customMem); return NULL; }
memcpy(internalBuffer, dictBuffer, dictSize); memcpy(internalBuffer, dictBuffer, dictSize);
cdict->dictBuffer = internalBuffer; cdict->dictBuffer = internalBuffer;
cdict->dictContent = internalBuffer; cdict->dictContent = internalBuffer;
cdict->byReference = 0;
} }
{ size_t const errorCode = ZSTD_compressBegin_advanced(cctx, cdict->dictContent, dictSize, params, 0); { size_t const errorCode = ZSTD_compressBegin_advanced(cctx, cdict->dictContent, dictSize, params, 0);
@ -2809,7 +2806,7 @@ size_t ZSTD_freeCDict(ZSTD_CDict* cdict)
if (cdict==NULL) return 0; /* support free on NULL */ if (cdict==NULL) return 0; /* support free on NULL */
{ ZSTD_customMem const cMem = cdict->refContext->customMem; { ZSTD_customMem const cMem = cdict->refContext->customMem;
ZSTD_freeCCtx(cdict->refContext); ZSTD_freeCCtx(cdict->refContext);
if (!cdict->byReference) ZSTD_free(cdict->dictBuffer, cMem); ZSTD_free(cdict->dictBuffer, cMem);
ZSTD_free(cdict, cMem); ZSTD_free(cdict, cMem);
return 0; return 0;
} }

35
lib/decompress/zstd_decompress.c
View File

@ -1713,39 +1713,44 @@ size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t
/* ====== ZSTD_DDict ====== */ /* ====== ZSTD_DDict ====== */
struct ZSTD_DDict_s { struct ZSTD_DDict_s {
void* dict; void* dictBuffer;
const void* dictContent;
size_t dictSize; size_t dictSize;
ZSTD_DCtx* refContext; ZSTD_DCtx* refContext;
}; /* typedef'd to ZSTD_DDict within "zstd.h" */ }; /* typedef'd to ZSTD_DDict within "zstd.h" */
ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize, ZSTD_customMem customMem) ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize, unsigned byReference, ZSTD_customMem customMem)
{ {
if (!customMem.customAlloc && !customMem.customFree) customMem = defaultCustomMem; if (!customMem.customAlloc && !customMem.customFree) customMem = defaultCustomMem;
if (!customMem.customAlloc || !customMem.customFree) return NULL; if (!customMem.customAlloc || !customMem.customFree) return NULL;
{ ZSTD_DDict* const ddict = (ZSTD_DDict*) ZSTD_malloc(sizeof(ZSTD_DDict), customMem); { ZSTD_DDict* const ddict = (ZSTD_DDict*) ZSTD_malloc(sizeof(ZSTD_DDict), customMem);
void* const dictContent = ZSTD_malloc(dictSize, customMem);
ZSTD_DCtx* const dctx = ZSTD_createDCtx_advanced(customMem); ZSTD_DCtx* const dctx = ZSTD_createDCtx_advanced(customMem);
if (!dictContent || !ddict || !dctx) { if (!ddict || !dctx) {
ZSTD_free(dictContent, customMem);
ZSTD_free(ddict, customMem); ZSTD_free(ddict, customMem);
ZSTD_free(dctx, customMem); ZSTD_free(dctx, customMem);
return NULL; return NULL;
} }
if (dictSize) { if ((byReference) || (!dict) || (!dictSize)) {
memcpy(dictContent, dict, dictSize); ddict->dictBuffer = NULL;
ddict->dictContent = dict;
} else {
void* const internalBuffer = ZSTD_malloc(dictSize, customMem);
if (!internalBuffer) { ZSTD_free(dctx, customMem); ZSTD_free(ddict, customMem); return NULL; }
memcpy(internalBuffer, dict, dictSize);
ddict->dictBuffer = internalBuffer;
ddict->dictContent = internalBuffer;
} }
{ size_t const errorCode = ZSTD_decompressBegin_usingDict(dctx, dictContent, dictSize); { size_t const errorCode = ZSTD_decompressBegin_usingDict(dctx, ddict->dictContent, dictSize);
if (ZSTD_isError(errorCode)) { if (ZSTD_isError(errorCode)) {
ZSTD_free(dictContent, customMem); ZSTD_free(ddict->dictBuffer, customMem);
ZSTD_free(ddict, customMem); ZSTD_free(ddict, customMem);
ZSTD_free(dctx, customMem); ZSTD_free(dctx, customMem);
return NULL; return NULL;
} } } }
ddict->dict = dictContent;
ddict->dictSize = dictSize; ddict->dictSize = dictSize;
ddict->refContext = dctx; ddict->refContext = dctx;
return ddict; return ddict;
@ -1758,7 +1763,7 @@ ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize, ZSTD_cu
ZSTD_DDict* ZSTD_createDDict(const void* dict, size_t dictSize) ZSTD_DDict* ZSTD_createDDict(const void* dict, size_t dictSize)
{ {
ZSTD_customMem const allocator = { NULL, NULL, NULL }; ZSTD_customMem const allocator = { NULL, NULL, NULL };
return ZSTD_createDDict_advanced(dict, dictSize, allocator); return ZSTD_createDDict_advanced(dict, dictSize, 0, allocator);
} }
size_t ZSTD_freeDDict(ZSTD_DDict* ddict) size_t ZSTD_freeDDict(ZSTD_DDict* ddict)
@ -1766,7 +1771,7 @@ size_t ZSTD_freeDDict(ZSTD_DDict* ddict)
if (ddict==NULL) return 0; /* support free on NULL */ if (ddict==NULL) return 0; /* support free on NULL */
{ ZSTD_customMem const cMem = ddict->refContext->customMem; { ZSTD_customMem const cMem = ddict->refContext->customMem;
ZSTD_freeDCtx(ddict->refContext); ZSTD_freeDCtx(ddict->refContext);
ZSTD_free(ddict->dict, cMem); ZSTD_free(ddict->dictBuffer, cMem);
ZSTD_free(ddict, cMem); ZSTD_free(ddict, cMem);
return 0; return 0;
} }
@ -1796,7 +1801,7 @@ unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize)
unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict) unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict)
{ {
if (ddict==NULL) return 0; if (ddict==NULL) return 0;
return ZSTD_getDictID_fromDict(ddict->dict, ddict->dictSize); return ZSTD_getDictID_fromDict(ddict->dictContent, ddict->dictSize);
} }
/*! ZSTD_getDictID_fromFrame() : /*! ZSTD_getDictID_fromFrame() :
@ -1827,7 +1832,7 @@ size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx,
const ZSTD_DDict* ddict) const ZSTD_DDict* ddict)
{ {
#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1) #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1)
if (ZSTD_isLegacy(src, srcSize)) return ZSTD_decompressLegacy(dst, dstCapacity, src, srcSize, ddict->dict, ddict->dictSize); if (ZSTD_isLegacy(src, srcSize)) return ZSTD_decompressLegacy(dst, dstCapacity, src, srcSize, ddict->dictContent, ddict->dictSize);
#endif #endif
ZSTD_refDCtx(dctx, ddict->refContext); ZSTD_refDCtx(dctx, ddict->refContext);
ZSTD_checkContinuity(dctx, dst); ZSTD_checkContinuity(dctx, dst);
@ -2007,7 +2012,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1) #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1)
{ U32 const legacyVersion = ZSTD_isLegacy(istart, iend-istart); { U32 const legacyVersion = ZSTD_isLegacy(istart, iend-istart);
if (legacyVersion) { if (legacyVersion) {
const void* const dict = zds->ddict ? zds->ddict->dict : NULL; const void* const dict = zds->ddict ? zds->ddict->dictContent : NULL;
size_t const dictSize = zds->ddict ? zds->ddict->dictSize : 0; size_t const dictSize = zds->ddict ? zds->ddict->dictSize : 0;
CHECK_F(ZSTD_initLegacyStream(&zds->legacyContext, zds->previousLegacyVersion, legacyVersion, CHECK_F(ZSTD_initLegacyStream(&zds->legacyContext, zds->previousLegacyVersion, legacyVersion,
dict, dictSize)); dict, dictSize));

15
lib/zstd.h
View File

@ -194,8 +194,8 @@ typedef struct ZSTD_DDict_s ZSTD_DDict;
/*! ZSTD_createDDict() : /*! ZSTD_createDDict() :
* Create a digested dictionary, ready to start decompression operation without startup delay. * Create a digested dictionary, ready to start decompression operation without startup delay.
* `dict` can be released after creation. */ * dictBuffer can be released after DDict creation, as its content is copied inside DDict */
ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict(const void* dict, size_t dictSize); ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict(const void* dictBuffer, size_t dictSize);
/*! ZSTD_freeDDict() : /*! ZSTD_freeDDict() :
* Function frees memory allocated with ZSTD_createDDict() */ * Function frees memory allocated with ZSTD_createDDict() */
@ -328,7 +328,7 @@ ZSTDLIB_API size_t ZSTD_DStreamOutSize(void); /*!< recommended size for output
* ***************************************************************************************/ * ***************************************************************************************/
/* --- Constants ---*/ /* --- Constants ---*/
#define ZSTD_MAGICNUMBER 0xFD2FB528 /* v0.8 */ #define ZSTD_MAGICNUMBER 0xFD2FB528 /* >= v0.8.0 */
#define ZSTD_MAGIC_SKIPPABLE_START 0x184D2A50U #define ZSTD_MAGIC_SKIPPABLE_START 0x184D2A50U
#define ZSTD_WINDOWLOG_MAX_32 25 #define ZSTD_WINDOWLOG_MAX_32 25
@ -464,6 +464,15 @@ ZSTDLIB_API ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem);
* Gives the amount of memory used by a given ZSTD_DCtx */ * Gives the amount of memory used by a given ZSTD_DCtx */
ZSTDLIB_API size_t ZSTD_sizeof_DCtx(const ZSTD_DCtx* dctx); ZSTDLIB_API size_t ZSTD_sizeof_DCtx(const ZSTD_DCtx* dctx);
/*! ZSTD_createDDict_byReference() :
* Create a digested dictionary, ready to start decompression operation without startup delay.
* Dictionary content is simply referenced, and therefore stays in dictBuffer.
* It is important that dictBuffer outlives DDict, it must remain read accessible throughout the lifetime of DDict */
ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict_byReference(const void* dictBuffer, size_t dictSize);
ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize,
unsigned byReference, ZSTD_customMem customMem);
/*! ZSTD_sizeof_DDict() : /*! ZSTD_sizeof_DDict() :
* Gives the amount of memory used by a given ZSTD_DDict */ * Gives the amount of memory used by a given ZSTD_DDict */
ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict); ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);