facebook
/
zstd
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

added : ZSTD_createCDict_byReference()

dev
Yann Collet 2016-12-21 16:20:11 +01:00
parent 8c1a042134
commit 1f57c2ed32
5 changed files with 49 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
Makefile
View File

@ -30,6 +30,8 @@ all:
$(MAKE) -C $(ZSTDDIR) $@
$(MAKE) -C $(PRGDIR) $@ zstd32
$(MAKE) -C $(TESTDIR) $@ all32
$(MAKE) -C $(ZWRAPDIR) $@
CPPFLAGS=-I../lib LDFLAGS=-L../lib $(MAKE) -C examples/ $@
.PHONY: lib
lib:
@ -54,6 +56,7 @@ clean:
@$(MAKE) -C $(PRGDIR) $@ > $(VOID)
@$(MAKE) -C $(TESTDIR) $@ > $(VOID)
@$(MAKE) -C $(ZWRAPDIR) $@ > $(VOID)
@$(MAKE) -C examples/ $@ > $(VOID)
@$(RM) zstd$(EXT) tmp*
@echo Cleaning completed

48
lib/compress/zstd_compress.c
View File

@ -2733,8 +2733,10 @@ size_t ZSTD_compress(void* dst, size_t dstCapacity, const void* src, size_t srcS
/* ===== Dictionary API ===== */
struct ZSTD_CDict_s {
void* dictContent;
void* dictBuffer;
const void* dictContent;
size_t dictContentSize;
unsigned byReference;
ZSTD_CCtx* refContext;
}; /* typedef'd tp ZSTD_CDict within "zstd.h" */
@ -2744,36 +2746,44 @@ size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict)
return ZSTD_sizeof_CCtx(cdict->refContext) + cdict->dictContentSize;
}
ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize, ZSTD_parameters params, ZSTD_customMem customMem)
ZSTD_CDict* ZSTD_createCDict_advanced(const void* dictBuffer, size_t dictSize, unsigned byReference,
ZSTD_parameters params, ZSTD_customMem customMem)
{
if (!customMem.customAlloc && !customMem.customFree) customMem = defaultCustomMem;
if (!customMem.customAlloc || !customMem.customFree) return NULL;
{ ZSTD_CDict* const cdict = (ZSTD_CDict*) ZSTD_malloc(sizeof(ZSTD_CDict), customMem);
void* const dictContent = ZSTD_malloc(dictSize, customMem);
ZSTD_CCtx* const cctx = ZSTD_createCCtx_advanced(customMem);
if (!dictContent || !cdict || !cctx) {
ZSTD_free(dictContent, customMem);
if (!cdict || !cctx) {
ZSTD_free(cdict, customMem);
ZSTD_free(cctx, customMem);
return NULL;
}
if (dictSize) {
memcpy(dictContent, dict, dictSize);
if ((byReference) || (!dictBuffer) || (!dictSize)) {
cdict->dictBuffer = NULL;
cdict->dictContent = dictBuffer;
cdict->byReference = 1;
} else {
void* const internalBuffer = ZSTD_malloc(dictSize, customMem);
if (!internalBuffer) return NULL;
memcpy(internalBuffer, dictBuffer, dictSize);
cdict->dictBuffer = internalBuffer;
cdict->dictContent = internalBuffer;
cdict->byReference = 0;
}
{ size_t const errorCode = ZSTD_compressBegin_advanced(cctx, dictContent, dictSize, params, 0);
{ size_t const errorCode = ZSTD_compressBegin_advanced(cctx, cdict->dictContent, dictSize, params, 0);
if (ZSTD_isError(errorCode)) {
ZSTD_free(dictContent, customMem);
ZSTD_free(cdict, customMem);
ZSTD_free(cdict->dictBuffer, customMem);
ZSTD_free(cctx, customMem);
ZSTD_free(cdict, customMem);
return NULL;
} }
cdict->dictContent = dictContent;
cdict->dictContentSize = dictSize;
cdict->refContext = cctx;
cdict->dictContentSize = dictSize;
return cdict;
}
}
@ -2783,7 +2793,15 @@ ZSTD_CDict* ZSTD_createCDict(const void* dict, size_t dictSize, int compressionL
ZSTD_customMem const allocator = { NULL, NULL, NULL };
ZSTD_parameters params = ZSTD_getParams(compressionLevel, 0, dictSize);
params.fParams.contentSizeFlag = 1;
return ZSTD_createCDict_advanced(dict, dictSize, params, allocator);
return ZSTD_createCDict_advanced(dict, dictSize, 0, params, allocator);
}
ZSTD_CDict* ZSTD_createCDict_byReference(const void* dict, size_t dictSize, int compressionLevel)
{
ZSTD_customMem const allocator = { NULL, NULL, NULL };
ZSTD_parameters params = ZSTD_getParams(compressionLevel, 0, dictSize);
params.fParams.contentSizeFlag = 1;
return ZSTD_createCDict_advanced(dict, dictSize, 1, params, allocator);
}
size_t ZSTD_freeCDict(ZSTD_CDict* cdict)
@ -2791,7 +2809,7 @@ size_t ZSTD_freeCDict(ZSTD_CDict* cdict)
if (cdict==NULL) return 0; /* support free on NULL */
{ ZSTD_customMem const cMem = cdict->refContext->customMem;
ZSTD_freeCCtx(cdict->refContext);
ZSTD_free(cdict->dictContent, cMem);
if (!cdict->byReference) ZSTD_free(cdict->dictBuffer, cMem);
ZSTD_free(cdict, cMem);
return 0;
}
@ -2939,7 +2957,7 @@ size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs,
if (dict) {
ZSTD_freeCDict(zcs->cdictLocal);
zcs->cdictLocal = ZSTD_createCDict_advanced(dict, dictSize, params, zcs->customMem);
zcs->cdictLocal = ZSTD_createCDict_advanced(dict, dictSize, 0, params, zcs->customMem);
if (zcs->cdictLocal == NULL) return ERROR(memory_allocation);
zcs->cdict = zcs->cdictLocal;
} else zcs->cdict = NULL;

12
lib/zstd.h
View File

@ -173,8 +173,8 @@ typedef struct ZSTD_CDict_s ZSTD_CDict;
* When compressing multiple messages / blocks with the same dictionary, it's recommended to load it just once.
* ZSTD_createCDict() will create a digested dictionary, ready to start future compression operations without startup delay.
* ZSTD_CDict can be created once and used by multiple threads concurrently, as its usage is read-only.
* `dict` can be released after ZSTD_CDict creation. */
ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict(const void* dict, size_t dictSize, int compressionLevel);
* `dictBuffer` can be released after ZSTD_CDict creation, as its content is copied within CDict */
ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict(const void* dictBuffer, size_t dictSize, int compressionLevel);
/*! ZSTD_freeCDict() :
* Function frees memory allocated by ZSTD_createCDict(). */
@ -400,9 +400,15 @@ ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem);
* Gives the amount of memory used by a given ZSTD_CCtx */
ZSTDLIB_API size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx);
/*! ZSTD_createCDict_byReference() :
* Create a digested dictionary for compression
* Dictionary content is simply referenced, and therefore stays in dictBuffer.
* It is important that dictBuffer outlives CDict, it must remain read accessible throughout the lifetime of CDict */
ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_byReference(const void* dictBuffer, size_t dictSize, int compressionLevel);
/*! ZSTD_createCDict_advanced() :
* Create a ZSTD_CDict using external alloc and free, and customized compression parameters */
ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize,
ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize, unsigned byReference,
ZSTD_parameters params, ZSTD_customMem customMem);
/*! ZSTD_sizeof_CDict() :

6
programs/bench.c
View File

@ -236,7 +236,7 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
ZSTD_customMem const cmem = { NULL, NULL, NULL };
U64 clockLoop = g_nbSeconds ? TIMELOOP_MICROSEC : 1;
U32 nbLoops = 0;
ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictBuffer, dictBufferSize, zparams, cmem);
ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictBuffer, dictBufferSize, 1, zparams, cmem);
if (cdict==NULL) EXM_THROW(1, "ZSTD_createCDict_advanced() allocation failure");
if (comprParams->windowLog) zparams.cParams.windowLog = comprParams->windowLog;
if (comprParams->chainLog) zparams.cParams.chainLog = comprParams->chainLog;
@ -452,7 +452,7 @@ static void BMK_loadFiles(void* buffer, size_t bufferSize,
if (totalSize == 0) EXM_THROW(12, "no data to bench");
}
static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles, const char* dictFileName,
static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles, const char* dictFileName,
int cLevel, int cLevelLast, ZSTD_compressionParameters *compressionParams)
{
void* srcBuffer;
@ -523,7 +523,7 @@ static void BMK_syntheticTest(int cLevel, int cLevelLast, double compressibility
}
int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles, const char* dictFileName,
int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles, const char* dictFileName,
int cLevel, int cLevelLast, ZSTD_compressionParameters* compressionParams)
{
double const compressibility = (double)g_compressibilityDefault / 100;

2
zlibWrapper/examples/zwrapbench.c
View File

@ -234,7 +234,7 @@ static int BMK_benchMem(z_const void* srcBuffer, size_t srcSize,
if (compressor == BMK_ZSTD) {
ZSTD_parameters const zparams = ZSTD_getParams(cLevel, avgSize, dictBufferSize);
ZSTD_customMem const cmem = { NULL, NULL, NULL };
ZSTD_CDict* cdict = ZSTD_createCDict_advanced(dictBuffer, dictBufferSize, zparams, cmem);
ZSTD_CDict* cdict = ZSTD_createCDict_advanced(dictBuffer, dictBufferSize, 1, zparams, cmem);
if (cdict==NULL) EXM_THROW(1, "ZSTD_createCDict_advanced() allocation failure");
do {