implemented ZSTD_DCtx_loadDictionary*()

this required updating ZSTD_createDDict_advanced()
to accept a dictContentType parameter (raw, full, auto).
dev
Yann Collet 2018-03-20 13:40:29 -07:00
parent 9e6ba88363
commit 353117c5d7
4 changed files with 64 additions and 28 deletions

View File

@ -1047,7 +1047,7 @@ size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const void* prefix, size_t
</p></pre><BR> </p></pre><BR>
<h3>Advanced parameters for decompression API</h3><pre></pre><b><pre></pre></b><BR> <h3>Advanced parameters for decompression API</h3><pre></pre><b><pre></pre></b><BR>
<pre><b>size_t ZSTD_DCtx_loadDictionary(ZSTD_DCtx* dctx, const void* dict, size_t dictSize); </b>/* not implemented */<b> <pre><b>size_t ZSTD_DCtx_loadDictionary(ZSTD_DCtx* dctx, const void* dict, size_t dictSize);
size_t ZSTD_DCtx_loadDictionary_byReference(ZSTD_DCtx* dctx, const void* dict, size_t dictSize); </b>/* not implemented */<b> size_t ZSTD_DCtx_loadDictionary_byReference(ZSTD_DCtx* dctx, const void* dict, size_t dictSize); </b>/* not implemented */<b>
size_t ZSTD_DCtx_loadDictionary_advanced(ZSTD_DCtx* dctx, const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictMode_e dictMode); </b>/* not implemented */<b> size_t ZSTD_DCtx_loadDictionary_advanced(ZSTD_DCtx* dctx, const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictMode_e dictMode); </b>/* not implemented */<b>
</b><p> Create an internal DDict from dict buffer, </b><p> Create an internal DDict from dict buffer,

View File

@ -14,8 +14,9 @@
*****************************************************************/ *****************************************************************/
/*! /*!
* HEAPMODE : * HEAPMODE :
* Select how default decompression function ZSTD_decompress() will allocate memory, * Select how default decompression function ZSTD_decompress() allocates its context,
* in memory stack (0), or in memory heap (1, requires malloc()) * on stack (0), or into heap (1, default; requires malloc()).
* Note that functions with explicit context such as ZSTD_decompressDCtx() are unaffected.
*/ */
#ifndef ZSTD_HEAPMODE #ifndef ZSTD_HEAPMODE
# define ZSTD_HEAPMODE 1 # define ZSTD_HEAPMODE 1
@ -30,10 +31,11 @@
#endif #endif
/*! /*!
* MAXWINDOWSIZE_DEFAULT : * MAXWINDOWSIZE_DEFAULT :
* maximum window size accepted by DStream, by default. * maximum window size accepted by DStream __by default__.
* Frames requiring more memory will be rejected. * Frames requiring more memory will be rejected.
*/ * It's possible to set a different limit using ZSTD_DCtx_setMaxWindowSize().
*/
#ifndef ZSTD_MAXWINDOWSIZE_DEFAULT #ifndef ZSTD_MAXWINDOWSIZE_DEFAULT
# define ZSTD_MAXWINDOWSIZE_DEFAULT (((U32)1 << ZSTD_WINDOWLOG_DEFAULTMAX) + 1) # define ZSTD_MAXWINDOWSIZE_DEFAULT (((U32)1 << ZSTD_WINDOWLOG_DEFAULTMAX) + 1)
#endif #endif
@ -2335,13 +2337,23 @@ size_t ZSTD_decompressBegin_usingDDict(ZSTD_DCtx* dstDCtx, const ZSTD_DDict* ddi
return 0; return 0;
} }
static size_t ZSTD_loadEntropy_inDDict(ZSTD_DDict* ddict) static size_t ZSTD_loadEntropy_inDDict(ZSTD_DDict* ddict, ZSTD_dictMode_e dictContentType)
{ {
ddict->dictID = 0; ddict->dictID = 0;
ddict->entropyPresent = 0; ddict->entropyPresent = 0;
if (ddict->dictSize < 8) return 0; if (dictContentType == ZSTD_dm_rawContent) return 0;
if (ddict->dictSize < 8) {
if (dictContentType == ZSTD_dm_fullDict)
return ERROR(dictionary_corrupted); /* only accept specified dictionaries */
return 0; /* pure content mode */
}
{ U32 const magic = MEM_readLE32(ddict->dictContent); { U32 const magic = MEM_readLE32(ddict->dictContent);
if (magic != ZSTD_MAGIC_DICTIONARY) return 0; /* pure content mode */ if (magic != ZSTD_MAGIC_DICTIONARY) {
if (dictContentType == ZSTD_dm_fullDict)
return ERROR(dictionary_corrupted); /* only accept specified dictionaries */
return 0; /* pure content mode */
}
} }
ddict->dictID = MEM_readLE32((const char*)ddict->dictContent + ZSTD_frameIdSize); ddict->dictID = MEM_readLE32((const char*)ddict->dictContent + ZSTD_frameIdSize);
@ -2352,7 +2364,10 @@ static size_t ZSTD_loadEntropy_inDDict(ZSTD_DDict* ddict)
} }
static size_t ZSTD_initDDict_internal(ZSTD_DDict* ddict, const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod) static size_t ZSTD_initDDict_internal(ZSTD_DDict* ddict,
const void* dict, size_t dictSize,
ZSTD_dictLoadMethod_e dictLoadMethod,
ZSTD_dictMode_e dictContentType)
{ {
if ((dictLoadMethod == ZSTD_dlm_byRef) || (!dict) || (!dictSize)) { if ((dictLoadMethod == ZSTD_dlm_byRef) || (!dict) || (!dictSize)) {
ddict->dictBuffer = NULL; ddict->dictBuffer = NULL;
@ -2368,12 +2383,15 @@ static size_t ZSTD_initDDict_internal(ZSTD_DDict* ddict, const void* dict, size_
ddict->entropy.hufTable[0] = (HUF_DTable)((HufLog)*0x1000001); /* cover both little and big endian */ ddict->entropy.hufTable[0] = (HUF_DTable)((HufLog)*0x1000001); /* cover both little and big endian */
/* parse dictionary content */ /* parse dictionary content */
CHECK_F( ZSTD_loadEntropy_inDDict(ddict) ); CHECK_F( ZSTD_loadEntropy_inDDict(ddict, dictContentType) );
return 0; return 0;
} }
ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_customMem customMem) ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize,
ZSTD_dictLoadMethod_e dictLoadMethod,
ZSTD_dictMode_e dictContentType,
ZSTD_customMem customMem)
{ {
if (!customMem.customAlloc ^ !customMem.customFree) return NULL; if (!customMem.customAlloc ^ !customMem.customFree) return NULL;
@ -2381,7 +2399,7 @@ ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize, ZSTD_di
if (!ddict) return NULL; if (!ddict) return NULL;
ddict->cMem = customMem; ddict->cMem = customMem;
if (ZSTD_isError( ZSTD_initDDict_internal(ddict, dict, dictSize, dictLoadMethod) )) { if (ZSTD_isError( ZSTD_initDDict_internal(ddict, dict, dictSize, dictLoadMethod, dictContentType) )) {
ZSTD_freeDDict(ddict); ZSTD_freeDDict(ddict);
return NULL; return NULL;
} }
@ -2397,7 +2415,7 @@ ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize, ZSTD_di
ZSTD_DDict* ZSTD_createDDict(const void* dict, size_t dictSize) ZSTD_DDict* ZSTD_createDDict(const void* dict, size_t dictSize)
{ {
ZSTD_customMem const allocator = { NULL, NULL, NULL }; ZSTD_customMem const allocator = { NULL, NULL, NULL };
return ZSTD_createDDict_advanced(dict, dictSize, ZSTD_dlm_byCopy, allocator); return ZSTD_createDDict_advanced(dict, dictSize, ZSTD_dlm_byCopy, ZSTD_dm_auto, allocator);
} }
/*! ZSTD_createDDict_byReference() : /*! ZSTD_createDDict_byReference() :
@ -2407,7 +2425,7 @@ ZSTD_DDict* ZSTD_createDDict(const void* dict, size_t dictSize)
ZSTD_DDict* ZSTD_createDDict_byReference(const void* dictBuffer, size_t dictSize) ZSTD_DDict* ZSTD_createDDict_byReference(const void* dictBuffer, size_t dictSize)
{ {
ZSTD_customMem const allocator = { NULL, NULL, NULL }; ZSTD_customMem const allocator = { NULL, NULL, NULL };
return ZSTD_createDDict_advanced(dictBuffer, dictSize, ZSTD_dlm_byRef, allocator); return ZSTD_createDDict_advanced(dictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dm_auto, allocator);
} }
@ -2427,7 +2445,7 @@ const ZSTD_DDict* ZSTD_initStaticDDict(
memcpy(ddict+1, dict, dictSize); /* local copy */ memcpy(ddict+1, dict, dictSize); /* local copy */
dict = ddict+1; dict = ddict+1;
} }
if (ZSTD_isError( ZSTD_initDDict_internal(ddict, dict, dictSize, ZSTD_dlm_byRef) )) if (ZSTD_isError( ZSTD_initDDict_internal(ddict, dict, dictSize, ZSTD_dlm_byRef, ZSTD_dm_auto) ))
return NULL; return NULL;
return ddict; return ddict;
} }
@ -2546,17 +2564,35 @@ size_t ZSTD_freeDStream(ZSTD_DStream* zds)
size_t ZSTD_DStreamInSize(void) { return ZSTD_BLOCKSIZE_MAX + ZSTD_blockHeaderSize; } size_t ZSTD_DStreamInSize(void) { return ZSTD_BLOCKSIZE_MAX + ZSTD_blockHeaderSize; }
size_t ZSTD_DStreamOutSize(void) { return ZSTD_BLOCKSIZE_MAX; } size_t ZSTD_DStreamOutSize(void) { return ZSTD_BLOCKSIZE_MAX; }
size_t ZSTD_DCtx_loadDictionary_advanced(ZSTD_DCtx* dctx, const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictMode_e dictMode)
{
ZSTD_freeDDict(dctx->ddictLocal);
if (dict && dictSize >= 8) {
dctx->ddictLocal = ZSTD_createDDict_advanced(dict, dictSize, dictLoadMethod, dictMode, dctx->customMem);
if (dctx->ddictLocal == NULL) return ERROR(memory_allocation);
} else {
dctx->ddictLocal = NULL;
}
dctx->ddict = dctx->ddictLocal;
return 0;
}
size_t ZSTD_DCtx_loadDictionary_byReference(ZSTD_DCtx* dctx, const void* dict, size_t dictSize)
{
return ZSTD_DCtx_loadDictionary_advanced(dctx, dict, dictSize, ZSTD_dlm_byRef, ZSTD_dm_auto);
}
size_t ZSTD_DCtx_loadDictionary(ZSTD_DCtx* dctx, const void* dict, size_t dictSize)
{
return ZSTD_DCtx_loadDictionary_advanced(dctx, dict, dictSize, ZSTD_dlm_byCopy, ZSTD_dm_auto);
}
size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize) size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize)
{ {
DEBUGLOG(4, "ZSTD_initDStream_usingDict"); DEBUGLOG(4, "ZSTD_initDStream_usingDict");
zds->streamStage = zdss_loadHeader; zds->streamStage = zdss_loadHeader;
zds->lhSize = zds->inPos = zds->outStart = zds->outEnd = 0; zds->lhSize = zds->inPos = zds->outStart = zds->outEnd = 0;
ZSTD_freeDDict(zds->ddictLocal); CHECK_F( ZSTD_DCtx_loadDictionary(zds, dict, dictSize) );
if (dict && dictSize >= 8) {
zds->ddictLocal = ZSTD_createDDict(dict, dictSize);
if (zds->ddictLocal == NULL) return ERROR(memory_allocation);
} else zds->ddictLocal = NULL;
zds->ddict = zds->ddictLocal;
zds->legacyVersion = 0; zds->legacyVersion = 0;
zds->hostageByte = 0; zds->hostageByte = 0;
return ZSTD_frameHeaderSize_prefix; return ZSTD_frameHeaderSize_prefix;

View File

@ -605,6 +605,7 @@ ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictS
ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize, ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize,
ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictLoadMethod_e dictLoadMethod,
ZSTD_dictMode_e dictContentType,
ZSTD_customMem customMem); ZSTD_customMem customMem);
@ -1270,9 +1271,9 @@ ZSTDLIB_API size_t ZSTD_CCtx_setParametersUsingCCtxParams(
* Note 3 : Use ZSTD_DCtx_loadDictionary_advanced() to select * Note 3 : Use ZSTD_DCtx_loadDictionary_advanced() to select
* how dictionary content will be interpreted and loaded. * how dictionary content will be interpreted and loaded.
*/ */
ZSTDLIB_API size_t ZSTD_DCtx_loadDictionary(ZSTD_DCtx* dctx, const void* dict, size_t dictSize); /* not implemented */ ZSTDLIB_API size_t ZSTD_DCtx_loadDictionary(ZSTD_DCtx* dctx, const void* dict, size_t dictSize);
ZSTDLIB_API size_t ZSTD_DCtx_loadDictionary_byReference(ZSTD_DCtx* dctx, const void* dict, size_t dictSize); /* not implemented */ ZSTDLIB_API size_t ZSTD_DCtx_loadDictionary_byReference(ZSTD_DCtx* dctx, const void* dict, size_t dictSize);
ZSTDLIB_API size_t ZSTD_DCtx_loadDictionary_advanced(ZSTD_DCtx* dctx, const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictMode_e dictMode); /* not implemented */ ZSTDLIB_API size_t ZSTD_DCtx_loadDictionary_advanced(ZSTD_DCtx* dctx, const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictMode_e dictMode);
/*! ZSTD_DCtx_refDDict() : /*! ZSTD_DCtx_refDDict() :

View File

@ -106,8 +106,7 @@ extern "C" {
# include <io.h> /* _isatty */ # include <io.h> /* _isatty */
# include <windows.h> /* DeviceIoControl, HANDLE, FSCTL_SET_SPARSE */ # include <windows.h> /* DeviceIoControl, HANDLE, FSCTL_SET_SPARSE */
# include <stdio.h> /* FILE */ # include <stdio.h> /* FILE */
static __inline int IS_CONSOLE(FILE* stdStream) static __inline int IS_CONSOLE(FILE* stdStream) {
{
DWORD dummy; DWORD dummy;
return _isatty(_fileno(stdStream)) && GetConsoleMode((HANDLE)_get_osfhandle(_fileno(stdStream)), &dummy); return _isatty(_fileno(stdStream)) && GetConsoleMode((HANDLE)_get_osfhandle(_fileno(stdStream)), &dummy);
} }