implemented ZSTD_DCtx_loadDictionary*()
this required updating ZSTD_createDDict_advanced() to accept a dictContentType parameter (raw, full, auto).dev
parent
9e6ba88363
commit
353117c5d7
|
@ -1047,7 +1047,7 @@ size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const void* prefix, size_t
|
||||||
</p></pre><BR>
|
</p></pre><BR>
|
||||||
|
|
||||||
<h3>Advanced parameters for decompression API</h3><pre></pre><b><pre></pre></b><BR>
|
<h3>Advanced parameters for decompression API</h3><pre></pre><b><pre></pre></b><BR>
|
||||||
<pre><b>size_t ZSTD_DCtx_loadDictionary(ZSTD_DCtx* dctx, const void* dict, size_t dictSize); </b>/* not implemented */<b>
|
<pre><b>size_t ZSTD_DCtx_loadDictionary(ZSTD_DCtx* dctx, const void* dict, size_t dictSize);
|
||||||
size_t ZSTD_DCtx_loadDictionary_byReference(ZSTD_DCtx* dctx, const void* dict, size_t dictSize); </b>/* not implemented */<b>
|
size_t ZSTD_DCtx_loadDictionary_byReference(ZSTD_DCtx* dctx, const void* dict, size_t dictSize); </b>/* not implemented */<b>
|
||||||
size_t ZSTD_DCtx_loadDictionary_advanced(ZSTD_DCtx* dctx, const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictMode_e dictMode); </b>/* not implemented */<b>
|
size_t ZSTD_DCtx_loadDictionary_advanced(ZSTD_DCtx* dctx, const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictMode_e dictMode); </b>/* not implemented */<b>
|
||||||
</b><p> Create an internal DDict from dict buffer,
|
</b><p> Create an internal DDict from dict buffer,
|
||||||
|
|
|
@ -14,8 +14,9 @@
|
||||||
*****************************************************************/
|
*****************************************************************/
|
||||||
/*!
|
/*!
|
||||||
* HEAPMODE :
|
* HEAPMODE :
|
||||||
* Select how default decompression function ZSTD_decompress() will allocate memory,
|
* Select how default decompression function ZSTD_decompress() allocates its context,
|
||||||
* in memory stack (0), or in memory heap (1, requires malloc())
|
* on stack (0), or into heap (1, default; requires malloc()).
|
||||||
|
* Note that functions with explicit context such as ZSTD_decompressDCtx() are unaffected.
|
||||||
*/
|
*/
|
||||||
#ifndef ZSTD_HEAPMODE
|
#ifndef ZSTD_HEAPMODE
|
||||||
# define ZSTD_HEAPMODE 1
|
# define ZSTD_HEAPMODE 1
|
||||||
|
@ -30,10 +31,11 @@
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
/*!
|
/*!
|
||||||
* MAXWINDOWSIZE_DEFAULT :
|
* MAXWINDOWSIZE_DEFAULT :
|
||||||
* maximum window size accepted by DStream, by default.
|
* maximum window size accepted by DStream __by default__.
|
||||||
* Frames requiring more memory will be rejected.
|
* Frames requiring more memory will be rejected.
|
||||||
*/
|
* It's possible to set a different limit using ZSTD_DCtx_setMaxWindowSize().
|
||||||
|
*/
|
||||||
#ifndef ZSTD_MAXWINDOWSIZE_DEFAULT
|
#ifndef ZSTD_MAXWINDOWSIZE_DEFAULT
|
||||||
# define ZSTD_MAXWINDOWSIZE_DEFAULT (((U32)1 << ZSTD_WINDOWLOG_DEFAULTMAX) + 1)
|
# define ZSTD_MAXWINDOWSIZE_DEFAULT (((U32)1 << ZSTD_WINDOWLOG_DEFAULTMAX) + 1)
|
||||||
#endif
|
#endif
|
||||||
|
@ -2335,13 +2337,23 @@ size_t ZSTD_decompressBegin_usingDDict(ZSTD_DCtx* dstDCtx, const ZSTD_DDict* ddi
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
static size_t ZSTD_loadEntropy_inDDict(ZSTD_DDict* ddict)
|
static size_t ZSTD_loadEntropy_inDDict(ZSTD_DDict* ddict, ZSTD_dictMode_e dictContentType)
|
||||||
{
|
{
|
||||||
ddict->dictID = 0;
|
ddict->dictID = 0;
|
||||||
ddict->entropyPresent = 0;
|
ddict->entropyPresent = 0;
|
||||||
if (ddict->dictSize < 8) return 0;
|
if (dictContentType == ZSTD_dm_rawContent) return 0;
|
||||||
|
|
||||||
|
if (ddict->dictSize < 8) {
|
||||||
|
if (dictContentType == ZSTD_dm_fullDict)
|
||||||
|
return ERROR(dictionary_corrupted); /* only accept specified dictionaries */
|
||||||
|
return 0; /* pure content mode */
|
||||||
|
}
|
||||||
{ U32 const magic = MEM_readLE32(ddict->dictContent);
|
{ U32 const magic = MEM_readLE32(ddict->dictContent);
|
||||||
if (magic != ZSTD_MAGIC_DICTIONARY) return 0; /* pure content mode */
|
if (magic != ZSTD_MAGIC_DICTIONARY) {
|
||||||
|
if (dictContentType == ZSTD_dm_fullDict)
|
||||||
|
return ERROR(dictionary_corrupted); /* only accept specified dictionaries */
|
||||||
|
return 0; /* pure content mode */
|
||||||
|
}
|
||||||
}
|
}
|
||||||
ddict->dictID = MEM_readLE32((const char*)ddict->dictContent + ZSTD_frameIdSize);
|
ddict->dictID = MEM_readLE32((const char*)ddict->dictContent + ZSTD_frameIdSize);
|
||||||
|
|
||||||
|
@ -2352,7 +2364,10 @@ static size_t ZSTD_loadEntropy_inDDict(ZSTD_DDict* ddict)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
static size_t ZSTD_initDDict_internal(ZSTD_DDict* ddict, const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod)
|
static size_t ZSTD_initDDict_internal(ZSTD_DDict* ddict,
|
||||||
|
const void* dict, size_t dictSize,
|
||||||
|
ZSTD_dictLoadMethod_e dictLoadMethod,
|
||||||
|
ZSTD_dictMode_e dictContentType)
|
||||||
{
|
{
|
||||||
if ((dictLoadMethod == ZSTD_dlm_byRef) || (!dict) || (!dictSize)) {
|
if ((dictLoadMethod == ZSTD_dlm_byRef) || (!dict) || (!dictSize)) {
|
||||||
ddict->dictBuffer = NULL;
|
ddict->dictBuffer = NULL;
|
||||||
|
@ -2368,12 +2383,15 @@ static size_t ZSTD_initDDict_internal(ZSTD_DDict* ddict, const void* dict, size_
|
||||||
ddict->entropy.hufTable[0] = (HUF_DTable)((HufLog)*0x1000001); /* cover both little and big endian */
|
ddict->entropy.hufTable[0] = (HUF_DTable)((HufLog)*0x1000001); /* cover both little and big endian */
|
||||||
|
|
||||||
/* parse dictionary content */
|
/* parse dictionary content */
|
||||||
CHECK_F( ZSTD_loadEntropy_inDDict(ddict) );
|
CHECK_F( ZSTD_loadEntropy_inDDict(ddict, dictContentType) );
|
||||||
|
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_customMem customMem)
|
ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize,
|
||||||
|
ZSTD_dictLoadMethod_e dictLoadMethod,
|
||||||
|
ZSTD_dictMode_e dictContentType,
|
||||||
|
ZSTD_customMem customMem)
|
||||||
{
|
{
|
||||||
if (!customMem.customAlloc ^ !customMem.customFree) return NULL;
|
if (!customMem.customAlloc ^ !customMem.customFree) return NULL;
|
||||||
|
|
||||||
|
@ -2381,7 +2399,7 @@ ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize, ZSTD_di
|
||||||
if (!ddict) return NULL;
|
if (!ddict) return NULL;
|
||||||
ddict->cMem = customMem;
|
ddict->cMem = customMem;
|
||||||
|
|
||||||
if (ZSTD_isError( ZSTD_initDDict_internal(ddict, dict, dictSize, dictLoadMethod) )) {
|
if (ZSTD_isError( ZSTD_initDDict_internal(ddict, dict, dictSize, dictLoadMethod, dictContentType) )) {
|
||||||
ZSTD_freeDDict(ddict);
|
ZSTD_freeDDict(ddict);
|
||||||
return NULL;
|
return NULL;
|
||||||
}
|
}
|
||||||
|
@ -2397,7 +2415,7 @@ ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize, ZSTD_di
|
||||||
ZSTD_DDict* ZSTD_createDDict(const void* dict, size_t dictSize)
|
ZSTD_DDict* ZSTD_createDDict(const void* dict, size_t dictSize)
|
||||||
{
|
{
|
||||||
ZSTD_customMem const allocator = { NULL, NULL, NULL };
|
ZSTD_customMem const allocator = { NULL, NULL, NULL };
|
||||||
return ZSTD_createDDict_advanced(dict, dictSize, ZSTD_dlm_byCopy, allocator);
|
return ZSTD_createDDict_advanced(dict, dictSize, ZSTD_dlm_byCopy, ZSTD_dm_auto, allocator);
|
||||||
}
|
}
|
||||||
|
|
||||||
/*! ZSTD_createDDict_byReference() :
|
/*! ZSTD_createDDict_byReference() :
|
||||||
|
@ -2407,7 +2425,7 @@ ZSTD_DDict* ZSTD_createDDict(const void* dict, size_t dictSize)
|
||||||
ZSTD_DDict* ZSTD_createDDict_byReference(const void* dictBuffer, size_t dictSize)
|
ZSTD_DDict* ZSTD_createDDict_byReference(const void* dictBuffer, size_t dictSize)
|
||||||
{
|
{
|
||||||
ZSTD_customMem const allocator = { NULL, NULL, NULL };
|
ZSTD_customMem const allocator = { NULL, NULL, NULL };
|
||||||
return ZSTD_createDDict_advanced(dictBuffer, dictSize, ZSTD_dlm_byRef, allocator);
|
return ZSTD_createDDict_advanced(dictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dm_auto, allocator);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@ -2427,7 +2445,7 @@ const ZSTD_DDict* ZSTD_initStaticDDict(
|
||||||
memcpy(ddict+1, dict, dictSize); /* local copy */
|
memcpy(ddict+1, dict, dictSize); /* local copy */
|
||||||
dict = ddict+1;
|
dict = ddict+1;
|
||||||
}
|
}
|
||||||
if (ZSTD_isError( ZSTD_initDDict_internal(ddict, dict, dictSize, ZSTD_dlm_byRef) ))
|
if (ZSTD_isError( ZSTD_initDDict_internal(ddict, dict, dictSize, ZSTD_dlm_byRef, ZSTD_dm_auto) ))
|
||||||
return NULL;
|
return NULL;
|
||||||
return ddict;
|
return ddict;
|
||||||
}
|
}
|
||||||
|
@ -2546,17 +2564,35 @@ size_t ZSTD_freeDStream(ZSTD_DStream* zds)
|
||||||
size_t ZSTD_DStreamInSize(void) { return ZSTD_BLOCKSIZE_MAX + ZSTD_blockHeaderSize; }
|
size_t ZSTD_DStreamInSize(void) { return ZSTD_BLOCKSIZE_MAX + ZSTD_blockHeaderSize; }
|
||||||
size_t ZSTD_DStreamOutSize(void) { return ZSTD_BLOCKSIZE_MAX; }
|
size_t ZSTD_DStreamOutSize(void) { return ZSTD_BLOCKSIZE_MAX; }
|
||||||
|
|
||||||
|
size_t ZSTD_DCtx_loadDictionary_advanced(ZSTD_DCtx* dctx, const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictMode_e dictMode)
|
||||||
|
{
|
||||||
|
ZSTD_freeDDict(dctx->ddictLocal);
|
||||||
|
if (dict && dictSize >= 8) {
|
||||||
|
dctx->ddictLocal = ZSTD_createDDict_advanced(dict, dictSize, dictLoadMethod, dictMode, dctx->customMem);
|
||||||
|
if (dctx->ddictLocal == NULL) return ERROR(memory_allocation);
|
||||||
|
} else {
|
||||||
|
dctx->ddictLocal = NULL;
|
||||||
|
}
|
||||||
|
dctx->ddict = dctx->ddictLocal;
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
size_t ZSTD_DCtx_loadDictionary_byReference(ZSTD_DCtx* dctx, const void* dict, size_t dictSize)
|
||||||
|
{
|
||||||
|
return ZSTD_DCtx_loadDictionary_advanced(dctx, dict, dictSize, ZSTD_dlm_byRef, ZSTD_dm_auto);
|
||||||
|
}
|
||||||
|
|
||||||
|
size_t ZSTD_DCtx_loadDictionary(ZSTD_DCtx* dctx, const void* dict, size_t dictSize)
|
||||||
|
{
|
||||||
|
return ZSTD_DCtx_loadDictionary_advanced(dctx, dict, dictSize, ZSTD_dlm_byCopy, ZSTD_dm_auto);
|
||||||
|
}
|
||||||
|
|
||||||
size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize)
|
size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize)
|
||||||
{
|
{
|
||||||
DEBUGLOG(4, "ZSTD_initDStream_usingDict");
|
DEBUGLOG(4, "ZSTD_initDStream_usingDict");
|
||||||
zds->streamStage = zdss_loadHeader;
|
zds->streamStage = zdss_loadHeader;
|
||||||
zds->lhSize = zds->inPos = zds->outStart = zds->outEnd = 0;
|
zds->lhSize = zds->inPos = zds->outStart = zds->outEnd = 0;
|
||||||
ZSTD_freeDDict(zds->ddictLocal);
|
CHECK_F( ZSTD_DCtx_loadDictionary(zds, dict, dictSize) );
|
||||||
if (dict && dictSize >= 8) {
|
|
||||||
zds->ddictLocal = ZSTD_createDDict(dict, dictSize);
|
|
||||||
if (zds->ddictLocal == NULL) return ERROR(memory_allocation);
|
|
||||||
} else zds->ddictLocal = NULL;
|
|
||||||
zds->ddict = zds->ddictLocal;
|
|
||||||
zds->legacyVersion = 0;
|
zds->legacyVersion = 0;
|
||||||
zds->hostageByte = 0;
|
zds->hostageByte = 0;
|
||||||
return ZSTD_frameHeaderSize_prefix;
|
return ZSTD_frameHeaderSize_prefix;
|
||||||
|
|
|
@ -605,6 +605,7 @@ ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictS
|
||||||
|
|
||||||
ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize,
|
ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize,
|
||||||
ZSTD_dictLoadMethod_e dictLoadMethod,
|
ZSTD_dictLoadMethod_e dictLoadMethod,
|
||||||
|
ZSTD_dictMode_e dictContentType,
|
||||||
ZSTD_customMem customMem);
|
ZSTD_customMem customMem);
|
||||||
|
|
||||||
|
|
||||||
|
@ -1270,9 +1271,9 @@ ZSTDLIB_API size_t ZSTD_CCtx_setParametersUsingCCtxParams(
|
||||||
* Note 3 : Use ZSTD_DCtx_loadDictionary_advanced() to select
|
* Note 3 : Use ZSTD_DCtx_loadDictionary_advanced() to select
|
||||||
* how dictionary content will be interpreted and loaded.
|
* how dictionary content will be interpreted and loaded.
|
||||||
*/
|
*/
|
||||||
ZSTDLIB_API size_t ZSTD_DCtx_loadDictionary(ZSTD_DCtx* dctx, const void* dict, size_t dictSize); /* not implemented */
|
ZSTDLIB_API size_t ZSTD_DCtx_loadDictionary(ZSTD_DCtx* dctx, const void* dict, size_t dictSize);
|
||||||
ZSTDLIB_API size_t ZSTD_DCtx_loadDictionary_byReference(ZSTD_DCtx* dctx, const void* dict, size_t dictSize); /* not implemented */
|
ZSTDLIB_API size_t ZSTD_DCtx_loadDictionary_byReference(ZSTD_DCtx* dctx, const void* dict, size_t dictSize);
|
||||||
ZSTDLIB_API size_t ZSTD_DCtx_loadDictionary_advanced(ZSTD_DCtx* dctx, const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictMode_e dictMode); /* not implemented */
|
ZSTDLIB_API size_t ZSTD_DCtx_loadDictionary_advanced(ZSTD_DCtx* dctx, const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictMode_e dictMode);
|
||||||
|
|
||||||
|
|
||||||
/*! ZSTD_DCtx_refDDict() :
|
/*! ZSTD_DCtx_refDDict() :
|
||||||
|
|
|
@ -106,8 +106,7 @@ extern "C" {
|
||||||
# include <io.h> /* _isatty */
|
# include <io.h> /* _isatty */
|
||||||
# include <windows.h> /* DeviceIoControl, HANDLE, FSCTL_SET_SPARSE */
|
# include <windows.h> /* DeviceIoControl, HANDLE, FSCTL_SET_SPARSE */
|
||||||
# include <stdio.h> /* FILE */
|
# include <stdio.h> /* FILE */
|
||||||
static __inline int IS_CONSOLE(FILE* stdStream)
|
static __inline int IS_CONSOLE(FILE* stdStream) {
|
||||||
{
|
|
||||||
DWORD dummy;
|
DWORD dummy;
|
||||||
return _isatty(_fileno(stdStream)) && GetConsoleMode((HANDLE)_get_osfhandle(_fileno(stdStream)), &dummy);
|
return _isatty(_fileno(stdStream)) && GetConsoleMode((HANDLE)_get_osfhandle(_fileno(stdStream)), &dummy);
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue