facebook
/
zstd
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Wrap the new advanced api completely

dev
Nick Terrell 2019-03-13 14:05:18 -07:00
parent 11e73576bb
commit e55da9e963
4 changed files with 70 additions and 66 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

112
lib/compress/zstd_compress.c
View File

@ -3769,13 +3769,17 @@ static size_t ZSTD_resetCStream_internal(ZSTD_CStream* cctx,
/* ZSTD_resetCStream():
* pledgedSrcSize == 0 means "unknown" */
size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize)
size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pss)
{
ZSTD_CCtx_params params = zcs->requestedParams;
/* temporary : 0 interpreted as "unknown" during transition period.
* Users willing to specify "unknown" **must** use ZSTD_CONTENTSIZE_UNKNOWN.
* 0 will be interpreted as "empty" in the future.
*/
U64 const pledgedSrcSize = (pss==0) ? ZSTD_CONTENTSIZE_UNKNOWN : pss;
DEBUGLOG(4, "ZSTD_resetCStream: pledgedSrcSize = %u", (unsigned)pledgedSrcSize);
if (pledgedSrcSize==0) pledgedSrcSize = ZSTD_CONTENTSIZE_UNKNOWN;
params.fParams.contentSizeFlag = 1;
return ZSTD_resetCStream_internal(zcs, NULL, 0, ZSTD_dct_auto, zcs->cdict, params, pledgedSrcSize);
FORWARD_IF_ERROR( ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only) );
FORWARD_IF_ERROR( ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize) );
return 0;
}
/*! ZSTD_initCStream_internal() :
@ -3787,31 +3791,18 @@ size_t ZSTD_initCStream_internal(ZSTD_CStream* zcs,
ZSTD_CCtx_params params, unsigned long long pledgedSrcSize)
{
DEBUGLOG(4, "ZSTD_initCStream_internal");
params.cParams = ZSTD_getCParamsFromCCtxParams(&params, pledgedSrcSize, dictSize);
FORWARD_IF_ERROR( ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only) );
FORWARD_IF_ERROR( ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize) );
assert(!ZSTD_isError(ZSTD_checkCParams(params.cParams)));
zcs->requestedParams = params;
assert(!((dict) && (cdict))); /* either dict or cdict, not both */
if (dict && dictSize >= 8) {
DEBUGLOG(4, "loading dictionary of size %u", (unsigned)dictSize);
RETURN_ERROR_IF(
zcs->staticSize, memory_allocation,
"static CCtx: incompatible with internal cdict creation");
ZSTD_freeCDict(zcs->cdictLocal);
zcs->cdictLocal = ZSTD_createCDict_advanced(dict, dictSize,
ZSTD_dlm_byCopy, ZSTD_dct_auto,
params.cParams, zcs->customMem);
zcs->cdict = zcs->cdictLocal;
RETURN_ERROR_IF(zcs->cdictLocal == NULL, memory_allocation);
if (dict) {
FORWARD_IF_ERROR( ZSTD_CCtx_loadDictionary(zcs, dict, dictSize) );
} else {
if (cdict) {
params.cParams = ZSTD_getCParamsFromCDict(cdict); /* cParams are enforced from cdict; it includes windowLog */
}
ZSTD_freeCDict(zcs->cdictLocal);
zcs->cdictLocal = NULL;
zcs->cdict = cdict;
/* Dictionary is cleared if !cdict */
FORWARD_IF_ERROR( ZSTD_CCtx_refCDict(zcs, cdict) );
}
return ZSTD_resetCStream_internal(zcs, NULL, 0, ZSTD_dct_auto, zcs->cdict, params, pledgedSrcSize);
return 0;
}
/* ZSTD_initCStream_usingCDict_advanced() :
@ -3822,23 +3813,20 @@ size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs,
unsigned long long pledgedSrcSize)
{
DEBUGLOG(4, "ZSTD_initCStream_usingCDict_advanced");
RETURN_ERROR_IF(!cdict, dictionary_wrong,
"cannot handle NULL cdict (does not know what to do)");
{ ZSTD_CCtx_params params = zcs->requestedParams;
params.cParams = ZSTD_getCParamsFromCDict(cdict);
params.fParams = fParams;
return ZSTD_initCStream_internal(zcs,
NULL, 0, cdict,
params, pledgedSrcSize);
}
FORWARD_IF_ERROR( ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only) );
FORWARD_IF_ERROR( ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize) );
zcs->requestedParams.fParams = fParams;
FORWARD_IF_ERROR( ZSTD_CCtx_refCDict(zcs, cdict) );
return 0;
}
/* note : cdict must outlive compression session */
size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict)
{
ZSTD_frameParameters const fParams = { 0 /* contentSizeFlag */, 0 /* checksum */, 0 /* hideDictID */ };
DEBUGLOG(4, "ZSTD_initCStream_usingCDict");
return ZSTD_initCStream_usingCDict_advanced(zcs, cdict, fParams, ZSTD_CONTENTSIZE_UNKNOWN); /* note : will check that cdict != NULL */
FORWARD_IF_ERROR( ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only) );
FORWARD_IF_ERROR( ZSTD_CCtx_refCDict(zcs, cdict) );
return 0;
}
@ -3848,33 +3836,53 @@ size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict)
* dict is loaded with default parameters ZSTD_dm_auto and ZSTD_dlm_byCopy. */
size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs,
const void* dict, size_t dictSize,
ZSTD_parameters params, unsigned long long pledgedSrcSize)
ZSTD_parameters params, unsigned long long pss)
{
DEBUGLOG(4, "ZSTD_initCStream_advanced: pledgedSrcSize=%u, flag=%u",
(unsigned)pledgedSrcSize, params.fParams.contentSizeFlag);
/* for compatibility with older programs relying on this behavior.
* Users should now specify ZSTD_CONTENTSIZE_UNKNOWN.
* This line will be removed in the future.
*/
U64 const pledgedSrcSize = (pss==0 && params.fParams.contentSizeFlag==0) ? ZSTD_CONTENTSIZE_UNKNOWN : pss;
DEBUGLOG(4, "ZSTD_initCStream_advanced");
FORWARD_IF_ERROR( ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only) );
FORWARD_IF_ERROR( ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize) );
FORWARD_IF_ERROR( ZSTD_checkCParams(params.cParams) );
if ((pledgedSrcSize==0) && (params.fParams.contentSizeFlag==0)) pledgedSrcSize = ZSTD_CONTENTSIZE_UNKNOWN; /* for compatibility with older programs relying on this behavior. Users should now specify ZSTD_CONTENTSIZE_UNKNOWN. This line will be removed in the future. */
zcs->requestedParams = ZSTD_assignParamsToCCtxParams(zcs->requestedParams, params);
return ZSTD_initCStream_internal(zcs, dict, dictSize, NULL /*cdict*/, zcs->requestedParams, pledgedSrcSize);
FORWARD_IF_ERROR( ZSTD_CCtx_loadDictionary(zcs, dict, dictSize) );
return 0;
}
size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel)
{
ZSTD_CCtxParams_init(&zcs->requestedParams, compressionLevel);
return ZSTD_initCStream_internal(zcs, dict, dictSize, NULL, zcs->requestedParams, ZSTD_CONTENTSIZE_UNKNOWN);
DEBUGLOG(4, "ZSTD_initCStream_usingDict");
FORWARD_IF_ERROR( ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only) );
FORWARD_IF_ERROR( ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel) );
FORWARD_IF_ERROR( ZSTD_CCtx_loadDictionary(zcs, dict, dictSize) );
return 0;
}
size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pss)
{
U64 const pledgedSrcSize = (pss==0) ? ZSTD_CONTENTSIZE_UNKNOWN : pss; /* temporary : 0 interpreted as "unknown" during transition period. Users willing to specify "unknown" **must** use ZSTD_CONTENTSIZE_UNKNOWN. `0` will be interpreted as "empty" in the future */
ZSTD_CCtxParams_init(&zcs->requestedParams, compressionLevel);
return ZSTD_initCStream_internal(zcs, NULL, 0, NULL, zcs->requestedParams, pledgedSrcSize);
/* temporary : 0 interpreted as "unknown" during transition period.
* Users willing to specify "unknown" **must** use ZSTD_CONTENTSIZE_UNKNOWN.
* 0 will be interpreted as "empty" in the future.
*/
U64 const pledgedSrcSize = (pss==0) ? ZSTD_CONTENTSIZE_UNKNOWN : pss;
DEBUGLOG(4, "ZSTD_initCStream_srcSize");
FORWARD_IF_ERROR( ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only) );
FORWARD_IF_ERROR( ZSTD_CCtx_refCDict(zcs, NULL) );
FORWARD_IF_ERROR( ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel) );
FORWARD_IF_ERROR( ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize) );
return 0;
}
size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel)
{
DEBUGLOG(4, "ZSTD_initCStream");
return ZSTD_initCStream_srcSize(zcs, compressionLevel, ZSTD_CONTENTSIZE_UNKNOWN);
FORWARD_IF_ERROR( ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only) );
FORWARD_IF_ERROR( ZSTD_CCtx_refCDict(zcs, NULL) );
FORWARD_IF_ERROR( ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel) );
return 0;
}
/*====== Compression ======*/
@ -3898,10 +3906,10 @@ static size_t ZSTD_limitCopy(void* dst, size_t dstCapacity,
* internal function for all *compressStream*() variants
* non-static, because can be called from zstdmt_compress.c
* @return : hint size for next input */
size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs,
ZSTD_outBuffer* output,
ZSTD_inBuffer* input,
ZSTD_EndDirective const flushMode)
static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs,
ZSTD_outBuffer* output,
ZSTD_inBuffer* input,
ZSTD_EndDirective const flushMode)
{
const char* const istart = (const char*)input->src;
const char* const iend = istart + input->size;

7
lib/compress/zstd_compress_internal.h
View File

@ -808,13 +808,6 @@ size_t ZSTD_initCStream_internal(ZSTD_CStream* zcs,
void ZSTD_resetSeqStore(seqStore_t* ssPtr);
/*! ZSTD_compressStream_generic() :
* Private use only. To be called from zstdmt_compress.c in single-thread mode. */
size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs,
ZSTD_outBuffer* output,
ZSTD_inBuffer* input,
ZSTD_EndDirective const flushMode);
/*! ZSTD_getCParamsFromCDict() :
* as the name implies */
ZSTD_compressionParameters ZSTD_getCParamsFromCDict(const ZSTD_CDict* cdict);

2
lib/compress/zstdmt_compress.c
View File

@ -1967,7 +1967,7 @@ size_t ZSTDMT_compressStream_generic(ZSTDMT_CCtx* mtctx,
assert(input->pos <= input->size);
if (mtctx->singleBlockingThread) { /* delegate to single-thread (synchronous) */
return ZSTD_compressStream_generic(mtctx->cctxPool->cctx[0], output, input, endOp);
return ZSTD_compressStream2(mtctx->cctxPool->cctx[0], output, input, endOp);
}
if ((mtctx->frameEnded) && (endOp==ZSTD_e_continue)) {

15
tests/zstreamtest.c
View File

@ -495,7 +495,7 @@ static int basicUnitTests(U32 seed, double compressibility)
/* _srcSize compression test */
DISPLAYLEVEL(3, "test%3i : compress_srcSize %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
ZSTD_initCStream_srcSize(zc, 1, CNBufferSize);
CHECK_Z( ZSTD_initCStream_srcSize(zc, 1, CNBufferSize) );
outBuff.dst = (char*)(compressedBuffer);
outBuff.size = compressedBufferSize;
outBuff.pos = 0;
@ -503,11 +503,14 @@ static int basicUnitTests(U32 seed, double compressibility)
inBuff.size = CNBufferSize;
inBuff.pos = 0;
CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) );
if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */
{ size_t const r = ZSTD_endStream(zc, &outBuff);
if (r != 0) goto _output_error; } /* error, or some data not flushed */
{ unsigned long long origSize = ZSTD_findDecompressedSize(outBuff.dst, outBuff.pos);
if ((size_t)origSize != CNBufferSize) goto _output_error; } /* exact original size must be present */
CHECK(inBuff.pos != inBuff.size, "Entire input should be consumed");
{ size_t const r = ZSTD_endStream(zc, &outBuff);
CHECK(r != 0, "Error or some data not flushed (ret=%zu)", r);
}
{ unsigned long long origSize = ZSTD_findDecompressedSize(outBuff.dst, outBuff.pos);
CHECK(origSize == ZSTD_CONTENTSIZE_UNKNOWN, "Unknown!");
CHECK((size_t)origSize != CNBufferSize, "Exact original size must be present (got %llu)", origSize);
}
DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/COMPRESSIBLE_NOISE_LENGTH*100);
/* wrong _srcSize compression test */