commit
8883af6a1e
@ -613,6 +613,7 @@ size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, const ZSTD_CDict*
|
||||
unsigned long long ingested;
|
||||
unsigned long long consumed;
|
||||
unsigned long long produced;
|
||||
unsigned currentJobID;
|
||||
} ZSTD_frameProgression;
|
||||
</b></pre><BR>
|
||||
<h3>Advanced Streaming decompression functions</h3><pre></pre><b><pre>typedef enum { DStream_p_maxWindowSize } ZSTD_DStreamParameter_e;
|
||||
|
@ -906,9 +906,27 @@ ZSTD_frameProgression ZSTD_getFrameProgression(const ZSTD_CCtx* cctx)
|
||||
fp.ingested = cctx->consumedSrcSize + buffered;
|
||||
fp.consumed = cctx->consumedSrcSize;
|
||||
fp.produced = cctx->producedCSize;
|
||||
fp.flushed = cctx->producedCSize; /* simplified; some data might still be left within streaming output buffer */
|
||||
fp.currentJobID = 0;
|
||||
fp.nbActiveWorkers = 0;
|
||||
return fp;
|
||||
} }
|
||||
|
||||
/*! ZSTD_toFlushNow()
|
||||
* Only useful for multithreading scenarios currently (nbWorkers >= 1).
|
||||
*/
|
||||
size_t ZSTD_toFlushNow(ZSTD_CCtx* cctx)
|
||||
{
|
||||
#ifdef ZSTD_MULTITHREAD
|
||||
if (cctx->appliedParams.nbWorkers > 0) {
|
||||
return ZSTDMT_toFlushNow(cctx->mtctx);
|
||||
}
|
||||
#endif
|
||||
(void)cctx;
|
||||
return 0; /* over-simplification; could also check if context is currently running in streaming mode, and in which case, report how many bytes are left to be flushed within output buffer */
|
||||
}
|
||||
|
||||
|
||||
|
||||
static U32 ZSTD_equivalentCParams(ZSTD_compressionParameters cParams1,
|
||||
ZSTD_compressionParameters cParams2)
|
||||
@ -3733,6 +3751,7 @@ size_t ZSTD_compress_generic (ZSTD_CCtx* cctx,
|
||||
|| (endOp == ZSTD_e_end && flushMin == 0) ) { /* compression completed */
|
||||
ZSTD_CCtx_reset(cctx);
|
||||
}
|
||||
DEBUGLOG(5, "completed ZSTD_compress_generic delegating to ZSTDMT_compressStream_generic");
|
||||
return flushMin;
|
||||
} }
|
||||
#endif
|
||||
|
@ -249,8 +249,8 @@ static buffer_t ZSTDMT_resizeBuffer(ZSTDMT_bufferPool* bufPool, buffer_t buffer)
|
||||
/* store buffer for later re-use, up to pool capacity */
|
||||
static void ZSTDMT_releaseBuffer(ZSTDMT_bufferPool* bufPool, buffer_t buf)
|
||||
{
|
||||
if (buf.start == NULL) return; /* compatible with release on NULL */
|
||||
DEBUGLOG(5, "ZSTDMT_releaseBuffer");
|
||||
if (buf.start == NULL) return; /* compatible with release on NULL */
|
||||
ZSTD_pthread_mutex_lock(&bufPool->poolMutex);
|
||||
if (bufPool->nbBuffers < bufPool->totalBuffers) {
|
||||
bufPool->bTable[bufPool->nbBuffers++] = buf; /* stored for later use */
|
||||
@ -542,6 +542,7 @@ static void ZSTDMT_serialState_update(serialState_t* serialState,
|
||||
/* Wait for our turn */
|
||||
ZSTD_PTHREAD_MUTEX_LOCK(&serialState->mutex);
|
||||
while (serialState->nextJobID < jobID) {
|
||||
DEBUGLOG(5, "wait for serialState->cond");
|
||||
ZSTD_pthread_cond_wait(&serialState->cond, &serialState->mutex);
|
||||
}
|
||||
/* A future job may error and skip our job */
|
||||
@ -631,6 +632,13 @@ typedef struct {
|
||||
unsigned frameChecksumNeeded; /* used only by mtctx */
|
||||
} ZSTDMT_jobDescription;
|
||||
|
||||
#define JOB_ERROR(e) { \
|
||||
ZSTD_PTHREAD_MUTEX_LOCK(&job->job_mutex); \
|
||||
job->cSize = e; \
|
||||
ZSTD_pthread_mutex_unlock(&job->job_mutex); \
|
||||
goto _endJob; \
|
||||
}
|
||||
|
||||
/* ZSTDMT_compressionJob() is a POOL_function type */
|
||||
void ZSTDMT_compressionJob(void* jobDescription)
|
||||
{
|
||||
@ -639,24 +647,17 @@ void ZSTDMT_compressionJob(void* jobDescription)
|
||||
ZSTD_CCtx* const cctx = ZSTDMT_getCCtx(job->cctxPool);
|
||||
rawSeqStore_t rawSeqStore = ZSTDMT_getSeq(job->seqPool);
|
||||
buffer_t dstBuff = job->dstBuff;
|
||||
size_t lastCBlockSize = 0;
|
||||
|
||||
/* ressources */
|
||||
if (cctx==NULL) {
|
||||
job->cSize = ERROR(memory_allocation);
|
||||
goto _endJob;
|
||||
}
|
||||
if (cctx==NULL) JOB_ERROR(ERROR(memory_allocation));
|
||||
if (dstBuff.start == NULL) { /* streaming job : doesn't provide a dstBuffer */
|
||||
dstBuff = ZSTDMT_getBuffer(job->bufPool);
|
||||
if (dstBuff.start==NULL) {
|
||||
job->cSize = ERROR(memory_allocation);
|
||||
goto _endJob;
|
||||
}
|
||||
if (dstBuff.start==NULL) JOB_ERROR(ERROR(memory_allocation));
|
||||
job->dstBuff = dstBuff; /* this value can be read in ZSTDMT_flush, when it copies the whole job */
|
||||
}
|
||||
if (jobParams.ldmParams.enableLdm && rawSeqStore.seq == NULL) {
|
||||
job->cSize = ERROR(memory_allocation);
|
||||
goto _endJob;
|
||||
}
|
||||
if (jobParams.ldmParams.enableLdm && rawSeqStore.seq == NULL)
|
||||
JOB_ERROR(ERROR(memory_allocation));
|
||||
|
||||
/* Don't compute the checksum for chunks, since we compute it externally,
|
||||
* but write it in the header.
|
||||
@ -670,30 +671,26 @@ void ZSTDMT_compressionJob(void* jobDescription)
|
||||
if (job->cdict) {
|
||||
size_t const initError = ZSTD_compressBegin_advanced_internal(cctx, NULL, 0, ZSTD_dct_auto, ZSTD_dtlm_fast, job->cdict, jobParams, job->fullFrameSize);
|
||||
assert(job->firstJob); /* only allowed for first job */
|
||||
if (ZSTD_isError(initError)) { job->cSize = initError; goto _endJob; }
|
||||
if (ZSTD_isError(initError)) JOB_ERROR(initError);
|
||||
} else { /* srcStart points at reloaded section */
|
||||
U64 const pledgedSrcSize = job->firstJob ? job->fullFrameSize : job->src.size;
|
||||
{ size_t const forceWindowError = ZSTD_CCtxParam_setParameter(&jobParams, ZSTD_p_forceMaxWindow, !job->firstJob);
|
||||
if (ZSTD_isError(forceWindowError)) {
|
||||
job->cSize = forceWindowError;
|
||||
goto _endJob;
|
||||
} }
|
||||
if (ZSTD_isError(forceWindowError)) JOB_ERROR(forceWindowError);
|
||||
}
|
||||
{ size_t const initError = ZSTD_compressBegin_advanced_internal(cctx,
|
||||
job->prefix.start, job->prefix.size, ZSTD_dct_rawContent, /* load dictionary in "content-only" mode (no header analysis) */
|
||||
ZSTD_dtlm_fast,
|
||||
NULL, /*cdict*/
|
||||
jobParams, pledgedSrcSize);
|
||||
if (ZSTD_isError(initError)) {
|
||||
job->cSize = initError;
|
||||
goto _endJob;
|
||||
} } }
|
||||
if (ZSTD_isError(initError)) JOB_ERROR(initError);
|
||||
} }
|
||||
|
||||
/* Perform serial step as early as possible, but after CCtx initialization */
|
||||
ZSTDMT_serialState_update(job->serial, cctx, rawSeqStore, job->src, job->jobID);
|
||||
|
||||
if (!job->firstJob) { /* flush and overwrite frame header when it's not first job */
|
||||
size_t const hSize = ZSTD_compressContinue(cctx, dstBuff.start, dstBuff.capacity, job->src.start, 0);
|
||||
if (ZSTD_isError(hSize)) { job->cSize = hSize; /* save error code */ goto _endJob; }
|
||||
if (ZSTD_isError(hSize)) JOB_ERROR(hSize);
|
||||
DEBUGLOG(5, "ZSTDMT_compressionJob: flush and overwrite %u bytes of frame header (not first job)", (U32)hSize);
|
||||
ZSTD_invalidateRepCodes(cctx);
|
||||
}
|
||||
@ -711,7 +708,7 @@ void ZSTDMT_compressionJob(void* jobDescription)
|
||||
assert(job->cSize == 0);
|
||||
for (chunkNb = 1; chunkNb < nbChunks; chunkNb++) {
|
||||
size_t const cSize = ZSTD_compressContinue(cctx, op, oend-op, ip, chunkSize);
|
||||
if (ZSTD_isError(cSize)) { job->cSize = cSize; goto _endJob; }
|
||||
if (ZSTD_isError(cSize)) JOB_ERROR(cSize);
|
||||
ip += chunkSize;
|
||||
op += cSize; assert(op < oend);
|
||||
/* stats */
|
||||
@ -724,18 +721,16 @@ void ZSTDMT_compressionJob(void* jobDescription)
|
||||
ZSTD_pthread_mutex_unlock(&job->job_mutex);
|
||||
}
|
||||
/* last block */
|
||||
assert(chunkSize > 0); assert((chunkSize & (chunkSize - 1)) == 0); /* chunkSize must be power of 2 for mask==(chunkSize-1) to work */
|
||||
assert(chunkSize > 0);
|
||||
assert((chunkSize & (chunkSize - 1)) == 0); /* chunkSize must be power of 2 for mask==(chunkSize-1) to work */
|
||||
if ((nbChunks > 0) | job->lastJob /*must output a "last block" flag*/ ) {
|
||||
size_t const lastBlockSize1 = job->src.size & (chunkSize-1);
|
||||
size_t const lastBlockSize = ((lastBlockSize1==0) & (job->src.size>=chunkSize)) ? chunkSize : lastBlockSize1;
|
||||
size_t const cSize = (job->lastJob) ?
|
||||
ZSTD_compressEnd (cctx, op, oend-op, ip, lastBlockSize) :
|
||||
ZSTD_compressContinue(cctx, op, oend-op, ip, lastBlockSize);
|
||||
if (ZSTD_isError(cSize)) { job->cSize = cSize; goto _endJob; }
|
||||
/* stats */
|
||||
ZSTD_PTHREAD_MUTEX_LOCK(&job->job_mutex);
|
||||
job->cSize += cSize;
|
||||
ZSTD_pthread_mutex_unlock(&job->job_mutex);
|
||||
if (ZSTD_isError(cSize)) JOB_ERROR(cSize);
|
||||
lastCBlockSize = cSize;
|
||||
} }
|
||||
|
||||
_endJob:
|
||||
@ -748,7 +743,9 @@ _endJob:
|
||||
ZSTDMT_releaseCCtx(job->cctxPool, cctx);
|
||||
/* report */
|
||||
ZSTD_PTHREAD_MUTEX_LOCK(&job->job_mutex);
|
||||
job->consumed = job->src.size;
|
||||
if (ZSTD_isError(job->cSize)) assert(lastCBlockSize == 0);
|
||||
job->cSize += lastCBlockSize;
|
||||
job->consumed = job->src.size; /* when job->consumed == job->src.size , compression job is presumed completed */
|
||||
ZSTD_pthread_cond_signal(&job->job_cond);
|
||||
ZSTD_pthread_mutex_unlock(&job->job_mutex);
|
||||
}
|
||||
@ -933,7 +930,7 @@ static void ZSTDMT_waitForAllJobsCompleted(ZSTDMT_CCtx* mtctx)
|
||||
unsigned const jobID = mtctx->doneJobID & mtctx->jobIDMask;
|
||||
ZSTD_PTHREAD_MUTEX_LOCK(&mtctx->jobs[jobID].job_mutex);
|
||||
while (mtctx->jobs[jobID].consumed < mtctx->jobs[jobID].src.size) {
|
||||
DEBUGLOG(5, "waiting for jobCompleted signal from job %u", mtctx->doneJobID); /* we want to block when waiting for data to flush */
|
||||
DEBUGLOG(4, "waiting for jobCompleted signal from job %u", mtctx->doneJobID); /* we want to block when waiting for data to flush */
|
||||
ZSTD_pthread_cond_wait(&mtctx->jobs[jobID].job_cond, &mtctx->jobs[jobID].job_mutex);
|
||||
}
|
||||
ZSTD_pthread_mutex_unlock(&mtctx->jobs[jobID].job_mutex);
|
||||
@ -1058,7 +1055,7 @@ static size_t ZSTDMT_resize(ZSTDMT_CCtx* mtctx, unsigned nbWorkers)
|
||||
|
||||
|
||||
/*! ZSTDMT_updateCParams_whileCompressing() :
|
||||
* Updates only a selected set of compression parameters, to remain compatible with current frame.
|
||||
* Updates a selected set of compression parameters, remaining compatible with currently active frame.
|
||||
* New parameters will be applied to next compression job. */
|
||||
void ZSTDMT_updateCParams_whileCompressing(ZSTDMT_CCtx* mtctx, const ZSTD_CCtx_params* cctxParams)
|
||||
{
|
||||
@ -1076,26 +1073,33 @@ void ZSTDMT_updateCParams_whileCompressing(ZSTDMT_CCtx* mtctx, const ZSTD_CCtx_p
|
||||
/* ZSTDMT_getFrameProgression():
|
||||
* tells how much data has been consumed (input) and produced (output) for current frame.
|
||||
* able to count progression inside worker threads.
|
||||
* Note : mutex will be acquired during statistics collection. */
|
||||
* Note : mutex will be acquired during statistics collection inside workers. */
|
||||
ZSTD_frameProgression ZSTDMT_getFrameProgression(ZSTDMT_CCtx* mtctx)
|
||||
{
|
||||
ZSTD_frameProgression fps;
|
||||
DEBUGLOG(6, "ZSTDMT_getFrameProgression");
|
||||
DEBUGLOG(5, "ZSTDMT_getFrameProgression");
|
||||
fps.ingested = mtctx->consumed + mtctx->inBuff.filled;
|
||||
fps.consumed = mtctx->consumed;
|
||||
fps.produced = mtctx->produced;
|
||||
fps.produced = fps.flushed = mtctx->produced;
|
||||
fps.currentJobID = mtctx->nextJobID;
|
||||
fps.nbActiveWorkers = 0;
|
||||
{ unsigned jobNb;
|
||||
unsigned lastJobNb = mtctx->nextJobID + mtctx->jobReady; assert(mtctx->jobReady <= 1);
|
||||
DEBUGLOG(6, "ZSTDMT_getFrameProgression: jobs: from %u to <%u (jobReady:%u)",
|
||||
mtctx->doneJobID, lastJobNb, mtctx->jobReady)
|
||||
for (jobNb = mtctx->doneJobID ; jobNb < lastJobNb ; jobNb++) {
|
||||
unsigned const wJobID = jobNb & mtctx->jobIDMask;
|
||||
ZSTD_pthread_mutex_lock(&mtctx->jobs[wJobID].job_mutex);
|
||||
{ size_t const cResult = mtctx->jobs[wJobID].cSize;
|
||||
ZSTDMT_jobDescription* jobPtr = &mtctx->jobs[wJobID];
|
||||
ZSTD_pthread_mutex_lock(&jobPtr->job_mutex);
|
||||
{ size_t const cResult = jobPtr->cSize;
|
||||
size_t const produced = ZSTD_isError(cResult) ? 0 : cResult;
|
||||
fps.ingested += mtctx->jobs[wJobID].src.size;
|
||||
fps.consumed += mtctx->jobs[wJobID].consumed;
|
||||
size_t const flushed = ZSTD_isError(cResult) ? 0 : jobPtr->dstFlushed;
|
||||
assert(flushed <= produced);
|
||||
fps.ingested += jobPtr->src.size;
|
||||
fps.consumed += jobPtr->consumed;
|
||||
fps.produced += produced;
|
||||
fps.flushed += flushed;
|
||||
fps.nbActiveWorkers += (jobPtr->consumed < jobPtr->src.size);
|
||||
}
|
||||
ZSTD_pthread_mutex_unlock(&mtctx->jobs[wJobID].job_mutex);
|
||||
}
|
||||
@ -1104,6 +1108,34 @@ ZSTD_frameProgression ZSTDMT_getFrameProgression(ZSTDMT_CCtx* mtctx)
|
||||
}
|
||||
|
||||
|
||||
size_t ZSTDMT_toFlushNow(ZSTDMT_CCtx* mtctx)
|
||||
{
|
||||
size_t toFlush;
|
||||
unsigned const jobID = mtctx->doneJobID;
|
||||
assert(jobID <= mtctx->nextJobID);
|
||||
if (jobID == mtctx->nextJobID) return 0; /* no active job => nothing to flush */
|
||||
|
||||
/* look into oldest non-fully-flushed job */
|
||||
{ unsigned const wJobID = jobID & mtctx->jobIDMask;
|
||||
ZSTDMT_jobDescription* const jobPtr = &mtctx->jobs[wJobID];
|
||||
ZSTD_pthread_mutex_lock(&jobPtr->job_mutex);
|
||||
{ size_t const cResult = jobPtr->cSize;
|
||||
size_t const produced = ZSTD_isError(cResult) ? 0 : cResult;
|
||||
size_t const flushed = ZSTD_isError(cResult) ? 0 : jobPtr->dstFlushed;
|
||||
assert(flushed <= produced);
|
||||
toFlush = produced - flushed;
|
||||
if (toFlush==0 && (jobPtr->consumed >= jobPtr->src.size)) {
|
||||
/* doneJobID is not-fully-flushed, but toFlush==0 : doneJobID should be compressing some more data */
|
||||
assert(jobPtr->consumed < jobPtr->src.size);
|
||||
}
|
||||
}
|
||||
ZSTD_pthread_mutex_unlock(&mtctx->jobs[wJobID].job_mutex);
|
||||
}
|
||||
|
||||
return toFlush;
|
||||
}
|
||||
|
||||
|
||||
/* ------------------------------------------ */
|
||||
/* ===== Multi-threaded compression ===== */
|
||||
/* ------------------------------------------ */
|
||||
@ -1498,7 +1530,7 @@ static size_t ZSTDMT_createCompressionJob(ZSTDMT_CCtx* mtctx, size_t srcSize, ZS
|
||||
mtctx->jobs[jobID].jobID = mtctx->nextJobID;
|
||||
mtctx->jobs[jobID].firstJob = (mtctx->nextJobID==0);
|
||||
mtctx->jobs[jobID].lastJob = endFrame;
|
||||
mtctx->jobs[jobID].frameChecksumNeeded = endFrame && (mtctx->nextJobID>0) && mtctx->params.fParams.checksumFlag;
|
||||
mtctx->jobs[jobID].frameChecksumNeeded = mtctx->params.fParams.checksumFlag && endFrame && (mtctx->nextJobID>0);
|
||||
mtctx->jobs[jobID].dstFlushed = 0;
|
||||
|
||||
/* Update the round buffer pos and clear the input buffer to be reset */
|
||||
@ -1576,7 +1608,7 @@ static size_t ZSTDMT_flushProduced(ZSTDMT_CCtx* mtctx, ZSTD_outBuffer* output, u
|
||||
/* try to flush something */
|
||||
{ size_t cSize = mtctx->jobs[wJobID].cSize; /* shared */
|
||||
size_t const srcConsumed = mtctx->jobs[wJobID].consumed; /* shared */
|
||||
size_t const srcSize = mtctx->jobs[wJobID].src.size; /* read-only, could be done after mutex lock, but no-declaration-after-statement */
|
||||
size_t const srcSize = mtctx->jobs[wJobID].src.size; /* read-only, could be done after mutex lock, but no-declaration-after-statement */
|
||||
ZSTD_pthread_mutex_unlock(&mtctx->jobs[wJobID].job_mutex);
|
||||
if (ZSTD_isError(cSize)) {
|
||||
DEBUGLOG(5, "ZSTDMT_flushProduced: job %u : compression error detected : %s",
|
||||
@ -1615,6 +1647,7 @@ static size_t ZSTDMT_flushProduced(ZSTDMT_CCtx* mtctx, ZSTD_outBuffer* output, u
|
||||
DEBUGLOG(5, "Job %u completed (%u bytes), moving to next one",
|
||||
mtctx->doneJobID, (U32)mtctx->jobs[wJobID].dstFlushed);
|
||||
ZSTDMT_releaseBuffer(mtctx->bufPool, mtctx->jobs[wJobID].dstBuff);
|
||||
DEBUGLOG(5, "dstBuffer released");
|
||||
mtctx->jobs[wJobID].dstBuff = g_nullBuffer;
|
||||
mtctx->jobs[wJobID].cSize = 0; /* ensure this job slot is considered "not started" in future check */
|
||||
mtctx->consumed += srcSize;
|
||||
@ -1691,6 +1724,7 @@ static int ZSTDMT_doesOverlapWindow(buffer_t buffer, ZSTD_window_t window)
|
||||
range_t extDict;
|
||||
range_t prefix;
|
||||
|
||||
DEBUGLOG(5, "ZSTDMT_doesOverlapWindow");
|
||||
extDict.start = window.dictBase + window.lowLimit;
|
||||
extDict.size = window.dictLimit - window.lowLimit;
|
||||
|
||||
@ -1711,12 +1745,13 @@ static void ZSTDMT_waitForLdmComplete(ZSTDMT_CCtx* mtctx, buffer_t buffer)
|
||||
{
|
||||
if (mtctx->params.ldmParams.enableLdm) {
|
||||
ZSTD_pthread_mutex_t* mutex = &mtctx->serial.ldmWindowMutex;
|
||||
DEBUGLOG(5, "ZSTDMT_waitForLdmComplete");
|
||||
DEBUGLOG(5, "source [0x%zx, 0x%zx)",
|
||||
(size_t)buffer.start,
|
||||
(size_t)buffer.start + buffer.capacity);
|
||||
ZSTD_PTHREAD_MUTEX_LOCK(mutex);
|
||||
while (ZSTDMT_doesOverlapWindow(buffer, mtctx->serial.ldmWindow)) {
|
||||
DEBUGLOG(6, "Waiting for LDM to finish...");
|
||||
DEBUGLOG(5, "Waiting for LDM to finish...");
|
||||
ZSTD_pthread_cond_wait(&mtctx->serial.ldmWindowCond, mutex);
|
||||
}
|
||||
DEBUGLOG(6, "Done waiting for LDM to finish");
|
||||
@ -1736,6 +1771,7 @@ static int ZSTDMT_tryGetInputRange(ZSTDMT_CCtx* mtctx)
|
||||
size_t const target = mtctx->targetSectionSize;
|
||||
buffer_t buffer;
|
||||
|
||||
DEBUGLOG(5, "ZSTDMT_tryGetInputRange");
|
||||
assert(mtctx->inBuff.buffer.start == NULL);
|
||||
assert(mtctx->roundBuff.capacity >= target);
|
||||
|
||||
@ -1749,7 +1785,7 @@ static int ZSTDMT_tryGetInputRange(ZSTDMT_CCtx* mtctx)
|
||||
buffer.start = start;
|
||||
buffer.capacity = prefixSize;
|
||||
if (ZSTDMT_isOverlapped(buffer, inUse)) {
|
||||
DEBUGLOG(6, "Waiting for buffer...");
|
||||
DEBUGLOG(5, "Waiting for buffer...");
|
||||
return 0;
|
||||
}
|
||||
ZSTDMT_waitForLdmComplete(mtctx, buffer);
|
||||
@ -1761,7 +1797,7 @@ static int ZSTDMT_tryGetInputRange(ZSTDMT_CCtx* mtctx)
|
||||
buffer.capacity = target;
|
||||
|
||||
if (ZSTDMT_isOverlapped(buffer, inUse)) {
|
||||
DEBUGLOG(6, "Waiting for buffer...");
|
||||
DEBUGLOG(5, "Waiting for buffer...");
|
||||
return 0;
|
||||
}
|
||||
assert(!ZSTDMT_isOverlapped(buffer, mtctx->inBuff.prefix));
|
||||
@ -1834,8 +1870,10 @@ size_t ZSTDMT_compressStream_generic(ZSTDMT_CCtx* mtctx,
|
||||
/* It is only possible for this operation to fail if there are
|
||||
* still compression jobs ongoing.
|
||||
*/
|
||||
DEBUGLOG(5, "ZSTDMT_tryGetInputRange failed");
|
||||
assert(mtctx->doneJobID != mtctx->nextJobID);
|
||||
}
|
||||
} else
|
||||
DEBUGLOG(5, "ZSTDMT_tryGetInputRange completed successfully : mtctx->inBuff.buffer.start = %p", mtctx->inBuff.buffer.start);
|
||||
}
|
||||
if (mtctx->inBuff.buffer.start != NULL) {
|
||||
size_t const toLoad = MIN(input->size - input->pos, mtctx->targetSectionSize - mtctx->inBuff.filled);
|
||||
@ -1863,6 +1901,7 @@ size_t ZSTDMT_compressStream_generic(ZSTDMT_CCtx* mtctx,
|
||||
/* check for potential compressed data ready to be flushed */
|
||||
{ size_t const remainingToFlush = ZSTDMT_flushProduced(mtctx, output, !forwardInputProgress, endOp); /* block if there was no forward input progress */
|
||||
if (input->pos < input->size) return MAX(remainingToFlush, 1); /* input not consumed : do not end flush yet */
|
||||
DEBUGLOG(5, "end of ZSTDMT_compressStream_generic: remainingToFlush = %u", (U32)remainingToFlush);
|
||||
return remainingToFlush;
|
||||
}
|
||||
}
|
||||
|
@ -119,11 +119,21 @@ ZSTDLIB_API size_t ZSTDMT_compressStream_generic(ZSTDMT_CCtx* mtctx,
|
||||
* === Not exposed in libzstd. Never invoke directly ===
|
||||
* ======================================================== */
|
||||
|
||||
/*! ZSTDMT_toFlushNow()
|
||||
* Tell how many bytes are ready to be flushed immediately.
|
||||
* Probe the oldest active job (not yet entirely flushed) and check its output buffer.
|
||||
* If return 0, it means there is no active job,
|
||||
* or, it means oldest job is still active, but everything produced has been flushed so far,
|
||||
* therefore flushing is limited by speed of oldest job. */
|
||||
size_t ZSTDMT_toFlushNow(ZSTDMT_CCtx* mtctx);
|
||||
|
||||
/*! ZSTDMT_CCtxParam_setMTCtxParameter()
|
||||
* like ZSTDMT_setMTCtxParameter(), but into a ZSTD_CCtx_Params */
|
||||
size_t ZSTDMT_CCtxParam_setMTCtxParameter(ZSTD_CCtx_params* params, ZSTDMT_parameter parameter, unsigned value);
|
||||
|
||||
/* ZSTDMT_CCtxParam_setNbWorkers()
|
||||
* Set nbWorkers, and clamp it.
|
||||
* Also reset jobSize and overlapLog */
|
||||
/*! ZSTDMT_CCtxParam_setNbWorkers()
|
||||
* Set nbWorkers, and clamp it.
|
||||
* Also reset jobSize and overlapLog */
|
||||
size_t ZSTDMT_CCtxParam_setNbWorkers(ZSTD_CCtx_params* params, unsigned nbWorkers);
|
||||
|
||||
/*! ZSTDMT_updateCParams_whileCompressing() :
|
||||
@ -131,9 +141,9 @@ size_t ZSTDMT_CCtxParam_setNbWorkers(ZSTD_CCtx_params* params, unsigned nbWorker
|
||||
* New parameters will be applied to next compression job. */
|
||||
void ZSTDMT_updateCParams_whileCompressing(ZSTDMT_CCtx* mtctx, const ZSTD_CCtx_params* cctxParams);
|
||||
|
||||
/* ZSTDMT_getFrameProgression():
|
||||
* tells how much data has been consumed (input) and produced (output) for current frame.
|
||||
* able to count progression inside worker threads.
|
||||
/*! ZSTDMT_getFrameProgression():
|
||||
* tells how much data has been consumed (input) and produced (output) for current frame.
|
||||
* able to count progression inside worker threads.
|
||||
*/
|
||||
ZSTD_frameProgression ZSTDMT_getFrameProgression(ZSTDMT_CCtx* mtctx);
|
||||
|
||||
|
37
lib/zstd.h
37
lib/zstd.h
@ -746,29 +746,48 @@ ZSTDLIB_API size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, const
|
||||
|
||||
/*! ZSTD_resetCStream() :
|
||||
* start a new compression job, using same parameters from previous job.
|
||||
* This is typically useful to skip dictionary loading stage, since it will re-use it in-place..
|
||||
* This is typically useful to skip dictionary loading stage, since it will re-use it in-place.
|
||||
* Note that zcs must be init at least once before using ZSTD_resetCStream().
|
||||
* If pledgedSrcSize is not known at reset time, use macro ZSTD_CONTENTSIZE_UNKNOWN.
|
||||
* If pledgedSrcSize > 0, its value must be correct, as it will be written in header, and controlled at the end.
|
||||
* For the time being, pledgedSrcSize==0 is interpreted as "srcSize unknown" for compatibility with older programs,
|
||||
* but it will change to mean "empty" in future version, so use macro ZSTD_CONTENTSIZE_UNKNOWN instead.
|
||||
* @return : 0, or an error code (which can be tested using ZSTD_isError()) */
|
||||
* @return : 0, or an error code (which can be tested using ZSTD_isError())
|
||||
*/
|
||||
ZSTDLIB_API size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize);
|
||||
|
||||
|
||||
typedef struct {
|
||||
unsigned long long ingested;
|
||||
unsigned long long consumed;
|
||||
unsigned long long produced;
|
||||
unsigned long long ingested; /* nb input bytes read and buffered */
|
||||
unsigned long long consumed; /* nb input bytes actually compressed */
|
||||
unsigned long long produced; /* nb of compressed bytes generated and buffered */
|
||||
unsigned long long flushed; /* nb of compressed bytes flushed : not provided; can be tracked from caller side */
|
||||
unsigned currentJobID; /* MT only : latest started job nb */
|
||||
unsigned nbActiveWorkers; /* MT only : nb of workers actively compressing at probe time */
|
||||
} ZSTD_frameProgression;
|
||||
|
||||
/* ZSTD_getFrameProgression():
|
||||
/* ZSTD_getFrameProgression() :
|
||||
* tells how much data has been ingested (read from input)
|
||||
* consumed (input actually compressed) and produced (output) for current frame.
|
||||
* Therefore, (ingested - consumed) is amount of input data buffered internally, not yet compressed.
|
||||
* Can report progression inside worker threads (multi-threading and non-blocking mode).
|
||||
* Note : (ingested - consumed) is amount of input data buffered internally, not yet compressed.
|
||||
* Aggregates progression inside active worker threads.
|
||||
*/
|
||||
ZSTD_frameProgression ZSTD_getFrameProgression(const ZSTD_CCtx* cctx);
|
||||
ZSTDLIB_API ZSTD_frameProgression ZSTD_getFrameProgression(const ZSTD_CCtx* cctx);
|
||||
|
||||
/*! ZSTD_toFlushNow() :
|
||||
* Tell how many bytes are ready to be flushed immediately.
|
||||
* Useful for multithreading scenarios (nbWorkers >= 1).
|
||||
* Probe the oldest active job, defined as oldest job not yet entirely flushed,
|
||||
* and check its output buffer.
|
||||
* @return : amount of data stored in oldest job and ready to be flushed immediately.
|
||||
* if @return == 0, it means either :
|
||||
* + there is no active job (could be checked with ZSTD_frameProgression()), or
|
||||
* + oldest job is still actively compressing data,
|
||||
* but everything it has produced has also been flushed so far,
|
||||
* therefore flushing speed is currently limited by production speed of oldest job
|
||||
* irrespective of the speed of concurrent newer jobs.
|
||||
*/
|
||||
ZSTDLIB_API size_t ZSTD_toFlushNow(ZSTD_CCtx* cctx);
|
||||
|
||||
|
||||
|
||||
|
@ -78,6 +78,7 @@
|
||||
#define MB *(1<<20)
|
||||
#define GB *(1U<<30)
|
||||
|
||||
#define ADAPT_WINDOWLOG_DEFAULT 23 /* 8 MB */
|
||||
#define DICTSIZE_MAX (32 MB) /* protection against large input (attack scenario) */
|
||||
|
||||
#define FNSPACE 30
|
||||
@ -285,6 +286,26 @@ void FIO_setOverlapLog(unsigned overlapLog){
|
||||
DISPLAYLEVEL(2, "Setting overlapLog is useless in single-thread mode \n");
|
||||
g_overlapLog = overlapLog;
|
||||
}
|
||||
static U32 g_adaptiveMode = 0;
|
||||
void FIO_setAdaptiveMode(unsigned adapt) {
|
||||
if ((adapt>0) && (g_nbWorkers==0))
|
||||
EXM_THROW(1, "Adaptive mode is not compatible with single thread mode \n");
|
||||
g_adaptiveMode = adapt;
|
||||
}
|
||||
static int g_minAdaptLevel = -50; /* initializing this value requires a constant, so ZSTD_minCLevel() doesn't work */
|
||||
void FIO_setAdaptMin(int minCLevel)
|
||||
{
|
||||
#ifndef ZSTD_NOCOMPRESS
|
||||
assert(minCLevel >= ZSTD_minCLevel());
|
||||
#endif
|
||||
g_minAdaptLevel = minCLevel;
|
||||
}
|
||||
static int g_maxAdaptLevel = 22; /* initializing this value requires a constant, so ZSTD_maxCLevel() doesn't work */
|
||||
void FIO_setAdaptMax(int maxCLevel)
|
||||
{
|
||||
g_maxAdaptLevel = maxCLevel;
|
||||
}
|
||||
|
||||
static U32 g_ldmFlag = 0;
|
||||
void FIO_setLdmFlag(unsigned ldmFlag) {
|
||||
g_ldmFlag = (ldmFlag>0);
|
||||
@ -463,7 +484,7 @@ typedef struct {
|
||||
|
||||
static cRess_t FIO_createCResources(const char* dictFileName, int cLevel,
|
||||
U64 srcSize,
|
||||
ZSTD_compressionParameters* comprParams) {
|
||||
ZSTD_compressionParameters comprParams) {
|
||||
cRess_t ress;
|
||||
memset(&ress, 0, sizeof(ress));
|
||||
|
||||
@ -484,6 +505,9 @@ static cRess_t FIO_createCResources(const char* dictFileName, int cLevel,
|
||||
if (dictFileName && (dictBuffer==NULL))
|
||||
EXM_THROW(32, "allocation error : can't create dictBuffer");
|
||||
|
||||
if (g_adaptiveMode && !g_ldmFlag && !comprParams.windowLog)
|
||||
comprParams.windowLog = ADAPT_WINDOWLOG_DEFAULT;
|
||||
|
||||
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_contentSizeFlag, 1) ); /* always enable content size when available (note: supposed to be default) */
|
||||
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_dictIDFlag, g_dictIDFlag) );
|
||||
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_checksumFlag, g_checksumFlag) );
|
||||
@ -500,13 +524,13 @@ static cRess_t FIO_createCResources(const char* dictFileName, int cLevel,
|
||||
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_ldmHashEveryLog, g_ldmHashEveryLog) );
|
||||
}
|
||||
/* compression parameters */
|
||||
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_windowLog, comprParams->windowLog) );
|
||||
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_chainLog, comprParams->chainLog) );
|
||||
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_hashLog, comprParams->hashLog) );
|
||||
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_searchLog, comprParams->searchLog) );
|
||||
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_minMatch, comprParams->searchLength) );
|
||||
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_targetLength, comprParams->targetLength) );
|
||||
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_compressionStrategy, (U32)comprParams->strategy) );
|
||||
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_windowLog, comprParams.windowLog) );
|
||||
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_chainLog, comprParams.chainLog) );
|
||||
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_hashLog, comprParams.hashLog) );
|
||||
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_searchLog, comprParams.searchLog) );
|
||||
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_minMatch, comprParams.searchLength) );
|
||||
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_targetLength, comprParams.targetLength) );
|
||||
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_compressionStrategy, (U32)comprParams.strategy) );
|
||||
/* multi-threading */
|
||||
#ifdef ZSTD_MULTITHREAD
|
||||
DISPLAYLEVEL(5,"set nb workers = %u \n", g_nbWorkers);
|
||||
@ -539,7 +563,8 @@ static void FIO_freeCResources(cRess_t ress)
|
||||
|
||||
|
||||
#ifdef ZSTD_GZCOMPRESS
|
||||
static unsigned long long FIO_compressGzFrame(cRess_t* ress,
|
||||
static unsigned long long
|
||||
FIO_compressGzFrame(cRess_t* ress,
|
||||
const char* srcFileName, U64 const srcFileSize,
|
||||
int compressionLevel, U64* readsize)
|
||||
{
|
||||
@ -621,9 +646,10 @@ static unsigned long long FIO_compressGzFrame(cRess_t* ress,
|
||||
|
||||
|
||||
#ifdef ZSTD_LZMACOMPRESS
|
||||
static unsigned long long FIO_compressLzmaFrame(cRess_t* ress,
|
||||
const char* srcFileName, U64 const srcFileSize,
|
||||
int compressionLevel, U64* readsize, int plain_lzma)
|
||||
static unsigned long long
|
||||
FIO_compressLzmaFrame(cRess_t* ress,
|
||||
const char* srcFileName, U64 const srcFileSize,
|
||||
int compressionLevel, U64* readsize, int plain_lzma)
|
||||
{
|
||||
unsigned long long inFileSize = 0, outFileSize = 0;
|
||||
lzma_stream strm = LZMA_STREAM_INIT;
|
||||
@ -696,9 +722,10 @@ static unsigned long long FIO_compressLzmaFrame(cRess_t* ress,
|
||||
#define LZ4F_max64KB max64KB
|
||||
#endif
|
||||
static int FIO_LZ4_GetBlockSize_FromBlockId (int id) { return (1 << (8 + (2 * id))); }
|
||||
static unsigned long long FIO_compressLz4Frame(cRess_t* ress,
|
||||
const char* srcFileName, U64 const srcFileSize,
|
||||
int compressionLevel, U64* readsize)
|
||||
static unsigned long long
|
||||
FIO_compressLz4Frame(cRess_t* ress,
|
||||
const char* srcFileName, U64 const srcFileSize,
|
||||
int compressionLevel, U64* readsize)
|
||||
{
|
||||
const size_t blockSize = FIO_LZ4_GetBlockSize_FromBlockId(LZ4F_max64KB);
|
||||
unsigned long long inFileSize = 0, outFileSize = 0;
|
||||
@ -796,17 +823,28 @@ FIO_compressZstdFrame(const cRess_t* ressPtr,
|
||||
FILE* const dstFile = ress.dstFile;
|
||||
U64 compressedfilesize = 0;
|
||||
ZSTD_EndDirective directive = ZSTD_e_continue;
|
||||
|
||||
/* stats */
|
||||
ZSTD_frameProgression previous_zfp_update = { 0, 0, 0, 0, 0, 0 };
|
||||
ZSTD_frameProgression previous_zfp_correction = { 0, 0, 0, 0, 0, 0 };
|
||||
typedef enum { noChange, slower, faster } speedChange_e;
|
||||
speedChange_e speedChange = noChange;
|
||||
unsigned flushWaiting = 0;
|
||||
unsigned inputPresented = 0;
|
||||
unsigned inputBlocked = 0;
|
||||
unsigned lastJobID = 0;
|
||||
|
||||
DISPLAYLEVEL(6, "compression using zstd format \n");
|
||||
|
||||
/* init */
|
||||
if (fileSize != UTIL_FILESIZE_UNKNOWN) {
|
||||
CHECK(ZSTD_CCtx_setPledgedSrcSize(ress.cctx, fileSize));
|
||||
}
|
||||
(void)compressionLevel; (void)srcFileName;
|
||||
(void)srcFileName;
|
||||
|
||||
/* Main compression loop */
|
||||
do {
|
||||
size_t result;
|
||||
size_t stillToFlush;
|
||||
/* Fill input Buffer */
|
||||
size_t const inSize = fread(ress.srcBuffer, (size_t)1, ress.srcBufferSize, srcFile);
|
||||
ZSTD_inBuffer inBuff = { ress.srcBuffer, inSize, 0 };
|
||||
@ -816,41 +854,139 @@ FIO_compressZstdFrame(const cRess_t* ressPtr,
|
||||
if ((inSize == 0) || (*readsize == fileSize))
|
||||
directive = ZSTD_e_end;
|
||||
|
||||
result = 1;
|
||||
stillToFlush = 1;
|
||||
while ((inBuff.pos != inBuff.size) /* input buffer must be entirely ingested */
|
||||
|| (directive == ZSTD_e_end && result != 0) ) {
|
||||
|| (directive == ZSTD_e_end && stillToFlush != 0) ) {
|
||||
|
||||
size_t const oldIPos = inBuff.pos;
|
||||
ZSTD_outBuffer outBuff = { ress.dstBuffer, ress.dstBufferSize, 0 };
|
||||
CHECK_V(result, ZSTD_compress_generic(ress.cctx, &outBuff, &inBuff, directive));
|
||||
size_t const toFlushNow = ZSTD_toFlushNow(ress.cctx);
|
||||
CHECK_V(stillToFlush, ZSTD_compress_generic(ress.cctx, &outBuff, &inBuff, directive));
|
||||
|
||||
/* count stats */
|
||||
inputPresented++;
|
||||
if (oldIPos == inBuff.pos) inputBlocked++; /* input buffer is full and can't take any more : input speed is faster than consumption rate */
|
||||
if (!toFlushNow) flushWaiting = 1;
|
||||
|
||||
/* Write compressed stream */
|
||||
DISPLAYLEVEL(6, "ZSTD_compress_generic(end:%u) => intput pos(%u)<=(%u)size ; output generated %u bytes \n",
|
||||
DISPLAYLEVEL(6, "ZSTD_compress_generic(end:%u) => input pos(%u)<=(%u)size ; output generated %u bytes \n",
|
||||
(U32)directive, (U32)inBuff.pos, (U32)inBuff.size, (U32)outBuff.pos);
|
||||
if (outBuff.pos) {
|
||||
size_t const sizeCheck = fwrite(ress.dstBuffer, 1, outBuff.pos, dstFile);
|
||||
if (sizeCheck!=outBuff.pos)
|
||||
if (sizeCheck != outBuff.pos)
|
||||
EXM_THROW(25, "Write error : cannot write compressed block");
|
||||
compressedfilesize += outBuff.pos;
|
||||
}
|
||||
|
||||
/* display notification; and adapt compression level */
|
||||
if (READY_FOR_UPDATE()) {
|
||||
ZSTD_frameProgression const zfp = ZSTD_getFrameProgression(ress.cctx);
|
||||
double const cShare = (double)zfp.produced / (zfp.consumed + !zfp.consumed/*avoid div0*/) * 100;
|
||||
|
||||
/* display progress notifications */
|
||||
if (g_displayLevel >= 3) {
|
||||
DISPLAYUPDATE(3, "\r(L%i) Buffered :%4u MB - Consumed :%4u MB - Compressed :%4u MB => %.2f%%",
|
||||
DISPLAYUPDATE(3, "\r(L%i) Buffered :%4u MB - Consumed :%4u MB - Compressed :%4u MB => %.2f%% ",
|
||||
compressionLevel,
|
||||
(U32)((zfp.ingested - zfp.consumed) >> 20),
|
||||
(U32)(zfp.consumed >> 20),
|
||||
(U32)(zfp.produced >> 20),
|
||||
cShare );
|
||||
} else {
|
||||
/* g_displayLevel <= 2; only display notifications if == 2; */
|
||||
} else { /* summarized notifications if == 2; */
|
||||
DISPLAYLEVEL(2, "\rRead : %u ", (U32)(zfp.consumed >> 20));
|
||||
if (fileSize != UTIL_FILESIZE_UNKNOWN)
|
||||
DISPLAYLEVEL(2, "/ %u ", (U32)(fileSize >> 20));
|
||||
DISPLAYLEVEL(2, "MB ==> %2.f%% ", cShare);
|
||||
DELAY_NEXT_UPDATE();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* adaptive mode : statistics measurement and speed correction */
|
||||
if (g_adaptiveMode) {
|
||||
|
||||
/* check output speed */
|
||||
if (zfp.currentJobID > 1) { /* only possible if nbWorkers >= 1 */
|
||||
|
||||
unsigned long long newlyProduced = zfp.produced - previous_zfp_update.produced;
|
||||
unsigned long long newlyFlushed = zfp.flushed - previous_zfp_update.flushed;
|
||||
assert(zfp.produced >= previous_zfp_update.produced);
|
||||
assert(g_nbWorkers >= 1);
|
||||
|
||||
/* test if compression is blocked
|
||||
* either because output is slow and all buffers are full
|
||||
* or because input is slow and no job can start while waiting for at least one buffer to be filled.
|
||||
* note : excluse starting part, since currentJobID > 1 */
|
||||
if ( (zfp.consumed == previous_zfp_update.consumed) /* no data compressed : no data available, or no more buffer to compress to, OR compression is really slow (compression of a single block is slower than update rate)*/
|
||||
&& (zfp.nbActiveWorkers == 0) /* confirmed : no compression ongoing */
|
||||
) {
|
||||
DISPLAYLEVEL(6, "all buffers full : compression stopped => slow down \n")
|
||||
speedChange = slower;
|
||||
}
|
||||
|
||||
previous_zfp_update = zfp;
|
||||
|
||||
if ( (newlyProduced > (newlyFlushed * 9 / 8)) /* compression produces more data than output can flush (though production can be spiky, due to work unit : (N==4)*block sizes) */
|
||||
&& (flushWaiting == 0) /* flush speed was never slowed by lack of production, so it's operating at max capacity */
|
||||
) {
|
||||
DISPLAYLEVEL(6, "compression faster than flush (%llu > %llu), and flushed was never slowed down by lack of production => slow down \n", newlyProduced, newlyFlushed);
|
||||
speedChange = slower;
|
||||
}
|
||||
flushWaiting = 0;
|
||||
}
|
||||
|
||||
/* course correct only if there is at least one new job completed */
|
||||
if (zfp.currentJobID > lastJobID) {
|
||||
DISPLAYLEVEL(6, "compression level adaptation check \n")
|
||||
|
||||
/* check input speed */
|
||||
if (zfp.currentJobID > g_nbWorkers+1) { /* warm up period, to fill all workers */
|
||||
if (inputBlocked <= 0) {
|
||||
DISPLAYLEVEL(6, "input is never blocked => input is slower than ingestion \n");
|
||||
speedChange = slower;
|
||||
} else if (speedChange == noChange) {
|
||||
unsigned long long newlyIngested = zfp.ingested - previous_zfp_correction.ingested;
|
||||
unsigned long long newlyConsumed = zfp.consumed - previous_zfp_correction.consumed;
|
||||
unsigned long long newlyProduced = zfp.produced - previous_zfp_correction.produced;
|
||||
unsigned long long newlyFlushed = zfp.flushed - previous_zfp_correction.flushed;
|
||||
previous_zfp_correction = zfp;
|
||||
assert(inputPresented > 0);
|
||||
DISPLAYLEVEL(6, "input blocked %u/%u(%.2f) - ingested:%u vs %u:consumed - flushed:%u vs %u:produced \n",
|
||||
inputBlocked, inputPresented, (double)inputBlocked/inputPresented*100,
|
||||
(U32)newlyIngested, (U32)newlyConsumed,
|
||||
(U32)newlyFlushed, (U32)newlyProduced);
|
||||
if ( (inputBlocked > inputPresented / 8) /* input is waiting often, because input buffers is full : compression or output too slow */
|
||||
&& (newlyFlushed * 33 / 32 > newlyProduced) /* flush everything that is produced */
|
||||
&& (newlyIngested * 33 / 32 > newlyConsumed) /* input speed as fast or faster than compression speed */
|
||||
) {
|
||||
DISPLAYLEVEL(6, "recommend faster as in(%llu) >= (%llu)comp(%llu) <= out(%llu) \n",
|
||||
newlyIngested, newlyConsumed, newlyProduced, newlyFlushed);
|
||||
speedChange = faster;
|
||||
}
|
||||
}
|
||||
inputBlocked = 0;
|
||||
inputPresented = 0;
|
||||
}
|
||||
|
||||
if (speedChange == slower) {
|
||||
DISPLAYLEVEL(6, "slower speed , higher compression \n")
|
||||
compressionLevel ++;
|
||||
if (compressionLevel > ZSTD_maxCLevel()) compressionLevel = ZSTD_maxCLevel();
|
||||
if (compressionLevel > g_maxAdaptLevel) compressionLevel = g_maxAdaptLevel;
|
||||
compressionLevel += (compressionLevel == 0); /* skip 0 */
|
||||
ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_compressionLevel, (unsigned)compressionLevel);
|
||||
}
|
||||
if (speedChange == faster) {
|
||||
DISPLAYLEVEL(6, "faster speed , lighter compression \n")
|
||||
compressionLevel --;
|
||||
if (compressionLevel < g_minAdaptLevel) compressionLevel = g_minAdaptLevel;
|
||||
compressionLevel -= (compressionLevel == 0); /* skip 0 */
|
||||
ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_compressionLevel, (unsigned)compressionLevel);
|
||||
}
|
||||
speedChange = noChange;
|
||||
|
||||
lastJobID = zfp.currentJobID;
|
||||
} /* if (zfp.currentJobID > lastJobID) */
|
||||
} /* if (g_adaptiveMode) */
|
||||
} /* if (READY_FOR_UPDATE()) */
|
||||
} /* while ((inBuff.pos != inBuff.size) */
|
||||
} while (directive != ZSTD_e_end);
|
||||
|
||||
if (ferror(srcFile)) {
|
||||
@ -1010,7 +1146,8 @@ static int FIO_compressFilename_dstFile(cRess_t ress,
|
||||
|
||||
|
||||
int FIO_compressFilename(const char* dstFileName, const char* srcFileName,
|
||||
const char* dictFileName, int compressionLevel, ZSTD_compressionParameters* comprParams)
|
||||
const char* dictFileName, int compressionLevel,
|
||||
ZSTD_compressionParameters comprParams)
|
||||
{
|
||||
clock_t const start = clock();
|
||||
U64 const fileSize = UTIL_getFileSize(srcFileName);
|
||||
@ -1030,7 +1167,7 @@ int FIO_compressFilename(const char* dstFileName, const char* srcFileName,
|
||||
int FIO_compressMultipleFilenames(const char** inFileNamesTable, unsigned nbFiles,
|
||||
const char* outFileName, const char* suffix,
|
||||
const char* dictFileName, int compressionLevel,
|
||||
ZSTD_compressionParameters* comprParams)
|
||||
ZSTD_compressionParameters comprParams)
|
||||
{
|
||||
int missed_files = 0;
|
||||
size_t dfnSize = FNSPACE;
|
||||
|
@ -48,20 +48,23 @@ typedef enum { FIO_zstdCompression, FIO_gzipCompression, FIO_xzCompression, FIO_
|
||||
***************************************/
|
||||
void FIO_setCompressionType(FIO_compressionType_t compressionType);
|
||||
void FIO_overwriteMode(void);
|
||||
void FIO_setNotificationLevel(unsigned level);
|
||||
void FIO_setSparseWrite(unsigned sparse); /**< 0: no sparse; 1: disable on stdout; 2: always enabled */
|
||||
void FIO_setDictIDFlag(unsigned dictIDFlag);
|
||||
void FIO_setChecksumFlag(unsigned checksumFlag);
|
||||
void FIO_setRemoveSrcFile(unsigned flag);
|
||||
void FIO_setMemLimit(unsigned memLimit);
|
||||
void FIO_setNbWorkers(unsigned nbWorkers);
|
||||
void FIO_setAdaptiveMode(unsigned adapt);
|
||||
void FIO_setAdaptMin(int minCLevel);
|
||||
void FIO_setAdaptMax(int maxCLevel);
|
||||
void FIO_setBlockSize(unsigned blockSize);
|
||||
void FIO_setOverlapLog(unsigned overlapLog);
|
||||
void FIO_setChecksumFlag(unsigned checksumFlag);
|
||||
void FIO_setDictIDFlag(unsigned dictIDFlag);
|
||||
void FIO_setLdmBucketSizeLog(unsigned ldmBucketSizeLog);
|
||||
void FIO_setLdmFlag(unsigned ldmFlag);
|
||||
void FIO_setLdmHashEveryLog(unsigned ldmHashEveryLog);
|
||||
void FIO_setLdmHashLog(unsigned ldmHashLog);
|
||||
void FIO_setLdmMinMatch(unsigned ldmMinMatch);
|
||||
void FIO_setLdmBucketSizeLog(unsigned ldmBucketSizeLog);
|
||||
void FIO_setLdmHashEveryLog(unsigned ldmHashEveryLog);
|
||||
void FIO_setMemLimit(unsigned memLimit);
|
||||
void FIO_setNbWorkers(unsigned nbWorkers);
|
||||
void FIO_setNotificationLevel(unsigned level);
|
||||
void FIO_setOverlapLog(unsigned overlapLog);
|
||||
void FIO_setRemoveSrcFile(unsigned flag);
|
||||
void FIO_setSparseWrite(unsigned sparse); /**< 0: no sparse; 1: disable on stdout; 2: always enabled */
|
||||
|
||||
|
||||
/*-*************************************
|
||||
@ -70,7 +73,7 @@ void FIO_setLdmHashEveryLog(unsigned ldmHashEveryLog);
|
||||
/** FIO_compressFilename() :
|
||||
@return : 0 == ok; 1 == pb with src file. */
|
||||
int FIO_compressFilename (const char* outfilename, const char* infilename, const char* dictFileName,
|
||||
int compressionLevel, ZSTD_compressionParameters* comprParams);
|
||||
int compressionLevel, ZSTD_compressionParameters comprParams);
|
||||
|
||||
/** FIO_decompressFilename() :
|
||||
@return : 0 == ok; 1 == pb with src file. */
|
||||
@ -78,6 +81,7 @@ int FIO_decompressFilename (const char* outfilename, const char* infilename, con
|
||||
|
||||
int FIO_listMultipleFiles(unsigned numFiles, const char** filenameTable, int displayLevel);
|
||||
|
||||
|
||||
/*-*************************************
|
||||
* Multiple File functions
|
||||
***************************************/
|
||||
@ -86,7 +90,7 @@ int FIO_listMultipleFiles(unsigned numFiles, const char** filenameTable, int dis
|
||||
int FIO_compressMultipleFilenames(const char** srcNamesTable, unsigned nbFiles,
|
||||
const char* outFileName, const char* suffix,
|
||||
const char* dictFileName, int compressionLevel,
|
||||
ZSTD_compressionParameters* comprParams);
|
||||
ZSTD_compressionParameters comprParams);
|
||||
|
||||
/** FIO_decompressMultipleFilenames() :
|
||||
@return : nb of missing or skipped files */
|
||||
@ -95,9 +99,15 @@ int FIO_decompressMultipleFilenames(const char** srcNamesTable, unsigned nbFiles
|
||||
const char* dictFileName);
|
||||
|
||||
|
||||
/*-*************************************
|
||||
* Advanced stuff (should actually be hosted elsewhere)
|
||||
***************************************/
|
||||
|
||||
/* custom crash signal handler */
|
||||
void FIO_addAbortHandler(void);
|
||||
|
||||
|
||||
|
||||
#if defined (__cplusplus)
|
||||
}
|
||||
#endif
|
||||
|
@ -1,5 +1,5 @@
|
||||
.
|
||||
.TH "ZSTD" "1" "2018-06-27" "zstd 1.3.5" "User Commands"
|
||||
.TH "ZSTD" "1" "September 2018" "zstd 1.3.5" "User Commands"
|
||||
.
|
||||
.SH "NAME"
|
||||
\fBzstd\fR \- zstd, zstdmt, unzstd, zstdcat \- Compress or decompress \.zst files
|
||||
@ -100,6 +100,10 @@ Display information related to a zstd compressed file, such as size, ratio, and
|
||||
\fB#\fR compression level [1\-19] (default: 3)
|
||||
.
|
||||
.TP
|
||||
\fB\-\-fast[=#]\fR
|
||||
switch to ultra\-fast compression levels\. If \fB=#\fR is not present, it defaults to \fB1\fR\. The higher the value, the faster the compression speed, at the cost of some compression ratio\. This setting overwrites compression level if one was set previously\. Similarly, if a compression level is set after \fB\-\-fast\fR, it overrides it\.
|
||||
.
|
||||
.TP
|
||||
\fB\-\-ultra\fR
|
||||
unlocks high compression levels 20+ (maximum 22), using a lot more memory\. Note that decompression will also require more memory when using these levels\.
|
||||
.
|
||||
@ -111,16 +115,16 @@ enables long distance matching with \fB#\fR \fBwindowLog\fR, if not \fB#\fR is n
|
||||
Note: If \fBwindowLog\fR is set to larger than 27, \fB\-\-long=windowLog\fR or \fB\-\-memory=windowSize\fR needs to be passed to the decompressor\.
|
||||
.
|
||||
.TP
|
||||
\fB\-\-fast[=#]\fR
|
||||
switch to ultra\-fast compression levels\. If \fB=#\fR is not present, it defaults to \fB1\fR\. The higher the value, the faster the compression speed, at the cost of some compression ratio\. This setting overwrites compression level if one was set previously\. Similarly, if a compression level is set after \fB\-\-fast\fR, it overrides it\.
|
||||
.
|
||||
.TP
|
||||
\fB\-T#\fR, \fB\-\-threads=#\fR
|
||||
Compress using \fB#\fR working threads (default: 1)\. If \fB#\fR is 0, attempt to detect and use the number of physical CPU cores\. In all cases, the nb of threads is capped to ZSTDMT_NBTHREADS_MAX==200\. This modifier does nothing if \fBzstd\fR is compiled without multithread support\.
|
||||
.
|
||||
.TP
|
||||
\fB\-\-single\-thread\fR
|
||||
Does not spawn a thread for compression, use caller thread instead\. This is the only available mode when multithread support is disabled\. In this mode, compression is serialized with I/O\. (This is different from \fB\-T1\fR, which spawns 1 compression thread in parallel of I/O)\. Single\-thread mode also features lower memory usage\.
|
||||
Does not spawn a thread for compression, use a single thread for both I/O and compression\. In this mode, compression is serialized with I/O, which is slightly slower\. (This is different from \fB\-T1\fR, which spawns 1 compression thread in parallel of I/O)\. This mode is the only one available when multithread support is disabled\. Single\-thread mode features lower memory usage\. Final compressed result is slightly different from \fB\-T1\fR\.
|
||||
.
|
||||
.TP
|
||||
\fB\-\-adapt\fR
|
||||
\fBzstd\fR will dynamically adapt compression level to perceived I/O conditions\. Compression level adaptation can be observed live by using command \fB\-v\fR\. The feature works when combined with multi\-threading and \fB\-\-long\fR mode\. It does not work with \fB\-\-single\-thread\fR\. It sets window size to 8 MB by default (can be changed manually, see \fBwlog\fR)\. Due to the chaotic nature of dynamic adaptation, compressed result is not reproducible\. \fInote\fR : at the time of this writing, \fB\-\-adapt\fR can remain stuck at low speed when combined with multiple worker threads (>=2)\.
|
||||
.
|
||||
.TP
|
||||
\fB\-D file\fR
|
||||
@ -194,7 +198,7 @@ All arguments after \fB\-\-\fR are treated as files
|
||||
Use FILEs as training set to create a dictionary\. The training set should contain a lot of small files (> 100), and weight typically 100x the target dictionary size (for example, 10 MB for a 100 KB dictionary)\.
|
||||
.
|
||||
.IP
|
||||
Supports multithreading if \fBzstd\fR is compiled with threading support\. Additional parameters can be specified with \fB\-\-train\-fastcover\fR\. The legacy dictionary builder can be accessed with \fB\-\-train\-legacy\fR\. The cover dictionary builder can be accessed with \fB\-\-train\-cover\fR\. Equivalent to \fB\-\-train\-fastCover=d=8,steps=4\fR\.
|
||||
Supports multithreading if \fBzstd\fR is compiled with threading support\. Additional parameters can be specified with \fB\-\-train\-fastcover\fR\. The legacy dictionary builder can be accessed with \fB\-\-train\-legacy\fR\. The cover dictionary builder can be accessed with \fB\-\-train\-cover\fR\. Equivalent to \fB\-\-train\-fastcover=d=8,steps=4\fR\.
|
||||
.
|
||||
.TP
|
||||
\fB\-o file\fR
|
||||
@ -218,11 +222,10 @@ A dictionary ID is a locally unique ID that a decoder can use to verify it is us
|
||||
.
|
||||
.TP
|
||||
\fB\-\-train\-cover[=k#,d=#,steps=#,split=#]\fR
|
||||
Select parameters for the default dictionary builder algorithm named cover\. If \fId\fR is not specified, then it tries \fId\fR = 6 and \fId\fR = 8\. If \fIk\fR is not specified, then it tries \fIsteps\fR values in the range [50, 2000]\. If \fIsteps\fR is not specified, then the default value of 40 is used\. If \fIsplit\fR is not specified or \fIsplit\fR <= 0, then the default value of 100 is used\. If \fIsplit\fR is 100, all input samples are used for both training and testing
|
||||
to find optimal _d_ and _k_ to build dictionary.Requires that \fId\fR <= \fIk\fR\.
|
||||
Select parameters for the default dictionary builder algorithm named cover\. If \fId\fR is not specified, then it tries \fId\fR = 6 and \fId\fR = 8\. If \fIk\fR is not specified, then it tries \fIsteps\fR values in the range [50, 2000]\. If \fIsteps\fR is not specified, then the default value of 40 is used\. If \fIsplit\fR is not specified or split <= 0, then the default value of 100 is used\. Requires that \fId\fR <= \fIk\fR\.
|
||||
.
|
||||
.IP
|
||||
Selects segments of size \fIk\fR with highest score to put in the dictionary\. The score of a segment is computed by the sum of the frequencies of all the subsegments of size \fId\fR\. Generally \fId\fR should be in the range [6, 8], occasionally up to 16, but the algorithm will run faster with d <= \fI8\fR\. Good values for \fIk\fR vary widely based on the input data, but a safe range is [2 * \fId\fR, 2000]\. Supports multithreading if \fBzstd\fR is compiled with threading support\.
|
||||
Selects segments of size \fIk\fR with highest score to put in the dictionary\. The score of a segment is computed by the sum of the frequencies of all the subsegments of size \fId\fR\. Generally \fId\fR should be in the range [6, 8], occasionally up to 16, but the algorithm will run faster with d <= \fI8\fR\. Good values for \fIk\fR vary widely based on the input data, but a safe range is [2 * \fId\fR, 2000]\. If \fIsplit\fR is 100, all input samples are used for both training and testing to find optimal \fId\fR and \fIk\fR to build dictionary\. Supports multithreading if \fBzstd\fR is compiled with threading support\.
|
||||
.
|
||||
.IP
|
||||
Examples:
|
||||
@ -239,15 +242,15 @@ Examples:
|
||||
.IP
|
||||
\fBzstd \-\-train\-cover=k=50 FILEs\fR
|
||||
.
|
||||
.IP
|
||||
\fBzstd \-\-train\-cover=k=50,split=60 FILEs\fR
|
||||
.
|
||||
.TP
|
||||
\fB\-\-train\-fastcover[=k#,d=#,f=#,steps=#,split=#,accel=#]\fR
|
||||
Same as cover but with extra parameters \fIf\fR and \fIaccel\fR and different default value of split
|
||||
Same as cover but with extra parameters \fIf\fR and \fIaccel\fR and different default value of split If \fIsplit\fR is not specified, then it tries \fIsplit\fR = 75\. If \fIf\fR is not specified, then it tries \fIf\fR = 20\. Requires that 0 < \fIf\fR < 32\. If \fIaccel\fR is not specified, then it tries \fIaccel\fR = 1\. Requires that 0 < \fIaccel\fR <= 10\. Requires that \fId\fR = 6 or \fId\fR = 8\.
|
||||
.
|
||||
.IP
|
||||
If \fIsplit\fR is not specified, then it tries \fIsplit\fR = 75. If \fIf\fR is not specified, then it tries \fIf\fR = 20. Requires that 0 < \fIf\fR < 32. If \fIaccel\fR is not specified, then it tries \fIaccel\fR = 1. Requires that 0 < \fIaccel\fR <= 10. Requires that \fId\fR = 6 or \fId\fR = 8.
|
||||
.
|
||||
.IP
|
||||
\fIf\fR is log of size of array that keeps track of frequency of subsegments of size \fId\fR. The subsegment is hashed to an index in the range [0,2^\fIf\fR - 1]. It is possible that 2 different subsegments are hashed to the same index, and they are considered as the same subsegment when computing frequency. Using a higher \fIf\fR reduces collision but takes longer.
|
||||
\fIf\fR is log of size of array that keeps track of frequency of subsegments of size \fId\fR\. The subsegment is hashed to an index in the range [0,2^\fIf\fR \- 1]\. It is possible that 2 different subsegments are hashed to the same index, and they are considered as the same subsegment when computing frequency\. Using a higher \fIf\fR reduces collision but takes longer\.
|
||||
.
|
||||
.IP
|
||||
Examples:
|
||||
|
@ -102,6 +102,13 @@ the last one takes effect.
|
||||
|
||||
* `-#`:
|
||||
`#` compression level \[1-19] (default: 3)
|
||||
* `--fast[=#]`:
|
||||
switch to ultra-fast compression levels.
|
||||
If `=#` is not present, it defaults to `1`.
|
||||
The higher the value, the faster the compression speed,
|
||||
at the cost of some compression ratio.
|
||||
This setting overwrites compression level if one was set previously.
|
||||
Similarly, if a compression level is set after `--fast`, it overrides it.
|
||||
* `--ultra`:
|
||||
unlocks high compression levels 20+ (maximum 22), using a lot more memory.
|
||||
Note that decompression will also require more memory when using these levels.
|
||||
@ -115,25 +122,28 @@ the last one takes effect.
|
||||
|
||||
Note: If `windowLog` is set to larger than 27, `--long=windowLog` or
|
||||
`--memory=windowSize` needs to be passed to the decompressor.
|
||||
* `--fast[=#]`:
|
||||
switch to ultra-fast compression levels.
|
||||
If `=#` is not present, it defaults to `1`.
|
||||
The higher the value, the faster the compression speed,
|
||||
at the cost of some compression ratio.
|
||||
This setting overwrites compression level if one was set previously.
|
||||
Similarly, if a compression level is set after `--fast`, it overrides it.
|
||||
|
||||
* `-T#`, `--threads=#`:
|
||||
Compress using `#` working threads (default: 1).
|
||||
If `#` is 0, attempt to detect and use the number of physical CPU cores.
|
||||
In all cases, the nb of threads is capped to ZSTDMT_NBTHREADS_MAX==200.
|
||||
This modifier does nothing if `zstd` is compiled without multithread support.
|
||||
* `--single-thread`:
|
||||
Does not spawn a thread for compression, use caller thread instead.
|
||||
This is the only available mode when multithread support is disabled.
|
||||
In this mode, compression is serialized with I/O.
|
||||
Does not spawn a thread for compression, use a single thread for both I/O and compression.
|
||||
In this mode, compression is serialized with I/O, which is slightly slower.
|
||||
(This is different from `-T1`, which spawns 1 compression thread in parallel of I/O).
|
||||
Single-thread mode also features lower memory usage.
|
||||
This mode is the only one available when multithread support is disabled.
|
||||
Single-thread mode features lower memory usage.
|
||||
Final compressed result is slightly different from `-T1`.
|
||||
* `--adapt[=min=#,max=#]` :
|
||||
`zstd` will dynamically adapt compression level to perceived I/O conditions.
|
||||
Compression level adaptation can be observed live by using command `-v`.
|
||||
Adaptation can be constrained between supplied `min` and `max` levels.
|
||||
The feature works when combined with multi-threading and `--long` mode.
|
||||
It does not work with `--single-thread`.
|
||||
It sets window size to 8 MB by default (can be changed manually, see `wlog`).
|
||||
Due to the chaotic nature of dynamic adaptation, compressed result is not reproducible.
|
||||
_note_ : at the time of this writing, `--adapt` can remain stuck at low speed
|
||||
when combined with multiple worker threads (>=2).
|
||||
* `-D file`:
|
||||
use `file` as Dictionary to compress or decompress FILE(s)
|
||||
* `--no-dictID`:
|
||||
|
@ -139,6 +139,7 @@ static int usage_advanced(const char* programName)
|
||||
DISPLAY( "--ultra : enable levels beyond %i, up to %i (requires more memory)\n", ZSTDCLI_CLEVEL_MAX, ZSTD_maxCLevel());
|
||||
DISPLAY( "--long[=#]: enable long distance matching with given window log (default: %u)\n", g_defaultMaxWindowLog);
|
||||
DISPLAY( "--fast[=#]: switch to ultra fast compression level (default: %u)\n", 1);
|
||||
DISPLAY( "--adapt : dynamically adapt compression level to I/O conditions \n");
|
||||
#ifdef ZSTD_MULTITHREAD
|
||||
DISPLAY( " -T# : spawns # compression threads (default: 1, 0==# cores) \n");
|
||||
DISPLAY( " -B# : select size of each job (default: 0==automatic) \n");
|
||||
@ -365,6 +366,30 @@ static ZDICT_fastCover_params_t defaultFastCoverParams(void)
|
||||
#endif
|
||||
|
||||
|
||||
/** parseAdaptParameters() :
|
||||
* reads adapt parameters from *stringPtr (e.g. "--zstd=min=1,max=19) and store them into adaptMinPtr and adaptMaxPtr.
|
||||
* Both adaptMinPtr and adaptMaxPtr must be already allocated and correctly initialized.
|
||||
* There is no guarantee that any of these values will be updated.
|
||||
* @return 1 means that parsing was successful,
|
||||
* @return 0 in case of malformed parameters
|
||||
*/
|
||||
static unsigned parseAdaptParameters(const char* stringPtr, int* adaptMinPtr, int* adaptMaxPtr)
|
||||
{
|
||||
for ( ; ;) {
|
||||
if (longCommandWArg(&stringPtr, "min=")) { *adaptMinPtr = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
|
||||
if (longCommandWArg(&stringPtr, "max=")) { *adaptMaxPtr = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
|
||||
DISPLAYLEVEL(4, "invalid compression parameter \n");
|
||||
return 0;
|
||||
}
|
||||
if (stringPtr[0] != 0) return 0; /* check the end of string */
|
||||
if (*adaptMinPtr > *adaptMaxPtr) {
|
||||
DISPLAYLEVEL(4, "incoherent adaptation limits \n");
|
||||
return 0;
|
||||
}
|
||||
return 1;
|
||||
}
|
||||
|
||||
|
||||
/** parseCompressionParameters() :
|
||||
* reads compression parameters from *stringPtr (e.g. "--zstd=wlog=23,clog=23,hlog=22,slog=6,slen=3,tlen=48,strat=6") into *params
|
||||
* @return 1 means that compression parameters were correct
|
||||
@ -429,6 +454,15 @@ typedef enum { zom_compress, zom_decompress, zom_test, zom_bench, zom_train, zom
|
||||
|
||||
#define CLEAN_RETURN(i) { operationResult = (i); goto _end; }
|
||||
|
||||
#ifdef ZSTD_NOCOMPRESS
|
||||
/* symbols from compression library are not defined and should not be invoked */
|
||||
# define MINCLEVEL -50
|
||||
# define MAXCLEVEL 22
|
||||
#else
|
||||
# define MINCLEVEL ZSTD_minCLevel()
|
||||
# define MAXCLEVEL ZSTD_maxCLevel()
|
||||
#endif
|
||||
|
||||
int main(int argCount, const char* argv[])
|
||||
{
|
||||
int argNb,
|
||||
@ -438,6 +472,9 @@ int main(int argCount, const char* argv[])
|
||||
ldmFlag = 0,
|
||||
main_pause = 0,
|
||||
nbWorkers = 0,
|
||||
adapt = 0,
|
||||
adaptMin = MINCLEVEL,
|
||||
adaptMax = MAXCLEVEL,
|
||||
nextArgumentIsOutFileName = 0,
|
||||
nextArgumentIsMaxDict = 0,
|
||||
nextArgumentIsDictID = 0,
|
||||
@ -556,6 +593,8 @@ int main(int argCount, const char* argv[])
|
||||
if (!strcmp(argument, "--keep")) { FIO_setRemoveSrcFile(0); continue; }
|
||||
if (!strcmp(argument, "--rm")) { FIO_setRemoveSrcFile(1); continue; }
|
||||
if (!strcmp(argument, "--priority=rt")) { setRealTimePrio = 1; continue; }
|
||||
if (!strcmp(argument, "--adapt")) { adapt = 1; continue; }
|
||||
if (longCommandWArg(&argument, "--adapt=")) { adapt = 1; if (!parseAdaptParameters(argument, &adaptMin, &adaptMax)) CLEAN_RETURN(badusage(programName)); continue; }
|
||||
if (!strcmp(argument, "--single-thread")) { nbWorkers = 0; singleThread = 1; continue; }
|
||||
if (!strcmp(argument, "--format=zstd")) { suffix = ZSTD_EXTENSION; FIO_setCompressionType(FIO_zstdCompression); continue; }
|
||||
#ifdef ZSTD_GZCOMPRESS
|
||||
@ -1004,24 +1043,25 @@ int main(int argCount, const char* argv[])
|
||||
#ifndef ZSTD_NOCOMPRESS
|
||||
FIO_setNbWorkers(nbWorkers);
|
||||
FIO_setBlockSize((U32)blockSize);
|
||||
if (g_overlapLog!=OVERLAP_LOG_DEFAULT) FIO_setOverlapLog(g_overlapLog);
|
||||
FIO_setLdmFlag(ldmFlag);
|
||||
FIO_setLdmHashLog(g_ldmHashLog);
|
||||
FIO_setLdmMinMatch(g_ldmMinMatch);
|
||||
if (g_ldmBucketSizeLog != LDM_PARAM_DEFAULT) {
|
||||
FIO_setLdmBucketSizeLog(g_ldmBucketSizeLog);
|
||||
}
|
||||
if (g_ldmHashEveryLog != LDM_PARAM_DEFAULT) {
|
||||
FIO_setLdmHashEveryLog(g_ldmHashEveryLog);
|
||||
}
|
||||
if (g_ldmBucketSizeLog != LDM_PARAM_DEFAULT) FIO_setLdmBucketSizeLog(g_ldmBucketSizeLog);
|
||||
if (g_ldmHashEveryLog != LDM_PARAM_DEFAULT) FIO_setLdmHashEveryLog(g_ldmHashEveryLog);
|
||||
FIO_setAdaptiveMode(adapt);
|
||||
FIO_setAdaptMin(adaptMin);
|
||||
FIO_setAdaptMax(adaptMax);
|
||||
if (adaptMin > cLevel) cLevel = adaptMin;
|
||||
if (adaptMax < cLevel) cLevel = adaptMax;
|
||||
|
||||
if (g_overlapLog!=OVERLAP_LOG_DEFAULT) FIO_setOverlapLog(g_overlapLog);
|
||||
if ((filenameIdx==1) && outFileName)
|
||||
operationResult = FIO_compressFilename(outFileName, filenameTable[0], dictFileName, cLevel, &compressionParams);
|
||||
operationResult = FIO_compressFilename(outFileName, filenameTable[0], dictFileName, cLevel, compressionParams);
|
||||
else
|
||||
operationResult = FIO_compressMultipleFilenames(filenameTable, filenameIdx, outFileName, suffix, dictFileName, cLevel, &compressionParams);
|
||||
operationResult = FIO_compressMultipleFilenames(filenameTable, filenameIdx, outFileName, suffix, dictFileName, cLevel, compressionParams);
|
||||
#else
|
||||
(void)suffix; (void)ultra; (void)cLevel; (void)ldmFlag; /* not used when ZSTD_NOCOMPRESS set */
|
||||
DISPLAY("Compression not supported\n");
|
||||
(void)suffix; (void)adapt; (void)ultra; (void)cLevel; (void)ldmFlag; /* not used when ZSTD_NOCOMPRESS set */
|
||||
DISPLAY("Compression not supported \n");
|
||||
#endif
|
||||
} else { /* decompression or test */
|
||||
#ifndef ZSTD_NODECOMPRESS
|
||||
@ -1038,7 +1078,7 @@ int main(int argCount, const char* argv[])
|
||||
else
|
||||
operationResult = FIO_decompressMultipleFilenames(filenameTable, filenameIdx, outFileName, dictFileName);
|
||||
#else
|
||||
DISPLAY("Decompression not supported\n");
|
||||
DISPLAY("Decompression not supported \n");
|
||||
#endif
|
||||
}
|
||||
|
||||
|
@ -103,6 +103,7 @@ else
|
||||
fi
|
||||
|
||||
|
||||
|
||||
$ECHO "\n===> simple tests "
|
||||
|
||||
./datagen > tmp
|
||||
@ -419,38 +420,6 @@ test -f dictionary
|
||||
rm tmp* dictionary
|
||||
|
||||
|
||||
$ECHO "\n===> cover dictionary builder : advanced options "
|
||||
|
||||
TESTFILE=../programs/zstdcli.c
|
||||
./datagen > tmpDict
|
||||
$ECHO "- Create first dictionary"
|
||||
$ZSTD --train-cover=k=46,d=8,split=80 *.c ../programs/*.c -o tmpDict
|
||||
cp $TESTFILE tmp
|
||||
$ZSTD -f tmp -D tmpDict
|
||||
$ZSTD -d tmp.zst -D tmpDict -fo result
|
||||
$DIFF $TESTFILE result
|
||||
$ECHO "- Create second (different) dictionary"
|
||||
$ZSTD --train-cover=k=56,d=8 *.c ../programs/*.c ../programs/*.h -o tmpDictC
|
||||
$ZSTD -d tmp.zst -D tmpDictC -fo result && die "wrong dictionary not detected!"
|
||||
$ECHO "- Create dictionary with short dictID"
|
||||
$ZSTD --train-cover=k=46,d=8,split=80 *.c ../programs/*.c --dictID=1 -o tmpDict1
|
||||
cmp tmpDict tmpDict1 && die "dictionaries should have different ID !"
|
||||
$ECHO "- Create dictionary with size limit"
|
||||
$ZSTD --train-cover=steps=8 *.c ../programs/*.c -o tmpDict2 --maxdict=4K
|
||||
$ECHO "- Compare size of dictionary from 90% training samples with 80% training samples"
|
||||
$ZSTD --train-cover=split=90 -r *.c ../programs/*.c
|
||||
$ZSTD --train-cover=split=80 -r *.c ../programs/*.c
|
||||
$ECHO "- Create dictionary using all samples for both training and testing"
|
||||
$ZSTD --train-cover=split=100 -r *.c ../programs/*.c
|
||||
$ECHO "- Test -o before --train-cover"
|
||||
rm -f tmpDict dictionary
|
||||
$ZSTD -o tmpDict --train-cover *.c ../programs/*.c
|
||||
test -f tmpDict
|
||||
$ZSTD --train-cover *.c ../programs/*.c
|
||||
test -f dictionary
|
||||
rm tmp* dictionary
|
||||
|
||||
|
||||
$ECHO "\n===> fastCover dictionary builder : advanced options "
|
||||
|
||||
TESTFILE=../programs/zstdcli.c
|
||||
@ -845,11 +814,20 @@ roundTripTest -g1M -P50 "1 --single-thread --long=29" " --long=28 --memory=512MB
|
||||
roundTripTest -g1M -P50 "1 --single-thread --long=29" " --zstd=wlog=28 --memory=512MB"
|
||||
|
||||
|
||||
$ECHO "\n===> adaptive mode "
|
||||
roundTripTest -g270000000 " --adapt"
|
||||
roundTripTest -g27000000 " --adapt=min=1,max=4"
|
||||
./datagen > tmp
|
||||
$ZSTD -f -vv --adapt=min=10,max=9 tmp && die "--adapt must fail on incoherent bounds"
|
||||
|
||||
|
||||
if [ "$1" != "--test-large-data" ]; then
|
||||
$ECHO "Skipping large data tests"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
|
||||
|
||||
$ECHO "\n===> large files tests "
|
||||
|
||||
roundTripTest -g270000000 1
|
||||
@ -904,4 +882,37 @@ else
|
||||
$ECHO "\n**** no multithreading, skipping zstdmt tests **** "
|
||||
fi
|
||||
|
||||
rm tmp*
|
||||
|
||||
$ECHO "\n===> cover dictionary builder : advanced options "
|
||||
|
||||
TESTFILE=../programs/zstdcli.c
|
||||
./datagen > tmpDict
|
||||
$ECHO "- Create first dictionary"
|
||||
$ZSTD --train-cover=k=46,d=8,split=80 *.c ../programs/*.c -o tmpDict
|
||||
cp $TESTFILE tmp
|
||||
$ZSTD -f tmp -D tmpDict
|
||||
$ZSTD -d tmp.zst -D tmpDict -fo result
|
||||
$DIFF $TESTFILE result
|
||||
$ECHO "- Create second (different) dictionary"
|
||||
$ZSTD --train-cover=k=56,d=8 *.c ../programs/*.c ../programs/*.h -o tmpDictC
|
||||
$ZSTD -d tmp.zst -D tmpDictC -fo result && die "wrong dictionary not detected!"
|
||||
$ECHO "- Create dictionary with short dictID"
|
||||
$ZSTD --train-cover=k=46,d=8,split=80 *.c ../programs/*.c --dictID=1 -o tmpDict1
|
||||
cmp tmpDict tmpDict1 && die "dictionaries should have different ID !"
|
||||
$ECHO "- Create dictionary with size limit"
|
||||
$ZSTD --train-cover=steps=8 *.c ../programs/*.c -o tmpDict2 --maxdict=4K
|
||||
$ECHO "- Compare size of dictionary from 90% training samples with 80% training samples"
|
||||
$ZSTD --train-cover=split=90 -r *.c ../programs/*.c
|
||||
$ZSTD --train-cover=split=80 -r *.c ../programs/*.c
|
||||
$ECHO "- Create dictionary using all samples for both training and testing"
|
||||
$ZSTD --train-cover=split=100 -r *.c ../programs/*.c
|
||||
$ECHO "- Test -o before --train-cover"
|
||||
rm -f tmpDict dictionary
|
||||
$ZSTD -o tmpDict --train-cover *.c ../programs/*.c
|
||||
test -f tmpDict
|
||||
$ZSTD --train-cover *.c ../programs/*.c
|
||||
test -f dictionary
|
||||
rm -f tmp* dictionary
|
||||
|
||||
|
||||
rm -f tmp*
|
||||
|
40
tests/rateLimiter.py
Executable file
40
tests/rateLimiter.py
Executable file
@ -0,0 +1,40 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
# ################################################################
|
||||
# Copyright (c) 2018-present, Facebook, Inc.
|
||||
# All rights reserved.
|
||||
#
|
||||
# This source code is licensed under both the BSD-style license (found in the
|
||||
# LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
# in the COPYING file in the root directory of this source tree).
|
||||
# ##########################################################################
|
||||
|
||||
# Rate limiter, replacement for pv
|
||||
# this rate limiter does not "catch up" after a blocking period
|
||||
# Limitations:
|
||||
# - only accepts limit speed in MB/s
|
||||
|
||||
import sys
|
||||
import time
|
||||
|
||||
MB = 1024 * 1024
|
||||
rate = float(sys.argv[1]) * MB
|
||||
start = time.time()
|
||||
total_read = 0
|
||||
|
||||
# sys.stderr.close() # remove error message, for Ctrl+C
|
||||
|
||||
try:
|
||||
buf = " "
|
||||
while len(buf):
|
||||
now = time.time()
|
||||
to_read = max(int(rate * (now - start)), 1)
|
||||
max_buf_size = 1 * MB
|
||||
to_read = min(to_read, max_buf_size)
|
||||
start = now
|
||||
|
||||
buf = sys.stdin.buffer.read(to_read)
|
||||
sys.stdout.buffer.write(buf)
|
||||
|
||||
except (KeyboardInterrupt, BrokenPipeError) as e:
|
||||
pass
|
Loading…
x
Reference in New Issue
Block a user