facebook/zstd
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

removed static variables

Browse Source 
so that --adapt can work on multiple input files too
This commit is contained in:
Yann Collet 2018-09-19 15:09:45 -07:00
parent 89bc309d90
commit ca02ebee07
2 changed files with 81 additions and 78 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
lib/compress/zstdmt_compress.c
View File

@ -1530,7 +1530,7 @@ static size_t ZSTDMT_createCompressionJob(ZSTDMT_CCtx* mtctx, size_t srcSize, ZS
mtctx->jobs[jobID].jobID = mtctx->nextJobID; mtctx->jobs[jobID].jobID = mtctx->nextJobID;
mtctx->jobs[jobID].firstJob = (mtctx->nextJobID==0); mtctx->jobs[jobID].firstJob = (mtctx->nextJobID==0);
mtctx->jobs[jobID].lastJob = endFrame; mtctx->jobs[jobID].lastJob = endFrame;
mtctx->jobs[jobID].frameChecksumNeeded = endFrame && (mtctx->nextJobID>0) && mtctx->params.fParams.checksumFlag; mtctx->jobs[jobID].frameChecksumNeeded = mtctx->params.fParams.checksumFlag && endFrame && (mtctx->nextJobID>0);
mtctx->jobs[jobID].dstFlushed = 0; mtctx->jobs[jobID].dstFlushed = 0;
/* Update the round buffer pos and clear the input buffer to be reset */ /* Update the round buffer pos and clear the input buffer to be reset */

73
programs/fileio.c
View File

@ -807,6 +807,8 @@ FIO_compressZstdFrame(const cRess_t* ressPtr,
ZSTD_EndDirective directive = ZSTD_e_continue; ZSTD_EndDirective directive = ZSTD_e_continue;
/* stats */ /* stats */
ZSTD_frameProgression previous_zfp_update = { 0, 0, 0, 0, 0, 0 };
ZSTD_frameProgression previous_zfp_correction = { 0, 0, 0, 0, 0, 0 };
typedef enum { noChange, slower, faster } speedChange_e; typedef enum { noChange, slower, faster } speedChange_e;
speedChange_e speedChange = noChange; speedChange_e speedChange = noChange;
unsigned flushWaiting = 0; unsigned flushWaiting = 0;
@ -820,7 +822,7 @@ FIO_compressZstdFrame(const cRess_t* ressPtr,
if (fileSize != UTIL_FILESIZE_UNKNOWN) { if (fileSize != UTIL_FILESIZE_UNKNOWN) {
CHECK(ZSTD_CCtx_setPledgedSrcSize(ress.cctx, fileSize)); CHECK(ZSTD_CCtx_setPledgedSrcSize(ress.cctx, fileSize));
} }
(void)compressionLevel; (void)srcFileName; (void)srcFileName;
/* Main compression loop */ /* Main compression loop */
do { do {
@ -863,24 +865,42 @@ FIO_compressZstdFrame(const cRess_t* ressPtr,
ZSTD_frameProgression const zfp = ZSTD_getFrameProgression(ress.cctx); ZSTD_frameProgression const zfp = ZSTD_getFrameProgression(ress.cctx);
double const cShare = (double)zfp.produced / (zfp.consumed + !zfp.consumed/*avoid div0*/) * 100; double const cShare = (double)zfp.produced / (zfp.consumed + !zfp.consumed/*avoid div0*/) * 100;
/* display progress notifications */
if (g_displayLevel >= 3) {
DISPLAYUPDATE(3, "\r(L%i) Buffered :%4u MB - Consumed :%4u MB - Compressed :%4u MB => %.2f%% ",
compressionLevel,
(U32)((zfp.ingested - zfp.consumed) >> 20),
(U32)(zfp.consumed >> 20),
(U32)(zfp.produced >> 20),
cShare );
} else { /* summarized notifications if == 2; */
DISPLAYLEVEL(2, "\rRead : %u ", (U32)(zfp.consumed >> 20));
if (fileSize != UTIL_FILESIZE_UNKNOWN)
DISPLAYLEVEL(2, "/ %u ", (U32)(fileSize >> 20));
DISPLAYLEVEL(2, "MB ==> %2.f%% ", cShare);
DELAY_NEXT_UPDATE();
}
/* adaptive mode : statistics measurement and speed correction */
if (g_adaptiveMode) {
/* check output speed */ /* check output speed */
if (zfp.currentJobID > 1) { /* only possible if nbWorkers >= 1 */ if (zfp.currentJobID > 1) { /* only possible if nbWorkers >= 1 */
static ZSTD_frameProgression cpszfp = { 0, 0, 0, 0, 0, 0 }; /* note : requires fileio to run main thread */
unsigned long long newlyProduced = zfp.produced - cpszfp.produced; unsigned long long newlyProduced = zfp.produced - previous_zfp_update.produced;
unsigned long long newlyFlushed = zfp.flushed - cpszfp.flushed; unsigned long long newlyFlushed = zfp.flushed - previous_zfp_update.flushed;
assert(zfp.produced >= cpszfp.produced); assert(zfp.produced >= previous_zfp_update.produced);
assert(g_nbWorkers >= 1); assert(g_nbWorkers >= 1);
if ( (zfp.ingested == cpszfp.ingested) /* no data read : input buffer full */ if ( (zfp.ingested == previous_zfp_update.ingested) /* no data read : input buffer full */
&& (zfp.consumed == cpszfp.consumed) /* no data compressed : no more buffer to compress OR compression is really slow */ && (zfp.consumed == previous_zfp_update.consumed) /* no data compressed : no more buffer to compress OR compression is really slow */
&& (zfp.nbActiveWorkers == 0) /* confirmed : no compression : either no more buffer to compress OR not enough data to start first worker */ && (zfp.nbActiveWorkers == 0) /* confirmed : no compression : either no more buffer to compress OR not enough data to start first worker */
) { ) {
DISPLAYLEVEL(6, "all buffers full : compression stopped => slow down \n") DISPLAYLEVEL(6, "all buffers full : compression stopped => slow down \n")
speedChange = slower; speedChange = slower;
} }
cpszfp = zfp; previous_zfp_update = zfp;
if ( (newlyProduced > (newlyFlushed * 9 / 8)) /* compression produces more data than output can flush (though production can be spiky, due to work unit : (N==4)*block sizes) */ if ( (newlyProduced > (newlyFlushed * 9 / 8)) /* compression produces more data than output can flush (though production can be spiky, due to work unit : (N==4)*block sizes) */
&& (flushWaiting == 0) /* flush speed was never slowed by lack of production, so it's operating at max capacity */ && (flushWaiting == 0) /* flush speed was never slowed by lack of production, so it's operating at max capacity */
@ -898,15 +918,14 @@ FIO_compressZstdFrame(const cRess_t* ressPtr,
/* check input speed */ /* check input speed */
if (zfp.currentJobID > g_nbWorkers+1) { /* warm up period, to fill all workers */ if (zfp.currentJobID > g_nbWorkers+1) { /* warm up period, to fill all workers */
if (inputBlocked <= 0) { if (inputBlocked <= 0) {
DISPLAYLEVEL(6, "input is never blocked => input is too slow \n"); DISPLAYLEVEL(6, "input is never blocked => input is slower than ingestion \n");
speedChange = slower; speedChange = slower;
} else if (speedChange == noChange) { } else if (speedChange == noChange) {
static ZSTD_frameProgression csuzfp = { 0, 0, 0, 0, 0, 0 }; unsigned long long newlyIngested = zfp.ingested - previous_zfp_correction.ingested;
unsigned long long newlyIngested = zfp.ingested - csuzfp.ingested; unsigned long long newlyConsumed = zfp.consumed - previous_zfp_correction.consumed;
unsigned long long newlyConsumed = zfp.consumed - csuzfp.consumed; unsigned long long newlyProduced = zfp.produced - previous_zfp_correction.produced;
unsigned long long newlyProduced = zfp.produced - csuzfp.produced; unsigned long long newlyFlushed = zfp.flushed - previous_zfp_correction.flushed;
unsigned long long newlyFlushed = zfp.flushed - csuzfp.flushed; previous_zfp_correction = zfp;
csuzfp = zfp;
assert(inputPresented > 0); assert(inputPresented > 0);
DISPLAYLEVEL(6, "input blocked %u/%u(%.2f) - ingested:%u vs %u:consumed - flushed:%u vs %u:produced \n", DISPLAYLEVEL(6, "input blocked %u/%u(%.2f) - ingested:%u vs %u:consumed - flushed:%u vs %u:produced \n",
inputBlocked, inputPresented, (double)inputBlocked/inputPresented*100, inputBlocked, inputPresented, (double)inputBlocked/inputPresented*100,
@ -925,7 +944,6 @@ FIO_compressZstdFrame(const cRess_t* ressPtr,
inputPresented = 0; inputPresented = 0;
} }
if (g_adaptiveMode) {
if (speedChange == slower) { if (speedChange == slower) {
DISPLAYLEVEL(6, "slower speed , higher compression \n") DISPLAYLEVEL(6, "slower speed , higher compression \n")
compressionLevel ++; compressionLevel ++;
@ -940,27 +958,12 @@ FIO_compressZstdFrame(const cRess_t* ressPtr,
ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_compressionLevel, (unsigned)compressionLevel); ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_compressionLevel, (unsigned)compressionLevel);
} }
speedChange = noChange; speedChange = noChange;
}
lastJobID = zfp.currentJobID; lastJobID = zfp.currentJobID;
} /* if (zfp.currentJobID > lastJobID) */ } /* if (zfp.currentJobID > lastJobID) */
} /* if (g_adaptiveMode) */
if (g_displayLevel >= 3) { } /* if (READY_FOR_UPDATE()) */
DISPLAYUPDATE(3, "\r(L%i) Buffered :%4u MB - Consumed :%4u MB - Compressed :%4u MB => %.2f%% ", } /* while ((inBuff.pos != inBuff.size) */
compressionLevel,
(U32)((zfp.ingested - zfp.consumed) >> 20),
(U32)(zfp.consumed >> 20),
(U32)(zfp.produced >> 20),
cShare );
} else {
/* g_displayLevel <= 2; only display notifications if == 2; */
DISPLAYLEVEL(2, "\rRead : %u ", (U32)(zfp.consumed >> 20));
if (fileSize != UTIL_FILESIZE_UNKNOWN)
DISPLAYLEVEL(2, "/ %u ", (U32)(fileSize >> 20));
DISPLAYLEVEL(2, "MB ==> %2.f%% ", cShare);
DELAY_NEXT_UPDATE();
}
}
}
} while (directive != ZSTD_e_end); } while (directive != ZSTD_e_end);
if (ferror(srcFile)) { if (ferror(srcFile)) {