facebook
/
zstd
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'progressiveMT' into progressiveFlush

dev
Yann Collet 2018-01-18 13:35:24 -08:00
parent 1b5d80d633 b6ab232f2d
commit ef97d5a287
5 changed files with 16 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
lib/compress/zstd_compress.c
View File

@ -2566,6 +2566,8 @@ static size_t ZSTD_initCDict_internal(
}
cdict->dictContentSize = dictSize;
/* Reset the state to no dictionary */
ZSTD_reset_compressedBlockState(&cdict->cBlockState);
{
void* const end = ZSTD_reset_matchState(
&cdict->matchState,
@ -2574,6 +2576,9 @@ static size_t ZSTD_initCDict_internal(
assert(end == (char*)cdict->workspace + cdict->workspaceSize);
(void)end;
}
/* (Maybe) load the dictionary
* Skips loading the dictionary if it is <= 8 bytes.
*/
{
ZSTD_CCtx_params params;
memset(&params, 0, sizeof(params));

6
lib/compress/zstd_compress_internal.h
View File

@ -170,9 +170,9 @@ struct ZSTD_CCtx_s {
void* workSpace;
size_t workSpaceSize;
size_t blockSize;
U64 pledgedSrcSizePlusOne; /* this way, 0 (default) == unknown */
U64 consumedSrcSize;
U64 producedCSize;
unsigned long long pledgedSrcSizePlusOne; /* this way, 0 (default) == unknown */
unsigned long long consumedSrcSize;
unsigned long long producedCSize;
XXH64_state_t xxhState;
ZSTD_customMem customMem;
size_t staticSize;

2
lib/compress/zstdmt_compress.c
View File

@ -664,7 +664,7 @@ unsigned ZSTDMT_getNbThreads(const ZSTDMT_CCtx* mtctx)
/* ZSTDMT_getFrameProgression():
* tells how much data has been consumed (input) and produced (output) for current frame.
* able to count progression inside worker threads.
* Note : mutex will be triggered during statistics collection. */
* Note : mutex will be acquired during statistics collection. */
ZSTD_frameProgression ZSTDMT_getFrameProgression(ZSTDMT_CCtx* mtctx)
{
ZSTD_frameProgression fs;

6
lib/zstd.h
View File

@ -723,8 +723,10 @@ typedef struct {
} ZSTD_frameProgression;
/* ZSTD_getFrameProgression():
* tells how much data has been consumed (input) and produced (output) for current frame.
* able to count progression inside worker threads (non-blocking mode).
* tells how much data has been ingested (read from input)
* consumed (input actually compressed) and produced (output) for current frame.
* Therefore, (ingested - consumed) is amount of input data buffered internally, not yet compressed.
* Can report progression inside worker threads (multi-threading and non-blocking mode).
*/
ZSTD_frameProgression ZSTD_getFrameProgression(const ZSTD_CCtx* cctx);

6
programs/fileio.c
View File

@ -84,10 +84,10 @@ void FIO_setNotificationLevel(unsigned level) { g_displayLevel=level; }
static const U64 g_refreshRate = SEC_TO_MICRO / 6;
static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER;
#define READY_FOR_UPDATE (UTIL_clockSpanMicro(g_displayClock) > g_refreshRate)
#define READY_FOR_UPDATE() (UTIL_clockSpanMicro(g_displayClock) > g_refreshRate)
#define DISPLAYUPDATE(l, ...) { \
if (g_displayLevel>=l) { \
if (READY_FOR_UPDATE || (g_displayLevel>=4)) { \
if (READY_FOR_UPDATE() || (g_displayLevel>=4)) { \
g_displayClock = UTIL_getTime(); DISPLAY(__VA_ARGS__); \
if (g_displayLevel>=4) fflush(stderr); \
} } }
@ -813,7 +813,7 @@ static int FIO_compressFilename_internal(cRess_t ress,
}
}
#if 1
if (READY_FOR_UPDATE) {
if (READY_FOR_UPDATE()) {
ZSTD_frameProgression const zfp = ZSTD_getFrameProgression(ress.cctx);
DISPLAYUPDATE(2, "\rRead :%6u MB - Consumed :%6u MB - Compressed :%6u MB => %.2f%%",
(U32)(zfp.ingested >> 20),