From 9f432926b44dc21a78942e192beefefcffbb3adc Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Mon, 9 Nov 2015 17:42:17 +0100 Subject: [PATCH] fileio can use small blocks params --- lib/zstdhc.c | 26 ++++++++++++++------------ lib/zstdhc_static.h | 6 +++--- programs/bench.c | 2 +- programs/fileio.c | 44 ++++++++++++++++++++++++++++++++------------ 4 files changed, 50 insertions(+), 28 deletions(-) diff --git a/lib/zstdhc.c b/lib/zstdhc.c index 866db183..fdf6dc95 100644 --- a/lib/zstdhc.c +++ b/lib/zstdhc.c @@ -111,7 +111,7 @@ size_t ZSTD_HC_freeCCtx(ZSTD_HC_CCtx* cctx) /** ZSTD_HC_validateParams correct params value to remain within authorized range optimize for srcSize if srcSize > 0 */ -void ZSTD_HC_validateParams(ZSTD_HC_parameters* params, size_t srcSize) +void ZSTD_HC_validateParams(ZSTD_HC_parameters* params, U64 srcSizeHint) { const U32 btPlus = (params->strategy == ZSTD_HC_btlazy2); @@ -120,9 +120,9 @@ void ZSTD_HC_validateParams(ZSTD_HC_parameters* params, size_t srcSize) if (params->windowLog < ZSTD_HC_WINDOWLOG_MIN) params->windowLog = ZSTD_HC_WINDOWLOG_MIN; /* correct params, to use less memory */ - if (srcSize > 0) + if ((srcSizeHint > 0) && (srcSizeHint < (1<windowLog > srcLog) params->windowLog = srcLog; } @@ -139,9 +139,10 @@ void ZSTD_HC_validateParams(ZSTD_HC_parameters* params, size_t srcSize) static size_t ZSTD_HC_resetCCtx_advanced (ZSTD_HC_CCtx* zc, - ZSTD_HC_parameters params) + ZSTD_HC_parameters params, + U64 srcSizeHint) { - ZSTD_HC_validateParams(¶ms, 0); + ZSTD_HC_validateParams(¶ms, srcSizeHint); /* reserve table memory */ { @@ -930,7 +931,7 @@ size_t ZSTD_HC_compressContinue (ZSTD_HC_CCtx* ctxPtr, if (ip != ctxPtr->end) { if (ctxPtr->end != NULL) - ZSTD_HC_resetCCtx_advanced(ctxPtr, ctxPtr->params); + ZSTD_HC_resetCCtx_advanced(ctxPtr, ctxPtr->params, srcSize); ctxPtr->base = ip; } @@ -941,23 +942,24 @@ size_t ZSTD_HC_compressContinue (ZSTD_HC_CCtx* ctxPtr, size_t ZSTD_HC_compressBegin_advanced(ZSTD_HC_CCtx* ctx, void* dst, size_t maxDstSize, - const ZSTD_HC_parameters params) + const ZSTD_HC_parameters params, + U64 srcSizeHint) { size_t errorCode; if (maxDstSize < 4) return ERROR(dstSize_tooSmall); - errorCode = ZSTD_HC_resetCCtx_advanced(ctx, params); + errorCode = ZSTD_HC_resetCCtx_advanced(ctx, params, srcSizeHint); if (ZSTD_isError(errorCode)) return errorCode; MEM_writeLE32(dst, ZSTD_magicNumber); /* Write Header */ return 4; } -size_t ZSTD_HC_compressBegin(ZSTD_HC_CCtx* ctx, void* dst, size_t maxDstSize, int compressionLevel) +size_t ZSTD_HC_compressBegin(ZSTD_HC_CCtx* ctx, void* dst, size_t maxDstSize, int compressionLevel, U64 srcSizeHint) { - int tableID = 1; + int tableID = ((srcSizeHint-1) > 128 KB); /* intentional underflow for 0 */ if (compressionLevel<=0) compressionLevel = 1; if (compressionLevel > ZSTD_HC_MAX_CLEVEL) compressionLevel = ZSTD_HC_MAX_CLEVEL; - return ZSTD_HC_compressBegin_advanced(ctx, dst, maxDstSize, ZSTD_HC_defaultParameters[tableID][compressionLevel]); + return ZSTD_HC_compressBegin_advanced(ctx, dst, maxDstSize, ZSTD_HC_defaultParameters[tableID][compressionLevel], srcSizeHint); } @@ -995,7 +997,7 @@ size_t ZSTD_HC_compress_advanced (ZSTD_HC_CCtx* ctx, } /* Header */ - oSize = ZSTD_HC_compressBegin_advanced(ctx, dst, maxDstSize, params); + oSize = ZSTD_HC_compressBegin_advanced(ctx, dst, maxDstSize, params, srcSize); if(ZSTD_isError(oSize)) return oSize; op += oSize; maxDstSize -= oSize; diff --git a/lib/zstdhc_static.h b/lib/zstdhc_static.h index 534f2a01..f1973fc0 100644 --- a/lib/zstdhc_static.h +++ b/lib/zstdhc_static.h @@ -83,14 +83,14 @@ size_t ZSTD_HC_compress_advanced (ZSTD_HC_CCtx* ctx, /** ZSTD_HC_validateParams correct params value to remain within authorized range - optimize for srcSize if srcSize > 0 */ -void ZSTD_HC_validateParams(ZSTD_HC_parameters* params, size_t srcSize); + srcSizeHint value is optional, select 0 if not known */ +void ZSTD_HC_validateParams(ZSTD_HC_parameters* params, U64 srcSizeHint); /* ************************************* * Streaming functions ***************************************/ -size_t ZSTD_HC_compressBegin(ZSTD_HC_CCtx* ctx, void* dst, size_t maxDstSize, int compressionLevel); +size_t ZSTD_HC_compressBegin(ZSTD_HC_CCtx* ctx, void* dst, size_t maxDstSize, int compressionLevel, U64 srcSizeHint); size_t ZSTD_HC_compressContinue(ZSTD_HC_CCtx* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize); size_t ZSTD_HC_compressEnd(ZSTD_HC_CCtx* ctx, void* dst, size_t maxDstSize); diff --git a/programs/bench.c b/programs/bench.c index 0009e4ae..5d10f9fb 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -380,7 +380,7 @@ static int BMK_benchMem(void* srcBuffer, size_t srcSize, const char* fileName, i } -static U64 BMK_GetFileSize(char* infilename) +static U64 BMK_GetFileSize(const char* infilename) { int r; #if defined(_MSC_VER) diff --git a/programs/fileio.c b/programs/fileio.c index d168f8a4..a7a394e0 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -36,7 +36,7 @@ /**LEGACY_SUPPORT : * decompressor can decode older formats (starting from Zstd 0.1+) */ # define ZSTD_LEGACY_SUPPORT 1 -#endif // ZSTD_LEGACY_SUPPORT +#endif /* ************************************* @@ -58,11 +58,13 @@ /* ************************************* * Includes ***************************************/ -#include /* fprintf, fopen, fread, _fileno, stdin, stdout */ -#include /* malloc, free */ -#include /* strcmp, strlen */ -#include /* clock */ -#include /* errno */ +#include /* fprintf, fopen, fread, _fileno, stdin, stdout */ +#include /* malloc, free */ +#include /* strcmp, strlen */ +#include /* clock */ +#include /* errno */ +#include /* stat64 */ +#include /* stat64 */ #include "mem.h" #include "fileio.h" #include "zstd_static.h" @@ -215,19 +217,35 @@ static void FIO_getFileHandles(FILE** pfinput, FILE** pfoutput, const char* inpu if ( *pfoutput==0) EXM_THROW(13, "Pb opening dst : %s", output_filename); } + +static U64 FIO_getFileSize(const char* infilename) +{ + int r; +#if defined(_MSC_VER) + struct _stat64 statbuf; + r = _stat64(infilename, &statbuf); +#else + struct stat statbuf; + r = stat(infilename, &statbuf); +#endif + if (r || !S_ISREG(statbuf.st_mode)) return 0; + return (U64)statbuf.st_size; +} + + typedef void* (*FIO_createC) (void); static void* local_ZSTD_createCCtx(void) { return (void*) ZSTD_createCCtx(); } static void* local_ZSTD_HC_createCCtx(void) { return (void*) ZSTD_HC_createCCtx(); } -typedef size_t (*FIO_initC) (void* ctx, void* dst, size_t maxDstSize, int cLevel); -static size_t local_ZSTD_compressBegin (void* ctx, void* dst, size_t maxDstSize, int cLevel) +typedef size_t (*FIO_initC) (void* ctx, void* dst, size_t maxDstSize, int cLevel, U64 srcSizeHint); +static size_t local_ZSTD_compressBegin (void* ctx, void* dst, size_t maxDstSize, int cLevel, U64 srcSizeHint) { - (void)cLevel; + (void)cLevel; (void)srcSizeHint; return ZSTD_compressBegin((ZSTD_CCtx*)ctx, dst, maxDstSize); } -static size_t local_ZSTD_HC_compressBegin (void* ctx, void* dst, size_t maxDstSize, int cLevel) +static size_t local_ZSTD_HC_compressBegin (void* ctx, void* dst, size_t maxDstSize, int cLevel, U64 srcSizeHint) { - return ZSTD_HC_compressBegin((ZSTD_HC_CCtx*)ctx, dst, maxDstSize, cLevel); + return ZSTD_HC_compressBegin((ZSTD_HC_CCtx*)ctx, dst, maxDstSize, cLevel, srcSizeHint); } typedef size_t (*FIO_continueC) (void* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize); @@ -294,6 +312,7 @@ unsigned long long FIO_compressFilename(const char* output_filename, const char* freeC = local_ZSTD_HC_freeCCtx; } FIO_getFileHandles(&finput, &foutput, input_filename, output_filename); + filesize = FIO_getFileSize(input_filename); /* Allocate Memory */ ctx = createC(); @@ -304,12 +323,13 @@ unsigned long long FIO_compressFilename(const char* output_filename, const char* inEnd = inBuff + inBuffSize; /* Write Frame Header */ - cSize = initC(ctx, outBuff, outBuffSize, cLevel); + cSize = initC(ctx, outBuff, outBuffSize, cLevel, filesize); if (ZSTD_isError(cSize)) EXM_THROW(22, "Compression error : cannot create frame header"); sizeCheck = fwrite(outBuff, 1, cSize, foutput); if (sizeCheck!=cSize) EXM_THROW(23, "Write error : cannot write header into %s", output_filename); compressedfilesize += cSize; + filesize = 0; /* Main compression loop */ while (1)