From 459a6b712feb742eba5b8e8ce3b64336bc6c20a8 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Mon, 15 Feb 2016 20:37:23 +0100 Subject: [PATCH] Fix corner case multiple files bug Added test cases --- NEWS | 2 ++ programs/fileio.c | 67 ++++++++++++++++++++++++++++--------------- programs/playTests.sh | 20 +++++++++++-- 3 files changed, 63 insertions(+), 26 deletions(-) diff --git a/NEWS b/NEWS index 170f0c82..f6b2d6c9 100644 --- a/NEWS +++ b/NEWS @@ -1,5 +1,7 @@ v0.5.1 New : Optimal parsing => Very high compression modes, thanks to Przemyslaw Skibinski +Changed : Dictionary builder integrated into libzstd and zstd cli +Fix : high compression modes for big-endian platforms v0.5.0 New : dictionary builder utility diff --git a/programs/fileio.c b/programs/fileio.c index e2ad9e84..028c7db4 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -120,7 +120,7 @@ #define MAX_DICT_SIZE (1 MB) /* protection against large input (attack scenario) ; can be changed */ -/* ************************************* +/*-************************************* * Macros ***************************************/ #define DISPLAY(...) fprintf(stderr, __VA_ARGS__) @@ -134,8 +134,10 @@ static U32 g_displayLevel = 2; /* 0 : no display; 1: errors; 2 : + result static const unsigned refreshRate = 150; static clock_t g_time = 0; +#define MAX(a,b) ((a)>(b)?(a):(b)) -/* ************************************* + +/*-************************************* * Local Parameters ***************************************/ static U32 g_overwrite = 0; @@ -143,7 +145,7 @@ void FIO_overwriteMode(void) { g_overwrite=1; } void FIO_setNotificationLevel(unsigned level) { g_displayLevel=level; } -/* ************************************* +/*-************************************* * Exceptions ***************************************/ #ifndef DEBUG @@ -160,7 +162,7 @@ void FIO_setNotificationLevel(unsigned level) { g_displayLevel=level; } } -/* ************************************* +/*-************************************* * Functions ***************************************/ static unsigned FIO_GetMilliSpan(clock_t nPrevious) @@ -284,6 +286,7 @@ typedef struct { size_t dictBufferSize; ZBUFF_CCtx* ctx; FILE* dstFile; + FILE* srcFile; } cRess_t; static cRess_t FIO_createCResources(const char* dictFileName) @@ -326,19 +329,15 @@ static int FIO_compressFilename_internal(cRess_t ress, const char* dstFileName, const char* srcFileName, int cLevel) { - FILE* srcFile; + FILE* srcFile = ress.srcFile; FILE* dstFile = ress.dstFile; U64 filesize = 0; U64 compressedfilesize = 0; size_t dictSize = ress.dictBufferSize; size_t sizeCheck, errorCode; - /* File check */ - srcFile = FIO_openSrcFile(srcFileName); - if (!srcFile) return 1; /* srcFile could not be opened */ - /* init */ - filesize = FIO_getFileSize(srcFileName) + dictSize; + filesize = MAX(FIO_getFileSize(srcFileName),dictSize); errorCode = ZBUFF_compressInit_advanced(ress.ctx, ress.dictBuffer, ress.dictBufferSize, ZSTD_getParams(cLevel, filesize)); if (ZBUFF_isError(errorCode)) EXM_THROW(21, "Error initializing compression : %s", ZBUFF_getErrorName(errorCode)); @@ -385,13 +384,33 @@ static int FIO_compressFilename_internal(cRess_t ress, DISPLAYLEVEL(2,"Compressed %llu bytes into %llu bytes ==> %.2f%%\n", (unsigned long long) filesize, (unsigned long long) compressedfilesize, (double)compressedfilesize/filesize*100); - /* clean */ - fclose(srcFile); - return 0; } +/*! FIO_compressFilename_internal() : + * same as FIO_compressFilename_extRess(), with ress.desFile already opened + * @return : 0 : compression completed correctly, + * 1 : missing or pb opening srcFileName + */ +static int FIO_compressFilename_srcFile(cRess_t ress, + const char* dstFileName, const char* srcFileName, + int cLevel) +{ + int result; + + /* File check */ + ress.srcFile = FIO_openSrcFile(srcFileName); + if (!ress.srcFile) return 1; /* srcFile could not be opened */ + + result = FIO_compressFilename_internal(ress, dstFileName, srcFileName, cLevel); + + /* clean */ + fclose(ress.srcFile); + return result; +} + + /*! FIO_compressFilename_extRess() : * @return : 0 : compression completed correctly, * 1 : missing or pb opening srcFileName @@ -402,11 +421,14 @@ static int FIO_compressFilename_extRess(cRess_t ress, { int result; + ress.srcFile = FIO_openSrcFile(srcFileName); + if (ress.srcFile==0) return 1; ress.dstFile = FIO_openDstFile(dstFileName); - if (ress.dstFile==0) return 1; + if (ress.dstFile==0) { fclose(ress.srcFile); return 1; } result = FIO_compressFilename_internal(ress, dstFileName, srcFileName, cLevel); + fclose(ress.srcFile); /* no pb to expect : only reading */ if (fclose(ress.dstFile)) EXM_THROW(28, "Write error : cannot properly close %s", dstFileName); return result; } @@ -456,7 +478,13 @@ int FIO_compressMultipleFilenames(const char** inFileNamesTable, unsigned nbFile ress = FIO_createCResources(dictFileName); /* loop on each file */ - if (suffix) { + if (!strcmp(suffix, stdoutmark)) { + ress.dstFile = stdout; + for (u=0; u tmpdec : " cp tmpall tmpall2 $ZSTD -dc tmpall* > tmpdec ls -ls tmp* +echo "compress multiple files including a missing one (notHere) : " $ZSTD -f tmp1 notHere tmp2 && die "missing file not detected!" -rm tmp* + +echo "\n**** integrity tests **** " +echo "test one file (tmp1.zst) " +$ZSTD -t tmp1.zst +echo "test multiple files (*.zst) " +$ZSTD -t *.zst +echo "test good and bad files (*) " +$ZSTD -t * && die "bad files not detected !" echo "\n**** zstd round-trip tests **** " roundTripTest -roundTripTest '' 6 -roundTripTest '' 16 +roundTripTest -g512K 6 # greedy, hash chain +roundTripTest -g512K 16 # btlazy2 +roundTripTest -g512K 19 # btopt + +rm tmp* if [ "$1" != "--test-large-data" ]; then echo "Skipping large data tests" @@ -118,3 +129,6 @@ roundTripTest -g50000000 -P94 19 roundTripTest -g99000000 -P99 20 roundTripTest -g6000000000 -P99 q + +rm tmp* +