From 01a1abfdb5a78cd4f63b0199be036b1546c4135c Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 5 May 2017 19:15:24 -0700 Subject: [PATCH] cli : -d and -t do not stop after a failed decompression The problematic srcfile will be named on console/log, but decompression/test will continue onto next file in the list. --- programs/fileio.c | 67 ++++++++++++++++++++++++++++++++++------------ tests/playTests.sh | 31 +++++++++++---------- 2 files changed, 67 insertions(+), 31 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index e188936b..ba15555d 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -206,8 +206,8 @@ void FIO_setOverlapLog(unsigned overlapLog){ static int FIO_remove(const char* path) { #if defined(_WIN32) || defined(WIN32) - /* windows doesn't allow remove read-only files, so try to make it - * writable first */ + /* windows doesn't allow remove read-only files, + * so try to make it writable first */ chmod(path, _S_IWRITE); #endif return remove(path); @@ -983,16 +983,19 @@ static unsigned FIO_passThrough(FILE* foutput, FILE* finput, void* buffer, size_ /** FIO_decompressFrame() : - @return : size of decoded frame + @return : size of decoded frame, or an error code */ +#define FIO_ERROR_ZSTD_DECODING ((unsigned long long)(-2)) unsigned long long FIO_decompressFrame(dRess_t* ress, FILE* finput, + const char* srcFileName, U64 alreadyDecoded) { U64 frameSize = 0; U32 storedSkips = 0; ZSTD_resetDStream(ress->dctx); + if (strlen(srcFileName)>20) srcFileName += strlen(srcFileName)-20; /* display last 20 characters */ /* Header loading (optional, saves one loop) */ { size_t const toRead = 9; @@ -1005,12 +1008,17 @@ unsigned long long FIO_decompressFrame(dRess_t* ress, ZSTD_inBuffer inBuff = { ress->srcBuffer, ress->srcBufferLoaded, 0 }; ZSTD_outBuffer outBuff= { ress->dstBuffer, ress->dstBufferSize, 0 }; size_t const readSizeHint = ZSTD_decompressStream(ress->dctx, &outBuff, &inBuff); - if (ZSTD_isError(readSizeHint)) EXM_THROW(36, "Decoding error : %s", ZSTD_getErrorName(readSizeHint)); + if (ZSTD_isError(readSizeHint)) { + DISPLAYLEVEL(1, "%s : Decoding error (36) : %s \n", + srcFileName, ZSTD_getErrorName(readSizeHint)); + return FIO_ERROR_ZSTD_DECODING; + } /* Write block */ storedSkips = FIO_fwriteSparse(ress->dstFile, ress->dstBuffer, outBuff.pos, storedSkips); frameSize += outBuff.pos; - DISPLAYUPDATE(2, "\rDecoded : %u MB... ", (U32)((alreadyDecoded+frameSize)>>20) ); + DISPLAYUPDATE(2, "\r%-20.20s : %u MB... ", + srcFileName, (U32)((alreadyDecoded+frameSize)>>20) ); if (inBuff.pos > 0) { memmove(ress->srcBuffer, (char*)ress->srcBuffer + inBuff.pos, inBuff.size - inBuff.pos); @@ -1018,14 +1026,22 @@ unsigned long long FIO_decompressFrame(dRess_t* ress, } if (readSizeHint == 0) break; /* end of frame */ - if (inBuff.size != inBuff.pos) EXM_THROW(37, "Decoding error : should consume entire input"); + if (inBuff.size != inBuff.pos) { + DISPLAYLEVEL(1, "%s : Decoding error (37) : should consume entire input \n", + srcFileName); + return FIO_ERROR_ZSTD_DECODING; + } /* Fill input buffer */ { size_t const toRead = MIN(readSizeHint, ress->srcBufferSize); /* support large skippable frames */ if (ress->srcBufferLoaded < toRead) - ress->srcBufferLoaded += fread(((char*)ress->srcBuffer) + ress->srcBufferLoaded, 1, toRead - ress->srcBufferLoaded, finput); - if (ress->srcBufferLoaded < toRead) EXM_THROW(39, "Read error : premature end"); - } } + ress->srcBufferLoaded += fread((char*)ress->srcBuffer + ress->srcBufferLoaded, + 1, toRead - ress->srcBufferLoaded, finput); + if (ress->srcBufferLoaded < toRead) { + DISPLAYLEVEL(1, "%s : Read error (39) : premature end \n", + srcFileName); + return FIO_ERROR_ZSTD_DECODING; + } } } FIO_fwriteSparseEnd(ress->dstFile, storedSkips); @@ -1221,9 +1237,14 @@ static int FIO_decompressSrcFile(dRess_t ress, const char* dstFileName, const ch size_t const toRead = 4; const BYTE* buf = (const BYTE*)ress.srcBuffer; if (ress.srcBufferLoaded < toRead) - ress.srcBufferLoaded += fread((char*)ress.srcBuffer + ress.srcBufferLoaded, (size_t)1, toRead - ress.srcBufferLoaded, srcFile); + ress.srcBufferLoaded += fread((char*)ress.srcBuffer + ress.srcBufferLoaded, + (size_t)1, toRead - ress.srcBufferLoaded, srcFile); if (ress.srcBufferLoaded==0) { - if (readSomething==0) { DISPLAY("zstd: %s: unexpected end of file \n", srcFileName); fclose(srcFile); return 1; } /* srcFileName is empty */ + if (readSomething==0) { + DISPLAY("zstd: %s: unexpected end of file \n", srcFileName); + fclose(srcFile); + return 1; + } /* srcFileName is empty */ break; /* no more input */ } readSomething = 1; /* there is at least >= 4 bytes in srcFile */ @@ -1259,7 +1280,8 @@ static int FIO_decompressSrcFile(dRess_t ress, const char* dstFileName, const ch } else { if (!ZSTD_isFrame(ress.srcBuffer, toRead)) { if ((g_overwrite) && !strcmp (dstFileName, stdoutmark)) { /* pass-through mode */ - unsigned const result = FIO_passThrough(ress.dstFile, srcFile, ress.srcBuffer, ress.srcBufferSize, ress.srcBufferLoaded); + unsigned const result = FIO_passThrough(ress.dstFile, srcFile, + ress.srcBuffer, ress.srcBufferSize, ress.srcBufferLoaded); if (fclose(srcFile)) EXM_THROW(32, "zstd: %s close error", srcFileName); /* error should never happen */ return result; } else { @@ -1267,9 +1289,15 @@ static int FIO_decompressSrcFile(dRess_t ress, const char* dstFileName, const ch fclose(srcFile); return 1; } } - filesize += FIO_decompressFrame(&ress, srcFile, filesize); + { unsigned long long const frameSize = FIO_decompressFrame(&ress, srcFile, srcFileName, filesize); + if (frameSize == FIO_ERROR_ZSTD_DECODING) { + fclose(srcFile); + return 1; + } + filesize += frameSize; + } } - } + } /* for each frame */ /* Final Status */ DISPLAYLEVEL(2, "\r%79s\r", ""); @@ -1364,15 +1392,20 @@ int FIO_decompressMultipleFilenames(const char** srcNamesTable, unsigned nbFiles dstFileName = (char*)malloc(dfnSize); if (dstFileName==NULL) EXM_THROW(74, "not enough memory for dstFileName"); } - if (sfnSize <= suffixSize || (strcmp(suffixPtr, GZ_EXTENSION) && strcmp(suffixPtr, XZ_EXTENSION) && strcmp(suffixPtr, ZSTD_EXTENSION) && strcmp(suffixPtr, LZMA_EXTENSION) && strcmp(suffixPtr, LZ4_EXTENSION))) { - DISPLAYLEVEL(1, "zstd: %s: unknown suffix (%s/%s/%s/%s expected) -- ignored \n", srcFileName, GZ_EXTENSION, XZ_EXTENSION, ZSTD_EXTENSION, LZMA_EXTENSION); + if (sfnSize <= suffixSize + || (strcmp(suffixPtr, GZ_EXTENSION) + && strcmp(suffixPtr, XZ_EXTENSION) + && strcmp(suffixPtr, ZSTD_EXTENSION) + && strcmp(suffixPtr, LZMA_EXTENSION) + && strcmp(suffixPtr, LZ4_EXTENSION)) ) { + DISPLAYLEVEL(1, "zstd: %s: unknown suffix (%s/%s/%s/%s/%s expected) -- ignored \n", + srcFileName, GZ_EXTENSION, XZ_EXTENSION, ZSTD_EXTENSION, LZMA_EXTENSION, LZ4_EXTENSION); skippedFiles++; continue; } else { memcpy(dstFileName, srcFileName, sfnSize - suffixSize); dstFileName[sfnSize-suffixSize] = '\0'; } - missingFiles += FIO_decompressDstFile(ress, dstFileName, srcFileName); } free(dstFileName); diff --git a/tests/playTests.sh b/tests/playTests.sh index 021fd59f..e6957458 100755 --- a/tests/playTests.sh +++ b/tests/playTests.sh @@ -92,7 +92,7 @@ $ZSTD tmp --stdout > tmpCompressed # long command format $ECHO "test : compress to named file" rm tmpCompressed $ZSTD tmp -o tmpCompressed -ls tmpCompressed # must work +test -f tmpCompressed # file must be created $ECHO "test : -o must be followed by filename (must fail)" $ZSTD tmp -of tmpCompressed && die "-o must be followed by filename " $ECHO "test : force write, correct order" @@ -142,21 +142,21 @@ $ZSTD -q -f tmpro rm -f tmpro tmpro.zst $ECHO "test : file removal" $ZSTD -f --rm tmp -ls tmp && die "tmp should no longer be present" +test ! -f tmp # tmp should no longer be present $ZSTD -f -d --rm tmp.zst -ls tmp.zst && die "tmp.zst should no longer be present" +test ! -f tmp.zst # tmp.zst should no longer be present $ECHO "test : --rm on stdin" $ECHO a | $ZSTD --rm > $INTOVOID # --rm should remain silent rm tmp $ZSTD -f tmp && die "tmp not present : should have failed" -ls tmp.zst && die "tmp.zst should not be created" +test ! -f tmp.zst # tmp.zst should not be created $ECHO "\n**** Advanced compression parameters **** " $ECHO "Hello world!" | $ZSTD --zstd=windowLog=21, - -o tmp.zst && die "wrong parameters not detected!" $ECHO "Hello world!" | $ZSTD --zstd=windowLo=21 - -o tmp.zst && die "wrong parameters not detected!" $ECHO "Hello world!" | $ZSTD --zstd=windowLog=21,slog - -o tmp.zst && die "wrong parameters not detected!" -ls tmp.zst && die "tmp.zst should not be created" +test ! -f tmp.zst # tmp.zst should not be created roundTripTest -g512K roundTripTest -g512K " --zstd=slen=3,tlen=48,strat=6" roundTripTest -g512K " --zstd=strat=6,wlog=23,clog=23,hlog=22,slog=6" @@ -201,16 +201,17 @@ $ECHO foo | $ZSTD > /dev/full && die "write error not detected!" $ECHO "$ECHO foo | $ZSTD | $ZSTD -d > /dev/full" $ECHO foo | $ZSTD | $ZSTD -d > /dev/full && die "write error not detected!" + $ECHO "\n**** symbolic link test **** " rm -f hello.tmp world.tmp hello.tmp.zst world.tmp.zst $ECHO "hello world" > hello.tmp ln -s hello.tmp world.tmp $ZSTD world.tmp hello.tmp -ls hello.tmp.zst || die "regular file should have been compressed!" -ls world.tmp.zst && die "symbolic link should not have been compressed!" +test -f hello.tmp.zst # regular file should have been compressed! +test ! -f world.tmp.zst # symbolic link should not have been compressed! $ZSTD world.tmp hello.tmp -f -ls world.tmp.zst || die "symbol link should have been compressed with --force" +test -f world.tmp.zst # symbolic link should have been compressed with --force rm -f hello.tmp world.tmp hello.tmp.zst world.tmp.zst fi @@ -225,10 +226,10 @@ $ZSTD tmpSparse -c | $ZSTD -dv --sparse -c > tmpOutSparse $DIFF -s tmpSparse tmpOutSparse $ZSTD tmpSparse -c | $ZSTD -dv --no-sparse -c > tmpOutNoSparse $DIFF -s tmpSparse tmpOutNoSparse -ls -ls tmpSparse* +ls -ls tmpSparse* # look at file size and block size on disk ./datagen -s1 -g1200007 -P100 | $ZSTD | $ZSTD -dv --sparse -c > tmpSparseOdd # Odd size file (to not finish on an exact nb of blocks) ./datagen -s1 -g1200007 -P100 | $DIFF -s - tmpSparseOdd -ls -ls tmpSparseOdd +ls -ls tmpSparseOdd # look at file size and block size on disk $ECHO "\n Sparse Compatibility with Console :" $ECHO "Hello World 1 !" | $ZSTD | $ZSTD -d -c $ECHO "Hello World 2 !" | $ZSTD | $ZSTD -d | cat @@ -238,7 +239,7 @@ cat tmpSparse1M tmpSparse1M > tmpSparse2M $ZSTD -v -f tmpSparse1M -o tmpSparseCompressed $ZSTD -d -v -f tmpSparseCompressed -o tmpSparseRegenerated $ZSTD -d -v -f tmpSparseCompressed -c >> tmpSparseRegenerated -ls -ls tmpSparse* +ls -ls tmpSparse* # look at file size and block size on disk $DIFF tmpSparse2M tmpSparseRegenerated rm tmpSparse* @@ -257,11 +258,11 @@ $ZSTD -df *.zst ls -ls tmp* $ECHO "compress tmp* into stdout > tmpall : " $ZSTD -c tmp1 tmp2 tmp3 > tmpall -ls -ls tmp* +ls -ls tmp* # check size of tmpall (should be tmp1.zst + tmp2.zst + tmp3.zst) $ECHO "decompress tmpall* into stdout > tmpdec : " cp tmpall tmpall2 $ZSTD -dc tmpall* > tmpdec -ls -ls tmp* +ls -ls tmp* # check size of tmpdec (should be 2*(tmp1 + tmp2 + tmp3)) $ECHO "compress multiple files including a missing one (notHere) : " $ZSTD -f tmp1 notHere tmp2 && die "missing file not detected!" @@ -379,7 +380,9 @@ $ZSTD -t tmp2.zst && die "bad file not detected !" $ZSTD -t tmp3 && die "bad file not detected !" # detects 0-sized files as bad $ECHO "test --rm and --test combined " $ZSTD -t --rm tmp1.zst -ls -ls tmp1.zst # check file is still present +test -f tmp1.zst # check file is still present +split -b16384 tmp1.zst tmpSplit. +$ZSTD -t tmpSplit.* && die "bad file not detected !" $ECHO "\n**** benchmark mode tests **** "