diff --git a/programs/fileio.c b/programs/fileio.c index 828878c6..ff3401b5 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -1453,7 +1453,12 @@ FIO_compressFilename_srcFile(FIO_prefs_t* const prefs, ress.srcFile = FIO_openSrcFile(srcFileName); if (ress.srcFile == NULL) return 1; /* srcFile could not be opened */ - + if (g_excludeCompressedFiles && !UTIL_isPrecompressedFile(srcFileName)) { /* precompressed file (--exclude-compressed). DO NOT COMPRESS */ + DISPLAYLEVEL(4, "Precompressed file: %s \n", srcFileName); + fclose(ress.srcFile); + ress.srcFile = NULL; + return 0; + } result = FIO_compressFilename_dstFile(prefs, ress, dstFileName, srcFileName, compressionLevel); fclose(ress.srcFile); diff --git a/programs/util.c b/programs/util.c index 58705880..830f2039 100644 --- a/programs/util.c +++ b/programs/util.c @@ -326,6 +326,27 @@ int UTIL_prepareFileList(const char *dirName, char** bufStart, size_t* pos, char #endif /* #ifdef _WIN32 */ +/* Check if the file is precompressed (.zst, .lz4, .gz, .xz). +YES => Skip the file (return 0) +NO => return 1 +*/ +int UTIL_isPrecompressedFile(const char *inputName) +{ + return compareExtensions(inputName,compressedFileExtensions); +} + +int compareExtensions(const char* infilename, const char extensionList[4][10]) +{ + int i=0; + //char* ext = strchr(infilename, '.'); + for(i=0;i<4;i++) + { + char* ext = strstr(infilename,extensionList[i]); + if(ext) + return 0; + } + return 1; +} /* * UTIL_createFileList - takes a list of files and directories (params: inputNames, inputNamesNb), scans directories, * and returns a new list of files (params: return value, allocatedBuffer, allocatedNamesNb). diff --git a/programs/util.h b/programs/util.h index 71ba0d4f..0da6255c 100644 --- a/programs/util.h +++ b/programs/util.h @@ -127,6 +127,8 @@ extern int g_utilDisplayLevel; typedef struct stat stat_t; #endif +int g_excludeCompressedFiles; +static const char compressedFileExtensions[4][10] = {".zst",".gz",".xz",".lz4"}; int UTIL_fileExist(const char* filename); int UTIL_isRegularFile(const char* infilename); @@ -135,6 +137,8 @@ U32 UTIL_isDirectory(const char* infilename); int UTIL_getFileStat(const char* infilename, stat_t* statbuf); int UTIL_isSameFile(const char* file1, const char* file2); int UTIL_compareStr(const void *p1, const void *p2); +int UTIL_isPrecompressedFile(const char* infilename); +int compareExtensions(const char* infilename, const char extensionList[4][10]); U32 UTIL_isFIFO(const char* infilename); U32 UTIL_isLink(const char* infilename); diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 3d5c4280..11116afc 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -118,6 +118,7 @@ static int usage(const char* programName) #endif DISPLAY( " -D file: use `file` as Dictionary \n"); DISPLAY( " -o file: result stored into `file` (only if 1 input file) \n"); + DISPLAY( "--exclude-compressed: only compress files that are not previously compressed \n"); DISPLAY( " -f : overwrite output without prompting and (de)compress links \n"); DISPLAY( "--rm : remove source file(s) after successful de/compression \n"); DISPLAY( " -k : preserve source file(s) (default) \n"); @@ -708,7 +709,7 @@ int main(int argCount, const char* argv[]) if (!strcmp(argument, "--compress-literals")) { literalCompressionMode = ZSTD_lcm_huffman; continue; } if (!strcmp(argument, "--no-compress-literals")) { literalCompressionMode = ZSTD_lcm_uncompressed; continue; } if (!strcmp(argument, "--no-progress")) { FIO_setNoProgress(1); continue; } - + if (!strcmp(argument, "--exclude-compressed")) { g_excludeCompressedFiles = 1; continue; } /* long commands with arguments */ #ifndef ZSTD_NODICT if (longCommandWArg(&argument, "--train-cover")) { diff --git a/tests/playTests.sh b/tests/playTests.sh index c1da1650..9294bf81 100755 --- a/tests/playTests.sh +++ b/tests/playTests.sh @@ -215,6 +215,25 @@ $ZSTD tmp -c --compress-literals -19 | $ZSTD -t $ZSTD -b --fast=1 -i0e1 tmp --compress-literals $ZSTD -b --fast=1 -i0e1 tmp --no-compress-literals +println "test: --exclude-compressed flag" +mkdir precompressedFilterTestDir +./datagen $size > precompressedFilterTestDir/input.5 +./datagen $size > precompressedFilterTestDir/input.6 +$ZSTD --exclude-compressed --long --rm -r precompressedFilterTestDir +sleep 5 +./datagen $size > precompressedFilterTestDir/input.7 +./datagen $size > precompressedFilterTestDir/input.8 +$ZSTD --exclude-compressed --long --rm -r precompressedFilterTestDir +file1timestamp=`date -r precompressedFilterTestDir/input.5.zst +%s` +file2timestamp=`date -r precompressedFilterTestDir/input.7.zst +%s` +if [[ $file2timestamp -ge $file1timestamp ]]; then + println "Test is successful. input.5.zst is not precompressed and therefore not compressed/modified again." +else + println "Test is not successful" +fi +println "Test completed" +sleep 5 + println "test : file removal" $ZSTD -f --rm tmp test ! -f tmp # tmp should no longer be present