Addressing comments, removing cyclic dependency with header file, updating tests

programs/fileio.c

@@ -319,6 +319,8 @@ struct FIO_prefs_s {
     /* Computation resources preferences */
     unsigned memLimit;
     int nbWorkers;
+
+    int excludeCompressedFiles;
 };
 
 
@@ -359,6 +361,7 @@ FIO_prefs_t* FIO_createPreferences(void)
     ret->srcSizeHint = 0;
     ret->testMode = 0;
     ret->literalCompressionMode = ZSTD_lcm_auto;
+    ret->excludeCompressedFiles = 0;
     return ret;
 }
 
@@ -402,6 +405,8 @@ void FIO_setNbWorkers(FIO_prefs_t* const prefs, int nbWorkers) {
     prefs->nbWorkers = nbWorkers;
 }
 
+void FIO_setExcludeCompressedFile(FIO_prefs_t* const prefs, int excludeCompressedFiles) { prefs->excludeCompressedFiles = excludeCompressedFiles; }
+
 void FIO_setBlockSize(FIO_prefs_t* const prefs, int blockSize) {
     if (blockSize && prefs->nbWorkers==0)
         DISPLAYLEVEL(2, "Setting block size is useless in single-thread mode \n");
@@ -1425,6 +1430,18 @@ static int FIO_compressFilename_dstFile(FIO_prefs_t* const prefs,
     return result;
 }
 
+static const char *compressedFileExtensions[] = {
+    ZSTD_EXTENSION,
+    TZSTD_EXTENSION,
+    GZ_EXTENSION,
+    TGZ_EXTENSION,
+    LZMA_EXTENSION,
+    XZ_EXTENSION,
+    TXZ_EXTENSION,
+    LZ4_EXTENSION,
+    TLZ4_EXTENSION,
+    NULL
+};
 
 /*! FIO_compressFilename_srcFile() :
  *  @return : 0 : compression completed correctly,
@@ -1451,19 +1468,18 @@ FIO_compressFilename_srcFile(FIO_prefs_t* const prefs,
         return 1;
     }
 
+    /* Check if "srcFile" is compressed. Only done if --exclude-compressed flag is used
+    * YES => ZSTD will skip compression of the file and will return 0.
+    * NO => ZSTD will resume with compress operation.
+    */
+    if (prefs->excludeCompressedFiles == 1 && UTIL_isCompressedFile(srcFileName, compressedFileExtensions)) {
+        DISPLAYLEVEL(4, "File is already compressed : %s \n", srcFileName);
+        return 0;
+    }
+
     ress.srcFile = FIO_openSrcFile(srcFileName);
     if (ress.srcFile == NULL) return 1;   /* srcFile could not be opened */
 
-    /* Check if "srcFile" is compressed. Only done if --exclude-compressed flag is used
-    * YES => ZSTD will not compress the file.
-    * NO => ZSTD will resume with compress operation.
-    */
-    if (g_excludeCompressedFiles && UTIL_isCompressedFile(srcFileName)) {  /* precompressed file (--exclude-compressed). DO NOT COMPRESS */
-        DISPLAYLEVEL(4, "File is already compressed : %s \n", srcFileName);
-        fclose(ress.srcFile);
-        ress.srcFile = NULL;
-        return 0;
-    }
     result = FIO_compressFilename_dstFile(prefs, ress, dstFileName, srcFileName, compressionLevel);
 
     fclose(ress.srcFile);

programs/fileio.h

@@ -93,6 +93,7 @@ void FIO_setLiteralCompressionMode(
 
 void FIO_setNoProgress(unsigned noProgress);
 void FIO_setNotificationLevel(int level);
+void FIO_setExcludeCompressedFile(FIO_prefs_t* const prefs, int excludeCompressedFiles);
 
 /*-*************************************
 *  Single File functions

programs/util.c

@@ -331,24 +331,18 @@ YES => Skip the file (return 0)
 NO => return 1
 */
 
-int UTIL_isCompressedFile(const char *inputName)
+int UTIL_isCompressedFile(const char *inputName, const char *extensionList[])
 {
-    return compareExtensions(inputName,g_compressedFileExtensions);
-}
-
-int compareExtensions(const char* infilename, const char* extensionList[])
-{
-  int i=0;
-   while(*extensionList != NULL)
+   while(*extensionList!=NULL)
    {
-     const char* ext = strstr(infilename,extensionList[i]);
+     const char* ext = strstr(inputName,*extensionList);
      if(ext)
         return 1;
       ++extensionList;
-      i++;
    }
    return 0;
 }
+
 /*
  * UTIL_createFileList - takes a list of files and directories (params: inputNames, inputNamesNb), scans directories,
  *                       and returns a new list of files (params: return value, allocatedBuffer, allocatedNamesNb).

programs/util.h

@@ -39,7 +39,6 @@ extern "C" {
 #endif
 #include <time.h>         /* clock_t, clock, CLOCKS_PER_SEC, nanosleep */
 #include "mem.h"          /* U32, U64 */
-#include "fileio.h"
 
 /*-************************************************************
 * Avoid fseek()'s 2GiB barrier with MSVC, macOS, *BSD, MinGW
@@ -127,19 +126,6 @@ extern int g_utilDisplayLevel;
     typedef struct stat stat_t;
 #endif
 
-int g_excludeCompressedFiles;
-static const char *g_compressedFileExtensions[] = {
-    ZSTD_EXTENSION,
-    TZSTD_EXTENSION,
-    GZ_EXTENSION,
-    TGZ_EXTENSION,
-    LZMA_EXTENSION,
-    XZ_EXTENSION,
-    TXZ_EXTENSION,
-    LZ4_EXTENSION,
-    TLZ4_EXTENSION,
-    NULL
-};
 
 int UTIL_fileExist(const char* filename);
 int UTIL_isRegularFile(const char* infilename);
@@ -148,9 +134,7 @@ U32 UTIL_isDirectory(const char* infilename);
 int UTIL_getFileStat(const char* infilename, stat_t* statbuf);
 int UTIL_isSameFile(const char* file1, const char* file2);
 int UTIL_compareStr(const void *p1, const void *p2);
-int UTIL_isCompressedFile(const char* infilename);
-int compareExtensions(const char* infilename, const char *extensionList[]);
-
+int UTIL_isCompressedFile(const char* infilename, const char *extensionList[]);
 U32 UTIL_isFIFO(const char* infilename);
 U32 UTIL_isLink(const char* infilename);
 #define UTIL_FILESIZE_UNKNOWN  ((U64)(-1))

programs/zstdcli.c

@@ -709,7 +709,7 @@ int main(int argCount, const char* argv[])
                     if (!strcmp(argument, "--compress-literals")) { literalCompressionMode = ZSTD_lcm_huffman; continue; }
                     if (!strcmp(argument, "--no-compress-literals")) { literalCompressionMode = ZSTD_lcm_uncompressed; continue; }
                     if (!strcmp(argument, "--no-progress")) { FIO_setNoProgress(1); continue; }
-                    if (!strcmp(argument, "--exclude-compressed")) { g_excludeCompressedFiles = 1; continue; }
+                    if (!strcmp(argument, "--exclude-compressed")) { FIO_setExcludeCompressedFile(prefs, 1); continue; }
                     /* long commands with arguments */
 #ifndef ZSTD_NODICT
                     if (longCommandWArg(&argument, "--train-cover")) {

tests/playTests.sh

@@ -224,6 +224,8 @@ sleep 5
 ./datagen $size > precompressedFilterTestDir/input.7
 ./datagen $size > precompressedFilterTestDir/input.8
 $ZSTD --exclude-compressed --long --rm -r precompressedFilterTestDir
+test ! -f input.5.zst.zst
+test ! -f input.6.zst.zst
 file1timestamp=`date -r precompressedFilterTestDir/input.5.zst +%s`
 file2timestamp=`date -r precompressedFilterTestDir/input.7.zst +%s`
 if [[ $file2timestamp -ge $file1timestamp ]]; then
@@ -232,6 +234,7 @@ else
   println "Test is not successful"
 fi
 println "Test completed"
+sleep 5
 
 println "test : file removal"
 $ZSTD -f --rm tmp