Add explicit --pass-through flag and default to enabled for *cat (#3223)

Fixes #3211.

Adds the `--[no-]pass-through` flag which enables/disables pass-through mode.

* `zstdcat`, `zcat`, and `gzcat` default to `--pass-through`.
  Pass-through mode can be disabled by passing `--no-pass-through`.
* All other binaries default to not setting pass-through mode.
  However, we preserve the legacy behavior of enabling pass-through
  mode when writing to stdout with `-f` set, unless pass-through
  mode is explicitly disabled with `--no-pass-through`.

Adds a new test for this behavior that should codify the behavior we want.

programs/fileio.c

@@ -290,6 +290,7 @@ FIO_prefs_t* FIO_createPreferences(void)
     ret->excludeCompressedFiles = 0;
     ret->allowBlockDevices = 0;
     ret->asyncIO = AIO_supported();
+    ret->passThrough = -1;
     return ret;
 }
 
@@ -463,6 +464,10 @@ void FIO_setAsyncIOFlag(FIO_prefs_t* const prefs, int value) {
 #endif
 }
 
+void FIO_setPassThroughFlag(FIO_prefs_t* const prefs, int value) {
+    prefs->passThrough = (value != 0);
+}
+
 /* FIO_ctx_t functions */
 
 void FIO_setHasStdoutOutput(FIO_ctx_t* const fCtx, int value) {
@@ -2336,6 +2341,16 @@ static int FIO_decompressFrames(FIO_ctx_t* const fCtx,
 {
     unsigned readSomething = 0;
     unsigned long long filesize = 0;
+    int passThrough = prefs->passThrough;
+
+    if (passThrough == -1) {
+        /* If pass-through mode is not explicitly enabled or disabled,
+         * default to the legacy behavior of enabling it if we are writing
+         * to stdout with the overwrite flag enabled.
+         */
+        passThrough = prefs->overwrite && !strcmp(dstFileName, stdoutmark);
+    }
+    assert(passThrough == 0 || passThrough == 1);
 
     /* for each frame */
     for ( ; ; ) {
@@ -2353,7 +2368,7 @@ static int FIO_decompressFrames(FIO_ctx_t* const fCtx,
         }
         readSomething = 1;   /* there is at least 1 byte in srcFile */
         if (ress.readCtx->srcBufferLoaded < toRead) { /* not enough input to check magic number */
-            if ((prefs->overwrite) && !strcmp (dstFileName, stdoutmark)) {  /* pass-through mode */
+            if (passThrough) {
                 return FIO_passThrough(&ress);
             }
             DISPLAYLEVEL(1, "zstd: %s: unknown header \n", srcFileName);
@@ -2391,7 +2406,7 @@ static int FIO_decompressFrames(FIO_ctx_t* const fCtx,
             DISPLAYLEVEL(1, "zstd: %s: lz4 file cannot be uncompressed (zstd compiled without HAVE_LZ4) -- ignored \n", srcFileName);
             return 1;
 #endif
-        } else if ((prefs->overwrite) && !strcmp (dstFileName, stdoutmark)) {  /* pass-through mode */
+        } else if (passThrough) {
             return FIO_passThrough(&ress);
         } else {
             DISPLAYLEVEL(1, "zstd: %s: unsupported format \n", srcFileName);

programs/fileio.h

@@ -105,6 +105,7 @@ void FIO_setPatchFromMode(FIO_prefs_t* const prefs, int value);
 void FIO_setContentSize(FIO_prefs_t* const prefs, int value);
 void FIO_displayCompressionParameters(const FIO_prefs_t* prefs);
 void FIO_setAsyncIOFlag(FIO_prefs_t* const prefs, int value);
+void FIO_setPassThroughFlag(FIO_prefs_t* const prefs, int value);
 
 /* FIO_ctx_t functions */
 void FIO_setNbFilesTotal(FIO_ctx_t* const fCtx, int value);

programs/fileio_types.h

@@ -68,6 +68,7 @@ typedef struct FIO_prefs_s {
     int patchFromMode;
     int contentSize;
     int allowBlockDevices;
+    int passThrough;
 } FIO_prefs_t;
 
 #endif /* FILEIO_TYPES_HEADER */

programs/zstdcli.c

@@ -264,6 +264,15 @@ static void usage_advanced(const char* programName)
 # else
     DISPLAYOUT("     --[no-]sparse     sparse mode (default: disabled)\n");
 # endif
+    {
+        char const* passThroughDefault = "disabled";
+        if (exeNameMatch(programName, ZSTD_CAT) ||
+            exeNameMatch(programName, ZSTD_ZCAT) ||
+            exeNameMatch(programName, ZSTD_GZCAT)) {
+            passThroughDefault = "enabled";
+        }
+        DISPLAYOUT("     --[no-]pass-through : passes through uncompressed files as-is (default: %s\n)", passThroughDefault);
+    }
 #endif  /* ZSTD_NODECOMPRESS */
 
 #ifndef ZSTD_NODICT
@@ -870,14 +879,14 @@ int main(int argCount, const char* argv[])
     /* preset behaviors */
     if (exeNameMatch(programName, ZSTD_ZSTDMT)) nbWorkers=0, singleThread=0;
     if (exeNameMatch(programName, ZSTD_UNZSTD)) operation=zom_decompress;
-    if (exeNameMatch(programName, ZSTD_CAT)) { operation=zom_decompress; FIO_overwriteMode(prefs); forceStdout=1; followLinks=1; outFileName=stdoutmark; g_displayLevel=1; }     /* supports multiple formats */
+    if (exeNameMatch(programName, ZSTD_CAT)) { operation=zom_decompress; FIO_overwriteMode(prefs); forceStdout=1; followLinks=1; FIO_setPassThroughFlag(prefs, 1); outFileName=stdoutmark; g_displayLevel=1; }     /* supports multiple formats */
-    if (exeNameMatch(programName, ZSTD_ZCAT)) { operation=zom_decompress; FIO_overwriteMode(prefs); forceStdout=1; followLinks=1; outFileName=stdoutmark; g_displayLevel=1; }    /* behave like zcat, also supports multiple formats */
+    if (exeNameMatch(programName, ZSTD_ZCAT)) { operation=zom_decompress; FIO_overwriteMode(prefs); forceStdout=1; followLinks=1; FIO_setPassThroughFlag(prefs, 1); outFileName=stdoutmark; g_displayLevel=1; }    /* behave like zcat, also supports multiple formats */
     if (exeNameMatch(programName, ZSTD_GZ)) {   /* behave like gzip */
         suffix = GZ_EXTENSION; FIO_setCompressionType(prefs, FIO_gzipCompression); FIO_setRemoveSrcFile(prefs, 1);
         dictCLevel = cLevel = 6;  /* gzip default is -6 */
     }
     if (exeNameMatch(programName, ZSTD_GUNZIP)) { operation=zom_decompress; FIO_setRemoveSrcFile(prefs, 1); }                                                     /* behave like gunzip, also supports multiple formats */
-    if (exeNameMatch(programName, ZSTD_GZCAT)) { operation=zom_decompress; FIO_overwriteMode(prefs); forceStdout=1; followLinks=1; outFileName=stdoutmark; g_displayLevel=1; }   /* behave like gzcat, also supports multiple formats */
+    if (exeNameMatch(programName, ZSTD_GZCAT)) { operation=zom_decompress; FIO_overwriteMode(prefs); forceStdout=1; followLinks=1; FIO_setPassThroughFlag(prefs, 1); outFileName=stdoutmark; g_displayLevel=1; }   /* behave like gzcat, also supports multiple formats */
     if (exeNameMatch(programName, ZSTD_LZMA)) { suffix = LZMA_EXTENSION; FIO_setCompressionType(prefs, FIO_lzmaCompression); FIO_setRemoveSrcFile(prefs, 1); }    /* behave like lzma */
     if (exeNameMatch(programName, ZSTD_UNLZMA)) { operation=zom_decompress; FIO_setCompressionType(prefs, FIO_lzmaCompression); FIO_setRemoveSrcFile(prefs, 1); } /* behave like unlzma, also supports multiple formats */
     if (exeNameMatch(programName, ZSTD_XZ)) { suffix = XZ_EXTENSION; FIO_setCompressionType(prefs, FIO_xzCompression); FIO_setRemoveSrcFile(prefs, 1); }          /* behave like xz */
@@ -926,6 +935,8 @@ int main(int argCount, const char* argv[])
                 if (!strcmp(argument, "--no-check")) { FIO_setChecksumFlag(prefs, 0); continue; }
                 if (!strcmp(argument, "--sparse")) { FIO_setSparseWrite(prefs, 2); continue; }
                 if (!strcmp(argument, "--no-sparse")) { FIO_setSparseWrite(prefs, 0); continue; }
+                if (!strcmp(argument, "--pass-through")) { FIO_setPassThroughFlag(prefs, 1); continue; }
+                if (!strcmp(argument, "--no-pass-through")) { FIO_setPassThroughFlag(prefs, 0); continue; }
                 if (!strcmp(argument, "--test")) { operation=zom_test; continue; }
                 if (!strcmp(argument, "--asyncio")) { FIO_setAsyncIOFlag(prefs, 1); continue;}
                 if (!strcmp(argument, "--no-asyncio")) { FIO_setAsyncIOFlag(prefs, 0); continue;}

tests/cli-tests/decompression/pass-through.sh

@@ -0,0 +1,57 @@
+#!/bin/sh
+
+set -e
+
+. "$COMMON/platform.sh"
+
+echo "" > 1
+echo "2" > 2
+echo "23" > 3
+echo "234" > 4
+echo "some data" > file
+
+println "+ passthrough enabled"
+
+zstd file
+
+# Test short files
+zstd -dc --pass-through 1 2 3 4
+
+# Test *cat symlinks
+zstdcat file
+"$ZSTD_SYMLINK_DIR/zcat" file
+"$ZSTD_SYMLINK_DIR/gzcat" file
+
+# Test multiple files with mix of compressed & not
+zstdcat file file.zst
+zstdcat file.zst file
+
+# Test --pass-through
+zstd -dc --pass-through file
+zstd -d --pass-through file -o pass-through-file
+
+# Test legacy implicit passthrough with -fc
+zstd -dcf file
+zstd -dcf file file.zst
+zstd -df < file
+zstd -dcf < file file.zst -
+zstd -dcf < file.zst file -
+
+$DIFF file pass-through-file
+
+println "+ passthrough disabled"
+
+# Test *cat
+zstdcat --no-pass-through file && die "should fail"
+"$ZSTD_SYMLINK_DIR/zcat" --no-pass-through file && die "should fail"
+"$ZSTD_SYMLINK_DIR/gzcat" --no-pass-through file && die "should fail"
+# Test zstd without implicit passthrough
+zstd -d file -o no-pass-through-file && die "should fail"
+zstd -d < file && die "should fail"
+
+# Test legacy implicit passthrough with -fc
+zstd --no-pass-through -dcf file && die "should fail"
+zstd --no-pass-through -dcf file file.zst && die "should fail"
+zstd --no-pass-through -df < file && die "should fail"
+zstd --no-pass-through -dcf < file file.zst - && die "should fail"
+zstd --no-pass-through -dcf < file.zst file - && die "should fail" ||:

tests/cli-tests/decompression/pass-through.sh.stderr.exact

@@ -0,0 +1,10 @@
+zstd: file: unsupported format 
+zstd: file: unsupported format 
+zstd: file: unsupported format 
+zstd: file: unsupported format 
+zstd: /*stdin*\: unsupported format 
+zstd: file: unsupported format 
+zstd: file: unsupported format 
+zstd: /*stdin*\: unsupported format 
+zstd: /*stdin*\: unsupported format 
+zstd: file: unsupported format 

tests/cli-tests/decompression/pass-through.sh.stdout.exact

@@ -0,0 +1,25 @@
++ passthrough enabled
+
+2
+23
+234
+some data
+some data
+some data
+some data
+some data
+some data
+some data
+some data
+some data
+some data
+some data
+some data
+some data
+some data
+some data
+some data
++ passthrough disabled
+some data
+some data
+some data