From 03cc84fddb9747ddc63f76b6f3171cd8f66da202 Mon Sep 17 00:00:00 2001 From: Nick Terrell Date: Thu, 4 Aug 2022 17:15:59 -0700 Subject: [PATCH] Add explicit --pass-through flag and default to enabled for *cat (#3223) Fixes #3211. Adds the `--[no-]pass-through` flag which enables/disables pass-through mode. * `zstdcat`, `zcat`, and `gzcat` default to `--pass-through`. Pass-through mode can be disabled by passing `--no-pass-through`. * All other binaries default to not setting pass-through mode. However, we preserve the legacy behavior of enabling pass-through mode when writing to stdout with `-f` set, unless pass-through mode is explicitly disabled with `--no-pass-through`. Adds a new test for this behavior that should codify the behavior we want. --- programs/fileio.c | 19 ++++++- programs/fileio.h | 1 + programs/fileio_types.h | 1 + programs/zstdcli.c | 17 +++++- tests/cli-tests/decompression/pass-through.sh | 57 +++++++++++++++++++ .../pass-through.sh.stderr.exact | 10 ++++ .../pass-through.sh.stdout.exact | 25 ++++++++ 7 files changed, 125 insertions(+), 5 deletions(-) create mode 100755 tests/cli-tests/decompression/pass-through.sh create mode 100644 tests/cli-tests/decompression/pass-through.sh.stderr.exact create mode 100644 tests/cli-tests/decompression/pass-through.sh.stdout.exact diff --git a/programs/fileio.c b/programs/fileio.c index 16518131..96cf602a 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -290,6 +290,7 @@ FIO_prefs_t* FIO_createPreferences(void) ret->excludeCompressedFiles = 0; ret->allowBlockDevices = 0; ret->asyncIO = AIO_supported(); + ret->passThrough = -1; return ret; } @@ -463,6 +464,10 @@ void FIO_setAsyncIOFlag(FIO_prefs_t* const prefs, int value) { #endif } +void FIO_setPassThroughFlag(FIO_prefs_t* const prefs, int value) { + prefs->passThrough = (value != 0); +} + /* FIO_ctx_t functions */ void FIO_setHasStdoutOutput(FIO_ctx_t* const fCtx, int value) { @@ -2336,6 +2341,16 @@ static int FIO_decompressFrames(FIO_ctx_t* const fCtx, { unsigned readSomething = 0; unsigned long long filesize = 0; + int passThrough = prefs->passThrough; + + if (passThrough == -1) { + /* If pass-through mode is not explicitly enabled or disabled, + * default to the legacy behavior of enabling it if we are writing + * to stdout with the overwrite flag enabled. + */ + passThrough = prefs->overwrite && !strcmp(dstFileName, stdoutmark); + } + assert(passThrough == 0 || passThrough == 1); /* for each frame */ for ( ; ; ) { @@ -2353,7 +2368,7 @@ static int FIO_decompressFrames(FIO_ctx_t* const fCtx, } readSomething = 1; /* there is at least 1 byte in srcFile */ if (ress.readCtx->srcBufferLoaded < toRead) { /* not enough input to check magic number */ - if ((prefs->overwrite) && !strcmp (dstFileName, stdoutmark)) { /* pass-through mode */ + if (passThrough) { return FIO_passThrough(&ress); } DISPLAYLEVEL(1, "zstd: %s: unknown header \n", srcFileName); @@ -2391,7 +2406,7 @@ static int FIO_decompressFrames(FIO_ctx_t* const fCtx, DISPLAYLEVEL(1, "zstd: %s: lz4 file cannot be uncompressed (zstd compiled without HAVE_LZ4) -- ignored \n", srcFileName); return 1; #endif - } else if ((prefs->overwrite) && !strcmp (dstFileName, stdoutmark)) { /* pass-through mode */ + } else if (passThrough) { return FIO_passThrough(&ress); } else { DISPLAYLEVEL(1, "zstd: %s: unsupported format \n", srcFileName); diff --git a/programs/fileio.h b/programs/fileio.h index f614aa04..b848934b 100644 --- a/programs/fileio.h +++ b/programs/fileio.h @@ -105,6 +105,7 @@ void FIO_setPatchFromMode(FIO_prefs_t* const prefs, int value); void FIO_setContentSize(FIO_prefs_t* const prefs, int value); void FIO_displayCompressionParameters(const FIO_prefs_t* prefs); void FIO_setAsyncIOFlag(FIO_prefs_t* const prefs, int value); +void FIO_setPassThroughFlag(FIO_prefs_t* const prefs, int value); /* FIO_ctx_t functions */ void FIO_setNbFilesTotal(FIO_ctx_t* const fCtx, int value); diff --git a/programs/fileio_types.h b/programs/fileio_types.h index c47adb3a..a1fac2ca 100644 --- a/programs/fileio_types.h +++ b/programs/fileio_types.h @@ -68,6 +68,7 @@ typedef struct FIO_prefs_s { int patchFromMode; int contentSize; int allowBlockDevices; + int passThrough; } FIO_prefs_t; #endif /* FILEIO_TYPES_HEADER */ diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 3e4510ab..47ef388f 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -264,6 +264,15 @@ static void usage_advanced(const char* programName) # else DISPLAYOUT(" --[no-]sparse sparse mode (default: disabled)\n"); # endif + { + char const* passThroughDefault = "disabled"; + if (exeNameMatch(programName, ZSTD_CAT) || + exeNameMatch(programName, ZSTD_ZCAT) || + exeNameMatch(programName, ZSTD_GZCAT)) { + passThroughDefault = "enabled"; + } + DISPLAYOUT(" --[no-]pass-through : passes through uncompressed files as-is (default: %s\n)", passThroughDefault); + } #endif /* ZSTD_NODECOMPRESS */ #ifndef ZSTD_NODICT @@ -870,14 +879,14 @@ int main(int argCount, const char* argv[]) /* preset behaviors */ if (exeNameMatch(programName, ZSTD_ZSTDMT)) nbWorkers=0, singleThread=0; if (exeNameMatch(programName, ZSTD_UNZSTD)) operation=zom_decompress; - if (exeNameMatch(programName, ZSTD_CAT)) { operation=zom_decompress; FIO_overwriteMode(prefs); forceStdout=1; followLinks=1; outFileName=stdoutmark; g_displayLevel=1; } /* supports multiple formats */ - if (exeNameMatch(programName, ZSTD_ZCAT)) { operation=zom_decompress; FIO_overwriteMode(prefs); forceStdout=1; followLinks=1; outFileName=stdoutmark; g_displayLevel=1; } /* behave like zcat, also supports multiple formats */ + if (exeNameMatch(programName, ZSTD_CAT)) { operation=zom_decompress; FIO_overwriteMode(prefs); forceStdout=1; followLinks=1; FIO_setPassThroughFlag(prefs, 1); outFileName=stdoutmark; g_displayLevel=1; } /* supports multiple formats */ + if (exeNameMatch(programName, ZSTD_ZCAT)) { operation=zom_decompress; FIO_overwriteMode(prefs); forceStdout=1; followLinks=1; FIO_setPassThroughFlag(prefs, 1); outFileName=stdoutmark; g_displayLevel=1; } /* behave like zcat, also supports multiple formats */ if (exeNameMatch(programName, ZSTD_GZ)) { /* behave like gzip */ suffix = GZ_EXTENSION; FIO_setCompressionType(prefs, FIO_gzipCompression); FIO_setRemoveSrcFile(prefs, 1); dictCLevel = cLevel = 6; /* gzip default is -6 */ } if (exeNameMatch(programName, ZSTD_GUNZIP)) { operation=zom_decompress; FIO_setRemoveSrcFile(prefs, 1); } /* behave like gunzip, also supports multiple formats */ - if (exeNameMatch(programName, ZSTD_GZCAT)) { operation=zom_decompress; FIO_overwriteMode(prefs); forceStdout=1; followLinks=1; outFileName=stdoutmark; g_displayLevel=1; } /* behave like gzcat, also supports multiple formats */ + if (exeNameMatch(programName, ZSTD_GZCAT)) { operation=zom_decompress; FIO_overwriteMode(prefs); forceStdout=1; followLinks=1; FIO_setPassThroughFlag(prefs, 1); outFileName=stdoutmark; g_displayLevel=1; } /* behave like gzcat, also supports multiple formats */ if (exeNameMatch(programName, ZSTD_LZMA)) { suffix = LZMA_EXTENSION; FIO_setCompressionType(prefs, FIO_lzmaCompression); FIO_setRemoveSrcFile(prefs, 1); } /* behave like lzma */ if (exeNameMatch(programName, ZSTD_UNLZMA)) { operation=zom_decompress; FIO_setCompressionType(prefs, FIO_lzmaCompression); FIO_setRemoveSrcFile(prefs, 1); } /* behave like unlzma, also supports multiple formats */ if (exeNameMatch(programName, ZSTD_XZ)) { suffix = XZ_EXTENSION; FIO_setCompressionType(prefs, FIO_xzCompression); FIO_setRemoveSrcFile(prefs, 1); } /* behave like xz */ @@ -926,6 +935,8 @@ int main(int argCount, const char* argv[]) if (!strcmp(argument, "--no-check")) { FIO_setChecksumFlag(prefs, 0); continue; } if (!strcmp(argument, "--sparse")) { FIO_setSparseWrite(prefs, 2); continue; } if (!strcmp(argument, "--no-sparse")) { FIO_setSparseWrite(prefs, 0); continue; } + if (!strcmp(argument, "--pass-through")) { FIO_setPassThroughFlag(prefs, 1); continue; } + if (!strcmp(argument, "--no-pass-through")) { FIO_setPassThroughFlag(prefs, 0); continue; } if (!strcmp(argument, "--test")) { operation=zom_test; continue; } if (!strcmp(argument, "--asyncio")) { FIO_setAsyncIOFlag(prefs, 1); continue;} if (!strcmp(argument, "--no-asyncio")) { FIO_setAsyncIOFlag(prefs, 0); continue;} diff --git a/tests/cli-tests/decompression/pass-through.sh b/tests/cli-tests/decompression/pass-through.sh new file mode 100755 index 00000000..2cab463f --- /dev/null +++ b/tests/cli-tests/decompression/pass-through.sh @@ -0,0 +1,57 @@ +#!/bin/sh + +set -e + +. "$COMMON/platform.sh" + +echo "" > 1 +echo "2" > 2 +echo "23" > 3 +echo "234" > 4 +echo "some data" > file + +println "+ passthrough enabled" + +zstd file + +# Test short files +zstd -dc --pass-through 1 2 3 4 + +# Test *cat symlinks +zstdcat file +"$ZSTD_SYMLINK_DIR/zcat" file +"$ZSTD_SYMLINK_DIR/gzcat" file + +# Test multiple files with mix of compressed & not +zstdcat file file.zst +zstdcat file.zst file + +# Test --pass-through +zstd -dc --pass-through file +zstd -d --pass-through file -o pass-through-file + +# Test legacy implicit passthrough with -fc +zstd -dcf file +zstd -dcf file file.zst +zstd -df < file +zstd -dcf < file file.zst - +zstd -dcf < file.zst file - + +$DIFF file pass-through-file + +println "+ passthrough disabled" + +# Test *cat +zstdcat --no-pass-through file && die "should fail" +"$ZSTD_SYMLINK_DIR/zcat" --no-pass-through file && die "should fail" +"$ZSTD_SYMLINK_DIR/gzcat" --no-pass-through file && die "should fail" +# Test zstd without implicit passthrough +zstd -d file -o no-pass-through-file && die "should fail" +zstd -d < file && die "should fail" + +# Test legacy implicit passthrough with -fc +zstd --no-pass-through -dcf file && die "should fail" +zstd --no-pass-through -dcf file file.zst && die "should fail" +zstd --no-pass-through -df < file && die "should fail" +zstd --no-pass-through -dcf < file file.zst - && die "should fail" +zstd --no-pass-through -dcf < file.zst file - && die "should fail" ||: diff --git a/tests/cli-tests/decompression/pass-through.sh.stderr.exact b/tests/cli-tests/decompression/pass-through.sh.stderr.exact new file mode 100644 index 00000000..f9ac13cb --- /dev/null +++ b/tests/cli-tests/decompression/pass-through.sh.stderr.exact @@ -0,0 +1,10 @@ +zstd: file: unsupported format +zstd: file: unsupported format +zstd: file: unsupported format +zstd: file: unsupported format +zstd: /*stdin*\: unsupported format +zstd: file: unsupported format +zstd: file: unsupported format +zstd: /*stdin*\: unsupported format +zstd: /*stdin*\: unsupported format +zstd: file: unsupported format diff --git a/tests/cli-tests/decompression/pass-through.sh.stdout.exact b/tests/cli-tests/decompression/pass-through.sh.stdout.exact new file mode 100644 index 00000000..b0d494c1 --- /dev/null +++ b/tests/cli-tests/decompression/pass-through.sh.stdout.exact @@ -0,0 +1,25 @@ ++ passthrough enabled + +2 +23 +234 +some data +some data +some data +some data +some data +some data +some data +some data +some data +some data +some data +some data +some data +some data +some data +some data ++ passthrough disabled +some data +some data +some data