facebook/zstd
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #1848 from felixhandte/tzst-ext-tmp

Browse Source 
Support Decompressing Short Tar Extensions
This commit is contained in:
Yann Collet 2019-10-28 10:08:20 -07:00 committed by GitHub
commit b4037b18ef
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 145 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
build/cmake/README.md
View File

@ -5,9 +5,9 @@ use case sensitivity that matches modern (ie. cmake version 2.6 and above)
conventions of using lower-case for commands, and upper-case for conventions of using lower-case for commands, and upper-case for
variables. variables.
# How to build ## How to build
As cmake doesn't support command like `cmake clean`, it's recommanded to perform a "out of source build". As cmake doesn't support command like `cmake clean`, it's recommended to perform a "out of source build".
To do this, you can create a new directory and build in it: To do this, you can create a new directory and build in it:
```sh ```sh
cd build/cmake cd build/cmake
@ -16,7 +16,7 @@ cd builddir
cmake .. cmake ..
make make
``` ```
Then you can clean all cmake caches by simpily delete the new directory: Then you can clean all cmake caches by simply delete the new directory:
```sh ```sh
rm -rf build/cmake/builddir rm -rf build/cmake/builddir
``` ```
@ -34,19 +34,19 @@ cd build/cmake/builddir
cmake -LH .. cmake -LH ..
``` ```
Bool options can be set to ON/OFF with -D\[option\]=\[ON/OFF\]. You can configure cmake options like this: Bool options can be set to `ON/OFF` with `-D[option]=[ON/OFF]`. You can configure cmake options like this:
```sh ```sh
cd build/cmake/builddir cd build/cmake/builddir
cmake -DZSTD_BUILD_TESTS=ON -DZSTD_LEGACY_SUPPORT=ON .. cmake -DZSTD_BUILD_TESTS=ON -DZSTD_LEGACY_SUPPORT=ON ..
make make
``` ```
## referring ### referring
[Looking for a 'cmake clean' command to clear up CMake output](https://stackoverflow.com/questions/9680420/looking-for-a-cmake-clean-command-to-clear-up-cmake-output) [Looking for a 'cmake clean' command to clear up CMake output](https://stackoverflow.com/questions/9680420/looking-for-a-cmake-clean-command-to-clear-up-cmake-output)
# CMake Style Recommendations ## CMake Style Recommendations
## Indent all code correctly, i.e. the body of ### Indent all code correctly, i.e. the body of
* if/else/endif * if/else/endif
* foreach/endforeach * foreach/endforeach
@ -57,7 +57,7 @@ make
Use spaces for indenting, 2, 3 or 4 spaces preferably. Use the same amount of Use spaces for indenting, 2, 3 or 4 spaces preferably. Use the same amount of
spaces for indenting as is used in the rest of the file. Do not use tabs. spaces for indenting as is used in the rest of the file. Do not use tabs.
## Upper/lower casing ### Upper/lower casing
Most important: use consistent upper- or lowercasing within one file ! Most important: use consistent upper- or lowercasing within one file !
@ -77,7 +77,7 @@ Add_Executable(hello hello.c)
aDd_ExEcUtAbLe(blub blub.c) aDd_ExEcUtAbLe(blub blub.c)
``` ```
## End commands ### End commands
To make the code easier to read, use empty commands for endforeach(), endif(), To make the code easier to read, use empty commands for endforeach(), endif(),
endfunction(), endmacro() and endwhile(). Also, use empty else() commands. endfunction(), endmacro() and endwhile(). Also, use empty else() commands.
@ -99,6 +99,6 @@ if(BARVAR)
endif(BARVAR) endif(BARVAR)
``` ```
## Other resources for best practices ### Other resources for best practices
`https://cmake.org/cmake/help/latest/manual/cmake-developer.7.html#modules` https://cmake.org/cmake/help/latest/manual/cmake-developer.7.html#modules

126
programs/fileio.c
View File

@ -1496,17 +1496,17 @@ FIO_determineCompressedName(const char* srcFileName, const char* outDirName, con
static char* dstFileNameBuffer = NULL; /* using static allocation : this function cannot be multi-threaded */ static char* dstFileNameBuffer = NULL; /* using static allocation : this function cannot be multi-threaded */
char* outDirFilename = NULL; char* outDirFilename = NULL;
size_t sfnSize = strlen(srcFileName); size_t sfnSize = strlen(srcFileName);
size_t const suffixSize = strlen(suffix); size_t const srcSuffixLen = strlen(suffix);
if (outDirName) { if (outDirName) {
outDirFilename = FIO_createFilename_fromOutDir(srcFileName, outDirName, suffixSize); outDirFilename = FIO_createFilename_fromOutDir(srcFileName, outDirName, srcSuffixLen);
sfnSize = strlen(outDirFilename); sfnSize = strlen(outDirFilename);
assert(outDirFilename != NULL); assert(outDirFilename != NULL);
} }
if (dfnbCapacity <= sfnSize+suffixSize+1) { if (dfnbCapacity <= sfnSize+srcSuffixLen+1) {
/* resize buffer for dstName */ /* resize buffer for dstName */
free(dstFileNameBuffer); free(dstFileNameBuffer);
dfnbCapacity = sfnSize + suffixSize + 30; dfnbCapacity = sfnSize + srcSuffixLen + 30;
dstFileNameBuffer = (char*)malloc(dfnbCapacity); dstFileNameBuffer = (char*)malloc(dfnbCapacity);
if (!dstFileNameBuffer) { if (!dstFileNameBuffer) {
EXM_THROW(30, "zstd: %s", strerror(errno)); EXM_THROW(30, "zstd: %s", strerror(errno));
@ -1520,7 +1520,7 @@ FIO_determineCompressedName(const char* srcFileName, const char* outDirName, con
} else { } else {
memcpy(dstFileNameBuffer, srcFileName, sfnSize); memcpy(dstFileNameBuffer, srcFileName, sfnSize);
} }
memcpy(dstFileNameBuffer+sfnSize, suffix, suffixSize+1 /* Include terminating null */); memcpy(dstFileNameBuffer+sfnSize, suffix, srcSuffixLen+1 /* Include terminating null */);
return dstFileNameBuffer; return dstFileNameBuffer;
} }
@ -2287,6 +2287,37 @@ int FIO_decompressFilename(FIO_prefs_t* const prefs,
return decodingError; return decodingError;
} }
static const char *suffixList[] = {
ZSTD_EXTENSION,
TZSTD_EXTENSION,
#ifdef ZSTD_GZDECOMPRESS
GZ_EXTENSION,
TGZ_EXTENSION,
#endif
#ifdef ZSTD_LZMADECOMPRESS
LZMA_EXTENSION,
XZ_EXTENSION,
TXZ_EXTENSION,
#endif
#ifdef ZSTD_LZ4DECOMPRESS
LZ4_EXTENSION,
TLZ4_EXTENSION,
#endif
NULL
};
static const char *suffixListStr =
ZSTD_EXTENSION "/" TZSTD_EXTENSION
#ifdef ZSTD_GZDECOMPRESS
"/" GZ_EXTENSION "/" TGZ_EXTENSION
#endif
#ifdef ZSTD_LZMADECOMPRESS
"/" LZMA_EXTENSION "/" XZ_EXTENSION "/" TXZ_EXTENSION
#endif
#ifdef ZSTD_LZ4DECOMPRESS
"/" LZ4_EXTENSION "/" TLZ4_EXTENSION
#endif
;
/* FIO_determineDstName() : /* FIO_determineDstName() :
* create a destination filename from a srcFileName. * create a destination filename from a srcFileName.
@ -2297,71 +2328,78 @@ FIO_determineDstName(const char* srcFileName, const char* outDirName)
{ {
static size_t dfnbCapacity = 0; static size_t dfnbCapacity = 0;
static char* dstFileNameBuffer = NULL; /* using static allocation : this function cannot be multi-threaded */ static char* dstFileNameBuffer = NULL; /* using static allocation : this function cannot be multi-threaded */
size_t dstFileNameEndPos;
char* outDirFilename = NULL; char* outDirFilename = NULL;
const char* dstSuffix = "";
size_t dstSuffixLen = 0;
size_t sfnSize = strlen(srcFileName); size_t sfnSize = strlen(srcFileName);
size_t suffixSize;
const char* const suffixPtr = strrchr(srcFileName, '.'); size_t srcSuffixLen;
if (suffixPtr == NULL) { const char* const srcSuffix = strrchr(srcFileName, '.');
DISPLAYLEVEL(1, "zstd: %s: unknown suffix -- ignored \n", if (srcSuffix == NULL) {
srcFileName); DISPLAYLEVEL(1,
"zstd: %s: unknown suffix (%s expected). "
"Can't derive the output file name. "
"Specify it with -o dstFileName. Ignoring.\n",
srcFileName, suffixListStr);
return NULL; return NULL;
} }
suffixSize = strlen(suffixPtr); srcSuffixLen = strlen(srcSuffix);
/* check suffix is authorized */ {
if (sfnSize <= suffixSize const char** matchedSuffixPtr;
|| ( strcmp(suffixPtr, ZSTD_EXTENSION) for (matchedSuffixPtr = suffixList; *matchedSuffixPtr != NULL; matchedSuffixPtr++) {
#ifdef ZSTD_GZDECOMPRESS if (!strcmp(*matchedSuffixPtr, srcSuffix)) {
&& strcmp(suffixPtr, GZ_EXTENSION) break;
#endif }
#ifdef ZSTD_LZMADECOMPRESS }
&& strcmp(suffixPtr, XZ_EXTENSION)
&& strcmp(suffixPtr, LZMA_EXTENSION) /* check suffix is authorized */
#endif if (sfnSize <= srcSuffixLen || *matchedSuffixPtr == NULL) {
#ifdef ZSTD_LZ4DECOMPRESS DISPLAYLEVEL(1,
&& strcmp(suffixPtr, LZ4_EXTENSION) "zstd: %s: unknown suffix (%s expected). "
#endif "Can't derive the output file name. "
) ) { "Specify it with -o dstFileName. Ignoring.\n",
const char* suffixlist = ZSTD_EXTENSION srcFileName, suffixListStr);
#ifdef ZSTD_GZDECOMPRESS return NULL;
"/" GZ_EXTENSION }
#endif
#ifdef ZSTD_LZMADECOMPRESS if ((*matchedSuffixPtr)[1] == 't') {
"/" XZ_EXTENSION "/" LZMA_EXTENSION dstSuffix = ".tar";
#endif dstSuffixLen = strlen(dstSuffix);
#ifdef ZSTD_LZ4DECOMPRESS }
"/" LZ4_EXTENSION
#endif
;
DISPLAYLEVEL(1, "zstd: %s: unknown suffix (%s expected) -- ignored \n",
srcFileName, suffixlist);
return NULL;
} }
if (outDirName) { if (outDirName) {
outDirFilename = FIO_createFilename_fromOutDir(srcFileName, outDirName, 0); outDirFilename = FIO_createFilename_fromOutDir(srcFileName, outDirName, 0);
sfnSize = strlen(outDirFilename); sfnSize = strlen(outDirFilename);
assert(outDirFilename != NULL); assert(outDirFilename != NULL);
} }
if (dfnbCapacity+suffixSize <= sfnSize+1) { if (dfnbCapacity+srcSuffixLen <= sfnSize+1+dstSuffixLen) {
/* allocate enough space to write dstFilename into it */ /* allocate enough space to write dstFilename into it */
free(dstFileNameBuffer); free(dstFileNameBuffer);
dfnbCapacity = sfnSize + 20; dfnbCapacity = sfnSize + 20;
dstFileNameBuffer = (char*)malloc(dfnbCapacity); dstFileNameBuffer = (char*)malloc(dfnbCapacity);
if (dstFileNameBuffer==NULL) if (dstFileNameBuffer==NULL)
EXM_THROW(74, "%s : not enough memory for dstFileName", strerror(errno)); EXM_THROW(74, "%s : not enough memory for dstFileName",
strerror(errno));
} }
/* return dst name == src name truncated from suffix */ /* return dst name == src name truncated from suffix */
assert(dstFileNameBuffer != NULL); assert(dstFileNameBuffer != NULL);
dstFileNameEndPos = sfnSize - srcSuffixLen;
if (outDirFilename) { if (outDirFilename) {
memcpy(dstFileNameBuffer, outDirFilename, sfnSize - suffixSize); memcpy(dstFileNameBuffer, outDirFilename, dstFileNameEndPos);
free(outDirFilename); free(outDirFilename);
} else { } else {
memcpy(dstFileNameBuffer, srcFileName, sfnSize - suffixSize); memcpy(dstFileNameBuffer, srcFileName, dstFileNameEndPos);
} }
dstFileNameBuffer[sfnSize-suffixSize] = '\0';
/* The short tar extensions tzst, tgz, txz and tlz4 files should have "tar"
* extension on decompression. Also writes terminating null. */
strcpy(dstFileNameBuffer + dstFileNameEndPos, dstSuffix);
return dstFileNameBuffer; return dstFileNameBuffer;
/* note : dstFileNameBuffer memory is not going to be free */ /* note : dstFileNameBuffer memory is not going to be free */

12
programs/fileio.h
View File

@ -30,11 +30,23 @@ extern "C" {
#else #else
# define nulmark "/dev/null" # define nulmark "/dev/null"
#endif #endif
/**
* We test whether the extension we found starts with 't', and if so, we append
* ".tar" to the end of the output name.
*/
#define LZMA_EXTENSION ".lzma" #define LZMA_EXTENSION ".lzma"
#define XZ_EXTENSION ".xz" #define XZ_EXTENSION ".xz"
#define TXZ_EXTENSION ".txz"
#define GZ_EXTENSION ".gz" #define GZ_EXTENSION ".gz"
#define TGZ_EXTENSION ".tgz"
#define ZSTD_EXTENSION ".zst" #define ZSTD_EXTENSION ".zst"
#define TZSTD_EXTENSION ".tzst"
#define LZ4_EXTENSION ".lz4" #define LZ4_EXTENSION ".lz4"
#define TLZ4_EXTENSION ".tlz4"
/*-************************************* /*-*************************************

40
tests/playTests.sh
View File

@ -845,6 +845,46 @@ if [ $LZ4MODE -ne 1 ]; then
grep ".lz4" tmplg > $INTOVOID && die "Unsupported suffix listed" grep ".lz4" tmplg > $INTOVOID && die "Unsupported suffix listed"
fi fi
println "\n===> tar extension tests "
rm -f tmp tmp.tar tmp.tzst tmp.tgz tmp.txz tmp.tlz4
./datagen > tmp
tar cf tmp.tar tmp
$ZSTD tmp.tar -o tmp.tzst
rm tmp.tar
$ZSTD -d tmp.tzst
[ -e tmp.tar ] || die ".tzst failed to decompress to .tar!"
rm -f tmp.tar tmp.tzst
if [ $GZIPMODE -eq 1 ]; then
tar czf tmp.tgz tmp
$ZSTD -d tmp.tgz
[ -e tmp.tar ] || die ".tgz failed to decompress to .tar!"
rm -f tmp.tar tmp.tgz
fi
if [ $LZMAMODE -eq 1 ]; then
tar c tmp | $ZSTD --format=xz > tmp.txz
$ZSTD -d tmp.txz
[ -e tmp.tar ] || die ".txz failed to decompress to .tar!"
rm -f tmp.tar tmp.txz
fi
if [ $LZ4MODE -eq 1 ]; then
tar c tmp | $ZSTD --format=lz4 > tmp.tlz4
$ZSTD -d tmp.tlz4
[ -e tmp.tar ] || die ".tlz4 failed to decompress to .tar!"
rm -f tmp.tar tmp.tlz4
fi
touch tmp.t tmp.tz tmp.tzs
! $ZSTD -d tmp.t
! $ZSTD -d tmp.tz
! $ZSTD -d tmp.tzs
exit
println "\n===> zstd round-trip tests " println "\n===> zstd round-trip tests "
roundTripTest roundTripTest