From 166645e7b39502493ce6df1f8025726e5cbee2d3 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 18 Aug 2017 18:30:41 -0700 Subject: [PATCH] fixed zstd-compress file-information is dependent on decompression functions. it should only be enabled when ZSTD_NODECOMPRESS is not set. also : added zstd-compress compilation test into `make shortest` --- Makefile | 1 + programs/Makefile | 24 ++- programs/fileio.c | 450 +++++++++++++++++++++++---------------------- programs/zstdcli.c | 16 +- 4 files changed, 261 insertions(+), 230 deletions(-) diff --git a/Makefile b/Makefile index d88104a0..a72f99fc 100644 --- a/Makefile +++ b/Makefile @@ -76,6 +76,7 @@ zlibwrapper: .PHONY: test shortest test shortest: + $(MAKE) -C $(PRGDIR) allVariants $(MAKE) -C $(TESTDIR) $@ .PHONY: examples diff --git a/programs/Makefile b/programs/Makefile index 2460a091..ee623e52 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -132,12 +132,15 @@ else LZ4_MSG := $(NO_LZ4_MSG) endif -.PHONY: default all clean clean_decomp_o install uninstall generate_res - +.PHONY: default default: zstd-release +.PHONY: all all: zstd +.PHONY: allVariants +allVariants: zstd zstd-compress + $(ZSTDDECOMP_O): CFLAGS += $(ALIGN_LOOP) zstd zstd4 : CPPFLAGS += $(THREAD_CPP) $(ZLIBCPP) $(LZMACPP) @@ -156,6 +159,7 @@ ifneq (,$(filter Windows%,$(OS))) endif $(CC) $(FLAGS) $^ $(RES_FILE) -o zstd$(EXT) $(LDFLAGS) +.PHONY: zstd-release zstd-release: DEBUGFLAGS := zstd-release: zstd @@ -166,7 +170,7 @@ ifneq (,$(filter Windows%,$(OS))) endif $(CC) -m32 $(FLAGS) $^ $(RES32_FILE) -o $@$(EXT) -zstd-nolegacy : clean_decomp_o +zstd-nolegacy : $(MAKE) zstd ZSTD_LEGACY_SUPPORT=0 zstd-nomt : THREAD_CPP := @@ -211,9 +215,11 @@ zstd-compress: $(ZSTDCOMMON_FILES) $(ZSTDCOMP_FILES) zstdcli.c fileio.c # zstd is now built with Multi-threading by default zstdmt: zstd +.PHONY: generate_res generate_res: windres/generate_res.bat +.PHONY: clean clean: $(MAKE) -C $(ZSTDDIR) clean @$(RM) $(ZSTDDIR)/decompress/*.o $(ZSTDDIR)/decompress/zstd_decompress.gcda @@ -222,20 +228,20 @@ clean: *.gcda default.profraw have_zlib$(EXT) @echo Cleaning completed -clean_decomp_o: - @$(RM) $(ZSTDDECOMP_O) - MD2ROFF = ronn MD2ROFF_FLAGS = --roff --warnings --manual="User Commands" --organization="zstd $(ZSTD_VERSION)" zstd.1: zstd.1.md cat $^ | $(MD2ROFF) $(MD2ROFF_FLAGS) | sed -n '/^\.\\\".*/!p' > $@ +.PHONY: man man: zstd.1 +.PHONY: clean-man clean-man: rm zstd.1 +.PHONY: preview-man preview-man: clean-man man man ./zstd.1 @@ -244,6 +250,10 @@ preview-man: clean-man man #----------------------------------------------------------------------------- ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS)) +.PHONY: list +list: + @$(MAKE) -pRrq -f $(lastword $(MAKEFILE_LIST)) : 2>/dev/null | awk -v RS= -F: '/^# File/,/^# Finished Make data base/ {if ($$1 !~ "^[#.]") {print $$1}}' | sort | egrep -v -e '^[^[:alnum:]]' -e '^$@$$' | xargs + ifneq (,$(filter $(shell uname),SunOS)) INSTALL ?= ginstall else @@ -264,6 +274,7 @@ INSTALL_PROGRAM ?= $(INSTALL) -m 755 INSTALL_SCRIPT ?= $(INSTALL) -m 755 INSTALL_MAN ?= $(INSTALL) -m 644 +.PHONY: install install: zstd @echo Installing binaries @$(INSTALL) -d -m 755 $(DESTDIR)$(BINDIR)/ $(DESTDIR)$(MANDIR)/ @@ -279,6 +290,7 @@ install: zstd @ln -sf zstd.1 $(DESTDIR)$(MANDIR)/unzstd.1 @echo zstd installation completed +.PHONY: uninstall uninstall: @$(RM) $(DESTDIR)$(BINDIR)/zstdgrep @$(RM) $(DESTDIR)$(BINDIR)/zstdless diff --git a/programs/fileio.c b/programs/fileio.c index 9f9b72fb..eb75c68a 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -925,223 +925,6 @@ int FIO_compressFilename(const char* dstFileName, const char* srcFileName, return result; } -typedef struct { - int numActualFrames; - int numSkippableFrames; - unsigned long long decompressedSize; - int decompUnavailable; - unsigned long long compressedSize; - int usesCheck; -} fileInfo_t; - -/* - * Reads information from file, stores in *info - * if successful, returns 0, returns 1 for frame analysis error, returns 2 for file not compressed with zstd - * returns 3 for cases in which file could not be opened. - */ -static int getFileInfo(fileInfo_t* info, const char* inFileName){ - int detectError = 0; - FILE* const srcFile = FIO_openSrcFile(inFileName); - if (srcFile == NULL) { - DISPLAY("Error: could not open source file %s\n", inFileName); - return 3; - } - info->compressedSize = (unsigned long long)UTIL_getFileSize(inFileName); - - /* begin analyzing frame */ - for ( ; ; ) { - BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX]; - size_t const numBytesRead = fread(headerBuffer, 1, sizeof(headerBuffer), srcFile); - if (numBytesRead < ZSTD_frameHeaderSize_min) { - if (feof(srcFile) && numBytesRead == 0 && info->compressedSize > 0) { - break; - } - else if (feof(srcFile)) { - DISPLAY("Error: reached end of file with incomplete frame\n"); - detectError = 2; - break; - } - else { - DISPLAY("Error: did not reach end of file but ran out of frames\n"); - detectError = 1; - break; - } - } - { U32 const magicNumber = MEM_readLE32(headerBuffer); - /* Zstandard frame */ - if (magicNumber == ZSTD_MAGICNUMBER) { - U64 const frameContentSize = ZSTD_getFrameContentSize(headerBuffer, numBytesRead); - if (frameContentSize == ZSTD_CONTENTSIZE_ERROR || frameContentSize == ZSTD_CONTENTSIZE_UNKNOWN) { - info->decompUnavailable = 1; - } else { - info->decompressedSize += frameContentSize; - } - /* move to the end of the frame header */ - { size_t const headerSize = ZSTD_frameHeaderSize(headerBuffer, numBytesRead); - if (ZSTD_isError(headerSize)) { - DISPLAY("Error: could not determine frame header size\n"); - detectError = 1; - break; - } - { int const ret = fseek(srcFile, ((long)headerSize)-((long)numBytesRead), SEEK_CUR); - if (ret != 0) { - DISPLAY("Error: could not move to end of frame header\n"); - detectError = 1; - break; - } } } - - /* skip the rest of the blocks in the frame */ - { int lastBlock = 0; - do { - BYTE blockHeaderBuffer[3]; - size_t const readBytes = fread(blockHeaderBuffer, 1, 3, srcFile); - if (readBytes != 3) { - DISPLAY("There was a problem reading the block header\n"); - detectError = 1; - break; - } - { U32 const blockHeader = MEM_readLE24(blockHeaderBuffer); - U32 const blockTypeID = (blockHeader >> 1) & 3; - U32 const isRLE = (blockTypeID == 1); - U32 const isWrongBlock = (blockTypeID == 3); - long const blockSize = isRLE ? 1 : (long)(blockHeader >> 3); - if (isWrongBlock) { - DISPLAY("Error: unsupported block type \n"); - detectError = 1; - break; - } - lastBlock = blockHeader & 1; - { int const ret = fseek(srcFile, blockSize, SEEK_CUR); - if (ret != 0) { - DISPLAY("Error: could not skip to end of block\n"); - detectError = 1; - break; - } } } - } while (lastBlock != 1); - - if (detectError) break; - } - - /* check if checksum is used */ - { BYTE const frameHeaderDescriptor = headerBuffer[4]; - int const contentChecksumFlag = (frameHeaderDescriptor & (1 << 2)) >> 2; - if (contentChecksumFlag) { - int const ret = fseek(srcFile, 4, SEEK_CUR); - info->usesCheck = 1; - if (ret != 0) { - DISPLAY("Error: could not skip past checksum\n"); - detectError = 1; - break; - } } } - info->numActualFrames++; - } - /* Skippable frame */ - else if ((magicNumber & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) { - U32 const frameSize = MEM_readLE32(headerBuffer + 4); - long const seek = (long)(8 + frameSize - numBytesRead); - int const ret = LONG_SEEK(srcFile, seek, SEEK_CUR); - if (ret != 0) { - DISPLAY("Error: could not find end of skippable frame\n"); - detectError = 1; - break; - } - info->numSkippableFrames++; - } - /* unknown content */ - else { - detectError = 2; - break; - } - } - } /* end analyzing frame */ - fclose(srcFile); - return detectError; -} - -static void displayInfo(const char* inFileName, fileInfo_t* info, int displayLevel){ - unsigned const unit = info->compressedSize < (1 MB) ? (1 KB) : (1 MB); - const char* const unitStr = info->compressedSize < (1 MB) ? "KB" : "MB"; - double const compressedSizeUnit = (double)info->compressedSize / unit; - double const decompressedSizeUnit = (double)info->decompressedSize / unit; - double const ratio = (info->compressedSize == 0) ? 0 : ((double)info->decompressedSize)/info->compressedSize; - const char* const checkString = (info->usesCheck ? "XXH64" : "None"); - if (displayLevel <= 2) { - if (!info->decompUnavailable) { - DISPLAYOUT("Skippable Non-Skippable Compressed Uncompressed Ratio Check Filename\n"); - DISPLAYOUT("%9d %13d %7.2f %2s %9.2f %2s %5.3f %5s %s\n", - info->numSkippableFrames, info->numActualFrames, - compressedSizeUnit, unitStr, decompressedSizeUnit, unitStr, - ratio, checkString, inFileName); - } else { - DISPLAYOUT("Skippable Non-Skippable Compressed Check Filename\n"); - DISPLAYOUT("%9d %13d %7.2f MB %5s %s\n", - info->numSkippableFrames, info->numActualFrames, - compressedSizeUnit, checkString, inFileName); - } - } else { - DISPLAYOUT("# Zstandard Frames: %d\n", info->numActualFrames); - DISPLAYOUT("# Skippable Frames: %d\n", info->numSkippableFrames); - DISPLAYOUT("Compressed Size: %.2f %2s (%llu B)\n", - compressedSizeUnit, unitStr, info->compressedSize); - if (!info->decompUnavailable) { - DISPLAYOUT("Decompressed Size: %.2f %2s (%llu B)\n", - decompressedSizeUnit, unitStr, info->decompressedSize); - DISPLAYOUT("Ratio: %.4f\n", ratio); - } - DISPLAYOUT("Check: %s\n", checkString); - DISPLAYOUT("\n"); - } -} - - -static int FIO_listFile(const char* inFileName, int displayLevel, unsigned fileNo, unsigned numFiles){ - /* initialize info to avoid warnings */ - fileInfo_t info; - memset(&info, 0, sizeof(info)); - DISPLAYOUT("%s (%u/%u):\n", inFileName, fileNo, numFiles); - { - int const error = getFileInfo(&info, inFileName); - if (error == 1) { - /* display error, but provide output */ - DISPLAY("An error occurred with getting file info\n"); - } - else if (error == 2) { - DISPLAYOUT("File %s not compressed with zstd\n", inFileName); - if (displayLevel > 2) { - DISPLAYOUT("\n"); - } - return 1; - } - else if (error == 3) { - /* error occurred with opening the file */ - if (displayLevel > 2) { - DISPLAYOUT("\n"); - } - return 1; - } - displayInfo(inFileName, &info, displayLevel); - return error; - } -} - -int FIO_listMultipleFiles(unsigned numFiles, const char** filenameTable, int displayLevel){ - if (numFiles == 0) { - DISPLAYOUT("No files given\n"); - return 0; - } - DISPLAYOUT("===========================================\n"); - DISPLAYOUT("Printing information about compressed files\n"); - DISPLAYOUT("===========================================\n"); - DISPLAYOUT("Number of files listed: %u\n", numFiles); - { - int error = 0; - unsigned u; - for (u=0; ucompressedSize = (unsigned long long)UTIL_getFileSize(inFileName); + + /* begin analyzing frame */ + for ( ; ; ) { + BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX]; + size_t const numBytesRead = fread(headerBuffer, 1, sizeof(headerBuffer), srcFile); + if (numBytesRead < ZSTD_frameHeaderSize_min) { + if (feof(srcFile) && numBytesRead == 0 && info->compressedSize > 0) { + break; + } + else if (feof(srcFile)) { + DISPLAY("Error: reached end of file with incomplete frame\n"); + detectError = 2; + break; + } + else { + DISPLAY("Error: did not reach end of file but ran out of frames\n"); + detectError = 1; + break; + } + } + { U32 const magicNumber = MEM_readLE32(headerBuffer); + /* Zstandard frame */ + if (magicNumber == ZSTD_MAGICNUMBER) { + U64 const frameContentSize = ZSTD_getFrameContentSize(headerBuffer, numBytesRead); + if (frameContentSize == ZSTD_CONTENTSIZE_ERROR || frameContentSize == ZSTD_CONTENTSIZE_UNKNOWN) { + info->decompUnavailable = 1; + } else { + info->decompressedSize += frameContentSize; + } + /* move to the end of the frame header */ + { size_t const headerSize = ZSTD_frameHeaderSize(headerBuffer, numBytesRead); + if (ZSTD_isError(headerSize)) { + DISPLAY("Error: could not determine frame header size\n"); + detectError = 1; + break; + } + { int const ret = fseek(srcFile, ((long)headerSize)-((long)numBytesRead), SEEK_CUR); + if (ret != 0) { + DISPLAY("Error: could not move to end of frame header\n"); + detectError = 1; + break; + } } } + + /* skip the rest of the blocks in the frame */ + { int lastBlock = 0; + do { + BYTE blockHeaderBuffer[3]; + size_t const readBytes = fread(blockHeaderBuffer, 1, 3, srcFile); + if (readBytes != 3) { + DISPLAY("There was a problem reading the block header\n"); + detectError = 1; + break; + } + { U32 const blockHeader = MEM_readLE24(blockHeaderBuffer); + U32 const blockTypeID = (blockHeader >> 1) & 3; + U32 const isRLE = (blockTypeID == 1); + U32 const isWrongBlock = (blockTypeID == 3); + long const blockSize = isRLE ? 1 : (long)(blockHeader >> 3); + if (isWrongBlock) { + DISPLAY("Error: unsupported block type \n"); + detectError = 1; + break; + } + lastBlock = blockHeader & 1; + { int const ret = fseek(srcFile, blockSize, SEEK_CUR); + if (ret != 0) { + DISPLAY("Error: could not skip to end of block\n"); + detectError = 1; + break; + } } } + } while (lastBlock != 1); + + if (detectError) break; + } + + /* check if checksum is used */ + { BYTE const frameHeaderDescriptor = headerBuffer[4]; + int const contentChecksumFlag = (frameHeaderDescriptor & (1 << 2)) >> 2; + if (contentChecksumFlag) { + int const ret = fseek(srcFile, 4, SEEK_CUR); + info->usesCheck = 1; + if (ret != 0) { + DISPLAY("Error: could not skip past checksum\n"); + detectError = 1; + break; + } } } + info->numActualFrames++; + } + /* Skippable frame */ + else if ((magicNumber & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) { + U32 const frameSize = MEM_readLE32(headerBuffer + 4); + long const seek = (long)(8 + frameSize - numBytesRead); + int const ret = LONG_SEEK(srcFile, seek, SEEK_CUR); + if (ret != 0) { + DISPLAY("Error: could not find end of skippable frame\n"); + detectError = 1; + break; + } + info->numSkippableFrames++; + } + /* unknown content */ + else { + detectError = 2; + break; + } + } + } /* end analyzing frame */ + fclose(srcFile); + return detectError; +} + +static void displayInfo(const char* inFileName, fileInfo_t* info, int displayLevel){ + unsigned const unit = info->compressedSize < (1 MB) ? (1 KB) : (1 MB); + const char* const unitStr = info->compressedSize < (1 MB) ? "KB" : "MB"; + double const compressedSizeUnit = (double)info->compressedSize / unit; + double const decompressedSizeUnit = (double)info->decompressedSize / unit; + double const ratio = (info->compressedSize == 0) ? 0 : ((double)info->decompressedSize)/info->compressedSize; + const char* const checkString = (info->usesCheck ? "XXH64" : "None"); + if (displayLevel <= 2) { + if (!info->decompUnavailable) { + DISPLAYOUT("Skippable Non-Skippable Compressed Uncompressed Ratio Check Filename\n"); + DISPLAYOUT("%9d %13d %7.2f %2s %9.2f %2s %5.3f %5s %s\n", + info->numSkippableFrames, info->numActualFrames, + compressedSizeUnit, unitStr, decompressedSizeUnit, unitStr, + ratio, checkString, inFileName); + } else { + DISPLAYOUT("Skippable Non-Skippable Compressed Check Filename\n"); + DISPLAYOUT("%9d %13d %7.2f MB %5s %s\n", + info->numSkippableFrames, info->numActualFrames, + compressedSizeUnit, checkString, inFileName); + } + } else { + DISPLAYOUT("# Zstandard Frames: %d\n", info->numActualFrames); + DISPLAYOUT("# Skippable Frames: %d\n", info->numSkippableFrames); + DISPLAYOUT("Compressed Size: %.2f %2s (%llu B)\n", + compressedSizeUnit, unitStr, info->compressedSize); + if (!info->decompUnavailable) { + DISPLAYOUT("Decompressed Size: %.2f %2s (%llu B)\n", + decompressedSizeUnit, unitStr, info->decompressedSize); + DISPLAYOUT("Ratio: %.4f\n", ratio); + } + DISPLAYOUT("Check: %s\n", checkString); + DISPLAYOUT("\n"); + } +} + + +static int FIO_listFile(const char* inFileName, int displayLevel, unsigned fileNo, unsigned numFiles){ + /* initialize info to avoid warnings */ + fileInfo_t info; + memset(&info, 0, sizeof(info)); + DISPLAYOUT("%s (%u/%u):\n", inFileName, fileNo, numFiles); + { + int const error = getFileInfo(&info, inFileName); + if (error == 1) { + /* display error, but provide output */ + DISPLAY("An error occurred with getting file info\n"); + } + else if (error == 2) { + DISPLAYOUT("File %s not compressed with zstd\n", inFileName); + if (displayLevel > 2) { + DISPLAYOUT("\n"); + } + return 1; + } + else if (error == 3) { + /* error occurred with opening the file */ + if (displayLevel > 2) { + DISPLAYOUT("\n"); + } + return 1; + } + displayInfo(inFileName, &info, displayLevel); + return error; + } +} + +int FIO_listMultipleFiles(unsigned numFiles, const char** filenameTable, int displayLevel){ + if (numFiles == 0) { + DISPLAYOUT("No files given\n"); + return 0; + } + DISPLAYOUT("===========================================\n"); + DISPLAYOUT("Printing information about compressed files\n"); + DISPLAYOUT("===========================================\n"); + DISPLAYOUT("Number of files listed: %u\n", numFiles); + { + int error = 0; + unsigned u; + for (u=0; u