fixed zstd-compress
file-information is dependent on decompression functions. it should only be enabled when ZSTD_NODECOMPRESS is not set. also : added zstd-compress compilation test into `make shortest`
This commit is contained in:
parent
f207b39f55
commit
166645e7b3
1
Makefile
1
Makefile
@ -76,6 +76,7 @@ zlibwrapper:
|
|||||||
|
|
||||||
.PHONY: test shortest
|
.PHONY: test shortest
|
||||||
test shortest:
|
test shortest:
|
||||||
|
$(MAKE) -C $(PRGDIR) allVariants
|
||||||
$(MAKE) -C $(TESTDIR) $@
|
$(MAKE) -C $(TESTDIR) $@
|
||||||
|
|
||||||
.PHONY: examples
|
.PHONY: examples
|
||||||
|
@ -132,12 +132,15 @@ else
|
|||||||
LZ4_MSG := $(NO_LZ4_MSG)
|
LZ4_MSG := $(NO_LZ4_MSG)
|
||||||
endif
|
endif
|
||||||
|
|
||||||
.PHONY: default all clean clean_decomp_o install uninstall generate_res
|
.PHONY: default
|
||||||
|
|
||||||
default: zstd-release
|
default: zstd-release
|
||||||
|
|
||||||
|
.PHONY: all
|
||||||
all: zstd
|
all: zstd
|
||||||
|
|
||||||
|
.PHONY: allVariants
|
||||||
|
allVariants: zstd zstd-compress
|
||||||
|
|
||||||
$(ZSTDDECOMP_O): CFLAGS += $(ALIGN_LOOP)
|
$(ZSTDDECOMP_O): CFLAGS += $(ALIGN_LOOP)
|
||||||
|
|
||||||
zstd zstd4 : CPPFLAGS += $(THREAD_CPP) $(ZLIBCPP) $(LZMACPP)
|
zstd zstd4 : CPPFLAGS += $(THREAD_CPP) $(ZLIBCPP) $(LZMACPP)
|
||||||
@ -156,6 +159,7 @@ ifneq (,$(filter Windows%,$(OS)))
|
|||||||
endif
|
endif
|
||||||
$(CC) $(FLAGS) $^ $(RES_FILE) -o zstd$(EXT) $(LDFLAGS)
|
$(CC) $(FLAGS) $^ $(RES_FILE) -o zstd$(EXT) $(LDFLAGS)
|
||||||
|
|
||||||
|
.PHONY: zstd-release
|
||||||
zstd-release: DEBUGFLAGS :=
|
zstd-release: DEBUGFLAGS :=
|
||||||
zstd-release: zstd
|
zstd-release: zstd
|
||||||
|
|
||||||
@ -166,7 +170,7 @@ ifneq (,$(filter Windows%,$(OS)))
|
|||||||
endif
|
endif
|
||||||
$(CC) -m32 $(FLAGS) $^ $(RES32_FILE) -o $@$(EXT)
|
$(CC) -m32 $(FLAGS) $^ $(RES32_FILE) -o $@$(EXT)
|
||||||
|
|
||||||
zstd-nolegacy : clean_decomp_o
|
zstd-nolegacy :
|
||||||
$(MAKE) zstd ZSTD_LEGACY_SUPPORT=0
|
$(MAKE) zstd ZSTD_LEGACY_SUPPORT=0
|
||||||
|
|
||||||
zstd-nomt : THREAD_CPP :=
|
zstd-nomt : THREAD_CPP :=
|
||||||
@ -211,9 +215,11 @@ zstd-compress: $(ZSTDCOMMON_FILES) $(ZSTDCOMP_FILES) zstdcli.c fileio.c
|
|||||||
# zstd is now built with Multi-threading by default
|
# zstd is now built with Multi-threading by default
|
||||||
zstdmt: zstd
|
zstdmt: zstd
|
||||||
|
|
||||||
|
.PHONY: generate_res
|
||||||
generate_res:
|
generate_res:
|
||||||
windres/generate_res.bat
|
windres/generate_res.bat
|
||||||
|
|
||||||
|
.PHONY: clean
|
||||||
clean:
|
clean:
|
||||||
$(MAKE) -C $(ZSTDDIR) clean
|
$(MAKE) -C $(ZSTDDIR) clean
|
||||||
@$(RM) $(ZSTDDIR)/decompress/*.o $(ZSTDDIR)/decompress/zstd_decompress.gcda
|
@$(RM) $(ZSTDDIR)/decompress/*.o $(ZSTDDIR)/decompress/zstd_decompress.gcda
|
||||||
@ -222,20 +228,20 @@ clean:
|
|||||||
*.gcda default.profraw have_zlib$(EXT)
|
*.gcda default.profraw have_zlib$(EXT)
|
||||||
@echo Cleaning completed
|
@echo Cleaning completed
|
||||||
|
|
||||||
clean_decomp_o:
|
|
||||||
@$(RM) $(ZSTDDECOMP_O)
|
|
||||||
|
|
||||||
MD2ROFF = ronn
|
MD2ROFF = ronn
|
||||||
MD2ROFF_FLAGS = --roff --warnings --manual="User Commands" --organization="zstd $(ZSTD_VERSION)"
|
MD2ROFF_FLAGS = --roff --warnings --manual="User Commands" --organization="zstd $(ZSTD_VERSION)"
|
||||||
|
|
||||||
zstd.1: zstd.1.md
|
zstd.1: zstd.1.md
|
||||||
cat $^ | $(MD2ROFF) $(MD2ROFF_FLAGS) | sed -n '/^\.\\\".*/!p' > $@
|
cat $^ | $(MD2ROFF) $(MD2ROFF_FLAGS) | sed -n '/^\.\\\".*/!p' > $@
|
||||||
|
|
||||||
|
.PHONY: man
|
||||||
man: zstd.1
|
man: zstd.1
|
||||||
|
|
||||||
|
.PHONY: clean-man
|
||||||
clean-man:
|
clean-man:
|
||||||
rm zstd.1
|
rm zstd.1
|
||||||
|
|
||||||
|
.PHONY: preview-man
|
||||||
preview-man: clean-man man
|
preview-man: clean-man man
|
||||||
man ./zstd.1
|
man ./zstd.1
|
||||||
|
|
||||||
@ -244,6 +250,10 @@ preview-man: clean-man man
|
|||||||
#-----------------------------------------------------------------------------
|
#-----------------------------------------------------------------------------
|
||||||
ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS))
|
ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS))
|
||||||
|
|
||||||
|
.PHONY: list
|
||||||
|
list:
|
||||||
|
@$(MAKE) -pRrq -f $(lastword $(MAKEFILE_LIST)) : 2>/dev/null | awk -v RS= -F: '/^# File/,/^# Finished Make data base/ {if ($$1 !~ "^[#.]") {print $$1}}' | sort | egrep -v -e '^[^[:alnum:]]' -e '^$@$$' | xargs
|
||||||
|
|
||||||
ifneq (,$(filter $(shell uname),SunOS))
|
ifneq (,$(filter $(shell uname),SunOS))
|
||||||
INSTALL ?= ginstall
|
INSTALL ?= ginstall
|
||||||
else
|
else
|
||||||
@ -264,6 +274,7 @@ INSTALL_PROGRAM ?= $(INSTALL) -m 755
|
|||||||
INSTALL_SCRIPT ?= $(INSTALL) -m 755
|
INSTALL_SCRIPT ?= $(INSTALL) -m 755
|
||||||
INSTALL_MAN ?= $(INSTALL) -m 644
|
INSTALL_MAN ?= $(INSTALL) -m 644
|
||||||
|
|
||||||
|
.PHONY: install
|
||||||
install: zstd
|
install: zstd
|
||||||
@echo Installing binaries
|
@echo Installing binaries
|
||||||
@$(INSTALL) -d -m 755 $(DESTDIR)$(BINDIR)/ $(DESTDIR)$(MANDIR)/
|
@$(INSTALL) -d -m 755 $(DESTDIR)$(BINDIR)/ $(DESTDIR)$(MANDIR)/
|
||||||
@ -279,6 +290,7 @@ install: zstd
|
|||||||
@ln -sf zstd.1 $(DESTDIR)$(MANDIR)/unzstd.1
|
@ln -sf zstd.1 $(DESTDIR)$(MANDIR)/unzstd.1
|
||||||
@echo zstd installation completed
|
@echo zstd installation completed
|
||||||
|
|
||||||
|
.PHONY: uninstall
|
||||||
uninstall:
|
uninstall:
|
||||||
@$(RM) $(DESTDIR)$(BINDIR)/zstdgrep
|
@$(RM) $(DESTDIR)$(BINDIR)/zstdgrep
|
||||||
@$(RM) $(DESTDIR)$(BINDIR)/zstdless
|
@$(RM) $(DESTDIR)$(BINDIR)/zstdless
|
||||||
|
@ -925,223 +925,6 @@ int FIO_compressFilename(const char* dstFileName, const char* srcFileName,
|
|||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
|
|
||||||
typedef struct {
|
|
||||||
int numActualFrames;
|
|
||||||
int numSkippableFrames;
|
|
||||||
unsigned long long decompressedSize;
|
|
||||||
int decompUnavailable;
|
|
||||||
unsigned long long compressedSize;
|
|
||||||
int usesCheck;
|
|
||||||
} fileInfo_t;
|
|
||||||
|
|
||||||
/*
|
|
||||||
* Reads information from file, stores in *info
|
|
||||||
* if successful, returns 0, returns 1 for frame analysis error, returns 2 for file not compressed with zstd
|
|
||||||
* returns 3 for cases in which file could not be opened.
|
|
||||||
*/
|
|
||||||
static int getFileInfo(fileInfo_t* info, const char* inFileName){
|
|
||||||
int detectError = 0;
|
|
||||||
FILE* const srcFile = FIO_openSrcFile(inFileName);
|
|
||||||
if (srcFile == NULL) {
|
|
||||||
DISPLAY("Error: could not open source file %s\n", inFileName);
|
|
||||||
return 3;
|
|
||||||
}
|
|
||||||
info->compressedSize = (unsigned long long)UTIL_getFileSize(inFileName);
|
|
||||||
|
|
||||||
/* begin analyzing frame */
|
|
||||||
for ( ; ; ) {
|
|
||||||
BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX];
|
|
||||||
size_t const numBytesRead = fread(headerBuffer, 1, sizeof(headerBuffer), srcFile);
|
|
||||||
if (numBytesRead < ZSTD_frameHeaderSize_min) {
|
|
||||||
if (feof(srcFile) && numBytesRead == 0 && info->compressedSize > 0) {
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
else if (feof(srcFile)) {
|
|
||||||
DISPLAY("Error: reached end of file with incomplete frame\n");
|
|
||||||
detectError = 2;
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
else {
|
|
||||||
DISPLAY("Error: did not reach end of file but ran out of frames\n");
|
|
||||||
detectError = 1;
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
{ U32 const magicNumber = MEM_readLE32(headerBuffer);
|
|
||||||
/* Zstandard frame */
|
|
||||||
if (magicNumber == ZSTD_MAGICNUMBER) {
|
|
||||||
U64 const frameContentSize = ZSTD_getFrameContentSize(headerBuffer, numBytesRead);
|
|
||||||
if (frameContentSize == ZSTD_CONTENTSIZE_ERROR || frameContentSize == ZSTD_CONTENTSIZE_UNKNOWN) {
|
|
||||||
info->decompUnavailable = 1;
|
|
||||||
} else {
|
|
||||||
info->decompressedSize += frameContentSize;
|
|
||||||
}
|
|
||||||
/* move to the end of the frame header */
|
|
||||||
{ size_t const headerSize = ZSTD_frameHeaderSize(headerBuffer, numBytesRead);
|
|
||||||
if (ZSTD_isError(headerSize)) {
|
|
||||||
DISPLAY("Error: could not determine frame header size\n");
|
|
||||||
detectError = 1;
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
{ int const ret = fseek(srcFile, ((long)headerSize)-((long)numBytesRead), SEEK_CUR);
|
|
||||||
if (ret != 0) {
|
|
||||||
DISPLAY("Error: could not move to end of frame header\n");
|
|
||||||
detectError = 1;
|
|
||||||
break;
|
|
||||||
} } }
|
|
||||||
|
|
||||||
/* skip the rest of the blocks in the frame */
|
|
||||||
{ int lastBlock = 0;
|
|
||||||
do {
|
|
||||||
BYTE blockHeaderBuffer[3];
|
|
||||||
size_t const readBytes = fread(blockHeaderBuffer, 1, 3, srcFile);
|
|
||||||
if (readBytes != 3) {
|
|
||||||
DISPLAY("There was a problem reading the block header\n");
|
|
||||||
detectError = 1;
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
{ U32 const blockHeader = MEM_readLE24(blockHeaderBuffer);
|
|
||||||
U32 const blockTypeID = (blockHeader >> 1) & 3;
|
|
||||||
U32 const isRLE = (blockTypeID == 1);
|
|
||||||
U32 const isWrongBlock = (blockTypeID == 3);
|
|
||||||
long const blockSize = isRLE ? 1 : (long)(blockHeader >> 3);
|
|
||||||
if (isWrongBlock) {
|
|
||||||
DISPLAY("Error: unsupported block type \n");
|
|
||||||
detectError = 1;
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
lastBlock = blockHeader & 1;
|
|
||||||
{ int const ret = fseek(srcFile, blockSize, SEEK_CUR);
|
|
||||||
if (ret != 0) {
|
|
||||||
DISPLAY("Error: could not skip to end of block\n");
|
|
||||||
detectError = 1;
|
|
||||||
break;
|
|
||||||
} } }
|
|
||||||
} while (lastBlock != 1);
|
|
||||||
|
|
||||||
if (detectError) break;
|
|
||||||
}
|
|
||||||
|
|
||||||
/* check if checksum is used */
|
|
||||||
{ BYTE const frameHeaderDescriptor = headerBuffer[4];
|
|
||||||
int const contentChecksumFlag = (frameHeaderDescriptor & (1 << 2)) >> 2;
|
|
||||||
if (contentChecksumFlag) {
|
|
||||||
int const ret = fseek(srcFile, 4, SEEK_CUR);
|
|
||||||
info->usesCheck = 1;
|
|
||||||
if (ret != 0) {
|
|
||||||
DISPLAY("Error: could not skip past checksum\n");
|
|
||||||
detectError = 1;
|
|
||||||
break;
|
|
||||||
} } }
|
|
||||||
info->numActualFrames++;
|
|
||||||
}
|
|
||||||
/* Skippable frame */
|
|
||||||
else if ((magicNumber & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) {
|
|
||||||
U32 const frameSize = MEM_readLE32(headerBuffer + 4);
|
|
||||||
long const seek = (long)(8 + frameSize - numBytesRead);
|
|
||||||
int const ret = LONG_SEEK(srcFile, seek, SEEK_CUR);
|
|
||||||
if (ret != 0) {
|
|
||||||
DISPLAY("Error: could not find end of skippable frame\n");
|
|
||||||
detectError = 1;
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
info->numSkippableFrames++;
|
|
||||||
}
|
|
||||||
/* unknown content */
|
|
||||||
else {
|
|
||||||
detectError = 2;
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} /* end analyzing frame */
|
|
||||||
fclose(srcFile);
|
|
||||||
return detectError;
|
|
||||||
}
|
|
||||||
|
|
||||||
static void displayInfo(const char* inFileName, fileInfo_t* info, int displayLevel){
|
|
||||||
unsigned const unit = info->compressedSize < (1 MB) ? (1 KB) : (1 MB);
|
|
||||||
const char* const unitStr = info->compressedSize < (1 MB) ? "KB" : "MB";
|
|
||||||
double const compressedSizeUnit = (double)info->compressedSize / unit;
|
|
||||||
double const decompressedSizeUnit = (double)info->decompressedSize / unit;
|
|
||||||
double const ratio = (info->compressedSize == 0) ? 0 : ((double)info->decompressedSize)/info->compressedSize;
|
|
||||||
const char* const checkString = (info->usesCheck ? "XXH64" : "None");
|
|
||||||
if (displayLevel <= 2) {
|
|
||||||
if (!info->decompUnavailable) {
|
|
||||||
DISPLAYOUT("Skippable Non-Skippable Compressed Uncompressed Ratio Check Filename\n");
|
|
||||||
DISPLAYOUT("%9d %13d %7.2f %2s %9.2f %2s %5.3f %5s %s\n",
|
|
||||||
info->numSkippableFrames, info->numActualFrames,
|
|
||||||
compressedSizeUnit, unitStr, decompressedSizeUnit, unitStr,
|
|
||||||
ratio, checkString, inFileName);
|
|
||||||
} else {
|
|
||||||
DISPLAYOUT("Skippable Non-Skippable Compressed Check Filename\n");
|
|
||||||
DISPLAYOUT("%9d %13d %7.2f MB %5s %s\n",
|
|
||||||
info->numSkippableFrames, info->numActualFrames,
|
|
||||||
compressedSizeUnit, checkString, inFileName);
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
DISPLAYOUT("# Zstandard Frames: %d\n", info->numActualFrames);
|
|
||||||
DISPLAYOUT("# Skippable Frames: %d\n", info->numSkippableFrames);
|
|
||||||
DISPLAYOUT("Compressed Size: %.2f %2s (%llu B)\n",
|
|
||||||
compressedSizeUnit, unitStr, info->compressedSize);
|
|
||||||
if (!info->decompUnavailable) {
|
|
||||||
DISPLAYOUT("Decompressed Size: %.2f %2s (%llu B)\n",
|
|
||||||
decompressedSizeUnit, unitStr, info->decompressedSize);
|
|
||||||
DISPLAYOUT("Ratio: %.4f\n", ratio);
|
|
||||||
}
|
|
||||||
DISPLAYOUT("Check: %s\n", checkString);
|
|
||||||
DISPLAYOUT("\n");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
static int FIO_listFile(const char* inFileName, int displayLevel, unsigned fileNo, unsigned numFiles){
|
|
||||||
/* initialize info to avoid warnings */
|
|
||||||
fileInfo_t info;
|
|
||||||
memset(&info, 0, sizeof(info));
|
|
||||||
DISPLAYOUT("%s (%u/%u):\n", inFileName, fileNo, numFiles);
|
|
||||||
{
|
|
||||||
int const error = getFileInfo(&info, inFileName);
|
|
||||||
if (error == 1) {
|
|
||||||
/* display error, but provide output */
|
|
||||||
DISPLAY("An error occurred with getting file info\n");
|
|
||||||
}
|
|
||||||
else if (error == 2) {
|
|
||||||
DISPLAYOUT("File %s not compressed with zstd\n", inFileName);
|
|
||||||
if (displayLevel > 2) {
|
|
||||||
DISPLAYOUT("\n");
|
|
||||||
}
|
|
||||||
return 1;
|
|
||||||
}
|
|
||||||
else if (error == 3) {
|
|
||||||
/* error occurred with opening the file */
|
|
||||||
if (displayLevel > 2) {
|
|
||||||
DISPLAYOUT("\n");
|
|
||||||
}
|
|
||||||
return 1;
|
|
||||||
}
|
|
||||||
displayInfo(inFileName, &info, displayLevel);
|
|
||||||
return error;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
int FIO_listMultipleFiles(unsigned numFiles, const char** filenameTable, int displayLevel){
|
|
||||||
if (numFiles == 0) {
|
|
||||||
DISPLAYOUT("No files given\n");
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
DISPLAYOUT("===========================================\n");
|
|
||||||
DISPLAYOUT("Printing information about compressed files\n");
|
|
||||||
DISPLAYOUT("===========================================\n");
|
|
||||||
DISPLAYOUT("Number of files listed: %u\n", numFiles);
|
|
||||||
{
|
|
||||||
int error = 0;
|
|
||||||
unsigned u;
|
|
||||||
for (u=0; u<numFiles;u++) {
|
|
||||||
error |= FIO_listFile(filenameTable[u], displayLevel, u+1, numFiles);
|
|
||||||
}
|
|
||||||
return error;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
int FIO_compressMultipleFilenames(const char** inFileNamesTable, unsigned nbFiles,
|
int FIO_compressMultipleFilenames(const char** inFileNamesTable, unsigned nbFiles,
|
||||||
const char* suffix,
|
const char* suffix,
|
||||||
@ -1187,10 +970,8 @@ int FIO_compressMultipleFilenames(const char** inFileNamesTable, unsigned nbFile
|
|||||||
missed_files += FIO_compressFilename_dstFile(ress, dstFileName, inFileNamesTable[u], compressionLevel);
|
missed_files += FIO_compressFilename_dstFile(ress, dstFileName, inFileNamesTable[u], compressionLevel);
|
||||||
} }
|
} }
|
||||||
|
|
||||||
/* Close & Free */
|
|
||||||
FIO_freeCResources(ress);
|
FIO_freeCResources(ress);
|
||||||
free(dstFileName);
|
free(dstFileName);
|
||||||
|
|
||||||
return missed_files;
|
return missed_files;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -1201,8 +982,8 @@ int FIO_compressMultipleFilenames(const char** inFileNamesTable, unsigned nbFile
|
|||||||
#ifndef ZSTD_NODECOMPRESS
|
#ifndef ZSTD_NODECOMPRESS
|
||||||
|
|
||||||
/* **************************************************************************
|
/* **************************************************************************
|
||||||
* Decompression
|
* Decompression
|
||||||
****************************************************************************/
|
***************************************************************************/
|
||||||
typedef struct {
|
typedef struct {
|
||||||
void* srcBuffer;
|
void* srcBuffer;
|
||||||
size_t srcBufferLoaded;
|
size_t srcBufferLoaded;
|
||||||
@ -1871,4 +1652,231 @@ int FIO_decompressMultipleFilenames(const char** srcNamesTable, unsigned nbFiles
|
|||||||
return missingFiles + skippedFiles;
|
return missingFiles + skippedFiles;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
/* **************************************************************************
|
||||||
|
* .zst file info (--list command)
|
||||||
|
***************************************************************************/
|
||||||
|
|
||||||
|
typedef struct {
|
||||||
|
int numActualFrames;
|
||||||
|
int numSkippableFrames;
|
||||||
|
unsigned long long decompressedSize;
|
||||||
|
int decompUnavailable;
|
||||||
|
unsigned long long compressedSize;
|
||||||
|
int usesCheck;
|
||||||
|
} fileInfo_t;
|
||||||
|
|
||||||
|
/** getFileInfo() :
|
||||||
|
* Reads information from file, stores in *info
|
||||||
|
* @return : 0 if successful
|
||||||
|
* 1 for frame analysis error
|
||||||
|
* 2 for file not compressed with zstd
|
||||||
|
* 3 for cases in which file could not be opened.
|
||||||
|
*/
|
||||||
|
static int getFileInfo(fileInfo_t* info, const char* inFileName){
|
||||||
|
int detectError = 0;
|
||||||
|
FILE* const srcFile = FIO_openSrcFile(inFileName);
|
||||||
|
if (srcFile == NULL) {
|
||||||
|
DISPLAY("Error: could not open source file %s\n", inFileName);
|
||||||
|
return 3;
|
||||||
|
}
|
||||||
|
info->compressedSize = (unsigned long long)UTIL_getFileSize(inFileName);
|
||||||
|
|
||||||
|
/* begin analyzing frame */
|
||||||
|
for ( ; ; ) {
|
||||||
|
BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX];
|
||||||
|
size_t const numBytesRead = fread(headerBuffer, 1, sizeof(headerBuffer), srcFile);
|
||||||
|
if (numBytesRead < ZSTD_frameHeaderSize_min) {
|
||||||
|
if (feof(srcFile) && numBytesRead == 0 && info->compressedSize > 0) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
else if (feof(srcFile)) {
|
||||||
|
DISPLAY("Error: reached end of file with incomplete frame\n");
|
||||||
|
detectError = 2;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
else {
|
||||||
|
DISPLAY("Error: did not reach end of file but ran out of frames\n");
|
||||||
|
detectError = 1;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
{ U32 const magicNumber = MEM_readLE32(headerBuffer);
|
||||||
|
/* Zstandard frame */
|
||||||
|
if (magicNumber == ZSTD_MAGICNUMBER) {
|
||||||
|
U64 const frameContentSize = ZSTD_getFrameContentSize(headerBuffer, numBytesRead);
|
||||||
|
if (frameContentSize == ZSTD_CONTENTSIZE_ERROR || frameContentSize == ZSTD_CONTENTSIZE_UNKNOWN) {
|
||||||
|
info->decompUnavailable = 1;
|
||||||
|
} else {
|
||||||
|
info->decompressedSize += frameContentSize;
|
||||||
|
}
|
||||||
|
/* move to the end of the frame header */
|
||||||
|
{ size_t const headerSize = ZSTD_frameHeaderSize(headerBuffer, numBytesRead);
|
||||||
|
if (ZSTD_isError(headerSize)) {
|
||||||
|
DISPLAY("Error: could not determine frame header size\n");
|
||||||
|
detectError = 1;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
{ int const ret = fseek(srcFile, ((long)headerSize)-((long)numBytesRead), SEEK_CUR);
|
||||||
|
if (ret != 0) {
|
||||||
|
DISPLAY("Error: could not move to end of frame header\n");
|
||||||
|
detectError = 1;
|
||||||
|
break;
|
||||||
|
} } }
|
||||||
|
|
||||||
|
/* skip the rest of the blocks in the frame */
|
||||||
|
{ int lastBlock = 0;
|
||||||
|
do {
|
||||||
|
BYTE blockHeaderBuffer[3];
|
||||||
|
size_t const readBytes = fread(blockHeaderBuffer, 1, 3, srcFile);
|
||||||
|
if (readBytes != 3) {
|
||||||
|
DISPLAY("There was a problem reading the block header\n");
|
||||||
|
detectError = 1;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
{ U32 const blockHeader = MEM_readLE24(blockHeaderBuffer);
|
||||||
|
U32 const blockTypeID = (blockHeader >> 1) & 3;
|
||||||
|
U32 const isRLE = (blockTypeID == 1);
|
||||||
|
U32 const isWrongBlock = (blockTypeID == 3);
|
||||||
|
long const blockSize = isRLE ? 1 : (long)(blockHeader >> 3);
|
||||||
|
if (isWrongBlock) {
|
||||||
|
DISPLAY("Error: unsupported block type \n");
|
||||||
|
detectError = 1;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
lastBlock = blockHeader & 1;
|
||||||
|
{ int const ret = fseek(srcFile, blockSize, SEEK_CUR);
|
||||||
|
if (ret != 0) {
|
||||||
|
DISPLAY("Error: could not skip to end of block\n");
|
||||||
|
detectError = 1;
|
||||||
|
break;
|
||||||
|
} } }
|
||||||
|
} while (lastBlock != 1);
|
||||||
|
|
||||||
|
if (detectError) break;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* check if checksum is used */
|
||||||
|
{ BYTE const frameHeaderDescriptor = headerBuffer[4];
|
||||||
|
int const contentChecksumFlag = (frameHeaderDescriptor & (1 << 2)) >> 2;
|
||||||
|
if (contentChecksumFlag) {
|
||||||
|
int const ret = fseek(srcFile, 4, SEEK_CUR);
|
||||||
|
info->usesCheck = 1;
|
||||||
|
if (ret != 0) {
|
||||||
|
DISPLAY("Error: could not skip past checksum\n");
|
||||||
|
detectError = 1;
|
||||||
|
break;
|
||||||
|
} } }
|
||||||
|
info->numActualFrames++;
|
||||||
|
}
|
||||||
|
/* Skippable frame */
|
||||||
|
else if ((magicNumber & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) {
|
||||||
|
U32 const frameSize = MEM_readLE32(headerBuffer + 4);
|
||||||
|
long const seek = (long)(8 + frameSize - numBytesRead);
|
||||||
|
int const ret = LONG_SEEK(srcFile, seek, SEEK_CUR);
|
||||||
|
if (ret != 0) {
|
||||||
|
DISPLAY("Error: could not find end of skippable frame\n");
|
||||||
|
detectError = 1;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
info->numSkippableFrames++;
|
||||||
|
}
|
||||||
|
/* unknown content */
|
||||||
|
else {
|
||||||
|
detectError = 2;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} /* end analyzing frame */
|
||||||
|
fclose(srcFile);
|
||||||
|
return detectError;
|
||||||
|
}
|
||||||
|
|
||||||
|
static void displayInfo(const char* inFileName, fileInfo_t* info, int displayLevel){
|
||||||
|
unsigned const unit = info->compressedSize < (1 MB) ? (1 KB) : (1 MB);
|
||||||
|
const char* const unitStr = info->compressedSize < (1 MB) ? "KB" : "MB";
|
||||||
|
double const compressedSizeUnit = (double)info->compressedSize / unit;
|
||||||
|
double const decompressedSizeUnit = (double)info->decompressedSize / unit;
|
||||||
|
double const ratio = (info->compressedSize == 0) ? 0 : ((double)info->decompressedSize)/info->compressedSize;
|
||||||
|
const char* const checkString = (info->usesCheck ? "XXH64" : "None");
|
||||||
|
if (displayLevel <= 2) {
|
||||||
|
if (!info->decompUnavailable) {
|
||||||
|
DISPLAYOUT("Skippable Non-Skippable Compressed Uncompressed Ratio Check Filename\n");
|
||||||
|
DISPLAYOUT("%9d %13d %7.2f %2s %9.2f %2s %5.3f %5s %s\n",
|
||||||
|
info->numSkippableFrames, info->numActualFrames,
|
||||||
|
compressedSizeUnit, unitStr, decompressedSizeUnit, unitStr,
|
||||||
|
ratio, checkString, inFileName);
|
||||||
|
} else {
|
||||||
|
DISPLAYOUT("Skippable Non-Skippable Compressed Check Filename\n");
|
||||||
|
DISPLAYOUT("%9d %13d %7.2f MB %5s %s\n",
|
||||||
|
info->numSkippableFrames, info->numActualFrames,
|
||||||
|
compressedSizeUnit, checkString, inFileName);
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
DISPLAYOUT("# Zstandard Frames: %d\n", info->numActualFrames);
|
||||||
|
DISPLAYOUT("# Skippable Frames: %d\n", info->numSkippableFrames);
|
||||||
|
DISPLAYOUT("Compressed Size: %.2f %2s (%llu B)\n",
|
||||||
|
compressedSizeUnit, unitStr, info->compressedSize);
|
||||||
|
if (!info->decompUnavailable) {
|
||||||
|
DISPLAYOUT("Decompressed Size: %.2f %2s (%llu B)\n",
|
||||||
|
decompressedSizeUnit, unitStr, info->decompressedSize);
|
||||||
|
DISPLAYOUT("Ratio: %.4f\n", ratio);
|
||||||
|
}
|
||||||
|
DISPLAYOUT("Check: %s\n", checkString);
|
||||||
|
DISPLAYOUT("\n");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
static int FIO_listFile(const char* inFileName, int displayLevel, unsigned fileNo, unsigned numFiles){
|
||||||
|
/* initialize info to avoid warnings */
|
||||||
|
fileInfo_t info;
|
||||||
|
memset(&info, 0, sizeof(info));
|
||||||
|
DISPLAYOUT("%s (%u/%u):\n", inFileName, fileNo, numFiles);
|
||||||
|
{
|
||||||
|
int const error = getFileInfo(&info, inFileName);
|
||||||
|
if (error == 1) {
|
||||||
|
/* display error, but provide output */
|
||||||
|
DISPLAY("An error occurred with getting file info\n");
|
||||||
|
}
|
||||||
|
else if (error == 2) {
|
||||||
|
DISPLAYOUT("File %s not compressed with zstd\n", inFileName);
|
||||||
|
if (displayLevel > 2) {
|
||||||
|
DISPLAYOUT("\n");
|
||||||
|
}
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
else if (error == 3) {
|
||||||
|
/* error occurred with opening the file */
|
||||||
|
if (displayLevel > 2) {
|
||||||
|
DISPLAYOUT("\n");
|
||||||
|
}
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
displayInfo(inFileName, &info, displayLevel);
|
||||||
|
return error;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
int FIO_listMultipleFiles(unsigned numFiles, const char** filenameTable, int displayLevel){
|
||||||
|
if (numFiles == 0) {
|
||||||
|
DISPLAYOUT("No files given\n");
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
DISPLAYOUT("===========================================\n");
|
||||||
|
DISPLAYOUT("Printing information about compressed files\n");
|
||||||
|
DISPLAYOUT("===========================================\n");
|
||||||
|
DISPLAYOUT("Number of files listed: %u\n", numFiles);
|
||||||
|
{
|
||||||
|
int error = 0;
|
||||||
|
unsigned u;
|
||||||
|
for (u=0; u<numFiles;u++) {
|
||||||
|
error |= FIO_listFile(filenameTable[u], displayLevel, u+1, numFiles);
|
||||||
|
}
|
||||||
|
return error;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
#endif /* #ifndef ZSTD_NODECOMPRESS */
|
#endif /* #ifndef ZSTD_NODECOMPRESS */
|
||||||
|
@ -634,7 +634,10 @@ int main(int argCount, const char* argv[])
|
|||||||
filenameTable[filenameIdx++] = argument;
|
filenameTable[filenameIdx++] = argument;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (lastCommand) { DISPLAY("error : command must be followed by argument \n"); CLEAN_RETURN(1); } /* forgotten argument */
|
if (lastCommand) { /* forgotten argument */
|
||||||
|
DISPLAY("error : command must be followed by argument \n");
|
||||||
|
CLEAN_RETURN(1);
|
||||||
|
}
|
||||||
|
|
||||||
/* Welcome message (if verbose) */
|
/* Welcome message (if verbose) */
|
||||||
DISPLAYLEVEL(3, WELCOME_MESSAGE);
|
DISPLAYLEVEL(3, WELCOME_MESSAGE);
|
||||||
@ -642,7 +645,7 @@ int main(int argCount, const char* argv[])
|
|||||||
DISPLAYLEVEL(4, "_POSIX_C_SOURCE defined: %ldL\n", (long) _POSIX_C_SOURCE);
|
DISPLAYLEVEL(4, "_POSIX_C_SOURCE defined: %ldL\n", (long) _POSIX_C_SOURCE);
|
||||||
#endif
|
#endif
|
||||||
#ifdef _POSIX_VERSION
|
#ifdef _POSIX_VERSION
|
||||||
DISPLAYLEVEL(4, "_POSIX_VERSION defined: %ldL\n", (long) _POSIX_VERSION);
|
DISPLAYLEVEL(4, "_POSIX_VERSION defined: %ldL \n", (long) _POSIX_VERSION);
|
||||||
#endif
|
#endif
|
||||||
#ifdef PLATFORM_POSIX_VERSION
|
#ifdef PLATFORM_POSIX_VERSION
|
||||||
DISPLAYLEVEL(4, "PLATFORM_POSIX_VERSION defined: %ldL\n", (long) PLATFORM_POSIX_VERSION);
|
DISPLAYLEVEL(4, "PLATFORM_POSIX_VERSION defined: %ldL\n", (long) PLATFORM_POSIX_VERSION);
|
||||||
@ -651,7 +654,7 @@ int main(int argCount, const char* argv[])
|
|||||||
if (nbThreads == 0) {
|
if (nbThreads == 0) {
|
||||||
/* try to guess */
|
/* try to guess */
|
||||||
nbThreads = UTIL_countPhysicalCores();
|
nbThreads = UTIL_countPhysicalCores();
|
||||||
DISPLAYLEVEL(3, "Note: %d physical core(s) detected\n", nbThreads);
|
DISPLAYLEVEL(3, "Note: %d physical core(s) detected \n", nbThreads);
|
||||||
}
|
}
|
||||||
|
|
||||||
g_utilDisplayLevel = g_displayLevel;
|
g_utilDisplayLevel = g_displayLevel;
|
||||||
@ -678,10 +681,17 @@ int main(int argCount, const char* argv[])
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
if (operation == zom_list) {
|
if (operation == zom_list) {
|
||||||
|
#ifndef ZSTD_NODECOMPRESS
|
||||||
int const ret = FIO_listMultipleFiles(filenameIdx, filenameTable, g_displayLevel);
|
int const ret = FIO_listMultipleFiles(filenameIdx, filenameTable, g_displayLevel);
|
||||||
CLEAN_RETURN(ret);
|
CLEAN_RETURN(ret);
|
||||||
|
#else
|
||||||
|
DISPLAY("file information is not supported \n");
|
||||||
|
CLEAN_RETURN(1);
|
||||||
|
#endif
|
||||||
}
|
}
|
||||||
|
|
||||||
/* Check if benchmark is selected */
|
/* Check if benchmark is selected */
|
||||||
if (operation==zom_bench) {
|
if (operation==zom_bench) {
|
||||||
#ifndef ZSTD_NOBENCH
|
#ifndef ZSTD_NOBENCH
|
||||||
|
Loading…
x
Reference in New Issue
Block a user