|
|
|
@ -91,6 +91,7 @@
|
|
|
|
|
* Macros
|
|
|
|
|
***************************************/
|
|
|
|
|
#define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
|
|
|
|
|
#define DISPLAYOUT(...) fprintf(stdout, __VA_ARGS__)
|
|
|
|
|
#define DISPLAYLEVEL(l, ...) { if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); } }
|
|
|
|
|
static int g_displayLevel = 2; /* 0 : no display; 1: errors; 2 : + result + interaction + warnings; 3 : + progression; 4 : + information */
|
|
|
|
|
void FIO_setNotificationLevel(unsigned level) { g_displayLevel=level; }
|
|
|
|
@ -871,182 +872,182 @@ typedef struct {
|
|
|
|
|
int usesCheck;
|
|
|
|
|
} fileInfo_t;
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
int calcFrameHeaderSize(BYTE frameHeaderDescriptor){
|
|
|
|
|
int frameContentSizeBytes = 0;
|
|
|
|
|
int windowDescriptorBytes;
|
|
|
|
|
int dictionaryIDBytes;
|
|
|
|
|
int frameContentSizeFlag = frameHeaderDescriptor >> 6;
|
|
|
|
|
int singleSegmentFlag = (frameHeaderDescriptor & (1 << 5)) >> 5;
|
|
|
|
|
int dictionaryIDFlag = frameHeaderDescriptor & 3;
|
|
|
|
|
if(frameContentSizeFlag!=0){
|
|
|
|
|
frameContentSizeBytes = 1 << frameContentSizeFlag;
|
|
|
|
|
}
|
|
|
|
|
else if(singleSegmentFlag){
|
|
|
|
|
frameContentSizeBytes = 1;
|
|
|
|
|
}
|
|
|
|
|
windowDescriptorBytes = singleSegmentFlag ? 0 : 1;
|
|
|
|
|
dictionaryIDBytes = dictionaryIDFlag ? 1 << (dictionaryIDFlag - 1): 0;
|
|
|
|
|
return 4 + 1 + windowDescriptorBytes + frameContentSizeBytes + dictionaryIDBytes;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* Reads information from file, stores in *info
|
|
|
|
|
* if successful, returns 0, otherwise returns 1
|
|
|
|
|
*/
|
|
|
|
|
int getFileInfo(fileInfo_t* info, const char* inFileName){
|
|
|
|
|
FILE* srcFile = FIO_openSrcFile(inFileName);
|
|
|
|
|
if(srcFile==NULL){
|
|
|
|
|
static int getFileInfo(fileInfo_t* info, const char* inFileName){
|
|
|
|
|
int detectError = 0;
|
|
|
|
|
FILE* const srcFile = FIO_openSrcFile(inFileName);
|
|
|
|
|
if (srcFile == NULL) {
|
|
|
|
|
DISPLAY("Error: could not open source file %s\n", inFileName);
|
|
|
|
|
return 1;
|
|
|
|
|
}
|
|
|
|
|
info->compressedSize = (unsigned long long)UTIL_getFileSize(inFileName);
|
|
|
|
|
info->decompressedSize = 0;
|
|
|
|
|
info->numActualFrames = 0;
|
|
|
|
|
info-> numSkippableFrames = 0;
|
|
|
|
|
info->numSkippableFrames = 0;
|
|
|
|
|
info->canComputeDecompSize = 1;
|
|
|
|
|
/* begin analyzing frame */
|
|
|
|
|
while(1){
|
|
|
|
|
BYTE magicNumberBuffer[4];
|
|
|
|
|
size_t numBytesRead = fread(magicNumberBuffer, 1, 4, srcFile);
|
|
|
|
|
U32 magicNumber;
|
|
|
|
|
if(numBytesRead != 4) break;
|
|
|
|
|
magicNumber = MEM_readLE32(magicNumberBuffer);
|
|
|
|
|
if(magicNumber==ZSTD_MAGICNUMBER){
|
|
|
|
|
BYTE frameHeaderDescriptor;
|
|
|
|
|
int totalFrameHeaderBytes;
|
|
|
|
|
BYTE* frameHeader;
|
|
|
|
|
int lastBlock = 0;
|
|
|
|
|
size_t readBytes = fread(&frameHeaderDescriptor, 1, 1, srcFile);
|
|
|
|
|
info->numActualFrames++;
|
|
|
|
|
if(readBytes != 1){
|
|
|
|
|
DISPLAY("There was an error with reading frame header descriptor\n");
|
|
|
|
|
exit(1);
|
|
|
|
|
for( ; ; ){
|
|
|
|
|
BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX];
|
|
|
|
|
size_t const numBytesRead = fread(headerBuffer, 1, sizeof(headerBuffer), srcFile);
|
|
|
|
|
if (numBytesRead < ZSTD_frameHeaderSize_min) {
|
|
|
|
|
if(feof(srcFile)){
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
/* calculate actual frame header size */
|
|
|
|
|
totalFrameHeaderBytes = calcFrameHeaderSize(frameHeaderDescriptor);
|
|
|
|
|
|
|
|
|
|
/* reset to beginning of from and read entire header */
|
|
|
|
|
fseek(srcFile, -5, SEEK_CUR);
|
|
|
|
|
frameHeader = (BYTE*)malloc(totalFrameHeaderBytes);
|
|
|
|
|
readBytes = fread(frameHeader, 1, totalFrameHeaderBytes, srcFile);
|
|
|
|
|
if(readBytes != (size_t)totalFrameHeaderBytes){
|
|
|
|
|
DISPLAY("There was an error reading the frame header\n");
|
|
|
|
|
exit(1);
|
|
|
|
|
else{
|
|
|
|
|
DISPLAY("Error: did not reach end of file but ran out of frames\n");
|
|
|
|
|
detectError = 1;
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/* get decompressed file size */
|
|
|
|
|
{
|
|
|
|
|
U64 additional = ZSTD_getFrameContentSize(frameHeader, totalFrameHeaderBytes);
|
|
|
|
|
if(additional!=ZSTD_CONTENTSIZE_UNKNOWN && additional!=ZSTD_CONTENTSIZE_ERROR){
|
|
|
|
|
info->decompressedSize += additional;
|
|
|
|
|
}
|
|
|
|
|
else{
|
|
|
|
|
}
|
|
|
|
|
{
|
|
|
|
|
U32 const magicNumber = MEM_readLE32(headerBuffer);
|
|
|
|
|
if (magicNumber == ZSTD_MAGICNUMBER) {
|
|
|
|
|
U64 const frameContentSize = ZSTD_getFrameContentSize(headerBuffer, numBytesRead);
|
|
|
|
|
if (frameContentSize == ZSTD_CONTENTSIZE_ERROR || frameContentSize == ZSTD_CONTENTSIZE_UNKNOWN) {
|
|
|
|
|
info->canComputeDecompSize = 0;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
else {
|
|
|
|
|
info->decompressedSize += frameContentSize;
|
|
|
|
|
}
|
|
|
|
|
{
|
|
|
|
|
/* move to the end of the frame header */
|
|
|
|
|
size_t const headerSize = ZSTD_frameHeaderSize(headerBuffer, numBytesRead);
|
|
|
|
|
if (ZSTD_isError(headerSize)) {
|
|
|
|
|
DISPLAY("Error: could not determine frame header size\n");
|
|
|
|
|
detectError = 1;
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
{
|
|
|
|
|
int const ret = fseek(srcFile, ((long)headerSize)-((long)numBytesRead), SEEK_CUR);
|
|
|
|
|
if (ret != 0) {
|
|
|
|
|
DISPLAY("Error: could not move to end of frame header\n");
|
|
|
|
|
detectError = 1;
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/* skip the rest of the blocks in the frame */
|
|
|
|
|
do{
|
|
|
|
|
BYTE blockHeaderBuffer[3];
|
|
|
|
|
U32 blockHeader;
|
|
|
|
|
int blockSize;
|
|
|
|
|
readBytes = fread(blockHeaderBuffer, 1, 3, srcFile);
|
|
|
|
|
if(readBytes != 3){
|
|
|
|
|
DISPLAY("There was a problem reading the block header\n");
|
|
|
|
|
exit(1);
|
|
|
|
|
/* skip the rest of the blocks in the frame */
|
|
|
|
|
{
|
|
|
|
|
int lastBlock = 0;
|
|
|
|
|
size_t readBytes = 0;
|
|
|
|
|
do{
|
|
|
|
|
BYTE blockHeaderBuffer[3];
|
|
|
|
|
U32 blockHeader;
|
|
|
|
|
int blockSize;
|
|
|
|
|
readBytes = fread(blockHeaderBuffer, 1, 3, srcFile);
|
|
|
|
|
if (readBytes != 3) {
|
|
|
|
|
DISPLAY("There was a problem reading the block header\n");
|
|
|
|
|
detectError = 1;
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
blockHeader = MEM_readLE24(blockHeaderBuffer);
|
|
|
|
|
lastBlock = blockHeader & 1;
|
|
|
|
|
blockSize = (blockHeader - (blockHeader & 7)) >> 3;
|
|
|
|
|
{
|
|
|
|
|
int const ret = fseek(srcFile, blockSize, SEEK_CUR);
|
|
|
|
|
if (ret != 0) {
|
|
|
|
|
DISPLAY("Error: could not skip to end of block\n");
|
|
|
|
|
detectError = 1;
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
} while (lastBlock != 1);
|
|
|
|
|
|
|
|
|
|
if (detectError) {
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
blockHeader = MEM_readLE24(blockHeaderBuffer);
|
|
|
|
|
lastBlock = blockHeader & 1;
|
|
|
|
|
blockSize = (blockHeader - (blockHeader & 7)) >> 3;
|
|
|
|
|
fseek(srcFile, blockSize, SEEK_CUR);
|
|
|
|
|
}while(lastBlock != 1);
|
|
|
|
|
{
|
|
|
|
|
/* check if checksum is used */
|
|
|
|
|
int contentChecksumFlag = (frameHeaderDescriptor & (1 << 2)) >> 2;
|
|
|
|
|
if(contentChecksumFlag){
|
|
|
|
|
info->usesCheck = 1;
|
|
|
|
|
{
|
|
|
|
|
/* check if checksum is used */
|
|
|
|
|
BYTE const frameHeaderDescriptor = headerBuffer[4];
|
|
|
|
|
int const contentChecksumFlag = (frameHeaderDescriptor & (1 << 2)) >> 2;
|
|
|
|
|
if (contentChecksumFlag) {
|
|
|
|
|
info->usesCheck = 1;
|
|
|
|
|
}
|
|
|
|
|
if (contentChecksumFlag) {
|
|
|
|
|
int const ret = fseek(srcFile, 4, SEEK_CUR);
|
|
|
|
|
if (ret != 0) {
|
|
|
|
|
DISPLAY("Error: could not skip past checksum\n");
|
|
|
|
|
detectError = 1;
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
if(contentChecksumFlag){
|
|
|
|
|
fseek(srcFile, 4, SEEK_CUR);
|
|
|
|
|
info->numActualFrames++;
|
|
|
|
|
}
|
|
|
|
|
else if (magicNumber == ZSTD_MAGIC_SKIPPABLE_START) {
|
|
|
|
|
BYTE frameSizeBuffer[4];
|
|
|
|
|
size_t readBytes = fread(frameSizeBuffer, 1, 4, srcFile);
|
|
|
|
|
if (readBytes != 4) {
|
|
|
|
|
DISPLAY("There was an error reading skippable frame size");
|
|
|
|
|
detectError = 1;
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
{
|
|
|
|
|
long const frameSize = MEM_readLE32(frameSizeBuffer);
|
|
|
|
|
int const ret = fseek(srcFile, frameSize, SEEK_CUR);
|
|
|
|
|
if (ret != 0) {
|
|
|
|
|
DISPLAY("Error: could not find end of skippable frame\n");
|
|
|
|
|
detectError = 1;
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
info->numSkippableFrames++;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
else if(magicNumber==ZSTD_MAGIC_SKIPPABLE_START){
|
|
|
|
|
BYTE frameSizeBuffer[4];
|
|
|
|
|
long frameSize;
|
|
|
|
|
size_t readBytes = fread(frameSizeBuffer, 1, 4, srcFile);
|
|
|
|
|
info->numSkippableFrames++;
|
|
|
|
|
if(readBytes != 4){
|
|
|
|
|
DISPLAY("There was an error reading skippable frame size");
|
|
|
|
|
exit(1);
|
|
|
|
|
}
|
|
|
|
|
frameSize = MEM_readLE32(frameSizeBuffer);
|
|
|
|
|
fseek(srcFile, frameSize, SEEK_CUR);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
}
|
|
|
|
|
return 0;
|
|
|
|
|
fclose(srcFile);
|
|
|
|
|
return detectError;
|
|
|
|
|
}
|
|
|
|
|
void displayInfo(const char* inFileName, fileInfo_t* info, int displayLevel){
|
|
|
|
|
double compressedSizeMB = (double)info->compressedSize/(1 MB);
|
|
|
|
|
double decompressedSizeMB = (double)info->decompressedSize/(1 MB);
|
|
|
|
|
|
|
|
|
|
static void displayInfo(const char* inFileName, fileInfo_t* info, int displayLevel){
|
|
|
|
|
double const compressedSizeMB = (double)info->compressedSize/(1 MB);
|
|
|
|
|
double const decompressedSizeMB = (double)info->decompressedSize/(1 MB);
|
|
|
|
|
const char* checkString = (info->usesCheck ? "XXH64" : "None");
|
|
|
|
|
if(displayLevel<=2){
|
|
|
|
|
if(info->usesCheck && info->canComputeDecompSize){
|
|
|
|
|
DISPLAY("Skippable Non-Skippable Compressed Uncompressed Ratio Check Filename\n");
|
|
|
|
|
DISPLAY("%9d %13d %7.2f MB %7.2f MB %5.3f XXH64 %s\n",
|
|
|
|
|
if (info->canComputeDecompSize) {
|
|
|
|
|
DISPLAYOUT("Skippable Non-Skippable Compressed Uncompressed Ratio Check Filename\n");
|
|
|
|
|
DISPLAYOUT("%9d %13d %7.2f MB %9.2f MB %5.3f %s %s\n",
|
|
|
|
|
info->numSkippableFrames, info->numActualFrames, compressedSizeMB, decompressedSizeMB,
|
|
|
|
|
compressedSizeMB/decompressedSizeMB, inFileName);
|
|
|
|
|
compressedSizeMB/decompressedSizeMB, checkString, inFileName);
|
|
|
|
|
}
|
|
|
|
|
else if(!info->usesCheck){
|
|
|
|
|
DISPLAY("Skippable Non-Skippable Compressed Uncompressed Ratio Check Filename\n");
|
|
|
|
|
DISPLAY("%9d %13d %7.2f MB %7.2f MB %5.3f %s\n",
|
|
|
|
|
info->numSkippableFrames, info->numActualFrames, compressedSizeMB, decompressedSizeMB,
|
|
|
|
|
compressedSizeMB/decompressedSizeMB, inFileName);
|
|
|
|
|
}
|
|
|
|
|
else if(!info->canComputeDecompSize){
|
|
|
|
|
DISPLAY("Skippable Non-Skippable Compressed Uncompressed Ratio Check Filename\n");
|
|
|
|
|
DISPLAY("%9d %13d %7.2f MB XXH64 %s\n",
|
|
|
|
|
info->numSkippableFrames, info->numActualFrames, compressedSizeMB, inFileName);
|
|
|
|
|
}
|
|
|
|
|
else{
|
|
|
|
|
DISPLAY("Skippable Non-Skippable Filename\n");
|
|
|
|
|
DISPLAY("%9d %13d %7.2f MB %s\n",
|
|
|
|
|
info->numSkippableFrames, info->numActualFrames, compressedSizeMB, inFileName);
|
|
|
|
|
else {
|
|
|
|
|
DISPLAYOUT("Skippable Non-Skippable Compressed Check Filename\n");
|
|
|
|
|
DISPLAYOUT("%9d %13d %7.2f MB %s %s\n",
|
|
|
|
|
info->numSkippableFrames, info->numActualFrames, compressedSizeMB, checkString, inFileName);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
else{
|
|
|
|
|
DISPLAY("# Zstandard Frames: %d\n", info->numActualFrames);
|
|
|
|
|
DISPLAY("# Skippable Frames: %d\n", info->numSkippableFrames);
|
|
|
|
|
DISPLAY("Compressed Size: %.2f MB (%llu B)\n", compressedSizeMB, info->compressedSize);
|
|
|
|
|
if(info->canComputeDecompSize){
|
|
|
|
|
DISPLAY("Decompressed Size: %.2f MB (%llu B)\n", decompressedSizeMB, info->decompressedSize);
|
|
|
|
|
DISPLAY("Ratio: %.4f\n", compressedSizeMB/decompressedSizeMB);
|
|
|
|
|
DISPLAYOUT("# Zstandard Frames: %d\n", info->numActualFrames);
|
|
|
|
|
DISPLAYOUT("# Skippable Frames: %d\n", info->numSkippableFrames);
|
|
|
|
|
DISPLAYOUT("Compressed Size: %.2f MB (%llu B)\n", compressedSizeMB, info->compressedSize);
|
|
|
|
|
if (info->canComputeDecompSize) {
|
|
|
|
|
DISPLAYOUT("Decompressed Size: %.2f MB (%llu B)\n", decompressedSizeMB, info->decompressedSize);
|
|
|
|
|
DISPLAYOUT("Ratio: %.4f\n", compressedSizeMB/decompressedSizeMB);
|
|
|
|
|
}
|
|
|
|
|
if(info->usesCheck){
|
|
|
|
|
DISPLAY("Check: XXH64\n");
|
|
|
|
|
if (info->usesCheck) {
|
|
|
|
|
DISPLAYOUT("Check: XXH64\n");
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
DISPLAYOUT("\n");
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
int FIO_listFile(const char* inFileName, int displayLevel){
|
|
|
|
|
const char* const suffixPtr = strrchr(inFileName, '.');
|
|
|
|
|
DISPLAY("File: %s\n", inFileName);
|
|
|
|
|
if(!suffixPtr || strcmp(suffixPtr, ZSTD_EXTENSION)){
|
|
|
|
|
DISPLAYLEVEL(1, "file %s was not compressed with zstd -- ignoring\n\n", inFileName);
|
|
|
|
|
return 1;
|
|
|
|
|
}
|
|
|
|
|
else{
|
|
|
|
|
fileInfo_t* info = (fileInfo_t*)malloc(sizeof(fileInfo_t));
|
|
|
|
|
int error = getFileInfo(info, inFileName);
|
|
|
|
|
if(error==1){
|
|
|
|
|
fileInfo_t info;
|
|
|
|
|
DISPLAYOUT("File: %s\n", inFileName);
|
|
|
|
|
{
|
|
|
|
|
int const error = getFileInfo(&info, inFileName);
|
|
|
|
|
if (error == 1) {
|
|
|
|
|
DISPLAY("An error occurred with getting file info\n");
|
|
|
|
|
exit(1);
|
|
|
|
|
return 1;
|
|
|
|
|
}
|
|
|
|
|
displayInfo(inFileName, info, displayLevel);
|
|
|
|
|
}
|
|
|
|
|
DISPLAY("\n");
|
|
|
|
|
displayInfo(inFileName, &info, displayLevel);
|
|
|
|
|
return 0;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|