facebook/zstd
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #1508 from facebook/fileio_lz4

Browse Source 
fixed fileio.c compilation with LZ4 enabled
This commit is contained in:
Yann Collet 2019-01-28 14:12:27 -08:00 committed by GitHub
commit 6cf253fb52
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 57 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

83
programs/fileio.c
View File

@ -195,7 +195,7 @@ static void ABRThandler(int sig) {
const char* name; const char* name;
void* addrlist[MAX_STACK_FRAMES]; void* addrlist[MAX_STACK_FRAMES];
char** symbollist; char** symbollist;
U32 addrlen, i; int addrlen, i;
switch (sig) { switch (sig) {
case SIGABRT: name = "SIGABRT"; break; case SIGABRT: name = "SIGABRT"; break;
@ -283,19 +283,19 @@ struct FIO_prefs_s {
/* Algorithm preferences */ /* Algorithm preferences */
FIO_compressionType_t compressionType; FIO_compressionType_t compressionType;
U32 sparseFileSupport; /* 0: no sparse allowed; 1: auto (file yes, stdout no); 2: force sparse */ U32 sparseFileSupport; /* 0: no sparse allowed; 1: auto (file yes, stdout no); 2: force sparse */
U32 dictIDFlag; int dictIDFlag;
U32 checksumFlag; int checksumFlag;
U32 blockSize; int blockSize;
unsigned overlapLog; int overlapLog;
U32 adaptiveMode; U32 adaptiveMode;
U32 rsyncable; int rsyncable;
int minAdaptLevel; int minAdaptLevel;
int maxAdaptLevel; int maxAdaptLevel;
U32 ldmFlag; int ldmFlag;
U32 ldmHashLog; int ldmHashLog;
U32 ldmMinMatch; int ldmMinMatch;
U32 ldmBucketSizeLog; int ldmBucketSizeLog;
U32 ldmHashRateLog; int ldmHashRateLog;
/* IO preferences */ /* IO preferences */
U32 removeSrcFile; U32 removeSrcFile;
@ -303,7 +303,7 @@ struct FIO_prefs_s {
/* Computation resources preferences */ /* Computation resources preferences */
unsigned memLimit; unsigned memLimit;
unsigned nbWorkers; int nbWorkers;
}; };
@ -352,7 +352,7 @@ void FIO_freePreferences(FIO_prefs_t* const prefs)
* Parameters: Display Options * Parameters: Display Options
***************************************/ ***************************************/
void FIO_setNotificationLevel(unsigned level) { g_display_prefs.displayLevel=level; } void FIO_setNotificationLevel(int level) { g_display_prefs.displayLevel=level; }
void FIO_setNoProgress(unsigned noProgress) { g_display_prefs.noProgress = noProgress; } void FIO_setNoProgress(unsigned noProgress) { g_display_prefs.noProgress = noProgress; }
@ -367,28 +367,28 @@ void FIO_overwriteMode(FIO_prefs_t* const prefs) { prefs->overwrite = 1; }
void FIO_setSparseWrite(FIO_prefs_t* const prefs, unsigned sparse) { prefs->sparseFileSupport = sparse; } void FIO_setSparseWrite(FIO_prefs_t* const prefs, unsigned sparse) { prefs->sparseFileSupport = sparse; }
void FIO_setDictIDFlag(FIO_prefs_t* const prefs, unsigned dictIDFlag) { prefs->dictIDFlag = dictIDFlag; } void FIO_setDictIDFlag(FIO_prefs_t* const prefs, int dictIDFlag) { prefs->dictIDFlag = dictIDFlag; }
void FIO_setChecksumFlag(FIO_prefs_t* const prefs, unsigned checksumFlag) { prefs->checksumFlag = checksumFlag; } void FIO_setChecksumFlag(FIO_prefs_t* const prefs, int checksumFlag) { prefs->checksumFlag = checksumFlag; }
void FIO_setRemoveSrcFile(FIO_prefs_t* const prefs, unsigned flag) { prefs->removeSrcFile = (flag>0); } void FIO_setRemoveSrcFile(FIO_prefs_t* const prefs, unsigned flag) { prefs->removeSrcFile = (flag>0); }
void FIO_setMemLimit(FIO_prefs_t* const prefs, unsigned memLimit) { prefs->memLimit = memLimit; } void FIO_setMemLimit(FIO_prefs_t* const prefs, unsigned memLimit) { prefs->memLimit = memLimit; }
void FIO_setNbWorkers(FIO_prefs_t* const prefs, unsigned nbWorkers) { void FIO_setNbWorkers(FIO_prefs_t* const prefs, int nbWorkers) {
#ifndef ZSTD_MULTITHREAD #ifndef ZSTD_MULTITHREAD
if (nbWorkers > 0) DISPLAYLEVEL(2, "Note : multi-threading is disabled \n"); if (nbWorkers > 0) DISPLAYLEVEL(2, "Note : multi-threading is disabled \n");
#endif #endif
prefs->nbWorkers = nbWorkers; prefs->nbWorkers = nbWorkers;
} }
void FIO_setBlockSize(FIO_prefs_t* const prefs, unsigned blockSize) { void FIO_setBlockSize(FIO_prefs_t* const prefs, int blockSize) {
if (blockSize && prefs->nbWorkers==0) if (blockSize && prefs->nbWorkers==0)
DISPLAYLEVEL(2, "Setting block size is useless in single-thread mode \n"); DISPLAYLEVEL(2, "Setting block size is useless in single-thread mode \n");
prefs->blockSize = blockSize; prefs->blockSize = blockSize;
} }
void FIO_setOverlapLog(FIO_prefs_t* const prefs, unsigned overlapLog){ void FIO_setOverlapLog(FIO_prefs_t* const prefs, int overlapLog){
if (overlapLog && prefs->nbWorkers==0) if (overlapLog && prefs->nbWorkers==0)
DISPLAYLEVEL(2, "Setting overlapLog is useless in single-thread mode \n"); DISPLAYLEVEL(2, "Setting overlapLog is useless in single-thread mode \n");
prefs->overlapLog = overlapLog; prefs->overlapLog = overlapLog;
@ -400,7 +400,7 @@ void FIO_setAdaptiveMode(FIO_prefs_t* const prefs, unsigned adapt) {
prefs->adaptiveMode = adapt; prefs->adaptiveMode = adapt;
} }
void FIO_setRsyncable(FIO_prefs_t* const prefs, unsigned rsyncable) { void FIO_setRsyncable(FIO_prefs_t* const prefs, int rsyncable) {
if ((rsyncable>0) && (prefs->nbWorkers==0)) if ((rsyncable>0) && (prefs->nbWorkers==0))
EXM_THROW(1, "Rsyncable mode is not compatible with single thread mode \n"); EXM_THROW(1, "Rsyncable mode is not compatible with single thread mode \n");
prefs->rsyncable = rsyncable; prefs->rsyncable = rsyncable;
@ -423,20 +423,20 @@ void FIO_setLdmFlag(FIO_prefs_t* const prefs, unsigned ldmFlag) {
prefs->ldmFlag = (ldmFlag>0); prefs->ldmFlag = (ldmFlag>0);
} }
void FIO_setLdmHashLog(FIO_prefs_t* const prefs, unsigned ldmHashLog) { void FIO_setLdmHashLog(FIO_prefs_t* const prefs, int ldmHashLog) {
prefs->ldmHashLog = ldmHashLog; prefs->ldmHashLog = ldmHashLog;
} }
void FIO_setLdmMinMatch(FIO_prefs_t* const prefs, unsigned ldmMinMatch) { void FIO_setLdmMinMatch(FIO_prefs_t* const prefs, int ldmMinMatch) {
prefs->ldmMinMatch = ldmMinMatch; prefs->ldmMinMatch = ldmMinMatch;
} }
void FIO_setLdmBucketSizeLog(FIO_prefs_t* const prefs, unsigned ldmBucketSizeLog) { void FIO_setLdmBucketSizeLog(FIO_prefs_t* const prefs, int ldmBucketSizeLog) {
prefs->ldmBucketSizeLog = ldmBucketSizeLog; prefs->ldmBucketSizeLog = ldmBucketSizeLog;
} }
void FIO_setLdmHashRateLog(FIO_prefs_t* const prefs, unsigned ldmHashRateLog) { void FIO_setLdmHashRateLog(FIO_prefs_t* const prefs, int ldmHashRateLog) {
prefs->ldmHashRateLog = ldmHashRateLog; prefs->ldmHashRateLog = ldmHashRateLog;
} }
@ -667,12 +667,12 @@ static cRess_t FIO_createCResources(FIO_prefs_t* const prefs,
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_ldmHashRateLog, prefs->ldmHashRateLog) ); CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_ldmHashRateLog, prefs->ldmHashRateLog) );
} }
/* compression parameters */ /* compression parameters */
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_windowLog, comprParams.windowLog) ); CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_windowLog, (int)comprParams.windowLog) );
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_chainLog, comprParams.chainLog) ); CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_chainLog, (int)comprParams.chainLog) );
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_hashLog, comprParams.hashLog) ); CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_hashLog, (int)comprParams.hashLog) );
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_searchLog, comprParams.searchLog) ); CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_searchLog, (int)comprParams.searchLog) );
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_minMatch, comprParams.minMatch) ); CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_minMatch, (int)comprParams.minMatch) );
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_targetLength, comprParams.targetLength) ); CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_targetLength, (int)comprParams.targetLength) );
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_strategy, comprParams.strategy) ); CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_strategy, comprParams.strategy) );
/* multi-threading */ /* multi-threading */
#ifdef ZSTD_MULTITHREAD #ifdef ZSTD_MULTITHREAD
@ -859,15 +859,19 @@ FIO_compressLzmaFrame(cRess_t* ress,
#endif #endif
#ifdef ZSTD_LZ4COMPRESS #ifdef ZSTD_LZ4COMPRESS
#if LZ4_VERSION_NUMBER <= 10600 #if LZ4_VERSION_NUMBER <= 10600
#define LZ4F_blockLinked blockLinked #define LZ4F_blockLinked blockLinked
#define LZ4F_max64KB max64KB #define LZ4F_max64KB max64KB
#endif #endif
static int FIO_LZ4_GetBlockSize_FromBlockId (int id) { return (1 << (8 + (2 * id))); } static int FIO_LZ4_GetBlockSize_FromBlockId (int id) { return (1 << (8 + (2 * id))); }
static unsigned long long static unsigned long long
FIO_compressLz4Frame(cRess_t* ress, FIO_compressLz4Frame(cRess_t* ress,
const char* srcFileName, U64 const srcFileSize, const char* srcFileName, U64 const srcFileSize,
int compressionLevel, U64* readsize) int compressionLevel, int checksumFlag,
U64* readsize)
{ {
const size_t blockSize = FIO_LZ4_GetBlockSize_FromBlockId(LZ4F_max64KB); const size_t blockSize = FIO_LZ4_GetBlockSize_FromBlockId(LZ4F_max64KB);
unsigned long long inFileSize = 0, outFileSize = 0; unsigned long long inFileSize = 0, outFileSize = 0;
@ -887,7 +891,7 @@ FIO_compressLz4Frame(cRess_t* ress,
prefs.compressionLevel = compressionLevel; prefs.compressionLevel = compressionLevel;
prefs.frameInfo.blockMode = LZ4F_blockLinked; prefs.frameInfo.blockMode = LZ4F_blockLinked;
prefs.frameInfo.blockSizeID = LZ4F_max64KB; prefs.frameInfo.blockSizeID = LZ4F_max64KB;
prefs.frameInfo.contentChecksumFlag = (contentChecksum_t)g_checksumFlag; prefs.frameInfo.contentChecksumFlag = (contentChecksum_t)checksumFlag;
#if LZ4_VERSION_NUMBER >= 10600 #if LZ4_VERSION_NUMBER >= 10600
prefs.frameInfo.contentSize = (srcFileSize==UTIL_FILESIZE_UNKNOWN) ? 0 : srcFileSize; prefs.frameInfo.contentSize = (srcFileSize==UTIL_FILESIZE_UNKNOWN) ? 0 : srcFileSize;
#endif #endif
@ -1086,7 +1090,7 @@ FIO_compressZstdFrame(FIO_prefs_t* const prefs,
DISPLAYLEVEL(6, "compression level adaptation check \n") DISPLAYLEVEL(6, "compression level adaptation check \n")
/* check input speed */ /* check input speed */
if (zfp.currentJobID > prefs->nbWorkers+1) { /* warm up period, to fill all workers */ if (zfp.currentJobID > (unsigned)(prefs->nbWorkers+1)) { /* warm up period, to fill all workers */
if (inputBlocked <= 0) { if (inputBlocked <= 0) {
DISPLAYLEVEL(6, "input is never blocked => input is slower than ingestion \n"); DISPLAYLEVEL(6, "input is never blocked => input is slower than ingestion \n");
speedChange = slower; speedChange = slower;
@ -1195,7 +1199,7 @@ FIO_compressFilename_internal(FIO_prefs_t* const prefs,
case FIO_lz4Compression: case FIO_lz4Compression:
#ifdef ZSTD_LZ4COMPRESS #ifdef ZSTD_LZ4COMPRESS
compressedfilesize = FIO_compressLz4Frame(&ress, srcFileName, fileSize, compressionLevel, &readsize); compressedfilesize = FIO_compressLz4Frame(&ress, srcFileName, fileSize, compressionLevel, prefs->checksumFlag, &readsize);
#else #else
(void)compressionLevel; (void)compressionLevel;
EXM_THROW(20, "zstd: %s: file cannot be compressed as lz4 (zstd compiled without ZSTD_LZ4COMPRESS) -- ignored \n", EXM_THROW(20, "zstd: %s: file cannot be compressed as lz4 (zstd compiled without ZSTD_LZ4COMPRESS) -- ignored \n",
@ -1535,7 +1539,7 @@ static unsigned FIO_fwriteSparse(FIO_prefs_t* const prefs, FILE* file, const voi
if (seekResult) if (seekResult)
EXM_THROW(74, "Sparse skip error ; try --no-sparse"); EXM_THROW(74, "Sparse skip error ; try --no-sparse");
storedSkips = 0; storedSkips = 0;
{ size_t const sizeCheck = fwrite(restPtr, 1, restEnd - restPtr, file); { size_t const sizeCheck = fwrite(restPtr, 1, (size_t)(restEnd - restPtr), file);
if (sizeCheck != (size_t)(restEnd - restPtr)) if (sizeCheck != (size_t)(restEnd - restPtr))
EXM_THROW(75, "Write error : cannot write decoded end of block"); EXM_THROW(75, "Write error : cannot write decoded end of block");
} } } } } } } }
@ -1560,7 +1564,10 @@ static void FIO_fwriteSparseEnd(FIO_prefs_t* const prefs, FILE* file, unsigned s
/** FIO_passThrough() : just copy input into output, for compatibility with gzip -df mode /** FIO_passThrough() : just copy input into output, for compatibility with gzip -df mode
@return : 0 (no error) */ @return : 0 (no error) */
static unsigned FIO_passThrough(FIO_prefs_t* const prefs, FILE* foutput, FILE* finput, void* buffer, size_t bufferSize, size_t alreadyLoaded) static int FIO_passThrough(FIO_prefs_t* const prefs,
FILE* foutput, FILE* finput,
void* buffer, size_t bufferSize,
size_t alreadyLoaded)
{ {
size_t const blockSize = MIN(64 KB, bufferSize); size_t const blockSize = MIN(64 KB, bufferSize);
size_t readFromInput = 1; size_t readFromInput = 1;
@ -1987,8 +1994,10 @@ static int FIO_decompressFrames(FIO_prefs_t* const prefs, dRess_t ress, FILE* sr
return 1; return 1;
#endif #endif
} else if ((prefs->overwrite) && !strcmp (dstFileName, stdoutmark)) { /* pass-through mode */ } else if ((prefs->overwrite) && !strcmp (dstFileName, stdoutmark)) { /* pass-through mode */
return FIO_passThrough(prefs, ress.dstFile, srcFile, return FIO_passThrough(prefs,
ress.srcBuffer, ress.srcBufferSize, ress.srcBufferLoaded); ress.dstFile, srcFile,
ress.srcBuffer, ress.srcBufferSize,
ress.srcBufferLoaded);
} else { } else {
DISPLAYLEVEL(1, "zstd: %s: unsupported format \n", srcFileName); DISPLAYLEVEL(1, "zstd: %s: unsupported format \n", srcFileName);
return 1; return 1;

22
programs/fileio.h
View File

@ -57,23 +57,23 @@ void FIO_overwriteMode(FIO_prefs_t* const prefs);
void FIO_setAdaptiveMode(FIO_prefs_t* const prefs, unsigned adapt); void FIO_setAdaptiveMode(FIO_prefs_t* const prefs, unsigned adapt);
void FIO_setAdaptMin(FIO_prefs_t* const prefs, int minCLevel); void FIO_setAdaptMin(FIO_prefs_t* const prefs, int minCLevel);
void FIO_setAdaptMax(FIO_prefs_t* const prefs, int maxCLevel); void FIO_setAdaptMax(FIO_prefs_t* const prefs, int maxCLevel);
void FIO_setBlockSize(FIO_prefs_t* const prefs, unsigned blockSize); void FIO_setBlockSize(FIO_prefs_t* const prefs, int blockSize);
void FIO_setChecksumFlag(FIO_prefs_t* const prefs, unsigned checksumFlag); void FIO_setChecksumFlag(FIO_prefs_t* const prefs, int checksumFlag);
void FIO_setDictIDFlag(FIO_prefs_t* const prefs, unsigned dictIDFlag); void FIO_setDictIDFlag(FIO_prefs_t* const prefs, int dictIDFlag);
void FIO_setLdmBucketSizeLog(FIO_prefs_t* const prefs, unsigned ldmBucketSizeLog); void FIO_setLdmBucketSizeLog(FIO_prefs_t* const prefs, int ldmBucketSizeLog);
void FIO_setLdmFlag(FIO_prefs_t* const prefs, unsigned ldmFlag); void FIO_setLdmFlag(FIO_prefs_t* const prefs, unsigned ldmFlag);
void FIO_setLdmHashRateLog(FIO_prefs_t* const prefs, unsigned ldmHashRateLog); void FIO_setLdmHashRateLog(FIO_prefs_t* const prefs, int ldmHashRateLog);
void FIO_setLdmHashLog(FIO_prefs_t* const prefs, unsigned ldmHashLog); void FIO_setLdmHashLog(FIO_prefs_t* const prefs, int ldmHashLog);
void FIO_setLdmMinMatch(FIO_prefs_t* const prefs, unsigned ldmMinMatch); void FIO_setLdmMinMatch(FIO_prefs_t* const prefs, int ldmMinMatch);
void FIO_setMemLimit(FIO_prefs_t* const prefs, unsigned memLimit); void FIO_setMemLimit(FIO_prefs_t* const prefs, unsigned memLimit);
void FIO_setNbWorkers(FIO_prefs_t* const prefs, unsigned nbWorkers); void FIO_setNbWorkers(FIO_prefs_t* const prefs, int nbWorkers);
void FIO_setOverlapLog(FIO_prefs_t* const prefs, unsigned overlapLog); void FIO_setOverlapLog(FIO_prefs_t* const prefs, int overlapLog);
void FIO_setRemoveSrcFile(FIO_prefs_t* const prefs, unsigned flag); void FIO_setRemoveSrcFile(FIO_prefs_t* const prefs, unsigned flag);
void FIO_setSparseWrite(FIO_prefs_t* const prefs, unsigned sparse); /**< 0: no sparse; 1: disable on stdout; 2: always enabled */ void FIO_setSparseWrite(FIO_prefs_t* const prefs, unsigned sparse); /**< 0: no sparse; 1: disable on stdout; 2: always enabled */
void FIO_setRsyncable(FIO_prefs_t* const prefs, unsigned rsyncable); void FIO_setRsyncable(FIO_prefs_t* const prefs, int rsyncable);
void FIO_setNoProgress(unsigned noProgress); void FIO_setNoProgress(unsigned noProgress);
void FIO_setNotificationLevel(unsigned level); void FIO_setNotificationLevel(int level);
/*-************************************* /*-*************************************
* Single File functions * Single File functions