Increase maximum window size
* Maximum window size in 32-bit mode is 1GB, since allocations for 2GB fail on my Mac. * Maximum window size in 64-bit mode is 2GB, since that is the largest power of 2 that works with the overflow prevention. * Allow `--long=windowLog` to set the window log, along with `--zstd=wlog=#`. These options also set the window size during decompression, but don't override `--memory=#` if it is set. * Present a helpful error message when the window size is too large during decompression. * The long range matcher defaults to a hash log 7 less than the window log, which keeps it at 20 for window log 27. * Keep the default long range matcher window size and the default maximum window size at 27 for the API and CLI. * Add tests that use the maximum window size and hash size for compression and decompression.dev
Nick Terrell
parent
8a4d0abc1c
commit
c233bdbaee
|
|
@ -102,6 +102,7 @@ static const U32 repStartValue[ZSTD_REP_NUM] = { 1, 4, 8 };
|
|||
#define BIT0 1
|
||||
|
||||
#define ZSTD_WINDOWLOG_ABSOLUTEMIN 10
|
||||
#define ZSTD_WINDOWLOG_DEFAULTMAX 27 /* Default maximum allowed window log */
|
||||
static const size_t ZSTD_fcs_fieldSize[4] = { 0, 2, 4, 8 };
|
||||
static const size_t ZSTD_did_fieldSize[4] = { 0, 1, 2, 4 };
|
||||
|
||||
|
|
|
|||
|
|
@ -429,7 +429,7 @@ size_t ZSTD_CCtxParam_setParameter(
|
|||
case ZSTD_p_enableLongDistanceMatching :
|
||||
if (value != 0) {
|
||||
ZSTD_cLevelToCCtxParams(params);
|
||||
params->cParams.windowLog = ZSTD_LDM_WINDOW_LOG;
|
||||
params->cParams.windowLog = ZSTD_LDM_DEFAULT_WINDOW_LOG;
|
||||
}
|
||||
return ZSTD_ldm_initializeParameters(¶ms->ldmParams, value);
|
||||
|
||||
|
|
@ -1599,7 +1599,7 @@ static size_t ZSTD_compress_frameChunk (ZSTD_CCtx* cctx,
|
|||
const BYTE* ip = (const BYTE*)src;
|
||||
BYTE* const ostart = (BYTE*)dst;
|
||||
BYTE* op = ostart;
|
||||
U32 const maxDist = 1 << cctx->appliedParams.cParams.windowLog;
|
||||
U32 const maxDist = (U32)1 << cctx->appliedParams.cParams.windowLog;
|
||||
|
||||
if (cctx->appliedParams.fParams.checksumFlag && srcSize)
|
||||
XXH64_update(&cctx->xxhState, src, srcSize);
|
||||
|
|
@ -1630,9 +1630,9 @@ static size_t ZSTD_compress_frameChunk (ZSTD_CCtx* cctx,
|
|||
* windowLog <= 31 ==> 3<<29 + 1<<windowLog < 7<<29 < 1<<32.
|
||||
*/
|
||||
if (cctx->lowLimit > (3U<<29)) {
|
||||
U32 const cycleMask = (1 << ZSTD_cycleLog(cctx->appliedParams.cParams.chainLog, cctx->appliedParams.cParams.strategy)) - 1;
|
||||
U32 const cycleMask = ((U32)1 << ZSTD_cycleLog(cctx->appliedParams.cParams.chainLog, cctx->appliedParams.cParams.strategy)) - 1;
|
||||
U32 const current = (U32)(ip - cctx->base);
|
||||
U32 const newCurrent = (current & cycleMask) + (1 << cctx->appliedParams.cParams.windowLog);
|
||||
U32 const newCurrent = (current & cycleMask) + ((U32)1 << cctx->appliedParams.cParams.windowLog);
|
||||
U32 const correction = current - newCurrent;
|
||||
ZSTD_STATIC_ASSERT(ZSTD_CHAINLOG_MAX <= 30);
|
||||
ZSTD_STATIC_ASSERT(ZSTD_WINDOWLOG_MAX_32 <= 30);
|
||||
|
|
@ -1688,7 +1688,7 @@ static size_t ZSTD_writeFrameHeader(void* dst, size_t dstCapacity,
|
|||
U32 const dictIDSizeCodeLength = (dictID>0) + (dictID>=256) + (dictID>=65536); /* 0-3 */
|
||||
U32 const dictIDSizeCode = params.fParams.noDictIDFlag ? 0 : dictIDSizeCodeLength; /* 0-3 */
|
||||
U32 const checksumFlag = params.fParams.checksumFlag>0;
|
||||
U32 const windowSize = 1U << params.cParams.windowLog;
|
||||
U32 const windowSize = (U32)1 << params.cParams.windowLog;
|
||||
U32 const singleSegment = params.fParams.contentSizeFlag && (windowSize >= pledgedSrcSize);
|
||||
BYTE const windowLogByte = (BYTE)((params.cParams.windowLog - ZSTD_WINDOWLOG_ABSOLUTEMIN) << 3);
|
||||
U32 const fcsCode = params.fParams.contentSizeFlag ?
|
||||
|
|
@ -1788,7 +1788,7 @@ size_t ZSTD_getBlockSize(const ZSTD_CCtx* cctx)
|
|||
{
|
||||
ZSTD_compressionParameters const cParams =
|
||||
ZSTD_getCParamsFromCCtxParams(cctx->appliedParams, 0, 0);
|
||||
return MIN (ZSTD_BLOCKSIZE_MAX, 1 << cParams.windowLog);
|
||||
return MIN (ZSTD_BLOCKSIZE_MAX, (U32)1 << cParams.windowLog);
|
||||
}
|
||||
|
||||
size_t ZSTD_compressBlock(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize)
|
||||
|
|
@ -1837,7 +1837,7 @@ static size_t ZSTD_loadDictionaryContent(ZSTD_CCtx* zc, const void* src, size_t
|
|||
case ZSTD_btopt:
|
||||
case ZSTD_btultra:
|
||||
if (srcSize >= HASH_READ_SIZE)
|
||||
ZSTD_updateTree(zc, iend-HASH_READ_SIZE, iend, 1 << zc->appliedParams.cParams.searchLog, zc->appliedParams.cParams.searchLength);
|
||||
ZSTD_updateTree(zc, iend-HASH_READ_SIZE, iend, (U32)1 << zc->appliedParams.cParams.searchLog, zc->appliedParams.cParams.searchLength);
|
||||
break;
|
||||
|
||||
default:
|
||||
|
|
|
|||
|
|
@ -14,14 +14,14 @@
|
|||
|
||||
#define LDM_BUCKET_SIZE_LOG 3
|
||||
#define LDM_MIN_MATCH_LENGTH 64
|
||||
#define LDM_HASH_LOG 20
|
||||
#define LDM_HASH_RLOG 7
|
||||
#define LDM_HASH_CHAR_OFFSET 10
|
||||
|
||||
size_t ZSTD_ldm_initializeParameters(ldmParams_t* params, U32 enableLdm)
|
||||
{
|
||||
ZSTD_STATIC_ASSERT(LDM_BUCKET_SIZE_LOG <= ZSTD_LDM_BUCKETSIZELOG_MAX);
|
||||
params->enableLdm = enableLdm>0;
|
||||
params->hashLog = LDM_HASH_LOG;
|
||||
params->hashLog = 0;
|
||||
params->bucketSizeLog = LDM_BUCKET_SIZE_LOG;
|
||||
params->minMatchLength = LDM_MIN_MATCH_LENGTH;
|
||||
params->hashEveryLog = ZSTD_LDM_HASHEVERYLOG_NOTSET;
|
||||
|
|
@ -30,6 +30,10 @@ size_t ZSTD_ldm_initializeParameters(ldmParams_t* params, U32 enableLdm)
|
|||
|
||||
void ZSTD_ldm_adjustParameters(ldmParams_t* params, U32 windowLog)
|
||||
{
|
||||
if (params->hashLog == 0) {
|
||||
params->hashLog = MAX(ZSTD_HASHLOG_MIN, windowLog - LDM_HASH_RLOG);
|
||||
assert(params->hashLog <= ZSTD_HASHLOG_MAX);
|
||||
}
|
||||
if (params->hashEveryLog == ZSTD_LDM_HASHEVERYLOG_NOTSET) {
|
||||
params->hashEveryLog =
|
||||
windowLog < params->hashLog ? 0 : windowLog - params->hashLog;
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@ extern "C" {
|
|||
* Long distance matching
|
||||
***************************************/
|
||||
|
||||
#define ZSTD_LDM_WINDOW_LOG 27
|
||||
#define ZSTD_LDM_DEFAULT_WINDOW_LOG ZSTD_WINDOWLOG_DEFAULTMAX
|
||||
#define ZSTD_LDM_HASHEVERYLOG_NOTSET 9999
|
||||
|
||||
/** ZSTD_compressBlock_ldm_generic() :
|
||||
|
|
|
|||
|
|
@ -35,7 +35,7 @@
|
|||
* Frames requiring more memory will be rejected.
|
||||
*/
|
||||
#ifndef ZSTD_MAXWINDOWSIZE_DEFAULT
|
||||
# define ZSTD_MAXWINDOWSIZE_DEFAULT (((U64)1 << ZSTD_WINDOWLOG_MAX) + 1) /* defined within zstd.h */
|
||||
# define ZSTD_MAXWINDOWSIZE_DEFAULT (((U32)1 << ZSTD_WINDOWLOG_DEFAULTMAX) + 1)
|
||||
#endif
|
||||
|
||||
|
||||
|
|
@ -913,7 +913,7 @@ static seq_t ZSTD_decodeSequence(seqState_t* seqState, const ZSTD_longOffset_e l
|
|||
offset = 0;
|
||||
else {
|
||||
ZSTD_STATIC_ASSERT(ZSTD_lo_isLongOffset == 1);
|
||||
ZSTD_STATIC_ASSERT(LONG_OFFSETS_MAX_EXTRA_BITS_32 == 2);
|
||||
ZSTD_STATIC_ASSERT(LONG_OFFSETS_MAX_EXTRA_BITS_32 == 5);
|
||||
assert(ofBits <= MaxOff);
|
||||
if (MEM_32bits() && longOffsets) {
|
||||
U32 const extraBits = ofBits - MIN(ofBits, STREAM_ACCUMULATOR_MIN_32-1);
|
||||
|
|
@ -1159,7 +1159,7 @@ seq_t ZSTD_decodeSequenceLong(seqState_t* seqState, ZSTD_longOffset_e const long
|
|||
offset = 0;
|
||||
else {
|
||||
ZSTD_STATIC_ASSERT(ZSTD_lo_isLongOffset == 1);
|
||||
ZSTD_STATIC_ASSERT(LONG_OFFSETS_MAX_EXTRA_BITS_32 == 2);
|
||||
ZSTD_STATIC_ASSERT(LONG_OFFSETS_MAX_EXTRA_BITS_32 == 5);
|
||||
assert(ofBits <= MaxOff);
|
||||
if (MEM_32bits() && longOffsets) {
|
||||
U32 const extraBits = ofBits - MIN(ofBits, STREAM_ACCUMULATOR_MIN_32-1);
|
||||
|
|
|
|||
10
lib/zstd.h
10
lib/zstd.h
|
|
@ -376,8 +376,8 @@ ZSTDLIB_API size_t ZSTD_DStreamOutSize(void); /*!< recommended size for output
|
|||
#define ZSTD_MAGIC_SKIPPABLE_START 0x184D2A50U
|
||||
#define ZSTD_MAGIC_DICTIONARY 0xEC30A437 /* v0.7+ */
|
||||
|
||||
#define ZSTD_WINDOWLOG_MAX_32 27
|
||||
#define ZSTD_WINDOWLOG_MAX_64 27
|
||||
#define ZSTD_WINDOWLOG_MAX_32 30
|
||||
#define ZSTD_WINDOWLOG_MAX_64 31
|
||||
#define ZSTD_WINDOWLOG_MAX ((unsigned)(sizeof(size_t) == 4 ? ZSTD_WINDOWLOG_MAX_32 : ZSTD_WINDOWLOG_MAX_64))
|
||||
#define ZSTD_WINDOWLOG_MIN 10
|
||||
#define ZSTD_HASHLOG_MAX MIN(ZSTD_WINDOWLOG_MAX, 30)
|
||||
|
|
@ -941,7 +941,9 @@ typedef enum {
|
|||
* Special: value 0 means "do not change cLevel". */
|
||||
ZSTD_p_windowLog, /* Maximum allowed back-reference distance, expressed as power of 2.
|
||||
* Must be clamped between ZSTD_WINDOWLOG_MIN and ZSTD_WINDOWLOG_MAX.
|
||||
* Special: value 0 means "do not change windowLog". */
|
||||
* Special: value 0 means "do not change windowLog".
|
||||
* Note: Using a window size greater than ZSTD_MAXWINDOWSIZE_DEFAULT (default: 2^27)
|
||||
* requires setting the maximum window size at least as large during decompression. */
|
||||
ZSTD_p_hashLog, /* Size of the probe table, as a power of 2.
|
||||
* Resulting table size is (1 << (hashLog+2)).
|
||||
* Must be clamped between ZSTD_HASHLOG_MIN and ZSTD_HASHLOG_MAX.
|
||||
|
|
@ -1009,7 +1011,7 @@ typedef enum {
|
|||
* Larger values increase memory usage and compression ratio, but decrease
|
||||
* compression speed.
|
||||
* Must be clamped between ZSTD_HASHLOG_MIN and ZSTD_HASHLOG_MAX
|
||||
* (default: 20). */
|
||||
* (default: windowlog - 7). */
|
||||
ZSTD_p_ldmMinMatch, /* Minimum size of searched matches for long distance matcher.
|
||||
* Larger/too small values usually decrease compression ratio.
|
||||
* Must be clamped between ZSTD_LDM_MINMATCH_MIN
|
||||
|
|
|
|||
|
|
@ -37,6 +37,7 @@
|
|||
# include <io.h>
|
||||
#endif
|
||||
|
||||
#include "bitstream.h"
|
||||
#include "mem.h"
|
||||
#include "fileio.h"
|
||||
#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_magicNumber, ZSTD_frameHeaderSize_max */
|
||||
|
|
@ -1167,6 +1168,35 @@ static unsigned FIO_passThrough(FILE* foutput, FILE* finput, void* buffer, size_
|
|||
return 0;
|
||||
}
|
||||
|
||||
static void FIO_zstdErrorHelp(dRess_t* ress, size_t ret, char const* srcFileName)
|
||||
{
|
||||
ZSTD_frameHeader header;
|
||||
/* No special help for these errors */
|
||||
if (ZSTD_getErrorCode(ret) != ZSTD_error_frameParameter_windowTooLarge)
|
||||
return;
|
||||
/* Try to decode the frame header */
|
||||
ret = ZSTD_getFrameHeader(&header, ress->srcBuffer, ress->srcBufferLoaded);
|
||||
if (ret == 0) {
|
||||
U32 const windowSize = (U32)header.windowSize;
|
||||
U32 const windowLog = BIT_highbit32(windowSize) + ((windowSize & (windowSize - 1)) != 0);
|
||||
U32 const windowMB = (windowSize >> 20) + (windowSize & ((1 MB) - 1));
|
||||
assert(header.windowSize <= (U64)((U32)-1));
|
||||
assert(g_memLimit > 0);
|
||||
DISPLAYLEVEL(1, "%s : Window size larger than maximum : %llu > %u\n",
|
||||
srcFileName, header.windowSize, g_memLimit);
|
||||
if (windowLog <= ZSTD_WINDOWLOG_MAX) {
|
||||
DISPLAYLEVEL(1, "%s : Use --long=%u or --memory=%uMB\n",
|
||||
srcFileName, windowLog, windowMB);
|
||||
return;
|
||||
}
|
||||
} else if (ZSTD_getErrorCode(ret) != ZSTD_error_frameParameter_windowTooLarge) {
|
||||
DISPLAYLEVEL(1, "%s : Error decoding frame header to read window size : %s\n",
|
||||
srcFileName, ZSTD_getErrorName(ret));
|
||||
return;
|
||||
}
|
||||
DISPLAYLEVEL(1, "%s : Window log larger than ZSTD_WINDOWLOG_MAX=%u not supported\n",
|
||||
srcFileName, ZSTD_WINDOWLOG_MAX);
|
||||
}
|
||||
|
||||
/** FIO_decompressFrame() :
|
||||
* @return : size of decoded zstd frame, or an error code
|
||||
|
|
@ -1183,8 +1213,8 @@ unsigned long long FIO_decompressZstdFrame(dRess_t* ress,
|
|||
ZSTD_resetDStream(ress->dctx);
|
||||
if (strlen(srcFileName)>20) srcFileName += strlen(srcFileName)-20; /* display last 20 characters */
|
||||
|
||||
/* Header loading (optional, saves one loop) */
|
||||
{ size_t const toRead = 9;
|
||||
/* Header loading : ensures ZSTD_getFrameHeader() will succeed */
|
||||
{ size_t const toRead = ZSTD_FRAMEHEADERSIZE_MAX;
|
||||
if (ress->srcBufferLoaded < toRead)
|
||||
ress->srcBufferLoaded += fread(((char*)ress->srcBuffer) + ress->srcBufferLoaded, 1, toRead - ress->srcBufferLoaded, finput);
|
||||
}
|
||||
|
|
@ -1197,6 +1227,7 @@ unsigned long long FIO_decompressZstdFrame(dRess_t* ress,
|
|||
if (ZSTD_isError(readSizeHint)) {
|
||||
DISPLAYLEVEL(1, "%s : Decoding error (36) : %s \n",
|
||||
srcFileName, ZSTD_getErrorName(readSizeHint));
|
||||
FIO_zstdErrorHelp(ress, readSizeHint, srcFileName);
|
||||
return FIO_ERROR_FRAME_DECODING;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -104,8 +104,11 @@ Display information related to a zstd compressed file, such as size, ratio, and
|
|||
unlocks high compression levels 20+ (maximum 22), using a lot more memory\. Note that decompression will also require more memory when using these levels\.
|
||||
.
|
||||
.TP
|
||||
\fB\-\-long\fR
|
||||
enables long distance matching\. This increases the window size (\fBwindowLog\fR) and memory usage for both the compressor and decompressor\. This setting is designed to improve the compression ratio for files with long matches at a large distance (up to the maximum window size, 128 MiB)\.
|
||||
\fB\-\-long[=#]\fR
|
||||
enables long distance matching with \fB#\fR \fBwindowLog\fR, if not \fB#\fR is not present it defaults to \fB27\fR\. This increases the window size (\fBwindowLog\fR) and memory usage for both the compressor and decompressor\. This setting is designed to improve the compression ratio for files with long matches at a large distance\.
|
||||
.
|
||||
.IP
|
||||
Note: If \fBwindowLog\fR is set to larger than 27, \fB\-\-long=windowLog\fR or \fB\-\-memory=windowSize\fR needs to be passed to the decompressor\.
|
||||
.
|
||||
.TP
|
||||
\fB\-T#\fR, \fB\-\-threads=#\fR
|
||||
|
|
@ -190,6 +193,10 @@ Dictionary saved into \fBfile\fR (default name: dictionary)\.
|
|||
Limit dictionary to specified size (default: 112640)\.
|
||||
.
|
||||
.TP
|
||||
\fB\-B#\fR
|
||||
Split input files in blocks of size # (default: no split)
|
||||
.
|
||||
.TP
|
||||
\fB\-\-dictID=#\fR
|
||||
A dictionary ID is a locally unique ID that a decoder can use to verify it is using the right dictionary\. By default, zstd will create a 4\-bytes random number ID\. It\'s possible to give a precise number instead\. Short numbers have an advantage : an ID < 256 will only need 1 byte in the compressed frame header, and an ID < 65536 will only need 2 bytes\. This compares favorably to 4 bytes default\. However, it\'s up to the dictionary manager to not assign twice the same ID to 2 different dictionaries\.
|
||||
.
|
||||
|
|
@ -267,7 +274,10 @@ There are 8 strategies numbered from 1 to 8, from faster to stronger: 1=ZSTD_fas
|
|||
Specify the maximum number of bits for a match distance\.
|
||||
.
|
||||
.IP
|
||||
The higher number of increases the chance to find a match which usually improves compression ratio\. It also increases memory requirements for the compressor and decompressor\. The minimum \fIwlog\fR is 10 (1 KiB) and the maximum is 27 (128 MiB)\.
|
||||
The higher number of increases the chance to find a match which usually improves compression ratio\. It also increases memory requirements for the compressor and decompressor\. The minimum \fIwlog\fR is 10 (1 KiB) and the maximum is 30 (1 GiB) on 32\-bit platforms and 31 (2 GiB) on 64\-bit platforms\.
|
||||
.
|
||||
.IP
|
||||
Note: If \fBwindowLog\fR is set to larger than 27, \fB\-\-long=windowLog\fR or \fB\-\-memory=windowSize\fR needs to be passed to the decompressor\.
|
||||
.
|
||||
.TP
|
||||
\fBhashLog\fR=\fIhlog\fR, \fBhlog\fR=\fIhlog\fR
|
||||
|
|
@ -340,7 +350,7 @@ Bigger hash tables usually improve compression ratio at the expense of more memo
|
|||
The minimum \fIldmhlog\fR is 6 and the maximum is 26 (default: 20)\.
|
||||
.
|
||||
.TP
|
||||
\fBldmSearchLength\fR=\fIldmslen\fR, \fBldmSlen\fR=\fIldmslen\fR
|
||||
\fBldmSearchLength\fR=\fIldmslen\fR, \fBldmslen\fR=\fIldmslen\fR
|
||||
Specify the minimum searched length of a match for long distance matching\.
|
||||
.
|
||||
.IP
|
||||
|
|
|
|||
|
|
@ -105,12 +105,16 @@ the last one takes effect.
|
|||
* `--ultra`:
|
||||
unlocks high compression levels 20+ (maximum 22), using a lot more memory.
|
||||
Note that decompression will also require more memory when using these levels.
|
||||
* `--long`:
|
||||
enables long distance matching.
|
||||
* `--long[=#]`:
|
||||
enables long distance matching with `#` `windowLog`, if not `#` is not
|
||||
present it defaults to `27`.
|
||||
This increases the window size (`windowLog`) and memory usage for both the
|
||||
compressor and decompressor. This setting is designed to improve the
|
||||
compression ratio for files with long matches at a large distance
|
||||
(up to the maximum window size, 128 MiB).
|
||||
compressor and decompressor.
|
||||
This setting is designed to improve the compression ratio for files with
|
||||
long matches at a large distance.
|
||||
|
||||
Note: If `windowLog` is set to larger than 27, `--long=windowLog` or
|
||||
`--memory=windowSize` needs to be passed to the decompressor.
|
||||
* `-T#`, `--threads=#`:
|
||||
Compress using `#` threads (default: 1).
|
||||
If `#` is 0, attempt to detect and use the number of physical CPU cores.
|
||||
|
|
@ -275,7 +279,11 @@ The list of available _options_:
|
|||
The higher number of increases the chance to find a match which usually
|
||||
improves compression ratio.
|
||||
It also increases memory requirements for the compressor and decompressor.
|
||||
The minimum _wlog_ is 10 (1 KiB) and the maximum is 27 (128 MiB).
|
||||
The minimum _wlog_ is 10 (1 KiB) and the maximum is 30 (1 GiB) on 32-bit
|
||||
platforms and 31 (2 GiB) on 64-bit platforms.
|
||||
|
||||
Note: If `windowLog` is set to larger than 27, `--long=windowLog` or
|
||||
`--memory=windowSize` needs to be passed to the decompressor.
|
||||
|
||||
- `hashLog`=_hlog_, `hlog`=_hlog_:
|
||||
Specify the maximum number of bits for a hash table.
|
||||
|
|
|
|||
|
|
@ -72,6 +72,7 @@ static const char* g_defaultDictName = "dictionary";
|
|||
static const unsigned g_defaultMaxDictSize = 110 KB;
|
||||
static const int g_defaultDictCLevel = 3;
|
||||
static const unsigned g_defaultSelectivityLevel = 9;
|
||||
static const unsigned g_defaultMaxWindowLog = 27;
|
||||
#define OVERLAP_LOG_DEFAULT 9999
|
||||
#define LDM_PARAM_DEFAULT 9999 /* Default for parameters where 0 is valid */
|
||||
static U32 g_overlapLog = OVERLAP_LOG_DEFAULT;
|
||||
|
|
@ -129,7 +130,7 @@ static int usage_advanced(const char* programName)
|
|||
DISPLAY( " -l : print information about zstd compressed files \n");
|
||||
#ifndef ZSTD_NOCOMPRESS
|
||||
DISPLAY( "--ultra : enable levels beyond %i, up to %i (requires more memory)\n", ZSTDCLI_CLEVEL_MAX, ZSTD_maxCLevel());
|
||||
DISPLAY( "--long : enable long distance matching (requires more memory)\n");
|
||||
DISPLAY( "--long[=#] : enable long distance matching with given window log (default : %u)\n", g_defaultMaxWindowLog);
|
||||
#ifdef ZSTD_MULTITHREAD
|
||||
DISPLAY( " -T# : use # threads for compression (default:1) \n");
|
||||
DISPLAY( " -B# : select size of each job (default:0==automatic) \n");
|
||||
|
|
@ -459,7 +460,6 @@ int main(int argCount, const char* argv[])
|
|||
if (!strcmp(argument, "--quiet")) { g_displayLevel--; continue; }
|
||||
if (!strcmp(argument, "--stdout")) { forceStdout=1; outFileName=stdoutmark; g_displayLevel-=(g_displayLevel==2); continue; }
|
||||
if (!strcmp(argument, "--ultra")) { ultra=1; continue; }
|
||||
if (!strcmp(argument, "--long")) { ldmFlag = 1; continue; }
|
||||
if (!strcmp(argument, "--check")) { FIO_setChecksumFlag(2); continue; }
|
||||
if (!strcmp(argument, "--no-check")) { FIO_setChecksumFlag(0); continue; }
|
||||
if (!strcmp(argument, "--sparse")) { FIO_setSparseWrite(2); continue; }
|
||||
|
|
@ -514,6 +514,22 @@ int main(int argCount, const char* argv[])
|
|||
if (longCommandWArg(&argument, "--maxdict=")) { maxDictSize = readU32FromChar(&argument); continue; }
|
||||
if (longCommandWArg(&argument, "--dictID=")) { dictID = readU32FromChar(&argument); continue; }
|
||||
if (longCommandWArg(&argument, "--zstd=")) { if (!parseCompressionParameters(argument, &compressionParams)) CLEAN_RETURN(badusage(programName)); continue; }
|
||||
if (longCommandWArg(&argument, "--long")) {
|
||||
unsigned ldmWindowLog = 0;
|
||||
ldmFlag = 1;
|
||||
/* Parse optional window log */
|
||||
if (*argument == '=') {
|
||||
++argument;
|
||||
ldmWindowLog = readU32FromChar(&argument);
|
||||
} else if (*argument != 0) {
|
||||
/* Invalid character following --long */
|
||||
CLEAN_RETURN(badusage(programName));
|
||||
}
|
||||
/* Only set windowLog if not already set by --zstd */
|
||||
if (compressionParams.windowLog == 0)
|
||||
compressionParams.windowLog = ldmWindowLog;
|
||||
continue;
|
||||
}
|
||||
/* fall-through, will trigger bad_usage() later on */
|
||||
}
|
||||
|
||||
|
|
@ -830,6 +846,13 @@ int main(int argCount, const char* argv[])
|
|||
#endif
|
||||
} else { /* decompression or test */
|
||||
#ifndef ZSTD_NODECOMPRESS
|
||||
if (memLimit == 0) {
|
||||
if (compressionParams.windowLog == 0)
|
||||
memLimit = (U32)1 << g_defaultMaxWindowLog;
|
||||
else {
|
||||
memLimit = (U32)1 << (compressionParams.windowLog & 31);
|
||||
}
|
||||
}
|
||||
FIO_setMemLimit(memLimit);
|
||||
if (filenameIdx==1 && outFileName)
|
||||
operationResult = FIO_decompressFilename(outFileName, filenameTable[0], dictFileName);
|
||||
|
|
|
|||
|
|
@ -13,11 +13,16 @@ roundTripTest() {
|
|||
cLevel="$2"
|
||||
proba=""
|
||||
fi
|
||||
if [ -n "$4" ]; then
|
||||
dLevel="$4"
|
||||
else
|
||||
dLevel="$cLevel"
|
||||
fi
|
||||
|
||||
rm -f tmp1 tmp2
|
||||
$ECHO "roundTripTest: ./datagen $1 $proba | $ZSTD -v$cLevel | $ZSTD -d"
|
||||
$ECHO "roundTripTest: ./datagen $1 $proba | $ZSTD -v$cLevel | $ZSTD -d$dLevel"
|
||||
./datagen $1 $proba | $MD5SUM > tmp1
|
||||
./datagen $1 $proba | $ZSTD --ultra -v$cLevel | $ZSTD -d | $MD5SUM > tmp2
|
||||
./datagen $1 $proba | $ZSTD --ultra -v$cLevel | $ZSTD -d$dLevel | $MD5SUM > tmp2
|
||||
$DIFF -q tmp1 tmp2
|
||||
}
|
||||
|
||||
|
|
@ -29,12 +34,17 @@ fileRoundTripTest() {
|
|||
local_c="$2"
|
||||
local_p=""
|
||||
fi
|
||||
if [ -n "$4" ]; then
|
||||
local_d="$4"
|
||||
else
|
||||
local_d="$local_c"
|
||||
fi
|
||||
|
||||
rm -f tmp.zstd tmp.md5.1 tmp.md5.2
|
||||
$ECHO "fileRoundTripTest: ./datagen $1 $local_p > tmp && $ZSTD -v$local_c -c tmp | $ZSTD -d"
|
||||
$ECHO "fileRoundTripTest: ./datagen $1 $local_p > tmp && $ZSTD -v$local_c -c tmp | $ZSTD -d$local_d"
|
||||
./datagen $1 $local_p > tmp
|
||||
cat tmp | $MD5SUM > tmp.md5.1
|
||||
$ZSTD --ultra -v$local_c -c tmp | $ZSTD -d | $MD5SUM > tmp.md5.2
|
||||
$ZSTD --ultra -v$local_c -c tmp | $ZSTD -d$local_d | $MD5SUM > tmp.md5.2
|
||||
$DIFF -q tmp.md5.1 tmp.md5.2
|
||||
}
|
||||
|
||||
|
|
@ -669,16 +679,24 @@ roundTripTest -g140000000 -P60 "5 --long"
|
|||
roundTripTest -g70000000 -P70 "8 --long"
|
||||
roundTripTest -g18000001 -P80 "18 --long"
|
||||
fileRoundTripTest -g4100M -P99 "1 --long"
|
||||
# Test large window logs
|
||||
roundTripTest -g4100M -P50 "1 --long=30"
|
||||
roundTripTest -g4100M -P50 "1 --long --zstd=wlog=30,clog=30"
|
||||
# Test parameter parsing
|
||||
roundTripTest -g1M -P50 "1 --long=30" " --memory=1024MB"
|
||||
roundTripTest -g1M -P50 "1 --long=30 --zstd=wlog=29" " --memory=512MB"
|
||||
roundTripTest -g1M -P50 "1 --long=30" " --long=29 --memory=1024MB"
|
||||
roundTripTest -g1M -P50 "1 --long=30" " --zstd=wlog=29 --memory=1024MB"
|
||||
|
||||
|
||||
if [ -n "$hasMT" ]
|
||||
then
|
||||
$ECHO "\n**** zstdmt long round-trip tests **** "
|
||||
roundTripTest -g99000000 -P99 "20 -T2"
|
||||
roundTripTest -g6000000000 -P99 "1 -T2"
|
||||
roundTripTest -g1500000000 -P97 "1 -T999"
|
||||
fileRoundTripTest -g4195M -P98 " -T0"
|
||||
roundTripTest -g1500000000 -P97 "1 --long -T999"
|
||||
roundTripTest -g99000000 -P99 "20 -T2" " "
|
||||
roundTripTest -g6000000000 -P99 "1 -T2" " "
|
||||
roundTripTest -g1500000000 -P97 "1 -T999" " "
|
||||
fileRoundTripTest -g4195M -P98 " -T0" " "
|
||||
roundTripTest -g1500000000 -P97 "1 --long=23 -T2" " "
|
||||
else
|
||||
$ECHO "\n**** no multithreading, skipping zstdmt tests **** "
|
||||
fi
|
||||
|
|
|
|||
|
|
@ -50,6 +50,7 @@ static const U32 g_cLevelMax_smallTests = 10;
|
|||
#define COMPRESSIBLE_NOISE_LENGTH (10 MB)
|
||||
#define FUZ_COMPRESSIBILITY_DEFAULT 50
|
||||
static const U32 prime32 = 2654435761U;
|
||||
static const U32 windowLogMax = 27;
|
||||
|
||||
|
||||
/*-************************************
|
||||
|
|
@ -1380,6 +1381,7 @@ static int fuzzerTests_newAPI(U32 seed, U32 nbTests, unsigned startTest, double
|
|||
|
||||
/* mess with compression parameters */
|
||||
cParams.windowLog += (FUZ_rand(&lseed) & 3) - 1;
|
||||
cParams.windowLog = MIN(windowLogMax, cParams.windowLog);
|
||||
cParams.hashLog += (FUZ_rand(&lseed) & 3) - 1;
|
||||
cParams.chainLog += (FUZ_rand(&lseed) & 3) - 1;
|
||||
cParams.searchLog += (FUZ_rand(&lseed) & 3) - 1;
|
||||
|
|
|
|||
Loading…
Reference in New Issue