Merge branch 'travisTest' of github.com:facebook/zstd into travisTest
Yann Collet
commit
064b8e6785
|
|
@ -10,8 +10,13 @@ addons:
|
|||
matrix:
|
||||
include:
|
||||
# Ubuntu 14.04
|
||||
<<<<<<< HEAD
|
||||
- env: Cmd='make gcc6install && CC=gcc-6 make -j all
|
||||
&& make clean && CC=gcc-6 make clean uasan-test-zstd'
|
||||
=======
|
||||
- env: Cmd='make test'
|
||||
- env: Cmd='make gcc6install && CC=gcc-6 make -j all && make clean && CC=gcc-6 make clean uasan-test-zstd </dev/null' # also test when stdin is not a tty
|
||||
>>>>>>> 0840d02ecf74eae656e0df0d900000d9b0154cde
|
||||
- env: Cmd='make gcc6install libc6install && CC=gcc-6 make clean uasan-test-zstd32'
|
||||
- env: Cmd='make gcc7install && CC=gcc-7 make clean uasan-test-zstd'
|
||||
- env: Cmd='make clang38install && CC=clang-3.8 make clean msan-test-zstd'
|
||||
|
|
|
|||
3
Makefile
3
Makefile
|
|
@ -63,8 +63,9 @@ zlibwrapper:
|
|||
$(MAKE) -C $(ZWRAPDIR) test
|
||||
|
||||
.PHONY: test
|
||||
test: MOREFLAGS += -g -DDEBUGLEVEL=1 -Werror
|
||||
test:
|
||||
$(MAKE) -C $(PRGDIR) allVariants MOREFLAGS+="-g -DDEBUGLEVEL=1"
|
||||
MOREFLAGS="$(MOREFLAGS)" $(MAKE) -C $(PRGDIR) allVariants
|
||||
$(MAKE) -C $(TESTDIR) $@
|
||||
|
||||
.PHONY: shortest
|
||||
|
|
|
|||
|
|
@ -167,11 +167,13 @@
|
|||
<ItemGroup>
|
||||
<ClCompile Include="..\..\..\lib\common\xxhash.c" />
|
||||
<ClCompile Include="..\..\..\programs\datagen.c" />
|
||||
<ClCompile Include="..\..\..\programs\bench.c" />
|
||||
<ClCompile Include="..\..\..\tests\fullbench.c" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<ClInclude Include="..\..\..\lib\zstd.h" />
|
||||
<ClInclude Include="..\..\..\programs\datagen.h" />
|
||||
<ClInclude Include="..\..\..\programs\bench.h" />
|
||||
<ClInclude Include="..\..\..\programs\util.h" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
|
|
|
|||
|
|
@ -176,6 +176,7 @@
|
|||
<ClCompile Include="..\..\..\lib\decompress\huf_decompress.c" />
|
||||
<ClCompile Include="..\..\..\lib\decompress\zstd_decompress.c" />
|
||||
<ClCompile Include="..\..\..\programs\datagen.c" />
|
||||
<ClCompile Include="..\..\..\programs\bench.c" />
|
||||
<ClCompile Include="..\..\..\tests\fullbench.c" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
|
|
@ -197,6 +198,7 @@
|
|||
<ClInclude Include="..\..\..\lib\legacy\zstd_legacy.h" />
|
||||
<ClInclude Include="..\..\..\programs\datagen.h" />
|
||||
<ClInclude Include="..\..\..\programs\util.h" />
|
||||
<ClInclude Include="..\..\..\programs\bench.h" />
|
||||
</ItemGroup>
|
||||
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
|
||||
<ImportGroup Label="ExtensionTargets">
|
||||
|
|
|
|||
|
|
@ -18,6 +18,7 @@ libzstd_srcs = [
|
|||
join_paths(common_dir, 'error_private.c'),
|
||||
join_paths(common_dir, 'xxhash.c'),
|
||||
join_paths(compress_dir, 'fse_compress.c'),
|
||||
join_paths(compress_dir, 'hist.c'),
|
||||
join_paths(compress_dir, 'huf_compress.c'),
|
||||
join_paths(compress_dir, 'zstd_compress.c'),
|
||||
join_paths(compress_dir, 'zstd_fast.c'),
|
||||
|
|
@ -130,6 +131,7 @@ test('fuzzer', fuzzer)
|
|||
if target_machine.system() != 'windows'
|
||||
paramgrill = executable('paramgrill',
|
||||
datagen_c, join_paths(tests_dir, 'paramgrill.c'),
|
||||
join_paths(programs_dir, 'bench.c'),
|
||||
include_directories: test_includes,
|
||||
link_with: libzstd,
|
||||
dependencies: libm)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,6 @@
|
|||
-- Include zstd.lua in your GENie or premake4 file, which exposes a project_zstd function
|
||||
dofile('zstd.lua')
|
||||
|
||||
solution 'example'
|
||||
configurations { 'Debug', 'Release' }
|
||||
project_zstd('../../lib/')
|
||||
|
|
@ -0,0 +1,80 @@
|
|||
-- This GENie/premake file copies the behavior of the Makefile in the lib folder.
|
||||
-- Basic usage: project_zstd(ZSTD_DIR)
|
||||
|
||||
function project_zstd(dir, compression, decompression, deprecated, dictbuilder, legacy)
|
||||
if compression == nil then compression = true end
|
||||
if decompression == nil then decompression = true end
|
||||
if deprecated == nil then deprecated = false end
|
||||
if dictbuilder == nil then dictbuilder = false end
|
||||
|
||||
if legacy == nil then legacy = 0 end
|
||||
|
||||
if not compression then
|
||||
dictbuilder = false
|
||||
deprecated = false
|
||||
end
|
||||
|
||||
if not decompression then
|
||||
legacy = 0
|
||||
deprecated = false
|
||||
end
|
||||
|
||||
project 'zstd'
|
||||
kind 'StaticLib'
|
||||
language 'C'
|
||||
|
||||
files {
|
||||
dir .. 'zstd.h',
|
||||
dir .. 'common/**.c',
|
||||
dir .. 'common/**.h'
|
||||
}
|
||||
|
||||
if compression then
|
||||
files {
|
||||
dir .. 'compress/**.c',
|
||||
dir .. 'compress/**.h'
|
||||
}
|
||||
end
|
||||
|
||||
if decompression then
|
||||
files {
|
||||
dir .. 'decompress/**.c',
|
||||
dir .. 'decompress/**.h'
|
||||
}
|
||||
end
|
||||
|
||||
if dictbuilder then
|
||||
files {
|
||||
dir .. 'dictBuilder/**.c',
|
||||
dir .. 'dictBuilder/**.h'
|
||||
}
|
||||
end
|
||||
|
||||
if deprecated then
|
||||
files {
|
||||
dir .. 'deprecated/**.c',
|
||||
dir .. 'deprecated/**.h'
|
||||
}
|
||||
end
|
||||
|
||||
if legacy ~= 0 then
|
||||
if legacy >= 8 then
|
||||
files {
|
||||
dir .. 'legacy/zstd_v0' .. (legacy - 7) .. '.*'
|
||||
}
|
||||
end
|
||||
includedirs {
|
||||
dir .. 'legacy'
|
||||
}
|
||||
end
|
||||
|
||||
includedirs {
|
||||
dir,
|
||||
dir .. 'common'
|
||||
}
|
||||
|
||||
defines {
|
||||
'XXH_NAMESPACE=ZSTD_',
|
||||
'ZSTD_LEGACY_SUPPORT=' .. legacy
|
||||
}
|
||||
end
|
||||
|
|
@ -488,20 +488,20 @@ For values spanning several bytes, convention is __little-endian__.
|
|||
__`Size_Format` for `Raw_Literals_Block` and `RLE_Literals_Block`__ :
|
||||
|
||||
`Size_Format` uses 1 _or_ 2 bits.
|
||||
Its value is : `Size_Format = (Header[0]>>2) & 3`
|
||||
Its value is : `Size_Format = (Literals_Section_Header[0]>>2) & 3`
|
||||
|
||||
- `Size_Format` == 00 or 10 : `Size_Format` uses 1 bit.
|
||||
`Regenerated_Size` uses 5 bits (0-31).
|
||||
`Literals_Section_Header` uses 1 byte.
|
||||
`Regenerated_Size = Header[0]>>3`
|
||||
`Regenerated_Size = Literals_Section_Header[0]>>3`
|
||||
- `Size_Format` == 01 : `Size_Format` uses 2 bits.
|
||||
`Regenerated_Size` uses 12 bits (0-4095).
|
||||
`Literals_Section_Header` uses 2 bytes.
|
||||
`Regenerated_Size = (Header[0]>>4) + (Header[1]<<4)`
|
||||
`Regenerated_Size = (Literals_Section_Header[0]>>4) + (Literals_Section_Header[1]<<4)`
|
||||
- `Size_Format` == 11 : `Size_Format` uses 2 bits.
|
||||
`Regenerated_Size` uses 20 bits (0-1048575).
|
||||
`Literals_Section_Header` uses 3 bytes.
|
||||
`Regenerated_Size = (Header[0]>>4) + (Header[1]<<4) + (Header[2]<<12)`
|
||||
`Regenerated_Size = (Literals_Section_Header[0]>>4) + (Literals_Section_Header[1]<<4) + (Literals_Section_Header[2]<<12)`
|
||||
|
||||
Only Stream1 is present for these cases.
|
||||
Note : it's allowed to represent a short value (for example `13`)
|
||||
|
|
|
|||
|
|
@ -88,15 +88,20 @@
|
|||
#endif
|
||||
#endif
|
||||
|
||||
/* prefetch */
|
||||
#if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_I86)) /* _mm_prefetch() is not defined outside of x86/x64 */
|
||||
# include <mmintrin.h> /* https://msdn.microsoft.com/fr-fr/library/84szxsww(v=vs.90).aspx */
|
||||
# define PREFETCH(ptr) _mm_prefetch((const char*)ptr, _MM_HINT_T0)
|
||||
#elif defined(__GNUC__) && ( (__GNUC__ >= 4) || ( (__GNUC__ == 3) && (__GNUC_MINOR__ >= 1) ) )
|
||||
# define PREFETCH(ptr) __builtin_prefetch(ptr, 0, 0)
|
||||
#else
|
||||
/* prefetch
|
||||
* can be disabled, by declaring NO_PREFETCH macro */
|
||||
#if defined(NO_PREFETCH)
|
||||
# define PREFETCH(ptr) /* disabled */
|
||||
#endif
|
||||
#else
|
||||
# if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_I86)) /* _mm_prefetch() is not defined outside of x86/x64 */
|
||||
# include <mmintrin.h> /* https://msdn.microsoft.com/fr-fr/library/84szxsww(v=vs.90).aspx */
|
||||
# define PREFETCH(ptr) _mm_prefetch((const char*)ptr, _MM_HINT_T0)
|
||||
# elif defined(__GNUC__) && ( (__GNUC__ >= 4) || ( (__GNUC__ == 3) && (__GNUC_MINOR__ >= 1) ) )
|
||||
# define PREFETCH(ptr) __builtin_prefetch(ptr, 0, 0)
|
||||
# else
|
||||
# define PREFETCH(ptr) /* disabled */
|
||||
# endif
|
||||
#endif /* NO_PREFETCH */
|
||||
|
||||
/* disable warnings */
|
||||
#ifdef _MSC_VER /* Visual Studio */
|
||||
|
|
|
|||
|
|
@ -39,6 +39,7 @@
|
|||
* Constants
|
||||
***************************************/
|
||||
#define COVER_MAX_SAMPLES_SIZE (sizeof(size_t) == 8 ? ((U32)-1) : ((U32)1 GB))
|
||||
#define DEFAULT_SPLITPOINT 1.0
|
||||
|
||||
/*-*************************************
|
||||
* Console display
|
||||
|
|
@ -203,6 +204,8 @@ typedef struct {
|
|||
size_t *offsets;
|
||||
const size_t *samplesSizes;
|
||||
size_t nbSamples;
|
||||
size_t nbTrainSamples;
|
||||
size_t nbTestSamples;
|
||||
U32 *suffix;
|
||||
size_t suffixSize;
|
||||
U32 *freqs;
|
||||
|
|
@ -222,7 +225,7 @@ static COVER_ctx_t *g_ctx = NULL;
|
|||
*/
|
||||
static size_t COVER_sum(const size_t *samplesSizes, unsigned nbSamples) {
|
||||
size_t sum = 0;
|
||||
size_t i;
|
||||
unsigned i;
|
||||
for (i = 0; i < nbSamples; ++i) {
|
||||
sum += samplesSizes[i];
|
||||
}
|
||||
|
|
@ -494,6 +497,10 @@ static int COVER_checkParameters(ZDICT_cover_params_t parameters,
|
|||
if (parameters.d > parameters.k) {
|
||||
return 0;
|
||||
}
|
||||
/* 0 < splitPoint <= 1 */
|
||||
if (parameters.splitPoint <= 0 || parameters.splitPoint > 1){
|
||||
return 0;
|
||||
}
|
||||
return 1;
|
||||
}
|
||||
|
||||
|
|
@ -531,9 +538,14 @@ static void COVER_ctx_destroy(COVER_ctx_t *ctx) {
|
|||
*/
|
||||
static int COVER_ctx_init(COVER_ctx_t *ctx, const void *samplesBuffer,
|
||||
const size_t *samplesSizes, unsigned nbSamples,
|
||||
unsigned d) {
|
||||
unsigned d, double splitPoint) {
|
||||
const BYTE *const samples = (const BYTE *)samplesBuffer;
|
||||
const size_t totalSamplesSize = COVER_sum(samplesSizes, nbSamples);
|
||||
/* Split samples into testing and training sets */
|
||||
const unsigned nbTrainSamples = splitPoint < 1.0 ? (unsigned)((double)nbSamples * splitPoint) : nbSamples;
|
||||
const unsigned nbTestSamples = splitPoint < 1.0 ? nbSamples - nbTrainSamples : nbSamples;
|
||||
const size_t trainingSamplesSize = splitPoint < 1.0 ? COVER_sum(samplesSizes, nbTrainSamples) : totalSamplesSize;
|
||||
const size_t testSamplesSize = splitPoint < 1.0 ? COVER_sum(samplesSizes + nbTrainSamples, nbTestSamples) : totalSamplesSize;
|
||||
/* Checks */
|
||||
if (totalSamplesSize < MAX(d, sizeof(U64)) ||
|
||||
totalSamplesSize >= (size_t)COVER_MAX_SAMPLES_SIZE) {
|
||||
|
|
@ -541,15 +553,29 @@ static int COVER_ctx_init(COVER_ctx_t *ctx, const void *samplesBuffer,
|
|||
(U32)(totalSamplesSize>>20), (COVER_MAX_SAMPLES_SIZE >> 20));
|
||||
return 0;
|
||||
}
|
||||
/* Check if there are at least 5 training samples */
|
||||
if (nbTrainSamples < 5) {
|
||||
DISPLAYLEVEL(1, "Total number of training samples is %u and is invalid.", nbTrainSamples);
|
||||
return 0;
|
||||
}
|
||||
/* Check if there's testing sample */
|
||||
if (nbTestSamples < 1) {
|
||||
DISPLAYLEVEL(1, "Total number of testing samples is %u and is invalid.", nbTestSamples);
|
||||
return 0;
|
||||
}
|
||||
/* Zero the context */
|
||||
memset(ctx, 0, sizeof(*ctx));
|
||||
DISPLAYLEVEL(2, "Training on %u samples of total size %u\n", nbSamples,
|
||||
(U32)totalSamplesSize);
|
||||
DISPLAYLEVEL(2, "Training on %u samples of total size %u\n", nbTrainSamples,
|
||||
(U32)trainingSamplesSize);
|
||||
DISPLAYLEVEL(2, "Testing on %u samples of total size %u\n", nbTestSamples,
|
||||
(U32)testSamplesSize);
|
||||
ctx->samples = samples;
|
||||
ctx->samplesSizes = samplesSizes;
|
||||
ctx->nbSamples = nbSamples;
|
||||
ctx->nbTrainSamples = nbTrainSamples;
|
||||
ctx->nbTestSamples = nbTestSamples;
|
||||
/* Partial suffix array */
|
||||
ctx->suffixSize = totalSamplesSize - MAX(d, sizeof(U64)) + 1;
|
||||
ctx->suffixSize = trainingSamplesSize - MAX(d, sizeof(U64)) + 1;
|
||||
ctx->suffix = (U32 *)malloc(ctx->suffixSize * sizeof(U32));
|
||||
/* Maps index to the dmerID */
|
||||
ctx->dmerAt = (U32 *)malloc(ctx->suffixSize * sizeof(U32));
|
||||
|
|
@ -563,7 +589,7 @@ static int COVER_ctx_init(COVER_ctx_t *ctx, const void *samplesBuffer,
|
|||
ctx->freqs = NULL;
|
||||
ctx->d = d;
|
||||
|
||||
/* Fill offsets from the samlesSizes */
|
||||
/* Fill offsets from the samplesSizes */
|
||||
{
|
||||
U32 i;
|
||||
ctx->offsets[0] = 0;
|
||||
|
|
@ -665,7 +691,7 @@ ZDICTLIB_API size_t ZDICT_trainFromBuffer_cover(
|
|||
BYTE* const dict = (BYTE*)dictBuffer;
|
||||
COVER_ctx_t ctx;
|
||||
COVER_map_t activeDmers;
|
||||
|
||||
parameters.splitPoint = 1.0;
|
||||
/* Initialize global data */
|
||||
g_displayLevel = parameters.zParams.notificationLevel;
|
||||
/* Checks */
|
||||
|
|
@ -684,7 +710,7 @@ ZDICTLIB_API size_t ZDICT_trainFromBuffer_cover(
|
|||
}
|
||||
/* Initialize context and activeDmers */
|
||||
if (!COVER_ctx_init(&ctx, samplesBuffer, samplesSizes, nbSamples,
|
||||
parameters.d)) {
|
||||
parameters.d, parameters.splitPoint)) {
|
||||
return ERROR(GENERIC);
|
||||
}
|
||||
if (!COVER_map_init(&activeDmers, parameters.k - parameters.d + 1)) {
|
||||
|
|
@ -839,7 +865,7 @@ typedef struct COVER_tryParameters_data_s {
|
|||
} COVER_tryParameters_data_t;
|
||||
|
||||
/**
|
||||
* Tries a set of parameters and upates the COVER_best_t with the results.
|
||||
* Tries a set of parameters and updates the COVER_best_t with the results.
|
||||
* This function is thread safe if zstd is compiled with multithreaded support.
|
||||
* It takes its parameters as an *OWNING* opaque pointer to support threading.
|
||||
*/
|
||||
|
|
@ -870,7 +896,7 @@ static void COVER_tryParameters(void *opaque) {
|
|||
dictBufferCapacity, parameters);
|
||||
dictBufferCapacity = ZDICT_finalizeDictionary(
|
||||
dict, dictBufferCapacity, dict + tail, dictBufferCapacity - tail,
|
||||
ctx->samples, ctx->samplesSizes, (unsigned)ctx->nbSamples,
|
||||
ctx->samples, ctx->samplesSizes, (unsigned)ctx->nbTrainSamples,
|
||||
parameters.zParams);
|
||||
if (ZDICT_isError(dictBufferCapacity)) {
|
||||
DISPLAYLEVEL(1, "Failed to finalize dictionary\n");
|
||||
|
|
@ -889,7 +915,8 @@ static void COVER_tryParameters(void *opaque) {
|
|||
/* Allocate dst with enough space to compress the maximum sized sample */
|
||||
{
|
||||
size_t maxSampleSize = 0;
|
||||
for (i = 0; i < ctx->nbSamples; ++i) {
|
||||
i = parameters.splitPoint < 1.0 ? ctx->nbTrainSamples : 0;
|
||||
for (; i < ctx->nbSamples; ++i) {
|
||||
maxSampleSize = MAX(ctx->samplesSizes[i], maxSampleSize);
|
||||
}
|
||||
dstCapacity = ZSTD_compressBound(maxSampleSize);
|
||||
|
|
@ -904,7 +931,8 @@ static void COVER_tryParameters(void *opaque) {
|
|||
}
|
||||
/* Compress each sample and sum their sizes (or error) */
|
||||
totalCompressedSize = dictBufferCapacity;
|
||||
for (i = 0; i < ctx->nbSamples; ++i) {
|
||||
i = parameters.splitPoint < 1.0 ? ctx->nbTrainSamples : 0;
|
||||
for (; i < ctx->nbSamples; ++i) {
|
||||
const size_t size = ZSTD_compress_usingCDict(
|
||||
cctx, dst, dstCapacity, ctx->samples + ctx->offsets[i],
|
||||
ctx->samplesSizes[i], cdict);
|
||||
|
|
@ -941,6 +969,8 @@ ZDICTLIB_API size_t ZDICT_optimizeTrainFromBuffer_cover(
|
|||
ZDICT_cover_params_t *parameters) {
|
||||
/* constants */
|
||||
const unsigned nbThreads = parameters->nbThreads;
|
||||
const double splitPoint =
|
||||
parameters->splitPoint <= 0.0 ? DEFAULT_SPLITPOINT : parameters->splitPoint;
|
||||
const unsigned kMinD = parameters->d == 0 ? 6 : parameters->d;
|
||||
const unsigned kMaxD = parameters->d == 0 ? 8 : parameters->d;
|
||||
const unsigned kMinK = parameters->k == 0 ? 50 : parameters->k;
|
||||
|
|
@ -958,6 +988,10 @@ ZDICTLIB_API size_t ZDICT_optimizeTrainFromBuffer_cover(
|
|||
POOL_ctx *pool = NULL;
|
||||
|
||||
/* Checks */
|
||||
if (splitPoint <= 0 || splitPoint > 1) {
|
||||
LOCALDISPLAYLEVEL(displayLevel, 1, "Incorrect parameters\n");
|
||||
return ERROR(GENERIC);
|
||||
}
|
||||
if (kMinK < kMaxD || kMaxK < kMinK) {
|
||||
LOCALDISPLAYLEVEL(displayLevel, 1, "Incorrect parameters\n");
|
||||
return ERROR(GENERIC);
|
||||
|
|
@ -988,7 +1022,7 @@ ZDICTLIB_API size_t ZDICT_optimizeTrainFromBuffer_cover(
|
|||
/* Initialize the context for this value of d */
|
||||
COVER_ctx_t ctx;
|
||||
LOCALDISPLAYLEVEL(displayLevel, 3, "d=%u\n", d);
|
||||
if (!COVER_ctx_init(&ctx, samplesBuffer, samplesSizes, nbSamples, d)) {
|
||||
if (!COVER_ctx_init(&ctx, samplesBuffer, samplesSizes, nbSamples, d, splitPoint)) {
|
||||
LOCALDISPLAYLEVEL(displayLevel, 1, "Failed to initialize context\n");
|
||||
COVER_best_destroy(&best);
|
||||
POOL_free(pool);
|
||||
|
|
@ -1013,6 +1047,7 @@ ZDICTLIB_API size_t ZDICT_optimizeTrainFromBuffer_cover(
|
|||
data->parameters = *parameters;
|
||||
data->parameters.k = k;
|
||||
data->parameters.d = d;
|
||||
data->parameters.splitPoint = splitPoint;
|
||||
data->parameters.steps = kSteps;
|
||||
data->parameters.zParams.notificationLevel = g_displayLevel;
|
||||
/* Check the parameters */
|
||||
|
|
|
|||
|
|
@ -86,6 +86,7 @@ typedef struct {
|
|||
unsigned d; /* dmer size : constraint: 0 < d <= k : Reasonable range [6, 16] */
|
||||
unsigned steps; /* Number of steps : Only used for optimization : 0 means default (32) : Higher means more parameters checked */
|
||||
unsigned nbThreads; /* Number of threads : constraint: 0 < nbThreads : 1 means single-threaded : Only used for optimization : Ignored if ZSTD_MULTITHREAD is not defined */
|
||||
double splitPoint; /* Percentage of samples used for training: the first nbSamples * splitPoint samples will be used to training, the last nbSamples * (1 - splitPoint) samples will be used for testing, 0 means default (1.0), 1.0 when all samples are used for both training and testing */
|
||||
ZDICT_params_t zParams;
|
||||
} ZDICT_cover_params_t;
|
||||
|
||||
|
|
|
|||
|
|
@ -150,7 +150,7 @@ Advanced arguments :
|
|||
|
||||
Dictionary builder :
|
||||
--train ## : create a dictionary from a training set of files
|
||||
--train-cover[=k=#,d=#,steps=#] : use the cover algorithm with optional args
|
||||
--train-cover[=k=#,d=#,steps=#,split=#] : use the cover algorithm with optional args
|
||||
--train-legacy[=s=#] : use the legacy algorithm with selectivity (default: 9)
|
||||
-o file : `file` is dictionary name (default: dictionary)
|
||||
--maxdict=# : limit dictionary to specified size (default: 112640)
|
||||
|
|
|
|||
1104
programs/bench.c
1104
programs/bench.c
File diff suppressed because it is too large
Load Diff
204
programs/bench.h
204
programs/bench.h
|
|
@ -19,25 +19,121 @@ extern "C" {
|
|||
#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_compressionParameters */
|
||||
#include "zstd.h" /* ZSTD_compressionParameters */
|
||||
|
||||
/* Creates a struct of type typeName with an int type .error field
|
||||
* and a .result field of some baseType. Functions with return
|
||||
* typeName pass a successful result with .error = 0 and .result
|
||||
* with the intended result, while returning an error will result
|
||||
* in .error != 0.
|
||||
*/
|
||||
#define ERROR_STRUCT(baseType, typeName) typedef struct { \
|
||||
baseType result; \
|
||||
int error; \
|
||||
} typeName
|
||||
|
||||
typedef struct {
|
||||
size_t cSize;
|
||||
double cSpeed; /* bytes / sec */
|
||||
double dSpeed;
|
||||
} BMK_result_t;
|
||||
|
||||
/* 0 = no Error */
|
||||
typedef struct {
|
||||
int errorCode;
|
||||
BMK_result_t result;
|
||||
} BMK_return_t;
|
||||
ERROR_STRUCT(BMK_result_t, BMK_return_t);
|
||||
|
||||
/* called in cli */
|
||||
int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles, const char* dictFileName,
|
||||
int cLevel, int cLevelLast, const ZSTD_compressionParameters* compressionParams,
|
||||
/* Loads files in fileNamesTable into memory, as well as a dictionary
|
||||
* from dictFileName, and then uses benchMem */
|
||||
/* fileNamesTable - name of files to benchmark
|
||||
* nbFiles - number of files (size of fileNamesTable), must be > 0
|
||||
* dictFileName - name of dictionary file to load
|
||||
* cLevel - compression level to benchmark, errors if invalid
|
||||
* compressionParams - basic compression Parameters
|
||||
* displayLevel - what gets printed
|
||||
* 0 : no display;
|
||||
* 1 : errors;
|
||||
* 2 : + result + interaction + warnings;
|
||||
* 3 : + progression;
|
||||
* 4 : + information
|
||||
* return
|
||||
* .error will give a nonzero error value if an error has occured
|
||||
* .result - if .error = 0, .result will return the time taken to compression speed
|
||||
* (.cSpeed), decompression speed (.dSpeed), and compressed size (.cSize) of the original
|
||||
* file
|
||||
*/
|
||||
BMK_return_t BMK_benchFiles(const char* const * const fileNamesTable, unsigned const nbFiles,
|
||||
const char* const dictFileName,
|
||||
int const cLevel, const ZSTD_compressionParameters* const compressionParams,
|
||||
int displayLevel);
|
||||
|
||||
/* basic benchmarking function, called in paramgrill
|
||||
* ctx, dctx must be valid */
|
||||
typedef enum {
|
||||
BMK_timeMode = 0,
|
||||
BMK_iterMode = 1
|
||||
} BMK_loopMode_t;
|
||||
|
||||
typedef enum {
|
||||
BMK_both = 0,
|
||||
BMK_decodeOnly = 1,
|
||||
BMK_compressOnly = 2
|
||||
} BMK_mode_t;
|
||||
|
||||
typedef struct {
|
||||
BMK_mode_t mode; /* 0: all, 1: compress only 2: decode only */
|
||||
BMK_loopMode_t loopMode; /* if loopmode, then nbSeconds = nbLoops */
|
||||
unsigned nbSeconds; /* default timing is in nbSeconds */
|
||||
size_t blockSize; /* Maximum allowable size of a block*/
|
||||
unsigned nbWorkers; /* multithreading */
|
||||
unsigned realTime; /* real time priority */
|
||||
int additionalParam; /* used by python speed benchmark */
|
||||
unsigned ldmFlag; /* enables long distance matching */
|
||||
unsigned ldmMinMatch; /* below: parameters for long distance matching, see zstd.1.md for meaning */
|
||||
unsigned ldmHashLog;
|
||||
unsigned ldmBucketSizeLog;
|
||||
unsigned ldmHashEveryLog;
|
||||
} BMK_advancedParams_t;
|
||||
|
||||
/* returns default parameters used by nonAdvanced functions */
|
||||
BMK_advancedParams_t BMK_initAdvancedParams(void);
|
||||
|
||||
/* See benchFiles for normal parameter uses and return, see advancedParams_t for adv */
|
||||
BMK_return_t BMK_benchFilesAdvanced(const char* const * const fileNamesTable, unsigned const nbFiles,
|
||||
const char* const dictFileName,
|
||||
int const cLevel, const ZSTD_compressionParameters* const compressionParams,
|
||||
int displayLevel, const BMK_advancedParams_t* const adv);
|
||||
|
||||
/* called in cli */
|
||||
/* Generates a sample with datagen with the compressibility argument*/
|
||||
/* cLevel - compression level to benchmark, errors if invalid
|
||||
* compressibility - determines compressibility of sample
|
||||
* compressionParams - basic compression Parameters
|
||||
* displayLevel - see benchFiles
|
||||
* adv - see advanced_Params_t
|
||||
* return
|
||||
* .error will give a nonzero error value if an error has occured
|
||||
* .result - if .error = 0, .result will return the time taken to compression speed
|
||||
* (.cSpeed), decompression speed (.dSpeed), and compressed size (.cSize) of the original
|
||||
* file
|
||||
*/
|
||||
BMK_return_t BMK_syntheticTest(int cLevel, double compressibility,
|
||||
const ZSTD_compressionParameters* compressionParams,
|
||||
int displayLevel, const BMK_advancedParams_t * const adv);
|
||||
|
||||
/* basic benchmarking function, called in paramgrill
|
||||
* applies ZSTD_compress_generic() and ZSTD_decompress_generic() on data in srcBuffer
|
||||
* with specific compression parameters specified by other arguments using benchFunction
|
||||
* (cLevel, comprParams + adv in advanced Mode) */
|
||||
/* srcBuffer - data source, expected to be valid compressed data if in Decode Only Mode
|
||||
* srcSize - size of data in srcBuffer
|
||||
* cLevel - compression level
|
||||
* comprParams - basic compression parameters
|
||||
* dictBuffer - a dictionary if used, null otherwise
|
||||
* dictBufferSize - size of dictBuffer, 0 otherwise
|
||||
* ctx - Compression Context (must be provided)
|
||||
* dctx - Decompression Context (must be provided)
|
||||
* diplayLevel - see BMK_benchFiles
|
||||
* displayName - name used by display
|
||||
* return
|
||||
* .error will give a nonzero value if an error has occured
|
||||
* .result - if .error = 0, will give the same results as benchFiles
|
||||
* but for the data stored in srcBuffer
|
||||
*/
|
||||
BMK_return_t BMK_benchMem(const void* srcBuffer, size_t srcSize,
|
||||
const size_t* fileSizes, unsigned nbFiles,
|
||||
const int cLevel, const ZSTD_compressionParameters* comprParams,
|
||||
|
|
@ -45,20 +141,82 @@ BMK_return_t BMK_benchMem(const void* srcBuffer, size_t srcSize,
|
|||
ZSTD_CCtx* ctx, ZSTD_DCtx* dctx,
|
||||
int displayLevel, const char* displayName);
|
||||
|
||||
/* Set Parameters */
|
||||
void BMK_setNbSeconds(unsigned nbLoops);
|
||||
void BMK_setBlockSize(size_t blockSize);
|
||||
void BMK_setNbWorkers(unsigned nbWorkers);
|
||||
void BMK_setRealTime(unsigned priority);
|
||||
void BMK_setNotificationLevel(unsigned level);
|
||||
void BMK_setSeparateFiles(unsigned separate);
|
||||
void BMK_setAdditionalParam(int additionalParam);
|
||||
void BMK_setDecodeOnlyMode(unsigned decodeFlag);
|
||||
void BMK_setLdmFlag(unsigned ldmFlag);
|
||||
void BMK_setLdmMinMatch(unsigned ldmMinMatch);
|
||||
void BMK_setLdmHashLog(unsigned ldmHashLog);
|
||||
void BMK_setLdmBucketSizeLog(unsigned ldmBucketSizeLog);
|
||||
void BMK_setLdmHashEveryLog(unsigned ldmHashEveryLog);
|
||||
/* See benchMem for normal parameter uses and return, see advancedParams_t for adv */
|
||||
BMK_return_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize,
|
||||
const size_t* fileSizes, unsigned nbFiles,
|
||||
const int cLevel, const ZSTD_compressionParameters* comprParams,
|
||||
const void* dictBuffer, size_t dictBufferSize,
|
||||
ZSTD_CCtx* ctx, ZSTD_DCtx* dctx,
|
||||
int displayLevel, const char* displayName,
|
||||
const BMK_advancedParams_t* adv);
|
||||
|
||||
typedef struct {
|
||||
size_t sumOfReturn; /* sum of return values */
|
||||
U64 nanoSecPerRun; /* time per iteration */
|
||||
} BMK_customResult_t;
|
||||
|
||||
ERROR_STRUCT(BMK_customResult_t, BMK_customReturn_t);
|
||||
|
||||
typedef size_t (*BMK_benchFn_t)(const void*, size_t, void*, size_t, void*);
|
||||
typedef size_t (*BMK_initFn_t)(void*);
|
||||
|
||||
/* This function times the execution of 2 argument functions, benchFn and initFn */
|
||||
|
||||
/* benchFn - (*benchFn)(srcBuffers[i], srcSizes[i], dstBuffers[i], dstCapacities[i], benchPayload)
|
||||
* is run nbLoops times
|
||||
* initFn - (*initFn)(initPayload) is run once per benchmark at the beginning. This argument can
|
||||
* be NULL, in which case nothing is run.
|
||||
* blockCount - number of blocks (size of srcBuffers, srcSizes, dstBuffers, dstCapacities)
|
||||
* srcBuffers - an array of buffers to be operated on by benchFn
|
||||
* srcSizes - an array of the sizes of above buffers
|
||||
* dstBuffers - an array of buffers to be written into by benchFn
|
||||
* dstCapacities - an array of the capacities of above buffers.
|
||||
* nbLoops - defines number of times benchFn is run.
|
||||
* return
|
||||
* .error will give a nonzero value if ZSTD_isError() is nonzero for any of the return
|
||||
* of the calls to initFn and benchFn, or if benchFunction errors internally
|
||||
* .result - if .error = 0, then .result will contain the sum of all return values of
|
||||
* benchFn on the first iteration through all of the blocks (.sumOfReturn) and also
|
||||
* the time per run of benchFn (.nanoSecPerRun). For the former, this
|
||||
* is generally intended to be used on functions which return the # of bytes written
|
||||
* into dstBuffer, hence this value will be the total amount of bytes written to
|
||||
* dstBuffer.
|
||||
*/
|
||||
BMK_customReturn_t BMK_benchFunction(
|
||||
BMK_benchFn_t benchFn, void* benchPayload,
|
||||
BMK_initFn_t initFn, void* initPayload,
|
||||
size_t blockCount,
|
||||
const void* const * const srcBuffers, const size_t* srcSizes,
|
||||
void* const * const dstBuffers, const size_t* dstCapacities,
|
||||
unsigned nbLoops);
|
||||
|
||||
|
||||
/* state information needed to advance computation for benchFunctionTimed */
|
||||
typedef struct BMK_timeState_t BMK_timedFnState_t;
|
||||
/* initializes timeState object with desired number of seconds */
|
||||
BMK_timedFnState_t* BMK_createTimeState(unsigned nbSeconds);
|
||||
/* resets existing timeState object */
|
||||
void BMK_resetTimeState(BMK_timedFnState_t*, unsigned nbSeconds);
|
||||
/* deletes timeState object */
|
||||
void BMK_freeTimeState(BMK_timedFnState_t* state);
|
||||
|
||||
typedef struct {
|
||||
BMK_customReturn_t result;
|
||||
int completed;
|
||||
} BMK_customTimedReturn_t;
|
||||
|
||||
/*
|
||||
* Benchmarks custom functions like BMK_benchFunction(), but runs for nbSeconds seconds rather than a fixed number of loops
|
||||
* arguments mostly the same other than BMK_benchFunction()
|
||||
* Usage - benchFunctionTimed will return in approximately one second. Keep calling BMK_benchFunctionTimed() until the return's completed field = 1.
|
||||
* to continue updating intermediate result. Intermediate return values are returned by the function.
|
||||
*/
|
||||
BMK_customTimedReturn_t BMK_benchFunctionTimed(BMK_timedFnState_t* cont,
|
||||
BMK_benchFn_t benchFn, void* benchPayload,
|
||||
BMK_initFn_t initFn, void* initPayload,
|
||||
size_t blockCount,
|
||||
const void* const * const srcBlockBuffers, const size_t* srcBlockSizes,
|
||||
void* const * const dstBlockBuffers, const size_t* dstBlockCapacities);
|
||||
|
||||
#endif /* BENCH_H_121279284357 */
|
||||
|
||||
|
|
|
|||
|
|
@ -323,7 +323,8 @@ int DiB_trainFromFiles(const char* dictFileName, unsigned maxDictSize,
|
|||
srcBuffer, sampleSizes, fs.nbSamples,
|
||||
coverParams);
|
||||
if (!ZDICT_isError(dictSize)) {
|
||||
DISPLAYLEVEL(2, "k=%u\nd=%u\nsteps=%u\n", coverParams->k, coverParams->d, coverParams->steps);
|
||||
unsigned splitPercentage = (unsigned)(coverParams->splitPoint * 100);
|
||||
DISPLAYLEVEL(2, "k=%u\nd=%u\nsteps=%u\nsplit=%u\n", coverParams->k, coverParams->d, coverParams->steps, splitPercentage);
|
||||
}
|
||||
} else {
|
||||
dictSize = ZDICT_trainFromBuffer_cover(dictBuffer, maxDictSize, srcBuffer,
|
||||
|
|
|
|||
|
|
@ -2017,21 +2017,25 @@ static int FIO_listFile(fileInfo_t* total, const char* inFileName, int displayLe
|
|||
}
|
||||
|
||||
int FIO_listMultipleFiles(unsigned numFiles, const char** filenameTable, int displayLevel){
|
||||
|
||||
if (!IS_CONSOLE(stdin)) {
|
||||
DISPLAYOUT("zstd: --list does not support reading from standard input\n");
|
||||
return 1;
|
||||
unsigned u;
|
||||
for (u=0; u<numFiles;u++) {
|
||||
if (!strcmp (filenameTable[u], stdinmark)) {
|
||||
DISPLAYOUT("zstd: --list does not support reading from standard input\n");
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
|
||||
if (numFiles == 0) {
|
||||
if (!IS_CONSOLE(stdin)) {
|
||||
DISPLAYOUT("zstd: --list does not support reading from standard input\n");
|
||||
}
|
||||
DISPLAYOUT("No files given\n");
|
||||
return 0;
|
||||
return 1;
|
||||
}
|
||||
if (displayLevel <= 2) {
|
||||
DISPLAYOUT("Frames Skips Compressed Uncompressed Ratio Check Filename\n");
|
||||
}
|
||||
{ int error = 0;
|
||||
unsigned u;
|
||||
fileInfo_t total;
|
||||
memset(&total, 0, sizeof(total));
|
||||
total.usesCheck = 1;
|
||||
|
|
|
|||
|
|
@ -217,8 +217,9 @@ Split input files in blocks of size # (default: no split)
|
|||
A dictionary ID is a locally unique ID that a decoder can use to verify it is using the right dictionary\. By default, zstd will create a 4\-bytes random number ID\. It\'s possible to give a precise number instead\. Short numbers have an advantage : an ID < 256 will only need 1 byte in the compressed frame header, and an ID < 65536 will only need 2 bytes\. This compares favorably to 4 bytes default\. However, it\'s up to the dictionary manager to not assign twice the same ID to 2 different dictionaries\.
|
||||
.
|
||||
.TP
|
||||
\fB\-\-train\-cover[=k#,d=#,steps=#]\fR
|
||||
Select parameters for the default dictionary builder algorithm named cover\. If \fId\fR is not specified, then it tries \fId\fR = 6 and \fId\fR = 8\. If \fIk\fR is not specified, then it tries \fIsteps\fR values in the range [50, 2000]\. If \fIsteps\fR is not specified, then the default value of 40 is used\. Requires that \fId\fR <= \fIk\fR\.
|
||||
\fB\-\-train\-cover[=k#,d=#,steps=#,split=#]\fR
|
||||
Select parameters for the default dictionary builder algorithm named cover\. If \fId\fR is not specified, then it tries \fId\fR = 6 and \fId\fR = 8\. If \fIk\fR is not specified, then it tries \fIsteps\fR values in the range [50, 2000]\. If \fIsteps\fR is not specified, then the default value of 40 is used\. If \fIsplit\fR is not specified or \fIsplit\fR <= 0, then the default value of 100 is used\. If \fIsplit\fR is 100, all input samples are used for both training and testing
|
||||
to find optimal _d_ and _k_ to build dictionary.Requires that \fId\fR <= \fIk\fR\.
|
||||
.
|
||||
.IP
|
||||
Selects segments of size \fIk\fR with highest score to put in the dictionary\. The score of a segment is computed by the sum of the frequencies of all the subsegments of size \fId\fR\. Generally \fId\fR should be in the range [6, 8], occasionally up to 16, but the algorithm will run faster with d <= \fI8\fR\. Good values for \fIk\fR vary widely based on the input data, but a safe range is [2 * \fId\fR, 2000]\. Supports multithreading if \fBzstd\fR is compiled with threading support\.
|
||||
|
|
|
|||
|
|
@ -223,11 +223,12 @@ Compression of small files similar to the sample set will be greatly improved.
|
|||
This compares favorably to 4 bytes default.
|
||||
However, it's up to the dictionary manager to not assign twice the same ID to
|
||||
2 different dictionaries.
|
||||
* `--train-cover[=k#,d=#,steps=#]`:
|
||||
* `--train-cover[=k#,d=#,steps=#,split=#]`:
|
||||
Select parameters for the default dictionary builder algorithm named cover.
|
||||
If _d_ is not specified, then it tries _d_ = 6 and _d_ = 8.
|
||||
If _k_ is not specified, then it tries _steps_ values in the range [50, 2000].
|
||||
If _steps_ is not specified, then the default value of 40 is used.
|
||||
If _split_ is not specified or split <= 0, then the default value of 100 is used.
|
||||
Requires that _d_ <= _k_.
|
||||
|
||||
Selects segments of size _k_ with highest score to put in the dictionary.
|
||||
|
|
@ -237,6 +238,8 @@ Compression of small files similar to the sample set will be greatly improved.
|
|||
algorithm will run faster with d <= _8_.
|
||||
Good values for _k_ vary widely based on the input data, but a safe range is
|
||||
[2 * _d_, 2000].
|
||||
If _split_ is 100, all input samples are used for both training and testing
|
||||
to find optimal _d_ and _k_ to build dictionary.
|
||||
Supports multithreading if `zstd` is compiled with threading support.
|
||||
|
||||
Examples:
|
||||
|
|
@ -249,6 +252,8 @@ Compression of small files similar to the sample set will be greatly improved.
|
|||
|
||||
`zstd --train-cover=k=50 FILEs`
|
||||
|
||||
`zstd --train-cover=k=50,split=60 FILEs`
|
||||
|
||||
* `--train-legacy[=selectivity=#]`:
|
||||
Use legacy dictionary builder algorithm with the given dictionary
|
||||
_selectivity_ (default: 9).
|
||||
|
|
|
|||
|
|
@ -32,7 +32,7 @@
|
|||
#include <errno.h> /* errno */
|
||||
#include "fileio.h" /* stdinmark, stdoutmark, ZSTD_EXTENSION */
|
||||
#ifndef ZSTD_NOBENCH
|
||||
# include "bench.h" /* BMK_benchFiles, BMK_SetNbSeconds */
|
||||
# include "bench.h" /* BMK_benchFiles */
|
||||
#endif
|
||||
#ifndef ZSTD_NODICT
|
||||
# include "dibio.h" /* ZDICT_cover_params_t, DiB_trainFromFiles() */
|
||||
|
|
@ -84,6 +84,7 @@ static U32 g_ldmMinMatch = 0;
|
|||
static U32 g_ldmHashEveryLog = LDM_PARAM_DEFAULT;
|
||||
static U32 g_ldmBucketSizeLog = LDM_PARAM_DEFAULT;
|
||||
|
||||
#define DEFAULT_SPLITPOINT 1.0
|
||||
|
||||
/*-************************************
|
||||
* Display Macros
|
||||
|
|
@ -170,7 +171,7 @@ static int usage_advanced(const char* programName)
|
|||
DISPLAY( "\n");
|
||||
DISPLAY( "Dictionary builder : \n");
|
||||
DISPLAY( "--train ## : create a dictionary from a training set of files \n");
|
||||
DISPLAY( "--train-cover[=k=#,d=#,steps=#] : use the cover algorithm with optional args\n");
|
||||
DISPLAY( "--train-cover[=k=#,d=#,steps=#,split=#] : use the cover algorithm with optional args\n");
|
||||
DISPLAY( "--train-legacy[=s=#] : use the legacy algorithm with selectivity (default: %u)\n", g_defaultSelectivityLevel);
|
||||
DISPLAY( " -o file : `file` is dictionary name (default: %s) \n", g_defaultDictName);
|
||||
DISPLAY( "--maxdict=# : limit dictionary to specified size (default: %u) \n", g_defaultMaxDictSize);
|
||||
|
|
@ -282,10 +283,15 @@ static unsigned parseCoverParameters(const char* stringPtr, ZDICT_cover_params_t
|
|||
if (longCommandWArg(&stringPtr, "k=")) { params->k = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
|
||||
if (longCommandWArg(&stringPtr, "d=")) { params->d = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
|
||||
if (longCommandWArg(&stringPtr, "steps=")) { params->steps = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
|
||||
if (longCommandWArg(&stringPtr, "split=")) {
|
||||
unsigned splitPercentage = readU32FromChar(&stringPtr);
|
||||
params->splitPoint = (double)splitPercentage / 100.0;
|
||||
if (stringPtr[0]==',') { stringPtr++; continue; } else break;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
if (stringPtr[0] != 0) return 0;
|
||||
DISPLAYLEVEL(4, "cover: k=%u\nd=%u\nsteps=%u\n", params->k, params->d, params->steps);
|
||||
DISPLAYLEVEL(4, "cover: k=%u\nd=%u\nsteps=%u\nsplit=%u\n", params->k, params->d, params->steps, (unsigned)(params->splitPoint * 100));
|
||||
return 1;
|
||||
}
|
||||
|
||||
|
|
@ -310,6 +316,7 @@ static ZDICT_cover_params_t defaultCoverParams(void)
|
|||
memset(¶ms, 0, sizeof(params));
|
||||
params.d = 8;
|
||||
params.steps = 4;
|
||||
params.splitPoint = DEFAULT_SPLITPOINT;
|
||||
return params;
|
||||
}
|
||||
#endif
|
||||
|
|
@ -398,6 +405,8 @@ int main(int argCount, const char* argv[])
|
|||
setRealTimePrio = 0,
|
||||
singleThread = 0,
|
||||
ultra=0;
|
||||
double compressibility = 0.5;
|
||||
BMK_advancedParams_t adv = BMK_initAdvancedParams();
|
||||
unsigned bench_nbSeconds = 3; /* would be better if this value was synchronized from bench */
|
||||
size_t blockSize = 0;
|
||||
zstd_operation_mode operation = zom_compress;
|
||||
|
|
@ -441,7 +450,7 @@ int main(int argCount, const char* argv[])
|
|||
#endif
|
||||
|
||||
/* preset behaviors */
|
||||
if (exeNameMatch(programName, ZSTD_ZSTDMT)) nbWorkers=0;
|
||||
if (exeNameMatch(programName, ZSTD_ZSTDMT)) nbWorkers=0, singleThread=0;
|
||||
if (exeNameMatch(programName, ZSTD_UNZSTD)) operation=zom_decompress;
|
||||
if (exeNameMatch(programName, ZSTD_CAT)) { operation=zom_decompress; forceStdout=1; FIO_overwriteMode(); outFileName=stdoutmark; g_displayLevel=1; } /* supports multiple formats */
|
||||
if (exeNameMatch(programName, ZSTD_ZCAT)) { operation=zom_decompress; forceStdout=1; FIO_overwriteMode(); outFileName=stdoutmark; g_displayLevel=1; } /* behave like zcat, also supports multiple formats */
|
||||
|
|
@ -609,7 +618,7 @@ int main(int argCount, const char* argv[])
|
|||
/* Decoding */
|
||||
case 'd':
|
||||
#ifndef ZSTD_NOBENCH
|
||||
BMK_setDecodeOnlyMode(1);
|
||||
adv.mode = BMK_decodeOnly;
|
||||
if (operation==zom_bench) { argument++; break; } /* benchmark decode (hidden option) */
|
||||
#endif
|
||||
operation=zom_decompress; argument++; break;
|
||||
|
|
@ -702,11 +711,19 @@ int main(int argCount, const char* argv[])
|
|||
case 'p': argument++;
|
||||
#ifndef ZSTD_NOBENCH
|
||||
if ((*argument>='0') && (*argument<='9')) {
|
||||
BMK_setAdditionalParam(readU32FromChar(&argument));
|
||||
adv.additionalParam = (int)readU32FromChar(&argument);
|
||||
} else
|
||||
#endif
|
||||
main_pause=1;
|
||||
break;
|
||||
|
||||
/* Select compressibility of synthetic sample */
|
||||
case 'P':
|
||||
{ argument++;
|
||||
compressibility = (double)readU32FromChar(&argument) / 100;
|
||||
}
|
||||
break;
|
||||
|
||||
/* unknown command */
|
||||
default : CLEAN_RETURN(badusage(programName));
|
||||
}
|
||||
|
|
@ -764,7 +781,7 @@ int main(int argCount, const char* argv[])
|
|||
DISPLAYLEVEL(3, "Note: %d physical core(s) detected \n", nbWorkers);
|
||||
}
|
||||
#else
|
||||
(void)singleThread;
|
||||
(void)singleThread; (void)nbWorkers;
|
||||
#endif
|
||||
|
||||
#ifdef UTIL_HAS_CREATEFILELIST
|
||||
|
|
@ -807,21 +824,46 @@ int main(int argCount, const char* argv[])
|
|||
/* Check if benchmark is selected */
|
||||
if (operation==zom_bench) {
|
||||
#ifndef ZSTD_NOBENCH
|
||||
BMK_setSeparateFiles(separateFiles);
|
||||
BMK_setBlockSize(blockSize);
|
||||
BMK_setNbWorkers(nbWorkers);
|
||||
BMK_setRealTime(setRealTimePrio);
|
||||
BMK_setNbSeconds(bench_nbSeconds);
|
||||
BMK_setLdmFlag(ldmFlag);
|
||||
BMK_setLdmMinMatch(g_ldmMinMatch);
|
||||
BMK_setLdmHashLog(g_ldmHashLog);
|
||||
adv.blockSize = blockSize;
|
||||
adv.nbWorkers = nbWorkers;
|
||||
adv.realTime = setRealTimePrio;
|
||||
adv.nbSeconds = bench_nbSeconds;
|
||||
adv.ldmFlag = ldmFlag;
|
||||
adv.ldmMinMatch = g_ldmMinMatch;
|
||||
adv.ldmHashLog = g_ldmHashLog;
|
||||
if (g_ldmBucketSizeLog != LDM_PARAM_DEFAULT) {
|
||||
BMK_setLdmBucketSizeLog(g_ldmBucketSizeLog);
|
||||
adv.ldmBucketSizeLog = g_ldmBucketSizeLog;
|
||||
}
|
||||
if (g_ldmHashEveryLog != LDM_PARAM_DEFAULT) {
|
||||
BMK_setLdmHashEveryLog(g_ldmHashEveryLog);
|
||||
adv.ldmHashEveryLog = g_ldmHashEveryLog;
|
||||
}
|
||||
BMK_benchFiles(filenameTable, filenameIdx, dictFileName, cLevel, cLevelLast, &compressionParams, g_displayLevel);
|
||||
|
||||
if (cLevel > ZSTD_maxCLevel()) cLevel = ZSTD_maxCLevel();
|
||||
if (cLevelLast > ZSTD_maxCLevel()) cLevelLast = ZSTD_maxCLevel();
|
||||
if (cLevelLast < cLevel) cLevelLast = cLevel;
|
||||
if (cLevelLast > cLevel)
|
||||
DISPLAYLEVEL(2, "Benchmarking levels from %d to %d\n", cLevel, cLevelLast);
|
||||
if(filenameIdx) {
|
||||
if(separateFiles) {
|
||||
unsigned i;
|
||||
for(i = 0; i < filenameIdx; i++) {
|
||||
int c;
|
||||
DISPLAYLEVEL(2, "Benchmarking %s \n", filenameTable[i]);
|
||||
for(c = cLevel; c <= cLevelLast; c++) {
|
||||
BMK_benchFilesAdvanced(&filenameTable[i], 1, dictFileName, c, &compressionParams, g_displayLevel, &adv);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
for(; cLevel <= cLevelLast; cLevel++) {
|
||||
BMK_benchFilesAdvanced(filenameTable, filenameIdx, dictFileName, cLevel, &compressionParams, g_displayLevel, &adv);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
for(; cLevel <= cLevelLast; cLevel++) {
|
||||
BMK_syntheticTest(cLevel, compressibility, &compressionParams, g_displayLevel, &adv);
|
||||
}
|
||||
}
|
||||
|
||||
#else
|
||||
(void)bench_nbSeconds; (void)blockSize; (void)setRealTimePrio; (void)separateFiles;
|
||||
#endif
|
||||
|
|
|
|||
|
|
@ -88,13 +88,8 @@ allnothread: fullbench fuzzer paramgrill datagen decodecorpus
|
|||
|
||||
dll: fuzzer-dll zstreamtest-dll
|
||||
|
||||
zstd:
|
||||
$(MAKE) -C $(PRGDIR) $@ MOREFLAGS+="$(DEBUGFLAGS)"
|
||||
|
||||
zstd32:
|
||||
$(MAKE) -C $(PRGDIR) $@ MOREFLAGS+="$(DEBUGFLAGS)"
|
||||
|
||||
zstd-nolegacy:
|
||||
PHONY: zstd zstd32 zstd-nolegacy # must be phony, only external makefile knows how to build them, or if they need an update
|
||||
zstd zstd32 zstd-nolegacy:
|
||||
$(MAKE) -C $(PRGDIR) $@ MOREFLAGS+="$(DEBUGFLAGS)"
|
||||
|
||||
gzstd:
|
||||
|
|
@ -133,7 +128,7 @@ fullbench fullbench32 : CPPFLAGS += $(MULTITHREAD_CPP)
|
|||
fullbench fullbench32 : LDFLAGS += $(MULTITHREAD_LD)
|
||||
fullbench fullbench32 : DEBUGFLAGS = # turn off assert() for speed measurements
|
||||
fullbench fullbench32 : $(ZSTD_FILES)
|
||||
fullbench fullbench32 : $(PRGDIR)/datagen.c fullbench.c
|
||||
fullbench fullbench32 : $(PRGDIR)/datagen.c $(PRGDIR)/bench.c fullbench.c
|
||||
$(CC) $(FLAGS) $^ -o $@$(EXT)
|
||||
|
||||
fullbench-lib : zstd-staticLib
|
||||
|
|
@ -245,13 +240,14 @@ checkTag: checkTag.c $(ZSTDDIR)/zstd.h
|
|||
|
||||
clean:
|
||||
$(MAKE) -C $(ZSTDDIR) clean
|
||||
$(MAKE) -C $(PRGDIR) clean
|
||||
@$(RM) -fR $(TESTARTEFACT)
|
||||
@$(RM) -f core *.o tmp* result* *.gcda dictionary *.zst \
|
||||
$(PRGDIR)/zstd$(EXT) $(PRGDIR)/zstd32$(EXT) \
|
||||
fullbench$(EXT) fullbench32$(EXT) \
|
||||
fullbench-lib$(EXT) fullbench-dll$(EXT) \
|
||||
fuzzer$(EXT) fuzzer32$(EXT) zbufftest$(EXT) zbufftest32$(EXT) \
|
||||
fuzzer-dll$(EXT) zstreamtest-dll$(EXT) zbufftest-dll$(EXT)\
|
||||
fuzzer-dll$(EXT) zstreamtest-dll$(EXT) zbufftest-dll$(EXT) \
|
||||
zstreamtest$(EXT) zstreamtest32$(EXT) \
|
||||
datagen$(EXT) paramgrill$(EXT) roundTripCrash$(EXT) longmatch$(EXT) \
|
||||
symbols$(EXT) invalidDictionaries$(EXT) legacy$(EXT) poolTests$(EXT) \
|
||||
|
|
@ -301,11 +297,6 @@ endif
|
|||
list:
|
||||
@$(MAKE) -pRrq -f $(lastword $(MAKEFILE_LIST)) : 2>/dev/null | awk -v RS= -F: '/^# File/,/^# Finished Make data base/ {if ($$1 !~ "^[#.]") {print $$1}}' | sort | egrep -v -e '^[^[:alnum:]]' -e '^$@$$' | xargs
|
||||
|
||||
.PHONY: zstd-playTests
|
||||
zstd-playTests: datagen
|
||||
file $(ZSTD)
|
||||
ZSTD="$(QEMU_SYS) $(ZSTD)" ./playTests.sh $(ZSTDRTTEST)
|
||||
|
||||
.PHONY: shortest
|
||||
shortest: ZSTDRTTEST=
|
||||
shortest: test-zstd
|
||||
|
|
@ -323,14 +314,21 @@ test32: test-zstd32 test-fullbench32 test-fuzzer32 test-zstream32
|
|||
|
||||
test-all: test test32 valgrindTest test-decodecorpus-cli
|
||||
|
||||
|
||||
.PHONY: test-zstd test-zstd32 test-zstd-nolegacy
|
||||
test-zstd: ZSTD = $(PRGDIR)/zstd
|
||||
test-zstd: zstd zstd-playTests
|
||||
test-zstd: zstd
|
||||
|
||||
test-zstd32: ZSTD = $(PRGDIR)/zstd32
|
||||
test-zstd32: zstd32 zstd-playTests
|
||||
test-zstd32: zstd32
|
||||
|
||||
test-zstd-nolegacy: ZSTD = $(PRGDIR)/zstd-nolegacy
|
||||
test-zstd-nolegacy: zstd-nolegacy zstd-playTests
|
||||
test-zstd-nolegacy: zstd-nolegacy
|
||||
|
||||
test-zstd test-zstd32 test-zstd-nolegacy: datagen
|
||||
file $(ZSTD)
|
||||
ZSTD="$(QEMU_SYS) $(ZSTD)" ./playTests.sh $(ZSTDRTTEST)
|
||||
|
||||
|
||||
test-gzstd: gzstd
|
||||
$(PRGDIR)/zstd -f README.md test-zstd-speed.py
|
||||
|
|
|
|||
|
|
@ -30,6 +30,7 @@
|
|||
#include "zstd.h" /* ZSTD_versionString */
|
||||
#include "util.h" /* time functions */
|
||||
#include "datagen.h"
|
||||
#include "bench.h" /* CustomBench*/
|
||||
|
||||
|
||||
/*_************************************
|
||||
|
|
@ -45,6 +46,8 @@
|
|||
#define KNUTH 2654435761U
|
||||
#define MAX_MEM (1984 MB)
|
||||
|
||||
#define DEFAULT_CLEVEL 1
|
||||
|
||||
#define COMPRESSIBILITY_DEFAULT 0.50
|
||||
static const size_t g_sampleSize = 10000000;
|
||||
|
||||
|
|
@ -89,18 +92,63 @@ static size_t BMK_findMaxMem(U64 requiredMem)
|
|||
return (size_t) requiredMem;
|
||||
}
|
||||
|
||||
/*_*******************************************************
|
||||
* Argument Parsing
|
||||
*********************************************************/
|
||||
|
||||
#define ERROR_OUT(msg) { DISPLAY("%s \n", msg); exit(1); }
|
||||
|
||||
static unsigned readU32FromChar(const char** stringPtr)
|
||||
{
|
||||
const char errorMsg[] = "error: numeric value too large";
|
||||
unsigned result = 0;
|
||||
while ((**stringPtr >='0') && (**stringPtr <='9')) {
|
||||
unsigned const max = (((unsigned)(-1)) / 10) - 1;
|
||||
if (result > max) ERROR_OUT(errorMsg);
|
||||
result *= 10, result += **stringPtr - '0', (*stringPtr)++ ;
|
||||
}
|
||||
if ((**stringPtr=='K') || (**stringPtr=='M')) {
|
||||
unsigned const maxK = ((unsigned)(-1)) >> 10;
|
||||
if (result > maxK) ERROR_OUT(errorMsg);
|
||||
result <<= 10;
|
||||
if (**stringPtr=='M') {
|
||||
if (result > maxK) ERROR_OUT(errorMsg);
|
||||
result <<= 10;
|
||||
}
|
||||
(*stringPtr)++; /* skip `K` or `M` */
|
||||
if (**stringPtr=='i') (*stringPtr)++;
|
||||
if (**stringPtr=='B') (*stringPtr)++;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
static unsigned longCommandWArg(const char** stringPtr, const char* longCommand)
|
||||
{
|
||||
size_t const comSize = strlen(longCommand);
|
||||
int const result = !strncmp(*stringPtr, longCommand, comSize);
|
||||
if (result) *stringPtr += comSize;
|
||||
return result;
|
||||
}
|
||||
|
||||
/*_*******************************************************
|
||||
* Benchmark wrappers
|
||||
*********************************************************/
|
||||
size_t local_ZSTD_compress(void* dst, size_t dstSize, void* buff2, const void* src, size_t srcSize)
|
||||
|
||||
|
||||
static ZSTD_CCtx* g_zcc = NULL;
|
||||
|
||||
size_t local_ZSTD_compress(const void* src, size_t srcSize, void* dst, size_t dstSize, void* buff2)
|
||||
{
|
||||
(void)buff2;
|
||||
return ZSTD_compress(dst, dstSize, src, srcSize, 1);
|
||||
ZSTD_parameters p;
|
||||
ZSTD_frameParameters f = {1 /* contentSizeHeader*/, 0, 0};
|
||||
p.fParams = f;
|
||||
p.cParams = *(ZSTD_compressionParameters*)buff2;
|
||||
return ZSTD_compress_advanced (g_zcc,dst, dstSize, src, srcSize, NULL ,0, p);
|
||||
//return ZSTD_compress(dst, dstSize, src, srcSize, cLevel);
|
||||
}
|
||||
|
||||
static size_t g_cSize = 0;
|
||||
size_t local_ZSTD_decompress(void* dst, size_t dstSize, void* buff2, const void* src, size_t srcSize)
|
||||
size_t local_ZSTD_decompress(const void* src, size_t srcSize, void* dst, size_t dstSize, void* buff2)
|
||||
{
|
||||
(void)src; (void)srcSize;
|
||||
return ZSTD_decompress(dst, dstSize, buff2, g_cSize);
|
||||
|
|
@ -110,14 +158,14 @@ static ZSTD_DCtx* g_zdc = NULL;
|
|||
|
||||
#ifndef ZSTD_DLL_IMPORT
|
||||
extern size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* ctx, const void* src, size_t srcSize);
|
||||
size_t local_ZSTD_decodeLiteralsBlock(void* dst, size_t dstSize, void* buff2, const void* src, size_t srcSize)
|
||||
size_t local_ZSTD_decodeLiteralsBlock(const void* src, size_t srcSize, void* dst, size_t dstSize, void* buff2)
|
||||
{
|
||||
(void)src; (void)srcSize; (void)dst; (void)dstSize;
|
||||
return ZSTD_decodeLiteralsBlock((ZSTD_DCtx*)g_zdc, buff2, g_cSize);
|
||||
}
|
||||
|
||||
extern size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeq, const void* src, size_t srcSize);
|
||||
size_t local_ZSTD_decodeSeqHeaders(void* dst, size_t dstSize, void* buff2, const void* src, size_t srcSize)
|
||||
size_t local_ZSTD_decodeSeqHeaders(const void* src, size_t srcSize, void* dst, size_t dstSize, void* buff2)
|
||||
{
|
||||
int nbSeq;
|
||||
(void)src; (void)srcSize; (void)dst; (void)dstSize;
|
||||
|
|
@ -126,12 +174,15 @@ size_t local_ZSTD_decodeSeqHeaders(void* dst, size_t dstSize, void* buff2, const
|
|||
#endif
|
||||
|
||||
static ZSTD_CStream* g_cstream= NULL;
|
||||
size_t local_ZSTD_compressStream(void* dst, size_t dstCapacity, void* buff2, const void* src, size_t srcSize)
|
||||
size_t local_ZSTD_compressStream(const void* src, size_t srcSize, void* dst, size_t dstCapacity, void* buff2)
|
||||
{
|
||||
ZSTD_outBuffer buffOut;
|
||||
ZSTD_inBuffer buffIn;
|
||||
(void)buff2;
|
||||
ZSTD_initCStream(g_cstream, 1);
|
||||
ZSTD_parameters p;
|
||||
ZSTD_frameParameters f = {1 /* contentSizeHeader*/, 0, 0};
|
||||
p.fParams = f;
|
||||
p.cParams = *(ZSTD_compressionParameters*)buff2;
|
||||
ZSTD_initCStream_advanced(g_cstream, NULL, 0, p, ZSTD_CONTENTSIZE_UNKNOWN);
|
||||
buffOut.dst = dst;
|
||||
buffOut.size = dstCapacity;
|
||||
buffOut.pos = 0;
|
||||
|
|
@ -143,12 +194,11 @@ size_t local_ZSTD_compressStream(void* dst, size_t dstCapacity, void* buff2, con
|
|||
return buffOut.pos;
|
||||
}
|
||||
|
||||
static size_t local_ZSTD_compress_generic_end(void* dst, size_t dstCapacity, void* buff2, const void* src, size_t srcSize)
|
||||
static size_t local_ZSTD_compress_generic_end(const void* src, size_t srcSize, void* dst, size_t dstCapacity, void* buff2)
|
||||
{
|
||||
ZSTD_outBuffer buffOut;
|
||||
ZSTD_inBuffer buffIn;
|
||||
(void)buff2;
|
||||
ZSTD_CCtx_setParameter(g_cstream, ZSTD_p_compressionLevel, 1);
|
||||
buffOut.dst = dst;
|
||||
buffOut.size = dstCapacity;
|
||||
buffOut.pos = 0;
|
||||
|
|
@ -159,12 +209,11 @@ static size_t local_ZSTD_compress_generic_end(void* dst, size_t dstCapacity, voi
|
|||
return buffOut.pos;
|
||||
}
|
||||
|
||||
static size_t local_ZSTD_compress_generic_continue(void* dst, size_t dstCapacity, void* buff2, const void* src, size_t srcSize)
|
||||
static size_t local_ZSTD_compress_generic_continue(const void* src, size_t srcSize, void* dst, size_t dstCapacity, void* buff2)
|
||||
{
|
||||
ZSTD_outBuffer buffOut;
|
||||
ZSTD_inBuffer buffIn;
|
||||
(void)buff2;
|
||||
ZSTD_CCtx_setParameter(g_cstream, ZSTD_p_compressionLevel, 1);
|
||||
buffOut.dst = dst;
|
||||
buffOut.size = dstCapacity;
|
||||
buffOut.pos = 0;
|
||||
|
|
@ -176,12 +225,11 @@ static size_t local_ZSTD_compress_generic_continue(void* dst, size_t dstCapacity
|
|||
return buffOut.pos;
|
||||
}
|
||||
|
||||
static size_t local_ZSTD_compress_generic_T2_end(void* dst, size_t dstCapacity, void* buff2, const void* src, size_t srcSize)
|
||||
static size_t local_ZSTD_compress_generic_T2_end(const void* src, size_t srcSize, void* dst, size_t dstCapacity, void* buff2)
|
||||
{
|
||||
ZSTD_outBuffer buffOut;
|
||||
ZSTD_inBuffer buffIn;
|
||||
(void)buff2;
|
||||
ZSTD_CCtx_setParameter(g_cstream, ZSTD_p_compressionLevel, 1);
|
||||
ZSTD_CCtx_setParameter(g_cstream, ZSTD_p_nbWorkers, 2);
|
||||
buffOut.dst = dst;
|
||||
buffOut.size = dstCapacity;
|
||||
|
|
@ -193,12 +241,11 @@ static size_t local_ZSTD_compress_generic_T2_end(void* dst, size_t dstCapacity,
|
|||
return buffOut.pos;
|
||||
}
|
||||
|
||||
static size_t local_ZSTD_compress_generic_T2_continue(void* dst, size_t dstCapacity, void* buff2, const void* src, size_t srcSize)
|
||||
static size_t local_ZSTD_compress_generic_T2_continue(const void* src, size_t srcSize, void* dst, size_t dstCapacity, void* buff2)
|
||||
{
|
||||
ZSTD_outBuffer buffOut;
|
||||
ZSTD_inBuffer buffIn;
|
||||
(void)buff2;
|
||||
ZSTD_CCtx_setParameter(g_cstream, ZSTD_p_compressionLevel, 1);
|
||||
ZSTD_CCtx_setParameter(g_cstream, ZSTD_p_nbWorkers, 2);
|
||||
buffOut.dst = dst;
|
||||
buffOut.size = dstCapacity;
|
||||
|
|
@ -212,7 +259,7 @@ static size_t local_ZSTD_compress_generic_T2_continue(void* dst, size_t dstCapac
|
|||
}
|
||||
|
||||
static ZSTD_DStream* g_dstream= NULL;
|
||||
static size_t local_ZSTD_decompressStream(void* dst, size_t dstCapacity, void* buff2, const void* src, size_t srcSize)
|
||||
static size_t local_ZSTD_decompressStream(const void* src, size_t srcSize, void* dst, size_t dstCapacity, void* buff2)
|
||||
{
|
||||
ZSTD_outBuffer buffOut;
|
||||
ZSTD_inBuffer buffIn;
|
||||
|
|
@ -228,24 +275,28 @@ static size_t local_ZSTD_decompressStream(void* dst, size_t dstCapacity, void* b
|
|||
return buffOut.pos;
|
||||
}
|
||||
|
||||
static ZSTD_CCtx* g_zcc = NULL;
|
||||
|
||||
#ifndef ZSTD_DLL_IMPORT
|
||||
size_t local_ZSTD_compressContinue(void* dst, size_t dstCapacity, void* buff2, const void* src, size_t srcSize)
|
||||
size_t local_ZSTD_compressContinue(const void* src, size_t srcSize, void* dst, size_t dstCapacity, void* buff2)
|
||||
{
|
||||
(void)buff2;
|
||||
ZSTD_compressBegin(g_zcc, 1 /* compressionLevel */);
|
||||
ZSTD_parameters p;
|
||||
ZSTD_frameParameters f = {1 /* contentSizeHeader*/, 0, 0};
|
||||
p.fParams = f;
|
||||
p.cParams = *(ZSTD_compressionParameters*)buff2;
|
||||
ZSTD_compressBegin_advanced(g_zcc, NULL, 0, p, srcSize);
|
||||
return ZSTD_compressEnd(g_zcc, dst, dstCapacity, src, srcSize);
|
||||
}
|
||||
|
||||
#define FIRST_BLOCK_SIZE 8
|
||||
size_t local_ZSTD_compressContinue_extDict(void* dst, size_t dstCapacity, void* buff2, const void* src, size_t srcSize)
|
||||
size_t local_ZSTD_compressContinue_extDict(const void* src, size_t srcSize, void* dst, size_t dstCapacity, void* buff2)
|
||||
{
|
||||
BYTE firstBlockBuf[FIRST_BLOCK_SIZE];
|
||||
|
||||
(void)buff2;
|
||||
|
||||
ZSTD_parameters p;
|
||||
ZSTD_frameParameters f = {1 , 0, 0};
|
||||
p.fParams = f;
|
||||
p.cParams = *(ZSTD_compressionParameters*)buff2;
|
||||
ZSTD_compressBegin_advanced(g_zcc, NULL, 0, p, srcSize);
|
||||
memcpy(firstBlockBuf, src, FIRST_BLOCK_SIZE);
|
||||
ZSTD_compressBegin(g_zcc, 1);
|
||||
|
||||
{ size_t const compressResult = ZSTD_compressContinue(g_zcc, dst, dstCapacity, firstBlockBuf, FIRST_BLOCK_SIZE);
|
||||
if (ZSTD_isError(compressResult)) { DISPLAY("local_ZSTD_compressContinue_extDict error : %s\n", ZSTD_getErrorName(compressResult)); return compressResult; }
|
||||
|
|
@ -255,7 +306,7 @@ size_t local_ZSTD_compressContinue_extDict(void* dst, size_t dstCapacity, void*
|
|||
return ZSTD_compressEnd(g_zcc, dst, dstCapacity, (const BYTE*)src + FIRST_BLOCK_SIZE, srcSize - FIRST_BLOCK_SIZE);
|
||||
}
|
||||
|
||||
size_t local_ZSTD_decompressContinue(void* dst, size_t dstCapacity, void* buff2, const void* src, size_t srcSize)
|
||||
size_t local_ZSTD_decompressContinue(const void* src, size_t srcSize, void* dst, size_t dstCapacity, void* buff2)
|
||||
{
|
||||
size_t regeneratedSize = 0;
|
||||
const BYTE* ip = (const BYTE*)buff2;
|
||||
|
|
@ -282,27 +333,28 @@ size_t local_ZSTD_decompressContinue(void* dst, size_t dstCapacity, void* buff2,
|
|||
/*_*******************************************************
|
||||
* Bench functions
|
||||
*********************************************************/
|
||||
static size_t benchMem(const void* src, size_t srcSize, U32 benchNb)
|
||||
static size_t benchMem(const void* src, size_t srcSize, U32 benchNb, int cLevel, ZSTD_compressionParameters* cparams)
|
||||
{
|
||||
BYTE* dstBuff;
|
||||
size_t const dstBuffSize = ZSTD_compressBound(srcSize);
|
||||
void* buff2;
|
||||
void* buff2, *buff1;
|
||||
const char* benchName;
|
||||
size_t (*benchFunction)(void* dst, size_t dstSize, void* verifBuff, const void* src, size_t srcSize);
|
||||
double bestTime = 100000000.;
|
||||
BMK_benchFn_t benchFunction;
|
||||
BMK_customReturn_t r;
|
||||
int errorcode = 0;
|
||||
|
||||
/* Selection */
|
||||
switch(benchNb)
|
||||
{
|
||||
case 1:
|
||||
benchFunction = local_ZSTD_compress; benchName = "compress(1)";
|
||||
benchFunction = local_ZSTD_compress; benchName = "compress";
|
||||
break;
|
||||
case 2:
|
||||
benchFunction = local_ZSTD_decompress; benchName = "decompress";
|
||||
break;
|
||||
#ifndef ZSTD_DLL_IMPORT
|
||||
case 11:
|
||||
benchFunction = local_ZSTD_compressContinue; benchName = "compressContinue(1)";
|
||||
benchFunction = local_ZSTD_compressContinue; benchName = "compressContinue";
|
||||
break;
|
||||
case 12:
|
||||
benchFunction = local_ZSTD_compressContinue_extDict; benchName = "compressContinue_extDict";
|
||||
|
|
@ -318,7 +370,7 @@ static size_t benchMem(const void* src, size_t srcSize, U32 benchNb)
|
|||
break;
|
||||
#endif
|
||||
case 41:
|
||||
benchFunction = local_ZSTD_compressStream; benchName = "compressStream(1)";
|
||||
benchFunction = local_ZSTD_compressStream; benchName = "compressStream";
|
||||
break;
|
||||
case 42:
|
||||
benchFunction = local_ZSTD_decompressStream; benchName = "decompressStream";
|
||||
|
|
@ -347,26 +399,59 @@ static size_t benchMem(const void* src, size_t srcSize, U32 benchNb)
|
|||
free(dstBuff); free(buff2);
|
||||
return 12;
|
||||
}
|
||||
buff1 = buff2;
|
||||
if (g_zcc==NULL) g_zcc = ZSTD_createCCtx();
|
||||
if (g_zdc==NULL) g_zdc = ZSTD_createDCtx();
|
||||
if (g_cstream==NULL) g_cstream = ZSTD_createCStream();
|
||||
if (g_dstream==NULL) g_dstream = ZSTD_createDStream();
|
||||
|
||||
/* DISPLAY("params: cLevel %d, wlog %d hlog %d clog %d slog %d slen %d tlen %d strat %d \n"
|
||||
, cLevel, cparams->windowLog, cparams->hashLog, cparams->chainLog, cparams->searchLog,
|
||||
cparams->searchLength, cparams->targetLength, cparams->strategy);*/
|
||||
|
||||
ZSTD_CCtx_setParameter(g_zcc, ZSTD_p_compressionLevel, cLevel);
|
||||
ZSTD_CCtx_setParameter(g_zcc, ZSTD_p_windowLog, cparams->windowLog);
|
||||
ZSTD_CCtx_setParameter(g_zcc, ZSTD_p_hashLog, cparams->hashLog);
|
||||
ZSTD_CCtx_setParameter(g_zcc, ZSTD_p_chainLog, cparams->chainLog);
|
||||
ZSTD_CCtx_setParameter(g_zcc, ZSTD_p_searchLog, cparams->searchLog);
|
||||
ZSTD_CCtx_setParameter(g_zcc, ZSTD_p_minMatch, cparams->searchLength);
|
||||
ZSTD_CCtx_setParameter(g_zcc, ZSTD_p_targetLength, cparams->targetLength);
|
||||
ZSTD_CCtx_setParameter(g_zcc, ZSTD_p_compressionStrategy, cparams->strategy);
|
||||
|
||||
|
||||
ZSTD_CCtx_setParameter(g_cstream, ZSTD_p_compressionLevel, cLevel);
|
||||
ZSTD_CCtx_setParameter(g_cstream, ZSTD_p_windowLog, cparams->windowLog);
|
||||
ZSTD_CCtx_setParameter(g_cstream, ZSTD_p_hashLog, cparams->hashLog);
|
||||
ZSTD_CCtx_setParameter(g_cstream, ZSTD_p_chainLog, cparams->chainLog);
|
||||
ZSTD_CCtx_setParameter(g_cstream, ZSTD_p_searchLog, cparams->searchLog);
|
||||
ZSTD_CCtx_setParameter(g_cstream, ZSTD_p_minMatch, cparams->searchLength);
|
||||
ZSTD_CCtx_setParameter(g_cstream, ZSTD_p_targetLength, cparams->targetLength);
|
||||
ZSTD_CCtx_setParameter(g_cstream, ZSTD_p_compressionStrategy, cparams->strategy);
|
||||
|
||||
/* Preparation */
|
||||
switch(benchNb)
|
||||
{
|
||||
case 1:
|
||||
buff2 = (void*)cparams;
|
||||
break;
|
||||
case 2:
|
||||
g_cSize = ZSTD_compress(buff2, dstBuffSize, src, srcSize, 1);
|
||||
g_cSize = ZSTD_compress(buff2, dstBuffSize, src, srcSize, cLevel);
|
||||
break;
|
||||
#ifndef ZSTD_DLL_IMPORT
|
||||
case 11:
|
||||
buff2 = (void*)cparams;
|
||||
break;
|
||||
case 12:
|
||||
buff2 = (void*)cparams;
|
||||
break;
|
||||
case 13 :
|
||||
g_cSize = ZSTD_compress(buff2, dstBuffSize, src, srcSize, 1);
|
||||
g_cSize = ZSTD_compress(buff2, dstBuffSize, src, srcSize, cLevel);
|
||||
break;
|
||||
case 31: /* ZSTD_decodeLiteralsBlock */
|
||||
{ blockProperties_t bp;
|
||||
ZSTD_frameHeader zfp;
|
||||
size_t frameHeaderSize, skippedSize;
|
||||
g_cSize = ZSTD_compress(dstBuff, dstBuffSize, src, srcSize, 1);
|
||||
g_cSize = ZSTD_compress(dstBuff, dstBuffSize, src, srcSize, cLevel);
|
||||
frameHeaderSize = ZSTD_getFrameHeader(&zfp, dstBuff, ZSTD_frameHeaderSize_min);
|
||||
if (frameHeaderSize==0) frameHeaderSize = ZSTD_frameHeaderSize_min;
|
||||
ZSTD_getcBlockSize(dstBuff+frameHeaderSize, dstBuffSize, &bp); /* Get 1st block type */
|
||||
|
|
@ -386,8 +471,8 @@ static size_t benchMem(const void* src, size_t srcSize, U32 benchNb)
|
|||
const BYTE* ip = dstBuff;
|
||||
const BYTE* iend;
|
||||
size_t frameHeaderSize, cBlockSize;
|
||||
ZSTD_compress(dstBuff, dstBuffSize, src, srcSize, 1); /* it would be better to use direct block compression here */
|
||||
g_cSize = ZSTD_compress(dstBuff, dstBuffSize, src, srcSize, 1);
|
||||
ZSTD_compress(dstBuff, dstBuffSize, src, srcSize, cLevel); /* it would be better to use direct block compression here */
|
||||
g_cSize = ZSTD_compress(dstBuff, dstBuffSize, src, srcSize, cLevel);
|
||||
frameHeaderSize = ZSTD_getFrameHeader(&zfp, dstBuff, ZSTD_frameHeaderSize_min);
|
||||
if (frameHeaderSize==0) frameHeaderSize = ZSTD_frameHeaderSize_min;
|
||||
ip += frameHeaderSize; /* Skip frame Header */
|
||||
|
|
@ -409,8 +494,11 @@ static size_t benchMem(const void* src, size_t srcSize, U32 benchNb)
|
|||
case 31:
|
||||
goto _cleanOut;
|
||||
#endif
|
||||
case 41 :
|
||||
buff2 = (void*)cparams;
|
||||
break;
|
||||
case 42 :
|
||||
g_cSize = ZSTD_compress(buff2, dstBuffSize, src, srcSize, 1);
|
||||
g_cSize = ZSTD_compress(buff2, dstBuffSize, src, srcSize, cLevel);
|
||||
break;
|
||||
|
||||
/* test functions */
|
||||
|
|
@ -419,58 +507,37 @@ static size_t benchMem(const void* src, size_t srcSize, U32 benchNb)
|
|||
default : ;
|
||||
}
|
||||
|
||||
|
||||
/* warming up memory */
|
||||
{ size_t i; for (i=0; i<dstBuffSize; i++) dstBuff[i]=(BYTE)i; }
|
||||
|
||||
|
||||
/* benchmark loop */
|
||||
{ U32 loopNb;
|
||||
U32 nbRounds = (U32)((50 MB) / (srcSize+1)) + 1; /* initial conservative speed estimate */
|
||||
# define TIME_SEC_MICROSEC (1*1000000ULL) /* 1 second */
|
||||
# define TIME_SEC_NANOSEC (1*1000000000ULL) /* 1 second */
|
||||
DISPLAY("%2i- %-30.30s : \r", benchNb, benchName);
|
||||
for (loopNb = 1; loopNb <= g_nbIterations; loopNb++) {
|
||||
UTIL_time_t clockStart;
|
||||
size_t benchResult=0;
|
||||
U32 roundNb;
|
||||
|
||||
UTIL_sleepMilli(5); /* give processor time to other processes */
|
||||
UTIL_waitForNextTick();
|
||||
clockStart = UTIL_getTime();
|
||||
for (roundNb=0; roundNb < nbRounds; roundNb++) {
|
||||
benchResult = benchFunction(dstBuff, dstBuffSize, buff2, src, srcSize);
|
||||
if (ZSTD_isError(benchResult)) {
|
||||
DISPLAY("ERROR ! %s() => %s !! \n", benchName, ZSTD_getErrorName(benchResult));
|
||||
exit(1);
|
||||
} }
|
||||
{ U64 const clockSpanNano = UTIL_clockSpanNano(clockStart);
|
||||
double const averageTime = (double)clockSpanNano / TIME_SEC_NANOSEC / nbRounds;
|
||||
if (clockSpanNano > 0) {
|
||||
if (averageTime < bestTime) bestTime = averageTime;
|
||||
assert(bestTime > (1./2000000000));
|
||||
nbRounds = (U32)(1. / bestTime); /* aim for 1 sec */
|
||||
DISPLAY("%2i- %-30.30s : %7.1f MB/s (%9u)\r",
|
||||
loopNb, benchName,
|
||||
(double)srcSize / (1 MB) / bestTime,
|
||||
(U32)benchResult);
|
||||
} else {
|
||||
assert(nbRounds < 40000000); /* avoid overflow */
|
||||
nbRounds *= 100;
|
||||
}
|
||||
} } }
|
||||
DISPLAY("%2u\n", benchNb);
|
||||
{
|
||||
r = BMK_benchFunction(benchFunction, buff2,
|
||||
NULL, NULL, 1, &src, &srcSize,
|
||||
(void * const * const)&dstBuff, &dstBuffSize, g_nbIterations);
|
||||
if(r.error) {
|
||||
DISPLAY("ERROR %d ! ! \n", r.error);
|
||||
errorcode = r.error;
|
||||
goto _cleanOut;
|
||||
}
|
||||
|
||||
DISPLAY("%2u#Speed: %f MB/s - Size: %f MB - %s\n", benchNb, (double)srcSize / r.result.nanoSecPerRun * 1000, (double)r.result.sumOfReturn / 1000000, benchName);
|
||||
}
|
||||
|
||||
_cleanOut:
|
||||
free(buff1);
|
||||
free(dstBuff);
|
||||
free(buff2);
|
||||
ZSTD_freeCCtx(g_zcc); g_zcc=NULL;
|
||||
ZSTD_freeDCtx(g_zdc); g_zdc=NULL;
|
||||
ZSTD_freeCStream(g_cstream); g_cstream=NULL;
|
||||
ZSTD_freeDStream(g_dstream); g_dstream=NULL;
|
||||
return 0;
|
||||
return errorcode;
|
||||
}
|
||||
|
||||
|
||||
static int benchSample(U32 benchNb)
|
||||
static int benchSample(U32 benchNb, int cLevel, ZSTD_compressionParameters* cparams)
|
||||
{
|
||||
size_t const benchedSize = g_sampleSize;
|
||||
const char* name = "Sample 10MiB";
|
||||
|
|
@ -486,16 +553,16 @@ static int benchSample(U32 benchNb)
|
|||
DISPLAY("\r%79s\r", "");
|
||||
DISPLAY(" %s : \n", name);
|
||||
if (benchNb)
|
||||
benchMem(origBuff, benchedSize, benchNb);
|
||||
benchMem(origBuff, benchedSize, benchNb, cLevel, cparams);
|
||||
else
|
||||
for (benchNb=0; benchNb<100; benchNb++) benchMem(origBuff, benchedSize, benchNb);
|
||||
for (benchNb=0; benchNb<100; benchNb++) benchMem(origBuff, benchedSize, benchNb, cLevel, cparams);
|
||||
|
||||
free(origBuff);
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
static int benchFiles(const char** fileNamesTable, const int nbFiles, U32 benchNb)
|
||||
static int benchFiles(const char** fileNamesTable, const int nbFiles, U32 benchNb, int cLevel, ZSTD_compressionParameters* cparams)
|
||||
{
|
||||
/* Loop for each file */
|
||||
int fileIdx;
|
||||
|
|
@ -540,9 +607,9 @@ static int benchFiles(const char** fileNamesTable, const int nbFiles, U32 benchN
|
|||
DISPLAY("\r%79s\r", "");
|
||||
DISPLAY(" %s : \n", inFileName);
|
||||
if (benchNb)
|
||||
benchMem(origBuff, benchedSize, benchNb);
|
||||
benchMem(origBuff, benchedSize, benchNb, cLevel, cparams);
|
||||
else
|
||||
for (benchNb=0; benchNb<100; benchNb++) benchMem(origBuff, benchedSize, benchNb);
|
||||
for (benchNb=0; benchNb<100; benchNb++) benchMem(origBuff, benchedSize, benchNb, cLevel, cparams);
|
||||
|
||||
free(origBuff);
|
||||
}
|
||||
|
|
@ -567,6 +634,8 @@ static int usage_advanced(const char* exename)
|
|||
DISPLAY( " -b# : test only function # \n");
|
||||
DISPLAY( " -i# : iteration loops [1-9](default : %i)\n", NBLOOPS);
|
||||
DISPLAY( " -P# : sample compressibility (default : %.1f%%)\n", COMPRESSIBILITY_DEFAULT * 100);
|
||||
DISPLAY( " -l# : benchmark functions at that compression level (default : %i)\n", DEFAULT_CLEVEL);
|
||||
DISPLAY( " --zstd : custom parameter selection. Format same as zstdcli \n");
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
|
@ -583,6 +652,8 @@ int main(int argc, const char** argv)
|
|||
const char* exename = argv[0];
|
||||
const char* input_filename = NULL;
|
||||
U32 benchNb = 0, main_pause = 0;
|
||||
int cLevel = DEFAULT_CLEVEL;
|
||||
ZSTD_compressionParameters cparams = ZSTD_getCParams(cLevel, 0, 0);
|
||||
|
||||
DISPLAY(WELCOME_MESSAGE);
|
||||
if (argc<1) return badusage(exename);
|
||||
|
|
@ -591,11 +662,29 @@ int main(int argc, const char** argv)
|
|||
const char* argument = argv[i];
|
||||
assert(argument != NULL);
|
||||
|
||||
/* Commands (note : aggregated commands are allowed) */
|
||||
if (argument[0]=='-') {
|
||||
if (longCommandWArg(&argument, "--zstd=")) {
|
||||
for ( ; ;) {
|
||||
if (longCommandWArg(&argument, "windowLog=") || longCommandWArg(&argument, "wlog=")) { cparams.windowLog = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; }
|
||||
if (longCommandWArg(&argument, "chainLog=") || longCommandWArg(&argument, "clog=")) { cparams.chainLog = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; }
|
||||
if (longCommandWArg(&argument, "hashLog=") || longCommandWArg(&argument, "hlog=")) { cparams.hashLog = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; }
|
||||
if (longCommandWArg(&argument, "searchLog=") || longCommandWArg(&argument, "slog=")) { cparams.searchLog = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; }
|
||||
if (longCommandWArg(&argument, "searchLength=") || longCommandWArg(&argument, "slen=")) { cparams.searchLength = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; }
|
||||
if (longCommandWArg(&argument, "targetLength=") || longCommandWArg(&argument, "tlen=")) { cparams.targetLength = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; }
|
||||
if (longCommandWArg(&argument, "strategy=") || longCommandWArg(&argument, "strat=")) { cparams.strategy = (ZSTD_strategy)(readU32FromChar(&argument)); if (argument[0]==',') { argument++; continue; } else break; }
|
||||
if (longCommandWArg(&argument, "level=") || longCommandWArg(&argument, "lvl=")) { cLevel = (int)readU32FromChar(&argument); cparams = ZSTD_getCParams(cLevel, 0, 0); if (argument[0]==',') { argument++; continue; } else break; }
|
||||
DISPLAY("invalid compression parameter \n");
|
||||
return 1;
|
||||
}
|
||||
|
||||
while (argument[1]!=0) {
|
||||
argument++;
|
||||
if (argument[0] != 0) {
|
||||
DISPLAY("invalid --zstd= format\n");
|
||||
return 1; // check the end of string
|
||||
} else {
|
||||
continue;
|
||||
}
|
||||
} else if (argument[0]=='-') { /* Commands (note : aggregated commands are allowed) */
|
||||
argument++;
|
||||
while (argument[0]!=0) {
|
||||
|
||||
switch(argument[0])
|
||||
{
|
||||
|
|
@ -608,34 +697,34 @@ int main(int argc, const char** argv)
|
|||
|
||||
/* Select specific algorithm to bench */
|
||||
case 'b':
|
||||
benchNb = 0;
|
||||
while ((argument[1]>= '0') && (argument[1]<= '9')) {
|
||||
benchNb *= 10;
|
||||
benchNb += argument[1] - '0';
|
||||
{
|
||||
argument++;
|
||||
benchNb = readU32FromChar(&argument);
|
||||
break;
|
||||
}
|
||||
break;
|
||||
|
||||
/* Modify Nb Iterations */
|
||||
case 'i':
|
||||
if ((argument[1] >='0') && (argument[1] <='9')) {
|
||||
int iters = argument[1] - '0';
|
||||
BMK_SetNbIterations(iters);
|
||||
{
|
||||
argument++;
|
||||
BMK_SetNbIterations((int)readU32FromChar(&argument));
|
||||
}
|
||||
break;
|
||||
|
||||
/* Select compressibility of synthetic sample */
|
||||
case 'P':
|
||||
{ U32 proba32 = 0;
|
||||
while ((argument[1]>= '0') && (argument[1]<= '9')) {
|
||||
proba32 *= 10;
|
||||
proba32 += argument[1] - '0';
|
||||
argument++;
|
||||
}
|
||||
g_compressibility = (double)proba32 / 100.;
|
||||
{ argument++;
|
||||
g_compressibility = (double)readU32FromChar(&argument) / 100.;
|
||||
}
|
||||
break;
|
||||
case 'l':
|
||||
{ argument++;
|
||||
cLevel = readU32FromChar(&argument);
|
||||
cparams = ZSTD_getCParams(cLevel, 0, 0);
|
||||
}
|
||||
break;
|
||||
|
||||
|
||||
|
||||
/* Unknown command */
|
||||
default : return badusage(exename);
|
||||
|
|
@ -648,10 +737,12 @@ int main(int argc, const char** argv)
|
|||
if (!input_filename) { input_filename=argument; filenamesStart=i; continue; }
|
||||
}
|
||||
|
||||
|
||||
|
||||
if (filenamesStart==0) /* no input file */
|
||||
result = benchSample(benchNb);
|
||||
result = benchSample(benchNb, cLevel, &cparams);
|
||||
else
|
||||
result = benchFiles(argv+filenamesStart, argc-filenamesStart, benchNb);
|
||||
result = benchFiles(argv+filenamesStart, argc-filenamesStart, benchNb, cLevel, &cparams);
|
||||
|
||||
if (main_pause) { int unused; printf("press enter...\n"); unused = getchar(); (void)unused; }
|
||||
|
||||
|
|
|
|||
|
|
@ -162,18 +162,15 @@ const char* g_stratName[ZSTD_btultra+1] = {
|
|||
"ZSTD_btlazy2 ", "ZSTD_btopt ", "ZSTD_btultra "};
|
||||
|
||||
/* TODO: support additional parameters (more files, fileSizes) */
|
||||
|
||||
//TODO: benchMem dctx can't = NULL in new system
|
||||
static size_t
|
||||
BMK_benchParam(BMK_result_t* resultPtr,
|
||||
const void* srcBuffer, size_t srcSize,
|
||||
ZSTD_CCtx* ctx, ZSTD_DCtx* dctx,
|
||||
const ZSTD_compressionParameters cParams) {
|
||||
|
||||
|
||||
BMK_return_t res = BMK_benchMem(srcBuffer,srcSize, &srcSize, 1, 0, &cParams, NULL, 0, ctx, dctx, 0, "File");
|
||||
*resultPtr = res.result;
|
||||
return res.errorCode;
|
||||
return res.error;
|
||||
}
|
||||
|
||||
static void BMK_printWinner(FILE* f, U32 cLevel, BMK_result_t result, ZSTD_compressionParameters params, size_t srcSize)
|
||||
|
|
|
|||
|
|
@ -48,6 +48,8 @@ fileRoundTripTest() {
|
|||
$DIFF -q tmp.md5.1 tmp.md5.2
|
||||
}
|
||||
|
||||
UNAME=$(uname)
|
||||
|
||||
isTerminal=false
|
||||
if [ -t 0 ] && [ -t 1 ]
|
||||
then
|
||||
|
|
@ -56,7 +58,10 @@ fi
|
|||
|
||||
isWindows=false
|
||||
INTOVOID="/dev/null"
|
||||
DEVDEVICE="/dev/random"
|
||||
case "$UNAME" in
|
||||
GNU) DEVDEVICE="/dev/random" ;;
|
||||
*) DEVDEVICE="/dev/zero" ;;
|
||||
esac
|
||||
case "$OS" in
|
||||
Windows*)
|
||||
isWindows=true
|
||||
|
|
@ -65,7 +70,6 @@ case "$OS" in
|
|||
;;
|
||||
esac
|
||||
|
||||
UNAME=$(uname)
|
||||
case "$UNAME" in
|
||||
Darwin) MD5SUM="md5 -r" ;;
|
||||
FreeBSD) MD5SUM="gmd5sum" ;;
|
||||
|
|
@ -408,7 +412,7 @@ $ECHO "\n===> cover dictionary builder : advanced options "
|
|||
TESTFILE=../programs/zstdcli.c
|
||||
./datagen > tmpDict
|
||||
$ECHO "- Create first dictionary"
|
||||
$ZSTD --train-cover=k=46,d=8 *.c ../programs/*.c -o tmpDict
|
||||
$ZSTD --train-cover=k=46,d=8,split=80 *.c ../programs/*.c -o tmpDict
|
||||
cp $TESTFILE tmp
|
||||
$ZSTD -f tmp -D tmpDict
|
||||
$ZSTD -d tmp.zst -D tmpDict -fo result
|
||||
|
|
@ -422,6 +426,11 @@ cmp tmpDict tmpDict1 && die "dictionaries should have different ID !"
|
|||
$ECHO "- Create dictionary with size limit"
|
||||
$ZSTD --train-cover=steps=8 *.c ../programs/*.c -o tmpDict2 --maxdict=4K
|
||||
rm tmp*
|
||||
$ECHO "- Compare size of dictionary from 90% training samples with 80% training samples"
|
||||
$ZSTD --train-cover=split=90 -r *.c ../programs/*.c
|
||||
$ZSTD --train-cover=split=80 -r *.c ../programs/*.c
|
||||
$ECHO "- Create dictionary using all samples for both training and testing"
|
||||
$ZSTD --train-cover=split=100 -r *.c ../programs/*.c
|
||||
|
||||
$ECHO "\n===> legacy dictionary builder "
|
||||
|
||||
|
|
@ -544,16 +553,16 @@ $ZSTD --format=xz -V || LZMAMODE=0
|
|||
if [ $LZMAMODE -eq 1 ]; then
|
||||
$ECHO "xz support detected"
|
||||
XZEXE=1
|
||||
xz -V && lzma -V || XZEXE=0
|
||||
xz -Q -V && lzma -Q -V || XZEXE=0
|
||||
if [ $XZEXE -eq 1 ]; then
|
||||
$ECHO "Testing zstd xz and lzma support"
|
||||
./datagen > tmp
|
||||
$ZSTD --format=lzma -f tmp
|
||||
$ZSTD --format=xz -f tmp
|
||||
xz -t -v tmp.xz
|
||||
xz -t -v tmp.lzma
|
||||
xz -f -k tmp
|
||||
lzma -f -k --lzma1 tmp
|
||||
xz -Q -t -v tmp.xz
|
||||
xz -Q -t -v tmp.lzma
|
||||
xz -Q -f -k tmp
|
||||
lzma -Q -f -k --lzma1 tmp
|
||||
$ZSTD -d -f -v tmp.xz
|
||||
$ZSTD -d -f -v tmp.lzma
|
||||
rm tmp*
|
||||
|
|
@ -565,13 +574,13 @@ if [ $LZMAMODE -eq 1 ]; then
|
|||
$ECHO "Testing xz and lzma symlinks"
|
||||
./datagen > tmp
|
||||
./xz tmp
|
||||
xz -d tmp.xz
|
||||
xz -Q -d tmp.xz
|
||||
./lzma tmp
|
||||
lzma -d tmp.lzma
|
||||
lzma -Q -d tmp.lzma
|
||||
$ECHO "Testing unxz and unlzma symlinks"
|
||||
xz tmp
|
||||
xz -Q tmp
|
||||
./xz -d tmp.xz
|
||||
lzma tmp
|
||||
lzma -Q tmp
|
||||
./lzma -d tmp.lzma
|
||||
rm xz unxz lzma unlzma
|
||||
rm tmp*
|
||||
|
|
@ -731,8 +740,14 @@ $ECHO "\n===> zstd --list/-l error detection tests "
|
|||
! $ZSTD -lv tmp1*
|
||||
! $ZSTD --list -v tmp2 tmp12.zst
|
||||
|
||||
$ECHO "\n===> zstd --list/-l exits 1 when stdin is piped in"
|
||||
! echo "piped STDIN" | $ZSTD --list
|
||||
$ECHO "\n===> zstd --list/-l errors when presented with stdin / no files"
|
||||
! $ZSTD -l
|
||||
! $ZSTD -l -
|
||||
! $ZSTD -l < tmp1.zst
|
||||
! $ZSTD -l - < tmp1.zst
|
||||
! $ZSTD -l - tmp1.zst
|
||||
! $ZSTD -l - tmp1.zst < tmp1.zst
|
||||
$ZSTD -l tmp1.zst < tmp1.zst # but doesn't error just because stdin is not a tty
|
||||
|
||||
$ECHO "\n===> zstd --list/-l test with null files "
|
||||
./datagen -g0 > tmp5
|
||||
|
|
|
|||
Loading…
Reference in New Issue