facebook/zstd
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'refs/remotes/facebook/dev' into v112

Browse Source 
# Conflicts:
#	appveyor.yml
This commit is contained in:
Przemyslaw Skibinski 2016-12-19 08:20:26 +01:00
commit f8046b8e72
26 changed files with 520 additions and 77 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
.travis.yml
View File

@ -12,7 +12,7 @@ matrix:
os: linux
sudo: false
- env: Ubu=12.04cont Cmd="make zlibwrapper && make clean && make -C tests test-zstd-nolegacy && make clean && make cmaketest && make clean && make -C contrib/pzstd googletest pzstd tests check && make -C contrib/pzstd clean"
- env: Ubu=12.04cont Cmd="make zlibwrapper && make clean && make -C tests test-symbols && make clean && make -C tests test-zstd-nolegacy && make clean && make cmaketest && make clean && make -C contrib/pzstd googletest pzstd tests check && make -C contrib/pzstd clean"
os: linux
sudo: false
language: cpp
@ -69,7 +69,7 @@ matrix:
# Ubuntu 14.04 LTS Server Edition 64 bit
- env: Ubu=14.04 Cmd="make armtest && make clean && make aarch64test"
- env: Ubu=14.04 Cmd="make armtest"
dist: trusty
sudo: required
addons:
@ -79,6 +79,16 @@ matrix:
- qemu-user-static
- gcc-arm-linux-gnueabi
- libc6-dev-armel-cross
# Ubuntu 14.04 LTS Server Edition 64 bit
- env: Ubu=14.04 Cmd="make aarch64test"
dist: trusty
sudo: required
addons:
apt:
packages:
- qemu-system-arm
- qemu-user-static
- gcc-aarch64-linux-gnu
- libc6-dev-arm64-cross
@ -91,6 +101,7 @@ matrix:
- qemu-system-ppc
- qemu-user-static
- gcc-powerpc-linux-gnu
- libc6-dev-armel-cross
- env: Ubu=14.04 Cmd='make -C lib all && CFLAGS="-O1 -g" make -C zlibWrapper valgrindTest && make -C tests valgrindTest'
os: linux
@ -101,7 +112,10 @@ matrix:
packages:
- valgrind
- env: Ubu=14.04 Cmd="make gpptest && make clean && make gnu90test && make clean && make c99test && make clean && make gnu99test && make clean && make clangtest && make clean && make -C contrib/pzstd googletest32 && make -C contrib/pzstd all32 && make -C contrib/pzstd check && make -C contrib/pzstd clean"
- env: Ubu=14.04 Cmd="make gpptest && make clean && make gnu90test && make clean
&& make c99test && make clean && make gnu99test && make clean
&& make clangtest && make clean && make -C contrib/pzstd googletest32
&& make -C contrib/pzstd all32 && make -C contrib/pzstd check && make -C contrib/pzstd clean"
os: linux
dist: trusty
sudo: required
@ -127,7 +141,7 @@ matrix:
- libc6-dev-i386
- gcc-multilib
- env: Ubu=14.04 Cmd="make gcc5test && make clean && make gcc6test"
- env: Ubu=14.04 Cmd="make gcc5test && make clean && make gcc6test && make clean && make -C tests dll"
os: linux
dist: trusty
sudo: required

4
NEWS
View File

@ -1,3 +1,7 @@
v1.1.3
cli : new : commands for advanced parameters, by Przemyslaw Skibinski
API : fix : all symbols properly exposed in libzstd, by Nick Terrell
v1.1.2
API : streaming : decompression : changed : automatic implicit reset when chain-decoding new frames without init
API : experimental : added : dictID retrieval functions, and ZSTD_initCStream_srcSize()

2
appveyor.yml
View File

@ -2,7 +2,7 @@ version: 1.0.{build}
environment:
matrix:
- COMPILER: "gcc"
MAKE_PARAMS: '"make test && make lib && make -C tests fullbench-dll fullbench-lib"'
MAKE_PARAMS: '"make test && make lib && make -C tests test-symbols fullbench-dll fullbench-lib"'
PLATFORM: "mingw64"
- COMPILER: "gcc"
MAKE_PARAMS: "make test"

4
build/VS2005/zstdlib/zstdlib.vcproj
View File

@ -359,6 +359,10 @@
RelativePath="..\..\..\lib\common\xxhash.c"
>
</File>
<File
RelativePath="..\..\..\lib\deprecated\zbuff_common.c"
>
</File>
<File
RelativePath="..\..\..\lib\deprecated\zbuff_compress.c"
>

1
build/VS2010/libzstd-dll/libzstd-dll.vcxproj
View File

@ -29,6 +29,7 @@
<ClCompile Include="..\..\..\lib\compress\zstd_compress.c" />
<ClCompile Include="..\..\..\lib\decompress\huf_decompress.c" />
<ClCompile Include="..\..\..\lib\decompress\zstd_decompress.c" />
<ClCompile Include="..\..\..\lib\deprecated\zbuff_common.c" />
<ClCompile Include="..\..\..\lib\deprecated\zbuff_compress.c" />
<ClCompile Include="..\..\..\lib\deprecated\zbuff_decompress.c" />
<ClCompile Include="..\..\..\lib\dictBuilder\divsufsort.c" />

1
build/VS2010/libzstd/libzstd.vcxproj
View File

@ -29,6 +29,7 @@
<ClCompile Include="..\..\..\lib\compress\zstd_compress.c" />
<ClCompile Include="..\..\..\lib\decompress\huf_decompress.c" />
<ClCompile Include="..\..\..\lib\decompress\zstd_decompress.c" />
<ClCompile Include="..\..\..\lib\deprecated\zbuff_common.c" />
<ClCompile Include="..\..\..\lib\deprecated\zbuff_compress.c" />
<ClCompile Include="..\..\..\lib\deprecated\zbuff_decompress.c" />
<ClCompile Include="..\..\..\lib\dictBuilder\divsufsort.c" />

1
build/cmake/lib/CMakeLists.txt
View File

@ -69,6 +69,7 @@ SET(Sources
${LIBRARY_DIR}/decompress/zstd_decompress.c
${LIBRARY_DIR}/dictBuilder/divsufsort.c
${LIBRARY_DIR}/dictBuilder/zdict.c
${LIBRARY_DIR}/deprecated/zbuff_common.c
${LIBRARY_DIR}/deprecated/zbuff_compress.c
${LIBRARY_DIR}/deprecated/zbuff_decompress.c)

4
lib/common/zstd_common.c
View File

@ -43,10 +43,6 @@ ZSTD_ErrorCode ZSTD_getErrorCode(size_t code) { return ERR_getErrorCode(code); }
* provides error code string from enum */
const char* ZSTD_getErrorString(ZSTD_ErrorCode code) { return ERR_getErrorName(code); }
/* --- ZBUFF Error Management (deprecated) --- */
unsigned ZBUFF_isError(size_t errorCode) { return ERR_isError(errorCode); }
const char* ZBUFF_getErrorName(size_t errorCode) { return ERR_getErrorName(errorCode); }
/*=**************************************************************
* Custom allocator

18
lib/common/zstd_errors.h
View File

@ -18,6 +18,20 @@ extern "C" {
#include <stddef.h> /* size_t */
/* ===== ZSTDERRORLIB_API : control library symbols visibility ===== */
#if defined(__GNUC__) && (__GNUC__ >= 4)
# define ZSTDERRORLIB_VISIBILITY __attribute__ ((visibility ("default")))
#else
# define ZSTDERRORLIB_VISIBILITY
#endif
#if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1)
# define ZSTDERRORLIB_API __declspec(dllexport) ZSTDERRORLIB_VISIBILITY
#elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1)
# define ZSTDERRORLIB_API __declspec(dllimport) ZSTDERRORLIB_VISIBILITY /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/
#else
# define ZSTDERRORLIB_API ZSTDERRORLIB_VISIBILITY
#endif
/*-****************************************
* error codes list
******************************************/
@ -49,8 +63,8 @@ typedef enum {
/*! ZSTD_getErrorCode() :
convert a `size_t` function result into a `ZSTD_ErrorCode` enum type,
which can be used to compare directly with enum list published into "error_public.h" */
ZSTD_ErrorCode ZSTD_getErrorCode(size_t functionResult);
const char* ZSTD_getErrorString(ZSTD_ErrorCode code);
ZSTDERRORLIB_API ZSTD_ErrorCode ZSTD_getErrorCode(size_t functionResult);
ZSTDERRORLIB_API const char* ZSTD_getErrorString(ZSTD_ErrorCode code);
#if defined (__cplusplus)

5
lib/compress/zstd_compress.c
View File

@ -2900,7 +2900,7 @@ size_t ZSTD_CStreamOutSize(void) { return ZSTD_compressBound(ZSTD_BLOCKSIZE_ABSO
size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize)
{
if (zcs->inBuffSize==0) return ERROR(stage_wrong); /* zcs has not been init at least once */
if (zcs->inBuffSize==0) return ERROR(stage_wrong); /* zcs has not been init at least once => can't reset */
if (zcs->cdict) CHECK_F(ZSTD_compressBegin_usingCDict(zcs->cctx, zcs->cdict, pledgedSrcSize))
else CHECK_F(ZSTD_compressBegin_advanced(zcs->cctx, NULL, 0, zcs->params, pledgedSrcSize));
@ -2967,7 +2967,8 @@ size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t di
size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize)
{
ZSTD_parameters const params = ZSTD_getParams(compressionLevel, pledgedSrcSize, 0);
ZSTD_parameters params = ZSTD_getParams(compressionLevel, pledgedSrcSize, 0);
if (pledgedSrcSize) params.fParams.contentSizeFlag = 1;
return ZSTD_initCStream_advanced(zcs, NULL, 0, params, pledgedSrcSize);
}

26
lib/deprecated/zbuff_common.c Normal file
View File

@ -0,0 +1,26 @@
/**
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under the BSD-style license found in the
* LICENSE file in the root directory of this source tree. An additional grant
* of patent rights can be found in the PATENTS file in the same directory.
*/
/*-*************************************
* Dependencies
***************************************/
#include "error_private.h"
#include "zbuff.h"
/*-****************************************
* ZBUFF Error Management (deprecated)
******************************************/
/*! ZBUFF_isError() :
* tells if a return value is an error code */
unsigned ZBUFF_isError(size_t errorCode) { return ERR_isError(errorCode); }
/*! ZBUFF_getErrorName() :
* provides error code string from function result (useful for debugging) */
const char* ZBUFF_getErrorName(size_t errorCode) { return ERR_getErrorName(errorCode); }

7
lib/dictBuilder/zdict.c
View File

@ -36,12 +36,11 @@
#include <time.h> /* clock */
#include "mem.h" /* read */
#include "error_private.h"
#include "fse.h" /* FSE_normalizeCount, FSE_writeNCount */
#define HUF_STATIC_LINKING_ONLY
#include "huf.h"
#include "huf.h" /* HUF_buildCTable, HUF_writeCTable */
#include "zstd_internal.h" /* includes zstd.h */
#include "xxhash.h"
#include "xxhash.h" /* XXH64 */
#include "divsufsort.h"
#ifndef ZDICT_STATIC_LINKING_ONLY
# define ZDICT_STATIC_LINKING_ONLY
@ -570,7 +569,7 @@ static void ZDICT_countEStats(EStats_ress_t esr, ZSTD_parameters params,
if (ZSTD_isError(errorCode)) { DISPLAYLEVEL(1, "warning : ZSTD_copyCCtx failed \n"); return; }
}
cSize = ZSTD_compressBlock(esr.zc, esr.workPlace, ZSTD_BLOCKSIZE_ABSOLUTEMAX, src, srcSize);
if (ZSTD_isError(cSize)) { DISPLAYLEVEL(1, "warning : could not compress sample size %u \n", (U32)srcSize); return; }
if (ZSTD_isError(cSize)) { DISPLAYLEVEL(3, "warning : could not compress sample size %u \n", (U32)srcSize); return; }
if (cSize) { /* if == 0; block is not compressible */
const seqStore_t* seqStorePtr = ZSTD_getSeqStore(esr.zc);

23
lib/dictBuilder/zdict.h
View File

@ -19,15 +19,18 @@ extern "C" {
#include <stddef.h> /* size_t */
/*====== Export for Windows ======*/
/*!
* ZSTD_DLL_EXPORT :
* Enable exporting of functions when building a Windows DLL
*/
#if defined(_WIN32) && defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1)
# define ZDICTLIB_API __declspec(dllexport)
/* ===== ZDICTLIB_API : control library symbols visibility ===== */
#if defined(__GNUC__) && (__GNUC__ >= 4)
# define ZDICTLIB_VISIBILITY __attribute__ ((visibility ("default")))
#else
# define ZDICTLIB_API
# define ZDICTLIB_VISIBILITY
#endif
#if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1)
# define ZDICTLIB_API __declspec(dllexport) ZDICTLIB_VISIBILITY
#elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1)
# define ZDICTLIB_API __declspec(dllimport) ZDICTLIB_VISIBILITY /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/
#else
# define ZDICTLIB_API ZDICTLIB_VISIBILITY
#endif
@ -79,7 +82,7 @@ typedef struct {
or an error code, which can be tested by ZDICT_isError().
note : ZDICT_trainFromBuffer_advanced() will send notifications into stderr if instructed to, using notificationLevel>0.
*/
size_t ZDICT_trainFromBuffer_advanced(void* dictBuffer, size_t dictBufferCapacity,
ZDICTLIB_API size_t ZDICT_trainFromBuffer_advanced(void* dictBuffer, size_t dictBufferCapacity,
const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples,
ZDICT_params_t parameters);
@ -97,7 +100,7 @@ size_t ZDICT_trainFromBuffer_advanced(void* dictBuffer, size_t dictBufferCapacit
starting from its beginning.
@return : size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`).
*/
size_t ZDICT_addEntropyTablesFromBuffer(void* dictBuffer, size_t dictContentSize, size_t dictBufferCapacity,
ZDICTLIB_API size_t ZDICT_addEntropyTablesFromBuffer(void* dictBuffer, size_t dictContentSize, size_t dictBufferCapacity,
const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples);

8
lib/zstd.h
View File

@ -56,7 +56,7 @@ extern "C" {
/*------ Version ------*/
#define ZSTD_VERSION_MAJOR 1
#define ZSTD_VERSION_MINOR 1
#define ZSTD_VERSION_RELEASE 2
#define ZSTD_VERSION_RELEASE 3
#define ZSTD_LIB_VERSION ZSTD_VERSION_MAJOR.ZSTD_VERSION_MINOR.ZSTD_VERSION_RELEASE
#define ZSTD_QUOTE(str) #str
@ -466,13 +466,13 @@ ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
* Provides the dictID stored within dictionary.
* if @return == 0, the dictionary is not conformant with Zstandard specification.
* It can still be loaded, but as a content-only dictionary. */
unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize);
ZSTDLIB_API unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize);
/*! ZSTD_getDictID_fromDDict() :
* Provides the dictID of the dictionary loaded into `ddict`.
* If @return == 0, the dictionary is not conformant to Zstandard specification, or empty.
* Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */
unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict);
ZSTDLIB_API unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict);
/*! ZSTD_getDictID_fromFrame() :
* Provides the dictID required to decompressed the frame stored within `src`.
@ -484,7 +484,7 @@ unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict);
* - `srcSize` is too small, and as a result, the frame header could not be decoded (only possible if `srcSize < ZSTD_FRAMEHEADERSIZE_MAX`).
* - This is not a Zstandard frame.
* When identifying the exact failure cause, it's possible to used ZSTD_getFrameParams(), which will provide a more precise error code. */
unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize);
ZSTDLIB_API unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize);
/********************************************************************

39
programs/bench.c
View File

@ -126,7 +126,8 @@ typedef struct {
static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
const char* displayName, int cLevel,
const size_t* fileSizes, U32 nbFiles,
const void* dictBuffer, size_t dictBufferSize)
const void* dictBuffer, size_t dictBufferSize,
ZSTD_compressionParameters *comprParams)
{
size_t const blockSize = ((g_blockSize>=32 && !g_decodeOnly) ? g_blockSize : srcSize) + (!srcSize) /* avoid div by 0 */ ;
size_t const avgSize = MIN(g_blockSize, (srcSize / nbFiles));
@ -232,12 +233,19 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
UTIL_getTime(&clockStart);
if (!cCompleted) { /* still some time to do compression tests */
ZSTD_parameters const zparams = ZSTD_getParams(cLevel, avgSize, dictBufferSize);
ZSTD_parameters zparams = ZSTD_getParams(cLevel, avgSize, dictBufferSize);
ZSTD_customMem const cmem = { NULL, NULL, NULL };
U64 clockLoop = g_nbSeconds ? TIMELOOP_MICROSEC : 1;
U32 nbLoops = 0;
ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictBuffer, dictBufferSize, zparams, cmem);
if (cdict==NULL) EXM_THROW(1, "ZSTD_createCDict_advanced() allocation failure");
if (comprParams->windowLog) zparams.cParams.windowLog = comprParams->windowLog;
if (comprParams->chainLog) zparams.cParams.chainLog = comprParams->chainLog;
if (comprParams->hashLog) zparams.cParams.hashLog = comprParams->hashLog;
if (comprParams->searchLog) zparams.cParams.searchLog = comprParams->searchLog;
if (comprParams->searchLength) zparams.cParams.searchLength = comprParams->searchLength;
if (comprParams->targetLength) zparams.cParams.targetLength = comprParams->targetLength;
if (comprParams->strategy) zparams.cParams.strategy = (ZSTD_strategy)(comprParams->strategy - 1);
do {
U32 blockNb;
size_t rSize;
@ -248,9 +256,9 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
blockTable[blockNb].srcPtr,blockTable[blockNb].srcSize,
cdict);
} else {
rSize = ZSTD_compressCCtx (ctx,
rSize = ZSTD_compress_advanced (ctx,
blockTable[blockNb].cPtr, blockTable[blockNb].cRoom,
blockTable[blockNb].srcPtr,blockTable[blockNb].srcSize, cLevel);
blockTable[blockNb].srcPtr,blockTable[blockNb].srcSize, NULL, 0, zparams);
}
if (ZSTD_isError(rSize)) EXM_THROW(1, "ZSTD_compress_usingCDict() failed : %s", ZSTD_getErrorName(rSize));
blockTable[blockNb].cSize = rSize;
@ -388,7 +396,8 @@ static size_t BMK_findMaxMem(U64 requiredMem)
static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize,
const char* displayName, int cLevel, int cLevelLast,
const size_t* fileSizes, unsigned nbFiles,
const void* dictBuffer, size_t dictBufferSize)
const void* dictBuffer, size_t dictBufferSize,
ZSTD_compressionParameters *compressionParams)
{
int l;
@ -407,7 +416,7 @@ static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize,
BMK_benchMem(srcBuffer, benchedSize,
displayName, l,
fileSizes, nbFiles,
dictBuffer, dictBufferSize);
dictBuffer, dictBufferSize, compressionParams);
}
}
@ -444,8 +453,8 @@ static void BMK_loadFiles(void* buffer, size_t bufferSize,
if (totalSize == 0) EXM_THROW(12, "no data to bench");
}
static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles,
const char* dictFileName, int cLevel, int cLevelLast)
static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles, const char* dictFileName,
int cLevel, int cLevelLast, ZSTD_compressionParameters *compressionParams)
{
void* srcBuffer;
size_t benchedSize;
@ -484,7 +493,7 @@ static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles,
BMK_benchCLevel(srcBuffer, benchedSize,
displayName, cLevel, cLevelLast,
fileSizes, nbFiles,
dictBuffer, dictBufferSize);
dictBuffer, dictBufferSize, compressionParams);
}
/* clean up */
@ -494,7 +503,7 @@ static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles,
}
static void BMK_syntheticTest(int cLevel, int cLevelLast, double compressibility)
static void BMK_syntheticTest(int cLevel, int cLevelLast, double compressibility, ZSTD_compressionParameters* compressionParams)
{
char name[20] = {0};
size_t benchedSize = 10000000;
@ -508,15 +517,15 @@ static void BMK_syntheticTest(int cLevel, int cLevelLast, double compressibility
/* Bench */
snprintf (name, sizeof(name), "Synthetic %2u%%", (unsigned)(compressibility*100));
BMK_benchCLevel(srcBuffer, benchedSize, name, cLevel, cLevelLast, &benchedSize, 1, NULL, 0);
BMK_benchCLevel(srcBuffer, benchedSize, name, cLevel, cLevelLast, &benchedSize, 1, NULL, 0, compressionParams);
/* clean up */
free(srcBuffer);
}
int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles,
const char* dictFileName, int cLevel, int cLevelLast)
int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles, const char* dictFileName,
int cLevel, int cLevelLast, ZSTD_compressionParameters* compressionParams)
{
double const compressibility = (double)g_compressibilityDefault / 100;
@ -526,8 +535,8 @@ int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles,
if (cLevelLast > cLevel) DISPLAYLEVEL(2, "Benchmarking levels from %d to %d\n", cLevel, cLevelLast);
if (nbFiles == 0)
BMK_syntheticTest(cLevel, cLevelLast, compressibility);
BMK_syntheticTest(cLevel, cLevelLast, compressibility, compressionParams);
else
BMK_benchFileTable(fileNamesTable, nbFiles, dictFileName, cLevel, cLevelLast);
BMK_benchFileTable(fileNamesTable, nbFiles, dictFileName, cLevel, cLevelLast, compressionParams);
return 0;
}

6
programs/bench.h
View File

@ -12,9 +12,11 @@
#define BENCH_H_121279284357
#include <stddef.h> /* size_t */
#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_compressionParameters */
#include "zstd.h" /* ZSTD_compressionParameters */
int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles,
const char* dictFileName, int cLevel, int cLevelLast);
int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles,const char* dictFileName,
int cLevel, int cLevelLast, ZSTD_compressionParameters* compressionParams);
/* Set Parameters */
void BMK_SetNbSeconds(unsigned nbLoops);

31
programs/dibio.c
View File

@ -32,7 +32,8 @@
#define MB *(1 <<20)
#define GB *(1U<<30)
#define MEMMULT 11
#define SAMPLESIZE_MAX (128 KB)
#define MEMMULT 11 /* rough estimation : memory cost to analyze 1 byte of sample */
static const size_t maxMemory = (sizeof(size_t) == 4) ? (2 GB - 64 MB) : ((size_t)(512 MB) << sizeof(size_t));
#define NOISELENGTH 32
@ -98,7 +99,7 @@ static unsigned DiB_loadFiles(void* buffer, size_t* bufferSizePtr,
for (n=0; n<nbFiles; n++) {
const char* const fileName = fileNamesTable[n];
unsigned long long const fs64 = UTIL_getFileSize(fileName);
size_t const fileSize = (size_t) MIN(fs64, 128 KB);
size_t const fileSize = (size_t) MIN(fs64, SAMPLESIZE_MAX);
if (fileSize > *bufferSizePtr-pos) break;
{ FILE* const f = fopen(fileName, "rb");
if (f==NULL) EXM_THROW(10, "zstd: dictBuilder: %s %s ", fileName, strerror(errno));
@ -164,6 +165,21 @@ static void DiB_saveDict(const char* dictFileName,
}
static int g_tooLargeSamples = 0;
static U64 DiB_getTotalCappedFileSize(const char** fileNamesTable, unsigned nbFiles)
{
U64 total = 0;
unsigned n;
for (n=0; n<nbFiles; n++) {
U64 const fileSize = UTIL_getFileSize(fileNamesTable[n]);
U64 const cappedFileSize = MIN(fileSize, SAMPLESIZE_MAX);
total += cappedFileSize;
g_tooLargeSamples |= (fileSize > 2*SAMPLESIZE_MAX);
}
return total;
}
/*! ZDICT_trainFromBuffer_unsafe() :
Strictly Internal use only !!
Same as ZDICT_trainFromBuffer_advanced(), but does not control `samplesBuffer`.
@ -182,16 +198,21 @@ int DiB_trainFromFiles(const char* dictFileName, unsigned maxDictSize,
{
void* const dictBuffer = malloc(maxDictSize);
size_t* const fileSizes = (size_t*)malloc(nbFiles * sizeof(size_t));
unsigned long long const totalSizeToLoad = UTIL_getTotalFileSize(fileNamesTable, nbFiles);
unsigned long long const totalSizeToLoad = DiB_getTotalCappedFileSize(fileNamesTable, nbFiles);
size_t const maxMem = DiB_findMaxMem(totalSizeToLoad * MEMMULT) / MEMMULT;
size_t benchedSize = MIN (maxMem, (size_t)totalSizeToLoad);
size_t benchedSize = (size_t) MIN ((unsigned long long)maxMem, totalSizeToLoad);
void* const srcBuffer = malloc(benchedSize+NOISELENGTH);
int result = 0;
/* Checks */
if ((!fileSizes) || (!srcBuffer) || (!dictBuffer)) EXM_THROW(12, "not enough memory for DiB_trainFiles"); /* should not happen */
g_displayLevel = params.notificationLevel;
if (nbFiles < 5) {
if (g_tooLargeSamples) {
DISPLAYLEVEL(2, "! Warning : some samples are very large \n");
DISPLAYLEVEL(2, "! Note that dictionary is only useful for small files or beginning of large files. \n");
DISPLAYLEVEL(2, "! As a consequence, only the first %u bytes of each file are loaded \n", SAMPLESIZE_MAX);
}
if ((nbFiles < 5) || (totalSizeToLoad < 9 * (unsigned long long)maxDictSize)) {
DISPLAYLEVEL(2, "! Warning : nb of samples too low for proper processing ! \n");
DISPLAYLEVEL(2, "! Please provide _one file per sample_. \n");
DISPLAYLEVEL(2, "! Do not concatenate samples together into a single file, \n");

19
programs/fileio.c
View File

@ -248,7 +248,8 @@ typedef struct {
FILE* srcFile;
} cRess_t;
static cRess_t FIO_createCResources(const char* dictFileName, int cLevel, U64 srcSize)
static cRess_t FIO_createCResources(const char* dictFileName, int cLevel,
U64 srcSize, ZSTD_compressionParameters* comprParams)
{
cRess_t ress;
memset(&ress, 0, sizeof(ress));
@ -269,6 +270,13 @@ static cRess_t FIO_createCResources(const char* dictFileName, int cLevel, U64 sr
params.fParams.contentSizeFlag = 1;
params.fParams.checksumFlag = g_checksumFlag;
params.fParams.noDictIDFlag = !g_dictIDFlag;
if (comprParams->windowLog) params.cParams.windowLog = comprParams->windowLog;
if (comprParams->chainLog) params.cParams.chainLog = comprParams->chainLog;
if (comprParams->hashLog) params.cParams.hashLog = comprParams->hashLog;
if (comprParams->searchLog) params.cParams.searchLog = comprParams->searchLog;
if (comprParams->searchLength) params.cParams.searchLength = comprParams->searchLength;
if (comprParams->targetLength) params.cParams.targetLength = comprParams->targetLength;
if (comprParams->strategy) params.cParams.strategy = (ZSTD_strategy)(comprParams->strategy - 1);
{ size_t const errorCode = ZSTD_initCStream_advanced(ress.cctx, dictBuffer, dictBuffSize, params, srcSize);
if (ZSTD_isError(errorCode)) EXM_THROW(33, "Error initializing CStream : %s", ZSTD_getErrorName(errorCode));
} }
@ -402,12 +410,12 @@ static int FIO_compressFilename_dstFile(cRess_t ress,
int FIO_compressFilename(const char* dstFileName, const char* srcFileName,
const char* dictFileName, int compressionLevel)
const char* dictFileName, int compressionLevel, ZSTD_compressionParameters* comprParams)
{
clock_t const start = clock();
U64 const srcSize = UTIL_getFileSize(srcFileName);
cRess_t const ress = FIO_createCResources(dictFileName, compressionLevel, srcSize);
cRess_t const ress = FIO_createCResources(dictFileName, compressionLevel, srcSize, comprParams);
int const result = FIO_compressFilename_dstFile(ress, dstFileName, srcFileName);
double const seconds = (double)(clock() - start) / CLOCKS_PER_SEC;
@ -420,14 +428,15 @@ int FIO_compressFilename(const char* dstFileName, const char* srcFileName,
int FIO_compressMultipleFilenames(const char** inFileNamesTable, unsigned nbFiles,
const char* suffix,
const char* dictFileName, int compressionLevel)
const char* dictFileName, int compressionLevel,
ZSTD_compressionParameters* comprParams)
{
int missed_files = 0;
size_t dfnSize = FNSPACE;
char* dstFileName = (char*)malloc(FNSPACE);
size_t const suffixSize = suffix ? strlen(suffix) : 0;
U64 const srcSize = (nbFiles != 1) ? 0 : UTIL_getFileSize(inFileNamesTable[0]) ;
cRess_t ress = FIO_createCResources(dictFileName, compressionLevel, srcSize);
cRess_t ress = FIO_createCResources(dictFileName, compressionLevel, srcSize, comprParams);
/* init */
if (dstFileName==NULL) EXM_THROW(27, "FIO_compressMultipleFilenames : allocation error for dstFileName");

9
programs/fileio.h
View File

@ -11,6 +11,9 @@
#ifndef FILEIO_H_23981798732
#define FILEIO_H_23981798732
#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_compressionParameters */
#include "zstd.h" /* ZSTD_compressionParameters */
#if defined (__cplusplus)
extern "C" {
#endif
@ -44,7 +47,8 @@ void FIO_setMemLimit(unsigned memLimit);
***************************************/
/** FIO_compressFilename() :
@return : 0 == ok; 1 == pb with src file. */
int FIO_compressFilename (const char* outfilename, const char* infilename, const char* dictFileName, int compressionLevel);
int FIO_compressFilename (const char* outfilename, const char* infilename, const char* dictFileName,
int compressionLevel, ZSTD_compressionParameters* comprParams);
/** FIO_decompressFilename() :
@return : 0 == ok; 1 == pb with src file. */
@ -58,7 +62,8 @@ int FIO_decompressFilename (const char* outfilename, const char* infilename, con
@return : nb of missing files */
int FIO_compressMultipleFilenames(const char** srcNamesTable, unsigned nbFiles,
const char* suffix,
const char* dictFileName, int compressionLevel);
const char* dictFileName, int compressionLevel,
ZSTD_compressionParameters* comprParams);
/** FIO_decompressMultipleFilenames() :
@return : nb of missing or skipped files */

109
programs/zstd.1
View File

@ -264,6 +264,115 @@ and weight typically 100x the target dictionary size (for example, 10 MB for a 1
cut file into independent blocks of size # (default: no block)
.SH ADVANCED COMPRESSION OPTIONS
.TP
.B \--zstd[=\fIoptions\fR]
.PD
\fBzstd\fR provides 22 predefined compression levels. The selected or default predefined compression level can be changed with advanced compression options.
The \fIoptions\fR are provided as a comma-separated list. You may specify only the \fIoptions\fR you want to change and the rest will be taken from the selected or default compression level.
The list of available \fIoptions\fR:
.RS
.TP
.BI strategy= strat
.PD 0
.TP
.BI strat= strat
.PD
Specify a strategy used by a match finder.
.IP ""
There are 8 strategies numbered from 0 to 7, from faster to stronger:
0=ZSTD_fast, 1=ZSTD_dfast, 2=ZSTD_greedy, 3=ZSTD_lazy, 4=ZSTD_lazy2, 5=ZSTD_btlazy2, 6=ZSTD_btopt, 7=ZSTD_btopt2.
.IP ""
.TP
.BI windowLog= wlog
.PD 0
.TP
.BI wlog= wlog
.PD
Specify the maximum number of bits for a match distance.
.IP ""
The higher number of bits increases the chance to find a match what usually improves compression ratio.
It also increases memory requirements for compressor and decompressor.
.IP ""
The minimum \fIwlog\fR is 10 (1 KiB) and the maximum is 25 (32 MiB) for 32-bit compilation and 27 (128 MiB) for 64-bit compilation.
.IP ""
.TP
.BI hashLog= hlog
.PD 0
.TP
.BI hlog= hlog
.PD
Specify the maximum number of bits for a hash table.
.IP ""
The bigger hash table causes less collisions what usually make compression faster but requires more memory during compression.
.IP ""
The minimum \fIhlog\fR is 6 (64 B) and the maximum is 25 (32 MiB) for 32-bit compilation and 27 (128 MiB) for 64-bit compilation.
.TP
.BI chainLog= clog
.PD 0
.TP
.BI clog= clog
.PD
Specify the maximum number of bits for a hash chain or a binary tree.
.IP ""
The higher number of bits increases the chance to find a match what usually improves compression ratio.
It also slows down compression speed and increases memory requirements for compression.
This option is ignored for the ZSTD_fast strategy.
.IP ""
The minimum \fIclog\fR is 6 (64 B) and the maximum is 26 (64 MiB) for 32-bit compilation and 28 (256 MiB) for 64-bit compilation.
.IP ""
.TP
.BI searchLog= slog
.PD 0
.TP
.BI slog= slog
.PD
Specify the maximum number of searches in a hash chain or a binary tree using logarithmic scale.
.IP ""
The bigger number of searches increases the chance to find a match what usually improves compression ratio but decreases compression speed.
.IP ""
The minimum \fIslog\fR is 1 and the maximum is 24 for 32-bit compilation and 26 for 64-bit compilation.
.IP ""
.TP
.BI searchLength= slen
.PD 0
.TP
.BI slen= slen
.PD
Specify the minimum searched length of a match in a hash table.
.IP ""
The bigger search length usually decreases compression ratio but improves decompression speed.
.IP ""
The minimum \fIslen\fR is 3 and the maximum is 7.
.IP ""
.TP
.BI targetLength= tlen
.PD 0
.TP
.BI tlen= tlen
.PD
Specify the minimum match length that causes a match finder to interrupt searching of better matches.
.IP ""
The bigger minimum match length usually improves compression ratio but decreases compression speed.
This option is used only with ZSTD_btopt and ZSTD_btopt2 strategies.
.IP ""
The minimum \fItlen\fR is 4 and the maximum is 999.
.IP ""
.PP
.B An example
.br
The following parameters sets advanced compression options to predefined level 19 for files bigger than 256 KB:
.IP ""
\fB--zstd=\fRwindowLog=23,chainLog=23,hashLog=22,searchLog=6,searchLength=3,targetLength=48,strategy=6
.SH BUGS
Report bugs at:- https://github.com/facebook/zstd/issues

35
programs/zstdcli.c
View File

@ -205,6 +205,32 @@ static unsigned longCommandWArg(const char** stringPtr, const char* longCommand)
return result;
}
/** parseCompressionParameters() :
* reads compression parameters from *stringPtr (e.g. "--zstd=wlog=23,clog=23,hlog=22,slog=6,slen=3,tlen=48,strat=6") into *params
* @return 1 means that compression parameters were correct
* @return 0 in case of malformed parameters
*/
static unsigned parseCompressionParameters(const char* stringPtr, ZSTD_compressionParameters* params)
{
for ( ; ;) {
if (longCommandWArg(&stringPtr, "windowLog=") || longCommandWArg(&stringPtr, "wlog=")) { params->windowLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
if (longCommandWArg(&stringPtr, "chainLog=") || longCommandWArg(&stringPtr, "clog=")) { params->chainLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
if (longCommandWArg(&stringPtr, "hashLog=") || longCommandWArg(&stringPtr, "hlog=")) { params->hashLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
if (longCommandWArg(&stringPtr, "searchLog=") || longCommandWArg(&stringPtr, "slog=")) { params->searchLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
if (longCommandWArg(&stringPtr, "searchLength=") || longCommandWArg(&stringPtr, "slen=")) { params->searchLength = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
if (longCommandWArg(&stringPtr, "targetLength=") || longCommandWArg(&stringPtr, "tlen=")) { params->targetLength = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
if (longCommandWArg(&stringPtr, "strategy=") || longCommandWArg(&stringPtr, "strat=")) { params->strategy = (ZSTD_strategy)(1 + readU32FromChar(&stringPtr)); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
return 0;
}
if (stringPtr[0] != 0) return 0; /* check the end of string */
DISPLAYLEVEL(4, "windowLog=%d\nchainLog=%d\nhashLog=%d\nsearchLog=%d\n", params->windowLog, params->chainLog, params->hashLog, params->searchLog);
DISPLAYLEVEL(4, "searchLength=%d\ntargetLength=%d\nstrategy=%d\n", params->searchLength, params->targetLength, params->strategy);
return 1;
}
typedef enum { zom_compress, zom_decompress, zom_test, zom_bench, zom_train } zstd_operation_mode;
#define CLEAN_RETURN(i) { operationResult = (i); goto _end; }
@ -223,6 +249,7 @@ int main(int argCount, const char* argv[])
ultra=0,
lastCommand = 0;
zstd_operation_mode operation = zom_compress;
ZSTD_compressionParameters compressionParams;
int cLevel = ZSTDCLI_CLEVEL_DEFAULT;
int cLevelLast = 1;
unsigned recursive = 0;
@ -259,6 +286,7 @@ int main(int argCount, const char* argv[])
/* preset behaviors */
if (!strcmp(programName, ZSTD_UNZSTD)) operation=zom_decompress;
if (!strcmp(programName, ZSTD_CAT)) { operation=zom_decompress; forceStdout=1; FIO_overwriteMode(); outFileName=stdoutmark; displayLevel=1; }
memset(&compressionParams, 0, sizeof(compressionParams));
/* command switches */
for (argNb=1; argNb<argCount; argNb++) {
@ -307,6 +335,7 @@ int main(int argCount, const char* argv[])
if (longCommandWArg(&argument, "--memlimit=")) { memLimit = readU32FromChar(&argument); continue; }
if (longCommandWArg(&argument, "--memory=")) { memLimit = readU32FromChar(&argument); continue; }
if (longCommandWArg(&argument, "--memlimit-decompress=")) { memLimit = readU32FromChar(&argument); continue; }
if (longCommandWArg(&argument, "--zstd=")) { if (!parseCompressionParameters(argument, &compressionParams)) CLEAN_RETURN(badusage(programName)); continue; }
/* fall-through, will trigger bad_usage() later on */
}
@ -497,7 +526,7 @@ int main(int argCount, const char* argv[])
if (operation==zom_bench) {
#ifndef ZSTD_NOBENCH
BMK_setNotificationLevel(displayLevel);
BMK_benchFiles(filenameTable, filenameIdx, dictFileName, cLevel, cLevelLast);
BMK_benchFiles(filenameTable, filenameIdx, dictFileName, cLevel, cLevelLast, &compressionParams);
#endif
goto _end;
}
@ -549,9 +578,9 @@ int main(int argCount, const char* argv[])
if (operation==zom_compress) {
#ifndef ZSTD_NOCOMPRESS
if ((filenameIdx==1) && outFileName)
operationResult = FIO_compressFilename(outFileName, filenameTable[0], dictFileName, cLevel);
operationResult = FIO_compressFilename(outFileName, filenameTable[0], dictFileName, cLevel, &compressionParams);
else
operationResult = FIO_compressMultipleFilenames(filenameTable, filenameIdx, outFileName ? outFileName : ZSTD_EXTENSION, dictFileName, cLevel);
operationResult = FIO_compressMultipleFilenames(filenameTable, filenameIdx, outFileName ? outFileName : ZSTD_EXTENSION, dictFileName, cLevel, &compressionParams);
#else
DISPLAY("Compression not supported\n");
#endif

3
tests/.gitignore vendored
View File

@ -3,15 +3,18 @@ fullbench
fullbench32
fuzzer
fuzzer32
fuzzer-dll
zbufftest
zbufftest32
zstreamtest
zstreamtest32
zstreamtest-dll
datagen
paramgrill
paramgrill32
roundTripCrash
longmatch
symbols
# Tmp test directory
zstdtest

39
tests/Makefile
View File

@ -28,7 +28,7 @@ PYTHON ?= python3
TESTARTEFACT := versionsTest namespaceTest
CPPFLAGS= -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/dictBuilder -I$(PRGDIR)
CPPFLAGS+= -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/dictBuilder -I$(ZSTDDIR)/deprecated -I$(PRGDIR)
CFLAGS ?= -O3
CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 \
-Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes -Wundef
@ -57,7 +57,7 @@ ZSTREAM_TESTTIME = -T2mn
FUZZERTEST= -T5mn
ZSTDRTTEST= --test-large-data
.PHONY: default all all32 clean test test32 test-all namespaceTest versionsTest
.PHONY: default all all32 dll clean test test32 test-all namespaceTest versionsTest
default: fullbench
@ -65,6 +65,8 @@ all: fullbench fuzzer zstreamtest paramgrill datagen zbufftest
all32: fullbench32 fuzzer32 zstreamtest32 zbufftest32
dll: fuzzer-dll zstreamtest-dll zbufftest-dll
zstd:
@ -99,6 +101,11 @@ fuzzer : $(ZSTD_FILES) $(ZDICT_FILES) $(PRGDIR)/datagen.c fuzzer.c
fuzzer32 : $(ZSTD_FILES) $(ZDICT_FILES) $(PRGDIR)/datagen.c fuzzer.c
$(CC) -m32 $(FLAGS) $^ -o $@$(EXT)
fuzzer-dll : LDFLAGS+= -L$(ZSTDDIR) -lzstd
fuzzer-dll : $(ZSTDDIR)/common/xxhash.c $(PRGDIR)/datagen.c fuzzer.c
$(MAKE) -C $(ZSTDDIR) libzstd
$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@$(EXT)
zbufftest : CPPFLAGS += -I$(ZSTDDIR)/deprecated
zbufftest : CFLAGS += -Wno-deprecated-declarations # required to silence deprecation warnings
zbufftest : $(ZSTD_FILES) $(ZBUFF_FILES) $(PRGDIR)/datagen.c zbufftest.c
@ -109,12 +116,24 @@ zbufftest32 : CFLAGS += -Wno-deprecated-declarations -m32
zbufftest32 : $(ZSTD_FILES) $(ZBUFF_FILES) $(PRGDIR)/datagen.c zbufftest.c
$(CC) $(FLAGS) $^ -o $@$(EXT)
zbufftest-dll : CPPFLAGS += -I$(ZSTDDIR)/deprecated
zbufftest-dll : CFLAGS += -Wno-deprecated-declarations # required to silence deprecation warnings
zbufftest-dll : LDFLAGS+= -L$(ZSTDDIR) -lzstd
zbufftest-dll : $(ZSTDDIR)/common/xxhash.c $(PRGDIR)/datagen.c zbufftest.c
$(MAKE) -C $(ZSTDDIR) libzstd
$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@$(EXT)
zstreamtest : $(ZSTD_FILES) $(PRGDIR)/datagen.c zstreamtest.c
$(CC) $(FLAGS) $^ -o $@$(EXT)
zstreamtest32 : $(ZSTD_FILES) $(PRGDIR)/datagen.c zstreamtest.c
$(CC) -m32 $(FLAGS) $^ -o $@$(EXT)
zstreamtest-dll : LDFLAGS+= -L$(ZSTDDIR) -lzstd
zstreamtest-dll : $(ZSTDDIR)/common/xxhash.c $(PRGDIR)/datagen.c zstreamtest.c
$(MAKE) -C $(ZSTDDIR) libzstd
$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@$(EXT)
paramgrill : $(ZSTD_FILES) $(PRGDIR)/datagen.c paramgrill.c
$(CC) $(FLAGS) $^ -lm -o $@$(EXT)
@ -127,6 +146,15 @@ roundTripCrash : $(ZSTD_FILES) roundTripCrash.c
longmatch : $(ZSTD_FILES) longmatch.c
$(CC) $(FLAGS) $^ -o $@$(EXT)
symbols : symbols.c
$(MAKE) -C $(ZSTDDIR) libzstd
ifneq (,$(filter Windows%,$(OS)))
cp $(ZSTDDIR)/dll/libzstd.dll .
$(CC) $(FLAGS) $^ -o $@$(EXT) -DZSTD_DLL_IMPORT=1 libzstd.dll
else
$(CC) $(FLAGS) $^ -o $@$(EXT) -Wl,-rpath=$(ZSTDDIR) $(ZSTDDIR)/libzstd.so
endif
namespaceTest:
if $(CC) namespaceTest.c ../lib/common/xxhash.c -o $@ ; then echo compilation should fail; exit 1 ; fi
$(RM) $@
@ -142,8 +170,10 @@ clean:
fullbench$(EXT) fullbench32$(EXT) \
fullbench-lib$(EXT) fullbench-dll$(EXT) \
fuzzer$(EXT) fuzzer32$(EXT) zbufftest$(EXT) zbufftest32$(EXT) \
fuzzer-dll$(EXT) zstreamtest-dll$(EXT) zbufftest-dll$(EXT)\
zstreamtest$(EXT) zstreamtest32$(EXT) \
datagen$(EXT) paramgrill$(EXT) roundTripCrash$(EXT) longmatch$(EXT)
datagen$(EXT) paramgrill$(EXT) roundTripCrash$(EXT) longmatch$(EXT) \
symbols$(EXT)
@echo Cleaning completed
@ -243,4 +273,7 @@ test-zstream32: zstreamtest32
test-longmatch: longmatch
$(QEMU_SYS) ./longmatch
test-symbols: symbols
$(QEMU_SYS) ./symbols
endif

13
tests/playTests.sh
View File

@ -27,7 +27,6 @@ case "$OS" in
Windows*)
isWindows=true
ECHO="echo -e"
INTOVOID="nul"
;;
esac
@ -101,6 +100,18 @@ $ZSTD -f tmp && die "tmp not present : should have failed"
ls tmp.zst && die "tmp.zst should not be created"
$ECHO "\n**** Advanced compression parameters **** "
$ECHO "Hello world!" | $ZSTD --zstd=windowLog=21, - -o tmp.zst && die "wrong parameters not detected!"
$ECHO "Hello world!" | $ZSTD --zstd=windowLo=21 - -o tmp.zst && die "wrong parameters not detected!"
$ECHO "Hello world!" | $ZSTD --zstd=windowLog=21,slog - -o tmp.zst && die "wrong parameters not detected!"
ls tmp.zst && die "tmp.zst should not be created"
roundTripTest -g512K
roundTripTest -g512K " --zstd=slen=3,tlen=48,strat=6"
roundTripTest -g512K " --zstd=strat=6,wlog=23,clog=23,hlog=22,slog=6"
roundTripTest -g512K " --zstd=windowLog=23,chainLog=23,hashLog=22,searchLog=6,searchLength=3,targetLength=48,strategy=6"
roundTripTest -g512K 19
$ECHO "\n**** Pass-Through mode **** "
$ECHO "Hello world 1!" | $ZSTD -df
$ECHO "Hello world 2!" | $ZSTD -dcf

144
tests/symbols.c Normal file
View File

@ -0,0 +1,144 @@
#include <stdio.h>
#include "zstd_errors.h"
#define ZSTD_STATIC_LINKING_ONLY
#include "zstd.h"
#define ZBUFF_DISABLE_DEPRECATE_WARNINGS
#define ZBUFF_STATIC_LINKING_ONLY
#include "zbuff.h"
#define ZDICT_STATIC_LINKING_ONLY
#include "zdict.h"
static const void *symbols[] = {
/* zstd.h */
&ZSTD_versionNumber,
&ZSTD_compress,
&ZSTD_decompress,
&ZSTD_getDecompressedSize,
&ZSTD_maxCLevel,
&ZSTD_compressBound,
&ZSTD_isError,
&ZSTD_getErrorName,
&ZSTD_createCCtx,
&ZSTD_freeCCtx,
&ZSTD_compressCCtx,
&ZSTD_createDCtx,
&ZSTD_freeDCtx,
&ZSTD_decompressDCtx,
&ZSTD_compress_usingDict,
&ZSTD_decompress_usingDict,
&ZSTD_createCDict,
&ZSTD_freeCDict,
&ZSTD_compress_usingCDict,
&ZSTD_createDDict,
&ZSTD_freeDDict,
&ZSTD_decompress_usingDDict,
&ZSTD_createCStream,
&ZSTD_freeCStream,
&ZSTD_initCStream,
&ZSTD_compressStream,
&ZSTD_flushStream,
&ZSTD_endStream,
&ZSTD_CStreamInSize,
&ZSTD_CStreamOutSize,
&ZSTD_createDStream,
&ZSTD_freeDStream,
&ZSTD_initDStream,
&ZSTD_decompressStream,
&ZSTD_DStreamInSize,
&ZSTD_DStreamOutSize,
/* zstd.h: advanced functions */
&ZSTD_estimateCCtxSize,
&ZSTD_createCCtx_advanced,
&ZSTD_sizeof_CCtx,
&ZSTD_createCDict_advanced,
&ZSTD_sizeof_CDict,
&ZSTD_getCParams,
&ZSTD_getParams,
&ZSTD_checkCParams,
&ZSTD_adjustCParams,
&ZSTD_compress_advanced,
&ZSTD_isFrame,
&ZSTD_estimateDCtxSize,
&ZSTD_createDCtx_advanced,
&ZSTD_sizeof_DCtx,
&ZSTD_sizeof_DDict,
&ZSTD_getDictID_fromDict,
&ZSTD_getDictID_fromDDict,
&ZSTD_getDictID_fromFrame,
&ZSTD_createCStream_advanced,
&ZSTD_initCStream_srcSize,
&ZSTD_initCStream_usingDict,
&ZSTD_initCStream_advanced,
&ZSTD_initCStream_usingCDict,
&ZSTD_resetCStream,
&ZSTD_sizeof_CStream,
&ZSTD_createDStream_advanced,
&ZSTD_initDStream_usingDict,
&ZSTD_setDStreamParameter,
&ZSTD_initDStream_usingDDict,
&ZSTD_resetDStream,
&ZSTD_sizeof_DStream,
&ZSTD_compressBegin,
&ZSTD_compressBegin_usingDict,
&ZSTD_compressBegin_advanced,
&ZSTD_copyCCtx,
&ZSTD_compressContinue,
&ZSTD_compressEnd,
&ZSTD_getFrameParams,
&ZSTD_decompressBegin,
&ZSTD_decompressBegin_usingDict,
&ZSTD_copyDCtx,
&ZSTD_nextSrcSizeToDecompress,
&ZSTD_decompressContinue,
&ZSTD_nextInputType,
&ZSTD_getBlockSizeMax,
&ZSTD_compressBlock,
&ZSTD_decompressBlock,
&ZSTD_insertBlock,
/* zstd_errors.h */
&ZSTD_getErrorCode,
&ZSTD_getErrorString,
/* zbuff.h */
&ZBUFF_createCCtx,
&ZBUFF_freeCCtx,
&ZBUFF_compressInit,
&ZBUFF_compressInitDictionary,
&ZBUFF_compressContinue,
&ZBUFF_compressFlush,
&ZBUFF_compressEnd,
&ZBUFF_createDCtx,
&ZBUFF_freeDCtx,
&ZBUFF_decompressInit,
&ZBUFF_decompressInitDictionary,
&ZBUFF_decompressContinue,
&ZBUFF_isError,
&ZBUFF_getErrorName,
&ZBUFF_recommendedCInSize,
&ZBUFF_recommendedCOutSize,
&ZBUFF_recommendedDInSize,
&ZBUFF_recommendedDOutSize,
/* zbuff.h: advanced functions */
&ZBUFF_createCCtx_advanced,
&ZBUFF_createDCtx_advanced,
&ZBUFF_compressInit_advanced,
/* zdict.h */
&ZDICT_trainFromBuffer,
&ZDICT_getDictID,
&ZDICT_isError,
&ZDICT_getErrorName,
/* zdict.h: advanced functions */
&ZDICT_trainFromBuffer_advanced,
&ZDICT_addEntropyTablesFromBuffer,
NULL,
};
int main(int argc, const char** argv) {
const void **symbol;
(void)argc;
(void)argv;
for (symbol = symbols; *symbol != NULL; ++symbol) {
printf("%p\n", *symbol);
}
return 0;
}

8
tests/zstreamtest.c
View File

@ -248,7 +248,7 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
/* _srcSize compression test */
DISPLAYLEVEL(4, "test%3i : compress_srcSize %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
ZSTD_initCStream_srcSize(zc, 1, CNBufferSize);
outBuff.dst = (char*)(compressedBuffer)+cSize;
outBuff.dst = (char*)(compressedBuffer);
outBuff.size = compressedBufferSize;
outBuff.pos = 0;
inBuff.src = CNBuffer;
@ -259,12 +259,16 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */
{ size_t const r = ZSTD_endStream(zc, &outBuff);
if (r != 0) goto _output_error; } /* error, or some data not flushed */
{ unsigned long long origSize = ZSTD_getDecompressedSize(outBuff.dst, outBuff.pos);
DISPLAY("outBuff.pos : %u \n", (U32)outBuff.pos);
DISPLAY("origSize = %u \n", (U32)origSize);
if ((size_t)origSize != CNBufferSize) goto _output_error; } /* exact original size must be present */
DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/COMPRESSIBLE_NOISE_LENGTH*100);
/* wrong _srcSize compression test */
DISPLAYLEVEL(4, "test%3i : wrong srcSize : %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH-1);
ZSTD_initCStream_srcSize(zc, 1, CNBufferSize-1);
outBuff.dst = (char*)(compressedBuffer)+cSize;
outBuff.dst = (char*)(compressedBuffer);
outBuff.size = compressedBufferSize;
outBuff.pos = 0;
inBuff.src = CNBuffer;