Merge remote-tracking branch 'refs/remotes/facebook/dev' into dev11
Przemyslaw Skibinski
commit
a875d3693a
12
.travis.yml
12
.travis.yml
|
|
@ -169,7 +169,13 @@ matrix:
|
|||
|
||||
script:
|
||||
- JOB_NUMBER=$(echo $TRAVIS_JOB_NUMBER | sed -e 's:[0-9][0-9]*\.\(.*\):\1:')
|
||||
|
||||
# cron & master => long tests, as this is the final step towards a Release
|
||||
# dev => normal tests; other feature branches => short tests (number > 11)
|
||||
- if [ "$TRAVIS_PULL_REQUEST" = "true" ] || [ $JOB_NUMBER -gt 11 ] || [ "$TRAVIS_BRANCH" = "dev" ] && [ "$TRAVIS_BRANCH" != "master" ]; then sh -c "$Cmd"; fi
|
||||
# master => long tests, as this is the final step towards a Release
|
||||
- if [ "$TRAVIS_BRANCH" = "master" ]; then FUZZERTEST=-T10mn sh -c "$Cmd"; fi
|
||||
- if [ "$TRAVIS_EVENT_TYPE" = "cron" ] || [ "$TRAVIS_BRANCH" = "master" ]; then
|
||||
FUZZERTEST=-T10mn sh -c "$Cmd" || travis_terminate 1;
|
||||
else
|
||||
if [ "$TRAVIS_PULL_REQUEST" = "true" ] || [ $JOB_NUMBER -gt 11 ] || [ "$TRAVIS_BRANCH" = "dev" ]; then
|
||||
sh -c "$Cmd" || travis_terminate 1;
|
||||
fi
|
||||
fi
|
||||
|
|
|
|||
|
|
@ -0,0 +1,57 @@
|
|||
dependencies:
|
||||
override:
|
||||
- sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test; sudo apt-get -y -qq update
|
||||
#- sudo apt-get -y install qemu-system-ppc qemu-user-static gcc-powerpc-linux-gnu valgrind
|
||||
#- sudo apt-get -y install qemu-system-arm gcc-arm-linux-gnueabi libc6-dev-armel-cross gcc-aarch64-linux-gnu libc6-dev-arm64-cross
|
||||
- sudo apt-get -y install libc6-dev-i386 clang gcc-5 gcc-6
|
||||
|
||||
# use default "parallel: true" for commands in the machine, checkout, dependencies and database build phase
|
||||
post:
|
||||
- |
|
||||
if [[ "$CIRCLE_NODE_INDEX" == "0" ]]; then make cmaketest && make clean; fi
|
||||
if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make -C tests test-invalidDictionaries && make clean; fi
|
||||
- |
|
||||
if [[ "$CIRCLE_NODE_INDEX" == "0" ]]; then g++ -v; make gpptest && make clean; fi
|
||||
if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make -C tests test-legacy && make clean; fi
|
||||
- |
|
||||
if [[ "$CIRCLE_NODE_INDEX" == "0" ]]; then gcc -v; make gnu90test && make clean; fi
|
||||
if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make -C tests test-symbols && make clean; fi
|
||||
- |
|
||||
if [[ "$CIRCLE_NODE_INDEX" == "0" ]]; then gcc -v; make c99test && make clean; fi
|
||||
if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make -C tests test-longmatch && make clean; fi
|
||||
- |
|
||||
if [[ "$CIRCLE_NODE_INDEX" == "0" ]]; then gcc -v; make gnu99test && make clean; fi
|
||||
if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make -C tests dll && make clean; fi
|
||||
- |
|
||||
if [[ "$CIRCLE_NODE_INDEX" == "0" ]]; then clang -v; make clangtest && make clean; fi
|
||||
if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make -C programs zstd-small zstd-decompress zstd-compress zstd32 MOREFLAGS="-I/usr/include/x86_64-linux-gnu" && make clean lib && make clean; fi
|
||||
- |
|
||||
if [[ "$CIRCLE_NODE_INDEX" == "0" ]]; then make travis-install && make clean; fi
|
||||
if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make -C tests test-fullbench && make clean; fi
|
||||
- |
|
||||
if [[ "$CIRCLE_NODE_INDEX" == "0" ]]; then gcc-5 -v; make gcc5test && gcc-6 -v && make gcc6test && make clean; fi
|
||||
if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make -C tests test-zstream && make clean; fi
|
||||
- |
|
||||
if [[ "$CIRCLE_NODE_INDEX" == "0" ]]; then make -C tests test-zstd && make clean; fi
|
||||
if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make -C tests test-fuzzer && make clean; fi
|
||||
|
||||
test:
|
||||
override:
|
||||
- echo Circle CI tests finished
|
||||
|
||||
# Longer tests
|
||||
#- make -C tests test-zstd-nolegacy && make clean
|
||||
#- pyenv global 3.4.4; make -C tests versionsTest && make clean
|
||||
#- make zlibwrapper && make clean
|
||||
#- gcc -v; make -C tests test32 MOREFLAGS="-I/usr/include/x86_64-linux-gnu" && make clean
|
||||
#- make uasan && make clean
|
||||
#- make asan32 && make clean
|
||||
#- make -C tests test32 CC=clang MOREFLAGS="-g -fsanitize=address -I/usr/include/x86_64-linux-gnu"
|
||||
# Valgrind tests
|
||||
#- CFLAGS="-O1 -g" make -C zlibWrapper valgrindTest && make clean
|
||||
#- make -C tests valgrindTest && make clean
|
||||
# ARM, AArch64, PowerPC, PowerPC64 tests
|
||||
#- make ppctest && make clean
|
||||
#- make ppc64test && make clean
|
||||
#- make armtest && make clean
|
||||
#- make aarch64test && make clean
|
||||
|
|
@ -13,6 +13,7 @@
|
|||
#include <string.h> // strerror
|
||||
#include <errno.h> // errno
|
||||
#include <sys/stat.h> // stat
|
||||
#define ZSTD_STATIC_LINKING_ONLY // ZSTD_findDecompressedSize
|
||||
#include <zstd.h> // presumes zstd library is installed
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -102,7 +102,7 @@ static void compress_orDie(const char* fname, const char* oname)
|
|||
}
|
||||
|
||||
|
||||
static const char* createOutFilename_orDie(const char* filename)
|
||||
static char* createOutFilename_orDie(const char* filename)
|
||||
{
|
||||
size_t const inL = strlen(filename);
|
||||
size_t const outL = inL + 5;
|
||||
|
|
@ -110,7 +110,7 @@ static const char* createOutFilename_orDie(const char* filename)
|
|||
memset(outSpace, 0, outL);
|
||||
strcat(outSpace, filename);
|
||||
strcat(outSpace, ".zst");
|
||||
return (const char*)outSpace;
|
||||
return (char*)outSpace;
|
||||
}
|
||||
|
||||
int main(int argc, const char** argv)
|
||||
|
|
@ -125,7 +125,7 @@ int main(int argc, const char** argv)
|
|||
return 1;
|
||||
}
|
||||
|
||||
const char* const outFilename = createOutFilename_orDie(inFilename);
|
||||
char* const outFilename = createOutFilename_orDie(inFilename);
|
||||
compress_orDie(inFilename, outFilename);
|
||||
free(outFilename);
|
||||
return 0;
|
||||
|
|
|
|||
|
|
@ -6,17 +6,16 @@
|
|||
* LICENSE-examples file in the root directory of this source tree.
|
||||
*/
|
||||
|
||||
|
||||
|
||||
#include <stdlib.h> // malloc, exit
|
||||
#include <stdio.h> // printf
|
||||
#include <string.h> // strerror
|
||||
#include <errno.h> // errno
|
||||
#include <sys/stat.h> // stat
|
||||
#define ZSTD_STATIC_LINKING_ONLY // ZSTD_findDecompressedSize
|
||||
#include <zstd.h> // presumes zstd library is installed
|
||||
|
||||
|
||||
static off_t fsize_X(const char *filename)
|
||||
static off_t fsize_orDie(const char *filename)
|
||||
{
|
||||
struct stat st;
|
||||
if (stat(filename, &st) == 0) return st.st_size;
|
||||
|
|
@ -25,7 +24,7 @@ static off_t fsize_X(const char *filename)
|
|||
exit(1);
|
||||
}
|
||||
|
||||
static FILE* fopen_X(const char *filename, const char *instruction)
|
||||
static FILE* fopen_orDie(const char *filename, const char *instruction)
|
||||
{
|
||||
FILE* const inFile = fopen(filename, instruction);
|
||||
if (inFile) return inFile;
|
||||
|
|
@ -34,7 +33,7 @@ static FILE* fopen_X(const char *filename, const char *instruction)
|
|||
exit(2);
|
||||
}
|
||||
|
||||
static void* malloc_X(size_t size)
|
||||
static void* malloc_orDie(size_t size)
|
||||
{
|
||||
void* const buff = malloc(size);
|
||||
if (buff) return buff;
|
||||
|
|
@ -43,11 +42,11 @@ static void* malloc_X(size_t size)
|
|||
exit(3);
|
||||
}
|
||||
|
||||
static void* loadFile_X(const char* fileName, size_t* size)
|
||||
static void* loadFile_orDie(const char* fileName, size_t* size)
|
||||
{
|
||||
off_t const buffSize = fsize_X(fileName);
|
||||
FILE* const inFile = fopen_X(fileName, "rb");
|
||||
void* const buffer = malloc_X(buffSize);
|
||||
off_t const buffSize = fsize_orDie(fileName);
|
||||
FILE* const inFile = fopen_orDie(fileName, "rb");
|
||||
void* const buffer = malloc_orDie(buffSize);
|
||||
size_t const readSize = fread(buffer, 1, buffSize, inFile);
|
||||
if (readSize != (size_t)buffSize) {
|
||||
printf("fread: %s : %s \n", fileName, strerror(errno));
|
||||
|
|
@ -62,13 +61,13 @@ static void* loadFile_X(const char* fileName, size_t* size)
|
|||
static void decompress(const char* fname)
|
||||
{
|
||||
size_t cSize;
|
||||
void* const cBuff = loadFile_X(fname, &cSize);
|
||||
void* const cBuff = loadFile_orDie(fname, &cSize);
|
||||
unsigned long long const rSize = ZSTD_findDecompressedSize(cBuff, cSize);
|
||||
if (rSize==0) {
|
||||
printf("%s : original size unknown. Use streaming decompression instead. \n", fname);
|
||||
exit(5);
|
||||
}
|
||||
void* const rBuff = malloc_X((size_t)rSize);
|
||||
void* const rBuff = malloc_orDie((size_t)rSize);
|
||||
|
||||
size_t const dSize = ZSTD_decompress(rBuff, rSize, cBuff, cSize);
|
||||
|
||||
|
|
|
|||
|
|
@ -43,6 +43,12 @@
|
|||
#include "huf.h"
|
||||
|
||||
|
||||
/*-****************************************
|
||||
* Version
|
||||
******************************************/
|
||||
unsigned FSE_versionNumber(void) { return FSE_VERSION_NUMBER; }
|
||||
|
||||
|
||||
/*-****************************************
|
||||
* FSE Error Management
|
||||
******************************************/
|
||||
|
|
@ -62,8 +68,6 @@ const char* HUF_getErrorName(size_t code) { return ERR_getErrorName(code); }
|
|||
/*-**************************************************************
|
||||
* FSE NCount encoding-decoding
|
||||
****************************************************************/
|
||||
static short FSE_abs(short a) { return (short)(a<0 ? -a : a); }
|
||||
|
||||
size_t FSE_readNCount (short* normalizedCounter, unsigned* maxSVPtr, unsigned* tableLogPtr,
|
||||
const void* headerBuffer, size_t hbSize)
|
||||
{
|
||||
|
|
@ -117,21 +121,21 @@ size_t FSE_readNCount (short* normalizedCounter, unsigned* maxSVPtr, unsigned* t
|
|||
} else {
|
||||
bitStream >>= 2;
|
||||
} }
|
||||
{ short const max = (short)((2*threshold-1)-remaining);
|
||||
short count;
|
||||
{ int const max = (2*threshold-1) - remaining;
|
||||
int count;
|
||||
|
||||
if ((bitStream & (threshold-1)) < (U32)max) {
|
||||
count = (short)(bitStream & (threshold-1));
|
||||
bitCount += nbBits-1;
|
||||
count = bitStream & (threshold-1);
|
||||
bitCount += nbBits-1;
|
||||
} else {
|
||||
count = (short)(bitStream & (2*threshold-1));
|
||||
count = bitStream & (2*threshold-1);
|
||||
if (count >= threshold) count -= max;
|
||||
bitCount += nbBits;
|
||||
bitCount += nbBits;
|
||||
}
|
||||
|
||||
count--; /* extra accuracy */
|
||||
remaining -= FSE_abs(count);
|
||||
normalizedCounter[charnum++] = count;
|
||||
remaining -= count < 0 ? -count : count; /* -1 means +1 */
|
||||
normalizedCounter[charnum++] = (short)count;
|
||||
previous0 = !count;
|
||||
while (remaining < threshold) {
|
||||
nbBits--;
|
||||
|
|
|
|||
|
|
@ -45,6 +45,32 @@ extern "C" {
|
|||
#include <stddef.h> /* size_t, ptrdiff_t */
|
||||
|
||||
|
||||
/*-*****************************************
|
||||
* FSE_PUBLIC_API : control library symbols visibility
|
||||
******************************************/
|
||||
#if defined(FSE_DLL_EXPORT) && (FSE_DLL_EXPORT==1) && defined(__GNUC__) && (__GNUC__ >= 4)
|
||||
# define FSE_PUBLIC_API __attribute__ ((visibility ("default")))
|
||||
#elif defined(FSE_DLL_EXPORT) && (FSE_DLL_EXPORT==1) /* Visual expected */
|
||||
# define FSE_PUBLIC_API __declspec(dllexport)
|
||||
#elif defined(FSE_DLL_IMPORT) && (FSE_DLL_IMPORT==1)
|
||||
# define FSE_PUBLIC_API __declspec(dllimport) /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/
|
||||
#else
|
||||
# define FSE_PUBLIC_API
|
||||
#endif
|
||||
|
||||
/*------ Version ------*/
|
||||
#define FSE_VERSION_MAJOR 0
|
||||
#define FSE_VERSION_MINOR 9
|
||||
#define FSE_VERSION_RELEASE 0
|
||||
|
||||
#define FSE_LIB_VERSION FSE_VERSION_MAJOR.FSE_VERSION_MINOR.FSE_VERSION_RELEASE
|
||||
#define FSE_QUOTE(str) #str
|
||||
#define FSE_EXPAND_AND_QUOTE(str) FSE_QUOTE(str)
|
||||
#define FSE_VERSION_STRING FSE_EXPAND_AND_QUOTE(FSE_LIB_VERSION)
|
||||
|
||||
#define FSE_VERSION_NUMBER (FSE_VERSION_MAJOR *100*100 + FSE_VERSION_MINOR *100 + FSE_VERSION_RELEASE)
|
||||
FSE_PUBLIC_API unsigned FSE_versionNumber(void); /**< library version number; to be used when checking dll version */
|
||||
|
||||
/*-****************************************
|
||||
* FSE simple functions
|
||||
******************************************/
|
||||
|
|
@ -56,8 +82,8 @@ extern "C" {
|
|||
if return == 1, srcData is a single byte symbol * srcSize times. Use RLE compression instead.
|
||||
if FSE_isError(return), compression failed (more details using FSE_getErrorName())
|
||||
*/
|
||||
size_t FSE_compress(void* dst, size_t dstCapacity,
|
||||
const void* src, size_t srcSize);
|
||||
FSE_PUBLIC_API size_t FSE_compress(void* dst, size_t dstCapacity,
|
||||
const void* src, size_t srcSize);
|
||||
|
||||
/*! FSE_decompress():
|
||||
Decompress FSE data from buffer 'cSrc', of size 'cSrcSize',
|
||||
|
|
@ -69,18 +95,18 @@ size_t FSE_compress(void* dst, size_t dstCapacity,
|
|||
Why ? : making this distinction requires a header.
|
||||
Header management is intentionally delegated to the user layer, which can better manage special cases.
|
||||
*/
|
||||
size_t FSE_decompress(void* dst, size_t dstCapacity,
|
||||
const void* cSrc, size_t cSrcSize);
|
||||
FSE_PUBLIC_API size_t FSE_decompress(void* dst, size_t dstCapacity,
|
||||
const void* cSrc, size_t cSrcSize);
|
||||
|
||||
|
||||
/*-*****************************************
|
||||
* Tool functions
|
||||
******************************************/
|
||||
size_t FSE_compressBound(size_t size); /* maximum compressed size */
|
||||
FSE_PUBLIC_API size_t FSE_compressBound(size_t size); /* maximum compressed size */
|
||||
|
||||
/* Error Management */
|
||||
unsigned FSE_isError(size_t code); /* tells if a return value is an error code */
|
||||
const char* FSE_getErrorName(size_t code); /* provides error code string (useful for debugging) */
|
||||
FSE_PUBLIC_API unsigned FSE_isError(size_t code); /* tells if a return value is an error code */
|
||||
FSE_PUBLIC_API const char* FSE_getErrorName(size_t code); /* provides error code string (useful for debugging) */
|
||||
|
||||
|
||||
/*-*****************************************
|
||||
|
|
@ -94,7 +120,7 @@ const char* FSE_getErrorName(size_t code); /* provides error code string (usef
|
|||
if return == 1, srcData is a single byte symbol * srcSize times. Use RLE compression.
|
||||
if FSE_isError(return), it's an error code.
|
||||
*/
|
||||
size_t FSE_compress2 (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog);
|
||||
FSE_PUBLIC_API size_t FSE_compress2 (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog);
|
||||
|
||||
|
||||
/*-*****************************************
|
||||
|
|
@ -127,50 +153,50 @@ or to save and provide normalized distribution using external method.
|
|||
@return : the count of the most frequent symbol (which is not identified).
|
||||
if return == srcSize, there is only one symbol.
|
||||
Can also return an error code, which can be tested with FSE_isError(). */
|
||||
size_t FSE_count(unsigned* count, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize);
|
||||
FSE_PUBLIC_API size_t FSE_count(unsigned* count, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize);
|
||||
|
||||
/*! FSE_optimalTableLog():
|
||||
dynamically downsize 'tableLog' when conditions are met.
|
||||
It saves CPU time, by using smaller tables, while preserving or even improving compression ratio.
|
||||
@return : recommended tableLog (necessarily <= 'maxTableLog') */
|
||||
unsigned FSE_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue);
|
||||
FSE_PUBLIC_API unsigned FSE_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue);
|
||||
|
||||
/*! FSE_normalizeCount():
|
||||
normalize counts so that sum(count[]) == Power_of_2 (2^tableLog)
|
||||
'normalizedCounter' is a table of short, of minimum size (maxSymbolValue+1).
|
||||
@return : tableLog,
|
||||
or an errorCode, which can be tested using FSE_isError() */
|
||||
size_t FSE_normalizeCount(short* normalizedCounter, unsigned tableLog, const unsigned* count, size_t srcSize, unsigned maxSymbolValue);
|
||||
FSE_PUBLIC_API size_t FSE_normalizeCount(short* normalizedCounter, unsigned tableLog, const unsigned* count, size_t srcSize, unsigned maxSymbolValue);
|
||||
|
||||
/*! FSE_NCountWriteBound():
|
||||
Provides the maximum possible size of an FSE normalized table, given 'maxSymbolValue' and 'tableLog'.
|
||||
Typically useful for allocation purpose. */
|
||||
size_t FSE_NCountWriteBound(unsigned maxSymbolValue, unsigned tableLog);
|
||||
FSE_PUBLIC_API size_t FSE_NCountWriteBound(unsigned maxSymbolValue, unsigned tableLog);
|
||||
|
||||
/*! FSE_writeNCount():
|
||||
Compactly save 'normalizedCounter' into 'buffer'.
|
||||
@return : size of the compressed table,
|
||||
or an errorCode, which can be tested using FSE_isError(). */
|
||||
size_t FSE_writeNCount (void* buffer, size_t bufferSize, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog);
|
||||
FSE_PUBLIC_API size_t FSE_writeNCount (void* buffer, size_t bufferSize, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog);
|
||||
|
||||
|
||||
/*! Constructor and Destructor of FSE_CTable.
|
||||
Note that FSE_CTable size depends on 'tableLog' and 'maxSymbolValue' */
|
||||
typedef unsigned FSE_CTable; /* don't allocate that. It's only meant to be more restrictive than void* */
|
||||
FSE_CTable* FSE_createCTable (unsigned tableLog, unsigned maxSymbolValue);
|
||||
void FSE_freeCTable (FSE_CTable* ct);
|
||||
FSE_PUBLIC_API FSE_CTable* FSE_createCTable (unsigned tableLog, unsigned maxSymbolValue);
|
||||
FSE_PUBLIC_API void FSE_freeCTable (FSE_CTable* ct);
|
||||
|
||||
/*! FSE_buildCTable():
|
||||
Builds `ct`, which must be already allocated, using FSE_createCTable().
|
||||
@return : 0, or an errorCode, which can be tested using FSE_isError() */
|
||||
size_t FSE_buildCTable(FSE_CTable* ct, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog);
|
||||
FSE_PUBLIC_API size_t FSE_buildCTable(FSE_CTable* ct, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog);
|
||||
|
||||
/*! FSE_compress_usingCTable():
|
||||
Compress `src` using `ct` into `dst` which must be already allocated.
|
||||
@return : size of compressed data (<= `dstCapacity`),
|
||||
or 0 if compressed data could not fit into `dst`,
|
||||
or an errorCode, which can be tested using FSE_isError() */
|
||||
size_t FSE_compress_usingCTable (void* dst, size_t dstCapacity, const void* src, size_t srcSize, const FSE_CTable* ct);
|
||||
FSE_PUBLIC_API size_t FSE_compress_usingCTable (void* dst, size_t dstCapacity, const void* src, size_t srcSize, const FSE_CTable* ct);
|
||||
|
||||
/*!
|
||||
Tutorial :
|
||||
|
|
@ -223,25 +249,25 @@ If there is an error, the function will return an ErrorCode (which can be tested
|
|||
@return : size read from 'rBuffer',
|
||||
or an errorCode, which can be tested using FSE_isError().
|
||||
maxSymbolValuePtr[0] and tableLogPtr[0] will also be updated with their respective values */
|
||||
size_t FSE_readNCount (short* normalizedCounter, unsigned* maxSymbolValuePtr, unsigned* tableLogPtr, const void* rBuffer, size_t rBuffSize);
|
||||
FSE_PUBLIC_API size_t FSE_readNCount (short* normalizedCounter, unsigned* maxSymbolValuePtr, unsigned* tableLogPtr, const void* rBuffer, size_t rBuffSize);
|
||||
|
||||
/*! Constructor and Destructor of FSE_DTable.
|
||||
Note that its size depends on 'tableLog' */
|
||||
typedef unsigned FSE_DTable; /* don't allocate that. It's just a way to be more restrictive than void* */
|
||||
FSE_DTable* FSE_createDTable(unsigned tableLog);
|
||||
void FSE_freeDTable(FSE_DTable* dt);
|
||||
FSE_PUBLIC_API FSE_DTable* FSE_createDTable(unsigned tableLog);
|
||||
FSE_PUBLIC_API void FSE_freeDTable(FSE_DTable* dt);
|
||||
|
||||
/*! FSE_buildDTable():
|
||||
Builds 'dt', which must be already allocated, using FSE_createDTable().
|
||||
return : 0, or an errorCode, which can be tested using FSE_isError() */
|
||||
size_t FSE_buildDTable (FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog);
|
||||
FSE_PUBLIC_API size_t FSE_buildDTable (FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog);
|
||||
|
||||
/*! FSE_decompress_usingDTable():
|
||||
Decompress compressed source `cSrc` of size `cSrcSize` using `dt`
|
||||
into `dst` which must be already allocated.
|
||||
@return : size of regenerated data (necessarily <= `dstCapacity`),
|
||||
or an errorCode, which can be tested using FSE_isError() */
|
||||
size_t FSE_decompress_usingDTable(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, const FSE_DTable* dt);
|
||||
FSE_PUBLIC_API size_t FSE_decompress_usingDTable(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, const FSE_DTable* dt);
|
||||
|
||||
/*!
|
||||
Tutorial :
|
||||
|
|
|
|||
|
|
@ -201,8 +201,6 @@ size_t FSE_NCountWriteBound(unsigned maxSymbolValue, unsigned tableLog)
|
|||
return maxSymbolValue ? maxHeaderSize : FSE_NCOUNTBOUND; /* maxSymbolValue==0 ? use default */
|
||||
}
|
||||
|
||||
static short FSE_abs(short a) { return (short)(a<0 ? -a : a); }
|
||||
|
||||
static size_t FSE_writeNCount_generic (void* header, size_t headerBufferSize,
|
||||
const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog,
|
||||
unsigned writeIsSafe)
|
||||
|
|
@ -258,16 +256,16 @@ static size_t FSE_writeNCount_generic (void* header, size_t headerBufferSize,
|
|||
bitStream >>= 16;
|
||||
bitCount -= 16;
|
||||
} }
|
||||
{ short count = normalizedCounter[charnum++];
|
||||
const short max = (short)((2*threshold-1)-remaining);
|
||||
remaining -= FSE_abs(count);
|
||||
if (remaining<1) return ERROR(GENERIC);
|
||||
{ int count = normalizedCounter[charnum++];
|
||||
int const max = (2*threshold-1)-remaining;
|
||||
remaining -= count < 0 ? -count : count;
|
||||
count++; /* +1 for extra accuracy */
|
||||
if (count>=threshold) count += max; /* [0..max[ [max..threshold[ (...) [threshold+max 2*threshold[ */
|
||||
bitStream += count << bitCount;
|
||||
bitCount += nbBits;
|
||||
bitCount -= (count<max);
|
||||
previous0 = (count==1);
|
||||
if (remaining<1) return ERROR(GENERIC);
|
||||
while (remaining<threshold) nbBits--, threshold>>=1;
|
||||
}
|
||||
if (bitCount>16) {
|
||||
|
|
|
|||
|
|
@ -2512,7 +2512,7 @@ static size_t ZSTD_loadDictionaryContent(ZSTD_CCtx* zc, const void* src, size_t
|
|||
return ERROR(GENERIC); /* strategy doesn't exist; impossible */
|
||||
}
|
||||
|
||||
zc->nextToUpdate = zc->loadedDictEnd;
|
||||
zc->nextToUpdate = (U32)(iend - zc->base);
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -2260,7 +2260,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
|
|||
|
||||
/* Adapt buffer sizes to frame header instructions */
|
||||
{ size_t const blockSize = MIN(zds->fParams.windowSize, ZSTD_BLOCKSIZE_ABSOLUTEMAX);
|
||||
size_t const neededOutSize = zds->fParams.windowSize + blockSize;
|
||||
size_t const neededOutSize = zds->fParams.windowSize + blockSize + WILDCOPY_OVERLENGTH * 2;
|
||||
zds->blockSize = blockSize;
|
||||
if (zds->inBuffSize < blockSize) {
|
||||
ZSTD_free(zds->inBuff, zds->customMem);
|
||||
|
|
|
|||
|
|
@ -4108,7 +4108,7 @@ size_t ZBUFFv06_decompressContinue(ZBUFFv06_DCtx* zbd,
|
|||
zbd->inBuff = (char*)malloc(blockSize);
|
||||
if (zbd->inBuff == NULL) return ERROR(memory_allocation);
|
||||
}
|
||||
{ size_t const neededOutSize = ((size_t)1 << zbd->fParams.windowLog) + blockSize;
|
||||
{ size_t const neededOutSize = ((size_t)1 << zbd->fParams.windowLog) + blockSize + WILDCOPY_OVERLENGTH * 2;
|
||||
if (zbd->outBuffSize < neededOutSize) {
|
||||
free(zbd->outBuff);
|
||||
zbd->outBuffSize = neededOutSize;
|
||||
|
|
|
|||
|
|
@ -4483,7 +4483,7 @@ size_t ZBUFFv07_decompressContinue(ZBUFFv07_DCtx* zbd,
|
|||
zbd->inBuff = (char*)zbd->customMem.customAlloc(zbd->customMem.opaque, blockSize);
|
||||
if (zbd->inBuff == NULL) return ERROR(memory_allocation);
|
||||
}
|
||||
{ size_t const neededOutSize = zbd->fParams.windowSize + blockSize;
|
||||
{ size_t const neededOutSize = zbd->fParams.windowSize + blockSize + WILDCOPY_OVERLENGTH * 2;
|
||||
if (zbd->outBuffSize < neededOutSize) {
|
||||
zbd->customMem.customFree(zbd->customMem.opaque, zbd->outBuff);
|
||||
zbd->outBuffSize = neededOutSize;
|
||||
|
|
|
|||
|
|
@ -184,7 +184,7 @@ clean:
|
|||
fuzzer-dll$(EXT) zstreamtest-dll$(EXT) zbufftest-dll$(EXT)\
|
||||
zstreamtest$(EXT) zstreamtest32$(EXT) \
|
||||
datagen$(EXT) paramgrill$(EXT) roundTripCrash$(EXT) longmatch$(EXT) \
|
||||
symbols$(EXT) invalidDictionaries$(EXT) pool$(EXT)
|
||||
symbols$(EXT) invalidDictionaries$(EXT) legacy$(EXT) pool$(EXT)
|
||||
@echo Cleaning completed
|
||||
|
||||
|
||||
|
|
@ -231,7 +231,7 @@ zstd-playTests: datagen
|
|||
ZSTD="$(QEMU_SYS) $(ZSTD)" ./playTests.sh $(ZSTDRTTEST)
|
||||
|
||||
test: test-zstd test-fullbench test-fuzzer test-zstream test-invalidDictionaries test-legacy
|
||||
ifneq ($(QEMU_SYS),qemu-ppc64-static)
|
||||
ifeq ($(QEMU_SYS),)
|
||||
test: test-pool
|
||||
endif
|
||||
|
||||
|
|
|
|||
|
|
@ -467,6 +467,38 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
|
|||
if (ZSTD_findDecompressedSize(compressedBuffer, cSize) != ZSTD_CONTENTSIZE_UNKNOWN) goto _output_error;
|
||||
DISPLAYLEVEL(3, "OK \n");
|
||||
|
||||
/* Overlen overwriting window data bug */
|
||||
DISPLAYLEVEL(3, "test%3i : wildcopy doesn't overwrite potential match data : ", testNb++);
|
||||
{ /* This test has a window size of 1024 bytes and consists of 3 blocks:
|
||||
1. 'a' repeated 517 times
|
||||
2. 'b' repeated 516 times
|
||||
3. a compressed block with no literals and 3 sequence commands:
|
||||
litlength = 0, offset = 24, match length = 24
|
||||
litlength = 0, offset = 24, match length = 3 (this one creates an overlength write of length 2*WILDCOPY_OVERLENGTH - 3)
|
||||
litlength = 0, offset = 1021, match length = 3 (this one will try to read from overwritten data if the buffer is too small) */
|
||||
|
||||
const char* testCase =
|
||||
"\x28\xB5\x2F\xFD\x04\x00\x4C\x00\x00\x10\x61\x61\x01\x00\x00\x2A"
|
||||
"\x80\x05\x44\x00\x00\x08\x62\x01\x00\x00\x2A\x20\x04\x5D\x00\x00"
|
||||
"\x00\x03\x40\x00\x00\x64\x60\x27\xB0\xE0\x0C\x67\x62\xCE\xE0";
|
||||
ZSTD_DStream* zds = ZSTD_createDStream();
|
||||
|
||||
ZSTD_initDStream(zds);
|
||||
inBuff.src = testCase;
|
||||
inBuff.size = 47;
|
||||
inBuff.pos = 0;
|
||||
outBuff.dst = decodedBuffer;
|
||||
outBuff.size = CNBufferSize;
|
||||
outBuff.pos = 0;
|
||||
|
||||
while (inBuff.pos < inBuff.size) {
|
||||
size_t const r = ZSTD_decompressStream(zds, &outBuff, &inBuff);
|
||||
/* Bug will cause checksum to fail */
|
||||
if (ZSTD_isError(r)) goto _output_error;
|
||||
}
|
||||
}
|
||||
DISPLAYLEVEL(3, "OK \n");
|
||||
|
||||
_end:
|
||||
FUZ_freeDictionary(dictionary);
|
||||
ZSTD_freeCStream(zc);
|
||||
|
|
|
|||
Loading…
Reference in New Issue