facebook
/
zstd
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #106 from Cyan4973/dev 

v0.4.6 candidate
dev
Yann Collet 2016-01-10 00:20:25 +01:00
parent ef774a68bb 523b594233
commit 8b8f3c16b2
25 changed files with 927 additions and 682 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
Makefile
View File

@ -31,8 +31,8 @@
# - Public forum : https://groups.google.com/forum/#!forum/lz4c
# ################################################################
# Version number
export VERSION := 0.4.5
# force a version number : uncomment below export (otherwise, default to the one declared into zstd.h)
#export VERSION := 0.4.6
PRGDIR = programs
ZSTDDIR = lib

6
NEWS
View File

@ -1,3 +1,9 @@
v0.4.6
fix : fast compression mode on Windows
Improved : high compression mode on repetitive data
New : block-level API
New : ZSTD_duplicateCCtx()
v0.4.5
new : -m/--multiple : compress/decompress multiple files

23
lib/Makefile
View File

@ -32,17 +32,21 @@
# ################################################################
# Version numbers
LIBVER_MAJOR=`sed -n '/define ZSTD_VERSION_MAJOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < zstd.h`
LIBVER_MINOR=`sed -n '/define ZSTD_VERSION_MINOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < zstd.h`
LIBVER_PATCH=`sed -n '/define ZSTD_VERSION_RELEASE/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < zstd.h`
LIBVER = $(LIBVER_MAJOR).$(LIBVER_MINOR).$(LIBVER_PATCH)
LIBVER_MAJOR_SCRIPT:=`sed -n '/define ZSTD_VERSION_MAJOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < zstd.h`
LIBVER_MINOR_SCRIPT:=`sed -n '/define ZSTD_VERSION_MINOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < zstd.h`
LIBVER_PATCH_SCRIPT:=`sed -n '/define ZSTD_VERSION_RELEASE/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < zstd.h`
LIBVER_SCRIPT:= $(LIBVER_MAJOR_SCRIPT).$(LIBVER_MINOR_SCRIPT).$(LIBVER_PATCH_SCRIPT)
LIBVER_MAJOR := $(shell echo $(LIBVER_MAJOR_SCRIPT))
LIBVER_MINOR := $(shell echo $(LIBVER_MINOR_SCRIPT))
LIBVER_PATCH := $(shell echo $(LIBVER_PATCH_SCRIPT))
LIBVER := $(shell echo $(LIBVER_SCRIPT))
VERSION?= $(LIBVER)
DESTDIR?=
PREFIX ?= /usr/local
CPPFLAGS= -I.
CFLAGS ?= -O3
CFLAGS += -std=c99 -Wall -Wextra -Wundef -Wshadow -Wcast-qual -Wcast-align -Wstrict-prototypes
CFLAGS += -std=c99 -Wall -Wextra -Wundef -Wshadow -Wcast-qual -Wcast-align -Wstrict-prototypes -Wstrict-aliasing=1
FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) $(MOREFLAGS)
LIBDIR ?= $(PREFIX)/lib
@ -98,12 +102,13 @@ clean:
#make install is validated only for Linux, OSX, kFreeBSD and Hurd targets
ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU))
libzstd.pc: libzstd.pc.in Makefile
libzstd.pc:
libzstd.pc: libzstd.pc.in
@echo creating pkgconfig
@sed -e 's|@PREFIX@|$(PREFIX)|' \
-e 's|@LIBDIR@|$(LIBDIR)|' \
-e 's|@INCLUDEDIR@|$(INCLUDEDIR)|' \
-e 's|@VERSION@|$(VERSION)|' \
-e 's|@LIBDIR@|$(LIBDIR)|' \
-e 's|@INCLUDEDIR@|$(INCLUDEDIR)|' \
-e 's|@VERSION@|$(VERSION)|' \
$< >$@
install: libzstd libzstd.pc

2
lib/error.h
View File

@ -69,7 +69,7 @@ extern "C" {
#define ERROR_LIST(ITEM) \
ITEM(PREFIX(No_Error)) ITEM(PREFIX(GENERIC)) \
ITEM(PREFIX(prefix_unknown)) ITEM(PREFIX(frameParameter_unsupported)) ITEM(PREFIX(frameParameter_unsupportedBy32bitsImplementation)) \
ITEM(PREFIX(init_missing)) ITEM(PREFIX(memory_allocation)) \
ITEM(PREFIX(init_missing)) ITEM(PREFIX(memory_allocation)) ITEM(PREFIX(stage_wrong)) \
ITEM(PREFIX(dstSize_tooSmall)) ITEM(PREFIX(srcSize_wrong)) \
ITEM(PREFIX(corruption_detected)) \
ITEM(PREFIX(tableLog_tooLarge)) ITEM(PREFIX(maxSymbolValue_tooLarge)) ITEM(PREFIX(maxSymbolValue_tooSmall)) \

97
lib/fse.c
View File

@ -34,10 +34,10 @@
#ifndef FSE_COMMONDEFS_ONLY
/****************************************************************
/* **************************************************************
* Tuning parameters
****************************************************************/
/* MEMORY_USAGE :
/*!MEMORY_USAGE :
* Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; etc.)
* Increasing memory usage improves compression ratio
* Reduced memory usage can improve speed, due to cache effect
@ -45,26 +45,23 @@
#define FSE_MAX_MEMORY_USAGE 14
#define FSE_DEFAULT_MEMORY_USAGE 13
/* FSE_MAX_SYMBOL_VALUE :
/*!FSE_MAX_SYMBOL_VALUE :
* Maximum symbol value authorized.
* Required for proper stack allocation */
#define FSE_MAX_SYMBOL_VALUE 255
/****************************************************************
/* **************************************************************
* template functions type & suffix
****************************************************************/
#define FSE_FUNCTION_TYPE BYTE
#define FSE_FUNCTION_EXTENSION
#define FSE_DECODE_TYPE FSE_decode_t
/****************************************************************
* Byte symbol type
****************************************************************/
#endif /* !FSE_COMMONDEFS_ONLY */
/****************************************************************
/* **************************************************************
* Compiler specifics
****************************************************************/
#ifdef _MSC_VER /* Visual Studio */
@ -82,7 +79,7 @@
#endif
/****************************************************************
/* **************************************************************
* Includes
****************************************************************/
#include <stdlib.h> /* malloc, free, qsort */
@ -92,7 +89,7 @@
#include "fse_static.h"
/****************************************************************
/* ***************************************************************
* Constants
*****************************************************************/
#define FSE_MAX_TABLELOG (FSE_MAX_MEMORY_USAGE-2)
@ -107,20 +104,20 @@
#endif
/****************************************************************
/* **************************************************************
* Error Management
****************************************************************/
#define FSE_STATIC_ASSERT(c) { enum { FSE_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */
/****************************************************************
/* **************************************************************
* Complex types
****************************************************************/
typedef U32 CTable_max_t[FSE_CTABLE_SIZE_U32(FSE_MAX_TABLELOG, FSE_MAX_SYMBOL_VALUE)];
typedef U32 DTable_max_t[FSE_DTABLE_SIZE_U32(FSE_MAX_TABLELOG)];
/****************************************************************
/* **************************************************************
* Templates
****************************************************************/
/*
@ -144,8 +141,7 @@ typedef U32 DTable_max_t[FSE_DTABLE_SIZE_U32(FSE_MAX_TABLELOG)];
/* Function templates */
size_t FSE_FUNCTION_NAME(FSE_count_generic, FSE_FUNCTION_EXTENSION)
(unsigned* count, unsigned* maxSymbolValuePtr, const FSE_FUNCTION_TYPE* source, size_t sourceSize, unsigned safe)
size_t FSE_count_generic(unsigned* count, unsigned* maxSymbolValuePtr, const FSE_FUNCTION_TYPE* source, size_t sourceSize, unsigned safe)
{
const FSE_FUNCTION_TYPE* ip = source;
const FSE_FUNCTION_TYPE* const iend = ip+sourceSize;
@ -226,7 +222,7 @@ size_t FSE_FUNCTION_NAME(FSE_count_generic, FSE_FUNCTION_EXTENSION)
size_t FSE_FUNCTION_NAME(FSE_countFast, FSE_FUNCTION_EXTENSION)
(unsigned* count, unsigned* maxSymbolValuePtr, const FSE_FUNCTION_TYPE* source, size_t sourceSize)
{
return FSE_FUNCTION_NAME(FSE_count_generic, FSE_FUNCTION_EXTENSION) (count, maxSymbolValuePtr, source, sourceSize, 0);
return FSE_count_generic(count, maxSymbolValuePtr, source, sourceSize, 0);
}
size_t FSE_FUNCTION_NAME(FSE_count, FSE_FUNCTION_EXTENSION)
@ -235,25 +231,26 @@ size_t FSE_FUNCTION_NAME(FSE_count, FSE_FUNCTION_EXTENSION)
if ((sizeof(FSE_FUNCTION_TYPE)==1) && (*maxSymbolValuePtr >= 255))
{
*maxSymbolValuePtr = 255;
return FSE_FUNCTION_NAME(FSE_count_generic, FSE_FUNCTION_EXTENSION) (count, maxSymbolValuePtr, source, sourceSize, 0);
return FSE_count_generic(count, maxSymbolValuePtr, source, sourceSize, 0);
}
return FSE_FUNCTION_NAME(FSE_count_generic, FSE_FUNCTION_EXTENSION) (count, maxSymbolValuePtr, source, sourceSize, 1);
return FSE_count_generic(count, maxSymbolValuePtr, source, sourceSize, 1);
}
static U32 FSE_tableStep(U32 tableSize) { return (tableSize>>1) + (tableSize>>3) + 3; }
size_t FSE_FUNCTION_NAME(FSE_buildCTable, FSE_FUNCTION_EXTENSION)
(FSE_CTable* ct, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog)
size_t FSE_buildCTable(FSE_CTable* ct, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog)
{
const unsigned tableSize = 1 << tableLog;
const unsigned tableMask = tableSize - 1;
U16* tableU16 = ( (U16*) ct) + 2;
FSE_symbolCompressionTransform* symbolTT = (FSE_symbolCompressionTransform*) (((U32*)ct) + 1 + (tableLog ? tableSize>>1 : 1) );
void* const ptr = ct;
U16* const tableU16 = ( (U16*) ptr) + 2;
void* const FSCT = ((U32*)ptr) + 1 /* header */ + (tableLog ? tableSize>>1 : 1) ;
FSE_symbolCompressionTransform* const symbolTT = (FSE_symbolCompressionTransform*) (FSCT);
const unsigned step = FSE_tableStep(tableSize);
unsigned cumul[FSE_MAX_SYMBOL_VALUE+2];
U32 position = 0;
FSE_FUNCTION_TYPE tableSymbol[FSE_MAX_TABLESIZE]; /* init isn't necessary, even if static analyzer complain about it */
FSE_FUNCTION_TYPE tableSymbol[FSE_MAX_TABLESIZE]; /* memset() is not necessary, even if static analyzer complain about it */
U32 highThreshold = tableSize-1;
unsigned symbol;
unsigned i;
@ -269,7 +266,7 @@ size_t FSE_FUNCTION_NAME(FSE_buildCTable, FSE_FUNCTION_EXTENSION)
cumul[0] = 0;
for (i=1; i<=maxSymbolValue+1; i++)
{
if (normalizedCounter[i-1]==-1) /* Low prob symbol */
if (normalizedCounter[i-1]==-1) /* Low proba symbol */
{
cumul[i] = cumul[i-1] + 1;
tableSymbol[highThreshold--] = (FSE_FUNCTION_TYPE)(i-1);
@ -287,7 +284,7 @@ size_t FSE_FUNCTION_NAME(FSE_buildCTable, FSE_FUNCTION_EXTENSION)
{
tableSymbol[position] = (FSE_FUNCTION_TYPE)symbol;
position = (position + step) & tableMask;
while (position > highThreshold) position = (position + step) & tableMask; /* Lowprob area */
while (position > highThreshold) position = (position + step) & tableMask; /* Low proba area */
}
}
@ -296,7 +293,7 @@ size_t FSE_FUNCTION_NAME(FSE_buildCTable, FSE_FUNCTION_EXTENSION)
/* Build table */
for (i=0; i<tableSize; i++)
{
FSE_FUNCTION_TYPE s = tableSymbol[i]; /* static analyzer doesn't understand tableSymbol is properly initialized */
FSE_FUNCTION_TYPE s = tableSymbol[i]; /* note : static analyzer may not understand tableSymbol is properly initialized */
tableU16[cumul[s]++] = (U16) (tableSize+i); /* TableU16 : sorted by symbol order; gives next state value */
}
@ -332,24 +329,22 @@ size_t FSE_FUNCTION_NAME(FSE_buildCTable, FSE_FUNCTION_EXTENSION)
}
#define FSE_DECODE_TYPE FSE_TYPE_NAME(FSE_decode_t, FSE_FUNCTION_EXTENSION)
FSE_DTable* FSE_FUNCTION_NAME(FSE_createDTable, FSE_FUNCTION_EXTENSION) (unsigned tableLog)
FSE_DTable* FSE_createDTable (unsigned tableLog)
{
if (tableLog > FSE_TABLELOG_ABSOLUTE_MAX) tableLog = FSE_TABLELOG_ABSOLUTE_MAX;
return (FSE_DTable*)malloc( FSE_DTABLE_SIZE_U32(tableLog) * sizeof (U32) );
}
void FSE_FUNCTION_NAME(FSE_freeDTable, FSE_FUNCTION_EXTENSION) (FSE_DTable* dt)
void FSE_freeDTable (FSE_DTable* dt)
{
free(dt);
}
size_t FSE_FUNCTION_NAME(FSE_buildDTable, FSE_FUNCTION_EXTENSION)
(FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog)
size_t FSE_buildDTable(FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog)
{
FSE_DTableHeader* const DTableH = (FSE_DTableHeader*)dt;
FSE_DECODE_TYPE* const tableDecode = (FSE_DECODE_TYPE*) (dt+1); /* because dt is unsigned, 32-bits aligned on 32-bits */
FSE_DTableHeader DTableH;
void* const tdPtr = dt+1; /* because dt is unsigned, 32-bits aligned on 32-bits */
FSE_DECODE_TYPE* const tableDecode = (FSE_DECODE_TYPE*) (tdPtr);
const U32 tableSize = 1 << tableLog;
const U32 tableMask = tableSize-1;
const U32 step = FSE_tableStep(tableSize);
@ -365,7 +360,7 @@ size_t FSE_FUNCTION_NAME(FSE_buildDTable, FSE_FUNCTION_EXTENSION)
if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge);
/* Init, lay down lowprob symbols */
DTableH[0].tableLog = (U16)tableLog;
DTableH.tableLog = (U16)tableLog;
for (s=0; s<=maxSymbolValue; s++)
{
if (normalizedCounter[s]==-1)
@ -406,7 +401,8 @@ size_t FSE_FUNCTION_NAME(FSE_buildDTable, FSE_FUNCTION_EXTENSION)
}
}
DTableH->fastMode = (U16)noLarge;
DTableH.fastMode = (U16)noLarge;
memcpy(dt, &DTableH, sizeof(DTableH));
return 0;
}
@ -890,8 +886,10 @@ size_t FSE_buildCTable_raw (FSE_CTable* ct, unsigned nbBits)
const unsigned tableSize = 1 << nbBits;
const unsigned tableMask = tableSize - 1;
const unsigned maxSymbolValue = tableMask;
U16* tableU16 = ( (U16*) ct) + 2;
FSE_symbolCompressionTransform* symbolTT = (FSE_symbolCompressionTransform*) ((((U32*)ct)+1) + (tableSize>>1));
void* const ptr = ct;
U16* const tableU16 = ( (U16*) ptr) + 2;
void* const FSCT = ((U32*)ptr) + 1 /* header */ + (tableSize>>1); /* assumption : tableLog >= 1 */
FSE_symbolCompressionTransform* const symbolTT = (FSE_symbolCompressionTransform*) (FSCT);
unsigned s;
/* Sanity checks */
@ -918,8 +916,10 @@ size_t FSE_buildCTable_raw (FSE_CTable* ct, unsigned nbBits)
/* fake FSE_CTable, for rle (100% always same symbol) input */
size_t FSE_buildCTable_rle (FSE_CTable* ct, BYTE symbolValue)
{
U16* tableU16 = ( (U16*) ct) + 2;
FSE_symbolCompressionTransform* symbolTT = (FSE_symbolCompressionTransform*) ((U32*)ct + 2);
void* ptr = ct;
U16* tableU16 = ( (U16*) ptr) + 2;
void* FSCTptr = (U32*)ptr + 2;
FSE_symbolCompressionTransform* symbolTT = (FSE_symbolCompressionTransform*) FSCTptr;
/* header */
tableU16[-2] = (U16) 0;
@ -1076,8 +1076,10 @@ size_t FSE_compress (void* dst, size_t dstSize, const void* src, size_t srcSize)
*********************************************************/
size_t FSE_buildDTable_rle (FSE_DTable* dt, BYTE symbolValue)
{
FSE_DTableHeader* const DTableH = (FSE_DTableHeader*)dt;
FSE_decode_t* const cell = (FSE_decode_t*)(dt + 1); /* because dt is unsigned */
void* ptr = dt;
FSE_DTableHeader* const DTableH = (FSE_DTableHeader*)ptr;
void* dPtr = dt + 1;
FSE_decode_t* const cell = (FSE_decode_t*)dPtr;
DTableH->tableLog = 0;
DTableH->fastMode = 0;
@ -1092,8 +1094,10 @@ size_t FSE_buildDTable_rle (FSE_DTable* dt, BYTE symbolValue)
size_t FSE_buildDTable_raw (FSE_DTable* dt, unsigned nbBits)
{
FSE_DTableHeader* const DTableH = (FSE_DTableHeader*)dt;
FSE_decode_t* const dinfo = (FSE_decode_t*)(dt + 1); /* because dt is unsigned */
void* ptr = dt;
FSE_DTableHeader* const DTableH = (FSE_DTableHeader*)ptr;
void* dPtr = dt + 1;
FSE_decode_t* const dinfo = (FSE_decode_t*)dPtr;
const unsigned tableSize = 1 << nbBits;
const unsigned tableMask = tableSize - 1;
const unsigned maxSymbolValue = tableMask;
@ -1189,7 +1193,8 @@ size_t FSE_decompress_usingDTable(void* dst, size_t originalSize,
const void* cSrc, size_t cSrcSize,
const FSE_DTable* dt)
{
const FSE_DTableHeader* DTableH = (const FSE_DTableHeader*)dt;
const void* ptr = dt;
const FSE_DTableHeader* DTableH = (const FSE_DTableHeader*)ptr;
const U32 fastMode = DTableH->fastMode;
/* select fast mode (static) */

46
lib/fse.h
View File

@ -40,20 +40,20 @@ extern "C" {
#endif
/******************************************
/* *****************************************
* Includes
******************************************/
#include <stddef.h> /* size_t, ptrdiff_t */
/******************************************
/* *****************************************
* FSE simple functions
******************************************/
size_t FSE_compress(void* dst, size_t maxDstSize,
const void* src, size_t srcSize);
size_t FSE_decompress(void* dst, size_t maxDstSize,
const void* cSrc, size_t cSrcSize);
/*
/*!
FSE_compress():
Compress content of buffer 'src', of size 'srcSize', into destination buffer 'dst'.
'dst' buffer must be already allocated. Compression runs faster is maxDstSize >= FSE_compressBound(srcSize)
@ -74,7 +74,7 @@ FSE_decompress():
*/
/******************************************
/* *****************************************
* Tool functions
******************************************/
size_t FSE_compressBound(size_t size); /* maximum compressed size */
@ -84,10 +84,10 @@ unsigned FSE_isError(size_t code); /* tells if a return value is an er
const char* FSE_getErrorName(size_t code); /* provides error code string (useful for debugging) */
/******************************************
/* *****************************************
* FSE advanced functions
******************************************/
/*
/*!
FSE_compress2():
Same as FSE_compress(), but allows the selection of 'maxSymbolValue' and 'tableLog'
Both parameters can be defined as '0' to mean : use default value
@ -99,10 +99,10 @@ FSE_compress2():
size_t FSE_compress2 (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog);
/******************************************
/* *****************************************
* FSE detailed API
******************************************/
/*
/*!
FSE_compress() does the following:
1. count symbol occurrence from source[] into table count[]
2. normalize counters so that sum(count[]) == Power_of_2 (2^tableLog)
@ -122,7 +122,7 @@ or to save and provide normalized distribution using external method.
/* *** COMPRESSION *** */
/*
/*!
FSE_count():
Provides the precise count of each symbol within a table 'count'
'count' is a table of unsigned int, of minimum size (maxSymbolValuePtr[0]+1).
@ -132,14 +132,14 @@ FSE_count():
if FSE_isError(return), it's an error code. */
size_t FSE_count(unsigned* count, unsigned* maxSymbolValuePtr, const unsigned char* src, size_t srcSize);
/*
/*!
FSE_optimalTableLog():
dynamically downsize 'tableLog' when conditions are met.
It saves CPU time, by using smaller tables, while preserving or even improving compression ratio.
return : recommended tableLog (necessarily <= initial 'tableLog') */
unsigned FSE_optimalTableLog(unsigned tableLog, size_t srcSize, unsigned maxSymbolValue);
/*
/*!
FSE_normalizeCount():
normalize counters so that sum(count[]) == Power_of_2 (2^tableLog)
'normalizedCounter' is a table of short, of minimum size (maxSymbolValue+1).
@ -147,13 +147,13 @@ FSE_normalizeCount():
or an errorCode, which can be tested using FSE_isError() */
size_t FSE_normalizeCount(short* normalizedCounter, unsigned tableLog, const unsigned* count, size_t srcSize, unsigned maxSymbolValue);
/*
/*!
FSE_NCountWriteBound():
Provides the maximum possible size of an FSE normalized table, given 'maxSymbolValue' and 'tableLog'
Typically useful for allocation purpose. */
size_t FSE_NCountWriteBound(unsigned maxSymbolValue, unsigned tableLog);
/*
/*!
FSE_writeNCount():
Compactly save 'normalizedCounter' into 'buffer'.
return : size of the compressed table
@ -161,21 +161,21 @@ FSE_writeNCount():
size_t FSE_writeNCount (void* buffer, size_t bufferSize, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog);
/*
/*!
Constructor and Destructor of type FSE_CTable
Note that its size depends on 'tableLog' and 'maxSymbolValue' */
typedef unsigned FSE_CTable; /* don't allocate that. It's just a way to be more restrictive than void* */
typedef unsigned FSE_CTable; /* don't allocate that. It's only meant to be more restrictive than void* */
FSE_CTable* FSE_createCTable (unsigned tableLog, unsigned maxSymbolValue);
void FSE_freeCTable (FSE_CTable* ct);
/*
/*!
FSE_buildCTable():
Builds 'ct', which must be already allocated, using FSE_createCTable()
return : 0
or an errorCode, which can be tested using FSE_isError() */
size_t FSE_buildCTable(FSE_CTable* ct, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog);
/*
/*!
FSE_compress_usingCTable():
Compress 'src' using 'ct' into 'dst' which must be already allocated
return : size of compressed data (<= maxDstSize)
@ -183,7 +183,7 @@ FSE_compress_usingCTable():
or an errorCode, which can be tested using FSE_isError() */
size_t FSE_compress_usingCTable (void* dst, size_t maxDstSize, const void* src, size_t srcSize, const FSE_CTable* ct);
/*
/*!
Tutorial :
----------
The first step is to count all symbols. FSE_count() does this job very fast.
@ -229,7 +229,7 @@ If there is an error, the function will return an ErrorCode (which can be tested
/* *** DECOMPRESSION *** */
/*
/*!
FSE_readNCount():
Read compactly saved 'normalizedCounter' from 'rBuffer'.
return : size read from 'rBuffer'
@ -237,21 +237,21 @@ FSE_readNCount():
maxSymbolValuePtr[0] and tableLogPtr[0] will also be updated with their respective values */
size_t FSE_readNCount (short* normalizedCounter, unsigned* maxSymbolValuePtr, unsigned* tableLogPtr, const void* rBuffer, size_t rBuffSize);
/*
/*!
Constructor and Destructor of type FSE_DTable
Note that its size depends on 'tableLog' */
typedef unsigned FSE_DTable; /* don't allocate that. It's just a way to be more restrictive than void* */
FSE_DTable* FSE_createDTable(unsigned tableLog);
void FSE_freeDTable(FSE_DTable* dt);
/*
/*!
FSE_buildDTable():
Builds 'dt', which must be already allocated, using FSE_createDTable()
return : 0,
or an errorCode, which can be tested using FSE_isError() */
size_t FSE_buildDTable (FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog);
/*
/*!
FSE_decompress_usingDTable():
Decompress compressed source 'cSrc' of size 'cSrcSize' using 'dt'
into 'dst' which must be already allocated.
@ -259,7 +259,7 @@ FSE_decompress_usingDTable():
or an errorCode, which can be tested using FSE_isError() */
size_t FSE_decompress_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const FSE_DTable* dt);
/*
/*!
Tutorial :
----------
(Note : these functions only decompress FSE-compressed blocks.

53
lib/fse_static.h
View File

@ -40,31 +40,31 @@ extern "C" {
#endif
/******************************************
* FSE API compatible with DLL
******************************************/
/* *****************************************
* Dependencies
*******************************************/
#include "fse.h"
#include "bitstream.h"
/******************************************
/* *****************************************
* Static allocation
******************************************/
*******************************************/
/* FSE buffer bounds */
#define FSE_NCOUNTBOUND 512
#define FSE_BLOCKBOUND(size) (size + (size>>7))
#define FSE_COMPRESSBOUND(size) (FSE_NCOUNTBOUND + FSE_BLOCKBOUND(size)) /* Macro version, useful for static allocation */
/* You can statically allocate FSE CTable/DTable as a table of unsigned using below macro */
/* It is possible to statically allocate FSE CTable/DTable as a table of unsigned using below macros */
#define FSE_CTABLE_SIZE_U32(maxTableLog, maxSymbolValue) (1 + (1<<(maxTableLog-1)) + ((maxSymbolValue+1)*2))
#define FSE_DTABLE_SIZE_U32(maxTableLog) (1 + (1<<maxTableLog))
/******************************************
/* *****************************************
* FSE advanced API
******************************************/
*******************************************/
size_t FSE_countFast(unsigned* count, unsigned* maxSymbolValuePtr, const unsigned char* src, size_t srcSize);
/* same as FSE_count(), but blindly trust that all values within src are <= maxSymbolValuePtr[0] */
/* same as FSE_count(), but blindly trust that all values within src are <= *maxSymbolValuePtr */
size_t FSE_buildCTable_raw (FSE_CTable* ct, unsigned nbBits);
/* build a fake FSE_CTable, designed to not compress an input, where each symbol uses nbBits */
@ -79,10 +79,10 @@ size_t FSE_buildDTable_rle (FSE_DTable* dt, unsigned char symbolValue);
/* build a fake FSE_DTable, designed to always generate the same symbolValue */
/******************************************
/* *****************************************
* FSE symbol compression API
******************************************/
/*
*******************************************/
/*!
This API consists of small unitary functions, which highly benefit from being inlined.
You will want to enable link-time-optimization to ensure these functions are properly inlined in your binary.
Visual seems to do it automatically.
@ -103,7 +103,7 @@ static void FSE_encodeSymbol(BIT_CStream_t* bitC, FSE_CState_t* CStatePtr, unsig
static void FSE_flushCState(BIT_CStream_t* bitC, const FSE_CState_t* CStatePtr);
/*
/*!
These functions are inner components of FSE_compress_usingCTable().
They allow the creation of custom streams, mixing multiple tables and bit sources.
@ -147,9 +147,9 @@ If there is an error, it returns an errorCode (which can be tested using FSE_isE
*/
/******************************************
/* *****************************************
* FSE symbol decompression API
******************************************/
*******************************************/
typedef struct
{
size_t state;
@ -163,7 +163,7 @@ static unsigned char FSE_decodeSymbol(FSE_DState_t* DStatePtr, BIT_DStream_t* bi
static unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr);
/*
/*!
Let's now decompose FSE_decompress_usingDTable() into its unitary components.
You will decode FSE-encoded symbols from the bitStream,
and also any other bitFields you put in, **in reverse order**.
@ -213,16 +213,16 @@ Check also the states. There might be some symbols left there, if some high prob
*/
/******************************************
/* *****************************************
* FSE unsafe API
******************************************/
*******************************************/
static unsigned char FSE_decodeSymbolFast(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD);
/* faster, but works only if nbBits is always >= 1 (otherwise, result will be corrupted) */
/******************************************
* Implementation of inline functions
******************************************/
/* *****************************************
* Implementation of inlined functions
*******************************************/
typedef struct
{
int deltaFindState;
@ -231,10 +231,12 @@ typedef struct
MEM_STATIC void FSE_initCState(FSE_CState_t* statePtr, const FSE_CTable* ct)
{
const U32 tableLog = ( (const U16*) ct) [0];
const void* ptr = ct;
const U16* u16ptr = (const U16*) ptr;
const U32 tableLog = MEM_read16(ptr);
statePtr->value = (ptrdiff_t)1<<tableLog;
statePtr->stateTable = ((const U16*) ct) + 2;
statePtr->symbolTT = (const void*)((const U32*)ct + 1 + (tableLog ? (1<<(tableLog-1)) : 1));
statePtr->stateTable = u16ptr+2;
statePtr->symbolTT = ((const U32*)ct + 1 + (tableLog ? (1<<(tableLog-1)) : 1));
statePtr->stateLog = tableLog;
}
@ -269,7 +271,8 @@ typedef struct
MEM_STATIC void FSE_initDState(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD, const FSE_DTable* dt)
{
const FSE_DTableHeader* const DTableH = (const FSE_DTableHeader*)dt;
const void* ptr = dt;
const FSE_DTableHeader* const DTableH = (const FSE_DTableHeader*)ptr;
DStatePtr->state = BIT_readBits(bitD, DTableH->tableLog);
BIT_reloadDStream(bitD);
DStatePtr->table = dt + 1;

81
lib/huff0.c
View File

@ -32,7 +32,7 @@
- Public forum : https://groups.google.com/forum/#!forum/lz4c
****************************************************************** */
/****************************************************************
/* **************************************************************
* Compiler specifics
****************************************************************/
#if defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
@ -57,7 +57,7 @@
#endif
/****************************************************************
/* **************************************************************
* Includes
****************************************************************/
#include <stdlib.h> /* malloc, free, qsort */
@ -68,7 +68,7 @@
#include "fse.h" /* header compression */
/****************************************************************
/* **************************************************************
* Constants
****************************************************************/
#define HUF_ABSOLUTEMAX_TABLELOG 16 /* absolute limit of HUF_MAX_TABLELOG. Beyond that value, code does not work */
@ -80,27 +80,21 @@
#endif
/****************************************************************
/* **************************************************************
* Error Management
****************************************************************/
unsigned HUF_isError(size_t code) { return ERR_isError(code); }
const char* HUF_getErrorName(size_t code) { return ERR_getErrorName(code); }
#define HUF_STATIC_ASSERT(c) { enum { HUF_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */
/******************************************
* Helper functions
******************************************/
unsigned HUF_isError(size_t code) { return ERR_isError(code); }
const char* HUF_getErrorName(size_t code) { return ERR_getErrorName(code); }
/*********************************************************
/* *******************************************************
* Huff0 : Huffman block compression
*********************************************************/
typedef struct HUF_CElt_s {
struct HUF_CElt_s {
U16 val;
BYTE nbBits;
} HUF_CElt ;
}; /* typedef'd to HUF_CElt within huff0_static.h */
typedef struct nodeElt_s {
U32 count;
@ -320,7 +314,7 @@ size_t HUF_buildCTable (HUF_CElt* tree, const U32* count, U32 maxSymbolValue, U3
/* sort, decreasing order */
HUF_sort(huffNode, count, maxSymbolValue);
// init for parents
/* init for parents */
nonNullRank = maxSymbolValue;
while(huffNode[nonNullRank].count == 0) nonNullRank--;
lowS = nonNullRank; nodeRoot = nodeNb + lowS - 1; lowN = nodeNb;
@ -330,7 +324,7 @@ size_t HUF_buildCTable (HUF_CElt* tree, const U32* count, U32 maxSymbolValue, U3
for (n=nodeNb; n<=nodeRoot; n++) huffNode[n].count = (U32)(1U<<30);
huffNode0[0].count = (U32)(1U<<31);
// create parents
/* create parents */
while (nodeNb <= nodeRoot)
{
U32 n1 = (huffNode[lowS].count < huffNode[lowN].count) ? lowS-- : lowN++;
@ -340,7 +334,7 @@ size_t HUF_buildCTable (HUF_CElt* tree, const U32* count, U32 maxSymbolValue, U3
nodeNb++;
}
// distribute weights (unlimited tree height)
/* distribute weights (unlimited tree height) */
huffNode[nodeRoot].nbBits = 0;
for (n=nodeRoot-1; n>=STARTNODE; n--)
huffNode[n].nbBits = huffNode[ huffNode[n].parent ].nbBits + 1;
@ -368,9 +362,9 @@ size_t HUF_buildCTable (HUF_CElt* tree, const U32* count, U32 maxSymbolValue, U3
}
}
for (n=0; n<=maxSymbolValue; n++)
tree[huffNode[n].byte].nbBits = huffNode[n].nbBits; // push nbBits per symbol, symbol order
tree[huffNode[n].byte].nbBits = huffNode[n].nbBits; /* push nbBits per symbol, symbol order */
for (n=0; n<=maxSymbolValue; n++)
tree[n].val = valPerRank[tree[n].nbBits]++; // assign value within rank, symbol order
tree[n].val = valPerRank[tree[n].nbBits]++; /* assign value within rank, symbol order */
}
return maxNbBits;
@ -636,12 +630,12 @@ size_t HUF_readDTableX2 (U16* DTable, const void* src, size_t srcSize)
BYTE huffWeight[HUF_MAX_SYMBOL_VALUE + 1];
U32 rankVal[HUF_ABSOLUTEMAX_TABLELOG + 1]; /* large enough for values from 0 to 16 */
U32 tableLog = 0;
const BYTE* ip = (const BYTE*) src;
size_t iSize = ip[0];
size_t iSize;
U32 nbSymbols = 0;
U32 n;
U32 nextRankStart;
HUF_DEltX2* const dt = (HUF_DEltX2*)(DTable + 1);
void* const dtPtr = DTable + 1;
HUF_DEltX2* const dt = (HUF_DEltX2*)dtPtr;
HUF_STATIC_ASSERT(sizeof(HUF_DEltX2) == sizeof(U16)); /* if compilation fails here, assertion is false */
//memset(huffWeight, 0, sizeof(huffWeight)); /* is not necessary, even though some analyzer complain ... */
@ -730,7 +724,8 @@ size_t HUF_decompress1X2_usingDTable(
BYTE* const oend = op + dstSize;
size_t errorCode;
const U32 dtLog = DTable[0];
const HUF_DEltX2* const dt = ((const HUF_DEltX2*)DTable) +1;
const void* dtPtr = DTable;
const HUF_DEltX2* const dt = ((const HUF_DEltX2*)dtPtr)+1;
BIT_DStream_t bitD;
errorCode = BIT_initDStream(&bitD, cSrc, cSrcSize);
if (HUF_isError(errorCode)) return errorCode;
@ -770,8 +765,8 @@ size_t HUF_decompress4X2_usingDTable(
const BYTE* const istart = (const BYTE*) cSrc;
BYTE* const ostart = (BYTE*) dst;
BYTE* const oend = ostart + dstSize;
const HUF_DEltX2* const dt = ((const HUF_DEltX2*)DTable) +1;
const void* const dtPtr = DTable;
const HUF_DEltX2* const dt = ((const HUF_DEltX2*)dtPtr) +1;
const U32 dtLog = DTable[0];
size_t errorCode;
@ -978,9 +973,9 @@ size_t HUF_readDTableX4 (U32* DTable, const void* src, size_t srcSize)
rankVal_t rankVal;
U32 tableLog, maxW, sizeOfSort, nbSymbols;
const U32 memLog = DTable[0];
const BYTE* ip = (const BYTE*) src;
size_t iSize = ip[0];
HUF_DEltX4* const dt = ((HUF_DEltX4*)DTable) + 1;
size_t iSize;
void* dtPtr = DTable;
HUF_DEltX4* const dt = ((HUF_DEltX4*)dtPtr) + 1;
HUF_STATIC_ASSERT(sizeof(HUF_DEltX4) == sizeof(U32)); /* if compilation fails here, assertion is false */
if (memLog > HUF_ABSOLUTEMAX_TABLELOG) return ERROR(tableLog_tooLarge);
@ -1127,7 +1122,8 @@ size_t HUF_decompress1X4_usingDTable(
BYTE* const oend = ostart + dstSize;
const U32 dtLog = DTable[0];
const HUF_DEltX4* const dt = ((const HUF_DEltX4*)DTable) +1;
const void* const dtPtr = DTable;
const HUF_DEltX4* const dt = ((const HUF_DEltX4*)dtPtr) +1;
size_t errorCode;
/* Init */
@ -1170,8 +1166,8 @@ size_t HUF_decompress4X4_usingDTable(
const BYTE* const istart = (const BYTE*) cSrc;
BYTE* const ostart = (BYTE*) dst;
BYTE* const oend = ostart + dstSize;
const HUF_DEltX4* const dt = ((const HUF_DEltX4*)DTable) +1;
const void* const dtPtr = DTable;
const HUF_DEltX4* const dt = ((const HUF_DEltX4*)dtPtr) +1;
const U32 dtLog = DTable[0];
size_t errorCode;
@ -1352,8 +1348,7 @@ size_t HUF_readDTableX6 (U32* DTable, const void* src, size_t srcSize)
U32 tableLog, maxW, sizeOfSort, nbSymbols;
rankVal_t rankVal;
const U32 memLog = DTable[0];
const BYTE* ip = (const BYTE*) src;
size_t iSize = ip[0];
size_t iSize;
if (memLog > HUF_ABSOLUTEMAX_TABLELOG) return ERROR(tableLog_tooLarge);
//memset(weightList, 0, sizeof(weightList)); /* is not necessary, even though some analyzer complain ... */
@ -1418,8 +1413,10 @@ size_t HUF_readDTableX6 (U32* DTable, const void* src, size_t srcSize)
/* fill tables */
{
HUF_DDescX6* DDescription = (HUF_DDescX6*)(DTable+1);
HUF_DSeqX6* DSequence = (HUF_DSeqX6*)(DTable + 1 + ((size_t)1<<(memLog-1)));
void* ddPtr = DTable+1;
HUF_DDescX6* DDescription = (HUF_DDescX6*)ddPtr;
void* dsPtr = DTable + 1 + ((size_t)1<<(memLog-1));
HUF_DSeqX6* DSequence = (HUF_DSeqX6*)dsPtr;
HUF_DSeqX6 DSeq;
HUF_DDescX6 DDesc;
DSeq.sequence = 0;
@ -1478,8 +1475,10 @@ static U32 HUF_decodeLastSymbolsX6(void* op, const U32 maxL, BIT_DStream_t* DStr
static inline size_t HUF_decodeStreamX6(BYTE* p, BIT_DStream_t* bitDPtr, BYTE* const pEnd, const U32* DTable, const U32 dtLog)
{
const HUF_DDescX6* dd = (const HUF_DDescX6*)(DTable+1);
const HUF_DSeqX6* ds = (const HUF_DSeqX6*)(DTable + 1 + ((size_t)1<<(dtLog-1)));
const void* const ddPtr = DTable+1;
const HUF_DDescX6* dd = (const HUF_DDescX6*)ddPtr;
const void* const dsPtr = DTable + 1 + ((size_t)1<<(dtLog-1));
const HUF_DSeqX6* ds = (const HUF_DSeqX6*)dsPtr;
BYTE* const pStart = p;
/* up to 16 symbols at a time */
@ -1557,8 +1556,10 @@ size_t HUF_decompress4X6_usingDTable(
BYTE* const oend = ostart + dstSize;
const U32 dtLog = DTable[0];
const HUF_DDescX6* dd = (const HUF_DDescX6*)(DTable+1);
const HUF_DSeqX6* ds = (const HUF_DSeqX6*)(DTable + 1 + ((size_t)1<<(dtLog-1)));
const void* const ddPtr = DTable+1;
const HUF_DDescX6* dd = (const HUF_DDescX6*)ddPtr;
const void* const dsPtr = DTable + 1 + ((size_t)1<<(dtLog-1));
const HUF_DSeqX6* ds = (const HUF_DSeqX6*)dsPtr;
size_t errorCode;
/* Init */

16
lib/huff0.h
View File

@ -40,25 +40,25 @@ extern "C" {
#endif
/******************************************
/* ****************************************
* Dependency
******************************************/
#include <stddef.h> /* size_t */
/******************************************
/* ****************************************
* Huff0 simple functions
******************************************/
size_t HUF_compress(void* dst, size_t maxDstSize,
const void* src, size_t srcSize);
size_t HUF_decompress(void* dst, size_t dstSize,
const void* cSrc, size_t cSrcSize);
/*
/*!
HUF_compress():
Compress content of buffer 'src', of size 'srcSize', into destination buffer 'dst'.
'dst' buffer must be already allocated. Compression runs faster if maxDstSize >= HUF_compressBound(srcSize).
Note : srcSize must be <= 128 KB
return : size of compressed data (<= maxDstSize)
@return : size of compressed data (<= maxDstSize)
Special values : if return == 0, srcData is not compressible => Nothing is stored within dst !!!
if return == 1, srcData is a single repeated byte symbol (RLE compression)
if HUF_isError(return), compression failed (more details using HUF_getErrorName())
@ -68,12 +68,12 @@ HUF_decompress():
into already allocated destination buffer 'dst', of size 'dstSize'.
'dstSize' must be the exact size of original (uncompressed) data.
Note : in contrast with FSE, HUF_decompress can regenerate RLE (cSrcSize==1) and uncompressed (cSrcSize==dstSize) data, because it knows size to regenerate.
return : size of regenerated data (== dstSize)
or an error code, which can be tested using HUF_isError()
@return : size of regenerated data (== dstSize)
or an error code, which can be tested using HUF_isError()
*/
/******************************************
/* ****************************************
* Tool functions
******************************************/
size_t HUF_compressBound(size_t size); /* maximum compressed size */
@ -83,7 +83,7 @@ unsigned HUF_isError(size_t code); /* tells if a return value is an er
const char* HUF_getErrorName(size_t code); /* provides error code string (useful for debugging) */
/******************************************
/* ****************************************
* Advanced functions
******************************************/
size_t HUF_compress2 (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog);

51
lib/huff0_static.h
View File

@ -40,13 +40,13 @@ extern "C" {
#endif
/******************************************
/* ****************************************
* Dependency
******************************************/
#include "huff0.h"
/******************************************
/* ****************************************
* Static allocation macros
******************************************/
/* Huff0 buffer bounds */
@ -64,14 +64,57 @@ extern "C" {
unsigned int DTable[HUF_DTABLE_SIZE(maxTableLog) * 3 / 2] = { maxTableLog }
/******************************************
* Advanced functions
/* ****************************************
* Advanced decompression functions
******************************************/
size_t HUF_decompress4X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* single-symbol decoder */
size_t HUF_decompress4X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* double-symbols decoder */
size_t HUF_decompress4X6 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* quad-symbols decoder */
/* ****************************************
* Huff0 detailed API
******************************************/
/*!
HUF_compress() does the following:
1. count symbol occurrence from source[] into table count[] using FSE_count()
2. build Huffman table from count using HUF_buildCTable()
3. save Huffman table to memory buffer using HUF_writeCTable()
4. encode the data stream using HUF_compress_usingCTable()
The following API allows targeting specific sub-functions for advanced tasks.
For example, it's possible to compress several blocks using the same 'CTable',
or to save and regenerate 'CTable' using external methods.
*/
/* FSE_count() : find it within "fse.h" */
typedef struct HUF_CElt_s HUF_CElt; /* incomplete type */
size_t HUF_buildCTable (HUF_CElt* tree, const unsigned* count, unsigned maxSymbolValue, unsigned maxNbBits);
size_t HUF_writeCTable (void* dst, size_t maxDstSize, const HUF_CElt* tree, unsigned maxSymbolValue, unsigned huffLog);
size_t HUF_compress_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable);
/*!
HUF_decompress() does the following:
1. select the decompression algorithm (X2, X4, X6) based on pre-computed heuristics
2. build Huffman table from save, using HUF_readDTableXn()
3. decode 1 or 4 segments in parallel using HUF_decompressSXn_usingDTable
*/
size_t HUF_readDTableX2 (unsigned short* DTable, const void* src, size_t srcSize);
size_t HUF_readDTableX4 (unsigned* DTable, const void* src, size_t srcSize);
size_t HUF_readDTableX6 (unsigned* DTable, const void* src, size_t srcSize);
size_t HUF_decompress1X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* single-symbol decoder */
size_t HUF_decompress1X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* double-symbol decoder */
size_t HUF_decompress1X6 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* quad-symbol decoder */
size_t HUF_decompress4X2_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const unsigned short* DTable);
size_t HUF_decompress4X4_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const unsigned* DTable);
size_t HUF_decompress4X6_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const unsigned* DTable);
#if defined (__cplusplus)
}
#endif

42
lib/legacy/zstd_v01.c
View File

@ -418,8 +418,9 @@ typedef struct {
static size_t FSE_FUNCTION_NAME(FSE_buildDTable, FSE_FUNCTION_EXTENSION)
(FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog)
{
FSE_DTableHeader* const DTableH = (FSE_DTableHeader*)dt;
FSE_DECODE_TYPE* const tableDecode = (FSE_DECODE_TYPE*) (dt+1); /* because dt is unsigned, 32-bits aligned on 32-bits */
void* ptr = dt;
FSE_DTableHeader* const DTableH = (FSE_DTableHeader*)ptr;
FSE_DECODE_TYPE* const tableDecode = (FSE_DECODE_TYPE*)(ptr) + 1; /* because dt is unsigned, 32-bits aligned on 32-bits */
const U32 tableSize = 1 << tableLog;
const U32 tableMask = tableSize-1;
const U32 step = FSE_tableStep(tableSize);
@ -615,8 +616,9 @@ static size_t FSE_readNCount (short* normalizedCounter, unsigned* maxSVPtr, unsi
*********************************************************/
static size_t FSE_buildDTable_rle (FSE_DTable* dt, BYTE symbolValue)
{
FSE_DTableHeader* const DTableH = (FSE_DTableHeader*)dt;
FSE_decode_t* const cell = (FSE_decode_t*)(dt + 1); /* because dt is unsigned */
void* ptr = dt;
FSE_DTableHeader* const DTableH = (FSE_DTableHeader*)ptr;
FSE_decode_t* const cell = (FSE_decode_t*)(ptr) + 1; /* because dt is unsigned */
DTableH->tableLog = 0;
DTableH->fastMode = 0;
@ -631,8 +633,9 @@ static size_t FSE_buildDTable_rle (FSE_DTable* dt, BYTE symbolValue)
static size_t FSE_buildDTable_raw (FSE_DTable* dt, unsigned nbBits)
{
FSE_DTableHeader* const DTableH = (FSE_DTableHeader*)dt;
FSE_decode_t* const dinfo = (FSE_decode_t*)(dt + 1); /* because dt is unsigned */
void* ptr = dt;
FSE_DTableHeader* const DTableH = (FSE_DTableHeader*)ptr;
FSE_decode_t* const dinfo = (FSE_decode_t*)(ptr) + 1; /* because dt is unsigned */
const unsigned tableSize = 1 << nbBits;
const unsigned tableMask = tableSize - 1;
const unsigned maxSymbolValue = tableMask;
@ -701,7 +704,7 @@ static size_t FSE_initDStream(FSE_DStream_t* bitD, const void* srcBuffer, size_t
}
/* FSE_lookBits
/*!FSE_lookBits
* Provides next n bits from the bitContainer.
* bitContainer is not modified (bits are still present for next read/look)
* On 32-bits, maxNbBits==25
@ -726,7 +729,7 @@ static void FSE_skipBits(FSE_DStream_t* bitD, U32 nbBits)
}
/* FSE_readBits
/*!FSE_readBits
* Read next n bits from the bitContainer.
* On 32-bits, don't read more than maxNbBits==25
* On 64-bits, don't read more than maxNbBits==57
@ -782,7 +785,8 @@ static unsigned FSE_reloadDStream(FSE_DStream_t* bitD)
static void FSE_initDState(FSE_DState_t* DStatePtr, FSE_DStream_t* bitD, const FSE_DTable* dt)
{
const FSE_DTableHeader* const DTableH = (const FSE_DTableHeader*)dt;
const void* ptr = dt;
const FSE_DTableHeader* const DTableH = (const FSE_DTableHeader*)ptr;
DStatePtr->state = FSE_readBits(bitD, DTableH->tableLog);
FSE_reloadDStream(bitD);
DStatePtr->table = dt + 1;
@ -898,11 +902,11 @@ static size_t FSE_decompress_usingDTable(void* dst, size_t originalSize,
const void* cSrc, size_t cSrcSize,
const FSE_DTable* dt)
{
const FSE_DTableHeader* DTableH = (const FSE_DTableHeader*)dt;
const U32 fastMode = DTableH->fastMode;
FSE_DTableHeader DTableH;
memcpy(&DTableH, dt, sizeof(DTableH)); /* memcpy() into local variable, to avoid strict aliasing warning */
/* select fast mode (static) */
if (fastMode) return FSE_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 1);
if (DTableH.fastMode) return FSE_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 1);
return FSE_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 0);
}
@ -935,7 +939,7 @@ static size_t FSE_decompress(void* dst, size_t maxDstSize, const void* cSrc, siz
/*********************************************************
/* *******************************************************
* Huff0 : Huffman block compression
*********************************************************/
#define HUF_MAX_SYMBOL_VALUE 255
@ -959,7 +963,7 @@ typedef struct nodeElt_s {
} nodeElt;
/*********************************************************
/* *******************************************************
* Huff0 : Huffman block decompression
*********************************************************/
typedef struct {
@ -978,7 +982,8 @@ static size_t HUF_readDTable (U16* DTable, const void* src, size_t srcSize)
size_t oSize;
U32 n;
U32 nextRankStart;
HUF_DElt* const dt = (HUF_DElt*)(DTable + 1);
void* ptr = DTable+1;
HUF_DElt* const dt = (HUF_DElt*)ptr;
FSE_STATIC_ASSERT(sizeof(HUF_DElt) == sizeof(U16)); /* if compilation fails here, assertion is false */
//memset(huffWeight, 0, sizeof(huffWeight)); /* should not be necessary, but some analyzer complain ... */
@ -1082,7 +1087,8 @@ static size_t HUF_decompress_usingDTable( /* -3% slower when non static */
BYTE* const omax = op + maxDstSize;
BYTE* const olimit = omax-15;
const HUF_DElt* const dt = (const HUF_DElt*)(DTable+1);
const void* ptr = DTable;
const HUF_DElt* const dt = (const HUF_DElt*)(ptr)+1;
const U32 dtLog = DTable[0];
size_t errorCode;
U32 reloadStatus;
@ -1988,8 +1994,8 @@ static size_t ZSTD_decompressBlock(
{
/* blockType == blockCompressed, srcSize is trusted */
const BYTE* ip = (const BYTE*)src;
const BYTE* litPtr;
size_t litSize;
const BYTE* litPtr = NULL;
size_t litSize = 0;
size_t errorCode;
/* Decode literals sub-block */

61
lib/legacy/zstd_v02.c
View File

@ -891,8 +891,9 @@ typedef struct
MEM_STATIC void FSE_initDState(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD, const FSE_DTable* dt)
{
const FSE_DTableHeader* const DTableH = (const FSE_DTableHeader*)dt;
DStatePtr->state = BIT_readBits(bitD, DTableH->tableLog);
FSE_DTableHeader DTableH;
memcpy(&DTableH, dt, sizeof(DTableH));
DStatePtr->state = BIT_readBits(bitD, DTableH.tableLog);
BIT_reloadDStream(bitD);
DStatePtr->table = dt + 1;
}
@ -1272,8 +1273,9 @@ static U32 FSE_tableStep(U32 tableSize) { return (tableSize>>1) + (tableSize>>3)
static size_t FSE_FUNCTION_NAME(FSE_buildDTable, FSE_FUNCTION_EXTENSION)
(FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog)
{
FSE_DTableHeader* const DTableH = (FSE_DTableHeader*)dt;
FSE_DECODE_TYPE* const tableDecode = (FSE_DECODE_TYPE*) (dt+1); /* because dt is unsigned, 32-bits aligned on 32-bits */
void* ptr = dt+1;
FSE_DECODE_TYPE* const tableDecode = (FSE_DECODE_TYPE*)ptr;
FSE_DTableHeader DTableH;
const U32 tableSize = 1 << tableLog;
const U32 tableMask = tableSize-1;
const U32 step = FSE_tableStep(tableSize);
@ -1289,7 +1291,7 @@ static size_t FSE_FUNCTION_NAME(FSE_buildDTable, FSE_FUNCTION_EXTENSION)
if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge);
/* Init, lay down lowprob symbols */
DTableH[0].tableLog = (U16)tableLog;
DTableH.tableLog = (U16)tableLog;
for (s=0; s<=maxSymbolValue; s++)
{
if (normalizedCounter[s]==-1)
@ -1330,7 +1332,8 @@ static size_t FSE_FUNCTION_NAME(FSE_buildDTable, FSE_FUNCTION_EXTENSION)
}
}
DTableH->fastMode = (U16)noLarge;
DTableH.fastMode = (U16)noLarge;
memcpy(dt, &DTableH, sizeof(DTableH)); /* memcpy(), to avoid strict aliasing warnings */
return 0;
}
@ -1468,8 +1471,9 @@ static size_t FSE_readNCount (short* normalizedCounter, unsigned* maxSVPtr, unsi
*********************************************************/
static size_t FSE_buildDTable_rle (FSE_DTable* dt, BYTE symbolValue)
{
FSE_DTableHeader* const DTableH = (FSE_DTableHeader*)dt;
FSE_decode_t* const cell = (FSE_decode_t*)(dt + 1); /* because dt is unsigned */
void* ptr = dt;
FSE_DTableHeader* const DTableH = (FSE_DTableHeader*)ptr;
FSE_decode_t* const cell = (FSE_decode_t*)(ptr) + 1; /* because dt is unsigned */
DTableH->tableLog = 0;
DTableH->fastMode = 0;
@ -1484,8 +1488,9 @@ static size_t FSE_buildDTable_rle (FSE_DTable* dt, BYTE symbolValue)
static size_t FSE_buildDTable_raw (FSE_DTable* dt, unsigned nbBits)
{
FSE_DTableHeader* const DTableH = (FSE_DTableHeader*)dt;
FSE_decode_t* const dinfo = (FSE_decode_t*)(dt + 1); /* because dt is unsigned */
void* ptr = dt;
FSE_DTableHeader* const DTableH = (FSE_DTableHeader*)ptr;
FSE_decode_t* const dinfo = (FSE_decode_t*)(ptr) + 1; /* because dt is unsigned */
const unsigned tableSize = 1 << nbBits;
const unsigned tableMask = tableSize - 1;
const unsigned maxSymbolValue = tableMask;
@ -1581,11 +1586,11 @@ static size_t FSE_decompress_usingDTable(void* dst, size_t originalSize,
const void* cSrc, size_t cSrcSize,
const FSE_DTable* dt)
{
const FSE_DTableHeader* DTableH = (const FSE_DTableHeader*)dt;
const U32 fastMode = DTableH->fastMode;
FSE_DTableHeader DTableH;
memcpy(&DTableH, dt, sizeof(DTableH));
/* select fast mode (static) */
if (fastMode) return FSE_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 1);
if (DTableH.fastMode) return FSE_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 1);
return FSE_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 0);
}
@ -1810,7 +1815,8 @@ static size_t HUF_readDTableX2 (U16* DTable, const void* src, size_t srcSize)
U32 nbSymbols = 0;
U32 n;
U32 nextRankStart;
HUF_DEltX2* const dt = (HUF_DEltX2*)(DTable + 1);
void* ptr = DTable+1;
HUF_DEltX2* const dt = (HUF_DEltX2*)ptr;
HUF_STATIC_ASSERT(sizeof(HUF_DEltX2) == sizeof(U16)); /* if compilation fails here, assertion is false */
//memset(huffWeight, 0, sizeof(huffWeight)); /* is not necessary, even though some analyzer complain ... */
@ -1903,7 +1909,8 @@ static size_t HUF_decompress4X2_usingDTable(
BYTE* const ostart = (BYTE*) dst;
BYTE* const oend = ostart + dstSize;
const HUF_DEltX2* const dt = ((const HUF_DEltX2*)DTable) +1;
const void* ptr = DTable;
const HUF_DEltX2* const dt = ((const HUF_DEltX2*)ptr) +1;
const U32 dtLog = DTable[0];
size_t errorCode;
@ -2112,7 +2119,8 @@ static size_t HUF_readDTableX4 (U32* DTable, const void* src, size_t srcSize)
const U32 memLog = DTable[0];
const BYTE* ip = (const BYTE*) src;
size_t iSize = ip[0];
HUF_DEltX4* const dt = ((HUF_DEltX4*)DTable) + 1;
void* ptr = DTable;
HUF_DEltX4* const dt = ((HUF_DEltX4*)ptr) + 1;
HUF_STATIC_ASSERT(sizeof(HUF_DEltX4) == sizeof(U32)); /* if compilation fails here, assertion is false */
if (memLog > HUF_ABSOLUTEMAX_TABLELOG) return ERROR(tableLog_tooLarge);
@ -2262,7 +2270,8 @@ static size_t HUF_decompress4X4_usingDTable(
BYTE* const ostart = (BYTE*) dst;
BYTE* const oend = ostart + dstSize;
const HUF_DEltX4* const dt = ((const HUF_DEltX4*)DTable) +1;
const void* ptr = DTable;
const HUF_DEltX4* const dt = ((const HUF_DEltX4*)ptr) +1;
const U32 dtLog = DTable[0];
size_t errorCode;
@ -2510,8 +2519,10 @@ static size_t HUF_readDTableX6 (U32* DTable, const void* src, size_t srcSize)
/* fill tables */
{
HUF_DDescX6* DDescription = (HUF_DDescX6*)(DTable+1);
HUF_DSeqX6* DSequence = (HUF_DSeqX6*)(DTable + 1 + ((size_t)1<<(memLog-1)));
void* ptr = DTable+1;
HUF_DDescX6* DDescription = (HUF_DDescX6*)(ptr);
void* dSeqStart = DTable + 1 + ((size_t)1<<(memLog-1));
HUF_DSeqX6* DSequence = (HUF_DSeqX6*)(dSeqStart);
HUF_DSeqX6 DSeq;
HUF_DDescX6 DDesc;
DSeq.sequence = 0;
@ -2570,8 +2581,10 @@ static U32 HUF_decodeLastSymbolsX6(void* op, const U32 maxL, BIT_DStream_t* DStr
static inline size_t HUF_decodeStreamX6(BYTE* p, BIT_DStream_t* bitDPtr, BYTE* const pEnd, const U32* DTable, const U32 dtLog)
{
const HUF_DDescX6* dd = (const HUF_DDescX6*)(DTable+1);
const HUF_DSeqX6* ds = (const HUF_DSeqX6*)(DTable + 1 + ((size_t)1<<(dtLog-1)));
const void* ddPtr = DTable+1;
const HUF_DDescX6* dd = (const HUF_DDescX6*)(ddPtr);
const void* dsPtr = DTable + 1 + ((size_t)1<<(dtLog-1));
const HUF_DSeqX6* ds = (const HUF_DSeqX6*)(dsPtr);
BYTE* const pStart = p;
/* up to 16 symbols at a time */
@ -2611,8 +2624,10 @@ static size_t HUF_decompress4X6_usingDTable(
BYTE* const oend = ostart + dstSize;
const U32 dtLog = DTable[0];
const HUF_DDescX6* dd = (const HUF_DDescX6*)(DTable+1);
const HUF_DSeqX6* ds = (const HUF_DSeqX6*)(DTable + 1 + ((size_t)1<<(dtLog-1)));
const void* ddPtr = DTable+1;
const HUF_DDescX6* dd = (const HUF_DDescX6*)(ddPtr);
const void* dsPtr = DTable + 1 + ((size_t)1<<(dtLog-1));
const HUF_DSeqX6* ds = (const HUF_DSeqX6*)(dsPtr);
size_t errorCode;
/* Init */

61
lib/legacy/zstd_v03.c
View File

@ -891,8 +891,9 @@ typedef struct
MEM_STATIC void FSE_initDState(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD, const FSE_DTable* dt)
{
const FSE_DTableHeader* const DTableH = (const FSE_DTableHeader*)dt;
DStatePtr->state = BIT_readBits(bitD, DTableH->tableLog);
FSE_DTableHeader DTableH;
memcpy(&DTableH, dt, sizeof(DTableH));
DStatePtr->state = BIT_readBits(bitD, DTableH.tableLog);
BIT_reloadDStream(bitD);
DStatePtr->table = dt + 1;
}
@ -1272,8 +1273,9 @@ static U32 FSE_tableStep(U32 tableSize) { return (tableSize>>1) + (tableSize>>3)
static size_t FSE_FUNCTION_NAME(FSE_buildDTable, FSE_FUNCTION_EXTENSION)
(FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog)
{
FSE_DTableHeader* const DTableH = (FSE_DTableHeader*)dt;
FSE_DECODE_TYPE* const tableDecode = (FSE_DECODE_TYPE*) (dt+1); /* because dt is unsigned, 32-bits aligned on 32-bits */
void* ptr = dt+1;
FSE_DTableHeader DTableH;
FSE_DECODE_TYPE* const tableDecode = (FSE_DECODE_TYPE*)ptr;
const U32 tableSize = 1 << tableLog;
const U32 tableMask = tableSize-1;
const U32 step = FSE_tableStep(tableSize);
@ -1289,7 +1291,7 @@ static size_t FSE_FUNCTION_NAME(FSE_buildDTable, FSE_FUNCTION_EXTENSION)
if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge);
/* Init, lay down lowprob symbols */
DTableH[0].tableLog = (U16)tableLog;
DTableH.tableLog = (U16)tableLog;
for (s=0; s<=maxSymbolValue; s++)
{
if (normalizedCounter[s]==-1)
@ -1330,7 +1332,8 @@ static size_t FSE_FUNCTION_NAME(FSE_buildDTable, FSE_FUNCTION_EXTENSION)
}
}
DTableH->fastMode = (U16)noLarge;
DTableH.fastMode = (U16)noLarge;
memcpy(dt, &DTableH, sizeof(DTableH));
return 0;
}
@ -1468,8 +1471,9 @@ static size_t FSE_readNCount (short* normalizedCounter, unsigned* maxSVPtr, unsi
*********************************************************/
static size_t FSE_buildDTable_rle (FSE_DTable* dt, BYTE symbolValue)
{
FSE_DTableHeader* const DTableH = (FSE_DTableHeader*)dt;
FSE_decode_t* const cell = (FSE_decode_t*)(dt + 1); /* because dt is unsigned */
void* ptr = dt;
FSE_DTableHeader* const DTableH = (FSE_DTableHeader*)ptr;
FSE_decode_t* const cell = (FSE_decode_t*)(ptr) + 1;
DTableH->tableLog = 0;
DTableH->fastMode = 0;
@ -1484,8 +1488,9 @@ static size_t FSE_buildDTable_rle (FSE_DTable* dt, BYTE symbolValue)
static size_t FSE_buildDTable_raw (FSE_DTable* dt, unsigned nbBits)
{
FSE_DTableHeader* const DTableH = (FSE_DTableHeader*)dt;
FSE_decode_t* const dinfo = (FSE_decode_t*)(dt + 1); /* because dt is unsigned */
void* ptr = dt;
FSE_DTableHeader* const DTableH = (FSE_DTableHeader*)ptr;
FSE_decode_t* const dinfo = (FSE_decode_t*)(ptr) + 1;
const unsigned tableSize = 1 << nbBits;
const unsigned tableMask = tableSize - 1;
const unsigned maxSymbolValue = tableMask;
@ -1581,11 +1586,11 @@ static size_t FSE_decompress_usingDTable(void* dst, size_t originalSize,
const void* cSrc, size_t cSrcSize,
const FSE_DTable* dt)
{
const FSE_DTableHeader* DTableH = (const FSE_DTableHeader*)dt;
const U32 fastMode = DTableH->fastMode;
FSE_DTableHeader DTableH;
memcpy(&DTableH, dt, sizeof(DTableH));
/* select fast mode (static) */
if (fastMode) return FSE_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 1);
if (DTableH.fastMode) return FSE_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 1);
return FSE_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 0);
}
@ -1810,7 +1815,8 @@ static size_t HUF_readDTableX2 (U16* DTable, const void* src, size_t srcSize)
U32 nbSymbols = 0;
U32 n;
U32 nextRankStart;
HUF_DEltX2* const dt = (HUF_DEltX2*)(DTable + 1);
void* ptr = DTable+1;
HUF_DEltX2* const dt = (HUF_DEltX2*)(ptr);
HUF_STATIC_ASSERT(sizeof(HUF_DEltX2) == sizeof(U16)); /* if compilation fails here, assertion is false */
//memset(huffWeight, 0, sizeof(huffWeight)); /* is not necessary, even though some analyzer complain ... */
@ -1903,7 +1909,8 @@ static size_t HUF_decompress4X2_usingDTable(
BYTE* const ostart = (BYTE*) dst;
BYTE* const oend = ostart + dstSize;
const HUF_DEltX2* const dt = ((const HUF_DEltX2*)DTable) +1;
const void* ptr = DTable;
const HUF_DEltX2* const dt = ((const HUF_DEltX2*)ptr) +1;
const U32 dtLog = DTable[0];
size_t errorCode;
@ -2112,7 +2119,8 @@ static size_t HUF_readDTableX4 (U32* DTable, const void* src, size_t srcSize)
const U32 memLog = DTable[0];
const BYTE* ip = (const BYTE*) src;
size_t iSize = ip[0];
HUF_DEltX4* const dt = ((HUF_DEltX4*)DTable) + 1;
void* ptr = DTable;
HUF_DEltX4* const dt = ((HUF_DEltX4*)ptr) + 1;
HUF_STATIC_ASSERT(sizeof(HUF_DEltX4) == sizeof(U32)); /* if compilation fails here, assertion is false */
if (memLog > HUF_ABSOLUTEMAX_TABLELOG) return ERROR(tableLog_tooLarge);
@ -2262,7 +2270,8 @@ static size_t HUF_decompress4X4_usingDTable(
BYTE* const ostart = (BYTE*) dst;
BYTE* const oend = ostart + dstSize;
const HUF_DEltX4* const dt = ((const HUF_DEltX4*)DTable) +1;
const void* ptr = DTable;
const HUF_DEltX4* const dt = ((const HUF_DEltX4*)ptr) +1;
const U32 dtLog = DTable[0];
size_t errorCode;
@ -2510,8 +2519,10 @@ static size_t HUF_readDTableX6 (U32* DTable, const void* src, size_t srcSize)
/* fill tables */
{
HUF_DDescX6* DDescription = (HUF_DDescX6*)(DTable+1);
HUF_DSeqX6* DSequence = (HUF_DSeqX6*)(DTable + 1 + ((size_t)1<<(memLog-1)));
void* ddPtr = DTable+1;
HUF_DDescX6* DDescription = (HUF_DDescX6*)(ddPtr);
void* dsPtr = DTable + 1 + ((size_t)1<<(memLog-1));
HUF_DSeqX6* DSequence = (HUF_DSeqX6*)(dsPtr);
HUF_DSeqX6 DSeq;
HUF_DDescX6 DDesc;
DSeq.sequence = 0;
@ -2570,8 +2581,10 @@ static U32 HUF_decodeLastSymbolsX6(void* op, const U32 maxL, BIT_DStream_t* DStr
static inline size_t HUF_decodeStreamX6(BYTE* p, BIT_DStream_t* bitDPtr, BYTE* const pEnd, const U32* DTable, const U32 dtLog)
{
const HUF_DDescX6* dd = (const HUF_DDescX6*)(DTable+1);
const HUF_DSeqX6* ds = (const HUF_DSeqX6*)(DTable + 1 + ((size_t)1<<(dtLog-1)));
const void* ddPtr = DTable+1;
const HUF_DDescX6* dd = (const HUF_DDescX6*)(ddPtr);
const void* dsPtr = DTable + 1 + ((size_t)1<<(dtLog-1));
const HUF_DSeqX6* ds = (const HUF_DSeqX6*)(dsPtr);
BYTE* const pStart = p;
/* up to 16 symbols at a time */
@ -2611,8 +2624,10 @@ static size_t HUF_decompress4X6_usingDTable(
BYTE* const oend = ostart + dstSize;
const U32 dtLog = DTable[0];
const HUF_DDescX6* dd = (const HUF_DDescX6*)(DTable+1);
const HUF_DSeqX6* ds = (const HUF_DSeqX6*)(DTable + 1 + ((size_t)1<<(dtLog-1)));
const void* ddPtr = DTable+1;
const HUF_DDescX6* dd = (const HUF_DDescX6*)(ddPtr);
const void* dsPtr = DTable + 1 + ((size_t)1<<(dtLog-1));
const HUF_DSeqX6* ds = (const HUF_DSeqX6*)(dsPtr);
size_t errorCode;
/* Init */

18
lib/zstd.h
View File

@ -62,7 +62,7 @@ extern "C" {
***************************************/
#define ZSTD_VERSION_MAJOR 0 /* for breaking interface changes */
#define ZSTD_VERSION_MINOR 4 /* for new (non-breaking) interface capabilities */
#define ZSTD_VERSION_RELEASE 5 /* for tweaks, bug-fixes, or development */
#define ZSTD_VERSION_RELEASE 6 /* for tweaks, bug-fixes, or development */
#define ZSTD_VERSION_NUMBER (ZSTD_VERSION_MAJOR *100*100 + ZSTD_VERSION_MINOR *100 + ZSTD_VERSION_RELEASE)
ZSTDLIB_API unsigned ZSTD_versionNumber (void);
@ -107,16 +107,24 @@ ZSTDLIB_API const char* ZSTD_getErrorName(size_t code); /** provides error co
/* *************************************
* Advanced functions
***************************************/
/** Compression context management */
typedef struct ZSTD_CCtx_s ZSTD_CCtx; /* incomplete type */
ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx(void);
ZSTDLIB_API size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx);
/**
ZSTD_compressCCtx() :
Same as ZSTD_compress(), but requires a ZSTD_CCtx working space already allocated
*/
/** ZSTD_compressCCtx() :
Same as ZSTD_compress(), but requires an already allocated ZSTD_CCtx (see ZSTD_createCCtx()) */
ZSTDLIB_API size_t ZSTD_compressCCtx(ZSTD_CCtx* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize, int compressionLevel);
/** Decompression context management */
typedef struct ZSTD_DCtx_s ZSTD_DCtx;
ZSTDLIB_API ZSTD_DCtx* ZSTD_createDCtx(void);
ZSTDLIB_API size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx);
/** ZSTD_decompressDCtx
* Same as ZSTD_decompress(), but requires an already allocated ZSTD_DCtx (see ZSTD_createDCtx()) */
ZSTDLIB_API size_t ZSTD_decompressDCtx(ZSTD_DCtx* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize);
#if defined (__cplusplus)
}

2
lib/zstd_buffered.c
View File

@ -143,7 +143,7 @@ size_t ZBUFF_compressInit_advanced(ZBUFF_CCtx* zbc, ZSTD_parameters params)
if (zbc->outBuff == NULL) return ERROR(memory_allocation);
}
zbc->outBuffContentSize = ZSTD_compressBegin_advanced(zbc->zc, zbc->outBuff, zbc->outBuffSize, params);
zbc->outBuffContentSize = ZSTD_compressBegin_advanced(zbc->zc, params);
if (ZSTD_isError(zbc->outBuffContentSize)) return zbc->outBuffContentSize;
zbc->inToCompress = 0;

428
lib/zstd_compress.c
View File

@ -65,7 +65,7 @@
/* *************************************
* Constants
***************************************/
unsigned ZSTD_maxCLevel(void) { return ZSTD_MAX_CLEVEL; }
ZSTDLIB_API unsigned ZSTD_maxCLevel(void) { return ZSTD_MAX_CLEVEL; }
static const U32 g_searchStrength = 8;
@ -109,10 +109,14 @@ struct ZSTD_CCtx_s
U32 dictLimit; /* below that point, need extDict */
U32 lowLimit; /* below that point, no more data */
U32 nextToUpdate; /* index from which to continue dictionary update */
U32 stage;
ZSTD_parameters params;
void* workSpace;
size_t workSpaceSize;
size_t blockSize;
size_t hbSize;
char headerBuffer[ZSTD_frameHeaderSize_max];
seqStore_t seqStore; /* sequences storage ptrs */
U32* hashTable;
@ -205,6 +209,8 @@ static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc,
zc->seqStore.litLengthStart = zc->seqStore.litStart + blockSize;
zc->seqStore.matchLengthStart = zc->seqStore.litLengthStart + (blockSize>>2);
zc->seqStore.dumpsStart = zc->seqStore.matchLengthStart + (blockSize>>2);
zc->hbSize = 0;
zc->stage = 0;
return 0;
}
@ -761,7 +767,8 @@ size_t ZSTD_compressBlock_fast_generic(ZSTD_CCtx* zc,
const BYTE* const istart = (const BYTE*)src;
const BYTE* ip = istart;
const BYTE* anchor = istart;
const BYTE* const lowest = base + zc->dictLimit;
const U32 lowIndex = zc->dictLimit;
const BYTE* const lowest = base + lowIndex;
const BYTE* const iend = istart + srcSize;
const BYTE* const ilimit = iend - 8;
@ -772,9 +779,9 @@ size_t ZSTD_compressBlock_fast_generic(ZSTD_CCtx* zc,
ZSTD_resetSeqStore(seqStorePtr);
if (ip < lowest+4)
{
hashTable[ZSTD_hashPtr(lowest+1, hBits, mls)] = zc->dictLimit+1;
hashTable[ZSTD_hashPtr(lowest+2, hBits, mls)] = zc->dictLimit+2;
hashTable[ZSTD_hashPtr(lowest+3, hBits, mls)] = zc->dictLimit+3;
hashTable[ZSTD_hashPtr(lowest+1, hBits, mls)] = lowIndex+1;
hashTable[ZSTD_hashPtr(lowest+2, hBits, mls)] = lowIndex+2;
hashTable[ZSTD_hashPtr(lowest+3, hBits, mls)] = lowIndex+3;
ip = lowest+4;
}
@ -784,10 +791,12 @@ size_t ZSTD_compressBlock_fast_generic(ZSTD_CCtx* zc,
size_t mlCode;
size_t offset;
const size_t h = ZSTD_hashPtr(ip, hBits, mls);
const BYTE* match = base + hashTable[h];
hashTable[h] = (U32)(ip-base);
const U32 matchIndex = hashTable[h];
const BYTE* match = base + matchIndex;
const U32 current = (U32)(ip-base);
hashTable[h] = current; /* update hash table */
if (MEM_read32(ip+1-offset_1) == MEM_read32(ip+1)) /* note : by construction, offset_1 <= (ip-base) */
if (MEM_read32(ip+1-offset_1) == MEM_read32(ip+1)) /* note : by construction, offset_1 <= current */
{
mlCode = ZSTD_count(ip+1+MINMATCH, ip+1+MINMATCH-offset_1, iend);
ip++;
@ -795,7 +804,7 @@ size_t ZSTD_compressBlock_fast_generic(ZSTD_CCtx* zc,
}
else
{
if ( (match <= lowest) ||
if ( (matchIndex <= lowIndex) ||
(MEM_read32(match) != MEM_read32(ip)) )
{
ip += ((ip-anchor) >> g_searchStrength) + 1;
@ -816,7 +825,7 @@ size_t ZSTD_compressBlock_fast_generic(ZSTD_CCtx* zc,
if (ip <= ilimit)
{
/* Fill Table */
hashTable[ZSTD_hashPtr(ip-(mlCode+MINMATCH)+2, hBits, mls)] = (U32)(ip-(mlCode+MINMATCH)+2-base); /* here because ip-(mlCode+MINMATCH)+2 could be > iend-8 without ip <= ilimit check*/
hashTable[ZSTD_hashPtr(base+current+2, hBits, mls)] = current+2; /* here because current+2 could be > iend-8 */
hashTable[ZSTD_hashPtr(ip-2, hBits, mls)] = (U32)(ip-2-base);
/* check immediate repcode */
while ( (ip <= ilimit)
@ -1014,191 +1023,8 @@ size_t ZSTD_compressBlock_fast_extDict(ZSTD_CCtx* ctx,
/** ZSTD_insertBt1 : add one or multiple positions to tree
* @ip : assumed <= iend-8
* @return : nb of positions added */
static U32 ZSTD_insertBt1(ZSTD_CCtx* zc, const BYTE* const ip, const U32 mls, const BYTE* const iend, U32 nbCompares)
{
U32* const hashTable = zc->hashTable;
const U32 hashLog = zc->params.hashLog;
const size_t h = ZSTD_hashPtr(ip, hashLog, mls);
U32* const bt = zc->contentTable;
const U32 btLog = zc->params.contentLog - 1;
const U32 btMask= (1 << btLog) - 1;
U32 matchIndex = hashTable[h];
size_t commonLengthSmaller=0, commonLengthLarger=0;
const BYTE* const base = zc->base;
const BYTE* match = base + matchIndex;
const U32 current = (U32)(ip-base);
const U32 btLow = btMask >= current ? 0 : current - btMask;
U32* smallerPtr = bt + 2*(current&btMask);
U32* largerPtr = bt + 2*(current&btMask) + 1;
U32 dummy32; /* to be nullified at the end */
const U32 windowLow = zc->lowLimit;
if ( (current-matchIndex == 1) /* RLE */
&& (matchIndex > windowLow)
&& (MEM_read64(match) == MEM_read64(ip)) )
{
size_t rleLength = ZSTD_count(ip+8, match+8, iend) + 8;
return (U32)(rleLength - mls);
}
hashTable[h] = current; /* Update Hash Table */
while (nbCompares-- && (matchIndex > windowLow))
{
U32* nextPtr = bt + 2*(matchIndex & btMask);
size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */
match = base + matchIndex;
if (match[matchLength] == ip[matchLength])
matchLength += ZSTD_count(ip+matchLength+1, match+matchLength+1, iend) +1;
if (ip+matchLength == iend) /* equal : no way to know if inf or sup */
break; /* drop , to guarantee consistency ; miss a bit of compression, but other solutions can corrupt the tree */
if (match[matchLength] < ip[matchLength])
{
/* match is smaller than current */
*smallerPtr = matchIndex; /* update smaller idx */
commonLengthSmaller = matchLength; /* all smaller will now have at least this guaranteed common length */
if (matchIndex <= btLow) { smallerPtr=&dummy32; break; } /* beyond tree size, stop the search */
smallerPtr = nextPtr+1; /* new "smaller" => larger of match */
matchIndex = nextPtr[1]; /* new matchIndex larger than previous (closer to current) */
}
else
{
/* match is larger than current */
*largerPtr = matchIndex;
commonLengthLarger = matchLength;
if (matchIndex <= btLow) { largerPtr=&dummy32; break; } /* beyond tree size, stop the search */
largerPtr = nextPtr;
matchIndex = nextPtr[0];
}
}
*smallerPtr = *largerPtr = 0;
return 1;
}
FORCE_INLINE /* inlining is important to hardwire a hot branch (template emulation) */
size_t ZSTD_insertBtAndFindBestMatch (
ZSTD_CCtx* zc,
const BYTE* const ip, const BYTE* const iend,
size_t* offsetPtr,
U32 nbCompares, const U32 mls)
{
U32* const hashTable = zc->hashTable;
const U32 hashLog = zc->params.hashLog;
const size_t h = ZSTD_hashPtr(ip, hashLog, mls);
U32* const bt = zc->contentTable;
const U32 btLog = zc->params.contentLog - 1;
const U32 btMask= (1 << btLog) - 1;
U32 matchIndex = hashTable[h];
size_t commonLengthSmaller=0, commonLengthLarger=0;
const BYTE* const base = zc->base;
const U32 current = (U32)(ip-base);
const U32 btLow = btMask >= current ? 0 : current - btMask;
const U32 windowLow = zc->lowLimit;
U32* smallerPtr = bt + 2*(current&btMask);
U32* largerPtr = bt + 2*(current&btMask) + 1;
size_t bestLength = 0;
U32 dummy32; /* to be nullified at the end */
hashTable[h] = current; /* Update Hash Table */
while (nbCompares-- && (matchIndex > windowLow))
{
U32* nextPtr = bt + 2*(matchIndex & btMask);
const BYTE* match = base + matchIndex;
size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */
if (match[matchLength] == ip[matchLength])
matchLength += ZSTD_count(ip+matchLength+1, match+matchLength+1, iend) +1;
if (matchLength > bestLength)
{
if ( (4*(int)(matchLength-bestLength)) > (int)(ZSTD_highbit(current-matchIndex+1) - ZSTD_highbit((U32)offsetPtr[0]+1)) )
bestLength = matchLength, *offsetPtr = current - matchIndex;
if (ip+matchLength == iend) /* equal : no way to know if inf or sup */
break; /* drop, to guarantee consistency (miss a little bit of compression) */
}
if (match[matchLength] < ip[matchLength])
{
/* match is smaller than current */
*smallerPtr = matchIndex; /* update smaller idx */
commonLengthSmaller = matchLength; /* all smaller will now have at least this guaranteed common length */
if (matchIndex <= btLow) { smallerPtr=&dummy32; break; } /* beyond tree size, stop the search */
smallerPtr = nextPtr+1; /* new "smaller" => larger of match */
matchIndex = nextPtr[1]; /* new matchIndex larger than previous (closer to current) */
}
else
{
/* match is larger than current */
*largerPtr = matchIndex;
commonLengthLarger = matchLength;
if (matchIndex <= btLow) { largerPtr=&dummy32; break; } /* beyond tree size, stop the search */
largerPtr = nextPtr;
matchIndex = nextPtr[0];
}
}
*smallerPtr = *largerPtr = 0;
zc->nextToUpdate = current+1; /* current has been inserted */
return bestLength;
}
static const BYTE* ZSTD_updateTree(ZSTD_CCtx* zc, const BYTE* const ip, const BYTE* const iend, const U32 nbCompares, const U32 mls)
{
const BYTE* const base = zc->base;
const U32 target = (U32)(ip - base);
U32 idx = zc->nextToUpdate;
for( ; idx < target ; )
idx += ZSTD_insertBt1(zc, base+idx, mls, iend, nbCompares);
zc->nextToUpdate = idx;
return base + idx;
}
/** Tree updater, providing best match */
FORCE_INLINE /* inlining is important to hardwire a hot branch (template emulation) */
size_t ZSTD_BtFindBestMatch (
ZSTD_CCtx* zc,
const BYTE* const ip, const BYTE* const iLimit,
size_t* offsetPtr,
const U32 maxNbAttempts, const U32 mls)
{
const BYTE* nextToUpdate = ZSTD_updateTree(zc, ip, iLimit, maxNbAttempts, mls);
if (nextToUpdate > ip) /* RLE data */
{ *offsetPtr = 1; return ZSTD_count(ip, ip-1, iLimit); }
return ZSTD_insertBtAndFindBestMatch(zc, ip, iLimit, offsetPtr, maxNbAttempts, mls);
}
FORCE_INLINE size_t ZSTD_BtFindBestMatch_selectMLS (
ZSTD_CCtx* zc, /* Index table will be updated */
const BYTE* ip, const BYTE* const iLimit,
size_t* offsetPtr,
const U32 maxNbAttempts, const U32 matchLengthSearch)
{
switch(matchLengthSearch)
{
default :
case 4 : return ZSTD_BtFindBestMatch(zc, ip, iLimit, offsetPtr, maxNbAttempts, 4);
case 5 : return ZSTD_BtFindBestMatch(zc, ip, iLimit, offsetPtr, maxNbAttempts, 5);
case 6 : return ZSTD_BtFindBestMatch(zc, ip, iLimit, offsetPtr, maxNbAttempts, 6);
}
}
/** ZSTD_insertBt1_extDict : add one or multiple positions to tree
* @ip : assumed <= iend-8
* @return : nb of positions added */
static U32 ZSTD_insertBt1_extDict(ZSTD_CCtx* zc, const BYTE* const ip, const U32 mls, const BYTE* const iend, U32 nbCompares)
static U32 ZSTD_insertBt1(ZSTD_CCtx* zc, const BYTE* const ip, const U32 mls, const BYTE* const iend, U32 nbCompares,
U32 extDict)
{
U32* const hashTable = zc->hashTable;
const U32 hashLog = zc->params.hashLog;
@ -1220,13 +1046,7 @@ static U32 ZSTD_insertBt1_extDict(ZSTD_CCtx* zc, const BYTE* const ip, const U32
U32* largerPtr = bt + 2*(current&btMask) + 1;
U32 dummy32; /* to be nullified at the end */
const U32 windowLow = zc->lowLimit;
if ( (current-matchIndex == 1) /* RLE */
&& (MEM_read64(match) == MEM_read64(ip)) )
{
size_t rleLength = ZSTD_count(ip+8, match+8, iend) + 8;
return (U32)(rleLength - mls);
}
U32 matchEndIdx = current+8;
hashTable[h] = current; /* Update Hash Table */
@ -1235,7 +1055,7 @@ static U32 ZSTD_insertBt1_extDict(ZSTD_CCtx* zc, const BYTE* const ip, const U32
U32* nextPtr = bt + 2*(matchIndex & btMask);
size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */
if (matchIndex+matchLength >= dictLimit)
if ((!extDict) || (matchIndex+matchLength >= dictLimit))
{
match = base + matchIndex;
if (match[matchLength] == ip[matchLength])
@ -1249,6 +1069,9 @@ static U32 ZSTD_insertBt1_extDict(ZSTD_CCtx* zc, const BYTE* const ip, const U32
match = base + matchIndex; /* to prepare for next usage of match[matchLength] */
}
if (matchLength > matchEndIdx - matchIndex)
matchEndIdx = matchIndex + (U32)matchLength;
if (ip+matchLength == iend) /* equal : no way to know if inf or sup */
break; /* drop , to guarantee consistency ; miss a bit of compression, but other solutions can corrupt the tree */
@ -1273,30 +1096,27 @@ static U32 ZSTD_insertBt1_extDict(ZSTD_CCtx* zc, const BYTE* const ip, const U32
}
*smallerPtr = *largerPtr = 0;
return 1;
return (matchEndIdx > current + 8) ? matchEndIdx - current - 8 : 1;
}
static const BYTE* ZSTD_updateTree_extDict(ZSTD_CCtx* zc, const BYTE* const ip, const BYTE* const iend, const U32 nbCompares, const U32 mls)
static void ZSTD_updateTree(ZSTD_CCtx* zc, const BYTE* const ip, const BYTE* const iend, const U32 nbCompares, const U32 mls)
{
const BYTE* const base = zc->base;
const U32 target = (U32)(ip - base);
U32 idx = zc->nextToUpdate;
for( ; idx < target ; )
idx += ZSTD_insertBt1_extDict(zc, base+idx, mls, iend, nbCompares);
zc->nextToUpdate = idx;
return base + idx;
idx += ZSTD_insertBt1(zc, base+idx, mls, iend, nbCompares, 0);
}
FORCE_INLINE /* inlining is important to hardwire a hot branch (template emulation) */
size_t ZSTD_insertBtAndFindBestMatch_extDict (
size_t ZSTD_insertBtAndFindBestMatch (
ZSTD_CCtx* zc,
const BYTE* const ip, const BYTE* const iend,
size_t* offsetPtr,
U32 nbCompares, const U32 mls)
U32 nbCompares, const U32 mls,
U32 extDict)
{
U32* const hashTable = zc->hashTable;
const U32 hashLog = zc->params.hashLog;
@ -1317,6 +1137,7 @@ size_t ZSTD_insertBtAndFindBestMatch_extDict (
U32* smallerPtr = bt + 2*(current&btMask);
U32* largerPtr = bt + 2*(current&btMask) + 1;
size_t bestLength = 0;
U32 matchEndIdx = current+8;
U32 dummy32; /* to be nullified at the end */
hashTable[h] = current; /* Update Hash Table */
@ -1327,7 +1148,7 @@ size_t ZSTD_insertBtAndFindBestMatch_extDict (
size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */
const BYTE* match;
if (matchIndex+matchLength >= dictLimit)
if ((!extDict) || (matchIndex+matchLength >= dictLimit))
{
match = base + matchIndex;
if (match[matchLength] == ip[matchLength])
@ -1343,6 +1164,8 @@ size_t ZSTD_insertBtAndFindBestMatch_extDict (
if (matchLength > bestLength)
{
if (matchLength > matchEndIdx - matchIndex)
matchEndIdx = matchIndex + (U32)matchLength;
if ( (4*(int)(matchLength-bestLength)) > (int)(ZSTD_highbit(current-matchIndex+1) - ZSTD_highbit((U32)offsetPtr[0]+1)) )
bestLength = matchLength, *offsetPtr = current - matchIndex;
if (ip+matchLength == iend) /* equal : no way to know if inf or sup */
@ -1371,10 +1194,52 @@ size_t ZSTD_insertBtAndFindBestMatch_extDict (
*smallerPtr = *largerPtr = 0;
zc->nextToUpdate = current+1; /* current has been inserted */
zc->nextToUpdate = (matchEndIdx > current + 8) ? matchEndIdx - 8 : current+1;
return bestLength;
}
/** Tree updater, providing best match */
FORCE_INLINE /* inlining is important to hardwire a hot branch (template emulation) */
size_t ZSTD_BtFindBestMatch (
ZSTD_CCtx* zc,
const BYTE* const ip, const BYTE* const iLimit,
size_t* offsetPtr,
const U32 maxNbAttempts, const U32 mls)
{
if (ip < zc->base + zc->nextToUpdate) return 0; /* skipped area */
ZSTD_updateTree(zc, ip, iLimit, maxNbAttempts, mls);
return ZSTD_insertBtAndFindBestMatch(zc, ip, iLimit, offsetPtr, maxNbAttempts, mls, 0);
}
FORCE_INLINE size_t ZSTD_BtFindBestMatch_selectMLS (
ZSTD_CCtx* zc, /* Index table will be updated */
const BYTE* ip, const BYTE* const iLimit,
size_t* offsetPtr,
const U32 maxNbAttempts, const U32 matchLengthSearch)
{
switch(matchLengthSearch)
{
default :
case 4 : return ZSTD_BtFindBestMatch(zc, ip, iLimit, offsetPtr, maxNbAttempts, 4);
case 5 : return ZSTD_BtFindBestMatch(zc, ip, iLimit, offsetPtr, maxNbAttempts, 5);
case 6 : return ZSTD_BtFindBestMatch(zc, ip, iLimit, offsetPtr, maxNbAttempts, 6);
}
}
static void ZSTD_updateTree_extDict(ZSTD_CCtx* zc, const BYTE* const ip, const BYTE* const iend, const U32 nbCompares, const U32 mls)
{
const BYTE* const base = zc->base;
const U32 target = (U32)(ip - base);
U32 idx = zc->nextToUpdate;
for( ; idx < target ; )
idx += ZSTD_insertBt1(zc, base+idx, mls, iend, nbCompares, 1);
}
/** Tree updater, providing best match */
FORCE_INLINE /* inlining is important to hardwire a hot branch (template emulation) */
size_t ZSTD_BtFindBestMatch_extDict (
@ -1383,10 +1248,9 @@ size_t ZSTD_BtFindBestMatch_extDict (
size_t* offsetPtr,
const U32 maxNbAttempts, const U32 mls)
{
const BYTE* nextToUpdate = ZSTD_updateTree_extDict(zc, ip, iLimit, maxNbAttempts, mls);
if (nextToUpdate > ip) /* RLE data */
{ *offsetPtr = 1; return ZSTD_count(ip, ip-1, iLimit); }
return ZSTD_insertBtAndFindBestMatch_extDict(zc, ip, iLimit, offsetPtr, maxNbAttempts, mls);
if (ip < zc->base + zc->nextToUpdate) return 0; /* skipped area */
ZSTD_updateTree_extDict(zc, ip, iLimit, maxNbAttempts, mls);
return ZSTD_insertBtAndFindBestMatch(zc, ip, iLimit, offsetPtr, maxNbAttempts, mls, 1);
}
@ -1930,7 +1794,7 @@ static ZSTD_blockCompressor ZSTD_selectBlockCompressor(ZSTD_strategy strat, int
}
size_t ZSTD_compressBlock(ZSTD_CCtx* zc, void* dst, size_t maxDstSize, const void* src, size_t srcSize)
static size_t ZSTD_compressBlock_internal(ZSTD_CCtx* zc, void* dst, size_t maxDstSize, const void* src, size_t srcSize)
{
ZSTD_blockCompressor blockCompressor = ZSTD_selectBlockCompressor(zc->params.strategy, zc->lowLimit < zc->dictLimit);
if (srcSize < MIN_CBLOCK_SIZE+3) return 0; /* don't even attempt compression below a certain srcSize */
@ -1963,12 +1827,13 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* ctxPtr,
if (ctxPtr->dictLimit < ctxPtr->lowLimit) ctxPtr->dictLimit = ctxPtr->lowLimit;
}
cSize = ZSTD_compressBlock(ctxPtr, op+3, maxDstSize-3, ip, blockSize);
cSize = ZSTD_compressBlock_internal(ctxPtr, op+3, maxDstSize-3, ip, blockSize);
if (ZSTD_isError(cSize)) return cSize;
if (cSize == 0)
{
cSize = ZSTD_noCompressBlock(op, maxDstSize, ip, blockSize); /* block is not compressible */
if (ZSTD_isError(cSize)) return cSize;
}
else
{
@ -1989,11 +1854,23 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* ctxPtr,
}
size_t ZSTD_compressContinue (ZSTD_CCtx* zc,
void* dst, size_t dstSize,
const void* src, size_t srcSize)
static size_t ZSTD_compressContinue_internal (ZSTD_CCtx* zc,
void* dst, size_t dstSize,
const void* src, size_t srcSize,
U32 frame)
{
const BYTE* const ip = (const BYTE*) src;
size_t hbSize = 0;
if (frame && (zc->stage==0))
{
hbSize = zc->hbSize;
if (dstSize <= hbSize) return ERROR(dstSize_tooSmall);
zc->stage = 1;
memcpy(dst, zc->headerBuffer, hbSize);
dstSize -= hbSize;
dst = (char*)dst + hbSize;
}
/* Check if blocks follow each other */
if (src != zc->nextSrc)
@ -2024,7 +1901,7 @@ size_t ZSTD_compressContinue (ZSTD_CCtx* zc,
else zc->nextToUpdate -= correction;
}
/* input-dictionary overlap */
/* if input and dictionary overlap : reduce dictionary (presumed modified by input) */
if ((ip+srcSize > zc->dictBase + zc->lowLimit) && (ip < zc->dictBase + zc->dictLimit))
{
zc->lowLimit = (U32)(ip + srcSize - zc->dictBase);
@ -2032,10 +1909,31 @@ size_t ZSTD_compressContinue (ZSTD_CCtx* zc,
}
zc->nextSrc = ip + srcSize;
return ZSTD_compress_generic (zc, dst, dstSize, src, srcSize);
{
size_t cSize;
if (frame) cSize = ZSTD_compress_generic (zc, dst, dstSize, src, srcSize);
else cSize = ZSTD_compressBlock_internal (zc, dst, dstSize, src, srcSize);
if (ZSTD_isError(cSize)) return cSize;
return cSize + hbSize;
}
}
size_t ZSTD_compressContinue (ZSTD_CCtx* zc,
void* dst, size_t dstSize,
const void* src, size_t srcSize)
{
return ZSTD_compressContinue_internal(zc, dst, dstSize, src, srcSize, 1);
}
size_t ZSTD_compressBlock(ZSTD_CCtx* zc, void* dst, size_t maxDstSize, const void* src, size_t srcSize)
{
if (srcSize > BLOCKSIZE) return ERROR(srcSize_wrong);
return ZSTD_compressContinue_internal(zc, dst, maxDstSize, src, srcSize, 0);
}
size_t ZSTD_compress_insertDictionary(ZSTD_CCtx* zc, const void* src, size_t srcSize)
{
const BYTE* const ip = (const BYTE*) src;
@ -2065,6 +1963,7 @@ size_t ZSTD_compress_insertDictionary(ZSTD_CCtx* zc, const void* src, size_t src
case ZSTD_btlazy2:
ZSTD_updateTree(zc, iend-8, iend, 1 << zc->params.searchLog, zc->params.searchLength);
zc->nextToUpdate = (U32)(iend - zc->base);
break;
default:
@ -2075,24 +1974,56 @@ size_t ZSTD_compress_insertDictionary(ZSTD_CCtx* zc, const void* src, size_t src
}
/*! ZSTD_duplicateCCtx
* Duplicate an existing context @srcCCtx into another one @dstCCtx.
* Only works during stage 0 (i.e. before first call to ZSTD_compressContinue())
* @return : 0, or an error code */
size_t ZSTD_duplicateCCtx(ZSTD_CCtx* dstCCtx, const ZSTD_CCtx* srcCCtx)
{
const U32 contentLog = (srcCCtx->params.strategy == ZSTD_fast) ? 1 : srcCCtx->params.contentLog;
const size_t tableSpace = ((1 << contentLog) + (1 << srcCCtx->params.hashLog)) * sizeof(U32);
if (srcCCtx->stage!=0) return ERROR(stage_wrong);
ZSTD_resetCCtx_advanced(dstCCtx, srcCCtx->params);
/* copy tables */
memcpy(dstCCtx->hashTable, srcCCtx->hashTable, tableSpace);
/* copy frame header */
dstCCtx->hbSize = srcCCtx->hbSize;
memcpy(dstCCtx->headerBuffer , srcCCtx->headerBuffer, srcCCtx->hbSize);
/* copy dictionary pointers */
dstCCtx->nextToUpdate= srcCCtx->nextToUpdate;
dstCCtx->nextSrc = srcCCtx->nextSrc;
dstCCtx->base = srcCCtx->base;
dstCCtx->dictBase = srcCCtx->dictBase;
dstCCtx->dictLimit = srcCCtx->dictLimit;
dstCCtx->lowLimit = srcCCtx->lowLimit;
return 0;
}
/*! ZSTD_compressBegin_advanced
* Write frame header, according to params
* @return : nb of bytes written */
* @return : 0, or an error code */
size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* ctx,
void* dst, size_t maxDstSize,
ZSTD_parameters params)
{
size_t errorCode;
ZSTD_validateParams(&params);
if (maxDstSize < ZSTD_frameHeaderSize_max) return ERROR(dstSize_tooSmall);
errorCode = ZSTD_resetCCtx_advanced(ctx, params);
if (ZSTD_isError(errorCode)) return errorCode;
MEM_writeLE32(dst, ZSTD_MAGICNUMBER); /* Write Header */
((BYTE*)dst)[4] = (BYTE)(params.windowLog - ZSTD_WINDOWLOG_ABSOLUTEMIN);
return ZSTD_frameHeaderSize_min;
MEM_writeLE32(ctx->headerBuffer, ZSTD_MAGICNUMBER); /* Write Header */
((BYTE*)ctx->headerBuffer)[4] = (BYTE)(params.windowLog - ZSTD_WINDOWLOG_ABSOLUTEMIN);
ctx->hbSize = ZSTD_frameHeaderSize_min;
ctx->stage = 0;
return 0;
}
@ -2111,29 +2042,38 @@ ZSTD_parameters ZSTD_getParams(int compressionLevel, U64 srcSizeHint)
}
size_t ZSTD_compressBegin(ZSTD_CCtx* ctx, void* dst, size_t maxDstSize, int compressionLevel)
size_t ZSTD_compressBegin(ZSTD_CCtx* ctx, int compressionLevel)
{
return ZSTD_compressBegin_advanced(ctx, dst, maxDstSize, ZSTD_getParams(compressionLevel, 0));
return ZSTD_compressBegin_advanced(ctx, ZSTD_getParams(compressionLevel, 0));
}
/*! ZSTD_compressEnd
* Write frame epilogue
* @return : nb of bytes written into dst (or an error code) */
size_t ZSTD_compressEnd(ZSTD_CCtx* ctx, void* dst, size_t maxDstSize)
size_t ZSTD_compressEnd(ZSTD_CCtx* zc, void* dst, size_t maxDstSize)
{
BYTE* op = (BYTE*)dst;
size_t hbSize = 0;
/* Sanity check */
(void)ctx;
/* empty frame */
if (zc->stage==0)
{
hbSize = zc->hbSize;
if (maxDstSize <= hbSize) return ERROR(dstSize_tooSmall);
zc->stage = 1;
memcpy(dst, zc->headerBuffer, hbSize);
maxDstSize -= hbSize;
op += hbSize;
}
/* frame epilogue */
if (maxDstSize < 3) return ERROR(dstSize_tooSmall);
/* End of frame */
op[0] = (BYTE)(bt_end << 6);
op[1] = 0;
op[2] = 0;
return 3;
return 3+hbSize;
}
size_t ZSTD_compress_advanced (ZSTD_CCtx* ctx,
@ -2147,10 +2087,8 @@ size_t ZSTD_compress_advanced (ZSTD_CCtx* ctx,
size_t oSize;
/* Header */
oSize = ZSTD_compressBegin_advanced(ctx, dst, maxDstSize, params);
oSize = ZSTD_compressBegin_advanced(ctx, params);
if(ZSTD_isError(oSize)) return oSize;
op += oSize;
maxDstSize -= oSize;
/* dictionary */
if (dict)
@ -2189,7 +2127,7 @@ size_t ZSTD_compress(void* dst, size_t maxDstSize, const void* src, size_t srcSi
ZSTD_CCtx ctxBody;
memset(&ctxBody, 0, sizeof(ctxBody));
result = ZSTD_compressCCtx(&ctxBody, dst, maxDstSize, src, srcSize, compressionLevel);
free(ctxBody.workSpace); /* can't free ctxBody, since it's on stack; free heap content */
free(ctxBody.workSpace); /* can't free ctxBody, since it's on stack; just free heap content */
return result;
}

44
lib/zstd_decompress.c
View File

@ -658,8 +658,19 @@ static size_t ZSTD_decompressSequences(
}
static size_t ZSTD_decompressBlock(
ZSTD_DCtx* dctx,
static void ZSTD_checkContinuity(ZSTD_DCtx* dctx, const void* dst)
{
if (dst != dctx->previousDstEnd) /* not contiguous */
{
dctx->dictEnd = dctx->previousDstEnd;
dctx->vBase = (const char*)dst - ((const char*)(dctx->previousDstEnd) - (const char*)(dctx->base));
dctx->base = dst;
dctx->previousDstEnd = dst;
}
}
static size_t ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx,
void* dst, size_t maxDstSize,
const void* src, size_t srcSize)
{
@ -676,6 +687,15 @@ static size_t ZSTD_decompressBlock(
}
size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx,
void* dst, size_t maxDstSize,
const void* src, size_t srcSize)
{
ZSTD_checkContinuity(dctx, dst);
return ZSTD_decompressBlock_internal(dctx, dst, maxDstSize, src, srcSize);
}
size_t ZSTD_decompress_usingDict(ZSTD_DCtx* ctx,
void* dst, size_t maxDstSize,
const void* src, size_t srcSize,
@ -736,7 +756,7 @@ size_t ZSTD_decompress_usingDict(ZSTD_DCtx* ctx,
switch(blockProperties.blockType)
{
case bt_compressed:
decodedSize = ZSTD_decompressBlock(ctx, op, oend-op, ip, cBlockSize);
decodedSize = ZSTD_decompressBlock_internal(ctx, op, oend-op, ip, cBlockSize);
break;
case bt_raw :
decodedSize = ZSTD_copyRawBlock(op, oend-op, ip, cBlockSize);
@ -787,13 +807,7 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* ctx, void* dst, size_t maxDstSize, con
{
/* Sanity check */
if (srcSize != ctx->expected) return ERROR(srcSize_wrong);
if (dst != ctx->previousDstEnd) /* not contiguous */
{
ctx->dictEnd = ctx->previousDstEnd;
ctx->vBase = (const char*)dst - ((const char*)(ctx->previousDstEnd) - (const char*)(ctx->base));
ctx->base = dst;
ctx->previousDstEnd = dst;
}
ZSTD_checkContinuity(ctx, dst);
/* Decompress : frame header; part 1 */
switch (ctx->stage)
@ -850,7 +864,7 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* ctx, void* dst, size_t maxDstSize, con
switch(ctx->bType)
{
case bt_compressed:
rSize = ZSTD_decompressBlock(ctx, dst, maxDstSize, src, srcSize);
rSize = ZSTD_decompressBlock_internal(ctx, dst, maxDstSize, src, srcSize);
break;
case bt_raw :
rSize = ZSTD_copyRawBlock(dst, maxDstSize, src, srcSize);
@ -875,10 +889,10 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* ctx, void* dst, size_t maxDstSize, con
}
void ZSTD_decompress_insertDictionary(ZSTD_DCtx* ctx, const void* src, size_t srcSize)
void ZSTD_decompress_insertDictionary(ZSTD_DCtx* ctx, const void* dict, size_t dictSize)
{
ctx->dictEnd = ctx->previousDstEnd;
ctx->vBase = (const char*)src - ((const char*)(ctx->previousDstEnd) - (const char*)(ctx->base));
ctx->base = src;
ctx->previousDstEnd = (const char*)src + srcSize;
ctx->vBase = (const char*)dict - ((const char*)(ctx->previousDstEnd) - (const char*)(ctx->base));
ctx->base = dict;
ctx->previousDstEnd = (const char*)dict + dictSize;
}

85
lib/zstd_static.h
View File

@ -33,9 +33,9 @@
#ifndef ZSTD_STATIC_H
#define ZSTD_STATIC_H
/* The objects defined into this file should be considered experimental.
* They are not labelled stable, as their prototype may change in the future.
* You can use them for tests, provide feedback, or if you can endure risk of future changes.
/* The objects defined into this file shall be considered experimental.
* They are not considered stable, as their prototype may change in the future.
* You can use them for tests, provide feedback, or if you can endure risks of future changes.
*/
#if defined (__cplusplus)
@ -108,40 +108,33 @@ ZSTDLIB_API size_t ZSTD_compress_advanced (ZSTD_CCtx* ctx,
const void* dict,size_t dictSize,
ZSTD_parameters params);
/** Decompression context management */
typedef struct ZSTD_DCtx_s ZSTD_DCtx;
ZSTDLIB_API ZSTD_DCtx* ZSTD_createDCtx(void);
ZSTDLIB_API size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx);
/** ZSTD_decompressDCtx
* Same as ZSTD_decompress, with pre-allocated DCtx structure */
size_t ZSTD_decompressDCtx(ZSTD_DCtx* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize);
/** ZSTD_decompress_usingDict
* Same as ZSTD_decompressDCtx, using a Dictionary content as prefix
* Note : dict can be NULL, in which case, it's equivalent to ZSTD_decompressDCtx() */
size_t ZSTD_decompress_usingDict(ZSTD_DCtx* ctx,
void* dst, size_t maxDstSize,
const void* src, size_t srcSize,
const void* dict, size_t dictSize);
ZSTDLIB_API size_t ZSTD_decompress_usingDict(ZSTD_DCtx* ctx,
void* dst, size_t maxDstSize,
const void* src, size_t srcSize,
const void* dict,size_t dictSize);
/* **************************************
* Streaming functions (direct mode)
****************************************/
ZSTDLIB_API size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, void* dst, size_t maxDstSize, int compressionLevel);
ZSTDLIB_API size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* ctx, void* dst, size_t maxDstSize, ZSTD_parameters params);
ZSTDLIB_API size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel);
ZSTDLIB_API size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* ctx, ZSTD_parameters params);
ZSTDLIB_API size_t ZSTD_compress_insertDictionary(ZSTD_CCtx* ctx, const void* src, size_t srcSize);
ZSTDLIB_API size_t ZSTD_duplicateCCtx(ZSTD_CCtx* dstCCtx, const ZSTD_CCtx* srcCCtx);
ZSTDLIB_API size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize);
ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t maxDstSize);
/**
Streaming compression, direct mode (bufferless)
Streaming compression, synchronous mode (bufferless)
A ZSTD_CCtx object is required to track streaming operations.
Use ZSTD_createCCtx() / ZSTD_freeCCtx() to manage it.
A ZSTD_CCtx object can be re-used multiple times.
ZSTD_CCtx object can be re-used multiple times within successive compression operations.
First operation is to start a new frame.
Use ZSTD_compressBegin().
@ -151,15 +144,20 @@ ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t maxDstSiz
Note that dictionary presence is a "hidden" information,
the decoder needs to be aware that it is required for proper decoding, or decoding will fail.
If you want to compress a lot of messages using same dictionary,
it can be beneficial to duplicate compression context rather than reloading dictionary each time.
In such case, use ZSTD_duplicateCCtx(), which will need an already created ZSTD_CCtx,
in order to duplicate compression context into it.
Then, consume your input using ZSTD_compressContinue().
The interface is synchronous, so all input will be consumed.
The interface is synchronous, so all input will be consumed and produce a compressed output.
You must ensure there is enough space in destination buffer to store compressed data under worst case scenario.
Worst case evaluation is provided by ZSTD_compressBound().
Finish a frame with ZSTD_compressEnd(), which will write the epilogue.
Without it, the frame will be considered incomplete by decoders.
You can then reuse ZSTD_CCtx to compress new frames.
You can then reuse ZSTD_CCtx to compress some new frame.
*/
@ -198,9 +196,36 @@ ZSTDLIB_API size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t ma
It can be zero, which is not an error; it just means ZSTD_decompressContinue() has decoded some header.
A frame is fully decoded when ZSTD_nextSrcSizeToDecompress() returns zero.
Context can then be reset to start a new decompression.
*/
/* **************************************
* Block functions
****************************************/
/*!Block functions produce and decode raw zstd blocks, without frame metadata.
It saves associated header sizes.
But user will have to save and regenerate fields required to regenerate data, such as block sizes.
A few rules to respect :
- Uncompressed block size must be <= 128 KB
- Compressing or decompressing require a context structure
+ Use ZSTD_createXCtx() to create them
- It is necessary to init context before starting
+ compression : ZSTD_compressBegin(), which allows selection of compression level or parameters
+ decompression : ZSTD_resetDCtx()
+ If you compress multiple blocks without resetting, next blocks will create references to previous ones
- Dictionary can optionally be inserted, using ZSTD_de/compress_insertDictionary()
- When a block is considered not compressible enough, ZSTD_compressBlock() result will be zero.
+ User must test for such outcome and be able to deal with uncompressed data
+ ZSTD_decompressBlock() doesn't accept uncompressed data as input
*/
size_t ZSTD_compressBlock (ZSTD_CCtx* cctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize);
size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize);
/* *************************************
* Pre-defined compression levels
***************************************/
@ -236,14 +261,14 @@ static const ZSTD_parameters ZSTD_defaultParameters[4][ZSTD_MAX_CLEVEL+1] = {
{ 0, 18, 13, 14, 1, 7, ZSTD_fast }, /* level 0 - never used */
{ 0, 18, 14, 15, 1, 6, ZSTD_fast }, /* level 1 */
{ 0, 18, 14, 15, 1, 5, ZSTD_fast }, /* level 2 */
{ 0, 18, 12, 15, 3, 7, ZSTD_greedy }, /* level 3 */
{ 0, 18, 13, 15, 4, 7, ZSTD_greedy }, /* level 4 */
{ 0, 18, 14, 15, 5, 7, ZSTD_greedy }, /* level 5 */
{ 0, 18, 13, 15, 4, 7, ZSTD_lazy }, /* level 6 */
{ 0, 18, 14, 16, 5, 7, ZSTD_lazy }, /* level 7 */
{ 0, 18, 15, 16, 6, 7, ZSTD_lazy }, /* level 8 */
{ 0, 18, 15, 15, 7, 7, ZSTD_lazy }, /* level 9 */
{ 0, 18, 16, 16, 7, 7, ZSTD_lazy }, /* level 10 */
{ 0, 18, 12, 15, 3, 4, ZSTD_greedy }, /* level 3 */
{ 0, 18, 13, 15, 4, 4, ZSTD_greedy }, /* level 4 */
{ 0, 18, 14, 15, 5, 4, ZSTD_greedy }, /* level 5 */
{ 0, 18, 13, 15, 4, 4, ZSTD_lazy }, /* level 6 */
{ 0, 18, 14, 16, 5, 4, ZSTD_lazy }, /* level 7 */
{ 0, 18, 15, 16, 6, 4, ZSTD_lazy }, /* level 8 */
{ 0, 18, 15, 15, 7, 4, ZSTD_lazy }, /* level 9 */
{ 0, 18, 16, 16, 7, 4, ZSTD_lazy }, /* level 10 */
{ 0, 18, 16, 16, 8, 4, ZSTD_lazy }, /* level 11 */
{ 0, 18, 17, 16, 8, 4, ZSTD_lazy }, /* level 12 */
{ 0, 18, 17, 16, 9, 4, ZSTD_lazy }, /* level 13 */

13
programs/Makefile
View File

@ -30,13 +30,22 @@
# fullbench32: Same as fullbench, but forced to compile in 32-bits mode
# ##########################################################################
VERSION?= 0.4.5
# Version numbers
LIBVER_MAJOR_SCRIPT:=`sed -n '/define ZSTD_VERSION_MAJOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < ../lib/zstd.h`
LIBVER_MINOR_SCRIPT:=`sed -n '/define ZSTD_VERSION_MINOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < ../lib/zstd.h`
LIBVER_PATCH_SCRIPT:=`sed -n '/define ZSTD_VERSION_RELEASE/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < ../lib/zstd.h`
LIBVER_SCRIPT:= $(LIBVER_MAJOR_SCRIPT).$(LIBVER_MINOR_SCRIPT).$(LIBVER_PATCH_SCRIPT)
LIBVER_MAJOR := $(shell echo $(LIBVER_MAJOR_SCRIPT))
LIBVER_MINOR := $(shell echo $(LIBVER_MINOR_SCRIPT))
LIBVER_PATCH := $(shell echo $(LIBVER_PATCH_SCRIPT))
LIBVER := $(shell echo $(LIBVER_SCRIPT))
VERSION?= $(LIBVER)
DESTDIR?=
PREFIX ?= /usr/local
CPPFLAGS= -I../lib -DZSTD_VERSION=\"$(VERSION)\"
CFLAGS ?= -O3 # -falign-loops=32 # not always beneficial
CFLAGS += -std=c99 -Wall -Wextra -Wundef -Wshadow -Wcast-qual -Wcast-align -Wstrict-prototypes
CFLAGS += -std=c99 -Wall -Wextra -Wundef -Wshadow -Wcast-qual -Wcast-align -Wstrict-prototypes -Wstrict-aliasing=1
FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) $(MOREFLAGS)
BINDIR = $(PREFIX)/bin

12
programs/fileio.c
View File

@ -225,7 +225,6 @@ static int FIO_getFiles(FILE** fileOutPtr, FILE** fileInPtr,
if (*fileOutPtr != 0)
{
/* prompt for overwrite authorization */
int ch = 'N';
fclose(*fileOutPtr);
DISPLAY("Warning : %s already exists \n", dstFileName);
if ((g_displayLevel <= 1) || (*fileInPtr == stdin))
@ -235,11 +234,14 @@ static int FIO_getFiles(FILE** fileOutPtr, FILE** fileInPtr,
return 1;
}
DISPLAY("Overwrite ? (y/N) : ");
while((ch = getchar()) != '\n' && ch != EOF); /* flush integrated */
if ((ch!='Y') && (ch!='y'))
{
DISPLAY("No. Operation aborted : %s already exists \n", dstFileName);
return 1;
int ch = getchar();
if ((ch!='Y') && (ch!='y'))
{
DISPLAY("No. Operation aborted : %s already exists \n", dstFileName);
return 1;
}
while ((ch!=EOF) && (ch!='\n')) ch = getchar(); /* flush rest of input line */
}
}
}

129
programs/fuzzer.c
View File

@ -194,6 +194,67 @@ static int basicUnitTests(U32 seed, double compressibility)
if (result != ERROR(srcSize_wrong)) goto _output_error;
DISPLAYLEVEL(4, "OK \n");
/* Dictionary and Duplication tests */
{
ZSTD_CCtx* ctxOrig = ZSTD_createCCtx();
ZSTD_CCtx* ctxDuplicated = ZSTD_createCCtx();
ZSTD_DCtx* dctx = ZSTD_createDCtx();
const size_t dictSize = 500;
size_t cSizeOrig;
DISPLAYLEVEL(4, "test%3i : load dictionary into context : ", testNb++);
result = ZSTD_compressBegin(ctxOrig, 2);
if (ZSTD_isError(result)) goto _output_error;
result = ZSTD_compress_insertDictionary(ctxOrig, CNBuffer, dictSize);
if (ZSTD_isError(result)) goto _output_error;
result = ZSTD_duplicateCCtx(ctxDuplicated, ctxOrig);
if (ZSTD_isError(result)) goto _output_error;
DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(4, "test%3i : compress with dictionary : ", testNb++);
cSize = 0;
result = ZSTD_compressContinue(ctxOrig, compressedBuffer, ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH), (const char*)CNBuffer + dictSize, COMPRESSIBLE_NOISE_LENGTH - dictSize);
if (ZSTD_isError(result)) goto _output_error;
cSize += result;
result = ZSTD_compressEnd(ctxOrig, (char*)compressedBuffer+cSize, ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH)-cSize);
if (ZSTD_isError(result)) goto _output_error;
cSize += result;
DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/COMPRESSIBLE_NOISE_LENGTH*100);
DISPLAYLEVEL(4, "test%3i : frame built with dictionary should be decompressible : ", testNb++);
result = ZSTD_decompress_usingDict(dctx,
decodedBuffer, COMPRESSIBLE_NOISE_LENGTH,
compressedBuffer, cSize,
CNBuffer, dictSize);
if (ZSTD_isError(result)) goto _output_error;
if (result != COMPRESSIBLE_NOISE_LENGTH - dictSize) goto _output_error;
ZSTD_freeCCtx(ctxOrig); /* if ctxOrig is read, will produce segfault */
DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(4, "test%3i : compress with duplicated context : ", testNb++);
cSizeOrig = cSize;
cSize = 0;
result = ZSTD_compressContinue(ctxDuplicated, compressedBuffer, ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH), (const char*)CNBuffer + dictSize, COMPRESSIBLE_NOISE_LENGTH - dictSize);
if (ZSTD_isError(result)) goto _output_error;
cSize += result;
result = ZSTD_compressEnd(ctxDuplicated, (char*)compressedBuffer+cSize, ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH)-cSize);
if (ZSTD_isError(result)) goto _output_error;
cSize += result;
if (cSize != cSizeOrig) goto _output_error; /* should be identical == have same size */
ZSTD_freeCCtx(ctxDuplicated);
DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/COMPRESSIBLE_NOISE_LENGTH*100);
DISPLAYLEVEL(4, "test%3i : frame built with duplicated context should be decompressible : ", testNb++);
result = ZSTD_decompress_usingDict(dctx,
decodedBuffer, COMPRESSIBLE_NOISE_LENGTH,
compressedBuffer, cSize,
CNBuffer, dictSize);
if (ZSTD_isError(result)) goto _output_error;
if (result != COMPRESSIBLE_NOISE_LENGTH - dictSize) goto _output_error;
ZSTD_freeDCtx(dctx);
DISPLAYLEVEL(4, "OK \n");
}
/* Decompression defense tests */
DISPLAYLEVEL(4, "test%3i : Check input length for magic number : ", testNb++);
result = ZSTD_decompress(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, CNBuffer, 3);
@ -207,6 +268,52 @@ static int basicUnitTests(U32 seed, double compressibility)
if (!ZSTD_isError(result)) goto _output_error;
DISPLAYLEVEL(4, "OK \n");
/* block API tests */
{
ZSTD_CCtx* const cctx = ZSTD_createCCtx();
ZSTD_DCtx* const dctx = ZSTD_createDCtx();
const size_t blockSize = 100 KB;
const size_t dictSize = 16 KB;
/* basic block compression */
DISPLAYLEVEL(4, "test%3i : Block compression test : ", testNb++);
result = ZSTD_compressBegin(cctx, 5);
if (ZSTD_isError(result)) goto _output_error;
cSize = ZSTD_compressBlock(cctx, compressedBuffer, ZSTD_compressBound(blockSize), CNBuffer, blockSize);
if (ZSTD_isError(cSize)) goto _output_error;
DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(4, "test%3i : Block decompression test : ", testNb++);
result = ZSTD_resetDCtx(dctx);
if (ZSTD_isError(result)) goto _output_error;
result = ZSTD_decompressBlock(dctx, decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, compressedBuffer, cSize);
if (ZSTD_isError(result)) goto _output_error;
if (result != blockSize) goto _output_error;
DISPLAYLEVEL(4, "OK \n");
/* dictionary block compression */
DISPLAYLEVEL(4, "test%3i : Dictionary Block compression test : ", testNb++);
result = ZSTD_compressBegin(cctx, 5);
if (ZSTD_isError(result)) goto _output_error;
result = ZSTD_compress_insertDictionary(cctx, CNBuffer, dictSize);
if (ZSTD_isError(result)) goto _output_error;
cSize = ZSTD_compressBlock(cctx, compressedBuffer, ZSTD_compressBound(blockSize), (char*)CNBuffer+dictSize, blockSize);
if (ZSTD_isError(cSize)) goto _output_error;
DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(4, "test%3i : Dictionary Block decompression test : ", testNb++);
result = ZSTD_resetDCtx(dctx);
if (ZSTD_isError(result)) goto _output_error;
ZSTD_decompress_insertDictionary(dctx, CNBuffer, dictSize);
result = ZSTD_decompressBlock(dctx, decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, compressedBuffer, cSize);
if (ZSTD_isError(result)) goto _output_error;
if (result != blockSize) goto _output_error;
DISPLAYLEVEL(4, "OK \n");
ZSTD_freeCCtx(cctx);
ZSTD_freeDCtx(dctx);
}
/* long rle test */
{
size_t sampleSize = 0;
@ -268,11 +375,13 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit
U32 result = 0;
U32 testNb = 0;
U32 coreSeed = seed, lseed = 0;
ZSTD_CCtx* refCtx;
ZSTD_CCtx* ctx;
ZSTD_DCtx* dctx;
U32 startTime = FUZ_GetMilliStart();
/* allocation */
refCtx = ZSTD_createCCtx();
ctx = ZSTD_createCCtx();
dctx= ZSTD_createDCtx();
cNoiseBuffer[0] = (BYTE*)malloc (srcBufferSize);
@ -284,7 +393,7 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit
mirrorBuffer = (BYTE*)malloc (dstBufferSize);
cBuffer = (BYTE*)malloc (cBufferSize);
CHECK (!cNoiseBuffer[0] || !cNoiseBuffer[1] || !cNoiseBuffer[2] || !cNoiseBuffer[3] || !cNoiseBuffer[4]
|| !dstBuffer || !mirrorBuffer || !cBuffer || !ctx || !dctx,
|| !dstBuffer || !mirrorBuffer || !cBuffer || !refCtx || !ctx || !dctx,
"Not enough memory, fuzzer tests cancelled");
/* Create initial samples */
@ -305,7 +414,7 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit
size_t sampleSize, sampleStart, maxTestSize, totalTestSize;
size_t cSize, dSize, dSupSize, errorCode, totalCSize, totalGenSize;
U32 sampleSizeLog, buffNb, cLevelMod, nbChunks, n;
XXH64_state_t crc64;
XXH64_CREATESTATE_STATIC(xxh64);
U64 crcOrig, crcDest;
int cLevel;
BYTE* sampleBuffer;
@ -447,7 +556,7 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit
}
/* Streaming compression of scattered segments test */
XXH64_reset(&crc64, 0);
XXH64_reset(xxh64, 0);
nbChunks = (FUZ_rand(&lseed) & 127) + 2;
sampleSizeLog = FUZ_rand(&lseed) % maxSrcLog;
maxTestSize = (size_t)1 << sampleSizeLog;
@ -461,10 +570,13 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit
dict = srcBuffer + sampleStart;
dictSize = sampleSize;
cSize = ZSTD_compressBegin(ctx, cBuffer, cBufferSize, (FUZ_rand(&lseed) % (20 - (sampleSizeLog/3))) + 1);
errorCode = ZSTD_compress_insertDictionary(ctx, dict, dictSize);
errorCode = ZSTD_compressBegin(refCtx, (FUZ_rand(&lseed) % (20 - (sampleSizeLog/3))) + 1);
CHECK (ZSTD_isError(errorCode), "start streaming error : %s", ZSTD_getErrorName(errorCode));
errorCode = ZSTD_compress_insertDictionary(refCtx, dict, dictSize);
CHECK (ZSTD_isError(errorCode), "dictionary insertion error : %s", ZSTD_getErrorName(errorCode));
totalTestSize = 0;
errorCode = ZSTD_duplicateCCtx(ctx, refCtx);
CHECK (ZSTD_isError(errorCode), "context duplication error : %s", ZSTD_getErrorName(errorCode));
totalTestSize = 0; cSize = 0;
for (n=0; n<nbChunks; n++)
{
sampleSizeLog = FUZ_rand(&lseed) % maxSampleLog;
@ -481,14 +593,14 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit
CHECK (ZSTD_isError(errorCode), "multi-segments compression error : %s", ZSTD_getErrorName(errorCode));
cSize += errorCode;
XXH64_update(&crc64, srcBuffer+sampleStart, sampleSize);
XXH64_update(xxh64, srcBuffer+sampleStart, sampleSize);
memcpy(mirrorBuffer + totalTestSize, srcBuffer+sampleStart, sampleSize);
totalTestSize += sampleSize;
}
errorCode = ZSTD_compressEnd(ctx, cBuffer+cSize, cBufferSize-cSize);
CHECK (ZSTD_isError(errorCode), "multi-segments epilogue error : %s", ZSTD_getErrorName(errorCode));
cSize += errorCode;
crcOrig = XXH64_digest(&crc64);
crcOrig = XXH64_digest(xxh64);
/* streaming decompression test */
errorCode = ZSTD_resetDCtx(dctx);
@ -517,6 +629,7 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit
DISPLAY("\r%u fuzzer tests completed \n", testNb-1);
_cleanup:
ZSTD_freeCCtx(refCtx);
ZSTD_freeCCtx(ctx);
ZSTD_freeDCtx(dctx);
free(cNoiseBuffer[0]);

171
programs/xxhash.c
View File

@ -1,6 +1,6 @@
/*
xxHash - Fast Hash algorithm
Copyright (C) 2012-2015, Yann Collet
Copyright (C) 2012-2016, Yann Collet
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
@ -32,18 +32,18 @@ You can contact the author at :
*/
/**************************************
/* *************************************
* Tuning parameters
**************************************/
/* XXH_FORCE_MEMORY_ACCESS
***************************************/
/*!XXH_FORCE_MEMORY_ACCESS
* By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable.
* Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal.
* The below switch allow to select different access method for improved performance.
* Method 0 (default) : use `memcpy()`. Safe and portable.
* Method 1 : `__packed` statement. It depends on compiler extension (ie, not portable).
* This method is safe if your compiler supports it, and *generally* as fast or faster than `memcpy`.
* Method 2 : direct access. This method is portable but violate C standard.
* It can generate buggy code on targets which generate assembly depending on alignment.
* Method 2 : direct access. This method doesn't depend on compiler but violate C standard.
* It can generate buggy code on targets which do not support unaligned memory accesses.
* But in some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6)
* See http://stackoverflow.com/a/32095106/646947 for details.
* Prefer these methods in priority order (0 > 1 > 2)
@ -57,14 +57,14 @@ You can contact the author at :
# endif
#endif
/* XXH_ACCEPT_NULL_INPUT_POINTER :
/*!XXH_ACCEPT_NULL_INPUT_POINTER :
* If the input pointer is a null pointer, xxHash default behavior is to trigger a memory access error, since it is a bad pointer.
* When this option is enabled, xxHash output for null input pointers will be the same as a null-length input.
* By default, this option is disabled. To enable it, uncomment below define :
*/
/* #define XXH_ACCEPT_NULL_INPUT_POINTER 1 */
/* XXH_FORCE_NATIVE_FORMAT :
/*!XXH_FORCE_NATIVE_FORMAT :
* By default, xxHash library provides endian-independant Hash values, based on little-endian convention.
* Results are therefore identical for little-endian and big-endian CPU.
* This comes at a performance cost for big-endian CPU, since some swapping is required to emulate little-endian format.
@ -74,9 +74,9 @@ You can contact the author at :
*/
#define XXH_FORCE_NATIVE_FORMAT 0
/* XXH_USELESS_ALIGN_BRANCH :
/*!XXH_USELESS_ALIGN_BRANCH :
* This is a minor performance trick, only useful with lots of very small keys.
* It means : don't make a test between aligned/unaligned, because performance will be the same.
* It means : don't check for aligned/unaligned input, because performance will be the same.
* It saves one initial branch per hash.
*/
#if defined(__i386) || defined(_M_IX86) || defined(__x86_64__) || defined(_M_X64)
@ -84,7 +84,7 @@ You can contact the author at :
#endif
/**************************************
/* *************************************
* Compiler Specific Options
***************************************/
#ifdef _MSC_VER /* Visual Studio */
@ -103,10 +103,9 @@ You can contact the author at :
#endif
/**************************************
/* *************************************
* Includes & Memory related functions
***************************************/
#include "xxhash.h"
/* Modify the local functions below should you wish to use some other memory routines */
/* for malloc(), free() */
#include <stdlib.h>
@ -116,23 +115,28 @@ static void XXH_free (void* p) { free(p); }
#include <string.h>
static void* XXH_memcpy(void* dest, const void* src, size_t size) { return memcpy(dest,src,size); }
#include "xxhash.h"
/**************************************
/* *************************************
* Basic Types
***************************************/
#if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */
# include <stdint.h>
typedef uint8_t BYTE;
typedef uint16_t U16;
typedef uint32_t U32;
typedef int32_t S32;
typedef uint64_t U64;
#else
typedef unsigned char BYTE;
typedef unsigned short U16;
typedef unsigned int U32;
typedef signed int S32;
typedef unsigned long long U64;
#ifndef MEM_MODULE
# define MEM_MODULE
# if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */
# include <stdint.h>
typedef uint8_t BYTE;
typedef uint16_t U16;
typedef uint32_t U32;
typedef int32_t S32;
typedef uint64_t U64;
# else
typedef unsigned char BYTE;
typedef unsigned short U16;
typedef unsigned int U32;
typedef signed int S32;
typedef unsigned long long U64;
# endif
#endif
@ -174,7 +178,7 @@ static U64 XXH_read64(const void* memPtr)
#endif // XXH_FORCE_DIRECT_MEMORY_ACCESS
/******************************************
/* ****************************************
* Compiler-specific Functions and Macros
******************************************/
#define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
@ -216,19 +220,19 @@ static U64 XXH_swap64 (U64 x)
#endif
/***************************************
/* *************************************
* Architecture Macros
***************************************/
typedef enum { XXH_bigEndian=0, XXH_littleEndian=1 } XXH_endianess;
/* XXH_CPU_LITTLE_ENDIAN can be defined externally, for example one the compiler command line */
/* XXH_CPU_LITTLE_ENDIAN can be defined externally, for example on the compiler command line */
#ifndef XXH_CPU_LITTLE_ENDIAN
static const int one = 1;
# define XXH_CPU_LITTLE_ENDIAN (*(const char*)(&one))
static const int g_one = 1;
# define XXH_CPU_LITTLE_ENDIAN (*(const char*)(&g_one))
#endif
/*****************************
/* ***************************
* Memory reads
*****************************/
typedef enum { XXH_aligned, XXH_unaligned } XXH_alignment;
@ -260,13 +264,13 @@ FORCE_INLINE U64 XXH_readLE64(const void* ptr, XXH_endianess endian)
}
/***************************************
/* *************************************
* Macros
***************************************/
#define XXH_STATIC_ASSERT(c) { enum { XXH_static_assert = 1/(!!(c)) }; } /* use only *after* variable declarations */
#define XXH_STATIC_ASSERT(c) { enum { XXH_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */
/***************************************
/* *************************************
* Constants
***************************************/
#define PRIME32_1 2654435761U
@ -281,8 +285,10 @@ FORCE_INLINE U64 XXH_readLE64(const void* ptr, XXH_endianess endian)
#define PRIME64_4 9650029242287828579ULL
#define PRIME64_5 2870177450012600261ULL
XXH_PUBLIC_API unsigned XXH_versionNumber (void) { return XXH_VERSION_NUMBER; }
/*****************************
/* ***************************
* Simple Hash Functions
*****************************/
FORCE_INLINE U32 XXH32_endian_align(const void* input, size_t len, U32 seed, XXH_endianess endian, XXH_alignment align)
@ -362,7 +368,7 @@ FORCE_INLINE U32 XXH32_endian_align(const void* input, size_t len, U32 seed, XXH
}
unsigned int XXH32 (const void* input, size_t len, unsigned int seed)
XXH_PUBLIC_API unsigned int XXH32 (const void* input, size_t len, unsigned int seed)
{
#if 0
/* Simple version, good for code maintenance, but unfortunately slow for small inputs */
@ -502,7 +508,7 @@ FORCE_INLINE U64 XXH64_endian_align(const void* input, size_t len, U64 seed, XXH
}
unsigned long long XXH64 (const void* input, size_t len, unsigned long long seed)
XXH_PUBLIC_API unsigned long long XXH64 (const void* input, size_t len, unsigned long long seed)
{
#if 0
/* Simple version, good for code maintenance, but unfortunately slow for small inputs */
@ -530,12 +536,12 @@ unsigned long long XXH64 (const void* input, size_t len, unsigned long long seed
#endif
}
/****************************************************
/* **************************************************
* Advanced Hash Functions
****************************************************/
/*** Allocation ***/
typedef struct
struct XXH32_state_s
{
U64 total_len;
U32 seed;
@ -545,9 +551,9 @@ typedef struct
U32 v4;
U32 mem32[4]; /* defined as U32 for alignment */
U32 memsize;
} XXH_istate32_t;
}; /* typedef'd to XXH32_state_t within xxhash.h */
typedef struct
struct XXH64_state_s
{
U64 total_len;
U64 seed;
@ -557,26 +563,26 @@ typedef struct
U64 v4;
U64 mem64[4]; /* defined as U64 for alignment */
U32 memsize;
} XXH_istate64_t;
}; /* typedef'd to XXH64_state_t within xxhash.h */
XXH32_state_t* XXH32_createState(void)
XXH_PUBLIC_API XXH32_state_t* XXH32_createState(void)
{
XXH_STATIC_ASSERT(sizeof(XXH32_state_t) >= sizeof(XXH_istate32_t)); /* A compilation error here means XXH32_state_t is not large enough */
XXH_STATIC_ASSERT(sizeof(XXH32_stateBody_t) >= sizeof(XXH32_state_t)); /* A compilation error here means XXH32_state_t is not large enough */
return (XXH32_state_t*)XXH_malloc(sizeof(XXH32_state_t));
}
XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr)
XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr)
{
XXH_free(statePtr);
return XXH_OK;
}
XXH64_state_t* XXH64_createState(void)
XXH_PUBLIC_API XXH64_state_t* XXH64_createState(void)
{
XXH_STATIC_ASSERT(sizeof(XXH64_state_t) >= sizeof(XXH_istate64_t)); /* A compilation error here means XXH64_state_t is not large enough */
XXH_STATIC_ASSERT(sizeof(XXH64_stateBody_t) >= sizeof(XXH64_state_t)); /* A compilation error here means XXH64_state_t is not large enough */
return (XXH64_state_t*)XXH_malloc(sizeof(XXH64_state_t));
}
XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr)
XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr)
{
XXH_free(statePtr);
return XXH_OK;
@ -585,36 +591,36 @@ XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr)
/*** Hash feed ***/
XXH_errorcode XXH32_reset(XXH32_state_t* state_in, unsigned int seed)
XXH_PUBLIC_API XXH_errorcode XXH32_reset(XXH32_state_t* statePtr, unsigned int seed)
{
XXH_istate32_t* state = (XXH_istate32_t*) state_in;
state->seed = seed;
state->v1 = seed + PRIME32_1 + PRIME32_2;
state->v2 = seed + PRIME32_2;
state->v3 = seed + 0;
state->v4 = seed - PRIME32_1;
state->total_len = 0;
state->memsize = 0;
return XXH_OK;
}
XXH_errorcode XXH64_reset(XXH64_state_t* state_in, unsigned long long seed)
{
XXH_istate64_t* state = (XXH_istate64_t*) state_in;
state->seed = seed;
state->v1 = seed + PRIME64_1 + PRIME64_2;
state->v2 = seed + PRIME64_2;
state->v3 = seed + 0;
state->v4 = seed - PRIME64_1;
state->total_len = 0;
state->memsize = 0;
XXH32_state_t state; /* using a local state to memcpy() in order to avoid strict-aliasing warnings */
memset(&state, 0, sizeof(state));
state.seed = seed;
state.v1 = seed + PRIME32_1 + PRIME32_2;
state.v2 = seed + PRIME32_2;
state.v3 = seed + 0;
state.v4 = seed - PRIME32_1;
memcpy(statePtr, &state, sizeof(state));
return XXH_OK;
}
FORCE_INLINE XXH_errorcode XXH32_update_endian (XXH32_state_t* state_in, const void* input, size_t len, XXH_endianess endian)
XXH_PUBLIC_API XXH_errorcode XXH64_reset(XXH64_state_t* statePtr, unsigned long long seed)
{
XXH64_state_t state; /* using a local state to memcpy() in order to avoid strict-aliasing warnings */
memset(&state, 0, sizeof(state));
state.seed = seed;
state.v1 = seed + PRIME64_1 + PRIME64_2;
state.v2 = seed + PRIME64_2;
state.v3 = seed + 0;
state.v4 = seed - PRIME64_1;
memcpy(statePtr, &state, sizeof(state));
return XXH_OK;
}
FORCE_INLINE XXH_errorcode XXH32_update_endian (XXH32_state_t* state, const void* input, size_t len, XXH_endianess endian)
{
XXH_istate32_t* state = (XXH_istate32_t *) state_in;
const BYTE* p = (const BYTE*)input;
const BYTE* const bEnd = p + len;
@ -701,7 +707,7 @@ FORCE_INLINE XXH_errorcode XXH32_update_endian (XXH32_state_t* state_in, const v
return XXH_OK;
}
XXH_errorcode XXH32_update (XXH32_state_t* state_in, const void* input, size_t len)
XXH_PUBLIC_API XXH_errorcode XXH32_update (XXH32_state_t* state_in, const void* input, size_t len)
{
XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN;
@ -713,9 +719,8 @@ XXH_errorcode XXH32_update (XXH32_state_t* state_in, const void* input, size_t l
FORCE_INLINE U32 XXH32_digest_endian (const XXH32_state_t* state_in, XXH_endianess endian)
FORCE_INLINE U32 XXH32_digest_endian (const XXH32_state_t* state, XXH_endianess endian)
{
const XXH_istate32_t* state = (const XXH_istate32_t*) state_in;
const BYTE * p = (const BYTE*)state->mem32;
const BYTE* bEnd = (const BYTE*)(state->mem32) + state->memsize;
U32 h32;
@ -755,7 +760,7 @@ FORCE_INLINE U32 XXH32_digest_endian (const XXH32_state_t* state_in, XXH_endiane
}
unsigned int XXH32_digest (const XXH32_state_t* state_in)
XXH_PUBLIC_API unsigned int XXH32_digest (const XXH32_state_t* state_in)
{
XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN;
@ -766,9 +771,8 @@ unsigned int XXH32_digest (const XXH32_state_t* state_in)
}
FORCE_INLINE XXH_errorcode XXH64_update_endian (XXH64_state_t* state_in, const void* input, size_t len, XXH_endianess endian)
FORCE_INLINE XXH_errorcode XXH64_update_endian (XXH64_state_t* state, const void* input, size_t len, XXH_endianess endian)
{
XXH_istate64_t * state = (XXH_istate64_t *) state_in;
const BYTE* p = (const BYTE*)input;
const BYTE* const bEnd = p + len;
@ -855,7 +859,7 @@ FORCE_INLINE XXH_errorcode XXH64_update_endian (XXH64_state_t* state_in, const v
return XXH_OK;
}
XXH_errorcode XXH64_update (XXH64_state_t* state_in, const void* input, size_t len)
XXH_PUBLIC_API XXH_errorcode XXH64_update (XXH64_state_t* state_in, const void* input, size_t len)
{
XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN;
@ -867,9 +871,8 @@ XXH_errorcode XXH64_update (XXH64_state_t* state_in, const void* input, size_t l
FORCE_INLINE U64 XXH64_digest_endian (const XXH64_state_t* state_in, XXH_endianess endian)
FORCE_INLINE U64 XXH64_digest_endian (const XXH64_state_t* state, XXH_endianess endian)
{
const XXH_istate64_t * state = (const XXH_istate64_t *) state_in;
const BYTE * p = (const BYTE*)state->mem64;
const BYTE* bEnd = (const BYTE*)state->mem64 + state->memsize;
U64 h64;
@ -949,7 +952,7 @@ FORCE_INLINE U64 XXH64_digest_endian (const XXH64_state_t* state_in, XXH_endiane
}
unsigned long long XXH64_digest (const XXH64_state_t* state_in)
XXH_PUBLIC_API unsigned long long XXH64_digest (const XXH64_state_t* state_in)
{
XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN;

156
programs/xxhash.h
View File

@ -1,7 +1,7 @@
/*
xxHash - Extremely Fast Hash algorithm
Header File
Copyright (C) 2012-2015, Yann Collet.
Copyright (C) 2012-2016, Yann Collet.
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
@ -71,34 +71,56 @@ extern "C" {
#endif
/*****************************
/* ****************************
* Definitions
*****************************/
******************************/
#include <stddef.h> /* size_t */
typedef enum { XXH_OK=0, XXH_ERROR } XXH_errorcode;
/*****************************
* Namespace Emulation
*****************************/
/* Motivations :
/* ****************************
* API modifier
******************************/
/*!XXH_PRIVATE_API
* Transforms all publics symbols within `xxhash.c` into private ones.
* Methodology :
* instead of : #include "xxhash.h"
* do :
* #define XXH_PRIVATE_API
* #include "xxhash.c" // note the .c , instead of .h
* also : don't compile and link xxhash.c separately
*/
#ifdef XXH_PRIVATE_API
# if defined(__GNUC__)
# define XXH_PUBLIC_API static __attribute__((unused))
# elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
# define XXH_PUBLIC_API static inline
# elif defined(_MSC_VER)
# define XXH_PUBLIC_API static __inline
# else
# define XXH_PUBLIC_API static /* this version may generate warnings for unused static functions; disable the relevant warning */
# endif
#else
# define XXH_PUBLIC_API /* do nothing */
#endif
If you need to include xxHash into your library,
but wish to avoid xxHash symbols to be present on your library interface
in an effort to avoid potential name collision if another library also includes xxHash,
/*!XXH_NAMESPACE, aka Namespace Emulation :
you can use XXH_NAMESPACE, which will automatically prefix any symbol from xxHash
with the value of XXH_NAMESPACE (so avoid to keep it NULL, and avoid numeric values).
If you want to include _and expose_ xxHash functions from within your own library,
but also want to avoid symbol collisions with another library which also includes xxHash,
Note that no change is required within the calling program :
it can still call xxHash functions using their regular name.
They will be automatically translated by this header.
you can use XXH_NAMESPACE, to automatically prefix any public symbol from `xxhash.c`
with the value of XXH_NAMESPACE (so avoid to keep it NULL and avoid numeric values).
Note that no change is required within the calling program as long as it also includes `xxhash.h` :
regular symbol name will be automatically translated by this header.
*/
#ifdef XXH_NAMESPACE
# define XXH_CAT(A,B) A##B
# define XXH_NAME2(A,B) XXH_CAT(A,B)
# define XXH32 XXH_NAME2(XXH_NAMESPACE, XXH32)
# define XXH64 XXH_NAME2(XXH_NAMESPACE, XXH64)
# define XXH_versionNumber XXH_NAME2(XXH_NAMESPACE, XXH_versionNumber)
# define XXH32_createState XXH_NAME2(XXH_NAMESPACE, XXH32_createState)
# define XXH64_createState XXH_NAME2(XXH_NAMESPACE, XXH64_createState)
# define XXH32_freeState XXH_NAME2(XXH_NAMESPACE, XXH32_freeState)
@ -112,78 +134,90 @@ They will be automatically translated by this header.
#endif
/*****************************
/* *************************************
* Version
***************************************/
#define XXH_VERSION_MAJOR 0
#define XXH_VERSION_MINOR 5
#define XXH_VERSION_RELEASE 0
#define XXH_VERSION_NUMBER (XXH_VERSION_MAJOR *100*100 + XXH_VERSION_MINOR *100 + XXH_VERSION_RELEASE)
XXH_PUBLIC_API unsigned XXH_versionNumber (void);
/* ****************************
* Simple Hash Functions
*****************************/
******************************/
unsigned int XXH32 (const void* input, size_t length, unsigned seed);
unsigned long long XXH64 (const void* input, size_t length, unsigned long long seed);
XXH_PUBLIC_API unsigned int XXH32 (const void* input, size_t length, unsigned int seed);
XXH_PUBLIC_API unsigned long long XXH64 (const void* input, size_t length, unsigned long long seed);
/*
/*!
XXH32() :
Calculate the 32-bits hash of sequence "length" bytes stored at memory address "input".
The memory between input & input+length must be valid (allocated and read-accessible).
"seed" can be used to alter the result predictably.
This function successfully passes all SMHasher tests.
Speed on Core 2 Duo @ 3 GHz (single thread, SMHasher benchmark) : 5.4 GB/s
XXH64() :
Calculate the 64-bits hash of sequence of length "len" stored at memory address "input".
Faster on 64-bits systems. Slower on 32-bits systems.
"seed" can be used to alter the result predictably.
This function runs faster on 64-bits systems, but slower on 32-bits systems (see benchmark).
*/
/*****************************
/* ****************************
* Advanced Hash Functions
*****************************/
typedef struct { long long ll[ 6]; } XXH32_state_t;
typedef struct { long long ll[11]; } XXH64_state_t;
/*
These structures allow static allocation of XXH states.
States must then be initialized using XXHnn_reset() before first use.
If you prefer dynamic allocation, please refer to functions below.
*/
XXH32_state_t* XXH32_createState(void);
XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr);
XXH64_state_t* XXH64_createState(void);
XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr);
/*
These functions create and release memory for XXH state.
States must then be initialized using XXHnn_reset() before first use.
*/
******************************/
typedef struct XXH32_state_s XXH32_state_t; /* incomplete */
typedef struct XXH64_state_s XXH64_state_t; /* incomplete */
XXH_errorcode XXH32_reset (XXH32_state_t* statePtr, unsigned seed);
XXH_errorcode XXH32_update (XXH32_state_t* statePtr, const void* input, size_t length);
unsigned int XXH32_digest (const XXH32_state_t* statePtr);
/*!Static allocation
For static linking only, do not use in the context of DLL ! */
typedef struct { long long ll[ 6]; } XXH32_stateBody_t;
typedef struct { long long ll[11]; } XXH64_stateBody_t;
XXH_errorcode XXH64_reset (XXH64_state_t* statePtr, unsigned long long seed);
XXH_errorcode XXH64_update (XXH64_state_t* statePtr, const void* input, size_t length);
unsigned long long XXH64_digest (const XXH64_state_t* statePtr);
#define XXH32_CREATESTATE_STATIC(name) XXH32_stateBody_t name##xxhbody; void* name##xxhvoid = &(name##xxhbody); XXH32_state_t* name = (XXH32_state_t*)(name##xxhvoid) /* no final ; */
#define XXH64_CREATESTATE_STATIC(name) XXH64_stateBody_t name##xxhbody; void* name##xxhvoid = &(name##xxhbody); XXH64_state_t* name = (XXH64_state_t*)(name##xxhvoid) /* no final ; */
/*
These functions calculate the xxHash of an input provided in multiple smaller packets,
as opposed to an input provided as a single block.
XXH state space must first be allocated, using either static or dynamic method provided above.
/*!Dynamic allocation
To be preferred in the context of DLL */
XXH_PUBLIC_API XXH32_state_t* XXH32_createState(void);
XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr);
XXH_PUBLIC_API XXH64_state_t* XXH64_createState(void);
XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr);
/* hash streaming */
XXH_PUBLIC_API XXH_errorcode XXH32_reset (XXH32_state_t* statePtr, unsigned int seed);
XXH_PUBLIC_API XXH_errorcode XXH32_update (XXH32_state_t* statePtr, const void* input, size_t length);
XXH_PUBLIC_API unsigned int XXH32_digest (const XXH32_state_t* statePtr);
XXH_PUBLIC_API XXH_errorcode XXH64_reset (XXH64_state_t* statePtr, unsigned long long seed);
XXH_PUBLIC_API XXH_errorcode XXH64_update (XXH64_state_t* statePtr, const void* input, size_t length);
XXH_PUBLIC_API unsigned long long XXH64_digest (const XXH64_state_t* statePtr);
/*!
These functions generate the xxHash of an input provided in multiple segments,
as opposed to provided as a single block.
XXH state must first be allocated, using either static or dynamic method provided above.
Start a new hash by initializing state with a seed, using XXHnn_reset().
Then, feed the hash state by calling XXHnn_update() as many times as necessary.
Obviously, input must be valid, meaning allocated and read accessible.
Obviously, input must be valid, hence allocated and read accessible.
The function returns an error code, with 0 meaning OK, and any other value meaning there is an error.
Finally, you can produce a hash anytime, by using XXHnn_digest().
This function returns the final nn-bits hash.
You can nonetheless continue feeding the hash state with more input,
and therefore get some new hashes, by calling again XXHnn_digest().
Finally, a hash value can be produced anytime, by using XXHnn_digest().
This function returns the nn-bits hash.
It's nonetheless possible to continue inserting input into the hash state
and later on generate some new hashes, by calling again XXHnn_digest().
When you are done, don't forget to free XXH state space, using typically XXHnn_freeState().
When done, free XXH state space if it was allocated dynamically.
*/

8
programs/zbufftest.c
View File

@ -286,7 +286,7 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit
size_t maxTestSize, totalTestSize, readSize, totalCSize, genSize, totalGenSize;
size_t errorCode;
U32 sampleSizeLog, buffNb, n, nbChunks;
XXH64_state_t crc64;
XXH64_CREATESTATE_STATIC(xxh64);
U64 crcOrig, crcDest;
/* init */
@ -313,7 +313,7 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit
srcBuffer = cNoiseBuffer[buffNb];
/* Multi - segments compression test */
XXH64_reset(&crc64, 0);
XXH64_reset(xxh64, 0);
nbChunks = (FUZ_rand(&lseed) & 127) + 2;
sampleSizeLog = FUZ_rand(&lseed) % maxSrcLog;
maxTestSize = (size_t)1 << sampleSizeLog;
@ -347,7 +347,7 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit
errorCode = ZBUFF_compressContinue(zc, cBuffer+cSize, &genSize, srcBuffer+sampleStart, &readSize);
CHECK (ZBUFF_isError(errorCode), "compression error : %s", ZBUFF_getErrorName(errorCode));
XXH64_update(&crc64, srcBuffer+sampleStart, readSize);
XXH64_update(xxh64, srcBuffer+sampleStart, readSize);
memcpy(copyBuffer+totalTestSize, srcBuffer+sampleStart, readSize);
cSize += genSize;
totalTestSize += readSize;
@ -371,7 +371,7 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit
CHECK (ZBUFF_isError(errorCode), "compression error : %s", ZBUFF_getErrorName(errorCode));
CHECK (errorCode != 0, "frame epilogue not fully consumed");
cSize += genSize;
crcOrig = XXH64_digest(&crc64);
crcOrig = XXH64_digest(xxh64);
/* multi - fragments decompression test */
ZBUFF_decompressInit(zd);

BIN
visual/2013/zstdlib/zstdlib.rc
View File

Binary file not shown.