facebook
/
zstd
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Updated FSE lib

dev
Yann Collet 2015-08-07 15:21:00 +01:00
parent 77c82b680b
commit a787550d1c
4 changed files with 310 additions and 280 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

334
lib/fse.c
View File

@ -496,7 +496,7 @@ size_t FSE_FUNCTION_NAME(FSE_buildCTable, FSE_FUNCTION_EXTENSION)
for (i=0; i<tableSize; i++)
{
FSE_FUNCTION_TYPE s = tableSymbol[i]; /* static analyzer doesn't understand tableSymbol is properly initialized */
tableU16[cumul[s]++] = (U16) (tableSize+i); /* Table U16 : sorted by symbol order; gives next state value */
tableU16[cumul[s]++] = (U16) (tableSize+i); /* TableU16 : sorted by symbol order; gives next state value */
}
/* Build Symbol Transformation Table */
@ -511,7 +511,7 @@ size_t FSE_FUNCTION_NAME(FSE_buildCTable, FSE_FUNCTION_EXTENSION)
break;
case -1:
case 1:
symbolTT[s].deltaNbBits = tableLog * (1 << 16);
symbolTT[s].deltaNbBits = tableLog << 16;
symbolTT[s].deltaFindState = total - 1;
total ++;
break;
@ -544,12 +544,16 @@ void FSE_FUNCTION_NAME(FSE_freeDTable, FSE_FUNCTION_EXTENSION) (FSE_DTable* dt)
free(dt);
}
typedef struct {
U16 tableLog;
U16 fastMode;
} FSE_DTableHeader; /* sizeof U32 */
size_t FSE_FUNCTION_NAME(FSE_buildDTable, FSE_FUNCTION_EXTENSION)
(FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog)
{
U32* const base32 = (U32*)dt;
FSE_DECODE_TYPE* const tableDecode = (FSE_DECODE_TYPE*) (base32+1);
FSE_DTableHeader* const DTableH = (FSE_DTableHeader*)dt;
FSE_DECODE_TYPE* const tableDecode = (FSE_DECODE_TYPE*) (dt+1); /* because dt is unsigned, 32-bits aligned on 32-bits */
const U32 tableSize = 1 << tableLog;
const U32 tableMask = tableSize-1;
const U32 step = FSE_tableStep(tableSize);
@ -565,7 +569,7 @@ size_t FSE_FUNCTION_NAME(FSE_buildDTable, FSE_FUNCTION_EXTENSION)
if (tableLog > FSE_MAX_TABLELOG) return (size_t)-FSE_ERROR_tableLog_tooLarge;
/* Init, lay down lowprob symbols */
base32[0] = tableLog;
DTableH[0].tableLog = (U16)tableLog;
for (s=0; s<=maxSymbolValue; s++)
{
if (normalizedCounter[s]==-1)
@ -606,7 +610,8 @@ size_t FSE_FUNCTION_NAME(FSE_buildDTable, FSE_FUNCTION_EXTENSION)
}
}
return noLarge;
DTableH->fastMode = (U16)noLarge;
return 0;
}
@ -639,7 +644,7 @@ static short FSE_abs(short a)
size_t FSE_NCountWriteBound(unsigned maxSymbolValue, unsigned tableLog)
{
size_t maxHeaderSize = (((maxSymbolValue+1) * tableLog) >> 3) + 1;
return maxSymbolValue ? maxHeaderSize : FSE_MAX_HEADERSIZE;
return maxSymbolValue ? maxHeaderSize : FSE_NCOUNTBOUND;
}
static size_t FSE_writeNCount_generic (void* header, size_t headerBufferSize,
@ -679,7 +684,7 @@ static size_t FSE_writeNCount_generic (void* header, size_t headerBufferSize,
{
start+=24;
bitStream += 0xFFFFU << bitCount;
if ((!safeWrite) && (out > oend-2)) return (size_t)-FSE_ERROR_GENERIC; /* Buffer overflow */
if ((!safeWrite) && (out > oend-2)) return (size_t)-FSE_ERROR_dstSize_tooSmall; /* Buffer overflow */
out[0] = (BYTE) bitStream;
out[1] = (BYTE)(bitStream>>8);
out+=2;
@ -695,7 +700,7 @@ static size_t FSE_writeNCount_generic (void* header, size_t headerBufferSize,
bitCount += 2;
if (bitCount>16)
{
if ((!safeWrite) && (out > oend - 2)) return (size_t)-FSE_ERROR_GENERIC; /* Buffer overflow */
if ((!safeWrite) && (out > oend - 2)) return (size_t)-FSE_ERROR_dstSize_tooSmall; /* Buffer overflow */
out[0] = (BYTE)bitStream;
out[1] = (BYTE)(bitStream>>8);
out += 2;
@ -718,7 +723,7 @@ static size_t FSE_writeNCount_generic (void* header, size_t headerBufferSize,
}
if (bitCount>16)
{
if ((!safeWrite) && (out > oend - 2)) return (size_t)-FSE_ERROR_GENERIC; /* Buffer overflow */
if ((!safeWrite) && (out > oend - 2)) return (size_t)-FSE_ERROR_dstSize_tooSmall; /* Buffer overflow */
out[0] = (BYTE)bitStream;
out[1] = (BYTE)(bitStream>>8);
out += 2;
@ -728,7 +733,7 @@ static size_t FSE_writeNCount_generic (void* header, size_t headerBufferSize,
}
/* flush remaining bitStream */
if ((!safeWrite) && (out > oend - 2)) return (size_t)-FSE_ERROR_GENERIC; /* Buffer overflow */
if ((!safeWrite) && (out > oend - 2)) return (size_t)-FSE_ERROR_dstSize_tooSmall; /* Buffer overflow */
out[0] = (BYTE)bitStream;
out[1] = (BYTE)(bitStream>>8);
out+= (bitCount+7) /8;
@ -899,21 +904,6 @@ unsigned FSE_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxS
}
typedef struct
{
U32 id;
U32 count;
} rank_t;
int FSE_compareRankT(const void* r1, const void* r2)
{
const rank_t* R1 = (const rank_t*)r1;
const rank_t* R2 = (const rank_t*)r2;
return 2 * (R1->count < R2->count) - 1;
}
/* Secondary normalization method.
To be used when primary method fails. */
@ -1140,12 +1130,15 @@ size_t FSE_buildCTable_rle (FSE_CTable* ct, BYTE symbolValue)
}
void FSE_initCStream(FSE_CStream_t* bitC, void* start)
size_t FSE_initCStream(FSE_CStream_t* bitC, void* start, size_t maxSize)
{
if (maxSize < 8) return (size_t)-FSE_ERROR_dstSize_tooSmall;
bitC->bitContainer = 0;
bitC->bitPos = 0; /* reserved for unusedBits */
bitC->bitPos = 0;
bitC->startPtr = (char*)start;
bitC->ptr = bitC->startPtr;
bitC->endPtr = bitC->startPtr + maxSize - sizeof(bitC->ptr);
return 0;
}
void FSE_initCState(FSE_CState_t* statePtr, const FSE_CTable* ct)
@ -1179,13 +1172,26 @@ void FSE_encodeSymbol(FSE_CStream_t* bitC, FSE_CState_t* statePtr, U32 symbol)
statePtr->value = stateTable[ (statePtr->value >> nbBitsOut) + symbolTT.deltaFindState];
}
void FSE_flushBitsFast(FSE_CStream_t* bitC) /* only if dst buffer is large enough ( >= FSE_compressBound()) */
{
size_t nbBytes = bitC->bitPos >> 3;
FSE_writeLEST(bitC->ptr, bitC->bitContainer);
bitC->ptr += nbBytes;
bitC->bitPos &= 7;
bitC->bitContainer >>= nbBytes*8;
}
void FSE_flushBits(FSE_CStream_t* bitC)
{
size_t nbBytes = bitC->bitPos >> 3;
FSE_writeLEST(bitC->ptr, bitC->bitContainer);
bitC->bitPos &= 7;
bitC->ptr += nbBytes;
bitC->bitContainer >>= nbBytes*8;
if (bitC->ptr <= bitC->endPtr)
{
bitC->bitPos &= 7;
bitC->bitContainer >>= nbBytes*8;
return;
}
}
void FSE_flushCState(FSE_CStream_t* bitC, const FSE_CState_t* statePtr)
@ -1199,9 +1205,12 @@ size_t FSE_closeCStream(FSE_CStream_t* bitC)
{
char* endPtr;
FSE_addBits(bitC, 1, 1);
FSE_addBitsFast(bitC, 1, 1);
FSE_flushBits(bitC);
if (bitC->bitPos > 7) /* still some data to flush => too close to buffer's end */
return 0; /* not compressible */
endPtr = bitC->ptr;
endPtr += bitC->bitPos > 0;
@ -1209,31 +1218,34 @@ size_t FSE_closeCStream(FSE_CStream_t* bitC)
}
size_t FSE_compress_usingCTable (void* dst, size_t dstSize,
static size_t FSE_compress_usingCTable_generic (void* dst, size_t dstSize,
const void* src, size_t srcSize,
const FSE_CTable* ct)
const FSE_CTable* ct, const unsigned fast)
{
const BYTE* const istart = (const BYTE*) src;
const BYTE* ip;
const BYTE* const iend = istart + srcSize;
size_t errorCode;
FSE_CStream_t bitC;
FSE_CState_t CState1, CState2;
/* init */
(void)dstSize; /* objective : ensure it fits into dstBuffer (Todo) */
FSE_initCStream(&bitC, dst);
errorCode = FSE_initCStream(&bitC, dst, dstSize);
if (FSE_isError(errorCode)) return 0;
FSE_initCState(&CState1, ct);
CState2 = CState1;
ip=iend;
#define FSE_FLUSHBITS(s) (fast ? FSE_flushBitsFast(s) : FSE_flushBits(s))
/* join to even */
if (srcSize & 1)
{
FSE_encodeSymbol(&bitC, &CState1, *--ip);
FSE_flushBits(&bitC);
FSE_FLUSHBITS(&bitC);
}
/* join to mod 4 */
@ -1241,16 +1253,16 @@ size_t FSE_compress_usingCTable (void* dst, size_t dstSize,
{
FSE_encodeSymbol(&bitC, &CState2, *--ip);
FSE_encodeSymbol(&bitC, &CState1, *--ip);
FSE_flushBits(&bitC);
FSE_FLUSHBITS(&bitC);
}
/* 2 or 4 encoding per loop */
while (ip>istart)
for ( ; ip>istart ; )
{
FSE_encodeSymbol(&bitC, &CState2, *--ip);
if (sizeof(bitC.bitContainer)*8 < FSE_MAX_TABLELOG*2+7 ) /* this test must be static */
FSE_flushBits(&bitC);
FSE_FLUSHBITS(&bitC);
FSE_encodeSymbol(&bitC, &CState1, *--ip);
@ -1260,7 +1272,7 @@ size_t FSE_compress_usingCTable (void* dst, size_t dstSize,
FSE_encodeSymbol(&bitC, &CState1, *--ip);
}
FSE_flushBits(&bitC);
FSE_FLUSHBITS(&bitC);
}
FSE_flushCState(&bitC, &CState2);
@ -1268,10 +1280,21 @@ size_t FSE_compress_usingCTable (void* dst, size_t dstSize,
return FSE_closeCStream(&bitC);
}
size_t FSE_compress_usingCTable (void* dst, size_t dstSize,
const void* src, size_t srcSize,
const FSE_CTable* ct)
{
const unsigned fast = (dstSize >= FSE_BLOCKBOUND(srcSize));
if (fast)
return FSE_compress_usingCTable_generic(dst, dstSize, src, srcSize, ct, 1);
else
return FSE_compress_usingCTable_generic(dst, dstSize, src, srcSize, ct, 0);
}
size_t FSE_compressBound(size_t size) { return FSE_COMPRESSBOUND(size); }
size_t FSE_compress2 (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog)
{
const BYTE* const istart = (const BYTE*) src;
@ -1286,9 +1309,8 @@ size_t FSE_compress2 (void* dst, size_t dstSize, const void* src, size_t srcSize
CTable_max_t ct;
size_t errorCode;
/* early out */
if (dstSize < FSE_compressBound(srcSize)) return (size_t)-FSE_ERROR_dstSize_tooSmall;
if (srcSize <= 1) return srcSize; /* Uncompressed or RLE */
/* init conditions */
if (srcSize <= 1) return 0; /* Uncompressible */
if (!maxSymbolValue) maxSymbolValue = FSE_MAX_SYMBOL_VALUE;
if (!tableLog) tableLog = FSE_DEFAULT_TABLELOG;
@ -1303,14 +1325,16 @@ size_t FSE_compress2 (void* dst, size_t dstSize, const void* src, size_t srcSize
if (FSE_isError(errorCode)) return errorCode;
/* Write table description header */
errorCode = FSE_writeNCount (op, FSE_MAX_HEADERSIZE, norm, maxSymbolValue, tableLog);
errorCode = FSE_writeNCount (op, oend-op, norm, maxSymbolValue, tableLog);
if (FSE_isError(errorCode)) return errorCode;
op += errorCode;
/* Compress */
errorCode = FSE_buildCTable (ct, norm, maxSymbolValue, tableLog);
if (FSE_isError(errorCode)) return errorCode;
op += FSE_compress_usingCTable(op, oend - op, ip, srcSize, ct);
errorCode = FSE_compress_usingCTable(op, oend - op, ip, srcSize, ct);
if (errorCode == 0) return 0; /* not enough space for compressed data */
op += errorCode;
/* check compressibility */
if ( (size_t)(op-ostart) >= srcSize-1 )
@ -1330,10 +1354,11 @@ size_t FSE_compress (void* dst, size_t dstSize, const void* src, size_t srcSize)
*********************************************************/
size_t FSE_buildDTable_rle (FSE_DTable* dt, BYTE symbolValue)
{
U32* const base32 = (U32*)dt;
FSE_decode_t* const cell = (FSE_decode_t*)(base32 + 1);
FSE_DTableHeader* const DTableH = (FSE_DTableHeader*)dt;
FSE_decode_t* const cell = (FSE_decode_t*)(dt + 1); /* because dt is unsigned */
base32[0] = 0;
DTableH->tableLog = 0;
DTableH->fastMode = 0;
cell->newState = 0;
cell->symbol = symbolValue;
@ -1345,8 +1370,8 @@ size_t FSE_buildDTable_rle (FSE_DTable* dt, BYTE symbolValue)
size_t FSE_buildDTable_raw (FSE_DTable* dt, unsigned nbBits)
{
U32* const base32 = (U32*)dt;
FSE_decode_t* dinfo = (FSE_decode_t*)(base32 + 1);
FSE_DTableHeader* const DTableH = (FSE_DTableHeader*)dt;
FSE_decode_t* const dinfo = (FSE_decode_t*)(dt + 1); /* because dt is unsigned */
const unsigned tableSize = 1 << nbBits;
const unsigned tableMask = tableSize - 1;
const unsigned maxSymbolValue = tableMask;
@ -1356,7 +1381,8 @@ size_t FSE_buildDTable_raw (FSE_DTable* dt, unsigned nbBits)
if (nbBits < 1) return (size_t)-FSE_ERROR_GENERIC; /* min size */
/* Build Decoding Table */
base32[0] = nbBits;
DTableH->tableLog = (U16)nbBits;
DTableH->fastMode = 1;
for (s=0; s<=maxSymbolValue; s++)
{
dinfo[s].newState = 0;
@ -1469,7 +1495,7 @@ unsigned FSE_reloadDStream(FSE_DStream_t* bitD)
}
if (bitD->ptr == bitD->start)
{
if (bitD->bitsConsumed < sizeof(bitD->bitContainer)*8) return FSE_DStream_partiallyFilled;
if (bitD->bitsConsumed < sizeof(bitD->bitContainer)*8) return FSE_DStream_endOfBuffer;
if (bitD->bitsConsumed == sizeof(bitD->bitContainer)*8) return FSE_DStream_completed;
return FSE_DStream_tooFar;
}
@ -1479,7 +1505,7 @@ unsigned FSE_reloadDStream(FSE_DStream_t* bitD)
if (bitD->ptr - nbBytes < bitD->start)
{
nbBytes = (U32)(bitD->ptr - bitD->start); /* note : necessarily ptr > start */
result = FSE_DStream_partiallyFilled;
result = FSE_DStream_endOfBuffer;
}
bitD->ptr -= nbBytes;
bitD->bitsConsumed -= nbBytes*8;
@ -1491,10 +1517,10 @@ unsigned FSE_reloadDStream(FSE_DStream_t* bitD)
void FSE_initDState(FSE_DState_t* DStatePtr, FSE_DStream_t* bitD, const FSE_DTable* dt)
{
const U32* const base32 = (const U32*)dt;
DStatePtr->state = FSE_readBits(bitD, base32[0]);
const FSE_DTableHeader* const DTableH = (const FSE_DTableHeader*)dt;
DStatePtr->state = FSE_readBits(bitD, DTableH->tableLog);
FSE_reloadDStream(bitD);
DStatePtr->table = base32 + 1;
DStatePtr->table = dt + 1;
}
BYTE FSE_decodeSymbol(FSE_DState_t* DStatePtr, FSE_DStream_t* bitD)
@ -1536,7 +1562,7 @@ unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr)
FORCE_INLINE size_t FSE_decompress_usingDTable_generic(
void* dst, size_t maxDstSize,
const void* cSrc, size_t cSrcSize,
const FSE_DTable* dt, unsigned fast)
const FSE_DTable* dt, const unsigned fast)
{
BYTE* const ostart = (BYTE*) dst;
BYTE* op = ostart;
@ -1605,8 +1631,11 @@ FORCE_INLINE size_t FSE_decompress_usingDTable_generic(
size_t FSE_decompress_usingDTable(void* dst, size_t originalSize,
const void* cSrc, size_t cSrcSize,
const FSE_DTable* dt, size_t fastMode)
const FSE_DTable* dt)
{
const FSE_DTableHeader* DTableH = (const FSE_DTableHeader*)dt;
const U32 fastMode = DTableH->fastMode;
/* select fast mode (static) */
if (fastMode) return FSE_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 1);
return FSE_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 0);
@ -1621,7 +1650,7 @@ size_t FSE_decompress(void* dst, size_t maxDstSize, const void* cSrc, size_t cSr
DTable_max_t dt; /* Static analyzer seems unable to understand this table will be properly initialized later */
unsigned tableLog;
unsigned maxSymbolValue = FSE_MAX_SYMBOL_VALUE;
size_t errorCode, fastMode;
size_t errorCode;
if (cSrcSize<2) return (size_t)-FSE_ERROR_srcSize_wrong; /* too small input size */
@ -1632,11 +1661,11 @@ size_t FSE_decompress(void* dst, size_t maxDstSize, const void* cSrc, size_t cSr
ip += errorCode;
cSrcSize -= errorCode;
fastMode = FSE_buildDTable (dt, counting, maxSymbolValue, tableLog);
if (FSE_isError(fastMode)) return fastMode;
errorCode = FSE_buildDTable (dt, counting, maxSymbolValue, tableLog);
if (FSE_isError(errorCode)) return errorCode;
/* always return, even if it is an error code */
return FSE_decompress_usingDTable (dst, maxDstSize, ip, cSrcSize, dt, fastMode);
return FSE_decompress_usingDTable (dst, maxDstSize, ip, cSrcSize, dt);
}
@ -1664,41 +1693,69 @@ typedef struct nodeElt_s {
BYTE nbBits;
} nodeElt;
/* HUF_writeCTable() :
return : size of saved CTable */
size_t HUF_writeCTable (void* dst, size_t maxDstSize, const HUF_CElt* tree, U32 maxSymbolValue, U32 huffLog)
{
BYTE bitsToWeight[HUF_ABSOLUTEMAX_TABLELOG + 1];
BYTE huffWeight[HUF_MAX_SYMBOL_VALUE + 1];
U32 n;
BYTE* op = (BYTE*)dst;
size_t size;
// check conditions
/* check conditions */
if (maxSymbolValue > HUF_MAX_SYMBOL_VALUE + 1)
return (size_t)-FSE_ERROR_GENERIC;
/* convert to weight */
bitsToWeight[0] = 0;
for (n=1; n<=huffLog; n++)
bitsToWeight[n] = (BYTE)(huffLog + 1 - n);
for (n=0; n<maxSymbolValue; n++)
huffWeight[n] = tree[n].nbBits ? (BYTE)(huffLog + 1 - tree[n].nbBits) : 0;
huffWeight[n] = bitsToWeight[tree[n].nbBits];
size = FSE_compress(op+1, maxDstSize-1, huffWeight, maxSymbolValue); // don't need last symbol stat : implied
size = FSE_compress(op+1, maxDstSize-1, huffWeight, maxSymbolValue); /* don't need last symbol stat : implied */
if (FSE_isError(size)) return size;
if (size >= 128) return (size_t)-FSE_ERROR_GENERIC; // should never happen, since maxSymbolValue <= 255
if (size >= 128) return (size_t)-FSE_ERROR_GENERIC; /* should never happen, since maxSymbolValue <= 255 */
if ((size <= 1) || (size >= maxSymbolValue/2))
{
if (maxSymbolValue > 64) return (size_t)-FSE_ERROR_GENERIC; // special case, not implemented (not possible)
if (size==1) // RLE
if (size==1) /* RLE */
{
op[0] = (BYTE)(128 /*special case*/ + 64 /* RLE */ + (maxSymbolValue-1));
op[1] = huffWeight[0];
return 2;
/* only possible case : serie of 1 (because there are at least 2) */
/* can only be 2^n or (2^n-1), otherwise not an huffman tree */
BYTE code;
switch(maxSymbolValue)
{
case 1: code = 0; break;
case 2: code = 1; break;
case 3: code = 2; break;
case 4: code = 3; break;
case 7: code = 4; break;
case 8: code = 5; break;
case 15: code = 6; break;
case 16: code = 7; break;
case 31: code = 8; break;
case 32: code = 9; break;
case 63: code = 10; break;
case 64: code = 11; break;
case 127: code = 12; break;
case 128: code = 13; break;
default : return (size_t)-FSE_ERROR_corruptionDetected;
}
op[0] = (BYTE)(255-13 + code);
return 1;
}
// Not compressible
/* Not compressible */
if (maxSymbolValue > (241-128)) return (size_t)-FSE_ERROR_GENERIC; /* not implemented (not possible with current format) */
if (((maxSymbolValue+1)/2) + 1 > maxDstSize) return (size_t)-FSE_ERROR_dstSize_tooSmall; /* not enough space within dst buffer */
op[0] = (BYTE)(128 /*special case*/ + 0 /* Not Compressible */ + (maxSymbolValue-1));
huffWeight[maxSymbolValue] = 0; /* to be sure it doesn't cause issue in final combination */
for (n=0; n<maxSymbolValue; n+=2)
op[(n/2)+1] = (BYTE)((huffWeight[n] << 4) + huffWeight[n+1]);
return ((maxSymbolValue+1)/2) + 1;
}
// normal case
/* normal header case */
op[0] = (BYTE)size;
return size+1;
}
@ -1709,7 +1766,7 @@ static U32 HUF_setMaxHeight(nodeElt* huffNode, U32 lastNonNull, U32 maxNbBits)
int totalCost = 0;
const U32 largestBits = huffNode[lastNonNull].nbBits;
// early exit : all is fine
/* early exit : all is fine */
if (largestBits <= maxNbBits) return largestBits;
// now we have a few too large elements (at least >= 2)
@ -1900,24 +1957,27 @@ static void HUF_encodeSymbol(FSE_CStream_t* bitCPtr, U32 symbol, const HUF_CElt*
}
#define FSE_FLUSHBITS_1(stream) \
if (sizeof((stream)->bitContainer)*8 < HUF_MAX_TABLELOG*2+7) FSE_flushBits(stream)
if (sizeof((stream)->bitContainer)*8 < HUF_MAX_TABLELOG*2+7) FSE_FLUSHBITS(stream)
#define FSE_FLUSHBITS_2(stream) \
if (sizeof((stream)->bitContainer)*8 < HUF_MAX_TABLELOG*4+7) FSE_flushBits(stream)
if (sizeof((stream)->bitContainer)*8 < HUF_MAX_TABLELOG*4+7) FSE_FLUSHBITS(stream)
static size_t HUF_compress_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, HUF_CElt* CTable)
size_t HUF_compress_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, HUF_CElt* CTable)
{
const BYTE* ip = (const BYTE*) src;
BYTE* const ostart = (BYTE*)dst;
BYTE* op = (BYTE*) ostart;
BYTE* const oend = ostart + dstSize;
U16* jumpTable = (U16*) dst;
size_t n, streamSize;
const unsigned fast = (dstSize >= HUF_BLOCKBOUND(srcSize));
size_t errorCode;
FSE_CStream_t bitC;
/* init */
(void)dstSize; /* objective : ensure it fits into dstBuffer (Todo) */
op += 6; /* jump Table -- could be optimized by delta / deviation */
FSE_initCStream(&bitC, op);
errorCode = FSE_initCStream(&bitC, op, dstSize);
if (FSE_isError(errorCode)) return 0;
n = srcSize & ~15; // mod 16
switch (srcSize & 15)
@ -1929,7 +1989,7 @@ static size_t HUF_compress_usingCTable(void* dst, size_t dstSize, const void* sr
case 13: HUF_encodeSymbol(&bitC, ip[n+12], CTable);
FSE_FLUSHBITS_1(&bitC);
case 12: HUF_encodeSymbol(&bitC, ip[n+11], CTable);
FSE_flushBits(&bitC);
FSE_FLUSHBITS(&bitC);
case 11: HUF_encodeSymbol(&bitC, ip[n+10], CTable);
FSE_FLUSHBITS_1(&bitC);
case 10: HUF_encodeSymbol(&bitC, ip[n+ 9], CTable);
@ -1937,7 +1997,7 @@ static size_t HUF_compress_usingCTable(void* dst, size_t dstSize, const void* sr
case 9 : HUF_encodeSymbol(&bitC, ip[n+ 8], CTable);
FSE_FLUSHBITS_1(&bitC);
case 8 : HUF_encodeSymbol(&bitC, ip[n+ 7], CTable);
FSE_flushBits(&bitC);
FSE_FLUSHBITS(&bitC);
case 7 : HUF_encodeSymbol(&bitC, ip[n+ 6], CTable);
FSE_FLUSHBITS_1(&bitC);
case 6 : HUF_encodeSymbol(&bitC, ip[n+ 5], CTable);
@ -1945,13 +2005,13 @@ static size_t HUF_compress_usingCTable(void* dst, size_t dstSize, const void* sr
case 5 : HUF_encodeSymbol(&bitC, ip[n+ 4], CTable);
FSE_FLUSHBITS_1(&bitC);
case 4 : HUF_encodeSymbol(&bitC, ip[n+ 3], CTable);
FSE_flushBits(&bitC);
FSE_FLUSHBITS(&bitC);
case 3 : HUF_encodeSymbol(&bitC, ip[n+ 2], CTable);
FSE_FLUSHBITS_2(&bitC);
case 2 : HUF_encodeSymbol(&bitC, ip[n+ 1], CTable);
FSE_FLUSHBITS_1(&bitC);
case 1 : HUF_encodeSymbol(&bitC, ip[n+ 0], CTable);
FSE_flushBits(&bitC);
FSE_FLUSHBITS(&bitC);
case 0 :
default: ;
}
@ -1965,13 +2025,15 @@ static size_t HUF_compress_usingCTable(void* dst, size_t dstSize, const void* sr
HUF_encodeSymbol(&bitC, ip[n-12], CTable);
FSE_FLUSHBITS_1(&bitC);
HUF_encodeSymbol(&bitC, ip[n-16], CTable);
FSE_flushBits(&bitC);
FSE_FLUSHBITS(&bitC);
}
streamSize = FSE_closeCStream(&bitC);
if (streamSize==0) return 0; /* not enough space within dst buffer == uncompressible */
FSE_writeLE16(jumpTable, (U16)streamSize);
op += streamSize;
FSE_initCStream(&bitC, op);
errorCode = FSE_initCStream(&bitC, op, oend-op);
if (FSE_isError(errorCode)) return 0;
n = srcSize & ~15; // mod 16
for (; n>0; n-=16)
{
@ -1982,13 +2044,15 @@ static size_t HUF_compress_usingCTable(void* dst, size_t dstSize, const void* sr
HUF_encodeSymbol(&bitC, ip[n-11], CTable);
FSE_FLUSHBITS_1(&bitC);
HUF_encodeSymbol(&bitC, ip[n-15], CTable);
FSE_flushBits(&bitC);
FSE_FLUSHBITS(&bitC);
}
streamSize = FSE_closeCStream(&bitC);
if (streamSize==0) return 0; /* not enough space within dst buffer == uncompressible */
FSE_writeLE16(jumpTable+1, (U16)streamSize);
op += streamSize;
FSE_initCStream(&bitC, op);
errorCode = FSE_initCStream(&bitC, op, oend-op);
if (FSE_isError(errorCode)) return 0;
n = srcSize & ~15; // mod 16
for (; n>0; n-=16)
{
@ -1999,13 +2063,15 @@ static size_t HUF_compress_usingCTable(void* dst, size_t dstSize, const void* sr
HUF_encodeSymbol(&bitC, ip[n-10], CTable);
FSE_FLUSHBITS_1(&bitC);
HUF_encodeSymbol(&bitC, ip[n-14], CTable);
FSE_flushBits(&bitC);
FSE_FLUSHBITS(&bitC);
}
streamSize = FSE_closeCStream(&bitC);
if (streamSize==0) return 0; /* not enough space within dst buffer == uncompressible */
FSE_writeLE16(jumpTable+2, (U16)streamSize);
op += streamSize;
FSE_initCStream(&bitC, op);
errorCode = FSE_initCStream(&bitC, op, oend-op);
if (FSE_isError(errorCode)) return 0;
n = srcSize & ~15; // mod 16
for (; n>0; n-=16)
{
@ -2016,9 +2082,10 @@ static size_t HUF_compress_usingCTable(void* dst, size_t dstSize, const void* sr
HUF_encodeSymbol(&bitC, ip[n- 9], CTable);
FSE_FLUSHBITS_1(&bitC);
HUF_encodeSymbol(&bitC, ip[n-13], CTable);
FSE_flushBits(&bitC);
FSE_FLUSHBITS(&bitC);
}
streamSize = FSE_closeCStream(&bitC);
if (streamSize==0) return 0; /* not enough space within dst buffer == uncompressible */
op += streamSize;
return op-ostart;
@ -2036,7 +2103,6 @@ size_t HUF_compress2 (void* dst, size_t dstSize, const void* src, size_t srcSize
size_t errorCode;
/* early out */
if (dstSize < FSE_compressBound(srcSize)) return (size_t)-FSE_ERROR_dstSize_tooSmall;
if (srcSize <= 1) return srcSize; /* Uncompressed or RLE */
if (!maxSymbolValue) maxSymbolValue = HUF_MAX_SYMBOL_VALUE;
if (!huffLog) huffLog = HUF_DEFAULT_TABLELOG;
@ -2062,7 +2128,7 @@ size_t HUF_compress2 (void* dst, size_t dstSize, const void* src, size_t srcSize
/* check compressibility */
if ((size_t)(op-ostart) >= srcSize-1)
return 0;
return op-ostart;
return op-ostart;
}
@ -2084,8 +2150,8 @@ typedef struct {
size_t HUF_readDTable (U16* DTable, const void* src, size_t srcSize)
{
BYTE huffWeight[HUF_MAX_SYMBOL_VALUE + 1];
U32 rankVal[HUF_ABSOLUTEMAX_TABLELOG + 1] = {0};
U32 weightTotal = 0;
U32 rankVal[HUF_ABSOLUTEMAX_TABLELOG + 1]; /* large enough for values from 0 to 16 */
U32 weightTotal;
U32 maxBits;
const BYTE* ip = (const BYTE*) src;
size_t iSize = ip[0];
@ -2094,58 +2160,65 @@ size_t HUF_readDTable (U16* DTable, const void* src, size_t srcSize)
U32 nextRankStart;
HUF_DElt* const dt = (HUF_DElt*)(DTable + 1);
FSE_STATIC_ASSERT(sizeof(HUF_DElt) == sizeof(U16)); // if compilation fails here, assertion is false
if (iSize >= 128) // special case
FSE_STATIC_ASSERT(sizeof(HUF_DElt) == sizeof(U16)); /* if compilation fails here, assertion is false */
//memset(huffWeight, 0, sizeof(huffWeight)); /* should not be necessary, but some analyzer complain ... */
if (iSize >= 128) /* special header */
{
if (iSize >= (128+64)) // RLE
if (iSize >= (242)) /* RLE */
{
if (srcSize < 2) return (size_t)-FSE_ERROR_srcSize_wrong;
oSize = (iSize & 63) + 1;
memset(huffWeight, ip[1], oSize);
iSize = 1;
static int l[14] = { 1, 2, 3, 4, 7, 8, 15, 16, 31, 32, 63, 64, 127, 128 };
oSize = l[iSize-242];
memset(huffWeight, 1, oSize);
iSize = 0;
}
else // Incompressible
else /* Incompressible */
{
oSize = (iSize & 63) + 1;
oSize = iSize - 127;
iSize = ((oSize+1)/2);
if (iSize+1 > srcSize) return (size_t)-FSE_ERROR_srcSize_wrong;
ip += 1;
for (n=0; n<oSize; n+=2)
{
huffWeight[n] = (ip[n/2] >> 4);
huffWeight[n+1] = (ip[n/2] & 15);
huffWeight[n] = ip[n/2] >> 4;
huffWeight[n+1] = ip[n/2] & 15;
}
}
}
else // normal case, header compressed with FSE
else /* header compressed with FSE (normal case) */
{
if (iSize+1 > srcSize) return (size_t)-FSE_ERROR_srcSize_wrong;
oSize = FSE_decompress(huffWeight, HUF_MAX_SYMBOL_VALUE, ip+1, iSize); // max 255 values stored, last is implied
oSize = FSE_decompress(huffWeight, HUF_MAX_SYMBOL_VALUE, ip+1, iSize); /* max 255 values decoded, last one is implied */
if (FSE_isError(oSize)) return oSize;
}
// stats on weights
/* collect weight stats */
memset(rankVal, 0, sizeof(rankVal));
weightTotal = 0;
for (n=0; n<oSize; n++)
{
if (huffWeight[n] >= HUF_ABSOLUTEMAX_TABLELOG) return (size_t)-FSE_ERROR_corruptionDetected;
rankVal[huffWeight[n]]++;
weightTotal += (1 << huffWeight[n]) >> 1;
}
// get last symbol weight(implied)
/* get last non-null symbol weight (implied, total must be 2^n) */
maxBits = FSE_highbit32(weightTotal) + 1;
if (maxBits > DTable[0]) return (size_t)-FSE_ERROR_GENERIC; // DTable is too small
if (maxBits > DTable[0]) return (size_t)-FSE_ERROR_tableLog_tooLarge; /* DTable is too small */
DTable[0] = (U16)maxBits;
{
U32 total = 1 << maxBits;
U32 rest = total - weightTotal;
U32 verif = 1 << FSE_highbit32(rest);
if (verif != rest) return (size_t)-FSE_ERROR_GENERIC; // last value must be a clean power of 2
huffWeight[oSize] = (BYTE)(FSE_highbit32(rest) + 1);
rankVal[huffWeight[oSize]]++;
U32 lastWeight = FSE_highbit32(rest) + 1;
if (verif != rest) return (size_t)-FSE_ERROR_corruptionDetected; /* last value must be a clean power of 2 */
huffWeight[oSize] = (BYTE)lastWeight;
rankVal[lastWeight]++;
}
// Prepare ranks
/* check tree construction validity */
if ((rankVal[1] < 2) || (rankVal[1] & 1)) return (size_t)-FSE_ERROR_corruptionDetected; /* by construction : at least 2 elts of rank 1, must be even */
/* Prepare ranks */
nextRankStart = 0;
for (n=1; n<=maxBits; n++)
{
@ -2154,12 +2227,12 @@ size_t HUF_readDTable (U16* DTable, const void* src, size_t srcSize)
rankVal[n] = current;
}
// fill table
/* fill DTable */
for (n=0; n<=oSize; n++)
{
U32 i;
const U32 w = huffWeight[n];
const U32 length = (1 << w) >> 1;
U32 i;
HUF_DElt D;
D.byte = (BYTE)n; D.nbBits = (BYTE)(maxBits + 1 - w);
for (i = rankVal[w]; i < rankVal[w] + length; i++)
@ -2170,15 +2243,16 @@ size_t HUF_readDTable (U16* DTable, const void* src, size_t srcSize)
return iSize+1;
}
static void HUF_decodeSymbol(BYTE* ptr, FSE_DStream_t* Dstream, const HUF_DElt* dt, U32 dtLog)
static BYTE HUF_decodeSymbol(FSE_DStream_t* Dstream, const HUF_DElt* dt, const U32 dtLog)
{
size_t val = FSE_lookBitsFast(Dstream, dtLog);
BYTE c = dt[val].byte;
const size_t val = FSE_lookBitsFast(Dstream, dtLog); /* note : dtLog >= 1 */
const BYTE c = dt[val].byte;
FSE_skipBits(Dstream, dt[val].nbBits);
*ptr = c;
return c;
}
static size_t HUF_decompress_usingDTable(
static size_t HUF_decompress_usingDTable( /* -3% slower when non static */
void* dst, size_t maxDstSize,
const void* cSrc, size_t cSrcSize,
const U16* DTable)
@ -2206,6 +2280,8 @@ static size_t HUF_decompress_usingDTable(
const char* const start4 = start3 + length3;
FSE_DStream_t bitD1, bitD2, bitD3, bitD4;
if (length1+length2+length3+6 >= cSrcSize) return (size_t)-FSE_ERROR_srcSize_wrong;
errorCode = FSE_initDStream(&bitD1, start1, length1);
if (FSE_isError(errorCode)) return errorCode;
errorCode = FSE_initDStream(&bitD2, start2, length2);
@ -2222,14 +2298,14 @@ static size_t HUF_decompress_usingDTable(
op+=16, reloadStatus = FSE_reloadDStream(&bitD2) | FSE_reloadDStream(&bitD3) | FSE_reloadDStream(&bitD4), FSE_reloadDStream(&bitD1))
{
#define HUF_DECODE_SYMBOL_0(n, Dstream) \
HUF_decodeSymbol(op+n, &Dstream, dt, dtLog);
op[n] = HUF_decodeSymbol(&Dstream, dt, dtLog);
#define HUF_DECODE_SYMBOL_1(n, Dstream) \
HUF_decodeSymbol(op+n, &Dstream, dt, dtLog); \
op[n] = HUF_decodeSymbol(&Dstream, dt, dtLog); \
if (FSE_32bits() && (HUF_MAX_TABLELOG>12)) FSE_reloadDStream(&Dstream)
#define HUF_DECODE_SYMBOL_2(n, Dstream) \
HUF_decodeSymbol(op+n, &Dstream, dt, dtLog); \
op[n] = HUF_decodeSymbol(&Dstream, dt, dtLog); \
if (FSE_32bits()) FSE_reloadDStream(&Dstream)
HUF_DECODE_SYMBOL_1( 0, bitD1);
@ -2250,7 +2326,7 @@ static size_t HUF_decompress_usingDTable(
HUF_DECODE_SYMBOL_0(15, bitD4);
}
if (reloadStatus!=FSE_DStream_completed) /* not complete : some bitStream might be 0 (unfinished) */
if (reloadStatus!=FSE_DStream_completed) /* not complete : some bitStream might be FSE_DStream_unfinished */
return (size_t)-FSE_ERROR_corruptionDetected;
/* tail */
@ -2259,7 +2335,7 @@ static size_t HUF_decompress_usingDTable(
FSE_DStream_t bitTail;
bitTail.ptr = bitD1.ptr;
bitTail.bitsConsumed = bitD1.bitsConsumed;
bitTail.bitContainer = bitD1.bitContainer; // required in case FSE_DStream_partiallyFilled
bitTail.bitContainer = bitD1.bitContainer; // required in case of FSE_DStream_endOfBuffer
bitTail.start = start1;
for ( ; (FSE_reloadDStream(&bitTail) < FSE_DStream_completed) && (op<omax) ; op++)
{

102
lib/fse.h
View File

@ -55,12 +55,11 @@ size_t FSE_decompress(void* dst, size_t maxDstSize,
/*
FSE_compress():
Compress content of buffer 'src', of size 'srcSize', into destination buffer 'dst'.
'dst' buffer must be already allocated, and sized to handle worst case situations.
Worst case size evaluation is provided by FSE_compressBound().
return : size of compressed data
Special values : if return == 0, srcData is not compressible => Nothing is stored within cSrc !!!
if return == 1, srcData is a single byte symbol * srcSize times. Use RLE compression.
if FSE_isError(return), it's an error code.
'dst' buffer must be already allocated. Compression runs faster is maxDstSize >= FSE_compressBound(srcSize)
return : size of compressed data (<= maxDstSize)
Special values : if return == 0, srcData is not compressible => Nothing is stored within dst !!!
if return == 1, srcData is a single byte symbol * srcSize times. Use RLE compression instead.
if FSE_isError(return), compression failed (more details using FSE_getErrorName())
FSE_decompress():
Decompress FSE data from buffer 'cSrc', of size 'cSrcSize',
@ -77,18 +76,18 @@ FSE_decompress():
/******************************************
* Huff0 simple functions
******************************************/
size_t HUF_compress (void* dst, size_t dstSize, const void* src, size_t srcSize);
size_t HUF_compress(void* dst, size_t maxDstSize,
const void* src, size_t srcSize);
size_t HUF_decompress(void* dst, size_t maxDstSize,
const void* cSrc, size_t cSrcSize);
/*
HUF_compress():
Compress content of buffer 'src', of size 'srcSize', into destination buffer 'dst'.
'dst' buffer must be already allocated, and sized to handle worst case situations.
Worst case size evaluation is provided by FSE_compressBound().
return : size of compressed data
Special values : if return == 0, srcData is not compressible => Nothing is stored within cSrc !!!
'dst' buffer must be already allocated. Compression runs faster is maxDstSize >= HUF_compressBound(srcSize)
return : size of compressed data (<= maxDstSize)
Special values : if return == 0, srcData is not compressible => Nothing is stored within dst !!!
if return == 1, srcData is a single byte symbol * srcSize times. Use RLE compression.
if FSE_isError(return), it's an error code.
if FSE_isError(return), compression failed (more details using FSE_getErrorName())
HUF_decompress():
Decompress Huff0 data from buffer 'cSrc', of size 'cSrcSize',
@ -134,18 +133,18 @@ size_t HUF_compress2 (void* dst, size_t dstSize, const void* src, size_t srcSize
FSE_compress() does the following:
1. count symbol occurrence from source[] into table count[]
2. normalize counters so that sum(count[]) == Power_of_2 (2^tableLog)
3. save normalized counters to memory buffer using writeHeader()
3. save normalized counters to memory buffer using writeNCount()
4. build encoding table 'CTable' from normalized counters
5. encode the data stream using encoding table 'CTable'
FSE_decompress() does the following:
1. read normalized counters with readHeader()
1. read normalized counters with readNCount()
2. build decoding table 'DTable' from normalized counters
3. decode the data stream using decoding table 'DTable'
The following API allows to trigger specific sub-functions for advanced tasks.
The following API allows targeting specific sub-functions for advanced tasks.
For example, it's possible to compress several blocks using the same 'CTable',
or to save and provide normalized distribution using one's own method.
or to save and provide normalized distribution using external method.
*/
/* *** COMPRESSION *** */
@ -191,8 +190,8 @@ size_t FSE_writeNCount (void* buffer, size_t bufferSize, const short* normalized
/*
Constructor and Destructor of type FSE_CTable
Not that its size depends on parameters 'tableLog' and 'maxSymbolValue' */
typedef unsigned FSE_CTable; /* don't allocate that. It's just a way to be more restrictive than void */
Note that its size depends on 'tableLog' and 'maxSymbolValue' */
typedef unsigned FSE_CTable; /* don't allocate that. It's just a way to be more restrictive than void* */
FSE_CTable* FSE_createCTable (unsigned tableLog, unsigned maxSymbolValue);
void FSE_freeCTable (FSE_CTable* ct);
@ -201,30 +200,32 @@ FSE_buildCTable():
Builds 'ct', which must be already allocated, using FSE_createCTable()
return : 0
or an errorCode, which can be tested using FSE_isError() */
size_t FSE_buildCTable(FSE_CTable* ct, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog);
size_t FSE_buildCTable(FSE_CTable* ct, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog);
/*
FSE_compress_usingCTable():
Compress 'src' using 'ct' into 'dst' which must be already allocated
return : size of compressed data
return : size of compressed data (<= maxDstSize)
or 0 if compressed data could not fit into 'dst'
or an errorCode, which can be tested using FSE_isError() */
size_t FSE_compress_usingCTable (void* dst, size_t dstSize, const void* src, size_t srcSize, const FSE_CTable* ct);
size_t FSE_compress_usingCTable (void* dst, size_t maxDstSize, const void* src, size_t srcSize, const FSE_CTable* ct);
/*
Tutorial :
----------
The first step is to count all symbols. FSE_count() provides one quick way to do this job.
The first step is to count all symbols. FSE_count() does this job very fast.
Result will be saved into 'count', a table of unsigned int, which must be already allocated, and have 'maxSymbolValuePtr[0]+1' cells.
'src' is a table of bytes of size 'srcSize'. All values within 'src' MUST be <= maxSymbolValuePtr[0]
maxSymbolValuePtr[0] will be updated, with its real value (necessarily <= original value)
FSE_count() will return the number of occurrence of the most frequent symbol.
This can be used to know if there is a single symbol within 'src', and to quickly evaluate its compressibility.
If there is an error, the function will return an ErrorCode (which can be tested using FSE_isError()).
The next step is to normalize the frequencies.
FSE_normalizeCount() will ensure that sum of frequencies is == 2 ^'tableLog'.
It also guarantees a minimum of 1 to any Symbol which frequency is >= 1.
You can use input 'tableLog'==0 to mean "use default tableLog value".
If you are unsure of which tableLog value to use, you can optionally call FSE_optimalTableLog(),
It also guarantees a minimum of 1 to any Symbol with frequency >= 1.
You can use 'tableLog'==0 to mean "use default tableLog value".
If you are unsure of which tableLog value to use, you can ask FSE_optimalTableLog(),
which will provide the optimal valid tableLog given sourceSize, maxSymbolValue, and a user-defined maximum (0 means "default").
The result of FSE_normalizeCount() will be saved into a table,
@ -232,23 +233,23 @@ called 'normalizedCounter', which is a table of signed short.
'normalizedCounter' must be already allocated, and have at least 'maxSymbolValue+1' cells.
The return value is tableLog if everything proceeded as expected.
It is 0 if there is a single symbol within distribution.
If there is an error(typically, invalid tableLog value), the function will return an ErrorCode (which can be tested using FSE_isError()).
If there is an error (ex: invalid tableLog value), the function will return an ErrorCode (which can be tested using FSE_isError()).
'normalizedCounter' can be saved in a compact manner to a memory area using FSE_writeHeader().
'header' buffer must be already allocated.
'normalizedCounter' can be saved in a compact manner to a memory area using FSE_writeNCount().
'buffer' must be already allocated.
For guaranteed success, buffer size must be at least FSE_headerBound().
The result of the function is the number of bytes written into 'header'.
If there is an error, the function will return an ErrorCode (which can be tested using FSE_isError()) (for example, buffer size too small).
The result of the function is the number of bytes written into 'buffer'.
If there is an error, the function will return an ErrorCode (which can be tested using FSE_isError(); ex : buffer size too small).
'normalizedCounter' can then be used to create the compression table 'CTable'.
The space required by 'CTable' must be already allocated. Its size is provided by FSE_sizeof_CTable().
'CTable' must be aligned of 4 bytes boundaries.
The space required by 'CTable' must be already allocated, using FSE_createCTable().
You can then use FSE_buildCTable() to fill 'CTable'.
In both cases, if there is an error, the function will return an ErrorCode (which can be tested using FSE_isError()).
If there is an error, both functions will return an ErrorCode (which can be tested using FSE_isError()).
'CTable' can then be used to compress 'src', with FSE_compress_usingCTable().
Similar to FSE_count(), the convention is that 'src' is assumed to be a table of char of size 'srcSize'
The function returns the size of compressed data (without header).
The function returns the size of compressed data (without header), necessarily <= maxDstSize.
If it returns '0', compressed data could not fit into 'dst'.
If there is an error, the function will return an ErrorCode (which can be tested using FSE_isError()).
*/
@ -265,26 +266,25 @@ size_t FSE_readNCount (short* normalizedCounter, unsigned* maxSymbolValuePtr, un
/*
Constructor and Destructor of type FSE_DTable
Note that its size depends on parameters 'tableLog' */
typedef unsigned FSE_DTable; /* don't allocate that. It's just a way to be more restrictive than void */
Note that its size depends on 'tableLog' */
typedef unsigned FSE_DTable; /* don't allocate that. It's just a way to be more restrictive than void* */
FSE_DTable* FSE_createDTable(unsigned tableLog);
void FSE_freeDTable(FSE_DTable* dt);
/*
FSE_buildDTable():
Builds 'dt', which must be already allocated, using FSE_createDTable()
return : 1 if 'dt' is compatible with fast mode, 0 otherwise,
return : 0,
or an errorCode, which can be tested using FSE_isError() */
size_t FSE_buildDTable (FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog);
/*
FSE_decompress_usingDTable():
Decompress compressed source 'cSrc' of size 'cSrcSize'
using 'dt' into 'dst' which must be already allocated.
Use fastMode==1 only if authorized by result of FSE_buildDTable().
Decompress compressed source 'cSrc' of size 'cSrcSize' using 'dt'
into 'dst' which must be already allocated.
return : size of regenerated data (necessarily <= maxDstSize)
or an errorCode, which can be tested using FSE_isError() */
size_t FSE_decompress_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const FSE_DTable* dt, size_t fastMode);
size_t FSE_decompress_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const FSE_DTable* dt);
/*
Tutorial :
@ -294,26 +294,24 @@ Tutorial :
If block is a single repeated byte, use memset() instead )
The first step is to obtain the normalized frequencies of symbols.
This can be performed by reading a header with FSE_readHeader().
'normalizedCounter' must be already allocated, and have at least 'maxSymbolValuePtr[0]+1' cells of short.
This can be performed by FSE_readNCount() if it was saved using FSE_writeNCount().
'normalizedCounter' must be already allocated, and have at least 'maxSymbolValuePtr[0]+1' cells of signed short.
In practice, that means it's necessary to know 'maxSymbolValue' beforehand,
or size the table to handle worst case situations (typically 256).
FSE_readHeader will provide 'tableLog' and 'maxSymbolValue' stored into the header.
The result of FSE_readHeader() is the number of bytes read from 'header'.
Note that 'headerSize' must be at least 4 bytes, even if useful information is less than that.
FSE_readNCount() will provide 'tableLog' and 'maxSymbolValue'.
The result of FSE_readNCount() is the number of bytes read from 'rBuffer'.
Note that 'rBufferSize' must be at least 4 bytes, even if useful information is less than that.
If there is an error, the function will return an error code, which can be tested using FSE_isError().
The next step is to create the decompression tables 'FSE_DTable' from 'normalizedCounter'.
The next step is to build the decompression tables 'FSE_DTable' from 'normalizedCounter'.
This is performed by the function FSE_buildDTable().
The space required by 'FSE_DTable' must be already allocated using FSE_createDTable().
The function will return 1 if FSE_DTable is compatible with fastMode, 0 otherwise.
If there is an error, the function will return an error code, which can be tested using FSE_isError().
'FSE_DTable' can then be used to decompress 'cSrc', with FSE_decompress_usingDTable().
Only trigger fastMode if it was authorized by the result of FSE_buildDTable(), otherwise decompression will fail.
cSrcSize must be correct, otherwise decompression will fail.
FSE_decompress_usingDTable() result will tell how many bytes were regenerated.
If there is an error, the function will return an error code, which can be tested using FSE_isError().
'cSrcSize' must be strictly correct, otherwise decompression will fail.
FSE_decompress_usingDTable() result will tell how many bytes were regenerated (<=maxDstSize).
If there is an error, the function will return an error code, which can be tested using FSE_isError(). (ex: dst buffer too small)
*/

59
lib/fse_static.h
View File

@ -48,14 +48,21 @@ extern "C" {
/******************************************
* Static allocation
******************************************/
#define FSE_MAX_HEADERSIZE 512
#define FSE_COMPRESSBOUND(size) (size + (size>>7) + FSE_MAX_HEADERSIZE) /* Macro can be useful for static allocation */
/* FSE buffer bounds */
#define FSE_NCOUNTBOUND 512
#define FSE_BLOCKBOUND(size) (size + (size>>7))
#define FSE_COMPRESSBOUND(size) (FSE_NCOUNTBOUND + FSE_BLOCKBOUND(size)) /* Macro version, useful for static allocation */
/* You can statically allocate CTable/DTable as a table of unsigned using below macro */
/* You can statically allocate FSE CTable/DTable as a table of unsigned using below macro */
#define FSE_CTABLE_SIZE_U32(maxTableLog, maxSymbolValue) (1 + (1<<(maxTableLog-1)) + ((maxSymbolValue+1)*2))
#define FSE_DTABLE_SIZE_U32(maxTableLog) (1 + (1<<maxTableLog))
/* You can statically allocate a Huff0 DTable as a table of unsigned char using below macro */
/* Huff0 buffer bounds */
#define HUF_CTABLEBOUND 129
#define HUF_BLOCKBOUND(size) (size + (size>>8) + 8) /* only true if pre-filtered with fast heuristic */
#define HUF_COMPRESSBOUND(size) (HUF_CTABLEBOUND + HUF_BLOCKBOUND(size)) /* Macro version, useful for static allocation */
/* You can statically allocate Huff0 DTable as a table of unsigned short using below macro */
#define HUF_DTABLE_SIZE_U16(maxTableLog) (1 + (1<<maxTableLog))
#define HUF_CREATE_STATIC_DTABLE(DTable, maxTableLog) \
unsigned short DTable[HUF_DTABLE_SIZE_U16(maxTableLog)] = { maxTableLog }
@ -102,6 +109,7 @@ size_t FSE_buildDTable_rle (FSE_DTable* dt, unsigned char symbolValue);
You will want to enable link-time-optimization to ensure these functions are properly inlined in your binary.
Visual seems to do it automatically.
For gcc or clang, you'll need to add -flto flag at compilation and linking stages.
If none of these solutions is applicable, include "fse.c" directly.
*/
typedef struct
@ -110,6 +118,7 @@ typedef struct
int bitPos;
char* startPtr;
char* ptr;
char* endPtr;
} FSE_CStream_t;
typedef struct
@ -120,7 +129,7 @@ typedef struct
unsigned stateLog;
} FSE_CState_t;
void FSE_initCStream(FSE_CStream_t* bitC, void* dstBuffer);
size_t FSE_initCStream(FSE_CStream_t* bitC, void* dstBuffer, size_t maxDstSize);
void FSE_initCState(FSE_CState_t* CStatePtr, const FSE_CTable* ct);
void FSE_encodeSymbol(FSE_CStream_t* bitC, FSE_CState_t* CStatePtr, unsigned symbol);
@ -139,17 +148,18 @@ So the first symbol you will encode is the last you will decode, like a LIFO sta
You will need a few variables to track your CStream. They are :
FSE_CTable ct; // Provided by FSE_buildCTable()
FSE_CStream_t bitC; // bitStream tracking structure
FSE_CState_t state; // State tracking structure (can have several)
FSE_CTable ct; // Provided by FSE_buildCTable()
FSE_CStream_t bitStream; // bitStream tracking structure
FSE_CState_t state; // State tracking structure (can have several)
The first thing to do is to init bitStream and state.
FSE_initCStream(&bitC, dstBuffer);
size_t errorCode = FSE_initCStream(&bitStream, dstBuffer, maxDstSize);
FSE_initCState(&state, ct);
Note that FSE_initCStream() can produce an error code, so its result should be tested, using FSE_isError();
You can then encode your input data, byte after byte.
FSE_encodeByte() outputs a maximum of 'tableLog' bits at a time.
FSE_encodeSymbol() outputs a maximum of 'tableLog' bits at a time.
Remember decoding will be done in reverse direction.
FSE_encodeByte(&bitStream, &state, symbol);
@ -165,8 +175,9 @@ Writing data to memory is a manual operation, performed by the flushBits functio
Your last FSE encoding operation shall be to flush your last state value(s).
FSE_flushState(&bitStream, &state);
Finally, you must then close the bitStream.
The function returns the size in bytes of CStream.
Finally, you must close the bitStream.
The function returns the size of CStream in bytes.
If data couldn't fit into dstBuffer, it will return a 0 ( == not compressible)
If there is an error, it returns an errorCode (which can be tested using FSE_isError()).
size_t size = FSE_closeCStream(&bitStream);
*/
@ -201,7 +212,7 @@ unsigned FSE_endOfDStream(const FSE_DStream_t* bitD);
unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr);
typedef enum { FSE_DStream_unfinished = 0,
FSE_DStream_partiallyFilled = 1,
FSE_DStream_endOfBuffer = 1,
FSE_DStream_completed = 2,
FSE_DStream_tooFar = 3 } FSE_DStream_status; /* result of FSE_reloadDStream() */
/* 1,2,4,8 would be better for bitmap combinations, but slows down performance a bit ... ?! */
@ -213,16 +224,16 @@ and also any other bitFields you put in, **in reverse order**.
You will need a few variables to track your bitStream. They are :
FSE_DStream_t DStream; // Stream context
FSE_DState_t DState; // State context. Multiple ones are possible
FSE_DTable dt; // Decoding table, provided by FSE_buildDTable()
U32 tableLog; // Provided by FSE_readHeader()
FSE_DStream_t DStream; // Stream context
FSE_DState_t DState; // State context. Multiple ones are possible
FSE_DTable* DTablePtr; // Decoding table, provided by FSE_buildDTable()
The first thing to do is to init the bitStream.
errorCode = FSE_initDStream(&DStream, &optionalId, srcBuffer, srcSize);
errorCode = FSE_initDStream(&DStream, srcBuffer, srcSize);
You should then retrieve your initial state(s) :
errorCode = FSE_initDState(&DState, &DStream, dt, tableLog);
You should then retrieve your initial state(s)
(in reverse flushing order if you have several ones) :
errorCode = FSE_initDState(&DState, &DStream, DTablePtr);
You can then decode your data, symbol after symbol.
For information the maximum number of bits read by FSE_decodeSymbol() is 'tableLog'.
@ -239,11 +250,11 @@ Refueling the register from memory is manually performed by the reload method.
FSE_reloadDStream() result tells if there is still some more data to read from DStream.
FSE_DStream_unfinished : there is still some data left into the DStream.
FSE_DStream_partiallyFilled : Dstream reached end of buffer. Its container may no longer be completely filled.
FSE_DStream_endOfBuffer : Dstream reached end of buffer. Its container may no longer be completely filled.
FSE_DStream_completed : Dstream reached its exact end, corresponding in general to decompression completed.
FSE_DStream_tooFar : Dstream went too far. Decompression result is corrupted.
When reaching end of buffer (FSE_DStream_partiallyFilled), progress slowly, notably if you decode multiple symbols per loop,
When reaching end of buffer (FSE_DStream_endOfBuffer), progress slowly, notably if you decode multiple symbols per loop,
to properly detect the exact end of stream.
After each decoded symbol, check if DStream is fully consumed using this simple test :
FSE_reloadDStream(&DStream) >= FSE_DStream_completed
@ -251,7 +262,7 @@ After each decoded symbol, check if DStream is fully consumed using this simple
When it's done, verify decompression is fully completed, by checking both DStream and the relevant states.
Checking if DStream has reached its end is performed by :
FSE_endOfDStream(&DStream);
Check also the states. There might be some entropy left there, able to decode some high probability (>50%) symbol.
Check also the states. There might be some symbols left there, if some high probability ones (>50%) are possible.
FSE_endOfDState(&DState);
*/
@ -263,7 +274,7 @@ size_t FSE_readBitsFast(FSE_DStream_t* bitD, unsigned nbBits);
/* faster, but works only if nbBits >= 1 (otherwise, result will be corrupted) */
unsigned char FSE_decodeSymbolFast(FSE_DState_t* DStatePtr, FSE_DStream_t* bitD);
/* faster, but works only if nbBits >= 1 (otherwise, result will be corrupted) */
/* faster, but works only if allways nbBits >= 1 (otherwise, result will be corrupted) */
#if defined (__cplusplus)

95
lib/zstd.c
View File

@ -536,64 +536,6 @@ static size_t ZSTD_noCompressBlock (void* dst, size_t maxDstSize, const void* sr
}
/* return : size of CStream in bits */
size_t ZSTD_compressLiterals_usingCTable(void* dst, size_t dstSize,
const void* src, size_t srcSize,
const FSE_CTable* CTable)
{
const BYTE* const istart = (const BYTE*)src;
const BYTE* ip = istart;
const BYTE* const iend = istart + srcSize;
FSE_CStream_t bitC;
FSE_CState_t CState1, CState2;
/* init */
(void)dstSize; // objective : ensure it fits into dstBuffer (Todo)
FSE_initCStream(&bitC, dst);
FSE_initCState(&CState1, CTable);
CState2 = CState1;
/* Note : at this stage, srcSize > LITERALS_NOENTROPY (checked by ZSTD_compressLiterals()) */
// join to mod 2
if (srcSize & 1)
{
FSE_encodeSymbol(&bitC, &CState1, *ip++);
FSE_flushBits(&bitC);
}
// join to mod 4
if ((sizeof(size_t)*8 > LitFSELog*4+7 ) && (srcSize & 2)) // test bit 2
{
FSE_encodeSymbol(&bitC, &CState2, *ip++);
FSE_encodeSymbol(&bitC, &CState1, *ip++);
FSE_flushBits(&bitC);
}
// 2 or 4 encoding per loop
while (ip<iend)
{
FSE_encodeSymbol(&bitC, &CState2, *ip++);
if (sizeof(size_t)*8 < LitFSELog*2+7 ) // this test must be static
FSE_flushBits(&bitC);
FSE_encodeSymbol(&bitC, &CState1, *ip++);
if (sizeof(size_t)*8 > LitFSELog*4+7 ) // this test must be static
{
FSE_encodeSymbol(&bitC, &CState2, *ip++);
FSE_encodeSymbol(&bitC, &CState1, *ip++);
}
FSE_flushBits(&bitC);
}
FSE_flushCState(&bitC, &CState2);
FSE_flushCState(&bitC, &CState1);
return FSE_closeCStream(&bitC);
}
size_t ZSTD_minGain(size_t srcSize)
{
return (srcSize >> 6) + 1;
@ -693,7 +635,6 @@ static size_t ZSTD_compressSequences(BYTE* dst, size_t maxDstSize,
const seqStore_t* seqStorePtr,
size_t lastLLSize, size_t srcSize)
{
FSE_CStream_t blockStream;
U32 count[256];
S16 norm[256];
size_t mostFrequent;
@ -710,8 +651,9 @@ static size_t ZSTD_compressSequences(BYTE* dst, size_t maxDstSize,
const U32* op_offset = seqStorePtr->offset;
const BYTE* op_matchLength = seqStorePtr->matchLength;
const size_t nbSeq = op_litLength - op_litLength_start;
BYTE* op;
BYTE* offsetBits_start = seqStorePtr->offCodeStart;
BYTE* op = dst;
BYTE* const oend = dst + maxDstSize;
BYTE* const offsetBits_start = seqStorePtr->offCodeStart;
BYTE* offsetBitsPtr = offsetBits_start;
const size_t minGain = ZSTD_minGain(srcSize);
const size_t maxCSize = srcSize - minGain;
@ -719,10 +661,8 @@ static size_t ZSTD_compressSequences(BYTE* dst, size_t maxDstSize,
const size_t maxLSize = maxCSize > minSeqSize ? maxCSize - minSeqSize : 0;
BYTE* seqHead;
/* init */
op = dst;
/* Encode literals */
/* Compress literals */
{
size_t cSize;
size_t litSize = op_lit - op_lit_start;
@ -768,7 +708,7 @@ static size_t ZSTD_compressSequences(BYTE* dst, size_t maxDstSize,
op += dumpsLength;
}
/* Encoding table of Literal Lengths */
/* CTable for Literal Lengths */
max = MaxLL;
mostFrequent = FSE_countFast(count, &max, seqStorePtr->litLengthStart, nbSeq);
if ((mostFrequent == nbSeq) && (nbSeq > 2))
@ -786,14 +726,14 @@ static size_t ZSTD_compressSequences(BYTE* dst, size_t maxDstSize,
{
tableLog = FSE_optimalTableLog(LLFSELog, nbSeq, max);
FSE_normalizeCount(norm, tableLog, count, nbSeq, max);
op += FSE_writeNCount(op, maxDstSize, norm, max, tableLog);
op += FSE_writeNCount(op, oend-op, norm, max, tableLog);
FSE_buildCTable(CTable_LitLength, norm, max, tableLog);
LLtype = bt_compressed;
}
/* Encoding table of Offsets */
/* CTable for Offsets codes */
{
/* create OffsetBits */
/* create Offset codes */
size_t i;
const U32* const op_offset_start = seqStorePtr->offsetStart;
max = MaxOff;
@ -820,12 +760,12 @@ static size_t ZSTD_compressSequences(BYTE* dst, size_t maxDstSize,
{
tableLog = FSE_optimalTableLog(OffFSELog, nbSeq, max);
FSE_normalizeCount(norm, tableLog, count, nbSeq, max);
op += FSE_writeNCount(op, maxDstSize, norm, max, tableLog);
op += FSE_writeNCount(op, oend-op, norm, max, tableLog);
FSE_buildCTable(CTable_OffsetBits, norm, max, tableLog);
Offtype = bt_compressed;
}
/* Encoding Table of MatchLengths */
/* CTable for MatchLengths */
max = MaxML;
mostFrequent = FSE_countFast(count, &max, seqStorePtr->matchLengthStart, nbSeq);
if ((mostFrequent == nbSeq) && (nbSeq > 2))
@ -843,20 +783,23 @@ static size_t ZSTD_compressSequences(BYTE* dst, size_t maxDstSize,
{
tableLog = FSE_optimalTableLog(MLFSELog, nbSeq, max);
FSE_normalizeCount(norm, tableLog, count, nbSeq, max);
op += FSE_writeNCount(op, maxDstSize, norm, max, tableLog);
op += FSE_writeNCount(op, oend-op, norm, max, tableLog);
FSE_buildCTable(CTable_MatchLength, norm, max, tableLog);
MLtype = bt_compressed;
}
seqHead[0] += (BYTE)((LLtype<<6) + (Offtype<<4) + (MLtype<<2));
/* Encoding */
/* Encoding Sequences */
{
size_t streamSize, errorCode;
FSE_CStream_t blockStream;
FSE_CState_t stateMatchLength;
FSE_CState_t stateOffsetBits;
FSE_CState_t stateLitLength;
FSE_initCStream(&blockStream, op);
errorCode = FSE_initCStream(&blockStream, op, oend-op);
if (FSE_isError(errorCode)) return 0; /* not enough space remaining */
FSE_initCState(&stateMatchLength, CTable_MatchLength);
FSE_initCState(&stateOffsetBits, CTable_OffsetBits);
FSE_initCState(&stateLitLength, CTable_LitLength);
@ -880,9 +823,11 @@ static size_t ZSTD_compressSequences(BYTE* dst, size_t maxDstSize,
FSE_flushCState(&blockStream, &stateMatchLength);
FSE_flushCState(&blockStream, &stateOffsetBits);
FSE_flushCState(&blockStream, &stateLitLength);
}
op += FSE_closeCStream(&blockStream);
streamSize = FSE_closeCStream(&blockStream);
if (streamSize==0) return 0; /* not enough space */
op += streamSize;
}
/* check compressibility */
if ((size_t)(op-dst) >= maxCSize) return 0;