facebook
/
zstd
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'refs/remotes/Cyan4973/dev060' into dev 

# Conflicts:
#	lib/zstd_compress.c
dev
inikep 2016-04-07 19:50:03 +02:00
parent 2bf79f9bff d7bee33caf
commit afe1f7994b
6 changed files with 50 additions and 69 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
lib/Makefile
View File

@ -118,8 +118,8 @@ install: libzstd libzstd.pc
@cp -a libzstd.pc $(DESTDIR)$(LIBDIR)/pkgconfig/
@install -m 644 libzstd.a $(DESTDIR)$(LIBDIR)/libzstd.a
@install -m 644 zstd.h $(DESTDIR)$(INCLUDEDIR)/zstd.h
@install -m 644 zstd.h $(DESTDIR)$(INCLUDEDIR)/zbuff.h
@install -m 644 zstd.h $(DESTDIR)$(INCLUDEDIR)/zdict.h
@install -m 644 zbuff.h $(DESTDIR)$(INCLUDEDIR)/zbuff.h
@install -m 644 zdict.h $(DESTDIR)$(INCLUDEDIR)/zdict.h
@echo zstd static and shared library installed
uninstall:
@ -129,8 +129,8 @@ uninstall:
@[ -x $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT_VER) ] && rm -f $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT_VER)
@[ -f $(DESTDIR)$(LIBDIR)/libzstd.a ] && rm -f $(DESTDIR)$(LIBDIR)/libzstd.a
@[ -f $(DESTDIR)$(INCLUDEDIR)/zstd.h ] && rm -f $(DESTDIR)$(INCLUDEDIR)/zstd.h
@[ -f $(DESTDIR)$(INCLUDEDIR)/zstd.h ] && rm -f $(DESTDIR)$(INCLUDEDIR)/zbuff.h
@[ -f $(DESTDIR)$(INCLUDEDIR)/zstd.h ] && rm -f $(DESTDIR)$(INCLUDEDIR)/zdict.h
@[ -f $(DESTDIR)$(INCLUDEDIR)/zbuff.h ] && rm -f $(DESTDIR)$(INCLUDEDIR)/zbuff.h
@[ -f $(DESTDIR)$(INCLUDEDIR)/zdict.h ] && rm -f $(DESTDIR)$(INCLUDEDIR)/zdict.h
@echo zstd libraries successfully uninstalled
endif

43
lib/zstd_compress.c
View File

@ -80,6 +80,7 @@ static void ZSTD_resetSeqStore(seqStore_t* ssPtr)
ssPtr->lit = ssPtr->litStart;
ssPtr->litLength = ssPtr->litLengthStart;
ssPtr->matchLength = ssPtr->matchLengthStart;
ssPtr->longLengthID = 0;
}
@ -183,7 +184,7 @@ void ZSTD_adjustCParams(ZSTD_compressionParameters* params, U64 srcSize, size_t
/* resize params, to use less memory when necessary */
{ U32 const minSrcSize = (srcSize==0) ? 500 : 0;
U64 const rSize = srcSize + dictSize + minSrcSize;
if (rSize < (1<<ZSTD_WINDOWLOG_MAX)) {
if (rSize < ((U64)1<<ZSTD_WINDOWLOG_MAX)) {
U32 const srcLog = ZSTD_highbit((U32)(rSize)-1) + 1;
if (params->windowLog > srcLog) params->windowLog = srcLog;
} }
@ -224,7 +225,7 @@ static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc,
const size_t tableSpace = (chainSize + hSize + h3Size) * sizeof(U32);
/* Check if workSpace is large enough, alloc a new one if needed */
{ size_t const optSpace = ((MaxML+1) + (MaxLL+1) + (1<<Offbits) + (1<<Litbits))*sizeof(U32)
{ size_t const optSpace = ((MaxML+1) + (MaxLL+1) + (MaxOff+1) + (1<<Litbits))*sizeof(U32)
+ (ZSTD_OPT_NUM+1)*(sizeof(ZSTD_match_t) + sizeof(ZSTD_optimal_t));
size_t const neededSpace = tableSpace + (256*sizeof(U32)) /* huffTable */ + tokenSpace
+ ((params.cParams.strategy == ZSTD_btopt) ? optSpace : 0);
@ -258,7 +259,7 @@ static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc,
zc->seqStore.litLengthFreq = zc->seqStore.litFreq + (1<<Litbits);
zc->seqStore.matchLengthFreq = zc->seqStore.litLengthFreq + (MaxLL+1);
zc->seqStore.offCodeFreq = zc->seqStore.matchLengthFreq + (MaxML+1);
zc->seqStore.matchTable = (ZSTD_match_t*)((void*)(zc->seqStore.offCodeFreq + (1<<Offbits)));
zc->seqStore.matchTable = (ZSTD_match_t*)((void*)(zc->seqStore.offCodeFreq + (MaxOff+1)));
zc->seqStore.priceTable = (ZSTD_optimal_t*)((void*)(zc->seqStore.matchTable + ZSTD_OPT_NUM+1));
zc->seqStore.buffer = zc->seqStore.priceTable + ZSTD_OPT_NUM+1;
zc->seqStore.litLengthSum = 0;
@ -620,14 +621,16 @@ void ZSTD_seqToCodes(const seqStore_t* seqStorePtr, size_t const nbSeq)
24, 24, 24, 24, 24, 24, 24, 24,
24, 24, 24, 24, 24, 24, 24, 24 };
const BYTE LL_deltaCode = 19;
U16* const llTable = seqStorePtr->litLengthStart;
const U16* const llTable = seqStorePtr->litLengthStart;
BYTE* const llCodeTable = seqStorePtr->llCodeStart;
size_t u;
for (u=0; u<nbSeq; u++) {
U32 ll = llTable[u];
if (llTable[u] == 65535) { ll = seqStorePtr->longLength; llTable[u] = (U16)ll; }
U32 const ll = llTable[u];
llCodeTable[u] = (ll>63) ? (BYTE)ZSTD_highbit(ll) + LL_deltaCode : LL_Code[ll];
} }
}
if (seqStorePtr->longLengthID==1)
llCodeTable[seqStorePtr->longLengthPos] = MaxLL;
}
/* Offset codes */
{ const U32* const offsetTable = seqStorePtr->offsetStart;
@ -646,14 +649,16 @@ void ZSTD_seqToCodes(const seqStore_t* seqStorePtr, size_t const nbSeq)
42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42,
42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42 };
const BYTE ML_deltaCode = 36;
U16* const mlTable = seqStorePtr->matchLengthStart;
const U16* const mlTable = seqStorePtr->matchLengthStart;
BYTE* const mlCodeTable = seqStorePtr->mlCodeStart;
size_t u;
for (u=0; u<nbSeq; u++) {
U32 ml = mlTable[u];
if (mlTable[u] == 65535) { ml = seqStorePtr->longLength; mlTable[u] = (U16)ml; }
U32 const ml = mlTable[u];
mlCodeTable[u] = (ml>127) ? (BYTE)ZSTD_highbit(ml) + ML_deltaCode : ML_Code[ml];
} }
}
if (seqStorePtr->longLengthID==2)
mlCodeTable[seqStorePtr->longLengthPos] = MaxML;
}
}
@ -738,8 +743,8 @@ size_t ZSTD_compressSequences(ZSTD_CCtx* zc,
Offtype = FSE_ENCODING_RLE;
} else if ((zc->flagStaticTables) && (nbSeq < MAX_SEQ_FOR_STATIC_FSE)) {
Offtype = FSE_ENCODING_STATIC;
} else if ((nbSeq < MIN_SEQ_FOR_DYNAMIC_FSE) || (mostFrequent < (nbSeq >> (Offbits-1)))) {
FSE_buildCTable_raw(CTable_OffsetBits, Offbits);
} else if ((nbSeq < MIN_SEQ_FOR_DYNAMIC_FSE) || (mostFrequent < (nbSeq >> (OF_defaultNormLog-1)))) {
FSE_buildCTable(CTable_OffsetBits, OF_defaultNorm, MaxOff, OF_defaultNormLog);
Offtype = FSE_ENCODING_RAW;
} else {
size_t nbSeq_1 = nbSeq;
@ -864,15 +869,15 @@ MEM_STATIC void ZSTD_storeSeq(seqStore_t* seqStorePtr, size_t litLength, const B
seqStorePtr->lit += litLength;
/* literal Length */
if (litLength>=65535) { *(seqStorePtr->litLength++) = 65535; seqStorePtr->longLength = (U32)litLength; }
else *seqStorePtr->litLength++ = (U16)litLength;
if (litLength>0xFFFF) { seqStorePtr->longLengthID = 1; seqStorePtr->longLengthPos = (U32)(seqStorePtr->litLength - seqStorePtr->litLengthStart); }
*seqStorePtr->litLength++ = (U16)litLength;
/* match offset */
*(seqStorePtr->offset++) = (U32)offsetCode + 1;
/* match Length */
if (matchCode>=65535) { *(seqStorePtr->matchLength++) = 65535; seqStorePtr->longLength = (U32)matchCode; }
else *seqStorePtr->matchLength++ = (U16)matchCode;
if (matchCode>0xFFFF) { seqStorePtr->longLengthID = 2; seqStorePtr->longLengthPos = (U32)(seqStorePtr->matchLength - seqStorePtr->matchLengthStart); }
*seqStorePtr->matchLength++ = (U16)matchCode;
}
@ -1508,13 +1513,13 @@ static size_t ZSTD_BtFindBestMatch_selectMLS_extDict (
/* ***********************
* Hash Chain
*************************/
#define NEXT_IN_CHAIN(d, mask) chainTable[(d) & mask]
/* Update chains up to ip (excluded)
Assumption : always within prefix (ie. not within extDict) */
FORCE_INLINE
@ -2251,7 +2256,7 @@ static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* zc,
const void* dict, size_t dictSize,
ZSTD_parameters params, U64 pledgedSrcSize)
{
U32 hashLog3 = (pledgedSrcSize || pledgedSrcSize >= 8192) ? ZSTD_HASHLOG3_MAX : ((pledgedSrcSize >= 2048) ? ZSTD_HASHLOG3_MIN + 1 : ZSTD_HASHLOG3_MIN);
U32 hashLog3 = (pledgedSrcSize || pledgedSrcSize >= 8192) ? ZSTD_HASHLOG3_MAX : ((pledgedSrcSize >= 2048) ? ZSTD_HASHLOG3_MIN + 1 : ZSTD_HASHLOG3_MIN);
zc->hashLog3 = (params.cParams.searchLength==3) ? hashLog3 : 0;
// printf("windowLog=%d hashLog=%d hashLog3=%d \n", params.windowLog, params.hashLog, zc->hashLog3);

36
lib/zstd_decompress.c
View File

@ -500,34 +500,6 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx,
@return : nb bytes read from src,
or an error code if it fails, testable with ZSTD_isError()
*/
FORCE_INLINE size_t ZSTD_buildSeqTableOff(FSE_DTable* DTable, U32 type, U32 rawBits, U32 maxLog,
const void* src, size_t srcSize)
{
U32 max = (1<<rawBits)-1;
switch(type)
{
case FSE_ENCODING_RLE :
if (!srcSize) return ERROR(srcSize_wrong);
FSE_buildDTable_rle(DTable, (*(const BYTE*)src) & max); /* if *src > max, data is corrupted */
return 1;
case FSE_ENCODING_RAW :
FSE_buildDTable_raw(DTable, rawBits);
return 0;
case FSE_ENCODING_STATIC:
return 0;
default : /* impossible */
case FSE_ENCODING_DYNAMIC :
{ U32 tableLog;
S16 norm[MaxSeq+1];
size_t const headerSize = FSE_readNCount(norm, &max, &tableLog, src, srcSize);
if (FSE_isError(headerSize)) return ERROR(corruption_detected);
if (tableLog > maxLog) return ERROR(corruption_detected);
FSE_buildDTable(DTable, norm, max, tableLog);
return headerSize;
} }
}
FORCE_INLINE size_t ZSTD_buildSeqTable(FSE_DTable* DTable, U32 type, U32 max, U32 maxLog,
const void* src, size_t srcSize,
const S16* defaultNorm, U32 defaultLog)
@ -594,7 +566,7 @@ size_t ZSTD_decodeSeqHeaders(int* nbSeqPtr,
if (ZSTD_isError(bhSize)) return ERROR(corruption_detected);
ip += bhSize;
}
{ size_t const bhSize = ZSTD_buildSeqTableOff(DTableOffb, Offtype, Offbits, OffFSELog, ip, iend-ip);
{ size_t const bhSize = ZSTD_buildSeqTable(DTableOffb, Offtype, MaxOff, OffFSELog, ip, iend-ip, OF_defaultNorm, OF_defaultNormLog);
if (ZSTD_isError(bhSize)) return ERROR(corruption_detected);
ip += bhSize;
}
@ -650,7 +622,7 @@ static void ZSTD_decodeSequence(seq_t* seq, seqState_t* seqState)
0, 1, 3, 7, 0xF, 0x1F, 0x3F, 0x7F,
0xFF, 0x1FF, 0x3FF, 0x7FF, 0xFFF, 0x1FFF, 0x3FFF, 0x7FFF,
0xFFFF, 0x1FFFF, 0x3FFFF, 0x7FFFF, 0xFFFFF, 0x1FFFFF, 0x3FFFFF, 0x7FFFFF,
0xFFFFFF, 0x1FFFFFF, 0x3FFFFFF, /*fake*/ 1, 1, 1, 1, 1 };
0xFFFFFF, 0x1FFFFFF, 0x3FFFFFF, /*fake*/ 1, 1 };
/* sequence */
{ size_t offset;
@ -660,7 +632,7 @@ static void ZSTD_decodeSequence(seq_t* seq, seqState_t* seqState)
offset = OF_base[ofCode] + BIT_readBits(&(seqState->DStream), ofBits); /* <= 26 bits */
if (MEM_32bits()) BIT_reloadDStream(&(seqState->DStream));
}
if (offset < ZSTD_REP_NUM) {
if (llCode == 0 && offset <= 1) offset = 1-offset;
@ -678,7 +650,7 @@ static void ZSTD_decodeSequence(seq_t* seq, seqState_t* seqState)
} else {
offset -= ZSTD_REP_MOVE;
seqState->prevOffset[2] = seqState->prevOffset[1];
seqState->prevOffset[1] = seqState->prevOffset[0];
seqState->prevOffset[1] = seqState->prevOffset[0];
seqState->prevOffset[0] = offset;
}
seq->offset = offset;

24
lib/zstd_internal.h
View File

@ -104,11 +104,10 @@ typedef enum { bt_compressed, bt_raw, bt_rle, bt_end } blockType_t;
#define REPCODE_STARTVALUE 1
#define Litbits 8
#define Offbits 5
#define MaxLit ((1<<Litbits) - 1)
#define MaxML 52
#define MaxLL 35
#define MaxOff ((1<<Offbits)- 1)
#define MaxOff 28
#define MaxSeq MAX(MaxLL, MaxML) /* Assumption : MaxOff < MaxLL,MaxML */
#define MLFSELog 9
#define LLFSELog 9
@ -122,21 +121,25 @@ typedef enum { bt_compressed, bt_raw, bt_rle, bt_end } blockType_t;
static const U32 LL_bits[MaxLL+1] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
1, 1, 1, 1, 2, 2, 3, 3, 4, 6, 7, 8, 9,10,11,12,
13,14,15,16 };
static const S16 LL_defaultNorm[MaxLL+1] = { 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1,
1, 1, 1, 1 };
static const S16 LL_defaultNorm[MaxLL+1] = { 4, 3, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1,
2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 2, 1, 1, 1, 1, 1,
-1,-1,-1,-1 };
static const U32 LL_defaultNormLog = 6;
static const U32 ML_bits[MaxML+1] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
1, 1, 1, 1, 2, 2, 3, 3, 4, 4, 5, 7, 8, 9,10,11,
12,13,14,15,16 };
static const S16 ML_defaultNorm[MaxML+1] = { 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1,
12,13,14,15,16 };
static const S16 ML_defaultNorm[MaxML+1] = { 1, 4, 3, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, };
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,-1,-1,
-1,-1,-1,-1,-1 };
static const U32 ML_defaultNormLog = 6;
static const S16 OF_defaultNorm[MaxOff+1] = { 1, 1, 1, 1, 1, 1, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1,-1,-1,-1,-1,-1 };
static const U32 OF_defaultNormLog = 5;
/*-*******************************************
* Shared functions to include for inlining
@ -219,7 +222,8 @@ typedef struct {
U16* matchLengthStart;
U16* matchLength;
BYTE* mlCodeStart;
U32 longLength;
U32 longLengthID; /* 0 == no longLength; 1 == Lit.longLength; 2 == Match.longLength; */
U32 longLengthPos;
/* opt */
ZSTD_optimal_t* priceTable;
ZSTD_match_t* matchTable;

6
lib/zstd_opt.h
View File

@ -57,7 +57,7 @@ MEM_STATIC void ZSTD_rescaleFreqs(seqStore_t* ssPtr)
ssPtr->litSum = (2<<Litbits);
ssPtr->litLengthSum = MaxLL+1;
ssPtr->matchLengthSum = MaxML+1;
ssPtr->offCodeSum = (1<<Offbits);
ssPtr->offCodeSum = (MaxOff+1);
ssPtr->matchSum = (2<<Litbits);
for (u=0; u<=MaxLit; u++)
@ -551,7 +551,7 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx,
if (opt[cur].off >= ZSTD_REP_NUM) {
opt[cur].rep[2] = opt[cur-mlen].rep[1];
opt[cur].rep[1] = opt[cur-mlen].rep[0];
opt[cur].rep[0] = opt[cur].off - ZSTD_REP_MOVE;
opt[cur].rep[0] = opt[cur].off - ZSTD_REP_MOVE;
ZSTD_LOG_ENCODE("%d: COPYREP_OFF cur=%d mlen=%d rep[0]=%d rep[1]=%d\n", (int)(inr-base), cur, mlen, opt[cur].rep[0], opt[cur].rep[1]);
} else {
opt[cur].rep[2] = (opt[cur].off > 1) ? opt[cur-mlen].rep[1] : opt[cur-mlen].rep[2];
@ -860,7 +860,7 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx,
if (opt[cur].off >= ZSTD_REP_NUM) {
opt[cur].rep[2] = opt[cur-mlen].rep[1];
opt[cur].rep[1] = opt[cur-mlen].rep[0];
opt[cur].rep[0] = opt[cur].off - ZSTD_REP_MOVE;
opt[cur].rep[0] = opt[cur].off - ZSTD_REP_MOVE;
ZSTD_LOG_ENCODE("%d: COPYREP_OFF cur=%d mlen=%d rep[0]=%d rep[1]=%d\n", (int)(inr-base), cur, mlen, opt[cur].rep[0], opt[cur].rep[1]);
} else {
opt[cur].rep[2] = (opt[cur].off > 1) ? opt[cur-mlen].rep[1] : opt[cur-mlen].rep[2];

2
lib/zstd_static.h
View File

@ -57,7 +57,7 @@ extern "C" {
/*-*************************************
* Types
***************************************/
#define ZSTD_WINDOWLOG_MAX (MEM_32bits() ? 25 : 27)
#define ZSTD_WINDOWLOG_MAX ((U32)(MEM_32bits() ? 25 : 27))
#define ZSTD_WINDOWLOG_MIN 18
#define ZSTD_CHAINLOG_MAX (ZSTD_WINDOWLOG_MAX+1)
#define ZSTD_CHAINLOG_MIN 4