diff --git a/NEWS b/NEWS index 84dc723d..23e15dda 100644 --- a/NEWS +++ b/NEWS @@ -1,6 +1,7 @@ v0.6.1 Fixed : Legacy codec v0.5 compatible with dictionary decompression Fixed : Decoder corruption error (#173) +Fixed : null-string roundtrip (#176) New : midipix support v0.6.0 diff --git a/lib/common/bitstream.h b/lib/common/bitstream.h index 6b3f13df..48021dfa 100644 --- a/lib/common/bitstream.h +++ b/lib/common/bitstream.h @@ -84,7 +84,7 @@ MEM_STATIC size_t BIT_closeCStream(BIT_CStream_t* bitC); /* Start with initCStream, providing the size of buffer to write into. * bitStream will never write outside of this buffer. -* `dstCapacity` must be >= sizeof(size_t), otherwise @return will be an error code. +* `dstCapacity` must be >= sizeof(bitD->bitContainer), otherwise @return will be an error code. * * bits are first added to a local register. * Local register is size_t, hence 64-bits on 64-bits systems, or 32-bits on 32-bits systems. @@ -128,7 +128,7 @@ MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* bitD); * Local register size is 64-bits on 64-bits systems, 32-bits on 32-bits systems (size_t). * You can then retrieve bitFields stored into the local register, **in reverse order**. * Local register is explicitly reloaded from memory by the BIT_reloadDStream() method. -* A reload guarantee a minimum of ((8*sizeof(size_t))-7) bits when its result is BIT_DStream_unfinished. +* A reload guarantee a minimum of ((8*sizeof(bitD->bitContainer))-7) bits when its result is BIT_DStream_unfinished. * Otherwise, it can be less than that, so proceed accordingly. * Checking if DStream has reached its end can be performed with BIT_endOfDStream(). */ @@ -263,9 +263,9 @@ MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, si { if (srcSize < 1) { memset(bitD, 0, sizeof(*bitD)); return ERROR(srcSize_wrong); } - if (srcSize >= sizeof(size_t)) { /* normal case */ + if (srcSize >= sizeof(bitD->bitContainer)) { /* normal case */ bitD->start = (const char*)srcBuffer; - bitD->ptr = (const char*)srcBuffer + srcSize - sizeof(size_t); + bitD->ptr = (const char*)srcBuffer + srcSize - sizeof(bitD->bitContainer); bitD->bitContainer = MEM_readLEST(bitD->ptr); { BYTE const lastByte = ((const BYTE*)srcBuffer)[srcSize-1]; if (lastByte == 0) return ERROR(GENERIC); /* endMark not present */ @@ -276,63 +276,58 @@ MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, si bitD->bitContainer = *(const BYTE*)(bitD->start); switch(srcSize) { - case 7: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[6]) << (sizeof(size_t)*8 - 16); - case 6: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[5]) << (sizeof(size_t)*8 - 24); - case 5: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[4]) << (sizeof(size_t)*8 - 32); - case 4: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[3]) << 24; - case 3: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[2]) << 16; - case 2: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[1]) << 8; + case 7: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[6]) << (sizeof(bitD->bitContainer)*8 - 16); + case 6: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[5]) << (sizeof(bitD->bitContainer)*8 - 24); + case 5: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[4]) << (sizeof(bitD->bitContainer)*8 - 32); + case 4: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[3]) << 24; + case 3: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[2]) << 16; + case 2: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[1]) << 8; default:; } { BYTE const lastByte = ((const BYTE*)srcBuffer)[srcSize-1]; if (lastByte == 0) return ERROR(GENERIC); /* endMark not present */ bitD->bitsConsumed = 8 - BIT_highbit32(lastByte); } - bitD->bitsConsumed += (U32)(sizeof(size_t) - srcSize)*8; + bitD->bitsConsumed += (U32)(sizeof(bitD->bitContainer) - srcSize)*8; } return srcSize; } -MEM_STATIC size_t BIT_getUpperBits(size_t bitD, U32 const start) +MEM_STATIC size_t BIT_getUpperBits(size_t bitContainer, U32 const start) { - return bitD >> start; + return bitContainer >> start; } -MEM_STATIC size_t BIT_getMiddleBits(size_t bitD, U32 const nbBits, U32 const start) +MEM_STATIC size_t BIT_getMiddleBits(size_t bitContainer, U32 const start, U32 const nbBits) { #if defined(__BMI__) && defined(__GNUC__) /* experimental */ # if defined(__x86_64__) - if (sizeof(bitD)==8) - return _bextr_u64(bitD, start, nbBits); + if (sizeof(bitContainer)==8) + return _bextr_u64(bitContainer, start, nbBits); else # endif - return _bextr_u32(bitD, start, nbBits); + return _bextr_u32(bitContainer, start, nbBits); #else - return (bitD >> start) & BIT_mask[nbBits]; + return (bitContainer >> start) & BIT_mask[nbBits]; #endif } -MEM_STATIC size_t BIT_getLowerBits(size_t bitD, U32 const nbBits) +MEM_STATIC size_t BIT_getLowerBits(size_t bitContainer, U32 const nbBits) { - return bitD & BIT_mask[nbBits]; + return bitContainer & BIT_mask[nbBits]; } /*! BIT_lookBits() : * Provides next n bits from local register. - * local register is not modified (bits are still present for next read/look). + * local register is not modified. * On 32-bits, maxNbBits==24. * On 64-bits, maxNbBits==56. * @return : value extracted */ MEM_STATIC size_t BIT_lookBits(const BIT_DStream_t* bitD, U32 nbBits) { -#if defined(__BMI__) && defined(__GNUC__) /* experimental */ -# if defined(__x86_64__) - if (sizeof(bitD->bitContainer)==8) - return _bextr_u64(bitD->bitContainer, 64 - bitD->bitsConsumed - nbBits, nbBits); - else -# endif - return _bextr_u32(bitD->bitContainer, 32 - bitD->bitsConsumed - nbBits, nbBits); +#if defined(__BMI__) && defined(__GNUC__) /* experimental; fails if bitD->bitsConsumed + nbBits > sizeof(bitD->bitContainer)*8 */ + return BIT_getMiddleBits(bitD->bitContainer, (sizeof(bitD->bitContainer)*8) - bitD->bitsConsumed - nbBits, nbBits); #else U32 const bitMask = sizeof(bitD->bitContainer)*8 - 1; return ((bitD->bitContainer << (bitD->bitsConsumed & bitMask)) >> 1) >> ((bitMask-nbBits) & bitMask); @@ -377,7 +372,7 @@ MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, U32 nbBits) * Refill `BIT_DStream_t` from src buffer previously defined (see BIT_initDStream() ). * This function is safe, it guarantees it will not read beyond src buffer. * @return : status of `BIT_DStream_t` internal register. - if status == unfinished, internal register is filled with >= (sizeof(size_t)*8 - 7) bits */ + if status == unfinished, internal register is filled with >= (sizeof(bitD->bitContainer)*8 - 7) bits */ MEM_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD) { if (bitD->bitsConsumed > (sizeof(bitD->bitContainer)*8)) /* should never happen */ diff --git a/lib/common/error_private.h b/lib/common/error_private.h index 7bd03065..3f039ae6 100644 --- a/lib/common/error_private.h +++ b/lib/common/error_private.h @@ -78,17 +78,17 @@ typedef ZSTD_ErrorCode ERR_enum; ERR_STATIC unsigned ERR_isError(size_t code) { return (code > ERROR(maxCode)); } -ERR_STATIC ERR_enum ERR_getError(size_t code) { if (!ERR_isError(code)) return (ERR_enum)0; return (ERR_enum) (0-code); } +ERR_STATIC ERR_enum ERR_getErrorCode(size_t code) { if (!ERR_isError(code)) return (ERR_enum)0; return (ERR_enum) (0-code); } /*-**************************************** * Error Strings ******************************************/ -ERR_STATIC const char* ERR_getErrorName(size_t code) +ERR_STATIC const char* ERR_getErrorString(ERR_enum code) { static const char* notErrorCode = "Unspecified error code"; - switch( ERR_getError(code) ) + switch( code ) { case PREFIX(no_error): return "No error detected"; case PREFIX(GENERIC): return "Error (generic)"; @@ -107,10 +107,14 @@ ERR_STATIC const char* ERR_getErrorName(size_t code) case PREFIX(maxSymbolValue_tooSmall): return "Specified maxSymbolValue is too small"; case PREFIX(dictionary_corrupted): return "Dictionary is corrupted"; case PREFIX(maxCode): - default: return notErrorCode; /* impossible, due to ERR_getError() */ + default: return notErrorCode; } } +ERR_STATIC const char* ERR_getErrorName(size_t code) +{ + return ERR_getErrorString(ERR_getErrorCode(code)); +} #if defined (__cplusplus) } diff --git a/lib/common/fse_decompress.c b/lib/common/fse_decompress.c index 8091c845..601edd74 100644 --- a/lib/common/fse_decompress.c +++ b/lib/common/fse_decompress.c @@ -64,6 +64,7 @@ /* ************************************************************** * Error Management ****************************************************************/ +#define FSE_isError ERR_isError #define FSE_STATIC_ASSERT(c) { enum { FSE_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */ @@ -110,7 +111,6 @@ void FSE_freeDTable (FSE_DTable* dt) size_t FSE_buildDTable(FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog) { - FSE_DTableHeader DTableH; void* const tdPtr = dt+1; /* because *dt is unsigned, 32-bits aligned on 32-bits */ FSE_DECODE_TYPE* const tableDecode = (FSE_DECODE_TYPE*) (tdPtr); U16 symbolNext[FSE_MAX_SYMBOL_VALUE+1]; @@ -124,19 +124,21 @@ size_t FSE_buildDTable(FSE_DTable* dt, const short* normalizedCounter, unsigned if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge); /* Init, lay down lowprob symbols */ - DTableH.tableLog = (U16)tableLog; - DTableH.fastMode = 1; - { S16 const largeLimit= (S16)(1 << (tableLog-1)); - U32 s; - for (s=0; s= largeLimit) DTableH.fastMode=0; - symbolNext[s] = normalizedCounter[s]; - } } } - memcpy(dt, &DTableH, sizeof(DTableH)); + { FSE_DTableHeader DTableH; + DTableH.tableLog = (U16)tableLog; + DTableH.fastMode = 1; + { S16 const largeLimit= (S16)(1 << (tableLog-1)); + U32 s; + for (s=0; s= largeLimit) DTableH.fastMode=0; + symbolNext[s] = normalizedCounter[s]; + } } } + memcpy(dt, &DTableH, sizeof(DTableH)); + } /* Spread symbols */ { U32 const tableMask = tableSize-1; diff --git a/lib/common/mem.h b/lib/common/mem.h index dbefcdb2..5abd83d1 100644 --- a/lib/common/mem.h +++ b/lib/common/mem.h @@ -63,7 +63,7 @@ extern "C" { /*-************************************************************** * Basic Types *****************************************************************/ -#if !defined (__VMS) && ( defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) +#if defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) # include typedef uint8_t BYTE; typedef uint16_t U16; @@ -108,8 +108,8 @@ extern "C" { # endif #endif -MEM_STATIC unsigned MEM_32bits(void) { return sizeof(void*)==4; } -MEM_STATIC unsigned MEM_64bits(void) { return sizeof(void*)==8; } +MEM_STATIC unsigned MEM_32bits(void) { return sizeof(size_t)==4; } +MEM_STATIC unsigned MEM_64bits(void) { return sizeof(size_t)==8; } MEM_STATIC unsigned MEM_isLittleEndian(void) { @@ -187,6 +187,48 @@ MEM_STATIC void MEM_write64(void* memPtr, U64 value) #endif /* MEM_FORCE_MEMORY_ACCESS */ +MEM_STATIC U32 MEM_swap32(U32 in) +{ +#if defined(_MSC_VER) /* Visual Studio */ + return _byteswap_ulong(in); +#elif defined (__GNUC__) + return __builtin_bswap32(in); +#else + return ((in << 24) & 0xff000000 ) | + ((in << 8) & 0x00ff0000 ) | + ((in >> 8) & 0x0000ff00 ) | + ((in >> 24) & 0x000000ff ); +#endif +} + +MEM_STATIC U64 MEM_swap64(U64 in) +{ +#if defined(_MSC_VER) /* Visual Studio */ + return _byteswap_uint64(in); +#elif defined (__GNUC__) + return __builtin_bswap64(in); +#else + return ((in << 56) & 0xff00000000000000ULL) | + ((in << 40) & 0x00ff000000000000ULL) | + ((in << 24) & 0x0000ff0000000000ULL) | + ((in << 8) & 0x000000ff00000000ULL) | + ((in >> 8) & 0x00000000ff000000ULL) | + ((in >> 24) & 0x0000000000ff0000ULL) | + ((in >> 40) & 0x000000000000ff00ULL) | + ((in >> 56) & 0x00000000000000ffULL); +#endif +} + +MEM_STATIC size_t MEM_swapST(size_t in) +{ + if (MEM_32bits()) + return (size_t)MEM_swap32((U32)in); + else + return (size_t)MEM_swap64((U64)in); +} + +/*=== Little endian r/w ===*/ + MEM_STATIC U16 MEM_readLE16(const void* memPtr) { if (MEM_isLittleEndian()) @@ -212,51 +254,32 @@ MEM_STATIC U32 MEM_readLE32(const void* memPtr) { if (MEM_isLittleEndian()) return MEM_read32(memPtr); - else { - const BYTE* p = (const BYTE*)memPtr; - return (U32)((U32)p[0] + ((U32)p[1]<<8) + ((U32)p[2]<<16) + ((U32)p[3]<<24)); - } + else + return MEM_swap32(MEM_read32(memPtr)); } MEM_STATIC void MEM_writeLE32(void* memPtr, U32 val32) { - if (MEM_isLittleEndian()) { + if (MEM_isLittleEndian()) MEM_write32(memPtr, val32); - } else { - BYTE* p = (BYTE*)memPtr; - p[0] = (BYTE)val32; - p[1] = (BYTE)(val32>>8); - p[2] = (BYTE)(val32>>16); - p[3] = (BYTE)(val32>>24); - } + else + MEM_write32(memPtr, MEM_swap32(val32)); } MEM_STATIC U64 MEM_readLE64(const void* memPtr) { if (MEM_isLittleEndian()) return MEM_read64(memPtr); - else { - const BYTE* p = (const BYTE*)memPtr; - return (U64)((U64)p[0] + ((U64)p[1]<<8) + ((U64)p[2]<<16) + ((U64)p[3]<<24) - + ((U64)p[4]<<32) + ((U64)p[5]<<40) + ((U64)p[6]<<48) + ((U64)p[7]<<56)); - } + else + return MEM_swap64(MEM_read64(memPtr)); } MEM_STATIC void MEM_writeLE64(void* memPtr, U64 val64) { - if (MEM_isLittleEndian()) { + if (MEM_isLittleEndian()) MEM_write64(memPtr, val64); - } else { - BYTE* p = (BYTE*)memPtr; - p[0] = (BYTE)val64; - p[1] = (BYTE)(val64>>8); - p[2] = (BYTE)(val64>>16); - p[3] = (BYTE)(val64>>24); - p[4] = (BYTE)(val64>>32); - p[5] = (BYTE)(val64>>40); - p[6] = (BYTE)(val64>>48); - p[7] = (BYTE)(val64>>56); - } + else + MEM_write64(memPtr, MEM_swap64(val64)); } MEM_STATIC size_t MEM_readLEST(const void* memPtr) @@ -275,7 +298,58 @@ MEM_STATIC void MEM_writeLEST(void* memPtr, size_t val) MEM_writeLE64(memPtr, (U64)val); } - /* function safe only for comparisons */ +/*=== Big endian r/w ===*/ + +MEM_STATIC U32 MEM_readBE32(const void* memPtr) +{ + if (MEM_isLittleEndian()) + return MEM_swap32(MEM_read32(memPtr)); + else + return MEM_read32(memPtr); +} + +MEM_STATIC void MEM_writeBE32(void* memPtr, U32 val32) +{ + if (MEM_isLittleEndian()) + MEM_write32(memPtr, MEM_swap32(val32)); + else + MEM_write32(memPtr, val32); +} + +MEM_STATIC U64 MEM_readBE64(const void* memPtr) +{ + if (MEM_isLittleEndian()) + return MEM_swap64(MEM_read64(memPtr)); + else + return MEM_read64(memPtr); +} + +MEM_STATIC void MEM_writeBE64(void* memPtr, U64 val64) +{ + if (MEM_isLittleEndian()) + MEM_write64(memPtr, MEM_swap64(val64)); + else + MEM_write64(memPtr, val64); +} + +MEM_STATIC size_t MEM_readBEST(const void* memPtr) +{ + if (MEM_32bits()) + return (size_t)MEM_readBE32(memPtr); + else + return (size_t)MEM_readBE64(memPtr); +} + +MEM_STATIC void MEM_writeBEST(void* memPtr, size_t val) +{ + if (MEM_32bits()) + MEM_writeBE32(memPtr, (U32)val); + else + MEM_writeBE64(memPtr, (U64)val); +} + + +/* function safe only for comparisons */ MEM_STATIC U32 MEM_readMINMATCH(const void* memPtr, U32 length) { switch (length) diff --git a/lib/common/zbuff.h b/lib/common/zbuff.h index e2482054..3caa39ca 100644 --- a/lib/common/zbuff.h +++ b/lib/common/zbuff.h @@ -153,7 +153,7 @@ ZSTDLIB_API unsigned ZBUFF_isError(size_t errorCode); ZSTDLIB_API const char* ZBUFF_getErrorName(size_t errorCode); /** Functions below provide recommended buffer sizes for Compression or Decompression operations. -* These sizes are just hints, and tend to offer better latency */ +* These sizes are just hints, they tend to offer better latency */ ZSTDLIB_API size_t ZBUFF_recommendedCInSize(void); ZSTDLIB_API size_t ZBUFF_recommendedCOutSize(void); ZSTDLIB_API size_t ZBUFF_recommendedDInSize(void); diff --git a/lib/common/zstd_common.c b/lib/common/zstd_common.c index 679e3ace..1b334088 100644 --- a/lib/common/zstd_common.c +++ b/lib/common/zstd_common.c @@ -30,17 +30,23 @@ */ -/* ************************************* +/*-************************************* * Dependencies ***************************************/ #include #include "mem.h" #include "fse_static.h" /* FSE_MIN_TABLELOG */ #include "error_private.h" -#include "zstd.h" /* declaration of ZSTD_isError, ZSTD_getErrorName */ +#include "zstd.h" /* declaration of ZSTD_isError, ZSTD_getErrorName */ #include "zbuff.h" /* declaration of ZBUFF_isError, ZBUFF_getErrorName */ -#include "fse.h" /* declaration of FSE_isError, FSE_getErrorName */ -#include "huf.h" /* declaration of HUF_isError, HUF_getErrorName */ +#include "fse.h" /* declaration of FSE_isError, FSE_getErrorName */ +#include "huf.h" /* declaration of HUF_isError, HUF_getErrorName */ + + +/*-**************************************** +* Version +******************************************/ +unsigned ZSTD_versionNumber (void) { return ZSTD_VERSION_NUMBER; } /*-**************************************** @@ -50,13 +56,17 @@ * tells if a return value is an error code */ unsigned ZSTD_isError(size_t code) { return ERR_isError(code); } +/*! ZSTD_getErrorName() : +* provides error code string from function result (useful for debugging) */ +const char* ZSTD_getErrorName(size_t code) { return ERR_getErrorName(code); } + /*! ZSTD_getError() : * convert a `size_t` function result into a proper ZSTD_errorCode enum */ -ZSTD_ErrorCode ZSTD_getError(size_t code) { return ERR_getError(code); } +ZSTD_ErrorCode ZSTD_getErrorCode(size_t code) { return ERR_getErrorCode(code); } -/*! ZSTD_getErrorName() : -* provides error code string (useful for debugging) */ -const char* ZSTD_getErrorName(size_t code) { return ERR_getErrorName(code); } +/*! ZSTD_getErrorString() : +* provides error code string from enum */ +const char* ZSTD_getErrorString(ZSTD_ErrorCode code) { return ERR_getErrorName(code); } /*-**************************************** @@ -84,7 +94,7 @@ const char* ZBUFF_getErrorName(size_t errorCode) { return ERR_getErrorName(error /*-************************************************************** -* FSE NCount encoding-decoding +* FSE NCount decoding ****************************************************************/ static short FSE_abs(short a) { return a<0 ? -a : a; } @@ -171,7 +181,7 @@ size_t FSE_readNCount (short* normalizedCounter, unsigned* maxSVPtr, unsigned* t ip = iend - 4; } bitStream = MEM_readLE32(ip) >> (bitCount & 31); - } } + } } /* while ((remaining>1) && (charnum<=*maxSVPtr)) */ if (remaining != 1) return ERROR(GENERIC); *maxSVPtr = charnum-1; diff --git a/lib/common/zstd_static.h b/lib/common/zstd_static.h index 0ec5b635..ef42b0a9 100644 --- a/lib/common/zstd_static.h +++ b/lib/common/zstd_static.h @@ -261,7 +261,8 @@ ZSTDLIB_API size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, void* dst, size_t dstCa /*! ZSTD_getErrorCode() : convert a `size_t` function result into a `ZSTD_ErrorCode` enum type, which can be used to compare directly with enum list published into "error_public.h" */ -ZSTDLIB_API ZSTD_ErrorCode ZSTD_getError(size_t code); +ZSTDLIB_API ZSTD_ErrorCode ZSTD_getErrorCode(size_t functionResult); +ZSTDLIB_API const char* ZSTD_getErrorString(ZSTD_ErrorCode code); #if defined (__cplusplus) diff --git a/lib/decompress/huf_decompress.c b/lib/decompress/huf_decompress.c index 55f417ba..e42f7630 100644 --- a/lib/decompress/huf_decompress.c +++ b/lib/decompress/huf_decompress.c @@ -1131,7 +1131,7 @@ size_t HUF_decompress (void* dst, size_t dstSize, const void* cSrc, size_t cSrcS { U32 algoNb = 0; if (Dtime[1] < Dtime[0]) algoNb = 1; - if (Dtime[2] < Dtime[algoNb]) algoNb = 2; + // if (Dtime[2] < Dtime[algoNb]) algoNb = 2; /* current speed of HUF_decompress4X6 is not good */ return decompress[algoNb](dst, dstSize, cSrc, cSrcSize); } diff --git a/lib/decompress/zbuff_decompress.c b/lib/decompress/zbuff_decompress.c index faa81370..858b442f 100644 --- a/lib/decompress/zbuff_decompress.c +++ b/lib/decompress/zbuff_decompress.c @@ -64,14 +64,14 @@ * just follow indications from ZBUFF_decompressContinue() to minimize latency. It should always be <= 128 KB + 3 . * *******************************************************************************/ -typedef enum { ZBUFFds_init, ZBUFFds_readHeader, +typedef enum { ZBUFFds_init, ZBUFFds_loadHeader, ZBUFFds_read, ZBUFFds_load, ZBUFFds_flush } ZBUFF_dStage; /* *** Resource management *** */ struct ZBUFF_DCtx_s { ZSTD_DCtx* zd; ZSTD_frameParams fParams; - size_t blockSize; + ZBUFF_dStage stage; char* inBuff; size_t inBuffSize; size_t inPos; @@ -79,7 +79,9 @@ struct ZBUFF_DCtx_s { size_t outBuffSize; size_t outStart; size_t outEnd; - ZBUFF_dStage stage; + size_t blockSize; + BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX]; + size_t lhSize; }; /* typedef'd to ZBUFF_DCtx within "zstd_buffered.h" */ @@ -108,8 +110,8 @@ size_t ZBUFF_freeDCtx(ZBUFF_DCtx* zbd) size_t ZBUFF_decompressInitDictionary(ZBUFF_DCtx* zbd, const void* dict, size_t dictSize) { - zbd->stage = ZBUFFds_readHeader; - zbd->inPos = zbd->outStart = zbd->outEnd = 0; + zbd->stage = ZBUFFds_loadHeader; + zbd->lhSize = zbd->inPos = zbd->outStart = zbd->outEnd = 0; return ZSTD_decompressBegin_usingDict(zbd->zd, dict, dictSize); } @@ -139,15 +141,29 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbd, case ZBUFFds_init : return ERROR(init_missing); - case ZBUFFds_readHeader : - /* read header from src */ - { size_t const headerSize = ZSTD_getFrameParams(&(zbd->fParams), src, *srcSizePtr); - if (ZSTD_isError(headerSize)) return headerSize; - if (headerSize) { - /* not enough input to decode header : needs headerSize > *srcSizePtr */ - *dstCapacityPtr = 0; - *srcSizePtr = 0; - return headerSize; + case ZBUFFds_loadHeader : + { size_t const hSize = ZSTD_getFrameParams(&(zbd->fParams), zbd->headerBuffer, zbd->lhSize); + if (hSize != 0) { + size_t const toLoad = hSize - zbd->lhSize; /* if hSize!=0, hSize > zbd->lhSize */ + if (ZSTD_isError(hSize)) return hSize; + if (toLoad > (size_t)(iend-ip)) { /* not enough input to load full header */ + memcpy(zbd->headerBuffer + zbd->lhSize, ip, iend-ip); + zbd->lhSize += iend-ip; ip = iend; notDone = 0; + *dstCapacityPtr = 0; + return (hSize - zbd->lhSize) + ZSTD_blockHeaderSize; /* remaining header bytes + next block header */ + } + memcpy(zbd->headerBuffer + zbd->lhSize, ip, toLoad); zbd->lhSize = hSize; ip += toLoad; + break; + } } + + /* Consume header */ + { size_t const h1Size = ZSTD_nextSrcSizeToDecompress(zbd->zd); /* == ZSTD_frameHeaderSize_min */ + size_t const h1Result = ZSTD_decompressContinue(zbd->zd, NULL, 0, zbd->headerBuffer, h1Size); + if (ZSTD_isError(h1Result)) return h1Result; + if (h1Size < zbd->lhSize) { /* long header */ + size_t const h2Size = ZSTD_nextSrcSizeToDecompress(zbd->zd); + size_t const h2Result = ZSTD_decompressContinue(zbd->zd, NULL, 0, zbd->headerBuffer+h1Size, h2Size); + if (ZSTD_isError(h2Result)) return h2Result; } } /* Frame header instruct buffer sizes */ @@ -175,8 +191,7 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbd, notDone = 0; break; } - if ((size_t)(iend-ip) >= neededInSize) { - /* directly decode from src */ + if ((size_t)(iend-ip) >= neededInSize) { /* decode directly from src */ size_t const decodedSize = ZSTD_decompressContinue(zbd->zd, zbd->outBuff + zbd->outStart, zbd->outBuffSize - zbd->outStart, ip, neededInSize); @@ -200,6 +215,7 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbd, ip += loadedSize; zbd->inPos += loadedSize; if (loadedSize < toLoad) { notDone = 0; break; } /* not enough input, wait for more */ + /* decode loaded input */ { size_t const decodedSize = ZSTD_decompressContinue(zbd->zd, zbd->outBuff + zbd->outStart, zbd->outBuffSize - zbd->outStart, diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index 72e52a45..177e9c8a 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -83,18 +83,20 @@ #endif +/*-************************************* +* Macros +***************************************/ +#define ZSTD_isError ERR_isError /* for inlining */ +#define FSE_isError ERR_isError +#define HUF_isError ERR_isError + + /*_******************************************************* * Memory operations **********************************************************/ static void ZSTD_copy4(void* dst, const void* src) { memcpy(dst, src, 4); } -/*-************************************* -* Error Management -***************************************/ -unsigned ZSTD_versionNumber (void) { return ZSTD_VERSION_NUMBER; } - - /*-************************************************************* * Context management ***************************************************************/ @@ -657,7 +659,8 @@ static void ZSTD_decodeSequence(seq_t* seq, seqState_t* seqState) } -FORCE_INLINE size_t ZSTD_execSequence(BYTE* op, +FORCE_INLINE +size_t ZSTD_execSequence(BYTE* op, BYTE* const oend, seq_t sequence, const BYTE** litPtr, const BYTE* const litLimit_8, const BYTE* const base, const BYTE* const vBase, const BYTE* const dictEnd) @@ -964,31 +967,29 @@ size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx) return dctx->expected; } -size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize) +size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize) { /* Sanity check */ if (srcSize != dctx->expected) return ERROR(srcSize_wrong); - ZSTD_checkContinuity(dctx, dst); + if (dstCapacity) ZSTD_checkContinuity(dctx, dst); /* Decompress : frame header; part 1 */ switch (dctx->stage) { case ZSTDds_getFrameHeaderSize : - { - if (srcSize != ZSTD_frameHeaderSize_min) return ERROR(srcSize_wrong); /* impossible */ - dctx->headerSize = ZSTD_frameHeaderSize(src, ZSTD_frameHeaderSize_min); - if (ZSTD_isError(dctx->headerSize)) return dctx->headerSize; - memcpy(dctx->headerBuffer, src, ZSTD_frameHeaderSize_min); - if (dctx->headerSize > ZSTD_frameHeaderSize_min) { - dctx->expected = dctx->headerSize - ZSTD_frameHeaderSize_min; - dctx->stage = ZSTDds_decodeFrameHeader; - return 0; - } - dctx->expected = 0; /* not necessary to copy more */ + if (srcSize != ZSTD_frameHeaderSize_min) return ERROR(srcSize_wrong); /* impossible */ + dctx->headerSize = ZSTD_frameHeaderSize(src, ZSTD_frameHeaderSize_min); + if (ZSTD_isError(dctx->headerSize)) return dctx->headerSize; + memcpy(dctx->headerBuffer, src, ZSTD_frameHeaderSize_min); + if (dctx->headerSize > ZSTD_frameHeaderSize_min) { + dctx->expected = dctx->headerSize - ZSTD_frameHeaderSize_min; + dctx->stage = ZSTDds_decodeFrameHeader; + return 0; } + dctx->expected = 0; /* not necessary to copy more */ + case ZSTDds_decodeFrameHeader: - { - size_t result; + { size_t result; memcpy(dctx->headerBuffer + ZSTD_frameHeaderSize_min, src, dctx->expected); result = ZSTD_decodeFrameHeader(dctx, dctx->headerBuffer, dctx->headerSize); if (ZSTD_isError(result)) return result; @@ -997,8 +998,7 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, co return 0; } case ZSTDds_decodeBlockHeader: - { - blockProperties_t bp; + { blockProperties_t bp; size_t const cBlockSize = ZSTD_getcBlockSize(src, ZSTD_blockHeaderSize, &bp); if (ZSTD_isError(cBlockSize)) return cBlockSize; if (bp.blockType == bt_end) { @@ -1012,16 +1012,14 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, co return 0; } case ZSTDds_decompressBlock: - { - /* Decompress : block content */ - size_t rSize; + { size_t rSize; switch(dctx->bType) { case bt_compressed: - rSize = ZSTD_decompressBlock_internal(dctx, dst, maxDstSize, src, srcSize); + rSize = ZSTD_decompressBlock_internal(dctx, dst, dstCapacity, src, srcSize); break; case bt_raw : - rSize = ZSTD_copyRawBlock(dst, maxDstSize, src, srcSize); + rSize = ZSTD_copyRawBlock(dst, dstCapacity, src, srcSize); break; case bt_rle : return ERROR(GENERIC); /* not yet handled */ diff --git a/lib/legacy/zstd_v05.c b/lib/legacy/zstd_v05.c index 6425ad2c..42e925c4 100644 --- a/lib/legacy/zstd_v05.c +++ b/lib/legacy/zstd_v05.c @@ -2885,8 +2885,7 @@ size_t HUFv05_decompress (void* dst, size_t dstSize, const void* cSrc, size_t cS /* validation checks */ if (dstSize == 0) return ERROR(dstSize_tooSmall); - if (cSrcSize > dstSize) return ERROR(corruption_detected); /* invalid */ - if (cSrcSize == dstSize) { memcpy(dst, cSrc, dstSize); return dstSize; } /* not compressed */ + if (cSrcSize >= dstSize) return ERROR(corruption_detected); /* invalid, or not compressed, but not compressed already dealt with */ if (cSrcSize == 1) { memset(dst, *(const BYTE*)cSrc, dstSize); return dstSize; } /* RLE */ /* decoder timing evaluation */ diff --git a/programs/datagen.c b/programs/datagen.c index 8d872f11..ecc783e1 100644 --- a/programs/datagen.c +++ b/programs/datagen.c @@ -59,7 +59,6 @@ typedef BYTE litDistribTable[LTSIZE]; - /*-******************************************************* * Local Functions *********************************************************/ diff --git a/programs/datagencli.c b/programs/datagencli.c index ce581baf..6122c29b 100644 --- a/programs/datagencli.c +++ b/programs/datagencli.c @@ -24,39 +24,17 @@ - Public forum : https://groups.google.com/forum/#!forum/lz4c */ -/************************************** +/*-************************************ * Includes **************************************/ #include /* fprintf, stderr */ +#include "mem.h" #include "datagen.h" /* RDG_generate */ -/************************************** -* Basic Types -**************************************/ -#if defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */ -# include - typedef uint8_t BYTE; - typedef uint16_t U16; - typedef uint32_t U32; - typedef int32_t S32; - typedef uint64_t U64; -#else - typedef unsigned char BYTE; - typedef unsigned short U16; - typedef unsigned int U32; - typedef signed int S32; - typedef unsigned long long U64; -#endif - - -/************************************** +/*-************************************ * Constants **************************************/ -#ifndef ZSTD_VERSION -# define ZSTD_VERSION "r1" -#endif - #define KB *(1 <<10) #define MB *(1 <<20) #define GB *(1U<<30) @@ -66,7 +44,7 @@ #define COMPRESSIBILITY_DEFAULT 50 -/************************************** +/*-************************************ * Macros **************************************/ #define DISPLAY(...) fprintf(stderr, __VA_ARGS__) @@ -74,10 +52,10 @@ static unsigned displayLevel = 2; -/********************************************************* +/*-******************************************************* * Command line *********************************************************/ -static int usage(char* programName) +static int usage(const char* programName) { DISPLAY( "Compressible data generator\n"); DISPLAY( "Usage :\n"); @@ -92,29 +70,26 @@ static int usage(char* programName) } -int main(int argc, char** argv) +int main(int argc, const char** argv) { int argNb; double proba = (double)COMPRESSIBILITY_DEFAULT / 100; double litProba = 0.0; U64 size = SIZE_DEFAULT; U32 seed = SEED_DEFAULT; - char* programName; + const char* programName; /* Check command line */ programName = argv[0]; - for(argNb=1; argNb='0') && (*argument<='9')) - { - size *= 10; - size += *argument - '0'; - argument++; - } + size *= 10, size += *argument++ - '0'; if (*argument=='K') { size <<= 10; argument++; } if (*argument=='M') { size <<= 20; argument++; } if (*argument=='G') { size <<= 30; argument++; } @@ -137,21 +108,13 @@ int main(int argc, char** argv) argument++; seed=0; while ((*argument>='0') && (*argument<='9')) - { - seed *= 10; - seed += *argument - '0'; - argument++; - } + seed *= 10, seed += *argument++ - '0'; break; case 'P': argument++; proba=0.0; while ((*argument>='0') && (*argument<='9')) - { - proba *= 10; - proba += *argument - '0'; - argument++; - } + proba *= 10, proba += *argument++ - '0'; if (proba>100.) proba=100.; proba /= 100.; break; @@ -159,11 +122,7 @@ int main(int argc, char** argv) argument++; litProba=0.; while ((*argument>='0') && (*argument<='9')) - { - litProba *= 10; - litProba += *argument - '0'; - argument++; - } + litProba *= 10, litProba += *argument++ - '0'; if (litProba>100.) litProba=100.; litProba /= 100.; break; @@ -174,12 +133,9 @@ int main(int argc, char** argv) default: return usage(programName); } - } + } } } /* for(argNb=1; argNb= alreadyLoaded */ + size_t const loadedSize = fread(((char*)ress.srcBuffer) + alreadyLoaded, 1, toLoad, finput); readSize = alreadyLoaded + loadedSize; } diff --git a/programs/zbufftest.c b/programs/zbufftest.c index f44f5d5e..f8d527ef 100644 --- a/programs/zbufftest.c +++ b/programs/zbufftest.c @@ -188,6 +188,33 @@ static int basicUnitTests(U32 seed, double compressibility) DISPLAYLEVEL(4, "OK \n"); } + /* Byte-by-byte decompression test */ + DISPLAYLEVEL(4, "test%3i : decompress byte-by-byte : ", testNb++); + ZBUFF_decompressInitDictionary(zd, CNBuffer, 128 KB); + { size_t r = 1, pIn=0, pOut=0; + while (r) { + size_t inS = 1; + size_t outS = 1; + r = ZBUFF_decompressContinue(zd, ((BYTE*)decodedBuffer)+pOut, &outS, ((BYTE*)compressedBuffer)+pIn, &inS); + pIn += inS; + pOut += outS; + } + readSize = pIn; + genSize = pOut; + } + if (genSize != CNBufferSize) goto _output_error; /* should regenerate the same amount */ + if (readSize != cSize) goto _output_error; /* should have read the entire frame */ + DISPLAYLEVEL(4, "OK \n"); + + /* check regenerated data is byte exact */ + { size_t i; + DISPLAYLEVEL(4, "test%3i : check decompressed result : ", testNb++); + for (i=0; i