facebook/zstd
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

updated streaming API

Browse Source
This commit is contained in:
Yann Collet 2016-08-17 01:39:22 +02:00
parent 94ca85d01b
commit 53e17fbd5e
5 changed files with 128 additions and 138 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

43
lib/compress/zstd_compress.c
View File

@ -2960,16 +2960,15 @@ static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs,
} }
} }
size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_wCursor* output, ZSTD_rCursor* input) size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input)
{ {
size_t sizeRead = input->size; size_t sizeRead = input->size - input->pos;
size_t sizeWritten = output->size; size_t sizeWritten = output->size - output->pos;
size_t const result = ZSTD_compressStream_generic(zcs, output->ptr, &sizeWritten, input->ptr, &sizeRead, zsf_gather); size_t const result = ZSTD_compressStream_generic(zcs,
input->ptr = (const char*)(input->ptr) + sizeRead; (char*)(output->dst) + output->pos, &sizeWritten,
input->size -= sizeRead; (const char*)(input->src) + input->pos, &sizeRead, zsf_gather);
output->ptr = (char*)(output->ptr) + sizeWritten; input->pos += sizeRead;
output->size -= sizeWritten; output->pos += sizeWritten;
output->nbBytesWritten += sizeWritten;
return result; return result;
} }
@ -2978,36 +2977,32 @@ size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_wCursor* output, ZSTD_rCursor
/*! ZSTD_flushStream() : /*! ZSTD_flushStream() :
* @return : amount of data remaining to flush */ * @return : amount of data remaining to flush */
size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_wCursor* output) size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output)
{ {
size_t srcSize = 0; size_t srcSize = 0;
size_t sizeWritten = output->size; size_t sizeWritten = output->size - output->pos;
size_t const result = ZSTD_compressStream_generic(zcs, output->ptr, &sizeWritten, &srcSize, &srcSize, zsf_flush); /* use a valid src address instead of NULL */ size_t const result = ZSTD_compressStream_generic(zcs, output->dst + output->pos, &sizeWritten, &srcSize, &srcSize, zsf_flush); /* use a valid src address instead of NULL */
output->ptr = (char*)(output->ptr) + sizeWritten; output->pos += sizeWritten;
output->size -= sizeWritten;
output->nbBytesWritten += sizeWritten;
if (ZSTD_isError(result)) return result; if (ZSTD_isError(result)) return result;
return zcs->outBuffContentSize - zcs->outBuffFlushedSize; /* remaining to flush */ return zcs->outBuffContentSize - zcs->outBuffFlushedSize; /* remaining to flush */
} }
size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_wCursor* output) size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output)
{ {
BYTE* const ostart = (BYTE*)(output->ptr); BYTE* const ostart = (BYTE*)(output->dst) + output->pos;
BYTE* const oend = ostart + output->size; BYTE* const oend = (BYTE*)(output->dst) + output->size;
BYTE* op = ostart; BYTE* op = ostart;
if (zcs->stage != zcss_final) { if (zcs->stage != zcss_final) {
/* flush whatever remains */ /* flush whatever remains */
size_t srcSize = 0; size_t srcSize = 0;
size_t sizeWritten = output->size; size_t sizeWritten = output->size - output->pos;
size_t const notEnded = ZSTD_compressStream_generic(zcs, ostart, &sizeWritten, &srcSize, &srcSize, zsf_end); /* use a valid src address instead of NULL */ size_t const notEnded = ZSTD_compressStream_generic(zcs, ostart, &sizeWritten, &srcSize, &srcSize, zsf_end); /* use a valid src address instead of NULL */
size_t const remainingToFlush = zcs->outBuffContentSize - zcs->outBuffFlushedSize; size_t const remainingToFlush = zcs->outBuffContentSize - zcs->outBuffFlushedSize;
op += sizeWritten; op += sizeWritten;
if (remainingToFlush) { if (remainingToFlush) {
output->ptr = op; output->pos += sizeWritten;
output->size -= sizeWritten;
output->nbBytesWritten += sizeWritten;
return remainingToFlush + ZSTD_BLOCKHEADERSIZE /* final empty block */ + (zcs->checksum * 4); return remainingToFlush + ZSTD_BLOCKHEADERSIZE /* final empty block */ + (zcs->checksum * 4);
} }
/* create epilogue */ /* create epilogue */
@ -3021,9 +3016,7 @@ size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_wCursor* output)
size_t const flushed = ZSTD_limitCopy(op, oend-op, zcs->outBuff + zcs->outBuffFlushedSize, toFlush); size_t const flushed = ZSTD_limitCopy(op, oend-op, zcs->outBuff + zcs->outBuffFlushedSize, toFlush);
op += flushed; op += flushed;
zcs->outBuffFlushedSize += flushed; zcs->outBuffFlushedSize += flushed;
output->ptr = op; output->pos += op-ostart;
output->size = oend-op;
output->nbBytesWritten += op-ostart;
if (toFlush==flushed) zcs->stage = zcss_init; /* end reached */ if (toFlush==flushed) zcs->stage = zcss_init; /* end reached */
return toFlush - flushed; return toFlush - flushed;
} }

20
lib/decompress/zstd_decompress.c
View File

@ -1393,13 +1393,13 @@ MEM_STATIC size_t ZSTD_limitCopy(void* dst, size_t dstCapacity, const void* src,
} }
size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_wCursor* output, ZSTD_rCursor* input) size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inBuffer* input)
{ {
const char* const istart = (const char*)(input->ptr); const char* const istart = (const char*)(input->src) + input->pos;
const char* const iend = istart + input->size; const char* const iend = (const char*)(input->src) + input->size;
const char* ip = istart; const char* ip = istart;
char* const ostart = (char*)(output->ptr); char* const ostart = (char*)(output->dst) + output->pos;
char* const oend = ostart + output->size; char* const oend = (char*)(output->dst) + output->size;
char* op = ostart; char* op = ostart;
U32 someMoreWork = 1; U32 someMoreWork = 1;
@ -1417,8 +1417,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_wCursor* output, ZSTD_rCurs
if (toLoad > (size_t)(iend-ip)) { /* not enough input to load full header */ if (toLoad > (size_t)(iend-ip)) { /* not enough input to load full header */
memcpy(zds->headerBuffer + zds->lhSize, ip, iend-ip); memcpy(zds->headerBuffer + zds->lhSize, ip, iend-ip);
zds->lhSize += iend-ip; zds->lhSize += iend-ip;
input->ptr = iend; input->pos = input->size;
input->size = 0;
return (hSize - zds->lhSize) + ZSTD_blockHeaderSize; /* remaining header bytes + next block header */ return (hSize - zds->lhSize) + ZSTD_blockHeaderSize; /* remaining header bytes + next block header */
} }
memcpy(zds->headerBuffer + zds->lhSize, ip, toLoad); zds->lhSize = hSize; ip += toLoad; memcpy(zds->headerBuffer + zds->lhSize, ip, toLoad); zds->lhSize = hSize; ip += toLoad;
@ -1522,11 +1521,8 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_wCursor* output, ZSTD_rCurs
} } } }
/* result */ /* result */
input->ptr = ip; input->pos += (size_t)(ip-istart);
input->size = (size_t)(iend-ip); output->pos += (size_t)(op-ostart);
output->ptr = op;
output->size = (size_t) (oend-op);
output->nbBytesWritten += (size_t)(op-ostart);
{ size_t nextSrcSizeHint = ZSTD_nextSrcSizeToDecompress(zds->zd); { size_t nextSrcSizeHint = ZSTD_nextSrcSizeToDecompress(zds->zd);
if (!nextSrcSizeHint) return (zds->outEnd != zds->outStart); /* return 0 only if fully flushed too */ if (!nextSrcSizeHint) return (zds->outEnd != zds->outStart); /* return 0 only if fully flushed too */
nextSrcSizeHint += ZSTD_blockHeaderSize * (ZSTD_nextInputType(zds->zd) == ZSTDnit_block); nextSrcSizeHint += ZSTD_blockHeaderSize * (ZSTD_nextInputType(zds->zd) == ZSTDnit_block);

37
lib/zstd.h
View File

@ -324,16 +324,17 @@ ZSTDLIB_API size_t ZSTD_sizeofDCtx(const ZSTD_DCtx* dctx);
* Streaming * Streaming
********************************************************************/ ********************************************************************/
typedef struct ZSTD_readCursor_s { typedef struct ZSTD_inBuffer_s {
const void* ptr; /**< position of cursor - update to new position */ const void* src; /**< start of input buffer */
size_t size; /**< remaining buffer size to read - update preserves end of buffer */ size_t size; /**< size of input buffer */
} ZSTD_rCursor; size_t pos; /**< position where reading stopped. Will be updated. Necessarily 0 <= pos <= size */
} ZSTD_inBuffer;
typedef struct ZSTD_writeCursor_s { typedef struct ZSTD_outBuffer_s {
void* ptr; /**< position of cursor - update to new position */ void* dst; /**< start of output buffer */
size_t size; /**< remaining buffer size to write - update preserves end of buffer */ size_t size; /**< size of output buffer */
size_t nbBytesWritten; /**< already written bytes - update adds bytes newly written (accumulator) */ size_t pos; /**< position where writing stopped. Will be updated. Necessarily 0 <= pos <= size */
} ZSTD_wCursor; } ZSTD_outBuffer;
/*====== compression ======*/ /*====== compression ======*/
@ -350,14 +351,14 @@ typedef struct ZSTD_writeCursor_s {
* Use ZSTD_initCStream_usingDict() for a compression which requires a dictionary. * Use ZSTD_initCStream_usingDict() for a compression which requires a dictionary.
* *
* Use ZSTD_compressStream() repetitively to consume input stream. * Use ZSTD_compressStream() repetitively to consume input stream.
* The function will automatically advance cursors. * The function will automatically update both `pos`.
* Note that it may not consume the entire input, in which case `input->size > 0`, * Note that it may not consume the entire input, in which case `pos < size`,
* and it's up to the caller to present again remaining data. * and it's up to the caller to present again remaining data.
* @return : a hint to preferred nb of bytes to use as input for next function call (it's just a hint, to improve latency) * @return : a hint to preferred nb of bytes to use as input for next function call (it's just a hint, to improve latency)
* or an error code, which can be tested using ZSTD_isError(). * or an error code, which can be tested using ZSTD_isError().
* *
* At any moment, it's possible to flush whatever data remains within buffer, using ZSTD_flushStream(). * At any moment, it's possible to flush whatever data remains within buffer, using ZSTD_flushStream().
* Cursor will be updated. * `output->pos` will be updated.
* Note some content might still be left within internal buffer if `output->size` is too small. * Note some content might still be left within internal buffer if `output->size` is too small.
* @return : nb of bytes still present within internal buffer (0 if it's empty) * @return : nb of bytes still present within internal buffer (0 if it's empty)
* or an error code, which can be tested using ZSTD_isError(). * or an error code, which can be tested using ZSTD_isError().
@ -380,9 +381,9 @@ size_t ZSTD_CStreamInSize(void); /**< recommended size for input buffer */
size_t ZSTD_CStreamOutSize(void); /**< recommended size for output buffer */ size_t ZSTD_CStreamOutSize(void); /**< recommended size for output buffer */
size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel); size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel);
size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_wCursor* output, ZSTD_rCursor* input); size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input);
size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_wCursor* output); size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output);
size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_wCursor* output); size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output);
/* advanced */ /* advanced */
ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem); ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem);
@ -404,8 +405,8 @@ size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dic
* or ZSTD_initDStream_usingDict() if decompression requires a dictionary. * or ZSTD_initDStream_usingDict() if decompression requires a dictionary.
* *
* Use ZSTD_decompressStream() repetitively to consume your input. * Use ZSTD_decompressStream() repetitively to consume your input.
* The function will update cursors. * The function will update both `pos`.
* Note that it may not consume the entire input (input->size > 0), * Note that it may not consume the entire input (pos < size),
* in which case it's up to the caller to present remaining input again. * in which case it's up to the caller to present remaining input again.
* @return : 0 when a frame is completely decoded and fully flushed, * @return : 0 when a frame is completely decoded and fully flushed,
* 1 when there is still some data left within internal buffer to flush, * 1 when there is still some data left within internal buffer to flush,
@ -422,7 +423,7 @@ size_t ZSTD_DStreamInSize(void); /*!< recommended size for input buffer */
size_t ZSTD_DStreamOutSize(void); /*!< recommended size for output buffer */ size_t ZSTD_DStreamOutSize(void); /*!< recommended size for output buffer */
size_t ZSTD_initDStream(ZSTD_DStream* zds); size_t ZSTD_initDStream(ZSTD_DStream* zds);
size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_wCursor* output, ZSTD_rCursor* input); size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inBuffer* input);
/* advanced */ /* advanced */
ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem); ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem);

37
programs/fileio.c
View File

@ -336,30 +336,30 @@ static int FIO_compressFilename_internal(cRess_t ress,
DISPLAYUPDATE(2, "\rRead : %u MB ", (U32)(readsize>>20)); DISPLAYUPDATE(2, "\rRead : %u MB ", (U32)(readsize>>20));
/* Compress using buffered streaming */ /* Compress using buffered streaming */
{ ZSTD_rCursor rCursor = { ress.srcBuffer, inSize }; { ZSTD_inBuffer inBuff = { ress.srcBuffer, inSize, 0 };
ZSTD_wCursor wCursor = { ress.dstBuffer, ress.dstBufferSize, 0 }; ZSTD_outBuffer outBuff= { ress.dstBuffer, ress.dstBufferSize, 0 };
{ size_t const result = ZSTD_compressStream(ress.cctx, &wCursor, &rCursor); { size_t const result = ZSTD_compressStream(ress.cctx, &outBuff, &inBuff);
if (ZSTD_isError(result)) EXM_THROW(23, "Compression error : %s ", ZSTD_getErrorName(result)); } if (ZSTD_isError(result)) EXM_THROW(23, "Compression error : %s ", ZSTD_getErrorName(result)); }
if (rCursor.size != 0) if (inBuff.pos != inBuff.size)
/* inBuff should be entirely consumed since buffer sizes are recommended ones */ /* inBuff should be entirely consumed since buffer sizes are recommended ones */
EXM_THROW(24, "Compression error : input block not fully consumed"); EXM_THROW(24, "Compression error : input block not fully consumed");
/* Write cBlock */ /* Write cBlock */
{ size_t const sizeCheck = fwrite(ress.dstBuffer, 1, wCursor.nbBytesWritten, dstFile); { size_t const sizeCheck = fwrite(ress.dstBuffer, 1, outBuff.pos, dstFile);
if (sizeCheck!=wCursor.nbBytesWritten) EXM_THROW(25, "Write error : cannot write compressed block into %s", dstFileName); } if (sizeCheck!=outBuff.pos) EXM_THROW(25, "Write error : cannot write compressed block into %s", dstFileName); }
compressedfilesize += wCursor.nbBytesWritten; compressedfilesize += outBuff.pos;
} }
DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%% ", (U32)(readsize>>20), (double)compressedfilesize/readsize*100); DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%% ", (U32)(readsize>>20), (double)compressedfilesize/readsize*100);
} }
/* End of Frame */ /* End of Frame */
{ ZSTD_wCursor wCursor = { ress.dstBuffer, ress.dstBufferSize, 0 }; { ZSTD_outBuffer outBuff = { ress.dstBuffer, ress.dstBufferSize, 0 };
size_t const result = ZSTD_endStream(ress.cctx, &wCursor); size_t const result = ZSTD_endStream(ress.cctx, &outBuff);
if (result!=0) EXM_THROW(26, "Compression error : cannot create frame end"); if (result!=0) EXM_THROW(26, "Compression error : cannot create frame end");
{ size_t const sizeCheck = fwrite(ress.dstBuffer, 1, wCursor.nbBytesWritten, dstFile); { size_t const sizeCheck = fwrite(ress.dstBuffer, 1, outBuff.pos, dstFile);
if (sizeCheck!=wCursor.nbBytesWritten) EXM_THROW(27, "Write error : cannot write frame end into %s", dstFileName); } if (sizeCheck!=outBuff.pos) EXM_THROW(27, "Write error : cannot write frame end into %s", dstFileName); }
compressedfilesize += wCursor.nbBytesWritten; compressedfilesize += outBuff.pos;
} }
/* Status */ /* Status */
@ -624,19 +624,18 @@ unsigned long long FIO_decompressFrame(dRess_t ress,
/* Main decompression Loop */ /* Main decompression Loop */
while (1) { while (1) {
ZSTD_rCursor rCursor = { ress.srcBuffer, readSize }; ZSTD_inBuffer inBuff = { ress.srcBuffer, readSize, 0 };
ZSTD_wCursor wCursor = { ress.dstBuffer, ress.dstBufferSize, 0 }; ZSTD_outBuffer outBuff= { ress.dstBuffer, ress.dstBufferSize, 0 };
size_t const toRead = ZSTD_decompressStream(ress.dctx, &wCursor, &rCursor ); size_t const toRead = ZSTD_decompressStream(ress.dctx, &outBuff, &inBuff );
if (ZSTD_isError(toRead)) EXM_THROW(36, "Decoding error : %s", ZSTD_getErrorName(toRead)); if (ZSTD_isError(toRead)) EXM_THROW(36, "Decoding error : %s", ZSTD_getErrorName(toRead));
readSize = rCursor.size;
/* Write block */ /* Write block */
storedSkips = FIO_fwriteSparse(foutput, ress.dstBuffer, wCursor.nbBytesWritten, storedSkips); storedSkips = FIO_fwriteSparse(foutput, ress.dstBuffer, outBuff.pos, storedSkips);
frameSize += wCursor.nbBytesWritten; frameSize += outBuff.pos;
DISPLAYUPDATE(2, "\rDecoded : %u MB... ", (U32)(frameSize>>20) ); DISPLAYUPDATE(2, "\rDecoded : %u MB... ", (U32)(frameSize>>20) );
if (toRead == 0) break; /* end of frame */ if (toRead == 0) break; /* end of frame */
if (readSize) EXM_THROW(37, "Decoding error : should consume entire input"); if (inBuff.size != inBuff.pos) EXM_THROW(37, "Decoding error : should consume entire input");
/* Fill input buffer */ /* Fill input buffer */
if (toRead > ress.srcBufferSize) EXM_THROW(38, "too large block"); if (toRead > ress.srcBufferSize) EXM_THROW(38, "too large block");

129
programs/zstreamtest.c
View File

@ -155,8 +155,8 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
U32 testNb=0; U32 testNb=0;
ZSTD_CStream* zc = ZSTD_createCStream_advanced(customMem); ZSTD_CStream* zc = ZSTD_createCStream_advanced(customMem);
ZSTD_DStream* zd = ZSTD_createDStream_advanced(customMem); ZSTD_DStream* zd = ZSTD_createDStream_advanced(customMem);
ZSTD_rCursor rCursor; ZSTD_inBuffer inBuff;
ZSTD_wCursor wCursor; ZSTD_outBuffer outBuff;
/* Create compressible test buffer */ /* Create compressible test buffer */
if (!CNBuffer || !compressedBuffer || !decodedBuffer || !zc || !zd) { if (!CNBuffer || !compressedBuffer || !decodedBuffer || !zc || !zd) {
@ -173,39 +173,41 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
/* Basic compression test */ /* Basic compression test */
DISPLAYLEVEL(4, "test%3i : compress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH); DISPLAYLEVEL(4, "test%3i : compress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
ZSTD_initCStream_usingDict(zc, CNBuffer, 128 KB, 1); ZSTD_initCStream_usingDict(zc, CNBuffer, 128 KB, 1);
wCursor.ptr = (char*)(compressedBuffer)+cSize; outBuff.dst = (char*)(compressedBuffer)+cSize;
wCursor.size = compressedBufferSize; outBuff.size = compressedBufferSize;
wCursor.nbBytesWritten = 0; outBuff.pos = 0;
rCursor.ptr = CNBuffer; inBuff.src = CNBuffer;
rCursor.size = CNBufferSize; inBuff.size = CNBufferSize;
{ size_t const r = ZSTD_compressStream(zc, &wCursor, &rCursor); inBuff.pos = 0;
{ size_t const r = ZSTD_compressStream(zc, &outBuff, &inBuff);
if (ZSTD_isError(r)) goto _output_error; } if (ZSTD_isError(r)) goto _output_error; }
if (rCursor.size != 0) goto _output_error; /* entire input should be consumed */ if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */
{ size_t const r = ZSTD_endStream(zc, &wCursor); { size_t const r = ZSTD_endStream(zc, &outBuff);
if (r != 0) goto _output_error; } /*< error, or some data not flushed */ if (r != 0) goto _output_error; } /*< error, or some data not flushed */
cSize += wCursor.nbBytesWritten; cSize += outBuff.pos;
DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/COMPRESSIBLE_NOISE_LENGTH*100); DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/COMPRESSIBLE_NOISE_LENGTH*100);
/* skippable frame test */ /* skippable frame test */
DISPLAYLEVEL(4, "test%3i : decompress skippable frame : ", testNb++); DISPLAYLEVEL(4, "test%3i : decompress skippable frame : ", testNb++);
ZSTD_initDStream_usingDict(zd, CNBuffer, 128 KB); ZSTD_initDStream_usingDict(zd, CNBuffer, 128 KB);
rCursor.ptr = compressedBuffer; inBuff.src = compressedBuffer;
rCursor.size = cSize; inBuff.size = cSize;
wCursor.ptr = decodedBuffer; inBuff.pos = 0;
wCursor.size = CNBufferSize; outBuff.dst = decodedBuffer;
wCursor.nbBytesWritten = 0; outBuff.size = CNBufferSize;
{ size_t const r = ZSTD_decompressStream(zd, &wCursor, &rCursor); outBuff.pos = 0;
{ size_t const r = ZSTD_decompressStream(zd, &outBuff, &inBuff);
if (r != 0) goto _output_error; } if (r != 0) goto _output_error; }
if (wCursor.nbBytesWritten != 0) goto _output_error; /* skippable frame len is 0 */ if (outBuff.pos != 0) goto _output_error; /* skippable frame len is 0 */
DISPLAYLEVEL(4, "OK \n"); DISPLAYLEVEL(4, "OK \n");
/* Basic decompression test */ /* Basic decompression test */
DISPLAYLEVEL(4, "test%3i : decompress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH); DISPLAYLEVEL(4, "test%3i : decompress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
ZSTD_initDStream_usingDict(zd, CNBuffer, 128 KB); ZSTD_initDStream_usingDict(zd, CNBuffer, 128 KB);
{ size_t const r = ZSTD_decompressStream(zd, &wCursor, &rCursor); { size_t const r = ZSTD_decompressStream(zd, &outBuff, &inBuff);
if (r != 0) goto _output_error; } /* should reach end of frame == 0; otherwise, some data left, or an error */ if (r != 0) goto _output_error; } /* should reach end of frame == 0; otherwise, some data left, or an error */
if (wCursor.nbBytesWritten != CNBufferSize) goto _output_error; /* should regenerate the same amount */ if (outBuff.pos != CNBufferSize) goto _output_error; /* should regenerate the same amount */
if (rCursor.size != 0) goto _output_error; /* should have read the entire frame */ if (inBuff.pos != inBuff.size) goto _output_error; /* should have read the entire frame */
DISPLAYLEVEL(4, "OK \n"); DISPLAYLEVEL(4, "OK \n");
/* check regenerated data is byte exact */ /* check regenerated data is byte exact */
@ -220,26 +222,27 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
DISPLAYLEVEL(4, "test%3i : decompress byte-by-byte : ", testNb++); DISPLAYLEVEL(4, "test%3i : decompress byte-by-byte : ", testNb++);
{ size_t r = 1; { size_t r = 1;
ZSTD_initDStream_usingDict(zd, CNBuffer, 128 KB); ZSTD_initDStream_usingDict(zd, CNBuffer, 128 KB);
rCursor.ptr = compressedBuffer; inBuff.src = compressedBuffer;
wCursor.ptr = decodedBuffer; outBuff.dst = decodedBuffer;
wCursor.nbBytesWritten = 0; inBuff.pos = 0;
outBuff.pos = 0;
while (r) { /* skippable frame */ while (r) { /* skippable frame */
rCursor.size = 1; inBuff.size = inBuff.pos + 1;
wCursor.size = 1; outBuff.size = outBuff.pos + 1;
r = ZSTD_decompressStream(zd, &wCursor, &rCursor); r = ZSTD_decompressStream(zd, &outBuff, &inBuff);
if (ZSTD_isError(r)) goto _output_error; if (ZSTD_isError(r)) goto _output_error;
} }
ZSTD_initDStream_usingDict(zd, CNBuffer, 128 KB); ZSTD_initDStream_usingDict(zd, CNBuffer, 128 KB);
r=1; r=1;
while (r) { /* normal frame */ while (r) { /* normal frame */
rCursor.size = 1; inBuff.size = inBuff.pos + 1;
wCursor.size = 1; outBuff.size = outBuff.pos + 1;
r = ZSTD_decompressStream(zd, &wCursor, &rCursor); r = ZSTD_decompressStream(zd, &outBuff, &inBuff);
if (ZSTD_isError(r)) goto _output_error; if (ZSTD_isError(r)) goto _output_error;
} }
} }
if (wCursor.nbBytesWritten != CNBufferSize) goto _output_error; /* should regenerate the same amount */ if (outBuff.pos != CNBufferSize) goto _output_error; /* should regenerate the same amount */
if ((size_t)(rCursor.ptr - compressedBuffer) != cSize) goto _output_error; /* should have read the entire frame */ if (inBuff.pos != cSize) goto _output_error; /* should have read the entire frame */
DISPLAYLEVEL(4, "OK \n"); DISPLAYLEVEL(4, "OK \n");
/* check regenerated data is byte exact */ /* check regenerated data is byte exact */
@ -345,7 +348,7 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres
const BYTE* srcBuffer; const BYTE* srcBuffer;
const BYTE* dict; const BYTE* dict;
size_t maxTestSize, dictSize; size_t maxTestSize, dictSize;
size_t cSize, totalTestSize, totalCSize, totalGenSize; size_t cSize, totalTestSize, totalGenSize;
U32 n, nbChunks; U32 n, nbChunks;
XXH64_state_t xxhState; XXH64_state_t xxhState;
U64 crcOrig; U64 crcOrig;
@ -395,31 +398,30 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres
/* multi-segments compression test */ /* multi-segments compression test */
XXH64_reset(&xxhState, 0); XXH64_reset(&xxhState, 0);
nbChunks = (FUZ_rand(&lseed) & 127) + 2; nbChunks = (FUZ_rand(&lseed) & 127) + 2;
{ ZSTD_wCursor wCursor = { cBuffer, cBufferSize, 0 } ; { ZSTD_outBuffer outBuff = { cBuffer, cBufferSize, 0 } ;
for (n=0, cSize=0, totalTestSize=0 ; (n<nbChunks) && (totalTestSize < maxTestSize) ; n++) { for (n=0, cSize=0, totalTestSize=0 ; (n<nbChunks) && (totalTestSize < maxTestSize) ; n++) {
/* compress random chunk into random size dst buffer */ /* compress random chunk into random size dst buffer */
{ size_t readChunkSize = FUZ_randomLength(&lseed, maxSampleLog); { size_t const readChunkSize = FUZ_randomLength(&lseed, maxSampleLog);
size_t const srcStart = FUZ_rand(&lseed) % (srcBufferSize - readChunkSize); size_t const srcStart = FUZ_rand(&lseed) % (srcBufferSize - readChunkSize);
size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog); size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog);
size_t const dstBuffSize = MIN(cBufferSize - cSize, randomDstSize); size_t const dstBuffSize = MIN(cBufferSize - cSize, randomDstSize);
ZSTD_rCursor rCursor = { srcBuffer+srcStart, readChunkSize }; ZSTD_inBuffer inBuff = { srcBuffer+srcStart, readChunkSize, 0 };
wCursor.size = dstBuffSize; outBuff.size = outBuff.pos + dstBuffSize;
{ size_t const compressionError = ZSTD_compressStream(zc, &wCursor, &rCursor); { size_t const compressionError = ZSTD_compressStream(zc, &outBuff, &inBuff);
CHECK (ZSTD_isError(compressionError), "compression error : %s", ZSTD_getErrorName(compressionError)); } CHECK (ZSTD_isError(compressionError), "compression error : %s", ZSTD_getErrorName(compressionError)); }
readChunkSize -= rCursor.size;
XXH64_update(&xxhState, srcBuffer+srcStart, readChunkSize); XXH64_update(&xxhState, srcBuffer+srcStart, inBuff.pos);
memcpy(copyBuffer+totalTestSize, srcBuffer+srcStart, readChunkSize); memcpy(copyBuffer+totalTestSize, srcBuffer+srcStart, inBuff.pos);
totalTestSize += readChunkSize; totalTestSize += inBuff.pos;
} }
/* random flush operation, to mess around */ /* random flush operation, to mess around */
if ((FUZ_rand(&lseed) & 15) == 0) { if ((FUZ_rand(&lseed) & 15) == 0) {
size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog); size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog);
size_t const dstBuffSize = MIN(cBufferSize - cSize, randomDstSize); size_t const adjustedDstSize = MIN(cBufferSize - cSize, randomDstSize);
wCursor.size = dstBuffSize; outBuff.size = outBuff.pos + adjustedDstSize;
{ size_t const flushError = ZSTD_flushStream(zc, &wCursor); { size_t const flushError = ZSTD_flushStream(zc, &outBuff);
CHECK (ZSTD_isError(flushError), "flush error : %s", ZSTD_getErrorName(flushError)); CHECK (ZSTD_isError(flushError), "flush error : %s", ZSTD_getErrorName(flushError));
} } } } } }
@ -429,33 +431,32 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres
size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog); size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog);
size_t const adjustedDstSize = MIN(cBufferSize - cSize, randomDstSize); size_t const adjustedDstSize = MIN(cBufferSize - cSize, randomDstSize);
U32 const enoughDstSize = (adjustedDstSize >= remainingToFlush); U32 const enoughDstSize = (adjustedDstSize >= remainingToFlush);
wCursor.size = adjustedDstSize; outBuff.size = outBuff.pos + adjustedDstSize;
remainingToFlush = ZSTD_endStream(zc, &wCursor); remainingToFlush = ZSTD_endStream(zc, &outBuff);
CHECK (ZSTD_isError(remainingToFlush), "flush error : %s", ZSTD_getErrorName(remainingToFlush)); CHECK (ZSTD_isError(remainingToFlush), "flush error : %s", ZSTD_getErrorName(remainingToFlush));
CHECK (enoughDstSize && remainingToFlush, "ZSTD_endStream() not fully flushed (%u remaining), but enough space available", (U32)remainingToFlush); CHECK (enoughDstSize && remainingToFlush, "ZSTD_endStream() not fully flushed (%u remaining), but enough space available", (U32)remainingToFlush);
} } } }
crcOrig = XXH64_digest(&xxhState); crcOrig = XXH64_digest(&xxhState);
cSize = wCursor.nbBytesWritten; cSize = outBuff.pos;
} }
/* multi - fragments decompression test */ /* multi - fragments decompression test */
ZSTD_initDStream_usingDict(zd, dict, dictSize); ZSTD_initDStream_usingDict(zd, dict, dictSize);
{ size_t decompressionResult = 1; { size_t decompressionResult = 1;
ZSTD_rCursor rCursor = { cBuffer, cSize }; ZSTD_inBuffer inBuff = { cBuffer, cSize, 0 };
ZSTD_wCursor wCursor = { dstBuffer, dstBufferSize, 0 }; ZSTD_outBuffer outBuff= { dstBuffer, dstBufferSize, 0 };
for (totalCSize = 0, totalGenSize = 0 ; decompressionResult ; ) { for (totalGenSize = 0 ; decompressionResult ; ) {
size_t const readCSrcSize = FUZ_randomLength(&lseed, maxSampleLog); size_t const readCSrcSize = FUZ_randomLength(&lseed, maxSampleLog);
size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog); size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog);
size_t const dstBuffSize = MIN(dstBufferSize - totalGenSize, randomDstSize); size_t const dstBuffSize = MIN(dstBufferSize - totalGenSize, randomDstSize);
rCursor.size = readCSrcSize; inBuff.size = inBuff.pos + readCSrcSize;
wCursor.size = dstBuffSize; outBuff.size = inBuff.pos + dstBuffSize;
decompressionResult = ZSTD_decompressStream(zd, &wCursor, &rCursor); decompressionResult = ZSTD_decompressStream(zd, &outBuff, &inBuff);
CHECK (ZSTD_isError(decompressionResult), "decompression error : %s", ZSTD_getErrorName(decompressionResult)); CHECK (ZSTD_isError(decompressionResult), "decompression error : %s", ZSTD_getErrorName(decompressionResult));
totalCSize += readCSrcSize - rCursor.size;
} }
CHECK (decompressionResult != 0, "frame not fully decoded"); CHECK (decompressionResult != 0, "frame not fully decoded");
CHECK (wCursor.nbBytesWritten != totalTestSize, "decompressed data : wrong size") CHECK (outBuff.pos != totalTestSize, "decompressed data : wrong size")
CHECK (totalCSize != cSize, "compressed data should be fully read") CHECK (inBuff.pos != cSize, "compressed data should be fully read")
{ U64 const crcDest = XXH64(dstBuffer, totalTestSize, 0); { U64 const crcDest = XXH64(dstBuffer, totalTestSize, 0);
if (crcDest!=crcOrig) findDiff(copyBuffer, dstBuffer, totalTestSize); if (crcDest!=crcOrig) findDiff(copyBuffer, dstBuffer, totalTestSize);
CHECK (crcDest!=crcOrig, "decompressed data corrupted"); CHECK (crcDest!=crcOrig, "decompressed data corrupted");
@ -475,15 +476,15 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres
/* try decompression on noisy data */ /* try decompression on noisy data */
ZSTD_initDStream(zd); ZSTD_initDStream(zd);
{ ZSTD_rCursor rCursor = { cBuffer, cSize }; { ZSTD_inBuffer inBuff = { cBuffer, cSize, 0 };
ZSTD_wCursor wCursor = { dstBuffer, dstBufferSize, 0 }; ZSTD_outBuffer outBuff= { dstBuffer, dstBufferSize, 0 };
while (wCursor.nbBytesWritten < dstBufferSize) { while (outBuff.pos < dstBufferSize) {
size_t const randomCSrcSize = FUZ_randomLength(&lseed, maxSampleLog); size_t const randomCSrcSize = FUZ_randomLength(&lseed, maxSampleLog);
size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog); size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog);
size_t const adjustedDstSize = MIN(dstBufferSize - wCursor.nbBytesWritten, randomDstSize); size_t const adjustedDstSize = MIN(dstBufferSize - outBuff.pos, randomDstSize);
wCursor.size = adjustedDstSize; outBuff.size = outBuff.pos + adjustedDstSize;
rCursor.size = randomCSrcSize; inBuff.size = inBuff.pos + randomCSrcSize;
{ size_t const decompressError = ZSTD_decompressStream(zd, &wCursor, &rCursor); { size_t const decompressError = ZSTD_decompressStream(zd, &outBuff, &inBuff);
if (ZSTD_isError(decompressError)) break; /* error correctly detected */ if (ZSTD_isError(decompressError)) break; /* error correctly detected */
} } } } } } } }
DISPLAY("\r%u fuzzer tests completed \n", testNb); DISPLAY("\r%u fuzzer tests completed \n", testNb);