facebook/zstd
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix fileio bug with new zbuff

Browse Source 
simplified zbuff decoding
This commit is contained in:
Yann Collet 2016-03-15 12:56:03 +01:00
parent 1c2c2bcaff
commit 09b21ee2f6
3 changed files with 62 additions and 92 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

115
lib/zbuff.c
View File

@ -40,7 +40,7 @@
***************************************/ ***************************************/
#include <stdlib.h> #include <stdlib.h>
#include "error_private.h" #include "error_private.h"
#include "zstd_static.h" #include "zstd_internal.h"
#include "zbuff_static.h" #include "zbuff_static.h"
@ -50,7 +50,7 @@
static size_t ZBUFF_blockHeaderSize = 3; static size_t ZBUFF_blockHeaderSize = 3;
static size_t ZBUFF_endFrameSize = 3; static size_t ZBUFF_endFrameSize = 3;
/** ************************************************ /*_**************************************************
* Streaming compression * Streaming compression
* *
* A ZBUFF_CCtx object is required to track streaming operation. * A ZBUFF_CCtx object is required to track streaming operation.
@ -88,13 +88,13 @@ typedef enum { ZBUFFcs_init, ZBUFFcs_load, ZBUFFcs_flush } ZBUFF_cStage;
/* *** Ressources *** */ /* *** Ressources *** */
struct ZBUFF_CCtx_s { struct ZBUFF_CCtx_s {
ZSTD_CCtx* zc; ZSTD_CCtx* zc;
char* inBuff; char* inBuff;
size_t inBuffSize; size_t inBuffSize;
size_t inToCompress; size_t inToCompress;
size_t inBuffPos; size_t inBuffPos;
size_t inBuffTarget; size_t inBuffTarget;
size_t blockSize; size_t blockSize;
char* outBuff; char* outBuff;
size_t outBuffSize; size_t outBuffSize;
size_t outBuffContentSize; size_t outBuffContentSize;
size_t outBuffFlushedSize; size_t outBuffFlushedSize;
@ -123,8 +123,6 @@ size_t ZBUFF_freeCCtx(ZBUFF_CCtx* zbc)
/* *** Initialization *** */ /* *** Initialization *** */
#define MIN(a,b) ( ((a)<(b)) ? (a) : (b) )
#define BLOCKSIZE (128 * 1024) /* a bit too "magic", should come from reference */
size_t ZBUFF_compressInit_advanced(ZBUFF_CCtx* zbc, const void* dict, size_t dictSize, ZSTD_parameters params) size_t ZBUFF_compressInit_advanced(ZBUFF_CCtx* zbc, const void* dict, size_t dictSize, ZSTD_parameters params)
{ {
size_t neededInBuffSize; size_t neededInBuffSize;
@ -172,9 +170,9 @@ ZSTDLIB_API size_t ZBUFF_compressInitDictionary(ZBUFF_CCtx* zbc, const void* dic
/* *** Compression *** */ /* *** Compression *** */
static size_t ZBUFF_limitCopy(void* dst, size_t maxDstSize, const void* src, size_t srcSize) static size_t ZBUFF_limitCopy(void* dst, size_t dstCapacity, const void* src, size_t srcSize)
{ {
size_t length = MIN(maxDstSize, srcSize); size_t length = MIN(dstCapacity, srcSize);
memcpy(dst, src, length); memcpy(dst, src, length);
return length; return length;
} }
@ -322,12 +320,10 @@ size_t ZBUFF_compressEnd(ZBUFF_CCtx* zbc, void* dst, size_t* maxDstSizePtr)
* just follow indications from ZBUFF_decompressContinue() to minimize latency. It should always be <= 128 KB + 3 . * just follow indications from ZBUFF_decompressContinue() to minimize latency. It should always be <= 128 KB + 3 .
* *******************************************************************************/ * *******************************************************************************/
typedef enum { ZBUFFds_init, ZBUFFds_readHeader, ZBUFFds_loadHeader, ZBUFFds_decodeHeader, typedef enum { ZBUFFds_init, ZBUFFds_readHeader,
ZBUFFds_read, ZBUFFds_load, ZBUFFds_flush } ZBUFF_dStage; ZBUFFds_read, ZBUFFds_load, ZBUFFds_flush } ZBUFF_dStage;
/* *** Resource management *** */ /* *** Resource management *** */
#define ZSTD_frameHeaderSize_max 5 /* too magical, should come from reference */
struct ZBUFF_DCtx_s { struct ZBUFF_DCtx_s {
ZSTD_DCtx* zc; ZSTD_DCtx* zc;
ZSTD_frameParams fParams; ZSTD_frameParams fParams;
@ -338,9 +334,7 @@ struct ZBUFF_DCtx_s {
size_t outBuffSize; size_t outBuffSize;
size_t outStart; size_t outStart;
size_t outEnd; size_t outEnd;
size_t hPos;
ZBUFF_dStage stage; ZBUFF_dStage stage;
unsigned char headerBuffer[ZSTD_frameHeaderSize_max];
}; /* typedef'd to ZBUFF_DCtx within "zstd_buffered.h" */ }; /* typedef'd to ZBUFF_DCtx within "zstd_buffered.h" */
@ -370,7 +364,7 @@ size_t ZBUFF_freeDCtx(ZBUFF_DCtx* zbc)
size_t ZBUFF_decompressInitDictionary(ZBUFF_DCtx* zbc, const void* dict, size_t dictSize) size_t ZBUFF_decompressInitDictionary(ZBUFF_DCtx* zbc, const void* dict, size_t dictSize)
{ {
zbc->stage = ZBUFFds_readHeader; zbc->stage = ZBUFFds_readHeader;
zbc->hPos = zbc->inPos = zbc->outStart = zbc->outEnd = 0; zbc->inPos = zbc->outStart = zbc->outEnd = 0;
return ZSTD_decompressBegin_usingDict(zbc->zc, dict, dictSize); return ZSTD_decompressBegin_usingDict(zbc->zc, dict, dictSize);
} }
@ -402,69 +396,36 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbc,
case ZBUFFds_readHeader : case ZBUFFds_readHeader :
/* read header from src */ /* read header from src */
{ { size_t const headerSize = ZSTD_getFrameParams(&(zbc->fParams), src, *srcSizePtr);
size_t headerSize = ZSTD_getFrameParams(&(zbc->fParams), src, *srcSizePtr);
if (ZSTD_isError(headerSize)) return headerSize; if (ZSTD_isError(headerSize)) return headerSize;
if (headerSize) { if (headerSize) {
/* not enough input to decode header : tell how many bytes would be necessary */ /* not enough input to decode header : needs headerSize > *srcSizePtr */
memcpy(zbc->headerBuffer+zbc->hPos, src, *srcSizePtr);
zbc->hPos += *srcSizePtr;
*dstCapacityPtr = 0; *dstCapacityPtr = 0;
zbc->stage = ZBUFFds_loadHeader; *srcSizePtr = 0;
return headerSize - zbc->hPos; return headerSize;
} } }
zbc->stage = ZBUFFds_decodeHeader;
break;
}
case ZBUFFds_loadHeader: /* Frame header provides buffer sizes */
/* complete header from src */ { size_t const neededInSize = BLOCKSIZE; /* a block is never > BLOCKSIZE */
if (zbc->inBuffSize < neededInSize) {
free(zbc->inBuff);
zbc->inBuffSize = neededInSize;
zbc->inBuff = (char*)malloc(neededInSize);
if (zbc->inBuff == NULL) return ERROR(memory_allocation);
} }
{ {
size_t headerSize = ZBUFF_limitCopy( size_t const neededOutSize = (size_t)1 << zbc->fParams.windowLog;
zbc->headerBuffer + zbc->hPos, ZSTD_frameHeaderSize_max - zbc->hPos, if (zbc->outBuffSize < neededOutSize) {
src, *srcSizePtr); free(zbc->outBuff);
zbc->hPos += headerSize; zbc->outBuffSize = neededOutSize;
ip += headerSize; zbc->outBuff = (char*)malloc(neededOutSize);
headerSize = ZSTD_getFrameParams(&(zbc->fParams), zbc->headerBuffer, zbc->hPos); if (zbc->outBuff == NULL) return ERROR(memory_allocation);
if (ZSTD_isError(headerSize)) return headerSize; } }
if (headerSize) { zbc->stage = ZBUFFds_read;
/* not enough input to decode header : tell how many bytes would be necessary */
*dstCapacityPtr = 0;
return headerSize - zbc->hPos;
}
// zbc->stage = ZBUFFds_decodeHeader; break; /* useless : stage follows */
}
case ZBUFFds_decodeHeader:
/* apply header to create / resize buffers */
{
size_t neededOutSize = (size_t)1 << zbc->fParams.windowLog;
size_t neededInSize = BLOCKSIZE; /* a block is never > BLOCKSIZE */
if (zbc->inBuffSize < neededInSize) {
free(zbc->inBuff);
zbc->inBuffSize = neededInSize;
zbc->inBuff = (char*)malloc(neededInSize);
if (zbc->inBuff == NULL) return ERROR(memory_allocation);
}
if (zbc->outBuffSize < neededOutSize) {
free(zbc->outBuff);
zbc->outBuffSize = neededOutSize;
zbc->outBuff = (char*)malloc(neededOutSize);
if (zbc->outBuff == NULL) return ERROR(memory_allocation);
} }
if (zbc->hPos) {
/* some data already loaded into headerBuffer : transfer into inBuff */
memcpy(zbc->inBuff, zbc->headerBuffer, zbc->hPos);
zbc->inPos = zbc->hPos;
zbc->hPos = 0;
zbc->stage = ZBUFFds_load;
break;
}
zbc->stage = ZBUFFds_read;
case ZBUFFds_read: case ZBUFFds_read:
{ {
size_t neededInSize = ZSTD_nextSrcSizeToDecompress(zbc->zc); size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zbc->zc);
if (neededInSize==0) { /* end of frame */ if (neededInSize==0) { /* end of frame */
zbc->stage = ZBUFFds_init; zbc->stage = ZBUFFds_init;
notDone = 0; notDone = 0;
@ -472,7 +433,7 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbc,
} }
if ((size_t)(iend-ip) >= neededInSize) { if ((size_t)(iend-ip) >= neededInSize) {
/* directly decode from src */ /* directly decode from src */
size_t decodedSize = ZSTD_decompressContinue(zbc->zc, size_t const decodedSize = ZSTD_decompressContinue(zbc->zc,
zbc->outBuff + zbc->outStart, zbc->outBuffSize - zbc->outStart, zbc->outBuff + zbc->outStart, zbc->outBuffSize - zbc->outStart,
ip, neededInSize); ip, neededInSize);
if (ZSTD_isError(decodedSize)) return decodedSize; if (ZSTD_isError(decodedSize)) return decodedSize;
@ -488,8 +449,8 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbc,
case ZBUFFds_load: case ZBUFFds_load:
{ {
size_t neededInSize = ZSTD_nextSrcSizeToDecompress(zbc->zc); size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zbc->zc);
size_t toLoad = neededInSize - zbc->inPos; /* should always be <= remaining space within inBuff */ size_t const toLoad = neededInSize - zbc->inPos; /* should always be <= remaining space within inBuff */
size_t loadedSize; size_t loadedSize;
if (toLoad > zbc->inBuffSize - zbc->inPos) return ERROR(corruption_detected); /* should never happen */ if (toLoad > zbc->inBuffSize - zbc->inPos) return ERROR(corruption_detected); /* should never happen */
loadedSize = ZBUFF_limitCopy(zbc->inBuff + zbc->inPos, toLoad, ip, iend-ip); loadedSize = ZBUFF_limitCopy(zbc->inBuff + zbc->inPos, toLoad, ip, iend-ip);
@ -497,7 +458,7 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbc,
zbc->inPos += loadedSize; zbc->inPos += loadedSize;
if (loadedSize < toLoad) { notDone = 0; break; } /* not enough input, wait for more */ if (loadedSize < toLoad) { notDone = 0; break; } /* not enough input, wait for more */
{ {
size_t decodedSize = ZSTD_decompressContinue(zbc->zc, size_t const decodedSize = ZSTD_decompressContinue(zbc->zc,
zbc->outBuff + zbc->outStart, zbc->outBuffSize - zbc->outStart, zbc->outBuff + zbc->outStart, zbc->outBuffSize - zbc->outStart,
zbc->inBuff, neededInSize); zbc->inBuff, neededInSize);
if (ZSTD_isError(decodedSize)) return decodedSize; if (ZSTD_isError(decodedSize)) return decodedSize;
@ -509,8 +470,8 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbc,
} } } }
case ZBUFFds_flush: case ZBUFFds_flush:
{ {
size_t toFlushSize = zbc->outEnd - zbc->outStart; size_t const toFlushSize = zbc->outEnd - zbc->outStart;
size_t flushedSize = ZBUFF_limitCopy(op, oend-op, zbc->outBuff + zbc->outStart, toFlushSize); size_t const flushedSize = ZBUFF_limitCopy(op, oend-op, zbc->outBuff + zbc->outStart, toFlushSize);
op += flushedSize; op += flushedSize;
zbc->outStart += flushedSize; zbc->outStart += flushedSize;
if (flushedSize == toFlushSize) { if (flushedSize == toFlushSize) {
@ -526,12 +487,12 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbc,
default: return ERROR(GENERIC); /* impossible */ default: return ERROR(GENERIC); /* impossible */
} } } }
/* result */
*srcSizePtr = ip-istart; *srcSizePtr = ip-istart;
*dstCapacityPtr = op-ostart; *dstCapacityPtr = op-ostart;
{ {
size_t nextSrcSizeHint = ZSTD_nextSrcSizeToDecompress(zbc->zc); size_t nextSrcSizeHint = ZSTD_nextSrcSizeToDecompress(zbc->zc);
if (nextSrcSizeHint > ZBUFF_blockHeaderSize) nextSrcSizeHint+= ZBUFF_blockHeaderSize; /* get next block header too */ if (nextSrcSizeHint > ZBUFF_blockHeaderSize) nextSrcSizeHint+= ZBUFF_blockHeaderSize; /* get following block header too */
nextSrcSizeHint -= zbc->inPos; /* already loaded*/ nextSrcSizeHint -= zbc->inPos; /* already loaded*/
return nextSrcSizeHint; return nextSrcSizeHint;
} }

2
lib/zstd_decompress.c
View File

@ -305,7 +305,7 @@ static size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize)
* @return : 0, `fparamsPtr` is correctly filled, * @return : 0, `fparamsPtr` is correctly filled,
* >0, `srcSize` is too small, result is expected `srcSize`, * >0, `srcSize` is too small, result is expected `srcSize`,
* or an error code, which can be tested using ZSTD_isError() */ * or an error code, which can be tested using ZSTD_isError() */
size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t const srcSize) size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t srcSize)
{ {
const BYTE* ip = (const BYTE*)src; const BYTE* ip = (const BYTE*)src;

37
programs/fileio.c
View File

@ -64,7 +64,7 @@
#include <sys/stat.h> /* stat64 */ #include <sys/stat.h> /* stat64 */
#include "mem.h" #include "mem.h"
#include "fileio.h" #include "fileio.h"
#include "zstd_static.h" /* ZSTD_magicNumber */ #include "zstd_static.h" /* ZSTD_magicNumber, ZSTD_frameHeaderSize_max */
#include "zbuff_static.h" #include "zbuff_static.h"
#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1) #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1)
@ -242,7 +242,7 @@ static FILE* FIO_openDstFile(const char* dstFileName)
/*! FIO_loadFile() : /*! FIO_loadFile() :
* creates a buffer, pointed by *bufferPtr, * creates a buffer, pointed by `*bufferPtr`,
* loads `filename` content into it, * loads `filename` content into it,
* up to MAX_DICT_SIZE bytes * up to MAX_DICT_SIZE bytes
*/ */
@ -342,6 +342,7 @@ static int FIO_compressFilename_internal(cRess_t ress,
/* init */ /* init */
filesize = MAX(FIO_getFileSize(srcFileName),dictSize); filesize = MAX(FIO_getFileSize(srcFileName),dictSize);
params = ZSTD_getParams(cLevel, filesize); params = ZSTD_getParams(cLevel, filesize);
params.srcSize = filesize;
if (g_maxWLog) if (params.windowLog > g_maxWLog) params.windowLog = g_maxWLog; if (g_maxWLog) if (params.windowLog > g_maxWLog) params.windowLog = g_maxWLog;
errorCode = ZBUFF_compressInit_advanced(ress.ctx, ress.dictBuffer, ress.dictBufferSize, params); errorCode = ZBUFF_compressInit_advanced(ress.ctx, ress.dictBuffer, ress.dictBufferSize, params);
if (ZBUFF_isError(errorCode)) EXM_THROW(21, "Error initializing compression : %s", ZBUFF_getErrorName(errorCode)); if (ZBUFF_isError(errorCode)) EXM_THROW(21, "Error initializing compression : %s", ZBUFF_getErrorName(errorCode));
@ -394,7 +395,7 @@ static int FIO_compressFilename_internal(cRess_t ress,
/*! FIO_compressFilename_internal() : /*! FIO_compressFilename_internal() :
* same as FIO_compressFilename_extRess(), with ress.desFile already opened * same as FIO_compressFilename_extRess(), with ress.destFile already opened (typically stdout)
* @return : 0 : compression completed correctly, * @return : 0 : compression completed correctly,
* 1 : missing or pb opening srcFileName * 1 : missing or pb opening srcFileName
*/ */
@ -432,10 +433,7 @@ static int FIO_compressFilename_extRess(cRess_t ress,
if (ress.dstFile==0) { fclose(ress.srcFile); return 1; } if (ress.dstFile==0) { fclose(ress.srcFile); return 1; }
result = FIO_compressFilename_internal(ress, dstFileName, srcFileName, cLevel); result = FIO_compressFilename_internal(ress, dstFileName, srcFileName, cLevel);
if (result!=0) remove(dstFileName); /* remove operation artefact */
if (result != 0) {
remove(dstFileName);
}
fclose(ress.srcFile); /* no pb to expect : only reading */ fclose(ress.srcFile); /* no pb to expect : only reading */
if (fclose(ress.dstFile)) EXM_THROW(28, "Write error : cannot properly close %s", dstFileName); if (fclose(ress.dstFile)) EXM_THROW(28, "Write error : cannot properly close %s", dstFileName);
@ -556,35 +554,46 @@ static void FIO_freeDResources(dRess_t ress)
} }
/** FIO_decompressFrame() :
@return : size of decoded frame
*/
unsigned long long FIO_decompressFrame(dRess_t ress, unsigned long long FIO_decompressFrame(dRess_t ress,
FILE* foutput, FILE* finput, size_t alreadyLoaded) FILE* foutput, FILE* finput, size_t alreadyLoaded)
{ {
U64 frameSize = 0; U64 frameSize = 0;
size_t readSize=alreadyLoaded; size_t readSize;
ZBUFF_decompressInitDictionary(ress.dctx, ress.dictBuffer, ress.dictBufferSize);
/* Complete Header loading */
{ size_t const toLoad = ZSTD_frameHeaderSize_max - alreadyLoaded; /* assumption : alreadyLoaded <= ZSTD_frameHeaderSize_max */
size_t const checkSize = fread(((char*)ress.srcBuffer) + alreadyLoaded, 1, toLoad, finput);
if (checkSize != toLoad) EXM_THROW(32, "Read error");
}
readSize = ZSTD_frameHeaderSize_max;
/* Main decompression Loop */ /* Main decompression Loop */
ZBUFF_decompressInitDictionary(ress.dctx, ress.dictBuffer, ress.dictBufferSize);
while (1) { while (1) {
/* Decode */ /* Decode */
size_t sizeCheck;
size_t inSize=readSize, decodedSize=ress.dstBufferSize; size_t inSize=readSize, decodedSize=ress.dstBufferSize;
size_t toRead = ZBUFF_decompressContinue(ress.dctx, ress.dstBuffer, &decodedSize, ress.srcBuffer, &inSize); size_t toRead = ZBUFF_decompressContinue(ress.dctx, ress.dstBuffer, &decodedSize, ress.srcBuffer, &inSize);
if (ZBUFF_isError(toRead)) EXM_THROW(36, "Decoding error : %s", ZBUFF_getErrorName(toRead)); if (ZBUFF_isError(toRead)) EXM_THROW(36, "Decoding error : %s", ZBUFF_getErrorName(toRead));
readSize -= inSize; readSize -= inSize;
/* Write block */ /* Write block */
sizeCheck = fwrite(ress.dstBuffer, 1, decodedSize, foutput); { size_t const sizeCheck = fwrite(ress.dstBuffer, 1, decodedSize, foutput);
if (sizeCheck != decodedSize) EXM_THROW(37, "Write error : unable to write data block to destination file"); if (sizeCheck != decodedSize) EXM_THROW(37, "Write error : unable to write data block into destination"); }
frameSize += decodedSize; frameSize += decodedSize;
DISPLAYUPDATE(2, "\rDecoded : %u MB... ", (U32)(frameSize>>20) ); DISPLAYUPDATE(2, "\rDecoded : %u MB... ", (U32)(frameSize>>20) );
if (toRead == 0) break; if (toRead == 0) break; /* end of frame */
if (readSize) EXM_THROW(38, "Decoding error : should consume entire input"); if (readSize) EXM_THROW(38, "Decoding error : should consume entire input");
/* Fill input buffer */ /* Fill input buffer */
if (toRead > ress.srcBufferSize) EXM_THROW(34, "too large block"); if (toRead > ress.srcBufferSize) EXM_THROW(34, "too large block");
readSize = fread(ress.srcBuffer, 1, toRead, finput); readSize = fread(ress.srcBuffer, 1, toRead, finput);
if (readSize != toRead) EXM_THROW(35, "Read error"); if (readSize != toRead)
EXM_THROW(35, "Read error");
} }
return frameSize; return frameSize;