facebook
/
zstd
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fileio can use small blocks params

dev
Yann Collet 2015-11-09 17:42:17 +01:00
parent 43e0cd512a
commit 9f432926b4
4 changed files with 50 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
lib/zstdhc.c
View File

@ -111,7 +111,7 @@ size_t ZSTD_HC_freeCCtx(ZSTD_HC_CCtx* cctx)
/** ZSTD_HC_validateParams
correct params value to remain within authorized range
optimize for srcSize if srcSize > 0 */
void ZSTD_HC_validateParams(ZSTD_HC_parameters* params, size_t srcSize)
void ZSTD_HC_validateParams(ZSTD_HC_parameters* params, U64 srcSizeHint)
{
const U32 btPlus = (params->strategy == ZSTD_HC_btlazy2);
@ -120,9 +120,9 @@ void ZSTD_HC_validateParams(ZSTD_HC_parameters* params, size_t srcSize)
if (params->windowLog < ZSTD_HC_WINDOWLOG_MIN) params->windowLog = ZSTD_HC_WINDOWLOG_MIN;
/* correct params, to use less memory */
if (srcSize > 0)
if ((srcSizeHint > 0) && (srcSizeHint < (1<<ZSTD_HC_WINDOWLOG_MAX)))
{
U32 srcLog = ZSTD_highbit((U32)srcSize-1) + 1;
U32 srcLog = ZSTD_highbit((U32)srcSizeHint-1) + 1;
if (params->windowLog > srcLog) params->windowLog = srcLog;
}
@ -139,9 +139,10 @@ void ZSTD_HC_validateParams(ZSTD_HC_parameters* params, size_t srcSize)
static size_t ZSTD_HC_resetCCtx_advanced (ZSTD_HC_CCtx* zc,
ZSTD_HC_parameters params)
ZSTD_HC_parameters params,
U64 srcSizeHint)
{
ZSTD_HC_validateParams(&params, 0);
ZSTD_HC_validateParams(&params, srcSizeHint);
/* reserve table memory */
{
@ -930,7 +931,7 @@ size_t ZSTD_HC_compressContinue (ZSTD_HC_CCtx* ctxPtr,
if (ip != ctxPtr->end)
{
if (ctxPtr->end != NULL)
ZSTD_HC_resetCCtx_advanced(ctxPtr, ctxPtr->params);
ZSTD_HC_resetCCtx_advanced(ctxPtr, ctxPtr->params, srcSize);
ctxPtr->base = ip;
}
@ -941,23 +942,24 @@ size_t ZSTD_HC_compressContinue (ZSTD_HC_CCtx* ctxPtr,
size_t ZSTD_HC_compressBegin_advanced(ZSTD_HC_CCtx* ctx,
void* dst, size_t maxDstSize,
const ZSTD_HC_parameters params)
const ZSTD_HC_parameters params,
U64 srcSizeHint)
{
size_t errorCode;
if (maxDstSize < 4) return ERROR(dstSize_tooSmall);
errorCode = ZSTD_HC_resetCCtx_advanced(ctx, params);
errorCode = ZSTD_HC_resetCCtx_advanced(ctx, params, srcSizeHint);
if (ZSTD_isError(errorCode)) return errorCode;
MEM_writeLE32(dst, ZSTD_magicNumber); /* Write Header */
return 4;
}
size_t ZSTD_HC_compressBegin(ZSTD_HC_CCtx* ctx, void* dst, size_t maxDstSize, int compressionLevel)
size_t ZSTD_HC_compressBegin(ZSTD_HC_CCtx* ctx, void* dst, size_t maxDstSize, int compressionLevel, U64 srcSizeHint)
{
int tableID = 1;
int tableID = ((srcSizeHint-1) > 128 KB); /* intentional underflow for 0 */
if (compressionLevel<=0) compressionLevel = 1;
if (compressionLevel > ZSTD_HC_MAX_CLEVEL) compressionLevel = ZSTD_HC_MAX_CLEVEL;
return ZSTD_HC_compressBegin_advanced(ctx, dst, maxDstSize, ZSTD_HC_defaultParameters[tableID][compressionLevel]);
return ZSTD_HC_compressBegin_advanced(ctx, dst, maxDstSize, ZSTD_HC_defaultParameters[tableID][compressionLevel], srcSizeHint);
}
@ -995,7 +997,7 @@ size_t ZSTD_HC_compress_advanced (ZSTD_HC_CCtx* ctx,
}
/* Header */
oSize = ZSTD_HC_compressBegin_advanced(ctx, dst, maxDstSize, params);
oSize = ZSTD_HC_compressBegin_advanced(ctx, dst, maxDstSize, params, srcSize);
if(ZSTD_isError(oSize)) return oSize;
op += oSize;
maxDstSize -= oSize;

6
lib/zstdhc_static.h
View File

@ -83,14 +83,14 @@ size_t ZSTD_HC_compress_advanced (ZSTD_HC_CCtx* ctx,
/** ZSTD_HC_validateParams
correct params value to remain within authorized range
optimize for srcSize if srcSize > 0 */
void ZSTD_HC_validateParams(ZSTD_HC_parameters* params, size_t srcSize);
srcSizeHint value is optional, select 0 if not known */
void ZSTD_HC_validateParams(ZSTD_HC_parameters* params, U64 srcSizeHint);
/* *************************************
* Streaming functions
***************************************/
size_t ZSTD_HC_compressBegin(ZSTD_HC_CCtx* ctx, void* dst, size_t maxDstSize, int compressionLevel);
size_t ZSTD_HC_compressBegin(ZSTD_HC_CCtx* ctx, void* dst, size_t maxDstSize, int compressionLevel, U64 srcSizeHint);
size_t ZSTD_HC_compressContinue(ZSTD_HC_CCtx* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize);
size_t ZSTD_HC_compressEnd(ZSTD_HC_CCtx* ctx, void* dst, size_t maxDstSize);

2
programs/bench.c
View File

@ -380,7 +380,7 @@ static int BMK_benchMem(void* srcBuffer, size_t srcSize, const char* fileName, i
}
static U64 BMK_GetFileSize(char* infilename)
static U64 BMK_GetFileSize(const char* infilename)
{
int r;
#if defined(_MSC_VER)

44
programs/fileio.c
View File

@ -36,7 +36,7 @@
/**LEGACY_SUPPORT :
* decompressor can decode older formats (starting from Zstd 0.1+) */
# define ZSTD_LEGACY_SUPPORT 1
#endif // ZSTD_LEGACY_SUPPORT
#endif
/* *************************************
@ -58,11 +58,13 @@
/* *************************************
* Includes
***************************************/
#include <stdio.h> /* fprintf, fopen, fread, _fileno, stdin, stdout */
#include <stdlib.h> /* malloc, free */
#include <string.h> /* strcmp, strlen */
#include <time.h> /* clock */
#include <errno.h> /* errno */
#include <stdio.h> /* fprintf, fopen, fread, _fileno, stdin, stdout */
#include <stdlib.h> /* malloc, free */
#include <string.h> /* strcmp, strlen */
#include <time.h> /* clock */
#include <errno.h> /* errno */
#include <sys/types.h> /* stat64 */
#include <sys/stat.h> /* stat64 */
#include "mem.h"
#include "fileio.h"
#include "zstd_static.h"
@ -215,19 +217,35 @@ static void FIO_getFileHandles(FILE** pfinput, FILE** pfoutput, const char* inpu
if ( *pfoutput==0) EXM_THROW(13, "Pb opening dst : %s", output_filename);
}
static U64 FIO_getFileSize(const char* infilename)
{
int r;
#if defined(_MSC_VER)
struct _stat64 statbuf;
r = _stat64(infilename, &statbuf);
#else
struct stat statbuf;
r = stat(infilename, &statbuf);
#endif
if (r || !S_ISREG(statbuf.st_mode)) return 0;
return (U64)statbuf.st_size;
}
typedef void* (*FIO_createC) (void);
static void* local_ZSTD_createCCtx(void) { return (void*) ZSTD_createCCtx(); }
static void* local_ZSTD_HC_createCCtx(void) { return (void*) ZSTD_HC_createCCtx(); }
typedef size_t (*FIO_initC) (void* ctx, void* dst, size_t maxDstSize, int cLevel);
static size_t local_ZSTD_compressBegin (void* ctx, void* dst, size_t maxDstSize, int cLevel)
typedef size_t (*FIO_initC) (void* ctx, void* dst, size_t maxDstSize, int cLevel, U64 srcSizeHint);
static size_t local_ZSTD_compressBegin (void* ctx, void* dst, size_t maxDstSize, int cLevel, U64 srcSizeHint)
{
(void)cLevel;
(void)cLevel; (void)srcSizeHint;
return ZSTD_compressBegin((ZSTD_CCtx*)ctx, dst, maxDstSize);
}
static size_t local_ZSTD_HC_compressBegin (void* ctx, void* dst, size_t maxDstSize, int cLevel)
static size_t local_ZSTD_HC_compressBegin (void* ctx, void* dst, size_t maxDstSize, int cLevel, U64 srcSizeHint)
{
return ZSTD_HC_compressBegin((ZSTD_HC_CCtx*)ctx, dst, maxDstSize, cLevel);
return ZSTD_HC_compressBegin((ZSTD_HC_CCtx*)ctx, dst, maxDstSize, cLevel, srcSizeHint);
}
typedef size_t (*FIO_continueC) (void* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize);
@ -294,6 +312,7 @@ unsigned long long FIO_compressFilename(const char* output_filename, const char*
freeC = local_ZSTD_HC_freeCCtx;
}
FIO_getFileHandles(&finput, &foutput, input_filename, output_filename);
filesize = FIO_getFileSize(input_filename);
/* Allocate Memory */
ctx = createC();
@ -304,12 +323,13 @@ unsigned long long FIO_compressFilename(const char* output_filename, const char*
inEnd = inBuff + inBuffSize;
/* Write Frame Header */
cSize = initC(ctx, outBuff, outBuffSize, cLevel);
cSize = initC(ctx, outBuff, outBuffSize, cLevel, filesize);
if (ZSTD_isError(cSize)) EXM_THROW(22, "Compression error : cannot create frame header");
sizeCheck = fwrite(outBuff, 1, cSize, foutput);
if (sizeCheck!=cSize) EXM_THROW(23, "Write error : cannot write header into %s", output_filename);
compressedfilesize += cSize;
filesize = 0;
/* Main compression loop */
while (1)