Yann Collet
commit
adcffd085c
14
lib/zstd.c
14
lib/zstd.c
|
|
@ -1004,9 +1004,9 @@ size_t ZSTD_compressContinue(ZSTD_cctx_t cctx, void* dst, size_t maxDstSize, con
|
|||
//U32 limit = 4 * BLOCKSIZE;
|
||||
//const U32 updateRate = 2 * BLOCKSIZE;
|
||||
|
||||
// Init
|
||||
/* Init */
|
||||
if (ctx->base==NULL) ctx->base = src, ctx->current=0;
|
||||
if (src != ctx->base + ctx->current) // not contiguous
|
||||
if (src != ctx->base + ctx->current) /* not contiguous */
|
||||
{
|
||||
ZSTD_resetCCtx(ctx);
|
||||
ctx->base = src;
|
||||
|
|
@ -1035,12 +1035,12 @@ size_t ZSTD_compressContinue(ZSTD_cctx_t cctx, void* dst, size_t maxDstSize, con
|
|||
}
|
||||
*/
|
||||
|
||||
// compress
|
||||
/* compress */
|
||||
if (maxDstSize < ZSTD_blockHeaderSize) return (size_t)-ZSTD_ERROR_maxDstSize_tooSmall;
|
||||
cSize = ZSTD_compressBlock(ctx, op+ZSTD_blockHeaderSize, maxDstSize-ZSTD_blockHeaderSize, ip, blockSize);
|
||||
if (cSize == 0)
|
||||
{
|
||||
cSize = ZSTD_noCompressBlock(op, maxDstSize, ip, blockSize);
|
||||
cSize = ZSTD_noCompressBlock(op, maxDstSize, ip, blockSize); /* block is not compressible */
|
||||
if (ZSTD_isError(cSize)) return cSize;
|
||||
}
|
||||
else
|
||||
|
|
@ -1084,7 +1084,7 @@ static size_t ZSTD_compressCCtx(void* ctx, void* dst, size_t maxDstSize, const v
|
|||
BYTE* const ostart = dst;
|
||||
BYTE* op = ostart;
|
||||
|
||||
// Header
|
||||
/* Header */
|
||||
{
|
||||
size_t headerSize = ZSTD_compressBegin(ctx, dst, maxDstSize);
|
||||
if(ZSTD_isError(headerSize)) return headerSize;
|
||||
|
|
@ -1092,7 +1092,7 @@ static size_t ZSTD_compressCCtx(void* ctx, void* dst, size_t maxDstSize, const v
|
|||
maxDstSize -= headerSize;
|
||||
}
|
||||
|
||||
// Compression
|
||||
/* Compression */
|
||||
{
|
||||
size_t cSize = ZSTD_compressContinue(ctx, op, maxDstSize, src, srcSize);
|
||||
if (ZSTD_isError(cSize)) return cSize;
|
||||
|
|
@ -1100,7 +1100,7 @@ static size_t ZSTD_compressCCtx(void* ctx, void* dst, size_t maxDstSize, const v
|
|||
maxDstSize -= cSize;
|
||||
}
|
||||
|
||||
// Close frame
|
||||
/* Close frame */
|
||||
{
|
||||
size_t endSize = ZSTD_compressEnd(ctx, op, maxDstSize);
|
||||
if(ZSTD_isError(endSize)) return endSize;
|
||||
|
|
|
|||
|
|
@ -127,11 +127,17 @@ test-zstd32: zstd32 datagen
|
|||
./datagen -g256MB | ./zstd32 -v | ./zstd32 -d > $(VOID)
|
||||
./datagen -g6GB | ./zstd32 -vq | ./zstd32 -d > $(VOID)
|
||||
|
||||
test-fullbench: fullbench
|
||||
test-fullbench: fullbench datagen
|
||||
./fullbench -i1
|
||||
./datagen -P0 -g516K > tmp
|
||||
./fullbench -i1 tmp
|
||||
@rm tmp
|
||||
|
||||
test-fullbench32: fullbench32
|
||||
test-fullbench32: fullbench32 datagen
|
||||
./fullbench32 -i1
|
||||
./datagen -P0 -g516K > tmp
|
||||
./fullbench32 -i1 tmp
|
||||
@rm tmp
|
||||
|
||||
test-fuzzer: fuzzer
|
||||
./fuzzer
|
||||
|
|
|
|||
|
|
@ -56,6 +56,24 @@
|
|||
#endif
|
||||
|
||||
|
||||
/**************************************
|
||||
* OS-specific Includes
|
||||
**************************************/
|
||||
#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) || defined(__CYGWIN__)
|
||||
# include <fcntl.h> /* _O_BINARY */
|
||||
# include <io.h> /* _setmode, _isatty */
|
||||
# ifdef __MINGW32__
|
||||
int _fileno(FILE *stream); /* MINGW somehow forgets to include this windows declaration into <stdio.h> */
|
||||
# endif
|
||||
# define SET_BINARY_MODE(file) _setmode(_fileno(file), _O_BINARY)
|
||||
# define IS_CONSOLE(stdStream) _isatty(_fileno(stdStream))
|
||||
#else
|
||||
# include <unistd.h> /* isatty */
|
||||
# define SET_BINARY_MODE(file)
|
||||
# define IS_CONSOLE(stdStream) isatty(fileno(stdStream))
|
||||
#endif
|
||||
|
||||
|
||||
/**************************************
|
||||
* Constants
|
||||
**************************************/
|
||||
|
|
@ -85,7 +103,6 @@
|
|||
* Local Parameters
|
||||
**************************************/
|
||||
static unsigned no_prompt = 0;
|
||||
static char* programName;
|
||||
static unsigned displayLevel = 2;
|
||||
|
||||
|
||||
|
|
@ -97,7 +114,7 @@ static unsigned int CDG_rand(U32* src)
|
|||
{
|
||||
U32 rand32 = *src;
|
||||
rand32 *= PRIME1;
|
||||
rand32 += PRIME2;
|
||||
rand32 ^= PRIME2;
|
||||
rand32 = CDG_rotl32(rand32, 13);
|
||||
*src = rand32;
|
||||
return rand32;
|
||||
|
|
@ -106,14 +123,20 @@ static unsigned int CDG_rand(U32* src)
|
|||
|
||||
#define LTSIZE 8192
|
||||
#define LTMASK (LTSIZE-1)
|
||||
static const char firstChar = '(';
|
||||
static const char lastChar = '}';
|
||||
static void* CDG_createLiteralDistrib(double ld)
|
||||
{
|
||||
char* lt = malloc(LTSIZE);
|
||||
BYTE* lt = malloc(LTSIZE);
|
||||
U32 i = 0;
|
||||
char character = '0';
|
||||
BYTE character = '0';
|
||||
BYTE firstChar = '(';
|
||||
BYTE lastChar = '}';
|
||||
|
||||
if (ld==0.0)
|
||||
{
|
||||
character = 0;
|
||||
firstChar = 0;
|
||||
lastChar =255;
|
||||
}
|
||||
while (i<LTSIZE)
|
||||
{
|
||||
U32 weight = (U32)((double)(LTSIZE - i) * ld) + 1;
|
||||
|
|
@ -137,7 +160,7 @@ static char CDG_genChar(U32* seed, const void* ltctx)
|
|||
#define CDG_RAND15BITS ((CDG_rand(seed) >> 3) & 32767)
|
||||
#define CDG_RANDLENGTH ( ((CDG_rand(seed) >> 7) & 7) ? (CDG_rand(seed) & 15) : (CDG_rand(seed) & 511) + 15)
|
||||
#define CDG_DICTSIZE (32 KB)
|
||||
static void CDG_generate(U64 size, U32* seed, double matchProba)
|
||||
static void CDG_generate(U64 size, U32* seed, double matchProba, double litProba)
|
||||
{
|
||||
BYTE fullbuff[CDG_DICTSIZE + 128 KB + 1];
|
||||
BYTE* buff = fullbuff + CDG_DICTSIZE;
|
||||
|
|
@ -145,10 +168,11 @@ static void CDG_generate(U64 size, U32* seed, double matchProba)
|
|||
U32 P32 = (U32)(32768 * matchProba);
|
||||
U32 pos=1;
|
||||
U32 genBlockSize = 128 KB;
|
||||
double literalDistrib = 0.13;
|
||||
void* ldctx = CDG_createLiteralDistrib(literalDistrib);
|
||||
void* ldctx = CDG_createLiteralDistrib(litProba);
|
||||
FILE* fout = stdout;
|
||||
|
||||
/* Build initial prefix */
|
||||
/* init */
|
||||
SET_BINARY_MODE(stdout);
|
||||
fullbuff[0] = CDG_genChar(seed, ldctx);
|
||||
while (pos<32 KB)
|
||||
{
|
||||
|
|
@ -207,11 +231,8 @@ static void CDG_generate(U64 size, U32* seed, double matchProba)
|
|||
}
|
||||
}
|
||||
|
||||
/* output datagen */
|
||||
pos=0;
|
||||
for (;pos+512<=genBlockSize;pos+=512)
|
||||
printf("%512.512s", buff+pos);
|
||||
for (;pos<genBlockSize;pos++) printf("%c", buff[pos]);
|
||||
/* output generated data */
|
||||
fwrite(buff, 1, genBlockSize, fout);
|
||||
/* Regenerate prefix */
|
||||
memcpy(fullbuff, buff + 96 KB, 32 KB);
|
||||
}
|
||||
|
|
@ -221,7 +242,7 @@ static void CDG_generate(U64 size, U32* seed, double matchProba)
|
|||
/*********************************************************
|
||||
* Command line
|
||||
*********************************************************/
|
||||
static int CDG_usage(void)
|
||||
static int CDG_usage(char* programName)
|
||||
{
|
||||
DISPLAY( "Compressible data generator\n");
|
||||
DISPLAY( "Usage :\n");
|
||||
|
|
@ -239,9 +260,11 @@ static int CDG_usage(void)
|
|||
int main(int argc, char** argv)
|
||||
{
|
||||
int argNb;
|
||||
int proba = CDG_COMPRESSIBILITY_DEFAULT;
|
||||
double proba = (double)CDG_COMPRESSIBILITY_DEFAULT / 100;
|
||||
double litProba = proba / 3.6;
|
||||
U64 size = CDG_SIZE_DEFAULT;
|
||||
U32 seed = CDG_SEED_DEFAULT;
|
||||
char* programName;
|
||||
|
||||
/* Check command line */
|
||||
programName = argv[0];
|
||||
|
|
@ -262,7 +285,7 @@ int main(int argc, char** argv)
|
|||
switch(*argument)
|
||||
{
|
||||
case 'h':
|
||||
return CDG_usage();
|
||||
return CDG_usage(programName);
|
||||
case 'g':
|
||||
argument++;
|
||||
size=0;
|
||||
|
|
@ -287,23 +310,37 @@ int main(int argc, char** argv)
|
|||
argument++;
|
||||
}
|
||||
break;
|
||||
case 'p':
|
||||
case 'P':
|
||||
argument++;
|
||||
proba=0;
|
||||
proba=0.0;
|
||||
while ((*argument>='0') && (*argument<='9'))
|
||||
{
|
||||
proba *= 10;
|
||||
proba += *argument - '0';
|
||||
argument++;
|
||||
}
|
||||
if (proba<0) proba=0;
|
||||
if (proba>100) proba=100;
|
||||
if (proba>100.) proba=100.;
|
||||
proba /= 100.;
|
||||
litProba = proba / 4.;
|
||||
break;
|
||||
case 'L':
|
||||
argument++;
|
||||
litProba=0.;
|
||||
while ((*argument>='0') && (*argument<='9'))
|
||||
{
|
||||
litProba *= 10;
|
||||
litProba += *argument - '0';
|
||||
argument++;
|
||||
}
|
||||
if (litProba>100.) litProba=100.;
|
||||
litProba /= 100.;
|
||||
break;
|
||||
case 'v':
|
||||
displayLevel = 4;
|
||||
argument++;
|
||||
break;
|
||||
default: ;
|
||||
default:
|
||||
return CDG_usage(programName);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -312,9 +349,9 @@ int main(int argc, char** argv)
|
|||
|
||||
DISPLAYLEVEL(4, "Data Generator %s \n", ZSTD_VERSION);
|
||||
DISPLAYLEVEL(3, "Seed = %u \n", seed);
|
||||
if (proba!=CDG_COMPRESSIBILITY_DEFAULT) DISPLAYLEVEL(3, "Compressibility : %i%%\n", proba);
|
||||
if (proba!=CDG_COMPRESSIBILITY_DEFAULT) DISPLAYLEVEL(3, "Compressibility : %i%%\n", (U32)(proba*100));
|
||||
|
||||
CDG_generate(size, &seed, ((double)proba) / 100);
|
||||
CDG_generate(size, &seed, proba, litProba);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -383,7 +383,7 @@ size_t benchMem(void* src, size_t srcSize, U32 benchNb)
|
|||
}
|
||||
|
||||
/* Allocation */
|
||||
dstBuffSize = srcSize + 512;
|
||||
dstBuffSize = ZSTD_compressBound(srcSize);
|
||||
dstBuff = malloc(dstBuffSize);
|
||||
buff2 = malloc(dstBuffSize);
|
||||
if ((!dstBuff) || (!buff2))
|
||||
|
|
@ -403,6 +403,14 @@ size_t benchMem(void* src, size_t srcSize, U32 benchNb)
|
|||
{
|
||||
blockProperties_t bp;
|
||||
ZSTD_compress(dstBuff, dstBuffSize, src, srcSize);
|
||||
ZSTD_getcBlockSize(dstBuff+4, dstBuffSize, &bp); // Get first block compressed size
|
||||
if (bp.blockType != bt_compressed)
|
||||
{
|
||||
DISPLAY("ZSTD_decodeLiteralsBlock : impossible to test on this sample (not compressible)\n");
|
||||
free(dstBuff);
|
||||
free(buff2);
|
||||
return 0;
|
||||
}
|
||||
g_cSize = ZSTD_getcBlockSize(dstBuff+7, dstBuffSize, &bp) + 3;
|
||||
memcpy(buff2, dstBuff+7, g_cSize);
|
||||
//srcSize = benchFunction(dstBuff, dstBuffSize, buff2, src, srcSize); // real speed
|
||||
|
|
@ -418,6 +426,13 @@ size_t benchMem(void* src, size_t srcSize, U32 benchNb)
|
|||
ZSTD_compress(dstBuff, dstBuffSize, src, srcSize);
|
||||
ip += 4; // Jump magic Number
|
||||
blockSize = ZSTD_getcBlockSize(ip, dstBuffSize, &bp); // Get first block compressed size
|
||||
if (bp.blockType != bt_compressed)
|
||||
{
|
||||
DISPLAY("ZSTD_decodeSeqHeaders : impossible to test on this sample (not compressible)\n");
|
||||
free(dstBuff);
|
||||
free(buff2);
|
||||
return 0;
|
||||
}
|
||||
iend = ip + 3 + blockSize; // Get end of first block
|
||||
ip += 3; // jump first block header
|
||||
ip += ZSTD_getcBlockSize(ip, iend - ip, &bp) + 3; // jump literal sub block and its header
|
||||
|
|
@ -450,6 +465,8 @@ size_t benchMem(void* src, size_t srcSize, U32 benchNb)
|
|||
default : ;
|
||||
}
|
||||
|
||||
{ size_t i; for (i=0; i<dstBuffSize; i++) dstBuff[i]=(BYTE)i; } /* warming up memory */
|
||||
|
||||
for (loopNb = 1; loopNb <= nbIterations; loopNb++)
|
||||
{
|
||||
double averageTime;
|
||||
|
|
@ -457,7 +474,6 @@ size_t benchMem(void* src, size_t srcSize, U32 benchNb)
|
|||
U32 nbRounds=0;
|
||||
|
||||
DISPLAY("%2i- %-30.30s : \r", loopNb, benchName);
|
||||
{ size_t i; for (i=0; i<dstBuffSize; i++) dstBuff[i]=(BYTE)i; } /* warming up memory */
|
||||
|
||||
milliTime = BMK_GetMilliStart();
|
||||
while(BMK_GetMilliStart() == milliTime);
|
||||
|
|
@ -545,7 +561,7 @@ int benchFiles(char** fileNamesTable, int nbFiles, U32 benchNb)
|
|||
DISPLAY("Not enough memory for '%s' full size; testing %i MB only...\n", inFileName, (int)(benchedSize>>20));
|
||||
}
|
||||
|
||||
// Alloc
|
||||
/* Alloc */
|
||||
origBuff = (char*) malloc((size_t)benchedSize);
|
||||
if(!origBuff)
|
||||
{
|
||||
|
|
@ -554,7 +570,7 @@ int benchFiles(char** fileNamesTable, int nbFiles, U32 benchNb)
|
|||
return 12;
|
||||
}
|
||||
|
||||
// Fill input buffer
|
||||
/* Fill input buffer */
|
||||
DISPLAY("Loading %s... \r", inFileName);
|
||||
readSize = fread(origBuff, 1, benchedSize, inFile);
|
||||
fclose(inFile);
|
||||
|
|
@ -566,7 +582,7 @@ int benchFiles(char** fileNamesTable, int nbFiles, U32 benchNb)
|
|||
return 13;
|
||||
}
|
||||
|
||||
// bench
|
||||
/* bench */
|
||||
DISPLAY("\r%79s\r", "");
|
||||
DISPLAY(" %s : \n", inFileName);
|
||||
if (benchNb)
|
||||
|
|
|
|||
Loading…
Reference in New Issue