facebook
/
zstd
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

blocksize & T

dev
Yann Collet 2015-10-27 12:18:00 +01:00
parent d55a7e0e4d
commit 2c6992e1dc
5 changed files with 82 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
lib/zstd_static.h
View File

@ -94,11 +94,11 @@ MEM_STATIC unsigned ZSTD_NbCommonBytes (register size_t val)
{
if (MEM_64bits())
{
# if defined(_MSC_VER) && defined(_WIN64) && !defined(LZ4_FORCE_SW_BITCOUNT)
# if defined(_MSC_VER) && defined(_WIN64)
unsigned long r = 0;
_BitScanForward64( &r, (U64)val );
return (int)(r>>3);
# elif defined(__GNUC__) && (__GNUC__ >= 3) && !defined(LZ4_FORCE_SW_BITCOUNT)
# elif defined(__GNUC__) && (__GNUC__ >= 3)
return (__builtin_ctzll((U64)val) >> 3);
# else
static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2, 0, 3, 1, 3, 1, 4, 2, 7, 0, 2, 3, 6, 1, 5, 3, 5, 1, 3, 4, 4, 2, 5, 6, 7, 7, 0, 1, 2, 3, 3, 4, 6, 2, 6, 5, 5, 3, 4, 5, 6, 7, 1, 2, 4, 6, 4, 4, 5, 7, 2, 6, 5, 7, 6, 7, 7 };
@ -107,11 +107,11 @@ MEM_STATIC unsigned ZSTD_NbCommonBytes (register size_t val)
}
else /* 32 bits */
{
# if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT)
# if defined(_MSC_VER)
unsigned long r;
_BitScanForward( &r, (U32)val );
return (int)(r>>3);
# elif defined(__GNUC__) && (__GNUC__ >= 3) && !defined(LZ4_FORCE_SW_BITCOUNT)
# elif defined(__GNUC__) && (__GNUC__ >= 3)
return (__builtin_ctz((U32)val) >> 3);
# else
static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0, 3, 2, 2, 1, 3, 2, 0, 1, 3, 3, 1, 2, 2, 2, 2, 0, 3, 1, 2, 0, 1, 0, 1, 1 };
@ -123,11 +123,11 @@ MEM_STATIC unsigned ZSTD_NbCommonBytes (register size_t val)
{
if (MEM_32bits())
{
# if defined(_MSC_VER) && defined(_WIN64) && !defined(LZ4_FORCE_SW_BITCOUNT)
# if defined(_MSC_VER) && defined(_WIN64)
unsigned long r = 0;
_BitScanReverse64( &r, val );
return (unsigned)(r>>3);
# elif defined(__GNUC__) && (__GNUC__ >= 3) && !defined(LZ4_FORCE_SW_BITCOUNT)
# elif defined(__GNUC__) && (__GNUC__ >= 3)
return (__builtin_clzll(val) >> 3);
# else
unsigned r;
@ -140,11 +140,11 @@ MEM_STATIC unsigned ZSTD_NbCommonBytes (register size_t val)
}
else /* 32 bits */
{
# if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT)
# if defined(_MSC_VER)
unsigned long r = 0;
_BitScanReverse( &r, (unsigned long)val );
return (unsigned)(r>>3);
# elif defined(__GNUC__) && (__GNUC__ >= 3) && !defined(LZ4_FORCE_SW_BITCOUNT)
# elif defined(__GNUC__) && (__GNUC__ >= 3)
return (__builtin_clz((U32)val) >> 3);
# else
unsigned r;

2
lib/zstdhc.c
View File

@ -305,7 +305,7 @@ static size_t ZSTD_HC_compressBlock(ZSTD_HC_CCtx* ctx, void* dst, size_t maxDstS
seqStorePtr->lit += lastLLSize;
}
/* Finale compression stage */
/* Final compression stage */
return ZSTD_compressSequences((BYTE*)dst, maxDstSize,
seqStorePtr, srcSize);
}

10
lib/zstdhc_static.h
View File

@ -53,6 +53,16 @@ typedef struct
U32 searchLog; /* nb of searches : larger == more compression, slower*/
} ZSTD_HC_parameters;
/* parameters boundaries */
#define ZSTD_HC_WINDOWLOG_MAX 26
#define ZSTD_HC_WINDOWLOG_MIN 17
#define ZSTD_HC_CHAINLOG_MAX ZSTD_HC_WINDOWLOG_MAX
#define ZSTD_HC_CHAINLOG_MIN 4
#define ZSTD_HC_HASHLOG_MAX 28
#define ZSTD_HC_HASHLOG_MIN 4
#define ZSTD_HC_SEARCHLOG_MAX (ZSTD_HC_CHAINLOG_MAX-1)
#define ZSTD_HC_SEARCHLOG_MIN 1
/* *************************************
* Functions

6
programs/bench.c
View File

@ -175,7 +175,6 @@ static U32 BMK_rand(U32* src)
return rand32 >> 9;
}
#define BMK_RAND15BITS ( BMK_rand(&seed) & 0x7FFF)
#define BMK_RANDLENGTH ((BMK_rand(&seed) & 3) ? (BMK_rand(&seed) % 15) : (BMK_rand(&seed) % 510) + 15)
#define BMK_RANDCHAR (BYTE)((BMK_rand(&seed) & 63) + '0')
@ -230,7 +229,6 @@ typedef struct
size_t resSize;
} blockParam_t;
typedef size_t (*compressor_t) (void* dst, size_t maxDstSize, const void* src, size_t srcSize, unsigned compressionLevel);
static size_t local_compress_fast (void* dst, size_t maxDstSize, const void* src, size_t srcSize, unsigned compressionLevel)
@ -249,7 +247,7 @@ static int BMK_benchMem(void* srcBuffer, size_t srcSize, const char* fileName, i
const size_t maxCompressedSize = (size_t)nbBlocks * ZSTD_compressBound(blockSize);
void* const compressedBuffer = malloc(maxCompressedSize);
void* const resultBuffer = malloc(srcSize);
const compressor_t compressor = (cLevel <= 1) ? local_compress_fast : ZSTD_HC_compress;
const compressor_t compressor = (cLevel == 0) ? local_compress_fast : ZSTD_HC_compress;
U64 crcOrig;
/* Memory allocation & restrictions */
@ -328,7 +326,7 @@ static int BMK_benchMem(void* srcBuffer, size_t srcSize, const char* fileName, i
ratio = (double)cSize / (double)srcSize*100.;
DISPLAY("%1i-%-14.14s : %9i -> %9i (%5.2f%%),%7.1f MB/s\r", loopNb, fileName, (int)srcSize, (int)cSize, ratio, (double)srcSize / fastestC / 1000.);
#if 1
#if 0
/* Decompression */
memset(resultBuffer, 0xD6, srcSize);

96
programs/paramgrill.c
View File

@ -100,10 +100,10 @@
#define COMPRESSIBILITY_DEFAULT 0.50
static const size_t sampleSize = 10000000;
static const int g_grillDuration = 30000000; /* about 8.5 hours */
static const int g_grillDuration = 50000000; /* about 13 hours */
static const int g_maxParamTime = 15000; /* 15 sec */
static const int g_maxVariationTime = 60000; /* 60 sec */
static const int g_maxNbVariations = 32;
static const int g_maxNbVariations = 64;
/**************************************
* Macros
@ -119,6 +119,7 @@ static double g_compressibility = COMPRESSIBILITY_DEFAULT;
static U32 g_blockSize = 0;
static U32 g_rand = 1;
static U32 g_singleRun = 0;
static U32 g_target = 0;
static ZSTD_HC_parameters g_params = { 0, 0, 0, 0 };
void BMK_SetNbIterations(int nbLoops)
@ -462,24 +463,18 @@ static int BMK_seed(winnerInfo_t* winners, ZSTD_HC_parameters params,
return better;
}
#define WINDOWLOG_MAX 26
#define WINDOWLOG_MIN 17
#define CHAINLOG_MAX WINDOWLOG_MAX
#define CHAINLOG_MIN 4
#define HASHLOG_MAX 28
#define HASHLOG_MIN 4
#define SEARCHLOG_MAX (CHAINLOG_MAX-1)
#define SEARCHLOG_MIN 1
#define MAX(a,b) ( (a) > (b) ? (a) : (b) )
static BYTE g_alreadyTested[WINDOWLOG_MAX+1-WINDOWLOG_MIN]
[CHAINLOG_MAX+1-CHAINLOG_MIN]
[HASHLOG_MAX+1-HASHLOG_MIN]
[SEARCHLOG_MAX+1-SEARCHLOG_MIN] = {}; /* init to zero */
static BYTE g_alreadyTested[ZSTD_HC_WINDOWLOG_MAX+1-ZSTD_HC_WINDOWLOG_MIN]
[ZSTD_HC_CHAINLOG_MAX+1-ZSTD_HC_CHAINLOG_MIN]
[ZSTD_HC_HASHLOG_MAX+1-ZSTD_HC_HASHLOG_MIN]
[ZSTD_HC_SEARCHLOG_MAX+1-ZSTD_HC_SEARCHLOG_MIN] = {}; /* init to zero */
#define NB_TESTS_PLAYED(p) \
g_alreadyTested[p.windowLog-WINDOWLOG_MIN][p.chainLog-CHAINLOG_MIN][p.hashLog-HASHLOG_MIN][p.searchLog-SEARCHLOG_MIN]
g_alreadyTested[p.windowLog-ZSTD_HC_WINDOWLOG_MIN] \
[p.chainLog-ZSTD_HC_CHAINLOG_MIN] \
[p.hashLog-ZSTD_HC_HASHLOG_MIN] \
[p.searchLog-ZSTD_HC_SEARCHLOG_MIN]
static void playAround(FILE* f, winnerInfo_t* winners,
@ -520,14 +515,14 @@ static void playAround(FILE* f, winnerInfo_t* winners,
}
/* validate new conf */
if (p.windowLog > WINDOWLOG_MAX) continue;
if (p.windowLog < MAX(WINDOWLOG_MIN, p.chainLog)) continue;
if (p.windowLog > ZSTD_HC_WINDOWLOG_MAX) continue;
if (p.windowLog < MAX(ZSTD_HC_WINDOWLOG_MIN, p.chainLog)) continue;
if (p.chainLog > p.windowLog) continue;
if (p.chainLog < CHAINLOG_MIN) continue;
if (p.hashLog > HASHLOG_MAX) continue;
if (p.hashLog < HASHLOG_MIN) continue;
if (p.chainLog < ZSTD_HC_CHAINLOG_MIN) continue;
if (p.hashLog > ZSTD_HC_HASHLOG_MAX) continue;
if (p.hashLog < ZSTD_HC_HASHLOG_MIN) continue;
if (p.searchLog > p.chainLog) continue;
if (p.searchLog < SEARCHLOG_MIN) continue;
if (p.searchLog < ZSTD_HC_SEARCHLOG_MIN) continue;
/* exclude faster if already played params */
if (FUZ_rand(&g_rand) & ((1 << NB_TESTS_PLAYED(p))-1)) continue;
@ -554,10 +549,10 @@ static void BMK_selectRandomStart(
{
/* totally random entry */
ZSTD_HC_parameters p;
p.chainLog = FUZ_rand(&g_rand) % (CHAINLOG_MAX+1 - CHAINLOG_MIN) + CHAINLOG_MIN;
p.hashLog = FUZ_rand(&g_rand) % (HASHLOG_MAX+1 - HASHLOG_MIN) + HASHLOG_MIN;
p.searchLog = FUZ_rand(&g_rand) % (SEARCHLOG_MAX+1 - SEARCHLOG_MIN) + SEARCHLOG_MIN;
p.windowLog = FUZ_rand(&g_rand) % (WINDOWLOG_MAX+1 - WINDOWLOG_MIN) + WINDOWLOG_MIN;
p.chainLog = FUZ_rand(&g_rand) % (ZSTD_HC_CHAINLOG_MAX+1 - ZSTD_HC_CHAINLOG_MIN) + ZSTD_HC_CHAINLOG_MIN;
p.hashLog = FUZ_rand(&g_rand) % (ZSTD_HC_HASHLOG_MAX+1 - ZSTD_HC_HASHLOG_MIN) + ZSTD_HC_HASHLOG_MIN;
p.searchLog = FUZ_rand(&g_rand) % (ZSTD_HC_SEARCHLOG_MAX+1 - ZSTD_HC_SEARCHLOG_MIN) + ZSTD_HC_SEARCHLOG_MIN;
p.windowLog = FUZ_rand(&g_rand) % (ZSTD_HC_WINDOWLOG_MAX+1 - ZSTD_HC_WINDOWLOG_MIN) + ZSTD_HC_WINDOWLOG_MIN;
playAround(f, winners, p, srcBuffer, srcSize, ctx);
}
else
@ -589,24 +584,29 @@ static void BMK_benchMem(void* srcBuffer, size_t srcSize)
f = fopen(rfName, "w");
if (f==NULL) { DISPLAY("error opening %s \n", rfName); exit(1); }
/* baseline config for level 9 */
params.windowLog = 19;
params.chainLog = 19;
params.hashLog = 19;
params.searchLog = 9;
BMK_benchParam(&testResult, srcBuffer, srcSize, ctx, params);
if (g_target)
g_cSpeedTarget[9] = g_target * 1000;
else
{
/* baseline config for level 9 */
params.windowLog = 19;
params.chainLog = 19;
params.hashLog = 19;
params.searchLog = 9;
BMK_benchParam(&testResult, srcBuffer, srcSize, ctx, params);
g_cSpeedTarget[9] = (testResult.cSpeed * 15) >> 4;
}
/* establish speed objectives (relative to current platform) */
g_cSpeedTarget[9] = (testResult.cSpeed * 15) >> 4;
/* establish speed objectives (relative to level9) */
g_cSpeedTarget[1] = g_cSpeedTarget[9] << 4;
g_cSpeedTarget[0] = (g_cSpeedTarget[1] * 181) >> 7; /* sqrt2 */
for (i=2; i<ZSTD_HC_MAX_CLEVEL; i++) /* note : last level no speed limit */
g_cSpeedTarget[i] = (g_cSpeedTarget[i-1] * 181) >> 8;
/* populate initial solution */
BMK_seed(winners, params, srcBuffer, srcSize, ctx);
for (i=1; i<=ZSTD_HC_MAX_CLEVEL; i++)
BMK_seed(winners, seedParams[i], srcBuffer, srcSize, ctx);
BMK_seed(winners, params, srcBuffer, srcSize, ctx);
BMK_printWinners(f, winners, srcSize);
/* start tests */
@ -622,6 +622,7 @@ static void BMK_benchMem(void* srcBuffer, size_t srcSize)
/* end summary */
BMK_printWinners(f, winners, srcSize);
DISPLAY("grillParams operations completed \n");
/* clean up*/
fclose(f);
@ -822,6 +823,31 @@ int main(int argc, char** argv)
break;
}
/* target level9 objective, in MB/s */
case 'T':
argument++;
g_target = 0;
while ((*argument >= '0') && (*argument <= '9'))
{
g_target *= 10;
g_target += *argument - '0';
argument++;
}
break;
/* cut input into blocks */
case 'B':
{
g_blockSize = 0;
argument++;
while ((*argument >='0') && (*argument <='9'))
g_blockSize *= 10, g_blockSize += *argument++ - '0';
if (*argument=='K') g_blockSize<<=10, argument++; /* allows using KB notation */
if (*argument=='M') g_blockSize<<=20, argument++;
if (*argument=='B') argument++;
}
break;
/* Unknown command */
default : return badusage(exename);
}