facebook
/
zstd
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

added additional tests for performance, allowed force compression level for testing purposes

dev
Paul Cruz 2017-07-13 14:46:54 -07:00
parent 9165e97fc6
commit 0d9665cef5
2 changed files with 62 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

61
contrib/adaptive-compression/adapt.c
View File

@ -33,6 +33,7 @@ static UTIL_time_t g_startTime;
static size_t g_streamedSize = 0; static size_t g_streamedSize = 0;
static unsigned g_useProgressBar = 0; static unsigned g_useProgressBar = 0;
static UTIL_freq_t g_ticksPerSecond; static UTIL_freq_t g_ticksPerSecond;
static unsigned g_forceCompressionLevel = 0;
typedef struct { typedef struct {
void* start; void* start;
@ -223,34 +224,39 @@ static void waitUntilAllJobsCompleted(adaptCCtx* ctx)
*/ */
static unsigned adaptCompressionLevel(adaptCCtx* ctx) static unsigned adaptCompressionLevel(adaptCCtx* ctx)
{ {
unsigned reset = 0; if (g_forceCompressionLevel) {
unsigned const allSlow = ctx->adaptParam < ctx->stats.compressedCounter && ctx->adaptParam < ctx->stats.writeCounter && ctx->adaptParam < ctx->stats.readyCounter; return g_compressionLevel;
unsigned const compressWaiting = ctx->adaptParam < ctx->stats.readyCounter;
unsigned const writeWaiting = ctx->adaptParam < ctx->stats.compressedCounter;
unsigned const createWaiting = ctx->adaptParam < ctx->stats.writeCounter;
unsigned const writeSlow = ((compressWaiting && createWaiting) || (createWaiting && !writeWaiting));
unsigned const compressSlow = ((writeWaiting && createWaiting) || (writeWaiting && !compressWaiting));
unsigned const createSlow = ((compressWaiting && writeWaiting) || (compressWaiting && !createWaiting));
DEBUG(3, "ready: %u compressed: %u write: %u\n", ctx->stats.readyCounter, ctx->stats.compressedCounter, ctx->stats.writeCounter);
if (allSlow) {
reset = 1;
} }
else if ((writeSlow || createSlow) && ctx->compressionLevel < (unsigned)ZSTD_maxCLevel()) { else {
DEBUG(3, "increasing compression level %u\n", ctx->compressionLevel); unsigned reset = 0;
ctx->compressionLevel++; unsigned const allSlow = ctx->adaptParam < ctx->stats.compressedCounter && ctx->adaptParam < ctx->stats.writeCounter && ctx->adaptParam < ctx->stats.readyCounter;
reset = 1; unsigned const compressWaiting = ctx->adaptParam < ctx->stats.readyCounter;
unsigned const writeWaiting = ctx->adaptParam < ctx->stats.compressedCounter;
unsigned const createWaiting = ctx->adaptParam < ctx->stats.writeCounter;
unsigned const writeSlow = ((compressWaiting && createWaiting) || (createWaiting && !writeWaiting));
unsigned const compressSlow = ((writeWaiting && createWaiting) || (writeWaiting && !compressWaiting));
unsigned const createSlow = ((compressWaiting && writeWaiting) || (compressWaiting && !createWaiting));
DEBUG(3, "ready: %u compressed: %u write: %u\n", ctx->stats.readyCounter, ctx->stats.compressedCounter, ctx->stats.writeCounter);
if (allSlow) {
reset = 1;
}
else if ((writeSlow || createSlow) && ctx->compressionLevel < (unsigned)ZSTD_maxCLevel()) {
DEBUG(3, "increasing compression level %u\n", ctx->compressionLevel);
ctx->compressionLevel++;
reset = 1;
}
else if (compressSlow && ctx->compressionLevel > 1) {
DEBUG(3, "decreasing compression level %u\n", ctx->compressionLevel);
ctx->compressionLevel--;
reset = 1;
}
if (reset) {
ctx->stats.readyCounter = 0;
ctx->stats.writeCounter = 0;
ctx->stats.compressedCounter = 0;
}
return ctx->compressionLevel;
} }
else if (compressSlow && ctx->compressionLevel > 1) {
DEBUG(3, "decreasing compression level %u\n", ctx->compressionLevel);
ctx->compressionLevel--;
reset = 1;
}
if (reset) {
ctx->stats.readyCounter = 0;
ctx->stats.writeCounter = 0;
ctx->stats.compressedCounter = 0;
}
return ctx->compressionLevel;
} }
static size_t getUseableDictSize(unsigned compressionLevel) static size_t getUseableDictSize(unsigned compressionLevel)
@ -649,6 +655,9 @@ int main(int argCount, const char* argv[])
forceStdout = 1; forceStdout = 1;
outFilename = stdoutmark; outFilename = stdoutmark;
break; break;
case 'f':
g_forceCompressionLevel = 1;
break;
default: default:
DISPLAY("Error: invalid argument provided\n"); DISPLAY("Error: invalid argument provided\n");
ret = 1; ret = 1;

29
contrib/adaptive-compression/test-performance.sh
View File

@ -1,4 +1,4 @@
echo "testing time" echo "testing time -- no limits set"
./datagen -g1GB > tmp ./datagen -g1GB > tmp
time ./adapt -otmp1.zst tmp time ./adapt -otmp1.zst tmp
time zstd -1 -o tmp2.zst tmp time zstd -1 -o tmp2.zst tmp
@ -14,7 +14,7 @@ time ./adapt -otmp1.zst tmp
time zstd -1 -o tmp2.zst tmp time zstd -1 -o tmp2.zst tmp
rm tmp* rm tmp*
echo -e "\ntesting compression ratio" echo -e "\ntesting compression ratio -- no limits set"
./datagen -g1GB > tmp ./datagen -g1GB > tmp
time ./adapt -otmp1.zst tmp time ./adapt -otmp1.zst tmp
time zstd -1 -o tmp2.zst tmp time zstd -1 -o tmp2.zst tmp
@ -32,3 +32,28 @@ time ./adapt -otmp1.zst tmp
time zstd -1 -o tmp2.zst tmp time zstd -1 -o tmp2.zst tmp
ls -l tmp1.zst tmp2.zst ls -l tmp1.zst tmp2.zst
rm tmp* rm tmp*
echo e "\ntesting performance at various compression levels -- no limits set"
./datagen -g1GB > tmp
echo "adapt"
time ./adapt -i5 -f tmp -otmp1.zst
echo "zstdcli"
time zstd -5 tmp -o tmp2.zst
ls -l tmp1.zst tmp2.zst
rm tmp*
./datagen -g1GB > tmp
echo "adapt"
time ./adapt -i10 -f tmp -otmp1.zst
echo "zstdcli"
time zstd -10 tmp -o tmp2.zst
ls -l tmp1.zst tmp2.zst
rm tmp*
./datagen -g1GB > tmp
echo "adapt"
time ./adapt -i15 -f tmp -otmp1.zst
echo "zstdcli"
time zstd -15 tmp -o tmp2.zst
ls -l tmp1.zst tmp2.zst
rm tmp*