added streaming_memory_usage example
Yann Collet
parent
cb8b471e8b
commit
9c1908a3cd
|
|
@ -6,6 +6,7 @@ dictionary_decompression
|
|||
streaming_compression
|
||||
streaming_decompression
|
||||
multiple_streaming_compression
|
||||
streaming_memory_usage
|
||||
|
||||
#test artefact
|
||||
tmp*
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@
|
|||
|
||||
# This Makefile presumes libzstd is installed, using `sudo make install`
|
||||
|
||||
LDFLAGS += -lzstd
|
||||
LIB = ../lib/libzstd.a
|
||||
|
||||
.PHONY: default all clean test
|
||||
|
||||
|
|
@ -18,27 +18,33 @@ default: all
|
|||
all: simple_compression simple_decompression \
|
||||
dictionary_compression dictionary_decompression \
|
||||
streaming_compression streaming_decompression \
|
||||
multiple_streaming_compression
|
||||
multiple_streaming_compression streaming_memory_usage
|
||||
|
||||
simple_compression : simple_compression.c
|
||||
$(LIB) :
|
||||
make -C ../lib libzstd.a
|
||||
|
||||
simple_compression : simple_compression.c $(LIB)
|
||||
$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
|
||||
|
||||
simple_decompression : simple_decompression.c
|
||||
simple_decompression : simple_decompression.c $(LIB)
|
||||
$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
|
||||
|
||||
dictionary_compression : dictionary_compression.c
|
||||
dictionary_compression : dictionary_compression.c $(LIB)
|
||||
$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
|
||||
|
||||
dictionary_decompression : dictionary_decompression.c
|
||||
dictionary_decompression : dictionary_decompression.c $(LIB)
|
||||
$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
|
||||
|
||||
streaming_compression : streaming_compression.c
|
||||
streaming_compression : streaming_compression.c $(LIB)
|
||||
$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
|
||||
|
||||
multiple_streaming_compression : multiple_streaming_compression.c
|
||||
multiple_streaming_compression : multiple_streaming_compression.c $(LIB)
|
||||
$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
|
||||
|
||||
streaming_decompression : streaming_decompression.c
|
||||
streaming_decompression : streaming_decompression.c $(LIB)
|
||||
$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
|
||||
|
||||
streaming_memory_usage : streaming_memory_usage.c $(LIB)
|
||||
$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
|
||||
|
||||
clean:
|
||||
|
|
@ -46,7 +52,7 @@ clean:
|
|||
simple_compression simple_decompression \
|
||||
dictionary_compression dictionary_decompression \
|
||||
streaming_compression streaming_decompression \
|
||||
multiple_streaming_compression
|
||||
multiple_streaming_compression streaming_memory_usage
|
||||
@echo Cleaning completed
|
||||
|
||||
test: all
|
||||
|
|
@ -56,6 +62,8 @@ test: all
|
|||
./simple_compression tmp
|
||||
./simple_decompression tmp.zst
|
||||
./streaming_decompression tmp.zst > /dev/null
|
||||
@echo -- Streaming memory usage
|
||||
./streaming_memory_usage
|
||||
@echo -- Streaming compression tests
|
||||
./streaming_compression tmp
|
||||
./streaming_decompression tmp.zst > /dev/null
|
||||
|
|
|
|||
|
|
@ -11,6 +11,10 @@ Zstandard library : usage examples
|
|||
Result remains in memory.
|
||||
Introduces usage of : `ZSTD_decompress()`
|
||||
|
||||
- [Streaming memory usage](streaming_memory_usage.c) :
|
||||
Provides amount of memory used by streaming context
|
||||
Introduces usage of : `ZSTD_sizeof_CStream()`
|
||||
|
||||
- [Streaming compression](streaming_compression.c) :
|
||||
Compress a single file.
|
||||
Introduces usage of : `ZSTD_compressStream()`
|
||||
|
|
|
|||
|
|
@ -0,0 +1,149 @@
|
|||
/*
|
||||
* Copyright (c) 2017-present, Yann Collet, Facebook, Inc.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
* in the COPYING file in the root directory of this source tree).
|
||||
* You may select, at your option, one of the above-listed licenses.
|
||||
*/
|
||||
|
||||
|
||||
/*=== Tuning parameter ===*/
|
||||
#ifndef MAX_TESTED_LEVEL
|
||||
#define MAX_TESTED_LEVEL 12
|
||||
#endif
|
||||
|
||||
|
||||
/*=== Dependencies ===*/
|
||||
#include <stdio.h> /* printf */
|
||||
#define ZSTD_STATIC_LINKING_ONLY
|
||||
#include "zstd.h"
|
||||
|
||||
|
||||
/*=== functions ===*/
|
||||
|
||||
/*! readU32FromChar() :
|
||||
@return : unsigned integer value read from input in `char` format
|
||||
allows and interprets K, KB, KiB, M, MB and MiB suffix.
|
||||
Will also modify `*stringPtr`, advancing it to position where it stopped reading.
|
||||
Note : function result can overflow if digit string > MAX_UINT */
|
||||
static unsigned readU32FromChar(const char** stringPtr)
|
||||
{
|
||||
unsigned result = 0;
|
||||
while ((**stringPtr >='0') && (**stringPtr <='9'))
|
||||
result *= 10, result += **stringPtr - '0', (*stringPtr)++ ;
|
||||
if ((**stringPtr=='K') || (**stringPtr=='M')) {
|
||||
result <<= 10;
|
||||
if (**stringPtr=='M') result <<= 10;
|
||||
(*stringPtr)++ ;
|
||||
if (**stringPtr=='i') (*stringPtr)++;
|
||||
if (**stringPtr=='B') (*stringPtr)++;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
|
||||
int main(int argc, char const *argv[]) {
|
||||
|
||||
printf("\n Zstandard (v%u) memory usage for streaming contexts : \n\n", ZSTD_versionNumber());
|
||||
|
||||
unsigned wLog = 0;
|
||||
if (argc > 1) {
|
||||
const char* valStr = argv[1];
|
||||
wLog = readU32FromChar(&valStr);
|
||||
}
|
||||
|
||||
int compressionLevel;
|
||||
for (compressionLevel = 1; compressionLevel <= MAX_TESTED_LEVEL; compressionLevel++) {
|
||||
#define INPUT_SIZE 5
|
||||
#define COMPRESSED_SIZE 128
|
||||
char const dataToCompress[INPUT_SIZE] = "abcde";
|
||||
char compressedData[COMPRESSED_SIZE];
|
||||
char decompressedData[INPUT_SIZE];
|
||||
ZSTD_CStream* const cstream = ZSTD_createCStream();
|
||||
if (cstream==NULL) {
|
||||
printf("Level %i : ZSTD_CStream Memory allocation failure \n", compressionLevel);
|
||||
return 1;
|
||||
}
|
||||
|
||||
/* forces compressor to use maximum memory size for given compression level,
|
||||
* by not providing any information on input size */
|
||||
ZSTD_parameters params = ZSTD_getParams(compressionLevel, 0, 0);
|
||||
if (wLog) { /* special mode : specific wLog */
|
||||
printf("Using custom compression parameter : level 1 + wLog=%u \n", wLog);
|
||||
params = ZSTD_getParams(1, 1 << wLog, 0);
|
||||
size_t const error = ZSTD_initCStream_advanced(cstream, NULL, 0, params, 0);
|
||||
if (ZSTD_isError(error)) {
|
||||
printf("ZSTD_initCStream_advanced error : %s \n", ZSTD_getErrorName(error));
|
||||
return 1;
|
||||
}
|
||||
} else {
|
||||
size_t const error = ZSTD_initCStream(cstream, compressionLevel);
|
||||
if (ZSTD_isError(error)) {
|
||||
printf("ZSTD_initCStream error : %s \n", ZSTD_getErrorName(error));
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
|
||||
size_t compressedSize;
|
||||
{ ZSTD_inBuffer inBuff = { dataToCompress, sizeof(dataToCompress), 0 };
|
||||
ZSTD_outBuffer outBuff = { compressedData, sizeof(compressedData), 0 };
|
||||
size_t const cError = ZSTD_compressStream(cstream, &outBuff, &inBuff);
|
||||
if (ZSTD_isError(cError)) {
|
||||
printf("ZSTD_compressStream error : %s \n", ZSTD_getErrorName(cError));
|
||||
return 1;
|
||||
}
|
||||
size_t const fError = ZSTD_endStream(cstream, &outBuff);
|
||||
if (ZSTD_isError(fError)) {
|
||||
printf("ZSTD_endStream error : %s \n", ZSTD_getErrorName(fError));
|
||||
return 1;
|
||||
}
|
||||
compressedSize = outBuff.pos;
|
||||
}
|
||||
|
||||
ZSTD_DStream* dstream = ZSTD_createDStream();
|
||||
if (dstream==NULL) {
|
||||
printf("Level %i : ZSTD_DStream Memory allocation failure \n", compressionLevel);
|
||||
return 1;
|
||||
}
|
||||
{ size_t const error = ZSTD_initDStream(dstream);
|
||||
if (ZSTD_isError(error)) {
|
||||
printf("ZSTD_initDStream error : %s \n", ZSTD_getErrorName(error));
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
/* forces decompressor to use maximum memory size, as decompressed size is not known */
|
||||
{ ZSTD_inBuffer inBuff = { compressedData, compressedSize, 0 };
|
||||
ZSTD_outBuffer outBuff = { decompressedData, sizeof(decompressedData), 0 };
|
||||
size_t const dResult = ZSTD_decompressStream(dstream, &outBuff, &inBuff);
|
||||
if (ZSTD_isError(dResult)) {
|
||||
printf("ZSTD_decompressStream error : %s \n", ZSTD_getErrorName(dResult));
|
||||
return 1;
|
||||
}
|
||||
if (dResult != 0) {
|
||||
printf("ZSTD_decompressStream error : unfinished decompression \n");
|
||||
return 1;
|
||||
}
|
||||
if (outBuff.pos != sizeof(dataToCompress)) {
|
||||
printf("ZSTD_decompressStream error : incorrect decompression \n");
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
|
||||
size_t const cstreamSize = ZSTD_sizeof_CStream(cstream);
|
||||
size_t const cstreamEstimatedSize = wLog ?
|
||||
ZSTD_estimateCStreamSize_advanced_usingCParams(params.cParams) :
|
||||
ZSTD_estimateCStreamSize(compressionLevel);
|
||||
size_t const dstreamSize = ZSTD_sizeof_DStream(dstream);
|
||||
|
||||
printf("Level %2i : Compression Mem = %5u KB (estimated : %5u KB) ; Decompression Mem = %4u KB \n",
|
||||
compressionLevel,
|
||||
(unsigned)(cstreamSize>>10), (unsigned)(cstreamEstimatedSize>>10), (unsigned)(dstreamSize>>10));
|
||||
|
||||
ZSTD_freeDStream(dstream);
|
||||
ZSTD_freeCStream(cstream);
|
||||
if (wLog) break; /* single test */
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
Loading…
Reference in New Issue