facebook/zstd
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #2813 from marxin/streaming-compress-enhance

Browse Source 
Enhance streaming_compression examples.
This commit is contained in:
sen 2021-10-12 13:05:54 -04:00 committed by GitHub
commit 66e554fdad
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 26 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
examples/streaming_compression.c
View File

@ -15,9 +15,12 @@
#include <zstd.h> // presumes zstd library is installed #include <zstd.h> // presumes zstd library is installed
#include "common.h" // Helper functions, CHECK(), and CHECK_ZSTD() #include "common.h" // Helper functions, CHECK(), and CHECK_ZSTD()
static void compressFile_orDie(const char* fname, const char* outName, int cLevel,
static void compressFile_orDie(const char* fname, const char* outName, int cLevel) int nbThreads)
{ {
fprintf (stderr, "Starting compression of %s with level %d, using %d threads\n",
fname, cLevel, nbThreads);
/* Open the input and output files. */ /* Open the input and output files. */
FILE* const fin = fopen_orDie(fname, "rb"); FILE* const fin = fopen_orDie(fname, "rb");
FILE* const fout = fopen_orDie(outName, "wb"); FILE* const fout = fopen_orDie(outName, "wb");
@ -39,7 +42,7 @@ static void compressFile_orDie(const char* fname, const char* outName, int cLeve
*/ */
CHECK_ZSTD( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, cLevel) ); CHECK_ZSTD( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, cLevel) );
CHECK_ZSTD( ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1) ); CHECK_ZSTD( ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1) );
ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 4); ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, nbThreads);
/* This loop read from the input file, compresses that entire chunk, /* This loop read from the input file, compresses that entire chunk,
* and writes all output produced to the output file. * and writes all output produced to the output file.
@ -106,17 +109,30 @@ int main(int argc, const char** argv)
{ {
const char* const exeName = argv[0]; const char* const exeName = argv[0];
if (argc!=2) { if (argc < 2) {
printf("wrong arguments\n"); printf("wrong arguments\n");
printf("usage:\n"); printf("usage:\n");
printf("%s FILE\n", exeName); printf("%s FILE [LEVEL] [THREADS]\n", exeName);
return 1; return 1;
} }
int cLevel = 1;
int nbThreads = 4;
if (argc >= 3) {
cLevel = atoi (argv[2]);
CHECK(cLevel != 0, "can't parse LEVEL!");
}
if (argc >= 4) {
nbThreads = atoi (argv[3]);
CHECK(nbThreads != 0, "can't parse THREADS!");
}
const char* const inFilename = argv[1]; const char* const inFilename = argv[1];
char* const outFilename = createOutFilename_orDie(inFilename); char* const outFilename = createOutFilename_orDie(inFilename);
compressFile_orDie(inFilename, outFilename, 1); compressFile_orDie(inFilename, outFilename, cLevel, nbThreads);
free(outFilename); /* not strictly required, since program execution stops there, free(outFilename); /* not strictly required, since program execution stops there,
* but some static analyzer main complain otherwise */ * but some static analyzer main complain otherwise */

6
examples/streaming_compression_thread_pool.c
View File

@ -28,8 +28,10 @@ typedef struct compress_args
static void *compressFile_orDie(void *data) static void *compressFile_orDie(void *data)
{ {
const int nbThreads = 16;
compress_args_t *args = (compress_args_t *)data; compress_args_t *args = (compress_args_t *)data;
fprintf (stderr, "Starting compression of %s with level %d\n", args->fname, args->cLevel); fprintf (stderr, "Starting compression of %s with level %d, using %d threads\n", args->fname, args->cLevel, nbThreads);
/* Open the input and output files. */ /* Open the input and output files. */
FILE* const fin = fopen_orDie(args->fname, "rb"); FILE* const fin = fopen_orDie(args->fname, "rb");
FILE* const fout = fopen_orDie(args->outName, "wb"); FILE* const fout = fopen_orDie(args->outName, "wb");
@ -56,7 +58,7 @@ static void *compressFile_orDie(void *data)
*/ */
CHECK_ZSTD( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, args->cLevel) ); CHECK_ZSTD( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, args->cLevel) );
CHECK_ZSTD( ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1) ); CHECK_ZSTD( ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1) );
ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 16); ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, nbThreads);
/* This loop read from the input file, compresses that entire chunk, /* This loop read from the input file, compresses that entire chunk,
* and writes all output produced to the output file. * and writes all output produced to the output file.