diff --git a/examples/.gitignore b/examples/.gitignore index 280feb36..c92c6989 100644 --- a/examples/.gitignore +++ b/examples/.gitignore @@ -1,6 +1,7 @@ #build simple_compression simple_decompression +simple_compressionCCtx dictionary_compression dictionary_decompression streaming_compression diff --git a/examples/Makefile b/examples/Makefile index 96af41b5..7bcbd120 100644 --- a/examples/Makefile +++ b/examples/Makefile @@ -17,6 +17,7 @@ LIB = ../lib/libzstd.a default: all all: simple_compression simple_decompression \ + simple_compressionCCtx\ dictionary_compression dictionary_decompression \ streaming_compression streaming_decompression \ multiple_streaming_compression streaming_memory_usage @@ -30,6 +31,9 @@ simple_compression : simple_compression.c $(LIB) simple_decompression : simple_decompression.c $(LIB) $(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@ +simple_compressionCCtx : simple_compressionCCtx.c $(LIB) + $(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@ + dictionary_compression : dictionary_compression.c $(LIB) $(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@ @@ -51,6 +55,7 @@ streaming_memory_usage : streaming_memory_usage.c $(LIB) clean: @rm -f core *.o tmp* result* *.zst \ simple_compression simple_decompression \ + simple_compressionCCtx \ dictionary_compression dictionary_decompression \ streaming_compression streaming_decompression \ multiple_streaming_compression streaming_memory_usage @@ -62,6 +67,7 @@ test: all @echo -- Simple compression tests ./simple_compression tmp ./simple_decompression tmp.zst + ./simple_compressionCCtx *.c ./streaming_decompression tmp.zst > /dev/null @echo -- Streaming memory usage ./streaming_memory_usage diff --git a/examples/dictionary_compression.c b/examples/dictionary_compression.c index 511b3567..2a3d6156 100644 --- a/examples/dictionary_compression.c +++ b/examples/dictionary_compression.c @@ -21,7 +21,7 @@ static ZSTD_CDict* createCDict_orDie(const char* dictFileName, int cLevel) { size_t dictSize; printf("loading dictionary %s \n", dictFileName); - void* const dictBuffer = loadFile_orDie(dictFileName, &dictSize); + void* const dictBuffer = loadFile_orDie(dictFileName, &dictSize, 0, 0); ZSTD_CDict* const cdict = ZSTD_createCDict(dictBuffer, dictSize, cLevel); if (!cdict) { fprintf(stderr, "ZSTD_createCDict error \n"); @@ -35,7 +35,7 @@ static ZSTD_CDict* createCDict_orDie(const char* dictFileName, int cLevel) static void compress(const char* fname, const char* oname, const ZSTD_CDict* cdict) { size_t fSize; - void* const fBuff = loadFile_orDie(fname, &fSize); + void* const fBuff = loadFile_orDie(fname, &fSize, 0, 0); size_t const cBuffSize = ZSTD_compressBound(fSize); void* const cBuff = malloc_orDie(cBuffSize); diff --git a/examples/dictionary_decompression.c b/examples/dictionary_decompression.c index 69f56d56..36d20c1b 100644 --- a/examples/dictionary_decompression.c +++ b/examples/dictionary_decompression.c @@ -25,7 +25,7 @@ static ZSTD_DDict* createDict_orDie(const char* dictFileName) { size_t dictSize; printf("loading dictionary %s \n", dictFileName); - void* const dictBuffer = loadFile_orDie(dictFileName, &dictSize); + void* const dictBuffer = loadFile_orDie(dictFileName, &dictSize, 0, 0); ZSTD_DDict* const ddict = ZSTD_createDDict(dictBuffer, dictSize); if (ddict==NULL) { fprintf(stderr, "ZSTD_createDDict error \n"); exit(5); } free(dictBuffer); @@ -35,7 +35,7 @@ static ZSTD_DDict* createDict_orDie(const char* dictFileName) static void decompress(const char* fname, const ZSTD_DDict* ddict) { size_t cSize; - void* const cBuff = loadFile_orDie(fname, &cSize); + void* const cBuff = loadFile_orDie(fname, &cSize, 0, 0); unsigned long long const rSize = ZSTD_findDecompressedSize(cBuff, cSize); if (rSize==ZSTD_CONTENTSIZE_ERROR) { fprintf(stderr, "%s : it was not compressed by zstd.\n", fname); diff --git a/examples/simple_compression.c b/examples/simple_compression.c index 0193dd40..7f1fa6cc 100644 --- a/examples/simple_compression.c +++ b/examples/simple_compression.c @@ -19,7 +19,7 @@ static void compress_orDie(const char* fname, const char* oname) { size_t fSize; - void* const fBuff = loadFile_orDie(fname, &fSize); + void* const fBuff = loadFile_orDie(fname, &fSize, 0, 0); size_t const cBuffSize = ZSTD_compressBound(fSize); void* const cBuff = malloc_orDie(cBuffSize); diff --git a/examples/simple_compressionCCtx.c b/examples/simple_compressionCCtx.c index 17255cba..a5447f06 100644 --- a/examples/simple_compressionCCtx.c +++ b/examples/simple_compressionCCtx.c @@ -16,14 +16,14 @@ #include // presumes zstd library is installed #include "utils.h" -static void compress_orDie(const char* fname, const char* oname) +/* compress with pre-allocated context (ZSTD_CCtx) and input/output buffers*/ +static void compressExpress_orDie(const char* fname, const char* oname, + ZSTD_CCtx* cctx, void* cBuff, size_t cBuffSize, void* fBuff, size_t fBuffSize) { size_t fSize; - void* const fBuff = loadFile_orDie(fname, &fSize); - size_t const cBuffSize = ZSTD_compressBound(fSize); - void* const cBuff = malloc_orDie(cBuffSize); + loadFile_orDie(fname, &fSize, fBuff, fBuffSize); - size_t const cSize = ZSTD_compress(cBuff, cBuffSize, fBuff, fSize, 1); + size_t const cSize = ZSTD_compressCCtx(cctx, cBuff, cBuffSize, fBuff, fBuffSize, 1); if (ZSTD_isError(cSize)) { fprintf(stderr, "error compressing %s : %s \n", fname, ZSTD_getErrorName(cSize)); exit(8); @@ -33,48 +33,65 @@ static void compress_orDie(const char* fname, const char* oname) /* success */ printf("%25s : %6u -> %7u - %s \n", fname, (unsigned)fSize, (unsigned)cSize, oname); - - free(fBuff); - free(cBuff); } -static char* createOutFilename_orDie(const char* filename) +static void getOutFilename(const char* const filename, char* const outFilename) { - size_t const inL = strlen(filename); - size_t const outL = inL + 5; - void* const outSpace = malloc_orDie(outL); - memset(outSpace, 0, outL); - strcat(outSpace, filename); - strcat(outSpace, ".zst"); - return (char*)outSpace; + memset(outFilename, 0, 1); + strcat(outFilename, filename); + strcat(outFilename, ".zst"); } int main(int argc, const char** argv) { const char* const exeName = argv[0]; - if (argc!=2) { + if (argc<2) { printf("wrong arguments\n"); printf("usage:\n"); - printf("%s FILE\n", exeName); + printf("%s FILE(s)\n", exeName); return 1; } - const char* const inFilename = argv[1]; + /* pre-calculate buffer sizes needed to handle all files */ + size_t maxFileNameLength=0; + size_t maxFileSize = 0; + size_t maxCBufferSize = 0; - /** copied code - ZSTD_CCtx* const cctx = ZSTD_createCCtx(); - if (cctx==NULL) { fprintf(stderr, "ZSTD_createCCtx() error \n"); exit(10); } - size_t const cSize = ZSTD_compress_usingCDict(cctx, cBuff, cBuffSize, fBuff, fSize, cdict); - if (ZSTD_isError(cSize)) { - fprintf(stderr, "error compressing %s : %s \n", fname, ZSTD_getErrorName(cSize)); - exit(7); + int argNb; + for (argNb = 1; argNb < argc; argNb++) { + const char* const fileName = argv[argNb]; + size_t const fileNameLength = strlen(fileName); + size_t const fileSize = fsize_orDie(fileName); + + if (fileNameLength > maxFileNameLength) maxFileNameLength = fileNameLength; + if (fileSize > maxFileSize) maxFileSize = fileSize; } - **/ + maxCBufferSize = ZSTD_compressBound(maxFileSize); + /* allocate memory for output file name, input/output buffers for all compression tasks */ + char* const outFilename = (char*)malloc_orDie(maxFileNameLength + 5); + void* const fBuffer = malloc_orDie(maxFileSize); + void* const cBuffer = malloc_orDie(maxCBufferSize); - char* const outFilename = createOutFilename_orDie(inFilename); - compress_orDie(inFilename, outFilename); + /* create a compression context (ZSTD_CCtx) for all compression tasks */ + ZSTD_CCtx* const cctx = ZSTD_createCCtx(); + if (cctx==NULL) { fprintf(stderr, "ZSTD_createCCtx() error \n"); exit(10); } + + /* compress files with shared context, input and output buffers */ + for (argNb = 1; argNb < argc; argNb++) { + const char* const inFilename = argv[argNb]; + getOutFilename(inFilename, outFilename); + compressExpress_orDie(inFilename, outFilename, cctx, cBuffer, maxCBufferSize, fBuffer, maxFileSize); + } + + /* free momery resources */ free(outFilename); + free(fBuffer); + free(cBuffer); + ZSTD_freeCCtx(cctx); /* never fails */ + + printf("compressed %i files \n", argc-1); + return 0; } diff --git a/examples/simple_decompression.c b/examples/simple_decompression.c index ee055dd6..00e02095 100644 --- a/examples/simple_decompression.c +++ b/examples/simple_decompression.c @@ -20,7 +20,7 @@ static void decompress(const char* fname) { size_t cSize; - void* const cBuff = loadFile_orDie(fname, &cSize); + void* const cBuff = loadFile_orDie(fname, &cSize, 0, 0); unsigned long long const rSize = ZSTD_findDecompressedSize(cBuff, cSize); if (rSize==ZSTD_CONTENTSIZE_ERROR) { fprintf(stderr, "%s : it was not compressed by zstd.\n", fname); diff --git a/examples/utils.h b/examples/utils.h index 55a329e2..506cc1ab 100644 --- a/examples/utils.h +++ b/examples/utils.h @@ -9,7 +9,7 @@ */ /* - * This header file has common utility functions used in examples. + * This header file has common utility functions used in examples. */ #ifndef UTILS_H #define UTILS_H @@ -21,7 +21,7 @@ #include // stat /* - * Define the returned error code from utility functions. + * Define the returned error code from utility functions. */ typedef enum { ERROR_fsize = 1, @@ -35,21 +35,33 @@ typedef enum { ERROR_largeFile = 9, } UTILS_ErrorCode; -/*! fsize_orDie() : +/*! fsize_orDie() : * Get the size of a given file path. - * + * * @return The size of a given file path. */ -static off_t fsize_orDie(const char *filename) +static size_t fsize_orDie(const char *filename) { struct stat st; - if (stat(filename, &st) == 0) return st.st_size; - /* error */ - perror(filename); - exit(ERROR_fsize); + if (stat(filename, &st) != 0) { + /* error */ + perror(filename); + exit(ERROR_fsize); + } + + off_t const fileSize = st.st_size; + size_t const size = (size_t)fileSize; + /* if off_t -> size_t conversion causes discrepancy, the file size is + * too big for at least 1 type to handle + */ + if (size != fileSize) { /* narrowcast overflow */ + fprintf(stderr, "%s : filesize too large \n", filename); + exit(ERROR_largeFile); + } + return size; } -/*! fopen_orDie() : +/*! fopen_orDie() : * Open a file using given file path and open option. * * @return If successful this function will return a FILE pointer to an @@ -64,7 +76,7 @@ static FILE* fopen_orDie(const char *filename, const char *instruction) exit(ERROR_fopen); } -/*! fclose_orDie() : +/*! fclose_orDie() : * Close an opened file using given FILE pointer. */ static void fclose_orDie(FILE* file) @@ -75,11 +87,11 @@ static void fclose_orDie(FILE* file) exit(ERROR_fclose); } -/*! fread_orDie() : - * +/*! fread_orDie() : + * * Read sizeToRead bytes from a given file, storing them at the * location given by buffer. - * + * * @return The number of bytes read. */ static size_t fread_orDie(void* buffer, size_t sizeToRead, FILE* file) @@ -93,12 +105,12 @@ static size_t fread_orDie(void* buffer, size_t sizeToRead, FILE* file) } /*! fwrite_orDie() : - * + * * Write sizeToWrite bytes to a file pointed to by file, obtaining * them from a location given by buffer. * * Note: This function will send an error to stderr and exit if it - * cannot write data to the given file pointer. + * cannot write data to the given file pointer. * * @return The number of bytes written. */ @@ -113,7 +125,7 @@ static size_t fwrite_orDie(const void* buffer, size_t sizeToWrite, FILE* file) /*! malloc_orDie() : * Allocate memory. - * + * * @return If successful this function returns a pointer to allo- * cated memory. If there is an error, this function will send that * error to stderr and exit. @@ -128,39 +140,41 @@ static void* malloc_orDie(size_t size) } /*! loadFile_orDie() : - * Read size bytes from a file. - * + * Read size bytes from a file. If buffer is not provided (i.e., buffer == null), + * malloc will be called to allocate one. + * * Note: This function will send an error to stderr and exit if it * cannot read data from the given file path. - * + * * @return If successful this function will return a pointer to read * data otherwise it will printout an error to stderr and exit. */ -static void* loadFile_orDie(const char* fileName, size_t* size) +static void* loadFile_orDie(const char* fileName, size_t* size, void* buffer, int bufferSize) { - off_t const fileSize = fsize_orDie(fileName); - size_t const buffSize = (size_t)fileSize; - if ((off_t)buffSize < fileSize) { /* narrowcast overflow */ - fprintf(stderr, "%s : filesize too large \n", fileName); - exit(ERROR_largeFile); - } + size_t const fileSize = fsize_orDie(fileName); FILE* const inFile = fopen_orDie(fileName, "rb"); - void* const buffer = malloc_orDie(buffSize); - size_t const readSize = fread(buffer, 1, buffSize, inFile); - if (readSize != (size_t)buffSize) { + if (!buffer) { + buffer = malloc_orDie(fileSize); + } + else if (bufferSize < fileSize) { + fprintf(stderr, "%s : filesize bigger than provided buffer.\n", fileName); + + } + size_t const readSize = fread(buffer, 1, fileSize, inFile); + if (readSize != (size_t)fileSize) { fprintf(stderr, "fread: %s : %s \n", fileName, strerror(errno)); exit(ERROR_fread); } fclose(inFile); /* can't fail, read only */ - *size = buffSize; + *size = fileSize; return buffer; } /*! saveFile_orDie() : - * + * * Save buffSize bytes to a given file path, obtaining them from a location pointed * to by buff. - * + * * Note: This function will send an error to stderr and exit if it * cannot write to a given file. */