Merge pull request #1574 from terrelln/examples

Stabilize ZSTD_getDictID_*() functions and clean up examples
dev
Nick Terrell 2019-04-05 23:29:32 -07:00 committed by GitHub
commit d5910a5d94
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 173 additions and 169 deletions

View File

@ -25,28 +25,28 @@ all: simple_compression simple_decompression \
$(LIB) : $(LIB) :
$(MAKE) -C ../lib libzstd.a $(MAKE) -C ../lib libzstd.a
simple_compression : simple_compression.c utils.h $(LIB) simple_compression : simple_compression.c common.h $(LIB)
$(CC) $(CPPFLAGS) $(CFLAGS) $< $(LIB) $(LDFLAGS) -o $@ $(CC) $(CPPFLAGS) $(CFLAGS) $< $(LIB) $(LDFLAGS) -o $@
simple_decompression : simple_decompression.c utils.h $(LIB) simple_decompression : simple_decompression.c common.h $(LIB)
$(CC) $(CPPFLAGS) $(CFLAGS) $< $(LIB) $(LDFLAGS) -o $@ $(CC) $(CPPFLAGS) $(CFLAGS) $< $(LIB) $(LDFLAGS) -o $@
multiple_simple_compression : multiple_simple_compression.c utils.h $(LIB) multiple_simple_compression : multiple_simple_compression.c common.h $(LIB)
$(CC) $(CPPFLAGS) $(CFLAGS) $< $(LIB) $(LDFLAGS) -o $@ $(CC) $(CPPFLAGS) $(CFLAGS) $< $(LIB) $(LDFLAGS) -o $@
dictionary_compression : dictionary_compression.c utils.h $(LIB) dictionary_compression : dictionary_compression.c common.h $(LIB)
$(CC) $(CPPFLAGS) $(CFLAGS) $< $(LIB) $(LDFLAGS) -o $@ $(CC) $(CPPFLAGS) $(CFLAGS) $< $(LIB) $(LDFLAGS) -o $@
dictionary_decompression : dictionary_decompression.c utils.h $(LIB) dictionary_decompression : dictionary_decompression.c common.h $(LIB)
$(CC) $(CPPFLAGS) $(CFLAGS) $< $(LIB) $(LDFLAGS) -o $@ $(CC) $(CPPFLAGS) $(CFLAGS) $< $(LIB) $(LDFLAGS) -o $@
streaming_compression : streaming_compression.c utils.h $(LIB) streaming_compression : streaming_compression.c common.h $(LIB)
$(CC) $(CPPFLAGS) $(CFLAGS) $< $(LIB) $(LDFLAGS) -o $@ $(CC) $(CPPFLAGS) $(CFLAGS) $< $(LIB) $(LDFLAGS) -o $@
multiple_streaming_compression : multiple_streaming_compression.c utils.h $(LIB) multiple_streaming_compression : multiple_streaming_compression.c common.h $(LIB)
$(CC) $(CPPFLAGS) $(CFLAGS) $< $(LIB) $(LDFLAGS) -o $@ $(CC) $(CPPFLAGS) $(CFLAGS) $< $(LIB) $(LDFLAGS) -o $@
streaming_decompression : streaming_decompression.c utils.h $(LIB) streaming_decompression : streaming_decompression.c common.h $(LIB)
$(CC) $(CPPFLAGS) $(CFLAGS) $< $(LIB) $(LDFLAGS) -o $@ $(CC) $(CPPFLAGS) $(CFLAGS) $< $(LIB) $(LDFLAGS) -o $@
streaming_memory_usage : streaming_memory_usage.c $(LIB) streaming_memory_usage : streaming_memory_usage.c $(LIB)

View File

@ -11,14 +11,13 @@
/* /*
* This header file has common utility functions used in examples. * This header file has common utility functions used in examples.
*/ */
#ifndef UTILS_H #ifndef COMMON_H
#define UTILS_H #define COMMON_H
#include <stdlib.h> // malloc, free, exit #include <stdlib.h> // malloc, free, exit
#include <stdio.h> // fprintf, perror, fopen, etc. #include <stdio.h> // fprintf, perror, fopen, etc.
#include <string.h> // strlen, strcat, memset, strerror #include <string.h> // strerror
#include <errno.h> // errno #include <errno.h> // errno
#include <assert.h> // assert
#include <sys/stat.h> // stat #include <sys/stat.h> // stat
#include <zstd.h> #include <zstd.h>
@ -35,7 +34,34 @@ typedef enum {
ERROR_saveFile = 7, ERROR_saveFile = 7,
ERROR_malloc = 8, ERROR_malloc = 8,
ERROR_largeFile = 9, ERROR_largeFile = 9,
} UTILS_ErrorCode; } COMMON_ErrorCode;
/*! CHECK
* Check that the condition holds. If it doesn't print a message and die.
*/
#define CHECK(cond, ...) \
do { \
if (!(cond)) { \
fprintf(stderr, \
"%s:%d CHECK(%s) failed: ", \
__FILE__, \
__LINE__, \
#cond); \
fprintf(stderr, "" __VA_ARGS__); \
fprintf(stderr, "\n"); \
exit(1); \
} \
} while (0)
/*! CHECK_ZSTD
* Check the zstd error code and die if an error occurred after printing a
* message.
*/
#define CHECK_ZSTD(fn, ...) \
do { \
size_t const err = (fn); \
CHECK(!ZSTD_isError(err), "%s", ZSTD_getErrorName(err)); \
} while (0)
/*! fsize_orDie() : /*! fsize_orDie() :
* Get the size of a given file path. * Get the size of a given file path.
@ -154,7 +180,7 @@ static void* malloc_orDie(size_t size)
static size_t loadFile_orDie(const char* fileName, void* buffer, size_t bufferSize) static size_t loadFile_orDie(const char* fileName, void* buffer, size_t bufferSize)
{ {
size_t const fileSize = fsize_orDie(fileName); size_t const fileSize = fsize_orDie(fileName);
assert(fileSize <= bufferSize); CHECK(fileSize <= bufferSize, "File too large!");
FILE* const inFile = fopen_orDie(fileName, "rb"); FILE* const inFile = fopen_orDie(fileName, "rb");
size_t const readSize = fread(buffer, 1, fileSize, inFile); size_t const readSize = fread(buffer, 1, fileSize, inFile);
@ -205,31 +231,4 @@ static void saveFile_orDie(const char* fileName, const void* buff, size_t buffSi
} }
} }
/*! CHECK
* Check that the condition holds. If it doesn't print a message and die.
*/
#define CHECK(cond, ...) \
do { \
if (!(cond)) { \
fprintf(stderr, \
"%s:%d CHECK(%s) failed: ", \
__FILE__, \
__LINE__, \
#cond); \
fprintf(stderr, "" __VA_ARGS__); \
fprintf(stderr, "\n"); \
exit(1); \
} \
} while (0)
/*! CHECK_ZSTD
* Check the zstd error code and die if an error occurred after printing a
* message.
*/
#define CHECK_ZSTD(fn, ...) \
do { \
size_t const err = (fn); \
CHECK(!ZSTD_isError(err), "%s", ZSTD_getErrorName(err)); \
} while (0)
#endif #endif

View File

@ -7,13 +7,11 @@
* in the COPYING file in the root directory of this source tree). * in the COPYING file in the root directory of this source tree).
* You may select, at your option, one of the above-listed licenses. * You may select, at your option, one of the above-listed licenses.
*/ */
#include <stdlib.h> // malloc, exit
#include <stdio.h> // printf #include <stdio.h> // printf
#include <string.h> // strerror #include <stdlib.h> // free
#include <errno.h> // errno #include <string.h> // memset, strcat
#include <sys/stat.h> // stat
#include <zstd.h> // presumes zstd library is installed #include <zstd.h> // presumes zstd library is installed
#include "utils.h" #include "common.h" // Helper functions, CHECK(), and CHECK_ZSTD()
/* createDict() : /* createDict() :
`dictFileName` is supposed to have been created using `zstd --train` */ `dictFileName` is supposed to have been created using `zstd --train` */
@ -23,10 +21,7 @@ static ZSTD_CDict* createCDict_orDie(const char* dictFileName, int cLevel)
printf("loading dictionary %s \n", dictFileName); printf("loading dictionary %s \n", dictFileName);
void* const dictBuffer = mallocAndLoadFile_orDie(dictFileName, &dictSize); void* const dictBuffer = mallocAndLoadFile_orDie(dictFileName, &dictSize);
ZSTD_CDict* const cdict = ZSTD_createCDict(dictBuffer, dictSize, cLevel); ZSTD_CDict* const cdict = ZSTD_createCDict(dictBuffer, dictSize, cLevel);
if (!cdict) { CHECK(cdict != NULL, "ZSTD_createCDict() failed!");
fprintf(stderr, "ZSTD_createCDict error \n");
exit(7);
}
free(dictBuffer); free(dictBuffer);
return cdict; return cdict;
} }
@ -39,13 +34,16 @@ static void compress(const char* fname, const char* oname, const ZSTD_CDict* cdi
size_t const cBuffSize = ZSTD_compressBound(fSize); size_t const cBuffSize = ZSTD_compressBound(fSize);
void* const cBuff = malloc_orDie(cBuffSize); void* const cBuff = malloc_orDie(cBuffSize);
/* Compress using the dictionary.
* This function writes the dictionary id, and content size into the header.
* But, it doesn't use a checksum. You can control these options using the
* advanced API: ZSTD_CCtx_setParameter(), ZSTD_CCtx_refCDict(),
* and ZSTD_compress2().
*/
ZSTD_CCtx* const cctx = ZSTD_createCCtx(); ZSTD_CCtx* const cctx = ZSTD_createCCtx();
if (cctx==NULL) { fprintf(stderr, "ZSTD_createCCtx() error \n"); exit(10); } CHECK(cctx != NULL, "ZSTD_createCCtx() failed!");
size_t const cSize = ZSTD_compress_usingCDict(cctx, cBuff, cBuffSize, fBuff, fSize, cdict); size_t const cSize = ZSTD_compress_usingCDict(cctx, cBuff, cBuffSize, fBuff, fSize, cdict);
if (ZSTD_isError(cSize)) { CHECK_ZSTD(cSize);
fprintf(stderr, "error compressing %s : %s \n", fname, ZSTD_getErrorName(cSize));
exit(7);
}
saveFile_orDie(oname, cBuff, cSize); saveFile_orDie(oname, cBuff, cSize);

View File

@ -9,15 +9,10 @@
*/ */
#include <stdlib.h> // malloc, exit
#include <stdio.h> // printf #include <stdio.h> // printf
#include <string.h> // strerror #include <stdlib.h> // free
#include <errno.h> // errno
#include <sys/stat.h> // stat
#define ZSTD_STATIC_LINKING_ONLY // ZSTD_findDecompressedSize
#include <zstd.h> // presumes zstd library is installed #include <zstd.h> // presumes zstd library is installed
#include "utils.h" #include "common.h" // Helper functions, CHECK(), and CHECK_ZSTD()
/* createDict() : /* createDict() :
`dictFileName` is supposed to have been created using `zstd --train` */ `dictFileName` is supposed to have been created using `zstd --train` */
@ -27,7 +22,7 @@ static ZSTD_DDict* createDict_orDie(const char* dictFileName)
printf("loading dictionary %s \n", dictFileName); printf("loading dictionary %s \n", dictFileName);
void* const dictBuffer = mallocAndLoadFile_orDie(dictFileName, &dictSize); void* const dictBuffer = mallocAndLoadFile_orDie(dictFileName, &dictSize);
ZSTD_DDict* const ddict = ZSTD_createDDict(dictBuffer, dictSize); ZSTD_DDict* const ddict = ZSTD_createDDict(dictBuffer, dictSize);
if (ddict==NULL) { fprintf(stderr, "ZSTD_createDDict error \n"); exit(5); } CHECK(ddict != NULL, "ZSTD_createDDict() failed!");
free(dictBuffer); free(dictBuffer);
return ddict; return ddict;
} }
@ -36,24 +31,40 @@ static void decompress(const char* fname, const ZSTD_DDict* ddict)
{ {
size_t cSize; size_t cSize;
void* const cBuff = mallocAndLoadFile_orDie(fname, &cSize); void* const cBuff = mallocAndLoadFile_orDie(fname, &cSize);
unsigned long long const rSize = ZSTD_findDecompressedSize(cBuff, cSize); /* Read the content size from the frame header. For simplicity we require
if (rSize==ZSTD_CONTENTSIZE_ERROR) { * that it is always present. By default, zstd will write the content size
fprintf(stderr, "%s : it was not compressed by zstd.\n", fname); * in the header when it is known. If you can't guarantee that the frame
exit(5); * content size is always written into the header, either use streaming
} else if (rSize==ZSTD_CONTENTSIZE_UNKNOWN) { * decompression, or ZSTD_decompressBound().
fprintf(stderr, "%s : original size unknown \n", fname); */
exit(6); unsigned long long const rSize = ZSTD_getFrameContentSize(cBuff, cSize);
} CHECK(rSize != ZSTD_CONTENTSIZE_ERROR, "%s: not compressed by zstd!", fname);
CHECK(rSize != ZSTD_CONTENTSIZE_UNKNOWN, "%s: original size unknown!", fname);
void* const rBuff = malloc_orDie((size_t)rSize); void* const rBuff = malloc_orDie((size_t)rSize);
/* Check that the dictionary ID matches.
* If a non-zstd dictionary is used, then both will be zero.
* By default zstd always writes the dictionary ID into the frame.
* Zstd will check if there is a dictionary ID mismatch as well.
*/
unsigned const expectedDictID = ZSTD_getDictID_fromDDict(ddict);
unsigned const actualDictID = ZSTD_getDictID_fromFrame(cBuff, cSize);
CHECK(actualDictID == expectedDictID,
"DictID mismatch: expected %u got %u",
expectedDictID,
actualDictID);
/* Decompress using the dictionary.
* If you need to control the decompression parameters, then use the
* advanced API: ZSTD_DCtx_setParameter(), ZSTD_DCtx_refDDict(), and
* ZSTD_decompressDCtx().
*/
ZSTD_DCtx* const dctx = ZSTD_createDCtx(); ZSTD_DCtx* const dctx = ZSTD_createDCtx();
if (dctx==NULL) { fprintf(stderr, "ZSTD_createDCtx() error \n"); exit(10); } CHECK(dctx != NULL, "ZSTD_createDCtx() failed!");
size_t const dSize = ZSTD_decompress_usingDDict(dctx, rBuff, rSize, cBuff, cSize, ddict); size_t const dSize = ZSTD_decompress_usingDDict(dctx, rBuff, rSize, cBuff, cSize, ddict);
if (dSize != rSize) { CHECK_ZSTD(dSize);
fprintf(stderr, "error decoding %s : %s \n", fname, ZSTD_getErrorName(dSize)); /* When zstd knows the content size, it will error if it doesn't match. */
exit(7); CHECK(dSize == rSize, "Impossible because zstd will check this condition!");
}
/* success */ /* success */
printf("%25s : %6u -> %7u \n", fname, (unsigned)cSize, (unsigned)rSize); printf("%25s : %6u -> %7u \n", fname, (unsigned)cSize, (unsigned)rSize);

View File

@ -8,13 +8,11 @@
* You may select, at your option, one of the above-listed licenses. * You may select, at your option, one of the above-listed licenses.
*/ */
#include <stdlib.h> // malloc, free, exit #include <stdio.h> // printf
#include <stdio.h> // fprintf, perror, fopen, etc. #include <stdlib.h> // free
#include <string.h> // strlen, strcat, memset, strerror #include <string.h> // memcpy, strlen
#include <errno.h> // errno
#include <sys/stat.h> // stat
#include <zstd.h> // presumes zstd library is installed #include <zstd.h> // presumes zstd library is installed
#include "utils.h" #include "common.h" // Helper functions, CHECK(), and CHECK_ZSTD()
typedef struct { typedef struct {
void* fBuffer; void* fBuffer;
@ -52,7 +50,7 @@ static resources createResources_orDie(int argc, const char** argv, char **ofn,
ress.fBuffer = malloc_orDie(ress.fBufferSize); ress.fBuffer = malloc_orDie(ress.fBufferSize);
ress.cBuffer = malloc_orDie(ress.cBufferSize); ress.cBuffer = malloc_orDie(ress.cBufferSize);
ress.cctx = ZSTD_createCCtx(); ress.cctx = ZSTD_createCCtx();
if (ress.cctx==NULL) { fprintf(stderr, "ZSTD_createCCtx() error \n"); exit(10); } CHECK(ress.cctx != NULL, "ZSTD_createCCtx() failed!");
return ress; return ress;
} }
@ -69,16 +67,17 @@ static void compressFile_orDie(resources ress, const char* fname, const char* on
{ {
size_t fSize = loadFile_orDie(fname, ress.fBuffer, ress.fBufferSize); size_t fSize = loadFile_orDie(fname, ress.fBuffer, ress.fBufferSize);
/* Compress using the context.
* If you need more control over parameters, use the advanced API:
* ZSTD_CCtx_setParameter(), and ZSTD_compress2().
*/
size_t const cSize = ZSTD_compressCCtx(ress.cctx, ress.cBuffer, ress.cBufferSize, ress.fBuffer, fSize, 1); size_t const cSize = ZSTD_compressCCtx(ress.cctx, ress.cBuffer, ress.cBufferSize, ress.fBuffer, fSize, 1);
if (ZSTD_isError(cSize)) { CHECK_ZSTD(cSize);
fprintf(stderr, "error compressing %s : %s \n", fname, ZSTD_getErrorName(cSize));
exit(8);
}
saveFile_orDie(oname, ress.cBuffer, cSize); saveFile_orDie(oname, ress.cBuffer, cSize);
/* success */ /* success */
// printf("%25s : %6u -> %7u - %s \n", fname, (unsigned)fSize, (unsigned)cSize, oname); printf("%25s : %6u -> %7u - %s \n", fname, (unsigned)fSize, (unsigned)cSize, oname);
} }
int main(int argc, const char** argv) int main(int argc, const char** argv)
@ -102,7 +101,7 @@ int main(int argc, const char** argv)
for (argNb = 1; argNb < argc; argNb++) { for (argNb = 1; argNb < argc; argNb++) {
const char* const inFilename = argv[argNb]; const char* const inFilename = argv[argNb];
size_t const inFilenameLen = strlen(inFilename); size_t const inFilenameLen = strlen(inFilename);
assert(inFilenameLen + 5 <= outFilenameBufferLen); CHECK(inFilenameLen + 5 <= outFilenameBufferLen, "File name too long!");
memcpy(outFilename, inFilename, inFilenameLen); memcpy(outFilename, inFilename, inFilenameLen);
memcpy(outFilename+inFilenameLen, ".zst", 5); memcpy(outFilename+inFilenameLen, ".zst", 5);
compressFile_orDie(ress, inFilename, outFilename); compressFile_orDie(ress, inFilename, outFilename);

View File

@ -14,13 +14,11 @@
* All structures and buffers will be created only once, * All structures and buffers will be created only once,
* and shared across all compression operations */ * and shared across all compression operations */
#include <stdlib.h> // malloc, exit #include <stdio.h> // printf
#include <stdio.h> // fprintf, perror, feof #include <stdlib.h> // free
#include <string.h> // strerror #include <string.h> // memset, strcat
#include <errno.h> // errno
#define ZSTD_STATIC_LINKING_ONLY // TODO: Remove once the API is stable
#include <zstd.h> // presumes zstd library is installed #include <zstd.h> // presumes zstd library is installed
#include "utils.h" #include "common.h" // Helper functions, CHECK(), and CHECK_ZSTD()
typedef struct { typedef struct {
void* buffIn; void* buffIn;
@ -86,7 +84,8 @@ static void compressFile_orDie(resources ress, const char* fname, const char* ou
fwrite_orDie(ress.buffOut, output.pos, fout); fwrite_orDie(ress.buffOut, output.pos, fout);
finished = lastChunk ? (remaining == 0) : (input.pos == input.size); finished = lastChunk ? (remaining == 0) : (input.pos == input.size);
} while (!finished); } while (!finished);
assert(input.pos == input.size); CHECK(input.pos == input.size,
"Impossible: zstd only returns 0 when the input is completely consumed!");
} }
fclose_orDie(fout); fclose_orDie(fout);

View File

@ -8,13 +8,11 @@
* You may select, at your option, one of the above-listed licenses. * You may select, at your option, one of the above-listed licenses.
*/ */
#include <stdlib.h> // malloc, free, exit #include <stdio.h> // printf
#include <stdio.h> // fprintf, perror, fopen, etc. #include <stdlib.h> // free
#include <string.h> // strlen, strcat, memset, strerror #include <string.h> // strlen, strcat, memset
#include <errno.h> // errno
#include <sys/stat.h> // stat
#include <zstd.h> // presumes zstd library is installed #include <zstd.h> // presumes zstd library is installed
#include "utils.h" #include "common.h" // Helper functions, CHECK(), and CHECK_ZSTD()
static void compress_orDie(const char* fname, const char* oname) static void compress_orDie(const char* fname, const char* oname)
{ {
@ -23,11 +21,12 @@ static void compress_orDie(const char* fname, const char* oname)
size_t const cBuffSize = ZSTD_compressBound(fSize); size_t const cBuffSize = ZSTD_compressBound(fSize);
void* const cBuff = malloc_orDie(cBuffSize); void* const cBuff = malloc_orDie(cBuffSize);
/* Compress.
* If you are doing many compressions, you may want to reuse the context.
* See the multiple_simple_compression.c example.
*/
size_t const cSize = ZSTD_compress(cBuff, cBuffSize, fBuff, fSize, 1); size_t const cSize = ZSTD_compress(cBuff, cBuffSize, fBuff, fSize, 1);
if (ZSTD_isError(cSize)) { CHECK_ZSTD(cSize);
fprintf(stderr, "error compressing %s : %s \n", fname, ZSTD_getErrorName(cSize));
exit(8);
}
saveFile_orDie(oname, cBuff, cSize); saveFile_orDie(oname, cBuff, cSize);

View File

@ -8,37 +8,36 @@
* You may select, at your option, one of the above-listed licenses. * You may select, at your option, one of the above-listed licenses.
*/ */
#include <stdlib.h> // malloc, exit
#include <stdio.h> // printf #include <stdio.h> // printf
#include <string.h> // strerror #include <stdlib.h> // free
#include <errno.h> // errno
#include <sys/stat.h> // stat
#define ZSTD_STATIC_LINKING_ONLY // ZSTD_findDecompressedSize
#include <zstd.h> // presumes zstd library is installed #include <zstd.h> // presumes zstd library is installed
#include "utils.h" #include "common.h" // Helper functions, CHECK(), and CHECK_ZSTD()
static void decompress(const char* fname) static void decompress(const char* fname)
{ {
size_t cSize; size_t cSize;
void* const cBuff = mallocAndLoadFile_orDie(fname, &cSize); void* const cBuff = mallocAndLoadFile_orDie(fname, &cSize);
unsigned long long const rSize = ZSTD_findDecompressedSize(cBuff, cSize); /* Read the content size from the frame header. For simplicity we require
if (rSize==ZSTD_CONTENTSIZE_ERROR) { * that it is always present. By default, zstd will write the content size
fprintf(stderr, "%s : it was not compressed by zstd.\n", fname); * in the header when it is known. If you can't guarantee that the frame
exit(5); * content size is always written into the header, either use streaming
} else if (rSize==ZSTD_CONTENTSIZE_UNKNOWN) { * decompression, or ZSTD_decompressBound().
fprintf(stderr, */
"%s : original size unknown. Use streaming decompression instead.\n", fname); unsigned long long const rSize = ZSTD_getFrameContentSize(cBuff, cSize);
exit(6); CHECK(rSize != ZSTD_CONTENTSIZE_ERROR, "%s: not compressed by zstd!", fname);
} CHECK(rSize != ZSTD_CONTENTSIZE_UNKNOWN, "%s: original size unknown!", fname);
void* const rBuff = malloc_orDie((size_t)rSize); void* const rBuff = malloc_orDie((size_t)rSize);
/* Decompress.
* If you are doing many decompressions, you may want to reuse the context
* and use ZSTD_decompressDCtx(). If you want to set advanced parameters,
* use ZSTD_DCtx_setParameter().
*/
size_t const dSize = ZSTD_decompress(rBuff, rSize, cBuff, cSize); size_t const dSize = ZSTD_decompress(rBuff, rSize, cBuff, cSize);
CHECK_ZSTD(dSize);
if (dSize != rSize) { /* When zstd knows the content size, it will error if it doesn't match. */
fprintf(stderr, "error decoding %s : %s \n", fname, ZSTD_getErrorName(dSize)); CHECK(dSize == rSize, "Impossible because zstd will check this condition!");
exit(7);
}
/* success */ /* success */
printf("%25s : %6u -> %7u \n", fname, (unsigned)cSize, (unsigned)rSize); printf("%25s : %6u -> %7u \n", fname, (unsigned)cSize, (unsigned)rSize);

View File

@ -9,12 +9,11 @@
*/ */
#include <stdlib.h> // malloc, free, exit #include <stdio.h> // printf
#include <stdio.h> // fprintf, perror, feof, fopen, etc. #include <stdlib.h> // free
#include <string.h> // strlen, memset, strcat #include <string.h> // memset, strcat, strlen
#define ZSTD_STATIC_LINKING_ONLY // TODO: Remove once the API is stable
#include <zstd.h> // presumes zstd library is installed #include <zstd.h> // presumes zstd library is installed
#include "utils.h" #include "common.h" // Helper functions, CHECK(), and CHECK_ZSTD()
static void compressFile_orDie(const char* fname, const char* outName, int cLevel) static void compressFile_orDie(const char* fname, const char* outName, int cLevel)
@ -75,7 +74,8 @@ static void compressFile_orDie(const char* fname, const char* outName, int cLeve
*/ */
finished = lastChunk ? (remaining == 0) : (input.pos == input.size); finished = lastChunk ? (remaining == 0) : (input.pos == input.size);
} while (!finished); } while (!finished);
assert(input.pos == input.size); CHECK(input.pos == input.size,
"Impossible: zstd only returns 0 when the input is completely consumed!");
} }
ZSTD_freeCCtx(cctx); ZSTD_freeCCtx(cctx);

View File

@ -9,12 +9,10 @@
*/ */
#include <stdlib.h> // malloc, exit #include <stdio.h> // fprintf
#include <stdio.h> // fprintf, perror, feof #include <stdlib.h> // free
#include <string.h> // strerror
#include <errno.h> // errno
#include <zstd.h> // presumes zstd library is installed #include <zstd.h> // presumes zstd library is installed
#include "utils.h" #include "common.h" // Helper functions, CHECK(), and CHECK_ZSTD()
static void decompressFile_orDie(const char* fname) static void decompressFile_orDie(const char* fname)
{ {

View File

@ -16,10 +16,10 @@
/*=== Dependencies ===*/ /*=== Dependencies ===*/
#include <stdio.h> /* printf */ #include <stdio.h> // printf
#define ZSTD_STATIC_LINKING_ONLY #define ZSTD_STATIC_LINKING_ONLY
#include "zstd.h" #include <zstd.h> // presumes zstd library is installed
#include "utils.h" #include "common.h" // Helper functions, CHECK(), and CHECK_ZSTD()
/*=== functions ===*/ /*=== functions ===*/

View File

@ -577,10 +577,8 @@ typedef struct ZSTD_outBuffer_s {
* The caller must check if input has been entirely consumed. * The caller must check if input has been entirely consumed.
* If not, the caller must make some room to receive more compressed data, * If not, the caller must make some room to receive more compressed data,
* and then present again remaining input data. * and then present again remaining input data.
* @return : a size hint, preferred nb of bytes to use as input for next function call * @return : provides a minimum amount of data remaining to be flushed from internal buffers
* or an error code, which can be tested using ZSTD_isError(). * or an error code, which can be tested using ZSTD_isError().
* Note 1 : it's just a hint, to help latency a little, any value will work fine.
* Note 2 : size hint is guaranteed to be <= ZSTD_CStreamInSize()
* *
* At any moment, it's possible to flush whatever data might remain stuck within internal buffer, * At any moment, it's possible to flush whatever data might remain stuck within internal buffer,
* using ZSTD_compressStream2() with ZSTD_e_flush. `output->pos` will be updated. * using ZSTD_compressStream2() with ZSTD_e_flush. `output->pos` will be updated.
@ -798,6 +796,35 @@ ZSTDLIB_API size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx,
const ZSTD_DDict* ddict); const ZSTD_DDict* ddict);
/********************************
* Dictionary helper functions
*******************************/
/*! ZSTD_getDictID_fromDict() :
* Provides the dictID stored within dictionary.
* if @return == 0, the dictionary is not conformant with Zstandard specification.
* It can still be loaded, but as a content-only dictionary. */
ZSTDLIB_API unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize);
/*! ZSTD_getDictID_fromDDict() :
* Provides the dictID of the dictionary loaded into `ddict`.
* If @return == 0, the dictionary is not conformant to Zstandard specification, or empty.
* Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */
ZSTDLIB_API unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict);
/*! ZSTD_getDictID_fromFrame() :
* Provides the dictID required to decompressed the frame stored within `src`.
* If @return == 0, the dictID could not be decoded.
* This could for one of the following reasons :
* - The frame does not require a dictionary to be decoded (most common case).
* - The frame was built with dictID intentionally removed. Whatever dictionary is necessary is a hidden information.
* Note : this use case also happens when using a non-conformant dictionary.
* - `srcSize` is too small, and as a result, the frame header could not be decoded (only possible if `srcSize < ZSTD_FRAMEHEADERSIZE_MAX`).
* - This is not a Zstandard frame.
* When identifying the exact failure cause, it's possible to use ZSTD_getFrameHeader(), which will provide a more precise error code. */
ZSTDLIB_API unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize);
/******************************************************************************* /*******************************************************************************
* Advanced dictionary and prefix API * Advanced dictionary and prefix API
* *
@ -1465,31 +1492,6 @@ ZSTDLIB_API unsigned ZSTD_isFrame(const void* buffer, size_t size);
* it must remain read accessible throughout the lifetime of DDict */ * it must remain read accessible throughout the lifetime of DDict */
ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict_byReference(const void* dictBuffer, size_t dictSize); ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict_byReference(const void* dictBuffer, size_t dictSize);
/*! ZSTD_getDictID_fromDict() :
* Provides the dictID stored within dictionary.
* if @return == 0, the dictionary is not conformant with Zstandard specification.
* It can still be loaded, but as a content-only dictionary. */
ZSTDLIB_API unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize);
/*! ZSTD_getDictID_fromDDict() :
* Provides the dictID of the dictionary loaded into `ddict`.
* If @return == 0, the dictionary is not conformant to Zstandard specification, or empty.
* Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */
ZSTDLIB_API unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict);
/*! ZSTD_getDictID_fromFrame() :
* Provides the dictID required to decompressed the frame stored within `src`.
* If @return == 0, the dictID could not be decoded.
* This could for one of the following reasons :
* - The frame does not require a dictionary to be decoded (most common case).
* - The frame was built with dictID intentionally removed. Whatever dictionary is necessary is a hidden information.
* Note : this use case also happens when using a non-conformant dictionary.
* - `srcSize` is too small, and as a result, the frame header could not be decoded (only possible if `srcSize < ZSTD_FRAMEHEADERSIZE_MAX`).
* - This is not a Zstandard frame.
* When identifying the exact failure cause, it's possible to use ZSTD_getFrameHeader(), which will provide a more precise error code. */
ZSTDLIB_API unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize);
/*! ZSTD_DCtx_loadDictionary_byReference() : /*! ZSTD_DCtx_loadDictionary_byReference() :
* Same as ZSTD_DCtx_loadDictionary(), * Same as ZSTD_DCtx_loadDictionary(),
* but references `dict` content instead of copying it into `dctx`. * but references `dict` content instead of copying it into `dctx`.