2016-10-06 04:23:52 -07:00
/*
2016-08-30 10:04:33 -07:00
* Copyright ( c ) 2016 - present , Yann Collet , Facebook , Inc .
* All rights reserved .
*
2017-08-18 16:52:05 -07:00
* This source code is licensed under both the BSD - style license ( found in the
* LICENSE file in the root directory of this source tree ) and the GPLv2 ( found
* in the COPYING file in the root directory of this source tree ) .
2017-09-08 00:09:23 -07:00
* You may select , at your option , one of the above - listed licenses .
2016-08-30 10:04:33 -07:00
*/
2015-01-23 16:58:16 -08:00
# if defined (__cplusplus)
extern " C " {
# endif
2016-11-29 11:46:37 -08:00
# ifndef ZSTD_H_235446
# define ZSTD_H_235446
2016-10-06 04:23:52 -07:00
/* ====== Dependency ======*/
2015-01-23 16:58:16 -08:00
# include <stddef.h> /* size_t */
2016-12-07 16:39:34 -08:00
/* ===== ZSTDLIB_API : control library symbols visibility ===== */
2017-05-19 18:01:59 -07:00
# ifndef ZSTDLIB_VISIBILITY
# if defined(__GNUC__) && (__GNUC__ >= 4)
# define ZSTDLIB_VISIBILITY __attribute__ ((visibility ("default")))
# else
# define ZSTDLIB_VISIBILITY
# endif
2016-12-16 13:27:30 -08:00
# endif
# if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1)
# define ZSTDLIB_API __declspec(dllexport) ZSTDLIB_VISIBILITY
2016-11-15 09:05:46 -08:00
# elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1)
2016-12-16 13:27:30 -08:00
# define ZSTDLIB_API __declspec(dllimport) ZSTDLIB_VISIBILITY /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/
2015-12-07 08:44:09 -08:00
# else
2016-12-16 13:27:30 -08:00
# define ZSTDLIB_API ZSTDLIB_VISIBILITY
2015-12-07 08:44:09 -08:00
# endif
2016-10-06 07:28:21 -07:00
/*******************************************************************************************************
Introduction
2017-05-12 16:29:19 -07:00
zstd , short for Zstandard , is a fast lossless compression algorithm ,
targeting real - time compression scenarios at zlib - level and better compression ratios .
The zstd compression library provides in - memory compression and decompression functions .
The library supports compression levels from 1 up to ZSTD_maxCLevel ( ) which is currently 22.
2017-02-20 12:08:59 -08:00
Levels > = 20 , labeled ` - - ultra ` , should be used with caution , as they require more memory .
2016-10-06 07:28:21 -07:00
Compression can be done in :
- a single step ( described as Simple API )
2018-01-10 08:47:19 -08:00
- a single step , reusing a context ( described as Explicit context )
2016-10-24 17:22:12 -07:00
- unbounded multiple steps ( described as Streaming compression )
2017-06-19 15:23:19 -07:00
The compression ratio achievable on small data can be highly improved using a dictionary in :
2016-10-06 07:28:21 -07:00
- a single step ( described as Simple dictionary API )
2018-01-10 08:47:19 -08:00
- a single step , reusing a dictionary ( described as Bulk - processing dictionary API )
2016-10-06 07:28:21 -07:00
2016-10-24 17:22:12 -07:00
Advanced experimental functions can be accessed using # define ZSTD_STATIC_LINKING_ONLY before including zstd . h .
2017-06-19 15:23:19 -07:00
Advanced experimental APIs shall never be used with a dynamic library .
2016-10-06 07:28:21 -07:00
They are not " stable " , their definition may change in the future . Only static linking is allowed .
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
/*------ Version ------*/
2016-08-31 07:51:25 -07:00
# define ZSTD_VERSION_MAJOR 1
2017-05-08 17:52:46 -07:00
# define ZSTD_VERSION_MINOR 3
2018-01-27 22:23:26 -08:00
# define ZSTD_VERSION_RELEASE 4
2016-04-20 13:46:16 -07:00
2017-05-16 16:12:23 -07:00
# define ZSTD_VERSION_NUMBER (ZSTD_VERSION_MAJOR *100*100 + ZSTD_VERSION_MINOR *100 + ZSTD_VERSION_RELEASE)
2017-06-19 15:23:19 -07:00
ZSTDLIB_API unsigned ZSTD_versionNumber ( void ) ; /**< useful to check dll version */
2017-05-16 16:12:23 -07:00
2016-04-20 13:46:16 -07:00
# define ZSTD_LIB_VERSION ZSTD_VERSION_MAJOR.ZSTD_VERSION_MINOR.ZSTD_VERSION_RELEASE
# define ZSTD_QUOTE(str) #str
# define ZSTD_EXPAND_AND_QUOTE(str) ZSTD_QUOTE(str)
# define ZSTD_VERSION_STRING ZSTD_EXPAND_AND_QUOTE(ZSTD_LIB_VERSION)
2018-01-10 08:47:19 -08:00
ZSTDLIB_API const char * ZSTD_versionString ( void ) ; /* added in v1.3.0 */
2015-01-23 16:58:16 -08:00
2016-10-06 04:23:52 -07:00
/***************************************
2016-07-18 07:52:10 -07:00
* Simple API
2015-10-21 01:07:25 -07:00
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
2016-02-04 06:28:14 -08:00
/*! ZSTD_compress() :
2017-04-20 22:54:54 -07:00
* Compresses ` src ` content as a single zstd compressed frame into already allocated ` dst ` .
* Hint : compression runs faster if ` dstCapacity ` > = ` ZSTD_compressBound ( srcSize ) ` .
* @ return : compressed size written into ` dst ` ( < = ` dstCapacity ) ,
* or an error code if it fails ( which can be tested using ZSTD_isError ( ) ) . */
2016-07-18 07:52:10 -07:00
ZSTDLIB_API size_t ZSTD_compress ( void * dst , size_t dstCapacity ,
const void * src , size_t srcSize ,
int compressionLevel ) ;
2015-01-23 16:58:16 -08:00
2016-02-04 06:28:14 -08:00
/*! ZSTD_decompress() :
2017-04-20 22:54:54 -07:00
* ` compressedSize ` : must be the _exact_ size of some number of compressed and / or skippable frames .
2017-06-19 15:23:19 -07:00
* ` dstCapacity ` is an upper bound of originalSize to regenerate .
2017-04-20 22:54:54 -07:00
* If user cannot imply a maximum upper bound , it ' s better to use streaming mode to decompress data .
* @ return : the number of bytes decompressed into ` dst ` ( < = ` dstCapacity ` ) ,
* or an errorCode if it fails ( which can be tested using ZSTD_isError ( ) ) . */
2016-02-02 17:11:32 -08:00
ZSTDLIB_API size_t ZSTD_decompress ( void * dst , size_t dstCapacity ,
2015-12-07 08:44:09 -08:00
const void * src , size_t compressedSize ) ;
2015-01-23 16:58:16 -08:00
2018-01-10 08:47:19 -08:00
/*! ZSTD_getFrameContentSize() : added in v1.3.0
2017-06-29 05:18:09 -07:00
* ` src ` should point to the start of a ZSTD encoded frame .
* ` srcSize ` must be at least as large as the frame header .
* hint : any size > = ` ZSTD_frameHeaderSize_max ` is large enough .
* @ return : - decompressed size of the frame in ` src ` , if known
* - ZSTD_CONTENTSIZE_UNKNOWN if the size cannot be determined
* - ZSTD_CONTENTSIZE_ERROR if an error occurred ( e . g . invalid magic number , srcSize too small )
* note 1 : a 0 return value means the frame is valid but " empty " .
* note 2 : decompressed size is an optional field , it may not be present , typically in streaming mode .
* When ` return = = ZSTD_CONTENTSIZE_UNKNOWN ` , data to decompress could be any size .
2017-04-20 22:54:54 -07:00
* In which case , it ' s necessary to use streaming mode to decompress data .
2017-06-29 05:18:09 -07:00
* Optionally , application can rely on some implicit limit ,
* as ZSTD_decompress ( ) only needs an upper bound of decompressed size .
* ( For example , data could be necessarily cut into blocks < = 16 KB ) .
* note 3 : decompressed size is always present when compression is done with ZSTD_compress ( )
* note 4 : decompressed size can be very large ( 64 - bits value ) ,
2017-04-20 22:54:54 -07:00
* potentially larger than what local system can handle as a single memory segment .
* In which case , it ' s necessary to use streaming mode to decompress data .
2017-06-29 05:18:09 -07:00
* note 5 : If source is untrusted , decompressed size could be wrong or intentionally modified .
* Always ensure return value fits within application ' s authorized limits .
2017-04-20 22:54:54 -07:00
* Each application can set its own limits .
2017-06-29 05:18:09 -07:00
* note 6 : This function replaces ZSTD_getDecompressedSize ( ) */
# define ZSTD_CONTENTSIZE_UNKNOWN (0ULL - 1)
# define ZSTD_CONTENTSIZE_ERROR (0ULL - 2)
ZSTDLIB_API unsigned long long ZSTD_getFrameContentSize ( const void * src , size_t srcSize ) ;
/*! ZSTD_getDecompressedSize() :
* NOTE : This function is now obsolete , in favor of ZSTD_getFrameContentSize ( ) .
2018-01-10 08:47:19 -08:00
* Both functions work the same way , but ZSTD_getDecompressedSize ( ) blends
* " empty " , " unknown " and " error " results to the same return value ( 0 ) ,
* while ZSTD_getFrameContentSize ( ) gives them separate return values .
* ` src ` is the start of a zstd compressed frame .
* @ return : content size to be decompressed , as a 64 - bits value _if known and not empty_ , 0 otherwise . */
2016-09-12 15:51:47 -07:00
ZSTDLIB_API unsigned long long ZSTD_getDecompressedSize ( const void * src , size_t srcSize ) ;
2016-07-27 06:09:11 -07:00
2016-07-18 07:52:10 -07:00
/*====== Helper functions ======*/
2017-11-18 11:16:39 -08:00
# define ZSTD_COMPRESSBOUND(srcSize) ((srcSize) + ((srcSize)>>8) + (((srcSize) < (128<<10)) ? (((128<<10) - (srcSize)) >> 11) /* margin, from 64 to 0 */ : 0)) /* this formula ensures that bound(A) + bound(B) <= bound(A+B) as long as A and B >= 128 KB */
2018-01-10 08:47:19 -08:00
ZSTDLIB_API size_t ZSTD_compressBound ( size_t srcSize ) ; /*!< maximum compressed size in worst case single-pass scenario */
2016-02-04 06:28:14 -08:00
ZSTDLIB_API unsigned ZSTD_isError ( size_t code ) ; /*!< tells if a `size_t` function result is an error code */
2016-07-18 07:52:10 -07:00
ZSTDLIB_API const char * ZSTD_getErrorName ( size_t code ) ; /*!< provides readable string from an error code */
2017-09-29 23:17:41 -07:00
ZSTDLIB_API int ZSTD_maxCLevel ( void ) ; /*!< maximum compression level available */
2015-10-21 01:07:25 -07:00
2016-10-06 04:23:52 -07:00
/***************************************
2018-01-10 08:47:19 -08:00
* Explicit context
2015-10-21 01:07:25 -07:00
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
2016-10-24 17:22:12 -07:00
/*= Compression context
2017-04-20 22:54:54 -07:00
* When compressing many times ,
* it is recommended to allocate a context just once , and re - use it for each successive compression operation .
* This will make workload friendlier for system ' s memory .
* Use one context per thread for parallel execution in multi - threaded environments . */
2016-08-25 16:43:47 -07:00
typedef struct ZSTD_CCtx_s ZSTD_CCtx ;
2015-12-07 08:44:09 -08:00
ZSTDLIB_API ZSTD_CCtx * ZSTD_createCCtx ( void ) ;
2016-07-27 18:47:45 -07:00
ZSTDLIB_API size_t ZSTD_freeCCtx ( ZSTD_CCtx * cctx ) ;
2015-10-21 01:07:25 -07:00
2016-10-06 04:23:52 -07:00
/*! ZSTD_compressCCtx() :
2017-04-20 22:54:54 -07:00
* Same as ZSTD_compress ( ) , requires an allocated ZSTD_CCtx ( see ZSTD_createCCtx ( ) ) . */
2017-05-12 16:29:19 -07:00
ZSTDLIB_API size_t ZSTD_compressCCtx ( ZSTD_CCtx * ctx ,
void * dst , size_t dstCapacity ,
const void * src , size_t srcSize ,
int compressionLevel ) ;
2015-01-23 16:58:16 -08:00
2017-03-14 11:24:09 -07:00
/*= Decompression context
2017-04-20 22:54:54 -07:00
* When decompressing many times ,
2017-05-12 16:29:19 -07:00
* it is recommended to allocate a context only once ,
* and re - use it for each successive compression operation .
2017-04-20 22:54:54 -07:00
* This will make workload friendlier for system ' s memory .
2017-05-12 16:29:19 -07:00
* Use one context per thread for parallel execution . */
2016-08-25 16:43:47 -07:00
typedef struct ZSTD_DCtx_s ZSTD_DCtx ;
2016-01-07 06:35:18 -08:00
ZSTDLIB_API ZSTD_DCtx * ZSTD_createDCtx ( void ) ;
2016-07-27 18:47:45 -07:00
ZSTDLIB_API size_t ZSTD_freeDCtx ( ZSTD_DCtx * dctx ) ;
2016-01-07 06:35:18 -08:00
2016-10-06 04:23:52 -07:00
/*! ZSTD_decompressDCtx() :
2017-05-12 16:29:19 -07:00
* Same as ZSTD_decompress ( ) , requires an allocated ZSTD_DCtx ( see ZSTD_createDCtx ( ) ) */
ZSTDLIB_API size_t ZSTD_decompressDCtx ( ZSTD_DCtx * ctx ,
void * dst , size_t dstCapacity ,
const void * src , size_t srcSize ) ;
2016-02-02 17:11:32 -08:00
2016-10-06 04:23:52 -07:00
/**************************
2016-06-07 03:16:49 -07:00
* Simple dictionary API
* * * * * * * * * * * * * * * * * * * * * * * * * * */
/*! ZSTD_compress_usingDict() :
2017-06-19 15:23:19 -07:00
* Compression using a predefined Dictionary ( see dictBuilder / zdict . h ) .
* Note : This function loads the dictionary , resulting in significant startup delay .
* Note : When ` dict = = NULL | | dictSize < 8 ` no dictionary is used . */
2016-06-07 03:16:49 -07:00
ZSTDLIB_API size_t ZSTD_compress_usingDict ( ZSTD_CCtx * ctx ,
void * dst , size_t dstCapacity ,
const void * src , size_t srcSize ,
const void * dict , size_t dictSize ,
int compressionLevel ) ;
/*! ZSTD_decompress_usingDict() :
2017-06-19 15:23:19 -07:00
* Decompression using a predefined Dictionary ( see dictBuilder / zdict . h ) .
* Dictionary must be identical to the one used during compression .
* Note : This function loads the dictionary , resulting in significant startup delay .
* Note : When ` dict = = NULL | | dictSize < 8 ` no dictionary is used . */
2016-06-07 03:16:49 -07:00
ZSTDLIB_API size_t ZSTD_decompress_usingDict ( ZSTD_DCtx * dctx ,
void * dst , size_t dstCapacity ,
const void * src , size_t srcSize ,
const void * dict , size_t dictSize ) ;
2017-06-19 15:23:19 -07:00
/**********************************
* Bulk processing dictionary API
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
2016-10-06 07:28:21 -07:00
typedef struct ZSTD_CDict_s ZSTD_CDict ;
2016-06-06 15:51:51 -07:00
/*! ZSTD_createCDict() :
2017-06-19 15:23:19 -07:00
* When compressing multiple messages / blocks with the same dictionary , it ' s recommended to load it just once .
* ZSTD_createCDict ( ) will create a digested dictionary , ready to start future compression operations without startup delay .
* ZSTD_CDict can be created once and shared by multiple threads concurrently , since its usage is read - only .
* ` dictBuffer ` can be released after ZSTD_CDict creation , since its content is copied within CDict */
2017-05-12 16:29:19 -07:00
ZSTDLIB_API ZSTD_CDict * ZSTD_createCDict ( const void * dictBuffer , size_t dictSize ,
int compressionLevel ) ;
2016-10-06 07:28:21 -07:00
/*! ZSTD_freeCDict() :
2017-06-19 15:23:19 -07:00
* Function frees memory allocated by ZSTD_createCDict ( ) . */
2016-06-06 15:51:51 -07:00
ZSTDLIB_API size_t ZSTD_freeCDict ( ZSTD_CDict * CDict ) ;
/*! ZSTD_compress_usingCDict() :
2017-04-17 17:57:35 -07:00
* Compression using a digested Dictionary .
* Faster startup than ZSTD_compress_usingDict ( ) , recommended when same dictionary is used multiple times .
* Note that compression level is decided during dictionary creation .
* Frame parameters are hardcoded ( dictID = yes , contentSize = yes , checksum = no ) */
2016-06-06 15:51:51 -07:00
ZSTDLIB_API size_t ZSTD_compress_usingCDict ( ZSTD_CCtx * cctx ,
2016-06-07 03:16:49 -07:00
void * dst , size_t dstCapacity ,
const void * src , size_t srcSize ,
const ZSTD_CDict * cdict ) ;
2016-06-06 15:51:51 -07:00
2016-10-06 07:28:21 -07:00
typedef struct ZSTD_DDict_s ZSTD_DDict ;
2016-06-06 15:51:51 -07:00
/*! ZSTD_createDDict() :
2017-06-19 15:23:19 -07:00
* Create a digested dictionary , ready to start decompression operation without startup delay .
* dictBuffer can be released after DDict creation , as its content is copied inside DDict */
2016-12-21 07:44:35 -08:00
ZSTDLIB_API ZSTD_DDict * ZSTD_createDDict ( const void * dictBuffer , size_t dictSize ) ;
2016-10-06 07:28:21 -07:00
/*! ZSTD_freeDDict() :
2017-06-19 15:23:19 -07:00
* Function frees memory allocated with ZSTD_createDDict ( ) */
2016-06-06 15:51:51 -07:00
ZSTDLIB_API size_t ZSTD_freeDDict ( ZSTD_DDict * ddict ) ;
/*! ZSTD_decompress_usingDDict() :
2017-06-19 15:23:19 -07:00
* Decompression using a digested Dictionary .
* Faster startup than ZSTD_decompress_usingDict ( ) , recommended when same dictionary is used multiple times . */
2016-06-06 15:51:51 -07:00
ZSTDLIB_API size_t ZSTD_decompress_usingDDict ( ZSTD_DCtx * dctx ,
2016-06-07 03:16:49 -07:00
void * dst , size_t dstCapacity ,
const void * src , size_t srcSize ,
const ZSTD_DDict * ddict ) ;
2016-02-02 17:11:32 -08:00
2016-01-07 06:35:18 -08:00
2016-10-06 04:23:52 -07:00
/****************************
2016-08-19 09:39:36 -07:00
* Streaming
* * * * * * * * * * * * * * * * * * * * * * * * * * * */
typedef struct ZSTD_inBuffer_s {
const void * src ; /**< start of input buffer */
size_t size ; /**< size of input buffer */
size_t pos ; /**< position where reading stopped. Will be updated. Necessarily 0 <= pos <= size */
} ZSTD_inBuffer ;
typedef struct ZSTD_outBuffer_s {
void * dst ; /**< start of output buffer */
size_t size ; /**< size of output buffer */
size_t pos ; /**< position where writing stopped. Will be updated. Necessarily 0 <= pos <= size */
} ZSTD_outBuffer ;
2016-10-06 05:22:48 -07:00
/*-***********************************************************************
2016-10-24 06:59:51 -07:00
* Streaming compression - HowTo
2016-08-19 09:39:36 -07:00
*
* A ZSTD_CStream object is required to track streaming operation .
* Use ZSTD_createCStream ( ) and ZSTD_freeCStream ( ) to create / release resources .
* ZSTD_CStream objects can be reused multiple times on consecutive compression operations .
2016-10-24 17:22:12 -07:00
* It is recommended to re - use ZSTD_CStream in situations where many streaming operations will be achieved consecutively ,
* since it will play nicer with system ' s memory , by re - using already allocated memory .
* Use one separate ZSTD_CStream per thread for parallel execution .
2016-08-19 09:39:36 -07:00
*
2016-10-24 17:22:12 -07:00
* Start a new compression by initializing ZSTD_CStream .
2016-08-19 09:39:36 -07:00
* Use ZSTD_initCStream ( ) to start a new compression operation .
2016-12-13 07:39:36 -08:00
* Use ZSTD_initCStream_usingDict ( ) or ZSTD_initCStream_usingCDict ( ) for a compression which requires a dictionary ( experimental section )
2016-08-19 09:39:36 -07:00
*
* Use ZSTD_compressStream ( ) repetitively to consume input stream .
2016-09-05 08:39:56 -07:00
* The function will automatically update both ` pos ` fields .
2016-08-19 09:39:36 -07:00
* Note that it may not consume the entire input , in which case ` pos < size ` ,
* and it ' s up to the caller to present again remaining data .
2016-08-28 07:43:34 -07:00
* @ return : a size hint , preferred nb of bytes to use as input for next function call
2016-08-19 09:39:36 -07:00
* or an error code , which can be tested using ZSTD_isError ( ) .
2016-12-13 07:39:36 -08:00
* Note 1 : it ' s just a hint , to help latency a little , any other value will work fine .
* Note 2 : size hint is guaranteed to be < = ZSTD_CStreamInSize ( )
2016-08-19 09:39:36 -07:00
*
2016-12-13 07:39:36 -08:00
* At any moment , it ' s possible to flush whatever data remains within internal buffer , using ZSTD_flushStream ( ) .
2016-08-19 09:39:36 -07:00
* ` output - > pos ` will be updated .
2016-12-13 07:39:36 -08:00
* Note that some content might still be left within internal buffer if ` output - > size ` is too small .
2016-08-19 09:39:36 -07:00
* @ return : nb of bytes still present within internal buffer ( 0 if it ' s empty )
* or an error code , which can be tested using ZSTD_isError ( ) .
*
* ZSTD_endStream ( ) instructs to finish a frame .
* It will perform a flush and write frame epilogue .
* The epilogue is required for decoders to consider a frame completed .
2017-05-19 10:17:59 -07:00
* ZSTD_endStream ( ) may not be able to flush full data if ` output - > size ` is too small .
2016-08-19 09:39:36 -07:00
* In which case , call again ZSTD_endStream ( ) to complete the flush .
2017-05-19 10:17:59 -07:00
* @ return : 0 if frame fully completed and fully flushed ,
or > 0 if some data is still present within internal buffer
( value is minimum size estimation for remaining data to flush , but it could be more )
2016-08-19 09:39:36 -07:00
* or an error code , which can be tested using ZSTD_isError ( ) .
*
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
2017-06-19 15:23:19 -07:00
typedef ZSTD_CCtx ZSTD_CStream ; /**< CCtx and CStream are now effectively same object (>= v1.3.0) */
2017-06-29 05:18:09 -07:00
/* Continue to distinguish them for compatibility with versions <= v1.2.0 */
2017-03-14 11:24:09 -07:00
/*===== ZSTD_CStream management functions =====*/
2016-08-22 16:18:06 -07:00
ZSTDLIB_API ZSTD_CStream * ZSTD_createCStream ( void ) ;
ZSTDLIB_API size_t ZSTD_freeCStream ( ZSTD_CStream * zcs ) ;
2016-12-13 07:39:36 -08:00
2017-03-14 11:24:09 -07:00
/*===== Streaming compression functions =====*/
2016-08-22 16:18:06 -07:00
ZSTDLIB_API size_t ZSTD_initCStream ( ZSTD_CStream * zcs , int compressionLevel ) ;
ZSTDLIB_API size_t ZSTD_compressStream ( ZSTD_CStream * zcs , ZSTD_outBuffer * output , ZSTD_inBuffer * input ) ;
ZSTDLIB_API size_t ZSTD_flushStream ( ZSTD_CStream * zcs , ZSTD_outBuffer * output ) ;
ZSTDLIB_API size_t ZSTD_endStream ( ZSTD_CStream * zcs , ZSTD_outBuffer * output ) ;
2016-08-19 09:39:36 -07:00
2016-10-06 04:23:52 -07:00
ZSTDLIB_API size_t ZSTD_CStreamInSize ( void ) ; /**< recommended size for input buffer */
ZSTDLIB_API size_t ZSTD_CStreamOutSize ( void ) ; /**< recommended size for output buffer. Guarantee to successfully flush at least one complete compressed block in all circumstances. */
2016-08-19 09:39:36 -07:00
2016-10-06 05:22:48 -07:00
/*-***************************************************************************
2016-10-24 06:59:51 -07:00
* Streaming decompression - HowTo
2016-08-19 09:39:36 -07:00
*
* A ZSTD_DStream object is required to track streaming operations .
* Use ZSTD_createDStream ( ) and ZSTD_freeDStream ( ) to create / release resources .
2016-08-23 07:58:10 -07:00
* ZSTD_DStream objects can be re - used multiple times .
2016-08-19 09:39:36 -07:00
*
* Use ZSTD_initDStream ( ) to start a new decompression operation ,
* or ZSTD_initDStream_usingDict ( ) if decompression requires a dictionary .
2016-09-05 10:47:43 -07:00
* @ return : recommended first input size
2016-08-19 09:39:36 -07:00
*
* Use ZSTD_decompressStream ( ) repetitively to consume your input .
2016-09-05 08:39:56 -07:00
* The function will update both ` pos ` fields .
2016-09-09 07:44:16 -07:00
* If ` input . pos < input . size ` , some input has not been consumed .
2016-09-05 20:16:40 -07:00
* It ' s up to the caller to present again remaining data .
2016-09-09 07:44:16 -07:00
* If ` output . pos < output . size ` , decoder has flushed everything it could .
2016-08-19 09:39:36 -07:00
* @ return : 0 when a frame is completely decoded and fully flushed ,
2016-09-05 08:39:56 -07:00
* an error code , which can be tested using ZSTD_isError ( ) ,
2016-12-02 18:37:38 -08:00
* any other value > 0 , which means there is still some decoding to do to complete current frame .
* The return value is a suggested next input size ( a hint to improve latency ) that will never load more than the current frame .
2016-08-19 09:39:36 -07:00
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
2017-06-19 15:23:19 -07:00
typedef ZSTD_DCtx ZSTD_DStream ; /**< DCtx and DStream are now effectively same object (>= v1.3.0) */
2018-01-10 08:47:19 -08:00
/* For compatibility with versions <= v1.2.0, continue to consider them separated. */
2017-03-14 11:24:09 -07:00
/*===== ZSTD_DStream management functions =====*/
2016-08-22 16:18:06 -07:00
ZSTDLIB_API ZSTD_DStream * ZSTD_createDStream ( void ) ;
ZSTDLIB_API size_t ZSTD_freeDStream ( ZSTD_DStream * zds ) ;
2016-12-13 07:39:36 -08:00
2017-03-14 11:24:09 -07:00
/*===== Streaming decompression functions =====*/
2016-10-06 04:23:52 -07:00
ZSTDLIB_API size_t ZSTD_initDStream ( ZSTD_DStream * zds ) ;
ZSTDLIB_API size_t ZSTD_decompressStream ( ZSTD_DStream * zds , ZSTD_outBuffer * output , ZSTD_inBuffer * input ) ;
2016-08-19 09:39:36 -07:00
2016-08-22 16:18:06 -07:00
ZSTDLIB_API size_t ZSTD_DStreamInSize ( void ) ; /*!< recommended size for input buffer */
2016-09-08 10:29:04 -07:00
ZSTDLIB_API size_t ZSTD_DStreamOutSize ( void ) ; /*!< recommended size for output buffer. Guarantee to successfully flush at least one complete block in all circumstances. */
2016-08-19 09:39:36 -07:00
2016-11-29 11:46:37 -08:00
# endif /* ZSTD_H_235446 */
2016-08-19 09:39:36 -07:00
2016-06-04 10:47:02 -07:00
2016-10-06 04:23:52 -07:00
/****************************************************************************************
2016-10-06 07:28:21 -07:00
* START OF ADVANCED AND EXPERIMENTAL FUNCTIONS
2016-06-04 10:47:02 -07:00
* The definitions in this section are considered experimental .
2017-06-29 05:18:09 -07:00
* They should never be used with a dynamic library , as prototypes may change in the future .
* They are provided for advanced scenarios .
2016-06-04 10:47:02 -07:00
* Use them only in association with static linking .
2016-10-06 04:23:52 -07:00
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
2016-06-04 10:47:02 -07:00
2017-05-25 15:07:37 -07:00
# if defined(ZSTD_STATIC_LINKING_ONLY) && !defined(ZSTD_H_ZSTD_STATIC_LINKING_ONLY)
# define ZSTD_H_ZSTD_STATIC_LINKING_ONLY
2016-10-06 04:23:52 -07:00
/* --- Constants ---*/
2016-12-21 07:44:35 -08:00
# define ZSTD_MAGICNUMBER 0xFD2FB528 /* >= v0.8.0 */
2016-06-04 10:47:02 -07:00
# define ZSTD_MAGIC_SKIPPABLE_START 0x184D2A50U
2018-01-10 08:47:19 -08:00
# define ZSTD_MAGIC_DICTIONARY 0xEC30A437 /* >= v0.7.0 */
2016-06-04 10:47:02 -07:00
2017-09-28 01:25:40 -07:00
# define ZSTD_WINDOWLOG_MAX_32 30
# define ZSTD_WINDOWLOG_MAX_64 31
2017-03-08 15:40:13 -08:00
# define ZSTD_WINDOWLOG_MAX ((unsigned)(sizeof(size_t) == 4 ? ZSTD_WINDOWLOG_MAX_32 : ZSTD_WINDOWLOG_MAX_64))
2017-09-28 01:25:40 -07:00
# define ZSTD_WINDOWLOG_MIN 10
2018-01-10 20:33:45 -08:00
# define ZSTD_HASHLOG_MAX ((ZSTD_WINDOWLOG_MAX < 30) ? ZSTD_WINDOWLOG_MAX : 30)
2017-09-28 01:25:40 -07:00
# define ZSTD_HASHLOG_MIN 6
2018-02-26 13:30:24 -08:00
# define ZSTD_CHAINLOG_MAX_32 29
# define ZSTD_CHAINLOG_MAX_64 30
# define ZSTD_CHAINLOG_MAX ((unsigned)(sizeof(size_t) == 4 ? ZSTD_CHAINLOG_MAX_32 : ZSTD_CHAINLOG_MAX_64))
2017-09-28 01:25:40 -07:00
# define ZSTD_CHAINLOG_MIN ZSTD_HASHLOG_MIN
# define ZSTD_HASHLOG3_MAX 17
# define ZSTD_SEARCHLOG_MAX (ZSTD_WINDOWLOG_MAX-1)
# define ZSTD_SEARCHLOG_MIN 1
# define ZSTD_SEARCHLENGTH_MAX 7 /* only for ZSTD_fast, other strategies are limited to 6 */
# define ZSTD_SEARCHLENGTH_MIN 3 /* only for ZSTD_btopt, other strategies are limited to 4 */
2018-03-13 14:23:14 -07:00
# define ZSTD_TARGETLENGTH_MIN 1 /* only used by btopt, btultra and btfast */
# define ZSTD_TARGETLENGTH_MAX 999 /* only used by btopt, btultra and btfast */
2017-09-28 01:25:40 -07:00
# define ZSTD_LDM_MINMATCH_MIN 4
2017-09-06 08:39:46 -07:00
# define ZSTD_LDM_MINMATCH_MAX 4096
2017-09-02 21:10:36 -07:00
# define ZSTD_LDM_BUCKETSIZELOG_MAX 8
2016-06-04 10:47:02 -07:00
2017-09-26 14:31:06 -07:00
# define ZSTD_FRAMEHEADERSIZE_PREFIX 5 /* minimum input size to know frame header size */
# define ZSTD_FRAMEHEADERSIZE_MIN 6
# define ZSTD_FRAMEHEADERSIZE_MAX 18 /* for static allocation */
static const size_t ZSTD_frameHeaderSize_prefix = ZSTD_FRAMEHEADERSIZE_PREFIX ;
2017-06-29 05:18:09 -07:00
static const size_t ZSTD_frameHeaderSize_min = ZSTD_FRAMEHEADERSIZE_MIN ;
2017-09-26 14:31:06 -07:00
static const size_t ZSTD_frameHeaderSize_max = ZSTD_FRAMEHEADERSIZE_MAX ;
2016-06-05 15:26:38 -07:00
static const size_t ZSTD_skippableHeaderSize = 8 ; /* magic number + skippable frame length */
2016-06-04 10:47:02 -07:00
2016-10-24 06:59:51 -07:00
/*--- Advanced types ---*/
2017-05-12 16:29:19 -07:00
typedef enum { ZSTD_fast = 1 , ZSTD_dfast , ZSTD_greedy , ZSTD_lazy , ZSTD_lazy2 ,
ZSTD_btlazy2 , ZSTD_btopt , ZSTD_btultra } ZSTD_strategy ; /* from faster to stronger */
2016-06-04 10:47:02 -07:00
typedef struct {
2016-08-13 15:16:20 -07:00
unsigned windowLog ; /**< largest match distance : larger == more compression, more memory needed during decompression */
unsigned chainLog ; /**< fully searched segment : larger == more compression, slower, more memory (useless for fast) */
unsigned hashLog ; /**< dispatch table : larger == faster, more memory */
unsigned searchLog ; /**< nb of searches : larger == more compression, slower */
unsigned searchLength ; /**< match length searched : larger == faster decompression, sometimes less compression */
unsigned targetLength ; /**< acceptable match size for optimal parser (only) : larger == more compression, slower */
2016-06-04 10:47:02 -07:00
ZSTD_strategy strategy ;
} ZSTD_compressionParameters ;
typedef struct {
2017-01-01 15:49:42 -08:00
unsigned contentSizeFlag ; /**< 1: content size will be in frame header (when known) */
unsigned checksumFlag ; /**< 1: generate a 32-bits checksum at end of frame, for error detection */
unsigned noDictIDFlag ; /**< 1: no dictID will be saved into frame header (if dictionary compression) */
2016-06-04 10:47:02 -07:00
} ZSTD_frameParameters ;
typedef struct {
ZSTD_compressionParameters cParams ;
ZSTD_frameParameters fParams ;
} ZSTD_parameters ;
2017-08-17 17:33:46 -07:00
typedef struct ZSTD_CCtx_params_s ZSTD_CCtx_params ;
2018-01-10 08:47:19 -08:00
typedef enum {
ZSTD_dm_auto = 0 , /* dictionary is "full" if it starts with ZSTD_MAGIC_DICTIONARY, otherwise it is "rawContent" */
ZSTD_dm_rawContent , /* ensures dictionary is always loaded as rawContent, even if it starts with ZSTD_MAGIC_DICTIONARY */
ZSTD_dm_fullDict /* refuses to load a dictionary if it does not respect Zstandard's specification */
} ZSTD_dictMode_e ;
typedef enum {
ZSTD_dlm_byCopy = 0 , /**< Copy dictionary content internally */
ZSTD_dlm_byRef , /**< Reference dictionary content -- the dictionary buffer must outlive its users. */
} ZSTD_dictLoadMethod_e ;
2016-06-04 10:47:02 -07:00
2017-02-10 11:38:57 -08:00
/***************************************
2017-05-09 15:11:30 -07:00
* Frame size functions
2017-02-10 11:38:57 -08:00
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
2017-02-22 12:12:32 -08:00
/*! ZSTD_findFrameCompressedSize() :
2017-02-22 12:27:15 -08:00
* ` src ` should point to the start of a ZSTD encoded frame or skippable frame
2017-11-27 12:15:23 -08:00
* ` srcSize ` must be > = first frame size
2017-06-29 05:18:09 -07:00
* @ return : the compressed size of the first frame starting at ` src ` ,
2017-05-09 15:11:30 -07:00
* suitable to pass to ` ZSTD_decompress ` or similar ,
2017-06-29 05:18:09 -07:00
* or an error code if input is invalid */
2017-02-22 12:12:32 -08:00
ZSTDLIB_API size_t ZSTD_findFrameCompressedSize ( const void * src , size_t srcSize ) ;
2017-02-10 11:38:57 -08:00
2017-02-09 10:50:43 -08:00
/*! ZSTD_findDecompressedSize() :
2017-05-09 15:11:30 -07:00
* ` src ` should point the start of a series of ZSTD encoded and / or skippable frames
2017-06-29 11:31:40 -07:00
* ` srcSize ` must be the _exact_ size of this series
2017-06-29 05:18:09 -07:00
* ( i . e . there should be a frame boundary exactly at ` srcSize ` bytes after ` src ` )
2017-05-09 15:11:30 -07:00
* @ return : - decompressed size of all data in all successive frames
* - if the decompressed size cannot be determined : ZSTD_CONTENTSIZE_UNKNOWN
* - if an error occurred : ZSTD_CONTENTSIZE_ERROR
*
* note 1 : decompressed size is an optional field , that may not be present , especially in streaming mode .
* When ` return = = ZSTD_CONTENTSIZE_UNKNOWN ` , data to decompress could be any size .
* In which case , it ' s necessary to use streaming mode to decompress data .
* note 2 : decompressed size is always present when compression is done with ZSTD_compress ( )
* note 3 : decompressed size can be very large ( 64 - bits value ) ,
* potentially larger than what local system can handle as a single memory segment .
* In which case , it ' s necessary to use streaming mode to decompress data .
* note 4 : If source is untrusted , decompressed size could be wrong or intentionally modified .
* Always ensure result fits within application ' s authorized limits .
* Each application can set its own limits .
* note 5 : ZSTD_findDecompressedSize handles multiple frames , and so it must traverse the input to
2017-06-29 05:18:09 -07:00
* read each contained frame header . This is fast as most of the data is skipped ,
2017-05-09 15:11:30 -07:00
* however it does mean that all frame data must be present and valid . */
2017-02-09 10:50:43 -08:00
ZSTDLIB_API unsigned long long ZSTD_findDecompressedSize ( const void * src , size_t srcSize ) ;
2017-06-15 14:13:28 -07:00
/*! ZSTD_frameHeaderSize() :
* ` src ` should point to the start of a ZSTD frame
* ` srcSize ` must be > = ZSTD_frameHeaderSize_prefix .
* @ return : size of the Frame Header */
2017-06-29 05:18:09 -07:00
ZSTDLIB_API size_t ZSTD_frameHeaderSize ( const void * src , size_t srcSize ) ;
2016-06-04 10:47:02 -07:00
2016-10-06 04:23:52 -07:00
/***************************************
2018-01-10 08:47:19 -08:00
* Memory management
2016-06-04 10:47:02 -07:00
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
2017-05-09 15:11:30 -07:00
/*! ZSTD_sizeof_*() :
* These functions give the current memory usage of selected object .
2018-01-10 08:47:19 -08:00
* Object memory usage can evolve when re - used . */
2017-05-09 15:11:30 -07:00
ZSTDLIB_API size_t ZSTD_sizeof_CCtx ( const ZSTD_CCtx * cctx ) ;
ZSTDLIB_API size_t ZSTD_sizeof_DCtx ( const ZSTD_DCtx * dctx ) ;
ZSTDLIB_API size_t ZSTD_sizeof_CStream ( const ZSTD_CStream * zcs ) ;
ZSTDLIB_API size_t ZSTD_sizeof_DStream ( const ZSTD_DStream * zds ) ;
ZSTDLIB_API size_t ZSTD_sizeof_CDict ( const ZSTD_CDict * cdict ) ;
ZSTDLIB_API size_t ZSTD_sizeof_DDict ( const ZSTD_DDict * ddict ) ;
/*! ZSTD_estimate*() :
* These functions make it possible to estimate memory usage
2017-06-26 15:52:39 -07:00
* of a future { D , C } Ctx , before its creation .
2017-06-28 15:34:56 -07:00
* ZSTD_estimateCCtxSize ( ) will provide a budget large enough for any compression level up to selected one .
* It will also consider src size to be arbitrarily " large " , which is worst case .
2017-09-24 16:47:02 -07:00
* If srcSize is known to always be small , ZSTD_estimateCCtxSize_usingCParams ( ) can provide a tighter estimation .
* ZSTD_estimateCCtxSize_usingCParams ( ) can be used in tandem with ZSTD_getCParams ( ) to create cParams from compressionLevel .
2018-02-01 19:29:30 -08:00
* ZSTD_estimateCCtxSize_usingCCtxParams ( ) can be used in tandem with ZSTD_CCtxParam_setParameter ( ) . Only single - threaded compression is supported . This function will return an error code if ZSTD_p_nbWorkers is > = 1.
2018-01-10 08:47:19 -08:00
* Note : CCtx size estimation is only correct for single - threaded compression . */
2017-06-26 15:52:39 -07:00
ZSTDLIB_API size_t ZSTD_estimateCCtxSize ( int compressionLevel ) ;
2017-09-24 16:47:02 -07:00
ZSTDLIB_API size_t ZSTD_estimateCCtxSize_usingCParams ( ZSTD_compressionParameters cParams ) ;
ZSTDLIB_API size_t ZSTD_estimateCCtxSize_usingCCtxParams ( const ZSTD_CCtx_params * params ) ;
2017-05-09 15:11:30 -07:00
ZSTDLIB_API size_t ZSTD_estimateDCtxSize ( void ) ;
2017-08-29 10:49:29 -07:00
/*! ZSTD_estimateCStreamSize() :
2017-06-28 15:34:56 -07:00
* ZSTD_estimateCStreamSize ( ) will provide a budget large enough for any compression level up to selected one .
* It will also consider src size to be arbitrarily " large " , which is worst case .
2017-09-24 16:47:02 -07:00
* If srcSize is known to always be small , ZSTD_estimateCStreamSize_usingCParams ( ) can provide a tighter estimation .
* ZSTD_estimateCStreamSize_usingCParams ( ) can be used in tandem with ZSTD_getCParams ( ) to create cParams from compressionLevel .
2018-02-01 19:29:30 -08:00
* ZSTD_estimateCStreamSize_usingCCtxParams ( ) can be used in tandem with ZSTD_CCtxParam_setParameter ( ) . Only single - threaded compression is supported . This function will return an error code if ZSTD_p_nbWorkers is > = 1.
2018-01-10 08:47:19 -08:00
* Note : CStream size estimation is only correct for single - threaded compression .
2017-06-26 17:44:26 -07:00
* ZSTD_DStream memory budget depends on window Size .
* This information can be passed manually , using ZSTD_estimateDStreamSize ,
* or deducted from a valid frame Header , using ZSTD_estimateDStreamSize_fromFrame ( ) ;
2017-05-09 15:11:30 -07:00
* Note : if streaming is init with function ZSTD_init ? Stream_usingDict ( ) ,
2017-06-26 17:44:26 -07:00
* an internal ? Dict will be created , which additional size is not estimated here .
2017-05-25 18:05:49 -07:00
* In this case , get total size by adding ZSTD_estimate ? DictSize */
2017-06-26 16:02:25 -07:00
ZSTDLIB_API size_t ZSTD_estimateCStreamSize ( int compressionLevel ) ;
2017-09-24 16:47:02 -07:00
ZSTDLIB_API size_t ZSTD_estimateCStreamSize_usingCParams ( ZSTD_compressionParameters cParams ) ;
ZSTDLIB_API size_t ZSTD_estimateCStreamSize_usingCCtxParams ( const ZSTD_CCtx_params * params ) ;
2017-06-26 17:44:26 -07:00
ZSTDLIB_API size_t ZSTD_estimateDStreamSize ( size_t windowSize ) ;
ZSTDLIB_API size_t ZSTD_estimateDStreamSize_fromFrame ( const void * src , size_t srcSize ) ;
2017-05-09 15:11:30 -07:00
/*! ZSTD_estimate?DictSize() :
2017-06-26 16:47:32 -07:00
* ZSTD_estimateCDictSize ( ) will bet that src size is relatively " small " , and content is copied , like ZSTD_createCDict ( ) .
2018-01-10 08:47:19 -08:00
* ZSTD_estimateCDictSize_advanced ( ) makes it possible to control compression parameters precisely , like ZSTD_createCDict_advanced ( ) .
* Note : dictionaries created by reference ( ` ZSTD_dlm_byRef ` ) are logically smaller .
2017-08-29 11:55:02 -07:00
*/
2017-06-26 16:47:32 -07:00
ZSTDLIB_API size_t ZSTD_estimateCDictSize ( size_t dictSize , int compressionLevel ) ;
2017-08-29 11:55:02 -07:00
ZSTDLIB_API size_t ZSTD_estimateCDictSize_advanced ( size_t dictSize , ZSTD_compressionParameters cParams , ZSTD_dictLoadMethod_e dictLoadMethod ) ;
ZSTDLIB_API size_t ZSTD_estimateDDictSize ( size_t dictSize , ZSTD_dictLoadMethod_e dictLoadMethod ) ;
2017-05-09 15:11:30 -07:00
2018-01-10 08:47:19 -08:00
/*! ZSTD_initStatic*() :
* Initialize an object using a pre - allocated fixed - size buffer .
* workspace : The memory area to emplace the object into .
* Provided pointer * must be 8 - bytes aligned * .
* Buffer must outlive object .
* workspaceSize : Use ZSTD_estimate * Size ( ) to determine
* how large workspace must be to support target scenario .
* @ return : pointer to object ( same address as workspace , just different type ) ,
* or NULL if error ( size too small , incorrect alignment , etc . )
* Note : zstd will never resize nor malloc ( ) when using a static buffer .
* If the object requires more memory than available ,
* zstd will just error out ( typically ZSTD_error_memory_allocation ) .
* Note 2 : there is no corresponding " free " function .
* Since workspace is allocated externally , it must be freed externally too .
* Note 3 : cParams : use ZSTD_getCParams ( ) to convert a compression level
* into its associated cParams .
* Limitation 1 : currently not compatible with internal dictionary creation , triggered by
* ZSTD_CCtx_loadDictionary ( ) , ZSTD_initCStream_usingDict ( ) or ZSTD_initDStream_usingDict ( ) .
* Limitation 2 : static cctx currently not compatible with multi - threading .
* Limitation 3 : static dctx is incompatible with legacy support .
*/
ZSTDLIB_API ZSTD_CCtx * ZSTD_initStaticCCtx ( void * workspace , size_t workspaceSize ) ;
ZSTDLIB_API ZSTD_CStream * ZSTD_initStaticCStream ( void * workspace , size_t workspaceSize ) ; /**< same as ZSTD_initStaticCCtx() */
ZSTDLIB_API ZSTD_DCtx * ZSTD_initStaticDCtx ( void * workspace , size_t workspaceSize ) ;
ZSTDLIB_API ZSTD_DStream * ZSTD_initStaticDStream ( void * workspace , size_t workspaceSize ) ; /**< same as ZSTD_initStaticDCtx() */
changed initStatic?Dict() return type to const ZSTD_?Dict*
ZSTD_create?Dict() is required to produce a ?Dict* return type
because `free()` does not accept a `const type*` argument.
If it wasn't for this restriction, I would have preferred to create a `const ?Dict*` object
to emphasize the fact that, once created, a dictionary never changes
(hence can be shared concurrently until the end of its lifetime).
There is no such limitation with initStatic?Dict() :
as stated in the doc, there is no corresponding free() function,
since `workspace` is provided, hence allocated, externally,
it can only be free() externally.
Which means, ZSTD_initStatic?Dict() can return a `const ZSTD_?Dict*` pointer.
Tested with `make all`, to catch initStatic's users,
which, incidentally, also updated zstd.h documentation.
2018-01-17 14:08:48 -08:00
ZSTDLIB_API const ZSTD_CDict * ZSTD_initStaticCDict (
void * workspace , size_t workspaceSize ,
const void * dict , size_t dictSize ,
ZSTD_dictLoadMethod_e dictLoadMethod ,
ZSTD_dictMode_e dictMode ,
ZSTD_compressionParameters cParams ) ;
ZSTDLIB_API const ZSTD_DDict * ZSTD_initStaticDDict (
void * workspace , size_t workspaceSize ,
const void * dict , size_t dictSize ,
ZSTD_dictLoadMethod_e dictLoadMethod ) ;
2018-01-10 08:47:19 -08:00
2018-01-10 09:07:47 -08:00
/*! Custom memory allocation :
* These prototypes make it possible to pass your own allocation / free functions .
* ZSTD_customMem is provided at creation time , using ZSTD_create * _advanced ( ) variants listed below .
* All allocation / free operations will be completed using these custom variants instead of regular < stdlib . h > ones .
*/
typedef void * ( * ZSTD_allocFunction ) ( void * opaque , size_t size ) ;
typedef void ( * ZSTD_freeFunction ) ( void * opaque , void * address ) ;
typedef struct { ZSTD_allocFunction customAlloc ; ZSTD_freeFunction customFree ; void * opaque ; } ZSTD_customMem ;
static ZSTD_customMem const ZSTD_defaultCMem = { NULL , NULL , NULL } ; /**< this constant defers to stdlib's functions */
ZSTDLIB_API ZSTD_CCtx * ZSTD_createCCtx_advanced ( ZSTD_customMem customMem ) ;
ZSTDLIB_API ZSTD_CStream * ZSTD_createCStream_advanced ( ZSTD_customMem customMem ) ;
ZSTDLIB_API ZSTD_DCtx * ZSTD_createDCtx_advanced ( ZSTD_customMem customMem ) ;
ZSTDLIB_API ZSTD_DStream * ZSTD_createDStream_advanced ( ZSTD_customMem customMem ) ;
ZSTDLIB_API ZSTD_CDict * ZSTD_createCDict_advanced ( const void * dict , size_t dictSize ,
ZSTD_dictLoadMethod_e dictLoadMethod ,
ZSTD_dictMode_e dictMode ,
ZSTD_compressionParameters cParams ,
ZSTD_customMem customMem ) ;
ZSTDLIB_API ZSTD_DDict * ZSTD_createDDict_advanced ( const void * dict , size_t dictSize ,
ZSTD_dictLoadMethod_e dictLoadMethod ,
ZSTD_customMem customMem ) ;
2018-01-10 08:47:19 -08:00
2016-07-10 18:12:17 -07:00
2016-10-06 04:23:52 -07:00
/***************************************
2016-06-06 15:51:51 -07:00
* Advanced compression functions
2016-06-04 10:47:02 -07:00
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
2016-12-21 07:20:11 -08:00
/*! ZSTD_createCDict_byReference() :
* Create a digested dictionary for compression
* Dictionary content is simply referenced , and therefore stays in dictBuffer .
* It is important that dictBuffer outlives CDict , it must remain read accessible throughout the lifetime of CDict */
ZSTDLIB_API ZSTD_CDict * ZSTD_createCDict_byReference ( const void * dictBuffer , size_t dictSize , int compressionLevel ) ;
2016-12-13 08:59:55 -08:00
/*! ZSTD_getCParams() :
* @ return ZSTD_compressionParameters structure for a selected compression level and estimated srcSize .
* ` estimatedSrcSize ` value is optional , select 0 if not known */
ZSTDLIB_API ZSTD_compressionParameters ZSTD_getCParams ( int compressionLevel , unsigned long long estimatedSrcSize , size_t dictSize ) ;
2016-06-27 06:28:45 -07:00
/*! ZSTD_getParams() :
2016-12-13 08:59:55 -08:00
* same as ZSTD_getCParams ( ) , but @ return a full ` ZSTD_parameters ` object instead of sub - component ` ZSTD_compressionParameters ` .
2017-10-18 11:18:27 -07:00
* All fields of ` ZSTD_frameParameters ` are set to default : contentSize = 1 , checksum = 0 , noDictID = 0 */
2016-12-13 08:59:55 -08:00
ZSTDLIB_API ZSTD_parameters ZSTD_getParams ( int compressionLevel , unsigned long long estimatedSrcSize , size_t dictSize ) ;
2016-06-04 10:47:02 -07:00
2016-06-27 06:12:26 -07:00
/*! ZSTD_checkCParams() :
2016-06-04 10:47:02 -07:00
* Ensure param values remain within authorized range */
ZSTDLIB_API size_t ZSTD_checkCParams ( ZSTD_compressionParameters params ) ;
2016-06-27 06:12:26 -07:00
/*! ZSTD_adjustCParams() :
2017-06-16 17:29:35 -07:00
* optimize params for a given ` srcSize ` and ` dictSize ` .
* both values are optional , select ` 0 ` if unknown . */
2016-07-07 02:53:18 -07:00
ZSTDLIB_API ZSTD_compressionParameters ZSTD_adjustCParams ( ZSTD_compressionParameters cPar , unsigned long long srcSize , size_t dictSize ) ;
2016-06-04 10:47:02 -07:00
/*! ZSTD_compress_advanced() :
2017-04-27 11:31:55 -07:00
* Same as ZSTD_compress_usingDict ( ) , with fine - tune control over each compression parameter */
ZSTDLIB_API size_t ZSTD_compress_advanced ( ZSTD_CCtx * cctx ,
void * dst , size_t dstCapacity ,
const void * src , size_t srcSize ,
const void * dict , size_t dictSize ,
ZSTD_parameters params ) ;
/*! ZSTD_compress_usingCDict_advanced() :
* Same as ZSTD_compress_usingCDict ( ) , with fine - tune control over frame parameters */
ZSTDLIB_API size_t ZSTD_compress_usingCDict_advanced ( ZSTD_CCtx * cctx ,
void * dst , size_t dstCapacity ,
const void * src , size_t srcSize ,
const ZSTD_CDict * cdict , ZSTD_frameParameters fParams ) ;
2016-06-04 10:47:02 -07:00
2016-06-14 04:46:11 -07:00
2016-10-24 06:59:51 -07:00
/*--- Advanced decompression functions ---*/
2016-06-04 10:47:02 -07:00
2016-11-02 17:30:49 -07:00
/*! ZSTD_isFrame() :
* Tells if the content of ` buffer ` starts with a valid Frame Identifier .
* Note : Frame Identifier is 4 bytes . If ` size < 4 ` , @ return will always be 0.
* Note 2 : Legacy Frame Identifiers are considered valid only if Legacy Support is enabled .
* Note 3 : Skippable Frame Identifiers are considered valid . */
ZSTDLIB_API unsigned ZSTD_isFrame ( const void * buffer , size_t size ) ;
2016-12-21 07:44:35 -08:00
/*! ZSTD_createDDict_byReference() :
* Create a digested dictionary , ready to start decompression operation without startup delay .
2017-05-25 15:44:06 -07:00
* Dictionary content is referenced , and therefore stays in dictBuffer .
* It is important that dictBuffer outlives DDict ,
* it must remain read accessible throughout the lifetime of DDict */
2016-12-21 07:44:35 -08:00
ZSTDLIB_API ZSTD_DDict * ZSTD_createDDict_byReference ( const void * dictBuffer , size_t dictSize ) ;
2016-12-05 16:21:06 -08:00
/*! ZSTD_getDictID_fromDict() :
* Provides the dictID stored within dictionary .
* if @ return = = 0 , the dictionary is not conformant with Zstandard specification .
* It can still be loaded , but as a content - only dictionary . */
2016-12-16 13:27:30 -08:00
ZSTDLIB_API unsigned ZSTD_getDictID_fromDict ( const void * dict , size_t dictSize ) ;
2016-12-05 16:21:06 -08:00
/*! ZSTD_getDictID_fromDDict() :
* Provides the dictID of the dictionary loaded into ` ddict ` .
* If @ return = = 0 , the dictionary is not conformant to Zstandard specification , or empty .
* Non - conformant dictionaries can still be loaded , but as content - only dictionaries . */
2016-12-16 13:27:30 -08:00
ZSTDLIB_API unsigned ZSTD_getDictID_fromDDict ( const ZSTD_DDict * ddict ) ;
2016-12-05 16:21:06 -08:00
/*! ZSTD_getDictID_fromFrame() :
* Provides the dictID required to decompressed the frame stored within ` src ` .
* If @ return = = 0 , the dictID could not be decoded .
* This could for one of the following reasons :
* - The frame does not require a dictionary to be decoded ( most common case ) .
* - The frame was built with dictID intentionally removed . Whatever dictionary is necessary is a hidden information .
* Note : this use case also happens when using a non - conformant dictionary .
* - ` srcSize ` is too small , and as a result , the frame header could not be decoded ( only possible if ` srcSize < ZSTD_FRAMEHEADERSIZE_MAX ` ) .
* - This is not a Zstandard frame .
2017-05-25 18:29:08 -07:00
* When identifying the exact failure cause , it ' s possible to use ZSTD_getFrameHeader ( ) , which will provide a more precise error code . */
2016-12-16 13:27:30 -08:00
ZSTDLIB_API unsigned ZSTD_getDictID_fromFrame ( const void * src , size_t srcSize ) ;
2016-12-05 16:21:06 -08:00
2016-06-04 10:47:02 -07:00
2016-10-06 04:23:52 -07:00
/********************************************************************
2016-10-24 06:59:51 -07:00
* Advanced streaming functions
2016-08-11 16:20:36 -07:00
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
2016-10-06 04:23:52 -07:00
/*===== Advanced Streaming compression functions =====*/
2017-12-07 14:13:10 -08:00
ZSTDLIB_API size_t ZSTD_initCStream_srcSize ( ZSTD_CStream * zcs , int compressionLevel , unsigned long long pledgedSrcSize ) ; /**< pledgedSrcSize must be correct. If it is not known at init time, use ZSTD_CONTENTSIZE_UNKNOWN. Note that, for compatibility with older programs, "0" also disables frame content size field. It may be enabled in the future. */
2017-08-30 14:36:54 -07:00
ZSTDLIB_API size_t ZSTD_initCStream_usingDict ( ZSTD_CStream * zcs , const void * dict , size_t dictSize , int compressionLevel ) ; /**< creates of an internal CDict (incompatible with static CCtx), except if dict == NULL or dictSize < 8, in which case no dict is used. Note: dict is loaded with ZSTD_dm_auto (treated as a full zstd dictionary if it begins with ZSTD_MAGIC_DICTIONARY, else as raw content) and ZSTD_dlm_byCopy.*/
2016-08-22 16:18:06 -07:00
ZSTDLIB_API size_t ZSTD_initCStream_advanced ( ZSTD_CStream * zcs , const void * dict , size_t dictSize ,
2017-12-08 09:16:49 -08:00
ZSTD_parameters params , unsigned long long pledgedSrcSize ) ; /**< pledgedSrcSize must be correct. If srcSize is not known at init time, use value ZSTD_CONTENTSIZE_UNKNOWN. dict is loaded with ZSTD_dm_auto and ZSTD_dlm_byCopy. */
2016-10-25 16:19:52 -07:00
ZSTDLIB_API size_t ZSTD_initCStream_usingCDict ( ZSTD_CStream * zcs , const ZSTD_CDict * cdict ) ; /**< note : cdict will just be referenced, and must outlive compression session */
2017-12-08 09:16:49 -08:00
ZSTDLIB_API size_t ZSTD_initCStream_usingCDict_advanced ( ZSTD_CStream * zcs , const ZSTD_CDict * cdict , ZSTD_frameParameters fParams , unsigned long long pledgedSrcSize ) ; /**< same as ZSTD_initCStream_usingCDict(), with control over frame parameters. pledgedSrcSize must be correct. If srcSize is not known at init time, use value ZSTD_CONTENTSIZE_UNKNOWN. */
2017-04-05 22:06:21 -07:00
/*! ZSTD_resetCStream() :
* start a new compression job , using same parameters from previous job .
* This is typically useful to skip dictionary loading stage , since it will re - use it in - place . .
* Note that zcs must be init at least once before using ZSTD_resetCStream ( ) .
2017-10-14 00:32:06 -07:00
* If pledgedSrcSize is not known at reset time , use macro ZSTD_CONTENTSIZE_UNKNOWN .
2017-04-05 22:06:21 -07:00
* If pledgedSrcSize > 0 , its value must be correct , as it will be written in header , and controlled at the end .
2017-10-14 00:32:06 -07:00
* For the time being , pledgedSrcSize = = 0 is interpreted as " srcSize unknown " for compatibility with older programs ,
2018-01-17 16:39:02 -08:00
* but it will change to mean " empty " in future version , so use macro ZSTD_CONTENTSIZE_UNKNOWN instead .
2017-10-14 00:32:06 -07:00
* @ return : 0 , or an error code ( which can be tested using ZSTD_isError ( ) ) */
2017-04-05 22:06:21 -07:00
ZSTDLIB_API size_t ZSTD_resetCStream ( ZSTD_CStream * zcs , unsigned long long pledgedSrcSize ) ;
2016-08-22 15:30:31 -07:00
2016-08-11 16:20:36 -07:00
2018-01-17 16:39:02 -08:00
typedef struct {
unsigned long long ingested ;
unsigned long long consumed ;
unsigned long long produced ;
} ZSTD_frameProgression ;
/* ZSTD_getFrameProgression():
2018-01-18 11:15:23 -08:00
* tells how much data has been ingested ( read from input )
* consumed ( input actually compressed ) and produced ( output ) for current frame .
* Therefore , ( ingested - consumed ) is amount of input data buffered internally , not yet compressed .
* Can report progression inside worker threads ( multi - threading and non - blocking mode ) .
2018-01-17 16:39:02 -08:00
*/
ZSTD_frameProgression ZSTD_getFrameProgression ( const ZSTD_CCtx * cctx ) ;
2016-10-06 04:23:52 -07:00
/*===== Advanced Streaming decompression functions =====*/
2017-09-24 15:57:29 -07:00
typedef enum { DStream_p_maxWindowSize } ZSTD_DStreamParameter_e ;
2017-09-25 16:21:17 -07:00
ZSTDLIB_API size_t ZSTD_setDStreamParameter ( ZSTD_DStream * zds , ZSTD_DStreamParameter_e paramType , unsigned paramValue ) ; /* obsolete : this API will be removed in a future version */
2017-09-24 15:57:29 -07:00
ZSTDLIB_API size_t ZSTD_initDStream_usingDict ( ZSTD_DStream * zds , const void * dict , size_t dictSize ) ; /**< note: no dictionary will be used if dict == NULL or dictSize < 8 */
ZSTDLIB_API size_t ZSTD_initDStream_usingDDict ( ZSTD_DStream * zds , const ZSTD_DDict * ddict ) ; /**< note : ddict is referenced, it must outlive decompression session */
2016-09-15 05:54:07 -07:00
ZSTDLIB_API size_t ZSTD_resetDStream ( ZSTD_DStream * zds ) ; /**< re-use decompression parameters from previous init; saves dictionary loading */
2016-08-11 16:20:36 -07:00
2016-10-06 04:23:52 -07:00
/*********************************************************************
2016-08-11 16:20:36 -07:00
* Buffer - less and synchronous inner streaming functions
2016-10-06 04:23:52 -07:00
*
* This is an advanced API , giving full control over buffer management , for users which need direct control over memory .
2017-09-24 16:47:02 -07:00
* But it ' s also a complex one , with several restrictions , documented below .
* Prefer normal streaming API for an easier experience .
2016-10-06 04:23:52 -07:00
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
2016-07-28 10:55:09 -07:00
2016-10-06 04:23:52 -07:00
/**
Buffer - less streaming compression ( synchronous mode )
2016-06-04 10:47:02 -07:00
A ZSTD_CCtx object is required to track streaming operations .
2016-06-14 04:46:11 -07:00
Use ZSTD_createCCtx ( ) / ZSTD_freeCCtx ( ) to manage resource .
2016-06-04 10:47:02 -07:00
ZSTD_CCtx object can be re - used multiple times within successive compression operations .
Start by initializing a context .
Use ZSTD_compressBegin ( ) , or ZSTD_compressBegin_usingDict ( ) for dictionary compression ,
or ZSTD_compressBegin_advanced ( ) , for finer parameter control .
It ' s also possible to duplicate a reference context which has already been initialized , using ZSTD_copyCCtx ( )
Then , consume your input using ZSTD_compressContinue ( ) .
2016-06-21 02:54:03 -07:00
There are some important considerations to keep in mind when using this advanced function :
2017-09-24 16:47:02 -07:00
- ZSTD_compressContinue ( ) has no internal buffer . It uses externally provided buffers only .
- Interface is synchronous : input is consumed entirely and produces 1 + compressed blocks .
2016-06-21 02:54:03 -07:00
- Caller must ensure there is enough space in ` dst ` to store compressed data under worst case scenario .
Worst case evaluation is provided by ZSTD_compressBound ( ) .
ZSTD_compressContinue ( ) doesn ' t guarantee recover after a failed compression .
- ZSTD_compressContinue ( ) presumes prior input * * * is still accessible and unmodified * * * ( up to maximum distance size , see WindowLog ) .
It remembers all previous contiguous blocks , plus one separated memory segment ( which can itself consists of multiple contiguous blocks )
- ZSTD_compressContinue ( ) detects that prior input has been overwritten when ` src ` buffer overlaps .
In which case , it will " discard " the relevant memory section from its history .
2016-07-28 06:29:08 -07:00
Finish a frame with ZSTD_compressEnd ( ) , which will write the last block ( s ) and optional checksum .
2017-01-11 09:21:25 -08:00
It ' s possible to use srcSize = = 0 , in which case , it will write a final empty block to end the frame .
2017-09-24 16:47:02 -07:00
Without last block mark , frames are considered unfinished ( hence corrupted ) by compliant decoders .
2016-06-04 10:47:02 -07:00
2017-09-24 16:47:02 -07:00
` ZSTD_CCtx ` object can be re - used ( ZSTD_compressBegin ( ) ) to compress again .
2016-06-04 10:47:02 -07:00
*/
2016-10-06 04:23:52 -07:00
/*===== Buffer-less streaming compression functions =====*/
ZSTDLIB_API size_t ZSTD_compressBegin ( ZSTD_CCtx * cctx , int compressionLevel ) ;
ZSTDLIB_API size_t ZSTD_compressBegin_usingDict ( ZSTD_CCtx * cctx , const void * dict , size_t dictSize , int compressionLevel ) ;
2017-10-14 00:32:06 -07:00
ZSTDLIB_API size_t ZSTD_compressBegin_advanced ( ZSTD_CCtx * cctx , const void * dict , size_t dictSize , ZSTD_parameters params , unsigned long long pledgedSrcSize ) ; /**< pledgedSrcSize : If srcSize is not known at init time, use ZSTD_CONTENTSIZE_UNKNOWN */
2017-04-26 15:42:10 -07:00
ZSTDLIB_API size_t ZSTD_compressBegin_usingCDict ( ZSTD_CCtx * cctx , const ZSTD_CDict * cdict ) ; /**< note: fails if cdict==NULL */
2017-10-14 00:32:06 -07:00
ZSTDLIB_API size_t ZSTD_compressBegin_usingCDict_advanced ( ZSTD_CCtx * const cctx , const ZSTD_CDict * const cdict , ZSTD_frameParameters const fParams , unsigned long long const pledgedSrcSize ) ; /* compression parameters are already set within cdict. pledgedSrcSize must be correct. If srcSize is not known, use macro ZSTD_CONTENTSIZE_UNKNOWN */
ZSTDLIB_API size_t ZSTD_copyCCtx ( ZSTD_CCtx * cctx , const ZSTD_CCtx * preparedCCtx , unsigned long long pledgedSrcSize ) ; /**< note: if pledgedSrcSize is not known, use ZSTD_CONTENTSIZE_UNKNOWN */
2017-04-18 13:55:53 -07:00
2016-10-06 04:23:52 -07:00
ZSTDLIB_API size_t ZSTD_compressContinue ( ZSTD_CCtx * cctx , void * dst , size_t dstCapacity , const void * src , size_t srcSize ) ;
ZSTDLIB_API size_t ZSTD_compressEnd ( ZSTD_CCtx * cctx , void * dst , size_t dstCapacity , const void * src , size_t srcSize ) ;
2016-06-04 10:47:02 -07:00
2016-10-06 05:22:48 -07:00
/*-
2016-07-18 07:52:10 -07:00
Buffer - less streaming decompression ( synchronous mode )
2016-06-04 10:47:02 -07:00
A ZSTD_DCtx object is required to track streaming operations .
Use ZSTD_createDCtx ( ) / ZSTD_freeDCtx ( ) to manage it .
A ZSTD_DCtx object can be re - used multiple times .
2017-05-25 18:29:08 -07:00
First typical operation is to retrieve frame parameters , using ZSTD_getFrameHeader ( ) .
2017-06-26 17:44:26 -07:00
Frame header is extracted from the beginning of compressed frame , so providing only the frame ' s beginning is enough .
Data fragment must be large enough to ensure successful decoding .
2017-09-09 01:03:29 -07:00
` ZSTD_frameHeaderSize_max ` bytes is guaranteed to always be large enough .
2017-06-26 17:44:26 -07:00
@ result : 0 : successful decoding , the ` ZSTD_frameHeader ` structure is correctly filled .
2016-07-29 10:40:37 -07:00
> 0 : ` srcSize ` is too small , please provide at least @ result bytes on next attempt .
2016-07-27 18:47:45 -07:00
errorCode , which can be tested using ZSTD_isError ( ) .
2016-06-04 10:47:02 -07:00
2017-09-09 01:03:29 -07:00
It fills a ZSTD_frameHeader structure with important information to correctly decode the frame ,
such as the dictionary ID , content size , or maximum back - reference distance ( ` windowSize ` ) .
Note that these values could be wrong , either because of data corruption , or because a 3 rd party deliberately spoofs false information .
As a consequence , check that values remain within valid application range .
For example , do not allocate memory blindly , check that ` windowSize ` is within expectation .
Each application can set its own limits , depending on local restrictions .
For extended interoperability , it is recommended to support ` windowSize ` of at least 8 MB .
ZSTD_decompressContinue ( ) needs previous data blocks during decompression , up to ` windowSize ` bytes .
ZSTD_decompressContinue ( ) is very sensitive to contiguity ,
if 2 blocks don ' t follow each other , make sure that either the compressor breaks contiguity at the same place ,
or that previous contiguous segment is large enough to properly handle maximum back - reference distance .
There are multiple ways to guarantee this condition .
The most memory efficient way is to use a round buffer of sufficient size .
Sufficient size is determined by invoking ZSTD_decodingBufferSize_min ( ) ,
which can @ return an error code if required value is too large for current system ( in 32 - bits mode ) .
In a round buffer methodology , ZSTD_decompressContinue ( ) decompresses each block next to previous one ,
up to the moment there is not enough room left in the buffer to guarantee decoding another full block ,
which maximum size is provided in ` ZSTD_frameHeader ` structure , field ` blockSizeMax ` .
At which point , decoding can resume from the beginning of the buffer .
Note that already decoded data stored in the buffer should be flushed before being overwritten .
There are alternatives possible , for example using two or more buffers of size ` windowSize ` each , though they consume more memory .
Finally , if you control the compression process , you can also ignore all buffer size rules ,
as long as the encoder and decoder progress in " lock-step " ,
aka use exactly the same buffer sizes , break contiguity at the same place , etc .
Once buffers are setup , start decompression , with ZSTD_decompressBegin ( ) .
2017-05-16 16:05:27 -07:00
If decompression requires a dictionary , use ZSTD_decompressBegin_usingDict ( ) or ZSTD_decompressBegin_usingDDict ( ) .
2016-06-04 10:47:02 -07:00
Then use ZSTD_nextSrcSizeToDecompress ( ) and ZSTD_decompressContinue ( ) alternatively .
2016-07-29 10:40:37 -07:00
ZSTD_nextSrcSizeToDecompress ( ) tells how many bytes to provide as ' srcSize ' to ZSTD_decompressContinue ( ) .
ZSTD_decompressContinue ( ) requires this _exact_ amount of bytes , or it will fail .
2016-06-04 10:47:02 -07:00
2017-09-09 01:03:29 -07:00
@ result of ZSTD_decompressContinue ( ) is the number of bytes regenerated within ' dst ' ( necessarily < = dstCapacity ) .
It can be zero : it just means ZSTD_decompressContinue ( ) has decoded some metadata item .
2016-07-29 10:40:37 -07:00
It can also be an error code , which can be tested with ZSTD_isError ( ) .
2016-06-04 10:47:02 -07:00
A frame is fully decoded when ZSTD_nextSrcSizeToDecompress ( ) returns zero .
Context can then be reset to start a new decompression .
2016-07-28 11:30:25 -07:00
Note : it ' s possible to know if next input to present is a header or a block , using ZSTD_nextInputType ( ) .
2016-07-29 10:40:37 -07:00
This information is not required to properly decode a frame .
2016-07-08 14:42:22 -07:00
2016-12-13 07:39:36 -08:00
= = Special case : skippable frames = =
2016-07-08 14:42:22 -07:00
2016-07-27 18:47:45 -07:00
Skippable frames allow integration of user - defined data into a flow of concatenated frames .
2017-09-09 01:03:29 -07:00
Skippable frames will be ignored ( skipped ) by decompressor .
The format of skippable frames is as follows :
2016-06-04 10:47:02 -07:00
a ) Skippable frame ID - 4 Bytes , Little endian format , any value from 0x184D2A50 to 0x184D2A5F
b ) Frame Size - 4 Bytes , Little endian format , unsigned 32 - bits
c ) Frame Content - any content ( User Data ) of length equal to Frame Size
2017-09-09 01:03:29 -07:00
For skippable frames ZSTD_getFrameHeader ( ) returns zfhPtr - > frameType = = ZSTD_skippableFrame .
For skippable frames ZSTD_decompressContinue ( ) always returns 0 : it only skips the content .
2016-06-04 10:47:02 -07:00
*/
2016-10-06 04:23:52 -07:00
/*===== Buffer-less streaming decompression functions =====*/
2017-07-07 15:51:24 -07:00
typedef enum { ZSTD_frame , ZSTD_skippableFrame } ZSTD_frameType_e ;
typedef struct {
2017-09-09 01:03:29 -07:00
unsigned long long frameContentSize ; /* if == ZSTD_CONTENTSIZE_UNKNOWN, it means this field is not available. 0 means "empty" */
2017-07-07 15:51:24 -07:00
unsigned long long windowSize ; /* can be very large, up to <= frameContentSize */
2017-09-09 01:03:29 -07:00
unsigned blockSizeMax ;
2017-07-07 15:51:24 -07:00
ZSTD_frameType_e frameType ; /* if == ZSTD_skippableFrame, frameContentSize is the size of skippable content */
unsigned headerSize ;
unsigned dictID ;
unsigned checksumFlag ;
} ZSTD_frameHeader ;
2017-06-26 17:44:26 -07:00
ZSTDLIB_API size_t ZSTD_getFrameHeader ( ZSTD_frameHeader * zfhPtr , const void * src , size_t srcSize ) ; /**< doesn't consume input */
2017-09-11 13:12:52 -07:00
ZSTDLIB_API size_t ZSTD_decodingBufferSize_min ( unsigned long long windowSize , unsigned long long frameContentSize ) ; /**< when frame content size is not known, pass in frameContentSize == ZSTD_CONTENTSIZE_UNKNOWN */
2017-09-09 01:03:29 -07:00
2016-10-06 04:23:52 -07:00
ZSTDLIB_API size_t ZSTD_decompressBegin ( ZSTD_DCtx * dctx ) ;
ZSTDLIB_API size_t ZSTD_decompressBegin_usingDict ( ZSTD_DCtx * dctx , const void * dict , size_t dictSize ) ;
2017-05-16 16:05:27 -07:00
ZSTDLIB_API size_t ZSTD_decompressBegin_usingDDict ( ZSTD_DCtx * dctx , const ZSTD_DDict * ddict ) ;
2016-10-06 04:23:52 -07:00
ZSTDLIB_API size_t ZSTD_nextSrcSizeToDecompress ( ZSTD_DCtx * dctx ) ;
ZSTDLIB_API size_t ZSTD_decompressContinue ( ZSTD_DCtx * dctx , void * dst , size_t dstCapacity , const void * src , size_t srcSize ) ;
2017-09-09 01:03:29 -07:00
/* misc */
ZSTDLIB_API void ZSTD_copyDCtx ( ZSTD_DCtx * dctx , const ZSTD_DCtx * preparedDCtx ) ;
2016-10-06 04:23:52 -07:00
typedef enum { ZSTDnit_frameHeader , ZSTDnit_blockHeader , ZSTDnit_block , ZSTDnit_lastBlock , ZSTDnit_checksum , ZSTDnit_skippableFrame } ZSTD_nextInputType_e ;
ZSTDLIB_API ZSTD_nextInputType_e ZSTD_nextInputType ( ZSTD_DCtx * dctx ) ;
2017-06-19 12:56:25 -07:00
2017-09-25 15:41:48 -07:00
/* ============================================ */
/** New advanced API (experimental) */
/* ============================================ */
2017-06-19 12:56:25 -07:00
/* notes on API design :
2017-09-24 15:57:29 -07:00
* In this proposal , parameters are pushed one by one into an existing context ,
2017-06-19 12:56:25 -07:00
* and then applied on all subsequent compression jobs .
* When no parameter is ever provided , CCtx is created with compression level ZSTD_CLEVEL_DEFAULT .
*
* This API is intended to replace all others experimental API .
* It can basically do all other use cases , and even new ones .
2018-01-10 09:07:47 -08:00
* It stands a reasonable chance to become " stable " , after a good testing period .
2017-06-19 12:56:25 -07:00
*/
/* note on naming convention :
* Initially , the API favored names like ZSTD_setCCtxParameter ( ) .
* In this proposal , convention is changed towards ZSTD_CCtx_setParameter ( ) .
2017-06-21 11:50:33 -07:00
* The main driver is that it identifies more clearly the target object type .
2017-09-27 10:29:31 -07:00
* It feels clearer when considering multiple targets :
2017-06-19 12:56:25 -07:00
* ZSTD_CDict_setParameter ( ) ( rather than ZSTD_setCDictParameter ( ) )
2017-09-21 16:07:29 -07:00
* ZSTD_CCtxParams_setParameter ( ) ( rather than ZSTD_setCCtxParamsParameter ( ) )
2017-09-25 16:21:17 -07:00
* etc . . .
2017-06-19 12:56:25 -07:00
*/
2017-06-21 11:50:33 -07:00
/* note on enum design :
2017-09-25 16:21:17 -07:00
* All enum will be pinned to explicit values before reaching " stable API " status */
2017-06-21 11:50:33 -07:00
2017-06-19 12:56:25 -07:00
typedef enum {
2017-09-27 10:29:31 -07:00
/* Question : should we have a format ZSTD_f_auto ?
* For the time being , it would mean exactly the same as ZSTD_f_zstd1 .
* But , in the future , should several formats be supported ,
* on the compression side , it would mean " default format " .
* On the decompression side , it would mean " multi format " ,
* and ZSTD_f_zstd1 could be reserved to mean " accept *only* zstd frames " .
* Since meaning is a little different , another option could be to define different enums for compression and decompression .
* This question could be kept for later , when there are actually multiple formats to support ,
* but there is also the question of pinning enum values , and pinning value ` 0 ` is especially important */
ZSTD_f_zstd1 = 0 , /* zstd frame format, specified in zstd_compression_format.md (default) */
2017-09-24 15:57:29 -07:00
ZSTD_f_zstd1_magicless , /* Variant of zstd frame format, without initial 4-bytes magic number.
* Useful to save 4 bytes per generated frame .
2017-09-27 10:29:31 -07:00
* Decoder cannot recognise automatically this format , requiring instructions . */
2017-09-24 15:57:29 -07:00
} ZSTD_format_e ;
2017-09-21 16:07:29 -07:00
typedef enum {
/* compression format */
2017-09-24 15:57:29 -07:00
ZSTD_p_format = 10 , /* See ZSTD_format_e enum definition.
* Cast selected format as unsigned for ZSTD_CCtx_setParameter ( ) compatibility . */
2017-09-21 16:07:29 -07:00
2017-06-19 12:56:25 -07:00
/* compression parameters */
ZSTD_p_compressionLevel = 100 , /* Update all compression parameters according to pre-defined cLevel table
* Default level is ZSTD_CLEVEL_DEFAULT = = 3.
2018-03-11 19:56:48 -07:00
* Special : value 0 means " do not change cLevel " .
* Note 1 : it ' s possible to pass a negative compression level by casting it to unsigned type .
2018-03-12 11:44:10 -07:00
* Note 2 : setting compressionLevel automatically updates ZSTD_p_compressLiterals . */
2017-06-19 12:56:25 -07:00
ZSTD_p_windowLog , /* Maximum allowed back-reference distance, expressed as power of 2.
* Must be clamped between ZSTD_WINDOWLOG_MIN and ZSTD_WINDOWLOG_MAX .
2017-09-22 14:04:39 -07:00
* Special : value 0 means " do not change windowLog " .
* Note : Using a window size greater than ZSTD_MAXWINDOWSIZE_DEFAULT ( default : 2 ^ 27 )
* requires setting the maximum window size at least as large during decompression . */
2017-06-19 12:56:25 -07:00
ZSTD_p_hashLog , /* Size of the probe table, as a power of 2.
* Resulting table size is ( 1 < < ( hashLog + 2 ) ) .
* Must be clamped between ZSTD_HASHLOG_MIN and ZSTD_HASHLOG_MAX .
* Larger tables improve compression ratio of strategies < = dFast ,
* and improve speed of strategies > dFast .
* Special : value 0 means " do not change hashLog " . */
ZSTD_p_chainLog , /* Size of the full-search table, as a power of 2.
* Resulting table size is ( 1 < < ( chainLog + 2 ) ) .
* Larger tables result in better and slower compression .
* This parameter is useless when using " fast " strategy .
* Special : value 0 means " do not change chainLog " . */
ZSTD_p_searchLog , /* Number of search attempts, as a power of 2.
* More attempts result in better and slower compression .
* This parameter is useless when using " fast " and " dFast " strategies .
* Special : value 0 means " do not change searchLog " . */
ZSTD_p_minMatch , /* Minimum size of searched matches (note : repCode matches can be smaller).
* Larger values make faster compression and decompression , but decrease ratio .
* Must be clamped between ZSTD_SEARCHLENGTH_MIN and ZSTD_SEARCHLENGTH_MAX .
* Note that currently , for all strategies < btopt , effective minimum is 4.
* Note that currently , for all strategies > fast , effective maximum is 6.
* Special : value 0 means " do not change minMatchLength " . */
2018-03-12 11:34:52 -07:00
ZSTD_p_targetLength , /* Impact of this field depends on strategy.
* For strategies btopt & btultra :
* Length of Match considered " good enough " to stop search .
* Larger values make compression stronger , and slower .
* For strategy fast :
* Distance between match sampling .
* Larger values make compression faster , and weaker .
2017-06-19 12:56:25 -07:00
* Special : value 0 means " do not change targetLength " . */
ZSTD_p_compressionStrategy , /* See ZSTD_strategy enum definition.
* Cast selected strategy as unsigned for ZSTD_CCtx_setParameter ( ) compatibility .
* The higher the value of selected strategy , the more complex it is ,
* resulting in stronger and slower compression .
* Special : value 0 means " do not change strategy " . */
/* frame parameters */
2017-10-13 17:39:13 -07:00
ZSTD_p_contentSizeFlag = 200 , /* Content size will be written into frame header _whenever known_ (default:1)
* Content size must be known at the beginning of compression ,
* it is provided using ZSTD_CCtx_setPledgedSrcSize ( ) */
2017-06-19 12:56:25 -07:00
ZSTD_p_checksumFlag , /* A 32-bits checksum of content is written at end of frame (default:0) */
2017-10-13 17:39:13 -07:00
ZSTD_p_dictIDFlag , /* When applicable, dictionary's ID is written into frame header (default:1) */
2017-06-19 12:56:25 -07:00
/* multi-threading parameters */
2018-01-16 16:15:47 -08:00
/* These parameters are only useful if multi-threading is enabled (ZSTD_MULTITHREAD).
* They return an error otherwise . */
2018-02-01 19:29:30 -08:00
ZSTD_p_nbWorkers = 400 , /* Select how many threads will be spawned to compress in parallel.
* When nbWorkers > = 1 , triggers asynchronous mode :
* ZSTD_compress_generic ( ) consumes some input , flush some output if possible , and immediately gives back control to caller ,
* while compression work is performed in parallel , within worker threads .
* ( note : a strong exception to this rule is when first invocation sets ZSTD_e_end : it becomes a blocking call ) .
* More workers improve speed , but also increase memory usage .
* Default value is ` 0 ` , aka " single-threaded mode " : no worker is spawned , compression is performed inside Caller ' s thread , all invocations are blocking */
2018-03-11 05:21:53 -07:00
ZSTD_p_jobSize , /* Size of a compression job. This value is enforced only in non-blocking mode.
* Each compression job is completed in parallel , so this value indirectly controls the nb of active threads .
2017-06-19 12:56:25 -07:00
* 0 means default , which is dynamically determined based on compression parameters .
2018-03-11 05:21:53 -07:00
* Job size must be a minimum of overlapSize , or 1 MB , whichever is largest .
2017-06-19 12:56:25 -07:00
* The minimum size is automatically and transparently enforced */
ZSTD_p_overlapSizeLog , /* Size of previous input reloaded at the beginning of each job.
* 0 = > no overlap , 6 ( default ) = > use 1 / 8 th of windowSize , > = 9 = > use full windowSize */
/* advanced parameters - may not remain available after API update */
2018-03-11 19:56:48 -07:00
2018-03-12 11:44:10 -07:00
ZSTD_p_compressLiterals = 1000 , /* control huffman compression of literals (enabled) by default.
2018-03-11 19:56:48 -07:00
* disabling it improves speed and decreases compression ratio by a large amount .
2018-03-13 14:23:14 -07:00
* note : this setting is automatically updated when changing compression level .
* positive compression levels set ZSTD_p_compressLiterals to 1.
* negative compression levels set ZSTD_p_compressLiterals to 0. */
2018-03-11 19:56:48 -07:00
2017-06-27 11:46:39 -07:00
ZSTD_p_forceMaxWindow = 1100 , /* Force back-reference distances to remain < windowSize,
* even when referencing into Dictionary content ( default : 0 ) */
2018-03-11 19:56:48 -07:00
2018-02-01 17:07:27 -08:00
ZSTD_p_enableLongDistanceMatching = 1200 , /* Enable long distance matching.
2017-09-05 11:05:57 -07:00
* This parameter is designed to improve the compression
* ratio for large inputs with long distance matches .
* This increases the memory usage as well as window size .
* Note : setting this parameter sets all the LDM parameters
* as well as ZSTD_p_windowLog . It should be set after
2017-09-02 21:10:36 -07:00
* ZSTD_p_compressionLevel and before ZSTD_p_windowLog and
* other LDM parameters . Setting the compression level
* after this parameter overrides the window log , though LDM
* will remain enabled until explicitly disabled . */
2018-02-01 17:07:27 -08:00
ZSTD_p_ldmHashLog , /* Size of the table for long distance matching, as a power of 2.
* Larger values increase memory usage and compression ratio , but decrease
* compression speed .
* Must be clamped between ZSTD_HASHLOG_MIN and ZSTD_HASHLOG_MAX
* ( default : windowlog - 7 ) .
* Special : value 0 means " do not change ldmHashLog " . */
ZSTD_p_ldmMinMatch , /* Minimum size of searched matches for long distance matcher.
* Larger / too small values usually decrease compression ratio .
* Must be clamped between ZSTD_LDM_MINMATCH_MIN
* and ZSTD_LDM_MINMATCH_MAX ( default : 64 ) .
* Special : value 0 means " do not change ldmMinMatch " . */
ZSTD_p_ldmBucketSizeLog , /* Log size of each bucket in the LDM hash table for collision resolution.
* Larger values usually improve collision resolution but may decrease
* compression speed .
* The maximum value is ZSTD_LDM_BUCKETSIZELOG_MAX ( default : 3 ) .
* note : 0 is a valid value */
2017-09-02 21:10:36 -07:00
ZSTD_p_ldmHashEveryLog , /* Frequency of inserting/looking up entries in the LDM hash table.
* The default is MAX ( 0 , ( windowLog - ldmHashLog ) ) to
* optimize hash table usage .
2017-09-05 11:05:57 -07:00
* Larger values improve compression speed . Deviating far from the
* default value will likely result in a decrease in compression ratio .
2018-02-01 17:07:27 -08:00
* Must be clamped between 0 and ZSTD_WINDOWLOG_MAX - ZSTD_HASHLOG_MIN .
* note : 0 is a valid value */
2017-06-27 11:46:39 -07:00
2017-06-19 12:56:25 -07:00
} ZSTD_cParameter ;
/*! ZSTD_CCtx_setParameter() :
* Set one compression parameter , selected by enum ZSTD_cParameter .
2018-02-02 14:24:56 -08:00
* Setting a parameter is generally only possible during frame initialization ( before starting compression ) ,
* except for a few exceptions which can be updated during compression : compressionLevel , hashLog , chainLog , searchLog , minMatch , targetLength and strategy .
2017-06-19 12:56:25 -07:00
* Note : when ` value ` is an enum , cast it to unsigned for proper type checking .
2018-02-02 14:24:56 -08:00
* @ result : informational value ( typically , value being set clamped correctly ) ,
Fixed Btree update
ZSTD_updateTree() expected to be followed by a Bt match finder, which would update zc->nextToUpdate.
With the new optimal match finder, it's not necessarily the case : a match might be found during repcode or hash3, and stops there because it reaches sufficient_len, without even entering the binary tree.
Previous policy was to nonetheless update zc->nextToUpdate, but the current position would not be inserted, creating "holes" in the btree, aka positions that will no longer be searched.
Now, when current position is not inserted, zc->nextToUpdate is not update, expecting ZSTD_updateTree() to fill the tree later on.
Solution selected is that ZSTD_updateTree() takes care of properly setting zc->nextToUpdate,
so that it no longer depends on a future function to do this job.
It took time to get there, as the issue started with a memory sanitizer error.
The pb would have been easier to spot with a proper `assert()`.
So this patch add a few of them.
Additionnally, I discovered that `make test` does not enable `assert()` during CLI tests.
This patch enables them.
Unfortunately, these `assert()` triggered other (unrelated) bugs during CLI tests, mostly within zstdmt.
So this patch also fixes them.
- Changed packed structure for gcc memory access : memory sanitizer would complain that a read "might" reach out-of-bound position on the ground that the `union` is larger than the type accessed.
Now, to avoid this issue, each type is independent.
- ZSTD_CCtxParams_setParameter() : @return provides the value of parameter, clamped/fixed appropriately.
- ZSTDMT : changed constant name to ZSTDMT_JOBSIZE_MIN
- ZSTDMT : multithreading is automatically disabled when srcSize <= ZSTDMT_JOBSIZE_MIN, since only one thread will be used in this case (saves memory and runtime).
- ZSTDMT : nbThreads is automatically clamped on setting the value.
2017-11-16 12:18:56 -08:00
* or an error code ( which can be tested with ZSTD_isError ( ) ) . */
2017-06-19 12:56:25 -07:00
ZSTDLIB_API size_t ZSTD_CCtx_setParameter ( ZSTD_CCtx * cctx , ZSTD_cParameter param , unsigned value ) ;
/*! ZSTD_CCtx_setPledgedSrcSize() :
* Total input data size to be compressed as a single frame .
* This value will be controlled at the end , and result in error if not respected .
* @ result : 0 , or an error code ( which can be tested with ZSTD_isError ( ) ) .
* Note 1 : 0 means zero , empty .
* In order to mean " unknown content size " , pass constant ZSTD_CONTENTSIZE_UNKNOWN .
2017-10-14 00:32:06 -07:00
* ZSTD_CONTENTSIZE_UNKNOWN is default value for any new compression job .
2017-06-19 12:56:25 -07:00
* Note 2 : If all data is provided and consumed in a single round ,
* this value is overriden by srcSize instead . */
ZSTDLIB_API size_t ZSTD_CCtx_setPledgedSrcSize ( ZSTD_CCtx * cctx , unsigned long long pledgedSrcSize ) ;
/*! ZSTD_CCtx_loadDictionary() :
2018-03-11 05:21:53 -07:00
* Create an internal CDict from ` dict ` buffer .
* Decompression will have to use same dictionary .
2017-06-19 12:56:25 -07:00
* @ result : 0 , or an error code ( which can be tested with ZSTD_isError ( ) ) .
2018-03-11 05:21:53 -07:00
* Special : Adding a NULL ( or 0 - size ) dictionary invalidates previous dictionary ,
* meaning " return to no-dictionary mode " .
* Note 1 : Dictionary will be used for all future compression jobs .
* To return to " no-dictionary " situation , load a NULL dictionary
2017-06-19 12:56:25 -07:00
* Note 2 : Loading a dictionary involves building tables , which are dependent on compression parameters .
2017-06-20 14:26:48 -07:00
* For this reason , compression parameters cannot be changed anymore after loading a dictionary .
2018-03-11 05:21:53 -07:00
* It ' s also a CPU consuming operation , with non - negligible impact on latency .
* Note 3 : ` dict ` content will be copied internally .
* Use ZSTD_CCtx_loadDictionary_byReference ( ) to reference dictionary content instead .
* In such a case , dictionary buffer must outlive its users .
* Note 4 : Use ZSTD_CCtx_loadDictionary_advanced ( )
* to precisely select how dictionary content must be interpreted . */
2017-06-19 12:56:25 -07:00
ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary ( ZSTD_CCtx * cctx , const void * dict , size_t dictSize ) ;
2017-08-25 10:48:07 -07:00
ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary_byReference ( ZSTD_CCtx * cctx , const void * dict , size_t dictSize ) ;
2017-08-29 15:10:42 -07:00
ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary_advanced ( ZSTD_CCtx * cctx , const void * dict , size_t dictSize , ZSTD_dictLoadMethod_e dictLoadMethod , ZSTD_dictMode_e dictMode ) ;
2017-08-25 10:48:07 -07:00
2017-06-19 12:56:25 -07:00
/*! ZSTD_CCtx_refCDict() :
2017-06-27 15:49:12 -07:00
* Reference a prepared dictionary , to be used for all next compression jobs .
2017-06-19 12:56:25 -07:00
* Note that compression parameters are enforced from within CDict ,
* and supercede any compression parameter previously set within CCtx .
* The dictionary will remain valid for future compression jobs using same CCtx .
* @ result : 0 , or an error code ( which can be tested with ZSTD_isError ( ) ) .
* Special : adding a NULL CDict means " return to no-dictionary mode " .
* Note 1 : Currently , only one dictionary can be managed .
* Adding a new dictionary effectively " discards " any previous one .
2018-03-11 05:21:53 -07:00
* Note 2 : CDict is just referenced , its lifetime must outlive CCtx . */
2017-06-19 12:56:25 -07:00
ZSTDLIB_API size_t ZSTD_CCtx_refCDict ( ZSTD_CCtx * cctx , const ZSTD_CDict * cdict ) ;
/*! ZSTD_CCtx_refPrefix() :
2017-06-27 15:49:12 -07:00
* Reference a prefix ( single - usage dictionary ) for next compression job .
* Decompression need same prefix to properly regenerate data .
* Prefix is * * only used once * * . Tables are discarded at end of compression job .
* Subsequent compression jobs will be done without prefix ( if none is explicitly referenced ) .
* If there is a need to use same prefix multiple times , consider embedding it into a ZSTD_CDict instead .
2017-06-19 12:56:25 -07:00
* @ result : 0 , or an error code ( which can be tested with ZSTD_isError ( ) ) .
2018-03-11 05:21:53 -07:00
* Special : Adding any prefix ( including NULL ) invalidates any previous prefix or dictionary
2017-06-19 12:56:25 -07:00
* Note 1 : Prefix buffer is referenced . It must outlive compression job .
* Note 2 : Referencing a prefix involves building tables , which are dependent on compression parameters .
2018-03-11 05:21:53 -07:00
* It ' s a CPU consuming operation , with non - negligible impact on latency .
* Note 3 : By default , the prefix is treated as raw content ( ZSTD_dm_rawContent ) .
* Use ZSTD_CCtx_refPrefix_advanced ( ) to alter dictMode . */
2017-06-27 15:49:12 -07:00
ZSTDLIB_API size_t ZSTD_CCtx_refPrefix ( ZSTD_CCtx * cctx , const void * prefix , size_t prefixSize ) ;
2017-08-29 15:10:42 -07:00
ZSTDLIB_API size_t ZSTD_CCtx_refPrefix_advanced ( ZSTD_CCtx * cctx , const void * prefix , size_t prefixSize , ZSTD_dictMode_e dictMode ) ;
2017-06-19 12:56:25 -07:00
typedef enum {
2018-01-23 15:52:40 -08:00
ZSTD_e_continue = 0 , /* collect more data, encoder decides when to output compressed result, for optimal conditions */
2017-06-19 12:56:25 -07:00
ZSTD_e_flush , /* flush any data provided so far - frame will continue, future data can still reference previous data for better compression */
2017-09-24 15:57:29 -07:00
ZSTD_e_end /* flush any remaining data and close current frame. Any additional data starts a new frame. */
2017-06-19 12:56:25 -07:00
} ZSTD_EndDirective ;
2017-06-20 14:26:48 -07:00
/*! ZSTD_compress_generic() :
2017-06-19 12:56:25 -07:00
* Behave about the same as ZSTD_compressStream . To note :
2017-06-20 14:26:48 -07:00
* - Compression parameters are pushed into CCtx before starting compression , using ZSTD_CCtx_setParameter ( )
2017-06-19 12:56:25 -07:00
* - Compression parameters cannot be changed once compression is started .
2017-09-24 15:57:29 -07:00
* - outpot - > pos must be < = dstCapacity , input - > pos must be < = srcSize
* - outpot - > pos and input - > pos will be updated . They are guaranteed to remain below their respective limit .
2017-12-12 14:01:54 -08:00
* - In single - thread mode ( default ) , function is blocking : it completed its job before returning to caller .
* - In multi - thread mode , function is non - blocking : it just acquires a copy of input , and distribute job to internal worker threads ,
* and then immediately returns , just indicating that there is some data remaining to be flushed .
* The function nonetheless guarantees forward progress : it will return only after it reads or write at least 1 + byte .
* - Exception : in multi - threading mode , if the first call requests a ZSTD_e_end directive , it is blocking : it will complete compression before giving back control to caller .
2018-01-23 15:19:11 -08:00
* - @ return provides a minimum amount of data remaining to be flushed from internal buffers
2017-06-19 12:56:25 -07:00
* or an error code , which can be tested using ZSTD_isError ( ) .
2017-12-12 14:01:54 -08:00
* if @ return ! = 0 , flush is not fully completed , there is still some data left within internal buffers .
2018-01-23 15:19:11 -08:00
* This is useful for ZSTD_e_flush , since in this case more flushes are necessary to empty all buffers .
* For ZSTD_e_end , @ return = = 0 when internal buffers are fully flushed and frame is completed .
2017-12-12 14:01:54 -08:00
* - after a ZSTD_e_end directive , if internal buffer is not fully flushed ( @ return ! = 0 ) ,
2017-06-20 14:26:48 -07:00
* only ZSTD_e_end or ZSTD_e_flush operations are allowed .
2017-12-12 14:01:54 -08:00
* Before starting a new compression job , or changing compression parameters ,
* it is required to fully flush internal buffers .
2017-06-19 12:56:25 -07:00
*/
ZSTDLIB_API size_t ZSTD_compress_generic ( ZSTD_CCtx * cctx ,
ZSTD_outBuffer * output ,
ZSTD_inBuffer * input ,
ZSTD_EndDirective endOp ) ;
/*! ZSTD_CCtx_reset() :
* Return a CCtx to clean state .
* Useful after an error , or to interrupt an ongoing compression job and start a new one .
* Any internal data not yet flushed is cancelled .
* Dictionary ( if any ) is dropped .
2017-09-24 15:57:29 -07:00
* All parameters are back to default values .
2017-06-19 12:56:25 -07:00
* It ' s possible to modify compression parameters after a reset .
*/
2018-01-10 09:07:47 -08:00
ZSTDLIB_API void ZSTD_CCtx_reset ( ZSTD_CCtx * cctx ) ;
2017-06-19 12:56:25 -07:00
/*! ZSTD_compress_generic_simpleArgs() :
* Same as ZSTD_compress_generic ( ) ,
* but using only integral types as arguments .
2017-09-24 15:57:29 -07:00
* Argument list is larger than ZSTD_ { in , out } Buffer ,
2017-06-19 12:56:25 -07:00
* but can be helpful for binders from dynamic languages
* which have troubles handling structures containing memory pointers .
*/
2017-09-25 15:44:48 -07:00
ZSTDLIB_API size_t ZSTD_compress_generic_simpleArgs (
2017-06-19 12:56:25 -07:00
ZSTD_CCtx * cctx ,
void * dst , size_t dstCapacity , size_t * dstPos ,
const void * src , size_t srcSize , size_t * srcPos ,
ZSTD_EndDirective endOp ) ;
2017-09-24 15:57:29 -07:00
/*! ZSTD_CCtx_params :
* Quick howto :
2017-08-21 13:18:00 -07:00
* - ZSTD_createCCtxParams ( ) : Create a ZSTD_CCtx_params structure
2017-09-24 15:57:29 -07:00
* - ZSTD_CCtxParam_setParameter ( ) : Push parameters one by one into
* an existing ZSTD_CCtx_params structure .
* This is similar to
* ZSTD_CCtx_setParameter ( ) .
* - ZSTD_CCtx_setParametersUsingCCtxParams ( ) : Apply parameters to
* an existing CCtx .
* These parameters will be applied to
* all subsequent compression jobs .
2017-08-21 13:18:00 -07:00
* - ZSTD_compress_generic ( ) : Do compression using the CCtx .
2017-08-23 12:03:30 -07:00
* - ZSTD_freeCCtxParams ( ) : Free the memory .
*
2017-09-24 15:57:29 -07:00
* This can be used with ZSTD_estimateCCtxSize_advanced_usingCCtxParams ( )
* for static allocation for single - threaded compression .
2017-08-25 13:58:41 -07:00
*/
2017-08-21 13:18:00 -07:00
ZSTDLIB_API ZSTD_CCtx_params * ZSTD_createCCtxParams ( void ) ;
/*! ZSTD_resetCCtxParams() :
2018-02-01 16:13:31 -08:00
* Reset params to default values .
2017-08-25 13:58:41 -07:00
*/
2017-08-21 13:18:00 -07:00
ZSTDLIB_API size_t ZSTD_resetCCtxParams ( ZSTD_CCtx_params * params ) ;
/*! ZSTD_initCCtxParams() :
2017-08-25 13:23:16 -07:00
* Initializes the compression parameters of cctxParams according to
2017-08-25 13:58:41 -07:00
* compression level . All other parameters are reset to their default values .
*/
2017-08-25 13:23:16 -07:00
ZSTDLIB_API size_t ZSTD_initCCtxParams ( ZSTD_CCtx_params * cctxParams , int compressionLevel ) ;
/*! ZSTD_initCCtxParams_advanced() :
* Initializes the compression and frame parameters of cctxParams according to
2017-08-25 13:58:41 -07:00
* params . All other parameters are reset to their default values .
*/
2017-08-25 13:23:16 -07:00
ZSTDLIB_API size_t ZSTD_initCCtxParams_advanced ( ZSTD_CCtx_params * cctxParams , ZSTD_parameters params ) ;
2017-08-21 13:18:00 -07:00
ZSTDLIB_API size_t ZSTD_freeCCtxParams ( ZSTD_CCtx_params * params ) ;
/*! ZSTD_CCtxParam_setParameter() :
* Similar to ZSTD_CCtx_setParameter .
* Set one compression parameter , selected by enum ZSTD_cParameter .
2017-08-29 18:03:06 -07:00
* Parameters must be applied to a ZSTD_CCtx using ZSTD_CCtx_setParametersUsingCCtxParams ( ) .
2017-08-21 13:18:00 -07:00
* Note : when ` value ` is an enum , cast it to unsigned for proper type checking .
2017-09-24 15:57:29 -07:00
* @ result : 0 , or an error code ( which can be tested with ZSTD_isError ( ) ) .
2017-08-25 13:58:41 -07:00
*/
2017-08-21 13:18:00 -07:00
ZSTDLIB_API size_t ZSTD_CCtxParam_setParameter ( ZSTD_CCtx_params * params , ZSTD_cParameter param , unsigned value ) ;
2017-08-29 18:03:06 -07:00
/*! ZSTD_CCtx_setParametersUsingCCtxParams() :
2017-09-24 15:57:29 -07:00
* Apply a set of ZSTD_CCtx_params to the compression context .
2018-02-01 19:29:30 -08:00
* This can be done even after compression is started ,
* if nbWorkers = = 0 , this will have no impact until a new compression is started .
* if nbWorkers > = 1 , new parameters will be picked up at next job ,
* with a few restrictions ( windowLog , pledgedSrcSize , nbWorkers , jobSize , and overlapLog are not updated ) .
2017-08-25 13:58:41 -07:00
*/
2017-08-29 18:03:06 -07:00
ZSTDLIB_API size_t ZSTD_CCtx_setParametersUsingCCtxParams (
ZSTD_CCtx * cctx , const ZSTD_CCtx_params * params ) ;
2017-06-19 12:56:25 -07:00
2016-06-04 10:47:02 -07:00
2017-09-24 15:57:29 -07:00
/*=== Advanced parameters for decompression API ===*/
/* The following parameters must be set after creating a ZSTD_DCtx* (or ZSTD_DStream*) object,
* but before starting decompression of a frame .
*/
/*! ZSTD_DCtx_loadDictionary() :
* Create an internal DDict from dict buffer ,
* to be used to decompress next frames .
* @ result : 0 , or an error code ( which can be tested with ZSTD_isError ( ) ) .
* Special : Adding a NULL ( or 0 - size ) dictionary invalidates any previous dictionary ,
* meaning " return to no-dictionary mode " .
* Note 1 : ` dict ` content will be copied internally .
* Use ZSTD_DCtx_loadDictionary_byReference ( )
* to reference dictionary content instead .
* In which case , the dictionary buffer must outlive its users .
* Note 2 : Loading a dictionary involves building tables ,
* which has a non - negligible impact on CPU usage and latency .
* Note 3 : Use ZSTD_DCtx_loadDictionary_advanced ( ) to select
* how dictionary content will be interpreted and loaded .
*/
2017-09-25 14:26:26 -07:00
ZSTDLIB_API size_t ZSTD_DCtx_loadDictionary ( ZSTD_DCtx * dctx , const void * dict , size_t dictSize ) ; /* not implemented */
ZSTDLIB_API size_t ZSTD_DCtx_loadDictionary_byReference ( ZSTD_DCtx * dctx , const void * dict , size_t dictSize ) ; /* not implemented */
ZSTDLIB_API size_t ZSTD_DCtx_loadDictionary_advanced ( ZSTD_DCtx * dctx , const void * dict , size_t dictSize , ZSTD_dictLoadMethod_e dictLoadMethod , ZSTD_dictMode_e dictMode ) ; /* not implemented */
2017-09-24 15:57:29 -07:00
/*! ZSTD_DCtx_refDDict() :
* Reference a prepared dictionary , to be used to decompress next frames .
* The dictionary remains active for decompression of future frames using same DCtx .
* @ result : 0 , or an error code ( which can be tested with ZSTD_isError ( ) ) .
* Note 1 : Currently , only one dictionary can be managed .
* Referencing a new dictionary effectively " discards " any previous one .
* Special : adding a NULL DDict means " return to no-dictionary mode " .
* Note 2 : DDict is just referenced , its lifetime must outlive its usage from DCtx .
*/
2017-09-25 14:26:26 -07:00
ZSTDLIB_API size_t ZSTD_DCtx_refDDict ( ZSTD_DCtx * dctx , const ZSTD_DDict * ddict ) ; /* not implemented */
2017-09-24 15:57:29 -07:00
/*! ZSTD_DCtx_refPrefix() :
* Reference a prefix ( single - usage dictionary ) for next compression job .
* Prefix is * * only used once * * . It must be explicitly referenced before each frame .
* If there is a need to use same prefix multiple times , consider embedding it into a ZSTD_DDict instead .
* @ result : 0 , or an error code ( which can be tested with ZSTD_isError ( ) ) .
* Note 1 : Adding any prefix ( including NULL ) invalidates any previously set prefix or dictionary
* Note 2 : Prefix buffer is referenced . It must outlive compression job .
* Note 3 : By default , the prefix is treated as raw content ( ZSTD_dm_rawContent ) .
* Use ZSTD_CCtx_refPrefix_advanced ( ) to alter dictMode .
2017-09-25 14:26:26 -07:00
* Note 4 : Referencing a raw content prefix has almost no cpu nor memory cost .
2017-09-24 15:57:29 -07:00
*/
2017-09-25 14:26:26 -07:00
ZSTDLIB_API size_t ZSTD_DCtx_refPrefix ( ZSTD_DCtx * dctx , const void * prefix , size_t prefixSize ) ; /* not implemented */
ZSTDLIB_API size_t ZSTD_DCtx_refPrefix_advanced ( ZSTD_DCtx * dctx , const void * prefix , size_t prefixSize , ZSTD_dictMode_e dictMode ) ; /* not implemented */
2017-09-24 15:57:29 -07:00
/*! ZSTD_DCtx_setMaxWindowSize() :
* Refuses allocating internal buffers for frames requiring a window size larger than provided limit .
* This is useful to prevent a decoder context from reserving too much memory for itself ( potential attack scenario ) .
* This parameter is only useful in streaming mode , since no internal buffer is allocated in direct mode .
* By default , a decompression context accepts all window sizes < = ( 1 < < ZSTD_WINDOWLOG_MAX )
* @ return : 0 , or an error code ( which can be tested using ZSTD_isError ( ) ) .
*/
ZSTDLIB_API size_t ZSTD_DCtx_setMaxWindowSize ( ZSTD_DCtx * dctx , size_t maxWindowSize ) ;
/*! ZSTD_DCtx_setFormat() :
* Instruct the decoder context about what kind of data to decode next .
* This instruction is mandatory to decode data without a fully - formed header ,
* such ZSTD_f_zstd1_magicless for example .
* @ return : 0 , or an error code ( which can be tested using ZSTD_isError ( ) ) .
*/
2017-09-25 15:41:48 -07:00
ZSTDLIB_API size_t ZSTD_DCtx_setFormat ( ZSTD_DCtx * dctx , ZSTD_format_e format ) ;
2017-09-24 15:57:29 -07:00
2017-09-25 15:41:48 -07:00
/*! ZSTD_decompress_generic() :
* Behave the same as ZSTD_decompressStream .
* Decompression parameters cannot be changed once decompression is started .
* @ return : an error code , which can be tested using ZSTD_isError ( )
* if > 0 , a hint , nb of expected input bytes for next invocation .
* ` 0 ` means : a frame has just been fully decoded and flushed .
*/
ZSTDLIB_API size_t ZSTD_decompress_generic ( ZSTD_DCtx * dctx ,
ZSTD_outBuffer * output ,
ZSTD_inBuffer * input ) ;
2017-09-25 15:44:48 -07:00
/*! ZSTD_decompress_generic_simpleArgs() :
* Same as ZSTD_decompress_generic ( ) ,
* but using only integral types as arguments .
* Argument list is larger than ZSTD_ { in , out } Buffer ,
* but can be helpful for binders from dynamic languages
* which have troubles handling structures containing memory pointers .
*/
ZSTDLIB_API size_t ZSTD_decompress_generic_simpleArgs (
ZSTD_DCtx * dctx ,
void * dst , size_t dstCapacity , size_t * dstPos ,
const void * src , size_t srcSize , size_t * srcPos ) ;
2017-09-25 16:21:17 -07:00
/*! ZSTD_DCtx_reset() :
* Return a DCtx to clean state .
* If a decompression was ongoing , any internal data not yet flushed is cancelled .
* All parameters are back to default values , including sticky ones .
* Dictionary ( if any ) is dropped .
* Parameters can be modified again after a reset .
2017-09-25 14:26:26 -07:00
*/
2017-09-25 16:21:17 -07:00
ZSTDLIB_API void ZSTD_DCtx_reset ( ZSTD_DCtx * dctx ) ;
2017-09-25 14:26:26 -07:00
2017-09-24 15:57:29 -07:00
2017-09-25 15:41:48 -07:00
/* ============================ */
/** Block level API */
/* ============================ */
2017-09-24 15:57:29 -07:00
/*!
2016-10-06 04:23:52 -07:00
Block functions produce and decode raw zstd blocks , without frame metadata .
2016-07-18 07:52:10 -07:00
Frame metadata cost is typically ~ 18 bytes , which can be non - negligible for very small blocks ( < 100 bytes ) .
2016-06-04 10:47:02 -07:00
User will have to take in charge required information to regenerate data , such as compressed and content sizes .
A few rules to respect :
2016-07-06 11:30:52 -07:00
- Compressing and decompressing require a context structure
2016-06-04 10:47:02 -07:00
+ Use ZSTD_createCCtx ( ) and ZSTD_createDCtx ( )
- It is necessary to init context before starting
2017-04-18 03:17:44 -07:00
+ compression : any ZSTD_compressBegin * ( ) variant , including with dictionary
+ decompression : any ZSTD_decompressBegin * ( ) variant , including with dictionary
+ copyCCtx ( ) and copyDCtx ( ) can be used too
2017-09-21 16:07:29 -07:00
- Block size is limited , it must be < = ZSTD_getBlockSize ( ) < = ZSTD_BLOCKSIZE_MAX = = 128 KB
2017-04-18 03:17:44 -07:00
+ If input is larger than a block size , it ' s necessary to split input data into multiple blocks
+ For inputs larger than a single block size , consider using the regular ZSTD_compress ( ) instead .
Frame metadata is not that costly , and quickly becomes negligible as source size grows larger .
2016-06-04 10:47:02 -07:00
- When a block is considered not compressible enough , ZSTD_compressBlock ( ) result will be zero .
In which case , nothing is produced into ` dst ` .
+ User must test for such outcome and deal directly with uncompressed data
2016-07-06 11:30:52 -07:00
+ ZSTD_decompressBlock ( ) doesn ' t accept uncompressed data as input ! ! !
2017-04-18 03:17:44 -07:00
+ In case of multiple successive blocks , should some of them be uncompressed ,
decoder must be informed of their existence in order to follow proper history .
Use ZSTD_insertBlock ( ) for such a case .
2016-06-04 10:47:02 -07:00
*/
2017-05-19 10:51:30 -07:00
# define ZSTD_BLOCKSIZELOG_MAX 17
# define ZSTD_BLOCKSIZE_MAX (1<<ZSTD_BLOCKSIZELOG_MAX) /* define, for static allocation */
2016-10-06 04:23:52 -07:00
/*===== Raw zstd block functions =====*/
2017-05-19 10:51:30 -07:00
ZSTDLIB_API size_t ZSTD_getBlockSize ( const ZSTD_CCtx * cctx ) ;
2016-06-04 10:47:02 -07:00
ZSTDLIB_API size_t ZSTD_compressBlock ( ZSTD_CCtx * cctx , void * dst , size_t dstCapacity , const void * src , size_t srcSize ) ;
ZSTDLIB_API size_t ZSTD_decompressBlock ( ZSTD_DCtx * dctx , void * dst , size_t dstCapacity , const void * src , size_t srcSize ) ;
2018-01-10 09:07:47 -08:00
ZSTDLIB_API size_t ZSTD_insertBlock ( ZSTD_DCtx * dctx , const void * blockStart , size_t blockSize ) ; /**< insert uncompressed block into `dctx` history. Useful for multi-blocks decompression. */
2016-06-04 10:47:02 -07:00
2016-11-29 11:46:37 -08:00
# endif /* ZSTD_H_ZSTD_STATIC_LINKING_ONLY */
2016-06-04 10:47:02 -07:00
2015-01-23 16:58:16 -08:00
# if defined (__cplusplus)
}
# endif