Merge remote-tracking branch 'refs/remotes/facebook/dev' into dev11
commit
b88e8c2e9b
18
Makefile
18
Makefile
|
@ -26,8 +26,7 @@ endif
|
||||||
default: lib zstd-release
|
default: lib zstd-release
|
||||||
|
|
||||||
.PHONY: all
|
.PHONY: all
|
||||||
all: allmost
|
all: | allmost examples manual
|
||||||
CPPFLAGS=-I../lib LDFLAGS=-L../lib $(MAKE) -C examples/ $@
|
|
||||||
|
|
||||||
.PHONY: allmost
|
.PHONY: allmost
|
||||||
allmost:
|
allmost:
|
||||||
|
@ -68,6 +67,14 @@ zlibwrapper:
|
||||||
test:
|
test:
|
||||||
$(MAKE) -C $(TESTDIR) $@
|
$(MAKE) -C $(TESTDIR) $@
|
||||||
|
|
||||||
|
.PHONY: examples
|
||||||
|
examples:
|
||||||
|
CPPFLAGS=-I../lib LDFLAGS=-L../lib $(MAKE) -C examples/ all
|
||||||
|
|
||||||
|
.PHONY: manual
|
||||||
|
manual:
|
||||||
|
$(MAKE) -C contrib/gen_html $@
|
||||||
|
|
||||||
.PHONY: clean
|
.PHONY: clean
|
||||||
clean:
|
clean:
|
||||||
@$(MAKE) -C $(ZSTDDIR) $@ > $(VOID)
|
@$(MAKE) -C $(ZSTDDIR) $@ > $(VOID)
|
||||||
|
@ -75,6 +82,7 @@ clean:
|
||||||
@$(MAKE) -C $(TESTDIR) $@ > $(VOID)
|
@$(MAKE) -C $(TESTDIR) $@ > $(VOID)
|
||||||
@$(MAKE) -C $(ZWRAPDIR) $@ > $(VOID)
|
@$(MAKE) -C $(ZWRAPDIR) $@ > $(VOID)
|
||||||
@$(MAKE) -C examples/ $@ > $(VOID)
|
@$(MAKE) -C examples/ $@ > $(VOID)
|
||||||
|
@$(MAKE) -C contrib/gen_html $@ > $(VOID)
|
||||||
@$(RM) zstd$(EXT) zstdmt$(EXT) tmp*
|
@$(RM) zstd$(EXT) zstdmt$(EXT) tmp*
|
||||||
@echo Cleaning completed
|
@echo Cleaning completed
|
||||||
|
|
||||||
|
@ -197,16 +205,16 @@ cmaketest:
|
||||||
cd $(BUILDIR)/cmake/build ; cmake -DPREFIX:STRING=~/install_test_dir $(CMAKE_PARAMS) .. ; $(MAKE) install ; $(MAKE) uninstall
|
cd $(BUILDIR)/cmake/build ; cmake -DPREFIX:STRING=~/install_test_dir $(CMAKE_PARAMS) .. ; $(MAKE) install ; $(MAKE) uninstall
|
||||||
|
|
||||||
c90test: clean
|
c90test: clean
|
||||||
CFLAGS="-std=c90" $(MAKE) all # will fail, due to // and long long
|
CFLAGS="-std=c90" $(MAKE) allmost # will fail, due to missing support for `long long`
|
||||||
|
|
||||||
gnu90test: clean
|
gnu90test: clean
|
||||||
CFLAGS="-std=gnu90" $(MAKE) all
|
CFLAGS="-std=gnu90" $(MAKE) allmost
|
||||||
|
|
||||||
c99test: clean
|
c99test: clean
|
||||||
CFLAGS="-std=c99" $(MAKE) allmost
|
CFLAGS="-std=c99" $(MAKE) allmost
|
||||||
|
|
||||||
gnu99test: clean
|
gnu99test: clean
|
||||||
CFLAGS="-std=gnu99" $(MAKE) all
|
CFLAGS="-std=gnu99" $(MAKE) allmost
|
||||||
|
|
||||||
c11test: clean
|
c11test: clean
|
||||||
CFLAGS="-std=c11" $(MAKE) allmost
|
CFLAGS="-std=c11" $(MAKE) allmost
|
||||||
|
|
2
NEWS
2
NEWS
|
@ -2,7 +2,7 @@ v1.1.4
|
||||||
cli : new : can compress in *.gz format, using --format=gzip command, by Przemyslaw Skibinski
|
cli : new : can compress in *.gz format, using --format=gzip command, by Przemyslaw Skibinski
|
||||||
cli : new : advanced benchmark command --priority=rt
|
cli : new : advanced benchmark command --priority=rt
|
||||||
cli : fix : write on sparse-enabled file systems in 32-bits mode, by @ds77
|
cli : fix : write on sparse-enabled file systems in 32-bits mode, by @ds77
|
||||||
API : new : ZSTD_getFrameCompressedSize(), ZSTD_getFrameContentSize(), ZSTD_findDecompressedSize(), by Sean Purcell
|
API : new : ZSTD_findFrameCompressedSize(), ZSTD_getFrameContentSize(), ZSTD_findDecompressedSize(), by Sean Purcell
|
||||||
API : change : ZSTD_compress*() with srcSize==0 create an empty-frame of known size
|
API : change : ZSTD_compress*() with srcSize==0 create an empty-frame of known size
|
||||||
build:new : meson build system in contrib/meson, by Dima Krasner
|
build:new : meson build system in contrib/meson, by Dima Krasner
|
||||||
doc : new : educational decoder, by Sean Purcell
|
doc : new : educational decoder, by Sean Purcell
|
||||||
|
|
|
@ -12,7 +12,7 @@ dependencies:
|
||||||
if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make -C tests test-invalidDictionaries && make clean; fi
|
if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make -C tests test-invalidDictionaries && make clean; fi
|
||||||
- |
|
- |
|
||||||
if [[ "$CIRCLE_NODE_INDEX" == "0" ]]; then g++ -v; make gpptest && make clean; fi
|
if [[ "$CIRCLE_NODE_INDEX" == "0" ]]; then g++ -v; make gpptest && make clean; fi
|
||||||
if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make -C tests test-legacy && make clean; fi
|
if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make -C tests test-legacy test-decodecorpus && make clean; fi
|
||||||
- |
|
- |
|
||||||
if [[ "$CIRCLE_NODE_INDEX" == "0" ]]; then gcc -v; make gnu90test && make clean; fi
|
if [[ "$CIRCLE_NODE_INDEX" == "0" ]]; then gcc -v; make gnu90test && make clean; fi
|
||||||
if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make -C tests test-symbols && make clean; fi
|
if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make -C tests test-symbols && make clean; fi
|
||||||
|
|
|
@ -7,12 +7,18 @@
|
||||||
# of patent rights can be found in the PATENTS file in the same directory.
|
# of patent rights can be found in the PATENTS file in the same directory.
|
||||||
# ##########################################################################
|
# ##########################################################################
|
||||||
|
|
||||||
|
|
||||||
CFLAGS ?= -O3
|
CFLAGS ?= -O3
|
||||||
CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 -Wswitch-enum -Wno-comment
|
CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 -Wswitch-enum -Wno-comment
|
||||||
CFLAGS += $(MOREFLAGS)
|
CFLAGS += $(MOREFLAGS)
|
||||||
FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS)
|
FLAGS = $(CPPFLAGS) $(CFLAGS) $(CXXFLAGS) $(LDFLAGS)
|
||||||
|
|
||||||
|
ZSTDAPI = ../../lib/zstd.h
|
||||||
|
ZSTDMANUAL = ../../doc/zstd_manual.html
|
||||||
|
LIBVER_MAJOR_SCRIPT:=`sed -n '/define ZSTD_VERSION_MAJOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < $(ZSTDAPI)`
|
||||||
|
LIBVER_MINOR_SCRIPT:=`sed -n '/define ZSTD_VERSION_MINOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < $(ZSTDAPI)`
|
||||||
|
LIBVER_PATCH_SCRIPT:=`sed -n '/define ZSTD_VERSION_RELEASE/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < $(ZSTDAPI)`
|
||||||
|
LIBVER_SCRIPT:= $(LIBVER_MAJOR_SCRIPT).$(LIBVER_MINOR_SCRIPT).$(LIBVER_PATCH_SCRIPT)
|
||||||
|
LIBVER := $(shell echo $(LIBVER_SCRIPT))
|
||||||
|
|
||||||
|
|
||||||
# Define *.exe as extension for Windows systems
|
# Define *.exe as extension for Windows systems
|
||||||
|
@ -23,14 +29,23 @@ EXT =
|
||||||
endif
|
endif
|
||||||
|
|
||||||
|
|
||||||
.PHONY: default gen_html
|
.PHONY: default
|
||||||
|
|
||||||
default: gen_html
|
default: gen_html
|
||||||
|
|
||||||
|
.PHONY: all
|
||||||
|
all: manual
|
||||||
|
|
||||||
gen_html: gen_html.cpp
|
gen_html: gen_html.cpp
|
||||||
$(CXX) $(FLAGS) $^ -o $@$(EXT)
|
$(CXX) $(FLAGS) $^ -o $@$(EXT)
|
||||||
|
|
||||||
|
$(ZSTDMANUAL): gen_html $(ZSTDAPI)
|
||||||
|
echo "Update zstd manual in /doc"
|
||||||
|
./gen_html $(LIBVER) $(ZSTDAPI) $(ZSTDMANUAL)
|
||||||
|
|
||||||
|
.PHONY: manual
|
||||||
|
manual: gen_html $(ZSTDMANUAL)
|
||||||
|
|
||||||
|
.PHONY: clean
|
||||||
clean:
|
clean:
|
||||||
@$(RM) gen_html$(EXT)
|
@$(RM) gen_html$(EXT)
|
||||||
@echo Cleaning completed
|
@echo Cleaning completed
|
||||||
|
|
|
@ -0,0 +1,20 @@
|
||||||
|
Zstandard Documentation
|
||||||
|
=======================
|
||||||
|
|
||||||
|
This directory contains material defining the Zstandard format,
|
||||||
|
as well as for help using the `zstd` library.
|
||||||
|
|
||||||
|
__`zstd_compression_format.md`__ : This document defines the Zstandard compression format.
|
||||||
|
Compliant decoders must adhere to this document,
|
||||||
|
and compliant encoders must generate data that follows it.
|
||||||
|
|
||||||
|
__`educational_decoder`__ : This directory contains an implementation of a Zstandard decoder,
|
||||||
|
compliant with the Zstandard compression format.
|
||||||
|
It can be used, for example, to better understand the format,
|
||||||
|
or as the basis for a separate implementation a Zstandard decoder/encoder.
|
||||||
|
|
||||||
|
__`zstd_manual.html`__ : Documentation on the functions found in `zstd.h`.
|
||||||
|
See [http://zstd.net/zstd_manual.html](http://zstd.net/zstd_manual.html) for
|
||||||
|
the manual released with the latest official `zstd` release.
|
||||||
|
|
||||||
|
|
|
@ -17,3 +17,13 @@ It also contains implementations of Huffman and FSE table decoding.
|
||||||
|
|
||||||
harness <input-file> <output-file> [dictionary]
|
harness <input-file> <output-file> [dictionary]
|
||||||
|
|
||||||
|
As an additional resource to be used with this decoder,
|
||||||
|
see the `decodecorpus` tool in the [tests] directory.
|
||||||
|
It generates valid Zstandard frames that can be used to verify
|
||||||
|
a Zstandard decoder implementation.
|
||||||
|
Note that to use the tool to verify this decoder implementation,
|
||||||
|
the --content-size flag should be set,
|
||||||
|
as this decoder does not handle streaming decoding,
|
||||||
|
and so it must know the decompressed size in advance.
|
||||||
|
|
||||||
|
[tests]: https://github.com/facebook/zstd/blob/dev/tests/
|
|
@ -799,7 +799,7 @@ static size_t decode_literals_simple(istream_t *const in, u8 **const literals,
|
||||||
case 2:
|
case 2:
|
||||||
// "Size_Format uses 1 bit. Regenerated_Size uses 5 bits (0-31)."
|
// "Size_Format uses 1 bit. Regenerated_Size uses 5 bits (0-31)."
|
||||||
IO_rewind_bits(in, 1);
|
IO_rewind_bits(in, 1);
|
||||||
size = IO_read_bits(in, 2);
|
size = IO_read_bits(in, 5);
|
||||||
break;
|
break;
|
||||||
case 1:
|
case 1:
|
||||||
// "Size_Format uses 2 bits. Regenerated_Size uses 12 bits (0-4095)."
|
// "Size_Format uses 2 bits. Regenerated_Size uses 12 bits (0-4095)."
|
||||||
|
@ -881,7 +881,7 @@ static size_t decode_literals_compressed(frame_context_t *const ctx,
|
||||||
IMPOSSIBLE();
|
IMPOSSIBLE();
|
||||||
}
|
}
|
||||||
if (regenerated_size > MAX_LITERALS_SIZE ||
|
if (regenerated_size > MAX_LITERALS_SIZE ||
|
||||||
compressed_size > regenerated_size) {
|
compressed_size >= regenerated_size) {
|
||||||
CORRUPTION();
|
CORRUPTION();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -1654,7 +1654,7 @@ static inline const u8 *IO_read_bytes(istream_t *const in, size_t len) {
|
||||||
/// Returns a pointer to write `len` bytes to, and advances the internal state
|
/// Returns a pointer to write `len` bytes to, and advances the internal state
|
||||||
static inline u8 *IO_write_bytes(ostream_t *const out, size_t len) {
|
static inline u8 *IO_write_bytes(ostream_t *const out, size_t len) {
|
||||||
if (len > out->len) {
|
if (len > out->len) {
|
||||||
INP_SIZE();
|
OUT_SIZE();
|
||||||
}
|
}
|
||||||
u8 *const ptr = out->ptr;
|
u8 *const ptr = out->ptr;
|
||||||
out->ptr += len;
|
out->ptr += len;
|
|
@ -1,10 +1,10 @@
|
||||||
<html>
|
<html>
|
||||||
<head>
|
<head>
|
||||||
<meta http-equiv="Content-Type" content="text/html; charset=ISO-8859-1">
|
<meta http-equiv="Content-Type" content="text/html; charset=ISO-8859-1">
|
||||||
<title>zstd 1.1.2 Manual</title>
|
<title>zstd 1.1.4 Manual</title>
|
||||||
</head>
|
</head>
|
||||||
<body>
|
<body>
|
||||||
<h1>zstd 1.1.2 Manual</h1>
|
<h1>zstd 1.1.4 Manual</h1>
|
||||||
<hr>
|
<hr>
|
||||||
<a name="Contents"></a><h2>Contents</h2>
|
<a name="Contents"></a><h2>Contents</h2>
|
||||||
<ol>
|
<ol>
|
||||||
|
@ -19,13 +19,15 @@
|
||||||
<li><a href="#Chapter9">Streaming decompression - HowTo</a></li>
|
<li><a href="#Chapter9">Streaming decompression - HowTo</a></li>
|
||||||
<li><a href="#Chapter10">START OF ADVANCED AND EXPERIMENTAL FUNCTIONS</a></li>
|
<li><a href="#Chapter10">START OF ADVANCED AND EXPERIMENTAL FUNCTIONS</a></li>
|
||||||
<li><a href="#Chapter11">Advanced types</a></li>
|
<li><a href="#Chapter11">Advanced types</a></li>
|
||||||
<li><a href="#Chapter12">Advanced compression functions</a></li>
|
<li><a href="#Chapter12">Compressed size functions</a></li>
|
||||||
<li><a href="#Chapter13">Advanced decompression functions</a></li>
|
<li><a href="#Chapter13">Decompressed size functions</a></li>
|
||||||
<li><a href="#Chapter14">Advanced streaming functions</a></li>
|
<li><a href="#Chapter14">Advanced compression functions</a></li>
|
||||||
<li><a href="#Chapter15">Buffer-less and synchronous inner streaming functions</a></li>
|
<li><a href="#Chapter15">Advanced decompression functions</a></li>
|
||||||
<li><a href="#Chapter16">Buffer-less streaming compression (synchronous mode)</a></li>
|
<li><a href="#Chapter16">Advanced streaming functions</a></li>
|
||||||
<li><a href="#Chapter17">Buffer-less streaming decompression (synchronous mode)</a></li>
|
<li><a href="#Chapter17">Buffer-less and synchronous inner streaming functions</a></li>
|
||||||
<li><a href="#Chapter18">Block functions</a></li>
|
<li><a href="#Chapter18">Buffer-less streaming compression (synchronous mode)</a></li>
|
||||||
|
<li><a href="#Chapter19">Buffer-less streaming decompression (synchronous mode)</a></li>
|
||||||
|
<li><a href="#Chapter20">Block functions</a></li>
|
||||||
</ol>
|
</ol>
|
||||||
<hr>
|
<hr>
|
||||||
<a name="Chapter1"></a><h2>Introduction</h2><pre>
|
<a name="Chapter1"></a><h2>Introduction</h2><pre>
|
||||||
|
@ -63,7 +65,7 @@
|
||||||
|
|
||||||
<pre><b>size_t ZSTD_decompress( void* dst, size_t dstCapacity,
|
<pre><b>size_t ZSTD_decompress( void* dst, size_t dstCapacity,
|
||||||
const void* src, size_t compressedSize);
|
const void* src, size_t compressedSize);
|
||||||
</b><p> `compressedSize` : must be the _exact_ size of a single compressed frame.
|
</b><p> `compressedSize` : must be the _exact_ size of some number of compressed and/or skippable frames.
|
||||||
`dstCapacity` is an upper bound of originalSize.
|
`dstCapacity` is an upper bound of originalSize.
|
||||||
If user cannot imply a maximum upper bound, it's better to use streaming mode to decompress data.
|
If user cannot imply a maximum upper bound, it's better to use streaming mode to decompress data.
|
||||||
@return : the number of bytes decompressed into `dst` (<= `dstCapacity`),
|
@return : the number of bytes decompressed into `dst` (<= `dstCapacity`),
|
||||||
|
@ -71,7 +73,16 @@
|
||||||
</p></pre><BR>
|
</p></pre><BR>
|
||||||
|
|
||||||
<pre><b>unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize);
|
<pre><b>unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize);
|
||||||
</b><p> 'src' is the start of a zstd compressed frame.
|
</b><p> NOTE: This function is planned to be obsolete, in favour of ZSTD_getFrameContentSize.
|
||||||
|
ZSTD_getFrameContentSize functions the same way, returning the decompressed size of a single
|
||||||
|
frame, but distinguishes empty frames from frames with an unknown size, or errors.
|
||||||
|
|
||||||
|
Additionally, ZSTD_findDecompressedSize can be used instead. It can handle multiple
|
||||||
|
concatenated frames in one buffer, and so is more general.
|
||||||
|
As a result however, it requires more computation and entire frames to be passed to it,
|
||||||
|
as opposed to ZSTD_getFrameContentSize which requires only a single frame's header.
|
||||||
|
|
||||||
|
'src' is the start of a zstd compressed frame.
|
||||||
@return : content size to be decompressed, as a 64-bits value _if known_, 0 otherwise.
|
@return : content size to be decompressed, as a 64-bits value _if known_, 0 otherwise.
|
||||||
note 1 : decompressed size is an optional field, that may not be present, especially in streaming mode.
|
note 1 : decompressed size is an optional field, that may not be present, especially in streaming mode.
|
||||||
When `return==0`, data to decompress could be any size.
|
When `return==0`, data to decompress could be any size.
|
||||||
|
@ -88,21 +99,29 @@
|
||||||
note 5 : when `return==0`, if precise failure cause is needed, use ZSTD_getFrameParams() to know more.
|
note 5 : when `return==0`, if precise failure cause is needed, use ZSTD_getFrameParams() to know more.
|
||||||
</p></pre><BR>
|
</p></pre><BR>
|
||||||
|
|
||||||
<h3>Helper functions</h3><pre><b>int ZSTD_maxCLevel(void); </b>/*!< maximum compression level available */<b>
|
<h3>Helper functions</h3><pre></pre><b><pre>int ZSTD_maxCLevel(void); </b>/*!< maximum compression level available */<b>
|
||||||
size_t ZSTD_compressBound(size_t srcSize); </b>/*!< maximum compressed size in worst case scenario */<b>
|
size_t ZSTD_compressBound(size_t srcSize); </b>/*!< maximum compressed size in worst case scenario */<b>
|
||||||
unsigned ZSTD_isError(size_t code); </b>/*!< tells if a `size_t` function result is an error code */<b>
|
unsigned ZSTD_isError(size_t code); </b>/*!< tells if a `size_t` function result is an error code */<b>
|
||||||
const char* ZSTD_getErrorName(size_t code); </b>/*!< provides readable string from an error code */<b>
|
const char* ZSTD_getErrorName(size_t code); </b>/*!< provides readable string from an error code */<b>
|
||||||
</b></pre><BR>
|
</pre></b><BR>
|
||||||
<a name="Chapter4"></a><h2>Explicit memory management</h2><pre></pre>
|
<a name="Chapter4"></a><h2>Explicit memory management</h2><pre></pre>
|
||||||
|
|
||||||
|
<h3>Compression context</h3><pre> When compressing many times,
|
||||||
|
it is recommended to allocate a context just once, and re-use it for each successive compression operation.
|
||||||
|
This will make workload friendlier for system's memory.
|
||||||
|
Use one context per thread for parallel execution in multi-threaded environments.
|
||||||
|
</pre><b><pre>typedef struct ZSTD_CCtx_s ZSTD_CCtx;
|
||||||
|
ZSTD_CCtx* ZSTD_createCCtx(void);
|
||||||
|
size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx);
|
||||||
|
</pre></b><BR>
|
||||||
<pre><b>size_t ZSTD_compressCCtx(ZSTD_CCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, int compressionLevel);
|
<pre><b>size_t ZSTD_compressCCtx(ZSTD_CCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, int compressionLevel);
|
||||||
</b><p> Same as ZSTD_compress(), requires an allocated ZSTD_CCtx (see ZSTD_createCCtx()).
|
</b><p> Same as ZSTD_compress(), requires an allocated ZSTD_CCtx (see ZSTD_createCCtx()).
|
||||||
</p></pre><BR>
|
</p></pre><BR>
|
||||||
|
|
||||||
<h3>Decompression context</h3><pre><b>typedef struct ZSTD_DCtx_s ZSTD_DCtx;
|
<h3>Decompression context</h3><pre></pre><b><pre>typedef struct ZSTD_DCtx_s ZSTD_DCtx;
|
||||||
ZSTD_DCtx* ZSTD_createDCtx(void);
|
ZSTD_DCtx* ZSTD_createDCtx(void);
|
||||||
size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx);
|
size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx);
|
||||||
</b></pre><BR>
|
</pre></b><BR>
|
||||||
<pre><b>size_t ZSTD_decompressDCtx(ZSTD_DCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
|
<pre><b>size_t ZSTD_decompressDCtx(ZSTD_DCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
|
||||||
</b><p> Same as ZSTD_decompress(), requires an allocated ZSTD_DCtx (see ZSTD_createDCtx()).
|
</b><p> Same as ZSTD_decompress(), requires an allocated ZSTD_DCtx (see ZSTD_createDCtx()).
|
||||||
</p></pre><BR>
|
</p></pre><BR>
|
||||||
|
@ -131,11 +150,11 @@ size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx);
|
||||||
|
|
||||||
<a name="Chapter6"></a><h2>Fast dictionary API</h2><pre></pre>
|
<a name="Chapter6"></a><h2>Fast dictionary API</h2><pre></pre>
|
||||||
|
|
||||||
<pre><b>ZSTD_CDict* ZSTD_createCDict(const void* dict, size_t dictSize, int compressionLevel);
|
<pre><b>ZSTD_CDict* ZSTD_createCDict(const void* dictBuffer, size_t dictSize, int compressionLevel);
|
||||||
</b><p> When compressing multiple messages / blocks with the same dictionary, it's recommended to load it just once.
|
</b><p> When compressing multiple messages / blocks with the same dictionary, it's recommended to load it just once.
|
||||||
ZSTD_createCDict() will create a digested dictionary, ready to start future compression operations without startup delay.
|
ZSTD_createCDict() will create a digested dictionary, ready to start future compression operations without startup delay.
|
||||||
ZSTD_CDict can be created once and used by multiple threads concurrently, as its usage is read-only.
|
ZSTD_CDict can be created once and used by multiple threads concurrently, as its usage is read-only.
|
||||||
`dict` can be released after ZSTD_CDict creation.
|
`dictBuffer` can be released after ZSTD_CDict creation, as its content is copied within CDict
|
||||||
</p></pre><BR>
|
</p></pre><BR>
|
||||||
|
|
||||||
<pre><b>size_t ZSTD_freeCDict(ZSTD_CDict* CDict);
|
<pre><b>size_t ZSTD_freeCDict(ZSTD_CDict* CDict);
|
||||||
|
@ -151,9 +170,9 @@ size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx);
|
||||||
Note that compression level is decided during dictionary creation.
|
Note that compression level is decided during dictionary creation.
|
||||||
</p></pre><BR>
|
</p></pre><BR>
|
||||||
|
|
||||||
<pre><b>ZSTD_DDict* ZSTD_createDDict(const void* dict, size_t dictSize);
|
<pre><b>ZSTD_DDict* ZSTD_createDDict(const void* dictBuffer, size_t dictSize);
|
||||||
</b><p> Create a digested dictionary, ready to start decompression operation without startup delay.
|
</b><p> Create a digested dictionary, ready to start decompression operation without startup delay.
|
||||||
`dict` can be released after creation.
|
dictBuffer can be released after DDict creation, as its content is copied inside DDict
|
||||||
</p></pre><BR>
|
</p></pre><BR>
|
||||||
|
|
||||||
<pre><b>size_t ZSTD_freeDDict(ZSTD_DDict* ddict);
|
<pre><b>size_t ZSTD_freeDDict(ZSTD_DDict* ddict);
|
||||||
|
@ -271,9 +290,9 @@ size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx);
|
||||||
} ZSTD_compressionParameters;
|
} ZSTD_compressionParameters;
|
||||||
</b></pre><BR>
|
</b></pre><BR>
|
||||||
<pre><b>typedef struct {
|
<pre><b>typedef struct {
|
||||||
unsigned contentSizeFlag; </b>/**< 1: content size will be in frame header (if known). */<b>
|
unsigned contentSizeFlag; </b>/**< 1: content size will be in frame header (when known) */<b>
|
||||||
unsigned checksumFlag; </b>/**< 1: will generate a 22-bits checksum at end of frame, to be used for error detection by decompressor */<b>
|
unsigned checksumFlag; </b>/**< 1: generate a 32-bits checksum at end of frame, for error detection */<b>
|
||||||
unsigned noDictIDFlag; </b>/**< 1: no dict ID will be saved into frame header (if dictionary compression) */<b>
|
unsigned noDictIDFlag; </b>/**< 1: no dictID will be saved into frame header (if dictionary compression) */<b>
|
||||||
} ZSTD_frameParameters;
|
} ZSTD_frameParameters;
|
||||||
</b></pre><BR>
|
</b></pre><BR>
|
||||||
<pre><b>typedef struct {
|
<pre><b>typedef struct {
|
||||||
|
@ -281,11 +300,56 @@ size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx);
|
||||||
ZSTD_frameParameters fParams;
|
ZSTD_frameParameters fParams;
|
||||||
} ZSTD_parameters;
|
} ZSTD_parameters;
|
||||||
</b></pre><BR>
|
</b></pre><BR>
|
||||||
<h3>Custom memory allocation functions</h3><pre><b>typedef void* (*ZSTD_allocFunction) (void* opaque, size_t size);
|
<h3>Custom memory allocation functions</h3><pre></pre><b><pre>typedef void* (*ZSTD_allocFunction) (void* opaque, size_t size);
|
||||||
typedef void (*ZSTD_freeFunction) (void* opaque, void* address);
|
typedef void (*ZSTD_freeFunction) (void* opaque, void* address);
|
||||||
typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; void* opaque; } ZSTD_customMem;
|
typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; void* opaque; } ZSTD_customMem;
|
||||||
</b></pre><BR>
|
</pre></b><BR>
|
||||||
<a name="Chapter12"></a><h2>Advanced compression functions</h2><pre></pre>
|
<a name="Chapter12"></a><h2>Compressed size functions</h2><pre></pre>
|
||||||
|
|
||||||
|
<pre><b>size_t ZSTD_getFrameCompressedSize(const void* src, size_t srcSize);
|
||||||
|
</b><p> `src` should point to the start of a ZSTD encoded frame
|
||||||
|
`srcSize` must be at least as large as the frame
|
||||||
|
@return : the compressed size of the frame pointed to by `src`, suitable to pass to
|
||||||
|
`ZSTD_decompress` or similar, or an error code if given invalid input.
|
||||||
|
</p></pre><BR>
|
||||||
|
|
||||||
|
<a name="Chapter13"></a><h2>Decompressed size functions</h2><pre></pre>
|
||||||
|
|
||||||
|
<pre><b>unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize);
|
||||||
|
</b><p> `src` should point to the start of a ZSTD encoded frame
|
||||||
|
`srcSize` must be at least as large as the frame header. A value greater than or equal
|
||||||
|
to `ZSTD_frameHeaderSize_max` is guaranteed to be large enough in all cases.
|
||||||
|
@return : decompressed size of the frame pointed to be `src` if known, otherwise
|
||||||
|
- ZSTD_CONTENTSIZE_UNKNOWN if the size cannot be determined
|
||||||
|
- ZSTD_CONTENTSIZE_ERROR if an error occured (e.g. invalid magic number, srcSize too small)
|
||||||
|
</p></pre><BR>
|
||||||
|
|
||||||
|
<pre><b>unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize);
|
||||||
|
</b><p> `src` should point the start of a series of ZSTD encoded and/or skippable frames
|
||||||
|
`srcSize` must be the _exact_ size of this series
|
||||||
|
(i.e. there should be a frame boundary exactly `srcSize` bytes after `src`)
|
||||||
|
@return : the decompressed size of all data in the contained frames, as a 64-bit value _if known_
|
||||||
|
- if the decompressed size cannot be determined: ZSTD_CONTENTSIZE_UNKNOWN
|
||||||
|
- if an error occurred: ZSTD_CONTENTSIZE_ERROR
|
||||||
|
|
||||||
|
note 1 : decompressed size is an optional field, that may not be present, especially in streaming mode.
|
||||||
|
When `return==ZSTD_CONTENTSIZE_UNKNOWN`, data to decompress could be any size.
|
||||||
|
In which case, it's necessary to use streaming mode to decompress data.
|
||||||
|
Optionally, application can still use ZSTD_decompress() while relying on implied limits.
|
||||||
|
(For example, data may be necessarily cut into blocks <= 16 KB).
|
||||||
|
note 2 : decompressed size is always present when compression is done with ZSTD_compress()
|
||||||
|
note 3 : decompressed size can be very large (64-bits value),
|
||||||
|
potentially larger than what local system can handle as a single memory segment.
|
||||||
|
In which case, it's necessary to use streaming mode to decompress data.
|
||||||
|
note 4 : If source is untrusted, decompressed size could be wrong or intentionally modified.
|
||||||
|
Always ensure result fits within application's authorized limits.
|
||||||
|
Each application can set its own limits.
|
||||||
|
note 5 : ZSTD_findDecompressedSize handles multiple frames, and so it must traverse the input to
|
||||||
|
read each contained frame header. This is efficient as most of the data is skipped,
|
||||||
|
however it does mean that all frame data must be present and valid.
|
||||||
|
</p></pre><BR>
|
||||||
|
|
||||||
|
<a name="Chapter14"></a><h2>Advanced compression functions</h2><pre></pre>
|
||||||
|
|
||||||
<pre><b>size_t ZSTD_estimateCCtxSize(ZSTD_compressionParameters cParams);
|
<pre><b>size_t ZSTD_estimateCCtxSize(ZSTD_compressionParameters cParams);
|
||||||
</b><p> Gives the amount of memory allocated for a ZSTD_CCtx given a set of compression parameters.
|
</b><p> Gives the amount of memory allocated for a ZSTD_CCtx given a set of compression parameters.
|
||||||
|
@ -300,7 +364,22 @@ typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; v
|
||||||
</b><p> Gives the amount of memory used by a given ZSTD_CCtx
|
</b><p> Gives the amount of memory used by a given ZSTD_CCtx
|
||||||
</p></pre><BR>
|
</p></pre><BR>
|
||||||
|
|
||||||
<pre><b>ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize,
|
<pre><b>typedef enum {
|
||||||
|
ZSTD_p_forceWindow </b>/* Force back-references to remain < windowSize, even when referencing Dictionary content (default:0)*/<b>
|
||||||
|
} ZSTD_CCtxParameter;
|
||||||
|
</b></pre><BR>
|
||||||
|
<pre><b>size_t ZSTD_setCCtxParameter(ZSTD_CCtx* cctx, ZSTD_CCtxParameter param, unsigned value);
|
||||||
|
</b><p> Set advanced parameters, selected through enum ZSTD_CCtxParameter
|
||||||
|
@result : 0, or an error code (which can be tested with ZSTD_isError())
|
||||||
|
</p></pre><BR>
|
||||||
|
|
||||||
|
<pre><b>ZSTD_CDict* ZSTD_createCDict_byReference(const void* dictBuffer, size_t dictSize, int compressionLevel);
|
||||||
|
</b><p> Create a digested dictionary for compression
|
||||||
|
Dictionary content is simply referenced, and therefore stays in dictBuffer.
|
||||||
|
It is important that dictBuffer outlives CDict, it must remain read accessible throughout the lifetime of CDict
|
||||||
|
</p></pre><BR>
|
||||||
|
|
||||||
|
<pre><b>ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize, unsigned byReference,
|
||||||
ZSTD_parameters params, ZSTD_customMem customMem);
|
ZSTD_parameters params, ZSTD_customMem customMem);
|
||||||
</b><p> Create a ZSTD_CDict using external alloc and free, and customized compression parameters
|
</b><p> Create a ZSTD_CDict using external alloc and free, and customized compression parameters
|
||||||
</p></pre><BR>
|
</p></pre><BR>
|
||||||
|
@ -336,7 +415,7 @@ typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; v
|
||||||
</b><p> Same as ZSTD_compress_usingDict(), with fine-tune control of each compression parameter
|
</b><p> Same as ZSTD_compress_usingDict(), with fine-tune control of each compression parameter
|
||||||
</p></pre><BR>
|
</p></pre><BR>
|
||||||
|
|
||||||
<a name="Chapter13"></a><h2>Advanced decompression functions</h2><pre></pre>
|
<a name="Chapter15"></a><h2>Advanced decompression functions</h2><pre></pre>
|
||||||
|
|
||||||
<pre><b>unsigned ZSTD_isFrame(const void* buffer, size_t size);
|
<pre><b>unsigned ZSTD_isFrame(const void* buffer, size_t size);
|
||||||
</b><p> Tells if the content of `buffer` starts with a valid Frame Identifier.
|
</b><p> Tells if the content of `buffer` starts with a valid Frame Identifier.
|
||||||
|
@ -357,6 +436,12 @@ typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; v
|
||||||
</b><p> Gives the amount of memory used by a given ZSTD_DCtx
|
</b><p> Gives the amount of memory used by a given ZSTD_DCtx
|
||||||
</p></pre><BR>
|
</p></pre><BR>
|
||||||
|
|
||||||
|
<pre><b>ZSTD_DDict* ZSTD_createDDict_byReference(const void* dictBuffer, size_t dictSize);
|
||||||
|
</b><p> Create a digested dictionary, ready to start decompression operation without startup delay.
|
||||||
|
Dictionary content is simply referenced, and therefore stays in dictBuffer.
|
||||||
|
It is important that dictBuffer outlives DDict, it must remain read accessible throughout the lifetime of DDict
|
||||||
|
</p></pre><BR>
|
||||||
|
|
||||||
<pre><b>size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
|
<pre><b>size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
|
||||||
</b><p> Gives the amount of memory used by a given ZSTD_DDict
|
</b><p> Gives the amount of memory used by a given ZSTD_DDict
|
||||||
</p></pre><BR>
|
</p></pre><BR>
|
||||||
|
@ -385,33 +470,33 @@ typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; v
|
||||||
When identifying the exact failure cause, it's possible to used ZSTD_getFrameParams(), which will provide a more precise error code.
|
When identifying the exact failure cause, it's possible to used ZSTD_getFrameParams(), which will provide a more precise error code.
|
||||||
</p></pre><BR>
|
</p></pre><BR>
|
||||||
|
|
||||||
<a name="Chapter14"></a><h2>Advanced streaming functions</h2><pre></pre>
|
<a name="Chapter16"></a><h2>Advanced streaming functions</h2><pre></pre>
|
||||||
|
|
||||||
<h3>Advanced Streaming compression functions</h3><pre><b>ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem);
|
<h3>Advanced Streaming compression functions</h3><pre></pre><b><pre>ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem);
|
||||||
size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize); </b>/**< pledgedSrcSize must be correct */<b>
|
size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize); </b>/**< pledgedSrcSize must be correct, a size of 0 means unknown. for a frame size of 0 use initCStream_advanced */<b>
|
||||||
size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel);
|
size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel); </b>/**< note: a dict will not be used if dict == NULL or dictSize < 8 */<b>
|
||||||
size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize,
|
size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize,
|
||||||
ZSTD_parameters params, unsigned long long pledgedSrcSize); </b>/**< pledgedSrcSize is optional and can be zero == unknown */<b>
|
ZSTD_parameters params, unsigned long long pledgedSrcSize); </b>/**< pledgedSrcSize is optional and can be 0 (meaning unknown). note: if the contentSizeFlag is set, pledgedSrcSize == 0 means the source size is actually 0 */<b>
|
||||||
size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict); </b>/**< note : cdict will just be referenced, and must outlive compression session */<b>
|
size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict); </b>/**< note : cdict will just be referenced, and must outlive compression session */<b>
|
||||||
size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize); </b>/**< re-use compression parameters from previous init; skip dictionary loading stage; zcs must be init at least once before */<b>
|
size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize); </b>/**< re-use compression parameters from previous init; skip dictionary loading stage; zcs must be init at least once before. note: pledgedSrcSize must be correct, a size of 0 means unknown. for a frame size of 0 use initCStream_advanced */<b>
|
||||||
size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs);
|
size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs);
|
||||||
</b></pre><BR>
|
</pre></b><BR>
|
||||||
<h3>Advanced Streaming decompression functions</h3><pre><b>typedef enum { ZSTDdsp_maxWindowSize } ZSTD_DStreamParameter_e;
|
<h3>Advanced Streaming decompression functions</h3><pre></pre><b><pre>typedef enum { DStream_p_maxWindowSize } ZSTD_DStreamParameter_e;
|
||||||
ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem);
|
ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem);
|
||||||
size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize);
|
size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize); </b>/**< note: a dict will not be used if dict == NULL or dictSize < 8 */<b>
|
||||||
size_t ZSTD_setDStreamParameter(ZSTD_DStream* zds, ZSTD_DStreamParameter_e paramType, unsigned paramValue);
|
size_t ZSTD_setDStreamParameter(ZSTD_DStream* zds, ZSTD_DStreamParameter_e paramType, unsigned paramValue);
|
||||||
size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict); </b>/**< note : ddict will just be referenced, and must outlive decompression session */<b>
|
size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict); </b>/**< note : ddict will just be referenced, and must outlive decompression session */<b>
|
||||||
size_t ZSTD_resetDStream(ZSTD_DStream* zds); </b>/**< re-use decompression parameters from previous init; saves dictionary loading */<b>
|
size_t ZSTD_resetDStream(ZSTD_DStream* zds); </b>/**< re-use decompression parameters from previous init; saves dictionary loading */<b>
|
||||||
size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds);
|
size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds);
|
||||||
</b></pre><BR>
|
</pre></b><BR>
|
||||||
<a name="Chapter15"></a><h2>Buffer-less and synchronous inner streaming functions</h2><pre>
|
<a name="Chapter17"></a><h2>Buffer-less and synchronous inner streaming functions</h2><pre>
|
||||||
This is an advanced API, giving full control over buffer management, for users which need direct control over memory.
|
This is an advanced API, giving full control over buffer management, for users which need direct control over memory.
|
||||||
But it's also a complex one, with many restrictions (documented below).
|
But it's also a complex one, with many restrictions (documented below).
|
||||||
Prefer using normal streaming API for an easier experience
|
Prefer using normal streaming API for an easier experience
|
||||||
|
|
||||||
<BR></pre>
|
<BR></pre>
|
||||||
|
|
||||||
<a name="Chapter16"></a><h2>Buffer-less streaming compression (synchronous mode)</h2><pre>
|
<a name="Chapter18"></a><h2>Buffer-less streaming compression (synchronous mode)</h2><pre>
|
||||||
A ZSTD_CCtx object is required to track streaming operations.
|
A ZSTD_CCtx object is required to track streaming operations.
|
||||||
Use ZSTD_createCCtx() / ZSTD_freeCCtx() to manage resource.
|
Use ZSTD_createCCtx() / ZSTD_freeCCtx() to manage resource.
|
||||||
ZSTD_CCtx object can be re-used multiple times within successive compression operations.
|
ZSTD_CCtx object can be re-used multiple times within successive compression operations.
|
||||||
|
@ -434,20 +519,21 @@ size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds);
|
||||||
In which case, it will "discard" the relevant memory section from its history.
|
In which case, it will "discard" the relevant memory section from its history.
|
||||||
|
|
||||||
Finish a frame with ZSTD_compressEnd(), which will write the last block(s) and optional checksum.
|
Finish a frame with ZSTD_compressEnd(), which will write the last block(s) and optional checksum.
|
||||||
It's possible to use a NULL,0 src content, in which case, it will write a final empty block to end the frame,
|
It's possible to use srcSize==0, in which case, it will write a final empty block to end the frame.
|
||||||
Without last block mark, frames will be considered unfinished (broken) by decoders.
|
Without last block mark, frames will be considered unfinished (corrupted) by decoders.
|
||||||
|
|
||||||
You can then reuse `ZSTD_CCtx` (ZSTD_compressBegin()) to compress some new frame.
|
`ZSTD_CCtx` object can be re-used (ZSTD_compressBegin()) to compress some new frame.
|
||||||
<BR></pre>
|
<BR></pre>
|
||||||
|
|
||||||
<h3>Buffer-less streaming compression functions</h3><pre><b>size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel);
|
<h3>Buffer-less streaming compression functions</h3><pre></pre><b><pre>size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel);
|
||||||
size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel);
|
size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel);
|
||||||
size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize);
|
size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize); </b>/**< pledgedSrcSize is optional and can be 0 (meaning unknown). note: if the contentSizeFlag is set, pledgedSrcSize == 0 means the source size is actually 0 */<b>
|
||||||
size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize);
|
size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize); </b>/**< note: if pledgedSrcSize can be 0, indicating unknown size. if it is non-zero, it must be accurate. for 0 size frames, use compressBegin_advanced */<b>
|
||||||
|
size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict, unsigned long long pledgedSrcSize); </b>/**< note: if pledgedSrcSize can be 0, indicating unknown size. if it is non-zero, it must be accurate. for 0 size frames, use compressBegin_advanced */<b>
|
||||||
size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
|
size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
|
||||||
size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
|
size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
|
||||||
</b></pre><BR>
|
</pre></b><BR>
|
||||||
<a name="Chapter17"></a><h2>Buffer-less streaming decompression (synchronous mode)</h2><pre>
|
<a name="Chapter19"></a><h2>Buffer-less streaming decompression (synchronous mode)</h2><pre>
|
||||||
A ZSTD_DCtx object is required to track streaming operations.
|
A ZSTD_DCtx object is required to track streaming operations.
|
||||||
Use ZSTD_createDCtx() / ZSTD_freeDCtx() to manage it.
|
Use ZSTD_createDCtx() / ZSTD_freeDCtx() to manage it.
|
||||||
A ZSTD_DCtx object can be re-used multiple times.
|
A ZSTD_DCtx object can be re-used multiple times.
|
||||||
|
@ -490,7 +576,7 @@ size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const vo
|
||||||
Note : it's possible to know if next input to present is a header or a block, using ZSTD_nextInputType().
|
Note : it's possible to know if next input to present is a header or a block, using ZSTD_nextInputType().
|
||||||
This information is not required to properly decode a frame.
|
This information is not required to properly decode a frame.
|
||||||
|
|
||||||
== Special case : skippable frames ==
|
== Special case : skippable frames
|
||||||
|
|
||||||
Skippable frames allow integration of user-defined data into a flow of concatenated frames.
|
Skippable frames allow integration of user-defined data into a flow of concatenated frames.
|
||||||
Skippable frames will be ignored (skipped) by a decompressor. The format of skippable frames is as follows :
|
Skippable frames will be ignored (skipped) by a decompressor. The format of skippable frames is as follows :
|
||||||
|
@ -509,7 +595,7 @@ size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const vo
|
||||||
unsigned checksumFlag;
|
unsigned checksumFlag;
|
||||||
} ZSTD_frameParams;
|
} ZSTD_frameParams;
|
||||||
</b></pre><BR>
|
</b></pre><BR>
|
||||||
<h3>Buffer-less streaming decompression functions</h3><pre><b>size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t srcSize); </b>/**< doesn't consume input, see details below */<b>
|
<h3>Buffer-less streaming decompression functions</h3><pre></pre><b><pre>size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t srcSize); </b>/**< doesn't consume input, see details below */<b>
|
||||||
size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx);
|
size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx);
|
||||||
size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t dictSize);
|
size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t dictSize);
|
||||||
void ZSTD_copyDCtx(ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx);
|
void ZSTD_copyDCtx(ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx);
|
||||||
|
@ -517,8 +603,8 @@ size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx);
|
||||||
size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
|
size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
|
||||||
typedef enum { ZSTDnit_frameHeader, ZSTDnit_blockHeader, ZSTDnit_block, ZSTDnit_lastBlock, ZSTDnit_checksum, ZSTDnit_skippableFrame } ZSTD_nextInputType_e;
|
typedef enum { ZSTDnit_frameHeader, ZSTDnit_blockHeader, ZSTDnit_block, ZSTDnit_lastBlock, ZSTDnit_checksum, ZSTDnit_skippableFrame } ZSTD_nextInputType_e;
|
||||||
ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx);
|
ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx);
|
||||||
</b></pre><BR>
|
</pre></b><BR>
|
||||||
<a name="Chapter18"></a><h2>Block functions</h2><pre>
|
<a name="Chapter20"></a><h2>Block functions</h2><pre>
|
||||||
Block functions produce and decode raw zstd blocks, without frame metadata.
|
Block functions produce and decode raw zstd blocks, without frame metadata.
|
||||||
Frame metadata cost is typically ~18 bytes, which can be non-negligible for very small blocks (< 100 bytes).
|
Frame metadata cost is typically ~18 bytes, which can be non-negligible for very small blocks (< 100 bytes).
|
||||||
User will have to take in charge required information to regenerate data, such as compressed and content sizes.
|
User will have to take in charge required information to regenerate data, such as compressed and content sizes.
|
||||||
|
@ -542,10 +628,10 @@ ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx);
|
||||||
Use ZSTD_insertBlock() in such a case.
|
Use ZSTD_insertBlock() in such a case.
|
||||||
<BR></pre>
|
<BR></pre>
|
||||||
|
|
||||||
<h3>Raw zstd block functions</h3><pre><b>size_t ZSTD_getBlockSizeMax(ZSTD_CCtx* cctx);
|
<h3>Raw zstd block functions</h3><pre></pre><b><pre>size_t ZSTD_getBlockSizeMax(ZSTD_CCtx* cctx);
|
||||||
size_t ZSTD_compressBlock (ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
|
size_t ZSTD_compressBlock (ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
|
||||||
size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
|
size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
|
||||||
size_t ZSTD_insertBlock(ZSTD_DCtx* dctx, const void* blockStart, size_t blockSize); </b>/**< insert block into `dctx` history. Useful for uncompressed blocks */<b>
|
size_t ZSTD_insertBlock(ZSTD_DCtx* dctx, const void* blockStart, size_t blockSize); </b>/**< insert block into `dctx` history. Useful for uncompressed blocks */<b>
|
||||||
</b></pre><BR>
|
</pre></b><BR>
|
||||||
</html>
|
</html>
|
||||||
</body>
|
</body>
|
||||||
|
|
|
@ -369,7 +369,7 @@ unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize)
|
||||||
totalDstSize += ret;
|
totalDstSize += ret;
|
||||||
}
|
}
|
||||||
{
|
{
|
||||||
size_t const frameSrcSize = ZSTD_getFrameCompressedSize(src, srcSize);
|
size_t const frameSrcSize = ZSTD_findFrameCompressedSize(src, srcSize);
|
||||||
if (ZSTD_isError(frameSrcSize)) {
|
if (ZSTD_isError(frameSrcSize)) {
|
||||||
return ZSTD_CONTENTSIZE_ERROR;
|
return ZSTD_CONTENTSIZE_ERROR;
|
||||||
}
|
}
|
||||||
|
@ -1437,17 +1437,20 @@ size_t ZSTD_generateNxBytes(void* dst, size_t dstCapacity, BYTE byte, size_t len
|
||||||
return length;
|
return length;
|
||||||
}
|
}
|
||||||
|
|
||||||
/** ZSTD_getFrameCompressedSize() :
|
/** ZSTD_findFrameCompressedSize() :
|
||||||
* compatible with legacy mode
|
* compatible with legacy mode
|
||||||
* `src` must point to the start of a ZSTD or ZSTD legacy frame
|
* `src` must point to the start of a ZSTD frame, ZSTD legacy frame, or skippable frame
|
||||||
* `srcSize` must be at least as large as the frame contained
|
* `srcSize` must be at least as large as the frame contained
|
||||||
* @return : the compressed size of the frame starting at `src` */
|
* @return : the compressed size of the frame starting at `src` */
|
||||||
size_t ZSTD_getFrameCompressedSize(const void *src, size_t srcSize)
|
size_t ZSTD_findFrameCompressedSize(const void *src, size_t srcSize)
|
||||||
{
|
{
|
||||||
#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1)
|
#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1)
|
||||||
if (ZSTD_isLegacy(src, srcSize)) return ZSTD_getFrameCompressedSizeLegacy(src, srcSize);
|
if (ZSTD_isLegacy(src, srcSize)) return ZSTD_findFrameCompressedSizeLegacy(src, srcSize);
|
||||||
#endif
|
#endif
|
||||||
{
|
if (srcSize >= ZSTD_skippableHeaderSize &&
|
||||||
|
(MEM_readLE32(src) & 0xFFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) {
|
||||||
|
return ZSTD_skippableHeaderSize + MEM_readLE32((const BYTE*)src + 4);
|
||||||
|
} else {
|
||||||
const BYTE* ip = (const BYTE*)src;
|
const BYTE* ip = (const BYTE*)src;
|
||||||
const BYTE* const ipstart = ip;
|
const BYTE* const ipstart = ip;
|
||||||
size_t remainingSize = srcSize;
|
size_t remainingSize = srcSize;
|
||||||
|
@ -1576,7 +1579,7 @@ static size_t ZSTD_decompressMultiFrame(ZSTD_DCtx* dctx,
|
||||||
|
|
||||||
#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1)
|
#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1)
|
||||||
if (ZSTD_isLegacy(src, srcSize)) {
|
if (ZSTD_isLegacy(src, srcSize)) {
|
||||||
size_t const frameSize = ZSTD_getFrameCompressedSizeLegacy(src, srcSize);
|
size_t const frameSize = ZSTD_findFrameCompressedSizeLegacy(src, srcSize);
|
||||||
size_t decodedSize;
|
size_t decodedSize;
|
||||||
if (ZSTD_isError(frameSize)) return frameSize;
|
if (ZSTD_isError(frameSize)) return frameSize;
|
||||||
|
|
||||||
|
|
|
@ -123,26 +123,26 @@ MEM_STATIC size_t ZSTD_decompressLegacy(
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
MEM_STATIC size_t ZSTD_getFrameCompressedSizeLegacy(const void *src,
|
MEM_STATIC size_t ZSTD_findFrameCompressedSizeLegacy(const void *src,
|
||||||
size_t compressedSize)
|
size_t compressedSize)
|
||||||
{
|
{
|
||||||
U32 const version = ZSTD_isLegacy(src, compressedSize);
|
U32 const version = ZSTD_isLegacy(src, compressedSize);
|
||||||
switch(version)
|
switch(version)
|
||||||
{
|
{
|
||||||
case 1 :
|
case 1 :
|
||||||
return ZSTDv01_getFrameCompressedSize(src, compressedSize);
|
return ZSTDv01_findFrameCompressedSize(src, compressedSize);
|
||||||
case 2 :
|
case 2 :
|
||||||
return ZSTDv02_getFrameCompressedSize(src, compressedSize);
|
return ZSTDv02_findFrameCompressedSize(src, compressedSize);
|
||||||
case 3 :
|
case 3 :
|
||||||
return ZSTDv03_getFrameCompressedSize(src, compressedSize);
|
return ZSTDv03_findFrameCompressedSize(src, compressedSize);
|
||||||
case 4 :
|
case 4 :
|
||||||
return ZSTDv04_getFrameCompressedSize(src, compressedSize);
|
return ZSTDv04_findFrameCompressedSize(src, compressedSize);
|
||||||
case 5 :
|
case 5 :
|
||||||
return ZSTDv05_getFrameCompressedSize(src, compressedSize);
|
return ZSTDv05_findFrameCompressedSize(src, compressedSize);
|
||||||
case 6 :
|
case 6 :
|
||||||
return ZSTDv06_getFrameCompressedSize(src, compressedSize);
|
return ZSTDv06_findFrameCompressedSize(src, compressedSize);
|
||||||
case 7 :
|
case 7 :
|
||||||
return ZSTDv07_getFrameCompressedSize(src, compressedSize);
|
return ZSTDv07_findFrameCompressedSize(src, compressedSize);
|
||||||
default :
|
default :
|
||||||
return ERROR(prefix_unknown);
|
return ERROR(prefix_unknown);
|
||||||
}
|
}
|
||||||
|
|
|
@ -1992,7 +1992,7 @@ size_t ZSTDv01_decompress(void* dst, size_t maxDstSize, const void* src, size_t
|
||||||
return ZSTDv01_decompressDCtx(&ctx, dst, maxDstSize, src, srcSize);
|
return ZSTDv01_decompressDCtx(&ctx, dst, maxDstSize, src, srcSize);
|
||||||
}
|
}
|
||||||
|
|
||||||
size_t ZSTDv01_getFrameCompressedSize(const void* src, size_t srcSize)
|
size_t ZSTDv01_findFrameCompressedSize(const void* src, size_t srcSize)
|
||||||
{
|
{
|
||||||
const BYTE* ip = (const BYTE*)src;
|
const BYTE* ip = (const BYTE*)src;
|
||||||
size_t remainingSize = srcSize;
|
size_t remainingSize = srcSize;
|
||||||
|
|
|
@ -40,7 +40,7 @@ ZSTDv01_getFrameSrcSize() : get the source length of a ZSTD frame compliant with
|
||||||
return : the number of bytes that would be read to decompress this frame
|
return : the number of bytes that would be read to decompress this frame
|
||||||
or an errorCode if it fails (which can be tested using ZSTDv01_isError())
|
or an errorCode if it fails (which can be tested using ZSTDv01_isError())
|
||||||
*/
|
*/
|
||||||
size_t ZSTDv01_getFrameCompressedSize(const void* src, size_t compressedSize);
|
size_t ZSTDv01_findFrameCompressedSize(const void* src, size_t compressedSize);
|
||||||
|
|
||||||
/**
|
/**
|
||||||
ZSTDv01_isError() : tells if the result of ZSTDv01_decompress() is an error
|
ZSTDv01_isError() : tells if the result of ZSTDv01_decompress() is an error
|
||||||
|
|
|
@ -3378,7 +3378,7 @@ static size_t ZSTD_decompress(void* dst, size_t maxDstSize, const void* src, siz
|
||||||
return ZSTD_decompressDCtx(&ctx, dst, maxDstSize, src, srcSize);
|
return ZSTD_decompressDCtx(&ctx, dst, maxDstSize, src, srcSize);
|
||||||
}
|
}
|
||||||
|
|
||||||
static size_t ZSTD_getFrameCompressedSize(const void *src, size_t srcSize)
|
static size_t ZSTD_findFrameCompressedSize(const void *src, size_t srcSize)
|
||||||
{
|
{
|
||||||
|
|
||||||
const BYTE* ip = (const BYTE*)src;
|
const BYTE* ip = (const BYTE*)src;
|
||||||
|
@ -3524,9 +3524,9 @@ size_t ZSTDv02_decompress( void* dst, size_t maxOriginalSize,
|
||||||
return ZSTD_decompress(dst, maxOriginalSize, src, compressedSize);
|
return ZSTD_decompress(dst, maxOriginalSize, src, compressedSize);
|
||||||
}
|
}
|
||||||
|
|
||||||
size_t ZSTDv02_getFrameCompressedSize(const void *src, size_t compressedSize)
|
size_t ZSTDv02_findFrameCompressedSize(const void *src, size_t compressedSize)
|
||||||
{
|
{
|
||||||
return ZSTD_getFrameCompressedSize(src, compressedSize);
|
return ZSTD_findFrameCompressedSize(src, compressedSize);
|
||||||
}
|
}
|
||||||
|
|
||||||
ZSTDv02_Dctx* ZSTDv02_createDCtx(void)
|
ZSTDv02_Dctx* ZSTDv02_createDCtx(void)
|
||||||
|
|
|
@ -40,7 +40,7 @@ ZSTDv02_getFrameSrcSize() : get the source length of a ZSTD frame compliant with
|
||||||
return : the number of bytes that would be read to decompress this frame
|
return : the number of bytes that would be read to decompress this frame
|
||||||
or an errorCode if it fails (which can be tested using ZSTDv02_isError())
|
or an errorCode if it fails (which can be tested using ZSTDv02_isError())
|
||||||
*/
|
*/
|
||||||
size_t ZSTDv02_getFrameCompressedSize(const void* src, size_t compressedSize);
|
size_t ZSTDv02_findFrameCompressedSize(const void* src, size_t compressedSize);
|
||||||
|
|
||||||
/**
|
/**
|
||||||
ZSTDv02_isError() : tells if the result of ZSTDv02_decompress() is an error
|
ZSTDv02_isError() : tells if the result of ZSTDv02_decompress() is an error
|
||||||
|
|
|
@ -3019,7 +3019,7 @@ static size_t ZSTD_decompress(void* dst, size_t maxDstSize, const void* src, siz
|
||||||
return ZSTD_decompressDCtx(&ctx, dst, maxDstSize, src, srcSize);
|
return ZSTD_decompressDCtx(&ctx, dst, maxDstSize, src, srcSize);
|
||||||
}
|
}
|
||||||
|
|
||||||
static size_t ZSTD_getFrameCompressedSize(const void* src, size_t srcSize)
|
static size_t ZSTD_findFrameCompressedSize(const void* src, size_t srcSize)
|
||||||
{
|
{
|
||||||
const BYTE* ip = (const BYTE*)src;
|
const BYTE* ip = (const BYTE*)src;
|
||||||
size_t remainingSize = srcSize;
|
size_t remainingSize = srcSize;
|
||||||
|
@ -3165,9 +3165,9 @@ size_t ZSTDv03_decompress( void* dst, size_t maxOriginalSize,
|
||||||
return ZSTD_decompress(dst, maxOriginalSize, src, compressedSize);
|
return ZSTD_decompress(dst, maxOriginalSize, src, compressedSize);
|
||||||
}
|
}
|
||||||
|
|
||||||
size_t ZSTDv03_getFrameCompressedSize(const void* src, size_t srcSize)
|
size_t ZSTDv03_findFrameCompressedSize(const void* src, size_t srcSize)
|
||||||
{
|
{
|
||||||
return ZSTD_getFrameCompressedSize(src, srcSize);
|
return ZSTD_findFrameCompressedSize(src, srcSize);
|
||||||
}
|
}
|
||||||
|
|
||||||
ZSTDv03_Dctx* ZSTDv03_createDCtx(void)
|
ZSTDv03_Dctx* ZSTDv03_createDCtx(void)
|
||||||
|
|
|
@ -40,7 +40,7 @@ ZSTDv03_getFrameSrcSize() : get the source length of a ZSTD frame compliant with
|
||||||
return : the number of bytes that would be read to decompress this frame
|
return : the number of bytes that would be read to decompress this frame
|
||||||
or an errorCode if it fails (which can be tested using ZSTDv03_isError())
|
or an errorCode if it fails (which can be tested using ZSTDv03_isError())
|
||||||
*/
|
*/
|
||||||
size_t ZSTDv03_getFrameCompressedSize(const void* src, size_t compressedSize);
|
size_t ZSTDv03_findFrameCompressedSize(const void* src, size_t compressedSize);
|
||||||
|
|
||||||
/**
|
/**
|
||||||
ZSTDv03_isError() : tells if the result of ZSTDv03_decompress() is an error
|
ZSTDv03_isError() : tells if the result of ZSTDv03_decompress() is an error
|
||||||
|
|
|
@ -3326,7 +3326,7 @@ static size_t ZSTD_decompress_usingDict(ZSTD_DCtx* ctx,
|
||||||
return op-ostart;
|
return op-ostart;
|
||||||
}
|
}
|
||||||
|
|
||||||
static size_t ZSTD_getFrameCompressedSize(const void* src, size_t srcSize)
|
static size_t ZSTD_findFrameCompressedSize(const void* src, size_t srcSize)
|
||||||
{
|
{
|
||||||
const BYTE* ip = (const BYTE*)src;
|
const BYTE* ip = (const BYTE*)src;
|
||||||
size_t remainingSize = srcSize;
|
size_t remainingSize = srcSize;
|
||||||
|
@ -3782,9 +3782,9 @@ size_t ZSTDv04_decompress(void* dst, size_t maxDstSize, const void* src, size_t
|
||||||
#endif
|
#endif
|
||||||
}
|
}
|
||||||
|
|
||||||
size_t ZSTDv04_getFrameCompressedSize(const void* src, size_t srcSize)
|
size_t ZSTDv04_findFrameCompressedSize(const void* src, size_t srcSize)
|
||||||
{
|
{
|
||||||
return ZSTD_getFrameCompressedSize(src, srcSize);
|
return ZSTD_findFrameCompressedSize(src, srcSize);
|
||||||
}
|
}
|
||||||
|
|
||||||
size_t ZSTDv04_resetDCtx(ZSTDv04_Dctx* dctx) { return ZSTD_resetDCtx(dctx); }
|
size_t ZSTDv04_resetDCtx(ZSTDv04_Dctx* dctx) { return ZSTD_resetDCtx(dctx); }
|
||||||
|
|
|
@ -40,7 +40,7 @@ ZSTDv04_getFrameSrcSize() : get the source length of a ZSTD frame compliant with
|
||||||
return : the number of bytes that would be read to decompress this frame
|
return : the number of bytes that would be read to decompress this frame
|
||||||
or an errorCode if it fails (which can be tested using ZSTDv04_isError())
|
or an errorCode if it fails (which can be tested using ZSTDv04_isError())
|
||||||
*/
|
*/
|
||||||
size_t ZSTDv04_getFrameCompressedSize(const void* src, size_t compressedSize);
|
size_t ZSTDv04_findFrameCompressedSize(const void* src, size_t compressedSize);
|
||||||
|
|
||||||
/**
|
/**
|
||||||
ZSTDv04_isError() : tells if the result of ZSTDv04_decompress() is an error
|
ZSTDv04_isError() : tells if the result of ZSTDv04_decompress() is an error
|
||||||
|
|
|
@ -3583,7 +3583,7 @@ size_t ZSTDv05_decompress(void* dst, size_t maxDstSize, const void* src, size_t
|
||||||
#endif
|
#endif
|
||||||
}
|
}
|
||||||
|
|
||||||
size_t ZSTDv05_getFrameCompressedSize(const void *src, size_t srcSize)
|
size_t ZSTDv05_findFrameCompressedSize(const void *src, size_t srcSize)
|
||||||
{
|
{
|
||||||
const BYTE* ip = (const BYTE*)src;
|
const BYTE* ip = (const BYTE*)src;
|
||||||
size_t remainingSize = srcSize;
|
size_t remainingSize = srcSize;
|
||||||
|
|
|
@ -38,7 +38,7 @@ ZSTDv05_getFrameSrcSize() : get the source length of a ZSTD frame
|
||||||
return : the number of bytes that would be read to decompress this frame
|
return : the number of bytes that would be read to decompress this frame
|
||||||
or an errorCode if it fails (which can be tested using ZSTDv05_isError())
|
or an errorCode if it fails (which can be tested using ZSTDv05_isError())
|
||||||
*/
|
*/
|
||||||
size_t ZSTDv05_getFrameCompressedSize(const void* src, size_t compressedSize);
|
size_t ZSTDv05_findFrameCompressedSize(const void* src, size_t compressedSize);
|
||||||
|
|
||||||
/* *************************************
|
/* *************************************
|
||||||
* Helper functions
|
* Helper functions
|
||||||
|
|
|
@ -3729,7 +3729,7 @@ size_t ZSTDv06_decompress(void* dst, size_t dstCapacity, const void* src, size_t
|
||||||
#endif
|
#endif
|
||||||
}
|
}
|
||||||
|
|
||||||
size_t ZSTDv06_getFrameCompressedSize(const void* src, size_t srcSize)
|
size_t ZSTDv06_findFrameCompressedSize(const void* src, size_t srcSize)
|
||||||
{
|
{
|
||||||
const BYTE* ip = (const BYTE*)src;
|
const BYTE* ip = (const BYTE*)src;
|
||||||
size_t remainingSize = srcSize;
|
size_t remainingSize = srcSize;
|
||||||
|
|
|
@ -47,7 +47,7 @@ ZSTDv06_getFrameSrcSize() : get the source length of a ZSTD frame
|
||||||
return : the number of bytes that would be read to decompress this frame
|
return : the number of bytes that would be read to decompress this frame
|
||||||
or an errorCode if it fails (which can be tested using ZSTDv06_isError())
|
or an errorCode if it fails (which can be tested using ZSTDv06_isError())
|
||||||
*/
|
*/
|
||||||
size_t ZSTDv06_getFrameCompressedSize(const void* src, size_t compressedSize);
|
size_t ZSTDv06_findFrameCompressedSize(const void* src, size_t compressedSize);
|
||||||
|
|
||||||
/* *************************************
|
/* *************************************
|
||||||
* Helper functions
|
* Helper functions
|
||||||
|
|
|
@ -3968,7 +3968,7 @@ size_t ZSTDv07_decompress(void* dst, size_t dstCapacity, const void* src, size_t
|
||||||
#endif
|
#endif
|
||||||
}
|
}
|
||||||
|
|
||||||
size_t ZSTDv07_getFrameCompressedSize(const void* src, size_t srcSize)
|
size_t ZSTDv07_findFrameCompressedSize(const void* src, size_t srcSize)
|
||||||
{
|
{
|
||||||
const BYTE* ip = (const BYTE*)src;
|
const BYTE* ip = (const BYTE*)src;
|
||||||
size_t remainingSize = srcSize;
|
size_t remainingSize = srcSize;
|
||||||
|
|
|
@ -54,7 +54,7 @@ ZSTDv07_getFrameSrcSize() : get the source length of a ZSTD frame
|
||||||
return : the number of bytes that would be read to decompress this frame
|
return : the number of bytes that would be read to decompress this frame
|
||||||
or an errorCode if it fails (which can be tested using ZSTDv07_isError())
|
or an errorCode if it fails (which can be tested using ZSTDv07_isError())
|
||||||
*/
|
*/
|
||||||
size_t ZSTDv07_getFrameCompressedSize(const void* src, size_t compressedSize);
|
size_t ZSTDv07_findFrameCompressedSize(const void* src, size_t compressedSize);
|
||||||
|
|
||||||
/*====== Helper functions ======*/
|
/*====== Helper functions ======*/
|
||||||
ZSTDLIBv07_API unsigned ZSTDv07_isError(size_t code); /*!< tells if a `size_t` function result is an error code */
|
ZSTDLIBv07_API unsigned ZSTDv07_isError(size_t code); /*!< tells if a `size_t` function result is an error code */
|
||||||
|
|
|
@ -400,12 +400,12 @@ typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; v
|
||||||
* Compressed size functions
|
* Compressed size functions
|
||||||
***************************************/
|
***************************************/
|
||||||
|
|
||||||
/*! ZSTD_getFrameCompressedSize() :
|
/*! ZSTD_findFrameCompressedSize() :
|
||||||
* `src` should point to the start of a ZSTD encoded frame
|
* `src` should point to the start of a ZSTD encoded frame or skippable frame
|
||||||
* `srcSize` must be at least as large as the frame
|
* `srcSize` must be at least as large as the frame
|
||||||
* @return : the compressed size of the frame pointed to by `src`, suitable to pass to
|
* @return : the compressed size of the frame pointed to by `src`, suitable to pass to
|
||||||
* `ZSTD_decompress` or similar, or an error code if given invalid input. */
|
* `ZSTD_decompress` or similar, or an error code if given invalid input. */
|
||||||
ZSTDLIB_API size_t ZSTD_getFrameCompressedSize(const void* src, size_t srcSize);
|
ZSTDLIB_API size_t ZSTD_findFrameCompressedSize(const void* src, size_t srcSize);
|
||||||
|
|
||||||
/***************************************
|
/***************************************
|
||||||
* Decompressed size functions
|
* Decompressed size functions
|
||||||
|
@ -700,6 +700,9 @@ ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapaci
|
||||||
c) Frame Content - any content (User Data) of length equal to Frame Size
|
c) Frame Content - any content (User Data) of length equal to Frame Size
|
||||||
For skippable frames ZSTD_decompressContinue() always returns 0.
|
For skippable frames ZSTD_decompressContinue() always returns 0.
|
||||||
For skippable frames ZSTD_getFrameParams() returns fparamsPtr->windowLog==0 what means that a frame is skippable.
|
For skippable frames ZSTD_getFrameParams() returns fparamsPtr->windowLog==0 what means that a frame is skippable.
|
||||||
|
Note : If fparamsPtr->frameContentSize==0, it is ambiguous: the frame might actually be a Zstd encoded frame with no content.
|
||||||
|
For purposes of decompression, it is valid in both cases to skip the frame using
|
||||||
|
ZSTD_findFrameCompressedSize to find its size in bytes.
|
||||||
It also returns Frame Size as fparamsPtr->frameContentSize.
|
It also returns Frame Size as fparamsPtr->frameContentSize.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
|
|
|
@ -17,6 +17,7 @@ roundTripCrash
|
||||||
longmatch
|
longmatch
|
||||||
symbols
|
symbols
|
||||||
legacy
|
legacy
|
||||||
|
decodecorpus
|
||||||
pool
|
pool
|
||||||
invalidDictionaries
|
invalidDictionaries
|
||||||
|
|
||||||
|
|
|
@ -56,6 +56,7 @@ VOID = /dev/null
|
||||||
ZSTREAM_TESTTIME = -T2mn
|
ZSTREAM_TESTTIME = -T2mn
|
||||||
FUZZERTEST ?= -T5mn
|
FUZZERTEST ?= -T5mn
|
||||||
ZSTDRTTEST = --test-large-data
|
ZSTDRTTEST = --test-large-data
|
||||||
|
DECODECORPUS_TESTTIME = -T30
|
||||||
|
|
||||||
.PHONY: default all all32 dll clean test test32 test-all namespaceTest versionsTest
|
.PHONY: default all all32 dll clean test test32 test-all namespaceTest versionsTest
|
||||||
|
|
||||||
|
@ -154,6 +155,9 @@ legacy : CPPFLAGS+= -I$(ZSTDDIR)/legacy
|
||||||
legacy : $(ZSTD_FILES) $(wildcard $(ZSTDDIR)/legacy/*.c) legacy.c
|
legacy : $(ZSTD_FILES) $(wildcard $(ZSTDDIR)/legacy/*.c) legacy.c
|
||||||
$(CC) $(FLAGS) $^ -o $@$(EXT)
|
$(CC) $(FLAGS) $^ -o $@$(EXT)
|
||||||
|
|
||||||
|
decodecorpus : $(filter-out $(ZSTDDIR)/compress/zstd_compress.c, $(wildcard $(ZSTD_FILES))) decodecorpus.c
|
||||||
|
$(CC) $(FLAGS) $^ -o $@$(EXT) -lm
|
||||||
|
|
||||||
symbols : symbols.c
|
symbols : symbols.c
|
||||||
$(MAKE) -C $(ZSTDDIR) libzstd
|
$(MAKE) -C $(ZSTDDIR) libzstd
|
||||||
ifneq (,$(filter Windows%,$(OS)))
|
ifneq (,$(filter Windows%,$(OS)))
|
||||||
|
@ -184,7 +188,8 @@ clean:
|
||||||
fuzzer-dll$(EXT) zstreamtest-dll$(EXT) zbufftest-dll$(EXT)\
|
fuzzer-dll$(EXT) zstreamtest-dll$(EXT) zbufftest-dll$(EXT)\
|
||||||
zstreamtest$(EXT) zstreamtest32$(EXT) \
|
zstreamtest$(EXT) zstreamtest32$(EXT) \
|
||||||
datagen$(EXT) paramgrill$(EXT) roundTripCrash$(EXT) longmatch$(EXT) \
|
datagen$(EXT) paramgrill$(EXT) roundTripCrash$(EXT) longmatch$(EXT) \
|
||||||
symbols$(EXT) invalidDictionaries$(EXT) legacy$(EXT) pool$(EXT)
|
symbols$(EXT) invalidDictionaries$(EXT) legacy$(EXT) pool$(EXT) \
|
||||||
|
decodecorpus$(EXT)
|
||||||
@echo Cleaning completed
|
@echo Cleaning completed
|
||||||
|
|
||||||
|
|
||||||
|
@ -230,7 +235,7 @@ zstd-playTests: datagen
|
||||||
file $(ZSTD)
|
file $(ZSTD)
|
||||||
ZSTD="$(QEMU_SYS) $(ZSTD)" ./playTests.sh $(ZSTDRTTEST)
|
ZSTD="$(QEMU_SYS) $(ZSTD)" ./playTests.sh $(ZSTDRTTEST)
|
||||||
|
|
||||||
test: test-zstd test-fullbench test-fuzzer test-zstream test-invalidDictionaries test-legacy
|
test: test-zstd test-fullbench test-fuzzer test-zstream test-invalidDictionaries test-legacy test-decodecorpus
|
||||||
ifeq ($(QEMU_SYS),)
|
ifeq ($(QEMU_SYS),)
|
||||||
test: test-pool
|
test: test-pool
|
||||||
endif
|
endif
|
||||||
|
@ -302,6 +307,9 @@ test-symbols: symbols
|
||||||
test-legacy: legacy
|
test-legacy: legacy
|
||||||
$(QEMU_SYS) ./legacy
|
$(QEMU_SYS) ./legacy
|
||||||
|
|
||||||
|
test-decodecorpus: decodecorpus
|
||||||
|
$(QEMU_SYS) ./decodecorpus -t $(DECODECORPUS_TESTTIME)
|
||||||
|
|
||||||
test-pool: pool
|
test-pool: pool
|
||||||
$(QEMU_SYS) ./pool
|
$(QEMU_SYS) ./pool
|
||||||
|
|
||||||
|
|
|
@ -10,12 +10,14 @@ This directory contains the following programs and scripts:
|
||||||
- `test-zstd-versions.py` : compatibility test between zstd versions stored on Github (v0.1+)
|
- `test-zstd-versions.py` : compatibility test between zstd versions stored on Github (v0.1+)
|
||||||
- `zbufftest` : Test tool to check ZBUFF (a buffered streaming API) integrity
|
- `zbufftest` : Test tool to check ZBUFF (a buffered streaming API) integrity
|
||||||
- `zstreamtest` : Fuzzer test tool for zstd streaming API
|
- `zstreamtest` : Fuzzer test tool for zstd streaming API
|
||||||
|
- `legacy` : Test tool to test decoding of legacy zstd frames
|
||||||
|
- `decodecorpus` : Tool to generate valid Zstandard frames, for verifying decoder implementations
|
||||||
|
|
||||||
|
|
||||||
#### `test-zstd-versions.py` - script for testing zstd interoperability between versions
|
#### `test-zstd-versions.py` - script for testing zstd interoperability between versions
|
||||||
|
|
||||||
This script creates `versionsTest` directory to which zstd repository is cloned.
|
This script creates `versionsTest` directory to which zstd repository is cloned.
|
||||||
Then all taged (released) versions of zstd are compiled.
|
Then all tagged (released) versions of zstd are compiled.
|
||||||
In the following step interoperability between zstd versions is checked.
|
In the following step interoperability between zstd versions is checked.
|
||||||
|
|
||||||
|
|
||||||
|
@ -64,3 +66,25 @@ optional arguments:
|
||||||
--sleepTime SLEEPTIME
|
--sleepTime SLEEPTIME
|
||||||
frequency of repository checking in seconds
|
frequency of repository checking in seconds
|
||||||
```
|
```
|
||||||
|
|
||||||
|
#### `decodecorpus` - tool to generate Zstandard frames for decoder testing
|
||||||
|
Command line tool to generate test .zst files.
|
||||||
|
|
||||||
|
This tool will generate .zst files with checksums,
|
||||||
|
as well as optionally output the corresponding correct uncompressed data for
|
||||||
|
extra verfication.
|
||||||
|
|
||||||
|
Example:
|
||||||
|
```
|
||||||
|
./decodecorpus -ptestfiles -otestfiles -n10000 -s5
|
||||||
|
```
|
||||||
|
will generate 10,000 sample .zst files using a seed of 5 in the `testfiles` directory,
|
||||||
|
with the zstd checksum field set,
|
||||||
|
as well as the 10,000 original files for more detailed comparison of decompression results.
|
||||||
|
|
||||||
|
```
|
||||||
|
./decodecorpus -t -T1mn
|
||||||
|
```
|
||||||
|
will choose a random seed, and for 1 minute,
|
||||||
|
generate random test frames and ensure that the
|
||||||
|
zstd library correctly decompresses them in both simple and streaming modes.
|
||||||
|
|
File diff suppressed because it is too large
Load Diff
|
@ -545,6 +545,13 @@ static int basicUnitTests(U32 seed, double compressibility)
|
||||||
if (r != _3BYTESTESTLENGTH) goto _output_error; }
|
if (r != _3BYTESTESTLENGTH) goto _output_error; }
|
||||||
DISPLAYLEVEL(4, "OK \n");
|
DISPLAYLEVEL(4, "OK \n");
|
||||||
|
|
||||||
|
/* findFrameCompressedSize on skippable frames */
|
||||||
|
DISPLAYLEVEL(4, "test%3i : frame compressed size of skippable frame : ", testNb++);
|
||||||
|
{ const char* frame = "\x50\x2a\x4d\x18\x05\x0\x0\0abcde";
|
||||||
|
size_t const frameSrcSize = 13;
|
||||||
|
if (ZSTD_findFrameCompressedSize(frame, frameSrcSize) != frameSrcSize) goto _output_error; }
|
||||||
|
DISPLAYLEVEL(4, "OK \n");
|
||||||
|
|
||||||
/* error string tests */
|
/* error string tests */
|
||||||
DISPLAYLEVEL(4, "test%3i : testing ZSTD error code strings : ", testNb++);
|
DISPLAYLEVEL(4, "test%3i : testing ZSTD error code strings : ", testNb++);
|
||||||
if (strcmp("No error detected", ZSTD_getErrorName((ZSTD_ErrorCode)(0-ZSTD_error_no_error))) != 0) goto _output_error;
|
if (strcmp("No error detected", ZSTD_getErrorName((ZSTD_ErrorCode)(0-ZSTD_error_no_error))) != 0) goto _output_error;
|
||||||
|
|
|
@ -16,6 +16,7 @@ static const void *symbols[] = {
|
||||||
&ZSTD_decompress,
|
&ZSTD_decompress,
|
||||||
&ZSTD_getDecompressedSize,
|
&ZSTD_getDecompressedSize,
|
||||||
&ZSTD_findDecompressedSize,
|
&ZSTD_findDecompressedSize,
|
||||||
|
&ZSTD_findFrameCompressedSize,
|
||||||
&ZSTD_getFrameContentSize,
|
&ZSTD_getFrameContentSize,
|
||||||
&ZSTD_maxCLevel,
|
&ZSTD_maxCLevel,
|
||||||
&ZSTD_compressBound,
|
&ZSTD_compressBound,
|
||||||
|
|
Loading…
Reference in New Issue