facebook
/
zstd
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'dev' into adding_read_files_from_file_feature

dev
AhmedAbdellah19 2019-10-23 20:29:01 +02:00 committed by GitHub
parent 779ea72953 3c8ff35013
commit 7a732e2bb2
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
70 changed files with 1864 additions and 1195 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
.travis.yml
View File

@ -54,7 +54,10 @@ matrix:
- name: Trusty (clang-3.8 + MSan + Test Zstd)
script:
- make clang38install
- CC=clang-3.8 make clean msan-test-zstd
# External libraries must be turned off when using MSAN tests,
# because they are not msan-instrumented,
# so any data coming from these libraries is always considered "uninitialized"
- CC=clang-3.8 make clean msan-test-zstd HAVE_ZLIB=0 HAVE_LZ4=0 HAVE_LZMA=0
- name: Trusty (Minimal Decompressor Macros)
script:

3
README.md
View File

@ -15,6 +15,7 @@ a list of known ports and bindings is provided on [Zstandard homepage](http://ww
[![Build status][AppveyorDevBadge]][AppveyorLink]
[![Build status][CircleDevBadge]][CircleLink]
[![Build status][CirrusDevBadge]][CirrusLink]
[![Fuzzing Status][OSSFuzzBadge]][OSSFuzzLink]
[travisDevBadge]: https://travis-ci.org/facebook/zstd.svg?branch=dev "Continuous Integration test suite"
[travisLink]: https://travis-ci.org/facebook/zstd
@ -24,6 +25,8 @@ a list of known ports and bindings is provided on [Zstandard homepage](http://ww
[CircleLink]: https://circleci.com/gh/facebook/zstd
[CirrusDevBadge]: https://api.cirrus-ci.com/github/facebook/zstd.svg?branch=dev
[CirrusLink]: https://cirrus-ci.com/github/facebook/zstd
[OSSFuzzBadge]: https://oss-fuzz-build-logs.storage.googleapis.com/badges/zstd.svg
[OSSFuzzLink]: https://bugs.chromium.org/p/oss-fuzz/issues/list?sort=-opened&can=1&q=proj:zstd
## Benchmarks

17
appveyor.yml
View File

@ -1,3 +1,7 @@
# Following tests are run _only_ on master branch
# To reproduce these tests, it's possible to push into a branch `appveyorTest`
# or a branch `visual*`, they will intentionnally trigger `master` tests
-
version: 1.0.{build}
branches:
@ -176,6 +180,11 @@
fuzzer_VS2015_%PLATFORM%_Release.exe %FUZZERTEST%
)
# The following tests are for regular pushes
# into `dev` or some feature branch
# There run less tests, for shorter feedback loop
-
version: 1.0.{build}
environment:
@ -249,3 +258,11 @@
COPY build\VS2010\bin\%PLATFORM%_%CONFIGURATION%\fuzzer.exe tests\fuzzer_VS2015_%PLATFORM%_%CONFIGURATION%.exe &&
COPY build\VS2010\bin\%PLATFORM%_%CONFIGURATION%\*.exe tests\
)
test_script:
- ECHO Testing %COMPILER% %PLATFORM% %CONFIGURATION%
- if [%HOST%]==[mingw] (
set "CC=%COMPILER%" &&
make check
)

0
build/LICENSE Normal file
View File

8
build/cmake/lib/CMakeLists.txt
View File

@ -133,8 +133,8 @@ endif ()
if (UNIX)
# pkg-config
set(PREFIX "${CMAKE_INSTALL_PREFIX}")
set(LIBDIR "${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}")
set(INCLUDEDIR "${CMAKE_INSTALL_PREFIX}/include")
set(LIBDIR "${CMAKE_INSTALL_FULL_LIBDIR}")
set(INCLUDEDIR "${CMAKE_INSTALL_FULL_INCLUDEDIR}")
set(VERSION "${zstd_VERSION_MAJOR}.${zstd_VERSION_MINOR}.${zstd_VERSION_PATCH}")
add_custom_target(libzstd.pc ALL
${CMAKE_COMMAND} -DIN="${LIBRARY_DIR}/libzstd.pc.in" -DOUT="libzstd.pc"
@ -152,10 +152,10 @@ install(FILES
${LIBRARY_DIR}/dictBuilder/zdict.h
${LIBRARY_DIR}/dictBuilder/cover.h
${LIBRARY_DIR}/common/zstd_errors.h
DESTINATION "include")
DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}")
if (ZSTD_BUILD_SHARED)
install(TARGETS libzstd_shared RUNTIME DESTINATION "bin"
install(TARGETS libzstd_shared RUNTIME DESTINATION "${CMAKE_INSTALL_BINDIR}"
LIBRARY DESTINATION "${CMAKE_INSTALL_LIBDIR}"
ARCHIVE DESTINATION "${CMAKE_INSTALL_LIBDIR}")
endif()

26
build/cmake/programs/CMakeLists.txt
View File

@ -31,15 +31,15 @@ target_link_libraries(zstd libzstd_static)
if (CMAKE_SYSTEM_NAME MATCHES "(Solaris|SunOS)")
target_link_libraries(zstd rt)
endif ()
install(TARGETS zstd RUNTIME DESTINATION "bin")
install(TARGETS zstd RUNTIME DESTINATION "${CMAKE_INSTALL_BINDIR}")
if (UNIX)
add_custom_target(zstdcat ALL ${CMAKE_COMMAND} -E create_symlink zstd zstdcat DEPENDS zstd COMMENT "Creating zstdcat symlink")
add_custom_target(unzstd ALL ${CMAKE_COMMAND} -E create_symlink zstd unzstd DEPENDS zstd COMMENT "Creating unzstd symlink")
install(FILES ${CMAKE_CURRENT_BINARY_DIR}/zstdcat DESTINATION "bin")
install(FILES ${CMAKE_CURRENT_BINARY_DIR}/unzstd DESTINATION "bin")
install(PROGRAMS ${PROGRAMS_DIR}/zstdgrep DESTINATION "bin")
install(PROGRAMS ${PROGRAMS_DIR}/zstdless DESTINATION "bin")
install(FILES ${CMAKE_CURRENT_BINARY_DIR}/zstdcat DESTINATION "${CMAKE_INSTALL_BINDIR}")
install(FILES ${CMAKE_CURRENT_BINARY_DIR}/unzstd DESTINATION "${CMAKE_INSTALL_BINDIR}")
install(PROGRAMS ${PROGRAMS_DIR}/zstdgrep DESTINATION "${CMAKE_INSTALL_BINDIR}")
install(PROGRAMS ${PROGRAMS_DIR}/zstdless DESTINATION "${CMAKE_INSTALL_BINDIR}")
add_custom_target(zstd.1 ALL
${CMAKE_COMMAND} -E copy ${PROGRAMS_DIR}/zstd.1 .
@ -56,14 +56,16 @@ if (UNIX)
# Define MAN_INSTALL_DIR if necessary
if (MAN_INSTALL_DIR)
else ()
set(MAN_INSTALL_DIR ${CMAKE_INSTALL_PREFIX}/share/man/man1)
set(MAN_INSTALL_DIR ${CMAKE_INSTALL_MANDIR}/man1)
endif ()
install(FILES ${CMAKE_CURRENT_BINARY_DIR}/zstd.1 DESTINATION "${MAN_INSTALL_DIR}")
install(FILES ${CMAKE_CURRENT_BINARY_DIR}/zstdcat.1 DESTINATION "${MAN_INSTALL_DIR}")
install(FILES ${CMAKE_CURRENT_BINARY_DIR}/unzstd.1 DESTINATION "${MAN_INSTALL_DIR}")
install(FILES ${CMAKE_CURRENT_BINARY_DIR}/zstdgrep.1 DESTINATION "${MAN_INSTALL_DIR}")
install(FILES ${CMAKE_CURRENT_BINARY_DIR}/zstdless.1 DESTINATION "${MAN_INSTALL_DIR}")
install(FILES
${CMAKE_CURRENT_BINARY_DIR}/zstd.1
${CMAKE_CURRENT_BINARY_DIR}/zstdcat.1
${CMAKE_CURRENT_BINARY_DIR}/unzstd.1
${CMAKE_CURRENT_BINARY_DIR}/zstdgrep.1
${CMAKE_CURRENT_BINARY_DIR}/zstdless.1
DESTINATION "${MAN_INSTALL_DIR}")
add_executable(zstd-frugal ${PROGRAMS_DIR}/zstdcli.c ${PROGRAMS_DIR}/util.c ${PROGRAMS_DIR}/timefn.c ${PROGRAMS_DIR}/fileio.c)
target_link_libraries(zstd-frugal libzstd_static)
@ -79,7 +81,7 @@ if (ZSTD_MULTITHREAD_SUPPORT)
target_link_libraries(zstd ${THREADS_LIBS})
add_custom_target(zstdmt ALL ${CMAKE_COMMAND} -E create_symlink zstd zstdmt DEPENDS zstd COMMENT "Creating zstdmt symlink")
install(FILES ${CMAKE_CURRENT_BINARY_DIR}/zstdmt DESTINATION "bin")
install(FILES ${CMAKE_CURRENT_BINARY_DIR}/zstdmt DESTINATION "${CMAKE_INSTALL_BINDIR}")
endif ()
endif ()

2
doc/educational_decoder/.gitignore vendored Normal file
View File

@ -0,0 +1,2 @@
# Build artifacts
harness

24
doc/educational_decoder/Makefile
View File

@ -7,8 +7,15 @@
# in the COPYING file in the root directory of this source tree).
# ################################################################
ZSTD ?= zstd # requires zstd installation on local system
ZSTD ?= zstd # note: requires zstd installation on local system
UNAME?= $(shell uname)
ifeq ($(UNAME), SunOS)
DIFF ?= gdiff
else
DIFF ?= diff
endif
HARNESS_FILES=*.c
MULTITHREAD_LDFLAGS = -pthread
@ -29,26 +36,27 @@ harness: $(HARNESS_FILES)
$(CC) $(FLAGS) $^ -o $@
clean:
@$(RM) -f harness
@$(RM) -rf harness.dSYM
@$(RM) harness
@$(RM) -rf harness.dSYM # MacOS specific
test: harness
#
# Testing single-file decompression with educational decoder
#
@$(ZSTD) README.md -o tmp.zst
@$(ZSTD) -f README.md -o tmp.zst
@./harness tmp.zst tmp
@$(DIFF) -s tmp README.md
@$(RM) -f tmp*
@$(RM) tmp*
#
# Testing dictionary decompression with education decoder
#
# note : files are presented multiple for training, to reach minimum threshold
@$(ZSTD) --train harness.c zstd_decompress.c zstd_decompress.h README.md \
harness.c zstd_decompress.c zstd_decompress.h README.md \
harness.c zstd_decompress.c zstd_decompress.h README.md
@$(ZSTD) -D dictionary README.md -o tmp.zst
harness.c zstd_decompress.c zstd_decompress.h README.md \
-o dictionary
@$(ZSTD) -f README.md -D dictionary -o tmp.zst
@./harness tmp.zst tmp dictionary
@$(DIFF) -s tmp README.md
@$(RM) -f tmp* dictionary
@$(RM) tmp* dictionary
@$(MAKE) clean

81
doc/educational_decoder/harness.c
View File

@ -21,88 +21,90 @@ typedef unsigned char u8;
// Protect against allocating too much memory for output
#define MAX_OUTPUT_SIZE ((size_t)1024 * 1024 * 1024)
u8 *input;
u8 *output;
u8 *dict;
size_t read_file(const char *path, u8 **ptr) {
FILE *f = fopen(path, "rb");
static size_t read_file(const char *path, u8 **ptr)
{
FILE* const f = fopen(path, "rb");
if (!f) {
fprintf(stderr, "failed to open file %s\n", path);
fprintf(stderr, "failed to open file %s \n", path);
exit(1);
}
fseek(f, 0L, SEEK_END);
size_t size = (size_t)ftell(f);
size_t const size = (size_t)ftell(f);
rewind(f);
*ptr = malloc(size);
if (!ptr) {
fprintf(stderr, "failed to allocate memory to hold %s\n", path);
fprintf(stderr, "failed to allocate memory to hold %s \n", path);
exit(1);
}
size_t pos = 0;
while (!feof(f)) {
size_t read = fread(&(*ptr)[pos], 1, size, f);
if (ferror(f)) {
fprintf(stderr, "error while reading file %s\n", path);
size_t const read = fread(*ptr, 1, size, f);
if (read != size) { /* must read everything in one pass */
fprintf(stderr, "error while reading file %s \n", path);
exit(1);
}
pos += read;
}
fclose(f);
return pos;
return read;
}
void write_file(const char *path, const u8 *ptr, size_t size) {
FILE *f = fopen(path, "wb");
static void write_file(const char *path, const u8 *ptr, size_t size)
{
FILE* const f = fopen(path, "wb");
if (!f) {
fprintf(stderr, "failed to open file %s \n", path);
exit(1);
}
size_t written = 0;
while (written < size) {
written += fwrite(&ptr[written], 1, size, f);
written += fwrite(ptr+written, 1, size, f);
if (ferror(f)) {
fprintf(stderr, "error while writing file %s\n", path);
exit(1);
}
}
} }
fclose(f);
}
int main(int argc, char **argv) {
int main(int argc, char **argv)
{
if (argc < 3) {
fprintf(stderr, "usage: %s <file.zst> <out_path> [dictionary]\n",
fprintf(stderr, "usage: %s <file.zst> <out_path> [dictionary] \n",
argv[0]);
return 1;
}
size_t input_size = read_file(argv[1], &input);
u8* input;
size_t const input_size = read_file(argv[1], &input);
u8* dict = NULL;
size_t dict_size = 0;
if (argc >= 4) {
dict_size = read_file(argv[3], &dict);
}
size_t decompressed_size = ZSTD_get_decompressed_size(input, input_size);
if (decompressed_size == (size_t)-1) {
decompressed_size = MAX_COMPRESSION_RATIO * input_size;
size_t out_capacity = ZSTD_get_decompressed_size(input, input_size);
if (out_capacity == (size_t)-1) {
out_capacity = MAX_COMPRESSION_RATIO * input_size;
fprintf(stderr, "WARNING: Compressed data does not contain "
"decompressed size, going to assume the compression "
"ratio is at most %d (decompressed size of at most "
"%zu)\n",
MAX_COMPRESSION_RATIO, decompressed_size);
"%u) \n",
MAX_COMPRESSION_RATIO, (unsigned)out_capacity);
}
if (decompressed_size > MAX_OUTPUT_SIZE) {
if (out_capacity > MAX_OUTPUT_SIZE) {
fprintf(stderr,
"Required output size too large for this implementation\n");
"Required output size too large for this implementation \n");
return 1;
}
output = malloc(decompressed_size);
u8* const output = malloc(out_capacity);
if (!output) {
fprintf(stderr, "failed to allocate memory\n");
fprintf(stderr, "failed to allocate memory \n");
return 1;
}
@ -110,16 +112,17 @@ int main(int argc, char **argv) {
if (dict) {
parse_dictionary(parsed_dict, dict, dict_size);
}
size_t decompressed =
ZSTD_decompress_with_dict(output, decompressed_size,
input, input_size, parsed_dict);
size_t const decompressed_size =
ZSTD_decompress_with_dict(output, out_capacity,
input, input_size,
parsed_dict);
free_dictionary(parsed_dict);
write_file(argv[2], output, decompressed);
write_file(argv[2], output, decompressed_size);
free(input);
free(output);
free(dict);
input = output = dict = NULL;
return 0;
}

9
doc/educational_decoder/zstd_decompress.c
View File

@ -856,8 +856,7 @@ static size_t decode_literals_compressed(frame_context_t *const ctx,
// Impossible
IMPOSSIBLE();
}
if (regenerated_size > MAX_LITERALS_SIZE ||
compressed_size >= regenerated_size) {
if (regenerated_size > MAX_LITERALS_SIZE) {
CORRUPTION();
}
@ -1530,7 +1529,7 @@ void free_dictionary(dictionary_t *const dict) {
/******* END DICTIONARY PARSING ***********************************************/
/******* IO STREAM OPERATIONS *************************************************/
#define UNALIGNED() ERROR("Attempting to operate on a non-byte aligned stream")
/// Reads `num` bits from a bitstream, and updates the internal offset
static inline u64 IO_read_bits(istream_t *const in, const int num_bits) {
if (num_bits > 64 || num_bits <= 0) {
@ -1609,7 +1608,7 @@ static inline const u8 *IO_get_read_ptr(istream_t *const in, size_t len) {
INP_SIZE();
}
if (in->bit_offset != 0) {
UNALIGNED();
ERROR("Attempting to operate on a non-byte aligned stream");
}
const u8 *const ptr = in->ptr;
in->ptr += len;
@ -1635,7 +1634,7 @@ static inline void IO_advance_input(istream_t *const in, size_t len) {
INP_SIZE();
}
if (in->bit_offset != 0) {
UNALIGNED();
ERROR("Attempting to operate on a non-byte aligned stream");
}
in->ptr += len;

46
doc/zstd_compression_format.md
View File

@ -16,7 +16,7 @@ Distribution of this document is unlimited.
### Version
0.3.3 (16/08/19)
0.3.4 (16/08/19)
Introduction
@ -1107,18 +1107,18 @@ It follows the following build rule :
The table has a size of `Table_Size = 1 << Accuracy_Log`.
Each cell describes the symbol decoded,
and instructions to get the next state.
and instructions to get the next state (`Number_of_Bits` and `Baseline`).
Symbols are scanned in their natural order for "less than 1" probabilities.
Symbols with this probability are being attributed a single cell,
starting from the end of the table and retreating.
These symbols define a full state reset, reading `Accuracy_Log` bits.
All remaining symbols are allocated in their natural order.
Starting from symbol `0` and table position `0`,
Then, all remaining symbols, sorted in natural order, are allocated cells.
Starting from symbol `0` (if it exists), and table position `0`,
each symbol gets allocated as many cells as its probability.
Cell allocation is spreaded, not linear :
each successor position follow this rule :
each successor position follows this rule :
```
position += (tableSize>>1) + (tableSize>>3) + 3;
@ -1130,40 +1130,41 @@ A position is skipped if already occupied by a "less than 1" probability symbol.
each position in the table, switching to the next symbol when enough
states have been allocated to the current one.
The result is a list of state values.
Each state will decode the current symbol.
The process guarantees that the table is entirely filled.
Each cell corresponds to a state value, which contains the symbol being decoded.
To get the `Number_of_Bits` and `Baseline` required for next state,
it's first necessary to sort all states in their natural order.
The lower states will need 1 more bit than higher ones.
To add the `Number_of_Bits` and `Baseline` required to retrieve next state,
it's first necessary to sort all occurrences of each symbol in state order.
Lower states will need 1 more bit than higher ones.
The process is repeated for each symbol.
__Example__ :
Presuming a symbol has a probability of 5.
It receives 5 state values. States are sorted in natural order.
Presuming a symbol has a probability of 5,
it receives 5 cells, corresponding to 5 state values.
These state values are then sorted in natural order.
Next power of 2 is 8.
Space of probabilities is divided into 8 equal parts.
Presuming the `Accuracy_Log` is 7, it defines 128 states.
Next power of 2 after 5 is 8.
Space of probabilities must be divided into 8 equal parts.
Presuming the `Accuracy_Log` is 7, it defines a space of 128 states.
Divided by 8, each share is 16 large.
In order to reach 8, 8-5=3 lowest states will count "double",
doubling the number of shares (32 in width),
requiring one more bit in the process.
In order to reach 8 shares, 8-5=3 lowest states will count "double",
doubling their shares (32 in width), hence requiring one more bit.
Baseline is assigned starting from the higher states using fewer bits,
and proceeding naturally, then resuming at the first state,
each takes its allocated width from Baseline.
increasing at each state, then resuming at the first state,
each state takes its allocated width from Baseline.
| state value | 1 | 39 | 77 | 84 | 122 |
| state order | 0 | 1 | 2 | 3 | 4 |
| ---------------- | ----- | ----- | ------ | ---- | ----- |
| ---------------- | ----- | ----- | ------ | ---- | ------ |
| width | 32 | 32 | 32 | 16 | 16 |
| `Number_of_Bits` | 5 | 5 | 5 | 4 | 4 |
| range number | 2 | 4 | 6 | 0 | 1 |
| `Baseline` | 32 | 64 | 96 | 0 | 16 |
| range | 32-63 | 64-95 | 96-127 | 0-15 | 16-31 |
The next state is determined from current state
During decoding, the next state value is determined from current state value,
by reading the required `Number_of_Bits`, and adding the specified `Baseline`.
See [Appendix A] for the results of this process applied to the default distributions.
@ -1657,6 +1658,7 @@ or at least provide a meaningful error code explaining for which reason it canno
Version changes
---------------
- 0.3.4 : clarifications for FSE decoding table
- 0.3.3 : clarifications for field Block_Size
- 0.3.2 : remove additional block size restriction on compressed blocks
- 0.3.1 : minor clarification regarding offset history update rules

288
doc/zstd_manual.html
View File

@ -1,10 +1,10 @@
<html>
<head>
<meta http-equiv="Content-Type" content="text/html; charset=ISO-8859-1">
<title>zstd 1.4.3 Manual</title>
<title>zstd 1.4.4 Manual</title>
</head>
<body>
<h1>zstd 1.4.3 Manual</h1>
<h1>zstd 1.4.4 Manual</h1>
<hr>
<a name="Contents"></a><h2>Contents</h2>
<ol>
@ -27,10 +27,16 @@
<li><a href="#Chapter17">Advanced compression functions</a></li>
<li><a href="#Chapter18">Advanced decompression functions</a></li>
<li><a href="#Chapter19">Advanced streaming functions</a></li>
<li><a href="#Chapter20">Buffer-less and synchronous inner streaming functions</a></li>
<li><a href="#Chapter21">Buffer-less streaming compression (synchronous mode)</a></li>
<li><a href="#Chapter22">Buffer-less streaming decompression (synchronous mode)</a></li>
<li><a href="#Chapter23">Block level API</a></li>
<li><a href="#Chapter20">! ZSTD_initCStream_usingDict() :</a></li>
<li><a href="#Chapter21">! ZSTD_initCStream_advanced() :</a></li>
<li><a href="#Chapter22">! ZSTD_initCStream_usingCDict() :</a></li>
<li><a href="#Chapter23">! ZSTD_initCStream_usingCDict_advanced() :</a></li>
<li><a href="#Chapter24">This function is deprecated, and is equivalent to:</a></li>
<li><a href="#Chapter25">This function is deprecated, and is equivalent to:</a></li>
<li><a href="#Chapter26">Buffer-less and synchronous inner streaming functions</a></li>
<li><a href="#Chapter27">Buffer-less streaming compression (synchronous mode)</a></li>
<li><a href="#Chapter28">Buffer-less streaming decompression (synchronous mode)</a></li>
<li><a href="#Chapter29">Block level API</a></li>
</ol>
<hr>
<a name="Chapter1"></a><h2>Introduction</h2><pre>
@ -157,9 +163,13 @@ size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx);
void* dst, size_t dstCapacity,
const void* src, size_t srcSize,
int compressionLevel);
</b><p> Same as ZSTD_compress(), using an explicit ZSTD_CCtx
The function will compress at requested compression level,
ignoring any other parameter
</b><p> Same as ZSTD_compress(), using an explicit ZSTD_CCtx.
Important : in order to behave similarly to `ZSTD_compress()`,
this function compresses at requested compression level,
__ignoring any other parameter__ .
If any advanced parameter was set using the advanced API,
they will all be reset. Only `compressionLevel` remains.
</p></pre><BR>
<h3>Decompression context</h3><pre> When decompressing many times,
@ -199,18 +209,26 @@ size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx);
</b>/* compression parameters<b>
* Note: When compressing with a ZSTD_CDict these parameters are superseded
* by the parameters used to construct the ZSTD_CDict. See ZSTD_CCtx_refCDict()
* for more info (superseded-by-cdict). */
ZSTD_c_compressionLevel=100, </b>/* Update all compression parameters according to pre-defined cLevel table<b>
* by the parameters used to construct the ZSTD_CDict.
* See ZSTD_CCtx_refCDict() for more info (superseded-by-cdict). */
ZSTD_c_compressionLevel=100, </b>/* Set compression parameters according to pre-defined cLevel table.<b>
* Note that exact compression parameters are dynamically determined,
* depending on both compression level and srcSize (when known).
* Default level is ZSTD_CLEVEL_DEFAULT==3.
* Special: value 0 means default, which is controlled by ZSTD_CLEVEL_DEFAULT.
* Note 1 : it's possible to pass a negative compression level.
* Note 2 : setting a level sets all default values of other compression parameters */
* Note 2 : setting a level resets all other compression parameters to default */
</b>/* Advanced compression parameters :<b>
* It's possible to pin down compression parameters to some specific values.
* In which case, these values are no longer dynamically selected by the compressor */
ZSTD_c_windowLog=101, </b>/* Maximum allowed back-reference distance, expressed as power of 2.<b>
* This will set a memory budget for streaming decompression,
* with larger values requiring more memory
* and typically compressing more.
* Must be clamped between ZSTD_WINDOWLOG_MIN and ZSTD_WINDOWLOG_MAX.
* Special: value 0 means "use default windowLog".
* Note: Using a windowLog greater than ZSTD_WINDOWLOG_LIMIT_DEFAULT
* requires explicitly allowing such window size at decompression stage if using streaming. */
* requires explicitly allowing such size at streaming decompression stage. */
ZSTD_c_hashLog=102, </b>/* Size of the initial probe table, as a power of 2.<b>
* Resulting memory usage is (1 << (hashLog+2)).
* Must be clamped between ZSTD_HASHLOG_MIN and ZSTD_HASHLOG_MAX.
@ -221,13 +239,13 @@ size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx);
* Resulting memory usage is (1 << (chainLog+2)).
* Must be clamped between ZSTD_CHAINLOG_MIN and ZSTD_CHAINLOG_MAX.
* Larger tables result in better and slower compression.
* This parameter is useless when using "fast" strategy.
* This parameter is useless for "fast" strategy.
* It's still useful when using "dfast" strategy,
* in which case it defines a secondary probe table.
* Special: value 0 means "use default chainLog". */
ZSTD_c_searchLog=104, </b>/* Number of search attempts, as a power of 2.<b>
* More attempts result in better and slower compression.
* This parameter is useless when using "fast" and "dFast" strategies.
* This parameter is useless for "fast" and "dFast" strategies.
* Special: value 0 means "use default searchLog". */
ZSTD_c_minMatch=105, </b>/* Minimum size of searched matches.<b>
* Note that Zstandard can still find matches of smaller size,
@ -282,7 +300,7 @@ size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx);
ZSTD_c_contentSizeFlag=200, </b>/* Content size will be written into frame header _whenever known_ (default:1)<b>
* Content size must be known at the beginning of compression.
* This is automatically the case when using ZSTD_compress2(),
* For streaming variants, content size must be provided with ZSTD_CCtx_setPledgedSrcSize() */
* For streaming scenarios, content size must be provided with ZSTD_CCtx_setPledgedSrcSize() */
ZSTD_c_checksumFlag=201, </b>/* A 32-bits checksum of content is written at end of frame (default:0) */<b>
ZSTD_c_dictIDFlag=202, </b>/* When applicable, dictionary's ID is written into frame header (default:1) */<b>
@ -301,7 +319,7 @@ size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx);
* Each compression job is completed in parallel, so this value can indirectly impact the nb of active threads.
* 0 means default, which is dynamically determined based on compression parameters.
* Job size must be a minimum of overlap size, or 1 MB, whichever is largest.
* The minimum size is automatically and transparently enforced */
* The minimum size is automatically and transparently enforced. */
ZSTD_c_overlapLog=402, </b>/* Control the overlap size, as a fraction of window size.<b>
* The overlap size is an amount of data reloaded from previous job at the beginning of a new job.
* It helps preserve compression ratio, while each job is compressed in parallel.
@ -324,6 +342,7 @@ size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx);
* ZSTD_c_forceAttachDict
* ZSTD_c_literalCompressionMode
* ZSTD_c_targetCBlockSize
* ZSTD_c_srcSizeHint
* Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them.
* note : never ever use experimentalParam? names directly;
* also, the enums values themselves are unstable and can still change.
@ -334,6 +353,7 @@ size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx);
ZSTD_c_experimentalParam4=1001,
ZSTD_c_experimentalParam5=1002,
ZSTD_c_experimentalParam6=1003,
ZSTD_c_experimentalParam7=1004
} ZSTD_cParameter;
</b></pre><BR>
<pre><b>typedef struct {
@ -794,7 +814,7 @@ size_t ZSTD_freeDStream(ZSTD_DStream* zds);
Note 3 : Referencing a prefix involves building tables, which are dependent on compression parameters.
It's a CPU consuming operation, with non-negligible impact on latency.
If there is a need to use the same prefix multiple times, consider loadDictionary instead.
Note 4 : By default, the prefix is interpreted as raw content (ZSTD_dm_rawContent).
Note 4 : By default, the prefix is interpreted as raw content (ZSTD_dct_rawContent).
Use experimental ZSTD_CCtx_refPrefix_advanced() to alter dictionary interpretation.
</p></pre><BR>
@ -838,7 +858,7 @@ size_t ZSTD_freeDStream(ZSTD_DStream* zds);
Note 2 : Prefix buffer is referenced. It **must** outlive decompression.
Prefix buffer must remain unmodified up to the end of frame,
reached when ZSTD_decompressStream() returns 0.
Note 3 : By default, the prefix is treated as raw content (ZSTD_dm_rawContent).
Note 3 : By default, the prefix is treated as raw content (ZSTD_dct_rawContent).
Use ZSTD_CCtx_refPrefix_advanced() to alter dictMode (Experimental section)
Note 4 : Referencing a raw content prefix has almost no cpu nor memory cost.
A full dictionary is more costly, as it requires building tables.
@ -864,6 +884,24 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
<BR></pre>
<pre><b>typedef struct {
unsigned int matchPos; </b>/* Match pos in dst */<b>
</b>/* If seqDef.offset > 3, then this is seqDef.offset - 3<b>
* If seqDef.offset < 3, then this is the corresponding repeat offset
* But if seqDef.offset < 3 and litLength == 0, this is the
* repeat offset before the corresponding repeat offset
* And if seqDef.offset == 3 and litLength == 0, this is the
* most recent repeat offset - 1
*/
unsigned int offset;
unsigned int litLength; </b>/* Literal length */<b>
unsigned int matchLength; </b>/* Match length */<b>
</b>/* 0 when seq not rep and seqDef.offset otherwise<b>
* when litLength == 0 this will be <= 4, otherwise <= 3 like normal
*/
unsigned int rep;
} ZSTD_Sequence;
</b></pre><BR>
<pre><b>typedef struct {
unsigned windowLog; </b>/**< largest match distance : larger == more compression, more memory needed during decompression */<b>
unsigned chainLog; </b>/**< fully searched segment : larger == more compression, slower, more memory (useless for fast) */<b>
@ -893,21 +931,12 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
</b></pre><BR>
<pre><b>typedef enum {
ZSTD_dlm_byCopy = 0, </b>/**< Copy dictionary content internally */<b>
ZSTD_dlm_byRef = 1, </b>/**< Reference dictionary content -- the dictionary buffer must outlive its users. */<b>
ZSTD_dlm_byRef = 1 </b>/**< Reference dictionary content -- the dictionary buffer must outlive its users. */<b>
} ZSTD_dictLoadMethod_e;
</b></pre><BR>
<pre><b>typedef enum {
</b>/* Opened question : should we have a format ZSTD_f_auto ?<b>
* Today, it would mean exactly the same as ZSTD_f_zstd1.
* But, in the future, should several formats become supported,
* on the compression side, it would mean "default format".
* On the decompression side, it would mean "automatic format detection",
* so that ZSTD_f_zstd1 would mean "accept *only* zstd frames".
* Since meaning is a little different, another option could be to define different enums for compression and decompression.
* This question could be kept for later, when there are actually multiple formats to support,
* but there is also the question of pinning enum values, and pinning value `0` is especially important */
ZSTD_f_zstd1 = 0, </b>/* zstd frame format, specified in zstd_compression_format.md (default) */<b>
ZSTD_f_zstd1_magicless = 1, </b>/* Variant of zstd frame format, without initial 4-bytes magic number.<b>
ZSTD_f_zstd1_magicless = 1 </b>/* Variant of zstd frame format, without initial 4-bytes magic number.<b>
* Useful to save 4 bytes per generated frame.
* Decoder cannot recognise automatically this format, requiring this instruction. */
} ZSTD_format_e;
@ -941,7 +970,7 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
*/
ZSTD_dictDefaultAttach = 0, </b>/* Use the default heuristic. */<b>
ZSTD_dictForceAttach = 1, </b>/* Never copy the dictionary. */<b>
ZSTD_dictForceCopy = 2, </b>/* Always copy the dictionary. */<b>
ZSTD_dictForceCopy = 2 </b>/* Always copy the dictionary. */<b>
} ZSTD_dictAttachPref_e;
</b></pre><BR>
<pre><b>typedef enum {
@ -950,7 +979,7 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
* levels will be compressed. */
ZSTD_lcm_huffman = 1, </b>/**< Always attempt Huffman compression. Uncompressed literals will still be<b>
* emitted if Huffman compression is not profitable. */
ZSTD_lcm_uncompressed = 2, </b>/**< Always emit uncompressed literals. */<b>
ZSTD_lcm_uncompressed = 2 </b>/**< Always emit uncompressed literals. */<b>
} ZSTD_literalCompressionMode_e;
</b></pre><BR>
<a name="Chapter15"></a><h2>Frame size functions</h2><pre></pre>
@ -999,20 +1028,38 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
or an error code (if srcSize is too small)
</p></pre><BR>
<pre><b>size_t ZSTD_getSequences(ZSTD_CCtx* zc, ZSTD_Sequence* outSeqs,
size_t outSeqsSize, const void* src, size_t srcSize);
</b><p> Extract sequences from the sequence store
zc can be used to insert custom compression params.
This function invokes ZSTD_compress2
@return : number of sequences extracted
</p></pre><BR>
<a name="Chapter16"></a><h2>Memory management</h2><pre></pre>
<pre><b>size_t ZSTD_estimateCCtxSize(int compressionLevel);
size_t ZSTD_estimateCCtxSize_usingCParams(ZSTD_compressionParameters cParams);
size_t ZSTD_estimateCCtxSize_usingCCtxParams(const ZSTD_CCtx_params* params);
size_t ZSTD_estimateDCtxSize(void);
</b><p> These functions make it possible to estimate memory usage
of a future {D,C}Ctx, before its creation.
ZSTD_estimateCCtxSize() will provide a budget large enough for any compression level up to selected one.
It will also consider src size to be arbitrarily "large", which is worst case.
If srcSize is known to always be small, ZSTD_estimateCCtxSize_usingCParams() can provide a tighter estimation.
ZSTD_estimateCCtxSize_usingCParams() can be used in tandem with ZSTD_getCParams() to create cParams from compressionLevel.
ZSTD_estimateCCtxSize_usingCCtxParams() can be used in tandem with ZSTD_CCtxParams_setParameter(). Only single-threaded compression is supported. This function will return an error code if ZSTD_c_nbWorkers is >= 1.
Note : CCtx size estimation is only correct for single-threaded compression.
</b><p> These functions make it possible to estimate memory usage of a future
{D,C}Ctx, before its creation.
ZSTD_estimateCCtxSize() will provide a budget large enough for any
compression level up to selected one. Unlike ZSTD_estimateCStreamSize*(),
this estimate does not include space for a window buffer, so this estimate
is guaranteed to be enough for single-shot compressions, but not streaming
compressions. It will however assume the input may be arbitrarily large,
which is the worst case. If srcSize is known to always be small,
ZSTD_estimateCCtxSize_usingCParams() can provide a tighter estimation.
ZSTD_estimateCCtxSize_usingCParams() can be used in tandem with
ZSTD_getCParams() to create cParams from compressionLevel.
ZSTD_estimateCCtxSize_usingCCtxParams() can be used in tandem with
ZSTD_CCtxParams_setParameter().
Note: only single-threaded compression is supported. This function will
return an error code if ZSTD_c_nbWorkers is >= 1.
</p></pre><BR>
<pre><b>size_t ZSTD_estimateCStreamSize(int compressionLevel);
@ -1086,6 +1133,7 @@ static ZSTD_customMem const ZSTD_defaultCMem = { NULL, NULL, NULL }; </b>/**< t
Dictionary content is just referenced, not duplicated.
As a consequence, `dictBuffer` **must** outlive CDict,
and its content must remain unmodified throughout the lifetime of CDict.
note: equivalent to ZSTD_createCDict_advanced(), with dictLoadMethod==ZSTD_dlm_byRef
</p></pre><BR>
<pre><b>ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, unsigned long long estimatedSrcSize, size_t dictSize);
@ -1116,7 +1164,9 @@ static ZSTD_customMem const ZSTD_defaultCMem = { NULL, NULL, NULL }; </b>/**< t
const void* src, size_t srcSize,
const void* dict,size_t dictSize,
ZSTD_parameters params);
</b><p> Same as ZSTD_compress_usingDict(), with fine-tune control over compression parameters (by structure)
</b><p> Note : this function is now DEPRECATED.
It can be replaced by ZSTD_compress2(), in combination with ZSTD_CCtx_setParameter() and other parameter setters.
This prototype will be marked as deprecated and generate compilation warning on reaching v1.5.x
</p></pre><BR>
<pre><b>size_t ZSTD_compress_usingCDict_advanced(ZSTD_CCtx* cctx,
@ -1124,7 +1174,9 @@ static ZSTD_customMem const ZSTD_defaultCMem = { NULL, NULL, NULL }; </b>/**< t
const void* src, size_t srcSize,
const ZSTD_CDict* cdict,
ZSTD_frameParameters fParams);
</b><p> Same as ZSTD_compress_usingCDict(), with fine-tune control over frame parameters
</b><p> Note : this function is now REDUNDANT.
It can be replaced by ZSTD_compress2(), in combination with ZSTD_CCtx_loadDictionary() and other parameter setters.
This prototype will be marked as deprecated and generate compilation warning in some future version
</p></pre><BR>
<pre><b>size_t ZSTD_CCtx_loadDictionary_byReference(ZSTD_CCtx* cctx, const void* dict, size_t dictSize);
@ -1301,53 +1353,67 @@ size_t ZSTD_freeCCtxParams(ZSTD_CCtx_params* params);
* pledgedSrcSize must be correct. If it is not known at init time, use
* ZSTD_CONTENTSIZE_UNKNOWN. Note that, for compatibility with older programs,
* "0" also disables frame content size field. It may be enabled in the future.
* Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x
*/
size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize);
</b>/**! ZSTD_initCStream_usingDict() :<b>
* This function is deprecated, and is equivalent to:
* ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
* ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel);
* ZSTD_CCtx_loadDictionary(zcs, dict, dictSize);
*
* Creates of an internal CDict (incompatible with static CCtx), except if
* dict == NULL or dictSize < 8, in which case no dict is used.
* Note: dict is loaded with ZSTD_dm_auto (treated as a full zstd dictionary if
* it begins with ZSTD_MAGIC_DICTIONARY, else as raw content) and ZSTD_dlm_byCopy.
*/
size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel);
</b>/**! ZSTD_initCStream_advanced() :<b>
* This function is deprecated, and is approximately equivalent to:
* ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
* ZSTD_CCtx_setZstdParams(zcs, params); // Set the zstd params and leave the rest as-is
* ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize);
* ZSTD_CCtx_loadDictionary(zcs, dict, dictSize);
*
* pledgedSrcSize must be correct. If srcSize is not known at init time, use
* value ZSTD_CONTENTSIZE_UNKNOWN. dict is loaded with ZSTD_dm_auto and ZSTD_dlm_byCopy.
*/
size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize,
ZSTD_parameters params, unsigned long long pledgedSrcSize);
</b>/**! ZSTD_initCStream_usingCDict() :<b>
* This function is deprecated, and equivalent to:
* ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
* ZSTD_CCtx_refCDict(zcs, cdict);
*
* note : cdict will just be referenced, and must outlive compression session
*/
size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict);
</b>/**! ZSTD_initCStream_usingCDict_advanced() :<b>
* This function is deprecated, and is approximately equivalent to:
* ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
* ZSTD_CCtx_setZstdFrameParams(zcs, fParams); // Set the zstd frame params and leave the rest as-is
* ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize);
* ZSTD_CCtx_refCDict(zcs, cdict);
*
* same as ZSTD_initCStream_usingCDict(), with control over frame parameters.
* pledgedSrcSize must be correct. If srcSize is not known at init time, use
* value ZSTD_CONTENTSIZE_UNKNOWN.
*/
size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, const ZSTD_CDict* cdict, ZSTD_frameParameters fParams, unsigned long long pledgedSrcSize);
size_t
ZSTD_initCStream_srcSize(ZSTD_CStream* zcs,
int compressionLevel,
unsigned long long pledgedSrcSize);
</pre></b><BR>
<a name="Chapter20"></a><h2>! ZSTD_initCStream_usingDict() :</h2><pre> This function is deprecated, and is equivalent to:
ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel);
ZSTD_CCtx_loadDictionary(zcs, dict, dictSize);
Creates of an internal CDict (incompatible with static CCtx), except if
dict == NULL or dictSize < 8, in which case no dict is used.
Note: dict is loaded with ZSTD_dct_auto (treated as a full zstd dictionary if
it begins with ZSTD_MAGIC_DICTIONARY, else as raw content) and ZSTD_dlm_byCopy.
Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x
<BR></pre>
<a name="Chapter21"></a><h2>! ZSTD_initCStream_advanced() :</h2><pre> This function is deprecated, and is approximately equivalent to:
ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
// Pseudocode: Set each zstd parameter and leave the rest as-is.
for ((param, value) : params) {
ZSTD_CCtx_setParameter(zcs, param, value);
}
ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize);
ZSTD_CCtx_loadDictionary(zcs, dict, dictSize);
dict is loaded with ZSTD_dct_auto and ZSTD_dlm_byCopy.
pledgedSrcSize must be correct.
If srcSize is not known at init time, use value ZSTD_CONTENTSIZE_UNKNOWN.
Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x
<BR></pre>
<a name="Chapter22"></a><h2>! ZSTD_initCStream_usingCDict() :</h2><pre> This function is deprecated, and equivalent to:
ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
ZSTD_CCtx_refCDict(zcs, cdict);
note : cdict will just be referenced, and must outlive compression session
Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x
<BR></pre>
<a name="Chapter23"></a><h2>! ZSTD_initCStream_usingCDict_advanced() :</h2><pre> This function is DEPRECATED, and is approximately equivalent to:
ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
// Pseudocode: Set each zstd frame parameter and leave the rest as-is.
for ((fParam, value) : fParams) {
ZSTD_CCtx_setParameter(zcs, fParam, value);
}
ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize);
ZSTD_CCtx_refCDict(zcs, cdict);
same as ZSTD_initCStream_usingCDict(), with control over frame parameters.
pledgedSrcSize must be correct. If srcSize is not known at init time, use
value ZSTD_CONTENTSIZE_UNKNOWN.
Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x
<BR></pre>
<pre><b>size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize);
</b><p> This function is deprecated, and is equivalent to:
ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
@ -1361,6 +1427,7 @@ size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, const ZSTD_CDict*
For the time being, pledgedSrcSize==0 is interpreted as "srcSize unknown" for compatibility with older programs,
but it will change to mean "empty" in future version, so use macro ZSTD_CONTENTSIZE_UNKNOWN instead.
@return : 0, or an error code (which can be tested using ZSTD_isError())
Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x
</p></pre><BR>
@ -1395,34 +1462,35 @@ size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, const ZSTD_CDict*
* ZSTD_DCtx_loadDictionary(zds, dict, dictSize);
*
* note: no dictionary will be used if dict == NULL or dictSize < 8
* Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x
*/
size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize);
</b>/**<b>
* This function is deprecated, and is equivalent to:
*
* ZSTD_DCtx_reset(zds, ZSTD_reset_session_only);
* ZSTD_DCtx_refDDict(zds, ddict);
*
* note : ddict is referenced, it must outlive decompression session
*/
size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict);
</b>/**<b>
* This function is deprecated, and is equivalent to:
*
* ZSTD_DCtx_reset(zds, ZSTD_reset_session_only);
*
* re-use decompression parameters from previous init; saves dictionary loading
*/
size_t ZSTD_resetDStream(ZSTD_DStream* zds);
</pre></b><BR>
<a name="Chapter20"></a><h2>Buffer-less and synchronous inner streaming functions</h2><pre>
<a name="Chapter24"></a><h2>This function is deprecated, and is equivalent to:</h2><pre>
ZSTD_DCtx_reset(zds, ZSTD_reset_session_only);
ZSTD_DCtx_refDDict(zds, ddict);
note : ddict is referenced, it must outlive decompression session
Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x
<BR></pre>
<a name="Chapter25"></a><h2>This function is deprecated, and is equivalent to:</h2><pre>
ZSTD_DCtx_reset(zds, ZSTD_reset_session_only);
re-use decompression parameters from previous init; saves dictionary loading
Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x
<BR></pre>
<a name="Chapter26"></a><h2>Buffer-less and synchronous inner streaming functions</h2><pre>
This is an advanced API, giving full control over buffer management, for users which need direct control over memory.
But it's also a complex one, with several restrictions, documented below.
Prefer normal streaming API for an easier experience.
<BR></pre>
<a name="Chapter21"></a><h2>Buffer-less streaming compression (synchronous mode)</h2><pre>
<a name="Chapter27"></a><h2>Buffer-less streaming compression (synchronous mode)</h2><pre>
A ZSTD_CCtx object is required to track streaming operations.
Use ZSTD_createCCtx() / ZSTD_freeCCtx() to manage resource.
ZSTD_CCtx object can be re-used multiple times within successive compression operations.
@ -1458,7 +1526,7 @@ size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict);
size_t ZSTD_compressBegin_usingCDict_advanced(ZSTD_CCtx* const cctx, const ZSTD_CDict* const cdict, ZSTD_frameParameters const fParams, unsigned long long const pledgedSrcSize); </b>/* compression parameters are already set within cdict. pledgedSrcSize must be correct. If srcSize is not known, use macro ZSTD_CONTENTSIZE_UNKNOWN */<b>
size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize); </b>/**< note: if pledgedSrcSize is not known, use ZSTD_CONTENTSIZE_UNKNOWN */<b>
</pre></b><BR>
<a name="Chapter22"></a><h2>Buffer-less streaming decompression (synchronous mode)</h2><pre>
<a name="Chapter28"></a><h2>Buffer-less streaming decompression (synchronous mode)</h2><pre>
A ZSTD_DCtx object is required to track streaming operations.
Use ZSTD_createDCtx() / ZSTD_freeDCtx() to manage it.
A ZSTD_DCtx object can be re-used multiple times.
@ -1554,9 +1622,9 @@ size_t ZSTD_decodingBufferSize_min(unsigned long long windowSize, unsigned long
<pre><b>typedef enum { ZSTDnit_frameHeader, ZSTDnit_blockHeader, ZSTDnit_block, ZSTDnit_lastBlock, ZSTDnit_checksum, ZSTDnit_skippableFrame } ZSTD_nextInputType_e;
</b></pre><BR>
<a name="Chapter23"></a><h2>Block level API</h2><pre></pre>
<a name="Chapter29"></a><h2>Block level API</h2><pre></pre>
<pre><b></b><p> Frame metadata cost is typically ~18 bytes, which can be non-negligible for very small blocks (< 100 bytes).
<pre><b></b><p> Frame metadata cost is typically ~12 bytes, which can be non-negligible for very small blocks (< 100 bytes).
But users will have to take in charge needed metadata to regenerate data, such as compressed and content sizes.
A few rules to respect :

15
lib/README.md
View File

@ -27,10 +27,10 @@ Enabling multithreading requires 2 conditions :
Both conditions are automatically applied when invoking `make lib-mt` target.
When linking a POSIX program with a multithreaded version of `libzstd`,
note that it's necessary to request the `-pthread` flag during link stage.
note that it's necessary to invoke the `-pthread` flag during link stage.
Multithreading capabilities are exposed
via the [advanced API defined in `lib/zstd.h`](https://github.com/facebook/zstd/blob/v1.3.8/lib/zstd.h#L592).
via the [advanced API defined in `lib/zstd.h`](https://github.com/facebook/zstd/blob/v1.4.3/lib/zstd.h#L351).
#### API
@ -112,6 +112,17 @@ The file structure is designed to make this selection manually achievable for an
will expose the deprecated `ZSTDMT` API exposed by `zstdmt_compress.h` in
the shared library, which is now hidden by default.
- The build macro `DYNAMIC_BMI2` can be set to 1 or 0 in order to generate binaries
which can detect at runtime the presence of BMI2 instructions, and use them only if present.
These instructions contribute to better performance, notably on the decoder side.
By default, this feature is automatically enabled on detecting
the right instruction set (x64) and compiler (clang or gcc >= 5).
It's obviously disabled for different cpus,
or when BMI2 instruction set is _required_ by the compiler command line
(in this case, only the BMI2 code path is generated).
Setting this macro will either force to generate the BMI2 dispatcher (1)
or prevent it (0). It overrides automatic detection.
#### Windows : using MinGW+MSYS to create DLL

2
lib/common/bitstream.h
View File

@ -164,7 +164,7 @@ MEM_STATIC unsigned BIT_highbit32 (U32 val)
_BitScanReverse ( &r, val );
return (unsigned) r;
# elif defined(__GNUC__) && (__GNUC__ >= 3) /* Use GCC Intrinsic */
return 31 - __builtin_clz (val);
return __builtin_clz (val) ^ 31;
# elif defined(__ICCARM__) /* IAR Intrinsic */
return 31 - __CLZ(val);
# else /* Software version */

7
lib/common/compiler.h
View File

@ -61,6 +61,13 @@
# define HINT_INLINE static INLINE_KEYWORD FORCE_INLINE_ATTR
#endif
/* UNUSED_ATTR tells the compiler it is okay if the function is unused. */
#if defined(__GNUC__)
# define UNUSED_ATTR __attribute__((unused))
#else
# define UNUSED_ATTR
#endif
/* force no inlining */
#ifdef _MSC_VER
# define FORCE_NOINLINE static __declspec(noinline)

48
lib/common/mem.h
View File

@ -74,10 +74,50 @@ void __msan_poison(const volatile void *a, size_t size);
intptr_t __msan_test_shadow(const volatile void *x, size_t size);
#endif
#if defined (MEMORY_SANITIZER)
# define MEM_SKIP_MSAN __attribute__((no_sanitize("memory")))
#else
# define MEM_SKIP_MSAN
/* detects whether we are being compiled under asan */
#if defined (__has_feature)
# if __has_feature(address_sanitizer)
# define ADDRESS_SANITIZER 1
# endif
#elif defined(__SANITIZE_ADDRESS__)
# define ADDRESS_SANITIZER 1
#endif
#if defined (ADDRESS_SANITIZER)
/* Not all platforms that support asan provide sanitizers/asan_interface.h.
* We therefore declare the functions we need ourselves, rather than trying to
* include the header file... */
/**
* Marks a memory region (<c>[addr, addr+size)</c>) as unaddressable.
*
* This memory must be previously allocated by your program. Instrumented
* code is forbidden from accessing addresses in this region until it is
* unpoisoned. This function is not guaranteed to poison the entire region -
* it could poison only a subregion of <c>[addr, addr+size)</c> due to ASan
* alignment restrictions.
*
* \note This function is not thread-safe because no two threads can poison or
* unpoison memory in the same memory region simultaneously.
*
* \param addr Start of memory region.
* \param size Size of memory region. */
void __asan_poison_memory_region(void const volatile *addr, size_t size);
/**
* Marks a memory region (<c>[addr, addr+size)</c>) as addressable.
*
* This memory must be previously allocated by your program. Accessing
* addresses in this region is allowed until this region is poisoned again.
* This function could unpoison a super-region of <c>[addr, addr+size)</c> due
* to ASan alignment restrictions.
*
* \note This function is not thread-safe because no two threads can
* poison or unpoison memory in the same memory region simultaneously.
*
* \param addr Start of memory region.
* \param size Size of memory region. */
void __asan_unpoison_memory_region(void const volatile *addr, size_t size);
#endif

10
lib/common/pool.c
View File

@ -127,9 +127,13 @@ POOL_ctx* POOL_create_advanced(size_t numThreads, size_t queueSize,
ctx->queueTail = 0;
ctx->numThreadsBusy = 0;
ctx->queueEmpty = 1;
(void)ZSTD_pthread_mutex_init(&ctx->queueMutex, NULL);
(void)ZSTD_pthread_cond_init(&ctx->queuePushCond, NULL);
(void)ZSTD_pthread_cond_init(&ctx->queuePopCond, NULL);
{
int error = 0;
error |= ZSTD_pthread_mutex_init(&ctx->queueMutex, NULL);
error |= ZSTD_pthread_cond_init(&ctx->queuePushCond, NULL);
error |= ZSTD_pthread_cond_init(&ctx->queuePopCond, NULL);
if (error) { POOL_free(ctx); return NULL; }
}
ctx->shutdown = 0;
/* Allocate space for the thread handles */
ctx->threads = (ZSTD_pthread_t*)ZSTD_malloc(numThreads * sizeof(ZSTD_pthread_t), customMem);

47
lib/common/threading.c
View File

@ -14,6 +14,8 @@
* This file will hold wrapper for systems, which do not support pthreads
*/
#include "threading.h"
/* create fake symbol to avoid empty translation unit warning */
int g_ZSTD_threading_useless_symbol;
@ -28,7 +30,6 @@ int g_ZSTD_threading_useless_symbol;
/* === Dependencies === */
#include <process.h>
#include <errno.h>
#include "threading.h"
/* === Implementation === */
@ -73,3 +74,47 @@ int ZSTD_pthread_join(ZSTD_pthread_t thread, void **value_ptr)
}
#endif /* ZSTD_MULTITHREAD */
#if defined(ZSTD_MULTITHREAD) && DEBUGLEVEL >= 1 && !defined(_WIN32)
#include <stdlib.h>
int ZSTD_pthread_mutex_init(ZSTD_pthread_mutex_t* mutex, pthread_mutexattr_t const* attr)
{
*mutex = (pthread_mutex_t*)malloc(sizeof(pthread_mutex_t));
if (!*mutex)
return 1;
return pthread_mutex_init(*mutex, attr);
}
int ZSTD_pthread_mutex_destroy(ZSTD_pthread_mutex_t* mutex)
{
if (!*mutex)
return 0;
{
int const ret = pthread_mutex_destroy(*mutex);
free(*mutex);
return ret;
}
}
int ZSTD_pthread_cond_init(ZSTD_pthread_cond_t* cond, pthread_condattr_t const* attr)
{
*cond = (pthread_cond_t*)malloc(sizeof(pthread_cond_t));
if (!*cond)
return 1;
return pthread_cond_init(*cond, attr);
}
int ZSTD_pthread_cond_destroy(ZSTD_pthread_cond_t* cond)
{
if (!*cond)
return 0;
{
int const ret = pthread_cond_destroy(*cond);
free(*cond);
return ret;
}
}
#endif

31
lib/common/threading.h
View File

@ -13,6 +13,8 @@
#ifndef THREADING_H_938743
#define THREADING_H_938743
#include "debug.h"
#if defined (__cplusplus)
extern "C" {
#endif
@ -79,6 +81,8 @@ int ZSTD_pthread_join(ZSTD_pthread_t thread, void** value_ptr);
/* === POSIX Systems === */
# include <pthread.h>
#if DEBUGLEVEL < 1
#define ZSTD_pthread_mutex_t pthread_mutex_t
#define ZSTD_pthread_mutex_init(a, b) pthread_mutex_init((a), (b))
#define ZSTD_pthread_mutex_destroy(a) pthread_mutex_destroy((a))
@ -96,6 +100,33 @@ int ZSTD_pthread_join(ZSTD_pthread_t thread, void** value_ptr);
#define ZSTD_pthread_create(a, b, c, d) pthread_create((a), (b), (c), (d))
#define ZSTD_pthread_join(a, b) pthread_join((a),(b))
#else /* DEBUGLEVEL >= 1 */
/* Debug implementation of threading.
* In this implementation we use pointers for mutexes and condition variables.
* This way, if we forget to init/destroy them the program will crash or ASAN
* will report leaks.
*/
#define ZSTD_pthread_mutex_t pthread_mutex_t*
int ZSTD_pthread_mutex_init(ZSTD_pthread_mutex_t* mutex, pthread_mutexattr_t const* attr);
int ZSTD_pthread_mutex_destroy(ZSTD_pthread_mutex_t* mutex);
#define ZSTD_pthread_mutex_lock(a) pthread_mutex_lock(*(a))
#define ZSTD_pthread_mutex_unlock(a) pthread_mutex_unlock(*(a))
#define ZSTD_pthread_cond_t pthread_cond_t*
int ZSTD_pthread_cond_init(ZSTD_pthread_cond_t* cond, pthread_condattr_t const* attr);
int ZSTD_pthread_cond_destroy(ZSTD_pthread_cond_t* cond);
#define ZSTD_pthread_cond_wait(a, b) pthread_cond_wait(*(a), *(b))
#define ZSTD_pthread_cond_signal(a) pthread_cond_signal(*(a))
#define ZSTD_pthread_cond_broadcast(a) pthread_cond_broadcast(*(a))
#define ZSTD_pthread_t pthread_t
#define ZSTD_pthread_create(a, b, c, d) pthread_create((a), (b), (c), (d))
#define ZSTD_pthread_join(a, b) pthread_join((a),(b))
#endif
#else /* ZSTD_MULTITHREAD not defined */
/* No multithreading support */

97
lib/common/zstd_internal.h
View File

@ -197,93 +197,54 @@ static void ZSTD_copy8(void* dst, const void* src) { memcpy(dst, src, 8); }
static void ZSTD_copy16(void* dst, const void* src) { memcpy(dst, src, 16); }
#define COPY16(d,s) { ZSTD_copy16(d,s); d+=16; s+=16; }
#define WILDCOPY_OVERLENGTH 8
#define VECLEN 16
#define WILDCOPY_OVERLENGTH 32
#define WILDCOPY_VECLEN 16
typedef enum {
ZSTD_no_overlap,
ZSTD_overlap_src_before_dst,
ZSTD_overlap_src_before_dst
/* ZSTD_overlap_dst_before_src, */
} ZSTD_overlap_e;
/*! ZSTD_wildcopy() :
* custom version of memcpy(), can overwrite up to WILDCOPY_OVERLENGTH bytes (if length==0) */
* Custom version of memcpy(), can over read/write up to WILDCOPY_OVERLENGTH bytes (if length==0)
* @param ovtype controls the overlap detection
* - ZSTD_no_overlap: The source and destination are guaranteed to be at least WILDCOPY_VECLEN bytes apart.
* - ZSTD_overlap_src_before_dst: The src and dst may overlap, but they MUST be at least 8 bytes apart.
* The src buffer must be before the dst buffer.
*/
MEM_STATIC FORCE_INLINE_ATTR DONT_VECTORIZE
void ZSTD_wildcopy(void* dst, const void* src, BYTE* oend_g, ptrdiff_t length, ZSTD_overlap_e ovtype)
void ZSTD_wildcopy(void* dst, const void* src, ptrdiff_t length, ZSTD_overlap_e const ovtype)
{
ptrdiff_t diff = (BYTE*)dst - (const BYTE*)src;
const BYTE* ip = (const BYTE*)src;
BYTE* op = (BYTE*)dst;
BYTE* const oend = op + length;
assert(diff >= 8 || (ovtype == ZSTD_no_overlap && diff < -8));
assert(diff >= 8 || (ovtype == ZSTD_no_overlap && diff <= -WILDCOPY_VECLEN));
if (length < VECLEN || (ovtype == ZSTD_overlap_src_before_dst && diff < VECLEN)) {
do
COPY8(op, ip)
while (op < oend);
}
else {
if (oend < oend_g-16) {
/* common case */
if (ovtype == ZSTD_overlap_src_before_dst && diff < WILDCOPY_VECLEN) {
/* Handle short offset copies. */
do {
COPY8(op, ip)
} while (op < oend);
} else {
assert(diff >= WILDCOPY_VECLEN || diff <= -WILDCOPY_VECLEN);
/* Separate out the first two COPY16() calls because the copy length is
* almost certain to be short, so the branches have different
* probabilities.
* On gcc-9 unrolling once is +1.6%, twice is +2%, thrice is +1.8%.
* On clang-8 unrolling once is +1.4%, twice is +3.3%, thrice is +3%.
*/
COPY16(op, ip);
COPY16(op, ip);
if (op >= oend) return;
do {
COPY16(op, ip);
COPY16(op, ip);
}
while (op < oend);
}
else {
do {
COPY8(op, ip);
}
while (op < oend);
}
}
}
/*! ZSTD_wildcopy_16min() :
* same semantics as ZSTD_wildcopy() except guaranteed to be able to copy 16 bytes at the start */
MEM_STATIC FORCE_INLINE_ATTR DONT_VECTORIZE
void ZSTD_wildcopy_16min(void* dst, const void* src, BYTE* oend_g, ptrdiff_t length, ZSTD_overlap_e ovtype)
{
ptrdiff_t diff = (BYTE*)dst - (const BYTE*)src;
const BYTE* ip = (const BYTE*)src;
BYTE* op = (BYTE*)dst;
BYTE* const oend = op + length;
assert(length >= 8);
assert(diff >= 8 || (ovtype == ZSTD_no_overlap && diff < -8));
if (ovtype == ZSTD_overlap_src_before_dst && diff < VECLEN) {
do {
COPY8(op, ip);
}
while (op < oend);
}
else {
if (oend < oend_g-16) {
/* common case */
do {
COPY16(op, ip);
}
while (op < oend);
}
else {
do {
COPY8(op, ip);
}
while (op < oend);
}
}
}
MEM_STATIC void ZSTD_wildcopy_e(void* dst, const void* src, void* dstEnd) /* should be faster for decoding, but strangely, not verified on all platform */
{
const BYTE* ip = (const BYTE*)src;
BYTE* op = (BYTE*)dst;
BYTE* const oend = (BYTE*)dstEnd;
do
COPY8(op, ip)
while (op < oend);
}
@ -339,7 +300,7 @@ MEM_STATIC U32 ZSTD_highbit32(U32 val) /* compress, dictBuilder, decodeCorpus
_BitScanReverse(&r, val);
return (unsigned)r;
# elif defined(__GNUC__) && (__GNUC__ >= 3) /* GCC Intrinsic */
return 31 - __builtin_clz(val);
return __builtin_clz (val) ^ 31;
# elif defined(__ICCARM__) /* IAR Intrinsic */
return 31 - __CLZ(val);
# else /* Software version */

226
lib/compress/zstd_compress.c
View File

@ -130,15 +130,11 @@ static void ZSTD_freeCCtxContent(ZSTD_CCtx* cctx)
{
assert(cctx != NULL);
assert(cctx->staticSize == 0);
/* Only free workspace if cctx not in workspace, otherwise the workspace
* will be freed when the cctx itself is freed. */
if ((void*)cctx->workspace.workspace != (void*)cctx) {
ZSTD_cwksp_free(&cctx->workspace, cctx->customMem);
}
ZSTD_clearAllDicts(cctx);
#ifdef ZSTD_MULTITHREAD
ZSTDMT_freeCCtx(cctx->mtctx); cctx->mtctx = NULL;
#endif
ZSTD_cwksp_free(&cctx->workspace, cctx->customMem);
}
size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx)
@ -146,8 +142,13 @@ size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx)
if (cctx==NULL) return 0; /* support free on NULL */
RETURN_ERROR_IF(cctx->staticSize, memory_allocation,
"not compatible with static CCtx");
{
int cctxInWorkspace = ZSTD_cwksp_owns_buffer(&cctx->workspace, cctx);
ZSTD_freeCCtxContent(cctx);
if (!cctxInWorkspace) {
ZSTD_free(cctx, cctx->customMem);
}
}
return 0;
}
@ -237,10 +238,10 @@ size_t ZSTD_CCtxParams_init_advanced(ZSTD_CCtx_params* cctxParams, ZSTD_paramete
RETURN_ERROR_IF(!cctxParams, GENERIC);
FORWARD_IF_ERROR( ZSTD_checkCParams(params.cParams) );
memset(cctxParams, 0, sizeof(*cctxParams));
assert(!ZSTD_checkCParams(params.cParams));
cctxParams->cParams = params.cParams;
cctxParams->fParams = params.fParams;
cctxParams->compressionLevel = ZSTD_CLEVEL_DEFAULT; /* should not matter, as all cParams are presumed properly defined */
assert(!ZSTD_checkCParams(params.cParams));
return 0;
}
@ -250,10 +251,10 @@ static ZSTD_CCtx_params ZSTD_assignParamsToCCtxParams(
const ZSTD_CCtx_params* cctxParams, ZSTD_parameters params)
{
ZSTD_CCtx_params ret = *cctxParams;
assert(!ZSTD_checkCParams(params.cParams));
ret.cParams = params.cParams;
ret.fParams = params.fParams;
ret.compressionLevel = ZSTD_CLEVEL_DEFAULT; /* should not matter, as all cParams are presumed properly defined */
assert(!ZSTD_checkCParams(params.cParams));
return ret;
}
@ -532,33 +533,33 @@ size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params* CCtxParams,
if (value) { /* 0 : does not change current level */
CCtxParams->compressionLevel = value;
}
if (CCtxParams->compressionLevel >= 0) return CCtxParams->compressionLevel;
if (CCtxParams->compressionLevel >= 0) return (size_t)CCtxParams->compressionLevel;
return 0; /* return type (size_t) cannot represent negative values */
}
case ZSTD_c_windowLog :
if (value!=0) /* 0 => use default */
BOUNDCHECK(ZSTD_c_windowLog, value);
CCtxParams->cParams.windowLog = value;
CCtxParams->cParams.windowLog = (U32)value;
return CCtxParams->cParams.windowLog;
case ZSTD_c_hashLog :
if (value!=0) /* 0 => use default */
BOUNDCHECK(ZSTD_c_hashLog, value);
CCtxParams->cParams.hashLog = value;
CCtxParams->cParams.hashLog = (U32)value;
return CCtxParams->cParams.hashLog;
case ZSTD_c_chainLog :
if (value!=0) /* 0 => use default */
BOUNDCHECK(ZSTD_c_chainLog, value);
CCtxParams->cParams.chainLog = value;
CCtxParams->cParams.chainLog = (U32)value;
return CCtxParams->cParams.chainLog;
case ZSTD_c_searchLog :
if (value!=0) /* 0 => use default */
BOUNDCHECK(ZSTD_c_searchLog, value);
CCtxParams->cParams.searchLog = value;
return value;
CCtxParams->cParams.searchLog = (U32)value;
return (size_t)value;
case ZSTD_c_minMatch :
if (value!=0) /* 0 => use default */
@ -1077,10 +1078,19 @@ ZSTD_sizeof_matchState(const ZSTD_compressionParameters* const cParams,
size_t const chainSize = (cParams->strategy == ZSTD_fast) ? 0 : ((size_t)1 << cParams->chainLog);
size_t const hSize = ((size_t)1) << cParams->hashLog;
U32 const hashLog3 = (forCCtx && cParams->minMatch==3) ? MIN(ZSTD_HASHLOG3_MAX, cParams->windowLog) : 0;
size_t const h3Size = ((size_t)1) << hashLog3;
size_t const tableSpace = (chainSize + hSize + h3Size) * sizeof(U32);
size_t const optPotentialSpace = ((MaxML+1) + (MaxLL+1) + (MaxOff+1) + (1<<Litbits)) * sizeof(U32)
+ (ZSTD_OPT_NUM+1) * (sizeof(ZSTD_match_t)+sizeof(ZSTD_optimal_t));
size_t const h3Size = hashLog3 ? ((size_t)1) << hashLog3 : 0;
/* We don't use ZSTD_cwksp_alloc_size() here because the tables aren't
* surrounded by redzones in ASAN. */
size_t const tableSpace = chainSize * sizeof(U32)
+ hSize * sizeof(U32)
+ h3Size * sizeof(U32);
size_t const optPotentialSpace =
ZSTD_cwksp_alloc_size((MaxML+1) * sizeof(U32))
+ ZSTD_cwksp_alloc_size((MaxLL+1) * sizeof(U32))
+ ZSTD_cwksp_alloc_size((MaxOff+1) * sizeof(U32))
+ ZSTD_cwksp_alloc_size((1<<Litbits) * sizeof(U32))
+ ZSTD_cwksp_alloc_size((ZSTD_OPT_NUM+1) * sizeof(ZSTD_match_t))
+ ZSTD_cwksp_alloc_size((ZSTD_OPT_NUM+1) * sizeof(ZSTD_optimal_t));
size_t const optSpace = (forCCtx && (cParams->strategy >= ZSTD_btopt))
? optPotentialSpace
: 0;
@ -1097,20 +1107,23 @@ size_t ZSTD_estimateCCtxSize_usingCCtxParams(const ZSTD_CCtx_params* params)
size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, (size_t)1 << cParams.windowLog);
U32 const divider = (cParams.minMatch==3) ? 3 : 4;
size_t const maxNbSeq = blockSize / divider;
size_t const tokenSpace = WILDCOPY_OVERLENGTH + blockSize + 11*maxNbSeq;
size_t const entropySpace = HUF_WORKSPACE_SIZE;
size_t const blockStateSpace = 2 * sizeof(ZSTD_compressedBlockState_t);
size_t const tokenSpace = ZSTD_cwksp_alloc_size(WILDCOPY_OVERLENGTH + blockSize)
+ ZSTD_cwksp_alloc_size(maxNbSeq * sizeof(seqDef))
+ 3 * ZSTD_cwksp_alloc_size(maxNbSeq * sizeof(BYTE));
size_t const entropySpace = ZSTD_cwksp_alloc_size(HUF_WORKSPACE_SIZE);
size_t const blockStateSpace = 2 * ZSTD_cwksp_alloc_size(sizeof(ZSTD_compressedBlockState_t));
size_t const matchStateSize = ZSTD_sizeof_matchState(&cParams, /* forCCtx */ 1);
size_t const ldmSpace = ZSTD_ldm_getTableSize(params->ldmParams);
size_t const ldmSeqSpace = ZSTD_ldm_getMaxNbSeq(params->ldmParams, blockSize) * sizeof(rawSeq);
size_t const ldmSeqSpace = ZSTD_cwksp_alloc_size(ZSTD_ldm_getMaxNbSeq(params->ldmParams, blockSize) * sizeof(rawSeq));
size_t const neededSpace = entropySpace + blockStateSpace + tokenSpace +
matchStateSize + ldmSpace + ldmSeqSpace;
size_t const cctxSpace = ZSTD_cwksp_alloc_size(sizeof(ZSTD_CCtx));
DEBUGLOG(5, "sizeof(ZSTD_CCtx) : %u", (U32)sizeof(ZSTD_CCtx));
DEBUGLOG(5, "sizeof(ZSTD_CCtx) : %u", (U32)cctxSpace);
DEBUGLOG(5, "estimate workspace : %u", (U32)neededSpace);
return sizeof(ZSTD_CCtx) + neededSpace;
return cctxSpace + neededSpace;
}
}
@ -1146,7 +1159,8 @@ size_t ZSTD_estimateCStreamSize_usingCCtxParams(const ZSTD_CCtx_params* params)
size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, (size_t)1 << cParams.windowLog);
size_t const inBuffSize = ((size_t)1 << cParams.windowLog) + blockSize;
size_t const outBuffSize = ZSTD_compressBound(blockSize) + 1;
size_t const streamingSize = inBuffSize + outBuffSize;
size_t const streamingSize = ZSTD_cwksp_alloc_size(inBuffSize)
+ ZSTD_cwksp_alloc_size(outBuffSize);
return CCtxSize + streamingSize;
}
@ -1393,7 +1407,9 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, windowSize);
U32 const divider = (params.cParams.minMatch==3) ? 3 : 4;
size_t const maxNbSeq = blockSize / divider;
size_t const tokenSpace = WILDCOPY_OVERLENGTH + blockSize + 11*maxNbSeq;
size_t const tokenSpace = ZSTD_cwksp_alloc_size(WILDCOPY_OVERLENGTH + blockSize)
+ ZSTD_cwksp_alloc_size(maxNbSeq * sizeof(seqDef))
+ 3 * ZSTD_cwksp_alloc_size(maxNbSeq * sizeof(BYTE));
size_t const buffOutSize = (zbuff==ZSTDb_buffered) ? ZSTD_compressBound(blockSize)+1 : 0;
size_t const buffInSize = (zbuff==ZSTDb_buffered) ? windowSize + blockSize : 0;
size_t const matchStateSize = ZSTD_sizeof_matchState(&params.cParams, /* forCCtx */ 1);
@ -1408,12 +1424,12 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
ZSTD_cwksp_bump_oversized_duration(ws, 0);
/* Check if workspace is large enough, alloc a new one if needed */
{ size_t const cctxSpace = zc->staticSize ? sizeof(ZSTD_CCtx) : 0;
size_t const entropySpace = HUF_WORKSPACE_SIZE;
size_t const blockStateSpace = 2 * sizeof(ZSTD_compressedBlockState_t);
size_t const bufferSpace = buffInSize + buffOutSize;
{ size_t const cctxSpace = zc->staticSize ? ZSTD_cwksp_alloc_size(sizeof(ZSTD_CCtx)) : 0;
size_t const entropySpace = ZSTD_cwksp_alloc_size(HUF_WORKSPACE_SIZE);
size_t const blockStateSpace = 2 * ZSTD_cwksp_alloc_size(sizeof(ZSTD_compressedBlockState_t));
size_t const bufferSpace = ZSTD_cwksp_alloc_size(buffInSize) + ZSTD_cwksp_alloc_size(buffOutSize);
size_t const ldmSpace = ZSTD_ldm_getTableSize(params.ldmParams);
size_t const ldmSeqSpace = maxNbLdmSeq * sizeof(rawSeq);
size_t const ldmSeqSpace = ZSTD_cwksp_alloc_size(maxNbLdmSeq * sizeof(rawSeq));
size_t const neededSpace =
cctxSpace +
@ -1649,12 +1665,13 @@ static size_t ZSTD_resetCCtx_byCopyingCDict(ZSTD_CCtx* cctx,
/* copy tables */
{ size_t const chainSize = (cdict_cParams->strategy == ZSTD_fast) ? 0 : ((size_t)1 << cdict_cParams->chainLog);
size_t const hSize = (size_t)1 << cdict_cParams->hashLog;
size_t const tableSpace = (chainSize + hSize) * sizeof(U32);
assert((U32*)cctx->blockState.matchState.chainTable == (U32*)cctx->blockState.matchState.hashTable + hSize); /* chainTable must follow hashTable */
assert((U32*)cctx->blockState.matchState.hashTable3 == (U32*)cctx->blockState.matchState.chainTable + chainSize);
assert((U32*)cdict->matchState.chainTable == (U32*)cdict->matchState.hashTable + hSize); /* chainTable must follow hashTable */
assert((U32*)cdict->matchState.hashTable3 == (U32*)cdict->matchState.chainTable + chainSize);
memcpy(cctx->blockState.matchState.hashTable, cdict->matchState.hashTable, tableSpace); /* presumes all tables follow each other */
memcpy(cctx->blockState.matchState.hashTable,
cdict->matchState.hashTable,
hSize * sizeof(U32));
memcpy(cctx->blockState.matchState.chainTable,
cdict->matchState.chainTable,
chainSize * sizeof(U32));
}
/* Zero the hashTable3, since the cdict never fills it */
@ -1741,10 +1758,16 @@ static size_t ZSTD_copyCCtx_internal(ZSTD_CCtx* dstCCtx,
size_t const hSize = (size_t)1 << srcCCtx->appliedParams.cParams.hashLog;
int const h3log = srcCCtx->blockState.matchState.hashLog3;
size_t const h3Size = h3log ? ((size_t)1 << h3log) : 0;
size_t const tableSpace = (chainSize + hSize + h3Size) * sizeof(U32);
assert((U32*)dstCCtx->blockState.matchState.chainTable == (U32*)dstCCtx->blockState.matchState.hashTable + hSize); /* chainTable must follow hashTable */
assert((U32*)dstCCtx->blockState.matchState.hashTable3 == (U32*)dstCCtx->blockState.matchState.chainTable + chainSize);
memcpy(dstCCtx->blockState.matchState.hashTable, srcCCtx->blockState.matchState.hashTable, tableSpace); /* presumes all tables follow each other */
memcpy(dstCCtx->blockState.matchState.hashTable,
srcCCtx->blockState.matchState.hashTable,
hSize * sizeof(U32));
memcpy(dstCCtx->blockState.matchState.chainTable,
srcCCtx->blockState.matchState.chainTable,
chainSize * sizeof(U32));
memcpy(dstCCtx->blockState.matchState.hashTable3,
srcCCtx->blockState.matchState.hashTable3,
h3Size * sizeof(U32));
}
ZSTD_cwksp_mark_tables_clean(&dstCCtx->workspace);
@ -2265,6 +2288,77 @@ static size_t ZSTD_buildSeqStore(ZSTD_CCtx* zc, const void* src, size_t srcSize)
return ZSTDbss_compress;
}
static void ZSTD_copyBlockSequences(ZSTD_CCtx* zc)
{
const seqStore_t* seqStore = ZSTD_getSeqStore(zc);
const seqDef* seqs = seqStore->sequencesStart;
size_t seqsSize = seqStore->sequences - seqs;
ZSTD_Sequence* outSeqs = &zc->seqCollector.seqStart[zc->seqCollector.seqIndex];
size_t i; size_t position; int repIdx;
assert(zc->seqCollector.seqIndex + 1 < zc->seqCollector.maxSequences);
for (i = 0, position = 0; i < seqsSize; ++i) {
outSeqs[i].offset = seqs[i].offset;
outSeqs[i].litLength = seqs[i].litLength;
outSeqs[i].matchLength = seqs[i].matchLength + MINMATCH;
if (i == seqStore->longLengthPos) {
if (seqStore->longLengthID == 1) {
outSeqs[i].litLength += 0x10000;
} else if (seqStore->longLengthID == 2) {
outSeqs[i].matchLength += 0x10000;
}
}
if (outSeqs[i].offset <= ZSTD_REP_NUM) {
outSeqs[i].rep = outSeqs[i].offset;
repIdx = (unsigned int)i - outSeqs[i].offset;
if (outSeqs[i].litLength == 0) {
if (outSeqs[i].offset < 3) {
--repIdx;
} else {
repIdx = (unsigned int)i - 1;
}
++outSeqs[i].rep;
}
assert(repIdx >= -3);
outSeqs[i].offset = repIdx >= 0 ? outSeqs[repIdx].offset : repStartValue[-repIdx - 1];
if (outSeqs[i].rep == 4) {
--outSeqs[i].offset;
}
} else {
outSeqs[i].offset -= ZSTD_REP_NUM;
}
position += outSeqs[i].litLength;
outSeqs[i].matchPos = (unsigned int)position;
position += outSeqs[i].matchLength;
}
zc->seqCollector.seqIndex += seqsSize;
}
size_t ZSTD_getSequences(ZSTD_CCtx* zc, ZSTD_Sequence* outSeqs,
size_t outSeqsSize, const void* src, size_t srcSize)
{
const size_t dstCapacity = ZSTD_compressBound(srcSize);
void* dst = ZSTD_malloc(dstCapacity, ZSTD_defaultCMem);
SeqCollector seqCollector;
RETURN_ERROR_IF(dst == NULL, memory_allocation);
seqCollector.collectSequences = 1;
seqCollector.seqStart = outSeqs;
seqCollector.seqIndex = 0;
seqCollector.maxSequences = outSeqsSize;
zc->seqCollector = seqCollector;
ZSTD_compress2(zc, dst, dstCapacity, src, srcSize);
ZSTD_free(dst, ZSTD_defaultCMem);
return zc->seqCollector.seqIndex;
}
/* Returns true if the given block is a RLE block */
static int ZSTD_isRLE(const BYTE *ip, size_t length) {
size_t i;
@ -2296,6 +2390,11 @@ static size_t ZSTD_compressBlock_internal(ZSTD_CCtx* zc,
if (bss == ZSTDbss_noCompress) { cSize = 0; goto out; }
}
if (zc->seqCollector.collectSequences) {
ZSTD_copyBlockSequences(zc);
return 0;
}
/* encode sequences and literals */
cSize = ZSTD_compressSequences(&zc->seqStore,
&zc->blockState.prevCBlock->entropy, &zc->blockState.nextCBlock->entropy,
@ -2360,7 +2459,6 @@ static void ZSTD_overflowCorrectIfNeeded(ZSTD_matchState_t* ms,
}
}
/*! ZSTD_compress_frameChunk() :
* Compress a chunk of data into one or multiple blocks.
* All blocks will be terminated, all input will be consumed.
@ -2405,7 +2503,6 @@ static size_t ZSTD_compress_frameChunk (ZSTD_CCtx* cctx,
op+ZSTD_blockHeaderSize, dstCapacity-ZSTD_blockHeaderSize,
ip, blockSize, 1 /* frame */);
FORWARD_IF_ERROR(cSize);
if (cSize == 0) { /* block is not compressible */
cSize = ZSTD_noCompressBlock(op, dstCapacity, ip, blockSize, lastBlock);
FORWARD_IF_ERROR(cSize);
@ -3053,8 +3150,11 @@ size_t ZSTD_estimateCDictSize_advanced(
ZSTD_dictLoadMethod_e dictLoadMethod)
{
DEBUGLOG(5, "sizeof(ZSTD_CDict) : %u", (unsigned)sizeof(ZSTD_CDict));
return sizeof(ZSTD_CDict) + HUF_WORKSPACE_SIZE + ZSTD_sizeof_matchState(&cParams, /* forCCtx */ 0)
+ (dictLoadMethod == ZSTD_dlm_byRef ? 0 : ZSTD_cwksp_align(dictSize, sizeof(void *)));
return ZSTD_cwksp_alloc_size(sizeof(ZSTD_CDict))
+ ZSTD_cwksp_alloc_size(HUF_WORKSPACE_SIZE)
+ ZSTD_sizeof_matchState(&cParams, /* forCCtx */ 0)
+ (dictLoadMethod == ZSTD_dlm_byRef ? 0
: ZSTD_cwksp_alloc_size(ZSTD_cwksp_align(dictSize, sizeof(void *))));
}
size_t ZSTD_estimateCDictSize(size_t dictSize, int compressionLevel)
@ -3134,11 +3234,11 @@ ZSTD_CDict* ZSTD_createCDict_advanced(const void* dictBuffer, size_t dictSize,
if (!customMem.customAlloc ^ !customMem.customFree) return NULL;
{ size_t const workspaceSize =
sizeof(ZSTD_CDict) +
HUF_WORKSPACE_SIZE +
ZSTD_cwksp_alloc_size(sizeof(ZSTD_CDict)) +
ZSTD_cwksp_alloc_size(HUF_WORKSPACE_SIZE) +
ZSTD_sizeof_matchState(&cParams, /* forCCtx */ 0) +
(dictLoadMethod == ZSTD_dlm_byRef ? 0
: ZSTD_cwksp_align(dictSize, sizeof(void*)));
: ZSTD_cwksp_alloc_size(ZSTD_cwksp_align(dictSize, sizeof(void*))));
void* const workspace = ZSTD_malloc(workspaceSize, customMem);
ZSTD_cwksp ws;
ZSTD_CDict* cdict;
@ -3186,12 +3286,11 @@ size_t ZSTD_freeCDict(ZSTD_CDict* cdict)
{
if (cdict==NULL) return 0; /* support free on NULL */
{ ZSTD_customMem const cMem = cdict->customMem;
/* Only free workspace if cdict not in workspace, otherwise the
* workspace will be freed when the cdict itself is freed. */
if ((void*)cdict->workspace.workspace != (void*)cdict) {
int cdictInWorkspace = ZSTD_cwksp_owns_buffer(&cdict->workspace, cdict);
ZSTD_cwksp_free(&cdict->workspace, cMem);
}
if (!cdictInWorkspace) {
ZSTD_free(cdict, cMem);
}
return 0;
}
}
@ -3217,8 +3316,11 @@ const ZSTD_CDict* ZSTD_initStaticCDict(
ZSTD_compressionParameters cParams)
{
size_t const matchStateSize = ZSTD_sizeof_matchState(&cParams, /* forCCtx */ 0);
size_t const neededSize = sizeof(ZSTD_CDict) + (dictLoadMethod == ZSTD_dlm_byRef ? 0 : ZSTD_cwksp_align(dictSize, sizeof(void*)))
+ HUF_WORKSPACE_SIZE + matchStateSize;
size_t const neededSize = ZSTD_cwksp_alloc_size(sizeof(ZSTD_CDict))
+ (dictLoadMethod == ZSTD_dlm_byRef ? 0
: ZSTD_cwksp_alloc_size(ZSTD_cwksp_align(dictSize, sizeof(void*))))
+ ZSTD_cwksp_alloc_size(HUF_WORKSPACE_SIZE)
+ matchStateSize;
ZSTD_CDict* cdict;
if ((size_t)workspace & 7) return NULL; /* 8-aligned */
@ -3444,7 +3546,7 @@ size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict)
/* ZSTD_initCStream_advanced() :
* pledgedSrcSize must be exact.
* if srcSize is not known at init time, use value ZSTD_CONTENTSIZE_UNKNOWN.
* dict is loaded with default parameters ZSTD_dm_auto and ZSTD_dlm_byCopy. */
* dict is loaded with default parameters ZSTD_dct_auto and ZSTD_dlm_byCopy. */
size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs,
const void* dict, size_t dictSize,
ZSTD_parameters params, unsigned long long pss)
@ -3708,7 +3810,7 @@ size_t ZSTD_compressStream2( ZSTD_CCtx* cctx,
if (cctx->mtctx == NULL) {
DEBUGLOG(4, "ZSTD_compressStream2: creating new mtctx for nbWorkers=%u",
params.nbWorkers);
cctx->mtctx = ZSTDMT_createCCtx_advanced(params.nbWorkers, cctx->customMem);
cctx->mtctx = ZSTDMT_createCCtx_advanced((U32)params.nbWorkers, cctx->customMem);
RETURN_ERROR_IF(cctx->mtctx == NULL, memory_allocation);
}
/* mt compression */
@ -3836,8 +3938,8 @@ static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEV
{ 19, 12, 13, 1, 6, 1, ZSTD_fast }, /* base for negative levels */
{ 19, 13, 14, 1, 7, 0, ZSTD_fast }, /* level 1 */
{ 20, 15, 16, 1, 6, 0, ZSTD_fast }, /* level 2 */
{ 21, 16, 17, 1, 5, 1, ZSTD_dfast }, /* level 3 */
{ 21, 18, 18, 1, 5, 1, ZSTD_dfast }, /* level 4 */
{ 21, 16, 17, 1, 5, 0, ZSTD_dfast }, /* level 3 */
{ 21, 18, 18, 1, 5, 0, ZSTD_dfast }, /* level 4 */
{ 21, 18, 19, 2, 5, 2, ZSTD_greedy }, /* level 5 */
{ 21, 19, 19, 3, 5, 4, ZSTD_greedy }, /* level 6 */
{ 21, 19, 19, 3, 5, 8, ZSTD_lazy }, /* level 7 */
@ -3861,8 +3963,8 @@ static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEV
/* W, C, H, S, L, T, strat */
{ 18, 12, 13, 1, 5, 1, ZSTD_fast }, /* base for negative levels */
{ 18, 13, 14, 1, 6, 0, ZSTD_fast }, /* level 1 */
{ 18, 14, 14, 1, 5, 1, ZSTD_dfast }, /* level 2 */
{ 18, 16, 16, 1, 4, 1, ZSTD_dfast }, /* level 3 */
{ 18, 14, 14, 1, 5, 0, ZSTD_dfast }, /* level 2 */
{ 18, 16, 16, 1, 4, 0, ZSTD_dfast }, /* level 3 */
{ 18, 16, 17, 2, 5, 2, ZSTD_greedy }, /* level 4.*/
{ 18, 18, 18, 3, 5, 2, ZSTD_greedy }, /* level 5.*/
{ 18, 18, 19, 3, 5, 4, ZSTD_lazy }, /* level 6.*/
@ -3888,8 +3990,8 @@ static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEV
{ 17, 12, 12, 1, 5, 1, ZSTD_fast }, /* base for negative levels */
{ 17, 12, 13, 1, 6, 0, ZSTD_fast }, /* level 1 */
{ 17, 13, 15, 1, 5, 0, ZSTD_fast }, /* level 2 */
{ 17, 15, 16, 2, 5, 1, ZSTD_dfast }, /* level 3 */
{ 17, 17, 17, 2, 4, 1, ZSTD_dfast }, /* level 4 */
{ 17, 15, 16, 2, 5, 0, ZSTD_dfast }, /* level 3 */
{ 17, 17, 17, 2, 4, 0, ZSTD_dfast }, /* level 4 */
{ 17, 16, 17, 3, 4, 2, ZSTD_greedy }, /* level 5 */
{ 17, 17, 17, 3, 4, 4, ZSTD_lazy }, /* level 6 */
{ 17, 17, 17, 3, 4, 8, ZSTD_lazy2 }, /* level 7 */
@ -3914,7 +4016,7 @@ static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEV
{ 14, 12, 13, 1, 5, 1, ZSTD_fast }, /* base for negative levels */
{ 14, 14, 15, 1, 5, 0, ZSTD_fast }, /* level 1 */
{ 14, 14, 15, 1, 4, 0, ZSTD_fast }, /* level 2 */
{ 14, 14, 15, 2, 4, 1, ZSTD_dfast }, /* level 3 */
{ 14, 14, 15, 2, 4, 0, ZSTD_dfast }, /* level 3 */
{ 14, 14, 14, 4, 4, 2, ZSTD_greedy }, /* level 4 */
{ 14, 14, 14, 3, 4, 4, ZSTD_lazy }, /* level 5.*/
{ 14, 14, 14, 4, 4, 8, ZSTD_lazy2 }, /* level 6 */

43
lib/compress/zstd_compress_internal.h
View File

@ -193,6 +193,13 @@ typedef struct {
size_t capacity; /* The capacity starting from `seq` pointer */
} rawSeqStore_t;
typedef struct {
int collectSequences;
ZSTD_Sequence* seqStart;
size_t seqIndex;
size_t maxSequences;
} SeqCollector;
struct ZSTD_CCtx_params_s {
ZSTD_format_e format;
ZSTD_compressionParameters cParams;
@ -240,6 +247,7 @@ struct ZSTD_CCtx_s {
XXH64_state_t xxhState;
ZSTD_customMem customMem;
size_t staticSize;
SeqCollector seqCollector;
int isFirstBlock;
seqStore_t seqStore; /* sequences storage ptrs */
@ -340,13 +348,32 @@ MEM_STATIC size_t ZSTD_minGain(size_t srcSize, ZSTD_strategy strat)
return (srcSize >> minlog) + 2;
}
/*! ZSTD_safecopyLiterals() :
* memcpy() function that won't read beyond more than WILDCOPY_OVERLENGTH bytes past ilimit_w.
* Only called when the sequence ends past ilimit_w, so it only needs to be optimized for single
* large copies.
*/
static void ZSTD_safecopyLiterals(BYTE* op, BYTE const* ip, BYTE const* const iend, BYTE const* ilimit_w) {
assert(iend > ilimit_w);
if (ip <= ilimit_w) {
ZSTD_wildcopy(op, ip, ilimit_w - ip, ZSTD_no_overlap);
op += ilimit_w - ip;
ip = ilimit_w;
}
while (ip < iend) *op++ = *ip++;
}
/*! ZSTD_storeSeq() :
* Store a sequence (litlen, litPtr, offCode and mlBase) into seqStore_t.
* `offCode` : distance to match + ZSTD_REP_MOVE (values <= ZSTD_REP_MOVE are repCodes).
* `mlBase` : matchLength - MINMATCH
* Allowed to overread literals up to litLimit.
*/
MEM_STATIC void ZSTD_storeSeq(seqStore_t* seqStorePtr, size_t litLength, const void* literals, U32 offCode, size_t mlBase)
HINT_INLINE UNUSED_ATTR
void ZSTD_storeSeq(seqStore_t* seqStorePtr, size_t litLength, const BYTE* literals, const BYTE* litLimit, U32 offCode, size_t mlBase)
{
BYTE const* const litLimit_w = litLimit - WILDCOPY_OVERLENGTH;
BYTE const* const litEnd = literals + litLength;
#if defined(DEBUGLEVEL) && (DEBUGLEVEL >= 6)
static const BYTE* g_start = NULL;
if (g_start==NULL) g_start = (const BYTE*)literals; /* note : index only works for compression within a single segment */
@ -359,7 +386,19 @@ MEM_STATIC void ZSTD_storeSeq(seqStore_t* seqStorePtr, size_t litLength, const v
/* copy Literals */
assert(seqStorePtr->maxNbLit <= 128 KB);
assert(seqStorePtr->lit + litLength <= seqStorePtr->litStart + seqStorePtr->maxNbLit);
ZSTD_wildcopy(seqStorePtr->lit, literals, seqStorePtr->lit + litLength + 8, (ptrdiff_t)litLength, ZSTD_no_overlap);
assert(literals + litLength <= litLimit);
if (litEnd <= litLimit_w) {
/* Common case we can use wildcopy.
* First copy 16 bytes, because literals are likely short.
*/
assert(WILDCOPY_OVERLENGTH >= 16);
ZSTD_copy16(seqStorePtr->lit, literals);
if (litLength > 16) {
ZSTD_wildcopy(seqStorePtr->lit+16, literals+16, (ptrdiff_t)litLength-16, ZSTD_no_overlap);
}
} else {
ZSTD_safecopyLiterals(seqStorePtr->lit, literals, litEnd, litLimit_w);
}
seqStorePtr->lit += litLength;
/* literal Length */

110
lib/compress/zstd_cwksp.h
View File

@ -34,6 +34,17 @@ extern "C" {
* In which case, resize it down to free some memory */
#define ZSTD_WORKSPACETOOLARGE_MAXDURATION 128
/* Since the workspace is effectively its own little malloc implementation /
* arena, when we run under ASAN, we should similarly insert redzones between
* each internal element of the workspace, so ASAN will catch overruns that
* reach outside an object but that stay inside the workspace.
*
* This defines the size of that redzone.
*/
#ifndef ZSTD_CWKSP_ASAN_REDZONE_SIZE
#define ZSTD_CWKSP_ASAN_REDZONE_SIZE 128
#endif
/*-*************************************
* Structures
***************************************/
@ -166,6 +177,24 @@ MEM_STATIC size_t ZSTD_cwksp_align(size_t size, size_t const align) {
return (size + mask) & ~mask;
}
/**
* Use this to determine how much space in the workspace we will consume to
* allocate this object. (Normally it should be exactly the size of the object,
* but under special conditions, like ASAN, where we pad each object, it might
* be larger.)
*
* Since tables aren't currently redzoned, you don't need to call through this
* to figure out how much space you need for the matchState tables. Everything
* else is though.
*/
MEM_STATIC size_t ZSTD_cwksp_alloc_size(size_t size) {
#if defined (ADDRESS_SANITIZER) && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE)
return size + 2 * ZSTD_CWKSP_ASAN_REDZONE_SIZE;
#else
return size;
#endif
}
MEM_STATIC void ZSTD_cwksp_internal_advance_phase(
ZSTD_cwksp* ws, ZSTD_cwksp_alloc_phase_e phase) {
assert(phase >= ws->phase);
@ -192,6 +221,13 @@ MEM_STATIC void ZSTD_cwksp_internal_advance_phase(
}
}
/**
* Returns whether this object/buffer/etc was allocated in this workspace.
*/
MEM_STATIC int ZSTD_cwksp_owns_buffer(const ZSTD_cwksp* ws, const void* ptr) {
return (ptr != NULL) && (ws->workspace <= ptr) && (ptr <= ws->workspaceEnd);
}
/**
* Internal function. Do not use directly.
*/
@ -201,8 +237,14 @@ MEM_STATIC void* ZSTD_cwksp_reserve_internal(
void* bottom = ws->tableEnd;
ZSTD_cwksp_internal_advance_phase(ws, phase);
alloc = (BYTE *)ws->allocStart - bytes;
DEBUGLOG(5, "cwksp: reserving %zd bytes, %zd bytes remaining",
bytes, ZSTD_cwksp_available_space(ws) - bytes);
#if defined (ADDRESS_SANITIZER) && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE)
/* over-reserve space */
alloc = (BYTE *)alloc - 2 * ZSTD_CWKSP_ASAN_REDZONE_SIZE;
#endif
DEBUGLOG(5, "cwksp: reserving %p %zd bytes, %zd bytes remaining",
alloc, bytes, ZSTD_cwksp_available_space(ws) - bytes);
ZSTD_cwksp_assert_internal_consistency(ws);
assert(alloc >= bottom);
if (alloc < bottom) {
@ -214,6 +256,14 @@ MEM_STATIC void* ZSTD_cwksp_reserve_internal(
ws->tableValidEnd = alloc;
}
ws->allocStart = alloc;
#if defined (ADDRESS_SANITIZER) && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE)
/* Move alloc so there's ZSTD_CWKSP_ASAN_REDZONE_SIZE unused space on
* either size. */
alloc = (BYTE *)alloc + ZSTD_CWKSP_ASAN_REDZONE_SIZE;
__asan_unpoison_memory_region(alloc, bytes);
#endif
return alloc;
}
@ -242,8 +292,9 @@ MEM_STATIC void* ZSTD_cwksp_reserve_table(ZSTD_cwksp* ws, size_t bytes) {
void* alloc = ws->tableEnd;
void* end = (BYTE *)alloc + bytes;
void* top = ws->allocStart;
DEBUGLOG(5, "cwksp: reserving table %zd bytes, %zd bytes remaining",
bytes, ZSTD_cwksp_available_space(ws) - bytes);
DEBUGLOG(5, "cwksp: reserving %p table %zd bytes, %zd bytes remaining",
alloc, bytes, ZSTD_cwksp_available_space(ws) - bytes);
assert((bytes & (sizeof(U32)-1)) == 0);
ZSTD_cwksp_internal_advance_phase(ws, phase);
ZSTD_cwksp_assert_internal_consistency(ws);
@ -254,6 +305,11 @@ MEM_STATIC void* ZSTD_cwksp_reserve_table(ZSTD_cwksp* ws, size_t bytes) {
return NULL;
}
ws->tableEnd = end;
#if defined (ADDRESS_SANITIZER) && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE)
__asan_unpoison_memory_region(alloc, bytes);
#endif
return alloc;
}
@ -262,12 +318,18 @@ MEM_STATIC void* ZSTD_cwksp_reserve_table(ZSTD_cwksp* ws, size_t bytes) {
*/
MEM_STATIC void* ZSTD_cwksp_reserve_object(ZSTD_cwksp* ws, size_t bytes) {
size_t roundedBytes = ZSTD_cwksp_align(bytes, sizeof(void*));
void* start = ws->objectEnd;
void* end = (BYTE*)start + roundedBytes;
void* alloc = ws->objectEnd;
void* end = (BYTE*)alloc + roundedBytes;
#if defined (ADDRESS_SANITIZER) && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE)
/* over-reserve space */
end = (BYTE *)end + 2 * ZSTD_CWKSP_ASAN_REDZONE_SIZE;
#endif
DEBUGLOG(5,
"cwksp: reserving object %zd bytes (rounded to %zd), %zd bytes remaining",
bytes, roundedBytes, ZSTD_cwksp_available_space(ws) - roundedBytes);
assert(((size_t)start & (sizeof(void*)-1)) == 0);
"cwksp: reserving %p object %zd bytes (rounded to %zd), %zd bytes remaining",
alloc, bytes, roundedBytes, ZSTD_cwksp_available_space(ws) - roundedBytes);
assert(((size_t)alloc & (sizeof(void*)-1)) == 0);
assert((bytes & (sizeof(void*)-1)) == 0);
ZSTD_cwksp_assert_internal_consistency(ws);
/* we must be in the first phase, no advance is possible */
@ -279,7 +341,15 @@ MEM_STATIC void* ZSTD_cwksp_reserve_object(ZSTD_cwksp* ws, size_t bytes) {
ws->objectEnd = end;
ws->tableEnd = end;
ws->tableValidEnd = end;
return start;
#if defined (ADDRESS_SANITIZER) && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE)
/* Move alloc so there's ZSTD_CWKSP_ASAN_REDZONE_SIZE unused space on
* either size. */
alloc = (BYTE *)alloc + ZSTD_CWKSP_ASAN_REDZONE_SIZE;
__asan_unpoison_memory_region(alloc, bytes);
#endif
return alloc;
}
MEM_STATIC void ZSTD_cwksp_mark_tables_dirty(ZSTD_cwksp* ws) {
@ -331,6 +401,14 @@ MEM_STATIC void ZSTD_cwksp_clean_tables(ZSTD_cwksp* ws) {
*/
MEM_STATIC void ZSTD_cwksp_clear_tables(ZSTD_cwksp* ws) {
DEBUGLOG(4, "cwksp: clearing tables!");
#if defined (ADDRESS_SANITIZER) && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE)
{
size_t size = (BYTE*)ws->tableValidEnd - (BYTE*)ws->objectEnd;
__asan_poison_memory_region(ws->objectEnd, size);
}
#endif
ws->tableEnd = ws->objectEnd;
ZSTD_cwksp_assert_internal_consistency(ws);
}
@ -353,6 +431,13 @@ MEM_STATIC void ZSTD_cwksp_clear(ZSTD_cwksp* ws) {
}
#endif
#if defined (ADDRESS_SANITIZER) && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE)
{
size_t size = (BYTE*)ws->workspaceEnd - (BYTE*)ws->objectEnd;
__asan_poison_memory_region(ws->objectEnd, size);
}
#endif
ws->tableEnd = ws->objectEnd;
ws->allocStart = ws->workspaceEnd;
ws->allocFailed = 0;
@ -389,9 +474,10 @@ MEM_STATIC size_t ZSTD_cwksp_create(ZSTD_cwksp* ws, size_t size, ZSTD_customMem
}
MEM_STATIC void ZSTD_cwksp_free(ZSTD_cwksp* ws, ZSTD_customMem customMem) {
void *ptr = ws->workspace;
DEBUGLOG(4, "cwksp: freeing workspace");
ZSTD_free(ws->workspace, customMem);
memset(ws, 0, sizeof(ZSTD_cwksp));
ZSTD_free(ptr, customMem);
}
/**
@ -404,7 +490,7 @@ MEM_STATIC void ZSTD_cwksp_move(ZSTD_cwksp* dst, ZSTD_cwksp* src) {
}
MEM_STATIC size_t ZSTD_cwksp_sizeof(const ZSTD_cwksp* ws) {
return (BYTE*)ws->workspaceEnd - (BYTE*)ws->workspace;
return (size_t)((BYTE*)ws->workspaceEnd - (BYTE*)ws->workspace);
}
MEM_STATIC int ZSTD_cwksp_reserve_failed(const ZSTD_cwksp* ws) {

18
lib/compress/zstd_double_fast.c
View File

@ -148,7 +148,7 @@ size_t ZSTD_compressBlock_doubleFast_generic(
const BYTE* repMatchEnd = repIndex < prefixLowestIndex ? dictEnd : iend;
mLength = ZSTD_count_2segments(ip+1+4, repMatch+4, iend, repMatchEnd, prefixLowest) + 4;
ip++;
ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, 0, mLength-MINMATCH);
ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, 0, mLength-MINMATCH);
goto _match_stored;
}
@ -157,7 +157,7 @@ size_t ZSTD_compressBlock_doubleFast_generic(
&& ((offset_1 > 0) & (MEM_read32(ip+1-offset_1) == MEM_read32(ip+1)))) {
mLength = ZSTD_count(ip+1+4, ip+1+4-offset_1, iend) + 4;
ip++;
ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, 0, mLength-MINMATCH);
ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, 0, mLength-MINMATCH);
goto _match_stored;
}
@ -247,7 +247,7 @@ _match_found:
offset_2 = offset_1;
offset_1 = offset;
ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH);
ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, offset + ZSTD_REP_MOVE, mLength-MINMATCH);
_match_stored:
/* match found */
@ -278,7 +278,7 @@ _match_stored:
const BYTE* const repEnd2 = repIndex2 < prefixLowestIndex ? dictEnd : iend;
size_t const repLength2 = ZSTD_count_2segments(ip+4, repMatch2+4, iend, repEnd2, prefixLowest) + 4;
U32 tmpOffset = offset_2; offset_2 = offset_1; offset_1 = tmpOffset; /* swap offset_2 <=> offset_1 */
ZSTD_storeSeq(seqStore, 0, anchor, 0, repLength2-MINMATCH);
ZSTD_storeSeq(seqStore, 0, anchor, iend, 0, repLength2-MINMATCH);
hashSmall[ZSTD_hashPtr(ip, hBitsS, mls)] = current2;
hashLong[ZSTD_hashPtr(ip, hBitsL, 8)] = current2;
ip += repLength2;
@ -297,7 +297,7 @@ _match_stored:
U32 const tmpOff = offset_2; offset_2 = offset_1; offset_1 = tmpOff; /* swap offset_2 <=> offset_1 */
hashSmall[ZSTD_hashPtr(ip, hBitsS, mls)] = (U32)(ip-base);
hashLong[ZSTD_hashPtr(ip, hBitsL, 8)] = (U32)(ip-base);
ZSTD_storeSeq(seqStore, 0, anchor, 0, rLength-MINMATCH);
ZSTD_storeSeq(seqStore, 0, anchor, iend, 0, rLength-MINMATCH);
ip += rLength;
anchor = ip;
continue; /* faster when present ... (?) */
@ -411,7 +411,7 @@ static size_t ZSTD_compressBlock_doubleFast_extDict_generic(
const BYTE* repMatchEnd = repIndex < prefixStartIndex ? dictEnd : iend;
mLength = ZSTD_count_2segments(ip+1+4, repMatch+4, iend, repMatchEnd, prefixStart) + 4;
ip++;
ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, 0, mLength-MINMATCH);
ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, 0, mLength-MINMATCH);
} else {
if ((matchLongIndex > dictStartIndex) && (MEM_read64(matchLong) == MEM_read64(ip))) {
const BYTE* const matchEnd = matchLongIndex < prefixStartIndex ? dictEnd : iend;
@ -422,7 +422,7 @@ static size_t ZSTD_compressBlock_doubleFast_extDict_generic(
while (((ip>anchor) & (matchLong>lowMatchPtr)) && (ip[-1] == matchLong[-1])) { ip--; matchLong--; mLength++; } /* catch up */
offset_2 = offset_1;
offset_1 = offset;
ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH);
ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, offset + ZSTD_REP_MOVE, mLength-MINMATCH);
} else if ((matchIndex > dictStartIndex) && (MEM_read32(match) == MEM_read32(ip))) {
size_t const h3 = ZSTD_hashPtr(ip+1, hBitsL, 8);
@ -447,7 +447,7 @@ static size_t ZSTD_compressBlock_doubleFast_extDict_generic(
}
offset_2 = offset_1;
offset_1 = offset;
ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH);
ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, offset + ZSTD_REP_MOVE, mLength-MINMATCH);
} else {
ip += ((ip-anchor) >> kSearchStrength) + 1;
@ -479,7 +479,7 @@ static size_t ZSTD_compressBlock_doubleFast_extDict_generic(
const BYTE* const repEnd2 = repIndex2 < prefixStartIndex ? dictEnd : iend;
size_t const repLength2 = ZSTD_count_2segments(ip+4, repMatch2+4, iend, repEnd2, prefixStart) + 4;
U32 const tmpOffset = offset_2; offset_2 = offset_1; offset_1 = tmpOffset; /* swap offset_2 <=> offset_1 */
ZSTD_storeSeq(seqStore, 0, anchor, 0, repLength2-MINMATCH);
ZSTD_storeSeq(seqStore, 0, anchor, iend, 0, repLength2-MINMATCH);
hashSmall[ZSTD_hashPtr(ip, hBitsS, mls)] = current2;
hashLong[ZSTD_hashPtr(ip, hBitsL, 8)] = current2;
ip += repLength2;

18
lib/compress/zstd_fast.c
View File

@ -136,7 +136,7 @@ _offset: /* Requires: ip0, match0 */
_match: /* Requires: ip0, match0, offcode */
/* Count the forward length */
mLength += ZSTD_count(ip0+mLength+4, match0+mLength+4, iend) + 4;
ZSTD_storeSeq(seqStore, (size_t)(ip0-anchor), anchor, offcode, mLength-MINMATCH);
ZSTD_storeSeq(seqStore, (size_t)(ip0-anchor), anchor, iend, offcode, mLength-MINMATCH);
/* match found */
ip0 += mLength;
anchor = ip0;
@ -156,7 +156,7 @@ _match: /* Requires: ip0, match0, offcode */
hashTable[ZSTD_hashPtr(ip0, hlog, mls)] = (U32)(ip0-base);
ip0 += rLength;
ip1 = ip0 + 1;
ZSTD_storeSeq(seqStore, 0 /*litLen*/, anchor, 0 /*offCode*/, rLength-MINMATCH);
ZSTD_storeSeq(seqStore, 0 /*litLen*/, anchor, iend, 0 /*offCode*/, rLength-MINMATCH);
anchor = ip0;
continue; /* faster when present (confirmed on gcc-8) ... (?) */
}
@ -261,7 +261,7 @@ size_t ZSTD_compressBlock_fast_dictMatchState_generic(
const BYTE* const repMatchEnd = repIndex < prefixStartIndex ? dictEnd : iend;
mLength = ZSTD_count_2segments(ip+1+4, repMatch+4, iend, repMatchEnd, prefixStart) + 4;
ip++;
ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, 0, mLength-MINMATCH);
ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, 0, mLength-MINMATCH);
} else if ( (matchIndex <= prefixStartIndex) ) {
size_t const dictHash = ZSTD_hashPtr(ip, dictHLog, mls);
U32 const dictMatchIndex = dictHashTable[dictHash];
@ -281,7 +281,7 @@ size_t ZSTD_compressBlock_fast_dictMatchState_generic(
} /* catch up */
offset_2 = offset_1;
offset_1 = offset;
ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH);
ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, offset + ZSTD_REP_MOVE, mLength-MINMATCH);
}
} else if (MEM_read32(match) != MEM_read32(ip)) {
/* it's not a match, and we're not going to check the dictionary */
@ -296,7 +296,7 @@ size_t ZSTD_compressBlock_fast_dictMatchState_generic(
&& (ip[-1] == match[-1])) { ip--; match--; mLength++; } /* catch up */
offset_2 = offset_1;
offset_1 = offset;
ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH);
ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, offset + ZSTD_REP_MOVE, mLength-MINMATCH);
}
/* match found */
@ -321,7 +321,7 @@ size_t ZSTD_compressBlock_fast_dictMatchState_generic(
const BYTE* const repEnd2 = repIndex2 < prefixStartIndex ? dictEnd : iend;
size_t const repLength2 = ZSTD_count_2segments(ip+4, repMatch2+4, iend, repEnd2, prefixStart) + 4;
U32 tmpOffset = offset_2; offset_2 = offset_1; offset_1 = tmpOffset; /* swap offset_2 <=> offset_1 */
ZSTD_storeSeq(seqStore, 0, anchor, 0, repLength2-MINMATCH);
ZSTD_storeSeq(seqStore, 0, anchor, iend, 0, repLength2-MINMATCH);
hashTable[ZSTD_hashPtr(ip, hlog, mls)] = current2;
ip += repLength2;
anchor = ip;
@ -411,7 +411,7 @@ static size_t ZSTD_compressBlock_fast_extDict_generic(
const BYTE* const repMatchEnd = repIndex < prefixStartIndex ? dictEnd : iend;
size_t const rLength = ZSTD_count_2segments(ip+1 +4, repMatch +4, iend, repMatchEnd, prefixStart) + 4;
ip++;
ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, 0, rLength-MINMATCH);
ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, 0, rLength-MINMATCH);
ip += rLength;
anchor = ip;
} else {
@ -427,7 +427,7 @@ static size_t ZSTD_compressBlock_fast_extDict_generic(
size_t mLength = ZSTD_count_2segments(ip+4, match+4, iend, matchEnd, prefixStart) + 4;
while (((ip>anchor) & (match>lowMatchPtr)) && (ip[-1] == match[-1])) { ip--; match--; mLength++; } /* catch up */
offset_2 = offset_1; offset_1 = offset; /* update offset history */
ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH);
ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, offset + ZSTD_REP_MOVE, mLength-MINMATCH);
ip += mLength;
anchor = ip;
} }
@ -446,7 +446,7 @@ static size_t ZSTD_compressBlock_fast_extDict_generic(
const BYTE* const repEnd2 = repIndex2 < prefixStartIndex ? dictEnd : iend;
size_t const repLength2 = ZSTD_count_2segments(ip+4, repMatch2+4, iend, repEnd2, prefixStart) + 4;
{ U32 const tmpOffset = offset_2; offset_2 = offset_1; offset_1 = tmpOffset; } /* swap offset_2 <=> offset_1 */
ZSTD_storeSeq(seqStore, 0 /*litlen*/, anchor, 0 /*offcode*/, repLength2-MINMATCH);
ZSTD_storeSeq(seqStore, 0 /*litlen*/, anchor, iend, 0 /*offcode*/, repLength2-MINMATCH);
hashTable[ZSTD_hashPtr(ip, hlog, mls)] = current2;
ip += repLength2;
anchor = ip;

10
lib/compress/zstd_lazy.c
View File

@ -810,7 +810,7 @@ ZSTD_compressBlock_lazy_generic(
/* store sequence */
_storeSequence:
{ size_t const litLength = start - anchor;
ZSTD_storeSeq(seqStore, litLength, anchor, (U32)offset, matchLength-MINMATCH);
ZSTD_storeSeq(seqStore, litLength, anchor, iend, (U32)offset, matchLength-MINMATCH);
anchor = ip = start + matchLength;
}
@ -828,7 +828,7 @@ _storeSequence:
const BYTE* const repEnd2 = repIndex < prefixLowestIndex ? dictEnd : iend;
matchLength = ZSTD_count_2segments(ip+4, repMatch+4, iend, repEnd2, prefixLowest) + 4;
offset = offset_2; offset_2 = offset_1; offset_1 = (U32)offset; /* swap offset_2 <=> offset_1 */
ZSTD_storeSeq(seqStore, 0, anchor, 0, matchLength-MINMATCH);
ZSTD_storeSeq(seqStore, 0, anchor, iend, 0, matchLength-MINMATCH);
ip += matchLength;
anchor = ip;
continue;
@ -843,7 +843,7 @@ _storeSequence:
/* store sequence */
matchLength = ZSTD_count(ip+4, ip+4-offset_2, iend) + 4;
offset = offset_2; offset_2 = offset_1; offset_1 = (U32)offset; /* swap repcodes */
ZSTD_storeSeq(seqStore, 0, anchor, 0, matchLength-MINMATCH);
ZSTD_storeSeq(seqStore, 0, anchor, iend, 0, matchLength-MINMATCH);
ip += matchLength;
anchor = ip;
continue; /* faster when present ... (?) */
@ -1051,7 +1051,7 @@ size_t ZSTD_compressBlock_lazy_extDict_generic(
/* store sequence */
_storeSequence:
{ size_t const litLength = start - anchor;
ZSTD_storeSeq(seqStore, litLength, anchor, (U32)offset, matchLength-MINMATCH);
ZSTD_storeSeq(seqStore, litLength, anchor, iend, (U32)offset, matchLength-MINMATCH);
anchor = ip = start + matchLength;
}
@ -1066,7 +1066,7 @@ _storeSequence:
const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend;
matchLength = ZSTD_count_2segments(ip+4, repMatch+4, iend, repEnd, prefixStart) + 4;
offset = offset_2; offset_2 = offset_1; offset_1 = (U32)offset; /* swap offset history */
ZSTD_storeSeq(seqStore, 0, anchor, 0, matchLength-MINMATCH);
ZSTD_storeSeq(seqStore, 0, anchor, iend, 0, matchLength-MINMATCH);
ip += matchLength;
anchor = ip;
continue; /* faster when present ... (?) */

8
lib/compress/zstd_ldm.c
View File

@ -49,9 +49,9 @@ size_t ZSTD_ldm_getTableSize(ldmParams_t params)
{
size_t const ldmHSize = ((size_t)1) << params.hashLog;
size_t const ldmBucketSizeLog = MIN(params.bucketSizeLog, params.hashLog);
size_t const ldmBucketSize =
((size_t)1) << (params.hashLog - ldmBucketSizeLog);
size_t const totalSize = ldmBucketSize + ldmHSize * sizeof(ldmEntry_t);
size_t const ldmBucketSize = ((size_t)1) << (params.hashLog - ldmBucketSizeLog);
size_t const totalSize = ZSTD_cwksp_alloc_size(ldmBucketSize)
+ ZSTD_cwksp_alloc_size(ldmHSize * sizeof(ldmEntry_t));
return params.enableLdm ? totalSize : 0;
}
@ -583,7 +583,7 @@ size_t ZSTD_ldm_blockCompress(rawSeqStore_t* rawSeqStore,
rep[i] = rep[i-1];
rep[0] = sequence.offset;
/* Store the sequence */
ZSTD_storeSeq(seqStore, newLitLength, ip - newLitLength,
ZSTD_storeSeq(seqStore, newLitLength, ip - newLitLength, iend,
sequence.offset + ZSTD_REP_MOVE,
sequence.matchLength - MINMATCH);
ip += sequence.matchLength;

2
lib/compress/zstd_opt.c
View File

@ -1098,7 +1098,7 @@ _shortestPath: /* cur, last_pos, best_mlen, best_off have to be set */
assert(anchor + llen <= iend);
ZSTD_updateStats(optStatePtr, llen, anchor, offCode, mlen);
ZSTD_storeSeq(seqStore, llen, anchor, offCode, mlen-MINMATCH);
ZSTD_storeSeq(seqStore, llen, anchor, iend, offCode, mlen-MINMATCH);
anchor += advance;
ip = anchor;
} }

12
lib/compress/zstdmt_compress.c
View File

@ -927,12 +927,18 @@ static void ZSTDMT_releaseAllJobResources(ZSTDMT_CCtx* mtctx)
unsigned jobID;
DEBUGLOG(3, "ZSTDMT_releaseAllJobResources");
for (jobID=0; jobID <= mtctx->jobIDMask; jobID++) {
/* Copy the mutex/cond out */
ZSTD_pthread_mutex_t const mutex = mtctx->jobs[jobID].job_mutex;
ZSTD_pthread_cond_t const cond = mtctx->jobs[jobID].job_cond;
DEBUGLOG(4, "job%02u: release dst address %08X", jobID, (U32)(size_t)mtctx->jobs[jobID].dstBuff.start);
ZSTDMT_releaseBuffer(mtctx->bufPool, mtctx->jobs[jobID].dstBuff);
mtctx->jobs[jobID].dstBuff = g_nullBuffer;
mtctx->jobs[jobID].cSize = 0;
/* Clear the job description, but keep the mutex/cond */
memset(&mtctx->jobs[jobID], 0, sizeof(mtctx->jobs[jobID]));
mtctx->jobs[jobID].job_mutex = mutex;
mtctx->jobs[jobID].job_cond = cond;
}
memset(mtctx->jobs, 0, (mtctx->jobIDMask+1)*sizeof(ZSTDMT_jobDescription));
mtctx->inBuff.buffer = g_nullBuffer;
mtctx->inBuff.filled = 0;
mtctx->allJobsCompleted = 1;

26
lib/decompress/zstd_decompress.c
View File

@ -88,10 +88,7 @@ size_t ZSTD_estimateDCtxSize(void) { return sizeof(ZSTD_DCtx); }
static size_t ZSTD_startingInputLength(ZSTD_format_e format)
{
size_t const startingInputLength = (format==ZSTD_f_zstd1_magicless) ?
ZSTD_FRAMEHEADERSIZE_PREFIX - ZSTD_FRAMEIDSIZE :
ZSTD_FRAMEHEADERSIZE_PREFIX;
ZSTD_STATIC_ASSERT(ZSTD_FRAMEHEADERSIZE_PREFIX >= ZSTD_FRAMEIDSIZE);
size_t const startingInputLength = ZSTD_FRAMEHEADERSIZE_PREFIX(format);
/* only supports formats ZSTD_f_zstd1 and ZSTD_f_zstd1_magicless */
assert( (format == ZSTD_f_zstd1) || (format == ZSTD_f_zstd1_magicless) );
return startingInputLength;
@ -376,7 +373,7 @@ unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize)
{
unsigned long long totalDstSize = 0;
while (srcSize >= ZSTD_FRAMEHEADERSIZE_PREFIX) {
while (srcSize >= ZSTD_startingInputLength(ZSTD_f_zstd1)) {
U32 const magicNumber = MEM_readLE32(src);
if ((magicNumber & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) {
@ -629,11 +626,12 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx,
/* check */
RETURN_ERROR_IF(
remainingSrcSize < ZSTD_FRAMEHEADERSIZE_MIN+ZSTD_blockHeaderSize,
remainingSrcSize < ZSTD_FRAMEHEADERSIZE_MIN(dctx->format)+ZSTD_blockHeaderSize,
srcSize_wrong);
/* Frame Header */
{ size_t const frameHeaderSize = ZSTD_frameHeaderSize(ip, ZSTD_FRAMEHEADERSIZE_PREFIX);
{ size_t const frameHeaderSize = ZSTD_frameHeaderSize_internal(
ip, ZSTD_FRAMEHEADERSIZE_PREFIX(dctx->format), dctx->format);
if (ZSTD_isError(frameHeaderSize)) return frameHeaderSize;
RETURN_ERROR_IF(remainingSrcSize < frameHeaderSize+ZSTD_blockHeaderSize,
srcSize_wrong);
@ -714,7 +712,7 @@ static size_t ZSTD_decompressMultiFrame(ZSTD_DCtx* dctx,
dictSize = ZSTD_DDict_dictSize(ddict);
}
while (srcSize >= ZSTD_FRAMEHEADERSIZE_PREFIX) {
while (srcSize >= ZSTD_startingInputLength(dctx->format)) {
#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1)
if (ZSTD_isLegacy(src, srcSize)) {
@ -1300,14 +1298,14 @@ size_t ZSTD_DCtx_refPrefix(ZSTD_DCtx* dctx, const void* prefix, size_t prefixSiz
/* ZSTD_initDStream_usingDict() :
* return : expected size, aka ZSTD_FRAMEHEADERSIZE_PREFIX.
* return : expected size, aka ZSTD_startingInputLength().
* this function cannot fail */
size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize)
{
DEBUGLOG(4, "ZSTD_initDStream_usingDict");
FORWARD_IF_ERROR( ZSTD_DCtx_reset(zds, ZSTD_reset_session_only) );
FORWARD_IF_ERROR( ZSTD_DCtx_loadDictionary(zds, dict, dictSize) );
return ZSTD_FRAMEHEADERSIZE_PREFIX;
return ZSTD_startingInputLength(zds->format);
}
/* note : this variant can't fail */
@ -1324,16 +1322,16 @@ size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* dctx, const ZSTD_DDict* ddict)
{
FORWARD_IF_ERROR( ZSTD_DCtx_reset(dctx, ZSTD_reset_session_only) );
FORWARD_IF_ERROR( ZSTD_DCtx_refDDict(dctx, ddict) );
return ZSTD_FRAMEHEADERSIZE_PREFIX;
return ZSTD_startingInputLength(dctx->format);
}
/* ZSTD_resetDStream() :
* return : expected size, aka ZSTD_FRAMEHEADERSIZE_PREFIX.
* return : expected size, aka ZSTD_startingInputLength().
* this function cannot fail */
size_t ZSTD_resetDStream(ZSTD_DStream* dctx)
{
FORWARD_IF_ERROR(ZSTD_DCtx_reset(dctx, ZSTD_reset_session_only));
return ZSTD_FRAMEHEADERSIZE_PREFIX;
return ZSTD_startingInputLength(dctx->format);
}
@ -1564,7 +1562,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
zds->lhSize += remainingInput;
}
input->pos = input->size;
return (MAX(ZSTD_FRAMEHEADERSIZE_MIN, hSize) - zds->lhSize) + ZSTD_blockHeaderSize; /* remaining header bytes + next block header */
return (MAX((size_t)ZSTD_FRAMEHEADERSIZE_MIN(zds->format), hSize) - zds->lhSize) + ZSTD_blockHeaderSize; /* remaining header bytes + next block header */
}
assert(ip != NULL);
memcpy(zds->headerBuffer + zds->lhSize, ip, toLoad); zds->lhSize = hSize; ip += toLoad;

280
lib/decompress/zstd_decompress_block.c
View File

@ -573,38 +573,118 @@ typedef struct {
size_t pos;
} seqState_t;
/*! ZSTD_overlapCopy8() :
* Copies 8 bytes from ip to op and updates op and ip where ip <= op.
* If the offset is < 8 then the offset is spread to at least 8 bytes.
*
* Precondition: *ip <= *op
* Postcondition: *op - *op >= 8
*/
static void ZSTD_overlapCopy8(BYTE** op, BYTE const** ip, size_t offset) {
assert(*ip <= *op);
if (offset < 8) {
/* close range match, overlap */
static const U32 dec32table[] = { 0, 1, 2, 1, 4, 4, 4, 4 }; /* added */
static const int dec64table[] = { 8, 8, 8, 7, 8, 9,10,11 }; /* subtracted */
int const sub2 = dec64table[offset];
(*op)[0] = (*ip)[0];
(*op)[1] = (*ip)[1];
(*op)[2] = (*ip)[2];
(*op)[3] = (*ip)[3];
*ip += dec32table[offset];
ZSTD_copy4(*op+4, *ip);
*ip -= sub2;
} else {
ZSTD_copy8(*op, *ip);
}
*ip += 8;
*op += 8;
assert(*op - *ip >= 8);
}
/* ZSTD_execSequenceLast7():
* exceptional case : decompress a match starting within last 7 bytes of output buffer.
* requires more careful checks, to ensure there is no overflow.
* performance does not matter though.
* note : this case is supposed to be never generated "naturally" by reference encoder,
* since in most cases it needs at least 8 bytes to look for a match.
* but it's allowed by the specification. */
/*! ZSTD_safecopy() :
* Specialized version of memcpy() that is allowed to READ up to WILDCOPY_OVERLENGTH past the input buffer
* and write up to 16 bytes past oend_w (op >= oend_w is allowed).
* This function is only called in the uncommon case where the sequence is near the end of the block. It
* should be fast for a single long sequence, but can be slow for several short sequences.
*
* @param ovtype controls the overlap detection
* - ZSTD_no_overlap: The source and destination are guaranteed to be at least WILDCOPY_VECLEN bytes apart.
* - ZSTD_overlap_src_before_dst: The src and dst may overlap and may be any distance apart.
* The src buffer must be before the dst buffer.
*/
static void ZSTD_safecopy(BYTE* op, BYTE* const oend_w, BYTE const* ip, ptrdiff_t length, ZSTD_overlap_e ovtype) {
ptrdiff_t const diff = op - ip;
BYTE* const oend = op + length;
assert((ovtype == ZSTD_no_overlap && (diff <= -8 || diff >= 8)) ||
(ovtype == ZSTD_overlap_src_before_dst && diff >= 0));
if (length < 8) {
/* Handle short lengths. */
while (op < oend) *op++ = *ip++;
return;
}
if (ovtype == ZSTD_overlap_src_before_dst) {
/* Copy 8 bytes and ensure the offset >= 8 when there can be overlap. */
assert(length >= 8);
ZSTD_overlapCopy8(&op, &ip, diff);
assert(op - ip >= 8);
assert(op <= oend);
}
if (oend <= oend_w) {
/* No risk of overwrite. */
ZSTD_wildcopy(op, ip, length, ovtype);
return;
}
if (op <= oend_w) {
/* Wildcopy until we get close to the end. */
assert(oend > oend_w);
ZSTD_wildcopy(op, ip, oend_w - op, ovtype);
ip += oend_w - op;
op = oend_w;
}
/* Handle the leftovers. */
while (op < oend) *op++ = *ip++;
}
/* ZSTD_execSequenceEnd():
* This version handles cases that are near the end of the output buffer. It requires
* more careful checks to make sure there is no overflow. By separating out these hard
* and unlikely cases, we can speed up the common cases.
*
* NOTE: This function needs to be fast for a single long sequence, but doesn't need
* to be optimized for many small sequences, since those fall into ZSTD_execSequence().
*/
FORCE_NOINLINE
size_t ZSTD_execSequenceLast7(BYTE* op,
size_t ZSTD_execSequenceEnd(BYTE* op,
BYTE* const oend, seq_t sequence,
const BYTE** litPtr, const BYTE* const litLimit,
const BYTE* const base, const BYTE* const vBase, const BYTE* const dictEnd)
const BYTE* const prefixStart, const BYTE* const virtualStart, const BYTE* const dictEnd)
{
BYTE* const oLitEnd = op + sequence.litLength;
size_t const sequenceLength = sequence.litLength + sequence.matchLength;
BYTE* const oMatchEnd = op + sequenceLength; /* risk : address space overflow (32-bits) */
const BYTE* const iLitEnd = *litPtr + sequence.litLength;
const BYTE* match = oLitEnd - sequence.offset;
BYTE* const oend_w = oend - WILDCOPY_OVERLENGTH;
/* check */
RETURN_ERROR_IF(oMatchEnd>oend, dstSize_tooSmall, "last match must fit within dstBuffer");
/* bounds checks */
assert(oLitEnd < oMatchEnd);
RETURN_ERROR_IF(oMatchEnd > oend, dstSize_tooSmall, "last match must fit within dstBuffer");
RETURN_ERROR_IF(iLitEnd > litLimit, corruption_detected, "try to read beyond literal buffer");
/* copy literals */
while (op < oLitEnd) *op++ = *(*litPtr)++;
ZSTD_safecopy(op, oend_w, *litPtr, sequence.litLength, ZSTD_no_overlap);
op = oLitEnd;
*litPtr = iLitEnd;
/* copy Match */
if (sequence.offset > (size_t)(oLitEnd - base)) {
if (sequence.offset > (size_t)(oLitEnd - prefixStart)) {
/* offset beyond prefix */
RETURN_ERROR_IF(sequence.offset > (size_t)(oLitEnd - vBase),corruption_detected);
match = dictEnd - (base-match);
RETURN_ERROR_IF(sequence.offset > (size_t)(oLitEnd - virtualStart), corruption_detected);
match = dictEnd - (prefixStart-match);
if (match + sequence.matchLength <= dictEnd) {
memmove(oLitEnd, match, sequence.matchLength);
return sequenceLength;
@ -614,13 +694,12 @@ size_t ZSTD_execSequenceLast7(BYTE* op,
memmove(oLitEnd, match, length1);
op = oLitEnd + length1;
sequence.matchLength -= length1;
match = base;
match = prefixStart;
} }
while (op < oMatchEnd) *op++ = *match++;
ZSTD_safecopy(op, oend_w, match, sequence.matchLength, ZSTD_overlap_src_before_dst);
return sequenceLength;
}
HINT_INLINE
size_t ZSTD_execSequence(BYTE* op,
BYTE* const oend, seq_t sequence,
@ -634,20 +713,29 @@ size_t ZSTD_execSequence(BYTE* op,
const BYTE* const iLitEnd = *litPtr + sequence.litLength;
const BYTE* match = oLitEnd - sequence.offset;
/* check */
RETURN_ERROR_IF(oMatchEnd>oend, dstSize_tooSmall, "last match must start at a minimum distance of WILDCOPY_OVERLENGTH from oend");
RETURN_ERROR_IF(iLitEnd > litLimit, corruption_detected, "over-read beyond lit buffer");
if (oLitEnd>oend_w) return ZSTD_execSequenceLast7(op, oend, sequence, litPtr, litLimit, prefixStart, virtualStart, dictEnd);
/* Errors and uncommon cases handled here. */
assert(oLitEnd < oMatchEnd);
if (iLitEnd > litLimit || oMatchEnd > oend_w)
return ZSTD_execSequenceEnd(op, oend, sequence, litPtr, litLimit, prefixStart, virtualStart, dictEnd);
/* copy Literals */
if (sequence.litLength > 8)
ZSTD_wildcopy_16min(op, (*litPtr), oend, sequence.litLength, ZSTD_no_overlap); /* note : since oLitEnd <= oend-WILDCOPY_OVERLENGTH, no risk of overwrite beyond oend */
else
ZSTD_copy8(op, *litPtr);
/* Assumptions (everything else goes into ZSTD_execSequenceEnd()) */
assert(iLitEnd <= litLimit /* Literal length is in bounds */);
assert(oLitEnd <= oend_w /* Can wildcopy literals */);
assert(oMatchEnd <= oend_w /* Can wildcopy matches */);
/* Copy Literals:
* Split out litLength <= 16 since it is nearly always true. +1.6% on gcc-9.
* We likely don't need the full 32-byte wildcopy.
*/
assert(WILDCOPY_OVERLENGTH >= 16);
ZSTD_copy16(op, (*litPtr));
if (sequence.litLength > 16) {
ZSTD_wildcopy(op+16, (*litPtr)+16, sequence.litLength-16, ZSTD_no_overlap);
}
op = oLitEnd;
*litPtr = iLitEnd; /* update for next sequence */
/* copy Match */
/* Copy Match */
if (sequence.offset > (size_t)(oLitEnd - prefixStart)) {
/* offset beyond prefix -> go into extDict */
RETURN_ERROR_IF(sequence.offset > (size_t)(oLitEnd - virtualStart), corruption_detected);
@ -662,123 +750,33 @@ size_t ZSTD_execSequence(BYTE* op,
op = oLitEnd + length1;
sequence.matchLength -= length1;
match = prefixStart;
if (op > oend_w || sequence.matchLength < MINMATCH) {
U32 i;
for (i = 0; i < sequence.matchLength; ++i) op[i] = match[i];
return sequenceLength;
}
} }
/* Requirement: op <= oend_w && sequence.matchLength >= MINMATCH */
/* Match within prefix of 1 or more bytes */
assert(op <= oMatchEnd);
assert(oMatchEnd <= oend_w);
assert(match >= prefixStart);
assert(sequence.matchLength >= 1);
/* match within prefix */
if (sequence.offset < 8) {
/* close range match, overlap */
static const U32 dec32table[] = { 0, 1, 2, 1, 4, 4, 4, 4 }; /* added */
static const int dec64table[] = { 8, 8, 8, 7, 8, 9,10,11 }; /* subtracted */
int const sub2 = dec64table[sequence.offset];
op[0] = match[0];
op[1] = match[1];
op[2] = match[2];
op[3] = match[3];
match += dec32table[sequence.offset];
ZSTD_copy4(op+4, match);
match -= sub2;
} else {
ZSTD_copy8(op, match);
}
op += 8; match += 8;
if (oMatchEnd > oend-(16-MINMATCH)) {
if (op < oend_w) {
ZSTD_wildcopy(op, match, oend, oend_w - op, ZSTD_overlap_src_before_dst);
match += oend_w - op;
op = oend_w;
}
while (op < oMatchEnd) *op++ = *match++;
} else {
ZSTD_wildcopy(op, match, oend, (ptrdiff_t)sequence.matchLength-8, ZSTD_overlap_src_before_dst); /* works even if matchLength < 8 */
}
return sequenceLength;
}
HINT_INLINE
size_t ZSTD_execSequenceLong(BYTE* op,
BYTE* const oend, seq_t sequence,
const BYTE** litPtr, const BYTE* const litLimit,
const BYTE* const prefixStart, const BYTE* const dictStart, const BYTE* const dictEnd)
{
BYTE* const oLitEnd = op + sequence.litLength;
size_t const sequenceLength = sequence.litLength + sequence.matchLength;
BYTE* const oMatchEnd = op + sequenceLength; /* risk : address space overflow (32-bits) */
BYTE* const oend_w = oend - WILDCOPY_OVERLENGTH;
const BYTE* const iLitEnd = *litPtr + sequence.litLength;
const BYTE* match = sequence.match;
/* check */
RETURN_ERROR_IF(oMatchEnd > oend, dstSize_tooSmall, "last match must start at a minimum distance of WILDCOPY_OVERLENGTH from oend");
RETURN_ERROR_IF(iLitEnd > litLimit, corruption_detected, "over-read beyond lit buffer");
if (oLitEnd > oend_w) return ZSTD_execSequenceLast7(op, oend, sequence, litPtr, litLimit, prefixStart, dictStart, dictEnd);
/* copy Literals */
if (sequence.litLength > 8)
ZSTD_wildcopy_16min(op, *litPtr, oend, sequence.litLength, ZSTD_no_overlap); /* note : since oLitEnd <= oend-WILDCOPY_OVERLENGTH, no risk of overwrite beyond oend */
else
ZSTD_copy8(op, *litPtr); /* note : op <= oLitEnd <= oend_w == oend - 8 */
op = oLitEnd;
*litPtr = iLitEnd; /* update for next sequence */
/* copy Match */
if (sequence.offset > (size_t)(oLitEnd - prefixStart)) {
/* offset beyond prefix */
RETURN_ERROR_IF(sequence.offset > (size_t)(oLitEnd - dictStart), corruption_detected);
if (match + sequence.matchLength <= dictEnd) {
memmove(oLitEnd, match, sequence.matchLength);
/* Nearly all offsets are >= WILDCOPY_VECLEN bytes, which means we can use wildcopy
* without overlap checking.
*/
if (sequence.offset >= WILDCOPY_VECLEN) {
/* We bet on a full wildcopy for matches, since we expect matches to be
* longer than literals (in general). In silesia, ~10% of matches are longer
* than 16 bytes.
*/
ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength, ZSTD_no_overlap);
return sequenceLength;
}
/* span extDict & currentPrefixSegment */
{ size_t const length1 = dictEnd - match;
memmove(oLitEnd, match, length1);
op = oLitEnd + length1;
sequence.matchLength -= length1;
match = prefixStart;
if (op > oend_w || sequence.matchLength < MINMATCH) {
U32 i;
for (i = 0; i < sequence.matchLength; ++i) op[i] = match[i];
return sequenceLength;
}
} }
assert(op <= oend_w);
assert(sequence.matchLength >= MINMATCH);
assert(sequence.offset < WILDCOPY_VECLEN);
/* match within prefix */
if (sequence.offset < 8) {
/* close range match, overlap */
static const U32 dec32table[] = { 0, 1, 2, 1, 4, 4, 4, 4 }; /* added */
static const int dec64table[] = { 8, 8, 8, 7, 8, 9,10,11 }; /* subtracted */
int const sub2 = dec64table[sequence.offset];
op[0] = match[0];
op[1] = match[1];
op[2] = match[2];
op[3] = match[3];
match += dec32table[sequence.offset];
ZSTD_copy4(op+4, match);
match -= sub2;
} else {
ZSTD_copy8(op, match);
}
op += 8; match += 8;
/* Copy 8 bytes and spread the offset to be >= 8. */
ZSTD_overlapCopy8(&op, &match, sequence.offset);
if (oMatchEnd > oend-(16-MINMATCH)) {
if (op < oend_w) {
ZSTD_wildcopy(op, match, oend, oend_w - op, ZSTD_overlap_src_before_dst);
match += oend_w - op;
op = oend_w;
}
while (op < oMatchEnd) *op++ = *match++;
} else {
ZSTD_wildcopy(op, match, oend, (ptrdiff_t)sequence.matchLength-8, ZSTD_overlap_src_before_dst); /* works even if matchLength < 8 */
/* If the match length is > 8 bytes, then continue with the wildcopy. */
if (sequence.matchLength > 8) {
assert(op < oMatchEnd);
ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength-8, ZSTD_overlap_src_before_dst);
}
return sequenceLength;
}
@ -1098,7 +1096,7 @@ ZSTD_decompressSequencesLong_body(
/* decode and decompress */
for ( ; (BIT_reloadDStream(&(seqState.DStream)) <= BIT_DStream_completed) && (seqNb<nbSeq) ; seqNb++) {
seq_t const sequence = ZSTD_decodeSequenceLong(&seqState, isLongOffset);
size_t const oneSeqSize = ZSTD_execSequenceLong(op, oend, sequences[(seqNb-ADVANCED_SEQS) & STORED_SEQS_MASK], &litPtr, litEnd, prefixStart, dictStart, dictEnd);
size_t const oneSeqSize = ZSTD_execSequence(op, oend, sequences[(seqNb-ADVANCED_SEQS) & STORED_SEQS_MASK], &litPtr, litEnd, prefixStart, dictStart, dictEnd);
if (ZSTD_isError(oneSeqSize)) return oneSeqSize;
PREFETCH_L1(sequence.match); PREFETCH_L1(sequence.match + sequence.matchLength - 1); /* note : it's safe to invoke PREFETCH() on any memory address, including invalid ones */
sequences[seqNb & STORED_SEQS_MASK] = sequence;
@ -1109,7 +1107,7 @@ ZSTD_decompressSequencesLong_body(
/* finish queue */
seqNb -= seqAdvance;
for ( ; seqNb<nbSeq ; seqNb++) {
size_t const oneSeqSize = ZSTD_execSequenceLong(op, oend, sequences[seqNb&STORED_SEQS_MASK], &litPtr, litEnd, prefixStart, dictStart, dictEnd);
size_t const oneSeqSize = ZSTD_execSequence(op, oend, sequences[seqNb&STORED_SEQS_MASK], &litPtr, litEnd, prefixStart, dictStart, dictEnd);
if (ZSTD_isError(oneSeqSize)) return oneSeqSize;
op += oneSeqSize;
}

2
lib/legacy/zstd_v01.c
View File

@ -346,7 +346,7 @@ FORCE_INLINE unsigned FSE_highbit32 (U32 val)
_BitScanReverse ( &r, val );
return (unsigned) r;
# elif defined(__GNUC__) && (GCC_VERSION >= 304) /* GCC Intrinsic */
return 31 - __builtin_clz (val);
return __builtin_clz (val) ^ 31;
# else /* Software version */
static const unsigned DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 };
U32 v = val;

2
lib/legacy/zstd_v02.c
View File

@ -353,7 +353,7 @@ MEM_STATIC unsigned BIT_highbit32 (U32 val)
_BitScanReverse ( &r, val );
return (unsigned) r;
# elif defined(__GNUC__) && (__GNUC__ >= 3) /* Use GCC Intrinsic */
return 31 - __builtin_clz (val);
return __builtin_clz (val) ^ 31;
# else /* Software version */
static const unsigned DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 };
U32 v = val;

2
lib/legacy/zstd_v03.c
View File

@ -356,7 +356,7 @@ MEM_STATIC unsigned BIT_highbit32 (U32 val)
_BitScanReverse ( &r, val );
return (unsigned) r;
# elif defined(__GNUC__) && (__GNUC__ >= 3) /* Use GCC Intrinsic */
return 31 - __builtin_clz (val);
return __builtin_clz (val) ^ 31;
# else /* Software version */
static const unsigned DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 };
U32 v = val;

2
lib/legacy/zstd_v04.c
View File

@ -627,7 +627,7 @@ MEM_STATIC unsigned BIT_highbit32 (U32 val)
_BitScanReverse ( &r, val );
return (unsigned) r;
# elif defined(__GNUC__) && (__GNUC__ >= 3) /* Use GCC Intrinsic */
return 31 - __builtin_clz (val);
return __builtin_clz (val) ^ 31;
# else /* Software version */
static const unsigned DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 };
U32 v = val;

2
lib/legacy/zstd_v05.c
View File

@ -756,7 +756,7 @@ MEM_STATIC unsigned BITv05_highbit32 (U32 val)
_BitScanReverse ( &r, val );
return (unsigned) r;
# elif defined(__GNUC__) && (__GNUC__ >= 3) /* Use GCC Intrinsic */
return 31 - __builtin_clz (val);
return __builtin_clz (val) ^ 31;
# else /* Software version */
static const unsigned DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 };
U32 v = val;

2
lib/legacy/zstd_v06.c
View File

@ -860,7 +860,7 @@ MEM_STATIC unsigned BITv06_highbit32 ( U32 val)
_BitScanReverse ( &r, val );
return (unsigned) r;
# elif defined(__GNUC__) && (__GNUC__ >= 3) /* Use GCC Intrinsic */
return 31 - __builtin_clz (val);
return __builtin_clz (val) ^ 31;
# else /* Software version */
static const unsigned DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 };
U32 v = val;

2
lib/legacy/zstd_v07.c
View File

@ -530,7 +530,7 @@ MEM_STATIC unsigned BITv07_highbit32 (U32 val)
_BitScanReverse ( &r, val );
return (unsigned) r;
# elif defined(__GNUC__) && (__GNUC__ >= 3) /* Use GCC Intrinsic */
return 31 - __builtin_clz (val);
return __builtin_clz (val) ^ 31;
# else /* Software version */
static const unsigned DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 };
U32 v = val;

179
lib/zstd.h
View File

@ -72,7 +72,7 @@ extern "C" {
/*------ Version ------*/
#define ZSTD_VERSION_MAJOR 1
#define ZSTD_VERSION_MINOR 4
#define ZSTD_VERSION_RELEASE 3
#define ZSTD_VERSION_RELEASE 4
#define ZSTD_VERSION_NUMBER (ZSTD_VERSION_MAJOR *100*100 + ZSTD_VERSION_MINOR *100 + ZSTD_VERSION_RELEASE)
ZSTDLIB_API unsigned ZSTD_versionNumber(void); /**< to check runtime library version */
@ -197,9 +197,13 @@ ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx(void);
ZSTDLIB_API size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx);
/*! ZSTD_compressCCtx() :
* Same as ZSTD_compress(), using an explicit ZSTD_CCtx
* The function will compress at requested compression level,
* ignoring any other parameter */
* Same as ZSTD_compress(), using an explicit ZSTD_CCtx.
* Important : in order to behave similarly to `ZSTD_compress()`,
* this function compresses at requested compression level,
* __ignoring any other parameter__ .
* If any advanced parameter was set using the advanced API,
* they will all be reset. Only `compressionLevel` remains.
*/
ZSTDLIB_API size_t ZSTD_compressCCtx(ZSTD_CCtx* cctx,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize,
@ -234,7 +238,7 @@ ZSTDLIB_API size_t ZSTD_decompressDCtx(ZSTD_DCtx* dctx,
* using ZSTD_CCtx_set*() functions.
* Pushed parameters are sticky : they are valid for next compressed frame, and any subsequent frame.
* "sticky" parameters are applicable to `ZSTD_compress2()` and `ZSTD_compressStream*()` !
* They do not apply to "simple" one-shot variants such as ZSTD_compressCCtx()
* __They do not apply to "simple" one-shot variants such as ZSTD_compressCCtx()__ .
*
* It's possible to reset all parameters to "default" using ZSTD_CCtx_reset().
*
@ -262,18 +266,26 @@ typedef enum {
/* compression parameters
* Note: When compressing with a ZSTD_CDict these parameters are superseded
* by the parameters used to construct the ZSTD_CDict. See ZSTD_CCtx_refCDict()
* for more info (superseded-by-cdict). */
ZSTD_c_compressionLevel=100, /* Update all compression parameters according to pre-defined cLevel table
* by the parameters used to construct the ZSTD_CDict.
* See ZSTD_CCtx_refCDict() for more info (superseded-by-cdict). */
ZSTD_c_compressionLevel=100, /* Set compression parameters according to pre-defined cLevel table.
* Note that exact compression parameters are dynamically determined,
* depending on both compression level and srcSize (when known).
* Default level is ZSTD_CLEVEL_DEFAULT==3.
* Special: value 0 means default, which is controlled by ZSTD_CLEVEL_DEFAULT.
* Note 1 : it's possible to pass a negative compression level.
* Note 2 : setting a level sets all default values of other compression parameters */
* Note 2 : setting a level resets all other compression parameters to default */
/* Advanced compression parameters :
* It's possible to pin down compression parameters to some specific values.
* In which case, these values are no longer dynamically selected by the compressor */
ZSTD_c_windowLog=101, /* Maximum allowed back-reference distance, expressed as power of 2.
* This will set a memory budget for streaming decompression,
* with larger values requiring more memory
* and typically compressing more.
* Must be clamped between ZSTD_WINDOWLOG_MIN and ZSTD_WINDOWLOG_MAX.
* Special: value 0 means "use default windowLog".
* Note: Using a windowLog greater than ZSTD_WINDOWLOG_LIMIT_DEFAULT
* requires explicitly allowing such window size at decompression stage if using streaming. */
* requires explicitly allowing such size at streaming decompression stage. */
ZSTD_c_hashLog=102, /* Size of the initial probe table, as a power of 2.
* Resulting memory usage is (1 << (hashLog+2)).
* Must be clamped between ZSTD_HASHLOG_MIN and ZSTD_HASHLOG_MAX.
@ -284,13 +296,13 @@ typedef enum {
* Resulting memory usage is (1 << (chainLog+2)).
* Must be clamped between ZSTD_CHAINLOG_MIN and ZSTD_CHAINLOG_MAX.
* Larger tables result in better and slower compression.
* This parameter is useless when using "fast" strategy.
* This parameter is useless for "fast" strategy.
* It's still useful when using "dfast" strategy,
* in which case it defines a secondary probe table.
* Special: value 0 means "use default chainLog". */
ZSTD_c_searchLog=104, /* Number of search attempts, as a power of 2.
* More attempts result in better and slower compression.
* This parameter is useless when using "fast" and "dFast" strategies.
* This parameter is useless for "fast" and "dFast" strategies.
* Special: value 0 means "use default searchLog". */
ZSTD_c_minMatch=105, /* Minimum size of searched matches.
* Note that Zstandard can still find matches of smaller size,
@ -345,7 +357,7 @@ typedef enum {
ZSTD_c_contentSizeFlag=200, /* Content size will be written into frame header _whenever known_ (default:1)
* Content size must be known at the beginning of compression.
* This is automatically the case when using ZSTD_compress2(),
* For streaming variants, content size must be provided with ZSTD_CCtx_setPledgedSrcSize() */
* For streaming scenarios, content size must be provided with ZSTD_CCtx_setPledgedSrcSize() */
ZSTD_c_checksumFlag=201, /* A 32-bits checksum of content is written at end of frame (default:0) */
ZSTD_c_dictIDFlag=202, /* When applicable, dictionary's ID is written into frame header (default:1) */
@ -364,7 +376,7 @@ typedef enum {
* Each compression job is completed in parallel, so this value can indirectly impact the nb of active threads.
* 0 means default, which is dynamically determined based on compression parameters.
* Job size must be a minimum of overlap size, or 1 MB, whichever is largest.
* The minimum size is automatically and transparently enforced */
* The minimum size is automatically and transparently enforced. */
ZSTD_c_overlapLog=402, /* Control the overlap size, as a fraction of window size.
* The overlap size is an amount of data reloaded from previous job at the beginning of a new job.
* It helps preserve compression ratio, while each job is compressed in parallel.
@ -398,7 +410,7 @@ typedef enum {
ZSTD_c_experimentalParam4=1001,
ZSTD_c_experimentalParam5=1002,
ZSTD_c_experimentalParam6=1003,
ZSTD_c_experimentalParam7=1004,
ZSTD_c_experimentalParam7=1004
} ZSTD_cParameter;
typedef struct {
@ -928,7 +940,7 @@ ZSTDLIB_API size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict);
* Note 3 : Referencing a prefix involves building tables, which are dependent on compression parameters.
* It's a CPU consuming operation, with non-negligible impact on latency.
* If there is a need to use the same prefix multiple times, consider loadDictionary instead.
* Note 4 : By default, the prefix is interpreted as raw content (ZSTD_dm_rawContent).
* Note 4 : By default, the prefix is interpreted as raw content (ZSTD_dct_rawContent).
* Use experimental ZSTD_CCtx_refPrefix_advanced() to alter dictionary interpretation. */
ZSTDLIB_API size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx,
const void* prefix, size_t prefixSize);
@ -972,7 +984,7 @@ ZSTDLIB_API size_t ZSTD_DCtx_refDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict);
* Note 2 : Prefix buffer is referenced. It **must** outlive decompression.
* Prefix buffer must remain unmodified up to the end of frame,
* reached when ZSTD_decompressStream() returns 0.
* Note 3 : By default, the prefix is treated as raw content (ZSTD_dm_rawContent).
* Note 3 : By default, the prefix is treated as raw content (ZSTD_dct_rawContent).
* Use ZSTD_CCtx_refPrefix_advanced() to alter dictMode (Experimental section)
* Note 4 : Referencing a raw content prefix has almost no cpu nor memory cost.
* A full dictionary is more costly, as it requires building tables.
@ -1017,8 +1029,8 @@ ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
* Some of them might be removed in the future (especially when redundant with existing stable functions)
* ***************************************************************************************/
#define ZSTD_FRAMEHEADERSIZE_PREFIX 5 /* minimum input size required to query frame header size */
#define ZSTD_FRAMEHEADERSIZE_MIN 6
#define ZSTD_FRAMEHEADERSIZE_PREFIX(format) ((format) == ZSTD_f_zstd1 ? 5 : 1) /* minimum input size required to query frame header size */
#define ZSTD_FRAMEHEADERSIZE_MIN(format) ((format) == ZSTD_f_zstd1 ? 6 : 2)
#define ZSTD_FRAMEHEADERSIZE_MAX 18 /* can be useful for static allocation */
#define ZSTD_SKIPPABLEHEADERSIZE 8
@ -1077,6 +1089,24 @@ ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
typedef struct ZSTD_CCtx_params_s ZSTD_CCtx_params;
typedef struct {
unsigned int matchPos; /* Match pos in dst */
/* If seqDef.offset > 3, then this is seqDef.offset - 3
* If seqDef.offset < 3, then this is the corresponding repeat offset
* But if seqDef.offset < 3 and litLength == 0, this is the
* repeat offset before the corresponding repeat offset
* And if seqDef.offset == 3 and litLength == 0, this is the
* most recent repeat offset - 1
*/
unsigned int offset;
unsigned int litLength; /* Literal length */
unsigned int matchLength; /* Match length */
/* 0 when seq not rep and seqDef.offset otherwise
* when litLength == 0 this will be <= 4, otherwise <= 3 like normal
*/
unsigned int rep;
} ZSTD_Sequence;
typedef struct {
unsigned windowLog; /**< largest match distance : larger == more compression, more memory needed during decompression */
unsigned chainLog; /**< fully searched segment : larger == more compression, slower, more memory (useless for fast) */
@ -1106,21 +1136,12 @@ typedef enum {
typedef enum {
ZSTD_dlm_byCopy = 0, /**< Copy dictionary content internally */
ZSTD_dlm_byRef = 1, /**< Reference dictionary content -- the dictionary buffer must outlive its users. */
ZSTD_dlm_byRef = 1 /**< Reference dictionary content -- the dictionary buffer must outlive its users. */
} ZSTD_dictLoadMethod_e;
typedef enum {
/* Opened question : should we have a format ZSTD_f_auto ?
* Today, it would mean exactly the same as ZSTD_f_zstd1.
* But, in the future, should several formats become supported,
* on the compression side, it would mean "default format".
* On the decompression side, it would mean "automatic format detection",
* so that ZSTD_f_zstd1 would mean "accept *only* zstd frames".
* Since meaning is a little different, another option could be to define different enums for compression and decompression.
* This question could be kept for later, when there are actually multiple formats to support,
* but there is also the question of pinning enum values, and pinning value `0` is especially important */
ZSTD_f_zstd1 = 0, /* zstd frame format, specified in zstd_compression_format.md (default) */
ZSTD_f_zstd1_magicless = 1, /* Variant of zstd frame format, without initial 4-bytes magic number.
ZSTD_f_zstd1_magicless = 1 /* Variant of zstd frame format, without initial 4-bytes magic number.
* Useful to save 4 bytes per generated frame.
* Decoder cannot recognise automatically this format, requiring this instruction. */
} ZSTD_format_e;
@ -1154,7 +1175,7 @@ typedef enum {
*/
ZSTD_dictDefaultAttach = 0, /* Use the default heuristic. */
ZSTD_dictForceAttach = 1, /* Never copy the dictionary. */
ZSTD_dictForceCopy = 2, /* Always copy the dictionary. */
ZSTD_dictForceCopy = 2 /* Always copy the dictionary. */
} ZSTD_dictAttachPref_e;
typedef enum {
@ -1163,7 +1184,7 @@ typedef enum {
* levels will be compressed. */
ZSTD_lcm_huffman = 1, /**< Always attempt Huffman compression. Uncompressed literals will still be
* emitted if Huffman compression is not profitable. */
ZSTD_lcm_uncompressed = 2, /**< Always emit uncompressed literals. */
ZSTD_lcm_uncompressed = 2 /**< Always emit uncompressed literals. */
} ZSTD_literalCompressionMode_e;
@ -1215,20 +1236,38 @@ ZSTDLIB_API unsigned long long ZSTD_decompressBound(const void* src, size_t srcS
* or an error code (if srcSize is too small) */
ZSTDLIB_API size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize);
/*! ZSTD_getSequences() :
* Extract sequences from the sequence store
* zc can be used to insert custom compression params.
* This function invokes ZSTD_compress2
* @return : number of sequences extracted
*/
ZSTDLIB_API size_t ZSTD_getSequences(ZSTD_CCtx* zc, ZSTD_Sequence* outSeqs,
size_t outSeqsSize, const void* src, size_t srcSize);
/***************************************
* Memory management
***************************************/
/*! ZSTD_estimate*() :
* These functions make it possible to estimate memory usage
* of a future {D,C}Ctx, before its creation.
* ZSTD_estimateCCtxSize() will provide a budget large enough for any compression level up to selected one.
* It will also consider src size to be arbitrarily "large", which is worst case.
* If srcSize is known to always be small, ZSTD_estimateCCtxSize_usingCParams() can provide a tighter estimation.
* ZSTD_estimateCCtxSize_usingCParams() can be used in tandem with ZSTD_getCParams() to create cParams from compressionLevel.
* ZSTD_estimateCCtxSize_usingCCtxParams() can be used in tandem with ZSTD_CCtxParams_setParameter(). Only single-threaded compression is supported. This function will return an error code if ZSTD_c_nbWorkers is >= 1.
* Note : CCtx size estimation is only correct for single-threaded compression. */
* These functions make it possible to estimate memory usage of a future
* {D,C}Ctx, before its creation.
*
* ZSTD_estimateCCtxSize() will provide a budget large enough for any
* compression level up to selected one. Unlike ZSTD_estimateCStreamSize*(),
* this estimate does not include space for a window buffer, so this estimate
* is guaranteed to be enough for single-shot compressions, but not streaming
* compressions. It will however assume the input may be arbitrarily large,
* which is the worst case. If srcSize is known to always be small,
* ZSTD_estimateCCtxSize_usingCParams() can provide a tighter estimation.
* ZSTD_estimateCCtxSize_usingCParams() can be used in tandem with
* ZSTD_getCParams() to create cParams from compressionLevel.
* ZSTD_estimateCCtxSize_usingCCtxParams() can be used in tandem with
* ZSTD_CCtxParams_setParameter().
*
* Note: only single-threaded compression is supported. This function will
* return an error code if ZSTD_c_nbWorkers is >= 1. */
ZSTDLIB_API size_t ZSTD_estimateCCtxSize(int compressionLevel);
ZSTDLIB_API size_t ZSTD_estimateCCtxSize_usingCParams(ZSTD_compressionParameters cParams);
ZSTDLIB_API size_t ZSTD_estimateCCtxSize_usingCCtxParams(const ZSTD_CCtx_params* params);
@ -1339,7 +1378,8 @@ ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictS
* Create a digested dictionary for compression
* Dictionary content is just referenced, not duplicated.
* As a consequence, `dictBuffer` **must** outlive CDict,
* and its content must remain unmodified throughout the lifetime of CDict. */
* and its content must remain unmodified throughout the lifetime of CDict.
* note: equivalent to ZSTD_createCDict_advanced(), with dictLoadMethod==ZSTD_dlm_byRef */
ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_byReference(const void* dictBuffer, size_t dictSize, int compressionLevel);
/*! ZSTD_getCParams() :
@ -1366,7 +1406,9 @@ ZSTDLIB_API size_t ZSTD_checkCParams(ZSTD_compressionParameters params);
ZSTDLIB_API ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, unsigned long long srcSize, size_t dictSize);
/*! ZSTD_compress_advanced() :
* Same as ZSTD_compress_usingDict(), with fine-tune control over compression parameters (by structure) */
* Note : this function is now DEPRECATED.
* It can be replaced by ZSTD_compress2(), in combination with ZSTD_CCtx_setParameter() and other parameter setters.
* This prototype will be marked as deprecated and generate compilation warning on reaching v1.5.x */
ZSTDLIB_API size_t ZSTD_compress_advanced(ZSTD_CCtx* cctx,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize,
@ -1374,7 +1416,9 @@ ZSTDLIB_API size_t ZSTD_compress_advanced(ZSTD_CCtx* cctx,
ZSTD_parameters params);
/*! ZSTD_compress_usingCDict_advanced() :
* Same as ZSTD_compress_usingCDict(), with fine-tune control over frame parameters */
* Note : this function is now REDUNDANT.
* It can be replaced by ZSTD_compress2(), in combination with ZSTD_CCtx_loadDictionary() and other parameter setters.
* This prototype will be marked as deprecated and generate compilation warning in some future version */
ZSTDLIB_API size_t ZSTD_compress_usingCDict_advanced(ZSTD_CCtx* cctx,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize,
@ -1624,8 +1668,13 @@ ZSTDLIB_API size_t ZSTD_decompressStream_simpleArgs (
* pledgedSrcSize must be correct. If it is not known at init time, use
* ZSTD_CONTENTSIZE_UNKNOWN. Note that, for compatibility with older programs,
* "0" also disables frame content size field. It may be enabled in the future.
* Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x
*/
ZSTDLIB_API size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize);
ZSTDLIB_API size_t
ZSTD_initCStream_srcSize(ZSTD_CStream* zcs,
int compressionLevel,
unsigned long long pledgedSrcSize);
/**! ZSTD_initCStream_usingDict() :
* This function is deprecated, and is equivalent to:
* ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
@ -1634,10 +1683,15 @@ ZSTDLIB_API size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLe
*
* Creates of an internal CDict (incompatible with static CCtx), except if
* dict == NULL or dictSize < 8, in which case no dict is used.
* Note: dict is loaded with ZSTD_dm_auto (treated as a full zstd dictionary if
* Note: dict is loaded with ZSTD_dct_auto (treated as a full zstd dictionary if
* it begins with ZSTD_MAGIC_DICTIONARY, else as raw content) and ZSTD_dlm_byCopy.
* Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x
*/
ZSTDLIB_API size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel);
ZSTDLIB_API size_t
ZSTD_initCStream_usingDict(ZSTD_CStream* zcs,
const void* dict, size_t dictSize,
int compressionLevel);
/**! ZSTD_initCStream_advanced() :
* This function is deprecated, and is approximately equivalent to:
* ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
@ -1648,21 +1702,29 @@ ZSTDLIB_API size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dic
* ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize);
* ZSTD_CCtx_loadDictionary(zcs, dict, dictSize);
*
* pledgedSrcSize must be correct. If srcSize is not known at init time, use
* value ZSTD_CONTENTSIZE_UNKNOWN. dict is loaded with ZSTD_dm_auto and ZSTD_dlm_byCopy.
* dict is loaded with ZSTD_dct_auto and ZSTD_dlm_byCopy.
* pledgedSrcSize must be correct.
* If srcSize is not known at init time, use value ZSTD_CONTENTSIZE_UNKNOWN.
* Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x
*/
ZSTDLIB_API size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize,
ZSTD_parameters params, unsigned long long pledgedSrcSize);
ZSTDLIB_API size_t
ZSTD_initCStream_advanced(ZSTD_CStream* zcs,
const void* dict, size_t dictSize,
ZSTD_parameters params,
unsigned long long pledgedSrcSize);
/**! ZSTD_initCStream_usingCDict() :
* This function is deprecated, and equivalent to:
* ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
* ZSTD_CCtx_refCDict(zcs, cdict);
*
* note : cdict will just be referenced, and must outlive compression session
* Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x
*/
ZSTDLIB_API size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict);
/**! ZSTD_initCStream_usingCDict_advanced() :
* This function is deprecated, and is approximately equivalent to:
* This function is DEPRECATED, and is approximately equivalent to:
* ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
* // Pseudocode: Set each zstd frame parameter and leave the rest as-is.
* for ((fParam, value) : fParams) {
@ -1674,8 +1736,13 @@ ZSTDLIB_API size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDi
* same as ZSTD_initCStream_usingCDict(), with control over frame parameters.
* pledgedSrcSize must be correct. If srcSize is not known at init time, use
* value ZSTD_CONTENTSIZE_UNKNOWN.
* Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x
*/
ZSTDLIB_API size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, const ZSTD_CDict* cdict, ZSTD_frameParameters fParams, unsigned long long pledgedSrcSize);
ZSTDLIB_API size_t
ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs,
const ZSTD_CDict* cdict,
ZSTD_frameParameters fParams,
unsigned long long pledgedSrcSize);
/*! ZSTD_resetCStream() :
* This function is deprecated, and is equivalent to:
@ -1690,6 +1757,7 @@ ZSTDLIB_API size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, const
* For the time being, pledgedSrcSize==0 is interpreted as "srcSize unknown" for compatibility with older programs,
* but it will change to mean "empty" in future version, so use macro ZSTD_CONTENTSIZE_UNKNOWN instead.
* @return : 0, or an error code (which can be tested using ZSTD_isError())
* Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x
*/
ZSTDLIB_API size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize);
@ -1735,8 +1803,10 @@ ZSTDLIB_API size_t ZSTD_toFlushNow(ZSTD_CCtx* cctx);
* ZSTD_DCtx_loadDictionary(zds, dict, dictSize);
*
* note: no dictionary will be used if dict == NULL or dictSize < 8
* Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x
*/
ZSTDLIB_API size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize);
/**
* This function is deprecated, and is equivalent to:
*
@ -1744,14 +1814,17 @@ ZSTDLIB_API size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dic
* ZSTD_DCtx_refDDict(zds, ddict);
*
* note : ddict is referenced, it must outlive decompression session
* Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x
*/
ZSTDLIB_API size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict);
/**
* This function is deprecated, and is equivalent to:
*
* ZSTD_DCtx_reset(zds, ZSTD_reset_session_only);
*
* re-use decompression parameters from previous init; saves dictionary loading
* Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x
*/
ZSTDLIB_API size_t ZSTD_resetDStream(ZSTD_DStream* zds);
@ -1925,7 +1998,7 @@ ZSTDLIB_API ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx);
/*!
Block functions produce and decode raw zstd blocks, without frame metadata.
Frame metadata cost is typically ~18 bytes, which can be non-negligible for very small blocks (< 100 bytes).
Frame metadata cost is typically ~12 bytes, which can be non-negligible for very small blocks (< 100 bytes).
But users will have to take in charge needed metadata to regenerate data, such as compressed and content sizes.
A few rules to respect :

9
programs/README.md
View File

@ -173,10 +173,13 @@ Benchmark arguments :
```
#### Restricted usage of Environment Variables
Using environment variables to set compression/decompression parameters has security implications. Therefore,
we intentionally restrict its usage. Currently, only `ZSTD_CLEVEL` is supported for setting compression level.
Using environment variables to set parameters has security implications.
Therefore, this avenue is intentionally restricted.
Only `ZSTD_CLEVEL` is supported currently, for setting compression level.
`ZSTD_CLEVEL` can be used to set the level between 1 and 19 (the "normal" range).
If the value of `ZSTD_CLEVEL` is not a valid integer, it will be ignored with a warning message.
Note that command line options will override corresponding environment variable settings.
`ZSTD_CLEVEL` just replaces the default compression level (`3`).
It can be overridden by corresponding command line arguments.
#### Long distance matching mode
The long distance matching mode, enabled with `--long`, is designed to improve

23
programs/benchzstd.c
View File

@ -88,7 +88,7 @@ static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER;
#endif
#define DEBUGOUTPUT(...) { if (DEBUG) DISPLAY(__VA_ARGS__); }
#define EXM_THROW_INT(errorNum, ...) { \
#define RETURN_ERROR_INT(errorNum, ...) { \
DEBUGOUTPUT("%s: %i: \n", __FILE__, __LINE__); \
DISPLAYLEVEL(1, "Error %i : ", errorNum); \
DISPLAYLEVEL(1, __VA_ARGS__); \
@ -401,9 +401,9 @@ BMK_benchMemAdvancedNoAlloc(
BMK_initCCtxArgs cctxprep;
BMK_initDCtxArgs dctxprep;
cbp.benchFn = local_defaultCompress;
cbp.benchFn = local_defaultCompress; /* ZSTD_compress2 */
cbp.benchPayload = cctx;
cbp.initFn = local_initCCtx;
cbp.initFn = local_initCCtx; /* BMK_initCCtx */
cbp.initPayload = &cctxprep;
cbp.errorFn = ZSTD_isError;
cbp.blockCount = nbBlocks;
@ -534,8 +534,8 @@ BMK_benchMemAdvancedNoAlloc(
if (u==srcSize-1) { /* should never happen */
DISPLAY("no difference detected\n");
}
}
}
} /* for (u=0; u<srcSize; u++) */
} /* if ((adv->mode == BMK_both) && (crcOrig!=crcCheck)) */
} /* CRC Checking */
if (displayLevel == 1) { /* hidden display mode -q, used by python speed benchmark */
@ -754,8 +754,7 @@ static int BMK_loadFiles(void* buffer, size_t bufferSize,
size_t pos = 0, totalSize = 0;
unsigned n;
for (n=0; n<nbFiles; n++) {
FILE* f;
U64 fileSize = UTIL_getFileSize(fileNamesTable[n]);
U64 fileSize = UTIL_getFileSize(fileNamesTable[n]); /* last file may be shortened */
if (UTIL_isDirectory(fileNamesTable[n])) {
DISPLAYLEVEL(2, "Ignoring %s directory... \n", fileNamesTable[n]);
fileSizes[n] = 0;
@ -766,20 +765,20 @@ static int BMK_loadFiles(void* buffer, size_t bufferSize,
fileSizes[n] = 0;
continue;
}
f = fopen(fileNamesTable[n], "rb");
if (f==NULL) EXM_THROW_INT(10, "impossible to open file %s", fileNamesTable[n]);
{ FILE* const f = fopen(fileNamesTable[n], "rb");
if (f==NULL) RETURN_ERROR_INT(10, "impossible to open file %s", fileNamesTable[n]);
DISPLAYUPDATE(2, "Loading %s... \r", fileNamesTable[n]);
if (fileSize > bufferSize-pos) fileSize = bufferSize-pos, nbFiles=n; /* buffer too small - stop after this file */
{ size_t const readSize = fread(((char*)buffer)+pos, 1, (size_t)fileSize, f);
if (readSize != (size_t)fileSize) EXM_THROW_INT(11, "could not read %s", fileNamesTable[n]);
if (readSize != (size_t)fileSize) RETURN_ERROR_INT(11, "could not read %s", fileNamesTable[n]);
pos += readSize;
}
fileSizes[n] = (size_t)fileSize;
totalSize += (size_t)fileSize;
fclose(f);
}
} }
if (totalSize == 0) EXM_THROW_INT(12, "no data to bench");
if (totalSize == 0) RETURN_ERROR_INT(12, "no data to bench");
return 0;
}

1
programs/benchzstd.h
View File

@ -205,7 +205,6 @@ BMK_benchOutcome_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize,
#endif /* BENCH_ZSTD_H_3242387 */
#if defined (__cplusplus)

354
programs/fileio.c
View File

@ -284,9 +284,10 @@ void FIO_addAbortHandler()
/*-*************************************
* Parameters: Typedefs
* Parameters: FIO_prefs_t
***************************************/
/* typedef'd to FIO_prefs_t within fileio.h */
struct FIO_prefs_s {
/* Algorithm preferences */
@ -308,6 +309,7 @@ struct FIO_prefs_s {
size_t streamSrcSize;
size_t targetCBlockSize;
int srcSizeHint;
int testMode;
ZSTD_literalCompressionMode_e literalCompressionMode;
/* IO preferences */
@ -355,6 +357,7 @@ FIO_prefs_t* FIO_createPreferences(void)
ret->streamSrcSize = 0;
ret->targetCBlockSize = 0;
ret->srcSizeHint = 0;
ret->testMode = 0;
ret->literalCompressionMode = ZSTD_lcm_auto;
return ret;
}
@ -435,6 +438,10 @@ void FIO_setSrcSizeHint(FIO_prefs_t* const prefs, size_t srcSizeHint) {
prefs->srcSizeHint = (int)MIN((size_t)INT_MAX, srcSizeHint);
}
void FIO_setTestMode(FIO_prefs_t* const prefs, int testMode) {
prefs->testMode = (testMode!=0);
}
void FIO_setLiteralCompressionMode(
FIO_prefs_t* const prefs,
ZSTD_literalCompressionMode_e mode) {
@ -529,8 +536,12 @@ static FILE* FIO_openSrcFile(const char* srcFileName)
/** FIO_openDstFile() :
* condition : `dstFileName` must be non-NULL.
* @result : FILE* to `dstFileName`, or NULL if it fails */
static FILE* FIO_openDstFile(FIO_prefs_t* const prefs, const char* srcFileName, const char* dstFileName)
static FILE*
FIO_openDstFile(FIO_prefs_t* const prefs,
const char* srcFileName, const char* dstFileName)
{
if (prefs->testMode) return NULL; /* do not open file in test mode */
assert(dstFileName != NULL);
if (!strcmp (dstFileName, stdoutmark)) {
DISPLAYLEVEL(4,"Using stdout for output \n");
@ -555,10 +566,14 @@ static FILE* FIO_openDstFile(FIO_prefs_t* const prefs, const char* srcFileName,
if (UTIL_isRegularFile(dstFileName)) {
/* Check if destination file already exists */
FILE* const fCheck = fopen( dstFileName, "rb" );
#if !defined(_WIN32)
/* this test does not work on Windows :
* `NUL` and `nul` are detected as regular files */
if (!strcmp(dstFileName, nulmark)) {
EXM_THROW(40, "%s is unexpectedly categorized as a regular file",
dstFileName);
}
#endif
if (fCheck != NULL) { /* dst file exists, authorization prompt */
fclose(fCheck);
if (!prefs->overwrite) {
@ -585,7 +600,7 @@ static FILE* FIO_openDstFile(FIO_prefs_t* const prefs, const char* srcFileName,
{ FILE* const f = fopen( dstFileName, "wb" );
if (f == NULL) {
DISPLAYLEVEL(1, "zstd: %s: %s\n", dstFileName, strerror(errno));
} else {
} else if(srcFileName != NULL && strcmp (srcFileName, stdinmark)) {
chmod(dstFileName, 00600);
}
return f;
@ -628,6 +643,96 @@ static size_t FIO_createDictBuffer(void** bufferPtr, const char* fileName)
return (size_t)fileSize;
}
/* FIO_checkFilenameCollisions() :
* Checks for and warns if there are any files that would have the same output path
*/
int FIO_checkFilenameCollisions(const char** filenameTable, unsigned nbFiles) {
const char **filenameTableSorted, *c, *prevElem, *filename;
unsigned u;
#if defined(_MSC_VER) || defined(__MINGW32__) || defined (__MSVCRT__) /* windows support */
c = "\\";
#else
c = "/";
#endif
filenameTableSorted = (const char**) malloc(sizeof(char*) * nbFiles);
if (!filenameTableSorted) {
DISPLAY("Unable to malloc new str array, not checking for name collisions\n");
return 1;
}
for (u = 0; u < nbFiles; ++u) {
filename = strrchr(filenameTable[u], c[0]);
if (filename == NULL) {
filenameTableSorted[u] = filenameTable[u];
} else {
filenameTableSorted[u] = filename+1;
}
}
qsort((void*)filenameTableSorted, nbFiles, sizeof(char*), UTIL_compareStr);
prevElem = filenameTableSorted[0];
for (u = 1; u < nbFiles; ++u) {
if (strcmp(prevElem, filenameTableSorted[u]) == 0) {
DISPLAY("WARNING: Two files have same filename: %s\n", prevElem);
}
prevElem = filenameTableSorted[u];
}
free((void*)filenameTableSorted);
return 0;
}
static const char*
extractFilename(const char* path, char separator)
{
const char* search = strrchr(path, separator);
if (search == NULL) return path;
return search+1;
}
/* FIO_createFilename_fromOutDir() :
* Takes a source file name and specified output directory, and
* allocates memory for and returns a pointer to final path.
* This function never returns an error (it may abort() in case of pb)
*/
static char*
FIO_createFilename_fromOutDir(const char* path, const char* outDirName, const size_t suffixLen)
{
const char* filenameStart;
char separator;
char* result;
#if defined(_MSC_VER) || defined(__MINGW32__) || defined (__MSVCRT__) /* windows support */
separator = '\\';
#else
separator = '/';
#endif
filenameStart = extractFilename(path, separator);
#if defined(_MSC_VER) || defined(__MINGW32__) || defined (__MSVCRT__) /* windows support */
filenameStart = extractFilename(filenameStart, '/'); /* sometimes, '/' separator is also used on Windows (mingw+msys2) */
#endif
result = (char*) calloc(1, strlen(outDirName) + 1 + strlen(filenameStart) + suffixLen + 1);
if (!result) {
EXM_THROW(30, "zstd: FIO_createFilename_fromOutDir: %s", strerror(errno));
}
memcpy(result, outDirName, strlen(outDirName));
if (outDirName[strlen(outDirName)-1] == separator) {
memcpy(result + strlen(outDirName), filenameStart, strlen(filenameStart));
} else {
memcpy(result + strlen(outDirName), &separator, 1);
memcpy(result + strlen(outDirName) + 1, filenameStart, strlen(filenameStart));
}
return result;
}
#ifndef ZSTD_NOCOMPRESS
/* **********************************************************************
@ -729,13 +834,12 @@ static void FIO_freeCResources(cRess_t ress)
#ifdef ZSTD_GZCOMPRESS
static unsigned long long
FIO_compressGzFrame(cRess_t* ress,
FIO_compressGzFrame(const cRess_t* ress, /* buffers & handlers are used, but not changed */
const char* srcFileName, U64 const srcFileSize,
int compressionLevel, U64* readsize)
{
unsigned long long inFileSize = 0, outFileSize = 0;
z_stream strm;
int ret;
if (compressionLevel > Z_BEST_COMPRESSION)
compressionLevel = Z_BEST_COMPRESSION;
@ -744,11 +848,12 @@ FIO_compressGzFrame(cRess_t* ress,
strm.zfree = Z_NULL;
strm.opaque = Z_NULL;
ret = deflateInit2(&strm, compressionLevel, Z_DEFLATED,
{ int const ret = deflateInit2(&strm, compressionLevel, Z_DEFLATED,
15 /* maxWindowLogSize */ + 16 /* gzip only */,
8, Z_DEFAULT_STRATEGY); /* see http://www.zlib.net/manual.html */
if (ret != Z_OK)
if (ret != Z_OK) {
EXM_THROW(71, "zstd: %s: deflateInit2 error %d \n", srcFileName, ret);
} }
strm.next_in = 0;
strm.avail_in = 0;
@ -756,6 +861,7 @@ FIO_compressGzFrame(cRess_t* ress,
strm.avail_out = (uInt)ress->dstBufferSize;
while (1) {
int ret;
if (strm.avail_in == 0) {
size_t const inSize = fread(ress->srcBuffer, 1, ress->srcBufferSize, ress->srcFile);
if (inSize == 0) break;
@ -766,32 +872,31 @@ FIO_compressGzFrame(cRess_t* ress,
ret = deflate(&strm, Z_NO_FLUSH);
if (ret != Z_OK)
EXM_THROW(72, "zstd: %s: deflate error %d \n", srcFileName, ret);
{ size_t const decompBytes = ress->dstBufferSize - strm.avail_out;
if (decompBytes) {
if (fwrite(ress->dstBuffer, 1, decompBytes, ress->dstFile) != decompBytes)
EXM_THROW(73, "Write error : cannot write to output file");
outFileSize += decompBytes;
{ size_t const cSize = ress->dstBufferSize - strm.avail_out;
if (cSize) {
if (fwrite(ress->dstBuffer, 1, cSize, ress->dstFile) != cSize)
EXM_THROW(73, "Write error : cannot write to output file : %s ", strerror(errno));
outFileSize += cSize;
strm.next_out = (Bytef*)ress->dstBuffer;
strm.avail_out = (uInt)ress->dstBufferSize;
}
}
if (srcFileSize == UTIL_FILESIZE_UNKNOWN)
DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%%",
} }
if (srcFileSize == UTIL_FILESIZE_UNKNOWN) {
DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%% ",
(unsigned)(inFileSize>>20),
(double)outFileSize/inFileSize*100)
else
DISPLAYUPDATE(2, "\rRead : %u / %u MB ==> %.2f%%",
} else {
DISPLAYUPDATE(2, "\rRead : %u / %u MB ==> %.2f%% ",
(unsigned)(inFileSize>>20), (unsigned)(srcFileSize>>20),
(double)outFileSize/inFileSize*100);
}
} }
while (1) {
ret = deflate(&strm, Z_FINISH);
{ size_t const decompBytes = ress->dstBufferSize - strm.avail_out;
if (decompBytes) {
if (fwrite(ress->dstBuffer, 1, decompBytes, ress->dstFile) != decompBytes)
EXM_THROW(75, "Write error : %s", strerror(errno));
outFileSize += decompBytes;
int const ret = deflate(&strm, Z_FINISH);
{ size_t const cSize = ress->dstBufferSize - strm.avail_out;
if (cSize) {
if (fwrite(ress->dstBuffer, 1, cSize, ress->dstFile) != cSize)
EXM_THROW(75, "Write error : %s ", strerror(errno));
outFileSize += cSize;
strm.next_out = (Bytef*)ress->dstBuffer;
strm.avail_out = (uInt)ress->dstBufferSize;
} }
@ -800,11 +905,11 @@ FIO_compressGzFrame(cRess_t* ress,
EXM_THROW(77, "zstd: %s: deflate error %d \n", srcFileName, ret);
}
ret = deflateEnd(&strm);
if (ret != Z_OK)
{ int const ret = deflateEnd(&strm);
if (ret != Z_OK) {
EXM_THROW(79, "zstd: %s: deflateEnd error %d \n", srcFileName, ret);
} }
*readsize = inFileSize;
return outFileSize;
}
#endif
@ -827,14 +932,14 @@ FIO_compressLzmaFrame(cRess_t* ress,
if (plain_lzma) {
lzma_options_lzma opt_lzma;
if (lzma_lzma_preset(&opt_lzma, compressionLevel))
EXM_THROW(71, "zstd: %s: lzma_lzma_preset error", srcFileName);
EXM_THROW(81, "zstd: %s: lzma_lzma_preset error", srcFileName);
ret = lzma_alone_encoder(&strm, &opt_lzma); /* LZMA */
if (ret != LZMA_OK)
EXM_THROW(71, "zstd: %s: lzma_alone_encoder error %d", srcFileName, ret);
EXM_THROW(82, "zstd: %s: lzma_alone_encoder error %d", srcFileName, ret);
} else {
ret = lzma_easy_encoder(&strm, compressionLevel, LZMA_CHECK_CRC64); /* XZ */
if (ret != LZMA_OK)
EXM_THROW(71, "zstd: %s: lzma_easy_encoder error %d", srcFileName, ret);
EXM_THROW(83, "zstd: %s: lzma_easy_encoder error %d", srcFileName, ret);
}
strm.next_in = 0;
@ -854,11 +959,11 @@ FIO_compressLzmaFrame(cRess_t* ress,
ret = lzma_code(&strm, action);
if (ret != LZMA_OK && ret != LZMA_STREAM_END)
EXM_THROW(72, "zstd: %s: lzma_code encoding error %d", srcFileName, ret);
EXM_THROW(84, "zstd: %s: lzma_code encoding error %d", srcFileName, ret);
{ size_t const compBytes = ress->dstBufferSize - strm.avail_out;
if (compBytes) {
if (fwrite(ress->dstBuffer, 1, compBytes, ress->dstFile) != compBytes)
EXM_THROW(73, "Write error : %s", strerror(errno));
EXM_THROW(85, "Write error : %s", strerror(errno));
outFileSize += compBytes;
strm.next_out = (BYTE*)ress->dstBuffer;
strm.avail_out = ress->dstBufferSize;
@ -1276,9 +1381,7 @@ static int FIO_compressFilename_dstFile(FIO_prefs_t* const prefs,
int result;
stat_t statbuf;
int transfer_permissions = 0;
assert(ress.srcFile != NULL);
if (ress.dstFile == NULL) {
closeDstFile = 1;
DISPLAYLEVEL(6, "FIO_compressFilename_dstFile: opening dst: %s", dstFileName);
@ -1369,11 +1472,9 @@ FIO_compressFilename_srcFile(FIO_prefs_t* const prefs,
return result;
}
int FIO_compressFilename(FIO_prefs_t* const prefs,
const char* dstFileName, const char* srcFileName,
const char* dictFileName, int compressionLevel,
ZSTD_compressionParameters comprParams)
int FIO_compressFilename(FIO_prefs_t* const prefs, const char* dstFileName,
const char* srcFileName, const char* dictFileName,
int compressionLevel, ZSTD_compressionParameters comprParams)
{
cRess_t const ress = FIO_createCResources(prefs, dictFileName, compressionLevel, comprParams);
int const result = FIO_compressFilename_srcFile(prefs, ress, dstFileName, srcFileName, compressionLevel);
@ -1383,20 +1484,24 @@ int FIO_compressFilename(FIO_prefs_t* const prefs,
return result;
}
/* FIO_determineCompressedName() :
* create a destination filename for compressed srcFileName.
* @return a pointer to it.
* This function never returns an error (it may abort() in case of pb)
*/
static const char*
FIO_determineCompressedName(const char* srcFileName, const char* suffix)
FIO_determineCompressedName(const char* srcFileName, const char* outDirName, const char* suffix)
{
static size_t dfnbCapacity = 0;
static char* dstFileNameBuffer = NULL; /* using static allocation : this function cannot be multi-threaded */
size_t const sfnSize = strlen(srcFileName);
char* outDirFilename = NULL;
size_t sfnSize = strlen(srcFileName);
size_t const suffixSize = strlen(suffix);
if (outDirName) {
outDirFilename = FIO_createFilename_fromOutDir(srcFileName, outDirName, suffixSize);
sfnSize = strlen(outDirFilename);
assert(outDirFilename != NULL);
}
if (dfnbCapacity <= sfnSize+suffixSize+1) {
/* resize buffer for dstName */
@ -1405,22 +1510,30 @@ FIO_determineCompressedName(const char* srcFileName, const char* suffix)
dstFileNameBuffer = (char*)malloc(dfnbCapacity);
if (!dstFileNameBuffer) {
EXM_THROW(30, "zstd: %s", strerror(errno));
} }
}
}
assert(dstFileNameBuffer != NULL);
memcpy(dstFileNameBuffer, srcFileName, sfnSize);
memcpy(dstFileNameBuffer+sfnSize, suffix, suffixSize+1 /* Include terminating null */);
if (outDirFilename) {
memcpy(dstFileNameBuffer, outDirFilename, sfnSize);
free(outDirFilename);
} else {
memcpy(dstFileNameBuffer, srcFileName, sfnSize);
}
memcpy(dstFileNameBuffer+sfnSize, suffix, suffixSize+1 /* Include terminating null */);
return dstFileNameBuffer;
}
/* FIO_compressMultipleFilenames() :
* compress nbFiles files
* into one destination (outFileName)
* or into one file each (outFileName == NULL, but suffix != NULL).
* into either one destination (outFileName),
* or into one file each (outFileName == NULL, but suffix != NULL),
* or into a destination folder (specified with -O)
*/
int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs,
const char** inFileNamesTable, unsigned nbFiles,
const char* outDirName,
const char* outFileName, const char* suffix,
const char* dictFileName, int compressionLevel,
ZSTD_compressionParameters comprParams)
@ -1430,7 +1543,6 @@ int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs,
/* init */
assert(outFileName != NULL || suffix != NULL);
if (outFileName != NULL) { /* output into a single destination (stdout typically) */
ress.dstFile = FIO_openDstFile(prefs, NULL, outFileName);
if (ress.dstFile == NULL) { /* could not open outFileName */
@ -1448,9 +1560,12 @@ int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs,
unsigned u;
for (u=0; u<nbFiles; u++) {
const char* const srcFileName = inFileNamesTable[u];
const char* const dstFileName = FIO_determineCompressedName(srcFileName, suffix); /* cannot fail */
const char* const dstFileName = FIO_determineCompressedName(srcFileName, outDirName, suffix); /* cannot fail */
error |= FIO_compressFilename_srcFile(prefs, ress, dstFileName, srcFileName, compressionLevel);
} }
}
if (outDirName)
FIO_checkFilenameCollisions(inFileNamesTable ,nbFiles);
}
FIO_freeCResources(ress);
return error;
@ -1512,7 +1627,11 @@ static void FIO_freeDResources(dRess_t ress)
/** FIO_fwriteSparse() :
* @return : storedSkips, to be provided to next call to FIO_fwriteSparse() of LZ4IO_fwriteSparseEnd() */
static unsigned FIO_fwriteSparse(FIO_prefs_t* const prefs, FILE* file, const void* buffer, size_t bufferSize, unsigned storedSkips)
static unsigned
FIO_fwriteSparse(const FIO_prefs_t* const prefs,
FILE* file,
const void* buffer, size_t bufferSize,
unsigned storedSkips)
{
const size_t* const bufferT = (const size_t*)buffer; /* Buffer is supposed malloc'ed, hence aligned on size_t */
size_t bufferSizeT = bufferSize / sizeof(size_t);
@ -1520,10 +1639,12 @@ static unsigned FIO_fwriteSparse(FIO_prefs_t* const prefs, FILE* file, const voi
const size_t* ptrT = bufferT;
static const size_t segmentSizeT = (32 KB) / sizeof(size_t); /* 0-test re-attempted every 32 KB */
if (prefs->testMode) return 0; /* do not output anything in test mode */
if (!prefs->sparseFileSupport) { /* normal write */
size_t const sizeCheck = fwrite(buffer, 1, bufferSize, file);
if (sizeCheck != bufferSize)
EXM_THROW(70, "Write error : %s (cannot write decoded block)",
EXM_THROW(70, "Write error : cannot write decoded block : %s",
strerror(errno));
return 0;
}
@ -1532,7 +1653,7 @@ static unsigned FIO_fwriteSparse(FIO_prefs_t* const prefs, FILE* file, const voi
if (storedSkips > 1 GB) {
int const seekResult = LONG_SEEK(file, 1 GB, SEEK_CUR);
if (seekResult != 0)
EXM_THROW(71, "1 GB skip error (sparse file support)");
EXM_THROW(91, "1 GB skip error (sparse file support)");
storedSkips -= 1 GB;
}
@ -1548,13 +1669,14 @@ static unsigned FIO_fwriteSparse(FIO_prefs_t* const prefs, FILE* file, const voi
if (nb0T != seg0SizeT) { /* not all 0s */
int const seekResult = LONG_SEEK(file, storedSkips, SEEK_CUR);
if (seekResult) EXM_THROW(72, "Sparse skip error ; try --no-sparse");
if (seekResult) EXM_THROW(92, "Sparse skip error ; try --no-sparse");
storedSkips = 0;
seg0SizeT -= nb0T;
ptrT += nb0T;
{ size_t const sizeCheck = fwrite(ptrT, sizeof(size_t), seg0SizeT, file);
if (sizeCheck != seg0SizeT)
EXM_THROW(73, "Write error : cannot write decoded block");
EXM_THROW(93, "Write error : cannot write decoded block : %s",
strerror(errno));
} }
ptrT += seg0SizeT;
}
@ -1571,19 +1693,21 @@ static unsigned FIO_fwriteSparse(FIO_prefs_t* const prefs, FILE* file, const voi
if (restPtr != restEnd) {
int seekResult = LONG_SEEK(file, storedSkips, SEEK_CUR);
if (seekResult)
EXM_THROW(74, "Sparse skip error ; try --no-sparse");
EXM_THROW(94, "Sparse skip error ; try --no-sparse");
storedSkips = 0;
{ size_t const sizeCheck = fwrite(restPtr, 1, (size_t)(restEnd - restPtr), file);
if (sizeCheck != (size_t)(restEnd - restPtr))
EXM_THROW(75, "Write error : cannot write decoded end of block");
EXM_THROW(95, "Write error : cannot write decoded end of block : %s",
strerror(errno));
} } } }
return storedSkips;
}
static void
FIO_fwriteSparseEnd(FIO_prefs_t* const prefs, FILE* file, unsigned storedSkips)
FIO_fwriteSparseEnd(const FIO_prefs_t* const prefs, FILE* file, unsigned storedSkips)
{
if (prefs->testMode) assert(storedSkips == 0);
if (storedSkips>0) {
assert(prefs->sparseFileSupport > 0); /* storedSkips>0 implies sparse support is enabled */
(void)prefs; /* assert can be disabled, in which case prefs becomes unused */
@ -1593,14 +1717,14 @@ FIO_fwriteSparseEnd(FIO_prefs_t* const prefs, FILE* file, unsigned storedSkips)
* so that skipped ones get implicitly translated as zero by FS */
{ const char lastZeroByte[1] = { 0 };
if (fwrite(lastZeroByte, 1, 1, file) != 1)
EXM_THROW(69, "Write error : cannot write last zero");
EXM_THROW(69, "Write error : cannot write last zero : %s", strerror(errno));
} }
}
/** FIO_passThrough() : just copy input into output, for compatibility with gzip -df mode
@return : 0 (no error) */
static int FIO_passThrough(FIO_prefs_t* const prefs,
static int FIO_passThrough(const FIO_prefs_t* const prefs,
FILE* foutput, FILE* finput,
void* buffer, size_t bufferSize,
size_t alreadyLoaded)
@ -1612,7 +1736,7 @@ static int FIO_passThrough(FIO_prefs_t* const prefs,
/* assumption : ress->srcBufferLoaded bytes already loaded and stored within buffer */
{ size_t const sizeCheck = fwrite(buffer, 1, alreadyLoaded, foutput);
if (sizeCheck != alreadyLoaded) {
DISPLAYLEVEL(1, "Pass-through write error \n");
DISPLAYLEVEL(1, "Pass-through write error : %s\n", strerror(errno));
return 1;
} }
@ -1640,7 +1764,10 @@ static unsigned FIO_highbit64(unsigned long long v)
/* FIO_zstdErrorHelp() :
* detailed error message when requested window size is too large */
static void FIO_zstdErrorHelp(FIO_prefs_t* const prefs, dRess_t* ress, size_t err, char const* srcFileName)
static void
FIO_zstdErrorHelp(const FIO_prefs_t* const prefs,
const dRess_t* ress,
size_t err, const char* srcFileName)
{
ZSTD_frameHeader header;
@ -1672,12 +1799,10 @@ static void FIO_zstdErrorHelp(FIO_prefs_t* const prefs, dRess_t* ress, size_t er
* @return : size of decoded zstd frame, or an error code
*/
#define FIO_ERROR_FRAME_DECODING ((unsigned long long)(-2))
static unsigned long long FIO_decompressZstdFrame(
FIO_prefs_t* const prefs,
dRess_t* ress,
FILE* finput,
const char* srcFileName,
U64 alreadyDecoded)
static unsigned long long
FIO_decompressZstdFrame(const FIO_prefs_t* const prefs,
dRess_t* ress, FILE* finput,
const char* srcFileName, U64 alreadyDecoded)
{
U64 frameSize = 0;
U32 storedSkips = 0;
@ -1741,13 +1866,16 @@ static unsigned long long FIO_decompressZstdFrame(
#ifdef ZSTD_GZDECOMPRESS
static unsigned long long FIO_decompressGzFrame(dRess_t* ress,
FILE* srcFile, const char* srcFileName)
static unsigned long long
FIO_decompressGzFrame(const FIO_prefs_t* const prefs,
dRess_t* ress, FILE* srcFile,
const char* srcFileName)
{
unsigned long long outFileSize = 0;
z_stream strm;
int flush = Z_NO_FLUSH;
int decodingError = 0;
unsigned storedSkips = 0;
strm.zalloc = Z_NULL;
strm.zfree = Z_NULL;
@ -1782,10 +1910,7 @@ static unsigned long long FIO_decompressGzFrame(dRess_t* ress,
}
{ size_t const decompBytes = ress->dstBufferSize - strm.avail_out;
if (decompBytes) {
if (fwrite(ress->dstBuffer, 1, decompBytes, ress->dstFile) != decompBytes) {
DISPLAYLEVEL(1, "zstd: %s \n", strerror(errno));
decodingError = 1; break;
}
storedSkips = FIO_fwriteSparse(prefs, ress->dstFile, ress->dstBuffer, decompBytes, storedSkips);
outFileSize += decompBytes;
strm.next_out = (Bytef*)ress->dstBuffer;
strm.avail_out = (uInt)ress->dstBufferSize;
@ -1802,19 +1927,24 @@ static unsigned long long FIO_decompressGzFrame(dRess_t* ress,
DISPLAYLEVEL(1, "zstd: %s: inflateEnd error \n", srcFileName);
decodingError = 1;
}
FIO_fwriteSparseEnd(prefs, ress->dstFile, storedSkips);
return decodingError ? FIO_ERROR_FRAME_DECODING : outFileSize;
}
#endif
#ifdef ZSTD_LZMADECOMPRESS
static unsigned long long FIO_decompressLzmaFrame(dRess_t* ress, FILE* srcFile, const char* srcFileName, int plain_lzma)
static unsigned long long
FIO_decompressLzmaFrame(const FIO_prefs_t* const prefs,
dRess_t* ress, FILE* srcFile,
const char* srcFileName, int plain_lzma)
{
unsigned long long outFileSize = 0;
lzma_stream strm = LZMA_STREAM_INIT;
lzma_action action = LZMA_RUN;
lzma_ret initRet;
int decodingError = 0;
unsigned storedSkips = 0;
strm.next_in = 0;
strm.avail_in = 0;
@ -1857,10 +1987,7 @@ static unsigned long long FIO_decompressLzmaFrame(dRess_t* ress, FILE* srcFile,
}
{ size_t const decompBytes = ress->dstBufferSize - strm.avail_out;
if (decompBytes) {
if (fwrite(ress->dstBuffer, 1, decompBytes, ress->dstFile) != decompBytes) {
DISPLAYLEVEL(1, "zstd: %s \n", strerror(errno));
decodingError = 1; break;
}
storedSkips = FIO_fwriteSparse(prefs, ress->dstFile, ress->dstBuffer, decompBytes, storedSkips);
outFileSize += decompBytes;
strm.next_out = (BYTE*)ress->dstBuffer;
strm.avail_out = ress->dstBufferSize;
@ -1872,19 +1999,23 @@ static unsigned long long FIO_decompressLzmaFrame(dRess_t* ress, FILE* srcFile,
memmove(ress->srcBuffer, strm.next_in, strm.avail_in);
ress->srcBufferLoaded = strm.avail_in;
lzma_end(&strm);
FIO_fwriteSparseEnd(prefs, ress->dstFile, storedSkips);
return decodingError ? FIO_ERROR_FRAME_DECODING : outFileSize;
}
#endif
#ifdef ZSTD_LZ4DECOMPRESS
static unsigned long long FIO_decompressLz4Frame(dRess_t* ress,
FILE* srcFile, const char* srcFileName)
static unsigned long long
FIO_decompressLz4Frame(const FIO_prefs_t* const prefs,
dRess_t* ress, FILE* srcFile,
const char* srcFileName)
{
unsigned long long filesize = 0;
LZ4F_errorCode_t nextToLoad;
LZ4F_decompressionContext_t dCtx;
LZ4F_errorCode_t const errorCode = LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION);
int decodingError = 0;
unsigned storedSkips = 0;
if (LZ4F_isError(errorCode)) {
DISPLAYLEVEL(1, "zstd: failed to create lz4 decompression context \n");
@ -1928,10 +2059,7 @@ static unsigned long long FIO_decompressLz4Frame(dRess_t* ress,
/* Write Block */
if (decodedBytes) {
if (fwrite(ress->dstBuffer, 1, decodedBytes, ress->dstFile) != decodedBytes) {
DISPLAYLEVEL(1, "zstd: %s \n", strerror(errno));
decodingError = 1; nextToLoad = 0; break;
}
storedSkips = FIO_fwriteSparse(prefs, ress->dstFile, ress->dstBuffer, decodedBytes, storedSkips);
filesize += decodedBytes;
DISPLAYUPDATE(2, "\rDecompressed : %u MB ", (unsigned)(filesize>>20));
}
@ -1952,6 +2080,7 @@ static unsigned long long FIO_decompressLz4Frame(dRess_t* ress,
LZ4F_freeDecompressionContext(dCtx);
ress->srcBufferLoaded = 0; /* LZ4F will reach exact frame boundary */
FIO_fwriteSparseEnd(prefs, ress->dstFile, storedSkips);
return decodingError ? FIO_ERROR_FRAME_DECODING : filesize;
}
@ -1965,7 +2094,8 @@ static unsigned long long FIO_decompressLz4Frame(dRess_t* ress,
* @return : 0 : OK
* 1 : error
*/
static int FIO_decompressFrames(FIO_prefs_t* const prefs, dRess_t ress, FILE* srcFile,
static int FIO_decompressFrames(const FIO_prefs_t* const prefs,
dRess_t ress, FILE* srcFile,
const char* dstFileName, const char* srcFileName)
{
unsigned readSomething = 0;
@ -1998,7 +2128,7 @@ static int FIO_decompressFrames(FIO_prefs_t* const prefs, dRess_t ress, FILE* sr
filesize += frameSize;
} else if (buf[0] == 31 && buf[1] == 139) { /* gz magic number */
#ifdef ZSTD_GZDECOMPRESS
unsigned long long const frameSize = FIO_decompressGzFrame(&ress, srcFile, srcFileName);
unsigned long long const frameSize = FIO_decompressGzFrame(prefs, &ress, srcFile, srcFileName);
if (frameSize == FIO_ERROR_FRAME_DECODING) return 1;
filesize += frameSize;
#else
@ -2008,7 +2138,7 @@ static int FIO_decompressFrames(FIO_prefs_t* const prefs, dRess_t ress, FILE* sr
} else if ((buf[0] == 0xFD && buf[1] == 0x37) /* xz magic number */
|| (buf[0] == 0x5D && buf[1] == 0x00)) { /* lzma header (no magic number) */
#ifdef ZSTD_LZMADECOMPRESS
unsigned long long const frameSize = FIO_decompressLzmaFrame(&ress, srcFile, srcFileName, buf[0] != 0xFD);
unsigned long long const frameSize = FIO_decompressLzmaFrame(prefs, &ress, srcFile, srcFileName, buf[0] != 0xFD);
if (frameSize == FIO_ERROR_FRAME_DECODING) return 1;
filesize += frameSize;
#else
@ -2017,7 +2147,7 @@ static int FIO_decompressFrames(FIO_prefs_t* const prefs, dRess_t ress, FILE* sr
#endif
} else if (MEM_readLE32(buf) == LZ4_MAGICNUMBER) {
#ifdef ZSTD_LZ4DECOMPRESS
unsigned long long const frameSize = FIO_decompressLz4Frame(&ress, srcFile, srcFileName);
unsigned long long const frameSize = FIO_decompressLz4Frame(prefs, &ress, srcFile, srcFileName);
if (frameSize == FIO_ERROR_FRAME_DECODING) return 1;
filesize += frameSize;
#else
@ -2057,11 +2187,11 @@ static int FIO_decompressDstFile(FIO_prefs_t* const prefs,
int transfer_permissions = 0;
int releaseDstFile = 0;
if (ress.dstFile == NULL) {
if ((ress.dstFile == NULL) && (prefs->testMode==0)) {
releaseDstFile = 1;
ress.dstFile = FIO_openDstFile(prefs, srcFileName, dstFileName);
if (ress.dstFile==0) return 1;
if (ress.dstFile==NULL) return 1;
/* Must only be added after FIO_openDstFile() succeeds.
* Otherwise we may delete the destination file if it already exists,
@ -2074,7 +2204,6 @@ static int FIO_decompressDstFile(FIO_prefs_t* const prefs,
transfer_permissions = 1;
}
result = FIO_decompressFrames(prefs, ress, srcFile, dstFileName, srcFileName);
if (releaseDstFile) {
@ -2164,13 +2293,14 @@ int FIO_decompressFilename(FIO_prefs_t* const prefs,
* @return a pointer to it.
* @return == NULL if there is an error */
static const char*
FIO_determineDstName(const char* srcFileName)
FIO_determineDstName(const char* srcFileName, const char* outDirName)
{
static size_t dfnbCapacity = 0;
static char* dstFileNameBuffer = NULL; /* using static allocation : this function cannot be multi-threaded */
size_t const sfnSize = strlen(srcFileName);
char* outDirFilename = NULL;
size_t sfnSize = strlen(srcFileName);
size_t suffixSize;
const char* const suffixPtr = strrchr(srcFileName, '.');
if (suffixPtr == NULL) {
DISPLAYLEVEL(1, "zstd: %s: unknown suffix -- ignored \n",
@ -2208,9 +2338,14 @@ FIO_determineDstName(const char* srcFileName)
srcFileName, suffixlist);
return NULL;
}
if (outDirName) {
outDirFilename = FIO_createFilename_fromOutDir(srcFileName, outDirName, 0);
sfnSize = strlen(outDirFilename);
assert(outDirFilename != NULL);
}
/* allocate enough space to write dstFilename into it */
if (dfnbCapacity+suffixSize <= sfnSize+1) {
/* allocate enough space to write dstFilename into it */
free(dstFileNameBuffer);
dfnbCapacity = sfnSize + 20;
dstFileNameBuffer = (char*)malloc(dfnbCapacity);
@ -2220,7 +2355,12 @@ FIO_determineDstName(const char* srcFileName)
/* return dst name == src name truncated from suffix */
assert(dstFileNameBuffer != NULL);
if (outDirFilename) {
memcpy(dstFileNameBuffer, outDirFilename, sfnSize - suffixSize);
free(outDirFilename);
} else {
memcpy(dstFileNameBuffer, srcFileName, sfnSize - suffixSize);
}
dstFileNameBuffer[sfnSize-suffixSize] = '\0';
return dstFileNameBuffer;
@ -2230,8 +2370,8 @@ FIO_determineDstName(const char* srcFileName)
int
FIO_decompressMultipleFilenames(FIO_prefs_t* const prefs,
const char* srcNamesTable[], unsigned nbFiles,
const char* outFileName,
const char** srcNamesTable, unsigned nbFiles,
const char* outDirName, const char* outFileName,
const char* dictFileName)
{
int error = 0;
@ -2239,30 +2379,32 @@ FIO_decompressMultipleFilenames(FIO_prefs_t* const prefs,
if (outFileName) {
unsigned u;
if (!prefs->testMode) {
ress.dstFile = FIO_openDstFile(prefs, NULL, outFileName);
if (ress.dstFile == 0) EXM_THROW(71, "cannot open %s", outFileName);
if (ress.dstFile == 0) EXM_THROW(19, "cannot open %s", outFileName);
}
for (u=0; u<nbFiles; u++)
error |= FIO_decompressSrcFile(prefs, ress, outFileName, srcNamesTable[u]);
if (fclose(ress.dstFile))
if ((!prefs->testMode) && (fclose(ress.dstFile)))
EXM_THROW(72, "Write error : %s : cannot properly close output file",
strerror(errno));
} else {
unsigned u;
for (u=0; u<nbFiles; u++) { /* create dstFileName */
const char* const srcFileName = srcNamesTable[u];
const char* const dstFileName = FIO_determineDstName(srcFileName);
const char* const dstFileName = FIO_determineDstName(srcFileName, outDirName);
if (dstFileName == NULL) { error=1; continue; }
error |= FIO_decompressSrcFile(prefs, ress, dstFileName, srcFileName);
}
if (outDirName)
FIO_checkFilenameCollisions(srcNamesTable ,nbFiles);
}
FIO_freeDResources(ress);
return error;
}
/* **************************************************************************
* .zst file info (--list command)
***************************************************************************/
@ -2301,7 +2443,7 @@ FIO_analyzeFrames(fileInfo_t* info, FILE* const srcFile)
for ( ; ; ) {
BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX];
size_t const numBytesRead = fread(headerBuffer, 1, sizeof(headerBuffer), srcFile);
if (numBytesRead < ZSTD_FRAMEHEADERSIZE_MIN) {
if (numBytesRead < ZSTD_FRAMEHEADERSIZE_MIN(ZSTD_f_zstd1)) {
if ( feof(srcFile)
&& (numBytesRead == 0)
&& (info->compressedSize > 0)

26
programs/fileio.h
View File

@ -26,7 +26,7 @@ extern "C" {
#define stdinmark "/*stdin*\\"
#define stdoutmark "/*stdout*\\"
#ifdef _WIN32
# define nulmark "nul"
# define nulmark "NUL"
#else
# define nulmark "/dev/null"
#endif
@ -74,6 +74,7 @@ void FIO_setRsyncable(FIO_prefs_t* const prefs, int rsyncable);
void FIO_setStreamSrcSize(FIO_prefs_t* const prefs, size_t streamSrcSize);
void FIO_setTargetCBlockSize(FIO_prefs_t* const prefs, size_t targetCBlockSize);
void FIO_setSrcSizeHint(FIO_prefs_t* const prefs, size_t srcSizeHint);
void FIO_setTestMode(FIO_prefs_t* const prefs, int testMode);
void FIO_setLiteralCompressionMode(
FIO_prefs_t* const prefs,
ZSTD_literalCompressionMode_e mode);
@ -85,13 +86,14 @@ void FIO_setNotificationLevel(int level);
* Single File functions
***************************************/
/** FIO_compressFilename() :
@return : 0 == ok; 1 == pb with src file. */
* @return : 0 == ok; 1 == pb with src file. */
int FIO_compressFilename (FIO_prefs_t* const prefs,
const char* outfilename, const char* infilename, const char* dictFileName,
int compressionLevel, ZSTD_compressionParameters comprParams);
const char* outfilename, const char* infilename,
const char* dictFileName, int compressionLevel,
ZSTD_compressionParameters comprParams);
/** FIO_decompressFilename() :
@return : 0 == ok; 1 == pb with src file. */
* @return : 0 == ok; 1 == pb with src file. */
int FIO_decompressFilename (FIO_prefs_t* const prefs,
const char* outfilename, const char* infilename, const char* dictFileName);
@ -102,20 +104,28 @@ int FIO_listMultipleFiles(unsigned numFiles, const char** filenameTable, int dis
* Multiple File functions
***************************************/
/** FIO_compressMultipleFilenames() :
@return : nb of missing files */
* @return : nb of missing files */
int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs,
const char** srcNamesTable, unsigned nbFiles,
const char** inFileNamesTable, unsigned nbFiles,
const char* outDirName,
const char* outFileName, const char* suffix,
const char* dictFileName, int compressionLevel,
ZSTD_compressionParameters comprParams);
/** FIO_decompressMultipleFilenames() :
@return : nb of missing or skipped files */
* @return : nb of missing or skipped files */
int FIO_decompressMultipleFilenames(FIO_prefs_t* const prefs,
const char** srcNamesTable, unsigned nbFiles,
const char* outDirName,
const char* outFileName,
const char* dictFileName);
/* FIO_checkFilenameCollisions() :
* Checks for and warns if there are any files that would have the same output path
*/
int FIO_checkFilenameCollisions(const char** filenameTable, unsigned nbFiles);
/*-*************************************
* Advanced stuff (should actually be hosted elsewhere)

58
programs/util.c
View File

@ -20,6 +20,9 @@ extern "C" {
#include <errno.h>
#include <assert.h>
#if defined(_MSC_VER) || defined(__MINGW32__) || defined (__MSVCRT__)
#include <direct.h> /* needed for _mkdir in windows */
#endif
int UTIL_fileExist(const char* filename)
{
@ -69,7 +72,8 @@ int UTIL_setFileStat(const char *filename, stat_t *statbuf)
#else
{
/* (atime, mtime) */
struct timespec timebuf[2] = { {0, UTIME_NOW}, statbuf->st_mtim };
struct timespec timebuf[2] = { {0, UTIME_NOW} };
timebuf[1] = statbuf->st_mtim;
res += utimensat(AT_FDCWD, filename, timebuf, 0);
}
#endif
@ -98,20 +102,27 @@ U32 UTIL_isDirectory(const char* infilename)
return 0;
}
int UTIL_isSameFile(const char* file1, const char* file2)
int UTIL_compareStr(const void *p1, const void *p2) {
return strcmp(* (char * const *) p1, * (char * const *) p2);
}
int UTIL_isSameFile(const char* fName1, const char* fName2)
{
#if defined(_MSC_VER)
assert(fName1 != NULL); assert(fName2 != NULL);
#if defined(_MSC_VER) || defined(_WIN32)
/* note : Visual does not support file identification by inode.
* inode does not work on Windows, even with a posix layer, like msys2.
* The following work-around is limited to detecting exact name repetition only,
* aka `filename` is considered different from `subdir/../filename` */
return !strcmp(file1, file2);
return !strcmp(fName1, fName2);
#else
stat_t file1Stat;
{ stat_t file1Stat;
stat_t file2Stat;
return UTIL_getFileStat(file1, &file1Stat)
&& UTIL_getFileStat(file2, &file2Stat)
return UTIL_getFileStat(fName1, &file1Stat)
&& UTIL_getFileStat(fName2, &file2Stat)
&& (file1Stat.st_dev == file2Stat.st_dev)
&& (file1Stat.st_ino == file2Stat.st_ino);
}
#endif
}
@ -461,19 +472,20 @@ int UTIL_prepareFileList(const char *dirName, char** bufStart, size_t* pos, char
DIR *dir;
struct dirent *entry;
char* path;
int dirLength, fnameLength, pathLength, nbFiles = 0;
size_t dirLength, fnameLength, pathLength;
int nbFiles = 0;
if (!(dir = opendir(dirName))) {
UTIL_DISPLAYLEVEL(1, "Cannot open directory '%s': %s\n", dirName, strerror(errno));
return 0;
}
dirLength = (int)strlen(dirName);
dirLength = strlen(dirName);
errno = 0;
while ((entry = readdir(dir)) != NULL) {
if (strcmp (entry->d_name, "..") == 0 ||
strcmp (entry->d_name, ".") == 0) continue;
fnameLength = (int)strlen(entry->d_name);
fnameLength = strlen(entry->d_name);
path = (char*) malloc(dirLength + fnameLength + 2);
if (!path) { closedir(dir); return 0; }
memcpy(path, dirName, dirLength);
@ -495,7 +507,8 @@ int UTIL_prepareFileList(const char *dirName, char** bufStart, size_t* pos, char
} else {
if (*bufStart + *pos + pathLength >= *bufEnd) {
ptrdiff_t newListSize = (*bufEnd - *bufStart) + LIST_SIZE_INCREASE;
*bufStart = (char*)UTIL_realloc(*bufStart, newListSize);
assert(newListSize >= 0);
*bufStart = (char*)UTIL_realloc(*bufStart, (size_t)newListSize);
*bufEnd = *bufStart + newListSize;
if (*bufStart == NULL) { free(path); closedir(dir); return 0; }
}
@ -544,7 +557,6 @@ UTIL_createFileList(const char **inputNames, unsigned inputNamesNb,
unsigned i, nbFiles;
char* buf = (char*)malloc(LIST_SIZE_INCREASE);
char* bufend = buf + LIST_SIZE_INCREASE;
const char** fileTable;
if (!buf) return NULL;
@ -553,36 +565,37 @@ UTIL_createFileList(const char **inputNames, unsigned inputNamesNb,
size_t const len = strlen(inputNames[i]);
if (buf + pos + len >= bufend) {
ptrdiff_t newListSize = (bufend - buf) + LIST_SIZE_INCREASE;
buf = (char*)UTIL_realloc(buf, newListSize);
assert(newListSize >= 0);
buf = (char*)UTIL_realloc(buf, (size_t)newListSize);
bufend = buf + newListSize;
if (!buf) return NULL;
}
if (buf + pos + len < bufend) {
memcpy(buf+pos, inputNames[i], len+1); /* with final \0 */
memcpy(buf+pos, inputNames[i], len+1); /* including final \0 */
pos += len + 1;
nbFiles++;
}
} else {
nbFiles += UTIL_prepareFileList(inputNames[i], &buf, &pos, &bufend, followLinks);
nbFiles += (unsigned)UTIL_prepareFileList(inputNames[i], &buf, &pos, &bufend, followLinks);
if (buf == NULL) return NULL;
} }
if (nbFiles == 0) { free(buf); return NULL; }
fileTable = (const char**)malloc((nbFiles+1) * sizeof(const char*));
{ const char** const fileTable = (const char**)malloc((nbFiles + 1) * sizeof(*fileTable));
if (!fileTable) { free(buf); return NULL; }
for (i=0, pos=0; i<nbFiles; i++) {
for (i = 0, pos = 0; i < nbFiles; i++) {
fileTable[i] = buf + pos;
if (buf + pos > bufend) { free(buf); free((void*)fileTable); return NULL; }
pos += strlen(fileTable[i]) + 1;
}
if (buf + pos > bufend) { free(buf); free((void*)fileTable); return NULL; }
*allocatedBuffer = buf;
*allocatedNamesNb = nbFiles;
return fileTable;
}
}
@ -615,7 +628,12 @@ int UTIL_countPhysicalCores(void)
DWORD returnLength = 0;
size_t byteOffset = 0;
glpi = (LPFN_GLPI)GetProcAddress(GetModuleHandle(TEXT("kernel32")),
#if defined(_MSC_VER)
/* Visual Studio does not like the following cast */
# pragma warning( disable : 4054 ) /* conversion from function ptr to data ptr */
# pragma warning( disable : 4055 ) /* conversion from data ptr to function ptr */
#endif
glpi = (LPFN_GLPI)(void*)GetProcAddress(GetModuleHandle(TEXT("kernel32")),
"GetLogicalProcessorInformation");
if (glpi == NULL) {

1
programs/util.h
View File

@ -134,6 +134,7 @@ int UTIL_setFileStat(const char* filename, stat_t* statbuf);
U32 UTIL_isDirectory(const char* infilename);
int UTIL_getFileStat(const char* infilename, stat_t* statbuf);
int UTIL_isSameFile(const char* file1, const char* file2);
int UTIL_compareStr(const void *p1, const void *p2);
U32 UTIL_isLink(const char* infilename);
#define UTIL_FILESIZE_UNKNOWN ((U64)(-1))

BIN
programs/windres/zstd32.res
View File

Binary file not shown.

BIN
programs/windres/zstd64.res
View File

Binary file not shown.

17
programs/zstd.1
View File

@ -1,5 +1,5 @@
.
.TH "ZSTD" "1" "August 2019" "zstd 1.4.3" "User Commands"
.TH "ZSTD" "1" "October 2019" "zstd 1.4.4" "User Commands"
.
.SH "NAME"
\fBzstd\fR \- zstd, zstdmt, unzstd, zstdcat \- Compress or decompress \.zst files
@ -127,6 +127,14 @@ Does not spawn a thread for compression, use a single thread for both I/O and co
\fBzstd\fR will dynamically adapt compression level to perceived I/O conditions\. Compression level adaptation can be observed live by using command \fB\-v\fR\. Adaptation can be constrained between supplied \fBmin\fR and \fBmax\fR levels\. The feature works when combined with multi\-threading and \fB\-\-long\fR mode\. It does not work with \fB\-\-single\-thread\fR\. It sets window size to 8 MB by default (can be changed manually, see \fBwlog\fR)\. Due to the chaotic nature of dynamic adaptation, compressed result is not reproducible\. \fInote\fR : at the time of this writing, \fB\-\-adapt\fR can remain stuck at low speed when combined with multiple worker threads (>=2)\.
.
.TP
\fB\-\-stream\-size=#\fR
Sets the pledged source size of input coming from a stream\. This value must be exact, as it will be included in the produced frame header\. Incorrect stream sizes will cause an error\. This information will be used to better optimize compression parameters, resulting in better and potentially faster compression, especially for smaller source sizes\.
.
.TP
\fB\-\-size\-hint=#\fR
When handling input from a stream, \fBzstd\fR must guess how large the source size will be when optimizing compression parameters\. If the stream size is relatively small, this guess may be a poor one, resulting in a higher compression ratio than expected\. This feature allows for controlling the guess when needed\. Exact guesses result in better compression ratios\. Overestimates result in slightly degraded compression ratios, while underestimates may result in significant degradation\.
.
.TP
\fB\-\-rsyncable\fR
\fBzstd\fR will periodically synchronize the compression state to make the compressed file more rsync\-friendly\. There is a negligible impact to compression ratio, and the faster compression levels will see a small compression speed hit\. This feature does not work with \fB\-\-single\-thread\fR\. You probably don\'t want to use it with long range mode, since it will decrease the effectiveness of the synchronization points, but your milage may vary\.
.
@ -167,6 +175,10 @@ keep source file(s) after successful compression or decompression\. This is the
operate recursively on directories
.
.TP
\fB\-\-output\-dir\-flat[=dir]\fR
resulting files are stored into target \fBdir\fR directory, instead of same directory as origin file\. Be aware that this command can introduce name collision issues, if multiple files, from different directories, end up having the same name\. Collision resolution ensures first file with a given name will be present in \fBdir\fR, while in combination with \fB\-f\fR, the last file will be present instead\.
.
.TP
\fB\-\-format=FORMAT\fR
compress and decompress in other formats\. If compiled with support, zstd can compress to or decompress from other compression algorithm formats\. Possibly available options are \fBzstd\fR, \fBgzip\fR, \fBxz\fR, \fBlzma\fR, and \fBlz4\fR\. If no such format is provided, \fBzstd\fR is the default\.
.
@ -198,6 +210,9 @@ add integrity check computed from uncompressed data (default: enabled)
\fB\-\-\fR
All arguments after \fB\-\-\fR are treated as files
.
.SS "Restricted usage of Environment Variables"
Using environment variables to set parameters has security implications\. Therefore, this avenue is intentionally restricted\. Only \fBZSTD_CLEVEL\fR is supported currently, for setting compression level\. \fBZSTD_CLEVEL\fR can be used to set the level between 1 and 19 (the "normal" range)\. If the value of \fBZSTD_CLEVEL\fR is not a valid integer, it will be ignored with a warning message\. \fBZSTD_CLEVEL\fR just replaces the default compression level (\fB3\fR)\. It can be overridden by corresponding command line arguments\.
.
.SH "DICTIONARY BUILDER"
\fBzstd\fR offers \fIdictionary\fR compression, which greatly improves efficiency on small files and messages\. It\'s possible to train \fBzstd\fR with a set of samples, the result of which is saved into a file called a \fBdictionary\fR\. Then during compression and decompression, reference the same dictionary, using command \fB\-D dictionaryFileName\fR\. Compression of small files similar to the sample set will be greatly improved\.
.

17
programs/zstd.1.md
View File

@ -191,6 +191,13 @@ the last one takes effect.
This is the default behavior.
* `-r`:
operate recursively on directories
* `--output-dir-flat[=dir]`:
resulting files are stored into target `dir` directory,
instead of same directory as origin file.
Be aware that this command can introduce name collision issues,
if multiple files, from different directories, end up having the same name.
Collision resolution ensures first file with a given name will be present in `dir`,
while in combination with `-f`, the last file will be present instead.
* `--format=FORMAT`:
compress and decompress in other formats. If compiled with
support, zstd can compress to or decompress from other compression algorithm
@ -214,6 +221,16 @@ the last one takes effect.
* `--`:
All arguments after `--` are treated as files
### Restricted usage of Environment Variables
Using environment variables to set parameters has security implications.
Therefore, this avenue is intentionally restricted.
Only `ZSTD_CLEVEL` is supported currently, for setting compression level.
`ZSTD_CLEVEL` can be used to set the level between 1 and 19 (the "normal" range).
If the value of `ZSTD_CLEVEL` is not a valid integer, it will be ignored with a warning message.
`ZSTD_CLEVEL` just replaces the default compression level (`3`).
It can be overridden by corresponding command line arguments.
DICTIONARY BUILDER
------------------

70
programs/zstdcli.c
View File

@ -139,7 +139,7 @@ static int usage_advanced(const char* programName)
#ifndef ZSTD_NOCOMPRESS
DISPLAY( "--ultra : enable levels beyond %i, up to %i (requires more memory)\n", ZSTDCLI_CLEVEL_MAX, ZSTD_maxCLevel());
DISPLAY( "--long[=#]: enable long distance matching with given window log (default: %u)\n", g_defaultMaxWindowLog);
DISPLAY( "--fast[=#]: switch to ultra fast compression level (default: %u)\n", 1);
DISPLAY( "--fast[=#]: switch to very fast compression levels (default: %u)\n", 1);
DISPLAY( "--adapt : dynamically adapt compression level to I/O conditions \n");
DISPLAY( "--stream-size=# : optimize compression parameters for streaming input of given number of bytes \n");
DISPLAY( "--size-hint=# optimize compression parameters for streaming input of approximately this size\n");
@ -147,7 +147,7 @@ static int usage_advanced(const char* programName)
#ifdef ZSTD_MULTITHREAD
DISPLAY( " -T# : spawns # compression threads (default: 1, 0==# cores) \n");
DISPLAY( " -B# : select size of each job (default: 0==automatic) \n");
DISPLAY( " --rsyncable : compress using a rsync-friendly method (-B sets block size) \n");
DISPLAY( "--rsyncable : compress using a rsync-friendly method (-B sets block size) \n");
#endif
DISPLAY( "--no-dictID : don't write dictID into header (dictionary compression)\n");
DISPLAY( "--[no-]check : integrity check (default: enabled) \n");
@ -155,6 +155,7 @@ static int usage_advanced(const char* programName)
#endif
#ifdef UTIL_HAS_CREATEFILELIST
DISPLAY( " -r : operate recursively on directories \n");
DISPLAY( "--output-dir-flat[=directory]: all resulting files stored into `directory`. \n");
#endif
DISPLAY( "--format=zstd : compress files to the .zst format (default) \n");
#ifdef ZSTD_GZCOMPRESS
@ -286,7 +287,7 @@ static unsigned readU32FromChar(const char** stringPtr) {
* If yes, @return 1 and advances *stringPtr to the position which immediately follows longCommand.
* @return 0 and doesn't modify *stringPtr otherwise.
*/
static unsigned longCommandWArg(const char** stringPtr, const char* longCommand)
static int longCommandWArg(const char** stringPtr, const char* longCommand)
{
size_t const comSize = strlen(longCommand);
int const result = !strncmp(*stringPtr, longCommand, comSize);
@ -429,8 +430,8 @@ static ZDICT_fastCover_params_t defaultFastCoverParams(void)
static unsigned parseAdaptParameters(const char* stringPtr, int* adaptMinPtr, int* adaptMaxPtr)
{
for ( ; ;) {
if (longCommandWArg(&stringPtr, "min=")) { *adaptMinPtr = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
if (longCommandWArg(&stringPtr, "max=")) { *adaptMaxPtr = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
if (longCommandWArg(&stringPtr, "min=")) { *adaptMinPtr = (int)readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
if (longCommandWArg(&stringPtr, "max=")) { *adaptMaxPtr = (int)readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
DISPLAYLEVEL(4, "invalid compression parameter \n");
return 0;
}
@ -525,7 +526,7 @@ static int init_cLevel(void) {
DISPLAYLEVEL(2, "Ignore environment variable setting %s=%s: numeric value too large\n", ENV_CLEVEL, env);
return ZSTDCLI_CLEVEL_DEFAULT;
} else if (*ptr == 0) {
return sign * absLevel;
return sign * (int)absLevel;
}
}
@ -562,6 +563,7 @@ int main(int argCount, const char* argv[])
adaptMax = MAXCLEVEL,
rsyncable = 0,
nextArgumentIsOutFileName = 0,
nextArgumentIsOutDirName = 0,
nextArgumentIsMaxDict = 0,
nextArgumentIsDictID = 0,
nextArgumentsAreFiles = 0,
@ -589,6 +591,7 @@ int main(int argCount, const char* argv[])
unsigned filenameIdx = 0;
const char* programName = argv[0];
const char* outFileName = NULL;
const char* outDirName = NULL;
const char* dictFileName = NULL;
const char* suffix = ZSTD_EXTENSION;
unsigned maxDictSize = g_defaultMaxDictSize;
@ -689,6 +692,7 @@ int main(int argCount, const char* argv[])
if (!strcmp(argument, "--keep")) { FIO_setRemoveSrcFile(prefs, 0); continue; }
if (!strcmp(argument, "--rm")) { FIO_setRemoveSrcFile(prefs, 1); continue; }
if (!strcmp(argument, "--priority=rt")) { setRealTimePrio = 1; continue; }
if (!strcmp(argument, "--output-dir-flat")) {nextArgumentIsOutDirName=1; lastCommand=1; continue; }
if (!strcmp(argument, "--adapt")) { adapt = 1; continue; }
if (longCommandWArg(&argument, "--adapt=")) { adapt = 1; if (!parseAdaptParameters(argument, &adaptMin, &adaptMax)) CLEAN_RETURN(badusage(programName)); continue; }
if (!strcmp(argument, "--single-thread")) { nbWorkers = 0; singleThread = 1; continue; }
@ -744,7 +748,7 @@ int main(int argCount, const char* argv[])
continue;
}
#endif
if (longCommandWArg(&argument, "--threads=")) { nbWorkers = readU32FromChar(&argument); continue; }
if (longCommandWArg(&argument, "--threads=")) { nbWorkers = (int)readU32FromChar(&argument); continue; }
if (longCommandWArg(&argument, "--memlimit=")) { memLimit = readU32FromChar(&argument); continue; }
if (longCommandWArg(&argument, "--memory=")) { memLimit = readU32FromChar(&argument); continue; }
if (longCommandWArg(&argument, "--memlimit-decompress=")) { memLimit = readU32FromChar(&argument); continue; }
@ -755,6 +759,7 @@ int main(int argCount, const char* argv[])
if (longCommandWArg(&argument, "--stream-size=")) { streamSrcSize = readU32FromChar(&argument); continue; }
if (longCommandWArg(&argument, "--target-compressed-block-size=")) { targetCBlockSize = readU32FromChar(&argument); continue; }
if (longCommandWArg(&argument, "--size-hint=")) { srcSizeHint = readU32FromChar(&argument); continue; }
if (longCommandWArg(&argument, "--output-dir-flat=")) { outDirName = argument; continue; }
if (longCommandWArg(&argument, "--long")) {
unsigned ldmWindowLog = 0;
ldmFlag = 1;
@ -866,7 +871,7 @@ int main(int argCount, const char* argv[])
#ifndef ZSTD_NOCOMPRESS
/* compression Level */
if ((*argument>='0') && (*argument<='9')) {
dictCLevel = cLevel = readU32FromChar(&argument);
dictCLevel = cLevel = (int)readU32FromChar(&argument);
continue;
}
#endif
@ -938,7 +943,7 @@ int main(int argCount, const char* argv[])
case 'e':
/* compression Level */
argument++;
cLevelLast = readU32FromChar(&argument);
cLevelLast = (int)readU32FromChar(&argument);
break;
/* Modify Nb Iterations (benchmark only) */
@ -964,7 +969,7 @@ int main(int argCount, const char* argv[])
/* nb of threads (hidden option) */
case 'T':
argument++;
nbWorkers = readU32FromChar(&argument);
nbWorkers = (int)readU32FromChar(&argument);
break;
/* Dictionary Selection level */
@ -1028,6 +1033,13 @@ int main(int argCount, const char* argv[])
continue;
}
if (nextArgumentIsOutDirName) {
nextArgumentIsOutDirName = 0;
lastCommand = 0;
outDirName = argument;
continue;
}
/* add filename to list */
filenameTable[filenameIdx++] = argument;
}
@ -1094,16 +1106,16 @@ int main(int argCount, const char* argv[])
#ifndef ZSTD_NOBENCH
benchParams.blockSize = blockSize;
benchParams.nbWorkers = nbWorkers;
benchParams.realTime = setRealTimePrio;
benchParams.realTime = (unsigned)setRealTimePrio;
benchParams.nbSeconds = bench_nbSeconds;
benchParams.ldmFlag = ldmFlag;
benchParams.ldmMinMatch = g_ldmMinMatch;
benchParams.ldmHashLog = g_ldmHashLog;
benchParams.ldmMinMatch = (int)g_ldmMinMatch;
benchParams.ldmHashLog = (int)g_ldmHashLog;
if (g_ldmBucketSizeLog != LDM_PARAM_DEFAULT) {
benchParams.ldmBucketSizeLog = g_ldmBucketSizeLog;
benchParams.ldmBucketSizeLog = (int)g_ldmBucketSizeLog;
}
if (g_ldmHashRateLog != LDM_PARAM_DEFAULT) {
benchParams.ldmHashRateLog = g_ldmHashRateLog;
benchParams.ldmHashRateLog = (int)g_ldmHashRateLog;
}
benchParams.literalCompressionMode = literalCompressionMode;
@ -1144,16 +1156,16 @@ int main(int argCount, const char* argv[])
#ifndef ZSTD_NODICT
ZDICT_params_t zParams;
zParams.compressionLevel = dictCLevel;
zParams.notificationLevel = g_displayLevel;
zParams.notificationLevel = (unsigned)g_displayLevel;
zParams.dictID = dictID;
if (dict == cover) {
int const optimize = !coverParams.k || !coverParams.d;
coverParams.nbThreads = nbWorkers;
coverParams.nbThreads = (unsigned)nbWorkers;
coverParams.zParams = zParams;
operationResult = DiB_trainFromFiles(outFileName, maxDictSize, filenameTable, filenameIdx, blockSize, NULL, &coverParams, NULL, optimize);
} else if (dict == fastCover) {
int const optimize = !fastCoverParams.k || !fastCoverParams.d;
fastCoverParams.nbThreads = nbWorkers;
fastCoverParams.nbThreads = (unsigned)nbWorkers;
fastCoverParams.zParams = zParams;
operationResult = DiB_trainFromFiles(outFileName, maxDictSize, filenameTable, filenameIdx, blockSize, NULL, NULL, &fastCoverParams, optimize);
} else {
@ -1172,7 +1184,7 @@ int main(int argCount, const char* argv[])
}
#ifndef ZSTD_NODECOMPRESS
if (operation==zom_test) { outFileName=nulmark; FIO_setRemoveSrcFile(prefs, 0); } /* test mode */
if (operation==zom_test) { FIO_setTestMode(prefs, 1); outFileName=nulmark; FIO_setRemoveSrcFile(prefs, 0); } /* test mode */
#endif
/* No input filename ==> use stdin and stdout */
@ -1208,14 +1220,14 @@ int main(int argCount, const char* argv[])
if (operation==zom_compress) {
#ifndef ZSTD_NOCOMPRESS
FIO_setNbWorkers(prefs, nbWorkers);
FIO_setBlockSize(prefs, (U32)blockSize);
if (g_overlapLog!=OVERLAP_LOG_DEFAULT) FIO_setOverlapLog(prefs, g_overlapLog);
FIO_setLdmFlag(prefs, ldmFlag);
FIO_setLdmHashLog(prefs, g_ldmHashLog);
FIO_setLdmMinMatch(prefs, g_ldmMinMatch);
if (g_ldmBucketSizeLog != LDM_PARAM_DEFAULT) FIO_setLdmBucketSizeLog(prefs, g_ldmBucketSizeLog);
if (g_ldmHashRateLog != LDM_PARAM_DEFAULT) FIO_setLdmHashRateLog(prefs, g_ldmHashRateLog);
FIO_setAdaptiveMode(prefs, adapt);
FIO_setBlockSize(prefs, (int)blockSize);
if (g_overlapLog!=OVERLAP_LOG_DEFAULT) FIO_setOverlapLog(prefs, (int)g_overlapLog);
FIO_setLdmFlag(prefs, (unsigned)ldmFlag);
FIO_setLdmHashLog(prefs, (int)g_ldmHashLog);
FIO_setLdmMinMatch(prefs, (int)g_ldmMinMatch);
if (g_ldmBucketSizeLog != LDM_PARAM_DEFAULT) FIO_setLdmBucketSizeLog(prefs, (int)g_ldmBucketSizeLog);
if (g_ldmHashRateLog != LDM_PARAM_DEFAULT) FIO_setLdmHashRateLog(prefs, (int)g_ldmHashRateLog);
FIO_setAdaptiveMode(prefs, (unsigned)adapt);
FIO_setAdaptMin(prefs, adaptMin);
FIO_setAdaptMax(prefs, adaptMax);
FIO_setRsyncable(prefs, rsyncable);
@ -1229,7 +1241,7 @@ int main(int argCount, const char* argv[])
if ((filenameIdx==1) && outFileName)
operationResult = FIO_compressFilename(prefs, outFileName, filenameTable[0], dictFileName, cLevel, compressionParams);
else
operationResult = FIO_compressMultipleFilenames(prefs, filenameTable, filenameIdx, outFileName, suffix, dictFileName, cLevel, compressionParams);
operationResult = FIO_compressMultipleFilenames(prefs, filenameTable, filenameIdx, outDirName, outFileName, suffix, dictFileName, cLevel, compressionParams);
#else
(void)suffix; (void)adapt; (void)rsyncable; (void)ultra; (void)cLevel; (void)ldmFlag; (void)literalCompressionMode; (void)targetCBlockSize; (void)streamSrcSize; (void)srcSizeHint; /* not used when ZSTD_NOCOMPRESS set */
DISPLAY("Compression not supported \n");
@ -1247,7 +1259,7 @@ int main(int argCount, const char* argv[])
if (filenameIdx==1 && outFileName)
operationResult = FIO_decompressFilename(prefs, outFileName, filenameTable[0], dictFileName);
else
operationResult = FIO_decompressMultipleFilenames(prefs, filenameTable, filenameIdx, outFileName, dictFileName);
operationResult = FIO_decompressMultipleFilenames(prefs, filenameTable, filenameIdx, outDirName, outFileName, dictFileName);
#else
DISPLAY("Decompression not supported \n");
#endif

2
programs/zstdgrep.1
View File

@ -1,5 +1,5 @@
.
.TH "ZSTDGREP" "1" "August 2019" "zstd 1.4.3" "User Commands"
.TH "ZSTDGREP" "1" "October 2019" "zstd 1.4.4" "User Commands"
.
.SH "NAME"
\fBzstdgrep\fR \- print lines matching a pattern in zstandard\-compressed files

2
programs/zstdless.1
View File

@ -1,5 +1,5 @@
.
.TH "ZSTDLESS" "1" "August 2019" "zstd 1.4.3" "User Commands"
.TH "ZSTDLESS" "1" "October 2019" "zstd 1.4.4" "User Commands"
.
.SH "NAME"
\fBzstdless\fR \- view zstandard\-compressed files

7
tests/Makefile
View File

@ -250,7 +250,8 @@ clean:
$(MAKE) -C $(ZSTDDIR) clean
$(MAKE) -C $(PRGDIR) clean
@$(RM) -fR $(TESTARTEFACT)
@$(RM) -f core *.o tmp* *.tmp result* *.gcda dictionary *.zst \
@$(RM) -rf tmp* # some test directories are named tmp*
@$(RM) core *.o *.tmp result* *.gcda dictionary *.zst \
$(PRGDIR)/zstd$(EXT) $(PRGDIR)/zstd32$(EXT) \
fullbench$(EXT) fullbench32$(EXT) \
fullbench-lib$(EXT) fullbench-dll$(EXT) \
@ -264,7 +265,7 @@ clean:
#----------------------------------------------------------------------------------
#make valgrindTest is validated only for Linux, macOS, BSD, Hurd and Solaris targets
# valgrind tests are validated only for some posix platforms
#----------------------------------------------------------------------------------
ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS))
HOST_OS = POSIX
@ -286,7 +287,7 @@ valgrindTest: zstd datagen fuzzer fullbench
endif
ifneq (,$(filter MSYS%,$(shell uname)))
ifneq (,$(filter MINGW% MSYS%,$(shell uname)))
HOST_OS = MSYS
endif

4
tests/decodecorpus.c
View File

@ -758,8 +758,8 @@ static U32 generateSequences(U32* seed, frame_t* frame, seqStore_t* seqStore,
DISPLAYLEVEL(7, " repeat offset: %d\n", (int)repIndex);
}
/* use libzstd sequence handling */
ZSTD_storeSeq(seqStore, literalLen, literals, offsetCode,
matchLen - MINMATCH);
ZSTD_storeSeq(seqStore, literalLen, literals, literals + literalLen,
offsetCode, matchLen - MINMATCH);
literalsSize -= literalLen;
excessMatch -= (matchLen - MIN_SEQ_LEN);

9
tests/fullbench.c
View File

@ -45,7 +45,6 @@
#define NBLOOPS 6
#define TIMELOOP_S 2
#define KNUTH 2654435761U
#define MAX_MEM (1984 MB)
#define DEFAULT_CLEVEL 1
@ -450,7 +449,7 @@ static int benchMem(unsigned benchNb,
case 31: /* ZSTD_decodeLiteralsBlock : starts literals block in dstBuff2 */
{ size_t frameHeaderSize;
g_cSize = ZSTD_compress(dstBuff, dstBuffSize, src, srcSize, cLevel);
frameHeaderSize = ZSTD_frameHeaderSize(dstBuff, ZSTD_FRAMEHEADERSIZE_PREFIX);
frameHeaderSize = ZSTD_frameHeaderSize(dstBuff, ZSTD_FRAMEHEADERSIZE_PREFIX(ZSTD_f_zstd1));
CONTROL(!ZSTD_isError(frameHeaderSize));
/* check block is compressible, hence contains a literals section */
{ blockProperties_t bp;
@ -471,10 +470,10 @@ static int benchMem(unsigned benchNb,
const BYTE* ip = dstBuff;
const BYTE* iend;
{ size_t const cSize = ZSTD_compress(dstBuff, dstBuffSize, src, srcSize, cLevel);
CONTROL(cSize > ZSTD_FRAMEHEADERSIZE_PREFIX);
CONTROL(cSize > ZSTD_FRAMEHEADERSIZE_PREFIX(ZSTD_f_zstd1));
}
/* Skip frame Header */
{ size_t const frameHeaderSize = ZSTD_frameHeaderSize(dstBuff, ZSTD_FRAMEHEADERSIZE_PREFIX);
{ size_t const frameHeaderSize = ZSTD_frameHeaderSize(dstBuff, ZSTD_FRAMEHEADERSIZE_PREFIX(ZSTD_f_zstd1));
CONTROL(!ZSTD_isError(frameHeaderSize));
ip += frameHeaderSize;
}
@ -722,7 +721,7 @@ static int usage_advanced(const char* exename)
DISPLAY( "\nAdvanced options :\n");
DISPLAY( " -b# : test only function # \n");
DISPLAY( " -l# : benchmark functions at that compression level (default : %i)\n", DEFAULT_CLEVEL);
DISPLAY( " --zstd : custom parameter selection. Format same as zstdcli \n");
DISPLAY( "--zstd= : custom parameter selection. Format same as zstdcli \n");
DISPLAY( " -P# : sample compressibility (default : %.1f%%)\n", COMPRESSIBILITY_DEFAULT * 100);
DISPLAY( " -B# : sample size (default : %u)\n", (unsigned)kSampleSizeDefault);
DISPLAY( " -i# : iteration loops [1-9](default : %i)\n", NBLOOPS);

5
tests/fuzz/stream_round_trip.c
View File

@ -125,13 +125,14 @@ static size_t compress(uint8_t *dst, size_t capacity,
int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)
{
size_t neededBufSize;
/* Give a random portion of src data to the producer, to use for
parameter generation. The rest will be used for (de)compression */
FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(src, size);
size = FUZZ_dataProducer_reserveDataPrefix(producer);
size_t neededBufSize;
neededBufSize = ZSTD_compressBound(size) * 5;
neededBufSize = ZSTD_compressBound(size) * 15;
/* Allocate all buffers and contexts if not already allocated */
if (neededBufSize > bufSize) {

94
tests/fuzzer.c
View File

@ -304,6 +304,28 @@ static int FUZ_mallocTests(unsigned seed, double compressibility, unsigned part)
#endif
static void FUZ_decodeSequences(BYTE* dst, ZSTD_Sequence* seqs, size_t seqsSize, BYTE* src, size_t size)
{
size_t i;
size_t j;
for(i = 0; i < seqsSize - 1; ++i) {
assert(dst + seqs[i].litLength + seqs[i].matchLength < dst + size);
assert(src + seqs[i].litLength + seqs[i].matchLength < src + size);
memcpy(dst, src, seqs[i].litLength);
dst += seqs[i].litLength;
src += seqs[i].litLength;
size -= seqs[i].litLength;
for (j = 0; j < seqs[i].matchLength; ++j)
dst[j] = dst[j - seqs[i].offset];
dst += seqs[i].matchLength;
src += seqs[i].matchLength;
size -= seqs[i].matchLength;
}
memcpy(dst, src, size);
}
/*=============================================
* Unit tests
=============================================*/
@ -1365,7 +1387,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
}
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(3, "test%3i : Building cdict w/ ZSTD_dm_fullDict on a good dictionary : ", testNb++);
DISPLAYLEVEL(3, "test%3i : Building cdict w/ ZSTD_dct_fullDict on a good dictionary : ", testNb++);
{ ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize);
ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_fullDict, cParams, ZSTD_defaultCMem);
if (cdict==NULL) goto _output_error;
@ -1373,7 +1395,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
}
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(3, "test%3i : Building cdict w/ ZSTD_dm_fullDict on a rawContent (must fail) : ", testNb++);
DISPLAYLEVEL(3, "test%3i : Building cdict w/ ZSTD_dct_fullDict on a rawContent (must fail) : ", testNb++);
{ ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize);
ZSTD_CDict* const cdict = ZSTD_createCDict_advanced((const char*)dictBuffer+1, dictSize-1, ZSTD_dlm_byRef, ZSTD_dct_fullDict, cParams, ZSTD_defaultCMem);
if (cdict!=NULL) goto _output_error;
@ -1381,7 +1403,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
}
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(3, "test%3i : Loading rawContent starting with dict header w/ ZSTD_dm_auto should fail : ", testNb++);
DISPLAYLEVEL(3, "test%3i : Loading rawContent starting with dict header w/ ZSTD_dct_auto should fail : ", testNb++);
{
size_t ret;
MEM_writeLE32((char*)dictBuffer+2, ZSTD_MAGIC_DICTIONARY);
@ -1395,7 +1417,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
}
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(3, "test%3i : Loading rawContent starting with dict header w/ ZSTD_dm_rawContent should pass : ", testNb++);
DISPLAYLEVEL(3, "test%3i : Loading rawContent starting with dict header w/ ZSTD_dct_rawContent should pass : ", testNb++);
{
size_t ret;
MEM_writeLE32((char*)dictBuffer+2, ZSTD_MAGIC_DICTIONARY);
@ -1864,6 +1886,36 @@ static int basicUnitTests(U32 const seed, double compressibility)
DISPLAYLEVEL(3, "streaming OK : regenerated %u bytes \n", (unsigned)out.pos);
}
/* basic block compression */
DISPLAYLEVEL(3, "test%3i : empty magic-less format test : ", testNb++);
CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_format, ZSTD_f_zstd1_magicless) );
{ ZSTD_inBuffer in = { CNBuffer, 0, 0 };
ZSTD_outBuffer out = { compressedBuffer, ZSTD_compressBound(0), 0 };
size_t const result = ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end);
if (result != 0) goto _output_error;
if (in.pos != in.size) goto _output_error;
cSize = out.pos;
}
DISPLAYLEVEL(3, "OK (compress : %u -> %u bytes)\n", (unsigned)0, (unsigned)cSize);
DISPLAYLEVEL(3, "test%3i : decompress of empty magic-less frame : ", testNb++);
ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
CHECK( ZSTD_DCtx_setParameter(dctx, ZSTD_d_format, ZSTD_f_zstd1_magicless) );
/* one shot */
{ size_t const result = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize);
if (result != 0) goto _output_error;
DISPLAYLEVEL(3, "one-shot OK, ");
}
/* streaming */
{ ZSTD_inBuffer in = { compressedBuffer, cSize, 0 };
ZSTD_outBuffer out = { decodedBuffer, CNBuffSize, 0 };
size_t const result = ZSTD_decompressStream(dctx, &out, &in);
if (result != 0) goto _output_error;
if (in.pos != in.size) goto _output_error;
if (out.pos != 0) goto _output_error;
DISPLAYLEVEL(3, "streaming OK : regenerated %u bytes \n", (unsigned)out.pos);
}
ZSTD_freeCCtx(cctx);
ZSTD_freeDCtx(dctx);
}
@ -1960,6 +2012,33 @@ static int basicUnitTests(U32 const seed, double compressibility)
DISPLAYLEVEL(3, "OK \n");
}
DISPLAYLEVEL(3, "test%3i : ZSTD_getSequences decode from sequences test : ", testNb++);
{
size_t srcSize = 100 KB;
BYTE* src = (BYTE*)CNBuffer;
BYTE* decoded = (BYTE*)compressedBuffer;
ZSTD_CCtx* cctx = ZSTD_createCCtx();
ZSTD_Sequence* seqs = (ZSTD_Sequence*)malloc(srcSize * sizeof(ZSTD_Sequence));
size_t seqsSize;
if (seqs == NULL) goto _output_error;
assert(cctx != NULL);
/* Populate src with random data */
RDG_genBuffer(CNBuffer, srcSize, compressibility, 0., seed);
/* get the sequences */
seqsSize = ZSTD_getSequences(cctx, seqs, srcSize, src, srcSize);
/* "decode" and compare the sequences */
FUZ_decodeSequences(decoded, seqs, seqsSize, src, srcSize);
assert(!memcmp(CNBuffer, compressedBuffer, srcSize));
ZSTD_freeCCtx(cctx);
free(seqs);
}
/* Multiple blocks of zeros test */
#define LONGZEROSLENGTH 1000000 /* 1MB of zeros */
DISPLAYLEVEL(3, "test%3i : compress %u zeroes : ", testNb++, LONGZEROSLENGTH);
@ -1972,7 +2051,6 @@ static int basicUnitTests(U32 const seed, double compressibility)
if (r != LONGZEROSLENGTH) goto _output_error; }
DISPLAYLEVEL(3, "OK \n");
/* All zeroes test (test bug #137) */
#define ZEROESLENGTH 100
DISPLAYLEVEL(3, "test%3i : compress %u zeroes : ", testNb++, ZEROESLENGTH);
@ -2156,9 +2234,9 @@ static int basicUnitTests(U32 const seed, double compressibility)
size_t approxIndex = 0;
size_t maxIndex = ((3U << 29) + (1U << ZSTD_WINDOWLOG_MAX)); /* ZSTD_CURRENT_MAX from zstd_compress_internal.h */
/* vastly overprovision space in a static context so that we can do all
* this without ever reallocating, which would reset the indices */
size_t const staticCCtxSize = 2 * ZSTD_estimateCCtxSize(22);
/* Provision enough space in a static context so that we can do all
* this without ever reallocating, which would reset the indices. */
size_t const staticCCtxSize = ZSTD_estimateCStreamSize(22);
void* const staticCCtxBuffer = malloc(staticCCtxSize);
ZSTD_CCtx* cctx = ZSTD_initStaticCCtx(staticCCtxBuffer, staticCCtxSize);

63
tests/playTests.sh
View File

@ -64,11 +64,12 @@ PRGDIR="$SCRIPT_DIR/../programs"
TESTDIR="$SCRIPT_DIR/../tests"
UNAME=$(uname)
isTerminal=false
detectedTerminal=false
if [ -t 0 ] && [ -t 1 ]
then
isTerminal=true
detectedTerminal=true
fi
isTerminal=${isTerminal:-$detectedTerminal}
isWindows=false
INTOVOID="/dev/null"
@ -211,8 +212,8 @@ $ZSTD tmp -c --no-compress-literals -19 | $ZSTD -t
$ZSTD tmp -c --compress-literals -1 | $ZSTD -t
$ZSTD tmp -c --compress-literals --fast=1 | $ZSTD -t
$ZSTD tmp -c --compress-literals -19 | $ZSTD -t
$ZSTD -b --fast=1 -i1e1 tmp --compress-literals
$ZSTD -b --fast=1 -i1e1 tmp --no-compress-literals
$ZSTD -b --fast=1 -i0e1 tmp --compress-literals
$ZSTD -b --fast=1 -i0e1 tmp --no-compress-literals
println "test : file removal"
$ZSTD -f --rm tmp
@ -239,7 +240,7 @@ rm tmp # erase source file
touch tmp.zst # create destination file
$ZSTD -f tmp && die "attempt to compress a non existing file"
test -f tmp.zst # destination file should still be present
rm tmp*
rm -rf tmp* # may also erase tmp* directory from previous failed run
println "\n===> decompression only tests "
head -c 1048576 /dev/zero > tmp
@ -269,6 +270,28 @@ if [ "$?" -eq 139 ]; then
fi
rm tmp*
println "test : compress multiple files into an output directory, --output-dir-flat"
println henlo > tmp1
mkdir tmpInputTestDir
mkdir tmpInputTestDir/we
mkdir tmpInputTestDir/we/must
mkdir tmpInputTestDir/we/must/go
mkdir tmpInputTestDir/we/must/go/deeper
println cool > tmpInputTestDir/we/must/go/deeper/tmp2
mkdir tmpOutDir
$ZSTD tmp1 tmpInputTestDir/we/must/go/deeper/tmp2 --output-dir-flat tmpOutDir
test -f tmpOutDir/tmp1.zst
test -f tmpOutDir/tmp2.zst
println "test : decompress multiple files into an output directory, --output-dir-flat"
mkdir tmpOutDirDecomp
$ZSTD tmpOutDir -r -d --output-dir-flat tmpOutDirDecomp
test -f tmpOutDirDecomp/tmp2
test -f tmpOutDirDecomp/tmp1
rm -f tmpOutDirDecomp/*
$ZSTD tmpOutDir -r -d --output-dir-flat=tmpOutDirDecomp
test -f tmpOutDirDecomp/tmp2
test -f tmpOutDirDecomp/tmp1
rm -rf tmp*
println "\n===> Advanced compression parameters "
println "Hello world!" | $ZSTD --zstd=windowLog=21, - -o tmp.zst && die "wrong parameters not detected!"
@ -412,7 +435,6 @@ ls -ls tmp* # check size of tmpdec (should be 2*(tmp1 + tmp2 + tmp3))
println "compress multiple files including a missing one (notHere) : "
$ZSTD -f tmp1 notHere tmp2 && die "missing file not detected!"
println "\n===> stream-size mode"
./datagen -g11000 > tmp
@ -560,30 +582,27 @@ println "- Create dictionary with short dictID"
$ZSTD --train-fastcover=k=46,d=8,f=15,split=80 "$TESTDIR"/*.c "$PRGDIR"/*.c --dictID=1 -o tmpDict1
cmp tmpDict tmpDict1 && die "dictionaries should have different ID !"
println "- Create dictionaries with shrink-dict flag enabled"
$ZSTD --train-fastcover=steps=256,shrink "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpShrinkDict
$ZSTD --train-fastcover=steps=256,shrink=1 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpShrinkDict1
$ZSTD --train-fastcover=steps=256,shrink=5 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpShrinkDict2
$ZSTD --train-fastcover=steps=1,shrink "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpShrinkDict
$ZSTD --train-fastcover=steps=1,shrink=1 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpShrinkDict1
$ZSTD --train-fastcover=steps=1,shrink=5 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpShrinkDict2
println "- Create dictionary with size limit"
$ZSTD --train-fastcover=steps=8 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict2 --maxdict=4K
println "- Compare size of dictionary from 90% training samples with 80% training samples"
$ZSTD --train-fastcover=split=90 -r "$TESTDIR"/*.c "$PRGDIR"/*.c
$ZSTD --train-fastcover=split=80 -r "$TESTDIR"/*.c "$PRGDIR"/*.c
$ZSTD --train-fastcover=steps=1 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict2 --maxdict=4K
println "- Create dictionary using all samples for both training and testing"
$ZSTD --train-fastcover=split=100 -r "$TESTDIR"/*.c "$PRGDIR"/*.c
$ZSTD --train-fastcover=k=56,d=8,split=100 -r "$TESTDIR"/*.c "$PRGDIR"/*.c
println "- Create dictionary using f=16"
$ZSTD --train-fastcover=f=16 -r "$TESTDIR"/*.c "$PRGDIR"/*.c
$ZSTD --train-fastcover=accel=15 -r "$TESTDIR"/*.c "$PRGDIR"/*.c && die "Created dictionary using accel=15"
$ZSTD --train-fastcover=k=56,d=8,f=16 -r "$TESTDIR"/*.c "$PRGDIR"/*.c
$ZSTD --train-fastcover=k=56,d=8,accel=15 -r "$TESTDIR"/*.c "$PRGDIR"/*.c && die "Created dictionary using accel=15"
println "- Create dictionary using accel=2"
$ZSTD --train-fastcover=accel=2 -r "$TESTDIR"/*.c "$PRGDIR"/*.c
$ZSTD --train-fastcover=k=56,d=8,accel=2 -r "$TESTDIR"/*.c "$PRGDIR"/*.c
println "- Create dictionary using accel=10"
$ZSTD --train-fastcover=accel=10 -r "$TESTDIR"/*.c "$PRGDIR"/*.c
$ZSTD --train-fastcover=k=56,d=8,accel=10 -r "$TESTDIR"/*.c "$PRGDIR"/*.c
println "- Create dictionary with multithreading"
$ZSTD --train-fastcover -T4 -r "$TESTDIR"/*.c "$PRGDIR"/*.c
println "- Test -o before --train-fastcover"
rm -f tmpDict dictionary
$ZSTD -o tmpDict --train-fastcover "$TESTDIR"/*.c "$PRGDIR"/*.c
$ZSTD -o tmpDict --train-fastcover=k=56,d=8 "$TESTDIR"/*.c "$PRGDIR"/*.c
test -f tmpDict
$ZSTD --train-fastcover "$TESTDIR"/*.c "$PRGDIR"/*.c
$ZSTD --train-fastcover=k=56,d=8 "$TESTDIR"/*.c "$PRGDIR"/*.c
test -f dictionary
rm tmp* dictionary
@ -657,10 +676,10 @@ $ZSTD -i0b0e3 tmp1
println "bench negative level"
$ZSTD -bi0 --fast tmp1
println "with recursive and quiet modes"
$ZSTD -rqi1b1e2 tmp1
$ZSTD -rqi0b1e2 tmp1
println "benchmark decompression only"
$ZSTD -f tmp1
$ZSTD -b -d -i1 tmp1.zst
$ZSTD -b -d -i0 tmp1.zst
println "\n===> zstd compatibility tests "

2
tests/poolTests.c
View File

@ -46,7 +46,7 @@ static int testOrder(size_t numThreads, size_t queueSize)
POOL_ctx* const ctx = POOL_create(numThreads, queueSize);
ASSERT_TRUE(ctx);
data.i = 0;
(void)ZSTD_pthread_mutex_init(&data.mutex, NULL);
ASSERT_FALSE(ZSTD_pthread_mutex_init(&data.mutex, NULL));
{ size_t i;
for (i = 0; i < 16; ++i) {
POOL_add(ctx, &fn, &data);

29
tests/regression/method.c
View File

@ -444,6 +444,8 @@ static int init_cstream(
ZSTD_parameters const params = config_get_zstd_params(config, 0, 0);
ZSTD_CDict* dict = NULL;
if (cdict) {
if (!config->use_dictionary)
return 1;
*cdict = ZSTD_createCDict_advanced(
state->dictionary.data,
state->dictionary.size,
@ -459,14 +461,18 @@ static int init_cstream(
} else {
zret = ZSTD_initCStream_advanced(
zcs,
state->dictionary.data,
state->dictionary.size,
config->use_dictionary ? state->dictionary.data : NULL,
config->use_dictionary ? state->dictionary.size : 0,
params,
ZSTD_CONTENTSIZE_UNKNOWN);
}
} else {
int const level = config_get_level(config);
if (level == CONFIG_NO_LEVEL)
return 1;
if (cdict) {
if (!config->use_dictionary)
return 1;
*cdict = ZSTD_createCDict(
state->dictionary.data,
state->dictionary.size,
@ -477,7 +483,10 @@ static int init_cstream(
zret = ZSTD_initCStream_usingCDict(zcs, *cdict);
} else if (config->use_dictionary) {
zret = ZSTD_initCStream_usingDict(
zcs, state->dictionary.data, state->dictionary.size, level);
zcs,
state->dictionary.data,
state->dictionary.size,
level);
} else {
zret = ZSTD_initCStream(zcs, level);
}
@ -506,6 +515,14 @@ static result_t old_streaming_compress_internal(
result = result_error(result_error_compression_error);
goto out;
}
if (!advanced && config_get_level(config) == CONFIG_NO_LEVEL) {
result = result_error(result_error_skip);
goto out;
}
if (cdict && !config->use_dictionary) {
result = result_error(result_error_skip);
goto out;
}
if (init_cstream(state, zcs, config, advanced, cdict ? &cd : NULL)) {
result = result_error(result_error_compression_error);
goto out;
@ -629,21 +646,21 @@ method_t const old_streaming = {
method_t const old_streaming_advanced = {
.name = "old streaming advanced",
.create = buffer_state_create,
.compress = old_streaming_compress,
.compress = old_streaming_compress_advanced,
.destroy = buffer_state_destroy,
};
method_t const old_streaming_cdict = {
.name = "old streaming cdcit",
.create = buffer_state_create,
.compress = old_streaming_compress,
.compress = old_streaming_compress_cdict,
.destroy = buffer_state_destroy,
};
method_t const old_streaming_advanced_cdict = {
.name = "old streaming advanced cdict",
.create = buffer_state_create,
.compress = old_streaming_compress,
.compress = old_streaming_compress_cdict_advanced,
.destroy = buffer_state_destroy,
};

310
tests/regression/results.csv
View File

@ -461,17 +461,9 @@ silesia, level 13, old stre
silesia, level 16, old streaming, 4377389
silesia, level 19, old streaming, 4293262
silesia, no source size, old streaming, 4849455
silesia, long distance mode, old streaming, 12000408
silesia, multithreaded, old streaming, 12000408
silesia, multithreaded long distance mode, old streaming, 12000408
silesia, small window log, old streaming, 12000408
silesia, small hash log, old streaming, 12000408
silesia, small chain log, old streaming, 12000408
silesia, explicit params, old streaming, 12000408
silesia, uncompressed literals, old streaming, 4849491
silesia, uncompressed literals optimal, old streaming, 4293262
silesia, huffman literals, old streaming, 6183385
silesia, multithreaded with advanced params, old streaming, 12000408
silesia.tar, level -5, old streaming, 6982738
silesia.tar, level -3, old streaming, 6641264
silesia.tar, level -1, old streaming, 6190789
@ -487,17 +479,9 @@ silesia.tar, level 13, old stre
silesia.tar, level 16, old streaming, 4381284
silesia.tar, level 19, old streaming, 4281511
silesia.tar, no source size, old streaming, 4861372
silesia.tar, long distance mode, old streaming, 12022046
silesia.tar, multithreaded, old streaming, 12022046
silesia.tar, multithreaded long distance mode, old streaming, 12022046
silesia.tar, small window log, old streaming, 12022046
silesia.tar, small hash log, old streaming, 12022046
silesia.tar, small chain log, old streaming, 12022046
silesia.tar, explicit params, old streaming, 12022046
silesia.tar, uncompressed literals, old streaming, 4861376
silesia.tar, uncompressed literals optimal, old streaming, 4281511
silesia.tar, huffman literals, old streaming, 6190789
silesia.tar, multithreaded with advanced params, old streaming, 12022046
github, level -5, old streaming, 205285
github, level -5 with dict, old streaming, 46718
github, level -3, old streaming, 190643
@ -527,17 +511,9 @@ github, level 16 with dict, old stre
github, level 19, old streaming, 133717
github, level 19 with dict, old streaming, 37576
github, no source size, old streaming, 140631
github, long distance mode, old streaming, 412933
github, multithreaded, old streaming, 412933
github, multithreaded long distance mode, old streaming, 412933
github, small window log, old streaming, 412933
github, small hash log, old streaming, 412933
github, small chain log, old streaming, 412933
github, explicit params, old streaming, 412933
github, uncompressed literals, old streaming, 136311
github, uncompressed literals optimal, old streaming, 133717
github, huffman literals, old streaming, 175568
github, multithreaded with advanced params, old streaming, 412933
silesia, level -5, old streaming advanced, 6882466
silesia, level -3, old streaming advanced, 6568358
silesia, level -1, old streaming advanced, 6183385
@ -553,17 +529,17 @@ silesia, level 13, old stre
silesia, level 16, old streaming advanced, 4377389
silesia, level 19, old streaming advanced, 4293262
silesia, no source size, old streaming advanced, 4849455
silesia, long distance mode, old streaming advanced, 12000408
silesia, multithreaded, old streaming advanced, 12000408
silesia, multithreaded long distance mode, old streaming advanced, 12000408
silesia, small window log, old streaming advanced, 12000408
silesia, small hash log, old streaming advanced, 12000408
silesia, small chain log, old streaming advanced, 12000408
silesia, explicit params, old streaming advanced, 12000408
silesia, long distance mode, old streaming advanced, 4849491
silesia, multithreaded, old streaming advanced, 4849491
silesia, multithreaded long distance mode, old streaming advanced, 4849491
silesia, small window log, old streaming advanced, 7123534
silesia, small hash log, old streaming advanced, 6554898
silesia, small chain log, old streaming advanced, 4931093
silesia, explicit params, old streaming advanced, 4797048
silesia, uncompressed literals, old streaming advanced, 4849491
silesia, uncompressed literals optimal, old streaming advanced, 4293262
silesia, huffman literals, old streaming advanced, 6183385
silesia, multithreaded with advanced params, old streaming advanced, 12000408
silesia, multithreaded with advanced params, old streaming advanced, 4849491
silesia.tar, level -5, old streaming advanced, 6982738
silesia.tar, level -3, old streaming advanced, 6641264
silesia.tar, level -1, old streaming advanced, 6190789
@ -579,238 +555,82 @@ silesia.tar, level 13, old stre
silesia.tar, level 16, old streaming advanced, 4381284
silesia.tar, level 19, old streaming advanced, 4281511
silesia.tar, no source size, old streaming advanced, 4861372
silesia.tar, long distance mode, old streaming advanced, 12022046
silesia.tar, multithreaded, old streaming advanced, 12022046
silesia.tar, multithreaded long distance mode, old streaming advanced, 12022046
silesia.tar, small window log, old streaming advanced, 12022046
silesia.tar, small hash log, old streaming advanced, 12022046
silesia.tar, small chain log, old streaming advanced, 12022046
silesia.tar, explicit params, old streaming advanced, 12022046
silesia.tar, long distance mode, old streaming advanced, 4861376
silesia.tar, multithreaded, old streaming advanced, 4861376
silesia.tar, multithreaded long distance mode, old streaming advanced, 4861376
silesia.tar, small window log, old streaming advanced, 7127552
silesia.tar, small hash log, old streaming advanced, 6587834
silesia.tar, small chain log, old streaming advanced, 4943271
silesia.tar, explicit params, old streaming advanced, 4808570
silesia.tar, uncompressed literals, old streaming advanced, 4861376
silesia.tar, uncompressed literals optimal, old streaming advanced, 4281511
silesia.tar, huffman literals, old streaming advanced, 6190789
silesia.tar, multithreaded with advanced params, old streaming advanced, 12022046
github, level -5, old streaming advanced, 205285
github, level -5 with dict, old streaming advanced, 46718
github, level -3, old streaming advanced, 190643
github, level -3 with dict, old streaming advanced, 45395
github, level -1, old streaming advanced, 175568
github, level -1 with dict, old streaming advanced, 43170
github, level 0, old streaming advanced, 136311
github, level 0 with dict, old streaming advanced, 41148
github, level 1, old streaming advanced, 142450
github, level 1 with dict, old streaming advanced, 41682
github, level 3, old streaming advanced, 136311
github, level 3 with dict, old streaming advanced, 41148
github, level 4, old streaming advanced, 136144
github, level 4 with dict, old streaming advanced, 41251
github, level 5, old streaming advanced, 135106
github, level 5 with dict, old streaming advanced, 38938
github, level 6, old streaming advanced, 135108
github, level 6 with dict, old streaming advanced, 38632
github, level 7, old streaming advanced, 135108
github, level 7 with dict, old streaming advanced, 38766
github, level 9, old streaming advanced, 135108
github, level 9 with dict, old streaming advanced, 39326
github, level 13, old streaming advanced, 133717
github, level 13 with dict, old streaming advanced, 39716
github, level 16, old streaming advanced, 133717
github, level 16 with dict, old streaming advanced, 37577
silesia.tar, multithreaded with advanced params, old streaming advanced, 4861376
github, level -5, old streaming advanced, 216734
github, level -5 with dict, old streaming advanced, 49562
github, level -3, old streaming advanced, 192160
github, level -3 with dict, old streaming advanced, 44956
github, level -1, old streaming advanced, 181108
github, level -1 with dict, old streaming advanced, 42383
github, level 0, old streaming advanced, 141090
github, level 0 with dict, old streaming advanced, 41113
github, level 1, old streaming advanced, 143682
github, level 1 with dict, old streaming advanced, 42430
github, level 3, old streaming advanced, 141090
github, level 3 with dict, old streaming advanced, 41113
github, level 4, old streaming advanced, 141090
github, level 4 with dict, old streaming advanced, 41084
github, level 5, old streaming advanced, 139391
github, level 5 with dict, old streaming advanced, 39159
github, level 6, old streaming advanced, 139394
github, level 6 with dict, old streaming advanced, 38749
github, level 7, old streaming advanced, 138675
github, level 7 with dict, old streaming advanced, 38746
github, level 9, old streaming advanced, 138675
github, level 9 with dict, old streaming advanced, 38987
github, level 13, old streaming advanced, 138675
github, level 13 with dict, old streaming advanced, 39724
github, level 16, old streaming advanced, 138675
github, level 16 with dict, old streaming advanced, 40771
github, level 19, old streaming advanced, 133717
github, level 19 with dict, old streaming advanced, 37576
github, no source size, old streaming advanced, 140631
github, long distance mode, old streaming advanced, 412933
github, multithreaded, old streaming advanced, 412933
github, multithreaded long distance mode, old streaming advanced, 412933
github, small window log, old streaming advanced, 412933
github, small hash log, old streaming advanced, 412933
github, small chain log, old streaming advanced, 412933
github, explicit params, old streaming advanced, 412933
github, uncompressed literals, old streaming advanced, 136311
github, long distance mode, old streaming advanced, 141090
github, multithreaded, old streaming advanced, 141090
github, multithreaded long distance mode, old streaming advanced, 141090
github, small window log, old streaming advanced, 141090
github, small hash log, old streaming advanced, 141578
github, small chain log, old streaming advanced, 139258
github, explicit params, old streaming advanced, 140930
github, uncompressed literals, old streaming advanced, 141090
github, uncompressed literals optimal, old streaming advanced, 133717
github, huffman literals, old streaming advanced, 175568
github, multithreaded with advanced params, old streaming advanced, 412933
silesia, level -5, old streaming cdcit, 6882466
silesia, level -3, old streaming cdcit, 6568358
silesia, level -1, old streaming cdcit, 6183385
silesia, level 0, old streaming cdcit, 4849491
silesia, level 1, old streaming cdcit, 5314109
silesia, level 3, old streaming cdcit, 4849491
silesia, level 4, old streaming cdcit, 4786913
silesia, level 5, old streaming cdcit, 4710178
silesia, level 6, old streaming cdcit, 4659996
silesia, level 7, old streaming cdcit, 4596234
silesia, level 9, old streaming cdcit, 4543862
silesia, level 13, old streaming cdcit, 4482073
silesia, level 16, old streaming cdcit, 4377389
silesia, level 19, old streaming cdcit, 4293262
silesia, no source size, old streaming cdcit, 4849455
silesia, long distance mode, old streaming cdcit, 12000408
silesia, multithreaded, old streaming cdcit, 12000408
silesia, multithreaded long distance mode, old streaming cdcit, 12000408
silesia, small window log, old streaming cdcit, 12000408
silesia, small hash log, old streaming cdcit, 12000408
silesia, small chain log, old streaming cdcit, 12000408
silesia, explicit params, old streaming cdcit, 12000408
silesia, uncompressed literals, old streaming cdcit, 4849491
silesia, uncompressed literals optimal, old streaming cdcit, 4293262
silesia, huffman literals, old streaming cdcit, 6183385
silesia, multithreaded with advanced params, old streaming cdcit, 12000408
silesia.tar, level -5, old streaming cdcit, 6982738
silesia.tar, level -3, old streaming cdcit, 6641264
silesia.tar, level -1, old streaming cdcit, 6190789
silesia.tar, level 0, old streaming cdcit, 4861376
silesia.tar, level 1, old streaming cdcit, 5336879
silesia.tar, level 3, old streaming cdcit, 4861376
silesia.tar, level 4, old streaming cdcit, 4799583
silesia.tar, level 5, old streaming cdcit, 4722276
silesia.tar, level 6, old streaming cdcit, 4672240
silesia.tar, level 7, old streaming cdcit, 4606657
silesia.tar, level 9, old streaming cdcit, 4554106
silesia.tar, level 13, old streaming cdcit, 4491707
silesia.tar, level 16, old streaming cdcit, 4381284
silesia.tar, level 19, old streaming cdcit, 4281511
silesia.tar, no source size, old streaming cdcit, 4861372
silesia.tar, long distance mode, old streaming cdcit, 12022046
silesia.tar, multithreaded, old streaming cdcit, 12022046
silesia.tar, multithreaded long distance mode, old streaming cdcit, 12022046
silesia.tar, small window log, old streaming cdcit, 12022046
silesia.tar, small hash log, old streaming cdcit, 12022046
silesia.tar, small chain log, old streaming cdcit, 12022046
silesia.tar, explicit params, old streaming cdcit, 12022046
silesia.tar, uncompressed literals, old streaming cdcit, 4861376
silesia.tar, uncompressed literals optimal, old streaming cdcit, 4281511
silesia.tar, huffman literals, old streaming cdcit, 6190789
silesia.tar, multithreaded with advanced params, old streaming cdcit, 12022046
github, level -5, old streaming cdcit, 205285
github, huffman literals, old streaming advanced, 181108
github, multithreaded with advanced params, old streaming advanced, 141090
github, level -5 with dict, old streaming cdcit, 46718
github, level -3, old streaming cdcit, 190643
github, level -3 with dict, old streaming cdcit, 45395
github, level -1, old streaming cdcit, 175568
github, level -1 with dict, old streaming cdcit, 43170
github, level 0, old streaming cdcit, 136311
github, level 0 with dict, old streaming cdcit, 41148
github, level 1, old streaming cdcit, 142450
github, level 1 with dict, old streaming cdcit, 41682
github, level 3, old streaming cdcit, 136311
github, level 3 with dict, old streaming cdcit, 41148
github, level 4, old streaming cdcit, 136144
github, level 4 with dict, old streaming cdcit, 41251
github, level 5, old streaming cdcit, 135106
github, level 5 with dict, old streaming cdcit, 38938
github, level 6, old streaming cdcit, 135108
github, level 6 with dict, old streaming cdcit, 38632
github, level 7, old streaming cdcit, 135108
github, level 7 with dict, old streaming cdcit, 38766
github, level 9, old streaming cdcit, 135108
github, level 9 with dict, old streaming cdcit, 39326
github, level 13, old streaming cdcit, 133717
github, level 13 with dict, old streaming cdcit, 39716
github, level 16, old streaming cdcit, 133717
github, level 16 with dict, old streaming cdcit, 37577
github, level 19, old streaming cdcit, 133717
github, level 19 with dict, old streaming cdcit, 37576
github, no source size, old streaming cdcit, 140631
github, long distance mode, old streaming cdcit, 412933
github, multithreaded, old streaming cdcit, 412933
github, multithreaded long distance mode, old streaming cdcit, 412933
github, small window log, old streaming cdcit, 412933
github, small hash log, old streaming cdcit, 412933
github, small chain log, old streaming cdcit, 412933
github, explicit params, old streaming cdcit, 412933
github, uncompressed literals, old streaming cdcit, 136311
github, uncompressed literals optimal, old streaming cdcit, 133717
github, huffman literals, old streaming cdcit, 175568
github, multithreaded with advanced params, old streaming cdcit, 412933
silesia, level -5, old streaming advanced cdict, 6882466
silesia, level -3, old streaming advanced cdict, 6568358
silesia, level -1, old streaming advanced cdict, 6183385
silesia, level 0, old streaming advanced cdict, 4849491
silesia, level 1, old streaming advanced cdict, 5314109
silesia, level 3, old streaming advanced cdict, 4849491
silesia, level 4, old streaming advanced cdict, 4786913
silesia, level 5, old streaming advanced cdict, 4710178
silesia, level 6, old streaming advanced cdict, 4659996
silesia, level 7, old streaming advanced cdict, 4596234
silesia, level 9, old streaming advanced cdict, 4543862
silesia, level 13, old streaming advanced cdict, 4482073
silesia, level 16, old streaming advanced cdict, 4377389
silesia, level 19, old streaming advanced cdict, 4293262
silesia, no source size, old streaming advanced cdict, 4849455
silesia, long distance mode, old streaming advanced cdict, 12000408
silesia, multithreaded, old streaming advanced cdict, 12000408
silesia, multithreaded long distance mode, old streaming advanced cdict, 12000408
silesia, small window log, old streaming advanced cdict, 12000408
silesia, small hash log, old streaming advanced cdict, 12000408
silesia, small chain log, old streaming advanced cdict, 12000408
silesia, explicit params, old streaming advanced cdict, 12000408
silesia, uncompressed literals, old streaming advanced cdict, 4849491
silesia, uncompressed literals optimal, old streaming advanced cdict, 4293262
silesia, huffman literals, old streaming advanced cdict, 6183385
silesia, multithreaded with advanced params, old streaming advanced cdict, 12000408
silesia.tar, level -5, old streaming advanced cdict, 6982738
silesia.tar, level -3, old streaming advanced cdict, 6641264
silesia.tar, level -1, old streaming advanced cdict, 6190789
silesia.tar, level 0, old streaming advanced cdict, 4861376
silesia.tar, level 1, old streaming advanced cdict, 5336879
silesia.tar, level 3, old streaming advanced cdict, 4861376
silesia.tar, level 4, old streaming advanced cdict, 4799583
silesia.tar, level 5, old streaming advanced cdict, 4722276
silesia.tar, level 6, old streaming advanced cdict, 4672240
silesia.tar, level 7, old streaming advanced cdict, 4606657
silesia.tar, level 9, old streaming advanced cdict, 4554106
silesia.tar, level 13, old streaming advanced cdict, 4491707
silesia.tar, level 16, old streaming advanced cdict, 4381284
silesia.tar, level 19, old streaming advanced cdict, 4281511
silesia.tar, no source size, old streaming advanced cdict, 4861372
silesia.tar, long distance mode, old streaming advanced cdict, 12022046
silesia.tar, multithreaded, old streaming advanced cdict, 12022046
silesia.tar, multithreaded long distance mode, old streaming advanced cdict, 12022046
silesia.tar, small window log, old streaming advanced cdict, 12022046
silesia.tar, small hash log, old streaming advanced cdict, 12022046
silesia.tar, small chain log, old streaming advanced cdict, 12022046
silesia.tar, explicit params, old streaming advanced cdict, 12022046
silesia.tar, uncompressed literals, old streaming advanced cdict, 4861376
silesia.tar, uncompressed literals optimal, old streaming advanced cdict, 4281511
silesia.tar, huffman literals, old streaming advanced cdict, 6190789
silesia.tar, multithreaded with advanced params, old streaming advanced cdict, 12022046
github, level -5, old streaming advanced cdict, 205285
github, level -5 with dict, old streaming advanced cdict, 46718
github, level -3, old streaming advanced cdict, 190643
github, level -3 with dict, old streaming advanced cdict, 45395
github, level -1, old streaming advanced cdict, 175568
github, level -1 with dict, old streaming advanced cdict, 43170
github, level 0, old streaming advanced cdict, 136311
github, level 0 with dict, old streaming advanced cdict, 41148
github, level 1, old streaming advanced cdict, 142450
github, level 1 with dict, old streaming advanced cdict, 41682
github, level 3, old streaming advanced cdict, 136311
github, level 3 with dict, old streaming advanced cdict, 41148
github, level 4, old streaming advanced cdict, 136144
github, level 4 with dict, old streaming advanced cdict, 41251
github, level 5, old streaming advanced cdict, 135106
github, level 5 with dict, old streaming advanced cdict, 38938
github, level 6, old streaming advanced cdict, 135108
github, level 6 with dict, old streaming advanced cdict, 38632
github, level 7, old streaming advanced cdict, 135108
github, level 7 with dict, old streaming advanced cdict, 38766
github, level 9, old streaming advanced cdict, 135108
github, level 9 with dict, old streaming advanced cdict, 39326
github, level 13, old streaming advanced cdict, 133717
github, level 13 with dict, old streaming advanced cdict, 39716
github, level 16, old streaming advanced cdict, 133717
github, level 16 with dict, old streaming advanced cdict, 37577
github, level 19, old streaming advanced cdict, 133717
github, level -5 with dict, old streaming advanced cdict, 49562
github, level -3 with dict, old streaming advanced cdict, 44956
github, level -1 with dict, old streaming advanced cdict, 42383
github, level 0 with dict, old streaming advanced cdict, 41113
github, level 1 with dict, old streaming advanced cdict, 42430
github, level 3 with dict, old streaming advanced cdict, 41113
github, level 4 with dict, old streaming advanced cdict, 41084
github, level 5 with dict, old streaming advanced cdict, 39158
github, level 6 with dict, old streaming advanced cdict, 38748
github, level 7 with dict, old streaming advanced cdict, 38744
github, level 9 with dict, old streaming advanced cdict, 38986
github, level 13 with dict, old streaming advanced cdict, 39724
github, level 16 with dict, old streaming advanced cdict, 40771
github, level 19 with dict, old streaming advanced cdict, 37576
github, no source size, old streaming advanced cdict, 140631
github, long distance mode, old streaming advanced cdict, 412933
github, multithreaded, old streaming advanced cdict, 412933
github, multithreaded long distance mode, old streaming advanced cdict, 412933
github, small window log, old streaming advanced cdict, 412933
github, small hash log, old streaming advanced cdict, 412933
github, small chain log, old streaming advanced cdict, 412933
github, explicit params, old streaming advanced cdict, 412933
github, uncompressed literals, old streaming advanced cdict, 136311
github, uncompressed literals optimal, old streaming advanced cdict, 133717
github, huffman literals, old streaming advanced cdict, 175568
github, multithreaded with advanced params, old streaming advanced cdict, 412933

1 Data Config Method Total compressed size
461 silesia level 16 old streaming 4377389
462 silesia level 19 old streaming 4293262
463 silesia no source size old streaming 4849455
silesia long distance mode old streaming 12000408
silesia multithreaded old streaming 12000408
silesia multithreaded long distance mode old streaming 12000408
silesia small window log old streaming 12000408
silesia small hash log old streaming 12000408
silesia small chain log old streaming 12000408
silesia explicit params old streaming 12000408
464 silesia uncompressed literals old streaming 4849491
465 silesia uncompressed literals optimal old streaming 4293262
466 silesia huffman literals old streaming 6183385
silesia multithreaded with advanced params old streaming 12000408
467 silesia.tar level -5 old streaming 6982738
468 silesia.tar level -3 old streaming 6641264
469 silesia.tar level -1 old streaming 6190789
479 silesia.tar level 16 old streaming 4381284
480 silesia.tar level 19 old streaming 4281511
481 silesia.tar no source size old streaming 4861372
silesia.tar long distance mode old streaming 12022046
silesia.tar multithreaded old streaming 12022046
silesia.tar multithreaded long distance mode old streaming 12022046
silesia.tar small window log old streaming 12022046
silesia.tar small hash log old streaming 12022046
silesia.tar small chain log old streaming 12022046
silesia.tar explicit params old streaming 12022046
482 silesia.tar uncompressed literals old streaming 4861376
483 silesia.tar uncompressed literals optimal old streaming 4281511
484 silesia.tar huffman literals old streaming 6190789
silesia.tar multithreaded with advanced params old streaming 12022046
485 github level -5 old streaming 205285
486 github level -5 with dict old streaming 46718
487 github level -3 old streaming 190643
511 github level 19 old streaming 133717
512 github level 19 with dict old streaming 37576
513 github no source size old streaming 140631
github long distance mode old streaming 412933
github multithreaded old streaming 412933
github multithreaded long distance mode old streaming 412933
github small window log old streaming 412933
github small hash log old streaming 412933
github small chain log old streaming 412933
github explicit params old streaming 412933
514 github uncompressed literals old streaming 136311
515 github uncompressed literals optimal old streaming 133717
516 github huffman literals old streaming 175568
github multithreaded with advanced params old streaming 412933
517 silesia level -5 old streaming advanced 6882466
518 silesia level -3 old streaming advanced 6568358
519 silesia level -1 old streaming advanced 6183385
529 silesia level 16 old streaming advanced 4377389
530 silesia level 19 old streaming advanced 4293262
531 silesia no source size old streaming advanced 4849455
532 silesia long distance mode old streaming advanced 12000408 4849491
533 silesia multithreaded old streaming advanced 12000408 4849491
534 silesia multithreaded long distance mode old streaming advanced 12000408 4849491
535 silesia small window log old streaming advanced 12000408 7123534
536 silesia small hash log old streaming advanced 12000408 6554898
537 silesia small chain log old streaming advanced 12000408 4931093
538 silesia explicit params old streaming advanced 12000408 4797048
539 silesia uncompressed literals old streaming advanced 4849491
540 silesia uncompressed literals optimal old streaming advanced 4293262
541 silesia huffman literals old streaming advanced 6183385
542 silesia multithreaded with advanced params old streaming advanced 12000408 4849491
543 silesia.tar level -5 old streaming advanced 6982738
544 silesia.tar level -3 old streaming advanced 6641264
545 silesia.tar level -1 old streaming advanced 6190789
555 silesia.tar level 16 old streaming advanced 4381284
556 silesia.tar level 19 old streaming advanced 4281511
557 silesia.tar no source size old streaming advanced 4861372
558 silesia.tar long distance mode old streaming advanced 12022046 4861376
559 silesia.tar multithreaded old streaming advanced 12022046 4861376
560 silesia.tar multithreaded long distance mode old streaming advanced 12022046 4861376
561 silesia.tar small window log old streaming advanced 12022046 7127552
562 silesia.tar small hash log old streaming advanced 12022046 6587834
563 silesia.tar small chain log old streaming advanced 12022046 4943271
564 silesia.tar explicit params old streaming advanced 12022046 4808570
565 silesia.tar uncompressed literals old streaming advanced 4861376
566 silesia.tar uncompressed literals optimal old streaming advanced 4281511
567 silesia.tar huffman literals old streaming advanced 6190789
568 silesia.tar multithreaded with advanced params old streaming advanced 12022046 4861376
569 github level -5 old streaming advanced 205285 216734
570 github level -5 with dict old streaming advanced 46718 49562
571 github level -3 old streaming advanced 190643 192160
572 github level -3 with dict old streaming advanced 45395 44956
573 github level -1 old streaming advanced 175568 181108
574 github level -1 with dict old streaming advanced 43170 42383
575 github level 0 old streaming advanced 136311 141090
576 github level 0 with dict old streaming advanced 41148 41113
577 github level 1 old streaming advanced 142450 143682
578 github level 1 with dict old streaming advanced 41682 42430
579 github level 3 old streaming advanced 136311 141090
580 github level 3 with dict old streaming advanced 41148 41113
581 github level 4 old streaming advanced 136144 141090
582 github level 4 with dict old streaming advanced 41251 41084
583 github level 5 old streaming advanced 135106 139391
584 github level 5 with dict old streaming advanced 38938 39159
585 github level 6 old streaming advanced 135108 139394
586 github level 6 with dict old streaming advanced 38632 38749
587 github level 7 old streaming advanced 135108 138675
588 github level 7 with dict old streaming advanced 38766 38746
589 github level 9 old streaming advanced 135108 138675
590 github level 9 with dict old streaming advanced 39326 38987
591 github level 13 old streaming advanced 133717 138675
592 github level 13 with dict old streaming advanced 39716 39724
593 github level 16 old streaming advanced 133717 138675
594 github level 16 with dict old streaming advanced 37577 40771
595 github level 19 old streaming advanced 133717
596 github level 19 with dict old streaming advanced 37576
597 github no source size old streaming advanced 140631
598 github long distance mode old streaming advanced 412933 141090
599 github multithreaded old streaming advanced 412933 141090
600 github multithreaded long distance mode old streaming advanced 412933 141090
601 github small window log old streaming advanced 412933 141090
602 github small hash log old streaming advanced 412933 141578
603 github small chain log old streaming advanced 412933 139258
604 github explicit params old streaming advanced 412933 140930
605 github uncompressed literals old streaming advanced 136311 141090
606 github uncompressed literals optimal old streaming advanced 133717
607 github huffman literals old streaming advanced 175568 181108
608 github multithreaded with advanced params old streaming advanced 412933 141090
silesia level -5 old streaming cdcit 6882466
silesia level -3 old streaming cdcit 6568358
silesia level -1 old streaming cdcit 6183385
silesia level 0 old streaming cdcit 4849491
silesia level 1 old streaming cdcit 5314109
silesia level 3 old streaming cdcit 4849491
silesia level 4 old streaming cdcit 4786913
silesia level 5 old streaming cdcit 4710178
silesia level 6 old streaming cdcit 4659996
silesia level 7 old streaming cdcit 4596234
silesia level 9 old streaming cdcit 4543862
silesia level 13 old streaming cdcit 4482073
silesia level 16 old streaming cdcit 4377389
silesia level 19 old streaming cdcit 4293262
silesia no source size old streaming cdcit 4849455
silesia long distance mode old streaming cdcit 12000408
silesia multithreaded old streaming cdcit 12000408
silesia multithreaded long distance mode old streaming cdcit 12000408
silesia small window log old streaming cdcit 12000408
silesia small hash log old streaming cdcit 12000408
silesia small chain log old streaming cdcit 12000408
silesia explicit params old streaming cdcit 12000408
silesia uncompressed literals old streaming cdcit 4849491
silesia uncompressed literals optimal old streaming cdcit 4293262
silesia huffman literals old streaming cdcit 6183385
silesia multithreaded with advanced params old streaming cdcit 12000408
silesia.tar level -5 old streaming cdcit 6982738
silesia.tar level -3 old streaming cdcit 6641264
silesia.tar level -1 old streaming cdcit 6190789
silesia.tar level 0 old streaming cdcit 4861376
silesia.tar level 1 old streaming cdcit 5336879
silesia.tar level 3 old streaming cdcit 4861376
silesia.tar level 4 old streaming cdcit 4799583
silesia.tar level 5 old streaming cdcit 4722276
silesia.tar level 6 old streaming cdcit 4672240
silesia.tar level 7 old streaming cdcit 4606657
silesia.tar level 9 old streaming cdcit 4554106
silesia.tar level 13 old streaming cdcit 4491707
silesia.tar level 16 old streaming cdcit 4381284
silesia.tar level 19 old streaming cdcit 4281511
silesia.tar no source size old streaming cdcit 4861372
silesia.tar long distance mode old streaming cdcit 12022046
silesia.tar multithreaded old streaming cdcit 12022046
silesia.tar multithreaded long distance mode old streaming cdcit 12022046
silesia.tar small window log old streaming cdcit 12022046
silesia.tar small hash log old streaming cdcit 12022046
silesia.tar small chain log old streaming cdcit 12022046
silesia.tar explicit params old streaming cdcit 12022046
silesia.tar uncompressed literals old streaming cdcit 4861376
silesia.tar uncompressed literals optimal old streaming cdcit 4281511
silesia.tar huffman literals old streaming cdcit 6190789
silesia.tar multithreaded with advanced params old streaming cdcit 12022046
github level -5 old streaming cdcit 205285
609 github level -5 with dict old streaming cdcit 46718
github level -3 old streaming cdcit 190643
610 github level -3 with dict old streaming cdcit 45395
github level -1 old streaming cdcit 175568
611 github level -1 with dict old streaming cdcit 43170
github level 0 old streaming cdcit 136311
612 github level 0 with dict old streaming cdcit 41148
github level 1 old streaming cdcit 142450
613 github level 1 with dict old streaming cdcit 41682
github level 3 old streaming cdcit 136311
614 github level 3 with dict old streaming cdcit 41148
github level 4 old streaming cdcit 136144
615 github level 4 with dict old streaming cdcit 41251
github level 5 old streaming cdcit 135106
616 github level 5 with dict old streaming cdcit 38938
github level 6 old streaming cdcit 135108
617 github level 6 with dict old streaming cdcit 38632
github level 7 old streaming cdcit 135108
618 github level 7 with dict old streaming cdcit 38766
github level 9 old streaming cdcit 135108
619 github level 9 with dict old streaming cdcit 39326
github level 13 old streaming cdcit 133717
620 github level 13 with dict old streaming cdcit 39716
github level 16 old streaming cdcit 133717
621 github level 16 with dict old streaming cdcit 37577
github level 19 old streaming cdcit 133717
622 github level 19 with dict old streaming cdcit 37576
623 github no source size level -5 with dict old streaming cdcit old streaming advanced cdict 140631 49562
624 github long distance mode level -3 with dict old streaming cdcit old streaming advanced cdict 412933 44956
625 github multithreaded level -1 with dict old streaming cdcit old streaming advanced cdict 412933 42383
626 github multithreaded long distance mode level 0 with dict old streaming cdcit old streaming advanced cdict 412933 41113
627 github small window log level 1 with dict old streaming cdcit old streaming advanced cdict 412933 42430
628 github small hash log level 3 with dict old streaming cdcit old streaming advanced cdict 412933 41113
629 github small chain log level 4 with dict old streaming cdcit old streaming advanced cdict 412933 41084
630 github explicit params level 5 with dict old streaming cdcit old streaming advanced cdict 412933 39158
631 github uncompressed literals level 6 with dict old streaming cdcit old streaming advanced cdict 136311 38748
632 github uncompressed literals optimal level 7 with dict old streaming cdcit old streaming advanced cdict 133717 38744
633 github huffman literals level 9 with dict old streaming cdcit old streaming advanced cdict 175568 38986
634 github multithreaded with advanced params level 13 with dict old streaming cdcit old streaming advanced cdict 412933 39724
635 silesia github level -5 level 16 with dict old streaming advanced cdict 6882466 40771
silesia level -3 old streaming advanced cdict 6568358
silesia level -1 old streaming advanced cdict 6183385
silesia level 0 old streaming advanced cdict 4849491
silesia level 1 old streaming advanced cdict 5314109
silesia level 3 old streaming advanced cdict 4849491
silesia level 4 old streaming advanced cdict 4786913
silesia level 5 old streaming advanced cdict 4710178
silesia level 6 old streaming advanced cdict 4659996
silesia level 7 old streaming advanced cdict 4596234
silesia level 9 old streaming advanced cdict 4543862
silesia level 13 old streaming advanced cdict 4482073
silesia level 16 old streaming advanced cdict 4377389
silesia level 19 old streaming advanced cdict 4293262
silesia no source size old streaming advanced cdict 4849455
silesia long distance mode old streaming advanced cdict 12000408
silesia multithreaded old streaming advanced cdict 12000408
silesia multithreaded long distance mode old streaming advanced cdict 12000408
silesia small window log old streaming advanced cdict 12000408
silesia small hash log old streaming advanced cdict 12000408
silesia small chain log old streaming advanced cdict 12000408
silesia explicit params old streaming advanced cdict 12000408
silesia uncompressed literals old streaming advanced cdict 4849491
silesia uncompressed literals optimal old streaming advanced cdict 4293262
silesia huffman literals old streaming advanced cdict 6183385
silesia multithreaded with advanced params old streaming advanced cdict 12000408
silesia.tar level -5 old streaming advanced cdict 6982738
silesia.tar level -3 old streaming advanced cdict 6641264
silesia.tar level -1 old streaming advanced cdict 6190789
silesia.tar level 0 old streaming advanced cdict 4861376
silesia.tar level 1 old streaming advanced cdict 5336879
silesia.tar level 3 old streaming advanced cdict 4861376
silesia.tar level 4 old streaming advanced cdict 4799583
silesia.tar level 5 old streaming advanced cdict 4722276
silesia.tar level 6 old streaming advanced cdict 4672240
silesia.tar level 7 old streaming advanced cdict 4606657
silesia.tar level 9 old streaming advanced cdict 4554106
silesia.tar level 13 old streaming advanced cdict 4491707
silesia.tar level 16 old streaming advanced cdict 4381284
silesia.tar level 19 old streaming advanced cdict 4281511
silesia.tar no source size old streaming advanced cdict 4861372
silesia.tar long distance mode old streaming advanced cdict 12022046
silesia.tar multithreaded old streaming advanced cdict 12022046
silesia.tar multithreaded long distance mode old streaming advanced cdict 12022046
silesia.tar small window log old streaming advanced cdict 12022046
silesia.tar small hash log old streaming advanced cdict 12022046
silesia.tar small chain log old streaming advanced cdict 12022046
silesia.tar explicit params old streaming advanced cdict 12022046
silesia.tar uncompressed literals old streaming advanced cdict 4861376
silesia.tar uncompressed literals optimal old streaming advanced cdict 4281511
silesia.tar huffman literals old streaming advanced cdict 6190789
silesia.tar multithreaded with advanced params old streaming advanced cdict 12022046
github level -5 old streaming advanced cdict 205285
github level -5 with dict old streaming advanced cdict 46718
github level -3 old streaming advanced cdict 190643
github level -3 with dict old streaming advanced cdict 45395
github level -1 old streaming advanced cdict 175568
github level -1 with dict old streaming advanced cdict 43170
github level 0 old streaming advanced cdict 136311
github level 0 with dict old streaming advanced cdict 41148
github level 1 old streaming advanced cdict 142450
github level 1 with dict old streaming advanced cdict 41682
github level 3 old streaming advanced cdict 136311
github level 3 with dict old streaming advanced cdict 41148
github level 4 old streaming advanced cdict 136144
github level 4 with dict old streaming advanced cdict 41251
github level 5 old streaming advanced cdict 135106
github level 5 with dict old streaming advanced cdict 38938
github level 6 old streaming advanced cdict 135108
github level 6 with dict old streaming advanced cdict 38632
github level 7 old streaming advanced cdict 135108
github level 7 with dict old streaming advanced cdict 38766
github level 9 old streaming advanced cdict 135108
github level 9 with dict old streaming advanced cdict 39326
github level 13 old streaming advanced cdict 133717
github level 13 with dict old streaming advanced cdict 39716
github level 16 old streaming advanced cdict 133717
github level 16 with dict old streaming advanced cdict 37577
github level 19 old streaming advanced cdict 133717
636 github level 19 with dict old streaming advanced cdict 37576
github no source size old streaming advanced cdict 140631
github long distance mode old streaming advanced cdict 412933
github multithreaded old streaming advanced cdict 412933
github multithreaded long distance mode old streaming advanced cdict 412933
github small window log old streaming advanced cdict 412933
github small hash log old streaming advanced cdict 412933
github small chain log old streaming advanced cdict 412933
github explicit params old streaming advanced cdict 412933
github uncompressed literals old streaming advanced cdict 136311
github uncompressed literals optimal old streaming advanced cdict 133717
github huffman literals old streaming advanced cdict 175568
github multithreaded with advanced params old streaming advanced cdict 412933

38
zlibWrapper/Makefile
View File

@ -18,13 +18,15 @@ EXAMPLE_PATH = examples
PROGRAMS_PATH = ../programs
TEST_FILE = ../doc/zstd_compression_format.md
CPPFLAGS = -DXXH_NAMESPACE=ZSTD_ -I$(ZLIB_PATH) -I$(PROGRAMS_PATH) \
CPPFLAGS += -DXXH_NAMESPACE=ZSTD_ -I$(ZLIB_PATH) -I$(PROGRAMS_PATH) \
-I$(ZSTDLIBDIR) -I$(ZSTDLIBDIR)/common -I$(ZLIBWRAPPER_PATH)
CFLAGS ?= $(MOREFLAGS) -O3 -std=gnu99
CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wswitch-enum \
STDFLAGS = -std=c90 -pedantic -Wno-long-long -Wno-variadic-macros -Wc++-compat \
-DNO_snprintf -DNO_vsnprintf # strict ISO C90 is missing these prototypes
DEBUGFLAGS= -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wswitch-enum \
-Wdeclaration-after-statement -Wstrict-prototypes -Wundef \
-Wstrict-aliasing=1
CFLAGS ?= -O3
CFLAGS += $(STDFLAGS) $(DEBUGFLAGS) $(MOREFLAGS)
# Define *.exe as extension for Windows systems
ifneq (,$(filter Windows%,$(OS)))
@ -33,6 +35,11 @@ else
EXT =
endif
default : release
release : STDFLAGS =
release : DEBUGFLAGS =
release : all
all: fitblk example zwrapbench minigzip
@ -68,35 +75,34 @@ valgrindTest: clean example fitblk example_zstd fitblk_zstd zwrapbench
$(VALGRIND) ./zwrapbench -rqi1b1e5 ../lib ../programs ../tests
#.c.o:
# $(CC) $(CFLAGS) $(CPPFLAGS) -c -o $@ $<
# $(CC) $(CFLAGS) $(CPPFLAGS) -c $< -o $@
minigzip: $(EXAMPLE_PATH)/minigzip.o $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.o $(GZFILES) $(ZSTDLIBRARY)
minigzip: $(EXAMPLE_PATH)/minigzip.o zstd_zlibwrapper.o $(GZFILES) $(ZSTDLIBRARY)
$(CC) $(CFLAGS) $(CPPFLAGS) $(LDFLAGS) $^ $(ZSTDLIBRARY) $(ZLIB_LIBRARY) -o $@
minigzip_zstd: $(EXAMPLE_PATH)/minigzip.o $(ZLIBWRAPPER_PATH)/zstdTurnedOn_zlibwrapper.o $(GZFILES) $(ZSTDLIBRARY)
minigzip_zstd: $(EXAMPLE_PATH)/minigzip.o zstdTurnedOn_zlibwrapper.o $(GZFILES) $(ZSTDLIBRARY)
$(CC) $(CFLAGS) $(CPPFLAGS) $(LDFLAGS) $^ $(ZSTDLIBRARY) $(ZLIB_LIBRARY) -o $@
example: $(EXAMPLE_PATH)/example.o $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.o $(GZFILES) $(ZSTDLIBRARY)
example: $(EXAMPLE_PATH)/example.o zstd_zlibwrapper.o $(GZFILES) $(ZSTDLIBRARY)
$(CC) $(CFLAGS) $(CPPFLAGS) $(LDFLAGS) $^ $(ZLIB_LIBRARY) -o $@
example_zstd: $(EXAMPLE_PATH)/example.o $(ZLIBWRAPPER_PATH)/zstdTurnedOn_zlibwrapper.o $(GZFILES) $(ZSTDLIBRARY)
example_zstd: $(EXAMPLE_PATH)/example.o zstdTurnedOn_zlibwrapper.o $(GZFILES) $(ZSTDLIBRARY)
$(CC) $(CFLAGS) $(CPPFLAGS) $(LDFLAGS) $^ $(ZLIB_LIBRARY) -o $@
fitblk: $(EXAMPLE_PATH)/fitblk.o $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.o $(ZSTDLIBRARY)
fitblk: $(EXAMPLE_PATH)/fitblk.o zstd_zlibwrapper.o $(ZSTDLIBRARY)
$(CC) $(CFLAGS) $(CPPFLAGS) $(LDFLAGS) $^ $(ZLIB_LIBRARY) -o $@
fitblk_zstd: $(EXAMPLE_PATH)/fitblk.o $(ZLIBWRAPPER_PATH)/zstdTurnedOn_zlibwrapper.o $(ZSTDLIBRARY)
fitblk_zstd: $(EXAMPLE_PATH)/fitblk.o zstdTurnedOn_zlibwrapper.o $(ZSTDLIBRARY)
$(CC) $(CFLAGS) $(CPPFLAGS) $(LDFLAGS) $^ $(ZLIB_LIBRARY) -o $@
zwrapbench: $(EXAMPLE_PATH)/zwrapbench.o $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.o $(PROGRAMS_PATH)/util.o $(PROGRAMS_PATH)/timefn.o $(PROGRAMS_PATH)/datagen.o $(ZSTDLIBRARY)
zwrapbench: $(EXAMPLE_PATH)/zwrapbench.o zstd_zlibwrapper.o $(PROGRAMS_PATH)/util.o $(PROGRAMS_PATH)/timefn.o $(PROGRAMS_PATH)/datagen.o $(ZSTDLIBRARY)
$(CC) $(CFLAGS) $(CPPFLAGS) $(LDFLAGS) $^ $(ZLIB_LIBRARY) -o $@
$(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.o: $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.c $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.h
$(CC) $(CFLAGS) $(CPPFLAGS) -I. -c -o $@ $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.c
zstd_zlibwrapper.o: $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.c $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.h
$(ZLIBWRAPPER_PATH)/zstdTurnedOn_zlibwrapper.o: $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.c $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.h
$(CC) $(CFLAGS) $(CPPFLAGS) -DZWRAP_USE_ZSTD=1 -I. -c -o $@ $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.c
zstdTurnedOn_zlibwrapper.o: CPPFLAGS += -DZWRAP_USE_ZSTD=1
zstdTurnedOn_zlibwrapper.o: $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.c $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.h
$(ZSTDLIBDIR)/libzstd.a:
$(MAKE) -C $(ZSTDLIBDIR) libzstd.a

12
zlibWrapper/examples/fitblk.c
View File

@ -159,7 +159,7 @@ int main(int argc, char **argv)
if (ZWRAP_isUsingZSTDcompression()) printf("zstd version %s\n", zstdVersion());
/* allocate memory for buffers and compression engine */
blk = malloc(size + EXCESS);
blk = (unsigned char*)malloc(size + EXCESS);
def.zalloc = Z_NULL;
def.zfree = Z_NULL;
def.opaque = Z_NULL;
@ -180,8 +180,8 @@ int main(int argc, char **argv)
if (ret == Z_STREAM_END && def.avail_out >= EXCESS) {
/* write block to stdout */
have = size + EXCESS - def.avail_out;
// if (fwrite(blk, 1, have, stdout) != have || ferror(stdout))
// quit("error writing output");
/* if (fwrite(blk, 1, have, stdout) != have || ferror(stdout))
* quit("error writing output"); */
/* clean up and print results to stderr */
ret = deflateEnd(&def);
@ -200,7 +200,7 @@ int main(int argc, char **argv)
inf.avail_in = 0;
inf.next_in = Z_NULL;
ret = inflateInit(&inf);
tmp = malloc(size + EXCESS);
tmp = (unsigned char*)malloc(size + EXCESS);
if (ret != Z_OK || tmp == NULL)
quit("out of memory");
ret = deflateReset(&def);
@ -237,8 +237,8 @@ int main(int argc, char **argv)
/* done -- write block to stdout */
have = size - def.avail_out;
// if (fwrite(blk, 1, have, stdout) != have || ferror(stdout))
// quit("error writing output");
/* if (fwrite(blk, 1, have, stdout) != have || ferror(stdout))
* quit("error writing output"); */
/* clean up and print results to stderr */
free(tmp);

24
zlibWrapper/examples/zwrapbench.c
View File

@ -311,14 +311,14 @@ static int BMK_benchMem(z_const void* srcBuffer, size_t srcSize,
ret = deflateReset(&def);
if (ret != Z_OK) EXM_THROW(1, "deflateReset failure");
if (useSetDict) {
ret = deflateSetDictionary(&def, dictBuffer, dictBufferSize);
ret = deflateSetDictionary(&def, (const z_Bytef*)dictBuffer, dictBufferSize);
if (ret != Z_OK) EXM_THROW(1, "deflateSetDictionary failure");
if (ZWRAP_isUsingZSTDcompression()) useSetDict = 0; /* zstd doesn't require deflateSetDictionary after ZWRAP_deflateReset_keepDict */
}
def.next_in = (z_const void*) blockTable[blockNb].srcPtr;
def.next_in = (z_const z_Bytef*) blockTable[blockNb].srcPtr;
def.avail_in = (uInt)blockTable[blockNb].srcSize;
def.total_in = 0;
def.next_out = (void*) blockTable[blockNb].cPtr;
def.next_out = (z_Bytef*) blockTable[blockNb].cPtr;
def.avail_out = (uInt)blockTable[blockNb].cRoom;
def.total_out = 0;
ret = deflate(&def, Z_FINISH);
@ -343,13 +343,13 @@ static int BMK_benchMem(z_const void* srcBuffer, size_t srcSize,
ret = deflateInit(&def, cLevel);
if (ret != Z_OK) EXM_THROW(1, "deflateInit failure");
if (dictBuffer) {
ret = deflateSetDictionary(&def, dictBuffer, dictBufferSize);
ret = deflateSetDictionary(&def, (const z_Bytef*)dictBuffer, dictBufferSize);
if (ret != Z_OK) EXM_THROW(1, "deflateSetDictionary failure");
}
def.next_in = (z_const void*) blockTable[blockNb].srcPtr;
def.next_in = (z_const z_Bytef*) blockTable[blockNb].srcPtr;
def.avail_in = (uInt)blockTable[blockNb].srcSize;
def.total_in = 0;
def.next_out = (void*) blockTable[blockNb].cPtr;
def.next_out = (z_Bytef*) blockTable[blockNb].cPtr;
def.avail_out = (uInt)blockTable[blockNb].cRoom;
def.total_out = 0;
ret = deflate(&def, Z_FINISH);
@ -451,15 +451,15 @@ static int BMK_benchMem(z_const void* srcBuffer, size_t srcSize,
else
ret = inflateReset(&inf);
if (ret != Z_OK) EXM_THROW(1, "inflateReset failure");
inf.next_in = (z_const void*) blockTable[blockNb].cPtr;
inf.next_in = (z_const z_Bytef*) blockTable[blockNb].cPtr;
inf.avail_in = (uInt)blockTable[blockNb].cSize;
inf.total_in = 0;
inf.next_out = (void*) blockTable[blockNb].resPtr;
inf.next_out = (z_Bytef*) blockTable[blockNb].resPtr;
inf.avail_out = (uInt)blockTable[blockNb].srcSize;
inf.total_out = 0;
ret = inflate(&inf, Z_FINISH);
if (ret == Z_NEED_DICT) {
ret = inflateSetDictionary(&inf, dictBuffer, dictBufferSize);
ret = inflateSetDictionary(&inf, (const z_Bytef*)dictBuffer, dictBufferSize);
if (ret != Z_OK) EXM_THROW(1, "inflateSetDictionary failure");
ret = inflate(&inf, Z_FINISH);
}
@ -483,15 +483,15 @@ static int BMK_benchMem(z_const void* srcBuffer, size_t srcSize,
inf.opaque = Z_NULL;
ret = inflateInit(&inf);
if (ret != Z_OK) EXM_THROW(1, "inflateInit failure");
inf.next_in = (z_const void*) blockTable[blockNb].cPtr;
inf.next_in = (z_const z_Bytef*) blockTable[blockNb].cPtr;
inf.avail_in = (uInt)blockTable[blockNb].cSize;
inf.total_in = 0;
inf.next_out = (void*) blockTable[blockNb].resPtr;
inf.next_out = (z_Bytef*) blockTable[blockNb].resPtr;
inf.avail_out = (uInt)blockTable[blockNb].srcSize;
inf.total_out = 0;
ret = inflate(&inf, Z_FINISH);
if (ret == Z_NEED_DICT) {
ret = inflateSetDictionary(&inf, dictBuffer, dictBufferSize);
ret = inflateSetDictionary(&inf, (const z_Bytef*) dictBuffer, dictBufferSize);
if (ret != Z_OK) EXM_THROW(1, "inflateSetDictionary failure");
ret = inflate(&inf, Z_FINISH);
}

2
zlibWrapper/gzclose.c
View File

@ -19,7 +19,7 @@ int ZEXPORT gzclose(file)
if (file == NULL)
return Z_STREAM_ERROR;
state = (gz_statep)file;
state.file = file;
return state.state->mode == GZ_READ ? gzclose_r(file) : gzclose_w(file);
#else

18
zlibWrapper/gzlib.c
View File

@ -216,7 +216,7 @@ local gzFile gz_open(path, fd, mode)
#if !defined(NO_snprintf) && !defined(NO_vsnprintf)
(void)snprintf(state.state->path, len + 1, "%s", (const char *)path);
#else
strcpy(state.state->path, path);
strcpy(state.state->path, (const char*)path);
#endif
/* compute the flags for open() */
@ -325,7 +325,7 @@ int ZEXPORT gzbuffer(file, size)
/* get internal structure and check integrity */
if (file == NULL)
return -1;
state = (gz_statep)file;
state.file = file;
if (state.state->mode != GZ_READ && state.state->mode != GZ_WRITE)
return -1;
@ -351,7 +351,7 @@ int ZEXPORT gzrewind(file)
/* get internal structure */
if (file == NULL)
return -1;
state = (gz_statep)file;
state.file = file;
/* check that we're reading and that there's no error */
if (state.state->mode != GZ_READ ||
@ -378,7 +378,7 @@ z_off64_t ZEXPORT gzseek64(file, offset, whence)
/* get internal structure and check integrity */
if (file == NULL)
return -1;
state = (gz_statep)file;
state.file = file;
if (state.state->mode != GZ_READ && state.state->mode != GZ_WRITE)
return -1;
@ -463,7 +463,7 @@ z_off64_t ZEXPORT gztell64(file)
/* get internal structure and check integrity */
if (file == NULL)
return -1;
state = (gz_statep)file;
state.file = file;
if (state.state->mode != GZ_READ && state.state->mode != GZ_WRITE)
return -1;
@ -491,7 +491,7 @@ z_off64_t ZEXPORT gzoffset64(file)
/* get internal structure and check integrity */
if (file == NULL)
return -1;
state = (gz_statep)file;
state.file = file;
if (state.state->mode != GZ_READ && state.state->mode != GZ_WRITE)
return -1;
@ -523,7 +523,7 @@ int ZEXPORT gzeof(file)
/* get internal structure and check integrity */
if (file == NULL)
return 0;
state = (gz_statep)file;
state.file = file;
if (state.state->mode != GZ_READ && state.state->mode != GZ_WRITE)
return 0;
@ -541,7 +541,7 @@ const char * ZEXPORT gzerror(file, errnum)
/* get internal structure and check integrity */
if (file == NULL)
return NULL;
state = (gz_statep)file;
state.file = file;
if (state.state->mode != GZ_READ && state.state->mode != GZ_WRITE)
return NULL;
@ -561,7 +561,7 @@ void ZEXPORT gzclearerr(file)
/* get internal structure and check integrity */
if (file == NULL)
return;
state = (gz_statep)file;
state.file = file;
if (state.state->mode != GZ_READ && state.state->mode != GZ_WRITE)
return;

16
zlibWrapper/gzread.c
View File

@ -386,7 +386,7 @@ int ZEXPORT gzread(file, buf, len)
/* get internal structure */
if (file == NULL)
return -1;
state = (gz_statep)file;
state.file = file;
/* check that we're reading and that there's no (serious) error */
if (state.state->mode != GZ_READ ||
@ -424,7 +424,7 @@ z_size_t ZEXPORT gzfread(buf, size, nitems, file)
/* get internal structure */
if (file == NULL)
return 0;
state = (gz_statep)file;
state.file = file;
/* check that we're reading and that there's no (serious) error */
if (state.state->mode != GZ_READ ||
@ -470,7 +470,7 @@ int ZEXPORT gzgetc(file)
/* get internal structure */
if (file == NULL)
return -1;
state = (gz_statep)file;
state.file = file;
/* check that we're reading and that there's no (serious) error */
if (state.state->mode != GZ_READ ||
@ -485,7 +485,7 @@ int ZEXPORT gzgetc(file)
}
/* nothing there -- try gz_read() */
ret = (unsigned)gz_read(state, buf, 1);
ret = (int)gz_read(state, buf, 1);
return ret < 1 ? -1 : buf[0];
}
@ -505,7 +505,7 @@ int ZEXPORT gzungetc(c, file)
/* get internal structure */
if (file == NULL)
return -1;
state = (gz_statep)file;
state.file = file;
/* check that we're reading and that there's no (serious) error */
if (state.state->mode != GZ_READ ||
@ -569,7 +569,7 @@ char * ZEXPORT gzgets(file, buf, len)
/* check parameters and get internal structure */
if (file == NULL || buf == NULL || len < 1)
return NULL;
state = (gz_statep)file;
state.file = file;
/* check that we're reading and that there's no (serious) error */
if (state.state->mode != GZ_READ ||
@ -628,7 +628,7 @@ int ZEXPORT gzdirect(file)
/* get internal structure */
if (file == NULL)
return 0;
state = (gz_statep)file;
state.file = file;
/* if the state is not known, but we can find out, then do so (this is
mainly for right after a gzopen() or gzdopen()) */
@ -649,7 +649,7 @@ int ZEXPORT gzclose_r(file)
/* get internal structure */
if (file == NULL)
return Z_STREAM_ERROR;
state = (gz_statep)file;
state.file = file;
/* check that we're reading */
if (state.state->mode != GZ_READ)

16
zlibWrapper/gzwrite.c
View File

@ -258,7 +258,7 @@ int ZEXPORT gzwrite(file, buf, len)
/* get internal structure */
if (file == NULL)
return 0;
state = (gz_statep)file;
state.file = file;
/* check that we're writing and that there's no error */
if (state.state->mode != GZ_WRITE || state.state->err != Z_OK)
@ -289,7 +289,7 @@ z_size_t ZEXPORT gzfwrite(buf, size, nitems, file)
assert(size != 0);
if (file == NULL)
return 0;
state = (gz_statep)file;
state.file = file;
/* check that we're writing and that there's no error */
if (state.state->mode != GZ_WRITE || state.state->err != Z_OK)
@ -319,7 +319,7 @@ int ZEXPORT gzputc(file, c)
/* get internal structure */
if (file == NULL)
return -1;
state = (gz_statep)file;
state.file = file;
strm = &(state.state->strm);
/* check that we're writing and that there's no error */
@ -366,7 +366,7 @@ int ZEXPORT gzputs(file, str)
/* get internal structure */
if (file == NULL)
return -1;
state = (gz_statep)file;
state.file = file;
/* check that we're writing and that there's no error */
if (state.state->mode != GZ_WRITE || state.state->err != Z_OK)
@ -393,7 +393,7 @@ int ZEXPORTVA gzvprintf(gzFile file, const char *format, va_list va)
/* get internal structure */
if (file == NULL)
return Z_STREAM_ERROR;
state = (gz_statep)file;
state.file = file;
strm = &(state.state->strm);
/* check that we're writing and that there's no error */
@ -565,7 +565,7 @@ int ZEXPORT gzflush(file, flush)
/* get internal structure */
if (file == NULL)
return Z_STREAM_ERROR;
state = (gz_statep)file;
state.file = file;
/* check that we're writing and that there's no error */
if (state.state->mode != GZ_WRITE || state.state->err != Z_OK)
@ -599,7 +599,7 @@ int ZEXPORT gzsetparams(file, level, strategy)
/* get internal structure */
if (file == NULL)
return Z_STREAM_ERROR;
state = (gz_statep)file;
state.file = file;
strm = &(state.state->strm);
/* check that we're writing and that there's no error */
@ -639,7 +639,7 @@ int ZEXPORT gzclose_w(file)
/* get internal structure */
if (file == NULL)
return Z_STREAM_ERROR;
state = (gz_statep)file;
state.file = file;
/* check that we're writing */
if (state.state->mode != GZ_WRITE)

26
zlibWrapper/zstd_zlibwrapper.c
View File

@ -31,7 +31,7 @@
/* === Constants === */
#define Z_INFLATE_SYNC 8
#define ZLIB_HEADERSIZE 4
#define ZSTD_HEADERSIZE ZSTD_FRAMEHEADERSIZE_MIN
#define ZSTD_HEADERSIZE ZSTD_FRAMEHEADERSIZE_MIN(ZSTD_f_zstd1)
#define ZWRAP_DEFAULT_CLEVEL 3 /* Z_DEFAULT_COMPRESSION is translated to ZWRAP_DEFAULT_CLEVEL for zstd */
@ -54,7 +54,7 @@ int ZWRAP_isUsingZSTDcompression(void) { return g_ZWRAP_useZSTDcompression; }
static ZWRAP_decompress_type g_ZWRAPdecompressionType = ZWRAP_AUTO;
void ZWRAP_setDecompressionType(ZWRAP_decompress_type type) { g_ZWRAPdecompressionType = type; };
void ZWRAP_setDecompressionType(ZWRAP_decompress_type type) { g_ZWRAPdecompressionType = type; }
ZWRAP_decompress_type ZWRAP_getDecompressionType(void) { return g_ZWRAPdecompressionType; }
@ -99,7 +99,7 @@ typedef struct {
unsigned long long pledgedSrcSize;
} ZWRAP_CCtx;
typedef ZWRAP_CCtx internal_state;
/* typedef ZWRAP_CCtx internal_state; */
@ -121,8 +121,10 @@ static ZWRAP_CCtx* ZWRAP_createCCtx(z_streamp strm)
if (zwc==NULL) return NULL;
memset(zwc, 0, sizeof(ZWRAP_CCtx));
memcpy(&zwc->allocFunc, strm, sizeof(z_stream));
{ ZSTD_customMem const ZWRAP_customMem = { ZWRAP_allocFunction, ZWRAP_freeFunction, &zwc->allocFunc };
zwc->customMem = ZWRAP_customMem; }
{ ZSTD_customMem ZWRAP_customMem = { ZWRAP_allocFunction, ZWRAP_freeFunction, NULL };
ZWRAP_customMem.opaque = &zwc->allocFunc;
zwc->customMem = ZWRAP_customMem;
}
} else {
zwc = (ZWRAP_CCtx*)calloc(1, sizeof(*zwc));
if (zwc==NULL) return NULL;
@ -455,15 +457,17 @@ static void ZWRAP_initDCtx(ZWRAP_DCtx* zwd)
static ZWRAP_DCtx* ZWRAP_createDCtx(z_streamp strm)
{
ZWRAP_DCtx* zwd;
MEM_STATIC_ASSERT(sizeof(zwd->headerBuf) >= ZSTD_FRAMEHEADERSIZE_MIN); /* check static buffer size condition */
MEM_STATIC_ASSERT(sizeof(zwd->headerBuf) >= ZSTD_HEADERSIZE); /* check static buffer size condition */
if (strm->zalloc && strm->zfree) {
zwd = (ZWRAP_DCtx*)strm->zalloc(strm->opaque, 1, sizeof(ZWRAP_DCtx));
if (zwd==NULL) return NULL;
memset(zwd, 0, sizeof(ZWRAP_DCtx));
zwd->allocFunc = *strm; /* just to copy zalloc, zfree & opaque */
{ ZSTD_customMem const ZWRAP_customMem = { ZWRAP_allocFunction, ZWRAP_freeFunction, &zwd->allocFunc };
zwd->customMem = ZWRAP_customMem; }
{ ZSTD_customMem ZWRAP_customMem = { ZWRAP_allocFunction, ZWRAP_freeFunction, NULL };
ZWRAP_customMem.opaque = &zwd->allocFunc;
zwd->customMem = ZWRAP_customMem;
}
} else {
zwd = (ZWRAP_DCtx*)calloc(1, sizeof(*zwd));
if (zwd==NULL) return NULL;
@ -509,7 +513,7 @@ static int ZWRAPD_finishWithErrorMsg(z_streamp strm, char* message)
ZEXTERN int ZEXPORT z_inflateInit_ OF((z_streamp strm,
const char *version, int stream_size))
const char* version, int stream_size))
{
if (g_ZWRAPdecompressionType == ZWRAP_FORCE_ZLIB) {
strm->reserved = ZWRAP_ZLIB_STREAM;
@ -520,7 +524,7 @@ ZEXTERN int ZEXPORT z_inflateInit_ OF((z_streamp strm,
LOG_WRAPPERD("- inflateInit\n");
if (zwd == NULL) return ZWRAPD_finishWithError(zwd, strm, 0);
zwd->version = ZSTD_malloc(strlen(version)+1, zwd->customMem);
zwd->version = (char*)ZSTD_malloc(strlen(version)+1, zwd->customMem);
if (zwd->version == NULL) return ZWRAPD_finishWithError(zwd, strm, 0);
strcpy(zwd->version, version);
@ -1003,7 +1007,7 @@ ZEXTERN int ZEXPORT z_inflateBackEnd OF((z_streamp strm))
}
ZEXTERN uLong ZEXPORT z_zlibCompileFlags OF((void)) { return zlibCompileFlags(); };
ZEXTERN uLong ZEXPORT z_zlibCompileFlags OF((void)) { return zlibCompileFlags(); }