Spelling (#1582)
* spelling: accidentally * spelling: across * spelling: additionally * spelling: addresses * spelling: appropriate * spelling: assumed * spelling: available * spelling: builder * spelling: capacity * spelling: compiler * spelling: compressibility * spelling: compressor * spelling: compression * spelling: contract * spelling: convenience * spelling: decompress * spelling: description * spelling: deflate * spelling: deterministically * spelling: dictionary * spelling: display * spelling: eliminate * spelling: preemptively * spelling: exclude * spelling: failure * spelling: independence * spelling: independent * spelling: intentionally * spelling: matching * spelling: maximum * spelling: meaning * spelling: mishandled * spelling: memory * spelling: occasionally * spelling: occurrence * spelling: official * spelling: offsets * spelling: original * spelling: output * spelling: overflow * spelling: overridden * spelling: parameter * spelling: performance * spelling: probability * spelling: receives * spelling: redundant * spelling: recompression * spelling: resources * spelling: sanity * spelling: segment * spelling: series * spelling: specified * spelling: specify * spelling: subtracted * spelling: successful * spelling: return * spelling: translation * spelling: update * spelling: unrelated * spelling: useless * spelling: variables * spelling: variety * spelling: verbatim * spelling: verification * spelling: visited * spelling: warming * spelling: workers * spelling: withdev
Josh Soref
Nick Terrell
parent
13fddac46d
commit
a880ca239b
|
|
@ -243,7 +243,7 @@ v1.0.0
|
|||
Change Licensing, all project is now BSD, Copyright Facebook
|
||||
Small decompression speed improvement
|
||||
API : Streaming API supports legacy format
|
||||
API : ZDICT_getDictID(), ZSTD_sizeof_{CCtx, DCtx, CStream, DStream}(), ZSTD_setDStreamParamter()
|
||||
API : ZDICT_getDictID(), ZSTD_sizeof_{CCtx, DCtx, CStream, DStream}(), ZSTD_setDStreamParameter()
|
||||
CLI supports legacy formats v0.4+
|
||||
Fixed : compression fails on certain huge files, reported by Jesse McGrew
|
||||
Enhanced documentation, by Przemyslaw Skibinski
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@
|
|||
Contributions to the cmake build configurations are welcome. Please
|
||||
use case sensitivity that matches modern (ie. cmake version 2.6 and above)
|
||||
conventions of using lower-case for commands, and upper-case for
|
||||
varibles.
|
||||
variables.
|
||||
|
||||
# CMake Style Recommendations
|
||||
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ The `Dockerfile` script requires a version of `docker` >= 17.05
|
|||
|
||||
## Installing docker
|
||||
|
||||
The officiel docker install docs use a ppa with a modern version available:
|
||||
The official docker install docs use a ppa with a modern version available:
|
||||
https://docs.docker.com/install/linux/docker-ce/ubuntu/
|
||||
|
||||
## How to run
|
||||
|
|
|
|||
|
|
@ -127,7 +127,7 @@ dictInfo* createDictFromFiles(sampleInfo *info, unsigned maxDictSize,
|
|||
|
||||
|
||||
/** compressWithDict() :
|
||||
* Compress samples from sample buffer given dicionary stored on dictionary buffer and compression level
|
||||
* Compress samples from sample buffer given dictionary stored on dictionary buffer and compression level
|
||||
* @return compression ratio
|
||||
*/
|
||||
double compressWithDict(sampleInfo *srcInfo, dictInfo* dInfo, int compressionLevel, int displayLevel) {
|
||||
|
|
@ -194,7 +194,7 @@ double compressWithDict(sampleInfo *srcInfo, dictInfo* dInfo, int compressionLev
|
|||
totalCompressedSize += compressedSize;
|
||||
}
|
||||
|
||||
/* Sum orignal sizes */
|
||||
/* Sum original sizes */
|
||||
for (i = 0; i<srcInfo->nbSamples; i++) {
|
||||
totalOriginalSize += srcInfo->samplesSizes[i];
|
||||
}
|
||||
|
|
|
|||
|
|
@ -125,7 +125,7 @@ typedef struct {
|
|||
*
|
||||
* Score(S) = F(S_1) + F(S_2) + ... + F(S_{k-d+1})
|
||||
*
|
||||
* Once the dmer with hash value d is in the dictionay we set F(d) = F(d)/2.
|
||||
* Once the dmer with hash value d is in the dictionary we set F(d) = F(d)/2.
|
||||
*/
|
||||
static FASTCOVER_segment_t FASTCOVER_selectSegment(const FASTCOVER_ctx_t *ctx,
|
||||
U32 *freqs, U32 begin,U32 end,
|
||||
|
|
@ -149,7 +149,7 @@ static FASTCOVER_segment_t FASTCOVER_selectSegment(const FASTCOVER_ctx_t *ctx,
|
|||
while (activeSegment.end < end) {
|
||||
/* Get hash value of current dmer */
|
||||
const size_t index = FASTCOVER_hashPtrToIndex(ctx->samples + activeSegment.end, parameters.f, ctx->d);
|
||||
/* Add frequency of this index to score if this is the first occurence of index in active segment */
|
||||
/* Add frequency of this index to score if this is the first occurrence of index in active segment */
|
||||
if (ctx->segmentFreqs[index] == 0) {
|
||||
activeSegment.score += freqs[index];
|
||||
}
|
||||
|
|
|
|||
|
|
@ -424,7 +424,7 @@ static ddict_collection_t createDDictCollection(const void* dictBuffer, size_t d
|
|||
}
|
||||
|
||||
|
||||
/* mess with adresses, so that linear scanning dictionaries != linear address scanning */
|
||||
/* mess with addresses, so that linear scanning dictionaries != linear address scanning */
|
||||
void shuffleDictionaries(ddict_collection_t dicts)
|
||||
{
|
||||
size_t const nbDicts = dicts.nbDDict;
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ Date: Mon, 17 Jul 2017 17:08:19 -0700
|
|||
Subject: [PATCH v5 2/5] lib: Add zstd modules
|
||||
|
||||
Add zstd compression and decompression kernel modules.
|
||||
zstd offers a wide varity of compression speed and quality trade-offs.
|
||||
zstd offers a wide variety of compression speed and quality trade-offs.
|
||||
It can compress at speeds approaching lz4, and quality approaching lzma.
|
||||
zstd decompressions at speeds more than twice as fast as zlib, and
|
||||
decompression speed remains roughly the same across all compression levels.
|
||||
|
|
@ -21,7 +21,7 @@ will be easier to keep the kernel zstd up to date.
|
|||
I benchmarked zstd compression as a special character device. I ran zstd
|
||||
and zlib compression at several levels, as well as performing no
|
||||
compression, which measure the time spent copying the data to kernel space.
|
||||
Data is passed to the compresser 4096 B at a time. The benchmark file is
|
||||
Data is passed to the compressor 4096 B at a time. The benchmark file is
|
||||
located in the upstream zstd source repository under
|
||||
`contrib/linux-kernel/zstd_compress_test.c` [2].
|
||||
|
||||
|
|
@ -86,7 +86,7 @@ Tested in userland using the test-suite in the zstd repo under
|
|||
`contrib/linux-kernel/test/UserlandTest.cpp` [5] by mocking the kernel
|
||||
functions. Fuzz tested using libfuzzer [6] with the fuzz harnesses under
|
||||
`contrib/linux-kernel/test/{RoundTripCrash.c,DecompressCrash.c}` [7] [8]
|
||||
with ASAN, UBSAN, and MSAN. Additionaly, it was tested while testing the
|
||||
with ASAN, UBSAN, and MSAN. Additionally, it was tested while testing the
|
||||
BtrFS and SquashFS patches coming next.
|
||||
|
||||
[1] https://clang.llvm.org/docs/ClangFormat.html
|
||||
|
|
@ -4200,14 +4200,14 @@ index 0000000..ff18ae6
|
|||
+ BYTE const windowLogByte = (BYTE)((params.cParams.windowLog - ZSTD_WINDOWLOG_ABSOLUTEMIN) << 3);
|
||||
+ U32 const fcsCode =
|
||||
+ params.fParams.contentSizeFlag ? (pledgedSrcSize >= 256) + (pledgedSrcSize >= 65536 + 256) + (pledgedSrcSize >= 0xFFFFFFFFU) : 0; /* 0-3 */
|
||||
+ BYTE const frameHeaderDecriptionByte = (BYTE)(dictIDSizeCode + (checksumFlag << 2) + (singleSegment << 5) + (fcsCode << 6));
|
||||
+ BYTE const frameHeaderDescriptionByte = (BYTE)(dictIDSizeCode + (checksumFlag << 2) + (singleSegment << 5) + (fcsCode << 6));
|
||||
+ size_t pos;
|
||||
+
|
||||
+ if (dstCapacity < ZSTD_frameHeaderSize_max)
|
||||
+ return ERROR(dstSize_tooSmall);
|
||||
+
|
||||
+ ZSTD_writeLE32(dst, ZSTD_MAGICNUMBER);
|
||||
+ op[4] = frameHeaderDecriptionByte;
|
||||
+ op[4] = frameHeaderDescriptionByte;
|
||||
+ pos = 5;
|
||||
+ if (!singleSegment)
|
||||
+ op[pos++] = windowLogByte;
|
||||
|
|
@ -8812,8 +8812,8 @@ index 0000000..ef3d174
|
|||
+ U32 position = 0;
|
||||
+ U32 symbol;
|
||||
+ for (symbol = 0; symbol <= maxSymbolValue; symbol++) {
|
||||
+ int nbOccurences;
|
||||
+ for (nbOccurences = 0; nbOccurences < normalizedCounter[symbol]; nbOccurences++) {
|
||||
+ int nbOccurrences;
|
||||
+ for (nbOccurrences = 0; nbOccurrences < normalizedCounter[symbol]; nbOccurrences++) {
|
||||
+ tableSymbol[position] = (FSE_FUNCTION_TYPE)symbol;
|
||||
+ position = (position + step) & tableMask;
|
||||
+ while (position > highThreshold)
|
||||
|
|
@ -9944,7 +9944,7 @@ index 0000000..2143da2
|
|||
+ HUF_repeat_none, /**< Cannot use the previous table */
|
||||
+ HUF_repeat_check, /**< Can use the previous table but it must be checked. Note : The previous table must have been constructed by HUF_compress{1,
|
||||
+ 4}X_repeat */
|
||||
+ HUF_repeat_valid /**< Can use the previous table and it is asumed to be valid */
|
||||
+ HUF_repeat_valid /**< Can use the previous table and it is assumed to be valid */
|
||||
+} HUF_repeat;
|
||||
+/** HUF_compress4X_repeat() :
|
||||
+* Same as HUF_compress4X_wksp(), but considers using hufTable if *repeat != HUF_repeat_none.
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ Signed-off-by: Nick Terrell <terrelln@fb.com>
|
|||
---
|
||||
v4 -> v5:
|
||||
- Fix patch documentation to reflect that Sean Purcell is the author
|
||||
- Don't strip trailing whitespace of unreleated code
|
||||
- Don't strip trailing whitespace of unrelated code
|
||||
- Make zstd_display_options() static
|
||||
|
||||
v5 -> v6:
|
||||
|
|
@ -224,7 +224,7 @@ index 0000000..dcab75a
|
|||
+ * set the default options, this is to ensure any user supplied
|
||||
+ * -X options on the appending mksquashfs command line are over-ridden.
|
||||
+ *
|
||||
+ * This function returns 0 on sucessful extraction of options, and -1 on error.
|
||||
+ * This function returns 0 on successful extraction of options, and -1 on error.
|
||||
+ */
|
||||
+static int zstd_extract_options(int block_size, void *buffer, int size)
|
||||
+{
|
||||
|
|
|
|||
|
|
@ -2436,14 +2436,14 @@ static size_t ZSTD_writeFrameHeader(void *dst, size_t dstCapacity, ZSTD_paramete
|
|||
BYTE const windowLogByte = (BYTE)((params.cParams.windowLog - ZSTD_WINDOWLOG_ABSOLUTEMIN) << 3);
|
||||
U32 const fcsCode =
|
||||
params.fParams.contentSizeFlag ? (pledgedSrcSize >= 256) + (pledgedSrcSize >= 65536 + 256) + (pledgedSrcSize >= 0xFFFFFFFFU) : 0; /* 0-3 */
|
||||
BYTE const frameHeaderDecriptionByte = (BYTE)(dictIDSizeCode + (checksumFlag << 2) + (singleSegment << 5) + (fcsCode << 6));
|
||||
BYTE const frameHeaderDescriptionByte = (BYTE)(dictIDSizeCode + (checksumFlag << 2) + (singleSegment << 5) + (fcsCode << 6));
|
||||
size_t pos;
|
||||
|
||||
if (dstCapacity < ZSTD_frameHeaderSize_max)
|
||||
return ERROR(dstSize_tooSmall);
|
||||
|
||||
ZSTD_writeLE32(dst, ZSTD_MAGICNUMBER);
|
||||
op[4] = frameHeaderDecriptionByte;
|
||||
op[4] = frameHeaderDescriptionByte;
|
||||
pos = 5;
|
||||
if (!singleSegment)
|
||||
op[pos++] = windowLogByte;
|
||||
|
|
|
|||
|
|
@ -141,8 +141,8 @@ size_t FSE_buildCTable_wksp(FSE_CTable *ct, const short *normalizedCounter, unsi
|
|||
U32 position = 0;
|
||||
U32 symbol;
|
||||
for (symbol = 0; symbol <= maxSymbolValue; symbol++) {
|
||||
int nbOccurences;
|
||||
for (nbOccurences = 0; nbOccurences < normalizedCounter[symbol]; nbOccurences++) {
|
||||
int nbOccurrences;
|
||||
for (nbOccurrences = 0; nbOccurrences < normalizedCounter[symbol]; nbOccurrences++) {
|
||||
tableSymbol[position] = (FSE_FUNCTION_TYPE)symbol;
|
||||
position = (position + step) & tableMask;
|
||||
while (position > highThreshold)
|
||||
|
|
|
|||
|
|
@ -134,7 +134,7 @@ typedef enum {
|
|||
HUF_repeat_none, /**< Cannot use the previous table */
|
||||
HUF_repeat_check, /**< Can use the previous table but it must be checked. Note : The previous table must have been constructed by HUF_compress{1,
|
||||
4}X_repeat */
|
||||
HUF_repeat_valid /**< Can use the previous table and it is asumed to be valid */
|
||||
HUF_repeat_valid /**< Can use the previous table and it is assumed to be valid */
|
||||
} HUF_repeat;
|
||||
/** HUF_compress4X_repeat() :
|
||||
* Same as HUF_compress4X_wksp(), but considers using hufTable if *repeat != HUF_repeat_none.
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
#ifndef LINUX_COMIPLER_H_
|
||||
#define LINUX_COMIPLER_H_
|
||||
#ifndef LINUX_COMPILER_H_
|
||||
#define LINUX_COMPILER_H_
|
||||
|
||||
#ifndef __always_inline
|
||||
# define __always_inline inline
|
||||
|
|
@ -9,4 +9,4 @@
|
|||
# define noinline __attribute__((__noinline__))
|
||||
#endif
|
||||
|
||||
#endif // LINUX_COMIPLER_H_
|
||||
#endif // LINUX_COMPILER_H_
|
||||
|
|
|
|||
|
|
@ -55,7 +55,7 @@ static std::uint64_t handleOneInput(const Options &options,
|
|||
SharedState& state) {
|
||||
auto inputSize = fileSizeOrZero(inputFile);
|
||||
// WorkQueue outlives ThreadPool so in the case of error we are certain
|
||||
// we don't accidently try to call push() on it after it is destroyed
|
||||
// we don't accidentally try to call push() on it after it is destroyed
|
||||
WorkQueue<std::shared_ptr<BufferWorkQueue>> outs{options.numThreads + 1};
|
||||
std::uint64_t bytesRead;
|
||||
std::uint64_t bytesWritten;
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@
|
|||
|
||||
/**
|
||||
* A subset of `folly/Range.h`.
|
||||
* All code copied verbatiam modulo formatting
|
||||
* All code copied verbatim modulo formatting
|
||||
*/
|
||||
#pragma once
|
||||
|
||||
|
|
|
|||
|
|
@ -54,7 +54,7 @@ class ResourcePool {
|
|||
|
||||
/**
|
||||
* @returns A unique pointer to a resource. The resource is null iff
|
||||
* there are no avaiable resources and `factory()` returns null.
|
||||
* there are no available resources and `factory()` returns null.
|
||||
*/
|
||||
UniquePtr get() {
|
||||
std::lock_guard<std::mutex> lock(mutex_);
|
||||
|
|
|
|||
|
|
@ -12,8 +12,8 @@ __`zstd_compression_format.md`__ : This document defines the Zstandard compressi
|
|||
Compliant decoders must adhere to this document,
|
||||
and compliant encoders must generate data that follows it.
|
||||
|
||||
Should you look for ressources to develop your own port of Zstandard algorithm,
|
||||
you may find the following ressources useful :
|
||||
Should you look for resources to develop your own port of Zstandard algorithm,
|
||||
you may find the following resources useful :
|
||||
|
||||
__`educational_decoder`__ : This directory contains an implementation of a Zstandard decoder,
|
||||
compliant with the Zstandard compression format.
|
||||
|
|
|
|||
|
|
@ -358,7 +358,7 @@ static u32 copy_literals(const size_t seq, istream_t *litstream,
|
|||
ostream_t *const out);
|
||||
|
||||
// Given an offset code from a sequence command (either an actual offset value
|
||||
// or an index for previous offset), computes the correct offset and udpates
|
||||
// or an index for previous offset), computes the correct offset and updates
|
||||
// the offset history
|
||||
static size_t compute_offset(sequence_command_t seq, u64 *const offset_hist);
|
||||
|
||||
|
|
|
|||
|
|
@ -384,7 +384,7 @@ size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx);
|
|||
Note 3 : Whenever all input data is provided and consumed in a single round,
|
||||
for example with ZSTD_compress2(),
|
||||
or invoking immediately ZSTD_compressStream2(,,,ZSTD_e_end),
|
||||
this value is automatically overriden by srcSize instead.
|
||||
this value is automatically overridden by srcSize instead.
|
||||
|
||||
</p></pre><BR>
|
||||
|
||||
|
|
@ -571,8 +571,8 @@ size_t ZSTD_freeCStream(ZSTD_CStream* zcs);
|
|||
</b><p> Behaves about the same as ZSTD_compressStream, with additional control on end directive.
|
||||
- Compression parameters are pushed into CCtx before starting compression, using ZSTD_CCtx_set*()
|
||||
- Compression parameters cannot be changed once compression is started (save a list of exceptions in multi-threading mode)
|
||||
- outpot->pos must be <= dstCapacity, input->pos must be <= srcSize
|
||||
- outpot->pos and input->pos will be updated. They are guaranteed to remain below their respective limit.
|
||||
- output->pos must be <= dstCapacity, input->pos must be <= srcSize
|
||||
- output->pos and input->pos will be updated. They are guaranteed to remain below their respective limit.
|
||||
- When nbWorkers==0 (default), function is blocking : it completes its job before returning to caller.
|
||||
- When nbWorkers>=1, function is non-blocking : it just acquires a copy of input, and distributes jobs to internal worker threads, flush whatever is available,
|
||||
and then immediately returns, just indicating that there is some data remaining to be flushed.
|
||||
|
|
|
|||
|
|
@ -107,7 +107,7 @@ int main(int argc, const char** argv)
|
|||
compressFile_orDie(ress, inFilename, outFilename);
|
||||
}
|
||||
|
||||
/* free momery */
|
||||
/* free memory */
|
||||
freeResources(ress,outFilename);
|
||||
|
||||
printf("compressed %i files \n", argc-1);
|
||||
|
|
|
|||
|
|
@ -40,7 +40,7 @@
|
|||
|
||||
/**
|
||||
* FORCE_INLINE_TEMPLATE is used to define C "templates", which take constant
|
||||
* parameters. They must be inlined for the compiler to elimininate the constant
|
||||
* parameters. They must be inlined for the compiler to eliminate the constant
|
||||
* branches.
|
||||
*/
|
||||
#define FORCE_INLINE_TEMPLATE static INLINE_KEYWORD FORCE_INLINE_ATTR
|
||||
|
|
|
|||
|
|
@ -358,7 +358,7 @@ size_t FSE_decompress_wksp(void* dst, size_t dstCapacity, const void* cSrc, size
|
|||
typedef enum {
|
||||
FSE_repeat_none, /**< Cannot use the previous table */
|
||||
FSE_repeat_check, /**< Can use the previous table but it must be checked */
|
||||
FSE_repeat_valid /**< Can use the previous table and it is asumed to be valid */
|
||||
FSE_repeat_valid /**< Can use the previous table and it is assumed to be valid */
|
||||
} FSE_repeat;
|
||||
|
||||
/* *****************************************
|
||||
|
|
|
|||
|
|
@ -14,8 +14,8 @@
|
|||
* This file will hold wrapper for systems, which do not support pthreads
|
||||
*/
|
||||
|
||||
/* create fake symbol to avoid empty trnaslation unit warning */
|
||||
int g_ZSTD_threading_useles_symbol;
|
||||
/* create fake symbol to avoid empty translation unit warning */
|
||||
int g_ZSTD_threading_useless_symbol;
|
||||
|
||||
#if defined(ZSTD_MULTITHREAD) && defined(_WIN32)
|
||||
|
||||
|
|
|
|||
|
|
@ -66,10 +66,10 @@
|
|||
/* #define XXH_ACCEPT_NULL_INPUT_POINTER 1 */
|
||||
|
||||
/*!XXH_FORCE_NATIVE_FORMAT :
|
||||
* By default, xxHash library provides endian-independant Hash values, based on little-endian convention.
|
||||
* By default, xxHash library provides endian-independent Hash values, based on little-endian convention.
|
||||
* Results are therefore identical for little-endian and big-endian CPU.
|
||||
* This comes at a performance cost for big-endian CPU, since some swapping is required to emulate little-endian format.
|
||||
* Should endian-independance be of no importance for your application, you may set the #define below to 1,
|
||||
* Should endian-independence be of no importance for your application, you may set the #define below to 1,
|
||||
* to improve speed for Big-endian CPU.
|
||||
* This option has no impact on Little_Endian CPU.
|
||||
*/
|
||||
|
|
|
|||
|
|
@ -129,9 +129,9 @@ size_t FSE_buildCTable_wksp(FSE_CTable* ct,
|
|||
{ U32 position = 0;
|
||||
U32 symbol;
|
||||
for (symbol=0; symbol<=maxSymbolValue; symbol++) {
|
||||
int nbOccurences;
|
||||
int nbOccurrences;
|
||||
int const freq = normalizedCounter[symbol];
|
||||
for (nbOccurences=0; nbOccurences<freq; nbOccurences++) {
|
||||
for (nbOccurrences=0; nbOccurrences<freq; nbOccurrences++) {
|
||||
tableSymbol[position] = (FSE_FUNCTION_TYPE)symbol;
|
||||
position = (position + step) & tableMask;
|
||||
while (position > highThreshold)
|
||||
|
|
|
|||
|
|
@ -2046,7 +2046,7 @@ void ZSTD_seqToCodes(const seqStore_t* seqStorePtr)
|
|||
* If x == 0: Return 0
|
||||
* Else: Return floor(-log2(x / 256) * 256)
|
||||
*/
|
||||
static unsigned const kInverseProbabiltyLog256[256] = {
|
||||
static unsigned const kInverseProbabilityLog256[256] = {
|
||||
0, 2048, 1792, 1642, 1536, 1453, 1386, 1329, 1280, 1236, 1197, 1162,
|
||||
1130, 1100, 1073, 1047, 1024, 1001, 980, 960, 941, 923, 906, 889,
|
||||
874, 859, 844, 830, 817, 804, 791, 779, 768, 756, 745, 734,
|
||||
|
|
@ -2085,7 +2085,7 @@ static size_t ZSTD_entropyCost(unsigned const* count, unsigned const max, size_t
|
|||
if (count[s] != 0 && norm == 0)
|
||||
norm = 1;
|
||||
assert(count[s] < total);
|
||||
cost += count[s] * kInverseProbabiltyLog256[norm];
|
||||
cost += count[s] * kInverseProbabilityLog256[norm];
|
||||
}
|
||||
return cost >> 8;
|
||||
}
|
||||
|
|
@ -2108,7 +2108,7 @@ static size_t ZSTD_crossEntropyCost(short const* norm, unsigned accuracyLog,
|
|||
unsigned const norm256 = normAcc << shift;
|
||||
assert(norm256 > 0);
|
||||
assert(norm256 < 256);
|
||||
cost += count[s] * kInverseProbabiltyLog256[norm256];
|
||||
cost += count[s] * kInverseProbabilityLog256[norm256];
|
||||
}
|
||||
return cost >> 8;
|
||||
}
|
||||
|
|
@ -2611,7 +2611,7 @@ ZSTD_compressSequences_internal(seqStore_t* seqStorePtr,
|
|||
FORWARD_IF_ERROR(bitstreamSize);
|
||||
op += bitstreamSize;
|
||||
/* zstd versions <= 1.3.4 mistakenly report corruption when
|
||||
* FSE_readNCount() recieves a buffer < 4 bytes.
|
||||
* FSE_readNCount() receives a buffer < 4 bytes.
|
||||
* Fixed by https://github.com/facebook/zstd/pull/1146.
|
||||
* This can happen when the last set_compressed table present is 2
|
||||
* bytes and the bitstream is only one byte.
|
||||
|
|
@ -2914,7 +2914,7 @@ static size_t ZSTD_writeFrameHeader(void* dst, size_t dstCapacity,
|
|||
BYTE const windowLogByte = (BYTE)((params.cParams.windowLog - ZSTD_WINDOWLOG_ABSOLUTEMIN) << 3);
|
||||
U32 const fcsCode = params.fParams.contentSizeFlag ?
|
||||
(pledgedSrcSize>=256) + (pledgedSrcSize>=65536+256) + (pledgedSrcSize>=0xFFFFFFFFU) : 0; /* 0-3 */
|
||||
BYTE const frameHeaderDecriptionByte = (BYTE)(dictIDSizeCode + (checksumFlag<<2) + (singleSegment<<5) + (fcsCode<<6) );
|
||||
BYTE const frameHeaderDescriptionByte = (BYTE)(dictIDSizeCode + (checksumFlag<<2) + (singleSegment<<5) + (fcsCode<<6) );
|
||||
size_t pos=0;
|
||||
|
||||
assert(!(params.fParams.contentSizeFlag && pledgedSrcSize == ZSTD_CONTENTSIZE_UNKNOWN));
|
||||
|
|
@ -2926,7 +2926,7 @@ static size_t ZSTD_writeFrameHeader(void* dst, size_t dstCapacity,
|
|||
MEM_writeLE32(dst, ZSTD_MAGICNUMBER);
|
||||
pos = 4;
|
||||
}
|
||||
op[pos++] = frameHeaderDecriptionByte;
|
||||
op[pos++] = frameHeaderDescriptionByte;
|
||||
if (!singleSegment) op[pos++] = windowLogByte;
|
||||
switch(dictIDSizeCode)
|
||||
{
|
||||
|
|
@ -2950,7 +2950,7 @@ static size_t ZSTD_writeFrameHeader(void* dst, size_t dstCapacity,
|
|||
/* ZSTD_writeLastEmptyBlock() :
|
||||
* output an empty Block with end-of-frame mark to complete a frame
|
||||
* @return : size of data written into `dst` (== ZSTD_blockHeaderSize (defined in zstd_internal.h))
|
||||
* or an error code if `dstCapcity` is too small (<ZSTD_blockHeaderSize)
|
||||
* or an error code if `dstCapacity` is too small (<ZSTD_blockHeaderSize)
|
||||
*/
|
||||
size_t ZSTD_writeLastEmptyBlock(void* dst, size_t dstCapacity)
|
||||
{
|
||||
|
|
|
|||
|
|
@ -36,9 +36,9 @@ extern "C" {
|
|||
#define ZSTD_DUBT_UNSORTED_MARK 1 /* For btlazy2 strategy, index 1 now means "unsorted".
|
||||
It could be confused for a real successor at index "1", if sorted as larger than its predecessor.
|
||||
It's not a big deal though : candidate will just be sorted again.
|
||||
Additionnally, candidate position 1 will be lost.
|
||||
Additionally, candidate position 1 will be lost.
|
||||
But candidate 1 cannot hide a large tree of candidates, so it's a minimal loss.
|
||||
The benefit is that ZSTD_DUBT_UNSORTED_MARK cannot be misdhandled after table re-use with a different strategy
|
||||
The benefit is that ZSTD_DUBT_UNSORTED_MARK cannot be mishandled after table re-use with a different strategy
|
||||
Constant required by ZSTD_compressBlock_btlazy2() and ZSTD_reduceTable_internal() */
|
||||
|
||||
|
||||
|
|
@ -842,7 +842,7 @@ size_t ZSTD_compress_advanced_internal(ZSTD_CCtx* cctx,
|
|||
/* ZSTD_writeLastEmptyBlock() :
|
||||
* output an empty Block with end-of-frame mark to complete a frame
|
||||
* @return : size of data written into `dst` (== ZSTD_blockHeaderSize (defined in zstd_internal.h))
|
||||
* or an error code if `dstCapcity` is too small (<ZSTD_blockHeaderSize)
|
||||
* or an error code if `dstCapacity` is too small (<ZSTD_blockHeaderSize)
|
||||
*/
|
||||
size_t ZSTD_writeLastEmptyBlock(void* dst, size_t dstCapacity);
|
||||
|
||||
|
|
|
|||
|
|
@ -19,7 +19,7 @@ extern "C" {
|
|||
|
||||
U32 ZSTD_insertAndFindFirstIndex(ZSTD_matchState_t* ms, const BYTE* ip);
|
||||
|
||||
void ZSTD_preserveUnsortedMark (U32* const table, U32 const size, U32 const reducerValue); /*! used in ZSTD_reduceIndex(). pre-emptively increase value of ZSTD_DUBT_UNSORTED_MARK */
|
||||
void ZSTD_preserveUnsortedMark (U32* const table, U32 const size, U32 const reducerValue); /*! used in ZSTD_reduceIndex(). preemptively increase value of ZSTD_DUBT_UNSORTED_MARK */
|
||||
|
||||
size_t ZSTD_compressBlock_btlazy2(
|
||||
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
|
|
|
|||
|
|
@ -429,7 +429,7 @@ size_t ZSTD_ldm_generateSequences(
|
|||
*/
|
||||
assert(ldmState->window.nextSrc >= (BYTE const*)src + srcSize);
|
||||
/* The input could be very large (in zstdmt), so it must be broken up into
|
||||
* chunks to enforce the maximmum distance and handle overflow correction.
|
||||
* chunks to enforce the maximum distance and handle overflow correction.
|
||||
*/
|
||||
assert(sequences->pos <= sequences->size);
|
||||
assert(sequences->size <= sequences->capacity);
|
||||
|
|
|
|||
|
|
@ -885,7 +885,7 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms,
|
|||
/* large match -> immediate encoding */
|
||||
{ U32 const maxML = matches[nbMatches-1].len;
|
||||
U32 const maxOffset = matches[nbMatches-1].off;
|
||||
DEBUGLOG(6, "found %u matches of maxLength=%u and maxOffCode=%u at cPos=%u => start new serie",
|
||||
DEBUGLOG(6, "found %u matches of maxLength=%u and maxOffCode=%u at cPos=%u => start new series",
|
||||
nbMatches, maxML, maxOffset, (U32)(ip-prefixStart));
|
||||
|
||||
if (maxML > sufficient_len) {
|
||||
|
|
@ -1133,7 +1133,7 @@ MEM_STATIC void ZSTD_upscaleStats(optState_t* optPtr)
|
|||
/* ZSTD_initStats_ultra():
|
||||
* make a first compression pass, just to seed stats with more accurate starting values.
|
||||
* only works on first block, with no dictionary and no ldm.
|
||||
* this function cannot error, hence its constract must be respected.
|
||||
* this function cannot error, hence its contract must be respected.
|
||||
*/
|
||||
static void
|
||||
ZSTD_initStats_ultra(ZSTD_matchState_t* ms,
|
||||
|
|
|
|||
|
|
@ -456,7 +456,7 @@ typedef struct {
|
|||
* Must be acquired after the main mutex when acquiring both.
|
||||
*/
|
||||
ZSTD_pthread_mutex_t ldmWindowMutex;
|
||||
ZSTD_pthread_cond_t ldmWindowCond; /* Signaled when ldmWindow is udpated */
|
||||
ZSTD_pthread_cond_t ldmWindowCond; /* Signaled when ldmWindow is updated */
|
||||
ZSTD_window_t ldmWindow; /* A thread-safe copy of ldmState.window */
|
||||
} serialState_t;
|
||||
|
||||
|
|
@ -647,7 +647,7 @@ static void ZSTDMT_compressionJob(void* jobDescription)
|
|||
buffer_t dstBuff = job->dstBuff;
|
||||
size_t lastCBlockSize = 0;
|
||||
|
||||
/* ressources */
|
||||
/* resources */
|
||||
if (cctx==NULL) JOB_ERROR(ERROR(memory_allocation));
|
||||
if (dstBuff.start == NULL) { /* streaming job : doesn't provide a dstBuffer */
|
||||
dstBuff = ZSTDMT_getBuffer(job->bufPool);
|
||||
|
|
@ -1527,7 +1527,7 @@ size_t ZSTDMT_initCStream(ZSTDMT_CCtx* mtctx, int compressionLevel) {
|
|||
/* ZSTDMT_writeLastEmptyBlock()
|
||||
* Write a single empty block with an end-of-frame to finish a frame.
|
||||
* Job must be created from streaming variant.
|
||||
* This function is always successfull if expected conditions are fulfilled.
|
||||
* This function is always successful if expected conditions are fulfilled.
|
||||
*/
|
||||
static void ZSTDMT_writeLastEmptyBlock(ZSTDMT_jobDescription* job)
|
||||
{
|
||||
|
|
|
|||
|
|
@ -103,7 +103,7 @@ ZSTDMT_API size_t ZSTDMT_initCStream_usingCDict(ZSTDMT_CCtx* mtctx,
|
|||
* List of parameters that can be set using ZSTDMT_setMTCtxParameter() */
|
||||
typedef enum {
|
||||
ZSTDMT_p_jobSize, /* Each job is compressed in parallel. By default, this value is dynamically determined depending on compression parameters. Can be set explicitly here. */
|
||||
ZSTDMT_p_overlapLog, /* Each job may reload a part of previous job to enhance compressionr ratio; 0 == no overlap, 6(default) == use 1/8th of window, >=9 == use full window. This is a "sticky" parameter : its value will be re-used on next compression job */
|
||||
ZSTDMT_p_overlapLog, /* Each job may reload a part of previous job to enhance compression ratio; 0 == no overlap, 6(default) == use 1/8th of window, >=9 == use full window. This is a "sticky" parameter : its value will be re-used on next compression job */
|
||||
ZSTDMT_p_rsyncable /* Enables rsyncable mode. */
|
||||
} ZSTDMT_parameter;
|
||||
|
||||
|
|
|
|||
|
|
@ -1187,7 +1187,7 @@ unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize)
|
|||
}
|
||||
|
||||
/*! ZSTD_getDictID_fromFrame() :
|
||||
* Provides the dictID required to decompresse frame stored within `src`.
|
||||
* Provides the dictID required to decompress frame stored within `src`.
|
||||
* If @return == 0, the dictID could not be decoded.
|
||||
* This could for one of the following reasons :
|
||||
* - The frame does not require a dictionary (most common case).
|
||||
|
|
|
|||
|
|
@ -801,7 +801,7 @@ ZSTD_updateFseState(ZSTD_fseState* DStatePtr, BIT_DStream_t* bitD)
|
|||
/* We need to add at most (ZSTD_WINDOWLOG_MAX_32 - 1) bits to read the maximum
|
||||
* offset bits. But we can only read at most (STREAM_ACCUMULATOR_MIN_32 - 1)
|
||||
* bits before reloading. This value is the maximum number of bytes we read
|
||||
* after reloading when we are decoding long offets.
|
||||
* after reloading when we are decoding long offsets.
|
||||
*/
|
||||
#define LONG_OFFSETS_MAX_EXTRA_BITS_32 \
|
||||
(ZSTD_WINDOWLOG_MAX_32 > STREAM_ACCUMULATOR_MIN_32 \
|
||||
|
|
@ -1180,7 +1180,7 @@ ZSTD_decompressSequences(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize,
|
|||
/* ZSTD_decompressSequencesLong() :
|
||||
* decompression function triggered when a minimum share of offsets is considered "long",
|
||||
* aka out of cache.
|
||||
* note : "long" definition seems overloaded here, sometimes meaning "wider than bitstream register", and sometimes mearning "farther than memory cache distance".
|
||||
* note : "long" definition seems overloaded here, sometimes meaning "wider than bitstream register", and sometimes meaning "farther than memory cache distance".
|
||||
* This function will try to mitigate main memory latency through the use of prefetching */
|
||||
static size_t
|
||||
ZSTD_decompressSequencesLong(ZSTD_DCtx* dctx,
|
||||
|
|
|
|||
|
|
@ -391,7 +391,7 @@ static void COVER_group(COVER_ctx_t *ctx, const void *group,
|
|||
*
|
||||
* Score(S) = F(S_1) + F(S_2) + ... + F(S_{k-d+1})
|
||||
*
|
||||
* Once the dmer d is in the dictionay we set F(d) = 0.
|
||||
* Once the dmer d is in the dictionary we set F(d) = 0.
|
||||
*/
|
||||
static COVER_segment_t COVER_selectSegment(const COVER_ctx_t *ctx, U32 *freqs,
|
||||
COVER_map_t *activeDmers, U32 begin,
|
||||
|
|
@ -435,7 +435,7 @@ static COVER_segment_t COVER_selectSegment(const COVER_ctx_t *ctx, U32 *freqs,
|
|||
U32 *delDmerOcc = COVER_map_at(activeDmers, delDmer);
|
||||
activeSegment.begin += 1;
|
||||
*delDmerOcc -= 1;
|
||||
/* If this is the last occurence of the dmer, subtract its score */
|
||||
/* If this is the last occurrence of the dmer, subtract its score */
|
||||
if (*delDmerOcc == 0) {
|
||||
COVER_map_remove(activeDmers, delDmer);
|
||||
activeSegment.score -= freqs[delDmer];
|
||||
|
|
|
|||
|
|
@ -51,9 +51,9 @@ typedef struct {
|
|||
* We will make sure that each epoch gets at least 10 * k bytes.
|
||||
*
|
||||
* The COVER algorithms divide the data up into epochs of equal size and
|
||||
* select one segemnt from each epoch.
|
||||
* select one segment from each epoch.
|
||||
*
|
||||
* @param maxDictSize The maximum allowed dictioary size.
|
||||
* @param maxDictSize The maximum allowed dictionary size.
|
||||
* @param nbDmers The number of dmers we are training on.
|
||||
* @param k The parameter k (segment size).
|
||||
* @param passes The target number of passes over the dmer corpus.
|
||||
|
|
|
|||
|
|
@ -132,7 +132,7 @@ typedef struct {
|
|||
*
|
||||
* Score(S) = F(S_1) + F(S_2) + ... + F(S_{k-d+1})
|
||||
*
|
||||
* Once the dmer with hash value d is in the dictionay we set F(d) = 0.
|
||||
* Once the dmer with hash value d is in the dictionary we set F(d) = 0.
|
||||
*/
|
||||
static COVER_segment_t FASTCOVER_selectSegment(const FASTCOVER_ctx_t *ctx,
|
||||
U32 *freqs, U32 begin, U32 end,
|
||||
|
|
@ -161,7 +161,7 @@ static COVER_segment_t FASTCOVER_selectSegment(const FASTCOVER_ctx_t *ctx,
|
|||
/* Get hash value of current dmer */
|
||||
const size_t idx = FASTCOVER_hashPtrToIndex(ctx->samples + activeSegment.end, f, d);
|
||||
|
||||
/* Add frequency of this index to score if this is the first occurence of index in active segment */
|
||||
/* Add frequency of this index to score if this is the first occurrence of index in active segment */
|
||||
if (segmentFreqs[idx] == 0) {
|
||||
activeSegment.score += freqs[idx];
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1759,7 +1759,7 @@ static size_t ZSTD_execSequence(BYTE* op,
|
|||
BYTE* const base, BYTE* const oend)
|
||||
{
|
||||
static const int dec32table[] = {0, 1, 2, 1, 4, 4, 4, 4}; /* added */
|
||||
static const int dec64table[] = {8, 8, 8, 7, 8, 9,10,11}; /* substracted */
|
||||
static const int dec64table[] = {8, 8, 8, 7, 8, 9,10,11}; /* subtracted */
|
||||
const BYTE* const ostart = op;
|
||||
const size_t litLength = sequence.litLength;
|
||||
BYTE* const endMatch = op + litLength + sequence.matchLength; /* risk : address space overflow (32-bits) */
|
||||
|
|
|
|||
|
|
@ -3098,7 +3098,7 @@ static size_t ZSTD_execSequence(BYTE* op,
|
|||
BYTE* const base, BYTE* const oend)
|
||||
{
|
||||
static const int dec32table[] = {0, 1, 2, 1, 4, 4, 4, 4}; /* added */
|
||||
static const int dec64table[] = {8, 8, 8, 7, 8, 9,10,11}; /* substracted */
|
||||
static const int dec64table[] = {8, 8, 8, 7, 8, 9,10,11}; /* subtracted */
|
||||
const BYTE* const ostart = op;
|
||||
BYTE* const oLitEnd = op + sequence.litLength;
|
||||
BYTE* const oMatchEnd = op + sequence.litLength + sequence.matchLength; /* risk : address space overflow (32-bits) */
|
||||
|
|
|
|||
|
|
@ -2739,7 +2739,7 @@ static size_t ZSTD_execSequence(BYTE* op,
|
|||
BYTE* const base, BYTE* const oend)
|
||||
{
|
||||
static const int dec32table[] = {0, 1, 2, 1, 4, 4, 4, 4}; /* added */
|
||||
static const int dec64table[] = {8, 8, 8, 7, 8, 9,10,11}; /* substracted */
|
||||
static const int dec64table[] = {8, 8, 8, 7, 8, 9,10,11}; /* subtracted */
|
||||
const BYTE* const ostart = op;
|
||||
BYTE* const oLitEnd = op + sequence.litLength;
|
||||
BYTE* const oMatchEnd = op + sequence.litLength + sequence.matchLength; /* risk : address space overflow (32-bits) */
|
||||
|
|
|
|||
|
|
@ -2862,7 +2862,7 @@ static size_t ZSTD_execSequence(BYTE* op,
|
|||
const BYTE* const base, const BYTE* const vBase, const BYTE* const dictEnd)
|
||||
{
|
||||
static const int dec32table[] = { 0, 1, 2, 1, 4, 4, 4, 4 }; /* added */
|
||||
static const int dec64table[] = { 8, 8, 8, 7, 8, 9,10,11 }; /* substracted */
|
||||
static const int dec64table[] = { 8, 8, 8, 7, 8, 9,10,11 }; /* subtracted */
|
||||
BYTE* const oLitEnd = op + sequence.litLength;
|
||||
const size_t sequenceLength = sequence.litLength + sequence.matchLength;
|
||||
BYTE* const oMatchEnd = op + sequenceLength; /* risk : address space overflow (32-bits) */
|
||||
|
|
|
|||
|
|
@ -3219,7 +3219,7 @@ static size_t ZSTDv05_execSequence(BYTE* op,
|
|||
const BYTE* const base, const BYTE* const vBase, const BYTE* const dictEnd)
|
||||
{
|
||||
static const int dec32table[] = { 0, 1, 2, 1, 4, 4, 4, 4 }; /* added */
|
||||
static const int dec64table[] = { 8, 8, 8, 7, 8, 9,10,11 }; /* substracted */
|
||||
static const int dec64table[] = { 8, 8, 8, 7, 8, 9,10,11 }; /* subtracted */
|
||||
BYTE* const oLitEnd = op + sequence.litLength;
|
||||
const size_t sequenceLength = sequence.litLength + sequence.matchLength;
|
||||
BYTE* const oMatchEnd = op + sequenceLength; /* risk : address space overflow (32-bits) */
|
||||
|
|
|
|||
|
|
@ -3408,7 +3408,7 @@ static size_t ZSTDv06_execSequence(BYTE* op,
|
|||
if (sequence.offset < 8) {
|
||||
/* close range match, overlap */
|
||||
static const U32 dec32table[] = { 0, 1, 2, 1, 4, 4, 4, 4 }; /* added */
|
||||
static const int dec64table[] = { 8, 8, 8, 7, 8, 9,10,11 }; /* substracted */
|
||||
static const int dec64table[] = { 8, 8, 8, 7, 8, 9,10,11 }; /* subtracted */
|
||||
int const sub2 = dec64table[sequence.offset];
|
||||
op[0] = match[0];
|
||||
op[1] = match[1];
|
||||
|
|
|
|||
|
|
@ -3633,7 +3633,7 @@ size_t ZSTDv07_execSequence(BYTE* op,
|
|||
if (sequence.offset < 8) {
|
||||
/* close range match, overlap */
|
||||
static const U32 dec32table[] = { 0, 1, 2, 1, 4, 4, 4, 4 }; /* added */
|
||||
static const int dec64table[] = { 8, 8, 8, 7, 8, 9,10,11 }; /* substracted */
|
||||
static const int dec64table[] = { 8, 8, 8, 7, 8, 9,10,11 }; /* subtracted */
|
||||
int const sub2 = dec64table[sequence.offset];
|
||||
op[0] = match[0];
|
||||
op[1] = match[1];
|
||||
|
|
|
|||
10
lib/zstd.h
10
lib/zstd.h
|
|
@ -432,7 +432,7 @@ ZSTDLIB_API size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param
|
|||
* Note 3 : Whenever all input data is provided and consumed in a single round,
|
||||
* for example with ZSTD_compress2(),
|
||||
* or invoking immediately ZSTD_compressStream2(,,,ZSTD_e_end),
|
||||
* this value is automatically overriden by srcSize instead.
|
||||
* this value is automatically overridden by srcSize instead.
|
||||
*/
|
||||
ZSTDLIB_API size_t ZSTD_CCtx_setPledgedSrcSize(ZSTD_CCtx* cctx, unsigned long long pledgedSrcSize);
|
||||
|
||||
|
|
@ -635,8 +635,8 @@ typedef enum {
|
|||
* Behaves about the same as ZSTD_compressStream, with additional control on end directive.
|
||||
* - Compression parameters are pushed into CCtx before starting compression, using ZSTD_CCtx_set*()
|
||||
* - Compression parameters cannot be changed once compression is started (save a list of exceptions in multi-threading mode)
|
||||
* - outpot->pos must be <= dstCapacity, input->pos must be <= srcSize
|
||||
* - outpot->pos and input->pos will be updated. They are guaranteed to remain below their respective limit.
|
||||
* - output->pos must be <= dstCapacity, input->pos must be <= srcSize
|
||||
* - output->pos and input->pos will be updated. They are guaranteed to remain below their respective limit.
|
||||
* - When nbWorkers==0 (default), function is blocking : it completes its job before returning to caller.
|
||||
* - When nbWorkers>=1, function is non-blocking : it just acquires a copy of input, and distributes jobs to internal worker threads, flush whatever is available,
|
||||
* and then immediately returns, just indicating that there is some data remaining to be flushed.
|
||||
|
|
@ -662,7 +662,7 @@ ZSTDLIB_API size_t ZSTD_CStreamOutSize(void); /**< recommended size for output
|
|||
|
||||
/*******************************************************************************
|
||||
* This is a legacy streaming API, and can be replaced by ZSTD_CCtx_reset() and
|
||||
* ZSTD_compressStream2(). It is redundent, but is still fully supported.
|
||||
* ZSTD_compressStream2(). It is redundant, but is still fully supported.
|
||||
* Advanced parameters and dictionary compression can only be used through the
|
||||
* new API.
|
||||
******************************************************************************/
|
||||
|
|
@ -1023,7 +1023,7 @@ ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
|
|||
#define ZSTD_WINDOWLOG_LIMIT_DEFAULT 27 /* by default, the streaming decoder will refuse any frame
|
||||
* requiring larger than (1<<ZSTD_WINDOWLOG_LIMIT_DEFAULT) window size,
|
||||
* to preserve host's memory from unreasonable requirements.
|
||||
* This limit can be overriden using ZSTD_DCtx_setParameter(,ZSTD_d_windowLogMax,).
|
||||
* This limit can be overridden using ZSTD_DCtx_setParameter(,ZSTD_d_windowLogMax,).
|
||||
* The limit does not apply for one-pass decoders (such as ZSTD_decompress()), since no additional memory is allocated */
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -383,7 +383,7 @@ BMK_benchMemAdvancedNoAlloc(
|
|||
}
|
||||
}
|
||||
|
||||
/* warmimg up `compressedBuffer` */
|
||||
/* warming up `compressedBuffer` */
|
||||
if (adv->mode == BMK_decodeOnly) {
|
||||
memcpy(compressedBuffer, srcBuffer, loadedCompressedSize);
|
||||
} else {
|
||||
|
|
|
|||
|
|
@ -170,7 +170,7 @@ BMK_benchOutcome_t BMK_syntheticTest(
|
|||
* comprParams - basic compression parameters
|
||||
* dictBuffer - a dictionary if used, null otherwise
|
||||
* dictBufferSize - size of dictBuffer, 0 otherwise
|
||||
* diplayLevel - see BMK_benchFiles
|
||||
* displayLevel - see BMK_benchFiles
|
||||
* displayName - name used by display
|
||||
* @return:
|
||||
* a variant, which expresses either an error, or a valid result.
|
||||
|
|
|
|||
|
|
@ -1060,7 +1060,7 @@ FIO_compressZstdFrame(FIO_prefs_t* const prefs,
|
|||
/* test if compression is blocked
|
||||
* either because output is slow and all buffers are full
|
||||
* or because input is slow and no job can start while waiting for at least one buffer to be filled.
|
||||
* note : excluse starting part, since currentJobID > 1 */
|
||||
* note : exclude starting part, since currentJobID > 1 */
|
||||
if ( (zfp.consumed == previous_zfp_update.consumed) /* no data compressed : no data available, or no more buffer to compress to, OR compression is really slow (compression of a single block is slower than update rate)*/
|
||||
&& (zfp.nbActiveWorkers == 0) /* confirmed : no compression ongoing */
|
||||
) {
|
||||
|
|
|
|||
|
|
@ -347,7 +347,7 @@ static unsigned parseFastCoverParameters(const char* stringPtr, ZDICT_fastCover_
|
|||
|
||||
/**
|
||||
* parseLegacyParameters() :
|
||||
* reads legacy dictioanry builter parameters from *stringPtr (e.g. "--train-legacy=selectivity=8") into *selectivity
|
||||
* reads legacy dictionary builder parameters from *stringPtr (e.g. "--train-legacy=selectivity=8") into *selectivity
|
||||
* @return 1 means that legacy dictionary builder parameters were correct
|
||||
* @return 0 in case of malformed parameters
|
||||
*/
|
||||
|
|
|
|||
|
|
@ -72,7 +72,7 @@ Command line tool to generate test .zst files.
|
|||
|
||||
This tool will generate .zst files with checksums,
|
||||
as well as optionally output the corresponding correct uncompressed data for
|
||||
extra verfication.
|
||||
extra verification.
|
||||
|
||||
Example:
|
||||
```
|
||||
|
|
@ -123,7 +123,7 @@ Full list of arguments
|
|||
Higher values will make optimizer run longer, more chances to find better solution.
|
||||
memLog : Limits the log of the size of each memotable (1 per strategy). Will use hash tables when state space is larger than max size.
|
||||
Setting memLog = 0 turns off memoization
|
||||
--display= : specifiy which parameters are included in the output
|
||||
--display= : specify which parameters are included in the output
|
||||
can use all --zstd parameter names and 'cParams' as a shorthand for all parameters used in ZSTD_compressionParameters
|
||||
(Default: display all params available)
|
||||
-P# : generated sample compressibility (when no file is provided)
|
||||
|
|
|
|||
|
|
@ -514,7 +514,7 @@ static size_t writeLiteralsBlockCompressed(U32* seed, frame_t* frame, size_t con
|
|||
if ((RAND(seed) & 3) || !frame->stats.hufInit) {
|
||||
do {
|
||||
if (RAND(seed) & 3) {
|
||||
/* add 10 to ensure some compressability */
|
||||
/* add 10 to ensure some compressibility */
|
||||
double const weight = ((RAND(seed) % 90) + 10) / 100.0;
|
||||
|
||||
DISPLAYLEVEL(5, " distribution weight: %d%%\n",
|
||||
|
|
|
|||
|
|
@ -104,7 +104,7 @@ libregression.a: $(FUZZ_HEADERS) $(PRGDIR)/util.h $(PRGDIR)/util.c regression_dr
|
|||
$(AR) $(FUZZ_ARFLAGS) $@ regression_driver.o
|
||||
|
||||
# Install libfuzzer (not usable for MSAN testing)
|
||||
# Provided for convienence. To use this library run make libFuzzer and
|
||||
# Provided for convenience. To use this library run make libFuzzer and
|
||||
# set LDFLAGS=-L.
|
||||
.PHONY: libFuzzer
|
||||
libFuzzer:
|
||||
|
|
|
|||
|
|
@ -37,8 +37,8 @@ The specific fuzzing engine is selected with `LIB_FUZZING_ENGINE` or
|
|||
`--lib-fuzzing-engine`, the default is `libregression.a`.
|
||||
It has flags that can easily set up sanitizers `--enable-{a,ub,m}san`, and
|
||||
coverage instrumentation `--enable-coverage`.
|
||||
It sets sane defaults which can be overriden with flags `--debug`,
|
||||
`--enable-ubsan-pointer-overlow`, etc.
|
||||
It sets sane defaults which can be overridden with flags `--debug`,
|
||||
`--enable-ubsan-pointer-overflow`, etc.
|
||||
Run `./fuzz.py build -h` for help.
|
||||
|
||||
### Running Fuzzers
|
||||
|
|
|
|||
|
|
@ -339,13 +339,13 @@ def build_parser(args):
|
|||
args = parse_env_flags(args, ' '.join(
|
||||
[args.cppflags, args.cflags, args.cxxflags, args.ldflags]))
|
||||
|
||||
# Check option sanitiy
|
||||
# Check option sanity
|
||||
if args.msan and (args.asan or args.ubsan):
|
||||
raise RuntimeError('MSAN may not be used with any other sanitizers')
|
||||
if args.msan_track_origins and not args.msan:
|
||||
raise RuntimeError('--enable-msan-track-origins requires MSAN')
|
||||
if args.ubsan_pointer_overflow and not args.ubsan:
|
||||
raise RuntimeError('--enable-ubsan-pointer-overlow requires UBSAN')
|
||||
raise RuntimeError('--enable-ubsan-pointer-overflow requires UBSAN')
|
||||
if args.sanitize_recover and not args.sanitize:
|
||||
raise RuntimeError('--enable-sanitize-recover but no sanitizers used')
|
||||
|
||||
|
|
@ -623,7 +623,7 @@ def regression(args):
|
|||
|
||||
def gen_parser(args):
|
||||
description = """
|
||||
Generate a seed corpus appropiate for TARGET with data generated with
|
||||
Generate a seed corpus appropriate for TARGET with data generated with
|
||||
decodecorpus.
|
||||
The fuzz inputs are prepended with a seed before the zstd data, so the
|
||||
output of decodecorpus shouldn't be used directly.
|
||||
|
|
|
|||
|
|
@ -55,7 +55,7 @@ extern "C" {
|
|||
#endif
|
||||
|
||||
/**
|
||||
* Determininistically constructs a seed based on the fuzz input.
|
||||
* Deterministically constructs a seed based on the fuzz input.
|
||||
* Consumes up to the first FUZZ_RNG_SEED_SIZE bytes of the input.
|
||||
*/
|
||||
FUZZ_STATIC uint32_t FUZZ_seed(uint8_t const **src, size_t* size) {
|
||||
|
|
|
|||
|
|
@ -75,7 +75,7 @@ void FUZZ_setRandomParameters(ZSTD_CCtx *cctx, size_t srcSize, uint32_t *state)
|
|||
setRand(cctx, ZSTD_c_contentSizeFlag, 0, 1, state);
|
||||
setRand(cctx, ZSTD_c_checksumFlag, 0, 1, state);
|
||||
setRand(cctx, ZSTD_c_dictIDFlag, 0, 1, state);
|
||||
/* Select long distance matchig parameters */
|
||||
/* Select long distance matching parameters */
|
||||
setRand(cctx, ZSTD_c_enableLongDistanceMatching, 0, 1, state);
|
||||
setRand(cctx, ZSTD_c_ldmHashLog, ZSTD_HASHLOG_MIN, 16, state);
|
||||
setRand(cctx, ZSTD_c_ldmMinMatch, ZSTD_LDM_MINMATCH_MIN,
|
||||
|
|
|
|||
|
|
@ -909,7 +909,7 @@ static int basicUnitTests(U32 seed, double compressibility)
|
|||
CHECK_EQ(value, 5);
|
||||
CHECK( ZSTD_CCtxParams_getParameter(params, ZSTD_c_jobSize, &value) );
|
||||
CHECK_EQ(value, 2 MB);
|
||||
/* Set the number of worksers and check the overlap log and job size. */
|
||||
/* Set the number of workers and check the overlap log and job size. */
|
||||
CHECK( ZSTD_CCtxParams_setParameter(params, ZSTD_c_nbWorkers, 2) );
|
||||
CHECK( ZSTD_CCtxParams_getParameter(params, ZSTD_c_overlapLog, &value) );
|
||||
CHECK_EQ(value, 5);
|
||||
|
|
|
|||
|
|
@ -1235,7 +1235,7 @@ static int createBuffersFromMemory(buffers_t* buff, void * srcBuffer, const size
|
|||
return 0;
|
||||
}
|
||||
|
||||
/* allocates buffer's arguments. returns success / failuere */
|
||||
/* allocates buffer's arguments. returns success / failure */
|
||||
static int createBuffers(buffers_t* buff, const char* const * const fileNamesTable,
|
||||
size_t nbFiles) {
|
||||
size_t pos = 0;
|
||||
|
|
@ -1508,7 +1508,7 @@ createMemoTableArray(const paramValues_t p,
|
|||
}
|
||||
|
||||
/* Sets pc to random unmeasured set of parameters */
|
||||
/* specifiy strategy */
|
||||
/* specify strategy */
|
||||
static void randomConstrainedParams(paramValues_t* pc, const memoTable_t* memoTableArray, const ZSTD_strategy st)
|
||||
{
|
||||
size_t j;
|
||||
|
|
@ -1573,7 +1573,7 @@ BMK_benchMemInvertible( buffers_t buf, contexts_t ctx,
|
|||
display_params_tested(*comprParams);
|
||||
memset(&bResult, 0, sizeof(bResult));
|
||||
|
||||
/* warmimg up memory */
|
||||
/* warming up memory */
|
||||
for (i = 0; i < buf.nbBlocks; i++) {
|
||||
if (mode != BMK_decodeOnly) {
|
||||
RDG_genBuffer(dstPtrs[i], dstCapacities[i], 0.10, 0.50, 1);
|
||||
|
|
@ -1992,7 +1992,7 @@ BMK_selectRandomStart( FILE* f,
|
|||
|
||||
/* BMK_generate_cLevelTable() :
|
||||
* test a large number of configurations
|
||||
* and distribute them accross compression levels according to speed conditions.
|
||||
* and distribute them across compression levels according to speed conditions.
|
||||
* display and save all intermediate results into rfName = "grillResults.txt".
|
||||
* the function automatically stops after g_timeLimit_s.
|
||||
* this function cannot error, it directly exit() in case of problem.
|
||||
|
|
@ -2270,7 +2270,7 @@ static winnerInfo_t climbOnce(const constraint_t target,
|
|||
|
||||
/* Optimizes for a fixed strategy */
|
||||
|
||||
/* flexible parameters: iterations of failed climbing (or if we do non-random, maybe this is when everything is close to visitied)
|
||||
/* flexible parameters: iterations of failed climbing (or if we do non-random, maybe this is when everything is close to visited)
|
||||
weight more on visit for bad results, less on good results/more on later results / ones with more failures.
|
||||
allocate memoTable here.
|
||||
*/
|
||||
|
|
|
|||
|
|
@ -153,7 +153,7 @@ static void print_help(void) {
|
|||
}
|
||||
}
|
||||
|
||||
/** Parse the arguments. Teturn 0 on success. Print help on failure. */
|
||||
/** Parse the arguments. Return 0 on success. Print help on failure. */
|
||||
static int parse_args(int argc, char** argv) {
|
||||
int option_index = 0;
|
||||
int c;
|
||||
|
|
|
|||
|
|
@ -771,7 +771,7 @@ static int basicUnitTests(U32 seed, double compressibility)
|
|||
}
|
||||
DISPLAYLEVEL(3, "OK \n");
|
||||
|
||||
DISPLAYLEVEL(3, "test%3i : ZSTD_resetDStream() wtih dictionary : ", testNb++);
|
||||
DISPLAYLEVEL(3, "test%3i : ZSTD_resetDStream() with dictionary : ", testNb++);
|
||||
{
|
||||
ZSTD_DCtx* dctx = ZSTD_createDCtx();
|
||||
/* We should succeed to decompress with the dictionary. */
|
||||
|
|
@ -1051,7 +1051,7 @@ static int basicUnitTests(U32 seed, double compressibility)
|
|||
inBuff.size = srcSize; assert(srcSize < COMPRESSIBLE_NOISE_LENGTH);
|
||||
inBuff.pos = 0;
|
||||
}
|
||||
{ ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, 4 KB, dictionary.filled); /* intentionnally lies on estimatedSrcSize, to push cdict into targeting a small window size */
|
||||
{ ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, 4 KB, dictionary.filled); /* intentionally lies on estimatedSrcSize, to push cdict into targeting a small window size */
|
||||
ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictionary.start, dictionary.filled, ZSTD_dlm_byRef, ZSTD_dct_fullDict, cParams, ZSTD_defaultCMem);
|
||||
DISPLAYLEVEL(5, "cParams.windowLog = %u : ", cParams.windowLog);
|
||||
CHECK_Z( ZSTD_CCtx_refCDict(zc, cdict) );
|
||||
|
|
@ -2069,7 +2069,7 @@ static int fuzzerTests_newAPI(U32 seed, int nbTests, int startTest,
|
|||
CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, pledgedSrcSize) );
|
||||
}
|
||||
|
||||
/* multi-threading parameters. Only adjust ocassionally for small tests. */
|
||||
/* multi-threading parameters. Only adjust occasionally for small tests. */
|
||||
if (bigTests || (FUZ_rand(&lseed) & 0xF) == 0xF) {
|
||||
U32 const nbThreadsCandidate = (FUZ_rand(&lseed) & 4) + 1;
|
||||
U32 const nbThreadsAdjusted = (windowLogMalus < nbThreadsCandidate) ? nbThreadsCandidate - windowLogMalus : 1;
|
||||
|
|
|
|||
|
|
@ -71,7 +71,7 @@ inflate with dictionary: hello, hello!
|
|||
The script used for compilation can be found at [zlibWrapper/Makefile](Makefile).
|
||||
|
||||
|
||||
#### The measurement of performace of Zstandard wrapper for zlib
|
||||
#### The measurement of performance of Zstandard wrapper for zlib
|
||||
|
||||
The zstd distribution contains a tool called `zwrapbench` which can measure speed and ratio of zlib, zstd, and the wrapper.
|
||||
The benchmark is conducted using given filenames or synthetic data if filenames are not provided.
|
||||
|
|
@ -96,8 +96,8 @@ as this case is automatically detected.
|
|||
#### Reusing contexts
|
||||
|
||||
The ordinary zlib compression of two files/streams allocates two contexts:
|
||||
- for the 1st file calls `deflateInit`, `deflate`, `...`, `deflate`, `defalateEnd`
|
||||
- for the 2nd file calls `deflateInit`, `deflate`, `...`, `deflate`, `defalateEnd`
|
||||
- for the 1st file calls `deflateInit`, `deflate`, `...`, `deflate`, `deflateEnd`
|
||||
- for the 2nd file calls `deflateInit`, `deflate`, `...`, `deflate`, `deflateEnd`
|
||||
|
||||
The speed of compression can be improved with reusing a single context with following steps:
|
||||
- initialize the context with `deflateInit`
|
||||
|
|
|
|||
|
|
@ -21,7 +21,7 @@
|
|||
data in order to determine how much of that input will compress to
|
||||
nearly the requested output block size. The first pass generates
|
||||
enough deflate blocks to produce output to fill the requested
|
||||
output size plus a specfied excess amount (see the EXCESS define
|
||||
output size plus a specified excess amount (see the EXCESS define
|
||||
below). The last deflate block may go quite a bit past that, but
|
||||
is discarded. The second pass decompresses and recompresses just
|
||||
the compressed data that fit in the requested plus excess sized
|
||||
|
|
@ -217,7 +217,7 @@ int main(int argc, char **argv)
|
|||
if (ret == Z_MEM_ERROR)
|
||||
quit("out of memory");
|
||||
|
||||
/* set up for next reocmpression */
|
||||
/* set up for next recompression */
|
||||
ret = inflateReset(&inf);
|
||||
assert(ret != Z_STREAM_ERROR);
|
||||
ret = deflateReset(&def);
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@
|
|||
data in order to determine how much of that input will compress to
|
||||
nearly the requested output block size. The first pass generates
|
||||
enough deflate blocks to produce output to fill the requested
|
||||
output size plus a specfied excess amount (see the EXCESS define
|
||||
output size plus a specified excess amount (see the EXCESS define
|
||||
below). The last deflate block may go quite a bit past that, but
|
||||
is discarded. The second pass decompresses and recompresses just
|
||||
the compressed data that fit in the requested plus excess sized
|
||||
|
|
@ -198,7 +198,7 @@ int main(int argc, char **argv)
|
|||
if (ret == Z_MEM_ERROR)
|
||||
quit("out of memory");
|
||||
|
||||
/* set up for next reocmpression */
|
||||
/* set up for next recompression */
|
||||
ret = inflateReset(&inf);
|
||||
assert(ret != Z_STREAM_ERROR);
|
||||
ret = deflateReset(&def);
|
||||
|
|
|
|||
|
|
@ -193,7 +193,7 @@ static int BMK_benchMem(z_const void* srcBuffer, size_t srcSize,
|
|||
remaining -= thisBlockSize;
|
||||
} } }
|
||||
|
||||
/* warmimg up memory */
|
||||
/* warming up memory */
|
||||
RDG_genBuffer(compressedBuffer, maxCompressedSize, 0.10, 0.50, 1);
|
||||
|
||||
/* Bench */
|
||||
|
|
|
|||
Loading…
Reference in New Issue