facebook/zstd
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add algorithm to remove all delimiters

Browse Source
This commit is contained in:
senhuang42 2020-11-02 10:46:52 -05:00
parent 435a3a0428
commit a36fdada57
2 changed files with 21 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
lib/compress/zstd_compress.c
View File

@ -2567,6 +2567,24 @@ size_t ZSTD_getSequences(ZSTD_CCtx* zc, ZSTD_Sequence* outSeqs,
ZSTD_compress2(zc, dst, dstCapacity, src, srcSize); ZSTD_compress2(zc, dst, dstCapacity, src, srcSize);
ZSTD_customFree(dst, ZSTD_defaultCMem); ZSTD_customFree(dst, ZSTD_defaultCMem);
if (format == ZSTD_sf_noBlockDelimiters) {
/* Merge the dummy block delimiters */
size_t i = 0;
size_t totalSeqs = zc->seqCollector.seqIndex;
for (; i < totalSeqs; ++i) {
if (seqCollector.seqStart[i].offset == 0 && seqCollector.seqStart[i].matchLength == 0) {
/* Merge the block boundary or last literals */
if (i != totalSeqs-1) {
/* Add last literals to next sequence, then "delete" this sequence */
seqCollector.seqStart[i+1].litLength += seqCollector.seqStart[i].litLength;
memmove(seqCollector.seqStart+i, seqCollector.seqStart+i+1, (totalSeqs-i-1)*sizeof(ZSTD_sequence));
}
totalSeqs--;
}
}
zc->seqCollector.seqIndex = totalSeqs;
}
return zc->seqCollector.seqIndex; return zc->seqCollector.seqIndex;
} }

6
lib/zstd.h
View File

@ -1309,9 +1309,9 @@ typedef enum {
* *
* If invoked with ZSTD_sf_noBlockDelimiters, sequences will still be generated * If invoked with ZSTD_sf_noBlockDelimiters, sequences will still be generated
* on a per-block basis, but any last literals of a block will be merged into the * on a per-block basis, but any last literals of a block will be merged into the
* last literals of the first sequence in the next block with the exception of the * last literals of the first sequence in the next block.
* final segment of last literals. As such, the final generated result has no * As such, the final generated result has no explicit representation of block boundaries,
* explicit representation of block boundaries. * and the final last literals segment is not represented in the sequences.
* *
* zc can be used to insert custom compression params. * zc can be used to insert custom compression params.
* This function invokes ZSTD_compress2 * This function invokes ZSTD_compress2