facebook/zstd
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add re-copying of ldmSeqStore after processing

Browse Source
This commit is contained in:
senhuang42 2020-09-26 22:14:21 -04:00
parent 42395a70c2
commit 84009a076a
2 changed files with 14 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
lib/compress/zstd_ldm.c
View File

@ -580,6 +580,7 @@ size_t ZSTD_ldm_blockCompress(rawSeqStore_t* rawSeqStore,
size_t lastLLSize; size_t lastLLSize;
ms->ldmSeqStore = *rawSeqStore; /* copy current seqStore */ ms->ldmSeqStore = *rawSeqStore; /* copy current seqStore */
lastLLSize = blockCompressor(ms, seqStore, rep, src, srcSize); lastLLSize = blockCompressor(ms, seqStore, rep, src, srcSize);
rawSeqStore->pos = ms->ldmSeqStore.pos;
return lastLLSize; return lastLLSize;
} }

22
lib/compress/zstd_opt.c
View File

@ -806,6 +806,7 @@ static rawSeq ldm_splitSequence(rawSeqStore_t* ldmSeqStore, U32 remainingBytes)
if (remainingBytes >= currSeq.litLength + currSeq.matchLength) { if (remainingBytes >= currSeq.litLength + currSeq.matchLength) {
printf("NO SPLIT\n"); printf("NO SPLIT\n");
ldmSeqStore->pos++; ldmSeqStore->pos++;
printf("pos is now: %u\n", ldmSeqStore->pos);
return currSeq; return currSeq;
} }
/* Need a split */ /* Need a split */
@ -831,8 +832,9 @@ static int ldm_getNextMatch(rawSeqStore_t* ldmSeqStore,
U32* matchOffset, U32 currPosInBlock, U32* matchOffset, U32 currPosInBlock,
U32 remainingBytes) { U32 remainingBytes) {
rawSeq seq = ldm_splitSequence(ldmSeqStore, remainingBytes); rawSeq seq = ldm_splitSequence(ldmSeqStore, remainingBytes);
if (seq.offset == 0) { if (seq.offset == 0 || ldmSeqStore->pos > ldmSeqStore->size) {
// Don't use the LDM for the rest of the block (there is none) // Don't use the LDM for the rest of the block (there is none)
printf("No ldm left in the block\n");
*matchStartPosInBlock = UINT32_MAX; *matchStartPosInBlock = UINT32_MAX;
*matchEndPosInBlock = UINT32_MAX; *matchEndPosInBlock = UINT32_MAX;
return 1; return 1;
@ -841,6 +843,7 @@ static int ldm_getNextMatch(rawSeqStore_t* ldmSeqStore,
*matchStartPosInBlock = currPosInBlock + seq.litLength; *matchStartPosInBlock = currPosInBlock + seq.litLength;
*matchEndPosInBlock = *matchStartPosInBlock + seq.matchLength; *matchEndPosInBlock = *matchStartPosInBlock + seq.matchLength;
*matchOffset = seq.offset; *matchOffset = seq.offset;
printf("New match range in block is: (%u, %u) with of: %u where currPosInBlock: %u\n", *matchStartPosInBlock, *matchEndPosInBlock, *matchOffset, currPosInBlock);
return 0; return 0;
} }
@ -860,6 +863,7 @@ static void ldm_maybeAddLdm(ZSTD_match_t* matches, U32* nbMatches,
if (matchLengthAdjusted >= matches[*nbMatches-1].len) { if (matchLengthAdjusted >= matches[*nbMatches-1].len) {
printf("Adding LDM with of: %u, ml: %u\n", matchOffsetAdjusted, matchLengthAdjusted); printf("Adding LDM with of: %u, ml: %u\n", matchOffsetAdjusted, matchLengthAdjusted);
/* Add sifting */
matches[*nbMatches].len = matchLengthAdjusted; matches[*nbMatches].len = matchLengthAdjusted;
matches[*nbMatches].off = matchOffsetAdjusted; matches[*nbMatches].off = matchOffsetAdjusted;
(*nbMatches)++; (*nbMatches)++;
@ -875,8 +879,7 @@ static void ldm_handleLdm(rawSeqStore_t* ldmSeqStore, ZSTD_match_t* matches, U32
matchEndPosInBlock, matchOffset, matchEndPosInBlock, matchOffset,
currPosInBlock, remainingBytes); currPosInBlock, remainingBytes);
} }
if (ldmSeqStore->pos <= ldmSeqStore->size) ldm_maybeAddLdm(matches, nbMatches, *matchStartPosInBlock, *matchEndPosInBlock, *matchOffset, currPosInBlock);
ldm_maybeAddLdm(matches, nbMatches, *matchStartPosInBlock, *matchEndPosInBlock, *matchOffset, currPosInBlock);
} }
@ -936,11 +939,7 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms,
U32 ldmStartPosInBlock = 0; U32 ldmStartPosInBlock = 0;
U32 ldmEndPosInBlock = 0; U32 ldmEndPosInBlock = 0;
U32 ldmOffset = 0; U32 ldmOffset = 0;
if (ms->ldmSeqStore.size != 0) {
ldm_getNextMatch(&ms->ldmSeqStore, &ldmStartPosInBlock,
&ldmEndPosInBlock, &ldmOffset,
(U32)(ip-istart), (U32)(iend - ip));
}
/* init */ /* init */
DEBUGLOG(5, "ZSTD_compressBlock_opt_generic: current=%u, prefix=%u, nextToUpdate=%u", DEBUGLOG(5, "ZSTD_compressBlock_opt_generic: current=%u, prefix=%u, nextToUpdate=%u",
@ -948,7 +947,12 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms,
assert(optLevel <= 2); assert(optLevel <= 2);
ZSTD_rescaleFreqs(optStatePtr, (const BYTE*)src, srcSize, optLevel); ZSTD_rescaleFreqs(optStatePtr, (const BYTE*)src, srcSize, optLevel);
ip += (ip==prefixStart); ip += (ip==prefixStart);
if (ms->ldmSeqStore.size != 0) {
ldm_getNextMatch(&ms->ldmSeqStore, &ldmStartPosInBlock,
&ldmEndPosInBlock, &ldmOffset,
(U32)(ip-istart), (U32)(iend - ip));
}
/* Match Loop */ /* Match Loop */
while (ip < ilimit) { while (ip < ilimit) {
U32 cur, last_pos = 0; U32 cur, last_pos = 0;