facebook/zstd
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add compression context

Browse Source
This commit is contained in:
Stella Lau 2017-07-10 12:38:27 -07:00
parent ae9cf235d6
commit 10a71d9f1c
1 changed files with 167 additions and 120 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

279
contrib/long_distance_matching/ldm.c
View File

@ -31,7 +31,10 @@ typedef uint32_t U32;
typedef int32_t S32; typedef int32_t S32;
typedef uint64_t U64; typedef uint64_t U64;
typedef uint64_t tag; typedef uint32_t offset_t;
typedef uint32_t hash_t;
// typedef uint64_t tag;
static unsigned LDM_isLittleEndian(void) { static unsigned LDM_isLittleEndian(void) {
const union { U32 u; BYTE c[4]; } one = { 1 }; const union { U32 u; BYTE c[4]; } one = { 1 };
@ -82,15 +85,18 @@ static U64 LDM_read64(const void *ptr) {
static void LDM_copy8(void *dst, const void *src) { static void LDM_copy8(void *dst, const void *src) {
memcpy(dst, src, 8); memcpy(dst, src, 8);
} }
typedef struct LDM_hashEntry {
offset_t offset;
} LDM_hashEntry;
typedef struct compress_stats { typedef struct LDM_compressStats {
U32 num_matches; U32 num_matches;
U32 total_match_length; U32 total_match_length;
U32 total_literal_length; U32 total_literal_length;
U64 total_offset; U64 total_offset;
} compress_stats; } LDM_compressStats;
static void LDM_printCompressStats(const compress_stats *stats) { static void LDM_printCompressStats(const LDM_compressStats *stats) {
printf("=====================\n"); printf("=====================\n");
printf("Compression statistics\n"); printf("Compression statistics\n");
printf("Total number of matches: %u\n", stats->num_matches); printf("Total number of matches: %u\n", stats->num_matches);
@ -103,53 +109,83 @@ static void LDM_printCompressStats(const compress_stats *stats) {
printf("=====================\n"); printf("=====================\n");
} }
// TODO: unused. typedef struct LDM_CCtx {
struct hash_entry { size_t isize; /* Input size */
U64 offset; size_t maxOSize; /* Maximum output size */
tag t;
};
static U32 LDM_hash(U32 sequence) { const BYTE *ibase; /* Base of input */
const BYTE *ip; /* Current input position */
const BYTE *iend; /* End of input */
// Maximum input position such that hashing at the position does not exceed
// end of input.
const BYTE *ihashLimit;
// Maximum input position such that finding a match of at least the minimum
// match length does not exceed end of input.
const BYTE *imatchLimit;
const BYTE *obase; /* Base of output */
BYTE *op; /* Output */
const BYTE *anchor; /* Anchor to start of current (match) block */
LDM_compressStats stats; /* Compression statistics */
LDM_hashEntry hashTable[LDM_HASHTABLESIZE_U32];
const BYTE *lastPosHashed; /* Last position hashed */
hash_t lastHash; /* Hash corresponding to lastPosHashed */
} LDM_CCtx;
static hash_t LDM_hash(U32 sequence) {
return ((sequence * 2654435761U) >> ((32)-LDM_HASHLOG)); return ((sequence * 2654435761U) >> ((32)-LDM_HASHLOG));
} }
static U32 LDM_hash5(U64 sequence) { static hash_t LDM_hash5(U64 sequence) {
static const U64 prime5bytes = 889523592379ULL; static const U64 prime5bytes = 889523592379ULL;
static const U64 prime8bytes = 11400714785074694791ULL; static const U64 prime8bytes = 11400714785074694791ULL;
const U32 hashLog = LDM_HASHLOG; const U32 hashLog = LDM_HASHLOG;
if (LDM_isLittleEndian()) if (LDM_isLittleEndian())
return (U32)(((sequence << 24) * prime5bytes) >> (64 - hashLog)); return (((sequence << 24) * prime5bytes) >> (64 - hashLog));
else else
return (U32)(((sequence >> 24) * prime8bytes) >> (64 - hashLog)); return (((sequence >> 24) * prime8bytes) >> (64 - hashLog));
} }
static U32 LDM_hash_position(const void * const p) { static hash_t LDM_hash_position(const void * const p) {
return LDM_hash(LDM_read32(p)); return LDM_hash(LDM_read32(p));
} }
static void LDM_put_position_on_hash(const BYTE *p, U32 h, void *tableBase, static void LDM_put_position_on_hash(const BYTE *p, hash_t h,
const BYTE *srcBase) { void *tableBase, const BYTE *srcBase) {
if (((p - srcBase) & HASH_EVERY) != HASH_EVERY) { if (((p - srcBase) & HASH_EVERY) != HASH_EVERY) {
return; return;
} }
U32 *hashTable = (U32 *) tableBase; LDM_hashEntry *hashTable = (LDM_hashEntry *) tableBase;
hashTable[h] = (U32)(p - srcBase); hashTable[h] = (LDM_hashEntry) { (hash_t )(p - srcBase) };
} }
static void LDM_put_position(const BYTE *p, void *tableBase, static void LDM_putPosition(const BYTE *p, void *tableBase,
const BYTE *srcBase) { const BYTE *srcBase) {
if (((p - srcBase) & HASH_EVERY) != HASH_EVERY) { if (((p - srcBase) & HASH_EVERY) != HASH_EVERY) {
return; return;
} }
U32 const h = LDM_hash_position(p); hash_t const h = LDM_hash_position(p);
LDM_put_position_on_hash(p, h, tableBase, srcBase); LDM_put_position_on_hash(p, h, tableBase, srcBase);
} }
static void LDM_putHashOfCurrentPosition(LDM_CCtx *const cctx) {
LDM_putPosition(cctx->ip, cctx->hashTable, cctx->ibase);
}
static const BYTE *LDM_get_position_on_hash( static const BYTE *LDM_get_position_on_hash(
U32 h, void *tableBase, const BYTE *srcBase) { hash_t h, void *tableBase, const BYTE *srcBase) {
const U32 * const hashTable = (U32*)tableBase; const LDM_hashEntry * const hashTable = (LDM_hashEntry *)tableBase;
return hashTable[h] + srcBase; return hashTable[h].offset + srcBase;
} }
static BYTE LDM_read_byte(const void *memPtr) { static BYTE LDM_read_byte(const void *memPtr) {
@ -180,140 +216,149 @@ void LDM_read_header(const void *src, size_t *compressSize,
*decompressSize = *ip; *decompressSize = *ip;
} }
// TODO: maxDstSize is unused static void LDM_initializeCCtx(LDM_CCtx *cctx,
const void *src, size_t srcSize,
void *dst, size_t maxDstSize) {
cctx->isize = srcSize;
cctx->maxOSize = maxDstSize;
cctx->ibase = (const BYTE *)src;
cctx->ip = cctx->ibase;
cctx->iend = cctx->ibase + srcSize;
cctx->ihashLimit = cctx->iend - HASH_SIZE;
cctx->imatchLimit = cctx->iend - MINMATCH;
cctx->obase = (BYTE *)dst;
cctx->op = (BYTE *)cctx->obase;
cctx->anchor = cctx->ibase;
memset(&(cctx->stats), 0, sizeof(cctx->stats));
memset(cctx->hashTable, 0, sizeof(cctx->hashTable));
cctx->lastPosHashed = NULL;
}
// TODO: srcSize and maxDstSize is unused
size_t LDM_compress(const void *src, size_t srcSize, size_t LDM_compress(const void *src, size_t srcSize,
void *dst, size_t maxDstSize) { void *dst, size_t maxDstSize) {
const BYTE * const istart = (const BYTE*)src; LDM_CCtx cctx;
const BYTE *ip = istart; LDM_initializeCCtx(&cctx, src, srcSize, dst, maxDstSize);
const BYTE * const iend = istart + srcSize;
const BYTE *ilimit = iend - HASH_SIZE;
const BYTE * const matchlimit = iend - HASH_SIZE;
const BYTE * const mflimit = iend - MINMATCH;
BYTE *op = (BYTE*) dst;
compress_stats compressStats = { 0 };
U32 hashTable[LDM_HASHTABLESIZE_U32];
memset(hashTable, 0, sizeof(hashTable));
const BYTE *anchor = (const BYTE *)src;
// struct LDM_cctx cctx;
size_t output_size = 0;
U32 forwardH; U32 forwardH;
/* Hash first byte: put into hash table */ /* Hash the first position and put it into the hash table. */
LDM_putHashOfCurrentPosition(&cctx);
const BYTE *lastHash = cctx.ip;
cctx.ip++;
forwardH = LDM_hash_position(cctx.ip);
LDM_put_position(ip, hashTable, istart); // TODO: loop condition is not accurate.
const BYTE *lastHash = ip; while (1) {
ip++;
forwardH = LDM_hash_position(ip);
//TODO Loop terminates before ip>=ilimit.
while (ip < ilimit) {
const BYTE *match; const BYTE *match;
BYTE *token; BYTE *token;
/* Find a match */ /* Find a match */
{ {
const BYTE *forwardIp = ip; const BYTE *forwardIp = cctx.ip;
unsigned step = 1; unsigned step = 1;
do { do {
U32 const h = forwardH; U32 const h = forwardH;
ip = forwardIp; cctx.ip = forwardIp;
forwardIp += step; forwardIp += step;
if (forwardIp > mflimit) { if (forwardIp > cctx.imatchLimit) {
goto _last_literals; goto _last_literals;
} }
match = LDM_get_position_on_hash(h, hashTable, istart); match = LDM_get_position_on_hash(h, cctx.hashTable, cctx.ibase);
forwardH = LDM_hash_position(forwardIp); forwardH = LDM_hash_position(forwardIp);
LDM_put_position_on_hash(ip, h, hashTable, istart); LDM_put_position_on_hash(cctx.ip, h, cctx.hashTable, cctx.ibase);
lastHash = ip; lastHash = cctx.ip;
} while (ip - match > WINDOW_SIZE || } while (cctx.ip - match > WINDOW_SIZE ||
LDM_read64(match) != LDM_read64(ip)); LDM_read64(match) != LDM_read64(cctx.ip));
} }
compressStats.num_matches++; cctx.stats.num_matches++;
/* Catchup: look back to extend match from found match */ /* Catchup: look back to extend match from found match */
while (ip > anchor && match > istart && ip[-1] == match[-1]) { while (cctx.ip > cctx.anchor && match > cctx.ibase && cctx.ip[-1] == match[-1]) {
ip--; cctx.ip--;
match--; match--;
} }
/* Encode literals */ /* Encode literals */
{ {
unsigned const litLength = (unsigned)(ip - anchor); unsigned const litLength = (unsigned)(cctx.ip - cctx.anchor);
token = op++; token = cctx.op++;
compressStats.total_literal_length += litLength; cctx.stats.total_literal_length += litLength;
#ifdef LDM_DEBUG #ifdef LDM_DEBUG
printf("Cur position: %zu\n", anchor - istart); printf("Cur position: %zu\n", cctx.anchor - cctx.ibase);
printf("LitLength %zu. (Match offset). %zu\n", litLength, ip - match); printf("LitLength %zu. (Match offset). %zu\n", litLength, cctx.ip - match);
#endif #endif
if (litLength >= RUN_MASK) { if (litLength >= RUN_MASK) {
int len = (int)litLength - RUN_MASK; int len = (int)litLength - RUN_MASK;
*token = (RUN_MASK << ML_BITS); *token = (RUN_MASK << ML_BITS);
for (; len >= 255; len -= 255) { for (; len >= 255; len -= 255) {
*op++ = 255; *(cctx.op)++ = 255;
} }
*op++ = (BYTE)len; *(cctx.op)++ = (BYTE)len;
} else { } else {
*token = (BYTE)(litLength << ML_BITS); *token = (BYTE)(litLength << ML_BITS);
} }
#ifdef LDM_DEBUG #ifdef LDM_DEBUG
printf("Literals "); printf("Literals ");
fwrite(anchor, litLength, 1, stdout); fwrite(cctx.anchor, litLength, 1, stdout);
printf("\n"); printf("\n");
#endif #endif
memcpy(op, anchor, litLength); memcpy(cctx.op, cctx.anchor, litLength);
op += litLength; cctx.op += litLength;
} }
_next_match: _next_match:
/* Encode offset */ /* Encode offset */
{ {
/* /*
LDM_writeLE16(op, ip-match); LDM_writeLE16(cctx.op, cctx.ip-match);
op += 2; cctx.op += 2;
*/ */
LDM_write32(op, ip - match); LDM_write32(cctx.op, cctx.ip - match);
op += 4; cctx.op += 4;
compressStats.total_offset += (ip - match); cctx.stats.total_offset += (cctx.ip - match);
} }
/* Encode Match Length */ /* Encode Match Length */
{ {
unsigned matchCode; unsigned matchCode;
matchCode = LDM_count(ip + MINMATCH, match + MINMATCH, matchCode = LDM_count(cctx.ip + MINMATCH, match + MINMATCH,
matchlimit); cctx.ihashLimit);
#ifdef LDM_DEBUG #ifdef LDM_DEBUG
printf("Match length %zu\n", matchCode + MINMATCH); printf("Match length %zu\n", matchCode + MINMATCH);
fwrite(ip, MINMATCH + matchCode, 1, stdout); fwrite(cctx.ip, MINMATCH + matchCode, 1, stdout);
printf("\n"); printf("\n");
#endif #endif
compressStats.total_match_length += matchCode + MINMATCH; cctx.stats.total_match_length += matchCode + MINMATCH;
unsigned ctr = 1; unsigned ctr = 1;
ip++; cctx.ip++;
for (; ctr < MINMATCH + matchCode; ip++, ctr++) { for (; ctr < MINMATCH + matchCode; cctx.ip++, ctr++) {
LDM_put_position(ip, hashTable, istart); LDM_putHashOfCurrentPosition(&cctx);
} }
// ip += MINMATCH + matchCode; // cctx.ip += MINMATCH + matchCode;
if (matchCode >= ML_MASK) { if (matchCode >= ML_MASK) {
*token += ML_MASK; *token += ML_MASK;
matchCode -= ML_MASK; matchCode -= ML_MASK;
LDM_write32(op, 0xFFFFFFFF); LDM_write32(cctx.op, 0xFFFFFFFF);
while (matchCode >= 4*0xFF) { while (matchCode >= 4*0xFF) {
op += 4; cctx.op += 4;
LDM_write32(op, 0xffffffff); LDM_write32(cctx.op, 0xffffffff);
matchCode -= 4*0xFF; matchCode -= 4*0xFF;
} }
op += matchCode / 255; cctx.op += matchCode / 255;
*op++ = (BYTE)(matchCode % 255); *(cctx.op)++ = (BYTE)(matchCode % 255);
} else { } else {
*token += (BYTE)(matchCode); *token += (BYTE)(matchCode);
} }
@ -323,57 +368,58 @@ _next_match:
#endif #endif
} }
anchor = ip; cctx.anchor = cctx.ip;
LDM_put_position(ip, hashTable, istart); LDM_putPosition(cctx.ip, cctx.hashTable, cctx.ibase);
forwardH = LDM_hash_position(++ip); forwardH = LDM_hash_position(++cctx.ip);
lastHash = ip; lastHash = cctx.ip;
} }
_last_literals: _last_literals:
/* Encode last literals */ /* Encode last literals */
{ {
size_t const lastRun = (size_t)(iend - anchor); size_t const lastRun = (size_t)(cctx.iend - cctx.anchor);
if (lastRun >= RUN_MASK) { if (lastRun >= RUN_MASK) {
size_t accumulator = lastRun - RUN_MASK; size_t accumulator = lastRun - RUN_MASK;
*op++ = RUN_MASK << ML_BITS; *(cctx.op)++ = RUN_MASK << ML_BITS;
for(; accumulator >= 255; accumulator -= 255) { for(; accumulator >= 255; accumulator -= 255) {
*op++ = 255; *(cctx.op)++ = 255;
} }
*op++ = (BYTE)accumulator; *(cctx.op)++ = (BYTE)accumulator;
} else { } else {
*op++ = (BYTE)(lastRun << ML_BITS); *(cctx.op)++ = (BYTE)(lastRun << ML_BITS);
} }
memcpy(op, anchor, lastRun); memcpy(cctx.op, cctx.anchor, lastRun);
op += lastRun; cctx.op += lastRun;
} }
LDM_printCompressStats(&compressStats); LDM_printCompressStats(&cctx.stats);
return (op - (BYTE *)dst); return (cctx.op - (BYTE *)cctx.obase);
} }
typedef struct LDM_DCtx { typedef struct LDM_DCtx {
const BYTE *ibase; /* Pointer to base of input */
const BYTE *ip; /* Pointer to current input position */
const BYTE *iend; /* End of source */
const BYTE *obase; /* Pointer to base of output */
BYTE *op; /* Pointer to output */
const BYTE *oend; /* Pointer to end of output */
size_t compressSize; size_t compressSize;
size_t maxDecompressSize; size_t maxDecompressSize;
const BYTE *ibase; /* Base of input */
const BYTE *ip; /* Current input position */
const BYTE *iend; /* End of source */
const BYTE *obase; /* Base of output */
BYTE *op; /* Current output position */
const BYTE *oend; /* End of output */
} LDM_DCtx; } LDM_DCtx;
static void LDM_initializeDCtx(LDM_DCtx *dctx, static void LDM_initializeDCtx(LDM_DCtx *dctx,
const void *src, size_t compressSize, const void *src, size_t compressSize,
void *dst, size_t maxDecompressSize) { void *dst, size_t maxDecompressSize) {
dctx->ibase = src;
dctx->ip = (const BYTE *)src;
dctx->iend = dctx->ip + compressSize;
dctx->op = dst;
dctx->oend = dctx->op + maxDecompressSize;
dctx->compressSize = compressSize; dctx->compressSize = compressSize;
dctx->maxDecompressSize = maxDecompressSize; dctx->maxDecompressSize = maxDecompressSize;
dctx->ibase = src;
dctx->ip = (const BYTE *)src;
dctx->iend = dctx->ip + dctx->compressSize;
dctx->op = dst;
dctx->oend = dctx->op + dctx->maxDecompressSize;
} }
size_t LDM_decompress(const void *src, size_t compressSize, size_t LDM_decompress(const void *src, size_t compressSize,
@ -382,12 +428,11 @@ size_t LDM_decompress(const void *src, size_t compressSize,
LDM_initializeDCtx(&dctx, src, compressSize, dst, maxDecompressSize); LDM_initializeDCtx(&dctx, src, compressSize, dst, maxDecompressSize);
BYTE *cpy; BYTE *cpy;
while (dctx.ip < dctx.iend) {
size_t length; size_t length;
const BYTE *match; const BYTE *match;
size_t offset; size_t offset;
while (dctx.ip < dctx.iend) {
/* get literal length */ /* get literal length */
unsigned const token = *(dctx.ip)++; unsigned const token = *(dctx.ip)++;
if ((length = (token >> ML_BITS)) == RUN_MASK) { if ((length = (token >> ML_BITS)) == RUN_MASK) {
@ -417,6 +462,8 @@ size_t LDM_decompress(const void *src, size_t compressSize,
offset = LDM_readLE16(dctx.ip); offset = LDM_readLE16(dctx.ip);
dctx.ip += 2; dctx.ip += 2;
*/ */
//TODO : dynamic offset size
offset = LDM_read32(dctx.ip); offset = LDM_read32(dctx.ip);
dctx.ip += 4; dctx.ip += 4;
#ifdef LDM_DEBUG #ifdef LDM_DEBUG