facebook/zstd
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Update compressor and decompressor

Browse Source
This commit is contained in:
Stella Lau 2017-07-06 16:47:08 -07:00
parent b96ad327a4
commit 3bbfa1249e
2 changed files with 33 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

40
contrib/long_distance_matching/ldm.c
View File

@ -20,6 +20,8 @@
#define RUN_BITS (8-ML_BITS) #define RUN_BITS (8-ML_BITS)
#define RUN_MASK ((1U<<RUN_BITS)-1) #define RUN_MASK ((1U<<RUN_BITS)-1)
//#define LDM_DEBUG
typedef uint8_t BYTE; typedef uint8_t BYTE;
typedef uint16_t U16; typedef uint16_t U16;
typedef uint32_t U32; typedef uint32_t U32;
@ -173,6 +175,7 @@ size_t LDM_compress(void const *source, void *dest, size_t source_size,
while (ip < ilimit) { while (ip < ilimit) {
const BYTE *match; const BYTE *match;
BYTE *token; BYTE *token;
/* Find a match */ /* Find a match */
{ {
const BYTE *forwardIp = ip; const BYTE *forwardIp = ip;
@ -196,8 +199,10 @@ size_t LDM_compress(void const *source, void *dest, size_t source_size,
unsigned const litLength = (unsigned)(ip - anchor); unsigned const litLength = (unsigned)(ip - anchor);
token = op++; token = op++;
#ifdef LDM_DEBUG
printf("Cur position: %zu\n", anchor - istart); printf("Cur position: %zu\n", anchor - istart);
printf("LitLength %zu. (Match offset). %zu\n", litLength, ip - match); printf("LitLength %zu. (Match offset). %zu\n", litLength, ip - match);
#endif
/* /*
fwrite(match, 4, 1, stdout); fwrite(match, 4, 1, stdout);
printf("\n"); printf("\n");
@ -207,16 +212,17 @@ size_t LDM_compress(void const *source, void *dest, size_t source_size,
int len = (int)litLength - RUN_MASK; int len = (int)litLength - RUN_MASK;
*token = (RUN_MASK << ML_BITS); *token = (RUN_MASK << ML_BITS);
for (; len >= 255; len -= 255) { for (; len >= 255; len -= 255) {
*op++ = (BYTE)len; *op++ = 255;
} }
*op++ = (BYTE)len;
} else { } else {
*token = (BYTE)(litLength << ML_BITS); *token = (BYTE)(litLength << ML_BITS);
} }
#ifdef LDM_DEBUG
printf("Literals "); printf("Literals ");
fwrite(anchor, litLength, 1, stdout); fwrite(anchor, litLength, 1, stdout);
printf("\n"); printf("\n");
#endif
LDM_wild_copy(op, anchor, op + litLength); LDM_wild_copy(op, anchor, op + litLength);
op += litLength; op += litLength;
} }
@ -232,10 +238,11 @@ _next_match:
unsigned matchCode; unsigned matchCode;
matchCode = LDM_count(ip + MINMATCH, match + MINMATCH, matchCode = LDM_count(ip + MINMATCH, match + MINMATCH,
matchlimit); matchlimit);
#ifdef LDM_DEBUG
printf("Match length %zu\n", matchCode + MINMATCH); printf("Match length %zu\n", matchCode + MINMATCH);
fwrite(ip, MINMATCH + matchCode, 1, stdout); fwrite(ip, MINMATCH + matchCode, 1, stdout);
printf("\n"); printf("\n");
#endif
ip += MINMATCH + matchCode; ip += MINMATCH + matchCode;
if (matchCode >= ML_MASK) { if (matchCode >= ML_MASK) {
*token += ML_MASK; *token += ML_MASK;
@ -251,7 +258,9 @@ _next_match:
} else { } else {
*token += (BYTE)(matchCode); *token += (BYTE)(matchCode);
} }
#ifdef LDM_DEBUG
printf("\n"); printf("\n");
#endif
} }
anchor = ip; anchor = ip;
@ -308,24 +317,33 @@ size_t LDM_decompress(void const *source, void *dest, size_t compressed_size,
length += s; length += s;
} while (s == 255); } while (s == 255);
} }
#ifdef LDM_DEBUG
printf("Literal length: %zu\n", length); printf("Literal length: %zu\n", length);
#endif
/* copy literals */ /* copy literals */
cpy = op + length; cpy = op + length;
#ifdef LDM_DEBUG
printf("Literals ");
fwrite(ip, length, 1, stdout);
printf("\n");
#endif
LDM_wild_copy(op, ip, cpy); LDM_wild_copy(op, ip, cpy);
ip += length; ip += length;
op = cpy; op = cpy;
/* get offset */ /* get offset */
offset = LDM_readLE16(ip); offset = LDM_readLE16(ip);
#ifdef LDM_DEBUG
printf("Offset: %zu\n", offset); printf("Offset: %zu\n", offset);
#endif
ip += 2; ip += 2;
match = op - offset; match = op - offset;
// LDM_write32(op, (U32)offset); // LDM_write32(op, (U32)offset);
/* get matchlength */ /* get matchlength */
length = token & ML_MASK; length = token & ML_MASK;
printf("Match length: %zu\n", length);
if (length == ML_MASK) { if (length == ML_MASK) {
unsigned s; unsigned s;
do { do {
@ -334,16 +352,20 @@ size_t LDM_decompress(void const *source, void *dest, size_t compressed_size,
} while (s == 255); } while (s == 255);
} }
length += MINMATCH; length += MINMATCH;
#ifdef LDM_DEBUG
printf("Match length: %zu\n", length);
#endif
/* copy match */ /* copy match */
cpy = op + length; cpy = op + length;
// Inefficient for now
while (match < cpy - offset) {
*op++ = *match++;
}
} }
// memcpy(dest, source, compressed_size); // memcpy(dest, source, compressed_size);
return compressed_size; return op - (BYTE *)dest;
} }

4
contrib/long_distance_matching/main-ldm.c
View File

@ -229,7 +229,7 @@ static size_t decompress(const char *fname, const char *oname) {
} }
/* go to the location corresponding to the last byte */ /* go to the location corresponding to the last byte */
if (lseek(fdout, statbuf.st_size - 1, SEEK_SET) == -1) { if (lseek(fdout, 2*statbuf.st_size - 1, SEEK_SET) == -1) {
perror("lseek error"); perror("lseek error");
return 1; return 1;
} }
@ -264,7 +264,7 @@ static size_t decompress(const char *fname, const char *oname) {
size_t size_out = LDM_decompress(src, dst, statbuf.st_size, size_t size_out = LDM_decompress(src, dst, statbuf.st_size,
statbuf.st_size); statbuf.st_size);
#endif #endif
ftruncate(fdout, size_out); //ftruncate(fdout, size_out);
close(fdin); close(fdin);
close(fdout); close(fdout);