facebook
/
zstd
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Update CLI and link from educational decoder

dev
Sean Purcell 2017-02-23 13:06:50 -08:00
parent 485ca8c352
commit 3cd8d50c34
2 changed files with 63 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
contrib/educational_decoder/README.md
View File

@ -17,3 +17,13 @@ It also contains implementations of Huffman and FSE table decoding.
harness <input-file> <output-file> [dictionary]
As an additional resource to be used with this decoder,
see the `decodecorpus` tool in the [tests] directory.
It generates valid Zstandard frames that can be used to verify
a Zstandard decoder implementation.
Note that to use the tool to verify this decoder implementation,
the --content-size flag should be set,
as this decoder does not handle streaming decoding,
and so it must know the decompressed size in advance.
[tests]: https://github.com/facebook/zstd/blob/dev/tests/

80
tests/decodecorpus.c
View File

@ -235,12 +235,16 @@ typedef struct {
* Generator Functions
*********************************************************/
struct {
int contentSize; /* force the content size to be present */
} opts; /* advanced options on generation */
/* Generate and write a random frame header */
static void writeFrameHeader(U32* seed, frame_t* frame)
{
BYTE* const op = frame->data;
size_t pos = 0;
frameHeader_t fl;
frameHeader_t fh;
BYTE windowByte = 0;
@ -248,7 +252,7 @@ static void writeFrameHeader(U32* seed, frame_t* frame)
int contentSizeFlag = 0;
int fcsCode = 0;
memset(&fl, 0, sizeof(fl));
memset(&fh, 0, sizeof(fh));
/* generate window size */
{
@ -256,8 +260,8 @@ static void writeFrameHeader(U32* seed, frame_t* frame)
int const exponent = RAND(seed) % (MAX_WINDOW_LOG - 10);
int const mantissa = RAND(seed) % 8;
windowByte = (exponent << 3) | mantissa;
fl.windowSize = (1U << (exponent + 10));
fl.windowSize += fl.windowSize / 8 * mantissa;
fh.windowSize = (1U << (exponent + 10));
fh.windowSize += fh.windowSize / 8 * mantissa;
}
{
@ -273,28 +277,28 @@ static void writeFrameHeader(U32* seed, frame_t* frame)
/* 0 size frame */
highBit = 0;
}
fl.contentSize = highBit ? highBit + (RAND(seed) % highBit) : 0;
fh.contentSize = highBit ? highBit + (RAND(seed) % highBit) : 0;
/* provide size sometimes */
contentSizeFlag = RAND(seed) & 1;
contentSizeFlag = opts.contentSize | (RAND(seed) & 1);
if (contentSizeFlag && !(RAND(seed) & 7)) {
if (contentSizeFlag && (fh.contentSize == 0 || !(RAND(seed) & 7))) {
/* do single segment sometimes */
fl.windowSize = fl.contentSize;
fh.windowSize = fh.contentSize;
singleSegment = 1;
}
}
if (contentSizeFlag) {
/* Determine how large fcs field has to be */
int minFcsCode = (fl.contentSize >= 256) +
(fl.contentSize >= 65536 + 256) +
(fl.contentSize > 0xFFFFFFFFU);
int minFcsCode = (fh.contentSize >= 256) +
(fh.contentSize >= 65536 + 256) +
(fh.contentSize > 0xFFFFFFFFU);
if (!singleSegment && !minFcsCode) {
minFcsCode = 1;
}
fcsCode = minFcsCode + (RAND(seed) % (4 - minFcsCode));
if (fcsCode == 1 && fl.contentSize < 256) fcsCode++;
if (fcsCode == 1 && fh.contentSize < 256) fcsCode++;
}
/* write out the header */
@ -314,20 +318,20 @@ static void writeFrameHeader(U32* seed, frame_t* frame)
if (contentSizeFlag) {
switch (fcsCode) {
default: /* Impossible */
case 0: op[pos++] = fl.contentSize; break;
case 1: MEM_writeLE16(op + pos, fl.contentSize - 256); pos += 2; break;
case 2: MEM_writeLE32(op + pos, fl.contentSize); pos += 4; break;
case 3: MEM_writeLE64(op + pos, fl.contentSize); pos += 8; break;
case 0: op[pos++] = fh.contentSize; break;
case 1: MEM_writeLE16(op + pos, fh.contentSize - 256); pos += 2; break;
case 2: MEM_writeLE32(op + pos, fh.contentSize); pos += 4; break;
case 3: MEM_writeLE64(op + pos, fh.contentSize); pos += 8; break;
}
}
DISPLAYLEVEL(2, " frame content size:\t%zu\n", fl.contentSize);
DISPLAYLEVEL(2, " frame window size:\t%u\n", fl.windowSize);
DISPLAYLEVEL(2, " frame content size:\t%zu\n", fh.contentSize);
DISPLAYLEVEL(2, " frame window size:\t%u\n", fh.windowSize);
DISPLAYLEVEL(2, " content size flag:\t%d\n", contentSizeFlag);
DISPLAYLEVEL(2, " single segment flag:\t%d\n", singleSegment);
frame->data = op + pos;
frame->header = fl;
frame->header = fh;
}
/* Write a literal block in either raw or RLE form, return the literals size */
@ -1245,6 +1249,8 @@ static int generateFile(U32 seed, const char* const path,
{
frame_t fr;
DISPLAY("seed: %u\n", seed);
generateFrame(seed, &fr);
outputBuffer(fr.dataStart, (BYTE*)fr.data - (BYTE*)fr.dataStart, path);
@ -1260,11 +1266,6 @@ static int generateCorpus(U32 seed, unsigned numFiles, const char* const path,
char outPath[MAX_PATH];
unsigned fnum;
if (!path) {
DISPLAY("Error: valid path is required in multiple files mode\n");
return 1;
}
DISPLAY("seed: %u\n", seed);
for (fnum = 0; fnum < numFiles; fnum++) {
@ -1330,7 +1331,15 @@ static void usage(const char* programName)
DISPLAY( " -t : activate test mode (test files against libzstd instead of outputting them)\n");
DISPLAY( " -T# : length of time to run tests for\n");
DISPLAY( " -v : increase verbosity level (default:0, max:7)\n");
DISPLAY( " -h : display help and exit\n");
DISPLAY( " -h/H : display help/long help and exit\n");
}
static void advancedUsage(const char* programName)
{
usage(programName);
DISPLAY( "\n");
DISPLAY( "Advanced arguments :\n");
DISPLAY( " --content-size : always include the content size in the frame header\n");
}
int main(int argc, char** argv)
@ -1359,6 +1368,9 @@ int main(int argc, char** argv)
case 'h':
usage(argv[0]);
return 0;
case 'H':
advancedUsage(argv[0]);
return 0;
case 'v':
argument++;
g_displayLevel++;
@ -1395,6 +1407,16 @@ int main(int argc, char** argv)
argument++;
testMode = 1;
break;
case '-':
argument++;
if (strcmp(argument, "content-size") == 0) {
opts.contentSize = 1;
} else {
advancedUsage(argv[0]);
return 1;
}
argument += strlen(argument);
break;
default:
usage(argv[0]);
return 1;
@ -1414,11 +1436,15 @@ int main(int argc, char** argv)
}
}
if (!path) {
DISPLAY("Error: path is required in file generation mode\n");
usage(argv[0]);
return 1;
}
if (numFiles == 0) {
return generateFile(seed, path, origPath);
} else {
return generateCorpus(seed, numFiles, path, origPath);
}
return 0;
}