facebook
/
zstd
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

updated pzstd and largeNbDicts to use the new FileNamesTable* abstraction

dev
Yann Collet 2019-11-06 09:10:05 -08:00
parent 7977899538
commit 31a0abbfda
7 changed files with 47 additions and 52 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
contrib/largeNbDicts/largeNbDicts.c
View File

@ -429,14 +429,14 @@ void shuffleDictionaries(ddict_collection_t dicts)
{
size_t const nbDicts = dicts.nbDDict;
for (size_t r=0; r<nbDicts; r++) {
size_t const d = rand() % nbDicts;
size_t const d = (size_t)rand() % nbDicts;
ZSTD_DDict* tmpd = dicts.ddicts[d];
dicts.ddicts[d] = dicts.ddicts[r];
dicts.ddicts[r] = tmpd;
}
for (size_t r=0; r<nbDicts; r++) {
size_t const d1 = rand() % nbDicts;
size_t const d2 = rand() % nbDicts;
size_t const d1 = (size_t)rand() % nbDicts;
size_t const d2 = (size_t)rand() % nbDicts;
ZSTD_DDict* tmpd = dicts.ddicts[d1];
dicts.ddicts[d1] = dicts.ddicts[d2];
dicts.ddicts[d2] = tmpd;
@ -528,7 +528,7 @@ size_t decompress(const void* src, size_t srcSize, void* dst, size_t dstCapacity
static int benchMem(slice_collection_t dstBlocks,
slice_collection_t srcBlocks,
ddict_collection_t dictionaries,
int nbRounds)
unsigned nbRounds)
{
assert(dstBlocks.nbSlices == srcBlocks.nbSlices);
@ -586,7 +586,7 @@ int bench(const char** fileNameTable, unsigned nbFiles,
const char* dictionary,
size_t blockSize, int clevel,
unsigned nbDictMax, unsigned nbBlocks,
int nbRounds)
unsigned nbRounds)
{
int result = 0;
@ -707,7 +707,7 @@ static unsigned readU32FromChar(const char** stringPtr)
while ((**stringPtr >='0') && (**stringPtr <='9')) {
unsigned const max = (((unsigned)(-1)) / 10) - 1;
assert(result <= max); /* check overflow */
result *= 10, result += **stringPtr - '0', (*stringPtr)++ ;
result *= 10, result += (unsigned)**stringPtr - '0', (*stringPtr)++ ;
}
if ((**stringPtr=='K') || (**stringPtr=='M')) {
unsigned const maxK = ((unsigned)(-1)) >> 10;
@ -729,7 +729,7 @@ static unsigned readU32FromChar(const char** stringPtr)
* If yes, @return 1 and advances *stringPtr to the position which immediately follows longCommand.
* @return 0 and doesn't modify *stringPtr otherwise.
*/
static unsigned longCommandWArg(const char** stringPtr, const char* longCommand)
static int longCommandWArg(const char** stringPtr, const char* longCommand)
{
size_t const comSize = strlen(longCommand);
int const result = !strncmp(*stringPtr, longCommand, comSize);
@ -765,7 +765,7 @@ int bad_usage(const char* exeName)
int main (int argc, const char** argv)
{
int recursiveMode = 0;
int nbRounds = BENCH_TIME_DEFAULT_S;
unsigned nbRounds = BENCH_TIME_DEFAULT_S;
const char* const exeName = argv[0];
if (argc < 2) return bad_usage(exeName);
@ -791,26 +791,27 @@ int main (int argc, const char** argv)
if (longCommandWArg(&argument, "--blockSize=")) { blockSize = readU32FromChar(&argument); continue; }
if (longCommandWArg(&argument, "--nbDicts=")) { nbDicts = readU32FromChar(&argument); continue; }
if (longCommandWArg(&argument, "--nbBlocks=")) { nbBlocks = readU32FromChar(&argument); continue; }
if (longCommandWArg(&argument, "--clevel=")) { cLevel = readU32FromChar(&argument); continue; }
if (longCommandWArg(&argument, "-")) { cLevel = readU32FromChar(&argument); continue; }
if (longCommandWArg(&argument, "--clevel=")) { cLevel = (int)readU32FromChar(&argument); continue; }
if (longCommandWArg(&argument, "-")) { cLevel = (int)readU32FromChar(&argument); continue; }
/* anything that's not a command is a filename */
nameTable[nameIdx++] = argument;
}
const char** filenameTable = nameTable;
unsigned nbFiles = nameIdx;
char* buffer_containing_filenames = NULL;
FileNamesTable* filenameTable;
if (recursiveMode) {
#ifndef UTIL_HAS_CREATEFILELIST
assert(0); /* missing capability, do not run */
#endif
filenameTable = UTIL_createFileList(nameTable, nameIdx, &buffer_containing_filenames, &nbFiles, 1 /* follow_links */);
filenameTable = UTIL_createExpandedFNT(nameTable, nameIdx, 1 /* follow_links */);
} else {
filenameTable = UTIL_createFileNamesTable(nameTable, nameIdx, NULL);
nameTable = NULL; /* UTIL_createFileNamesTable() takes ownership of nameTable */
}
int result = bench(filenameTable, nbFiles, dictionary, blockSize, cLevel, nbDicts, nbBlocks, nbRounds);
int result = bench(filenameTable->fileNames, (unsigned)filenameTable->tableSize, dictionary, blockSize, cLevel, nbDicts, nbBlocks, nbRounds);
free(buffer_containing_filenames);
UTIL_freeFileNamesTable(filenameTable);
free(nameTable);
return result;

14
contrib/pzstd/Options.cpp
View File

@ -337,23 +337,19 @@ Options::Status Options::parse(int argc, const char **argv) {
// Translate input files/directories into files to (de)compress
if (recursive) {
char *scratchBuffer = nullptr;
unsigned numFiles = 0;
const char **files =
UTIL_createFileList(localInputFiles.data(), localInputFiles.size(),
&scratchBuffer, &numFiles, followLinks);
FileNamesTable* const files = UTIL_createExpandedFNT(localInputFiles.data(), localInputFiles.size(), followLinks);
if (files == nullptr) {
std::fprintf(stderr, "Error traversing directories\n");
return Status::Failure;
}
auto guard =
makeScopeGuard([&] { UTIL_freeFileList(files, scratchBuffer); });
if (numFiles == 0) {
makeScopeGuard([&] { UTIL_freeFileNamesTable(files); });
if (files->tableSize == 0) {
std::fprintf(stderr, "No files found\n");
return Status::Failure;
}
inputFiles.resize(numFiles);
std::copy(files, files + numFiles, inputFiles.begin());
inputFiles.resize(files->tableSize);
std::copy(files->fileNames, files->fileNames + files->tableSize, inputFiles.begin());
} else {
inputFiles.resize(localInputFiles.size());
std::copy(localInputFiles.begin(), localInputFiles.end(),

6
doc/zstd_manual.html
View File

@ -967,12 +967,6 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
* tables. However, this model incurs no start-up cost (as long as the
* working context's tables can be reused). For small inputs, this can be
* faster than copying the CDict's tables.
*
* - The CDict's tables are not used at all, and instead we use the working
* context alone to reload the dictionary and use params based on the source
* size. See ZSTD_compress_insertDictionary() and ZSTD_compress_usingDict().
* This method is effective when the dictionary sizes are very small relative
* to the input size, and the input size is fairly large to begin with.
*
* - The CDict's tables are not used at all, and instead we use the working
* context alone to reload the dictionary and use params based on the source

17
programs/util.c
View File

@ -374,7 +374,7 @@ static size_t getTotalTableSize(FileNamesTable* table)
}
FileNamesTable*
UTIL_concatenateTwoTables(FileNamesTable* table1, FileNamesTable* table2)
UTIL_mergeFileNamesTable(FileNamesTable* table1, FileNamesTable* table2)
{
unsigned newTableIdx = 0;
size_t pos = 0;
@ -587,13 +587,11 @@ const char* UTIL_getFileExtension(const char* infilename)
}
static FileNamesTable*
createFNT_fromFNT(FileNamesTable* fnt, int followLinks)
FileNamesTable*
UTIL_createExpandedFNT(const char** inputNames, size_t nbIfns, int followLinks)
{
size_t pos;
size_t const nbIfns = fnt->tableSize;
unsigned nbFiles;
const char** const inputNames = fnt->fileNames;
char* buf = (char*)malloc(LIST_SIZE_INCREASE);
char* bufend = buf + LIST_SIZE_INCREASE;
@ -637,12 +635,11 @@ createFNT_fromFNT(FileNamesTable* fnt, int followLinks)
}
FileNamesTable*
UTIL_expandFileNamesTable(FileNamesTable* fnt, int followLinks)
void UTIL_expandFNT(FileNamesTable** fnt, int followLinks)
{
FileNamesTable* const newFNT = createFNT_fromFNT(fnt, followLinks);
UTIL_freeFileNamesTable(fnt);
return newFNT;
FileNamesTable* const newFNT = UTIL_createExpandedFNT((*fnt)->fileNames, (*fnt)->tableSize, followLinks);
UTIL_freeFileNamesTable(*fnt);
*fnt = newFNT;
}

23
programs/util.h
View File

@ -188,21 +188,28 @@ UTIL_createFileNamesTable(const char** filenames, size_t tableSize, char* buf);
*/
void UTIL_freeFileNamesTable(FileNamesTable* table);
/*! UTIL_concatenateTwoTables():
/*! UTIL_mergeFileNamesTable():
* @return : FileNamesTable*, concatenation of @table1 and @table2
* note: @table1 and @table2 are consumed (freed) by this operation
*/
FileNamesTable*
UTIL_concatenateTwoTables(FileNamesTable* table1, FileNamesTable* table2);
UTIL_mergeFileNamesTable(FileNamesTable* table1, FileNamesTable* table2);
/*! UTIL_expandFileNamesTable() :
* read names from @fnt, expand those corresponding to directories
* @return : an expanded FileNamesTable*, with only file names,
* or NULL in case of error.
* Note: the function takes ownership of fnt, and consumes it (free it)
/*! UTIL_expandFNT() :
* read names from @fnt, and expand those corresponding to directories
* update @fnt, now containing only file names,
* @return : 0 in case of success, 1 if error
* note : in case of error, @fnt[0] is NULL
*/
FileNamesTable* UTIL_expandFileNamesTable(FileNamesTable* fnt, int followLinks);
void UTIL_expandFNT(FileNamesTable** fnt, int followLinks);
/*! UTIL_createExpandedFNT() :
* read names from @filenames, and expand those corresponding to directories
* @return : an expanded FileNamesTable*, where each name is a file
* or NULL in case of error
*/
FileNamesTable* UTIL_createExpandedFNT(const char** filenames, size_t nbFilenames, int followLinks);
/*! UTIL_allocateFileNamesTable() :

4
programs/zstdcli.c
View File

@ -1029,12 +1029,12 @@ int main(int argCount, const char* argv[])
DISPLAYLEVEL(1, "zstd: error reading %s \n", file_of_names->fileNames[flNb]);
CLEAN_RETURN(1);
}
filenames = UTIL_concatenateTwoTables(filenames, fnt);
filenames = UTIL_mergeFileNamesTable(filenames, fnt);
}
}
if (recursive) { /* at this stage, filenameTable is a list of paths, which can contain both files and directories */
filenames = UTIL_expandFileNamesTable(filenames, followLinks);
UTIL_expandFNT(&filenames, followLinks);
}
#else
(void)followLinks;

2
zlibWrapper/examples/zwrapbench.c
View File

@ -986,7 +986,7 @@ int main(int argCount, char** argv)
#ifdef UTIL_HAS_CREATEFILELIST
if (recursive) {
filenames = UTIL_expandFileNamesTable(filenames, 1);
UTIL_expandFNT(&filenames, 1);
}
#endif