facebook/zstd
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #2206 from bimbashrestha/stream-size

Browse Source 
[cli] Allowing stdin on --patch-from using --stream-size
This commit is contained in:
Bimba Shrestha 2020-06-18 13:10:11 -05:00 committed by GitHub
commit c496fd261d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 13 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
programs/fileio.c
View File

@ -772,6 +772,8 @@ static void FIO_adjustMemLimitForPatchFromMode(FIO_prefs_t* const prefs,
{
unsigned long long maxSize = MAX(prefs->memLimit, MAX(dictSize, maxSrcFileSize));
unsigned const maxWindowSize = (1U << ZSTD_WINDOWLOG_MAX);
if (maxSize == UTIL_FILESIZE_UNKNOWN)
EXM_THROW(42, "Using --patch-from with stdin requires --stream-size");
assert(maxSize != UTIL_FILESIZE_UNKNOWN);
if (maxSize > maxWindowSize)
EXM_THROW(42, "Can't handle files larger than %u GB\n", maxWindowSize/(1 GB));
@ -839,8 +841,10 @@ static cRess_t FIO_createCResources(FIO_prefs_t* const prefs,
/* need to update memLimit before calling createDictBuffer
* because of memLimit check inside it */
if (prefs->patchFromMode)
FIO_adjustParamsForPatchFromMode(prefs, &comprParams, UTIL_getFileSize(dictFileName), maxSrcFileSize, cLevel);
if (prefs->patchFromMode) {
unsigned long long const ssSize = (unsigned long long)prefs->streamSrcSize;
FIO_adjustParamsForPatchFromMode(prefs, &comprParams, UTIL_getFileSize(dictFileName), ssSize > 0 ? ssSize : maxSrcFileSize, cLevel);
}
ress.dstBuffer = malloc(ress.dstBufferSize);
ress.dictBufferSize = FIO_createDictBuffer(&ress.dictBuffer, dictFileName, prefs); /* works with dictFileName==NULL */
if (!ress.srcBuffer || !ress.dstBuffer)

7
tests/playTests.sh
View File

@ -1304,6 +1304,13 @@ datagen -g5000000 > tmp_patch
zstd -15 --patch-from=tmp_dict tmp_patch 2>&1 | grep "long mode automatically triggered"
rm -rf tmp*
println "\n===> patch-from --stream-size test"
datagen -g1000 -P50 > tmp_dict
datagen -g1000 -P10 > tmp_patch
cat tmp_patch | zstd -f --patch-from=tmp_dict -c -o tmp_patch_diff && die
cat tmp_patch | zstd -f --patch-from=tmp_dict --stream-size=1000 -c -o tmp_patch_diff
rm -rf tmp*
println "\n===> large files tests "
roundTripTest -g270000000 1