From 9ad11bea4e83f008919218581488420b539a1a05 Mon Sep 17 00:00:00 2001 From: Nick Terrell Date: Mon, 19 Jun 2017 12:24:14 -0700 Subject: [PATCH] [linux] Fix decompression memory allocation --- contrib/linux-kernel/lib/zstd/decompress.c | 34 +++++++++++-------- contrib/linux-kernel/test/UserlandTest.cpp | 15 +++++++-- contrib/linux-kernel/zstd.diff | 38 +++++++++++++--------- 3 files changed, 55 insertions(+), 32 deletions(-) diff --git a/contrib/linux-kernel/lib/zstd/decompress.c b/contrib/linux-kernel/lib/zstd/decompress.c index def10ea4..ec673d7e 100644 --- a/contrib/linux-kernel/lib/zstd/decompress.c +++ b/contrib/linux-kernel/lib/zstd/decompress.c @@ -2212,6 +2212,20 @@ ZSTD_DStream *ZSTD_initDStream(size_t maxWindowSize, void *workspace, size_t wor zds->ddict = zds->ddictLocal; zds->legacyVersion = 0; zds->hostageByte = 0; + + { + size_t const blockSize = MIN(zds->maxWindowSize, ZSTD_BLOCKSIZE_ABSOLUTEMAX); + size_t const neededOutSize = zds->maxWindowSize + blockSize + WILDCOPY_OVERLENGTH * 2; + + zds->inBuff = (char *)ZSTD_malloc(blockSize, zds->customMem); + zds->inBuffSize = blockSize; + zds->outBuff = (char *)ZSTD_malloc(neededOutSize, zds->customMem); + zds->outBuffSize = neededOutSize; + if (zds->inBuff == NULL || zds->outBuff == NULL) { + ZSTD_freeDStream(zds); + return NULL; + } + } return zds; } @@ -2333,25 +2347,17 @@ size_t ZSTD_decompressStream(ZSTD_DStream *zds, ZSTD_outBuffer *output, ZSTD_inB if (zds->fParams.windowSize > zds->maxWindowSize) return ERROR(frameParameter_windowTooLarge); - /* Adapt buffer sizes to frame header instructions */ + /* Buffers are preallocated, but double check */ { - size_t const blockSize = MIN(zds->fParams.windowSize, ZSTD_BLOCKSIZE_ABSOLUTEMAX); - size_t const neededOutSize = zds->fParams.windowSize + blockSize + WILDCOPY_OVERLENGTH * 2; - zds->blockSize = blockSize; + size_t const blockSize = MIN(zds->maxWindowSize, ZSTD_BLOCKSIZE_ABSOLUTEMAX); + size_t const neededOutSize = zds->maxWindowSize + blockSize + WILDCOPY_OVERLENGTH * 2; if (zds->inBuffSize < blockSize) { - ZSTD_free(zds->inBuff, zds->customMem); - zds->inBuffSize = blockSize; - zds->inBuff = (char *)ZSTD_malloc(blockSize, zds->customMem); - if (zds->inBuff == NULL) - return ERROR(memory_allocation); + return ERROR(GENERIC); } if (zds->outBuffSize < neededOutSize) { - ZSTD_free(zds->outBuff, zds->customMem); - zds->outBuffSize = neededOutSize; - zds->outBuff = (char *)ZSTD_malloc(neededOutSize, zds->customMem); - if (zds->outBuff == NULL) - return ERROR(memory_allocation); + return ERROR(GENERIC); } + zds->blockSize = blockSize; } zds->stage = zdss_read; } diff --git a/contrib/linux-kernel/test/UserlandTest.cpp b/contrib/linux-kernel/test/UserlandTest.cpp index 73b30be4..03058382 100644 --- a/contrib/linux-kernel/test/UserlandTest.cpp +++ b/contrib/linux-kernel/test/UserlandTest.cpp @@ -280,9 +280,9 @@ TEST(Block, ContentSize) { TEST(Block, CCtxLevelIncrease) { std::string c; - auto cctx = createCCtx(6); + auto cctx = createCCtx(22); auto dctx = createDCtx(); - for (int level = 1; level <= 6; ++level) { + for (int level = 1; level <= 22; ++level) { auto compressed = compress(*cctx, kData, level); auto const decompressed = decompress(*dctx, compressed, kData.size()); EXPECT_EQ(kData, decompressed); @@ -478,6 +478,17 @@ TEST(Stream, Flush) { EXPECT_EQ(kData, decompressed); } +TEST(Stream, DStreamLevelIncrease) { + auto zds = createDStream(); + for (int level = 1; level <= 22; ++level) { + auto zcs = createCStream(level); + auto compressed = compress(*zcs, kData); + ZSTD_resetDStream(zds.get()); + auto const decompressed = decompress(*zds, compressed, kData.size()); + EXPECT_EQ(kData, decompressed); + } +} + #define TEST_SYMBOL(symbol) \ do { \ extern void *__##symbol; \ diff --git a/contrib/linux-kernel/zstd.diff b/contrib/linux-kernel/zstd.diff index 285961ab..1fa64ff8 100644 --- a/contrib/linux-kernel/zstd.diff +++ b/contrib/linux-kernel/zstd.diff @@ -5068,10 +5068,10 @@ index 0000000..42236a3 +MODULE_DESCRIPTION("Zstd Compressor"); diff --git a/lib/zstd/decompress.c b/lib/zstd/decompress.c new file mode 100644 -index 0000000..def10ea +index 0000000..ec673d7 --- /dev/null +++ b/lib/zstd/decompress.c -@@ -0,0 +1,2508 @@ +@@ -0,0 +1,2514 @@ +/** + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. @@ -7286,6 +7286,20 @@ index 0000000..def10ea + zds->ddict = zds->ddictLocal; + zds->legacyVersion = 0; + zds->hostageByte = 0; ++ ++ { ++ size_t const blockSize = MIN(zds->maxWindowSize, ZSTD_BLOCKSIZE_ABSOLUTEMAX); ++ size_t const neededOutSize = zds->maxWindowSize + blockSize + WILDCOPY_OVERLENGTH * 2; ++ ++ zds->inBuff = (char *)ZSTD_malloc(blockSize, zds->customMem); ++ zds->inBuffSize = blockSize; ++ zds->outBuff = (char *)ZSTD_malloc(neededOutSize, zds->customMem); ++ zds->outBuffSize = neededOutSize; ++ if (zds->inBuff == NULL || zds->outBuff == NULL) { ++ ZSTD_freeDStream(zds); ++ return NULL; ++ } ++ } + return zds; +} + @@ -7407,25 +7421,17 @@ index 0000000..def10ea + if (zds->fParams.windowSize > zds->maxWindowSize) + return ERROR(frameParameter_windowTooLarge); + -+ /* Adapt buffer sizes to frame header instructions */ ++ /* Buffers are preallocated, but double check */ + { -+ size_t const blockSize = MIN(zds->fParams.windowSize, ZSTD_BLOCKSIZE_ABSOLUTEMAX); -+ size_t const neededOutSize = zds->fParams.windowSize + blockSize + WILDCOPY_OVERLENGTH * 2; -+ zds->blockSize = blockSize; ++ size_t const blockSize = MIN(zds->maxWindowSize, ZSTD_BLOCKSIZE_ABSOLUTEMAX); ++ size_t const neededOutSize = zds->maxWindowSize + blockSize + WILDCOPY_OVERLENGTH * 2; + if (zds->inBuffSize < blockSize) { -+ ZSTD_free(zds->inBuff, zds->customMem); -+ zds->inBuffSize = blockSize; -+ zds->inBuff = (char *)ZSTD_malloc(blockSize, zds->customMem); -+ if (zds->inBuff == NULL) -+ return ERROR(memory_allocation); ++ return ERROR(GENERIC); + } + if (zds->outBuffSize < neededOutSize) { -+ ZSTD_free(zds->outBuff, zds->customMem); -+ zds->outBuffSize = neededOutSize; -+ zds->outBuff = (char *)ZSTD_malloc(neededOutSize, zds->customMem); -+ if (zds->outBuff == NULL) -+ return ERROR(memory_allocation); ++ return ERROR(GENERIC); + } ++ zds->blockSize = blockSize; + } + zds->stage = zdss_read; + }