Yann Collet
GitHub
commit
983e16a269
|
|
@ -1978,10 +1978,15 @@ void ZSTD_compressBlock_lazy_generic(ZSTD_CCtx *ctx, const void *src, size_t src
|
|||
break; /* nothing found : store previous solution */
|
||||
}
|
||||
|
||||
/* NOTE:
|
||||
* start[-offset+ZSTD_REP_MOVE-1] is undefined behavior.
|
||||
* (-offset+ZSTD_REP_MOVE-1) is unsigned, and is added to start, which
|
||||
* overflows the pointer, which is undefined behavior.
|
||||
*/
|
||||
/* catch up */
|
||||
if (offset) {
|
||||
while ((start > anchor) && (start > base + offset - ZSTD_REP_MOVE) &&
|
||||
(start[-1] == start[-1 - offset + ZSTD_REP_MOVE])) /* only search for offset within prefix */
|
||||
(start[-1] == (start-offset+ZSTD_REP_MOVE)[-1])) /* only search for offset within prefix */
|
||||
{
|
||||
start--;
|
||||
matchLength++;
|
||||
|
|
|
|||
|
|
@ -0,0 +1,85 @@
|
|||
/**
|
||||
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under the BSD-style license found in the
|
||||
* LICENSE file in the root directory of this source tree. An additional grant
|
||||
* of patent rights can be found in the PATENTS file in the same directory.
|
||||
*/
|
||||
|
||||
/*
|
||||
This program takes a file in input,
|
||||
performs a zstd round-trip test (compression - decompress)
|
||||
compares the result with original
|
||||
and generates a crash (double free) on corruption detection.
|
||||
*/
|
||||
|
||||
/*===========================================
|
||||
* Dependencies
|
||||
*==========================================*/
|
||||
#include <stddef.h> /* size_t */
|
||||
#include <stdlib.h> /* malloc, free, exit */
|
||||
#include <stdio.h> /* fprintf */
|
||||
#include <linux/zstd.h>
|
||||
|
||||
/*===========================================
|
||||
* Macros
|
||||
*==========================================*/
|
||||
#define MIN(a,b) ( (a) < (b) ? (a) : (b) )
|
||||
|
||||
static ZSTD_DCtx *dctx = NULL;
|
||||
void *dws = NULL;
|
||||
static void* rBuff = NULL;
|
||||
static size_t buffSize = 0;
|
||||
|
||||
static void crash(int errorCode){
|
||||
/* abort if AFL/libfuzzer, exit otherwise */
|
||||
#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION /* could also use __AFL_COMPILER */
|
||||
abort();
|
||||
#else
|
||||
exit(errorCode);
|
||||
#endif
|
||||
}
|
||||
|
||||
static void decompressCheck(const void* srcBuff, size_t srcBuffSize)
|
||||
{
|
||||
size_t const neededBuffSize = 20 * srcBuffSize;
|
||||
|
||||
/* Allocate all buffers and contexts if not already allocated */
|
||||
if (neededBuffSize > buffSize) {
|
||||
free(rBuff);
|
||||
buffSize = 0;
|
||||
|
||||
rBuff = malloc(neededBuffSize);
|
||||
if (!rBuff) {
|
||||
fprintf(stderr, "not enough memory ! \n");
|
||||
crash(1);
|
||||
}
|
||||
buffSize = neededBuffSize;
|
||||
}
|
||||
if (!dctx) {
|
||||
size_t const workspaceSize = ZSTD_DCtxWorkspaceBound();
|
||||
dws = malloc(workspaceSize);
|
||||
if (!dws) {
|
||||
fprintf(stderr, "not enough memory ! \n");
|
||||
crash(1);
|
||||
}
|
||||
dctx = ZSTD_initDCtx(dws, workspaceSize);
|
||||
if (!dctx) {
|
||||
fprintf(stderr, "not enough memory ! \n");
|
||||
crash(1);
|
||||
}
|
||||
}
|
||||
ZSTD_decompressDCtx(dctx, rBuff, buffSize, srcBuff, srcBuffSize);
|
||||
|
||||
#ifndef SKIP_FREE
|
||||
free(dws); dws = NULL; dctx = NULL;
|
||||
free(rBuff); rBuff = NULL;
|
||||
buffSize = 0;
|
||||
#endif
|
||||
}
|
||||
|
||||
int LLVMFuzzerTestOneInput(const unsigned char *srcBuff, size_t srcBuffSize) {
|
||||
decompressCheck(srcBuff, srcBuffSize);
|
||||
return 0;
|
||||
}
|
||||
|
|
@ -12,12 +12,24 @@ CPPFLAGS += $(IFLAGS)
|
|||
../lib/zstd/libzstd.a: $(OBJECTS)
|
||||
$(AR) $(ARFLAGS) $@ $^
|
||||
|
||||
DecompressCrash: DecompressCrash.o $(OBJECTS) libFuzzer.a
|
||||
$(CXX) $(TEST_CPPFLAGS) $(TEST_CXXFLAGS) $(LDFLAGS) $^ -o $@
|
||||
|
||||
RoundTripCrash: RoundTripCrash.o $(OBJECTS) ../lib/xxhash.o libFuzzer.a
|
||||
$(CXX) $(TEST_CPPFLAGS) $(TEST_CXXFLAGS) $(LDFLAGS) $^ -o $@
|
||||
|
||||
UserlandTest: UserlandTest.cpp ../lib/zstd/libzstd.a ../lib/xxhash.o
|
||||
$(CXX) $(CXXFLAGS) $(CFLAGS) $(CPPFLAGS) $^ googletest/build/googlemock/gtest/libgtest.a googletest/build/googlemock/gtest/libgtest_main.a -o $@
|
||||
|
||||
XXHashUserlandTest: XXHashUserlandTest.cpp ../lib/xxhash.o ../../../lib/common/xxhash.o
|
||||
$(CXX) $(CXXFLAGS) $(CFLAGS) $(CPPFLAGS) $^ googletest/build/googlemock/gtest/libgtest.a googletest/build/googlemock/gtest/libgtest_main.a -o $@
|
||||
|
||||
# Install libfuzzer
|
||||
libFuzzer.a:
|
||||
@$(RM) -rf Fuzzer
|
||||
@git clone https://chromium.googlesource.com/chromium/llvm-project/llvm/lib/Fuzzer
|
||||
@./Fuzzer/build.sh
|
||||
|
||||
# Install googletest
|
||||
.PHONY: googletest
|
||||
googletest:
|
||||
|
|
@ -28,3 +40,4 @@ googletest:
|
|||
|
||||
clean:
|
||||
$(RM) -f *.{o,a} ../lib/zstd/*.{o,a}
|
||||
$(RM) -f DecompressCrash RoundTripCrash UserlandTest XXHashUserlandTest
|
||||
|
|
|
|||
|
|
@ -0,0 +1,162 @@
|
|||
/**
|
||||
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under the BSD-style license found in the
|
||||
* LICENSE file in the root directory of this source tree. An additional grant
|
||||
* of patent rights can be found in the PATENTS file in the same directory.
|
||||
*/
|
||||
|
||||
/*
|
||||
This program takes a file in input,
|
||||
performs a zstd round-trip test (compression - decompress)
|
||||
compares the result with original
|
||||
and generates a crash (double free) on corruption detection.
|
||||
*/
|
||||
|
||||
/*===========================================
|
||||
* Dependencies
|
||||
*==========================================*/
|
||||
#include <stddef.h> /* size_t */
|
||||
#include <stdlib.h> /* malloc, free, exit */
|
||||
#include <stdio.h> /* fprintf */
|
||||
#include <linux/xxhash.h>
|
||||
#include <linux/zstd.h>
|
||||
|
||||
/*===========================================
|
||||
* Macros
|
||||
*==========================================*/
|
||||
#define MIN(a,b) ( (a) < (b) ? (a) : (b) )
|
||||
|
||||
static const int kMaxClevel = 22;
|
||||
|
||||
static ZSTD_CCtx *cctx = NULL;
|
||||
void *cws = NULL;
|
||||
static ZSTD_DCtx *dctx = NULL;
|
||||
void *dws = NULL;
|
||||
static void* cBuff = NULL;
|
||||
static void* rBuff = NULL;
|
||||
static size_t buffSize = 0;
|
||||
|
||||
|
||||
/** roundTripTest() :
|
||||
* Compresses `srcBuff` into `compressedBuff`,
|
||||
* then decompresses `compressedBuff` into `resultBuff`.
|
||||
* Compression level used is derived from first content byte.
|
||||
* @return : result of decompression, which should be == `srcSize`
|
||||
* or an error code if either compression or decompression fails.
|
||||
* Note : `compressedBuffCapacity` should be `>= ZSTD_compressBound(srcSize)`
|
||||
* for compression to be guaranteed to work */
|
||||
static size_t roundTripTest(void* resultBuff, size_t resultBuffCapacity,
|
||||
void* compressedBuff, size_t compressedBuffCapacity,
|
||||
const void* srcBuff, size_t srcBuffSize)
|
||||
{
|
||||
size_t const hashLength = MIN(128, srcBuffSize);
|
||||
unsigned const h32 = xxh32(srcBuff, hashLength, 0);
|
||||
int const cLevel = h32 % kMaxClevel;
|
||||
ZSTD_parameters const params = ZSTD_getParams(cLevel, srcBuffSize, 0);
|
||||
size_t const cSize = ZSTD_compressCCtx(cctx, compressedBuff, compressedBuffCapacity, srcBuff, srcBuffSize, params);
|
||||
if (ZSTD_isError(cSize)) {
|
||||
fprintf(stderr, "Compression error : %u \n", ZSTD_getErrorCode(cSize));
|
||||
return cSize;
|
||||
}
|
||||
return ZSTD_decompressDCtx(dctx, resultBuff, resultBuffCapacity, compressedBuff, cSize);
|
||||
}
|
||||
|
||||
|
||||
static size_t checkBuffers(const void* buff1, const void* buff2, size_t buffSize)
|
||||
{
|
||||
const char* ip1 = (const char*)buff1;
|
||||
const char* ip2 = (const char*)buff2;
|
||||
size_t pos;
|
||||
|
||||
for (pos=0; pos<buffSize; pos++)
|
||||
if (ip1[pos]!=ip2[pos])
|
||||
break;
|
||||
|
||||
return pos;
|
||||
}
|
||||
|
||||
static void crash(int errorCode){
|
||||
/* abort if AFL/libfuzzer, exit otherwise */
|
||||
#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION /* could also use __AFL_COMPILER */
|
||||
abort();
|
||||
#else
|
||||
exit(errorCode);
|
||||
#endif
|
||||
}
|
||||
|
||||
static void roundTripCheck(const void* srcBuff, size_t srcBuffSize)
|
||||
{
|
||||
size_t const neededBuffSize = ZSTD_compressBound(srcBuffSize);
|
||||
|
||||
/* Allocate all buffers and contexts if not already allocated */
|
||||
if (neededBuffSize > buffSize) {
|
||||
free(cBuff);
|
||||
free(rBuff);
|
||||
buffSize = 0;
|
||||
|
||||
cBuff = malloc(neededBuffSize);
|
||||
rBuff = malloc(neededBuffSize);
|
||||
if (!cBuff || !rBuff) {
|
||||
fprintf(stderr, "not enough memory ! \n");
|
||||
crash(1);
|
||||
}
|
||||
buffSize = neededBuffSize;
|
||||
}
|
||||
if (!cctx) {
|
||||
ZSTD_compressionParameters const params = ZSTD_getCParams(kMaxClevel, 0, 0);
|
||||
size_t const workspaceSize = ZSTD_CCtxWorkspaceBound(params);
|
||||
cws = malloc(workspaceSize);
|
||||
if (!cws) {
|
||||
fprintf(stderr, "not enough memory ! \n");
|
||||
crash(1);
|
||||
}
|
||||
cctx = ZSTD_initCCtx(cws, workspaceSize);
|
||||
if (!cctx) {
|
||||
fprintf(stderr, "not enough memory ! \n");
|
||||
crash(1);
|
||||
}
|
||||
}
|
||||
if (!dctx) {
|
||||
size_t const workspaceSize = ZSTD_DCtxWorkspaceBound();
|
||||
dws = malloc(workspaceSize);
|
||||
if (!dws) {
|
||||
fprintf(stderr, "not enough memory ! \n");
|
||||
crash(1);
|
||||
}
|
||||
dctx = ZSTD_initDCtx(dws, workspaceSize);
|
||||
if (!dctx) {
|
||||
fprintf(stderr, "not enough memory ! \n");
|
||||
crash(1);
|
||||
}
|
||||
}
|
||||
|
||||
{ size_t const result = roundTripTest(rBuff, buffSize, cBuff, buffSize, srcBuff, srcBuffSize);
|
||||
if (ZSTD_isError(result)) {
|
||||
fprintf(stderr, "roundTripTest error : %u \n", ZSTD_getErrorCode(result));
|
||||
crash(1);
|
||||
}
|
||||
if (result != srcBuffSize) {
|
||||
fprintf(stderr, "Incorrect regenerated size : %u != %u\n", (unsigned)result, (unsigned)srcBuffSize);
|
||||
crash(1);
|
||||
}
|
||||
if (checkBuffers(srcBuff, rBuff, srcBuffSize) != srcBuffSize) {
|
||||
fprintf(stderr, "Silent decoding corruption !!!");
|
||||
crash(1);
|
||||
}
|
||||
}
|
||||
|
||||
#ifndef SKIP_FREE
|
||||
free(cws); cws = NULL; cctx = NULL;
|
||||
free(dws); dws = NULL; dctx = NULL;
|
||||
free(cBuff); cBuff = NULL;
|
||||
free(rBuff); rBuff = NULL;
|
||||
buffSize = 0;
|
||||
#endif
|
||||
}
|
||||
|
||||
int LLVMFuzzerTestOneInput(const unsigned char *srcBuff, size_t srcBuffSize) {
|
||||
roundTripCheck(srcBuff, srcBuffSize);
|
||||
return 0;
|
||||
}
|
||||
|
|
@ -0,0 +1,191 @@
|
|||
/*
|
||||
* Copyright (c) 2016-present, Facebook, Inc.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under the BSD-style license found in the
|
||||
* LICENSE file in the root directory of this source tree. An additional grant
|
||||
* of patent rights can be found in the PATENTS file in the same directory.
|
||||
*
|
||||
* This program is free software; you can redistribute it and/or modify it under
|
||||
* the terms of the GNU General Public License version 2 as published by the
|
||||
* Free Software Foundation. This program is dual-licensed; you may select
|
||||
* either version 2 of the GNU General Public License ("GPL") or BSD license
|
||||
* ("BSD").
|
||||
*/
|
||||
|
||||
/* DO_XXH should be 32 or 64 for xxh32 and xxh64 respectively */
|
||||
#define DO_XXH 0
|
||||
/* DO_CRC should be 0 or 1 */
|
||||
#define DO_CRC 0
|
||||
/* Buffer size */
|
||||
#define BUFFER_SIZE 4096
|
||||
|
||||
#include <linux/kernel.h>
|
||||
#include <linux/module.h>
|
||||
#include <linux/fs.h>
|
||||
#include <linux/uaccess.h>
|
||||
|
||||
#if DO_XXH
|
||||
#include <linux/xxhash.h>
|
||||
#endif
|
||||
|
||||
#if DO_CRC
|
||||
#include <linux/crc32.h>
|
||||
#endif
|
||||
|
||||
/* Device name to pass to register_chrdev(). */
|
||||
#define DEVICE_NAME "xxhash_test"
|
||||
|
||||
/* Dynamically allocated device major number */
|
||||
static int device_major;
|
||||
|
||||
/*
|
||||
* We reuse the same hash state, and thus can hash only one
|
||||
* file at a time.
|
||||
*/
|
||||
static bool device_is_open;
|
||||
|
||||
static uint64_t total_length;
|
||||
|
||||
|
||||
#if (DO_XXH == 32)
|
||||
|
||||
#define xxh_state xxh32_state
|
||||
#define xxh_reset xxh32_reset
|
||||
#define xxh_update xxh32_update
|
||||
#define xxh_digest xxh32_digest
|
||||
#define XXH_FORMAT "XXH32 = 0x%x"
|
||||
|
||||
#elif (DO_XXH == 64)
|
||||
|
||||
#define xxh_state xxh64_state
|
||||
#define xxh_reset xxh64_reset
|
||||
#define xxh_update xxh64_update
|
||||
#define xxh_digest xxh64_digest
|
||||
#define XXH_FORMAT "XXH64 = 0x%llx"
|
||||
|
||||
#elif DO_XXH
|
||||
|
||||
#error "Invalid value of DO_XXH"
|
||||
|
||||
#endif
|
||||
|
||||
#if DO_XXH
|
||||
|
||||
/* XXH state */
|
||||
static struct xxh_state state;
|
||||
|
||||
#endif /* DO_XXH */
|
||||
|
||||
#if DO_CRC
|
||||
|
||||
static uint32_t crc;
|
||||
|
||||
#endif /* DO_CRC */
|
||||
|
||||
/*
|
||||
* Input buffer used to put data coming from userspace.
|
||||
*/
|
||||
static uint8_t buffer_in[BUFFER_SIZE];
|
||||
|
||||
static int xxhash_test_open(struct inode *i, struct file *f)
|
||||
{
|
||||
if (device_is_open)
|
||||
return -EBUSY;
|
||||
|
||||
device_is_open = true;
|
||||
|
||||
total_length = 0;
|
||||
#if DO_XXH
|
||||
xxh_reset(&state, 0);
|
||||
#endif
|
||||
#if DO_CRC
|
||||
crc = 0xFFFFFFFF;
|
||||
#endif
|
||||
|
||||
printk(KERN_INFO DEVICE_NAME ": opened\n");
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int xxhash_test_release(struct inode *i, struct file *f)
|
||||
{
|
||||
device_is_open = false;
|
||||
|
||||
printk(KERN_INFO DEVICE_NAME ": total_len = %llu\n", total_length);
|
||||
#if DO_XXH
|
||||
printk(KERN_INFO DEVICE_NAME ": " XXH_FORMAT "\n", xxh_digest(&state));
|
||||
#endif
|
||||
#if DO_CRC
|
||||
printk(KERN_INFO DEVICE_NAME ": CRC32 = 0x%08x\n", ~crc);
|
||||
#endif
|
||||
printk(KERN_INFO DEVICE_NAME ": closed\n");
|
||||
return 0;
|
||||
}
|
||||
|
||||
/*
|
||||
* Hash the data given to us from userspace.
|
||||
*/
|
||||
static ssize_t xxhash_test_write(struct file *file, const char __user *buf,
|
||||
size_t size, loff_t *pos)
|
||||
{
|
||||
size_t remaining = size;
|
||||
|
||||
while (remaining > 0) {
|
||||
#if DO_XXH
|
||||
int ret;
|
||||
#endif
|
||||
size_t const copy_size = min(remaining, sizeof(buffer_in));
|
||||
|
||||
if (copy_from_user(buffer_in, buf, copy_size))
|
||||
return -EFAULT;
|
||||
buf += copy_size;
|
||||
remaining -= copy_size;
|
||||
total_length += copy_size;
|
||||
#if DO_XXH
|
||||
if ((ret = xxh_update(&state, buffer_in, copy_size))) {
|
||||
printk(KERN_INFO DEVICE_NAME ": xxh failure.");
|
||||
return ret;
|
||||
}
|
||||
#endif
|
||||
#if DO_CRC
|
||||
crc = crc32(crc, buffer_in, copy_size);
|
||||
#endif
|
||||
}
|
||||
return size;
|
||||
}
|
||||
/* register the character device. */
|
||||
static int __init xxhash_test_init(void)
|
||||
{
|
||||
static const struct file_operations fileops = {
|
||||
.owner = THIS_MODULE,
|
||||
.open = &xxhash_test_open,
|
||||
.release = &xxhash_test_release,
|
||||
.write = &xxhash_test_write
|
||||
};
|
||||
|
||||
device_major = register_chrdev(0, DEVICE_NAME, &fileops);
|
||||
if (device_major < 0) {
|
||||
return device_major;
|
||||
}
|
||||
|
||||
printk(KERN_INFO DEVICE_NAME ": module loaded\n");
|
||||
printk(KERN_INFO DEVICE_NAME ": Create a device node with "
|
||||
"'mknod " DEVICE_NAME " c %d 0' and write data "
|
||||
"to it.\n", device_major);
|
||||
return 0;
|
||||
}
|
||||
|
||||
static void __exit xxhash_test_exit(void)
|
||||
{
|
||||
unregister_chrdev(device_major, DEVICE_NAME);
|
||||
printk(KERN_INFO DEVICE_NAME ": module unloaded\n");
|
||||
}
|
||||
|
||||
module_init(xxhash_test_init);
|
||||
module_exit(xxhash_test_exit);
|
||||
|
||||
MODULE_DESCRIPTION("XXHash tester");
|
||||
MODULE_VERSION("1.0");
|
||||
|
||||
|
||||
MODULE_LICENSE("Dual BSD/GPL");
|
||||
|
|
@ -1599,10 +1599,10 @@ index 0000000..a826b99
|
|||
+#endif /* BITSTREAM_H_MODULE */
|
||||
diff --git a/lib/zstd/compress.c b/lib/zstd/compress.c
|
||||
new file mode 100644
|
||||
index 0000000..42236a3
|
||||
index 0000000..1aff542
|
||||
--- /dev/null
|
||||
+++ b/lib/zstd/compress.c
|
||||
@@ -0,0 +1,3463 @@
|
||||
@@ -0,0 +1,3468 @@
|
||||
+/**
|
||||
+ * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
|
||||
+ * All rights reserved.
|
||||
|
|
@ -3583,10 +3583,15 @@ index 0000000..42236a3
|
|||
+ break; /* nothing found : store previous solution */
|
||||
+ }
|
||||
+
|
||||
+ /* NOTE:
|
||||
+ * start[-offset+ZSTD_REP_MOVE-1] is undefined behavior.
|
||||
+ * (-offset+ZSTD_REP_MOVE-1) is unsigned, and is added to start, which
|
||||
+ * overflows the pointer, which is undefined behavior.
|
||||
+ */
|
||||
+ /* catch up */
|
||||
+ if (offset) {
|
||||
+ while ((start > anchor) && (start > base + offset - ZSTD_REP_MOVE) &&
|
||||
+ (start[-1] == start[-1 - offset + ZSTD_REP_MOVE])) /* only search for offset within prefix */
|
||||
+ (start[-1] == (start-offset+ZSTD_REP_MOVE)[-1])) /* only search for offset within prefix */
|
||||
+ {
|
||||
+ start--;
|
||||
+ matchLength++;
|
||||
|
|
|
|||
|
|
@ -0,0 +1,285 @@
|
|||
/*
|
||||
* Copyright (c) 2016-present, Facebook, Inc.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under the BSD-style license found in the
|
||||
* LICENSE file in the root directory of this source tree. An additional grant
|
||||
* of patent rights can be found in the PATENTS file in the same directory.
|
||||
*
|
||||
* This program is free software; you can redistribute it and/or modify it under
|
||||
* the terms of the GNU General Public License version 2 as published by the
|
||||
* Free Software Foundation. This program is dual-licensed; you may select
|
||||
* either version 2 of the GNU General Public License ("GPL") or BSD license
|
||||
* ("BSD").
|
||||
*/
|
||||
|
||||
/* Compression level or 0 to disable */
|
||||
#define DO_ZLIB 9
|
||||
/* Compression level or 0 to disable */
|
||||
#define DO_ZSTD 0
|
||||
/* Buffer size */
|
||||
#define BUFFER_SIZE 4096
|
||||
|
||||
#include <linux/kernel.h>
|
||||
#include <linux/module.h>
|
||||
#include <linux/vmalloc.h>
|
||||
#include <linux/fs.h>
|
||||
#include <linux/uaccess.h>
|
||||
|
||||
#if DO_ZSTD
|
||||
#include <linux/zstd.h>
|
||||
#endif
|
||||
|
||||
#if DO_ZLIB
|
||||
#include <linux/zlib.h>
|
||||
#endif
|
||||
|
||||
/* Device name to pass to register_chrdev(). */
|
||||
#define DEVICE_NAME "zstd_compress_test"
|
||||
|
||||
/* Dynamically allocated device major number */
|
||||
static int device_major;
|
||||
|
||||
/*
|
||||
* We reuse the same state, and thus can compress only one file at a time.
|
||||
*/
|
||||
static bool device_is_open;
|
||||
|
||||
|
||||
static void *workspace = NULL;
|
||||
|
||||
/*
|
||||
* Input buffer used to put data coming from userspace.
|
||||
*/
|
||||
static uint8_t buffer_in[BUFFER_SIZE];
|
||||
static uint8_t buffer_out[BUFFER_SIZE];
|
||||
|
||||
static uint64_t uncompressed_len;
|
||||
static uint64_t compressed_len;
|
||||
|
||||
#if DO_ZSTD
|
||||
|
||||
static ZSTD_CStream *state;
|
||||
|
||||
static ZSTD_inBuffer input = {
|
||||
.src = buffer_in,
|
||||
.size = sizeof(buffer_in),
|
||||
.pos = sizeof(buffer_in),
|
||||
};
|
||||
|
||||
static ZSTD_outBuffer output = {
|
||||
.dst = buffer_out,
|
||||
.size = sizeof(buffer_out),
|
||||
.pos = sizeof(buffer_out),
|
||||
};
|
||||
|
||||
#endif /* DO_ZSTD */
|
||||
|
||||
#if DO_ZLIB
|
||||
|
||||
static z_stream state = {
|
||||
.next_in = buffer_in,
|
||||
.avail_in = 0,
|
||||
.total_in = 0,
|
||||
|
||||
.next_out = buffer_out,
|
||||
.avail_out = sizeof(buffer_out),
|
||||
.total_out = 0,
|
||||
|
||||
.msg = NULL,
|
||||
.state = NULL,
|
||||
.workspace = NULL,
|
||||
};
|
||||
|
||||
#endif /* DO_ZLIB */
|
||||
|
||||
static int zstd_compress_test_open(struct inode *i, struct file *f)
|
||||
{
|
||||
if (device_is_open)
|
||||
return -EBUSY;
|
||||
|
||||
device_is_open = true;
|
||||
|
||||
uncompressed_len = compressed_len = 0;
|
||||
|
||||
#if DO_ZSTD
|
||||
if (ZSTD_isError(ZSTD_resetCStream(state, 0)))
|
||||
return -EIO;
|
||||
#endif
|
||||
|
||||
#if DO_ZLIB
|
||||
if (zlib_deflateReset(&state) != Z_OK)
|
||||
return -EIO;
|
||||
#endif
|
||||
|
||||
printk(KERN_INFO DEVICE_NAME ": opened\n");
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int zstd_compress_test_release(struct inode *i, struct file *f)
|
||||
{
|
||||
device_is_open = false;
|
||||
|
||||
#if DO_ZSTD
|
||||
do {
|
||||
size_t ret;
|
||||
|
||||
output.pos = 0;
|
||||
ret = ZSTD_endStream(state, &output);
|
||||
if (ZSTD_isError(ret)) {
|
||||
printk(KERN_INFO DEVICE_NAME ": zstd end error %u\n", ZSTD_getErrorCode(ret));
|
||||
return -EIO;
|
||||
}
|
||||
compressed_len += output.pos;
|
||||
} while (output.pos != output.size);
|
||||
#endif
|
||||
|
||||
#if DO_ZLIB
|
||||
for (;;) {
|
||||
int ret;
|
||||
|
||||
state.next_out = buffer_out;
|
||||
state.avail_out = sizeof(buffer_out);
|
||||
ret = zlib_deflate(&state, Z_FINISH);
|
||||
compressed_len += sizeof(buffer_out) - state.avail_out;
|
||||
if (ret == Z_STREAM_END)
|
||||
break;
|
||||
if (ret != Z_OK) {
|
||||
printk(KERN_INFO DEVICE_NAME ": zlib end error %d: %s\n", ret, state.msg);
|
||||
return -EIO;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
printk(KERN_INFO DEVICE_NAME ": uncompressed_len = %llu\n", uncompressed_len);
|
||||
printk(KERN_INFO DEVICE_NAME ": compressed_len = %llu\n", compressed_len);
|
||||
printk(KERN_INFO DEVICE_NAME ": closed\n");
|
||||
return 0;
|
||||
}
|
||||
|
||||
/*
|
||||
* Hash the data given to us from userspace.
|
||||
*/
|
||||
static ssize_t zstd_compress_test_write(struct file *file,
|
||||
const char __user *buf, size_t size, loff_t *pos)
|
||||
{
|
||||
size_t remaining = size;
|
||||
|
||||
while (remaining > 0) {
|
||||
size_t const copy_size = min(remaining, sizeof(buffer_in));
|
||||
|
||||
if (copy_from_user(buffer_in, buf, copy_size))
|
||||
return -EFAULT;
|
||||
buf += copy_size;
|
||||
remaining -= copy_size;
|
||||
uncompressed_len += copy_size;
|
||||
|
||||
#if DO_ZSTD
|
||||
input.pos = 0;
|
||||
input.size = copy_size;
|
||||
while (input.pos != input.size) {
|
||||
size_t ret;
|
||||
|
||||
output.pos = 0;
|
||||
ret = ZSTD_compressStream(state, &output, &input);
|
||||
if (ZSTD_isError(ret)) {
|
||||
printk(KERN_INFO DEVICE_NAME ": zstd compress error %u\n", ZSTD_getErrorCode(ret));
|
||||
return -EIO;
|
||||
}
|
||||
compressed_len += output.pos;
|
||||
}
|
||||
#endif
|
||||
#if DO_ZLIB
|
||||
state.next_in = buffer_in;
|
||||
state.avail_in = copy_size;
|
||||
while (state.avail_in > 0) {
|
||||
int ret;
|
||||
|
||||
state.next_out = buffer_out;
|
||||
state.avail_out = sizeof(buffer_out);
|
||||
ret = zlib_deflate(&state, Z_NO_FLUSH);
|
||||
compressed_len += sizeof(buffer_out) - state.avail_out;
|
||||
if (ret != Z_OK) {
|
||||
printk(KERN_INFO DEVICE_NAME ": zlib end error %d: %s\n", ret, state.msg);
|
||||
return -EIO;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
}
|
||||
return size;
|
||||
}
|
||||
/* register the character device. */
|
||||
static int __init zstd_compress_test_init(void)
|
||||
{
|
||||
static const struct file_operations fileops = {
|
||||
.owner = THIS_MODULE,
|
||||
.open = &zstd_compress_test_open,
|
||||
.release = &zstd_compress_test_release,
|
||||
.write = &zstd_compress_test_write
|
||||
};
|
||||
size_t workspace_size = 0;
|
||||
#if DO_ZSTD
|
||||
ZSTD_parameters params;
|
||||
#endif
|
||||
|
||||
device_major = register_chrdev(0, DEVICE_NAME, &fileops);
|
||||
if (device_major < 0) {
|
||||
return device_major;
|
||||
}
|
||||
|
||||
#if DO_ZSTD
|
||||
params = ZSTD_getParams(DO_ZSTD, 0, 0);
|
||||
workspace_size = ZSTD_CStreamWorkspaceBound(params.cParams);
|
||||
|
||||
if (!(workspace = vmalloc(workspace_size)))
|
||||
goto fail;
|
||||
if (!(state = ZSTD_initCStream(params, 0, workspace, workspace_size)))
|
||||
goto fail;
|
||||
#endif
|
||||
|
||||
#if DO_ZLIB
|
||||
workspace_size = zlib_deflate_workspacesize(MAX_WBITS, MAX_MEM_LEVEL);
|
||||
|
||||
if (!(workspace = vmalloc(workspace_size)))
|
||||
goto fail;
|
||||
state.workspace = workspace;
|
||||
if (zlib_deflateInit(&state, DO_ZLIB) != Z_OK)
|
||||
goto fail;
|
||||
#endif
|
||||
|
||||
printk(KERN_INFO DEVICE_NAME ": module loaded\n");
|
||||
printk(KERN_INFO DEVICE_NAME ": compression requires %zu bytes of memory\n", workspace_size);
|
||||
printk(KERN_INFO DEVICE_NAME ": Create a device node with "
|
||||
"'mknod " DEVICE_NAME " c %d 0' and write data "
|
||||
"to it.\n", device_major);
|
||||
return 0;
|
||||
|
||||
fail:
|
||||
printk(KERN_INFO DEVICE_NAME ": failed to load module\n");
|
||||
if (workspace) {
|
||||
vfree(workspace);
|
||||
workspace = NULL;
|
||||
}
|
||||
return -ENOMEM;
|
||||
}
|
||||
|
||||
static void __exit zstd_compress_test_exit(void)
|
||||
{
|
||||
unregister_chrdev(device_major, DEVICE_NAME);
|
||||
#if DO_ZLIB
|
||||
zlib_deflateEnd(&state);
|
||||
#endif
|
||||
if (workspace) {
|
||||
vfree(workspace);
|
||||
workspace = NULL;
|
||||
}
|
||||
printk(KERN_INFO DEVICE_NAME ": module unloaded\n");
|
||||
}
|
||||
|
||||
module_init(zstd_compress_test_init);
|
||||
module_exit(zstd_compress_test_exit);
|
||||
|
||||
MODULE_DESCRIPTION("Zstd compression tester");
|
||||
MODULE_VERSION("1.0");
|
||||
|
||||
MODULE_LICENSE("Dual BSD/GPL");
|
||||
|
|
@ -0,0 +1,256 @@
|
|||
/*
|
||||
* Copyright (c) 2016-present, Facebook, Inc.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under the BSD-style license found in the
|
||||
* LICENSE file in the root directory of this source tree. An additional grant
|
||||
* of patent rights can be found in the PATENTS file in the same directory.
|
||||
*
|
||||
* This program is free software; you can redistribute it and/or modify it under
|
||||
* the terms of the GNU General Public License version 2 as published by the
|
||||
* Free Software Foundation. This program is dual-licensed; you may select
|
||||
* either version 2 of the GNU General Public License ("GPL") or BSD license
|
||||
* ("BSD").
|
||||
*/
|
||||
|
||||
/* Compression level or 0 to disable */
|
||||
#define DO_ZLIB 1
|
||||
/* Compression level or 0 to disable */
|
||||
#define DO_ZSTD 0
|
||||
/* Buffer size */
|
||||
#define BUFFER_SIZE 4096
|
||||
|
||||
#include <linux/kernel.h>
|
||||
#include <linux/module.h>
|
||||
#include <linux/vmalloc.h>
|
||||
#include <linux/fs.h>
|
||||
#include <linux/uaccess.h>
|
||||
|
||||
#if DO_ZSTD
|
||||
#include <linux/zstd.h>
|
||||
#endif
|
||||
|
||||
#if DO_ZLIB
|
||||
#include <linux/zlib.h>
|
||||
#endif
|
||||
|
||||
/* Device name to pass to register_chrdev(). */
|
||||
#define DEVICE_NAME "zstd_decompress_test"
|
||||
|
||||
/* Dynamically allocated device major number */
|
||||
static int device_major;
|
||||
|
||||
/*
|
||||
* We reuse the same state, and thus can compress only one file at a time.
|
||||
*/
|
||||
static bool device_is_open;
|
||||
|
||||
|
||||
static void *workspace = NULL;
|
||||
|
||||
/*
|
||||
* Input buffer used to put data coming from userspace.
|
||||
*/
|
||||
static uint8_t buffer_in[BUFFER_SIZE];
|
||||
static uint8_t buffer_out[BUFFER_SIZE];
|
||||
|
||||
static uint64_t uncompressed_len;
|
||||
static uint64_t compressed_len;
|
||||
|
||||
#if DO_ZSTD
|
||||
|
||||
static ZSTD_DStream *state;
|
||||
|
||||
static ZSTD_inBuffer input = {
|
||||
.src = buffer_in,
|
||||
.size = sizeof(buffer_in),
|
||||
.pos = sizeof(buffer_in),
|
||||
};
|
||||
|
||||
static ZSTD_outBuffer output = {
|
||||
.dst = buffer_out,
|
||||
.size = sizeof(buffer_out),
|
||||
.pos = sizeof(buffer_out),
|
||||
};
|
||||
|
||||
#endif /* DO_ZSTD */
|
||||
|
||||
#if DO_ZLIB
|
||||
|
||||
static z_stream state = {
|
||||
.next_in = buffer_in,
|
||||
.avail_in = 0,
|
||||
.total_in = 0,
|
||||
|
||||
.next_out = buffer_out,
|
||||
.avail_out = sizeof(buffer_out),
|
||||
.total_out = 0,
|
||||
|
||||
.msg = NULL,
|
||||
.state = NULL,
|
||||
.workspace = NULL,
|
||||
};
|
||||
|
||||
#endif /* DO_ZLIB */
|
||||
|
||||
static int zstd_decompress_test_open(struct inode *i, struct file *f)
|
||||
{
|
||||
if (device_is_open)
|
||||
return -EBUSY;
|
||||
|
||||
device_is_open = true;
|
||||
|
||||
uncompressed_len = compressed_len = 0;
|
||||
|
||||
#if DO_ZSTD
|
||||
if (ZSTD_isError(ZSTD_resetDStream(state)))
|
||||
return -EIO;
|
||||
#endif
|
||||
|
||||
#if DO_ZLIB
|
||||
if (zlib_inflateReset(&state) != Z_OK)
|
||||
return -EIO;
|
||||
#endif
|
||||
|
||||
printk(KERN_INFO DEVICE_NAME ": opened\n");
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int zstd_decompress_test_release(struct inode *i, struct file *f)
|
||||
{
|
||||
device_is_open = false;
|
||||
|
||||
printk(KERN_INFO DEVICE_NAME ": uncompressed_len = %llu\n", uncompressed_len);
|
||||
printk(KERN_INFO DEVICE_NAME ": compressed_len = %llu\n", compressed_len);
|
||||
printk(KERN_INFO DEVICE_NAME ": closed\n");
|
||||
return 0;
|
||||
}
|
||||
|
||||
/*
|
||||
* Hash the data given to us from userspace.
|
||||
*/
|
||||
static ssize_t zstd_decompress_test_write(struct file *file,
|
||||
const char __user *buf, size_t size, loff_t *pos)
|
||||
{
|
||||
size_t remaining = size;
|
||||
|
||||
while (remaining > 0) {
|
||||
size_t const copy_size = min(remaining, sizeof(buffer_in));
|
||||
|
||||
if (copy_from_user(buffer_in, buf, copy_size))
|
||||
return -EFAULT;
|
||||
buf += copy_size;
|
||||
remaining -= copy_size;
|
||||
compressed_len += copy_size;
|
||||
|
||||
#if DO_ZSTD
|
||||
input.pos = 0;
|
||||
input.size = copy_size;
|
||||
while (input.pos != input.size) {
|
||||
size_t ret;
|
||||
|
||||
output.pos = 0;
|
||||
ret = ZSTD_decompressStream(state, &output, &input);
|
||||
if (ZSTD_isError(ret)) {
|
||||
printk(KERN_INFO DEVICE_NAME ": zstd decompress error %u\n", ZSTD_getErrorCode(ret));
|
||||
return -EIO;
|
||||
}
|
||||
uncompressed_len += output.pos;
|
||||
}
|
||||
#endif
|
||||
#if DO_ZLIB
|
||||
state.next_in = buffer_in;
|
||||
state.avail_in = copy_size;
|
||||
while (state.avail_in > 0) {
|
||||
int ret;
|
||||
|
||||
state.next_out = buffer_out;
|
||||
state.avail_out = sizeof(buffer_out);
|
||||
ret = zlib_inflate(&state, Z_NO_FLUSH);
|
||||
uncompressed_len += sizeof(buffer_out) - state.avail_out;
|
||||
if (ret != Z_OK && ret != Z_STREAM_END) {
|
||||
printk(KERN_INFO DEVICE_NAME ": zlib decompress error %d: %s\n", ret, state.msg);
|
||||
return -EIO;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
}
|
||||
return size;
|
||||
}
|
||||
/* register the character device. */
|
||||
static int __init zstd_decompress_test_init(void)
|
||||
{
|
||||
static const struct file_operations fileops = {
|
||||
.owner = THIS_MODULE,
|
||||
.open = &zstd_decompress_test_open,
|
||||
.release = &zstd_decompress_test_release,
|
||||
.write = &zstd_decompress_test_write
|
||||
};
|
||||
size_t workspace_size = 0;
|
||||
#if DO_ZSTD
|
||||
ZSTD_parameters params;
|
||||
size_t max_window_size;
|
||||
#endif
|
||||
|
||||
device_major = register_chrdev(0, DEVICE_NAME, &fileops);
|
||||
if (device_major < 0) {
|
||||
return device_major;
|
||||
}
|
||||
|
||||
#if DO_ZSTD
|
||||
params = ZSTD_getParams(DO_ZSTD, 0, 0);
|
||||
max_window_size = (size_t)1 << params.cParams.windowLog;
|
||||
workspace_size = ZSTD_DStreamWorkspaceBound(max_window_size);
|
||||
|
||||
if (!(workspace = vmalloc(workspace_size)))
|
||||
goto fail;
|
||||
if (!(state = ZSTD_initDStream(max_window_size, workspace, workspace_size)))
|
||||
goto fail;
|
||||
#endif
|
||||
|
||||
#if DO_ZLIB
|
||||
workspace_size = zlib_inflate_workspacesize();
|
||||
|
||||
if (!(workspace = vmalloc(workspace_size)))
|
||||
goto fail;
|
||||
state.workspace = workspace;
|
||||
if (zlib_inflateInit(&state) != Z_OK)
|
||||
goto fail;
|
||||
#endif
|
||||
|
||||
printk(KERN_INFO DEVICE_NAME ": module loaded\n");
|
||||
printk(KERN_INFO DEVICE_NAME ": decompression requires %zu bytes of memory\n", workspace_size);
|
||||
printk(KERN_INFO DEVICE_NAME ": Create a device node with "
|
||||
"'mknod " DEVICE_NAME " c %d 0' and write data "
|
||||
"to it.\n", device_major);
|
||||
return 0;
|
||||
|
||||
fail:
|
||||
printk(KERN_INFO DEVICE_NAME ": failed to load module\n");
|
||||
if (workspace) {
|
||||
vfree(workspace);
|
||||
workspace = NULL;
|
||||
}
|
||||
return -ENOMEM;
|
||||
}
|
||||
|
||||
static void __exit zstd_decompress_test_exit(void)
|
||||
{
|
||||
unregister_chrdev(device_major, DEVICE_NAME);
|
||||
#if DO_ZLIB
|
||||
zlib_deflateEnd(&state);
|
||||
#endif
|
||||
if (workspace) {
|
||||
vfree(workspace);
|
||||
workspace = NULL;
|
||||
}
|
||||
printk(KERN_INFO DEVICE_NAME ": module unloaded\n");
|
||||
}
|
||||
|
||||
module_init(zstd_decompress_test_init);
|
||||
module_exit(zstd_decompress_test_exit);
|
||||
|
||||
MODULE_DESCRIPTION("Zstd decompression tester");
|
||||
MODULE_VERSION("1.0");
|
||||
|
||||
MODULE_LICENSE("Dual BSD/GPL");
|
||||
Loading…
Reference in New Issue