diff --git a/contrib/linux-kernel/0000-cover-letter.patch b/contrib/linux-kernel/0000-cover-letter.patch index 763b5a9c..33a5189b 100644 --- a/contrib/linux-kernel/0000-cover-letter.patch +++ b/contrib/linux-kernel/0000-cover-letter.patch @@ -1,7 +1,7 @@ -From 8bc9a0ae5c86a6d02d9a5274b9965ddac0e8d330 Mon Sep 17 00:00:00 2001 +From 0cd63464d182bb9708f8b25f7da3dc8e5ec6b4fa Mon Sep 17 00:00:00 2001 From: Nick Terrell -Date: Wed, 28 Jun 2017 22:00:00 -0700 -Subject: [PATCH v2 0/4] Add xxhash and zstd modules +Date: Thu, 20 Jul 2017 13:18:30 -0700 +Subject: [PATCH v3 0/4] Add xxhash and zstd modules Hi all, @@ -24,6 +24,13 @@ v1 -> v2: HUF_compressWeights(), HUF_readDTableX2(), and HUF_readDTableX4() (2/4) - No zstd function uses more than 400 B of stack space (2/4) +v2 -> v3: +- Work around gcc-7 bug https://gcc.gnu.org/bugzilla/show_bug.cgi?id=81388 + (2/4) +- Fix bug in dictionary compression from upstream commit cc1522351f (2/4) +- Port upstream BtrFS commits e1ddce71d6, 389a6cfc2a, and 6acafd1eff (3/4) +- Change default compression level for BtrFS to 3 (3/4) + Nick Terrell (4): lib: Add xxhash module lib: Add zstd modules @@ -40,7 +47,7 @@ Nick Terrell (4): fs/btrfs/props.c | 6 + fs/btrfs/super.c | 12 +- fs/btrfs/sysfs.c | 2 + - fs/btrfs/zstd.c | 433 ++++++ + fs/btrfs/zstd.c | 435 ++++++ fs/squashfs/Kconfig | 14 + fs/squashfs/Makefile | 1 + fs/squashfs/decompressor.c | 7 + @@ -63,13 +70,13 @@ Nick Terrell (4): lib/zstd/fse_compress.c | 795 ++++++++++ lib/zstd/fse_decompress.c | 332 +++++ lib/zstd/huf.h | 212 +++ - lib/zstd/huf_compress.c | 771 ++++++++++ + lib/zstd/huf_compress.c | 770 ++++++++++ lib/zstd/huf_decompress.c | 960 ++++++++++++ lib/zstd/mem.h | 151 ++ lib/zstd/zstd_common.c | 75 + - lib/zstd/zstd_internal.h | 269 ++++ + lib/zstd/zstd_internal.h | 250 ++++ lib/zstd/zstd_opt.h | 1014 +++++++++++++ - 39 files changed, 14400 insertions(+), 12 deletions(-) + 39 files changed, 14382 insertions(+), 12 deletions(-) create mode 100644 fs/btrfs/zstd.c create mode 100644 fs/squashfs/zstd_wrapper.c create mode 100644 include/linux/xxhash.h diff --git a/contrib/linux-kernel/0001-lib-Add-xxhash-module.patch b/contrib/linux-kernel/0001-lib-Add-xxhash-module.patch index 84a2c53c..f86731ce 100644 --- a/contrib/linux-kernel/0001-lib-Add-xxhash-module.patch +++ b/contrib/linux-kernel/0001-lib-Add-xxhash-module.patch @@ -1,7 +1,7 @@ -From 5ac909c415ab4a18fd90794793c96e450795e8c6 Mon Sep 17 00:00:00 2001 +From fc7f26acbabda35f1c61dfc357dbb207dc8ed23d Mon Sep 17 00:00:00 2001 From: Nick Terrell -Date: Wed, 21 Jun 2017 17:37:36 -0700 -Subject: [PATCH v2 1/4] lib: Add xxhash module +Date: Mon, 17 Jul 2017 17:07:18 -0700 +Subject: [PATCH v3 1/4] lib: Add xxhash module Adds xxhash kernel module with xxh32 and xxh64 hashes. xxhash is an extremely fast non-cryptographic hash algorithm for checksumming. @@ -327,10 +327,10 @@ index 0000000..9e1f42c + +#endif /* XXHASH_H */ diff --git a/lib/Kconfig b/lib/Kconfig -index 0c8b78a..b6009d7 100644 +index 6762529..5e7541f 100644 --- a/lib/Kconfig +++ b/lib/Kconfig -@@ -184,6 +184,9 @@ config CRC8 +@@ -192,6 +192,9 @@ config CRC8 when they need to do cyclic redundancy check according CRC8 algorithm. Module will be called crc8. @@ -341,10 +341,10 @@ index 0c8b78a..b6009d7 100644 bool depends on AUDIT && !AUDIT_ARCH diff --git a/lib/Makefile b/lib/Makefile -index 0166fbc..1338226 100644 +index 40c1837..d06b68a 100644 --- a/lib/Makefile +++ b/lib/Makefile -@@ -102,6 +102,7 @@ obj-$(CONFIG_CRC32_SELFTEST) += crc32test.o +@@ -102,6 +102,7 @@ obj-$(CONFIG_CRC4) += crc4.o obj-$(CONFIG_CRC7) += crc7.o obj-$(CONFIG_LIBCRC32C) += libcrc32c.o obj-$(CONFIG_CRC8) += crc8.o diff --git a/contrib/linux-kernel/0002-lib-Add-zstd-modules.patch b/contrib/linux-kernel/0002-lib-Add-zstd-modules.patch index 97109399..268307cf 100644 --- a/contrib/linux-kernel/0002-lib-Add-zstd-modules.patch +++ b/contrib/linux-kernel/0002-lib-Add-zstd-modules.patch @@ -1,7 +1,7 @@ -From d2626127c6d6e60e940dd9a3ed58323bdcdc4930 Mon Sep 17 00:00:00 2001 +From 686a6149b98250d66b5951e3ae05e79063e9de98 Mon Sep 17 00:00:00 2001 From: Nick Terrell -Date: Tue, 16 May 2017 14:55:36 -0700 -Subject: [PATCH v2 2/4] lib: Add zstd modules +Date: Mon, 17 Jul 2017 17:08:19 -0700 +Subject: [PATCH v3 2/4] lib: Add zstd modules Add zstd compression and decompression kernel modules. zstd offers a wide varity of compression speed and quality trade-offs. @@ -110,6 +110,10 @@ v1 -> v2: HUF_compressWeights(), HUF_readDTableX2(), and HUF_readDTableX4() - No function uses more than 400 B of stack space +v2 -> v3: +- Work around gcc-7 bug https://gcc.gnu.org/bugzilla/show_bug.cgi?id=81388 +- Fix bug in dictionary compression from upstream commit cc1522351f + include/linux/zstd.h | 1157 +++++++++++++++ lib/Kconfig | 8 + lib/Makefile | 2 + @@ -123,13 +127,13 @@ v1 -> v2: lib/zstd/fse_compress.c | 795 +++++++++++ lib/zstd/fse_decompress.c | 332 +++++ lib/zstd/huf.h | 212 +++ - lib/zstd/huf_compress.c | 771 ++++++++++ + lib/zstd/huf_compress.c | 770 ++++++++++ lib/zstd/huf_decompress.c | 960 +++++++++++++ lib/zstd/mem.h | 151 ++ lib/zstd/zstd_common.c | 75 + - lib/zstd/zstd_internal.h | 269 ++++ + lib/zstd/zstd_internal.h | 250 ++++ lib/zstd/zstd_opt.h | 1014 +++++++++++++ - 19 files changed, 13014 insertions(+) + 19 files changed, 12994 insertions(+) create mode 100644 include/linux/zstd.h create mode 100644 lib/zstd/Makefile create mode 100644 lib/zstd/bitstream.h @@ -1312,10 +1316,10 @@ index 0000000..249575e + +#endif /* ZSTD_H */ diff --git a/lib/Kconfig b/lib/Kconfig -index b6009d7..f00ddab 100644 +index 5e7541f..0d49ed0 100644 --- a/lib/Kconfig +++ b/lib/Kconfig -@@ -241,6 +241,14 @@ config LZ4HC_COMPRESS +@@ -249,6 +249,14 @@ config LZ4HC_COMPRESS config LZ4_DECOMPRESS tristate @@ -1331,7 +1335,7 @@ index b6009d7..f00ddab 100644 # diff --git a/lib/Makefile b/lib/Makefile -index 1338226..4fcef16 100644 +index d06b68a..d5c8a4f 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -116,6 +116,8 @@ obj-$(CONFIG_LZO_DECOMPRESS) += lzo/ @@ -10012,10 +10016,10 @@ index 0000000..2143da2 +#endif /* HUF_H_298734234 */ diff --git a/lib/zstd/huf_compress.c b/lib/zstd/huf_compress.c new file mode 100644 -index 0000000..0361f38 +index 0000000..40055a7 --- /dev/null +++ b/lib/zstd/huf_compress.c -@@ -0,0 +1,771 @@ +@@ -0,0 +1,770 @@ +/* + * Huffman encoder, part of New Generation Entropy library + * Copyright (C) 2013-2016, Yann Collet. @@ -10543,7 +10547,7 @@ index 0000000..0361f38 + +size_t HUF_compressBound(size_t size) { return HUF_COMPRESSBOUND(size); } + -+#define HUF_FLUSHBITS(s) (fast ? BIT_flushBitsFast(s) : BIT_flushBits(s)) ++#define HUF_FLUSHBITS(s) BIT_flushBits(s) + +#define HUF_FLUSHBITS_1(stream) \ + if (sizeof((stream)->bitContainer) * 8 < HUF_TABLELOG_MAX * 2 + 7) \ @@ -10560,7 +10564,6 @@ index 0000000..0361f38 + BYTE *const oend = ostart + dstSize; + BYTE *op = ostart; + size_t n; -+ const unsigned fast = (dstSize >= HUF_BLOCKBOUND(srcSize)); + BIT_CStream_t bitC; + + /* init */ @@ -11993,10 +11996,10 @@ index 0000000..a282624 +} diff --git a/lib/zstd/zstd_internal.h b/lib/zstd/zstd_internal.h new file mode 100644 -index 0000000..6748719 +index 0000000..f0ba474 --- /dev/null +++ b/lib/zstd/zstd_internal.h -@@ -0,0 +1,269 @@ +@@ -0,0 +1,250 @@ +/** + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. @@ -12125,35 +12128,16 @@ index 0000000..6748719 +/*-******************************************* +* Shared functions to include for inlining +*********************************************/ -+static void ZSTD_copy8(void *dst, const void *src) { memcpy(dst, src, 8); } -+#define COPY8(d, s) \ -+ { \ -+ ZSTD_copy8(d, s); \ -+ d += 8; \ -+ s += 8; \ -+ } -+ ++static void ZSTD_copy8(void *dst, const void *src) { ++ memcpy(dst, src, 8); ++} +/*! ZSTD_wildcopy() : +* custom version of memcpy(), can copy up to 7 bytes too many (8 bytes if length==0) */ +#define WILDCOPY_OVERLENGTH 8 +ZSTD_STATIC void ZSTD_wildcopy(void *dst, const void *src, ptrdiff_t length) +{ -+ const BYTE *ip = (const BYTE *)src; -+ BYTE *op = (BYTE *)dst; -+ BYTE *const oend = op + length; -+ do -+ COPY8(op, ip) -+ while (op < oend); -+} -+ -+ZSTD_STATIC void ZSTD_wildcopy_e(void *dst, const void *src, void *dstEnd) /* should be faster for decoding, but strangely, not verified on all platform */ -+{ -+ const BYTE *ip = (const BYTE *)src; -+ BYTE *op = (BYTE *)dst; -+ BYTE *const oend = (BYTE *)dstEnd; -+ do -+ COPY8(op, ip) -+ while (op < oend); ++ if (length > 0) ++ memcpy(dst, src, length); +} + +/*-******************************************* diff --git a/contrib/linux-kernel/0003-btrfs-Add-zstd-support.patch b/contrib/linux-kernel/0003-btrfs-Add-zstd-support.patch index abc8326c..5578fa38 100644 --- a/contrib/linux-kernel/0003-btrfs-Add-zstd-support.patch +++ b/contrib/linux-kernel/0003-btrfs-Add-zstd-support.patch @@ -1,7 +1,7 @@ -From 599f8f2aaace3df939cb145368574a52268d82d0 Mon Sep 17 00:00:00 2001 +From b0ef8fc63c9ca251ceca632f53aa1de8f1f17772 Mon Sep 17 00:00:00 2001 From: Nick Terrell -Date: Wed, 21 Jun 2017 17:31:39 -0700 -Subject: [PATCH v2 3/4] btrfs: Add zstd support +Date: Mon, 17 Jul 2017 17:08:39 -0700 +Subject: [PATCH v3 3/4] btrfs: Add zstd support Add zstd compression and decompression support to BtrFS. zstd at its fastest level compresses almost as well as zlib, while offering much @@ -63,6 +63,10 @@ zstd source repository: https://github.com/facebook/zstd Signed-off-by: Nick Terrell --- +v2 -> v3: +- Port upstream BtrFS commits e1ddce71d6, 389a6cfc2a, and 6acafd1eff +- Change default compression level for BtrFS to 3 + fs/btrfs/Kconfig | 2 + fs/btrfs/Makefile | 2 +- fs/btrfs/compression.c | 1 + @@ -73,9 +77,9 @@ Signed-off-by: Nick Terrell fs/btrfs/props.c | 6 + fs/btrfs/super.c | 12 +- fs/btrfs/sysfs.c | 2 + - fs/btrfs/zstd.c | 433 +++++++++++++++++++++++++++++++++++++++++++++ + fs/btrfs/zstd.c | 435 +++++++++++++++++++++++++++++++++++++++++++++ include/uapi/linux/btrfs.h | 8 +- - 12 files changed, 469 insertions(+), 12 deletions(-) + 12 files changed, 471 insertions(+), 12 deletions(-) create mode 100644 fs/btrfs/zstd.c diff --git a/fs/btrfs/Kconfig b/fs/btrfs/Kconfig @@ -105,10 +109,10 @@ index 128ce17..962a95a 100644 reada.o backref.o ulist.o qgroup.o send.o dev-replace.o raid56.o \ uuid-tree.o props.o hash.o free-space-tree.o diff --git a/fs/btrfs/compression.c b/fs/btrfs/compression.c -index 10e6b28..3beb0d0 100644 +index d2ef9ac..4ff42d1 100644 --- a/fs/btrfs/compression.c +++ b/fs/btrfs/compression.c -@@ -761,6 +761,7 @@ static struct { +@@ -704,6 +704,7 @@ static struct { static const struct btrfs_compress_op * const btrfs_compress_op[] = { &btrfs_zlib_compress, &btrfs_lzo_compress, @@ -117,10 +121,10 @@ index 10e6b28..3beb0d0 100644 void __init btrfs_init_compress(void) diff --git a/fs/btrfs/compression.h b/fs/btrfs/compression.h -index 39ec43a..d99fc21 100644 +index 87f6d33..2269e00 100644 --- a/fs/btrfs/compression.h +++ b/fs/btrfs/compression.h -@@ -60,8 +60,9 @@ enum btrfs_compression_type { +@@ -99,8 +99,9 @@ enum btrfs_compression_type { BTRFS_COMPRESS_NONE = 0, BTRFS_COMPRESS_ZLIB = 1, BTRFS_COMPRESS_LZO = 2, @@ -132,7 +136,7 @@ index 39ec43a..d99fc21 100644 }; struct btrfs_compress_op { -@@ -92,5 +93,6 @@ struct btrfs_compress_op { +@@ -128,5 +129,6 @@ struct btrfs_compress_op { extern const struct btrfs_compress_op btrfs_zlib_compress; extern const struct btrfs_compress_op btrfs_lzo_compress; @@ -140,10 +144,10 @@ index 39ec43a..d99fc21 100644 #endif diff --git a/fs/btrfs/ctree.h b/fs/btrfs/ctree.h -index 4f8f75d..61dd3dd 100644 +index 3f3eb7b..845d77c 100644 --- a/fs/btrfs/ctree.h +++ b/fs/btrfs/ctree.h -@@ -271,6 +271,7 @@ struct btrfs_super_block { +@@ -270,6 +270,7 @@ struct btrfs_super_block { BTRFS_FEATURE_INCOMPAT_MIXED_GROUPS | \ BTRFS_FEATURE_INCOMPAT_BIG_METADATA | \ BTRFS_FEATURE_INCOMPAT_COMPRESS_LZO | \ @@ -152,10 +156,10 @@ index 4f8f75d..61dd3dd 100644 BTRFS_FEATURE_INCOMPAT_EXTENDED_IREF | \ BTRFS_FEATURE_INCOMPAT_SKINNY_METADATA | \ diff --git a/fs/btrfs/disk-io.c b/fs/btrfs/disk-io.c -index 5f678dc..49c0e91 100644 +index 080e2eb..04632f4 100644 --- a/fs/btrfs/disk-io.c +++ b/fs/btrfs/disk-io.c -@@ -2831,6 +2831,8 @@ int open_ctree(struct super_block *sb, +@@ -2828,6 +2828,8 @@ int open_ctree(struct super_block *sb, features |= BTRFS_FEATURE_INCOMPAT_MIXED_BACKREF; if (fs_info->compress_type == BTRFS_COMPRESS_LZO) features |= BTRFS_FEATURE_INCOMPAT_COMPRESS_LZO; @@ -165,7 +169,7 @@ index 5f678dc..49c0e91 100644 if (features & BTRFS_FEATURE_INCOMPAT_SKINNY_METADATA) btrfs_info(fs_info, "has skinny extents"); diff --git a/fs/btrfs/ioctl.c b/fs/btrfs/ioctl.c -index e176375..f732cfd 100644 +index fa1b78c..b9963d9 100644 --- a/fs/btrfs/ioctl.c +++ b/fs/btrfs/ioctl.c @@ -327,8 +327,10 @@ static int btrfs_ioctl_setflags(struct file *file, void __user *arg) @@ -180,7 +184,7 @@ index e176375..f732cfd 100644 ret = btrfs_set_prop(inode, "btrfs.compression", comp, strlen(comp), 0); if (ret) -@@ -1463,6 +1465,8 @@ int btrfs_defrag_file(struct inode *inode, struct file *file, +@@ -1466,6 +1468,8 @@ int btrfs_defrag_file(struct inode *inode, struct file *file, if (range->compress_type == BTRFS_COMPRESS_LZO) { btrfs_set_fs_incompat(fs_info, COMPRESS_LZO); @@ -190,10 +194,10 @@ index e176375..f732cfd 100644 ret = defrag_count; diff --git a/fs/btrfs/props.c b/fs/btrfs/props.c -index d6cb155..162105f 100644 +index 4b23ae5..20631e9 100644 --- a/fs/btrfs/props.c +++ b/fs/btrfs/props.c -@@ -383,6 +383,8 @@ static int prop_compression_validate(const char *value, size_t len) +@@ -390,6 +390,8 @@ static int prop_compression_validate(const char *value, size_t len) return 0; else if (!strncmp("zlib", value, len)) return 0; @@ -202,7 +206,7 @@ index d6cb155..162105f 100644 return -EINVAL; } -@@ -405,6 +407,8 @@ static int prop_compression_apply(struct inode *inode, +@@ -412,6 +414,8 @@ static int prop_compression_apply(struct inode *inode, type = BTRFS_COMPRESS_LZO; else if (!strncmp("zlib", value, len)) type = BTRFS_COMPRESS_ZLIB; @@ -211,7 +215,7 @@ index d6cb155..162105f 100644 else return -EINVAL; -@@ -422,6 +426,8 @@ static const char *prop_compression_extract(struct inode *inode) +@@ -429,6 +433,8 @@ static const char *prop_compression_extract(struct inode *inode) return "zlib"; case BTRFS_COMPRESS_LZO: return "lzo"; @@ -221,7 +225,7 @@ index d6cb155..162105f 100644 return NULL; diff --git a/fs/btrfs/super.c b/fs/btrfs/super.c -index 4f1cdd5..4f792d5 100644 +index 12540b6..c370dea 100644 --- a/fs/btrfs/super.c +++ b/fs/btrfs/super.c @@ -513,6 +513,14 @@ int btrfs_parse_options(struct btrfs_fs_info *info, char *options, @@ -239,7 +243,7 @@ index 4f1cdd5..4f792d5 100644 } else if (strncmp(args[0].from, "no", 2) == 0) { compress_type = "no"; btrfs_clear_opt(info->mount_opt, COMPRESS); -@@ -1240,8 +1248,10 @@ static int btrfs_show_options(struct seq_file *seq, struct dentry *dentry) +@@ -1227,8 +1235,10 @@ static int btrfs_show_options(struct seq_file *seq, struct dentry *dentry) if (btrfs_test_opt(info, COMPRESS)) { if (info->compress_type == BTRFS_COMPRESS_ZLIB) compress_type = "zlib"; @@ -252,7 +256,7 @@ index 4f1cdd5..4f792d5 100644 seq_printf(seq, ",compress-force=%s", compress_type); else diff --git a/fs/btrfs/sysfs.c b/fs/btrfs/sysfs.c -index 1f157fb..b0dec90 100644 +index c2d5f35..2b6d37c 100644 --- a/fs/btrfs/sysfs.c +++ b/fs/btrfs/sysfs.c @@ -200,6 +200,7 @@ BTRFS_FEAT_ATTR_INCOMPAT(mixed_backref, MIXED_BACKREF); @@ -273,10 +277,10 @@ index 1f157fb..b0dec90 100644 BTRFS_FEAT_ATTR_PTR(raid56), diff --git a/fs/btrfs/zstd.c b/fs/btrfs/zstd.c new file mode 100644 -index 0000000..838741b +index 0000000..1822068 --- /dev/null +++ b/fs/btrfs/zstd.c -@@ -0,0 +1,433 @@ +@@ -0,0 +1,435 @@ +/* + * Copyright (c) 2016-present, Facebook, Inc. + * All rights reserved. @@ -308,10 +312,11 @@ index 0000000..838741b + +#define ZSTD_BTRFS_MAX_WINDOWLOG 17 +#define ZSTD_BTRFS_MAX_INPUT (1 << ZSTD_BTRFS_MAX_WINDOWLOG) ++#define ZSTD_BTRFS_DEFAULT_LEVEL 3 + +static ZSTD_parameters zstd_get_btrfs_parameters(size_t src_len) +{ -+ ZSTD_parameters params = ZSTD_getParams(1, src_len, 0); ++ ZSTD_parameters params = ZSTD_getParams(ZSTD_BTRFS_DEFAULT_LEVEL, src_len, 0); + + if (params.cParams.windowLog > ZSTD_BTRFS_MAX_WINDOWLOG) + params.cParams.windowLog = ZSTD_BTRFS_MAX_WINDOWLOG; @@ -330,7 +335,7 @@ index 0000000..838741b +{ + struct workspace *workspace = list_entry(ws, struct workspace, list); + -+ vfree(workspace->mem); ++ kvfree(workspace->mem); + kfree(workspace->buf); + kfree(workspace); +} @@ -341,15 +346,15 @@ index 0000000..838741b + zstd_get_btrfs_parameters(ZSTD_BTRFS_MAX_INPUT); + struct workspace *workspace; + -+ workspace = kzalloc(sizeof(*workspace), GFP_NOFS); ++ workspace = kzalloc(sizeof(*workspace), GFP_KERNEL); + if (!workspace) + return ERR_PTR(-ENOMEM); + + workspace->size = max_t(size_t, + ZSTD_CStreamWorkspaceBound(params.cParams), + ZSTD_DStreamWorkspaceBound(ZSTD_BTRFS_MAX_INPUT)); -+ workspace->mem = vmalloc(workspace->size); -+ workspace->buf = kmalloc(PAGE_SIZE, GFP_NOFS); ++ workspace->mem = kvmalloc(workspace->size, GFP_KERNEL); ++ workspace->buf = kmalloc(PAGE_SIZE, GFP_KERNEL); + if (!workspace->mem || !workspace->buf) + goto fail; + @@ -541,12 +546,13 @@ index 0000000..838741b + return ret; +} + -+static int zstd_decompress_bio(struct list_head *ws, struct page **pages_in, -+ u64 disk_start, -+ struct bio *orig_bio, -+ size_t srclen) ++static int zstd_decompress_bio(struct list_head *ws, struct compressed_bio *cb) +{ + struct workspace *workspace = list_entry(ws, struct workspace, list); ++ struct page **pages_in = cb->compressed_pages; ++ u64 disk_start = cb->start; ++ struct bio *orig_bio = cb->orig_bio; ++ size_t srclen = cb->compressed_len; + ZSTD_DStream *stream; + int ret = 0; + unsigned long page_in_index = 0; @@ -711,7 +717,7 @@ index 0000000..838741b + .decompress = zstd_decompress, +}; diff --git a/include/uapi/linux/btrfs.h b/include/uapi/linux/btrfs.h -index a456e53..992c150 100644 +index 9aa74f3..378230c 100644 --- a/include/uapi/linux/btrfs.h +++ b/include/uapi/linux/btrfs.h @@ -255,13 +255,7 @@ struct btrfs_ioctl_fs_info_args { diff --git a/contrib/linux-kernel/0004-squashfs-Add-zstd-support.patch b/contrib/linux-kernel/0004-squashfs-Add-zstd-support.patch index b638194f..02bd1073 100644 --- a/contrib/linux-kernel/0004-squashfs-Add-zstd-support.patch +++ b/contrib/linux-kernel/0004-squashfs-Add-zstd-support.patch @@ -1,7 +1,7 @@ -From 5ff6a64abaea7b7f11d37cb0fdf08642316a3a90 Mon Sep 17 00:00:00 2001 +From 0cd63464d182bb9708f8b25f7da3dc8e5ec6b4fa Mon Sep 17 00:00:00 2001 From: Nick Terrell -Date: Mon, 12 Jun 2017 12:18:23 -0700 -Subject: [PATCH v2 4/4] squashfs: Add zstd support +Date: Mon, 17 Jul 2017 17:08:59 -0700 +Subject: [PATCH v3 4/4] squashfs: Add zstd support Add zstd compression and decompression support to SquashFS. zstd is a great fit for SquashFS because it can compress at ratios approaching xz, diff --git a/contrib/linux-kernel/fs/btrfs/zstd.c b/contrib/linux-kernel/fs/btrfs/zstd.c index 838741b3..18220687 100644 --- a/contrib/linux-kernel/fs/btrfs/zstd.c +++ b/contrib/linux-kernel/fs/btrfs/zstd.c @@ -29,10 +29,11 @@ #define ZSTD_BTRFS_MAX_WINDOWLOG 17 #define ZSTD_BTRFS_MAX_INPUT (1 << ZSTD_BTRFS_MAX_WINDOWLOG) +#define ZSTD_BTRFS_DEFAULT_LEVEL 3 static ZSTD_parameters zstd_get_btrfs_parameters(size_t src_len) { - ZSTD_parameters params = ZSTD_getParams(1, src_len, 0); + ZSTD_parameters params = ZSTD_getParams(ZSTD_BTRFS_DEFAULT_LEVEL, src_len, 0); if (params.cParams.windowLog > ZSTD_BTRFS_MAX_WINDOWLOG) params.cParams.windowLog = ZSTD_BTRFS_MAX_WINDOWLOG; @@ -51,7 +52,7 @@ static void zstd_free_workspace(struct list_head *ws) { struct workspace *workspace = list_entry(ws, struct workspace, list); - vfree(workspace->mem); + kvfree(workspace->mem); kfree(workspace->buf); kfree(workspace); } @@ -62,15 +63,15 @@ static struct list_head *zstd_alloc_workspace(void) zstd_get_btrfs_parameters(ZSTD_BTRFS_MAX_INPUT); struct workspace *workspace; - workspace = kzalloc(sizeof(*workspace), GFP_NOFS); + workspace = kzalloc(sizeof(*workspace), GFP_KERNEL); if (!workspace) return ERR_PTR(-ENOMEM); workspace->size = max_t(size_t, ZSTD_CStreamWorkspaceBound(params.cParams), ZSTD_DStreamWorkspaceBound(ZSTD_BTRFS_MAX_INPUT)); - workspace->mem = vmalloc(workspace->size); - workspace->buf = kmalloc(PAGE_SIZE, GFP_NOFS); + workspace->mem = kvmalloc(workspace->size, GFP_KERNEL); + workspace->buf = kmalloc(PAGE_SIZE, GFP_KERNEL); if (!workspace->mem || !workspace->buf) goto fail; @@ -262,12 +263,13 @@ out: return ret; } -static int zstd_decompress_bio(struct list_head *ws, struct page **pages_in, - u64 disk_start, - struct bio *orig_bio, - size_t srclen) +static int zstd_decompress_bio(struct list_head *ws, struct compressed_bio *cb) { struct workspace *workspace = list_entry(ws, struct workspace, list); + struct page **pages_in = cb->compressed_pages; + u64 disk_start = cb->start; + struct bio *orig_bio = cb->orig_bio; + size_t srclen = cb->compressed_len; ZSTD_DStream *stream; int ret = 0; unsigned long page_in_index = 0; diff --git a/contrib/linux-kernel/lib/zstd/huf_compress.c b/contrib/linux-kernel/lib/zstd/huf_compress.c index 0361f387..40055a70 100644 --- a/contrib/linux-kernel/lib/zstd/huf_compress.c +++ b/contrib/linux-kernel/lib/zstd/huf_compress.c @@ -525,7 +525,7 @@ static void HUF_encodeSymbol(BIT_CStream_t *bitCPtr, U32 symbol, const HUF_CElt size_t HUF_compressBound(size_t size) { return HUF_COMPRESSBOUND(size); } -#define HUF_FLUSHBITS(s) (fast ? BIT_flushBitsFast(s) : BIT_flushBits(s)) +#define HUF_FLUSHBITS(s) BIT_flushBits(s) #define HUF_FLUSHBITS_1(stream) \ if (sizeof((stream)->bitContainer) * 8 < HUF_TABLELOG_MAX * 2 + 7) \ @@ -542,7 +542,6 @@ size_t HUF_compress1X_usingCTable(void *dst, size_t dstSize, const void *src, si BYTE *const oend = ostart + dstSize; BYTE *op = ostart; size_t n; - const unsigned fast = (dstSize >= HUF_BLOCKBOUND(srcSize)); BIT_CStream_t bitC; /* init */ diff --git a/contrib/linux-kernel/lib/zstd/zstd_internal.h b/contrib/linux-kernel/lib/zstd/zstd_internal.h index 67487199..f0ba4744 100644 --- a/contrib/linux-kernel/lib/zstd/zstd_internal.h +++ b/contrib/linux-kernel/lib/zstd/zstd_internal.h @@ -126,35 +126,16 @@ static const U32 OF_defaultNormLog = OF_DEFAULTNORMLOG; /*-******************************************* * Shared functions to include for inlining *********************************************/ -static void ZSTD_copy8(void *dst, const void *src) { memcpy(dst, src, 8); } -#define COPY8(d, s) \ - { \ - ZSTD_copy8(d, s); \ - d += 8; \ - s += 8; \ - } - +static void ZSTD_copy8(void *dst, const void *src) { + memcpy(dst, src, 8); +} /*! ZSTD_wildcopy() : * custom version of memcpy(), can copy up to 7 bytes too many (8 bytes if length==0) */ #define WILDCOPY_OVERLENGTH 8 ZSTD_STATIC void ZSTD_wildcopy(void *dst, const void *src, ptrdiff_t length) { - const BYTE *ip = (const BYTE *)src; - BYTE *op = (BYTE *)dst; - BYTE *const oend = op + length; - do - COPY8(op, ip) - while (op < oend); -} - -ZSTD_STATIC void ZSTD_wildcopy_e(void *dst, const void *src, void *dstEnd) /* should be faster for decoding, but strangely, not verified on all platform */ -{ - const BYTE *ip = (const BYTE *)src; - BYTE *op = (BYTE *)dst; - BYTE *const oend = (BYTE *)dstEnd; - do - COPY8(op, ip) - while (op < oend); + if (length > 0) + memcpy(dst, src, length); } /*-*******************************************