facebook
/
zstd
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'dev' of github.com:facebook/zstd into dev

dev
Yann Collet 2017-07-20 15:12:21 -07:00
parent 5e6c5203f3 57251f52cf
commit 4542d7118c
8 changed files with 105 additions and 126 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
contrib/linux-kernel/0000-cover-letter.patch
View File

@ -1,7 +1,7 @@
From 8bc9a0ae5c86a6d02d9a5274b9965ddac0e8d330 Mon Sep 17 00:00:00 2001 From 0cd63464d182bb9708f8b25f7da3dc8e5ec6b4fa Mon Sep 17 00:00:00 2001
From: Nick Terrell <terrelln@fb.com> From: Nick Terrell <terrelln@fb.com>
Date: Wed, 28 Jun 2017 22:00:00 -0700 Date: Thu, 20 Jul 2017 13:18:30 -0700
Subject: [PATCH v2 0/4] Add xxhash and zstd modules Subject: [PATCH v3 0/4] Add xxhash and zstd modules
Hi all, Hi all,
@ -24,6 +24,13 @@ v1 -> v2:
HUF_compressWeights(), HUF_readDTableX2(), and HUF_readDTableX4() (2/4) HUF_compressWeights(), HUF_readDTableX2(), and HUF_readDTableX4() (2/4)
- No zstd function uses more than 400 B of stack space (2/4) - No zstd function uses more than 400 B of stack space (2/4)
v2 -> v3:
- Work around gcc-7 bug https://gcc.gnu.org/bugzilla/show_bug.cgi?id=81388
(2/4)
- Fix bug in dictionary compression from upstream commit cc1522351f (2/4)
- Port upstream BtrFS commits e1ddce71d6, 389a6cfc2a, and 6acafd1eff (3/4)
- Change default compression level for BtrFS to 3 (3/4)
Nick Terrell (4): Nick Terrell (4):
lib: Add xxhash module lib: Add xxhash module
lib: Add zstd modules lib: Add zstd modules
@ -40,7 +47,7 @@ Nick Terrell (4):
fs/btrfs/props.c | 6 + fs/btrfs/props.c | 6 +
fs/btrfs/super.c | 12 +- fs/btrfs/super.c | 12 +-
fs/btrfs/sysfs.c | 2 + fs/btrfs/sysfs.c | 2 +
fs/btrfs/zstd.c | 433 ++++++ fs/btrfs/zstd.c | 435 ++++++
fs/squashfs/Kconfig | 14 + fs/squashfs/Kconfig | 14 +
fs/squashfs/Makefile | 1 + fs/squashfs/Makefile | 1 +
fs/squashfs/decompressor.c | 7 + fs/squashfs/decompressor.c | 7 +
@ -63,13 +70,13 @@ Nick Terrell (4):
lib/zstd/fse_compress.c | 795 ++++++++++ lib/zstd/fse_compress.c | 795 ++++++++++
lib/zstd/fse_decompress.c | 332 +++++ lib/zstd/fse_decompress.c | 332 +++++
lib/zstd/huf.h | 212 +++ lib/zstd/huf.h | 212 +++
lib/zstd/huf_compress.c | 771 ++++++++++ lib/zstd/huf_compress.c | 770 ++++++++++
lib/zstd/huf_decompress.c | 960 ++++++++++++ lib/zstd/huf_decompress.c | 960 ++++++++++++
lib/zstd/mem.h | 151 ++ lib/zstd/mem.h | 151 ++
lib/zstd/zstd_common.c | 75 + lib/zstd/zstd_common.c | 75 +
lib/zstd/zstd_internal.h | 269 ++++ lib/zstd/zstd_internal.h | 250 ++++
lib/zstd/zstd_opt.h | 1014 +++++++++++++ lib/zstd/zstd_opt.h | 1014 +++++++++++++
39 files changed, 14400 insertions(+), 12 deletions(-) 39 files changed, 14382 insertions(+), 12 deletions(-)
create mode 100644 fs/btrfs/zstd.c create mode 100644 fs/btrfs/zstd.c
create mode 100644 fs/squashfs/zstd_wrapper.c create mode 100644 fs/squashfs/zstd_wrapper.c
create mode 100644 include/linux/xxhash.h create mode 100644 include/linux/xxhash.h

14
contrib/linux-kernel/0001-lib-Add-xxhash-module.patch
View File

@ -1,7 +1,7 @@
From 5ac909c415ab4a18fd90794793c96e450795e8c6 Mon Sep 17 00:00:00 2001 From fc7f26acbabda35f1c61dfc357dbb207dc8ed23d Mon Sep 17 00:00:00 2001
From: Nick Terrell <terrelln@fb.com> From: Nick Terrell <terrelln@fb.com>
Date: Wed, 21 Jun 2017 17:37:36 -0700 Date: Mon, 17 Jul 2017 17:07:18 -0700
Subject: [PATCH v2 1/4] lib: Add xxhash module Subject: [PATCH v3 1/4] lib: Add xxhash module
Adds xxhash kernel module with xxh32 and xxh64 hashes. xxhash is an Adds xxhash kernel module with xxh32 and xxh64 hashes. xxhash is an
extremely fast non-cryptographic hash algorithm for checksumming. extremely fast non-cryptographic hash algorithm for checksumming.
@ -327,10 +327,10 @@ index 0000000..9e1f42c
+ +
+#endif /* XXHASH_H */ +#endif /* XXHASH_H */
diff --git a/lib/Kconfig b/lib/Kconfig diff --git a/lib/Kconfig b/lib/Kconfig
index 0c8b78a..b6009d7 100644 index 6762529..5e7541f 100644
--- a/lib/Kconfig --- a/lib/Kconfig
+++ b/lib/Kconfig +++ b/lib/Kconfig
@@ -184,6 +184,9 @@ config CRC8 @@ -192,6 +192,9 @@ config CRC8
when they need to do cyclic redundancy check according CRC8 when they need to do cyclic redundancy check according CRC8
algorithm. Module will be called crc8. algorithm. Module will be called crc8.
@ -341,10 +341,10 @@ index 0c8b78a..b6009d7 100644
bool bool
depends on AUDIT && !AUDIT_ARCH depends on AUDIT && !AUDIT_ARCH
diff --git a/lib/Makefile b/lib/Makefile diff --git a/lib/Makefile b/lib/Makefile
index 0166fbc..1338226 100644 index 40c1837..d06b68a 100644
--- a/lib/Makefile --- a/lib/Makefile
+++ b/lib/Makefile +++ b/lib/Makefile
@@ -102,6 +102,7 @@ obj-$(CONFIG_CRC32_SELFTEST) += crc32test.o @@ -102,6 +102,7 @@ obj-$(CONFIG_CRC4) += crc4.o
obj-$(CONFIG_CRC7) += crc7.o obj-$(CONFIG_CRC7) += crc7.o
obj-$(CONFIG_LIBCRC32C) += libcrc32c.o obj-$(CONFIG_LIBCRC32C) += libcrc32c.o
obj-$(CONFIG_CRC8) += crc8.o obj-$(CONFIG_CRC8) += crc8.o

60
contrib/linux-kernel/0002-lib-Add-zstd-modules.patch
View File

@ -1,7 +1,7 @@
From d2626127c6d6e60e940dd9a3ed58323bdcdc4930 Mon Sep 17 00:00:00 2001 From 686a6149b98250d66b5951e3ae05e79063e9de98 Mon Sep 17 00:00:00 2001
From: Nick Terrell <terrelln@fb.com> From: Nick Terrell <terrelln@fb.com>
Date: Tue, 16 May 2017 14:55:36 -0700 Date: Mon, 17 Jul 2017 17:08:19 -0700
Subject: [PATCH v2 2/4] lib: Add zstd modules Subject: [PATCH v3 2/4] lib: Add zstd modules
Add zstd compression and decompression kernel modules. Add zstd compression and decompression kernel modules.
zstd offers a wide varity of compression speed and quality trade-offs. zstd offers a wide varity of compression speed and quality trade-offs.
@ -110,6 +110,10 @@ v1 -> v2:
HUF_compressWeights(), HUF_readDTableX2(), and HUF_readDTableX4() HUF_compressWeights(), HUF_readDTableX2(), and HUF_readDTableX4()
- No function uses more than 400 B of stack space - No function uses more than 400 B of stack space
v2 -> v3:
- Work around gcc-7 bug https://gcc.gnu.org/bugzilla/show_bug.cgi?id=81388
- Fix bug in dictionary compression from upstream commit cc1522351f
include/linux/zstd.h | 1157 +++++++++++++++ include/linux/zstd.h | 1157 +++++++++++++++
lib/Kconfig | 8 + lib/Kconfig | 8 +
lib/Makefile | 2 + lib/Makefile | 2 +
@ -123,13 +127,13 @@ v1 -> v2:
lib/zstd/fse_compress.c | 795 +++++++++++ lib/zstd/fse_compress.c | 795 +++++++++++
lib/zstd/fse_decompress.c | 332 +++++ lib/zstd/fse_decompress.c | 332 +++++
lib/zstd/huf.h | 212 +++ lib/zstd/huf.h | 212 +++
lib/zstd/huf_compress.c | 771 ++++++++++ lib/zstd/huf_compress.c | 770 ++++++++++
lib/zstd/huf_decompress.c | 960 +++++++++++++ lib/zstd/huf_decompress.c | 960 +++++++++++++
lib/zstd/mem.h | 151 ++ lib/zstd/mem.h | 151 ++
lib/zstd/zstd_common.c | 75 + lib/zstd/zstd_common.c | 75 +
lib/zstd/zstd_internal.h | 269 ++++ lib/zstd/zstd_internal.h | 250 ++++
lib/zstd/zstd_opt.h | 1014 +++++++++++++ lib/zstd/zstd_opt.h | 1014 +++++++++++++
19 files changed, 13014 insertions(+) 19 files changed, 12994 insertions(+)
create mode 100644 include/linux/zstd.h create mode 100644 include/linux/zstd.h
create mode 100644 lib/zstd/Makefile create mode 100644 lib/zstd/Makefile
create mode 100644 lib/zstd/bitstream.h create mode 100644 lib/zstd/bitstream.h
@ -1312,10 +1316,10 @@ index 0000000..249575e
+ +
+#endif /* ZSTD_H */ +#endif /* ZSTD_H */
diff --git a/lib/Kconfig b/lib/Kconfig diff --git a/lib/Kconfig b/lib/Kconfig
index b6009d7..f00ddab 100644 index 5e7541f..0d49ed0 100644
--- a/lib/Kconfig --- a/lib/Kconfig
+++ b/lib/Kconfig +++ b/lib/Kconfig
@@ -241,6 +241,14 @@ config LZ4HC_COMPRESS @@ -249,6 +249,14 @@ config LZ4HC_COMPRESS
config LZ4_DECOMPRESS config LZ4_DECOMPRESS
tristate tristate
@ -1331,7 +1335,7 @@ index b6009d7..f00ddab 100644
# #
diff --git a/lib/Makefile b/lib/Makefile diff --git a/lib/Makefile b/lib/Makefile
index 1338226..4fcef16 100644 index d06b68a..d5c8a4f 100644
--- a/lib/Makefile --- a/lib/Makefile
+++ b/lib/Makefile +++ b/lib/Makefile
@@ -116,6 +116,8 @@ obj-$(CONFIG_LZO_DECOMPRESS) += lzo/ @@ -116,6 +116,8 @@ obj-$(CONFIG_LZO_DECOMPRESS) += lzo/
@ -10012,10 +10016,10 @@ index 0000000..2143da2
+#endif /* HUF_H_298734234 */ +#endif /* HUF_H_298734234 */
diff --git a/lib/zstd/huf_compress.c b/lib/zstd/huf_compress.c diff --git a/lib/zstd/huf_compress.c b/lib/zstd/huf_compress.c
new file mode 100644 new file mode 100644
index 0000000..0361f38 index 0000000..40055a7
--- /dev/null --- /dev/null
+++ b/lib/zstd/huf_compress.c +++ b/lib/zstd/huf_compress.c
@@ -0,0 +1,771 @@ @@ -0,0 +1,770 @@
+/* +/*
+ * Huffman encoder, part of New Generation Entropy library + * Huffman encoder, part of New Generation Entropy library
+ * Copyright (C) 2013-2016, Yann Collet. + * Copyright (C) 2013-2016, Yann Collet.
@ -10543,7 +10547,7 @@ index 0000000..0361f38
+ +
+size_t HUF_compressBound(size_t size) { return HUF_COMPRESSBOUND(size); } +size_t HUF_compressBound(size_t size) { return HUF_COMPRESSBOUND(size); }
+ +
+#define HUF_FLUSHBITS(s) (fast ? BIT_flushBitsFast(s) : BIT_flushBits(s)) +#define HUF_FLUSHBITS(s) BIT_flushBits(s)
+ +
+#define HUF_FLUSHBITS_1(stream) \ +#define HUF_FLUSHBITS_1(stream) \
+ if (sizeof((stream)->bitContainer) * 8 < HUF_TABLELOG_MAX * 2 + 7) \ + if (sizeof((stream)->bitContainer) * 8 < HUF_TABLELOG_MAX * 2 + 7) \
@ -10560,7 +10564,6 @@ index 0000000..0361f38
+ BYTE *const oend = ostart + dstSize; + BYTE *const oend = ostart + dstSize;
+ BYTE *op = ostart; + BYTE *op = ostart;
+ size_t n; + size_t n;
+ const unsigned fast = (dstSize >= HUF_BLOCKBOUND(srcSize));
+ BIT_CStream_t bitC; + BIT_CStream_t bitC;
+ +
+ /* init */ + /* init */
@ -11993,10 +11996,10 @@ index 0000000..a282624
+} +}
diff --git a/lib/zstd/zstd_internal.h b/lib/zstd/zstd_internal.h diff --git a/lib/zstd/zstd_internal.h b/lib/zstd/zstd_internal.h
new file mode 100644 new file mode 100644
index 0000000..6748719 index 0000000..f0ba474
--- /dev/null --- /dev/null
+++ b/lib/zstd/zstd_internal.h +++ b/lib/zstd/zstd_internal.h
@@ -0,0 +1,269 @@ @@ -0,0 +1,250 @@
+/** +/**
+ * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
+ * All rights reserved. + * All rights reserved.
@ -12125,35 +12128,16 @@ index 0000000..6748719
+/*-******************************************* +/*-*******************************************
+* Shared functions to include for inlining +* Shared functions to include for inlining
+*********************************************/ +*********************************************/
+static void ZSTD_copy8(void *dst, const void *src) { memcpy(dst, src, 8); } +static void ZSTD_copy8(void *dst, const void *src) {
+#define COPY8(d, s) \ + memcpy(dst, src, 8);
+ { \
+ ZSTD_copy8(d, s); \
+ d += 8; \
+ s += 8; \
+} +}
+
+/*! ZSTD_wildcopy() : +/*! ZSTD_wildcopy() :
+* custom version of memcpy(), can copy up to 7 bytes too many (8 bytes if length==0) */ +* custom version of memcpy(), can copy up to 7 bytes too many (8 bytes if length==0) */
+#define WILDCOPY_OVERLENGTH 8 +#define WILDCOPY_OVERLENGTH 8
+ZSTD_STATIC void ZSTD_wildcopy(void *dst, const void *src, ptrdiff_t length) +ZSTD_STATIC void ZSTD_wildcopy(void *dst, const void *src, ptrdiff_t length)
+{ +{
+ const BYTE *ip = (const BYTE *)src; + if (length > 0)
+ BYTE *op = (BYTE *)dst; + memcpy(dst, src, length);
+ BYTE *const oend = op + length;
+ do
+ COPY8(op, ip)
+ while (op < oend);
+}
+
+ZSTD_STATIC void ZSTD_wildcopy_e(void *dst, const void *src, void *dstEnd) /* should be faster for decoding, but strangely, not verified on all platform */
+{
+ const BYTE *ip = (const BYTE *)src;
+ BYTE *op = (BYTE *)dst;
+ BYTE *const oend = (BYTE *)dstEnd;
+ do
+ COPY8(op, ip)
+ while (op < oend);
+} +}
+ +
+/*-******************************************* +/*-*******************************************

76
contrib/linux-kernel/0003-btrfs-Add-zstd-support.patch
View File

@ -1,7 +1,7 @@
From 599f8f2aaace3df939cb145368574a52268d82d0 Mon Sep 17 00:00:00 2001 From b0ef8fc63c9ca251ceca632f53aa1de8f1f17772 Mon Sep 17 00:00:00 2001
From: Nick Terrell <terrelln@fb.com> From: Nick Terrell <terrelln@fb.com>
Date: Wed, 21 Jun 2017 17:31:39 -0700 Date: Mon, 17 Jul 2017 17:08:39 -0700
Subject: [PATCH v2 3/4] btrfs: Add zstd support Subject: [PATCH v3 3/4] btrfs: Add zstd support
Add zstd compression and decompression support to BtrFS. zstd at its Add zstd compression and decompression support to BtrFS. zstd at its
fastest level compresses almost as well as zlib, while offering much fastest level compresses almost as well as zlib, while offering much
@ -63,6 +63,10 @@ zstd source repository: https://github.com/facebook/zstd
Signed-off-by: Nick Terrell <terrelln@fb.com> Signed-off-by: Nick Terrell <terrelln@fb.com>
--- ---
v2 -> v3:
- Port upstream BtrFS commits e1ddce71d6, 389a6cfc2a, and 6acafd1eff
- Change default compression level for BtrFS to 3
fs/btrfs/Kconfig | 2 + fs/btrfs/Kconfig | 2 +
fs/btrfs/Makefile | 2 +- fs/btrfs/Makefile | 2 +-
fs/btrfs/compression.c | 1 + fs/btrfs/compression.c | 1 +
@ -73,9 +77,9 @@ Signed-off-by: Nick Terrell <terrelln@fb.com>
fs/btrfs/props.c | 6 + fs/btrfs/props.c | 6 +
fs/btrfs/super.c | 12 +- fs/btrfs/super.c | 12 +-
fs/btrfs/sysfs.c | 2 + fs/btrfs/sysfs.c | 2 +
fs/btrfs/zstd.c | 433 +++++++++++++++++++++++++++++++++++++++++++++ fs/btrfs/zstd.c | 435 +++++++++++++++++++++++++++++++++++++++++++++
include/uapi/linux/btrfs.h | 8 +- include/uapi/linux/btrfs.h | 8 +-
12 files changed, 469 insertions(+), 12 deletions(-) 12 files changed, 471 insertions(+), 12 deletions(-)
create mode 100644 fs/btrfs/zstd.c create mode 100644 fs/btrfs/zstd.c
diff --git a/fs/btrfs/Kconfig b/fs/btrfs/Kconfig diff --git a/fs/btrfs/Kconfig b/fs/btrfs/Kconfig
@ -105,10 +109,10 @@ index 128ce17..962a95a 100644
reada.o backref.o ulist.o qgroup.o send.o dev-replace.o raid56.o \ reada.o backref.o ulist.o qgroup.o send.o dev-replace.o raid56.o \
uuid-tree.o props.o hash.o free-space-tree.o uuid-tree.o props.o hash.o free-space-tree.o
diff --git a/fs/btrfs/compression.c b/fs/btrfs/compression.c diff --git a/fs/btrfs/compression.c b/fs/btrfs/compression.c
index 10e6b28..3beb0d0 100644 index d2ef9ac..4ff42d1 100644
--- a/fs/btrfs/compression.c --- a/fs/btrfs/compression.c
+++ b/fs/btrfs/compression.c +++ b/fs/btrfs/compression.c
@@ -761,6 +761,7 @@ static struct { @@ -704,6 +704,7 @@ static struct {
static const struct btrfs_compress_op * const btrfs_compress_op[] = { static const struct btrfs_compress_op * const btrfs_compress_op[] = {
&btrfs_zlib_compress, &btrfs_zlib_compress,
&btrfs_lzo_compress, &btrfs_lzo_compress,
@ -117,10 +121,10 @@ index 10e6b28..3beb0d0 100644
void __init btrfs_init_compress(void) void __init btrfs_init_compress(void)
diff --git a/fs/btrfs/compression.h b/fs/btrfs/compression.h diff --git a/fs/btrfs/compression.h b/fs/btrfs/compression.h
index 39ec43a..d99fc21 100644 index 87f6d33..2269e00 100644
--- a/fs/btrfs/compression.h --- a/fs/btrfs/compression.h
+++ b/fs/btrfs/compression.h +++ b/fs/btrfs/compression.h
@@ -60,8 +60,9 @@ enum btrfs_compression_type { @@ -99,8 +99,9 @@ enum btrfs_compression_type {
BTRFS_COMPRESS_NONE = 0, BTRFS_COMPRESS_NONE = 0,
BTRFS_COMPRESS_ZLIB = 1, BTRFS_COMPRESS_ZLIB = 1,
BTRFS_COMPRESS_LZO = 2, BTRFS_COMPRESS_LZO = 2,
@ -132,7 +136,7 @@ index 39ec43a..d99fc21 100644
}; };
struct btrfs_compress_op { struct btrfs_compress_op {
@@ -92,5 +93,6 @@ struct btrfs_compress_op { @@ -128,5 +129,6 @@ struct btrfs_compress_op {
extern const struct btrfs_compress_op btrfs_zlib_compress; extern const struct btrfs_compress_op btrfs_zlib_compress;
extern const struct btrfs_compress_op btrfs_lzo_compress; extern const struct btrfs_compress_op btrfs_lzo_compress;
@ -140,10 +144,10 @@ index 39ec43a..d99fc21 100644
#endif #endif
diff --git a/fs/btrfs/ctree.h b/fs/btrfs/ctree.h diff --git a/fs/btrfs/ctree.h b/fs/btrfs/ctree.h
index 4f8f75d..61dd3dd 100644 index 3f3eb7b..845d77c 100644
--- a/fs/btrfs/ctree.h --- a/fs/btrfs/ctree.h
+++ b/fs/btrfs/ctree.h +++ b/fs/btrfs/ctree.h
@@ -271,6 +271,7 @@ struct btrfs_super_block { @@ -270,6 +270,7 @@ struct btrfs_super_block {
BTRFS_FEATURE_INCOMPAT_MIXED_GROUPS | \ BTRFS_FEATURE_INCOMPAT_MIXED_GROUPS | \
BTRFS_FEATURE_INCOMPAT_BIG_METADATA | \ BTRFS_FEATURE_INCOMPAT_BIG_METADATA | \
BTRFS_FEATURE_INCOMPAT_COMPRESS_LZO | \ BTRFS_FEATURE_INCOMPAT_COMPRESS_LZO | \
@ -152,10 +156,10 @@ index 4f8f75d..61dd3dd 100644
BTRFS_FEATURE_INCOMPAT_EXTENDED_IREF | \ BTRFS_FEATURE_INCOMPAT_EXTENDED_IREF | \
BTRFS_FEATURE_INCOMPAT_SKINNY_METADATA | \ BTRFS_FEATURE_INCOMPAT_SKINNY_METADATA | \
diff --git a/fs/btrfs/disk-io.c b/fs/btrfs/disk-io.c diff --git a/fs/btrfs/disk-io.c b/fs/btrfs/disk-io.c
index 5f678dc..49c0e91 100644 index 080e2eb..04632f4 100644
--- a/fs/btrfs/disk-io.c --- a/fs/btrfs/disk-io.c
+++ b/fs/btrfs/disk-io.c +++ b/fs/btrfs/disk-io.c
@@ -2831,6 +2831,8 @@ int open_ctree(struct super_block *sb, @@ -2828,6 +2828,8 @@ int open_ctree(struct super_block *sb,
features |= BTRFS_FEATURE_INCOMPAT_MIXED_BACKREF; features |= BTRFS_FEATURE_INCOMPAT_MIXED_BACKREF;
if (fs_info->compress_type == BTRFS_COMPRESS_LZO) if (fs_info->compress_type == BTRFS_COMPRESS_LZO)
features |= BTRFS_FEATURE_INCOMPAT_COMPRESS_LZO; features |= BTRFS_FEATURE_INCOMPAT_COMPRESS_LZO;
@ -165,7 +169,7 @@ index 5f678dc..49c0e91 100644
if (features & BTRFS_FEATURE_INCOMPAT_SKINNY_METADATA) if (features & BTRFS_FEATURE_INCOMPAT_SKINNY_METADATA)
btrfs_info(fs_info, "has skinny extents"); btrfs_info(fs_info, "has skinny extents");
diff --git a/fs/btrfs/ioctl.c b/fs/btrfs/ioctl.c diff --git a/fs/btrfs/ioctl.c b/fs/btrfs/ioctl.c
index e176375..f732cfd 100644 index fa1b78c..b9963d9 100644
--- a/fs/btrfs/ioctl.c --- a/fs/btrfs/ioctl.c
+++ b/fs/btrfs/ioctl.c +++ b/fs/btrfs/ioctl.c
@@ -327,8 +327,10 @@ static int btrfs_ioctl_setflags(struct file *file, void __user *arg) @@ -327,8 +327,10 @@ static int btrfs_ioctl_setflags(struct file *file, void __user *arg)
@ -180,7 +184,7 @@ index e176375..f732cfd 100644
ret = btrfs_set_prop(inode, "btrfs.compression", ret = btrfs_set_prop(inode, "btrfs.compression",
comp, strlen(comp), 0); comp, strlen(comp), 0);
if (ret) if (ret)
@@ -1463,6 +1465,8 @@ int btrfs_defrag_file(struct inode *inode, struct file *file, @@ -1466,6 +1468,8 @@ int btrfs_defrag_file(struct inode *inode, struct file *file,
if (range->compress_type == BTRFS_COMPRESS_LZO) { if (range->compress_type == BTRFS_COMPRESS_LZO) {
btrfs_set_fs_incompat(fs_info, COMPRESS_LZO); btrfs_set_fs_incompat(fs_info, COMPRESS_LZO);
@ -190,10 +194,10 @@ index e176375..f732cfd 100644
ret = defrag_count; ret = defrag_count;
diff --git a/fs/btrfs/props.c b/fs/btrfs/props.c diff --git a/fs/btrfs/props.c b/fs/btrfs/props.c
index d6cb155..162105f 100644 index 4b23ae5..20631e9 100644
--- a/fs/btrfs/props.c --- a/fs/btrfs/props.c
+++ b/fs/btrfs/props.c +++ b/fs/btrfs/props.c
@@ -383,6 +383,8 @@ static int prop_compression_validate(const char *value, size_t len) @@ -390,6 +390,8 @@ static int prop_compression_validate(const char *value, size_t len)
return 0; return 0;
else if (!strncmp("zlib", value, len)) else if (!strncmp("zlib", value, len))
return 0; return 0;
@ -202,7 +206,7 @@ index d6cb155..162105f 100644
return -EINVAL; return -EINVAL;
} }
@@ -405,6 +407,8 @@ static int prop_compression_apply(struct inode *inode, @@ -412,6 +414,8 @@ static int prop_compression_apply(struct inode *inode,
type = BTRFS_COMPRESS_LZO; type = BTRFS_COMPRESS_LZO;
else if (!strncmp("zlib", value, len)) else if (!strncmp("zlib", value, len))
type = BTRFS_COMPRESS_ZLIB; type = BTRFS_COMPRESS_ZLIB;
@ -211,7 +215,7 @@ index d6cb155..162105f 100644
else else
return -EINVAL; return -EINVAL;
@@ -422,6 +426,8 @@ static const char *prop_compression_extract(struct inode *inode) @@ -429,6 +433,8 @@ static const char *prop_compression_extract(struct inode *inode)
return "zlib"; return "zlib";
case BTRFS_COMPRESS_LZO: case BTRFS_COMPRESS_LZO:
return "lzo"; return "lzo";
@ -221,7 +225,7 @@ index d6cb155..162105f 100644
return NULL; return NULL;
diff --git a/fs/btrfs/super.c b/fs/btrfs/super.c diff --git a/fs/btrfs/super.c b/fs/btrfs/super.c
index 4f1cdd5..4f792d5 100644 index 12540b6..c370dea 100644
--- a/fs/btrfs/super.c --- a/fs/btrfs/super.c
+++ b/fs/btrfs/super.c +++ b/fs/btrfs/super.c
@@ -513,6 +513,14 @@ int btrfs_parse_options(struct btrfs_fs_info *info, char *options, @@ -513,6 +513,14 @@ int btrfs_parse_options(struct btrfs_fs_info *info, char *options,
@ -239,7 +243,7 @@ index 4f1cdd5..4f792d5 100644
} else if (strncmp(args[0].from, "no", 2) == 0) { } else if (strncmp(args[0].from, "no", 2) == 0) {
compress_type = "no"; compress_type = "no";
btrfs_clear_opt(info->mount_opt, COMPRESS); btrfs_clear_opt(info->mount_opt, COMPRESS);
@@ -1240,8 +1248,10 @@ static int btrfs_show_options(struct seq_file *seq, struct dentry *dentry) @@ -1227,8 +1235,10 @@ static int btrfs_show_options(struct seq_file *seq, struct dentry *dentry)
if (btrfs_test_opt(info, COMPRESS)) { if (btrfs_test_opt(info, COMPRESS)) {
if (info->compress_type == BTRFS_COMPRESS_ZLIB) if (info->compress_type == BTRFS_COMPRESS_ZLIB)
compress_type = "zlib"; compress_type = "zlib";
@ -252,7 +256,7 @@ index 4f1cdd5..4f792d5 100644
seq_printf(seq, ",compress-force=%s", compress_type); seq_printf(seq, ",compress-force=%s", compress_type);
else else
diff --git a/fs/btrfs/sysfs.c b/fs/btrfs/sysfs.c diff --git a/fs/btrfs/sysfs.c b/fs/btrfs/sysfs.c
index 1f157fb..b0dec90 100644 index c2d5f35..2b6d37c 100644
--- a/fs/btrfs/sysfs.c --- a/fs/btrfs/sysfs.c
+++ b/fs/btrfs/sysfs.c +++ b/fs/btrfs/sysfs.c
@@ -200,6 +200,7 @@ BTRFS_FEAT_ATTR_INCOMPAT(mixed_backref, MIXED_BACKREF); @@ -200,6 +200,7 @@ BTRFS_FEAT_ATTR_INCOMPAT(mixed_backref, MIXED_BACKREF);
@ -273,10 +277,10 @@ index 1f157fb..b0dec90 100644
BTRFS_FEAT_ATTR_PTR(raid56), BTRFS_FEAT_ATTR_PTR(raid56),
diff --git a/fs/btrfs/zstd.c b/fs/btrfs/zstd.c diff --git a/fs/btrfs/zstd.c b/fs/btrfs/zstd.c
new file mode 100644 new file mode 100644
index 0000000..838741b index 0000000..1822068
--- /dev/null --- /dev/null
+++ b/fs/btrfs/zstd.c +++ b/fs/btrfs/zstd.c
@@ -0,0 +1,433 @@ @@ -0,0 +1,435 @@
+/* +/*
+ * Copyright (c) 2016-present, Facebook, Inc. + * Copyright (c) 2016-present, Facebook, Inc.
+ * All rights reserved. + * All rights reserved.
@ -308,10 +312,11 @@ index 0000000..838741b
+ +
+#define ZSTD_BTRFS_MAX_WINDOWLOG 17 +#define ZSTD_BTRFS_MAX_WINDOWLOG 17
+#define ZSTD_BTRFS_MAX_INPUT (1 << ZSTD_BTRFS_MAX_WINDOWLOG) +#define ZSTD_BTRFS_MAX_INPUT (1 << ZSTD_BTRFS_MAX_WINDOWLOG)
+#define ZSTD_BTRFS_DEFAULT_LEVEL 3
+ +
+static ZSTD_parameters zstd_get_btrfs_parameters(size_t src_len) +static ZSTD_parameters zstd_get_btrfs_parameters(size_t src_len)
+{ +{
+ ZSTD_parameters params = ZSTD_getParams(1, src_len, 0); + ZSTD_parameters params = ZSTD_getParams(ZSTD_BTRFS_DEFAULT_LEVEL, src_len, 0);
+ +
+ if (params.cParams.windowLog > ZSTD_BTRFS_MAX_WINDOWLOG) + if (params.cParams.windowLog > ZSTD_BTRFS_MAX_WINDOWLOG)
+ params.cParams.windowLog = ZSTD_BTRFS_MAX_WINDOWLOG; + params.cParams.windowLog = ZSTD_BTRFS_MAX_WINDOWLOG;
@ -330,7 +335,7 @@ index 0000000..838741b
+{ +{
+ struct workspace *workspace = list_entry(ws, struct workspace, list); + struct workspace *workspace = list_entry(ws, struct workspace, list);
+ +
+ vfree(workspace->mem); + kvfree(workspace->mem);
+ kfree(workspace->buf); + kfree(workspace->buf);
+ kfree(workspace); + kfree(workspace);
+} +}
@ -341,15 +346,15 @@ index 0000000..838741b
+ zstd_get_btrfs_parameters(ZSTD_BTRFS_MAX_INPUT); + zstd_get_btrfs_parameters(ZSTD_BTRFS_MAX_INPUT);
+ struct workspace *workspace; + struct workspace *workspace;
+ +
+ workspace = kzalloc(sizeof(*workspace), GFP_NOFS); + workspace = kzalloc(sizeof(*workspace), GFP_KERNEL);
+ if (!workspace) + if (!workspace)
+ return ERR_PTR(-ENOMEM); + return ERR_PTR(-ENOMEM);
+ +
+ workspace->size = max_t(size_t, + workspace->size = max_t(size_t,
+ ZSTD_CStreamWorkspaceBound(params.cParams), + ZSTD_CStreamWorkspaceBound(params.cParams),
+ ZSTD_DStreamWorkspaceBound(ZSTD_BTRFS_MAX_INPUT)); + ZSTD_DStreamWorkspaceBound(ZSTD_BTRFS_MAX_INPUT));
+ workspace->mem = vmalloc(workspace->size); + workspace->mem = kvmalloc(workspace->size, GFP_KERNEL);
+ workspace->buf = kmalloc(PAGE_SIZE, GFP_NOFS); + workspace->buf = kmalloc(PAGE_SIZE, GFP_KERNEL);
+ if (!workspace->mem || !workspace->buf) + if (!workspace->mem || !workspace->buf)
+ goto fail; + goto fail;
+ +
@ -541,12 +546,13 @@ index 0000000..838741b
+ return ret; + return ret;
+} +}
+ +
+static int zstd_decompress_bio(struct list_head *ws, struct page **pages_in, +static int zstd_decompress_bio(struct list_head *ws, struct compressed_bio *cb)
+ u64 disk_start,
+ struct bio *orig_bio,
+ size_t srclen)
+{ +{
+ struct workspace *workspace = list_entry(ws, struct workspace, list); + struct workspace *workspace = list_entry(ws, struct workspace, list);
+ struct page **pages_in = cb->compressed_pages;
+ u64 disk_start = cb->start;
+ struct bio *orig_bio = cb->orig_bio;
+ size_t srclen = cb->compressed_len;
+ ZSTD_DStream *stream; + ZSTD_DStream *stream;
+ int ret = 0; + int ret = 0;
+ unsigned long page_in_index = 0; + unsigned long page_in_index = 0;
@ -711,7 +717,7 @@ index 0000000..838741b
+ .decompress = zstd_decompress, + .decompress = zstd_decompress,
+}; +};
diff --git a/include/uapi/linux/btrfs.h b/include/uapi/linux/btrfs.h diff --git a/include/uapi/linux/btrfs.h b/include/uapi/linux/btrfs.h
index a456e53..992c150 100644 index 9aa74f3..378230c 100644
--- a/include/uapi/linux/btrfs.h --- a/include/uapi/linux/btrfs.h
+++ b/include/uapi/linux/btrfs.h +++ b/include/uapi/linux/btrfs.h
@@ -255,13 +255,7 @@ struct btrfs_ioctl_fs_info_args { @@ -255,13 +255,7 @@ struct btrfs_ioctl_fs_info_args {

6
contrib/linux-kernel/0004-squashfs-Add-zstd-support.patch
View File

@ -1,7 +1,7 @@
From 5ff6a64abaea7b7f11d37cb0fdf08642316a3a90 Mon Sep 17 00:00:00 2001 From 0cd63464d182bb9708f8b25f7da3dc8e5ec6b4fa Mon Sep 17 00:00:00 2001
From: Nick Terrell <terrelln@fb.com> From: Nick Terrell <terrelln@fb.com>
Date: Mon, 12 Jun 2017 12:18:23 -0700 Date: Mon, 17 Jul 2017 17:08:59 -0700
Subject: [PATCH v2 4/4] squashfs: Add zstd support Subject: [PATCH v3 4/4] squashfs: Add zstd support
Add zstd compression and decompression support to SquashFS. zstd is a Add zstd compression and decompression support to SquashFS. zstd is a
great fit for SquashFS because it can compress at ratios approaching xz, great fit for SquashFS because it can compress at ratios approaching xz,

20
contrib/linux-kernel/fs/btrfs/zstd.c
View File

@ -29,10 +29,11 @@
#define ZSTD_BTRFS_MAX_WINDOWLOG 17 #define ZSTD_BTRFS_MAX_WINDOWLOG 17
#define ZSTD_BTRFS_MAX_INPUT (1 << ZSTD_BTRFS_MAX_WINDOWLOG) #define ZSTD_BTRFS_MAX_INPUT (1 << ZSTD_BTRFS_MAX_WINDOWLOG)
#define ZSTD_BTRFS_DEFAULT_LEVEL 3
static ZSTD_parameters zstd_get_btrfs_parameters(size_t src_len) static ZSTD_parameters zstd_get_btrfs_parameters(size_t src_len)
{ {
ZSTD_parameters params = ZSTD_getParams(1, src_len, 0); ZSTD_parameters params = ZSTD_getParams(ZSTD_BTRFS_DEFAULT_LEVEL, src_len, 0);
if (params.cParams.windowLog > ZSTD_BTRFS_MAX_WINDOWLOG) if (params.cParams.windowLog > ZSTD_BTRFS_MAX_WINDOWLOG)
params.cParams.windowLog = ZSTD_BTRFS_MAX_WINDOWLOG; params.cParams.windowLog = ZSTD_BTRFS_MAX_WINDOWLOG;
@ -51,7 +52,7 @@ static void zstd_free_workspace(struct list_head *ws)
{ {
struct workspace *workspace = list_entry(ws, struct workspace, list); struct workspace *workspace = list_entry(ws, struct workspace, list);
vfree(workspace->mem); kvfree(workspace->mem);
kfree(workspace->buf); kfree(workspace->buf);
kfree(workspace); kfree(workspace);
} }
@ -62,15 +63,15 @@ static struct list_head *zstd_alloc_workspace(void)
zstd_get_btrfs_parameters(ZSTD_BTRFS_MAX_INPUT); zstd_get_btrfs_parameters(ZSTD_BTRFS_MAX_INPUT);
struct workspace *workspace; struct workspace *workspace;
workspace = kzalloc(sizeof(*workspace), GFP_NOFS); workspace = kzalloc(sizeof(*workspace), GFP_KERNEL);
if (!workspace) if (!workspace)
return ERR_PTR(-ENOMEM); return ERR_PTR(-ENOMEM);
workspace->size = max_t(size_t, workspace->size = max_t(size_t,
ZSTD_CStreamWorkspaceBound(params.cParams), ZSTD_CStreamWorkspaceBound(params.cParams),
ZSTD_DStreamWorkspaceBound(ZSTD_BTRFS_MAX_INPUT)); ZSTD_DStreamWorkspaceBound(ZSTD_BTRFS_MAX_INPUT));
workspace->mem = vmalloc(workspace->size); workspace->mem = kvmalloc(workspace->size, GFP_KERNEL);
workspace->buf = kmalloc(PAGE_SIZE, GFP_NOFS); workspace->buf = kmalloc(PAGE_SIZE, GFP_KERNEL);
if (!workspace->mem || !workspace->buf) if (!workspace->mem || !workspace->buf)
goto fail; goto fail;
@ -262,12 +263,13 @@ out:
return ret; return ret;
} }
static int zstd_decompress_bio(struct list_head *ws, struct page **pages_in, static int zstd_decompress_bio(struct list_head *ws, struct compressed_bio *cb)
u64 disk_start,
struct bio *orig_bio,
size_t srclen)
{ {
struct workspace *workspace = list_entry(ws, struct workspace, list); struct workspace *workspace = list_entry(ws, struct workspace, list);
struct page **pages_in = cb->compressed_pages;
u64 disk_start = cb->start;
struct bio *orig_bio = cb->orig_bio;
size_t srclen = cb->compressed_len;
ZSTD_DStream *stream; ZSTD_DStream *stream;
int ret = 0; int ret = 0;
unsigned long page_in_index = 0; unsigned long page_in_index = 0;

3
contrib/linux-kernel/lib/zstd/huf_compress.c
View File

@ -525,7 +525,7 @@ static void HUF_encodeSymbol(BIT_CStream_t *bitCPtr, U32 symbol, const HUF_CElt
size_t HUF_compressBound(size_t size) { return HUF_COMPRESSBOUND(size); } size_t HUF_compressBound(size_t size) { return HUF_COMPRESSBOUND(size); }
#define HUF_FLUSHBITS(s) (fast ? BIT_flushBitsFast(s) : BIT_flushBits(s)) #define HUF_FLUSHBITS(s) BIT_flushBits(s)
#define HUF_FLUSHBITS_1(stream) \ #define HUF_FLUSHBITS_1(stream) \
if (sizeof((stream)->bitContainer) * 8 < HUF_TABLELOG_MAX * 2 + 7) \ if (sizeof((stream)->bitContainer) * 8 < HUF_TABLELOG_MAX * 2 + 7) \
@ -542,7 +542,6 @@ size_t HUF_compress1X_usingCTable(void *dst, size_t dstSize, const void *src, si
BYTE *const oend = ostart + dstSize; BYTE *const oend = ostart + dstSize;
BYTE *op = ostart; BYTE *op = ostart;
size_t n; size_t n;
const unsigned fast = (dstSize >= HUF_BLOCKBOUND(srcSize));
BIT_CStream_t bitC; BIT_CStream_t bitC;
/* init */ /* init */

27
contrib/linux-kernel/lib/zstd/zstd_internal.h
View File

@ -126,35 +126,16 @@ static const U32 OF_defaultNormLog = OF_DEFAULTNORMLOG;
/*-******************************************* /*-*******************************************
* Shared functions to include for inlining * Shared functions to include for inlining
*********************************************/ *********************************************/
static void ZSTD_copy8(void *dst, const void *src) { memcpy(dst, src, 8); } static void ZSTD_copy8(void *dst, const void *src) {
#define COPY8(d, s) \ memcpy(dst, src, 8);
{ \
ZSTD_copy8(d, s); \
d += 8; \
s += 8; \
} }
/*! ZSTD_wildcopy() : /*! ZSTD_wildcopy() :
* custom version of memcpy(), can copy up to 7 bytes too many (8 bytes if length==0) */ * custom version of memcpy(), can copy up to 7 bytes too many (8 bytes if length==0) */
#define WILDCOPY_OVERLENGTH 8 #define WILDCOPY_OVERLENGTH 8
ZSTD_STATIC void ZSTD_wildcopy(void *dst, const void *src, ptrdiff_t length) ZSTD_STATIC void ZSTD_wildcopy(void *dst, const void *src, ptrdiff_t length)
{ {
const BYTE *ip = (const BYTE *)src; if (length > 0)
BYTE *op = (BYTE *)dst; memcpy(dst, src, length);
BYTE *const oend = op + length;
do
COPY8(op, ip)
while (op < oend);
}
ZSTD_STATIC void ZSTD_wildcopy_e(void *dst, const void *src, void *dstEnd) /* should be faster for decoding, but strangely, not verified on all platform */
{
const BYTE *ip = (const BYTE *)src;
BYTE *op = (BYTE *)dst;
BYTE *const oend = (BYTE *)dstEnd;
do
COPY8(op, ip)
while (op < oend);
} }
/*-******************************************* /*-*******************************************