307 lines
9.3 KiB
Diff
307 lines
9.3 KiB
Diff
From 46bf8f6d30d6ddf2446c110f122482b5e5e16933 Mon Sep 17 00:00:00 2001
|
|
From: Sean Purcell <me@seanp.xyz>
|
|
Date: Mon, 17 Jul 2017 17:08:59 -0700
|
|
Subject: [PATCH v5 4/5] squashfs: Add zstd support
|
|
|
|
Add zstd compression and decompression support to SquashFS. zstd is a
|
|
great fit for SquashFS because it can compress at ratios approaching xz,
|
|
while decompressing twice as fast as zlib. For SquashFS in particular,
|
|
it can decompress as fast as lzo and lz4. It also has the flexibility
|
|
to turn down the compression ratio for faster compression times.
|
|
|
|
The compression benchmark is run on the file tree from the SquashFS archive
|
|
found in ubuntu-16.10-desktop-amd64.iso [1]. It uses `mksquashfs` with the
|
|
default block size (128 KB) and and various compression algorithms/levels.
|
|
xz and zstd are also benchmarked with 256 KB blocks. The decompression
|
|
benchmark times how long it takes to `tar` the file tree into `/dev/null`.
|
|
See the benchmark file in the upstream zstd source repository located under
|
|
`contrib/linux-kernel/squashfs-benchmark.sh` [2] for details.
|
|
|
|
I ran the benchmarks on a Ubuntu 14.04 VM with 2 cores and 4 GiB of RAM.
|
|
The VM is running on a MacBook Pro with a 3.1 GHz Intel Core i7 processor,
|
|
16 GB of RAM, and a SSD.
|
|
|
|
| Method | Ratio | Compression MB/s | Decompression MB/s |
|
|
|----------------|-------|------------------|--------------------|
|
|
| gzip | 2.92 | 15 | 128 |
|
|
| lzo | 2.64 | 9.5 | 217 |
|
|
| lz4 | 2.12 | 94 | 218 |
|
|
| xz | 3.43 | 5.5 | 35 |
|
|
| xz 256 KB | 3.53 | 5.4 | 40 |
|
|
| zstd 1 | 2.71 | 96 | 210 |
|
|
| zstd 5 | 2.93 | 69 | 198 |
|
|
| zstd 10 | 3.01 | 41 | 225 |
|
|
| zstd 15 | 3.13 | 11.4 | 224 |
|
|
| zstd 16 256 KB | 3.24 | 8.1 | 210 |
|
|
|
|
This patch was written by Sean Purcell <me@seanp.xyz>, but I will be
|
|
taking over the submission process.
|
|
|
|
[1] http://releases.ubuntu.com/16.10/
|
|
[2] https://github.com/facebook/zstd/blob/dev/contrib/linux-kernel/squashfs-benchmark.sh
|
|
|
|
zstd source repository: https://github.com/facebook/zstd
|
|
|
|
Signed-off-by: Sean Purcell <me@seanp.xyz>
|
|
Signed-off-by: Nick Terrell <terrelln@fb.com>
|
|
---
|
|
v3 -> v4:
|
|
- Fix minor linter warnings
|
|
|
|
v4 -> v5:
|
|
- Fix ZSTD_DStream initialization code in squashfs
|
|
- Fix patch documentation to reflect that Sean Purcell is the author
|
|
|
|
fs/squashfs/Kconfig | 14 +++++
|
|
fs/squashfs/Makefile | 1 +
|
|
fs/squashfs/decompressor.c | 7 +++
|
|
fs/squashfs/decompressor.h | 4 ++
|
|
fs/squashfs/squashfs_fs.h | 1 +
|
|
fs/squashfs/zstd_wrapper.c | 151 +++++++++++++++++++++++++++++++++++++++++++++
|
|
6 files changed, 178 insertions(+)
|
|
create mode 100644 fs/squashfs/zstd_wrapper.c
|
|
|
|
diff --git a/fs/squashfs/Kconfig b/fs/squashfs/Kconfig
|
|
index ffb093e..1adb334 100644
|
|
--- a/fs/squashfs/Kconfig
|
|
+++ b/fs/squashfs/Kconfig
|
|
@@ -165,6 +165,20 @@ config SQUASHFS_XZ
|
|
|
|
If unsure, say N.
|
|
|
|
+config SQUASHFS_ZSTD
|
|
+ bool "Include support for ZSTD compressed file systems"
|
|
+ depends on SQUASHFS
|
|
+ select ZSTD_DECOMPRESS
|
|
+ help
|
|
+ Saying Y here includes support for reading Squashfs file systems
|
|
+ compressed with ZSTD compression. ZSTD gives better compression than
|
|
+ the default ZLIB compression, while using less CPU.
|
|
+
|
|
+ ZSTD is not the standard compression used in Squashfs and so most
|
|
+ file systems will be readable without selecting this option.
|
|
+
|
|
+ If unsure, say N.
|
|
+
|
|
config SQUASHFS_4K_DEVBLK_SIZE
|
|
bool "Use 4K device block size?"
|
|
depends on SQUASHFS
|
|
diff --git a/fs/squashfs/Makefile b/fs/squashfs/Makefile
|
|
index 246a6f3..6655631 100644
|
|
--- a/fs/squashfs/Makefile
|
|
+++ b/fs/squashfs/Makefile
|
|
@@ -15,3 +15,4 @@ squashfs-$(CONFIG_SQUASHFS_LZ4) += lz4_wrapper.o
|
|
squashfs-$(CONFIG_SQUASHFS_LZO) += lzo_wrapper.o
|
|
squashfs-$(CONFIG_SQUASHFS_XZ) += xz_wrapper.o
|
|
squashfs-$(CONFIG_SQUASHFS_ZLIB) += zlib_wrapper.o
|
|
+squashfs-$(CONFIG_SQUASHFS_ZSTD) += zstd_wrapper.o
|
|
diff --git a/fs/squashfs/decompressor.c b/fs/squashfs/decompressor.c
|
|
index d2bc136..8366398 100644
|
|
--- a/fs/squashfs/decompressor.c
|
|
+++ b/fs/squashfs/decompressor.c
|
|
@@ -65,6 +65,12 @@ static const struct squashfs_decompressor squashfs_zlib_comp_ops = {
|
|
};
|
|
#endif
|
|
|
|
+#ifndef CONFIG_SQUASHFS_ZSTD
|
|
+static const struct squashfs_decompressor squashfs_zstd_comp_ops = {
|
|
+ NULL, NULL, NULL, NULL, ZSTD_COMPRESSION, "zstd", 0
|
|
+};
|
|
+#endif
|
|
+
|
|
static const struct squashfs_decompressor squashfs_unknown_comp_ops = {
|
|
NULL, NULL, NULL, NULL, 0, "unknown", 0
|
|
};
|
|
@@ -75,6 +81,7 @@ static const struct squashfs_decompressor *decompressor[] = {
|
|
&squashfs_lzo_comp_ops,
|
|
&squashfs_xz_comp_ops,
|
|
&squashfs_lzma_unsupported_comp_ops,
|
|
+ &squashfs_zstd_comp_ops,
|
|
&squashfs_unknown_comp_ops
|
|
};
|
|
|
|
diff --git a/fs/squashfs/decompressor.h b/fs/squashfs/decompressor.h
|
|
index a25713c..0f5a8e4 100644
|
|
--- a/fs/squashfs/decompressor.h
|
|
+++ b/fs/squashfs/decompressor.h
|
|
@@ -58,4 +58,8 @@ extern const struct squashfs_decompressor squashfs_lzo_comp_ops;
|
|
extern const struct squashfs_decompressor squashfs_zlib_comp_ops;
|
|
#endif
|
|
|
|
+#ifdef CONFIG_SQUASHFS_ZSTD
|
|
+extern const struct squashfs_decompressor squashfs_zstd_comp_ops;
|
|
+#endif
|
|
+
|
|
#endif
|
|
diff --git a/fs/squashfs/squashfs_fs.h b/fs/squashfs/squashfs_fs.h
|
|
index 506f4ba..24d12fd 100644
|
|
--- a/fs/squashfs/squashfs_fs.h
|
|
+++ b/fs/squashfs/squashfs_fs.h
|
|
@@ -241,6 +241,7 @@ struct meta_index {
|
|
#define LZO_COMPRESSION 3
|
|
#define XZ_COMPRESSION 4
|
|
#define LZ4_COMPRESSION 5
|
|
+#define ZSTD_COMPRESSION 6
|
|
|
|
struct squashfs_super_block {
|
|
__le32 s_magic;
|
|
diff --git a/fs/squashfs/zstd_wrapper.c b/fs/squashfs/zstd_wrapper.c
|
|
new file mode 100644
|
|
index 0000000..eeaabf8
|
|
--- /dev/null
|
|
+++ b/fs/squashfs/zstd_wrapper.c
|
|
@@ -0,0 +1,151 @@
|
|
+/*
|
|
+ * Squashfs - a compressed read only filesystem for Linux
|
|
+ *
|
|
+ * Copyright (c) 2016-present, Facebook, Inc.
|
|
+ * All rights reserved.
|
|
+ *
|
|
+ * This program is free software; you can redistribute it and/or
|
|
+ * modify it under the terms of the GNU General Public License
|
|
+ * as published by the Free Software Foundation; either version 2,
|
|
+ * or (at your option) any later version.
|
|
+ *
|
|
+ * This program is distributed in the hope that it will be useful,
|
|
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
+ * GNU General Public License for more details.
|
|
+ *
|
|
+ * zstd_wrapper.c
|
|
+ */
|
|
+
|
|
+#include <linux/mutex.h>
|
|
+#include <linux/buffer_head.h>
|
|
+#include <linux/slab.h>
|
|
+#include <linux/zstd.h>
|
|
+#include <linux/vmalloc.h>
|
|
+
|
|
+#include "squashfs_fs.h"
|
|
+#include "squashfs_fs_sb.h"
|
|
+#include "squashfs.h"
|
|
+#include "decompressor.h"
|
|
+#include "page_actor.h"
|
|
+
|
|
+struct workspace {
|
|
+ void *mem;
|
|
+ size_t mem_size;
|
|
+ size_t window_size;
|
|
+};
|
|
+
|
|
+static void *zstd_init(struct squashfs_sb_info *msblk, void *buff)
|
|
+{
|
|
+ struct workspace *wksp = kmalloc(sizeof(*wksp), GFP_KERNEL);
|
|
+
|
|
+ if (wksp == NULL)
|
|
+ goto failed;
|
|
+ wksp->window_size = max_t(size_t,
|
|
+ msblk->block_size, SQUASHFS_METADATA_SIZE);
|
|
+ wksp->mem_size = ZSTD_DStreamWorkspaceBound(wksp->window_size);
|
|
+ wksp->mem = vmalloc(wksp->mem_size);
|
|
+ if (wksp->mem == NULL)
|
|
+ goto failed;
|
|
+
|
|
+ return wksp;
|
|
+
|
|
+failed:
|
|
+ ERROR("Failed to allocate zstd workspace\n");
|
|
+ kfree(wksp);
|
|
+ return ERR_PTR(-ENOMEM);
|
|
+}
|
|
+
|
|
+
|
|
+static void zstd_free(void *strm)
|
|
+{
|
|
+ struct workspace *wksp = strm;
|
|
+
|
|
+ if (wksp)
|
|
+ vfree(wksp->mem);
|
|
+ kfree(wksp);
|
|
+}
|
|
+
|
|
+
|
|
+static int zstd_uncompress(struct squashfs_sb_info *msblk, void *strm,
|
|
+ struct buffer_head **bh, int b, int offset, int length,
|
|
+ struct squashfs_page_actor *output)
|
|
+{
|
|
+ struct workspace *wksp = strm;
|
|
+ ZSTD_DStream *stream;
|
|
+ size_t total_out = 0;
|
|
+ size_t zstd_err;
|
|
+ int k = 0;
|
|
+ ZSTD_inBuffer in_buf = { NULL, 0, 0 };
|
|
+ ZSTD_outBuffer out_buf = { NULL, 0, 0 };
|
|
+
|
|
+ stream = ZSTD_initDStream(wksp->window_size, wksp->mem, wksp->mem_size);
|
|
+
|
|
+ if (!stream) {
|
|
+ ERROR("Failed to initialize zstd decompressor\n");
|
|
+ goto out;
|
|
+ }
|
|
+
|
|
+ out_buf.size = PAGE_SIZE;
|
|
+ out_buf.dst = squashfs_first_page(output);
|
|
+
|
|
+ do {
|
|
+ if (in_buf.pos == in_buf.size && k < b) {
|
|
+ int avail = min(length, msblk->devblksize - offset);
|
|
+
|
|
+ length -= avail;
|
|
+ in_buf.src = bh[k]->b_data + offset;
|
|
+ in_buf.size = avail;
|
|
+ in_buf.pos = 0;
|
|
+ offset = 0;
|
|
+ }
|
|
+
|
|
+ if (out_buf.pos == out_buf.size) {
|
|
+ out_buf.dst = squashfs_next_page(output);
|
|
+ if (out_buf.dst == NULL) {
|
|
+ /* Shouldn't run out of pages
|
|
+ * before stream is done.
|
|
+ */
|
|
+ squashfs_finish_page(output);
|
|
+ goto out;
|
|
+ }
|
|
+ out_buf.pos = 0;
|
|
+ out_buf.size = PAGE_SIZE;
|
|
+ }
|
|
+
|
|
+ total_out -= out_buf.pos;
|
|
+ zstd_err = ZSTD_decompressStream(stream, &out_buf, &in_buf);
|
|
+ total_out += out_buf.pos; /* add the additional data produced */
|
|
+
|
|
+ if (in_buf.pos == in_buf.size && k < b)
|
|
+ put_bh(bh[k++]);
|
|
+ } while (zstd_err != 0 && !ZSTD_isError(zstd_err));
|
|
+
|
|
+ squashfs_finish_page(output);
|
|
+
|
|
+ if (ZSTD_isError(zstd_err)) {
|
|
+ ERROR("zstd decompression error: %d\n",
|
|
+ (int)ZSTD_getErrorCode(zstd_err));
|
|
+ goto out;
|
|
+ }
|
|
+
|
|
+ if (k < b)
|
|
+ goto out;
|
|
+
|
|
+ return (int)total_out;
|
|
+
|
|
+out:
|
|
+ for (; k < b; k++)
|
|
+ put_bh(bh[k]);
|
|
+
|
|
+ return -EIO;
|
|
+}
|
|
+
|
|
+const struct squashfs_decompressor squashfs_zstd_comp_ops = {
|
|
+ .init = zstd_init,
|
|
+ .free = zstd_free,
|
|
+ .decompress = zstd_uncompress,
|
|
+ .id = ZSTD_COMPRESSION,
|
|
+ .name = "zstd",
|
|
+ .supported = 1
|
|
+};
|
|
--
|
|
2.9.3
|