This patch introdueces the use of the Intel QAT to decompress compressed
data in the EROFS filesystem, aiming to improve the decompression speed
of compressed datea.
We created a 285MiB compressed file and then used the following command to
create EROFS images with different cluster size.
# mkfs.erofs -zdeflate,level=9 -C16384
fio command was used to test random read and small random read(~5%) and
sequential read performance.
# fio -filename=testfile -bs=4k -rw=read -name=job1
# fio -filename=testfile -bs=4k -rw=randread -name=job1
# fio -filename=testfile -bs=4k -rw=randread --io_size=14m -name=job1
Here are some performance numbers for reference:
Processors: Intel(R) Xeon(R) 6766E(144 core)
Memory: 521 GiB
|-----------------------------------------------------------------------------|
| | Cluster size | sequential read | randread | small randread(5%) |
|-----------|--------------|-----------------|-----------|--------------------|
| Intel QAT | 4096 | 538 MiB/s | 112 MiB/s | 20.76 MiB/s |
| Intel QAT | 16384 | 699 MiB/s | 158 MiB/s | 21.02 MiB/s |
| Intel QAT | 65536 | 917 MiB/s | 278 MiB/s | 20.90 MiB/s |
| Intel QAT | 131072 | 1056 MiB/s | 351 MiB/s | 23.36 MiB/s |
| Intel QAT | 262144 | 1145 MiB/s | 431 MiB/s | 26.66 MiB/s |
| deflate | 4096 | 499 MiB/s | 108 MiB/s | 21.50 MiB/s |
| deflate | 16384 | 422 MiB/s | 125 MiB/s | 18.94 MiB/s |
| deflate | 65536 | 452 MiB/s | 159 MiB/s | 13.02 MiB/s |
| deflate | 131072 | 452 MiB/s | 177 MiB/s | 11.44 MiB/s |
| deflate | 262144 | 466 MiB/s | 194 MiB/s | 10.60 MiB/s |
Signed-off-by: Bo Liu <[email protected]>
---
v1:
https://lore.kernel.org/linux-erofs/[email protected]/
Changes since v1:
- Move the code to the decompress_crypto.c file.
- Add a struct z_erofs_crypto_engine to maintain accelerator information.
- Add a sysfs interface to enable/disable the accelerator.
fs/erofs/Makefile | 2 +-
fs/erofs/compress.h | 13 ++
fs/erofs/decompressor_crypto.c | 219 ++++++++++++++++++++++++++++++++
fs/erofs/decompressor_deflate.c | 14 +-
fs/erofs/sysfs.c | 19 +++
5 files changed, 265 insertions(+), 2 deletions(-)
create mode 100644 fs/erofs/decompressor_crypto.c
diff --git a/fs/erofs/Makefile b/fs/erofs/Makefile
index 4331d53c7109..8462e16a8356 100644
--- a/fs/erofs/Makefile
+++ b/fs/erofs/Makefile
@@ -3,7 +3,7 @@
obj-$(CONFIG_EROFS_FS) += erofs.o
erofs-objs := super.o inode.o data.o namei.o dir.o sysfs.o
erofs-$(CONFIG_EROFS_FS_XATTR) += xattr.o
-erofs-$(CONFIG_EROFS_FS_ZIP) += decompressor.o zmap.o zdata.o zutil.o
+erofs-$(CONFIG_EROFS_FS_ZIP) += decompressor.o zmap.o zdata.o zutil.o
decompressor_crypto.o
erofs-$(CONFIG_EROFS_FS_ZIP_LZMA) += decompressor_lzma.o
erofs-$(CONFIG_EROFS_FS_ZIP_DEFLATE) += decompressor_deflate.o
erofs-$(CONFIG_EROFS_FS_ZIP_ZSTD) += decompressor_zstd.o
diff --git a/fs/erofs/compress.h b/fs/erofs/compress.h
index 2704d7a592a5..909fab195d93 100644
--- a/fs/erofs/compress.h
+++ b/fs/erofs/compress.h
@@ -70,10 +70,23 @@ struct z_erofs_stream_dctx {
bool bounced; /* is the bounce buffer used now? */
};
+struct z_erofs_crypto_engine {
+ char *crypto_name;
+ bool enabled;
+ struct crypto_acomp *erofs_tfm;
+};
+
+extern struct z_erofs_crypto_engine *z_erofs_crypto[];
+
int z_erofs_stream_switch_bufs(struct z_erofs_stream_dctx *dctx, void **dst,
void **src, struct page **pgpl);
int z_erofs_fixup_insize(struct z_erofs_decompress_req *rq, const char *padbuf,
unsigned int padbufsize);
int __init z_erofs_init_decompressor(void);
void z_erofs_exit_decompressor(void);
+int z_erofs_crypto_decompress(struct z_erofs_decompress_req *rq,
+ struct crypto_acomp *erofs_tfm,
struct page **pgpl);
+struct crypto_acomp *z_erofs_crypto_get_enbale_engine(int type);
+int z_erofs_crypto_enable_engine(const char *name);
+int z_erofs_crypto_disable_engine(const char *name);
#endif
diff --git a/fs/erofs/decompressor_crypto.c b/fs/erofs/decompressor_crypto.c
new file mode 100644
index 000000000000..500cff5e9b17
--- /dev/null
+++ b/fs/erofs/decompressor_crypto.c
@@ -0,0 +1,219 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+
+#include <linux/scatterlist.h>
+#include <crypto/acompress.h>
+
+#include "compress.h"
+
+static int z_erofs_crypto_decompress_mem(struct z_erofs_decompress_req *rq,
+ struct crypto_acomp *erofs_tfm)
+{
+ unsigned int nrpages_out = rq->outpages, nrpages_in = rq->inpages;
+ struct sg_table st_src, st_dst;
+ struct scatterlist *sg_src, *sg_dst;
+ struct acomp_req *req;
+ struct crypto_wait wait;
+ int ret, i;
+ u8 *headpage;
+
+
+ WARN_ON(!*rq->in);
+ headpage = kmap_local_page(*rq->in);
+ ret = z_erofs_fixup_insize(rq, headpage + rq->pageofs_in,
+ min_t(unsigned int, rq->inputsize,
+ rq->sb->s_blocksize -
rq->pageofs_in));
+ kunmap_local(headpage);
+ if (ret)
+ return ret;
+
+ req = acomp_request_alloc(erofs_tfm);
+ if (!req) {
+ erofs_err(rq->sb, "failed to alloc decompress request");
+ return -ENOMEM;
+ }
+
+ if (sg_alloc_table(&st_src, nrpages_in, GFP_KERNEL)) {
+ acomp_request_free(req);
+ return -ENOMEM;
+ }
+
+ if (sg_alloc_table(&st_dst, nrpages_out, GFP_KERNEL)) {
+ acomp_request_free(req);
+ sg_free_table(&st_src);
+ return -ENOMEM;
+ }
+
+ for_each_sg(st_src.sgl, sg_src, nrpages_in, i) {
+ if (i == 0)
+ sg_set_page(sg_src, rq->in[0],
+ PAGE_SIZE - rq->pageofs_in, rq->pageofs_in);
+ else if (i == nrpages_in - 1)
+ sg_set_page(sg_src, rq->in[i],
+ rq->pageofs_in + rq->inputsize - (nrpages_in -
1) * PAGE_SIZE, 0);
+ else
+ sg_set_page(sg_src, rq->in[i], PAGE_SIZE, 0);
+ }
+
+ i = 0;
+ for_each_sg(st_dst.sgl, sg_dst, nrpages_out, i) {
+ if (i == 0)
+ sg_set_page(sg_dst, rq->out[0],
+ PAGE_SIZE - rq->pageofs_out, rq->pageofs_out);
+ else if (i == nrpages_out)
+ sg_set_page(sg_dst, rq->out[i],
+ rq->pageofs_out + rq->outputsize - (nrpages_out
- 1) * PAGE_SIZE, 0);
+ else
+ sg_set_page(sg_dst, rq->out[i], PAGE_SIZE, 0);
+ }
+
+ acomp_request_set_params(req, st_src.sgl,
+ st_dst.sgl, rq->inputsize, rq->outputsize);
+
+ crypto_init_wait(&wait);
+ acomp_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
+ crypto_req_done, &wait);
+
+ ret = crypto_wait_req(crypto_acomp_decompress(req), &wait);
+ if (ret < 0) {
+ erofs_err(rq->sb, "failed to decompress %d in[%u, %u] out[%u]",
+ ret, rq->inputsize, rq->pageofs_in,
rq->outputsize);
+ ret = -EIO;
+ } else
+ ret = 0;
+
+ acomp_request_free(req);
+ sg_free_table(&st_src);
+ sg_free_table(&st_dst);
+ return ret;
+}
+
+int z_erofs_crypto_decompress(struct z_erofs_decompress_req *rq,
+ struct crypto_acomp *erofs_tfm, struct page
**pgpl)
+{
+ const unsigned int nrpages_out =
+ PAGE_ALIGN(rq->pageofs_out + rq->outputsize) >> PAGE_SHIFT;
+ int i;
+
+ /* one optimized fast path only for non bigpcluster cases yet */
+ if (rq->inputsize <= PAGE_SIZE && nrpages_out == 1 && !rq->inplace_io) {
+ WARN_ON(!*rq->out);
+ goto dstmap_out;
+ }
+
+ for (i = 0; i < rq->outpages; i++) {
+ struct page *const page = rq->out[i];
+ struct page *victim;
+
+ if (!page) {
+ victim = __erofs_allocpage(pgpl, rq->gfp, true);
+ if (!victim)
+ return -ENOMEM;
+ set_page_private(victim, Z_EROFS_SHORTLIVED_PAGE);
+ rq->out[i] = victim;
+ }
+ }
+
+dstmap_out:
+ return z_erofs_crypto_decompress_mem(rq, erofs_tfm);
+}
+
+struct crypto_acomp *z_erofs_crypto_get_enbale_engine(int type)
+{
+ int i = 0;
+
+ while (z_erofs_crypto[type][i].crypto_name) {
+ if (z_erofs_crypto[type][i].enabled)
+ return z_erofs_crypto[type][i].erofs_tfm;
+ i++;
+ }
+
+ return NULL;
+}
+
+static int z_erofs_crypto_get_compress_type(const char *name)
+{
+ int i, j;
+
+ for (i = 0; i < Z_EROFS_COMPRESSION_MAX; i++) {
+ j = 0;
+ while (z_erofs_crypto[i][j].crypto_name) {
+ if (!strncmp(name, z_erofs_crypto[i][j].crypto_name,
+
strlen(z_erofs_crypto[i][j].crypto_name))) {
+ return i;
+ }
+ j++;
+ }
+ }
+
+ return -EINVAL;
+}
+
+int z_erofs_crypto_enable_engine(const char *name)
+{
+ int i = 0, type;
+
+ type = z_erofs_crypto_get_compress_type(name);
+ if (type < 0)
+ return -EINVAL;
+
+ while (z_erofs_crypto[type][i].crypto_name) {
+ if (!strncmp(z_erofs_crypto[type][i].crypto_name, name,
+ strlen(z_erofs_crypto[type][i].crypto_name))) {
+ if (z_erofs_crypto[type][i].enabled)
+ break;
+
+ z_erofs_crypto[type][i].erofs_tfm =
+
crypto_alloc_acomp(z_erofs_crypto[type][i].crypto_name, 0, 0);
+ if (IS_ERR(z_erofs_crypto[type][i].erofs_tfm)) {
+ z_erofs_crypto[type][i].erofs_tfm = NULL;
+ break;
+ }
+ z_erofs_crypto[type][i].enabled = true;
+ } else if (z_erofs_crypto[type][i].enabled) {
+ if (z_erofs_crypto[type][i].erofs_tfm)
+
crypto_free_acomp(z_erofs_crypto[type][i].erofs_tfm);
+ z_erofs_crypto[type][i].enabled = false;
+ }
+ i++;
+ }
+
+ return 0;
+}
+
+int z_erofs_crypto_disable_engine(const char *name)
+{
+ int i = 0, type;
+
+ type = z_erofs_crypto_get_compress_type(name);
+ if (type < 0)
+ return -EINVAL;
+
+ while (z_erofs_crypto[type][i].crypto_name) {
+ if (!strncmp(z_erofs_crypto[type][i].crypto_name, name,
+ strlen(z_erofs_crypto[type][i].crypto_name))) {
+ if (z_erofs_crypto[type][i].enabled &&
+ z_erofs_crypto[type][i].erofs_tfm) {
+
crypto_free_acomp(z_erofs_crypto[type][i].erofs_tfm);
+ z_erofs_crypto[type][i].erofs_tfm = NULL;
+ z_erofs_crypto[type][i].enabled = false;
+ }
+ }
+ i++;
+ }
+
+ return 0;
+
+}
+
+struct z_erofs_crypto_engine *z_erofs_crypto[] = {
+ [Z_EROFS_COMPRESSION_LZ4] = &(struct z_erofs_crypto_engine) {NULL},
+ [Z_EROFS_COMPRESSION_LZMA] = &(struct z_erofs_crypto_engine) {NULL},
+ [Z_EROFS_COMPRESSION_DEFLATE] = {&(struct z_erofs_crypto_engine) {
+ .crypto_name = "qat_deflate",
+ .enabled = false,
+ .erofs_tfm = NULL,
+ },
+ &(const struct z_erofs_crypto_engine) { NULL },
+ },
+ [Z_EROFS_COMPRESSION_ZSTD] = &(struct z_erofs_crypto_engine) {NULL},
+};
diff --git a/fs/erofs/decompressor_deflate.c b/fs/erofs/decompressor_deflate.c
index c6908a487054..35a5889880f4 100644
--- a/fs/erofs/decompressor_deflate.c
+++ b/fs/erofs/decompressor_deflate.c
@@ -97,7 +97,7 @@ static int z_erofs_load_deflate_config(struct super_block *sb,
return -ENOMEM;
}
-static int z_erofs_deflate_decompress(struct z_erofs_decompress_req *rq,
+static int __z_erofs_deflate_decompress(struct z_erofs_decompress_req *rq,
struct page **pgpl)
{
struct super_block *sb = rq->sb;
@@ -178,6 +178,18 @@ static int z_erofs_deflate_decompress(struct
z_erofs_decompress_req *rq,
return err;
}
+static int z_erofs_deflate_decompress(struct z_erofs_decompress_req *rq,
+ struct page **pgpl)
+{
+ struct crypto_acomp *erofs_tfm = NULL;
+
+ erofs_tfm =
z_erofs_crypto_get_enbale_engine(Z_EROFS_COMPRESSION_DEFLATE);
+ if (erofs_tfm && !rq->partial_decoding)
+ return z_erofs_crypto_decompress(rq, erofs_tfm, pgpl);
+ else
+ return __z_erofs_deflate_decompress(rq, pgpl);
+}
+
const struct z_erofs_decompressor z_erofs_deflate_decomp = {
.config = z_erofs_load_deflate_config,
.decompress = z_erofs_deflate_decompress,
diff --git a/fs/erofs/sysfs.c b/fs/erofs/sysfs.c
index dad4e6c6c155..a9c0aad01264 100644
--- a/fs/erofs/sysfs.c
+++ b/fs/erofs/sysfs.c
@@ -7,12 +7,15 @@
#include <linux/kobject.h>
#include "internal.h"
+#include "compress.h"
enum {
attr_feature,
attr_drop_caches,
attr_pointer_ui,
attr_pointer_bool,
+ attr_crypto_enable,
+ attr_crypto_disable,
};
enum {
@@ -59,6 +62,8 @@ static struct erofs_attr erofs_attr_##_name = {
\
#ifdef CONFIG_EROFS_FS_ZIP
EROFS_ATTR_RW_UI(sync_decompress, erofs_mount_opts);
EROFS_ATTR_FUNC(drop_caches, 0200);
+EROFS_ATTR_FUNC(crypto_enable, 0644);
+EROFS_ATTR_FUNC(crypto_disable, 0644);
#endif
static struct attribute *erofs_attrs[] = {
@@ -95,6 +100,8 @@ static struct attribute *erofs_feat_attrs[] = {
ATTR_LIST(fragments),
ATTR_LIST(dedupe),
ATTR_LIST(48bit),
+ ATTR_LIST(crypto_enable),
+ ATTR_LIST(crypto_disable),
NULL,
};
ATTRIBUTE_GROUPS(erofs_feat);
@@ -181,6 +188,18 @@ static ssize_t erofs_attr_store(struct kobject *kobj,
struct attribute *attr,
if (t & 1)
invalidate_mapping_pages(MNGD_MAPPING(sbi), 0, -1);
return len;
+ case attr_crypto_enable:
+ buf = skip_spaces(buf);
+ if (z_erofs_crypto_enable_engine(buf))
+ return -EINVAL;
+
+ return len;
+ case attr_crypto_disable:
+ buf = skip_spaces(buf);
+ if (z_erofs_crypto_disable_engine(buf))
+ return -EINVAL;
+
+ return len;
#endif
}
return 0;
--
2.31.1
