Commit 5c1aab1d authored by Nick Terrell's avatar Nick Terrell Committed by Chris Mason

btrfs: Add zstd support

Add zstd compression and decompression support to BtrFS. zstd at its
fastest level compresses almost as well as zlib, while offering much
faster compression and decompression, approaching lzo speeds.

I benchmarked btrfs with zstd compression against no compression, lzo
compression, and zlib compression. I benchmarked two scenarios. Copying
a set of files to btrfs, and then reading the files. Copying a tarball
to btrfs, extracting it to btrfs, and then reading the extracted files.
After every operation, I call `sync` and include the sync time.
Between every pair of operations I unmount and remount the filesystem
to avoid caching. The benchmark files can be found in the upstream
zstd source repository under
`contrib/linux-kernel/{btrfs-benchmark.sh,btrfs-extract-benchmark.sh}`
[1] [2].

I ran the benchmarks on a Ubuntu 14.04 VM with 2 cores and 4 GiB of RAM.
The VM is running on a MacBook Pro with a 3.1 GHz Intel Core i7 processor,
16 GB of RAM, and a SSD.

The first compression benchmark is copying 10 copies of the unzipped
Silesia corpus [3] into a BtrFS filesystem mounted with
`-o compress-force=Method`. The decompression benchmark times how long
it takes to `tar` all 10 copies into `/dev/null`. The compression ratio is
measured by comparing the output of `df` and `du`. See the benchmark file
[1] for details. I benchmarked multiple zstd compression levels, although
the patch uses zstd level 1.

| Method  | Ratio | Compression MB/s | Decompression speed |
|---------|-------|------------------|---------------------|
| None    |  0.99 |              504 |                 686 |
| lzo     |  1.66 |              398 |                 442 |
| zlib    |  2.58 |               65 |                 241 |
| zstd 1  |  2.57 |              260 |                 383 |
| zstd 3  |  2.71 |              174 |                 408 |
| zstd 6  |  2.87 |               70 |                 398 |
| zstd 9  |  2.92 |               43 |                 406 |
| zstd 12 |  2.93 |               21 |                 408 |
| zstd 15 |  3.01 |               11 |                 354 |

The next benchmark first copies `linux-4.11.6.tar` [4] to btrfs. Then it
measures the compression ratio, extracts the tar, and deletes the tar.
Then it measures the compression ratio again, and `tar`s the extracted
files into `/dev/null`. See the benchmark file [2] for details.

| Method | Tar Ratio | Extract Ratio | Copy (s) | Extract (s)| Read (s) |
|--------|-----------|---------------|----------|------------|----------|
| None   |      0.97 |          0.78 |    0.981 |      5.501 |    8.807 |
| lzo    |      2.06 |          1.38 |    1.631 |      8.458 |    8.585 |
| zlib   |      3.40 |          1.86 |    7.750 |     21.544 |   11.744 |
| zstd 1 |      3.57 |          1.85 |    2.579 |     11.479 |    9.389 |

[1] https://github.com/facebook/zstd/blob/dev/contrib/linux-kernel/btrfs-benchmark.sh
[2] https://github.com/facebook/zstd/blob/dev/contrib/linux-kernel/btrfs-extract-benchmark.sh
[3] http://sun.aei.polsl.pl/~sdeor/index.php?page=silesia
[4] https://cdn.kernel.org/pub/linux/kernel/v4.x/linux-4.11.6.tar.xz

zstd source repository: https://github.com/facebook/zstdSigned-off-by: default avatarNick Terrell <terrelln@fb.com>
Signed-off-by: default avatarChris Mason <clm@fb.com>
parent 73f3d1b4
...@@ -6,6 +6,8 @@ config BTRFS_FS ...@@ -6,6 +6,8 @@ config BTRFS_FS
select ZLIB_DEFLATE select ZLIB_DEFLATE
select LZO_COMPRESS select LZO_COMPRESS
select LZO_DECOMPRESS select LZO_DECOMPRESS
select ZSTD_COMPRESS
select ZSTD_DECOMPRESS
select RAID6_PQ select RAID6_PQ
select XOR_BLOCKS select XOR_BLOCKS
select SRCU select SRCU
......
...@@ -6,7 +6,7 @@ btrfs-y += super.o ctree.o extent-tree.o print-tree.o root-tree.o dir-item.o \ ...@@ -6,7 +6,7 @@ btrfs-y += super.o ctree.o extent-tree.o print-tree.o root-tree.o dir-item.o \
transaction.o inode.o file.o tree-defrag.o \ transaction.o inode.o file.o tree-defrag.o \
extent_map.o sysfs.o struct-funcs.o xattr.o ordered-data.o \ extent_map.o sysfs.o struct-funcs.o xattr.o ordered-data.o \
extent_io.o volumes.o async-thread.o ioctl.o locking.o orphan.o \ extent_io.o volumes.o async-thread.o ioctl.o locking.o orphan.o \
export.o tree-log.o free-space-cache.o zlib.o lzo.o \ export.o tree-log.o free-space-cache.o zlib.o lzo.o zstd.o \
compression.o delayed-ref.o relocation.o delayed-inode.o scrub.o \ compression.o delayed-ref.o relocation.o delayed-inode.o scrub.o \
reada.o backref.o ulist.o qgroup.o send.o dev-replace.o raid56.o \ reada.o backref.o ulist.o qgroup.o send.o dev-replace.o raid56.o \
uuid-tree.o props.o hash.o free-space-tree.o uuid-tree.o props.o hash.o free-space-tree.o
......
...@@ -704,6 +704,7 @@ static struct { ...@@ -704,6 +704,7 @@ static struct {
static const struct btrfs_compress_op * const btrfs_compress_op[] = { static const struct btrfs_compress_op * const btrfs_compress_op[] = {
&btrfs_zlib_compress, &btrfs_zlib_compress,
&btrfs_lzo_compress, &btrfs_lzo_compress,
&btrfs_zstd_compress,
}; };
void __init btrfs_init_compress(void) void __init btrfs_init_compress(void)
......
...@@ -99,8 +99,9 @@ enum btrfs_compression_type { ...@@ -99,8 +99,9 @@ enum btrfs_compression_type {
BTRFS_COMPRESS_NONE = 0, BTRFS_COMPRESS_NONE = 0,
BTRFS_COMPRESS_ZLIB = 1, BTRFS_COMPRESS_ZLIB = 1,
BTRFS_COMPRESS_LZO = 2, BTRFS_COMPRESS_LZO = 2,
BTRFS_COMPRESS_TYPES = 2, BTRFS_COMPRESS_ZSTD = 3,
BTRFS_COMPRESS_LAST = 3, BTRFS_COMPRESS_TYPES = 3,
BTRFS_COMPRESS_LAST = 4,
}; };
struct btrfs_compress_op { struct btrfs_compress_op {
...@@ -128,5 +129,6 @@ struct btrfs_compress_op { ...@@ -128,5 +129,6 @@ struct btrfs_compress_op {
extern const struct btrfs_compress_op btrfs_zlib_compress; extern const struct btrfs_compress_op btrfs_zlib_compress;
extern const struct btrfs_compress_op btrfs_lzo_compress; extern const struct btrfs_compress_op btrfs_lzo_compress;
extern const struct btrfs_compress_op btrfs_zstd_compress;
#endif #endif
...@@ -270,6 +270,7 @@ struct btrfs_super_block { ...@@ -270,6 +270,7 @@ struct btrfs_super_block {
BTRFS_FEATURE_INCOMPAT_MIXED_GROUPS | \ BTRFS_FEATURE_INCOMPAT_MIXED_GROUPS | \
BTRFS_FEATURE_INCOMPAT_BIG_METADATA | \ BTRFS_FEATURE_INCOMPAT_BIG_METADATA | \
BTRFS_FEATURE_INCOMPAT_COMPRESS_LZO | \ BTRFS_FEATURE_INCOMPAT_COMPRESS_LZO | \
BTRFS_FEATURE_INCOMPAT_COMPRESS_ZSTD | \
BTRFS_FEATURE_INCOMPAT_RAID56 | \ BTRFS_FEATURE_INCOMPAT_RAID56 | \
BTRFS_FEATURE_INCOMPAT_EXTENDED_IREF | \ BTRFS_FEATURE_INCOMPAT_EXTENDED_IREF | \
BTRFS_FEATURE_INCOMPAT_SKINNY_METADATA | \ BTRFS_FEATURE_INCOMPAT_SKINNY_METADATA | \
......
...@@ -2828,6 +2828,8 @@ int open_ctree(struct super_block *sb, ...@@ -2828,6 +2828,8 @@ int open_ctree(struct super_block *sb,
features |= BTRFS_FEATURE_INCOMPAT_MIXED_BACKREF; features |= BTRFS_FEATURE_INCOMPAT_MIXED_BACKREF;
if (fs_info->compress_type == BTRFS_COMPRESS_LZO) if (fs_info->compress_type == BTRFS_COMPRESS_LZO)
features |= BTRFS_FEATURE_INCOMPAT_COMPRESS_LZO; features |= BTRFS_FEATURE_INCOMPAT_COMPRESS_LZO;
else if (fs_info->compress_type == BTRFS_COMPRESS_ZSTD)
features |= BTRFS_FEATURE_INCOMPAT_COMPRESS_ZSTD;
if (features & BTRFS_FEATURE_INCOMPAT_SKINNY_METADATA) if (features & BTRFS_FEATURE_INCOMPAT_SKINNY_METADATA)
btrfs_info(fs_info, "has skinny extents"); btrfs_info(fs_info, "has skinny extents");
......
...@@ -327,8 +327,10 @@ static int btrfs_ioctl_setflags(struct file *file, void __user *arg) ...@@ -327,8 +327,10 @@ static int btrfs_ioctl_setflags(struct file *file, void __user *arg)
if (fs_info->compress_type == BTRFS_COMPRESS_LZO) if (fs_info->compress_type == BTRFS_COMPRESS_LZO)
comp = "lzo"; comp = "lzo";
else else if (fs_info->compress_type == BTRFS_COMPRESS_ZLIB)
comp = "zlib"; comp = "zlib";
else
comp = "zstd";
ret = btrfs_set_prop(inode, "btrfs.compression", ret = btrfs_set_prop(inode, "btrfs.compression",
comp, strlen(comp), 0); comp, strlen(comp), 0);
if (ret) if (ret)
...@@ -1466,6 +1468,8 @@ int btrfs_defrag_file(struct inode *inode, struct file *file, ...@@ -1466,6 +1468,8 @@ int btrfs_defrag_file(struct inode *inode, struct file *file,
if (range->compress_type == BTRFS_COMPRESS_LZO) { if (range->compress_type == BTRFS_COMPRESS_LZO) {
btrfs_set_fs_incompat(fs_info, COMPRESS_LZO); btrfs_set_fs_incompat(fs_info, COMPRESS_LZO);
} else if (range->compress_type == BTRFS_COMPRESS_ZSTD) {
btrfs_set_fs_incompat(fs_info, COMPRESS_ZSTD);
} }
ret = defrag_count; ret = defrag_count;
......
...@@ -390,6 +390,8 @@ static int prop_compression_validate(const char *value, size_t len) ...@@ -390,6 +390,8 @@ static int prop_compression_validate(const char *value, size_t len)
return 0; return 0;
else if (!strncmp("zlib", value, len)) else if (!strncmp("zlib", value, len))
return 0; return 0;
else if (!strncmp("zstd", value, len))
return 0;
return -EINVAL; return -EINVAL;
} }
...@@ -412,6 +414,8 @@ static int prop_compression_apply(struct inode *inode, ...@@ -412,6 +414,8 @@ static int prop_compression_apply(struct inode *inode,
type = BTRFS_COMPRESS_LZO; type = BTRFS_COMPRESS_LZO;
else if (!strncmp("zlib", value, len)) else if (!strncmp("zlib", value, len))
type = BTRFS_COMPRESS_ZLIB; type = BTRFS_COMPRESS_ZLIB;
else if (!strncmp("zstd", value, len))
type = BTRFS_COMPRESS_ZSTD;
else else
return -EINVAL; return -EINVAL;
...@@ -429,6 +433,8 @@ static const char *prop_compression_extract(struct inode *inode) ...@@ -429,6 +433,8 @@ static const char *prop_compression_extract(struct inode *inode)
return "zlib"; return "zlib";
case BTRFS_COMPRESS_LZO: case BTRFS_COMPRESS_LZO:
return "lzo"; return "lzo";
case BTRFS_COMPRESS_ZSTD:
return "zstd";
} }
return NULL; return NULL;
......
...@@ -513,6 +513,14 @@ int btrfs_parse_options(struct btrfs_fs_info *info, char *options, ...@@ -513,6 +513,14 @@ int btrfs_parse_options(struct btrfs_fs_info *info, char *options,
btrfs_clear_opt(info->mount_opt, NODATASUM); btrfs_clear_opt(info->mount_opt, NODATASUM);
btrfs_set_fs_incompat(info, COMPRESS_LZO); btrfs_set_fs_incompat(info, COMPRESS_LZO);
no_compress = 0; no_compress = 0;
} else if (strcmp(args[0].from, "zstd") == 0) {
compress_type = "zstd";
info->compress_type = BTRFS_COMPRESS_ZSTD;
btrfs_set_opt(info->mount_opt, COMPRESS);
btrfs_clear_opt(info->mount_opt, NODATACOW);
btrfs_clear_opt(info->mount_opt, NODATASUM);
btrfs_set_fs_incompat(info, COMPRESS_ZSTD);
no_compress = 0;
} else if (strncmp(args[0].from, "no", 2) == 0) { } else if (strncmp(args[0].from, "no", 2) == 0) {
compress_type = "no"; compress_type = "no";
btrfs_clear_opt(info->mount_opt, COMPRESS); btrfs_clear_opt(info->mount_opt, COMPRESS);
...@@ -1227,8 +1235,10 @@ static int btrfs_show_options(struct seq_file *seq, struct dentry *dentry) ...@@ -1227,8 +1235,10 @@ static int btrfs_show_options(struct seq_file *seq, struct dentry *dentry)
if (btrfs_test_opt(info, COMPRESS)) { if (btrfs_test_opt(info, COMPRESS)) {
if (info->compress_type == BTRFS_COMPRESS_ZLIB) if (info->compress_type == BTRFS_COMPRESS_ZLIB)
compress_type = "zlib"; compress_type = "zlib";
else else if (info->compress_type == BTRFS_COMPRESS_LZO)
compress_type = "lzo"; compress_type = "lzo";
else
compress_type = "zstd";
if (btrfs_test_opt(info, FORCE_COMPRESS)) if (btrfs_test_opt(info, FORCE_COMPRESS))
seq_printf(seq, ",compress-force=%s", compress_type); seq_printf(seq, ",compress-force=%s", compress_type);
else else
......
...@@ -200,6 +200,7 @@ BTRFS_FEAT_ATTR_INCOMPAT(mixed_backref, MIXED_BACKREF); ...@@ -200,6 +200,7 @@ BTRFS_FEAT_ATTR_INCOMPAT(mixed_backref, MIXED_BACKREF);
BTRFS_FEAT_ATTR_INCOMPAT(default_subvol, DEFAULT_SUBVOL); BTRFS_FEAT_ATTR_INCOMPAT(default_subvol, DEFAULT_SUBVOL);
BTRFS_FEAT_ATTR_INCOMPAT(mixed_groups, MIXED_GROUPS); BTRFS_FEAT_ATTR_INCOMPAT(mixed_groups, MIXED_GROUPS);
BTRFS_FEAT_ATTR_INCOMPAT(compress_lzo, COMPRESS_LZO); BTRFS_FEAT_ATTR_INCOMPAT(compress_lzo, COMPRESS_LZO);
BTRFS_FEAT_ATTR_INCOMPAT(compress_zstd, COMPRESS_ZSTD);
BTRFS_FEAT_ATTR_INCOMPAT(big_metadata, BIG_METADATA); BTRFS_FEAT_ATTR_INCOMPAT(big_metadata, BIG_METADATA);
BTRFS_FEAT_ATTR_INCOMPAT(extended_iref, EXTENDED_IREF); BTRFS_FEAT_ATTR_INCOMPAT(extended_iref, EXTENDED_IREF);
BTRFS_FEAT_ATTR_INCOMPAT(raid56, RAID56); BTRFS_FEAT_ATTR_INCOMPAT(raid56, RAID56);
...@@ -212,6 +213,7 @@ static struct attribute *btrfs_supported_feature_attrs[] = { ...@@ -212,6 +213,7 @@ static struct attribute *btrfs_supported_feature_attrs[] = {
BTRFS_FEAT_ATTR_PTR(default_subvol), BTRFS_FEAT_ATTR_PTR(default_subvol),
BTRFS_FEAT_ATTR_PTR(mixed_groups), BTRFS_FEAT_ATTR_PTR(mixed_groups),
BTRFS_FEAT_ATTR_PTR(compress_lzo), BTRFS_FEAT_ATTR_PTR(compress_lzo),
BTRFS_FEAT_ATTR_PTR(compress_zstd),
BTRFS_FEAT_ATTR_PTR(big_metadata), BTRFS_FEAT_ATTR_PTR(big_metadata),
BTRFS_FEAT_ATTR_PTR(extended_iref), BTRFS_FEAT_ATTR_PTR(extended_iref),
BTRFS_FEAT_ATTR_PTR(raid56), BTRFS_FEAT_ATTR_PTR(raid56),
......
This diff is collapsed.
...@@ -255,13 +255,7 @@ struct btrfs_ioctl_fs_info_args { ...@@ -255,13 +255,7 @@ struct btrfs_ioctl_fs_info_args {
#define BTRFS_FEATURE_INCOMPAT_DEFAULT_SUBVOL (1ULL << 1) #define BTRFS_FEATURE_INCOMPAT_DEFAULT_SUBVOL (1ULL << 1)
#define BTRFS_FEATURE_INCOMPAT_MIXED_GROUPS (1ULL << 2) #define BTRFS_FEATURE_INCOMPAT_MIXED_GROUPS (1ULL << 2)
#define BTRFS_FEATURE_INCOMPAT_COMPRESS_LZO (1ULL << 3) #define BTRFS_FEATURE_INCOMPAT_COMPRESS_LZO (1ULL << 3)
/* #define BTRFS_FEATURE_INCOMPAT_COMPRESS_ZSTD (1ULL << 4)
* some patches floated around with a second compression method
* lets save that incompat here for when they do get in
* Note we don't actually support it, we're just reserving the
* number
*/
#define BTRFS_FEATURE_INCOMPAT_COMPRESS_LZOv2 (1ULL << 4)
/* /*
* older kernels tried to do bigger metadata blocks, but the * older kernels tried to do bigger metadata blocks, but the
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment