bcachefs: Merge extents with checksums

Signed-off-by: Kent Overstreet <kent.overstreet@linux.dev>

fs/bcachefs/bkey_methods.c

@@ -202,15 +202,20 @@ enum merge_result bch2_bkey_merge(struct bch_fs *c,
 				  struct bkey_i *l, struct bkey_i *r)
 {
 	const struct bkey_ops *ops = &bch2_bkey_ops[l->k.type];
+	enum merge_result ret;
 
-	if (!key_merging_disabled(c) &&
-	    ops->key_merge &&
-	    l->k.type == r->k.type &&
-	    !bversion_cmp(l->k.version, r->k.version) &&
-	    !bkey_cmp(l->k.p, bkey_start_pos(&r->k)))
-		return ops->key_merge(c, l, r);
+	if (key_merging_disabled(c) ||
+	    !ops->key_merge ||
+	    l->k.type != r->k.type ||
+	    bversion_cmp(l->k.version, r->k.version) ||
+	    bkey_cmp(l->k.p, bkey_start_pos(&r->k)))
+		return BCH_MERGE_NOMERGE;
 
-	return BCH_MERGE_NOMERGE;
+	ret = ops->key_merge(c, l, r);
+
+	if (ret != BCH_MERGE_NOMERGE)
+		l->k.needs_whiteout |= r->k.needs_whiteout;
+	return ret;
 }
 
 static const struct old_bkey_type {

fs/bcachefs/checksum.c

@@ -281,22 +281,8 @@ void bch2_encrypt_bio(struct bch_fs *c, unsigned type,
 	do_encrypt_sg(c->chacha20, nonce, sgl, bytes);
 }
 
-static inline bool bch2_checksum_mergeable(unsigned type)
-{
-
-	switch (type) {
-	case BCH_CSUM_NONE:
-	case BCH_CSUM_CRC32C:
-	case BCH_CSUM_CRC64:
-		return true;
-	default:
-		return false;
-	}
-}
-
-static struct bch_csum bch2_checksum_merge(unsigned type,
-					   struct bch_csum a,
-					   struct bch_csum b, size_t b_len)
+struct bch_csum bch2_checksum_merge(unsigned type, struct bch_csum a,
+				    struct bch_csum b, size_t b_len)
 {
 	BUG_ON(!bch2_checksum_mergeable(type));
 

fs/bcachefs/checksum.h

@@ -9,6 +9,22 @@
 #include <linux/crc64.h>
 #include <crypto/chacha.h>
 
+static inline bool bch2_checksum_mergeable(unsigned type)
+{
+
+	switch (type) {
+	case BCH_CSUM_NONE:
+	case BCH_CSUM_CRC32C:
+	case BCH_CSUM_CRC64:
+		return true;
+	default:
+		return false;
+	}
+}
+
+struct bch_csum bch2_checksum_merge(unsigned, struct bch_csum,
+				    struct bch_csum, size_t);
+
 static inline u64 bch2_crc64_update(u64 crc, const void *p, size_t len)
 {
 	return crc64_be(crc, p, len);

fs/bcachefs/extents.c

@@ -1358,53 +1358,63 @@ void bch2_extent_to_text(struct printbuf *out, struct bch_fs *c,
 	bch2_bkey_ptrs_to_text(out, c, k);
 }
 
+static unsigned bch2_crc_field_size_max[] = {
+	[BCH_EXTENT_ENTRY_crc32] = CRC32_SIZE_MAX,
+	[BCH_EXTENT_ENTRY_crc64] = CRC64_SIZE_MAX,
+	[BCH_EXTENT_ENTRY_crc128] = CRC128_SIZE_MAX,
+};
+
+static void bch2_extent_crc_pack(union bch_extent_crc *dst,
+				 struct bch_extent_crc_unpacked src)
+{
+#define set_common_fields(_dst, _src)					\
+		_dst.csum_type		= _src.csum_type,		\
+		_dst.compression_type	= _src.compression_type,	\
+		_dst._compressed_size	= _src.compressed_size - 1,	\
+		_dst._uncompressed_size	= _src.uncompressed_size - 1,	\
+		_dst.offset		= _src.offset
+
+	switch (extent_entry_type(to_entry(dst))) {
+	case BCH_EXTENT_ENTRY_crc32:
+		set_common_fields(dst->crc32, src);
+		dst->crc32.csum	 = *((__le32 *) &src.csum.lo);
+		break;
+	case BCH_EXTENT_ENTRY_crc64:
+		set_common_fields(dst->crc64, src);
+		dst->crc64.nonce	= src.nonce;
+		dst->crc64.csum_lo	= src.csum.lo;
+		dst->crc64.csum_hi	= *((__le16 *) &src.csum.hi);
+		break;
+	case BCH_EXTENT_ENTRY_crc128:
+		set_common_fields(dst->crc128, src);
+		dst->crc128.nonce	= src.nonce;
+		dst->crc128.csum	= src.csum;
+		break;
+	default:
+		BUG();
+	}
+#undef set_common_fields
+}
+
 static void bch2_extent_crc_init(union bch_extent_crc *crc,
 				 struct bch_extent_crc_unpacked new)
 {
-#define common_fields(_crc)						\
-		.csum_type		= _crc.csum_type,		\
-		.compression_type	= _crc.compression_type,	\
-		._compressed_size	= _crc.compressed_size - 1,	\
-		._uncompressed_size	= _crc.uncompressed_size - 1,	\
-		.offset			= _crc.offset
-
 	if (bch_crc_bytes[new.csum_type]	<= 4 &&
-	    new.uncompressed_size		<= CRC32_SIZE_MAX &&
-	    new.nonce				<= CRC32_NONCE_MAX) {
-		crc->crc32 = (struct bch_extent_crc32) {
-			.type = 1 << BCH_EXTENT_ENTRY_crc32,
-			common_fields(new),
-			.csum			= *((__le32 *) &new.csum.lo),
-		};
-		return;
-	}
-
-	if (bch_crc_bytes[new.csum_type]	<= 10 &&
-	    new.uncompressed_size		<= CRC64_SIZE_MAX &&
-	    new.nonce				<= CRC64_NONCE_MAX) {
-		crc->crc64 = (struct bch_extent_crc64) {
-			.type = 1 << BCH_EXTENT_ENTRY_crc64,
-			common_fields(new),
-			.nonce			= new.nonce,
-			.csum_lo		= new.csum.lo,
-			.csum_hi		= *((__le16 *) &new.csum.hi),
-		};
-		return;
-	}
+	    new.uncompressed_size - 1		<= CRC32_SIZE_MAX &&
+	    new.nonce				<= CRC32_NONCE_MAX)
+		crc->type = 1 << BCH_EXTENT_ENTRY_crc32;
+	else if (bch_crc_bytes[new.csum_type]	<= 10 &&
+		   new.uncompressed_size - 1	<= CRC64_SIZE_MAX &&
+		   new.nonce			<= CRC64_NONCE_MAX)
+		crc->type = 1 << BCH_EXTENT_ENTRY_crc64;
+	else if (bch_crc_bytes[new.csum_type]	<= 16 &&
+		   new.uncompressed_size - 1	<= CRC128_SIZE_MAX &&
+		   new.nonce			<= CRC128_NONCE_MAX)
+		crc->type = 1 << BCH_EXTENT_ENTRY_crc128;
+	else
+		BUG();
 
-	if (bch_crc_bytes[new.csum_type]	<= 16 &&
-	    new.uncompressed_size		<= CRC128_SIZE_MAX &&
-	    new.nonce				<= CRC128_NONCE_MAX) {
-		crc->crc128 = (struct bch_extent_crc128) {
-			.type = 1 << BCH_EXTENT_ENTRY_crc128,
-			common_fields(new),
-			.nonce			= new.nonce,
-			.csum			= new.csum,
-		};
-		return;
-	}
-#undef common_fields
-	BUG();
+	bch2_extent_crc_pack(crc, new);
 }
 
 void bch2_extent_crc_append(struct bkey_i_extent *e,
@@ -1515,46 +1525,98 @@ enum merge_result bch2_extent_merge(struct bch_fs *c,
 {
 	struct bkey_s_extent el = bkey_i_to_s_extent(l);
 	struct bkey_s_extent er = bkey_i_to_s_extent(r);
-	union bch_extent_entry *en_l, *en_r;
+	union bch_extent_entry *en_l = el.v->start;
+	union bch_extent_entry *en_r = er.v->start;
+	struct bch_extent_crc_unpacked crc_l, crc_r;
 
-	if (bkey_val_u64s(&l->k) != bkey_val_u64s(&r->k))
-		return BCH_MERGE_NOMERGE;
+	crc_l = bch2_extent_crc_unpack(el.k, NULL);
 
 	extent_for_each_entry(el, en_l) {
-		struct bch_extent_ptr *lp, *rp;
-		struct bch_dev *ca;
-
 		en_r = vstruct_idx(er.v, (u64 *) en_l - el.v->_data);
 
-		if ((extent_entry_type(en_l) !=
-		     extent_entry_type(en_r)) ||
-		    !extent_entry_is_ptr(en_l))
+		if (extent_entry_type(en_l) != extent_entry_type(en_r))
 			return BCH_MERGE_NOMERGE;
 
-		lp = &en_l->ptr;
-		rp = &en_r->ptr;
+		switch (extent_entry_type(en_l)) {
+		case BCH_EXTENT_ENTRY_ptr: {
+			const struct bch_extent_ptr *lp = &en_l->ptr;
+			const struct bch_extent_ptr *rp = &en_r->ptr;
+			struct bch_dev *ca;
 
-		if (lp->offset + el.k->size	!= rp->offset ||
-		    lp->dev			!= rp->dev ||
-		    lp->gen			!= rp->gen)
-			return BCH_MERGE_NOMERGE;
+			if (lp->offset + crc_l.compressed_size != rp->offset ||
+			    lp->dev			!= rp->dev ||
+			    lp->gen			!= rp->gen)
+				return BCH_MERGE_NOMERGE;
 
-		/* We don't allow extents to straddle buckets: */
-		ca = bch_dev_bkey_exists(c, lp->dev);
+			/* We don't allow extents to straddle buckets: */
+			ca = bch_dev_bkey_exists(c, lp->dev);
 
-		if (PTR_BUCKET_NR(ca, lp) != PTR_BUCKET_NR(ca, rp))
+			if (PTR_BUCKET_NR(ca, lp) != PTR_BUCKET_NR(ca, rp))
+				return BCH_MERGE_NOMERGE;
+
+			break;
+		}
+		case BCH_EXTENT_ENTRY_stripe_ptr:
+			if (en_l->stripe_ptr.block	!= en_r->stripe_ptr.block ||
+			    en_l->stripe_ptr.idx	!= en_r->stripe_ptr.idx)
+				return BCH_MERGE_NOMERGE;
+			break;
+		case BCH_EXTENT_ENTRY_crc32:
+		case BCH_EXTENT_ENTRY_crc64:
+		case BCH_EXTENT_ENTRY_crc128:
+			crc_l = bch2_extent_crc_unpack(el.k, entry_to_crc(en_l));
+			crc_r = bch2_extent_crc_unpack(er.k, entry_to_crc(en_r));
+
+			if (crc_l.csum_type		!= crc_r.csum_type ||
+			    crc_l.compression_type	!= crc_r.compression_type ||
+			    crc_l.nonce			!= crc_r.nonce)
+				return BCH_MERGE_NOMERGE;
+
+			if (crc_l.offset + crc_l.live_size != crc_l.compressed_size ||
+			    crc_r.offset)
+				return BCH_MERGE_NOMERGE;
+
+			if (!bch2_checksum_mergeable(crc_l.csum_type))
+				return BCH_MERGE_NOMERGE;
+
+			if (crc_l.compression_type)
+				return BCH_MERGE_NOMERGE;
+
+			if (crc_l.csum_type &&
+			    crc_l.uncompressed_size +
+			    crc_r.uncompressed_size > c->sb.encoded_extent_max)
+				return BCH_MERGE_NOMERGE;
+
+			if (crc_l.uncompressed_size + crc_r.uncompressed_size - 1 >
+			    bch2_crc_field_size_max[extent_entry_type(en_l)])
+				return BCH_MERGE_NOMERGE;
+
+			break;
+		default:
 			return BCH_MERGE_NOMERGE;
+		}
 	}
 
-	l->k.needs_whiteout |= r->k.needs_whiteout;
+	extent_for_each_entry(el, en_l) {
+		struct bch_extent_crc_unpacked crc_l, crc_r;
+
+		en_r = vstruct_idx(er.v, (u64 *) en_l - el.v->_data);
 
-	/* Keys with no pointers aren't restricted to one bucket and could
-	 * overflow KEY_SIZE
-	 */
-	if ((u64) l->k.size + r->k.size > KEY_SIZE_MAX) {
-		bch2_key_resize(&l->k, KEY_SIZE_MAX);
-		bch2_cut_front(l->k.p, r);
-		return BCH_MERGE_PARTIAL;
+		if (!extent_entry_is_crc(en_l))
+			continue;
+
+		crc_l = bch2_extent_crc_unpack(el.k, entry_to_crc(en_l));
+		crc_r = bch2_extent_crc_unpack(er.k, entry_to_crc(en_r));
+
+		crc_l.csum = bch2_checksum_merge(crc_l.csum_type,
+						 crc_l.csum,
+						 crc_r.csum,
+						 crc_r.uncompressed_size << 9);
+
+		crc_l.uncompressed_size	+= crc_r.uncompressed_size;
+		crc_l.compressed_size	+= crc_r.compressed_size;
+
+		bch2_extent_crc_pack(entry_to_crc(en_l), crc_l);
 	}
 
 	bch2_key_resize(&l->k, l->k.size + r->k.size);
@@ -1725,11 +1787,6 @@ enum merge_result bch2_reservation_merge(struct bch_fs *c,
 	    li->v.nr_replicas != ri->v.nr_replicas)
 		return BCH_MERGE_NOMERGE;
 
-	l->k.needs_whiteout |= r->k.needs_whiteout;
-
-	/* Keys with no pointers aren't restricted to one bucket and could
-	 * overflow KEY_SIZE
-	 */
 	if ((u64) l->k.size + r->k.size > KEY_SIZE_MAX) {
 		bch2_key_resize(&l->k, KEY_SIZE_MAX);
 		bch2_cut_front(l->k.p, r);