bcachefs: Mark btree nodes as needing rewrite when not all replicas are RW

This fixes a bug where recovery fails when one of the devices is read
only.

Also - consolidate the "must rewrite this node to insert it" behind a
new btree node flag.
Signed-off-by: Kent Overstreet <kent.overstreet@gmail.com>
Signed-off-by: Kent Overstreet <kent.overstreet@linux.dev>

fs/bcachefs/btree_io.c

@@ -917,6 +917,7 @@ int bch2_btree_node_read_done(struct bch_fs *c, struct btree *b, bool have_retry
 	struct sort_iter *iter;
 	struct btree_node *sorted;
 	struct bkey_packed *k;
+	struct bch_extent_ptr *ptr;
 	struct bset *i;
 	bool used_mempool, blacklisted;
 	unsigned u64s;
@@ -971,8 +972,10 @@ int bch2_btree_node_read_done(struct bch_fs *c, struct btree *b, bool have_retry
 			bset_encrypt(c, i, b->written << 9);
 
 			if (btree_node_is_extents(b) &&
-			    !BTREE_NODE_NEW_EXTENT_OVERWRITE(b->data))
+			    !BTREE_NODE_NEW_EXTENT_OVERWRITE(b->data)) {
 				set_btree_node_old_extent_overwrite(b);
+				set_btree_node_need_rewrite(b);
+			}
 
 			sectors = vstruct_sectors(b->data, c->block_bits);
 		} else {
@@ -1098,6 +1101,13 @@ int bch2_btree_node_read_done(struct bch_fs *c, struct btree *b, bool have_retry
 	set_needs_whiteout(btree_bset_first(b), true);
 
 	btree_node_reset_sib_u64s(b);
+
+	bkey_for_each_ptr(bch2_bkey_ptrs(bkey_i_to_s(&b->key)), ptr) {
+		struct bch_dev *ca = bch_dev_bkey_exists(c, ptr->dev);
+
+		if (ca->mi.state != BCH_MEMBER_STATE_RW)
+			set_btree_node_need_rewrite(b);
+	}
 out:
 	mempool_free(iter, &c->fill_iter);
 	return retry_read;

fs/bcachefs/btree_types.h

@@ -409,6 +409,7 @@ enum btree_flags {
 	BTREE_NODE_dying,
 	BTREE_NODE_fake,
 	BTREE_NODE_old_extent_overwrite,
+	BTREE_NODE_need_rewrite,
 };
 
 BTREE_FLAG(read_in_flight);
@@ -423,6 +424,7 @@ BTREE_FLAG(just_written);
 BTREE_FLAG(dying);
 BTREE_FLAG(fake);
 BTREE_FLAG(old_extent_overwrite);
+BTREE_FLAG(need_rewrite);
 
 static inline struct btree_write *btree_current_write(struct btree *b)
 {

fs/bcachefs/btree_update_interior.c

@@ -290,8 +290,10 @@ static struct btree *bch2_btree_node_alloc(struct btree_update *as, unsigned lev
 		SET_BTREE_NODE_NEW_EXTENT_OVERWRITE(b->data, true);
 
 	if (btree_node_is_extents(b) &&
-	    !BTREE_NODE_NEW_EXTENT_OVERWRITE(b->data))
+	    !BTREE_NODE_NEW_EXTENT_OVERWRITE(b->data)) {
 		set_btree_node_old_extent_overwrite(b);
+		set_btree_node_need_rewrite(b);
+	}
 
 	bch2_btree_build_aux_trees(b);
 
@@ -1943,6 +1945,7 @@ void bch2_btree_root_alloc(struct bch_fs *c, enum btree_id id)
 	bch2_btree_cache_cannibalize_unlock(c);
 
 	set_btree_node_fake(b);
+	set_btree_node_need_rewrite(b);
 	b->c.level	= 0;
 	b->c.btree_id	= id;
 

fs/bcachefs/btree_update_leaf.c

@@ -270,17 +270,8 @@ btree_key_can_insert(struct btree_trans *trans,
 	struct bch_fs *c = trans->c;
 	struct btree *b = iter_l(iter)->b;
 
-	if (unlikely(btree_node_fake(b)))
-		return BTREE_INSERT_BTREE_NODE_FULL;
-
-	/*
-	 * old bch2_extent_sort_fix_overlapping() algorithm won't work with new
-	 * style extent updates:
-	 */
-	if (unlikely(btree_node_old_extent_overwrite(b)))
-		return BTREE_INSERT_BTREE_NODE_FULL;
-
-	if (unlikely(u64s > bch_btree_keys_u64s_remaining(c, b)))
+	if (unlikely(btree_node_need_rewrite(b)) ||
+	    unlikely(u64s > bch_btree_keys_u64s_remaining(c, b)))
 		return BTREE_INSERT_BTREE_NODE_FULL;
 
 	return BTREE_INSERT_OK;