bcachefs: Refactor dev usage

This is to make it more amenable for serialization.
Signed-off-by: Kent Overstreet <kent.overstreet@gmail.com>
Signed-off-by: Kent Overstreet <kent.overstreet@linux.dev>

fs/bcachefs/alloc_background.c

@@ -54,10 +54,10 @@ static void pd_controllers_update(struct work_struct *work)
 		 * reclaimed by copy GC
 		 */
 		fragmented += max_t(s64, 0, (bucket_to_sector(ca,
-					stats.buckets[BCH_DATA_user] +
-					stats.buckets[BCH_DATA_cached]) -
-				  (stats.sectors[BCH_DATA_user] +
-				   stats.sectors[BCH_DATA_cached])) << 9);
+					stats.d[BCH_DATA_user].buckets +
+					stats.d[BCH_DATA_cached].buckets) -
+				  (stats.d[BCH_DATA_user].sectors +
+				   stats.d[BCH_DATA_cached].sectors)) << 9);
 	}
 
 	bch2_pd_controller_update(&c->copygc_pd, free, fragmented, -1);
@@ -217,7 +217,7 @@ static int bch2_alloc_read_fn(struct bch_fs *c, enum btree_id id,
 		return 0;
 
 	ca = bch_dev_bkey_exists(c, k.k->p.inode);
-	g = __bucket(ca, k.k->p.offset, 0);
+	g = bucket(ca, k.k->p.offset);
 	u = bch2_alloc_unpack(k);
 
 	g->_mark.gen		= u.gen;
@@ -278,7 +278,6 @@ static int bch2_alloc_write_key(struct btree_trans *trans,
 	struct bch_fs *c = trans->c;
 	struct bkey_s_c k;
 	struct bch_dev *ca;
-	struct bucket_array *ba;
 	struct bucket *g;
 	struct bucket_mark m;
 	struct bkey_alloc_unpacked old_u, new_u;
@@ -302,9 +301,7 @@ static int bch2_alloc_write_key(struct btree_trans *trans,
 
 	percpu_down_read(&c->mark_lock);
 	ca	= bch_dev_bkey_exists(c, iter->pos.inode);
-	ba	= bucket_array(ca);
-
-	g	= &ba->b[iter->pos.offset];
+	g	= bucket(ca, iter->pos.offset);
 	m	= READ_ONCE(g->mark);
 	new_u	= alloc_mem_to_key(g, m);
 	percpu_up_read(&c->mark_lock);
@@ -330,16 +327,10 @@ int bch2_dev_alloc_write(struct bch_fs *c, struct bch_dev *ca, unsigned flags)
 {
 	struct btree_trans trans;
 	struct btree_iter *iter;
-	u64 first_bucket, nbuckets;
+	u64 first_bucket	= ca->mi.first_bucket;
+	u64 nbuckets		= ca->mi.nbuckets;
 	int ret = 0;
 
-	percpu_down_read(&c->mark_lock);
-	first_bucket	= bucket_array(ca)->first_bucket;
-	nbuckets	= bucket_array(ca)->nbuckets;
-	percpu_up_read(&c->mark_lock);
-
-	BUG_ON(BKEY_ALLOC_VAL_U64s_MAX > 8);
-
 	bch2_trans_init(&trans, c, BTREE_ITER_MAX, 0);
 
 	iter = bch2_trans_get_iter(&trans, BTREE_ID_ALLOC,
@@ -552,7 +543,8 @@ int bch2_bucket_io_time_reset(struct btree_trans *trans, unsigned dev,
 static int wait_buckets_available(struct bch_fs *c, struct bch_dev *ca)
 {
 	unsigned long gc_count = c->gc_count;
-	u64 available;
+	s64 available;
+	unsigned i;
 	int ret = 0;
 
 	ca->allocator_state = ALLOCATOR_BLOCKED;
@@ -568,8 +560,15 @@ static int wait_buckets_available(struct bch_fs *c, struct bch_dev *ca)
 		if (gc_count != c->gc_count)
 			ca->inc_gen_really_needs_gc = 0;
 
-		available = max_t(s64, 0, dev_buckets_available(ca) -
-				  ca->inc_gen_really_needs_gc);
+		available  = dev_buckets_available(ca);
+		available -= ca->inc_gen_really_needs_gc;
+
+		spin_lock(&c->freelist_lock);
+		for (i = 0; i < RESERVE_NR; i++)
+			available -= fifo_used(&ca->free[i]);
+		spin_unlock(&c->freelist_lock);
+
+		available = max(available, 0LL);
 
 		if (available > fifo_free(&ca->free_inc) ||
 		    (available &&
@@ -598,6 +597,9 @@ static bool bch2_can_invalidate_bucket(struct bch_dev *ca,
 	if (!is_available_bucket(mark))
 		return false;
 
+	if (mark.owned_by_allocator)
+		return false;
+
 	if (ca->buckets_nouse &&
 	    test_bit(bucket, ca->buckets_nouse))
 		return false;

fs/bcachefs/alloc_foreground.c

@@ -192,8 +192,9 @@ long bch2_bucket_alloc_new_fs(struct bch_dev *ca)
 	rcu_read_lock();
 	buckets = bucket_array(ca);
 
-	for (b = ca->mi.first_bucket; b < ca->mi.nbuckets; b++)
-		if (is_available_bucket(buckets->b[b].mark))
+	for (b = buckets->first_bucket; b < buckets->nbuckets; b++)
+		if (is_available_bucket(buckets->b[b].mark) &&
+		    !buckets->b[b].mark.owned_by_allocator)
 			goto success;
 	b = -1;
 success:
@@ -224,9 +225,8 @@ struct open_bucket *bch2_bucket_alloc(struct bch_fs *c, struct bch_dev *ca,
 				      bool may_alloc_partial,
 				      struct closure *cl)
 {
-	struct bucket_array *buckets;
 	struct open_bucket *ob;
-	long bucket = 0;
+	long b = 0;
 
 	spin_lock(&c->freelist_lock);
 
@@ -260,13 +260,13 @@ struct open_bucket *bch2_bucket_alloc(struct bch_fs *c, struct bch_dev *ca,
 		return ERR_PTR(-OPEN_BUCKETS_EMPTY);
 	}
 
-	if (likely(fifo_pop(&ca->free[RESERVE_NONE], bucket)))
+	if (likely(fifo_pop(&ca->free[RESERVE_NONE], b)))
 		goto out;
 
 	switch (reserve) {
 	case RESERVE_BTREE_MOVINGGC:
 	case RESERVE_MOVINGGC:
-		if (fifo_pop(&ca->free[RESERVE_MOVINGGC], bucket))
+		if (fifo_pop(&ca->free[RESERVE_MOVINGGC], b))
 			goto out;
 		break;
 	default:
@@ -284,20 +284,19 @@ struct open_bucket *bch2_bucket_alloc(struct bch_fs *c, struct bch_dev *ca,
 	trace_bucket_alloc_fail(ca, reserve);
 	return ERR_PTR(-FREELIST_EMPTY);
 out:
-	verify_not_on_freelist(c, ca, bucket);
+	verify_not_on_freelist(c, ca, b);
 
 	ob = bch2_open_bucket_alloc(c);
 
 	spin_lock(&ob->lock);
-	buckets = bucket_array(ca);
 
 	ob->valid	= true;
 	ob->sectors_free = ca->mi.bucket_size;
 	ob->alloc_reserve = reserve;
 	ob->ptr		= (struct bch_extent_ptr) {
 		.type	= 1 << BCH_EXTENT_ENTRY_ptr,
-		.gen	= buckets->b[bucket].mark.gen,
-		.offset	= bucket_to_sector(ca, bucket),
+		.gen	= bucket(ca, b)->mark.gen,
+		.offset	= bucket_to_sector(ca, b),
 		.dev	= ca->dev_idx,
 	};
 

fs/bcachefs/buckets.c

@@ -379,15 +379,12 @@ static inline int is_unavailable_bucket(struct bucket_mark m)
 	return !is_available_bucket(m);
 }
 
-static inline int is_fragmented_bucket(struct bucket_mark m,
-				       struct bch_dev *ca)
-{
-	if (!m.owned_by_allocator &&
-	    m.data_type == BCH_DATA_user &&
-	    bucket_sectors_used(m))
-		return max_t(int, 0, (int) ca->mi.bucket_size -
-			     bucket_sectors_used(m));
-	return 0;
+static inline int bucket_sectors_fragmented(struct bch_dev *ca,
+					    struct bucket_mark m)
+{
+	return bucket_sectors_used(m)
+		? max(0, (int) ca->mi.bucket_size - (int) bucket_sectors_used(m))
+		: 0;
 }
 
 static inline int is_stripe_data_bucket(struct bucket_mark m)
@@ -395,11 +392,6 @@ static inline int is_stripe_data_bucket(struct bucket_mark m)
 	return m.stripe && m.data_type != BCH_DATA_parity;
 }
 
-static inline int bucket_stripe_sectors(struct bucket_mark m)
-{
-	return is_stripe_data_bucket(m) ? m.dirty_sectors : 0;
-}
-
 static inline enum bch_data_type bucket_type(struct bucket_mark m)
 {
 	return m.cached_sectors && !m.dirty_sectors
@@ -461,7 +453,7 @@ static inline void account_bucket(struct bch_fs_usage *fs_usage,
 	if (type == BCH_DATA_sb || type == BCH_DATA_journal)
 		fs_usage->hidden	+= size;
 
-	dev_usage->buckets[type]	+= nr;
+	dev_usage->d[type].buckets	+= nr;
 }
 
 static void bch2_dev_usage_update(struct bch_fs *c, struct bch_dev *ca,
@@ -487,16 +479,14 @@ static void bch2_dev_usage_update(struct bch_fs *c, struct bch_dev *ca,
 	u->buckets_unavailable +=
 		is_unavailable_bucket(new) - is_unavailable_bucket(old);
 
-	u->buckets_ec += (int) new.stripe - (int) old.stripe;
-	u->sectors_ec += bucket_stripe_sectors(new) -
-			 bucket_stripe_sectors(old);
-
-	u->sectors[old.data_type] -= old.dirty_sectors;
-	u->sectors[new.data_type] += new.dirty_sectors;
-	u->sectors[BCH_DATA_cached] +=
+	u->d[old.data_type].sectors -= old.dirty_sectors;
+	u->d[new.data_type].sectors += new.dirty_sectors;
+	u->d[BCH_DATA_cached].sectors +=
 		(int) new.cached_sectors - (int) old.cached_sectors;
-	u->sectors_fragmented +=
-		is_fragmented_bucket(new, ca) - is_fragmented_bucket(old, ca);
+
+	u->d[old.data_type].fragmented -= bucket_sectors_fragmented(ca, old);
+	u->d[new.data_type].fragmented += bucket_sectors_fragmented(ca, new);
+
 	preempt_enable();
 
 	if (!is_available_bucket(old) && is_available_bucket(new))

fs/bcachefs/buckets.h

@@ -153,18 +153,9 @@ static inline unsigned bucket_sectors_used(struct bucket_mark mark)
 	return mark.dirty_sectors + mark.cached_sectors;
 }
 
-static inline bool bucket_unused(struct bucket_mark mark)
-{
-	return !mark.owned_by_allocator &&
-		!mark.data_type &&
-		!bucket_sectors_used(mark);
-}
-
 static inline bool is_available_bucket(struct bucket_mark mark)
 {
-	return (!mark.owned_by_allocator &&
-		!mark.dirty_sectors &&
-		!mark.stripe);
+	return !mark.dirty_sectors && !mark.stripe;
 }
 
 static inline bool bucket_needs_journal_commit(struct bucket_mark m,

fs/bcachefs/buckets_types.h

@@ -52,15 +52,14 @@ struct bucket_array {
 };
 
 struct bch_dev_usage {
-	u64			buckets[BCH_DATA_NR];
+	u64			buckets_ec;
 	u64			buckets_unavailable;
 
-	/* _compressed_ sectors: */
-	u64			sectors[BCH_DATA_NR];
-	u64			sectors_fragmented;
-
-	u64			buckets_ec;
-	u64			sectors_ec;
+	struct {
+		u64		buckets;
+		u64		sectors; /* _compressed_ sectors: */
+		u64		fragmented;
+	}			d[BCH_DATA_NR];
 };
 
 struct bch_fs_usage {

fs/bcachefs/chardev.c

@@ -477,11 +477,11 @@ static long bch2_ioctl_dev_usage(struct bch_fs *c,
 	arg.nr_buckets		= ca->mi.nbuckets - ca->mi.first_bucket;
 	arg.available_buckets	= arg.nr_buckets - src.buckets_unavailable;
 	arg.ec_buckets		= src.buckets_ec;
-	arg.ec_sectors		= src.sectors_ec;
+	arg.ec_sectors		= 0;
 
 	for (i = 0; i < BCH_DATA_NR; i++) {
-		arg.buckets[i] = src.buckets[i];
-		arg.sectors[i] = src.sectors[i];
+		arg.buckets[i] = src.d[i].buckets;
+		arg.sectors[i] = src.d[i].sectors;
 	}
 
 	percpu_ref_put(&ca->ref);

fs/bcachefs/movinggc.c

@@ -291,7 +291,7 @@ unsigned long bch2_copygc_wait_amount(struct bch_fs *c)
 
 		fragmented_allowed += ((__dev_buckets_available(ca, usage) *
 					ca->mi.bucket_size) >> 1);
-		fragmented += usage.sectors_fragmented;
+		fragmented += usage.d[BCH_DATA_user].fragmented;
 	}
 
 	return max_t(s64, 0, fragmented_allowed - fragmented);

fs/bcachefs/super.c

@@ -1221,7 +1221,7 @@ static int bch2_dev_attach_bdev(struct bch_fs *c, struct bch_sb_handle *sb)
 		return ret;
 
 	if (test_bit(BCH_FS_ALLOC_READ_DONE, &c->flags) &&
-	    !percpu_u64_get(&ca->usage[0]->buckets[BCH_DATA_sb])) {
+	    !percpu_u64_get(&ca->usage[0]->d[BCH_DATA_sb].buckets)) {
 		mutex_lock(&c->sb_lock);
 		bch2_mark_dev_superblock(ca->fs, ca, 0);
 		mutex_unlock(&c->sb_lock);

fs/bcachefs/sysfs.c

@@ -797,59 +797,40 @@ static void dev_alloc_debug_to_text(struct printbuf *out, struct bch_dev *ca)
 		nr[c->open_buckets[i].type]++;
 
 	pr_buf(out,
-		"free_inc:               %zu/%zu\n"
-		"free[RESERVE_MOVINGGC]: %zu/%zu\n"
-		"free[RESERVE_NONE]:     %zu/%zu\n"
-		"buckets:\n"
-		"    capacity:           %llu\n"
-		"    sb:                 %llu\n"
-		"    journal:            %llu\n"
-		"    meta:               %llu\n"
-		"    user:               %llu\n"
-		"    cached:             %llu\n"
-		"    erasure coded:      %llu\n"
-		"    available:          %lli\n"
-		"sectors:\n"
-		"    sb:                 %llu\n"
-		"    journal:            %llu\n"
-		"    meta:               %llu\n"
-		"    user:               %llu\n"
-		"    cached:             %llu\n"
-		"    erasure coded:      %llu\n"
-		"    fragmented:         %llu\n"
-		"    copygc threshold:   %llu\n"
-		"freelist_wait:          %s\n"
-		"open buckets:           %u/%u (reserved %u)\n"
-		"open_buckets_wait:      %s\n"
-		"open_buckets_btree:     %u\n"
-		"open_buckets_user:      %u\n"
-		"btree reserve cache:    %u\n",
-		fifo_used(&ca->free_inc),		ca->free_inc.size,
-		fifo_used(&ca->free[RESERVE_MOVINGGC]),	ca->free[RESERVE_MOVINGGC].size,
-		fifo_used(&ca->free[RESERVE_NONE]),	ca->free[RESERVE_NONE].size,
-		ca->mi.nbuckets - ca->mi.first_bucket,
-		stats.buckets[BCH_DATA_sb],
-		stats.buckets[BCH_DATA_journal],
-		stats.buckets[BCH_DATA_btree],
-		stats.buckets[BCH_DATA_user],
-		stats.buckets[BCH_DATA_cached],
-		stats.buckets_ec,
-		__dev_buckets_available(ca, stats),
-		stats.sectors[BCH_DATA_sb],
-		stats.sectors[BCH_DATA_journal],
-		stats.sectors[BCH_DATA_btree],
-		stats.sectors[BCH_DATA_user],
-		stats.sectors[BCH_DATA_cached],
-		stats.sectors_ec,
-		stats.sectors_fragmented,
-		c->copygc_threshold,
-		c->freelist_wait.list.first		? "waiting" : "empty",
-		c->open_buckets_nr_free, OPEN_BUCKETS_COUNT,
-		BTREE_NODE_OPEN_BUCKET_RESERVE,
-		c->open_buckets_wait.list.first		? "waiting" : "empty",
-		nr[BCH_DATA_btree],
-		nr[BCH_DATA_user],
-		c->btree_reserve_cache_nr);
+	       "\t\t buckets\t sectors      fragmented\n"
+	       "capacity%16llu\n",
+	       ca->mi.nbuckets - ca->mi.first_bucket);
+
+	for (i = 1; i < BCH_DATA_NR; i++)
+		pr_buf(out, "%-8s%16llu%16llu%16llu\n",
+		       bch2_data_types[i], stats.d[i].buckets,
+		       stats.d[i].sectors, stats.d[i].fragmented);
+
+	pr_buf(out,
+	       "ec\t%16llu\n"
+	       "available%15llu\n"
+	       "\n"
+	       "free_inc\t\t%zu/%zu\n"
+	       "free[RESERVE_MOVINGGC]\t%zu/%zu\n"
+	       "free[RESERVE_NONE]\t%zu/%zu\n"
+	       "freelist_wait\t\t%s\n"
+	       "open buckets\t\t%u/%u (reserved %u)\n"
+	       "open_buckets_wait\t%s\n"
+	       "open_buckets_btree\t%u\n"
+	       "open_buckets_user\t%u\n"
+	       "btree reserve cache\t%u\n",
+	       stats.buckets_ec,
+	       __dev_buckets_available(ca, stats),
+	       fifo_used(&ca->free_inc),		ca->free_inc.size,
+	       fifo_used(&ca->free[RESERVE_MOVINGGC]),	ca->free[RESERVE_MOVINGGC].size,
+	       fifo_used(&ca->free[RESERVE_NONE]),	ca->free[RESERVE_NONE].size,
+	       c->freelist_wait.list.first		? "waiting" : "empty",
+	       c->open_buckets_nr_free, OPEN_BUCKETS_COUNT,
+	       BTREE_NODE_OPEN_BUCKET_RESERVE,
+	       c->open_buckets_wait.list.first		? "waiting" : "empty",
+	       nr[BCH_DATA_btree],
+	       nr[BCH_DATA_user],
+	       c->btree_reserve_cache_nr);
 }
 
 static const char * const bch2_rw[] = {