Commit 44871b1b authored by Chris Mason's avatar Chris Mason

Btrfs: reduce stack usage in some crucial tree balancing functions

Many of the tree balancing functions follow the same pattern.

1) cow a block
2) do something to the result

This commit breaks them up into two functions so the variables and
code required for part two don't suck down stack during part one.
Signed-off-by: default avatarChris Mason <chris.mason@oracle.com>
parent 56bec294
...@@ -2266,66 +2266,27 @@ noinline int btrfs_leaf_free_space(struct btrfs_root *root, ...@@ -2266,66 +2266,27 @@ noinline int btrfs_leaf_free_space(struct btrfs_root *root,
return ret; return ret;
} }
/* static noinline int __push_leaf_right(struct btrfs_trans_handle *trans,
* push some data in the path leaf to the right, trying to free up at struct btrfs_root *root,
* least data_size bytes. returns zero if the push worked, nonzero otherwise struct btrfs_path *path,
* int data_size, int empty,
* returns 1 if the push failed because the other node didn't have enough struct extent_buffer *right,
* room, 0 if everything worked out and < 0 if there were major errors. int free_space, u32 left_nritems)
*/
static int push_leaf_right(struct btrfs_trans_handle *trans, struct btrfs_root
*root, struct btrfs_path *path, int data_size,
int empty)
{ {
struct extent_buffer *left = path->nodes[0]; struct extent_buffer *left = path->nodes[0];
struct extent_buffer *right; struct extent_buffer *upper = path->nodes[1];
struct extent_buffer *upper;
struct btrfs_disk_key disk_key; struct btrfs_disk_key disk_key;
int slot; int slot;
u32 i; u32 i;
int free_space;
int push_space = 0; int push_space = 0;
int push_items = 0; int push_items = 0;
struct btrfs_item *item; struct btrfs_item *item;
u32 left_nritems;
u32 nr; u32 nr;
u32 right_nritems; u32 right_nritems;
u32 data_end; u32 data_end;
u32 this_item_size; u32 this_item_size;
int ret; int ret;
slot = path->slots[1];
if (!path->nodes[1])
return 1;
upper = path->nodes[1];
if (slot >= btrfs_header_nritems(upper) - 1)
return 1;
btrfs_assert_tree_locked(path->nodes[1]);
right = read_node_slot(root, upper, slot + 1);
btrfs_tree_lock(right);
btrfs_set_lock_blocking(right);
free_space = btrfs_leaf_free_space(root, right);
if (free_space < data_size)
goto out_unlock;
/* cow and double check */
ret = btrfs_cow_block(trans, root, right, upper,
slot + 1, &right);
if (ret)
goto out_unlock;
free_space = btrfs_leaf_free_space(root, right);
if (free_space < data_size)
goto out_unlock;
left_nritems = btrfs_header_nritems(left);
if (left_nritems == 0)
goto out_unlock;
if (empty) if (empty)
nr = 0; nr = 0;
else else
...@@ -2334,6 +2295,7 @@ static int push_leaf_right(struct btrfs_trans_handle *trans, struct btrfs_root ...@@ -2334,6 +2295,7 @@ static int push_leaf_right(struct btrfs_trans_handle *trans, struct btrfs_root
if (path->slots[0] >= left_nritems) if (path->slots[0] >= left_nritems)
push_space += data_size; push_space += data_size;
slot = path->slots[1];
i = left_nritems - 1; i = left_nritems - 1;
while (i >= nr) { while (i >= nr) {
item = btrfs_item_nr(left, i); item = btrfs_item_nr(left, i);
...@@ -2464,25 +2426,83 @@ static int push_leaf_right(struct btrfs_trans_handle *trans, struct btrfs_root ...@@ -2464,25 +2426,83 @@ static int push_leaf_right(struct btrfs_trans_handle *trans, struct btrfs_root
return 1; return 1;
} }
/*
* push some data in the path leaf to the right, trying to free up at
* least data_size bytes. returns zero if the push worked, nonzero otherwise
*
* returns 1 if the push failed because the other node didn't have enough
* room, 0 if everything worked out and < 0 if there were major errors.
*/
static int push_leaf_right(struct btrfs_trans_handle *trans, struct btrfs_root
*root, struct btrfs_path *path, int data_size,
int empty)
{
struct extent_buffer *left = path->nodes[0];
struct extent_buffer *right;
struct extent_buffer *upper;
int slot;
int free_space;
u32 left_nritems;
int ret;
if (!path->nodes[1])
return 1;
slot = path->slots[1];
upper = path->nodes[1];
if (slot >= btrfs_header_nritems(upper) - 1)
return 1;
btrfs_assert_tree_locked(path->nodes[1]);
right = read_node_slot(root, upper, slot + 1);
btrfs_tree_lock(right);
btrfs_set_lock_blocking(right);
free_space = btrfs_leaf_free_space(root, right);
if (free_space < data_size)
goto out_unlock;
/* cow and double check */
ret = btrfs_cow_block(trans, root, right, upper,
slot + 1, &right);
if (ret)
goto out_unlock;
free_space = btrfs_leaf_free_space(root, right);
if (free_space < data_size)
goto out_unlock;
left_nritems = btrfs_header_nritems(left);
if (left_nritems == 0)
goto out_unlock;
return __push_leaf_right(trans, root, path, data_size, empty,
right, free_space, left_nritems);
out_unlock:
btrfs_tree_unlock(right);
free_extent_buffer(right);
return 1;
}
/* /*
* push some data in the path leaf to the left, trying to free up at * push some data in the path leaf to the left, trying to free up at
* least data_size bytes. returns zero if the push worked, nonzero otherwise * least data_size bytes. returns zero if the push worked, nonzero otherwise
*/ */
static int push_leaf_left(struct btrfs_trans_handle *trans, struct btrfs_root static noinline int __push_leaf_left(struct btrfs_trans_handle *trans,
*root, struct btrfs_path *path, int data_size, struct btrfs_root *root,
int empty) struct btrfs_path *path, int data_size,
int empty, struct extent_buffer *left,
int free_space, int right_nritems)
{ {
struct btrfs_disk_key disk_key; struct btrfs_disk_key disk_key;
struct extent_buffer *right = path->nodes[0]; struct extent_buffer *right = path->nodes[0];
struct extent_buffer *left;
int slot; int slot;
int i; int i;
int free_space;
int push_space = 0; int push_space = 0;
int push_items = 0; int push_items = 0;
struct btrfs_item *item; struct btrfs_item *item;
u32 old_left_nritems; u32 old_left_nritems;
u32 right_nritems;
u32 nr; u32 nr;
int ret = 0; int ret = 0;
int wret; int wret;
...@@ -2490,41 +2510,6 @@ static int push_leaf_left(struct btrfs_trans_handle *trans, struct btrfs_root ...@@ -2490,41 +2510,6 @@ static int push_leaf_left(struct btrfs_trans_handle *trans, struct btrfs_root
u32 old_left_item_size; u32 old_left_item_size;
slot = path->slots[1]; slot = path->slots[1];
if (slot == 0)
return 1;
if (!path->nodes[1])
return 1;
right_nritems = btrfs_header_nritems(right);
if (right_nritems == 0)
return 1;
btrfs_assert_tree_locked(path->nodes[1]);
left = read_node_slot(root, path->nodes[1], slot - 1);
btrfs_tree_lock(left);
btrfs_set_lock_blocking(left);
free_space = btrfs_leaf_free_space(root, left);
if (free_space < data_size) {
ret = 1;
goto out;
}
/* cow and double check */
ret = btrfs_cow_block(trans, root, left,
path->nodes[1], slot - 1, &left);
if (ret) {
/* we hit -ENOSPC, but it isn't fatal here */
ret = 1;
goto out;
}
free_space = btrfs_leaf_free_space(root, left);
if (free_space < data_size) {
ret = 1;
goto out;
}
if (empty) if (empty)
nr = right_nritems; nr = right_nritems;
...@@ -2691,6 +2676,154 @@ static int push_leaf_left(struct btrfs_trans_handle *trans, struct btrfs_root ...@@ -2691,6 +2676,154 @@ static int push_leaf_left(struct btrfs_trans_handle *trans, struct btrfs_root
return ret; return ret;
} }
/*
* push some data in the path leaf to the left, trying to free up at
* least data_size bytes. returns zero if the push worked, nonzero otherwise
*/
static int push_leaf_left(struct btrfs_trans_handle *trans, struct btrfs_root
*root, struct btrfs_path *path, int data_size,
int empty)
{
struct extent_buffer *right = path->nodes[0];
struct extent_buffer *left;
int slot;
int free_space;
u32 right_nritems;
int ret = 0;
slot = path->slots[1];
if (slot == 0)
return 1;
if (!path->nodes[1])
return 1;
right_nritems = btrfs_header_nritems(right);
if (right_nritems == 0)
return 1;
btrfs_assert_tree_locked(path->nodes[1]);
left = read_node_slot(root, path->nodes[1], slot - 1);
btrfs_tree_lock(left);
btrfs_set_lock_blocking(left);
free_space = btrfs_leaf_free_space(root, left);
if (free_space < data_size) {
ret = 1;
goto out;
}
/* cow and double check */
ret = btrfs_cow_block(trans, root, left,
path->nodes[1], slot - 1, &left);
if (ret) {
/* we hit -ENOSPC, but it isn't fatal here */
ret = 1;
goto out;
}
free_space = btrfs_leaf_free_space(root, left);
if (free_space < data_size) {
ret = 1;
goto out;
}
return __push_leaf_left(trans, root, path, data_size,
empty, left, free_space, right_nritems);
out:
btrfs_tree_unlock(left);
free_extent_buffer(left);
return ret;
}
/*
* split the path's leaf in two, making sure there is at least data_size
* available for the resulting leaf level of the path.
*
* returns 0 if all went well and < 0 on failure.
*/
static noinline int copy_for_split(struct btrfs_trans_handle *trans,
struct btrfs_root *root,
struct btrfs_path *path,
struct extent_buffer *l,
struct extent_buffer *right,
int slot, int mid, int nritems)
{
int data_copy_size;
int rt_data_off;
int i;
int ret = 0;
int wret;
struct btrfs_disk_key disk_key;
nritems = nritems - mid;
btrfs_set_header_nritems(right, nritems);
data_copy_size = btrfs_item_end_nr(l, mid) - leaf_data_end(root, l);
copy_extent_buffer(right, l, btrfs_item_nr_offset(0),
btrfs_item_nr_offset(mid),
nritems * sizeof(struct btrfs_item));
copy_extent_buffer(right, l,
btrfs_leaf_data(right) + BTRFS_LEAF_DATA_SIZE(root) -
data_copy_size, btrfs_leaf_data(l) +
leaf_data_end(root, l), data_copy_size);
rt_data_off = BTRFS_LEAF_DATA_SIZE(root) -
btrfs_item_end_nr(l, mid);
for (i = 0; i < nritems; i++) {
struct btrfs_item *item = btrfs_item_nr(right, i);
u32 ioff;
if (!right->map_token) {
map_extent_buffer(right, (unsigned long)item,
sizeof(struct btrfs_item),
&right->map_token, &right->kaddr,
&right->map_start, &right->map_len,
KM_USER1);
}
ioff = btrfs_item_offset(right, item);
btrfs_set_item_offset(right, item, ioff + rt_data_off);
}
if (right->map_token) {
unmap_extent_buffer(right, right->map_token, KM_USER1);
right->map_token = NULL;
}
btrfs_set_header_nritems(l, mid);
ret = 0;
btrfs_item_key(right, &disk_key, 0);
wret = insert_ptr(trans, root, path, &disk_key, right->start,
path->slots[1] + 1, 1);
if (wret)
ret = wret;
btrfs_mark_buffer_dirty(right);
btrfs_mark_buffer_dirty(l);
BUG_ON(path->slots[0] != slot);
ret = btrfs_update_ref(trans, root, l, right, 0, nritems);
BUG_ON(ret);
if (mid <= slot) {
btrfs_tree_unlock(path->nodes[0]);
free_extent_buffer(path->nodes[0]);
path->nodes[0] = right;
path->slots[0] -= mid;
path->slots[1] += 1;
} else {
btrfs_tree_unlock(right);
free_extent_buffer(right);
}
BUG_ON(path->slots[0] < 0);
return ret;
}
/* /*
* split the path's leaf in two, making sure there is at least data_size * split the path's leaf in two, making sure there is at least data_size
* available for the resulting leaf level of the path. * available for the resulting leaf level of the path.
...@@ -2708,14 +2841,10 @@ static noinline int split_leaf(struct btrfs_trans_handle *trans, ...@@ -2708,14 +2841,10 @@ static noinline int split_leaf(struct btrfs_trans_handle *trans,
int mid; int mid;
int slot; int slot;
struct extent_buffer *right; struct extent_buffer *right;
int data_copy_size;
int rt_data_off;
int i;
int ret = 0; int ret = 0;
int wret; int wret;
int double_split; int double_split;
int num_doubles = 0; int num_doubles = 0;
struct btrfs_disk_key disk_key;
/* first try to make some room by pushing left and right */ /* first try to make some room by pushing left and right */
if (data_size && ins_key->type != BTRFS_DIR_ITEM_KEY) { if (data_size && ins_key->type != BTRFS_DIR_ITEM_KEY) {
...@@ -2767,11 +2896,14 @@ static noinline int split_leaf(struct btrfs_trans_handle *trans, ...@@ -2767,11 +2896,14 @@ static noinline int split_leaf(struct btrfs_trans_handle *trans,
write_extent_buffer(right, root->fs_info->chunk_tree_uuid, write_extent_buffer(right, root->fs_info->chunk_tree_uuid,
(unsigned long)btrfs_header_chunk_tree_uuid(right), (unsigned long)btrfs_header_chunk_tree_uuid(right),
BTRFS_UUID_SIZE); BTRFS_UUID_SIZE);
if (mid <= slot) { if (mid <= slot) {
if (nritems == 1 || if (nritems == 1 ||
leaf_space_used(l, mid, nritems - mid) + data_size > leaf_space_used(l, mid, nritems - mid) + data_size >
BTRFS_LEAF_DATA_SIZE(root)) { BTRFS_LEAF_DATA_SIZE(root)) {
if (slot >= nritems) { if (slot >= nritems) {
struct btrfs_disk_key disk_key;
btrfs_cpu_key_to_disk(&disk_key, ins_key); btrfs_cpu_key_to_disk(&disk_key, ins_key);
btrfs_set_header_nritems(right, 0); btrfs_set_header_nritems(right, 0);
wret = insert_ptr(trans, root, path, wret = insert_ptr(trans, root, path,
...@@ -2799,6 +2931,8 @@ static noinline int split_leaf(struct btrfs_trans_handle *trans, ...@@ -2799,6 +2931,8 @@ static noinline int split_leaf(struct btrfs_trans_handle *trans,
if (leaf_space_used(l, 0, mid) + data_size > if (leaf_space_used(l, 0, mid) + data_size >
BTRFS_LEAF_DATA_SIZE(root)) { BTRFS_LEAF_DATA_SIZE(root)) {
if (!extend && data_size && slot == 0) { if (!extend && data_size && slot == 0) {
struct btrfs_disk_key disk_key;
btrfs_cpu_key_to_disk(&disk_key, ins_key); btrfs_cpu_key_to_disk(&disk_key, ins_key);
btrfs_set_header_nritems(right, 0); btrfs_set_header_nritems(right, 0);
wret = insert_ptr(trans, root, path, wret = insert_ptr(trans, root, path,
...@@ -2831,76 +2965,16 @@ static noinline int split_leaf(struct btrfs_trans_handle *trans, ...@@ -2831,76 +2965,16 @@ static noinline int split_leaf(struct btrfs_trans_handle *trans,
} }
} }
} }
nritems = nritems - mid;
btrfs_set_header_nritems(right, nritems);
data_copy_size = btrfs_item_end_nr(l, mid) - leaf_data_end(root, l);
copy_extent_buffer(right, l, btrfs_item_nr_offset(0),
btrfs_item_nr_offset(mid),
nritems * sizeof(struct btrfs_item));
copy_extent_buffer(right, l,
btrfs_leaf_data(right) + BTRFS_LEAF_DATA_SIZE(root) -
data_copy_size, btrfs_leaf_data(l) +
leaf_data_end(root, l), data_copy_size);
rt_data_off = BTRFS_LEAF_DATA_SIZE(root) -
btrfs_item_end_nr(l, mid);
for (i = 0; i < nritems; i++) { ret = copy_for_split(trans, root, path, l, right, slot, mid, nritems);
struct btrfs_item *item = btrfs_item_nr(right, i);
u32 ioff;
if (!right->map_token) {
map_extent_buffer(right, (unsigned long)item,
sizeof(struct btrfs_item),
&right->map_token, &right->kaddr,
&right->map_start, &right->map_len,
KM_USER1);
}
ioff = btrfs_item_offset(right, item);
btrfs_set_item_offset(right, item, ioff + rt_data_off);
}
if (right->map_token) {
unmap_extent_buffer(right, right->map_token, KM_USER1);
right->map_token = NULL;
}
btrfs_set_header_nritems(l, mid);
ret = 0;
btrfs_item_key(right, &disk_key, 0);
wret = insert_ptr(trans, root, path, &disk_key, right->start,
path->slots[1] + 1, 1);
if (wret)
ret = wret;
btrfs_mark_buffer_dirty(right);
btrfs_mark_buffer_dirty(l);
BUG_ON(path->slots[0] != slot);
ret = btrfs_update_ref(trans, root, l, right, 0, nritems);
BUG_ON(ret); BUG_ON(ret);
if (mid <= slot) {
btrfs_tree_unlock(path->nodes[0]);
free_extent_buffer(path->nodes[0]);
path->nodes[0] = right;
path->slots[0] -= mid;
path->slots[1] += 1;
} else {
btrfs_tree_unlock(right);
free_extent_buffer(right);
}
BUG_ON(path->slots[0] < 0);
if (double_split) { if (double_split) {
BUG_ON(num_doubles != 0); BUG_ON(num_doubles != 0);
num_doubles++; num_doubles++;
goto again; goto again;
} }
return ret; return ret;
} }
...@@ -3382,39 +3456,27 @@ int btrfs_insert_some_items(struct btrfs_trans_handle *trans, ...@@ -3382,39 +3456,27 @@ int btrfs_insert_some_items(struct btrfs_trans_handle *trans,
} }
/* /*
* Given a key and some data, insert items into the tree. * this is a helper for btrfs_insert_empty_items, the main goal here is
* This does all the path init required, making room in the tree if needed. * to save stack depth by doing the bulk of the work in a function
* that doesn't call btrfs_search_slot
*/ */
int btrfs_insert_empty_items(struct btrfs_trans_handle *trans, static noinline_for_stack int
struct btrfs_root *root, setup_items_for_insert(struct btrfs_trans_handle *trans,
struct btrfs_path *path, struct btrfs_root *root, struct btrfs_path *path,
struct btrfs_key *cpu_key, u32 *data_size, struct btrfs_key *cpu_key, u32 *data_size,
int nr) u32 total_data, u32 total_size, int nr)
{ {
struct extent_buffer *leaf;
struct btrfs_item *item; struct btrfs_item *item;
int ret = 0;
int slot;
int slot_orig;
int i; int i;
u32 nritems; u32 nritems;
u32 total_size = 0;
u32 total_data = 0;
unsigned int data_end; unsigned int data_end;
struct btrfs_disk_key disk_key; struct btrfs_disk_key disk_key;
int ret;
struct extent_buffer *leaf;
int slot;
for (i = 0; i < nr; i++)
total_data += data_size[i];
total_size = total_data + (nr * sizeof(struct btrfs_item));
ret = btrfs_search_slot(trans, root, cpu_key, path, total_size, 1);
if (ret == 0)
return -EEXIST;
if (ret < 0)
goto out;
slot_orig = path->slots[0];
leaf = path->nodes[0]; leaf = path->nodes[0];
slot = path->slots[0];
nritems = btrfs_header_nritems(leaf); nritems = btrfs_header_nritems(leaf);
data_end = leaf_data_end(root, leaf); data_end = leaf_data_end(root, leaf);
...@@ -3426,9 +3488,6 @@ int btrfs_insert_empty_items(struct btrfs_trans_handle *trans, ...@@ -3426,9 +3488,6 @@ int btrfs_insert_empty_items(struct btrfs_trans_handle *trans,
BUG(); BUG();
} }
slot = path->slots[0];
BUG_ON(slot < 0);
if (slot != nritems) { if (slot != nritems) {
unsigned int old_data = btrfs_item_end_nr(leaf, slot); unsigned int old_data = btrfs_item_end_nr(leaf, slot);
...@@ -3484,11 +3543,13 @@ int btrfs_insert_empty_items(struct btrfs_trans_handle *trans, ...@@ -3484,11 +3543,13 @@ int btrfs_insert_empty_items(struct btrfs_trans_handle *trans,
data_end -= data_size[i]; data_end -= data_size[i];
btrfs_set_item_size(leaf, item, data_size[i]); btrfs_set_item_size(leaf, item, data_size[i]);
} }
btrfs_set_header_nritems(leaf, nritems + nr); btrfs_set_header_nritems(leaf, nritems + nr);
btrfs_mark_buffer_dirty(leaf); btrfs_mark_buffer_dirty(leaf);
ret = 0; ret = 0;
if (slot == 0) { if (slot == 0) {
struct btrfs_disk_key disk_key;
btrfs_cpu_key_to_disk(&disk_key, cpu_key); btrfs_cpu_key_to_disk(&disk_key, cpu_key);
ret = fixup_low_keys(trans, root, path, &disk_key, 1); ret = fixup_low_keys(trans, root, path, &disk_key, 1);
} }
...@@ -3497,6 +3558,43 @@ int btrfs_insert_empty_items(struct btrfs_trans_handle *trans, ...@@ -3497,6 +3558,43 @@ int btrfs_insert_empty_items(struct btrfs_trans_handle *trans,
btrfs_print_leaf(root, leaf); btrfs_print_leaf(root, leaf);
BUG(); BUG();
} }
return ret;
}
/*
* Given a key and some data, insert items into the tree.
* This does all the path init required, making room in the tree if needed.
*/
int btrfs_insert_empty_items(struct btrfs_trans_handle *trans,
struct btrfs_root *root,
struct btrfs_path *path,
struct btrfs_key *cpu_key, u32 *data_size,
int nr)
{
struct extent_buffer *leaf;
int ret = 0;
int slot;
int i;
u32 total_size = 0;
u32 total_data = 0;
for (i = 0; i < nr; i++)
total_data += data_size[i];
total_size = total_data + (nr * sizeof(struct btrfs_item));
ret = btrfs_search_slot(trans, root, cpu_key, path, total_size, 1);
if (ret == 0)
return -EEXIST;
if (ret < 0)
goto out;
leaf = path->nodes[0];
slot = path->slots[0];
BUG_ON(slot < 0);
ret = setup_items_for_insert(trans, root, path, cpu_key, data_size,
total_data, total_size, nr);
out: out:
btrfs_unlock_up_safe(path, 1); btrfs_unlock_up_safe(path, 1);
return ret; return ret;
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment