71#define BALANCE_UNIT 0x100000
84 ERR(
"out of memory\n");
96 ERR(
"delete_tree_item returned %08lx\n",
Status);
107 c->used -=
Vcb->superblock.node_size;
143 ERR(
"out of memory\n");
156 ERR(
"unexpected tree type %x\n", secttype);
169 if (inline_rc < ei->refcount) {
179 ERR(
"out of memory\n");
191 ERR(
"delete_tree_item returned %08lx\n",
Status);
197 ERR(
"out of memory\n");
209 ERR(
"delete_tree_item returned %08lx\n",
Status);
235 while (le !=
items) {
248 searchkey.
offset = 0xffffffffffffffff;
252 ERR(
"find_item returned %08lx\n",
Status);
263 ERR(
"EXTENT_ITEM for %I64x found, but tree flag not set\n",
address);
267 ERR(
"could not find valid EXTENT_ITEM for address %I64x\n",
address);
273 ERR(
"add_metadata_reloc returned %08lx\n",
Status);
292 bool inserted =
false;
295 ref->hash =
ref->tbr.offset;
297 ref->hash =
ref->parent->new_address;
300 while (le != &newlist) {
327 bool all_inline =
true;
339 while (le != &mr->
refs) {
351 if ((
ULONG)(inline_len + 1 + extlen) > (
Vcb->superblock.node_size >> 2)) {
353 first_noninline =
ref;
355 inline_len += extlen + 1;
363 ERR(
"out of memory\n");
382 while (le != &mr->
refs) {
385 if (
ref == first_noninline)
414 ERR(
"insert_tree_item returned %08lx\n",
Status);
422 while (le != &mr->
refs) {
428 ERR(
"insert_tree_item returned %08lx\n",
Status);
434 ERR(
"insert_tree_item returned %08lx\n",
Status);
458 ERR(
"increase_extent_refcount returned %08lx\n",
Status);
467 ERR(
"decrease_extent_refcount returned %08lx\n",
Status);
495 ERR(
"increase_extent_refcount returned %08lx\n",
Status);
504 ERR(
"decrease_extent_refcount returned %08lx\n",
Status);
515 le =
c->changed_extents.
Flink;
517 while (le != &
c->changed_extents) {
524 while (le2 != &ce->
refs) {
577 while (le !=
items) {
584 ERR(
"out of memory\n");
591 ERR(
"read_data returned %08lx\n",
Status);
598 if (data_items && mr->
data->
level == 0) {
599 le2 = data_items->
Flink;
600 while (le2 != data_items) {
626 while (le2 != &mr->
refs) {
635 firstitem = (
KEY*)&mr->
data[1];
637 le3 =
Vcb->roots.Flink;
641 if (
r2->id ==
ref->tbr.offset) {
650 ERR(
"could not find subvol with id %I64x\n",
ref->tbr.offset);
656 ERR(
"find_item_to_level returned %08lx\n",
Status);
661 while (
t &&
t->header.level < mr->
data->
level + 1) {
672 ERR(
"add_metadata_reloc_parent returned %08lx\n",
Status);
683 ERR(
"add_metadata_reloc_parent returned %08lx\n",
Status);
697 while (le !=
items) {
706 le2 =
Vcb->trees_ptrs[
hash >> 24];
709 while (le2 != &
Vcb->trees_hash) {
712 if (
t->header.address == mr->
address) {
715 }
else if (
t->hash >
hash)
727 while (le !=
items) {
748 newchunk->
used +=
Vcb->superblock.node_size;
759 le2 =
Vcb->chunks.Flink;
760 while (le2 != &
Vcb->chunks) {
768 c2->
used +=
Vcb->superblock.node_size;
788 ERR(
"alloc_chunk returned %08lx\n",
Status);
800 ERR(
"could not find address in new chunk\n");
804 newchunk->
used +=
Vcb->superblock.node_size;
816 while (le2 != &mr->
refs) {
823 for (
i = 0;
i <
ref->parent->data->num_items;
i++) {
830 if (
ref->parent->t) {
833 le3 =
ref->parent->t->itemlist.Flink;
834 while (le3 != &
ref->parent->t->itemlist) {
849 le3 =
Vcb->roots.Flink;
850 while (le3 != &
Vcb->roots) {
853 if (
r2->id ==
ref->tbr.offset) {
864 if (
r ==
Vcb->root_root)
866 else if (
r ==
Vcb->chunk_root)
868 else if (
r->root_item.block_number == mr->
address) {
876 searchkey.
offset = 0xffffffffffffffff;
880 ERR(
"find_item returned %08lx\n",
Status);
885 ERR(
"could not find ROOT_ITEM for tree %I64x\n", searchkey.
obj_id);
892 ERR(
"out of memory\n");
901 ERR(
"delete_tree_item returned %08lx\n",
Status);
907 ERR(
"insert_tree_item returned %08lx\n",
Status);
944 if (t2->
hash >> 24 ==
h)
956 if (!
Vcb->trees_ptrs[
h]) {
959 le2 =
Vcb->trees_hash.Flink;
964 if (
Vcb->trees_ptrs[h2]) {
965 le2 =
Vcb->trees_ptrs[h2];
973 le2 =
Vcb->trees_ptrs[
h];
976 while (le2 != &
Vcb->trees_hash) {
994 if (data_items &&
level == 0) {
995 le2 = data_items->
Flink;
997 while (le2 != data_items) {
1029 ERR(
"out of memory\n");
1042 bool inserted =
false;
1044 le2 = tree_writes.
Flink;
1045 while (le2 != &tree_writes) {
1068 ERR(
"do_tree_writes returned %08lx\n",
Status);
1073 while (le !=
items) {
1078 ERR(
"add_metadata_reloc_extent_item returned %08lx\n",
Status);
1108 TRACE(
"chunk %I64x\n",
c->offset);
1117 searchkey.
offset = 0xffffffffffffffff;
1121 ERR(
"find_item returned %08lx\n",
Status);
1132 bool tree =
false, skinny =
false;
1149 ERR(
"add_metadata_reloc returned %08lx\n",
Status);
1175 ERR(
"write_metadata_items returned %08lx\n",
Status);
1181 Vcb->need_write =
true;
1187 ERR(
"do_write returned %08lx\n",
Status);
1228 le =
Vcb->roots.Flink;
1229 while (le != &
Vcb->roots) {
1232 if (
r2->id == edr->
root) {
1241 ERR(
"could not find subvol %I64x\n", edr->
root);
1251 ERR(
"find_item returned %08lx\n",
Status);
1261 ERR(
"could not find EXTENT_DATA for inode %I64x in root %I64x\n", searchkey.
obj_id,
r->id);
1283 ERR(
"out of memory\n");
1293 ERR(
"add_metadata_reloc_parent returned %08lx\n",
Status);
1327 ERR(
"out of memory\n");
1338 ERR(
"delete_tree_item returned %08lx\n",
Status);
1367 if (sectlen >
len) {
1380 inline_rc += edr->
count;
1384 ERR(
"data_reloc_add_tree_edr returned %08lx\n",
Status);
1393 ERR(
"out of memory\n");
1399 inline_rc +=
ref->sdr.count;
1403 ERR(
"add_metadata_reloc_parent returned %08lx\n",
Status);
1412 ERR(
"unexpected tree type %x\n", secttype);
1421 if (inline_rc < ei->refcount) {
1431 ERR(
"data_reloc_add_tree_edr returned %08lx\n",
Status);
1437 ERR(
"delete_tree_item returned %08lx\n",
Status);
1446 ERR(
"out of memory\n");
1456 ERR(
"add_metadata_reloc_parent returned %08lx\n",
Status);
1466 ERR(
"delete_tree_item returned %08lx\n",
Status);
1492 bool inserted =
false;
1497 ref->hash =
ref->parent->new_address;
1500 while (le != &newlist) {
1517 while (le != &newlist) {
1520 if (le->
Flink != &newlist) {
1546 bool all_inline =
true;
1556 while (le != &dr->
refs) {
1562 rc +=
ref->edr.count;
1569 if ((
ULONG)(inline_len + 1 + extlen) > (
Vcb->superblock.node_size >> 2)) {
1571 first_noninline =
ref;
1573 inline_len += extlen + 1;
1581 ERR(
"out of memory\n");
1591 while (le != &dr->
refs) {
1594 if (
ref == first_noninline)
1620 ERR(
"insert_tree_item returned %08lx\n",
Status);
1627 while (le != &dr->
refs) {
1635 ERR(
"out of memory\n");
1643 ERR(
"insert_tree_item returned %08lx\n",
Status);
1651 ERR(
"out of memory\n");
1655 *sdr =
ref->sdr.count;
1659 ERR(
"insert_tree_item returned %08lx\n",
Status);
1681 TRACE(
"chunk %I64x\n",
c->offset);
1691 searchkey.
offset = 0xffffffffffffffff;
1695 ERR(
"find_item returned %08lx\n",
Status);
1719 ERR(
"add_data_reloc returned %08lx\n",
Status);
1726 if (
loaded >= 0x1000000 || num_loaded >= 100)
1746 ERR(
"out of memory\n");
1752 while (le != &
items) {
1776 le2 =
Vcb->chunks.Flink;
1777 while (le2 != &
Vcb->chunks) {
1805 ERR(
"alloc_chunk returned %08lx\n",
Status);
1817 ERR(
"could not find address in new chunk\n");
1837 ERR(
"out of memory\n");
1844 ERR(
"out of memory\n");
1859 ERR(
"find_item returned %08lx\n",
Status);
1878 (
ULONG)(((ce -
cs) *
Vcb->csum_size) >>
Vcb->sector_shift));
1897 while (runlength != 0) {
1898 if (
index >= bmplen)
1901 if (
index + runlength >= bmplen) {
1902 runlength = bmplen -
index;
1908 if (
index > lastoff) {
1909 ULONG off = lastoff;
1924 ERR(
"read_data returned %08lx\n",
Status);
1933 ERR(
"write_data_complete returned %08lx\n",
Status);
1959 ERR(
"read_data returned %08lx\n",
Status);
1968 ERR(
"write_data_complete returned %08lx\n",
Status);
1976 }
while (runlength > 0);
1986 if (lastoff < dr->
size >>
Vcb->sector_shift) {
1987 ULONG off = lastoff;
2001 ERR(
"read_data returned %08lx\n",
Status);
2008 ERR(
"write_data_complete returned %08lx\n",
Status);
2025 ERR(
"write_metadata_items returned %08lx\n",
Status);
2030 while (le != &
items) {
2035 ERR(
"add_data_reloc_extent_item returned %08lx\n",
Status);
2042 le =
c->changed_extents.
Flink;
2043 while (le != &
c->changed_extents) {
2050 while (le2 != &
items) {
2068 Vcb->need_write =
true;
2073 le =
Vcb->chunks.Flink;
2074 while (le != &
Vcb->chunks) {
2092 while (le3 != &
items) {
2117 ERR(
"do_write returned %08lx\n",
Status);
2126 le =
Vcb->all_fcbs.Flink;
2127 while (le != &
Vcb->all_fcbs) {
2143 while (le3 != &
items) {
2228 opts = &
Vcb->balance.opts[
sort];
2245 for (
i = 0;
i <
c->chunk_item->num_stripes;
i++) {
2246 if (cis[
i].dev_id == opts->
devid) {
2263 factor =
c->chunk_item->num_stripes;
2265 factor =
c->chunk_item->num_stripes /
c->chunk_item->sub_stripes;
2267 factor =
c->chunk_item->num_stripes - 1;
2269 factor =
c->chunk_item->num_stripes - 2;
2273 physsize =
c->chunk_item->size /
factor;
2275 for (
i = 0;
i <
c->chunk_item->num_stripes;
i++) {
2301 if (
c->used > 0 &&
usage == 0)
2325 if (
args->usage_start == 0) {
2361 if (
args->limit_start == 0) {
2392 ERR(
"find_item returned %08lx\n",
Status);
2399 ERR(
"delete_tree_item returned %08lx\n",
Status);
2406 ERR(
"out of memory\n");
2430 ERR(
"insert_tree_item returned %08lx\n",
Status);
2441 ERR(
"do_write returned %08lx\n",
Status);
2464 ERR(
"find_item returned %08lx\n",
Status);
2471 ERR(
"delete_tree_item returned %08lx\n",
Status);
2477 ERR(
"do_write returned %08lx\n",
Status);
2564 ERR(
"out of memory\n");
2593 if (
Vcb->need_write) {
2597 ERR(
"do_write returned %08lx\n",
Status);
2614 ERR(
"find_item returned %08lx\n",
Status);
2622 ERR(
"delete_tree_item returned %08lx\n",
Status);
2635 ERR(
"find_item returned %08lx\n",
Status);
2643 ERR(
"delete_tree_item returned %08lx\n",
Status);
2650 Vcb->superblock.num_devices--;
2651 Vcb->superblock.total_bytes -=
dev->devitem.num_bytes;
2652 Vcb->devices_loaded--;
2660 ERR(
"do_write returned %08lx\n",
Status);
2667 if (!
dev->readonly &&
dev->devobj) {
2670 WARN(
"remove_superblocks returned %08lx\n",
Status);
2697 ERR(
"IoGetDeviceObjectPointer returned %08lx\n",
Status);
2703 ERR(
"IOCTL_MOUNTDEV_QUERY_DEVICE_NAME returned %08lx\n",
Status);
2710 ERR(
"out of memory\n");
2714 ERR(
"IOCTL_MOUNTDEV_QUERY_DEVICE_NAME returned %08lx\n",
Status);
2723 WARN(
"mountmgr_add_drive_letter returned %08lx\n",
Status);
2747 vde->
device->Characteristics &= ~FILE_REMOVABLE_MEDIA;
2768 if (
dev->trim && !
dev->readonly && !
Vcb->options.no_trim)
2784 le =
Vcb->devices.Flink;
2785 while (le != &
Vcb->devices) {
2813 dev->num_trim_entries = 0;
2821 ERR(
"find_item returned %08lx\n",
Status);
2851 if (lastoff < dev->devitem.num_bytes)
2854 if (
dev->num_trim_entries == 0)
2861 ERR(
"out of memory\n");
2876 le =
dev->trim_list.Flink;
2877 while (le != &
dev->trim_list) {
2889 WARN(
"IOCTL_STORAGE_MANAGE_DATA_SET_ATTRIBUTES returned %08lx\n",
Status);
2899 dev->num_trim_entries = 0;
2918 le =
Vcb->chunks.Flink;
2919 while (le != &
Vcb->chunks) {
2938 Vcb->balance.chunks_left--;
2951 ERR(
"balance_data_chunk returned %08lx\n",
Status);
2961 Vcb->balance.stopping =
true;
2963 if (
Vcb->balance.stopping)
2975 ERR(
"do_write returned %08lx\n",
Status);
2992 ERR(
"alloc_chunk returned %08lx\n",
Status);
3010 le =
Vcb->chunks.Flink;
3011 while (le != &
Vcb->chunks) {
3016 for (
n = 0;
n <
c->chunk_item->num_stripes;
n++) {
3020 if (stripe_size == 0) {
3024 factor =
c->chunk_item->num_stripes;
3026 factor =
c->chunk_item->num_stripes /
c->chunk_item->sub_stripes;
3028 factor =
c->chunk_item->num_stripes - 1;
3030 factor =
c->chunk_item->num_stripes - 2;
3034 stripe_size =
c->chunk_item->size /
factor;
3052 uint64_t num_chunks[3], okay_metadata_chunks = 0, okay_data_chunks = 0, okay_system_chunks = 0;
3053 uint64_t old_data_flags = 0, old_metadata_flags = 0, old_system_flags = 0;
3056 Vcb->balance.balance_num++;
3058 Vcb->balance.stopping =
false;
3062 old_data_flags =
Vcb->data_flags;
3069 old_metadata_flags =
Vcb->metadata_flags;
3074 old_system_flags =
Vcb->system_flags;
3085 num_chunks[0] = num_chunks[1] = num_chunks[2] = 0;
3086 Vcb->balance.total_chunks =
Vcb->balance.chunks_left = 0;
3092 if (!
Vcb->readonly) {
3093 if (!
Vcb->balance.removing && !
Vcb->balance.shrinking) {
3096 ERR(
"add_balance_item returned %08lx\n",
Status);
3101 if (
Vcb->need_write) {
3107 ERR(
"do_write returned %08lx\n",
Status);
3117 if (
Vcb->balance.stopping)
3122 le =
Vcb->chunks.Flink;
3123 while (le != &
Vcb->chunks) {
3136 ERR(
"unexpected chunk type %I64x\n",
c->chunk_item->type);
3146 Vcb->balance.total_chunks++;
3147 Vcb->balance.chunks_left++;
3149 okay_metadata_chunks++;
3153 okay_system_chunks++;
3155 if (!
c->cache_loaded) {
3159 ERR(
"load_cache_chunk returned %08lx\n",
Status);
3175 if (okay_metadata_chunks == 0 || okay_data_chunks == 0 || okay_system_chunks == 0) {
3176 bool consolidated =
false;
3179 if (okay_metadata_chunks == 0) {
3184 c->balance_num =
Vcb->balance.balance_num;
3186 ERR(
"alloc_chunk returned %08lx\n",
Status);
3197 ERR(
"try_consolidation returned %08lx\n",
Status);
3201 c->balance_num =
Vcb->balance.balance_num;
3203 consolidated =
true;
3205 if (
Vcb->balance.stopping)
3210 if (okay_data_chunks == 0) {
3215 c->balance_num =
Vcb->balance.balance_num;
3217 ERR(
"alloc_chunk returned %08lx\n",
Status);
3228 ERR(
"try_consolidation returned %08lx\n",
Status);
3232 c->balance_num =
Vcb->balance.balance_num;
3234 consolidated =
true;
3236 if (
Vcb->balance.stopping)
3241 if (okay_system_chunks == 0) {
3246 c->balance_num =
Vcb->balance.balance_num;
3248 ERR(
"alloc_chunk returned %08lx\n",
Status);
3259 ERR(
"try_consolidation returned %08lx\n",
Status);
3263 c->balance_num =
Vcb->balance.balance_num;
3265 consolidated =
true;
3267 if (
Vcb->balance.stopping)
3276 while (le != &chunks) {
3288 while (le != &chunks) {
3300 ERR(
"balance_data_chunk returned %08lx\n",
Status);
3308 Vcb->balance.stopping =
true;
3310 if (
Vcb->balance.stopping)
3315 c->space_changed =
true;
3318 if (
Vcb->balance.stopping)
3324 c->list_entry_balance.Flink =
NULL;
3326 Vcb->balance.chunks_left--;
3344 ERR(
"balance_metadata_chunk returned %08lx\n",
Status);
3352 Vcb->balance.stopping =
true;
3354 if (
Vcb->balance.stopping)
3359 c->space_changed =
true;
3362 if (
Vcb->balance.stopping)
3365 c->list_entry_balance.Flink =
NULL;
3367 Vcb->balance.chunks_left--;
3371 if (!
Vcb->readonly) {
3374 while (le != &chunks) {
3379 c->list_entry_balance.Flink =
NULL;
3382 if (old_data_flags != 0)
3383 Vcb->data_flags = old_data_flags;
3385 if (old_metadata_flags != 0)
3386 Vcb->metadata_flags = old_metadata_flags;
3388 if (old_system_flags != 0)
3389 Vcb->system_flags = old_system_flags;
3392 if (
Vcb->balance.removing) {
3397 le =
Vcb->devices.Flink;
3398 while (le != &
Vcb->devices) {
3410 if (
Vcb->balance.chunks_left == 0) {
3414 ERR(
"finish_removing_device returned %08lx\n",
Status);
3422 }
else if (
Vcb->balance.shrinking) {
3427 le =
Vcb->devices.Flink;
3428 while (le != &
Vcb->devices) {
3440 ERR(
"could not find device %I64x\n",
Vcb->balance.opts[0].devid);
3448 WARN(
"regenerate_space_list returned %08lx\n",
Status);
3454 dev->devitem.num_bytes =
Vcb->balance.opts[0].drange_start;
3458 ERR(
"update_dev_item returned %08lx\n",
Status);
3464 WARN(
"regenerate_space_list returned %08lx\n",
Status);
3466 Vcb->superblock.total_bytes -=
old_size -
dev->devitem.num_bytes;
3470 ERR(
"do_write returned %08lx\n",
Status);
3483 ERR(
"remove_balance_item returned %08lx\n",
Status);
3488 if (
Vcb->trim && !
Vcb->options.no_trim) {
3491 le =
Vcb->devices.Flink;
3492 while (le != &
Vcb->devices) {
3524 WARN(
"cannot start balance while locked\n");
3528 if (
Vcb->scrub.thread) {
3529 WARN(
"cannot start balance while scrub running\n");
3533 if (
Vcb->balance.thread) {
3534 WARN(
"balance already running\n");
3546 for (
i = 0;
i < 3;
i++) {