71 #ifndef _MSC_VER // not in mingw yet 72 #define DEVICE_DSM_FLAG_TRIM_NOT_FS_ALLOCATED 0x80000000 75 #define BALANCE_UNIT 0x100000 // only read 1 MB at a time 88 ERR(
"out of memory\n");
100 ERR(
"delete_tree_item returned %08lx\n",
Status);
111 c->used -=
Vcb->superblock.node_size;
147 ERR(
"out of memory\n");
160 ERR(
"unexpected tree type %x\n", secttype);
173 if (inline_rc < ei->refcount) {
183 ERR(
"out of memory\n");
195 ERR(
"delete_tree_item returned %08lx\n",
Status);
201 ERR(
"out of memory\n");
213 ERR(
"delete_tree_item returned %08lx\n",
Status);
239 while (le !=
items) {
252 searchkey.
offset = 0xffffffffffffffff;
256 ERR(
"find_item returned %08lx\n",
Status);
267 ERR(
"EXTENT_ITEM for %I64x found, but tree flag not set\n",
address);
271 ERR(
"could not find valid EXTENT_ITEM for address %I64x\n",
address);
277 ERR(
"add_metadata_reloc returned %08lx\n",
Status);
296 bool inserted =
false;
299 ref->hash =
ref->tbr.offset;
301 ref->hash =
ref->parent->new_address;
304 while (le != &newlist) {
331 bool all_inline =
true;
343 while (le != &mr->
refs) {
355 if ((
ULONG)(inline_len + 1 + extlen) > (
Vcb->superblock.node_size >> 2)) {
357 first_noninline =
ref;
359 inline_len += extlen + 1;
367 ERR(
"out of memory\n");
386 while (le != &mr->
refs) {
389 if (
ref == first_noninline)
418 ERR(
"insert_tree_item returned %08lx\n",
Status);
426 while (le != &mr->
refs) {
432 ERR(
"insert_tree_item returned %08lx\n",
Status);
438 ERR(
"insert_tree_item returned %08lx\n",
Status);
462 ERR(
"increase_extent_refcount returned %08lx\n",
Status);
471 ERR(
"decrease_extent_refcount returned %08lx\n",
Status);
499 ERR(
"increase_extent_refcount returned %08lx\n",
Status);
508 ERR(
"decrease_extent_refcount returned %08lx\n",
Status);
519 le =
c->changed_extents.
Flink;
521 while (le != &
c->changed_extents) {
528 while (le2 != &ce->
refs) {
581 while (le !=
items) {
588 ERR(
"out of memory\n");
595 ERR(
"read_data returned %08lx\n",
Status);
602 if (data_items && mr->
data->
level == 0) {
603 le2 = data_items->
Flink;
604 while (le2 != data_items) {
630 while (le2 != &mr->
refs) {
639 firstitem = (
KEY*)&mr->
data[1];
641 le3 =
Vcb->roots.Flink;
645 if (
r2->id ==
ref->tbr.offset) {
654 ERR(
"could not find subvol with id %I64x\n",
ref->tbr.offset);
660 ERR(
"find_item_to_level returned %08lx\n",
Status);
665 while (
t &&
t->header.level < mr->
data->
level + 1) {
676 ERR(
"add_metadata_reloc_parent returned %08lx\n",
Status);
687 ERR(
"add_metadata_reloc_parent returned %08lx\n",
Status);
701 while (le !=
items) {
710 le2 =
Vcb->trees_ptrs[
hash >> 24];
713 while (le2 != &
Vcb->trees_hash) {
716 if (
t->header.address == mr->
address) {
719 }
else if (
t->hash >
hash)
731 while (le !=
items) {
752 newchunk->
used +=
Vcb->superblock.node_size;
763 le2 =
Vcb->chunks.Flink;
764 while (le2 != &
Vcb->chunks) {
772 c2->
used +=
Vcb->superblock.node_size;
792 ERR(
"alloc_chunk returned %08lx\n",
Status);
804 ERR(
"could not find address in new chunk\n");
808 newchunk->
used +=
Vcb->superblock.node_size;
820 while (le2 != &mr->
refs) {
827 for (
i = 0;
i <
ref->parent->data->num_items;
i++) {
834 if (
ref->parent->t) {
837 le3 =
ref->parent->t->itemlist.Flink;
838 while (le3 != &
ref->parent->t->itemlist) {
853 le3 =
Vcb->roots.Flink;
854 while (le3 != &
Vcb->roots) {
857 if (
r2->id ==
ref->tbr.offset) {
868 if (
r ==
Vcb->root_root)
870 else if (
r ==
Vcb->chunk_root)
872 else if (
r->root_item.block_number == mr->
address) {
880 searchkey.
offset = 0xffffffffffffffff;
884 ERR(
"find_item returned %08lx\n",
Status);
889 ERR(
"could not find ROOT_ITEM for tree %I64x\n", searchkey.
obj_id);
896 ERR(
"out of memory\n");
905 ERR(
"delete_tree_item returned %08lx\n",
Status);
911 ERR(
"insert_tree_item returned %08lx\n",
Status);
948 if (t2->
hash >> 24 ==
h)
960 if (!
Vcb->trees_ptrs[
h]) {
963 le2 =
Vcb->trees_hash.Flink;
968 if (
Vcb->trees_ptrs[h2]) {
969 le2 =
Vcb->trees_ptrs[h2];
977 le2 =
Vcb->trees_ptrs[
h];
980 while (le2 != &
Vcb->trees_hash) {
998 if (data_items &&
level == 0) {
999 le2 = data_items->
Flink;
1001 while (le2 != data_items) {
1033 ERR(
"out of memory\n");
1046 bool inserted =
false;
1048 le2 = tree_writes.
Flink;
1049 while (le2 != &tree_writes) {
1072 ERR(
"do_tree_writes returned %08lx\n",
Status);
1077 while (le !=
items) {
1082 ERR(
"add_metadata_reloc_extent_item returned %08lx\n",
Status);
1112 TRACE(
"chunk %I64x\n",
c->offset);
1121 searchkey.
offset = 0xffffffffffffffff;
1125 ERR(
"find_item returned %08lx\n",
Status);
1136 bool tree =
false, skinny =
false;
1153 ERR(
"add_metadata_reloc returned %08lx\n",
Status);
1179 ERR(
"write_metadata_items returned %08lx\n",
Status);
1185 Vcb->need_write =
true;
1191 ERR(
"do_write returned %08lx\n",
Status);
1232 le =
Vcb->roots.Flink;
1233 while (le != &
Vcb->roots) {
1236 if (
r2->id == edr->
root) {
1245 ERR(
"could not find subvol %I64x\n", edr->
root);
1255 ERR(
"find_item returned %08lx\n",
Status);
1265 ERR(
"could not find EXTENT_DATA for inode %I64x in root %I64x\n", searchkey.
obj_id,
r->id);
1287 ERR(
"out of memory\n");
1297 ERR(
"add_metadata_reloc_parent returned %08lx\n",
Status);
1331 ERR(
"out of memory\n");
1342 ERR(
"delete_tree_item returned %08lx\n",
Status);
1371 if (sectlen >
len) {
1384 inline_rc += edr->
count;
1388 ERR(
"data_reloc_add_tree_edr returned %08lx\n",
Status);
1397 ERR(
"out of memory\n");
1403 inline_rc +=
ref->sdr.count;
1407 ERR(
"add_metadata_reloc_parent returned %08lx\n",
Status);
1416 ERR(
"unexpected tree type %x\n", secttype);
1425 if (inline_rc < ei->refcount) {
1435 ERR(
"data_reloc_add_tree_edr returned %08lx\n",
Status);
1441 ERR(
"delete_tree_item returned %08lx\n",
Status);
1450 ERR(
"out of memory\n");
1460 ERR(
"add_metadata_reloc_parent returned %08lx\n",
Status);
1470 ERR(
"delete_tree_item returned %08lx\n",
Status);
1496 bool inserted =
false;
1501 ref->hash =
ref->parent->new_address;
1504 while (le != &newlist) {
1521 while (le != &newlist) {
1524 if (le->
Flink != &newlist) {
1550 bool all_inline =
true;
1560 while (le != &dr->
refs) {
1566 rc +=
ref->edr.count;
1573 if ((
ULONG)(inline_len + 1 + extlen) > (
Vcb->superblock.node_size >> 2)) {
1575 first_noninline =
ref;
1577 inline_len += extlen + 1;
1585 ERR(
"out of memory\n");
1595 while (le != &dr->
refs) {
1598 if (
ref == first_noninline)
1624 ERR(
"insert_tree_item returned %08lx\n",
Status);
1631 while (le != &dr->
refs) {
1639 ERR(
"out of memory\n");
1647 ERR(
"insert_tree_item returned %08lx\n",
Status);
1655 ERR(
"out of memory\n");
1659 *sdr =
ref->sdr.count;
1663 ERR(
"insert_tree_item returned %08lx\n",
Status);
1685 TRACE(
"chunk %I64x\n",
c->offset);
1695 searchkey.
offset = 0xffffffffffffffff;
1699 ERR(
"find_item returned %08lx\n",
Status);
1723 ERR(
"add_data_reloc returned %08lx\n",
Status);
1730 if (
loaded >= 0x1000000 || num_loaded >= 100)
1750 ERR(
"out of memory\n");
1756 while (le != &
items) {
1780 le2 =
Vcb->chunks.Flink;
1781 while (le2 != &
Vcb->chunks) {
1809 ERR(
"alloc_chunk returned %08lx\n",
Status);
1821 ERR(
"could not find address in new chunk\n");
1837 bmplen = (
ULONG)(dr->
size /
Vcb->superblock.sector_size);
1841 ERR(
"out of memory\n");
1848 ERR(
"out of memory\n");
1863 ERR(
"find_item returned %08lx\n",
Status);
1882 (
ULONG)((ce -
cs) *
Vcb->csum_size /
Vcb->superblock.sector_size));
1901 while (runlength != 0) {
1902 if (
index >= bmplen)
1905 if (
index + runlength >= bmplen) {
1906 runlength = bmplen -
index;
1912 if (
index > lastoff) {
1913 ULONG off = lastoff;
1928 ERR(
"read_data returned %08lx\n",
Status);
1937 ERR(
"write_data_complete returned %08lx\n",
Status);
1963 ERR(
"read_data returned %08lx\n",
Status);
1972 ERR(
"write_data_complete returned %08lx\n",
Status);
1980 }
while (runlength > 0);
1990 if (lastoff < dr->
size /
Vcb->superblock.sector_size) {
1991 ULONG off = lastoff;
2005 ERR(
"read_data returned %08lx\n",
Status);
2012 ERR(
"write_data_complete returned %08lx\n",
Status);
2029 ERR(
"write_metadata_items returned %08lx\n",
Status);
2034 while (le != &
items) {
2039 ERR(
"add_data_reloc_extent_item returned %08lx\n",
Status);
2046 le =
c->changed_extents.
Flink;
2047 while (le != &
c->changed_extents) {
2054 while (le2 != &
items) {
2072 Vcb->need_write =
true;
2077 le =
Vcb->chunks.Flink;
2078 while (le != &
Vcb->chunks) {
2096 while (le3 != &
items) {
2121 ERR(
"do_write returned %08lx\n",
Status);
2130 le =
Vcb->all_fcbs.Flink;
2131 while (le != &
Vcb->all_fcbs) {
2147 while (le3 != &
items) {
2232 opts = &
Vcb->balance.opts[
sort];
2249 for (
i = 0;
i <
c->chunk_item->num_stripes;
i++) {
2250 if (cis[
i].dev_id == opts->
devid) {
2267 factor =
c->chunk_item->num_stripes;
2269 factor =
c->chunk_item->num_stripes /
c->chunk_item->sub_stripes;
2271 factor =
c->chunk_item->num_stripes - 1;
2273 factor =
c->chunk_item->num_stripes - 2;
2277 physsize =
c->chunk_item->size /
factor;
2279 for (
i = 0;
i <
c->chunk_item->num_stripes;
i++) {
2305 if (
c->used > 0 &&
usage == 0)
2329 if (
args->usage_start == 0) {
2365 if (
args->limit_start == 0) {
2396 ERR(
"find_item returned %08lx\n",
Status);
2403 ERR(
"delete_tree_item returned %08lx\n",
Status);
2410 ERR(
"out of memory\n");
2434 ERR(
"insert_tree_item returned %08lx\n",
Status);
2445 ERR(
"do_write returned %08lx\n",
Status);
2468 ERR(
"find_item returned %08lx\n",
Status);
2475 ERR(
"delete_tree_item returned %08lx\n",
Status);
2481 ERR(
"do_write returned %08lx\n",
Status);
2568 ERR(
"out of memory\n");
2597 if (
Vcb->need_write) {
2601 ERR(
"do_write returned %08lx\n",
Status);
2618 ERR(
"find_item returned %08lx\n",
Status);
2626 ERR(
"delete_tree_item returned %08lx\n",
Status);
2639 ERR(
"find_item returned %08lx\n",
Status);
2647 ERR(
"delete_tree_item returned %08lx\n",
Status);
2654 Vcb->superblock.num_devices--;
2655 Vcb->superblock.total_bytes -=
dev->devitem.num_bytes;
2656 Vcb->devices_loaded--;
2664 ERR(
"do_write returned %08lx\n",
Status);
2671 if (!
dev->readonly &&
dev->devobj) {
2674 WARN(
"remove_superblocks returned %08lx\n",
Status);
2701 ERR(
"IoGetDeviceObjectPointer returned %08lx\n",
Status);
2707 ERR(
"IOCTL_MOUNTDEV_QUERY_DEVICE_NAME returned %08lx\n",
Status);
2714 ERR(
"out of memory\n");
2718 ERR(
"IOCTL_MOUNTDEV_QUERY_DEVICE_NAME returned %08lx\n",
Status);
2727 WARN(
"mountmgr_add_drive_letter returned %08lx\n",
Status);
2772 if (
dev->trim && !
dev->readonly && !
Vcb->options.no_trim)
2788 le =
Vcb->devices.Flink;
2789 while (le != &
Vcb->devices) {
2817 dev->num_trim_entries = 0;
2825 ERR(
"find_item returned %08lx\n",
Status);
2855 if (lastoff < dev->devitem.num_bytes)
2858 if (
dev->num_trim_entries == 0)
2865 ERR(
"out of memory\n");
2880 le =
dev->trim_list.Flink;
2881 while (le != &
dev->trim_list) {
2893 WARN(
"IOCTL_STORAGE_MANAGE_DATA_SET_ATTRIBUTES returned %08lx\n",
Status);
2903 dev->num_trim_entries = 0;
2922 le =
Vcb->chunks.Flink;
2923 while (le != &
Vcb->chunks) {
2942 Vcb->balance.chunks_left--;
2955 ERR(
"balance_data_chunk returned %08lx\n",
Status);
2965 Vcb->balance.stopping =
true;
2967 if (
Vcb->balance.stopping)
2979 ERR(
"do_write returned %08lx\n",
Status);
2996 ERR(
"alloc_chunk returned %08lx\n",
Status);
3014 le =
Vcb->chunks.Flink;
3015 while (le != &
Vcb->chunks) {
3020 for (
n = 0;
n <
c->chunk_item->num_stripes;
n++) {
3024 if (stripe_size == 0) {
3028 factor =
c->chunk_item->num_stripes;
3030 factor =
c->chunk_item->num_stripes /
c->chunk_item->sub_stripes;
3032 factor =
c->chunk_item->num_stripes - 1;
3034 factor =
c->chunk_item->num_stripes - 2;
3038 stripe_size =
c->chunk_item->size /
factor;
3056 uint64_t num_chunks[3], okay_metadata_chunks = 0, okay_data_chunks = 0, okay_system_chunks = 0;
3057 uint64_t old_data_flags = 0, old_metadata_flags = 0, old_system_flags = 0;
3060 Vcb->balance.balance_num++;
3062 Vcb->balance.stopping =
false;
3066 old_data_flags =
Vcb->data_flags;
3073 old_metadata_flags =
Vcb->metadata_flags;
3078 old_system_flags =
Vcb->system_flags;
3089 num_chunks[0] = num_chunks[1] = num_chunks[2] = 0;
3090 Vcb->balance.total_chunks =
Vcb->balance.chunks_left = 0;
3096 if (!
Vcb->readonly) {
3097 if (!
Vcb->balance.removing && !
Vcb->balance.shrinking) {
3100 ERR(
"add_balance_item returned %08lx\n",
Status);
3105 if (
Vcb->need_write) {
3111 ERR(
"do_write returned %08lx\n",
Status);
3121 if (
Vcb->balance.stopping)
3126 le =
Vcb->chunks.Flink;
3127 while (le != &
Vcb->chunks) {
3140 ERR(
"unexpected chunk type %I64x\n",
c->chunk_item->type);
3150 Vcb->balance.total_chunks++;
3151 Vcb->balance.chunks_left++;
3153 okay_metadata_chunks++;
3157 okay_system_chunks++;
3159 if (!
c->cache_loaded) {
3163 ERR(
"load_cache_chunk returned %08lx\n",
Status);
3179 if (okay_metadata_chunks == 0 || okay_data_chunks == 0 || okay_system_chunks == 0) {
3180 bool consolidated =
false;
3183 if (okay_metadata_chunks == 0) {
3188 c->balance_num =
Vcb->balance.balance_num;
3190 ERR(
"alloc_chunk returned %08lx\n",
Status);
3201 ERR(
"try_consolidation returned %08lx\n",
Status);
3205 c->balance_num =
Vcb->balance.balance_num;
3207 consolidated =
true;
3209 if (
Vcb->balance.stopping)
3214 if (okay_data_chunks == 0) {
3219 c->balance_num =
Vcb->balance.balance_num;
3221 ERR(
"alloc_chunk returned %08lx\n",
Status);
3232 ERR(
"try_consolidation returned %08lx\n",
Status);
3236 c->balance_num =
Vcb->balance.balance_num;
3238 consolidated =
true;
3240 if (
Vcb->balance.stopping)
3245 if (okay_system_chunks == 0) {
3250 c->balance_num =
Vcb->balance.balance_num;
3252 ERR(
"alloc_chunk returned %08lx\n",
Status);
3263 ERR(
"try_consolidation returned %08lx\n",
Status);
3267 c->balance_num =
Vcb->balance.balance_num;
3269 consolidated =
true;
3271 if (
Vcb->balance.stopping)
3280 while (le != &chunks) {
3292 while (le != &chunks) {
3304 ERR(
"balance_data_chunk returned %08lx\n",
Status);
3312 Vcb->balance.stopping =
true;
3314 if (
Vcb->balance.stopping)
3319 c->space_changed =
true;
3322 if (
Vcb->balance.stopping)
3328 c->list_entry_balance.Flink =
NULL;
3330 Vcb->balance.chunks_left--;
3348 ERR(
"balance_metadata_chunk returned %08lx\n",
Status);
3356 Vcb->balance.stopping =
true;
3358 if (
Vcb->balance.stopping)
3363 c->space_changed =
true;
3366 if (
Vcb->balance.stopping)
3369 c->list_entry_balance.Flink =
NULL;
3371 Vcb->balance.chunks_left--;
3375 if (!
Vcb->readonly) {
3378 while (le != &chunks) {
3383 c->list_entry_balance.Flink =
NULL;
3386 if (old_data_flags != 0)
3387 Vcb->data_flags = old_data_flags;
3389 if (old_metadata_flags != 0)
3390 Vcb->metadata_flags = old_metadata_flags;
3392 if (old_system_flags != 0)
3393 Vcb->system_flags = old_system_flags;
3396 if (
Vcb->balance.removing) {
3401 le =
Vcb->devices.Flink;
3402 while (le != &
Vcb->devices) {
3414 if (
Vcb->balance.chunks_left == 0) {
3418 ERR(
"finish_removing_device returned %08lx\n",
Status);
3426 }
else if (
Vcb->balance.shrinking) {
3431 le =
Vcb->devices.Flink;
3432 while (le != &
Vcb->devices) {
3444 ERR(
"could not find device %I64x\n",
Vcb->balance.opts[0].devid);
3452 WARN(
"regenerate_space_list returned %08lx\n",
Status);
3457 old_size =
dev->devitem.num_bytes;
3458 dev->devitem.num_bytes =
Vcb->balance.opts[0].drange_start;
3462 ERR(
"update_dev_item returned %08lx\n",
Status);
3463 dev->devitem.num_bytes = old_size;
3468 WARN(
"regenerate_space_list returned %08lx\n",
Status);
3470 Vcb->superblock.total_bytes -= old_size -
dev->devitem.num_bytes;
3474 ERR(
"do_write returned %08lx\n",
Status);
3487 ERR(
"remove_balance_item returned %08lx\n",
Status);
3492 if (
Vcb->trim && !
Vcb->options.no_trim) {
3495 le =
Vcb->devices.Flink;
3496 while (le != &
Vcb->devices) {
3528 WARN(
"cannot start balance while locked\n");
3532 if (
Vcb->scrub.thread) {
3533 WARN(
"cannot start balance while scrub running\n");
3537 if (
Vcb->balance.thread) {
3538 WARN(
"balance already running\n");
3550 for (
i = 0;
i < 3;
i++) {
3615 Vcb->balance.paused =
false;
3616 Vcb->balance.removing =
false;
3617 Vcb->balance.shrinking =
false;
3625 ERR(
"PsCreateSystemThread returned %08lx\n",
Status);
3646 ERR(
"find_item returned %08lx\n",
Status);
3651 TRACE(
"no balance item found\n");
3674 for (
i = 0;
i < 3;
i++) {
3688 Vcb->balance.opts[
i].usage_start = 0;
3689 Vcb->balance.opts[
i].usage_end = 90;
3694 if (
Vcb->readonly ||
Vcb->options.skip_balance)
3695 Vcb->balance.paused =
true;
3697 Vcb->balance.paused =
false;
3699 Vcb->balance.removing =
false;
3700 Vcb->balance.shrinking =
false;
3708 ERR(
"PsCreateSystemThread returned %08lx\n",
Status);
3721 if (!
Vcb->balance.thread) {
3734 if (
Vcb->balance.removing)
3737 if (
Vcb->balance.shrinking)
3757 if (!
Vcb->balance.thread)
3760 if (
Vcb->balance.paused)
3763 Vcb->balance.paused =
true;
3773 if (!
Vcb->balance.thread)
3776 if (!
Vcb->balance.paused)
3782 Vcb->balance.paused =
false;
3792 if (!
Vcb->balance.thread)
3795 Vcb->balance.paused =
false;
3796 Vcb->balance.stopping =
true;
3824 if (
Vcb->readonly) {
3831 le =
Vcb->devices.Flink;
3832 while (le != &
Vcb->devices) {
3846 WARN(
"device %I64x not found\n", devid);
3850 if (!
dev->readonly) {
3851 if (num_rw_devices == 1) {
3853 WARN(
"not removing last non-readonly device\n");
3857 if (num_rw_devices == 4 &&