65#define LZO_PAGE_SIZE 4096
92 ERR(
"out of memory\n");
109 switch (
Vcb->superblock.csum_type) {
129 switch (
Vcb->superblock.csum_type) {
160 WARN(
"hash was invalid\n");
173 WARN(
"hash was invalid\n");
183 switch (
Vcb->superblock.csum_type) {
203 switch (
Vcb->superblock.csum_type) {
238 bool checksum_error =
false;
245 WARN(
"stripe %u returned error %08lx\n",
j,
context->stripes[
j].iosb.Status);
247 return context->stripes[
j].iosb.Status;
261 checksum_error =
true;
264 checksum_error =
true;
271 checksum_error =
true;
274 ERR(
"check_csum returned %08lx\n",
Status);
287 bool recovered =
false;
291 ERR(
"out of memory\n");
298 Vcb->superblock.node_size, (
uint8_t*)t2,
false);
300 WARN(
"sync_read_phys returned %08lx\n",
Status);
307 ERR(
"recovering from checksum error at %I64x, device %I64x\n",
addr,
devices[
stripe]->devitem.dev_id);
312 t2,
Vcb->superblock.node_size);
314 WARN(
"write_data_phys returned %08lx\n",
Status);
320 }
else if (t2->address !=
addr || checksum_error)
329 ERR(
"unrecoverable checksum error at %I64x\n",
addr);
342 ERR(
"out of memory\n");
348 bool recovered =
false;
354 Vcb->superblock.sector_size,
sector,
false);
356 WARN(
"sync_read_phys returned %08lx\n",
Status);
369 WARN(
"write_data_phys returned %08lx\n",
Status);
401 WARN(
"stripe %u returned error %08lx\n",
i,
context->stripes[
i].iosb.Status);
403 return context->stripes[
i].iosb.Status;
417 ERR(
"unrecoverable checksum error at %I64x, device %I64x\n",
addr,
devices[
stripe]->devitem.dev_id);
419 if (checksum_error) {
423 WARN(
"address of tree was %I64x, not %I64x as expected\n", th->
address,
addr);
447 ERR(
"unrecoverable checksum error at %I64x, device %I64x\n",
addr,
devices[
stripe]->devitem.dev_id);
459 ERR(
"check_csum returned %08lx\n",
Status);
471 bool checksum_error =
false;
476 WARN(
"stripe %u returned error %08lx\n",
j,
context->stripes[
j].iosb.Status);
478 return context->stripes[
j].iosb.Status;
487 checksum_error =
true;
490 WARN(
"address of tree was %I64x, not %I64x as expected\n", th->
address,
addr);
491 checksum_error =
true;
495 checksum_error =
true;
502 checksum_error =
true;
504 ERR(
"check_csum returned %08lx\n",
Status);
516 bool recovered =
false;
520 ERR(
"out of memory\n");
538 Vcb->superblock.node_size, (
uint8_t*)t2,
false);
540 WARN(
"sync_read_phys returned %08lx\n",
Status);
547 ERR(
"recovering from checksum error at %I64x, device %I64x\n",
addr,
devices[
stripe +
j]->devitem.dev_id);
552 cis[
stripe + badsubstripe].
offset + off, t2,
Vcb->superblock.node_size);
554 WARN(
"write_data_phys returned %08lx\n",
Status);
560 }
else if (t2->address !=
addr || checksum_error)
569 ERR(
"unrecoverable checksum error at %I64x\n",
addr);
582 ERR(
"out of memory\n");
590 bool recovered =
false;
609 Vcb->superblock.sector_size,
sector,
false);
611 WARN(
"sync_read_phys returned %08lx\n",
Status);
616 ERR(
"recovering from checksum error at %I64x, device %I64x\n",
addr + ((
uint64_t)
i <<
Vcb->sector_shift),
devices[stripe2 +
j]->devitem.dev_id);
619 if (!
Vcb->readonly && !
devices[stripe2 + badsubstripe]->readonly &&
devices[stripe2 + badsubstripe]->devobj) {
621 cis[stripe2 + badsubstripe].
offset + off,
sector,
Vcb->superblock.sector_size);
623 WARN(
"write_data_phys returned %08lx\n",
Status);
654 bool checksum_error =
false;
657 bool no_success =
true;
661 WARN(
"stripe %u returned error %08lx\n",
j,
context->stripes[
j].iosb.Status);
663 return context->stripes[
j].iosb.Status;
676 le =
c->partial_stripes.
Flink;
677 while (le != &
c->partial_stripes) {
685 while (runlength != 0) {
697 uint64_t runend = runstart + (runlength <<
Vcb->sector_shift);
719 checksum_error =
true;
720 if (!no_success && !degraded)
723 checksum_error =
true;
724 if (!no_success && !degraded)
732 WARN(
"checksum error\n");
733 checksum_error =
true;
735 ERR(
"check_csum returned %08lx\n",
Status);
739 checksum_error =
true;
747 bool recovered =
false,
first =
true, failed =
false;
752 ERR(
"out of memory\n");
768 ERR(
"sync_read_phys returned %08lx\n",
Status);
778 ERR(
"sync_read_phys returned %08lx\n",
Status);
784 do_xor(t2, t2 +
Vcb->superblock.node_size,
Vcb->superblock.node_size);
800 ERR(
"recovering from checksum error at %I64x, device %I64x\n",
addr,
devices[
stripe]->devitem.dev_id);
807 WARN(
"write_data_phys returned %08lx\n",
Status);
815 ERR(
"unrecoverable checksum error at %I64x\n",
addr);
828 ERR(
"out of memory\n");
844 bool recovered =
false,
first =
true, failed =
false;
855 ERR(
"sync_read_phys returned %08lx\n",
Status);
864 sector +
Vcb->superblock.sector_size,
false);
866 ERR(
"sync_read_phys returned %08lx\n",
Status);
894 WARN(
"write_data_phys returned %08lx\n",
Status);
919 if (missing1 == num_stripes - 2 || missing2 == num_stripes - 2) {
920 uint16_t missing = missing1 == (num_stripes - 2) ? missing2 : missing1;
975 denom =
gdiv(1, gyx ^ 1);
976 a =
gmul(gyx, denom);
999 bool checksum_error =
false;
1002 bool no_success =
true;
1006 WARN(
"stripe %u returned error %08lx\n",
j,
context->stripes[
j].iosb.Status);
1010 return context->stripes[
j].iosb.Status;
1023 le =
c->partial_stripes.
Flink;
1024 while (le != &
c->partial_stripes) {
1032 while (runlength != 0) {
1044 uint64_t runend = runstart + (runlength <<
Vcb->sector_shift);
1066 checksum_error =
true;
1070 checksum_error =
true;
1079 WARN(
"checksum error\n");
1080 checksum_error =
true;
1082 ERR(
"check_csum returned %08lx\n",
Status);
1085 }
else if (degraded)
1086 checksum_error =
true;
1088 if (!checksum_error)
1093 uint16_t k, physstripe, parity1, parity2, error_stripe = 0;
1095 bool recovered =
false, failed =
false;
1100 ERR(
"out of memory\n");
1114 if (
j != physstripe) {
1117 sector + (
k *
Vcb->superblock.node_size),
false);
1119 ERR(
"sync_read_phys returned %08lx\n",
Status);
1148 Vcb->superblock.node_size);
1159 ERR(
"recovering from checksum error at %I64x, device %I64x\n",
addr,
devices[physstripe]->devitem.dev_id);
1167 WARN(
"write_data_phys returned %08lx\n",
Status);
1176 bool read_q =
false;
1182 ERR(
"sync_read_phys returned %08lx\n",
Status);
1213 ERR(
"recovering from checksum error at %I64x, device %I64x\n",
addr,
devices[physstripe]->devitem.dev_id);
1221 WARN(
"write_data_phys returned %08lx\n",
Status);
1226 if (
devices[error_stripe_phys] &&
devices[error_stripe_phys]->devobj) {
1228 ERR(
"recovering from parity error at %I64x, device %I64x\n",
addr,
devices[error_stripe_phys]->devitem.dev_id);
1237 Vcb->superblock.node_size);
1240 Vcb->superblock.node_size);
1245 devices[error_stripe_phys]->devitem.dev_id);
1254 if (!
Vcb->readonly &&
devices[error_stripe_phys] &&
devices[error_stripe_phys]->devobj && !
devices[error_stripe_phys]->readonly) {
1256 sector + (error_stripe *
Vcb->superblock.node_size),
Vcb->superblock.node_size);
1258 WARN(
"write_data_phys returned %08lx\n",
Status);
1267 ERR(
"unrecoverable checksum error at %I64x\n",
addr);
1280 ERR(
"out of memory\n");
1286 uint16_t physstripe, parity1, parity2;
1298 bool recovered =
false, failed =
false;
1307 if (
j != physstripe) {
1312 ERR(
"sync_read_phys returned %08lx\n",
Status);
1349 ERR(
"recovering from checksum error at %I64x, device %I64x\n",
addr + ((
uint64_t)
i <<
Vcb->sector_shift),
1350 devices[physstripe]->devitem.dev_id);
1358 WARN(
"write_data_phys returned %08lx\n",
Status);
1366 bool read_q =
false;
1372 ERR(
"sync_read_phys returned %08lx\n",
Status);
1405 ERR(
"recovering from checksum error at %I64x, device %I64x\n",
1414 WARN(
"write_data_phys returned %08lx\n",
Status);
1419 if (
devices[error_stripe_phys] &&
devices[error_stripe_phys]->devobj) {
1421 ERR(
"recovering from parity error at %I64x, device %I64x\n",
addr + ((
uint64_t)
i <<
Vcb->sector_shift),
1422 devices[error_stripe_phys]->devitem.dev_id);
1431 Vcb->superblock.sector_size);
1434 Vcb->superblock.sector_size);
1438 ERR(
"recovering from checksum error at %I64x, device %I64x\n",
1440 devices[error_stripe_phys]->devitem.dev_id);
1449 if (!
Vcb->readonly &&
devices[error_stripe_phys] &&
devices[error_stripe_phys]->devobj && !
devices[error_stripe_phys]->readonly) {
1451 sector + ((
unsigned int)error_stripe <<
Vcb->sector_shift),
Vcb->superblock.sector_size);
1453 WARN(
"write_data_phys returned %08lx\n",
Status);
1462 ERR(
"unrecoverable checksum error at %I64x\n",
addr + ((
uint64_t)
i <<
Vcb->sector_shift));
1487 uint16_t i, startoffstripe, allowed_missing, missing_devices = 0;
1493 if (
Vcb->log_to_phys_loaded) {
1498 ERR(
"get_chunk_from_address failed\n");
1515 while (le != &
Vcb->sys_chunks) {
1528 ERR(
"out of memory\n");
1544 ERR(
"could not find chunk for %I64x in bootstrap\n",
addr);
1557 allowed_missing = 0;
1560 allowed_missing = 1;
1563 allowed_missing = 1;
1566 allowed_missing = 1;
1569 allowed_missing = 2;
1572 allowed_missing = 2;
1575 allowed_missing = 3;
1578 allowed_missing = 0;
1588 ERR(
"out of memory\n");
1602 context.sector_size =
Vcb->superblock.sector_size;
1632 ERR(
"out of memory\n");
1641 ERR(
"out of memory\n");
1655 ERR(
"MmProbeAndLockPages threw exception %08lx\n",
Status);
1663 if (startoffstripe >
i)
1665 else if (startoffstripe ==
i)
1666 context.stripes[
i].stripestart = startoff;
1670 if (endoffstripe >
i)
1672 else if (endoffstripe ==
i)
1673 context.stripes[
i].stripeend = endoff + 1;
1681 ERR(
"IoAllocateMdl failed\n");
1692 ERR(
"out of memory\n");
1711 stripeoff[
stripe] += readlen;
1713 }
else if (
length - pos < ci->stripe_length) {
1741 orig_ls =
c->last_stripe;
1758 ERR(
"out of memory\n");
1766 context.startoffstripe = startoffstripe;
1777 ERR(
"out of memory\n");
1791 ERR(
"MmProbeAndLockPages threw exception %08lx\n",
Status);
1800 ERR(
"out of memory\n");
1811 bool stripeset =
false;
1813 if (startoffstripe >
i)
1815 else if (startoffstripe ==
i)
1820 if (endoffstripe >
i)
1822 else if (endoffstripe ==
i)
1829 context.stripes[
i+
j].stripestart = sstart;
1833 if (sstart !=
send) {
1837 ERR(
"IoAllocateMdl failed\n");
1853 context.stripes[
i+
j].stripestart = sstart;
1858 if (sstart !=
send) {
1862 ERR(
"IoAllocateMdl failed\n");
1876 ERR(
"could not find stripe to read\n");
1885 ERR(
"out of memory\n");
1905 stripeoff[
stripe] += readlen;
1907 }
else if (
length - pos < ci->stripe_length) {
1930 orig_ls =
i =
c->last_stripe;
1938 ERR(
"no devices available to service request\n");
1954 ERR(
"out of memory\n");
1961 ERR(
"IoAllocateMdl failed\n");
1971 ERR(
"IoAllocateMdl failed\n");
1985 ERR(
"MmProbeAndLockPages threw exception %08lx\n",
Status);
1995 bool need_dummy =
false;
2004 ERR(
"out of memory\n");
2013 ERR(
"out of memory\n");
2027 ERR(
"MmProbeAndLockPages threw exception %08lx\n",
Status);
2044 if (
i == startoffstripe) {
2073 for (
i = 0;
i < startoffstripe;
i++) {
2103 if (endoffstripe ==
i) {
2106 }
else if (endoffstripe >
i)
2120 false,
false,
NULL);
2123 ERR(
"IoAllocateMdl failed\n");
2135 ERR(
"out of memory\n");
2144 ERR(
"IoAllocateMdl failed\n");
2158 ERR(
"out of memory\n");
2183 stripeoff[
stripe] = readlen;
2197 stripeoff[
stripe] = readlen;
2236 stripeoff[
stripe] += readlen;
2254 bool need_dummy =
false;
2263 ERR(
"out of memory\n");
2272 ERR(
"out of memory\n");
2286 ERR(
"MmProbeAndLockPages threw exception %08lx\n",
Status);
2303 if (
i == startoffstripe) {
2332 for (
i = 0;
i < startoffstripe;
i++) {
2365 if (endoffstripe ==
i) {
2368 }
else if (endoffstripe >
i)
2384 ERR(
"IoAllocateMdl failed\n");
2396 ERR(
"out of memory\n");
2405 ERR(
"IoAllocateMdl failed\n");
2419 ERR(
"out of memory\n");
2444 stripeoff[
stripe] = readlen;
2458 stripeoff[
stripe] = readlen;
2468 while (
stripe != parity1) {
2505 stripeoff[
stripe] += readlen;
2531 if (missing_devices > allowed_missing) {
2532 ERR(
"not enough devices to service request (%u missing)\n", missing_devices);
2551 ERR(
"IoAllocateIrp failed\n");
2559 ERR(
"IoMakeAssociatedIrp failed\n");
2572 if (!
context.stripes[
i].Irp->AssociatedIrp.SystemBuffer) {
2573 ERR(
"out of memory\n");
2599 need_to_wait =
false;
2603 need_to_wait =
true;
2625 ERR(
"read_data_raid0 returned %08lx\n",
Status);
2641 ERR(
"read_data_raid10 returned %08lx\n",
Status);
2656 ERR(
"read_data_dup returned %08lx\n",
Status);
2669 Status =
read_data_raid5(
Vcb, file_read ?
context.va :
buf,
addr,
length, &
context, ci,
devices,
offset,
generation,
c, missing_devices > 0 ?
true :
false);
2671 ERR(
"read_data_raid5 returned %08lx\n",
Status);
2684 Status =
read_data_raid6(
Vcb, file_read ?
context.va :
buf,
addr,
length, &
context, ci,
devices,
offset,
generation,
c, missing_devices > 0 ?
true :
false);
2686 ERR(
"read_data_raid6 returned %08lx\n",
Status);
2724 if (!
Vcb->log_to_phys_loaded)
2739 TRACE(
"tried to read beyond end of stream\n");
2744 WARN(
"tried to read zero bytes\n");
2748 if (
start + length < fcb->adsdata.Length)
2756 if (pbr) *pbr = readlen;
2810 WARN(
"Tried to read beyond end of file\n");
2836 last_end =
ext->offset +
len;
2840 if (
ext->offset > last_end &&
ext->offset >
start + bytes_read) {
2852 WARN(
"Encryption not supported\n");
2858 WARN(
"Other encodings not supported\n");
2889 ERR(
"out of memory\n");
2894 decomp_alloc =
true;
2896 decomp =
data + bytes_read;
2897 decomp_alloc =
false;
2903 ERR(
"zlib_decompress returned %08lx\n",
Status);
2909 ERR(
"extent data was truncated\n");
2918 ERR(
"lzo_decompress returned %08lx\n",
Status);
2925 ERR(
"zstd_decompress returned %08lx\n",
Status);
2954 ERR(
"out of memory\n");
2959 rp->mdl = (
Irp &&
Irp->MdlAddress) ?
true :
false;
2960 rp->extents[0].off =
start + bytes_read -
ext->offset;
2962 rp->num_extents = 1;
2963 rp->csum_free =
false;
2972 if (rp->addr & (
fcb->
Vcb->superblock.sector_size - 1)) {
2973 rp->bumpoff = rp->addr & (
fcb->
Vcb->superblock.sector_size - 1);
2974 rp->addr -= rp->bumpoff;
2983 (
length & (
fcb->
Vcb->superblock.sector_size - 1)) == 0) {
2984 rp->buf =
data + bytes_read;
2985 rp->buf_free =
false;
2988 rp->buf_free =
true;
2991 ERR(
"out of memory\n");
3003 ERR(
"get_chunk_from_address(%I64x) failed\n", rp->addr);
3018 rp->csum =
ext->csum;
3022 rp->data =
data + bytes_read;
3024 rp->extents[0].ed_offset = ed2->
offset;
3025 rp->extents[0].ed_size = ed2->
size;
3026 rp->extents[0].ed_num_bytes = ed2->
num_bytes;
3030 bytes_read += rp->read;
3052 WARN(
"Unsupported extent data type %u\n", ed->
type);
3057 last_end =
ext->offset +
len;
3070 le = read_parts.
Flink->Flink;
3071 while (le != &read_parts) {
3083 rp2->
c = last_rp->
c;
3088 if (last_rp->
csum) {
3093 ERR(
"out of memory\n");
3109 ERR(
"out of memory\n");
3159 le = read_parts.
Flink;
3160 while (le != &read_parts) {
3163 Status =
read_data(
fcb->
Vcb, rp->
addr, rp->
to_read, rp->
csum,
false, rp->
buf, rp->
c,
NULL,
Irp, 0, rp->
mdl,
3166 ERR(
"read_data returned %08lx\n",
Status);
3178 ULONG outlen, inlen, off2;
3198 partlen = *(
uint32_t*)(buf2 + inoff);
3200 if (partlen < inlen) {
3202 inoff += partlen +
sizeof(
uint32_t);
3203 inlen -= partlen +
sizeof(
uint32_t);
3211 buf2 = &buf2[inoff];
3226 ERR(
"out of memory\n");
3233 ERR(
"out of memory\n");
3248 inpageoff, &ccj->
cj);
3250 ERR(
"add_calc_job_decomp returned %08lx\n",
Status);
3269 if (
length > 0 &&
start + bytes_read < fcb->inode_item.st_size) {