read.c File Reference

#include "btrfs_drv.h"

Include dependency graph for read.c:

Go to the source code of this file.

Classes
struct	read_data_stripe
struct	read_data_context

Macros
#define	LINUX_PAGE_SIZE   4096

Enumerations
enum	read_data_status {   ReadDataStatus_Pending, ReadDataStatus_Success, ReadDataStatus_Error, ReadDataStatus_MissingDevice,   ReadDataStatus_Skip }

Functions
	_Function_class_ (IO_COMPLETION_ROUTINE)
NTSTATUS	check_csum (device_extension *Vcb, UINT8 *data, UINT32 sectors, UINT32 *csum)
static NTSTATUS	read_data_dup (device_extension *Vcb, UINT8 *buf, UINT64 addr, read_data_context *context, CHUNK_ITEM *ci, device **devices, UINT64 generation)
static NTSTATUS	read_data_raid0 (device_extension *Vcb, UINT8 *buf, UINT64 addr, UINT32 length, read_data_context *context, CHUNK_ITEM *ci, device **devices, UINT64 generation, UINT64 offset)
static NTSTATUS	read_data_raid10 (device_extension *Vcb, UINT8 *buf, UINT64 addr, UINT32 length, read_data_context *context, CHUNK_ITEM *ci, device **devices, UINT64 generation, UINT64 offset)
static NTSTATUS	read_data_raid5 (device_extension *Vcb, UINT8 *buf, UINT64 addr, UINT32 length, read_data_context *context, CHUNK_ITEM *ci, device **devices, UINT64 offset, UINT64 generation, chunk *c, BOOL degraded)
void	raid6_recover2 (UINT8 *sectors, UINT16 num_stripes, ULONG sector_size, UINT16 missing1, UINT16 missing2, UINT8 *out)
static NTSTATUS	read_data_raid6 (device_extension *Vcb, UINT8 *buf, UINT64 addr, UINT32 length, read_data_context *context, CHUNK_ITEM *ci, device **devices, UINT64 offset, UINT64 generation, chunk *c, BOOL degraded)
NTSTATUS	read_data (_In_ device_extension *Vcb, _In_ UINT64 addr, _In_ UINT32 length, _In_reads_bytes_opt_(length *sizeof(UINT32)/Vcb->superblock.sector_size) UINT32 *csum, _In_ BOOL is_tree, _Out_writes_bytes_(length) UINT8 *buf, _In_opt_ chunk *c, _Out_opt_ chunk **pc, _In_opt_ PIRP Irp, _In_ UINT64 generation, _In_ BOOL file_read, _In_ ULONG priority)
NTSTATUS	read_stream (fcb *fcb, UINT8 *data, UINT64 start, ULONG length, ULONG *pbr)
NTSTATUS	read_file (fcb *fcb, UINT8 *data, UINT64 start, UINT64 length, ULONG *pbr, PIRP Irp)
NTSTATUS	do_read (PIRP Irp, BOOLEAN wait, ULONG *bytes_read)
	_Dispatch_type_ (IRP_MJ_READ)

Variables
BOOL	diskacc
tPsUpdateDiskCounters	fPsUpdateDiskCounters
tCcCopyReadEx	fCcCopyReadEx
tFsRtlUpdateDiskCounters	fFsRtlUpdateDiskCounters

Macro Definition Documentation

LINUX_PAGE_SIZE

#define LINUX_PAGE_SIZE   4096

Definition at line 63 of file read.c.

Enumeration Type Documentation

read_data_status

enum read_data_status

Enumerator
ReadDataStatus_Pending
ReadDataStatus_Success
ReadDataStatus_Error
ReadDataStatus_MissingDevice
ReadDataStatus_Skip

Definition at line 20 of file read.c.

                      {
    ReadDataStatus_Pending,
    ReadDataStatus_Success,
    ReadDataStatus_Error,
    ReadDataStatus_MissingDevice,
    ReadDataStatus_Skip
};

Function Documentation

_Dispatch_type_()

_Dispatch_type_

(

IRP_MJ_READ

)

Definition at line 3285 of file read.c.

                                                               {
    device_extension* Vcb = DeviceObject->DeviceExtension;
    PIO_STACK_LOCATION IrpSp = IoGetCurrentIrpStackLocation(Irp);
    PFILE_OBJECT FileObject = IrpSp->FileObject;
    ULONG bytes_read = 0;
    NTSTATUS Status;
    BOOL top_level;
    fcb* fcb;
    ccb* ccb;
    BOOLEAN fcb_lock = FALSE, wait;

    FsRtlEnterFileSystem();

    top_level = is_top_level(Irp);

    TRACE("read\n");

    if (Vcb && Vcb->type == VCB_TYPE_VOLUME) {
        Status = vol_read(DeviceObject, Irp);
        goto exit2;
    } else if (!Vcb || Vcb->type != VCB_TYPE_FS) {
        Status = STATUS_INVALID_PARAMETER;
        goto end;
    }

    Irp->IoStatus.Information = 0;

    if (IrpSp->MinorFunction & IRP_MN_COMPLETE) {
        CcMdlReadComplete(IrpSp->FileObject, Irp->MdlAddress);

        Irp->MdlAddress = NULL;
        Status = STATUS_SUCCESS;

        goto exit;
    }

    fcb = FileObject->FsContext;

    if (!fcb) {
        ERR("fcb was NULL\n");
        Status = STATUS_INVALID_PARAMETER;
        goto exit;
    }

    ccb = FileObject->FsContext2;

    if (!ccb) {
        ERR("ccb was NULL\n");
        Status = STATUS_INVALID_PARAMETER;
        goto exit;
    }

    if (Irp->RequestorMode == UserMode && !(ccb->access & FILE_READ_DATA)) {
        WARN("insufficient privileges\n");
        Status = STATUS_ACCESS_DENIED;
        goto exit;
    }

    if (fcb == Vcb->volume_fcb) {
        TRACE("reading volume FCB\n");

        IoSkipCurrentIrpStackLocation(Irp);

        Status = IoCallDriver(Vcb->Vpb->RealDevice, Irp);

        goto exit2;
    }

    wait = IoIsOperationSynchronous(Irp);

    // Don't offload jobs when doing paging IO - otherwise this can lead to
    // deadlocks in CcCopyRead.
    if (Irp->Flags & IRP_PAGING_IO)
        wait = TRUE;

    if (!(Irp->Flags & IRP_PAGING_IO) && FileObject->SectionObjectPointer->DataSectionObject) {
        IO_STATUS_BLOCK iosb;

        CcFlushCache(FileObject->SectionObjectPointer, &IrpSp->Parameters.Read.ByteOffset, IrpSp->Parameters.Read.Length, &iosb);
        if (!NT_SUCCESS(iosb.Status)) {
            ERR("CcFlushCache returned %08x\n", iosb.Status);
            return iosb.Status;
        }
    }

    if (!ExIsResourceAcquiredSharedLite(fcb->Header.Resource)) {
        if (!ExAcquireResourceSharedLite(fcb->Header.Resource, wait)) {
            Status = STATUS_PENDING;
            IoMarkIrpPending(Irp);
            goto exit;
        }

        fcb_lock = TRUE;
    }

    Status = do_read(Irp, wait, &bytes_read);

    if (fcb_lock)
        ExReleaseResourceLite(fcb->Header.Resource);

exit:
    if (FileObject->Flags & FO_SYNCHRONOUS_IO && !(Irp->Flags & IRP_PAGING_IO))
        FileObject->CurrentByteOffset.QuadPart = IrpSp->Parameters.Read.ByteOffset.QuadPart + (NT_SUCCESS(Status) ? bytes_read : 0);

end:
    Irp->IoStatus.Status = Status;

    TRACE("Irp->IoStatus.Status = %08x\n", Irp->IoStatus.Status);
    TRACE("Irp->IoStatus.Information = %lu\n", Irp->IoStatus.Information);
    TRACE("returning %08x\n", Status);

    if (Status != STATUS_PENDING)
        IoCompleteRequest(Irp, IO_NO_INCREMENT);
    else {
        if (!add_thread_job(Vcb, Irp))
            do_read_job(Irp);
    }

exit2:
    if (top_level)
        IoSetTopLevelIrp(NULL);

    FsRtlExitFileSystem();

    return Status;
}

_Function_class_()

_Function_class_

(

IO_COMPLETION_ROUTINE

)

Definition at line 65 of file read.c.

                                                                                                {
#else
static NTSTATUS read_data_completion(PDEVICE_OBJECT DeviceObject, PIRP Irp, PVOID conptr) {
#endif
    read_data_stripe* stripe = conptr;
    read_data_context* context = (read_data_context*)stripe->context;

    UNUSED(DeviceObject);

    stripe->iosb = Irp->IoStatus;

    if (NT_SUCCESS(Irp->IoStatus.Status))
        stripe->status = ReadDataStatus_Success;
    else
        stripe->status = ReadDataStatus_Error;

    if (InterlockedDecrement(&context->stripes_left) == 0)
        KeSetEvent(&context->Event, 0, FALSE);

    return STATUS_MORE_PROCESSING_REQUIRED;
}

check_csum()

NTSTATUS check_csum	(	device_extension *	Vcb,
		UINT8 *	data,
		UINT32	sectors,
		UINT32 *	csum
	)

Definition at line 89 of file read.c.

                                                                                      {
    NTSTATUS Status;
    calc_job* cj;
    UINT32* csum2;

    // From experimenting, it seems that 40 sectors is roughly the crossover
    // point where offloading the crc32 calculation becomes worth it.

    if (sectors < 40 || KeQueryActiveProcessorCount(NULL) < 2) {
        ULONG j;

        for (j = 0; j < sectors; j++) {
            UINT32 crc32 = ~calc_crc32c(0xffffffff, data + (j * Vcb->superblock.sector_size), Vcb->superblock.sector_size);

            if (crc32 != csum[j]) {
                return STATUS_CRC_ERROR;
            }
        }

        return STATUS_SUCCESS;
    }

    csum2 = ExAllocatePoolWithTag(PagedPool, sizeof(UINT32) * sectors, ALLOC_TAG);
    if (!csum2) {
        ERR("out of memory\n");
        return STATUS_INSUFFICIENT_RESOURCES;
    }

    Status = add_calc_job(Vcb, data, sectors, csum2, &cj);
    if (!NT_SUCCESS(Status)) {
        ERR("add_calc_job returned %08x\n", Status);
        ExFreePool(csum2);
        return Status;
    }

    KeWaitForSingleObject(&cj->event, Executive, KernelMode, FALSE, NULL);

    if (RtlCompareMemory(csum2, csum, sectors * sizeof(UINT32)) != sectors * sizeof(UINT32)) {
        free_calc_job(cj);
        ExFreePool(csum2);
        return STATUS_CRC_ERROR;
    }

    free_calc_job(cj);
    ExFreePool(csum2);

    return STATUS_SUCCESS;
}

Referenced by read_data_dup(), read_data_raid0(), read_data_raid10(), read_data_raid5(), and read_data_raid6().

do_read()

NTSTATUS do_read	(	PIRP	Irp,
		BOOLEAN	wait,
		ULONG *	bytes_read
	)

Definition at line 3138 of file read.c.

                                                            {
    PIO_STACK_LOCATION IrpSp = IoGetCurrentIrpStackLocation(Irp);
    PFILE_OBJECT FileObject = IrpSp->FileObject;
    fcb* fcb = FileObject->FsContext;
    UINT8* data = NULL;
    ULONG length = IrpSp->Parameters.Read.Length, addon = 0;
    UINT64 start = IrpSp->Parameters.Read.ByteOffset.QuadPart;

    *bytes_read = 0;

    if (!fcb || !fcb->Vcb || !fcb->subvol)
        return STATUS_INTERNAL_ERROR;

    TRACE("file = %S (fcb = %p)\n", file_desc(FileObject), fcb);
    TRACE("offset = %llx, length = %x\n", start, length);
    TRACE("paging_io = %s, no cache = %s\n", Irp->Flags & IRP_PAGING_IO ? "TRUE" : "FALSE", Irp->Flags & IRP_NOCACHE ? "TRUE" : "FALSE");

    if (!fcb->ads && fcb->type == BTRFS_TYPE_DIRECTORY)
        return STATUS_INVALID_DEVICE_REQUEST;

    if (!(Irp->Flags & IRP_PAGING_IO) && !FsRtlCheckLockForReadAccess(&fcb->lock, Irp)) {
        WARN("tried to read locked region\n");
        return STATUS_FILE_LOCK_CONFLICT;
    }

    if (length == 0) {
        TRACE("tried to read zero bytes\n");
        return STATUS_SUCCESS;
    }

    if (start >= (UINT64)fcb->Header.FileSize.QuadPart) {
        TRACE("tried to read with offset after file end (%llx >= %llx)\n", start, fcb->Header.FileSize.QuadPart);
        return STATUS_END_OF_FILE;
    }

    TRACE("FileObject %p fcb %p FileSize = %llx st_size = %llx (%p)\n", FileObject, fcb, fcb->Header.FileSize.QuadPart, fcb->inode_item.st_size, &fcb->inode_item.st_size);

    if (Irp->Flags & IRP_NOCACHE || !(IrpSp->MinorFunction & IRP_MN_MDL)) {
        data = map_user_buffer(Irp, fcb->Header.Flags2 & FSRTL_FLAG2_IS_PAGING_FILE ? HighPagePriority : NormalPagePriority);

        if (Irp->MdlAddress && !data) {
            ERR("MmGetSystemAddressForMdlSafe returned NULL\n");
            return STATUS_INSUFFICIENT_RESOURCES;
        }

        if (start >= (UINT64)fcb->Header.ValidDataLength.QuadPart) {
            length = (ULONG)min(length, min(start + length, (UINT64)fcb->Header.FileSize.QuadPart) - fcb->Header.ValidDataLength.QuadPart);
            RtlZeroMemory(data, length);
            Irp->IoStatus.Information = *bytes_read = length;
            return STATUS_SUCCESS;
        }

        if (length + start > (UINT64)fcb->Header.ValidDataLength.QuadPart) {
            addon = (ULONG)(min(start + length, (UINT64)fcb->Header.FileSize.QuadPart) - fcb->Header.ValidDataLength.QuadPart);
            RtlZeroMemory(data + (fcb->Header.ValidDataLength.QuadPart - start), addon);
            length = (ULONG)(fcb->Header.ValidDataLength.QuadPart - start);
        }
    }

    if (!(Irp->Flags & IRP_NOCACHE)) {
        NTSTATUS Status = STATUS_SUCCESS;

        _SEH2_TRY {
            if (!FileObject->PrivateCacheMap) {
                CC_FILE_SIZES ccfs;

                ccfs.AllocationSize = fcb->Header.AllocationSize;
                ccfs.FileSize = fcb->Header.FileSize;
                ccfs.ValidDataLength = fcb->Header.ValidDataLength;

                init_file_cache(FileObject, &ccfs);
            }

            if (IrpSp->MinorFunction & IRP_MN_MDL) {
                CcMdlRead(FileObject,&IrpSp->Parameters.Read.ByteOffset, length, &Irp->MdlAddress, &Irp->IoStatus);
            } else {
                if (fCcCopyReadEx) {
                    TRACE("CcCopyReadEx(%p, %llx, %x, %u, %p, %p, %p, %p)\n", FileObject, IrpSp->Parameters.Read.ByteOffset.QuadPart,
                          length, wait, data, &Irp->IoStatus, Irp->Tail.Overlay.Thread);
                    TRACE("sizes = %llx, %llx, %llx\n", fcb->Header.AllocationSize, fcb->Header.FileSize, fcb->Header.ValidDataLength);
                    if (!fCcCopyReadEx(FileObject, &IrpSp->Parameters.Read.ByteOffset, length, wait, data, &Irp->IoStatus, Irp->Tail.Overlay.Thread)) {
                        TRACE("CcCopyReadEx could not wait\n");

                        IoMarkIrpPending(Irp);
                        return STATUS_PENDING;
                    }
                    TRACE("CcCopyReadEx finished\n");
                } else {
                    TRACE("CcCopyRead(%p, %llx, %x, %u, %p, %p)\n", FileObject, IrpSp->Parameters.Read.ByteOffset.QuadPart, length, wait, data, &Irp->IoStatus);
                    TRACE("sizes = %llx, %llx, %llx\n", fcb->Header.AllocationSize, fcb->Header.FileSize, fcb->Header.ValidDataLength);
                    if (!CcCopyRead(FileObject, &IrpSp->Parameters.Read.ByteOffset, length, wait, data, &Irp->IoStatus)) {
                        TRACE("CcCopyRead could not wait\n");

                        IoMarkIrpPending(Irp);
                        return STATUS_PENDING;
                    }
                    TRACE("CcCopyRead finished\n");
                }
            }
        } _SEH2_EXCEPT (EXCEPTION_EXECUTE_HANDLER) {
            Status = _SEH2_GetExceptionCode();
        } _SEH2_END;

        if (NT_SUCCESS(Status)) {
            Status = Irp->IoStatus.Status;
            Irp->IoStatus.Information += addon;
            *bytes_read = (ULONG)Irp->IoStatus.Information;
        } else
            ERR("EXCEPTION - %08x\n", Status);

        return Status;
    } else {
        NTSTATUS Status;

        if (!wait) {
            IoMarkIrpPending(Irp);
            return STATUS_PENDING;
        }

        if (fcb->ads)
            Status = read_stream(fcb, data, start, length, bytes_read);
        else
            Status = read_file(fcb, data, start, length, bytes_read, Irp);

        *bytes_read += addon;
        TRACE("read %u bytes\n", *bytes_read);

        Irp->IoStatus.Information = *bytes_read;

        if (diskacc && Status != STATUS_PENDING) {
            PETHREAD thread = NULL;

            if (Irp->Tail.Overlay.Thread && !IoIsSystemThread(Irp->Tail.Overlay.Thread))
                thread = Irp->Tail.Overlay.Thread;
            else if (!IoIsSystemThread(PsGetCurrentThread()))
                thread = PsGetCurrentThread();
            else if (IoIsSystemThread(PsGetCurrentThread()) && IoGetTopLevelIrp() == Irp)
                thread = PsGetCurrentThread();

            if (thread)
                fPsUpdateDiskCounters(PsGetThreadProcess(thread), *bytes_read, 0, 1, 0, 0);
        }

        return Status;
    }
}

Referenced by _Dispatch_type_(), and do_read_job().

raid6_recover2()

void raid6_recover2	(	UINT8 *	sectors,
		UINT16	num_stripes,
		ULONG	sector_size,
		UINT16	missing1,
		UINT16	missing2,
		UINT8 *	out
	)

Definition at line 862 of file read.c.

                                                                                                                         {
    if (missing1 == num_stripes - 2 || missing2 == num_stripes - 2) { // reconstruct from q and data
        UINT16 missing = missing1 == (num_stripes - 2) ? missing2 : missing1;
        UINT16 stripe;

        stripe = num_stripes - 3;

        if (stripe == missing)
            RtlZeroMemory(out, sector_size);
        else
            RtlCopyMemory(out, sectors + (stripe * sector_size), sector_size);

        do {
            stripe--;

            galois_double(out, sector_size);

            if (stripe != missing)
                do_xor(out, sectors + (stripe * sector_size), sector_size);
        } while (stripe > 0);

        do_xor(out, sectors + ((num_stripes - 1) * sector_size), sector_size);

        if (missing != 0)
            galois_divpower(out, (UINT8)missing, sector_size);
    } else { // reconstruct from p and q
        UINT16 x, y, stripe;
        UINT8 gyx, gx, denom, a, b, *p, *q, *pxy, *qxy;
        UINT32 j;

        stripe = num_stripes - 3;

        pxy = out + sector_size;
        qxy = out;

        if (stripe == missing1 || stripe == missing2) {
            RtlZeroMemory(qxy, sector_size);
            RtlZeroMemory(pxy, sector_size);

            if (stripe == missing1)
                x = stripe;
            else
                y = stripe;
        } else {
            RtlCopyMemory(qxy, sectors + (stripe * sector_size), sector_size);
            RtlCopyMemory(pxy, sectors + (stripe * sector_size), sector_size);
        }

        do {
            stripe--;

            galois_double(qxy, sector_size);

            if (stripe != missing1 && stripe != missing2) {
                do_xor(qxy, sectors + (stripe * sector_size), sector_size);
                do_xor(pxy, sectors + (stripe * sector_size), sector_size);
            } else if (stripe == missing1)
                x = stripe;
            else if (stripe == missing2)
                y = stripe;
        } while (stripe > 0);

        gyx = gpow2(y > x ? (y-x) : (255-x+y));
        gx = gpow2(255-x);

        denom = gdiv(1, gyx ^ 1);
        a = gmul(gyx, denom);
        b = gmul(gx, denom);

        p = sectors + ((num_stripes - 2) * sector_size);
        q = sectors + ((num_stripes - 1) * sector_size);

        for (j = 0; j < sector_size; j++) {
            *qxy = gmul(a, *p ^ *pxy) ^ gmul(b, *q ^ *qxy);

            p++;
            q++;
            pxy++;
            qxy++;
        }

        do_xor(out + sector_size, out, sector_size);
        do_xor(out + sector_size, sectors + ((num_stripes - 2) * sector_size), sector_size);
    }
}

Referenced by partial_stripe_read(), and read_data_raid6().

read_data()

NTSTATUS read_data	(	_In_ device_extension *	Vcb,
		_In_ UINT64	addr,
		_In_ UINT32	length,
		_In_reads_bytes_opt_(length *sizeof(UINT32)/Vcb->superblock.sector_size) UINT32 *	csum,
		_In_ BOOL	is_tree,
		_Out_writes_bytes_(length) UINT8 *	buf,
		_In_opt_ chunk *	c,
		_Out_opt_ chunk **	pc,
		_In_opt_ PIRP	Irp,
		_In_ UINT64	generation,
		_In_ BOOL	file_read,
		_In_ ULONG	priority
	)

Definition at line 1447 of file read.c.

                                        {
    CHUNK_ITEM* ci;
    CHUNK_ITEM_STRIPE* cis;
    read_data_context context;
    UINT64 type, offset, total_reading = 0;
    NTSTATUS Status;
    device** devices = NULL;
    UINT16 i, startoffstripe, allowed_missing, missing_devices = 0;
    UINT8* dummypage = NULL;
    PMDL dummy_mdl = NULL;
    BOOL need_to_wait;
    UINT64 lockaddr, locklen;
#ifdef DEBUG_STATS
    LARGE_INTEGER time1, time2;
#endif

    if (Vcb->log_to_phys_loaded) {
        if (!c) {
            c = get_chunk_from_address(Vcb, addr);

            if (!c) {
                ERR("get_chunk_from_address failed\n");
                return STATUS_INTERNAL_ERROR;
            }
        }

        ci = c->chunk_item;
        offset = c->offset;
        devices = c->devices;

        if (pc)
            *pc = c;
    } else {
        LIST_ENTRY* le = Vcb->sys_chunks.Flink;

        ci = NULL;

        c = NULL;
        while (le != &Vcb->sys_chunks) {
            sys_chunk* sc = CONTAINING_RECORD(le, sys_chunk, list_entry);

            if (sc->key.obj_id == 0x100 && sc->key.obj_type == TYPE_CHUNK_ITEM && sc->key.offset <= addr) {
                CHUNK_ITEM* chunk_item = sc->data;

                if ((addr - sc->key.offset) < chunk_item->size && chunk_item->num_stripes > 0) {
                    ci = chunk_item;
                    offset = sc->key.offset;
                    cis = (CHUNK_ITEM_STRIPE*)&chunk_item[1];

                    devices = ExAllocatePoolWithTag(PagedPool, sizeof(device*) * ci->num_stripes, ALLOC_TAG);
                    if (!devices) {
                        ERR("out of memory\n");
                        return STATUS_INSUFFICIENT_RESOURCES;
                    }

                    for (i = 0; i < ci->num_stripes; i++) {
                        devices[i] = find_device_from_uuid(Vcb, &cis[i].dev_uuid);
                    }

                    break;
                }
            }

            le = le->Flink;
        }

        if (!ci) {
            ERR("could not find chunk for %llx in bootstrap\n", addr);
            return STATUS_INTERNAL_ERROR;
        }

        if (pc)
            *pc = NULL;
    }

    if (ci->type & BLOCK_FLAG_DUPLICATE) {
        type = BLOCK_FLAG_DUPLICATE;
        allowed_missing = ci->num_stripes - 1;
    } else if (ci->type & BLOCK_FLAG_RAID0) {
        type = BLOCK_FLAG_RAID0;
        allowed_missing = 0;
    } else if (ci->type & BLOCK_FLAG_RAID1) {
        type = BLOCK_FLAG_DUPLICATE;
        allowed_missing = 1;
    } else if (ci->type & BLOCK_FLAG_RAID10) {
        type = BLOCK_FLAG_RAID10;
        allowed_missing = 1;
    } else if (ci->type & BLOCK_FLAG_RAID5) {
        type = BLOCK_FLAG_RAID5;
        allowed_missing = 1;
    } else if (ci->type & BLOCK_FLAG_RAID6) {
        type = BLOCK_FLAG_RAID6;
        allowed_missing = 2;
    } else { // SINGLE
        type = BLOCK_FLAG_DUPLICATE;
        allowed_missing = 0;
    }

    cis = (CHUNK_ITEM_STRIPE*)&ci[1];

    RtlZeroMemory(&context, sizeof(read_data_context));
    KeInitializeEvent(&context.Event, NotificationEvent, FALSE);

    context.stripes = ExAllocatePoolWithTag(NonPagedPool, sizeof(read_data_stripe) * ci->num_stripes, ALLOC_TAG);
    if (!context.stripes) {
        ERR("out of memory\n");
        return STATUS_INSUFFICIENT_RESOURCES;
    }

    if (c && (type == BLOCK_FLAG_RAID5 || type == BLOCK_FLAG_RAID6)) {
        get_raid56_lock_range(c, addr, length, &lockaddr, &locklen);
        chunk_lock_range(Vcb, c, lockaddr, locklen);
    }

    RtlZeroMemory(context.stripes, sizeof(read_data_stripe) * ci->num_stripes);

    context.buflen = length;
    context.num_stripes = ci->num_stripes;
    context.stripes_left = context.num_stripes;
    context.sector_size = Vcb->superblock.sector_size;
    context.csum = csum;
    context.tree = is_tree;
    context.type = type;

    if (type == BLOCK_FLAG_RAID0) {
        UINT64 startoff, endoff;
        UINT16 endoffstripe, stripe;
        UINT32 *stripeoff, pos;
        PMDL master_mdl;
        PFN_NUMBER* pfns;

        // FIXME - test this still works if page size isn't the same as sector size

        // This relies on the fact that MDLs are followed in memory by the page file numbers,
        // so with a bit of jiggery-pokery you can trick your disks into deinterlacing your RAID0
        // data for you without doing a memcpy yourself.
        // MDLs are officially opaque, so this might very well break in future versions of Windows.

        get_raid0_offset(addr - offset, ci->stripe_length, ci->num_stripes, &startoff, &startoffstripe);
        get_raid0_offset(addr + length - offset - 1, ci->stripe_length, ci->num_stripes, &endoff, &endoffstripe);

        if (file_read) {
            // Unfortunately we can't avoid doing at least one memcpy, as Windows can give us an MDL
            // with duplicated dummy PFNs, which confuse check_csum. Ah well.
            // See https://msdn.microsoft.com/en-us/library/windows/hardware/Dn614012.aspx if you're interested.

            context.va = ExAllocatePoolWithTag(NonPagedPool, length, ALLOC_TAG);

            if (!context.va) {
                ERR("out of memory\n");
                Status = STATUS_INSUFFICIENT_RESOURCES;
                goto exit;
            }
        } else
            context.va = buf;

        master_mdl = IoAllocateMdl(context.va, length, FALSE, FALSE, NULL);
        if (!master_mdl) {
            ERR("out of memory\n");
            Status = STATUS_INSUFFICIENT_RESOURCES;
            goto exit;
        }

        Status = STATUS_SUCCESS;

        _SEH2_TRY {
            MmProbeAndLockPages(master_mdl, KernelMode, IoWriteAccess);
        } _SEH2_EXCEPT (EXCEPTION_EXECUTE_HANDLER) {
            Status = _SEH2_GetExceptionCode();
        } _SEH2_END;

        if (!NT_SUCCESS(Status)) {
            ERR("MmProbeAndLockPages threw exception %08x\n", Status);
            IoFreeMdl(master_mdl);
            goto exit;
        }

        pfns = (PFN_NUMBER*)(master_mdl + 1);

        for (i = 0; i < ci->num_stripes; i++) {
            if (startoffstripe > i)
                context.stripes[i].stripestart = startoff - (startoff % ci->stripe_length) + ci->stripe_length;
            else if (startoffstripe == i)
                context.stripes[i].stripestart = startoff;
            else
                context.stripes[i].stripestart = startoff - (startoff % ci->stripe_length);

            if (endoffstripe > i)
                context.stripes[i].stripeend = endoff - (endoff % ci->stripe_length) + ci->stripe_length;
            else if (endoffstripe == i)
                context.stripes[i].stripeend = endoff + 1;
            else
                context.stripes[i].stripeend = endoff - (endoff % ci->stripe_length);

            if (context.stripes[i].stripestart != context.stripes[i].stripeend) {
                context.stripes[i].mdl = IoAllocateMdl(context.va, (ULONG)(context.stripes[i].stripeend - context.stripes[i].stripestart), FALSE, FALSE, NULL);

                if (!context.stripes[i].mdl) {
                    ERR("IoAllocateMdl failed\n");
                    MmUnlockPages(master_mdl);
                    IoFreeMdl(master_mdl);
                    Status = STATUS_INSUFFICIENT_RESOURCES;
                    goto exit;
                }
            }
        }

        stripeoff = ExAllocatePoolWithTag(NonPagedPool, sizeof(UINT32) * ci->num_stripes, ALLOC_TAG);
        if (!stripeoff) {
            ERR("out of memory\n");
            MmUnlockPages(master_mdl);
            IoFreeMdl(master_mdl);
            Status = STATUS_INSUFFICIENT_RESOURCES;
            goto exit;
        }

        RtlZeroMemory(stripeoff, sizeof(UINT32) * ci->num_stripes);

        pos = 0;
        stripe = startoffstripe;
        while (pos < length) {
            PFN_NUMBER* stripe_pfns = (PFN_NUMBER*)(context.stripes[stripe].mdl + 1);

            if (pos == 0) {
                UINT32 readlen = (UINT32)min(context.stripes[stripe].stripeend - context.stripes[stripe].stripestart, ci->stripe_length - (context.stripes[stripe].stripestart % ci->stripe_length));

                RtlCopyMemory(stripe_pfns, pfns, readlen * sizeof(PFN_NUMBER) >> PAGE_SHIFT);

                stripeoff[stripe] += readlen;
                pos += readlen;
            } else if (length - pos < ci->stripe_length) {
                RtlCopyMemory(&stripe_pfns[stripeoff[stripe] >> PAGE_SHIFT], &pfns[pos >> PAGE_SHIFT], (length - pos) * sizeof(PFN_NUMBER) >> PAGE_SHIFT);

                pos = length;
            } else {
                RtlCopyMemory(&stripe_pfns[stripeoff[stripe] >> PAGE_SHIFT], &pfns[pos >> PAGE_SHIFT], (ULONG)(ci->stripe_length * sizeof(PFN_NUMBER) >> PAGE_SHIFT));

                stripeoff[stripe] += (UINT32)ci->stripe_length;
                pos += (UINT32)ci->stripe_length;
            }

            stripe = (stripe + 1) % ci->num_stripes;
        }

        MmUnlockPages(master_mdl);
        IoFreeMdl(master_mdl);

        ExFreePool(stripeoff);
    } else if (type == BLOCK_FLAG_RAID10) {
        UINT64 startoff, endoff;
        UINT16 endoffstripe, j, stripe;
        ULONG orig_ls;
        PMDL master_mdl;
        PFN_NUMBER* pfns;
        UINT32* stripeoff, pos;
        read_data_stripe** stripes;

        if (c)
            orig_ls = c->last_stripe;
        else
            orig_ls = 0;

        get_raid0_offset(addr - offset, ci->stripe_length, ci->num_stripes / ci->sub_stripes, &startoff, &startoffstripe);
        get_raid0_offset(addr + length - offset - 1, ci->stripe_length, ci->num_stripes / ci->sub_stripes, &endoff, &endoffstripe);

        if ((ci->num_stripes % ci->sub_stripes) != 0) {
            ERR("chunk %llx: num_stripes %x was not a multiple of sub_stripes %x!\n", offset, ci->num_stripes, ci->sub_stripes);
            Status = STATUS_INTERNAL_ERROR;
            goto exit;
        }

        if (file_read) {
            context.va = ExAllocatePoolWithTag(NonPagedPool, length, ALLOC_TAG);

            if (!context.va) {
                ERR("out of memory\n");
                Status = STATUS_INSUFFICIENT_RESOURCES;
                goto exit;
            }
        } else
            context.va = buf;

        context.firstoff = (UINT16)((startoff % ci->stripe_length) / Vcb->superblock.sector_size);
        context.startoffstripe = startoffstripe;
        context.sectors_per_stripe = (UINT16)(ci->stripe_length / Vcb->superblock.sector_size);

        startoffstripe *= ci->sub_stripes;
        endoffstripe *= ci->sub_stripes;

        if (c)
            c->last_stripe = (orig_ls + 1) % ci->sub_stripes;

        master_mdl = IoAllocateMdl(context.va, length, FALSE, FALSE, NULL);
        if (!master_mdl) {
            ERR("out of memory\n");
            Status = STATUS_INSUFFICIENT_RESOURCES;
            goto exit;
        }

        Status = STATUS_SUCCESS;

        _SEH2_TRY {
            MmProbeAndLockPages(master_mdl, KernelMode, IoWriteAccess);
        } _SEH2_EXCEPT (EXCEPTION_EXECUTE_HANDLER) {
            Status = _SEH2_GetExceptionCode();
        } _SEH2_END;

        if (!NT_SUCCESS(Status)) {
            ERR("MmProbeAndLockPages threw exception %08x\n", Status);
            IoFreeMdl(master_mdl);
            goto exit;
        }

        pfns = (PFN_NUMBER*)(master_mdl + 1);

        stripes = ExAllocatePoolWithTag(NonPagedPool, sizeof(read_data_stripe*) * ci->num_stripes / ci->sub_stripes, ALLOC_TAG);
        if (!stripes) {
            ERR("out of memory\n");
            MmUnlockPages(master_mdl);
            IoFreeMdl(master_mdl);
            Status = STATUS_INSUFFICIENT_RESOURCES;
            goto exit;
        }

        RtlZeroMemory(stripes, sizeof(read_data_stripe*) * ci->num_stripes / ci->sub_stripes);

        for (i = 0; i < ci->num_stripes; i += ci->sub_stripes) {
            UINT64 sstart, send;
            BOOL stripeset = FALSE;

            if (startoffstripe > i)
                sstart = startoff - (startoff % ci->stripe_length) + ci->stripe_length;
            else if (startoffstripe == i)
                sstart = startoff;
            else
                sstart = startoff - (startoff % ci->stripe_length);

            if (endoffstripe > i)
                send = endoff - (endoff % ci->stripe_length) + ci->stripe_length;
            else if (endoffstripe == i)
                send = endoff + 1;
            else
                send = endoff - (endoff % ci->stripe_length);

            for (j = 0; j < ci->sub_stripes; j++) {
                if (j == orig_ls && devices[i+j] && devices[i+j]->devobj) {
                    context.stripes[i+j].stripestart = sstart;
                    context.stripes[i+j].stripeend = send;
                    stripes[i / ci->sub_stripes] = &context.stripes[i+j];

                    if (sstart != send) {
                        context.stripes[i+j].mdl = IoAllocateMdl(context.va, (ULONG)(send - sstart), FALSE, FALSE, NULL);

                        if (!context.stripes[i+j].mdl) {
                            ERR("IoAllocateMdl failed\n");
                            MmUnlockPages(master_mdl);
                            IoFreeMdl(master_mdl);
                            Status = STATUS_INSUFFICIENT_RESOURCES;
                            goto exit;
                        }
                    }

                    stripeset = TRUE;
                } else
                    context.stripes[i+j].status = ReadDataStatus_Skip;
            }

            if (!stripeset) {
                for (j = 0; j < ci->sub_stripes; j++) {
                    if (devices[i+j] && devices[i+j]->devobj) {
                        context.stripes[i+j].stripestart = sstart;
                        context.stripes[i+j].stripeend = send;
                        context.stripes[i+j].status = ReadDataStatus_Pending;
                        stripes[i / ci->sub_stripes] = &context.stripes[i+j];

                        if (sstart != send) {
                            context.stripes[i+j].mdl = IoAllocateMdl(context.va, (ULONG)(send - sstart), FALSE, FALSE, NULL);

                            if (!context.stripes[i+j].mdl) {
                                ERR("IoAllocateMdl failed\n");
                                MmUnlockPages(master_mdl);
                                IoFreeMdl(master_mdl);
                                Status = STATUS_INSUFFICIENT_RESOURCES;
                                goto exit;
                            }
                        }

                        stripeset = TRUE;
                        break;
                    }
                }

                if (!stripeset) {
                    ERR("could not find stripe to read\n");
                    Status = STATUS_DEVICE_NOT_READY;
                    goto exit;
                }
            }
        }

        stripeoff = ExAllocatePoolWithTag(NonPagedPool, sizeof(UINT32) * ci->num_stripes / ci->sub_stripes, ALLOC_TAG);
        if (!stripeoff) {
            ERR("out of memory\n");
            MmUnlockPages(master_mdl);
            IoFreeMdl(master_mdl);
            Status = STATUS_INSUFFICIENT_RESOURCES;
            goto exit;
        }

        RtlZeroMemory(stripeoff, sizeof(UINT32) * ci->num_stripes / ci->sub_stripes);

        pos = 0;
        stripe = startoffstripe / ci->sub_stripes;
        while (pos < length) {
            PFN_NUMBER* stripe_pfns = (PFN_NUMBER*)(stripes[stripe]->mdl + 1);

            if (pos == 0) {
                UINT32 readlen = (UINT32)min(stripes[stripe]->stripeend - stripes[stripe]->stripestart,
                                             ci->stripe_length - (stripes[stripe]->stripestart % ci->stripe_length));

                RtlCopyMemory(stripe_pfns, pfns, readlen * sizeof(PFN_NUMBER) >> PAGE_SHIFT);

                stripeoff[stripe] += readlen;
                pos += readlen;
            } else if (length - pos < ci->stripe_length) {
                RtlCopyMemory(&stripe_pfns[stripeoff[stripe] >> PAGE_SHIFT], &pfns[pos >> PAGE_SHIFT], (length - pos) * sizeof(PFN_NUMBER) >> PAGE_SHIFT);

                pos = length;
            } else {
                RtlCopyMemory(&stripe_pfns[stripeoff[stripe] >> PAGE_SHIFT], &pfns[pos >> PAGE_SHIFT], (ULONG)(ci->stripe_length * sizeof(PFN_NUMBER) >> PAGE_SHIFT));

                stripeoff[stripe] += (ULONG)ci->stripe_length;
                pos += (ULONG)ci->stripe_length;
            }

            stripe = (stripe + 1) % (ci->num_stripes / ci->sub_stripes);
        }

        MmUnlockPages(master_mdl);
        IoFreeMdl(master_mdl);

        ExFreePool(stripeoff);
        ExFreePool(stripes);
    } else if (type == BLOCK_FLAG_DUPLICATE) {
        UINT64 orig_ls;

        if (c)
            orig_ls = i = c->last_stripe;
        else
            orig_ls = i = 0;

        while (!devices[i] || !devices[i]->devobj) {
            i = (i + 1) % ci->num_stripes;

            if (i == orig_ls) {
                ERR("no devices available to service request\n");
                Status = STATUS_DEVICE_NOT_READY;
                goto exit;
            }
        }

        if (c)
            c->last_stripe = (i + 1) % ci->num_stripes;

        context.stripes[i].stripestart = addr - offset;
        context.stripes[i].stripeend = context.stripes[i].stripestart + length;

        if (file_read) {
            context.va = ExAllocatePoolWithTag(NonPagedPool, length, ALLOC_TAG);

            if (!context.va) {
                ERR("out of memory\n");
                Status = STATUS_INSUFFICIENT_RESOURCES;
                goto exit;
            }

            context.stripes[i].mdl = IoAllocateMdl(context.va, length, FALSE, FALSE, NULL);
            if (!context.stripes[i].mdl) {
                ERR("IoAllocateMdl failed\n");
                Status = STATUS_INSUFFICIENT_RESOURCES;
                goto exit;
            }

            MmBuildMdlForNonPagedPool(context.stripes[i].mdl);
        } else {
            context.stripes[i].mdl = IoAllocateMdl(buf, length, FALSE, FALSE, NULL);

            if (!context.stripes[i].mdl) {
                ERR("IoAllocateMdl failed\n");
                Status = STATUS_INSUFFICIENT_RESOURCES;
                goto exit;
            }

            Status = STATUS_SUCCESS;

            _SEH2_TRY {
                MmProbeAndLockPages(context.stripes[i].mdl, KernelMode, IoWriteAccess);
            } _SEH2_EXCEPT (EXCEPTION_EXECUTE_HANDLER) {
                Status = _SEH2_GetExceptionCode();
            } _SEH2_END;

            if (!NT_SUCCESS(Status)) {
                ERR("MmProbeAndLockPages threw exception %08x\n", Status);
                goto exit;
            }
        }
    } else if (type == BLOCK_FLAG_RAID5) {
        UINT64 startoff, endoff;
        UINT16 endoffstripe, parity;
        UINT32 *stripeoff, pos;
        PMDL master_mdl;
        PFN_NUMBER *pfns, dummy;
        BOOL need_dummy = FALSE;

        get_raid0_offset(addr - offset, ci->stripe_length, ci->num_stripes - 1, &startoff, &startoffstripe);
        get_raid0_offset(addr + length - offset - 1, ci->stripe_length, ci->num_stripes - 1, &endoff, &endoffstripe);

        if (file_read) {
            context.va = ExAllocatePoolWithTag(NonPagedPool, length, ALLOC_TAG);

            if (!context.va) {
                ERR("out of memory\n");
                Status = STATUS_INSUFFICIENT_RESOURCES;
                goto exit;
            }
        } else
            context.va = buf;

        master_mdl = IoAllocateMdl(context.va, length, FALSE, FALSE, NULL);
        if (!master_mdl) {
            ERR("out of memory\n");
            Status = STATUS_INSUFFICIENT_RESOURCES;
            goto exit;
        }

        Status = STATUS_SUCCESS;

        _SEH2_TRY {
            MmProbeAndLockPages(master_mdl, KernelMode, IoWriteAccess);
        } _SEH2_EXCEPT (EXCEPTION_EXECUTE_HANDLER) {
            Status = _SEH2_GetExceptionCode();
        } _SEH2_END;

        if (!NT_SUCCESS(Status)) {
            ERR("MmProbeAndLockPages threw exception %08x\n", Status);
            IoFreeMdl(master_mdl);
            goto exit;
        }

        pfns = (PFN_NUMBER*)(master_mdl + 1);

        pos = 0;
        while (pos < length) {
            parity = (((addr - offset + pos) / ((ci->num_stripes - 1) * ci->stripe_length)) + ci->num_stripes - 1) % ci->num_stripes;

            if (pos == 0) {
                UINT16 stripe = (parity + startoffstripe + 1) % ci->num_stripes;
                ULONG skip, readlen;

                i = startoffstripe;
                while (stripe != parity) {
                    if (i == startoffstripe) {
                        readlen = min(length, (ULONG)(ci->stripe_length - (startoff % ci->stripe_length)));

                        context.stripes[stripe].stripestart = startoff;
                        context.stripes[stripe].stripeend = startoff + readlen;

                        pos += readlen;

                        if (pos == length)
                            break;
                    } else {
                        readlen = min(length - pos, (ULONG)ci->stripe_length);

                        context.stripes[stripe].stripestart = startoff - (startoff % ci->stripe_length);
                        context.stripes[stripe].stripeend = context.stripes[stripe].stripestart + readlen;

                        pos += readlen;

                        if (pos == length)
                            break;
                    }

                    i++;
                    stripe = (stripe + 1) % ci->num_stripes;
                }

                if (pos == length)
                    break;

                for (i = 0; i < startoffstripe; i++) {
                    UINT16 stripe2 = (parity + i + 1) % ci->num_stripes;

                    context.stripes[stripe2].stripestart = context.stripes[stripe2].stripeend = startoff - (startoff % ci->stripe_length) + ci->stripe_length;
                }

                context.stripes[parity].stripestart = context.stripes[parity].stripeend = startoff - (startoff % ci->stripe_length) + ci->stripe_length;

                if (length - pos > ci->num_stripes * (ci->num_stripes - 1) * ci->stripe_length) {
                    skip = (ULONG)(((length - pos) / (ci->num_stripes * (ci->num_stripes - 1) * ci->stripe_length)) - 1);

                    for (i = 0; i < ci->num_stripes; i++) {
                        context.stripes[i].stripeend += skip * ci->num_stripes * ci->stripe_length;
                    }

                    pos += (UINT32)(skip * (ci->num_stripes - 1) * ci->num_stripes * ci->stripe_length);
                    need_dummy = TRUE;
                }
            } else if (length - pos >= ci->stripe_length * (ci->num_stripes - 1)) {
                for (i = 0; i < ci->num_stripes; i++) {
                    context.stripes[i].stripeend += ci->stripe_length;
                }

                pos += (UINT32)(ci->stripe_length * (ci->num_stripes - 1));
                need_dummy = TRUE;
            } else {
                UINT16 stripe = (parity + 1) % ci->num_stripes;

                i = 0;
                while (stripe != parity) {
                    if (endoffstripe == i) {
                        context.stripes[stripe].stripeend = endoff + 1;
                        break;
                    } else if (endoffstripe > i)
                        context.stripes[stripe].stripeend = endoff - (endoff % ci->stripe_length) + ci->stripe_length;

                    i++;
                    stripe = (stripe + 1) % ci->num_stripes;
                }

                break;
            }
        }

        for (i = 0; i < ci->num_stripes; i++) {
            if (context.stripes[i].stripestart != context.stripes[i].stripeend) {
                context.stripes[i].mdl = IoAllocateMdl(context.va, (ULONG)(context.stripes[i].stripeend - context.stripes[i].stripestart),
                                                       FALSE, FALSE, NULL);

                if (!context.stripes[i].mdl) {
                    ERR("IoAllocateMdl failed\n");
                    MmUnlockPages(master_mdl);
                    IoFreeMdl(master_mdl);
                    Status = STATUS_INSUFFICIENT_RESOURCES;
                    goto exit;
                }
            }
        }

        if (need_dummy) {
            dummypage = ExAllocatePoolWithTag(NonPagedPool, PAGE_SIZE, ALLOC_TAG);
            if (!dummypage) {
                ERR("out of memory\n");
                MmUnlockPages(master_mdl);
                IoFreeMdl(master_mdl);
                Status = STATUS_INSUFFICIENT_RESOURCES;
                goto exit;
            }

            dummy_mdl = IoAllocateMdl(dummypage, PAGE_SIZE, FALSE, FALSE, NULL);
            if (!dummy_mdl) {
                ERR("IoAllocateMdl failed\n");
                MmUnlockPages(master_mdl);
                IoFreeMdl(master_mdl);
                Status = STATUS_INSUFFICIENT_RESOURCES;
                goto exit;
            }

            MmBuildMdlForNonPagedPool(dummy_mdl);

            dummy = *(PFN_NUMBER*)(dummy_mdl + 1);
        }

        stripeoff = ExAllocatePoolWithTag(NonPagedPool, sizeof(UINT32) * ci->num_stripes, ALLOC_TAG);
        if (!stripeoff) {
            ERR("out of memory\n");
            MmUnlockPages(master_mdl);
            IoFreeMdl(master_mdl);
            Status = STATUS_INSUFFICIENT_RESOURCES;
            goto exit;
        }

        RtlZeroMemory(stripeoff, sizeof(UINT32) * ci->num_stripes);

        pos = 0;

        while (pos < length) {
            PFN_NUMBER* stripe_pfns;

            parity = (((addr - offset + pos) / ((ci->num_stripes - 1) * ci->stripe_length)) + ci->num_stripes - 1) % ci->num_stripes;

            if (pos == 0) {
                UINT16 stripe = (parity + startoffstripe + 1) % ci->num_stripes;
                UINT32 readlen = min(length - pos, (UINT32)min(context.stripes[stripe].stripeend - context.stripes[stripe].stripestart,
                                                       ci->stripe_length - (context.stripes[stripe].stripestart % ci->stripe_length)));

                stripe_pfns = (PFN_NUMBER*)(context.stripes[stripe].mdl + 1);

                RtlCopyMemory(stripe_pfns, pfns, readlen * sizeof(PFN_NUMBER) >> PAGE_SHIFT);

                stripeoff[stripe] = readlen;
                pos += readlen;

                stripe = (stripe + 1) % ci->num_stripes;

                while (stripe != parity) {
                    stripe_pfns = (PFN_NUMBER*)(context.stripes[stripe].mdl + 1);
                    readlen = min(length - pos, (UINT32)min(context.stripes[stripe].stripeend - context.stripes[stripe].stripestart, ci->stripe_length));

                    if (readlen == 0)
                        break;

                    RtlCopyMemory(stripe_pfns, &pfns[pos >> PAGE_SHIFT], readlen * sizeof(PFN_NUMBER) >> PAGE_SHIFT);

                    stripeoff[stripe] = readlen;
                    pos += readlen;

                    stripe = (stripe + 1) % ci->num_stripes;
                }
            } else if (length - pos >= ci->stripe_length * (ci->num_stripes - 1)) {
                UINT16 stripe = (parity + 1) % ci->num_stripes;
                ULONG k;

                while (stripe != parity) {
                    stripe_pfns = (PFN_NUMBER*)(context.stripes[stripe].mdl + 1);

                    RtlCopyMemory(&stripe_pfns[stripeoff[stripe] >> PAGE_SHIFT], &pfns[pos >> PAGE_SHIFT], (ULONG)(ci->stripe_length * sizeof(PFN_NUMBER) >> PAGE_SHIFT));

                    stripeoff[stripe] += (UINT32)ci->stripe_length;
                    pos += (UINT32)ci->stripe_length;

                    stripe = (stripe + 1) % ci->num_stripes;
                }

                stripe_pfns = (PFN_NUMBER*)(context.stripes[parity].mdl + 1);

                for (k = 0; k < ci->stripe_length >> PAGE_SHIFT; k++) {
                    stripe_pfns[stripeoff[parity] >> PAGE_SHIFT] = dummy;
                    stripeoff[parity] += PAGE_SIZE;
                }
            } else {
                UINT16 stripe = (parity + 1) % ci->num_stripes;
                UINT32 readlen;

                while (pos < length) {
                    stripe_pfns = (PFN_NUMBER*)(context.stripes[stripe].mdl + 1);
                    readlen = min(length - pos, (ULONG)min(context.stripes[stripe].stripeend - context.stripes[stripe].stripestart, ci->stripe_length));

                    if (readlen == 0)
                        break;

                    RtlCopyMemory(&stripe_pfns[stripeoff[stripe] >> PAGE_SHIFT], &pfns[pos >> PAGE_SHIFT], readlen * sizeof(PFN_NUMBER) >> PAGE_SHIFT);

                    stripeoff[stripe] += readlen;
                    pos += readlen;

                    stripe = (stripe + 1) % ci->num_stripes;
                }
            }
        }

        MmUnlockPages(master_mdl);
        IoFreeMdl(master_mdl);

        ExFreePool(stripeoff);
    } else if (type == BLOCK_FLAG_RAID6) {
        UINT64 startoff, endoff;
        UINT16 endoffstripe, parity1;
        UINT32 *stripeoff, pos;
        PMDL master_mdl;
        PFN_NUMBER *pfns, dummy;
        BOOL need_dummy = FALSE;

        get_raid0_offset(addr - offset, ci->stripe_length, ci->num_stripes - 2, &startoff, &startoffstripe);
        get_raid0_offset(addr + length - offset - 1, ci->stripe_length, ci->num_stripes - 2, &endoff, &endoffstripe);

        if (file_read) {
            context.va = ExAllocatePoolWithTag(NonPagedPool, length, ALLOC_TAG);

            if (!context.va) {
                ERR("out of memory\n");
                Status = STATUS_INSUFFICIENT_RESOURCES;
                goto exit;
            }
        } else
            context.va = buf;

        master_mdl = IoAllocateMdl(context.va, length, FALSE, FALSE, NULL);
        if (!master_mdl) {
            ERR("out of memory\n");
            Status = STATUS_INSUFFICIENT_RESOURCES;
            goto exit;
        }

        Status = STATUS_SUCCESS;

        _SEH2_TRY {
            MmProbeAndLockPages(master_mdl, KernelMode, IoWriteAccess);
        } _SEH2_EXCEPT (EXCEPTION_EXECUTE_HANDLER) {
            Status = _SEH2_GetExceptionCode();
        } _SEH2_END;

        if (!NT_SUCCESS(Status)) {
            ERR("MmProbeAndLockPages threw exception %08x\n", Status);
            IoFreeMdl(master_mdl);
            goto exit;
        }

        pfns = (PFN_NUMBER*)(master_mdl + 1);

        pos = 0;
        while (pos < length) {
            parity1 = (((addr - offset + pos) / ((ci->num_stripes - 2) * ci->stripe_length)) + ci->num_stripes - 2) % ci->num_stripes;

            if (pos == 0) {
                UINT16 stripe = (parity1 + startoffstripe + 2) % ci->num_stripes, parity2;
                ULONG skip, readlen;

                i = startoffstripe;
                while (stripe != parity1) {
                    if (i == startoffstripe) {
                        readlen = (ULONG)min(length, ci->stripe_length - (startoff % ci->stripe_length));

                        context.stripes[stripe].stripestart = startoff;
                        context.stripes[stripe].stripeend = startoff + readlen;

                        pos += readlen;

                        if (pos == length)
                            break;
                    } else {
                        readlen = min(length - pos, (ULONG)ci->stripe_length);

                        context.stripes[stripe].stripestart = startoff - (startoff % ci->stripe_length);
                        context.stripes[stripe].stripeend = context.stripes[stripe].stripestart + readlen;

                        pos += readlen;

                        if (pos == length)
                            break;
                    }

                    i++;
                    stripe = (stripe + 1) % ci->num_stripes;
                }

                if (pos == length)
                    break;

                for (i = 0; i < startoffstripe; i++) {
                    UINT16 stripe2 = (parity1 + i + 2) % ci->num_stripes;

                    context.stripes[stripe2].stripestart = context.stripes[stripe2].stripeend = startoff - (startoff % ci->stripe_length) + ci->stripe_length;
                }

                context.stripes[parity1].stripestart = context.stripes[parity1].stripeend = startoff - (startoff % ci->stripe_length) + ci->stripe_length;

                parity2 = (parity1 + 1) % ci->num_stripes;
                context.stripes[parity2].stripestart = context.stripes[parity2].stripeend = startoff - (startoff % ci->stripe_length) + ci->stripe_length;

                if (length - pos > ci->num_stripes * (ci->num_stripes - 2) * ci->stripe_length) {
                    skip = (ULONG)(((length - pos) / (ci->num_stripes * (ci->num_stripes - 2) * ci->stripe_length)) - 1);

                    for (i = 0; i < ci->num_stripes; i++) {
                        context.stripes[i].stripeend += skip * ci->num_stripes * ci->stripe_length;
                    }

                    pos += (UINT32)(skip * (ci->num_stripes - 2) * ci->num_stripes * ci->stripe_length);
                    need_dummy = TRUE;
                }
            } else if (length - pos >= ci->stripe_length * (ci->num_stripes - 2)) {
                for (i = 0; i < ci->num_stripes; i++) {
                    context.stripes[i].stripeend += ci->stripe_length;
                }

                pos += (UINT32)(ci->stripe_length * (ci->num_stripes - 2));
                need_dummy = TRUE;
            } else {
                UINT16 stripe = (parity1 + 2) % ci->num_stripes;

                i = 0;
                while (stripe != parity1) {
                    if (endoffstripe == i) {
                        context.stripes[stripe].stripeend = endoff + 1;
                        break;
                    } else if (endoffstripe > i)
                        context.stripes[stripe].stripeend = endoff - (endoff % ci->stripe_length) + ci->stripe_length;

                    i++;
                    stripe = (stripe + 1) % ci->num_stripes;
                }

                break;
            }
        }

        for (i = 0; i < ci->num_stripes; i++) {
            if (context.stripes[i].stripestart != context.stripes[i].stripeend) {
                context.stripes[i].mdl = IoAllocateMdl(context.va, (ULONG)(context.stripes[i].stripeend - context.stripes[i].stripestart), FALSE, FALSE, NULL);

                if (!context.stripes[i].mdl) {
                    ERR("IoAllocateMdl failed\n");
                    MmUnlockPages(master_mdl);
                    IoFreeMdl(master_mdl);
                    Status = STATUS_INSUFFICIENT_RESOURCES;
                    goto exit;
                }
            }
        }

        if (need_dummy) {
            dummypage = ExAllocatePoolWithTag(NonPagedPool, PAGE_SIZE, ALLOC_TAG);
            if (!dummypage) {
                ERR("out of memory\n");
                MmUnlockPages(master_mdl);
                IoFreeMdl(master_mdl);
                Status = STATUS_INSUFFICIENT_RESOURCES;
                goto exit;
            }

            dummy_mdl = IoAllocateMdl(dummypage, PAGE_SIZE, FALSE, FALSE, NULL);
            if (!dummy_mdl) {
                ERR("IoAllocateMdl failed\n");
                MmUnlockPages(master_mdl);
                IoFreeMdl(master_mdl);
                Status = STATUS_INSUFFICIENT_RESOURCES;
                goto exit;
            }

            MmBuildMdlForNonPagedPool(dummy_mdl);

            dummy = *(PFN_NUMBER*)(dummy_mdl + 1);
        }

        stripeoff = ExAllocatePoolWithTag(NonPagedPool, sizeof(UINT32) * ci->num_stripes, ALLOC_TAG);
        if (!stripeoff) {
            ERR("out of memory\n");
            MmUnlockPages(master_mdl);
            IoFreeMdl(master_mdl);
            Status = STATUS_INSUFFICIENT_RESOURCES;
            goto exit;
        }

        RtlZeroMemory(stripeoff, sizeof(UINT32) * ci->num_stripes);

        pos = 0;

        while (pos < length) {
            PFN_NUMBER* stripe_pfns;

            parity1 = (((addr - offset + pos) / ((ci->num_stripes - 2) * ci->stripe_length)) + ci->num_stripes - 2) % ci->num_stripes;

            if (pos == 0) {
                UINT16 stripe = (parity1 + startoffstripe + 2) % ci->num_stripes;
                UINT32 readlen = min(length - pos, (UINT32)min(context.stripes[stripe].stripeend - context.stripes[stripe].stripestart,
                                                       ci->stripe_length - (context.stripes[stripe].stripestart % ci->stripe_length)));

                stripe_pfns = (PFN_NUMBER*)(context.stripes[stripe].mdl + 1);

                RtlCopyMemory(stripe_pfns, pfns, readlen * sizeof(PFN_NUMBER) >> PAGE_SHIFT);

                stripeoff[stripe] = readlen;
                pos += readlen;

                stripe = (stripe + 1) % ci->num_stripes;

                while (stripe != parity1) {
                    stripe_pfns = (PFN_NUMBER*)(context.stripes[stripe].mdl + 1);
                    readlen = (UINT32)min(length - pos, min(context.stripes[stripe].stripeend - context.stripes[stripe].stripestart, ci->stripe_length));

                    if (readlen == 0)
                        break;

                    RtlCopyMemory(stripe_pfns, &pfns[pos >> PAGE_SHIFT], readlen * sizeof(PFN_NUMBER) >> PAGE_SHIFT);

                    stripeoff[stripe] = readlen;
                    pos += readlen;

                    stripe = (stripe + 1) % ci->num_stripes;
                }
            } else if (length - pos >= ci->stripe_length * (ci->num_stripes - 2)) {
                UINT16 stripe = (parity1 + 2) % ci->num_stripes;
                UINT16 parity2 = (parity1 + 1) % ci->num_stripes;
                ULONG k;

                while (stripe != parity1) {
                    stripe_pfns = (PFN_NUMBER*)(context.stripes[stripe].mdl + 1);

                    RtlCopyMemory(&stripe_pfns[stripeoff[stripe] >> PAGE_SHIFT], &pfns[pos >> PAGE_SHIFT], (ULONG)(ci->stripe_length * sizeof(PFN_NUMBER) >> PAGE_SHIFT));

                    stripeoff[stripe] += (UINT32)ci->stripe_length;
                    pos += (UINT32)ci->stripe_length;

                    stripe = (stripe + 1) % ci->num_stripes;
                }

                stripe_pfns = (PFN_NUMBER*)(context.stripes[parity1].mdl + 1);

                for (k = 0; k < ci->stripe_length >> PAGE_SHIFT; k++) {
                    stripe_pfns[stripeoff[parity1] >> PAGE_SHIFT] = dummy;
                    stripeoff[parity1] += PAGE_SIZE;
                }

                stripe_pfns = (PFN_NUMBER*)(context.stripes[parity2].mdl + 1);

                for (k = 0; k < ci->stripe_length >> PAGE_SHIFT; k++) {
                    stripe_pfns[stripeoff[parity2] >> PAGE_SHIFT] = dummy;
                    stripeoff[parity2] += PAGE_SIZE;
                }
            } else {
                UINT16 stripe = (parity1 + 2) % ci->num_stripes;
                UINT32 readlen;

                while (pos < length) {
                    stripe_pfns = (PFN_NUMBER*)(context.stripes[stripe].mdl + 1);
                    readlen = (UINT32)min(length - pos, min(context.stripes[stripe].stripeend - context.stripes[stripe].stripestart, ci->stripe_length));

                    if (readlen == 0)
                        break;

                    RtlCopyMemory(&stripe_pfns[stripeoff[stripe] >> PAGE_SHIFT], &pfns[pos >> PAGE_SHIFT], readlen * sizeof(PFN_NUMBER) >> PAGE_SHIFT);

                    stripeoff[stripe] += readlen;
                    pos += readlen;

                    stripe = (stripe + 1) % ci->num_stripes;
                }
            }
        }

        MmUnlockPages(master_mdl);
        IoFreeMdl(master_mdl);

        ExFreePool(stripeoff);
    }

    context.address = addr;

    for (i = 0; i < ci->num_stripes; i++) {
        if (!devices[i] || !devices[i]->devobj || context.stripes[i].stripestart == context.stripes[i].stripeend) {
            context.stripes[i].status = ReadDataStatus_MissingDevice;
            context.stripes_left--;

            if (!devices[i] || !devices[i]->devobj)
                missing_devices++;
        }
    }

    if (missing_devices > allowed_missing) {
        ERR("not enough devices to service request (%u missing)\n", missing_devices);
        Status = STATUS_UNEXPECTED_IO_ERROR;
        goto exit;
    }

    for (i = 0; i < ci->num_stripes; i++) {
        PIO_STACK_LOCATION IrpSp;

        if (devices[i] && devices[i]->devobj && context.stripes[i].stripestart != context.stripes[i].stripeend && context.stripes[i].status != ReadDataStatus_Skip) {
            context.stripes[i].context = (struct read_data_context*)&context;

            if (type == BLOCK_FLAG_RAID10) {
                context.stripes[i].stripenum = i / ci->sub_stripes;
            }

            if (!Irp) {
                context.stripes[i].Irp = IoAllocateIrp(devices[i]->devobj->StackSize, FALSE);

                if (!context.stripes[i].Irp) {
                    ERR("IoAllocateIrp failed\n");
                    Status = STATUS_INSUFFICIENT_RESOURCES;
                    goto exit;
                }
            } else {
                context.stripes[i].Irp = IoMakeAssociatedIrp(Irp, devices[i]->devobj->StackSize);

                if (!context.stripes[i].Irp) {
                    ERR("IoMakeAssociatedIrp failed\n");
                    Status = STATUS_INSUFFICIENT_RESOURCES;
                    goto exit;
                }
            }

            IrpSp = IoGetNextIrpStackLocation(context.stripes[i].Irp);
            IrpSp->MajorFunction = IRP_MJ_READ;

            if (devices[i]->devobj->Flags & DO_BUFFERED_IO) {
                context.stripes[i].Irp->AssociatedIrp.SystemBuffer = ExAllocatePoolWithTag(NonPagedPool, (ULONG)(context.stripes[i].stripeend - context.stripes[i].stripestart), ALLOC_TAG);
                if (!context.stripes[i].Irp->AssociatedIrp.SystemBuffer) {
                    ERR("out of memory\n");
                    Status = STATUS_INSUFFICIENT_RESOURCES;
                    goto exit;
                }

                context.stripes[i].Irp->Flags |= IRP_BUFFERED_IO | IRP_DEALLOCATE_BUFFER | IRP_INPUT_OPERATION;

                context.stripes[i].Irp->UserBuffer = MmGetSystemAddressForMdlSafe(context.stripes[i].mdl, priority);
            } else if (devices[i]->devobj->Flags & DO_DIRECT_IO)
                context.stripes[i].Irp->MdlAddress = context.stripes[i].mdl;
            else
                context.stripes[i].Irp->UserBuffer = MmGetSystemAddressForMdlSafe(context.stripes[i].mdl, priority);

            IrpSp->Parameters.Read.Length = (ULONG)(context.stripes[i].stripeend - context.stripes[i].stripestart);
            IrpSp->Parameters.Read.ByteOffset.QuadPart = context.stripes[i].stripestart + cis[i].offset;

            total_reading += IrpSp->Parameters.Read.Length;

            context.stripes[i].Irp->UserIosb = &context.stripes[i].iosb;

            IoSetCompletionRoutine(context.stripes[i].Irp, read_data_completion, &context.stripes[i], TRUE, TRUE, TRUE);

            context.stripes[i].status = ReadDataStatus_Pending;
        }
    }

#ifdef DEBUG_STATS
    if (!is_tree)
        time1 = KeQueryPerformanceCounter(NULL);
#endif

    need_to_wait = FALSE;
    for (i = 0; i < ci->num_stripes; i++) {
        if (context.stripes[i].status != ReadDataStatus_MissingDevice && context.stripes[i].status != ReadDataStatus_Skip) {
            IoCallDriver(devices[i]->devobj, context.stripes[i].Irp);
            need_to_wait = TRUE;
        }
    }

    if (need_to_wait)
        KeWaitForSingleObject(&context.Event, Executive, KernelMode, FALSE, NULL);

#ifdef DEBUG_STATS
    if (!is_tree) {
        time2 = KeQueryPerformanceCounter(NULL);

        Vcb->stats.read_disk_time += time2.QuadPart - time1.QuadPart;
    }
#endif

    if (diskacc)
        fFsRtlUpdateDiskCounters(total_reading, 0);

    // check if any of the devices return a "user-induced" error

    for (i = 0; i < ci->num_stripes; i++) {
        if (context.stripes[i].status == ReadDataStatus_Error && IoIsErrorUserInduced(context.stripes[i].iosb.Status)) {
            Status = context.stripes[i].iosb.Status;
            goto exit;
        }
    }

    if (type == BLOCK_FLAG_RAID0) {
        Status = read_data_raid0(Vcb, file_read ? context.va : buf, addr, length, &context, ci, devices, generation, offset);
        if (!NT_SUCCESS(Status)) {
            ERR("read_data_raid0 returned %08x\n", Status);

            if (file_read)
                ExFreePool(context.va);

            goto exit;
        }

        if (file_read) {
            RtlCopyMemory(buf, context.va, length);
            ExFreePool(context.va);
        }
    } else if (type == BLOCK_FLAG_RAID10) {
        Status = read_data_raid10(Vcb, file_read ? context.va : buf, addr, length, &context, ci, devices, generation, offset);

        if (!NT_SUCCESS(Status)) {
            ERR("read_data_raid10 returned %08x\n", Status);

            if (file_read)
                ExFreePool(context.va);

            goto exit;
        }

        if (file_read) {
            RtlCopyMemory(buf, context.va, length);
            ExFreePool(context.va);
        }
    } else if (type == BLOCK_FLAG_DUPLICATE) {
        Status = read_data_dup(Vcb, file_read ? context.va : buf, addr, &context, ci, devices, generation);
        if (!NT_SUCCESS(Status)) {
            ERR("read_data_dup returned %08x\n", Status);

            if (file_read)
                ExFreePool(context.va);

            goto exit;
        }

        if (file_read) {
            RtlCopyMemory(buf, context.va, length);
            ExFreePool(context.va);
        }
    } else if (type == BLOCK_FLAG_RAID5) {
        Status = read_data_raid5(Vcb, file_read ? context.va : buf, addr, length, &context, ci, devices, offset, generation, c, missing_devices > 0 ? TRUE : FALSE);
        if (!NT_SUCCESS(Status)) {
            ERR("read_data_raid5 returned %08x\n", Status);

            if (file_read)
                ExFreePool(context.va);

            goto exit;
        }

        if (file_read) {
            RtlCopyMemory(buf, context.va, length);
            ExFreePool(context.va);
        }
    } else if (type == BLOCK_FLAG_RAID6) {
        Status = read_data_raid6(Vcb, file_read ? context.va : buf, addr, length, &context, ci, devices, offset, generation, c, missing_devices > 0 ? TRUE : FALSE);
        if (!NT_SUCCESS(Status)) {
            ERR("read_data_raid6 returned %08x\n", Status);

            if (file_read)
                ExFreePool(context.va);

            goto exit;
        }

        if (file_read) {
            RtlCopyMemory(buf, context.va, length);
            ExFreePool(context.va);
        }
    }

exit:
    if (c && (type == BLOCK_FLAG_RAID5 || type == BLOCK_FLAG_RAID6))
        chunk_unlock_range(Vcb, c, lockaddr, locklen);

    if (dummy_mdl)
        IoFreeMdl(dummy_mdl);

    if (dummypage)
        ExFreePool(dummypage);

    for (i = 0; i < ci->num_stripes; i++) {
        if (context.stripes[i].mdl) {
            if (context.stripes[i].mdl->MdlFlags & MDL_PAGES_LOCKED)
                MmUnlockPages(context.stripes[i].mdl);

            IoFreeMdl(context.stripes[i].mdl);
        }

        if (context.stripes[i].Irp)
            IoFreeIrp(context.stripes[i].Irp);
    }

    ExFreePool(context.stripes);

    if (!Vcb->log_to_phys_loaded)
        ExFreePool(devices);

    return Status;
}

Referenced by read_file().

read_data_dup()

static NTSTATUS read_data_dup ( device_extension *  Vcb, UINT8 *  buf, UINT64  addr, read_data_context *  context, CHUNK_ITEM *  ci, device **  devices, UINT64  generation  )

static

Definition at line 138 of file read.c.

                                                                   {
    ULONG i;
    BOOL checksum_error = FALSE;
    UINT16 j, stripe = 0;
    NTSTATUS Status;
    CHUNK_ITEM_STRIPE* cis = (CHUNK_ITEM_STRIPE*)&ci[1];

    for (j = 0; j < ci->num_stripes; j++) {
        if (context->stripes[j].status == ReadDataStatus_Error) {
            WARN("stripe %u returned error %08x\n", j, context->stripes[j].iosb.Status);
            log_device_error(Vcb, devices[j], BTRFS_DEV_STAT_READ_ERRORS);
            return context->stripes[j].iosb.Status;
        } else if (context->stripes[j].status == ReadDataStatus_Success) {
            stripe = j;
            break;
        }
    }

    if (context->stripes[stripe].status != ReadDataStatus_Success)
        return STATUS_INTERNAL_ERROR;

    if (context->tree) {
        tree_header* th = (tree_header*)buf;
        UINT32 crc32;

        crc32 = ~calc_crc32c(0xffffffff, (UINT8*)&th->fs_uuid, context->buflen - sizeof(th->csum));

        if (th->address != context->address || crc32 != *((UINT32*)th->csum)) {
            checksum_error = TRUE;
            log_device_error(Vcb, devices[stripe], BTRFS_DEV_STAT_CORRUPTION_ERRORS);
        } else if (generation != 0 && th->generation != generation) {
            checksum_error = TRUE;
            log_device_error(Vcb, devices[stripe], BTRFS_DEV_STAT_GENERATION_ERRORS);
        }
    } else if (context->csum) {
#ifdef DEBUG_STATS
        LARGE_INTEGER time1, time2;

        time1 = KeQueryPerformanceCounter(NULL);
#endif
        Status = check_csum(Vcb, buf, (ULONG)context->stripes[stripe].Irp->IoStatus.Information / context->sector_size, context->csum);

        if (Status == STATUS_CRC_ERROR) {
            checksum_error = TRUE;
            log_device_error(Vcb, devices[stripe], BTRFS_DEV_STAT_CORRUPTION_ERRORS);
        } else if (!NT_SUCCESS(Status)) {
            ERR("check_csum returned %08x\n", Status);
            return Status;
        }
#ifdef DEBUG_STATS
        time2 = KeQueryPerformanceCounter(NULL);

        Vcb->stats.read_csum_time += time2.QuadPart - time1.QuadPart;
#endif
    }

    if (!checksum_error)
        return STATUS_SUCCESS;

    if (ci->num_stripes == 1)
        return STATUS_CRC_ERROR;

    if (context->tree) {
        tree_header* t2;
        BOOL recovered = FALSE;

        t2 = ExAllocatePoolWithTag(NonPagedPool, Vcb->superblock.node_size, ALLOC_TAG);
        if (!t2) {
            ERR("out of memory\n");
            return STATUS_INSUFFICIENT_RESOURCES;
        }

        for (j = 0; j < ci->num_stripes; j++) {
            if (j != stripe && devices[j] && devices[j]->devobj) {
                Status = sync_read_phys(devices[j]->devobj, cis[j].offset + context->stripes[stripe].stripestart, Vcb->superblock.node_size, (UINT8*)t2, FALSE);
                if (!NT_SUCCESS(Status)) {
                    WARN("sync_read_phys returned %08x\n", Status);
                    log_device_error(Vcb, devices[j], BTRFS_DEV_STAT_READ_ERRORS);
                } else {
                    UINT32 crc32 = ~calc_crc32c(0xffffffff, (UINT8*)&t2->fs_uuid, Vcb->superblock.node_size - sizeof(t2->csum));

                    if (t2->address == addr && crc32 == *((UINT32*)t2->csum) && (generation == 0 || t2->generation == generation)) {
                        RtlCopyMemory(buf, t2, Vcb->superblock.node_size);
                        ERR("recovering from checksum error at %llx, device %llx\n", addr, devices[stripe]->devitem.dev_id);
                        recovered = TRUE;

                        if (!Vcb->readonly && !devices[stripe]->readonly) { // write good data over bad
                            Status = write_data_phys(devices[stripe]->devobj, cis[stripe].offset + context->stripes[stripe].stripestart,
                                                     t2, Vcb->superblock.node_size);
                            if (!NT_SUCCESS(Status)) {
                                WARN("write_data_phys returned %08x\n", Status);
                                log_device_error(Vcb, devices[stripe], BTRFS_DEV_STAT_WRITE_ERRORS);
                            }
                        }

                        break;
                    } else if (t2->address != addr || crc32 != *((UINT32*)t2->csum))
                        log_device_error(Vcb, devices[j], BTRFS_DEV_STAT_CORRUPTION_ERRORS);
                    else
                        log_device_error(Vcb, devices[j], BTRFS_DEV_STAT_GENERATION_ERRORS);
                }
            }
        }

        if (!recovered) {
            ERR("unrecoverable checksum error at %llx\n", addr);
            ExFreePool(t2);
            return STATUS_CRC_ERROR;
        }

        ExFreePool(t2);
    } else {
        ULONG sectors = (ULONG)context->stripes[stripe].Irp->IoStatus.Information / Vcb->superblock.sector_size;
        UINT8* sector;

        sector = ExAllocatePoolWithTag(NonPagedPool, Vcb->superblock.sector_size, ALLOC_TAG);
        if (!sector) {
            ERR("out of memory\n");
            return STATUS_INSUFFICIENT_RESOURCES;
        }

        for (i = 0; i < sectors; i++) {
            UINT32 crc32 = ~calc_crc32c(0xffffffff, buf + (i * Vcb->superblock.sector_size), Vcb->superblock.sector_size);

            if (context->csum[i] != crc32) {
                BOOL recovered = FALSE;

                for (j = 0; j < ci->num_stripes; j++) {
                    if (j != stripe && devices[j] && devices[j]->devobj) {
                        Status = sync_read_phys(devices[j]->devobj, cis[j].offset + context->stripes[stripe].stripestart + UInt32x32To64(i, Vcb->superblock.sector_size),
                                                Vcb->superblock.sector_size, sector, FALSE);
                        if (!NT_SUCCESS(Status)) {
                            WARN("sync_read_phys returned %08x\n", Status);
                            log_device_error(Vcb, devices[j], BTRFS_DEV_STAT_READ_ERRORS);
                        } else {
                            UINT32 crc32b = ~calc_crc32c(0xffffffff, sector, Vcb->superblock.sector_size);

                            if (crc32b == context->csum[i]) {
                                RtlCopyMemory(buf + (i * Vcb->superblock.sector_size), sector, Vcb->superblock.sector_size);
                                ERR("recovering from checksum error at %llx, device %llx\n", addr + UInt32x32To64(i, Vcb->superblock.sector_size), devices[stripe]->devitem.dev_id);
                                recovered = TRUE;

                                if (!Vcb->readonly && !devices[stripe]->readonly) { // write good data over bad
                                    Status = write_data_phys(devices[stripe]->devobj, cis[stripe].offset + context->stripes[stripe].stripestart + UInt32x32To64(i, Vcb->superblock.sector_size),
                                                             sector, Vcb->superblock.sector_size);
                                    if (!NT_SUCCESS(Status)) {
                                        WARN("write_data_phys returned %08x\n", Status);
                                        log_device_error(Vcb, devices[stripe], BTRFS_DEV_STAT_WRITE_ERRORS);
                                    }
                                }

                                break;
                            } else
                                log_device_error(Vcb, devices[j], BTRFS_DEV_STAT_CORRUPTION_ERRORS);
                        }
                    }
                }

                if (!recovered) {
                    ERR("unrecoverable checksum error at %llx\n", addr + UInt32x32To64(i, Vcb->superblock.sector_size));
                    ExFreePool(sector);
                    return STATUS_CRC_ERROR;
                }
            }
        }

        ExFreePool(sector);
    }

    return STATUS_SUCCESS;
}

Referenced by read_data().

read_data_raid0()

static NTSTATUS read_data_raid0 ( device_extension *  Vcb, UINT8 *  buf, UINT64  addr, UINT32  length, read_data_context *  context, CHUNK_ITEM *  ci, device **  devices, UINT64  generation, UINT64  offset  )

static

Definition at line 311 of file read.c.

                                                                                                    {
    UINT64 i;

    for (i = 0; i < ci->num_stripes; i++) {
        if (context->stripes[i].status == ReadDataStatus_Error) {
            WARN("stripe %llu returned error %08x\n", i, context->stripes[i].iosb.Status);
            log_device_error(Vcb, devices[i], BTRFS_DEV_STAT_READ_ERRORS);
            return context->stripes[i].iosb.Status;
        }
    }

    if (context->tree) { // shouldn't happen, as trees shouldn't cross stripe boundaries
        tree_header* th = (tree_header*)buf;
        UINT32 crc32 = ~calc_crc32c(0xffffffff, (UINT8*)&th->fs_uuid, Vcb->superblock.node_size - sizeof(th->csum));

        if (crc32 != *((UINT32*)th->csum) || addr != th->address || (generation != 0 && generation != th->generation)) {
            UINT64 off;
            UINT16 stripe;

            get_raid0_offset(addr - offset, ci->stripe_length, ci->num_stripes, &off, &stripe);

            ERR("unrecoverable checksum error at %llx, device %llx\n", addr, devices[stripe]->devitem.dev_id);

            if (crc32 != *((UINT32*)th->csum)) {
                WARN("crc32 was %08x, expected %08x\n", crc32, *((UINT32*)th->csum));
                log_device_error(Vcb, devices[stripe], BTRFS_DEV_STAT_CORRUPTION_ERRORS);
                return STATUS_CRC_ERROR;
            } else if (addr != th->address) {
                WARN("address of tree was %llx, not %llx as expected\n", th->address, addr);
                log_device_error(Vcb, devices[stripe], BTRFS_DEV_STAT_CORRUPTION_ERRORS);
                return STATUS_CRC_ERROR;
            } else if (generation != 0 && generation != th->generation) {
                WARN("generation of tree was %llx, not %llx as expected\n", th->generation, generation);
                log_device_error(Vcb, devices[stripe], BTRFS_DEV_STAT_GENERATION_ERRORS);
                return STATUS_CRC_ERROR;
            }
        }
    } else if (context->csum) {
        NTSTATUS Status;
#ifdef DEBUG_STATS
        LARGE_INTEGER time1, time2;

        time1 = KeQueryPerformanceCounter(NULL);
#endif
        Status = check_csum(Vcb, buf, length / Vcb->superblock.sector_size, context->csum);

        if (Status == STATUS_CRC_ERROR) {
            for (i = 0; i < length / Vcb->superblock.sector_size; i++) {
                UINT32 crc32 = ~calc_crc32c(0xffffffff, buf + (i * Vcb->superblock.sector_size), Vcb->superblock.sector_size);

                if (context->csum[i] != crc32) {
                    UINT64 off;
                    UINT16 stripe;

                    get_raid0_offset(addr - offset + UInt32x32To64(i, Vcb->superblock.sector_size), ci->stripe_length, ci->num_stripes, &off, &stripe);

                    ERR("unrecoverable checksum error at %llx, device %llx\n", addr, devices[stripe]->devitem.dev_id);

                    log_device_error(Vcb, devices[stripe], BTRFS_DEV_STAT_CORRUPTION_ERRORS);

                    return Status;
                }
            }

            return Status;
        } else if (!NT_SUCCESS(Status)) {
            ERR("check_csum returned %08x\n", Status);
            return Status;
        }
#ifdef DEBUG_STATS
        time2 = KeQueryPerformanceCounter(NULL);

        Vcb->stats.read_csum_time += time2.QuadPart - time1.QuadPart;
#endif
    }

    return STATUS_SUCCESS;
}

Referenced by read_data().

read_data_raid10()

static NTSTATUS read_data_raid10 ( device_extension *  Vcb, UINT8 *  buf, UINT64  addr, UINT32  length, read_data_context *  context, CHUNK_ITEM *  ci, device **  devices, UINT64  generation, UINT64  offset  )

static

Definition at line 391 of file read.c.

                                                                                                     {
    UINT64 i;
    UINT16 j, stripe;
    NTSTATUS Status;
    BOOL checksum_error = FALSE;
    CHUNK_ITEM_STRIPE* cis = (CHUNK_ITEM_STRIPE*)&ci[1];

    for (j = 0; j < ci->num_stripes; j++) {
        if (context->stripes[j].status == ReadDataStatus_Error) {
            WARN("stripe %llu returned error %08x\n", j, context->stripes[j].iosb.Status);
            log_device_error(Vcb, devices[j], BTRFS_DEV_STAT_READ_ERRORS);
            return context->stripes[j].iosb.Status;
        } else if (context->stripes[j].status == ReadDataStatus_Success)
            stripe = j;
    }

    if (context->tree) {
        tree_header* th = (tree_header*)buf;
        UINT32 crc32 = ~calc_crc32c(0xffffffff, (UINT8*)&th->fs_uuid, Vcb->superblock.node_size - sizeof(th->csum));

        if (crc32 != *((UINT32*)th->csum)) {
            WARN("crc32 was %08x, expected %08x\n", crc32, *((UINT32*)th->csum));
            checksum_error = TRUE;
            log_device_error(Vcb, devices[stripe], BTRFS_DEV_STAT_CORRUPTION_ERRORS);
        } else if (addr != th->address) {
            WARN("address of tree was %llx, not %llx as expected\n", th->address, addr);
            checksum_error = TRUE;
            log_device_error(Vcb, devices[stripe], BTRFS_DEV_STAT_CORRUPTION_ERRORS);
        } else if (generation != 0 && generation != th->generation) {
            WARN("generation of tree was %llx, not %llx as expected\n", th->generation, generation);
            checksum_error = TRUE;
            log_device_error(Vcb, devices[stripe], BTRFS_DEV_STAT_GENERATION_ERRORS);
        }
    } else if (context->csum) {
#ifdef DEBUG_STATS
        LARGE_INTEGER time1, time2;

        time1 = KeQueryPerformanceCounter(NULL);
#endif
        Status = check_csum(Vcb, buf, length / Vcb->superblock.sector_size, context->csum);

        if (Status == STATUS_CRC_ERROR)
            checksum_error = TRUE;
        else if (!NT_SUCCESS(Status)) {
            ERR("check_csum returned %08x\n", Status);
            return Status;
        }
#ifdef DEBUG_STATS
        time2 = KeQueryPerformanceCounter(NULL);

        Vcb->stats.read_csum_time += time2.QuadPart - time1.QuadPart;
#endif
    }

    if (!checksum_error)
        return STATUS_SUCCESS;

    if (context->tree) {
        tree_header* t2;
        UINT64 off;
        UINT16 badsubstripe = 0;
        BOOL recovered = FALSE;

        t2 = ExAllocatePoolWithTag(NonPagedPool, Vcb->superblock.node_size, ALLOC_TAG);
        if (!t2) {
            ERR("out of memory\n");
            return STATUS_INSUFFICIENT_RESOURCES;
        }

        get_raid0_offset(addr - offset, ci->stripe_length, ci->num_stripes / ci->sub_stripes, &off, &stripe);

        stripe *= ci->sub_stripes;

        for (j = 0; j < ci->sub_stripes; j++) {
            if (context->stripes[stripe + j].status == ReadDataStatus_Success) {
                badsubstripe = j;
                break;
            }
        }

        for (j = 0; j < ci->sub_stripes; j++) {
            if (context->stripes[stripe + j].status != ReadDataStatus_Success && devices[stripe + j] && devices[stripe + j]->devobj) {
                Status = sync_read_phys(devices[stripe + j]->devobj, cis[stripe + j].offset + off,
                                        Vcb->superblock.node_size, (UINT8*)t2, FALSE);
                if (!NT_SUCCESS(Status)) {
                    WARN("sync_read_phys returned %08x\n", Status);
                    log_device_error(Vcb, devices[stripe + j], BTRFS_DEV_STAT_READ_ERRORS);
                } else {
                    UINT32 crc32 = ~calc_crc32c(0xffffffff, (UINT8*)&t2->fs_uuid, Vcb->superblock.node_size - sizeof(t2->csum));

                    if (t2->address == addr && crc32 == *((UINT32*)t2->csum) && (generation == 0 || t2->generation == generation)) {
                        RtlCopyMemory(buf, t2, Vcb->superblock.node_size);
                        ERR("recovering from checksum error at %llx, device %llx\n", addr, devices[stripe + j]->devitem.dev_id);
                        recovered = TRUE;

                        if (!Vcb->readonly && !devices[stripe + badsubstripe]->readonly && devices[stripe + badsubstripe]->devobj) { // write good data over bad
                            Status = write_data_phys(devices[stripe + badsubstripe]->devobj, cis[stripe + badsubstripe].offset + off,
                                                     t2, Vcb->superblock.node_size);
                            if (!NT_SUCCESS(Status)) {
                                WARN("write_data_phys returned %08x\n", Status);
                                log_device_error(Vcb, devices[stripe + badsubstripe], BTRFS_DEV_STAT_WRITE_ERRORS);
                            }
                        }

                        break;
                    } else if (t2->address != addr || crc32 != *((UINT32*)t2->csum))
                        log_device_error(Vcb, devices[stripe + j], BTRFS_DEV_STAT_CORRUPTION_ERRORS);
                    else
                        log_device_error(Vcb, devices[stripe + j], BTRFS_DEV_STAT_GENERATION_ERRORS);
                }
            }
        }

        if (!recovered) {
            ERR("unrecoverable checksum error at %llx\n", addr);
            ExFreePool(t2);
            return STATUS_CRC_ERROR;
        }

        ExFreePool(t2);
    } else {
        ULONG sectors = length / Vcb->superblock.sector_size;
        UINT8* sector;

        sector = ExAllocatePoolWithTag(NonPagedPool, Vcb->superblock.sector_size, ALLOC_TAG);
        if (!sector) {
            ERR("out of memory\n");
            return STATUS_INSUFFICIENT_RESOURCES;
        }

        for (i = 0; i < sectors; i++) {
            UINT32 crc32 = ~calc_crc32c(0xffffffff, buf + (i * Vcb->superblock.sector_size), Vcb->superblock.sector_size);

            if (context->csum[i] != crc32) {
                UINT64 off;
                UINT16 stripe2, badsubstripe = 0;
                BOOL recovered = FALSE;

                get_raid0_offset(addr - offset + UInt32x32To64(i, Vcb->superblock.sector_size), ci->stripe_length,
                                 ci->num_stripes / ci->sub_stripes, &off, &stripe2);

                stripe2 *= ci->sub_stripes;

                for (j = 0; j < ci->sub_stripes; j++) {
                    if (context->stripes[stripe2 + j].status == ReadDataStatus_Success) {
                        badsubstripe = j;
                        break;
                    }
                }

                log_device_error(Vcb, devices[stripe2 + badsubstripe], BTRFS_DEV_STAT_CORRUPTION_ERRORS);

                for (j = 0; j < ci->sub_stripes; j++) {
                    if (context->stripes[stripe2 + j].status != ReadDataStatus_Success && devices[stripe2 + j] && devices[stripe2 + j]->devobj) {
                        Status = sync_read_phys(devices[stripe2 + j]->devobj, cis[stripe2 + j].offset + off,
                                                Vcb->superblock.sector_size, sector, FALSE);
                        if (!NT_SUCCESS(Status)) {
                            WARN("sync_read_phys returned %08x\n", Status);
                            log_device_error(Vcb, devices[stripe2 + j], BTRFS_DEV_STAT_READ_ERRORS);
                        } else {
                            UINT32 crc32b = ~calc_crc32c(0xffffffff, sector, Vcb->superblock.sector_size);

                            if (crc32b == context->csum[i]) {
                                RtlCopyMemory(buf + (i * Vcb->superblock.sector_size), sector, Vcb->superblock.sector_size);
                                ERR("recovering from checksum error at %llx, device %llx\n", addr + UInt32x32To64(i, Vcb->superblock.sector_size), devices[stripe2 + j]->devitem.dev_id);
                                recovered = TRUE;

                                if (!Vcb->readonly && !devices[stripe2 + badsubstripe]->readonly && devices[stripe2 + badsubstripe]->devobj) { // write good data over bad
                                    Status = write_data_phys(devices[stripe2 + badsubstripe]->devobj, cis[stripe2 + badsubstripe].offset + off,
                                                             sector, Vcb->superblock.sector_size);
                                    if (!NT_SUCCESS(Status)) {
                                        WARN("write_data_phys returned %08x\n", Status);
                                        log_device_error(Vcb, devices[stripe2 + badsubstripe], BTRFS_DEV_STAT_READ_ERRORS);
                                    }
                                }

                                break;
                            } else
                                log_device_error(Vcb, devices[stripe2 + j], BTRFS_DEV_STAT_CORRUPTION_ERRORS);
                        }
                    }
                }

                if (!recovered) {
                    ERR("unrecoverable checksum error at %llx\n", addr + UInt32x32To64(i, Vcb->superblock.sector_size));
                    ExFreePool(sector);
                    return STATUS_CRC_ERROR;
                }
            }
        }

        ExFreePool(sector);
    }

    return STATUS_SUCCESS;
}

Referenced by read_data().

read_data_raid5()

static NTSTATUS read_data_raid5 ( device_extension *  Vcb, UINT8 *  buf, UINT64  addr, UINT32  length, read_data_context *  context, CHUNK_ITEM *  ci, device **  devices, UINT64  offset, UINT64  generation, chunk *  c, BOOL  degraded  )

static

Definition at line 589 of file read.c.

                                                                                                             {
    ULONG i;
    NTSTATUS Status;
    BOOL checksum_error = FALSE;
    CHUNK_ITEM_STRIPE* cis = (CHUNK_ITEM_STRIPE*)&ci[1];
    UINT16 j, stripe;
    BOOL no_success = TRUE;

    for (j = 0; j < ci->num_stripes; j++) {
        if (context->stripes[j].status == ReadDataStatus_Error) {
            WARN("stripe %u returned error %08x\n", j, context->stripes[j].iosb.Status);
            log_device_error(Vcb, devices[j], BTRFS_DEV_STAT_READ_ERRORS);
            return context->stripes[j].iosb.Status;
        } else if (context->stripes[j].status == ReadDataStatus_Success) {
            stripe = j;
            no_success = FALSE;
        }
    }

    if (c) {    // check partial stripes
        LIST_ENTRY* le;
        UINT64 ps_length = (ci->num_stripes - 1) * ci->stripe_length;

        ExAcquireResourceSharedLite(&c->partial_stripes_lock, TRUE);

        le = c->partial_stripes.Flink;
        while (le != &c->partial_stripes) {
            partial_stripe* ps = CONTAINING_RECORD(le, partial_stripe, list_entry);

            if (ps->address + ps_length > addr && ps->address < addr + length) {
                ULONG runlength, index;

                runlength = RtlFindFirstRunClear(&ps->bmp, &index);

                while (runlength != 0) {
                    UINT64 runstart = ps->address + (index * Vcb->superblock.sector_size);
                    UINT64 runend = runstart + (runlength * Vcb->superblock.sector_size);
                    UINT64 start = max(runstart, addr);
                    UINT64 end = min(runend, addr + length);

                    if (end > start)
                        RtlCopyMemory(buf + start - addr, &ps->data[start - ps->address], (ULONG)(end - start));

                    runlength = RtlFindNextForwardRunClear(&ps->bmp, index + runlength, &index);
                }
            } else if (ps->address >= addr + length)
                break;

            le = le->Flink;
        }

        ExReleaseResourceLite(&c->partial_stripes_lock);
    }

    if (context->tree) {
        tree_header* th = (tree_header*)buf;
        UINT32 crc32 = ~calc_crc32c(0xffffffff, (UINT8*)&th->fs_uuid, Vcb->superblock.node_size - sizeof(th->csum));

        if (addr != th->address || crc32 != *((UINT32*)th->csum)) {
            checksum_error = TRUE;
            if (!no_success && !degraded)
                log_device_error(Vcb, devices[stripe], BTRFS_DEV_STAT_CORRUPTION_ERRORS);
        } else if (generation != 0 && generation != th->generation) {
            checksum_error = TRUE;
            if (!no_success && !degraded)
                log_device_error(Vcb, devices[stripe], BTRFS_DEV_STAT_GENERATION_ERRORS);
        }
    } else if (context->csum) {
#ifdef DEBUG_STATS
        LARGE_INTEGER time1, time2;

        time1 = KeQueryPerformanceCounter(NULL);
#endif
        Status = check_csum(Vcb, buf, length / Vcb->superblock.sector_size, context->csum);

        if (Status == STATUS_CRC_ERROR) {
            if (!degraded)
                WARN("checksum error\n");
            checksum_error = TRUE;
        } else if (!NT_SUCCESS(Status)) {
            ERR("check_csum returned %08x\n", Status);
            return Status;
        }

#ifdef DEBUG_STATS
        time2 = KeQueryPerformanceCounter(NULL);

        Vcb->stats.read_csum_time += time2.QuadPart - time1.QuadPart;
#endif
    } else if (degraded)
        checksum_error = TRUE;

    if (!checksum_error)
        return STATUS_SUCCESS;

    if (context->tree) {
        UINT16 parity;
        UINT64 off;
        BOOL recovered = FALSE, first = TRUE, failed = FALSE;
        UINT8* t2;

        t2 = ExAllocatePoolWithTag(NonPagedPool, Vcb->superblock.node_size * 2, ALLOC_TAG);
        if (!t2) {
            ERR("out of memory\n");
            return STATUS_INSUFFICIENT_RESOURCES;
        }

        get_raid0_offset(addr - offset, ci->stripe_length, ci->num_stripes - 1, &off, &stripe);

        parity = (((addr - offset) / ((ci->num_stripes - 1) * ci->stripe_length)) + ci->num_stripes - 1) % ci->num_stripes;

        stripe = (parity + stripe + 1) % ci->num_stripes;

        for (j = 0; j < ci->num_stripes; j++) {
            if (j != stripe) {
                if (devices[j] && devices[j]->devobj) {
                    if (first) {
                        Status = sync_read_phys(devices[j]->devobj, cis[j].offset + off, Vcb->superblock.node_size, t2, FALSE);
                        if (!NT_SUCCESS(Status)) {
                            ERR("sync_read_phys returned %08x\n", Status);
                            log_device_error(Vcb, devices[j], BTRFS_DEV_STAT_READ_ERRORS);
                            failed = TRUE;
                            break;
                        }

                        first = FALSE;
                    } else {
                        Status = sync_read_phys(devices[j]->devobj, cis[j].offset + off, Vcb->superblock.node_size, t2 + Vcb->superblock.node_size, FALSE);
                        if (!NT_SUCCESS(Status)) {
                            ERR("sync_read_phys returned %08x\n", Status);
                            log_device_error(Vcb, devices[j], BTRFS_DEV_STAT_READ_ERRORS);
                            failed = TRUE;
                            break;
                        }

                        do_xor(t2, t2 + Vcb->superblock.node_size, Vcb->superblock.node_size);
                    }
                } else {
                    failed = TRUE;
                    break;
                }
            }
        }

        if (!failed) {
            tree_header* t3 = (tree_header*)t2;
            UINT32 crc32 = ~calc_crc32c(0xffffffff, (UINT8*)&t3->fs_uuid, Vcb->superblock.node_size - sizeof(t3->csum));

            if (t3->address == addr && crc32 == *((UINT32*)t3->csum) && (generation == 0 || t3->generation == generation)) {
                RtlCopyMemory(buf, t2, Vcb->superblock.node_size);

                if (!degraded)
                    ERR("recovering from checksum error at %llx, device %llx\n", addr, devices[stripe]->devitem.dev_id);

                recovered = TRUE;

                if (!Vcb->readonly && devices[stripe] && !devices[stripe]->readonly && devices[stripe]->devobj) { // write good data over bad
                    Status = write_data_phys(devices[stripe]->devobj, cis[stripe].offset + off, t2, Vcb->superblock.node_size);
                    if (!NT_SUCCESS(Status)) {
                        WARN("write_data_phys returned %08x\n", Status);
                        log_device_error(Vcb, devices[stripe], BTRFS_DEV_STAT_WRITE_ERRORS);
                    }
                }
            }
        }

        if (!recovered) {
            ERR("unrecoverable checksum error at %llx\n", addr);
            ExFreePool(t2);
            return STATUS_CRC_ERROR;
        }

        ExFreePool(t2);
    } else {
        ULONG sectors = length / Vcb->superblock.sector_size;
        UINT8* sector;

        sector = ExAllocatePoolWithTag(NonPagedPool, Vcb->superblock.sector_size * 2, ALLOC_TAG);
        if (!sector) {
            ERR("out of memory\n");
            return STATUS_INSUFFICIENT_RESOURCES;
        }

        for (i = 0; i < sectors; i++) {
            UINT16 parity;
            UINT64 off;
            UINT32 crc32;

            if (context->csum)
                crc32 = ~calc_crc32c(0xffffffff, buf + (i * Vcb->superblock.sector_size), Vcb->superblock.sector_size);

            get_raid0_offset(addr - offset + UInt32x32To64(i, Vcb->superblock.sector_size), ci->stripe_length,
                             ci->num_stripes - 1, &off, &stripe);

            parity = (((addr - offset + UInt32x32To64(i, Vcb->superblock.sector_size)) / ((ci->num_stripes - 1) * ci->stripe_length)) + ci->num_stripes - 1) % ci->num_stripes;

            stripe = (parity + stripe + 1) % ci->num_stripes;

            if (!devices[stripe] || !devices[stripe]->devobj || (context->csum && context->csum[i] != crc32)) {
                BOOL recovered = FALSE, first = TRUE, failed = FALSE;

                if (devices[stripe] && devices[stripe]->devobj)
                    log_device_error(Vcb, devices[stripe], BTRFS_DEV_STAT_READ_ERRORS);

                for (j = 0; j < ci->num_stripes; j++) {
                    if (j != stripe) {
                        if (devices[j] && devices[j]->devobj) {
                            if (first) {
                                Status = sync_read_phys(devices[j]->devobj, cis[j].offset + off, Vcb->superblock.sector_size, sector, FALSE);
                                if (!NT_SUCCESS(Status)) {
                                    ERR("sync_read_phys returned %08x\n", Status);
                                    failed = TRUE;
                                    log_device_error(Vcb, devices[j], BTRFS_DEV_STAT_READ_ERRORS);
                                    break;
                                }

                                first = FALSE;
                            } else {
                                Status = sync_read_phys(devices[j]->devobj, cis[j].offset + off, Vcb->superblock.sector_size, sector + Vcb->superblock.sector_size, FALSE);
                                if (!NT_SUCCESS(Status)) {
                                    ERR("sync_read_phys returned %08x\n", Status);
                                    failed = TRUE;
                                    log_device_error(Vcb, devices[j], BTRFS_DEV_STAT_READ_ERRORS);
                                    break;
                                }

                                do_xor(sector, sector + Vcb->superblock.sector_size, Vcb->superblock.sector_size);
                            }
                        } else {
                            failed = TRUE;
                            break;
                        }
                    }
                }

                if (!failed) {
                    if (context->csum)
                        crc32 = ~calc_crc32c(0xffffffff, sector, Vcb->superblock.sector_size);

                    if (!context->csum || crc32 == context->csum[i]) {
                        RtlCopyMemory(buf + (i * Vcb->superblock.sector_size), sector, Vcb->superblock.sector_size);

                        if (!degraded)
                            ERR("recovering from checksum error at %llx, device %llx\n", addr + UInt32x32To64(i, Vcb->superblock.sector_size), devices[stripe]->devitem.dev_id);

                        recovered = TRUE;

                        if (!Vcb->readonly && devices[stripe] && !devices[stripe]->readonly && devices[stripe]->devobj) { // write good data over bad
                            Status = write_data_phys(devices[stripe]->devobj, cis[stripe].offset + off,
                                                     sector, Vcb->superblock.sector_size);
                            if (!NT_SUCCESS(Status)) {
                                WARN("write_data_phys returned %08x\n", Status);
                                log_device_error(Vcb, devices[stripe], BTRFS_DEV_STAT_WRITE_ERRORS);
                            }
                        }
                    }
                }

                if (!recovered) {
                    ERR("unrecoverable checksum error at %llx\n", addr + UInt32x32To64(i, Vcb->superblock.sector_size));
                    ExFreePool(sector);
                    return STATUS_CRC_ERROR;
                }
            }
        }

        ExFreePool(sector);
    }

    return STATUS_SUCCESS;
}

Referenced by read_data().

read_data_raid6()

static NTSTATUS read_data_raid6 ( device_extension *  Vcb, UINT8 *  buf, UINT64  addr, UINT32  length, read_data_context *  context, CHUNK_ITEM *  ci, device **  devices, UINT64  offset, UINT64  generation, chunk *  c, BOOL  degraded  )

static

Definition at line 948 of file read.c.

                                                                                                             {
    NTSTATUS Status;
    ULONG i;
    BOOL checksum_error = FALSE;
    CHUNK_ITEM_STRIPE* cis = (CHUNK_ITEM_STRIPE*)&ci[1];
    UINT16 stripe, j;
    BOOL no_success = TRUE;

    for (j = 0; j < ci->num_stripes; j++) {
        if (context->stripes[j].status == ReadDataStatus_Error) {
            WARN("stripe %u returned error %08x\n", j, context->stripes[j].iosb.Status);

            if (devices[j])
                log_device_error(Vcb, devices[j], BTRFS_DEV_STAT_READ_ERRORS);
            return context->stripes[j].iosb.Status;
        } else if (context->stripes[j].status == ReadDataStatus_Success) {
            stripe = j;
            no_success = FALSE;
        }
    }

    if (c) {    // check partial stripes
        LIST_ENTRY* le;
        UINT64 ps_length = (ci->num_stripes - 2) * ci->stripe_length;

        ExAcquireResourceSharedLite(&c->partial_stripes_lock, TRUE);

        le = c->partial_stripes.Flink;
        while (le != &c->partial_stripes) {
            partial_stripe* ps = CONTAINING_RECORD(le, partial_stripe, list_entry);

            if (ps->address + ps_length > addr && ps->address < addr + length) {
                ULONG runlength, index;

                runlength = RtlFindFirstRunClear(&ps->bmp, &index);

                while (runlength != 0) {
                    UINT64 runstart = ps->address + (index * Vcb->superblock.sector_size);
                    UINT64 runend = runstart + (runlength * Vcb->superblock.sector_size);
                    UINT64 start = max(runstart, addr);
                    UINT64 end = min(runend, addr + length);

                    if (end > start)
                        RtlCopyMemory(buf + start - addr, &ps->data[start - ps->address], (ULONG)(end - start));

                    runlength = RtlFindNextForwardRunClear(&ps->bmp, index + runlength, &index);
                }
            } else if (ps->address >= addr + length)
                break;

            le = le->Flink;
        }

        ExReleaseResourceLite(&c->partial_stripes_lock);
    }

    if (context->tree) {
        tree_header* th = (tree_header*)buf;
        UINT32 crc32 = ~calc_crc32c(0xffffffff, (UINT8*)&th->fs_uuid, Vcb->superblock.node_size - sizeof(th->csum));

        if (addr != th->address || crc32 != *((UINT32*)th->csum)) {
            checksum_error = TRUE;
            if (!no_success && !degraded && devices[stripe])
                log_device_error(Vcb, devices[stripe], BTRFS_DEV_STAT_CORRUPTION_ERRORS);
        } else if (generation != 0 && generation != th->generation) {
            checksum_error = TRUE;
            if (!no_success && !degraded && devices[stripe])
                log_device_error(