scrub.c File Reference

#include "btrfs_drv.h"

Include dependency graph for scrub.c:

Go to the source code of this file.

Classes
struct	scrub_context_stripe
struct	_scrub_context
struct	path_part
struct	scrub_context_raid56_stripe
struct	scrub_context_raid56

Macros
#define	SCRUB_UNIT   0x100000

Typedefs
typedef struct _scrub_context	scrub_context

Functions
static void	log_file_checksum_error (device_extension *Vcb, uint64_t addr, uint64_t devid, uint64_t subvol, uint64_t inode, uint64_t offset)
static void	log_file_checksum_error_shared (device_extension *Vcb, uint64_t treeaddr, uint64_t addr, uint64_t devid, uint64_t extent)
static void	log_tree_checksum_error (device_extension *Vcb, uint64_t addr, uint64_t devid, uint64_t root, uint8_t level, KEY *firstitem)
static void	log_tree_checksum_error_shared (device_extension *Vcb, uint64_t offset, uint64_t address, uint64_t devid)
static void	log_unrecoverable_error (device_extension *Vcb, uint64_t address, uint64_t devid)
static void	log_error (device_extension *Vcb, uint64_t addr, uint64_t devid, bool metadata, bool recoverable, bool parity)
	_Function_class_ (IO_COMPLETION_ROUTINE)
static NTSTATUS	scrub_extent_dup (device_extension *Vcb, chunk *c, uint64_t offset, void *csum, scrub_context *context)
static NTSTATUS	scrub_extent_raid0 (device_extension *Vcb, chunk *c, uint64_t offset, uint32_t length, uint16_t startoffstripe, void *csum, scrub_context *context)
static NTSTATUS	scrub_extent_raid10 (device_extension *Vcb, chunk *c, uint64_t offset, uint32_t length, uint16_t startoffstripe, void *csum, scrub_context *context)
static NTSTATUS	scrub_extent (device_extension *Vcb, chunk *c, ULONG type, uint64_t offset, uint32_t size, void *csum)
static NTSTATUS	scrub_data_extent (device_extension *Vcb, chunk *c, uint64_t offset, ULONG type, void *csum, RTL_BITMAP *bmp, ULONG bmplen)
static void	scrub_raid5_stripe (device_extension *Vcb, chunk *c, scrub_context_raid56 *context, uint64_t stripe_start, uint64_t bit_start, uint64_t num, uint16_t missing_devices)
static void	scrub_raid6_stripe (device_extension *Vcb, chunk *c, scrub_context_raid56 *context, uint64_t stripe_start, uint64_t bit_start, uint64_t num, uint16_t missing_devices)
static NTSTATUS	scrub_chunk_raid56_stripe_run (device_extension *Vcb, chunk *c, uint64_t stripe_start, uint64_t stripe_end)
static NTSTATUS	scrub_chunk_raid56 (device_extension *Vcb, chunk *c, uint64_t *offset, bool *changed)
static NTSTATUS	scrub_chunk (device_extension *Vcb, chunk *c, uint64_t *offset, bool *changed)
	_Function_class_ (KSTART_ROUTINE)
NTSTATUS	start_scrub (device_extension *Vcb, KPROCESSOR_MODE processor_mode)
NTSTATUS	query_scrub (device_extension *Vcb, KPROCESSOR_MODE processor_mode, void *data, ULONG length)
NTSTATUS	pause_scrub (device_extension *Vcb, KPROCESSOR_MODE processor_mode)
NTSTATUS	resume_scrub (device_extension *Vcb, KPROCESSOR_MODE processor_mode)
NTSTATUS	stop_scrub (device_extension *Vcb, KPROCESSOR_MODE processor_mode)

Macro Definition Documentation

SCRUB_UNIT

#define SCRUB_UNIT   0x100000

Definition at line 20 of file scrub.c.

Typedef Documentation

scrub_context

typedef struct _scrub_context scrub_context

Function Documentation

_Function_class_() [1/2]

_Function_class_

(

IO_COMPLETION_ROUTINE

)

Definition at line 651 of file scrub.c.

                                                                                                     {
    scrub_context_stripe* stripe = conptr;
    scrub_context* context = (scrub_context*)stripe->context;
    ULONG left = InterlockedDecrement(&context->stripes_left);

    UNUSED(DeviceObject);

    stripe->iosb = Irp->IoStatus;

    if (left == 0)
        KeSetEvent(&context->Event, 0, false);

    return STATUS_MORE_PROCESSING_REQUIRED;
}

_Function_class_() [2/2]

_Function_class_

(

KSTART_ROUTINE

)

Definition at line 3139 of file scrub.c.

                                                  {
    device_extension* Vcb = context;
    LIST_ENTRY chunks, *le;
    NTSTATUS Status;
    LARGE_INTEGER time;

    KeInitializeEvent(&Vcb->scrub.finished, NotificationEvent, false);

    InitializeListHead(&chunks);

    ExAcquireResourceExclusiveLite(&Vcb->tree_lock, true);

    if (Vcb->need_write && !Vcb->readonly)
        Status = do_write(Vcb, NULL);
    else
        Status = STATUS_SUCCESS;

    free_trees(Vcb);

    if (!NT_SUCCESS(Status)) {
        ExReleaseResourceLite(&Vcb->tree_lock);
        ERR("do_write returned %08lx\n", Status);
        Vcb->scrub.error = Status;
        goto end;
    }

    ExConvertExclusiveToSharedLite(&Vcb->tree_lock);

    ExAcquireResourceExclusiveLite(&Vcb->scrub.stats_lock, true);

    KeQuerySystemTime(&Vcb->scrub.start_time);
    Vcb->scrub.finish_time.QuadPart = 0;
    Vcb->scrub.resume_time.QuadPart = Vcb->scrub.start_time.QuadPart;
    Vcb->scrub.duration.QuadPart = 0;
    Vcb->scrub.total_chunks = 0;
    Vcb->scrub.chunks_left = 0;
    Vcb->scrub.data_scrubbed = 0;
    Vcb->scrub.num_errors = 0;

    while (!IsListEmpty(&Vcb->scrub.errors)) {
        scrub_error* err = CONTAINING_RECORD(RemoveHeadList(&Vcb->scrub.errors), scrub_error, list_entry);
        ExFreePool(err);
    }

    ExAcquireResourceSharedLite(&Vcb->chunk_lock, true);

    le = Vcb->chunks.Flink;
    while (le != &Vcb->chunks) {
        chunk* c = CONTAINING_RECORD(le, chunk, list_entry);

        acquire_chunk_lock(c, Vcb);

        if (!c->readonly) {
            InsertTailList(&chunks, &c->list_entry_balance);
            Vcb->scrub.total_chunks++;
            Vcb->scrub.chunks_left++;
        }

        release_chunk_lock(c, Vcb);

        le = le->Flink;
    }

    ExReleaseResourceLite(&Vcb->chunk_lock);

    ExReleaseResource(&Vcb->scrub.stats_lock);

    ExReleaseResourceLite(&Vcb->tree_lock);

    while (!IsListEmpty(&chunks)) {
        chunk* c = CONTAINING_RECORD(RemoveHeadList(&chunks), chunk, list_entry_balance);
        uint64_t offset = c->offset;
        bool changed;

        c->reloc = true;

        KeWaitForSingleObject(&Vcb->scrub.event, Executive, KernelMode, false, NULL);

        if (!Vcb->scrub.stopping) {
            do {
                changed = false;

                Status = scrub_chunk(Vcb, c, &offset, &changed);
                if (!NT_SUCCESS(Status)) {
                    ERR("scrub_chunk returned %08lx\n", Status);
                    Vcb->scrub.stopping = true;
                    Vcb->scrub.error = Status;
                    break;
                }

                if (offset == c->offset + c->chunk_item->size || Vcb->scrub.stopping)
                    break;

                KeWaitForSingleObject(&Vcb->scrub.event, Executive, KernelMode, false, NULL);
            } while (changed);
        }

        ExAcquireResourceExclusiveLite(&Vcb->scrub.stats_lock, true);

        if (!Vcb->scrub.stopping)
            Vcb->scrub.chunks_left--;

        if (IsListEmpty(&chunks))
            KeQuerySystemTime(&Vcb->scrub.finish_time);

        ExReleaseResource(&Vcb->scrub.stats_lock);

        c->reloc = false;
        c->list_entry_balance.Flink = NULL;
    }

    KeQuerySystemTime(&time);
    Vcb->scrub.duration.QuadPart += time.QuadPart - Vcb->scrub.resume_time.QuadPart;

end:
    ZwClose(Vcb->scrub.thread);
    Vcb->scrub.thread = NULL;

    KeSetEvent(&Vcb->scrub.finished, 0, false);
}

log_error()

static void log_error ( device_extension *  Vcb, uint64_t  addr, uint64_t  devid, bool  metadata, bool  recoverable, bool  parity  )

static

Definition at line 605 of file scrub.c.

                                                                                                                          {
    if (recoverable) {
        scrub_error* err;

        if (parity) {
            ERR("recovering from parity error at %I64x on device %I64x\n", addr, devid);
        } else {
            if (metadata)
                ERR("recovering from metadata checksum error at %I64x on device %I64x\n", addr, devid);
            else
                ERR("recovering from data checksum error at %I64x on device %I64x\n", addr, devid);
        }

        err = ExAllocatePoolWithTag(PagedPool, sizeof(scrub_error), ALLOC_TAG);
        if (!err) {
            ERR("out of memory\n");
            return;
        }

        err->address = addr;
        err->device = devid;
        err->recovered = true;
        err->is_metadata = metadata;
        err->parity = parity;

        if (metadata)
            RtlZeroMemory(&err->metadata, sizeof(err->metadata));
        else
            RtlZeroMemory(&err->data, sizeof(err->data));

        ExAcquireResourceExclusiveLite(&Vcb->scrub.stats_lock, true);

        Vcb->scrub.num_errors++;
        InsertTailList(&Vcb->scrub.errors, &err->list_entry);

        ExReleaseResourceLite(&Vcb->scrub.stats_lock);
    } else {
        if (metadata)
            ERR("unrecoverable metadata checksum error at %I64x\n", addr);
        else
            ERR("unrecoverable data checksum error at %I64x\n", addr);

        log_unrecoverable_error(Vcb, addr, devid);
    }
}

Referenced by scrub_extent_dup(), scrub_extent_raid0(), scrub_extent_raid10(), scrub_raid5_stripe(), and scrub_raid6_stripe().

log_file_checksum_error()

static void log_file_checksum_error ( device_extension *  Vcb, uint64_t  addr, uint64_t  devid, uint64_t  subvol, uint64_t  inode, uint64_t  offset  )

static

Definition at line 47 of file scrub.c.

                                                                                                                                            {
    LIST_ENTRY *le, parts;
    root* r = NULL;
    KEY searchkey;
    traverse_ptr tp;
    uint64_t dir;
    bool orig_subvol = true, not_in_tree = false;
    ANSI_STRING fn;
    scrub_error* err;
    NTSTATUS Status;
    ULONG utf16len;

    le = Vcb->roots.Flink;
    while (le != &Vcb->roots) {
        root* r2 = CONTAINING_RECORD(le, root, list_entry);

        if (r2->id == subvol) {
            r = r2;
            break;
        }

        le = le->Flink;
    }

    if (!r) {
        ERR("could not find subvol %I64x\n", subvol);
        return;
    }

    InitializeListHead(&parts);

    dir = inode;

    while (true) {
        if (dir == r->root_item.objid) {
            if (r == Vcb->root_fileref->fcb->subvol)
                break;

            searchkey.obj_id = r->id;
            searchkey.obj_type = TYPE_ROOT_BACKREF;
            searchkey.offset = 0xffffffffffffffff;

            Status = find_item(Vcb, Vcb->root_root, &tp, &searchkey, false, NULL);
            if (!NT_SUCCESS(Status)) {
                ERR("find_item returned %08lx\n", Status);
                goto end;
            }

            if (tp.item->key.obj_id == searchkey.obj_id && tp.item->key.obj_type == searchkey.obj_type) {
                ROOT_REF* rr = (ROOT_REF*)tp.item->data;
                path_part* pp;

                if (tp.item->size < sizeof(ROOT_REF)) {
                    ERR("(%I64x,%x,%I64x) was %u bytes, expected at least %Iu\n", tp.item->key.obj_id, tp.item->key.obj_type, tp.item->key.offset, tp.item->size, sizeof(ROOT_REF));
                    goto end;
                }

                if (tp.item->size < offsetof(ROOT_REF, name[0]) + rr->n) {
                    ERR("(%I64x,%x,%I64x) was %u bytes, expected at least %Iu\n", tp.item->key.obj_id, tp.item->key.obj_type, tp.item->key.offset,
                        tp.item->size, offsetof(ROOT_REF, name[0]) + rr->n);
                    goto end;
                }

                pp = ExAllocatePoolWithTag(PagedPool, sizeof(path_part), ALLOC_TAG);
                if (!pp) {
                    ERR("out of memory\n");
                    goto end;
                }

                pp->name.Buffer = rr->name;
                pp->name.Length = pp->name.MaximumLength = rr->n;
                pp->orig_subvol = false;

                InsertTailList(&parts, &pp->list_entry);

                r = NULL;

                le = Vcb->roots.Flink;
                while (le != &Vcb->roots) {
                    root* r2 = CONTAINING_RECORD(le, root, list_entry);

                    if (r2->id == tp.item->key.offset) {
                        r = r2;
                        break;
                    }

                    le = le->Flink;
                }

                if (!r) {
                    ERR("could not find subvol %I64x\n", tp.item->key.offset);
                    goto end;
                }

                dir = rr->dir;
                orig_subvol = false;
            } else {
                not_in_tree = true;
                break;
            }
        } else {
            searchkey.obj_id = dir;
            searchkey.obj_type = TYPE_INODE_EXTREF;
            searchkey.offset = 0xffffffffffffffff;

            Status = find_item(Vcb, r, &tp, &searchkey, false, NULL);
            if (!NT_SUCCESS(Status)) {
                ERR("find_item returned %08lx\n", Status);
                goto end;
            }

            if (tp.item->key.obj_id == searchkey.obj_id && tp.item->key.obj_type == TYPE_INODE_REF) {
                INODE_REF* ir = (INODE_REF*)tp.item->data;
                path_part* pp;

                if (tp.item->size < sizeof(INODE_REF)) {
                    ERR("(%I64x,%x,%I64x) was %u bytes, expected at least %Iu\n", tp.item->key.obj_id, tp.item->key.obj_type, tp.item->key.offset, tp.item->size, sizeof(INODE_REF));
                    goto end;
                }

                if (tp.item->size < offsetof(INODE_REF, name[0]) + ir->n) {
                    ERR("(%I64x,%x,%I64x) was %u bytes, expected at least %Iu\n", tp.item->key.obj_id, tp.item->key.obj_type, tp.item->key.offset,
                        tp.item->size, offsetof(INODE_REF, name[0]) + ir->n);
                    goto end;
                }

                pp = ExAllocatePoolWithTag(PagedPool, sizeof(path_part), ALLOC_TAG);
                if (!pp) {
                    ERR("out of memory\n");
                    goto end;
                }

                pp->name.Buffer = ir->name;
                pp->name.Length = pp->name.MaximumLength = ir->n;
                pp->orig_subvol = orig_subvol;

                InsertTailList(&parts, &pp->list_entry);

                if (dir == tp.item->key.offset)
                    break;

                dir = tp.item->key.offset;
            } else if (tp.item->key.obj_id == searchkey.obj_id && tp.item->key.obj_type == TYPE_INODE_EXTREF) {
                INODE_EXTREF* ier = (INODE_EXTREF*)tp.item->data;
                path_part* pp;

                if (tp.item->size < sizeof(INODE_EXTREF)) {
                    ERR("(%I64x,%x,%I64x) was %u bytes, expected at least %Iu\n", tp.item->key.obj_id, tp.item->key.obj_type, tp.item->key.offset,
                                                                                  tp.item->size, sizeof(INODE_EXTREF));
                    goto end;
                }

                if (tp.item->size < offsetof(INODE_EXTREF, name[0]) + ier->n) {
                    ERR("(%I64x,%x,%I64x) was %u bytes, expected at least %Iu\n", tp.item->key.obj_id, tp.item->key.obj_type, tp.item->key.offset,
                        tp.item->size, offsetof(INODE_EXTREF, name[0]) + ier->n);
                    goto end;
                }

                pp = ExAllocatePoolWithTag(PagedPool, sizeof(path_part), ALLOC_TAG);
                if (!pp) {
                    ERR("out of memory\n");
                    goto end;
                }

                pp->name.Buffer = ier->name;
                pp->name.Length = pp->name.MaximumLength = ier->n;
                pp->orig_subvol = orig_subvol;

                InsertTailList(&parts, &pp->list_entry);

                if (dir == ier->dir)
                    break;

                dir = ier->dir;
            } else {
                ERR("could not find INODE_REF for inode %I64x in subvol %I64x\n", dir, r->id);
                goto end;
            }
        }
    }

    fn.MaximumLength = 0;

    if (not_in_tree) {
        le = parts.Blink;
        while (le != &parts) {
            path_part* pp = CONTAINING_RECORD(le, path_part, list_entry);
            LIST_ENTRY* le2 = le->Blink;

            if (pp->orig_subvol)
                break;

            RemoveTailList(&parts);
            ExFreePool(pp);

            le = le2;
        }
    }

    le = parts.Flink;
    while (le != &parts) {
        path_part* pp = CONTAINING_RECORD(le, path_part, list_entry);

        fn.MaximumLength += pp->name.Length + 1;

        le = le->Flink;
    }

    fn.Buffer = ExAllocatePoolWithTag(PagedPool, fn.MaximumLength, ALLOC_TAG);
    if (!fn.Buffer) {
        ERR("out of memory\n");
        goto end;
    }

    fn.Length = 0;

    le = parts.Blink;
    while (le != &parts) {
        path_part* pp = CONTAINING_RECORD(le, path_part, list_entry);

        fn.Buffer[fn.Length] = '\\';
        fn.Length++;

        RtlCopyMemory(&fn.Buffer[fn.Length], pp->name.Buffer, pp->name.Length);
        fn.Length += pp->name.Length;

        le = le->Blink;
    }

    if (not_in_tree)
        ERR("subvol %I64x, %.*s, offset %I64x\n", subvol, fn.Length, fn.Buffer, offset);
    else
        ERR("%.*s, offset %I64x\n", fn.Length, fn.Buffer, offset);

    Status = utf8_to_utf16(NULL, 0, &utf16len, fn.Buffer, fn.Length);
    if (!NT_SUCCESS(Status)) {
        ERR("utf8_to_utf16 1 returned %08lx\n", Status);
        ExFreePool(fn.Buffer);
        goto end;
    }

    err = ExAllocatePoolWithTag(PagedPool, offsetof(scrub_error, data.filename[0]) + utf16len, ALLOC_TAG);
    if (!err) {
        ERR("out of memory\n");
        ExFreePool(fn.Buffer);
        goto end;
    }

    err->address = addr;
    err->device = devid;
    err->recovered = false;
    err->is_metadata = false;
    err->parity = false;

    err->data.subvol = not_in_tree ? subvol : 0;
    err->data.offset = offset;
    err->data.filename_length = (uint16_t)utf16len;

    Status = utf8_to_utf16(err->data.filename, utf16len, &utf16len, fn.Buffer, fn.Length);
    if (!NT_SUCCESS(Status)) {
        ERR("utf8_to_utf16 2 returned %08lx\n", Status);
        ExFreePool(fn.Buffer);
        ExFreePool(err);
        goto end;
    }

    ExAcquireResourceExclusiveLite(&Vcb->scrub.stats_lock, true);

    Vcb->scrub.num_errors++;
    InsertTailList(&Vcb->scrub.errors, &err->list_entry);

    ExReleaseResourceLite(&Vcb->scrub.stats_lock);

    ExFreePool(fn.Buffer);

end:
    while (!IsListEmpty(&parts)) {
        path_part* pp = CONTAINING_RECORD(RemoveHeadList(&parts), path_part, list_entry);

        ExFreePool(pp);
    }
}

Referenced by log_file_checksum_error_shared(), and log_unrecoverable_error().

log_file_checksum_error_shared()

static void log_file_checksum_error_shared ( device_extension *  Vcb, uint64_t  treeaddr, uint64_t  addr, uint64_t  devid, uint64_t  extent  )

static

Definition at line 330 of file scrub.c.

                                                                                                                                     {
    tree_header* tree;
    NTSTATUS Status;
    leaf_node* ln;
    ULONG i;

    tree = ExAllocatePoolWithTag(PagedPool, Vcb->superblock.node_size, ALLOC_TAG);
    if (!tree) {
        ERR("out of memory\n");
        return;
    }

    Status = read_data(Vcb, treeaddr, Vcb->superblock.node_size, NULL, true, (uint8_t*)tree, NULL, NULL, NULL, 0, false, NormalPagePriority);
    if (!NT_SUCCESS(Status)) {
        ERR("read_data returned %08lx\n", Status);
        goto end;
    }

    if (tree->level != 0) {
        ERR("tree level was %x, expected 0\n", tree->level);
        goto end;
    }

    ln = (leaf_node*)&tree[1];

    for (i = 0; i < tree->num_items; i++) {
        if (ln[i].key.obj_type == TYPE_EXTENT_DATA && ln[i].size >= sizeof(EXTENT_DATA) - 1 + sizeof(EXTENT_DATA2)) {
            EXTENT_DATA* ed = (EXTENT_DATA*)((uint8_t*)tree + sizeof(tree_header) + ln[i].offset);
            EXTENT_DATA2* ed2 = (EXTENT_DATA2*)ed->data;

            if (ed->type == EXTENT_TYPE_REGULAR && ed2->size != 0 && ed2->address == addr)
                log_file_checksum_error(Vcb, addr, devid, tree->tree_id, ln[i].key.obj_id, ln[i].key.offset + addr - extent);
        }
    }

end:
    ExFreePool(tree);
}

Referenced by log_unrecoverable_error().

log_tree_checksum_error()

static void log_tree_checksum_error ( device_extension *  Vcb, uint64_t  addr, uint64_t  devid, uint64_t  root, uint8_t  level, KEY *  firstitem  )

static

Definition at line 369 of file scrub.c.

                                                                                                                                        {
    scrub_error* err;

    err = ExAllocatePoolWithTag(PagedPool, sizeof(scrub_error), ALLOC_TAG);
    if (!err) {
        ERR("out of memory\n");
        return;
    }

    err->address = addr;
    err->device = devid;
    err->recovered = false;
    err->is_metadata = true;
    err->parity = false;

    err->metadata.root = root;
    err->metadata.level = level;

    if (firstitem) {
        ERR("root %I64x, level %u, first item (%I64x,%x,%I64x)\n", root, level, firstitem->obj_id,
                                                                firstitem->obj_type, firstitem->offset);

        err->metadata.firstitem = *firstitem;
    } else {
        ERR("root %I64x, level %u\n", root, level);

        RtlZeroMemory(&err->metadata.firstitem, sizeof(KEY));
    }

    ExAcquireResourceExclusiveLite(&Vcb->scrub.stats_lock, true);

    Vcb->scrub.num_errors++;
    InsertTailList(&Vcb->scrub.errors, &err->list_entry);

    ExReleaseResourceLite(&Vcb->scrub.stats_lock);
}

Referenced by log_tree_checksum_error_shared(), and log_unrecoverable_error().

log_tree_checksum_error_shared()

static void log_tree_checksum_error_shared ( device_extension *  Vcb, uint64_t  offset, uint64_t  address, uint64_t  devid  )

static

Definition at line 406 of file scrub.c.

                                                                                                                     {
    tree_header* tree;
    NTSTATUS Status;
    internal_node* in;
    ULONG i;

    tree = ExAllocatePoolWithTag(PagedPool, Vcb->superblock.node_size, ALLOC_TAG);
    if (!tree) {
        ERR("out of memory\n");
        return;
    }

    Status = read_data(Vcb, offset, Vcb->superblock.node_size, NULL, true, (uint8_t*)tree, NULL, NULL, NULL, 0, false, NormalPagePriority);
    if (!NT_SUCCESS(Status)) {
        ERR("read_data returned %08lx\n", Status);
        goto end;
    }

    if (tree->level == 0) {
        ERR("tree level was 0\n");
        goto end;
    }

    in = (internal_node*)&tree[1];

    for (i = 0; i < tree->num_items; i++) {
        if (in[i].address == address) {
            log_tree_checksum_error(Vcb, address, devid, tree->tree_id, tree->level - 1, &in[i].key);
            break;
        }
    }

end:
    ExFreePool(tree);
}

Referenced by log_unrecoverable_error().

log_unrecoverable_error()

static void log_unrecoverable_error ( device_extension *  Vcb, uint64_t  address, uint64_t  devid  )

static

Definition at line 442 of file scrub.c.

                                                                                             {
    KEY searchkey;
    traverse_ptr tp;
    NTSTATUS Status;
    EXTENT_ITEM* ei;
    EXTENT_ITEM2* ei2 = NULL;
    uint8_t* ptr;
    ULONG len;
    uint64_t rc;

    // FIXME - still log even if rest of this function fails

    searchkey.obj_id = address;
    searchkey.obj_type = TYPE_METADATA_ITEM;
    searchkey.offset = 0xffffffffffffffff;

    Status = find_item(Vcb, Vcb->extent_root, &tp, &searchkey, false, NULL);
    if (!NT_SUCCESS(Status)) {
        ERR("find_item returned %08lx\n", Status);
        return;
    }

    if ((tp.item->key.obj_type != TYPE_EXTENT_ITEM && tp.item->key.obj_type != TYPE_METADATA_ITEM) ||
        tp.item->key.obj_id >= address + Vcb->superblock.sector_size ||
        (tp.item->key.obj_type == TYPE_EXTENT_ITEM && tp.item->key.obj_id + tp.item->key.offset <= address) ||
        (tp.item->key.obj_type == TYPE_METADATA_ITEM && tp.item->key.obj_id + Vcb->superblock.node_size <= address)
    )
        return;

    if (tp.item->size < sizeof(EXTENT_ITEM)) {
        ERR("(%I64x,%x,%I64x) was %u bytes, expected at least %Iu\n", tp.item->key.obj_id, tp.item->key.obj_type, tp.item->key.offset, tp.item->size, sizeof(EXTENT_ITEM));
        return;
    }

    ei = (EXTENT_ITEM*)tp.item->data;
    ptr = (uint8_t*)&ei[1];
    len = tp.item->size - sizeof(EXTENT_ITEM);

    if (tp.item->key.obj_id == TYPE_EXTENT_ITEM && ei->flags & EXTENT_ITEM_TREE_BLOCK) {
        if (tp.item->size < sizeof(EXTENT_ITEM) + sizeof(EXTENT_ITEM2)) {
            ERR("(%I64x,%x,%I64x) was %u bytes, expected at least %Iu\n", tp.item->key.obj_id, tp.item->key.obj_type, tp.item->key.offset,
                                                                          tp.item->size, sizeof(EXTENT_ITEM) + sizeof(EXTENT_ITEM2));
            return;
        }

        ei2 = (EXTENT_ITEM2*)ptr;

        ptr += sizeof(EXTENT_ITEM2);
        len -= sizeof(EXTENT_ITEM2);
    }

    rc = 0;

    while (len > 0) {
        uint8_t type = *ptr;

        ptr++;
        len--;

        if (type == TYPE_TREE_BLOCK_REF) {
            TREE_BLOCK_REF* tbr;

            if (len < sizeof(TREE_BLOCK_REF)) {
                ERR("TREE_BLOCK_REF takes up %Iu bytes, but only %lu remaining\n", sizeof(TREE_BLOCK_REF), len);
                break;
            }

            tbr = (TREE_BLOCK_REF*)ptr;

            log_tree_checksum_error(Vcb, address, devid, tbr->offset, ei2 ? ei2->level : (uint8_t)tp.item->key.offset, ei2 ? &ei2->firstitem : NULL);

            rc++;

            ptr += sizeof(TREE_BLOCK_REF);
            len -= sizeof(TREE_BLOCK_REF);
        } else if (type == TYPE_EXTENT_DATA_REF) {
            EXTENT_DATA_REF* edr;

            if (len < sizeof(EXTENT_DATA_REF)) {
                ERR("EXTENT_DATA_REF takes up %Iu bytes, but only %lu remaining\n", sizeof(EXTENT_DATA_REF), len);
                break;
            }

            edr = (EXTENT_DATA_REF*)ptr;

            log_file_checksum_error(Vcb, address, devid, edr->root, edr->objid, edr->offset + address - tp.item->key.obj_id);

            rc += edr->count;

            ptr += sizeof(EXTENT_DATA_REF);
            len -= sizeof(EXTENT_DATA_REF);
        } else if (type == TYPE_SHARED_BLOCK_REF) {
            SHARED_BLOCK_REF* sbr;

            if (len < sizeof(SHARED_BLOCK_REF)) {
                ERR("SHARED_BLOCK_REF takes up %Iu bytes, but only %lu remaining\n", sizeof(SHARED_BLOCK_REF), len);
                break;
            }

            sbr = (SHARED_BLOCK_REF*)ptr;

            log_tree_checksum_error_shared(Vcb, sbr->offset, address, devid);

            rc++;

            ptr += sizeof(SHARED_BLOCK_REF);
            len -= sizeof(SHARED_BLOCK_REF);
        } else if (type == TYPE_SHARED_DATA_REF) {
            SHARED_DATA_REF* sdr;

            if (len < sizeof(SHARED_DATA_REF)) {
                ERR("SHARED_DATA_REF takes up %Iu bytes, but only %lu remaining\n", sizeof(SHARED_DATA_REF), len);
                break;
            }

            sdr = (SHARED_DATA_REF*)ptr;

            log_file_checksum_error_shared(Vcb, sdr->offset, address, devid, tp.item->key.obj_id);

            rc += sdr->count;

            ptr += sizeof(SHARED_DATA_REF);
            len -= sizeof(SHARED_DATA_REF);
        } else {
            ERR("unknown extent type %x\n", type);
            break;
        }
    }

    if (rc < ei->refcount) {
        do {
            traverse_ptr next_tp;

            if (find_next_item(Vcb, &tp, &next_tp, false, NULL))
                tp = next_tp;
            else
                break;

            if (tp.item->key.obj_id == address) {
                if (tp.item->key.obj_type == TYPE_TREE_BLOCK_REF)
                    log_tree_checksum_error(Vcb, address, devid, tp.item->key.offset, ei2 ? ei2->level : (uint8_t)tp.item->key.offset, ei2 ? &ei2->firstitem : NULL);
                else if (tp.item->key.obj_type == TYPE_EXTENT_DATA_REF) {
                    EXTENT_DATA_REF* edr;

                    if (tp.item->size < sizeof(EXTENT_DATA_REF)) {
                        ERR("(%I64x,%x,%I64x) was %u bytes, expected %Iu\n", tp.item->key.obj_id, tp.item->key.obj_type, tp.item->key.offset,
                                                                             tp.item->size, sizeof(EXTENT_DATA_REF));
                        break;
                    }

                    edr = (EXTENT_DATA_REF*)tp.item->data;

                    log_file_checksum_error(Vcb, address, devid, edr->root, edr->objid, edr->offset + address - tp.item->key.obj_id);
                } else if (tp.item->key.obj_type == TYPE_SHARED_BLOCK_REF)
                    log_tree_checksum_error_shared(Vcb, tp.item->key.offset, address, devid);
                else if (tp.item->key.obj_type == TYPE_SHARED_DATA_REF)
                    log_file_checksum_error_shared(Vcb, tp.item->key.offset, address, devid, tp.item->key.obj_id);
            } else
                break;
        } while (true);
    }
}

Referenced by log_error().

pause_scrub()

NTSTATUS pause_scrub	(	device_extension *	Vcb,
		KPROCESSOR_MODE	processor_mode
	)

Definition at line 3402 of file scrub.c.

                                                                            {
    LARGE_INTEGER time;

    if (!SeSinglePrivilegeCheck(RtlConvertLongToLuid(SE_MANAGE_VOLUME_PRIVILEGE), processor_mode))
        return STATUS_PRIVILEGE_NOT_HELD;

    if (!Vcb->scrub.thread)
        return STATUS_DEVICE_NOT_READY;

    if (Vcb->scrub.paused)
        return STATUS_DEVICE_NOT_READY;

    Vcb->scrub.paused = true;
    KeClearEvent(&Vcb->scrub.event);

    KeQuerySystemTime(&time);
    Vcb->scrub.duration.QuadPart += time.QuadPart - Vcb->scrub.resume_time.QuadPart;

    return STATUS_SUCCESS;
}

Referenced by fsctl_request().

query_scrub()

NTSTATUS query_scrub	(	device_extension *	Vcb,
		KPROCESSOR_MODE	processor_mode,
		void *	data,
		ULONG	length
	)

Definition at line 3302 of file scrub.c.

                                                                                                      {
    btrfs_query_scrub* bqs = (btrfs_query_scrub*)data;
    ULONG len;
    NTSTATUS Status;
    LIST_ENTRY* le;
    btrfs_scrub_error* bse = NULL;

    if (!SeSinglePrivilegeCheck(RtlConvertLongToLuid(SE_MANAGE_VOLUME_PRIVILEGE), processor_mode))
        return STATUS_PRIVILEGE_NOT_HELD;

    if (length < offsetof(btrfs_query_scrub, errors))
        return STATUS_BUFFER_TOO_SMALL;

    ExAcquireResourceSharedLite(&Vcb->scrub.stats_lock, true);

    if (Vcb->scrub.thread && Vcb->scrub.chunks_left > 0)
        bqs->status = Vcb->scrub.paused ? BTRFS_SCRUB_PAUSED : BTRFS_SCRUB_RUNNING;
    else
        bqs->status = BTRFS_SCRUB_STOPPED;

    bqs->start_time.QuadPart = Vcb->scrub.start_time.QuadPart;
    bqs->finish_time.QuadPart = Vcb->scrub.finish_time.QuadPart;
    bqs->chunks_left = Vcb->scrub.chunks_left;
    bqs->total_chunks = Vcb->scrub.total_chunks;
    bqs->data_scrubbed = Vcb->scrub.data_scrubbed;

    bqs->duration = Vcb->scrub.duration.QuadPart;

    if (bqs->status == BTRFS_SCRUB_RUNNING) {
        LARGE_INTEGER time;

        KeQuerySystemTime(&time);
        bqs->duration += time.QuadPart - Vcb->scrub.resume_time.QuadPart;
    }

    bqs->error = Vcb->scrub.error;

    bqs->num_errors = Vcb->scrub.num_errors;

    len = length - offsetof(btrfs_query_scrub, errors);

    le = Vcb->scrub.errors.Flink;
    while (le != &Vcb->scrub.errors) {
        scrub_error* err = CONTAINING_RECORD(le, scrub_error, list_entry);
        ULONG errlen;

        if (err->is_metadata)
            errlen = offsetof(btrfs_scrub_error, metadata.firstitem) + sizeof(KEY);
        else
            errlen = offsetof(btrfs_scrub_error, data.filename) + err->data.filename_length;

        if (len < errlen) {
            Status = STATUS_BUFFER_OVERFLOW;
            goto end;
        }

        if (!bse)
            bse = &bqs->errors;
        else {
            ULONG lastlen;

            if (bse->is_metadata)
                lastlen = offsetof(btrfs_scrub_error, metadata.firstitem) + sizeof(KEY);
            else
                lastlen = offsetof(btrfs_scrub_error, data.filename) + bse->data.filename_length;

            bse->next_entry = lastlen;
            bse = (btrfs_scrub_error*)(((uint8_t*)bse) + lastlen);
        }

        bse->next_entry = 0;
        bse->address = err->address;
        bse->device = err->device;
        bse->recovered = err->recovered;
        bse->is_metadata = err->is_metadata;
        bse->parity = err->parity;

        if (err->is_metadata) {
            bse->metadata.root = err->metadata.root;
            bse->metadata.level = err->metadata.level;
            bse->metadata.firstitem = err->metadata.firstitem;
        } else {
            bse->data.subvol = err->data.subvol;
            bse->data.offset = err->data.offset;
            bse->data.filename_length = err->data.filename_length;
            RtlCopyMemory(bse->data.filename, err->data.filename, err->data.filename_length);
        }

        len -= errlen;
        le = le->Flink;
    }

    Status = STATUS_SUCCESS;

end:
    ExReleaseResourceLite(&Vcb->scrub.stats_lock);

    return Status;
}

Referenced by fsctl_request().

resume_scrub()

NTSTATUS resume_scrub	(	device_extension *	Vcb,
		KPROCESSOR_MODE	processor_mode
	)

Definition at line 3423 of file scrub.c.

                                                                             {
    if (!SeSinglePrivilegeCheck(RtlConvertLongToLuid(SE_MANAGE_VOLUME_PRIVILEGE), processor_mode))
        return STATUS_PRIVILEGE_NOT_HELD;

    if (!Vcb->scrub.thread)
        return STATUS_DEVICE_NOT_READY;

    if (!Vcb->scrub.paused)
        return STATUS_DEVICE_NOT_READY;

    Vcb->scrub.paused = false;
    KeSetEvent(&Vcb->scrub.event, 0, false);

    KeQuerySystemTime(&Vcb->scrub.resume_time);

    return STATUS_SUCCESS;
}

Referenced by fsctl_request().

scrub_chunk()

static NTSTATUS scrub_chunk ( device_extension *  Vcb, chunk *  c, uint64_t *  offset, bool *  changed  )

static

Definition at line 2924 of file scrub.c.

                                                                                              {
    NTSTATUS Status;
    KEY searchkey;
    traverse_ptr tp;
    bool b = false, tree_run = false;
    ULONG type, num_extents = 0;
    uint64_t total_data = 0, tree_run_start, tree_run_end;

    TRACE("chunk %I64x\n", c->offset);

    ExAcquireResourceSharedLite(&Vcb->tree_lock, true);

    if (c->chunk_item->type & BLOCK_FLAG_DUPLICATE)
        type = BLOCK_FLAG_DUPLICATE;
    else if (c->chunk_item->type & BLOCK_FLAG_RAID0)
        type = BLOCK_FLAG_RAID0;
    else if (c->chunk_item->type & BLOCK_FLAG_RAID1)
        type = BLOCK_FLAG_DUPLICATE;
    else if (c->chunk_item->type & BLOCK_FLAG_RAID10)
        type = BLOCK_FLAG_RAID10;
    else if (c->chunk_item->type & BLOCK_FLAG_RAID5) {
        Status = scrub_chunk_raid56(Vcb, c, offset, changed);
        goto end;
    } else if (c->chunk_item->type & BLOCK_FLAG_RAID6) {
        Status = scrub_chunk_raid56(Vcb, c, offset, changed);
        goto end;
    } else if (c->chunk_item->type & BLOCK_FLAG_RAID1C3)
        type = BLOCK_FLAG_DUPLICATE;
    else if (c->chunk_item->type & BLOCK_FLAG_RAID1C4)
        type = BLOCK_FLAG_DUPLICATE;
    else // SINGLE
        type = BLOCK_FLAG_DUPLICATE;

    searchkey.obj_id = *offset;
    searchkey.obj_type = TYPE_METADATA_ITEM;
    searchkey.offset = 0xffffffffffffffff;

    Status = find_item(Vcb, Vcb->extent_root, &tp, &searchkey, false, NULL);
    if (!NT_SUCCESS(Status)) {
        ERR("error - find_item returned %08lx\n", Status);
        goto end;
    }

    do {
        traverse_ptr next_tp;

        if (tp.item->key.obj_id >= c->offset + c->chunk_item->size)
            break;

        if (tp.item->key.obj_id >= *offset && (tp.item->key.obj_type == TYPE_EXTENT_ITEM || tp.item->key.obj_type == TYPE_METADATA_ITEM)) {
            uint64_t size = tp.item->key.obj_type == TYPE_METADATA_ITEM ? Vcb->superblock.node_size : tp.item->key.offset;
            bool is_tree;
            void* csum = NULL;
            RTL_BITMAP bmp;
            ULONG* bmparr = NULL, bmplen;

            TRACE("%I64x\n", tp.item->key.obj_id);

            is_tree = false;

            if (tp.item->key.obj_type == TYPE_METADATA_ITEM)
                is_tree = true;
            else {
                EXTENT_ITEM* ei = (EXTENT_ITEM*)tp.item->data;

                if (tp.item->size < sizeof(EXTENT_ITEM)) {
                    ERR("(%I64x,%x,%I64x) was %u bytes, expected at least %Iu\n", tp.item->key.obj_id, tp.item->key.obj_type, tp.item->key.offset, tp.item->size, sizeof(EXTENT_ITEM));
                    Status = STATUS_INTERNAL_ERROR;
                    goto end;
                }

                if (ei->flags & EXTENT_ITEM_TREE_BLOCK)
                    is_tree = true;
            }

            if (size < Vcb->superblock.sector_size) {
                ERR("extent %I64x has size less than sector_size (%I64x < %x)\n", tp.item->key.obj_id, size, Vcb->superblock.sector_size);
                Status = STATUS_INTERNAL_ERROR;
                goto end;
            }

            // load csum
            if (!is_tree) {
                traverse_ptr tp2;

                csum = ExAllocatePoolWithTag(PagedPool, (ULONG)(Vcb->csum_size * size / Vcb->superblock.sector_size), ALLOC_TAG);
                if (!csum) {
                    ERR("out of memory\n");
                    Status = STATUS_INSUFFICIENT_RESOURCES;
                    goto end;
                }

                bmplen = (ULONG)(size / Vcb->superblock.sector_size);

                bmparr = ExAllocatePoolWithTag(PagedPool, (ULONG)(sector_align((bmplen >> 3) + 1, sizeof(ULONG))), ALLOC_TAG);
                if (!bmparr) {
                    ERR("out of memory\n");
                    ExFreePool(csum);
                    Status = STATUS_INSUFFICIENT_RESOURCES;
                    goto end;
                }

                RtlInitializeBitMap(&bmp, bmparr, bmplen);
                RtlSetAllBits(&bmp); // 1 = no csum, 0 = csum

                searchkey.obj_id = EXTENT_CSUM_ID;
                searchkey.obj_type = TYPE_EXTENT_CSUM;
                searchkey.offset = tp.item->key.obj_id;

                Status = find_item(Vcb, Vcb->checksum_root, &tp2, &searchkey, false, NULL);
                if (!NT_SUCCESS(Status) && Status != STATUS_NOT_FOUND) {
                    ERR("find_item returned %08lx\n", Status);
                    ExFreePool(csum);
                    ExFreePool(bmparr);
                    goto end;
                }

                if (Status != STATUS_NOT_FOUND) {
                    do {
                        traverse_ptr next_tp2;

                        if (tp2.item->key.obj_type == TYPE_EXTENT_CSUM) {
                            if (tp2.item->key.offset >= tp.item->key.obj_id + size)
                                break;
                            else if (tp2.item->size >= Vcb->csum_size && tp2.item->key.offset + (tp2.item->size * Vcb->superblock.sector_size / Vcb->csum_size) >= tp.item->key.obj_id) {
                                uint64_t cs = max(tp.item->key.obj_id, tp2.item->key.offset);
                                uint64_t ce = min(tp.item->key.obj_id + size, tp2.item->key.offset + (tp2.item->size * Vcb->superblock.sector_size / Vcb->csum_size));

                                RtlCopyMemory((uint8_t*)csum + ((cs - tp.item->key.obj_id) * Vcb->csum_size / Vcb->superblock.sector_size),
                                              tp2.item->data + ((cs - tp2.item->key.offset) * Vcb->csum_size / Vcb->superblock.sector_size),
                                              (ULONG)((ce - cs) * Vcb->csum_size / Vcb->superblock.sector_size));

                                RtlClearBits(&bmp, (ULONG)((cs - tp.item->key.obj_id) / Vcb->superblock.sector_size), (ULONG)((ce - cs) / Vcb->superblock.sector_size));

                                if (ce == tp.item->key.obj_id + size)
                                    break;
                            }
                        }

                        if (find_next_item(Vcb, &tp2, &next_tp2, false, NULL))
                            tp2 = next_tp2;
                        else
                            break;
                    } while (true);
                }
            }

            if (tree_run) {
                if (!is_tree || tp.item->key.obj_id > tree_run_end) {
                    Status = scrub_extent(Vcb, c, type, tree_run_start, (uint32_t)(tree_run_end - tree_run_start), NULL);
                    if (!NT_SUCCESS(Status)) {
                        ERR("scrub_extent returned %08lx\n", Status);
                        goto end;
                    }

                    if (!is_tree)
                        tree_run = false;
                    else {
                        tree_run_start = tp.item->key.obj_id;
                        tree_run_end = tp.item->key.obj_id + Vcb->superblock.node_size;
                    }
                } else
                    tree_run_end = tp.item->key.obj_id + Vcb->superblock.node_size;
            } else if (is_tree) {
                tree_run = true;
                tree_run_start = tp.item->key.obj_id;
                tree_run_end = tp.item->key.obj_id + Vcb->superblock.node_size;
            }

            if (!is_tree) {
                Status = scrub_data_extent(Vcb, c, tp.item->key.obj_id, type, csum, &bmp, bmplen);
                if (!NT_SUCCESS(Status)) {
                    ERR("scrub_data_extent returned %08lx\n", Status);
                    ExFreePool(csum);
                    ExFreePool(bmparr);
                    goto end;
                }

                ExFreePool(csum);
                ExFreePool(bmparr);
            }

            *offset = tp.item->key.obj_id + size;
            *changed = true;

            total_data += size;
            num_extents++;

            // only do so much at a time
            if (num_extents >= 64 || total_data >= 0x8000000) // 128 MB
                break;
        }

        b = find_next_item(Vcb, &tp, &next_tp, false, NULL);

        if (b)
            tp = next_tp;
    } while (b);

    if (tree_run) {
        Status = scrub_extent(Vcb, c, type, tree_run_start, (uint32_t)(tree_run_end - tree_run_start), NULL);
        if (!NT_SUCCESS(Status)) {
            ERR("scrub_extent returned %08lx\n", Status);
            goto end;
        }
    }

    Status = STATUS_SUCCESS;

end:
    ExReleaseResourceLite(&Vcb->tree_lock);

    return Status;
}

Referenced by _Function_class_().

scrub_chunk_raid56()

static NTSTATUS scrub_chunk_raid56 ( device_extension *  Vcb, chunk *  c, uint64_t *  offset, bool *  changed  )

static

Definition at line 2837 of file scrub.c.

                                                                                                     {
    NTSTATUS Status;
    KEY searchkey;
    traverse_ptr tp;
    bool b;
    uint64_t full_stripe_len, stripe, stripe_start, stripe_end, total_data = 0;
    ULONG num_extents = 0, num_parity_stripes = c->chunk_item->type & BLOCK_FLAG_RAID6 ? 2 : 1;

    full_stripe_len = (c->chunk_item->num_stripes - num_parity_stripes) * c->chunk_item->stripe_length;
    stripe = (*offset - c->offset) / full_stripe_len;

    *offset = c->offset + (stripe * full_stripe_len);

    searchkey.obj_id = *offset;
    searchkey.obj_type = TYPE_METADATA_ITEM;
    searchkey.offset = 0xffffffffffffffff;

    Status = find_item(Vcb, Vcb->extent_root, &tp, &searchkey, false, NULL);
    if (!NT_SUCCESS(Status)) {
        ERR("find_item returned %08lx\n", Status);
        return Status;
    }

    *changed = false;

    do {
        traverse_ptr next_tp;

        if (tp.item->key.obj_id >= c->offset + c->chunk_item->size)
            break;

        if (tp.item->key.obj_id >= *offset && (tp.item->key.obj_type == TYPE_EXTENT_ITEM || tp.item->key.obj_type == TYPE_METADATA_ITEM)) {
            uint64_t size = tp.item->key.obj_type == TYPE_METADATA_ITEM ? Vcb->superblock.node_size : tp.item->key.offset;

            TRACE("%I64x\n", tp.item->key.obj_id);

            if (size < Vcb->superblock.sector_size) {
                ERR("extent %I64x has size less than sector_size (%I64x < %x)\n", tp.item->key.obj_id, size, Vcb->superblock.sector_size);
                return STATUS_INTERNAL_ERROR;
            }

            stripe = (tp.item->key.obj_id - c->offset) / full_stripe_len;

            if (*changed) {
                if (stripe > stripe_end + 1) {
                    Status = scrub_chunk_raid56_stripe_run(Vcb, c, stripe_start, stripe_end);
                    if (!NT_SUCCESS(Status)) {
                        ERR("scrub_chunk_raid56_stripe_run returned %08lx\n", Status);
                        return Status;
                    }

                    stripe_start = stripe;
                }
            } else
                stripe_start = stripe;

            stripe_end = (tp.item->key.obj_id + size - 1 - c->offset) / full_stripe_len;

            *changed = true;

            total_data += size;
            num_extents++;

            // only do so much at a time
            if (num_extents >= 64 || total_data >= 0x8000000) // 128 MB
                break;
        }

        b = find_next_item(Vcb, &tp, &next_tp, false, NULL);

        if (b)
            tp = next_tp;
    } while (b);

    if (*changed) {
        Status = scrub_chunk_raid56_stripe_run(Vcb, c, stripe_start, stripe_end);
        if (!NT_SUCCESS(Status)) {
            ERR("scrub_chunk_raid56_stripe_run returned %08lx\n", Status);
            return Status;
        }

        *offset = c->offset + ((stripe_end + 1) * full_stripe_len);
    }

    return STATUS_SUCCESS;
}

Referenced by scrub_chunk().

scrub_chunk_raid56_stripe_run()

static NTSTATUS scrub_chunk_raid56_stripe_run ( device_extension *  Vcb, chunk *  c, uint64_t  stripe_start, uint64_t  stripe_end  )

static

Definition at line 2426 of file scrub.c.

                                                                                                                           {
    NTSTATUS Status;
    KEY searchkey;
    traverse_ptr tp;
    bool b;
    uint64_t run_start, run_end, full_stripe_len, stripe;
    uint32_t max_read, num_sectors;
    ULONG arrlen, *allocarr, *csumarr = NULL, *treearr, num_parity_stripes = c->chunk_item->type & BLOCK_FLAG_RAID6 ? 2 : 1;
    scrub_context_raid56 context;
    uint16_t i;
    CHUNK_ITEM_STRIPE* cis = (CHUNK_ITEM_STRIPE*)&c->chunk_item[1];

    TRACE("(%p, %p, %I64x, %I64x)\n", Vcb, c, stripe_start, stripe_end);

    full_stripe_len = (c->chunk_item->num_stripes - num_parity_stripes) * c->chunk_item->stripe_length;
    run_start = c->offset + (stripe_start * full_stripe_len);
    run_end = c->offset + ((stripe_end + 1) * full_stripe_len);

    searchkey.obj_id = run_start;
    searchkey.obj_type = TYPE_METADATA_ITEM;
    searchkey.offset = 0xffffffffffffffff;

    Status = find_item(Vcb, Vcb->extent_root, &tp, &searchkey, false, NULL);
    if (!NT_SUCCESS(Status)) {
        ERR("find_item returned %08lx\n", Status);
        return Status;
    }

    num_sectors = (uint32_t)((stripe_end - stripe_start + 1) * full_stripe_len / Vcb->superblock.sector_size);
    arrlen = (ULONG)sector_align((num_sectors / 8) + 1, sizeof(ULONG));

    allocarr = ExAllocatePoolWithTag(PagedPool, arrlen, ALLOC_TAG);
    if (!allocarr) {
        ERR("out of memory\n");
        return STATUS_INSUFFICIENT_RESOURCES;
    }

    treearr = ExAllocatePoolWithTag(PagedPool, arrlen, ALLOC_TAG);
    if (!treearr) {
        ERR("out of memory\n");
        ExFreePool(allocarr);
        return STATUS_INSUFFICIENT_RESOURCES;
    }

    RtlInitializeBitMap(&context.alloc, allocarr, num_sectors);
    RtlClearAllBits(&context.alloc);

    RtlInitializeBitMap(&context.is_tree, treearr, num_sectors);
    RtlClearAllBits(&context.is_tree);

    context.parity_scratch = ExAllocatePoolWithTag(PagedPool, (ULONG)c->chunk_item->stripe_length, ALLOC_TAG);
    if (!context.parity_scratch) {
        ERR("out of memory\n");
        ExFreePool(allocarr);
        ExFreePool(treearr);
        return STATUS_INSUFFICIENT_RESOURCES;
    }

    if (c->chunk_item->type & BLOCK_FLAG_DATA) {
        csumarr = ExAllocatePoolWithTag(PagedPool, arrlen, ALLOC_TAG);
        if (!csumarr) {
            ERR("out of memory\n");
            ExFreePool(allocarr);
            ExFreePool(treearr);
            ExFreePool(context.parity_scratch);
            return STATUS_INSUFFICIENT_RESOURCES;
        }

        RtlInitializeBitMap(&context.has_csum, csumarr, num_sectors);
        RtlClearAllBits(&context.has_csum);

        context.csum = ExAllocatePoolWithTag(PagedPool, num_sectors * Vcb->csum_size, ALLOC_TAG);
        if (!context.csum) {
            ERR("out of memory\n");
            ExFreePool(allocarr);
            ExFreePool(treearr);
            ExFreePool(context.parity_scratch);
            ExFreePool(csumarr);
            return STATUS_INSUFFICIENT_RESOURCES;
        }
    }

    if (c->chunk_item->type & BLOCK_FLAG_RAID6) {
        context.parity_scratch2 = ExAllocatePoolWithTag(PagedPool, (ULONG)c->chunk_item->stripe_length, ALLOC_TAG);
        if (!context.parity_scratch2) {
            ERR("out of memory\n");
            ExFreePool(allocarr);
            ExFreePool(treearr);
            ExFreePool(context.parity_scratch);

            if (c->chunk_item->type & BLOCK_FLAG_DATA) {
                ExFreePool(csumarr);
                ExFreePool(context.csum);
            }

            return STATUS_INSUFFICIENT_RESOURCES;
        }
    }

    do {
        traverse_ptr next_tp;

        if (tp.item->key.obj_id >= run_end)
            break;

        if (tp.item->key.obj_type == TYPE_EXTENT_ITEM || tp.item->key.obj_type == TYPE_METADATA_ITEM) {
            uint64_t size = tp.item->key.obj_type == TYPE_METADATA_ITEM ? Vcb->superblock.node_size : tp.item->key.offset;

            if (tp.item->key.obj_id + size > run_start) {
                uint64_t extent_start = max(run_start, tp.item->key.obj_id);
                uint64_t extent_end = min(tp.item->key.obj_id + size, run_end);
                bool extent_is_tree = false;

                RtlSetBits(&context.alloc, (ULONG)((extent_start - run_start) / Vcb->superblock.sector_size), (ULONG)((extent_end - extent_start) / Vcb->superblock.sector_size));

                if (tp.item->key.obj_type == TYPE_METADATA_ITEM)
                    extent_is_tree = true;
                else {
                    EXTENT_ITEM* ei = (EXTENT_ITEM*)tp.item->data;

                    if (tp.item->size < sizeof(EXTENT_ITEM)) {
                        ERR("(%I64x,%x,%I64x) was %u bytes, expected at least %Iu\n", tp.item->key.obj_id, tp.item->key.obj_type, tp.item->key.offset, tp.item->size, sizeof(EXTENT_ITEM));
                        Status = STATUS_INTERNAL_ERROR;
                        goto end;
                    }

                    if (ei->flags & EXTENT_ITEM_TREE_BLOCK)
                        extent_is_tree = true;
                }

                if (extent_is_tree)
                    RtlSetBits(&context.is_tree, (ULONG)((extent_start - run_start) / Vcb->superblock.sector_size), (ULONG)((extent_end - extent_start) / Vcb->superblock.sector_size));
                else if (c->chunk_item->type & BLOCK_FLAG_DATA) {
                    traverse_ptr tp2;
                    bool b2;

                    searchkey.obj_id = EXTENT_CSUM_ID;
                    searchkey.obj_type = TYPE_EXTENT_CSUM;
                    searchkey.offset = extent_start;

                    Status = find_item(Vcb, Vcb->checksum_root, &tp2, &searchkey, false, NULL);
                    if (!NT_SUCCESS(Status) && Status != STATUS_NOT_FOUND) {
                        ERR("find_item returned %08lx\n", Status);
                        goto end;
                    }

                    do {
                        traverse_ptr next_tp2;

                        if (tp2.item->key.offset >= extent_end)
                            break;

                        if (tp2.item->key.offset >= extent_start) {
                            uint64_t csum_start = max(extent_start, tp2.item->key.offset);
                            uint64_t csum_end = min(extent_end, tp2.item->key.offset + (tp2.item->size * Vcb->superblock.sector_size / Vcb->csum_size));

                            RtlSetBits(&context.has_csum, (ULONG)((csum_start - run_start) / Vcb->superblock.sector_size), (ULONG)((csum_end - csum_start) / Vcb->superblock.sector_size));

                            RtlCopyMemory((uint8_t*)context.csum + ((csum_start - run_start) * Vcb->csum_size / Vcb->superblock.sector_size),
                                          tp2.item->data + ((csum_start - tp2.item->key.offset) * Vcb->csum_size / Vcb->superblock.sector_size),
                                          (ULONG)((csum_end - csum_start) * Vcb->csum_size / Vcb->superblock.sector_size));
                        }

                        b2 = find_next_item(Vcb, &tp2, &next_tp2, false, NULL);

                        if (b2)
                            tp2 = next_tp2;
                    } while (b2);
                }
            }
        }

        b = find_next_item(Vcb, &tp, &next_tp, false, NULL);

        if (b)
            tp = next_tp;
    } while (b);

    context.stripes = ExAllocatePoolWithTag(PagedPool, sizeof(scrub_context_raid56_stripe) * c->chunk_item->num_stripes, ALLOC_TAG);
    if (!context.stripes) {
        ERR("out of memory\n");
        Status = STATUS_INSUFFICIENT_RESOURCES;
        goto end;
    }

    max_read = (uint32_t)min(1048576 / c->chunk_item->stripe_length, stripe_end - stripe_start + 1); // only process 1 MB of data at a time

    for (i = 0; i < c->chunk_item->num_stripes; i++) {
        context.stripes[i].buf = ExAllocatePoolWithTag(PagedPool, (ULONG)(max_read * c->chunk_item->stripe_length), ALLOC_TAG);
        if (!context.stripes[i].buf) {
            uint64_t j;

            ERR("out of memory\n");

            for (j = 0; j < i; j++) {
                ExFreePool(context.stripes[j].buf);
            }
            ExFreePool(context.stripes);

            Status = STATUS_INSUFFICIENT_RESOURCES;
            goto end;
        }

        context.stripes[i].errorarr = ExAllocatePoolWithTag(PagedPool, (ULONG)sector_align(((c->chunk_item->stripe_length / Vcb->superblock.sector_size) / 8) + 1, sizeof(ULONG)), ALLOC_TAG);
        if (!context.stripes[i].errorarr) {
            uint64_t j;

            ERR("out of memory\n");

            ExFreePool(context.stripes[i].buf);

            for (j = 0; j < i; j++) {
                ExFreePool(context.stripes[j].buf);
            }
            ExFreePool(context.stripes);

            Status = STATUS_INSUFFICIENT_RESOURCES;
            goto end;
        }

        RtlInitializeBitMap(&context.stripes[i].error, context.stripes[i].errorarr, (ULONG)(c->chunk_item->stripe_length / Vcb->superblock.sector_size));

        context.stripes[i].context = &context;
        context.stripes[i].rewrite = false;
    }

    stripe = stripe_start;

    Status = STATUS_SUCCESS;

    chunk_lock_range(Vcb, c, run_start, run_end - run_start);

    do {
        ULONG read_stripes;
        uint16_t missing_devices = 0;
        bool need_wait = false;

        if (max_read < stripe_end + 1 - stripe)
            read_stripes = max_read;
        else
            read_stripes = (ULONG)(stripe_end + 1 - stripe);

        context.stripes_left = c->chunk_item->num_stripes;

        // read megabyte by megabyte
        for (i = 0; i < c->chunk_item->num_stripes; i++) {
            if (c->devices[i]->devobj) {
                PIO_STACK_LOCATION IrpSp;

                context.stripes[i].Irp = IoAllocateIrp(c->devices[i]->devobj->StackSize, false);

                if (!context.stripes[i].Irp) {
                    ERR("IoAllocateIrp failed\n");
                    Status = STATUS_INSUFFICIENT_RESOURCES;
                    goto end3;
                }

                context.stripes[i].Irp->MdlAddress = NULL;

                IrpSp = IoGetNextIrpStackLocation(context.stripes[i].Irp);
                IrpSp->MajorFunction = IRP_MJ_READ;
                IrpSp->FileObject = c->devices[i]->fileobj;

                if (c->devices[i]->devobj->Flags & DO_BUFFERED_IO) {
                    context.stripes[i].Irp->AssociatedIrp.SystemBuffer = ExAllocatePoolWithTag(NonPagedPool, (ULONG)(read_stripes * c->chunk_item->stripe_length), ALLOC_TAG);
                    if (!context.stripes[i].Irp->AssociatedIrp.SystemBuffer) {
                        ERR("out of memory\n");
                        Status = STATUS_INSUFFICIENT_RESOURCES;
                        goto end3;
                    }

                    context.stripes[i].Irp->Flags |= IRP_BUFFERED_IO | IRP_DEALLOCATE_BUFFER | IRP_INPUT_OPERATION;

                    context.stripes[i].Irp->UserBuffer = context.stripes[i].buf;
                } else if (c->devices[i]->devobj->Flags & DO_DIRECT_IO) {
                    context.stripes[i].Irp->MdlAddress = IoAllocateMdl(context.stripes[i].buf, (ULONG)(read_stripes * c->chunk_item->stripe_length), false, false, NULL);
                    if (!context.stripes[i].Irp->MdlAddress) {
                        ERR("IoAllocateMdl failed\n");
                        Status = STATUS_INSUFFICIENT_RESOURCES;
                        goto end3;
                    }

                    Status = STATUS_SUCCESS;

                    _SEH2_TRY {
                        MmProbeAndLockPages(context.stripes[i].Irp->MdlAddress, KernelMode, IoWriteAccess);
                    } _SEH2_EXCEPT (EXCEPTION_EXECUTE_HANDLER) {
                        Status = _SEH2_GetExceptionCode();
                    } _SEH2_END;

                    if (!NT_SUCCESS(Status)) {
                        ERR("MmProbeAndLockPages threw exception %08lx\n", Status);
                        IoFreeMdl(context.stripes[i].Irp->MdlAddress);
                        goto end3;
                    }
                } else
                    context.stripes[i].Irp->UserBuffer = context.stripes[i].buf;

                context.stripes[i].offset = stripe * c->chunk_item->stripe_length;

                IrpSp->Parameters.Read.Length = (ULONG)(read_stripes * c->chunk_item->stripe_length);
                IrpSp->Parameters.Read.ByteOffset.QuadPart = cis[i].offset + context.stripes[i].offset;

                context.stripes[i].Irp->UserIosb = &context.stripes[i].iosb;
                context.stripes[i].missing = false;

                IoSetCompletionRoutine(context.stripes[i].Irp, scrub_read_completion_raid56, &context.stripes[i], true, true, true);

                Vcb->scrub.data_scrubbed += read_stripes * c->chunk_item->stripe_length;
                need_wait = true;
            } else {
                context.stripes[i].Irp = NULL;
                context.stripes[i].missing = true;
                missing_devices++;
                InterlockedDecrement(&context.stripes_left);
            }
        }

        if (c->chunk_item->type & BLOCK_FLAG_RAID5 && missing_devices > 1) {
            ERR("too many missing devices (%u, maximum 1)\n", missing_devices);
            Status = STATUS_UNEXPECTED_IO_ERROR;
            goto end3;
        } else if (c->chunk_item->type & BLOCK_FLAG_RAID6 && missing_devices > 2) {
            ERR("too many missing devices (%u, maximum 2)\n", missing_devices);
            Status = STATUS_UNEXPECTED_IO_ERROR;
            goto end3;
        }

        if (need_wait) {
            KeInitializeEvent(&context.Event, NotificationEvent, false);

            for (i = 0; i < c->chunk_item->num_stripes; i++) {
                if (c->devices[i]->devobj)
                    IoCallDriver(c->devices[i]->devobj, context.stripes[i].Irp);
            }

            KeWaitForSingleObject(&context.Event, Executive, KernelMode, false, NULL);
        }

        // return an error if any of the stripes returned an error
        for (i = 0; i < c->chunk_item->num_stripes; i++) {
            if (!context.stripes[i].missing && !NT_SUCCESS(context.stripes[i].iosb.Status)) {
                Status = context.stripes[i].iosb.Status;
                log_device_error(Vcb, c->devices[i], BTRFS_DEV_STAT_READ_ERRORS);
                goto end3;
            }
        }

        if (c->chunk_item->type & BLOCK_FLAG_RAID6) {
            for (i = 0; i < read_stripes; i++) {
                scrub_raid6_stripe(Vcb, c, &context, stripe_start, stripe, i, missing_devices);
            }
        } else {
            for (i = 0; i < read_stripes; i++) {
                scrub_raid5_stripe(Vcb, c, &context, stripe_start, stripe, i, missing_devices);
            }
        }
        stripe += read_stripes;

end3:
        for (i = 0; i < c->chunk_item->num_stripes; i++) {
            if (context.stripes[i].Irp) {
                if (c->devices[i]->devobj->Flags & DO_DIRECT_IO && context.stripes[i].Irp->MdlAddress) {
                    MmUnlockPages(context.stripes[i].Irp->MdlAddress);
                    IoFreeMdl(context.stripes[i].Irp->MdlAddress);
                }
                IoFreeIrp(context.stripes[i].Irp);
                context.stripes[i].Irp = NULL;

                if (context.stripes[i].rewrite) {
                    Status = write_data_phys(c->devices[i]->devobj, c->devices[i]->fileobj, cis[i].offset + context.stripes[i].offset,
                                             context.stripes[i].buf, (uint32_t)(read_stripes * c->chunk_item->stripe_length));

                    if (!NT_SUCCESS(Status)) {
                        ERR("write_data_phys returned %08lx\n", Status);
                        log_device_error(Vcb, c->devices[i], BTRFS_DEV_STAT_WRITE_ERRORS);
                        goto end2;
                    }
                }
            }
        }

        if (!NT_SUCCESS(Status))
            break;
    } while (stripe < stripe_end);

end2:
    chunk_unlock_range(Vcb, c, run_start, run_end - run_start);

    for (i = 0; i < c->chunk_item->num_stripes; i++) {
        ExFreePool(context.stripes[i].buf);
        ExFreePool(context.stripes[i].errorarr);
    }
    ExFreePool(context.stripes);

end:
    ExFreePool(treearr);
    ExFreePool(allocarr);
    ExFreePool(context.parity_scratch);

    if (c->chunk_item->type & BLOCK_FLAG_RAID6)
        ExFreePool(context.parity_scratch2);

    if (c->chunk_item->type & BLOCK_FLAG_DATA) {
        ExFreePool(csumarr);
        ExFreePool(context.csum);
    }

    return Status;
}

Referenced by scrub_chunk_raid56().

scrub_data_extent()

static NTSTATUS scrub_data_extent ( device_extension *  Vcb, chunk *  c, uint64_t  offset, ULONG  type, void *  csum, RTL_BITMAP *  bmp, ULONG  bmplen  )

static

Definition at line 1627 of file scrub.c.

                                                                                                                                           {
    NTSTATUS Status;
    ULONG runlength, index;

    runlength = RtlFindFirstRunClear(bmp, &index);

    while (runlength != 0) {
        if (index >= bmplen)
            break;

        if (index + runlength >= bmplen) {
            runlength = bmplen - index;

            if (runlength == 0)
                break;
        }

        do {
            ULONG rl;

            if (runlength * Vcb->superblock.sector_size > SCRUB_UNIT)
                rl = SCRUB_UNIT / Vcb->superblock.sector_size;
            else
                rl = runlength;

            Status = scrub_extent(Vcb, c, type, offset + UInt32x32To64(index, Vcb->superblock.sector_size),
                                  rl * Vcb->superblock.sector_size, (uint8_t*)csum + (index * Vcb->csum_size));
            if (!NT_SUCCESS(Status)) {
                ERR("scrub_data_extent_dup returned %08lx\n", Status);
                return Status;
            }

            runlength -= rl;
            index += rl;
        } while (runlength > 0);

        runlength = RtlFindNextForwardRunClear(bmp, index, &index);
    }

    return STATUS_SUCCESS;
}

Referenced by scrub_chunk().

scrub_extent()

static NTSTATUS scrub_extent ( device_extension *  Vcb, chunk *  c, ULONG  type, uint64_t  offset, uint32_t  size, void *  csum  )

static

Definition at line 1377 of file scrub.c.

                                                                                                                      {
    ULONG i;
    scrub_context context;
    CHUNK_ITEM_STRIPE* cis;
    NTSTATUS Status;
    uint16_t startoffstripe, num_missing, allowed_missing;

    TRACE("(%p, %p, %lx, %I64x, %x, %p)\n", Vcb, c, type, offset, size, csum);

    context.stripes = ExAllocatePoolWithTag(NonPagedPool, sizeof(scrub_context_stripe) * c->chunk_item->num_stripes, ALLOC_TAG);
    if (!context.stripes) {
        ERR("out of memory\n");
        Status = STATUS_INSUFFICIENT_RESOURCES;
        goto end;
    }

    RtlZeroMemory(context.stripes, sizeof(scrub_context_stripe) * c->chunk_item->num_stripes);

    context.stripes_left = 0;

    cis = (CHUNK_ITEM_STRIPE*)&c->chunk_item[1];

    if (type == BLOCK_FLAG_RAID0) {
        uint64_t startoff, endoff;
        uint16_t endoffstripe;

        get_raid0_offset(offset - c->offset, c->chunk_item->stripe_length, c->chunk_item->num_stripes, &startoff, &startoffstripe);
        get_raid0_offset(offset + size - c->offset - 1, c->chunk_item->stripe_length, c->chunk_item->num_stripes, &endoff, &endoffstripe);

        for (i = 0; i < c->chunk_item->num_stripes; i++) {
            if (startoffstripe > i)
                context.stripes[i].start = startoff - (startoff % c->chunk_item->stripe_length) + c->chunk_item->stripe_length;
            else if (startoffstripe == i)
                context.stripes[i].start = startoff;
            else
                context.stripes[i].start = startoff - (startoff % c->chunk_item->stripe_length);

            if (endoffstripe > i)
                context.stripes[i].length = (uint32_t)(endoff - (endoff % c->chunk_item->stripe_length) + c->chunk_item->stripe_length - context.stripes[i].start);
            else if (endoffstripe == i)
                context.stripes[i].length = (uint32_t)(endoff + 1 - context.stripes[i].start);
            else
                context.stripes[i].length = (uint32_t)(endoff - (endoff % c->chunk_item->stripe_length) - context.stripes[i].start);
        }

        allowed_missing = 0;
    } else if (type == BLOCK_FLAG_RAID10) {
        uint64_t startoff, endoff;
        uint16_t endoffstripe, j, sub_stripes = max(c->chunk_item->sub_stripes, 1);

        get_raid0_offset(offset - c->offset, c->chunk_item->stripe_length, c->chunk_item->num_stripes / sub_stripes, &startoff, &startoffstripe);
        get_raid0_offset(offset + size - c->offset - 1, c->chunk_item->stripe_length, c->chunk_item->num_stripes / sub_stripes, &endoff, &endoffstripe);

        if ((c->chunk_item->num_stripes % sub_stripes) != 0) {
            ERR("chunk %I64x: num_stripes %x was not a multiple of sub_stripes %x!\n", c->offset, c->chunk_item->num_stripes, sub_stripes);
            Status = STATUS_INTERNAL_ERROR;
            goto end;
        }

        startoffstripe *= sub_stripes;
        endoffstripe *= sub_stripes;

        for (i = 0; i < c->chunk_item->num_stripes; i += sub_stripes) {
            if (startoffstripe > i)
                context.stripes[i].start = startoff - (startoff % c->chunk_item->stripe_length) + c->chunk_item->stripe_length;
            else if (startoffstripe == i)
                context.stripes[i].start = startoff;
            else
                context.stripes[i].start = startoff - (startoff % c->chunk_item->stripe_length);

            if (endoffstripe > i)
                context.stripes[i].length = (uint32_t)(endoff - (endoff % c->chunk_item->stripe_length) + c->chunk_item->stripe_length - context.stripes[i].start);
            else if (endoffstripe == i)
                context.stripes[i].length = (uint32_t)(endoff + 1 - context.stripes[i].start);
            else
                context.stripes[i].length = (uint32_t)(endoff - (endoff % c->chunk_item->stripe_length) - context.stripes[i].start);

            for (j = 1; j < sub_stripes; j++) {
                context.stripes[i+j].start = context.stripes[i].start;
                context.stripes[i+j].length = context.stripes[i].length;
            }
        }

        startoffstripe /= sub_stripes;
        allowed_missing = 1;
    } else
        allowed_missing = c->chunk_item->num_stripes - 1;

    num_missing = 0;

    for (i = 0; i < c->chunk_item->num_stripes; i++) {
        PIO_STACK_LOCATION IrpSp;

        context.stripes[i].context = (struct _scrub_context*)&context;

        if (type == BLOCK_FLAG_DUPLICATE) {
            context.stripes[i].start = offset - c->offset;
            context.stripes[i].length = size;
        } else if (type != BLOCK_FLAG_RAID0 && type != BLOCK_FLAG_RAID10) {
            ERR("unexpected chunk type %lx\n", type);
            Status = STATUS_INTERNAL_ERROR;
            goto end;
        }

        if (!c->devices[i]->devobj) {
            num_missing++;

            if (num_missing > allowed_missing) {
                ERR("too many missing devices (at least %u, maximum allowed %u)\n", num_missing, allowed_missing);
                Status = STATUS_INTERNAL_ERROR;
                goto end;
            }
        } else if (context.stripes[i].length > 0) {
            context.stripes[i].buf = ExAllocatePoolWithTag(NonPagedPool, context.stripes[i].length, ALLOC_TAG);

            if (!context.stripes[i].buf) {
                ERR("out of memory\n");
                Status = STATUS_INSUFFICIENT_RESOURCES;
                goto end;
            }

            context.stripes[i].Irp = IoAllocateIrp(c->devices[i]->devobj->StackSize, false);

            if (!context.stripes[i].Irp) {
                ERR("IoAllocateIrp failed\n");
                Status = STATUS_INSUFFICIENT_RESOURCES;
                goto end;
            }

            IrpSp = IoGetNextIrpStackLocation(context.stripes[i].Irp);
            IrpSp->MajorFunction = IRP_MJ_READ;
            IrpSp->FileObject = c->devices[i]->fileobj;

            if (c->devices[i]->devobj->Flags & DO_BUFFERED_IO) {
                context.stripes[i].Irp->AssociatedIrp.SystemBuffer = ExAllocatePoolWithTag(NonPagedPool, context.stripes[i].length, ALLOC_TAG);
                if (!context.stripes[i].Irp->AssociatedIrp.SystemBuffer) {
                    ERR("out of memory\n");
                    Status = STATUS_INSUFFICIENT_RESOURCES;
                    goto end;
                }

                context.stripes[i].Irp->Flags |= IRP_BUFFERED_IO | IRP_DEALLOCATE_BUFFER | IRP_INPUT_OPERATION;

                context.stripes[i].Irp->UserBuffer = context.stripes[i].buf;
            } else if (c->devices[i]->devobj->Flags & DO_DIRECT_IO) {
                context.stripes[i].Irp->MdlAddress = IoAllocateMdl(context.stripes[i].buf, context.stripes[i].length, false, false, NULL);
                if (!context.stripes[i].Irp->MdlAddress) {
                    ERR("IoAllocateMdl failed\n");
                    Status = STATUS_INSUFFICIENT_RESOURCES;
                    goto end;
                }

                Status = STATUS_SUCCESS;

                _SEH2_TRY {
                    MmProbeAndLockPages(context.stripes[i].Irp->MdlAddress, KernelMode, IoWriteAccess);
                } _SEH2_EXCEPT (EXCEPTION_EXECUTE_HANDLER) {
                    Status = _SEH2_GetExceptionCode();
                } _SEH2_END;

                if (!NT_SUCCESS(Status)) {
                    ERR("MmProbeAndLockPages threw exception %08lx\n", Status);
                    IoFreeMdl(context.stripes[i].Irp->MdlAddress);
                    context.stripes[i].Irp->MdlAddress = NULL;
                    goto end;
                }
            } else
                context.stripes[i].Irp->UserBuffer = context.stripes[i].buf;

            IrpSp->Parameters.Read.Length = context.stripes[i].length;
            IrpSp->Parameters.Read.ByteOffset.QuadPart = context.stripes[i].start + cis[i].offset;

            context.stripes[i].Irp->UserIosb = &context.stripes[i].iosb;

            IoSetCompletionRoutine(context.stripes[i].Irp, scrub_read_completion, &context.stripes[i], true, true, true);

            context.stripes_left++;

            Vcb->scrub.data_scrubbed += context.stripes[i].length;
        }
    }

    if (context.stripes_left == 0) {
        ERR("error - not reading any stripes\n");
        Status = STATUS_INTERNAL_ERROR;
        goto end;
    }

    KeInitializeEvent(&context.Event, NotificationEvent, false);

    for (i = 0; i < c->chunk_item->num_stripes; i++) {
        if (c->devices[i]->devobj && context.stripes[i].length > 0)
            IoCallDriver(c->devices[i]->devobj, context.stripes[i].Irp);
    }

    KeWaitForSingleObject(&context.Event, Executive, KernelMode, false, NULL);

    // return an error if any of the stripes returned an error
    for (i = 0; i < c->chunk_item->num_stripes; i++) {
        if (!NT_SUCCESS(context.stripes[i].iosb.Status)) {
            Status = context.stripes[i].iosb.Status;
            log_device_error(Vcb, c->devices[i], BTRFS_DEV_STAT_READ_ERRORS);
            goto end;
        }
    }

    if (type == BLOCK_FLAG_DUPLICATE) {
        Status = scrub_extent_dup(Vcb, c, offset, csum, &context);
        if (!NT_SUCCESS(Status)) {
            ERR("scrub_extent_dup returned %08lx\n", Status);
            goto end;
        }
    } else if (type == BLOCK_FLAG_RAID0) {
        Status = scrub_extent_raid0(Vcb, c, offset, size, startoffstripe, csum, &context);
        if (!NT_SUCCESS(Status)) {
            ERR("scrub_extent_raid0 returned %08lx\n", Status);
            goto end;
        }
    } else if (type == BLOCK_FLAG_RAID10) {
        Status = scrub_extent_raid10(Vcb, c, offset, size, startoffstripe, csum, &context);
        if (!NT_SUCCESS(Status)) {
            ERR("scrub_extent_raid10 returned %08lx\n", Status);
            goto end;
        }
    }

end:
    if (context.stripes) {
        for (i = 0; i < c->chunk_item->num_stripes; i++) {
            if (context.stripes[i].Irp) {
                if (c->devices[i]->devobj->Flags & DO_DIRECT_IO && context.stripes[i].Irp->MdlAddress) {
                    MmUnlockPages(context.stripes[i].Irp->MdlAddress);
                    IoFreeMdl(context.stripes[i].Irp->MdlAddress);
                }
                IoFreeIrp(context.stripes[i].Irp);
            }

            if (context.stripes[i].buf)
                ExFreePool(context.stripes[i].buf);

            if (context.stripes[i].bad_csums)
                ExFreePool(context.stripes[i].bad_csums);
        }

        ExFreePool(context.stripes);
    }

    return Status;
}

Referenced by scrub_chunk(), and scrub_data_extent().

scrub_extent_dup()

static NTSTATUS scrub_extent_dup ( device_extension *  Vcb, chunk *  c, uint64_t  offset, void *  csum, scrub_context *  context  )

static

Definition at line 667 of file scrub.c.

                                                                                                                       {
    NTSTATUS Status;
    bool csum_error = false;
    ULONG i;
    CHUNK_ITEM_STRIPE* cis = (CHUNK_ITEM_STRIPE*)&c->chunk_item[1];
    uint16_t present_devices = 0;

    if (csum) {
        ULONG good_stripe = 0xffffffff;

        for (i = 0; i < c->chunk_item->num_stripes; i++) {
            if (c->devices[i]->devobj) {
                present_devices++;

                // if first stripe is okay, we only need to check that the others are identical to it
                if (good_stripe != 0xffffffff) {
                    if (RtlCompareMemory(context->stripes[i].buf, context->stripes[good_stripe].buf,
                                        context->stripes[good_stripe].length) != context->stripes[i].length) {
                        context->stripes[i].csum_error = true;
                        csum_error = true;
                        log_device_error(Vcb, c->devices[i], BTRFS_DEV_STAT_CORRUPTION_ERRORS);
                    }
                } else {
                    Status = check_csum(Vcb, context->stripes[i].buf, context->stripes[i].length / Vcb->superblock.sector_size, csum);
                    if (Status == STATUS_CRC_ERROR) {
                        context->stripes[i].csum_error = true;
                        csum_error = true;
                        log_device_error(Vcb, c->devices[i], BTRFS_DEV_STAT_CORRUPTION_ERRORS);
                    } else if (!NT_SUCCESS(Status)) {
                        ERR("check_csum returned %08lx\n", Status);
                        return Status;
                    } else
                        good_stripe = i;
                }
            }
        }
    } else {
        ULONG good_stripe = 0xffffffff;

        for (i = 0; i < c->chunk_item->num_stripes; i++) {
            ULONG j;

            if (c->devices[i]->devobj) {
                // if first stripe is okay, we only need to check that the others are identical to it
                if (good_stripe != 0xffffffff) {
                    if (RtlCompareMemory(context->stripes[i].buf, context->stripes[good_stripe].buf,
                                         context->stripes[good_stripe].length) != context->stripes[i].length) {
                        context->stripes[i].csum_error = true;
                        csum_error = true;
                        log_device_error(Vcb, c->devices[i], BTRFS_DEV_STAT_CORRUPTION_ERRORS);
                    }
                } else {
                    for (j = 0; j < context->stripes[i].length / Vcb->superblock.node_size; j++) {
                        tree_header* th = (tree_header*)&context->stripes[i].buf[j * Vcb->superblock.node_size];

                        if (!check_tree_checksum(Vcb, th) || th->address != offset + UInt32x32To64(j, Vcb->superblock.node_size)) {
                            context->stripes[i].csum_error = true;
                            csum_error = true;
                            log_device_error(Vcb, c->devices[i], BTRFS_DEV_STAT_CORRUPTION_ERRORS);
                        }
                    }

                    if (!context->stripes[i].csum_error)
                        good_stripe = i;
                }
            }
        }
    }

    if (!csum_error)
        return STATUS_SUCCESS;

    // handle checksum error

    for (i = 0; i < c->chunk_item->num_stripes; i++) {
        if (context->stripes[i].csum_error) {
            if (csum) {
                context->stripes[i].bad_csums = ExAllocatePoolWithTag(PagedPool, context->stripes[i].length * Vcb->csum_size / Vcb->superblock.sector_size, ALLOC_TAG);
                if (!context->stripes[i].bad_csums) {
                    ERR("out of memory\n");
                    return STATUS_INSUFFICIENT_RESOURCES;
                }

                do_calc_job(Vcb, context->stripes[i].buf, context->stripes[i].length / Vcb->superblock.sector_size, context->stripes[i].bad_csums);
            } else {
                ULONG j;

                context->stripes[i].bad_csums = ExAllocatePoolWithTag(PagedPool, context->stripes[i].length * Vcb->csum_size / Vcb->superblock.node_size, ALLOC_TAG);
                if (!context->stripes[i].bad_csums) {
                    ERR("out of memory\n");
                    return STATUS_INSUFFICIENT_RESOURCES;
                }

                for (j = 0; j < context->stripes[i].length / Vcb->superblock.node_size; j++) {
                    tree_header* th = (tree_header*)&context->stripes[i].buf[j * Vcb->superblock.node_size];

                    get_tree_checksum(Vcb, th, (uint8_t*)context->stripes[i].bad_csums + (Vcb->csum_size * j));
                }
            }
        }
    }

    if (present_devices > 1) {
        ULONG good_stripe = 0xffffffff;

        for (i = 0; i < c->chunk_item->num_stripes; i++) {
            if (c->devices[i]->devobj && !context->stripes[i].csum_error) {
                good_stripe = i;
                break;
            }
        }

        if (good_stripe != 0xffffffff) {
            // log

            for (i = 0; i < c->chunk_item->num_stripes; i++) {
                if (context->stripes[i].csum_error) {
                    ULONG j;

                    if (csum) {
                        for (j = 0; j < context->stripes[i].length / Vcb->superblock.sector_size; j++) {
                            if (RtlCompareMemory((uint8_t*)context->stripes[i].bad_csums + (j * Vcb->csum_size), (uint8_t*)csum + (j + Vcb->csum_size), Vcb->csum_size) != Vcb->csum_size) {
                                uint64_t addr = offset + UInt32x32To64(j, Vcb->superblock.sector_size);

                                log_error(Vcb, addr, c->devices[i]->devitem.dev_id, false, true, false);
                                log_device_error(Vcb, c->devices[i], BTRFS_DEV_STAT_CORRUPTION_ERRORS);
                            }
                        }
                    } else {
                        for (j = 0; j < context->stripes[i].length / Vcb->superblock.node_size; j++) {
                            tree_header* th = (tree_header*)&context->stripes[i].buf[j * Vcb->superblock.node_size];
                            uint64_t addr = offset + UInt32x32To64(j, Vcb->superblock.node_size);

                            if (RtlCompareMemory((uint8_t*)context->stripes[i].bad_csums + (j * Vcb->csum_size), th, Vcb->csum_size) != Vcb->csum_size || th->address != addr) {
                                log_error(Vcb, addr, c->devices[i]->devitem.dev_id, true, true, false);
                                log_device_error(Vcb, c->devices[i], BTRFS_DEV_STAT_CORRUPTION_ERRORS);
                            }
                        }
                    }
                }
            }

            // write good data over bad

            for (i = 0; i < c->chunk_item->num_stripes; i++) {
                if (context->stripes[i].csum_error && !c->devices[i]->readonly) {
                    Status = write_data_phys(c->devices[i]->devobj, c->devices[i]->fileobj, cis[i].offset + offset - c->offset,
                                             context->stripes[good_stripe].buf, context->stripes[i].length);

                    if (!NT_SUCCESS(Status)) {
                        ERR("write_data_phys returned %08lx\n", Status);
                        log_device_error(Vcb, c->devices[i], BTRFS_DEV_STAT_WRITE_ERRORS);
                        return Status;
                    }
                }
            }

            return STATUS_SUCCESS;
        }

        // if csum errors on all stripes, check sector by sector

        for (i = 0; i < c->chunk_item->num_stripes; i++) {
            ULONG j;

            if (c->devices[i]->devobj) {
                if (csum) {
                    for (j = 0; j < context->stripes[i].length / Vcb->superblock.sector_size; j++) {
                        if (RtlCompareMemory((uint8_t*)context->stripes[i].bad_csums + (j * Vcb->csum_size), (uint8_t*)csum + (j * Vcb->csum_size), Vcb->csum_size) != Vcb->csum_size) {
                            ULONG k;
                            uint64_t addr = offset + UInt32x32To64(j, Vcb->superblock.sector_size);
                            bool recovered = false;

                            for (k = 0; k < c->chunk_item->num_stripes; k++) {
                                if (i != k && c->devices[k]->devobj &&
                                    RtlCompareMemory((uint8_t*)context->stripes[k].bad_csums + (j * Vcb->csum_size),
                                                     (uint8_t*)csum + (j * Vcb->csum_size), Vcb->csum_size) == Vcb->csum_size) {
                                    log_error(Vcb, addr, c->devices[i]->devitem.dev_id, false, true, false);
                                    log_device_error(Vcb, c->devices[i], BTRFS_DEV_STAT_CORRUPTION_ERRORS);

                                    RtlCopyMemory(context->stripes[i].buf + (j * Vcb->superblock.sector_size),
                                                  context->stripes[k].buf + (j * Vcb->superblock.sector_size), Vcb->superblock.sector_size);

                                    recovered = true;
                                    break;
                                }
                            }

                            if (!recovered) {
                                log_error(Vcb, addr, c->devices[i]->devitem.dev_id, false, false, false);
                                log_device_error(Vcb, c->devices[i], BTRFS_DEV_STAT_CORRUPTION_ERRORS);
                            }
                        }
                    }
                } else {
                    for (j = 0; j < context->stripes[i].length / Vcb->superblock.node_size; j++) {
                        tree_header* th = (tree_header*)&context->stripes[i].buf[j * Vcb->superblock.node_size];
                        uint64_t addr = offset + UInt32x32To64(j, Vcb->superblock.node_size);

                        if (RtlCompareMemory((uint8_t*)context->stripes[i].bad_csums + (j * Vcb->csum_size), th, Vcb->csum_size) != Vcb->csum_size || th->address != addr) {
                            ULONG k;
                            bool recovered = false;

                            for (k = 0; k < c->chunk_item->num_stripes; k++) {
                                if (i != k && c->devices[k]->devobj) {
                                    tree_header* th2 = (tree_header*)&context->stripes[k].buf[j * Vcb->superblock.node_size];

                                    if (RtlCompareMemory((uint8_t*)context->stripes[k].bad_csums + (j * Vcb->csum_size), th2, Vcb->csum_size) == Vcb->csum_size && th2->address == addr) {
                                        log_error(Vcb, addr, c->devices[i]->devitem.dev_id, true, true, false);
                                        log_device_error(Vcb, c->devices[i], BTRFS_DEV_STAT_CORRUPTION_ERRORS);

                                        RtlCopyMemory(th, th2, Vcb->superblock.node_size);

                                        recovered = true;
                                        break;
                                    }
                                }
                            }

                            if (!recovered) {
                                log_error(Vcb, addr, c->devices[i]->devitem.dev_id, true, false, false);
                                log_device_error(Vcb, c->devices[i], BTRFS_DEV_STAT_CORRUPTION_ERRORS);
                            }
                        }
                    }
                }
            }
        }

        // write good data over bad

        for (i = 0; i < c->chunk_item->num_stripes; i++) {
            if (c->devices[i]->devobj && !c->devices[i]->readonly) {
                Status = write_data_phys(c->devices[i]->devobj, c->devices[i]->fileobj, cis[i].offset + offset - c->offset,
                                         context->stripes[i].buf, context->stripes[i].length);
                if (!NT_SUCCESS(Status)) {
                    ERR("write_data_phys returned %08lx\n", Status);
                    log_device_error(Vcb, c->devices[i], BTRFS_DEV_STAT_CORRUPTION_ERRORS);
                    return Status;
                }
            }
        }

        return STATUS_SUCCESS;
    }

    for (i = 0; i < c->chunk_item->num_stripes; i++) {
        if (c->devices[i]->devobj) {
            ULONG j;

            if (csum) {
                for (j = 0; j < context->stripes[i].length / Vcb->superblock.sector_size; j++) {
                    if (RtlCompareMemory((uint8_t*)context->stripes[i].bad_csums + (j * Vcb->csum_size), (uint8_t*)csum + (j + Vcb->csum_size), Vcb->csum_size) != Vcb->csum_size) {
                        uint64_t addr = offset + UInt32x32To64(j, Vcb->superblock.sector_size);

                        log_error(Vcb, addr, c->devices[i]->devitem.dev_id, false, false, false);
                    }
                }
            } else {
                for (j = 0; j < context->stripes[i].length / Vcb->superblock.node_size; j++) {
                    tree_header* th = (tree_header*)&context->stripes[i].buf[j * Vcb->superblock.node_size];
                    uint64_t addr = offset + UInt32x32To64(j, Vcb->superblock.node_size);

                    if (RtlCompareMemory((uint8_t*)context->stripes[i].bad_csums + (j * Vcb->csum_size), th, Vcb->csum_size) != Vcb->csum_size || th->address != addr)
                        log_error(Vcb, addr, c->devices[i]->devitem.dev_id, true, false, false);
                }
            }
        }
    }

    return STATUS_SUCCESS;
}

Referenced by scrub_extent().

scrub_extent_raid0()

static NTSTATUS scrub_extent_raid0 ( device_extension *  Vcb, chunk *  c, uint64_t  offset, uint32_t  length, uint16_t  startoffstripe, void *  csum, scrub_context *  context  )

static

Definition at line 940 of file scrub.c.

                                                                                                                                                                   {
    ULONG j;
    uint16_t stripe;
    uint32_t pos, *stripeoff;

    pos = 0;
    stripeoff = ExAllocatePoolWithTag(NonPagedPool, sizeof(uint32_t) * c->chunk_item->num_stripes, ALLOC_TAG);
    if (!stripeoff) {
        ERR("out of memory\n");
        return STATUS_INSUFFICIENT_RESOURCES;
    }

    RtlZeroMemory(stripeoff, sizeof(uint32_t) * c->chunk_item->num_stripes);

    stripe = startoffstripe;
    while (pos < length) {
        uint32_t readlen;

        if (pos == 0)
            readlen = (uint32_t)min(context->stripes[stripe].length, c->chunk_item->stripe_length - (context->stripes[stripe].start % c->chunk_item->stripe_length));
        else
            readlen = min(length - pos, (uint32_t)c->chunk_item->stripe_length);

        if (csum) {
            for (j = 0; j < readlen; j += Vcb->superblock.sector_size) {
                if (!check_sector_csum(Vcb, context->stripes[stripe].buf + stripeoff[stripe], (uint8_t*)csum + (pos * Vcb->csum_size / Vcb->superblock.sector_size))) {
                    uint64_t addr = offset + pos;

                    log_error(Vcb, addr, c->devices[stripe]->devitem.dev_id, false, false, false);
                    log_device_error(Vcb, c->devices[stripe], BTRFS_DEV_STAT_CORRUPTION_ERRORS);
                }

                pos += Vcb->superblock.sector_size;
                stripeoff[stripe] += Vcb->superblock.sector_size;
            }
        } else {
            for (j = 0; j < readlen; j += Vcb->superblock.node_size) {
                tree_header* th = (tree_header*)(context->stripes[stripe].buf + stripeoff[stripe]);
                uint64_t addr = offset + pos;

                if (!check_tree_checksum(Vcb, th) || th->address != addr) {
                    log_error(Vcb, addr, c->devices[stripe]->devitem.dev_id, true, false, false);
                    log_device_error(Vcb, c->devices[stripe], BTRFS_DEV_STAT_CORRUPTION_ERRORS);
                }

                pos += Vcb->superblock.node_size;
                stripeoff[stripe] += Vcb->superblock.node_size;
            }
        }

        stripe = (stripe + 1) % c->chunk_item->num_stripes;
    }

    ExFreePool(stripeoff);

    return STATUS_SUCCESS;
}

Referenced by scrub_extent().

scrub_extent_raid10()

static NTSTATUS scrub_extent_raid10 ( device_extension *  Vcb, chunk *  c, uint64_t  offset, uint32_t  length, uint16_t  startoffstripe, void *  csum, scrub_context *  context  )

static

Definition at line 998 of file scrub.c.

                                                                                                                                                                    {
    ULONG j;
    uint16_t stripe, sub_stripes = max(c->chunk_item->sub_stripes, 1);
    uint32_t pos, *stripeoff;
    bool csum_error = false;
    NTSTATUS Status;

    pos = 0;
    stripeoff = ExAllocatePoolWithTag(NonPagedPool, sizeof(uint32_t) * c->chunk_item->num_stripes / sub_stripes, ALLOC_TAG);
    if (!stripeoff) {
        ERR("out of memory\n");
        return STATUS_INSUFFICIENT_RESOURCES;
    }

    RtlZeroMemory(stripeoff, sizeof(uint32_t) * c->chunk_item->num_stripes / sub_stripes);

    stripe = startoffstripe;
    while (pos < length) {
        uint32_t readlen;

        if (pos == 0)
            readlen = (uint32_t)min(context->stripes[stripe * sub_stripes].length,
                                  c->chunk_item->stripe_length - (context->stripes[stripe * sub_stripes].start % c->chunk_item->stripe_length));
        else
            readlen = min(length - pos, (uint32_t)c->chunk_item->stripe_length);

        if (csum) {
            ULONG good_stripe = 0xffffffff;
            uint16_t k;

            for (k = 0; k < sub_stripes; k++) {
                if (c->devices[(stripe * sub_stripes) + k]->devobj) {
                    // if first stripe is okay, we only need to check that the others are identical to it
                    if (good_stripe != 0xffffffff) {
                        if (RtlCompareMemory(context->stripes[(stripe * sub_stripes) + k].buf + stripeoff[stripe],
                                            context->stripes[(stripe * sub_stripes) + good_stripe].buf + stripeoff[stripe],
                                            readlen) != readlen) {
                            context->stripes[(stripe * sub_stripes) + k].csum_error = true;
                            csum_error = true;
                            log_device_error(Vcb, c->devices[(stripe * sub_stripes) + k], BTRFS_DEV_STAT_CORRUPTION_ERRORS);
                        }
                    } else {
                        for (j = 0; j < readlen; j += Vcb->superblock.sector_size) {
                            if (!check_sector_csum(Vcb, context->stripes[(stripe * sub_stripes) + k].buf + stripeoff[stripe] + j,
                                                   (uint8_t*)csum + ((pos + j) * Vcb->csum_size / Vcb->superblock.sector_size))) {
                                csum_error = true;
                                context->stripes[(stripe * sub_stripes) + k].csum_error = true;
                                log_device_error(Vcb, c->devices[(stripe * sub_stripes) + k], BTRFS_DEV_STAT_CORRUPTION_ERRORS);
                                break;
                            }
                        }

                        if (!context->stripes[(stripe * sub_stripes) + k].csum_error)
                            good_stripe = k;
                    }
                }
            }

            pos += readlen;
            stripeoff[stripe] += readlen;
        } else {
            ULONG good_stripe = 0xffffffff;
            uint16_t k;

            for (k = 0; k < sub_stripes; k++) {
                if (c->devices[(stripe * sub_stripes) + k]->devobj) {
                    // if first stripe is okay, we only need to check that the others are identical to it
                    if (good_stripe != 0xffffffff) {
                        if (RtlCompareMemory(context->stripes[(stripe * sub_stripes) + k].buf + stripeoff[stripe],
                                            context->stripes[(stripe * sub_stripes) + good_stripe].buf + stripeoff[stripe],
                                            readlen) != readlen) {
                            context->stripes[(<