swapout.c File Reference

#include <ntoskrnl.h>
#include "newmm.h"
#include <debug.h>

Include dependency graph for swapout.c:

Go to the source code of this file.

Macros
#define	NDEBUG
#define	DPRINTC   DPRINT

Functions
PFN_NUMBER NTAPI	MmWithdrawSectionPage (PMM_SECTION_SEGMENT Segment, PLARGE_INTEGER FileOffset, BOOLEAN *Dirty)
NTSTATUS NTAPI	MmFinalizeSectionPageOut (PMM_SECTION_SEGMENT Segment, PLARGE_INTEGER FileOffset, PFN_NUMBER Page, BOOLEAN Dirty)
NTSTATUS NTAPI	MmPageOutCacheSection (PMMSUPPORT AddressSpace, MEMORY_AREA *MemoryArea, PVOID Address, PBOOLEAN Dirty, PMM_REQUIRED_RESOURCES Required)
NTSTATUS NTAPI	MmpPageOutPhysicalAddress (PFN_NUMBER Page)
ULONG NTAPI	MiCacheEvictPages (PMM_SECTION_SEGMENT Segment, ULONG Target)
NTSTATUS	MiRosTrimCache (ULONG Target, ULONG Priority, PULONG NrFreed)

Variables
KEVENT	MmWaitPageEvent
FAST_MUTEX	RmapListLock
PMMWSL	MmWorkingSetList
FAST_MUTEX	MiGlobalPageOperation
LIST_ENTRY	MiSegmentList

Macro Definition Documentation

DPRINTC

#define DPRINTC   DPRINT

Definition at line 68 of file swapout.c.

NDEBUG

#define NDEBUG

Definition at line 65 of file swapout.c.

Function Documentation

MiCacheEvictPages()

ULONG NTAPI MiCacheEvictPages	(	PMM_SECTION_SEGMENT	Segment,
		ULONG	Target
	)

Definition at line 590 of file swapout.c.

{
    ULONG_PTR Entry;
    ULONG Result = 0, i, j;
    NTSTATUS Status;
    PFN_NUMBER Page;
    LARGE_INTEGER Offset;
 
    MmLockSectionSegment(Segment);
 
    for (i = 0; i < RtlNumberGenericTableElements(&Segment->PageTable); i++) {
 
        PCACHE_SECTION_PAGE_TABLE Element = RtlGetElementGenericTable(&Segment->PageTable,
                                                                      i);
 
        ASSERT(Element);
 
        Offset = Element->FileOffset;
        for (j = 0; j < ENTRIES_PER_ELEMENT; j++, Offset.QuadPart += PAGE_SIZE) {
            Entry = MmGetPageEntrySectionSegment(Segment, &Offset);
            if (Entry && !IS_SWAP_FROM_SSE(Entry)) {
                Page = PFN_FROM_SSE(Entry);
                MmUnlockSectionSegment(Segment);
                Status = MmpPageOutPhysicalAddress(Page);
                if (NT_SUCCESS(Status))
                    Result++;
                MmLockSectionSegment(Segment);
            }
        }
    }
 
    MmUnlockSectionSegment(Segment);
 
    return Result;
}

Referenced by MiRosTrimCache().

MiRosTrimCache()

NTSTATUS MiRosTrimCache	(	ULONG	Target,
		ULONG	Priority,
		PULONG	NrFreed
	)

Definition at line 633 of file swapout.c.

{
    ULONG Freed;
    PLIST_ENTRY Entry;
    PMM_SECTION_SEGMENT Segment;
    *NrFreed = 0;
 
    DPRINT1("Need to trim %lu cache pages\n", Target);
    for (Entry = MiSegmentList.Flink;
         *NrFreed < Target && Entry != &MiSegmentList;
         Entry = Entry->Flink) {
        Segment = CONTAINING_RECORD(Entry, MM_SECTION_SEGMENT, ListOfSegments);
        /* Defer to MM to try recovering pages from it */
        Freed = MiCacheEvictPages(Segment, Target);
        *NrFreed += Freed;
    }
    DPRINT1("Evicted %lu cache pages\n", Target);
 
    if (!IsListEmpty(&MiSegmentList)) {
        Entry = MiSegmentList.Flink;
        RemoveEntryList(Entry);
        InsertTailList(&MiSegmentList, Entry);
    }
 
    return STATUS_SUCCESS;
}

Referenced by CcShutdownSystem(), and MmInitSystem().

MmFinalizeSectionPageOut()

NTSTATUS NTAPI MmFinalizeSectionPageOut	(	PMM_SECTION_SEGMENT	Segment,
		PLARGE_INTEGER	FileOffset,
		PFN_NUMBER	Page,
		BOOLEAN	Dirty
	)

Definition at line 165 of file swapout.c.

{
    NTSTATUS Status = STATUS_SUCCESS;
    BOOLEAN WriteZero = FALSE, WritePage = FALSE;
    SWAPENTRY Swap = MmGetSavedSwapEntryPage(Page);
 
    /* Bail early if the reference count isn't where we need it */
    if (MmGetReferenceCountPageWithoutLock(Page) != 1)
    {
        DPRINT1("Cannot page out locked page %x with ref count %lu\n",
                Page,
                MmGetReferenceCountPageWithoutLock(Page));
        return STATUS_UNSUCCESSFUL;
    }
 
    MmLockSectionSegment(Segment);
    (void)InterlockedIncrementUL(&Segment->ReferenceCount);
 
    if (Dirty)
    {
        DPRINT("Finalize (dirty) Segment %p Page %x\n", Segment, Page);
        DPRINT("Segment->FileObject %p\n", Segment->FileObject);
        DPRINT("Segment->Flags %x\n", Segment->Flags);
 
        WriteZero = TRUE;
        WritePage = TRUE;
    }
    else
    {
        WriteZero = TRUE;
    }
 
    DPRINT("Status %x\n", Status);
 
    MmUnlockSectionSegment(Segment);
 
    if (WritePage)
    {
        DPRINT("MiWriteBackPage(Segment %p FileObject %p Offset %x)\n",
               Segment,
               Segment->FileObject,
               FileOffset->LowPart);
 
        Status = MiWriteBackPage(Segment->FileObject,
                                 FileOffset,
                                 PAGE_SIZE,
                                 Page);
    }
 
    MmLockSectionSegment(Segment);
 
    if (WriteZero && NT_SUCCESS(Status))
    {
        DPRINT("Setting page entry in segment %p:%x to swap %x\n",
               Segment,
               FileOffset->LowPart,
               Swap);
 
        MmSetPageEntrySectionSegment(Segment,
                                     FileOffset,
                                     Swap ? MAKE_SWAP_SSE(Swap) : 0);
    }
    else
    {
        DPRINT("Setting page entry in segment %p:%x to page %x\n",
               Segment,
               FileOffset->LowPart,
               Page);
 
        MmSetPageEntrySectionSegment(Segment,
                                     FileOffset,
                                     Page ? (Dirty ? DIRTY_SSE(MAKE_PFN_SSE(Page)) : MAKE_PFN_SSE(Page)) : 0);
    }
 
    if (NT_SUCCESS(Status))
    {
        DPRINT("Removing page %x for real\n", Page);
        MmSetSavedSwapEntryPage(Page, 0);
        MmReleasePageMemoryConsumer(MC_CACHE, Page);
    }
 
    MmUnlockSectionSegment(Segment);
 
    if (InterlockedDecrementUL(&Segment->ReferenceCount) == 0)
    {
        MmFinalizeSegment(Segment);
    }
 
    /* Note: Writing may evict the segment... Nothing is guaranteed from here down */
    MiSetPageEvent(Segment, (ULONG_PTR)FileOffset->QuadPart);
 
    DPRINT("Status %x\n", Status);
    return Status;
}

Referenced by MmpPageOutPhysicalAddress().

MmPageOutCacheSection()

NTSTATUS NTAPI MmPageOutCacheSection	(	PMMSUPPORT	AddressSpace,
		MEMORY_AREA *	MemoryArea,
		PVOID	Address,
		PBOOLEAN	Dirty,
		PMM_REQUIRED_RESOURCES	Required
	)

Definition at line 279 of file swapout.c.

{
    ULONG_PTR Entry;
    PFN_NUMBER OurPage;
    PEPROCESS Process = MmGetAddressSpaceOwner(AddressSpace);
    LARGE_INTEGER TotalOffset;
    PMM_SECTION_SEGMENT Segment;
    PVOID PAddress = MM_ROUND_DOWN(Address, PAGE_SIZE);
 
    TotalOffset.QuadPart = (ULONG_PTR)PAddress -
                           MA_GetStartingAddress(MemoryArea) +
                           MemoryArea->Data.SectionData.ViewOffset.QuadPart;
 
    Segment = MemoryArea->Data.SectionData.Segment;
 
    MmLockSectionSegment(Segment);
    ASSERT(KeGetCurrentIrql() <= APC_LEVEL);
 
    Entry = MmGetPageEntrySectionSegment(Segment, &TotalOffset);
    DBG_UNREFERENCED_LOCAL_VARIABLE(Entry);
 
    if (MmIsPageSwapEntry(Process, PAddress))
    {
        SWAPENTRY SwapEntry;
        MmGetPageFileMapping(Process, PAddress, &SwapEntry);
        MmUnlockSectionSegment(Segment);
        return SwapEntry == MM_WAIT_ENTRY ? STATUS_SUCCESS + 1 : STATUS_UNSUCCESSFUL;
    }
 
    MmDeleteRmap(Required->Page[0], Process, Address);
    MmDeleteVirtualMapping(Process, Address, Dirty, &OurPage);
    ASSERT(OurPage == Required->Page[0]);
 
    /* Note: this releases the reference held by this address space only. */
    MmReleasePageMemoryConsumer(MC_CACHE, Required->Page[0]);
 
    MmUnlockSectionSegment(Segment);
    MiSetPageEvent(Process, Address);
    return STATUS_SUCCESS;
}

Referenced by MmpPageOutPhysicalAddress().

MmpPageOutPhysicalAddress()

NTSTATUS NTAPI MmpPageOutPhysicalAddress

(

PFN_NUMBER

Page

)

Definition at line 345 of file swapout.c.

{
    BOOLEAN ProcRef = FALSE, PageDirty;
    PFN_NUMBER SectionPage = 0;
    PMM_RMAP_ENTRY entry;
    PMM_SECTION_SEGMENT Segment = NULL;
    LARGE_INTEGER FileOffset;
    PMEMORY_AREA MemoryArea;
    PMMSUPPORT AddressSpace = NULL;
    BOOLEAN Dirty = FALSE;
    PVOID Address = NULL;
    PEPROCESS Process = NULL;
    NTSTATUS Status = STATUS_SUCCESS;
    MM_REQUIRED_RESOURCES Resources = { 0 };
 
    DPRINTC("Page out %x (ref ct %x)\n", Page, MmGetReferenceCountPageWithoutLock(Page));
 
    ExAcquireFastMutex(&MiGlobalPageOperation);
    if ((Segment = MmGetSectionAssociation(Page, &FileOffset)))
    {
        DPRINTC("Withdrawing page (%x) %p:%x\n",
                Page,
                Segment,
                FileOffset.LowPart);
 
        SectionPage = MmWithdrawSectionPage(Segment, &FileOffset, &Dirty);
        DPRINTC("SectionPage %x\n", SectionPage);
 
        if (SectionPage == MM_WAIT_ENTRY || SectionPage == 0)
        {
            DPRINT1("In progress page out %x\n", SectionPage);
            ExReleaseFastMutex(&MiGlobalPageOperation);
            return STATUS_UNSUCCESSFUL;
        }
        else
        {
            ASSERT(SectionPage == Page);
        }
        Resources.State = Dirty ? 1 : 0;
    }
    else
    {
        DPRINT("No segment association for %x\n", Page);
    }
 
    Dirty = MmIsDirtyPageRmap(Page);
 
    DPRINTC("Trying to unmap all instances of %x\n", Page);
    ExAcquireFastMutex(&RmapListLock);
    entry = MmGetRmapListHeadPage(Page);
 
    // Entry and Segment might be null here in the case that the page
    // is new and is in the process of being swapped in
    if (!entry && !Segment)
    {
        Status = STATUS_UNSUCCESSFUL;
        DPRINT1("Page %x is in transit\n", Page);
        ExReleaseFastMutex(&RmapListLock);
        goto bail;
    }
 
    while (entry != NULL && NT_SUCCESS(Status))
    {
        Process = entry->Process;
        Address = entry->Address;
 
        DPRINTC("Process %p Address %p Page %x\n", Process, Address, Page);
 
        if (RMAP_IS_SEGMENT(Address))
        {
            entry = entry->Next;
            continue;
        }
 
        if (Process && Address < MmSystemRangeStart)
        {
            /* Make sure we don't try to page out part of an exiting process */
            if (PspIsProcessExiting(Process))
            {
                DPRINT("bail\n");
                ExReleaseFastMutex(&RmapListLock);
                goto bail;
            }
            ObReferenceObject(Process);
            ProcRef = TRUE;
            AddressSpace = &Process->Vm;
        }
        else
        {
            AddressSpace = MmGetKernelAddressSpace();
        }
        ExReleaseFastMutex(&RmapListLock);
 
        RtlZeroMemory(&Resources, sizeof(Resources));
 
        if ((((ULONG_PTR)Address) & 0xFFF) != 0)
        {
            KeBugCheck(MEMORY_MANAGEMENT);
        }
 
        do
        {
            MmLockAddressSpace(AddressSpace);
 
            MemoryArea = MmLocateMemoryAreaByAddress(AddressSpace, Address);
            if (MemoryArea == NULL || MemoryArea->DeleteInProgress)
            {
                Status = STATUS_UNSUCCESSFUL;
                MmUnlockAddressSpace(AddressSpace);
                DPRINTC("bail\n");
                goto bail;
            }
 
            DPRINTC("Type %x (%p -> %p)\n",
                    MemoryArea->Type,
                    MA_GetStartingAddress(MemoryArea),
                    MA_GetEndingAddress(MemoryArea));
 
            Resources.DoAcquisition = NULL;
            Resources.Page[0] = Page;
 
            ASSERT(KeGetCurrentIrql() <= APC_LEVEL);
 
            DPRINT("%p:%p, page %x %x\n",
                   Process,
                   Address,
                   Page,
                   Resources.Page[0]);
 
            PageDirty = FALSE;
 
            Status = MmPageOutCacheSection(AddressSpace,
                                           MemoryArea,
                                           Address,
                                           &PageDirty,
                                           &Resources);
 
            Dirty |= PageDirty;
            DPRINT("%x\n", Status);
 
            ASSERT(KeGetCurrentIrql() <= APC_LEVEL);
 
            MmUnlockAddressSpace(AddressSpace);
 
            if (Status == STATUS_SUCCESS + 1)
            {
                // Wait page ... the other guy has it, so we'll just fail for now
                DPRINT1("Wait entry ... can't continue\n");
                Status = STATUS_UNSUCCESSFUL;
                goto bail;
            }
            else if (Status == STATUS_MORE_PROCESSING_REQUIRED)
            {
                DPRINTC("DoAcquisition %p\n", Resources.DoAcquisition);
 
                Status = Resources.DoAcquisition(AddressSpace,
                                                 MemoryArea,
                                                 &Resources);
 
                DPRINTC("Status %x\n", Status);
                if (!NT_SUCCESS(Status))
                {
                    DPRINT1("bail\n");
                    goto bail;
                }
                else
                {
                    Status = STATUS_MM_RESTART_OPERATION;
                }
            }
        }
        while (Status == STATUS_MM_RESTART_OPERATION);
 
        if (ProcRef)
        {
            ObDereferenceObject(Process);
            ProcRef = FALSE;
        }
 
        ExAcquireFastMutex(&RmapListLock);
        ASSERT(!MM_IS_WAIT_PTE(MmGetPfnForProcess(Process, Address)));
        entry = MmGetRmapListHeadPage(Page);
 
        DPRINTC("Entry %p\n", entry);
    }
 
    ExReleaseFastMutex(&RmapListLock);
 
bail:
    DPRINTC("BAIL %x\n", Status);
 
    if (Segment)
    {
        ULONG RefCount;
 
        DPRINTC("About to finalize section page %x (%p:%x) Status %x %s\n",
                Page,
                Segment,
                FileOffset.LowPart,
                Status,
                Dirty ? "dirty" : "clean");
 
        if (!NT_SUCCESS(Status) ||
            !NT_SUCCESS(Status = MmFinalizeSectionPageOut(Segment,
                                                          &FileOffset,
                                                          Page,
                                                          Dirty)))
        {
            DPRINTC("Failed to page out %x, replacing %x at %x in segment %x\n",
                    SectionPage,
                    FileOffset.LowPart,
                    Segment);
 
            MmLockSectionSegment(Segment);
 
            MmSetPageEntrySectionSegment(Segment,
                                         &FileOffset,
                                         Dirty ? MAKE_PFN_SSE(Page) : DIRTY_SSE(MAKE_PFN_SSE(Page)));
 
            MmUnlockSectionSegment(Segment);
        }
 
        /* Alas, we had the last reference */
        if ((RefCount = InterlockedDecrementUL(&Segment->ReferenceCount)) == 0)
            MmFinalizeSegment(Segment);
    }
 
    if (ProcRef)
    {
        DPRINTC("Dereferencing process...\n");
        ObDereferenceObject(Process);
    }
 
    ExReleaseFastMutex(&MiGlobalPageOperation);
 
    DPRINTC("%s %x %x\n",
            NT_SUCCESS(Status) ? "Evicted" : "Spared",
            Page,
            Status);
 
    return NT_SUCCESS(Status) ? STATUS_SUCCESS : STATUS_UNSUCCESSFUL;
}

Referenced by MiCacheEvictPages(), and MmPageOutPhysicalAddress().

MmWithdrawSectionPage()

PFN_NUMBER NTAPI MmWithdrawSectionPage	(	PMM_SECTION_SEGMENT	Segment,
		PLARGE_INTEGER	FileOffset,
		BOOLEAN *	Dirty
	)

Definition at line 87 of file swapout.c.

{
    ULONG_PTR Entry;
 
    DPRINT("MmWithdrawSectionPage(%p,%08x%08x,%p)\n",
           Segment,
           FileOffset->HighPart,
           FileOffset->LowPart,
           Dirty);
 
    MmLockSectionSegment(Segment);
    Entry = MmGetPageEntrySectionSegment(Segment, FileOffset);
 
    *Dirty = !!IS_DIRTY_SSE(Entry);
 
    DPRINT("Withdraw %x (%x) of %wZ\n",
           FileOffset->LowPart,
           Entry,
           Segment->FileObject ? &Segment->FileObject->FileName : NULL);
 
    if (!Entry)
    {
        DPRINT("Stoeled!\n");
        MmUnlockSectionSegment(Segment);
        return 0;
    }
    else if (MM_IS_WAIT_PTE(Entry))
    {
        DPRINT("WAIT\n");
        MmUnlockSectionSegment(Segment);
        return MM_WAIT_ENTRY;
    }
    else if (Entry && !IS_SWAP_FROM_SSE(Entry))
    {
        DPRINT("Page %x\n", PFN_FROM_SSE(Entry));
 
        *Dirty |= (Entry & 2);
 
        MmSetPageEntrySectionSegment(Segment,
                                     FileOffset,
                                     MAKE_SWAP_SSE(MM_WAIT_ENTRY));
 
        MmUnlockSectionSegment(Segment);
        return PFN_FROM_SSE(Entry);
    }
    else
    {
        DPRINT1("SWAP ENTRY?! (%p:%08x%08x)\n",
                Segment,
                FileOffset->HighPart,
                FileOffset->LowPart);
 
        ASSERT(FALSE);
        MmUnlockSectionSegment(Segment);
        return 0;
    }
}

Referenced by MmpPageOutPhysicalAddress().

Variable Documentation

MiGlobalPageOperation

FAST_MUTEX MiGlobalPageOperation

Definition at line 74 of file swapout.c.

Referenced by MmInitSystem(), and MmpPageOutPhysicalAddress().

MiSegmentList

LIST_ENTRY MiSegmentList

extern

Definition at line 86 of file data.c.

Referenced by MiRosTrimCache().

MmWaitPageEvent

KEVENT MmWaitPageEvent

extern

Definition at line 461 of file fault.c.

MmWorkingSetList

PMMWSL MmWorkingSetList

extern

Definition at line 19 of file wslist.cpp.

RmapListLock

FAST_MUTEX RmapListLock

extern

Referenced by MmpPageOutPhysicalAddress().