balance.c File Reference

#include <ntoskrnl.h>
#include <debug.h>
#include "ARM3/miarm.h"

Include dependency graph for balance.c:

Go to the source code of this file.

Classes
struct	_MM_ALLOCATION_REQUEST

Macros
#define	NDEBUG

Typedefs
typedef struct _MM_ALLOCATION_REQUEST	MM_ALLOCATION_REQUEST
typedef struct _MM_ALLOCATION_REQUEST *	PMM_ALLOCATION_REQUEST

Functions
VOID NTAPI	MmInitializeBalancer (ULONG NrAvailablePages, ULONG NrSystemPages)
VOID NTAPI	MmInitializeMemoryConsumer (ULONG Consumer, NTSTATUS(*Trim)(ULONG Target, ULONG Priority, PULONG NrFreed))
VOID NTAPI	MiZeroPhysicalPage (IN PFN_NUMBER PageFrameIndex)
NTSTATUS NTAPI	MmReleasePageMemoryConsumer (ULONG Consumer, PFN_NUMBER Page)
ULONG NTAPI	MiTrimMemoryConsumer (ULONG Consumer, ULONG InitialTarget)
NTSTATUS	MmTrimUserMemory (ULONG Target, ULONG Priority, PULONG NrFreedPages)
VOID NTAPI	MmRebalanceMemoryConsumers (VOID)
VOID NTAPI	MmRebalanceMemoryConsumersAndWait (VOID)
NTSTATUS NTAPI	MmRequestPageMemoryConsumer (ULONG Consumer, BOOLEAN CanWait, PPFN_NUMBER AllocatedPage)
VOID	CcRosTrimCache (_In_ ULONG Target, _Out_ PULONG NrFreed)
VOID NTAPI	MiBalancerThread (PVOID Unused)
VOID NTAPI	MiInitBalancerThread (VOID)

Variables
MM_MEMORY_CONSUMER	MiMemoryConsumers [MC_MAXIMUM]
static ULONG	MiMinimumAvailablePages
static ULONG	MiMinimumPagesPerRun
static CLIENT_ID	MiBalancerThreadId
static HANDLE	MiBalancerThreadHandle = NULL
static KEVENT	MiBalancerEvent
static KEVENT	MiBalancerDoneEvent
static KTIMER	MiBalancerTimer
static LONG	PageOutThreadActive

Macro Definition Documentation

NDEBUG

#define NDEBUG

Definition at line 12 of file balance.c.

Typedef Documentation

MM_ALLOCATION_REQUEST

typedef struct _MM_ALLOCATION_REQUEST MM_ALLOCATION_REQUEST

PMM_ALLOCATION_REQUEST

typedef struct _MM_ALLOCATION_REQUEST * PMM_ALLOCATION_REQUEST

Function Documentation

CcRosTrimCache()

VOID CcRosTrimCache	(	_In_ ULONG	Target,
		_Out_ PULONG	NrFreed
	)

Definition at line 457 of file view.c.

{
    PLIST_ENTRY current_entry;
    PROS_VACB current;
    ULONG PagesFreed;
    KIRQL oldIrql;
    LIST_ENTRY FreeList;
    BOOLEAN FlushedPages = FALSE;
 
    DPRINT("CcRosTrimCache(Target %lu)\n", Target);
 
    InitializeListHead(&FreeList);
 
    *NrFreed = 0;
 
retry:
    oldIrql = KeAcquireQueuedSpinLock(LockQueueMasterLock);
 
    current_entry = VacbLruListHead.Flink;
    while (current_entry != &VacbLruListHead)
    {
        ULONG Refs;
 
        current = CONTAINING_RECORD(current_entry,
                                    ROS_VACB,
                                    VacbLruListEntry);
 
        KeAcquireSpinLockAtDpcLevel(&current->SharedCacheMap->CacheMapLock);
 
        /* Reference the VACB */
        CcRosVacbIncRefCount(current);
 
        /* Check if it's mapped and not dirty */
        if (InterlockedCompareExchange((PLONG)&current->MappedCount, 0, 0) > 0 && !current->Dirty)
        {
            /* This code is never executed. It is left for reference only. */
#if 1
            DPRINT1("MmPageOutPhysicalAddress unexpectedly called\n");
            ASSERT(FALSE);
#else
            ULONG i;
            PFN_NUMBER Page;
 
            /* We have to break these locks to call MmPageOutPhysicalAddress */
            KeReleaseSpinLockFromDpcLevel(&current->SharedCacheMap->CacheMapLock);
            KeReleaseQueuedSpinLock(LockQueueMasterLock, oldIrql);
 
            /* Page out the VACB */
            for (i = 0; i < VACB_MAPPING_GRANULARITY / PAGE_SIZE; i++)
            {
                Page = (PFN_NUMBER)(MmGetPhysicalAddress((PUCHAR)current->BaseAddress + (i * PAGE_SIZE)).QuadPart >> PAGE_SHIFT);
 
                MmPageOutPhysicalAddress(Page);
            }
 
            /* Reacquire the locks */
            oldIrql = KeAcquireQueuedSpinLock(LockQueueMasterLock);
            KeAcquireSpinLockAtDpcLevel(&current->SharedCacheMap->CacheMapLock);
#endif
        }
 
        /* Only keep iterating though the loop while the lock is held */
        current_entry = current_entry->Flink;
 
        /* Dereference the VACB */
        Refs = CcRosVacbDecRefCount(current);
 
        /* Check if we can free this entry now */
        if (Refs < 2)
        {
            ASSERT(!current->Dirty);
            ASSERT(!current->MappedCount);
            ASSERT(Refs == 1);
 
            RemoveEntryList(&current->CacheMapVacbListEntry);
            RemoveEntryList(&current->VacbLruListEntry);
            InitializeListHead(&current->VacbLruListEntry);
            InsertHeadList(&FreeList, &current->CacheMapVacbListEntry);
 
            /* Calculate how many pages we freed for Mm */
            PagesFreed = min(VACB_MAPPING_GRANULARITY / PAGE_SIZE, Target);
            Target -= PagesFreed;
            (*NrFreed) += PagesFreed;
        }
 
        KeReleaseSpinLockFromDpcLevel(&current->SharedCacheMap->CacheMapLock);
    }
 
    KeReleaseQueuedSpinLock(LockQueueMasterLock, oldIrql);
 
    /* Try flushing pages if we haven't met our target */
    if ((Target > 0) && !FlushedPages)
    {
        /* Flush dirty pages to disk */
        CcRosFlushDirtyPages(Target, &PagesFreed, FALSE, FALSE);
        FlushedPages = TRUE;
 
        /* We can only swap as many pages as we flushed */
        if (PagesFreed < Target) Target = PagesFreed;
 
        /* Check if we flushed anything */
        if (PagesFreed != 0)
        {
            /* Try again after flushing dirty pages */
            DPRINT("Flushed %lu dirty cache pages to disk\n", PagesFreed);
            goto retry;
        }
    }
 
    while (!IsListEmpty(&FreeList))
    {
        ULONG Refs;
 
        current_entry = RemoveHeadList(&FreeList);
        current = CONTAINING_RECORD(current_entry,
                                    ROS_VACB,
                                    CacheMapVacbListEntry);
        InitializeListHead(&current->CacheMapVacbListEntry);
        Refs = CcRosVacbDecRefCount(current);
        ASSERT(Refs == 0);
    }
 
    DPRINT("Evicted %lu cache pages\n", (*NrFreed));
}

Referenced by MiBalancerThread().

MiBalancerThread()

VOID NTAPI MiBalancerThread

(

PVOID

Unused

)

Definition at line 354 of file balance.c.

{
    PVOID WaitObjects[2];
    NTSTATUS Status;
 
    WaitObjects[0] = &MiBalancerEvent;
    WaitObjects[1] = &MiBalancerTimer;
 
    while (TRUE)
    {
        KeSetEvent(&MiBalancerDoneEvent, IO_NO_INCREMENT, FALSE);
        Status = KeWaitForMultipleObjects(_countof(WaitObjects),
                                          WaitObjects,
                                          WaitAny,
                                          Executive,
                                          KernelMode,
                                          FALSE,
                                          NULL,
                                          NULL);
 
        if (Status == STATUS_WAIT_0 || Status == STATUS_WAIT_1)
        {
            ULONG InitialTarget = 0;
            ULONG Target;
            ULONG NrFreedPages;
 
            do
            {
                ULONG OldTarget = InitialTarget;
 
                /* Trim each consumer */
                for (ULONG i = 0; i < MC_MAXIMUM; i++)
                {
                    InitialTarget = MiTrimMemoryConsumer(i, InitialTarget);
                }
 
                /* Trim cache */
                Target = max(InitialTarget, abs(MiMinimumAvailablePages - MmAvailablePages));
                if (Target)
                {
                    CcRosTrimCache(Target, &NrFreedPages);
                    InitialTarget -= min(NrFreedPages, InitialTarget);
                }
 
                /* No pages left to swap! */
                if (InitialTarget != 0 &&
                    InitialTarget == OldTarget)
                {
                    /* Game over */
                    KeBugCheck(NO_PAGES_AVAILABLE);
                }
            }
            while (InitialTarget != 0);
 
            if (Status == STATUS_WAIT_0)
            {
                LONG Active = InterlockedExchange(&PageOutThreadActive, 0);
                ASSERT(Active == 1);
                DBG_UNREFERENCED_LOCAL_VARIABLE(Active);
            }
        }
        else
        {
            DPRINT1("KeWaitForMultipleObjects failed, status = %x\n", Status);
            KeBugCheck(MEMORY_MANAGEMENT);
        }
    }
}

Referenced by MiInitBalancerThread().

MiInitBalancerThread()

VOID NTAPI MiInitBalancerThread

(

VOID

)

Definition at line 426 of file balance.c.

{
    KPRIORITY Priority;
    NTSTATUS Status;
    LARGE_INTEGER Timeout;
 
    KeInitializeEvent(&MiBalancerEvent, SynchronizationEvent, FALSE);
    KeInitializeEvent(&MiBalancerDoneEvent, SynchronizationEvent, FALSE);
    KeInitializeTimerEx(&MiBalancerTimer, SynchronizationTimer);
 
    Timeout.QuadPart = -20000000; /* 2 sec */
    KeSetTimerEx(&MiBalancerTimer,
                 Timeout,
                 2000,         /* 2 sec */
                 NULL);
 
    Status = PsCreateSystemThread(&MiBalancerThreadHandle,
                                  THREAD_ALL_ACCESS,
                                  NULL,
                                  NULL,
                                  &MiBalancerThreadId,
                                  MiBalancerThread,
                                  NULL);
    if (!NT_SUCCESS(Status))
    {
        KeBugCheck(MEMORY_MANAGEMENT);
    }
 
    Priority = LOW_REALTIME_PRIORITY + 1;
    NtSetInformationThread(MiBalancerThreadHandle,
                           ThreadPriority,
                           &Priority,
                           sizeof(Priority));
 
}

Referenced by MmInitSystem().

MiTrimMemoryConsumer()

ULONG NTAPI MiTrimMemoryConsumer	(	ULONG	Consumer,
		ULONG	InitialTarget
	)

Definition at line 96 of file balance.c.

{
    ULONG Target = InitialTarget;
    ULONG NrFreedPages = 0;
    NTSTATUS Status;
 
    /* Make sure we can trim this consumer */
    if (!MiMemoryConsumers[Consumer].Trim)
    {
        /* Return the unmodified initial target */
        return InitialTarget;
    }
 
    if (MmAvailablePages < MiMinimumAvailablePages)
    {
        /* Global page limit exceeded */
        Target = (ULONG)max(Target, MiMinimumAvailablePages - MmAvailablePages);
    }
    else if (MiMemoryConsumers[Consumer].PagesUsed > MiMemoryConsumers[Consumer].PagesTarget)
    {
        /* Consumer page limit exceeded */
        Target = max(Target, MiMemoryConsumers[Consumer].PagesUsed - MiMemoryConsumers[Consumer].PagesTarget);
    }
 
    if (Target)
    {
        /* Now swap the pages out */
        Status = MiMemoryConsumers[Consumer].Trim(Target, MmAvailablePages < MiMinimumAvailablePages, &NrFreedPages);
 
        DPRINT("Trimming consumer %lu: Freed %lu pages with a target of %lu pages\n", Consumer, NrFreedPages, Target);
 
        if (!NT_SUCCESS(Status))
        {
            KeBugCheck(MEMORY_MANAGEMENT);
        }
    }
 
    /* Return the page count needed to be freed to meet the initial target */
    return (InitialTarget > NrFreedPages) ? (InitialTarget - NrFreedPages) : 0;
}

Referenced by MiBalancerThread().

MiZeroPhysicalPage()

VOID NTAPI MiZeroPhysicalPage

(

IN PFN_NUMBER

PageFrameIndex

)

Definition at line 122 of file pfnlist.c.

{
    KIRQL OldIrql;
    PVOID VirtualAddress;
    PEPROCESS Process = PsGetCurrentProcess();
 
    /* Map in hyperspace, then wipe it using XMMI or MEMSET */
    VirtualAddress = MiMapPageInHyperSpace(Process, PageFrameIndex, &OldIrql);
    ASSERT(VirtualAddress);
    KeZeroPages(VirtualAddress, PAGE_SIZE);
    MiUnmapPageInHyperSpace(Process, VirtualAddress, OldIrql);
}

Referenced by MiRemoveZeroPage().

MmInitializeBalancer()

VOID NTAPI MmInitializeBalancer	(	ULONG	NrAvailablePages,
		ULONG	NrSystemPages
	)

Definition at line 44 of file balance.c.

{
    memset(MiMemoryConsumers, 0, sizeof(MiMemoryConsumers));
 
    /* Set up targets. */
    MiMinimumAvailablePages = 256;
    MiMinimumPagesPerRun = 256;
    MiMemoryConsumers[MC_USER].PagesTarget = NrAvailablePages / 2;
}

Referenced by MiInitMachineDependent().

MmInitializeMemoryConsumer()

VOID NTAPI MmInitializeMemoryConsumer	(	ULONG	Consumer,
		NTSTATUS(*)(ULONG Target, ULONG Priority, PULONG NrFreed)	Trim
	)

Definition at line 57 of file balance.c.

{
    MiMemoryConsumers[Consumer].Trim = Trim;
}

Referenced by MmInitSystem().

MmRebalanceMemoryConsumers()

VOID NTAPI MmRebalanceMemoryConsumers

(

VOID

)

Definition at line 290 of file balance.c.

{
    if (InterlockedCompareExchange(&PageOutThreadActive, 1, 0) == 0)
    {
        KeSetEvent(&MiBalancerEvent, IO_NO_INCREMENT, FALSE);
    }
}

Referenced by MiDecrementAvailablePages(), and MmRebalanceMemoryConsumersAndWait().

MmRebalanceMemoryConsumersAndWait()

VOID NTAPI MmRebalanceMemoryConsumersAndWait

(

VOID

)

Definition at line 300 of file balance.c.

{
    ASSERT(PsGetCurrentProcess()->AddressCreationLock.Owner != KeGetCurrentThread());
    ASSERT(!MM_ANY_WS_LOCK_HELD(PsGetCurrentThread()));
    ASSERT(KeGetCurrentIrql() < DISPATCH_LEVEL);
 
    KeResetEvent(&MiBalancerDoneEvent);
    MmRebalanceMemoryConsumers();
    KeWaitForSingleObject(&MiBalancerDoneEvent, Executive, KernelMode, FALSE, NULL);
}

Referenced by MmAccessFault(), and MmArmAccessFault().

MmReleasePageMemoryConsumer()

NTSTATUS NTAPI MmReleasePageMemoryConsumer	(	ULONG	Consumer,
		PFN_NUMBER	Page
	)

Definition at line 72 of file balance.c.

{
    KIRQL OldIrql;
 
    if (Page == 0)
    {
        DPRINT1("Tried to release page zero.\n");
        KeBugCheck(MEMORY_MANAGEMENT);
    }
 
    (void)InterlockedDecrementUL(&MiMemoryConsumers[Consumer].PagesUsed);
    UpdateTotalCommittedPages(-1);
 
    OldIrql = MiAcquirePfnLock();
 
    MmDereferencePage(Page);
 
    MiReleasePfnLock(OldIrql);
 
    return(STATUS_SUCCESS);
}

Referenced by FreeSegmentPage(), if(), MiFreeSegmentPage(), MiGetOnePage(), MiPurgeImageSegment(), MiReadFilePage(), MmCreateProcessAddressSpace(), MmDeleteVirtualMapping(), MmFinalizeSectionPageOut(), MmFreeCacheSectionPage(), MmFreePageTable(), MmFreeSectionPage(), MmGetPageTableForProcess(), MmGetPageTableForProcessForPAE(), MmMakeSegmentResident(), MmPageOutCacheSection(), MmPageOutPhysicalAddress(), MmpFreePageFileSegment(), MmPurgeSegment(), and MmUnsharePageEntrySectionSegment().

MmRequestPageMemoryConsumer()

NTSTATUS NTAPI MmRequestPageMemoryConsumer	(	ULONG	Consumer,
		BOOLEAN	CanWait,
		PPFN_NUMBER	AllocatedPage
	)

Definition at line 313 of file balance.c.

{
    PFN_NUMBER Page;
 
    /* Delay some requests for the Memory Manager to recover pages (CORE-17624).
     * FIXME: This is suboptimal.
     */
    static INT i = 0;
    static LARGE_INTEGER TinyTime = {{-1L, -1L}};
    if (i++ >= 100)
    {
        KeDelayExecutionThread(KernelMode, FALSE, &TinyTime);
        i = 0;
    }
 
    /*
     * Actually allocate the page.
     */
    Page = MmAllocPage(Consumer);
    if (Page == 0)
    {
        *AllocatedPage = 0;
        return STATUS_NO_MEMORY;
    }
    *AllocatedPage = Page;
 
    /* Update the target */
    InterlockedIncrementUL(&MiMemoryConsumers[Consumer].PagesUsed);
    UpdateTotalCommittedPages(1);
 
    return(STATUS_SUCCESS);
}

Referenced by CcInitCacheZeroPage(), MiGetOnePage(), MiGetPageTableForProcess(), MiReadFilePage(), MiSwapInPage(), MmAccessFaultSectionView(), MmCreateProcessAddressSpace(), MmGetPageTableForProcess(), MmGetPageTableForProcessForPAE(), MmMakeSegmentResident(), and MmNotPresentFaultSectionView().

MmTrimUserMemory()

NTSTATUS MmTrimUserMemory	(	ULONG	Target,
		ULONG	Priority,
		PULONG	NrFreedPages
	)

Definition at line 138 of file balance.c.

{
    PFN_NUMBER FirstPage, CurrentPage;
    NTSTATUS Status;
 
    (*NrFreedPages) = 0;
 
    DPRINT("MM BALANCER: %s\n", Priority ? "Paging out!" : "Removing access bit!");
 
    FirstPage = MmGetLRUFirstUserPage();
    CurrentPage = FirstPage;
    while (CurrentPage != 0 && Target > 0)
    {
        if (Priority)
        {
            Status = MmPageOutPhysicalAddress(CurrentPage);
            if (NT_SUCCESS(Status))
            {
                DPRINT("Succeeded\n");
                Target--;
                (*NrFreedPages)++;
                if (CurrentPage == FirstPage)
                {
                    FirstPage = 0;
                }
            }
        }
        else
        {
            /* When not paging-out agressively, just reset the accessed bit */
            PEPROCESS Process = NULL;
            PVOID Address = NULL;
            BOOLEAN Accessed = FALSE;
 
            /*
             * We have a lock-ordering problem here. We cant lock the PFN DB before the Process address space.
             * So we must use circonvoluted loops.
             * Well...
             */
            while (TRUE)
            {
                KAPC_STATE ApcState;
                KIRQL OldIrql = MiAcquirePfnLock();
                PMM_RMAP_ENTRY Entry = MmGetRmapListHeadPage(CurrentPage);
                while (Entry)
                {
                    if (RMAP_IS_SEGMENT(Entry->Address))
                    {
                        Entry = Entry->Next;
                        continue;
                    }
 
                    /* Check that we didn't treat this entry before */
                    if (Entry->Address < Address)
                    {
                        Entry = Entry->Next;
                        continue;
                    }
 
                    if ((Entry->Address == Address) && (Entry->Process <= Process))
                    {
                        Entry = Entry->Next;
                        continue;
                    }
 
                    break;
                }
 
                if (!Entry)
                {
                    MiReleasePfnLock(OldIrql);
                    break;
                }
 
                Process = Entry->Process;
                Address = Entry->Address;
 
                ObReferenceObject(Process);
 
                if (!ExAcquireRundownProtection(&Process->RundownProtect))
                {
                    ObDereferenceObject(Process);
                    MiReleasePfnLock(OldIrql);
                    continue;
                }
 
                MiReleasePfnLock(OldIrql);
 
                KeStackAttachProcess(&Process->Pcb, &ApcState);
                MiLockProcessWorkingSet(Process, PsGetCurrentThread());
 
                /* Be sure this is still valid. */
                if (MmIsAddressValid(Address))
                {
                    PMMPTE Pte = MiAddressToPte(Address);
                    Accessed = Accessed || Pte->u.Hard.Accessed;
                    Pte->u.Hard.Accessed = 0;
 
                    /* There is no need to invalidate, the balancer thread is never on a user process */
                    //KeInvalidateTlbEntry(Address);
                }
 
                MiUnlockProcessWorkingSet(Process, PsGetCurrentThread());
 
                KeUnstackDetachProcess(&ApcState);
                ExReleaseRundownProtection(&Process->RundownProtect);
                ObDereferenceObject(Process);
            }
 
            if (!Accessed)
            {
                /* Nobody accessed this page since the last time we check. Time to clean up */
 
                Status = MmPageOutPhysicalAddress(CurrentPage);
                if (NT_SUCCESS(Status))
                {
                    if (CurrentPage == FirstPage)
                    {
                        FirstPage = 0;
                    }
                }
                // DPRINT1("Paged-out one page: %s\n", NT_SUCCESS(Status) ? "Yes" : "No");
            }
 
            /* Done for this page. */
            Target--;
        }
 
        CurrentPage = MmGetLRUNextUserPage(CurrentPage, TRUE);
        if (FirstPage == 0)
        {
            FirstPage = CurrentPage;
        }
        else if (CurrentPage == FirstPage)
        {
            DPRINT1("We are back at the start, abort!\n");
            return STATUS_SUCCESS;
        }
    }
 
    if (CurrentPage)
    {
        KIRQL OldIrql = MiAcquirePfnLock();
        MmDereferencePage(CurrentPage);
        MiReleasePfnLock(OldIrql);
    }
 
    return STATUS_SUCCESS;
}

Referenced by MmInitSystem().

Variable Documentation

MiBalancerDoneEvent

KEVENT MiBalancerDoneEvent

static

Definition at line 34 of file balance.c.

Referenced by MiBalancerThread(), MiInitBalancerThread(), and MmRebalanceMemoryConsumersAndWait().

MiBalancerEvent

KEVENT MiBalancerEvent

static

Definition at line 33 of file balance.c.

Referenced by MiBalancerThread(), MiInitBalancerThread(), and MmRebalanceMemoryConsumers().

MiBalancerThreadHandle

HANDLE MiBalancerThreadHandle = NULL

static

Definition at line 32 of file balance.c.

Referenced by MiInitBalancerThread().

MiBalancerThreadId

CLIENT_ID MiBalancerThreadId

static

Definition at line 31 of file balance.c.

Referenced by MiInitBalancerThread().

MiBalancerTimer

KTIMER MiBalancerTimer

static

Definition at line 35 of file balance.c.

Referenced by MiBalancerThread(), and MiInitBalancerThread().

MiMemoryConsumers

MM_MEMORY_CONSUMER MiMemoryConsumers[MC_MAXIMUM]

Definition at line 28 of file balance.c.

Referenced by MiTrimMemoryConsumer(), MmInitializeBalancer(), MmInitializeMemoryConsumer(), MmReleasePageMemoryConsumer(), MmRequestPageMemoryConsumer(), and QSI_DEF().

MiMinimumAvailablePages

ULONG MiMinimumAvailablePages

static

Definition at line 29 of file balance.c.

Referenced by MiBalancerThread(), MiTrimMemoryConsumer(), and MmInitializeBalancer().

MiMinimumPagesPerRun

ULONG MiMinimumPagesPerRun

static

Definition at line 30 of file balance.c.

Referenced by MmInitializeBalancer().

PageOutThreadActive

LONG PageOutThreadActive

static

Definition at line 37 of file balance.c.

Referenced by MiBalancerThread(), and MmRebalanceMemoryConsumers().