mm.h File Reference

#include <internal/arch/mm.h>

Include dependency graph for mm.h:

Go to the source code of this file.

Classes
struct	_MM_SECTION_SEGMENT
struct	_MM_IMAGE_SECTION_OBJECT
struct	_ROS_SECTION_OBJECT
struct	_MEMORY_AREA
struct	_MM_RMAP_ENTRY
struct	_MMPFNENTRY
struct	_MMPFN
struct	_MMPFNLIST
struct	_MM_MEMORY_CONSUMER
struct	_MM_REGION
struct	_MMFREE_POOL_ENTRY
struct	_MM_PAGED_POOL_INFO
struct	_MMPAGING_FILE

Macros
#define	MMDBG_COPY_WRITE   0x00000001
#define	MMDBG_COPY_PHYSICAL   0x00000002
#define	MMDBG_COPY_UNSAFE   0x00000004
#define	MMDBG_COPY_CACHED   0x00000008
#define	MMDBG_COPY_UNCACHED   0x00000010
#define	MMDBG_COPY_WRITE_COMBINED   0x00000020
#define	MMDBG_COPY_MAX_SIZE   0x8
#define	MI_STATIC_MEMORY_AREAS   (13)
#define	MEMORY_AREA_SECTION_VIEW   (1)
#define	MEMORY_AREA_CACHE   (2)
#define	MEMORY_AREA_OWNED_BY_ARM3   (15)
#define	MEMORY_AREA_STATIC   (0x80000000)
#define	MM_VIRTMEM_GRANULARITY   (64 * 1024)
#define	STATUS_MM_RESTART_OPERATION   ((NTSTATUS)0xD0000001)
#define	PAGE_WRITETHROUGH   (1024)
#define	PAGE_SYSTEM   (2048)
#define	SEC_PHYSICALMEMORY   (0x80000000)
#define	MM_PAGEFILE_SEGMENT   (0x1)
#define	MM_DATAFILE_SEGMENT   (0x2)
#define	MC_CACHE   (0)
#define	MC_USER   (1)
#define	MC_SYSTEM   (2)
#define	MC_MAXIMUM   (3)
#define	PAGED_POOL_MASK   1
#define	MUST_SUCCEED_POOL_MASK   2
#define	CACHE_ALIGNED_POOL_MASK   4
#define	QUOTA_POOL_MASK   8
#define	SESSION_POOL_MASK   32
#define	VERIFIER_POOL_MASK   64
#define	MAX_PAGING_FILES   (16)
#define	MM_ROUND_UP(x, s)   ((PVOID)(((ULONG_PTR)(x)+(s)-1) & ~((ULONG_PTR)(s)-1)))
#define	MM_ROUND_DOWN(x, s)   ((PVOID)(((ULONG_PTR)(x)) & ~((ULONG_PTR)(s)-1)))
#define	PAGE_FLAGS_VALID_FOR_SECTION
#define	PAGE_IS_READABLE
#define	PAGE_IS_WRITABLE
#define	PAGE_IS_EXECUTABLE
#define	PAGE_IS_WRITECOPY
#define	MM_WAIT_ENTRY   0x7ffffc00
#define	InterlockedCompareExchangePte(PointerPte, Exchange, Comperand)   InterlockedCompareExchange((PLONG)(PointerPte), Exchange, Comperand)
#define	InterlockedExchangePte(PointerPte, Value)   InterlockedExchange((PLONG)(PointerPte), Value)
#define	MA_GetStartingAddress(_MemoryArea)   ((_MemoryArea)->VadNode.StartingVpn << PAGE_SHIFT)
#define	MA_GetEndingAddress(_MemoryArea)   (((_MemoryArea)->VadNode.EndingVpn + 1) << PAGE_SHIFT)
#define	MI_SET_USAGE(x)
#define	MI_SET_PROCESS2(x)
#define	StartOfAllocation   ReadInProgress
#define	EndOfAllocation   WriteInProgress
#define	MM_FREE_POOL_SIGNATURE   'ARM3'
#define	MI_ASSERT_PFN_LOCK_HELD()   ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL)
#define	MmDeleteHyperspaceMapping(x)   MiUnmapPageInHyperSpace(HyperProcess, x, HyperIrql);

Typedefs
typedef ULONG_PTR	SWAPENTRY
typedef struct _MM_SECTION_SEGMENT	MM_SECTION_SEGMENT
typedef struct _MM_SECTION_SEGMENT *	PMM_SECTION_SEGMENT
typedef struct _MM_IMAGE_SECTION_OBJECT	MM_IMAGE_SECTION_OBJECT
typedef struct _MM_IMAGE_SECTION_OBJECT *	PMM_IMAGE_SECTION_OBJECT
typedef struct _ROS_SECTION_OBJECT	ROS_SECTION_OBJECT
typedef struct _ROS_SECTION_OBJECT *	PROS_SECTION_OBJECT
typedef struct _MEMORY_AREA	MEMORY_AREA
typedef struct _MEMORY_AREA *	PMEMORY_AREA
typedef struct _MM_RMAP_ENTRY	MM_RMAP_ENTRY
typedef struct _MM_RMAP_ENTRY *	PMM_RMAP_ENTRY
typedef enum _MI_PFN_USAGES	MI_PFN_USAGES
typedef struct _MMPFNENTRY	MMPFNENTRY
typedef struct _MMPFN	MMPFN
typedef struct _MMPFN *	PMMPFN
typedef struct _MMPFNLIST	MMPFNLIST
typedef struct _MMPFNLIST *	PMMPFNLIST
typedef struct _MM_MEMORY_CONSUMER	MM_MEMORY_CONSUMER
typedef struct _MM_MEMORY_CONSUMER *	PMM_MEMORY_CONSUMER
typedef struct _MM_REGION	MM_REGION
typedef struct _MM_REGION *	PMM_REGION
typedef struct _MMFREE_POOL_ENTRY	MMFREE_POOL_ENTRY
typedef struct _MMFREE_POOL_ENTRY *	PMMFREE_POOL_ENTRY
typedef struct _MM_PAGED_POOL_INFO	MM_PAGED_POOL_INFO
typedef struct _MM_PAGED_POOL_INFO *	PMM_PAGED_POOL_INFO
typedef struct _MMPAGING_FILE	MMPAGING_FILE
typedef struct _MMPAGING_FILE *	PMMPAGING_FILE
typedef VOID(*	PMM_ALTER_REGION_FUNC) (PMMSUPPORT AddressSpace, PVOID BaseAddress, SIZE_T Length, ULONG OldType, ULONG OldProtect, ULONG NewType, ULONG NewProtect)
typedef VOID(*	PMM_FREE_PAGE_FUNC) (PVOID Context, PMEMORY_AREA MemoryArea, PVOID Address, PFN_NUMBER Page, SWAPENTRY SwapEntry, BOOLEAN Dirty)

Enumerations
enum	_MI_PFN_USAGES {   MI_USAGE_NOT_SET = 0, MI_USAGE_PAGED_POOL, MI_USAGE_NONPAGED_POOL, MI_USAGE_NONPAGED_POOL_EXPANSION,   MI_USAGE_KERNEL_STACK, MI_USAGE_KERNEL_STACK_EXPANSION, MI_USAGE_SYSTEM_PTE, MI_USAGE_VAD,   MI_USAGE_PEB_TEB, MI_USAGE_SECTION, MI_USAGE_PAGE_TABLE, MI_USAGE_PAGE_DIRECTORY,   MI_USAGE_LEGACY_PAGE_DIRECTORY, MI_USAGE_DRIVER_PAGE, MI_USAGE_CONTINOUS_ALLOCATION, MI_USAGE_MDL,   MI_USAGE_DEMAND_ZERO, MI_USAGE_ZERO_LOOP, MI_USAGE_CACHE, MI_USAGE_PFN_DATABASE,   MI_USAGE_BOOT_DRIVER, MI_USAGE_INIT_MEMORY, MI_USAGE_FREE_PAGE }

Functions
NTSTATUS NTAPI	MmDbgCopyMemory (IN ULONG64 Address, IN PVOID Buffer, IN ULONG Size, IN ULONG Flags)
BOOLEAN NTAPI	MmIsSessionAddress (IN PVOID Address)
ULONG NTAPI	MmGetSessionId (IN PEPROCESS Process)
ULONG NTAPI	MmGetSessionIdEx (IN PEPROCESS Process)
NTSTATUS NTAPI	MmCreateMemoryArea (PMMSUPPORT AddressSpace, ULONG Type, PVOID *BaseAddress, SIZE_T Length, ULONG Protection, PMEMORY_AREA *Result, ULONG AllocationFlags, ULONG AllocationGranularity)
PMEMORY_AREA NTAPI	MmLocateMemoryAreaByAddress (PMMSUPPORT AddressSpace, PVOID Address)
NTSTATUS NTAPI	MmFreeMemoryArea (PMMSUPPORT AddressSpace, PMEMORY_AREA MemoryArea, PMM_FREE_PAGE_FUNC FreePage, PVOID FreePageContext)
VOID NTAPI	MiRosCleanupMemoryArea (PEPROCESS Process, PMMVAD Vad)
PMEMORY_AREA NTAPI	MmLocateMemoryAreaByRegion (PMMSUPPORT AddressSpace, PVOID Address, SIZE_T Length)
PVOID NTAPI	MmFindGap (PMMSUPPORT AddressSpace, SIZE_T Length, ULONG_PTR Granularity, BOOLEAN TopDown)
VOID NTAPI	MiRosCheckMemoryAreas (PMMSUPPORT AddressSpace)
VOID NTAPI	MiCheckAllProcessMemoryAreas (VOID)
INIT_FUNCTION VOID NTAPI	MiInitializeNonPagedPool (VOID)
PVOID NTAPI	MiAllocatePoolPages (IN POOL_TYPE PoolType, IN SIZE_T SizeInBytes)
POOL_TYPE NTAPI	MmDeterminePoolType (IN PVOID VirtualAddress)
ULONG NTAPI	MiFreePoolPages (IN PVOID StartingAddress)
BOOLEAN NTAPI	MiRaisePoolQuota (IN POOL_TYPE PoolType, IN ULONG CurrentMaxQuota, OUT PULONG NewMaxQuota)
VOID NTAPI	MmBuildMdlFromPages (PMDL Mdl, PPFN_NUMBER Pages)
VOID NTAPI	MmInit1 (VOID)
INIT_FUNCTION BOOLEAN NTAPI	MmInitSystem (IN ULONG Phase, IN PLOADER_PARAMETER_BLOCK LoaderBlock)
SWAPENTRY NTAPI	MmAllocSwapPage (VOID)
VOID NTAPI	MmFreeSwapPage (SWAPENTRY Entry)
INIT_FUNCTION VOID NTAPI	MmInitPagingFile (VOID)
BOOLEAN NTAPI	MmIsFileObjectAPagingFile (PFILE_OBJECT FileObject)
NTSTATUS NTAPI	MmReadFromSwapPage (SWAPENTRY SwapEntry, PFN_NUMBER Page)
NTSTATUS NTAPI	MmWriteToSwapPage (SWAPENTRY SwapEntry, PFN_NUMBER Page)
VOID NTAPI	MmShowOutOfSpaceMessagePagingFile (VOID)
NTSTATUS NTAPI	MiReadPageFile (_In_ PFN_NUMBER Page, _In_ ULONG PageFileIndex, _In_ ULONG_PTR PageFileOffset)
NTSTATUS NTAPI	MmInitializeProcessAddressSpace (IN PEPROCESS Process, IN PEPROCESS Clone OPTIONAL, IN PVOID Section OPTIONAL, IN OUT PULONG Flags, IN POBJECT_NAME_INFORMATION *AuditName OPTIONAL)
NTSTATUS NTAPI	MmCreatePeb (IN PEPROCESS Process, IN PINITIAL_PEB InitialPeb, OUT PPEB *BasePeb)
NTSTATUS NTAPI	MmCreateTeb (IN PEPROCESS Process, IN PCLIENT_ID ClientId, IN PINITIAL_TEB InitialTeb, OUT PTEB *BaseTeb)
VOID NTAPI	MmDeleteTeb (struct _EPROCESS *Process, PTEB Teb)
VOID NTAPI	MmCleanProcessAddressSpace (IN PEPROCESS Process)
NTSTATUS NTAPI	MmDeleteProcessAddressSpace (IN PEPROCESS Process)
ULONG NTAPI	MmGetSessionLocaleId (VOID)
NTSTATUS NTAPI	MmSetMemoryPriorityProcess (IN PEPROCESS Process, IN UCHAR MemoryPriority)
NTSTATUS NTAPI	MmPageFault (ULONG Cs, PULONG Eip, PULONG Eax, ULONG Cr2, ULONG ErrorCode)
VOID NTAPI	MiInitializeSpecialPool (VOID)
BOOLEAN NTAPI	MmUseSpecialPool (IN SIZE_T NumberOfBytes, IN ULONG Tag)
BOOLEAN NTAPI	MmIsSpecialPoolAddress (IN PVOID P)
BOOLEAN NTAPI	MmIsSpecialPoolAddressFree (IN PVOID P)
PVOID NTAPI	MmAllocateSpecialPool (IN SIZE_T NumberOfBytes, IN ULONG Tag, IN POOL_TYPE PoolType, IN ULONG SpecialType)
VOID NTAPI	MmFreeSpecialPool (IN PVOID P)
NTSTATUS NTAPI	MmAccessFault (IN ULONG FaultCode, IN PVOID Address, IN KPROCESSOR_MODE Mode, IN PVOID TrapInformation)
NTSTATUS NTAPI	MiCopyFromUserPage (PFN_NUMBER DestPage, const VOID *SrcAddress)
PVOID NTAPI	MmCreateKernelStack (BOOLEAN GuiStack, UCHAR Node)
VOID NTAPI	MmDeleteKernelStack (PVOID Stack, BOOLEAN GuiStack)
INIT_FUNCTION VOID NTAPI	MmInitializeMemoryConsumer (ULONG Consumer, NTSTATUS(*Trim)(ULONG Target, ULONG Priority, PULONG NrFreed))
INIT_FUNCTION VOID NTAPI	MmInitializeBalancer (ULONG NrAvailablePages, ULONG NrSystemPages)
NTSTATUS NTAPI	MmReleasePageMemoryConsumer (ULONG Consumer, PFN_NUMBER Page)
NTSTATUS NTAPI	MmRequestPageMemoryConsumer (ULONG Consumer, BOOLEAN MyWait, PPFN_NUMBER AllocatedPage)
INIT_FUNCTION VOID NTAPI	MiInitBalancerThread (VOID)
VOID NTAPI	MmRebalanceMemoryConsumers (VOID)
VOID NTAPI	MmSetRmapListHeadPage (PFN_NUMBER Page, struct _MM_RMAP_ENTRY *ListHead)
struct _MM_RMAP_ENTRY *NTAPI	MmGetRmapListHeadPage (PFN_NUMBER Page)
VOID NTAPI	MmInsertRmap (PFN_NUMBER Page, struct _EPROCESS *Process, PVOID Address)
VOID NTAPI	MmDeleteAllRmaps (PFN_NUMBER Page, PVOID Context, VOID(*DeleteMapping)(PVOID Context, struct _EPROCESS *Process, PVOID Address))
VOID NTAPI	MmDeleteRmap (PFN_NUMBER Page, struct _EPROCESS *Process, PVOID Address)
INIT_FUNCTION VOID NTAPI	MmInitializeRmapList (VOID)
VOID NTAPI	MmSetCleanAllRmaps (PFN_NUMBER Page)
VOID NTAPI	MmSetDirtyAllRmaps (PFN_NUMBER Page)
BOOLEAN NTAPI	MmIsDirtyPageRmap (PFN_NUMBER Page)
NTSTATUS NTAPI	MmPageOutPhysicalAddress (PFN_NUMBER Page)
FORCEINLINE KIRQL	MiAcquirePfnLock (VOID)
FORCEINLINE VOID	MiReleasePfnLock (_In_ KIRQL OldIrql)
FORCEINLINE VOID	MiAcquirePfnLockAtDpcLevel (VOID)
FORCEINLINE VOID	MiReleasePfnLockFromDpcLevel (VOID)
FORCEINLINE PMMPFN	MiGetPfnEntry (IN PFN_NUMBER Pfn)
FORCEINLINE PFN_NUMBER	MiGetPfnEntryIndex (IN PMMPFN Pfn1)
PFN_NUMBER NTAPI	MmGetLRUNextUserPage (PFN_NUMBER PreviousPage)
PFN_NUMBER NTAPI	MmGetLRUFirstUserPage (VOID)
VOID NTAPI	MmInsertLRULastUserPage (PFN_NUMBER Page)
VOID NTAPI	MmRemoveLRUUserPage (PFN_NUMBER Page)
VOID NTAPI	MmDumpArmPfnDatabase (IN BOOLEAN StatusOnly)
VOID NTAPI	MmZeroPageThread (VOID)
PVOID NTAPI	MiMapPageInHyperSpace (IN PEPROCESS Process, IN PFN_NUMBER Page, IN PKIRQL OldIrql)
VOID NTAPI	MiUnmapPageInHyperSpace (IN PEPROCESS Process, IN PVOID Address, IN KIRQL OldIrql)
PVOID NTAPI	MiMapPagesInZeroSpace (IN PMMPFN Pfn1, IN PFN_NUMBER NumberOfPages)
VOID NTAPI	MiUnmapPagesInZeroSpace (IN PVOID VirtualAddress, IN PFN_NUMBER NumberOfPages)
FORCEINLINE PVOID	MmCreateHyperspaceMapping (IN PFN_NUMBER Page)
NTSTATUS NTAPI	MmCreateVirtualMapping (struct _EPROCESS *Process, PVOID Address, ULONG flProtect, PPFN_NUMBER Pages, ULONG PageCount)
NTSTATUS NTAPI	MmCreateVirtualMappingUnsafe (struct _EPROCESS *Process, PVOID Address, ULONG flProtect, PPFN_NUMBER Pages, ULONG PageCount)
ULONG NTAPI	MmGetPageProtect (struct _EPROCESS *Process, PVOID Address)
VOID NTAPI	MmSetPageProtect (struct _EPROCESS *Process, PVOID Address, ULONG flProtect)
BOOLEAN NTAPI	MmIsPagePresent (struct _EPROCESS *Process, PVOID Address)
BOOLEAN NTAPI	MmIsDisabledPage (struct _EPROCESS *Process, PVOID Address)
INIT_FUNCTION VOID NTAPI	MmInitGlobalKernelPageDirectory (VOID)
VOID NTAPI	MmGetPageFileMapping (struct _EPROCESS *Process, PVOID Address, SWAPENTRY *SwapEntry)
VOID NTAPI	MmDeletePageFileMapping (struct _EPROCESS *Process, PVOID Address, SWAPENTRY *SwapEntry)
NTSTATUS NTAPI	MmCreatePageFileMapping (struct _EPROCESS *Process, PVOID Address, SWAPENTRY SwapEntry)
BOOLEAN NTAPI	MmIsPageSwapEntry (struct _EPROCESS *Process, PVOID Address)
VOID NTAPI	MmSetDirtyPage (struct _EPROCESS *Process, PVOID Address)
PFN_NUMBER NTAPI	MmAllocPage (ULONG Consumer)
VOID NTAPI	MmDereferencePage (PFN_NUMBER Page)
VOID NTAPI	MmReferencePage (PFN_NUMBER Page)
ULONG NTAPI	MmGetReferenceCountPage (PFN_NUMBER Page)
BOOLEAN NTAPI	MmIsPageInUse (PFN_NUMBER Page)
VOID NTAPI	MmSetSavedSwapEntryPage (PFN_NUMBER Page, SWAPENTRY SavedSwapEntry)
SWAPENTRY NTAPI	MmGetSavedSwapEntryPage (PFN_NUMBER Page)
VOID NTAPI	MmSetCleanPage (struct _EPROCESS *Process, PVOID Address)
VOID NTAPI	MmDeletePageTable (struct _EPROCESS *Process, PVOID Address)
PFN_NUMBER NTAPI	MmGetPfnForProcess (struct _EPROCESS *Process, PVOID Address)
BOOLEAN NTAPI	MmCreateProcessAddressSpace (IN ULONG MinWs, IN PEPROCESS Dest, IN PULONG_PTR DirectoryTableBase)
INIT_FUNCTION NTSTATUS NTAPI	MmInitializeHandBuiltProcess (IN PEPROCESS Process, IN PULONG_PTR DirectoryTableBase)
INIT_FUNCTION NTSTATUS NTAPI	MmInitializeHandBuiltProcess2 (IN PEPROCESS Process)
NTSTATUS NTAPI	MmSetExecuteOptions (IN ULONG ExecuteOptions)
NTSTATUS NTAPI	MmGetExecuteOptions (IN PULONG ExecuteOptions)
VOID NTAPI	MmDeleteVirtualMapping (struct _EPROCESS *Process, PVOID Address, BOOLEAN *WasDirty, PPFN_NUMBER Page)
BOOLEAN NTAPI	MmIsDirtyPage (struct _EPROCESS *Process, PVOID Address)
NTSTATUS	MmTrimUserMemory (ULONG Target, ULONG Priority, PULONG NrFreedPages)
NTSTATUS NTAPI	MmAlterRegion (PMMSUPPORT AddressSpace, PVOID BaseAddress, PLIST_ENTRY RegionListHead, PVOID StartAddress, SIZE_T Length, ULONG NewType, ULONG NewProtect, PMM_ALTER_REGION_FUNC AlterFunc)
VOID NTAPI	MmInitializeRegion (PLIST_ENTRY RegionListHead, SIZE_T Length, ULONG Type, ULONG Protect)
PMM_REGION NTAPI	MmFindRegion (PVOID BaseAddress, PLIST_ENTRY RegionListHead, PVOID Address, PVOID *RegionBaseAddress)
VOID NTAPI	MmGetImageInformation (OUT PSECTION_IMAGE_INFORMATION ImageInformation)
PFILE_OBJECT NTAPI	MmGetFileObjectForSection (IN PVOID Section)
NTSTATUS NTAPI	MmGetFileNameForAddress (IN PVOID Address, OUT PUNICODE_STRING ModuleName)
NTSTATUS NTAPI	MmGetFileNameForSection (IN PVOID Section, OUT POBJECT_NAME_INFORMATION *ModuleName)
NTSTATUS NTAPI	MmQuerySectionView (PMEMORY_AREA MemoryArea, PVOID Address, PMEMORY_BASIC_INFORMATION Info, PSIZE_T ResultLength)
NTSTATUS NTAPI	MmProtectSectionView (PMMSUPPORT AddressSpace, PMEMORY_AREA MemoryArea, PVOID BaseAddress, SIZE_T Length, ULONG Protect, PULONG OldProtect)
INIT_FUNCTION NTSTATUS NTAPI	MmInitSectionImplementation (VOID)
NTSTATUS NTAPI	MmNotPresentFaultSectionView (PMMSUPPORT AddressSpace, MEMORY_AREA *MemoryArea, PVOID Address, BOOLEAN Locked)
NTSTATUS NTAPI	MmPageOutSectionView (PMMSUPPORT AddressSpace, PMEMORY_AREA MemoryArea, PVOID Address, ULONG_PTR Entry)
INIT_FUNCTION NTSTATUS NTAPI	MmCreatePhysicalMemorySection (VOID)
NTSTATUS NTAPI	MmAccessFaultSectionView (PMMSUPPORT AddressSpace, MEMORY_AREA *MemoryArea, PVOID Address)
VOID NTAPI	MmFreeSectionSegments (PFILE_OBJECT FileObject)
INIT_FUNCTION VOID NTAPI	MiReloadBootLoadedDrivers (IN PLOADER_PARAMETER_BLOCK LoaderBlock)
INIT_FUNCTION BOOLEAN NTAPI	MiInitializeLoadedModuleList (IN PLOADER_PARAMETER_BLOCK LoaderBlock)
BOOLEAN NTAPI	MmChangeKernelResourceSectionProtection (IN ULONG_PTR ProtectionMask)
VOID NTAPI	MmMakeKernelResourceSectionWritable (VOID)
NTSTATUS NTAPI	MmLoadSystemImage (IN PUNICODE_STRING FileName, IN PUNICODE_STRING NamePrefix OPTIONAL, IN PUNICODE_STRING LoadedName OPTIONAL, IN ULONG Flags, OUT PVOID *ModuleObject, OUT PVOID *ImageBaseAddress)
NTSTATUS NTAPI	MmUnloadSystemImage (IN PVOID ImageHandle)
NTSTATUS NTAPI	MmCheckSystemImage (IN HANDLE ImageHandle, IN BOOLEAN PurgeSection)
NTSTATUS NTAPI	MmCallDllInitialize (IN PLDR_DATA_TABLE_ENTRY LdrEntry, IN PLIST_ENTRY ListHead)
NTSTATUS NTAPI	MmGrowKernelStack (IN PVOID StackPointer)
FORCEINLINE VOID	MmLockAddressSpace (PMMSUPPORT AddressSpace)
FORCEINLINE VOID	MmUnlockAddressSpace (PMMSUPPORT AddressSpace)
FORCEINLINE PEPROCESS	MmGetAddressSpaceOwner (IN PMMSUPPORT AddressSpace)
FORCEINLINE PMMSUPPORT	MmGetCurrentAddressSpace (VOID)
FORCEINLINE PMMSUPPORT	MmGetKernelAddressSpace (VOID)
VOID NTAPI	ExpCheckPoolAllocation (PVOID P, POOL_TYPE PoolType, ULONG Tag)
VOID NTAPI	ExReturnPoolQuota (IN PVOID P)
NTSTATUS NTAPI	MmAdjustWorkingSetSize (IN SIZE_T WorkingSetMinimumInBytes, IN SIZE_T WorkingSetMaximumInBytes, IN ULONG SystemCache, IN BOOLEAN IncreaseOkay)
	_IRQL_requires_max_ (APC_LEVEL) NTSTATUS NTAPI MmAttachSession(_Inout_ PVOID SessionEntry
VOID NTAPI	MmQuitNextSession (_Inout_ PVOID SessionEntry)
PVOID NTAPI	MmGetSessionById (_In_ ULONG SessionId)
VOID	MmShutdownSystem (IN ULONG Phase)
NTSTATUS NTAPI	MmCopyVirtualMemory (IN PEPROCESS SourceProcess, IN PVOID SourceAddress, IN PEPROCESS TargetProcess, OUT PVOID TargetAddress, IN SIZE_T BufferSize, IN KPROCESSOR_MODE PreviousMode, OUT PSIZE_T ReturnSize)

Variables
PMMSUPPORT	MmKernelAddressSpace
PFN_COUNT	MiFreeSwapPages
PFN_COUNT	MiUsedSwapPages
PFN_COUNT	MmNumberOfPhysicalPages
UCHAR	MmDisablePagingExecutive
PFN_NUMBER	MmLowestPhysicalPage
PFN_NUMBER	MmHighestPhysicalPage
PFN_NUMBER	MmAvailablePages
PFN_NUMBER	MmResidentAvailablePages
ULONG	MmThrottleTop
ULONG	MmThrottleBottom
LIST_ENTRY	MmLoadedUserImageList
KMUTANT	MmSystemLoadLock
ULONG	MmNumberOfPagingFiles
PVOID	MmUnloadedDrivers
PVOID	MmLastUnloadedDrivers
PVOID	MmTriageActionTaken
PVOID	KernelVerifier
MM_DRIVER_VERIFIER_DATA	MmVerifierData
SIZE_T	MmTotalCommitLimit
SIZE_T	MmTotalCommittedPages
SIZE_T	MmSharedCommit
SIZE_T	MmDriverCommit
SIZE_T	MmProcessCommit
SIZE_T	MmPagedPoolCommit
SIZE_T	MmPeakCommitment
SIZE_T	MmtotalCommitLimitMaximum
PVOID	MiDebugMapping
PMMPTE	MmDebugPte
PMMPFN	MmPfnDatabase
MMPFNLIST	MmZeroedPageListHead
MMPFNLIST	MmFreePageListHead
MMPFNLIST	MmStandbyPageListHead
MMPFNLIST	MmModifiedPageListHead
MMPFNLIST	MmModifiedNoWritePageListHead
MM_MEMORY_CONSUMER	MiMemoryConsumers [MC_MAXIMUM]
PMMPAGING_FILE	MmPagingFile [MAX_PAGING_FILES]
PEPROCESS	HyperProcess
KIRQL	HyperIrql
_Out_ PKAPC_STATE	ApcState

Macro Definition Documentation

CACHE_ALIGNED_POOL_MASK

#define CACHE_ALIGNED_POOL_MASK   4

Definition at line 100 of file mm.h.

EndOfAllocation

#define EndOfAllocation   WriteInProgress

Definition at line 288 of file mm.h.

InterlockedCompareExchangePte

#define InterlockedCompareExchangePte	(		PointerPte,
			Exchange,
			Comperand
	)		InterlockedCompareExchange((PLONG)(PointerPte), Exchange, Comperand)

Definition at line 154 of file mm.h.

InterlockedExchangePte

#define InterlockedExchangePte	(		PointerPte,
			Value
	)		InterlockedExchange((PLONG)(PointerPte), Value)

Definition at line 157 of file mm.h.

MA_GetEndingAddress

#define MA_GetEndingAddress

(

_MemoryArea

)

(((_MemoryArea)->VadNode.EndingVpn + 1) << PAGE_SHIFT)

Definition at line 208 of file mm.h.

MA_GetStartingAddress

#define MA_GetStartingAddress

(

_MemoryArea

)

((_MemoryArea)->VadNode.StartingVpn << PAGE_SHIFT)

Definition at line 207 of file mm.h.

MAX_PAGING_FILES

#define MAX_PAGING_FILES   (16)

Definition at line 105 of file mm.h.

MC_CACHE

#define MC_CACHE   (0)

Definition at line 93 of file mm.h.

MC_MAXIMUM

#define MC_MAXIMUM   (3)

Definition at line 96 of file mm.h.

MC_SYSTEM

#define MC_SYSTEM   (2)

Definition at line 95 of file mm.h.

MC_USER

#define MC_USER   (1)

Definition at line 94 of file mm.h.

MEMORY_AREA_CACHE

#define MEMORY_AREA_CACHE   (2)

Definition at line 72 of file mm.h.

MEMORY_AREA_OWNED_BY_ARM3

#define MEMORY_AREA_OWNED_BY_ARM3   (15)

Definition at line 73 of file mm.h.

MEMORY_AREA_SECTION_VIEW

#define MEMORY_AREA_SECTION_VIEW   (1)

Definition at line 71 of file mm.h.

MEMORY_AREA_STATIC

#define MEMORY_AREA_STATIC   (0x80000000)

Definition at line 74 of file mm.h.

MI_ASSERT_PFN_LOCK_HELD

#define MI_ASSERT_PFN_LOCK_HELD

(

)

ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL)

Definition at line 936 of file mm.h.

MI_SET_PROCESS2

#define MI_SET_PROCESS2

(

x

)

Definition at line 254 of file mm.h.

MI_SET_USAGE

#define MI_SET_USAGE

(

x

)

Definition at line 253 of file mm.h.

MI_STATIC_MEMORY_AREAS

#define MI_STATIC_MEMORY_AREAS   (13)

Definition at line 68 of file mm.h.

MM_DATAFILE_SEGMENT

#define MM_DATAFILE_SEGMENT   (0x2)

Definition at line 91 of file mm.h.

MM_FREE_POOL_SIGNATURE

#define MM_FREE_POOL_SIGNATURE   'ARM3'

Definition at line 409 of file mm.h.

MM_PAGEFILE_SEGMENT

#define MM_PAGEFILE_SEGMENT   (0x1)

Definition at line 90 of file mm.h.

MM_ROUND_DOWN

#define MM_ROUND_DOWN	(		x,
			s
	)		((PVOID)(((ULONG_PTR)(x)) & ~((ULONG_PTR)(s)-1)))

Definition at line 111 of file mm.h.

MM_ROUND_UP

#define MM_ROUND_UP	(		x,
			s
	)		((PVOID)(((ULONG_PTR)(x)+(s)-1) & ~((ULONG_PTR)(s)-1)))

Definition at line 108 of file mm.h.

MM_VIRTMEM_GRANULARITY

#define MM_VIRTMEM_GRANULARITY   (64 * 1024)

Definition at line 78 of file mm.h.

MM_WAIT_ENTRY

#define MM_WAIT_ENTRY   0x7ffffc00

Definition at line 152 of file mm.h.

MMDBG_COPY_CACHED

#define MMDBG_COPY_CACHED   0x00000008

Definition at line 56 of file mm.h.

MMDBG_COPY_MAX_SIZE

#define MMDBG_COPY_MAX_SIZE   0x8

Definition at line 63 of file mm.h.

MMDBG_COPY_PHYSICAL

#define MMDBG_COPY_PHYSICAL   0x00000002

Definition at line 54 of file mm.h.

MMDBG_COPY_UNCACHED

#define MMDBG_COPY_UNCACHED   0x00000010

Definition at line 57 of file mm.h.

MMDBG_COPY_UNSAFE

#define MMDBG_COPY_UNSAFE   0x00000004

Definition at line 55 of file mm.h.

MMDBG_COPY_WRITE

#define MMDBG_COPY_WRITE   0x00000001

Definition at line 53 of file mm.h.

MMDBG_COPY_WRITE_COMBINED

#define MMDBG_COPY_WRITE_COMBINED   0x00000020

Definition at line 58 of file mm.h.

MmDeleteHyperspaceMapping

#define MmDeleteHyperspaceMapping

(

x

)

MiUnmapPageInHyperSpace(HyperProcess, x, HyperIrql);

Definition at line 1034 of file mm.h.

MUST_SUCCEED_POOL_MASK

#define MUST_SUCCEED_POOL_MASK   2

Definition at line 99 of file mm.h.

PAGE_FLAGS_VALID_FOR_SECTION

#define PAGE_FLAGS_VALID_FOR_SECTION

Value:

(PAGE_READONLY | \
     PAGE_READWRITE | \
     PAGE_WRITECOPY | \
     PAGE_EXECUTE | \
     PAGE_EXECUTE_READ | \
     PAGE_EXECUTE_READWRITE | \
     PAGE_EXECUTE_WRITECOPY | \
     PAGE_NOACCESS | \
     PAGE_NOCACHE)

Definition at line 114 of file mm.h.

PAGE_IS_EXECUTABLE

#define PAGE_IS_EXECUTABLE

Value:

(PAGE_EXECUTE | \
    PAGE_EXECUTE_READ | \
    PAGE_EXECUTE_READWRITE | \
    PAGE_EXECUTE_WRITECOPY)

Definition at line 139 of file mm.h.

PAGE_IS_READABLE

#define PAGE_IS_READABLE

Value:

(PAGE_READONLY | \
    PAGE_READWRITE | \
    PAGE_WRITECOPY | \
    PAGE_EXECUTE_READ | \
    PAGE_EXECUTE_READWRITE | \
    PAGE_EXECUTE_WRITECOPY)

Definition at line 125 of file mm.h.

PAGE_IS_WRITABLE

#define PAGE_IS_WRITABLE

Value:

(PAGE_READWRITE | \
    PAGE_WRITECOPY | \
    PAGE_EXECUTE_READWRITE | \
    PAGE_EXECUTE_WRITECOPY)

Definition at line 133 of file mm.h.

PAGE_IS_WRITECOPY

#define PAGE_IS_WRITECOPY

Value:

(PAGE_WRITECOPY | \
    PAGE_EXECUTE_WRITECOPY)

Definition at line 145 of file mm.h.

PAGE_SYSTEM

#define PAGE_SYSTEM   (2048)

Definition at line 86 of file mm.h.

PAGE_WRITETHROUGH

#define PAGE_WRITETHROUGH   (1024)

Definition at line 85 of file mm.h.

PAGED_POOL_MASK

#define PAGED_POOL_MASK   1

Definition at line 98 of file mm.h.

QUOTA_POOL_MASK

#define QUOTA_POOL_MASK   8

Definition at line 101 of file mm.h.

SEC_PHYSICALMEMORY

#define SEC_PHYSICALMEMORY   (0x80000000)

Definition at line 88 of file mm.h.

SESSION_POOL_MASK

#define SESSION_POOL_MASK   32

Definition at line 102 of file mm.h.

StartOfAllocation

#define StartOfAllocation   ReadInProgress

Definition at line 287 of file mm.h.

STATUS_MM_RESTART_OPERATION

#define STATUS_MM_RESTART_OPERATION   ((NTSTATUS)0xD0000001)

Definition at line 80 of file mm.h.

VERIFIER_POOL_MASK

#define VERIFIER_POOL_MASK   64

Definition at line 103 of file mm.h.

Typedef Documentation

MEMORY_AREA

typedef struct _MEMORY_AREA MEMORY_AREA

MI_PFN_USAGES

typedef enum _MI_PFN_USAGES MI_PFN_USAGES

MM_IMAGE_SECTION_OBJECT

typedef struct _MM_IMAGE_SECTION_OBJECT MM_IMAGE_SECTION_OBJECT

MM_MEMORY_CONSUMER

typedef struct _MM_MEMORY_CONSUMER MM_MEMORY_CONSUMER

MM_PAGED_POOL_INFO

typedef struct _MM_PAGED_POOL_INFO MM_PAGED_POOL_INFO

MM_REGION

typedef struct _MM_REGION MM_REGION

MM_RMAP_ENTRY

typedef struct _MM_RMAP_ENTRY MM_RMAP_ENTRY

MM_SECTION_SEGMENT

typedef struct _MM_SECTION_SEGMENT MM_SECTION_SEGMENT

MMFREE_POOL_ENTRY

typedef struct _MMFREE_POOL_ENTRY MMFREE_POOL_ENTRY

MMPAGING_FILE

typedef struct _MMPAGING_FILE MMPAGING_FILE

MMPFN

typedef struct _MMPFN MMPFN

MMPFNENTRY

typedef struct _MMPFNENTRY MMPFNENTRY

MMPFNLIST

typedef struct _MMPFNLIST MMPFNLIST

PMEMORY_AREA

typedef struct _MEMORY_AREA * PMEMORY_AREA

PMM_ALTER_REGION_FUNC

typedef VOID(* PMM_ALTER_REGION_FUNC) (PMMSUPPORT AddressSpace, PVOID BaseAddress, SIZE_T Length, ULONG OldType, ULONG OldProtect, ULONG NewType, ULONG NewProtect)

Definition at line 444 of file mm.h.

PMM_FREE_PAGE_FUNC

typedef VOID(* PMM_FREE_PAGE_FUNC) (PVOID Context, PMEMORY_AREA MemoryArea, PVOID Address, PFN_NUMBER Page, SWAPENTRY SwapEntry, BOOLEAN Dirty)

Definition at line 455 of file mm.h.

PMM_IMAGE_SECTION_OBJECT

typedef struct _MM_IMAGE_SECTION_OBJECT * PMM_IMAGE_SECTION_OBJECT

PMM_MEMORY_CONSUMER

typedef struct _MM_MEMORY_CONSUMER * PMM_MEMORY_CONSUMER

PMM_PAGED_POOL_INFO

typedef struct _MM_PAGED_POOL_INFO * PMM_PAGED_POOL_INFO

PMM_REGION

typedef struct _MM_REGION * PMM_REGION

PMM_RMAP_ENTRY

typedef struct _MM_RMAP_ENTRY * PMM_RMAP_ENTRY

PMM_SECTION_SEGMENT

typedef struct _MM_SECTION_SEGMENT * PMM_SECTION_SEGMENT

PMMFREE_POOL_ENTRY

typedef struct _MMFREE_POOL_ENTRY * PMMFREE_POOL_ENTRY

PMMPAGING_FILE

typedef struct _MMPAGING_FILE * PMMPAGING_FILE

PMMPFN

typedef struct _MMPFN * PMMPFN

PMMPFNLIST

typedef struct _MMPFNLIST * PMMPFNLIST

PROS_SECTION_OBJECT

typedef struct _ROS_SECTION_OBJECT * PROS_SECTION_OBJECT

ROS_SECTION_OBJECT

typedef struct _ROS_SECTION_OBJECT ROS_SECTION_OBJECT

SWAPENTRY

typedef ULONG_PTR SWAPENTRY

Definition at line 47 of file mm.h.

Enumeration Type Documentation

_MI_PFN_USAGES

enum _MI_PFN_USAGES

Enumerator
MI_USAGE_NOT_SET
MI_USAGE_PAGED_POOL
MI_USAGE_NONPAGED_POOL
MI_USAGE_NONPAGED_POOL_EXPANSION
MI_USAGE_KERNEL_STACK
MI_USAGE_KERNEL_STACK_EXPANSION
MI_USAGE_SYSTEM_PTE
MI_USAGE_VAD
MI_USAGE_PEB_TEB
MI_USAGE_SECTION
MI_USAGE_PAGE_TABLE
MI_USAGE_PAGE_DIRECTORY
MI_USAGE_LEGACY_PAGE_DIRECTORY
MI_USAGE_DRIVER_PAGE
MI_USAGE_CONTINOUS_ALLOCATION
MI_USAGE_MDL
MI_USAGE_DEMAND_ZERO
MI_USAGE_ZERO_LOOP
MI_USAGE_CACHE
MI_USAGE_PFN_DATABASE
MI_USAGE_BOOT_DRIVER
MI_USAGE_INIT_MEMORY
MI_USAGE_FREE_PAGE

Definition at line 257 of file mm.h.

{
    MI_USAGE_NOT_SET = 0,
    MI_USAGE_PAGED_POOL,
    MI_USAGE_NONPAGED_POOL,
    MI_USAGE_NONPAGED_POOL_EXPANSION,
    MI_USAGE_KERNEL_STACK,
    MI_USAGE_KERNEL_STACK_EXPANSION,
    MI_USAGE_SYSTEM_PTE,
    MI_USAGE_VAD,
    MI_USAGE_PEB_TEB,
    MI_USAGE_SECTION,
    MI_USAGE_PAGE_TABLE,
    MI_USAGE_PAGE_DIRECTORY,
    MI_USAGE_LEGACY_PAGE_DIRECTORY,
    MI_USAGE_DRIVER_PAGE,
    MI_USAGE_CONTINOUS_ALLOCATION,
    MI_USAGE_MDL,
    MI_USAGE_DEMAND_ZERO,
    MI_USAGE_ZERO_LOOP,
    MI_USAGE_CACHE,
    MI_USAGE_PFN_DATABASE,
    MI_USAGE_BOOT_DRIVER,
    MI_USAGE_INIT_MEMORY,
    MI_USAGE_FREE_PAGE
} MI_PFN_USAGES;

Function Documentation

_IRQL_requires_max_()

_IRQL_requires_max_

(

APC_LEVEL

)

Definition at line 197 of file cddata.c.

{
    THREAD_CONTEXT ThreadContext = {0};
    PIRP_CONTEXT IrpContext = NULL;
    BOOLEAN Wait;

#ifdef CD_SANITY
    PVOID PreviousTopLevel;
#endif

    NTSTATUS Status;

#if DBG

    KIRQL SaveIrql = KeGetCurrentIrql();

#endif

    ASSERT_OPTIONAL_IRP( Irp );

    UNREFERENCED_PARAMETER( DeviceObject );

    FsRtlEnterFileSystem();

#ifdef CD_SANITY
    PreviousTopLevel = IoGetTopLevelIrp();
#endif

    //
    //  Loop until this request has been completed or posted.
    //

    do {

        //
        //  Use a try-except to handle the exception cases.
        //

        _SEH2_TRY {

            //
            //  If the IrpContext is NULL then this is the first pass through
            //  this loop.
            //

            if (IrpContext == NULL) {

                //
                //  Decide if this request is waitable an allocate the IrpContext.
                //  If the file object in the stack location is NULL then this
                //  is a mount which is always waitable.  Otherwise we look at
                //  the file object flags.
                //

                if (IoGetCurrentIrpStackLocation( Irp )->FileObject == NULL) {

                    Wait = TRUE;

                } else {

                    Wait = CanFsdWait( Irp );
                }

                IrpContext = CdCreateIrpContext( Irp, Wait );

                //
                //  Update the thread context information.
                //

                CdSetThreadContext( IrpContext, &ThreadContext );

#ifdef CD_SANITY
                NT_ASSERT( !CdTestTopLevel ||
                        SafeNodeType( IrpContext->TopLevel ) == CDFS_NTC_IRP_CONTEXT );
#endif

            //
            //  Otherwise cleanup the IrpContext for the retry.
            //

            } else {

                //
                //  Set the MORE_PROCESSING flag to make sure the IrpContext
                //  isn't inadvertently deleted here.  Then cleanup the
                //  IrpContext to perform the retry.
                //

                SetFlag( IrpContext->Flags, IRP_CONTEXT_FLAG_MORE_PROCESSING );
                CdCleanupIrpContext( IrpContext, FALSE );
            }

            //
            //  Case on the major irp code.
            //

            switch (IrpContext->MajorFunction) {

            case IRP_MJ_CREATE :

                Status = CdCommonCreate( IrpContext, Irp );
                break;

            case IRP_MJ_CLOSE :

                Status = CdCommonClose( IrpContext, Irp );
                break;

            case IRP_MJ_READ :

                //
                //  If this is an Mdl complete request, don't go through
                //  common read.
                //

                if (FlagOn( IrpContext->MinorFunction, IRP_MN_COMPLETE )) {

                    Status = CdCompleteMdl( IrpContext, Irp );

                } else {

                    Status = CdCommonRead( IrpContext, Irp );
                }

                break;

            case IRP_MJ_WRITE :

                Status = CdCommonWrite( IrpContext, Irp );
                break;

            case IRP_MJ_QUERY_INFORMATION :

                Status = CdCommonQueryInfo( IrpContext, Irp );
                break;

            case IRP_MJ_SET_INFORMATION :

                Status = CdCommonSetInfo( IrpContext, Irp );
                break;

            case IRP_MJ_QUERY_VOLUME_INFORMATION :

                Status = CdCommonQueryVolInfo( IrpContext, Irp );
                break;

            case IRP_MJ_DIRECTORY_CONTROL :

                Status = CdCommonDirControl( IrpContext, Irp );
                break;

            case IRP_MJ_FILE_SYSTEM_CONTROL :

                Status = CdCommonFsControl( IrpContext, Irp );
                break;

            case IRP_MJ_DEVICE_CONTROL :

                Status = CdCommonDevControl( IrpContext, Irp );
                break;

            case IRP_MJ_LOCK_CONTROL :

                Status = CdCommonLockControl( IrpContext, Irp );
                break;

            case IRP_MJ_CLEANUP :

                Status = CdCommonCleanup( IrpContext, Irp );
                break;

            case IRP_MJ_PNP :

                Status = CdCommonPnp( IrpContext, Irp );
                break;

            case IRP_MJ_SHUTDOWN :
            
                Status = CdCommonShutdown( IrpContext, Irp );
                break;

            default :

                Status = STATUS_INVALID_DEVICE_REQUEST;
                CdCompleteRequest( IrpContext, Irp, Status );
            }

        } _SEH2_EXCEPT( CdExceptionFilter( IrpContext, _SEH2_GetExceptionInformation() )) {

            Status = CdProcessException( IrpContext, Irp, _SEH2_GetExceptionCode() );
        } _SEH2_END;

    } while (Status == STATUS_CANT_WAIT);

#ifdef CD_SANITY
    NT_ASSERT( !CdTestTopLevel ||
            (PreviousTopLevel == IoGetTopLevelIrp()) );
#endif

    FsRtlExitFileSystem();

    NT_ASSERT( SaveIrql == KeGetCurrentIrql( ));

    return Status;
}

ExpCheckPoolAllocation()

VOID NTAPI ExpCheckPoolAllocation	(	PVOID	P,
		POOL_TYPE	PoolType,
		ULONG	Tag
	)

Definition at line 288 of file expool.c.

{
    PPOOL_HEADER Entry;
    ULONG i;
    KIRQL OldIrql;
    POOL_TYPE RealPoolType;

    /* Get the pool header */
    Entry = ((PPOOL_HEADER)P) - 1;

    /* Check if this is a large allocation */
    if (PAGE_ALIGN(P) == P)
    {
        /* Lock the pool table */
        KeAcquireSpinLock(&ExpLargePoolTableLock, &OldIrql);

        /* Find the pool tag */
        for (i = 0; i < PoolBigPageTableSize; i++)
        {
            /* Check if this is our allocation */
            if (PoolBigPageTable[i].Va == P)
            {
                /* Make sure the tag is ok */
                if (PoolBigPageTable[i].Key != Tag)
                {
                    KeBugCheckEx(BAD_POOL_CALLER, 0x0A, (ULONG_PTR)P, PoolBigPageTable[i].Key, Tag);
                }

                break;
            }
        }

        /* Release the lock */
        KeReleaseSpinLock(&ExpLargePoolTableLock, OldIrql);

        if (i == PoolBigPageTableSize)
        {
            /* Did not find the allocation */
            //ASSERT(FALSE);
        }

        /* Get Pool type by address */
        RealPoolType = MmDeterminePoolType(P);
    }
    else
    {
        /* Verify the tag */
        if (Entry->PoolTag != Tag)
        {
            DPRINT1("Allocation has wrong pool tag! Expected '%.4s', got '%.4s' (0x%08lx)\n",
                    &Tag, &Entry->PoolTag, Entry->PoolTag);
            KeBugCheckEx(BAD_POOL_CALLER, 0x0A, (ULONG_PTR)P, Entry->PoolTag, Tag);
        }

        /* Check the rest of the header */
        ExpCheckPoolHeader(Entry);

        /* Get Pool type from entry */
        RealPoolType = (Entry->PoolType - 1);
    }

    /* Should we check the pool type? */
    if (PoolType != -1)
    {
        /* Verify the pool type */
        if (RealPoolType != PoolType)
        {
            DPRINT1("Wrong pool type! Expected %s, got %s\n",
                    PoolType & BASE_POOL_TYPE_MASK ? "PagedPool" : "NonPagedPool",
                    (Entry->PoolType - 1) & BASE_POOL_TYPE_MASK ? "PagedPool" : "NonPagedPool");
            KeBugCheckEx(BAD_POOL_CALLER, 0xCC, (ULONG_PTR)P, Entry->PoolTag, Tag);
        }
    }
}

ExReturnPoolQuota()

VOID NTAPI ExReturnPoolQuota

(

IN PVOID

P

)

Definition at line 1798 of file expool.c.

{
    PPOOL_HEADER Entry;
    POOL_TYPE PoolType;
    USHORT BlockSize;
    PEPROCESS Process;

    if ((ExpPoolFlags & POOL_FLAG_SPECIAL_POOL) &&
        (MmIsSpecialPoolAddress(P)))
    {
        return;
    }

    Entry = P;
    Entry--;
    ASSERT((ULONG_PTR)Entry % POOL_BLOCK_SIZE == 0);

    PoolType = Entry->PoolType - 1;
    BlockSize = Entry->BlockSize;

    if (PoolType & QUOTA_POOL_MASK)
    {
        Process = ((PVOID *)POOL_NEXT_BLOCK(Entry))[-1];
        ASSERT(Process != NULL);
        if (Process)
        {
            if (Process->Pcb.Header.Type != ProcessObject)
            {
                DPRINT1("Object %p is not a process. Type %u, pool type 0x%x, block size %u\n",
                        Process, Process->Pcb.Header.Type, Entry->PoolType, BlockSize);
                KeBugCheckEx(BAD_POOL_CALLER,
                             POOL_BILLED_PROCESS_INVALID,
                             (ULONG_PTR)P,
                             Entry->PoolTag,
                             (ULONG_PTR)Process);
            }
            ((PVOID *)POOL_NEXT_BLOCK(Entry))[-1] = NULL;
            PsReturnPoolQuota(Process,
                              PoolType & BASE_POOL_TYPE_MASK,
                              BlockSize * POOL_BLOCK_SIZE);
            ObDereferenceObject(Process);
        }
    }
}

Referenced by IoFreeIrp().

MiAcquirePfnLock()

FORCEINLINE KIRQL MiAcquirePfnLock

(

VOID

)

Definition at line 901 of file mm.h.

{
    return KeAcquireQueuedSpinLock(LockQueuePfnLock);
}

Referenced by _MiFlushMappedSection(), MiAddDescriptorToDatabase(), MiAllocatePagesForMdl(), MiAllocatePoolPages(), MiBalancerThread(), MiBuildPagedPool(), MiBuildPfnDatabaseFromLoaderBlock(), MiCheckPurgeAndUpMapCount(), MiDeleteARM3Section(), MiDeleteSystemPageableVm(), MiDeleteVirtualAddresses(), MiDereferenceControlArea(), MiDispatchFault(), MiFillSystemPageDirectory(), MiFindContiguousPages(), MiFlushTbAndCapture(), MiFreeContiguousMemory(), MiGetPageProtection(), MiGetPteForProcess(), MiInitializeAndChargePfn(), MiInitMachineDependent(), MiLoadImageSection(), MiMakeSystemAddressValidPfn(), MiMapLockedPagesInUserSpace(), MiProcessValidPteList(), MiProtectVirtualMemory(), MiReleaseProcessReferenceToSessionDataPage(), MiRemoveMappedPtes(), MiRemoveMappedView(), MiResolveDemandZeroFault(), MiResolvePageFileFault(), MiResolveProtoPteFault(), MiSegmentDelete(), MiSessionCommitPageTables(), MiSessionCreateInternal(), MiSessionInitializeWorkingSetList(), MiSetControlAreaSymbolsLoaded(), MiUnmapLockedPagesInUserSpace(), MmAllocateSpecialPool(), MmAllocPage(), MmArmAccessFault(), MmCreateArm3Section(), MmCreateKernelStack(), MmCreateProcessAddressSpace(), MmDeleteKernelStack(), MmDeleteProcessAddressSpace(), MmDeleteProcessAddressSpace2(), MmDereferencePage(), MmFreeLoaderBlock(), MmFreeMemoryArea(), MmFreePagesFromMdl(), MmFreeSpecialPool(), MmGetLRUFirstUserPage(), MmGetLRUNextUserPage(), MmGetPhysicalMemoryRanges(), MmGetReferenceCountPage(), MmGetRmapListHeadPage(), MmGetSavedSwapEntryPage(), MmGrowKernelStackEx(), MmInitializeProcessAddressSpace(), MmInsertLRULastUserPage(), MmNotPresentFaultCachePage(), MmPageOutSectionView(), MmProbeAndLockPages(), MmRemoveLRUUserPage(), MmSetRmapListHeadPage(), MmSetSavedSwapEntryPage(), MmUnlockPages(), and MmZeroPageThread().

MiAcquirePfnLockAtDpcLevel()

FORCEINLINE VOID MiAcquirePfnLockAtDpcLevel

(

VOID

)

Definition at line 916 of file mm.h.

{
    PKSPIN_LOCK_QUEUE LockQueue;

    ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL);
    LockQueue = &KeGetCurrentPrcb()->LockQueue[LockQueuePfnLock];
    KeAcquireQueuedSpinLockAtDpcLevel(LockQueue);
}

Referenced by MiAllocatePoolPages().

MiAllocatePoolPages()

PVOID NTAPI MiAllocatePoolPages	(	IN POOL_TYPE	PoolType,
		IN SIZE_T	SizeInBytes
	)

Definition at line 420 of file pool.c.

{
    PFN_NUMBER PageFrameNumber;
    PFN_COUNT SizeInPages, PageTableCount;
    ULONG i;
    KIRQL OldIrql;
    PLIST_ENTRY NextEntry, NextHead, LastHead;
    PMMPTE PointerPte, StartPte;
    PMMPDE PointerPde;
    ULONG EndAllocation;
    MMPTE TempPte;
    MMPDE TempPde;
    PMMPFN Pfn1;
    PVOID BaseVa, BaseVaStart;
    PMMFREE_POOL_ENTRY FreeEntry;

    //
    // Figure out how big the allocation is in pages
    //
    SizeInPages = (PFN_COUNT)BYTES_TO_PAGES(SizeInBytes);

    //
    // Check for overflow
    //
    if (SizeInPages == 0)
    {
        //
        // Fail
        //
        return NULL;
    }

    //
    // Handle paged pool
    //
    if ((PoolType & BASE_POOL_TYPE_MASK) == PagedPool)
    {
        //
        // If only one page is being requested, try to grab it from the S-LIST
        //
        if ((SizeInPages == 1) && (ExQueryDepthSList(&MiPagedPoolSListHead)))
        {
            BaseVa = InterlockedPopEntrySList(&MiPagedPoolSListHead);
            if (BaseVa) return BaseVa;
        }

        //
        // Lock the paged pool mutex
        //
        KeAcquireGuardedMutex(&MmPagedPoolMutex);

        //
        // Find some empty allocation space
        //
        i = RtlFindClearBitsAndSet(MmPagedPoolInfo.PagedPoolAllocationMap,
                                   SizeInPages,
                                   MmPagedPoolInfo.PagedPoolHint);
        if (i == 0xFFFFFFFF)
        {
            //
            // Get the page bit count
            //
            i = ((SizeInPages - 1) / PTE_PER_PAGE) + 1;
            DPRINT("Paged pool expansion: %lu %x\n", i, SizeInPages);

            //
            // Check if there is enougn paged pool expansion space left
            //
            if (MmPagedPoolInfo.NextPdeForPagedPoolExpansion >
                (PMMPDE)MiAddressToPte(MmPagedPoolInfo.LastPteForPagedPool))
            {
                //
                // Out of memory!
                //
                DPRINT1("FAILED to allocate %Iu bytes from paged pool\n", SizeInBytes);
                KeReleaseGuardedMutex(&MmPagedPoolMutex);
                return NULL;
            }

            //
            // Check if we'll have to expand past the last PTE we have available
            //
            if (((i - 1) + MmPagedPoolInfo.NextPdeForPagedPoolExpansion) >
                 (PMMPDE)MiAddressToPte(MmPagedPoolInfo.LastPteForPagedPool))
            {
                //
                // We can only support this much then
                //
                PointerPde = MiPteToPde(MmPagedPoolInfo.LastPteForPagedPool);
                PageTableCount = (PFN_COUNT)(PointerPde + 1 -
                                 MmPagedPoolInfo.NextPdeForPagedPoolExpansion);
                ASSERT(PageTableCount < i);
                i = PageTableCount;
            }
            else
            {
                //
                // Otherwise, there is plenty of space left for this expansion
                //
                PageTableCount = i;
            }

            //
            // Get the template PDE we'll use to expand
            //
            TempPde = ValidKernelPde;

            //
            // Get the first PTE in expansion space
            //
            PointerPde = MmPagedPoolInfo.NextPdeForPagedPoolExpansion;
            BaseVa = MiPdeToPte(PointerPde);
            BaseVaStart = BaseVa;

            //
            // Lock the PFN database and loop pages
            //
            OldIrql = MiAcquirePfnLock();
            do
            {
                //
                // It should not already be valid
                //
                ASSERT(PointerPde->u.Hard.Valid == 0);

                /* Request a page */
                MI_SET_USAGE(MI_USAGE_PAGED_POOL);
                MI_SET_PROCESS2("Kernel");
                PageFrameNumber = MiRemoveAnyPage(MI_GET_NEXT_COLOR());
                TempPde.u.Hard.PageFrameNumber = PageFrameNumber;
#if (_MI_PAGING_LEVELS >= 3)
                /* On PAE/x64 systems, there's no double-buffering */
                /* Initialize the PFN entry for it */
                MiInitializePfnForOtherProcess(PageFrameNumber,
                                               (PMMPTE)PointerPde,
                                               PFN_FROM_PTE(MiAddressToPte(PointerPde)));

                /* Write the actual PDE now */
                MI_WRITE_VALID_PDE(PointerPde, TempPde);
#else
                //
                // Save it into our double-buffered system page directory
                //
                MmSystemPagePtes[((ULONG_PTR)PointerPde & (SYSTEM_PD_SIZE - 1)) / sizeof(MMPTE)] = TempPde;

                /* Initialize the PFN */
                MiInitializePfnForOtherProcess(PageFrameNumber,
                                               (PMMPTE)PointerPde,
                                               MmSystemPageDirectory[(PointerPde - MiAddressToPde(NULL)) / PDE_PER_PAGE]);
#endif

                //
                // Move on to the next expansion address
                //
                PointerPde++;
                BaseVa = (PVOID)((ULONG_PTR)BaseVa + PAGE_SIZE);
                i--;
            } while (i > 0);

            //
            // Release the PFN database lock
            //
            MiReleasePfnLock(OldIrql);

            //
            // These pages are now available, clear their availablity bits
            //
            EndAllocation = (ULONG)(MmPagedPoolInfo.NextPdeForPagedPoolExpansion -
                                    (PMMPDE)MiAddressToPte(MmPagedPoolInfo.FirstPteForPagedPool)) *
                            PTE_PER_PAGE;
            RtlClearBits(MmPagedPoolInfo.PagedPoolAllocationMap,
                         EndAllocation,
                         PageTableCount * PTE_PER_PAGE);

            //
            // Update the next expansion location
            //
            MmPagedPoolInfo.NextPdeForPagedPoolExpansion += PageTableCount;

            //
            // Zero out the newly available memory
            //
            RtlZeroMemory(BaseVaStart, PageTableCount * PAGE_SIZE);

            //
            // Now try consuming the pages again
            //
            i = RtlFindClearBitsAndSet(MmPagedPoolInfo.PagedPoolAllocationMap,
                                       SizeInPages,
                                       0);
            if (i == 0xFFFFFFFF)
            {
                //
                // Out of memory!
                //
                DPRINT1("FAILED to allocate %Iu bytes from paged pool\n", SizeInBytes);
                KeReleaseGuardedMutex(&MmPagedPoolMutex);
                return NULL;
            }
        }

        //
        // Update the pool hint if the request was just one page
        //
        if (SizeInPages == 1) MmPagedPoolInfo.PagedPoolHint = i + 1;

        //
        // Update the end bitmap so we know the bounds of this allocation when
        // the time comes to free it
        //
        EndAllocation = i + SizeInPages - 1;
        RtlSetBit(MmPagedPoolInfo.EndOfPagedPoolBitmap, EndAllocation);

        //
        // Now we can release the lock (it mainly protects the bitmap)
        //
        KeReleaseGuardedMutex(&MmPagedPoolMutex);

        //
        // Now figure out where this allocation starts
        //
        BaseVa = (PVOID)((ULONG_PTR)MmPagedPoolStart + (i << PAGE_SHIFT));

        //
        // Flush the TLB
        //
        KeFlushEntireTb(TRUE, TRUE);

        /* Setup a demand-zero writable PTE */
        MI_MAKE_SOFTWARE_PTE(&TempPte, MM_READWRITE);

        //
        // Find the first and last PTE, then loop them all
        //
        PointerPte = MiAddressToPte(BaseVa);
        StartPte = PointerPte + SizeInPages;
        do
        {
            //
            // Write the demand zero PTE and keep going
            //
            MI_WRITE_INVALID_PTE(PointerPte, TempPte);
        } while (++PointerPte < StartPte);

        //
        // Return the allocation address to the caller
        //
        return BaseVa;
    }

    //
    // If only one page is being requested, try to grab it from the S-LIST
    //
    if ((SizeInPages == 1) && (ExQueryDepthSList(&MiNonPagedPoolSListHead)))
    {
        BaseVa = InterlockedPopEntrySList(&MiNonPagedPoolSListHead);
        if (BaseVa) return BaseVa;
    }

    //
    // Allocations of less than 4 pages go into their individual buckets
    //
    i = SizeInPages - 1;
    if (i >= MI_MAX_FREE_PAGE_LISTS) i = MI_MAX_FREE_PAGE_LISTS - 1;

    //
    // Loop through all the free page lists based on the page index
    //
    NextHead = &MmNonPagedPoolFreeListHead[i];
    LastHead = &MmNonPagedPoolFreeListHead[MI_MAX_FREE_PAGE_LISTS];

    //
    // Acquire the nonpaged pool lock
    //
    OldIrql = KeAcquireQueuedSpinLock(LockQueueMmNonPagedPoolLock);
    do
    {
        //
        // Now loop through all the free page entries in this given list
        //
        NextEntry = NextHead->Flink;
        while (NextEntry != NextHead)
        {
            /* Is freed non paged pool enabled */
            if (MmProtectFreedNonPagedPool)
            {
                /* We need to be able to touch this page, unprotect it */
                MiUnProtectFreeNonPagedPool(NextEntry, 0);
            }

            //
            // Grab the entry and see if it can handle our allocation
            //
            FreeEntry = CONTAINING_RECORD(NextEntry, MMFREE_POOL_ENTRY, List);
            ASSERT(FreeEntry->Signature == MM_FREE_POOL_SIGNATURE);
            if (FreeEntry->Size >= SizeInPages)
            {
                //
                // It does, so consume the pages from here
                //
                FreeEntry->Size -= SizeInPages;

                //
                // The allocation will begin in this free page area
                //
                BaseVa = (PVOID)((ULONG_PTR)FreeEntry +
                                 (FreeEntry->Size  << PAGE_SHIFT));

                /* Remove the item from the list, depending if pool is protected */
                if (MmProtectFreedNonPagedPool)
                    MiProtectedPoolRemoveEntryList(&FreeEntry->List);
                else
                    RemoveEntryList(&FreeEntry->List);

                //
                // However, check if its' still got space left
                //
                if (FreeEntry->Size != 0)
                {
                    /* Check which list to insert this entry into */
                    i = FreeEntry->Size - 1;
                    if (i >= MI_MAX_FREE_PAGE_LISTS) i = MI_MAX_FREE_PAGE_LISTS - 1;

                    /* Insert the entry into the free list head, check for prot. pool */
                    if (MmProtectFreedNonPagedPool)
                        MiProtectedPoolInsertList(&MmNonPagedPoolFreeListHead[i], &FreeEntry->List, TRUE);
                    else
                        InsertTailList(&MmNonPagedPoolFreeListHead[i], &FreeEntry->List);

                    /* Is freed non paged pool protected? */
                    if (MmProtectFreedNonPagedPool)
                    {
                        /* Protect the freed pool! */
                        MiProtectFreeNonPagedPool(FreeEntry, FreeEntry->Size);
                    }
                }

                //
                // Grab the PTE for this allocation
                //
                PointerPte = MiAddressToPte(BaseVa);
                ASSERT(PointerPte->u.Hard.Valid == 1);

                //
                // Grab the PFN NextEntry and index
                //
                Pfn1 = MiGetPfnEntry(PFN_FROM_PTE(PointerPte));

                //
                // Now mark it as the beginning of an allocation
                //
                ASSERT(Pfn1->u3.e1.StartOfAllocation == 0);
                Pfn1->u3.e1.StartOfAllocation = 1;

                /* Mark it as special pool if needed */
                ASSERT(Pfn1->u4.VerifierAllocation == 0);
                if (PoolType & VERIFIER_POOL_MASK)
                {
                    Pfn1->u4.VerifierAllocation = 1;
                }

                //
                // Check if the allocation is larger than one page
                //
                if (SizeInPages != 1)
                {
                    //
                    // Navigate to the last PFN entry and PTE
                    //
                    PointerPte += SizeInPages - 1;
                    ASSERT(PointerPte->u.Hard.Valid == 1);
                    Pfn1 = MiGetPfnEntry(PointerPte->u.Hard.PageFrameNumber);
                }

                //
                // Mark this PFN as the last (might be the same as the first)
                //
                ASSERT(Pfn1->u3.e1.EndOfAllocation == 0);
                Pfn1->u3.e1.EndOfAllocation = 1;

                //
                // Release the nonpaged pool lock, and return the allocation
                //
                KeReleaseQueuedSpinLock(LockQueueMmNonPagedPoolLock, OldIrql);
                return BaseVa;
            }

            //
            // Try the next free page entry
            //
            NextEntry = FreeEntry->List.Flink;

            /* Is freed non paged pool protected? */
            if (MmProtectFreedNonPagedPool)
            {
                /* Protect the freed pool! */
                MiProtectFreeNonPagedPool(FreeEntry, FreeEntry->Size);
            }
        }
    } while (++NextHead < LastHead);

    //
    // If we got here, we're out of space.
    // Start by releasing the lock
    //
    KeReleaseQueuedSpinLock(LockQueueMmNonPagedPoolLock, OldIrql);

    //
    // Allocate some system PTEs
    //
    StartPte = MiReserveSystemPtes(SizeInPages, NonPagedPoolExpansion);
    PointerPte = StartPte;
    if (StartPte == NULL)
    {
        //
        // Ran out of memory
        //
        DPRINT1("Out of NP Expansion Pool\n");
        return NULL;
    }

    //
    // Acquire the pool lock now
    //
    OldIrql = KeAcquireQueuedSpinLock(LockQueueMmNonPagedPoolLock);

    //
    // Lock the PFN database too
    //
    MiAcquirePfnLockAtDpcLevel();

    //
    // Loop the pages
    //
    TempPte = ValidKernelPte;
    do
    {
        /* Allocate a page */
        MI_SET_USAGE(MI_USAGE_PAGED_POOL);
        MI_SET_PROCESS2("Kernel");
        PageFrameNumber = MiRemoveAnyPage(MI_GET_NEXT_COLOR());

        /* Get the PFN entry for it and fill it out */
        Pfn1 = MiGetPfnEntry(PageFrameNumber);
        Pfn1->u3.e2.ReferenceCount = 1;
        Pfn1->u2.ShareCount = 1;
        Pfn1->PteAddress = PointerPte;
        Pfn1->u3.e1.PageLocation = ActiveAndValid;
        Pfn1->u4.VerifierAllocation = 0;

        /* Write the PTE for it */
        TempPte.u.Hard.PageFrameNumber = PageFrameNumber;
        MI_WRITE_VALID_PTE(PointerPte++, TempPte);
    } while (--SizeInPages > 0);

    //
    // This is the last page
    //
    Pfn1->u3.e1.EndOfAllocation = 1;

    //
    // Get the first page and mark it as such
    //
    Pfn1 = MiGetPfnEntry(StartPte->u.Hard.PageFrameNumber);
    Pfn1->u3.e1.StartOfAllocation = 1;

    /* Mark it as a verifier allocation if needed */
    ASSERT(Pfn1->u4.VerifierAllocation == 0);
    if (PoolType & VERIFIER_POOL_MASK) Pfn1->u4.VerifierAllocation = 1;

    //
    // Release the PFN and nonpaged pool lock
    //
    MiReleasePfnLockFromDpcLevel();
    KeReleaseQueuedSpinLock(LockQueueMmNonPagedPoolLock, OldIrql);

    //
    // Return the address
    //
    return MiPteToAddress(StartPte);
}

Referenced by _IRQL_requires_(), ExAllocatePoolWithTag(), and InitializePool().

MiCheckAllProcessMemoryAreas()

VOID NTAPI MiCheckAllProcessMemoryAreas

(

VOID

)

MiCopyFromUserPage()

NTSTATUS NTAPI MiCopyFromUserPage	(	PFN_NUMBER	DestPage,
		const VOID *	SrcAddress
	)

Definition at line 1046 of file section.c.

{
    PEPROCESS Process;
    KIRQL Irql;
    PVOID DestAddress;

    Process = PsGetCurrentProcess();
    DestAddress = MiMapPageInHyperSpace(Process, DestPage, &Irql);
    if (DestAddress == NULL)
    {
        return(STATUS_NO_MEMORY);
    }
    ASSERT((ULONG_PTR)DestAddress % PAGE_SIZE == 0);
    ASSERT((ULONG_PTR)SrcAddress % PAGE_SIZE == 0);
    RtlCopyMemory(DestAddress, SrcAddress, PAGE_SIZE);
    MiUnmapPageInHyperSpace(Process, DestAddress, Irql);
    return(STATUS_SUCCESS);
}

Referenced by MmAccessFaultSectionView().

MiFreePoolPages()

ULONG NTAPI MiFreePoolPages

(

IN PVOID

StartingAddress

)

Definition at line 905 of file pool.c.

{
    PMMPTE PointerPte, StartPte;
    PMMPFN Pfn1, StartPfn;
    PFN_COUNT FreePages, NumberOfPages;
    KIRQL OldIrql;
    PMMFREE_POOL_ENTRY FreeEntry, NextEntry, LastEntry;
    ULONG i, End;
    ULONG_PTR Offset;

    //
    // Handle paged pool
    //
    if ((StartingVa >= MmPagedPoolStart) && (StartingVa <= MmPagedPoolEnd))
    {
        //
        // Calculate the offset from the beginning of paged pool, and convert it
        // into pages
        //
        Offset = (ULONG_PTR)StartingVa - (ULONG_PTR)MmPagedPoolStart;
        i = (ULONG)(Offset >> PAGE_SHIFT);
        End = i;

        //
        // Now use the end bitmap to scan until we find a set bit, meaning that
        // this allocation finishes here
        //
        while (!RtlTestBit(MmPagedPoolInfo.EndOfPagedPoolBitmap, End)) End++;

        //
        // Now calculate the total number of pages this allocation spans. If it's
        // only one page, add it to the S-LIST instead of freeing it
        //
        NumberOfPages = End - i + 1;
        if ((NumberOfPages == 1) &&
            (ExQueryDepthSList(&MiPagedPoolSListHead) < MiPagedPoolSListMaximum))
        {
            InterlockedPushEntrySList(&MiPagedPoolSListHead, StartingVa);
            return 1;
        }

        /* Delete the actual pages */
        PointerPte = MmPagedPoolInfo.FirstPteForPagedPool + i;
        FreePages = MiDeleteSystemPageableVm(PointerPte, NumberOfPages, 0, NULL);
        ASSERT(FreePages == NumberOfPages);

        //
        // Acquire the paged pool lock
        //
        KeAcquireGuardedMutex(&MmPagedPoolMutex);

        //
        // Clear the allocation and free bits
        //
        RtlClearBit(MmPagedPoolInfo.EndOfPagedPoolBitmap, End);
        RtlClearBits(MmPagedPoolInfo.PagedPoolAllocationMap, i, NumberOfPages);

        //
        // Update the hint if we need to
        //
        if (i < MmPagedPoolInfo.PagedPoolHint) MmPagedPoolInfo.PagedPoolHint = i;

        //
        // Release the lock protecting the bitmaps
        //
        KeReleaseGuardedMutex(&MmPagedPoolMutex);

        //
        // And finally return the number of pages freed
        //
        return NumberOfPages;
    }

    //
    // Get the first PTE and its corresponding PFN entry. If this is also the
    // last PTE, meaning that this allocation was only for one page, push it into
    // the S-LIST instead of freeing it
    //
    StartPte = PointerPte = MiAddressToPte(StartingVa);
    StartPfn = Pfn1 = MiGetPfnEntry(PointerPte->u.Hard.PageFrameNumber);
    if ((Pfn1->u3.e1.EndOfAllocation == 1) &&
        (ExQueryDepthSList(&MiNonPagedPoolSListHead) < MiNonPagedPoolSListMaximum))
    {
        InterlockedPushEntrySList(&MiNonPagedPoolSListHead, StartingVa);
        return 1;
    }

    //
    // Loop until we find the last PTE
    //
    while (Pfn1->u3.e1.EndOfAllocation == 0)
    {
        //
        // Keep going
        //
        PointerPte++;
        Pfn1 = MiGetPfnEntry(PointerPte->u.Hard.PageFrameNumber);
    }

    //
    // Now we know how many pages we have
    //
    NumberOfPages = (PFN_COUNT)(PointerPte - StartPte + 1);

    //
    // Acquire the nonpaged pool lock
    //
    OldIrql = KeAcquireQueuedSpinLock(LockQueueMmNonPagedPoolLock);

    //
    // Mark the first and last PTEs as not part of an allocation anymore
    //
    StartPfn->u3.e1.StartOfAllocation = 0;
    Pfn1->u3.e1.EndOfAllocation = 0;

    //
    // Assume we will free as many pages as the allocation was
    //
    FreePages = NumberOfPages;

    //
    // Peek one page past the end of the allocation
    //
    PointerPte++;

    //
    // Guard against going past initial nonpaged pool
    //
    if (MiGetPfnEntryIndex(Pfn1) == MiEndOfInitialPoolFrame)
    {
        //
        // This page is on the outskirts of initial nonpaged pool, so ignore it
        //
        Pfn1 = NULL;
    }
    else
    {
        /* Sanity check */
        ASSERT((ULONG_PTR)StartingVa + NumberOfPages <= (ULONG_PTR)MmNonPagedPoolEnd);

        /* Check if protected pool is enabled */
        if (MmProtectFreedNonPagedPool)
        {
            /* The freed block will be merged, it must be made accessible */
            MiUnProtectFreeNonPagedPool(MiPteToAddress(PointerPte), 0);
        }

        //
        // Otherwise, our entire allocation must've fit within the initial non
        // paged pool, or the expansion nonpaged pool, so get the PFN entry of
        // the next allocation
        //
        if (PointerPte->u.Hard.Valid == 1)
        {
            //
            // It's either expansion or initial: get the PFN entry
            //
            Pfn1 = MiGetPfnEntry(PointerPte->u.Hard.PageFrameNumber);
        }
        else
        {
            //
            // This means we've reached the guard page that protects the end of
            // the expansion nonpaged pool
            //
            Pfn1 = NULL;
        }

    }

    //
    // Check if this allocation actually exists
    //
    if ((Pfn1) && (Pfn1->u3.e1.StartOfAllocation == 0))
    {
        //
        // It doesn't, so we should actually locate a free entry descriptor
        //
        FreeEntry = (PMMFREE_POOL_ENTRY)((ULONG_PTR)StartingVa +
                                         (NumberOfPages << PAGE_SHIFT));
        ASSERT(FreeEntry->Signature == MM_FREE_POOL_SIGNATURE);
        ASSERT(FreeEntry->Owner == FreeEntry);

        /* Consume this entry's pages */
        FreePages += FreeEntry->Size;

        /* Remove the item from the list, depending if pool is protected */
        if (MmProtectFreedNonPagedPool)
            MiProtectedPoolRemoveEntryList(&FreeEntry->List);
        else
            RemoveEntryList(&FreeEntry->List);
    }

    //
    // Now get the official free entry we'll create for the caller's allocation
    //
    FreeEntry = StartingVa;

    //
    // Check if the our allocation is the very first page
    //
    if (MiGetPfnEntryIndex(StartPfn) == MiStartOfInitialPoolFrame)
    {
        //
        // Then we can't do anything or we'll risk underflowing
        //
        Pfn1 = NULL;
    }
    else
    {
        //
        // Otherwise, get the PTE for the page right before our allocation
        //
        PointerPte -= NumberOfPages + 1;

        /* Check if protected pool is enabled */
        if (MmProtectFreedNonPagedPool)
        {
            /* The freed block will be merged, it must be made accessible */
            MiUnProtectFreeNonPagedPool(MiPteToAddress(PointerPte), 0);
        }

        /* Check if this is valid pool, or a guard page */
        if (PointerPte->u.Hard.Valid == 1)
        {
            //
            // It's either expansion or initial nonpaged pool, get the PFN entry
            //
            Pfn1 = MiGetPfnEntry(PointerPte->u.Hard.PageFrameNumber);
        }
        else
        {
            //
            // We must've reached the guard page, so don't risk touching it
            //
            Pfn1 = NULL;
        }
    }

    //
    // Check if there is a valid PFN entry for the page before the allocation
    // and then check if this page was actually the end of an allocation.
    // If it wasn't, then we know for sure it's a free page
    //
    if ((Pfn1) && (Pfn1->u3.e1.EndOfAllocation == 0))
    {
        //
        // Get the free entry descriptor for that given page range
        //
        FreeEntry = (PMMFREE_POOL_ENTRY)((ULONG_PTR)StartingVa - PAGE_SIZE);
        ASSERT(FreeEntry->Signature == MM_FREE_POOL_SIGNATURE);
        FreeEntry = FreeEntry->Owner;

        /* Check if protected pool is enabled */
        if (MmProtectFreedNonPagedPool)
        {
            /* The freed block will be merged, it must be made accessible */
            MiUnProtectFreeNonPagedPool(FreeEntry, 0);
        }

        //
        // Check if the entry is small enough to be indexed on a free list
        // If it is, we'll want to re-insert it, since we're about to
        // collapse our pages on top of it, which will change its count
        //
        if (FreeEntry->Size < (MI_MAX_FREE_PAGE_LISTS - 1))
        {
            /* Remove the item from the list, depending if pool is protected */
            if (MmProtectFreedNonPagedPool)
                MiProtectedPoolRemoveEntryList(&FreeEntry->List);
            else
                RemoveEntryList(&FreeEntry->List);

            //
            // Update its size
            //
            FreeEntry->Size += FreePages;

            //
            // And now find the new appropriate list to place it in
            //
            i = (ULONG)(FreeEntry->Size - 1);
            if (i >= MI_MAX_FREE_PAGE_LISTS) i = MI_MAX_FREE_PAGE_LISTS - 1;

            /* Insert the entry into the free list head, check for prot. pool */
            if (MmProtectFreedNonPagedPool)
                MiProtectedPoolInsertList(&MmNonPagedPoolFreeListHead[i], &FreeEntry->List, TRUE);
            else
                InsertTailList(&MmNonPagedPoolFreeListHead[i], &FreeEntry->List);
        }
        else
        {
            //
            // Otherwise, just combine our free pages into this entry
            //
            FreeEntry->Size += FreePages;
        }
    }

    //
    // Check if we were unable to do any compaction, and we'll stick with this
    //
    if (FreeEntry == StartingVa)
    {
        //
        // Well, now we are a free entry. At worse we just have our newly freed
        // pages, at best we have our pages plus whatever entry came after us
        //
        FreeEntry->Size = FreePages;

        //
        // Find the appropriate list we should be on
        //
        i = FreeEntry->Size - 1;
        if (i >= MI_MAX_FREE_PAGE_LISTS) i = MI_MAX_FREE_PAGE_LISTS - 1;

        /* Insert the entry into the free list head, check for prot. pool */
        if (MmProtectFreedNonPagedPool)
            MiProtectedPoolInsertList(&MmNonPagedPoolFreeListHead[i], &FreeEntry->List, TRUE);
        else
            InsertTailList(&MmNonPagedPoolFreeListHead[i], &FreeEntry->List);
    }

    //
    // Just a sanity check
    //
    ASSERT(FreePages != 0);

    //
    // Get all the pages between our allocation and its end. These will all now
    // become free page chunks.
    //
    NextEntry = StartingVa;
    LastEntry = (PMMFREE_POOL_ENTRY)((ULONG_PTR)NextEntry + (FreePages << PAGE_SHIFT));
    do
    {
        //
        // Link back to the parent free entry, and keep going
        //
        NextEntry->Owner = FreeEntry;
        NextEntry->Signature = MM_FREE_POOL_SIGNATURE;
        NextEntry = (PMMFREE_POOL_ENTRY)((ULONG_PTR)NextEntry + PAGE_SIZE);
    } while (NextEntry != LastEntry);

    /* Is freed non paged pool protected? */
    if (MmProtectFreedNonPagedPool)
    {
        /* Protect the freed pool! */
        MiProtectFreeNonPagedPool(FreeEntry, FreeEntry->Size);
    }

    //
    // We're done, release the lock and let the caller know how much we freed
    //
    KeReleaseQueuedSpinLock(LockQueueMmNonPagedPoolLock, OldIrql);
    return NumberOfPages;
}

Referenced by _IRQL_requires_(), and ExFreePoolWithTag().

MiGetPfnEntry()

FORCEINLINE PMMPFN MiGetPfnEntry

(

IN PFN_NUMBER

Pfn

)

Definition at line 940 of file mm.h.

{
    PMMPFN Page;
    extern RTL_BITMAP MiPfnBitMap;

    /* Make sure the PFN number is valid */
    if (Pfn > MmHighestPhysicalPage) return NULL;

    /* Make sure this page actually has a PFN entry */
    if ((MiPfnBitMap.Buffer) && !(RtlTestBit(&MiPfnBitMap, (ULONG)Pfn))) return NULL;

    /* Get the entry */
    Page = &MmPfnDatabase[Pfn];

    /* Return it */
    return Page;
};

Referenced by MiAddHalIoMappings(), MiAllocatePagesForMdl(), MiAllocatePoolPages(), MiBuildPfnDatabaseFromLoaderBlock(), MiBuildPfnDatabaseFromPages(), MiBuildPfnDatabaseFromPageTables(), MiBuildPfnDatabaseSelf(), MiBuildPfnDatabaseZeroPage(), MiCompleteProtoPteFault(), MiCopyPfn(), MiDbgTranslatePhysicalAddress(), MiDecommitPages(), MiDeletePte(), MiDeleteSystemPageableVm(), MiFindContiguousMemory(), MiFreeContiguousMemory(), MiFreePoolPages(), MiGetPageProtection(), MiInitializeWorkingSetList(), MiInitMachineDependent(), MiLockVirtualMemory(), MiMapLockedPagesInUserSpace(), MiMapPageInHyperSpace(), MiProcessValidPteList(), MiProtectVirtualMemory(), MiReloadBootLoadedDrivers(), MiRemoveMappedPtes(), MiResolveProtoPteFault(), MiResolveTransitionFault(), MiSetPagingOfDriver(), MiSetProtectionOnSection(), MiSetupPfnForPageTable(), MiUnlockVirtualMemory(), MiUnmapLockedPagesInUserSpace(), MiZeroPfn(), MmAllocPage(), MmBuildMdlForNonPagedPool(), MmDeleteKernelStack(), MmDeleteProcessAddressSpace2(), MmDereferencePage(), MmDumpArmPfnDatabase(), MmFreeLoaderBlock(), MmFreePagesFromMdl(), MmGetLRUFirstUserPage(), MmGetLRUNextUserPage(), MmGetReferenceCountPage(), MmGetRmapListHeadPage(), MmGetSavedSwapEntryPage(), MmInsertLRULastUserPage(), MmIsPageInUse(), MmMapIoSpace(), MmNotPresentFaultCachePage(), MmProbeAndLockPages(), MmReferencePage(), MmRemoveLRUUserPage(), MmRosNotifyAvailablePage(), MmSetRmapListHeadPage(), MmSetSavedSwapEntryPage(), MmUnlockPages(), MmUnmapIoSpace(), MmZeroPageThread(), and PspCreateProcess().

MiGetPfnEntryIndex()

FORCEINLINE PFN_NUMBER MiGetPfnEntryIndex

(

IN PMMPFN

Pfn1

)

Definition at line 960 of file mm.h.

{
    //
    // This will return the Page Frame Number (PFN) from the MMPFN
    //
    return Pfn1 - MmPfnDatabase;
}

Referenced by MiAllocatePagesForMdl(), MiDecrementShareCount(), MiDereferencePfnAndDropLockCount(), MiFreePoolPages(), MiInitializeWorkingSetList(), MiInsertPageInFreeList(), MiMapPagesInZeroSpace(), and MiUnlinkFreeOrZeroedPage().

MiInitBalancerThread()

INIT_FUNCTION VOID NTAPI MiInitBalancerThread

(

VOID

)

Definition at line 449 of file balance.c.

{
    KPRIORITY Priority;
    NTSTATUS Status;
#if !defined(__GNUC__)

    LARGE_INTEGER dummyJunkNeeded;
    dummyJunkNeeded.QuadPart = -20000000; /* 2 sec */
    ;
#endif


    KeInitializeEvent(&MiBalancerEvent, SynchronizationEvent, FALSE);
    KeInitializeTimerEx(&MiBalancerTimer, SynchronizationTimer);
    KeSetTimerEx(&MiBalancerTimer,
#if defined(__GNUC__)
                 (LARGE_INTEGER)(LONGLONG)-20000000LL,     /* 2 sec */
#else
                 dummyJunkNeeded,
#endif
                 2000,         /* 2 sec */
                 NULL);

    Status = PsCreateSystemThread(&MiBalancerThreadHandle,
                                  THREAD_ALL_ACCESS,
                                  NULL,
                                  NULL,
                                  &MiBalancerThreadId,
                                  MiBalancerThread,
                                  NULL);
    if (!NT_SUCCESS(Status))
    {
        KeBugCheck(MEMORY_MANAGEMENT);
    }

    Priority = LOW_REALTIME_PRIORITY + 1;
    NtSetInformationThread(MiBalancerThreadHandle,
                           ThreadPriority,
                           &Priority,
                           sizeof(Priority));

}

Referenced by MmInitSystem().

MiInitializeLoadedModuleList()

INIT_FUNCTION BOOLEAN NTAPI MiInitializeLoadedModuleList

(

IN PLOADER_PARAMETER_BLOCK

LoaderBlock

)

Definition at line 2197 of file sysldr.c.

{
    PLDR_DATA_TABLE_ENTRY LdrEntry, NewEntry;
    PLIST_ENTRY ListHead, NextEntry;
    ULONG EntrySize;

    /* Setup the loaded module list and locks */
    ExInitializeResourceLite(&PsLoadedModuleResource);
    KeInitializeSpinLock(&PsLoadedModuleSpinLock);
    InitializeListHead(&PsLoadedModuleList);

    /* Get loop variables and the kernel entry */
    ListHead = &LoaderBlock->LoadOrderListHead;
    NextEntry = ListHead->Flink;
    LdrEntry = CONTAINING_RECORD(NextEntry,
                                 LDR_DATA_TABLE_ENTRY,
                                 InLoadOrderLinks);
    PsNtosImageBase = (ULONG_PTR)LdrEntry->DllBase;

    /* Locate resource section, pool code, and system pte code */
    MiLocateKernelSections(LdrEntry);

    /* Loop the loader block */
    while (NextEntry != ListHead)
    {
        /* Get the loader entry */
        LdrEntry = CONTAINING_RECORD(NextEntry,
                                     LDR_DATA_TABLE_ENTRY,
                                     InLoadOrderLinks);

        /* FIXME: ROS HACK. Make sure this is a driver */
        if (!RtlImageNtHeader(LdrEntry->DllBase))
        {
            /* Skip this entry */
            NextEntry = NextEntry->Flink;
            continue;
        }

        /* Calculate the size we'll need and allocate a copy */
        EntrySize = sizeof(LDR_DATA_TABLE_ENTRY) +
                    LdrEntry->BaseDllName.MaximumLength +
                    sizeof(UNICODE_NULL);
        NewEntry = ExAllocatePoolWithTag(NonPagedPool, EntrySize, TAG_MODULE_OBJECT);
        if (!NewEntry) return FALSE;

        /* Copy the entry over */
        *NewEntry = *LdrEntry;

        /* Allocate the name */
        NewEntry->FullDllName.Buffer =
            ExAllocatePoolWithTag(PagedPool,
                                  LdrEntry->FullDllName.MaximumLength +
                                      sizeof(UNICODE_NULL),
                                  TAG_LDR_WSTR);
        if (!NewEntry->FullDllName.Buffer)
        {
            ExFreePoolWithTag(NewEntry, TAG_MODULE_OBJECT);
            return FALSE;
        }

        /* Set the base name */
        NewEntry->BaseDllName.Buffer = (PVOID)(NewEntry + 1);

        /* Copy the full and base name */
        RtlCopyMemory(NewEntry->FullDllName.Buffer,
                      LdrEntry->FullDllName.Buffer,
                      LdrEntry->FullDllName.MaximumLength);
        RtlCopyMemory(NewEntry->BaseDllName.Buffer,
                      LdrEntry->BaseDllName.Buffer,
                      LdrEntry->BaseDllName.MaximumLength);

        /* Null-terminate the base name */
        NewEntry->BaseDllName.Buffer[NewEntry->BaseDllName.Length /
                                     sizeof(WCHAR)] = UNICODE_NULL;

        /* Insert the entry into the list */
        InsertTailList(&PsLoadedModuleList, &NewEntry->InLoadOrderLinks);
        NextEntry = NextEntry->Flink;
    }

    /* Build the import lists for the boot drivers */
    MiBuildImportsForBootDrivers();

    /* We're done */
    return TRUE;
}

Referenced by MmArmInitSystem().

MiInitializeNonPagedPool()

INIT_FUNCTION VOID NTAPI MiInitializeNonPagedPool

(

VOID

)

Definition at line 276 of file pool.c.

{
    ULONG i;
    PFN_COUNT PoolPages;
    PMMFREE_POOL_ENTRY FreeEntry, FirstEntry;
    PMMPTE PointerPte;
    PAGED_CODE();

    //
    // Initialize the pool S-LISTs as well as their maximum count. In general,
    // we'll allow 8 times the default on a 2GB system, and two times the default
    // on a 1GB system.
    //
    InitializeSListHead(&MiPagedPoolSListHead);
    InitializeSListHead(&MiNonPagedPoolSListHead);
    if (MmNumberOfPhysicalPages >= ((2 * _1GB) /PAGE_SIZE))
    {
        MiNonPagedPoolSListMaximum *= 8;
        MiPagedPoolSListMaximum *= 8;
    }
    else if (MmNumberOfPhysicalPages >= (_1GB /PAGE_SIZE))
    {
        MiNonPagedPoolSListMaximum *= 2;
        MiPagedPoolSListMaximum *= 2;
    }

    //
    // However if debugging options for the pool are enabled, turn off the S-LIST
    // to reduce the risk of messing things up even more
    //
    if (MmProtectFreedNonPagedPool)
    {
        MiNonPagedPoolSListMaximum = 0;
        MiPagedPoolSListMaximum = 0;
    }

    //
    // We keep 4 lists of free pages (4 lists help avoid contention)
    //
    for (i = 0; i < MI_MAX_FREE_PAGE_LISTS; i++)
    {
        //
        // Initialize each of them
        //
        InitializeListHead(&MmNonPagedPoolFreeListHead[i]);
    }

    //
    // Calculate how many pages the initial nonpaged pool has
    //
    PoolPages = (PFN_COUNT)BYTES_TO_PAGES(MmSizeOfNonPagedPoolInBytes);
    MmNumberOfFreeNonPagedPool = PoolPages;

    //
    // Initialize the first free entry
    //
    FreeEntry = MmNonPagedPoolStart;
    FirstEntry = FreeEntry;
    FreeEntry->Size = PoolPages;
    FreeEntry->Signature = MM_FREE_POOL_SIGNATURE;
    FreeEntry->Owner = FirstEntry;

    //
    // Insert it into the last list
    //
    InsertHeadList(&MmNonPagedPoolFreeListHead[MI_MAX_FREE_PAGE_LISTS - 1],
                   &FreeEntry->List);

    //
    // Now create free entries for every single other page
    //
    while (PoolPages-- > 1)
    {
        //
        // Link them all back to the original entry
        //
        FreeEntry = (PMMFREE_POOL_ENTRY)((ULONG_PTR)FreeEntry + PAGE_SIZE);
        FreeEntry->Owner = FirstEntry;
        FreeEntry->Signature = MM_FREE_POOL_SIGNATURE;
    }

    //
    // Validate and remember first allocated pool page
    //
    PointerPte = MiAddressToPte(MmNonPagedPoolStart);
    ASSERT(PointerPte->u.Hard.Valid == 1);
    MiStartOfInitialPoolFrame = PFN_FROM_PTE(PointerPte);

    //
    // Keep track of where initial nonpaged pool ends
    //
    MmNonPagedPoolEnd0 = (PVOID)((ULONG_PTR)MmNonPagedPoolStart +
                                 MmSizeOfNonPagedPoolInBytes);

    //
    // Validate and remember last allocated pool page
    //
    PointerPte = MiAddressToPte((PVOID)((ULONG_PTR)MmNonPagedPoolEnd0 - 1));
    ASSERT(PointerPte->u.Hard.Valid == 1);
    MiEndOfInitialPoolFrame = PFN_FROM_PTE(PointerPte);

    //
    // Validate the first nonpaged pool expansion page (which is a guard page)
    //
    PointerPte = MiAddressToPte(MmNonPagedPoolExpansionStart);
    ASSERT(PointerPte->u.Hard.Valid == 0);

    //
    // Calculate the size of the expansion region alone
    //
    MiExpansionPoolPagesInitialCharge = (PFN_COUNT)
    BYTES_TO_PAGES(MmMaximumNonPagedPoolInBytes - MmSizeOfNonPagedPoolInBytes);

    //
    // Remove 2 pages, since there's a guard page on top and on the bottom
    //
    MiExpansionPoolPagesInitialCharge -= 2;

    //
    // Now initialize the nonpaged pool expansion PTE space. Remember there's a
    // guard page on top so make sure to skip it. The bottom guard page will be
    // guaranteed by the fact our size is off by one.
    //
    MiInitializeSystemPtes(PointerPte + 1,
                           MiExpansionPoolPagesInitialCharge,
                           NonPagedPoolExpansion);
}

Referenced by MiBuildNonPagedPool(), and MiInitMachineDependent().

MiInitializeSpecialPool()

VOID NTAPI MiInitializeSpecialPool

(

VOID

)

Definition at line 123 of file special.c.

{
    ULONG SpecialPoolPtes, i;
    PMMPTE PointerPte;

    /* Check if there is a special pool tag */
    if ((MmSpecialPoolTag == 0) ||
        (MmSpecialPoolTag == -1)) return;

    /* Calculate number of system PTEs for the special pool */
    if (MmNumberOfSystemPtes >= 0x3000)
        SpecialPoolPtes = MmNumberOfSystemPtes / 3;
    else
        SpecialPoolPtes = MmNumberOfSystemPtes / 6;

    /* Don't let the number go too high */
    if (SpecialPoolPtes > 0x6000) SpecialPoolPtes = 0x6000;

    /* Round up to the page size */
    SpecialPoolPtes = PAGE_ROUND_UP(SpecialPoolPtes);

    ASSERT((SpecialPoolPtes & (PTE_PER_PAGE - 1)) == 0);

    /* Reserve those PTEs */
    do
    {
        PointerPte = MiReserveAlignedSystemPtes(SpecialPoolPtes,
                                                SystemPteSpace,
                                                /*0x400000*/0); // FIXME:
        if (PointerPte) break;

        /* Reserving didn't work, so try to reduce the requested size */
        ASSERT(SpecialPoolPtes >= PTE_PER_PAGE);
        SpecialPoolPtes -= PTE_PER_PAGE;
    } while (SpecialPoolPtes);

    /* Fail if we couldn't reserve them at all */
    if (!SpecialPoolPtes) return;

    /* Make sure we got enough */
    ASSERT(SpecialPoolPtes >= PTE_PER_PAGE);

    /* Save first PTE and its address */
    MiSpecialPoolFirstPte = PointerPte;
    MmSpecialPoolStart = MiPteToAddress(PointerPte);

    for (i = 0; i < PTE_PER_PAGE / 2; i++)
    {
        /* Point it to the next entry */
        PointerPte->u.List.NextEntry = &PointerPte[2] - MmSystemPteBase;

        /* Move to the next pair */
        PointerPte += 2;
    }

    /* Save extra values */
    MiSpecialPoolExtra = PointerPte;
    MiSpecialPoolExtraCount = SpecialPoolPtes - PTE_PER_PAGE;

    /* Mark the previous PTE as the last one */
    MiSpecialPoolLastPte = PointerPte - 2;
    MiSpecialPoolLastPte->u.List.NextEntry = MM_EMPTY_PTE_LIST;

    /* Save end address of the special pool */
    MmSpecialPoolEnd = MiPteToAddress(MiSpecialPoolLastPte + 1);

    /* Calculate maximum non-paged part of the special pool */
    MiSpecialPagesNonPagedMaximum = MmResidentAvailablePages >> 4;

    /* And limit it if it turned out to be too big */
    if (MmNumberOfPhysicalPages > 0x3FFF)
        MiSpecialPagesNonPagedMaximum = MmResidentAvailablePages >> 3;

    DPRINT1("Special pool start %p - end %p\n", MmSpecialPoolStart, MmSpecialPoolEnd);
    ExpPoolFlags |= POOL_FLAG_SPECIAL_POOL;

    //MiTestSpecialPool();
}

Referenced by MiBuildPagedPool().

MiMapPageInHyperSpace()

PVOID NTAPI MiMapPageInHyperSpace	(	IN PEPROCESS	Process,
		IN PFN_NUMBER	Page,
		IN PKIRQL	OldIrql
	)

Definition at line 30 of file hypermap.c.

{
    MMPTE TempPte;
    PMMPTE PointerPte;
    PFN_NUMBER Offset;

    //
    // Never accept page 0 or non-physical pages
    //
    ASSERT(Page != 0);
    ASSERT(MiGetPfnEntry(Page) != NULL);

    //
    // Build the PTE
    //
    TempPte = ValidKernelPteLocal;
    TempPte.u.Hard.PageFrameNumber = Page;

    //
    // Pick the first hyperspace PTE
    //
    PointerPte = MmFirstReservedMappingPte;

    //
    // Acquire the hyperlock
    //
    ASSERT(Process == PsGetCurrentProcess());
    KeAcquireSpinLock(&Process->HyperSpaceLock, OldIrql);

    //
    // Now get the first free PTE
    //
    Offset = PFN_FROM_PTE(PointerPte);
    if (!Offset)
    {
        //
        // Reset the PTEs
        //
        Offset = MI_HYPERSPACE_PTES;
        KeFlushProcessTb();
    }

    //
    // Prepare the next PTE
    //
    PointerPte->u.Hard.PageFrameNumber = Offset - 1;

    //
    // Write the current PTE
    //
    PointerPte += Offset;
    MI_WRITE_VALID_PTE(PointerPte, TempPte);

    //
    // Return the address
    //
    return MiPteToAddress(PointerPte);
}

Referenced by _MiWriteBackPage(), MiCopyFromUserPage(), MiCopyPageToPage(), MiReadFilePage(), MiReadPage(), MiZeroPhysicalPage(), and MmCreateHyperspaceMapping().

MiMapPagesInZeroSpace()

PVOID NTAPI MiMapPagesInZeroSpace	(	IN PMMPFN	Pfn1,
		IN PFN_NUMBER	NumberOfPages
	)

Definition at line 113 of file hypermap.c.

{
    MMPTE TempPte;
    PMMPTE PointerPte;
    PFN_NUMBER Offset, PageFrameIndex;

    //
    // Sanity checks
    //
    ASSERT(KeGetCurrentIrql() == PASSIVE_LEVEL);
    ASSERT(NumberOfPages != 0);
    ASSERT(NumberOfPages <= MI_ZERO_PTES);

    //
    // Pick the first zeroing PTE
    //
    PointerPte = MiFirstReservedZeroingPte;

    //
    // Now get the first free PTE
    //
    Offset = PFN_FROM_PTE(PointerPte);
    if (NumberOfPages > Offset)
    {
        //
        // Reset the PTEs
        //
        Offset = MI_ZERO_PTES;
        PointerPte->u.Hard.PageFrameNumber = Offset;
        KeFlushProcessTb();
    }

    //
    // Prepare the next PTE
    //
    PointerPte->u.Hard.PageFrameNumber = Offset - NumberOfPages;

    /* Choose the correct PTE to use, and which template */
    PointerPte += (Offset + 1);
    TempPte = ValidKernelPte;

    /* Make sure the list isn't empty and loop it */
    ASSERT(Pfn1 != (PVOID)LIST_HEAD);
    while (Pfn1 != (PVOID)LIST_HEAD)
    {
        /* Get the page index for this PFN */
        PageFrameIndex = MiGetPfnEntryIndex(Pfn1);

        //
        // Write the PFN
        //
        TempPte.u.Hard.PageFrameNumber = PageFrameIndex;

        //
        // Set the correct PTE to write to, and set its new value
        //
        PointerPte--;
        MI_WRITE_VALID_PTE(PointerPte, TempPte);

        /* Move to the next PFN */
        Pfn1 = (PMMPFN)Pfn1->u1.Flink;
    }

    //
    // Return the address
    //
    return MiPteToAddress(PointerPte);
}

Referenced by MmZeroPageThread().

MiRaisePoolQuota()

BOOLEAN NTAPI MiRaisePoolQuota	(	IN POOL_TYPE	PoolType,
		IN ULONG	CurrentMaxQuota,
		OUT PULONG	NewMaxQuota
	)

Definition at line 1266 of file pool.c.

{
    //
    // Not implemented
    //
    UNIMPLEMENTED;
    *NewMaxQuota = CurrentMaxQuota + 65536;
    return TRUE;
}

MiReadPageFile()

NTSTATUS NTAPI MiReadPageFile	(	_In_ PFN_NUMBER	Page,
		_In_ ULONG	PageFileIndex,
		_In_ ULONG_PTR	PageFileOffset
	)

Definition at line 201 of file pagefile.c.

{
    LARGE_INTEGER file_offset;
    IO_STATUS_BLOCK Iosb;
    NTSTATUS Status;
    KEVENT Event;
    UCHAR MdlBase[sizeof(MDL) + sizeof(ULONG)];
    PMDL Mdl = (PMDL)MdlBase;
    PMMPAGING_FILE PagingFile;

    DPRINT("MiReadSwapFile\n");

    if (PageFileOffset == 0)
    {
        KeBugCheck(MEMORY_MANAGEMENT);
        return(STATUS_UNSUCCESSFUL);
    }

    ASSERT(PageFileIndex < MAX_PAGING_FILES);

    PagingFile = MmPagingFile[PageFileIndex];

    if (PagingFile->FileObject == NULL || PagingFile->FileObject->DeviceObject == NULL)
    {
        DPRINT1("Bad paging file %u\n", PageFileIndex);
        KeBugCheck(MEMORY_MANAGEMENT);
    }

    MmInitializeMdl(Mdl, NULL, PAGE_SIZE);
    MmBuildMdlFromPages(Mdl, &Page);
    Mdl->MdlFlags |= MDL_PAGES_LOCKED;

    file_offset.QuadPart = PageFileOffset * PAGE_SIZE;

    KeInitializeEvent(&Event, NotificationEvent, FALSE);
    Status = IoPageRead(PagingFile->FileObject,
                        Mdl,
                        &file_offset,
                        &Event,
                        &Iosb);
    if (Status == STATUS_PENDING)
    {
        KeWaitForSingleObject(&Event, Executive, KernelMode, FALSE, NULL);
        Status = Iosb.Status;
    }
    if (Mdl->MdlFlags & MDL_MAPPED_TO_SYSTEM_VA)
    {
        MmUnmapLockedPages (Mdl->MappedSystemVa, Mdl);
    }
    return(Status);
}

Referenced by MiResolvePageFileFault(), and MmReadFromSwapPage().

MiReleasePfnLock()

FORCEINLINE VOID MiReleasePfnLock

(

_In_ KIRQL

OldIrql

)

Definition at line 908 of file mm.h.

{
    KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
}

Referenced by _MiFlushMappedSection(), MiAddDescriptorToDatabase(), MiAllocatePagesForMdl(), MiAllocatePoolPages(), MiBalancerThread(), MiBuildPagedPool(), MiBuildPfnDatabaseFromLoaderBlock(), MiCheckControlArea(), MiCheckPurgeAndUpMapCount(), MiCompleteProtoPteFault(), MiDeleteSystemPageableVm(), MiDeleteVirtualAddresses(), MiDispatchFault(), MiFillSystemPageDirectory(), MiFindContiguousPages(), MiFlushTbAndCapture(), MiFreeContiguousMemory(), MiGetPageProtection(), MiGetPteForProcess(), MiInitializeAndChargePfn(), MiInitMachineDependent(), MiLoadImageSection(), MiMakeSystemAddressValidPfn(), MiMapLockedPagesInUserSpace(), MiProcessValidPteList(), MiProtectVirtualMemory(), MiReleaseProcessReferenceToSessionDataPage(), MiRemoveMappedPtes(), MiResolveDemandZeroFault(), MiResolvePageFileFault(), MiResolveProtoPteFault(), MiSegmentDelete(), MiSessionCommitPageTables(), MiSessionCreateInternal(), MiSessionInitializeWorkingSetList(), MiSetControlAreaSymbolsLoaded(), MiUnmapLockedPagesInUserSpace(), MmAllocateSpecialPool(), MmAllocPage(), MmArmAccessFault(), MmCreateArm3Section(), MmCreateKernelStack(), MmCreateProcessAddressSpace(), MmDeleteKernelStack(), MmDeleteProcessAddressSpace(), MmDeleteProcessAddressSpace2(), MmDereferencePage(), MmFreeLoaderBlock(), MmFreeMemoryArea(), MmFreePagesFromMdl(), MmFreeSpecialPool(), MmGetLRUFirstUserPage(), MmGetLRUNextUserPage(), MmGetPhysicalMemoryRanges(), MmGetReferenceCountPage(), MmGetRmapListHeadPage(), MmGetSavedSwapEntryPage(), MmGrowKernelStackEx(), MmInitializeProcessAddressSpace(), MmInsertLRULastUserPage(), MmNotPresentFaultCachePage(), MmPageOutSectionView(), MmProbeAndLockPages(), MmRemoveLRUUserPage(), MmSetRmapListHeadPage(), MmSetSavedSwapEntryPage(), MmUnlockPages(), and MmZeroPageThread().

MiReleasePfnLockFromDpcLevel()

FORCEINLINE VOID MiReleasePfnLockFromDpcLevel

(

VOID

)

Definition at line 927 of file mm.h.

{
    PKSPIN_LOCK_QUEUE LockQueue;

    LockQueue = &KeGetCurrentPrcb()->LockQueue[LockQueuePfnLock];
    KeReleaseQueuedSpinLockFromDpcLevel(LockQueue);
    ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL);
}

Referenced by MiAllocatePoolPages().

MiReloadBootLoadedDrivers()

INIT_FUNCTION VOID NTAPI MiReloadBootLoadedDrivers

(

IN PLOADER_PARAMETER_BLOCK

LoaderBlock

)

Definition at line 1686 of file sysldr.c.

{
    PLIST_ENTRY NextEntry;
    ULONG i = 0;
    PIMAGE_NT_HEADERS NtHeader;
    PLDR_DATA_TABLE_ENTRY LdrEntry;
    PIMAGE_FILE_HEADER FileHeader;
    BOOLEAN ValidRelocs;
    PIMAGE_DATA_DIRECTORY DataDirectory;
    PVOID DllBase, NewImageAddress;
    NTSTATUS Status;
    PMMPTE PointerPte, StartPte, LastPte;
    PFN_COUNT PteCount;
    PMMPFN Pfn1;
    MMPTE TempPte, OldPte;

    /* Loop driver list */
    for (NextEntry = LoaderBlock->LoadOrderListHead.Flink;
         NextEntry != &LoaderBlock->LoadOrderListHead;
         NextEntry = NextEntry->Flink)
    {
        /* Get the loader entry and NT header */
        LdrEntry = CONTAINING_RECORD(NextEntry,
                                     LDR_DATA_TABLE_ENTRY,
                                     InLoadOrderLinks);
        NtHeader = RtlImageNtHeader(LdrEntry->DllBase);

        /* Debug info */
        DPRINT("[Mm0]: Driver at: %p ending at: %p for module: %wZ\n",
                LdrEntry->DllBase,
                (ULONG_PTR)LdrEntry->DllBase + LdrEntry->SizeOfImage,
                &LdrEntry->FullDllName);

        /* Get the first PTE and the number of PTEs we'll need */
        PointerPte = StartPte = MiAddressToPte(LdrEntry->DllBase);
        PteCount = ROUND_TO_PAGES(LdrEntry->SizeOfImage) >> PAGE_SHIFT;
        LastPte = StartPte + PteCount;

#if MI_TRACE_PFNS
        /* Loop the PTEs */
        while (PointerPte < LastPte)
        {
            ULONG len;
            ASSERT(PointerPte->u.Hard.Valid == 1);
            Pfn1 = MiGetPfnEntry(PFN_FROM_PTE(PointerPte));
            len = wcslen(LdrEntry->BaseDllName.Buffer) * sizeof(WCHAR);
            snprintf(Pfn1->ProcessName, min(16, len), "%S", LdrEntry->BaseDllName.Buffer);
            PointerPte++;
        }
#endif
        /* Skip kernel and HAL */
        /* ROS HACK: Skip BOOTVID/KDCOM too */
        i++;
        if (i <= 4) continue;

        /* Skip non-drivers */
        if (!NtHeader) continue;

        /* Get the file header and make sure we can relocate */
        FileHeader = &NtHeader->FileHeader;
        if (FileHeader->Characteristics & IMAGE_FILE_RELOCS_STRIPPED) continue;
        if (NtHeader->OptionalHeader.NumberOfRvaAndSizes <
            IMAGE_DIRECTORY_ENTRY_BASERELOC) continue;

        /* Everything made sense until now, check the relocation section too */
        DataDirectory = &NtHeader->OptionalHeader.
                        DataDirectory[IMAGE_DIRECTORY_ENTRY_BASERELOC];
        if (!DataDirectory->VirtualAddress)
        {
            /* We don't really have relocations */
            ValidRelocs = FALSE;
        }
        else
        {
            /* Make sure the size is valid */
            if ((DataDirectory->VirtualAddress + DataDirectory->Size) >
                LdrEntry->SizeOfImage)
            {
                /* They're not, skip */
                continue;
            }

            /* We have relocations */
            ValidRelocs = TRUE;
        }

        /* Remember the original address */
        DllBase = LdrEntry->DllBase;

        /* Loop the PTEs */
        PointerPte = StartPte;
        while (PointerPte < LastPte)
        {
            /* Mark the page modified in the PFN database */
            ASSERT(PointerPte->u.Hard.Valid == 1);
            Pfn1 = MiGetPfnEntry(PFN_FROM_PTE(PointerPte));
            ASSERT(Pfn1->u3.e1.Rom == 0);
            Pfn1->u3.e1.Modified = TRUE;

            /* Next */
            PointerPte++;
        }

        /* Now reserve system PTEs for the image */
        PointerPte = MiReserveSystemPtes(PteCount, SystemPteSpace);
        if (!PointerPte)
        {
            /* Shouldn't happen */
            ERROR_FATAL("[Mm0]: Couldn't allocate driver section!\n");
            return;
        }

        /* This is the new virtual address for the module */
        LastPte = PointerPte + PteCount;
        NewImageAddress = MiPteToAddress(PointerPte);

        /* Sanity check */
        DPRINT("[Mm0]: Copying from: %p to: %p\n", DllBase, NewImageAddress);
        ASSERT(ExpInitializationPhase == 0);

        /* Loop the new driver PTEs */
        TempPte = ValidKernelPte;
        while (PointerPte < LastPte)
        {
            /* Copy the old data */
            OldPte = *StartPte;
            ASSERT(OldPte.u.Hard.Valid == 1);

            /* Set page number from the loader's memory */
            TempPte.u.Hard.PageFrameNumber = OldPte.u.Hard.PageFrameNumber;

            /* Write it */
            MI_WRITE_VALID_PTE(PointerPte, TempPte);

            /* Move on */
            PointerPte++;
            StartPte++;
        }

        /* Update position */
        PointerPte -= PteCount;

        /* Sanity check */
        ASSERT(*(PULONG)NewImageAddress == *(PULONG)DllBase);

        /* Set the image base to the address where the loader put it */
        NtHeader->OptionalHeader.ImageBase = (ULONG_PTR)DllBase;

        /* Check if we had relocations */
        if (ValidRelocs)
        {
            /* Relocate the image */
            Status = LdrRelocateImageWithBias(NewImageAddress,
                                              0,
                                              "SYSLDR",
                                              STATUS_SUCCESS,
                                              STATUS_CONFLICTING_ADDRESSES,
                                              STATUS_INVALID_IMAGE_FORMAT);
            if (!NT_SUCCESS(Status))
            {
                /* This shouldn't happen */
                ERROR_FATAL("Relocations failed!\n");
                return;
            }
        }

        /* Update the loader entry */
        LdrEntry->DllBase = NewImageAddress;

        /* Update the thunks */
        DPRINT("[Mm0]: Updating thunks to: %wZ\n", &LdrEntry->BaseDllName);
        MiUpdateThunks(LoaderBlock,
                       DllBase,
                       NewImageAddress,
                       LdrEntry->SizeOfImage);

        /* Update the loader entry */
        LdrEntry->Flags |= LDRP_SYSTEM_MAPPED;
        LdrEntry->EntryPoint = (PVOID)((ULONG_PTR)NewImageAddress +
                                NtHeader->OptionalHeader.AddressOfEntryPoint);
        LdrEntry->SizeOfImage = PteCount << PAGE_SHIFT;

        /* FIXME: We'll need to fixup the PFN linkage when switching to ARM3 */
    }
}

Referenced by MmArmInitSystem().

MiRosCheckMemoryAreas()

VOID NTAPI MiRosCheckMemoryAreas

(

PMMSUPPORT

AddressSpace

)

MiRosCleanupMemoryArea()

VOID NTAPI MiRosCleanupMemoryArea	(	PEPROCESS	Process,
		PMMVAD	Vad
	)

Definition at line 522 of file marea.c.

{
    PMEMORY_AREA MemoryArea;
    PVOID BaseAddress;
    NTSTATUS Status;

    /* We must be called from MmCleanupAddressSpace and nowhere else!
       Make sure things are as expected... */
    ASSERT(Process == PsGetCurrentProcess());
    ASSERT(Process->VmDeleted == TRUE);
    ASSERT(((PsGetCurrentThread()->ThreadsProcess == Process) &&
            (Process->ActiveThreads == 1)) ||
           (Process->ActiveThreads == 0));

    /* We are in cleanup, we don't need to synchronize */
    MmUnlockAddressSpace(&Process->Vm);

    MemoryArea = (PMEMORY_AREA)Vad;
    BaseAddress = (PVOID)MA_GetStartingAddress(MemoryArea);

    if (MemoryArea->Type == MEMORY_AREA_SECTION_VIEW)
    {
        Status = MiRosUnmapViewOfSection(Process, BaseAddress, Process->ProcessExiting);
    }
    else if (MemoryArea->Type == MEMORY_AREA_CACHE)
    {
        Status = MmUnmapViewOfCacheSegment(&Process->Vm, BaseAddress);
    }
    else
    {
        /* There shouldn't be anything else! */
        ASSERT(FALSE);
    }

    /* Make sure this worked! */
    ASSERT(NT_SUCCESS(Status));

    /* Lock the address space again */
    MmLockAddressSpace(&Process->Vm);
}

Referenced by MmCleanProcessAddressSpace().

MiUnmapPageInHyperSpace()

VOID NTAPI MiUnmapPageInHyperSpace	(	IN PEPROCESS	Process,
		IN PVOID	Address,
		IN KIRQL	OldIrql
	)

Definition at line 93 of file hypermap.c.

{
    ASSERT(Process == PsGetCurrentProcess());

    //
    // Blow away the mapping
    //
    MiAddressToPte(Address)->u.Long = 0;

    //
    // Release the hyperlock
    //
    ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
    KeReleaseSpinLock(&Process->HyperSpaceLock, OldIrql);
}

Referenced by _MiWriteBackPage(), MiCopyFromUserPage(), MiCopyPageToPage(), MiReadFilePage(), MiReadPage(), and MiZeroPhysicalPage().

MiUnmapPagesInZeroSpace()

VOID NTAPI MiUnmapPagesInZeroSpace	(	IN PVOID	VirtualAddress,
		IN PFN_NUMBER	NumberOfPages
	)

Definition at line 185 of file hypermap.c.

{
    PMMPTE PointerPte;

    //
    // Sanity checks
    //
    ASSERT(KeGetCurrentIrql() == PASSIVE_LEVEL);
    ASSERT (NumberOfPages != 0);
    ASSERT(NumberOfPages <= MI_ZERO_PTES);

    //
    // Get the first PTE for the mapped zero VA
    //
    PointerPte = MiAddressToPte(VirtualAddress);

    //
    // Blow away the mapped zero PTEs
    //
    RtlZeroMemory(PointerPte, NumberOfPages * sizeof(MMPTE));
}

Referenced by MmZeroPageThread().

MmAccessFault()

NTSTATUS NTAPI MmAccessFault	(	IN ULONG	FaultCode,
		IN PVOID	Address,
		IN KPROCESSOR_MODE	Mode,
		IN PVOID	TrapInformation
	)

Definition at line 204 of file mmfault.c.

{
    PMEMORY_AREA MemoryArea = NULL;

    /* Cute little hack for ROS */
    if ((ULONG_PTR)Address >= (ULONG_PTR)MmSystemRangeStart)
    {
#ifdef _M_IX86
        /* Check for an invalid page directory in kernel mode */
        if (Mmi386MakeKernelPageTableGlobal(Address))
        {
            /* All is well with the world */
            return STATUS_SUCCESS;
        }