This graph shows which files directly or indirectly include this file:

Classes
struct	_POOL_DESCRIPTOR

struct	_POOL_HEADER

struct	_POOL_TRACKER_TABLE

struct	_POOL_TRACKER_BIG_PAGES

struct	_MI_LARGE_PAGE_DRIVER_ENTRY

struct	_PHYSICAL_MEMORY_RUN

struct	_PHYSICAL_MEMORY_DESCRIPTOR

struct	_MMCOLOR_TABLES

struct	_MI_LARGE_PAGE_RANGES

struct	_MMVIEW

struct	_MMSESSION

struct	_MM_SESSION_SPACE_FLAGS

struct	_MM_SESSION_SPACE

Macros
#define	MI_LOWEST_VAD_ADDRESS (PVOID)MM_LOWEST_USER_ADDRESS

#define	_1KB (1024u)

#define	_1MB (1024 * _1KB)

#define	_1GB (1024 * _1MB)

#define	_64K (64 * _1KB)

#define	PT_SIZE (PTE_PER_PAGE * sizeof(MMPTE))

#define	PD_SIZE (PDE_PER_PAGE * sizeof(MMPDE))

#define	SYSTEM_PD_SIZE (PPE_PER_PAGE * PD_SIZE)

#define	MM_ZERO_ACCESS 0

#define	MM_READONLY 1

#define	MM_EXECUTE 2

#define	MM_EXECUTE_READ 3

#define	MM_READWRITE 4

#define	MM_WRITECOPY 5

#define	MM_EXECUTE_READWRITE 6

#define	MM_EXECUTE_WRITECOPY 7

#define	MM_PROTECT_ACCESS 7

#define	MM_NOCACHE 0x08

#define	MM_GUARDPAGE 0x10

#define	MM_WRITECOMBINE 0x18

#define	MM_PROTECT_SPECIAL 0x18

#define	MM_DECOMMIT (MM_ZERO_ACCESS \| MM_GUARDPAGE)

#define	MM_NOACCESS (MM_ZERO_ACCESS \| MM_WRITECOMBINE)

#define	MM_OUTSWAPPED_KSTACK (MM_EXECUTE_WRITECOPY \| MM_WRITECOMBINE)

#define	MM_INVALID_PROTECTION 0xFFFFFFFF

#define	PTE_MAPPING_NONE 0x100

#define	PTE_MAPPING_NOT_OWNED 0x101

#define	PTE_MAPPING_EMPTY 0x102

#define	PTE_MAPPING_RESERVED 0x103

#define	PTE_MAPPING_ADDRESS_NOT_OWNED 0x104

#define	PTE_MAPPING_ADDRESS_INVALID 0x105

#define	PTE_UNMAPPING_ADDRESS_NOT_OWNED 0x108

#define	PTE_MAPPING_ADDRESS_EMPTY 0x109

#define	IMAGE_SCN_PROTECTION_MASK (IMAGE_SCN_MEM_WRITE \| IMAGE_SCN_MEM_READ \| IMAGE_SCN_MEM_EXECUTE)

#define	MI_IS_SESSION_IMAGE_ADDRESS(Address) (((Address) >= MiSessionImageStart) && ((Address) < MiSessionImageEnd))

#define	MI_IS_SESSION_ADDRESS(Address) (((Address) >= MmSessionBase) && ((Address) < MiSessionSpaceEnd))

#define	MI_IS_SESSION_PTE(Pte) ((((PMMPTE)Pte) >= MiSessionBasePte) && (((PMMPTE)Pte) < MiSessionLastPte))

#define	MI_IS_PAGE_TABLE_ADDRESS(Address) (((PVOID)(Address) >= (PVOID)PTE_BASE) && ((PVOID)(Address) <= (PVOID)PTE_TOP))

#define	MI_IS_SYSTEM_PAGE_TABLE_ADDRESS(Address) (((Address) >= (PVOID)MiAddressToPte(MmSystemRangeStart)) && ((Address) <= (PVOID)PTE_TOP))

#define	MI_IS_PAGE_TABLE_OR_HYPER_ADDRESS(Address) (((PVOID)(Address) >= (PVOID)PTE_BASE) && ((PVOID)(Address) <= (PVOID)MmHyperSpaceEnd))

#define	MI_MAKE_SOFTWARE_PTE(p, x) ((p)->u.Long = (x << MM_PTE_SOFTWARE_PROTECTION_BITS))

#define	MI_SET_PFN_DELETED(x) ((x)->PteAddress = (PMMPTE)((ULONG_PTR)(x)->PteAddress \| 1))

#define	MI_IS_PFN_DELETED(x) ((ULONG_PTR)((x)->PteAddress) & 1)

#define	MM_SYSLDR_NO_IMPORTS ((PVOID)(ULONG_PTR)-2)

#define	MM_SYSLDR_BOOT_LOADED ((PVOID)(ULONG_PTR)-1)

#define	MM_SYSLDR_SINGLE_ENTRY 0x1

#define	MI_INITIAL_SESSION_IDS 64

#define	MI_GET_PAGE_COLOR(x) ((x) & MmSecondaryColorMask)

#define	MI_GET_NEXT_COLOR() (MI_GET_PAGE_COLOR(++MmSystemPageColor))

#define	MI_GET_NEXT_PROCESS_COLOR(x) (MI_GET_PAGE_COLOR(++(x)->NextPageColor))

#define	MI_PTE_LOOKUP_NEEDED 0xFFFFF

#define	MI_SESSION_DATA_PAGES_MAXIMUM (MM_ALLOCATION_GRANULARITY / PAGE_SIZE)

#define	MI_SESSION_TAG_PAGES_MAXIMUM (MM_ALLOCATION_GRANULARITY / PAGE_SIZE)

#define	MM_READ_WRITE_ALLOWED 11

#define	MM_READ_ONLY_ALLOWED 10

#define	MM_NO_ACCESS_ALLOWED 01

#define	MM_DELETE_CHECK 85

#define	MI_SYSTEM_VIEW_BUCKET_SIZE _64K

#define	POOL_BLOCK_SIZE 8

#define	POOL_LISTS_PER_PAGE (PAGE_SIZE / POOL_BLOCK_SIZE)

#define	BASE_POOL_TYPE_MASK 1

#define	POOL_MAX_ALLOC (PAGE_SIZE - (sizeof(POOL_HEADER) + POOL_BLOCK_SIZE))

#define	POOL_FLAG_CHECK_TIMERS 0x1

#define	POOL_FLAG_CHECK_WORKERS 0x2

#define	POOL_FLAG_CHECK_RESOURCES 0x4

#define	POOL_FLAG_VERIFIER 0x8

#define	POOL_FLAG_CHECK_DEADLOCK 0x10

#define	POOL_FLAG_SPECIAL_POOL 0x20

#define	POOL_FLAG_DBGPRINT_ON_FAILURE 0x40

#define	POOL_FLAG_CRASH_ON_FAILURE 0x80

#define	POOL_CORRUPTED_LIST 3

#define	POOL_SIZE_OR_INDEX_MISMATCH 5

#define	POOL_ENTRIES_NOT_ALIGNED_PREVIOUS 6

#define	POOL_HEADER_NOT_ALIGNED 7

#define	POOL_HEADER_IS_ZERO 8

#define	POOL_ENTRIES_NOT_ALIGNED_NEXT 9

#define	POOL_ENTRY_NOT_FOUND 10

#define	POOL_ENTRY_CORRUPTED 1

#define	POOL_ENTRY_ALREADY_FREE 6

#define	POOL_ENTRY_NOT_ALLOCATED 7

#define	POOL_ALLOC_IRQL_INVALID 8

#define	POOL_FREE_IRQL_INVALID 9

#define	POOL_BILLED_PROCESS_INVALID 13

#define	POOL_HEADER_SIZE_INVALID 32

#define	MI_IS_ROS_PFN(x) ((x)->u4.AweAllocation == TRUE)

Typedefs
typedef struct _POOL_DESCRIPTOR	POOL_DESCRIPTOR

typedef struct _POOL_DESCRIPTOR *	PPOOL_DESCRIPTOR

typedef struct _POOL_HEADER	POOL_HEADER

typedef struct _POOL_HEADER *	PPOOL_HEADER

typedef struct _POOL_TRACKER_TABLE	POOL_TRACKER_TABLE

typedef struct _POOL_TRACKER_TABLE *	PPOOL_TRACKER_TABLE

typedef struct _POOL_TRACKER_BIG_PAGES	POOL_TRACKER_BIG_PAGES

typedef struct _POOL_TRACKER_BIG_PAGES *	PPOOL_TRACKER_BIG_PAGES

typedef struct _MI_LARGE_PAGE_DRIVER_ENTRY	MI_LARGE_PAGE_DRIVER_ENTRY

typedef struct _MI_LARGE_PAGE_DRIVER_ENTRY *	PMI_LARGE_PAGE_DRIVER_ENTRY

typedef enum _MMSYSTEM_PTE_POOL_TYPE	MMSYSTEM_PTE_POOL_TYPE

typedef enum _MI_PFN_CACHE_ATTRIBUTE	MI_PFN_CACHE_ATTRIBUTE

typedef enum _MI_PFN_CACHE_ATTRIBUTE *	PMI_PFN_CACHE_ATTRIBUTE

typedef struct _PHYSICAL_MEMORY_RUN	PHYSICAL_MEMORY_RUN

typedef struct _PHYSICAL_MEMORY_RUN *	PPHYSICAL_MEMORY_RUN

typedef struct _PHYSICAL_MEMORY_DESCRIPTOR	PHYSICAL_MEMORY_DESCRIPTOR

typedef struct _PHYSICAL_MEMORY_DESCRIPTOR *	PPHYSICAL_MEMORY_DESCRIPTOR

typedef struct _MMCOLOR_TABLES	MMCOLOR_TABLES

typedef struct _MMCOLOR_TABLES *	PMMCOLOR_TABLES

typedef struct _MI_LARGE_PAGE_RANGES	MI_LARGE_PAGE_RANGES

typedef struct _MI_LARGE_PAGE_RANGES *	PMI_LARGE_PAGE_RANGES

typedef struct _MMVIEW	MMVIEW

typedef struct _MMVIEW *	PMMVIEW

typedef struct _MMSESSION	MMSESSION

typedef struct _MMSESSION *	PMMSESSION

typedef struct _MM_SESSION_SPACE_FLAGS	MM_SESSION_SPACE_FLAGS

typedef struct _MM_SESSION_SPACE	MM_SESSION_SPACE

typedef struct _MM_SESSION_SPACE *	PMM_SESSION_SPACE

Enumerations
enum	_MMSYSTEM_PTE_POOL_TYPE { SystemPteSpace , NonPagedPoolExpansion , MaximumPtePoolTypes }

enum	_MI_PFN_CACHE_ATTRIBUTE { MiNonCached , MiCached , MiWriteCombined , MiNotMapped }

Functions
	C_ASSERT (sizeof(POOL_HEADER)==POOL_BLOCK_SIZE)

	C_ASSERT (POOL_BLOCK_SIZE==sizeof(LIST_ENTRY))

FORCEINLINE BOOLEAN	MI_IS_PROCESS_WORKING_SET (PMMSUPPORT WorkingSet)

FORCEINLINE BOOLEAN	MiIsMemoryTypeFree (TYPE_OF_MEMORY MemoryType)

FORCEINLINE BOOLEAN	MiIsMemoryTypeInvisible (TYPE_OF_MEMORY MemoryType)

FORCEINLINE BOOLEAN	MiIsUserPde (PVOID Address)

FORCEINLINE BOOLEAN	MiIsUserPte (PVOID Address)

FORCEINLINE ULONG_PTR	MiDetermineUserGlobalPteMask (IN PVOID PointerPte)

FORCEINLINE VOID	MI_MAKE_HARDWARE_PTE_KERNEL (IN PMMPTE NewPte, IN PMMPTE MappingPte, IN ULONG_PTR ProtectionMask, IN PFN_NUMBER PageFrameNumber)

FORCEINLINE VOID	MI_MAKE_HARDWARE_PTE (IN PMMPTE NewPte, IN PMMPTE MappingPte, IN ULONG_PTR ProtectionMask, IN PFN_NUMBER PageFrameNumber)

FORCEINLINE VOID	MI_MAKE_HARDWARE_PTE_USER (IN PMMPTE NewPte, IN PMMPTE MappingPte, IN ULONG_PTR ProtectionMask, IN PFN_NUMBER PageFrameNumber)

FORCEINLINE VOID	MI_MAKE_PROTOTYPE_PTE (IN PMMPTE NewPte, IN PMMPTE PointerPte)

FORCEINLINE VOID	MI_MAKE_SUBSECTION_PTE (IN PMMPTE NewPte, IN PVOID Segment)

FORCEINLINE BOOLEAN	MI_IS_MAPPED_PTE (PMMPTE PointerPte)

FORCEINLINE VOID	MI_MAKE_TRANSITION_PTE (_Out_ PMMPTE NewPte, _In_ PFN_NUMBER Page, _In_ ULONG Protection)

FORCEINLINE BOOLEAN	MI_IS_PHYSICAL_ADDRESS (IN PVOID Address)

FORCEINLINE VOID	MI_WRITE_VALID_PTE (IN PMMPTE PointerPte, IN MMPTE TempPte)

FORCEINLINE VOID	MI_UPDATE_VALID_PTE (IN PMMPTE PointerPte, IN MMPTE TempPte)

FORCEINLINE VOID	MI_WRITE_INVALID_PTE (IN PMMPTE PointerPte, IN MMPTE InvalidPte)

FORCEINLINE VOID	MI_ERASE_PTE (IN PMMPTE PointerPte)

FORCEINLINE VOID	MI_WRITE_VALID_PDE (IN PMMPDE PointerPde, IN MMPDE TempPde)

FORCEINLINE VOID	MI_WRITE_INVALID_PDE (IN PMMPDE PointerPde, IN MMPDE InvalidPde)

FORCEINLINE BOOLEAN	MM_ANY_WS_LOCK_HELD (IN PETHREAD Thread)

FORCEINLINE BOOLEAN	MM_ANY_WS_LOCK_HELD_EXCLUSIVE (_In_ PETHREAD Thread)

FORCEINLINE BOOLEAN	MI_WS_OWNER (IN PEPROCESS Process)

FORCEINLINE BOOLEAN	MiIsRosSectionObject (IN PSECTION Section)

VOID NTAPI	MiDecrementReferenceCount (IN PMMPFN Pfn1, IN PFN_NUMBER PageFrameIndex)

FORCEINLINE BOOLEAN	MI_IS_WS_UNSAFE (IN PEPROCESS Process)

FORCEINLINE VOID	MiLockProcessWorkingSet (IN PEPROCESS Process, IN PETHREAD Thread)

FORCEINLINE VOID	MiLockProcessWorkingSetShared (IN PEPROCESS Process, IN PETHREAD Thread)

FORCEINLINE VOID	MiLockProcessWorkingSetUnsafe (IN PEPROCESS Process, IN PETHREAD Thread)

FORCEINLINE VOID	MiUnlockProcessWorkingSet (IN PEPROCESS Process, IN PETHREAD Thread)

FORCEINLINE VOID	MiUnlockProcessWorkingSetShared (IN PEPROCESS Process, IN PETHREAD Thread)

FORCEINLINE VOID	MiUnlockProcessWorkingSetUnsafe (IN PEPROCESS Process, IN PETHREAD Thread)

FORCEINLINE VOID	MiLockWorkingSet (IN PETHREAD Thread, IN PMMSUPPORT WorkingSet)

FORCEINLINE VOID	MiLockWorkingSetShared (_In_ PETHREAD Thread, _In_ PMMSUPPORT WorkingSet)

FORCEINLINE VOID	MiUnlockWorkingSet (IN PETHREAD Thread, IN PMMSUPPORT WorkingSet)

FORCEINLINE VOID	MiUnlockWorkingSetShared (_In_ PETHREAD Thread, _In_ PMMSUPPORT WorkingSet)

FORCEINLINE BOOLEAN	MiConvertSharedWorkingSetLockToExclusive (_In_ PETHREAD Thread, _In_ PMMSUPPORT Vm)

FORCEINLINE VOID	MiUnlockProcessWorkingSetForFault (IN PEPROCESS Process, IN PETHREAD Thread, OUT PBOOLEAN Safe, OUT PBOOLEAN Shared)

FORCEINLINE VOID	MiLockProcessWorkingSetForFault (IN PEPROCESS Process, IN PETHREAD Thread, IN BOOLEAN Safe, IN BOOLEAN Shared)

FORCEINLINE KIRQL	MiAcquireExpansionLock (VOID)

FORCEINLINE VOID	MiReleaseExpansionLock (KIRQL OldIrql)

FORCEINLINE PMMPTE	MI_GET_PROTOTYPE_PTE_FOR_VPN (IN PMMVAD Vad, IN ULONG_PTR Vpn)

FORCEINLINE PMMPFN	MI_PFN_ELEMENT (IN PFN_NUMBER Pfn)

FORCEINLINE VOID	MiDropLockCount (IN PMMPFN Pfn1)

FORCEINLINE VOID	MiDereferencePfnAndDropLockCount (IN PMMPFN Pfn1)

FORCEINLINE VOID	MiReferenceProbedPageAndBumpLockCount (IN PMMPFN Pfn1)

FORCEINLINE VOID	MiReferenceUsedPageAndBumpLockCount (IN PMMPFN Pfn1)

FORCEINLINE VOID	MiReferenceUnusedPageAndBumpLockCount (IN PMMPFN Pfn1)

BOOLEAN NTAPI	MmArmInitSystem (IN ULONG Phase, IN PLOADER_PARAMETER_BLOCK LoaderBlock)

VOID NTAPI	MiInitializeSessionSpaceLayout (VOID)

NTSTATUS NTAPI	MiInitMachineDependent (IN PLOADER_PARAMETER_BLOCK LoaderBlock)

VOID NTAPI	MiComputeColorInformation (VOID)

VOID NTAPI	MiMapPfnDatabase (IN PLOADER_PARAMETER_BLOCK LoaderBlock)

VOID NTAPI	MiInitializeColorTables (VOID)

VOID NTAPI	MiInitializePfnDatabase (IN PLOADER_PARAMETER_BLOCK LoaderBlock)

VOID NTAPI	MiInitializeSessionWsSupport (VOID)

VOID NTAPI	MiInitializeSessionIds (VOID)

BOOLEAN NTAPI	MiInitializeMemoryEvents (VOID)

PFN_NUMBER NTAPI	MxGetNextPage (IN PFN_NUMBER PageCount)

PPHYSICAL_MEMORY_DESCRIPTOR NTAPI	MmInitializeMemoryLimits (IN PLOADER_PARAMETER_BLOCK LoaderBlock, IN PBOOLEAN IncludeType)

PFN_NUMBER NTAPI	MiPagesInLoaderBlock (IN PLOADER_PARAMETER_BLOCK LoaderBlock, IN PBOOLEAN IncludeType)

VOID FASTCALL	MiSyncARM3WithROS (IN PVOID AddressStart, IN PVOID AddressEnd)

NTSTATUS NTAPI	MiRosProtectVirtualMemory (IN PEPROCESS Process, IN OUT PVOID *BaseAddress, IN OUT PSIZE_T NumberOfBytesToProtect, IN ULONG NewAccessProtection, OUT PULONG OldAccessProtection OPTIONAL)

NTSTATUS NTAPI	MmArmAccessFault (IN ULONG FaultCode, IN PVOID Address, IN KPROCESSOR_MODE Mode, IN PVOID TrapInformation)

NTSTATUS FASTCALL	MiCheckPdeForPagedPool (IN PVOID Address)

VOID NTAPI	MiInitializeNonPagedPoolThresholds (VOID)

VOID NTAPI	MiInitializePoolEvents (VOID)

VOID NTAPI	InitializePool (IN POOL_TYPE PoolType, IN ULONG Threshold)

VOID NTAPI	ExInitializePoolDescriptor (IN PPOOL_DESCRIPTOR PoolDescriptor, IN POOL_TYPE PoolType, IN ULONG PoolIndex, IN ULONG Threshold, IN PVOID PoolLock)

NTSTATUS NTAPI	MiInitializeSessionPool (VOID)

VOID NTAPI	MiInitializeSystemPtes (IN PMMPTE StartingPte, IN ULONG NumberOfPtes, IN MMSYSTEM_PTE_POOL_TYPE PoolType)

PMMPTE NTAPI	MiReserveSystemPtes (IN ULONG NumberOfPtes, IN MMSYSTEM_PTE_POOL_TYPE SystemPtePoolType)

VOID NTAPI	MiReleaseSystemPtes (IN PMMPTE StartingPte, IN ULONG NumberOfPtes, IN MMSYSTEM_PTE_POOL_TYPE SystemPtePoolType)

PFN_NUMBER NTAPI	MiFindContiguousPages (IN PFN_NUMBER LowestPfn, IN PFN_NUMBER HighestPfn, IN PFN_NUMBER BoundaryPfn, IN PFN_NUMBER SizeInPages, IN MEMORY_CACHING_TYPE CacheType)

PVOID NTAPI	MiCheckForContiguousMemory (IN PVOID BaseAddress, IN PFN_NUMBER BaseAddressPages, IN PFN_NUMBER SizeInPages, IN PFN_NUMBER LowestPfn, IN PFN_NUMBER HighestPfn, IN PFN_NUMBER BoundaryPfn, IN MI_PFN_CACHE_ATTRIBUTE CacheAttribute)

PMDL NTAPI	MiAllocatePagesForMdl (IN PHYSICAL_ADDRESS LowAddress, IN PHYSICAL_ADDRESS HighAddress, IN PHYSICAL_ADDRESS SkipBytes, IN SIZE_T TotalBytes, IN MI_PFN_CACHE_ATTRIBUTE CacheAttribute, IN ULONG Flags)

VOID NTAPI	MiInsertPageInList (IN PMMPFNLIST ListHead, IN PFN_NUMBER PageFrameIndex)

VOID NTAPI	MiUnlinkFreeOrZeroedPage (IN PMMPFN Entry)

VOID NTAPI	MiUnlinkPageFromList (IN PMMPFN Pfn)

VOID NTAPI	MiInitializePfn (IN PFN_NUMBER PageFrameIndex, IN PMMPTE PointerPte, IN BOOLEAN Modified)

NTSTATUS NTAPI	MiInitializeAndChargePfn (OUT PPFN_NUMBER PageFrameIndex, IN PMMPDE PointerPde, IN PFN_NUMBER ContainingPageFrame, IN BOOLEAN SessionAllocation)

VOID NTAPI	MiInitializePfnAndMakePteValid (IN PFN_NUMBER PageFrameIndex, IN PMMPTE PointerPte, IN MMPTE TempPte)

VOID NTAPI	MiInitializePfnForOtherProcess (IN PFN_NUMBER PageFrameIndex, IN PVOID PteAddress, IN PFN_NUMBER PteFrame)

VOID NTAPI	MiDecrementShareCount (IN PMMPFN Pfn1, IN PFN_NUMBER PageFrameIndex)

PFN_NUMBER NTAPI	MiRemoveAnyPage (IN ULONG Color)

PFN_NUMBER NTAPI	MiRemoveZeroPage (IN ULONG Color)

VOID NTAPI	MiZeroPhysicalPage (IN PFN_NUMBER PageFrameIndex)

VOID NTAPI	MiInsertPageInFreeList (IN PFN_NUMBER PageFrameIndex)

PFN_COUNT NTAPI	MiDeleteSystemPageableVm (IN PMMPTE PointerPte, IN PFN_NUMBER PageCount, IN ULONG Flags, OUT PPFN_NUMBER ValidPages)

ULONG NTAPI	MiGetPageProtection (IN PMMPTE PointerPte)

PLDR_DATA_TABLE_ENTRY NTAPI	MiLookupDataTableEntry (IN PVOID Address)

VOID NTAPI	MiInitializeDriverLargePageList (VOID)

VOID NTAPI	MiInitializeLargePageSupport (VOID)

VOID NTAPI	MiSyncCachedRanges (VOID)

BOOLEAN NTAPI	MiIsPfnInUse (IN PMMPFN Pfn1)

PMMVAD NTAPI	MiLocateAddress (IN PVOID VirtualAddress)

TABLE_SEARCH_RESULT NTAPI	MiCheckForConflictingNode (IN ULONG_PTR StartVpn, IN ULONG_PTR EndVpn, IN PMM_AVL_TABLE Table, OUT PMMADDRESS_NODE *NodeOrParent)

TABLE_SEARCH_RESULT NTAPI	MiFindEmptyAddressRangeDownTree (IN SIZE_T Length, IN ULONG_PTR BoundaryAddress, IN ULONG_PTR Alignment, IN PMM_AVL_TABLE Table, OUT PULONG_PTR Base, OUT PMMADDRESS_NODE *Parent)

NTSTATUS NTAPI	MiFindEmptyAddressRangeDownBasedTree (IN SIZE_T Length, IN ULONG_PTR BoundaryAddress, IN ULONG_PTR Alignment, IN PMM_AVL_TABLE Table, OUT PULONG_PTR Base)

TABLE_SEARCH_RESULT NTAPI	MiFindEmptyAddressRangeInTree (IN SIZE_T Length, IN ULONG_PTR Alignment, IN PMM_AVL_TABLE Table, OUT PMMADDRESS_NODE *PreviousVad, OUT PULONG_PTR Base)

NTSTATUS NTAPI	MiCheckSecuredVad (IN PMMVAD Vad, IN PVOID Base, IN SIZE_T Size, IN ULONG ProtectionMask)

VOID NTAPI	MiInsertVad (_Inout_ PMMVAD Vad, _Inout_ PMM_AVL_TABLE VadRoot)

NTSTATUS NTAPI	MiInsertVadEx (_Inout_ PMMVAD Vad, _In_ ULONG_PTR *BaseAddress, _In_ SIZE_T ViewSize, _In_ ULONG_PTR HighestAddress, _In_ ULONG_PTR Alignment, _In_ ULONG AllocationType)

VOID NTAPI	MiInsertBasedSection (IN PSECTION Section)

NTSTATUS NTAPI	MiRosUnmapViewOfSection (IN PEPROCESS Process, IN PVOID BaseAddress, IN BOOLEAN SkipDebuggerNotify)

VOID NTAPI	MiInsertNode (IN PMM_AVL_TABLE Table, IN PMMADDRESS_NODE NewNode, PMMADDRESS_NODE Parent, TABLE_SEARCH_RESULT Result)

VOID NTAPI	MiRemoveNode (IN PMMADDRESS_NODE Node, IN PMM_AVL_TABLE Table)

PMMADDRESS_NODE NTAPI	MiGetPreviousNode (IN PMMADDRESS_NODE Node)

PMMADDRESS_NODE NTAPI	MiGetNextNode (IN PMMADDRESS_NODE Node)

BOOLEAN NTAPI	MiInitializeSystemSpaceMap (IN PMMSESSION InputSession OPTIONAL)

VOID NTAPI	MiSessionRemoveProcess (VOID)

VOID NTAPI	MiReleaseProcessReferenceToSessionDataPage (IN PMM_SESSION_SPACE SessionGlobal)

VOID NTAPI	MiSessionAddProcess (IN PEPROCESS NewProcess)

ULONG NTAPI	MiMakeProtectionMask (IN ULONG Protect)

VOID NTAPI	MiDeleteVirtualAddresses (IN ULONG_PTR Va, IN ULONG_PTR EndingAddress, IN PMMVAD Vad)

VOID NTAPI	MiDeletePte (IN PMMPTE PointerPte, IN PVOID VirtualAddress, IN PEPROCESS CurrentProcess, IN PMMPTE PrototypePte)

ULONG NTAPI	MiMakeSystemAddressValid (IN PVOID PageTableVirtualAddress, IN PEPROCESS CurrentProcess)

ULONG NTAPI	MiMakeSystemAddressValidPfn (IN PVOID VirtualAddress, IN KIRQL OldIrql)

VOID NTAPI	MiRemoveMappedView (IN PEPROCESS CurrentProcess, IN PMMVAD Vad)

PSUBSECTION NTAPI	MiLocateSubsection (IN PMMVAD Vad, IN ULONG_PTR Vpn)

VOID NTAPI	MiDeleteARM3Section (PVOID ObjectBody)

NTSTATUS NTAPI	MiQueryMemorySectionName (IN HANDLE ProcessHandle, IN PVOID BaseAddress, OUT PVOID MemoryInformation, IN SIZE_T MemoryInformationLength, OUT PSIZE_T ReturnLength)

NTSTATUS NTAPI	MiRosUnmapViewInSystemSpace (IN PVOID MappedBase)

VOID NTAPI	MiMakePdeExistAndMakeValid (IN PMMPDE PointerPde, IN PEPROCESS TargetProcess, IN KIRQL OldIrql)

VOID NTAPI	MiWriteProtectSystemImage (_In_ PVOID ImageBase)

FORCEINLINE PFN_NUMBER	MiRemoveZeroPageSafe (IN ULONG Color)

FORCEINLINE USHORT	MiIncrementPageTableReferences (IN PVOID Address)

FORCEINLINE USHORT	MiDecrementPageTableReferences (IN PVOID Address)

FORCEINLINE VOID	MiDeletePde (_In_ PMMPDE PointerPde, _In_ PEPROCESS CurrentProcess)

Variables
const ULONG_PTR	MmProtectToPteMask [32]

const ULONG	MmProtectToValue [32]

ULONG	ExpNumberOfPagedPools

POOL_DESCRIPTOR	NonPagedPoolDescriptor

PPOOL_DESCRIPTOR	ExpPagedPoolDescriptor [16+1]

PPOOL_TRACKER_TABLE	PoolTrackTable

PMM_SESSION_SPACE	MmSessionSpace

MMPTE	HyperTemplatePte

MMPDE	ValidKernelPde

MMPTE	ValidKernelPte

MMPDE	ValidKernelPdeLocal

MMPTE	ValidKernelPteLocal

MMPDE	DemandZeroPde

MMPTE	DemandZeroPte

MMPTE	PrototypePte

MMPTE	MmDecommittedPte

BOOLEAN	MmLargeSystemCache

BOOLEAN	MmZeroPageFile

BOOLEAN	MmProtectFreedNonPagedPool

BOOLEAN	MmTrackLockedPages

BOOLEAN	MmTrackPtes

BOOLEAN	MmDynamicPfn

BOOLEAN	MmMirroring

BOOLEAN	MmMakeLowMemory

BOOLEAN	MmEnforceWriteProtection

SIZE_T	MmAllocationFragment

ULONG	MmConsumedPoolPercentage

ULONG	MmVerifyDriverBufferType

ULONG	MmVerifyDriverLevel

WCHAR	MmVerifyDriverBuffer [512]

WCHAR	MmLargePageDriverBuffer [512]

LIST_ENTRY	MiLargePageDriverList

BOOLEAN	MiLargePageAllDrivers

ULONG	MmVerifyDriverBufferLength

ULONG	MmLargePageDriverBufferLength

SIZE_T	MmSizeOfNonPagedPoolInBytes

SIZE_T	MmMaximumNonPagedPoolInBytes

PFN_NUMBER	MmMaximumNonPagedPoolInPages

PFN_NUMBER	MmSizeOfPagedPoolInPages

PVOID	MmNonPagedSystemStart

PVOID	MmNonPagedPoolStart

PVOID	MmNonPagedPoolExpansionStart

PVOID	MmNonPagedPoolEnd

SIZE_T	MmSizeOfPagedPoolInBytes

PVOID	MmPagedPoolStart

PVOID	MmPagedPoolEnd

PVOID	MmSessionBase

SIZE_T	MmSessionSize

PMMPTE	MmFirstReservedMappingPte

PMMPTE	MmLastReservedMappingPte

PMMPTE	MiFirstReservedZeroingPte

MI_PFN_CACHE_ATTRIBUTE	MiPlatformCacheAttributes [2][MmMaximumCacheType]

PPHYSICAL_MEMORY_DESCRIPTOR	MmPhysicalMemoryBlock

SIZE_T	MmBootImageSize

PMMPTE	MmSystemPtesStart [MaximumPtePoolTypes]

PMMPTE	MmSystemPtesEnd [MaximumPtePoolTypes]

PMEMORY_ALLOCATION_DESCRIPTOR	MxFreeDescriptor

MEMORY_ALLOCATION_DESCRIPTOR	MxOldFreeDescriptor

ULONG_PTR	MxPfnAllocation

MM_PAGED_POOL_INFO	MmPagedPoolInfo

KGUARDED_MUTEX	MmPagedPoolMutex

KGUARDED_MUTEX	MmSectionCommitMutex

PVOID	MiSystemViewStart

SIZE_T	MmSystemViewSize

PVOID	MiSessionSpaceEnd

PMMPTE	MiSessionImagePteStart

PMMPTE	MiSessionImagePteEnd

PMMPTE	MiSessionBasePte

PMMPTE	MiSessionLastPte

PMMPDE	MmSystemPagePtes

PVOID	MmSystemCacheStart

PVOID	MmSystemCacheEnd

MMSUPPORT	MmSystemCacheWs

SIZE_T	MmAllocatedNonPagedPool

ULONG	MmSpecialPoolTag

PVOID	MmHyperSpaceEnd

PMMWSL	MmSystemCacheWorkingSetList

SIZE_T	MmMinimumNonPagedPoolSize

ULONG	MmMinAdditionNonPagedPoolPerMb

SIZE_T	MmDefaultMaximumNonPagedPool

ULONG	MmMaxAdditionNonPagedPoolPerMb

ULONG	MmSecondaryColors

ULONG	MmSecondaryColorMask

ULONG	MmNumberOfSystemPtes

ULONG	MmMaximumNonPagedPoolPercent

ULONG	MmLargeStackSize

PMMCOLOR_TABLES	MmFreePagesByColor [FreePageList+1]

MMPFNLIST	MmStandbyPageListByPriority [8]

ULONG	MmProductType

MM_SYSTEMSIZE	MmSystemSize

PKEVENT	MiLowMemoryEvent

PKEVENT	MiHighMemoryEvent

PKEVENT	MiLowPagedPoolEvent

PKEVENT	MiHighPagedPoolEvent

PKEVENT	MiLowNonPagedPoolEvent

PKEVENT	MiHighNonPagedPoolEvent

PFN_NUMBER	MmLowMemoryThreshold

PFN_NUMBER	MmHighMemoryThreshold

PFN_NUMBER	MiLowPagedPoolThreshold

PFN_NUMBER	MiHighPagedPoolThreshold

PFN_NUMBER	MiLowNonPagedPoolThreshold

PFN_NUMBER	MiHighNonPagedPoolThreshold

PFN_NUMBER	MmMinimumFreePages

PFN_NUMBER	MmPlentyFreePages

SIZE_T	MmMinimumStackCommitInBytes

PFN_COUNT	MiExpansionPoolPagesInitialCharge

PFN_NUMBER	MmResidentAvailableAtInit

ULONG	MmTotalFreeSystemPtes [MaximumPtePoolTypes]

PFN_NUMBER	MmTotalSystemDriverPages

ULONG	MmCritsectTimeoutSeconds

PVOID	MiSessionImageStart

PVOID	MiSessionImageEnd

PMMPTE	MiHighestUserPte

PMMPDE	MiHighestUserPde

PFN_NUMBER	MmSystemPageDirectory [PPE_PER_PAGE]

PMMPTE	MmSharedUserDataPte

LIST_ENTRY	MmProcessList

KEVENT	MmZeroingPageEvent

ULONG	MmSystemPageColor

ULONG	MmProcessColorSeed

PMMWSL	MmWorkingSetList

PFN_NUMBER	MiNumberOfFreePages

SIZE_T	MmSessionViewSize

SIZE_T	MmSessionPoolSize

SIZE_T	MmSessionImageSize

PVOID	MiSessionPoolEnd

PVOID	MiSessionPoolStart

PVOID	MiSessionViewStart

PVOID	MiSessionSpaceWs

ULONG	MmMaximumDeadKernelStacks

SLIST_HEADER	MmDeadStackSListHead

MM_AVL_TABLE	MmSectionBasedRoot

KGUARDED_MUTEX	MmSectionBasedMutex

PVOID	MmHighSectionBase

SIZE_T	MmSystemLockPagesCount

ULONG_PTR	MmSubsectionBase

LARGE_INTEGER	MmCriticalSectionTimeout

LIST_ENTRY	MmWorkingSetExpansionHead

KSPIN_LOCK	MmExpansionLock

PETHREAD	MiExpansionLockOwner

Macro Definition Documentation

◆ _1GB

#define _1GB (1024 * _1MB)

Definition at line 20 of file miarm.h.

◆ _1KB

#define _1KB (1024u)

Definition at line 18 of file miarm.h.

◆ _1MB

#define _1MB (1024 * _1KB)

Definition at line 19 of file miarm.h.

◆ _64K

#define _64K (64 * _1KB)

Definition at line 23 of file miarm.h.

◆ BASE_POOL_TYPE_MASK

#define BASE_POOL_TYPE_MASK 1

Definition at line 286 of file miarm.h.

◆ IMAGE_SCN_PROTECTION_MASK

#define IMAGE_SCN_PROTECTION_MASK (IMAGE_SCN_MEM_WRITE | IMAGE_SCN_MEM_READ | IMAGE_SCN_MEM_EXECUTE)

Definition at line 174 of file miarm.h.

◆ MI_GET_NEXT_COLOR

#define MI_GET_NEXT_COLOR ( ) (MI_GET_PAGE_COLOR(++MmSystemPageColor))

Definition at line 246 of file miarm.h.

◆ MI_GET_NEXT_PROCESS_COLOR

#define MI_GET_NEXT_PROCESS_COLOR ( x ) (MI_GET_PAGE_COLOR(++(x)->NextPageColor))

Definition at line 247 of file miarm.h.

◆ MI_GET_PAGE_COLOR

#define MI_GET_PAGE_COLOR ( x ) ((x) & MmSecondaryColorMask)

Definition at line 245 of file miarm.h.

◆ MI_INITIAL_SESSION_IDS

#define MI_INITIAL_SESSION_IDS 64

Definition at line 221 of file miarm.h.

◆ MI_IS_PAGE_TABLE_ADDRESS

#define MI_IS_PAGE_TABLE_ADDRESS ( Address ) (((PVOID)(Address) >= (PVOID)PTE_BASE) && ((PVOID)(Address) <= (PVOID)PTE_TOP))

Definition at line 191 of file miarm.h.

◆ MI_IS_PAGE_TABLE_OR_HYPER_ADDRESS

#define MI_IS_PAGE_TABLE_OR_HYPER_ADDRESS ( Address ) (((PVOID)(Address) >= (PVOID)PTE_BASE) && ((PVOID)(Address) <= (PVOID)MmHyperSpaceEnd))

Definition at line 197 of file miarm.h.

◆ MI_IS_PFN_DELETED

#define MI_IS_PFN_DELETED ( x ) ((ULONG_PTR)((x)->PteAddress) & 1)

Definition at line 209 of file miarm.h.

◆ MI_IS_ROS_PFN

#define MI_IS_ROS_PFN ( x ) ((x)->u4.AweAllocation == TRUE)

Definition at line 1116 of file miarm.h.

◆ MI_IS_SESSION_ADDRESS

#define MI_IS_SESSION_ADDRESS ( Address ) (((Address) >= MmSessionBase) && ((Address) < MiSessionSpaceEnd))

Definition at line 185 of file miarm.h.

◆ MI_IS_SESSION_IMAGE_ADDRESS

#define MI_IS_SESSION_IMAGE_ADDRESS ( Address ) (((Address) >= MiSessionImageStart) && ((Address) < MiSessionImageEnd))

Definition at line 182 of file miarm.h.

◆ MI_IS_SESSION_PTE

#define MI_IS_SESSION_PTE ( Pte ) ((((PMMPTE)Pte) >= MiSessionBasePte) && (((PMMPTE)Pte) < MiSessionLastPte))

Definition at line 188 of file miarm.h.

◆ MI_IS_SYSTEM_PAGE_TABLE_ADDRESS

#define MI_IS_SYSTEM_PAGE_TABLE_ADDRESS ( Address ) (((Address) >= (PVOID)MiAddressToPte(MmSystemRangeStart)) && ((Address) <= (PVOID)PTE_TOP))

Definition at line 194 of file miarm.h.

◆ MI_LOWEST_VAD_ADDRESS

#define MI_LOWEST_VAD_ADDRESS (PVOID)MM_LOWEST_USER_ADDRESS

Definition at line 15 of file miarm.h.

◆ MI_MAKE_SOFTWARE_PTE

#define MI_MAKE_SOFTWARE_PTE	(	p,
		x
	)	((p)->u.Long = (x << MM_PTE_SOFTWARE_PROTECTION_BITS))

Definition at line 203 of file miarm.h.

◆ MI_PTE_LOOKUP_NEEDED

#define MI_PTE_LOOKUP_NEEDED 0xFFFFF

Definition at line 255 of file miarm.h.

◆ MI_SESSION_DATA_PAGES_MAXIMUM

#define MI_SESSION_DATA_PAGES_MAXIMUM (MM_ALLOCATION_GRANULARITY / PAGE_SIZE)

Definition at line 261 of file miarm.h.

◆ MI_SESSION_TAG_PAGES_MAXIMUM

#define MI_SESSION_TAG_PAGES_MAXIMUM (MM_ALLOCATION_GRANULARITY / PAGE_SIZE)

Definition at line 262 of file miarm.h.

◆ MI_SET_PFN_DELETED

#define MI_SET_PFN_DELETED ( x ) ((x)->PteAddress = (PMMPTE)((ULONG_PTR)(x)->PteAddress | 1))

Definition at line 208 of file miarm.h.

◆ MI_SYSTEM_VIEW_BUCKET_SIZE

#define MI_SYSTEM_VIEW_BUCKET_SIZE _64K

Definition at line 275 of file miarm.h.

◆ MM_DECOMMIT

#define MM_DECOMMIT (MM_ZERO_ACCESS | MM_GUARDPAGE)

Definition at line 64 of file miarm.h.

◆ MM_DELETE_CHECK

#define MM_DELETE_CHECK 85

Definition at line 270 of file miarm.h.

◆ MM_EXECUTE

#define MM_EXECUTE 2

Definition at line 45 of file miarm.h.

◆ MM_EXECUTE_READ

#define MM_EXECUTE_READ 3

Definition at line 46 of file miarm.h.

◆ MM_EXECUTE_READWRITE

#define MM_EXECUTE_READWRITE 6

Definition at line 49 of file miarm.h.

◆ MM_EXECUTE_WRITECOPY

#define MM_EXECUTE_WRITECOPY 7

Definition at line 50 of file miarm.h.

◆ MM_GUARDPAGE

#define MM_GUARDPAGE 0x10

Definition at line 57 of file miarm.h.

◆ MM_INVALID_PROTECTION

#define MM_INVALID_PROTECTION 0xFFFFFFFF

Definition at line 67 of file miarm.h.

◆ MM_NO_ACCESS_ALLOWED

#define MM_NO_ACCESS_ALLOWED 01

Definition at line 269 of file miarm.h.

◆ MM_NOACCESS

#define MM_NOACCESS (MM_ZERO_ACCESS | MM_WRITECOMBINE)

Definition at line 65 of file miarm.h.

◆ MM_NOCACHE

#define MM_NOCACHE 0x08

Definition at line 56 of file miarm.h.

◆ MM_OUTSWAPPED_KSTACK

#define MM_OUTSWAPPED_KSTACK (MM_EXECUTE_WRITECOPY | MM_WRITECOMBINE)

Definition at line 66 of file miarm.h.

◆ MM_PROTECT_ACCESS

#define MM_PROTECT_ACCESS 7

Definition at line 51 of file miarm.h.

◆ MM_PROTECT_SPECIAL

#define MM_PROTECT_SPECIAL 0x18

Definition at line 59 of file miarm.h.

◆ MM_READ_ONLY_ALLOWED

#define MM_READ_ONLY_ALLOWED 10

Definition at line 268 of file miarm.h.

◆ MM_READ_WRITE_ALLOWED

#define MM_READ_WRITE_ALLOWED 11

Definition at line 267 of file miarm.h.

◆ MM_READONLY

#define MM_READONLY 1

Definition at line 44 of file miarm.h.

◆ MM_READWRITE

#define MM_READWRITE 4

Definition at line 47 of file miarm.h.

◆ MM_SYSLDR_BOOT_LOADED

#define MM_SYSLDR_BOOT_LOADED ((PVOID)(ULONG_PTR)-1)

Definition at line 215 of file miarm.h.

◆ MM_SYSLDR_NO_IMPORTS

#define MM_SYSLDR_NO_IMPORTS ((PVOID)(ULONG_PTR)-2)

Definition at line 214 of file miarm.h.

◆ MM_SYSLDR_SINGLE_ENTRY

#define MM_SYSLDR_SINGLE_ENTRY 0x1

Definition at line 216 of file miarm.h.

◆ MM_WRITECOMBINE

#define MM_WRITECOMBINE 0x18

Definition at line 58 of file miarm.h.

◆ MM_WRITECOPY

#define MM_WRITECOPY 5

Definition at line 48 of file miarm.h.

◆ MM_ZERO_ACCESS

#define MM_ZERO_ACCESS 0

Definition at line 43 of file miarm.h.

◆ PD_SIZE

#define PD_SIZE (PDE_PER_PAGE * sizeof(MMPDE))

Definition at line 29 of file miarm.h.

◆ POOL_ALLOC_IRQL_INVALID

#define POOL_ALLOC_IRQL_INVALID 8

Definition at line 318 of file miarm.h.

◆ POOL_BILLED_PROCESS_INVALID

#define POOL_BILLED_PROCESS_INVALID 13

Definition at line 320 of file miarm.h.

◆ POOL_BLOCK_SIZE

#define POOL_BLOCK_SIZE 8

Definition at line 283 of file miarm.h.

◆ POOL_CORRUPTED_LIST

#define POOL_CORRUPTED_LIST 3

Definition at line 304 of file miarm.h.

◆ POOL_ENTRIES_NOT_ALIGNED_NEXT

#define POOL_ENTRIES_NOT_ALIGNED_NEXT 9

Definition at line 309 of file miarm.h.

◆ POOL_ENTRIES_NOT_ALIGNED_PREVIOUS

#define POOL_ENTRIES_NOT_ALIGNED_PREVIOUS 6

Definition at line 306 of file miarm.h.

◆ POOL_ENTRY_ALREADY_FREE

#define POOL_ENTRY_ALREADY_FREE 6

Definition at line 316 of file miarm.h.

◆ POOL_ENTRY_CORRUPTED

#define POOL_ENTRY_CORRUPTED 1

Definition at line 315 of file miarm.h.

◆ POOL_ENTRY_NOT_ALLOCATED

#define POOL_ENTRY_NOT_ALLOCATED 7

Definition at line 317 of file miarm.h.

◆ POOL_ENTRY_NOT_FOUND

#define POOL_ENTRY_NOT_FOUND 10

Definition at line 310 of file miarm.h.

◆ POOL_FLAG_CHECK_DEADLOCK

#define POOL_FLAG_CHECK_DEADLOCK 0x10

Definition at line 296 of file miarm.h.

◆ POOL_FLAG_CHECK_RESOURCES

#define POOL_FLAG_CHECK_RESOURCES 0x4

Definition at line 294 of file miarm.h.

◆ POOL_FLAG_CHECK_TIMERS

#define POOL_FLAG_CHECK_TIMERS 0x1

Definition at line 292 of file miarm.h.

◆ POOL_FLAG_CHECK_WORKERS

#define POOL_FLAG_CHECK_WORKERS 0x2

Definition at line 293 of file miarm.h.

◆ POOL_FLAG_CRASH_ON_FAILURE

#define POOL_FLAG_CRASH_ON_FAILURE 0x80

Definition at line 299 of file miarm.h.

◆ POOL_FLAG_DBGPRINT_ON_FAILURE

#define POOL_FLAG_DBGPRINT_ON_FAILURE 0x40

Definition at line 298 of file miarm.h.

◆ POOL_FLAG_SPECIAL_POOL

#define POOL_FLAG_SPECIAL_POOL 0x20

Definition at line 297 of file miarm.h.

◆ POOL_FLAG_VERIFIER

#define POOL_FLAG_VERIFIER 0x8

Definition at line 295 of file miarm.h.

◆ POOL_FREE_IRQL_INVALID

#define POOL_FREE_IRQL_INVALID 9

Definition at line 319 of file miarm.h.

◆ POOL_HEADER_IS_ZERO

#define POOL_HEADER_IS_ZERO 8

Definition at line 308 of file miarm.h.

◆ POOL_HEADER_NOT_ALIGNED

#define POOL_HEADER_NOT_ALIGNED 7

Definition at line 307 of file miarm.h.

◆ POOL_HEADER_SIZE_INVALID

#define POOL_HEADER_SIZE_INVALID 32

Definition at line 321 of file miarm.h.

◆ POOL_LISTS_PER_PAGE

#define POOL_LISTS_PER_PAGE (PAGE_SIZE / POOL_BLOCK_SIZE)

Definition at line 285 of file miarm.h.

◆ POOL_MAX_ALLOC

#define POOL_MAX_ALLOC (PAGE_SIZE - (sizeof(POOL_HEADER) + POOL_BLOCK_SIZE))

Definition at line 287 of file miarm.h.

◆ POOL_SIZE_OR_INDEX_MISMATCH

#define POOL_SIZE_OR_INDEX_MISMATCH 5

Definition at line 305 of file miarm.h.

◆ PT_SIZE

#define PT_SIZE (PTE_PER_PAGE * sizeof(MMPTE))

Definition at line 26 of file miarm.h.

◆ PTE_MAPPING_ADDRESS_EMPTY

#define PTE_MAPPING_ADDRESS_EMPTY 0x109

Definition at line 169 of file miarm.h.

◆ PTE_MAPPING_ADDRESS_INVALID

#define PTE_MAPPING_ADDRESS_INVALID 0x105

Definition at line 167 of file miarm.h.

◆ PTE_MAPPING_ADDRESS_NOT_OWNED

#define PTE_MAPPING_ADDRESS_NOT_OWNED 0x104

Definition at line 166 of file miarm.h.

◆ PTE_MAPPING_EMPTY

#define PTE_MAPPING_EMPTY 0x102

Definition at line 164 of file miarm.h.

◆ PTE_MAPPING_NONE

#define PTE_MAPPING_NONE 0x100

Definition at line 162 of file miarm.h.

◆ PTE_MAPPING_NOT_OWNED

#define PTE_MAPPING_NOT_OWNED 0x101

Definition at line 163 of file miarm.h.

◆ PTE_MAPPING_RESERVED

#define PTE_MAPPING_RESERVED 0x103

Definition at line 165 of file miarm.h.

◆ PTE_UNMAPPING_ADDRESS_NOT_OWNED

#define PTE_UNMAPPING_ADDRESS_NOT_OWNED 0x108

Definition at line 168 of file miarm.h.

◆ SYSTEM_PD_SIZE

#define SYSTEM_PD_SIZE (PPE_PER_PAGE * PD_SIZE)

Definition at line 32 of file miarm.h.

Typedef Documentation

◆ MI_LARGE_PAGE_DRIVER_ENTRY

typedef struct _MI_LARGE_PAGE_DRIVER_ENTRY MI_LARGE_PAGE_DRIVER_ENTRY

◆ MI_LARGE_PAGE_RANGES

typedef struct _MI_LARGE_PAGE_RANGES MI_LARGE_PAGE_RANGES

◆ MI_PFN_CACHE_ATTRIBUTE

typedef enum _MI_PFN_CACHE_ATTRIBUTE MI_PFN_CACHE_ATTRIBUTE

◆ MM_SESSION_SPACE

typedef struct _MM_SESSION_SPACE MM_SESSION_SPACE

◆ MM_SESSION_SPACE_FLAGS

typedef struct _MM_SESSION_SPACE_FLAGS MM_SESSION_SPACE_FLAGS

◆ MMCOLOR_TABLES

typedef struct _MMCOLOR_TABLES MMCOLOR_TABLES

◆ MMSESSION

typedef struct _MMSESSION MMSESSION

◆ MMSYSTEM_PTE_POOL_TYPE

typedef enum _MMSYSTEM_PTE_POOL_TYPE MMSYSTEM_PTE_POOL_TYPE

◆ MMVIEW

typedef struct _MMVIEW MMVIEW

◆ PHYSICAL_MEMORY_DESCRIPTOR

typedef struct _PHYSICAL_MEMORY_DESCRIPTOR PHYSICAL_MEMORY_DESCRIPTOR

◆ PHYSICAL_MEMORY_RUN

typedef struct _PHYSICAL_MEMORY_RUN PHYSICAL_MEMORY_RUN

◆ PMI_LARGE_PAGE_DRIVER_ENTRY

typedef struct _MI_LARGE_PAGE_DRIVER_ENTRY * PMI_LARGE_PAGE_DRIVER_ENTRY

◆ PMI_LARGE_PAGE_RANGES

typedef struct _MI_LARGE_PAGE_RANGES * PMI_LARGE_PAGE_RANGES

◆ PMI_PFN_CACHE_ATTRIBUTE

typedef enum _MI_PFN_CACHE_ATTRIBUTE * PMI_PFN_CACHE_ATTRIBUTE

◆ PMM_SESSION_SPACE

typedef struct _MM_SESSION_SPACE * PMM_SESSION_SPACE

◆ PMMCOLOR_TABLES

typedef struct _MMCOLOR_TABLES * PMMCOLOR_TABLES

◆ PMMSESSION

typedef struct _MMSESSION * PMMSESSION

◆ PMMVIEW

typedef struct _MMVIEW * PMMVIEW

◆ POOL_DESCRIPTOR

typedef struct _POOL_DESCRIPTOR POOL_DESCRIPTOR

◆ POOL_HEADER

typedef struct _POOL_HEADER POOL_HEADER

◆ POOL_TRACKER_BIG_PAGES

typedef struct _POOL_TRACKER_BIG_PAGES POOL_TRACKER_BIG_PAGES

◆ POOL_TRACKER_TABLE

typedef struct _POOL_TRACKER_TABLE POOL_TRACKER_TABLE

◆ PPHYSICAL_MEMORY_DESCRIPTOR

typedef struct _PHYSICAL_MEMORY_DESCRIPTOR * PPHYSICAL_MEMORY_DESCRIPTOR

◆ PPHYSICAL_MEMORY_RUN

typedef struct _PHYSICAL_MEMORY_RUN * PPHYSICAL_MEMORY_RUN

◆ PPOOL_DESCRIPTOR

typedef struct _POOL_DESCRIPTOR * PPOOL_DESCRIPTOR

◆ PPOOL_HEADER

typedef struct _POOL_HEADER * PPOOL_HEADER

◆ PPOOL_TRACKER_BIG_PAGES

typedef struct _POOL_TRACKER_BIG_PAGES * PPOOL_TRACKER_BIG_PAGES

◆ PPOOL_TRACKER_TABLE

typedef struct _POOL_TRACKER_TABLE * PPOOL_TRACKER_TABLE

Enumeration Type Documentation

◆ _MI_PFN_CACHE_ATTRIBUTE

enum _MI_PFN_CACHE_ATTRIBUTE

Enumerator
MiNonCached
MiCached
MiWriteCombined
MiNotMapped

Definition at line 422 of file miarm.h.

{
    MiNonCached,
    MiCached,
    MiWriteCombined,
    MiNotMapped
} MI_PFN_CACHE_ATTRIBUTE, *PMI_PFN_CACHE_ATTRIBUTE;

◆ _MMSYSTEM_PTE_POOL_TYPE

enum _MMSYSTEM_PTE_POOL_TYPE

Enumerator
SystemPteSpace
NonPagedPoolExpansion
MaximumPtePoolTypes

Definition at line 415 of file miarm.h.

{
    SystemPteSpace,
    NonPagedPoolExpansion,
    MaximumPtePoolTypes
} MMSYSTEM_PTE_POOL_TYPE;

Function Documentation

◆ C_ASSERT() [1/2]

C_ASSERT ( POOL_BLOCK_SIZE = =sizeof(LIST_ENTRY) )

◆ C_ASSERT() [2/2]

C_ASSERT ( sizeof(POOL_HEADER) = =POOL_BLOCK_SIZE )

◆ ExInitializePoolDescriptor()

VOID NTAPI ExInitializePoolDescriptor	(	IN PPOOL_DESCRIPTOR	PoolDescriptor,
		IN POOL_TYPE	PoolType,
		IN ULONG	PoolIndex,
		IN ULONG	Threshold,
		IN PVOID	PoolLock
	)

Definition at line 969 of file expool.c.

{
    PLIST_ENTRY NextEntry, LastEntry;
 
    //
    // Setup the descriptor based on the caller's request
    //
    PoolDescriptor->PoolType = PoolType;
    PoolDescriptor->PoolIndex = PoolIndex;
    PoolDescriptor->Threshold = Threshold;
    PoolDescriptor->LockAddress = PoolLock;
 
    //
    // Initialize accounting data
    //
    PoolDescriptor->RunningAllocs = 0;
    PoolDescriptor->RunningDeAllocs = 0;
    PoolDescriptor->TotalPages = 0;
    PoolDescriptor->TotalBytes = 0;
    PoolDescriptor->TotalBigPages = 0;
 
    //
    // Nothing pending for now
    //
    PoolDescriptor->PendingFrees = NULL;
    PoolDescriptor->PendingFreeDepth = 0;
 
    //
    // Loop all the descriptor's allocation lists and initialize them
    //
    NextEntry = PoolDescriptor->ListHeads;
    LastEntry = NextEntry + POOL_LISTS_PER_PAGE;
    while (NextEntry < LastEntry)
    {
        ExpInitializePoolListHead(NextEntry);
        NextEntry++;
    }
 
    //
    // Note that ReactOS does not support Session Pool Yet
    //
    ASSERT(PoolType != PagedPoolSession);
}

Referenced by InitializePool(), and MiInitializeSessionPool().

◆ InitializePool()

VOID NTAPI InitializePool	(	IN POOL_TYPE	PoolType,
		IN ULONG	Threshold
	)

Definition at line 1020 of file expool.c.

{
    PPOOL_DESCRIPTOR Descriptor;
    SIZE_T TableSize;
    ULONG i;
 
    //
    // Check what kind of pool this is
    //
    if (PoolType == NonPagedPool)
    {
        //
        // Compute the track table size and convert it from a power of two to an
        // actual byte size
        //
        // NOTE: On checked builds, we'll assert if the registry table size was
        // invalid, while on retail builds we'll just break out of the loop at
        // that point.
        //
        TableSize = min(PoolTrackTableSize, MmSizeOfNonPagedPoolInBytes >> 8);
        for (i = 0; i < 32; i++)
        {
            if (TableSize & 1)
            {
                ASSERT((TableSize & ~1) == 0);
                if (!(TableSize & ~1)) break;
            }
            TableSize >>= 1;
        }
 
        //
        // If we hit bit 32, than no size was defined in the registry, so
        // we'll use the default size of 2048 entries.
        //
        // Otherwise, use the size from the registry, as long as it's not
        // smaller than 64 entries.
        //
        if (i == 32)
        {
            PoolTrackTableSize = 2048;
        }
        else
        {
            PoolTrackTableSize = max(1 << i, 64);
        }
 
        //
        // Loop trying with the biggest specified size first, and cut it down
        // by a power of two each iteration in case not enough memory exist
        //
        while (TRUE)
        {
            //
            // Do not allow overflow
            //
            if ((PoolTrackTableSize + 1) > (MAXULONG_PTR / sizeof(POOL_TRACKER_TABLE)))
            {
                PoolTrackTableSize >>= 1;
                continue;
            }
 
            //
            // Allocate the tracker table and exit the loop if this worked
            //
            PoolTrackTable = MiAllocatePoolPages(NonPagedPool,
                                                 (PoolTrackTableSize + 1) *
                                                 sizeof(POOL_TRACKER_TABLE));
            if (PoolTrackTable) break;
 
            //
            // Otherwise, as long as we're not down to the last bit, keep
            // iterating
            //
            if (PoolTrackTableSize == 1)
            {
                KeBugCheckEx(MUST_SUCCEED_POOL_EMPTY,
                             TableSize,
                             0xFFFFFFFF,
                             0xFFFFFFFF,
                             0xFFFFFFFF);
            }
            PoolTrackTableSize >>= 1;
        }
 
        //
        // Add one entry, compute the hash, and zero the table
        //
        PoolTrackTableSize++;
        PoolTrackTableMask = PoolTrackTableSize - 2;
 
        RtlZeroMemory(PoolTrackTable,
                      PoolTrackTableSize * sizeof(POOL_TRACKER_TABLE));
 
        //
        // Finally, add the most used tags to speed up those allocations
        //
        ExpSeedHotTags();
 
        //
        // We now do the exact same thing with the tracker table for big pages
        //
        TableSize = min(PoolBigPageTableSize, MmSizeOfNonPagedPoolInBytes >> 8);
        for (i = 0; i < 32; i++)
        {
            if (TableSize & 1)
            {
                ASSERT((TableSize & ~1) == 0);
                if (!(TableSize & ~1)) break;
            }
            TableSize >>= 1;
        }
 
        //
        // For big pages, the default tracker table is 4096 entries, while the
        // minimum is still 64
        //
        if (i == 32)
        {
            PoolBigPageTableSize = 4096;
        }
        else
        {
            PoolBigPageTableSize = max(1 << i, 64);
        }
 
        //
        // Again, run the exact same loop we ran earlier, but this time for the
        // big pool tracker instead
        //
        while (TRUE)
        {
            if ((PoolBigPageTableSize + 1) > (MAXULONG_PTR / sizeof(POOL_TRACKER_BIG_PAGES)))
            {
                PoolBigPageTableSize >>= 1;
                continue;
            }
 
            PoolBigPageTable = MiAllocatePoolPages(NonPagedPool,
                                                   PoolBigPageTableSize *
                                                   sizeof(POOL_TRACKER_BIG_PAGES));
            if (PoolBigPageTable) break;
 
            if (PoolBigPageTableSize == 1)
            {
                KeBugCheckEx(MUST_SUCCEED_POOL_EMPTY,
                             TableSize,
                             0xFFFFFFFF,
                             0xFFFFFFFF,
                             0xFFFFFFFF);
            }
 
            PoolBigPageTableSize >>= 1;
        }
 
        //
        // An extra entry is not needed for for the big pool tracker, so just
        // compute the hash and zero it
        //
        PoolBigPageTableHash = PoolBigPageTableSize - 1;
        RtlZeroMemory(PoolBigPageTable,
                      PoolBigPageTableSize * sizeof(POOL_TRACKER_BIG_PAGES));
        for (i = 0; i < PoolBigPageTableSize; i++)
        {
            PoolBigPageTable[i].Va = (PVOID)POOL_BIG_TABLE_ENTRY_FREE;
        }
 
        //
        // During development, print this out so we can see what's happening
        //
        DPRINT("EXPOOL: Pool Tracker Table at: 0x%p with 0x%lx bytes\n",
                PoolTrackTable, PoolTrackTableSize * sizeof(POOL_TRACKER_TABLE));
        DPRINT("EXPOOL: Big Pool Tracker Table at: 0x%p with 0x%lx bytes\n",
                PoolBigPageTable, PoolBigPageTableSize * sizeof(POOL_TRACKER_BIG_PAGES));
 
        //
        // Insert the generic tracker for all of big pool
        //
        ExpInsertPoolTracker('looP',
                             ROUND_TO_PAGES(PoolBigPageTableSize *
                                            sizeof(POOL_TRACKER_BIG_PAGES)),
                             NonPagedPool);
 
        //
        // No support for NUMA systems at this time
        //
        ASSERT(KeNumberNodes == 1);
 
        //
        // Initialize the tag spinlock
        //
        KeInitializeSpinLock(&ExpTaggedPoolLock);
 
        //
        // Initialize the nonpaged pool descriptor
        //
        PoolVector[NonPagedPool] = &NonPagedPoolDescriptor;
        ExInitializePoolDescriptor(PoolVector[NonPagedPool],
                                   NonPagedPool,
                                   0,
                                   Threshold,
                                   NULL);
    }
    else
    {
        //
        // No support for NUMA systems at this time
        //
        ASSERT(KeNumberNodes == 1);
 
        //
        // Allocate the pool descriptor
        //
        Descriptor = ExAllocatePoolWithTag(NonPagedPool,
                                           sizeof(KGUARDED_MUTEX) +
                                           sizeof(POOL_DESCRIPTOR),
                                           'looP');
        if (!Descriptor)
        {
            //
            // This is really bad...
            //
            KeBugCheckEx(MUST_SUCCEED_POOL_EMPTY,
                         0,
                         -1,
                         -1,
                         -1);
        }
 
        //
        // Setup the vector and guarded mutex for paged pool
        //
        PoolVector[PagedPool] = Descriptor;
        ExpPagedPoolMutex = (PKGUARDED_MUTEX)(Descriptor + 1);
        ExpPagedPoolDescriptor[0] = Descriptor;
        KeInitializeGuardedMutex(ExpPagedPoolMutex);
        ExInitializePoolDescriptor(Descriptor,
                                   PagedPool,
                                   0,
                                   Threshold,
                                   ExpPagedPoolMutex);
 
        //
        // Insert the generic tracker for all of nonpaged pool
        //
        ExpInsertPoolTracker('looP',
                             ROUND_TO_PAGES(PoolTrackTableSize * sizeof(POOL_TRACKER_TABLE)),
                             NonPagedPool);
    }
}

Referenced by MiBuildPagedPool(), and MiInitMachineDependent().

◆ MI_ERASE_PTE()

FORCEINLINE VOID MI_ERASE_PTE ( IN PMMPTE PointerPte )

Definition at line 1019 of file miarm.h.

{
    /* Zero out the PTE */
    ASSERT(PointerPte->u.Long != 0);
    PointerPte->u.Long = 0;
}

Referenced by MiDeletePte(), MiDeleteSystemPageableVm(), MiDeleteVirtualAddresses(), MiUnmapLockedPagesInUserSpace(), and MmFreeSpecialPool().

◆ MI_GET_PROTOTYPE_PTE_FOR_VPN()

FORCEINLINE PMMPTE MI_GET_PROTOTYPE_PTE_FOR_VPN	(	IN PMMVAD	Vad,
		IN ULONG_PTR	Vpn
	)

Definition at line 1570 of file miarm.h.

{
    PMMPTE ProtoPte;
 
    /* Find the offset within the VAD's prototype PTEs */
    ProtoPte = Vad->FirstPrototypePte + (Vpn - Vad->StartingVpn);
    ASSERT(ProtoPte <= Vad->LastContiguousPte);
    return ProtoPte;
}

Referenced by MiDeleteVirtualAddresses(), MiQueryAddressState(), and NtAllocateVirtualMemory().

◆ MI_IS_MAPPED_PTE()

FORCEINLINE BOOLEAN MI_IS_MAPPED_PTE ( PMMPTE PointerPte )

Todo:: Make this reasonable code, this is UGLY!

Definition at line 933 of file miarm.h.

{
    return ((PointerPte->u.Long & 0xFFFFFC01) != 0);
}

◆ MI_IS_PHYSICAL_ADDRESS()

FORCEINLINE BOOLEAN MI_IS_PHYSICAL_ADDRESS ( IN PVOID Address )

Definition at line 959 of file miarm.h.

{
    PMMPDE PointerPde;
 
    /* Large pages are never paged out, always physically resident */
    PointerPde = MiAddressToPde(Address);
    return ((PointerPde->u.Hard.LargePage) && (PointerPde->u.Hard.Valid));
}

Referenced by MiDispatchFault(), MiFindInitializationCode(), MiFreeInitializationCode(), MiProtectFreeNonPagedPool(), MiReleaseProcessReferenceToSessionDataPage(), MiSetPagingOfDriver(), MiUnProtectFreeNonPagedPool(), MiWriteProtectSystemImage(), MmChangeKernelResourceSectionProtection(), MmFreeDriverInitialization(), MmFreeLoaderBlock(), MmMakeKernelResourceSectionWritable(), and MmProbeAndLockPages().

◆ MI_IS_PROCESS_WORKING_SET()

FORCEINLINE BOOLEAN MI_IS_PROCESS_WORKING_SET ( PMMSUPPORT WorkingSet )

Definition at line 675 of file miarm.h.

{
    return (WorkingSet != &MmSystemCacheWs) && !WorkingSet->Flags.SessionSpace;
}

Referenced by MiInitializeWorkingSetList(), and MmWorkingSetManager().

◆ MI_IS_WS_UNSAFE()

FORCEINLINE BOOLEAN MI_IS_WS_UNSAFE ( IN PEPROCESS Process )

Definition at line 1127 of file miarm.h.

{
    return (Process->Vm.Flags.AcquiredUnsafe == TRUE);
}

Referenced by MiLockProcessWorkingSet(), MiLockProcessWorkingSetShared(), MiLockProcessWorkingSetUnsafe(), MiUnlockProcessWorkingSet(), MiUnlockProcessWorkingSetForFault(), MiUnlockProcessWorkingSetShared(), and MiUnlockProcessWorkingSetUnsafe().

◆ MI_MAKE_HARDWARE_PTE()

FORCEINLINE VOID MI_MAKE_HARDWARE_PTE	(	IN PMMPTE	NewPte,
		IN PMMPTE	MappingPte,
		IN ULONG_PTR	ProtectionMask,
		IN PFN_NUMBER	PageFrameNumber
	)

Definition at line 821 of file miarm.h.

{
    /* Check that we are not setting valid a page that should not be */
    ASSERT(ProtectionMask & MM_PROTECT_ACCESS);
    ASSERT((ProtectionMask & MM_GUARDPAGE) == 0);
 
    /* Set the protection and page */
    NewPte->u.Long = MiDetermineUserGlobalPteMask(MappingPte);
    NewPte->u.Long |= MmProtectToPteMask[ProtectionMask];
    NewPte->u.Hard.PageFrameNumber = PageFrameNumber;
}

Referenced by MiCompleteProtoPteFault(), MiResolveDemandZeroFault(), MiResolvePageFileFault(), MiResolveProtoPteFault(), MmArmAccessFault(), and MmCreateVirtualMappingUnsafeEx().

◆ MI_MAKE_HARDWARE_PTE_KERNEL()

FORCEINLINE VOID MI_MAKE_HARDWARE_PTE_KERNEL	(	IN PMMPTE	NewPte,
		IN PMMPTE	MappingPte,
		IN ULONG_PTR	ProtectionMask,
		IN PFN_NUMBER	PageFrameNumber
	)

Definition at line 787 of file miarm.h.

{
    /* Only valid for kernel, non-session PTEs */
    ASSERT(MappingPte > MiHighestUserPte);
    ASSERT(!MI_IS_SESSION_PTE(MappingPte));
    ASSERT((MappingPte < (PMMPTE)PDE_BASE) || (MappingPte > (PMMPTE)PDE_TOP));
 
    /* Check that we are not setting valid a page that should not be */
    ASSERT(ProtectionMask & MM_PROTECT_ACCESS);
    ASSERT((ProtectionMask & MM_GUARDPAGE) == 0);
 
    /* Start fresh */
    NewPte->u.Long = 0;
 
    /* Set the protection and page */
    NewPte->u.Hard.PageFrameNumber = PageFrameNumber;
    NewPte->u.Long |= MmProtectToPteMask[ProtectionMask];
 
    /* Make this valid & global */
#ifdef _GLOBAL_PAGES_ARE_AWESOME_
    if (KeFeatureBits & KF_GLOBAL_PAGE)
        NewPte->u.Hard.Global = 1;
#endif
    NewPte->u.Hard.Valid = 1;
}

Referenced by MiArchCreateProcessAddressSpace(), MmChangeKernelResourceSectionProtection(), MmCreateKernelStack(), MmGrowKernelStackEx(), and MmInitSystem().

◆ MI_MAKE_HARDWARE_PTE_USER()

FORCEINLINE VOID MI_MAKE_HARDWARE_PTE_USER	(	IN PMMPTE	NewPte,
		IN PMMPTE	MappingPte,
		IN ULONG_PTR	ProtectionMask,
		IN PFN_NUMBER	PageFrameNumber
	)

Definition at line 841 of file miarm.h.

{
    /* Only valid for kernel, non-session PTEs */
    ASSERT(MappingPte <= MiHighestUserPte);
 
    /* Start fresh */
    NewPte->u.Long = 0;
 
    /* Check that we are not setting valid a page that should not be */
    ASSERT(ProtectionMask & MM_PROTECT_ACCESS);
    ASSERT((ProtectionMask & MM_GUARDPAGE) == 0);
 
    NewPte->u.Hard.Valid = TRUE;
    NewPte->u.Hard.Owner = TRUE;
    NewPte->u.Hard.PageFrameNumber = PageFrameNumber;
    NewPte->u.Long |= MmProtectToPteMask[ProtectionMask];
}

Referenced by MiCompleteProtoPteFault(), MiFlushTbAndCapture(), MiMapLockedPagesInUserSpace(), MiResolveDemandZeroFault(), MiResolveProtoPteFault(), and MmArmAccessFault().

◆ MI_MAKE_PROTOTYPE_PTE()

FORCEINLINE VOID MI_MAKE_PROTOTYPE_PTE	(	IN PMMPTE	NewPte,
		IN PMMPTE	PointerPte
	)

Definition at line 868 of file miarm.h.

{
    ULONG_PTR Offset;
 
    /* Mark this as a prototype */
    NewPte->u.Long = 0;
    NewPte->u.Proto.Prototype = 1;
 
    /*
     * Prototype PTEs are only valid in paged pool by design, this little trick
     * lets us only use 30 bits for the address of the PTE, as long as the area
     * stays 1024MB At most.
     */
    Offset = (ULONG_PTR)PointerPte - (ULONG_PTR)MmPagedPoolStart;
 
    /*
     * 7 bits go in the "low" (but we assume the bottom 2 are zero)
     * and the other 21 bits go in the "high"
     */
    NewPte->u.Proto.ProtoAddressLow = (Offset & 0x1FC) >> 2;
    NewPte->u.Proto.ProtoAddressHigh = (Offset & 0x3FFFFE00) >> 9;
}

◆ MI_MAKE_SUBSECTION_PTE()

FORCEINLINE VOID MI_MAKE_SUBSECTION_PTE	(	IN PMMPTE	NewPte,
		IN PVOID	Segment
	)

Definition at line 897 of file miarm.h.

{
    ULONG_PTR Offset;
 
    /* Mark this as a prototype */
    NewPte->u.Long = 0;
    NewPte->u.Subsect.Prototype = 1;
 
    /*
     * Segments are only valid either in nonpaged pool. We store the 20 bit
     * difference either from the top or bottom of nonpaged pool, giving a
     * maximum of 128MB to each delta, meaning nonpaged pool cannot exceed
     * 256MB.
     */
    if ((ULONG_PTR)Segment < ((ULONG_PTR)MmSubsectionBase + (128 * _1MB)))
    {
        Offset = (ULONG_PTR)Segment - (ULONG_PTR)MmSubsectionBase;
        NewPte->u.Subsect.WhichPool = PagedPool;
    }
    else
    {
        Offset = (ULONG_PTR)MmNonPagedPoolEnd - (ULONG_PTR)Segment;
        NewPte->u.Subsect.WhichPool = NonPagedPool;
    }
 
    /*
     * 4 bits go in the "low" (but we assume the bottom 3 are zero)
     * and the other 20 bits go in the "high"
     */
    NewPte->u.Subsect.SubsectionAddressLow = (Offset & 0x78) >> 3;
    NewPte->u.Subsect.SubsectionAddressHigh = (Offset & 0xFFFFF80) >> 7;
}

◆ MI_MAKE_TRANSITION_PTE()

FORCEINLINE VOID MI_MAKE_TRANSITION_PTE	(	_Out_ PMMPTE	NewPte,
		_In_ PFN_NUMBER	Page,
		_In_ ULONG	Protection
	)

Definition at line 943 of file miarm.h.

{
    NewPte->u.Long = 0;
    NewPte->u.Trans.Transition = 1;
    NewPte->u.Trans.Protection = Protection;
    NewPte->u.Trans.PageFrameNumber = Page;
}

Referenced by MiResolvePageFileFault(), and TrimWsList().

◆ MI_PFN_ELEMENT()

FORCEINLINE PMMPFN MI_PFN_ELEMENT ( IN PFN_NUMBER Pfn )

Definition at line 1587 of file miarm.h.

{
    /* Get the entry */
    return &MmPfnDatabase[Pfn];
};

Referenced by MiDecrementReferenceCount(), MiDecrementShareCount(), MiDispatchFault(), MiFindContiguousPages(), MiGetPageProtection(), MiInitializeAndChargePfn(), MiInitializePfn(), MiInitializePfnAndMakePteValid(), MiInitializePfnForOtherProcess(), MiInsertPageInFreeList(), MiInsertPageInList(), MiInsertStandbyListAtFront(), MiMapLockedPagesInUserSpace(), MiReleaseProcessReferenceToSessionDataPage(), MiRemoveAnyPage(), MiRemovePageByColor(), MiRemoveZeroPage(), MiResolveDemandZeroFault(), MiResolvePageFileFault(), MiResolveProtoPteFault(), MiSegmentDelete(), MiSessionCommitPageTables(), MiSessionCreateInternal(), MiUnlinkFreeOrZeroedPage(), MiUnlinkPageFromList(), MmArmAccessFault(), and MmFreeSpecialPool().

◆ MI_UPDATE_VALID_PTE()

FORCEINLINE VOID MI_UPDATE_VALID_PTE	(	IN PMMPTE	PointerPte,
		IN MMPTE	TempPte
	)

Definition at line 991 of file miarm.h.

{
    /* Write the valid PTE */
    ASSERT(PointerPte->u.Hard.Valid == 1);
    ASSERT(TempPte.u.Hard.Valid == 1);
    ASSERT(PointerPte->u.Hard.PageFrameNumber == TempPte.u.Hard.PageFrameNumber);
    *PointerPte = TempPte;
}

Referenced by MiFlushTbAndCapture(), MiSetSystemCodeProtection(), MmArmAccessFault(), and MmChangeKernelResourceSectionProtection().

◆ MI_WRITE_INVALID_PDE()

FORCEINLINE VOID MI_WRITE_INVALID_PDE	(	IN PMMPDE	PointerPde,
		IN MMPDE	InvalidPde
	)

Definition at line 1048 of file miarm.h.

{
    /* Write the invalid PDE */
    ASSERT(InvalidPde.u.Hard.Valid == 0);
    ASSERT(InvalidPde.u.Long != 0);
#ifdef _M_AMD64
    ASSERT(InvalidPde.u.Soft.Protection == MM_EXECUTE_READWRITE);
#endif
    *PointerPde = InvalidPde;
}

Referenced by MmArmAccessFault().

◆ MI_WRITE_INVALID_PTE()

FORCEINLINE VOID MI_WRITE_INVALID_PTE	(	IN PMMPTE	PointerPte,
		IN MMPTE	InvalidPte
	)

Definition at line 1006 of file miarm.h.

{
    /* Write the invalid PTE */
    ASSERT(InvalidPte.u.Hard.Valid == 0);
    *PointerPte = InvalidPte;
}

Referenced by _IRQL_requires_max_(), MiAccessCheck(), MiAddMappedPtes(), MiAllocatePoolPages(), MiDecommitPages(), MiDecrementShareCount(), MiMapViewOfDataSection(), MiProcessValidPteList(), MiProtectFreeNonPagedPool(), MiProtectVirtualMemory(), MiResolvePageFileFault(), MmArmAccessFault(), MmCommitSessionMappedView(), MmCreateKernelStack(), MmGrowKernelStackEx(), and NtAllocateVirtualMemory().

◆ MI_WRITE_VALID_PDE()

FORCEINLINE VOID MI_WRITE_VALID_PDE	(	IN PMMPDE	PointerPde,
		IN MMPDE	TempPde
	)

Definition at line 1031 of file miarm.h.

{
    /* Write the valid PDE */
    ASSERT(PointerPde->u.Hard.Valid == 0);
#ifdef _M_AMD64
    ASSERT(PointerPde->u.Hard.NoExecute == 0);
#endif
    ASSERT(TempPde.u.Hard.Valid == 1);
    *PointerPde = TempPde;
}

Referenced by MiAllocatePoolPages(), MiBuildPagedPool(), MiFillSystemPageDirectory(), MiInitializeAndChargePfn(), MiRemoveMappedPtes(), MiSessionCommitPageTables(), MiSessionCreateInternal(), and MiSessionInitializeWorkingSetList().

◆ MI_WRITE_VALID_PTE()

FORCEINLINE VOID MI_WRITE_VALID_PTE	(	IN PMMPTE	PointerPte,
		IN MMPTE	TempPte
	)

Definition at line 973 of file miarm.h.

{
    /* Write the valid PTE */
    ASSERT(PointerPte->u.Hard.Valid == 0);
    ASSERT(TempPte.u.Hard.Valid == 1);
#if _M_AMD64
    ASSERT(!MI_IS_PAGE_TABLE_ADDRESS(MiPteToAddress(PointerPte)) ||
           (TempPte.u.Hard.NoExecute == 0));
#endif
    *PointerPte = TempPte;
}

Referenced by MiAllocatePoolPages(), MiArchCreateProcessAddressSpace(), MiBuildPagedPool(), MiCompleteProtoPteFault(), MiCopyPfn(), MiDispatchFault(), MiInitializeColorTables(), MiInitializePfnAndMakePteValid(), MiInitMachineDependent(), MiLoadImageSection(), MiMapLockedPagesInUserSpace(), MiMapPageInHyperSpace(), MiMapPagesInZeroSpace(), MiMapPDEs(), MiMapPfnDatabase(), MiMapPPEs(), MiMapPTEs(), MiReloadBootLoadedDrivers(), MiResolveDemandZeroFault(), MiResolvePageFileFault(), MiResolveProtoPteFault(), MiResolveTransitionFault(), MiSessionCreateInternal(), MiUnProtectFreeNonPagedPool(), MiZeroPfn(), MmAllocateNonCachedMemory(), MmArmAccessFault(), MmCreateKernelStack(), MmGrowKernelStackEx(), MmMapIoSpace(), MmMapLockedPagesSpecifyCache(), and MmMapLockedPagesWithReservedMapping().

◆ MI_WS_OWNER()

FORCEINLINE BOOLEAN MI_WS_OWNER ( IN PEPROCESS Process )

Definition at line 1090 of file miarm.h.

{
    /* Check if this process is the owner, and that the thread owns the WS */
    if (PsGetCurrentThread()->OwnsProcessWorkingSetExclusive == 0)
    {
        DPRINT("Thread: %p is not an owner\n", PsGetCurrentThread());
    }
    if (KeGetCurrentThread()->ApcState.Process != &Process->Pcb)
    {
        DPRINT("Current thread %p is attached to another process %p\n", PsGetCurrentThread(), Process);
    }
    return ((KeGetCurrentThread()->ApcState.Process == &Process->Pcb) &&
            ((PsGetCurrentThread()->OwnsProcessWorkingSetExclusive) ||
             (PsGetCurrentThread()->OwnsProcessWorkingSetShared)));
}

Referenced by MiUnlockProcessWorkingSet(), MiUnlockProcessWorkingSetForFault(), MiUnlockProcessWorkingSetShared(), and MiUnlockProcessWorkingSetUnsafe().

◆ MiAcquireExpansionLock()

FORCEINLINE KIRQL MiAcquireExpansionLock ( VOID )

Definition at line 1544 of file miarm.h.

{
    KIRQL OldIrql;
 
    ASSERT(KeGetCurrentIrql() <= APC_LEVEL);
    KeAcquireSpinLock(&MmExpansionLock, &OldIrql);
    ASSERT(MiExpansionLockOwner == NULL);
    MiExpansionLockOwner = PsGetCurrentThread();
    return OldIrql;
}

Referenced by MiArchCreateProcessAddressSpace(), MiDereferenceSessionFinal(), MiSessionAddProcess(), MiSessionInitializeWorkingSetList(), MiSessionLeader(), MiSessionRemoveProcess(), MmDeleteProcessAddressSpace(), MmGetSessionById(), and MmWorkingSetManager().

◆ MiAllocatePagesForMdl()

PMDL NTAPI MiAllocatePagesForMdl	(	IN PHYSICAL_ADDRESS	LowAddress,
		IN PHYSICAL_ADDRESS	HighAddress,
		IN PHYSICAL_ADDRESS	SkipBytes,
		IN SIZE_T	TotalBytes,
		IN MI_PFN_CACHE_ATTRIBUTE	CacheAttribute,
		IN ULONG	Flags
	)

Definition at line 182 of file freelist.c.

{
    PMDL Mdl;
    PFN_NUMBER PageCount, LowPage, HighPage, SkipPages, PagesFound = 0, Page;
    PPFN_NUMBER MdlPage, LastMdlPage;
    KIRQL OldIrql;
    PMMPFN Pfn1;
    INT LookForZeroedPages;
 
    ASSERT(KeGetCurrentIrql() <= APC_LEVEL);
    DPRINT("ARM3-DEBUG: Being called with %I64x %I64x %I64x %lx %d %lu\n", LowAddress, HighAddress, SkipBytes, TotalBytes, CacheAttribute, MdlFlags);
 
    //
    // Convert the low address into a PFN
    //
    LowPage = (PFN_NUMBER)(LowAddress.QuadPart >> PAGE_SHIFT);
 
    //
    // Convert, and normalize, the high address into a PFN
    //
    HighPage = (PFN_NUMBER)(HighAddress.QuadPart >> PAGE_SHIFT);
    if (HighPage > MmHighestPhysicalPage) HighPage = MmHighestPhysicalPage;
 
    //
    // Validate skipbytes and convert them into pages
    //
    if (BYTE_OFFSET(SkipBytes.LowPart)) return NULL;
    SkipPages = (PFN_NUMBER)(SkipBytes.QuadPart >> PAGE_SHIFT);
 
    /* This isn't supported at all */
    if (SkipPages) DPRINT1("WARNING: Caller requesting SkipBytes, MDL might be mismatched\n");
 
    //
    // Now compute the number of pages the MDL will cover
    //
    PageCount = (PFN_NUMBER)ADDRESS_AND_SIZE_TO_SPAN_PAGES(0, TotalBytes);
    do
    {
        //
        // Try creating an MDL for these many pages
        //
        Mdl = MmCreateMdl(NULL, NULL, PageCount << PAGE_SHIFT);
        if (Mdl) break;
 
        //
        // This function is not required to return the amount of pages requested
        // In fact, it can return as little as 1 page, and callers are supposed
        // to deal with this scenario. So re-attempt the allocation with less
        // pages than before, and see if it worked this time.
        //
        PageCount -= (PageCount >> 4);
    } while (PageCount);
 
    //
    // Wow, not even a single page was around!
    //
    if (!Mdl) return NULL;
 
    //
    // This is where the page array starts....
    //
    MdlPage = (PPFN_NUMBER)(Mdl + 1);
 
    //
    // Lock the PFN database
    //
    OldIrql = MiAcquirePfnLock();
 
    //
    // Are we looking for any pages, without discriminating?
    //
    if ((LowPage == 0) && (HighPage == MmHighestPhysicalPage))
    {
        //
        // Well then, let's go shopping
        //
        while (PagesFound < PageCount)
        {
            /* Grab a page */
            MI_SET_USAGE(MI_USAGE_MDL);
            MI_SET_PROCESS2("Kernel");
 
            /* FIXME: This check should be smarter */
            Page = 0;
            if (MmAvailablePages != 0)
                Page = MiRemoveAnyPage(0);
 
            if (Page == 0)
            {
                /* This is not good... hopefully we have at least SOME pages */
                ASSERT(PagesFound);
                break;
            }
 
            /* Grab the page entry for it */
            Pfn1 = MiGetPfnEntry(Page);
 
            //
            // Make sure it's really free
            //
            ASSERT(Pfn1->u3.e2.ReferenceCount == 0);
 
            /* Now setup the page and mark it */
            Pfn1->u3.e2.ReferenceCount = 1;
            Pfn1->u2.ShareCount = 1;
            MI_SET_PFN_DELETED(Pfn1);
            Pfn1->u4.PteFrame = 0x1FFEDCB;
            Pfn1->u3.e1.StartOfAllocation = 1;
            Pfn1->u3.e1.EndOfAllocation = 1;
            Pfn1->u4.VerifierAllocation = 0;
 
            //
            // Save it into the MDL
            //
            *MdlPage++ = MiGetPfnEntryIndex(Pfn1);
            PagesFound++;
        }
    }
    else
    {
        //
        // You want specific range of pages. We'll do this in two runs
        //
        for (LookForZeroedPages = 1; LookForZeroedPages >= 0; LookForZeroedPages--)
        {
            //
            // Scan the range you specified
            //
            for (Page = LowPage; Page < HighPage; Page++)
            {
                //
                // Get the PFN entry for this page
                //
                Pfn1 = MiGetPfnEntry(Page);
                if (!Pfn1) continue;
 
                //
                // Make sure it's free and if this is our first pass, zeroed
                //
                if (MiIsPfnInUse(Pfn1)) continue;
                if ((Pfn1->u3.e1.PageLocation == ZeroedPageList) != LookForZeroedPages) continue;
 
                /* Remove the page from the free or zero list */
                ASSERT(Pfn1->u3.e1.ReadInProgress == 0);
                MI_SET_USAGE(MI_USAGE_MDL);
                MI_SET_PROCESS2("Kernel");
                MiUnlinkFreeOrZeroedPage(Pfn1);
 
                //
                // Sanity checks
                //
                ASSERT(Pfn1->u3.e2.ReferenceCount == 0);
 
                //
                // Now setup the page and mark it
                //
                Pfn1->u3.e2.ReferenceCount = 1;
                Pfn1->u2.ShareCount = 1;
                MI_SET_PFN_DELETED(Pfn1);
                Pfn1->u4.PteFrame = 0x1FFEDCB;
                Pfn1->u3.e1.StartOfAllocation = 1;
                Pfn1->u3.e1.EndOfAllocation = 1;
                Pfn1->u4.VerifierAllocation = 0;
 
                //
                // Save this page into the MDL
                //
                *MdlPage++ = Page;
                if (++PagesFound == PageCount) break;
            }
 
            //
            // If the first pass was enough, don't keep going, otherwise, go again
            //
            if (PagesFound == PageCount) break;
        }
    }
 
    //
    // Now release the PFN count
    //
    MiReleasePfnLock(OldIrql);
 
    //
    // We might've found less pages, but not more ;-)
    //
    if (PagesFound != PageCount) ASSERT(PagesFound < PageCount);
    if (!PagesFound)
    {
        //
        // If we didn' tfind any pages at all, fail
        //
        DPRINT1("NO MDL PAGES!\n");
        ExFreePoolWithTag(Mdl, TAG_MDL);
        return NULL;
    }
 
    //
    // Write out how many pages we found
    //
    Mdl->ByteCount = (ULONG)(PagesFound << PAGE_SHIFT);
 
    //
    // Terminate the MDL array if there's certain missing pages
    //
    if (PagesFound != PageCount) *MdlPage = LIST_HEAD;
 
    //
    // Now go back and loop over all the MDL pages
    //
    MdlPage = (PPFN_NUMBER)(Mdl + 1);
    LastMdlPage = MdlPage + PagesFound;
    while (MdlPage < LastMdlPage)
    {
        //
        // Check if we've reached the end
        //
        Page = *MdlPage++;
        if (Page == LIST_HEAD) break;
 
        //
        // Get the PFN entry for the page and check if we should zero it out
        //
        Pfn1 = MiGetPfnEntry(Page);
        ASSERT(Pfn1);
        if (Pfn1->u3.e1.PageLocation != ZeroedPageList) MiZeroPhysicalPage(Page);
        Pfn1->u3.e1.PageLocation = ActiveAndValid;
    }
 
    //
    // We're done, mark the pages as locked
    //
    Mdl->Process = NULL;
    Mdl->MdlFlags |= MDL_PAGES_LOCKED;
    return Mdl;
}

Referenced by MmAllocateNonCachedMemory(), MmAllocatePagesForMdl(), and MmAllocatePagesForMdlEx().

◆ MiCheckForConflictingNode()

TABLE_SEARCH_RESULT NTAPI MiCheckForConflictingNode	(	IN ULONG_PTR	StartVpn,
		IN ULONG_PTR	EndVpn,
		IN PMM_AVL_TABLE	Table,
		OUT PMMADDRESS_NODE *	NodeOrParent
	)

Definition at line 150 of file vadnode.c.

{
    PMMADDRESS_NODE ParentNode, CurrentNode;
 
    ASSERT_LOCKED_FOR_READ(Table);
 
    /* If the tree is empty, there is no conflict */
    if (Table->NumberGenericTableElements == 0) return TableEmptyTree;
 
    /* Start looping from the root node */
    CurrentNode = RtlRightChildAvl(&Table->BalancedRoot);
    ASSERT(CurrentNode != NULL);
    while (CurrentNode)
    {
        ParentNode = CurrentNode;
 
        /* This address comes after */
        if (StartVpn > CurrentNode->EndingVpn)
        {
            /* Keep searching on the right */
            CurrentNode = RtlRightChildAvl(CurrentNode);
        }
        else if (EndVpn < CurrentNode->StartingVpn)
        {
            /* This address ends before the node starts, search on the left */
            CurrentNode = RtlLeftChildAvl(CurrentNode);
        }
        else
        {
            /* This address is part of this node, return it */
            *NodeOrParent = ParentNode;
            return TableFoundNode;
        }
    }
 
    /* There is no more child, save the current node as parent */
    *NodeOrParent = ParentNode;
    if (StartVpn > ParentNode->EndingVpn)
    {
        return TableInsertAsRight;
    }
    else
    {
        return TableInsertAsLeft;
    }
}

Referenced by MiInsertVadEx(), MiMapLockedPagesInUserSpace(), MiProtectVirtualMemory(), MmLocateMemoryAreaByAddress(), MmLocateMemoryAreaByRegion(), and NtAllocateVirtualMemory().

◆ MiCheckForContiguousMemory()

PVOID NTAPI MiCheckForContiguousMemory	(	IN PVOID	BaseAddress,
		IN PFN_NUMBER	BaseAddressPages,
		IN PFN_NUMBER	SizeInPages,
		IN PFN_NUMBER	LowestPfn,
		IN PFN_NUMBER	HighestPfn,
		IN PFN_NUMBER	BoundaryPfn,
		IN MI_PFN_CACHE_ATTRIBUTE	CacheAttribute
	)

Definition at line 214 of file contmem.c.

{
    PMMPTE StartPte, EndPte;
    PFN_NUMBER PreviousPage = 0, Page, HighPage, BoundaryMask, Pages = 0;
 
    //
    // Okay, first of all check if the PFNs match our restrictions
    //
    if (LowestPfn > HighestPfn) return NULL;
    if (LowestPfn + SizeInPages <= LowestPfn) return NULL;
    if (LowestPfn + SizeInPages - 1 > HighestPfn) return NULL;
    if (BaseAddressPages < SizeInPages) return NULL;
 
    //
    // This is the last page we need to get to and the boundary requested
    //
    HighPage = HighestPfn + 1 - SizeInPages;
    BoundaryMask = ~(BoundaryPfn - 1);
 
    //
    // And here's the PTEs for this allocation. Let's go scan them.
    //
    StartPte = MiAddressToPte(BaseAddress);
    EndPte = StartPte + BaseAddressPages;
    while (StartPte < EndPte)
    {
        //
        // Get this PTE's page number
        //
        ASSERT (StartPte->u.Hard.Valid == 1);
        Page = PFN_FROM_PTE(StartPte);
 
        //
        // Is this the beginning of our adventure?
        //
        if (!Pages)
        {
            //
            // Check if this PFN is within our range
            //
            if ((Page >= LowestPfn) && (Page <= HighPage))
            {
                //
                // It is! Do you care about boundary (alignment)?
                //
                if (!(BoundaryPfn) ||
                    (!((Page ^ (Page + SizeInPages - 1)) & BoundaryMask)))
                {
                    //
                    // You don't care, or you do care but we deliver
                    //
                    Pages++;
                }
            }
 
            //
            // Have we found all the pages we need by now?
            // Incidently, this means you only wanted one page
            //
            if (Pages == SizeInPages)
            {
                //
                // Mission complete
                //
                return MiPteToAddress(StartPte);
            }
        }
        else
        {
            //
            // Have we found a page that doesn't seem to be contiguous?
            //
            if (Page != (PreviousPage + 1))
            {
                //
                // Ah crap, we have to start over
                //
                Pages = 0;
                continue;
            }
 
            //
            // Otherwise, we're still in the game. Do we have all our pages?
            //
            if (++Pages == SizeInPages)
            {
                //
                // We do! This entire range was contiguous, so we'll return it!
                //
                return MiPteToAddress(StartPte - Pages + 1);
            }
        }
 
        //
        // Try with the next PTE, remember this PFN
        //
        PreviousPage = Page;
        StartPte++;
        continue;
    }
 
    //
    // All good returns are within the loop...
    //
    return NULL;
}

Referenced by MiAllocateContiguousMemory().

◆ MiCheckPdeForPagedPool()

NTSTATUS FASTCALL MiCheckPdeForPagedPool ( IN PVOID Address )

Definition at line 479 of file pagfault.c.

{
    return STATUS_ACCESS_VIOLATION;
}

Referenced by MiDeletePte(), MiInitializePfn(), MiInitializePfnAndMakePteValid(), and MmArmAccessFault().

◆ MiCheckSecuredVad()

NTSTATUS NTAPI MiCheckSecuredVad	(	IN PMMVAD	Vad,
		IN PVOID	Base,
		IN SIZE_T	Size,
		IN ULONG	ProtectionMask
	)

Definition at line 903 of file vadnode.c.

{
    ULONG_PTR StartAddress, EndAddress;
 
    /* Compute start and end address */
    StartAddress = (ULONG_PTR)Base;
    EndAddress = StartAddress + Size - 1;
 
    /* Are we deleting/unmapping, or changing? */
    if (ProtectionMask < MM_DELETE_CHECK)
    {
        /* Changing... are we allowed to do so? */
        if ((Vad->u.VadFlags.NoChange == 1) &&
            (Vad->u2.VadFlags2.SecNoChange == 1) &&
            (Vad->u.VadFlags.Protection != ProtectionMask))
        {
            /* Nope, bail out */
            DPRINT1("Trying to mess with a no-change VAD!\n");
            return STATUS_INVALID_PAGE_PROTECTION;
        }
    }
    else
    {
        /* This is allowed */
        ProtectionMask = 0;
    }
 
    /* ARM3 doesn't support this yet */
    ASSERT(Vad->u2.VadFlags2.MultipleSecured == 0);
 
    /* Is this a one-secured VAD, like a TEB or PEB? */
    if (Vad->u2.VadFlags2.OneSecured)
    {
        /* Is this allocation being described by the VAD? */
        if ((StartAddress <= ((PMMVAD_LONG)Vad)->u3.Secured.EndVpn) &&
            (EndAddress >= ((PMMVAD_LONG)Vad)->u3.Secured.StartVpn))
        {
            /* Guard page? */
            if (ProtectionMask & MM_DECOMMIT)
            {
                DPRINT1("Not allowed to change protection on guard page!\n");
                return STATUS_INVALID_PAGE_PROTECTION;
            }
 
            /* ARM3 doesn't have read-only VADs yet */
            ASSERT(Vad->u2.VadFlags2.ReadOnly == 0);
 
            /* Check if read-write protections are allowed */
            if (MmReadWrite[ProtectionMask] < MM_READ_WRITE_ALLOWED)
            {
                DPRINT1("Invalid protection mask for RW access!\n");
                return STATUS_INVALID_PAGE_PROTECTION;
            }
        }
    }
 
    /* All good, allow the change */
    return STATUS_SUCCESS;
}

Referenced by MiUnmapViewOfSection(), and NtAllocateVirtualMemory().

◆ MiComputeColorInformation()

VOID NTAPI MiComputeColorInformation ( VOID )

Definition at line 499 of file mminit.c.

{
    ULONG L2Associativity;
 
    /* Check if no setting was provided already */
    if (!MmSecondaryColors)
    {
        /* Get L2 cache information */
        L2Associativity = KeGetPcr()->SecondLevelCacheAssociativity;
 
        /* The number of colors is the number of cache bytes by set/way */
        MmSecondaryColors = KeGetPcr()->SecondLevelCacheSize;
        if (L2Associativity) MmSecondaryColors /= L2Associativity;
    }
 
    /* Now convert cache bytes into pages */
    MmSecondaryColors >>= PAGE_SHIFT;
    if (!MmSecondaryColors)
    {
        /* If there was no cache data from the KPCR, use the default colors */
        MmSecondaryColors = MI_SECONDARY_COLORS;
    }
    else
    {
        /* Otherwise, make sure there aren't too many colors */
        if (MmSecondaryColors > MI_MAX_SECONDARY_COLORS)
        {
            /* Set the maximum */
            MmSecondaryColors = MI_MAX_SECONDARY_COLORS;
        }
 
        /* Make sure there aren't too little colors */
        if (MmSecondaryColors < MI_MIN_SECONDARY_COLORS)
        {
            /* Set the default */
            MmSecondaryColors = MI_SECONDARY_COLORS;
        }
 
        /* Finally make sure the colors are a power of two */
        if (MmSecondaryColors & (MmSecondaryColors - 1))
        {
            /* Set the default */
            MmSecondaryColors = MI_SECONDARY_COLORS;
        }
    }
 
    /* Compute the mask and store it */
    MmSecondaryColorMask = MmSecondaryColors - 1;
    KeGetCurrentPrcb()->SecondaryColorMask = MmSecondaryColorMask;
}

Referenced by MmArmInitSystem().

◆ MiConvertSharedWorkingSetLockToExclusive()

FORCEINLINE BOOLEAN MiConvertSharedWorkingSetLockToExclusive	(	_In_ PETHREAD	Thread,
		_In_ PMMSUPPORT	Vm
	)

Definition at line 1441 of file miarm.h.

{
    /* Sanity check: No exclusive lock. */
    ASSERT(!Thread->OwnsProcessWorkingSetExclusive);
    ASSERT(!Thread->OwnsSessionWorkingSetExclusive);
    ASSERT(!Thread->OwnsSystemWorkingSetExclusive);
 
    /* And it should have one and only one shared lock */
    ASSERT((Thread->OwnsProcessWorkingSetShared + Thread->OwnsSessionWorkingSetShared + Thread->OwnsSystemWorkingSetShared) == 1);
 
    /* Try. */
    if (!ExConvertPushLockSharedToExclusive(&Vm->WorkingSetMutex))
        return FALSE;
 
    if (Vm == &MmSystemCacheWs)
    {
        ASSERT(Thread->OwnsSystemWorkingSetShared);
        Thread->OwnsSystemWorkingSetShared = FALSE;
        Thread->OwnsSystemWorkingSetExclusive = TRUE;
    }
    else if (Vm->Flags.SessionSpace)
    {
        ASSERT(Thread->OwnsSessionWorkingSetShared);
        Thread->OwnsSessionWorkingSetShared = FALSE;
        Thread->OwnsSessionWorkingSetExclusive = TRUE;
    }
    else
    {
        ASSERT(Thread->OwnsProcessWorkingSetShared);
        Thread->OwnsProcessWorkingSetShared = FALSE;
        Thread->OwnsProcessWorkingSetExclusive = TRUE;
    }
 
    return TRUE;
}

Referenced by MmWorkingSetManager().

◆ MiDecrementPageTableReferences()

FORCEINLINE USHORT MiDecrementPageTableReferences ( IN PVOID Address )

Definition at line 2505 of file miarm.h.

{
    PMMPDE PointerPde = MiAddressToPde(Address);
    PMMPFN Pfn;
 
    /* We should not tinker with this one. */
    ASSERT(PointerPde != (PMMPDE)PXE_SELFMAP);
 
    DPRINT("Decrementing %p from %p\n", PointerPde, _ReturnAddress());
 
    /* Make sure we're locked */
    ASSERT(PsGetCurrentThread()->OwnsProcessWorkingSetExclusive);
 
    /* If we're decreasing refcount, then it must be valid! */
    ASSERT(PointerPde->u.Hard.Valid == 1);
 
    /* This lies on the PFN */
    Pfn = MiGetPfnEntry(PFN_FROM_PDE(PointerPde));
 
    ASSERT(Pfn->OriginalPte.u.Soft.UsedPageTableEntries != 0);
    Pfn->OriginalPte.u.Soft.UsedPageTableEntries--;
 
    ASSERT(Pfn->OriginalPte.u.Soft.UsedPageTableEntries < PTE_PER_PAGE);
 
    return Pfn->OriginalPte.u.Soft.UsedPageTableEntries;
}

Referenced by _Success_(), MiDeletePde(), MiDeleteVirtualAddresses(), MiUnmapLockedPagesInUserSpace(), and MmDeletePageFileMapping().

◆ MiDecrementReferenceCount()

VOID NTAPI MiDecrementReferenceCount	(	IN PMMPFN	Pfn1,
		IN PFN_NUMBER	PageFrameIndex
	)

Definition at line 1236 of file pfnlist.c.

{
    /* PFN lock must be held */
    MI_ASSERT_PFN_LOCK_HELD();
 
    /* Handle RosMm PFNs here, too (in case they got locked/unlocked by ARM3) */
    if (MI_IS_ROS_PFN(Pfn1))
    {
        MmDereferencePage(PageFrameIndex);
        return;
    }
 
    /* Sanity checks on the page */
    if (PageFrameIndex > MmHighestPhysicalPage ||
        Pfn1 != MI_PFN_ELEMENT(PageFrameIndex) ||
        Pfn1->u3.e2.ReferenceCount == 0 ||
        Pfn1->u3.e2.ReferenceCount >= 2500)
    {
        DPRINT1("PageFrameIndex=0x%lx, MmHighestPhysicalPage=0x%lx\n", PageFrameIndex, MmHighestPhysicalPage);
        DPRINT1("Pfn1=%p, Element=%p, RefCount=%u\n", Pfn1, MI_PFN_ELEMENT(PageFrameIndex), Pfn1->u3.e2.ReferenceCount);
        ASSERT(PageFrameIndex <= MmHighestPhysicalPage);
        ASSERT(Pfn1 == MI_PFN_ELEMENT(PageFrameIndex));
        ASSERT(Pfn1->u3.e2.ReferenceCount != 0);
        ASSERT(Pfn1->u3.e2.ReferenceCount < 2500);
    }
 
    /* Dereference the page, bail out if it's still alive */
    InterlockedDecrement16((PSHORT)&Pfn1->u3.e2.ReferenceCount);
    if (Pfn1->u3.e2.ReferenceCount) return;
 
    /* Nobody should still have reference to this page */
    if (Pfn1->u2.ShareCount != 0)
    {
        /* Otherwise something's really wrong */
        KeBugCheckEx(PFN_LIST_CORRUPT, 7, PageFrameIndex, Pfn1->u2.ShareCount, 0);
    }
 
    /* And it should be lying on some page list */
    ASSERT(Pfn1->u3.e1.PageLocation != ActiveAndValid);
 
    /* Did someone set the delete flag? */
    if (MI_IS_PFN_DELETED(Pfn1))
    {
        /* Insert it into the free list, there's nothing left to do */
        MiInsertPageInFreeList(PageFrameIndex);
        return;
    }
 
    /* Check to see which list this page should go into */
    if (Pfn1->u3.e1.Modified == 1)
    {
        /* Push it into the modified page list */
        MiInsertPageInList(&MmModifiedPageListHead, PageFrameIndex);
    }
    else
    {
        /* Otherwise, insert this page into the standby list */
        ASSERT(Pfn1->u3.e1.RemovalRequested == 0);
        MiInsertStandbyListAtFront(PageFrameIndex);
    }
}

Referenced by MiDecrementShareCount(), MiDeletePte(), MiDereferencePfnAndDropLockCount(), and MmFreePagesFromMdl().

◆ MiDecrementShareCount()

VOID NTAPI MiDecrementShareCount	(	IN PMMPFN	Pfn1,
		IN PFN_NUMBER	PageFrameIndex
	)

Definition at line 1141 of file pfnlist.c.

{
    PMMPTE PointerPte;
    MMPTE TempPte;
 
    ASSERT(PageFrameIndex > 0);
    ASSERT(MI_PFN_ELEMENT(PageFrameIndex) != NULL);
    ASSERT(Pfn1 == MI_PFN_ELEMENT(PageFrameIndex));
    ASSERT(MI_IS_ROS_PFN(Pfn1) == FALSE);
 
    DPRINT("Decrementing %p from %p\n", Pfn1, _ReturnAddress());
 
    /* Page must be in-use */
    if ((Pfn1->u3.e1.PageLocation != ActiveAndValid) &&
        (Pfn1->u3.e1.PageLocation != StandbyPageList))
    {
        /* Otherwise we have PFN corruption */
        KeBugCheckEx(PFN_LIST_CORRUPT,
                     0x99,
                     PageFrameIndex,
                     Pfn1->u3.e1.PageLocation,
                     0);
    }
 
    /* Page should at least have one reference */
    ASSERT(Pfn1->u3.e2.ReferenceCount != 0);
 
    /* Check if the share count is now 0 */
    ASSERT(Pfn1->u2.ShareCount < 0xF000000);
    if (!--Pfn1->u2.ShareCount)
    {
        /* Was this a prototype PTE? */
        if (Pfn1->u3.e1.PrototypePte)
        {
            /* Grab the PTE address and make sure it's in prototype pool */
            PointerPte = Pfn1->PteAddress;
            ASSERT((PointerPte >= (PMMPTE)MmPagedPoolStart) && (PointerPte <= (PMMPTE)MmPagedPoolEnd));
 
            /* The PTE that backs it should also be valdi */
            PointerPte = MiAddressToPte(PointerPte);
            ASSERT(PointerPte->u.Hard.Valid == 1);
 
            /* Get the original prototype PTE and turn it into a transition PTE */
            PointerPte = Pfn1->PteAddress;
            TempPte = *PointerPte;
            TempPte.u.Soft.Transition = 1;
            TempPte.u.Soft.Valid = 0;
            TempPte.u.Soft.Prototype = 0;
            TempPte.u.Soft.Protection = Pfn1->OriginalPte.u.Soft.Protection;
            MI_WRITE_INVALID_PTE(PointerPte, TempPte);
            DPRINT("Marking PTE: %p as transition (%p - %lx)\n", PointerPte, Pfn1, MiGetPfnEntryIndex(Pfn1));
        }
 
        /* Put the page in transition */
        Pfn1->u3.e1.PageLocation = TransitionPage;
 
        /* PFN lock must be held */
        MI_ASSERT_PFN_LOCK_HELD();
 
        if (Pfn1->u3.e2.ReferenceCount == 1)
        {
            /* Is there still a PFN for this page? */
            if (MI_IS_PFN_DELETED(Pfn1))
            {
                /* Clear the last reference */
                Pfn1->u3.e2.ReferenceCount = 0;
                ASSERT(Pfn1->OriginalPte.u.Soft.Prototype == 0);
 
                /* Mark the page temporarily as valid, we're going to make it free soon */
                Pfn1->u3.e1.PageLocation = ActiveAndValid;
 
                /* Bring it back into the free list */
                MiInsertPageInFreeList(PageFrameIndex);
            }
            else
            {
                /* PFN not yet deleted, drop a ref count */
                MiDecrementReferenceCount(Pfn1, PageFrameIndex);
            }
        }
        else
        {
            /* Otherwise, just drop the reference count */
            InterlockedDecrement16((PSHORT)&Pfn1->u3.e2.ReferenceCount);
        }
    }
}

Referenced by FreeWsleIndex(), MiDeletePte(), MiDeleteSystemPageableVm(), MiFreeContiguousMemory(), MiProcessValidPteList(), MiProtectVirtualMemory(), MiReleaseProcessReferenceToSessionDataPage(), MiRemoveMappedPtes(), MiUnmapLockedPagesInUserSpace(), MmDeleteKernelStack(), MmDeleteProcessAddressSpace(), MmFreeLoaderBlock(), MmFreeSpecialPool(), and TrimWsList().

◆ MiDeleteARM3Section()

VOID NTAPI MiDeleteARM3Section ( PVOID ObjectBody )

Definition at line 2949 of file section.c.

{
    PSECTION SectionObject;
    PCONTROL_AREA ControlArea;
    KIRQL OldIrql;
 
    SectionObject = (PSECTION)ObjectBody;
 
    if (SectionObject->u.Flags.Based == 1)
    {
        /* Remove the node from the global section address tree */
        KeAcquireGuardedMutex(&MmSectionBasedMutex);
        MiRemoveNode(&SectionObject->Address, &MmSectionBasedRoot);
        KeReleaseGuardedMutex(&MmSectionBasedMutex);
    }
 
    /* Lock the PFN database */
    OldIrql = MiAcquirePfnLock();
 
    ASSERT(SectionObject->Segment);
    ASSERT(SectionObject->Segment->ControlArea);
 
    ControlArea = SectionObject->Segment->ControlArea;
 
    /* Dereference */
    ControlArea->NumberOfSectionReferences--;
    ControlArea->NumberOfUserReferences--;
 
    ASSERT(ControlArea->u.Flags.BeingDeleted == 0);
 
    /* Check it. It will delete it if there is no more reference to it */
    MiCheckControlArea(ControlArea, OldIrql);
}

Referenced by MmpDeleteSection().

◆ MiDeletePde()

FORCEINLINE VOID MiDeletePde	(	_In_ PMMPDE	PointerPde,
		_In_ PEPROCESS	CurrentProcess
	)

Definition at line 2539 of file miarm.h.

{
    /* Only for user-mode ones */
    ASSERT(MiIsUserPde(PointerPde));
 
    /* Kill this one as a PTE */
    MiDeletePte((PMMPTE)PointerPde, MiPdeToPte(PointerPde), CurrentProcess, NULL);
#if _MI_PAGING_LEVELS >= 3
    /* Cascade down */
    if (MiDecrementPageTableReferences(MiPdeToPte(PointerPde)) == 0)
    {
        MiDeletePte(MiPdeToPpe(PointerPde), PointerPde, CurrentProcess, NULL);
#if _MI_PAGING_LEVELS == 4
        if (MiDecrementPageTableReferences(PointerPde) == 0)
        {
            MiDeletePte(MiPdeToPxe(PointerPde), MiPdeToPpe(PointerPde), CurrentProcess, NULL);
        }
#endif
    }
#endif
}

Referenced by _Success_(), MiDeleteVirtualAddresses(), MiUnmapLockedPagesInUserSpace(), and MmDeletePageFileMapping().

◆ MiDeletePte()

VOID NTAPI MiDeletePte	(	IN PMMPTE	PointerPte,
		IN PVOID	VirtualAddress,
		IN PEPROCESS	CurrentProcess,
		IN PMMPTE	PrototypePte
	)

Definition at line 369 of file virtual.c.

{
    PMMPFN Pfn1;
    MMPTE TempPte;
    PFN_NUMBER PageFrameIndex;
    PMMPDE PointerPde;
 
    /* PFN lock must be held */
    MI_ASSERT_PFN_LOCK_HELD();
 
    /* WorkingSet must be exclusively locked */
    ASSERT(MM_ANY_WS_LOCK_HELD_EXCLUSIVE(PsGetCurrentThread()));
 
    /* This must be current process. */
    ASSERT(CurrentProcess == PsGetCurrentProcess());
 
    /* Capture the PTE */
    TempPte = *PointerPte;
 
    /* See if the PTE is valid */
    if (TempPte.u.Hard.Valid == 0)
    {
        /* Prototype and paged out PTEs not supported yet */
        ASSERT(TempPte.u.Soft.Prototype == 0);
        ASSERT((TempPte.u.Soft.PageFileHigh == 0) || (TempPte.u.Soft.Transition == 1));
 
        if (TempPte.u.Soft.Transition)
        {
            /* Get the PFN entry */
            PageFrameIndex = PFN_FROM_PTE(&TempPte);
            Pfn1 = MiGetPfnEntry(PageFrameIndex);
 
            DPRINT("Pte %p is transitional!\n", PointerPte);
 
            /* Make sure the saved PTE address is valid */
            ASSERT((PMMPTE)((ULONG_PTR)Pfn1->PteAddress & ~0x1) == PointerPte);
 
            /* Destroy the PTE */
            MI_ERASE_PTE(PointerPte);
 
            /* Drop the reference on the page table. */
            MiDecrementShareCount(MiGetPfnEntry(Pfn1->u4.PteFrame), Pfn1->u4.PteFrame);
 
            /* In case of shared page, the prototype PTE must be in transition, not the process one */
            ASSERT(Pfn1->u3.e1.PrototypePte == 0);
 
            /* Delete the PFN */
            MI_SET_PFN_DELETED(Pfn1);
 
            /* It must be either free (refcount == 0) or being written (refcount == 1) */
            ASSERT(Pfn1->u3.e2.ReferenceCount == Pfn1->u3.e1.WriteInProgress);
 
            /* See if we must free it ourselves, or if it will be freed once I/O is over */
            if (Pfn1->u3.e2.ReferenceCount == 0)
            {
                /* And it should be in standby or modified list */
                ASSERT((Pfn1->u3.e1.PageLocation == ModifiedPageList) || (Pfn1->u3.e1.PageLocation == StandbyPageList));
 
                /* Unlink it and set its reference count to one */
                MiUnlinkPageFromList(Pfn1);
                Pfn1->u3.e2.ReferenceCount++;
 
                /* This will put it back in free list and clean properly up */
                MiDecrementReferenceCount(Pfn1, PageFrameIndex);
            }
            return;
        }
    }
 
    /* Get the PFN entry */
    PageFrameIndex = PFN_FROM_PTE(&TempPte);
    Pfn1 = MiGetPfnEntry(PageFrameIndex);
 
    /* Check if this is a valid, prototype PTE */
    if (Pfn1->u3.e1.PrototypePte == 1)
    {
        /* Get the PDE and make sure it's faulted in */
        PointerPde = MiPteToPde(PointerPte);
        if (PointerPde->u.Hard.Valid == 0)
        {
#if (_MI_PAGING_LEVELS == 2)
            /* Could be paged pool access from a new process -- synchronize the page directories */
            if (!NT_SUCCESS(MiCheckPdeForPagedPool(VirtualAddress)))
            {
#endif
                /* The PDE must be valid at this point */
                KeBugCheckEx(MEMORY_MANAGEMENT,
                             0x61940,
                             (ULONG_PTR)PointerPte,
                             PointerPte->u.Long,
                             (ULONG_PTR)VirtualAddress);
            }
#if (_MI_PAGING_LEVELS == 2)
        }
#endif
        /* Drop the share count on the page table */
        PointerPde = MiPteToPde(PointerPte);
        MiDecrementShareCount(MiGetPfnEntry(PointerPde->u.Hard.PageFrameNumber),
            PointerPde->u.Hard.PageFrameNumber);
 
        /* Drop the share count */
        MiDecrementShareCount(Pfn1, PageFrameIndex);
 
        /* Either a fork, or this is the shared user data page */
        if ((PointerPte <= MiHighestUserPte) && (PrototypePte != Pfn1->PteAddress))
        {
            /* If it's not the shared user page, then crash, since there's no fork() yet */
            if ((PAGE_ALIGN(VirtualAddress) != (PVOID)USER_SHARED_DATA) ||
                 (MmHighestUserAddress <= (PVOID)USER_SHARED_DATA))
            {
                /* Must be some sort of memory corruption */
                KeBugCheckEx(MEMORY_MANAGEMENT,
                             0x400,
                             (ULONG_PTR)PointerPte,
                             (ULONG_PTR)PrototypePte,
                             (ULONG_PTR)Pfn1->PteAddress);
            }
        }
 
        /* Erase it */
        MI_ERASE_PTE(PointerPte);
    }
    else
    {
        /* Make sure the saved PTE address is valid */
        if ((PMMPTE)((ULONG_PTR)Pfn1->PteAddress & ~0x1) != PointerPte)
        {
            /* The PFN entry is illegal, or invalid */
            KeBugCheckEx(MEMORY_MANAGEMENT,
                         0x401,
                         (ULONG_PTR)PointerPte,
                         PointerPte->u.Long,
                         (ULONG_PTR)Pfn1->PteAddress);
        }
 
        /* Erase the PTE */
        MI_ERASE_PTE(PointerPte);
 
        /* There should only be 1 shared reference count */
        ASSERT(Pfn1->u2.ShareCount == 1);
 
        /* Drop the reference on the page table. */
        MiDecrementShareCount(MiGetPfnEntry(Pfn1->u4.PteFrame), Pfn1->u4.PteFrame);
 
        /* Mark the PFN for deletion and dereference what should be the last ref */
        MI_SET_PFN_DELETED(Pfn1);
        MiDecrementShareCount(Pfn1, PageFrameIndex);
 
        /* We should eventually do this */
        //CurrentProcess->NumberOfPrivatePages--;
    }
 
    /* Flush the TLB */
    KeFlushCurrentTb();
}

Referenced by MiDeletePde(), MiDeleteVirtualAddresses(), MiResolveProtoPteFault(), and MmArmAccessFault().

◆ MiDeleteSystemPageableVm()

PFN_COUNT NTAPI MiDeleteSystemPageableVm	(	IN PMMPTE	PointerPte,
		IN PFN_NUMBER	PageCount,
		IN ULONG	Flags,
		OUT PPFN_NUMBER	ValidPages
	)

Definition at line 275 of file virtual.c.

{
    PFN_COUNT ActualPages = 0;
    PETHREAD CurrentThread = PsGetCurrentThread();
    PMMPFN Pfn1, Pfn2;
    PFN_NUMBER PageFrameIndex, PageTableIndex;
    KIRQL OldIrql;
    ASSERT(KeGetCurrentIrql() <= APC_LEVEL);
 
    /* Lock the system working set */
    MiLockWorkingSet(CurrentThread, &MmSystemCacheWs);
 
    /* Loop all pages */
    while (PageCount)
    {
        /* Make sure there's some data about the page */
        if (PointerPte->u.Long)
        {
            /* Normally this is one possibility -- freeing a valid page */
            if (PointerPte->u.Hard.Valid)
            {
                /* Get the page PFN */
                PageFrameIndex = PFN_FROM_PTE(PointerPte);
                Pfn1 = MiGetPfnEntry(PageFrameIndex);
 
                /* Should not have any working set data yet */
                ASSERT(Pfn1->u1.WsIndex == 0);
 
                /* Actual valid, legitimate, pages */
                if (ValidPages) (*ValidPages)++;
 
                /* Get the page table entry */
                PageTableIndex = Pfn1->u4.PteFrame;
                Pfn2 = MiGetPfnEntry(PageTableIndex);
 
                /* Lock the PFN database */
                OldIrql = MiAcquirePfnLock();
 
                /* Delete it the page */
                MI_SET_PFN_DELETED(Pfn1);
                MiDecrementShareCount(Pfn1, PageFrameIndex);
 
                /* Decrement the page table too */
                MiDecrementShareCount(Pfn2, PageTableIndex);
 
                /* Release the PFN database */
                MiReleasePfnLock(OldIrql);
 
                /* Destroy the PTE */
                MI_ERASE_PTE(PointerPte);
            }
            else
            {
                /* As always, only handle current ARM3 scenarios */
                ASSERT(PointerPte->u.Soft.Prototype == 0);
                ASSERT(PointerPte->u.Soft.Transition == 0);
 
                /*
                 * The only other ARM3 possibility is a demand zero page, which would
                 * mean freeing some of the paged pool pages that haven't even been
                 * touched yet, as part of a larger allocation.
                 *
                 * Right now, we shouldn't expect any page file information in the PTE
                 */
                ASSERT(PointerPte->u.Soft.PageFileHigh == 0);
 
                /* Destroy the PTE */
                MI_ERASE_PTE(PointerPte);
            }
 
            /* Actual legitimate pages */
            ActualPages++;
        }
 
        /* Keep going */
        PointerPte++;
        PageCount--;
    }
 
    /* Release the working set */
    MiUnlockWorkingSet(CurrentThread, &MmSystemCacheWs);
 
    /* Flush the entire TLB */
    KeFlushEntireTb(TRUE, TRUE);
 
    /* Done */
    return ActualPages;
}

Referenced by MiFreeInitializationCode(), MiFreePoolPages(), MmFreeDriverInitialization(), and MmFreeSpecialPool().

◆ MiDeleteVirtualAddresses()

VOID NTAPI MiDeleteVirtualAddresses	(	IN ULONG_PTR	Va,
		IN ULONG_PTR	EndingAddress,
		IN PMMVAD	Vad
	)

Definition at line 530 of file virtual.c.

{
    PMMPTE PointerPte, PrototypePte, LastPrototypePte;
    PMMPDE PointerPde;
#if (_MI_PAGING_LEVELS >= 3)
    PMMPPE PointerPpe;
#endif
#if (_MI_PAGING_LEVELS >= 4)
    PMMPPE PointerPxe;
#endif
    MMPTE TempPte;
    PEPROCESS CurrentProcess;
    KIRQL OldIrql;
    BOOLEAN AddressGap = FALSE;
    PSUBSECTION Subsection;
 
    /* Get out if this is a fake VAD, RosMm will free the marea pages */
    if ((Vad) && (Vad->u.VadFlags.Spare == 1)) return;
 
    /* Get the current process */
    CurrentProcess = PsGetCurrentProcess();
 
    /* Check if this is a section VAD or a VM VAD */
    if (!(Vad) || (Vad->u.VadFlags.PrivateMemory) || !(Vad->FirstPrototypePte))
    {
        /* Don't worry about prototypes */
        PrototypePte = LastPrototypePte = NULL;
    }
    else
    {
        /* Get the prototype PTE */
        PrototypePte = Vad->FirstPrototypePte;
        LastPrototypePte = Vad->FirstPrototypePte + 1;
    }
 
    /* In all cases, we don't support fork() yet */
    ASSERT(CurrentProcess->CloneRoot == NULL);
 
    /* Loop the PTE for each VA (EndingAddress is inclusive!) */
    while (Va <= EndingAddress)
    {
#if (_MI_PAGING_LEVELS >= 4)
        /* Get the PXE and check if it's valid */
        PointerPxe = MiAddressToPxe((PVOID)Va);
        if (!PointerPxe->u.Hard.Valid)
        {
            /* Check for unmapped range and skip it */
            if (!PointerPxe->u.Long)
            {
                /* There are gaps in the address space */
                AddressGap = TRUE;
 
                /* Update Va and continue looping */
                Va = (ULONG_PTR)MiPxeToAddress(PointerPxe + 1);
                continue;
            }
 
            /* Make the PXE valid */
            MiMakeSystemAddressValid(MiPteToAddress(PointerPxe), CurrentProcess);
        }
#endif
#if (_MI_PAGING_LEVELS >= 3)
        /* Get the PPE and check if it's valid */
        PointerPpe = MiAddressToPpe((PVOID)Va);
        if (!PointerPpe->u.Hard.Valid)
        {
            /* Check for unmapped range and skip it */
            if (!PointerPpe->u.Long)
            {
                /* There are gaps in the address space */
                AddressGap = TRUE;
 
                /* Update Va and continue looping */
                Va = (ULONG_PTR)MiPpeToAddress(PointerPpe + 1);
                continue;
            }
 
            /* Make the PPE valid */
            MiMakeSystemAddressValid(MiPteToAddress(PointerPpe), CurrentProcess);
        }
#endif
        /* Skip invalid PDEs */
        PointerPde = MiAddressToPde((PVOID)Va);
        if (!PointerPde->u.Long)
        {
            /* There are gaps in the address space */
            AddressGap = TRUE;
 
            /* Check if all the PDEs are invalid, so there's nothing to free */
            Va = (ULONG_PTR)MiPdeToAddress(PointerPde + 1);
            continue;
        }
 
        /* Now check if the PDE is mapped in */
        if (!PointerPde->u.Hard.Valid)
        {
            /* It isn't, so map it in */
            PointerPte = MiPteToAddress(PointerPde);
            MiMakeSystemAddressValid(PointerPte, CurrentProcess);
        }
 
        /* Now we should have a valid PDE, mapped in, and still have some VA */
        ASSERT(PointerPde->u.Hard.Valid == 1);
        ASSERT(Va <= EndingAddress);
 
        /* Check if this is a section VAD with gaps in it */
        if ((AddressGap) && (LastPrototypePte))
        {
            /* We need to skip to the next correct prototype PTE */
            PrototypePte = MI_GET_PROTOTYPE_PTE_FOR_VPN(Vad, Va >> PAGE_SHIFT);
 
            /* And we need the subsection to skip to the next last prototype PTE */
            Subsection = MiLocateSubsection(Vad, Va >> PAGE_SHIFT);
            if (Subsection)
            {
                /* Found it! */
                LastPrototypePte = &Subsection->SubsectionBase[Subsection->PtesInSubsection];
            }
            else
            {
                /* No more subsections, we are done with prototype PTEs */
                PrototypePte = NULL;
            }
        }
 
        /* Lock the PFN Database while we delete the PTEs */
        OldIrql = MiAcquirePfnLock();
        PointerPte = MiAddressToPte(Va);
        do
        {
            /* Making sure the PDE is still valid */
            ASSERT(PointerPde->u.Hard.Valid == 1);
 
            /* Capture the PDE and make sure it exists */
            TempPte = *PointerPte;
            if (TempPte.u.Long)
            {
                /* Check if the PTE is actually mapped in */
                if (MI_IS_MAPPED_PTE(&TempPte))
                {
                    /* Are we dealing with section VAD? */
                    if ((LastPrototypePte) && (PrototypePte > LastPrototypePte))
                    {
                        /* We need to skip to the next correct prototype PTE */
                        PrototypePte = MI_GET_PROTOTYPE_PTE_FOR_VPN(Vad, Va >> PAGE_SHIFT);
 
                        /* And we need the subsection to skip to the next last prototype PTE */
                        Subsection = MiLocateSubsection(Vad, Va >> PAGE_SHIFT);
                        if (Subsection)
                        {
                            /* Found it! */
                            LastPrototypePte = &Subsection->SubsectionBase[Subsection->PtesInSubsection];
                        }
                        else
                        {
                            /* No more subsections, we are done with prototype PTEs */
                            PrototypePte = NULL;
                        }
                    }
 
                    /* Check for prototype PTE */
                    if ((TempPte.u.Hard.Valid == 0) &&
                        (TempPte.u.Soft.Prototype == 1))
                    {
                        /* Just nuke it */
                        MI_ERASE_PTE(PointerPte);
                    }
                    else
                    {
                        /* Delete the PTE proper */
                        MiDeletePte(PointerPte,
                                    (PVOID)Va,
                                    CurrentProcess,
                                    PrototypePte);
                    }
                }
                else
                {
                    /* The PTE was never mapped, just nuke it here */
                    MI_ERASE_PTE(PointerPte);
                }
 
                if (MiDecrementPageTableReferences((PVOID)Va) == 0)
                {
                    ASSERT(PointerPde->u.Long != 0);
 
                    /* Delete the PDE proper */
                    MiDeletePde(PointerPde, CurrentProcess);
 
                    /* Continue with the next PDE */
                    Va = (ULONG_PTR)MiPdeToAddress(PointerPde + 1);
 
                    /* Use this to detect address gaps */
                    PointerPte++;
 
                    PrototypePte++;
                    break;
                }
            }
 
            /* Update the address and PTE for it */
            Va += PAGE_SIZE;
            PointerPte++;
            PrototypePte++;
        } while ((Va & (PDE_MAPPED_VA - 1)) && (Va <= EndingAddress));
 
        /* Release the lock */
        MiReleasePfnLock(OldIrql);
 
        if (Va > EndingAddress) return;
 
        /* Check if we exited the loop regularly */
        AddressGap = (PointerPte != MiAddressToPte(Va));
    }
}

Referenced by MiRemoveMappedView(), MmCleanProcessAddressSpace(), MmDeleteTeb(), and NtFreeVirtualMemory().

◆ MiDereferencePfnAndDropLockCount()

FORCEINLINE VOID MiDereferencePfnAndDropLockCount ( IN PMMPFN Pfn1 )

Definition at line 1628 of file miarm.h.

{
    USHORT RefCount, OldRefCount;
    PFN_NUMBER PageFrameIndex;
 
    /* Loop while we decrement the page successfully */
    do
    {
        /* There should be at least one reference */
        OldRefCount = Pfn1->u3.e2.ReferenceCount;
        ASSERT(OldRefCount != 0);
 
        /* Are we the last one */
        if (OldRefCount == 1)
        {
            /* The page shoudln't be shared not active at this point */
            ASSERT(Pfn1->u3.e2.ReferenceCount == 1);
            ASSERT(Pfn1->u3.e1.PageLocation != ActiveAndValid);
            ASSERT(Pfn1->u2.ShareCount == 0);
 
            /* Is it a prototype PTE? */
            if ((Pfn1->u3.e1.PrototypePte == 1) &&
                (Pfn1->OriginalPte.u.Soft.Prototype == 1))
            {
                /* FIXME: We should return commit */
                DPRINT1("Not returning commit for prototype PTE\n");
            }
 
            /* Update the counter, and drop a reference the long way */
            InterlockedDecrementSizeT(&MmSystemLockPagesCount);
            PageFrameIndex = MiGetPfnEntryIndex(Pfn1);
            MiDecrementReferenceCount(Pfn1, PageFrameIndex);
            return;
        }
 
        /* Drop a reference the short way, and that's it */
        RefCount = InterlockedCompareExchange16((PSHORT)&Pfn1->u3.e2.ReferenceCount,
                                                OldRefCount - 1,
                                                OldRefCount);
        ASSERT(RefCount != 0);
    } while (OldRefCount != RefCount);
 
    /* If we got here, there should be more than one reference */
    ASSERT(RefCount > 1);
    if (RefCount == 2)
    {
        /* Is it still being shared? */
        if (Pfn1->u2.ShareCount >= 1)
        {
            /* Then it should be valid */
            ASSERT(Pfn1->u3.e1.PageLocation == ActiveAndValid);
 
            /* Is it a prototype PTE? */
            if ((Pfn1->u3.e1.PrototypePte == 1) &&
                (Pfn1->OriginalPte.u.Soft.Prototype == 1))
            {
                /* We don't handle ethis */
                ASSERT(FALSE);
            }
 
            /* Update the counter */
            InterlockedDecrementSizeT(&MmSystemLockPagesCount);
        }
    }
}

Referenced by MI_UNLOCK_VA(), MiCompleteProtoPteFault(), MiDispatchFault(), and MmUnlockPages().

◆ MiDetermineUserGlobalPteMask()

FORCEINLINE ULONG_PTR MiDetermineUserGlobalPteMask ( IN PVOID PointerPte )

Definition at line 750 of file miarm.h.

{
    MMPTE TempPte;
 
    /* Start fresh */
    TempPte.u.Long = 0;
 
    /* Make it valid and accessed */
    TempPte.u.Hard.Valid = TRUE;
    MI_MAKE_ACCESSED_PAGE(&TempPte);
 
    /* Is this for user-mode? */
    if (
#if (_MI_PAGING_LEVELS == 4)
        MiIsUserPxe(PointerPte) ||
#endif
#if (_MI_PAGING_LEVELS >= 3)
        MiIsUserPpe(PointerPte) ||
#endif
        MiIsUserPde(PointerPte) ||
        MiIsUserPte(PointerPte))
    {
        /* Set the owner bit */
        MI_MAKE_OWNER_PAGE(&TempPte);
    }
 
    /* FIXME: We should also set the global bit */
 
    /* Return the protection */
    return TempPte.u.Long;
}

Referenced by MI_MAKE_HARDWARE_PTE(), and MiResolveTransitionFault().

◆ MiDropLockCount()

FORCEINLINE VOID MiDropLockCount ( IN PMMPFN Pfn1 )

Definition at line 1598 of file miarm.h.

{
    /* This page shouldn't be locked, but it should be valid */
    ASSERT(Pfn1->u3.e2.ReferenceCount != 0);
    ASSERT(Pfn1->u2.ShareCount == 0);
 
    /* Is this the last reference to the page */
    if (Pfn1->u3.e2.ReferenceCount == 1)
    {
        /* It better not be valid */
        ASSERT(Pfn1->u3.e1.PageLocation != ActiveAndValid);
 
        /* Is it a prototype PTE? */
        if ((Pfn1->u3.e1.PrototypePte == 1) &&
            (Pfn1->OriginalPte.u.Soft.Prototype == 1))
        {
            /* FIXME: We should return commit */
            DPRINT1("Not returning commit for prototype PTE\n");
        }
 
        /* Update the counter */
        InterlockedDecrementSizeT(&MmSystemLockPagesCount);
    }
}

Referenced by MiResolveTransitionFault().

◆ MiFindContiguousPages()

PFN_NUMBER NTAPI MiFindContiguousPages	(	IN PFN_NUMBER	LowestPfn,
		IN PFN_NUMBER	HighestPfn,
		IN PFN_NUMBER	BoundaryPfn,
		IN PFN_NUMBER	SizeInPages,
		IN MEMORY_CACHING_TYPE	CacheType
	)

Definition at line 22 of file contmem.c.

{
    PFN_NUMBER Page, PageCount, LastPage, Length, BoundaryMask;
    ULONG i = 0;
    PMMPFN Pfn1, EndPfn;
    KIRQL OldIrql;
    PAGED_CODE();
    ASSERT(SizeInPages != 0);
 
    //
    // Convert the boundary PFN into an alignment mask
    //
    BoundaryMask = ~(BoundaryPfn - 1);
 
    /* Disable APCs */
    KeEnterGuardedRegion();
 
    //
    // Loop all the physical memory blocks
    //
    do
    {
        //
        // Capture the base page and length of this memory block
        //
        Page = MmPhysicalMemoryBlock->Run[i].BasePage;
        PageCount = MmPhysicalMemoryBlock->Run[i].PageCount;
 
        //
        // Check how far this memory block will go
        //
        LastPage = Page + PageCount;
 
        //
        // Trim it down to only the PFNs we're actually interested in
        //
        if ((LastPage - 1) > HighestPfn) LastPage = HighestPfn + 1;
        if (Page < LowestPfn) Page = LowestPfn;
 
        //
        // Skip this run if it's empty or fails to contain all the pages we need
        //
        if (!(PageCount) || ((Page + SizeInPages) > LastPage)) continue;
 
        //
        // Now scan all the relevant PFNs in this run
        //
        Length = 0;
        for (Pfn1 = MI_PFN_ELEMENT(Page); Page < LastPage; Page++, Pfn1++)
        {
            //
            // If this PFN is in use, ignore it
            //
            if (MiIsPfnInUse(Pfn1))
            {
                Length = 0;
                continue;
            }
 
            //
            // If we haven't chosen a start PFN yet and the caller specified an
            // alignment, make sure the page matches the alignment restriction
            //
            if ((!(Length) && (BoundaryPfn)) &&
                (((Page ^ (Page + SizeInPages - 1)) & BoundaryMask)))
            {
                //
                // It does not, so bail out
                //
                continue;
            }
 
            //
            // Increase the number of valid pages, and check if we have enough
            //
            if (++Length == SizeInPages)
            {
                //
                // It appears we've amassed enough legitimate pages, rollback
                //
                Pfn1 -= (Length - 1);
                Page -= (Length - 1);
 
                //
                // Acquire the PFN lock
                //
                OldIrql = MiAcquirePfnLock();
                do
                {
                    //
                    // Things might've changed for us. Is the page still free?
                    //
                    if (MiIsPfnInUse(Pfn1)) break;
 
                    //
                    // So far so good. Is this the last confirmed valid page?
                    //
                    if (!--Length)
                    {
                        //
                        // Sanity check that we didn't go out of bounds
                        //
                        ASSERT(i != MmPhysicalMemoryBlock->NumberOfRuns);
 
                        //
                        // Loop until all PFN entries have been processed
                        //
                        EndPfn = Pfn1 - SizeInPages + 1;
                        do
                        {
                            //
                            // This PFN is now a used page, set it up
                            //
                            MI_SET_USAGE(MI_USAGE_CONTINOUS_ALLOCATION);
                            MI_SET_PROCESS2("Kernel Driver");
                            MiUnlinkFreeOrZeroedPage(Pfn1);
                            Pfn1->u3.e2.ReferenceCount = 1;
                            Pfn1->u2.ShareCount = 1;
                            Pfn1->u3.e1.PageLocation = ActiveAndValid;
                            Pfn1->u3.e1.StartOfAllocation = 0;
                            Pfn1->u3.e1.EndOfAllocation = 0;
                            Pfn1->u3.e1.PrototypePte = 0;
                            Pfn1->u4.VerifierAllocation = 0;
                            Pfn1->PteAddress = (PVOID)(ULONG_PTR)0xBAADF00DBAADF00DULL;
 
                            //
                            // Check if this is the last PFN, otherwise go on
                            //
                            if (Pfn1 == EndPfn) break;
                            Pfn1--;
                        } while (TRUE);
 
                        //
                        // Mark the first and last PFN so we can find them later
                        //
                        Pfn1->u3.e1.StartOfAllocation = 1;
                        (Pfn1 + SizeInPages - 1)->u3.e1.EndOfAllocation = 1;
 
                        //
                        // Now it's safe to let go of the PFN lock
                        //
                        MiReleasePfnLock(OldIrql);
 
                        //
                        // Quick sanity check that the last PFN is consistent
                        //
                        EndPfn = Pfn1 + SizeInPages;
                        ASSERT(EndPfn == MI_PFN_ELEMENT(Page + 1));
 
                        //
                        // Compute the first page, and make sure it's consistent
                        //
                        Page = Page - SizeInPages + 1;
                        ASSERT(Pfn1 == MI_PFN_ELEMENT(Page));
                        ASSERT(Page != 0);
 
                        /* Enable APCs and return the page */
                        KeLeaveGuardedRegion();
                        return Page;
                    }
 
                    //
                    // Keep going. The purpose of this loop is to reconfirm that
                    // after acquiring the PFN lock these pages are still usable
                    //
                    Pfn1++;
                    Page++;
                } while (TRUE);
 
                //
                // If we got here, something changed while we hadn't acquired
                // the PFN lock yet, so we'll have to restart
                //
                MiReleasePfnLock(OldIrql);
                Length = 0;
            }
        }
    } while (++i != MmPhysicalMemoryBlock->NumberOfRuns);
 
    //
    // And if we get here, it means no suitable physical memory runs were found
    //
    KeLeaveGuardedRegion();
    return 0;
}

Referenced by MiFindContiguousMemory().

◆ MiFindEmptyAddressRangeDownBasedTree()

NTSTATUS NTAPI MiFindEmptyAddressRangeDownBasedTree	(	IN SIZE_T	Length,
		IN ULONG_PTR	BoundaryAddress,
		IN ULONG_PTR	Alignment,
		IN PMM_AVL_TABLE	Table,
		OUT PULONG_PTR	Base
	)

Definition at line 799 of file vadnode.c.

{
    PMMADDRESS_NODE Node, LowestNode;
    ULONG_PTR LowVpn, BestVpn;
 
    ASSERT_LOCKED_FOR_READ(Table);
 
    /* Sanity checks */
    ASSERT(Table == &MmSectionBasedRoot);
    ASSERT(BoundaryAddress);
    ASSERT(BoundaryAddress <= ((ULONG_PTR)MM_HIGHEST_VAD_ADDRESS + 1));
 
    /* Compute page length, make sure the boundary address is valid */
    Length = ROUND_TO_PAGES(Length);
    if ((BoundaryAddress + 1) < Length) return STATUS_NO_MEMORY;
 
    /* Check if the table is empty */
    BestVpn = ROUND_DOWN(BoundaryAddress + 1 - Length, Alignment);
    if (Table->NumberGenericTableElements == 0)
    {
        /* Tree is empty, the candidate address is already the best one */
        *Base = BestVpn;
        return STATUS_SUCCESS;
    }
 
    /* Go to the right-most node which should be the biggest address */
    Node = Table->BalancedRoot.RightChild;
    while (RtlRightChildAvl(Node)) Node = RtlRightChildAvl(Node);
 
    /* Check if we can fit in here */
    LowVpn = ROUND_UP(Node->EndingVpn + 1, Alignment);
    if ((LowVpn < BoundaryAddress) && (Length <= (BoundaryAddress - LowVpn)))
    {
#if (NTDDI_VERSION >= NTDDI_VISTA)
        /* Return the address. */
        *Base = BestVpn;
#else
        /* Note: this is a compatibility hack that mimics a bug in the 2k3
           kernel. It will can waste up to Alignment bytes of memory above
           the allocation. This bug was fixed in Windows Vista */
        *Base = ROUND_DOWN(BoundaryAddress - Length, Alignment);
#endif
        return STATUS_SUCCESS;
    }
 
    /* Now loop the Vad nodes */
    do
    {
        /* Break out if we've reached the last node */
        LowestNode = MiGetPreviousNode(Node);
        if (!LowestNode) break;
 
        /* Check if this node could contain the requested address */
        LowVpn = ROUND_UP(LowestNode->EndingVpn + 1, Alignment);
        if ((LowestNode->EndingVpn < BestVpn) &&
            (LowVpn < Node->StartingVpn) &&
            (Length <= (Node->StartingVpn - LowVpn)))
        {
            /* Check if we need to take BoundaryAddress into account */
            if (BoundaryAddress < Node->StartingVpn)
            {
                /* Return the optimal VPN address */
                *Base = BestVpn;
                return STATUS_SUCCESS;
            }
            else
            {
                /* The upper margin is given by the Node's starting address */
                *Base = ROUND_DOWN(Node->StartingVpn - Length, Alignment);
                return STATUS_SUCCESS;
            }
        }
 
        /* Move to the next node */
        Node = LowestNode;
    } while (TRUE);
 
    /* Check if there's enough space before the lowest Vad */
    if ((Node->StartingVpn > (ULONG_PTR)MI_LOWEST_VAD_ADDRESS) &&
        ((Node->StartingVpn - (ULONG_PTR)MI_LOWEST_VAD_ADDRESS) >= Length))
    {
        /* Check if it fits in perfectly */
        if (BoundaryAddress < Node->StartingVpn)
        {
            /* Return the optimal VPN address */
            *Base = BestVpn;
            return STATUS_SUCCESS;
        }
 
        /* Return an aligned base address within this node */
        *Base = ROUND_DOWN(Node->StartingVpn - Length, Alignment);
        return STATUS_SUCCESS;
    }
 
    /* No address space left at all */
    return STATUS_NO_MEMORY;
}

Referenced by MmCreateArm3Section().

◆ MiFindEmptyAddressRangeDownTree()

TABLE_SEARCH_RESULT NTAPI MiFindEmptyAddressRangeDownTree	(	IN SIZE_T	Length,
		IN ULONG_PTR	BoundaryAddress,
		IN ULONG_PTR	Alignment,
		IN PMM_AVL_TABLE	Table,
		OUT PULONG_PTR	Base,
		OUT PMMADDRESS_NODE *	Parent
	)

Definition at line 681 of file vadnode.c.

{
    PMMADDRESS_NODE Node, OldNode = NULL, Child;
    ULONG_PTR LowVpn, HighVpn, AlignmentVpn;
    PFN_NUMBER PageCount;
 
    ASSERT_LOCKED_FOR_READ(Table);
 
    /* Sanity checks */
    ASSERT(BoundaryAddress);
    ASSERT(BoundaryAddress <= ((ULONG_PTR)MM_HIGHEST_VAD_ADDRESS));
    ASSERT((Alignment & (PAGE_SIZE - 1)) == 0);
 
    /* Calculate page numbers for the length and alignment */
    Length = ROUND_TO_PAGES(Length);
    PageCount = Length >> PAGE_SHIFT;
    AlignmentVpn = Alignment / PAGE_SIZE;
 
    /* Check for kernel mode table (memory areas) */
    if (Table->Unused == 1)
    {
        LowVpn = ALIGN_UP_BY((ULONG_PTR)MmSystemRangeStart >> PAGE_SHIFT, AlignmentVpn);
    }
    else
    {
        LowVpn = ALIGN_UP_BY((ULONG_PTR)MM_LOWEST_USER_ADDRESS, Alignment);
    }
 
    /* Check if there is enough space below the boundary */
    if ((LowVpn + Length) > (BoundaryAddress + 1))
    {
        return TableFoundNode;
    }
 
    /* Check if the table is empty */
    if (Table->NumberGenericTableElements == 0)
    {
        /* Tree is empty, the candidate address is already the best one */
        *Base = ALIGN_DOWN_BY(BoundaryAddress + 1 - Length, Alignment);
        return TableEmptyTree;
    }
 
    /* Calculate the initial upper margin */
    HighVpn = (BoundaryAddress + 1) >> PAGE_SHIFT;
 
    /* Starting from the root, follow the right children until we found a node
       that ends above the boundary */
    Node = RtlRightChildAvl(&Table->BalancedRoot);
    while ((Node->EndingVpn < HighVpn) &&
           ((Child = RtlRightChildAvl(Node)) != NULL)) Node = Child;
 
    /* Now loop the Vad nodes */
    while (Node)
    {
        /* Calculate the lower margin */
        LowVpn = ALIGN_UP_BY(Node->EndingVpn + 1, AlignmentVpn);
 
        /* Check if the current bounds are suitable */
        if ((HighVpn > LowVpn) && ((HighVpn - LowVpn) >= PageCount))
        {
            /* There is enough space to add our node */
            LowVpn = ALIGN_DOWN_BY(HighVpn - PageCount, AlignmentVpn);
            *Base = LowVpn << PAGE_SHIFT;
 
            /* Can we use the current node as parent? */
            if (!RtlRightChildAvl(Node))
            {
                /* Node has no right child, so use it as parent */
                *Parent = Node;
                return TableInsertAsRight;
            }
            else
            {
                /* Node has a right child. This means we must have already
                   moved one node left from the right-most node we started
                   with, thus we already have an OldNode! */
                ASSERT(OldNode != NULL);
 
                /* The node we had before is the most left grandchild of
                   that right child, use it as parent. */
                ASSERT(RtlLeftChildAvl(OldNode) == NULL);
                *Parent = OldNode;
                return TableInsertAsLeft;
            }
        }
 
        /* Update the upper margin if necessary */
        if (Node->StartingVpn < HighVpn) HighVpn = Node->StartingVpn;
 
        /* Remember the current node and go to the previous node */
        OldNode = Node;
        Node = MiGetPreviousNode(Node);
    }
 
    /* Check if there's enough space before the lowest Vad */
    LowVpn = ALIGN_UP_BY((ULONG_PTR)MI_LOWEST_VAD_ADDRESS, Alignment) / PAGE_SIZE;
    if ((HighVpn > LowVpn) && ((HighVpn - LowVpn) >= PageCount))
    {
        /* There is enough space to add our address */
        LowVpn = ALIGN_DOWN_BY(HighVpn - PageCount, Alignment >> PAGE_SHIFT);
        *Base = LowVpn << PAGE_SHIFT;
        *Parent = OldNode;
        return TableInsertAsLeft;
    }
 
    /* No address space left at all */
    *Base = 0;
    *Parent = NULL;
    return TableFoundNode;
}

Referenced by MiInsertVadEx(), and MmFindGap().

◆ MiFindEmptyAddressRangeInTree()

TABLE_SEARCH_RESULT NTAPI MiFindEmptyAddressRangeInTree	(	IN SIZE_T	Length,
		IN ULONG_PTR	Alignment,
		IN PMM_AVL_TABLE	Table,
		OUT PMMADDRESS_NODE *	PreviousVad,
		OUT PULONG_PTR	Base
	)

Definition at line 584 of file vadnode.c.

{
    PMMADDRESS_NODE Node, PreviousNode;
    ULONG_PTR PageCount, AlignmentVpn, LowVpn, HighestVpn;
    ASSERT(Length != 0);
 
    ASSERT_LOCKED_FOR_READ(Table);
 
    /* Calculate page numbers for the length, alignment, and starting address */
    PageCount = BYTES_TO_PAGES(Length);
    AlignmentVpn = Alignment >> PAGE_SHIFT;
    LowVpn = ALIGN_UP_BY((ULONG_PTR)MM_LOWEST_USER_ADDRESS >> PAGE_SHIFT, AlignmentVpn);
 
    /* Check for kernel mode table (memory areas) */
    if (Table->Unused == 1)
    {
        LowVpn = ALIGN_UP_BY((ULONG_PTR)MmSystemRangeStart >> PAGE_SHIFT, AlignmentVpn);
    }
 
    /* Check if the table is empty */
    if (Table->NumberGenericTableElements == 0)
    {
        /* Tree is empty, the candidate address is already the best one */
        *Base = LowVpn << PAGE_SHIFT;
        return TableEmptyTree;
    }
 
    /* Otherwise, follow the leftmost child of the right root node's child */
    Node = RtlRightChildAvl(&Table->BalancedRoot);
    while (RtlLeftChildAvl(Node)) Node = RtlLeftChildAvl(Node);
 
    /* Start a search to find a gap */
    PreviousNode = NULL;
    while (Node != NULL)
    {
        /* Check if the gap below the current node is suitable */
        if (Node->StartingVpn >= LowVpn + PageCount)
        {
            /* There is enough space to add our node */
            *Base = LowVpn << PAGE_SHIFT;
 
            /* Can we use the current node as parent? */
            if (RtlLeftChildAvl(Node) == NULL)
            {
                /* Node has no left child, so use it as parent */
                *PreviousVad = Node;
                return TableInsertAsLeft;
            }
            else
            {
                /* Node has a left child, this means that the previous node is
                   the right-most child of it's left child and can be used as
                   the parent. In case we use the space before the left-most
                   node, it's left child must be NULL. */
                ASSERT(PreviousNode != NULL);
                ASSERT(RtlRightChildAvl(PreviousNode) == NULL);
                *PreviousVad = PreviousNode;
                return TableInsertAsRight;
            }
        }
 
        /* The next candidate is above the current node */
        if (Node->EndingVpn >= LowVpn)
            LowVpn = ALIGN_UP_BY(Node->EndingVpn + 1, AlignmentVpn);
 
        /* Remember the current node and go to the next node */
        PreviousNode = Node;
        Node = MiGetNextNode(Node);
    }
 
    /* We're up to the highest VAD, will this allocation fit above it? */
    HighestVpn = ((ULONG_PTR)MM_HIGHEST_VAD_ADDRESS + 1) / PAGE_SIZE;
 
    /* Check for kernel mode table (memory areas) */
    if (Table->Unused == 1)
    {
        HighestVpn = ALIGN_UP_BY((ULONG_PTR)(LONG_PTR)-1 >> PAGE_SHIFT, AlignmentVpn);
    }
 
    if (HighestVpn >= LowVpn + PageCount)
    {
        /* Yes! Use this VAD to store the allocation */
        *PreviousVad = PreviousNode;
        *Base = LowVpn << PAGE_SHIFT;
        return TableInsertAsRight;
    }
 
    /* Nyet, there's no free address space for this allocation, so we'll fail */
    return TableFoundNode;
}

Referenced by MiInsertVadEx(), MiMapLockedPagesInUserSpace(), and MmFindGap().

◆ MiGetNextNode()

PMMADDRESS_NODE NTAPI MiGetNextNode ( IN PMMADDRESS_NODE Node )

Definition at line 549 of file vadnode.c.

{
    PMMADDRESS_NODE Parent;
 
    /* Get the right child */
    if (RtlRightChildAvl(Node))
    {
        /* Get left-most child */
        Node = RtlRightChildAvl(Node);
        while (RtlLeftChildAvl(Node)) Node = RtlLeftChildAvl(Node);
        return Node;
    }
 
    Parent = RtlParentAvl(Node);
    ASSERT(Parent != NULL);
    while (Parent != Node)
    {
        /* The parent should be a left child, return the real predecessor */
        if (RtlIsLeftChildAvl(Node))
        {
            /* Return it */
            return Parent;
        }
 
        /* Keep lopping until we find our parent */
        Node = Parent;
        Parent = RtlParentAvl(Node);
    }
 
    /* Nothing found */
    return NULL;
}

Referenced by MiFindEmptyAddressRangeInTree(), and MiQueryMemoryBasicInformation().

◆ MiGetPageProtection()

ULONG NTAPI MiGetPageProtection ( IN PMMPTE PointerPte )

Definition at line 1355 of file virtual.c.

{
    MMPTE TempPte;
    PMMPFN Pfn;
    PEPROCESS CurrentProcess;
    PETHREAD CurrentThread;
    BOOLEAN WsSafe, WsShared;
    ULONG Protect;
    KIRQL OldIrql;
    PAGED_CODE();
 
    /* Copy this PTE's contents */
    TempPte = *PointerPte;
 
    /* Assure it's not totally zero */
    ASSERT(TempPte.u.Long);
 
    /* Check for a special prototype format */
    if ((TempPte.u.Soft.Valid == 0) &&
        (TempPte.u.Soft.Prototype == 1))
    {
        /* Check if the prototype PTE is not yet pointing to a PTE */
        if (TempPte.u.Soft.PageFileHigh == MI_PTE_LOOKUP_NEEDED)
        {
            /* The prototype PTE contains the protection */
            return MmProtectToValue[TempPte.u.Soft.Protection];
        }
 
        /* Get a pointer to the underlying shared PTE */
        PointerPte = MiProtoPteToPte(&TempPte);
 
        /* Since the PTE we want to read can be paged out at any time, we need
           to release the working set lock first, so that it can be paged in */
        CurrentThread = PsGetCurrentThread();
        CurrentProcess = PsGetCurrentProcess();
        MiUnlockProcessWorkingSetForFault(CurrentProcess,
                                          CurrentThread,
                                          &WsSafe,
                                          &WsShared);
 
        /* Now read the PTE value */
        TempPte = *PointerPte;
 
        /* Check if that one is invalid */
        if (!TempPte.u.Hard.Valid)
        {
            /* We get the protection directly from this PTE */
            Protect = MmProtectToValue[TempPte.u.Soft.Protection];
        }
        else
        {
            /* The PTE is valid, so we might need to get the protection from
               the PFN. Lock the PFN database */
            OldIrql = MiAcquirePfnLock();
 
            /* Check if the PDE is still valid */
            if (MiAddressToPte(PointerPte)->u.Hard.Valid == 0)
            {
                /* It's not, make it valid */
                MiMakeSystemAddressValidPfn(PointerPte, OldIrql);
            }
 
            /* Now it's safe to read the PTE value again */
            TempPte = *PointerPte;
            ASSERT(TempPte.u.Long != 0);
 
            /* Check again if the PTE is invalid */
            if (!TempPte.u.Hard.Valid)
            {
                /* The PTE is not valid, so we can use it's protection field */
                Protect = MmProtectToValue[TempPte.u.Soft.Protection];
            }
            else
            {
                /* The PTE is valid, so we can find the protection in the
                   OriginalPte field of the PFN */
                Pfn = MI_PFN_ELEMENT(TempPte.u.Hard.PageFrameNumber);
                Protect = MmProtectToValue[Pfn->OriginalPte.u.Soft.Protection];
            }
 
            /* Release the PFN database */
            MiReleasePfnLock(OldIrql);
        }
 
        /* Lock the working set again */
        MiLockProcessWorkingSetForFault(CurrentProcess,
                                        CurrentThread,
                                        WsSafe,
                                        WsShared);
 
        return Protect;
    }
 
    /* In the easy case of transition or demand zero PTE just return its protection */
    if (!TempPte.u.Hard.Valid) return MmProtectToValue[TempPte.u.Soft.Protection];
 
    /* If we get here, the PTE is valid, so look up the page in PFN database */
    Pfn = MiGetPfnEntry(TempPte.u.Hard.PageFrameNumber);
    if (!Pfn->u3.e1.PrototypePte)
    {
        /* Return protection of the original pte */
        ASSERT(Pfn->u4.AweAllocation == 0);
        return MmProtectToValue[Pfn->OriginalPte.u.Soft.Protection];
    }
 
    /* This is software PTE */
    DPRINT("Prototype PTE: %lx %p\n", TempPte.u.Hard.PageFrameNumber, Pfn);
    DPRINT("VA: %p\n", MiPteToAddress(&TempPte));
    DPRINT("Mask: %lx\n", TempPte.u.Soft.Protection);
    DPRINT("Mask2: %lx\n", Pfn->OriginalPte.u.Soft.Protection);
    return MmProtectToValue[TempPte.u.Soft.Protection];
}

Referenced by MiProtectVirtualMemory(), MiQueryAddressState(), and NtAllocateVirtualMemory().

◆ MiGetPreviousNode()

PMMADDRESS_NODE NTAPI MiGetPreviousNode ( IN PMMADDRESS_NODE Node )

Definition at line 513 of file vadnode.c.

{
    PMMADDRESS_NODE Parent;
 
    /* Get the left child */
    if (RtlLeftChildAvl(Node))
    {
        /* Get right-most child */
        Node = RtlLeftChildAvl(Node);
        while (RtlRightChildAvl(Node)) Node = RtlRightChildAvl(Node);
        return Node;
    }
 
    Parent = RtlParentAvl(Node);
    ASSERT(Parent != NULL);
    while (Parent != Node)
    {
        /* The parent should be a right child, return the real predecessor */
        if (RtlIsRightChildAvl(Node))
        {
            /* Return it unless it's the root */
            if (Parent == RtlParentAvl(Parent)) Parent = NULL;
            return Parent;
        }
 
        /* Keep lopping until we find our parent */
        Node = Parent;
        Parent = RtlParentAvl(Node);
    }
 
    /* Nothing found */
    return NULL;
}

Referenced by MiFindEmptyAddressRangeDownBasedTree(), and MiFindEmptyAddressRangeDownTree().

◆ MiIncrementPageTableReferences()

FORCEINLINE USHORT MiIncrementPageTableReferences ( IN PVOID Address )

Definition at line 2479 of file miarm.h.

{
    PMMPDE PointerPde = MiAddressToPde(Address);
    PMMPFN Pfn;
 
    /* We should not tinker with this one. */
    ASSERT(PointerPde != (PMMPDE)PXE_SELFMAP);
    DPRINT("Incrementing %p from %p\n", Address, _ReturnAddress());
 
    /* Make sure we're locked */
    ASSERT(PsGetCurrentThread()->OwnsProcessWorkingSetExclusive);
 
    /* If we're bumping refcount, then it must be valid! */
    ASSERT(PointerPde->u.Hard.Valid == 1);
 
    /* This lies on the PFN */
    Pfn = MiGetPfnEntry(PFN_FROM_PDE(PointerPde));
    Pfn->OriginalPte.u.Soft.UsedPageTableEntries++;
 
    ASSERT(Pfn->OriginalPte.u.Soft.UsedPageTableEntries <= PTE_PER_PAGE);
 
    return Pfn->OriginalPte.u.Soft.UsedPageTableEntries;
}

Referenced by MiDecommitPages(), MiMapLockedPagesInUserSpace(), MiProtectVirtualMemory(), MmArmAccessFault(), MmCreatePageFileMapping(), MmCreateVirtualMappingUnsafeEx(), and NtAllocateVirtualMemory().

◆ MiInitializeAndChargePfn()

NTSTATUS NTAPI MiInitializeAndChargePfn	(	OUT PPFN_NUMBER	PageFrameIndex,
		IN PMMPDE	PointerPde,
		IN PFN_NUMBER	ContainingPageFrame,
		IN BOOLEAN	SessionAllocation
	)

Definition at line 1101 of file pfnlist.c.

{
    MMPDE TempPde;
    KIRQL OldIrql;
 
    /* Use either a global or local PDE */
    TempPde = SessionAllocation ? ValidKernelPdeLocal : ValidKernelPde;
 
    /* Lock the PFN database */
    OldIrql = MiAcquirePfnLock();
 
    /* Make sure nobody is racing us */
    if (PointerPde->u.Hard.Valid == 1)
    {
        /* Return special error if that was the case */
        MiReleasePfnLock(OldIrql);
        return STATUS_RETRY;
    }
 
    /* Grab a zero page and set the PFN, then make it valid */
    *PageFrameIndex = MiRemoveZeroPage(MI_GET_NEXT_COLOR());
    TempPde.u.Hard.PageFrameNumber = *PageFrameIndex;
    MI_WRITE_VALID_PDE(PointerPde, TempPde);
 
    /* Initialize the PFN */
    MiInitializePfnForOtherProcess(*PageFrameIndex,
                                   PointerPde,
                                   ContainingPageFrame);
    ASSERT(MI_PFN_ELEMENT(*PageFrameIndex)->u1.WsIndex == 0);
 
    /* Release the lock and return success */
    MiReleasePfnLock(OldIrql);
    return STATUS_SUCCESS;
}

Referenced by MiInitializeSessionPool().

◆ MiInitializeColorTables()

VOID NTAPI MiInitializeColorTables ( VOID )

Definition at line 553 of file mminit.c.

{
    ULONG i;
    PMMPTE PointerPte, LastPte;
    MMPTE TempPte = ValidKernelPte;
 
    /* The color table starts after the ARM3 PFN database */
    MmFreePagesByColor[0] = (PMMCOLOR_TABLES)&MmPfnDatabase[MmHighestPhysicalPage + 1];
 
    /* Loop the PTEs. We have two color tables for each secondary color */
    PointerPte = MiAddressToPte(&MmFreePagesByColor[0][0]);
    LastPte = MiAddressToPte((ULONG_PTR)MmFreePagesByColor[0] +
                             (2 * MmSecondaryColors * sizeof(MMCOLOR_TABLES))
                             - 1);
    while (PointerPte <= LastPte)
    {
        /* Check for valid PTE */
        if (PointerPte->u.Hard.Valid == 0)
        {
            /* Get a page and map it */
            TempPte.u.Hard.PageFrameNumber = MxGetNextPage(1);
            MI_WRITE_VALID_PTE(PointerPte, TempPte);
 
            /* Zero out the page */
            RtlZeroMemory(MiPteToAddress(PointerPte), PAGE_SIZE);
        }
 
        /* Next */
        PointerPte++;
    }
 
    /* Now set the address of the next list, right after this one */
    MmFreePagesByColor[1] = &MmFreePagesByColor[0][MmSecondaryColors];
 
    /* Now loop the lists to set them up */
    for (i = 0; i < MmSecondaryColors; i++)
    {
        /* Set both free and zero lists for each color */
        MmFreePagesByColor[ZeroedPageList][i].Flink = LIST_HEAD;
        MmFreePagesByColor[ZeroedPageList][i].Blink = (PVOID)LIST_HEAD;
        MmFreePagesByColor[ZeroedPageList][i].Count = 0;
        MmFreePagesByColor[FreePageList][i].Flink = LIST_HEAD;
        MmFreePagesByColor[FreePageList][i].Blink = (PVOID)LIST_HEAD;
        MmFreePagesByColor[FreePageList][i].Count = 0;
    }
}

Referenced by MiBuildPfnDatabase(), and MiInitMachineDependent().

◆ MiInitializeDriverLargePageList()

VOID NTAPI MiInitializeDriverLargePageList ( VOID )

Definition at line 68 of file largepag.c.

{
    PWCHAR p, pp;
 
    /* Initialize the list */
    InitializeListHead(&MiLargePageDriverList);
 
    /* Bail out if there's nothing */
    if (MmLargePageDriverBufferLength == 0xFFFFFFFF) return;
 
    /* Loop from start to finish */
    p = MmLargePageDriverBuffer;
    pp = MmLargePageDriverBuffer + (MmLargePageDriverBufferLength / sizeof(WCHAR));
    while (p < pp)
    {
        /* Skip whitespaces */
        if ((*p == L' ') || (*p == L'\n') || (*p == L'\r') || (*p == L'\t'))
        {
            /* Skip the character */
            p++;
            continue;
        }
 
        /* A star means everything */
        if (*p == L'*')
        {
            /* No need to keep going */
            MiLargePageAllDrivers = TRUE;
            break;
        }
 
        DPRINT1("Large page drivers not supported\n");
        ASSERT(FALSE);
    }
}

Referenced by MmArmInitSystem().

◆ MiInitializeLargePageSupport()

VOID NTAPI MiInitializeLargePageSupport ( VOID )

Definition at line 34 of file largepag.c.

{
#if _MI_PAGING_LEVELS > 2
    DPRINT1("MiInitializeLargePageSupport: PAE/x64 Not Implemented\n");
    //ASSERT(FALSE);
#else
    /* Initialize the large-page hyperspace PTE used for initial mapping */
    MiLargePageHyperPte = MiReserveSystemPtes(1, SystemPteSpace);
    ASSERT(MiLargePageHyperPte);
    MiLargePageHyperPte->u.Long = 0;
 
    /* Initialize the process tracking list, and insert the system process */
    InitializeListHead(&MmProcessList);
    InsertTailList(&MmProcessList, &PsGetCurrentProcess()->MmProcessLinks);
#endif
}

Referenced by MmArmInitSystem().

◆ MiInitializeMemoryEvents()

BOOLEAN NTAPI MiInitializeMemoryEvents ( VOID )

Definition at line 1330 of file mminit.c.

{
    UNICODE_STRING LowString = RTL_CONSTANT_STRING(L"\\KernelObjects\\LowMemoryCondition");
    UNICODE_STRING HighString = RTL_CONSTANT_STRING(L"\\KernelObjects\\HighMemoryCondition");
    UNICODE_STRING LowPagedPoolString = RTL_CONSTANT_STRING(L"\\KernelObjects\\LowPagedPoolCondition");
    UNICODE_STRING HighPagedPoolString = RTL_CONSTANT_STRING(L"\\KernelObjects\\HighPagedPoolCondition");
    UNICODE_STRING LowNonPagedPoolString = RTL_CONSTANT_STRING(L"\\KernelObjects\\LowNonPagedPoolCondition");
    UNICODE_STRING HighNonPagedPoolString = RTL_CONSTANT_STRING(L"\\KernelObjects\\HighNonPagedPoolCondition");
    NTSTATUS Status;
 
    /* Check if we have a registry setting */
    if (MmLowMemoryThreshold)
    {
        /* Convert it to pages */
        MmLowMemoryThreshold *= (_1MB / PAGE_SIZE);
    }
    else
    {
        /* The low memory threshold is hit when we don't consider that we have "plenty" of free pages anymore */
        MmLowMemoryThreshold = MmPlentyFreePages;
 
        /* More than one GB of memory? */
        if (MmNumberOfPhysicalPages > 0x40000)
        {
            /* Start at 32MB, and add another 16MB for each GB */
            MmLowMemoryThreshold = (32 * _1MB) / PAGE_SIZE;
            MmLowMemoryThreshold += ((MmNumberOfPhysicalPages - 0x40000) >> 7);
        }
        else if (MmNumberOfPhysicalPages > 0x8000)
        {
            /* For systems with > 128MB RAM, add another 4MB for each 128MB */
            MmLowMemoryThreshold += ((MmNumberOfPhysicalPages - 0x8000) >> 5);
        }
 
        /* Don't let the minimum threshold go past 64MB */
        MmLowMemoryThreshold = min(MmLowMemoryThreshold, (64 * _1MB) / PAGE_SIZE);
    }
 
    /* Check if we have a registry setting */
    if (MmHighMemoryThreshold)
    {
        /* Convert it into pages */
        MmHighMemoryThreshold *= (_1MB / PAGE_SIZE);
    }
    else
    {
        /* Otherwise, the default is three times the low memory threshold */
        MmHighMemoryThreshold = 3 * MmLowMemoryThreshold;
        ASSERT(MmHighMemoryThreshold > MmLowMemoryThreshold);
    }
 
    /* Make sure high threshold is actually higher than the low */
    MmHighMemoryThreshold = max(MmHighMemoryThreshold, MmLowMemoryThreshold);
 
    /* Create the memory events for all the thresholds */
    Status = MiCreateMemoryEvent(&LowString, &MiLowMemoryEvent);
    if (!NT_SUCCESS(Status)) return FALSE;
    Status = MiCreateMemoryEvent(&HighString, &MiHighMemoryEvent);
    if (!NT_SUCCESS(Status)) return FALSE;
    Status = MiCreateMemoryEvent(&LowPagedPoolString, &MiLowPagedPoolEvent);
    if (!NT_SUCCESS(Status)) return FALSE;
    Status = MiCreateMemoryEvent(&HighPagedPoolString, &MiHighPagedPoolEvent);
    if (!NT_SUCCESS(Status)) return FALSE;
    Status = MiCreateMemoryEvent(&LowNonPagedPoolString, &MiLowNonPagedPoolEvent);
    if (!NT_SUCCESS(Status)) return FALSE;
    Status = MiCreateMemoryEvent(&HighNonPagedPoolString, &MiHighNonPagedPoolEvent);
    if (!NT_SUCCESS(Status)) return FALSE;
 
    /* Now setup the pool events */
    MiInitializePoolEvents();
 
    /* Set the initial event state */
    MiNotifyMemoryEvents();
    return TRUE;
}

Referenced by MmInitSystem().

◆ MiInitializeNonPagedPoolThresholds()

VOID NTAPI MiInitializeNonPagedPoolThresholds ( VOID )

Definition at line 186 of file pool.c.

{
    PFN_NUMBER Size = MmMaximumNonPagedPoolInPages;
 
    /* Default low threshold of 8MB or one third of nonpaged pool */
    MiLowNonPagedPoolThreshold = (8 * _1MB) >> PAGE_SHIFT;
    MiLowNonPagedPoolThreshold = min(MiLowNonPagedPoolThreshold, Size / 3);
 
    /* Default high threshold of 20MB or 50% */
    MiHighNonPagedPoolThreshold = (20 * _1MB) >> PAGE_SHIFT;
    MiHighNonPagedPoolThreshold = min(MiHighNonPagedPoolThreshold, Size / 2);
    ASSERT(MiLowNonPagedPoolThreshold < MiHighNonPagedPoolThreshold);
}

Referenced by MiBuildNonPagedPool(), and MiInitMachineDependent().

◆ MiInitializePfn()

VOID NTAPI MiInitializePfn	(	IN PFN_NUMBER	PageFrameIndex,
		IN PMMPTE	PointerPte,
		IN BOOLEAN	Modified
	)

Definition at line 970 of file pfnlist.c.

{
    PMMPFN Pfn1;
    NTSTATUS Status;
    PMMPTE PointerPtePte;
    MI_ASSERT_PFN_LOCK_HELD();
 
    /* Setup the PTE */
    Pfn1 = MI_PFN_ELEMENT(PageFrameIndex);
    Pfn1->PteAddress = PointerPte;
 
    DPRINT("Called for %p from %p\n", Pfn1, _ReturnAddress());
 
    /* Check if this PFN is part of a valid address space */
    if (PointerPte->u.Hard.Valid == 1)
    {
        /* Only valid from MmCreateProcessAddressSpace path */
        ASSERT(PsGetCurrentProcess()->Vm.WorkingSetSize == 0);
 
        /* Make this a demand zero PTE */
        MI_MAKE_SOFTWARE_PTE(&Pfn1->OriginalPte, MM_READWRITE);
    }
    else
    {
        /* Copy the PTE data */
        Pfn1->OriginalPte = *PointerPte;
        ASSERT(!((Pfn1->OriginalPte.u.Soft.Prototype == 0) &&
                 (Pfn1->OriginalPte.u.Soft.Transition == 1)));
    }
 
    /* Otherwise this is a fresh page -- set it up */
    ASSERT(Pfn1->u3.e2.ReferenceCount == 0);
    Pfn1->u3.e2.ReferenceCount = 1;
    Pfn1->u2.ShareCount = 1;
    Pfn1->u3.e1.PageLocation = ActiveAndValid;
    ASSERT(Pfn1->u3.e1.Rom == 0);
    Pfn1->u3.e1.Modified = Modified;
 
    /* Get the page table for the PTE */
    PointerPtePte = MiAddressToPte(PointerPte);
    if (PointerPtePte->u.Hard.Valid == 0)
    {
        /* Make sure the PDE gets paged in properly */
        Status = MiCheckPdeForPagedPool(PointerPte);
        if (!NT_SUCCESS(Status))
        {
            /* Crash */
            KeBugCheckEx(MEMORY_MANAGEMENT,
                         0x61940,
                         (ULONG_PTR)PointerPte,
                         (ULONG_PTR)PointerPtePte->u.Long,
                         (ULONG_PTR)MiPteToAddress(PointerPte));
        }
    }
 
    /* Get the PFN for the page table */
    PageFrameIndex = PFN_FROM_PTE(PointerPtePte);
    ASSERT(PageFrameIndex != 0);
    Pfn1->u4.PteFrame = PageFrameIndex;
 
    DPRINT("Incrementing share count of %lp from %p\n", PageFrameIndex, _ReturnAddress());
 
    /* Increase its share count so we don't get rid of it */
    Pfn1 = MI_PFN_ELEMENT(PageFrameIndex);
    Pfn1->u2.ShareCount++;
}

Referenced by MiLoadImageSection(), MiResolveDemandZeroFault(), MiResolvePageFileFault(), MiResolveProtoPteFault(), MmArmAccessFault(), MmCreateKernelStack(), MmGrowKernelStackEx(), and MmInitializeProcessAddressSpace().

◆ MiInitializePfnAndMakePteValid()

VOID NTAPI MiInitializePfnAndMakePteValid	(	IN PFN_NUMBER	PageFrameIndex,
		IN PMMPTE	PointerPte,
		IN MMPTE	TempPte
	)

Definition at line 1041 of file pfnlist.c.

{
    PMMPFN Pfn1;
    NTSTATUS Status;
    PMMPTE PointerPtePte;
    MI_ASSERT_PFN_LOCK_HELD();
 
    /* PTE must be invalid */
    ASSERT(PointerPte->u.Hard.Valid == 0);
 
    /* Setup the PTE */
    Pfn1 = MI_PFN_ELEMENT(PageFrameIndex);
    Pfn1->PteAddress = PointerPte;
    Pfn1->OriginalPte = DemandZeroPte;
 
    DPRINT("Incrementing %p from %p\n", Pfn1, _ReturnAddress());
 
    /* Otherwise this is a fresh page -- set it up */
    ASSERT(Pfn1->u3.e2.ReferenceCount == 0);
    Pfn1->u3.e2.ReferenceCount++;
    Pfn1->u2.ShareCount++;
    Pfn1->u3.e1.PageLocation = ActiveAndValid;
    ASSERT(Pfn1->u3.e1.Rom == 0);
    Pfn1->u3.e1.Modified = 1;
 
    /* Get the page table for the PTE */
    PointerPtePte = MiAddressToPte(PointerPte);
    if (PointerPtePte->u.Hard.Valid == 0)
    {
        /* Make sure the PDE gets paged in properly */
        Status = MiCheckPdeForPagedPool(PointerPte);
        if (!NT_SUCCESS(Status))
        {
            /* Crash */
            KeBugCheckEx(MEMORY_MANAGEMENT,
                         0x61940,
                         (ULONG_PTR)PointerPte,
                         (ULONG_PTR)PointerPtePte->u.Long,
                         (ULONG_PTR)MiPteToAddress(PointerPte));
        }
    }
 
    /* Get the PFN for the page table */
    PageFrameIndex = PFN_FROM_PTE(PointerPtePte);
    ASSERT(PageFrameIndex != 0);
    Pfn1->u4.PteFrame = PageFrameIndex;
 
    /* Increase its share count so we don't get rid of it */
    Pfn1 = MI_PFN_ELEMENT(PageFrameIndex);
    Pfn1->u2.ShareCount++;
    DPRINT("Incrementing %p from %p\n", Pfn1, _ReturnAddress());
 
    /* Write valid PTE */
    MI_WRITE_VALID_PTE(PointerPte, TempPte);
}

Referenced by GetFreeWsleIndex(), MiSessionCreateInternal(), MiSessionInitializeWorkingSetList(), and MmAllocateSpecialPool().

◆ MiInitializePfnDatabase()

VOID NTAPI MiInitializePfnDatabase ( IN PLOADER_PARAMETER_BLOCK LoaderBlock )

Definition at line 1065 of file mminit.c.

{
    /* Scan memory and start setting up PFN entries */
    MiBuildPfnDatabaseFromPages(LoaderBlock);
 
    /* Add the zero page */
    MiBuildPfnDatabaseZeroPage();
 
    /* Scan the loader block and build the rest of the PFN database */
    MiBuildPfnDatabaseFromLoaderBlock(LoaderBlock);
 
    /* Finally add the pages for the PFN database itself */
    MiBuildPfnDatabaseSelf();
}

Referenced by MiInitMachineDependent().

◆ MiInitializePfnForOtherProcess()

VOID NTAPI MiInitializePfnForOtherProcess	(	IN PFN_NUMBER	PageFrameIndex,
		IN PVOID	PteAddress,
		IN PFN_NUMBER	PteFrame
	)

Definition at line 1301 of file pfnlist.c.

{
    PMMPFN Pfn1;
 
    /* Setup the PTE */
    Pfn1 = MI_PFN_ELEMENT(PageFrameIndex);
    Pfn1->PteAddress = PteAddress;
 
    /* Make this a software PTE */
    MI_MAKE_SOFTWARE_PTE(&Pfn1->OriginalPte, MM_READWRITE);
 
    DPRINT("Called for %p from %p\n", Pfn1, _ReturnAddress());
 
    /* Setup the page */
    ASSERT(Pfn1->u3.e2.ReferenceCount == 0);
    Pfn1->u3.e2.ReferenceCount = 1;
    Pfn1->u2.ShareCount = 1;
    Pfn1->u3.e1.PageLocation = ActiveAndValid;
    Pfn1->u3.e1.Modified = TRUE;
    Pfn1->u4.InPageError = FALSE;
 
    /* Did we get a PFN for the page table */
    if (PteFrame)
    {
        /* Store it */
        Pfn1->u4.PteFrame = PteFrame;
 
        /* Increase its share count so we don't get rid of it */
        Pfn1 = MI_PFN_ELEMENT(PteFrame);
 
        DPRINT("Incrementing %p from %p\n", Pfn1, _ReturnAddress());
        Pfn1->u2.ShareCount++;
    }
}

Referenced by MiAllocatePoolPages(), MiBuildPagedPool(), MiFillSystemPageDirectory(), MiInitializeAndChargePfn(), MiSessionCommitPageTables(), MiSessionCreateInternal(), and MiSessionInitializeWorkingSetList().

◆ MiInitializePoolEvents()

VOID NTAPI MiInitializePoolEvents ( VOID )

Definition at line 203 of file pool.c.

{
    KIRQL OldIrql;
    PFN_NUMBER FreePoolInPages;
 
    /* Lock paged pool */
    KeAcquireGuardedMutex(&MmPagedPoolMutex);
 
    /* Total size of the paged pool minus the allocated size, is free */
    FreePoolInPages = MmSizeOfPagedPoolInPages - MmPagedPoolInfo.AllocatedPagedPool;
 
    /* Check the initial state high state */
    if (FreePoolInPages >= MiHighPagedPoolThreshold)
    {
        /* We have plenty of pool */
        KeSetEvent(MiHighPagedPoolEvent, 0, FALSE);
    }
    else
    {
        /* We don't */
        KeClearEvent(MiHighPagedPoolEvent);
    }
 
    /* Check the initial low state */
    if (FreePoolInPages <= MiLowPagedPoolThreshold)
    {
        /* We're very low in free pool memory */
        KeSetEvent(MiLowPagedPoolEvent, 0, FALSE);
    }
    else
    {
        /* We're not */
        KeClearEvent(MiLowPagedPoolEvent);
    }
 
    /* Release the paged pool lock */
    KeReleaseGuardedMutex(&MmPagedPoolMutex);
 
    /* Now it's time for the nonpaged pool lock */
    OldIrql = KeAcquireQueuedSpinLock(LockQueueMmNonPagedPoolLock);
 
    /* Free pages are the maximum minus what's been allocated */
    FreePoolInPages = MmMaximumNonPagedPoolInPages - MmAllocatedNonPagedPool;
 
    /* Check if we have plenty */
    if (FreePoolInPages >= MiHighNonPagedPoolThreshold)
    {
        /* We do, set the event */
        KeSetEvent(MiHighNonPagedPoolEvent, 0, FALSE);
    }
    else
    {
        /* We don't, clear the event */
        KeClearEvent(MiHighNonPagedPoolEvent);
    }
 
    /* Check if we have very little */
    if (FreePoolInPages <= MiLowNonPagedPoolThreshold)
    {
        /* We do, set the event */
        KeSetEvent(MiLowNonPagedPoolEvent, 0, FALSE);
    }
    else
    {
        /* We don't, clear it */
        KeClearEvent(MiLowNonPagedPoolEvent);
    }
 
    /* We're done, release the nonpaged pool lock */
    KeReleaseQueuedSpinLock(LockQueueMmNonPagedPoolLock, OldIrql);
}

Referenced by MiInitializeMemoryEvents().

◆ MiInitializeSessionIds()

VOID NTAPI MiInitializeSessionIds ( VOID )

Definition at line 116 of file session.c.

{
    ULONG Size, BitmapSize;
    PFN_NUMBER TotalPages;
 
    /* Setup the total number of data pages needed for the structure */
    TotalPages = MI_SESSION_DATA_PAGES_MAXIMUM;
    MiSessionDataPages = ROUND_TO_PAGES(sizeof(MM_SESSION_SPACE)) >> PAGE_SHIFT;
    ASSERT(MiSessionDataPages <= MI_SESSION_DATA_PAGES_MAXIMUM - 3);
    TotalPages -= MiSessionDataPages;
 
    /* Setup the number of pages needed for session pool tags */
    MiSessionTagSizePages = 2;
    MiSessionBigPoolPages = 1;
    MiSessionTagPages = MiSessionTagSizePages + MiSessionBigPoolPages;
    ASSERT(MiSessionTagPages <= TotalPages);
    ASSERT(MiSessionTagPages < MI_SESSION_TAG_PAGES_MAXIMUM);
 
    /* Total pages needed for a session (FIXME: Probably different on PAE/x64) */
    MiSessionCreateCharge = 1 + MiSessionDataPages + MiSessionTagPages;
 
    /* Initialize the lock */
    KeInitializeGuardedMutex(&MiSessionIdMutex);
 
    /* Allocate the bitmap */
    Size = MI_INITIAL_SESSION_IDS;
    BitmapSize = ((Size + 31) / 32) * sizeof(ULONG);
    MiSessionIdBitmap = ExAllocatePoolWithTag(PagedPool,
                                              sizeof(RTL_BITMAP) + BitmapSize,
                                              TAG_MM);
    if (MiSessionIdBitmap)
    {
        /* Free all the bits */
        RtlInitializeBitMap(MiSessionIdBitmap,
                            (PVOID)(MiSessionIdBitmap + 1),
                            Size);
        RtlClearAllBits(MiSessionIdBitmap);
    }
    else
    {
        /* Die if we couldn't allocate the bitmap */
        KeBugCheckEx(INSTALL_MORE_MEMORY,
                     MmNumberOfPhysicalPages,
                     MmLowestPhysicalPage,
                     MmHighestPhysicalPage,
                     0x200);
    }
}

Referenced by MmInitSystem().

◆ MiInitializeSessionPool()

NTSTATUS NTAPI MiInitializeSessionPool ( VOID )

Definition at line 1276 of file pool.c.

{
    PMMPTE PointerPte, LastPte;
    PMMPDE PointerPde, LastPde;
    PFN_NUMBER PageFrameIndex, PdeCount;
    PPOOL_DESCRIPTOR PoolDescriptor;
    PMM_SESSION_SPACE SessionGlobal;
    PMM_PAGED_POOL_INFO PagedPoolInfo;
    NTSTATUS Status;
    ULONG Index, PoolSize, BitmapSize;
    PAGED_CODE();
 
    /* Lock session pool */
    SessionGlobal = MmSessionSpace->GlobalVirtualAddress;
    KeInitializeGuardedMutex(&SessionGlobal->PagedPoolMutex);
 
    /* Setup a valid pool descriptor */
    PoolDescriptor = &MmSessionSpace->PagedPool;
    ExInitializePoolDescriptor(PoolDescriptor,
                               PagedPoolSession,
                               0,
                               0,
                               &SessionGlobal->PagedPoolMutex);
 
    /* Setup the pool addresses */
    MmSessionSpace->PagedPoolStart = (PVOID)MiSessionPoolStart;
    MmSessionSpace->PagedPoolEnd = (PVOID)((ULONG_PTR)MiSessionPoolEnd - 1);
    DPRINT1("Session Pool Start: 0x%p End: 0x%p\n",
            MmSessionSpace->PagedPoolStart, MmSessionSpace->PagedPoolEnd);
 
    /* Reset all the counters */
    PagedPoolInfo = &MmSessionSpace->PagedPoolInfo;
    PagedPoolInfo->PagedPoolCommit = 0;
    PagedPoolInfo->PagedPoolHint = 0;
    PagedPoolInfo->AllocatedPagedPool = 0;
 
    /* Compute PDE and PTE addresses */
    PointerPde = MiAddressToPde(MmSessionSpace->PagedPoolStart);
    PointerPte = MiAddressToPte(MmSessionSpace->PagedPoolStart);
    LastPde = MiAddressToPde(MmSessionSpace->PagedPoolEnd);
    LastPte = MiAddressToPte(MmSessionSpace->PagedPoolEnd);
 
    /* Write them down */
    MmSessionSpace->PagedPoolBasePde = PointerPde;
    PagedPoolInfo->FirstPteForPagedPool = PointerPte;
    PagedPoolInfo->LastPteForPagedPool = LastPte;
    PagedPoolInfo->NextPdeForPagedPoolExpansion = PointerPde + 1;
 
    /* Zero the PDEs */
    PdeCount = LastPde - PointerPde;
    RtlZeroMemory(PointerPde, (PdeCount + 1) * sizeof(MMPTE));
 
    /* Initialize the PFN for the PDE */
    Status = MiInitializeAndChargePfn(&PageFrameIndex,
                                      PointerPde,
                                      MmSessionSpace->SessionPageDirectoryIndex,
                                      TRUE);
    ASSERT(NT_SUCCESS(Status) == TRUE);
 
    /* Initialize the first page table */
    Index = (ULONG_PTR)MmSessionSpace->PagedPoolStart - (ULONG_PTR)MmSessionBase;
    Index >>= 22;
#ifndef _M_AMD64 // FIXME
    ASSERT(MmSessionSpace->PageTables[Index].u.Long == 0);
    MmSessionSpace->PageTables[Index] = *PointerPde;
#endif
 
    /* Bump up counters */
    InterlockedIncrementSizeT(&MmSessionSpace->NonPageablePages);
    InterlockedIncrementSizeT(&MmSessionSpace->CommittedPages);
 
    /* Compute the size of the pool in pages, and of the bitmap for it */
    PoolSize = MmSessionPoolSize >> PAGE_SHIFT;
    BitmapSize = sizeof(RTL_BITMAP) + ((PoolSize + 31) / 32) * sizeof(ULONG);
 
    /* Allocate and initialize the bitmap to track allocations */
    PagedPoolInfo->PagedPoolAllocationMap = ExAllocatePoolWithTag(NonPagedPool,
                                                                  BitmapSize,
                                                                  TAG_MM);
    ASSERT(PagedPoolInfo->PagedPoolAllocationMap != NULL);
    RtlInitializeBitMap(PagedPoolInfo->PagedPoolAllocationMap,
                        (PULONG)(PagedPoolInfo->PagedPoolAllocationMap + 1),
                        PoolSize);
 
    /* Set all bits, but clear the first page table's worth */
    RtlSetAllBits(PagedPoolInfo->PagedPoolAllocationMap);
    RtlClearBits(PagedPoolInfo->PagedPoolAllocationMap, 0, PTE_PER_PAGE);
 
    /* Allocate and initialize the bitmap to track free space */
    PagedPoolInfo->EndOfPagedPoolBitmap = ExAllocatePoolWithTag(NonPagedPool,
                                                                BitmapSize,
                                                                TAG_MM);
    ASSERT(PagedPoolInfo->EndOfPagedPoolBitmap != NULL);
    RtlInitializeBitMap(PagedPoolInfo->EndOfPagedPoolBitmap,
                        (PULONG)(PagedPoolInfo->EndOfPagedPoolBitmap + 1),
                        PoolSize);
 
    /* Clear all the bits and return success */
    RtlClearAllBits(PagedPoolInfo->EndOfPagedPoolBitmap);
    return STATUS_SUCCESS;
}

◆ MiInitializeSessionSpaceLayout()

VOID NTAPI MiInitializeSessionSpaceLayout ( VOID )

Definition at line 63 of file init.c.

{
    /* This is the entire size */
    MmSessionSize = MI_SESSION_SIZE;
 
    /* Start with session space end */
    MiSessionSpaceEnd = (PVOID)MI_SESSION_SPACE_END;
 
    /* The highest range is the session image range */
    MmSessionImageSize = MI_SESSION_IMAGE_SIZE;
    MiSessionImageEnd = MiSessionSpaceEnd;
    MiSessionImageStart = (PUCHAR)MiSessionImageEnd - MmSessionImageSize;
    ASSERT(IS_PAGE_ALIGNED(MiSessionImageStart));
 
    /* Session working set is below the session image range */
    MiSessionSpaceWs = (PUCHAR)MiSessionImageStart - MI_SESSION_WORKING_SET_SIZE;
 
    /* Session view is below the session working set */
    MmSessionViewSize = MI_SESSION_VIEW_SIZE;
    MiSessionViewEnd = MiSessionSpaceWs;
    MiSessionViewStart = (PUCHAR)MiSessionViewEnd - MmSessionViewSize;
    ASSERT(IS_PAGE_ALIGNED(MiSessionViewStart));
 
    /* Session pool is below session view */
    MmSessionPoolSize = MI_SESSION_POOL_SIZE;
    MiSessionPoolEnd = MiSessionViewStart;
    MiSessionPoolStart = (PUCHAR)MiSessionPoolEnd - MmSessionPoolSize;
    ASSERT(IS_PAGE_ALIGNED(MiSessionPoolStart));
 
    /* And it all begins here */
    MmSessionBase = MiSessionPoolStart;
 
    /* System view space ends at session space, so now that we know where
     * this is, we can compute the base address of system view space itself. */
    MmSystemViewSize = MI_SYSTEM_VIEW_SIZE;
    MiSystemViewStart = (PUCHAR)MmSessionBase - MmSystemViewSize;
    ASSERT(IS_PAGE_ALIGNED(MiSystemViewStart));
 
    /* Sanity checks */
    ASSERT(Add2Ptr(MmSessionBase, MmSessionSize) == MiSessionSpaceEnd);
    ASSERT(MiSessionViewEnd <= MiSessionImageStart);
    ASSERT(MmSessionBase <= MiSessionPoolStart);
 
    /* Compute the PTE addresses for all the addresses we carved out */
    MiSessionImagePteStart = MiAddressToPte(MiSessionImageStart);
    MiSessionImagePteEnd = MiAddressToPte(MiSessionImageEnd);
    MiSessionBasePte = MiAddressToPte(MmSessionBase);
    MiSessionLastPte = MiAddressToPte(MiSessionSpaceEnd);
 
    /* Initialize the pointer to the session space structure */
    MmSessionSpace = (PMM_SESSION_SPACE)Add2Ptr(MiSessionImageStart, 0x10000);
}

Referenced by MmArmInitSystem().

◆ MiInitializeSessionWsSupport()

VOID NTAPI MiInitializeSessionWsSupport ( VOID )

Definition at line 40 of file session.c.

{
    /* Initialize the list heads */
    InitializeListHead(&MiSessionWsList);
}

Referenced by MmInitSystem().

◆ MiInitializeSystemPtes()

VOID NTAPI MiInitializeSystemPtes	(	IN PMMPTE	StartingPte,
		IN ULONG	NumberOfPtes,
		IN MMSYSTEM_PTE_POOL_TYPE	PoolType
	)

Definition at line 388 of file syspte.c.

{
    //
    // Sanity checks
    //
    ASSERT(NumberOfPtes >= 1);
 
    //
    // Set the starting and ending PTE addresses for this space
    //
    MmSystemPteBase = MI_SYSTEM_PTE_BASE;
    MmSystemPtesStart[PoolType] = StartingPte;
    MmSystemPtesEnd[PoolType] = StartingPte + NumberOfPtes - 1;
    DPRINT("System PTE space for %d starting at: %p and ending at: %p\n",
           PoolType, MmSystemPtesStart[PoolType], MmSystemPtesEnd[PoolType]);
 
    //
    // Clear all the PTEs to start with
    //
    RtlZeroMemory(StartingPte, NumberOfPtes * sizeof(MMPTE));
 
    //
    // Make the first entry free and link it
    //
    StartingPte->u.List.NextEntry = MM_EMPTY_PTE_LIST;
    MmFirstFreeSystemPte[PoolType].u.Long = 0;
    MmFirstFreeSystemPte[PoolType].u.List.NextEntry = StartingPte -
                                                      MmSystemPteBase;
 
    //
    // The second entry stores the size of this PTE space
    //
    StartingPte++;
    StartingPte->u.Long = 0;
    StartingPte->u.List.NextEntry = NumberOfPtes;
 
    //
    // We also keep a global for it
    //
    MmTotalFreeSystemPtes[PoolType] = NumberOfPtes;
 
    //
    // Check if this is the system PTE space
    //
    if (PoolType == SystemPteSpace)
    {
        //
        // Remember how many PTEs we have
        //
        MmTotalSystemPtes = NumberOfPtes;
    }
}

Referenced by MiBuildSystemPteSpace(), MiInitializeNonPagedPool(), and MiInitMachineDependent().

◆ MiInitializeSystemSpaceMap()

BOOLEAN NTAPI MiInitializeSystemSpaceMap ( IN PMMSESSION InputSession OPTIONAL )

Definition at line 222 of file section.c.

{
    SIZE_T AllocSize, BitmapSize, Size;
    PVOID ViewStart;
    PMMSESSION Session;
 
    /* Check if this a session or system space */
    if (InputSession)
    {
        /* Use the input session */
        Session = InputSession;
        ViewStart = MiSessionViewStart;
        Size = MmSessionViewSize;
    }
    else
    {
        /* Use the system space "session" */
        Session = &MmSession;
        ViewStart = MiSystemViewStart;
        Size = MmSystemViewSize;
    }
 
    /* Initialize the system space lock */
    Session->SystemSpaceViewLockPointer = &Session->SystemSpaceViewLock;
    KeInitializeGuardedMutex(Session->SystemSpaceViewLockPointer);
 
    /* Set the start address */
    Session->SystemSpaceViewStart = ViewStart;
 
    /* Create a bitmap to describe system space */
    BitmapSize = sizeof(RTL_BITMAP) + ((((Size / MI_SYSTEM_VIEW_BUCKET_SIZE) + 31) / 32) * sizeof(ULONG));
    Session->SystemSpaceBitMap = ExAllocatePoolWithTag(NonPagedPool,
                                                       BitmapSize,
                                                       TAG_MM);
    ASSERT(Session->SystemSpaceBitMap);
    RtlInitializeBitMap(Session->SystemSpaceBitMap,
                        (PULONG)(Session->SystemSpaceBitMap + 1),
                        (ULONG)(Size / MI_SYSTEM_VIEW_BUCKET_SIZE));
 
    /* Set system space fully empty to begin with */
    RtlClearAllBits(Session->SystemSpaceBitMap);
 
    /* Set default hash flags */
    Session->SystemSpaceHashSize = 31;
    Session->SystemSpaceHashKey = Session->SystemSpaceHashSize - 1;
    Session->SystemSpaceHashEntries = 0;
 
    /* Calculate how much space for the hash views we'll need */
    AllocSize = sizeof(MMVIEW) * Session->SystemSpaceHashSize;
    ASSERT(AllocSize < PAGE_SIZE);
 
    /* Allocate and zero the view table */
    Session->SystemSpaceViewTable = ExAllocatePoolWithTag(Session == &MmSession ?
                                                          NonPagedPool :
                                                          PagedPool,
                                                          AllocSize,
                                                          TAG_MM);
    ASSERT(Session->SystemSpaceViewTable != NULL);
    RtlZeroMemory(Session->SystemSpaceViewTable, AllocSize);
 
    /* Success */
    return TRUE;
}

Referenced by MiBuildPagedPool(), and MiSessionCreateInternal().

◆ MiInitMachineDependent()

NTSTATUS NTAPI MiInitMachineDependent ( IN PLOADER_PARAMETER_BLOCK LoaderBlock )

Definition at line 697 of file init.c.

{
    NTSTATUS Status;
    ULONG Flags;
 
    ASSERT(MxPfnAllocation != 0);
 
    /* Set some hardcoded addresses */
    MmHyperSpaceEnd = (PVOID)HYPER_SPACE_END;
    MmNonPagedSystemStart = (PVOID)MM_SYSTEM_SPACE_START;
    MmPfnDatabase = (PVOID)MI_PFN_DATABASE;
    MmWorkingSetList = (PVOID)MI_WORKING_SET_LIST;
 
 
//    PrototypePte.u.Proto.Valid = 1
//    PrototypePte.u.ReadOnly
//    PrototypePte.u.Prototype
//    PrototypePte.u.Protection = MM_READWRITE;
//    PrototypePte.u.ProtoAddress
    PrototypePte.u.Soft.PageFileHigh = MI_PTE_LOOKUP_NEEDED;
 
    MiInitializePageTable();
 
    MiBuildNonPagedPool();
 
    MiBuildSystemPteSpace();
 
    /* Map the PFN database pages */
    MiBuildPfnDatabase(LoaderBlock);
 
    /* Reset the ref/share count so that MmInitializeProcessAddressSpace works */
    PMMPFN Pfn = MiGetPfnEntry(PFN_FROM_PTE((PMMPTE)PXE_SELFMAP));
    Pfn->u2.ShareCount = 0;
    Pfn->u3.e2.ReferenceCount = 0;
 
    Pfn = MiGetPfnEntry(PFN_FROM_PDE(MiAddressToPde((PVOID)HYPER_SPACE)));
    Pfn->u2.ShareCount = 0;
    Pfn->u3.e2.ReferenceCount = 0;
 
    Pfn = MiGetPfnEntry(PFN_FROM_PPE(MiAddressToPpe((PVOID)HYPER_SPACE)));
    Pfn->u2.ShareCount = 0;
    Pfn->u3.e2.ReferenceCount = 0;
 
    Pfn = MiGetPfnEntry(PFN_FROM_PXE(MiAddressToPxe((PVOID)HYPER_SPACE)));
    Pfn->u2.ShareCount = 0;
    Pfn->u3.e2.ReferenceCount = 0;
 
    Pfn = MiGetPfnEntry(PFN_FROM_PTE(MiAddressToPte(MmWorkingSetList)));
    Pfn->u2.ShareCount = 0;
    Pfn->u3.e2.ReferenceCount = 0;
 
    /* Initialize the nonpaged pool */
    InitializePool(NonPagedPool, 0);
 
    /* Initialize the bogus address space */
    Flags = 0;
    Status = MmInitializeProcessAddressSpace(PsGetCurrentProcess(), NULL, NULL, &Flags, NULL);
    if (!NT_SUCCESS(Status))
    {
        DPRINT1("MmInitializeProcessAddressSpace(9 failed: 0x%lx\n", Status);
        return Status;
    }
 
    /* Initialize the balancer */
    MmInitializeBalancer((ULONG)MmAvailablePages, 0);
 
    /* Make sure we have everything we need */
    ASSERT(MmPfnDatabase);
    ASSERT(MmNonPagedSystemStart);
    ASSERT(MmNonPagedPoolStart);
    ASSERT(MmSizeOfNonPagedPoolInBytes);
    ASSERT(MmMaximumNonPagedPoolInBytes);
    ASSERT(MmNonPagedPoolExpansionStart);
    ASSERT(MmHyperSpaceEnd);
    ASSERT(MmNumberOfSystemPtes);
    ASSERT(MiAddressToPde(MmNonPagedPoolStart)->u.Hard.Valid);
    ASSERT(MiAddressToPte(MmNonPagedPoolStart)->u.Hard.Valid);
 
    return STATUS_SUCCESS;
}

Referenced by MmArmInitSystem().

◆ MiInsertBasedSection()

VOID NTAPI MiInsertBasedSection ( IN PSECTION Section )

Definition at line 423 of file vadnode.c.

{
    TABLE_SEARCH_RESULT Result;
    PMMADDRESS_NODE Parent = NULL;
    ASSERT(Section->Address.EndingVpn >= Section->Address.StartingVpn);
 
    ASSERT_LOCKED_FOR_WRITE(&MmSectionBasedRoot);
 
    /* Find the parent VAD and where this child should be inserted */
    Result = RtlpFindAvlTableNodeOrParent(&MmSectionBasedRoot, (PVOID)Section->Address.StartingVpn, &Parent);
    ASSERT(Result != TableFoundNode);
    ASSERT((Parent != NULL) || (Result == TableEmptyTree));
    MiInsertNode(&MmSectionBasedRoot, &Section->Address, Parent, Result);
}

Referenced by MmCreateArm3Section().

◆ MiInsertNode()

VOID NTAPI MiInsertNode	(	IN PMM_AVL_TABLE	Table,
		IN PMMADDRESS_NODE	NewNode,
		PMMADDRESS_NODE	Parent,
		TABLE_SEARCH_RESULT	Result
	)

◆ MiInsertPageInFreeList()

VOID NTAPI MiInsertPageInFreeList ( IN PFN_NUMBER PageFrameIndex )

Definition at line 611 of file pfnlist.c.

{
    PMMPFNLIST ListHead;
    PFN_NUMBER LastPage;
    PMMPFN Pfn1;
    ULONG Color;
    PMMPFN Blink;
    PMMCOLOR_TABLES ColorTable;
 
    /* Make sure the page index is valid */
    MI_ASSERT_PFN_LOCK_HELD();
    ASSERT((PageFrameIndex != 0) &&
           (PageFrameIndex <= MmHighestPhysicalPage) &&
           (PageFrameIndex >= MmLowestPhysicalPage));
 
    /* Get the PFN entry */
    Pfn1 = MI_PFN_ELEMENT(PageFrameIndex);
 
    /* Sanity checks that a right kind of page is being inserted here */
    ASSERT(Pfn1->u4.MustBeCached == 0);
    ASSERT(Pfn1->u3.e1.Rom != 1);
    ASSERT(Pfn1->u3.e1.RemovalRequested == 0);
    ASSERT(Pfn1->u4.VerifierAllocation == 0);
    ASSERT(Pfn1->u3.e2.ReferenceCount == 0);
 
    /* Get the free page list and increment its count */
    ListHead = &MmFreePageListHead;
    ASSERT_LIST_INVARIANT(ListHead);
    ListHead->Total++;
 
    /* Get the last page on the list */
    LastPage = ListHead->Blink;
    if (LastPage != LIST_HEAD)
    {
        /* Link us with the previous page, so we're at the end now */
        MI_PFN_ELEMENT(LastPage)->u1.Flink = PageFrameIndex;
    }
    else
    {
        /* The list is empty, so we are the first page */
        ListHead->Flink = PageFrameIndex;
    }
 
    /* Now make the list head point back to us (since we go at the end) */
    ListHead->Blink = PageFrameIndex;
    ASSERT_LIST_INVARIANT(ListHead);
 
    /* And initialize our own list pointers */
    Pfn1->u1.Flink = LIST_HEAD;
    Pfn1->u2.Blink = LastPage;
 
    /* Set the list name and default priority */
    Pfn1->u3.e1.PageLocation = FreePageList;
    Pfn1->u4.Priority = 3;
 
    /* Clear some status fields */
    Pfn1->u4.InPageError = 0;
    Pfn1->u4.AweAllocation = 0;
 
    /* Increment number of available pages */
    MiIncrementAvailablePages();
 
    /* Get the page color */
    Color = PageFrameIndex & MmSecondaryColorMask;
 
    /* Get the first page on the color list */
    ColorTable = &MmFreePagesByColor[FreePageList][Color];
    if (ColorTable->Flink == LIST_HEAD)
    {
        /* The list is empty, so we are the first page */
        Pfn1->u4.PteFrame = COLORED_LIST_HEAD;
        ColorTable->Flink = PageFrameIndex;
    }
    else
    {
        /* Get the previous page */
        Blink = (PMMPFN)ColorTable->Blink;
 
        /* Make it link to us, and link back to it */
        Blink->OriginalPte.u.Long = PageFrameIndex;
        Pfn1->u4.PteFrame = MiGetPfnEntryIndex(Blink);
    }
 
    /* Now initialize our own list pointers */
    ColorTable->Blink = Pfn1;
 
    /* This page is now the last */
    Pfn1->OriginalPte.u.Long = LIST_HEAD;
 
    /* And increase the count in the colored list */
    ColorTable->Count++;
 
    /* Notify zero page thread if enough pages are on the free list now */
    if (ListHead->Total >= 8)
    {
        /* Set the event */
        KeSetEvent(&MmZeroingPageEvent, IO_NO_INCREMENT, FALSE);
    }
 
#if MI_TRACE_PFNS
    Pfn1->PfnUsage = MI_USAGE_FREE_PAGE;
    RtlZeroMemory(Pfn1->ProcessName, 16);
    Pfn1->CallSite = NULL;
#endif
}

Referenced by MiAddDescriptorToDatabase(), MiBuildPfnDatabaseFromLoaderBlock(), MiDecrementReferenceCount(), MiDecrementShareCount(), MiSegmentDelete(), MmCreateProcessAddressSpace(), MmDereferencePage(), and MmFreeLoaderBlock().

◆ MiInsertPageInList()

VOID NTAPI MiInsertPageInList	(	IN PMMPFNLIST	ListHead,
		IN PFN_NUMBER	PageFrameIndex
	)

Definition at line 779 of file pfnlist.c.

{
    PFN_NUMBER Flink, LastPage;
    PMMPFN Pfn1, Pfn2;
    MMLISTS ListName;
    PMMCOLOR_TABLES ColorHead;
    ULONG Color;
 
    /* For free pages, use MiInsertPageInFreeList */
    ASSERT(ListHead != &MmFreePageListHead);
 
    /* Make sure the lock is held */
    MI_ASSERT_PFN_LOCK_HELD();
 
    /* Make sure the PFN is valid */
    ASSERT((PageFrameIndex) &&
           (PageFrameIndex <= MmHighestPhysicalPage) &&
           (PageFrameIndex >= MmLowestPhysicalPage));
 
    /* Page should be unused */
    Pfn1 = MI_PFN_ELEMENT(PageFrameIndex);
    ASSERT(Pfn1->u3.e2.ReferenceCount == 0);
    ASSERT(Pfn1->u3.e1.Rom != 1);
 
    /* Is a standby or modified page being inserted? */
    ListName = ListHead->ListName;
    if ((ListName == StandbyPageList) || (ListName == ModifiedPageList))
    {
        /* If the page is in transition, it must also be a prototype page */
        if ((Pfn1->OriginalPte.u.Soft.Prototype == 0) &&
            (Pfn1->OriginalPte.u.Soft.Transition == 1))
        {
            /* Crash the system on inconsistency */
            KeBugCheckEx(MEMORY_MANAGEMENT, 0x8888, 0, 0, 0);
        }
    }
 
    /* Standby pages are prioritized, so we need to get the real head */
    if (ListHead == &MmStandbyPageListHead)
    {
        /* Obviously the prioritized list should still have the same name */
        ListHead = &MmStandbyPageListByPriority[Pfn1->u4.Priority];
        ASSERT(ListHead->ListName == ListName);
    }
 
    /* Increment the list count */
    ListHead->Total++;
 
    /* Is a modified page being inserted? */
    if (ListHead == &MmModifiedPageListHead)
    {
        /* For now, only single-prototype pages should end up in this path */
        DPRINT("Modified page being added: %lx\n", PageFrameIndex);
        ASSERT(Pfn1->OriginalPte.u.Soft.Prototype == 0);
 
        /* Modified pages are colored when they are selected for page file */
        ListHead = &MmModifiedPageListByColor[0];
        ASSERT (ListHead->ListName == ListName);
        ListHead->Total++;
 
        /* Increment the number of paging file modified pages */
        MmTotalPagesForPagingFile++;
    }
 
    /* Don't handle bad pages yet yet */
    ASSERT(Pfn1->u3.e1.RemovalRequested == 0);
 
    /* Zero pages go to the head, all other pages go to the end */
    if (ListName == ZeroedPageList)
    {
        /* Make the head of the list point to this page now */
        Flink = ListHead->Flink;
        ListHead->Flink = PageFrameIndex;
 
        /* Make the page point to the previous head, and back to the list */
        Pfn1->u1.Flink = Flink;
        Pfn1->u2.Blink = LIST_HEAD;
 
        /* Was the list empty? */
        if (Flink != LIST_HEAD)
        {
            /* It wasn't, so update the backlink of the previous head page */
            Pfn2 = MI_PFN_ELEMENT(Flink);
            Pfn2->u2.Blink = PageFrameIndex;
        }
        else
        {
            /* It was empty, so have it loop back around to this new page */
            ListHead->Blink = PageFrameIndex;
        }
    }
    else
    {
        /* Get the last page on the list */
        LastPage = ListHead->Blink;
        if (LastPage != LIST_HEAD)
        {
            /* Link us with the previous page, so we're at the end now */
            MI_PFN_ELEMENT(LastPage)->u1.Flink = PageFrameIndex;
        }
        else
        {
            /* The list is empty, so we are the first page */
            ListHead->Flink = PageFrameIndex;
        }
 
        /* Now make the list head point back to us (since we go at the end) */
        ListHead->Blink = PageFrameIndex;
        ASSERT_LIST_INVARIANT(ListHead);
 
        /* And initialize our own list pointers */
        Pfn1->u1.Flink = LIST_HEAD;
        Pfn1->u2.Blink = LastPage;
    }
 
    /* Move the page onto its new location */
    Pfn1->u3.e1.PageLocation = ListName;
 
    /* For zero/free pages, we also have to handle the colored lists */
    if (ListName <= StandbyPageList)
    {
        /* Increment number of available pages */
        MiIncrementAvailablePages();
 
        /* Sanity checks */
        ASSERT(ListName == ZeroedPageList);
        ASSERT(Pfn1->u4.InPageError == 0);
 
        /* Get the page color */
        Color = PageFrameIndex & MmSecondaryColorMask;
 
        /* Get the list for this color */
        ColorHead = &MmFreePagesByColor[ZeroedPageList][Color];
 
        /* Get the old head */
        Flink = ColorHead->Flink;
 
        /* Make this page point back to the list, and point forwards to the old head */
        Pfn1->OriginalPte.u.Long = Flink;
        Pfn1->u4.PteFrame = COLORED_LIST_HEAD;
 
        /* Set the new head */
        ColorHead->Flink = PageFrameIndex;
 
        /* Was the head empty? */
        if (Flink != LIST_HEAD)
        {
            /* No, so make the old head point to this page */
            Pfn2 = MI_PFN_ELEMENT(Flink);
            Pfn2->u4.PteFrame = PageFrameIndex;
        }
        else
        {
            /* Yes, make it loop back to this page */
            ColorHead->Blink = (PVOID)Pfn1;
        }
 
        /* One more paged on the colored list */
        ColorHead->Count++;
 
#if MI_TRACE_PFNS
            ASSERT(MI_PFN_CURRENT_USAGE == MI_USAGE_NOT_SET);
            Pfn1->PfnUsage = MI_USAGE_FREE_PAGE;
            RtlZeroMemory(Pfn1->ProcessName, 16);
            Pfn1->CallSite = NULL;
#endif
    }
    else if (ListName == ModifiedPageList)
    {
        /* In ARM3, page must be destined for page file, and not yet written out */
        ASSERT(Pfn1->OriginalPte.u.Soft.Prototype == 0);
        ASSERT(Pfn1->OriginalPte.u.Soft.PageFileHigh == 0);
 
        /* One more transition page */
        MmTransitionSharedPages++;
 
        /* Increment the number of per-process modified pages */
        PsGetCurrentProcess()->ModifiedPageCount++;
 
        /* FIXME: Wake up modified page writer if there are not enough free pages */
    }
    else if (ListName == ModifiedNoWritePageList)
    {
        /* This list is not yet implemented */
        ASSERT(FALSE);
    }
}

Referenced by MiDecrementReferenceCount(), and MmZeroPageThread().

◆ MiInsertVad()

VOID NTAPI MiInsertVad	(	_Inout_ PMMVAD	Vad,
		_Inout_ PMM_AVL_TABLE	VadRoot
	)

Referenced by MiMapLockedPagesInUserSpace(), and NtFreeVirtualMemory().

◆ MiInsertVadEx()

NTSTATUS NTAPI MiInsertVadEx	(	_Inout_ PMMVAD	Vad,
		_In_ ULONG_PTR *	BaseAddress,
		_In_ SIZE_T	ViewSize,
		_In_ ULONG_PTR	HighestAddress,
		_In_ ULONG_PTR	Alignment,
		_In_ ULONG	AllocationType
	)

Definition at line 282 of file vadnode.c.

{
    ULONG_PTR StartingAddress, EndingAddress;
    PEPROCESS CurrentProcess;
    PETHREAD CurrentThread;
    TABLE_SEARCH_RESULT Result;
    PMMADDRESS_NODE Parent;
 
    /* Align the view size to pages */
    ViewSize = ALIGN_UP_BY(ViewSize, PAGE_SIZE);
 
    /* Get the current process */
    CurrentProcess = PsGetCurrentProcess();
 
    /* Acquire the address creation lock and make sure the process is alive */
    KeAcquireGuardedMutex(&CurrentProcess->AddressCreationLock);
    if (CurrentProcess->VmDeleted)
    {
        KeReleaseGuardedMutex(&CurrentProcess->AddressCreationLock);
        DPRINT1("The process is dying\n");
        return STATUS_PROCESS_IS_TERMINATING;
    }
 
    /* Did the caller specify an address? */
    if (*BaseAddress == 0)
    {
        /* Make sure HighestAddress is not too large */
        HighestAddress = min(HighestAddress, (ULONG_PTR)MM_HIGHEST_VAD_ADDRESS);
 
        /* Which way should we search? */
        if ((AllocationType & MEM_TOP_DOWN) || CurrentProcess->VmTopDown)
        {
            /* Find an address top-down */
            Result = MiFindEmptyAddressRangeDownTree(ViewSize,
                                                     HighestAddress,
                                                     Alignment,
                                                     &CurrentProcess->VadRoot,
                                                     &StartingAddress,
                                                     &Parent);
        }
        else
        {
            /* Find an address bottom-up */
            Result = MiFindEmptyAddressRangeInTree(ViewSize,
                                                   Alignment,
                                                   &CurrentProcess->VadRoot,
                                                   &Parent,
                                                   &StartingAddress);
        }
 
        /* Get the ending address, which is the last piece we need for the VAD */
        EndingAddress = StartingAddress + ViewSize - 1;
 
        /* Check if we found a suitable location */
        if ((Result == TableFoundNode) || (EndingAddress > HighestAddress))
        {
            DPRINT1("Not enough free space to insert this VAD node!\n");
            KeReleaseGuardedMutex(&CurrentProcess->AddressCreationLock);
            return STATUS_NO_MEMORY;
        }
 
        ASSERT(StartingAddress != 0);
        ASSERT(StartingAddress < (ULONG_PTR)HighestAddress);
        ASSERT(EndingAddress > StartingAddress);
    }
    else
    {
        /* Calculate the starting and ending address */
        StartingAddress = ALIGN_DOWN_BY(*BaseAddress, Alignment);
        EndingAddress = StartingAddress + ViewSize - 1;
 
        /* Make sure it doesn't conflict with an existing allocation */
        Result = MiCheckForConflictingNode(StartingAddress >> PAGE_SHIFT,
                                           EndingAddress >> PAGE_SHIFT,
                                           &CurrentProcess->VadRoot,
                                           &Parent);
        if (Result == TableFoundNode)
        {
            DPRINT("Given address conflicts with existing node\n");
            KeReleaseGuardedMutex(&CurrentProcess->AddressCreationLock);
            return STATUS_CONFLICTING_ADDRESSES;
        }
    }
 
    /* Now set the VAD address */
    Vad->StartingVpn = StartingAddress >> PAGE_SHIFT;
    Vad->EndingVpn = EndingAddress >> PAGE_SHIFT;
 
    /* Check if we already need to charge for the pages */
    if ((Vad->u.VadFlags.PrivateMemory && Vad->u.VadFlags.MemCommit) ||
        (!Vad->u.VadFlags.PrivateMemory &&
         (Vad->u.VadFlags.Protection & PAGE_WRITECOPY)))
    {
        /* Set the commit charge */
        Vad->u.VadFlags.CommitCharge = ViewSize / PAGE_SIZE;
    }
 
    /* Check if the VAD is to be secured */
    if (Vad->u2.VadFlags2.OneSecured)
    {
        /* This *must* be a long VAD! */
        ASSERT(Vad->u2.VadFlags2.LongVad);
 
        /* Yeah this is retarded, I didn't invent it! */
        ((PMMVAD_LONG)Vad)->u3.Secured.StartVpn = StartingAddress;
        ((PMMVAD_LONG)Vad)->u3.Secured.EndVpn = EndingAddress;
    }
 
    /* Lock the working set */
    CurrentThread = PsGetCurrentThread();
    MiLockProcessWorkingSetUnsafe(CurrentProcess, CurrentThread);
 
    /* Insert the VAD */
    CurrentProcess->VadRoot.NodeHint = Vad;
    MiInsertNode(&CurrentProcess->VadRoot, (PVOID)Vad, Parent, Result);
 
    /* Release the working set */
    MiUnlockProcessWorkingSetUnsafe(CurrentProcess, CurrentThread);
 
    /* Update the process' virtual size, and peak virtual size */
    CurrentProcess->VirtualSize += ViewSize;
    if (CurrentProcess->VirtualSize > CurrentProcess->PeakVirtualSize)
    {
        CurrentProcess->PeakVirtualSize = CurrentProcess->VirtualSize;
    }
 
    /* Unlock the address space */
    KeReleaseGuardedMutex(&CurrentProcess->AddressCreationLock);
 
    *BaseAddress = StartingAddress;
    return STATUS_SUCCESS;
}

Referenced by MiCreatePebOrTeb(), MiMapViewOfDataSection(), and NtAllocateVirtualMemory().

◆ MiIsMemoryTypeFree()

FORCEINLINE BOOLEAN MiIsMemoryTypeFree ( TYPE_OF_MEMORY MemoryType )

Definition at line 682 of file miarm.h.

{
    return ((MemoryType == LoaderFree) ||
            (MemoryType == LoaderLoadedProgram) ||
            (MemoryType == LoaderFirmwareTemporary) ||
            (MemoryType == LoaderOsloaderStack));
}

Referenced by MiAddDescriptorToDatabase(), and MiScanMemoryDescriptors().

◆ MiIsMemoryTypeInvisible()

FORCEINLINE BOOLEAN MiIsMemoryTypeInvisible ( TYPE_OF_MEMORY MemoryType )

Definition at line 692 of file miarm.h.

{
    return ((MemoryType == LoaderFirmwarePermanent) ||
            (MemoryType == LoaderSpecialMemory) ||
            (MemoryType == LoaderHALCachedMemory) ||
            (MemoryType == LoaderBBTMemory));
}

Referenced by KdpGetMemorySizeInMBs(), MiAddDescriptorToDatabase(), MiBuildPfnDatabase(), and MiScanMemoryDescriptors().

◆ MiIsPfnInUse()

BOOLEAN NTAPI MiIsPfnInUse ( IN PMMPFN Pfn1 )

Definition at line 174 of file freelist.c.

{
    /* Standby list or higher, unlinked, and with references */
    return !MiIsPfnFree(Pfn1);
}

Referenced by MiAllocatePagesForMdl(), MiFindContiguousPages(), MmGetRmapListHeadPage(), MmIsPageInUse(), and MmSetRmapListHeadPage().

◆ MiIsRosSectionObject()

FORCEINLINE BOOLEAN MiIsRosSectionObject ( IN PSECTION Section )

Definition at line 1111 of file miarm.h.

{
    return Section->u.Flags.filler;
}

Referenced by MmExtendSection(), MmGetFileObjectForSection(), MmGetImageInformation(), MmMapViewInSessionSpace(), MmMapViewInSystemSpaceEx(), MmMapViewOfSection(), MmpDeleteSection(), and NtQuerySection().

◆ MiIsUserPde()

FORCEINLINE BOOLEAN MiIsUserPde ( PVOID Address )

Definition at line 731 of file miarm.h.

{
    return ((Address >= (PVOID)MiAddressToPde(NULL)) &&
            (Address <= (PVOID)MiHighestUserPde));
}

Referenced by MiDeletePde(), MiDetermineUserGlobalPteMask(), and MmArmAccessFault().

◆ MiIsUserPte()

FORCEINLINE BOOLEAN MiIsUserPte ( PVOID Address )

Definition at line 739 of file miarm.h.

{
    return (Address >= (PVOID)PTE_BASE) && (Address <= (PVOID)MiHighestUserPte);
}

Referenced by MiDetermineUserGlobalPteMask(), and MmArmAccessFault().

◆ MiLocateAddress()

PMMVAD NTAPI MiLocateAddress ( IN PVOID VirtualAddress )

Definition at line 116 of file vadnode.c.

{
    PMMVAD FoundVad;
    ULONG_PTR Vpn;
    PMM_AVL_TABLE Table = &PsGetCurrentProcess()->VadRoot;
    TABLE_SEARCH_RESULT SearchResult;
 
    ASSERT_LOCKED_FOR_READ(Table);
 
    /* Start with the the hint */
    FoundVad = (PMMVAD)Table->NodeHint;
    if (!FoundVad) return NULL;
 
    /* Check if this VPN is in the hint, if so, use it */
    Vpn = (ULONG_PTR)VirtualAddress >> PAGE_SHIFT;
    if ((Vpn >= FoundVad->StartingVpn) && (Vpn <= FoundVad->EndingVpn)) return FoundVad;
 
    /* VAD hint didn't work, go look for it */
    SearchResult = RtlpFindAvlTableNodeOrParent(Table,
                                                (PVOID)Vpn,
                                                (PMMADDRESS_NODE*)&FoundVad);
    if (SearchResult != TableFoundNode) return NULL;
 
    /* We found it, update the hint */
    ASSERT(FoundVad != NULL);
    ASSERT((Vpn >= FoundVad->StartingVpn) && (Vpn <= FoundVad->EndingVpn));
 
    /* We allow this (atomic) update without exclusive lock, because it's a hint only */
    Table->NodeHint = FoundVad;
    return FoundVad;
}

Referenced by MiCheckVadsForLockOperation(), MiCheckVirtualAddress(), MiUnmapLockedPagesInUserSpace(), MiUnmapViewOfSection(), MmDeleteTeb(), MmGetFileNameForAddress(), NtAreMappedFilesTheSame(), and NtFreeVirtualMemory().

◆ MiLocateSubsection()

PSUBSECTION NTAPI MiLocateSubsection	(	IN PMMVAD	Vad,
		IN ULONG_PTR	Vpn
	)

Definition at line 556 of file section.c.

{
    PSUBSECTION Subsection;
    PCONTROL_AREA ControlArea;
    ULONG_PTR PteOffset;
 
    /* Get the control area */
    ControlArea = Vad->ControlArea;
    ASSERT(ControlArea->u.Flags.Rom == 0);
    ASSERT(ControlArea->u.Flags.Image == 0);
    ASSERT(ControlArea->u.Flags.GlobalOnlyPerSession == 0);
 
    /* Get the subsection */
    Subsection = (PSUBSECTION)(ControlArea + 1);
 
    /* We only support single-subsection segments */
    ASSERT(Subsection->SubsectionBase != NULL);
    ASSERT(Vad->FirstPrototypePte >= Subsection->SubsectionBase);
    ASSERT(Vad->FirstPrototypePte < &Subsection->SubsectionBase[Subsection->PtesInSubsection]);
 
    /* Compute the PTE offset */
    PteOffset = Vpn - Vad->StartingVpn;
    PteOffset += Vad->FirstPrototypePte - Subsection->SubsectionBase;
 
    /* Again, we only support single-subsection segments */
    ASSERT(PteOffset < 0xF0000000);
    ASSERT(PteOffset < Subsection->PtesInSubsection);
 
    /* Return the subsection */
    return Subsection;
}

Referenced by MiDeleteVirtualAddresses().

◆ MiLockProcessWorkingSet()

FORCEINLINE VOID MiLockProcessWorkingSet	(	IN PEPROCESS	Process,
		IN PETHREAD	Thread
	)

Definition at line 1137 of file miarm.h.

{
    /* Shouldn't already be owning the process working set */
    ASSERT(Thread->OwnsProcessWorkingSetShared == FALSE);
    ASSERT(Thread->OwnsProcessWorkingSetExclusive == FALSE);
 
    /* Block APCs, make sure that still nothing is already held */
    KeEnterGuardedRegion();
    ASSERT(!MM_ANY_WS_LOCK_HELD(Thread));
 
    /* Lock the working set */
    ExAcquirePushLockExclusive(&Process->Vm.WorkingSetMutex);
 
    /* Now claim that we own the lock */
    ASSERT(!MI_IS_WS_UNSAFE(Process));
    ASSERT(Thread->OwnsProcessWorkingSetExclusive == FALSE);
    Thread->OwnsProcessWorkingSetExclusive = TRUE;
}

Referenced by MiLockProcessWorkingSetForFault(), MiLockVirtualMemory(), MiUnlockVirtualMemory(), MmArmAccessFault(), MmFreeMemoryArea(), MmInitializeProcessAddressSpace(), MmProbeAndLockPages(), and MmTrimUserMemory().

◆ MiLockProcessWorkingSetForFault()

FORCEINLINE VOID MiLockProcessWorkingSetForFault	(	IN PEPROCESS	Process,
		IN PETHREAD	Thread,
		IN BOOLEAN	Safe,
		IN BOOLEAN	Shared
	)

Definition at line 1514 of file miarm.h.

{
    /* Check if this was a safe lock or not */
    if (Safe)
    {
        if (Shared)
        {
            /* Reacquire safely & shared */
            MiLockProcessWorkingSetShared(Process, Thread);
        }
        else
        {
            /* Reacquire safely */
            MiLockProcessWorkingSet(Process, Thread);
        }
    }
    else
    {
        /* Unsafe lock cannot be shared */
        ASSERT(Shared == FALSE);
        /* Reacquire unsafely */
        MiLockProcessWorkingSetUnsafe(Process, Thread);
    }
}

Referenced by MiGetPageProtection(), and MiMakeSystemAddressValid().

◆ MiLockProcessWorkingSetShared()

FORCEINLINE VOID MiLockProcessWorkingSetShared	(	IN PEPROCESS	Process,
		IN PETHREAD	Thread
	)

Definition at line 1159 of file miarm.h.

{
    /* Shouldn't already be owning the process working set */
    ASSERT(Thread->OwnsProcessWorkingSetShared == FALSE);
    ASSERT(Thread->OwnsProcessWorkingSetExclusive == FALSE);
 
    /* Block APCs, make sure that still nothing is already held */
    KeEnterGuardedRegion();
    ASSERT(!MM_ANY_WS_LOCK_HELD(Thread));
 
    /* Lock the working set */
    ExAcquirePushLockShared(&Process->Vm.WorkingSetMutex);
 
    /* Now claim that we own the lock */
    ASSERT(!MI_IS_WS_UNSAFE(Process));
    ASSERT(Thread->OwnsProcessWorkingSetShared == FALSE);
    ASSERT(Thread->OwnsProcessWorkingSetExclusive == FALSE);
    Thread->OwnsProcessWorkingSetShared = TRUE;
}

Referenced by MiLockProcessWorkingSetForFault(), MiQueryMemoryBasicInformation(), MmAccessFault(), MmGetPageFileMapping(), MmGetPageProtect(), MmGetPfnForProcess(), MmIsDisabledPage(), MmIsPagePresent(), and MmIsPageSwapEntry().

◆ MiLockProcessWorkingSetUnsafe()

FORCEINLINE VOID MiLockProcessWorkingSetUnsafe	(	IN PEPROCESS	Process,
		IN PETHREAD	Thread
	)

Definition at line 1182 of file miarm.h.

{
    /* Shouldn't already be owning the process working set */
    ASSERT(Thread->OwnsProcessWorkingSetExclusive == FALSE);
 
    /* APCs must be blocked, make sure that still nothing is already held */
    ASSERT(KeAreAllApcsDisabled() == TRUE);
    ASSERT(!MM_ANY_WS_LOCK_HELD(Thread));
 
    /* Lock the working set */
    ExAcquirePushLockExclusive(&Process->Vm.WorkingSetMutex);
 
    /* Now claim that we own the lock */
    ASSERT(!MI_IS_WS_UNSAFE(Process));
    Process->Vm.Flags.AcquiredUnsafe = 1;
    ASSERT(Thread->OwnsProcessWorkingSetExclusive == FALSE);
    Thread->OwnsProcessWorkingSetExclusive = TRUE;
}

Referenced by _Success_(), MiDecommitPages(), MiInsertVadEx(), MiLockProcessWorkingSetForFault(), MiMapLockedPagesInUserSpace(), MiProtectVirtualMemory(), MiUnmapLockedPagesInUserSpace(), MiUnmapViewOfSection(), MmCleanProcessAddressSpace(), MmCreatePageFileMapping(), MmCreateVirtualMappingUnsafeEx(), MmDeletePageFileMapping(), MmDeleteTeb(), MmInsertMemoryArea(), MmSetDirtyBit(), MmSetPageProtect(), NtAllocateVirtualMemory(), and NtFreeVirtualMemory().

◆ MiLockWorkingSet()

FORCEINLINE VOID MiLockWorkingSet	(	IN PETHREAD	Thread,
		IN PMMSUPPORT	WorkingSet
	)

Definition at line 1278 of file miarm.h.

{
    /* Block APCs */
    KeEnterGuardedRegion();
 
    /* Working set should be in global memory */
    ASSERT(MI_IS_SESSION_ADDRESS((PVOID)WorkingSet) == FALSE);
 
    /* Thread shouldn't already be owning something */
    ASSERT(!MM_ANY_WS_LOCK_HELD(Thread));
 
    /* Lock this working set */
    ExAcquirePushLockExclusive(&WorkingSet->WorkingSetMutex);
 
    /* Which working set is this? */
    if (WorkingSet == &MmSystemCacheWs)
    {
        /* Own the system working set */
        ASSERT((Thread->OwnsSystemWorkingSetExclusive == FALSE) &&
               (Thread->OwnsSystemWorkingSetShared == FALSE));
        Thread->OwnsSystemWorkingSetExclusive = TRUE;
    }
    else if (WorkingSet->Flags.SessionSpace)
    {
        /* Own the session working set */
        ASSERT((Thread->OwnsSessionWorkingSetExclusive == FALSE) &&
               (Thread->OwnsSessionWorkingSetShared == FALSE));
        Thread->OwnsSessionWorkingSetExclusive = TRUE;
    }
    else
    {
        /* Own the process working set */
        ASSERT((Thread->OwnsProcessWorkingSetExclusive == FALSE) &&
               (Thread->OwnsProcessWorkingSetShared == FALSE));
        Thread->OwnsProcessWorkingSetExclusive = TRUE;
    }
}

Referenced by MiDeleteSystemPageableVm(), MiSetPagingOfDriver(), MmAdjustWorkingSetSize(), MmArmAccessFault(), MmFreeMemoryArea(), and MmInsertMemoryArea().

◆ MiLockWorkingSetShared()

FORCEINLINE VOID MiLockWorkingSetShared	(	_In_ PETHREAD	Thread,
		_In_ PMMSUPPORT	WorkingSet
	)

Definition at line 1319 of file miarm.h.

{
    /* Block APCs */
    KeEnterGuardedRegion();
 
    /* Working set should be in global memory */
    ASSERT(MI_IS_SESSION_ADDRESS((PVOID)WorkingSet) == FALSE);
 
    /* Thread shouldn't already be owning something */
    ASSERT(!MM_ANY_WS_LOCK_HELD(Thread));
 
    /* Lock this working set */
    ExAcquirePushLockShared(&WorkingSet->WorkingSetMutex);
 
    /* Which working set is this? */
    if (WorkingSet == &MmSystemCacheWs)
    {
        /* Own the system working set */
        ASSERT((Thread->OwnsSystemWorkingSetExclusive == FALSE) &&
               (Thread->OwnsSystemWorkingSetShared == FALSE));
        Thread->OwnsSystemWorkingSetShared = TRUE;
    }
    else if (WorkingSet->Flags.SessionSpace)
    {
        /* Own the session working set */
        ASSERT((Thread->OwnsSessionWorkingSetExclusive == FALSE) &&
               (Thread->OwnsSessionWorkingSetShared == FALSE));
        Thread->OwnsSessionWorkingSetShared = TRUE;
    }
    else
    {
        /* Own the process working set */
        ASSERT((Thread->OwnsProcessWorkingSetExclusive == FALSE) &&
               (Thread->OwnsProcessWorkingSetShared == FALSE));
        Thread->OwnsProcessWorkingSetShared = TRUE;
    }
}

Referenced by MmAccessFault(), and MmWorkingSetManager().

◆ MiLookupDataTableEntry()

PLDR_DATA_TABLE_ENTRY NTAPI MiLookupDataTableEntry ( IN PVOID Address )

Definition at line 3528 of file sysldr.c.

{
    PLDR_DATA_TABLE_ENTRY LdrEntry, FoundEntry = NULL;
    PLIST_ENTRY NextEntry;
    PAGED_CODE();
 
    /* Loop entries */
    NextEntry = PsLoadedModuleList.Flink;
    do
    {
        /* Get the loader entry */
        LdrEntry =  CONTAINING_RECORD(NextEntry,
                                      LDR_DATA_TABLE_ENTRY,
                                      InLoadOrderLinks);
 
        /* Check if the address matches */
        if ((Address >= LdrEntry->DllBase) &&
            (Address < (PVOID)((ULONG_PTR)LdrEntry->DllBase +
                               LdrEntry->SizeOfImage)))
        {
            /* Found a match */
            FoundEntry = LdrEntry;
            break;
        }
 
        /* Move on */
        NextEntry = NextEntry->Flink;
    } while(NextEntry != &PsLoadedModuleList);
 
    /* Return the entry */
    return FoundEntry;
}

Referenced by ExAllocatePool(), MmAddVerifierThunks(), and MmPageEntireDriver().

◆ MiMakePdeExistAndMakeValid()

VOID NTAPI MiMakePdeExistAndMakeValid	(	IN PMMPDE	PointerPde,
		IN PEPROCESS	TargetProcess,
		IN KIRQL	OldIrql
	)

Definition at line 2481 of file virtual.c.

{
   PMMPTE PointerPte;
#if _MI_PAGING_LEVELS >= 3
   PMMPPE PointerPpe = MiPdeToPpe(PointerPde);
#if _MI_PAGING_LEVELS == 4
   PMMPXE PointerPxe = MiPdeToPxe(PointerPde);
#endif
#endif
 
   //
   // Sanity checks. The latter is because we only use this function with the
   // PFN lock not held, so it may go away in the future.
   //
   ASSERT(KeAreAllApcsDisabled() == TRUE);
   ASSERT(OldIrql == MM_NOIRQL);
 
   //
   // If everything is already valid, there is nothing to do.
   //
   if (
#if _MI_PAGING_LEVELS == 4
       (PointerPxe->u.Hard.Valid) &&
#endif
#if _MI_PAGING_LEVELS >= 3
       (PointerPpe->u.Hard.Valid) &&
#endif
       (PointerPde->u.Hard.Valid))
   {
       return;
   }
 
   //
   // At least something is invalid, so begin by getting the PTE for the PDE itself
   // and then lookup each additional level. We must do it in this precise order
   // because the pagfault.c code (as well as in Windows) depends that the next
   // level up (higher) must be valid when faulting a lower level
   //
   PointerPte = MiPteToAddress(PointerPde);
   do
   {
       //
       // Make sure APCs continued to be disabled
       //
       ASSERT(KeAreAllApcsDisabled() == TRUE);
 
#if _MI_PAGING_LEVELS == 4
       //
       // First, make the PXE valid if needed
       //
       if (!PointerPxe->u.Hard.Valid)
       {
           MiMakeSystemAddressValid(PointerPpe, TargetProcess);
           ASSERT(PointerPxe->u.Hard.Valid == 1);
       }
#endif
 
#if _MI_PAGING_LEVELS >= 3
       //
       // Next, the PPE
       //
       if (!PointerPpe->u.Hard.Valid)
       {
           MiMakeSystemAddressValid(PointerPde, TargetProcess);
           ASSERT(PointerPpe->u.Hard.Valid == 1);
       }
#endif
 
       //
       // And finally, make the PDE itself valid.
       //
       MiMakeSystemAddressValid(PointerPte, TargetProcess);
 
       /* Do not increment Page table refcount here for the PDE, this must be managed by caller */
 
       //
       // This should've worked the first time so the loop is really just for
       // show -- ASSERT that we're actually NOT going to be looping.
       //
       ASSERT(PointerPde->u.Hard.Valid == 1);
   } while (
#if _MI_PAGING_LEVELS == 4
        !PointerPxe->u.Hard.Valid ||
#endif
#if _MI_PAGING_LEVELS >= 3
        !PointerPpe->u.Hard.Valid ||
#endif
        !PointerPde->u.Hard.Valid);
}

Referenced by _Success_(), MiDecommitPages(), MiMapLockedPagesInUserSpace(), MiProtectVirtualMemory(), MmCreatePageFileMapping(), MmCreateVirtualMappingUnsafeEx(), MmDeletePageFileMapping(), MmGetPageFileMapping(), MmGetPageProtect(), MmGetPfnForProcess(), MmIsDisabledPage(), MmIsPagePresent(), MmIsPageSwapEntry(), MmSetDirtyBit(), MmSetPageProtect(), and NtAllocateVirtualMemory().

◆ MiMakeProtectionMask()

ULONG NTAPI MiMakeProtectionMask ( IN ULONG Protect )

Definition at line 140 of file section.c.

{
    ULONG Mask1, Mask2, ProtectMask;
 
    /* PAGE_EXECUTE_WRITECOMBINE is theoretically the maximum */
    if (Protect >= (PAGE_WRITECOMBINE * 2)) return MM_INVALID_PROTECTION;
 
    /*
     * Windows API protection mask can be understood as two bitfields, differing
     * by whether or not execute rights are being requested
     */
    Mask1 = Protect & 0xF;
    Mask2 = (Protect >> 4) & 0xF;
 
    /* Check which field is there */
    if (!Mask1)
    {
        /* Mask2 must be there, use it to determine the PTE protection */
        if (!Mask2) return MM_INVALID_PROTECTION;
        ProtectMask = MmUserProtectionToMask2[Mask2];
    }
    else
    {
        /* Mask2 should not be there, use Mask1 to determine the PTE mask */
        if (Mask2) return MM_INVALID_PROTECTION;
        ProtectMask = MmUserProtectionToMask1[Mask1];
    }
 
    /* Make sure the final mask is a valid one */
    if (ProtectMask == MM_INVALID_PROTECTION) return MM_INVALID_PROTECTION;
 
    /* Check for PAGE_GUARD option */
    if (Protect & PAGE_GUARD)
    {
        /* It's not valid on no-access, nocache, or writecombine pages */
        if ((ProtectMask == MM_NOACCESS) ||
            (Protect & (PAGE_NOCACHE | PAGE_WRITECOMBINE)))
        {
            /* Fail such requests */
            return MM_INVALID_PROTECTION;
        }
 
        /* This actually turns on guard page in this scenario! */
        ProtectMask |= MM_GUARDPAGE;
    }
 
    /* Check for nocache option */
    if (Protect & PAGE_NOCACHE)
    {
        /* The earlier check should've eliminated this possibility */
        ASSERT((Protect & PAGE_GUARD) == 0);
 
        /* Check for no-access page or write combine page */
        if ((ProtectMask == MM_NOACCESS) || (Protect & PAGE_WRITECOMBINE))
        {
            /* Such a request is invalid */
            return MM_INVALID_PROTECTION;
        }
 
        /* Add the PTE flag */
        ProtectMask |= MM_NOCACHE;
    }
 
    /* Check for write combine option */
    if (Protect & PAGE_WRITECOMBINE)
    {
        /* The two earlier scenarios should've caught this */
        ASSERT((Protect & (PAGE_GUARD | PAGE_NOACCESS)) == 0);
 
        /* Don't allow on no-access pages */
        if (ProtectMask == MM_NOACCESS) return MM_INVALID_PROTECTION;
 
        /* This actually turns on write-combine in this scenario! */
        ProtectMask |= MM_NOACCESS;
    }
 
    /* Return the final MM PTE protection mask */
    return ProtectMask;
}

Referenced by MiProtectVirtualMemory(), MmCreateVirtualMappingUnsafeEx(), MmSetPageProtect(), and NtAllocateVirtualMemory().

◆ MiMakeSystemAddressValid()

ULONG NTAPI MiMakeSystemAddressValid	(	IN PVOID	PageTableVirtualAddress,
		IN PEPROCESS	CurrentProcess
	)

Definition at line 183 of file virtual.c.

{
    NTSTATUS Status;
    BOOLEAN WsShared = FALSE, WsSafe = FALSE, LockChange = FALSE;
    PETHREAD CurrentThread = PsGetCurrentThread();
 
    /* Must be a non-pool page table, since those are double-mapped already */
    ASSERT(PageTableVirtualAddress > MM_HIGHEST_USER_ADDRESS);
    ASSERT((PageTableVirtualAddress < MmPagedPoolStart) ||
           (PageTableVirtualAddress > MmPagedPoolEnd));
 
    /* Working set lock or PFN lock should be held */
    ASSERT(KeAreAllApcsDisabled() == TRUE);
 
    /* Check if the page table is valid */
    while (!MmIsAddressValid(PageTableVirtualAddress))
    {
        /* Release the working set lock */
        MiUnlockProcessWorkingSetForFault(CurrentProcess,
                                          CurrentThread,
                                          &WsSafe,
                                          &WsShared);
 
        /* Fault it in */
        Status = MmAccessFault(FALSE, PageTableVirtualAddress, KernelMode, NULL);
        if (!NT_SUCCESS(Status))
        {
            /* This should not fail */
            KeBugCheckEx(KERNEL_DATA_INPAGE_ERROR,
                         1,
                         Status,
                         (ULONG_PTR)CurrentProcess,
                         (ULONG_PTR)PageTableVirtualAddress);
        }
 
        /* Lock the working set again */
        MiLockProcessWorkingSetForFault(CurrentProcess,
                                        CurrentThread,
                                        WsSafe,
                                        WsShared);
 
        /* This flag will be useful later when we do better locking */
        LockChange = TRUE;
    }
 
    /* Let caller know what the lock state is */
    return LockChange;
}

Referenced by MiCalculatePageCommitment(), MiDeleteVirtualAddresses(), MiIsEntireRangeCommitted(), MiIsPageTablePresent(), MiMakePdeExistAndMakeValid(), and MiQueryAddressState().

◆ MiMakeSystemAddressValidPfn()

ULONG NTAPI MiMakeSystemAddressValidPfn	(	IN PVOID	VirtualAddress,
		IN KIRQL	OldIrql
	)

Definition at line 235 of file virtual.c.

{
    NTSTATUS Status;
    BOOLEAN LockChange = FALSE;
 
    /* Must be e kernel address */
    ASSERT(VirtualAddress > MM_HIGHEST_USER_ADDRESS);
 
    /* Check if the page is valid */
    while (!MmIsAddressValid(VirtualAddress))
    {
        /* Release the PFN database */
        MiReleasePfnLock(OldIrql);
 
        /* Fault it in */
        Status = MmAccessFault(FALSE, VirtualAddress, KernelMode, NULL);
        if (!NT_SUCCESS(Status))
        {
            /* This should not fail */
            KeBugCheckEx(KERNEL_DATA_INPAGE_ERROR,
                         3,
                         Status,
                         0,
                         (ULONG_PTR)VirtualAddress);
        }
 
        /* This flag will be useful later when we do better locking */
        LockChange = TRUE;
 
        /* Lock the PFN database */
        OldIrql = MiAcquirePfnLock();
    }
 
    /* Let caller know what the lock state is */
    return LockChange;
}

Referenced by MiGetPageProtection(), and MiSegmentDelete().

◆ MiMapPfnDatabase()

VOID NTAPI MiMapPfnDatabase ( IN PLOADER_PARAMETER_BLOCK LoaderBlock )

Definition at line 663 of file mminit.c.

{
    PFN_NUMBER FreePage, FreePageCount, PagesLeft, BasePage, PageCount;
    PLIST_ENTRY NextEntry;
    PMEMORY_ALLOCATION_DESCRIPTOR MdBlock;
    PMMPTE PointerPte, LastPte;
    MMPTE TempPte = ValidKernelPte;
 
    /* Get current page data, since we won't be using MxGetNextPage as it would corrupt our state */
    FreePage = MxFreeDescriptor->BasePage;
    FreePageCount = MxFreeDescriptor->PageCount;
    PagesLeft = 0;
 
    /* Loop the memory descriptors */
    NextEntry = LoaderBlock->MemoryDescriptorListHead.Flink;
    while (NextEntry != &LoaderBlock->MemoryDescriptorListHead)
    {
        /* Get the descriptor */
        MdBlock = CONTAINING_RECORD(NextEntry,
                                    MEMORY_ALLOCATION_DESCRIPTOR,
                                    ListEntry);
        if ((MdBlock->MemoryType == LoaderFirmwarePermanent) ||
            (MdBlock->MemoryType == LoaderBBTMemory) ||
            (MdBlock->MemoryType == LoaderSpecialMemory))
        {
            /* These pages are not part of the PFN database */
            NextEntry = MdBlock->ListEntry.Flink;
            continue;
        }
 
        /* Next, check if this is our special free descriptor we've found */
        if (MdBlock == MxFreeDescriptor)
        {
            /* Use the real numbers instead */
            BasePage = MxOldFreeDescriptor.BasePage;
            PageCount = MxOldFreeDescriptor.PageCount;
        }
        else
        {
            /* Use the descriptor's numbers */
            BasePage = MdBlock->BasePage;
            PageCount = MdBlock->PageCount;
        }
 
        /* Get the PTEs for this range */
        PointerPte = MiAddressToPte(&MmPfnDatabase[BasePage]);
        LastPte = MiAddressToPte(((ULONG_PTR)&MmPfnDatabase[BasePage + PageCount]) - 1);
        DPRINT("MD Type: %lx Base: %lx Count: %lx\n", MdBlock->MemoryType, BasePage, PageCount);
 
        /* Loop them */
        while (PointerPte <= LastPte)
        {
            /* We'll only touch PTEs that aren't already valid */
            if (PointerPte->u.Hard.Valid == 0)
            {
                /* Use the next free page */
                TempPte.u.Hard.PageFrameNumber = FreePage;
                ASSERT(FreePageCount != 0);
 
                /* Consume free pages */
                FreePage++;
                FreePageCount--;
                if (!FreePageCount)
                {
                    /* Out of memory */
                    KeBugCheckEx(INSTALL_MORE_MEMORY,
                                 MmNumberOfPhysicalPages,
                                 FreePageCount,
                                 MxOldFreeDescriptor.PageCount,
                                 1);
                }
 
                /* Write out this PTE */
                PagesLeft++;
                MI_WRITE_VALID_PTE(PointerPte, TempPte);
 
                /* Zero this page */
                RtlZeroMemory(MiPteToAddress(PointerPte), PAGE_SIZE);
            }
 
            /* Next! */
            PointerPte++;
        }
 
        /* Do the next address range */
        NextEntry = MdBlock->ListEntry.Flink;
    }
 
    /* Now update the free descriptors to consume the pages we used up during the PFN allocation loop */
    MxFreeDescriptor->BasePage = FreePage;
    MxFreeDescriptor->PageCount = FreePageCount;
}

Referenced by MiInitMachineDependent().

◆ MiPagesInLoaderBlock()

PFN_NUMBER NTAPI MiPagesInLoaderBlock	(	IN PLOADER_PARAMETER_BLOCK	LoaderBlock,
		IN PBOOLEAN	IncludeType
	)

◆ MiQueryMemorySectionName()

NTSTATUS NTAPI MiQueryMemorySectionName	(	IN HANDLE	ProcessHandle,
		IN PVOID	BaseAddress,
		OUT PVOID	MemoryInformation,
		IN SIZE_T	MemoryInformationLength,
		OUT PSIZE_T	ReturnLength
	)

Definition at line 1752 of file section.c.

{
    PEPROCESS Process;
    NTSTATUS Status;
    UNICODE_STRING ModuleFileName;
    PMEMORY_SECTION_NAME SectionName = NULL;
    KPROCESSOR_MODE PreviousMode = ExGetPreviousMode();
 
    Status = ObReferenceObjectByHandle(ProcessHandle,
                                       PROCESS_QUERY_INFORMATION,
                                       NULL,
                                       PreviousMode,
                                       (PVOID*)(&Process),
                                       NULL);
 
    if (!NT_SUCCESS(Status))
    {
        DPRINT("MiQueryMemorySectionName: ObReferenceObjectByHandle returned %x\n",Status);
        return Status;
    }
 
    Status = MmGetFileNameForAddress(BaseAddress, &ModuleFileName);
 
    if (NT_SUCCESS(Status))
    {
        SectionName = MemoryInformation;
        if (PreviousMode != KernelMode)
        {
            _SEH2_TRY
            {
                RtlInitEmptyUnicodeString(&SectionName->SectionFileName,
                                          (PWSTR)(SectionName + 1),
                                          MemoryInformationLength - sizeof(MEMORY_SECTION_NAME));
                RtlCopyUnicodeString(&SectionName->SectionFileName, &ModuleFileName);
 
                if (ReturnLength) *ReturnLength = ModuleFileName.Length + sizeof(MEMORY_SECTION_NAME);
 
            }
            _SEH2_EXCEPT(EXCEPTION_EXECUTE_HANDLER)
            {
                Status = _SEH2_GetExceptionCode();
            }
            _SEH2_END;
        }
        else
        {
            RtlInitEmptyUnicodeString(&SectionName->SectionFileName,
                                      (PWSTR)(SectionName + 1),
                                      MemoryInformationLength - sizeof(MEMORY_SECTION_NAME));
            RtlCopyUnicodeString(&SectionName->SectionFileName, &ModuleFileName);
 
            if (ReturnLength) *ReturnLength = ModuleFileName.Length + sizeof(MEMORY_SECTION_NAME);
 
        }
 
        RtlFreeUnicodeString(&ModuleFileName);
    }
    ObDereferenceObject(Process);
    return Status;
}

Referenced by NtQueryVirtualMemory().

◆ MiReferenceProbedPageAndBumpLockCount()

FORCEINLINE VOID MiReferenceProbedPageAndBumpLockCount ( IN PMMPFN Pfn1 )

Definition at line 1700 of file miarm.h.

{
    USHORT RefCount, OldRefCount;
 
    /* Sanity check */
    ASSERT(Pfn1->u3.e2.ReferenceCount != 0);
 
    /* Does ARM3 own the page? */
    if (MI_IS_ROS_PFN(Pfn1))
    {
        /* ReactOS Mm doesn't track share count */
        ASSERT(Pfn1->u3.e1.PageLocation == ActiveAndValid);
    }
    else
    {
        /* On ARM3 pages, we should see a valid share count */
        ASSERT((Pfn1->u2.ShareCount != 0) && (Pfn1->u3.e1.PageLocation == ActiveAndValid));
 
        /* Is it a prototype PTE? */
        if ((Pfn1->u3.e1.PrototypePte == 1) &&
            (Pfn1->OriginalPte.u.Soft.Prototype == 1))
        {
            /* FIXME: We should charge commit */
            DPRINT1("Not charging commit for prototype PTE\n");
        }
    }
 
    /* More locked pages! */
    InterlockedIncrementSizeT(&MmSystemLockPagesCount);
 
    /* Loop trying to update the reference count */
    do
    {
        /* Get the current reference count, make sure it's valid */
        OldRefCount = Pfn1->u3.e2.ReferenceCount;
        ASSERT(OldRefCount != 0);
        ASSERT(OldRefCount < 2500);
 
        /* Bump it up by one */
        RefCount = InterlockedCompareExchange16((PSHORT)&Pfn1->u3.e2.ReferenceCount,
                                                OldRefCount + 1,
                                                OldRefCount);
        ASSERT(RefCount != 0);
    } while (OldRefCount != RefCount);
 
    /* Was this the first lock attempt? If not, undo our bump */
    if (OldRefCount != 1) InterlockedDecrementSizeT(&MmSystemLockPagesCount);
}

Referenced by MI_LOCK_VA(), and MmProbeAndLockPages().

◆ MiReferenceUnusedPageAndBumpLockCount()

FORCEINLINE VOID MiReferenceUnusedPageAndBumpLockCount ( IN PMMPFN Pfn1 )

Definition at line 1800 of file miarm.h.

{
    USHORT NewRefCount;
 
    /* Make sure the page isn't used yet */
    ASSERT(Pfn1->u2.ShareCount == 0);
    ASSERT(Pfn1->u3.e1.PageLocation != ActiveAndValid);
 
    /* Is it a prototype PTE? */
    if ((Pfn1->u3.e1.PrototypePte == 1) &&
        (Pfn1->OriginalPte.u.Soft.Prototype == 1))
    {
        /* FIXME: We should charge commit */
        DPRINT1("Not charging commit for prototype PTE\n");
    }
 
    /* More locked pages! */
    InterlockedIncrementSizeT(&MmSystemLockPagesCount);
 
    /* Update the reference count */
    NewRefCount = InterlockedIncrement16((PSHORT)&Pfn1->u3.e2.ReferenceCount);
    if (NewRefCount != 1)
    {
        /* Someone had already locked the page, so undo our bump */
        ASSERT(NewRefCount < 2500);
        InterlockedDecrementSizeT(&MmSystemLockPagesCount);
    }
}

Referenced by MiResolveTransitionFault().

◆ MiReferenceUsedPageAndBumpLockCount()

FORCEINLINE VOID MiReferenceUsedPageAndBumpLockCount ( IN PMMPFN Pfn1 )

Definition at line 1755 of file miarm.h.

{
    USHORT NewRefCount;
 
    /* Is it a prototype PTE? */
    if ((Pfn1->u3.e1.PrototypePte == 1) &&
        (Pfn1->OriginalPte.u.Soft.Prototype == 1))
    {
        /* FIXME: We should charge commit */
        DPRINT1("Not charging commit for prototype PTE\n");
    }
 
    /* More locked pages! */
    InterlockedIncrementSizeT(&MmSystemLockPagesCount);
 
    /* Update the reference count */
    NewRefCount = InterlockedIncrement16((PSHORT)&Pfn1->u3.e2.ReferenceCount);
    if (NewRefCount == 2)
    {
        /* Is it locked or shared? */
        if (Pfn1->u2.ShareCount)
        {
            /* It's shared, so make sure it's active */
            ASSERT(Pfn1->u3.e1.PageLocation == ActiveAndValid);
        }
        else
        {
            /* It's locked, so we shouldn't lock again */
            InterlockedDecrementSizeT(&MmSystemLockPagesCount);
        }
    }
    else
    {
        /* Someone had already locked the page, so undo our bump */
        ASSERT(NewRefCount < 2500);
        InterlockedDecrementSizeT(&MmSystemLockPagesCount);
    }
}

Referenced by MiDispatchFault().

◆ MiReleaseExpansionLock()

FORCEINLINE VOID MiReleaseExpansionLock ( KIRQL OldIrql )

Definition at line 1557 of file miarm.h.

{
    ASSERT(MiExpansionLockOwner == PsGetCurrentThread());
    MiExpansionLockOwner = NULL;
    KeReleaseSpinLock(&MmExpansionLock, OldIrql);
    ASSERT(KeGetCurrentIrql() <= APC_LEVEL);
}

Referenced by MiArchCreateProcessAddressSpace(), MiDereferenceSessionFinal(), MiSessionAddProcess(), MiSessionInitializeWorkingSetList(), MiSessionLeader(), MiSessionRemoveProcess(), MmDeleteProcessAddressSpace(), MmGetSessionById(), and MmWorkingSetManager().

◆ MiReleaseProcessReferenceToSessionDataPage()

VOID NTAPI MiReleaseProcessReferenceToSessionDataPage ( IN PMM_SESSION_SPACE SessionGlobal )

Definition at line 209 of file session.c.

{
    ULONG i, SessionId;
    PMMPTE PointerPte;
    PFN_NUMBER PageFrameIndex[MI_SESSION_DATA_PAGES_MAXIMUM];
    PMMPFN Pfn1;
    KIRQL OldIrql;
 
    /* Is there more than just this reference? If so, bail out */
    if (InterlockedDecrement(&SessionGlobal->ProcessReferenceToSession)) return;
 
    /* Get the session ID */
    SessionId = SessionGlobal->SessionId;
    DPRINT1("Last process in session %lu going down!!!\n", SessionId);
 
    /* Free the session page tables */
#ifndef _M_AMD64
    ExFreePoolWithTag(SessionGlobal->PageTables, 'tHmM');
#endif
    ASSERT(!MI_IS_PHYSICAL_ADDRESS(SessionGlobal));
 
    /* Capture the data page PFNs */
    PointerPte = MiAddressToPte(SessionGlobal);
    for (i = 0; i < MiSessionDataPages; i++)
    {
        PageFrameIndex[i] = PFN_FROM_PTE(PointerPte + i);
    }
 
    /* Release them */
    MiReleaseSystemPtes(PointerPte, MiSessionDataPages, SystemPteSpace);
 
    /* Mark them as deleted */
    for (i = 0; i < MiSessionDataPages; i++)
    {
        Pfn1 = MI_PFN_ELEMENT(PageFrameIndex[i]);
        MI_SET_PFN_DELETED(Pfn1);
    }
 
    /* Loop every data page and drop a reference count */
    OldIrql = MiAcquirePfnLock();
    for (i = 0; i < MiSessionDataPages; i++)
    {
        /* Sanity check that the page is correct, then decrement it */
        Pfn1 = MI_PFN_ELEMENT(PageFrameIndex[i]);
        ASSERT(Pfn1->u2.ShareCount == 1);
        ASSERT(Pfn1->u3.e2.ReferenceCount == 1);
        MiDecrementShareCount(Pfn1, PageFrameIndex[i]);
    }
 
    /* Done playing with pages, release the lock */
    MiReleasePfnLock(OldIrql);
 
    /* Decrement the number of data pages */
    InterlockedDecrement(&MmSessionDataPages);
 
    /* Free this session ID from the session bitmap */
    KeAcquireGuardedMutex(&MiSessionIdMutex);
    ASSERT(RtlCheckBit(MiSessionIdBitmap, SessionId));
    RtlClearBit(MiSessionIdBitmap, SessionId);
    KeReleaseGuardedMutex(&MiSessionIdMutex);
}

Referenced by MiDereferenceSession(), and MmDeleteProcessAddressSpace().

◆ MiReleaseSystemPtes()

VOID NTAPI MiReleaseSystemPtes	(	IN PMMPTE	StartingPte,
		IN ULONG	NumberOfPtes,
		IN MMSYSTEM_PTE_POOL_TYPE	SystemPtePoolType
	)

Definition at line 264 of file syspte.c.

{
    KIRQL OldIrql;
    ULONG ClusterSize;
    PMMPTE PreviousPte, NextPte, InsertPte;
 
    //
    // Check to make sure the PTE address is within bounds
    //
    ASSERT(NumberOfPtes != 0);
    ASSERT(StartingPte >= MmSystemPtesStart[SystemPtePoolType]);
    ASSERT(StartingPte + NumberOfPtes - 1 <= MmSystemPtesEnd[SystemPtePoolType]);
 
    //
    // Zero PTEs
    //
    RtlZeroMemory(StartingPte, NumberOfPtes * sizeof(MMPTE));
 
    //
    // Acquire the System PTE lock
    //
    OldIrql = KeAcquireQueuedSpinLock(LockQueueSystemSpaceLock);
 
    //
    // Increase availability
    //
    MmTotalFreeSystemPtes[SystemPtePoolType] += NumberOfPtes;
 
    //
    // Step through the cluster list to find where to insert the PTEs
    //
    PreviousPte = &MmFirstFreeSystemPte[SystemPtePoolType];
    InsertPte = NULL;
 
    while (PreviousPte->u.List.NextEntry != MM_EMPTY_PTE_LIST)
    {
        //
        // Get the next cluster and its size
        //
        NextPte = MmSystemPteBase + PreviousPte->u.List.NextEntry;
        ClusterSize = MI_GET_CLUSTER_SIZE(NextPte);
 
        //
        // Check if this cluster is adjacent to the PTEs being released
        //
        if ((NextPte + ClusterSize == StartingPte) ||
            (StartingPte + NumberOfPtes == NextPte))
        {
            //
            // Add the PTEs in the cluster to the PTEs being released
            //
            NumberOfPtes += ClusterSize;
 
            if (NextPte < StartingPte)
                StartingPte = NextPte;
 
            //
            // Unlink this cluster and zero it
            //
            PreviousPte->u.List.NextEntry = NextPte->u.List.NextEntry;
 
            if (NextPte->u.List.OneEntry == 0)
            {
                NextPte->u.Long = 0;
                NextPte++;
            }
            NextPte->u.Long = 0;
 
            //
            // Invalidate the previously found insertion location, if any
            //
            InsertPte = NULL;
        }
        else
        {
            //
            // Check if the insertion location is right before this cluster
            //
            if ((InsertPte == NULL) && (NumberOfPtes <= ClusterSize))
                InsertPte = PreviousPte;
 
            //
            // On to the next cluster
            //
            PreviousPte = NextPte;
        }
    }
 
    //
    // If no insertion location was found, use the tail of the list
    //
    if (InsertPte == NULL)
        InsertPte = PreviousPte;
 
    //
    // Create a new cluster using the PTEs being released
    //
    if (NumberOfPtes != 1)
    {
        StartingPte->u.List.OneEntry = 0;
 
        NextPte = StartingPte + 1;
        NextPte->u.List.NextEntry = NumberOfPtes;
    }
    else
        StartingPte->u.List.OneEntry = 1;
 
    //
    // Link the new cluster into the cluster list at the insertion location
    //
    StartingPte->u.List.NextEntry = InsertPte->u.List.NextEntry;
    InsertPte->u.List.NextEntry = StartingPte - MmSystemPteBase;
 
    //
    // Release the System PTE lock
    //
    KeReleaseQueuedSpinLock(LockQueueSystemSpaceLock, OldIrql);
}

Referenced by MiAllocatePoolPages(), MiArchCreateProcessAddressSpace(), MiCopyPfn(), MiReleaseProcessReferenceToSessionDataPage(), MiZeroPfn(), MmDeleteKernelStack(), MmFreeNonCachedMemory(), MmUnmapIoSpace(), and MmUnmapLockedPages().

◆ MiRemoveAnyPage()

PFN_NUMBER NTAPI MiRemoveAnyPage ( IN ULONG Color )

Definition at line 477 of file pfnlist.c.

{
    PFN_NUMBER PageIndex;
    PMMPFN Pfn1;
 
    /* Make sure PFN lock is held and we have pages */
    MI_ASSERT_PFN_LOCK_HELD();
    ASSERT(Color < MmSecondaryColors);
    if (MmAvailablePages == 0)
    {
        return 0;
    }
 
    /* Check the colored free list */
    PageIndex = MmFreePagesByColor[FreePageList][Color].Flink;
    if (PageIndex == LIST_HEAD)
    {
        /* Check the colored zero list */
        PageIndex = MmFreePagesByColor[ZeroedPageList][Color].Flink;
        if (PageIndex == LIST_HEAD)
        {
            /* Check the free list */
            ASSERT_LIST_INVARIANT(&MmFreePageListHead);
            PageIndex = MmFreePageListHead.Flink;
            Color = PageIndex & MmSecondaryColorMask;
            if (PageIndex == LIST_HEAD)
            {
                /* Check the zero list */
                ASSERT_LIST_INVARIANT(&MmZeroedPageListHead);
                PageIndex = MmZeroedPageListHead.Flink;
                Color = PageIndex & MmSecondaryColorMask;
                ASSERT(PageIndex != LIST_HEAD);
                if (PageIndex == LIST_HEAD)
                {
                    /* FIXME: Should check the standby list */
                    ASSERT(MmZeroedPageListHead.Total == 0);
                }
            }
        }
    }
 
    /* Remove the page from its list */
    PageIndex = MiRemovePageByColor(PageIndex, Color);
    ASSERT(PageIndex != 0);
 
    /* Sanity checks */
    Pfn1 = MI_PFN_ELEMENT(PageIndex);
    ASSERT((Pfn1->u3.e1.PageLocation == FreePageList) ||
           (Pfn1->u3.e1.PageLocation == ZeroedPageList));
    ASSERT(Pfn1->u3.e2.ReferenceCount == 0);
    ASSERT(Pfn1->u2.ShareCount == 0);
    ASSERT_LIST_INVARIANT(&MmFreePageListHead);
    ASSERT_LIST_INVARIANT(&MmZeroedPageListHead);
 
    /* Return the page */
    return PageIndex;
}

Referenced by GetFreeWsleIndex(), MiAllocatePagesForMdl(), MiAllocatePoolPages(), MiArchCreateProcessAddressSpace(), MiInitMachineDependent(), MiLoadImageSection(), MiResolveDemandZeroFault(), MiResolvePageFileFault(), MiResolveProtoPteFault(), MiSessionCreateInternal(), MiSessionInitializeWorkingSetList(), MmAllocateSpecialPool(), MmArmAccessFault(), MmCreateKernelStack(), MmCreateProcessAddressSpace(), MmGrowKernelStackEx(), and MmZeroPageThread().

◆ MiRemoveMappedView()

VOID NTAPI MiRemoveMappedView	(	IN PEPROCESS	CurrentProcess,
		IN PMMVAD	Vad
	)

Definition at line 766 of file section.c.

{
    KIRQL OldIrql;
    PCONTROL_AREA ControlArea;
    PETHREAD CurrentThread = PsGetCurrentThread();
 
    /* Get the control area */
    ControlArea = Vad->ControlArea;
 
    /* We only support non-extendable, non-image, pagefile-backed regular sections */
    ASSERT(Vad->u.VadFlags.VadType == VadNone);
    ASSERT(Vad->u2.VadFlags2.ExtendableFile == FALSE);
    ASSERT(ControlArea);
    ASSERT(ControlArea->FilePointer == NULL);
 
    /* Delete the actual virtual memory pages */
    MiDeleteVirtualAddresses(Vad->StartingVpn << PAGE_SHIFT,
                             (Vad->EndingVpn << PAGE_SHIFT) | (PAGE_SIZE - 1),
                             Vad);
 
    /* Release the working set */
    MiUnlockProcessWorkingSetUnsafe(CurrentProcess, CurrentThread);
 
    /* Lock the PFN database */
    OldIrql = MiAcquirePfnLock();
 
    /* Remove references */
    ControlArea->NumberOfMappedViews--;
    ControlArea->NumberOfUserReferences--;
 
    /* Check if it should be destroyed */
    MiCheckControlArea(ControlArea, OldIrql);
}

Referenced by MiUnmapViewOfSection(), and MmCleanProcessAddressSpace().

◆ MiRemoveNode()

VOID NTAPI MiRemoveNode	(	IN PMMADDRESS_NODE	Node,
		IN PMM_AVL_TABLE	Table
	)

Definition at line 440 of file vadnode.c.

{
    PMMVAD_LONG Vad;
 
    ASSERT_LOCKED_FOR_WRITE(Table);
 
    /* Call the AVL code */
    RtlpDeleteAvlTreeNode(Table, Node);
 
    /* Decrease element count */
    Table->NumberGenericTableElements--;
 
    /* Check if this node was the hint */
    if (Table->NodeHint == Node)
    {
        /* Get a new hint, unless we're empty now, in which case nothing */
        if (!Table->NumberGenericTableElements) Table->NodeHint = NULL;
        else Table->NodeHint = Table->BalancedRoot.RightChild;
    }
 
    /* Free the node from ReactOS view as well */
    Vad = (PMMVAD_LONG)Node;
    if ((Table != &MmSectionBasedRoot) && (Vad->u.VadFlags.Spare == 0))
    {
        PMEMORY_AREA MemoryArea;
        PEPROCESS Process;
 
        /* Check if this is VM VAD */
        if (Vad->ControlArea == NULL)
        {
            /* We store the ReactOS MEMORY_AREA here */
            MemoryArea = (PMEMORY_AREA)Vad->FirstPrototypePte;
        }
        else
        {
            /* This is a section VAD. We store the ReactOS MEMORY_AREA here */
            MemoryArea = (PMEMORY_AREA)Vad->u4.Banked;
        }
 
        /* Make sure one actually still exists */
        if (MemoryArea)
        {
            /* Make sure we have not already freed it */
            ASSERT(MemoryArea != (PVOID)(ULONG_PTR)0xDEADBAB1DEADBAB1ULL);
 
            /* Get the process */
            Process = CONTAINING_RECORD(Table, EPROCESS, VadRoot);
 
            /* We only create fake memory-areas for ARM3 VADs */
            ASSERT(MemoryArea->Type == MEMORY_AREA_OWNED_BY_ARM3);
            ASSERT(MemoryArea->Vad == NULL);
 
            /* Free it */
            MmFreeMemoryArea(&Process->Vm, MemoryArea, NULL, NULL);
 
            /* Check if this is VM VAD */
            if (Vad->ControlArea == NULL)
            {
                /* Delete the pointer to it */
                Vad->FirstPrototypePte = (PVOID)(ULONG_PTR)0xDEADBAB1DEADBAB1ULL;
            }
            else
            {
                /* Delete the pointer to it */
                Vad->u4.Banked = (PVOID)(ULONG_PTR)0xDEADBAB1DEADBAB1ULL;
            }
        }
    }
}

Referenced by MiDeleteARM3Section(), MiUnmapLockedPagesInUserSpace(), MiUnmapViewOfSection(), MmCleanProcessAddressSpace(), MmDeleteTeb(), and NtFreeVirtualMemory().

◆ MiRemoveZeroPage()

PFN_NUMBER NTAPI MiRemoveZeroPage ( IN ULONG Color )

Definition at line 537 of file pfnlist.c.

{
    PFN_NUMBER PageIndex;
    PMMPFN Pfn1;
    BOOLEAN Zero = FALSE;
 
    /* Make sure PFN lock is held and we have pages */
    MI_ASSERT_PFN_LOCK_HELD();
    ASSERT(Color < MmSecondaryColors);
    if (MmAvailablePages == 0)
    {
        return 0;
    }
 
    /* Check the colored zero list */
    PageIndex = MmFreePagesByColor[ZeroedPageList][Color].Flink;
    if (PageIndex == LIST_HEAD)
    {
        /* Check the zero list */
        ASSERT_LIST_INVARIANT(&MmZeroedPageListHead);
        PageIndex = MmZeroedPageListHead.Flink;
        if (PageIndex == LIST_HEAD)
        {
            /* This means there's no zero pages, we have to look for free ones */
            ASSERT(MmZeroedPageListHead.Total == 0);
            Zero = TRUE;
 
            /* Check the colored free list */
            PageIndex = MmFreePagesByColor[FreePageList][Color].Flink;
            if (PageIndex == LIST_HEAD)
            {
                /* Check the free list */
                ASSERT_LIST_INVARIANT(&MmFreePageListHead);
                PageIndex = MmFreePageListHead.Flink;
                Color = PageIndex & MmSecondaryColorMask;
                ASSERT(PageIndex != LIST_HEAD);
                if (PageIndex == LIST_HEAD)
                {
                    /* FIXME: Should check the standby list */
                    ASSERT(MmZeroedPageListHead.Total == 0);
                }
            }
        }
        else
        {
            Color = PageIndex & MmSecondaryColorMask;
        }
    }
 
    /* Sanity checks */
    Pfn1 = MI_PFN_ELEMENT(PageIndex);
    ASSERT((Pfn1->u3.e1.PageLocation == FreePageList) ||
           (Pfn1->u3.e1.PageLocation == ZeroedPageList));
 
    /* Remove the page from its list */
    PageIndex = MiRemovePageByColor(PageIndex, Color);
    ASSERT(PageIndex != 0);
    ASSERT(Pfn1 == MI_PFN_ELEMENT(PageIndex));
 
    /* Zero it, if needed */
    if (Zero) MiZeroPhysicalPage(PageIndex);
 
    /* Sanity checks */
    ASSERT(Pfn1->u3.e2.ReferenceCount == 0);
    ASSERT(Pfn1->u2.ShareCount == 0);
    ASSERT_LIST_INVARIANT(&MmFreePageListHead);
    ASSERT_LIST_INVARIANT(&MmZeroedPageListHead);
 
    /* Return the page */
    return PageIndex;
}

Referenced by MiBuildPagedPool(), MiFillSystemPageDirectory(), MiInitializeAndChargePfn(), MiRemoveZeroPageSafe(), MiResolveDemandZeroFault(), MiSessionCommitPageTables(), and MmAllocPage().

◆ MiRemoveZeroPageSafe()

FORCEINLINE PFN_NUMBER MiRemoveZeroPageSafe ( IN ULONG Color )

Definition at line 2413 of file miarm.h.

{
    if (MmFreePagesByColor[ZeroedPageList][Color].Flink != LIST_HEAD) return MiRemoveZeroPage(Color);
    return 0;
}

Referenced by MiArchCreateProcessAddressSpace(), MiResolveDemandZeroFault(), MiSessionCreateInternal(), MiSessionInitializeWorkingSetList(), MmArmAccessFault(), and MmCreateProcessAddressSpace().

◆ MiReserveSystemPtes()

PMMPTE NTAPI MiReserveSystemPtes	(	IN ULONG	NumberOfPtes,
		IN MMSYSTEM_PTE_POOL_TYPE	SystemPtePoolType
	)

Definition at line 246 of file syspte.c.

{
    PMMPTE PointerPte;
 
    //
    // Use the extended function
    //
    PointerPte = MiReserveAlignedSystemPtes(NumberOfPtes, SystemPtePoolType, 0);
 
    //
    // Return the PTE Pointer
    //
    return PointerPte;
}

Referenced by _IRQL_requires_max_(), MiAllocatePoolPages(), MiArchCreateProcessAddressSpace(), MiBuildPagedPool(), MiBuildSystemPteSpace(), MiCopyPfn(), MiInitializeLargePageSupport(), MiInitMachineDependent(), MiLoadImageSection(), MiReloadBootLoadedDrivers(), MiSessionCreateInternal(), MiZeroPfn(), MmAllocateNonCachedMemory(), MmCreateKernelStack(), MmMapIoSpace(), and MmMapLockedPagesSpecifyCache().

◆ MiRosProtectVirtualMemory()

NTSTATUS NTAPI MiRosProtectVirtualMemory	(	IN PEPROCESS	Process,
		IN OUT PVOID *	BaseAddress,
		IN OUT PSIZE_T	NumberOfBytesToProtect,
		IN ULONG	NewAccessProtection,
		OUT PULONG OldAccessProtection	OPTIONAL
	)

Definition at line 2158 of file virtual.c.

{
    PMEMORY_AREA MemoryArea;
    PMMSUPPORT AddressSpace;
    ULONG OldAccessProtection_;
    NTSTATUS Status;
 
    *NumberOfBytesToProtect = PAGE_ROUND_UP((ULONG_PTR)(*BaseAddress) + (*NumberOfBytesToProtect)) - PAGE_ROUND_DOWN(*BaseAddress);
    *BaseAddress = (PVOID)PAGE_ROUND_DOWN(*BaseAddress);
 
    AddressSpace = &Process->Vm;
    MmLockAddressSpace(AddressSpace);
    MemoryArea = MmLocateMemoryAreaByAddress(AddressSpace, *BaseAddress);
    if (MemoryArea == NULL || MemoryArea->DeleteInProgress)
    {
        MmUnlockAddressSpace(AddressSpace);
        return STATUS_UNSUCCESSFUL;
    }
 
    if (OldAccessProtection == NULL) OldAccessProtection = &OldAccessProtection_;
 
    ASSERT(MemoryArea->Type == MEMORY_AREA_SECTION_VIEW);
    Status = MmProtectSectionView(AddressSpace,
                                  MemoryArea,
                                  *BaseAddress,
                                  *NumberOfBytesToProtect,
                                  NewAccessProtection,
                                  OldAccessProtection);
 
    MmUnlockAddressSpace(AddressSpace);
 
    return Status;
}

Referenced by MiProtectVirtualMemory().

◆ MiRosUnmapViewInSystemSpace()

NTSTATUS NTAPI MiRosUnmapViewInSystemSpace ( IN PVOID MappedBase )

Definition at line 4562 of file section.c.

{
    DPRINT("MmUnmapViewInSystemSpace() called\n");
 
    return MmUnmapViewOfSegment(MmGetKernelAddressSpace(), MappedBase);
}

Referenced by MmUnmapViewInSystemSpace().

◆ MiRosUnmapViewOfSection()

NTSTATUS NTAPI MiRosUnmapViewOfSection	(	IN PEPROCESS	Process,
		IN PVOID	BaseAddress,
		IN BOOLEAN	SkipDebuggerNotify
	)

Definition at line 3594 of file section.c.

{
    NTSTATUS Status;
    PMEMORY_AREA MemoryArea;
    PMMSUPPORT AddressSpace;
    PVOID ImageBaseAddress = 0;
 
    DPRINT("Opening memory area Process %p BaseAddress %p\n",
           Process, BaseAddress);
 
    ASSERT(Process);
 
    AddressSpace = &Process->Vm;
 
    MemoryArea = MmLocateMemoryAreaByAddress(AddressSpace,
                 BaseAddress);
    if (MemoryArea == NULL ||
#ifdef NEWCC
            ((MemoryArea->Type != MEMORY_AREA_SECTION_VIEW) && (MemoryArea->Type != MEMORY_AREA_CACHE)) ||
#else
            (MemoryArea->Type != MEMORY_AREA_SECTION_VIEW) ||
#endif
            MemoryArea->DeleteInProgress)
 
    {
        if (MemoryArea) ASSERT(MemoryArea->Type != MEMORY_AREA_OWNED_BY_ARM3);
 
        DPRINT1("Unable to find memory area at address %p.\n", BaseAddress);
        return STATUS_NOT_MAPPED_VIEW;
    }
 
    if (MemoryArea->VadNode.u.VadFlags.VadType == VadImageMap)
    {
        ULONG i;
        ULONG NrSegments;
        PMM_IMAGE_SECTION_OBJECT ImageSectionObject;
        PMM_SECTION_SEGMENT SectionSegments;
        PMM_SECTION_SEGMENT Segment;
 
        Segment = MemoryArea->SectionData.Segment;
        ImageSectionObject = ImageSectionObjectFromSegment(Segment);
        SectionSegments = ImageSectionObject->Segments;
        NrSegments = ImageSectionObject->NrSegments;
 
        MemoryArea->DeleteInProgress = TRUE;
 
        /* Search for the current segment within the section segments
         * and calculate the image base address */
        for (i = 0; i < NrSegments; i++)
        {
            if (Segment == &SectionSegments[i])
            {
                ImageBaseAddress = (char*)BaseAddress - (ULONG_PTR)SectionSegments[i].Image.VirtualAddress;
                break;
            }
        }
        if (i >= NrSegments)
        {
            KeBugCheck(MEMORY_MANAGEMENT);
        }
 
        for (i = 0; i < NrSegments; i++)
        {
            PVOID SBaseAddress = (PVOID)
                                 ((char*)ImageBaseAddress + (ULONG_PTR)SectionSegments[i].Image.VirtualAddress);
 
            Status = MmUnmapViewOfSegment(AddressSpace, SBaseAddress);
            if (!NT_SUCCESS(Status))
            {
                DPRINT1("MmUnmapViewOfSegment failed for %p (Process %p) with %lx\n",
                        SBaseAddress, Process, Status);
                ASSERT(NT_SUCCESS(Status));
            }
        }
        DPRINT("One mapping less for %p\n", ImageSectionObject->FileObject->SectionObjectPointer);
        InterlockedDecrement(&ImageSectionObject->MapCount);
    }
    else
    {
        PMM_SECTION_SEGMENT Segment = MemoryArea->SectionData.Segment;
        PMMVAD Vad = &MemoryArea->VadNode;
        PCONTROL_AREA ControlArea  = Vad->ControlArea;
        PFILE_OBJECT FileObject;
        SIZE_T ViewSize;
        LARGE_INTEGER ViewOffset;
        ViewOffset.QuadPart = MemoryArea->SectionData.ViewOffset;
 
        InterlockedIncrement64(Segment->ReferenceCount);
 
        ViewSize = PAGE_SIZE + ((Vad->EndingVpn - Vad->StartingVpn) << PAGE_SHIFT);
 
        Status = MmUnmapViewOfSegment(AddressSpace, BaseAddress);
        if (!NT_SUCCESS(Status))
        {
            DPRINT1("MmUnmapViewOfSegment failed for %p (Process %p) with %lx\n",
                    BaseAddress, Process, Status);
            ASSERT(NT_SUCCESS(Status));
        }
 
        /* These might be deleted now */
        Vad = NULL;
        MemoryArea = NULL;
 
        if (FlagOn(*Segment->Flags, MM_PHYSICALMEMORY_SEGMENT))
        {
            /* Don't bother */
            MmDereferenceSegment(Segment);
            return STATUS_SUCCESS;
        }
        ASSERT(FlagOn(*Segment->Flags, MM_DATAFILE_SEGMENT));
 
        FileObject = Segment->FileObject;
        FsRtlAcquireFileExclusive(FileObject);
 
        /* Don't bother for auto-delete closed file. */
        if (FlagOn(FileObject->Flags, FO_DELETE_ON_CLOSE) && FlagOn(FileObject->Flags, FO_CLEANUP_COMPLETE))
        {
            FsRtlReleaseFile(FileObject);
            MmDereferenceSegment(Segment);
            return STATUS_SUCCESS;
        }
 
        /*
         * Flush only when last mapping is deleted.
         * FIXME: Why ControlArea == NULL? Or rather: is ControlArea ever not NULL here?
         */
        if (ControlArea == NULL || ControlArea->NumberOfMappedViews == 1)
        {
            while (ViewSize > 0)
            {
                ULONG FlushSize = min(ViewSize, PAGE_ROUND_DOWN(MAXULONG));
                MmFlushSegment(FileObject->SectionObjectPointer,
                               &ViewOffset,
                               FlushSize,
                               NULL);
                ViewSize -= FlushSize;
                ViewOffset.QuadPart += FlushSize;
            }
        }
 
        FsRtlReleaseFile(FileObject);
        MmDereferenceSegment(Segment);
    }
 
    /* Notify debugger */
    if (ImageBaseAddress && !SkipDebuggerNotify) DbgkUnMapViewOfSection(ImageBaseAddress);
 
    return STATUS_SUCCESS;
}

Referenced by MiRosCleanupMemoryArea(), and MiUnmapViewOfSection().

◆ MiSessionAddProcess()

VOID NTAPI MiSessionAddProcess ( IN PEPROCESS NewProcess )

Definition at line 423 of file session.c.

{
    PMM_SESSION_SPACE SessionGlobal;
    KIRQL OldIrql;
 
    /* The current process must already be in a session */
    if (!(PsGetCurrentProcess()->Flags & PSF_PROCESS_IN_SESSION_BIT)) return;
 
    /* Sanity check */
    ASSERT(MmIsAddressValid(MmSessionSpace) == TRUE);
 
    /* Get the global session */
    SessionGlobal = MmSessionSpace->GlobalVirtualAddress;
 
    /* Increment counters */
    InterlockedIncrement((PLONG)&SessionGlobal->ReferenceCount);
    InterlockedIncrement(&SessionGlobal->ResidentProcessCount);
    InterlockedIncrement(&SessionGlobal->ProcessReferenceToSession);
 
    /* Set the session pointer */
    ASSERT(NewProcess->Session == NULL);
    NewProcess->Session = SessionGlobal;
 
    /* Acquire the expansion lock while touching the session */
    OldIrql = MiAcquireExpansionLock();
 
    /* Insert it into the process list */
    InsertTailList(&SessionGlobal->ProcessList, &NewProcess->SessionProcessLinks);
 
    /* Release the lock again */
    MiReleaseExpansionLock(OldIrql);
 
    /* Set the flag */
    PspSetProcessFlag(NewProcess, PSF_PROCESS_IN_SESSION_BIT);
}

Referenced by MmCreateProcessAddressSpace().

◆ MiSessionRemoveProcess()

VOID NTAPI MiSessionRemoveProcess ( VOID )

Definition at line 392 of file session.c.

{
    PEPROCESS CurrentProcess = PsGetCurrentProcess();
    KIRQL OldIrql;
 
    /* If the process isn't already in a session, or if it's the leader... */
    if (!(CurrentProcess->Flags & PSF_PROCESS_IN_SESSION_BIT) ||
        (CurrentProcess->Vm.Flags.SessionLeader))
    {
        /* Then there's nothing to do */
        return;
    }
 
    /* Sanity check */
    ASSERT(MmIsAddressValid(MmSessionSpace) == TRUE);
 
    /* Acquire the expansion lock while touching the session */
    OldIrql = MiAcquireExpansionLock();
 
    /* Remove the process from the list */
    RemoveEntryList(&CurrentProcess->SessionProcessLinks);
 
    /* Release the lock again */
    MiReleaseExpansionLock(OldIrql);
 
    /* Dereference the session */
    MiDereferenceSession();
}

Referenced by MmCleanProcessAddressSpace().

◆ MiSyncARM3WithROS()

VOID FASTCALL MiSyncARM3WithROS	(	IN PVOID	AddressStart,
		IN PVOID	AddressEnd
	)

◆ MiSyncCachedRanges()

VOID NTAPI MiSyncCachedRanges ( VOID )

Definition at line 54 of file largepag.c.

{
    ULONG i;
 
    /* Scan every range */
    for (i = 0; i < MiLargePageRangeIndex; i++)
    {
        UNIMPLEMENTED_DBGBREAK("No support for large pages\n");
    }
}

Referenced by MmArmInitSystem().

◆ MiUnlinkFreeOrZeroedPage()

VOID NTAPI MiUnlinkFreeOrZeroedPage ( IN PMMPFN Entry )

Definition at line 137 of file pfnlist.c.

{
    PFN_NUMBER OldFlink, OldBlink;
    PMMPFNLIST ListHead;
    MMLISTS ListName;
    ULONG Color;
    PMMCOLOR_TABLES ColorTable;
    PMMPFN Pfn1;
 
    /* Make sure the PFN lock is held */
    MI_ASSERT_PFN_LOCK_HELD();
 
    /* Make sure the PFN entry isn't in-use */
    ASSERT(Entry->u3.e1.WriteInProgress == 0);
    ASSERT(Entry->u3.e1.ReadInProgress == 0);
 
    /* Find the list for this entry, make sure it's the free or zero list */
    ListHead = MmPageLocationList[Entry->u3.e1.PageLocation];
    ListName = ListHead->ListName;
    ASSERT(ListHead != NULL);
    ASSERT(ListName <= FreePageList);
    ASSERT_LIST_INVARIANT(ListHead);
 
    /* Remove one count */
    ASSERT(ListHead->Total != 0);
    ListHead->Total--;
 
    /* Get the forward and back pointers */
    OldFlink = Entry->u1.Flink;
    OldBlink = Entry->u2.Blink;
 
    /* Check if the next entry is the list head */
    if (OldFlink != LIST_HEAD)
    {
        /* It is not, so set the backlink of the actual entry, to our backlink */
        MI_PFN_ELEMENT(OldFlink)->u2.Blink = OldBlink;
    }
    else
    {
        /* Set the list head's backlink instead */
        ListHead->Blink = OldBlink;
    }
 
    /* Check if the back entry is the list head */
    if (OldBlink != LIST_HEAD)
    {
        /* It is not, so set the backlink of the actual entry, to our backlink */
        MI_PFN_ELEMENT(OldBlink)->u1.Flink = OldFlink;
    }
    else
    {
        /* Set the list head's backlink instead */
        ListHead->Flink = OldFlink;
    }
 
    /* Get the page color */
    OldBlink = MiGetPfnEntryIndex(Entry);
    Color = OldBlink & MmSecondaryColorMask;
 
    /* Get the first page on the color list */
    ColorTable = &MmFreePagesByColor[ListName][Color];
 
    /* Check if this was was actually the head */
    OldFlink = ColorTable->Flink;
    if (OldFlink == OldBlink)
    {
        /* Make the table point to the next page this page was linking to */
        ColorTable->Flink = Entry->OriginalPte.u.Long;
        if (ColorTable->Flink != LIST_HEAD)
        {
            /* And make the previous link point to the head now */
            MI_PFN_ELEMENT(ColorTable->Flink)->u4.PteFrame = COLORED_LIST_HEAD;
        }
        else
        {
            /* And if that page was the head, loop the list back around */
            ColorTable->Blink = (PVOID)LIST_HEAD;
        }
    }
    else
    {
        /* This page shouldn't be pointing back to the head */
        ASSERT(Entry->u4.PteFrame != COLORED_LIST_HEAD);
 
        /* Make the back link point to whoever the next page is */
        Pfn1 = MI_PFN_ELEMENT(Entry->u4.PteFrame);
        Pfn1->OriginalPte.u.Long = Entry->OriginalPte.u.Long;
 
        /* Check if this page was pointing to the head */
        if (Entry->OriginalPte.u.Long != LIST_HEAD)
        {
            /* Make the back link point to the head */
            Pfn1 = MI_PFN_ELEMENT(Entry->OriginalPte.u.Long);
            Pfn1->u4.PteFrame = Entry->u4.PteFrame;
        }
        else
        {
            /* Then the table is directly back pointing to this page now */
            ColorTable->Blink = Pfn1;
        }
    }
 
    /* One less colored page */
    ASSERT(ColorTable->Count >= 1);
    ColorTable->Count--;
 
    /* ReactOS Hack */
    Entry->OriginalPte.u.Long = 0;
 
    /* We are not on a list anymore */
    Entry->u1.Flink = Entry->u2.Blink = 0;
    ASSERT_LIST_INVARIANT(ListHead);
 
    /* Decrement number of available pages */
    MiDecrementAvailablePages();
 
#if MI_TRACE_PFNS
    ASSERT(MI_PFN_CURRENT_USAGE != MI_USAGE_NOT_SET);
    Entry->PfnUsage = MI_PFN_CURRENT_USAGE;
    memcpy(Entry->ProcessName, MI_PFN_CURRENT_PROCESS_NAME, 16);
    Entry->CallSite = _ReturnAddress();
    MI_PFN_CURRENT_USAGE = MI_USAGE_NOT_SET;
    MI_SET_PROCESS2("Not Set");
#endif
}

Referenced by MiAllocatePagesForMdl(), and MiFindContiguousPages().

◆ MiUnlinkPageFromList()

VOID NTAPI MiUnlinkPageFromList ( IN PMMPFN Pfn )

Definition at line 265 of file pfnlist.c.

{
    PMMPFNLIST ListHead;
    PFN_NUMBER OldFlink, OldBlink;
 
    /* Make sure the PFN lock is held */
    MI_ASSERT_PFN_LOCK_HELD();
 
    /* ARM3 should only call this for dead pages */
    ASSERT(Pfn->u3.e2.ReferenceCount == 0);
 
    /* Transition pages are supposed to be standby/modified/nowrite */
    ListHead = MmPageLocationList[Pfn->u3.e1.PageLocation];
    ASSERT(ListHead->ListName >= StandbyPageList);
 
    /* Check if this was standby, or modified */
    if (ListHead == &MmStandbyPageListHead)
    {
        /* Should not be a ROM page */
        ASSERT(Pfn->u3.e1.Rom == 0);
 
        /* Get the exact list */
        ListHead = &MmStandbyPageListByPriority[Pfn->u4.Priority];
 
        /* Decrement number of available pages */
        MiDecrementAvailablePages();
 
        /* Decrease transition page counter */
        ASSERT(Pfn->u3.e1.PrototypePte == 1); /* Only supported ARM3 case */
        MmTransitionSharedPages--;
    }
    else if (ListHead == &MmModifiedPageListHead)
    {
        /* Only shared memory (page-file backed) modified pages are supported */
        ASSERT(Pfn->OriginalPte.u.Soft.Prototype == 0);
 
        /* Decrement the counters */
        ListHead->Total--;
        MmTotalPagesForPagingFile--;
 
        /* Pick the correct colored list */
        ListHead = &MmModifiedPageListByColor[0];
 
        /* Decrease transition page counter */
        MmTransitionSharedPages--;
    }
    else if (ListHead == &MmModifiedNoWritePageListHead)
    {
        /* List not yet supported */
        ASSERT(FALSE);
    }
 
    /* Nothing should be in progress and the list should not be empty */
    ASSERT(Pfn->u3.e1.WriteInProgress == 0);
    ASSERT(Pfn->u3.e1.ReadInProgress == 0);
    ASSERT(ListHead->Total != 0);
 
    /* Get the forward and back pointers */
    OldFlink = Pfn->u1.Flink;
    OldBlink = Pfn->u2.Blink;
 
    /* Check if the next entry is the list head */
    if (OldFlink != LIST_HEAD)
    {
        /* It is not, so set the backlink of the actual entry, to our backlink */
        MI_PFN_ELEMENT(OldFlink)->u2.Blink = OldBlink;
    }
    else
    {
        /* Set the list head's backlink instead */
        ListHead->Blink = OldBlink;
    }
 
    /* Check if the back entry is the list head */
    if (OldBlink != LIST_HEAD)
    {
        /* It is not, so set the backlink of the actual entry, to our backlink */
        MI_PFN_ELEMENT(OldBlink)->u1.Flink = OldFlink;
    }
    else
    {
        /* Set the list head's backlink instead */
        ListHead->Flink = OldFlink;
    }
 
    /* ReactOS Hack */
    Pfn->OriginalPte.u.Long = 0;
 
    /* We are not on a list anymore */
    Pfn->u1.Flink = Pfn->u2.Blink = 0;
 
    /* Remove one entry from the list */
    ListHead->Total--;
 
    ASSERT_LIST_INVARIANT(ListHead);
}

Referenced by MiDeletePte(), MiDispatchFault(), MiResolveTransitionFault(), and MiSegmentDelete().

◆ MiUnlockProcessWorkingSet()

FORCEINLINE VOID MiUnlockProcessWorkingSet	(	IN PEPROCESS	Process,
		IN PETHREAD	Thread
	)

Definition at line 1207 of file miarm.h.

{
    /* Make sure we are the owner of a safe acquisition */
    ASSERT(MI_WS_OWNER(Process));
    ASSERT(!MI_IS_WS_UNSAFE(Process));
 
    /* The thread doesn't own it anymore */
    ASSERT(Thread->OwnsProcessWorkingSetExclusive == TRUE);
    Thread->OwnsProcessWorkingSetExclusive = FALSE;
 
    /* Release the lock and re-enable APCs */
    ExReleasePushLockExclusive(&Process->Vm.WorkingSetMutex);
    KeLeaveGuardedRegion();
}

Referenced by MiLockVirtualMemory(), MiUnlockProcessWorkingSetForFault(), MiUnlockVirtualMemory(), MmArmAccessFault(), MmFreeMemoryArea(), MmInitializeProcessAddressSpace(), MmProbeAndLockPages(), and MmTrimUserMemory().

◆ MiUnlockProcessWorkingSetForFault()

FORCEINLINE VOID MiUnlockProcessWorkingSetForFault	(	IN PEPROCESS	Process,
		IN PETHREAD	Thread,
		OUT PBOOLEAN	Safe,
		OUT PBOOLEAN	Shared
	)

Definition at line 1481 of file miarm.h.

{
    ASSERT(MI_WS_OWNER(Process));
 
    /* Check if the current owner is unsafe */
    if (MI_IS_WS_UNSAFE(Process))
    {
        /* Release unsafely */
        MiUnlockProcessWorkingSetUnsafe(Process, Thread);
        *Safe = FALSE;
        *Shared = FALSE;
    }
    else if (Thread->OwnsProcessWorkingSetExclusive == 1)
    {
        /* Owner is safe and exclusive, release normally */
        MiUnlockProcessWorkingSet(Process, Thread);
        *Safe = TRUE;
        *Shared = FALSE;
    }
    else
    {
        /* Owner is shared (implies safe), release normally */
        MiUnlockProcessWorkingSetShared(Process, Thread);
        *Safe = TRUE;
        *Shared = TRUE;
    }
}

Referenced by MiGetPageProtection(), and MiMakeSystemAddressValid().

◆ MiUnlockProcessWorkingSetShared()

FORCEINLINE VOID MiUnlockProcessWorkingSetShared	(	IN PEPROCESS	Process,
		IN PETHREAD	Thread
	)

Definition at line 1228 of file miarm.h.

{
    /* Make sure we are the owner of a safe acquisition (because shared) */
    ASSERT(MI_WS_OWNER(Process));
    ASSERT(!MI_IS_WS_UNSAFE(Process));
 
    /* Ensure we are in a shared acquisition */
    ASSERT(Thread->OwnsProcessWorkingSetShared == TRUE);
    ASSERT(Thread->OwnsProcessWorkingSetExclusive == FALSE);
 
    /* Don't claim the lock anylonger */
    Thread->OwnsProcessWorkingSetShared = FALSE;
 
    /* Release the lock and re-enable APCs */
    ExReleasePushLockShared(&Process->Vm.WorkingSetMutex);
    KeLeaveGuardedRegion();
}

Referenced by MiQueryMemoryBasicInformation(), MiUnlockProcessWorkingSetForFault(), MmAccessFault(), MmGetPageFileMapping(), MmGetPageProtect(), MmGetPfnForProcess(), MmIsDisabledPage(), MmIsPagePresent(), and MmIsPageSwapEntry().

◆ MiUnlockProcessWorkingSetUnsafe()

FORCEINLINE VOID MiUnlockProcessWorkingSetUnsafe	(	IN PEPROCESS	Process,
		IN PETHREAD	Thread
	)

Definition at line 1252 of file miarm.h.

{
    /* Make sure we are the owner of an unsafe acquisition */
    ASSERT(KeGetCurrentIrql() <= APC_LEVEL);
    ASSERT(KeAreAllApcsDisabled() == TRUE);
    ASSERT(MI_WS_OWNER(Process));
    ASSERT(MI_IS_WS_UNSAFE(Process));
 
    /* No longer unsafe */
    Process->Vm.Flags.AcquiredUnsafe = 0;
 
    /* The thread doesn't own it anymore */
    ASSERT(Thread->OwnsProcessWorkingSetExclusive == TRUE);
    Thread->OwnsProcessWorkingSetExclusive = FALSE;
 
    /* Release the lock but don't touch APC state */
    ExReleasePushLockExclusive(&Process->Vm.WorkingSetMutex);
    ASSERT(KeGetCurrentIrql() <= APC_LEVEL);
}

Referenced by _Success_(), MiDecommitPages(), MiInsertVadEx(), MiMapLockedPagesInUserSpace(), MiProtectVirtualMemory(), MiRemoveMappedView(), MiUnlockProcessWorkingSetForFault(), MiUnmapLockedPagesInUserSpace(), MmCleanProcessAddressSpace(), MmCreatePageFileMapping(), MmCreateVirtualMappingUnsafeEx(), MmDeletePageFileMapping(), MmDeleteTeb(), MmInsertMemoryArea(), MmSetDirtyBit(), MmSetPageProtect(), NtAllocateVirtualMemory(), and NtFreeVirtualMemory().

◆ MiUnlockWorkingSet()

FORCEINLINE VOID MiUnlockWorkingSet	(	IN PETHREAD	Thread,
		IN PMMSUPPORT	WorkingSet
	)

Definition at line 1364 of file miarm.h.

{
    /* Working set should be in global memory */
    ASSERT(MI_IS_SESSION_ADDRESS((PVOID)WorkingSet) == FALSE);
 
    /* Which working set is this? */
    if (WorkingSet == &MmSystemCacheWs)
    {
        /* Release the system working set */
        ASSERT((Thread->OwnsSystemWorkingSetExclusive == TRUE) &&
               (Thread->OwnsSystemWorkingSetShared == FALSE));
        Thread->OwnsSystemWorkingSetExclusive = FALSE;
    }
    else if (WorkingSet->Flags.SessionSpace)
    {
        /* Release the session working set */
        ASSERT((Thread->OwnsSessionWorkingSetExclusive == TRUE) &&
               (Thread->OwnsSessionWorkingSetShared == FALSE));
        Thread->OwnsSessionWorkingSetExclusive = FALSE;
    }
    else
    {
        /* Release the process working set */
        ASSERT((Thread->OwnsProcessWorkingSetExclusive == TRUE) &&
               (Thread->OwnsProcessWorkingSetShared == FALSE));
        Thread->OwnsProcessWorkingSetExclusive = FALSE;
    }
 
    /* Release the working set lock */
    ExReleasePushLockExclusive(&WorkingSet->WorkingSetMutex);
 
    /* Unblock APCs */
    KeLeaveGuardedRegion();
}

Referenced by MiDeleteSystemPageableVm(), MiSetPagingOfDriver(), MmAdjustWorkingSetSize(), MmArmAccessFault(), MmFreeMemoryArea(), MmInsertMemoryArea(), and MmWorkingSetManager().

◆ MiUnlockWorkingSetShared()

FORCEINLINE VOID MiUnlockWorkingSetShared	(	_In_ PETHREAD	Thread,
		_In_ PMMSUPPORT	WorkingSet
	)

Definition at line 1402 of file miarm.h.

{
    /* Working set should be in global memory */
    ASSERT(MI_IS_SESSION_ADDRESS((PVOID)WorkingSet) == FALSE);
 
    /* Which working set is this? */
    if (WorkingSet == &MmSystemCacheWs)
    {
        /* Release the system working set */
        ASSERT((Thread->OwnsSystemWorkingSetExclusive == FALSE) &&
               (Thread->OwnsSystemWorkingSetShared == TRUE));
        Thread->OwnsSystemWorkingSetShared = FALSE;
    }
    else if (WorkingSet->Flags.SessionSpace)
    {
        /* Release the session working set */
        ASSERT((Thread->OwnsSessionWorkingSetExclusive == FALSE) &&
               (Thread->OwnsSessionWorkingSetShared == TRUE));
        Thread->OwnsSessionWorkingSetShared = FALSE;
    }
    else
    {
        /* Release the process working set */
        ASSERT((Thread->OwnsProcessWorkingSetExclusive == FALSE) &&
               (Thread->OwnsProcessWorkingSetShared == TRUE));
        Thread->OwnsProcessWorkingSetShared = FALSE;
    }
 
    /* Release the working set lock */
    ExReleasePushLockShared(&WorkingSet->WorkingSetMutex);
 
    /* Unblock APCs */
    KeLeaveGuardedRegion();
}

Referenced by MmAccessFault(), and MmWorkingSetManager().

◆ MiWriteProtectSystemImage()

VOID NTAPI MiWriteProtectSystemImage ( _In_ PVOID ImageBase )

Definition at line 2481 of file sysldr.c.

{
    PIMAGE_NT_HEADERS NtHeaders;
    PIMAGE_SECTION_HEADER SectionHeaders, Section;
    ULONG i;
    PVOID SectionBase, SectionEnd;
    ULONG SectionSize;
    ULONG Protection;
    PMMPTE FirstPte, LastPte;
 
    /* Check if the registry setting is on or not */
    if (MmEnforceWriteProtection == FALSE)
    {
        /* Ignore section protection */
        return;
    }
 
    /* Large page mapped images are not supported */
    NT_ASSERT(!MI_IS_PHYSICAL_ADDRESS(ImageBase));
 
    /* Session images are not yet supported */
    NT_ASSERT(!MI_IS_SESSION_ADDRESS(ImageBase));
 
    /* Get the NT headers */
    NtHeaders = RtlImageNtHeader(ImageBase);
    if (NtHeaders == NULL)
    {
        DPRINT1("Failed to get NT headers for image @ %p\n", ImageBase);
        return;
    }
 
    /* Don't touch NT4 drivers */
    if ((NtHeaders->OptionalHeader.MajorOperatingSystemVersion < 5) ||
        (NtHeaders->OptionalHeader.MajorSubsystemVersion < 5))
    {
        DPRINT1("Skipping NT 4 driver @ %p\n", ImageBase);
        return;
    }
 
    /* Get the section headers */
    SectionHeaders = IMAGE_FIRST_SECTION(NtHeaders);
 
    /* Get the base address of the first section */
    SectionBase = Add2Ptr(ImageBase, SectionHeaders[0].VirtualAddress);
 
    /* Start protecting the image header as R/W */
    FirstPte = MiAddressToPte(ImageBase);
    LastPte = MiAddressToPte(SectionBase) - 1;
    Protection = IMAGE_SCN_MEM_READ | IMAGE_SCN_MEM_WRITE;
    if (LastPte >= FirstPte)
    {
        MiSetSystemCodeProtection(FirstPte, LastPte, Protection);
    }
 
    /* Loop the sections */
    for (i = 0; i < NtHeaders->FileHeader.NumberOfSections; i++)
    {
        /* Get the section base address and size */
        Section = &SectionHeaders[i];
        SectionBase = Add2Ptr(ImageBase, Section->VirtualAddress);
        SectionSize = max(Section->SizeOfRawData, Section->Misc.VirtualSize);
 
        /* Get the first PTE of this section */
        FirstPte = MiAddressToPte(SectionBase);
 
        /* Check for overlap with the previous range */
        if (FirstPte == LastPte)
        {
            /* Combine the old and new protection by ORing them */
            Protection |= (Section->Characteristics & IMAGE_SCN_PROTECTION_MASK);
 
            /* Update the protection for this PTE */
            MiSetSystemCodeProtection(FirstPte, FirstPte, Protection);
 
            /* Skip this PTE */
            FirstPte++;
        }
 
        /* There can not be gaps! */
        NT_ASSERT(FirstPte == (LastPte + 1));
 
        /* Get the end of the section and the last PTE */
        SectionEnd = Add2Ptr(SectionBase, SectionSize - 1);
        NT_ASSERT(SectionEnd < Add2Ptr(ImageBase, NtHeaders->OptionalHeader.SizeOfImage));
        LastPte = MiAddressToPte(SectionEnd);
 
        /* If there are no more pages (after an overlap), skip this section */
        if (LastPte < FirstPte)
        {
            NT_ASSERT(FirstPte == (LastPte + 1));
            continue;
        }
 
        /* Get the section protection */
        Protection = (Section->Characteristics & IMAGE_SCN_PROTECTION_MASK);
 
        /* Update the protection for this section */
        MiSetSystemCodeProtection(FirstPte, LastPte, Protection);
    }
 
    /* Image should end with the last section */
    if (ALIGN_UP_POINTER_BY(SectionEnd, PAGE_SIZE) !=
        Add2Ptr(ImageBase, NtHeaders->OptionalHeader.SizeOfImage))
    {
        DPRINT1("ImageBase 0x%p ImageSize 0x%lx Section %u VA 0x%lx Raw 0x%lx virt 0x%lx\n",
            ImageBase,
            NtHeaders->OptionalHeader.SizeOfImage,
            i,
            Section->VirtualAddress,
            Section->SizeOfRawData,
            Section->Misc.VirtualSize);
    }
}

Referenced by MmInitSystem(), and MmLoadSystemImage().

◆ MiZeroPhysicalPage()

VOID NTAPI MiZeroPhysicalPage ( IN PFN_NUMBER PageFrameIndex )

Definition at line 122 of file pfnlist.c.

{
    KIRQL OldIrql;
    PVOID VirtualAddress;
    PEPROCESS Process = PsGetCurrentProcess();
 
    /* Map in hyperspace, then wipe it using XMMI or MEMSET */
    VirtualAddress = MiMapPageInHyperSpace(Process, PageFrameIndex, &OldIrql);
    ASSERT(VirtualAddress);
    KeZeroPages(VirtualAddress, PAGE_SIZE);
    MiUnmapPageInHyperSpace(Process, VirtualAddress, OldIrql);
}

◆ MM_ANY_WS_LOCK_HELD()

FORCEINLINE BOOLEAN MM_ANY_WS_LOCK_HELD ( IN PETHREAD Thread )

Definition at line 1065 of file miarm.h.

{
    /* If any of these are held, return TRUE */
    return ((Thread->OwnsProcessWorkingSetExclusive) ||
            (Thread->OwnsProcessWorkingSetShared) ||
            (Thread->OwnsSystemWorkingSetExclusive) ||
            (Thread->OwnsSystemWorkingSetShared) ||
            (Thread->OwnsSessionWorkingSetExclusive) ||
            (Thread->OwnsSessionWorkingSetShared));
}

Referenced by MiLockProcessWorkingSet(), MiLockProcessWorkingSetShared(), MiLockProcessWorkingSetUnsafe(), MiLockWorkingSet(), MiLockWorkingSetShared(), MmRebalanceMemoryConsumersAndWait(), and TrimWsList().

◆ MM_ANY_WS_LOCK_HELD_EXCLUSIVE()

FORCEINLINE BOOLEAN MM_ANY_WS_LOCK_HELD_EXCLUSIVE ( _In_ PETHREAD Thread )

Definition at line 1078 of file miarm.h.

{
    return ((Thread->OwnsProcessWorkingSetExclusive) ||
            (Thread->OwnsSystemWorkingSetExclusive) ||
            (Thread->OwnsSessionWorkingSetExclusive));
}

Referenced by MiDeletePte(), MiInsertInWorkingSetList(), and RemoveFromWsList().

◆ MmArmAccessFault()

NTSTATUS NTAPI MmArmAccessFault	(	IN ULONG	FaultCode,
		IN PVOID	Address,
		IN KPROCESSOR_MODE	Mode,
		IN PVOID	TrapInformation
	)

Definition at line 1698 of file pagfault.c.

{
    KIRQL OldIrql = KeGetCurrentIrql(), LockIrql;
    PMMPTE ProtoPte = NULL;
    PMMPTE PointerPte = MiAddressToPte(Address);
    PMMPDE PointerPde = MiAddressToPde(Address);
#if (_MI_PAGING_LEVELS >= 3)
    PMMPDE PointerPpe = MiAddressToPpe(Address);
#if (_MI_PAGING_LEVELS == 4)
    PMMPDE PointerPxe = MiAddressToPxe(Address);
#endif
#endif
    MMPTE TempPte;
    PETHREAD CurrentThread;
    PEPROCESS CurrentProcess;
    NTSTATUS Status;
    PMMSUPPORT WorkingSet;
    ULONG ProtectionCode;
    PMMVAD Vad = NULL;
    PFN_NUMBER PageFrameIndex;
    ULONG Color;
    BOOLEAN IsSessionAddress;
    PMMPFN Pfn1;
    DPRINT("ARM3 FAULT AT: %p\n", Address);
 
    /* Check for page fault on high IRQL */
    if (OldIrql > APC_LEVEL)
    {
#if (_MI_PAGING_LEVELS < 3)
        /* Could be a page table for paged pool, which we'll allow */
        if (MI_IS_SYSTEM_PAGE_TABLE_ADDRESS(Address)) MiSynchronizeSystemPde((PMMPDE)PointerPte);
        MiCheckPdeForPagedPool(Address);
#endif
        /* Check if any of the top-level pages are invalid */
        if (
#if (_MI_PAGING_LEVELS == 4)
            (PointerPxe->u.Hard.Valid == 0) ||
#endif
#if (_MI_PAGING_LEVELS >= 3)
            (PointerPpe->u.Hard.Valid == 0) ||
#endif
            (PointerPde->u.Hard.Valid == 0) ||
            (PointerPte->u.Hard.Valid == 0))
        {
            /* This fault is not valid, print out some debugging help */
            DbgPrint("MM:***PAGE FAULT AT IRQL > 1  Va %p, IRQL %lx\n",
                     Address,
                     OldIrql);
            if (TrapInformation)
            {
                PKTRAP_FRAME TrapFrame = TrapInformation;
#ifdef _M_IX86
                DbgPrint("MM:***EIP %p, EFL %p\n", TrapFrame->Eip, TrapFrame->EFlags);
                DbgPrint("MM:***EAX %p, ECX %p EDX %p\n", TrapFrame->Eax, TrapFrame->Ecx, TrapFrame->Edx);
                DbgPrint("MM:***EBX %p, ESI %p EDI %p\n", TrapFrame->Ebx, TrapFrame->Esi, TrapFrame->Edi);
#elif defined(_M_AMD64)
                DbgPrint("MM:***RIP %p, EFL %p\n", TrapFrame->Rip, TrapFrame->EFlags);
                DbgPrint("MM:***RAX %p, RCX %p RDX %p\n", TrapFrame->Rax, TrapFrame->Rcx, TrapFrame->Rdx);
                DbgPrint("MM:***RBX %p, RSI %p RDI %p\n", TrapFrame->Rbx, TrapFrame->Rsi, TrapFrame->Rdi);
#elif defined(_M_ARM)
                DbgPrint("MM:***PC %p\n", TrapFrame->Pc);
                DbgPrint("MM:***R0 %p, R1 %p R2 %p, R3 %p\n", TrapFrame->R0, TrapFrame->R1, TrapFrame->R2, TrapFrame->R3);
                DbgPrint("MM:***R11 %p, R12 %p SP %p, LR %p\n", TrapFrame->R11, TrapFrame->R12, TrapFrame->Sp, TrapFrame->Lr);
#endif
            }
 
            /* Tell the trap handler to fail */
            return STATUS_IN_PAGE_ERROR | 0x10000000;
        }
 
        /* Not yet implemented in ReactOS */
        ASSERT(MI_IS_PAGE_LARGE(PointerPde) == FALSE);
        ASSERT((!MI_IS_NOT_PRESENT_FAULT(FaultCode) && MI_IS_PAGE_COPY_ON_WRITE(PointerPte)) == FALSE);
 
        /* Check if this was a write */
        if (MI_IS_WRITE_ACCESS(FaultCode))
        {
            /* Was it to a read-only page? */
            Pfn1 = MI_PFN_ELEMENT(PointerPte->u.Hard.PageFrameNumber);
            if (!(PointerPte->u.Long & PTE_READWRITE) &&
                !(Pfn1->OriginalPte.u.Soft.Protection & MM_READWRITE))
            {
                /* Crash with distinguished bugcheck code */
                KeBugCheckEx(ATTEMPTED_WRITE_TO_READONLY_MEMORY,
                             (ULONG_PTR)Address,
                             PointerPte->u.Long,
                             (ULONG_PTR)TrapInformation,
                             10);
            }
        }
 
        /* Nothing is actually wrong */
        DPRINT1("Fault at IRQL %u is ok (%p)\n", OldIrql, Address);
        return STATUS_SUCCESS;
    }
 
    /* Check for kernel fault address */
    if (Address >= MmSystemRangeStart)
    {
        /* Bail out, if the fault came from user mode */
        if (Mode == UserMode) return STATUS_ACCESS_VIOLATION;
 
#if (_MI_PAGING_LEVELS == 2)
        if (MI_IS_SYSTEM_PAGE_TABLE_ADDRESS(Address)) MiSynchronizeSystemPde((PMMPDE)PointerPte);
        MiCheckPdeForPagedPool(Address);
#endif
 
        /* Check if the higher page table entries are invalid */
        if (
#if (_MI_PAGING_LEVELS == 4)
            /* AMD64 system, check if PXE is invalid */
            (PointerPxe->u.Hard.Valid == 0) ||
#endif
#if (_MI_PAGING_LEVELS >= 3)
            /* PAE/AMD64 system, check if PPE is invalid */
            (PointerPpe->u.Hard.Valid == 0) ||
#endif
            /* Always check if the PDE is valid */
            (PointerPde->u.Hard.Valid == 0))
        {
            /* PXE/PPE/PDE (still) not valid, kill the system */
            KeBugCheckEx(PAGE_FAULT_IN_NONPAGED_AREA,
                         (ULONG_PTR)Address,
                         FaultCode,
                         (ULONG_PTR)TrapInformation,
                         2);
        }
 
        /* Not handling session faults yet */
        IsSessionAddress = MI_IS_SESSION_ADDRESS(Address);
 
        /* The PDE is valid, so read the PTE */
        TempPte = *PointerPte;
        if (TempPte.u.Hard.Valid == 1)
        {
            /* Check if this was system space or session space */
            if (!IsSessionAddress)
            {
                /* Check if the PTE is still valid under PFN lock */
                OldIrql = MiAcquirePfnLock();
                TempPte = *PointerPte;
                if (TempPte.u.Hard.Valid)
                {
                    /* Check if this was a write */
                    if (MI_IS_WRITE_ACCESS(FaultCode))
                    {
                        /* Was it to a read-only page? */
                        Pfn1 = MI_PFN_ELEMENT(PointerPte->u.Hard.PageFrameNumber);
                        if (!(PointerPte->u.Long & PTE_READWRITE) &&
                            !(Pfn1->OriginalPte.u.Soft.Protection & MM_READWRITE))
                        {
                            /* Crash with distinguished bugcheck code */
                            KeBugCheckEx(ATTEMPTED_WRITE_TO_READONLY_MEMORY,
                                         (ULONG_PTR)Address,
                                         PointerPte->u.Long,
                                         (ULONG_PTR)TrapInformation,
                                         11);
                        }
                    }
 
                    /* Check for execution of non-executable memory */
                    if (MI_IS_INSTRUCTION_FETCH(FaultCode) &&
                        !MI_IS_PAGE_EXECUTABLE(&TempPte))
                    {
                        KeBugCheckEx(ATTEMPTED_EXECUTE_OF_NOEXECUTE_MEMORY,
                                     (ULONG_PTR)Address,
                                     (ULONG_PTR)TempPte.u.Long,
                                     (ULONG_PTR)TrapInformation,
                                     1);
                    }
                }
 
                /* Release PFN lock and return all good */
                MiReleasePfnLock(OldIrql);
                return STATUS_SUCCESS;
            }
        }
#if (_MI_PAGING_LEVELS == 2)
        /* Check if this was a session PTE that needs to remap the session PDE */
        if (MI_IS_SESSION_PTE(Address))
        {
            /* Do the remapping */
            Status = MiCheckPdeForSessionSpace(Address);
            if (!NT_SUCCESS(Status))
            {
                /* It failed, this address is invalid */
                KeBugCheckEx(PAGE_FAULT_IN_NONPAGED_AREA,
                             (ULONG_PTR)Address,
                             FaultCode,
                             (ULONG_PTR)TrapInformation,
                             6);
            }
        }
#else
 
_WARN("Session space stuff is not implemented yet!")
 
#endif
 
        /* Check for a fault on the page table or hyperspace */
        if (MI_IS_PAGE_TABLE_OR_HYPER_ADDRESS(Address))
        {
#if (_MI_PAGING_LEVELS < 3)
            /* Windows does this check but I don't understand why -- it's done above! */
            ASSERT(MiCheckPdeForPagedPool(Address) != STATUS_WAIT_1);
#endif
            /* Handle this as a user mode fault */
            goto UserFault;
        }
 
        /* Get the current thread */
        CurrentThread = PsGetCurrentThread();
 
        /* What kind of address is this */
        if (!IsSessionAddress)
        {
            /* Use the system working set */
            WorkingSet = &MmSystemCacheWs;
            CurrentProcess = NULL;
 
            /* Make sure we don't have a recursive working set lock */
            if ((CurrentThread->OwnsProcessWorkingSetExclusive) ||
                (CurrentThread->OwnsProcessWorkingSetShared) ||
                (CurrentThread->OwnsSystemWorkingSetExclusive) ||
                (CurrentThread->OwnsSystemWorkingSetShared) ||
                (CurrentThread->OwnsSessionWorkingSetExclusive) ||
                (CurrentThread->OwnsSessionWorkingSetShared))
            {
                /* Fail */
                return STATUS_IN_PAGE_ERROR | 0x10000000;
            }
        }
        else
        {
            /* Use the session process and working set */
            CurrentProcess = HYDRA_PROCESS;
            WorkingSet = &MmSessionSpace->GlobalVirtualAddress->Vm;
 
            /* Make sure we don't have a recursive working set lock */
            if ((CurrentThread->OwnsSessionWorkingSetExclusive) ||
                (CurrentThread->OwnsSessionWorkingSetShared))
            {
                /* Fail */
                return STATUS_IN_PAGE_ERROR | 0x10000000;
            }
        }
RetryKernel:
        /* Acquire the working set lock */
        KeRaiseIrql(APC_LEVEL, &LockIrql);
        MiLockWorkingSet(CurrentThread, WorkingSet);
 
        /* Re-read PTE now that we own the lock */
        TempPte = *PointerPte;
        if (TempPte.u.Hard.Valid == 1)
        {
            /* Check if this was a write */
            if (MI_IS_WRITE_ACCESS(FaultCode))
            {
                /* Was it to a read-only page that is not copy on write? */
                Pfn1 = MI_PFN_ELEMENT(PointerPte->u.Hard.PageFrameNumber);
                if (!(TempPte.u.Long & PTE_READWRITE) &&
                    !(Pfn1->OriginalPte.u.Soft.Protection & MM_READWRITE) &&
                    !MI_IS_PAGE_COPY_ON_WRITE(&TempPte))
                {
                    /* Case not yet handled */
                    ASSERT(!IsSessionAddress);
 
                    /* Crash with distinguished bugcheck code */
                    KeBugCheckEx(ATTEMPTED_WRITE_TO_READONLY_MEMORY,
                                 (ULONG_PTR)Address,
                                 TempPte.u.Long,
                                 (ULONG_PTR)TrapInformation,
                                 12);
                }
            }
 
            /* Check for execution of non-executable memory */
            if (MI_IS_INSTRUCTION_FETCH(FaultCode) &&
                !MI_IS_PAGE_EXECUTABLE(&TempPte))
            {
                KeBugCheckEx(ATTEMPTED_EXECUTE_OF_NOEXECUTE_MEMORY,
                             (ULONG_PTR)Address,
                             (ULONG_PTR)TempPte.u.Long,
                             (ULONG_PTR)TrapInformation,
                             2);
            }
 
            /* Check for read-only write in session space */
            if ((IsSessionAddress) &&
                MI_IS_WRITE_ACCESS(FaultCode) &&
                !MI_IS_PAGE_WRITEABLE(&TempPte))
            {
                /* Sanity check */
                ASSERT(MI_IS_SESSION_IMAGE_ADDRESS(Address));
 
                /* Was this COW? */
                if (!MI_IS_PAGE_COPY_ON_WRITE(&TempPte))
                {
                    /* Then this is not allowed */
                    KeBugCheckEx(ATTEMPTED_WRITE_TO_READONLY_MEMORY,
                                 (ULONG_PTR)Address,
                                 (ULONG_PTR)TempPte.u.Long,
                                 (ULONG_PTR)TrapInformation,
                                 13);
                }
 
                /* Otherwise, handle COW */
                ASSERT(FALSE);
            }
 
            /* Release the working set */
            MiUnlockWorkingSet(CurrentThread, WorkingSet);
            KeLowerIrql(LockIrql);
 
            /* Otherwise, the PDE was probably invalid, and all is good now */
            return STATUS_SUCCESS;
        }
 
        /* Check one kind of prototype PTE */
        if (TempPte.u.Soft.Prototype)
        {
            /* Make sure protected pool is on, and that this is a pool address */
            if ((MmProtectFreedNonPagedPool) &&
                (((Address >= MmNonPagedPoolStart) &&
                  (Address < (PVOID)((ULONG_PTR)MmNonPagedPoolStart +
                                     MmSizeOfNonPagedPoolInBytes))) ||
                 ((Address >= MmNonPagedPoolExpansionStart) &&
                  (Address < MmNonPagedPoolEnd))))
            {
                /* Bad boy, bad boy, whatcha gonna do, whatcha gonna do when ARM3 comes for you! */
                KeBugCheckEx(DRIVER_CAUGHT_MODIFYING_FREED_POOL,
                             (ULONG_PTR)Address,
                             FaultCode,
                             Mode,
                             4);
            }
 
            /* Get the prototype PTE! */
            ProtoPte = MiProtoPteToPte(&TempPte);
 
            /* Do we need to locate the prototype PTE in session space? */
            if ((IsSessionAddress) &&
                (TempPte.u.Soft.PageFileHigh == MI_PTE_LOOKUP_NEEDED))
            {
                /* Yep, go find it as well as the VAD for it */
                ProtoPte = MiCheckVirtualAddress(Address,
                                                 &ProtectionCode,
                                                 &Vad);
                ASSERT(ProtoPte != NULL);
            }
        }
        else
        {
            /* We don't implement transition PTEs */
            ASSERT(TempPte.u.Soft.Transition == 0);
 
            /* Check for no-access PTE */
            if (TempPte.u.Soft.Protection == MM_NOACCESS)
            {
                /* Bugcheck the system! */
                KeBugCheckEx(PAGE_FAULT_IN_NONPAGED_AREA,
                             (ULONG_PTR)Address,
                             FaultCode,
                             (ULONG_PTR)TrapInformation,
                             1);
            }
 
            /* Check for no protecton at all */
            if (TempPte.u.Soft.Protection == MM_ZERO_ACCESS)
            {
                /* Bugcheck the system! */
                KeBugCheckEx(PAGE_FAULT_IN_NONPAGED_AREA,
                             (ULONG_PTR)Address,
                             FaultCode,
                             (ULONG_PTR)TrapInformation,
                             0);
            }
        }
 
        /* Check for demand page */
        if (MI_IS_WRITE_ACCESS(FaultCode) &&
            !(ProtoPte) &&
            !(IsSessionAddress) &&
            !(TempPte.u.Hard.Valid))
        {
            /* Get the protection code */
            ASSERT(TempPte.u.Soft.Transition == 0);
            if (!(TempPte.u.Soft.Protection & MM_READWRITE))
            {
                /* Bugcheck the system! */
                KeBugCheckEx(ATTEMPTED_WRITE_TO_READONLY_MEMORY,
                             (ULONG_PTR)Address,
                             TempPte.u.Long,
                             (ULONG_PTR)TrapInformation,
                             14);
            }
        }
 
        /* Now do the real fault handling */
        Status = MiDispatchFault(FaultCode,
                                 Address,
                                 PointerPte,
                                 ProtoPte,
                                 FALSE,
                                 CurrentProcess,
                                 TrapInformation,
                                 NULL);
 
        /* Release the working set */
        ASSERT(KeAreAllApcsDisabled() == TRUE);
        MiUnlockWorkingSet(CurrentThread, WorkingSet);
        KeLowerIrql(LockIrql);
 
        if (Status == STATUS_NO_MEMORY)
        {
            MmRebalanceMemoryConsumersAndWait();
            goto RetryKernel;
        }
 
        /* We are done! */
        DPRINT("Fault resolved with status: %lx\n", Status);
        return Status;
    }
 
    /* This is a user fault */
UserFault:
    CurrentThread = PsGetCurrentThread();
    CurrentProcess = (PEPROCESS)CurrentThread->Tcb.ApcState.Process;
 
    /* Lock the working set */
    MiLockProcessWorkingSet(CurrentProcess, CurrentThread);
 
    ProtectionCode = MM_INVALID_PROTECTION;
 
#if (_MI_PAGING_LEVELS == 4)
    /* Check if the PXE is valid */
    if (PointerPxe->u.Hard.Valid == 0)
    {
        /* Right now, we only handle scenarios where the PXE is totally empty */
        ASSERT(PointerPxe->u.Long == 0);
 
        /* This is only possible for user mode addresses! */
        ASSERT(PointerPte <= MiHighestUserPte);
 
        /* Check if we have a VAD */
        MiCheckVirtualAddress(Address, &ProtectionCode, &Vad);
        if (ProtectionCode == MM_NOACCESS)
        {
            MiUnlockProcessWorkingSet(CurrentProcess, CurrentThread);
            return STATUS_ACCESS_VIOLATION;
        }
 
        /* Resolve a demand zero fault */
        Status = MiResolveDemandZeroFault(PointerPpe,
                                 PointerPxe,
                                 MM_EXECUTE_READWRITE,
                                 CurrentProcess,
                                 MM_NOIRQL);
        if (!NT_SUCCESS(Status))
        {
            goto ExitUser;
        }
 
        /* We should come back with a valid PXE */
        ASSERT(PointerPxe->u.Hard.Valid == 1);
    }
#endif
 
#if (_MI_PAGING_LEVELS >= 3)
    /* Check if the PPE is valid */
    if (PointerPpe->u.Hard.Valid == 0)
    {
        /* Right now, we only handle scenarios where the PPE is totally empty */
        ASSERT(PointerPpe->u.Long == 0);
 
        /* This is only possible for user mode addresses! */
        ASSERT(PointerPte <= MiHighestUserPte);
 
        /* Check if we have a VAD, unless we did this already */
        if (ProtectionCode == MM_INVALID_PROTECTION)
        {
            MiCheckVirtualAddress(Address, &ProtectionCode, &Vad);
        }
 
        if (ProtectionCode == MM_NOACCESS)
        {
            MiUnlockProcessWorkingSet(CurrentProcess, CurrentThread);
            return STATUS_ACCESS_VIOLATION;
        }
 
        /* Resolve a demand zero fault */
        Status = MiResolveDemandZeroFault(PointerPde,
                                 PointerPpe,
                                 MM_EXECUTE_READWRITE,
                                 CurrentProcess,
                                 MM_NOIRQL);
        if (!NT_SUCCESS(Status))
        {
            goto ExitUser;
        }
 
        /* We should come back with a valid PPE */
        ASSERT(PointerPpe->u.Hard.Valid == 1);
        MiIncrementPageTableReferences(PointerPde);
    }
#endif
 
    /* Check if the PDE is invalid */
    if (PointerPde->u.Hard.Valid == 0)
    {
        /* Right now, we only handle scenarios where the PDE is totally empty */
        ASSERT(PointerPde->u.Long == 0);
 
        /* And go dispatch the fault on the PDE. This should handle the demand-zero */
#if MI_TRACE_PFNS
        UserPdeFault = TRUE;
#endif
        /* Check if we have a VAD, unless we did this already */
        if (ProtectionCode == MM_INVALID_PROTECTION)
        {
            MiCheckVirtualAddress(Address, &ProtectionCode, &Vad);
        }
 
        if (ProtectionCode == MM_NOACCESS)
        {
#if (_MI_PAGING_LEVELS == 2)
            /* Could be a page table for paged pool */
            MiCheckPdeForPagedPool(Address);
#endif
            /* Has the code above changed anything -- is this now a valid PTE? */
            Status = (PointerPde->u.Hard.Valid == 1) ? STATUS_SUCCESS : STATUS_ACCESS_VIOLATION;
 
            /* Either this was a bogus VA or we've fixed up a paged pool PDE */
            MiUnlockProcessWorkingSet(CurrentProcess, CurrentThread);
            return Status;
        }
 
        /* Resolve a demand zero fault */
        Status = MiResolveDemandZeroFault(PointerPte,
                                 PointerPde,
                                 MM_EXECUTE_READWRITE,
                                 CurrentProcess,
                                 MM_NOIRQL);
        if (!NT_SUCCESS(Status))
        {
            goto ExitUser;
        }
 
#if _MI_PAGING_LEVELS >= 3
        MiIncrementPageTableReferences(PointerPte);
#endif
 
#if MI_TRACE_PFNS
        UserPdeFault = FALSE;
        /* Update debug info */
        if (TrapInformation)
            MiGetPfnEntry(PointerPde->u.Hard.PageFrameNumber)->CallSite = (PVOID)((PKTRAP_FRAME)TrapInformation)->Eip;
        else
            MiGetPfnEntry(PointerPde->u.Hard.PageFrameNumber)->CallSite = _ReturnAddress();
#endif
        /* We should come back with APCs enabled, and with a valid PDE */
        ASSERT(KeAreAllApcsDisabled() == TRUE);
        ASSERT(PointerPde->u.Hard.Valid == 1);
    }
    else
    {
        /* Not yet implemented in ReactOS */
        ASSERT(MI_IS_PAGE_LARGE(PointerPde) == FALSE);
    }
 
    /* Now capture the PTE. */
    TempPte = *PointerPte;
 
    /* Check if the PTE is valid */
    if (TempPte.u.Hard.Valid)
    {
        /* Check if this is a write on a readonly PTE */
        if (MI_IS_WRITE_ACCESS(FaultCode))
        {
            /* Is this a copy on write PTE? */
            if (MI_IS_PAGE_COPY_ON_WRITE(&TempPte))
            {
                PFN_NUMBER PageFrameIndex, OldPageFrameIndex;
                PMMPFN Pfn1;
 
                LockIrql = MiAcquirePfnLock();
 
                ASSERT(MmAvailablePages > 0);
 
                MI_SET_USAGE(MI_USAGE_COW);
                MI_SET_PROCESS(CurrentProcess);
 
                /* Allocate a new page and copy it */
                PageFrameIndex = MiRemoveAnyPage(MI_GET_NEXT_PROCESS_COLOR(CurrentProcess));
                if (PageFrameIndex == 0)
                {
                    MiReleasePfnLock(LockIrql);
                    Status = STATUS_NO_MEMORY;
                    goto ExitUser;
                }
                OldPageFrameIndex = PFN_FROM_PTE(&TempPte);
 
                MiCopyPfn(PageFrameIndex, OldPageFrameIndex);
 
                /* Dereference whatever this PTE is referencing */
                Pfn1 = MI_PFN_ELEMENT(OldPageFrameIndex);
                ASSERT(Pfn1->u3.e1.PrototypePte == 1);
                ASSERT(!MI_IS_PFN_DELETED(Pfn1));
                ProtoPte = Pfn1->PteAddress;
                MiDeletePte(PointerPte, Address, CurrentProcess, ProtoPte);
 
                /* And make a new shiny one with our page */
                MiInitializePfn(PageFrameIndex, PointerPte, TRUE);
                TempPte.u.Hard.PageFrameNumber = PageFrameIndex;
                TempPte.u.Hard.Write = 1;
                TempPte.u.Hard.CopyOnWrite = 0;
 
                MI_WRITE_VALID_PTE(PointerPte, TempPte);
 
                MiReleasePfnLock(LockIrql);
 
                /* Return the status */
                MiUnlockProcessWorkingSet(CurrentProcess, CurrentThread);
                return STATUS_PAGE_FAULT_COPY_ON_WRITE;
            }
 
            /* Is this a read-only PTE? */
            if (!MI_IS_PAGE_WRITEABLE(&TempPte))
            {
                /* Return the status */
                MiUnlockProcessWorkingSet(CurrentProcess, CurrentThread);
                return STATUS_ACCESS_VIOLATION;
            }
        }
 
#if _MI_HAS_NO_EXECUTE
        /* Check for execution of non-executable memory */
        if (MI_IS_INSTRUCTION_FETCH(FaultCode) &&
            !MI_IS_PAGE_EXECUTABLE(&TempPte))
        {
            /* Check if execute enable was set */
            if (CurrentProcess->Pcb.Flags.ExecuteEnable)
            {
                /* Fix up the PTE to be executable */
                TempPte.u.Hard.NoExecute = 0;
                MI_UPDATE_VALID_PTE(PointerPte, TempPte);
                MiUnlockProcessWorkingSet(CurrentProcess, CurrentThread);
                return STATUS_SUCCESS;
            }
 
            /* Return the status */
            MiUnlockProcessWorkingSet(CurrentProcess, CurrentThread);
            return STATUS_ACCESS_VIOLATION;
        }
#endif
 
        /* The fault has already been resolved by a different thread */
        MiUnlockProcessWorkingSet(CurrentProcess, CurrentThread);
        return STATUS_SUCCESS;
    }
 
    /* Quick check for demand-zero */
    if ((TempPte.u.Long == (MM_READWRITE << MM_PTE_SOFTWARE_PROTECTION_BITS)) ||
        (TempPte.u.Long == (MM_EXECUTE_READWRITE << MM_PTE_SOFTWARE_PROTECTION_BITS)))
    {
        /* Resolve the fault */
        Status = MiResolveDemandZeroFault(Address,
                                 PointerPte,
                                 TempPte.u.Soft.Protection,
                                 CurrentProcess,
                                 MM_NOIRQL);
        if (!NT_SUCCESS(Status))
        {
            goto ExitUser;
        }
 
#if MI_TRACE_PFNS
        /* Update debug info */
        if (TrapInformation)
            MiGetPfnEntry(PointerPte->u.Hard.PageFrameNumber)->CallSite = (PVOID)((PKTRAP_FRAME)TrapInformation)->Eip;
        else
            MiGetPfnEntry(PointerPte->u.Hard.PageFrameNumber)->CallSite = _ReturnAddress();
#endif
 
        /* Return the status */
        MiUnlockProcessWorkingSet(CurrentProcess, CurrentThread);
        return STATUS_PAGE_FAULT_DEMAND_ZERO;
    }
 
    /* Check for zero PTE */
    if (TempPte.u.Long == 0)
    {
        /* Check if this address range belongs to a valid allocation (VAD) */
        ProtoPte = MiCheckVirtualAddress(Address, &ProtectionCode, &Vad);
        if (ProtectionCode == MM_NOACCESS)
        {
#if (_MI_PAGING_LEVELS == 2)
            /* Could be a page table for paged pool */
            MiCheckPdeForPagedPool(Address);
#endif
            /* Has the code above changed anything -- is this now a valid PTE? */
            Status = (PointerPte->u.Hard.Valid == 1) ? STATUS_SUCCESS : STATUS_ACCESS_VIOLATION;
 
            /* Either this was a bogus VA or we've fixed up a paged pool PDE */
            MiUnlockProcessWorkingSet(CurrentProcess, CurrentThread);
            return Status;
        }
 
        /*
         * Check if this is a real user-mode address or actually a kernel-mode
         * page table for a user mode address
         */
        if (Address <= MM_HIGHEST_USER_ADDRESS
#if _MI_PAGING_LEVELS >= 3
            || MiIsUserPte(Address)
#if _MI_PAGING_LEVELS == 4
            || MiIsUserPde(Address)
#endif
#endif
        )
        {
            /* Add an additional page table reference */
            MiIncrementPageTableReferences(Address);
        }
 
        /* Is this a guard page? */
        if ((ProtectionCode & MM_PROTECT_SPECIAL) == MM_GUARDPAGE)
        {
            /* The VAD protection cannot be MM_DECOMMIT! */
            ASSERT(ProtectionCode != MM_DECOMMIT);
 
            /* Remove the bit */
            TempPte.u.Soft.Protection = ProtectionCode & ~MM_GUARDPAGE;
            MI_WRITE_INVALID_PTE(PointerPte, TempPte);
 
            /* Not supported */
            ASSERT(ProtoPte == NULL);
            ASSERT(CurrentThread->ApcNeeded == 0);
 
            /* Drop the working set lock */
            MiUnlockProcessWorkingSet(CurrentProcess, CurrentThread);
            ASSERT(KeGetCurrentIrql() == OldIrql);
 
            /* Handle stack expansion */
            return MiCheckForUserStackOverflow(Address, TrapInformation);
        }
 
        /* Did we get a prototype PTE back? */
        if (!ProtoPte)
        {
            /* Is this PTE actually part of the PDE-PTE self-mapping directory? */
            if (PointerPde == MiAddressToPde(PTE_BASE))
            {
                /* Then it's really a demand-zero PDE (on behalf of user-mode) */
#ifdef _M_ARM
                _WARN("This is probably completely broken!");
                MI_WRITE_INVALID_PDE((PMMPDE)PointerPte, DemandZeroPde);
#else
                MI_WRITE_INVALID_PDE(PointerPte, DemandZeroPde);
#endif
            }
            else
            {
                /* No, create a new PTE. First, write the protection */
                TempPte.u.Soft.Protection = ProtectionCode;
                MI_WRITE_INVALID_PTE(PointerPte, TempPte);
            }
 
            /* Lock the PFN database since we're going to grab a page */
            OldIrql = MiAcquirePfnLock();
 
            /* Make sure we have enough pages */
            //ASSERT(MmAvailablePages >= 32);
 
            /* Try to get a zero page */
            MI_SET_USAGE(MI_USAGE_PEB_TEB);
            MI_SET_PROCESS2(CurrentProcess->ImageFileName);
            Color = MI_GET_NEXT_PROCESS_COLOR(CurrentProcess);
            PageFrameIndex = MiRemoveZeroPageSafe(Color);
            if (!PageFrameIndex)
            {
                /* Grab a page out of there. Later we should grab a colored zero page */
                PageFrameIndex = MiRemoveAnyPage(Color);
 
                /* Release the lock since we need to do some zeroing */
                MiReleasePfnLock(OldIrql);
 
                if (PageFrameIndex == 0)
                {
                    Status = STATUS_NO_MEMORY;
                    goto ExitUser;
                }
 
                /* Zero out the page, since it's for user-mode */
                MiZeroPfn(PageFrameIndex);
 
                /* Grab the lock again so we can initialize the PFN entry */
                OldIrql = MiAcquirePfnLock();
            }
 
            /* Initialize the PFN entry now */
            MiInitializePfn(PageFrameIndex, PointerPte, 1);
 
            /* Increment the count of pages in the process */
            CurrentProcess->NumberOfPrivatePages++;
 
            /* One more demand-zero fault */
            KeGetCurrentPrcb()->MmDemandZeroCount++;
 
            /* And we're done with the lock */
            MiReleasePfnLock(OldIrql);
 
            /* Fault on user PDE, or fault on user PTE? */
            if (PointerPte <= MiHighestUserPte)
            {
                /* User fault, build a user PTE */
                MI_MAKE_HARDWARE_PTE_USER(&TempPte,
                                          PointerPte,
                                          PointerPte->u.Soft.Protection,
                                          PageFrameIndex);
            }
            else
            {
                /* This is a user-mode PDE, create a kernel PTE for it */
                MI_MAKE_HARDWARE_PTE(&TempPte,
                                     PointerPte,
                                     PointerPte->u.Soft.Protection,
                                     PageFrameIndex);
            }
 
            /* Write the dirty bit for writeable pages */
            if (MI_IS_PAGE_WRITEABLE(&TempPte)) MI_MAKE_DIRTY_PAGE(&TempPte);
 
            /* And now write down the PTE, making the address valid */
            MI_WRITE_VALID_PTE(PointerPte, TempPte);
            Pfn1 = MI_PFN_ELEMENT(PageFrameIndex);
            ASSERT(Pfn1->u1.Event == NULL);
 
            /* Demand zero */
            ASSERT(KeGetCurrentIrql() <= APC_LEVEL);
            MiUnlockProcessWorkingSet(CurrentProcess, CurrentThread);
            return STATUS_PAGE_FAULT_DEMAND_ZERO;
        }
 
        /* We should have a valid protection here */
        ASSERT(ProtectionCode != 0x100);
 
        /* Write the prototype PTE */
        TempPte = PrototypePte;
        TempPte.u.Soft.Protection = ProtectionCode;
        ASSERT(TempPte.u.Long != 0);
        MI_WRITE_INVALID_PTE(PointerPte, TempPte);
    }
    else
    {
        /* Get the protection code and check if this is a proto PTE */
        ProtectionCode = (ULONG)TempPte.u.Soft.Protection;
        if (TempPte.u.Soft.Prototype)
        {
            /* Do we need to go find the real PTE? */
            if (TempPte.u.Soft.PageFileHigh == MI_PTE_LOOKUP_NEEDED)
            {
                /* Get the prototype pte and VAD for it */
                ProtoPte = MiCheckVirtualAddress(Address,
                                                 &ProtectionCode,
                                                 &Vad);
                if (!ProtoPte)
                {
                    ASSERT(KeGetCurrentIrql() <= APC_LEVEL);
                    MiUnlockProcessWorkingSet(CurrentProcess, CurrentThread);
                    return STATUS_ACCESS_VIOLATION;
                }
            }
            else
            {
                /* Get the prototype PTE! */
                ProtoPte = MiProtoPteToPte(&TempPte);
 
                /* Is it read-only */
                if (TempPte.u.Proto.ReadOnly)
                {
                    /* Set read-only code */
                    ProtectionCode = MM_READONLY;
                }
                else
                {
                    /* Set unknown protection */
                    ProtectionCode = 0x100;
                    ASSERT(CurrentProcess->CloneRoot != NULL);
                }
            }
        }
    }
 
    /* Do we have a valid protection code? */
    if (ProtectionCode != 0x100)
    {
        /* Run a software access check first, including to detect guard pages */
        Status = MiAccessCheck(PointerPte,
                               !MI_IS_NOT_PRESENT_FAULT(FaultCode),
                               Mode,
                               ProtectionCode,
                               TrapInformation,
                               FALSE);
        if (Status != STATUS_SUCCESS)
        {
            /* Not supported */
            ASSERT(CurrentThread->ApcNeeded == 0);
 
            /* Drop the working set lock */
            MiUnlockProcessWorkingSet(CurrentProcess, CurrentThread);
            ASSERT(KeGetCurrentIrql() == OldIrql);
 
            /* Did we hit a guard page? */
            if (Status == STATUS_GUARD_PAGE_VIOLATION)
            {
                /* Handle stack expansion */
                return MiCheckForUserStackOverflow(Address, TrapInformation);
            }
 
            /* Otherwise, fail back to the caller directly */
            return Status;
        }
    }
 
    /* Dispatch the fault */
    Status = MiDispatchFault(FaultCode,
                             Address,
                             PointerPte,
                             ProtoPte,
                             FALSE,
                             CurrentProcess,
                             TrapInformation,
                             Vad);
 
ExitUser:
 
    /* Return the status */
    ASSERT(KeGetCurrentIrql() <= APC_LEVEL);
    MiUnlockProcessWorkingSet(CurrentProcess, CurrentThread);
 
    if (Status == STATUS_NO_MEMORY)
    {
        MmRebalanceMemoryConsumersAndWait();
        goto UserFault;
    }
 
    return Status;
}

Referenced by MmAccessFault().

◆ MmArmInitSystem()

BOOLEAN NTAPI MmArmInitSystem	(	IN ULONG	Phase,
		IN PLOADER_PARAMETER_BLOCK	LoaderBlock
	)

Definition at line 2037 of file mminit.c.

{
    ULONG i;
    BOOLEAN IncludeType[LoaderMaximum];
    PVOID Bitmap;
    PPHYSICAL_MEMORY_RUN Run;
    PFN_NUMBER PageCount;
#if DBG
    ULONG j;
    PMMPTE PointerPte, TestPte;
    MMPTE TempPte;
#endif
 
    /* Dump memory descriptors */
    if (MiDbgEnableMdDump) MiDbgDumpMemoryDescriptors();
 
    //
    // Instantiate memory that we don't consider RAM/usable
    // We use the same exclusions that Windows does, in order to try to be
    // compatible with WinLDR-style booting
    //
    for (i = 0; i < LoaderMaximum; i++) IncludeType[i] = TRUE;
    IncludeType[LoaderBad] = FALSE;
    IncludeType[LoaderFirmwarePermanent] = FALSE;
    IncludeType[LoaderSpecialMemory] = FALSE;
    IncludeType[LoaderBBTMemory] = FALSE;
    if (Phase == 0)
    {
        /* Count physical pages on the system */
        MiScanMemoryDescriptors(LoaderBlock);
 
        /* Initialize the phase 0 temporary event */
        KeInitializeEvent(&MiTempEvent, NotificationEvent, FALSE);
 
        /* Set all the events to use the temporary event for now */
        MiLowMemoryEvent = &MiTempEvent;
        MiHighMemoryEvent = &MiTempEvent;
        MiLowPagedPoolEvent = &MiTempEvent;
        MiHighPagedPoolEvent = &MiTempEvent;
        MiLowNonPagedPoolEvent = &MiTempEvent;
        MiHighNonPagedPoolEvent = &MiTempEvent;
 
        //
        // Default throttling limits for Cc
        // May be ajusted later on depending on system type
        //
        MmThrottleTop = 450;
        MmThrottleBottom = 127;
 
        //
        // Define the basic user vs. kernel address space separation
        //
        MmSystemRangeStart = (PVOID)MI_DEFAULT_SYSTEM_RANGE_START;
        MmUserProbeAddress = (ULONG_PTR)MI_USER_PROBE_ADDRESS;
        MmHighestUserAddress = (PVOID)MI_HIGHEST_USER_ADDRESS;
 
        /* Highest PTE and PDE based on the addresses above */
        MiHighestUserPte = MiAddressToPte(MmHighestUserAddress);
        MiHighestUserPde = MiAddressToPde(MmHighestUserAddress);
#if (_MI_PAGING_LEVELS >= 3)
        MiHighestUserPpe = MiAddressToPpe(MmHighestUserAddress);
#if (_MI_PAGING_LEVELS >= 4)
        MiHighestUserPxe = MiAddressToPxe(MmHighestUserAddress);
#endif
#endif
        //
        // Get the size of the boot loader's image allocations and then round
        // that region up to a PDE size, so that any PDEs we might create for
        // whatever follows are separate from the PDEs that boot loader might've
        // already created (and later, we can blow all that away if we want to).
        //
        MmBootImageSize = KeLoaderBlock->Extension->LoaderPagesSpanned;
        MmBootImageSize *= PAGE_SIZE;
        MmBootImageSize = (MmBootImageSize + PDE_MAPPED_VA - 1) & ~(PDE_MAPPED_VA - 1);
        ASSERT((MmBootImageSize % PDE_MAPPED_VA) == 0);
 
        /* Initialize session space address layout */
        MiInitializeSessionSpaceLayout();
 
        /* Set the based section highest address */
        MmHighSectionBase = (PVOID)((ULONG_PTR)MmHighestUserAddress - 0x800000);
 
        /* Loop all 8 standby lists */
        for (i = 0; i < 8; i++)
        {
            /* Initialize them */
            MmStandbyPageListByPriority[i].Total = 0;
            MmStandbyPageListByPriority[i].ListName = StandbyPageList;
            MmStandbyPageListByPriority[i].Flink = MM_EMPTY_LIST;
            MmStandbyPageListByPriority[i].Blink = MM_EMPTY_LIST;
        }
 
        /* Initialize the user mode image list */
        InitializeListHead(&MmLoadedUserImageList);
 
        /* Initalize the Working set list */
        InitializeListHead(&MmWorkingSetExpansionHead);
 
        /* Initialize critical section timeout value (relative time is negative) */
        MmCriticalSectionTimeout.QuadPart = MmCritsectTimeoutSeconds * (-10000000LL);
 
        /* Initialize the paged pool mutex and the section commit mutex */
        KeInitializeGuardedMutex(&MmPagedPoolMutex);
        KeInitializeGuardedMutex(&MmSectionCommitMutex);
        KeInitializeGuardedMutex(&MmSectionBasedMutex);
 
        /* Initialize the Loader Lock */
        KeInitializeMutant(&MmSystemLoadLock, FALSE);
 
        /* Set up the zero page event */
        KeInitializeEvent(&MmZeroingPageEvent, NotificationEvent, FALSE);
 
        /* Initialize the dead stack S-LIST */
        InitializeSListHead(&MmDeadStackSListHead);
 
        //
        // Check if this is a machine with less than 19MB of RAM
        //
        PageCount = MmNumberOfPhysicalPages;
        if (PageCount < MI_MIN_PAGES_FOR_SYSPTE_TUNING)
        {
            //
            // Use the very minimum of system PTEs
            //
            MmNumberOfSystemPtes = 7000;
        }
        else
        {
            //
            // Use the default
            //
            MmNumberOfSystemPtes = 11000;
            if (PageCount > MI_MIN_PAGES_FOR_SYSPTE_BOOST)
            {
                //
                // Double the amount of system PTEs
                //
                MmNumberOfSystemPtes <<= 1;
            }
            if (PageCount > MI_MIN_PAGES_FOR_SYSPTE_BOOST_BOOST)
            {
                //
                // Double the amount of system PTEs
                //
                MmNumberOfSystemPtes <<= 1;
            }
            if (MmSpecialPoolTag != 0 && MmSpecialPoolTag != -1)
            {
                //
                // Add some extra PTEs for special pool
                //
                MmNumberOfSystemPtes += 0x6000;
            }
        }
 
        DPRINT("System PTE count has been tuned to %lu (%lu bytes)\n",
               MmNumberOfSystemPtes, MmNumberOfSystemPtes * PAGE_SIZE);
 
        /* Check if no values are set for the heap limits */
        if (MmHeapSegmentReserve == 0)
        {
            MmHeapSegmentReserve = 2 * _1MB;
        }
 
        if (MmHeapSegmentCommit == 0)
        {
            MmHeapSegmentCommit = 2 * PAGE_SIZE;
        }
 
        if (MmHeapDeCommitTotalFreeThreshold == 0)
        {
            MmHeapDeCommitTotalFreeThreshold = 64 * _1KB;
        }
 
        if (MmHeapDeCommitFreeBlockThreshold == 0)
        {
            MmHeapDeCommitFreeBlockThreshold = PAGE_SIZE;
        }
 
        /* Initialize the working set lock */
        ExInitializePushLock(&MmSystemCacheWs.WorkingSetMutex);
 
        /* Set commit limit */
        MmTotalCommitLimit = (2 * _1GB) >> PAGE_SHIFT;
        MmTotalCommitLimitMaximum = MmTotalCommitLimit;
 
        /* Has the allocation fragment been setup? */
        if (!MmAllocationFragment)
        {
            /* Use the default value */
            MmAllocationFragment = MI_ALLOCATION_FRAGMENT;
            if (PageCount < ((256 * _1MB) / PAGE_SIZE))
            {
                /* On memory systems with less than 256MB, divide by 4 */
                MmAllocationFragment = MI_ALLOCATION_FRAGMENT / 4;
            }
            else if (PageCount < (_1GB / PAGE_SIZE))
            {
                /* On systems with less than 1GB, divide by 2 */
                MmAllocationFragment = MI_ALLOCATION_FRAGMENT / 2;
            }
        }
        else
        {
            /* Convert from 1KB fragments to pages */
            MmAllocationFragment *= _1KB;
            MmAllocationFragment = ROUND_TO_PAGES(MmAllocationFragment);
 
            /* Don't let it past the maximum */
            MmAllocationFragment = min(MmAllocationFragment,
                                       MI_MAX_ALLOCATION_FRAGMENT);
 
            /* Don't let it too small either */
            MmAllocationFragment = max(MmAllocationFragment,
                                       MI_MIN_ALLOCATION_FRAGMENT);
        }
 
        /* Check for kernel stack size that's too big */
        if (MmLargeStackSize > (KERNEL_LARGE_STACK_SIZE / _1KB))
        {
            /* Sanitize to default value */
            MmLargeStackSize = KERNEL_LARGE_STACK_SIZE;
        }
        else
        {
            /* Take the registry setting, and convert it into bytes */
            MmLargeStackSize *= _1KB;
 
            /* Now align it to a page boundary */
            MmLargeStackSize = PAGE_ROUND_UP(MmLargeStackSize);
 
            /* Sanity checks */
            ASSERT(MmLargeStackSize <= KERNEL_LARGE_STACK_SIZE);
            ASSERT((MmLargeStackSize & (PAGE_SIZE - 1)) == 0);
 
            /* Make sure it's not too low */
            if (MmLargeStackSize < KERNEL_STACK_SIZE) MmLargeStackSize = KERNEL_STACK_SIZE;
        }
 
        /* Compute color information (L2 cache-separated paging lists) */
        MiComputeColorInformation();
 
        // Calculate the number of bytes for the PFN database
        // then add the color tables and convert to pages
        MxPfnAllocation = (MmHighestPhysicalPage + 1) * sizeof(MMPFN);
        MxPfnAllocation += (MmSecondaryColors * sizeof(MMCOLOR_TABLES) * 2);
        MxPfnAllocation >>= PAGE_SHIFT;
 
        // We have to add one to the count here, because in the process of
        // shifting down to the page size, we actually ended up getting the
        // lower aligned size (so say, 0x5FFFF bytes is now 0x5F pages).
        // Later on, we'll shift this number back into bytes, which would cause
        // us to end up with only 0x5F000 bytes -- when we actually want to have
        // 0x60000 bytes.
        MxPfnAllocation++;
 
        /* Initialize the platform-specific parts */
        MiInitMachineDependent(LoaderBlock);
 
#if DBG
        /* Prototype PTEs are assumed to be in paged pool, so check if the math works */
        PointerPte = (PMMPTE)MmPagedPoolStart;
        MI_MAKE_PROTOTYPE_PTE(&TempPte, PointerPte);
        TestPte = MiProtoPteToPte(&TempPte);
        ASSERT(PointerPte == TestPte);
 
        /* Try the last nonpaged pool address */
        PointerPte = (PMMPTE)MI_NONPAGED_POOL_END;
        MI_MAKE_PROTOTYPE_PTE(&TempPte, PointerPte);
        TestPte = MiProtoPteToPte(&TempPte);
        ASSERT(PointerPte == TestPte);
 
        /* Try a bunch of random addresses near the end of the address space */
        PointerPte = (PMMPTE)((ULONG_PTR)MI_HIGHEST_SYSTEM_ADDRESS - 0x37FFF);
        for (j = 0; j < 20; j += 1)
        {
            MI_MAKE_PROTOTYPE_PTE(&TempPte, PointerPte);
            TestPte = MiProtoPteToPte(&TempPte);
            ASSERT(PointerPte == TestPte);
            PointerPte++;
        }
 
        /* Subsection PTEs are always in nonpaged pool, pick a random address to try */
        PointerPte = (PMMPTE)((ULONG_PTR)MmNonPagedPoolStart + (MmSizeOfNonPagedPoolInBytes / 2));
        MI_MAKE_SUBSECTION_PTE(&TempPte, PointerPte);
        TestPte = MiSubsectionPteToSubsection(&TempPte);
        ASSERT(PointerPte == TestPte);
#endif
 
        //
        // Build the physical memory block
        //
        MmPhysicalMemoryBlock = MmInitializeMemoryLimits(LoaderBlock,
                                                         IncludeType);
 
        //
        // Allocate enough buffer for the PFN bitmap
        // Align it up to a 32-bit boundary
        //
        Bitmap = ExAllocatePoolWithTag(NonPagedPool,
                                       (((MmHighestPhysicalPage + 1) + 31) / 32) * 4,
                                       TAG_MM);
        if (!Bitmap)
        {
            //
            // This is critical
            //
            KeBugCheckEx(INSTALL_MORE_MEMORY,
                         MmNumberOfPhysicalPages,
                         MmLowestPhysicalPage,
                         MmHighestPhysicalPage,
                         0x101);
        }
 
        //
        // Initialize it and clear all the bits to begin with
        //
        RtlInitializeBitMap(&MiPfnBitMap,
                            Bitmap,
                            (ULONG)MmHighestPhysicalPage + 1);
        RtlClearAllBits(&MiPfnBitMap);
 
        //
        // Loop physical memory runs
        //
        for (i = 0; i < MmPhysicalMemoryBlock->NumberOfRuns; i++)
        {
            //
            // Get the run
            //
            Run = &MmPhysicalMemoryBlock->Run[i];
            DPRINT("PHYSICAL RAM [0x%08p to 0x%08p]\n",
                   Run->BasePage << PAGE_SHIFT,
                   (Run->BasePage + Run->PageCount) << PAGE_SHIFT);
 
            //
            // Make sure it has pages inside it
            //
            if (Run->PageCount)
            {
                //
                // Set the bits in the PFN bitmap
                //
                RtlSetBits(&MiPfnBitMap, (ULONG)Run->BasePage, (ULONG)Run->PageCount);
            }
        }
 
        /* Look for large page cache entries that need caching */
        MiSyncCachedRanges();
 
        /* Loop for HAL Heap I/O device mappings that need coherency tracking */
        MiAddHalIoMappings();
 
        /* Set the initial resident page count */
        MmResidentAvailablePages = MmAvailablePages - 32;
 
        /* Initialize large page structures on PAE/x64, and MmProcessList on x86 */
        MiInitializeLargePageSupport();
 
        /* Check if the registry says any drivers should be loaded with large pages */
        MiInitializeDriverLargePageList();
 
        /* Relocate the boot drivers into system PTE space and fixup their PFNs */
        MiReloadBootLoadedDrivers(LoaderBlock);
 
        /* FIXME: Call out into Driver Verifier for initialization  */
 
        /* Check how many pages the system has */
        if (MmNumberOfPhysicalPages <= ((13 * _1MB) / PAGE_SIZE))
        {
            /* Set small system */
            MmSystemSize = MmSmallSystem;
            MmMaximumDeadKernelStacks = 0;
        }
        else if (MmNumberOfPhysicalPages <= ((19 * _1MB) / PAGE_SIZE))
        {
            /* Set small system and add 100 pages for the cache */
            MmSystemSize = MmSmallSystem;
            MmSystemCacheWsMinimum += 100;
            MmMaximumDeadKernelStacks = 2;
        }
        else
        {
            /* Set medium system and add 400 pages for the cache */
            MmSystemSize = MmMediumSystem;
            MmSystemCacheWsMinimum += 400;
            MmMaximumDeadKernelStacks = 5;
        }
 
        /* Check for less than 24MB */
        if (MmNumberOfPhysicalPages < ((24 * _1MB) / PAGE_SIZE))
        {
            /* No more than 32 pages */
            MmSystemCacheWsMinimum = 32;
        }
 
        /* Check for more than 32MB */
        if (MmNumberOfPhysicalPages >= ((32 * _1MB) / PAGE_SIZE))
        {
            /* Check for product type being "Wi" for WinNT */
            if (MmProductType == '\0i\0W')
            {
                /* Then this is a large system */
                MmSystemSize = MmLargeSystem;
            }
            else
            {
                /* For servers, we need 64MB to consider this as being large */
                if (MmNumberOfPhysicalPages >= ((64 * _1MB) / PAGE_SIZE))
                {
                    /* Set it as large */
                    MmSystemSize = MmLargeSystem;
                }
            }
        }
 
        /* Check for more than 33 MB */
        if (MmNumberOfPhysicalPages > ((33 * _1MB) / PAGE_SIZE))
        {
            /* Add another 500 pages to the cache */
            MmSystemCacheWsMinimum += 500;
        }
 
        /* Now setup the shared user data fields */
        ASSERT(SharedUserData->NumberOfPhysicalPages == 0);
        SharedUserData->NumberOfPhysicalPages = MmNumberOfPhysicalPages;
        SharedUserData->LargePageMinimum = 0;
 
        /* Check for workstation (Wi for WinNT) */
        if (MmProductType == '\0i\0W')
        {
            /* Set Windows NT Workstation product type */
            SharedUserData->NtProductType = NtProductWinNt;
            MmProductType = 0;
 
            /* For this product, we wait till the last moment to throttle */
            MmThrottleTop = 250;
            MmThrottleBottom = 30;
        }
        else
        {
            /* Check for LanMan server (La for LanmanNT) */
            if (MmProductType == '\0a\0L')
            {
                /* This is a domain controller */
                SharedUserData->NtProductType = NtProductLanManNt;
            }
            else
            {
                /* Otherwise it must be a normal server (Se for ServerNT) */
                SharedUserData->NtProductType = NtProductServer;
            }
 
            /* Set the product type, and make the system more aggressive with low memory */
            MmProductType = 1;
            MmMinimumFreePages = 81;
 
            /* We will throttle earlier to preserve memory */
            MmThrottleTop = 450;
            MmThrottleBottom = 80;
        }
 
        /* Update working set tuning parameters */
        MiAdjustWorkingSetManagerParameters(!MmProductType);
 
        /* Finetune the page count by removing working set and NP expansion */
        MmResidentAvailablePages -= MiExpansionPoolPagesInitialCharge;
        MmResidentAvailablePages -= MmSystemCacheWsMinimum;
        MmResidentAvailableAtInit = MmResidentAvailablePages;
        if (MmResidentAvailablePages <= 0)
        {
            /* This should not happen */
            DPRINT1("System cache working set too big\n");
            return FALSE;
        }
 
        /* Define limits for system cache */
#ifdef _M_AMD64
        MmSizeOfSystemCacheInPages = ((MI_SYSTEM_CACHE_END + 1) - MI_SYSTEM_CACHE_START) / PAGE_SIZE;
#else
        MmSizeOfSystemCacheInPages = ((ULONG_PTR)MI_PAGED_POOL_START - (ULONG_PTR)MI_SYSTEM_CACHE_START) / PAGE_SIZE;
#endif
        MmSystemCacheEnd = (PVOID)((ULONG_PTR)MmSystemCacheStart + (MmSizeOfSystemCacheInPages * PAGE_SIZE) - 1);
#ifdef _M_AMD64
        ASSERT(MmSystemCacheEnd == (PVOID)MI_SYSTEM_CACHE_END);
#else
        ASSERT(MmSystemCacheEnd == (PVOID)((ULONG_PTR)MI_PAGED_POOL_START - 1));
#endif
 
        /* Initialize the system cache */
        //MiInitializeSystemCache(MmSystemCacheWsMinimum, MmAvailablePages);
 
        /* Update the commit limit */
        MmTotalCommitLimit = MmAvailablePages;
        if (MmTotalCommitLimit > 1024) MmTotalCommitLimit -= 1024;
        MmTotalCommitLimitMaximum = MmTotalCommitLimit;
 
        /* Size up paged pool and build the shadow system page directory */
        MiBuildPagedPool();
 
        /* Debugger physical memory support is now ready to be used */
        MmDebugPte = MiAddressToPte(MiDebugMapping);
 
        /* Initialize the loaded module list */
        MiInitializeLoadedModuleList(LoaderBlock);
    }
 
    //
    // Always return success for now
    //
    return TRUE;
}

◆ MmInitializeMemoryLimits()

PPHYSICAL_MEMORY_DESCRIPTOR NTAPI MmInitializeMemoryLimits	(	IN PLOADER_PARAMETER_BLOCK	LoaderBlock,
		IN PBOOLEAN	IncludeType
	)

Definition at line 1613 of file mminit.c.

{
    PLIST_ENTRY NextEntry;
    ULONG Run = 0, InitialRuns;
    PFN_NUMBER NextPage = -1, PageCount = 0;
    PPHYSICAL_MEMORY_DESCRIPTOR Buffer, NewBuffer;
    PMEMORY_ALLOCATION_DESCRIPTOR MdBlock;
 
    //
    // Start with the maximum we might need
    //
    InitialRuns = MiNumberDescriptors;
 
    //
    // Allocate the maximum we'll ever need
    //
    Buffer = ExAllocatePoolWithTag(NonPagedPool,
                                   sizeof(PHYSICAL_MEMORY_DESCRIPTOR) +
                                   sizeof(PHYSICAL_MEMORY_RUN) *
                                   (InitialRuns - 1),
                                   'lMmM');
    if (!Buffer) return NULL;
 
    //
    // For now that's how many runs we have
    //
    Buffer->NumberOfRuns = InitialRuns;
 
    //
    // Now loop through the descriptors again
    //
    NextEntry = LoaderBlock->MemoryDescriptorListHead.Flink;
    while (NextEntry != &LoaderBlock->MemoryDescriptorListHead)
    {
        //
        // Grab each one, and check if it's one we should include
        //
        MdBlock = CONTAINING_RECORD(NextEntry,
                                    MEMORY_ALLOCATION_DESCRIPTOR,
                                    ListEntry);
        if ((MdBlock->MemoryType < LoaderMaximum) &&
            (IncludeType[MdBlock->MemoryType]))
        {
            //
            // Add this to our running total
            //
            PageCount += MdBlock->PageCount;
 
            //
            // Check if the next page is described by the next descriptor
            //
            if (MdBlock->BasePage == NextPage)
            {
                //
                // Combine it into the same physical run
                //
                ASSERT(MdBlock->PageCount != 0);
                Buffer->Run[Run - 1].PageCount += MdBlock->PageCount;
                NextPage += MdBlock->PageCount;
            }
            else
            {
                //
                // Otherwise just duplicate the descriptor's contents
                //
                Buffer->Run[Run].BasePage = MdBlock->BasePage;
                Buffer->Run[Run].PageCount = MdBlock->PageCount;
                NextPage = Buffer->Run[Run].BasePage + Buffer->Run[Run].PageCount;
 
                //
                // And in this case, increase the number of runs
                //
                Run++;
            }
        }
 
        //
        // Try the next descriptor
        //
        NextEntry = MdBlock->ListEntry.Flink;
    }
 
    //
    // We should not have been able to go past our initial estimate
    //
    ASSERT(Run <= Buffer->NumberOfRuns);
 
    //
    // Our guess was probably exaggerated...
    //
    if (InitialRuns > Run)
    {
        //
        // Allocate a more accurately sized buffer
        //
        NewBuffer = ExAllocatePoolWithTag(NonPagedPool,
                                          sizeof(PHYSICAL_MEMORY_DESCRIPTOR) +
                                          sizeof(PHYSICAL_MEMORY_RUN) *
                                          (Run - 1),
                                          'lMmM');
        if (NewBuffer)
        {
            //
            // Copy the old buffer into the new, then free it
            //
            RtlCopyMemory(NewBuffer->Run,
                          Buffer->Run,
                          sizeof(PHYSICAL_MEMORY_RUN) * Run);
            ExFreePoolWithTag(Buffer, 'lMmM');
 
            //
            // Now use the new buffer
            //
            Buffer = NewBuffer;
        }
    }
 
    //
    // Write the final numbers, and return it
    //
    Buffer->NumberOfRuns = Run;
    Buffer->NumberOfPages = PageCount;
    return Buffer;
}

Referenced by MmArmInitSystem().

◆ MxGetNextPage()

PFN_NUMBER NTAPI MxGetNextPage ( IN PFN_NUMBER PageCount )

Definition at line 474 of file mminit.c.

{
    PFN_NUMBER Pfn;
 
    /* Make sure we have enough pages */
    if (PageCount > MxFreeDescriptor->PageCount)
    {
        /* Crash the system */
        KeBugCheckEx(INSTALL_MORE_MEMORY,
                     MmNumberOfPhysicalPages,
                     MxFreeDescriptor->PageCount,
                     MxOldFreeDescriptor.PageCount,
                     PageCount);
    }
 
    /* Use our lowest usable free pages */
    Pfn = MxFreeDescriptor->BasePage;
    MxFreeDescriptor->BasePage += PageCount;
    MxFreeDescriptor->PageCount -= PageCount;
    return Pfn;
}