15 #define MODULE_INVOLVED_IN_ARM3 44 PMMPTE PointerPte, LastPte;
52 LastPte = PointerPte + PageCount;
62 }
while (++PointerPte < LastPte);
93 if (++UnprotectedPages == PageCount)
break;
113 *PoolFlink = *PoolBlink =
NULL;
119 PoolVa = Links->Flink;
123 if (Safe) *PoolFlink = PoolVa;
127 if (Links != Links->Blink)
130 PoolVa = Links->Blink;
134 if (Safe) *PoolBlink = PoolVa;
154 PVOID PoolFlink, PoolBlink;
170 PVOID PoolFlink, PoolBlink;
335 FirstEntry = FreeEntry;
336 FreeEntry->
Size = PoolPages;
338 FreeEntry->
Owner = FirstEntry;
349 while (PoolPages-- > 1)
355 FreeEntry->
Owner = FirstEntry;
430 PMMPTE PointerPte, StartPte;
436 PVOID BaseVa, BaseVaStart;
447 if (SizeInPages == 0)
466 if (BaseVa)
return BaseVa;
486 DPRINT(
"Paged pool expansion: %lu %x\n",
i, SizeInPages);
497 DPRINT1(
"FAILED to allocate %Iu bytes from paged pool\n", SizeInBytes);
512 PageTableCount = (
PFN_COUNT)(PointerPde + 1 -
535 BaseVaStart = BaseVa;
553 #if (_MI_PAGING_LEVELS >= 3) 618 DPRINT1(
"FAILED to allocate %Iu bytes from paged pool\n", SizeInBytes);
633 EndAllocation =
i + SizeInPages - 1;
658 StartPte = PointerPte + SizeInPages;
665 }
while (++PointerPte < StartPte);
679 if (BaseVa)
return BaseVa;
702 NextEntry = NextHead->
Flink;
703 while (NextEntry != NextHead)
717 if (FreeEntry->
Size >= SizeInPages)
722 FreeEntry->
Size -= SizeInPages;
739 if (FreeEntry->
Size != 0)
773 Pfn1->
u3.
e1.StartOfAllocation = 1;
785 if (SizeInPages != 1)
790 PointerPte += SizeInPages - 1;
799 Pfn1->
u3.
e1.EndOfAllocation = 1;
820 }
while (++NextHead < LastHead);
832 PointerPte = StartPte;
833 if (StartPte ==
NULL)
838 DPRINT(
"Out of NP Expansion Pool\n");
850 MiAcquirePfnLockAtDpcLevel();
855 MiReleasePfnLockFromDpcLevel();
878 Pfn1->
u3.
e2.ReferenceCount = 1;
887 }
while (--SizeInPages > 0);
892 Pfn1->
u3.
e1.EndOfAllocation = 1;
898 Pfn1->
u3.
e1.StartOfAllocation = 1;
907 MiReleasePfnLockFromDpcLevel();
920 PMMPTE PointerPte, StartPte;
951 NumberOfPages = End -
i + 1;
952 if ((NumberOfPages == 1) &&
962 ASSERT(FreePages == NumberOfPages);
988 return NumberOfPages;
998 if ((Pfn1->
u3.
e1.EndOfAllocation == 1) &&
1008 while (Pfn1->
u3.
e1.EndOfAllocation == 0)
1020 NumberOfPages = (
PFN_COUNT)(PointerPte - StartPte + 1);
1030 StartPfn->
u3.
e1.StartOfAllocation = 0;
1031 Pfn1->
u3.
e1.EndOfAllocation = 0;
1036 FreePages = NumberOfPages;
1091 if ((Pfn1) && (Pfn1->
u3.
e1.StartOfAllocation == 0))
1102 FreePages += FreeEntry->
Size;
1114 FreeEntry = StartingVa;
1131 PointerPte -= NumberOfPages + 1;
1162 if ((Pfn1) && (Pfn1->
u3.
e1.EndOfAllocation == 0))
1169 FreeEntry = FreeEntry->
Owner;
1194 FreeEntry->
Size += FreePages;
1212 FreeEntry->
Size += FreePages;
1219 if (FreeEntry == StartingVa)
1225 FreeEntry->
Size = FreePages;
1248 NextEntry = StartingVa;
1255 NextEntry->
Owner = FreeEntry;
1258 }
while (NextEntry != LastEntry);
1271 return NumberOfPages;
1278 PMMPTE PointerPte, LastPte;
1279 PMMPDE PointerPde, LastPde;
1303 DPRINT1(
"Session Pool Start: 0x%p End: 0x%p\n",
1325 PdeCount = LastPde - PointerPde;
1338 #ifndef _M_AMD64 // FIXME 1349 BitmapSize =
sizeof(
RTL_BITMAP) + ((PoolSize + 31) / 32) *
sizeof(
ULONG);
1440 DPRINT1(
"MmRaisePoolQuota(): Failed to increase pool quota, not enough non paged pool space (current size -- %lu || allocated size -- %lu)\n",
1448 DPRINT1(
"MmRaisePoolQuota(): Failed to increase pool quota, not enough resident pages available (current available pages -- %lu)\n",
1474 DPRINT1(
"MmRaisePoolQuota(): Failed to increase pool quota, not enough paged pool space (current size -- %lu || allocated size -- %lu)\n",
VOID NTAPI MiInitializePoolEvents(VOID)
#define MI_CHARGE_PAGED_POOL_QUOTA
#define MI_MAKE_SOFTWARE_PTE(p, x)
#define DEFAULT_UNREACHABLE
PRTL_BITMAP PagedPoolAllocationMap
SIZE_T MmAllocatedNonPagedPool
POOL_TYPE NTAPI MmDeterminePoolType(IN PVOID PoolAddress)
PFN_NUMBER MmResidentAvailablePages
BOOLEAN NTAPI MiUnProtectFreeNonPagedPool(IN PVOID VirtualAddress, IN ULONG PageCount)
PFN_NUMBER MiEndOfInitialPoolFrame
struct _RTL_BITMAP RTL_BITMAP
#define MiAddressToPde(x)
PKEVENT MiHighNonPagedPoolEvent
VOID FASTCALL KeAcquireGuardedMutex(IN PKGUARDED_MUTEX GuardedMutex)
PMMPTE NTAPI MiReserveSystemPtes(IN ULONG NumberOfPtes, IN MMSYSTEM_PTE_POOL_TYPE SystemPtePoolType)
FORCEINLINE VOID InsertHeadList(_Inout_ PLIST_ENTRY ListHead, _Inout_ __drv_aliasesMem PLIST_ENTRY Entry)
#define ASSERT_IRQL_EQUAL(x)
#define VERIFIER_POOL_MASK
NTSTATUS NTAPI MiInitializeAndChargePfn(OUT PPFN_NUMBER PageFrameIndex, IN PMMPDE PointerPde, IN PFN_NUMBER ContainingPageFrame, IN BOOLEAN SessionAllocation)
NTSYSAPI void WINAPI RtlInitializeBitMap(PRTL_BITMAP, PULONG, ULONG)
NTSYSAPI VOID NTAPI RtlSetBit(_In_ PRTL_BITMAP BitMapHeader, _In_range_(<, BitMapHeader->SizeOfBitMap) ULONG BitNumber)
PVOID MmNonPagedPoolExpansionStart
#define MI_GET_NEXT_COLOR()
HARDWARE_PDE_ARMV6 TempPde
PMM_SESSION_SPACE MmSessionSpace
struct _MMPFN::@1752::@1758 e2
IN PVOID IN PVOID IN USHORT IN USHORT Size
VOID NTAPI MiInitializePfnForOtherProcess(IN PFN_NUMBER PageFrameIndex, IN PVOID PteAddress, IN PFN_NUMBER PteFrame)
NTSTATUS NTAPI MiInitializeSessionPool(VOID)
#define InsertTailList(ListHead, Entry)
PFN_NUMBER MmAvailablePages
LONG NTAPI KeSetEvent(IN PKEVENT Event, IN KPRIORITY Increment, IN BOOLEAN Wait)
FORCEINLINE PMMPDE MiPteToPde(PMMPTE PointerPte)
_Must_inspect_result_ FORCEINLINE BOOLEAN IsListEmpty(_In_ const LIST_ENTRY *ListHead)
SIZE_T MmTotalNonPagedPoolQuota
MM_PAGED_POOL_INFO PagedPoolInfo
SLIST_HEADER MiNonPagedPoolSListHead
FORCEINLINE BOOLEAN RemoveEntryList(_In_ PLIST_ENTRY Entry)
PFN_COUNT MmNumberOfFreeNonPagedPool
#define MiAddressToPte(x)
NTSTATUS(* NTAPI)(IN PFILE_FULL_EA_INFORMATION EaBuffer, IN ULONG EaLength, OUT PULONG ErrorOffset)
PMMPTE LastPteForPagedPool
FORCEINLINE VOID MI_WRITE_VALID_PTE(IN PMMPTE PointerPte, IN MMPTE TempPte)
#define InterlockedIncrementSizeT(a)
VOID NTAPI MiInitializeSystemPtes(IN PMMPTE StartingPte, IN ULONG NumberOfPtes, IN MMSYSTEM_PTE_POOL_TYPE PoolType)
VOID NTAPI MiProtectedPoolRemoveEntryList(IN PLIST_ENTRY Entry)
PKEVENT MiLowNonPagedPoolEvent
VOID NTAPI ExInitializePoolDescriptor(IN PPOOL_DESCRIPTOR PoolDescriptor, IN POOL_TYPE PoolType, IN ULONG PoolIndex, IN ULONG Threshold, IN PVOID PoolLock)
#define MI_SET_PROCESS2(x)
PFN_NUMBER MiStartOfInitialPoolFrame
NTSYSAPI void WINAPI RtlSetAllBits(PRTL_BITMAP)
VOID NTAPI MiInitializeNonPagedPoolThresholds(VOID)
PFN_NUMBER MmSystemPageDirectory[PPE_PER_PAGE]
POOL_DESCRIPTOR PagedPool
SIZE_T AllocatedPagedPool
#define MI_MAX_FREE_PAGE_LISTS
FORCEINLINE PMMPTE MiPdeToPte(PMMPDE PointerPde)
ULONG_PTR VerifierAllocation
PFN_NUMBER MiHighPagedPoolThreshold
NTSYSAPI ULONG WINAPI RtlFindClearBitsAndSet(PRTL_BITMAP, ULONG, ULONG)
PFN_COUNT MiExpansionPoolPagesInitialCharge
#define BASE_POOL_TYPE_MASK
FORCEINLINE VOID MI_WRITE_VALID_PDE(IN PMMPDE PointerPde, IN MMPDE TempPde)
_Must_inspect_result_ _In_ WDFDMATRANSACTION _In_ PFN_WDF_PROGRAM_DMA _In_ WDF_DMA_DIRECTION _In_ PMDL _In_ PVOID VirtualAddress
VOID NTAPI MiReleaseSystemPtes(IN PMMPTE StartingPte, IN ULONG NumberOfPtes, IN MMSYSTEM_PTE_POOL_TYPE SystemPtePoolType)
PFLT_MESSAGE_WAITER_QUEUE CONTAINING_RECORD(Csq, DEVICE_EXTENSION, IrpQueue)) -> WaiterQ.mLock) _IRQL_raises_(DISPATCH_LEVEL) VOID NTAPI FltpAcquireMessageWaiterLock(_In_ PIO_CSQ Csq, _Out_ PKIRQL Irql)
VOID NTAPI KeFlushEntireTb(IN BOOLEAN Invalid, IN BOOLEAN AllProcessors)
FORCEINLINE VOID InitializeSListHead(_Out_ PSLIST_HEADER SListHead)
ULONG MmConsumedPoolPercentage
ULONG NTAPI MiFreePoolPages(IN PVOID StartingVa)
KGUARDED_MUTEX MmPagedPoolMutex
struct _LIST_ENTRY * Flink
_In_ HANDLE _Outptr_result_bytebuffer_ ViewSize PVOID * BaseAddress
_Requires_lock_held_(PspQuotaLock)
Raises the quota limit, depending on the given pool type of the quota in question....
struct _MM_SESSION_SPACE * GlobalVirtualAddress
#define MM_FREE_POOL_SIGNATURE
#define NT_SUCCESS(StatCode)
_In_ WDFCOLLECTION _In_ ULONG Index
PVOID NTAPI MiAllocatePoolPages(IN POOL_TYPE PoolType, IN SIZE_T SizeInBytes)
MM_PAGED_POOL_INFO MmPagedPoolInfo
VOID NTAPI RtlClearBit(_In_ PRTL_BITMAP BitMapHeader, _In_ BITMAP_INDEX BitNumber)
VOID FASTCALL KeReleaseQueuedSpinLock(IN KSPIN_LOCK_QUEUE_NUMBER LockNumber, IN KIRQL OldIrql)
HARDWARE_PTE_ARMV6 TempPte
FORCEINLINE USHORT ExQueryDepthSList(_In_ PSLIST_HEADER SListHead)
_Must_inspect_result_ NTSYSAPI BOOLEAN NTAPI RtlTestBit(_In_ PRTL_BITMAP BitMapHeader, _In_range_(<, BitMapHeader->SizeOfBitMap) ULONG BitNumber)
PFN_NUMBER MiHighNonPagedPoolThreshold
ULONG MiNonPagedPoolSListMaximum
_Must_inspect_result_ _In_opt_ PWDF_OBJECT_ATTRIBUTES _In_ _Strict_type_match_ POOL_TYPE _In_opt_ ULONG PoolTag
#define ExAllocatePoolWithTag(hernya, size, tag)
VOID NTAPI MiProtectedPoolInsertList(IN PLIST_ENTRY ListHead, IN PLIST_ENTRY Entry, IN BOOLEAN Critical)
struct _MMFREE_POOL_ENTRY * Owner
#define BYTES_TO_PAGES(Size)
ULONG MiPagedPoolSListMaximum
_In_ SIZE_T _Out_ PSIZE_T NewMaxQuota
KGUARDED_MUTEX PagedPoolMutex
_Requires_lock_held_ Interrupt _Releases_lock_ Interrupt _In_ _IRQL_restores_ KIRQL OldIrql
PFN_NUMBER MmMaximumNonPagedPoolInPages
_In_ SIZE_T CurrentMaxQuota
_Must_inspect_result_ _In_ WDFCMRESLIST List
#define InterlockedPushEntrySList(SListHead, SListEntry)
VOID NTAPI MiInitializeNonPagedPool(VOID)
KIRQL FASTCALL KeAcquireQueuedSpinLock(IN KSPIN_LOCK_QUEUE_NUMBER LockNumber)
FORCEINLINE PMMPFN MiGetPfnEntry(IN PFN_NUMBER Pfn)
FORCEINLINE VOID MiProtectedPoolUnProtectLinks(IN PLIST_ENTRY Links, OUT PVOID *PoolFlink, OUT PVOID *PoolBlink)
PVOID MmNonPagedPoolStart
PFN_NUMBER MiLowPagedPoolThreshold
PFN_COUNT NTAPI MiDeleteSystemPageableVm(IN PMMPTE PointerPte, IN PFN_NUMBER PageCount, IN ULONG Flags, OUT PPFN_NUMBER ValidPages)
_In_ ULONG _In_ ULONG Offset
NTSYSAPI void WINAPI RtlClearBits(PRTL_BITMAP, ULONG, ULONG)
PFN_NUMBER NTAPI MiRemoveAnyPage(IN ULONG Color)
FORCEINLINE VOID MI_WRITE_INVALID_PTE(IN PMMPTE PointerPte, IN MMPTE InvalidPte)
PFN_COUNT MmNumberOfPhysicalPages
BOOLEAN MmProtectFreedNonPagedPool
VOID FASTCALL KeInitializeGuardedMutex(OUT PKGUARDED_MUTEX GuardedMutex)
GLsizei GLenum const GLvoid GLsizei GLenum GLbyte GLbyte GLbyte GLdouble GLdouble GLdouble GLfloat GLfloat GLfloat GLint GLint GLint GLshort GLshort GLshort GLubyte GLubyte GLubyte GLuint GLuint GLuint GLushort GLushort GLushort GLbyte GLbyte GLbyte GLbyte GLdouble GLdouble GLdouble GLdouble GLfloat GLfloat GLfloat GLfloat GLint GLint GLint GLint GLshort GLshort GLshort GLshort GLubyte GLubyte GLubyte GLubyte GLuint GLuint GLuint GLuint GLushort GLushort GLushort GLushort GLboolean const GLdouble const GLfloat const GLint const GLshort const GLbyte const GLdouble const GLfloat const GLint const GLshort const GLdouble const GLfloat const GLint const GLshort const GLdouble const GLfloat const GLint const GLshort const GLdouble const GLfloat const GLint const GLshort const GLdouble const GLdouble const GLfloat const GLfloat const GLint const GLint const GLshort const GLshort const GLdouble const GLfloat const GLint const GLshort const GLdouble const GLfloat const GLint const GLshort const GLdouble const GLfloat const GLint const GLshort const GLdouble const GLfloat const GLint const GLshort const GLdouble const GLfloat const GLint const GLshort const GLdouble const GLfloat const GLint const GLshort const GLdouble const GLfloat const GLint const GLshort GLenum GLenum GLenum GLfloat GLenum GLint GLenum GLenum GLenum GLfloat GLenum GLenum GLint GLenum GLfloat GLenum GLint GLint GLushort GLenum GLenum GLfloat GLenum GLenum GLint GLfloat const GLubyte GLenum GLenum GLenum const GLfloat GLenum GLenum const GLint GLenum GLint GLint GLsizei GLsizei GLint GLenum GLenum const GLvoid GLenum GLenum const GLfloat GLenum GLenum const GLint GLenum GLenum const GLdouble GLenum GLenum const GLfloat GLenum GLenum const GLint GLsizei GLuint GLfloat GLuint GLbitfield GLfloat GLint GLuint GLboolean GLenum GLfloat GLenum GLbitfield GLenum GLfloat GLfloat GLint GLint const GLfloat GLenum GLfloat GLfloat GLint GLint GLfloat GLfloat GLint GLint const GLfloat GLint GLfloat GLfloat GLint GLfloat GLfloat GLint GLfloat GLfloat const GLdouble const GLfloat const GLdouble const GLfloat GLint i
struct _MMFREE_POOL_ENTRY * PMMFREE_POOL_ENTRY
PRTL_BITMAP EndOfPagedPoolBitmap
#define InitializeListHead(ListHead)
ULONG MmMaximumNonPagedPoolInBytes
_Must_inspect_result_ typedef _In_ PHYSICAL_ADDRESS _Inout_ PLARGE_INTEGER NumberOfBytes
_In_ SIZE_T QuotaToReturn
NTSYSAPI void WINAPI RtlClearAllBits(PRTL_BITMAP)
VOID FASTCALL KeReleaseGuardedMutex(IN OUT PKGUARDED_MUTEX GuardedMutex)
PKEVENT MiLowPagedPoolEvent
#define MI_NON_PAGED_QUOTA_MIN_RESIDENT_PAGES
SLIST_HEADER MiPagedPoolSListHead
FORCEINLINE PFN_NUMBER MiGetPfnEntryIndex(IN PMMPFN Pfn1)
_Must_inspect_result_ _In_ WDFDEVICE _In_ DEVICE_REGISTRY_PROPERTY _In_ _Strict_type_match_ POOL_TYPE PoolType
PKEVENT MiHighPagedPoolEvent
#define RtlZeroMemory(Destination, Length)
VOID NTAPI MiProtectFreeNonPagedPool(IN PVOID VirtualAddress, IN ULONG PageCount)
SIZE_T MmTotalPagedPoolQuota
PFN_NUMBER MiLowNonPagedPoolThreshold
ULONG MmSizeOfNonPagedPoolInBytes
PFN_NUMBER MmSizeOfPagedPoolInPages
#define InterlockedPopEntrySList(SListHead)
PMMPTE FirstPteForPagedPool
FORCEINLINE PVOID MiPteToAddress(PMMPTE PointerPte)
LIST_ENTRY MmNonPagedPoolFreeListHead[MI_MAX_FREE_PAGE_LISTS]
VOID NTAPI KeClearEvent(IN PKEVENT Event)
#define MI_CHARGE_NON_PAGED_POOL_QUOTA
FORCEINLINE BOOLEAN MI_IS_PHYSICAL_ADDRESS(IN PVOID Address)
PVOID NTAPI MmAllocateMappingAddress(IN SIZE_T NumberOfBytes, IN ULONG PoolTag)
base of all file and directory entries
VOID NTAPI KeBugCheckEx(_In_ ULONG BugCheckCode, _In_ ULONG_PTR BugCheckParameter1, _In_ ULONG_PTR BugCheckParameter2, _In_ ULONG_PTR BugCheckParameter3, _In_ ULONG_PTR BugCheckParameter4)
#define MI_QUOTA_NON_PAGED_NEEDED_PAGES
VOID NTAPI MmFreeMappingAddress(IN PVOID BaseAddress, IN ULONG PoolTag)
PMMPDE NextPdeForPagedPoolExpansion
FORCEINLINE VOID MiProtectedPoolProtectLinks(IN PVOID PoolFlink, IN PVOID PoolBlink)
PFN_NUMBER SessionPageDirectoryIndex
static KSPIN_LOCK PspQuotaLock