ReactOS 0.4.15-dev-8131-g4988de4
expool.c File Reference
#include <ntoskrnl.h>
#include <debug.h>
#include <mm/ARM3/miarm.h>
Include dependency graph for expool.c:

Go to the source code of this file.

Classes

struct  _POOL_DPC_CONTEXT
 

Macros

#define NDEBUG
 
#define MODULE_INVOLVED_IN_ARM3
 
#define POOL_BIG_TABLE_ENTRY_FREE   0x1
 
#define POOL_BIG_TABLE_USE_RATE   4
 
#define POOL_ENTRY(x)   (PPOOL_HEADER)((ULONG_PTR)(x) - sizeof(POOL_HEADER))
 
#define POOL_FREE_BLOCK(x)   (PLIST_ENTRY)((ULONG_PTR)(x) + sizeof(POOL_HEADER))
 
#define POOL_BLOCK(x, i)   (PPOOL_HEADER)((ULONG_PTR)(x) + ((i) * POOL_BLOCK_SIZE))
 
#define POOL_NEXT_BLOCK(x)   POOL_BLOCK((x), (x)->BlockSize)
 
#define POOL_PREV_BLOCK(x)   POOL_BLOCK((x), -((x)->PreviousSize))
 

Typedefs

typedef struct _POOL_DPC_CONTEXT POOL_DPC_CONTEXT
 
typedef struct _POOL_DPC_CONTEXTPPOOL_DPC_CONTEXT
 

Functions

PLIST_ENTRY NTAPI ExpDecodePoolLink (IN PLIST_ENTRY Link)
 
PLIST_ENTRY NTAPI ExpEncodePoolLink (IN PLIST_ENTRY Link)
 
VOID NTAPI ExpCheckPoolLinks (IN PLIST_ENTRY ListHead)
 
VOID NTAPI ExpInitializePoolListHead (IN PLIST_ENTRY ListHead)
 
BOOLEAN NTAPI ExpIsPoolListEmpty (IN PLIST_ENTRY ListHead)
 
VOID NTAPI ExpRemovePoolEntryList (IN PLIST_ENTRY Entry)
 
PLIST_ENTRY NTAPI ExpRemovePoolHeadList (IN PLIST_ENTRY ListHead)
 
PLIST_ENTRY NTAPI ExpRemovePoolTailList (IN PLIST_ENTRY ListHead)
 
VOID NTAPI ExpInsertPoolTailList (IN PLIST_ENTRY ListHead, IN PLIST_ENTRY Entry)
 
VOID NTAPI ExpInsertPoolHeadList (IN PLIST_ENTRY ListHead, IN PLIST_ENTRY Entry)
 
VOID NTAPI ExpCheckPoolHeader (IN PPOOL_HEADER Entry)
 
VOID NTAPI ExpCheckPoolAllocation (PVOID P, POOL_TYPE PoolType, ULONG Tag)
 
VOID NTAPI ExpCheckPoolBlocks (IN PVOID Block)
 
FORCEINLINE VOID ExpCheckPoolIrqlLevel (IN POOL_TYPE PoolType, IN SIZE_T NumberOfBytes, IN PVOID Entry)
 
FORCEINLINE ULONG ExpComputeHashForTag (IN ULONG Tag, IN SIZE_T BucketMask)
 
FORCEINLINE ULONG ExpComputePartialHashForAddress (IN PVOID BaseAddress)
 
VOID NTAPI ExpSeedHotTags (VOID)
 
VOID NTAPI ExpRemovePoolTracker (IN ULONG Key, IN SIZE_T NumberOfBytes, IN POOL_TYPE PoolType)
 
VOID NTAPI ExpInsertPoolTracker (IN ULONG Key, IN SIZE_T NumberOfBytes, IN POOL_TYPE PoolType)
 
VOID NTAPI ExInitializePoolDescriptor (IN PPOOL_DESCRIPTOR PoolDescriptor, IN POOL_TYPE PoolType, IN ULONG PoolIndex, IN ULONG Threshold, IN PVOID PoolLock)
 
VOID NTAPI InitializePool (IN POOL_TYPE PoolType, IN ULONG Threshold)
 
FORCEINLINE KIRQL ExLockPool (IN PPOOL_DESCRIPTOR Descriptor)
 
FORCEINLINE VOID ExUnlockPool (IN PPOOL_DESCRIPTOR Descriptor, IN KIRQL OldIrql)
 
VOID NTAPI ExpGetPoolTagInfoTarget (IN PKDPC Dpc, IN PVOID DeferredContext, IN PVOID SystemArgument1, IN PVOID SystemArgument2)
 
NTSTATUS NTAPI ExGetPoolTagInfo (IN PSYSTEM_POOLTAG_INFORMATION SystemInformation, IN ULONG SystemInformationLength, IN OUT PULONG ReturnLength OPTIONAL)
 
 _IRQL_requires_ (DISPATCH_LEVEL)
 
BOOLEAN NTAPI ExpAddTagForBigPages (IN PVOID Va, IN ULONG Key, IN ULONG NumberOfPages, IN POOL_TYPE PoolType)
 
ULONG NTAPI ExpFindAndRemoveTagBigPages (IN PVOID Va, OUT PULONG_PTR BigPages, IN POOL_TYPE PoolType)
 
VOID NTAPI ExQueryPoolUsage (OUT PULONG PagedPoolPages, OUT PULONG NonPagedPoolPages, OUT PULONG PagedPoolAllocs, OUT PULONG PagedPoolFrees, OUT PULONG PagedPoolLookasideHits, OUT PULONG NonPagedPoolAllocs, OUT PULONG NonPagedPoolFrees, OUT PULONG NonPagedPoolLookasideHits)
 
VOID NTAPI ExReturnPoolQuota (IN PVOID P)
 
PVOID NTAPI ExAllocatePoolWithTag (IN POOL_TYPE PoolType, IN SIZE_T NumberOfBytes, IN ULONG Tag)
 
PVOID NTAPI ExAllocatePool (POOL_TYPE PoolType, SIZE_T NumberOfBytes)
 
VOID NTAPI ExFreePoolWithTag (IN PVOID P, IN ULONG TagToFree)
 
VOID NTAPI ExFreePool (PVOID P)
 
SIZE_T NTAPI ExQueryPoolBlockSize (IN PVOID PoolBlock, OUT PBOOLEAN QuotaCharged)
 
PVOID NTAPI ExAllocatePoolWithQuota (IN POOL_TYPE PoolType, IN SIZE_T NumberOfBytes)
 
PVOID NTAPI ExAllocatePoolWithTagPriority (IN POOL_TYPE PoolType, IN SIZE_T NumberOfBytes, IN ULONG Tag, IN EX_POOL_PRIORITY Priority)
 
PVOID NTAPI ExAllocatePoolWithQuotaTag (IN POOL_TYPE PoolType, IN SIZE_T NumberOfBytes, IN ULONG Tag)
 

Variables

ULONG ExpNumberOfPagedPools
 
POOL_DESCRIPTOR NonPagedPoolDescriptor
 
PPOOL_DESCRIPTOR ExpPagedPoolDescriptor [16+1]
 
PPOOL_DESCRIPTOR PoolVector [2]
 
PKGUARDED_MUTEX ExpPagedPoolMutex
 
SIZE_T PoolTrackTableSize
 
SIZE_T PoolTrackTableMask
 
SIZE_T PoolBigPageTableSize
 
SIZE_T PoolBigPageTableHash
 
ULONG ExpBigTableExpansionFailed
 
PPOOL_TRACKER_TABLE PoolTrackTable
 
PPOOL_TRACKER_BIG_PAGES PoolBigPageTable
 
KSPIN_LOCK ExpTaggedPoolLock
 
ULONG PoolHitTag
 
BOOLEAN ExStopBadTags
 
KSPIN_LOCK ExpLargePoolTableLock
 
ULONG ExpPoolBigEntriesInUse
 
ULONG ExpPoolFlags
 
ULONG ExPoolFailures
 
ULONGLONG MiLastPoolDumpTime
 

Macro Definition Documentation

◆ MODULE_INVOLVED_IN_ARM3

#define MODULE_INVOLVED_IN_ARM3

Definition at line 15 of file expool.c.

◆ NDEBUG

#define NDEBUG

Definition at line 12 of file expool.c.

◆ POOL_BIG_TABLE_ENTRY_FREE

#define POOL_BIG_TABLE_ENTRY_FREE   0x1

Definition at line 23 of file expool.c.

◆ POOL_BIG_TABLE_USE_RATE

#define POOL_BIG_TABLE_USE_RATE   4

Definition at line 31 of file expool.c.

◆ POOL_BLOCK

#define POOL_BLOCK (   x,
  i 
)    (PPOOL_HEADER)((ULONG_PTR)(x) + ((i) * POOL_BLOCK_SIZE))

Definition at line 63 of file expool.c.

◆ POOL_ENTRY

#define POOL_ENTRY (   x)    (PPOOL_HEADER)((ULONG_PTR)(x) - sizeof(POOL_HEADER))

Definition at line 61 of file expool.c.

◆ POOL_FREE_BLOCK

#define POOL_FREE_BLOCK (   x)    (PLIST_ENTRY)((ULONG_PTR)(x) + sizeof(POOL_HEADER))

Definition at line 62 of file expool.c.

◆ POOL_NEXT_BLOCK

#define POOL_NEXT_BLOCK (   x)    POOL_BLOCK((x), (x)->BlockSize)

Definition at line 64 of file expool.c.

◆ POOL_PREV_BLOCK

#define POOL_PREV_BLOCK (   x)    POOL_BLOCK((x), -((x)->PreviousSize))

Definition at line 65 of file expool.c.

Typedef Documentation

◆ POOL_DPC_CONTEXT

◆ PPOOL_DPC_CONTEXT

Function Documentation

◆ _IRQL_requires_()

_IRQL_requires_ ( DISPATCH_LEVEL  )

Definition at line 1459 of file expool.c.

1465{
1466 SIZE_T OldSize = PoolBigPageTableSize;
1467 SIZE_T NewSize, NewSizeInBytes;
1468 PPOOL_TRACKER_BIG_PAGES NewTable;
1469 PPOOL_TRACKER_BIG_PAGES OldTable;
1470 ULONG i;
1471 ULONG PagesFreed;
1472 ULONG Hash;
1473 ULONG HashMask;
1474
1475 /* Must be holding ExpLargePoolTableLock */
1477
1478 /* Make sure we don't overflow */
1479 if (Shrink)
1480 {
1481 NewSize = OldSize / 2;
1482
1483 /* Make sure we don't shrink too much. */
1485
1487 ASSERT(NewSize <= OldSize);
1488
1489 /* If there is only one page left, then keep it around. Not a failure either. */
1490 if (NewSize == OldSize)
1491 {
1494 return TRUE;
1495 }
1496 }
1497 else
1498 {
1499 if (!NT_SUCCESS(RtlSIZETMult(2, OldSize, &NewSize)))
1500 {
1501 DPRINT1("Overflow expanding big page table. Size=%lu\n", OldSize);
1503 return FALSE;
1504 }
1505
1506 /* Make sure we don't stupidly waste pages */
1508 ASSERT(NewSize > OldSize);
1509 }
1510
1511 if (!NT_SUCCESS(RtlSIZETMult(sizeof(POOL_TRACKER_BIG_PAGES), NewSize, &NewSizeInBytes)))
1512 {
1513 DPRINT1("Overflow while calculating big page table size. Size=%lu\n", OldSize);
1515 return FALSE;
1516 }
1517
1518 NewTable = MiAllocatePoolPages(NonPagedPool, NewSizeInBytes);
1519 if (NewTable == NULL)
1520 {
1521 DPRINT("Could not allocate %lu bytes for new big page table\n", NewSizeInBytes);
1523 return FALSE;
1524 }
1525
1526 DPRINT("%s big pool tracker table to %lu entries\n", Shrink ? "Shrinking" : "Expanding", NewSize);
1527
1528 /* Initialize the new table */
1529 RtlZeroMemory(NewTable, NewSizeInBytes);
1530 for (i = 0; i < NewSize; i++)
1531 {
1532 NewTable[i].Va = (PVOID)POOL_BIG_TABLE_ENTRY_FREE;
1533 }
1534
1535 /* Copy over all items */
1536 OldTable = PoolBigPageTable;
1537 HashMask = NewSize - 1;
1538 for (i = 0; i < OldSize; i++)
1539 {
1540 /* Skip over empty items */
1541 if ((ULONG_PTR)OldTable[i].Va & POOL_BIG_TABLE_ENTRY_FREE)
1542 {
1543 continue;
1544 }
1545
1546 /* Recalculate the hash due to the new table size */
1547 Hash = ExpComputePartialHashForAddress(OldTable[i].Va) % HashMask;
1548
1549 /* Find the location in the new table */
1550 while (!((ULONG_PTR)NewTable[Hash].Va & POOL_BIG_TABLE_ENTRY_FREE))
1551 {
1552 if (++Hash == NewSize)
1553 Hash = 0;
1554 }
1555
1556 /* We must have space */
1558
1559 /* Finally, copy the item */
1560 NewTable[Hash] = OldTable[i];
1561 }
1562
1563 /* Activate the new table */
1564 PoolBigPageTable = NewTable;
1567
1568 /* Release the lock, we're done changing global state */
1570
1571 /* Free the old table and update our tracker */
1572 PagesFreed = MiFreePoolPages(OldTable);
1573 ExpRemovePoolTracker('looP', PagesFreed << PAGE_SHIFT, 0);
1574 ExpInsertPoolTracker('looP', ALIGN_UP_BY(NewSizeInBytes, PAGE_SIZE), 0);
1575
1576 return TRUE;
1577}
#define ALIGN_DOWN_BY(size, align)
#define ALIGN_UP_BY(size, align)
#define DPRINT1
Definition: precomp.h:8
#define NULL
Definition: types.h:112
#define TRUE
Definition: types.h:120
#define FALSE
Definition: types.h:117
#define NT_SUCCESS(StatCode)
Definition: apphelp.c:32
static int Hash(const char *)
Definition: reader.c:2257
#define KeReleaseSpinLock(sl, irql)
Definition: env_spec_w32.h:627
#define PAGE_SIZE
Definition: env_spec_w32.h:49
#define PAGE_SHIFT
Definition: env_spec_w32.h:45
#define KeGetCurrentIrql()
Definition: env_spec_w32.h:706
#define NonPagedPool
Definition: env_spec_w32.h:307
#define DISPATCH_LEVEL
Definition: env_spec_w32.h:696
ULONG ExpPoolBigEntriesInUse
Definition: expool.c:55
VOID NTAPI ExpRemovePoolTracker(IN ULONG Key, IN SIZE_T NumberOfBytes, IN POOL_TYPE PoolType)
Definition: expool.c:760
SIZE_T PoolBigPageTableSize
Definition: expool.c:47
FORCEINLINE ULONG ExpComputePartialHashForAddress(IN PVOID BaseAddress)
Definition: expool.c:457
PPOOL_TRACKER_BIG_PAGES PoolBigPageTable
Definition: expool.c:50
#define POOL_BIG_TABLE_ENTRY_FREE
Definition: expool.c:23
KSPIN_LOCK ExpLargePoolTableLock
Definition: expool.c:54
VOID NTAPI ExpInsertPoolTracker(IN ULONG Key, IN SIZE_T NumberOfBytes, IN POOL_TYPE PoolType)
Definition: expool.c:851
SIZE_T PoolBigPageTableHash
Definition: expool.c:47
_Must_inspect_result_ _In_ USHORT NewSize
Definition: fltkernel.h:975
GLsizei GLenum const GLvoid GLsizei GLenum GLbyte GLbyte GLbyte GLdouble GLdouble GLdouble GLfloat GLfloat GLfloat GLint GLint GLint GLshort GLshort GLshort GLubyte GLubyte GLubyte GLuint GLuint GLuint GLushort GLushort GLushort GLbyte GLbyte GLbyte GLbyte GLdouble GLdouble GLdouble GLdouble GLfloat GLfloat GLfloat GLfloat GLint GLint GLint GLint GLshort GLshort GLshort GLshort GLubyte GLubyte GLubyte GLubyte GLuint GLuint GLuint GLuint GLushort GLushort GLushort GLushort GLboolean const GLdouble const GLfloat const GLint const GLshort const GLbyte const GLdouble const GLfloat const GLint const GLshort const GLdouble const GLfloat const GLint const GLshort const GLdouble const GLfloat const GLint const GLshort const GLdouble const GLfloat const GLint const GLshort const GLdouble const GLdouble const GLfloat const GLfloat const GLint const GLint const GLshort const GLshort const GLdouble const GLfloat const GLint const GLshort const GLdouble const GLfloat const GLint const GLshort const GLdouble const GLfloat const GLint const GLshort const GLdouble const GLfloat const GLint const GLshort const GLdouble const GLfloat const GLint const GLshort const GLdouble const GLfloat const GLint const GLshort const GLdouble const GLfloat const GLint const GLshort GLenum GLenum GLenum GLfloat GLenum GLint GLenum GLenum GLenum GLfloat GLenum GLenum GLint GLenum GLfloat GLenum GLint GLint GLushort GLenum GLenum GLfloat GLenum GLenum GLint GLfloat const GLubyte GLenum GLenum GLenum const GLfloat GLenum GLenum const GLint GLenum GLint GLint GLsizei GLsizei GLint GLenum GLenum const GLvoid GLenum GLenum const GLfloat GLenum GLenum const GLint GLenum GLenum const GLdouble GLenum GLenum const GLfloat GLenum GLenum const GLint GLsizei GLuint GLfloat GLuint GLbitfield GLfloat GLint GLuint GLboolean GLenum GLfloat GLenum GLbitfield GLenum GLfloat GLfloat GLint GLint const GLfloat GLenum GLfloat GLfloat GLint GLint GLfloat GLfloat GLint GLint const GLfloat GLint GLfloat GLfloat GLint GLfloat GLfloat GLint GLfloat GLfloat const GLdouble const GLfloat const GLdouble const GLfloat GLint i
Definition: glfuncs.h:248
#define ASSERT(a)
Definition: mode.c:44
PVOID NTAPI MiAllocatePoolPages(IN POOL_TYPE PoolType, IN SIZE_T SizeInBytes)
Definition: pool.c:422
ULONG NTAPI MiFreePoolPages(IN PVOID StartingAddress)
Definition: pool.c:918
#define DPRINT
Definition: sndvol32.h:73
void * PVOID
Definition: typedefs.h:50
ULONG_PTR SIZE_T
Definition: typedefs.h:80
#define RtlZeroMemory(Destination, Length)
Definition: typedefs.h:262
uint32_t ULONG_PTR
Definition: typedefs.h:65
uint32_t ULONG
Definition: typedefs.h:59
_Requires_lock_held_ Interrupt _Releases_lock_ Interrupt _In_ _IRQL_restores_ KIRQL OldIrql
Definition: kefuncs.h:778

◆ ExAllocatePool()

PVOID NTAPI ExAllocatePool ( POOL_TYPE  PoolType,
SIZE_T  NumberOfBytes 
)

Definition at line 2465 of file expool.c.

2467{
2468 ULONG Tag = TAG_NONE;
2469#if 0 && DBG
2470 PLDR_DATA_TABLE_ENTRY LdrEntry;
2471
2472 /* Use the first four letters of the driver name, or "None" if unavailable */
2473 LdrEntry = KeGetCurrentIrql() <= APC_LEVEL
2475 : NULL;
2476 if (LdrEntry)
2477 {
2478 ULONG i;
2479 Tag = 0;
2480 for (i = 0; i < min(4, LdrEntry->BaseDllName.Length / sizeof(WCHAR)); i++)
2481 Tag = Tag >> 8 | (LdrEntry->BaseDllName.Buffer[i] & 0xff) << 24;
2482 for (; i < 4; i++)
2483 Tag = Tag >> 8 | ' ' << 24;
2484 }
2485#endif
2487}
#define ExAllocatePoolWithTag(hernya, size, tag)
Definition: env_spec_w32.h:350
#define APC_LEVEL
Definition: env_spec_w32.h:695
#define _ReturnAddress()
Definition: intrin_arm.h:35
PLDR_DATA_TABLE_ENTRY NTAPI MiLookupDataTableEntry(IN PVOID Address)
Definition: sysldr.c:3517
#define min(a, b)
Definition: monoChain.cc:55
Definition: btrfs_drv.h:1876
UNICODE_STRING BaseDllName
Definition: ldrtypes.h:145
#define TAG_NONE
Definition: tag.h:110
_Must_inspect_result_ _In_ WDFDEVICE _In_ BOOLEAN _In_opt_ PVOID Tag
Definition: wdfdevice.h:4065
_Must_inspect_result_ _In_ WDFDEVICE _In_ DEVICE_REGISTRY_PROPERTY _In_ _Strict_type_match_ POOL_TYPE PoolType
Definition: wdfdevice.h:3815
_Must_inspect_result_ typedef _In_ PHYSICAL_ADDRESS _Inout_ PLARGE_INTEGER NumberOfBytes
Definition: iotypes.h:1036
__wchar_t WCHAR
Definition: xmlstorage.h:180

◆ ExAllocatePoolWithQuota()

PVOID NTAPI ExAllocatePoolWithQuota ( IN POOL_TYPE  PoolType,
IN SIZE_T  NumberOfBytes 
)

Definition at line 2949 of file expool.c.

2951{
2952 //
2953 // Allocate the pool
2954 //
2956}
#define ExAllocatePoolWithQuotaTag(a, b, c)
Definition: exfuncs.h:530

◆ ExAllocatePoolWithQuotaTag()

PVOID NTAPI ExAllocatePoolWithQuotaTag ( IN POOL_TYPE  PoolType,
IN SIZE_T  NumberOfBytes,
IN ULONG  Tag 
)

Definition at line 2987 of file expool.c.

2990{
2991 BOOLEAN Raise = TRUE;
2992 PVOID Buffer;
2996
2997 //
2998 // Check if we should fail instead of raising an exception
2999 //
3001 {
3002 Raise = FALSE;
3003 PoolType &= ~POOL_QUOTA_FAIL_INSTEAD_OF_RAISE;
3004 }
3005
3006 //
3007 // Inject the pool quota mask
3008 //
3010
3011 //
3012 // Check if we have enough space to add the quota owner process, as long as
3013 // this isn't the system process, which never gets charged quota
3014 //
3015 ASSERT(NumberOfBytes != 0);
3016 if ((NumberOfBytes <= (PAGE_SIZE - POOL_BLOCK_SIZE - sizeof(PVOID))) &&
3018 {
3019 //
3020 // Add space for our EPROCESS pointer
3021 //
3022 NumberOfBytes += sizeof(PEPROCESS);
3023 }
3024 else
3025 {
3026 //
3027 // We won't be able to store the pointer, so don't use quota for this
3028 //
3030 }
3031
3032 //
3033 // Allocate the pool buffer now
3034 //
3036
3037 //
3038 // If the buffer is page-aligned, this is a large page allocation and we
3039 // won't touch it
3040 //
3041 if (PAGE_ALIGN(Buffer) != Buffer)
3042 {
3043 //
3044 // Also if special pool is enabled, and this was allocated from there,
3045 // we won't touch it either
3046 //
3049 {
3050 return Buffer;
3051 }
3052
3053 //
3054 // If it wasn't actually allocated with quota charges, ignore it too
3055 //
3056 if (!(PoolType & QUOTA_POOL_MASK)) return Buffer;
3057
3058 //
3059 // If this is the system process, we don't charge quota, so ignore
3060 //
3061 if (Process == PsInitialSystemProcess) return Buffer;
3062
3063 //
3064 // Actually go and charge quota for the process now
3065 //
3069 Entry->BlockSize * POOL_BLOCK_SIZE);
3070 if (!NT_SUCCESS(Status))
3071 {
3072 //
3073 // Quota failed, back out the allocation, clear the owner, and fail
3074 //
3075 ((PVOID *)POOL_NEXT_BLOCK(Entry))[-1] = NULL;
3077 if (Raise) RtlRaiseStatus(Status);
3078 return NULL;
3079 }
3080
3081 //
3082 // Quota worked, write the owner and then reference it before returning
3083 //
3084 ((PVOID *)POOL_NEXT_BLOCK(Entry))[-1] = Process;
3086 }
3087 else if (!(Buffer) && (Raise))
3088 {
3089 //
3090 // The allocation failed, raise an error if we are in raise mode
3091 //
3093 }
3094
3095 //
3096 // Return the allocated buffer
3097 //
3098 return Buffer;
3099}
#define BASE_POOL_TYPE_MASK
Definition: ExPools.c:15
#define QUOTA_POOL_MASK
Definition: ExPools.c:16
unsigned char BOOLEAN
LONG NTSTATUS
Definition: precomp.h:26
Definition: bufpool.h:45
#define POOL_ENTRY(x)
Definition: expool.c:61
#define POOL_NEXT_BLOCK(x)
Definition: expool.c:64
ULONG ExpPoolFlags
Definition: expool.c:56
_Must_inspect_result_ _In_ PLARGE_INTEGER _In_ PLARGE_INTEGER _In_ ULONG _In_ PFILE_OBJECT _In_ PVOID Process
Definition: fsrtlfuncs.h:223
Status
Definition: gdiplustypes.h:25
#define POOL_BLOCK_SIZE
Definition: miarm.h:269
#define POOL_FLAG_SPECIAL_POOL
Definition: miarm.h:283
#define ExFreePoolWithTag(_P, _T)
Definition: module.h:1109
DECLSPEC_NORETURN NTSYSAPI VOID NTAPI RtlRaiseStatus(_In_ NTSTATUS Status)
struct _EPROCESS * PEPROCESS
Definition: nt_native.h:30
BOOLEAN NTAPI MmIsSpecialPoolAddress(IN PVOID P)
PEPROCESS PsInitialSystemProcess
Definition: psmgr.c:50
NTSTATUS NTAPI PsChargeProcessPoolQuota(_In_ PEPROCESS Process, _In_ POOL_TYPE PoolType, _In_ SIZE_T Amount)
Charges the process' quota pool. The type of quota to be charged depends upon the PoolType parameter.
Definition: quota.c:872
base of all file and directory entries
Definition: entries.h:83
#define STATUS_INSUFFICIENT_RESOURCES
Definition: udferr_usr.h:158
#define POOL_QUOTA_FAIL_INSTEAD_OF_RAISE
#define PAGE_ALIGN(Va)
#define ObReferenceObject
Definition: obfuncs.h:204
#define PsGetCurrentProcess
Definition: psfuncs.h:17

◆ ExAllocatePoolWithTag()

PVOID NTAPI ExAllocatePoolWithTag ( IN POOL_TYPE  PoolType,
IN SIZE_T  NumberOfBytes,
IN ULONG  Tag 
)

Definition at line 1904 of file expool.c.

1907{
1908 PPOOL_DESCRIPTOR PoolDesc;
1909 PLIST_ENTRY ListHead;
1910 PPOOL_HEADER Entry, NextEntry, FragmentEntry;
1911 KIRQL OldIrql;
1912 USHORT BlockSize, i;
1913 ULONG OriginalType;
1914 PKPRCB Prcb = KeGetCurrentPrcb();
1916
1917 //
1918 // Some sanity checks
1919 //
1920 ASSERT(Tag != 0);
1921 ASSERT(Tag != ' GIB');
1922 ASSERT(NumberOfBytes != 0);
1924
1925 //
1926 // Not supported in ReactOS
1927 //
1929
1930 //
1931 // Check if verifier or special pool is enabled
1932 //
1934 {
1935 //
1936 // For verifier, we should call the verification routine
1937 //
1939 {
1940 DPRINT1("Driver Verifier is not yet supported\n");
1941 }
1942
1943 //
1944 // For special pool, we check if this is a suitable allocation and do
1945 // the special allocation if needed
1946 //
1948 {
1949 //
1950 // Check if this is a special pool allocation
1951 //
1953 {
1954 //
1955 // Try to allocate using special pool
1956 //
1958 if (Entry) return Entry;
1959 }
1960 }
1961 }
1962
1963 //
1964 // Get the pool type and its corresponding vector for this request
1965 //
1966 OriginalType = PoolType;
1968 PoolDesc = PoolVector[PoolType];
1969 ASSERT(PoolDesc != NULL);
1970
1971 //
1972 // Check if this is a big page allocation
1973 //
1975 {
1976 //
1977 // Allocate pages for it
1978 //
1979 Entry = MiAllocatePoolPages(OriginalType, NumberOfBytes);
1980 if (!Entry)
1981 {
1982#if DBG
1983 //
1984 // Out of memory, display current consumption
1985 // Let's consider that if the caller wanted more
1986 // than a hundred pages, that's a bogus caller
1987 // and we are not out of memory. Dump at most
1988 // once a second to avoid spamming the log.
1989 //
1990 if (NumberOfBytes < 100 * PAGE_SIZE &&
1992 {
1993 MiDumpPoolConsumers(FALSE, 0, 0, 0);
1995 }
1996#endif
1997
1998 //
1999 // Must succeed pool is deprecated, but still supported. These allocation
2000 // failures must cause an immediate bugcheck
2001 //
2002 if (OriginalType & MUST_SUCCEED_POOL_MASK)
2003 {
2004 KeBugCheckEx(MUST_SUCCEED_POOL_EMPTY,
2008 0);
2009 }
2010
2011 //
2012 // Internal debugging
2013 //
2015
2016 //
2017 // This flag requests printing failures, and can also further specify
2018 // breaking on failures
2019 //
2021 {
2022 DPRINT1("EX: ExAllocatePool (%lu, 0x%x) returning NULL\n",
2024 OriginalType);
2026 }
2027
2028 //
2029 // Finally, this flag requests an exception, which we are more than
2030 // happy to raise!
2031 //
2032 if (OriginalType & POOL_RAISE_IF_ALLOCATION_FAILURE)
2033 {
2035 }
2036
2037 return NULL;
2038 }
2039
2040 //
2041 // Increment required counters
2042 //
2047
2048 //
2049 // Add a tag for the big page allocation and switch to the generic "BIG"
2050 // tag if we failed to do so, then insert a tracker for this alloation.
2051 //
2053 Tag,
2055 OriginalType))
2056 {
2057 Tag = ' GIB';
2058 }
2060 return Entry;
2061 }
2062
2063 //
2064 // Should never request 0 bytes from the pool, but since so many drivers do
2065 // it, we'll just assume they want 1 byte, based on NT's similar behavior
2066 //
2067 if (!NumberOfBytes) NumberOfBytes = 1;
2068
2069 //
2070 // A pool allocation is defined by its data, a linked list to connect it to
2071 // the free list (if necessary), and a pool header to store accounting info.
2072 // Calculate this size, then convert it into a block size (units of pool
2073 // headers)
2074 //
2075 // Note that i cannot overflow (past POOL_LISTS_PER_PAGE) because any such
2076 // request would've been treated as a POOL_MAX_ALLOC earlier and resulted in
2077 // the direct allocation of pages.
2078 //
2079 i = (USHORT)((NumberOfBytes + sizeof(POOL_HEADER) + (POOL_BLOCK_SIZE - 1))
2080 / POOL_BLOCK_SIZE);
2082
2083 //
2084 // Handle lookaside list optimization for both paged and nonpaged pool
2085 //
2087 {
2088 //
2089 // Try popping it from the per-CPU lookaside list
2090 //
2092 Prcb->PPPagedLookasideList[i - 1].P :
2093 Prcb->PPNPagedLookasideList[i - 1].P;
2094 LookasideList->TotalAllocates++;
2096 if (!Entry)
2097 {
2098 //
2099 // We failed, try popping it from the global list
2100 //
2102 Prcb->PPPagedLookasideList[i - 1].L :
2103 Prcb->PPNPagedLookasideList[i - 1].L;
2104 LookasideList->TotalAllocates++;
2106 }
2107
2108 //
2109 // If we were able to pop it, update the accounting and return the block
2110 //
2111 if (Entry)
2112 {
2113 LookasideList->AllocateHits++;
2114
2115 //
2116 // Get the real entry, write down its pool type, and track it
2117 //
2118 Entry--;
2119 Entry->PoolType = OriginalType + 1;
2121 Entry->BlockSize * POOL_BLOCK_SIZE,
2122 OriginalType);
2123
2124 //
2125 // Return the pool allocation
2126 //
2127 Entry->PoolTag = Tag;
2128 (POOL_FREE_BLOCK(Entry))->Flink = NULL;
2129 (POOL_FREE_BLOCK(Entry))->Blink = NULL;
2130 return POOL_FREE_BLOCK(Entry);
2131 }
2132 }
2133
2134 //
2135 // Loop in the free lists looking for a block if this size. Start with the
2136 // list optimized for this kind of size lookup
2137 //
2138 ListHead = &PoolDesc->ListHeads[i];
2139 do
2140 {
2141 //
2142 // Are there any free entries available on this list?
2143 //
2144 if (!ExpIsPoolListEmpty(ListHead))
2145 {
2146 //
2147 // Acquire the pool lock now
2148 //
2149 OldIrql = ExLockPool(PoolDesc);
2150
2151 //
2152 // And make sure the list still has entries
2153 //
2154 if (ExpIsPoolListEmpty(ListHead))
2155 {
2156 //
2157 // Someone raced us (and won) before we had a chance to acquire
2158 // the lock.
2159 //
2160 // Try again!
2161 //
2162 ExUnlockPool(PoolDesc, OldIrql);
2163 continue;
2164 }
2165
2166 //
2167 // Remove a free entry from the list
2168 // Note that due to the way we insert free blocks into multiple lists
2169 // there is a guarantee that any block on this list will either be
2170 // of the correct size, or perhaps larger.
2171 //
2172 ExpCheckPoolLinks(ListHead);
2174 ExpCheckPoolLinks(ListHead);
2176 ASSERT(Entry->BlockSize >= i);
2177 ASSERT(Entry->PoolType == 0);
2178
2179 //
2180 // Check if this block is larger that what we need. The block could
2181 // not possibly be smaller, due to the reason explained above (and
2182 // we would've asserted on a checked build if this was the case).
2183 //
2184 if (Entry->BlockSize != i)
2185 {
2186 //
2187 // Is there an entry before this one?
2188 //
2189 if (Entry->PreviousSize == 0)
2190 {
2191 //
2192 // There isn't anyone before us, so take the next block and
2193 // turn it into a fragment that contains the leftover data
2194 // that we don't need to satisfy the caller's request
2195 //
2196 FragmentEntry = POOL_BLOCK(Entry, i);
2197 FragmentEntry->BlockSize = Entry->BlockSize - i;
2198
2199 //
2200 // And make it point back to us
2201 //
2202 FragmentEntry->PreviousSize = i;
2203
2204 //
2205 // Now get the block that follows the new fragment and check
2206 // if it's still on the same page as us (and not at the end)
2207 //
2208 NextEntry = POOL_NEXT_BLOCK(FragmentEntry);
2209 if (PAGE_ALIGN(NextEntry) != NextEntry)
2210 {
2211 //
2212 // Adjust this next block to point to our newly created
2213 // fragment block
2214 //
2215 NextEntry->PreviousSize = FragmentEntry->BlockSize;
2216 }
2217 }
2218 else
2219 {
2220 //
2221 // There is a free entry before us, which we know is smaller
2222 // so we'll make this entry the fragment instead
2223 //
2224 FragmentEntry = Entry;
2225
2226 //
2227 // And then we'll remove from it the actual size required.
2228 // Now the entry is a leftover free fragment
2229 //
2230 Entry->BlockSize -= i;
2231
2232 //
2233 // Now let's go to the next entry after the fragment (which
2234 // used to point to our original free entry) and make it
2235 // reference the new fragment entry instead.
2236 //
2237 // This is the entry that will actually end up holding the
2238 // allocation!
2239 //
2241 Entry->PreviousSize = FragmentEntry->BlockSize;
2242
2243 //
2244 // And now let's go to the entry after that one and check if
2245 // it's still on the same page, and not at the end
2246 //
2247 NextEntry = POOL_BLOCK(Entry, i);
2248 if (PAGE_ALIGN(NextEntry) != NextEntry)
2249 {
2250 //
2251 // Make it reference the allocation entry
2252 //
2253 NextEntry->PreviousSize = i;
2254 }
2255 }
2256
2257 //
2258 // Now our (allocation) entry is the right size
2259 //
2260 Entry->BlockSize = i;
2261
2262 //
2263 // And the next entry is now the free fragment which contains
2264 // the remaining difference between how big the original entry
2265 // was, and the actual size the caller needs/requested.
2266 //
2267 FragmentEntry->PoolType = 0;
2268 BlockSize = FragmentEntry->BlockSize;
2269
2270 //
2271 // Now check if enough free bytes remained for us to have a
2272 // "full" entry, which contains enough bytes for a linked list
2273 // and thus can be used for allocations (up to 8 bytes...)
2274 //
2275 ExpCheckPoolLinks(&PoolDesc->ListHeads[BlockSize - 1]);
2276 if (BlockSize != 1)
2277 {
2278 //
2279 // Insert the free entry into the free list for this size
2280 //
2281 ExpInsertPoolTailList(&PoolDesc->ListHeads[BlockSize - 1],
2282 POOL_FREE_BLOCK(FragmentEntry));
2283 ExpCheckPoolLinks(POOL_FREE_BLOCK(FragmentEntry));
2284 }
2285 }
2286
2287 //
2288 // We have found an entry for this allocation, so set the pool type
2289 // and release the lock since we're done
2290 //
2291 Entry->PoolType = OriginalType + 1;
2293 ExUnlockPool(PoolDesc, OldIrql);
2294
2295 //
2296 // Increment required counters
2297 //
2300
2301 //
2302 // Track this allocation
2303 //
2305 Entry->BlockSize * POOL_BLOCK_SIZE,
2306 OriginalType);
2307
2308 //
2309 // Return the pool allocation
2310 //
2311 Entry->PoolTag = Tag;
2312 (POOL_FREE_BLOCK(Entry))->Flink = NULL;
2313 (POOL_FREE_BLOCK(Entry))->Blink = NULL;
2314 return POOL_FREE_BLOCK(Entry);
2315 }
2316 } while (++ListHead != &PoolDesc->ListHeads[POOL_LISTS_PER_PAGE]);
2317
2318 //
2319 // There were no free entries left, so we have to allocate a new fresh page
2320 //
2321 Entry = MiAllocatePoolPages(OriginalType, PAGE_SIZE);
2322 if (!Entry)
2323 {
2324#if DBG
2325 //
2326 // Out of memory, display current consumption
2327 // Let's consider that if the caller wanted more
2328 // than a hundred pages, that's a bogus caller
2329 // and we are not out of memory. Dump at most
2330 // once a second to avoid spamming the log.
2331 //
2332 if (NumberOfBytes < 100 * PAGE_SIZE &&
2334 {
2335 MiDumpPoolConsumers(FALSE, 0, 0, 0);
2337 }
2338#endif
2339
2340 //
2341 // Must succeed pool is deprecated, but still supported. These allocation
2342 // failures must cause an immediate bugcheck
2343 //
2344 if (OriginalType & MUST_SUCCEED_POOL_MASK)
2345 {
2346 KeBugCheckEx(MUST_SUCCEED_POOL_EMPTY,
2347 PAGE_SIZE,
2350 0);
2351 }
2352
2353 //
2354 // Internal debugging
2355 //
2357
2358 //
2359 // This flag requests printing failures, and can also further specify
2360 // breaking on failures
2361 //
2363 {
2364 DPRINT1("EX: ExAllocatePool (%lu, 0x%x) returning NULL\n",
2366 OriginalType);
2368 }
2369
2370 //
2371 // Finally, this flag requests an exception, which we are more than
2372 // happy to raise!
2373 //
2374 if (OriginalType & POOL_RAISE_IF_ALLOCATION_FAILURE)
2375 {
2377 }
2378
2379 //
2380 // Return NULL to the caller in all other cases
2381 //
2382 return NULL;
2383 }
2384
2385 //
2386 // Setup the entry data
2387 //
2388 Entry->Ulong1 = 0;
2389 Entry->BlockSize = i;
2390 Entry->PoolType = OriginalType + 1;
2391
2392 //
2393 // This page will have two entries -- one for the allocation (which we just
2394 // created above), and one for the remaining free bytes, which we're about
2395 // to create now. The free bytes are the whole page minus what was allocated
2396 // and then converted into units of block headers.
2397 //
2398 BlockSize = (PAGE_SIZE / POOL_BLOCK_SIZE) - i;
2399 FragmentEntry = POOL_BLOCK(Entry, i);
2400 FragmentEntry->Ulong1 = 0;
2401 FragmentEntry->BlockSize = BlockSize;
2402 FragmentEntry->PreviousSize = i;
2403
2404 //
2405 // Increment required counters
2406 //
2409
2410 //
2411 // Now check if enough free bytes remained for us to have a "full" entry,
2412 // which contains enough bytes for a linked list and thus can be used for
2413 // allocations (up to 8 bytes...)
2414 //
2415 if (FragmentEntry->BlockSize != 1)
2416 {
2417 //
2418 // Excellent -- acquire the pool lock
2419 //
2420 OldIrql = ExLockPool(PoolDesc);
2421
2422 //
2423 // And insert the free entry into the free list for this block size
2424 //
2425 ExpCheckPoolLinks(&PoolDesc->ListHeads[BlockSize - 1]);
2426 ExpInsertPoolTailList(&PoolDesc->ListHeads[BlockSize - 1],
2427 POOL_FREE_BLOCK(FragmentEntry));
2428 ExpCheckPoolLinks(POOL_FREE_BLOCK(FragmentEntry));
2429
2430 //
2431 // Release the pool lock
2432 //
2434 ExUnlockPool(PoolDesc, OldIrql);
2435 }
2436 else
2437 {
2438 //
2439 // Simply do a sanity check
2440 //
2442 }
2443
2444 //
2445 // Increment performance counters and track this allocation
2446 //
2449 Entry->BlockSize * POOL_BLOCK_SIZE,
2450 OriginalType);
2451
2452 //
2453 // And return the pool allocation
2454 //
2456 Entry->PoolTag = Tag;
2457 return POOL_FREE_BLOCK(Entry);
2458}
#define InterlockedIncrement
Definition: armddk.h:53
UCHAR KIRQL
Definition: env_spec_w32.h:591
#define PagedPool
Definition: env_spec_w32.h:308
ULONG ExPoolFailures
Definition: expool.c:57
#define POOL_BLOCK(x, i)
Definition: expool.c:63
#define POOL_FREE_BLOCK(x)
Definition: expool.c:62
VOID NTAPI ExpCheckPoolBlocks(IN PVOID Block)
Definition: expool.c:376
PPOOL_DESCRIPTOR PoolVector[2]
Definition: expool.c:44
VOID NTAPI ExpInsertPoolTailList(IN PLIST_ENTRY ListHead, IN PLIST_ENTRY Entry)
Definition: expool.c:163
VOID NTAPI ExpCheckPoolLinks(IN PLIST_ENTRY ListHead)
Definition: expool.c:99
FORCEINLINE VOID ExpCheckPoolIrqlLevel(IN POOL_TYPE PoolType, IN SIZE_T NumberOfBytes, IN PVOID Entry)
Definition: expool.c:416
PLIST_ENTRY NTAPI ExpRemovePoolHeadList(IN PLIST_ENTRY ListHead)
Definition: expool.c:139
FORCEINLINE VOID ExUnlockPool(IN PPOOL_DESCRIPTOR Descriptor, IN KIRQL OldIrql)
Definition: expool.c:1297
POOL_DESCRIPTOR NonPagedPoolDescriptor
Definition: expool.c:42
BOOLEAN NTAPI ExpAddTagForBigPages(IN PVOID Va, IN ULONG Key, IN ULONG NumberOfPages, IN POOL_TYPE PoolType)
Definition: expool.c:1581
FORCEINLINE KIRQL ExLockPool(IN PPOOL_DESCRIPTOR Descriptor)
Definition: expool.c:1273
ULONGLONG MiLastPoolDumpTime
Definition: expool.c:58
BOOLEAN NTAPI ExpIsPoolListEmpty(IN PLIST_ENTRY ListHead)
Definition: expool.c:121
_Must_inspect_result_ _In_ LPCGUID _In_ ULONG _In_ FSRTL_ALLOCATE_ECP_FLAGS _In_opt_ PFSRTL_EXTRA_CREATE_PARAMETER_CLEANUP_CALLBACK _Inout_ PVOID LookasideList
Definition: fltkernel.h:2554
NTSYSAPI void WINAPI DbgBreakPoint(void)
#define InterlockedExchangeAdd
Definition: interlocked.h:181
#define InterlockedExchangeAddSizeT(a, b)
Definition: interlocked.h:196
#define POOL_LISTS_PER_PAGE
Definition: miarm.h:271
#define POOL_FLAG_DBGPRINT_ON_FAILURE
Definition: miarm.h:284
#define POOL_FLAG_CRASH_ON_FAILURE
Definition: miarm.h:285
#define POOL_MAX_ALLOC
Definition: miarm.h:273
#define POOL_FLAG_VERIFIER
Definition: miarm.h:281
#define NUMBER_POOL_LOOKASIDE_LISTS
Definition: ketypes.h:371
FORCEINLINE struct _KPRCB * KeGetCurrentPrcb(VOID)
Definition: ketypes.h:1161
#define MUST_SUCCEED_POOL_MASK
Definition: mm.h:119
#define SESSION_POOL_MASK
Definition: mm.h:122
BOOLEAN NTAPI MmUseSpecialPool(IN SIZE_T NumberOfBytes, IN ULONG Tag)
PVOID NTAPI MmAllocateSpecialPool(IN SIZE_T NumberOfBytes, IN ULONG Tag, IN POOL_TYPE PoolType, IN ULONG SpecialType)
#define ExRaiseStatus
Definition: ntoskrnl.h:114
VOID MiDumpPoolConsumers(BOOLEAN CalledFromDbg, ULONG Tag, ULONG Mask, ULONG Flags)
long LONG
Definition: pedump.c:60
unsigned short USHORT
Definition: pedump.c:61
VOID NTAPI KeBugCheckEx(_In_ ULONG BugCheckCode, _In_ ULONG_PTR BugCheckParameter1, _In_ ULONG_PTR BugCheckParameter2, _In_ ULONG_PTR BugCheckParameter3, _In_ ULONG_PTR BugCheckParameter4)
Definition: rtlcompat.c:108
#define KeQueryInterruptTime()
Definition: ke.h:37
GENERAL_LOOKASIDE_POOL PPNPagedLookasideList[NUMBER_POOL_LOOKASIDE_LISTS]
Definition: ketypes.h:709
GENERAL_LOOKASIDE_POOL PPPagedLookasideList[NUMBER_POOL_LOOKASIDE_LISTS]
Definition: ketypes.h:710
Definition: typedefs.h:120
LIST_ENTRY ListHeads[POOL_LISTS_PER_PAGE]
Definition: miarm.h:323
ULONG RunningAllocs
Definition: miarm.h:313
ULONG TotalPages
Definition: miarm.h:315
SIZE_T TotalBytes
Definition: miarm.h:321
ULONG TotalBigPages
Definition: miarm.h:316
USHORT PreviousSize
int32_t * PLONG
Definition: typedefs.h:58
struct LOOKASIDE_ALIGN _GENERAL_LOOKASIDE * PGENERAL_LOOKASIDE
#define POOL_RAISE_IF_ALLOCATION_FAILURE
#define ROUND_TO_PAGES(Size)
#define BYTES_TO_PAGES(Size)
#define InterlockedPopEntrySList(SListHead)
Definition: rtlfuncs.h:3392

◆ ExAllocatePoolWithTagPriority()

PVOID NTAPI ExAllocatePoolWithTagPriority ( IN POOL_TYPE  PoolType,
IN SIZE_T  NumberOfBytes,
IN ULONG  Tag,
IN EX_POOL_PRIORITY  Priority 
)

Definition at line 2963 of file expool.c.

2967{
2968 PVOID Buffer;
2969
2970 //
2971 // Allocate the pool
2972 //
2974 if (Buffer == NULL)
2975 {
2977 }
2978
2979 return Buffer;
2980}
#define UNIMPLEMENTED
Definition: debug.h:118

Referenced by _RxAllocatePoolWithTag(), InitializeMemoryManagement(), and MyAllocatePool().

◆ ExFreePool()

VOID NTAPI ExFreePool ( PVOID  P)

Definition at line 2920 of file expool.c.

2921{
2922 //
2923 // Just free without checking for the tag
2924 //
2925 ExFreePoolWithTag(P, 0);
2926}
#define P(row, col)

◆ ExFreePoolWithTag()

VOID NTAPI ExFreePoolWithTag ( IN PVOID  P,
IN ULONG  TagToFree 
)

Definition at line 2494 of file expool.c.

2496{
2497 PPOOL_HEADER Entry, NextEntry;
2498 USHORT BlockSize;
2499 KIRQL OldIrql;
2501 PPOOL_DESCRIPTOR PoolDesc;
2502 ULONG Tag;
2503 BOOLEAN Combined = FALSE;
2504 PFN_NUMBER PageCount, RealPageCount;
2505 PKPRCB Prcb = KeGetCurrentPrcb();
2508
2509 //
2510 // Check if any of the debug flags are enabled
2511 //
2518 {
2519 //
2520 // Check if special pool is enabled
2521 //
2523 {
2524 //
2525 // Check if it was allocated from a special pool
2526 //
2528 {
2529 //
2530 // Was deadlock verification also enabled? We can do some extra
2531 // checks at this point
2532 //
2534 {
2535 DPRINT1("Verifier not yet supported\n");
2536 }
2537
2538 //
2539 // It is, so handle it via special pool free routine
2540 //
2542 return;
2543 }
2544 }
2545
2546 //
2547 // For non-big page allocations, we'll do a bunch of checks in here
2548 //
2549 if (PAGE_ALIGN(P) != P)
2550 {
2551 //
2552 // Get the entry for this pool allocation
2553 // The pointer math here may look wrong or confusing, but it is quite right
2554 //
2555 Entry = P;
2556 Entry--;
2557
2558 //
2559 // Get the pool type
2560 //
2561 PoolType = (Entry->PoolType - 1) & BASE_POOL_TYPE_MASK;
2562
2563 //
2564 // FIXME: Many other debugging checks go here
2565 //
2567 }
2568 }
2569
2570 //
2571 // Check if this is a big page allocation
2572 //
2573 if (PAGE_ALIGN(P) == P)
2574 {
2575 //
2576 // We need to find the tag for it, so first we need to find out what
2577 // kind of allocation this was (paged or nonpaged), then we can go
2578 // ahead and try finding the tag for it. Remember to get rid of the
2579 // PROTECTED_POOL tag if it's found.
2580 //
2581 // Note that if at insertion time, we failed to add the tag for a big
2582 // pool allocation, we used a special tag called 'BIG' to identify the
2583 // allocation, and we may get this tag back. In this scenario, we must
2584 // manually get the size of the allocation by actually counting through
2585 // the PFN database.
2586 //
2589 Tag = ExpFindAndRemoveTagBigPages(P, &PageCount, PoolType);
2590 if (!Tag)
2591 {
2592 DPRINT1("We do not know the size of this allocation. This is not yet supported\n");
2593 ASSERT(Tag == ' GIB');
2594 PageCount = 1; // We are going to lie! This might screw up accounting?
2595 }
2596 else if (Tag & PROTECTED_POOL)
2597 {
2598 Tag &= ~PROTECTED_POOL;
2599 }
2600
2601 //
2602 // Check block tag
2603 //
2604 if (TagToFree && TagToFree != Tag)
2605 {
2606 DPRINT1("Freeing pool - invalid tag specified: %.4s != %.4s\n", (char*)&TagToFree, (char*)&Tag);
2607#if DBG
2608 /* Do not bugcheck in case this is a big allocation for which we didn't manage to insert the tag */
2609 if (Tag != ' GIB')
2610 KeBugCheckEx(BAD_POOL_CALLER, 0x0A, (ULONG_PTR)P, Tag, TagToFree);
2611#endif
2612 }
2613
2614 //
2615 // We have our tag and our page count, so we can go ahead and remove this
2616 // tracker now
2617 //
2619
2620 //
2621 // Check if any of the debug flags are enabled
2622 //
2627 {
2628 //
2629 // Was deadlock verification also enabled? We can do some extra
2630 // checks at this point
2631 //
2633 {
2634 DPRINT1("Verifier not yet supported\n");
2635 }
2636
2637 //
2638 // FIXME: Many debugging checks go here
2639 //
2640 }
2641
2642 //
2643 // Update counters
2644 //
2645 PoolDesc = PoolVector[PoolType];
2648 -(LONG_PTR)(PageCount << PAGE_SHIFT));
2649
2650 //
2651 // Do the real free now and update the last counter with the big page count
2652 //
2653 RealPageCount = MiFreePoolPages(P);
2654 ASSERT(RealPageCount == PageCount);
2656 -(LONG)RealPageCount);
2657 return;
2658 }
2659
2660 //
2661 // Get the entry for this pool allocation
2662 // The pointer math here may look wrong or confusing, but it is quite right
2663 //
2664 Entry = P;
2665 Entry--;
2667
2668 //
2669 // Get the size of the entry, and it's pool type, then load the descriptor
2670 // for this pool type
2671 //
2672 BlockSize = Entry->BlockSize;
2673 PoolType = (Entry->PoolType - 1) & BASE_POOL_TYPE_MASK;
2674 PoolDesc = PoolVector[PoolType];
2675
2676 //
2677 // Make sure that the IRQL makes sense
2678 //
2680
2681 //
2682 // Get the pool tag and get rid of the PROTECTED_POOL flag
2683 //
2684 Tag = Entry->PoolTag;
2685 if (Tag & PROTECTED_POOL) Tag &= ~PROTECTED_POOL;
2686
2687 //
2688 // Check block tag
2689 //
2690 if (TagToFree && TagToFree != Tag)
2691 {
2692 DPRINT1("Freeing pool - invalid tag specified: %.4s != %.4s\n", (char*)&TagToFree, (char*)&Tag);
2693#if DBG
2694 KeBugCheckEx(BAD_POOL_CALLER, 0x0A, (ULONG_PTR)P, Tag, TagToFree);
2695#endif
2696 }
2697
2698 //
2699 // Track the removal of this allocation
2700 //
2702 BlockSize * POOL_BLOCK_SIZE,
2703 Entry->PoolType - 1);
2704
2705 //
2706 // Release pool quota, if any
2707 //
2708 if ((Entry->PoolType - 1) & QUOTA_POOL_MASK)
2709 {
2710 Process = ((PVOID *)POOL_NEXT_BLOCK(Entry))[-1];
2711 if (Process)
2712 {
2713 if (Process->Pcb.Header.Type != ProcessObject)
2714 {
2715 DPRINT1("Object %p is not a process. Type %u, pool type 0x%x, block size %u\n",
2716 Process, Process->Pcb.Header.Type, Entry->PoolType, BlockSize);
2717 KeBugCheckEx(BAD_POOL_CALLER,
2719 (ULONG_PTR)P,
2720 Tag,
2722 }
2725 }
2726 }
2727
2728 //
2729 // Is this allocation small enough to have come from a lookaside list?
2730 //
2731 if (BlockSize <= NUMBER_POOL_LOOKASIDE_LISTS)
2732 {
2733 //
2734 // Try pushing it into the per-CPU lookaside list
2735 //
2737 Prcb->PPPagedLookasideList[BlockSize - 1].P :
2738 Prcb->PPNPagedLookasideList[BlockSize - 1].P;
2739 LookasideList->TotalFrees++;
2740 if (ExQueryDepthSList(&LookasideList->ListHead) < LookasideList->Depth)
2741 {
2742 LookasideList->FreeHits++;
2744 return;
2745 }
2746
2747 //
2748 // We failed, try to push it into the global lookaside list
2749 //
2751 Prcb->PPPagedLookasideList[BlockSize - 1].L :
2752 Prcb->PPNPagedLookasideList[BlockSize - 1].L;
2753 LookasideList->TotalFrees++;
2754 if (ExQueryDepthSList(&LookasideList->ListHead) < LookasideList->Depth)
2755 {
2756 LookasideList->FreeHits++;
2758 return;
2759 }
2760 }
2761
2762 //
2763 // Get the pointer to the next entry
2764 //
2765 NextEntry = POOL_BLOCK(Entry, BlockSize);
2766
2767 //
2768 // Update performance counters
2769 //
2771 InterlockedExchangeAddSizeT(&PoolDesc->TotalBytes, -BlockSize * POOL_BLOCK_SIZE);
2772
2773 //
2774 // Acquire the pool lock
2775 //
2776 OldIrql = ExLockPool(PoolDesc);
2777
2778 //
2779 // Check if the next allocation is at the end of the page
2780 //
2782 if (PAGE_ALIGN(NextEntry) != NextEntry)
2783 {
2784 //
2785 // We may be able to combine the block if it's free
2786 //
2787 if (NextEntry->PoolType == 0)
2788 {
2789 //
2790 // The next block is free, so we'll do a combine
2791 //
2792 Combined = TRUE;
2793
2794 //
2795 // Make sure there's actual data in the block -- anything smaller
2796 // than this means we only have the header, so there's no linked list
2797 // for us to remove
2798 //
2799 if ((NextEntry->BlockSize != 1))
2800 {
2801 //
2802 // The block is at least big enough to have a linked list, so go
2803 // ahead and remove it
2804 //
2809 }
2810
2811 //
2812 // Our entry is now combined with the next entry
2813 //
2814 Entry->BlockSize = Entry->BlockSize + NextEntry->BlockSize;
2815 }
2816 }
2817
2818 //
2819 // Now check if there was a previous entry on the same page as us
2820 //
2821 if (Entry->PreviousSize)
2822 {
2823 //
2824 // Great, grab that entry and check if it's free
2825 //
2826 NextEntry = POOL_PREV_BLOCK(Entry);
2827 if (NextEntry->PoolType == 0)
2828 {
2829 //
2830 // It is, so we can do a combine
2831 //
2832 Combined = TRUE;
2833
2834 //
2835 // Make sure there's actual data in the block -- anything smaller
2836 // than this means we only have the header so there's no linked list
2837 // for us to remove
2838 //
2839 if ((NextEntry->BlockSize != 1))
2840 {
2841 //
2842 // The block is at least big enough to have a linked list, so go
2843 // ahead and remove it
2844 //
2849 }
2850
2851 //
2852 // Combine our original block (which might've already been combined
2853 // with the next block), into the previous block
2854 //
2855 NextEntry->BlockSize = NextEntry->BlockSize + Entry->BlockSize;
2856
2857 //
2858 // And now we'll work with the previous block instead
2859 //
2860 Entry = NextEntry;
2861 }
2862 }
2863
2864 //
2865 // By now, it may have been possible for our combined blocks to actually
2866 // have made up a full page (if there were only 2-3 allocations on the
2867 // page, they could've all been combined).
2868 //
2869 if ((PAGE_ALIGN(Entry) == Entry) &&
2871 {
2872 //
2873 // In this case, release the pool lock, update the performance counter,
2874 // and free the page
2875 //
2876 ExUnlockPool(PoolDesc, OldIrql);
2877 InterlockedExchangeAdd((PLONG)&PoolDesc->TotalPages, -1);
2879 return;
2880 }
2881
2882 //
2883 // Otherwise, we now have a free block (or a combination of 2 or 3)
2884 //
2885 Entry->PoolType = 0;
2886 BlockSize = Entry->BlockSize;
2887 ASSERT(BlockSize != 1);
2888
2889 //
2890 // Check if we actually did combine it with anyone
2891 //
2892 if (Combined)
2893 {
2894 //
2895 // Get the first combined block (either our original to begin with, or
2896 // the one after the original, depending if we combined with the previous)
2897 //
2898 NextEntry = POOL_NEXT_BLOCK(Entry);
2899
2900 //
2901 // As long as the next block isn't on a page boundary, have it point
2902 // back to us
2903 //
2904 if (PAGE_ALIGN(NextEntry) != NextEntry) NextEntry->PreviousSize = BlockSize;
2905 }
2906
2907 //
2908 // Insert this new free block, and release the pool lock
2909 //
2910 ExpInsertPoolHeadList(&PoolDesc->ListHeads[BlockSize - 1], POOL_FREE_BLOCK(Entry));
2912 ExUnlockPool(PoolDesc, OldIrql);
2913}
#define POOL_PREV_BLOCK(x)
Definition: expool.c:65
PLIST_ENTRY NTAPI ExpDecodePoolLink(IN PLIST_ENTRY Link)
Definition: expool.c:85
VOID NTAPI ExpRemovePoolEntryList(IN PLIST_ENTRY Entry)
Definition: expool.c:128
VOID NTAPI ExpInsertPoolHeadList(IN PLIST_ENTRY ListHead, IN PLIST_ENTRY Entry)
Definition: expool.c:178
ULONG NTAPI ExpFindAndRemoveTagBigPages(IN PVOID Va, OUT PULONG_PTR BigPages, IN POOL_TYPE PoolType)
Definition: expool.c:1682
if(dx< 0)
Definition: linetemp.h:194
#define POOL_BILLED_PROCESS_INVALID
Definition: miarm.h:306
#define POOL_FLAG_CHECK_WORKERS
Definition: miarm.h:279
#define POOL_FLAG_CHECK_RESOURCES
Definition: miarm.h:280
#define POOL_FLAG_CHECK_TIMERS
Definition: miarm.h:278
#define POOL_FLAG_CHECK_DEADLOCK
Definition: miarm.h:282
__int3264 LONG_PTR
Definition: mstsclib_h.h:276
@ ProcessObject
Definition: ketypes.h:409
POOL_TYPE NTAPI MmDeterminePoolType(IN PVOID VirtualAddress)
Definition: pool.c:408
VOID NTAPI MmFreeSpecialPool(IN PVOID P)
VOID NTAPI PsReturnPoolQuota(_In_ PEPROCESS Process, _In_ POOL_TYPE PoolType, _In_ SIZE_T Amount)
Returns the pool quota that the process was taking up.
Definition: quota.c:907
ULONG PFN_NUMBER
Definition: ke.h:9
ULONG RunningDeAllocs
Definition: miarm.h:314
INT POOL_TYPE
Definition: typedefs.h:78
FORCEINLINE USHORT ExQueryDepthSList(_In_ PSLIST_HEADER SListHead)
Definition: exfuncs.h:153
#define PROTECTED_POOL
Definition: extypes.h:340
#define ObDereferenceObject
Definition: obfuncs.h:203
#define InterlockedPushEntrySList(SListHead, SListEntry)
Definition: rtlfuncs.h:3389

◆ ExGetPoolTagInfo()

NTSTATUS NTAPI ExGetPoolTagInfo ( IN PSYSTEM_POOLTAG_INFORMATION  SystemInformation,
IN ULONG  SystemInformationLength,
IN OUT PULONG ReturnLength  OPTIONAL 
)

Definition at line 1356 of file expool.c.

1359{
1360 ULONG TableSize, CurrentLength;
1361 ULONG EntryCount;
1363 PSYSTEM_POOLTAG TagEntry;
1364 PPOOL_TRACKER_TABLE Buffer, TrackerEntry;
1367
1368 //
1369 // Keep track of how much data the caller's buffer must hold
1370 //
1371 CurrentLength = FIELD_OFFSET(SYSTEM_POOLTAG_INFORMATION, TagInfo);
1372
1373 //
1374 // Initialize the caller's buffer
1375 //
1376 TagEntry = &SystemInformation->TagInfo[0];
1377 SystemInformation->Count = 0;
1378
1379 //
1380 // Capture the number of entries, and the total size needed to make a copy
1381 // of the table
1382 //
1383 EntryCount = (ULONG)PoolTrackTableSize;
1384 TableSize = EntryCount * sizeof(POOL_TRACKER_TABLE);
1385
1386 //
1387 // Allocate the "Generic DPC" temporary buffer
1388 //
1391
1392 //
1393 // Do a "Generic DPC" to atomically retrieve the tag and allocation data
1394 //
1395 Context.PoolTrackTable = Buffer;
1396 Context.PoolTrackTableSize = PoolTrackTableSize;
1397 Context.PoolTrackTableExpansion = NULL;
1398 Context.PoolTrackTableSizeExpansion = 0;
1400
1401 //
1402 // Now parse the results
1403 //
1404 for (TrackerEntry = Buffer; TrackerEntry < (Buffer + EntryCount); TrackerEntry++)
1405 {
1406 //
1407 // If the entry is empty, skip it
1408 //
1409 if (!TrackerEntry->Key) continue;
1410
1411 //
1412 // Otherwise, add one more entry to the caller's buffer, and ensure that
1413 // enough space has been allocated in it
1414 //
1415 SystemInformation->Count++;
1416 CurrentLength += sizeof(*TagEntry);
1417 if (SystemInformationLength < CurrentLength)
1418 {
1419 //
1420 // The caller's buffer is too small, so set a failure code. The
1421 // caller will know the count, as well as how much space is needed.
1422 //
1423 // We do NOT break out of the loop, because we want to keep incrementing
1424 // the Count as well as CurrentLength so that the caller can know the
1425 // final numbers
1426 //
1428 }
1429 else
1430 {
1431 //
1432 // Small sanity check that our accounting is working correctly
1433 //
1434 ASSERT(TrackerEntry->PagedAllocs >= TrackerEntry->PagedFrees);
1435 ASSERT(TrackerEntry->NonPagedAllocs >= TrackerEntry->NonPagedFrees);
1436
1437 //
1438 // Return the data into the caller's buffer
1439 //
1440 TagEntry->TagUlong = TrackerEntry->Key;
1441 TagEntry->PagedAllocs = TrackerEntry->PagedAllocs;
1442 TagEntry->PagedFrees = TrackerEntry->PagedFrees;
1443 TagEntry->PagedUsed = TrackerEntry->PagedBytes;
1444 TagEntry->NonPagedAllocs = TrackerEntry->NonPagedAllocs;
1445 TagEntry->NonPagedFrees = TrackerEntry->NonPagedFrees;
1446 TagEntry->NonPagedUsed = TrackerEntry->NonPagedBytes;
1447 TagEntry++;
1448 }
1449 }
1450
1451 //
1452 // Free the "Generic DPC" temporary buffer, return the buffer length and status
1453 //
1454 ExFreePoolWithTag(Buffer, 'ofnI');
1455 if (ReturnLength) *ReturnLength = CurrentLength;
1456 return Status;
1457}
VOID NTAPI KeGenericCallDpc(IN PKDEFERRED_ROUTINE Routine, IN PVOID Context)
Definition: dpc.c:984
IN CINT OUT PVOID IN ULONG OUT PULONG ReturnLength
Definition: dumpinfo.c:43
#define PASSIVE_LEVEL
Definition: env_spec_w32.h:693
VOID NTAPI ExpGetPoolTagInfoTarget(IN PKDPC Dpc, IN PVOID DeferredContext, IN PVOID SystemArgument1, IN PVOID SystemArgument2)
Definition: expool.c:1321
SIZE_T PoolTrackTableSize
Definition: expool.c:46
struct _POOL_TRACKER_TABLE POOL_TRACKER_TABLE
#define STATUS_SUCCESS
Definition: shellext.h:65
LONG NonPagedAllocs
Definition: miarm.h:370
SIZE_T NonPagedBytes
Definition: miarm.h:372
LONG NonPagedFrees
Definition: miarm.h:371
SIZE_T PagedBytes
Definition: miarm.h:375
SIZE_T PagedUsed
Definition: extypes.h:1143
ULONG TagUlong
Definition: extypes.h:1139
ULONG PagedFrees
Definition: extypes.h:1142
ULONG PagedAllocs
Definition: extypes.h:1141
ULONG NonPagedAllocs
Definition: extypes.h:1144
SIZE_T NonPagedUsed
Definition: extypes.h:1146
ULONG NonPagedFrees
Definition: extypes.h:1145
#define FIELD_OFFSET(t, f)
Definition: typedefs.h:255
#define STATUS_INFO_LENGTH_MISMATCH
Definition: udferr_usr.h:133
_Must_inspect_result_ typedef _Out_ PULONG TableSize
Definition: iotypes.h:4327

Referenced by QSI_DEF().

◆ ExInitializePoolDescriptor()

VOID NTAPI ExInitializePoolDescriptor ( IN PPOOL_DESCRIPTOR  PoolDescriptor,
IN POOL_TYPE  PoolType,
IN ULONG  PoolIndex,
IN ULONG  Threshold,
IN PVOID  PoolLock 
)

Definition at line 969 of file expool.c.

974{
975 PLIST_ENTRY NextEntry, LastEntry;
976
977 //
978 // Setup the descriptor based on the caller's request
979 //
980 PoolDescriptor->PoolType = PoolType;
981 PoolDescriptor->PoolIndex = PoolIndex;
982 PoolDescriptor->Threshold = Threshold;
983 PoolDescriptor->LockAddress = PoolLock;
984
985 //
986 // Initialize accounting data
987 //
988 PoolDescriptor->RunningAllocs = 0;
989 PoolDescriptor->RunningDeAllocs = 0;
990 PoolDescriptor->TotalPages = 0;
991 PoolDescriptor->TotalBytes = 0;
992 PoolDescriptor->TotalBigPages = 0;
993
994 //
995 // Nothing pending for now
996 //
997 PoolDescriptor->PendingFrees = NULL;
998 PoolDescriptor->PendingFreeDepth = 0;
999
1000 //
1001 // Loop all the descriptor's allocation lists and initialize them
1002 //
1003 NextEntry = PoolDescriptor->ListHeads;
1004 LastEntry = NextEntry + POOL_LISTS_PER_PAGE;
1005 while (NextEntry < LastEntry)
1006 {
1007 ExpInitializePoolListHead(NextEntry);
1008 NextEntry++;
1009 }
1010
1011 //
1012 // Note that ReactOS does not support Session Pool Yet
1013 //
1015}
VOID NTAPI ExpInitializePoolListHead(IN PLIST_ENTRY ListHead)
Definition: expool.c:114
@ PagedPoolSession
Definition: ketypes.h:893

Referenced by InitializePool(), and MiInitializeSessionPool().

◆ ExLockPool()

FORCEINLINE KIRQL ExLockPool ( IN PPOOL_DESCRIPTOR  Descriptor)

Definition at line 1273 of file expool.c.

1274{
1275 //
1276 // Check if this is nonpaged pool
1277 //
1278 if ((Descriptor->PoolType & BASE_POOL_TYPE_MASK) == NonPagedPool)
1279 {
1280 //
1281 // Use the queued spin lock
1282 //
1284 }
1285 else
1286 {
1287 //
1288 // Use the guarded mutex
1289 //
1290 KeAcquireGuardedMutex(Descriptor->LockAddress);
1291 return APC_LEVEL;
1292 }
1293}
VOID FASTCALL KeAcquireGuardedMutex(IN PKGUARDED_MUTEX GuardedMutex)
Definition: gmutex.c:42
KIRQL FASTCALL KeAcquireQueuedSpinLock(IN KSPIN_LOCK_QUEUE_NUMBER LockNumber)
Definition: spinlock.c:108
_Must_inspect_result_ _In_ WDFIORESLIST _In_ PIO_RESOURCE_DESCRIPTOR Descriptor
Definition: wdfresource.h:342
@ LockQueueNonPagedPoolLock
Definition: ketypes.h:664

Referenced by ExAllocatePoolWithTag(), and ExFreePoolWithTag().

◆ ExpAddTagForBigPages()

BOOLEAN NTAPI ExpAddTagForBigPages ( IN PVOID  Va,
IN ULONG  Key,
IN ULONG  NumberOfPages,
IN POOL_TYPE  PoolType 
)

Definition at line 1581 of file expool.c.

1585{
1586 ULONG Hash, i = 0;
1587 PVOID OldVa;
1588 KIRQL OldIrql;
1590 PPOOL_TRACKER_BIG_PAGES Entry, EntryEnd, EntryStart;
1593
1594 //
1595 // As the table is expandable, these values must only be read after acquiring
1596 // the lock to avoid a teared access during an expansion
1597 // NOTE: Windows uses a special reader/writer SpinLock to improve
1598 // performance in the common case (add/remove a tracker entry)
1599 //
1600Retry:
1605
1606 //
1607 // We loop from the current hash bucket to the end of the table, and then
1608 // rollover to hash bucket 0 and keep going from there. If we return back
1609 // to the beginning, then we attempt expansion at the bottom of the loop
1610 //
1611 EntryStart = Entry = &PoolBigPageTable[Hash];
1612 EntryEnd = &PoolBigPageTable[TableSize];
1613 do
1614 {
1615 //
1616 // Make sure that this is a free entry and attempt to atomically make the
1617 // entry busy now
1618 // NOTE: the Interlocked operation cannot fail with an exclusive SpinLock
1619 //
1620 OldVa = Entry->Va;
1621 if (((ULONG_PTR)OldVa & POOL_BIG_TABLE_ENTRY_FREE) &&
1622 (NT_VERIFY(InterlockedCompareExchangePointer(&Entry->Va, Va, OldVa) == OldVa)))
1623 {
1624 //
1625 // We now own this entry, write down the size and the pool tag
1626 //
1627 Entry->Key = Key;
1628 Entry->NumberOfPages = NumberOfPages;
1629
1630 //
1631 // Add one more entry to the count, and see if we're getting within
1632 // 75% of the table size, at which point we'll do an expansion now
1633 // to avoid blocking too hard later on.
1634 //
1635 // Note that we only do this if it's also been the 16th time that we
1636 // keep losing the race or that we are not finding a free entry anymore,
1637 // which implies a massive number of concurrent big pool allocations.
1638 //
1641 {
1642 DPRINT("Attempting expansion since we now have %lu entries\n",
1645 ExpReallocateBigPageTable(OldIrql, FALSE);
1646 return TRUE;
1647 }
1648
1649 //
1650 // We have our entry, return
1651 //
1653 return TRUE;
1654 }
1655
1656 //
1657 // We don't have our entry yet, so keep trying, making the entry list
1658 // circular if we reach the last entry. We'll eventually break out of
1659 // the loop once we've rolled over and returned back to our original
1660 // hash bucket
1661 //
1662 i++;
1663 if (++Entry >= EntryEnd) Entry = &PoolBigPageTable[0];
1664 } while (Entry != EntryStart);
1665
1666 //
1667 // This means there's no free hash buckets whatsoever, so we now have
1668 // to attempt expanding the table
1669 //
1671 if (ExpReallocateBigPageTable(OldIrql, FALSE))
1672 {
1673 goto Retry;
1674 }
1676 DPRINT1("Big pool table expansion failed\n");
1677 return FALSE;
1678}
_In_ PSCSI_REQUEST_BLOCK _Out_ NTSTATUS _Inout_ BOOLEAN * Retry
Definition: classpnp.h:312
#define KeAcquireSpinLock(sl, irql)
Definition: env_spec_w32.h:609
ULONG ExpBigTableExpansionFailed
Definition: expool.c:48
#define POOL_BIG_TABLE_USE_RATE
Definition: expool.c:31
#define InterlockedCompareExchangePointer
Definition: interlocked.h:129
#define NT_VERIFY(exp)
Definition: rtlfuncs.h:3287

Referenced by ExAllocatePoolWithTag().

◆ ExpCheckPoolAllocation()

VOID NTAPI ExpCheckPoolAllocation ( PVOID  P,
POOL_TYPE  PoolType,
ULONG  Tag 
)

Definition at line 296 of file expool.c.

300{
302 ULONG i;
304 POOL_TYPE RealPoolType;
305
306 /* Get the pool header */
307 Entry = ((PPOOL_HEADER)P) - 1;
308
309 /* Check if this is a large allocation */
310 if (PAGE_ALIGN(P) == P)
311 {
312 /* Lock the pool table */
314
315 /* Find the pool tag */
316 for (i = 0; i < PoolBigPageTableSize; i++)
317 {
318 /* Check if this is our allocation */
319 if (PoolBigPageTable[i].Va == P)
320 {
321 /* Make sure the tag is ok */
322 if (PoolBigPageTable[i].Key != Tag)
323 {
324 KeBugCheckEx(BAD_POOL_CALLER, 0x0A, (ULONG_PTR)P, PoolBigPageTable[i].Key, Tag);
325 }
326
327 break;
328 }
329 }
330
331 /* Release the lock */
333
334 if (i == PoolBigPageTableSize)
335 {
336 /* Did not find the allocation */
337 //ASSERT(FALSE);
338 }
339
340 /* Get Pool type by address */
341 RealPoolType = MmDeterminePoolType(P);
342 }
343 else
344 {
345 /* Verify the tag */
346 if (Entry->PoolTag != Tag)
347 {
348 DPRINT1("Allocation has wrong pool tag! Expected '%.4s', got '%.4s' (0x%08lx)\n",
349 &Tag, &Entry->PoolTag, Entry->PoolTag);
350 KeBugCheckEx(BAD_POOL_CALLER, 0x0A, (ULONG_PTR)P, Entry->PoolTag, Tag);
351 }
352
353 /* Check the rest of the header */
355
356 /* Get Pool type from entry */
357 RealPoolType = (Entry->PoolType - 1);
358 }
359
360 /* Should we check the pool type? */
361 if (PoolType != -1)
362 {
363 /* Verify the pool type */
364 if (RealPoolType != PoolType)
365 {
366 DPRINT1("Wrong pool type! Expected %s, got %s\n",
367 PoolType & BASE_POOL_TYPE_MASK ? "PagedPool" : "NonPagedPool",
368 (Entry->PoolType - 1) & BASE_POOL_TYPE_MASK ? "PagedPool" : "NonPagedPool");
369 KeBugCheckEx(BAD_POOL_CALLER, 0xCC, (ULONG_PTR)P, Entry->PoolTag, Tag);
370 }
371 }
372}
VOID NTAPI ExpCheckPoolHeader(IN PPOOL_HEADER Entry)
Definition: expool.c:193
struct _POOL_HEADER * PPOOL_HEADER

◆ ExpCheckPoolBlocks()

VOID NTAPI ExpCheckPoolBlocks ( IN PVOID  Block)

Definition at line 376 of file expool.c.

377{
378 BOOLEAN FoundBlock = FALSE;
379 SIZE_T Size = 0;
381
382 /* Get the first entry for this page, make sure it really is the first */
383 Entry = PAGE_ALIGN(Block);
384 ASSERT(Entry->PreviousSize == 0);
385
386 /* Now scan each entry */
387 while (TRUE)
388 {
389 /* When we actually found our block, remember this */
390 if (Entry == Block) FoundBlock = TRUE;
391
392 /* Now validate this block header */
394
395 /* And go to the next one, keeping track of our size */
396 Size += Entry->BlockSize;
398
399 /* If we hit the last block, stop */
400 if (Size >= (PAGE_SIZE / POOL_BLOCK_SIZE)) break;
401
402 /* If we hit the end of the page, stop */
403 if (PAGE_ALIGN(Entry) == Entry) break;
404 }
405
406 /* We must've found our block, and we must have hit the end of the page */
407 if ((PAGE_ALIGN(Entry) != Entry) || !(FoundBlock))
408 {
409 /* Otherwise, the blocks are messed up */
410 KeBugCheckEx(BAD_POOL_HEADER, 10, (ULONG_PTR)Block, __LINE__, (ULONG_PTR)Entry);
411 }
412}
_Must_inspect_result_ _In_ WDFDEVICE _In_ PWDF_DEVICE_PROPERTY_DATA _In_ DEVPROPTYPE _In_ ULONG Size
Definition: wdfdevice.h:4533

Referenced by ExAllocatePoolWithTag(), and ExFreePoolWithTag().

◆ ExpCheckPoolHeader()

VOID NTAPI ExpCheckPoolHeader ( IN PPOOL_HEADER  Entry)

Definition at line 193 of file expool.c.

194{
195 PPOOL_HEADER PreviousEntry, NextEntry;
196
197 /* Is there a block before this one? */
198 if (Entry->PreviousSize)
199 {
200 /* Get it */
201 PreviousEntry = POOL_PREV_BLOCK(Entry);
202
203 /* The two blocks must be on the same page! */
204 if (PAGE_ALIGN(Entry) != PAGE_ALIGN(PreviousEntry))
205 {
206 /* Something is awry */
207 KeBugCheckEx(BAD_POOL_HEADER,
208 6,
209 (ULONG_PTR)PreviousEntry,
210 __LINE__,
212 }
213
214 /* This block should also indicate that it's as large as we think it is */
215 if (PreviousEntry->BlockSize != Entry->PreviousSize)
216 {
217 /* Otherwise, someone corrupted one of the sizes */
218 DPRINT1("PreviousEntry BlockSize %lu, tag %.4s. Entry PreviousSize %lu, tag %.4s\n",
219 PreviousEntry->BlockSize, (char *)&PreviousEntry->PoolTag,
220 Entry->PreviousSize, (char *)&Entry->PoolTag);
221 KeBugCheckEx(BAD_POOL_HEADER,
222 5,
223 (ULONG_PTR)PreviousEntry,
224 __LINE__,
226 }
227 }
228 else if (PAGE_ALIGN(Entry) != Entry)
229 {
230 /* If there's no block before us, we are the first block, so we should be on a page boundary */
231 KeBugCheckEx(BAD_POOL_HEADER,
232 7,
233 0,
234 __LINE__,
236 }
237
238 /* This block must have a size */
239 if (!Entry->BlockSize)
240 {
241 /* Someone must've corrupted this field */
242 if (Entry->PreviousSize)
243 {
244 PreviousEntry = POOL_PREV_BLOCK(Entry);
245 DPRINT1("PreviousEntry tag %.4s. Entry tag %.4s\n",
246 (char *)&PreviousEntry->PoolTag,
247 (char *)&Entry->PoolTag);
248 }
249 else
250 {
251 DPRINT1("Entry tag %.4s\n",
252 (char *)&Entry->PoolTag);
253 }
254 KeBugCheckEx(BAD_POOL_HEADER,
255 8,
256 0,
257 __LINE__,
259 }
260
261 /* Okay, now get the next block */
262 NextEntry = POOL_NEXT_BLOCK(Entry);
263
264 /* If this is the last block, then we'll be page-aligned, otherwise, check this block */
265 if (PAGE_ALIGN(NextEntry) != NextEntry)
266 {
267 /* The two blocks must be on the same page! */
268 if (PAGE_ALIGN(Entry) != PAGE_ALIGN(NextEntry))
269 {
270 /* Something is messed up */
271 KeBugCheckEx(BAD_POOL_HEADER,
272 9,
273 (ULONG_PTR)NextEntry,
274 __LINE__,
276 }
277
278 /* And this block should think we are as large as we truly are */
279 if (NextEntry->PreviousSize != Entry->BlockSize)
280 {
281 /* Otherwise, someone corrupted the field */
282 DPRINT1("Entry BlockSize %lu, tag %.4s. NextEntry PreviousSize %lu, tag %.4s\n",
283 Entry->BlockSize, (char *)&Entry->PoolTag,
284 NextEntry->PreviousSize, (char *)&NextEntry->PoolTag);
285 KeBugCheckEx(BAD_POOL_HEADER,
286 5,
287 (ULONG_PTR)NextEntry,
288 __LINE__,
290 }
291 }
292}

Referenced by ExpCheckPoolAllocation(), and ExpCheckPoolBlocks().

◆ ExpCheckPoolIrqlLevel()

FORCEINLINE VOID ExpCheckPoolIrqlLevel ( IN POOL_TYPE  PoolType,
IN SIZE_T  NumberOfBytes,
IN PVOID  Entry 
)

Definition at line 416 of file expool.c.

419{
420 //
421 // Validate IRQL: It must be APC_LEVEL or lower for Paged Pool, and it must
422 // be DISPATCH_LEVEL or lower for Non Paged Pool
423 //
427 {
428 //
429 // Take the system down
430 //
431 KeBugCheckEx(BAD_POOL_CALLER,
434 PoolType,
436 }
437}
#define POOL_FREE_IRQL_INVALID
Definition: miarm.h:305
#define POOL_ALLOC_IRQL_INVALID
Definition: miarm.h:304

Referenced by ExAllocatePoolWithTag(), and ExFreePoolWithTag().

◆ ExpCheckPoolLinks()

VOID NTAPI ExpCheckPoolLinks ( IN PLIST_ENTRY  ListHead)

Definition at line 99 of file expool.c.

100{
101 if ((ExpDecodePoolLink(ExpDecodePoolLink(ListHead->Flink)->Blink) != ListHead) ||
102 (ExpDecodePoolLink(ExpDecodePoolLink(ListHead->Blink)->Flink) != ListHead))
103 {
104 KeBugCheckEx(BAD_POOL_HEADER,
105 3,
106 (ULONG_PTR)ListHead,
109 }
110}
struct _LIST_ENTRY * Blink
Definition: typedefs.h:122
struct _LIST_ENTRY * Flink
Definition: typedefs.h:121

Referenced by ExAllocatePoolWithTag(), ExFreePoolWithTag(), ExpInsertPoolHeadList(), and ExpInsertPoolTailList().

◆ ExpComputeHashForTag()

FORCEINLINE ULONG ExpComputeHashForTag ( IN ULONG  Tag,
IN SIZE_T  BucketMask 
)

Definition at line 441 of file expool.c.

443{
444 //
445 // Compute the hash by multiplying with a large prime number and then XORing
446 // with the HIDWORD of the result.
447 //
448 // Finally, AND with the bucket mask to generate a valid index/bucket into
449 // the table
450 //
451 ULONGLONG Result = (ULONGLONG)40543 * Tag;
452 return (ULONG)BucketMask & ((ULONG)Result ^ (Result >> 32));
453}
uint64_t ULONGLONG
Definition: typedefs.h:67
_At_(*)(_In_ PWSK_CLIENT Client, _In_opt_ PUNICODE_STRING NodeName, _In_opt_ PUNICODE_STRING ServiceName, _In_opt_ ULONG NameSpace, _In_opt_ GUID *Provider, _In_opt_ PADDRINFOEXW Hints, _Outptr_ PADDRINFOEXW *Result, _In_opt_ PEPROCESS OwningProcess, _In_opt_ PETHREAD OwningThread, _Inout_ PIRP Irp Result)(Mem)) NTSTATUS(WSKAPI *PFN_WSK_GET_ADDRESS_INFO
Definition: wsk.h:409

Referenced by ExpInsertPoolTracker(), ExpRemovePoolTracker(), and ExpSeedHotTags().

◆ ExpComputePartialHashForAddress()

FORCEINLINE ULONG ExpComputePartialHashForAddress ( IN PVOID  BaseAddress)

Definition at line 457 of file expool.c.

458{
460 //
461 // Compute the hash by converting the address into a page number, and then
462 // XORing each nibble with the next one.
463 //
464 // We do *NOT* AND with the bucket mask at this point because big table expansion
465 // might happen. Therefore, the final step of the hash must be performed
466 // while holding the expansion pushlock, and this is why we call this a
467 // "partial" hash only.
468 //
470 return (Result >> 24) ^ (Result >> 16) ^ (Result >> 8) ^ Result;
471}
_In_ HANDLE _Outptr_result_bytebuffer_ ViewSize PVOID * BaseAddress
Definition: mmfuncs.h:404

Referenced by _IRQL_requires_(), ExpAddTagForBigPages(), and ExpFindAndRemoveTagBigPages().

◆ ExpDecodePoolLink()

PLIST_ENTRY NTAPI ExpDecodePoolLink ( IN PLIST_ENTRY  Link)

Definition at line 85 of file expool.c.

86{
87 return (PLIST_ENTRY)((ULONG_PTR)Link & ~1);
88}
#define ULONG_PTR
Definition: config.h:101
static int Link(const char **args)
Definition: vfdcmd.c:2414

Referenced by ExFreePoolWithTag(), ExpCheckPoolLinks(), ExpInsertPoolHeadList(), ExpInsertPoolTailList(), ExpIsPoolListEmpty(), ExpRemovePoolEntryList(), ExpRemovePoolHeadList(), and ExpRemovePoolTailList().

◆ ExpEncodePoolLink()

◆ ExpFindAndRemoveTagBigPages()

ULONG NTAPI ExpFindAndRemoveTagBigPages ( IN PVOID  Va,
OUT PULONG_PTR  BigPages,
IN POOL_TYPE  PoolType 
)

Definition at line 1682 of file expool.c.

1685{
1688 KIRQL OldIrql;
1693
1694 //
1695 // As the table is expandable, these values must only be read after acquiring
1696 // the lock to avoid a teared access during an expansion
1697 //
1702
1703 //
1704 // Loop while trying to find this big page allocation
1705 //
1706 while (PoolBigPageTable[Hash].Va != Va)
1707 {
1708 //
1709 // Increment the size until we go past the end of the table
1710 //
1711 if (++Hash >= TableSize)
1712 {
1713 //
1714 // Is this the second time we've tried?
1715 //
1716 if (!FirstTry)
1717 {
1718 //
1719 // This means it was never inserted into the pool table and it
1720 // received the special "BIG" tag -- return that and return 0
1721 // so that the code can ask Mm for the page count instead
1722 //
1724 *BigPages = 0;
1725 return ' GIB';
1726 }
1727
1728 //
1729 // The first time this happens, reset the hash index and try again
1730 //
1731 Hash = 0;
1732 FirstTry = FALSE;
1733 }
1734 }
1735
1736 //
1737 // Now capture all the information we need from the entry, since after we
1738 // release the lock, the data can change
1739 //
1741 *BigPages = Entry->NumberOfPages;
1742 PoolTag = Entry->Key;
1743
1744 //
1745 // Set the free bit, and decrement the number of allocations. Finally, release
1746 // the lock and return the tag that was located
1747 //
1749
1751
1752 /* If reaching 12.5% of the size (or whatever integer rounding gets us to),
1753 * halve the allocation size, which will get us to 25% of space used. */
1755 {
1756 /* Shrink the table. */
1757 ExpReallocateBigPageTable(OldIrql, TRUE);
1758 }
1759 else
1760 {
1762 }
1763 return PoolTag;
1764}
@ FirstTry
Definition: copy.c:25
_Must_inspect_result_ _In_opt_ PWDF_OBJECT_ATTRIBUTES _In_ _Strict_type_match_ POOL_TYPE _In_opt_ ULONG PoolTag
Definition: wdfmemory.h:164

Referenced by ExFreePoolWithTag().

◆ ExpGetPoolTagInfoTarget()

VOID NTAPI ExpGetPoolTagInfoTarget ( IN PKDPC  Dpc,
IN PVOID  DeferredContext,
IN PVOID  SystemArgument1,
IN PVOID  SystemArgument2 
)

Definition at line 1321 of file expool.c.

1325{
1329
1330 //
1331 // Make sure we win the race, and if we did, copy the data atomically
1332 //
1334 {
1335 RtlCopyMemory(Context->PoolTrackTable,
1337 Context->PoolTrackTableSize * sizeof(POOL_TRACKER_TABLE));
1338
1339 //
1340 // This is here because ReactOS does not yet support expansion
1341 //
1342 ASSERT(Context->PoolTrackTableSizeExpansion == 0);
1343 }
1344
1345 //
1346 // Regardless of whether we won or not, we must now synchronize and then
1347 // decrement the barrier since this is one more processor that has completed
1348 // the callback.
1349 //
1352}
BOOLEAN NTAPI KeSignalCallDpcSynchronize(IN PVOID SystemArgument2)
Definition: dpc.c:1026
VOID NTAPI KeSignalCallDpcDone(IN PVOID SystemArgument1)
Definition: dpc.c:1013
PPOOL_TRACKER_TABLE PoolTrackTable
Definition: expool.c:49
#define UNREFERENCED_PARAMETER(P)
Definition: ntbasedef.h:317
#define RtlCopyMemory(Destination, Source, Length)
Definition: typedefs.h:263
_Must_inspect_result_ _In_ PWDF_DPC_CONFIG _In_ PWDF_OBJECT_ATTRIBUTES _Out_ WDFDPC * Dpc
Definition: wdfdpc.h:112
_In_opt_ PVOID _In_opt_ PVOID SystemArgument1
Definition: ketypes.h:688
_In_opt_ PVOID DeferredContext
Definition: ketypes.h:687
_In_opt_ PVOID _In_opt_ PVOID _In_opt_ PVOID SystemArgument2
Definition: ketypes.h:689

Referenced by ExGetPoolTagInfo().

◆ ExpInitializePoolListHead()

VOID NTAPI ExpInitializePoolListHead ( IN PLIST_ENTRY  ListHead)

Definition at line 114 of file expool.c.

115{
116 ListHead->Flink = ListHead->Blink = ExpEncodePoolLink(ListHead);
117}
PLIST_ENTRY NTAPI ExpEncodePoolLink(IN PLIST_ENTRY Link)
Definition: expool.c:92

Referenced by ExInitializePoolDescriptor().

◆ ExpInsertPoolHeadList()

VOID NTAPI ExpInsertPoolHeadList ( IN PLIST_ENTRY  ListHead,
IN PLIST_ENTRY  Entry 
)

Definition at line 178 of file expool.c.

180{
181 PLIST_ENTRY Flink;
182 ExpCheckPoolLinks(ListHead);
183 Flink = ExpDecodePoolLink(ListHead->Flink);
184 Entry->Flink = ExpEncodePoolLink(Flink);
185 Entry->Blink = ExpEncodePoolLink(ListHead);
186 Flink->Blink = ExpEncodePoolLink(Entry);
187 ListHead->Flink = ExpEncodePoolLink(Entry);
188 ExpCheckPoolLinks(ListHead);
189}

Referenced by ExFreePoolWithTag().

◆ ExpInsertPoolTailList()

VOID NTAPI ExpInsertPoolTailList ( IN PLIST_ENTRY  ListHead,
IN PLIST_ENTRY  Entry 
)

Definition at line 163 of file expool.c.

165{
166 PLIST_ENTRY Blink;
167 ExpCheckPoolLinks(ListHead);
168 Blink = ExpDecodePoolLink(ListHead->Blink);
169 Entry->Flink = ExpEncodePoolLink(ListHead);
170 Entry->Blink = ExpEncodePoolLink(Blink);
171 Blink->Flink = ExpEncodePoolLink(Entry);
172 ListHead->Blink = ExpEncodePoolLink(Entry);
173 ExpCheckPoolLinks(ListHead);
174}

Referenced by ExAllocatePoolWithTag().

◆ ExpInsertPoolTracker()

VOID NTAPI ExpInsertPoolTracker ( IN ULONG  Key,
IN SIZE_T  NumberOfBytes,
IN POOL_TYPE  PoolType 
)

Definition at line 851 of file expool.c.

854{
858 SIZE_T TableMask, TableSize;
859
860 //
861 // Remove the PROTECTED_POOL flag which is not part of the tag
862 //
863 Key &= ~PROTECTED_POOL;
864
865 //
866 // With WinDBG you can set a tag you want to break on when an allocation is
867 // attempted
868 //
869 if (Key == PoolHitTag) DbgBreakPoint();
870
871 //
872 // There is also an internal flag you can set to break on malformed tags
873 //
874 if (ExStopBadTags) ASSERT(Key & 0xFFFFFF00);
875
876 //
877 // ASSERT on ReactOS features not yet supported
878 //
880
881 //
882 // Why the double indirection? Because normally this function is also used
883 // when doing session pool allocations, which has another set of tables,
884 // sizes, and masks that live in session pool. Now we don't support session
885 // pool so we only ever use the regular tables, but I'm keeping the code this
886 // way so that the day we DO support session pool, it won't require that
887 // many changes
888 //
890 TableMask = PoolTrackTableMask;
893
894 //
895 // Compute the hash for this key, and loop all the possible buckets
896 //
897 Hash = ExpComputeHashForTag(Key, TableMask);
898 Index = Hash;
899 while (TRUE)
900 {
901 //
902 // Do we already have an entry for this tag? */
903 //
905 if (TableEntry->Key == Key)
906 {
907 //
908 // Increment the counters depending on if this was paged or nonpaged
909 // pool
910 //
912 {
913 InterlockedIncrement(&TableEntry->NonPagedAllocs);
915 return;
916 }
917 InterlockedIncrement(&TableEntry->PagedAllocs);
919 return;
920 }
921
922 //
923 // We don't have an entry yet, but we've found a free bucket for it
924 //
925 if (!(TableEntry->Key) && (Hash != PoolTrackTableSize - 1))
926 {
927 //
928 // We need to hold the lock while creating a new entry, since other
929 // processors might be in this code path as well
930 //
932 if (!PoolTrackTable[Hash].Key)
933 {
934 //
935 // We've won the race, so now create this entry in the bucket
936 //
937 ASSERT(Table[Hash].Key == 0);
939 TableEntry->Key = Key;
940 }
942
943 //
944 // Now we force the loop to run again, and we should now end up in
945 // the code path above which does the interlocked increments...
946 //
947 continue;
948 }
949
950 //
951 // This path is hit when we don't have an entry, and the current bucket
952 // is full, so we simply try the next one
953 //
954 Hash = (Hash + 1) & TableMask;
955 if (Hash == Index) break;
956 }
957
958 //
959 // And finally this path is hit when all the buckets are full, and we need
960 // some expansion. This path is not yet supported in ReactOS and so we'll
961 // ignore the tag
962 //
963 DPRINT1("Out of pool tag space, ignoring...\n");
964}
FORCEINLINE ULONG ExpComputeHashForTag(IN ULONG Tag, IN SIZE_T BucketMask)
Definition: expool.c:441
BOOLEAN ExStopBadTags
Definition: expool.c:53
SIZE_T PoolTrackTableMask
Definition: expool.c:46
KSPIN_LOCK ExpTaggedPoolLock
Definition: expool.c:51
ULONG PoolHitTag
Definition: expool.c:52
ASMGENDATA Table[]
Definition: genincdata.c:61
#define DBG_UNREFERENCED_LOCAL_VARIABLE(L)
Definition: ntbasedef.h:319
_In_ WDFCOLLECTION _In_ ULONG Index
#define ExReleaseSpinLock(Lock, OldIrql)
#define ExAcquireSpinLock(Lock, OldIrql)
_Must_inspect_result_ typedef _In_ ULONG TableEntry
Definition: iotypes.h:4303

Referenced by _IRQL_requires_(), ExAllocatePoolWithTag(), and InitializePool().

◆ ExpIsPoolListEmpty()

BOOLEAN NTAPI ExpIsPoolListEmpty ( IN PLIST_ENTRY  ListHead)

Definition at line 121 of file expool.c.

122{
123 return (ExpDecodePoolLink(ListHead->Flink) == ListHead);
124}

Referenced by ExAllocatePoolWithTag().

◆ ExpRemovePoolEntryList()

VOID NTAPI ExpRemovePoolEntryList ( IN PLIST_ENTRY  Entry)

Definition at line 128 of file expool.c.

129{
130 PLIST_ENTRY Blink, Flink;
131 Flink = ExpDecodePoolLink(Entry->Flink);
132 Blink = ExpDecodePoolLink(Entry->Blink);
133 Flink->Blink = ExpEncodePoolLink(Blink);
134 Blink->Flink = ExpEncodePoolLink(Flink);
135}

Referenced by ExFreePoolWithTag().

◆ ExpRemovePoolHeadList()

PLIST_ENTRY NTAPI ExpRemovePoolHeadList ( IN PLIST_ENTRY  ListHead)

Definition at line 139 of file expool.c.

140{
141 PLIST_ENTRY Entry, Flink;
142 Entry = ExpDecodePoolLink(ListHead->Flink);
143 Flink = ExpDecodePoolLink(Entry->Flink);
144 ListHead->Flink = ExpEncodePoolLink(Flink);
145 Flink->Blink = ExpEncodePoolLink(ListHead);
146 return Entry;
147}

Referenced by ExAllocatePoolWithTag().

◆ ExpRemovePoolTailList()

PLIST_ENTRY NTAPI ExpRemovePoolTailList ( IN PLIST_ENTRY  ListHead)

Definition at line 151 of file expool.c.

152{
153 PLIST_ENTRY Entry, Blink;
154 Entry = ExpDecodePoolLink(ListHead->Blink);
155 Blink = ExpDecodePoolLink(Entry->Blink);
156 ListHead->Blink = ExpEncodePoolLink(Blink);
157 Blink->Flink = ExpEncodePoolLink(ListHead);
158 return Entry;
159}

◆ ExpRemovePoolTracker()

VOID NTAPI ExpRemovePoolTracker ( IN ULONG  Key,
IN SIZE_T  NumberOfBytes,
IN POOL_TYPE  PoolType 
)

Definition at line 760 of file expool.c.

763{
766 SIZE_T TableMask, TableSize;
767
768 //
769 // Remove the PROTECTED_POOL flag which is not part of the tag
770 //
771 Key &= ~PROTECTED_POOL;
772
773 //
774 // With WinDBG you can set a tag you want to break on when an allocation is
775 // attempted
776 //
777 if (Key == PoolHitTag) DbgBreakPoint();
778
779 //
780 // Why the double indirection? Because normally this function is also used
781 // when doing session pool allocations, which has another set of tables,
782 // sizes, and masks that live in session pool. Now we don't support session
783 // pool so we only ever use the regular tables, but I'm keeping the code this
784 // way so that the day we DO support session pool, it won't require that
785 // many changes
786 //
788 TableMask = PoolTrackTableMask;
791
792 //
793 // Compute the hash for this key, and loop all the possible buckets
794 //
795 Hash = ExpComputeHashForTag(Key, TableMask);
796 Index = Hash;
797 while (TRUE)
798 {
799 //
800 // Have we found the entry for this tag? */
801 //
803 if (TableEntry->Key == Key)
804 {
805 //
806 // Decrement the counters depending on if this was paged or nonpaged
807 // pool
808 //
810 {
811 InterlockedIncrement(&TableEntry->NonPagedFrees);
814 return;
815 }
816 InterlockedIncrement(&TableEntry->PagedFrees);
819 return;
820 }
821
822 //
823 // We should have only ended up with an empty entry if we've reached
824 // the last bucket
825 //
826 if (!TableEntry->Key)
827 {
828 DPRINT1("Empty item reached in tracker table. Hash=0x%lx, TableMask=0x%lx, Tag=0x%08lx, NumberOfBytes=%lu, PoolType=%d\n",
829 Hash, TableMask, Key, (ULONG)NumberOfBytes, PoolType);
830 ASSERT(Hash == TableMask);
831 }
832
833 //
834 // This path is hit when we don't have an entry, and the current bucket
835 // is full, so we simply try the next one
836 //
837 Hash = (Hash + 1) & TableMask;
838 if (Hash == Index) break;
839 }
840
841 //
842 // And finally this path is hit when all the buckets are full, and we need
843 // some expansion. This path is not yet supported in ReactOS and so we'll
844 // ignore the tag
845 //
846 DPRINT1("Out of pool tag space, ignoring...\n");
847}
LONG_PTR SSIZE_T
Definition: basetsd.h:181

Referenced by _IRQL_requires_(), and ExFreePoolWithTag().

◆ ExpSeedHotTags()

VOID NTAPI ExpSeedHotTags ( VOID  )

Definition at line 640 of file expool.c.

641{
642 ULONG i, Key, Hash, Index;
644 ULONG TagList[] =
645 {
646 ' oI',
647 ' laH',
648 'PldM',
649 'LooP',
650 'tSbO',
651 ' prI',
652 'bdDN',
653 'LprI',
654 'pOoI',
655 ' ldM',
656 'eliF',
657 'aVMC',
658 'dSeS',
659 'CFtN',
660 'looP',
661 'rPCT',
662 'bNMC',
663 'dTeS',
664 'sFtN',
665 'TPCT',
666 'CPCT',
667 ' yeK',
668 'qSbO',
669 'mNoI',
670 'aEoI',
671 'cPCT',
672 'aFtN',
673 '0ftN',
674 'tceS',
675 'SprI',
676 'ekoT',
677 ' eS',
678 'lCbO',
679 'cScC',
680 'lFtN',
681 'cAeS',
682 'mfSF',
683 'kWcC',
684 'miSF',
685 'CdfA',
686 'EdfA',
687 'orSF',
688 'nftN',
689 'PRIU',
690 'rFpN',
691 'RFpN',
692 'aPeS',
693 'sUeS',
694 'FpcA',
695 'MpcA',
696 'cSeS',
697 'mNbO',
698 'sFpN',
699 'uLeS',
700 'DPcS',
701 'nevE',
702 'vrqR',
703 'ldaV',
704 ' pP',
705 'SdaV',
706 ' daV',
707 'LdaV',
708 'FdaV',
709 ' GIB',
710 };
711
712 //
713 // Loop all 64 hot tags
714 //
715 ASSERT((sizeof(TagList) / sizeof(ULONG)) == 64);
716 for (i = 0; i < sizeof(TagList) / sizeof(ULONG); i++)
717 {
718 //
719 // Get the current tag, and compute its hash in the tracker table
720 //
721 Key = TagList[i];
723
724 //
725 // Loop all the hashes in this index/bucket
726 //
727 Index = Hash;
728 while (TRUE)
729 {
730 //
731 // Find an empty entry, and make sure this isn't the last hash that
732 // can fit.
733 //
734 // On checked builds, also make sure this is the first time we are
735 // seeding this tag.
736 //
737 ASSERT(TrackTable[Hash].Key != Key);
738 if (!(TrackTable[Hash].Key) && (Hash != PoolTrackTableSize - 1))
739 {
740 //
741 // It has been seeded, move on to the next tag
742 //
743 TrackTable[Hash].Key = Key;
744 break;
745 }
746
747 //
748 // This entry was already taken, compute the next possible hash while
749 // making sure we're not back at our initial index.
750 //
751 ASSERT(TrackTable[Hash].Key != Key);
752 Hash = (Hash + 1) & PoolTrackTableMask;
753 if (Hash == Index) break;
754 }
755 }
756}

Referenced by InitializePool().

◆ ExQueryPoolBlockSize()

SIZE_T NTAPI ExQueryPoolBlockSize ( IN PVOID  PoolBlock,
OUT PBOOLEAN  QuotaCharged 
)

Definition at line 2933 of file expool.c.

2935{
2936 //
2937 // Not implemented
2938 //
2940 return FALSE;
2941}

◆ ExQueryPoolUsage()

VOID NTAPI ExQueryPoolUsage ( OUT PULONG  PagedPoolPages,
OUT PULONG  NonPagedPoolPages,
OUT PULONG  PagedPoolAllocs,
OUT PULONG  PagedPoolFrees,
OUT PULONG  PagedPoolLookasideHits,
OUT PULONG  NonPagedPoolAllocs,
OUT PULONG  NonPagedPoolFrees,
OUT PULONG  NonPagedPoolLookasideHits 
)

Definition at line 1768 of file expool.c.

1776{
1777 ULONG i;
1778 PPOOL_DESCRIPTOR PoolDesc;
1779
1780 //
1781 // Assume all failures
1782 //
1783 *PagedPoolPages = 0;
1784 *PagedPoolAllocs = 0;
1785 *PagedPoolFrees = 0;
1786
1787 //
1788 // Tally up the totals for all the apged pool
1789 //
1790 for (i = 0; i < ExpNumberOfPagedPools + 1; i++)
1791 {
1792 PoolDesc = ExpPagedPoolDescriptor[i];
1793 *PagedPoolPages += PoolDesc->TotalPages + PoolDesc->TotalBigPages;
1794 *PagedPoolAllocs += PoolDesc->RunningAllocs;
1795 *PagedPoolFrees += PoolDesc->RunningDeAllocs;
1796 }
1797
1798 //
1799 // The first non-paged pool has a hardcoded well-known descriptor name
1800 //
1801 PoolDesc = &NonPagedPoolDescriptor;
1802 *NonPagedPoolPages = PoolDesc->TotalPages + PoolDesc->TotalBigPages;
1803 *NonPagedPoolAllocs = PoolDesc->RunningAllocs;
1804 *NonPagedPoolFrees = PoolDesc->RunningDeAllocs;
1805
1806 //
1807 // If the system has more than one non-paged pool, copy the other descriptor
1808 // totals as well
1809 //
1810#if 0
1811 if (ExpNumberOfNonPagedPools > 1)
1812 {
1813 for (i = 0; i < ExpNumberOfNonPagedPools; i++)
1814 {
1815 PoolDesc = ExpNonPagedPoolDescriptor[i];
1816 *NonPagedPoolPages += PoolDesc->TotalPages + PoolDesc->TotalBigPages;
1817 *NonPagedPoolAllocs += PoolDesc->RunningAllocs;
1818 *NonPagedPoolFrees += PoolDesc->RunningDeAllocs;
1819 }
1820 }
1821#endif
1822
1823 //
1824 // Get the amount of hits in the system lookaside lists
1825 //
1827 {
1828 PLIST_ENTRY ListEntry;
1829
1830 for (ListEntry = ExPoolLookasideListHead.Flink;
1831 ListEntry != &ExPoolLookasideListHead;
1832 ListEntry = ListEntry->Flink)
1833 {
1835
1836 Lookaside = CONTAINING_RECORD(ListEntry, GENERAL_LOOKASIDE, ListEntry);
1837
1838 if (Lookaside->Type == NonPagedPool)
1839 {
1840 *NonPagedPoolLookasideHits += Lookaside->AllocateHits;
1841 }
1842 else
1843 {
1844 *PagedPoolLookasideHits += Lookaside->AllocateHits;
1845 }
1846 }
1847 }
1848}
#define IsListEmpty(ListHead)
Definition: env_spec_w32.h:954
ULONG ExpNumberOfPagedPools
Definition: expool.c:41
PPOOL_DESCRIPTOR ExpPagedPoolDescriptor[16+1]
Definition: expool.c:43
LIST_ENTRY ExPoolLookasideListHead
Definition: lookas.c:22
#define CONTAINING_RECORD(address, type, field)
Definition: typedefs.h:260
_Must_inspect_result_ _In_opt_ PWDF_OBJECT_ATTRIBUTES _In_ _In_ _Strict_type_match_ POOL_TYPE _In_opt_ PWDF_OBJECT_ATTRIBUTES _In_opt_ ULONG _Out_ WDFLOOKASIDE * Lookaside
Definition: wdfmemory.h:414
struct LOOKASIDE_ALIGN _GENERAL_LOOKASIDE GENERAL_LOOKASIDE

Referenced by QSI_DEF().

◆ ExReturnPoolQuota()

VOID NTAPI ExReturnPoolQuota ( IN PVOID  P)

Definition at line 1852 of file expool.c.

1853{
1856 USHORT BlockSize;
1858
1861 {
1862 return;
1863 }
1864
1865 Entry = P;
1866 Entry--;
1868
1869 PoolType = Entry->PoolType - 1;
1870 BlockSize = Entry->BlockSize;
1871
1873 {
1874 Process = ((PVOID *)POOL_NEXT_BLOCK(Entry))[-1];
1875 ASSERT(Process != NULL);
1876 if (Process)
1877 {
1878 if (Process->Pcb.Header.Type != ProcessObject)
1879 {
1880 DPRINT1("Object %p is not a process. Type %u, pool type 0x%x, block size %u\n",
1881 Process, Process->Pcb.Header.Type, Entry->PoolType, BlockSize);
1882 KeBugCheckEx(BAD_POOL_CALLER,
1884 (ULONG_PTR)P,
1885 Entry->PoolTag,
1887 }
1888 ((PVOID *)POOL_NEXT_BLOCK(Entry))[-1] = NULL;
1891 BlockSize * POOL_BLOCK_SIZE);
1893 }
1894 }
1895}

Referenced by IoFreeIrp().

◆ ExUnlockPool()

FORCEINLINE VOID ExUnlockPool ( IN PPOOL_DESCRIPTOR  Descriptor,
IN KIRQL  OldIrql 
)

Definition at line 1297 of file expool.c.

1299{
1300 //
1301 // Check if this is nonpaged pool
1302 //
1303 if ((Descriptor->PoolType & BASE_POOL_TYPE_MASK) == NonPagedPool)
1304 {
1305 //
1306 // Use the queued spin lock
1307 //
1309 }
1310 else
1311 {
1312 //
1313 // Use the guarded mutex
1314 //
1315 KeReleaseGuardedMutex(Descriptor->LockAddress);
1316 }
1317}
VOID FASTCALL KeReleaseGuardedMutex(IN OUT PKGUARDED_MUTEX GuardedMutex)
Definition: gmutex.c:53
VOID FASTCALL KeReleaseQueuedSpinLock(IN KSPIN_LOCK_QUEUE_NUMBER LockNumber, IN KIRQL OldIrql)
Definition: spinlock.c:154

Referenced by ExAllocatePoolWithTag(), and ExFreePoolWithTag().

◆ InitializePool()

VOID NTAPI InitializePool ( IN POOL_TYPE  PoolType,
IN ULONG  Threshold 
)

Definition at line 1020 of file expool.c.

1022{
1025 ULONG i;
1026
1027 //
1028 // Check what kind of pool this is
1029 //
1030 if (PoolType == NonPagedPool)
1031 {
1032 //
1033 // Compute the track table size and convert it from a power of two to an
1034 // actual byte size
1035 //
1036 // NOTE: On checked builds, we'll assert if the registry table size was
1037 // invalid, while on retail builds we'll just break out of the loop at
1038 // that point.
1039 //
1041 for (i = 0; i < 32; i++)
1042 {
1043 if (TableSize & 1)
1044 {
1045 ASSERT((TableSize & ~1) == 0);
1046 if (!(TableSize & ~1)) break;
1047 }
1048 TableSize >>= 1;
1049 }
1050
1051 //
1052 // If we hit bit 32, than no size was defined in the registry, so
1053 // we'll use the default size of 2048 entries.
1054 //
1055 // Otherwise, use the size from the registry, as long as it's not
1056 // smaller than 64 entries.
1057 //
1058 if (i == 32)
1059 {
1060 PoolTrackTableSize = 2048;
1061 }
1062 else
1063 {
1064 PoolTrackTableSize = max(1 << i, 64);
1065 }
1066
1067 //
1068 // Loop trying with the biggest specified size first, and cut it down
1069 // by a power of two each iteration in case not enough memory exist
1070 //
1071 while (TRUE)
1072 {
1073 //
1074 // Do not allow overflow
1075 //
1076 if ((PoolTrackTableSize + 1) > (MAXULONG_PTR / sizeof(POOL_TRACKER_TABLE)))
1077 {
1078 PoolTrackTableSize >>= 1;
1079 continue;
1080 }
1081
1082 //
1083 // Allocate the tracker table and exit the loop if this worked
1084 //
1086 (PoolTrackTableSize + 1) *
1087 sizeof(POOL_TRACKER_TABLE));
1088 if (PoolTrackTable) break;
1089
1090 //
1091 // Otherwise, as long as we're not down to the last bit, keep
1092 // iterating
1093 //
1094 if (PoolTrackTableSize == 1)
1095 {
1096 KeBugCheckEx(MUST_SUCCEED_POOL_EMPTY,
1097 TableSize,
1098 0xFFFFFFFF,
1099 0xFFFFFFFF,
1100 0xFFFFFFFF);
1101 }
1102 PoolTrackTableSize >>= 1;
1103 }
1104
1105 //
1106 // Add one entry, compute the hash, and zero the table
1107 //
1110
1113
1114 //
1115 // Finally, add the most used tags to speed up those allocations
1116 //
1118
1119 //
1120 // We now do the exact same thing with the tracker table for big pages
1121 //
1123 for (i = 0; i < 32; i++)
1124 {
1125 if (TableSize & 1)
1126 {
1127 ASSERT((TableSize & ~1) == 0);
1128 if (!(TableSize & ~1)) break;
1129 }
1130 TableSize >>= 1;
1131 }
1132
1133 //
1134 // For big pages, the default tracker table is 4096 entries, while the
1135 // minimum is still 64
1136 //
1137 if (i == 32)
1138 {
1139 PoolBigPageTableSize = 4096;
1140 }
1141 else
1142 {
1143 PoolBigPageTableSize = max(1 << i, 64);
1144 }
1145
1146 //
1147 // Again, run the exact same loop we ran earlier, but this time for the
1148 // big pool tracker instead
1149 //
1150 while (TRUE)
1151 {
1153 {
1155 continue;
1156 }
1157
1160 sizeof(POOL_TRACKER_BIG_PAGES));
1161 if (PoolBigPageTable) break;
1162
1163 if (PoolBigPageTableSize == 1)
1164 {
1165 KeBugCheckEx(MUST_SUCCEED_POOL_EMPTY,
1166 TableSize,
1167 0xFFFFFFFF,
1168 0xFFFFFFFF,
1169 0xFFFFFFFF);
1170 }
1171
1173 }
1174
1175 //
1176 // An extra entry is not needed for for the big pool tracker, so just
1177 // compute the hash and zero it
1178 //
1182 for (i = 0; i < PoolBigPageTableSize; i++)
1183 {
1185 }
1186
1187 //
1188 // During development, print this out so we can see what's happening
1189 //
1190 DPRINT("EXPOOL: Pool Tracker Table at: 0x%p with 0x%lx bytes\n",
1192 DPRINT("EXPOOL: Big Pool Tracker Table at: 0x%p with 0x%lx bytes\n",
1194
1195 //
1196 // Insert the generic tracker for all of big pool
1197 //
1198 ExpInsertPoolTracker('looP',
1200 sizeof(POOL_TRACKER_BIG_PAGES)),
1201 NonPagedPool);
1202
1203 //
1204 // No support for NUMA systems at this time
1205 //
1206 ASSERT(KeNumberNodes == 1);
1207
1208 //
1209 // Initialize the tag spinlock
1210 //
1212
1213 //
1214 // Initialize the nonpaged pool descriptor
1215 //
1219 0,
1220 Threshold,
1221 NULL);
1222 }
1223 else
1224 {
1225 //
1226 // No support for NUMA systems at this time
1227 //
1228 ASSERT(KeNumberNodes == 1);
1229
1230 //
1231 // Allocate the pool descriptor
1232 //
1234 sizeof(KGUARDED_MUTEX) +
1235 sizeof(POOL_DESCRIPTOR),
1236 'looP');
1237 if (!Descriptor)
1238 {
1239 //
1240 // This is really bad...
1241 //
1242 KeBugCheckEx(MUST_SUCCEED_POOL_EMPTY,
1243 0,
1244 -1,
1245 -1,
1246 -1);
1247 }
1248
1249 //
1250 // Setup the vector and guarded mutex for paged pool
1251 //
1257 PagedPool,
1258 0,
1259 Threshold,
1261
1262 //
1263 // Insert the generic tracker for all of nonpaged pool
1264 //
1265 ExpInsertPoolTracker('looP',
1267 NonPagedPool);
1268 }
1269}
#define MAXULONG_PTR
Definition: basetsd.h:103
#define KeInitializeSpinLock(sl)
Definition: env_spec_w32.h:604
VOID NTAPI ExInitializePoolDescriptor(IN PPOOL_DESCRIPTOR PoolDescriptor, IN POOL_TYPE PoolType, IN ULONG PoolIndex, IN ULONG Threshold, IN PVOID PoolLock)
Definition: expool.c:969
PKGUARDED_MUTEX ExpPagedPoolMutex
Definition: expool.c:45
VOID NTAPI ExpSeedHotTags(VOID)
Definition: expool.c:640
VOID FASTCALL KeInitializeGuardedMutex(OUT PKGUARDED_MUTEX GuardedMutex)
Definition: gmutex.c:31
UCHAR KeNumberNodes
Definition: krnlinit.c:40
ULONG MmSizeOfNonPagedPoolInBytes
Definition: init.c:21
#define max(a, b)
Definition: svc.c:63
struct _KGUARDED_MUTEX * PKGUARDED_MUTEX

Referenced by MiBuildPagedPool(), and MiInitMachineDependent().

Variable Documentation

◆ ExpBigTableExpansionFailed

ULONG ExpBigTableExpansionFailed

Definition at line 48 of file expool.c.

Referenced by ExpAddTagForBigPages().

◆ ExpLargePoolTableLock

KSPIN_LOCK ExpLargePoolTableLock

◆ ExpNumberOfPagedPools

ULONG ExpNumberOfPagedPools

Definition at line 41 of file expool.c.

Referenced by ExQueryPoolUsage().

◆ ExPoolFailures

ULONG ExPoolFailures

Definition at line 57 of file expool.c.

Referenced by ExAllocatePoolWithTag().

◆ ExpPagedPoolDescriptor

PPOOL_DESCRIPTOR ExpPagedPoolDescriptor[16+1]

Definition at line 43 of file expool.c.

Referenced by ExQueryPoolUsage(), and InitializePool().

◆ ExpPagedPoolMutex

PKGUARDED_MUTEX ExpPagedPoolMutex

Definition at line 45 of file expool.c.

Referenced by InitializePool().

◆ ExpPoolBigEntriesInUse

ULONG ExpPoolBigEntriesInUse

Definition at line 55 of file expool.c.

Referenced by _IRQL_requires_(), ExpAddTagForBigPages(), and ExpFindAndRemoveTagBigPages().

◆ ExpPoolFlags

◆ ExpTaggedPoolLock

KSPIN_LOCK ExpTaggedPoolLock

Definition at line 51 of file expool.c.

Referenced by ExpInsertPoolTracker(), and InitializePool().

◆ ExStopBadTags

BOOLEAN ExStopBadTags

Definition at line 53 of file expool.c.

Referenced by ExpInsertPoolTracker().

◆ MiLastPoolDumpTime

ULONGLONG MiLastPoolDumpTime

Definition at line 58 of file expool.c.

Referenced by ExAllocatePoolWithTag().

◆ NonPagedPoolDescriptor

POOL_DESCRIPTOR NonPagedPoolDescriptor

Definition at line 42 of file expool.c.

Referenced by ExAllocatePoolWithTag(), ExQueryPoolUsage(), and InitializePool().

◆ PoolBigPageTable

◆ PoolBigPageTableHash

SIZE_T PoolBigPageTableHash

◆ PoolBigPageTableSize

◆ PoolHitTag

ULONG PoolHitTag

Definition at line 52 of file expool.c.

Referenced by ExpInsertPoolTracker(), and ExpRemovePoolTracker().

◆ PoolTrackTable

◆ PoolTrackTableMask

SIZE_T PoolTrackTableMask

◆ PoolTrackTableSize

SIZE_T PoolTrackTableSize

◆ PoolVector

PPOOL_DESCRIPTOR PoolVector[2]

Definition at line 44 of file expool.c.

Referenced by ExAllocatePoolWithTag(), ExFreePoolWithTag(), and InitializePool().