expool.c File Reference

#include <ntoskrnl.h>
#include <debug.h>
#include <mm/ARM3/miarm.h>

Include dependency graph for expool.c:

Go to the source code of this file.

Classes
struct	_POOL_DPC_CONTEXT

Macros
#define	NDEBUG
#define	MODULE_INVOLVED_IN_ARM3
#define	POOL_BIG_TABLE_ENTRY_FREE   0x1
#define	POOL_BIG_TABLE_USE_RATE   4
#define	POOL_ENTRY(x)   (PPOOL_HEADER)((ULONG_PTR)(x) - sizeof(POOL_HEADER))
#define	POOL_FREE_BLOCK(x)   (PLIST_ENTRY)((ULONG_PTR)(x) + sizeof(POOL_HEADER))
#define	POOL_BLOCK(x, i)   (PPOOL_HEADER)((ULONG_PTR)(x) + ((i) * POOL_BLOCK_SIZE))
#define	POOL_NEXT_BLOCK(x)   POOL_BLOCK((x), (x)->BlockSize)
#define	POOL_PREV_BLOCK(x)   POOL_BLOCK((x), -((x)->PreviousSize))

Typedefs
typedef struct _POOL_DPC_CONTEXT	POOL_DPC_CONTEXT
typedef struct _POOL_DPC_CONTEXT *	PPOOL_DPC_CONTEXT

Functions
PLIST_ENTRY NTAPI	ExpDecodePoolLink (IN PLIST_ENTRY Link)
PLIST_ENTRY NTAPI	ExpEncodePoolLink (IN PLIST_ENTRY Link)
VOID NTAPI	ExpCheckPoolLinks (IN PLIST_ENTRY ListHead)
VOID NTAPI	ExpInitializePoolListHead (IN PLIST_ENTRY ListHead)
BOOLEAN NTAPI	ExpIsPoolListEmpty (IN PLIST_ENTRY ListHead)
VOID NTAPI	ExpRemovePoolEntryList (IN PLIST_ENTRY Entry)
PLIST_ENTRY NTAPI	ExpRemovePoolHeadList (IN PLIST_ENTRY ListHead)
PLIST_ENTRY NTAPI	ExpRemovePoolTailList (IN PLIST_ENTRY ListHead)
VOID NTAPI	ExpInsertPoolTailList (IN PLIST_ENTRY ListHead, IN PLIST_ENTRY Entry)
VOID NTAPI	ExpInsertPoolHeadList (IN PLIST_ENTRY ListHead, IN PLIST_ENTRY Entry)
VOID NTAPI	ExpCheckPoolHeader (IN PPOOL_HEADER Entry)
VOID NTAPI	ExpCheckPoolAllocation (PVOID P, POOL_TYPE PoolType, ULONG Tag)
VOID NTAPI	ExpCheckPoolBlocks (IN PVOID Block)
FORCEINLINE VOID	ExpCheckPoolIrqlLevel (IN POOL_TYPE PoolType, IN SIZE_T NumberOfBytes, IN PVOID Entry)
FORCEINLINE ULONG	ExpComputeHashForTag (IN ULONG Tag, IN SIZE_T BucketMask)
FORCEINLINE ULONG	ExpComputePartialHashForAddress (IN PVOID BaseAddress)
VOID NTAPI	ExpSeedHotTags (VOID)
VOID NTAPI	ExpRemovePoolTracker (IN ULONG Key, IN SIZE_T NumberOfBytes, IN POOL_TYPE PoolType)
VOID NTAPI	ExpInsertPoolTracker (IN ULONG Key, IN SIZE_T NumberOfBytes, IN POOL_TYPE PoolType)
VOID NTAPI	ExInitializePoolDescriptor (IN PPOOL_DESCRIPTOR PoolDescriptor, IN POOL_TYPE PoolType, IN ULONG PoolIndex, IN ULONG Threshold, IN PVOID PoolLock)
VOID NTAPI	InitializePool (IN POOL_TYPE PoolType, IN ULONG Threshold)
FORCEINLINE KIRQL	ExLockPool (IN PPOOL_DESCRIPTOR Descriptor)
FORCEINLINE VOID	ExUnlockPool (IN PPOOL_DESCRIPTOR Descriptor, IN KIRQL OldIrql)
VOID NTAPI	ExpGetPoolTagInfoTarget (IN PKDPC Dpc, IN PVOID DeferredContext, IN PVOID SystemArgument1, IN PVOID SystemArgument2)
NTSTATUS NTAPI	ExGetPoolTagInfo (IN PSYSTEM_POOLTAG_INFORMATION SystemInformation, IN ULONG SystemInformationLength, IN OUT PULONG ReturnLength OPTIONAL)
	_IRQL_requires_ (DISPATCH_LEVEL)
BOOLEAN NTAPI	ExpAddTagForBigPages (IN PVOID Va, IN ULONG Key, IN ULONG NumberOfPages, IN POOL_TYPE PoolType)
ULONG NTAPI	ExpFindAndRemoveTagBigPages (IN PVOID Va, OUT PULONG_PTR BigPages, IN POOL_TYPE PoolType)
VOID NTAPI	ExQueryPoolUsage (OUT PULONG PagedPoolPages, OUT PULONG NonPagedPoolPages, OUT PULONG PagedPoolAllocs, OUT PULONG PagedPoolFrees, OUT PULONG PagedPoolLookasideHits, OUT PULONG NonPagedPoolAllocs, OUT PULONG NonPagedPoolFrees, OUT PULONG NonPagedPoolLookasideHits)
VOID NTAPI	ExReturnPoolQuota (IN PVOID P)
PVOID NTAPI	ExAllocatePoolWithTag (IN POOL_TYPE PoolType, IN SIZE_T NumberOfBytes, IN ULONG Tag)
PVOID NTAPI	ExAllocatePool (POOL_TYPE PoolType, SIZE_T NumberOfBytes)
VOID NTAPI	ExFreePoolWithTag (IN PVOID P, IN ULONG TagToFree)
VOID NTAPI	ExFreePool (PVOID P)
SIZE_T NTAPI	ExQueryPoolBlockSize (IN PVOID PoolBlock, OUT PBOOLEAN QuotaCharged)
PVOID NTAPI	ExAllocatePoolWithQuota (IN POOL_TYPE PoolType, IN SIZE_T NumberOfBytes)
PVOID NTAPI	ExAllocatePoolWithTagPriority (IN POOL_TYPE PoolType, IN SIZE_T NumberOfBytes, IN ULONG Tag, IN EX_POOL_PRIORITY Priority)
PVOID NTAPI	ExAllocatePoolWithQuotaTag (IN POOL_TYPE PoolType, IN SIZE_T NumberOfBytes, IN ULONG Tag)

Variables
ULONG	ExpNumberOfPagedPools
POOL_DESCRIPTOR	NonPagedPoolDescriptor
PPOOL_DESCRIPTOR	ExpPagedPoolDescriptor [16+1]
PPOOL_DESCRIPTOR	PoolVector [2]
PKGUARDED_MUTEX	ExpPagedPoolMutex
SIZE_T	PoolTrackTableSize
SIZE_T	PoolTrackTableMask
SIZE_T	PoolBigPageTableSize
SIZE_T	PoolBigPageTableHash
ULONG	ExpBigTableExpansionFailed
PPOOL_TRACKER_TABLE	PoolTrackTable
PPOOL_TRACKER_BIG_PAGES	PoolBigPageTable
KSPIN_LOCK	ExpTaggedPoolLock
ULONG	PoolHitTag
BOOLEAN	ExStopBadTags
KSPIN_LOCK	ExpLargePoolTableLock
ULONG	ExpPoolBigEntriesInUse
ULONG	ExpPoolFlags
ULONG	ExPoolFailures
ULONGLONG	MiLastPoolDumpTime

Macro Definition Documentation

MODULE_INVOLVED_IN_ARM3

#define MODULE_INVOLVED_IN_ARM3

Definition at line 15 of file expool.c.

NDEBUG

#define NDEBUG

Definition at line 12 of file expool.c.

POOL_BIG_TABLE_ENTRY_FREE

#define POOL_BIG_TABLE_ENTRY_FREE   0x1

Definition at line 23 of file expool.c.

POOL_BIG_TABLE_USE_RATE

#define POOL_BIG_TABLE_USE_RATE   4

Definition at line 31 of file expool.c.

POOL_BLOCK

#define POOL_BLOCK	(		x,
			i
	)		(PPOOL_HEADER)((ULONG_PTR)(x) + ((i) * POOL_BLOCK_SIZE))

Definition at line 63 of file expool.c.

POOL_ENTRY

#define POOL_ENTRY

(

x

)

(PPOOL_HEADER)((ULONG_PTR)(x) - sizeof(POOL_HEADER))

Definition at line 61 of file expool.c.

POOL_FREE_BLOCK

#define POOL_FREE_BLOCK

(

x

)

(PLIST_ENTRY)((ULONG_PTR)(x) + sizeof(POOL_HEADER))

Definition at line 62 of file expool.c.

POOL_NEXT_BLOCK

#define POOL_NEXT_BLOCK

(

x

)

POOL_BLOCK((x), (x)->BlockSize)

Definition at line 64 of file expool.c.

POOL_PREV_BLOCK

#define POOL_PREV_BLOCK

(

x

)

POOL_BLOCK((x), -((x)->PreviousSize))

Definition at line 65 of file expool.c.

Typedef Documentation

POOL_DPC_CONTEXT

typedef struct _POOL_DPC_CONTEXT POOL_DPC_CONTEXT

PPOOL_DPC_CONTEXT

typedef struct _POOL_DPC_CONTEXT * PPOOL_DPC_CONTEXT

Function Documentation

_IRQL_requires_()

_IRQL_requires_

(

DISPATCH_LEVEL

)

Definition at line 1459 of file expool.c.

{
    SIZE_T OldSize = PoolBigPageTableSize;
    SIZE_T NewSize, NewSizeInBytes;
    PPOOL_TRACKER_BIG_PAGES NewTable;
    PPOOL_TRACKER_BIG_PAGES OldTable;
    ULONG i;
    ULONG PagesFreed;
    ULONG Hash;
    ULONG HashMask;
 
    /* Must be holding ExpLargePoolTableLock */
    ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
 
    /* Make sure we don't overflow */
    if (Shrink)
    {
        NewSize = OldSize / 2;
 
        /* Make sure we don't shrink too much. */
        ASSERT(NewSize >= ExpPoolBigEntriesInUse);
 
        NewSize = ALIGN_UP_BY(NewSize, PAGE_SIZE / sizeof(POOL_TRACKER_BIG_PAGES));
        ASSERT(NewSize <= OldSize);
 
        /* If there is only one page left, then keep it around. Not a failure either. */
        if (NewSize == OldSize)
        {
            ASSERT(NewSize == (PAGE_SIZE / sizeof(POOL_TRACKER_BIG_PAGES)));
            KeReleaseSpinLock(&ExpLargePoolTableLock, OldIrql);
            return TRUE;
        }
    }
    else
    {
        if (!NT_SUCCESS(RtlSIZETMult(2, OldSize, &NewSize)))
        {
            DPRINT1("Overflow expanding big page table. Size=%lu\n", OldSize);
            KeReleaseSpinLock(&ExpLargePoolTableLock, OldIrql);
            return FALSE;
        }
 
        /* Make sure we don't stupidly waste pages */
        NewSize = ALIGN_DOWN_BY(NewSize, PAGE_SIZE / sizeof(POOL_TRACKER_BIG_PAGES));
        ASSERT(NewSize > OldSize);
    }
 
    if (!NT_SUCCESS(RtlSIZETMult(sizeof(POOL_TRACKER_BIG_PAGES), NewSize, &NewSizeInBytes)))
    {
        DPRINT1("Overflow while calculating big page table size. Size=%lu\n", OldSize);
        KeReleaseSpinLock(&ExpLargePoolTableLock, OldIrql);
        return FALSE;
    }
 
    NewTable = MiAllocatePoolPages(NonPagedPool, NewSizeInBytes);
    if (NewTable == NULL)
    {
        DPRINT("Could not allocate %lu bytes for new big page table\n", NewSizeInBytes);
        KeReleaseSpinLock(&ExpLargePoolTableLock, OldIrql);
        return FALSE;
    }
 
    DPRINT("%s big pool tracker table to %lu entries\n", Shrink ? "Shrinking" : "Expanding", NewSize);
 
    /* Initialize the new table */
    RtlZeroMemory(NewTable, NewSizeInBytes);
    for (i = 0; i < NewSize; i++)
    {
        NewTable[i].Va = (PVOID)POOL_BIG_TABLE_ENTRY_FREE;
    }
 
    /* Copy over all items */
    OldTable = PoolBigPageTable;
    HashMask = NewSize - 1;
    for (i = 0; i < OldSize; i++)
    {
        /* Skip over empty items */
        if ((ULONG_PTR)OldTable[i].Va & POOL_BIG_TABLE_ENTRY_FREE)
        {
            continue;
        }
 
        /* Recalculate the hash due to the new table size */
        Hash = ExpComputePartialHashForAddress(OldTable[i].Va) % HashMask;
 
        /* Find the location in the new table */
        while (!((ULONG_PTR)NewTable[Hash].Va & POOL_BIG_TABLE_ENTRY_FREE))
        {
            if (++Hash == NewSize)
                Hash = 0;
        }
 
        /* We must have space */
        ASSERT((ULONG_PTR)NewTable[Hash].Va & POOL_BIG_TABLE_ENTRY_FREE);
 
        /* Finally, copy the item */
        NewTable[Hash] = OldTable[i];
    }
 
    /* Activate the new table */
    PoolBigPageTable = NewTable;
    PoolBigPageTableSize = NewSize;
    PoolBigPageTableHash = PoolBigPageTableSize - 1;
 
    /* Release the lock, we're done changing global state */
    KeReleaseSpinLock(&ExpLargePoolTableLock, OldIrql);
 
    /* Free the old table and update our tracker */
    PagesFreed = MiFreePoolPages(OldTable);
    ExpRemovePoolTracker('looP', PagesFreed << PAGE_SHIFT, 0);
    ExpInsertPoolTracker('looP', ALIGN_UP_BY(NewSizeInBytes, PAGE_SIZE), 0);
 
    return TRUE;
}

ExAllocatePool()

PVOID NTAPI ExAllocatePool	(	POOL_TYPE	PoolType,
		SIZE_T	NumberOfBytes
	)

Definition at line 2465 of file expool.c.

{
    ULONG Tag = TAG_NONE;
#if 0 && DBG
    PLDR_DATA_TABLE_ENTRY LdrEntry;
 
    /* Use the first four letters of the driver name, or "None" if unavailable */
    LdrEntry = KeGetCurrentIrql() <= APC_LEVEL
                ? MiLookupDataTableEntry(_ReturnAddress())
                : NULL;
    if (LdrEntry)
    {
        ULONG i;
        Tag = 0;
        for (i = 0; i < min(4, LdrEntry->BaseDllName.Length / sizeof(WCHAR)); i++)
            Tag = Tag >> 8 | (LdrEntry->BaseDllName.Buffer[i] & 0xff) << 24;
        for (; i < 4; i++)
            Tag = Tag >> 8 | ' ' << 24;
    }
#endif
    return ExAllocatePoolWithTag(PoolType, NumberOfBytes, Tag);
}

ExAllocatePoolWithQuota()

PVOID NTAPI ExAllocatePoolWithQuota	(	IN POOL_TYPE	PoolType,
		IN SIZE_T	NumberOfBytes
	)

Definition at line 2949 of file expool.c.

{
    //
    // Allocate the pool
    //
    return ExAllocatePoolWithQuotaTag(PoolType, NumberOfBytes, TAG_NONE);
}

ExAllocatePoolWithQuotaTag()

PVOID NTAPI ExAllocatePoolWithQuotaTag	(	IN POOL_TYPE	PoolType,
		IN SIZE_T	NumberOfBytes,
		IN ULONG	Tag
	)

Definition at line 2987 of file expool.c.

{
    BOOLEAN Raise = TRUE;
    PVOID Buffer;
    PPOOL_HEADER Entry;
    NTSTATUS Status;
    PEPROCESS Process = PsGetCurrentProcess();
 
    //
    // Check if we should fail instead of raising an exception
    //
    if (PoolType & POOL_QUOTA_FAIL_INSTEAD_OF_RAISE)
    {
        Raise = FALSE;
        PoolType &= ~POOL_QUOTA_FAIL_INSTEAD_OF_RAISE;
    }
 
    //
    // Inject the pool quota mask
    //
    PoolType += QUOTA_POOL_MASK;
 
    //
    // Check if we have enough space to add the quota owner process, as long as
    // this isn't the system process, which never gets charged quota
    //
    ASSERT(NumberOfBytes != 0);
    if ((NumberOfBytes <= (PAGE_SIZE - POOL_BLOCK_SIZE - sizeof(PVOID))) &&
        (Process != PsInitialSystemProcess))
    {
        //
        // Add space for our EPROCESS pointer
        //
        NumberOfBytes += sizeof(PEPROCESS);
    }
    else
    {
        //
        // We won't be able to store the pointer, so don't use quota for this
        //
        PoolType -= QUOTA_POOL_MASK;
    }
 
    //
    // Allocate the pool buffer now
    //
    Buffer = ExAllocatePoolWithTag(PoolType, NumberOfBytes, Tag);
 
    //
    // If the buffer is page-aligned, this is a large page allocation and we
    // won't touch it
    //
    if (PAGE_ALIGN(Buffer) != Buffer)
    {
        //
        // Also if special pool is enabled, and this was allocated from there,
        // we won't touch it either
        //
        if ((ExpPoolFlags & POOL_FLAG_SPECIAL_POOL) &&
            (MmIsSpecialPoolAddress(Buffer)))
        {
            return Buffer;
        }
 
        //
        // If it wasn't actually allocated with quota charges, ignore it too
        //
        if (!(PoolType & QUOTA_POOL_MASK)) return Buffer;
 
        //
        // If this is the system process, we don't charge quota, so ignore
        //
        if (Process == PsInitialSystemProcess) return Buffer;
 
        //
        // Actually go and charge quota for the process now
        //
        Entry = POOL_ENTRY(Buffer);
        Status = PsChargeProcessPoolQuota(Process,
                                          PoolType & BASE_POOL_TYPE_MASK,
                                          Entry->BlockSize * POOL_BLOCK_SIZE);
        if (!NT_SUCCESS(Status))
        {
            //
            // Quota failed, back out the allocation, clear the owner, and fail
            //
            ((PVOID *)POOL_NEXT_BLOCK(Entry))[-1] = NULL;
            ExFreePoolWithTag(Buffer, Tag);
            if (Raise) RtlRaiseStatus(Status);
            return NULL;
        }
 
        //
        // Quota worked, write the owner and then reference it before returning
        //
        ((PVOID *)POOL_NEXT_BLOCK(Entry))[-1] = Process;
        ObReferenceObject(Process);
    }
    else if (!(Buffer) && (Raise))
    {
        //
        // The allocation failed, raise an error if we are in raise mode
        //
        RtlRaiseStatus(STATUS_INSUFFICIENT_RESOURCES);
    }
 
    //
    // Return the allocated buffer
    //
    return Buffer;
}

ExAllocatePoolWithTag()

PVOID NTAPI ExAllocatePoolWithTag	(	IN POOL_TYPE	PoolType,
		IN SIZE_T	NumberOfBytes,
		IN ULONG	Tag
	)

Definition at line 1904 of file expool.c.

{
    PPOOL_DESCRIPTOR PoolDesc;
    PLIST_ENTRY ListHead;
    PPOOL_HEADER Entry, NextEntry, FragmentEntry;
    KIRQL OldIrql;
    USHORT BlockSize, i;
    ULONG OriginalType;
    PKPRCB Prcb = KeGetCurrentPrcb();
    PGENERAL_LOOKASIDE LookasideList;
 
    //
    // Some sanity checks
    //
    ASSERT(Tag != 0);
    ASSERT(Tag != ' GIB');
    ASSERT(NumberOfBytes != 0);
    ExpCheckPoolIrqlLevel(PoolType, NumberOfBytes, NULL);
 
    //
    // Not supported in ReactOS
    //
    ASSERT(!(PoolType & SESSION_POOL_MASK));
 
    //
    // Check if verifier or special pool is enabled
    //
    if (ExpPoolFlags & (POOL_FLAG_VERIFIER | POOL_FLAG_SPECIAL_POOL))
    {
        //
        // For verifier, we should call the verification routine
        //
        if (ExpPoolFlags & POOL_FLAG_VERIFIER)
        {
            DPRINT1("Driver Verifier is not yet supported\n");
        }
 
        //
        // For special pool, we check if this is a suitable allocation and do
        // the special allocation if needed
        //
        if (ExpPoolFlags & POOL_FLAG_SPECIAL_POOL)
        {
            //
            // Check if this is a special pool allocation
            //
            if (MmUseSpecialPool(NumberOfBytes, Tag))
            {
                //
                // Try to allocate using special pool
                //
                Entry = MmAllocateSpecialPool(NumberOfBytes, Tag, PoolType, 2);
                if (Entry) return Entry;
            }
        }
    }
 
    //
    // Get the pool type and its corresponding vector for this request
    //
    OriginalType = PoolType;
    PoolType = PoolType & BASE_POOL_TYPE_MASK;
    PoolDesc = PoolVector[PoolType];
    ASSERT(PoolDesc != NULL);
 
    //
    // Check if this is a big page allocation
    //
    if (NumberOfBytes > POOL_MAX_ALLOC)
    {
        //
        // Allocate pages for it
        //
        Entry = MiAllocatePoolPages(OriginalType, NumberOfBytes);
        if (!Entry)
        {
#if DBG
            //
            // Out of memory, display current consumption
            // Let's consider that if the caller wanted more
            // than a hundred pages, that's a bogus caller
            // and we are not out of memory. Dump at most
            // once a second to avoid spamming the log.
            //
            if (NumberOfBytes < 100 * PAGE_SIZE &&
                KeQueryInterruptTime() >= MiLastPoolDumpTime + 10000000)
            {
                MiDumpPoolConsumers(FALSE, 0, 0, 0);
                MiLastPoolDumpTime = KeQueryInterruptTime();
            }
#endif
 
            //
            // Must succeed pool is deprecated, but still supported. These allocation
            // failures must cause an immediate bugcheck
            //
            if (OriginalType & MUST_SUCCEED_POOL_MASK)
            {
                KeBugCheckEx(MUST_SUCCEED_POOL_EMPTY,
                             NumberOfBytes,
                             NonPagedPoolDescriptor.TotalPages,
                             NonPagedPoolDescriptor.TotalBigPages,
                             0);
            }
 
            //
            // Internal debugging
            //
            ExPoolFailures++;
 
            //
            // This flag requests printing failures, and can also further specify
            // breaking on failures
            //
            if (ExpPoolFlags & POOL_FLAG_DBGPRINT_ON_FAILURE)
            {
                DPRINT1("EX: ExAllocatePool (%lu, 0x%x) returning NULL\n",
                        NumberOfBytes,
                        OriginalType);
                if (ExpPoolFlags & POOL_FLAG_CRASH_ON_FAILURE) DbgBreakPoint();
            }
 
            //
            // Finally, this flag requests an exception, which we are more than
            // happy to raise!
            //
            if (OriginalType & POOL_RAISE_IF_ALLOCATION_FAILURE)
            {
                ExRaiseStatus(STATUS_INSUFFICIENT_RESOURCES);
            }
 
            return NULL;
        }
 
        //
        // Increment required counters
        //
        InterlockedExchangeAdd((PLONG)&PoolDesc->TotalBigPages,
                               (LONG)BYTES_TO_PAGES(NumberOfBytes));
        InterlockedExchangeAddSizeT(&PoolDesc->TotalBytes, NumberOfBytes);
        InterlockedIncrement((PLONG)&PoolDesc->RunningAllocs);
 
        //
        // Add a tag for the big page allocation and switch to the generic "BIG"
        // tag if we failed to do so, then insert a tracker for this alloation.
        //
        if (!ExpAddTagForBigPages(Entry,
                                  Tag,
                                  (ULONG)BYTES_TO_PAGES(NumberOfBytes),
                                  OriginalType))
        {
            Tag = ' GIB';
        }
        ExpInsertPoolTracker(Tag, ROUND_TO_PAGES(NumberOfBytes), OriginalType);
        return Entry;
    }
 
    //
    // Should never request 0 bytes from the pool, but since so many drivers do
    // it, we'll just assume they want 1 byte, based on NT's similar behavior
    //
    if (!NumberOfBytes) NumberOfBytes = 1;
 
    //
    // A pool allocation is defined by its data, a linked list to connect it to
    // the free list (if necessary), and a pool header to store accounting info.
    // Calculate this size, then convert it into a block size (units of pool
    // headers)
    //
    // Note that i cannot overflow (past POOL_LISTS_PER_PAGE) because any such
    // request would've been treated as a POOL_MAX_ALLOC earlier and resulted in
    // the direct allocation of pages.
    //
    i = (USHORT)((NumberOfBytes + sizeof(POOL_HEADER) + (POOL_BLOCK_SIZE - 1))
                 / POOL_BLOCK_SIZE);
    ASSERT(i < POOL_LISTS_PER_PAGE);
 
    //
    // Handle lookaside list optimization for both paged and nonpaged pool
    //
    if (i <= NUMBER_POOL_LOOKASIDE_LISTS)
    {
        //
        // Try popping it from the per-CPU lookaside list
        //
        LookasideList = (PoolType == PagedPool) ?
                         Prcb->PPPagedLookasideList[i - 1].P :
                         Prcb->PPNPagedLookasideList[i - 1].P;
        LookasideList->TotalAllocates++;
        Entry = (PPOOL_HEADER)InterlockedPopEntrySList(&LookasideList->ListHead);
        if (!Entry)
        {
            //
            // We failed, try popping it from the global list
            //
            LookasideList = (PoolType == PagedPool) ?
                             Prcb->PPPagedLookasideList[i - 1].L :
                             Prcb->PPNPagedLookasideList[i - 1].L;
            LookasideList->TotalAllocates++;
            Entry = (PPOOL_HEADER)InterlockedPopEntrySList(&LookasideList->ListHead);
        }
 
        //
        // If we were able to pop it, update the accounting and return the block
        //
        if (Entry)
        {
            LookasideList->AllocateHits++;
 
            //
            // Get the real entry, write down its pool type, and track it
            //
            Entry--;
            Entry->PoolType = OriginalType + 1;
            ExpInsertPoolTracker(Tag,
                                 Entry->BlockSize * POOL_BLOCK_SIZE,
                                 OriginalType);
 
            //
            // Return the pool allocation
            //
            Entry->PoolTag = Tag;
            (POOL_FREE_BLOCK(Entry))->Flink = NULL;
            (POOL_FREE_BLOCK(Entry))->Blink = NULL;
            return POOL_FREE_BLOCK(Entry);
        }
    }
 
    //
    // Loop in the free lists looking for a block if this size. Start with the
    // list optimized for this kind of size lookup
    //
    ListHead = &PoolDesc->ListHeads[i];
    do
    {
        //
        // Are there any free entries available on this list?
        //
        if (!ExpIsPoolListEmpty(ListHead))
        {
            //
            // Acquire the pool lock now
            //
            OldIrql = ExLockPool(PoolDesc);
 
            //
            // And make sure the list still has entries
            //
            if (ExpIsPoolListEmpty(ListHead))
            {
                //
                // Someone raced us (and won) before we had a chance to acquire
                // the lock.
                //
                // Try again!
                //
                ExUnlockPool(PoolDesc, OldIrql);
                continue;
            }
 
            //
            // Remove a free entry from the list
            // Note that due to the way we insert free blocks into multiple lists
            // there is a guarantee that any block on this list will either be
            // of the correct size, or perhaps larger.
            //
            ExpCheckPoolLinks(ListHead);
            Entry = POOL_ENTRY(ExpRemovePoolHeadList(ListHead));
            ExpCheckPoolLinks(ListHead);
            ExpCheckPoolBlocks(Entry);
            ASSERT(Entry->BlockSize >= i);
            ASSERT(Entry->PoolType == 0);
 
            //
            // Check if this block is larger that what we need. The block could
            // not possibly be smaller, due to the reason explained above (and
            // we would've asserted on a checked build if this was the case).
            //
            if (Entry->BlockSize != i)
            {
                //
                // Is there an entry before this one?
                //
                if (Entry->PreviousSize == 0)
                {
                    //
                    // There isn't anyone before us, so take the next block and
                    // turn it into a fragment that contains the leftover data
                    // that we don't need to satisfy the caller's request
                    //
                    FragmentEntry = POOL_BLOCK(Entry, i);
                    FragmentEntry->BlockSize = Entry->BlockSize - i;
 
                    //
                    // And make it point back to us
                    //
                    FragmentEntry->PreviousSize = i;
 
                    //
                    // Now get the block that follows the new fragment and check
                    // if it's still on the same page as us (and not at the end)
                    //
                    NextEntry = POOL_NEXT_BLOCK(FragmentEntry);
                    if (PAGE_ALIGN(NextEntry) != NextEntry)
                    {
                        //
                        // Adjust this next block to point to our newly created
                        // fragment block
                        //
                        NextEntry->PreviousSize = FragmentEntry->BlockSize;
                    }
                }
                else
                {
                    //
                    // There is a free entry before us, which we know is smaller
                    // so we'll make this entry the fragment instead
                    //
                    FragmentEntry = Entry;
 
                    //
                    // And then we'll remove from it the actual size required.
                    // Now the entry is a leftover free fragment
                    //
                    Entry->BlockSize -= i;
 
                    //
                    // Now let's go to the next entry after the fragment (which
                    // used to point to our original free entry) and make it
                    // reference the new fragment entry instead.
                    //
                    // This is the entry that will actually end up holding the
                    // allocation!
                    //
                    Entry = POOL_NEXT_BLOCK(Entry);
                    Entry->PreviousSize = FragmentEntry->BlockSize;
 
                    //
                    // And now let's go to the entry after that one and check if
                    // it's still on the same page, and not at the end
                    //
                    NextEntry = POOL_BLOCK(Entry, i);
                    if (PAGE_ALIGN(NextEntry) != NextEntry)
                    {
                        //
                        // Make it reference the allocation entry
                        //
                        NextEntry->PreviousSize = i;
                    }
                }
 
                //
                // Now our (allocation) entry is the right size
                //
                Entry->BlockSize = i;
 
                //
                // And the next entry is now the free fragment which contains
                // the remaining difference between how big the original entry
                // was, and the actual size the caller needs/requested.
                //
                FragmentEntry->PoolType = 0;
                BlockSize = FragmentEntry->BlockSize;
 
                //
                // Now check if enough free bytes remained for us to have a
                // "full" entry, which contains enough bytes for a linked list
                // and thus can be used for allocations (up to 8 bytes...)
                //
                ExpCheckPoolLinks(&PoolDesc->ListHeads[BlockSize - 1]);
                if (BlockSize != 1)
                {
                    //
                    // Insert the free entry into the free list for this size
                    //
                    ExpInsertPoolTailList(&PoolDesc->ListHeads[BlockSize - 1],
                                          POOL_FREE_BLOCK(FragmentEntry));
                    ExpCheckPoolLinks(POOL_FREE_BLOCK(FragmentEntry));
                }
            }
 
            //
            // We have found an entry for this allocation, so set the pool type
            // and release the lock since we're done
            //
            Entry->PoolType = OriginalType + 1;
            ExpCheckPoolBlocks(Entry);
            ExUnlockPool(PoolDesc, OldIrql);
 
            //
            // Increment required counters
            //
            InterlockedExchangeAddSizeT(&PoolDesc->TotalBytes, Entry->BlockSize * POOL_BLOCK_SIZE);
            InterlockedIncrement((PLONG)&PoolDesc->RunningAllocs);
 
            //
            // Track this allocation
            //
            ExpInsertPoolTracker(Tag,
                                 Entry->BlockSize * POOL_BLOCK_SIZE,
                                 OriginalType);
 
            //
            // Return the pool allocation
            //
            Entry->PoolTag = Tag;
            (POOL_FREE_BLOCK(Entry))->Flink = NULL;
            (POOL_FREE_BLOCK(Entry))->Blink = NULL;
            return POOL_FREE_BLOCK(Entry);
        }
    } while (++ListHead != &PoolDesc->ListHeads[POOL_LISTS_PER_PAGE]);
 
    //
    // There were no free entries left, so we have to allocate a new fresh page
    //
    Entry = MiAllocatePoolPages(OriginalType, PAGE_SIZE);
    if (!Entry)
    {
#if DBG
        //
        // Out of memory, display current consumption
        // Let's consider that if the caller wanted more
        // than a hundred pages, that's a bogus caller
        // and we are not out of memory. Dump at most
        // once a second to avoid spamming the log.
        //
        if (NumberOfBytes < 100 * PAGE_SIZE &&
            KeQueryInterruptTime() >= MiLastPoolDumpTime + 10000000)
        {
            MiDumpPoolConsumers(FALSE, 0, 0, 0);
            MiLastPoolDumpTime = KeQueryInterruptTime();
        }
#endif
 
        //
        // Must succeed pool is deprecated, but still supported. These allocation
        // failures must cause an immediate bugcheck
        //
        if (OriginalType & MUST_SUCCEED_POOL_MASK)
        {
            KeBugCheckEx(MUST_SUCCEED_POOL_EMPTY,
                         PAGE_SIZE,
                         NonPagedPoolDescriptor.TotalPages,
                         NonPagedPoolDescriptor.TotalBigPages,
                         0);
        }
 
        //
        // Internal debugging
        //
        ExPoolFailures++;
 
        //
        // This flag requests printing failures, and can also further specify
        // breaking on failures
        //
        if (ExpPoolFlags & POOL_FLAG_DBGPRINT_ON_FAILURE)
        {
            DPRINT1("EX: ExAllocatePool (%lu, 0x%x) returning NULL\n",
                    NumberOfBytes,
                    OriginalType);
            if (ExpPoolFlags & POOL_FLAG_CRASH_ON_FAILURE) DbgBreakPoint();
        }
 
        //
        // Finally, this flag requests an exception, which we are more than
        // happy to raise!
        //
        if (OriginalType & POOL_RAISE_IF_ALLOCATION_FAILURE)
        {
            ExRaiseStatus(STATUS_INSUFFICIENT_RESOURCES);
        }
 
        //
        // Return NULL to the caller in all other cases
        //
        return NULL;
    }
 
    //
    // Setup the entry data
    //
    Entry->Ulong1 = 0;
    Entry->BlockSize = i;
    Entry->PoolType = OriginalType + 1;
 
    //
    // This page will have two entries -- one for the allocation (which we just
    // created above), and one for the remaining free bytes, which we're about
    // to create now. The free bytes are the whole page minus what was allocated
    // and then converted into units of block headers.
    //
    BlockSize = (PAGE_SIZE / POOL_BLOCK_SIZE) - i;
    FragmentEntry = POOL_BLOCK(Entry, i);
    FragmentEntry->Ulong1 = 0;
    FragmentEntry->BlockSize = BlockSize;
    FragmentEntry->PreviousSize = i;
 
    //
    // Increment required counters
    //
    InterlockedIncrement((PLONG)&PoolDesc->TotalPages);
    InterlockedExchangeAddSizeT(&PoolDesc->TotalBytes, Entry->BlockSize * POOL_BLOCK_SIZE);
 
    //
    // Now check if enough free bytes remained for us to have a "full" entry,
    // which contains enough bytes for a linked list and thus can be used for
    // allocations (up to 8 bytes...)
    //
    if (FragmentEntry->BlockSize != 1)
    {
        //
        // Excellent -- acquire the pool lock
        //
        OldIrql = ExLockPool(PoolDesc);
 
        //
        // And insert the free entry into the free list for this block size
        //
        ExpCheckPoolLinks(&PoolDesc->ListHeads[BlockSize - 1]);
        ExpInsertPoolTailList(&PoolDesc->ListHeads[BlockSize - 1],
                              POOL_FREE_BLOCK(FragmentEntry));
        ExpCheckPoolLinks(POOL_FREE_BLOCK(FragmentEntry));
 
        //
        // Release the pool lock
        //
        ExpCheckPoolBlocks(Entry);
        ExUnlockPool(PoolDesc, OldIrql);
    }
    else
    {
        //
        // Simply do a sanity check
        //
        ExpCheckPoolBlocks(Entry);
    }
 
    //
    // Increment performance counters and track this allocation
    //
    InterlockedIncrement((PLONG)&PoolDesc->RunningAllocs);
    ExpInsertPoolTracker(Tag,
                         Entry->BlockSize * POOL_BLOCK_SIZE,
                         OriginalType);
 
    //
    // And return the pool allocation
    //
    ExpCheckPoolBlocks(Entry);
    Entry->PoolTag = Tag;
    return POOL_FREE_BLOCK(Entry);
}

ExAllocatePoolWithTagPriority()

PVOID NTAPI ExAllocatePoolWithTagPriority	(	IN POOL_TYPE	PoolType,
		IN SIZE_T	NumberOfBytes,
		IN ULONG	Tag,
		IN EX_POOL_PRIORITY	Priority
	)

Definition at line 2963 of file expool.c.

{
    PVOID Buffer;
 
    //
    // Allocate the pool
    //
    Buffer = ExAllocatePoolWithTag(PoolType, NumberOfBytes, Tag);
    if (Buffer == NULL)
    {
        UNIMPLEMENTED;
    }
 
    return Buffer;
}

Referenced by _RxAllocatePoolWithTag(), InitializeMemoryManagement(), and MyAllocatePool().

ExFreePool()

VOID NTAPI ExFreePool

(

PVOID

P

)

Definition at line 2920 of file expool.c.

{
    //
    // Just free without checking for the tag
    //
    ExFreePoolWithTag(P, 0);
}

ExFreePoolWithTag()

VOID NTAPI ExFreePoolWithTag	(	IN PVOID	P,
		IN ULONG	TagToFree
	)

Definition at line 2494 of file expool.c.

{
    PPOOL_HEADER Entry, NextEntry;
    USHORT BlockSize;
    KIRQL OldIrql;
    POOL_TYPE PoolType;
    PPOOL_DESCRIPTOR PoolDesc;
    ULONG Tag;
    BOOLEAN Combined = FALSE;
    PFN_NUMBER PageCount, RealPageCount;
    PKPRCB Prcb = KeGetCurrentPrcb();
    PGENERAL_LOOKASIDE LookasideList;
    PEPROCESS Process;
 
    //
    // Check if any of the debug flags are enabled
    //
    if (ExpPoolFlags & (POOL_FLAG_CHECK_TIMERS |
                        POOL_FLAG_CHECK_WORKERS |
                        POOL_FLAG_CHECK_RESOURCES |
                        POOL_FLAG_VERIFIER |
                        POOL_FLAG_CHECK_DEADLOCK |
                        POOL_FLAG_SPECIAL_POOL))
    {
        //
        // Check if special pool is enabled
        //
        if (ExpPoolFlags & POOL_FLAG_SPECIAL_POOL)
        {
            //
            // Check if it was allocated from a special pool
            //
            if (MmIsSpecialPoolAddress(P))
            {
                //
                // Was deadlock verification also enabled? We can do some extra
                // checks at this point
                //
                if (ExpPoolFlags & POOL_FLAG_CHECK_DEADLOCK)
                {
                    DPRINT1("Verifier not yet supported\n");
                }
 
                //
                // It is, so handle it via special pool free routine
                //
                MmFreeSpecialPool(P);
                return;
            }
        }
 
        //
        // For non-big page allocations, we'll do a bunch of checks in here
        //
        if (PAGE_ALIGN(P) != P)
        {
            //
            // Get the entry for this pool allocation
            // The pointer math here may look wrong or confusing, but it is quite right
            //
            Entry = P;
            Entry--;
 
            //
            // Get the pool type
            //
            PoolType = (Entry->PoolType - 1) & BASE_POOL_TYPE_MASK;
 
            //
            // FIXME: Many other debugging checks go here
            //
            ExpCheckPoolIrqlLevel(PoolType, 0, P);
        }
    }
 
    //
    // Check if this is a big page allocation
    //
    if (PAGE_ALIGN(P) == P)
    {
        //
        // We need to find the tag for it, so first we need to find out what
        // kind of allocation this was (paged or nonpaged), then we can go
        // ahead and try finding the tag for it. Remember to get rid of the
        // PROTECTED_POOL tag if it's found.
        //
        // Note that if at insertion time, we failed to add the tag for a big
        // pool allocation, we used a special tag called 'BIG' to identify the
        // allocation, and we may get this tag back. In this scenario, we must
        // manually get the size of the allocation by actually counting through
        // the PFN database.
        //
        PoolType = MmDeterminePoolType(P);
        ExpCheckPoolIrqlLevel(PoolType, 0, P);
        Tag = ExpFindAndRemoveTagBigPages(P, &PageCount, PoolType);
        if (!Tag)
        {
            DPRINT1("We do not know the size of this allocation. This is not yet supported\n");
            ASSERT(Tag == ' GIB');
            PageCount = 1; // We are going to lie! This might screw up accounting?
        }
        else if (Tag & PROTECTED_POOL)
        {
            Tag &= ~PROTECTED_POOL;
        }
 
        //
        // Check block tag
        //
        if (TagToFree && TagToFree != Tag)
        {
            DPRINT1("Freeing pool - invalid tag specified: %.4s != %.4s\n", (char*)&TagToFree, (char*)&Tag);
#if DBG
            /* Do not bugcheck in case this is a big allocation for which we didn't manage to insert the tag */
            if (Tag != ' GIB')
                KeBugCheckEx(BAD_POOL_CALLER, 0x0A, (ULONG_PTR)P, Tag, TagToFree);
#endif
        }
 
        //
        // We have our tag and our page count, so we can go ahead and remove this
        // tracker now
        //
        ExpRemovePoolTracker(Tag, PageCount << PAGE_SHIFT, PoolType);
 
        //
        // Check if any of the debug flags are enabled
        //
        if (ExpPoolFlags & (POOL_FLAG_CHECK_TIMERS |
                            POOL_FLAG_CHECK_WORKERS |
                            POOL_FLAG_CHECK_RESOURCES |
                            POOL_FLAG_CHECK_DEADLOCK))
        {
            //
            // Was deadlock verification also enabled? We can do some extra
            // checks at this point
            //
            if (ExpPoolFlags & POOL_FLAG_CHECK_DEADLOCK)
            {
                DPRINT1("Verifier not yet supported\n");
            }
 
            //
            // FIXME: Many debugging checks go here
            //
        }
 
        //
        // Update counters
        //
        PoolDesc = PoolVector[PoolType];
        InterlockedIncrement((PLONG)&PoolDesc->RunningDeAllocs);
        InterlockedExchangeAddSizeT(&PoolDesc->TotalBytes,
                                    -(LONG_PTR)(PageCount << PAGE_SHIFT));
 
        //
        // Do the real free now and update the last counter with the big page count
        //
        RealPageCount = MiFreePoolPages(P);
        ASSERT(RealPageCount == PageCount);
        InterlockedExchangeAdd((PLONG)&PoolDesc->TotalBigPages,
                               -(LONG)RealPageCount);
        return;
    }
 
    //
    // Get the entry for this pool allocation
    // The pointer math here may look wrong or confusing, but it is quite right
    //
    Entry = P;
    Entry--;
    ASSERT((ULONG_PTR)Entry % POOL_BLOCK_SIZE == 0);
 
    //
    // Get the size of the entry, and it's pool type, then load the descriptor
    // for this pool type
    //
    BlockSize = Entry->BlockSize;
    PoolType = (Entry->PoolType - 1) & BASE_POOL_TYPE_MASK;
    PoolDesc = PoolVector[PoolType];
 
    //
    // Make sure that the IRQL makes sense
    //
    ExpCheckPoolIrqlLevel(PoolType, 0, P);
 
    //
    // Get the pool tag and get rid of the PROTECTED_POOL flag
    //
    Tag = Entry->PoolTag;
    if (Tag & PROTECTED_POOL) Tag &= ~PROTECTED_POOL;
 
    //
    // Check block tag
    //
    if (TagToFree && TagToFree != Tag)
    {
        DPRINT1("Freeing pool - invalid tag specified: %.4s != %.4s\n", (char*)&TagToFree, (char*)&Tag);
#if DBG
        KeBugCheckEx(BAD_POOL_CALLER, 0x0A, (ULONG_PTR)P, Tag, TagToFree);
#endif
    }
 
    //
    // Track the removal of this allocation
    //
    ExpRemovePoolTracker(Tag,
                         BlockSize * POOL_BLOCK_SIZE,
                         Entry->PoolType - 1);
 
    //
    // Release pool quota, if any
    //
    if ((Entry->PoolType - 1) & QUOTA_POOL_MASK)
    {
        Process = ((PVOID *)POOL_NEXT_BLOCK(Entry))[-1];
        if (Process)
        {
            if (Process->Pcb.Header.Type != ProcessObject)
            {
                DPRINT1("Object %p is not a process. Type %u, pool type 0x%x, block size %u\n",
                        Process, Process->Pcb.Header.Type, Entry->PoolType, BlockSize);
                KeBugCheckEx(BAD_POOL_CALLER,
                             POOL_BILLED_PROCESS_INVALID,
                             (ULONG_PTR)P,
                             Tag,
                             (ULONG_PTR)Process);
            }
            PsReturnPoolQuota(Process, PoolType, BlockSize * POOL_BLOCK_SIZE);
            ObDereferenceObject(Process);
        }
    }
 
    //
    // Is this allocation small enough to have come from a lookaside list?
    //
    if (BlockSize <= NUMBER_POOL_LOOKASIDE_LISTS)
    {
        //
        // Try pushing it into the per-CPU lookaside list
        //
        LookasideList = (PoolType == PagedPool) ?
                         Prcb->PPPagedLookasideList[BlockSize - 1].P :
                         Prcb->PPNPagedLookasideList[BlockSize - 1].P;
        LookasideList->TotalFrees++;
        if (ExQueryDepthSList(&LookasideList->ListHead) < LookasideList->Depth)
        {
            LookasideList->FreeHits++;
            InterlockedPushEntrySList(&LookasideList->ListHead, P);
            return;
        }
 
        //
        // We failed, try to push it into the global lookaside list
        //
        LookasideList = (PoolType == PagedPool) ?
                         Prcb->PPPagedLookasideList[BlockSize - 1].L :
                         Prcb->PPNPagedLookasideList[BlockSize - 1].L;
        LookasideList->TotalFrees++;
        if (ExQueryDepthSList(&LookasideList->ListHead) < LookasideList->Depth)
        {
            LookasideList->FreeHits++;
            InterlockedPushEntrySList(&LookasideList->ListHead, P);
            return;
        }
    }
 
    //
    // Get the pointer to the next entry
    //
    NextEntry = POOL_BLOCK(Entry, BlockSize);
 
    //
    // Update performance counters
    //
    InterlockedIncrement((PLONG)&PoolDesc->RunningDeAllocs);
    InterlockedExchangeAddSizeT(&PoolDesc->TotalBytes, -BlockSize * POOL_BLOCK_SIZE);
 
    //
    // Acquire the pool lock
    //
    OldIrql = ExLockPool(PoolDesc);
 
    //
    // Check if the next allocation is at the end of the page
    //
    ExpCheckPoolBlocks(Entry);
    if (PAGE_ALIGN(NextEntry) != NextEntry)
    {
        //
        // We may be able to combine the block if it's free
        //
        if (NextEntry->PoolType == 0)
        {
            //
            // The next block is free, so we'll do a combine
            //
            Combined = TRUE;
 
            //
            // Make sure there's actual data in the block -- anything smaller
            // than this means we only have the header, so there's no linked list
            // for us to remove
            //
            if ((NextEntry->BlockSize != 1))
            {
                //
                // The block is at least big enough to have a linked list, so go
                // ahead and remove it
                //
                ExpCheckPoolLinks(POOL_FREE_BLOCK(NextEntry));
                ExpRemovePoolEntryList(POOL_FREE_BLOCK(NextEntry));
                ExpCheckPoolLinks(ExpDecodePoolLink((POOL_FREE_BLOCK(NextEntry))->Flink));
                ExpCheckPoolLinks(ExpDecodePoolLink((POOL_FREE_BLOCK(NextEntry))->Blink));
            }
 
            //
            // Our entry is now combined with the next entry
            //
            Entry->BlockSize = Entry->BlockSize + NextEntry->BlockSize;
        }
    }
 
    //
    // Now check if there was a previous entry on the same page as us
    //
    if (Entry->PreviousSize)
    {
        //
        // Great, grab that entry and check if it's free
        //
        NextEntry = POOL_PREV_BLOCK(Entry);
        if (NextEntry->PoolType == 0)
        {
            //
            // It is, so we can do a combine
            //
            Combined = TRUE;
 
            //
            // Make sure there's actual data in the block -- anything smaller
            // than this means we only have the header so there's no linked list
            // for us to remove
            //
            if ((NextEntry->BlockSize != 1))
            {
                //
                // The block is at least big enough to have a linked list, so go
                // ahead and remove it
                //
                ExpCheckPoolLinks(POOL_FREE_BLOCK(NextEntry));
                ExpRemovePoolEntryList(POOL_FREE_BLOCK(NextEntry));
                ExpCheckPoolLinks(ExpDecodePoolLink((POOL_FREE_BLOCK(NextEntry))->Flink));
                ExpCheckPoolLinks(ExpDecodePoolLink((POOL_FREE_BLOCK(NextEntry))->Blink));
            }
 
            //
            // Combine our original block (which might've already been combined
            // with the next block), into the previous block
            //
            NextEntry->BlockSize = NextEntry->BlockSize + Entry->BlockSize;
 
            //
            // And now we'll work with the previous block instead
            //
            Entry = NextEntry;
        }
    }
 
    //
    // By now, it may have been possible for our combined blocks to actually
    // have made up a full page (if there were only 2-3 allocations on the
    // page, they could've all been combined).
    //
    if ((PAGE_ALIGN(Entry) == Entry) &&
        (PAGE_ALIGN(POOL_NEXT_BLOCK(Entry)) == POOL_NEXT_BLOCK(Entry)))
    {
        //
        // In this case, release the pool lock, update the performance counter,
        // and free the page
        //
        ExUnlockPool(PoolDesc, OldIrql);
        InterlockedExchangeAdd((PLONG)&PoolDesc->TotalPages, -1);
        MiFreePoolPages(Entry);
        return;
    }
 
    //
    // Otherwise, we now have a free block (or a combination of 2 or 3)
    //
    Entry->PoolType = 0;
    BlockSize = Entry->BlockSize;
    ASSERT(BlockSize != 1);
 
    //
    // Check if we actually did combine it with anyone
    //
    if (Combined)
    {
        //
        // Get the first combined block (either our original to begin with, or
        // the one after the original, depending if we combined with the previous)
        //
        NextEntry = POOL_NEXT_BLOCK(Entry);
 
        //
        // As long as the next block isn't on a page boundary, have it point
        // back to us
        //
        if (PAGE_ALIGN(NextEntry) != NextEntry) NextEntry->PreviousSize = BlockSize;
    }
 
    //
    // Insert this new free block, and release the pool lock
    //
    ExpInsertPoolHeadList(&PoolDesc->ListHeads[BlockSize - 1], POOL_FREE_BLOCK(Entry));
    ExpCheckPoolLinks(POOL_FREE_BLOCK(Entry));
    ExUnlockPool(PoolDesc, OldIrql);
}

ExGetPoolTagInfo()

NTSTATUS NTAPI ExGetPoolTagInfo	(	IN PSYSTEM_POOLTAG_INFORMATION	SystemInformation,
		IN ULONG	SystemInformationLength,
		IN OUT PULONG ReturnLength	OPTIONAL
	)

Definition at line 1356 of file expool.c.

{
    ULONG TableSize, CurrentLength;
    ULONG EntryCount;
    NTSTATUS Status = STATUS_SUCCESS;
    PSYSTEM_POOLTAG TagEntry;
    PPOOL_TRACKER_TABLE Buffer, TrackerEntry;
    POOL_DPC_CONTEXT Context;
    ASSERT(KeGetCurrentIrql() == PASSIVE_LEVEL);
 
    //
    // Keep track of how much data the caller's buffer must hold
    //
    CurrentLength = FIELD_OFFSET(SYSTEM_POOLTAG_INFORMATION, TagInfo);
 
    //
    // Initialize the caller's buffer
    //
    TagEntry = &SystemInformation->TagInfo[0];
    SystemInformation->Count = 0;
 
    //
    // Capture the number of entries, and the total size needed to make a copy
    // of the table
    //
    EntryCount = (ULONG)PoolTrackTableSize;
    TableSize = EntryCount * sizeof(POOL_TRACKER_TABLE);
 
    //
    // Allocate the "Generic DPC" temporary buffer
    //
    Buffer = ExAllocatePoolWithTag(NonPagedPool, TableSize, 'ofnI');
    if (!Buffer) return STATUS_INSUFFICIENT_RESOURCES;
 
    //
    // Do a "Generic DPC" to atomically retrieve the tag and allocation data
    //
    Context.PoolTrackTable = Buffer;
    Context.PoolTrackTableSize = PoolTrackTableSize;
    Context.PoolTrackTableExpansion = NULL;
    Context.PoolTrackTableSizeExpansion = 0;
    KeGenericCallDpc(ExpGetPoolTagInfoTarget, &Context);
 
    //
    // Now parse the results
    //
    for (TrackerEntry = Buffer; TrackerEntry < (Buffer + EntryCount); TrackerEntry++)
    {
        //
        // If the entry is empty, skip it
        //
        if (!TrackerEntry->Key) continue;
 
        //
        // Otherwise, add one more entry to the caller's buffer, and ensure that
        // enough space has been allocated in it
        //
        SystemInformation->Count++;
        CurrentLength += sizeof(*TagEntry);
        if (SystemInformationLength < CurrentLength)
        {
            //
            // The caller's buffer is too small, so set a failure code. The
            // caller will know the count, as well as how much space is needed.
            //
            // We do NOT break out of the loop, because we want to keep incrementing
            // the Count as well as CurrentLength so that the caller can know the
            // final numbers
            //
            Status = STATUS_INFO_LENGTH_MISMATCH;
        }
        else
        {
            //
            // Small sanity check that our accounting is working correctly
            //
            ASSERT(TrackerEntry->PagedAllocs >= TrackerEntry->PagedFrees);
            ASSERT(TrackerEntry->NonPagedAllocs >= TrackerEntry->NonPagedFrees);
 
            //
            // Return the data into the caller's buffer
            //
            TagEntry->TagUlong = TrackerEntry->Key;
            TagEntry->PagedAllocs = TrackerEntry->PagedAllocs;
            TagEntry->PagedFrees = TrackerEntry->PagedFrees;
            TagEntry->PagedUsed = TrackerEntry->PagedBytes;
            TagEntry->NonPagedAllocs = TrackerEntry->NonPagedAllocs;
            TagEntry->NonPagedFrees = TrackerEntry->NonPagedFrees;
            TagEntry->NonPagedUsed = TrackerEntry->NonPagedBytes;
            TagEntry++;
        }
    }
 
    //
    // Free the "Generic DPC" temporary buffer, return the buffer length and status
    //
    ExFreePoolWithTag(Buffer, 'ofnI');
    if (ReturnLength) *ReturnLength = CurrentLength;
    return Status;
}

Referenced by QSI_DEF().

ExInitializePoolDescriptor()

VOID NTAPI ExInitializePoolDescriptor	(	IN PPOOL_DESCRIPTOR	PoolDescriptor,
		IN POOL_TYPE	PoolType,
		IN ULONG	PoolIndex,
		IN ULONG	Threshold,
		IN PVOID	PoolLock
	)

Definition at line 969 of file expool.c.

{
    PLIST_ENTRY NextEntry, LastEntry;
 
    //
    // Setup the descriptor based on the caller's request
    //
    PoolDescriptor->PoolType = PoolType;
    PoolDescriptor->PoolIndex = PoolIndex;
    PoolDescriptor->Threshold = Threshold;
    PoolDescriptor->LockAddress = PoolLock;
 
    //
    // Initialize accounting data
    //
    PoolDescriptor->RunningAllocs = 0;
    PoolDescriptor->RunningDeAllocs = 0;
    PoolDescriptor->TotalPages = 0;
    PoolDescriptor->TotalBytes = 0;
    PoolDescriptor->TotalBigPages = 0;
 
    //
    // Nothing pending for now
    //
    PoolDescriptor->PendingFrees = NULL;
    PoolDescriptor->PendingFreeDepth = 0;
 
    //
    // Loop all the descriptor's allocation lists and initialize them
    //
    NextEntry = PoolDescriptor->ListHeads;
    LastEntry = NextEntry + POOL_LISTS_PER_PAGE;
    while (NextEntry < LastEntry)
    {
        ExpInitializePoolListHead(NextEntry);
        NextEntry++;
    }
 
    //
    // Note that ReactOS does not support Session Pool Yet
    //
    ASSERT(PoolType != PagedPoolSession);
}

Referenced by InitializePool(), and MiInitializeSessionPool().

ExLockPool()

FORCEINLINE KIRQL ExLockPool

(

IN PPOOL_DESCRIPTOR

Descriptor

)

Definition at line 1273 of file expool.c.

{
    //
    // Check if this is nonpaged pool
    //
    if ((Descriptor->PoolType & BASE_POOL_TYPE_MASK) == NonPagedPool)
    {
        //
        // Use the queued spin lock
        //
        return KeAcquireQueuedSpinLock(LockQueueNonPagedPoolLock);
    }
    else
    {
        //
        // Use the guarded mutex
        //
        KeAcquireGuardedMutex(Descriptor->LockAddress);
        return APC_LEVEL;
    }
}

Referenced by ExAllocatePoolWithTag(), and ExFreePoolWithTag().

ExpAddTagForBigPages()

BOOLEAN NTAPI ExpAddTagForBigPages	(	IN PVOID	Va,
		IN ULONG	Key,
		IN ULONG	NumberOfPages,
		IN POOL_TYPE	PoolType
	)

Definition at line 1581 of file expool.c.

{
    ULONG Hash, i = 0;
    PVOID OldVa;
    KIRQL OldIrql;
    SIZE_T TableSize;
    PPOOL_TRACKER_BIG_PAGES Entry, EntryEnd, EntryStart;
    ASSERT(((ULONG_PTR)Va & POOL_BIG_TABLE_ENTRY_FREE) == 0);
    ASSERT(!(PoolType & SESSION_POOL_MASK));
 
    //
    // As the table is expandable, these values must only be read after acquiring
    // the lock to avoid a teared access during an expansion
    // NOTE: Windows uses a special reader/writer SpinLock to improve
    // performance in the common case (add/remove a tracker entry)
    //
Retry:
    Hash = ExpComputePartialHashForAddress(Va);
    KeAcquireSpinLock(&ExpLargePoolTableLock, &OldIrql);
    Hash &= PoolBigPageTableHash;
    TableSize = PoolBigPageTableSize;
 
    //
    // We loop from the current hash bucket to the end of the table, and then
    // rollover to hash bucket 0 and keep going from there. If we return back
    // to the beginning, then we attempt expansion at the bottom of the loop
    //
    EntryStart = Entry = &PoolBigPageTable[Hash];
    EntryEnd = &PoolBigPageTable[TableSize];
    do
    {
        //
        // Make sure that this is a free entry and attempt to atomically make the
        // entry busy now
        // NOTE: the Interlocked operation cannot fail with an exclusive SpinLock
        //
        OldVa = Entry->Va;
        if (((ULONG_PTR)OldVa & POOL_BIG_TABLE_ENTRY_FREE) &&
            (NT_VERIFY(InterlockedCompareExchangePointer(&Entry->Va, Va, OldVa) == OldVa)))
        {
            //
            // We now own this entry, write down the size and the pool tag
            //
            Entry->Key = Key;
            Entry->NumberOfPages = NumberOfPages;
 
            //
            // Add one more entry to the count, and see if we're getting within
            // 75% of the table size, at which point we'll do an expansion now
            // to avoid blocking too hard later on.
            //
            // Note that we only do this if it's also been the 16th time that we
            // keep losing the race or that we are not finding a free entry anymore,
            // which implies a massive number of concurrent big pool allocations.
            //
            ExpPoolBigEntriesInUse++;
            if ((i >= 16) && (ExpPoolBigEntriesInUse > (TableSize * (POOL_BIG_TABLE_USE_RATE - 1) / POOL_BIG_TABLE_USE_RATE)))
            {
                DPRINT("Attempting expansion since we now have %lu entries\n",
                        ExpPoolBigEntriesInUse);
                ASSERT(TableSize == PoolBigPageTableSize);
                ExpReallocateBigPageTable(OldIrql, FALSE);
                return TRUE;
            }
 
            //
            // We have our entry, return
            //
            KeReleaseSpinLock(&ExpLargePoolTableLock, OldIrql);
            return TRUE;
        }
 
        //
        // We don't have our entry yet, so keep trying, making the entry list
        // circular if we reach the last entry. We'll eventually break out of
        // the loop once we've rolled over and returned back to our original
        // hash bucket
        //
        i++;
        if (++Entry >= EntryEnd) Entry = &PoolBigPageTable[0];
    } while (Entry != EntryStart);
 
    //
    // This means there's no free hash buckets whatsoever, so we now have
    // to attempt expanding the table
    //
    ASSERT(TableSize == PoolBigPageTableSize);
    if (ExpReallocateBigPageTable(OldIrql, FALSE))
    {
        goto Retry;
    }
    ExpBigTableExpansionFailed++;
    DPRINT1("Big pool table expansion failed\n");
    return FALSE;
}

Referenced by ExAllocatePoolWithTag().

ExpCheckPoolAllocation()

VOID NTAPI ExpCheckPoolAllocation	(	PVOID	P,
		POOL_TYPE	PoolType,
		ULONG	Tag
	)

Definition at line 296 of file expool.c.

{
    PPOOL_HEADER Entry;
    ULONG i;
    KIRQL OldIrql;
    POOL_TYPE RealPoolType;
 
    /* Get the pool header */
    Entry = ((PPOOL_HEADER)P) - 1;
 
    /* Check if this is a large allocation */
    if (PAGE_ALIGN(P) == P)
    {
        /* Lock the pool table */
        KeAcquireSpinLock(&ExpLargePoolTableLock, &OldIrql);
 
        /* Find the pool tag */
        for (i = 0; i < PoolBigPageTableSize; i++)
        {
            /* Check if this is our allocation */
            if (PoolBigPageTable[i].Va == P)
            {
                /* Make sure the tag is ok */
                if (PoolBigPageTable[i].Key != Tag)
                {
                    KeBugCheckEx(BAD_POOL_CALLER, 0x0A, (ULONG_PTR)P, PoolBigPageTable[i].Key, Tag);
                }
 
                break;
            }
        }
 
        /* Release the lock */
        KeReleaseSpinLock(&ExpLargePoolTableLock, OldIrql);
 
        if (i == PoolBigPageTableSize)
        {
            /* Did not find the allocation */
            //ASSERT(FALSE);
        }
 
        /* Get Pool type by address */
        RealPoolType = MmDeterminePoolType(P);
    }
    else
    {
        /* Verify the tag */
        if (Entry->PoolTag != Tag)
        {
            DPRINT1("Allocation has wrong pool tag! Expected '%.4s', got '%.4s' (0x%08lx)\n",
                    &Tag, &Entry->PoolTag, Entry->PoolTag);
            KeBugCheckEx(BAD_POOL_CALLER, 0x0A, (ULONG_PTR)P, Entry->PoolTag, Tag);
        }
 
        /* Check the rest of the header */
        ExpCheckPoolHeader(Entry);
 
        /* Get Pool type from entry */
        RealPoolType = (Entry->PoolType - 1);
    }
 
    /* Should we check the pool type? */
    if (PoolType != -1)
    {
        /* Verify the pool type */
        if (RealPoolType != PoolType)
        {
            DPRINT1("Wrong pool type! Expected %s, got %s\n",
                    PoolType & BASE_POOL_TYPE_MASK ? "PagedPool" : "NonPagedPool",
                    (Entry->PoolType - 1) & BASE_POOL_TYPE_MASK ? "PagedPool" : "NonPagedPool");
            KeBugCheckEx(BAD_POOL_CALLER, 0xCC, (ULONG_PTR)P, Entry->PoolTag, Tag);
        }
    }
}

ExpCheckPoolBlocks()

VOID NTAPI ExpCheckPoolBlocks

(

IN PVOID

Block

)

Definition at line 376 of file expool.c.

{
    BOOLEAN FoundBlock = FALSE;
    SIZE_T Size = 0;
    PPOOL_HEADER Entry;
 
    /* Get the first entry for this page, make sure it really is the first */
    Entry = PAGE_ALIGN(Block);
    ASSERT(Entry->PreviousSize == 0);
 
    /* Now scan each entry */
    while (TRUE)
    {
        /* When we actually found our block, remember this */
        if (Entry == Block) FoundBlock = TRUE;
 
        /* Now validate this block header */
        ExpCheckPoolHeader(Entry);
 
        /* And go to the next one, keeping track of our size */
        Size += Entry->BlockSize;
        Entry = POOL_NEXT_BLOCK(Entry);
 
        /* If we hit the last block, stop */
        if (Size >= (PAGE_SIZE / POOL_BLOCK_SIZE)) break;
 
        /* If we hit the end of the page, stop */
        if (PAGE_ALIGN(Entry) == Entry) break;
    }
 
    /* We must've found our block, and we must have hit the end of the page */
    if ((PAGE_ALIGN(Entry) != Entry) || !(FoundBlock))
    {
        /* Otherwise, the blocks are messed up */
        KeBugCheckEx(BAD_POOL_HEADER, 10, (ULONG_PTR)Block, __LINE__, (ULONG_PTR)Entry);
    }
}

Referenced by ExAllocatePoolWithTag(), and ExFreePoolWithTag().

ExpCheckPoolHeader()

VOID NTAPI ExpCheckPoolHeader

(

IN PPOOL_HEADER

Entry

)

Definition at line 193 of file expool.c.

{
    PPOOL_HEADER PreviousEntry, NextEntry;
 
    /* Is there a block before this one? */
    if (Entry->PreviousSize)
    {
        /* Get it */
        PreviousEntry = POOL_PREV_BLOCK(Entry);
 
        /* The two blocks must be on the same page! */
        if (PAGE_ALIGN(Entry) != PAGE_ALIGN(PreviousEntry))
        {
            /* Something is awry */
            KeBugCheckEx(BAD_POOL_HEADER,
                         6,
                         (ULONG_PTR)PreviousEntry,
                         __LINE__,
                         (ULONG_PTR)Entry);
        }
 
        /* This block should also indicate that it's as large as we think it is */
        if (PreviousEntry->BlockSize != Entry->PreviousSize)
        {
            /* Otherwise, someone corrupted one of the sizes */
            DPRINT1("PreviousEntry BlockSize %lu, tag %.4s. Entry PreviousSize %lu, tag %.4s\n",
                    PreviousEntry->BlockSize, (char *)&PreviousEntry->PoolTag,
                    Entry->PreviousSize, (char *)&Entry->PoolTag);
            KeBugCheckEx(BAD_POOL_HEADER,
                         5,
                         (ULONG_PTR)PreviousEntry,
                         __LINE__,
                         (ULONG_PTR)Entry);
        }
    }
    else if (PAGE_ALIGN(Entry) != Entry)
    {
        /* If there's no block before us, we are the first block, so we should be on a page boundary */
        KeBugCheckEx(BAD_POOL_HEADER,
                     7,
                     0,
                     __LINE__,
                     (ULONG_PTR)Entry);
    }
 
    /* This block must have a size */
    if (!Entry->BlockSize)
    {
        /* Someone must've corrupted this field */
        if (Entry->PreviousSize)
        {
            PreviousEntry = POOL_PREV_BLOCK(Entry);
            DPRINT1("PreviousEntry tag %.4s. Entry tag %.4s\n",
                    (char *)&PreviousEntry->PoolTag,
                    (char *)&Entry->PoolTag);
        }
        else
        {
            DPRINT1("Entry tag %.4s\n",
                    (char *)&Entry->PoolTag);
        }
        KeBugCheckEx(BAD_POOL_HEADER,
                     8,
                     0,
                     __LINE__,
                     (ULONG_PTR)Entry);
    }
 
    /* Okay, now get the next block */
    NextEntry = POOL_NEXT_BLOCK(Entry);
 
    /* If this is the last block, then we'll be page-aligned, otherwise, check this block */
    if (PAGE_ALIGN(NextEntry) != NextEntry)
    {
        /* The two blocks must be on the same page! */
        if (PAGE_ALIGN(Entry) != PAGE_ALIGN(NextEntry))
        {
            /* Something is messed up */
            KeBugCheckEx(BAD_POOL_HEADER,
                         9,
                         (ULONG_PTR)NextEntry,
                         __LINE__,
                         (ULONG_PTR)Entry);
        }
 
        /* And this block should think we are as large as we truly are */
        if (NextEntry->PreviousSize != Entry->BlockSize)
        {
            /* Otherwise, someone corrupted the field */
            DPRINT1("Entry BlockSize %lu, tag %.4s. NextEntry PreviousSize %lu, tag %.4s\n",
                    Entry->BlockSize, (char *)&Entry->PoolTag,
                    NextEntry->PreviousSize, (char *)&NextEntry->PoolTag);
            KeBugCheckEx(BAD_POOL_HEADER,
                         5,
                         (ULONG_PTR)NextEntry,
                         __LINE__,
                         (ULONG_PTR)Entry);
        }
    }
}

Referenced by ExpCheckPoolAllocation(), and ExpCheckPoolBlocks().

ExpCheckPoolIrqlLevel()

FORCEINLINE VOID ExpCheckPoolIrqlLevel	(	IN POOL_TYPE	PoolType,
		IN SIZE_T	NumberOfBytes,
		IN PVOID	Entry
	)

Definition at line 416 of file expool.c.

{
    //
    // Validate IRQL: It must be APC_LEVEL or lower for Paged Pool, and it must
    // be DISPATCH_LEVEL or lower for Non Paged Pool
    //
    if (((PoolType & BASE_POOL_TYPE_MASK) == PagedPool) ?
        (KeGetCurrentIrql() > APC_LEVEL) :
        (KeGetCurrentIrql() > DISPATCH_LEVEL))
    {
        //
        // Take the system down
        //
        KeBugCheckEx(BAD_POOL_CALLER,
                     !Entry ? POOL_ALLOC_IRQL_INVALID : POOL_FREE_IRQL_INVALID,
                     KeGetCurrentIrql(),
                     PoolType,
                     !Entry ? NumberOfBytes : (ULONG_PTR)Entry);
    }
}

Referenced by ExAllocatePoolWithTag(), and ExFreePoolWithTag().

ExpCheckPoolLinks()

VOID NTAPI ExpCheckPoolLinks

(

IN PLIST_ENTRY

ListHead

)

Definition at line 99 of file expool.c.

{
    if ((ExpDecodePoolLink(ExpDecodePoolLink(ListHead->Flink)->Blink) != ListHead) ||
        (ExpDecodePoolLink(ExpDecodePoolLink(ListHead->Blink)->Flink) != ListHead))
    {
        KeBugCheckEx(BAD_POOL_HEADER,
                     3,
                     (ULONG_PTR)ListHead,
                     (ULONG_PTR)ExpDecodePoolLink(ExpDecodePoolLink(ListHead->Flink)->Blink),
                     (ULONG_PTR)ExpDecodePoolLink(ExpDecodePoolLink(ListHead->Blink)->Flink));
    }
}

Referenced by ExAllocatePoolWithTag(), ExFreePoolWithTag(), ExpInsertPoolHeadList(), and ExpInsertPoolTailList().

ExpComputeHashForTag()

FORCEINLINE ULONG ExpComputeHashForTag	(	IN ULONG	Tag,
		IN SIZE_T	BucketMask
	)

Definition at line 441 of file expool.c.

{
    //
    // Compute the hash by multiplying with a large prime number and then XORing
    // with the HIDWORD of the result.
    //
    // Finally, AND with the bucket mask to generate a valid index/bucket into
    // the table
    //
    ULONGLONG Result = (ULONGLONG)40543 * Tag;
    return (ULONG)BucketMask & ((ULONG)Result ^ (Result >> 32));
}

Referenced by ExpInsertPoolTracker(), ExpRemovePoolTracker(), and ExpSeedHotTags().

ExpComputePartialHashForAddress()

FORCEINLINE ULONG ExpComputePartialHashForAddress

(

IN PVOID

BaseAddress

)

Definition at line 457 of file expool.c.

{
    ULONG Result;
    //
    // Compute the hash by converting the address into a page number, and then
    // XORing each nibble with the next one.
    //
    // We do *NOT* AND with the bucket mask at this point because big table expansion
    // might happen. Therefore, the final step of the hash must be performed
    // while holding the expansion pushlock, and this is why we call this a
    // "partial" hash only.
    //
    Result = (ULONG)((ULONG_PTR)BaseAddress >> PAGE_SHIFT);
    return (Result >> 24) ^ (Result >> 16) ^ (Result >> 8) ^ Result;
}

Referenced by _IRQL_requires_(), ExpAddTagForBigPages(), and ExpFindAndRemoveTagBigPages().

ExpDecodePoolLink()

PLIST_ENTRY NTAPI ExpDecodePoolLink

(

IN PLIST_ENTRY

Link

)

Definition at line 85 of file expool.c.

{
    return (PLIST_ENTRY)((ULONG_PTR)Link & ~1);
}

Referenced by ExFreePoolWithTag(), ExpCheckPoolLinks(), ExpInsertPoolHeadList(), ExpInsertPoolTailList(), ExpIsPoolListEmpty(), ExpRemovePoolEntryList(), ExpRemovePoolHeadList(), and ExpRemovePoolTailList().

ExpEncodePoolLink()

PLIST_ENTRY NTAPI ExpEncodePoolLink

(

IN PLIST_ENTRY

Link

)

Definition at line 92 of file expool.c.

{
    return (PLIST_ENTRY)((ULONG_PTR)Link | 1);
}

Referenced by ExpInitializePoolListHead(), ExpInsertPoolHeadList(), ExpInsertPoolTailList(), ExpRemovePoolEntryList(), ExpRemovePoolHeadList(), and ExpRemovePoolTailList().

ExpFindAndRemoveTagBigPages()

ULONG NTAPI ExpFindAndRemoveTagBigPages	(	IN PVOID	Va,
		OUT PULONG_PTR	BigPages,
		IN POOL_TYPE	PoolType
	)

Definition at line 1682 of file expool.c.

{
    BOOLEAN FirstTry = TRUE;
    SIZE_T TableSize;
    KIRQL OldIrql;
    ULONG PoolTag, Hash;
    PPOOL_TRACKER_BIG_PAGES Entry;
    ASSERT(((ULONG_PTR)Va & POOL_BIG_TABLE_ENTRY_FREE) == 0);
    ASSERT(!(PoolType & SESSION_POOL_MASK));
 
    //
    // As the table is expandable, these values must only be read after acquiring
    // the lock to avoid a teared access during an expansion
    //
    Hash = ExpComputePartialHashForAddress(Va);
    KeAcquireSpinLock(&ExpLargePoolTableLock, &OldIrql);
    Hash &= PoolBigPageTableHash;
    TableSize = PoolBigPageTableSize;
 
    //
    // Loop while trying to find this big page allocation
    //
    while (PoolBigPageTable[Hash].Va != Va)
    {
        //
        // Increment the size until we go past the end of the table
        //
        if (++Hash >= TableSize)
        {
            //
            // Is this the second time we've tried?
            //
            if (!FirstTry)
            {
                //
                // This means it was never inserted into the pool table and it
                // received the special "BIG" tag -- return that and return 0
                // so that the code can ask Mm for the page count instead
                //
                KeReleaseSpinLock(&ExpLargePoolTableLock, OldIrql);
                *BigPages = 0;
                return ' GIB';
            }
 
            //
            // The first time this happens, reset the hash index and try again
            //
            Hash = 0;
            FirstTry = FALSE;
        }
    }
 
    //
    // Now capture all the information we need from the entry, since after we
    // release the lock, the data can change
    //
    Entry = &PoolBigPageTable[Hash];
    *BigPages = Entry->NumberOfPages;
    PoolTag = Entry->Key;
 
    //
    // Set the free bit, and decrement the number of allocations. Finally, release
    // the lock and return the tag that was located
    //
    Entry->Va = (PVOID)((ULONG_PTR)Entry->Va | POOL_BIG_TABLE_ENTRY_FREE);
 
    ExpPoolBigEntriesInUse--;
 
    /* If reaching 12.5% of the size (or whatever integer rounding gets us to),
     * halve the allocation size, which will get us to 25% of space used. */
    if (ExpPoolBigEntriesInUse < (PoolBigPageTableSize / (POOL_BIG_TABLE_USE_RATE * 2)))
    {
        /* Shrink the table. */
        ExpReallocateBigPageTable(OldIrql, TRUE);
    }
    else
    {
        KeReleaseSpinLock(&ExpLargePoolTableLock, OldIrql);
    }
    return PoolTag;
}

Referenced by ExFreePoolWithTag().

ExpGetPoolTagInfoTarget()

VOID NTAPI ExpGetPoolTagInfoTarget	(	IN PKDPC	Dpc,
		IN PVOID	DeferredContext,
		IN PVOID	SystemArgument1,
		IN PVOID	SystemArgument2
	)

Definition at line 1321 of file expool.c.

{
    PPOOL_DPC_CONTEXT Context = DeferredContext;
    UNREFERENCED_PARAMETER(Dpc);
    ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
 
    //
    // Make sure we win the race, and if we did, copy the data atomically
    //
    if (KeSignalCallDpcSynchronize(SystemArgument2))
    {
        RtlCopyMemory(Context->PoolTrackTable,
                      PoolTrackTable,
                      Context->PoolTrackTableSize * sizeof(POOL_TRACKER_TABLE));
 
        //
        // This is here because ReactOS does not yet support expansion
        //
        ASSERT(Context->PoolTrackTableSizeExpansion == 0);
    }
 
    //
    // Regardless of whether we won or not, we must now synchronize and then
    // decrement the barrier since this is one more processor that has completed
    // the callback.
    //
    KeSignalCallDpcSynchronize(SystemArgument2);
    KeSignalCallDpcDone(SystemArgument1);
}

Referenced by ExGetPoolTagInfo().

ExpInitializePoolListHead()

VOID NTAPI ExpInitializePoolListHead

(

IN PLIST_ENTRY

ListHead

)

Definition at line 114 of file expool.c.

{
    ListHead->Flink = ListHead->Blink = ExpEncodePoolLink(ListHead);
}

Referenced by ExInitializePoolDescriptor().

ExpInsertPoolHeadList()

VOID NTAPI ExpInsertPoolHeadList	(	IN PLIST_ENTRY	ListHead,
		IN PLIST_ENTRY	Entry
	)

Definition at line 178 of file expool.c.

{
    PLIST_ENTRY Flink;
    ExpCheckPoolLinks(ListHead);
    Flink = ExpDecodePoolLink(ListHead->Flink);
    Entry->Flink = ExpEncodePoolLink(Flink);
    Entry->Blink = ExpEncodePoolLink(ListHead);
    Flink->Blink = ExpEncodePoolLink(Entry);
    ListHead->Flink = ExpEncodePoolLink(Entry);
    ExpCheckPoolLinks(ListHead);
}

Referenced by ExFreePoolWithTag().

ExpInsertPoolTailList()

VOID NTAPI ExpInsertPoolTailList	(	IN PLIST_ENTRY	ListHead,
		IN PLIST_ENTRY	Entry
	)

Definition at line 163 of file expool.c.

{
    PLIST_ENTRY Blink;
    ExpCheckPoolLinks(ListHead);
    Blink = ExpDecodePoolLink(ListHead->Blink);
    Entry->Flink = ExpEncodePoolLink(ListHead);
    Entry->Blink = ExpEncodePoolLink(Blink);
    Blink->Flink = ExpEncodePoolLink(Entry);
    ListHead->Blink = ExpEncodePoolLink(Entry);
    ExpCheckPoolLinks(ListHead);
}

Referenced by ExAllocatePoolWithTag().

ExpInsertPoolTracker()

VOID NTAPI ExpInsertPoolTracker	(	IN ULONG	Key,
		IN SIZE_T	NumberOfBytes,
		IN POOL_TYPE	PoolType
	)

Definition at line 851 of file expool.c.

{
    ULONG Hash, Index;
    KIRQL OldIrql;
    PPOOL_TRACKER_TABLE Table, TableEntry;
    SIZE_T TableMask, TableSize;
 
    //
    // Remove the PROTECTED_POOL flag which is not part of the tag
    //
    Key &= ~PROTECTED_POOL;
 
    //
    // With WinDBG you can set a tag you want to break on when an allocation is
    // attempted
    //
    if (Key == PoolHitTag) DbgBreakPoint();
 
    //
    // There is also an internal flag you can set to break on malformed tags
    //
    if (ExStopBadTags) ASSERT(Key & 0xFFFFFF00);
 
    //
    // ASSERT on ReactOS features not yet supported
    //
    ASSERT(!(PoolType & SESSION_POOL_MASK));
 
    //
    // Why the double indirection? Because normally this function is also used
    // when doing session pool allocations, which has another set of tables,
    // sizes, and masks that live in session pool. Now we don't support session
    // pool so we only ever use the regular tables, but I'm keeping the code this
    // way so that the day we DO support session pool, it won't require that
    // many changes
    //
    Table = PoolTrackTable;
    TableMask = PoolTrackTableMask;
    TableSize = PoolTrackTableSize;
    DBG_UNREFERENCED_LOCAL_VARIABLE(TableSize);
 
    //
    // Compute the hash for this key, and loop all the possible buckets
    //
    Hash = ExpComputeHashForTag(Key, TableMask);
    Index = Hash;
    while (TRUE)
    {
        //
        // Do we already have an entry for this tag? */
        //
        TableEntry = &Table[Hash];
        if (TableEntry->Key == Key)
        {
            //
            // Increment the counters depending on if this was paged or nonpaged
            // pool
            //
            if ((PoolType & BASE_POOL_TYPE_MASK) == NonPagedPool)
            {
                InterlockedIncrement(&TableEntry->NonPagedAllocs);
                InterlockedExchangeAddSizeT(&TableEntry->NonPagedBytes, NumberOfBytes);
                return;
            }
            InterlockedIncrement(&TableEntry->PagedAllocs);
            InterlockedExchangeAddSizeT(&TableEntry->PagedBytes, NumberOfBytes);
            return;
        }
 
        //
        // We don't have an entry yet, but we've found a free bucket for it
        //
        if (!(TableEntry->Key) && (Hash != PoolTrackTableSize - 1))
        {
            //
            // We need to hold the lock while creating a new entry, since other
            // processors might be in this code path as well
            //
            ExAcquireSpinLock(&ExpTaggedPoolLock, &OldIrql);
            if (!PoolTrackTable[Hash].Key)
            {
                //
                // We've won the race, so now create this entry in the bucket
                //
                ASSERT(Table[Hash].Key == 0);
                PoolTrackTable[Hash].Key = Key;
                TableEntry->Key = Key;
            }
            ExReleaseSpinLock(&ExpTaggedPoolLock, OldIrql);
 
            //
            // Now we force the loop to run again, and we should now end up in
            // the code path above which does the interlocked increments...
            //
            continue;
        }
 
        //
        // This path is hit when we don't have an entry, and the current bucket
        // is full, so we simply try the next one
        //
        Hash = (Hash + 1) & TableMask;
        if (Hash == Index) break;
    }
 
    //
    // And finally this path is hit when all the buckets are full, and we need
    // some expansion. This path is not yet supported in ReactOS and so we'll
    // ignore the tag
    //
    DPRINT1("Out of pool tag space, ignoring...\n");
}

Referenced by _IRQL_requires_(), ExAllocatePoolWithTag(), and InitializePool().

ExpIsPoolListEmpty()

BOOLEAN NTAPI ExpIsPoolListEmpty

(

IN PLIST_ENTRY

ListHead

)

Definition at line 121 of file expool.c.

{
    return (ExpDecodePoolLink(ListHead->Flink) == ListHead);
}

Referenced by ExAllocatePoolWithTag().

ExpRemovePoolEntryList()

VOID NTAPI ExpRemovePoolEntryList

(

IN PLIST_ENTRY

Entry

)

Definition at line 128 of file expool.c.

{
    PLIST_ENTRY Blink, Flink;
    Flink = ExpDecodePoolLink(Entry->Flink);
    Blink = ExpDecodePoolLink(Entry->Blink);
    Flink->Blink = ExpEncodePoolLink(Blink);
    Blink->Flink = ExpEncodePoolLink(Flink);
}

Referenced by ExFreePoolWithTag().

ExpRemovePoolHeadList()

PLIST_ENTRY NTAPI ExpRemovePoolHeadList

(

IN PLIST_ENTRY

ListHead

)

Definition at line 139 of file expool.c.

{
    PLIST_ENTRY Entry, Flink;
    Entry = ExpDecodePoolLink(ListHead->Flink);
    Flink = ExpDecodePoolLink(Entry->Flink);
    ListHead->Flink = ExpEncodePoolLink(Flink);
    Flink->Blink = ExpEncodePoolLink(ListHead);
    return Entry;
}

Referenced by ExAllocatePoolWithTag().

ExpRemovePoolTailList()

PLIST_ENTRY NTAPI ExpRemovePoolTailList

(

IN PLIST_ENTRY

ListHead

)

Definition at line 151 of file expool.c.

{
    PLIST_ENTRY Entry, Blink;
    Entry = ExpDecodePoolLink(ListHead->Blink);
    Blink = ExpDecodePoolLink(Entry->Blink);
    ListHead->Blink = ExpEncodePoolLink(Blink);
    Blink->Flink = ExpEncodePoolLink(ListHead);
    return Entry;
}

ExpRemovePoolTracker()

VOID NTAPI ExpRemovePoolTracker	(	IN ULONG	Key,
		IN SIZE_T	NumberOfBytes,
		IN POOL_TYPE	PoolType
	)

Definition at line 760 of file expool.c.

{
    ULONG Hash, Index;
    PPOOL_TRACKER_TABLE Table, TableEntry;
    SIZE_T TableMask, TableSize;
 
    //
    // Remove the PROTECTED_POOL flag which is not part of the tag
    //
    Key &= ~PROTECTED_POOL;
 
    //
    // With WinDBG you can set a tag you want to break on when an allocation is
    // attempted
    //
    if (Key == PoolHitTag) DbgBreakPoint();
 
    //
    // Why the double indirection? Because normally this function is also used
    // when doing session pool allocations, which has another set of tables,
    // sizes, and masks that live in session pool. Now we don't support session
    // pool so we only ever use the regular tables, but I'm keeping the code this
    // way so that the day we DO support session pool, it won't require that
    // many changes
    //
    Table = PoolTrackTable;
    TableMask = PoolTrackTableMask;
    TableSize = PoolTrackTableSize;
    DBG_UNREFERENCED_LOCAL_VARIABLE(TableSize);
 
    //
    // Compute the hash for this key, and loop all the possible buckets
    //
    Hash = ExpComputeHashForTag(Key, TableMask);
    Index = Hash;
    while (TRUE)
    {
        //
        // Have we found the entry for this tag? */
        //
        TableEntry = &Table[Hash];
        if (TableEntry->Key == Key)
        {
            //
            // Decrement the counters depending on if this was paged or nonpaged
            // pool
            //
            if ((PoolType & BASE_POOL_TYPE_MASK) == NonPagedPool)
            {
                InterlockedIncrement(&TableEntry->NonPagedFrees);
                InterlockedExchangeAddSizeT(&TableEntry->NonPagedBytes,
                                            -(SSIZE_T)NumberOfBytes);
                return;
            }
            InterlockedIncrement(&TableEntry->PagedFrees);
            InterlockedExchangeAddSizeT(&TableEntry->PagedBytes,
                                        -(SSIZE_T)NumberOfBytes);
            return;
        }
 
        //
        // We should have only ended up with an empty entry if we've reached
        // the last bucket
        //
        if (!TableEntry->Key)
        {
            DPRINT1("Empty item reached in tracker table. Hash=0x%lx, TableMask=0x%lx, Tag=0x%08lx, NumberOfBytes=%lu, PoolType=%d\n",
                    Hash, TableMask, Key, (ULONG)NumberOfBytes, PoolType);
            ASSERT(Hash == TableMask);
        }
 
        //
        // This path is hit when we don't have an entry, and the current bucket
        // is full, so we simply try the next one
        //
        Hash = (Hash + 1) & TableMask;
        if (Hash == Index) break;
    }
 
    //
    // And finally this path is hit when all the buckets are full, and we need
    // some expansion. This path is not yet supported in ReactOS and so we'll
    // ignore the tag
    //
    DPRINT1("Out of pool tag space, ignoring...\n");
}

Referenced by _IRQL_requires_(), and ExFreePoolWithTag().

ExpSeedHotTags()

VOID NTAPI ExpSeedHotTags

(

VOID

)

Definition at line 640 of file expool.c.

{
    ULONG i, Key, Hash, Index;
    PPOOL_TRACKER_TABLE TrackTable = PoolTrackTable;
    ULONG TagList[] =
    {
        '  oI',
        ' laH',
        'PldM',
        'LooP',
        'tSbO',
        ' prI',
        'bdDN',
        'LprI',
        'pOoI',
        ' ldM',
        'eliF',
        'aVMC',
        'dSeS',
        'CFtN',
        'looP',
        'rPCT',
        'bNMC',
        'dTeS',
        'sFtN',
        'TPCT',
        'CPCT',
        ' yeK',
        'qSbO',
        'mNoI',
        'aEoI',
        'cPCT',
        'aFtN',
        '0ftN',
        'tceS',
        'SprI',
        'ekoT',
        '  eS',
        'lCbO',
        'cScC',
        'lFtN',
        'cAeS',
        'mfSF',
        'kWcC',
        'miSF',
        'CdfA',
        'EdfA',
        'orSF',
        'nftN',
        'PRIU',
        'rFpN',
        'RFpN',
        'aPeS',
        'sUeS',
        'FpcA',
        'MpcA',
        'cSeS',
        'mNbO',
        'sFpN',
        'uLeS',
        'DPcS',
        'nevE',
        'vrqR',
        'ldaV',
        '  pP',
        'SdaV',
        ' daV',
        'LdaV',
        'FdaV',
        ' GIB',
    };
 
    //
    // Loop all 64 hot tags
    //
    ASSERT((sizeof(TagList) / sizeof(ULONG)) == 64);
    for (i = 0; i < sizeof(TagList) / sizeof(ULONG); i++)
    {
        //
        // Get the current tag, and compute its hash in the tracker table
        //
        Key = TagList[i];
        Hash = ExpComputeHashForTag(Key, PoolTrackTableMask);
 
        //
        // Loop all the hashes in this index/bucket
        //
        Index = Hash;
        while (TRUE)
        {
            //
            // Find an empty entry, and make sure this isn't the last hash that
            // can fit.
            //
            // On checked builds, also make sure this is the first time we are
            // seeding this tag.
            //
            ASSERT(TrackTable[Hash].Key != Key);
            if (!(TrackTable[Hash].Key) && (Hash != PoolTrackTableSize - 1))
            {
                //
                // It has been seeded, move on to the next tag
                //
                TrackTable[Hash].Key = Key;
                break;
            }
 
            //
            // This entry was already taken, compute the next possible hash while
            // making sure we're not back at our initial index.
            //
            ASSERT(TrackTable[Hash].Key != Key);
            Hash = (Hash + 1) & PoolTrackTableMask;
            if (Hash == Index) break;
        }
    }
}

Referenced by InitializePool().

ExQueryPoolBlockSize()

SIZE_T NTAPI ExQueryPoolBlockSize	(	IN PVOID	PoolBlock,
		OUT PBOOLEAN	QuotaCharged
	)

Definition at line 2933 of file expool.c.

{
    //
    // Not implemented
    //
    UNIMPLEMENTED;
    return FALSE;
}

ExQueryPoolUsage()

VOID NTAPI ExQueryPoolUsage	(	OUT PULONG	PagedPoolPages,
		OUT PULONG	NonPagedPoolPages,
		OUT PULONG	PagedPoolAllocs,
		OUT PULONG	PagedPoolFrees,
		OUT PULONG	PagedPoolLookasideHits,
		OUT PULONG	NonPagedPoolAllocs,
		OUT PULONG	NonPagedPoolFrees,
		OUT PULONG	NonPagedPoolLookasideHits
	)

Definition at line 1768 of file expool.c.

{
    ULONG i;
    PPOOL_DESCRIPTOR PoolDesc;
 
    //
    // Assume all failures
    //
    *PagedPoolPages = 0;
    *PagedPoolAllocs = 0;
    *PagedPoolFrees = 0;
 
    //
    // Tally up the totals for all the apged pool
    //
    for (i = 0; i < ExpNumberOfPagedPools + 1; i++)
    {
        PoolDesc = ExpPagedPoolDescriptor[i];
        *PagedPoolPages += PoolDesc->TotalPages + PoolDesc->TotalBigPages;
        *PagedPoolAllocs += PoolDesc->RunningAllocs;
        *PagedPoolFrees += PoolDesc->RunningDeAllocs;
    }
 
    //
    // The first non-paged pool has a hardcoded well-known descriptor name
    //
    PoolDesc = &NonPagedPoolDescriptor;
    *NonPagedPoolPages = PoolDesc->TotalPages + PoolDesc->TotalBigPages;
    *NonPagedPoolAllocs = PoolDesc->RunningAllocs;
    *NonPagedPoolFrees = PoolDesc->RunningDeAllocs;
 
    //
    // If the system has more than one non-paged pool, copy the other descriptor
    // totals as well
    //
#if 0
    if (ExpNumberOfNonPagedPools > 1)
    {
        for (i = 0; i < ExpNumberOfNonPagedPools; i++)
        {
            PoolDesc = ExpNonPagedPoolDescriptor[i];
            *NonPagedPoolPages += PoolDesc->TotalPages + PoolDesc->TotalBigPages;
            *NonPagedPoolAllocs += PoolDesc->RunningAllocs;
            *NonPagedPoolFrees += PoolDesc->RunningDeAllocs;
        }
    }
#endif
 
    //
    // Get the amount of hits in the system lookaside lists
    //
    if (!IsListEmpty(&ExPoolLookasideListHead))
    {
        PLIST_ENTRY ListEntry;
 
        for (ListEntry = ExPoolLookasideListHead.Flink;
             ListEntry != &ExPoolLookasideListHead;
             ListEntry = ListEntry->Flink)
        {
            PGENERAL_LOOKASIDE Lookaside;
 
            Lookaside = CONTAINING_RECORD(ListEntry, GENERAL_LOOKASIDE, ListEntry);
 
            if (Lookaside->Type == NonPagedPool)
            {
                *NonPagedPoolLookasideHits += Lookaside->AllocateHits;
            }
            else
            {
                *PagedPoolLookasideHits += Lookaside->AllocateHits;
            }
        }
    }
}

Referenced by QSI_DEF().

ExReturnPoolQuota()

VOID NTAPI ExReturnPoolQuota

(

IN PVOID

P

)

Definition at line 1852 of file expool.c.

{
    PPOOL_HEADER Entry;
    POOL_TYPE PoolType;
    USHORT BlockSize;
    PEPROCESS Process;
 
    if ((ExpPoolFlags & POOL_FLAG_SPECIAL_POOL) &&
        (MmIsSpecialPoolAddress(P)))
    {
        return;
    }
 
    Entry = P;
    Entry--;
    ASSERT((ULONG_PTR)Entry % POOL_BLOCK_SIZE == 0);
 
    PoolType = Entry->PoolType - 1;
    BlockSize = Entry->BlockSize;
 
    if (PoolType & QUOTA_POOL_MASK)
    {
        Process = ((PVOID *)POOL_NEXT_BLOCK(Entry))[-1];
        ASSERT(Process != NULL);
        if (Process)
        {
            if (Process->Pcb.Header.Type != ProcessObject)
            {
                DPRINT1("Object %p is not a process. Type %u, pool type 0x%x, block size %u\n",
                        Process, Process->Pcb.Header.Type, Entry->PoolType, BlockSize);
                KeBugCheckEx(BAD_POOL_CALLER,
                             POOL_BILLED_PROCESS_INVALID,
                             (ULONG_PTR)P,
                             Entry->PoolTag,
                             (ULONG_PTR)Process);
            }
            ((PVOID *)POOL_NEXT_BLOCK(Entry))[-1] = NULL;
            PsReturnPoolQuota(Process,
                              PoolType & BASE_POOL_TYPE_MASK,
                              BlockSize * POOL_BLOCK_SIZE);
            ObDereferenceObject(Process);
        }
    }
}

Referenced by IoFreeIrp().

ExUnlockPool()

FORCEINLINE VOID ExUnlockPool	(	IN PPOOL_DESCRIPTOR	Descriptor,
		IN KIRQL	OldIrql
	)

Definition at line 1297 of file expool.c.

{
    //
    // Check if this is nonpaged pool
    //
    if ((Descriptor->PoolType & BASE_POOL_TYPE_MASK) == NonPagedPool)
    {
        //
        // Use the queued spin lock
        //
        KeReleaseQueuedSpinLock(LockQueueNonPagedPoolLock, OldIrql);
    }
    else
    {
        //
        // Use the guarded mutex
        //
        KeReleaseGuardedMutex(Descriptor->LockAddress);
    }
}

Referenced by ExAllocatePoolWithTag(), and ExFreePoolWithTag().

InitializePool()

VOID NTAPI InitializePool	(	IN POOL_TYPE	PoolType,
		IN ULONG	Threshold
	)

Definition at line 1020 of file expool.c.

{
    PPOOL_DESCRIPTOR Descriptor;
    SIZE_T TableSize;
    ULONG i;
 
    //
    // Check what kind of pool this is
    //
    if (PoolType == NonPagedPool)
    {
        //
        // Compute the track table size and convert it from a power of two to an
        // actual byte size
        //
        // NOTE: On checked builds, we'll assert if the registry table size was
        // invalid, while on retail builds we'll just break out of the loop at
        // that point.
        //
        TableSize = min(PoolTrackTableSize, MmSizeOfNonPagedPoolInBytes >> 8);
        for (i = 0; i < 32; i++)
        {
            if (TableSize & 1)
            {
                ASSERT((TableSize & ~1) == 0);
                if (!(TableSize & ~1)) break;
            }
            TableSize >>= 1;
        }
 
        //
        // If we hit bit 32, than no size was defined in the registry, so
        // we'll use the default size of 2048 entries.
        //
        // Otherwise, use the size from the registry, as long as it's not
        // smaller than 64 entries.
        //
        if (i == 32)
        {
            PoolTrackTableSize = 2048;
        }
        else
        {
            PoolTrackTableSize = max(1 << i, 64);
        }
 
        //
        // Loop trying with the biggest specified size first, and cut it down
        // by a power of two each iteration in case not enough memory exist
        //
        while (TRUE)
        {
            //
            // Do not allow overflow
            //
            if ((PoolTrackTableSize + 1) > (MAXULONG_PTR / sizeof(POOL_TRACKER_TABLE)))
            {
                PoolTrackTableSize >>= 1;
                continue;
            }
 
            //
            // Allocate the tracker table and exit the loop if this worked
            //
            PoolTrackTable = MiAllocatePoolPages(NonPagedPool,
                                                 (PoolTrackTableSize + 1) *
                                                 sizeof(POOL_TRACKER_TABLE));
            if (PoolTrackTable) break;
 
            //
            // Otherwise, as long as we're not down to the last bit, keep
            // iterating
            //
            if (PoolTrackTableSize == 1)
            {
                KeBugCheckEx(MUST_SUCCEED_POOL_EMPTY,
                             TableSize,
                             0xFFFFFFFF,
                             0xFFFFFFFF,
                             0xFFFFFFFF);
            }
            PoolTrackTableSize >>= 1;
        }
 
        //
        // Add one entry, compute the hash, and zero the table
        //
        PoolTrackTableSize++;
        PoolTrackTableMask = PoolTrackTableSize - 2;
 
        RtlZeroMemory(PoolTrackTable,
                      PoolTrackTableSize * sizeof(POOL_TRACKER_TABLE));
 
        //
        // Finally, add the most used tags to speed up those allocations
        //
        ExpSeedHotTags();
 
        //
        // We now do the exact same thing with the tracker table for big pages
        //
        TableSize = min(PoolBigPageTableSize, MmSizeOfNonPagedPoolInBytes >> 8);
        for (i = 0; i < 32; i++)
        {
            if (TableSize & 1)
            {
                ASSERT((TableSize & ~1) == 0);
                if (!(TableSize & ~1)) break;
            }
            TableSize >>= 1;
        }
 
        //
        // For big pages, the default tracker table is 4096 entries, while the
        // minimum is still 64
        //
        if (i == 32)
        {
            PoolBigPageTableSize = 4096;
        }
        else
        {
            PoolBigPageTableSize = max(1 << i, 64);
        }
 
        //
        // Again, run the exact same loop we ran earlier, but this time for the
        // big pool tracker instead
        //
        while (TRUE)
        {
            if ((PoolBigPageTableSize + 1) > (MAXULONG_PTR / sizeof(POOL_TRACKER_BIG_PAGES)))
            {
                PoolBigPageTableSize >>= 1;
                continue;
            }
 
            PoolBigPageTable = MiAllocatePoolPages(NonPagedPool,
                                                   PoolBigPageTableSize *
                                                   sizeof(POOL_TRACKER_BIG_PAGES));
            if (PoolBigPageTable) break;
 
            if (PoolBigPageTableSize == 1)
            {
                KeBugCheckEx(MUST_SUCCEED_POOL_EMPTY,
                             TableSize,
                             0xFFFFFFFF,
                             0xFFFFFFFF,
                             0xFFFFFFFF);
            }
 
            PoolBigPageTableSize >>= 1;
        }
 
        //
        // An extra entry is not needed for for the big pool tracker, so just
        // compute the hash and zero it
        //
        PoolBigPageTableHash = PoolBigPageTableSize - 1;
        RtlZeroMemory(PoolBigPageTable,
                      PoolBigPageTableSize * sizeof(POOL_TRACKER_BIG_PAGES));
        for (i = 0; i < PoolBigPageTableSize; i++)
        {
            PoolBigPageTable[i].Va = (PVOID)POOL_BIG_TABLE_ENTRY_FREE;
        }
 
        //
        // During development, print this out so we can see what's happening
        //
        DPRINT("EXPOOL: Pool Tracker Table at: 0x%p with 0x%lx bytes\n",
                PoolTrackTable, PoolTrackTableSize * sizeof(POOL_TRACKER_TABLE));
        DPRINT("EXPOOL: Big Pool Tracker Table at: 0x%p with 0x%lx bytes\n",
                PoolBigPageTable, PoolBigPageTableSize * sizeof(POOL_TRACKER_BIG_PAGES));
 
        //
        // Insert the generic tracker for all of big pool
        //
        ExpInsertPoolTracker('looP',
                             ROUND_TO_PAGES(PoolBigPageTableSize *
                                            sizeof(POOL_TRACKER_BIG_PAGES)),
                             NonPagedPool);
 
        //
        // No support for NUMA systems at this time
        //
        ASSERT(KeNumberNodes == 1);
 
        //
        // Initialize the tag spinlock
        //
        KeInitializeSpinLock(&ExpTaggedPoolLock);
 
        //
        // Initialize the nonpaged pool descriptor
        //
        PoolVector[NonPagedPool] = &NonPagedPoolDescriptor;
        ExInitializePoolDescriptor(PoolVector[NonPagedPool],
                                   NonPagedPool,
                                   0,
                                   Threshold,
                                   NULL);
    }
    else
    {
        //
        // No support for NUMA systems at this time
        //
        ASSERT(KeNumberNodes == 1);
 
        //
        // Allocate the pool descriptor
        //
        Descriptor = ExAllocatePoolWithTag(NonPagedPool,
                                           sizeof(KGUARDED_MUTEX) +
                                           sizeof(POOL_DESCRIPTOR),
                                           'looP');
        if (!Descriptor)
        {
            //
            // This is really bad...
            //
            KeBugCheckEx(MUST_SUCCEED_POOL_EMPTY,
                         0,
                         -1,
                         -1,
                         -1);
        }
 
        //
        // Setup the vector and guarded mutex for paged pool
        //
        PoolVector[PagedPool] = Descriptor;
        ExpPagedPoolMutex = (PKGUARDED_MUTEX)(Descriptor + 1);
        ExpPagedPoolDescriptor[0] = Descriptor;
        KeInitializeGuardedMutex(ExpPagedPoolMutex);
        ExInitializePoolDescriptor(Descriptor,
                                   PagedPool,
                                   0,
                                   Threshold,
                                   ExpPagedPoolMutex);
 
        //
        // Insert the generic tracker for all of nonpaged pool
        //
        ExpInsertPoolTracker('looP',
                             ROUND_TO_PAGES(PoolTrackTableSize * sizeof(POOL_TRACKER_TABLE)),
                             NonPagedPool);
    }
}

Referenced by MiBuildPagedPool(), and MiInitMachineDependent().

Variable Documentation

ExpBigTableExpansionFailed

ULONG ExpBigTableExpansionFailed

Definition at line 48 of file expool.c.

Referenced by ExpAddTagForBigPages().

ExpLargePoolTableLock

KSPIN_LOCK ExpLargePoolTableLock

Definition at line 54 of file expool.c.

Referenced by _IRQL_requires_(), ExpAddTagForBigPages(), ExpCheckPoolAllocation(), and ExpFindAndRemoveTagBigPages().

ExpNumberOfPagedPools

ULONG ExpNumberOfPagedPools

Definition at line 41 of file expool.c.

Referenced by ExQueryPoolUsage().

ExPoolFailures

ULONG ExPoolFailures

Definition at line 57 of file expool.c.

Referenced by ExAllocatePoolWithTag().

ExpPagedPoolDescriptor

PPOOL_DESCRIPTOR ExpPagedPoolDescriptor[16+1]

Definition at line 43 of file expool.c.

Referenced by ExQueryPoolUsage(), and InitializePool().

ExpPagedPoolMutex

PKGUARDED_MUTEX ExpPagedPoolMutex

Definition at line 45 of file expool.c.

Referenced by InitializePool().

ExpPoolBigEntriesInUse

ULONG ExpPoolBigEntriesInUse

Definition at line 55 of file expool.c.

Referenced by _IRQL_requires_(), ExpAddTagForBigPages(), and ExpFindAndRemoveTagBigPages().

ExpPoolFlags

ULONG ExpPoolFlags

Definition at line 56 of file expool.c.

Referenced by ExAllocatePoolWithQuotaTag(), ExAllocatePoolWithTag(), ExFreePoolWithTag(), ExReturnPoolQuota(), and MiInitializeSpecialPool().

ExpTaggedPoolLock

KSPIN_LOCK ExpTaggedPoolLock

Definition at line 51 of file expool.c.

Referenced by ExpInsertPoolTracker(), and InitializePool().

ExStopBadTags

BOOLEAN ExStopBadTags

Definition at line 53 of file expool.c.

Referenced by ExpInsertPoolTracker().

MiLastPoolDumpTime

ULONGLONG MiLastPoolDumpTime

Definition at line 58 of file expool.c.

Referenced by ExAllocatePoolWithTag().

NonPagedPoolDescriptor

POOL_DESCRIPTOR NonPagedPoolDescriptor

Definition at line 42 of file expool.c.

Referenced by ExAllocatePoolWithTag(), ExQueryPoolUsage(), and InitializePool().

PoolBigPageTable

PPOOL_TRACKER_BIG_PAGES PoolBigPageTable

Definition at line 50 of file expool.c.

Referenced by _IRQL_requires_(), ExpAddTagForBigPages(), ExpCheckPoolAllocation(), ExpFindAndRemoveTagBigPages(), and InitializePool().

PoolBigPageTableHash

SIZE_T PoolBigPageTableHash

Definition at line 47 of file expool.c.

Referenced by _IRQL_requires_(), ExpAddTagForBigPages(), ExpFindAndRemoveTagBigPages(), and InitializePool().

PoolBigPageTableSize

SIZE_T PoolBigPageTableSize

Definition at line 47 of file expool.c.

Referenced by _IRQL_requires_(), ExpAddTagForBigPages(), ExpCheckPoolAllocation(), ExpFindAndRemoveTagBigPages(), and InitializePool().

PoolHitTag

ULONG PoolHitTag

Definition at line 52 of file expool.c.

Referenced by ExpInsertPoolTracker(), and ExpRemovePoolTracker().

PoolTrackTable

PPOOL_TRACKER_TABLE PoolTrackTable

Definition at line 49 of file expool.c.

Referenced by ExpGetPoolTagInfoTarget(), ExpInsertPoolTracker(), ExpRemovePoolTracker(), ExpSeedHotTags(), and InitializePool().

PoolTrackTableMask

SIZE_T PoolTrackTableMask

Definition at line 46 of file expool.c.

Referenced by ExpInsertPoolTracker(), ExpRemovePoolTracker(), ExpSeedHotTags(), and InitializePool().

PoolTrackTableSize

SIZE_T PoolTrackTableSize

Definition at line 46 of file expool.c.

Referenced by ExGetPoolTagInfo(), ExpInsertPoolTracker(), ExpRemovePoolTracker(), ExpSeedHotTags(), and InitializePool().

PoolVector

PPOOL_DESCRIPTOR PoolVector[2]

Definition at line 44 of file expool.c.

Referenced by ExAllocatePoolWithTag(), ExFreePoolWithTag(), and InitializePool().