rundown.c File Reference

#include <ntoskrnl.h>
#include <debug.h>

Include dependency graph for rundown.c:

Go to the source code of this file.

Macros
#define	NDEBUG

Functions
BOOLEAN FASTCALL	ExfAcquireRundownProtection (IN PEX_RUNDOWN_REF RunRef)
BOOLEAN FASTCALL	ExfAcquireRundownProtectionEx (IN PEX_RUNDOWN_REF RunRef, IN ULONG Count)
VOID FASTCALL	ExfInitializeRundownProtection (IN PEX_RUNDOWN_REF RunRef)
VOID FASTCALL	ExfReInitializeRundownProtection (IN PEX_RUNDOWN_REF RunRef)
VOID FASTCALL	ExfRundownCompleted (IN PEX_RUNDOWN_REF RunRef)
VOID FASTCALL	ExfReleaseRundownProtection (IN PEX_RUNDOWN_REF RunRef)
VOID FASTCALL	ExfReleaseRundownProtectionEx (IN PEX_RUNDOWN_REF RunRef, IN ULONG Count)
VOID FASTCALL	ExfWaitForRundownProtectionRelease (IN PEX_RUNDOWN_REF RunRef)
BOOLEAN FASTCALL	ExfAcquireRundownProtectionCacheAware (IN PEX_RUNDOWN_REF_CACHE_AWARE RunRefCacheAware)
BOOLEAN FASTCALL	ExfAcquireRundownProtectionCacheAwareEx (IN PEX_RUNDOWN_REF_CACHE_AWARE RunRefCacheAware, IN ULONG Count)
VOID FASTCALL	ExfReleaseRundownProtectionCacheAware (IN PEX_RUNDOWN_REF_CACHE_AWARE RunRefCacheAware)
VOID FASTCALL	ExfReleaseRundownProtectionCacheAwareEx (IN PEX_RUNDOWN_REF_CACHE_AWARE RunRefCacheAware, IN ULONG Count)
VOID FASTCALL	ExfWaitForRundownProtectionReleaseCacheAware (IN PEX_RUNDOWN_REF_CACHE_AWARE RunRefCacheAware)
VOID FASTCALL	ExfRundownCompletedCacheAware (IN PEX_RUNDOWN_REF_CACHE_AWARE RunRefCacheAware)
VOID FASTCALL	ExfReInitializeRundownProtectionCacheAware (IN PEX_RUNDOWN_REF_CACHE_AWARE RunRefCacheAware)
PEX_RUNDOWN_REF_CACHE_AWARE NTAPI	ExAllocateCacheAwareRundownProtection (IN POOL_TYPE PoolType, IN ULONG Tag)
VOID NTAPI	ExFreeCacheAwareRundownProtection (IN PEX_RUNDOWN_REF_CACHE_AWARE RunRefCacheAware)
VOID NTAPI	ExInitializeRundownProtectionCacheAware (IN PEX_RUNDOWN_REF_CACHE_AWARE RunRefCacheAware, IN SIZE_T Size)
SIZE_T NTAPI	ExSizeOfRundownProtectionCacheAware (VOID)

Macro Definition Documentation

NDEBUG

#define NDEBUG

Definition at line 14 of file rundown.c.

Function Documentation

ExAllocateCacheAwareRundownProtection()

PEX_RUNDOWN_REF_CACHE_AWARE NTAPI ExAllocateCacheAwareRundownProtection	(	IN POOL_TYPE	PoolType,
		IN ULONG	Tag
	)

Definition at line 562 of file rundown.c.

{
    PEX_RUNDOWN_REF RunRef;
    PVOID PoolToFree, RunRefs;
    ULONG RunRefSize, Count, Align;
    PEX_RUNDOWN_REF_CACHE_AWARE RunRefCacheAware;

    PAGED_CODE();

    /* Allocate the master structure */
    RunRefCacheAware = ExAllocatePoolWithTag(PoolType, sizeof(EX_RUNDOWN_REF_CACHE_AWARE), Tag);
    if (RunRefCacheAware == NULL)
    {
        return NULL;
    }

    /* Compute the size of each runref */
    RunRefCacheAware->Number = KeNumberProcessors;
    if (KeNumberProcessors <= 1)
    {
        RunRefSize = sizeof(EX_RUNDOWN_REF);
    }
    else
    {
        Align = KeGetRecommendedSharedDataAlignment();
        RunRefSize = Align;
        ASSERT((RunRefSize & (RunRefSize - 1)) == 0);
    }

    /* It must at least hold a EX_RUNDOWN_REF structure */
    ASSERT(sizeof(EX_RUNDOWN_REF) <= RunRefSize);
    RunRefCacheAware->RunRefSize = RunRefSize;

    /* Allocate our runref pool */
    PoolToFree = ExAllocatePoolWithTag(PoolType, RunRefSize * RunRefCacheAware->Number, Tag);
    if (PoolToFree == NULL)
    {
        ExFreePoolWithTag(RunRefCacheAware, Tag);
        return NULL;
    }

    /* On SMP, check for alignment */
    if (RunRefCacheAware->Number > 1 && (ULONG_PTR)PoolToFree & (Align - 1))
    {
        /* Not properly aligned, do it again! */
        ExFreePoolWithTag(PoolToFree, Tag);

        /* Allocate a bigger buffer to be able to align properly */
        PoolToFree = ExAllocatePoolWithTag(PoolType, RunRefSize * (RunRefCacheAware->Number + 1), Tag);
        if (PoolToFree == NULL)
        {
            ExFreePoolWithTag(RunRefCacheAware, Tag);
            return NULL;
        }

        RunRefs = (PVOID)ALIGN_UP_BY(PoolToFree, Align);
    }
    else
    {
        RunRefs = PoolToFree;
    }

    RunRefCacheAware->RunRefs = RunRefs;
    RunRefCacheAware->PoolToFree = PoolToFree;

    /* And initialize runref */
    if (RunRefCacheAware->Number != 0)
    {
        for (Count = 0; Count < RunRefCacheAware->Number; ++Count)
        {
            RunRef = ExGetRunRefForGivenProcessor(RunRefCacheAware, Count);
            _ExInitializeRundownProtection(RunRef);
        }
    }

    return RunRefCacheAware;
}

ExfAcquireRundownProtection()

BOOLEAN FASTCALL ExfAcquireRundownProtection

(

IN PEX_RUNDOWN_REF

RunRef

)

Definition at line 36 of file rundown.c.

{
    ULONG_PTR Value = RunRef->Count, NewValue;

    /* Loop until successfully incremented the counter */
    for (;;)
    {
        /* Make sure a rundown is not active */
        if (Value & EX_RUNDOWN_ACTIVE) return FALSE;

        /* Add a reference */
        NewValue = Value + EX_RUNDOWN_COUNT_INC;

        /* Change the value */
        NewValue = ExpChangeRundown(RunRef, NewValue, Value);
        if (NewValue == Value) return TRUE;

        /* Update it */
        Value = NewValue;
    }
}

ExfAcquireRundownProtectionCacheAware()

BOOLEAN FASTCALL ExfAcquireRundownProtectionCacheAware

(

IN PEX_RUNDOWN_REF_CACHE_AWARE

RunRefCacheAware

)

Definition at line 384 of file rundown.c.

{
    PEX_RUNDOWN_REF RunRef;

    RunRef = ExGetRunRefForGivenProcessor(RunRefCacheAware, KeGetCurrentProcessorNumber());
    return _ExAcquireRundownProtection(RunRef);
}

ExfAcquireRundownProtectionCacheAwareEx()

BOOLEAN FASTCALL ExfAcquireRundownProtectionCacheAwareEx	(	IN PEX_RUNDOWN_REF_CACHE_AWARE	RunRefCacheAware,
		IN ULONG	Count
	)

Definition at line 397 of file rundown.c.

{
    PEX_RUNDOWN_REF RunRef;

    RunRef = ExGetRunRefForGivenProcessor(RunRefCacheAware, KeGetCurrentProcessorNumber());
    return ExfAcquireRundownProtectionEx(RunRef, Count);
}

ExfAcquireRundownProtectionEx()

BOOLEAN FASTCALL ExfAcquireRundownProtectionEx	(	IN PEX_RUNDOWN_REF	RunRef,
		IN ULONG	Count
	)

Definition at line 78 of file rundown.c.

{
    ULONG_PTR Value = RunRef->Count, NewValue;

    /* Loop until successfully incremented the counter */
    for (;;)
    {
        /* Make sure a rundown is not active */
        if (Value & EX_RUNDOWN_ACTIVE) return FALSE;

        /* Add references */
        NewValue = Value + EX_RUNDOWN_COUNT_INC * Count;

        /* Change the value */
        NewValue = ExpChangeRundown(RunRef, NewValue, Value);
        if (NewValue == Value) return TRUE;

        /* Update the value */
        Value = NewValue;
    }
}

Referenced by ExCompareExchangeCallBack(), ExfAcquireRundownProtectionCacheAwareEx(), and ExReferenceCallBackBlock().

ExfInitializeRundownProtection()

VOID FASTCALL ExfInitializeRundownProtection

(

IN PEX_RUNDOWN_REF

RunRef

)

Definition at line 118 of file rundown.c.

{
    /* Set the count to zero */
    RunRef->Count = 0;
}

ExFreeCacheAwareRundownProtection()

VOID NTAPI ExFreeCacheAwareRundownProtection

(

IN PEX_RUNDOWN_REF_CACHE_AWARE

RunRefCacheAware

)

Definition at line 646 of file rundown.c.

{
    PAGED_CODE();

    /*
     * This is to be called for RunRefCacheAware that were allocated with
     * ExAllocateCacheAwareRundownProtection and not for user-allocated
     * ones
     */
    ASSERT(RunRefCacheAware->PoolToFree != (PVOID)0xBADCA11);

    /* We don't know the tag that as used for allocation */
    ExFreePoolWithTag(RunRefCacheAware->PoolToFree, 0);
    ExFreePoolWithTag(RunRefCacheAware, 0);
}

ExfReInitializeRundownProtection()

VOID FASTCALL ExfReInitializeRundownProtection

(

IN PEX_RUNDOWN_REF

RunRef

)

Definition at line 141 of file rundown.c.

{
    PAGED_CODE();

    /* Sanity check */
    ASSERT((RunRef->Count & EX_RUNDOWN_ACTIVE) != 0);

    /* Reset the count */
    ExpSetRundown(RunRef, 0);
}

ExfReInitializeRundownProtectionCacheAware()

VOID FASTCALL ExfReInitializeRundownProtectionCacheAware

(

IN PEX_RUNDOWN_REF_CACHE_AWARE

RunRefCacheAware

)

Definition at line 534 of file rundown.c.

{
    PEX_RUNDOWN_REF RunRef;
    ULONG ProcCount, Current;

    ProcCount = RunRefCacheAware->Number;
    /* No proc, nothing to do */
    if (ProcCount == 0)
    {
        return;
    }

    /* We will mark all our runrefs inactive */
    for (Current = 0; Current < ProcCount; ++Current)
    {
        /* Get the runref for the proc */
        RunRef = ExGetRunRefForGivenProcessor(RunRefCacheAware, Current);
        ASSERT((RunRef->Count & EX_RUNDOWN_ACTIVE) != 0);

        ExpSetRundown(RunRef, 0);
    }
}

ExfReleaseRundownProtection()

VOID FASTCALL ExfReleaseRundownProtection

(

IN PEX_RUNDOWN_REF

RunRef

)

Definition at line 197 of file rundown.c.

{
    ULONG_PTR Value = RunRef->Count, NewValue;
    PEX_RUNDOWN_WAIT_BLOCK WaitBlock;

    /* Loop until successfully incremented the counter */
    for (;;)
    {
        /* Check if rundown is not active */
        if (!(Value & EX_RUNDOWN_ACTIVE))
        {
            /* Sanity check */
            ASSERT((Value >= EX_RUNDOWN_COUNT_INC) || (KeNumberProcessors > 1));

            /* Get the new value */
            NewValue = Value - EX_RUNDOWN_COUNT_INC;

            /* Change the value */
            NewValue = ExpChangeRundown(RunRef, NewValue, Value);
            if (NewValue == Value) break;

            /* Update value */
            Value = NewValue;
        }
        else
        {
            /* Get the wait block */
            WaitBlock = (PEX_RUNDOWN_WAIT_BLOCK)(Value & ~EX_RUNDOWN_ACTIVE);
            ASSERT((WaitBlock->Count > 0) || (KeNumberProcessors > 1));

            /* Remove the one count */
            if (!InterlockedDecrementSizeT(&WaitBlock->Count))
            {
                /* We're down to 0 now, so signal the event */
                KeSetEvent(&WaitBlock->WakeEvent, IO_NO_INCREMENT, FALSE);
            }

            /* We're all done */
            break;
        }
    }
}

ExfReleaseRundownProtectionCacheAware()

VOID FASTCALL ExfReleaseRundownProtectionCacheAware

(

IN PEX_RUNDOWN_REF_CACHE_AWARE

RunRefCacheAware

)

Definition at line 411 of file rundown.c.

{
    PEX_RUNDOWN_REF RunRef;

    RunRef = ExGetRunRefForGivenProcessor(RunRefCacheAware, KeGetCurrentProcessorNumber());
    _ExReleaseRundownProtection(RunRef);
}

ExfReleaseRundownProtectionCacheAwareEx()

VOID FASTCALL ExfReleaseRundownProtectionCacheAwareEx	(	IN PEX_RUNDOWN_REF_CACHE_AWARE	RunRefCacheAware,
		IN ULONG	Count
	)

Definition at line 424 of file rundown.c.

{
    PEX_RUNDOWN_REF RunRef;

    RunRef = ExGetRunRefForGivenProcessor(RunRefCacheAware, KeGetCurrentProcessorNumber());
    ExfReleaseRundownProtectionEx(RunRef, Count);
}

ExfReleaseRundownProtectionEx()

VOID FASTCALL ExfReleaseRundownProtectionEx	(	IN PEX_RUNDOWN_REF	RunRef,
		IN ULONG	Count
	)

Definition at line 260 of file rundown.c.

{
    ULONG_PTR Value = RunRef->Count, NewValue;
    PEX_RUNDOWN_WAIT_BLOCK WaitBlock;

    /* Loop until successfully incremented the counter */
    for (;;)
    {
        /* Check if rundown is not active */
        if (!(Value & EX_RUNDOWN_ACTIVE))
        {
            /* Sanity check */
            ASSERT((Value >= EX_RUNDOWN_COUNT_INC * Count) ||
                   (KeNumberProcessors > 1));

            /* Get the new value */
            NewValue = Value - EX_RUNDOWN_COUNT_INC * Count;

            /* Change the value */
            NewValue = ExpChangeRundown(RunRef, NewValue, Value);
            if (NewValue == Value) break;

            /* Update value */
            Value = NewValue;
        }
        else
        {
            /* Get the wait block */
            WaitBlock = (PEX_RUNDOWN_WAIT_BLOCK)(Value & ~EX_RUNDOWN_ACTIVE);
            ASSERT((WaitBlock->Count >= Count) || (KeNumberProcessors > 1));

            /* Remove the counts */
            if (InterlockedExchangeAddSizeT(&WaitBlock->Count,
                                            -(LONG)Count) == (LONG)Count)
            {
                /* We're down to 0 now, so signal the event */
                KeSetEvent(&WaitBlock->WakeEvent, IO_NO_INCREMENT, FALSE);
            }

            /* We're all done */
            break;
        }
    }
}

Referenced by ExCompareExchangeCallBack(), ExfReleaseRundownProtectionCacheAwareEx(), and ExReferenceCallBackBlock().

ExfRundownCompleted()

VOID FASTCALL ExfRundownCompleted

(

IN PEX_RUNDOWN_REF

RunRef

)

Definition at line 169 of file rundown.c.

{
    PAGED_CODE();

    /* Sanity check */
    ASSERT((RunRef->Count & EX_RUNDOWN_ACTIVE) != 0);

    /* Mark the counter as active */
    ExpSetRundown(RunRef, EX_RUNDOWN_ACTIVE);
}

ExfRundownCompletedCacheAware()

VOID FASTCALL ExfRundownCompletedCacheAware

(

IN PEX_RUNDOWN_REF_CACHE_AWARE

RunRefCacheAware

)

Definition at line 506 of file rundown.c.

{
    PEX_RUNDOWN_REF RunRef;
    ULONG ProcCount, Current;

    ProcCount = RunRefCacheAware->Number;
    /* No proc, nothing to do */
    if (ProcCount == 0)
    {
        return;
    }

    /* We will mark all our runrefs active */
    for (Current = 0; Current < ProcCount; ++Current)
    {
        /* Get the runref for the proc */
        RunRef = ExGetRunRefForGivenProcessor(RunRefCacheAware, Current);
        ASSERT((RunRef->Count & EX_RUNDOWN_ACTIVE) != 0);

        ExpSetRundown(RunRef, EX_RUNDOWN_ACTIVE);
    }
}

ExfWaitForRundownProtectionRelease()

VOID FASTCALL ExfWaitForRundownProtectionRelease

(

IN PEX_RUNDOWN_REF

RunRef

)

Definition at line 324 of file rundown.c.

{
    ULONG_PTR Value, Count, NewValue;
    EX_RUNDOWN_WAIT_BLOCK WaitBlock;
    PEX_RUNDOWN_WAIT_BLOCK WaitBlockPointer;
    PKEVENT Event;
    PAGED_CODE();

    /* Set the active bit */
    Value = ExpChangeRundown(RunRef, EX_RUNDOWN_ACTIVE, 0);
    if ((Value == 0) || (Value == EX_RUNDOWN_ACTIVE)) return;

    /* No event for now */
    Event = NULL;
    WaitBlockPointer = (PEX_RUNDOWN_WAIT_BLOCK)((ULONG_PTR)&WaitBlock |
                                                EX_RUNDOWN_ACTIVE);

    /* Start waitblock set loop */
    for (;;)
    {
        /* Save the count */
        Count = Value >> EX_RUNDOWN_COUNT_SHIFT;

        /* If the count is over one and we don't have en event yet, create it */
        if ((Count) && !(Event))
        {
            /* Initialize the event */
            KeInitializeEvent(&WaitBlock.WakeEvent,
                              SynchronizationEvent,
                              FALSE);

            /* Set the pointer */
            Event = &WaitBlock.WakeEvent;
        }

        /* Set the count */
        WaitBlock.Count = Count;

        /* Now set the pointer */
        NewValue = ExpChangeRundown(RunRef, (ULONG_PTR)WaitBlockPointer, Value);
        if (NewValue == Value) break;

        /* Loop again */
        Value = NewValue;
        ASSERT((Value & EX_RUNDOWN_ACTIVE) == 0);
    }

    /* If the count was 0, we're done */
    if (!Count) return;

    /* Wait for whoever needs to release to notify us */
    KeWaitForSingleObject(Event, Executive, KernelMode, FALSE, NULL);
    ASSERT(WaitBlock.Count == 0);
}

ExfWaitForRundownProtectionReleaseCacheAware()

VOID FASTCALL ExfWaitForRundownProtectionReleaseCacheAware

(

IN PEX_RUNDOWN_REF_CACHE_AWARE

RunRefCacheAware

)

Definition at line 438 of file rundown.c.

{
    PEX_RUNDOWN_REF RunRef;
    EX_RUNDOWN_WAIT_BLOCK WaitBlock;
    PEX_RUNDOWN_WAIT_BLOCK WaitBlockPointer;
    ULONG_PTR ProcCount, Current, Value, OldValue, TotalCount;

    ProcCount = RunRefCacheAware->Number;
    /* No proc, nothing to do */
    if (ProcCount == 0)
    {
        return;
    }

    TotalCount = 0;
    WaitBlock.Count = 0;
    WaitBlockPointer = (PEX_RUNDOWN_WAIT_BLOCK)((ULONG_PTR)&WaitBlock |
                                                EX_RUNDOWN_ACTIVE);
    /* We will check all our runrefs */
    for (Current = 0; Current < ProcCount; ++Current)
    {
        /* Get the runref for the proc */
        RunRef = ExGetRunRefForGivenProcessor(RunRefCacheAware, Current);
        /* Loop for setting the wait block */
        do
        {
            Value = RunRef->Count;
            ASSERT((Value & EX_RUNDOWN_ACTIVE) == 0);

            /* Remove old value and set our waitblock instead */
            OldValue = ExpChangeRundown(RunRef, WaitBlockPointer, Value);
            if (OldValue == Value)
            {
                break;
            }

            Value = OldValue;
        }
        while (TRUE);

        /* Count the deleted values */
        TotalCount += Value;
    }

    /* Sanity check: we didn't overflow */
    ASSERT((LONG_PTR)TotalCount >= 0);
    if (TotalCount != 0)
    {
        /* Init the waitblock event */
        KeInitializeEvent(&WaitBlock.WakeEvent,
                          SynchronizationEvent,
                          FALSE);

        /* Do we have to wait? If so, go ahead! */
        if (InterlockedExchangeAddSizeT(&WaitBlock.Count,
                                        (LONG_PTR)TotalCount >> EX_RUNDOWN_COUNT_SHIFT) ==
                                       -(LONG_PTR)(TotalCount >> EX_RUNDOWN_COUNT_SHIFT))
        {
            KeWaitForSingleObject(&WaitBlock.WakeEvent, Executive, KernelMode, FALSE, NULL);
        }
    }
}

ExInitializeRundownProtectionCacheAware()

VOID NTAPI ExInitializeRundownProtectionCacheAware	(	IN PEX_RUNDOWN_REF_CACHE_AWARE	RunRefCacheAware,
		IN SIZE_T	Size
	)

Definition at line 667 of file rundown.c.

{
    PVOID Pool;
    PEX_RUNDOWN_REF RunRef;
    ULONG Count, RunRefSize, Align;

    PAGED_CODE();

    /* Get the user allocate pool for runrefs */
    Pool = (PVOID)((ULONG_PTR)RunRefCacheAware + sizeof(EX_RUNDOWN_REF_CACHE_AWARE));

    /* By default a runref is structure-sized */
    RunRefSize = sizeof(EX_RUNDOWN_REF);

    /*
     * If we just have enough room for a single runref, deduce were on a single
     * processor machine
     */
    if (Size == sizeof(EX_RUNDOWN_REF_CACHE_AWARE) + sizeof(EX_RUNDOWN_REF))
    {
        Count = 1;
    }
    else
    {
        /* Get alignment constraint */
        Align = KeGetRecommendedSharedDataAlignment();

        /* How many runrefs given the alignment? */
        RunRefSize = Align;
        Count = ((Size - sizeof(EX_RUNDOWN_REF_CACHE_AWARE)) / Align) - 1;
        Pool = (PVOID)ALIGN_UP_BY(Pool, Align);
    }

    /* Initialize the structure */
    RunRefCacheAware->RunRefs = Pool;
    RunRefCacheAware->RunRefSize = RunRefSize;
    RunRefCacheAware->Number = Count;

    /* There is no allocated pool! */
    RunRefCacheAware->PoolToFree = (PVOID)0xBADCA11u;

    /* Initialize runref */
    if (RunRefCacheAware->Number != 0)
    {
        for (Count = 0; Count < RunRefCacheAware->Number; ++Count)
        {
            RunRef = ExGetRunRefForGivenProcessor(RunRefCacheAware, Count);
            _ExInitializeRundownProtection(RunRef);
        }
    }
}

ExSizeOfRundownProtectionCacheAware()

SIZE_T NTAPI ExSizeOfRundownProtectionCacheAware

(

VOID

)

Definition at line 725 of file rundown.c.

{
    SIZE_T Size;

    PAGED_CODE();

    /* Compute the needed size for runrefs */
    if (KeNumberProcessors <= 1)
    {
        Size = sizeof(EX_RUNDOWN_REF);
    }
    else
    {
        /* We +1, to have enough room for alignment */
        Size = (KeNumberProcessors + 1) * KeGetRecommendedSharedDataAlignment();
    }

    /* Return total size (master structure and runrefs) */
    return Size + sizeof(EX_RUNDOWN_REF_CACHE_AWARE);
}