ke_x.h File Reference

This graph shows which files directly or indirectly include this file:

Go to the source code of this file.

Macros
#define	KeEnterGuardedRegionThread(_Thread)
#define	KeEnterGuardedRegion()
#define	KeLeaveGuardedRegionThread(_Thread)
#define	KeLeaveGuardedRegion()
#define	KeEnterCriticalRegionThread(_Thread)
#define	KeEnterCriticalRegion()
#define	KeLeaveCriticalRegionThread(_Thread)
#define	KeLeaveCriticalRegion()
#define	KiSatisfyMutantWait(Object, Thread)
#define	KiSatisfyNonMutantWait(Object)
#define	KiSatisfyObjectWait(Object, Thread)
#define	KiAddThreadToWaitList(Thread, Swappable)
#define	KxDelayThreadWait()
#define	KxMultiThreadWait()
#define	KxSingleThreadWait()
#define	KxQueueThreadWait()

Functions
FORCEINLINE KPROCESSOR_MODE	KeGetPreviousMode (VOID)
FORCEINLINE VOID	KiAcquireDispatcherObject (IN DISPATCHER_HEADER *Object)
FORCEINLINE VOID	KiReleaseDispatcherObject (IN DISPATCHER_HEADER *Object)
FORCEINLINE KIRQL	KiAcquireDispatcherLock (VOID)
FORCEINLINE VOID	KiReleaseDispatcherLock (IN KIRQL OldIrql)
FORCEINLINE VOID	KiAcquireDispatcherLockAtSynchLevel (VOID)
FORCEINLINE VOID	KiReleaseDispatcherLockFromSynchLevel (VOID)
FORCEINLINE VOID	KiInsertDeferredReadyList (IN PKTHREAD Thread)
FORCEINLINE VOID	KiRescheduleThread (IN BOOLEAN NewThread, IN ULONG Cpu)
FORCEINLINE VOID	KiSetThreadSwapBusy (IN PKTHREAD Thread)
FORCEINLINE VOID	KiAcquirePrcbLock (IN PKPRCB Prcb)
FORCEINLINE VOID	KiReleasePrcbLock (IN PKPRCB Prcb)
FORCEINLINE VOID	KiAcquireThreadLock (IN PKTHREAD Thread)
FORCEINLINE VOID	KiReleaseThreadLock (IN PKTHREAD Thread)
FORCEINLINE BOOLEAN	KiTryThreadLock (IN PKTHREAD Thread)
FORCEINLINE VOID	KiCheckDeferredReadyList (IN PKPRCB Prcb)
FORCEINLINE VOID	KiRequestApcInterrupt (IN BOOLEAN NeedApc, IN UCHAR Processor)
FORCEINLINE PKSPIN_LOCK_QUEUE	KiAcquireTimerLock (IN ULONG Hand)
FORCEINLINE VOID	KiReleaseTimerLock (IN PKSPIN_LOCK_QUEUE LockQueue)
FORCEINLINE VOID	KiAcquireApcLockRaiseToSynch (IN PKTHREAD Thread, IN PKLOCK_QUEUE_HANDLE Handle)
FORCEINLINE VOID	KiAcquireApcLockAtSynchLevel (IN PKTHREAD Thread, IN PKLOCK_QUEUE_HANDLE Handle)
FORCEINLINE VOID	KiAcquireApcLockRaiseToDpc (IN PKTHREAD Thread, IN PKLOCK_QUEUE_HANDLE Handle)
FORCEINLINE VOID	KiReleaseApcLock (IN PKLOCK_QUEUE_HANDLE Handle)
FORCEINLINE VOID	KiReleaseApcLockFromSynchLevel (IN PKLOCK_QUEUE_HANDLE Handle)
FORCEINLINE VOID	KiAcquireProcessLockRaiseToSynch (IN PKPROCESS Process, IN PKLOCK_QUEUE_HANDLE Handle)
FORCEINLINE VOID	KiReleaseProcessLock (IN PKLOCK_QUEUE_HANDLE Handle)
FORCEINLINE VOID	KiReleaseProcessLockFromSynchLevel (IN PKLOCK_QUEUE_HANDLE Handle)
FORCEINLINE VOID	KiAcquireDeviceQueueLock (IN PKDEVICE_QUEUE DeviceQueue, IN PKLOCK_QUEUE_HANDLE DeviceLock)
FORCEINLINE VOID	KiReleaseDeviceQueueLock (IN PKLOCK_QUEUE_HANDLE DeviceLock)
FORCEINLINE PLARGE_INTEGER	KiRecalculateDueTime (IN PLARGE_INTEGER OriginalDueTime, IN PLARGE_INTEGER DueTime, IN OUT PLARGE_INTEGER NewDueTime)
FORCEINLINE BOOLEAN	KiCheckThreadStackSwap (IN PKTHREAD Thread, IN KPROCESSOR_MODE WaitMode)
FORCEINLINE NTSTATUS	KiCheckAlertability (IN PKTHREAD Thread, IN BOOLEAN Alertable, IN KPROCESSOR_MODE WaitMode)
FORCEINLINE ULONG	KiComputeTimerTableIndex (IN ULONGLONG DueTime)
FORCEINLINE VOID	KiRemoveEntryTimer (IN PKTIMER Timer)
FORCEINLINE VOID	KxInsertTimer (IN PKTIMER Timer, IN ULONG Hand)
FORCEINLINE BOOLEAN	KiComputeDueTime (IN PKTIMER Timer, IN LARGE_INTEGER DueTime, OUT PULONG Hand)
FORCEINLINE VOID	KxRemoveTreeTimer (IN PKTIMER Timer)
FORCEINLINE VOID	KxSetTimerForThreadWait (IN PKTIMER Timer, IN LARGE_INTEGER Interval, OUT PULONG Hand)
FORCEINLINE VOID	KxUnwaitThread (IN DISPATCHER_HEADER *Object, IN KPRIORITY Increment)
FORCEINLINE VOID	KxUnwaitThreadForEvent (IN PKEVENT Event, IN KPRIORITY Increment)
	_Requires_lock_held_ (Prcb->PrcbLock) _Releases_lock_(Prcb -> PrcbLock) FORCEINLINE VOID KxQueueReadyThread(IN PKTHREAD Thread, IN PKPRCB Prcb)
FORCEINLINE PKTHREAD	KiSelectReadyThread (IN KPRIORITY Priority, IN PKPRCB Prcb)
FORCEINLINE SCHAR	KiComputeNewPriority (IN PKTHREAD Thread, IN SCHAR Adjustment)
FORCEINLINE VOID	_KeInitializeGuardedMutex (OUT PKGUARDED_MUTEX GuardedMutex)
FORCEINLINE VOID	_KeAcquireGuardedMutexUnsafe (IN OUT PKGUARDED_MUTEX GuardedMutex)
FORCEINLINE VOID	_KeReleaseGuardedMutexUnsafe (IN OUT PKGUARDED_MUTEX GuardedMutex)
FORCEINLINE VOID	_KeAcquireGuardedMutex (IN PKGUARDED_MUTEX GuardedMutex)
FORCEINLINE VOID	_KeReleaseGuardedMutex (IN OUT PKGUARDED_MUTEX GuardedMutex)
FORCEINLINE BOOLEAN	_KeTryToAcquireGuardedMutex (IN OUT PKGUARDED_MUTEX GuardedMutex)
FORCEINLINE VOID	KiAcquireNmiListLock (OUT PKIRQL OldIrql)
FORCEINLINE VOID	KiReleaseNmiListLock (IN KIRQL OldIrql)

Macro Definition Documentation

KeEnterCriticalRegion

#define KeEnterCriticalRegion

(

)

Value:

{                                                                           \
    PKTHREAD _Thread = KeGetCurrentThread();                                \
    KeEnterCriticalRegionThread(_Thread);                                   \
}

Definition at line 88 of file ke_x.h.

KeEnterCriticalRegionThread

#define KeEnterCriticalRegionThread

(

_Thread

)

Value:

{                                                                           \
    /* Sanity checks */                                                     \
    ASSERT((_Thread) == KeGetCurrentThread());                              \
    ASSERT(((_Thread)->KernelApcDisable <= 0) &&                            \
           ((_Thread)->KernelApcDisable != -32768));                        \
                                                                            \
    /* Disable Kernel APCs */                                               \
    (_Thread)->KernelApcDisable--;                                          \
}

Definition at line 77 of file ke_x.h.

KeEnterGuardedRegion

#define KeEnterGuardedRegion

(

)

Value:

{                                                                           \
    PKTHREAD _Thread = KeGetCurrentThread();                                \
    KeEnterGuardedRegionThread(_Thread);                                    \
}

Definition at line 39 of file ke_x.h.

KeEnterGuardedRegionThread

#define KeEnterGuardedRegionThread

(

_Thread

)

Value:

{                                                                           \
    /* Sanity checks */                                                     \
    ASSERT(KeGetCurrentIrql() <= APC_LEVEL);                                \
    ASSERT((_Thread) == KeGetCurrentThread());                              \
    ASSERT(((_Thread)->SpecialApcDisable <= 0) &&                           \
           ((_Thread)->SpecialApcDisable != -32768));                       \
                                                                            \
    /* Disable Special APCs */                                              \
    (_Thread)->SpecialApcDisable--;                                         \
}

Definition at line 27 of file ke_x.h.

KeLeaveCriticalRegion

#define KeLeaveCriticalRegion

(

)

Value:

{                                                                           \
    PKTHREAD _Thread = KeGetCurrentThread();                                \
    KeLeaveCriticalRegionThread(_Thread);                                   \
}

Definition at line 119 of file ke_x.h.

KeLeaveCriticalRegionThread

#define KeLeaveCriticalRegionThread

(

_Thread

)

Value:

{                                                                           \
    /* Sanity checks */                                                     \
    ASSERT((_Thread) == KeGetCurrentThread());                              \
    ASSERT((_Thread)->KernelApcDisable < 0);                                \
                                                                            \
    /* Enable Kernel APCs */                                                \
    (_Thread)->KernelApcDisable++;                                          \
                                                                            \
    /* Check if Kernel APCs are now enabled */                              \
    if (!((_Thread)->KernelApcDisable))                                     \
    {                                                                       \
        /* Check if we need to request an APC Delivery */                   \
        if (!(IsListEmpty(&(_Thread)->ApcState.ApcListHead[KernelMode])) && \
            !((_Thread)->SpecialApcDisable))                                \
        {                                                                   \
            /* Check for the right environment */                           \
            KiCheckForKernelApcDelivery();                                  \
        }                                                                   \
    }                                                                       \
}

Definition at line 97 of file ke_x.h.

KeLeaveGuardedRegion

#define KeLeaveGuardedRegion

(

)

Value:

{                                                                           \
    PKTHREAD _Thread = KeGetCurrentThread();                                \
    KeLeaveGuardedRegionThread(_Thread);                                    \
}

Definition at line 68 of file ke_x.h.

KeLeaveGuardedRegionThread

#define KeLeaveGuardedRegionThread

(

_Thread

)

Value:

{                                                                           \
    /* Sanity checks */                                                     \
    ASSERT(KeGetCurrentIrql() <= APC_LEVEL);                                \
    ASSERT((_Thread) == KeGetCurrentThread());                              \
    ASSERT((_Thread)->SpecialApcDisable < 0);                               \
                                                                            \
    /* Leave region and check if APCs are OK now */                         \
    if (!(++(_Thread)->SpecialApcDisable))                                  \
    {                                                                       \
        /* Check for Kernel APCs on the list */                             \
        if (!IsListEmpty(&(_Thread)->ApcState.                              \
                         ApcListHead[KernelMode]))                          \
        {                                                                   \
            /* Check for APC Delivery */                                    \
            KiCheckForKernelApcDelivery();                                  \
        }                                                                   \
    }                                                                       \
}

Definition at line 48 of file ke_x.h.

KiAddThreadToWaitList

#define KiAddThreadToWaitList	(		Thread,
			Swappable
	)

Value:

{                                                                           \
    /* Make sure it's swappable */                                          \
    if (Swappable)                                                          \
    {                                                                       \
        /* Insert it into the PRCB's List */                                \
        InsertTailList(&KeGetCurrentPrcb()->WaitListHead,                   \
                       &Thread->WaitListEntry);                             \
    }                                                                       \
}

Definition at line 824 of file ke_x.h.

KiSatisfyMutantWait

#define KiSatisfyMutantWait	(		Object,
			Thread
	)

Value:

{                                                                           \
    /* Decrease the Signal State */                                         \
    (Object)->Header.SignalState--;                                         \
                                                                            \
    /* Check if it's now non-signaled */                                    \
    if (!(Object)->Header.SignalState)                                      \
    {                                                                       \
        /* Set the Owner Thread */                                          \
        (Object)->OwnerThread = Thread;                                     \
                                                                            \
        /* Disable APCs if needed */                                        \
        Thread->KernelApcDisable = Thread->KernelApcDisable -               \
                                   (Object)->ApcDisable;                    \
                                                                            \
        /* Check if it's abandoned */                                       \
        if ((Object)->Abandoned)                                            \
        {                                                                   \
            /* Unabandon it */                                              \
            (Object)->Abandoned = FALSE;                                    \
                                                                            \
            /* Return Status */                                             \
            Thread->WaitStatus = STATUS_ABANDONED;                          \
        }                                                                   \
                                                                            \
        /* Insert it into the Mutant List */                                \
        InsertHeadList(Thread->MutantListHead.Blink,                        \
                       &(Object)->MutantListEntry);                         \
    }                                                                       \
}

Definition at line 715 of file ke_x.h.

KiSatisfyNonMutantWait

#define KiSatisfyNonMutantWait

(

Object

)

Value:

{                                                                           \
    if (((Object)->Header.Type & TIMER_OR_EVENT_TYPE) ==                    \
             EventSynchronizationObject)                                    \
    {                                                                       \
        /* Synchronization Timers and Events just get un-signaled */        \
        (Object)->Header.SignalState = 0;                                   \
    }                                                                       \
    else if ((Object)->Header.Type == SemaphoreObject)                      \
    {                                                                       \
        /* These ones can have multiple states, so we only decrease it */   \
        (Object)->Header.SignalState--;                                     \
    }                                                                       \
}

Definition at line 749 of file ke_x.h.

KiSatisfyObjectWait

#define KiSatisfyObjectWait	(		Object,
			Thread
	)

Value:

{                                                                           \
    /* Special case for Mutants */                                          \
    if ((Object)->Header.Type == MutantObject)                              \
    {                                                                       \
        KiSatisfyMutantWait((Object), (Thread));                            \
    }                                                                       \
    else                                                                    \
    {                                                                       \
        KiSatisfyNonMutantWait(Object);                                     \
    }                                                                       \
}

Definition at line 767 of file ke_x.h.

KxDelayThreadWait

#define KxDelayThreadWait

(

)

Value:

\
    /* Setup the Wait Block */                                              \
    Thread->WaitBlockList = TimerBlock;                                     \
                                                                            \
    /* Setup the timer */                                                   \
    KxSetTimerForThreadWait(Timer, *Interval, &Hand);                       \
                                                                            \
    /* Save the due time for the caller */                                  \
    DueTime.QuadPart = Timer->DueTime.QuadPart;                             \
                                                                            \
    /* Link the timer to this Wait Block */                                 \
    TimerBlock->NextWaitBlock = TimerBlock;                                 \
    Timer->Header.WaitListHead.Flink = &TimerBlock->WaitListEntry;          \
    Timer->Header.WaitListHead.Blink = &TimerBlock->WaitListEntry;          \
                                                                            \
    /* Clear wait status */                                                 \
    Thread->WaitStatus = STATUS_SUCCESS;                                    \
                                                                            \
    /* Setup wait fields */                                                 \
    Thread->Alertable = Alertable;                                          \
    Thread->WaitReason = DelayExecution;                                    \
    Thread->WaitMode = WaitMode;                                            \
                                                                            \
    /* Check if we can swap the thread's stack */                           \
    Thread->WaitListEntry.Flink = NULL;                                     \
    Swappable = KiCheckThreadStackSwap(Thread, WaitMode);                   \
                                                                            \
    /* Set the wait time */                                                 \
    Thread->WaitTime = KeTickCount.LowPart;

Definition at line 1076 of file ke_x.h.

KxMultiThreadWait

#define KxMultiThreadWait

(

)

Definition at line 1107 of file ke_x.h.

KxQueueThreadWait

#define KxQueueThreadWait

(

)

Definition at line 1208 of file ke_x.h.

KxSingleThreadWait

#define KxSingleThreadWait

(

)

Definition at line 1163 of file ke_x.h.

Function Documentation

_KeAcquireGuardedMutex()

FORCEINLINE VOID _KeAcquireGuardedMutex

(

IN PKGUARDED_MUTEX

GuardedMutex

)

Definition at line 1584 of file ke_x.h.

{
    PKTHREAD Thread = KeGetCurrentThread();

    /* Sanity checks */
    ASSERT(KeGetCurrentIrql() <= APC_LEVEL);
    ASSERT(GuardedMutex->Owner != Thread);

    /* Disable Special APCs */
    KeEnterGuardedRegionThread(Thread);

    /* Remove the lock */
    if (!InterlockedBitTestAndReset(&GuardedMutex->Count, GM_LOCK_BIT_V))
    {
        /* The Guarded Mutex was already locked, enter contented case */
        KiAcquireGuardedMutex(GuardedMutex);
    }

    /* Set the Owner and Special APC Disable state */
    GuardedMutex->Owner = Thread;
    GuardedMutex->SpecialApcDisable = Thread->SpecialApcDisable;
}

Referenced by KeAcquireGuardedMutex().

_KeAcquireGuardedMutexUnsafe()

FORCEINLINE VOID _KeAcquireGuardedMutexUnsafe

(

IN OUT PKGUARDED_MUTEX

GuardedMutex

)

Definition at line 1519 of file ke_x.h.

{
    PKTHREAD Thread = KeGetCurrentThread();

    /* Sanity checks */
    ASSERT((KeGetCurrentIrql() == APC_LEVEL) ||
           (Thread->SpecialApcDisable < 0) ||
           (Thread->Teb == NULL) ||
           (Thread->Teb >= (PTEB)MM_SYSTEM_RANGE_START));
    ASSERT(GuardedMutex->Owner != Thread);

    /* Remove the lock */
    if (!InterlockedBitTestAndReset(&GuardedMutex->Count, GM_LOCK_BIT_V))
    {
        /* The Guarded Mutex was already locked, enter contented case */
        KiAcquireGuardedMutex(GuardedMutex);
    }

    /* Set the Owner */
    GuardedMutex->Owner = Thread;
}

Referenced by KeAcquireGuardedMutexUnsafe().

_KeInitializeGuardedMutex()

FORCEINLINE VOID _KeInitializeGuardedMutex

(

OUT PKGUARDED_MUTEX

GuardedMutex

)

Definition at line 1506 of file ke_x.h.

{
    /* Setup the Initial Data */
    GuardedMutex->Count = GM_LOCK_BIT;
    GuardedMutex->Owner = NULL;
    GuardedMutex->Contention = 0;

    /* Initialize the Wait Gate */
    KeInitializeGate(&GuardedMutex->Gate);
}

Referenced by KeInitializeGuardedMutex().

_KeReleaseGuardedMutex()

FORCEINLINE VOID _KeReleaseGuardedMutex

(

IN OUT PKGUARDED_MUTEX

GuardedMutex

)

Definition at line 1609 of file ke_x.h.

{
    PKTHREAD Thread = KeGetCurrentThread();
    LONG OldValue, NewValue;

    /* Sanity checks */
    ASSERT(KeGetCurrentIrql() <= APC_LEVEL);
    ASSERT(GuardedMutex->Owner == Thread);
    ASSERT(Thread->SpecialApcDisable == GuardedMutex->SpecialApcDisable);

    /* Destroy the Owner */
    GuardedMutex->Owner = NULL;

    /* Add the Lock Bit */
    OldValue = InterlockedExchangeAdd(&GuardedMutex->Count, GM_LOCK_BIT);
    ASSERT((OldValue & GM_LOCK_BIT) == 0);

    /* Check if it was already locked, but not woken */
    if ((OldValue) && !(OldValue & GM_LOCK_WAITER_WOKEN))
    {
        /* Update the Oldvalue to what it should be now */
        OldValue += GM_LOCK_BIT;

        /* The mutex will be woken, minus one waiter */
        NewValue = OldValue + GM_LOCK_WAITER_WOKEN -
            GM_LOCK_WAITER_INC;

        /* Remove the Woken bit */
        if (InterlockedCompareExchange(&GuardedMutex->Count,
                                       NewValue,
                                       OldValue) == OldValue)
        {
            /* Signal the Gate */
            KeSignalGateBoostPriority(&GuardedMutex->Gate);
        }
    }

    /* Re-enable APCs */
    KeLeaveGuardedRegionThread(Thread);
}

Referenced by KeReleaseGuardedMutex().

_KeReleaseGuardedMutexUnsafe()

FORCEINLINE VOID _KeReleaseGuardedMutexUnsafe

(

IN OUT PKGUARDED_MUTEX

GuardedMutex

)

Definition at line 1543 of file ke_x.h.

{
    LONG OldValue, NewValue;

    /* Sanity checks */
    ASSERT((KeGetCurrentIrql() == APC_LEVEL) ||
           (KeGetCurrentThread()->SpecialApcDisable < 0) ||
           (KeGetCurrentThread()->Teb == NULL) ||
           (KeGetCurrentThread()->Teb >= (PTEB)MM_SYSTEM_RANGE_START));
    ASSERT(GuardedMutex->Owner == KeGetCurrentThread());

    /* Destroy the Owner */
    GuardedMutex->Owner = NULL;

    /* Add the Lock Bit */
    OldValue = InterlockedExchangeAdd(&GuardedMutex->Count, GM_LOCK_BIT);
    ASSERT((OldValue & GM_LOCK_BIT) == 0);

    /* Check if it was already locked, but not woken */
    if ((OldValue) && !(OldValue & GM_LOCK_WAITER_WOKEN))
    {
        /* Update the Oldvalue to what it should be now */
        OldValue += GM_LOCK_BIT;

        /* The mutex will be woken, minus one waiter */
        NewValue = OldValue + GM_LOCK_WAITER_WOKEN -
            GM_LOCK_WAITER_INC;

        /* Remove the Woken bit */
        if (InterlockedCompareExchange(&GuardedMutex->Count,
                                       NewValue,
                                       OldValue) == OldValue)
        {
            /* Signal the Gate */
            KeSignalGateBoostPriority(&GuardedMutex->Gate);
        }
    }
}

Referenced by KeReleaseGuardedMutexUnsafe().

_KeTryToAcquireGuardedMutex()

FORCEINLINE BOOLEAN _KeTryToAcquireGuardedMutex

(

IN OUT PKGUARDED_MUTEX

GuardedMutex

)

Definition at line 1652 of file ke_x.h.

{
    PKTHREAD Thread = KeGetCurrentThread();

    /* Block APCs */
    KeEnterGuardedRegionThread(Thread);

    /* Remove the lock */
    if (!InterlockedBitTestAndReset(&GuardedMutex->Count, GM_LOCK_BIT_V))
    {
        /* Re-enable APCs */
        KeLeaveGuardedRegionThread(Thread);
        YieldProcessor();

        /* Return failure */
        return FALSE;
    }

    /* Set the Owner and APC State */
    GuardedMutex->Owner = Thread;
    GuardedMutex->SpecialApcDisable = Thread->SpecialApcDisable;
    return TRUE;
}

Referenced by KeTryToAcquireGuardedMutex().

_Requires_lock_held_()

_Requires_lock_held_

(

Prcb->

PrcbLock

)

-> PrcbLock) FORCEINLINE VOID KxQueueReadyThread(IN PKTHREAD Thread, IN PKPRCB Prcb)

Definition at line 1350 of file ke_x.h.

{
    BOOLEAN Preempted;
    KPRIORITY Priority;

    /* Sanity checks */
    ASSERT(Prcb == KeGetCurrentPrcb());
    ASSERT(Thread->State == Running);
    ASSERT(Thread->NextProcessor == Prcb->Number);

    /* Check if this thread is allowed to run in this CPU */
#ifdef CONFIG_SMP
    if ((Thread->Affinity) & (Prcb->SetMember))
#else
    if (TRUE)
#endif
    {
        /* Set thread ready for execution */
        Thread->State = Ready;

        /* Save current priority and if someone had pre-empted it */
        Priority = Thread->Priority;
        Preempted = Thread->Preempted;

        /* We're not pre-empting now, and set the wait time */
        Thread->Preempted = FALSE;
        Thread->WaitTime = KeTickCount.LowPart;

        /* Sanity check */
        ASSERT((Priority >= 0) && (Priority <= HIGH_PRIORITY));

        /* Insert this thread in the appropriate order */
        Preempted ? InsertHeadList(&Prcb->DispatcherReadyListHead[Priority],
                                   &Thread->WaitListEntry) :
                    InsertTailList(&Prcb->DispatcherReadyListHead[Priority],
                                   &Thread->WaitListEntry);

        /* Update the ready summary */
        Prcb->ReadySummary |= PRIORITY_MASK(Priority);

        /* Sanity check */
        ASSERT(Priority == Thread->Priority);

        /* Release the PRCB lock */
        KiReleasePrcbLock(Prcb);
    }
    else
    {
        /* Otherwise, prepare this thread to be deferred */
        Thread->State = DeferredReady;
        Thread->DeferredProcessor = Prcb->Number;

        /* Release the lock and defer scheduling */
        KiReleasePrcbLock(Prcb);
        KiDeferredReadyThread(Thread);
    }
}

KeGetPreviousMode()

FORCEINLINE KPROCESSOR_MODE KeGetPreviousMode

(

VOID

)

Definition at line 17 of file ke_x.h.

{
    /* Return the current mode */
    return KeGetCurrentThread()->PreviousMode;
}

KiAcquireApcLockAtSynchLevel()

FORCEINLINE VOID KiAcquireApcLockAtSynchLevel	(	IN PKTHREAD	Thread,
		IN PKLOCK_QUEUE_HANDLE	Handle
	)

Definition at line 616 of file ke_x.h.

{
    /* Acquire the lock */
    ASSERT(KeGetCurrentIrql() >= SYNCH_LEVEL);
    KeAcquireInStackQueuedSpinLockAtDpcLevel(&Thread->ApcQueueLock, Handle);
}

Referenced by KeFreezeAllThreads(), and KeThawAllThreads().

KiAcquireApcLockRaiseToDpc()

FORCEINLINE VOID KiAcquireApcLockRaiseToDpc	(	IN PKTHREAD	Thread,
		IN PKLOCK_QUEUE_HANDLE	Handle
	)

Definition at line 626 of file ke_x.h.

{
    /* Acquire the lock */
    KeAcquireInStackQueuedSpinLock(&Thread->ApcQueueLock, Handle);
}

Referenced by KiDeliverApc().

KiAcquireApcLockRaiseToSynch()

FORCEINLINE VOID KiAcquireApcLockRaiseToSynch	(	IN PKTHREAD	Thread,
		IN PKLOCK_QUEUE_HANDLE	Handle
	)

Definition at line 607 of file ke_x.h.

{
    /* Acquire the lock and raise to synchronization level */
    KeAcquireInStackQueuedSpinLockRaiseToSynch(&Thread->ApcQueueLock, Handle);
}

Referenced by KeAlertResumeThread(), KeAlertThread(), KeAttachProcess(), KeDetachProcess(), KeFlushQueueApc(), KeForceResumeThread(), KeInsertQueueApc(), KeRemoveQueueApc(), KeResumeThread(), KeStackAttachProcess(), KeSuspendThread(), KeTestAlertThread(), KeUnstackDetachProcess(), and KeWaitForGate().

KiAcquireDeviceQueueLock()

FORCEINLINE VOID KiAcquireDeviceQueueLock	(	IN PKDEVICE_QUEUE	DeviceQueue,
		IN PKLOCK_QUEUE_HANDLE	DeviceLock
	)

Definition at line 676 of file ke_x.h.

{
    /* Check if we were called from a threaded DPC */
    if (KeGetCurrentPrcb()->DpcThreadActive)
    {
        /* Lock the Queue, we're not at DPC level */
        KeAcquireInStackQueuedSpinLock(&DeviceQueue->Lock, DeviceLock);
    }
    else
    {
        /* We must be at DPC level, acquire the lock safely */
        ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
        KeAcquireInStackQueuedSpinLockAtDpcLevel(&DeviceQueue->Lock,
                                                 DeviceLock);
    }
}

Referenced by KeInsertByKeyDeviceQueue(), KeInsertDeviceQueue(), KeRemoveByKeyDeviceQueue(), KeRemoveByKeyDeviceQueueIfBusy(), KeRemoveDeviceQueue(), and KeRemoveEntryDeviceQueue().

KiAcquireDispatcherLock()

FORCEINLINE KIRQL KiAcquireDispatcherLock

(

VOID

)

Definition at line 149 of file ke_x.h.

{
    /* Raise to synch level */
    return KfRaiseIrql(SYNCH_LEVEL);
}

Referenced by ExpExpandResourceOwnerTable(), ExpWaitForResource(), KeBoostPriorityThread(), KeCancelTimer(), KeConnectInterrupt(), KeDisconnectInterrupt(), KeInitializeMutant(), KeInsertHeadQueue(), KeInsertQueue(), KePulseEvent(), KeReadyThread(), KeReleaseMutant(), KeReleaseSemaphore(), KeResetEvent(), KeRevertToUserAffinityThread(), KeRundownQueue(), KeRundownThread(), KeSetAffinityThread(), KeSetBasePriorityThread(), KeSetEvent(), KeSetEventBoostPriority(), KeSetIdealProcessorThread(), KeSetPriorityThread(), KeSetProcess(), KeSetSystemAffinityThread(), KeSetSystemTime(), KeSetTimerEx(), KiInsertQueueApc(), KiScanReadyQueues(), and KiTimerExpiration().

KiAcquireDispatcherLockAtSynchLevel()

FORCEINLINE VOID KiAcquireDispatcherLockAtSynchLevel

(

VOID

)

Definition at line 165 of file ke_x.h.

{
    /* This is a no-op at SYNCH_LEVEL for UP systems */
    ASSERT(KeGetCurrentIrql() >= SYNCH_LEVEL);
    return;
}

Referenced by KeAlertResumeThread(), KeAlertThread(), KeAttachProcess(), KeDelayExecutionThread(), KeDetachProcess(), KeForceResumeThread(), KeFreezeAllThreads(), KeRemoveQueue(), KeRemoveQueueApc(), KeResumeThread(), KeSetAffinityProcess(), KeSetPriorityAndQuantumProcess(), KeSignalGateBoostPriority(), KeStackAttachProcess(), KeStartThread(), KeSuspendThread(), KeTerminateThread(), KeThawAllThreads(), KeUnstackDetachProcess(), KeWaitForGate(), KeWaitForMultipleObjects(), KeWaitForSingleObject(), and KiCompleteTimer().

KiAcquireDispatcherObject()

FORCEINLINE VOID KiAcquireDispatcherObject

(

IN DISPATCHER_HEADER *

Object

)

Definition at line 132 of file ke_x.h.

{
    UNREFERENCED_PARAMETER(Object);
}

Referenced by KeSignalGateBoostPriority(), KeWaitForGate(), and KiInsertQueueApc().

KiAcquireNmiListLock()

FORCEINLINE VOID KiAcquireNmiListLock

(

OUT PKIRQL

OldIrql

)

Definition at line 1679 of file ke_x.h.

{
    KeAcquireSpinLock(&KiNmiCallbackListLock, OldIrql);
}

Referenced by KeDeregisterNmiCallback(), and KeRegisterNmiCallback().

KiAcquirePrcbLock()

FORCEINLINE VOID KiAcquirePrcbLock

(

IN PKPRCB

Prcb

)

Definition at line 220 of file ke_x.h.

{
    UNREFERENCED_PARAMETER(Prcb);
}

Referenced by _Requires_lock_not_held_(), KeRevertToUserAffinityThread(), KeSetSystemAffinityThread(), KiAdjustQuantumThread(), KiDeferredReadyThread(), KiExitDispatcher(), KiInitializeKernel(), KiScanReadyQueues(), KiSetPriorityThread(), KiSwapThread(), KiSystemStartupBootStack(), and NtYieldExecution().

KiAcquireProcessLockRaiseToSynch()

FORCEINLINE VOID KiAcquireProcessLockRaiseToSynch	(	IN PKPROCESS	Process,
		IN PKLOCK_QUEUE_HANDLE	Handle
	)

Definition at line 651 of file ke_x.h.

{
    /* Acquire the lock and raise to synchronization level */
    KeAcquireInStackQueuedSpinLockRaiseToSynch(&Process->ProcessLock, Handle);
}

Referenced by KeFreezeAllThreads(), KeQueryRuntimeProcess(), KeQueryValuesProcess(), KeSetAffinityProcess(), KeSetPriorityAndQuantumProcess(), KeSetQuantumProcess(), KeStartThread(), KeTerminateThread(), KeThawAllThreads(), and MmSetExecuteOptions().

KiAcquireThreadLock()

FORCEINLINE VOID KiAcquireThreadLock

(

IN PKTHREAD

Thread

)

Definition at line 240 of file ke_x.h.

{
    UNREFERENCED_PARAMETER(Thread);
}

Referenced by _Requires_lock_not_held_(), ExpBoostOwnerThread(), KeBoostPriorityThread(), KeQueryBasePriorityThread(), KeSetBasePriorityThread(), KeSetPriorityAndQuantumProcess(), KeSetPriorityThread(), KeWaitForGate(), KiAdjustQuantumThread(), KiDeferredReadyThread(), KiInsertQueueApc(), and NtYieldExecution().

KiAcquireTimerLock()

FORCEINLINE PKSPIN_LOCK_QUEUE KiAcquireTimerLock

(

IN ULONG

Hand

)

Definition at line 286 of file ke_x.h.

{
    ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL);

    /* Nothing to do on UP */
    UNREFERENCED_PARAMETER(Hand);
    return NULL;
}

Referenced by KeSetSystemTime(), KiInsertTreeTimer(), KiTimerExpiration(), KxInsertTimer(), and KxRemoveTreeTimer().

KiCheckAlertability()

FORCEINLINE NTSTATUS KiCheckAlertability	(	IN PKTHREAD	Thread,
		IN BOOLEAN	Alertable,
		IN KPROCESSOR_MODE	WaitMode
	)

Definition at line 841 of file ke_x.h.

{
    /* Check if the wait is alertable */
    if (Alertable)
    {
        /* It is, first check if the thread is alerted in this mode */
        if (Thread->Alerted[WaitMode])
        {
            /* It is, so bail out of the wait */
            Thread->Alerted[WaitMode] = FALSE;
            return STATUS_ALERTED;
        }
        else if ((WaitMode != KernelMode) &&
                (!IsListEmpty(&Thread->ApcState.ApcListHead[UserMode])))
        {
            /* It's isn't, but this is a user wait with queued user APCs */
            Thread->ApcState.UserApcPending = TRUE;
            return STATUS_USER_APC;
        }
        else if (Thread->Alerted[KernelMode])
        {
            /* It isn't that either, but we're alered in kernel mode */
            Thread->Alerted[KernelMode] = FALSE;
            return STATUS_ALERTED;
        }
    }
    else if ((WaitMode != KernelMode) && (Thread->ApcState.UserApcPending))
    {
        /* Not alertable, but this is a user wait with pending user APCs */
        return STATUS_USER_APC;
    }

    /* Otherwise, we're fine */
    return STATUS_WAIT_0;
}

Referenced by KeDelayExecutionThread(), KeWaitForMultipleObjects(), and KeWaitForSingleObject().

KiCheckDeferredReadyList()

FORCEINLINE VOID KiCheckDeferredReadyList

(

IN PKPRCB

Prcb

)

Definition at line 268 of file ke_x.h.

{
    /* There are no deferred ready lists on UP systems */
    UNREFERENCED_PARAMETER(Prcb);
}

Referenced by KiExitDispatcher().

KiCheckThreadStackSwap()

FORCEINLINE BOOLEAN KiCheckThreadStackSwap	(	IN PKTHREAD	Thread,
		IN KPROCESSOR_MODE	WaitMode
	)

Definition at line 803 of file ke_x.h.

{
    /* Check the required conditions */
    if ((WaitMode != KernelMode) &&
        (Thread->EnableStackSwap) &&
        (Thread->Priority >= (LOW_REALTIME_PRIORITY + 9)))
    {
        /* We are go for swap */
        return TRUE;
    }
    else
    {
        /* Don't swap the thread */
        return FALSE;
    }
}

KiComputeDueTime()

FORCEINLINE BOOLEAN KiComputeDueTime	(	IN PKTIMER	Timer,
		IN LARGE_INTEGER	DueTime,
		OUT PULONG	Hand
	)

Definition at line 952 of file ke_x.h.

{
    LARGE_INTEGER InterruptTime, SystemTime, DifferenceTime;

    /* Convert to relative time if needed */
    Timer->Header.Absolute = FALSE;
    if (DueTime.HighPart >= 0)
    {
        /* Get System Time */
        KeQuerySystemTime(&SystemTime);

        /* Do the conversion */
        DifferenceTime.QuadPart = SystemTime.QuadPart - DueTime.QuadPart;

        /* Make sure it hasn't already expired */
        Timer->Header.Absolute = TRUE;
        if (DifferenceTime.HighPart >= 0)
        {
            /* Cancel everything */
            Timer->Header.SignalState = TRUE;
            Timer->Header.Hand = 0;
            Timer->DueTime.QuadPart = 0;
            *Hand = 0;
            return FALSE;
        }

        /* Set the time as Absolute */
        DueTime = DifferenceTime;
    }

    /* Get the Interrupt Time */
    InterruptTime.QuadPart = KeQueryInterruptTime();

    /* Recalculate due time */
    Timer->DueTime.QuadPart = InterruptTime.QuadPart - DueTime.QuadPart;

    /* Get the handle */
    *Hand = KiComputeTimerTableIndex(Timer->DueTime.QuadPart);
    Timer->Header.Hand = (UCHAR)*Hand;
    Timer->Header.Inserted = TRUE;
    return TRUE;
}

Referenced by KeSetTimerEx(), and KiInsertTreeTimer().

KiComputeNewPriority()

FORCEINLINE SCHAR KiComputeNewPriority	(	IN PKTHREAD	Thread,
		IN SCHAR	Adjustment
	)

Definition at line 1469 of file ke_x.h.

{
    SCHAR Priority;

    /* Priority sanity checks */
    ASSERT((Thread->PriorityDecrement >= 0) &&
           (Thread->PriorityDecrement <= Thread->Priority));
    ASSERT((Thread->Priority < LOW_REALTIME_PRIORITY) ?
            TRUE : (Thread->PriorityDecrement == 0));

    /* Get the current priority */
    Priority = Thread->Priority;
    if (Priority < LOW_REALTIME_PRIORITY)
    {
        /* Decrease priority by the priority decrement */
        Priority -= (Thread->PriorityDecrement + Adjustment);

        /* Don't go out of bounds */
        if (Priority < Thread->BasePriority) Priority = Thread->BasePriority;

        /* Reset the priority decrement */
        Thread->PriorityDecrement = 0;
    }

    /* Sanity check */
    ASSERT((Thread->BasePriority == 0) || (Priority != 0));

    /* Return the new priority */
    return Priority;
}

Referenced by _Requires_lock_not_held_(), KeSetBasePriorityThread(), KeSetEventBoostPriority(), KiAdjustQuantumThread(), KiDeferredReadyThread(), and NtYieldExecution().

KiComputeTimerTableIndex()

FORCEINLINE ULONG KiComputeTimerTableIndex

(

IN ULONGLONG

DueTime

)

Definition at line 881 of file ke_x.h.

{
    return (DueTime / KeMaximumIncrement) & (TIMER_TABLE_SIZE - 1);
}

Referenced by KeSetSystemTime(), KiComputeDueTime(), KiInsertTimerTable(), and KxSetTimerForThreadWait().

KiInsertDeferredReadyList()

FORCEINLINE VOID KiInsertDeferredReadyList

(

IN PKTHREAD

Thread

)

Definition at line 185 of file ke_x.h.

{
    /* Set the thread to deferred state and boot CPU */
    Thread->State = DeferredReady;
    Thread->DeferredProcessor = 0;

    /* Make the thread ready immediately */
    KiDeferredReadyThread(Thread);
}

Referenced by KiInsertQueueApc(), KiReadyThread(), KiScanReadyQueues(), and KiSetPriorityThread().

KiRecalculateDueTime()

FORCEINLINE PLARGE_INTEGER KiRecalculateDueTime	(	IN PLARGE_INTEGER	OriginalDueTime,
		IN PLARGE_INTEGER	DueTime,
		IN OUT PLARGE_INTEGER	NewDueTime
	)

Definition at line 785 of file ke_x.h.

{
    /* Don't do anything for absolute waits */
    if (OriginalDueTime->QuadPart >= 0) return OriginalDueTime;

    /* Otherwise, query the interrupt time and recalculate */
    NewDueTime->QuadPart = KeQueryInterruptTime();
    NewDueTime->QuadPart -= DueTime->QuadPart;
    return NewDueTime;
}

Referenced by KeDelayExecutionThread(), KeRemoveQueue(), KeWaitForMultipleObjects(), and KeWaitForSingleObject().

KiReleaseApcLock()

FORCEINLINE VOID KiReleaseApcLock

(

IN PKLOCK_QUEUE_HANDLE

Handle

)

Definition at line 635 of file ke_x.h.

{
    /* Release the lock */
    KeReleaseInStackQueuedSpinLock(Handle);
}

Referenced by KeFlushQueueApc(), KeRemoveQueueApc(), KeSuspendThread(), KeTestAlertThread(), KeUnstackDetachProcess(), KeWaitForGate(), and KiDeliverApc().

KiReleaseApcLockFromSynchLevel()

FORCEINLINE VOID KiReleaseApcLockFromSynchLevel

(

IN PKLOCK_QUEUE_HANDLE

Handle

)

Definition at line 643 of file ke_x.h.

{
    /* Release the lock */
    KeReleaseInStackQueuedSpinLockFromDpcLevel(Handle);
}

Referenced by KeAlertResumeThread(), KeAlertThread(), KeDetachProcess(), KeForceResumeThread(), KeFreezeAllThreads(), KeInsertQueueApc(), KeResumeThread(), KeSuspendThread(), KeThawAllThreads(), KeUnstackDetachProcess(), KeWaitForGate(), and KiAttachProcess().

KiReleaseDeviceQueueLock()

FORCEINLINE VOID KiReleaseDeviceQueueLock

(

IN PKLOCK_QUEUE_HANDLE

DeviceLock

)

Definition at line 696 of file ke_x.h.

{
    /* Check if we were called from a threaded DPC */
    if (KeGetCurrentPrcb()->DpcThreadActive)
    {
        /* Unlock the Queue, we're not at DPC level */
        KeReleaseInStackQueuedSpinLock(DeviceLock);
    }
    else
    {
        /* We must be at DPC level, release the lock safely */
        ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
        KeReleaseInStackQueuedSpinLockFromDpcLevel(DeviceLock);
    }
}

Referenced by KeInsertByKeyDeviceQueue(), KeInsertDeviceQueue(), KeRemoveByKeyDeviceQueue(), KeRemoveByKeyDeviceQueueIfBusy(), KeRemoveDeviceQueue(), and KeRemoveEntryDeviceQueue().

KiReleaseDispatcherLock()

FORCEINLINE VOID KiReleaseDispatcherLock

(

IN KIRQL

OldIrql

)

Definition at line 157 of file ke_x.h.

{
    /* Just exit the dispatcher */
    KiExitDispatcher(OldIrql);
}

Referenced by ExpExpandResourceOwnerTable(), KeBoostPriorityThread(), KeCancelTimer(), KeConnectInterrupt(), KeDelayExecutionThread(), KeDisconnectInterrupt(), KeInitializeMutant(), KeInsertHeadQueue(), KeInsertQueue(), KePulseEvent(), KeReadyThread(), KeReleaseMutant(), KeReleaseSemaphore(), KeResetEvent(), KeRevertToUserAffinityThread(), KeRundownThread(), KeSetAffinityThread(), KeSetBasePriorityThread(), KeSetEvent(), KeSetEventBoostPriority(), KeSetIdealProcessorThread(), KeSetPriorityThread(), KeSetProcess(), KeSetSystemAffinityThread(), KeWaitForMultipleObjects(), KeWaitForSingleObject(), KiScanReadyQueues(), KiTimerExpiration(), and KiTimerListExpire().

KiReleaseDispatcherLockFromSynchLevel()

FORCEINLINE VOID KiReleaseDispatcherLockFromSynchLevel

(

VOID

)

Definition at line 174 of file ke_x.h.

{
    /* This is a no-op at SYNCH_LEVEL for UP systems */
    return;
}

Referenced by KeAlertResumeThread(), KeAlertThread(), KeDelayExecutionThread(), KeDetachProcess(), KeForceResumeThread(), KeFreezeAllThreads(), KeRemoveQueue(), KeRemoveQueueApc(), KeResumeThread(), KeRundownQueue(), KeSetAffinityProcess(), KeSetPriorityAndQuantumProcess(), KeSetTimerEx(), KeSignalGateBoostPriority(), KeStartThread(), KeSuspendThread(), KeTerminateThread(), KeThawAllThreads(), KeUnstackDetachProcess(), KeWaitForGate(), KeWaitForMultipleObjects(), KeWaitForSingleObject(), KiAttachProcess(), KiCompleteTimer(), KiInsertQueueApc(), and KxInsertTimer().

KiReleaseDispatcherObject()

FORCEINLINE VOID KiReleaseDispatcherObject

(

IN DISPATCHER_HEADER *

Object

)

Definition at line 142 of file ke_x.h.

{
    UNREFERENCED_PARAMETER(Object);
}

Referenced by KeSignalGateBoostPriority(), KeWaitForGate(), and KiInsertQueueApc().

KiReleaseNmiListLock()

FORCEINLINE VOID KiReleaseNmiListLock

(

IN KIRQL

OldIrql

)

Definition at line 1686 of file ke_x.h.

{
    KeReleaseSpinLock(&KiNmiCallbackListLock, OldIrql);
}

Referenced by KeDeregisterNmiCallback(), and KeRegisterNmiCallback().

KiReleasePrcbLock()

FORCEINLINE VOID KiReleasePrcbLock

(

IN PKPRCB

Prcb

)

Definition at line 230 of file ke_x.h.

{
    UNREFERENCED_PARAMETER(Prcb);
}

Referenced by _Requires_lock_held_(), _Requires_lock_not_held_(), KeRevertToUserAffinityThread(), KeSetSystemAffinityThread(), KiAdjustQuantumThread(), KiDeferredReadyThread(), KiInitializeKernel(), KiScanReadyQueues(), KiSetPriorityThread(), KiSwapThread(), KiSystemStartupBootStack(), and NtYieldExecution().

KiReleaseProcessLock()

FORCEINLINE VOID KiReleaseProcessLock

(

IN PKLOCK_QUEUE_HANDLE

Handle

)

Definition at line 660 of file ke_x.h.

{
    /* Release the lock and restore previous IRQL */
    KeReleaseInStackQueuedSpinLock(Handle);
}

Referenced by KeFreezeAllThreads(), KeQueryRuntimeProcess(), KeQueryValuesProcess(), KeSetQuantumProcess(), KeStartThread(), and MmSetExecuteOptions().

KiReleaseProcessLockFromSynchLevel()

FORCEINLINE VOID KiReleaseProcessLockFromSynchLevel

(

IN PKLOCK_QUEUE_HANDLE

Handle

)

Definition at line 668 of file ke_x.h.

{
    /* Release the lock without lowering IRQL */
    KeReleaseInStackQueuedSpinLockFromDpcLevel(Handle);
}

Referenced by KeFreezeAllThreads(), KeSetAffinityProcess(), KeSetPriorityAndQuantumProcess(), KeTerminateThread(), and KeThawAllThreads().

KiReleaseThreadLock()

FORCEINLINE VOID KiReleaseThreadLock

(

IN PKTHREAD

Thread

)

Definition at line 250 of file ke_x.h.

{
    UNREFERENCED_PARAMETER(Thread);
}

Referenced by _Requires_lock_not_held_(), ExpBoostOwnerThread(), KeBoostPriorityThread(), KeQueryBasePriorityThread(), KeSetBasePriorityThread(), KeSetPriorityAndQuantumProcess(), KeSetPriorityThread(), KeSignalGateBoostPriority(), KeWaitForGate(), KiAdjustQuantumThread(), KiDeferredReadyThread(), KiInsertQueueApc(), and NtYieldExecution().

KiReleaseTimerLock()

FORCEINLINE VOID KiReleaseTimerLock

(

IN PKSPIN_LOCK_QUEUE

LockQueue

)

Definition at line 297 of file ke_x.h.

{
    ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL);

    /* Nothing to do on UP */
    UNREFERENCED_PARAMETER(LockQueue);
}

Referenced by KeSetSystemTime(), KiCompleteTimer(), KiInsertTreeTimer(), KiTimerExpiration(), KxInsertTimer(), and KxRemoveTreeTimer().

KiRemoveEntryTimer()

FORCEINLINE VOID KiRemoveEntryTimer

(

IN PKTIMER

Timer

)

Definition at line 892 of file ke_x.h.

{
    ULONG Hand;
    PKTIMER_TABLE_ENTRY TableEntry;

    /* Remove the timer from the timer list and check if it's empty */
    Hand = Timer->Header.Hand;
    if (RemoveEntryList(&Timer->TimerListEntry))
    {
        /* Get the respective timer table entry */
        TableEntry = &KiTimerTableListHead[Hand];
        if (&TableEntry->Entry == TableEntry->Entry.Flink)
        {
            /* Set the entry to an infinite absolute time */
            TableEntry->Time.HighPart = 0xFFFFFFFF;
        }
    }

    /* Clear the list entries on dbg builds so we can tell the timer is gone */
#if DBG
    Timer->TimerListEntry.Flink = NULL;
    Timer->TimerListEntry.Blink = NULL;
#endif
}

Referenced by KeSetSystemTime(), KiCompleteTimer(), KiInsertTreeTimer(), and KiTimerExpiration().

KiRequestApcInterrupt()

FORCEINLINE VOID KiRequestApcInterrupt	(	IN BOOLEAN	NeedApc,
		IN UCHAR	Processor
	)

Definition at line 276 of file ke_x.h.

{
    /* We deliver instantly on UP */
    UNREFERENCED_PARAMETER(NeedApc);
    UNREFERENCED_PARAMETER(Processor);
}

Referenced by KiInsertQueueApc().

KiRescheduleThread()

FORCEINLINE VOID KiRescheduleThread	(	IN BOOLEAN	NewThread,
		IN ULONG	Cpu
	)

Definition at line 197 of file ke_x.h.

{
    /* This is meaningless on UP systems */
    UNREFERENCED_PARAMETER(NewThread);
    UNREFERENCED_PARAMETER(Cpu);
}

KiSelectReadyThread()

FORCEINLINE PKTHREAD KiSelectReadyThread	(	IN KPRIORITY	Priority,
		IN PKPRCB	Prcb
	)

Definition at line 1419 of file ke_x.h.

{
    ULONG PrioritySet;
    LONG HighPriority;
    PLIST_ENTRY ListEntry;
    PKTHREAD Thread = NULL;

    /* Save the current mask and get the priority set for the CPU */
    PrioritySet = Prcb->ReadySummary >> Priority;
    if (!PrioritySet) goto Quickie;

    /* Get the highest priority possible */
    BitScanReverse((PULONG)&HighPriority, PrioritySet);
    ASSERT((PrioritySet & PRIORITY_MASK(HighPriority)) != 0);
    HighPriority += Priority;

    /* Make sure the list isn't empty at the highest priority */
    ASSERT(IsListEmpty(&Prcb->DispatcherReadyListHead[HighPriority]) == FALSE);

    /* Get the first thread on the list */
    ListEntry = Prcb->DispatcherReadyListHead[HighPriority].Flink;
    Thread = CONTAINING_RECORD(ListEntry, KTHREAD, WaitListEntry);

    /* Make sure this thread is here for a reason */
    ASSERT(HighPriority == Thread->Priority);
    ASSERT(Thread->Affinity & AFFINITY_MASK(Prcb->Number));
    ASSERT(Thread->NextProcessor == Prcb->Number);

    /* Remove it from the list */
    if (RemoveEntryList(&Thread->WaitListEntry))
    {
        /* The list is empty now, reset the ready summary */
        Prcb->ReadySummary ^= PRIORITY_MASK(HighPriority);
    }

    /* Sanity check and return the thread */
Quickie:
    ASSERT((Thread == NULL) ||
           (Thread->BasePriority == 0) ||
           (Thread->Priority != 0));
    return Thread;
}

Referenced by _Requires_lock_not_held_(), KiAdjustQuantumThread(), KiSelectNextThread(), KiSetPriorityThread(), KiSwapThread(), and NtYieldExecution().

KiSetThreadSwapBusy()

FORCEINLINE VOID KiSetThreadSwapBusy

(

IN PKTHREAD

Thread

)

Definition at line 210 of file ke_x.h.

{
    UNREFERENCED_PARAMETER(Thread);
}

Referenced by _Requires_lock_not_held_(), KeDelayExecutionThread(), KeRemoveQueue(), KeTerminateThread(), KeWaitForGate(), KeWaitForMultipleObjects(), KeWaitForSingleObject(), KiExitDispatcher(), and NtYieldExecution().

KiTryThreadLock()

FORCEINLINE BOOLEAN KiTryThreadLock

(

IN PKTHREAD

Thread

)

Definition at line 260 of file ke_x.h.

{
    UNREFERENCED_PARAMETER(Thread);
    return FALSE;
}

Referenced by KeSignalGateBoostPriority().

KxInsertTimer()

FORCEINLINE VOID KxInsertTimer	(	IN PKTIMER	Timer,
		IN ULONG	Hand
	)

Definition at line 923 of file ke_x.h.

{
    PKSPIN_LOCK_QUEUE LockQueue;
    ASSERT(KeGetCurrentIrql() >= SYNCH_LEVEL);

    /* Acquire the lock and release the dispatcher lock */
    LockQueue = KiAcquireTimerLock(Hand);
    KiReleaseDispatcherLockFromSynchLevel();

    /* Try to insert the timer */
    if (KiInsertTimerTable(Timer, Hand))
    {
        /* Complete it */
        KiCompleteTimer(Timer, LockQueue);
    }
    else
    {
        /* Do nothing, just release the lock */
        KiReleaseTimerLock(LockQueue);
    }
}

Referenced by KeDelayExecutionThread(), KeRemoveQueue(), KeSetTimerEx(), KeWaitForMultipleObjects(), and KeWaitForSingleObject().

KxRemoveTreeTimer()

FORCEINLINE VOID KxRemoveTreeTimer

(

IN PKTIMER

Timer

)

Definition at line 1004 of file ke_x.h.

{
    ULONG Hand = Timer->Header.Hand;
    PKSPIN_LOCK_QUEUE LockQueue;
    PKTIMER_TABLE_ENTRY TimerEntry;

    /* Acquire timer lock */
    LockQueue = KiAcquireTimerLock(Hand);

    /* Set the timer as non-inserted */
    Timer->Header.Inserted = FALSE;

    /* Remove it from the timer list */
    if (RemoveEntryList(&Timer->TimerListEntry))
    {
        /* Get the entry and check if it's empty */
        TimerEntry = &KiTimerTableListHead[Hand];
        if (IsListEmpty(&TimerEntry->Entry))
        {
            /* Clear the time then */
            TimerEntry->Time.HighPart = 0xFFFFFFFF;
        }
    }

    /* Release the timer lock */
    KiReleaseTimerLock(LockQueue);
}

Referenced by KeCancelTimer(), KeSetTimerEx(), KiInsertQueue(), and KiUnlinkThread().

KxSetTimerForThreadWait()

FORCEINLINE VOID KxSetTimerForThreadWait	(	IN PKTIMER	Timer,
		IN LARGE_INTEGER	Interval,
		OUT PULONG	Hand
	)

Definition at line 1034 of file ke_x.h.

{
    ULONGLONG DueTime;
    LARGE_INTEGER InterruptTime, SystemTime, TimeDifference;

    /* Check the timer's interval to see if it's absolute */
    Timer->Header.Absolute = FALSE;
    if (Interval.HighPart >= 0)
    {
        /* Get the system time and calculate the relative time */
        KeQuerySystemTime(&SystemTime);
        TimeDifference.QuadPart = SystemTime.QuadPart - Interval.QuadPart;
        Timer->Header.Absolute = TRUE;

        /* Check if we've already expired */
        if (TimeDifference.HighPart >= 0)
        {
            /* Reset everything */
            Timer->DueTime.QuadPart = 0;
            *Hand = 0;
            Timer->Header.Hand = 0;
            return;
        }
        else
        {
            /* Update the interval */
            Interval = TimeDifference;
        }
    }

    /* Calculate the due time */
    InterruptTime.QuadPart = KeQueryInterruptTime();
    DueTime = InterruptTime.QuadPart - Interval.QuadPart;
    Timer->DueTime.QuadPart = DueTime;

    /* Calculate the timer handle */
    *Hand = KiComputeTimerTableIndex(DueTime);
    Timer->Header.Hand = (UCHAR)*Hand;
}

KxUnwaitThread()

FORCEINLINE VOID KxUnwaitThread	(	IN DISPATCHER_HEADER *	Object,
		IN KPRIORITY	Increment
	)

Definition at line 1259 of file ke_x.h.

{
    PLIST_ENTRY WaitEntry, WaitList;
    PKWAIT_BLOCK WaitBlock;
    PKTHREAD WaitThread;
    ULONG WaitKey;

    /* Loop the Wait Entries */
    WaitList = &Object->WaitListHead;
    ASSERT(IsListEmpty(&Object->WaitListHead) == FALSE);
    WaitEntry = WaitList->Flink;
    do
    {
        /* Get the current wait block */
        WaitBlock = CONTAINING_RECORD(WaitEntry, KWAIT_BLOCK, WaitListEntry);

        /* Get the waiting thread */
        WaitThread = WaitBlock->Thread;

        /* Check the current Wait Mode */
        if (WaitBlock->WaitType == WaitAny)
        {
            /* Use the actual wait key */
            WaitKey = WaitBlock->WaitKey;
        }
        else
        {
            /* Otherwise, use STATUS_KERNEL_APC */
            WaitKey = STATUS_KERNEL_APC;
        }

        /* Unwait the thread */
        KiUnwaitThread(WaitThread, WaitKey, Increment);

        /* Next entry */
        WaitEntry = WaitList->Flink;
    } while (WaitEntry != WaitList);
}

Referenced by KeSetEvent(), KeSetProcess(), KeTerminateThread(), KiSignalTimer(), KiTimerExpiration(), and KiTimerListExpire().

KxUnwaitThreadForEvent()

FORCEINLINE VOID KxUnwaitThreadForEvent	(	IN PKEVENT	Event,
		IN KPRIORITY	Increment
	)

Definition at line 1304 of file ke_x.h.

{
    PLIST_ENTRY WaitEntry, WaitList;
    PKWAIT_BLOCK WaitBlock;
    PKTHREAD WaitThread;

    /* Loop the Wait Entries */
    WaitList = &Event->Header.WaitListHead;
    ASSERT(IsListEmpty(&Event->Header.WaitListHead) == FALSE);
    WaitEntry = WaitList->Flink;
    do
    {
        /* Get the current wait block */
        WaitBlock = CONTAINING_RECORD(WaitEntry, KWAIT_BLOCK, WaitListEntry);

        /* Get the waiting thread */
        WaitThread = WaitBlock->Thread;

        /* Check the current Wait Mode */
        if (WaitBlock->WaitType == WaitAny)
        {
            /* Un-signal it */
            Event->Header.SignalState = 0;

            /* Un-signal the event and unwait the thread */
            KiUnwaitThread(WaitThread, WaitBlock->WaitKey, Increment);
            break;
        }

        /* Unwait the thread with STATUS_KERNEL_APC */
        KiUnwaitThread(WaitThread, STATUS_KERNEL_APC, Increment);

        /* Next entry */
        WaitEntry = WaitList->Flink;
    } while (WaitEntry != WaitList);
}

Referenced by KeSetEvent(), KeSetEventBoostPriority(), KiSignalTimer(), KiTimerExpiration(), and KiTimerListExpire().