ke_x.h File Reference

This graph shows which files directly or indirectly include this file:

Go to the source code of this file.

Macros
#define	KeEnterGuardedRegionThread(_Thread)
#define	KeEnterGuardedRegion()
#define	KeLeaveGuardedRegionThread(_Thread)
#define	KeLeaveGuardedRegion()
#define	KeEnterCriticalRegionThread(_Thread)
#define	KeEnterCriticalRegion()
#define	KeLeaveCriticalRegionThread(_Thread)
#define	KeLeaveCriticalRegion()
#define	KiSatisfyMutantWait(Object, Thread)
#define	KiSatisfyNonMutantWait(Object)
#define	KiSatisfyObjectWait(Object, Thread)
#define	KiAddThreadToWaitList(Thread, Swappable)
#define	KxDelayThreadWait()
#define	KxMultiThreadWait()
#define	KxSingleThreadWait()
#define	KxQueueThreadWait()

Functions
FORCEINLINE KPROCESSOR_MODE	KeGetPreviousMode (VOID)
FORCEINLINE VOID	KiAcquireDispatcherObject (IN DISPATCHER_HEADER *Object)
FORCEINLINE VOID	KiReleaseDispatcherObject (IN DISPATCHER_HEADER *Object)
FORCEINLINE KIRQL	KiAcquireDispatcherLock (VOID)
FORCEINLINE VOID	KiReleaseDispatcherLock (IN KIRQL OldIrql)
FORCEINLINE VOID	KiAcquireDispatcherLockAtSynchLevel (VOID)
FORCEINLINE VOID	KiReleaseDispatcherLockFromSynchLevel (VOID)
FORCEINLINE VOID	KiInsertDeferredReadyList (IN PKTHREAD Thread)
FORCEINLINE VOID	KiRescheduleThread (IN BOOLEAN NewThread, IN ULONG Cpu)
FORCEINLINE VOID	KiSetThreadSwapBusy (IN PKTHREAD Thread)
FORCEINLINE VOID	KiAcquirePrcbLock (IN PKPRCB Prcb)
FORCEINLINE VOID	KiReleasePrcbLock (IN PKPRCB Prcb)
FORCEINLINE VOID	KiAcquireThreadLock (IN PKTHREAD Thread)
FORCEINLINE VOID	KiReleaseThreadLock (IN PKTHREAD Thread)
FORCEINLINE BOOLEAN	KiTryThreadLock (IN PKTHREAD Thread)
FORCEINLINE VOID	KiCheckDeferredReadyList (IN PKPRCB Prcb)
FORCEINLINE VOID	KiRequestApcInterrupt (IN BOOLEAN NeedApc, IN UCHAR Processor)
FORCEINLINE PKSPIN_LOCK_QUEUE	KiAcquireTimerLock (IN ULONG Hand)
FORCEINLINE VOID	KiReleaseTimerLock (IN PKSPIN_LOCK_QUEUE LockQueue)
FORCEINLINE VOID	KiAcquireApcLockRaiseToSynch (IN PKTHREAD Thread, IN PKLOCK_QUEUE_HANDLE Handle)
FORCEINLINE VOID	KiAcquireApcLockAtSynchLevel (IN PKTHREAD Thread, IN PKLOCK_QUEUE_HANDLE Handle)
FORCEINLINE VOID	KiAcquireApcLockRaiseToDpc (IN PKTHREAD Thread, IN PKLOCK_QUEUE_HANDLE Handle)
FORCEINLINE VOID	KiReleaseApcLock (IN PKLOCK_QUEUE_HANDLE Handle)
FORCEINLINE VOID	KiReleaseApcLockFromSynchLevel (IN PKLOCK_QUEUE_HANDLE Handle)
FORCEINLINE VOID	KiAcquireProcessLockRaiseToSynch (IN PKPROCESS Process, IN PKLOCK_QUEUE_HANDLE Handle)
FORCEINLINE VOID	KiReleaseProcessLock (IN PKLOCK_QUEUE_HANDLE Handle)
FORCEINLINE VOID	KiReleaseProcessLockFromSynchLevel (IN PKLOCK_QUEUE_HANDLE Handle)
FORCEINLINE VOID	KiAcquireDeviceQueueLock (IN PKDEVICE_QUEUE DeviceQueue, IN PKLOCK_QUEUE_HANDLE DeviceLock)
FORCEINLINE VOID	KiReleaseDeviceQueueLock (IN PKLOCK_QUEUE_HANDLE DeviceLock)
FORCEINLINE PLARGE_INTEGER	KiRecalculateDueTime (IN PLARGE_INTEGER OriginalDueTime, IN PLARGE_INTEGER DueTime, IN OUT PLARGE_INTEGER NewDueTime)
FORCEINLINE BOOLEAN	KiCheckThreadStackSwap (IN PKTHREAD Thread, IN KPROCESSOR_MODE WaitMode)
FORCEINLINE NTSTATUS	KiCheckAlertability (IN PKTHREAD Thread, IN BOOLEAN Alertable, IN KPROCESSOR_MODE WaitMode)
FORCEINLINE ULONG	KiComputeTimerTableIndex (IN ULONGLONG DueTime)
FORCEINLINE VOID	KiRemoveEntryTimer (IN PKTIMER Timer)
FORCEINLINE VOID	KxInsertTimer (IN PKTIMER Timer, IN ULONG Hand)
FORCEINLINE BOOLEAN	KiComputeDueTime (IN PKTIMER Timer, IN LARGE_INTEGER DueTime, OUT PULONG Hand)
FORCEINLINE VOID	KxRemoveTreeTimer (IN PKTIMER Timer)
FORCEINLINE VOID	KxSetTimerForThreadWait (IN PKTIMER Timer, IN LARGE_INTEGER Interval, OUT PULONG Hand)
FORCEINLINE VOID	KxUnwaitThread (IN DISPATCHER_HEADER *Object, IN KPRIORITY Increment)
FORCEINLINE VOID	KxUnwaitThreadForEvent (IN PKEVENT Event, IN KPRIORITY Increment)
FORCEINLINE VOID	KxQueueReadyThread (IN PKTHREAD Thread, IN PKPRCB Prcb)
FORCEINLINE PKTHREAD	KiSelectReadyThread (IN KPRIORITY Priority, IN PKPRCB Prcb)
FORCEINLINE SCHAR	KiComputeNewPriority (IN PKTHREAD Thread, IN SCHAR Adjustment)
FORCEINLINE VOID	_KeInitializeGuardedMutex (OUT PKGUARDED_MUTEX GuardedMutex)
FORCEINLINE VOID	_KeAcquireGuardedMutexUnsafe (IN OUT PKGUARDED_MUTEX GuardedMutex)
FORCEINLINE VOID	_KeReleaseGuardedMutexUnsafe (IN OUT PKGUARDED_MUTEX GuardedMutex)
FORCEINLINE VOID	_KeAcquireGuardedMutex (IN PKGUARDED_MUTEX GuardedMutex)
FORCEINLINE VOID	_KeReleaseGuardedMutex (IN OUT PKGUARDED_MUTEX GuardedMutex)
FORCEINLINE BOOLEAN	_KeTryToAcquireGuardedMutex (IN OUT PKGUARDED_MUTEX GuardedMutex)
FORCEINLINE VOID	KiAcquireNmiListLock (OUT PKIRQL OldIrql)
FORCEINLINE VOID	KiReleaseNmiListLock (IN KIRQL OldIrql)

Macro Definition Documentation

KeEnterCriticalRegion

#define KeEnterCriticalRegion

(

)

Value:

{                                                                           \
    PKTHREAD _Thread = KeGetCurrentThread();                                \
    KeEnterCriticalRegionThread(_Thread);                                   \
}

Definition at line 83 of file ke_x.h.

KeEnterCriticalRegionThread

#define KeEnterCriticalRegionThread

(

_Thread

)

Value:

{                                                                           \
    /* Sanity checks */                                                     \
    ASSERT(_Thread == KeGetCurrentThread());                                \
    ASSERT((_Thread->KernelApcDisable <= 0) &&                              \
           (_Thread->KernelApcDisable != -32768));                          \
                                                                            \
    /* Disable Kernel APCs */                                               \
    _Thread->KernelApcDisable--;                                            \
}

Definition at line 72 of file ke_x.h.

KeEnterGuardedRegion

#define KeEnterGuardedRegion

(

)

Value:

{                                                                           \
    PKTHREAD _Thread = KeGetCurrentThread();                                \
    KeEnterGuardedRegionThread(_Thread);                                    \
}

Definition at line 34 of file ke_x.h.

KeEnterGuardedRegionThread

#define KeEnterGuardedRegionThread

(

_Thread

)

Value:

{                                                                           \
    /* Sanity checks */                                                     \
    ASSERT(KeGetCurrentIrql() <= APC_LEVEL);                                \
    ASSERT(_Thread == KeGetCurrentThread());                                \
    ASSERT((_Thread->SpecialApcDisable <= 0) &&                             \
           (_Thread->SpecialApcDisable != -32768));                         \
                                                                            \
    /* Disable Special APCs */                                              \
    _Thread->SpecialApcDisable--;                                           \
}

Definition at line 22 of file ke_x.h.

KeLeaveCriticalRegion

#define KeLeaveCriticalRegion

(

)

Value:

{                                                                           \
    PKTHREAD _Thread = KeGetCurrentThread();                                \
    KeLeaveCriticalRegionThread(_Thread);                                   \
}

Definition at line 114 of file ke_x.h.

KeLeaveCriticalRegionThread

#define KeLeaveCriticalRegionThread

(

_Thread

)

Value:

{                                                                           \
    /* Sanity checks */                                                     \
    ASSERT(_Thread == KeGetCurrentThread());                                \
    ASSERT(_Thread->KernelApcDisable < 0);                                  \
                                                                            \
    /* Enable Kernel APCs */                                                \
    _Thread->KernelApcDisable++;                                            \
                                                                            \
    /* Check if Kernel APCs are now enabled */                              \
    if (!(_Thread->KernelApcDisable))                                       \
    {                                                                       \
        /* Check if we need to request an APC Delivery */                   \
        if (!(IsListEmpty(&_Thread->ApcState.ApcListHead[KernelMode])) &&   \
            !(_Thread->SpecialApcDisable))                                  \
        {                                                                   \
            /* Check for the right environment */                           \
            KiCheckForKernelApcDelivery();                                  \
        }                                                                   \
    }                                                                       \
}

Definition at line 92 of file ke_x.h.

KeLeaveGuardedRegion

#define KeLeaveGuardedRegion

(

)

Value:

{                                                                           \
    PKTHREAD _Thread = KeGetCurrentThread();                                \
    KeLeaveGuardedRegionThread(_Thread);                                    \
}

Definition at line 63 of file ke_x.h.

KeLeaveGuardedRegionThread

#define KeLeaveGuardedRegionThread

(

_Thread

)

Value:

{                                                                           \
    /* Sanity checks */                                                     \
    ASSERT(KeGetCurrentIrql() <= APC_LEVEL);                                \
    ASSERT(_Thread == KeGetCurrentThread());                                \
    ASSERT(_Thread->SpecialApcDisable < 0);                                 \
                                                                            \
    /* Leave region and check if APCs are OK now */                         \
    if (!(++_Thread->SpecialApcDisable))                                    \
    {                                                                       \
        /* Check for Kernel APCs on the list */                             \
        if (!IsListEmpty(&_Thread->ApcState.                                \
                         ApcListHead[KernelMode]))                          \
        {                                                                   \
            /* Check for APC Delivery */                                    \
            KiCheckForKernelApcDelivery();                                  \
        }                                                                   \
    }                                                                       \
}

Definition at line 43 of file ke_x.h.

KiAddThreadToWaitList

#define KiAddThreadToWaitList	(		Thread,
			Swappable
	)

Value:

{                                                                           \
    /* Make sure it's swappable */                                          \
    if (Swappable)                                                          \
    {                                                                       \
        /* Insert it into the PRCB's List */                                \
        InsertTailList(&KeGetCurrentPrcb()->WaitListHead,                   \
                       &Thread->WaitListEntry);                             \
    }                                                                       \
}

Definition at line 819 of file ke_x.h.

KiSatisfyMutantWait

#define KiSatisfyMutantWait	(		Object,
			Thread
	)

Value:

{                                                                           \
    /* Decrease the Signal State */                                         \
    (Object)->Header.SignalState--;                                         \
                                                                            \
    /* Check if it's now non-signaled */                                    \
    if (!(Object)->Header.SignalState)                                      \
    {                                                                       \
        /* Set the Owner Thread */                                          \
        (Object)->OwnerThread = Thread;                                     \
                                                                            \
        /* Disable APCs if needed */                                        \
        Thread->KernelApcDisable = Thread->KernelApcDisable -               \
                                   (Object)->ApcDisable;                    \
                                                                            \
        /* Check if it's abandoned */                                       \
        if ((Object)->Abandoned)                                            \
        {                                                                   \
            /* Unabandon it */                                              \
            (Object)->Abandoned = FALSE;                                    \
                                                                            \
            /* Return Status */                                             \
            Thread->WaitStatus = STATUS_ABANDONED;                          \
        }                                                                   \
                                                                            \
        /* Insert it into the Mutant List */                                \
        InsertHeadList(Thread->MutantListHead.Blink,                        \
                       &(Object)->MutantListEntry);                         \
    }                                                                       \
}

Definition at line 710 of file ke_x.h.

KiSatisfyNonMutantWait

#define KiSatisfyNonMutantWait

(

Object

)

Value:

{                                                                           \
    if (((Object)->Header.Type & TIMER_OR_EVENT_TYPE) ==                    \
             EventSynchronizationObject)                                    \
    {                                                                       \
        /* Synchronization Timers and Events just get un-signaled */        \
        (Object)->Header.SignalState = 0;                                   \
    }                                                                       \
    else if ((Object)->Header.Type == SemaphoreObject)                      \
    {                                                                       \
        /* These ones can have multiple states, so we only decrease it */   \
        (Object)->Header.SignalState--;                                     \
    }                                                                       \
}

Definition at line 744 of file ke_x.h.

KiSatisfyObjectWait

#define KiSatisfyObjectWait	(		Object,
			Thread
	)

Value:

{                                                                           \
    /* Special case for Mutants */                                          \
    if ((Object)->Header.Type == MutantObject)                              \
    {                                                                       \
        KiSatisfyMutantWait((Object), (Thread));                            \
    }                                                                       \
    else                                                                    \
    {                                                                       \
        KiSatisfyNonMutantWait(Object);                                     \
    }                                                                       \
}

Definition at line 762 of file ke_x.h.

KxDelayThreadWait

#define KxDelayThreadWait

(

)

Value:

\
    /* Setup the Wait Block */                                              \
    Thread->WaitBlockList = TimerBlock;                                     \
                                                                            \
    /* Setup the timer */                                                   \
    KxSetTimerForThreadWait(Timer, *Interval, &Hand);                       \
                                                                            \
    /* Save the due time for the caller */                                  \
    DueTime.QuadPart = Timer->DueTime.QuadPart;                             \
                                                                            \
    /* Link the timer to this Wait Block */                                 \
    TimerBlock->NextWaitBlock = TimerBlock;                                 \
    Timer->Header.WaitListHead.Flink = &TimerBlock->WaitListEntry;          \
    Timer->Header.WaitListHead.Blink = &TimerBlock->WaitListEntry;          \
                                                                            \
    /* Clear wait status */                                                 \
    Thread->WaitStatus = STATUS_SUCCESS;                                    \
                                                                            \
    /* Setup wait fields */                                                 \
    Thread->Alertable = Alertable;                                          \
    Thread->WaitReason = DelayExecution;                                    \
    Thread->WaitMode = WaitMode;                                            \
                                                                            \
    /* Check if we can swap the thread's stack */                           \
    Thread->WaitListEntry.Flink = NULL;                                     \
    Swappable = KiCheckThreadStackSwap(Thread, WaitMode);                   \
                                                                            \
    /* Set the wait time */                                                 \
    Thread->WaitTime = KeTickCount.LowPart;

Definition at line 1071 of file ke_x.h.

KxMultiThreadWait

#define KxMultiThreadWait

(

)

Definition at line 1102 of file ke_x.h.

KxQueueThreadWait

#define KxQueueThreadWait

(

)

Definition at line 1203 of file ke_x.h.

KxSingleThreadWait

#define KxSingleThreadWait

(

)

Definition at line 1158 of file ke_x.h.

Function Documentation

_KeAcquireGuardedMutex()

FORCEINLINE VOID _KeAcquireGuardedMutex

(

IN PKGUARDED_MUTEX

GuardedMutex

)

Definition at line 1577 of file ke_x.h.

{
    PKTHREAD Thread = KeGetCurrentThread();

    /* Sanity checks */
    ASSERT(KeGetCurrentIrql() <= APC_LEVEL);
    ASSERT(GuardedMutex->Owner != Thread);

    /* Disable Special APCs */
    KeEnterGuardedRegionThread(Thread);

    /* Remove the lock */
    if (!InterlockedBitTestAndReset(&GuardedMutex->Count, GM_LOCK_BIT_V))
    {
        /* The Guarded Mutex was already locked, enter contented case */
        KiAcquireGuardedMutex(GuardedMutex);
    }

    /* Set the Owner and Special APC Disable state */
    GuardedMutex->Owner = Thread;
    GuardedMutex->SpecialApcDisable = Thread->SpecialApcDisable;
}

Referenced by KeAcquireGuardedMutex().

_KeAcquireGuardedMutexUnsafe()

FORCEINLINE VOID _KeAcquireGuardedMutexUnsafe

(

IN OUT PKGUARDED_MUTEX

GuardedMutex

)

Definition at line 1512 of file ke_x.h.

{
    PKTHREAD Thread = KeGetCurrentThread();

    /* Sanity checks */
    ASSERT((KeGetCurrentIrql() == APC_LEVEL) ||
           (Thread->SpecialApcDisable < 0) ||
           (Thread->Teb == NULL) ||
           (Thread->Teb >= (PTEB)MM_SYSTEM_RANGE_START));
    ASSERT(GuardedMutex->Owner != Thread);

    /* Remove the lock */
    if (!InterlockedBitTestAndReset(&GuardedMutex->Count, GM_LOCK_BIT_V))
    {
        /* The Guarded Mutex was already locked, enter contented case */
        KiAcquireGuardedMutex(GuardedMutex);
    }

    /* Set the Owner */
    GuardedMutex->Owner = Thread;
}

Referenced by KeAcquireGuardedMutexUnsafe().

_KeInitializeGuardedMutex()

FORCEINLINE VOID _KeInitializeGuardedMutex

(

OUT PKGUARDED_MUTEX

GuardedMutex

)

Definition at line 1499 of file ke_x.h.

{
    /* Setup the Initial Data */
    GuardedMutex->Count = GM_LOCK_BIT;
    GuardedMutex->Owner = NULL;
    GuardedMutex->Contention = 0;

    /* Initialize the Wait Gate */
    KeInitializeGate(&GuardedMutex->Gate);
}

Referenced by KeInitializeGuardedMutex().

_KeReleaseGuardedMutex()

FORCEINLINE VOID _KeReleaseGuardedMutex

(

IN OUT PKGUARDED_MUTEX

GuardedMutex

)

Definition at line 1602 of file ke_x.h.

{
    PKTHREAD Thread = KeGetCurrentThread();
    LONG OldValue, NewValue;

    /* Sanity checks */
    ASSERT(KeGetCurrentIrql() <= APC_LEVEL);
    ASSERT(GuardedMutex->Owner == Thread);
    ASSERT(Thread->SpecialApcDisable == GuardedMutex->SpecialApcDisable);

    /* Destroy the Owner */
    GuardedMutex->Owner = NULL;

    /* Add the Lock Bit */
    OldValue = InterlockedExchangeAdd(&GuardedMutex->Count, GM_LOCK_BIT);
    ASSERT((OldValue & GM_LOCK_BIT) == 0);

    /* Check if it was already locked, but not woken */
    if ((OldValue) && !(OldValue & GM_LOCK_WAITER_WOKEN))
    {
        /* Update the Oldvalue to what it should be now */
        OldValue += GM_LOCK_BIT;

        /* The mutex will be woken, minus one waiter */
        NewValue = OldValue + GM_LOCK_WAITER_WOKEN -
            GM_LOCK_WAITER_INC;

        /* Remove the Woken bit */
        if (InterlockedCompareExchange(&GuardedMutex->Count,
                                       NewValue,
                                       OldValue) == OldValue)
        {
            /* Signal the Gate */
            KeSignalGateBoostPriority(&GuardedMutex->Gate);
        }
    }

    /* Re-enable APCs */
    KeLeaveGuardedRegionThread(Thread);
}

Referenced by KeReleaseGuardedMutex().

_KeReleaseGuardedMutexUnsafe()

FORCEINLINE VOID _KeReleaseGuardedMutexUnsafe

(

IN OUT PKGUARDED_MUTEX

GuardedMutex

)

Definition at line 1536 of file ke_x.h.

{
    LONG OldValue, NewValue;

    /* Sanity checks */
    ASSERT((KeGetCurrentIrql() == APC_LEVEL) ||
           (KeGetCurrentThread()->SpecialApcDisable < 0) ||
           (KeGetCurrentThread()->Teb == NULL) ||
           (KeGetCurrentThread()->Teb >= (PTEB)MM_SYSTEM_RANGE_START));
    ASSERT(GuardedMutex->Owner == KeGetCurrentThread());

    /* Destroy the Owner */
    GuardedMutex->Owner = NULL;

    /* Add the Lock Bit */
    OldValue = InterlockedExchangeAdd(&GuardedMutex->Count, GM_LOCK_BIT);
    ASSERT((OldValue & GM_LOCK_BIT) == 0);

    /* Check if it was already locked, but not woken */
    if ((OldValue) && !(OldValue & GM_LOCK_WAITER_WOKEN))
    {
        /* Update the Oldvalue to what it should be now */
        OldValue += GM_LOCK_BIT;

        /* The mutex will be woken, minus one waiter */
        NewValue = OldValue + GM_LOCK_WAITER_WOKEN -
            GM_LOCK_WAITER_INC;

        /* Remove the Woken bit */
        if (InterlockedCompareExchange(&GuardedMutex->Count,
                                       NewValue,
                                       OldValue) == OldValue)
        {
            /* Signal the Gate */
            KeSignalGateBoostPriority(&GuardedMutex->Gate);
        }
    }
}

Referenced by KeReleaseGuardedMutexUnsafe().

_KeTryToAcquireGuardedMutex()

FORCEINLINE BOOLEAN _KeTryToAcquireGuardedMutex

(

IN OUT PKGUARDED_MUTEX

GuardedMutex

)

Definition at line 1645 of file ke_x.h.

{
    PKTHREAD Thread = KeGetCurrentThread();

    /* Block APCs */
    KeEnterGuardedRegionThread(Thread);

    /* Remove the lock */
    if (!InterlockedBitTestAndReset(&GuardedMutex->Count, GM_LOCK_BIT_V))
    {
        /* Re-enable APCs */
        KeLeaveGuardedRegionThread(Thread);
        YieldProcessor();

        /* Return failure */
        return FALSE;
    }

    /* Set the Owner and APC State */
    GuardedMutex->Owner = Thread;
    GuardedMutex->SpecialApcDisable = Thread->SpecialApcDisable;
    return TRUE;
}

Referenced by KeTryToAcquireGuardedMutex().

KeGetPreviousMode()

FORCEINLINE KPROCESSOR_MODE KeGetPreviousMode

(

VOID

)

Definition at line 12 of file ke_x.h.

{
    /* Return the current mode */
    return KeGetCurrentThread()->PreviousMode;
}

KiAcquireApcLockAtSynchLevel()

FORCEINLINE VOID KiAcquireApcLockAtSynchLevel	(	IN PKTHREAD	Thread,
		IN PKLOCK_QUEUE_HANDLE	Handle
	)

Definition at line 611 of file ke_x.h.

{
    /* Acquire the lock */
    ASSERT(KeGetCurrentIrql() >= SYNCH_LEVEL);
    KeAcquireInStackQueuedSpinLockAtDpcLevel(&Thread->ApcQueueLock, Handle);
}

Referenced by KeFreezeAllThreads(), and KeThawAllThreads().

KiAcquireApcLockRaiseToDpc()

FORCEINLINE VOID KiAcquireApcLockRaiseToDpc	(	IN PKTHREAD	Thread,
		IN PKLOCK_QUEUE_HANDLE	Handle
	)

Definition at line 621 of file ke_x.h.

{
    /* Acquire the lock */
    KeAcquireInStackQueuedSpinLock(&Thread->ApcQueueLock, Handle);
}

Referenced by KiDeliverApc().

KiAcquireApcLockRaiseToSynch()

FORCEINLINE VOID KiAcquireApcLockRaiseToSynch	(	IN PKTHREAD	Thread,
		IN PKLOCK_QUEUE_HANDLE	Handle
	)

Definition at line 602 of file ke_x.h.

{
    /* Acquire the lock and raise to synchronization level */
    KeAcquireInStackQueuedSpinLockRaiseToSynch(&Thread->ApcQueueLock, Handle);
}

Referenced by KeAlertResumeThread(), KeAlertThread(), KeAttachProcess(), KeDetachProcess(), KeFlushQueueApc(), KeForceResumeThread(), KeInsertQueueApc(), KeRemoveQueueApc(), KeResumeThread(), KeStackAttachProcess(), KeSuspendThread(), KeTestAlertThread(), KeUnstackDetachProcess(), and KeWaitForGate().

KiAcquireDeviceQueueLock()

FORCEINLINE VOID KiAcquireDeviceQueueLock	(	IN PKDEVICE_QUEUE	DeviceQueue,
		IN PKLOCK_QUEUE_HANDLE	DeviceLock
	)

Definition at line 671 of file ke_x.h.

{
    /* Check if we were called from a threaded DPC */
    if (KeGetCurrentPrcb()->DpcThreadActive)
    {
        /* Lock the Queue, we're not at DPC level */
        KeAcquireInStackQueuedSpinLock(&DeviceQueue->Lock, DeviceLock);
    }
    else
    {
        /* We must be at DPC level, acquire the lock safely */
        ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
        KeAcquireInStackQueuedSpinLockAtDpcLevel(&DeviceQueue->Lock,
                                                 DeviceLock);
    }
}

Referenced by KeInsertByKeyDeviceQueue(), KeInsertDeviceQueue(), KeRemoveByKeyDeviceQueue(), KeRemoveByKeyDeviceQueueIfBusy(), KeRemoveDeviceQueue(), and KeRemoveEntryDeviceQueue().

KiAcquireDispatcherLock()

FORCEINLINE KIRQL KiAcquireDispatcherLock

(

VOID

)

Definition at line 144 of file ke_x.h.

{
    /* Raise to synch level */
    return KfRaiseIrql(SYNCH_LEVEL);
}

Referenced by ExpExpandResourceOwnerTable(), ExpWaitForResource(), KeBoostPriorityThread(), KeCancelTimer(), KeConnectInterrupt(), KeDisconnectInterrupt(), KeInitializeMutant(), KeInsertHeadQueue(), KeInsertQueue(), KePulseEvent(), KeReadyThread(), KeReleaseMutant(), KeReleaseSemaphore(), KeResetEvent(), KeRevertToUserAffinityThread(), KeRundownQueue(), KeRundownThread(), KeSetAffinityThread(), KeSetBasePriorityThread(), KeSetEvent(), KeSetEventBoostPriority(), KeSetIdealProcessorThread(), KeSetPriorityThread(), KeSetProcess(), KeSetSystemAffinityThread(), KeSetSystemTime(), KeSetTimerEx(), KiInsertQueueApc(), KiScanReadyQueues(), and KiTimerExpiration().

KiAcquireDispatcherLockAtSynchLevel()

FORCEINLINE VOID KiAcquireDispatcherLockAtSynchLevel

(

VOID

)

Definition at line 160 of file ke_x.h.

{
    /* This is a no-op at SYNCH_LEVEL for UP systems */
    ASSERT(KeGetCurrentIrql() >= SYNCH_LEVEL);
    return;
}

Referenced by KeAlertResumeThread(), KeAlertThread(), KeAttachProcess(), KeDelayExecutionThread(), KeDetachProcess(), KeForceResumeThread(), KeFreezeAllThreads(), KeRemoveQueue(), KeRemoveQueueApc(), KeResumeThread(), KeSetAffinityProcess(), KeSignalGateBoostPriority(), KeStackAttachProcess(), KeStartThread(), KeSuspendThread(), KeTerminateThread(), KeThawAllThreads(), KeUnstackDetachProcess(), KeWaitForGate(), KeWaitForMultipleObjects(), KeWaitForSingleObject(), and KiCompleteTimer().

KiAcquireDispatcherObject()

FORCEINLINE VOID KiAcquireDispatcherObject

(

IN DISPATCHER_HEADER *

Object

)

Definition at line 127 of file ke_x.h.

{
    UNREFERENCED_PARAMETER(Object);
}

Referenced by KeSignalGateBoostPriority(), KeWaitForGate(), and KiInsertQueueApc().

KiAcquireNmiListLock()

FORCEINLINE VOID KiAcquireNmiListLock

(

OUT PKIRQL

OldIrql

)

Definition at line 1672 of file ke_x.h.

{
    KeAcquireSpinLock(&KiNmiCallbackListLock, OldIrql);
}

Referenced by KeDeregisterNmiCallback(), and KeRegisterNmiCallback().

KiAcquirePrcbLock()

FORCEINLINE VOID KiAcquirePrcbLock

(

IN PKPRCB

Prcb

)

Definition at line 215 of file ke_x.h.

{
    UNREFERENCED_PARAMETER(Prcb);
}

Referenced by KeRevertToUserAffinityThread(), KeSetSystemAffinityThread(), KiAdjustQuantumThread(), KiDeferredReadyThread(), KiExitDispatcher(), KiInitializeKernel(), KiQuantumEnd(), KiScanReadyQueues(), KiSetPriorityThread(), KiSwapThread(), KiSystemStartupBootStack(), and NtYieldExecution().

KiAcquireProcessLockRaiseToSynch()

FORCEINLINE VOID KiAcquireProcessLockRaiseToSynch	(	IN PKPROCESS	Process,
		IN PKLOCK_QUEUE_HANDLE	Handle
	)

Definition at line 646 of file ke_x.h.

{
    /* Acquire the lock and raise to synchronization level */
    KeAcquireInStackQueuedSpinLockRaiseToSynch(&Process->ProcessLock, Handle);
}

Referenced by KeFreezeAllThreads(), KeQueryRuntimeProcess(), KeQueryValuesProcess(), KeSetAffinityProcess(), KeSetPriorityAndQuantumProcess(), KeSetQuantumProcess(), KeStartThread(), KeTerminateThread(), KeThawAllThreads(), and MmSetExecuteOptions().

KiAcquireThreadLock()

FORCEINLINE VOID KiAcquireThreadLock

(

IN PKTHREAD

Thread

)

Definition at line 235 of file ke_x.h.

{
    UNREFERENCED_PARAMETER(Thread);
}

Referenced by ExpBoostOwnerThread(), KeBoostPriorityThread(), KeQueryBasePriorityThread(), KeSetBasePriorityThread(), KeSetPriorityAndQuantumProcess(), KeSetPriorityThread(), KeWaitForGate(), KiAdjustQuantumThread(), KiDeferredReadyThread(), KiInsertQueueApc(), KiQuantumEnd(), and NtYieldExecution().

KiAcquireTimerLock()

FORCEINLINE PKSPIN_LOCK_QUEUE KiAcquireTimerLock

(

IN ULONG

Hand

)

Definition at line 281 of file ke_x.h.

{
    ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL);

    /* Nothing to do on UP */
    UNREFERENCED_PARAMETER(Hand);
    return NULL;
}

Referenced by KeSetSystemTime(), KiInsertTreeTimer(), KiTimerExpiration(), KxInsertTimer(), and KxRemoveTreeTimer().

KiCheckAlertability()

FORCEINLINE NTSTATUS KiCheckAlertability	(	IN PKTHREAD	Thread,
		IN BOOLEAN	Alertable,
		IN KPROCESSOR_MODE	WaitMode
	)

Definition at line 836 of file ke_x.h.

{
    /* Check if the wait is alertable */
    if (Alertable)
    {
        /* It is, first check if the thread is alerted in this mode */
        if (Thread->Alerted[WaitMode])
        {
            /* It is, so bail out of the wait */
            Thread->Alerted[WaitMode] = FALSE;
            return STATUS_ALERTED;
        }
        else if ((WaitMode != KernelMode) &&
                (!IsListEmpty(&Thread->ApcState.ApcListHead[UserMode])))
        {
            /* It's isn't, but this is a user wait with queued user APCs */
            Thread->ApcState.UserApcPending = TRUE;
            return STATUS_USER_APC;
        }
        else if (Thread->Alerted[KernelMode])
        {
            /* It isn't that either, but we're alered in kernel mode */
            Thread->Alerted[KernelMode] = FALSE;
            return STATUS_ALERTED;
        }
    }
    else if ((WaitMode != KernelMode) && (Thread->ApcState.UserApcPending))
    {
        /* Not alertable, but this is a user wait with pending user APCs */
        return STATUS_USER_APC;
    }

    /* Otherwise, we're fine */
    return STATUS_WAIT_0;
}

Referenced by KeDelayExecutionThread(), KeWaitForMultipleObjects(), and KeWaitForSingleObject().

KiCheckDeferredReadyList()

FORCEINLINE VOID KiCheckDeferredReadyList

(

IN PKPRCB

Prcb

)

Definition at line 263 of file ke_x.h.

{
    /* There are no deferred ready lists on UP systems */
    UNREFERENCED_PARAMETER(Prcb);
}

Referenced by KiExitDispatcher().

KiCheckThreadStackSwap()

FORCEINLINE BOOLEAN KiCheckThreadStackSwap	(	IN PKTHREAD	Thread,
		IN KPROCESSOR_MODE	WaitMode
	)

Definition at line 798 of file ke_x.h.

{
    /* Check the required conditions */
    if ((WaitMode != KernelMode) &&
        (Thread->EnableStackSwap) &&
        (Thread->Priority >= (LOW_REALTIME_PRIORITY + 9)))
    {
        /* We are go for swap */
        return TRUE;
    }
    else
    {
        /* Don't swap the thread */
        return FALSE;
    }
}

KiComputeDueTime()

FORCEINLINE BOOLEAN KiComputeDueTime	(	IN PKTIMER	Timer,
		IN LARGE_INTEGER	DueTime,
		OUT PULONG	Hand
	)

Definition at line 947 of file ke_x.h.

{
    LARGE_INTEGER InterruptTime, SystemTime, DifferenceTime;

    /* Convert to relative time if needed */
    Timer->Header.Absolute = FALSE;
    if (DueTime.HighPart >= 0)
    {
        /* Get System Time */
        KeQuerySystemTime(&SystemTime);

        /* Do the conversion */
        DifferenceTime.QuadPart = SystemTime.QuadPart - DueTime.QuadPart;

        /* Make sure it hasn't already expired */
        Timer->Header.Absolute = TRUE;
        if (DifferenceTime.HighPart >= 0)
        {
            /* Cancel everything */
            Timer->Header.SignalState = TRUE;
            Timer->Header.Hand = 0;
            Timer->DueTime.QuadPart = 0;
            *Hand = 0;
            return FALSE;
        }

        /* Set the time as Absolute */
        DueTime = DifferenceTime;
    }

    /* Get the Interrupt Time */
    InterruptTime.QuadPart = KeQueryInterruptTime();

    /* Recalculate due time */
    Timer->DueTime.QuadPart = InterruptTime.QuadPart - DueTime.QuadPart;

    /* Get the handle */
    *Hand = KiComputeTimerTableIndex(Timer->DueTime.QuadPart);
    Timer->Header.Hand = (UCHAR)*Hand;
    Timer->Header.Inserted = TRUE;
    return TRUE;
}

Referenced by KeSetTimerEx(), and KiInsertTreeTimer().

KiComputeNewPriority()

FORCEINLINE SCHAR KiComputeNewPriority	(	IN PKTHREAD	Thread,
		IN SCHAR	Adjustment
	)

Definition at line 1462 of file ke_x.h.

{
    SCHAR Priority;

    /* Priority sanity checks */
    ASSERT((Thread->PriorityDecrement >= 0) &&
           (Thread->PriorityDecrement <= Thread->Priority));
    ASSERT((Thread->Priority < LOW_REALTIME_PRIORITY) ?
            TRUE : (Thread->PriorityDecrement == 0));

    /* Get the current priority */
    Priority = Thread->Priority;
    if (Priority < LOW_REALTIME_PRIORITY)
    {
        /* Decrease priority by the priority decrement */
        Priority -= (Thread->PriorityDecrement + Adjustment);

        /* Don't go out of bounds */
        if (Priority < Thread->BasePriority) Priority = Thread->BasePriority;

        /* Reset the priority decrement */
        Thread->PriorityDecrement = 0;
    }

    /* Sanity check */
    ASSERT((Thread->BasePriority == 0) || (Priority != 0));

    /* Return the new priority */
    return Priority;
}

Referenced by KeSetBasePriorityThread(), KeSetEventBoostPriority(), KiAdjustQuantumThread(), KiDeferredReadyThread(), KiQuantumEnd(), and NtYieldExecution().

KiComputeTimerTableIndex()

FORCEINLINE ULONG KiComputeTimerTableIndex

(

IN ULONGLONG

DueTime

)

Definition at line 876 of file ke_x.h.

{
    return (DueTime / KeMaximumIncrement) & (TIMER_TABLE_SIZE - 1);
}

Referenced by KeSetSystemTime(), KiComputeDueTime(), KiInsertTimerTable(), and KxSetTimerForThreadWait().

KiInsertDeferredReadyList()

FORCEINLINE VOID KiInsertDeferredReadyList

(

IN PKTHREAD

Thread

)

Definition at line 180 of file ke_x.h.

{
    /* Set the thread to deferred state and boot CPU */
    Thread->State = DeferredReady;
    Thread->DeferredProcessor = 0;

    /* Make the thread ready immediately */
    KiDeferredReadyThread(Thread);
}

Referenced by KiInsertQueueApc(), KiReadyThread(), KiScanReadyQueues(), and KiSetPriorityThread().

KiRecalculateDueTime()

FORCEINLINE PLARGE_INTEGER KiRecalculateDueTime	(	IN PLARGE_INTEGER	OriginalDueTime,
		IN PLARGE_INTEGER	DueTime,
		IN OUT PLARGE_INTEGER	NewDueTime
	)

Definition at line 780 of file ke_x.h.

{
    /* Don't do anything for absolute waits */
    if (OriginalDueTime->QuadPart >= 0) return OriginalDueTime;

    /* Otherwise, query the interrupt time and recalculate */
    NewDueTime->QuadPart = KeQueryInterruptTime();
    NewDueTime->QuadPart -= DueTime->QuadPart;
    return NewDueTime;
}

Referenced by KeDelayExecutionThread(), KeRemoveQueue(), KeWaitForMultipleObjects(), and KeWaitForSingleObject().

KiReleaseApcLock()

FORCEINLINE VOID KiReleaseApcLock

(

IN PKLOCK_QUEUE_HANDLE

Handle

)

Definition at line 630 of file ke_x.h.

{
    /* Release the lock */
    KeReleaseInStackQueuedSpinLock(Handle);
}

Referenced by KeFlushQueueApc(), KeRemoveQueueApc(), KeSuspendThread(), KeTestAlertThread(), KeUnstackDetachProcess(), KeWaitForGate(), and KiDeliverApc().

KiReleaseApcLockFromSynchLevel()

FORCEINLINE VOID KiReleaseApcLockFromSynchLevel

(

IN PKLOCK_QUEUE_HANDLE

Handle

)

Definition at line 638 of file ke_x.h.

{
    /* Release the lock */
    KeReleaseInStackQueuedSpinLockFromDpcLevel(Handle);
}

Referenced by KeAlertResumeThread(), KeAlertThread(), KeDetachProcess(), KeForceResumeThread(), KeFreezeAllThreads(), KeInsertQueueApc(), KeResumeThread(), KeSuspendThread(), KeThawAllThreads(), KeUnstackDetachProcess(), KeWaitForGate(), and KiAttachProcess().

KiReleaseDeviceQueueLock()

FORCEINLINE VOID KiReleaseDeviceQueueLock

(

IN PKLOCK_QUEUE_HANDLE

DeviceLock

)

Definition at line 691 of file ke_x.h.

{
    /* Check if we were called from a threaded DPC */
    if (KeGetCurrentPrcb()->DpcThreadActive)
    {
        /* Unlock the Queue, we're not at DPC level */
        KeReleaseInStackQueuedSpinLock(DeviceLock);
    }
    else
    {
        /* We must be at DPC level, release the lock safely */
        ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
        KeReleaseInStackQueuedSpinLockFromDpcLevel(DeviceLock);
    }
}

Referenced by KeInsertByKeyDeviceQueue(), KeInsertDeviceQueue(), KeRemoveByKeyDeviceQueue(), KeRemoveByKeyDeviceQueueIfBusy(), KeRemoveDeviceQueue(), and KeRemoveEntryDeviceQueue().

KiReleaseDispatcherLock()

FORCEINLINE VOID KiReleaseDispatcherLock

(

IN KIRQL

OldIrql

)

Definition at line 152 of file ke_x.h.

{
    /* Just exit the dispatcher */
    KiExitDispatcher(OldIrql);
}

Referenced by ExpExpandResourceOwnerTable(), KeBoostPriorityThread(), KeCancelTimer(), KeConnectInterrupt(), KeDelayExecutionThread(), KeDisconnectInterrupt(), KeInitializeMutant(), KeInsertHeadQueue(), KeInsertQueue(), KePulseEvent(), KeReadyThread(), KeReleaseMutant(), KeReleaseSemaphore(), KeResetEvent(), KeRevertToUserAffinityThread(), KeRundownThread(), KeSetAffinityThread(), KeSetBasePriorityThread(), KeSetEvent(), KeSetEventBoostPriority(), KeSetIdealProcessorThread(), KeSetPriorityThread(), KeSetProcess(), KeSetSystemAffinityThread(), KeWaitForMultipleObjects(), KeWaitForSingleObject(), KiScanReadyQueues(), KiTimerExpiration(), and KiTimerListExpire().

KiReleaseDispatcherLockFromSynchLevel()

FORCEINLINE VOID KiReleaseDispatcherLockFromSynchLevel

(

VOID

)

Definition at line 169 of file ke_x.h.

{
    /* This is a no-op at SYNCH_LEVEL for UP systems */
    return;
}

Referenced by KeAlertResumeThread(), KeAlertThread(), KeDelayExecutionThread(), KeDetachProcess(), KeForceResumeThread(), KeFreezeAllThreads(), KeRemoveQueue(), KeRemoveQueueApc(), KeResumeThread(), KeRundownQueue(), KeSetAffinityProcess(), KeSetPriorityAndQuantumProcess(), KeSetTimerEx(), KeSignalGateBoostPriority(), KeStartThread(), KeSuspendThread(), KeTerminateThread(), KeThawAllThreads(), KeUnstackDetachProcess(), KeWaitForGate(), KeWaitForMultipleObjects(), KeWaitForSingleObject(), KiAttachProcess(), KiCompleteTimer(), KiInsertQueueApc(), and KxInsertTimer().

KiReleaseDispatcherObject()

FORCEINLINE VOID KiReleaseDispatcherObject

(

IN DISPATCHER_HEADER *

Object

)

Definition at line 137 of file ke_x.h.

{
    UNREFERENCED_PARAMETER(Object);
}

Referenced by KeSignalGateBoostPriority(), KeWaitForGate(), and KiInsertQueueApc().

KiReleaseNmiListLock()

FORCEINLINE VOID KiReleaseNmiListLock

(

IN KIRQL

OldIrql

)

Definition at line 1679 of file ke_x.h.

{
    KeReleaseSpinLock(&KiNmiCallbackListLock, OldIrql);
}

Referenced by KeDeregisterNmiCallback(), and KeRegisterNmiCallback().

KiReleasePrcbLock()

FORCEINLINE VOID KiReleasePrcbLock

(

IN PKPRCB

Prcb

)

Definition at line 225 of file ke_x.h.

{
    UNREFERENCED_PARAMETER(Prcb);
}

Referenced by KeRevertToUserAffinityThread(), KeSetSystemAffinityThread(), KiAdjustQuantumThread(), KiDeferredReadyThread(), KiInitializeKernel(), KiQuantumEnd(), KiScanReadyQueues(), KiSetPriorityThread(), KiSwapThread(), KiSystemStartupBootStack(), KxQueueReadyThread(), and NtYieldExecution().

KiReleaseProcessLock()

FORCEINLINE VOID KiReleaseProcessLock

(

IN PKLOCK_QUEUE_HANDLE

Handle

)

Definition at line 655 of file ke_x.h.

{
    /* Release the lock and restore previous IRQL */
    KeReleaseInStackQueuedSpinLock(Handle);
}

Referenced by KeFreezeAllThreads(), KeQueryRuntimeProcess(), KeQueryValuesProcess(), KeSetQuantumProcess(), KeStartThread(), and MmSetExecuteOptions().

KiReleaseProcessLockFromSynchLevel()

FORCEINLINE VOID KiReleaseProcessLockFromSynchLevel

(

IN PKLOCK_QUEUE_HANDLE

Handle

)

Definition at line 663 of file ke_x.h.

{
    /* Release the lock without lowering IRQL */
    KeReleaseInStackQueuedSpinLockFromDpcLevel(Handle);
}

Referenced by KeFreezeAllThreads(), KeSetAffinityProcess(), KeSetPriorityAndQuantumProcess(), KeTerminateThread(), and KeThawAllThreads().

KiReleaseThreadLock()

FORCEINLINE VOID KiReleaseThreadLock

(

IN PKTHREAD

Thread

)

Definition at line 245 of file ke_x.h.

{
    UNREFERENCED_PARAMETER(Thread);
}

Referenced by ExpBoostOwnerThread(), KeBoostPriorityThread(), KeQueryBasePriorityThread(), KeSetBasePriorityThread(), KeSetPriorityAndQuantumProcess(), KeSetPriorityThread(), KeSignalGateBoostPriority(), KeWaitForGate(), KiAdjustQuantumThread(), KiDeferredReadyThread(), KiInsertQueueApc(), KiQuantumEnd(), and NtYieldExecution().

KiReleaseTimerLock()

FORCEINLINE VOID KiReleaseTimerLock

(

IN PKSPIN_LOCK_QUEUE

LockQueue

)

Definition at line 292 of file ke_x.h.

{
    ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL);

    /* Nothing to do on UP */
    UNREFERENCED_PARAMETER(LockQueue);
}

Referenced by KeSetSystemTime(), KiCompleteTimer(), KiInsertTreeTimer(), KiTimerExpiration(), KxInsertTimer(), and KxRemoveTreeTimer().

KiRemoveEntryTimer()

FORCEINLINE VOID KiRemoveEntryTimer

(

IN PKTIMER

Timer

)

Definition at line 887 of file ke_x.h.

{
    ULONG Hand;
    PKTIMER_TABLE_ENTRY TableEntry;

    /* Remove the timer from the timer list and check if it's empty */
    Hand = Timer->Header.Hand;
    if (RemoveEntryList(&Timer->TimerListEntry))
    {
        /* Get the respective timer table entry */
        TableEntry = &KiTimerTableListHead[Hand];
        if (&TableEntry->Entry == TableEntry->Entry.Flink)
        {
            /* Set the entry to an infinite absolute time */
            TableEntry->Time.HighPart = 0xFFFFFFFF;
        }
    }

    /* Clear the list entries on dbg builds so we can tell the timer is gone */
#if DBG
    Timer->TimerListEntry.Flink = NULL;
    Timer->TimerListEntry.Blink = NULL;
#endif
}

Referenced by KeSetSystemTime(), KiCompleteTimer(), KiInsertTreeTimer(), and KiTimerExpiration().

KiRequestApcInterrupt()

FORCEINLINE VOID KiRequestApcInterrupt	(	IN BOOLEAN	NeedApc,
		IN UCHAR	Processor
	)

Definition at line 271 of file ke_x.h.

{
    /* We deliver instantly on UP */
    UNREFERENCED_PARAMETER(NeedApc);
    UNREFERENCED_PARAMETER(Processor);
}

Referenced by KiInsertQueueApc().

KiRescheduleThread()

FORCEINLINE VOID KiRescheduleThread	(	IN BOOLEAN	NewThread,
		IN ULONG	Cpu
	)

Definition at line 192 of file ke_x.h.

{
    /* This is meaningless on UP systems */
    UNREFERENCED_PARAMETER(NewThread);
    UNREFERENCED_PARAMETER(Cpu);
}

KiSelectReadyThread()

FORCEINLINE PKTHREAD KiSelectReadyThread	(	IN KPRIORITY	Priority,
		IN PKPRCB	Prcb
	)

Definition at line 1412 of file ke_x.h.

{
    ULONG PrioritySet;
    LONG HighPriority;
    PLIST_ENTRY ListEntry;
    PKTHREAD Thread = NULL;

    /* Save the current mask and get the priority set for the CPU */
    PrioritySet = Prcb->ReadySummary >> Priority;
    if (!PrioritySet) goto Quickie;

    /* Get the highest priority possible */
    BitScanReverse((PULONG)&HighPriority, PrioritySet);
    ASSERT((PrioritySet & PRIORITY_MASK(HighPriority)) != 0);
    HighPriority += Priority;

    /* Make sure the list isn't empty at the highest priority */
    ASSERT(IsListEmpty(&Prcb->DispatcherReadyListHead[HighPriority]) == FALSE);

    /* Get the first thread on the list */
    ListEntry = Prcb->DispatcherReadyListHead[HighPriority].Flink;
    Thread = CONTAINING_RECORD(ListEntry, KTHREAD, WaitListEntry);

    /* Make sure this thread is here for a reason */
    ASSERT(HighPriority == Thread->Priority);
    ASSERT(Thread->Affinity & AFFINITY_MASK(Prcb->Number));
    ASSERT(Thread->NextProcessor == Prcb->Number);

    /* Remove it from the list */
    if (RemoveEntryList(&Thread->WaitListEntry))
    {
        /* The list is empty now, reset the ready summary */
        Prcb->ReadySummary ^= PRIORITY_MASK(HighPriority);
    }

    /* Sanity check and return the thread */
Quickie:
    ASSERT((Thread == NULL) ||
           (Thread->BasePriority == 0) ||
           (Thread->Priority != 0));
    return Thread;
}

Referenced by KiAdjustQuantumThread(), KiQuantumEnd(), KiSelectNextThread(), KiSetPriorityThread(), KiSwapThread(), and NtYieldExecution().

KiSetThreadSwapBusy()

FORCEINLINE VOID KiSetThreadSwapBusy

(

IN PKTHREAD

Thread

)

Definition at line 205 of file ke_x.h.

{
    UNREFERENCED_PARAMETER(Thread);
}

Referenced by KeDelayExecutionThread(), KeRemoveQueue(), KeTerminateThread(), KeWaitForGate(), KeWaitForMultipleObjects(), KeWaitForSingleObject(), KiExitDispatcher(), KiQuantumEnd(), and NtYieldExecution().

KiTryThreadLock()

FORCEINLINE BOOLEAN KiTryThreadLock

(

IN PKTHREAD

Thread

)

Definition at line 255 of file ke_x.h.

{
    UNREFERENCED_PARAMETER(Thread);
    return FALSE;
}

Referenced by KeSignalGateBoostPriority().

KxInsertTimer()

FORCEINLINE VOID KxInsertTimer	(	IN PKTIMER	Timer,
		IN ULONG	Hand
	)

Definition at line 918 of file ke_x.h.

{
    PKSPIN_LOCK_QUEUE LockQueue;
    ASSERT(KeGetCurrentIrql() >= SYNCH_LEVEL);

    /* Acquire the lock and release the dispatcher lock */
    LockQueue = KiAcquireTimerLock(Hand);
    KiReleaseDispatcherLockFromSynchLevel();

    /* Try to insert the timer */
    if (KiInsertTimerTable(Timer, Hand))
    {
        /* Complete it */
        KiCompleteTimer(Timer, LockQueue);
    }
    else
    {
        /* Do nothing, just release the lock */
        KiReleaseTimerLock(LockQueue);
    }
}

Referenced by KeDelayExecutionThread(), KeRemoveQueue(), KeSetTimerEx(), KeWaitForMultipleObjects(), and KeWaitForSingleObject().

KxQueueReadyThread()

FORCEINLINE VOID KxQueueReadyThread	(	IN PKTHREAD	Thread,
		IN PKPRCB	Prcb
	)

Definition at line 1347 of file ke_x.h.

{
    BOOLEAN Preempted;
    KPRIORITY Priority;

    /* Sanity checks */
    ASSERT(Prcb == KeGetCurrentPrcb());
    ASSERT(Thread->State == Running);
    ASSERT(Thread->NextProcessor == Prcb->Number);

    /* Check if this thread is allowed to run in this CPU */
#ifdef CONFIG_SMP
    if ((Thread->Affinity) & (Prcb->SetMember))
#else
    if (TRUE)
#endif
    {
        /* Set thread ready for execution */
        Thread->State = Ready;

        /* Save current priority and if someone had pre-empted it */
        Priority = Thread->Priority;
        Preempted = Thread->Preempted;

        /* We're not pre-empting now, and set the wait time */
        Thread->Preempted = FALSE;
        Thread->WaitTime = KeTickCount.LowPart;

        /* Sanity check */
        ASSERT((Priority >= 0) && (Priority <= HIGH_PRIORITY));

        /* Insert this thread in the appropriate order */
        Preempted ? InsertHeadList(&Prcb->DispatcherReadyListHead[Priority],
                                   &Thread->WaitListEntry) :
                    InsertTailList(&Prcb->DispatcherReadyListHead[Priority],
                                   &Thread->WaitListEntry);

        /* Update the ready summary */
        Prcb->ReadySummary |= PRIORITY_MASK(Priority);

        /* Sanity check */
        ASSERT(Priority == Thread->Priority);

        /* Release the PRCB lock */
        KiReleasePrcbLock(Prcb);
    }
    else
    {
        /* Otherwise, prepare this thread to be deferred */
        Thread->State = DeferredReady;
        Thread->DeferredProcessor = Prcb->Number;

        /* Release the lock and defer scheduling */
        KiReleasePrcbLock(Prcb);
        KiDeferredReadyThread(Thread);
    }
}

Referenced by KiDispatchInterrupt(), KiDpcInterruptHandler(), KiExitDispatcher(), KiQuantumEnd(), KiQueueReadyThread(), and NtYieldExecution().

KxRemoveTreeTimer()

FORCEINLINE VOID KxRemoveTreeTimer

(

IN PKTIMER

Timer

)

Definition at line 999 of file ke_x.h.

{
    ULONG Hand = Timer->Header.Hand;
    PKSPIN_LOCK_QUEUE LockQueue;
    PKTIMER_TABLE_ENTRY TimerEntry;

    /* Acquire timer lock */
    LockQueue = KiAcquireTimerLock(Hand);

    /* Set the timer as non-inserted */
    Timer->Header.Inserted = FALSE;

    /* Remove it from the timer list */
    if (RemoveEntryList(&Timer->TimerListEntry))
    {
        /* Get the entry and check if it's empty */
        TimerEntry = &KiTimerTableListHead[Hand];
        if (IsListEmpty(&TimerEntry->Entry))
        {
            /* Clear the time then */
            TimerEntry->Time.HighPart = 0xFFFFFFFF;
        }
    }

    /* Release the timer lock */
    KiReleaseTimerLock(LockQueue);
}

Referenced by KeCancelTimer(), KeSetTimerEx(), KiInsertQueue(), and KiUnlinkThread().

KxSetTimerForThreadWait()

FORCEINLINE VOID KxSetTimerForThreadWait	(	IN PKTIMER	Timer,
		IN LARGE_INTEGER	Interval,
		OUT PULONG	Hand
	)

Definition at line 1029 of file ke_x.h.

{
    ULONGLONG DueTime;
    LARGE_INTEGER InterruptTime, SystemTime, TimeDifference;

    /* Check the timer's interval to see if it's absolute */
    Timer->Header.Absolute = FALSE;
    if (Interval.HighPart >= 0)
    {
        /* Get the system time and calculate the relative time */
        KeQuerySystemTime(&SystemTime);
        TimeDifference.QuadPart = SystemTime.QuadPart - Interval.QuadPart;
        Timer->Header.Absolute = TRUE;

        /* Check if we've already expired */
        if (TimeDifference.HighPart >= 0)
        {
            /* Reset everything */
            Timer->DueTime.QuadPart = 0;
            *Hand = 0;
            Timer->Header.Hand = 0;
            return;
        }
        else
        {
            /* Update the interval */
            Interval = TimeDifference;
        }
    }

    /* Calculate the due time */
    InterruptTime.QuadPart = KeQueryInterruptTime();
    DueTime = InterruptTime.QuadPart - Interval.QuadPart;
    Timer->DueTime.QuadPart = DueTime;

    /* Calculate the timer handle */
    *Hand = KiComputeTimerTableIndex(DueTime);
    Timer->Header.Hand = (UCHAR)*Hand;
}

KxUnwaitThread()

FORCEINLINE VOID KxUnwaitThread	(	IN DISPATCHER_HEADER *	Object,
		IN KPRIORITY	Increment
	)

Definition at line 1254 of file ke_x.h.

{
    PLIST_ENTRY WaitEntry, WaitList;
    PKWAIT_BLOCK WaitBlock;
    PKTHREAD WaitThread;
    ULONG WaitKey;

    /* Loop the Wait Entries */
    WaitList = &Object->WaitListHead;
    ASSERT(IsListEmpty(&Object->WaitListHead) == FALSE);
    WaitEntry = WaitList->Flink;
    do
    {
        /* Get the current wait block */
        WaitBlock = CONTAINING_RECORD(WaitEntry, KWAIT_BLOCK, WaitListEntry);

        /* Get the waiting thread */
        WaitThread = WaitBlock->Thread;

        /* Check the current Wait Mode */
        if (WaitBlock->WaitType == WaitAny)
        {
            /* Use the actual wait key */
            WaitKey = WaitBlock->WaitKey;
        }
        else
        {
            /* Otherwise, use STATUS_KERNEL_APC */
            WaitKey = STATUS_KERNEL_APC;
        }

        /* Unwait the thread */
        KiUnwaitThread(WaitThread, WaitKey, Increment);

        /* Next entry */
        WaitEntry = WaitList->Flink;
    } while (WaitEntry != WaitList);
}

Referenced by KeSetEvent(), KeSetProcess(), KeTerminateThread(), KiSignalTimer(), KiTimerExpiration(), and KiTimerListExpire().

KxUnwaitThreadForEvent()

FORCEINLINE VOID KxUnwaitThreadForEvent	(	IN PKEVENT	Event,
		IN KPRIORITY	Increment
	)

Definition at line 1299 of file ke_x.h.

{
    PLIST_ENTRY WaitEntry, WaitList;
    PKWAIT_BLOCK WaitBlock;
    PKTHREAD WaitThread;

    /* Loop the Wait Entries */
    WaitList = &Event->Header.WaitListHead;
    ASSERT(IsListEmpty(&Event->Header.WaitListHead) == FALSE);
    WaitEntry = WaitList->Flink;
    do
    {
        /* Get the current wait block */
        WaitBlock = CONTAINING_RECORD(WaitEntry, KWAIT_BLOCK, WaitListEntry);

        /* Get the waiting thread */
        WaitThread = WaitBlock->Thread;

        /* Check the current Wait Mode */
        if (WaitBlock->WaitType == WaitAny)
        {
            /* Un-signal it */
            Event->Header.SignalState = 0;

            /* Un-signal the event and unwait the thread */
            KiUnwaitThread(WaitThread, WaitBlock->WaitKey, Increment);
            break;
        }

        /* Unwait the thread with STATUS_KERNEL_APC */
        KiUnwaitThread(WaitThread, STATUS_KERNEL_APC, Increment);

        /* Next entry */
        WaitEntry = WaitList->Flink;
    } while (WaitEntry != WaitList);
}

Referenced by KeSetEvent(), KeSetEventBoostPriority(), KiSignalTimer(), KiTimerExpiration(), and KiTimerListExpire().