ReactOS 0.4.15-dev-5863-g1fe3ab7
ke_x.h
Go to the documentation of this file.
1/*
2* PROJECT: ReactOS Kernel
3* LICENSE: GPL - See COPYING in the top level directory
4* FILE: ntoskrnl/include/internal/ke_x.h
5* PURPOSE: Internal Inlined Functions for the Kernel
6* PROGRAMMERS: Alex Ionescu (alex.ionescu@reactos.org)
7*/
8
9#ifdef __cplusplus
10extern "C"
11{
12#endif
13
14#ifndef _M_ARM
18{
19 /* Return the current mode */
20 return KeGetCurrentThread()->PreviousMode;
21}
22#endif
23
24//
25// Enters a Guarded Region
26//
27#define KeEnterGuardedRegionThread(_Thread) \
28{ \
29 /* Sanity checks */ \
30 ASSERT(KeGetCurrentIrql() <= APC_LEVEL); \
31 ASSERT((_Thread) == KeGetCurrentThread()); \
32 ASSERT(((_Thread)->SpecialApcDisable <= 0) && \
33 ((_Thread)->SpecialApcDisable != -32768)); \
34 \
35 /* Disable Special APCs */ \
36 (_Thread)->SpecialApcDisable--; \
37}
38
39#define KeEnterGuardedRegion() \
40{ \
41 PKTHREAD _Thread = KeGetCurrentThread(); \
42 KeEnterGuardedRegionThread(_Thread); \
43}
44
45//
46// Leaves a Guarded Region
47//
48#define KeLeaveGuardedRegionThread(_Thread) \
49{ \
50 /* Sanity checks */ \
51 ASSERT(KeGetCurrentIrql() <= APC_LEVEL); \
52 ASSERT((_Thread) == KeGetCurrentThread()); \
53 ASSERT((_Thread)->SpecialApcDisable < 0); \
54 \
55 /* Leave region and check if APCs are OK now */ \
56 if (!(++(_Thread)->SpecialApcDisable)) \
57 { \
58 /* Check for Kernel APCs on the list */ \
59 if (!IsListEmpty(&(_Thread)->ApcState. \
60 ApcListHead[KernelMode])) \
61 { \
62 /* Check for APC Delivery */ \
63 KiCheckForKernelApcDelivery(); \
64 } \
65 } \
66}
67
68#define KeLeaveGuardedRegion() \
69{ \
70 PKTHREAD _Thread = KeGetCurrentThread(); \
71 KeLeaveGuardedRegionThread(_Thread); \
72}
73
74//
75// Enters a Critical Region
76//
77#define KeEnterCriticalRegionThread(_Thread) \
78{ \
79 /* Sanity checks */ \
80 ASSERT((_Thread) == KeGetCurrentThread()); \
81 ASSERT(((_Thread)->KernelApcDisable <= 0) && \
82 ((_Thread)->KernelApcDisable != -32768)); \
83 \
84 /* Disable Kernel APCs */ \
85 (_Thread)->KernelApcDisable--; \
86}
87
88#define KeEnterCriticalRegion() \
89{ \
90 PKTHREAD _Thread = KeGetCurrentThread(); \
91 KeEnterCriticalRegionThread(_Thread); \
92}
93
94//
95// Leaves a Critical Region
96//
97#define KeLeaveCriticalRegionThread(_Thread) \
98{ \
99 /* Sanity checks */ \
100 ASSERT((_Thread) == KeGetCurrentThread()); \
101 ASSERT((_Thread)->KernelApcDisable < 0); \
102 \
103 /* Enable Kernel APCs */ \
104 (_Thread)->KernelApcDisable++; \
105 \
106 /* Check if Kernel APCs are now enabled */ \
107 if (!((_Thread)->KernelApcDisable)) \
108 { \
109 /* Check if we need to request an APC Delivery */ \
110 if (!(IsListEmpty(&(_Thread)->ApcState.ApcListHead[KernelMode])) && \
111 !((_Thread)->SpecialApcDisable)) \
112 { \
113 /* Check for the right environment */ \
114 KiCheckForKernelApcDelivery(); \
115 } \
116 } \
117}
118
119#define KeLeaveCriticalRegion() \
120{ \
121 PKTHREAD _Thread = KeGetCurrentThread(); \
122 KeLeaveCriticalRegionThread(_Thread); \
123}
124
125#ifndef CONFIG_SMP
126
127//
128// This routine protects against multiple CPU acquires, it's meaningless on UP.
129//
131VOID
133{
135}
136
137//
138// This routine protects against multiple CPU acquires, it's meaningless on UP.
139//
141VOID
143{
145}
146
148KIRQL
150{
151 /* Raise to synch level */
152 return KfRaiseIrql(SYNCH_LEVEL);
153}
154
156VOID
158{
159 /* Just exit the dispatcher */
161}
162
164VOID
166{
167 /* This is a no-op at SYNCH_LEVEL for UP systems */
169 return;
170}
171
173VOID
175{
176 /* This is a no-op at SYNCH_LEVEL for UP systems */
177 return;
178}
179
180//
181// This routine makes the thread deferred ready on the boot CPU.
182//
184VOID
186{
187 /* Set the thread to deferred state and boot CPU */
188 Thread->State = DeferredReady;
189 Thread->DeferredProcessor = 0;
190
191 /* Make the thread ready immediately */
193}
194
196VOID
198 IN ULONG Cpu)
199{
200 /* This is meaningless on UP systems */
201 UNREFERENCED_PARAMETER(NewThread);
203}
204
205//
206// This routine protects against multiple CPU acquires, it's meaningless on UP.
207//
209VOID
211{
213}
214
215//
216// This routine protects against multiple CPU acquires, it's meaningless on UP.
217//
219VOID
221{
223}
224
225//
226// This routine protects against multiple CPU acquires, it's meaningless on UP.
227//
229VOID
231{
233}
234
235//
236// This routine protects against multiple CPU acquires, it's meaningless on UP.
237//
239VOID
241{
243}
244
245//
246// This routine protects against multiple CPU acquires, it's meaningless on UP.
247//
249VOID
251{
253}
254
255//
256// This routine protects against multiple CPU acquires, it's meaningless on UP.
257//
261{
263 return FALSE;
264}
265
267VOID
269{
270 /* There are no deferred ready lists on UP systems */
272}
273
275VOID
278{
279 /* We deliver instantly on UP */
280 UNREFERENCED_PARAMETER(NeedApc);
282}
283
287{
289
290 /* Nothing to do on UP */
292 return NULL;
293}
294
296VOID
298{
300
301 /* Nothing to do on UP */
302 UNREFERENCED_PARAMETER(LockQueue);
303}
304
305#else
306
308VOID
310{
311 LONG OldValue;
312
313 /* Make sure we're at a safe level to touch the lock */
315
316 /* Start acquire loop */
317 do
318 {
319 /* Loop until the other CPU releases it */
320 while (TRUE)
321 {
322 /* Check if it got released */
323 OldValue = Object->Lock;
324 if ((OldValue & KOBJECT_LOCK_BIT) == 0) break;
325
326 /* Let the CPU know that this is a loop */
328 }
329
330 /* Try acquiring the lock now */
331 } while (InterlockedCompareExchange(&Object->Lock,
332 OldValue | KOBJECT_LOCK_BIT,
333 OldValue) != OldValue);
334}
335
337VOID
339{
340 /* Make sure we're at a safe level to touch the lock */
342
343 /* Release it */
345}
346
348KIRQL
350{
351 /* Raise to synchronization level and acquire the dispatcher lock */
353}
354
356VOID
358{
359 /* First release the lock */
360 KeReleaseQueuedSpinLockFromDpcLevel(&KeGetCurrentPrcb()->
361 LockQueue[LockQueueDispatcherLock]);
362
363 /* Then exit the dispatcher */
365}
366
368VOID
370{
371 /* Acquire the dispatcher lock */
373 KeAcquireQueuedSpinLockAtDpcLevel(&KeGetCurrentPrcb()->
374 LockQueue[LockQueueDispatcherLock]);
375}
376
378VOID
380{
381 /* Release the dispatcher lock */
382 KeReleaseQueuedSpinLockFromDpcLevel(&KeGetCurrentPrcb()->
383 LockQueue[LockQueueDispatcherLock]);
384}
385
386//
387// This routine inserts a thread into the deferred ready list of the current CPU
388//
390VOID
392{
393 PKPRCB Prcb = KeGetCurrentPrcb();
394
395 /* Set the thread to deferred state and CPU */
396 Thread->State = DeferredReady;
397 Thread->DeferredProcessor = Prcb->Number;
398
399 /* Add it on the list */
400 PushEntryList(&Prcb->DeferredReadyListHead, &Thread->SwapListEntry);
401}
402
404VOID
406 IN ULONG Cpu)
407{
408 /* Check if a new thread needs to be scheduled on a different CPU */
409 if ((NewThread) && !(KeGetCurrentPrcb()->Number == Cpu))
410 {
411 /* Send an IPI to request delivery */
413 }
414}
415
416//
417// This routine sets the current thread in a swap busy state, which ensure that
418// nobody else tries to swap it concurrently.
419//
421VOID
423{
424 /* Make sure nobody already set it */
425 ASSERT(Thread->SwapBusy == FALSE);
426
427 /* Set it ourselves */
428 Thread->SwapBusy = TRUE;
429}
430
431//
432// This routine acquires the PRCB lock so that only one caller can touch
433// volatile PRCB data.
434//
435// Since this is a simple optimized spin-lock, it must only be acquired
436// at dispatcher level or higher!
437//
439VOID
441{
442 /* Make sure we're at a safe level to touch the PRCB lock */
444
445 /* Start acquire loop */
446 for (;;)
447 {
448 /* Acquire the lock and break out if we acquired it first */
449 if (!InterlockedExchange((PLONG)&Prcb->PrcbLock, 1)) break;
450
451 /* Loop until the other CPU releases it */
452 do
453 {
454 /* Let the CPU know that this is a loop */
456 } while (Prcb->PrcbLock);
457 }
458}
459
460//
461// This routine releases the PRCB lock so that other callers can touch
462// volatile PRCB data.
463//
464// Since this is a simple optimized spin-lock, it must be be only acquired
465// at dispatcher level or higher!
466//
468VOID
470{
471 /* Make sure we are above dispatch and the lock is acquired! */
473 ASSERT(Prcb->PrcbLock != 0);
474
475 /* Release it */
476 InterlockedAnd((PLONG)&Prcb->PrcbLock, 0);
477}
478
479//
480// This routine acquires the thread lock so that only one caller can touch
481// volatile thread data.
482//
483// Since this is a simple optimized spin-lock, it must be be only acquired
484// at dispatcher level or higher!
485//
487VOID
489{
490 /* Make sure we're at a safe level to touch the thread lock */
492
493 /* Start acquire loop */
494 for (;;)
495 {
496 /* Acquire the lock and break out if we acquired it first */
497 if (!InterlockedExchange((PLONG)&Thread->ThreadLock, 1)) break;
498
499 /* Loop until the other CPU releases it */
500 do
501 {
502 /* Let the CPU know that this is a loop */
504 } while (Thread->ThreadLock);
505 }
506}
507
508//
509// This routine releases the thread lock so that other callers can touch
510// volatile thread data.
511//
512// Since this is a simple optimized spin-lock, it must be be only acquired
513// at dispatcher level or higher!
514//
516VOID
518{
519 /* Make sure we are still above dispatch */
521
522 /* Release it */
524}
525
529{
530 LONG Value;
531
532 /* If the lock isn't acquired, return false */
533 if (!Thread->ThreadLock) return FALSE;
534
535 /* Otherwise, try to acquire it and check the result */
536 Value = 1;
538
539 /* Return the lock state */
540 return (Value == 1);
541}
542
544VOID
546{
547 /* Scan the deferred ready lists if required */
548 if (Prcb->DeferredReadyListHead.Next) KiProcessDeferredReadyList(Prcb);
549}
550
552VOID
555{
556 /* Check if we need to request APC delivery */
557 if (NeedApc)
558 {
559 /* Check if it's on another CPU */
561 {
562 /* Send an IPI to request delivery */
564 }
565 else
566 {
567 /* Request a software interrupt */
569 }
570 }
571}
572
576{
577 PKSPIN_LOCK_QUEUE LockQueue;
578 ULONG LockIndex;
580
581 /* Get the lock index */
582 LockIndex = Hand >> LOCK_QUEUE_TIMER_LOCK_SHIFT;
583 LockIndex &= (LOCK_QUEUE_TIMER_TABLE_LOCKS - 1);
584
585 /* Now get the lock */
586 LockQueue = &KeGetCurrentPrcb()->LockQueue[LockQueueTimerTableLock + LockIndex];
587
588 /* Acquire it and return */
589 KeAcquireQueuedSpinLockAtDpcLevel(LockQueue);
590 return LockQueue;
591}
592
594VOID
596{
598
599 /* Release the lock */
600 KeReleaseQueuedSpinLockFromDpcLevel(LockQueue);
601}
602
603#endif
604
606VOID
609{
610 /* Acquire the lock and raise to synchronization level */
612}
613
615VOID
618{
619 /* Acquire the lock */
622}
623
625VOID
628{
629 /* Acquire the lock */
631}
632
634VOID
636{
637 /* Release the lock */
639}
640
642VOID
644{
645 /* Release the lock */
647}
648
650VOID
653{
654 /* Acquire the lock and raise to synchronization level */
656}
657
659VOID
661{
662 /* Release the lock and restore previous IRQL */
664}
665
667VOID
669{
670 /* Release the lock without lowering IRQL */
672}
673
675VOID
677 IN PKLOCK_QUEUE_HANDLE DeviceLock)
678{
679 /* Check if we were called from a threaded DPC */
680 if (KeGetCurrentPrcb()->DpcThreadActive)
681 {
682 /* Lock the Queue, we're not at DPC level */
683 KeAcquireInStackQueuedSpinLock(&DeviceQueue->Lock, DeviceLock);
684 }
685 else
686 {
687 /* We must be at DPC level, acquire the lock safely */
690 DeviceLock);
691 }
692}
693
695VOID
697{
698 /* Check if we were called from a threaded DPC */
699 if (KeGetCurrentPrcb()->DpcThreadActive)
700 {
701 /* Unlock the Queue, we're not at DPC level */
703 }
704 else
705 {
706 /* We must be at DPC level, release the lock safely */
709 }
710}
711
712//
713// Satisfies the wait of a mutant dispatcher object
714//
715#define KiSatisfyMutantWait(Object, Thread) \
716{ \
717 /* Decrease the Signal State */ \
718 (Object)->Header.SignalState--; \
719 \
720 /* Check if it's now non-signaled */ \
721 if (!(Object)->Header.SignalState) \
722 { \
723 /* Set the Owner Thread */ \
724 (Object)->OwnerThread = Thread; \
725 \
726 /* Disable APCs if needed */ \
727 Thread->KernelApcDisable = Thread->KernelApcDisable - \
728 (Object)->ApcDisable; \
729 \
730 /* Check if it's abandoned */ \
731 if ((Object)->Abandoned) \
732 { \
733 /* Unabandon it */ \
734 (Object)->Abandoned = FALSE; \
735 \
736 /* Return Status */ \
737 Thread->WaitStatus = STATUS_ABANDONED; \
738 } \
739 \
740 /* Insert it into the Mutant List */ \
741 InsertHeadList(Thread->MutantListHead.Blink, \
742 &(Object)->MutantListEntry); \
743 } \
744}
745
746//
747// Satisfies the wait of any nonmutant dispatcher object
748//
749#define KiSatisfyNonMutantWait(Object) \
750{ \
751 if (((Object)->Header.Type & TIMER_OR_EVENT_TYPE) == \
752 EventSynchronizationObject) \
753 { \
754 /* Synchronization Timers and Events just get un-signaled */ \
755 (Object)->Header.SignalState = 0; \
756 } \
757 else if ((Object)->Header.Type == SemaphoreObject) \
758 { \
759 /* These ones can have multiple states, so we only decrease it */ \
760 (Object)->Header.SignalState--; \
761 } \
762}
763
764//
765// Satisfies the wait of any dispatcher object
766//
767#define KiSatisfyObjectWait(Object, Thread) \
768{ \
769 /* Special case for Mutants */ \
770 if ((Object)->Header.Type == MutantObject) \
771 { \
772 KiSatisfyMutantWait((Object), (Thread)); \
773 } \
774 else \
775 { \
776 KiSatisfyNonMutantWait(Object); \
777 } \
778}
779
780//
781// Recalculates the due time
782//
787 IN OUT PLARGE_INTEGER NewDueTime)
788{
789 /* Don't do anything for absolute waits */
790 if (OriginalDueTime->QuadPart >= 0) return OriginalDueTime;
791
792 /* Otherwise, query the interrupt time and recalculate */
793 NewDueTime->QuadPart = KeQueryInterruptTime();
794 NewDueTime->QuadPart -= DueTime->QuadPart;
795 return NewDueTime;
796}
797
798//
799// Determines whether a thread should be added to the wait list
800//
804 IN KPROCESSOR_MODE WaitMode)
805{
806 /* Check the required conditions */
807 if ((WaitMode != KernelMode) &&
808 (Thread->EnableStackSwap) &&
809 (Thread->Priority >= (LOW_REALTIME_PRIORITY + 9)))
810 {
811 /* We are go for swap */
812 return TRUE;
813 }
814 else
815 {
816 /* Don't swap the thread */
817 return FALSE;
818 }
819}
820
821//
822// Adds a thread to the wait list
823//
824#define KiAddThreadToWaitList(Thread, Swappable) \
825{ \
826 /* Make sure it's swappable */ \
827 if (Swappable) \
828 { \
829 /* Insert it into the PRCB's List */ \
830 InsertTailList(&KeGetCurrentPrcb()->WaitListHead, \
831 &Thread->WaitListEntry); \
832 } \
833}
834
835//
836// Checks if a wait in progress should be interrupted by APCs or an alertable
837// state.
838//
843 IN KPROCESSOR_MODE WaitMode)
844{
845 /* Check if the wait is alertable */
846 if (Alertable)
847 {
848 /* It is, first check if the thread is alerted in this mode */
849 if (Thread->Alerted[WaitMode])
850 {
851 /* It is, so bail out of the wait */
852 Thread->Alerted[WaitMode] = FALSE;
853 return STATUS_ALERTED;
854 }
855 else if ((WaitMode != KernelMode) &&
856 (!IsListEmpty(&Thread->ApcState.ApcListHead[UserMode])))
857 {
858 /* It's isn't, but this is a user wait with queued user APCs */
859 Thread->ApcState.UserApcPending = TRUE;
860 return STATUS_USER_APC;
861 }
862 else if (Thread->Alerted[KernelMode])
863 {
864 /* It isn't that either, but we're alered in kernel mode */
865 Thread->Alerted[KernelMode] = FALSE;
866 return STATUS_ALERTED;
867 }
868 }
869 else if ((WaitMode != KernelMode) && (Thread->ApcState.UserApcPending))
870 {
871 /* Not alertable, but this is a user wait with pending user APCs */
872 return STATUS_USER_APC;
873 }
874
875 /* Otherwise, we're fine */
876 return STATUS_WAIT_0;
877}
878
880ULONG
882{
884}
885
886//
887// Called from KiCompleteTimer, KiInsertTreeTimer, KeSetSystemTime
888// to remove timer entries
889// See Windows HPI blog for more information.
891VOID
893{
894 ULONG Hand;
896
897 /* Remove the timer from the timer list and check if it's empty */
898 Hand = Timer->Header.Hand;
899 if (RemoveEntryList(&Timer->TimerListEntry))
900 {
901 /* Get the respective timer table entry */
903 if (&TableEntry->Entry == TableEntry->Entry.Flink)
904 {
905 /* Set the entry to an infinite absolute time */
906 TableEntry->Time.HighPart = 0xFFFFFFFF;
907 }
908 }
909
910 /* Clear the list entries on dbg builds so we can tell the timer is gone */
911#if DBG
912 Timer->TimerListEntry.Flink = NULL;
913 Timer->TimerListEntry.Blink = NULL;
914#endif
915}
916
917//
918// Called by Wait and Queue code to insert a timer for dispatching.
919// Also called by KeSetTimerEx to insert a timer from the caller.
920//
922VOID
924 IN ULONG Hand)
925{
926 PKSPIN_LOCK_QUEUE LockQueue;
928
929 /* Acquire the lock and release the dispatcher lock */
930 LockQueue = KiAcquireTimerLock(Hand);
932
933 /* Try to insert the timer */
934 if (KiInsertTimerTable(Timer, Hand))
935 {
936 /* Complete it */
937 KiCompleteTimer(Timer, LockQueue);
938 }
939 else
940 {
941 /* Do nothing, just release the lock */
942 KiReleaseTimerLock(LockQueue);
943 }
944}
945
946//
947// Called by KeSetTimerEx and KiInsertTreeTimer to calculate Due Time
948// See the Windows HPI Blog for more information
949//
954 OUT PULONG Hand)
955{
956 LARGE_INTEGER InterruptTime, SystemTime, DifferenceTime;
957
958 /* Convert to relative time if needed */
959 Timer->Header.Absolute = FALSE;
960 if (DueTime.HighPart >= 0)
961 {
962 /* Get System Time */
963 KeQuerySystemTime(&SystemTime);
964
965 /* Do the conversion */
966 DifferenceTime.QuadPart = SystemTime.QuadPart - DueTime.QuadPart;
967
968 /* Make sure it hasn't already expired */
969 Timer->Header.Absolute = TRUE;
970 if (DifferenceTime.HighPart >= 0)
971 {
972 /* Cancel everything */
973 Timer->Header.SignalState = TRUE;
974 Timer->Header.Hand = 0;
975 Timer->DueTime.QuadPart = 0;
976 *Hand = 0;
977 return FALSE;
978 }
979
980 /* Set the time as Absolute */
981 DueTime = DifferenceTime;
982 }
983
984 /* Get the Interrupt Time */
985 InterruptTime.QuadPart = KeQueryInterruptTime();
986
987 /* Recalculate due time */
988 Timer->DueTime.QuadPart = InterruptTime.QuadPart - DueTime.QuadPart;
989
990 /* Get the handle */
991 *Hand = KiComputeTimerTableIndex(Timer->DueTime.QuadPart);
992 Timer->Header.Hand = (UCHAR)*Hand;
993 Timer->Header.Inserted = TRUE;
994 return TRUE;
995}
996
997//
998// Called from Unlink and Queue Insert Code.
999// Also called by timer code when canceling an inserted timer.
1000// Removes a timer from it's tree.
1001//
1003VOID
1005{
1006 ULONG Hand = Timer->Header.Hand;
1007 PKSPIN_LOCK_QUEUE LockQueue;
1008 PKTIMER_TABLE_ENTRY TimerEntry;
1009
1010 /* Acquire timer lock */
1011 LockQueue = KiAcquireTimerLock(Hand);
1012
1013 /* Set the timer as non-inserted */
1014 Timer->Header.Inserted = FALSE;
1015
1016 /* Remove it from the timer list */
1017 if (RemoveEntryList(&Timer->TimerListEntry))
1018 {
1019 /* Get the entry and check if it's empty */
1020 TimerEntry = &KiTimerTableListHead[Hand];
1021 if (IsListEmpty(&TimerEntry->Entry))
1022 {
1023 /* Clear the time then */
1024 TimerEntry->Time.HighPart = 0xFFFFFFFF;
1025 }
1026 }
1027
1028 /* Release the timer lock */
1029 KiReleaseTimerLock(LockQueue);
1030}
1031
1033VOID
1036 OUT PULONG Hand)
1037{
1039 LARGE_INTEGER InterruptTime, SystemTime, TimeDifference;
1040
1041 /* Check the timer's interval to see if it's absolute */
1042 Timer->Header.Absolute = FALSE;
1043 if (Interval.HighPart >= 0)
1044 {
1045 /* Get the system time and calculate the relative time */
1046 KeQuerySystemTime(&SystemTime);
1047 TimeDifference.QuadPart = SystemTime.QuadPart - Interval.QuadPart;
1048 Timer->Header.Absolute = TRUE;
1049
1050 /* Check if we've already expired */
1051 if (TimeDifference.HighPart >= 0)
1052 {
1053 /* Reset everything */
1054 Timer->DueTime.QuadPart = 0;
1055 *Hand = 0;
1056 Timer->Header.Hand = 0;
1057 return;
1058 }
1059 else
1060 {
1061 /* Update the interval */
1062 Interval = TimeDifference;
1063 }
1064 }
1065
1066 /* Calculate the due time */
1067 InterruptTime.QuadPart = KeQueryInterruptTime();
1068 DueTime = InterruptTime.QuadPart - Interval.QuadPart;
1069 Timer->DueTime.QuadPart = DueTime;
1070
1071 /* Calculate the timer handle */
1073 Timer->Header.Hand = (UCHAR)*Hand;
1074}
1075
1076#define KxDelayThreadWait() \
1077 \
1078 /* Setup the Wait Block */ \
1079 Thread->WaitBlockList = TimerBlock; \
1080 \
1081 /* Setup the timer */ \
1082 KxSetTimerForThreadWait(Timer, *Interval, &Hand); \
1083 \
1084 /* Save the due time for the caller */ \
1085 DueTime.QuadPart = Timer->DueTime.QuadPart; \
1086 \
1087 /* Link the timer to this Wait Block */ \
1088 TimerBlock->NextWaitBlock = TimerBlock; \
1089 Timer->Header.WaitListHead.Flink = &TimerBlock->WaitListEntry; \
1090 Timer->Header.WaitListHead.Blink = &TimerBlock->WaitListEntry; \
1091 \
1092 /* Clear wait status */ \
1093 Thread->WaitStatus = STATUS_SUCCESS; \
1094 \
1095 /* Setup wait fields */ \
1096 Thread->Alertable = Alertable; \
1097 Thread->WaitReason = DelayExecution; \
1098 Thread->WaitMode = WaitMode; \
1099 \
1100 /* Check if we can swap the thread's stack */ \
1101 Thread->WaitListEntry.Flink = NULL; \
1102 Swappable = KiCheckThreadStackSwap(Thread, WaitMode); \
1103 \
1104 /* Set the wait time */ \
1105 Thread->WaitTime = KeTickCount.LowPart;
1106
1107#define KxMultiThreadWait() \
1108 /* Link wait block array to the thread */ \
1109 Thread->WaitBlockList = WaitBlockArray; \
1110 \
1111 /* Reset the index */ \
1112 Index = 0; \
1113 \
1114 /* Loop wait blocks */ \
1115 do \
1116 { \
1117 /* Fill out the wait block */ \
1118 WaitBlock = &WaitBlockArray[Index]; \
1119 WaitBlock->Object = Object[Index]; \
1120 WaitBlock->WaitKey = (USHORT)Index; \
1121 WaitBlock->WaitType = WaitType; \
1122 WaitBlock->Thread = Thread; \
1123 \
1124 /* Link to next block */ \
1125 WaitBlock->NextWaitBlock = &WaitBlockArray[Index + 1]; \
1126 Index++; \
1127 } while (Index < Count); \
1128 \
1129 /* Link the last block */ \
1130 WaitBlock->NextWaitBlock = WaitBlockArray; \
1131 \
1132 /* Set default wait status */ \
1133 Thread->WaitStatus = STATUS_WAIT_0; \
1134 \
1135 /* Check if we have a timer */ \
1136 if (Timeout) \
1137 { \
1138 /* Link to the block */ \
1139 TimerBlock->NextWaitBlock = WaitBlockArray; \
1140 \
1141 /* Setup the timer */ \
1142 KxSetTimerForThreadWait(Timer, *Timeout, &Hand); \
1143 \
1144 /* Save the due time for the caller */ \
1145 DueTime.QuadPart = Timer->DueTime.QuadPart; \
1146 \
1147 /* Initialize the list */ \
1148 InitializeListHead(&Timer->Header.WaitListHead); \
1149 } \
1150 \
1151 /* Set wait settings */ \
1152 Thread->Alertable = Alertable; \
1153 Thread->WaitMode = WaitMode; \
1154 Thread->WaitReason = WaitReason; \
1155 \
1156 /* Check if we can swap the thread's stack */ \
1157 Thread->WaitListEntry.Flink = NULL; \
1158 Swappable = KiCheckThreadStackSwap(Thread, WaitMode); \
1159 \
1160 /* Set the wait time */ \
1161 Thread->WaitTime = KeTickCount.LowPart;
1162
1163#define KxSingleThreadWait() \
1164 /* Setup the Wait Block */ \
1165 Thread->WaitBlockList = WaitBlock; \
1166 WaitBlock->WaitKey = STATUS_SUCCESS; \
1167 WaitBlock->Object = Object; \
1168 WaitBlock->WaitType = WaitAny; \
1169 \
1170 /* Clear wait status */ \
1171 Thread->WaitStatus = STATUS_SUCCESS; \
1172 \
1173 /* Check if we have a timer */ \
1174 if (Timeout) \
1175 { \
1176 /* Setup the timer */ \
1177 KxSetTimerForThreadWait(Timer, *Timeout, &Hand); \
1178 \
1179 /* Save the due time for the caller */ \
1180 DueTime.QuadPart = Timer->DueTime.QuadPart; \
1181 \
1182 /* Pointer to timer block */ \
1183 WaitBlock->NextWaitBlock = TimerBlock; \
1184 TimerBlock->NextWaitBlock = WaitBlock; \
1185 \
1186 /* Link the timer to this Wait Block */ \
1187 Timer->Header.WaitListHead.Flink = &TimerBlock->WaitListEntry; \
1188 Timer->Header.WaitListHead.Blink = &TimerBlock->WaitListEntry; \
1189 } \
1190 else \
1191 { \
1192 /* No timer block, just ourselves */ \
1193 WaitBlock->NextWaitBlock = WaitBlock; \
1194 } \
1195 \
1196 /* Set wait settings */ \
1197 Thread->Alertable = Alertable; \
1198 Thread->WaitMode = WaitMode; \
1199 Thread->WaitReason = WaitReason; \
1200 \
1201 /* Check if we can swap the thread's stack */ \
1202 Thread->WaitListEntry.Flink = NULL; \
1203 Swappable = KiCheckThreadStackSwap(Thread, WaitMode); \
1204 \
1205 /* Set the wait time */ \
1206 Thread->WaitTime = KeTickCount.LowPart;
1207
1208#define KxQueueThreadWait() \
1209 /* Setup the Wait Block */ \
1210 Thread->WaitBlockList = WaitBlock; \
1211 WaitBlock->WaitKey = STATUS_SUCCESS; \
1212 WaitBlock->Object = Queue; \
1213 WaitBlock->WaitType = WaitAny; \
1214 WaitBlock->Thread = Thread; \
1215 \
1216 /* Clear wait status */ \
1217 Thread->WaitStatus = STATUS_SUCCESS; \
1218 \
1219 /* Check if we have a timer */ \
1220 if (Timeout) \
1221 { \
1222 /* Setup the timer */ \
1223 KxSetTimerForThreadWait(Timer, *Timeout, &Hand); \
1224 \
1225 /* Save the due time for the caller */ \
1226 DueTime.QuadPart = Timer->DueTime.QuadPart; \
1227 \
1228 /* Pointer to timer block */ \
1229 WaitBlock->NextWaitBlock = TimerBlock; \
1230 TimerBlock->NextWaitBlock = WaitBlock; \
1231 \
1232 /* Link the timer to this Wait Block */ \
1233 Timer->Header.WaitListHead.Flink = &TimerBlock->WaitListEntry; \
1234 Timer->Header.WaitListHead.Blink = &TimerBlock->WaitListEntry; \
1235 } \
1236 else \
1237 { \
1238 /* No timer block, just ourselves */ \
1239 WaitBlock->NextWaitBlock = WaitBlock; \
1240 } \
1241 \
1242 /* Set wait settings */ \
1243 Thread->Alertable = FALSE; \
1244 Thread->WaitMode = WaitMode; \
1245 Thread->WaitReason = WrQueue; \
1246 \
1247 /* Check if we can swap the thread's stack */ \
1248 Thread->WaitListEntry.Flink = NULL; \
1249 Swappable = KiCheckThreadStackSwap(Thread, WaitMode); \
1250 \
1251 /* Set the wait time */ \
1252 Thread->WaitTime = KeTickCount.LowPart;
1253
1254//
1255// Unwaits a Thread
1256//
1258VOID
1261{
1262 PLIST_ENTRY WaitEntry, WaitList;
1263 PKWAIT_BLOCK WaitBlock;
1264 PKTHREAD WaitThread;
1265 ULONG WaitKey;
1266
1267 /* Loop the Wait Entries */
1268 WaitList = &Object->WaitListHead;
1269 ASSERT(IsListEmpty(&Object->WaitListHead) == FALSE);
1270 WaitEntry = WaitList->Flink;
1271 do
1272 {
1273 /* Get the current wait block */
1274 WaitBlock = CONTAINING_RECORD(WaitEntry, KWAIT_BLOCK, WaitListEntry);
1275
1276 /* Get the waiting thread */
1277 WaitThread = WaitBlock->Thread;
1278
1279 /* Check the current Wait Mode */
1280 if (WaitBlock->WaitType == WaitAny)
1281 {
1282 /* Use the actual wait key */
1283 WaitKey = WaitBlock->WaitKey;
1284 }
1285 else
1286 {
1287 /* Otherwise, use STATUS_KERNEL_APC */
1288 WaitKey = STATUS_KERNEL_APC;
1289 }
1290
1291 /* Unwait the thread */
1292 KiUnwaitThread(WaitThread, WaitKey, Increment);
1293
1294 /* Next entry */
1295 WaitEntry = WaitList->Flink;
1296 } while (WaitEntry != WaitList);
1297}
1298
1299//
1300// Unwaits a Thread waiting on an event
1301//
1303VOID
1306{
1307 PLIST_ENTRY WaitEntry, WaitList;
1308 PKWAIT_BLOCK WaitBlock;
1309 PKTHREAD WaitThread;
1310
1311 /* Loop the Wait Entries */
1312 WaitList = &Event->Header.WaitListHead;
1313 ASSERT(IsListEmpty(&Event->Header.WaitListHead) == FALSE);
1314 WaitEntry = WaitList->Flink;
1315 do
1316 {
1317 /* Get the current wait block */
1318 WaitBlock = CONTAINING_RECORD(WaitEntry, KWAIT_BLOCK, WaitListEntry);
1319
1320 /* Get the waiting thread */
1321 WaitThread = WaitBlock->Thread;
1322
1323 /* Check the current Wait Mode */
1324 if (WaitBlock->WaitType == WaitAny)
1325 {
1326 /* Un-signal it */
1327 Event->Header.SignalState = 0;
1328
1329 /* Un-signal the event and unwait the thread */
1330 KiUnwaitThread(WaitThread, WaitBlock->WaitKey, Increment);
1331 break;
1332 }
1333
1334 /* Unwait the thread with STATUS_KERNEL_APC */
1336
1337 /* Next entry */
1338 WaitEntry = WaitList->Flink;
1339 } while (WaitEntry != WaitList);
1340}
1341
1342//
1343// This routine queues a thread that is ready on the PRCB's ready lists.
1344// If this thread cannot currently run on this CPU, then the thread is
1345// added to the deferred ready list instead.
1346//
1347// This routine must be entered with the PRCB lock held and it will exit
1348// with the PRCB lock released!
1349//
1351_Releases_lock_(Prcb->PrcbLock)
1353VOID
1354KxQueueReadyThread(IN PKTHREAD Thread,
1355 IN PKPRCB Prcb)
1356{
1357 BOOLEAN Preempted;
1359
1360 /* Sanity checks */
1361 ASSERT(Prcb == KeGetCurrentPrcb());
1362 ASSERT(Thread->State == Running);
1363 ASSERT(Thread->NextProcessor == Prcb->Number);
1364
1365 /* Check if this thread is allowed to run in this CPU */
1366#ifdef CONFIG_SMP
1367 if ((Thread->Affinity) & (Prcb->SetMember))
1368#else
1369 if (TRUE)
1370#endif
1371 {
1372 /* Set thread ready for execution */
1373 Thread->State = Ready;
1374
1375 /* Save current priority and if someone had pre-empted it */
1376 Priority = Thread->Priority;
1377 Preempted = Thread->Preempted;
1378
1379 /* We're not pre-empting now, and set the wait time */
1380 Thread->Preempted = FALSE;
1381 Thread->WaitTime = KeTickCount.LowPart;
1382
1383 /* Sanity check */
1384 ASSERT((Priority >= 0) && (Priority <= HIGH_PRIORITY));
1385
1386 /* Insert this thread in the appropriate order */
1387 Preempted ? InsertHeadList(&Prcb->DispatcherReadyListHead[Priority],
1388 &Thread->WaitListEntry) :
1389 InsertTailList(&Prcb->DispatcherReadyListHead[Priority],
1390 &Thread->WaitListEntry);
1391
1392 /* Update the ready summary */
1393 Prcb->ReadySummary |= PRIORITY_MASK(Priority);
1394
1395 /* Sanity check */
1396 ASSERT(Priority == Thread->Priority);
1397
1398 /* Release the PRCB lock */
1399 KiReleasePrcbLock(Prcb);
1400 }
1401 else
1402 {
1403 /* Otherwise, prepare this thread to be deferred */
1404 Thread->State = DeferredReady;
1405 Thread->DeferredProcessor = Prcb->Number;
1406
1407 /* Release the lock and defer scheduling */
1408 KiReleasePrcbLock(Prcb);
1410 }
1411}
1412
1413//
1414// This routine scans for an appropriate ready thread to select at the
1415// given priority and for the given CPU.
1416//
1420 IN PKPRCB Prcb)
1421{
1422 ULONG PrioritySet;
1423 LONG HighPriority;
1424 PLIST_ENTRY ListEntry;
1426
1427 /* Save the current mask and get the priority set for the CPU */
1428 PrioritySet = Prcb->ReadySummary >> Priority;
1429 if (!PrioritySet) goto Quickie;
1430
1431 /* Get the highest priority possible */
1432 BitScanReverse((PULONG)&HighPriority, PrioritySet);
1433 ASSERT((PrioritySet & PRIORITY_MASK(HighPriority)) != 0);
1434 HighPriority += Priority;
1435
1436 /* Make sure the list isn't empty at the highest priority */
1437 ASSERT(IsListEmpty(&Prcb->DispatcherReadyListHead[HighPriority]) == FALSE);
1438
1439 /* Get the first thread on the list */
1440 ListEntry = Prcb->DispatcherReadyListHead[HighPriority].Flink;
1441 Thread = CONTAINING_RECORD(ListEntry, KTHREAD, WaitListEntry);
1442
1443 /* Make sure this thread is here for a reason */
1444 ASSERT(HighPriority == Thread->Priority);
1445 ASSERT(Thread->Affinity & AFFINITY_MASK(Prcb->Number));
1446 ASSERT(Thread->NextProcessor == Prcb->Number);
1447
1448 /* Remove it from the list */
1449 if (RemoveEntryList(&Thread->WaitListEntry))
1450 {
1451 /* The list is empty now, reset the ready summary */
1452 Prcb->ReadySummary ^= PRIORITY_MASK(HighPriority);
1453 }
1454
1455 /* Sanity check and return the thread */
1456Quickie:
1457 ASSERT((Thread == NULL) ||
1458 (Thread->BasePriority == 0) ||
1459 (Thread->Priority != 0));
1460 return Thread;
1461}
1462
1463//
1464// This routine computes the new priority for a thread. It is only valid for
1465// threads with priorities in the dynamic priority range.
1466//
1468SCHAR
1471{
1473
1474 /* Priority sanity checks */
1475 ASSERT((Thread->PriorityDecrement >= 0) &&
1476 (Thread->PriorityDecrement <= Thread->Priority));
1477 ASSERT((Thread->Priority < LOW_REALTIME_PRIORITY) ?
1478 TRUE : (Thread->PriorityDecrement == 0));
1479
1480 /* Get the current priority */
1481 Priority = Thread->Priority;
1483 {
1484 /* Decrease priority by the priority decrement */
1485 Priority -= (Thread->PriorityDecrement + Adjustment);
1486
1487 /* Don't go out of bounds */
1488 if (Priority < Thread->BasePriority) Priority = Thread->BasePriority;
1489
1490 /* Reset the priority decrement */
1491 Thread->PriorityDecrement = 0;
1492 }
1493
1494 /* Sanity check */
1495 ASSERT((Thread->BasePriority == 0) || (Priority != 0));
1496
1497 /* Return the new priority */
1498 return Priority;
1499}
1500
1501//
1502// Guarded Mutex Routines
1503//
1505VOID
1507{
1508 /* Setup the Initial Data */
1509 GuardedMutex->Count = GM_LOCK_BIT;
1510 GuardedMutex->Owner = NULL;
1511 GuardedMutex->Contention = 0;
1512
1513 /* Initialize the Wait Gate */
1514 KeInitializeGate(&GuardedMutex->Gate);
1515}
1516
1518VOID
1520{
1522
1523 /* Sanity checks */
1525 (Thread->SpecialApcDisable < 0) ||
1526 (Thread->Teb == NULL) ||
1527 (Thread->Teb >= (PTEB)MM_SYSTEM_RANGE_START));
1528 ASSERT(GuardedMutex->Owner != Thread);
1529
1530 /* Remove the lock */
1531 if (!InterlockedBitTestAndReset(&GuardedMutex->Count, GM_LOCK_BIT_V))
1532 {
1533 /* The Guarded Mutex was already locked, enter contented case */
1534 KiAcquireGuardedMutex(GuardedMutex);
1535 }
1536
1537 /* Set the Owner */
1538 GuardedMutex->Owner = Thread;
1539}
1540
1542VOID
1544{
1545 LONG OldValue, NewValue;
1546
1547 /* Sanity checks */
1549 (KeGetCurrentThread()->SpecialApcDisable < 0) ||
1550 (KeGetCurrentThread()->Teb == NULL) ||
1552 ASSERT(GuardedMutex->Owner == KeGetCurrentThread());
1553
1554 /* Destroy the Owner */
1555 GuardedMutex->Owner = NULL;
1556
1557 /* Add the Lock Bit */
1558 OldValue = InterlockedExchangeAdd(&GuardedMutex->Count, GM_LOCK_BIT);
1559 ASSERT((OldValue & GM_LOCK_BIT) == 0);
1560
1561 /* Check if it was already locked, but not woken */
1562 if ((OldValue) && !(OldValue & GM_LOCK_WAITER_WOKEN))
1563 {
1564 /* Update the Oldvalue to what it should be now */
1565 OldValue += GM_LOCK_BIT;
1566
1567 /* The mutex will be woken, minus one waiter */
1568 NewValue = OldValue + GM_LOCK_WAITER_WOKEN -
1570
1571 /* Remove the Woken bit */
1572 if (InterlockedCompareExchange(&GuardedMutex->Count,
1573 NewValue,
1574 OldValue) == OldValue)
1575 {
1576 /* Signal the Gate */
1577 KeSignalGateBoostPriority(&GuardedMutex->Gate);
1578 }
1579 }
1580}
1581
1583VOID
1585{
1587
1588 /* Sanity checks */
1590 ASSERT(GuardedMutex->Owner != Thread);
1591
1592 /* Disable Special APCs */
1594
1595 /* Remove the lock */
1596 if (!InterlockedBitTestAndReset(&GuardedMutex->Count, GM_LOCK_BIT_V))
1597 {
1598 /* The Guarded Mutex was already locked, enter contented case */
1599 KiAcquireGuardedMutex(GuardedMutex);
1600 }
1601
1602 /* Set the Owner and Special APC Disable state */
1603 GuardedMutex->Owner = Thread;
1604 GuardedMutex->SpecialApcDisable = Thread->SpecialApcDisable;
1605}
1606
1608VOID
1610{
1612 LONG OldValue, NewValue;
1613
1614 /* Sanity checks */
1616 ASSERT(GuardedMutex->Owner == Thread);
1617 ASSERT(Thread->SpecialApcDisable == GuardedMutex->SpecialApcDisable);
1618
1619 /* Destroy the Owner */
1620 GuardedMutex->Owner = NULL;
1621
1622 /* Add the Lock Bit */
1623 OldValue = InterlockedExchangeAdd(&GuardedMutex->Count, GM_LOCK_BIT);
1624 ASSERT((OldValue & GM_LOCK_BIT) == 0);
1625
1626 /* Check if it was already locked, but not woken */
1627 if ((OldValue) && !(OldValue & GM_LOCK_WAITER_WOKEN))
1628 {
1629 /* Update the Oldvalue to what it should be now */
1630 OldValue += GM_LOCK_BIT;
1631
1632 /* The mutex will be woken, minus one waiter */
1633 NewValue = OldValue + GM_LOCK_WAITER_WOKEN -
1635
1636 /* Remove the Woken bit */
1637 if (InterlockedCompareExchange(&GuardedMutex->Count,
1638 NewValue,
1639 OldValue) == OldValue)
1640 {
1641 /* Signal the Gate */
1642 KeSignalGateBoostPriority(&GuardedMutex->Gate);
1643 }
1644 }
1645
1646 /* Re-enable APCs */
1648}
1649
1651BOOLEAN
1653{
1655
1656 /* Block APCs */
1658
1659 /* Remove the lock */
1660 if (!InterlockedBitTestAndReset(&GuardedMutex->Count, GM_LOCK_BIT_V))
1661 {
1662 /* Re-enable APCs */
1665
1666 /* Return failure */
1667 return FALSE;
1668 }
1669
1670 /* Set the Owner and APC State */
1671 GuardedMutex->Owner = Thread;
1672 GuardedMutex->SpecialApcDisable = Thread->SpecialApcDisable;
1673 return TRUE;
1674}
1675
1676
1678VOID
1680{
1682}
1683
1685VOID
1687{
1689}
1690
1691#if defined(_M_IX86) || defined(_M_AMD64)
1693VOID
1694KiCpuId(
1695 PCPU_INFO CpuInfo,
1697{
1698 __cpuid((INT*)CpuInfo->AsUINT32, Function);
1699}
1700
1702VOID
1703KiCpuIdEx(
1704 PCPU_INFO CpuInfo,
1706 ULONG SubFunction)
1707{
1708 __cpuidex((INT*)CpuInfo->AsUINT32, Function, SubFunction);
1709}
1710#endif /* _M_IX86 || _M_AMD64 */
1711
1712#ifdef __cplusplus
1713} // extern "C"
1714#endif
unsigned char BOOLEAN
#define MM_SYSTEM_RANGE_START
Definition: armddk.h:18
#define InterlockedExchange
Definition: armddk.h:54
LONG NTSTATUS
Definition: precomp.h:26
_In_ CDROM_SCAN_FOR_SPECIAL_INFO _In_ PCDROM_SCAN_FOR_SPECIAL_HANDLER Function
Definition: cdrom.h:1156
#define _Requires_lock_held_(lock)
#define _Releases_lock_(lock)
#define NULL
Definition: types.h:112
#define TRUE
Definition: types.h:120
#define FALSE
Definition: types.h:117
LONG KPRIORITY
Definition: compat.h:803
#define RemoveEntryList(Entry)
Definition: env_spec_w32.h:986
#define InsertTailList(ListHead, Entry)
#define InsertHeadList(ListHead, Entry)
#define SYNCH_LEVEL
Definition: env_spec_w32.h:704
#define IsListEmpty(ListHead)
Definition: env_spec_w32.h:954
UCHAR KIRQL
Definition: env_spec_w32.h:591
#define KeReleaseSpinLock(sl, irql)
Definition: env_spec_w32.h:627
#define APC_LEVEL
Definition: env_spec_w32.h:695
#define KeAcquireSpinLock(sl, irql)
Definition: env_spec_w32.h:609
#define KeQuerySystemTime(t)
Definition: env_spec_w32.h:570
#define KeGetCurrentIrql()
Definition: env_spec_w32.h:706
KIRQL * PKIRQL
Definition: env_spec_w32.h:592
#define DISPATCH_LEVEL
Definition: env_spec_w32.h:696
IN OUT PLONG IN OUT PLONG Addend IN OUT PLONG IN LONG Increment
Definition: CrNtStubs.h:46
_In_opt_ PFILE_OBJECT _In_opt_ PETHREAD Thread
Definition: fltkernel.h:2653
_Must_inspect_result_ _In_ PLARGE_INTEGER _In_ PLARGE_INTEGER _In_ ULONG _In_ PFILE_OBJECT _In_ PVOID Process
Definition: fsrtlfuncs.h:223
ULONG Handle
Definition: gdb_input.c:15
VOID FASTCALL HalRequestSoftwareInterrupt(IN KIRQL Irql)
Definition: pic.c:271
KIRQL FASTCALL KfRaiseIrql(IN KIRQL NewIrql)
Definition: pic.c:187
VOID FASTCALL KeAcquireInStackQueuedSpinLock(IN PKSPIN_LOCK SpinLock, IN PKLOCK_QUEUE_HANDLE LockHandle)
Definition: spinlock.c:130
KIRQL FASTCALL KeAcquireQueuedSpinLockRaiseToSynch(IN KSPIN_LOCK_QUEUE_NUMBER LockNumber)
Definition: spinlock.c:119
VOID FASTCALL KeAcquireInStackQueuedSpinLockRaiseToSynch(IN PKSPIN_LOCK SpinLock, IN PKLOCK_QUEUE_HANDLE LockHandle)
Definition: spinlock.c:142
VOID FASTCALL KeReleaseInStackQueuedSpinLock(IN PKLOCK_QUEUE_HANDLE LockHandle)
Definition: spinlock.c:166
#define KeGetCurrentThread
Definition: hal.h:55
#define LOW_REALTIME_PRIORITY
#define HIGH_PRIORITY
#define BitScanReverse
Definition: interlocked.h:6
#define InterlockedExchangeAdd
Definition: interlocked.h:181
#define InterlockedCompareExchange
Definition: interlocked.h:104
#define InterlockedBitTestAndReset
Definition: interlocked.h:35
#define InterlockedAnd
Definition: interlocked.h:62
PPC_QUAL void __cpuid(int CPUInfo[], const int InfoType)
Definition: intrin_ppc.h:682
__INTRIN_INLINE void __cpuidex(int CPUInfo[4], int InfoType, int ECXValue)
Definition: intrin_x86.h:1649
FORCEINLINE VOID _KeAcquireGuardedMutexUnsafe(IN OUT PKGUARDED_MUTEX GuardedMutex)
Definition: ke_x.h:1519
FORCEINLINE VOID KiAcquireNmiListLock(OUT PKIRQL OldIrql)
Definition: ke_x.h:1679
FORCEINLINE VOID KiAcquireApcLockRaiseToSynch(IN PKTHREAD Thread, IN PKLOCK_QUEUE_HANDLE Handle)
Definition: ke_x.h:607
FORCEINLINE BOOLEAN KiComputeDueTime(IN PKTIMER Timer, IN LARGE_INTEGER DueTime, OUT PULONG Hand)
Definition: ke_x.h:952
FORCEINLINE VOID KiReleaseDispatcherLock(IN KIRQL OldIrql)
Definition: ke_x.h:157
FORCEINLINE VOID KiCheckDeferredReadyList(IN PKPRCB Prcb)
Definition: ke_x.h:268
FORCEINLINE VOID KiAcquireApcLockRaiseToDpc(IN PKTHREAD Thread, IN PKLOCK_QUEUE_HANDLE Handle)
Definition: ke_x.h:626
FORCEINLINE VOID _KeReleaseGuardedMutexUnsafe(IN OUT PKGUARDED_MUTEX GuardedMutex)
Definition: ke_x.h:1543
FORCEINLINE PKTHREAD KiSelectReadyThread(IN KPRIORITY Priority, IN PKPRCB Prcb)
Definition: ke_x.h:1419
#define KeEnterGuardedRegionThread(_Thread)
Definition: ke_x.h:27
FORCEINLINE VOID KiSetThreadSwapBusy(IN PKTHREAD Thread)
Definition: ke_x.h:210
FORCEINLINE VOID KxRemoveTreeTimer(IN PKTIMER Timer)
Definition: ke_x.h:1004
FORCEINLINE VOID KiRemoveEntryTimer(IN PKTIMER Timer)
Definition: ke_x.h:892
FORCEINLINE VOID KiReleaseProcessLockFromSynchLevel(IN PKLOCK_QUEUE_HANDLE Handle)
Definition: ke_x.h:668
FORCEINLINE VOID KiReleaseTimerLock(IN PKSPIN_LOCK_QUEUE LockQueue)
Definition: ke_x.h:297
FORCEINLINE PLARGE_INTEGER KiRecalculateDueTime(IN PLARGE_INTEGER OriginalDueTime, IN PLARGE_INTEGER DueTime, IN OUT PLARGE_INTEGER NewDueTime)
Definition: ke_x.h:785
FORCEINLINE VOID KiRequestApcInterrupt(IN BOOLEAN NeedApc, IN UCHAR Processor)
Definition: ke_x.h:276
FORCEINLINE VOID KiReleasePrcbLock(IN PKPRCB Prcb)
Definition: ke_x.h:230
FORCEINLINE ULONG KiComputeTimerTableIndex(IN ULONGLONG DueTime)
Definition: ke_x.h:881
FORCEINLINE VOID KiAcquireThreadLock(IN PKTHREAD Thread)
Definition: ke_x.h:240
FORCEINLINE VOID KxUnwaitThread(IN DISPATCHER_HEADER *Object, IN KPRIORITY Increment)
Definition: ke_x.h:1259
FORCEINLINE BOOLEAN _KeTryToAcquireGuardedMutex(IN OUT PKGUARDED_MUTEX GuardedMutex)
Definition: ke_x.h:1652
FORCEINLINE VOID _KeAcquireGuardedMutex(IN PKGUARDED_MUTEX GuardedMutex)
Definition: ke_x.h:1584
FORCEINLINE VOID KiReleaseNmiListLock(IN KIRQL OldIrql)
Definition: ke_x.h:1686
FORCEINLINE PKSPIN_LOCK_QUEUE KiAcquireTimerLock(IN ULONG Hand)
Definition: ke_x.h:286
FORCEINLINE VOID KiReleaseDispatcherObject(IN DISPATCHER_HEADER *Object)
Definition: ke_x.h:142
FORCEINLINE VOID _KeInitializeGuardedMutex(OUT PKGUARDED_MUTEX GuardedMutex)
Definition: ke_x.h:1506
FORCEINLINE VOID KiInsertDeferredReadyList(IN PKTHREAD Thread)
Definition: ke_x.h:185
FORCEINLINE VOID _KeReleaseGuardedMutex(IN OUT PKGUARDED_MUTEX GuardedMutex)
Definition: ke_x.h:1609
FORCEINLINE BOOLEAN KiCheckThreadStackSwap(IN PKTHREAD Thread, IN KPROCESSOR_MODE WaitMode)
Definition: ke_x.h:803
#define KeLeaveGuardedRegionThread(_Thread)
Definition: ke_x.h:48
FORCEINLINE VOID KiAcquireDispatcherObject(IN DISPATCHER_HEADER *Object)
Definition: ke_x.h:132
FORCEINLINE VOID KiReleaseDispatcherLockFromSynchLevel(VOID)
Definition: ke_x.h:174
FORCEINLINE VOID KiReleaseProcessLock(IN PKLOCK_QUEUE_HANDLE Handle)
Definition: ke_x.h:660
FORCEINLINE VOID KiReleaseThreadLock(IN PKTHREAD Thread)
Definition: ke_x.h:250
FORCEINLINE VOID KxSetTimerForThreadWait(IN PKTIMER Timer, IN LARGE_INTEGER Interval, OUT PULONG Hand)
Definition: ke_x.h:1034
FORCEINLINE VOID KiAcquireDispatcherLockAtSynchLevel(VOID)
Definition: ke_x.h:165
FORCEINLINE VOID KiAcquireApcLockAtSynchLevel(IN PKTHREAD Thread, IN PKLOCK_QUEUE_HANDLE Handle)
Definition: ke_x.h:616
FORCEINLINE VOID KiRescheduleThread(IN BOOLEAN NewThread, IN ULONG Cpu)
Definition: ke_x.h:197
FORCEINLINE VOID KiReleaseApcLockFromSynchLevel(IN PKLOCK_QUEUE_HANDLE Handle)
Definition: ke_x.h:643
FORCEINLINE VOID KiAcquirePrcbLock(IN PKPRCB Prcb)
Definition: ke_x.h:220
FORCEINLINE VOID KiAcquireDeviceQueueLock(IN PKDEVICE_QUEUE DeviceQueue, IN PKLOCK_QUEUE_HANDLE DeviceLock)
Definition: ke_x.h:676
FORCEINLINE KIRQL KiAcquireDispatcherLock(VOID)
Definition: ke_x.h:149
FORCEINLINE VOID KiAcquireProcessLockRaiseToSynch(IN PKPROCESS Process, IN PKLOCK_QUEUE_HANDLE Handle)
Definition: ke_x.h:651
FORCEINLINE VOID KxInsertTimer(IN PKTIMER Timer, IN ULONG Hand)
Definition: ke_x.h:923
FORCEINLINE NTSTATUS KiCheckAlertability(IN PKTHREAD Thread, IN BOOLEAN Alertable, IN KPROCESSOR_MODE WaitMode)
Definition: ke_x.h:841
FORCEINLINE SCHAR KiComputeNewPriority(IN PKTHREAD Thread, IN SCHAR Adjustment)
Definition: ke_x.h:1469
FORCEINLINE VOID KiReleaseApcLock(IN PKLOCK_QUEUE_HANDLE Handle)
Definition: ke_x.h:635
FORCEINLINE VOID KiReleaseDeviceQueueLock(IN PKLOCK_QUEUE_HANDLE DeviceLock)
Definition: ke_x.h:696
FORCEINLINE VOID KxUnwaitThreadForEvent(IN PKEVENT Event, IN KPRIORITY Increment)
Definition: ke_x.h:1304
FORCEINLINE BOOLEAN KiTryThreadLock(IN PKTHREAD Thread)
Definition: ke_x.h:260
#define ASSERT(a)
Definition: mode.c:44
#define KernelMode
Definition: asm.h:34
#define UserMode
Definition: asm.h:35
FORCEINLINE struct _KPRCB * KeGetCurrentPrcb(VOID)
Definition: ketypes.h:1080
#define IPI_DPC
Definition: ketypes.h:237
#define IPI_APC
Definition: ketypes.h:236
#define KeGetPreviousMode()
Definition: ketypes.h:1108
_In_ PVOID _In_ BOOLEAN Alertable
Definition: exfuncs.h:453
@ Ready
Definition: ketypes.h:422
@ Running
Definition: ketypes.h:423
@ DeferredReady
Definition: ketypes.h:428
#define KOBJECT_LOCK_BIT
Definition: ketypes.h:121
DWORD Interval
Definition: netstat.c:33
#define UNREFERENCED_PARAMETER(P)
Definition: ntbasedef.h:317
@ WaitAny
NTKERNELAPI volatile KSYSTEM_TIME KeTickCount
Definition: clock.c:19
KTIMER_TABLE_ENTRY KiTimerTableListHead[TIMER_TABLE_SIZE]
Definition: timerobj.c:17
#define PRIORITY_MASK(Id)
Definition: ke.h:160
VOID FASTCALL KiCompleteTimer(IN PKTIMER Timer, IN PKSPIN_LOCK_QUEUE LockQueue)
Definition: timerobj.c:167
VOID FASTCALL KiAcquireGuardedMutex(IN OUT PKGUARDED_MUTEX GuardedMutex)
Definition: wait.c:122
KSPIN_LOCK KiNmiCallbackListLock
Definition: bug.c:30
VOID FASTCALL KiProcessDeferredReadyList(IN PKPRCB Prcb)
Definition: thrdschd.c:41
BOOLEAN FASTCALL KiInsertTimerTable(IN PKTIMER Timer, IN ULONG Hand)
Definition: timerobj.c:63
VOID FASTCALL KiDeferredReadyThread(IN PKTHREAD Thread)
Definition: thrdschd.c:79
VOID FASTCALL KiIpiSend(KAFFINITY TargetSet, ULONG IpiRequest)
VOID FASTCALL KeSignalGateBoostPriority(PKGATE Gate)
VOID FASTCALL KeInitializeGate(PKGATE Gate)
VOID FASTCALL KiUnwaitThread(IN PKTHREAD Thread, IN LONG_PTR WaitStatus, IN KPRIORITY Increment)
Definition: wait.c:89
#define AFFINITY_MASK(Id)
Definition: ke.h:159
VOID FASTCALL KiExitDispatcher(KIRQL OldIrql)
ULONG KeMaximumIncrement
Definition: clock.c:20
VOID FASTCALL KeAcquireInStackQueuedSpinLockAtDpcLevel(IN PKSPIN_LOCK SpinLock, IN PKLOCK_QUEUE_HANDLE LockHandle)
Definition: spinlock.c:363
VOID FASTCALL KeReleaseInStackQueuedSpinLockFromDpcLevel(IN PKLOCK_QUEUE_HANDLE LockHandle)
Definition: spinlock.c:395
_In_opt_ PENTER_STATE_SYSTEM_HANDLER _In_opt_ PVOID _In_ LONG _In_opt_ LONG volatile * Number
Definition: ntpoapi.h:207
#define STATUS_WAIT_0
Definition: ntstatus.h:237
#define STATUS_ALERTED
Definition: ntstatus.h:80
#define STATUS_USER_APC
Definition: ntstatus.h:78
#define STATUS_KERNEL_APC
Definition: ntstatus.h:79
long LONG
Definition: pedump.c:60
#define YieldProcessor
Definition: ke.h:48
#define KeQueryInterruptTime()
Definition: ke.h:37
signed char SCHAR
Definition: sqltypes.h:14
EX_PUSH_LOCK ThreadLock
Definition: pstypes.h:1160
USHORT Number
Definition: ketypes.h:564
SINGLE_LIST_ENTRY DeferredReadyListHead
Definition: ketypes.h:633
ULONG LowPart
Definition: ketypes.h:917
Definition: ketypes.h:714
ULARGE_INTEGER Time
Definition: ketypes.h:719
LIST_ENTRY Entry
Definition: ketypes.h:718
struct _KTHREAD * Thread
Definition: ketypes.h:453
UCHAR WaitType
Definition: ketypes.h:446
USHORT WaitKey
Definition: ketypes.h:448
Definition: typedefs.h:120
struct _LIST_ENTRY * Flink
Definition: typedefs.h:121
Definition: compat.h:836
$ULONG HighPart
Definition: ntbasedef.h:570
uint32_t * PULONG
Definition: typedefs.h:59
int32_t INT
Definition: typedefs.h:58
#define IN
Definition: typedefs.h:39
int32_t * PLONG
Definition: typedefs.h:58
#define CONTAINING_RECORD(address, type, field)
Definition: typedefs.h:260
uint32_t ULONG
Definition: typedefs.h:59
uint64_t ULONGLONG
Definition: typedefs.h:67
#define OUT
Definition: typedefs.h:40
UINT32 AsUINT32[4]
Definition: ketypes.h:298
LONGLONG QuadPart
Definition: typedefs.h:114
_Must_inspect_result_ _In_ WDFCOLLECTION _In_ WDFOBJECT Object
_In_ WDFINTERRUPT _In_ WDF_INTERRUPT_POLICY _In_ WDF_INTERRUPT_PRIORITY Priority
Definition: wdfinterrupt.h:655
_Must_inspect_result_ _In_ WDFKEY _In_ PCUNICODE_STRING _Out_opt_ PUSHORT _Inout_opt_ PUNICODE_STRING Value
Definition: wdfregistry.h:413
_In_ WDFTIMER _In_ LONGLONG DueTime
Definition: wdftimer.h:190
#define FORCEINLINE
Definition: wdftypes.h:67
_Must_inspect_result_ typedef _In_ ULONG TableEntry
Definition: iotypes.h:4303
_In_ KPRIORITY _In_ LONG Adjustment
Definition: kefuncs.h:441
_Requires_lock_held_ Interrupt _Releases_lock_ Interrupt _In_ _IRQL_restores_ KIRQL OldIrql
Definition: kefuncs.h:792
_In_ UCHAR Processor
Definition: kefuncs.h:684
#define GM_LOCK_BIT_V
CCHAR KPROCESSOR_MODE
Definition: ketypes.h:7
#define GM_LOCK_WAITER_WOKEN
#define LOCK_QUEUE_TIMER_TABLE_LOCKS
#define LOCK_QUEUE_TIMER_LOCK_SHIFT
#define TIMER_TABLE_SIZE
Definition: ketypes.h:836
@ LockQueueTimerTableLock
Definition: ketypes.h:663
@ LockQueueDispatcherLock
Definition: ketypes.h:646
#define GM_LOCK_BIT
#define GM_LOCK_WAITER_INC
FORCEINLINE VOID PushEntryList(_Inout_ PSINGLE_LIST_ENTRY ListHead, _Inout_ __drv_aliasesMem PSINGLE_LIST_ENTRY Entry)
Definition: rtlfuncs.h:253
unsigned char UCHAR
Definition: xmlstorage.h:181