ReactOS  0.4.15-dev-2537-g99d7768
ke_x.h
Go to the documentation of this file.
1 /*
2 * PROJECT: ReactOS Kernel
3 * LICENSE: GPL - See COPYING in the top level directory
4 * FILE: ntoskrnl/include/internal/ke_x.h
5 * PURPOSE: Internal Inlined Functions for the Kernel
6 * PROGRAMMERS: Alex Ionescu (alex.ionescu@reactos.org)
7 */
8 
9 #ifdef __cplusplus
10 extern "C"
11 {
12 #endif
13 
14 #ifndef _M_ARM
18 {
19  /* Return the current mode */
20  return KeGetCurrentThread()->PreviousMode;
21 }
22 #endif
23 
24 //
25 // Enters a Guarded Region
26 //
27 #define KeEnterGuardedRegionThread(_Thread) \
28 { \
29  /* Sanity checks */ \
30  ASSERT(KeGetCurrentIrql() <= APC_LEVEL); \
31  ASSERT(_Thread == KeGetCurrentThread()); \
32  ASSERT((_Thread->SpecialApcDisable <= 0) && \
33  (_Thread->SpecialApcDisable != -32768)); \
34  \
35  /* Disable Special APCs */ \
36  _Thread->SpecialApcDisable--; \
37 }
38 
39 #define KeEnterGuardedRegion() \
40 { \
41  PKTHREAD _Thread = KeGetCurrentThread(); \
42  KeEnterGuardedRegionThread(_Thread); \
43 }
44 
45 //
46 // Leaves a Guarded Region
47 //
48 #define KeLeaveGuardedRegionThread(_Thread) \
49 { \
50  /* Sanity checks */ \
51  ASSERT(KeGetCurrentIrql() <= APC_LEVEL); \
52  ASSERT(_Thread == KeGetCurrentThread()); \
53  ASSERT(_Thread->SpecialApcDisable < 0); \
54  \
55  /* Leave region and check if APCs are OK now */ \
56  if (!(++_Thread->SpecialApcDisable)) \
57  { \
58  /* Check for Kernel APCs on the list */ \
59  if (!IsListEmpty(&_Thread->ApcState. \
60  ApcListHead[KernelMode])) \
61  { \
62  /* Check for APC Delivery */ \
63  KiCheckForKernelApcDelivery(); \
64  } \
65  } \
66 }
67 
68 #define KeLeaveGuardedRegion() \
69 { \
70  PKTHREAD _Thread = KeGetCurrentThread(); \
71  KeLeaveGuardedRegionThread(_Thread); \
72 }
73 
74 //
75 // Enters a Critical Region
76 //
77 #define KeEnterCriticalRegionThread(_Thread) \
78 { \
79  /* Sanity checks */ \
80  ASSERT(_Thread == KeGetCurrentThread()); \
81  ASSERT((_Thread->KernelApcDisable <= 0) && \
82  (_Thread->KernelApcDisable != -32768)); \
83  \
84  /* Disable Kernel APCs */ \
85  _Thread->KernelApcDisable--; \
86 }
87 
88 #define KeEnterCriticalRegion() \
89 { \
90  PKTHREAD _Thread = KeGetCurrentThread(); \
91  KeEnterCriticalRegionThread(_Thread); \
92 }
93 
94 //
95 // Leaves a Critical Region
96 //
97 #define KeLeaveCriticalRegionThread(_Thread) \
98 { \
99  /* Sanity checks */ \
100  ASSERT(_Thread == KeGetCurrentThread()); \
101  ASSERT(_Thread->KernelApcDisable < 0); \
102  \
103  /* Enable Kernel APCs */ \
104  _Thread->KernelApcDisable++; \
105  \
106  /* Check if Kernel APCs are now enabled */ \
107  if (!(_Thread->KernelApcDisable)) \
108  { \
109  /* Check if we need to request an APC Delivery */ \
110  if (!(IsListEmpty(&_Thread->ApcState.ApcListHead[KernelMode])) && \
111  !(_Thread->SpecialApcDisable)) \
112  { \
113  /* Check for the right environment */ \
114  KiCheckForKernelApcDelivery(); \
115  } \
116  } \
117 }
118 
119 #define KeLeaveCriticalRegion() \
120 { \
121  PKTHREAD _Thread = KeGetCurrentThread(); \
122  KeLeaveCriticalRegionThread(_Thread); \
123 }
124 
125 #ifndef CONFIG_SMP
126 
127 //
128 // This routine protects against multiple CPU acquires, it's meaningless on UP.
129 //
131 VOID
133 {
135 }
136 
137 //
138 // This routine protects against multiple CPU acquires, it's meaningless on UP.
139 //
141 VOID
143 {
145 }
146 
148 KIRQL
150 {
151  /* Raise to synch level */
152  return KfRaiseIrql(SYNCH_LEVEL);
153 }
154 
156 VOID
158 {
159  /* Just exit the dispatcher */
161 }
162 
164 VOID
166 {
167  /* This is a no-op at SYNCH_LEVEL for UP systems */
169  return;
170 }
171 
173 VOID
175 {
176  /* This is a no-op at SYNCH_LEVEL for UP systems */
177  return;
178 }
179 
180 //
181 // This routine makes the thread deferred ready on the boot CPU.
182 //
184 VOID
186 {
187  /* Set the thread to deferred state and boot CPU */
188  Thread->State = DeferredReady;
189  Thread->DeferredProcessor = 0;
190 
191  /* Make the thread ready immediately */
193 }
194 
196 VOID
198  IN ULONG Cpu)
199 {
200  /* This is meaningless on UP systems */
201  UNREFERENCED_PARAMETER(NewThread);
203 }
204 
205 //
206 // This routine protects against multiple CPU acquires, it's meaningless on UP.
207 //
209 VOID
211 {
213 }
214 
215 //
216 // This routine protects against multiple CPU acquires, it's meaningless on UP.
217 //
219 VOID
221 {
223 }
224 
225 //
226 // This routine protects against multiple CPU acquires, it's meaningless on UP.
227 //
229 VOID
231 {
233 }
234 
235 //
236 // This routine protects against multiple CPU acquires, it's meaningless on UP.
237 //
239 VOID
241 {
243 }
244 
245 //
246 // This routine protects against multiple CPU acquires, it's meaningless on UP.
247 //
249 VOID
251 {
253 }
254 
255 //
256 // This routine protects against multiple CPU acquires, it's meaningless on UP.
257 //
259 BOOLEAN
261 {
263  return FALSE;
264 }
265 
267 VOID
269 {
270  /* There are no deferred ready lists on UP systems */
272 }
273 
275 VOID
278 {
279  /* We deliver instantly on UP */
280  UNREFERENCED_PARAMETER(NeedApc);
282 }
283 
287 {
289 
290  /* Nothing to do on UP */
292  return NULL;
293 }
294 
296 VOID
298 {
300 
301  /* Nothing to do on UP */
302  UNREFERENCED_PARAMETER(LockQueue);
303 }
304 
305 #else
306 
308 VOID
310 {
311  LONG OldValue;
312 
313  /* Make sure we're at a safe level to touch the lock */
315 
316  /* Start acquire loop */
317  do
318  {
319  /* Loop until the other CPU releases it */
320  while (TRUE)
321  {
322  /* Check if it got released */
323  OldValue = Object->Lock;
324  if ((OldValue & KOBJECT_LOCK_BIT) == 0) break;
325 
326  /* Let the CPU know that this is a loop */
327  YieldProcessor();
328  }
329 
330  /* Try acquiring the lock now */
331  } while (InterlockedCompareExchange(&Object->Lock,
332  OldValue | KOBJECT_LOCK_BIT,
333  OldValue) != OldValue);
334 }
335 
337 VOID
339 {
340  /* Make sure we're at a safe level to touch the lock */
342 
343  /* Release it */
345 }
346 
348 KIRQL
350 {
351  /* Raise to synchronization level and acquire the dispatcher lock */
353 }
354 
356 VOID
358 {
359  /* First release the lock */
361  LockQueue[LockQueueDispatcherLock]);
362 
363  /* Then exit the dispatcher */
365 }
366 
368 VOID
370 {
371  /* Acquire the dispatcher lock */
374  LockQueue[LockQueueDispatcherLock]);
375 }
376 
378 VOID
380 {
381  /* Release the dispatcher lock */
383  LockQueue[LockQueueDispatcherLock]);
384 }
385 
386 //
387 // This routine inserts a thread into the deferred ready list of the current CPU
388 //
390 VOID
392 {
393  PKPRCB Prcb = KeGetCurrentPrcb();
394 
395  /* Set the thread to deferred state and CPU */
396  Thread->State = DeferredReady;
397  Thread->DeferredProcessor = Prcb->Number;
398 
399  /* Add it on the list */
400  PushEntryList(&Prcb->DeferredReadyListHead, &Thread->SwapListEntry);
401 }
402 
404 VOID
405 KiRescheduleThread(IN BOOLEAN NewThread,
406  IN ULONG Cpu)
407 {
408  /* Check if a new thread needs to be scheduled on a different CPU */
409  if ((NewThread) && !(KeGetCurrentPrcb()->Number == Cpu))
410  {
411  /* Send an IPI to request delivery */
413  }
414 }
415 
416 //
417 // This routine sets the current thread in a swap busy state, which ensure that
418 // nobody else tries to swap it concurrently.
419 //
421 VOID
423 {
424  /* Make sure nobody already set it */
425  ASSERT(Thread->SwapBusy == FALSE);
426 
427  /* Set it ourselves */
428  Thread->SwapBusy = TRUE;
429 }
430 
431 //
432 // This routine acquires the PRCB lock so that only one caller can touch
433 // volatile PRCB data.
434 //
435 // Since this is a simple optimized spin-lock, it must only be acquired
436 // at dispatcher level or higher!
437 //
439 VOID
441 {
442  /* Make sure we're at a safe level to touch the PRCB lock */
444 
445  /* Start acquire loop */
446  for (;;)
447  {
448  /* Acquire the lock and break out if we acquired it first */
449  if (!InterlockedExchange((PLONG)&Prcb->PrcbLock, 1)) break;
450 
451  /* Loop until the other CPU releases it */
452  do
453  {
454  /* Let the CPU know that this is a loop */
455  YieldProcessor();
456  } while (Prcb->PrcbLock);
457  }
458 }
459 
460 //
461 // This routine releases the PRCB lock so that other callers can touch
462 // volatile PRCB data.
463 //
464 // Since this is a simple optimized spin-lock, it must be be only acquired
465 // at dispatcher level or higher!
466 //
468 VOID
470 {
471  /* Make sure we are above dispatch and the lock is acquired! */
473  ASSERT(Prcb->PrcbLock != 0);
474 
475  /* Release it */
476  InterlockedAnd((PLONG)&Prcb->PrcbLock, 0);
477 }
478 
479 //
480 // This routine acquires the thread lock so that only one caller can touch
481 // volatile thread data.
482 //
483 // Since this is a simple optimized spin-lock, it must be be only acquired
484 // at dispatcher level or higher!
485 //
487 VOID
489 {
490  /* Make sure we're at a safe level to touch the thread lock */
492 
493  /* Start acquire loop */
494  for (;;)
495  {
496  /* Acquire the lock and break out if we acquired it first */
497  if (!InterlockedExchange((PLONG)&Thread->ThreadLock, 1)) break;
498 
499  /* Loop until the other CPU releases it */
500  do
501  {
502  /* Let the CPU know that this is a loop */
503  YieldProcessor();
504  } while (Thread->ThreadLock);
505  }
506 }
507 
508 //
509 // This routine releases the thread lock so that other callers can touch
510 // volatile thread data.
511 //
512 // Since this is a simple optimized spin-lock, it must be be only acquired
513 // at dispatcher level or higher!
514 //
516 VOID
518 {
519  /* Make sure we are still above dispatch */
521 
522  /* Release it */
524 }
525 
527 BOOLEAN
529 {
530  LONG Value;
531 
532  /* If the lock isn't acquired, return false */
533  if (!Thread->ThreadLock) return FALSE;
534 
535  /* Otherwise, try to acquire it and check the result */
536  Value = 1;
538 
539  /* Return the lock state */
540  return (Value == 1);
541 }
542 
544 VOID
546 {
547  /* Scan the deferred ready lists if required */
548  if (Prcb->DeferredReadyListHead.Next) KiProcessDeferredReadyList(Prcb);
549 }
550 
552 VOID
555 {
556  /* Check if we need to request APC delivery */
557  if (NeedApc)
558  {
559  /* Check if it's on another CPU */
561  {
562  /* Send an IPI to request delivery */
564  }
565  else
566  {
567  /* Request a software interrupt */
569  }
570  }
571 }
572 
576 {
577  PKSPIN_LOCK_QUEUE LockQueue;
578  ULONG LockIndex;
580 
581  /* Get the lock index */
582  LockIndex = Hand >> LOCK_QUEUE_TIMER_LOCK_SHIFT;
583  LockIndex &= (LOCK_QUEUE_TIMER_TABLE_LOCKS - 1);
584 
585  /* Now get the lock */
586  LockQueue = &KeGetCurrentPrcb()->LockQueue[LockQueueTimerTableLock + LockIndex];
587 
588  /* Acquire it and return */
590  return LockQueue;
591 }
592 
594 VOID
596 {
598 
599  /* Release the lock */
601 }
602 
603 #endif
604 
606 VOID
609 {
610  /* Acquire the lock and raise to synchronization level */
612 }
613 
615 VOID
618 {
619  /* Acquire the lock */
622 }
623 
625 VOID
628 {
629  /* Acquire the lock */
631 }
632 
634 VOID
636 {
637  /* Release the lock */
639 }
640 
642 VOID
644 {
645  /* Release the lock */
647 }
648 
650 VOID
653 {
654  /* Acquire the lock and raise to synchronization level */
656 }
657 
659 VOID
661 {
662  /* Release the lock and restore previous IRQL */
664 }
665 
667 VOID
669 {
670  /* Release the lock without lowering IRQL */
672 }
673 
675 VOID
677  IN PKLOCK_QUEUE_HANDLE DeviceLock)
678 {
679  /* Check if we were called from a threaded DPC */
680  if (KeGetCurrentPrcb()->DpcThreadActive)
681  {
682  /* Lock the Queue, we're not at DPC level */
683  KeAcquireInStackQueuedSpinLock(&DeviceQueue->Lock, DeviceLock);
684  }
685  else
686  {
687  /* We must be at DPC level, acquire the lock safely */
689  KeAcquireInStackQueuedSpinLockAtDpcLevel(&DeviceQueue->Lock,
690  DeviceLock);
691  }
692 }
693 
695 VOID
697 {
698  /* Check if we were called from a threaded DPC */
699  if (KeGetCurrentPrcb()->DpcThreadActive)
700  {
701  /* Unlock the Queue, we're not at DPC level */
702  KeReleaseInStackQueuedSpinLock(DeviceLock);
703  }
704  else
705  {
706  /* We must be at DPC level, release the lock safely */
709  }
710 }
711 
712 //
713 // Satisfies the wait of a mutant dispatcher object
714 //
715 #define KiSatisfyMutantWait(Object, Thread) \
716 { \
717  /* Decrease the Signal State */ \
718  (Object)->Header.SignalState--; \
719  \
720  /* Check if it's now non-signaled */ \
721  if (!(Object)->Header.SignalState) \
722  { \
723  /* Set the Owner Thread */ \
724  (Object)->OwnerThread = Thread; \
725  \
726  /* Disable APCs if needed */ \
727  Thread->KernelApcDisable = Thread->KernelApcDisable - \
728  (Object)->ApcDisable; \
729  \
730  /* Check if it's abandoned */ \
731  if ((Object)->Abandoned) \
732  { \
733  /* Unabandon it */ \
734  (Object)->Abandoned = FALSE; \
735  \
736  /* Return Status */ \
737  Thread->WaitStatus = STATUS_ABANDONED; \
738  } \
739  \
740  /* Insert it into the Mutant List */ \
741  InsertHeadList(Thread->MutantListHead.Blink, \
742  &(Object)->MutantListEntry); \
743  } \
744 }
745 
746 //
747 // Satisfies the wait of any nonmutant dispatcher object
748 //
749 #define KiSatisfyNonMutantWait(Object) \
750 { \
751  if (((Object)->Header.Type & TIMER_OR_EVENT_TYPE) == \
752  EventSynchronizationObject) \
753  { \
754  /* Synchronization Timers and Events just get un-signaled */ \
755  (Object)->Header.SignalState = 0; \
756  } \
757  else if ((Object)->Header.Type == SemaphoreObject) \
758  { \
759  /* These ones can have multiple states, so we only decrease it */ \
760  (Object)->Header.SignalState--; \
761  } \
762 }
763 
764 //
765 // Satisfies the wait of any dispatcher object
766 //
767 #define KiSatisfyObjectWait(Object, Thread) \
768 { \
769  /* Special case for Mutants */ \
770  if ((Object)->Header.Type == MutantObject) \
771  { \
772  KiSatisfyMutantWait((Object), (Thread)); \
773  } \
774  else \
775  { \
776  KiSatisfyNonMutantWait(Object); \
777  } \
778 }
779 
780 //
781 // Recalculates the due time
782 //
787  IN OUT PLARGE_INTEGER NewDueTime)
788 {
789  /* Don't do anything for absolute waits */
790  if (OriginalDueTime->QuadPart >= 0) return OriginalDueTime;
791 
792  /* Otherwise, query the interrupt time and recalculate */
793  NewDueTime->QuadPart = KeQueryInterruptTime();
794  NewDueTime->QuadPart -= DueTime->QuadPart;
795  return NewDueTime;
796 }
797 
798 //
799 // Determines whether a thread should be added to the wait list
800 //
802 BOOLEAN
804  IN KPROCESSOR_MODE WaitMode)
805 {
806  /* Check the required conditions */
807  if ((WaitMode != KernelMode) &&
808  (Thread->EnableStackSwap) &&
809  (Thread->Priority >= (LOW_REALTIME_PRIORITY + 9)))
810  {
811  /* We are go for swap */
812  return TRUE;
813  }
814  else
815  {
816  /* Don't swap the thread */
817  return FALSE;
818  }
819 }
820 
821 //
822 // Adds a thread to the wait list
823 //
824 #define KiAddThreadToWaitList(Thread, Swappable) \
825 { \
826  /* Make sure it's swappable */ \
827  if (Swappable) \
828  { \
829  /* Insert it into the PRCB's List */ \
830  InsertTailList(&KeGetCurrentPrcb()->WaitListHead, \
831  &Thread->WaitListEntry); \
832  } \
833 }
834 
835 //
836 // Checks if a wait in progress should be interrupted by APCs or an alertable
837 // state.
838 //
840 NTSTATUS
843  IN KPROCESSOR_MODE WaitMode)
844 {
845  /* Check if the wait is alertable */
846  if (Alertable)
847  {
848  /* It is, first check if the thread is alerted in this mode */
849  if (Thread->Alerted[WaitMode])
850  {
851  /* It is, so bail out of the wait */
852  Thread->Alerted[WaitMode] = FALSE;
853  return STATUS_ALERTED;
854  }
855  else if ((WaitMode != KernelMode) &&
856  (!IsListEmpty(&Thread->ApcState.ApcListHead[UserMode])))
857  {
858  /* It's isn't, but this is a user wait with queued user APCs */
859  Thread->ApcState.UserApcPending = TRUE;
860  return STATUS_USER_APC;
861  }
862  else if (Thread->Alerted[KernelMode])
863  {
864  /* It isn't that either, but we're alered in kernel mode */
865  Thread->Alerted[KernelMode] = FALSE;
866  return STATUS_ALERTED;
867  }
868  }
869  else if ((WaitMode != KernelMode) && (Thread->ApcState.UserApcPending))
870  {
871  /* Not alertable, but this is a user wait with pending user APCs */
872  return STATUS_USER_APC;
873  }
874 
875  /* Otherwise, we're fine */
876  return STATUS_WAIT_0;
877 }
878 
880 ULONG
882 {
883  return (DueTime / KeMaximumIncrement) & (TIMER_TABLE_SIZE - 1);
884 }
885 
886 //
887 // Called from KiCompleteTimer, KiInsertTreeTimer, KeSetSystemTime
888 // to remove timer entries
889 // See Windows HPI blog for more information.
891 VOID
893 {
894  ULONG Hand;
896 
897  /* Remove the timer from the timer list and check if it's empty */
898  Hand = Timer->Header.Hand;
899  if (RemoveEntryList(&Timer->TimerListEntry))
900  {
901  /* Get the respective timer table entry */
903  if (&TableEntry->Entry == TableEntry->Entry.Flink)
904  {
905  /* Set the entry to an infinite absolute time */
906  TableEntry->Time.HighPart = 0xFFFFFFFF;
907  }
908  }
909 
910  /* Clear the list entries on dbg builds so we can tell the timer is gone */
911 #if DBG
912  Timer->TimerListEntry.Flink = NULL;
913  Timer->TimerListEntry.Blink = NULL;
914 #endif
915 }
916 
917 //
918 // Called by Wait and Queue code to insert a timer for dispatching.
919 // Also called by KeSetTimerEx to insert a timer from the caller.
920 //
922 VOID
924  IN ULONG Hand)
925 {
926  PKSPIN_LOCK_QUEUE LockQueue;
928 
929  /* Acquire the lock and release the dispatcher lock */
930  LockQueue = KiAcquireTimerLock(Hand);
932 
933  /* Try to insert the timer */
934  if (KiInsertTimerTable(Timer, Hand))
935  {
936  /* Complete it */
937  KiCompleteTimer(Timer, LockQueue);
938  }
939  else
940  {
941  /* Do nothing, just release the lock */
942  KiReleaseTimerLock(LockQueue);
943  }
944 }
945 
946 //
947 // Called by KeSetTimerEx and KiInsertTreeTimer to calculate Due Time
948 // See the Windows HPI Blog for more information
949 //
951 BOOLEAN
954  OUT PULONG Hand)
955 {
956  LARGE_INTEGER InterruptTime, SystemTime, DifferenceTime;
957 
958  /* Convert to relative time if needed */
959  Timer->Header.Absolute = FALSE;
960  if (DueTime.HighPart >= 0)
961  {
962  /* Get System Time */
963  KeQuerySystemTime(&SystemTime);
964 
965  /* Do the conversion */
966  DifferenceTime.QuadPart = SystemTime.QuadPart - DueTime.QuadPart;
967 
968  /* Make sure it hasn't already expired */
969  Timer->Header.Absolute = TRUE;
970  if (DifferenceTime.HighPart >= 0)
971  {
972  /* Cancel everything */
973  Timer->Header.SignalState = TRUE;
974  Timer->Header.Hand = 0;
975  Timer->DueTime.QuadPart = 0;
976  *Hand = 0;
977  return FALSE;
978  }
979 
980  /* Set the time as Absolute */
981  DueTime = DifferenceTime;
982  }
983 
984  /* Get the Interrupt Time */
985  InterruptTime.QuadPart = KeQueryInterruptTime();
986 
987  /* Recalculate due time */
988  Timer->DueTime.QuadPart = InterruptTime.QuadPart - DueTime.QuadPart;
989 
990  /* Get the handle */
991  *Hand = KiComputeTimerTableIndex(Timer->DueTime.QuadPart);
992  Timer->Header.Hand = (UCHAR)*Hand;
993  Timer->Header.Inserted = TRUE;
994  return TRUE;
995 }
996 
997 //
998 // Called from Unlink and Queue Insert Code.
999 // Also called by timer code when canceling an inserted timer.
1000 // Removes a timer from it's tree.
1001 //
1003 VOID
1005 {
1006  ULONG Hand = Timer->Header.Hand;
1007  PKSPIN_LOCK_QUEUE LockQueue;
1008  PKTIMER_TABLE_ENTRY TimerEntry;
1009 
1010  /* Acquire timer lock */
1011  LockQueue = KiAcquireTimerLock(Hand);
1012 
1013  /* Set the timer as non-inserted */
1014  Timer->Header.Inserted = FALSE;
1015 
1016  /* Remove it from the timer list */
1017  if (RemoveEntryList(&Timer->TimerListEntry))
1018  {
1019  /* Get the entry and check if it's empty */
1020  TimerEntry = &KiTimerTableListHead[Hand];
1021  if (IsListEmpty(&TimerEntry->Entry))
1022  {
1023  /* Clear the time then */
1024  TimerEntry->Time.HighPart = 0xFFFFFFFF;
1025  }
1026  }
1027 
1028  /* Release the timer lock */
1029  KiReleaseTimerLock(LockQueue);
1030 }
1031 
1033 VOID
1036  OUT PULONG Hand)
1037 {
1039  LARGE_INTEGER InterruptTime, SystemTime, TimeDifference;
1040 
1041  /* Check the timer's interval to see if it's absolute */
1042  Timer->Header.Absolute = FALSE;
1043  if (Interval.HighPart >= 0)
1044  {
1045  /* Get the system time and calculate the relative time */
1046  KeQuerySystemTime(&SystemTime);
1047  TimeDifference.QuadPart = SystemTime.QuadPart - Interval.QuadPart;
1048  Timer->Header.Absolute = TRUE;
1049 
1050  /* Check if we've already expired */
1051  if (TimeDifference.HighPart >= 0)
1052  {
1053  /* Reset everything */
1054  Timer->DueTime.QuadPart = 0;
1055  *Hand = 0;
1056  Timer->Header.Hand = 0;
1057  return;
1058  }
1059  else
1060  {
1061  /* Update the interval */
1062  Interval = TimeDifference;
1063  }
1064  }
1065 
1066  /* Calculate the due time */
1067  InterruptTime.QuadPart = KeQueryInterruptTime();
1068  DueTime = InterruptTime.QuadPart - Interval.QuadPart;
1069  Timer->DueTime.QuadPart = DueTime;
1070 
1071  /* Calculate the timer handle */
1073  Timer->Header.Hand = (UCHAR)*Hand;
1074 }
1075 
1076 #define KxDelayThreadWait() \
1077  \
1078  /* Setup the Wait Block */ \
1079  Thread->WaitBlockList = TimerBlock; \
1080  \
1081  /* Setup the timer */ \
1082  KxSetTimerForThreadWait(Timer, *Interval, &Hand); \
1083  \
1084  /* Save the due time for the caller */ \
1085  DueTime.QuadPart = Timer->DueTime.QuadPart; \
1086  \
1087  /* Link the timer to this Wait Block */ \
1088  TimerBlock->NextWaitBlock = TimerBlock; \
1089  Timer->Header.WaitListHead.Flink = &TimerBlock->WaitListEntry; \
1090  Timer->Header.WaitListHead.Blink = &TimerBlock->WaitListEntry; \
1091  \
1092  /* Clear wait status */ \
1093  Thread->WaitStatus = STATUS_SUCCESS; \
1094  \
1095  /* Setup wait fields */ \
1096  Thread->Alertable = Alertable; \
1097  Thread->WaitReason = DelayExecution; \
1098  Thread->WaitMode = WaitMode; \
1099  \
1100  /* Check if we can swap the thread's stack */ \
1101  Thread->WaitListEntry.Flink = NULL; \
1102  Swappable = KiCheckThreadStackSwap(Thread, WaitMode); \
1103  \
1104  /* Set the wait time */ \
1105  Thread->WaitTime = KeTickCount.LowPart;
1106 
1107 #define KxMultiThreadWait() \
1108  /* Link wait block array to the thread */ \
1109  Thread->WaitBlockList = WaitBlockArray; \
1110  \
1111  /* Reset the index */ \
1112  Index = 0; \
1113  \
1114  /* Loop wait blocks */ \
1115  do \
1116  { \
1117  /* Fill out the wait block */ \
1118  WaitBlock = &WaitBlockArray[Index]; \
1119  WaitBlock->Object = Object[Index]; \
1120  WaitBlock->WaitKey = (USHORT)Index; \
1121  WaitBlock->WaitType = WaitType; \
1122  WaitBlock->Thread = Thread; \
1123  \
1124  /* Link to next block */ \
1125  WaitBlock->NextWaitBlock = &WaitBlockArray[Index + 1]; \
1126  Index++; \
1127  } while (Index < Count); \
1128  \
1129  /* Link the last block */ \
1130  WaitBlock->NextWaitBlock = WaitBlockArray; \
1131  \
1132  /* Set default wait status */ \
1133  Thread->WaitStatus = STATUS_WAIT_0; \
1134  \
1135  /* Check if we have a timer */ \
1136  if (Timeout) \
1137  { \
1138  /* Link to the block */ \
1139  TimerBlock->NextWaitBlock = WaitBlockArray; \
1140  \
1141  /* Setup the timer */ \
1142  KxSetTimerForThreadWait(Timer, *Timeout, &Hand); \
1143  \
1144  /* Save the due time for the caller */ \
1145  DueTime.QuadPart = Timer->DueTime.QuadPart; \
1146  \
1147  /* Initialize the list */ \
1148  InitializeListHead(&Timer->Header.WaitListHead); \
1149  } \
1150  \
1151  /* Set wait settings */ \
1152  Thread->Alertable = Alertable; \
1153  Thread->WaitMode = WaitMode; \
1154  Thread->WaitReason = WaitReason; \
1155  \
1156  /* Check if we can swap the thread's stack */ \
1157  Thread->WaitListEntry.Flink = NULL; \
1158  Swappable = KiCheckThreadStackSwap(Thread, WaitMode); \
1159  \
1160  /* Set the wait time */ \
1161  Thread->WaitTime = KeTickCount.LowPart;
1162 
1163 #define KxSingleThreadWait() \
1164  /* Setup the Wait Block */ \
1165  Thread->WaitBlockList = WaitBlock; \
1166  WaitBlock->WaitKey = STATUS_SUCCESS; \
1167  WaitBlock->Object = Object; \
1168  WaitBlock->WaitType = WaitAny; \
1169  \
1170  /* Clear wait status */ \
1171  Thread->WaitStatus = STATUS_SUCCESS; \
1172  \
1173  /* Check if we have a timer */ \
1174  if (Timeout) \
1175  { \
1176  /* Setup the timer */ \
1177  KxSetTimerForThreadWait(Timer, *Timeout, &Hand); \
1178  \
1179  /* Save the due time for the caller */ \
1180  DueTime.QuadPart = Timer->DueTime.QuadPart; \
1181  \
1182  /* Pointer to timer block */ \
1183  WaitBlock->NextWaitBlock = TimerBlock; \
1184  TimerBlock->NextWaitBlock = WaitBlock; \
1185  \
1186  /* Link the timer to this Wait Block */ \
1187  Timer->Header.WaitListHead.Flink = &TimerBlock->WaitListEntry; \
1188  Timer->Header.WaitListHead.Blink = &TimerBlock->WaitListEntry; \
1189  } \
1190  else \
1191  { \
1192  /* No timer block, just ourselves */ \
1193  WaitBlock->NextWaitBlock = WaitBlock; \
1194  } \
1195  \
1196  /* Set wait settings */ \
1197  Thread->Alertable = Alertable; \
1198  Thread->WaitMode = WaitMode; \
1199  Thread->WaitReason = WaitReason; \
1200  \
1201  /* Check if we can swap the thread's stack */ \
1202  Thread->WaitListEntry.Flink = NULL; \
1203  Swappable = KiCheckThreadStackSwap(Thread, WaitMode); \
1204  \
1205  /* Set the wait time */ \
1206  Thread->WaitTime = KeTickCount.LowPart;
1207 
1208 #define KxQueueThreadWait() \
1209  /* Setup the Wait Block */ \
1210  Thread->WaitBlockList = WaitBlock; \
1211  WaitBlock->WaitKey = STATUS_SUCCESS; \
1212  WaitBlock->Object = Queue; \
1213  WaitBlock->WaitType = WaitAny; \
1214  WaitBlock->Thread = Thread; \
1215  \
1216  /* Clear wait status */ \
1217  Thread->WaitStatus = STATUS_SUCCESS; \
1218  \
1219  /* Check if we have a timer */ \
1220  if (Timeout) \
1221  { \
1222  /* Setup the timer */ \
1223  KxSetTimerForThreadWait(Timer, *Timeout, &Hand); \
1224  \
1225  /* Save the due time for the caller */ \
1226  DueTime.QuadPart = Timer->DueTime.QuadPart; \
1227  \
1228  /* Pointer to timer block */ \
1229  WaitBlock->NextWaitBlock = TimerBlock; \
1230  TimerBlock->NextWaitBlock = WaitBlock; \
1231  \
1232  /* Link the timer to this Wait Block */ \
1233  Timer->Header.WaitListHead.Flink = &TimerBlock->WaitListEntry; \
1234  Timer->Header.WaitListHead.Blink = &TimerBlock->WaitListEntry; \
1235  } \
1236  else \
1237  { \
1238  /* No timer block, just ourselves */ \
1239  WaitBlock->NextWaitBlock = WaitBlock; \
1240  } \
1241  \
1242  /* Set wait settings */ \
1243  Thread->Alertable = FALSE; \
1244  Thread->WaitMode = WaitMode; \
1245  Thread->WaitReason = WrQueue; \
1246  \
1247  /* Check if we can swap the thread's stack */ \
1248  Thread->WaitListEntry.Flink = NULL; \
1249  Swappable = KiCheckThreadStackSwap(Thread, WaitMode); \
1250  \
1251  /* Set the wait time */ \
1252  Thread->WaitTime = KeTickCount.LowPart;
1253 
1254 //
1255 // Unwaits a Thread
1256 //
1258 VOID
1261 {
1262  PLIST_ENTRY WaitEntry, WaitList;
1263  PKWAIT_BLOCK WaitBlock;
1264  PKTHREAD WaitThread;
1265  ULONG WaitKey;
1266 
1267  /* Loop the Wait Entries */
1268  WaitList = &Object->WaitListHead;
1269  ASSERT(IsListEmpty(&Object->WaitListHead) == FALSE);
1270  WaitEntry = WaitList->Flink;
1271  do
1272  {
1273  /* Get the current wait block */
1274  WaitBlock = CONTAINING_RECORD(WaitEntry, KWAIT_BLOCK, WaitListEntry);
1275 
1276  /* Get the waiting thread */
1277  WaitThread = WaitBlock->Thread;
1278 
1279  /* Check the current Wait Mode */
1280  if (WaitBlock->WaitType == WaitAny)
1281  {
1282  /* Use the actual wait key */
1283  WaitKey = WaitBlock->WaitKey;
1284  }
1285  else
1286  {
1287  /* Otherwise, use STATUS_KERNEL_APC */
1288  WaitKey = STATUS_KERNEL_APC;
1289  }
1290 
1291  /* Unwait the thread */
1292  KiUnwaitThread(WaitThread, WaitKey, Increment);
1293 
1294  /* Next entry */
1295  WaitEntry = WaitList->Flink;
1296  } while (WaitEntry != WaitList);
1297 }
1298 
1299 //
1300 // Unwaits a Thread waiting on an event
1301 //
1303 VOID
1306 {
1307  PLIST_ENTRY WaitEntry, WaitList;
1308  PKWAIT_BLOCK WaitBlock;
1309  PKTHREAD WaitThread;
1310 
1311  /* Loop the Wait Entries */
1312  WaitList = &Event->Header.WaitListHead;
1313  ASSERT(IsListEmpty(&Event->Header.WaitListHead) == FALSE);
1314  WaitEntry = WaitList->Flink;
1315  do
1316  {
1317  /* Get the current wait block */
1318  WaitBlock = CONTAINING_RECORD(WaitEntry, KWAIT_BLOCK, WaitListEntry);
1319 
1320  /* Get the waiting thread */
1321  WaitThread = WaitBlock->Thread;
1322 
1323  /* Check the current Wait Mode */
1324  if (WaitBlock->WaitType == WaitAny)
1325  {
1326  /* Un-signal it */
1327  Event->Header.SignalState = 0;
1328 
1329  /* Un-signal the event and unwait the thread */
1330  KiUnwaitThread(WaitThread, WaitBlock->WaitKey, Increment);
1331  break;
1332  }
1333 
1334  /* Unwait the thread with STATUS_KERNEL_APC */
1336 
1337  /* Next entry */
1338  WaitEntry = WaitList->Flink;
1339  } while (WaitEntry != WaitList);
1340 }
1341 
1342 //
1343 // This routine queues a thread that is ready on the PRCB's ready lists.
1344 // If this thread cannot currently run on this CPU, then the thread is
1345 // added to the deferred ready list instead.
1346 //
1347 // This routine must be entered with the PRCB lock held and it will exit
1348 // with the PRCB lock released!
1349 //
1351 VOID
1353  IN PKPRCB Prcb)
1354 {
1355  BOOLEAN Preempted;
1357 
1358  /* Sanity checks */
1359  ASSERT(Prcb == KeGetCurrentPrcb());
1360  ASSERT(Thread->State == Running);
1361  ASSERT(Thread->NextProcessor == Prcb->Number);
1362 
1363  /* Check if this thread is allowed to run in this CPU */
1364 #ifdef CONFIG_SMP
1365  if ((Thread->Affinity) & (Prcb->SetMember))
1366 #else
1367  if (TRUE)
1368 #endif
1369  {
1370  /* Set thread ready for execution */
1371  Thread->State = Ready;
1372 
1373  /* Save current priority and if someone had pre-empted it */
1374  Priority = Thread->Priority;
1375  Preempted = Thread->Preempted;
1376 
1377  /* We're not pre-empting now, and set the wait time */
1378  Thread->Preempted = FALSE;
1379  Thread->WaitTime = KeTickCount.LowPart;
1380 
1381  /* Sanity check */
1382  ASSERT((Priority >= 0) && (Priority <= HIGH_PRIORITY));
1383 
1384  /* Insert this thread in the appropriate order */
1385  Preempted ? InsertHeadList(&Prcb->DispatcherReadyListHead[Priority],
1386  &Thread->WaitListEntry) :
1387  InsertTailList(&Prcb->DispatcherReadyListHead[Priority],
1388  &Thread->WaitListEntry);
1389 
1390  /* Update the ready summary */
1391  Prcb->ReadySummary |= PRIORITY_MASK(Priority);
1392 
1393  /* Sanity check */
1394  ASSERT(Priority == Thread->Priority);
1395 
1396  /* Release the PRCB lock */
1397  KiReleasePrcbLock(Prcb);
1398  }
1399  else
1400  {
1401  /* Otherwise, prepare this thread to be deferred */
1402  Thread->State = DeferredReady;
1403  Thread->DeferredProcessor = Prcb->Number;
1404 
1405  /* Release the lock and defer scheduling */
1406  KiReleasePrcbLock(Prcb);
1408  }
1409 }
1410 
1411 //
1412 // This routine scans for an appropriate ready thread to select at the
1413 // given priority and for the given CPU.
1414 //
1416 PKTHREAD
1418  IN PKPRCB Prcb)
1419 {
1420  ULONG PrioritySet;
1421  LONG HighPriority;
1422  PLIST_ENTRY ListEntry;
1423  PKTHREAD Thread = NULL;
1424 
1425  /* Save the current mask and get the priority set for the CPU */
1426  PrioritySet = Prcb->ReadySummary >> Priority;
1427  if (!PrioritySet) goto Quickie;
1428 
1429  /* Get the highest priority possible */
1430  BitScanReverse((PULONG)&HighPriority, PrioritySet);
1431  ASSERT((PrioritySet & PRIORITY_MASK(HighPriority)) != 0);
1432  HighPriority += Priority;
1433 
1434  /* Make sure the list isn't empty at the highest priority */
1435  ASSERT(IsListEmpty(&Prcb->DispatcherReadyListHead[HighPriority]) == FALSE);
1436 
1437  /* Get the first thread on the list */
1438  ListEntry = Prcb->DispatcherReadyListHead[HighPriority].Flink;
1439  Thread = CONTAINING_RECORD(ListEntry, KTHREAD, WaitListEntry);
1440 
1441  /* Make sure this thread is here for a reason */
1442  ASSERT(HighPriority == Thread->Priority);
1443  ASSERT(Thread->Affinity & AFFINITY_MASK(Prcb->Number));
1444  ASSERT(Thread->NextProcessor == Prcb->Number);
1445 
1446  /* Remove it from the list */
1447  if (RemoveEntryList(&Thread->WaitListEntry))
1448  {
1449  /* The list is empty now, reset the ready summary */
1450  Prcb->ReadySummary ^= PRIORITY_MASK(HighPriority);
1451  }
1452 
1453  /* Sanity check and return the thread */
1454 Quickie:
1455  ASSERT((Thread == NULL) ||
1456  (Thread->BasePriority == 0) ||
1457  (Thread->Priority != 0));
1458  return Thread;
1459 }
1460 
1461 //
1462 // This routine computes the new priority for a thread. It is only valid for
1463 // threads with priorities in the dynamic priority range.
1464 //
1466 SCHAR
1469 {
1470  SCHAR Priority;
1471 
1472  /* Priority sanity checks */
1473  ASSERT((Thread->PriorityDecrement >= 0) &&
1474  (Thread->PriorityDecrement <= Thread->Priority));
1475  ASSERT((Thread->Priority < LOW_REALTIME_PRIORITY) ?
1476  TRUE : (Thread->PriorityDecrement == 0));
1477 
1478  /* Get the current priority */
1479  Priority = Thread->Priority;
1481  {
1482  /* Decrease priority by the priority decrement */
1483  Priority -= (Thread->PriorityDecrement + Adjustment);
1484 
1485  /* Don't go out of bounds */
1486  if (Priority < Thread->BasePriority) Priority = Thread->BasePriority;
1487 
1488  /* Reset the priority decrement */
1489  Thread->PriorityDecrement = 0;
1490  }
1491 
1492  /* Sanity check */
1493  ASSERT((Thread->BasePriority == 0) || (Priority != 0));
1494 
1495  /* Return the new priority */
1496  return Priority;
1497 }
1498 
1499 //
1500 // Guarded Mutex Routines
1501 //
1503 VOID
1505 {
1506  /* Setup the Initial Data */
1507  GuardedMutex->Count = GM_LOCK_BIT;
1508  GuardedMutex->Owner = NULL;
1509  GuardedMutex->Contention = 0;
1510 
1511  /* Initialize the Wait Gate */
1512  KeInitializeGate(&GuardedMutex->Gate);
1513 }
1514 
1516 VOID
1518 {
1520 
1521  /* Sanity checks */
1523  (Thread->SpecialApcDisable < 0) ||
1524  (Thread->Teb == NULL) ||
1525  (Thread->Teb >= (PTEB)MM_SYSTEM_RANGE_START));
1526  ASSERT(GuardedMutex->Owner != Thread);
1527 
1528  /* Remove the lock */
1529  if (!InterlockedBitTestAndReset(&GuardedMutex->Count, GM_LOCK_BIT_V))
1530  {
1531  /* The Guarded Mutex was already locked, enter contented case */
1532  KiAcquireGuardedMutex(GuardedMutex);
1533  }
1534 
1535  /* Set the Owner */
1536  GuardedMutex->Owner = Thread;
1537 }
1538 
1540 VOID
1542 {
1543  LONG OldValue, NewValue;
1544 
1545  /* Sanity checks */
1547  (KeGetCurrentThread()->SpecialApcDisable < 0) ||
1548  (KeGetCurrentThread()->Teb == NULL) ||
1550  ASSERT(GuardedMutex->Owner == KeGetCurrentThread());
1551 
1552  /* Destroy the Owner */
1553  GuardedMutex->Owner = NULL;
1554 
1555  /* Add the Lock Bit */
1556  OldValue = InterlockedExchangeAdd(&GuardedMutex->Count, GM_LOCK_BIT);
1557  ASSERT((OldValue & GM_LOCK_BIT) == 0);
1558 
1559  /* Check if it was already locked, but not woken */
1560  if ((OldValue) && !(OldValue & GM_LOCK_WAITER_WOKEN))
1561  {
1562  /* Update the Oldvalue to what it should be now */
1563  OldValue += GM_LOCK_BIT;
1564 
1565  /* The mutex will be woken, minus one waiter */
1566  NewValue = OldValue + GM_LOCK_WAITER_WOKEN -
1568 
1569  /* Remove the Woken bit */
1570  if (InterlockedCompareExchange(&GuardedMutex->Count,
1571  NewValue,
1572  OldValue) == OldValue)
1573  {
1574  /* Signal the Gate */
1575  KeSignalGateBoostPriority(&GuardedMutex->Gate);
1576  }
1577  }
1578 }
1579 
1581 VOID
1583 {
1585 
1586  /* Sanity checks */
1588  ASSERT(GuardedMutex->Owner != Thread);
1589 
1590  /* Disable Special APCs */
1592 
1593  /* Remove the lock */
1594  if (!InterlockedBitTestAndReset(&GuardedMutex->Count, GM_LOCK_BIT_V))
1595  {
1596  /* The Guarded Mutex was already locked, enter contented case */
1597  KiAcquireGuardedMutex(GuardedMutex);
1598  }
1599 
1600  /* Set the Owner and Special APC Disable state */
1601  GuardedMutex->Owner = Thread;
1602  GuardedMutex->SpecialApcDisable = Thread->SpecialApcDisable;
1603 }
1604 
1606 VOID
1608 {
1610  LONG OldValue, NewValue;
1611 
1612  /* Sanity checks */
1614  ASSERT(GuardedMutex->Owner == Thread);
1615  ASSERT(Thread->SpecialApcDisable == GuardedMutex->SpecialApcDisable);
1616 
1617  /* Destroy the Owner */
1618  GuardedMutex->Owner = NULL;
1619 
1620  /* Add the Lock Bit */
1621  OldValue = InterlockedExchangeAdd(&GuardedMutex->Count, GM_LOCK_BIT);
1622  ASSERT((OldValue & GM_LOCK_BIT) == 0);
1623 
1624  /* Check if it was already locked, but not woken */
1625  if ((OldValue) && !(OldValue & GM_LOCK_WAITER_WOKEN))
1626  {
1627  /* Update the Oldvalue to what it should be now */
1628  OldValue += GM_LOCK_BIT;
1629 
1630  /* The mutex will be woken, minus one waiter */
1631  NewValue = OldValue + GM_LOCK_WAITER_WOKEN -
1633 
1634  /* Remove the Woken bit */
1635  if (InterlockedCompareExchange(&GuardedMutex->Count,
1636  NewValue,
1637  OldValue) == OldValue)
1638  {
1639  /* Signal the Gate */
1640  KeSignalGateBoostPriority(&GuardedMutex->Gate);
1641  }
1642  }
1643 
1644  /* Re-enable APCs */
1646 }
1647 
1649 BOOLEAN
1651 {
1653 
1654  /* Block APCs */
1656 
1657  /* Remove the lock */
1658  if (!InterlockedBitTestAndReset(&GuardedMutex->Count, GM_LOCK_BIT_V))
1659  {
1660  /* Re-enable APCs */
1662  YieldProcessor();
1663 
1664  /* Return failure */
1665  return FALSE;
1666  }
1667 
1668  /* Set the Owner and APC State */
1669  GuardedMutex->Owner = Thread;
1670  GuardedMutex->SpecialApcDisable = Thread->SpecialApcDisable;
1671  return TRUE;
1672 }
1673 
1674 
1676 VOID
1678 {
1680 }
1681 
1683 VOID
1685 {
1687 }
1688 
1689 #if defined(_M_IX86) || defined(_M_AMD64)
1691 VOID
1692 KiCpuId(
1693  PCPU_INFO CpuInfo,
1694  ULONG Function)
1695 {
1696  __cpuid((INT*)CpuInfo->AsUINT32, Function);
1697 }
1698 
1700 VOID
1701 KiCpuIdEx(
1702  PCPU_INFO CpuInfo,
1703  ULONG Function,
1704  ULONG SubFunction)
1705 {
1706  __cpuidex((INT*)CpuInfo->AsUINT32, Function, SubFunction);
1707 }
1708 #endif /* _M_IX86 || _M_AMD64 */
1709 
1710 #ifdef __cplusplus
1711 } // extern "C"
1712 #endif
#define KeQuerySystemTime(t)
Definition: env_spec_w32.h:570
#define KeGetCurrentIrql()
Definition: env_spec_w32.h:706
FORCEINLINE BOOLEAN KiCheckThreadStackSwap(IN PKTHREAD Thread, IN KPROCESSOR_MODE WaitMode)
Definition: ke_x.h:803
FORCEINLINE VOID KiReleaseThreadLock(IN PKTHREAD Thread)
Definition: ke_x.h:250
VOID FASTCALL KeReleaseQueuedSpinLockFromDpcLevel(IN OUT PKSPIN_LOCK_QUEUE LockQueue)
ULONG KeMaximumIncrement
Definition: clock.c:20
ULONG LowPart
Definition: ketypes.h:917
_In_ PVOID _In_ BOOLEAN Alertable
Definition: exfuncs.h:452
VOID FASTCALL KeAcquireInStackQueuedSpinLockRaiseToSynch(IN PKSPIN_LOCK SpinLock, IN PKLOCK_QUEUE_HANDLE LockHandle)
Definition: spinlock.c:142
#define IN
Definition: typedefs.h:39
FORCEINLINE VOID KxSetTimerForThreadWait(IN PKTIMER Timer, IN LARGE_INTEGER Interval, OUT PULONG Hand)
Definition: ke_x.h:1034
VOID FASTCALL KiUnwaitThread(IN PKTHREAD Thread, IN LONG_PTR WaitStatus, IN KPRIORITY Increment)
Definition: wait.c:89
#define InterlockedAnd
Definition: interlocked.h:62
#define PRIORITY_MASK(Id)
Definition: ke.h:160
FORCEINLINE VOID KiAcquireProcessLockRaiseToSynch(IN PKPROCESS Process, IN PKLOCK_QUEUE_HANDLE Handle)
Definition: ke_x.h:651
_In_ CDROM_SCAN_FOR_SPECIAL_INFO _In_ PCDROM_SCAN_FOR_SPECIAL_HANDLER Function
Definition: cdrom.h:1154
FORCEINLINE VOID KiReleasePrcbLock(IN PKPRCB Prcb)
Definition: ke_x.h:230
FORCEINLINE VOID _KeAcquireGuardedMutex(IN PKGUARDED_MUTEX GuardedMutex)
Definition: ke_x.h:1582
FORCEINLINE KPROCESSOR_MODE KeGetPreviousMode(VOID)
Definition: ke_x.h:17
FORCEINLINE VOID KiCheckDeferredReadyList(IN PKPRCB Prcb)
Definition: ke_x.h:268
#define TRUE
Definition: types.h:120
#define UNREFERENCED_PARAMETER(P)
Definition: ntbasedef.h:317
FORCEINLINE VOID KiReleaseNmiListLock(IN KIRQL OldIrql)
Definition: ke_x.h:1684
FORCEINLINE VOID InsertHeadList(_Inout_ PLIST_ENTRY ListHead, _Inout_ __drv_aliasesMem PLIST_ENTRY Entry)
Definition: rtlfuncs.h:201
FORCEINLINE BOOLEAN KiComputeDueTime(IN PKTIMER Timer, IN LARGE_INTEGER DueTime, OUT PULONG Hand)
Definition: ke_x.h:952
FORCEINLINE VOID KxUnwaitThreadForEvent(IN PKEVENT Event, IN KPRIORITY Increment)
Definition: ke_x.h:1304
FORCEINLINE PKSPIN_LOCK_QUEUE KiAcquireTimerLock(IN ULONG Hand)
Definition: ke_x.h:286
LONG NTSTATUS
Definition: precomp.h:26
FORCEINLINE VOID KiInsertDeferredReadyList(IN PKTHREAD Thread)
Definition: ke_x.h:185
FORCEINLINE VOID KiSetThreadSwapBusy(IN PKTHREAD Thread)
Definition: ke_x.h:210
FORCEINLINE VOID KiAcquireDeviceQueueLock(IN PKDEVICE_QUEUE DeviceQueue, IN PKLOCK_QUEUE_HANDLE DeviceLock)
Definition: ke_x.h:676
FORCEINLINE struct _KPRCB * KeGetCurrentPrcb(VOID)
Definition: ketypes.h:1079
FORCEINLINE VOID KxRemoveTreeTimer(IN PKTIMER Timer)
Definition: ke_x.h:1004
struct _KTHREAD * Thread
Definition: ketypes.h:453
VOID FASTCALL KiDeferredReadyThread(IN PKTHREAD Thread)
Definition: thrdschd.c:79
#define InterlockedCompareExchange
Definition: interlocked.h:104
_In_ UCHAR Processor
Definition: kefuncs.h:682
VOID FASTCALL KeAcquireQueuedSpinLockAtDpcLevel(IN OUT PKSPIN_LOCK_QUEUE LockQueue)
#define LOW_REALTIME_PRIORITY
#define AFFINITY_MASK(Id)
Definition: ke.h:159
#define InsertTailList(ListHead, Entry)
VOID FASTCALL KeAcquireInStackQueuedSpinLock(IN PKSPIN_LOCK SpinLock, IN PKLOCK_QUEUE_HANDLE LockHandle)
Definition: spinlock.c:130
#define STATUS_ALERTED
Definition: ntstatus.h:80
FORCEINLINE VOID KiReleaseApcLock(IN PKLOCK_QUEUE_HANDLE Handle)
Definition: ke_x.h:635
#define LOCK_QUEUE_TIMER_TABLE_LOCKS
FORCEINLINE BOOLEAN KiTryThreadLock(IN PKTHREAD Thread)
Definition: ke_x.h:260
int32_t INT
Definition: typedefs.h:58
_Must_inspect_result_ FORCEINLINE BOOLEAN IsListEmpty(_In_ const LIST_ENTRY *ListHead)
Definition: rtlfuncs.h:57
FORCEINLINE VOID _KeReleaseGuardedMutexUnsafe(IN OUT PKGUARDED_MUTEX GuardedMutex)
Definition: ke_x.h:1541
LONG KPRIORITY
Definition: compat.h:662
FORCEINLINE VOID _KeInitializeGuardedMutex(OUT PKGUARDED_MUTEX GuardedMutex)
Definition: ke_x.h:1504
Definition: ketypes.h:703
FORCEINLINE BOOLEAN RemoveEntryList(_In_ PLIST_ENTRY Entry)
Definition: rtlfuncs.h:105
FORCEINLINE VOID YieldProcessor(VOID)
Definition: ke.h:32
FORCEINLINE SCHAR KiComputeNewPriority(IN PKTHREAD Thread, IN SCHAR Adjustment)
Definition: ke_x.h:1467
FORCEINLINE VOID PushEntryList(_Inout_ PSINGLE_LIST_ENTRY ListHead, _Inout_ __drv_aliasesMem PSINGLE_LIST_ENTRY Entry)
Definition: rtlfuncs.h:253
UCHAR KIRQL
Definition: env_spec_w32.h:591
FORCEINLINE VOID KxQueueReadyThread(IN PKTHREAD Thread, IN PKPRCB Prcb)
Definition: ke_x.h:1352
#define STATUS_WAIT_0
Definition: ntstatus.h:237
FORCEINLINE VOID _KeReleaseGuardedMutex(IN OUT PKGUARDED_MUTEX GuardedMutex)
Definition: ke_x.h:1607
FORCEINLINE VOID KiAcquireApcLockAtSynchLevel(IN PKTHREAD Thread, IN PKLOCK_QUEUE_HANDLE Handle)
Definition: ke_x.h:616
#define FALSE
Definition: types.h:117
#define GM_LOCK_BIT
_In_ KPRIORITY _In_ LONG Adjustment
Definition: kefuncs.h:438
long LONG
Definition: pedump.c:60
#define InterlockedBitTestAndReset
Definition: interlocked.h:35
#define GM_LOCK_WAITER_INC
unsigned char BOOLEAN
FORCEINLINE VOID KiAcquirePrcbLock(IN PKPRCB Prcb)
Definition: ke_x.h:220
FORCEINLINE VOID KxInsertTimer(IN PKTIMER Timer, IN ULONG Hand)
Definition: ke_x.h:923
#define STATUS_KERNEL_APC
Definition: ntstatus.h:79
#define IPI_DPC
Definition: ketypes.h:237
#define HIGH_PRIORITY
#define InterlockedExchangeAdd
Definition: interlocked.h:181
PFLT_MESSAGE_WAITER_QUEUE CONTAINING_RECORD(Csq, DEVICE_EXTENSION, IrpQueue)) -> WaiterQ.mLock) _IRQL_raises_(DISPATCH_LEVEL) VOID NTAPI FltpAcquireMessageWaiterLock(_In_ PIO_CSQ Csq, _Out_ PKIRQL Irql)
Definition: Messaging.c:560
VOID FASTCALL KiExitDispatcher(KIRQL OldIrql)
ULARGE_INTEGER Time
Definition: ketypes.h:709
FORCEINLINE VOID KiRescheduleThread(IN BOOLEAN NewThread, IN ULONG Cpu)
Definition: ke_x.h:197
BOOLEAN FASTCALL KiInsertTimerTable(IN PKTIMER Timer, IN ULONG Hand)
Definition: timerobj.c:63
_Must_inspect_result_ _In_ WDFKEY _In_ PCUNICODE_STRING _Out_opt_ PUSHORT _Inout_opt_ PUNICODE_STRING Value
Definition: wdfregistry.h:406
DWORD Interval
Definition: netstat.c:33
KIRQL FASTCALL KfRaiseIrql(IN KIRQL NewIrql)
Definition: pic.c:187
FORCEINLINE VOID KiAcquireThreadLock(IN PKTHREAD Thread)
Definition: ke_x.h:240
struct _LIST_ENTRY * Flink
Definition: typedefs.h:121
VOID FASTCALL KeReleaseInStackQueuedSpinLock(IN PKLOCK_QUEUE_HANDLE LockHandle)
Definition: spinlock.c:166
VOID FASTCALL KeAcquireInStackQueuedSpinLockAtDpcLevel(IN PKSPIN_LOCK SpinLock, IN PKLOCK_QUEUE_HANDLE LockHandle)
Definition: spinlock.c:337
#define TIMER_TABLE_SIZE
Definition: ketypes.h:836
LIST_ENTRY Entry
Definition: ketypes.h:708
$ULONG HighPart
Definition: ntbasedef.h:570
UCHAR WaitType
Definition: ketypes.h:446
#define ASSERT(a)
Definition: mode.c:45
FORCEINLINE VOID KiAcquireDispatcherObject(IN DISPATCHER_HEADER *Object)
Definition: ke_x.h:132
uint64_t ULONGLONG
Definition: typedefs.h:67
USHORT WaitKey
Definition: ketypes.h:448
FORCEINLINE VOID KiRequestApcInterrupt(IN BOOLEAN NeedApc, IN UCHAR Processor)
Definition: ke_x.h:276
#define KeAcquireSpinLock(sl, irql)
Definition: env_spec_w32.h:609
FORCEINLINE VOID KiReleaseDispatcherObject(IN DISPATCHER_HEADER *Object)
Definition: ke_x.h:142
signed char SCHAR
Definition: sqltypes.h:14
_In_opt_ PFILE_OBJECT _In_opt_ PETHREAD Thread
Definition: fltkernel.h:2653
CCHAR KPROCESSOR_MODE
Definition: ketypes.h:7
KSPIN_LOCK KiNmiCallbackListLock
Definition: bug.c:30
FORCEINLINE VOID KiReleaseProcessLock(IN PKLOCK_QUEUE_HANDLE Handle)
Definition: ke_x.h:660
KTIMER_TABLE_ENTRY KiTimerTableListHead[TIMER_TABLE_SIZE]
Definition: timerobj.c:17
#define KeEnterGuardedRegionThread(_Thread)
Definition: ke_x.h:27
_In_ WDFINTERRUPT _In_ WDF_INTERRUPT_POLICY _In_ WDF_INTERRUPT_PRIORITY Priority
Definition: wdfinterrupt.h:651
unsigned char UCHAR
Definition: xmlstorage.h:181
VOID FASTCALL KiCompleteTimer(IN PKTIMER Timer, IN PKSPIN_LOCK_QUEUE LockQueue)
Definition: timerobj.c:167
NTKERNELAPI volatile KSYSTEM_TIME KeTickCount
Definition: clock.c:19
_Requires_lock_held_ Interrupt _Releases_lock_ Interrupt _In_ _IRQL_restores_ KIRQL OldIrql
Definition: kefuncs.h:790
#define BitScanReverse
Definition: interlocked.h:6
Definition: ketypes.h:412
KIRQL * PKIRQL
Definition: env_spec_w32.h:592
FORCEINLINE VOID KiReleaseDispatcherLockFromSynchLevel(VOID)
Definition: ke_x.h:174
FORCEINLINE VOID KiAcquireApcLockRaiseToDpc(IN PKTHREAD Thread, IN PKLOCK_QUEUE_HANDLE Handle)
Definition: ke_x.h:626
Definition: typedefs.h:119
#define SYNCH_LEVEL
Definition: env_spec_w32.h:704
#define MM_SYSTEM_RANGE_START
Definition: armddk.h:18
FORCEINLINE VOID KiAcquireApcLockRaiseToSynch(IN PKTHREAD Thread, IN PKLOCK_QUEUE_HANDLE Handle)
Definition: ke_x.h:607
VOID FASTCALL KiIpiSend(KAFFINITY TargetSet, ULONG IpiRequest)
_In_opt_ PENTER_STATE_SYSTEM_HANDLER _In_opt_ PVOID _In_ LONG _In_opt_ LONG volatile * Number
Definition: ntpoapi.h:204
#define LOCK_QUEUE_TIMER_LOCK_SHIFT
#define InterlockedExchange
Definition: armddk.h:54
FORCEINLINE VOID KiReleaseProcessLockFromSynchLevel(IN PKLOCK_QUEUE_HANDLE Handle)
Definition: ke_x.h:668
_Must_inspect_result_ _In_ WDFCOLLECTION _In_ WDFOBJECT Object
#define DISPATCH_LEVEL
Definition: env_spec_w32.h:696
#define STATUS_USER_APC
Definition: ntstatus.h:78
UINT32 AsUINT32[4]
Definition: ketypes.h:297
SINGLE_LIST_ENTRY DeferredReadyListHead
Definition: ketypes.h:632
VOID FASTCALL HalRequestSoftwareInterrupt(IN KIRQL Irql)
Definition: pic.c:271
FORCEINLINE VOID KiReleaseApcLockFromSynchLevel(IN PKLOCK_QUEUE_HANDLE Handle)
Definition: ke_x.h:643
Definition: compat.h:694
FORCEINLINE PLARGE_INTEGER KiRecalculateDueTime(IN PLARGE_INTEGER OriginalDueTime, IN PLARGE_INTEGER DueTime, IN OUT PLARGE_INTEGER NewDueTime)
Definition: ke_x.h:785
__INTRIN_INLINE void __cpuidex(int CPUInfo[4], int InfoType, int ECXValue)
Definition: intrin_x86.h:1650
PPC_QUAL void __cpuid(int CPUInfo[], const int InfoType)
Definition: intrin_ppc.h:682
VOID FASTCALL KiAcquireGuardedMutex(IN OUT PKGUARDED_MUTEX GuardedMutex)
Definition: wait.c:122
FORCEINLINE VOID KiReleaseDispatcherLock(IN KIRQL OldIrql)
Definition: ke_x.h:157
FORCEINLINE PKTHREAD KiSelectReadyThread(IN KPRIORITY Priority, IN PKPRCB Prcb)
Definition: ke_x.h:1417
VOID FASTCALL KeSignalGateBoostPriority(PKGATE Gate)
FORCEINLINE KIRQL KiAcquireDispatcherLock(VOID)
Definition: ke_x.h:149
FORCEINLINE BOOLEAN _KeTryToAcquireGuardedMutex(IN OUT PKGUARDED_MUTEX GuardedMutex)
Definition: ke_x.h:1650
#define FORCEINLINE
Definition: wdftypes.h:67
FORCEINLINE VOID KiAcquireNmiListLock(OUT PKIRQL OldIrql)
Definition: ke_x.h:1677
unsigned int * PULONG
Definition: retypes.h:1
#define NULL
Definition: types.h:112
FORCEINLINE VOID KiRemoveEntryTimer(IN PKTIMER Timer)
Definition: ke_x.h:892
#define KeReleaseSpinLock(sl, irql)
Definition: env_spec_w32.h:627
_In_ WDFTIMER _In_ LONGLONG DueTime
Definition: wdftimer.h:187
_Must_inspect_result_ typedef _In_ ULONG TableEntry
Definition: iotypes.h:4302
FORCEINLINE VOID KiReleaseDeviceQueueLock(IN PKLOCK_QUEUE_HANDLE DeviceLock)
Definition: ke_x.h:696
_Must_inspect_result_ _In_ PLARGE_INTEGER _In_ PLARGE_INTEGER _In_ ULONG _In_ PFILE_OBJECT _In_ PVOID Process
Definition: fsrtlfuncs.h:219
_In_ HANDLE Handle
Definition: extypes.h:390
FORCEINLINE VOID _KeAcquireGuardedMutexUnsafe(IN OUT PKGUARDED_MUTEX GuardedMutex)
Definition: ke_x.h:1517
#define OUT
Definition: typedefs.h:40
USHORT Number
Definition: ketypes.h:563
unsigned int ULONG
Definition: retypes.h:1
#define IPI_APC
Definition: ketypes.h:236
#define KeLeaveGuardedRegionThread(_Thread)
Definition: ke_x.h:48
VOID FASTCALL KeReleaseInStackQueuedSpinLockFromDpcLevel(IN PKLOCK_QUEUE_HANDLE LockHandle)
Definition: spinlock.c:369
#define KOBJECT_LOCK_BIT
Definition: ketypes.h:121
FORCEINLINE VOID KxUnwaitThread(IN DISPATCHER_HEADER *Object, IN KPRIORITY Increment)
Definition: ke_x.h:1259
EX_PUSH_LOCK ThreadLock
Definition: pstypes.h:1160
#define KeGetCurrentThread
Definition: hal.h:50
FORCEINLINE NTSTATUS KiCheckAlertability(IN PKTHREAD Thread, IN BOOLEAN Alertable, IN KPROCESSOR_MODE WaitMode)
Definition: ke_x.h:841
IN OUT PLONG IN OUT PLONG Addend IN OUT PLONG IN LONG Increment
Definition: CrNtStubs.h:42
VOID FASTCALL KeInitializeGate(PKGATE Gate)
#define GM_LOCK_BIT_V
signed int * PLONG
Definition: retypes.h:5
#define APC_LEVEL
Definition: env_spec_w32.h:695
ULONGLONG NTAPI KeQueryInterruptTime(VOID)
Definition: clock.c:203
FORCEINLINE ULONG KiComputeTimerTableIndex(IN ULONGLONG DueTime)
Definition: ke_x.h:881
#define GM_LOCK_WAITER_WOKEN
KIRQL FASTCALL KeAcquireQueuedSpinLockRaiseToSynch(IN KSPIN_LOCK_QUEUE_NUMBER LockNumber)
Definition: spinlock.c:119
FORCEINLINE VOID KiReleaseTimerLock(IN PKSPIN_LOCK_QUEUE LockQueue)
Definition: ke_x.h:297
LONGLONG QuadPart
Definition: typedefs.h:114
FORCEINLINE VOID KiAcquireDispatcherLockAtSynchLevel(VOID)
Definition: ke_x.h:165
VOID FASTCALL KiProcessDeferredReadyList(IN PKPRCB Prcb)
Definition: thrdschd.c:41