ReactOS  r76032
ke_x.h
Go to the documentation of this file.
1 /*
2 * PROJECT: ReactOS Kernel
3 * LICENSE: GPL - See COPYING in the top level directory
4 * FILE: ntoskrnl/include/internal/ke_x.h
5 * PURPOSE: Internal Inlined Functions for the Kernel
6 * PROGRAMMERS: Alex Ionescu (alex.ionescu@reactos.org)
7 */
8 
9 #ifndef _M_ARM
13 {
14  /* Return the current mode */
15  return KeGetCurrentThread()->PreviousMode;
16 }
17 #endif
18 
19 //
20 // Enters a Guarded Region
21 //
22 #define KeEnterGuardedRegionThread(_Thread) \
23 { \
24  /* Sanity checks */ \
25  ASSERT(KeGetCurrentIrql() <= APC_LEVEL); \
26  ASSERT(_Thread == KeGetCurrentThread()); \
27  ASSERT((_Thread->SpecialApcDisable <= 0) && \
28  (_Thread->SpecialApcDisable != -32768)); \
29  \
30  /* Disable Special APCs */ \
31  _Thread->SpecialApcDisable--; \
32 }
33 
34 #define KeEnterGuardedRegion() \
35 { \
36  PKTHREAD _Thread = KeGetCurrentThread(); \
37  KeEnterGuardedRegionThread(_Thread); \
38 }
39 
40 //
41 // Leaves a Guarded Region
42 //
43 #define KeLeaveGuardedRegionThread(_Thread) \
44 { \
45  /* Sanity checks */ \
46  ASSERT(KeGetCurrentIrql() <= APC_LEVEL); \
47  ASSERT(_Thread == KeGetCurrentThread()); \
48  ASSERT(_Thread->SpecialApcDisable < 0); \
49  \
50  /* Leave region and check if APCs are OK now */ \
51  if (!(++_Thread->SpecialApcDisable)) \
52  { \
53  /* Check for Kernel APCs on the list */ \
54  if (!IsListEmpty(&_Thread->ApcState. \
55  ApcListHead[KernelMode])) \
56  { \
57  /* Check for APC Delivery */ \
58  KiCheckForKernelApcDelivery(); \
59  } \
60  } \
61 }
62 
63 #define KeLeaveGuardedRegion() \
64 { \
65  PKTHREAD _Thread = KeGetCurrentThread(); \
66  KeLeaveGuardedRegionThread(_Thread); \
67 }
68 
69 //
70 // Enters a Critical Region
71 //
72 #define KeEnterCriticalRegionThread(_Thread) \
73 { \
74  /* Sanity checks */ \
75  ASSERT(_Thread == KeGetCurrentThread()); \
76  ASSERT((_Thread->KernelApcDisable <= 0) && \
77  (_Thread->KernelApcDisable != -32768)); \
78  \
79  /* Disable Kernel APCs */ \
80  _Thread->KernelApcDisable--; \
81 }
82 
83 #define KeEnterCriticalRegion() \
84 { \
85  PKTHREAD _Thread = KeGetCurrentThread(); \
86  KeEnterCriticalRegionThread(_Thread); \
87 }
88 
89 //
90 // Leaves a Critical Region
91 //
92 #define KeLeaveCriticalRegionThread(_Thread) \
93 { \
94  /* Sanity checks */ \
95  ASSERT(_Thread == KeGetCurrentThread()); \
96  ASSERT(_Thread->KernelApcDisable < 0); \
97  \
98  /* Enable Kernel APCs */ \
99  _Thread->KernelApcDisable++; \
100  \
101  /* Check if Kernel APCs are now enabled */ \
102  if (!(_Thread->KernelApcDisable)) \
103  { \
104  /* Check if we need to request an APC Delivery */ \
105  if (!(IsListEmpty(&_Thread->ApcState.ApcListHead[KernelMode])) && \
106  !(_Thread->SpecialApcDisable)) \
107  { \
108  /* Check for the right environment */ \
109  KiCheckForKernelApcDelivery(); \
110  } \
111  } \
112 }
113 
114 #define KeLeaveCriticalRegion() \
115 { \
116  PKTHREAD _Thread = KeGetCurrentThread(); \
117  KeLeaveCriticalRegionThread(_Thread); \
118 }
119 
120 #ifndef CONFIG_SMP
121 
122 //
123 // This routine protects against multiple CPU acquires, it's meaningless on UP.
124 //
126 VOID
128 {
129  UNREFERENCED_PARAMETER(Object);
130 }
131 
132 //
133 // This routine protects against multiple CPU acquires, it's meaningless on UP.
134 //
136 VOID
138 {
139  UNREFERENCED_PARAMETER(Object);
140 }
141 
143 KIRQL
145 {
146  /* Raise to synch level */
147  return KfRaiseIrql(SYNCH_LEVEL);
148 }
149 
151 VOID
153 {
154  /* Just exit the dispatcher */
155  KiExitDispatcher(OldIrql);
156 }
157 
159 VOID
161 {
162  /* This is a no-op at DPC Level for UP systems */
163  return;
164 }
165 
167 VOID
169 {
170  /* This is a no-op at DPC Level for UP systems */
171  return;
172 }
173 
174 //
175 // This routine makes the thread deferred ready on the boot CPU.
176 //
178 VOID
180 {
181  /* Set the thread to deferred state and boot CPU */
182  Thread->State = DeferredReady;
183  Thread->DeferredProcessor = 0;
184 
185  /* Make the thread ready immediately */
186  KiDeferredReadyThread(Thread);
187 }
188 
190 VOID
192  IN ULONG Cpu)
193 {
194  /* This is meaningless on UP systems */
195  UNREFERENCED_PARAMETER(NewThread);
197 }
198 
199 //
200 // This routine protects against multiple CPU acquires, it's meaningless on UP.
201 //
203 VOID
205 {
206  UNREFERENCED_PARAMETER(Thread);
207 }
208 
209 //
210 // This routine protects against multiple CPU acquires, it's meaningless on UP.
211 //
213 VOID
215 {
217 }
218 
219 //
220 // This routine protects against multiple CPU acquires, it's meaningless on UP.
221 //
223 VOID
225 {
227 }
228 
229 //
230 // This routine protects against multiple CPU acquires, it's meaningless on UP.
231 //
233 VOID
235 {
236  UNREFERENCED_PARAMETER(Thread);
237 }
238 
239 //
240 // This routine protects against multiple CPU acquires, it's meaningless on UP.
241 //
243 VOID
245 {
246  UNREFERENCED_PARAMETER(Thread);
247 }
248 
249 //
250 // This routine protects against multiple CPU acquires, it's meaningless on UP.
251 //
253 BOOLEAN
255 {
256  UNREFERENCED_PARAMETER(Thread);
257  return FALSE;
258 }
259 
261 VOID
263 {
264  /* There are no deferred ready lists on UP systems */
266 }
267 
269 VOID
272 {
273  /* We deliver instantly on UP */
274  UNREFERENCED_PARAMETER(NeedApc);
275  UNREFERENCED_PARAMETER(Processor);
276 }
277 
281 {
283 
284  /* Nothing to do on UP */
286  return NULL;
287 }
288 
290 VOID
292 {
294 
295  /* Nothing to do on UP */
296  UNREFERENCED_PARAMETER(LockQueue);
297 }
298 
299 #else
300 
302 VOID
304 {
305  LONG OldValue;
306 
307  /* Make sure we're at a safe level to touch the lock */
309 
310  /* Start acquire loop */
311  do
312  {
313  /* Loop until the other CPU releases it */
314  while (TRUE)
315  {
316  /* Check if it got released */
317  OldValue = Object->Lock;
318  if ((OldValue & KOBJECT_LOCK_BIT) == 0) break;
319 
320  /* Let the CPU know that this is a loop */
321  YieldProcessor();
322  }
323 
324  /* Try acquiring the lock now */
325  } while (InterlockedCompareExchange(&Object->Lock,
326  OldValue | KOBJECT_LOCK_BIT,
327  OldValue) != OldValue);
328 }
329 
331 VOID
333 {
334  /* Make sure we're at a safe level to touch the lock */
336 
337  /* Release it */
338  InterlockedAnd(&Object->Lock, ~KOBJECT_LOCK_BIT);
339 }
340 
342 KIRQL
344 {
345  /* Raise to synchronization level and acquire the dispatcher lock */
347 }
348 
350 VOID
352 {
353  /* First release the lock */
355  LockQueue[LockQueueDispatcherLock]);
356 
357  /* Then exit the dispatcher */
358  KiExitDispatcher(OldIrql);
359 }
360 
362 VOID
364 {
365  /* Acquire the dispatcher lock */
367  LockQueue[LockQueueDispatcherLock]);
368 }
369 
371 VOID
373 {
374  /* Release the dispatcher lock */
376  LockQueue[LockQueueDispatcherLock]);
377 }
378 
379 //
380 // This routine inserts a thread into the deferred ready list of the current CPU
381 //
383 VOID
385 {
386  PKPRCB Prcb = KeGetCurrentPrcb();
387 
388  /* Set the thread to deferred state and CPU */
389  Thread->State = DeferredReady;
390  Thread->DeferredProcessor = Prcb->Number;
391 
392  /* Add it on the list */
393  PushEntryList(&Prcb->DeferredReadyListHead, &Thread->SwapListEntry);
394 }
395 
397 VOID
398 KiRescheduleThread(IN BOOLEAN NewThread,
399  IN ULONG Cpu)
400 {
401  /* Check if a new thread needs to be scheduled on a different CPU */
402  if ((NewThread) && !(KeGetCurrentPrcb()->Number == Cpu))
403  {
404  /* Send an IPI to request delivery */
406  }
407 }
408 
409 //
410 // This routine sets the current thread in a swap busy state, which ensure that
411 // nobody else tries to swap it concurrently.
412 //
414 VOID
416 {
417  /* Make sure nobody already set it */
418  ASSERT(Thread->SwapBusy == FALSE);
419 
420  /* Set it ourselves */
421  Thread->SwapBusy = TRUE;
422 }
423 
424 //
425 // This routine acquires the PRCB lock so that only one caller can touch
426 // volatile PRCB data.
427 //
428 // Since this is a simple optimized spin-lock, it must only be acquired
429 // at dispatcher level or higher!
430 //
432 VOID
434 {
435  /* Make sure we're at a safe level to touch the PRCB lock */
437 
438  /* Start acquire loop */
439  for (;;)
440  {
441  /* Acquire the lock and break out if we acquired it first */
442  if (!InterlockedExchange((PLONG)&Prcb->PrcbLock, 1)) break;
443 
444  /* Loop until the other CPU releases it */
445  do
446  {
447  /* Let the CPU know that this is a loop */
448  YieldProcessor();
449  } while (Prcb->PrcbLock);
450  }
451 }
452 
453 //
454 // This routine releases the PRCB lock so that other callers can touch
455 // volatile PRCB data.
456 //
457 // Since this is a simple optimized spin-lock, it must be be only acquired
458 // at dispatcher level or higher!
459 //
461 VOID
463 {
464  /* Make sure we are above dispatch and the lock is acquired! */
466  ASSERT(Prcb->PrcbLock != 0);
467 
468  /* Release it */
469  InterlockedAnd((PLONG)&Prcb->PrcbLock, 0);
470 }
471 
472 //
473 // This routine acquires the thread lock so that only one caller can touch
474 // volatile thread data.
475 //
476 // Since this is a simple optimized spin-lock, it must be be only acquired
477 // at dispatcher level or higher!
478 //
480 VOID
482 {
483  /* Make sure we're at a safe level to touch the thread lock */
485 
486  /* Start acquire loop */
487  for (;;)
488  {
489  /* Acquire the lock and break out if we acquired it first */
490  if (!InterlockedExchange((PLONG)&Thread->ThreadLock, 1)) break;
491 
492  /* Loop until the other CPU releases it */
493  do
494  {
495  /* Let the CPU know that this is a loop */
496  YieldProcessor();
497  } while (Thread->ThreadLock);
498  }
499 }
500 
501 //
502 // This routine releases the thread lock so that other callers can touch
503 // volatile thread data.
504 //
505 // Since this is a simple optimized spin-lock, it must be be only acquired
506 // at dispatcher level or higher!
507 //
509 VOID
511 {
512  /* Make sure we are still above dispatch */
514 
515  /* Release it */
516  InterlockedAnd((PLONG)&Thread->ThreadLock, 0);
517 }
518 
520 BOOLEAN
522 {
523  LONG Value;
524 
525  /* If the lock isn't acquired, return false */
526  if (!Thread->ThreadLock) return FALSE;
527 
528  /* Otherwise, try to acquire it and check the result */
529  Value = 1;
530  Value = InterlockedExchange((PLONG)&Thread->ThreadLock, Value);
531 
532  /* Return the lock state */
533  return (Value == 1);
534 }
535 
537 VOID
539 {
540  /* Scan the deferred ready lists if required */
541  if (Prcb->DeferredReadyListHead.Next) KiProcessDeferredReadyList(Prcb);
542 }
543 
545 VOID
548 {
549  /* Check if we need to request APC delivery */
550  if (NeedApc)
551  {
552  /* Check if it's on another CPU */
553  if (KeGetCurrentPrcb()->Number != Processor)
554  {
555  /* Send an IPI to request delivery */
556  KiIpiSend(AFFINITY_MASK(Processor), IPI_APC);
557  }
558  else
559  {
560  /* Request a software interrupt */
562  }
563  }
564 }
565 
569 {
570  PKSPIN_LOCK_QUEUE LockQueue;
571  ULONG LockIndex;
573 
574  /* Get the lock index */
575  LockIndex = Hand >> LOCK_QUEUE_TIMER_LOCK_SHIFT;
576  LockIndex &= (LOCK_QUEUE_TIMER_TABLE_LOCKS - 1);
577 
578  /* Now get the lock */
579  LockQueue = &KeGetCurrentPrcb()->LockQueue[LockQueueTimerTableLock + LockIndex];
580 
581  /* Acquire it and return */
583  return LockQueue;
584 }
585 
587 VOID
589 {
591 
592  /* Release the lock */
594 }
595 
596 #endif
597 
599 VOID
602 {
603  /* Acquire the lock and raise to synchronization level */
604  KeAcquireInStackQueuedSpinLockRaiseToSynch(&Thread->ApcQueueLock, Handle);
605 }
606 
608 VOID
611 {
612  /* Acquire the lock */
613  KeAcquireInStackQueuedSpinLockAtDpcLevel(&Thread->ApcQueueLock, Handle);
614 }
615 
617 VOID
620 {
621  /* Acquire the lock */
622  KeAcquireInStackQueuedSpinLock(&Thread->ApcQueueLock, Handle);
623 }
624 
626 VOID
628 {
629  /* Release the lock */
631 }
632 
634 VOID
636 {
637  /* Release the lock */
639 }
640 
642 VOID
645 {
646  /* Acquire the lock and raise to synchronization level */
647  KeAcquireInStackQueuedSpinLockRaiseToSynch(&Process->ProcessLock, Handle);
648 }
649 
651 VOID
653 {
654  /* Release the lock */
656 }
657 
659 VOID
661 {
662  /* Release the lock */
664 }
665 
667 VOID
669  IN PKLOCK_QUEUE_HANDLE DeviceLock)
670 {
671  /* Check if we were called from a threaded DPC */
672  if (KeGetCurrentPrcb()->DpcThreadActive)
673  {
674  /* Lock the Queue, we're not at DPC level */
675  KeAcquireInStackQueuedSpinLock(&DeviceQueue->Lock, DeviceLock);
676  }
677  else
678  {
679  /* We must be at DPC level, acquire the lock safely */
681  KeAcquireInStackQueuedSpinLockAtDpcLevel(&DeviceQueue->Lock,
682  DeviceLock);
683  }
684 }
685 
687 VOID
689 {
690  /* Check if we were called from a threaded DPC */
691  if (KeGetCurrentPrcb()->DpcThreadActive)
692  {
693  /* Unlock the Queue, we're not at DPC level */
694  KeReleaseInStackQueuedSpinLock(DeviceLock);
695  }
696  else
697  {
698  /* We must be at DPC level, release the lock safely */
701  }
702 }
703 
704 //
705 // Satisfies the wait of a mutant dispatcher object
706 //
707 #define KiSatisfyMutantWait(Object, Thread) \
708 { \
709  /* Decrease the Signal State */ \
710  (Object)->Header.SignalState--; \
711  \
712  /* Check if it's now non-signaled */ \
713  if (!(Object)->Header.SignalState) \
714  { \
715  /* Set the Owner Thread */ \
716  (Object)->OwnerThread = Thread; \
717  \
718  /* Disable APCs if needed */ \
719  Thread->KernelApcDisable = Thread->KernelApcDisable - \
720  (Object)->ApcDisable; \
721  \
722  /* Check if it's abandoned */ \
723  if ((Object)->Abandoned) \
724  { \
725  /* Unabandon it */ \
726  (Object)->Abandoned = FALSE; \
727  \
728  /* Return Status */ \
729  Thread->WaitStatus = STATUS_ABANDONED; \
730  } \
731  \
732  /* Insert it into the Mutant List */ \
733  InsertHeadList(Thread->MutantListHead.Blink, \
734  &(Object)->MutantListEntry); \
735  } \
736 }
737 
738 //
739 // Satisfies the wait of any nonmutant dispatcher object
740 //
741 #define KiSatisfyNonMutantWait(Object) \
742 { \
743  if (((Object)->Header.Type & TIMER_OR_EVENT_TYPE) == \
744  EventSynchronizationObject) \
745  { \
746  /* Synchronization Timers and Events just get un-signaled */ \
747  (Object)->Header.SignalState = 0; \
748  } \
749  else if ((Object)->Header.Type == SemaphoreObject) \
750  { \
751  /* These ones can have multiple states, so we only decrease it */ \
752  (Object)->Header.SignalState--; \
753  } \
754 }
755 
756 //
757 // Satisfies the wait of any dispatcher object
758 //
759 #define KiSatisfyObjectWait(Object, Thread) \
760 { \
761  /* Special case for Mutants */ \
762  if ((Object)->Header.Type == MutantObject) \
763  { \
764  KiSatisfyMutantWait((Object), (Thread)); \
765  } \
766  else \
767  { \
768  KiSatisfyNonMutantWait(Object); \
769  } \
770 }
771 
772 //
773 // Recalculates the due time
774 //
779  IN OUT PLARGE_INTEGER NewDueTime)
780 {
781  /* Don't do anything for absolute waits */
782  if (OriginalDueTime->QuadPart >= 0) return OriginalDueTime;
783 
784  /* Otherwise, query the interrupt time and recalculate */
785  NewDueTime->QuadPart = KeQueryInterruptTime();
786  NewDueTime->QuadPart -= DueTime->QuadPart;
787  return NewDueTime;
788 }
789 
790 //
791 // Determines whether a thread should be added to the wait list
792 //
794 BOOLEAN
796  IN KPROCESSOR_MODE WaitMode)
797 {
798  /* Check the required conditions */
799  if ((WaitMode != KernelMode) &&
800  (Thread->EnableStackSwap) &&
801  (Thread->Priority >= (LOW_REALTIME_PRIORITY + 9)))
802  {
803  /* We are go for swap */
804  return TRUE;
805  }
806  else
807  {
808  /* Don't swap the thread */
809  return FALSE;
810  }
811 }
812 
813 //
814 // Adds a thread to the wait list
815 //
816 #define KiAddThreadToWaitList(Thread, Swappable) \
817 { \
818  /* Make sure it's swappable */ \
819  if (Swappable) \
820  { \
821  /* Insert it into the PRCB's List */ \
822  InsertTailList(&KeGetCurrentPrcb()->WaitListHead, \
823  &Thread->WaitListEntry); \
824  } \
825 }
826 
827 //
828 // Checks if a wait in progress should be interrupted by APCs or an alertable
829 // state.
830 //
832 NTSTATUS
835  IN KPROCESSOR_MODE WaitMode)
836 {
837  /* Check if the wait is alertable */
838  if (Alertable)
839  {
840  /* It is, first check if the thread is alerted in this mode */
841  if (Thread->Alerted[WaitMode])
842  {
843  /* It is, so bail out of the wait */
844  Thread->Alerted[WaitMode] = FALSE;
845  return STATUS_ALERTED;
846  }
847  else if ((WaitMode != KernelMode) &&
848  (!IsListEmpty(&Thread->ApcState.ApcListHead[UserMode])))
849  {
850  /* It's isn't, but this is a user wait with queued user APCs */
851  Thread->ApcState.UserApcPending = TRUE;
852  return STATUS_USER_APC;
853  }
854  else if (Thread->Alerted[KernelMode])
855  {
856  /* It isn't that either, but we're alered in kernel mode */
857  Thread->Alerted[KernelMode] = FALSE;
858  return STATUS_ALERTED;
859  }
860  }
861  else if ((WaitMode != KernelMode) && (Thread->ApcState.UserApcPending))
862  {
863  /* Not alertable, but this is a user wait with pending user APCs */
864  return STATUS_USER_APC;
865  }
866 
867  /* Otherwise, we're fine */
868  return STATUS_WAIT_0;
869 }
870 
872 ULONG
874 {
875  return (DueTime / KeMaximumIncrement) & (TIMER_TABLE_SIZE - 1);
876 }
877 
878 //
879 // Called from KiCompleteTimer, KiInsertTreeTimer, KeSetSystemTime
880 // to remove timer entries
881 // See Windows HPI blog for more information.
883 VOID
885 {
886  ULONG Hand;
888 
889  /* Remove the timer from the timer list and check if it's empty */
890  Hand = Timer->Header.Hand;
891  if (RemoveEntryList(&Timer->TimerListEntry))
892  {
893  /* Get the respective timer table entry */
894  TableEntry = &KiTimerTableListHead[Hand];
895  if (&TableEntry->Entry == TableEntry->Entry.Flink)
896  {
897  /* Set the entry to an infinite absolute time */
898  TableEntry->Time.HighPart = 0xFFFFFFFF;
899  }
900  }
901 
902  /* Clear the list entries on dbg builds so we can tell the timer is gone */
903 #if DBG
904  Timer->TimerListEntry.Flink = NULL;
905  Timer->TimerListEntry.Blink = NULL;
906 #endif
907 }
908 
909 //
910 // Called by Wait and Queue code to insert a timer for dispatching.
911 // Also called by KeSetTimerEx to insert a timer from the caller.
912 //
914 VOID
916  IN ULONG Hand)
917 {
918  PKSPIN_LOCK_QUEUE LockQueue;
919 
920  /* Acquire the lock and release the dispatcher lock */
921  LockQueue = KiAcquireTimerLock(Hand);
923 
924  /* Try to insert the timer */
925  if (KiInsertTimerTable(Timer, Hand))
926  {
927  /* Complete it */
928  KiCompleteTimer(Timer, LockQueue);
929  }
930  else
931  {
932  /* Do nothing, just release the lock */
933  KiReleaseTimerLock(LockQueue);
934  }
935 }
936 
937 //
938 // Called by KeSetTimerEx and KiInsertTreeTimer to calculate Due Time
939 // See the Windows HPI Blog for more information
940 //
942 BOOLEAN
945  OUT PULONG Hand)
946 {
947  LARGE_INTEGER InterruptTime, SystemTime, DifferenceTime;
948 
949  /* Convert to relative time if needed */
950  Timer->Header.Absolute = FALSE;
951  if (DueTime.HighPart >= 0)
952  {
953  /* Get System Time */
954  KeQuerySystemTime(&SystemTime);
955 
956  /* Do the conversion */
957  DifferenceTime.QuadPart = SystemTime.QuadPart - DueTime.QuadPart;
958 
959  /* Make sure it hasn't already expired */
960  Timer->Header.Absolute = TRUE;
961  if (DifferenceTime.HighPart >= 0)
962  {
963  /* Cancel everything */
964  Timer->Header.SignalState = TRUE;
965  Timer->Header.Hand = 0;
966  Timer->DueTime.QuadPart = 0;
967  *Hand = 0;
968  return FALSE;
969  }
970 
971  /* Set the time as Absolute */
972  DueTime = DifferenceTime;
973  }
974 
975  /* Get the Interrupt Time */
976  InterruptTime.QuadPart = KeQueryInterruptTime();
977 
978  /* Recalculate due time */
979  Timer->DueTime.QuadPart = InterruptTime.QuadPart - DueTime.QuadPart;
980 
981  /* Get the handle */
982  *Hand = KiComputeTimerTableIndex(Timer->DueTime.QuadPart);
983  Timer->Header.Hand = (UCHAR)*Hand;
984  Timer->Header.Inserted = TRUE;
985  return TRUE;
986 }
987 
988 //
989 // Called from Unlink and Queue Insert Code.
990 // Also called by timer code when canceling an inserted timer.
991 // Removes a timer from it's tree.
992 //
994 VOID
996 {
997  ULONG Hand = Timer->Header.Hand;
998  PKSPIN_LOCK_QUEUE LockQueue;
999  PKTIMER_TABLE_ENTRY TimerEntry;
1000 
1001  /* Acquire timer lock */
1002  LockQueue = KiAcquireTimerLock(Hand);
1003 
1004  /* Set the timer as non-inserted */
1005  Timer->Header.Inserted = FALSE;
1006 
1007  /* Remove it from the timer list */
1008  if (RemoveEntryList(&Timer->TimerListEntry))
1009  {
1010  /* Get the entry and check if it's empty */
1011  TimerEntry = &KiTimerTableListHead[Hand];
1012  if (IsListEmpty(&TimerEntry->Entry))
1013  {
1014  /* Clear the time then */
1015  TimerEntry->Time.HighPart = 0xFFFFFFFF;
1016  }
1017  }
1018 
1019  /* Release the timer lock */
1020  KiReleaseTimerLock(LockQueue);
1021 }
1022 
1024 VOID
1027  OUT PULONG Hand)
1028 {
1030  LARGE_INTEGER InterruptTime, SystemTime, TimeDifference;
1031 
1032  /* Check the timer's interval to see if it's absolute */
1033  Timer->Header.Absolute = FALSE;
1034  if (Interval.HighPart >= 0)
1035  {
1036  /* Get the system time and calculate the relative time */
1037  KeQuerySystemTime(&SystemTime);
1038  TimeDifference.QuadPart = SystemTime.QuadPart - Interval.QuadPart;
1039  Timer->Header.Absolute = TRUE;
1040 
1041  /* Check if we've already expired */
1042  if (TimeDifference.HighPart >= 0)
1043  {
1044  /* Reset everything */
1045  Timer->DueTime.QuadPart = 0;
1046  *Hand = 0;
1047  Timer->Header.Hand = 0;
1048  return;
1049  }
1050  else
1051  {
1052  /* Update the interval */
1053  Interval = TimeDifference;
1054  }
1055  }
1056 
1057  /* Calculate the due time */
1058  InterruptTime.QuadPart = KeQueryInterruptTime();
1059  DueTime = InterruptTime.QuadPart - Interval.QuadPart;
1060  Timer->DueTime.QuadPart = DueTime;
1061 
1062  /* Calculate the timer handle */
1063  *Hand = KiComputeTimerTableIndex(DueTime);
1064  Timer->Header.Hand = (UCHAR)*Hand;
1065 }
1066 
1067 #define KxDelayThreadWait() \
1068  \
1069  /* Setup the Wait Block */ \
1070  Thread->WaitBlockList = TimerBlock; \
1071  \
1072  /* Setup the timer */ \
1073  KxSetTimerForThreadWait(Timer, *Interval, &Hand); \
1074  \
1075  /* Save the due time for the caller */ \
1076  DueTime.QuadPart = Timer->DueTime.QuadPart; \
1077  \
1078  /* Link the timer to this Wait Block */ \
1079  TimerBlock->NextWaitBlock = TimerBlock; \
1080  Timer->Header.WaitListHead.Flink = &TimerBlock->WaitListEntry; \
1081  Timer->Header.WaitListHead.Blink = &TimerBlock->WaitListEntry; \
1082  \
1083  /* Clear wait status */ \
1084  Thread->WaitStatus = STATUS_SUCCESS; \
1085  \
1086  /* Setup wait fields */ \
1087  Thread->Alertable = Alertable; \
1088  Thread->WaitReason = DelayExecution; \
1089  Thread->WaitMode = WaitMode; \
1090  \
1091  /* Check if we can swap the thread's stack */ \
1092  Thread->WaitListEntry.Flink = NULL; \
1093  Swappable = KiCheckThreadStackSwap(Thread, WaitMode); \
1094  \
1095  /* Set the wait time */ \
1096  Thread->WaitTime = KeTickCount.LowPart;
1097 
1098 #define KxMultiThreadWait() \
1099  /* Link wait block array to the thread */ \
1100  Thread->WaitBlockList = WaitBlockArray; \
1101  \
1102  /* Reset the index */ \
1103  Index = 0; \
1104  \
1105  /* Loop wait blocks */ \
1106  do \
1107  { \
1108  /* Fill out the wait block */ \
1109  WaitBlock = &WaitBlockArray[Index]; \
1110  WaitBlock->Object = Object[Index]; \
1111  WaitBlock->WaitKey = (USHORT)Index; \
1112  WaitBlock->WaitType = WaitType; \
1113  WaitBlock->Thread = Thread; \
1114  \
1115  /* Link to next block */ \
1116  WaitBlock->NextWaitBlock = &WaitBlockArray[Index + 1]; \
1117  Index++; \
1118  } while (Index < Count); \
1119  \
1120  /* Link the last block */ \
1121  WaitBlock->NextWaitBlock = WaitBlockArray; \
1122  \
1123  /* Set default wait status */ \
1124  Thread->WaitStatus = STATUS_WAIT_0; \
1125  \
1126  /* Check if we have a timer */ \
1127  if (Timeout) \
1128  { \
1129  /* Link to the block */ \
1130  TimerBlock->NextWaitBlock = WaitBlockArray; \
1131  \
1132  /* Setup the timer */ \
1133  KxSetTimerForThreadWait(Timer, *Timeout, &Hand); \
1134  \
1135  /* Save the due time for the caller */ \
1136  DueTime.QuadPart = Timer->DueTime.QuadPart; \
1137  \
1138  /* Initialize the list */ \
1139  InitializeListHead(&Timer->Header.WaitListHead); \
1140  } \
1141  \
1142  /* Set wait settings */ \
1143  Thread->Alertable = Alertable; \
1144  Thread->WaitMode = WaitMode; \
1145  Thread->WaitReason = WaitReason; \
1146  \
1147  /* Check if we can swap the thread's stack */ \
1148  Thread->WaitListEntry.Flink = NULL; \
1149  Swappable = KiCheckThreadStackSwap(Thread, WaitMode); \
1150  \
1151  /* Set the wait time */ \
1152  Thread->WaitTime = KeTickCount.LowPart;
1153 
1154 #define KxSingleThreadWait() \
1155  /* Setup the Wait Block */ \
1156  Thread->WaitBlockList = WaitBlock; \
1157  WaitBlock->WaitKey = STATUS_SUCCESS; \
1158  WaitBlock->Object = Object; \
1159  WaitBlock->WaitType = WaitAny; \
1160  \
1161  /* Clear wait status */ \
1162  Thread->WaitStatus = STATUS_SUCCESS; \
1163  \
1164  /* Check if we have a timer */ \
1165  if (Timeout) \
1166  { \
1167  /* Setup the timer */ \
1168  KxSetTimerForThreadWait(Timer, *Timeout, &Hand); \
1169  \
1170  /* Save the due time for the caller */ \
1171  DueTime.QuadPart = Timer->DueTime.QuadPart; \
1172  \
1173  /* Pointer to timer block */ \
1174  WaitBlock->NextWaitBlock = TimerBlock; \
1175  TimerBlock->NextWaitBlock = WaitBlock; \
1176  \
1177  /* Link the timer to this Wait Block */ \
1178  Timer->Header.WaitListHead.Flink = &TimerBlock->WaitListEntry; \
1179  Timer->Header.WaitListHead.Blink = &TimerBlock->WaitListEntry; \
1180  } \
1181  else \
1182  { \
1183  /* No timer block, just ourselves */ \
1184  WaitBlock->NextWaitBlock = WaitBlock; \
1185  } \
1186  \
1187  /* Set wait settings */ \
1188  Thread->Alertable = Alertable; \
1189  Thread->WaitMode = WaitMode; \
1190  Thread->WaitReason = WaitReason; \
1191  \
1192  /* Check if we can swap the thread's stack */ \
1193  Thread->WaitListEntry.Flink = NULL; \
1194  Swappable = KiCheckThreadStackSwap(Thread, WaitMode); \
1195  \
1196  /* Set the wait time */ \
1197  Thread->WaitTime = KeTickCount.LowPart;
1198 
1199 #define KxQueueThreadWait() \
1200  /* Setup the Wait Block */ \
1201  Thread->WaitBlockList = WaitBlock; \
1202  WaitBlock->WaitKey = STATUS_SUCCESS; \
1203  WaitBlock->Object = Queue; \
1204  WaitBlock->WaitType = WaitAny; \
1205  WaitBlock->Thread = Thread; \
1206  \
1207  /* Clear wait status */ \
1208  Thread->WaitStatus = STATUS_SUCCESS; \
1209  \
1210  /* Check if we have a timer */ \
1211  if (Timeout) \
1212  { \
1213  /* Setup the timer */ \
1214  KxSetTimerForThreadWait(Timer, *Timeout, &Hand); \
1215  \
1216  /* Save the due time for the caller */ \
1217  DueTime.QuadPart = Timer->DueTime.QuadPart; \
1218  \
1219  /* Pointer to timer block */ \
1220  WaitBlock->NextWaitBlock = TimerBlock; \
1221  TimerBlock->NextWaitBlock = WaitBlock; \
1222  \
1223  /* Link the timer to this Wait Block */ \
1224  Timer->Header.WaitListHead.Flink = &TimerBlock->WaitListEntry; \
1225  Timer->Header.WaitListHead.Blink = &TimerBlock->WaitListEntry; \
1226  } \
1227  else \
1228  { \
1229  /* No timer block, just ourselves */ \
1230  WaitBlock->NextWaitBlock = WaitBlock; \
1231  } \
1232  \
1233  /* Set wait settings */ \
1234  Thread->Alertable = FALSE; \
1235  Thread->WaitMode = WaitMode; \
1236  Thread->WaitReason = WrQueue; \
1237  \
1238  /* Check if we can swap the thread's stack */ \
1239  Thread->WaitListEntry.Flink = NULL; \
1240  Swappable = KiCheckThreadStackSwap(Thread, WaitMode); \
1241  \
1242  /* Set the wait time */ \
1243  Thread->WaitTime = KeTickCount.LowPart;
1244 
1245 //
1246 // Unwaits a Thread
1247 //
1249 VOID
1252 {
1253  PLIST_ENTRY WaitEntry, WaitList;
1254  PKWAIT_BLOCK WaitBlock;
1255  PKTHREAD WaitThread;
1256  ULONG WaitKey;
1257 
1258  /* Loop the Wait Entries */
1259  WaitList = &Object->WaitListHead;
1260  ASSERT(IsListEmpty(&Object->WaitListHead) == FALSE);
1261  WaitEntry = WaitList->Flink;
1262  do
1263  {
1264  /* Get the current wait block */
1265  WaitBlock = CONTAINING_RECORD(WaitEntry, KWAIT_BLOCK, WaitListEntry);
1266 
1267  /* Get the waiting thread */
1268  WaitThread = WaitBlock->Thread;
1269 
1270  /* Check the current Wait Mode */
1271  if (WaitBlock->WaitType == WaitAny)
1272  {
1273  /* Use the actual wait key */
1274  WaitKey = WaitBlock->WaitKey;
1275  }
1276  else
1277  {
1278  /* Otherwise, use STATUS_KERNEL_APC */
1279  WaitKey = STATUS_KERNEL_APC;
1280  }
1281 
1282  /* Unwait the thread */
1283  KiUnwaitThread(WaitThread, WaitKey, Increment);
1284 
1285  /* Next entry */
1286  WaitEntry = WaitList->Flink;
1287  } while (WaitEntry != WaitList);
1288 }
1289 
1290 //
1291 // Unwaits a Thread waiting on an event
1292 //
1294 VOID
1297 {
1298  PLIST_ENTRY WaitEntry, WaitList;
1299  PKWAIT_BLOCK WaitBlock;
1300  PKTHREAD WaitThread;
1301 
1302  /* Loop the Wait Entries */
1303  WaitList = &Event->Header.WaitListHead;
1304  ASSERT(IsListEmpty(&Event->Header.WaitListHead) == FALSE);
1305  WaitEntry = WaitList->Flink;
1306  do
1307  {
1308  /* Get the current wait block */
1309  WaitBlock = CONTAINING_RECORD(WaitEntry, KWAIT_BLOCK, WaitListEntry);
1310 
1311  /* Get the waiting thread */
1312  WaitThread = WaitBlock->Thread;
1313 
1314  /* Check the current Wait Mode */
1315  if (WaitBlock->WaitType == WaitAny)
1316  {
1317  /* Un-signal it */
1318  Event->Header.SignalState = 0;
1319 
1320  /* Un-signal the event and unwait the thread */
1321  KiUnwaitThread(WaitThread, WaitBlock->WaitKey, Increment);
1322  break;
1323  }
1324 
1325  /* Unwait the thread with STATUS_KERNEL_APC */
1326  KiUnwaitThread(WaitThread, STATUS_KERNEL_APC, Increment);
1327 
1328  /* Next entry */
1329  WaitEntry = WaitList->Flink;
1330  } while (WaitEntry != WaitList);
1331 }
1332 
1333 //
1334 // This routine queues a thread that is ready on the PRCB's ready lists.
1335 // If this thread cannot currently run on this CPU, then the thread is
1336 // added to the deferred ready list instead.
1337 //
1338 // This routine must be entered with the PRCB lock held and it will exit
1339 // with the PRCB lock released!
1340 //
1342 VOID
1344  IN PKPRCB Prcb)
1345 {
1346  BOOLEAN Preempted;
1348 
1349  /* Sanity checks */
1350  ASSERT(Prcb == KeGetCurrentPrcb());
1351  ASSERT(Thread->State == Running);
1352  ASSERT(Thread->NextProcessor == Prcb->Number);
1353 
1354  /* Check if this thread is allowed to run in this CPU */
1355 #ifdef CONFIG_SMP
1356  if ((Thread->Affinity) & (Prcb->SetMember))
1357 #else
1358  if (TRUE)
1359 #endif
1360  {
1361  /* Set thread ready for execution */
1362  Thread->State = Ready;
1363 
1364  /* Save current priority and if someone had pre-empted it */
1365  Priority = Thread->Priority;
1366  Preempted = Thread->Preempted;
1367 
1368  /* We're not pre-empting now, and set the wait time */
1369  Thread->Preempted = FALSE;
1370  Thread->WaitTime = KeTickCount.LowPart;
1371 
1372  /* Sanity check */
1373  ASSERT((Priority >= 0) && (Priority <= HIGH_PRIORITY));
1374 
1375  /* Insert this thread in the appropriate order */
1376  Preempted ? InsertHeadList(&Prcb->DispatcherReadyListHead[Priority],
1377  &Thread->WaitListEntry) :
1378  InsertTailList(&Prcb->DispatcherReadyListHead[Priority],
1379  &Thread->WaitListEntry);
1380 
1381  /* Update the ready summary */
1382  Prcb->ReadySummary |= PRIORITY_MASK(Priority);
1383 
1384  /* Sanity check */
1385  ASSERT(Priority == Thread->Priority);
1386 
1387  /* Release the PRCB lock */
1388  KiReleasePrcbLock(Prcb);
1389  }
1390  else
1391  {
1392  /* Otherwise, prepare this thread to be deferred */
1393  Thread->State = DeferredReady;
1394  Thread->DeferredProcessor = Prcb->Number;
1395 
1396  /* Release the lock and defer scheduling */
1397  KiReleasePrcbLock(Prcb);
1398  KiDeferredReadyThread(Thread);
1399  }
1400 }
1401 
1402 //
1403 // This routine scans for an appropriate ready thread to select at the
1404 // given priority and for the given CPU.
1405 //
1407 PKTHREAD
1409  IN PKPRCB Prcb)
1410 {
1411  ULONG PrioritySet;
1412  LONG HighPriority;
1413  PLIST_ENTRY ListEntry;
1414  PKTHREAD Thread = NULL;
1415 
1416  /* Save the current mask and get the priority set for the CPU */
1417  PrioritySet = Prcb->ReadySummary >> Priority;
1418  if (!PrioritySet) goto Quickie;
1419 
1420  /* Get the highest priority possible */
1421  BitScanReverse((PULONG)&HighPriority, PrioritySet);
1422  ASSERT((PrioritySet & PRIORITY_MASK(HighPriority)) != 0);
1423  HighPriority += Priority;
1424 
1425  /* Make sure the list isn't empty at the highest priority */
1426  ASSERT(IsListEmpty(&Prcb->DispatcherReadyListHead[HighPriority]) == FALSE);
1427 
1428  /* Get the first thread on the list */
1429  ListEntry = Prcb->DispatcherReadyListHead[HighPriority].Flink;
1430  Thread = CONTAINING_RECORD(ListEntry, KTHREAD, WaitListEntry);
1431 
1432  /* Make sure this thread is here for a reason */
1433  ASSERT(HighPriority == Thread->Priority);
1434  ASSERT(Thread->Affinity & AFFINITY_MASK(Prcb->Number));
1435  ASSERT(Thread->NextProcessor == Prcb->Number);
1436 
1437  /* Remove it from the list */
1438  if (RemoveEntryList(&Thread->WaitListEntry))
1439  {
1440  /* The list is empty now, reset the ready summary */
1441  Prcb->ReadySummary ^= PRIORITY_MASK(HighPriority);
1442  }
1443 
1444  /* Sanity check and return the thread */
1445 Quickie:
1446  ASSERT((Thread == NULL) ||
1447  (Thread->BasePriority == 0) ||
1448  (Thread->Priority != 0));
1449  return Thread;
1450 }
1451 
1452 //
1453 // This routine computes the new priority for a thread. It is only valid for
1454 // threads with priorities in the dynamic priority range.
1455 //
1457 SCHAR
1460 {
1461  SCHAR Priority;
1462 
1463  /* Priority sanity checks */
1464  ASSERT((Thread->PriorityDecrement >= 0) &&
1465  (Thread->PriorityDecrement <= Thread->Priority));
1466  ASSERT((Thread->Priority < LOW_REALTIME_PRIORITY) ?
1467  TRUE : (Thread->PriorityDecrement == 0));
1468 
1469  /* Get the current priority */
1470  Priority = Thread->Priority;
1471  if (Priority < LOW_REALTIME_PRIORITY)
1472  {
1473  /* Decrease priority by the priority decrement */
1474  Priority -= (Thread->PriorityDecrement + Adjustment);
1475 
1476  /* Don't go out of bounds */
1477  if (Priority < Thread->BasePriority) Priority = Thread->BasePriority;
1478 
1479  /* Reset the priority decrement */
1480  Thread->PriorityDecrement = 0;
1481  }
1482 
1483  /* Sanity check */
1484  ASSERT((Thread->BasePriority == 0) || (Priority != 0));
1485 
1486  /* Return the new priority */
1487  return Priority;
1488 }
1489 
1490 //
1491 // Guarded Mutex Routines
1492 //
1494 VOID
1496 {
1497  /* Setup the Initial Data */
1498  GuardedMutex->Count = GM_LOCK_BIT;
1499  GuardedMutex->Owner = NULL;
1500  GuardedMutex->Contention = 0;
1501 
1502  /* Initialize the Wait Gate */
1503  KeInitializeGate(&GuardedMutex->Gate);
1504 }
1505 
1507 VOID
1509 {
1510  PKTHREAD Thread = KeGetCurrentThread();
1511 
1512  /* Sanity checks */
1514  (Thread->SpecialApcDisable < 0) ||
1515  (Thread->Teb == NULL) ||
1516  (Thread->Teb >= (PTEB)MM_SYSTEM_RANGE_START));
1517  ASSERT(GuardedMutex->Owner != Thread);
1518 
1519  /* Remove the lock */
1520  if (!InterlockedBitTestAndReset(&GuardedMutex->Count, GM_LOCK_BIT_V))
1521  {
1522  /* The Guarded Mutex was already locked, enter contented case */
1523  KiAcquireGuardedMutex(GuardedMutex);
1524  }
1525 
1526  /* Set the Owner */
1527  GuardedMutex->Owner = Thread;
1528 }
1529 
1531 VOID
1533 {
1534  LONG OldValue, NewValue;
1535 
1536  /* Sanity checks */
1538  (KeGetCurrentThread()->SpecialApcDisable < 0) ||
1539  (KeGetCurrentThread()->Teb == NULL) ||
1541  ASSERT(GuardedMutex->Owner == KeGetCurrentThread());
1542 
1543  /* Destroy the Owner */
1544  GuardedMutex->Owner = NULL;
1545 
1546  /* Add the Lock Bit */
1547  OldValue = InterlockedExchangeAdd(&GuardedMutex->Count, GM_LOCK_BIT);
1548  ASSERT((OldValue & GM_LOCK_BIT) == 0);
1549 
1550  /* Check if it was already locked, but not woken */
1551  if ((OldValue) && !(OldValue & GM_LOCK_WAITER_WOKEN))
1552  {
1553  /* Update the Oldvalue to what it should be now */
1554  OldValue += GM_LOCK_BIT;
1555 
1556  /* The mutex will be woken, minus one waiter */
1557  NewValue = OldValue + GM_LOCK_WAITER_WOKEN -
1559 
1560  /* Remove the Woken bit */
1561  if (InterlockedCompareExchange(&GuardedMutex->Count,
1562  NewValue,
1563  OldValue) == OldValue)
1564  {
1565  /* Signal the Gate */
1566  KeSignalGateBoostPriority(&GuardedMutex->Gate);
1567  }
1568  }
1569 }
1570 
1572 VOID
1574 {
1575  PKTHREAD Thread = KeGetCurrentThread();
1576 
1577  /* Sanity checks */
1579  ASSERT(GuardedMutex->Owner != Thread);
1580 
1581  /* Disable Special APCs */
1583 
1584  /* Remove the lock */
1585  if (!InterlockedBitTestAndReset(&GuardedMutex->Count, GM_LOCK_BIT_V))
1586  {
1587  /* The Guarded Mutex was already locked, enter contented case */
1588  KiAcquireGuardedMutex(GuardedMutex);
1589  }
1590 
1591  /* Set the Owner and Special APC Disable state */
1592  GuardedMutex->Owner = Thread;
1593  GuardedMutex->SpecialApcDisable = Thread->SpecialApcDisable;
1594 }
1595 
1597 VOID
1599 {
1600  PKTHREAD Thread = KeGetCurrentThread();
1601  LONG OldValue, NewValue;
1602 
1603  /* Sanity checks */
1605  ASSERT(GuardedMutex->Owner == Thread);
1606  ASSERT(Thread->SpecialApcDisable == GuardedMutex->SpecialApcDisable);
1607 
1608  /* Destroy the Owner */
1609  GuardedMutex->Owner = NULL;
1610 
1611  /* Add the Lock Bit */
1612  OldValue = InterlockedExchangeAdd(&GuardedMutex->Count, GM_LOCK_BIT);
1613  ASSERT((OldValue & GM_LOCK_BIT) == 0);
1614 
1615  /* Check if it was already locked, but not woken */
1616  if ((OldValue) && !(OldValue & GM_LOCK_WAITER_WOKEN))
1617  {
1618  /* Update the Oldvalue to what it should be now */
1619  OldValue += GM_LOCK_BIT;
1620 
1621  /* The mutex will be woken, minus one waiter */
1622  NewValue = OldValue + GM_LOCK_WAITER_WOKEN -
1624 
1625  /* Remove the Woken bit */
1626  if (InterlockedCompareExchange(&GuardedMutex->Count,
1627  NewValue,
1628  OldValue) == OldValue)
1629  {
1630  /* Signal the Gate */
1631  KeSignalGateBoostPriority(&GuardedMutex->Gate);
1632  }
1633  }
1634 
1635  /* Re-enable APCs */
1637 }
1638 
1640 BOOLEAN
1642 {
1643  PKTHREAD Thread = KeGetCurrentThread();
1644 
1645  /* Block APCs */
1647 
1648  /* Remove the lock */
1649  if (!InterlockedBitTestAndReset(&GuardedMutex->Count, GM_LOCK_BIT_V))
1650  {
1651  /* Re-enable APCs */
1653  YieldProcessor();
1654 
1655  /* Return failure */
1656  return FALSE;
1657  }
1658 
1659  /* Set the Owner and APC State */
1660  GuardedMutex->Owner = Thread;
1661  GuardedMutex->SpecialApcDisable = Thread->SpecialApcDisable;
1662  return TRUE;
1663 }
1664 
1665 
1667 VOID
1669 {
1671 }
1672 
1674 VOID
1676 {
1678 }
1679 
1680 #if defined(_M_IX86) || defined(_M_AMD64)
1682 VOID
1683 KiCpuId(
1684  PCPU_INFO CpuInfo,
1685  ULONG Function)
1686 {
1687  __cpuid((INT*)CpuInfo->AsUINT32, Function);
1688 }
1689 
1691 VOID
1692 KiCpuIdEx(
1693  PCPU_INFO CpuInfo,
1694  ULONG Function,
1695  ULONG SubFunction)
1696 {
1697  __cpuidex((INT*)CpuInfo->AsUINT32, Function, SubFunction);
1698 }
1699 #endif /* _M_IX86 || _M_AMD64 */
#define KeQuerySystemTime(t)
Definition: env_spec_w32.h:570
#define KeGetCurrentIrql()
Definition: env_spec_w32.h:706
FORCEINLINE BOOLEAN KiCheckThreadStackSwap(IN PKTHREAD Thread, IN KPROCESSOR_MODE WaitMode)
Definition: ke_x.h:795
FORCEINLINE VOID KiReleaseThreadLock(IN PKTHREAD Thread)
Definition: ke_x.h:244
VOID FASTCALL KeReleaseQueuedSpinLockFromDpcLevel(IN OUT PKSPIN_LOCK_QUEUE LockQueue)
ULONG KeMaximumIncrement
Definition: clock.c:20
ULONG LowPart
Definition: ketypes.h:895
_In_ PVOID _In_ BOOLEAN Alertable
Definition: exfuncs.h:452
VOID FASTCALL KeAcquireInStackQueuedSpinLockRaiseToSynch(IN PKSPIN_LOCK SpinLock, IN PKLOCK_QUEUE_HANDLE LockHandle)
Definition: spinlock.c:142
#define IN
Definition: typedefs.h:38
FORCEINLINE VOID KxSetTimerForThreadWait(IN PKTIMER Timer, IN LARGE_INTEGER Interval, OUT PULONG Hand)
Definition: ke_x.h:1025
VOID FASTCALL KiUnwaitThread(IN PKTHREAD Thread, IN LONG_PTR WaitStatus, IN KPRIORITY Increment)
Definition: wait.c:89
#define InterlockedAnd
Definition: interlocked.h:62
#define TRUE
Definition: types.h:120
#define PRIORITY_MASK(Id)
Definition: ke.h:149
FORCEINLINE VOID KiReleasePrcbLock(IN PKPRCB Prcb)
Definition: ke_x.h:224
SHORT SpecialApcDisable
Definition: ketypes.h:1052
FORCEINLINE VOID _KeAcquireGuardedMutex(IN PKGUARDED_MUTEX GuardedMutex)
Definition: ke_x.h:1573
FORCEINLINE KPROCESSOR_MODE KeGetPreviousMode(VOID)
Definition: ke_x.h:12
FORCEINLINE VOID KiCheckDeferredReadyList(IN PKPRCB Prcb)
Definition: ke_x.h:262
FORCEINLINE VOID KiReleaseProcessLockFromDpcLevel(IN PKLOCK_QUEUE_HANDLE Handle)
Definition: ke_x.h:660
ASSERT((InvokeOnSuccess||InvokeOnError||InvokeOnCancel)?(CompletionRoutine!=NULL):TRUE)
#define UNREFERENCED_PARAMETER(P)
Definition: ntbasedef.h:315
FORCEINLINE VOID KiReleaseNmiListLock(IN KIRQL OldIrql)
Definition: ke_x.h:1675
_In_ LARGE_INTEGER DueTime
Definition: kefuncs.h:524
FORCEINLINE VOID InsertHeadList(_Inout_ PLIST_ENTRY ListHead, _Inout_ __drv_aliasesMem PLIST_ENTRY Entry)
Definition: rtlfuncs.h:201
FORCEINLINE BOOLEAN KiComputeDueTime(IN PKTIMER Timer, IN LARGE_INTEGER DueTime, OUT PULONG Hand)
Definition: ke_x.h:943
IN OUT PIRP IN ULONG IN WMIENABLEDISABLECONTROL Function
Definition: wmilib.h:37
FORCEINLINE VOID KxUnwaitThreadForEvent(IN PKEVENT Event, IN KPRIORITY Increment)
Definition: ke_x.h:1295
FORCEINLINE PKSPIN_LOCK_QUEUE KiAcquireTimerLock(IN ULONG Hand)
Definition: ke_x.h:280
FORCEINLINE VOID KiInsertDeferredReadyList(IN PKTHREAD Thread)
Definition: ke_x.h:179
FORCEINLINE VOID KiSetThreadSwapBusy(IN PKTHREAD Thread)
Definition: ke_x.h:204
FORCEINLINE VOID KiAcquireDeviceQueueLock(IN PKDEVICE_QUEUE DeviceQueue, IN PKLOCK_QUEUE_HANDLE DeviceLock)
Definition: ke_x.h:668
FORCEINLINE struct _KPRCB * KeGetCurrentPrcb(VOID)
Definition: ketypes.h:1054
_In_ KPRIORITY Priority
Definition: kefuncs.h:516
FORCEINLINE VOID KxRemoveTreeTimer(IN PKTIMER Timer)
Definition: ke_x.h:995
struct _KTHREAD * Thread
Definition: ketypes.h:445
SCHAR Priority
Definition: ketypes.h:974
FORCEINLINE VOID KiAcquireApcLock(IN PKTHREAD Thread, IN PKLOCK_QUEUE_HANDLE Handle)
Definition: ke_x.h:600
VOID FASTCALL KiDeferredReadyThread(IN PKTHREAD Thread)
Definition: thrdschd.c:79
#define InterlockedCompareExchange
Definition: interlocked.h:104
_In_ UCHAR Processor
Definition: kefuncs.h:695
VOID FASTCALL KeAcquireQueuedSpinLockAtDpcLevel(IN OUT PKSPIN_LOCK_QUEUE LockQueue)
#define AFFINITY_MASK(Id)
Definition: ke.h:148
FORCEINLINE VOID KiAcquireApcLockAtDpcLevel(IN PKTHREAD Thread, IN PKLOCK_QUEUE_HANDLE Handle)
Definition: ke_x.h:609
#define InsertTailList(ListHead, Entry)
VOID FASTCALL KeAcquireInStackQueuedSpinLock(IN PKSPIN_LOCK SpinLock, IN PKLOCK_QUEUE_HANDLE LockHandle)
Definition: spinlock.c:130
#define LOW_REALTIME_PRIORITY
#define STATUS_ALERTED
Definition: ntstatus.h:80
FORCEINLINE VOID KiReleaseApcLock(IN PKLOCK_QUEUE_HANDLE Handle)
Definition: ke_x.h:627
struct _TEB * Teb
Definition: ketypes.h:1057
#define LOCK_QUEUE_TIMER_TABLE_LOCKS
FORCEINLINE BOOLEAN KiTryThreadLock(IN PKTHREAD Thread)
Definition: ke_x.h:254
int32_t INT
Definition: typedefs.h:56
_Must_inspect_result_ FORCEINLINE BOOLEAN IsListEmpty(_In_ const LIST_ENTRY *ListHead)
Definition: rtlfuncs.h:57
FORCEINLINE VOID _KeReleaseGuardedMutexUnsafe(IN OUT PKGUARDED_MUTEX GuardedMutex)
Definition: ke_x.h:1532
LONG KPRIORITY
Definition: compat.h:454
FORCEINLINE VOID _KeInitializeGuardedMutex(OUT PKGUARDED_MUTEX GuardedMutex)
Definition: ke_x.h:1495
Definition: ketypes.h:661
FORCEINLINE BOOLEAN RemoveEntryList(_In_ PLIST_ENTRY Entry)
Definition: rtlfuncs.h:105
FORCEINLINE VOID YieldProcessor(VOID)
Definition: ke.h:32
FORCEINLINE SCHAR KiComputeNewPriority(IN PKTHREAD Thread, IN SCHAR Adjustment)
Definition: ke_x.h:1458
FORCEINLINE VOID PushEntryList(_Inout_ PSINGLE_LIST_ENTRY ListHead, _Inout_ __drv_aliasesMem PSINGLE_LIST_ENTRY Entry)
Definition: rtlfuncs.h:253
UCHAR KIRQL
Definition: env_spec_w32.h:591
FORCEINLINE VOID KxQueueReadyThread(IN PKTHREAD Thread, IN PKPRCB Prcb)
Definition: ke_x.h:1343
#define STATUS_WAIT_0
Definition: ntstatus.h:223
FORCEINLINE VOID _KeReleaseGuardedMutex(IN OUT PKGUARDED_MUTEX GuardedMutex)
Definition: ke_x.h:1598
#define FALSE
Definition: types.h:117
#define GM_LOCK_BIT
_In_ KPRIORITY _In_ LONG Adjustment
Definition: kefuncs.h:451
long LONG
Definition: pedump.c:60
#define InterlockedBitTestAndReset
Definition: interlocked.h:35
#define GM_LOCK_WAITER_INC
FORCEINLINE VOID KiAcquirePrcbLock(IN PKPRCB Prcb)
Definition: ke_x.h:214
smooth NULL
Definition: ftsmooth.c:557
#define FORCEINLINE
Definition: ntbasedef.h:213
FORCEINLINE VOID KiReleaseApcLockFromDpcLevel(IN PKLOCK_QUEUE_HANDLE Handle)
Definition: ke_x.h:635
FORCEINLINE VOID KxInsertTimer(IN PKTIMER Timer, IN ULONG Hand)
Definition: ke_x.h:915
#define STATUS_KERNEL_APC
Definition: ntstatus.h:79
#define IPI_DPC
Definition: ketypes.h:233
#define InterlockedExchangeAdd
Definition: interlocked.h:181
VOID FASTCALL KiExitDispatcher(KIRQL OldIrql)
ULARGE_INTEGER Time
Definition: ketypes.h:667
FORCEINLINE VOID KiRescheduleThread(IN BOOLEAN NewThread, IN ULONG Cpu)
Definition: ke_x.h:191
BOOLEAN FASTCALL KiInsertTimerTable(IN PKTIMER Timer, IN ULONG Hand)
Definition: timerobj.c:63
DWORD Interval
Definition: netstat.c:30
KIRQL FASTCALL KfRaiseIrql(IN KIRQL NewIrql)
Definition: pic.c:187
FORCEINLINE VOID KiAcquireThreadLock(IN PKTHREAD Thread)
Definition: ke_x.h:234
SCHAR BasePriority
Definition: ketypes.h:1209
struct _LIST_ENTRY * Flink
Definition: typedefs.h:119
_In_ HANDLE Handle
Definition: extypes.h:390
VOID FASTCALL KeReleaseInStackQueuedSpinLock(IN PKLOCK_QUEUE_HANDLE LockHandle)
Definition: spinlock.c:166
unsigned char BOOLEAN
VOID FASTCALL KeAcquireInStackQueuedSpinLockAtDpcLevel(IN PKSPIN_LOCK SpinLock, IN PKLOCK_QUEUE_HANDLE LockHandle)
Definition: spinlock.c:337
#define TIMER_TABLE_SIZE
Definition: ketypes.h:821
LIST_ENTRY Entry
Definition: ketypes.h:666
$ULONG HighPart
Definition: ntbasedef.h:569
UCHAR WaitType
Definition: ketypes.h:449
FORCEINLINE VOID KiAcquireDispatcherObject(IN DISPATCHER_HEADER *Object)
Definition: ke_x.h:127
uint64_t ULONGLONG
Definition: typedefs.h:65
USHORT WaitKey
Definition: ketypes.h:448
FORCEINLINE VOID KiRequestApcInterrupt(IN BOOLEAN NeedApc, IN UCHAR Processor)
Definition: ke_x.h:270
#define KeAcquireSpinLock(sl, irql)
Definition: env_spec_w32.h:609
FORCEINLINE VOID KiReleaseDispatcherObject(IN DISPATCHER_HEADER *Object)
Definition: ke_x.h:137
signed char SCHAR
Definition: sqltypes.h:14
_In_opt_ PFILE_OBJECT _In_opt_ PETHREAD Thread
Definition: fltkernel.h:2653
CCHAR KPROCESSOR_MODE
Definition: ketypes.h:7
KSPIN_LOCK KiNmiCallbackListLock
Definition: bug.c:33
static IUnknown Object
Definition: main.c:512
LIST_ENTRY WaitListEntry
Definition: ketypes.h:1041
FORCEINLINE VOID KiReleaseProcessLock(IN PKLOCK_QUEUE_HANDLE Handle)
Definition: ke_x.h:652
KTIMER_TABLE_ENTRY KiTimerTableListHead[TIMER_TABLE_SIZE]
Definition: timerobj.c:17
#define KeEnterGuardedRegionThread(_Thread)
Definition: ke_x.h:22
unsigned char UCHAR
Definition: xmlstorage.h:181
VOID FASTCALL KiCompleteTimer(IN PKTIMER Timer, IN PKSPIN_LOCK_QUEUE LockQueue)
Definition: timerobj.c:167
NTKERNELAPI volatile KSYSTEM_TIME KeTickCount
Definition: clock.c:19
_Requires_lock_held_ Interrupt _Releases_lock_ Interrupt _In_ _IRQL_restores_ KIRQL OldIrql
Definition: kefuncs.h:803
#define BitScanReverse
Definition: interlocked.h:6
FORCEINLINE VOID KiAcquireProcessLock(IN PKPROCESS Process, IN PKLOCK_QUEUE_HANDLE Handle)
Definition: ke_x.h:643
Definition: ketypes.h:370
KIRQL * PKIRQL
Definition: env_spec_w32.h:592
Definition: typedefs.h:117
#define SYNCH_LEVEL
Definition: env_spec_w32.h:704
volatile ULONG NextProcessor
Definition: ketypes.h:977
#define MM_SYSTEM_RANGE_START
Definition: armddk.h:18
VOID FASTCALL KiIpiSend(KAFFINITY TargetSet, ULONG IpiRequest)
_In_opt_ PENTER_STATE_SYSTEM_HANDLER _In_opt_ PVOID _In_ LONG _In_opt_ LONG volatile * Number
Definition: ntpoapi.h:204
#define LOCK_QUEUE_TIMER_LOCK_SHIFT
UINTN UINT8 Value
Definition: acefiex.h:751
#define InterlockedExchange
Definition: armddk.h:54
#define DISPATCH_LEVEL
Definition: env_spec_w32.h:696
#define STATUS_USER_APC
Definition: ntstatus.h:78
UINT32 AsUINT32[4]
Definition: ketypes.h:293
SINGLE_LIST_ENTRY DeferredReadyListHead
Definition: ketypes.h:628
VOID FASTCALL HalRequestSoftwareInterrupt(IN KIRQL Irql)
Definition: pic.c:271
Definition: compat.h:484
FORCEINLINE PLARGE_INTEGER KiRecalculateDueTime(IN PLARGE_INTEGER OriginalDueTime, IN PLARGE_INTEGER DueTime, IN OUT PLARGE_INTEGER NewDueTime)
Definition: ke_x.h:777
__INTRIN_INLINE void __cpuidex(int CPUInfo[4], int InfoType, int ECXValue)
Definition: intrin_x86.h:1433
PPC_QUAL void __cpuid(int CPUInfo[], const int InfoType)
Definition: intrin_ppc.h:682
VOID FASTCALL KiAcquireGuardedMutex(IN OUT PKGUARDED_MUTEX GuardedMutex)
Definition: wait.c:122
LONG NTSTATUS
Definition: DriverTester.h:11
FORCEINLINE VOID KiReleaseDispatcherLock(IN KIRQL OldIrql)
Definition: ke_x.h:152
FORCEINLINE PKTHREAD KiSelectReadyThread(IN KPRIORITY Priority, IN PKPRCB Prcb)
Definition: ke_x.h:1408
VOID FASTCALL KeSignalGateBoostPriority(PKGATE Gate)
FORCEINLINE KIRQL KiAcquireDispatcherLock(VOID)
Definition: ke_x.h:144
FORCEINLINE BOOLEAN _KeTryToAcquireGuardedMutex(IN OUT PKGUARDED_MUTEX GuardedMutex)
Definition: ke_x.h:1641
FORCEINLINE VOID KiAcquireNmiListLock(OUT PKIRQL OldIrql)
Definition: ke_x.h:1668
unsigned int * PULONG
Definition: retypes.h:1
FORCEINLINE VOID KiRemoveEntryTimer(IN PKTIMER Timer)
Definition: ke_x.h:884
#define KeReleaseSpinLock(sl, irql)
Definition: env_spec_w32.h:627
_Must_inspect_result_ typedef _In_ ULONG TableEntry
Definition: iotypes.h:3914
FORCEINLINE VOID KiReleaseDeviceQueueLock(IN PKLOCK_QUEUE_HANDLE DeviceLock)
Definition: ke_x.h:688
FORCEINLINE VOID KiReleaseDispatcherLockFromDpcLevel(VOID)
Definition: ke_x.h:168
FORCEINLINE VOID KiAcquireApcLockAtApcLevel(IN PKTHREAD Thread, IN PKLOCK_QUEUE_HANDLE Handle)
Definition: ke_x.h:618
_Must_inspect_result_ _In_ PLARGE_INTEGER _In_ PLARGE_INTEGER _In_ ULONG _In_ PFILE_OBJECT _In_ PVOID Process
Definition: fsrtlfuncs.h:219
FORCEINLINE VOID _KeAcquireGuardedMutexUnsafe(IN OUT PKGUARDED_MUTEX GuardedMutex)
Definition: ke_x.h:1508
#define OUT
Definition: typedefs.h:39
USHORT Number
Definition: ketypes.h:559
GROUP_AFFINITY Affinity
Definition: ketypes.h:1233
unsigned int ULONG
Definition: retypes.h:1
#define CONTAINING_RECORD(address, type, field)
Definition: typedefs.h:259
#define IPI_APC
Definition: ketypes.h:232
#define KeLeaveGuardedRegionThread(_Thread)
Definition: ke_x.h:43
#define HIGH_PRIORITY
VOID FASTCALL KeReleaseInStackQueuedSpinLockFromDpcLevel(IN PKLOCK_QUEUE_HANDLE LockHandle)
Definition: spinlock.c:369
#define KOBJECT_LOCK_BIT
Definition: ketypes.h:121
FORCEINLINE VOID KxUnwaitThread(IN DISPATCHER_HEADER *Object, IN KPRIORITY Increment)
Definition: ke_x.h:1250
#define KeGetCurrentThread
Definition: hal.h:44
FORCEINLINE NTSTATUS KiCheckAlertability(IN PKTHREAD Thread, IN BOOLEAN Alertable, IN KPROCESSOR_MODE WaitMode)
Definition: ke_x.h:833
IN OUT PLONG IN OUT PLONG Addend IN OUT PLONG IN LONG Increment
Definition: CrNtStubs.h:42
VOID FASTCALL KeInitializeGate(PKGATE Gate)
#define GM_LOCK_BIT_V
signed int * PLONG
Definition: retypes.h:5
#define APC_LEVEL
Definition: env_spec_w32.h:695
ULONGLONG NTAPI KeQueryInterruptTime(VOID)
Definition: clock.c:203
FORCEINLINE ULONG KiComputeTimerTableIndex(IN ULONGLONG DueTime)
Definition: ke_x.h:873
#define GM_LOCK_WAITER_WOKEN
KIRQL FASTCALL KeAcquireQueuedSpinLockRaiseToSynch(IN KSPIN_LOCK_QUEUE_NUMBER LockNumber)
Definition: spinlock.c:119
FORCEINLINE VOID KiReleaseTimerLock(IN PKSPIN_LOCK_QUEUE LockQueue)
Definition: ke_x.h:291
FORCEINLINE VOID KiAcquireDispatcherLockAtDpcLevel(VOID)
Definition: ke_x.h:160
LONGLONG QuadPart
Definition: typedefs.h:112
VOID FASTCALL KiProcessDeferredReadyList(IN PKPRCB Prcb)
Definition: thrdschd.c:41