ReactOS  0.4.13-dev-982-g9853eab
rundown.c File Reference
#include <ntoskrnl.h>
#include <debug.h>
Include dependency graph for rundown.c:

Go to the source code of this file.

Macros

#define NDEBUG
 

Functions

BOOLEAN FASTCALL ExfAcquireRundownProtection (IN PEX_RUNDOWN_REF RunRef)
 
BOOLEAN FASTCALL ExfAcquireRundownProtectionEx (IN PEX_RUNDOWN_REF RunRef, IN ULONG Count)
 
VOID FASTCALL ExfInitializeRundownProtection (IN PEX_RUNDOWN_REF RunRef)
 
VOID FASTCALL ExfReInitializeRundownProtection (IN PEX_RUNDOWN_REF RunRef)
 
VOID FASTCALL ExfRundownCompleted (IN PEX_RUNDOWN_REF RunRef)
 
VOID FASTCALL ExfReleaseRundownProtection (IN PEX_RUNDOWN_REF RunRef)
 
VOID FASTCALL ExfReleaseRundownProtectionEx (IN PEX_RUNDOWN_REF RunRef, IN ULONG Count)
 
VOID FASTCALL ExfWaitForRundownProtectionRelease (IN PEX_RUNDOWN_REF RunRef)
 
BOOLEAN FASTCALL ExfAcquireRundownProtectionCacheAware (IN PEX_RUNDOWN_REF_CACHE_AWARE RunRefCacheAware)
 
BOOLEAN FASTCALL ExfAcquireRundownProtectionCacheAwareEx (IN PEX_RUNDOWN_REF_CACHE_AWARE RunRefCacheAware, IN ULONG Count)
 
VOID FASTCALL ExfReleaseRundownProtectionCacheAware (IN PEX_RUNDOWN_REF_CACHE_AWARE RunRefCacheAware)
 
VOID FASTCALL ExfReleaseRundownProtectionCacheAwareEx (IN PEX_RUNDOWN_REF_CACHE_AWARE RunRefCacheAware, IN ULONG Count)
 
VOID FASTCALL ExfWaitForRundownProtectionReleaseCacheAware (IN PEX_RUNDOWN_REF_CACHE_AWARE RunRefCacheAware)
 
VOID FASTCALL ExfRundownCompletedCacheAware (IN PEX_RUNDOWN_REF_CACHE_AWARE RunRefCacheAware)
 
VOID FASTCALL ExfReInitializeRundownProtectionCacheAware (IN PEX_RUNDOWN_REF_CACHE_AWARE RunRefCacheAware)
 
PEX_RUNDOWN_REF_CACHE_AWARE NTAPI ExAllocateCacheAwareRundownProtection (IN POOL_TYPE PoolType, IN ULONG Tag)
 
VOID NTAPI ExFreeCacheAwareRundownProtection (IN PEX_RUNDOWN_REF_CACHE_AWARE RunRefCacheAware)
 
VOID NTAPI ExInitializeRundownProtectionCacheAware (IN PEX_RUNDOWN_REF_CACHE_AWARE RunRefCacheAware, IN SIZE_T Size)
 
SIZE_T NTAPI ExSizeOfRundownProtectionCacheAware (VOID)
 

Macro Definition Documentation

◆ NDEBUG

#define NDEBUG

Definition at line 14 of file rundown.c.

Function Documentation

◆ ExAllocateCacheAwareRundownProtection()

PEX_RUNDOWN_REF_CACHE_AWARE NTAPI ExAllocateCacheAwareRundownProtection ( IN POOL_TYPE  PoolType,
IN ULONG  Tag 
)

Definition at line 562 of file rundown.c.

564 {
565  PEX_RUNDOWN_REF RunRef;
566  PVOID PoolToFree, RunRefs;
567  ULONG RunRefSize, Count, Align;
568  PEX_RUNDOWN_REF_CACHE_AWARE RunRefCacheAware;
569 
570  PAGED_CODE();
571 
572  /* Allocate the master structure */
573  RunRefCacheAware = ExAllocatePoolWithTag(PoolType, sizeof(EX_RUNDOWN_REF_CACHE_AWARE), Tag);
574  if (RunRefCacheAware == NULL)
575  {
576  return NULL;
577  }
578 
579  /* Compute the size of each runref */
580  RunRefCacheAware->Number = KeNumberProcessors;
581  if (KeNumberProcessors <= 1)
582  {
583  RunRefSize = sizeof(EX_RUNDOWN_REF);
584  }
585  else
586  {
588  RunRefSize = Align;
589  ASSERT((RunRefSize & (RunRefSize - 1)) == 0);
590  }
591 
592  /* It must at least hold a EX_RUNDOWN_REF structure */
593  ASSERT(sizeof(EX_RUNDOWN_REF) <= RunRefSize);
594  RunRefCacheAware->RunRefSize = RunRefSize;
595 
596  /* Allocate our runref pool */
597  PoolToFree = ExAllocatePoolWithTag(PoolType, RunRefSize * RunRefCacheAware->Number, Tag);
598  if (PoolToFree == NULL)
599  {
600  ExFreePoolWithTag(RunRefCacheAware, Tag);
601  return NULL;
602  }
603 
604  /* On SMP, check for alignment */
605  if (RunRefCacheAware->Number > 1 && (ULONG_PTR)PoolToFree & (Align - 1))
606  {
607  /* Not properly aligned, do it again! */
608  ExFreePoolWithTag(PoolToFree, Tag);
609 
610  /* Allocate a bigger buffer to be able to align properly */
611  PoolToFree = ExAllocatePoolWithTag(PoolType, RunRefSize * (RunRefCacheAware->Number + 1), Tag);
612  if (PoolToFree == NULL)
613  {
614  ExFreePoolWithTag(RunRefCacheAware, Tag);
615  return NULL;
616  }
617 
618  RunRefs = (PVOID)ALIGN_UP_BY(PoolToFree, Align);
619  }
620  else
621  {
622  RunRefs = PoolToFree;
623  }
624 
625  RunRefCacheAware->RunRefs = RunRefs;
626  RunRefCacheAware->PoolToFree = PoolToFree;
627 
628  /* And initialize runref */
629  if (RunRefCacheAware->Number != 0)
630  {
631  for (Count = 0; Count < RunRefCacheAware->Number; ++Count)
632  {
633  RunRef = ExGetRunRefForGivenProcessor(RunRefCacheAware, Count);
635  }
636  }
637 
638  return RunRefCacheAware;
639 }
_In_ SIZE_T RunRefSize
Definition: exfuncs.h:1103
PEX_RUNDOWN_REF RunRefs
Definition: extypes.h:433
FORCEINLINE VOID _ExInitializeRundownProtection(IN PEX_RUNDOWN_REF RunRef)
Definition: ex.h:890
struct _EX_RUNDOWN_REF EX_RUNDOWN_REF
_Inout_ __drv_aliasesMem PSLIST_ENTRY _Inout_ PSLIST_ENTRY _In_ ULONG Count
Definition: exfuncs.h:1015
#define PAGED_CODE()
Definition: video.h:57
ULONG NTAPI KeGetRecommendedSharedDataAlignment(VOID)
Definition: cpu.c:471
uint32_t ULONG_PTR
Definition: typedefs.h:63
smooth NULL
Definition: ftsmooth.c:416
void * PVOID
Definition: retypes.h:9
ASSERT((InvokeOnSuccess||InvokeOnError||InvokeOnCancel) ?(CompletionRoutine !=NULL) :TRUE)
#define ExAllocatePoolWithTag(hernya, size, tag)
Definition: env_spec_w32.h:350
CCHAR KeNumberProcessors
Definition: krnlinit.c:35
FORCEINLINE PEX_RUNDOWN_REF ExGetRunRefForGivenProcessor(IN PEX_RUNDOWN_REF_CACHE_AWARE RunRefCacheAware, IN ULONG ProcNumber)
Definition: ex.h:779
IN ULONG IN ULONG Tag
Definition: evtlib.h:159
unsigned int ULONG
Definition: retypes.h:1
#define ALIGN_UP_BY(size, align)
#define ExFreePoolWithTag(_P, _T)
Definition: module.h:1099
_Must_inspect_result_ _In_ FLT_CONTEXT_TYPE _In_ SIZE_T _In_ POOL_TYPE PoolType
Definition: fltkernel.h:1444

◆ ExfAcquireRundownProtection()

BOOLEAN FASTCALL ExfAcquireRundownProtection ( IN PEX_RUNDOWN_REF  RunRef)

Definition at line 36 of file rundown.c.

37 {
38  ULONG_PTR Value = RunRef->Count, NewValue;
39 
40  /* Loop until successfully incremented the counter */
41  for (;;)
42  {
43  /* Make sure a rundown is not active */
44  if (Value & EX_RUNDOWN_ACTIVE) return FALSE;
45 
46  /* Add a reference */
47  NewValue = Value + EX_RUNDOWN_COUNT_INC;
48 
49  /* Change the value */
50  NewValue = ExpChangeRundown(RunRef, NewValue, Value);
51  if (NewValue == Value) return TRUE;
52 
53  /* Update it */
54  Value = NewValue;
55  }
56 }
_In_opt_ ULONG _Out_ PULONG Value
Definition: rtlfuncs.h:2343
#define TRUE
Definition: types.h:120
#define EX_RUNDOWN_COUNT_INC
uint32_t ULONG_PTR
Definition: typedefs.h:63
#define ExpChangeRundown(x, y, z)
Definition: ex.h:158
#define EX_RUNDOWN_ACTIVE

◆ ExfAcquireRundownProtectionCacheAware()

BOOLEAN FASTCALL ExfAcquireRundownProtectionCacheAware ( IN PEX_RUNDOWN_REF_CACHE_AWARE  RunRefCacheAware)

Definition at line 384 of file rundown.c.

385 {
386  PEX_RUNDOWN_REF RunRef;
387 
388  RunRef = ExGetRunRefForGivenProcessor(RunRefCacheAware, KeGetCurrentProcessorNumber());
389  return _ExAcquireRundownProtection(RunRef);
390 }
FORCEINLINE ULONG KeGetCurrentProcessorNumber(VOID)
Definition: ke.h:325
FORCEINLINE BOOLEAN _ExAcquireRundownProtection(IN PEX_RUNDOWN_REF RunRef)
Definition: ex.h:806
FORCEINLINE PEX_RUNDOWN_REF ExGetRunRefForGivenProcessor(IN PEX_RUNDOWN_REF_CACHE_AWARE RunRefCacheAware, IN ULONG ProcNumber)
Definition: ex.h:779

◆ ExfAcquireRundownProtectionCacheAwareEx()

BOOLEAN FASTCALL ExfAcquireRundownProtectionCacheAwareEx ( IN PEX_RUNDOWN_REF_CACHE_AWARE  RunRefCacheAware,
IN ULONG  Count 
)

Definition at line 397 of file rundown.c.

399 {
400  PEX_RUNDOWN_REF RunRef;
401 
402  RunRef = ExGetRunRefForGivenProcessor(RunRefCacheAware, KeGetCurrentProcessorNumber());
403  return ExfAcquireRundownProtectionEx(RunRef, Count);
404 }
_Inout_ __drv_aliasesMem PSLIST_ENTRY _Inout_ PSLIST_ENTRY _In_ ULONG Count
Definition: exfuncs.h:1015
FORCEINLINE ULONG KeGetCurrentProcessorNumber(VOID)
Definition: ke.h:325
FORCEINLINE PEX_RUNDOWN_REF ExGetRunRefForGivenProcessor(IN PEX_RUNDOWN_REF_CACHE_AWARE RunRefCacheAware, IN ULONG ProcNumber)
Definition: ex.h:779
BOOLEAN FASTCALL ExfAcquireRundownProtectionEx(IN PEX_RUNDOWN_REF RunRef, IN ULONG Count)
Definition: rundown.c:78

◆ ExfAcquireRundownProtectionEx()

BOOLEAN FASTCALL ExfAcquireRundownProtectionEx ( IN PEX_RUNDOWN_REF  RunRef,
IN ULONG  Count 
)

Definition at line 78 of file rundown.c.

80 {
81  ULONG_PTR Value = RunRef->Count, NewValue;
82 
83  /* Loop until successfully incremented the counter */
84  for (;;)
85  {
86  /* Make sure a rundown is not active */
87  if (Value & EX_RUNDOWN_ACTIVE) return FALSE;
88 
89  /* Add references */
90  NewValue = Value + EX_RUNDOWN_COUNT_INC * Count;
91 
92  /* Change the value */
93  NewValue = ExpChangeRundown(RunRef, NewValue, Value);
94  if (NewValue == Value) return TRUE;
95 
96  /* Update the value */
97  Value = NewValue;
98  }
99 }
_In_opt_ ULONG _Out_ PULONG Value
Definition: rtlfuncs.h:2343
#define TRUE
Definition: types.h:120
_Inout_ __drv_aliasesMem PSLIST_ENTRY _Inout_ PSLIST_ENTRY _In_ ULONG Count
Definition: exfuncs.h:1015
#define EX_RUNDOWN_COUNT_INC
uint32_t ULONG_PTR
Definition: typedefs.h:63
#define ExpChangeRundown(x, y, z)
Definition: ex.h:158
#define EX_RUNDOWN_ACTIVE

Referenced by ExCompareExchangeCallBack(), ExfAcquireRundownProtectionCacheAwareEx(), and ExReferenceCallBackBlock().

◆ ExfInitializeRundownProtection()

VOID FASTCALL ExfInitializeRundownProtection ( IN PEX_RUNDOWN_REF  RunRef)

Definition at line 118 of file rundown.c.

119 {
120  /* Set the count to zero */
121  RunRef->Count = 0;
122 }

◆ ExFreeCacheAwareRundownProtection()

VOID NTAPI ExFreeCacheAwareRundownProtection ( IN PEX_RUNDOWN_REF_CACHE_AWARE  RunRefCacheAware)

Definition at line 646 of file rundown.c.

647 {
648  PAGED_CODE();
649 
650  /*
651  * This is to be called for RunRefCacheAware that were allocated with
652  * ExAllocateCacheAwareRundownProtection and not for user-allocated
653  * ones
654  */
655  ASSERT(RunRefCacheAware->PoolToFree != (PVOID)0xBADCA11);
656 
657  /* We don't know the tag that as used for allocation */
658  ExFreePoolWithTag(RunRefCacheAware->PoolToFree, 0);
659  ExFreePoolWithTag(RunRefCacheAware, 0);
660 }
#define PAGED_CODE()
Definition: video.h:57
ASSERT((InvokeOnSuccess||InvokeOnError||InvokeOnCancel) ?(CompletionRoutine !=NULL) :TRUE)
#define ExFreePoolWithTag(_P, _T)
Definition: module.h:1099

◆ ExfReInitializeRundownProtection()

VOID FASTCALL ExfReInitializeRundownProtection ( IN PEX_RUNDOWN_REF  RunRef)

Definition at line 141 of file rundown.c.

142 {
143  PAGED_CODE();
144 
145  /* Sanity check */
146  ASSERT((RunRef->Count & EX_RUNDOWN_ACTIVE) != 0);
147 
148  /* Reset the count */
149  ExpSetRundown(RunRef, 0);
150 }
#define ExpSetRundown(x, y)
Definition: ex.h:160
#define PAGED_CODE()
Definition: video.h:57
ASSERT((InvokeOnSuccess||InvokeOnError||InvokeOnCancel) ?(CompletionRoutine !=NULL) :TRUE)
#define EX_RUNDOWN_ACTIVE

◆ ExfReInitializeRundownProtectionCacheAware()

VOID FASTCALL ExfReInitializeRundownProtectionCacheAware ( IN PEX_RUNDOWN_REF_CACHE_AWARE  RunRefCacheAware)

Definition at line 534 of file rundown.c.

535 {
536  PEX_RUNDOWN_REF RunRef;
537  ULONG ProcCount, Current;
538 
539  ProcCount = RunRefCacheAware->Number;
540  /* No proc, nothing to do */
541  if (ProcCount == 0)
542  {
543  return;
544  }
545 
546  /* We will mark all our runrefs inactive */
547  for (Current = 0; Current < ProcCount; ++Current)
548  {
549  /* Get the runref for the proc */
550  RunRef = ExGetRunRefForGivenProcessor(RunRefCacheAware, Current);
551  ASSERT((RunRef->Count & EX_RUNDOWN_ACTIVE) != 0);
552 
553  ExpSetRundown(RunRef, 0);
554  }
555 }
#define ExpSetRundown(x, y)
Definition: ex.h:160
volatile ULONG_PTR Count
Definition: extypes.h:181
ASSERT((InvokeOnSuccess||InvokeOnError||InvokeOnCancel) ?(CompletionRoutine !=NULL) :TRUE)
FORCEINLINE PEX_RUNDOWN_REF ExGetRunRefForGivenProcessor(IN PEX_RUNDOWN_REF_CACHE_AWARE RunRefCacheAware, IN ULONG ProcNumber)
Definition: ex.h:779
unsigned int ULONG
Definition: retypes.h:1
#define EX_RUNDOWN_ACTIVE

◆ ExfReleaseRundownProtection()

VOID FASTCALL ExfReleaseRundownProtection ( IN PEX_RUNDOWN_REF  RunRef)

Definition at line 197 of file rundown.c.

198 {
199  ULONG_PTR Value = RunRef->Count, NewValue;
200  PEX_RUNDOWN_WAIT_BLOCK WaitBlock;
201 
202  /* Loop until successfully incremented the counter */
203  for (;;)
204  {
205  /* Check if rundown is not active */
206  if (!(Value & EX_RUNDOWN_ACTIVE))
207  {
208  /* Sanity check */
210 
211  /* Get the new value */
212  NewValue = Value - EX_RUNDOWN_COUNT_INC;
213 
214  /* Change the value */
215  NewValue = ExpChangeRundown(RunRef, NewValue, Value);
216  if (NewValue == Value) break;
217 
218  /* Update value */
219  Value = NewValue;
220  }
221  else
222  {
223  /* Get the wait block */
225  ASSERT((WaitBlock->Count > 0) || (KeNumberProcessors > 1));
226 
227  /* Remove the one count */
228  if (!InterlockedDecrementSizeT(&WaitBlock->Count))
229  {
230  /* We're down to 0 now, so signal the event */
231  KeSetEvent(&WaitBlock->WakeEvent, IO_NO_INCREMENT, FALSE);
232  }
233 
234  /* We're all done */
235  break;
236  }
237  }
238 }
_In_opt_ ULONG _Out_ PULONG Value
Definition: rtlfuncs.h:2343
LONG NTAPI KeSetEvent(IN PKEVENT Event, IN KPRIORITY Increment, IN BOOLEAN Wait)
Definition: eventobj.c:159
#define EX_RUNDOWN_COUNT_INC
uint32_t ULONG_PTR
Definition: typedefs.h:63
#define ExpChangeRundown(x, y, z)
Definition: ex.h:158
ASSERT((InvokeOnSuccess||InvokeOnError||InvokeOnCancel) ?(CompletionRoutine !=NULL) :TRUE)
CCHAR KeNumberProcessors
Definition: krnlinit.c:35
struct _EX_RUNDOWN_WAIT_BLOCK * PEX_RUNDOWN_WAIT_BLOCK
#define IO_NO_INCREMENT
Definition: iotypes.h:565
#define InterlockedDecrementSizeT(a)
Definition: interlocked.h:153
#define EX_RUNDOWN_ACTIVE

◆ ExfReleaseRundownProtectionCacheAware()

VOID FASTCALL ExfReleaseRundownProtectionCacheAware ( IN PEX_RUNDOWN_REF_CACHE_AWARE  RunRefCacheAware)

Definition at line 411 of file rundown.c.

412 {
413  PEX_RUNDOWN_REF RunRef;
414 
415  RunRef = ExGetRunRefForGivenProcessor(RunRefCacheAware, KeGetCurrentProcessorNumber());
417 }
FORCEINLINE VOID _ExReleaseRundownProtection(IN PEX_RUNDOWN_REF RunRef)
Definition: ex.h:847
FORCEINLINE ULONG KeGetCurrentProcessorNumber(VOID)
Definition: ke.h:325
FORCEINLINE PEX_RUNDOWN_REF ExGetRunRefForGivenProcessor(IN PEX_RUNDOWN_REF_CACHE_AWARE RunRefCacheAware, IN ULONG ProcNumber)
Definition: ex.h:779

◆ ExfReleaseRundownProtectionCacheAwareEx()

VOID FASTCALL ExfReleaseRundownProtectionCacheAwareEx ( IN PEX_RUNDOWN_REF_CACHE_AWARE  RunRefCacheAware,
IN ULONG  Count 
)

Definition at line 424 of file rundown.c.

426 {
427  PEX_RUNDOWN_REF RunRef;
428 
429  RunRef = ExGetRunRefForGivenProcessor(RunRefCacheAware, KeGetCurrentProcessorNumber());
431 }
_Inout_ __drv_aliasesMem PSLIST_ENTRY _Inout_ PSLIST_ENTRY _In_ ULONG Count
Definition: exfuncs.h:1015
FORCEINLINE ULONG KeGetCurrentProcessorNumber(VOID)
Definition: ke.h:325
VOID FASTCALL ExfReleaseRundownProtectionEx(IN PEX_RUNDOWN_REF RunRef, IN ULONG Count)
Definition: rundown.c:260
FORCEINLINE PEX_RUNDOWN_REF ExGetRunRefForGivenProcessor(IN PEX_RUNDOWN_REF_CACHE_AWARE RunRefCacheAware, IN ULONG ProcNumber)
Definition: ex.h:779

◆ ExfReleaseRundownProtectionEx()

VOID FASTCALL ExfReleaseRundownProtectionEx ( IN PEX_RUNDOWN_REF  RunRef,
IN ULONG  Count 
)

Definition at line 260 of file rundown.c.

262 {
263  ULONG_PTR Value = RunRef->Count, NewValue;
264  PEX_RUNDOWN_WAIT_BLOCK WaitBlock;
265 
266  /* Loop until successfully incremented the counter */
267  for (;;)
268  {
269  /* Check if rundown is not active */
270  if (!(Value & EX_RUNDOWN_ACTIVE))
271  {
272  /* Sanity check */
274  (KeNumberProcessors > 1));
275 
276  /* Get the new value */
277  NewValue = Value - EX_RUNDOWN_COUNT_INC * Count;
278 
279  /* Change the value */
280  NewValue = ExpChangeRundown(RunRef, NewValue, Value);
281  if (NewValue == Value) break;
282 
283  /* Update value */
284  Value = NewValue;
285  }
286  else
287  {
288  /* Get the wait block */
290  ASSERT((WaitBlock->Count >= Count) || (KeNumberProcessors > 1));
291 
292  /* Remove the counts */
293  if (InterlockedExchangeAddSizeT(&WaitBlock->Count,
294  -(LONG)Count) == (LONG)Count)
295  {
296  /* We're down to 0 now, so signal the event */
297  KeSetEvent(&WaitBlock->WakeEvent, IO_NO_INCREMENT, FALSE);
298  }
299 
300  /* We're all done */
301  break;
302  }
303  }
304 }
_In_opt_ ULONG _Out_ PULONG Value
Definition: rtlfuncs.h:2343
_Inout_ __drv_aliasesMem PSLIST_ENTRY _Inout_ PSLIST_ENTRY _In_ ULONG Count
Definition: exfuncs.h:1015
LONG NTAPI KeSetEvent(IN PKEVENT Event, IN KPRIORITY Increment, IN BOOLEAN Wait)
Definition: eventobj.c:159
#define EX_RUNDOWN_COUNT_INC
uint32_t ULONG_PTR
Definition: typedefs.h:63
long LONG
Definition: pedump.c:60
#define ExpChangeRundown(x, y, z)
Definition: ex.h:158
ASSERT((InvokeOnSuccess||InvokeOnError||InvokeOnCancel) ?(CompletionRoutine !=NULL) :TRUE)
#define InterlockedExchangeAddSizeT(a, b)
Definition: interlocked.h:196
CCHAR KeNumberProcessors
Definition: krnlinit.c:35
struct _EX_RUNDOWN_WAIT_BLOCK * PEX_RUNDOWN_WAIT_BLOCK
#define IO_NO_INCREMENT
Definition: iotypes.h:565
#define EX_RUNDOWN_ACTIVE

Referenced by ExCompareExchangeCallBack(), ExfReleaseRundownProtectionCacheAwareEx(), and ExReferenceCallBackBlock().

◆ ExfRundownCompleted()

VOID FASTCALL ExfRundownCompleted ( IN PEX_RUNDOWN_REF  RunRef)

Definition at line 169 of file rundown.c.

170 {
171  PAGED_CODE();
172 
173  /* Sanity check */
174  ASSERT((RunRef->Count & EX_RUNDOWN_ACTIVE) != 0);
175 
176  /* Mark the counter as active */
178 }
#define ExpSetRundown(x, y)
Definition: ex.h:160
#define PAGED_CODE()
Definition: video.h:57
ASSERT((InvokeOnSuccess||InvokeOnError||InvokeOnCancel) ?(CompletionRoutine !=NULL) :TRUE)
#define EX_RUNDOWN_ACTIVE

◆ ExfRundownCompletedCacheAware()

VOID FASTCALL ExfRundownCompletedCacheAware ( IN PEX_RUNDOWN_REF_CACHE_AWARE  RunRefCacheAware)

Definition at line 506 of file rundown.c.

507 {
508  PEX_RUNDOWN_REF RunRef;
509  ULONG ProcCount, Current;
510 
511  ProcCount = RunRefCacheAware->Number;
512  /* No proc, nothing to do */
513  if (ProcCount == 0)
514  {
515  return;
516  }
517 
518  /* We will mark all our runrefs active */
519  for (Current = 0; Current < ProcCount; ++Current)
520  {
521  /* Get the runref for the proc */
522  RunRef = ExGetRunRefForGivenProcessor(RunRefCacheAware, Current);
523  ASSERT((RunRef->Count & EX_RUNDOWN_ACTIVE) != 0);
524 
526  }
527 }
#define ExpSetRundown(x, y)
Definition: ex.h:160
volatile ULONG_PTR Count
Definition: extypes.h:181
ASSERT((InvokeOnSuccess||InvokeOnError||InvokeOnCancel) ?(CompletionRoutine !=NULL) :TRUE)
FORCEINLINE PEX_RUNDOWN_REF ExGetRunRefForGivenProcessor(IN PEX_RUNDOWN_REF_CACHE_AWARE RunRefCacheAware, IN ULONG ProcNumber)
Definition: ex.h:779
unsigned int ULONG
Definition: retypes.h:1
#define EX_RUNDOWN_ACTIVE

◆ ExfWaitForRundownProtectionRelease()

VOID FASTCALL ExfWaitForRundownProtectionRelease ( IN PEX_RUNDOWN_REF  RunRef)

Definition at line 324 of file rundown.c.

325 {
326  ULONG_PTR Value, Count, NewValue;
327  EX_RUNDOWN_WAIT_BLOCK WaitBlock;
328  PEX_RUNDOWN_WAIT_BLOCK WaitBlockPointer;
329  PKEVENT Event;
330  PAGED_CODE();
331 
332  /* Set the active bit */
334  if ((Value == 0) || (Value == EX_RUNDOWN_ACTIVE)) return;
335 
336  /* No event for now */
337  Event = NULL;
338  WaitBlockPointer = (PEX_RUNDOWN_WAIT_BLOCK)((ULONG_PTR)&WaitBlock |
340 
341  /* Start waitblock set loop */
342  for (;;)
343  {
344  /* Save the count */
346 
347  /* If the count is over one and we don't have en event yet, create it */
348  if ((Count) && !(Event))
349  {
350  /* Initialize the event */
351  KeInitializeEvent(&WaitBlock.WakeEvent,
353  FALSE);
354 
355  /* Set the pointer */
356  Event = &WaitBlock.WakeEvent;
357  }
358 
359  /* Set the count */
360  WaitBlock.Count = Count;
361 
362  /* Now set the pointer */
363  NewValue = ExpChangeRundown(RunRef, (ULONG_PTR)WaitBlockPointer, Value);
364  if (NewValue == Value) break;
365 
366  /* Loop again */
367  Value = NewValue;
368  ASSERT((Value & EX_RUNDOWN_ACTIVE) == 0);
369  }
370 
371  /* If the count was 0, we're done */
372  if (!Count) return;
373 
374  /* Wait for whoever needs to release to notify us */
376  ASSERT(WaitBlock.Count == 0);
377 }
_In_opt_ ULONG _Out_ PULONG Value
Definition: rtlfuncs.h:2343
#define EX_RUNDOWN_COUNT_SHIFT
_Inout_ __drv_aliasesMem PSLIST_ENTRY _Inout_ PSLIST_ENTRY _In_ ULONG Count
Definition: exfuncs.h:1015
NTSTATUS NTAPI KeWaitForSingleObject(IN PVOID Object, IN KWAIT_REASON WaitReason, IN KPROCESSOR_MODE WaitMode, IN BOOLEAN Alertable, IN PLARGE_INTEGER Timeout OPTIONAL)
Definition: wait.c:416
#define PAGED_CODE()
Definition: video.h:57
uint32_t ULONG_PTR
Definition: typedefs.h:63
_In_ PVOID _In_ ULONG Event
Definition: iotypes.h:434
smooth NULL
Definition: ftsmooth.c:416
#define ExpChangeRundown(x, y, z)
Definition: ex.h:158
ASSERT((InvokeOnSuccess||InvokeOnError||InvokeOnCancel) ?(CompletionRoutine !=NULL) :TRUE)
#define KeInitializeEvent(pEvt, foo, foo2)
Definition: env_spec_w32.h:477
struct _EX_RUNDOWN_WAIT_BLOCK * PEX_RUNDOWN_WAIT_BLOCK
#define EX_RUNDOWN_ACTIVE

◆ ExfWaitForRundownProtectionReleaseCacheAware()

VOID FASTCALL ExfWaitForRundownProtectionReleaseCacheAware ( IN PEX_RUNDOWN_REF_CACHE_AWARE  RunRefCacheAware)

Definition at line 438 of file rundown.c.

439 {
440  PEX_RUNDOWN_REF RunRef;
441  EX_RUNDOWN_WAIT_BLOCK WaitBlock;
442  PEX_RUNDOWN_WAIT_BLOCK WaitBlockPointer;
443  ULONG_PTR ProcCount, Current, Value, OldValue, TotalCount;
444 
445  ProcCount = RunRefCacheAware->Number;
446  /* No proc, nothing to do */
447  if (ProcCount == 0)
448  {
449  return;
450  }
451 
452  TotalCount = 0;
453  WaitBlock.Count = 0;
454  WaitBlockPointer = (PEX_RUNDOWN_WAIT_BLOCK)((ULONG_PTR)&WaitBlock |
456  /* We will check all our runrefs */
457  for (Current = 0; Current < ProcCount; ++Current)
458  {
459  /* Get the runref for the proc */
460  RunRef = ExGetRunRefForGivenProcessor(RunRefCacheAware, Current);
461  /* Loop for setting the wait block */
462  do
463  {
464  Value = RunRef->Count;
465  ASSERT((Value & EX_RUNDOWN_ACTIVE) == 0);
466 
467  /* Remove old value and set our waitblock instead */
468  OldValue = ExpChangeRundown(RunRef, WaitBlockPointer, Value);
469  if (OldValue == Value)
470  {
471  break;
472  }
473 
474  Value = OldValue;
475  }
476  while (TRUE);
477 
478  /* Count the deleted values */
479  TotalCount += Value;
480  }
481 
482  /* Sanity check: we didn't overflow */
483  ASSERT((LONG_PTR)TotalCount >= 0);
484  if (TotalCount != 0)
485  {
486  /* Init the waitblock event */
487  KeInitializeEvent(&WaitBlock.WakeEvent,
489  FALSE);
490 
491  /* Do we have to wait? If so, go ahead! */
492  if (InterlockedExchangeAddSizeT(&WaitBlock.Count,
493  (LONG_PTR)TotalCount >> EX_RUNDOWN_COUNT_SHIFT) ==
494  -(LONG_PTR)(TotalCount >> EX_RUNDOWN_COUNT_SHIFT))
495  {
497  }
498  }
499 }
_In_opt_ ULONG _Out_ PULONG Value
Definition: rtlfuncs.h:2343
#define TRUE
Definition: types.h:120
volatile ULONG_PTR Count
Definition: extypes.h:181
#define EX_RUNDOWN_COUNT_SHIFT
NTSTATUS NTAPI KeWaitForSingleObject(IN PVOID Object, IN KWAIT_REASON WaitReason, IN KPROCESSOR_MODE WaitMode, IN BOOLEAN Alertable, IN PLARGE_INTEGER Timeout OPTIONAL)
Definition: wait.c:416
uint32_t ULONG_PTR
Definition: typedefs.h:63
smooth NULL
Definition: ftsmooth.c:416
#define ExpChangeRundown(x, y, z)
Definition: ex.h:158
ASSERT((InvokeOnSuccess||InvokeOnError||InvokeOnCancel) ?(CompletionRoutine !=NULL) :TRUE)
#define InterlockedExchangeAddSizeT(a, b)
Definition: interlocked.h:196
FORCEINLINE PEX_RUNDOWN_REF ExGetRunRefForGivenProcessor(IN PEX_RUNDOWN_REF_CACHE_AWARE RunRefCacheAware, IN ULONG ProcNumber)
Definition: ex.h:779
#define KeInitializeEvent(pEvt, foo, foo2)
Definition: env_spec_w32.h:477
__int3264 LONG_PTR
Definition: mstsclib_h.h:276
struct _EX_RUNDOWN_WAIT_BLOCK * PEX_RUNDOWN_WAIT_BLOCK
#define EX_RUNDOWN_ACTIVE

◆ ExInitializeRundownProtectionCacheAware()

VOID NTAPI ExInitializeRundownProtectionCacheAware ( IN PEX_RUNDOWN_REF_CACHE_AWARE  RunRefCacheAware,
IN SIZE_T  Size 
)

Definition at line 667 of file rundown.c.

669 {
670  PVOID Pool;
671  PEX_RUNDOWN_REF RunRef;
672  ULONG Count, RunRefSize, Align;
673 
674  PAGED_CODE();
675 
676  /* Get the user allocate pool for runrefs */
677  Pool = (PVOID)((ULONG_PTR)RunRefCacheAware + sizeof(EX_RUNDOWN_REF_CACHE_AWARE));
678 
679  /* By default a runref is structure-sized */
680  RunRefSize = sizeof(EX_RUNDOWN_REF);
681 
682  /*
683  * If we just have enough room for a single runref, deduce were on a single
684  * processor machine
685  */
686  if (Size == sizeof(EX_RUNDOWN_REF_CACHE_AWARE) + sizeof(EX_RUNDOWN_REF))
687  {
688  Count = 1;
689  }
690  else
691  {
692  /* Get alignment constraint */
694 
695  /* How many runrefs given the alignment? */
696  RunRefSize = Align;
697  Count = ((Size - sizeof(EX_RUNDOWN_REF_CACHE_AWARE)) / Align) - 1;
698  Pool = (PVOID)ALIGN_UP_BY(Pool, Align);
699  }
700 
701  /* Initialize the structure */
702  RunRefCacheAware->RunRefs = Pool;
703  RunRefCacheAware->RunRefSize = RunRefSize;
704  RunRefCacheAware->Number = Count;
705 
706  /* There is no allocated pool! */
707  RunRefCacheAware->PoolToFree = (PVOID)0xBADCA11u;
708 
709  /* Initialize runref */
710  if (RunRefCacheAware->Number != 0)
711  {
712  for (Count = 0; Count < RunRefCacheAware->Number; ++Count)
713  {
714  RunRef = ExGetRunRefForGivenProcessor(RunRefCacheAware, Count);
716  }
717  }
718 }
_In_ SIZE_T RunRefSize
Definition: exfuncs.h:1103
FORCEINLINE VOID _ExInitializeRundownProtection(IN PEX_RUNDOWN_REF RunRef)
Definition: ex.h:890
struct _EX_RUNDOWN_REF EX_RUNDOWN_REF
_Inout_ __drv_aliasesMem PSLIST_ENTRY _Inout_ PSLIST_ENTRY _In_ ULONG Count
Definition: exfuncs.h:1015
#define PAGED_CODE()
Definition: video.h:57
ULONG NTAPI KeGetRecommendedSharedDataAlignment(VOID)
Definition: cpu.c:471
uint32_t ULONG_PTR
Definition: typedefs.h:63
struct _EX_RUNDOWN_REF_CACHE_AWARE EX_RUNDOWN_REF_CACHE_AWARE
void * PVOID
Definition: retypes.h:9
Definition: bufpool.h:50
IN PVOID IN PVOID IN USHORT IN USHORT Size
Definition: pci.h:359
FORCEINLINE PEX_RUNDOWN_REF ExGetRunRefForGivenProcessor(IN PEX_RUNDOWN_REF_CACHE_AWARE RunRefCacheAware, IN ULONG ProcNumber)
Definition: ex.h:779
unsigned int ULONG
Definition: retypes.h:1
#define ALIGN_UP_BY(size, align)

◆ ExSizeOfRundownProtectionCacheAware()

SIZE_T NTAPI ExSizeOfRundownProtectionCacheAware ( VOID  )

Definition at line 725 of file rundown.c.

726 {
727  SIZE_T Size;
728 
729  PAGED_CODE();
730 
731  /* Compute the needed size for runrefs */
732  if (KeNumberProcessors <= 1)
733  {
734  Size = sizeof(EX_RUNDOWN_REF);
735  }
736  else
737  {
738  /* We +1, to have enough room for alignment */
740  }
741 
742  /* Return total size (master structure and runrefs) */
743  return Size + sizeof(EX_RUNDOWN_REF_CACHE_AWARE);
744 }
struct _EX_RUNDOWN_REF EX_RUNDOWN_REF
#define PAGED_CODE()
Definition: video.h:57
ULONG NTAPI KeGetRecommendedSharedDataAlignment(VOID)
Definition: cpu.c:471
struct _EX_RUNDOWN_REF_CACHE_AWARE EX_RUNDOWN_REF_CACHE_AWARE
IN PVOID IN PVOID IN USHORT IN USHORT Size
Definition: pci.h:359
CCHAR KeNumberProcessors
Definition: krnlinit.c:35
ULONG_PTR SIZE_T
Definition: typedefs.h:78