#include <ntoskrnl.h>
#include <debug.h>

Include dependency graph for work.c:

Macros
#define	NDEBUG

#define	EX_HYPERCRITICAL_WORK_THREADS 1

#define	EX_DELAYED_WORK_THREADS 3

#define	EX_CRITICAL_WORK_THREADS 5

#define	EX_DYNAMIC_WORK_THREAD 0x80000000

#define	EX_HYPERCRITICAL_QUEUE_PRIORITY_INCREMENT 7

#define	EX_CRITICAL_QUEUE_PRIORITY_INCREMENT 5

#define	EX_DELAYED_QUEUE_PRIORITY_INCREMENT 4

Functions
VOID NTAPI	ExpWorkerThreadEntryPoint (IN PVOID Context)

VOID NTAPI	ExpCreateWorkerThread (WORK_QUEUE_TYPE WorkQueueType, IN BOOLEAN Dynamic)

VOID NTAPI	ExpDetectWorkerThreadDeadlock (VOID)

VOID NTAPI	ExpCheckDynamicThreadCount (VOID)

VOID NTAPI	ExpWorkerThreadBalanceManager (IN PVOID Context)

VOID NTAPI	ExpInitializeWorkerThreads (VOID)

VOID NTAPI	ExpSetSwappingKernelApc (IN PKAPC Apc, OUT PKNORMAL_ROUTINE NormalRoutine, IN OUT PVOID NormalContext, IN OUT PVOID SystemArgument1, IN OUT PVOID SystemArgument2)

VOID NTAPI	ExSwapinWorkerThreads (IN BOOLEAN AllowSwap)

VOID NTAPI	ExQueueWorkItem (IN PWORK_QUEUE_ITEM WorkItem, IN WORK_QUEUE_TYPE QueueType)

Macro Definition Documentation

◆ EX_CRITICAL_QUEUE_PRIORITY_INCREMENT

#define EX_CRITICAL_QUEUE_PRIORITY_INCREMENT 5

Definition at line 27 of file work.c.

◆ EX_CRITICAL_WORK_THREADS

#define EX_CRITICAL_WORK_THREADS 5

Definition at line 20 of file work.c.

◆ EX_DELAYED_QUEUE_PRIORITY_INCREMENT

#define EX_DELAYED_QUEUE_PRIORITY_INCREMENT 4

Definition at line 28 of file work.c.

◆ EX_DELAYED_WORK_THREADS

#define EX_DELAYED_WORK_THREADS 3

Definition at line 19 of file work.c.

◆ EX_DYNAMIC_WORK_THREAD

#define EX_DYNAMIC_WORK_THREAD 0x80000000

Definition at line 23 of file work.c.

◆ EX_HYPERCRITICAL_QUEUE_PRIORITY_INCREMENT

#define EX_HYPERCRITICAL_QUEUE_PRIORITY_INCREMENT 7

Definition at line 26 of file work.c.

◆ EX_HYPERCRITICAL_WORK_THREADS

#define EX_HYPERCRITICAL_WORK_THREADS 1

Definition at line 18 of file work.c.

◆ NDEBUG

#define NDEBUG

Definition at line 12 of file work.c.

Function Documentation

◆ ExpCheckDynamicThreadCount()

VOID NTAPI ExpCheckDynamicThreadCount ( VOID )

Definition at line 382 of file work.c.

{
    ULONG i;
    PEX_WORK_QUEUE Queue;
 
    /* Loop the 3 queues */
    for (i = 0; i < MaximumWorkQueue; i++)
    {
        /* Get the queue */
        Queue = &ExWorkerQueue[i];
 
        /* Check if still need a new thread. See ExQueueWorkItem */
        if ((Queue->Info.MakeThreadsAsNecessary) &&
            (!IsListEmpty(&Queue->WorkerQueue.EntryListHead)) &&
            (Queue->WorkerQueue.CurrentCount <
             Queue->WorkerQueue.MaximumCount) &&
            (Queue->DynamicThreadCount < 16))
        {
            /* Create a new thread */
            DPRINT1("EX: Creating new dynamic thread as requested\n");
            ExpCreateWorkerThread(i, TRUE);
        }
    }
}

Referenced by ExpWorkerThreadBalanceManager().

◆ ExpCreateWorkerThread()

VOID NTAPI ExpCreateWorkerThread	(	WORK_QUEUE_TYPE	WorkQueueType,
		IN BOOLEAN	Dynamic
	)

Definition at line 250 of file work.c.

{
    PETHREAD Thread;
    HANDLE hThread;
    ULONG Context;
    KPRIORITY Priority;
    NTSTATUS Status;
 
    /* Check if this is going to be a dynamic thread */
    Context = WorkQueueType;
 
    /* Add the dynamic mask */
    if (Dynamic) Context |= EX_DYNAMIC_WORK_THREAD;
 
    /* Create the System Thread */
    Status = PsCreateSystemThread(&hThread,
                                  THREAD_ALL_ACCESS,
                                  NULL,
                                  NULL,
                                  NULL,
                                  ExpWorkerThreadEntryPoint,
                                  UlongToPtr(Context));
    if (!NT_SUCCESS(Status))
    {
        /* Well... */
        DPRINT1("Failed to create worker thread: 0x%08x\n", Status);
        return;
    }
 
    /* If the thread is dynamic */
    if (Dynamic)
    {
        /* Increase the count */
        InterlockedIncrement(&ExWorkerQueue[WorkQueueType].DynamicThreadCount);
    }
 
    /* Set the priority */
    if (WorkQueueType == DelayedWorkQueue)
    {
        /* Priority == 4 */
        Priority = EX_DELAYED_QUEUE_PRIORITY_INCREMENT;
    }
    else if (WorkQueueType == CriticalWorkQueue)
    {
        /* Priority == 5 */
        Priority = EX_CRITICAL_QUEUE_PRIORITY_INCREMENT;
    }
    else
    {
        /* Priority == 7 */
        Priority = EX_HYPERCRITICAL_QUEUE_PRIORITY_INCREMENT;
    }
 
    /* Get the Thread */
    ObReferenceObjectByHandle(hThread,
                              THREAD_SET_INFORMATION,
                              PsThreadType,
                              KernelMode,
                              (PVOID*)&Thread,
                              NULL);
 
    /* Set the Priority */
    KeSetBasePriorityThread(&Thread->Tcb, Priority);
 
    /* Dereference and close handle */
    ObDereferenceObject(Thread);
    ObCloseHandle(hThread, KernelMode);
}

Referenced by ExpCheckDynamicThreadCount(), ExpDetectWorkerThreadDeadlock(), and ExpInitializeWorkerThreads().

◆ ExpDetectWorkerThreadDeadlock()

VOID NTAPI ExpDetectWorkerThreadDeadlock ( VOID )

Definition at line 337 of file work.c.

{
    ULONG i;
    PEX_WORK_QUEUE Queue;
 
    /* Loop the 3 queues */
    for (i = 0; i < MaximumWorkQueue; i++)
    {
        /* Get the queue */
        Queue = &ExWorkerQueue[i];
        ASSERT(Queue->DynamicThreadCount <= 16);
 
        /* Check if stuff is on the queue that still is unprocessed */
        if ((Queue->QueueDepthLastPass) &&
            (Queue->WorkItemsProcessed == Queue->WorkItemsProcessedLastPass) &&
            (Queue->DynamicThreadCount < 16))
        {
            /* Stuff is still on the queue and nobody did anything about it */
            DPRINT1("EX: Work Queue Deadlock detected: %lu\n", i);
            ExpCreateWorkerThread(i, TRUE);
            DPRINT1("Dynamic threads queued %d\n", Queue->DynamicThreadCount);
        }
 
        /* Update our data */
        Queue->WorkItemsProcessedLastPass = Queue->WorkItemsProcessed;
        Queue->QueueDepthLastPass = KeReadStateQueue(&Queue->WorkerQueue);
    }
}

Referenced by ExpWorkerThreadBalanceManager().

◆ ExpInitializeWorkerThreads()

VOID NTAPI ExpInitializeWorkerThreads ( VOID )

Definition at line 522 of file work.c.

{
    ULONG WorkQueueType;
    ULONG CriticalThreads, DelayedThreads;
    HANDLE ThreadHandle;
    PETHREAD Thread;
    ULONG i;
    NTSTATUS Status;
 
    /* Setup the stack swap support */
    ExInitializeFastMutex(&ExpWorkerSwapinMutex);
    InitializeListHead(&ExpWorkerListHead);
    ExpWorkersCanSwap = TRUE;
 
    /* Set the number of critical and delayed threads. We shouldn't hardcode */
    DelayedThreads = EX_DELAYED_WORK_THREADS;
    CriticalThreads = EX_CRITICAL_WORK_THREADS;
 
    /* Protect against greedy registry modifications */
    ExpAdditionalDelayedWorkerThreads =
        min(ExpAdditionalDelayedWorkerThreads, 16);
    ExpAdditionalCriticalWorkerThreads =
        min(ExpAdditionalCriticalWorkerThreads, 16);
 
    /* Calculate final count */
    DelayedThreads += ExpAdditionalDelayedWorkerThreads;
    CriticalThreads += ExpAdditionalCriticalWorkerThreads;
 
    /* Initialize the Array */
    for (WorkQueueType = 0; WorkQueueType < MaximumWorkQueue; WorkQueueType++)
    {
        /* Clear the structure and initialize the queue */
        RtlZeroMemory(&ExWorkerQueue[WorkQueueType], sizeof(EX_WORK_QUEUE));
        KeInitializeQueue(&ExWorkerQueue[WorkQueueType].WorkerQueue, 0);
    }
 
    /* Dynamic threads are only used for the critical queue */
    ExWorkerQueue[CriticalWorkQueue].Info.MakeThreadsAsNecessary = TRUE;
 
    /* Initialize the balance set manager events */
    KeInitializeEvent(&ExpThreadSetManagerEvent, SynchronizationEvent, FALSE);
    KeInitializeEvent(&ExpThreadSetManagerShutdownEvent,
                      NotificationEvent,
                      FALSE);
 
    /* Create the built-in worker threads for the critical queue */
    for (i = 0; i < CriticalThreads; i++)
    {
        /* Create the thread */
        ExpCreateWorkerThread(CriticalWorkQueue, FALSE);
        ExCriticalWorkerThreads++;
    }
 
    /* Create the built-in worker threads for the delayed queue */
    for (i = 0; i < DelayedThreads; i++)
    {
        /* Create the thread */
        ExpCreateWorkerThread(DelayedWorkQueue, FALSE);
        ExDelayedWorkerThreads++;
    }
 
    /* Create the built-in worker thread for the hypercritical queue */
    ExpCreateWorkerThread(HyperCriticalWorkQueue, FALSE);
 
    /* Create the balance set manager thread */
    Status = PsCreateSystemThread(&ThreadHandle,
                                  THREAD_ALL_ACCESS,
                                  NULL,
                                  0,
                                  NULL,
                                  ExpWorkerThreadBalanceManager,
                                  NULL);
    if (!NT_SUCCESS(Status))
    {
        KeBugCheckEx(PHASE1_INITIALIZATION_FAILED, Status, 0, 0, 0);
    }
 
    /* Get a pointer to it for the shutdown process */
    ObReferenceObjectByHandle(ThreadHandle,
                              THREAD_ALL_ACCESS,
                              NULL,
                              KernelMode,
                              (PVOID*)&Thread,
                              NULL);
    ExpWorkerThreadBalanceManagerPtr = Thread;
 
    /* Close the handle and return */
    ObCloseHandle(ThreadHandle, KernelMode);
}

Referenced by ExpInitSystemPhase1().

◆ ExpSetSwappingKernelApc()

VOID NTAPI ExpSetSwappingKernelApc	(	IN PKAPC	Apc,
		OUT PKNORMAL_ROUTINE *	NormalRoutine,
		IN OUT PVOID *	NormalContext,
		IN OUT PVOID *	SystemArgument1,
		IN OUT PVOID *	SystemArgument2
	)

Definition at line 614 of file work.c.

{
    PBOOLEAN AllowSwap;
    PKEVENT Event = (PKEVENT)*SystemArgument1;
 
    /* Make sure it's an active worker */
    if (PsGetCurrentThread()->ActiveExWorker)
    {
        /* Read the setting from the context flag */
        AllowSwap = (PBOOLEAN)NormalContext;
        KeSetKernelStackSwapEnable(*AllowSwap);
    }
 
    /* Let caller know that we're done */
    KeSetEvent(Event, 0, FALSE);
}

Referenced by ExSwapinWorkerThreads().

◆ ExpWorkerThreadBalanceManager()

VOID NTAPI ExpWorkerThreadBalanceManager ( IN PVOID Context )

Definition at line 427 of file work.c.

{
    KTIMER Timer;
    LARGE_INTEGER Timeout;
    NTSTATUS Status;
    PVOID WaitEvents[3];
    PAGED_CODE();
    UNREFERENCED_PARAMETER(Context);
 
    /* Raise our priority above all other worker threads */
    KeSetBasePriorityThread(KeGetCurrentThread(),
                            EX_CRITICAL_QUEUE_PRIORITY_INCREMENT + 1);
 
    /* Setup the timer */
    KeInitializeTimer(&Timer);
    Timeout.QuadPart = Int32x32To64(-1, 10000000);
 
    /* We'll wait on the periodic timer and also the emergency event */
    WaitEvents[0] = &Timer;
    WaitEvents[1] = &ExpThreadSetManagerEvent;
    WaitEvents[2] = &ExpThreadSetManagerShutdownEvent;
 
    /* Start wait loop */
    for (;;)
    {
        /* Wait for the timer */
        KeSetTimer(&Timer, Timeout, NULL);
        Status = KeWaitForMultipleObjects(3,
                                          WaitEvents,
                                          WaitAny,
                                          Executive,
                                          KernelMode,
                                          FALSE,
                                          NULL,
                                          NULL);
        if (Status == 0)
        {
            /* Our timer expired. Check for deadlocks */
            ExpDetectWorkerThreadDeadlock();
        }
        else if (Status == 1)
        {
            /* Someone notified us, verify if we should create a new thread */
            ExpCheckDynamicThreadCount();
        }
        else if (Status == 2)
        {
            /* We are shutting down. Cancel the timer */
            DPRINT1("System shutdown\n");
            KeCancelTimer(&Timer);
 
            /* Make sure we have a final thread */
            ASSERT(ExpLastWorkerThread);
 
            /* Wait for it */
            KeWaitForSingleObject(ExpLastWorkerThread,
                                  Executive,
                                  KernelMode,
                                  FALSE,
                                  NULL);
 
            /* Dereference it and kill us */
            ObDereferenceObject(ExpLastWorkerThread);
            PsTerminateSystemThread(STATUS_SYSTEM_SHUTDOWN);
        }
 
        /*
         * If WinDBG wants to attach or kill a user-mode process, and/or
         * page-in an address region, queue a debugger worker thread.
         */
        if (ExpDebuggerWork == WinKdWorkerStart)
        {
             ExInitializeWorkItem(&ExpDebuggerWorkItem, ExpDebuggerWorker, NULL);
             ExpDebuggerWork = WinKdWorkerInitialized;
             ExQueueWorkItem(&ExpDebuggerWorkItem, DelayedWorkQueue);
        }
    }
}

Referenced by ExpInitializeWorkerThreads().

◆ ExpWorkerThreadEntryPoint()

VOID NTAPI ExpWorkerThreadEntryPoint ( IN PVOID Context )

Definition at line 78 of file work.c.

{
    PWORK_QUEUE_ITEM WorkItem;
    PLIST_ENTRY QueueEntry;
    WORK_QUEUE_TYPE WorkQueueType;
    PEX_WORK_QUEUE WorkQueue;
    LARGE_INTEGER Timeout;
    PLARGE_INTEGER TimeoutPointer = NULL;
    PETHREAD Thread = PsGetCurrentThread();
    KPROCESSOR_MODE WaitMode;
    EX_QUEUE_WORKER_INFO OldValue, NewValue;
 
    /* Check if this is a dyamic thread */
    if ((ULONG_PTR)Context & EX_DYNAMIC_WORK_THREAD)
    {
        /* It is, which means we will eventually time out after 10 minutes */
        Timeout.QuadPart = Int32x32To64(10, -10000000 * 60);
        TimeoutPointer = &Timeout;
    }
 
    /* Get Queue Type and Worker Queue */
    WorkQueueType = (WORK_QUEUE_TYPE)((ULONG_PTR)Context &
                                      ~EX_DYNAMIC_WORK_THREAD);
    WorkQueue = &ExWorkerQueue[WorkQueueType];
 
    /* Select the wait mode */
    WaitMode = (UCHAR)WorkQueue->Info.WaitMode;
 
    /* Nobody should have initialized this yet, do it now */
    ASSERT(Thread->ExWorkerCanWaitUser == 0);
    if (WaitMode == UserMode) Thread->ExWorkerCanWaitUser = TRUE;
 
    /* If we shouldn't swap, disable that feature */
    if (!ExpWorkersCanSwap) KeSetKernelStackSwapEnable(FALSE);
 
    /* Set the worker flags */
    do
    {
        /* Check if the queue is being disabled */
        if (WorkQueue->Info.QueueDisabled)
        {
            /* Re-enable stack swapping and kill us */
            KeSetKernelStackSwapEnable(TRUE);
            PsTerminateSystemThread(STATUS_SYSTEM_SHUTDOWN);
        }
 
        /* Increase the worker count */
        OldValue = WorkQueue->Info;
        NewValue = OldValue;
        NewValue.WorkerCount++;
    }
    while (InterlockedCompareExchange((PLONG)&WorkQueue->Info,
                                      *(PLONG)&NewValue,
                                      *(PLONG)&OldValue) != *(PLONG)&OldValue);
 
    /* Success, you are now officially a worker thread! */
    Thread->ActiveExWorker = TRUE;
 
    /* Loop forever */
ProcessLoop:
    for (;;)
    {
        /* Wait for something to happen on the queue */
        QueueEntry = KeRemoveQueue(&WorkQueue->WorkerQueue,
                                   WaitMode,
                                   TimeoutPointer);
 
        /* Check if we timed out and quit this loop in that case */
        if ((NTSTATUS)(ULONG_PTR)QueueEntry == STATUS_TIMEOUT) break;
 
        /* Increment Processed Work Items */
        InterlockedIncrement((PLONG)&WorkQueue->WorkItemsProcessed);
 
        /* Get the Work Item */
        WorkItem = CONTAINING_RECORD(QueueEntry, WORK_QUEUE_ITEM, List);
 
        /* Make sure nobody is trying to play smart with us */
        ASSERT((ULONG_PTR)WorkItem->WorkerRoutine > MmUserProbeAddress);
 
        /* Call the Worker Routine */
        WorkItem->WorkerRoutine(WorkItem->Parameter);
 
        /* Make sure APCs are not disabled */
        if (Thread->Tcb.CombinedApcDisable != 0)
        {
            /* We're nice and do it behind your back */
            DPRINT1("Warning: Broken Worker Thread: %p %p %p came back "
                    "with APCs disabled!\n",
                    WorkItem->WorkerRoutine,
                    WorkItem->Parameter,
                    WorkItem);
            ASSERT(Thread->Tcb.CombinedApcDisable == 0);
            Thread->Tcb.CombinedApcDisable = 0;
        }
 
        /* Make sure it returned at right IRQL */
        if (KeGetCurrentIrql() != PASSIVE_LEVEL)
        {
            /* It didn't, bugcheck! */
            KeBugCheckEx(WORKER_THREAD_RETURNED_AT_BAD_IRQL,
                         (ULONG_PTR)WorkItem->WorkerRoutine,
                         KeGetCurrentIrql(),
                         (ULONG_PTR)WorkItem->Parameter,
                         (ULONG_PTR)WorkItem);
        }
 
        /* Make sure it returned with Impersionation Disabled */
        if (Thread->ActiveImpersonationInfo)
        {
            /* It didn't, bugcheck! */
            KeBugCheckEx(IMPERSONATING_WORKER_THREAD,
                         (ULONG_PTR)WorkItem->WorkerRoutine,
                         (ULONG_PTR)WorkItem->Parameter,
                         (ULONG_PTR)WorkItem,
                         0);
        }
    }
 
    /* This is a dynamic thread. Terminate it unless IRPs are pending */
    if (!IsListEmpty(&Thread->IrpList)) goto ProcessLoop;
 
    /* Don't terminate it if the queue is disabled either */
    if (WorkQueue->Info.QueueDisabled) goto ProcessLoop;
 
    /* Set the worker flags */
    do
    {
        /* Decrease the worker count */
        OldValue = WorkQueue->Info;
        NewValue = OldValue;
        NewValue.WorkerCount--;
    }
    while (InterlockedCompareExchange((PLONG)&WorkQueue->Info,
                                      *(PLONG)&NewValue,
                                      *(PLONG)&OldValue) != *(PLONG)&OldValue);
 
    /* Decrement dynamic thread count */
    InterlockedDecrement(&WorkQueue->DynamicThreadCount);
 
    /* We're not a worker thread anymore */
    Thread->ActiveExWorker = FALSE;
 
    /* Re-enable the stack swap */
    KeSetKernelStackSwapEnable(TRUE);
    return;
}

Referenced by ExpCreateWorkerThread().

◆ ExQueueWorkItem()

VOID NTAPI ExQueueWorkItem	(	IN PWORK_QUEUE_ITEM	WorkItem,
		IN WORK_QUEUE_TYPE	QueueType
	)

Definition at line 723 of file work.c.

{
    PEX_WORK_QUEUE WorkQueue = &ExWorkerQueue[QueueType];
    ASSERT(QueueType < MaximumWorkQueue);
    ASSERT(WorkItem->List.Flink == NULL);
 
    /* Don't try to trick us */
    if ((ULONG_PTR)WorkItem->WorkerRoutine < MmUserProbeAddress)
    {
        /* Bugcheck the system */
        KeBugCheckEx(WORKER_INVALID,
                     1,
                     (ULONG_PTR)WorkItem,
                     (ULONG_PTR)WorkItem->WorkerRoutine,
                     0);
    }
 
    /* Insert the Queue */
    KeInsertQueue(&WorkQueue->WorkerQueue, &WorkItem->List);
    ASSERT(!WorkQueue->Info.QueueDisabled);
 
    /*
     * Check if we need a new thread. Our decision is as follows:
     *  - This queue type must support Dynamic Threads (duh!)
     *  - It actually has to have unprocessed items
     *  - We have CPUs which could be handling another thread
     *  - We haven't abused our usage of dynamic threads.
     */
    if ((WorkQueue->Info.MakeThreadsAsNecessary) &&
        (!IsListEmpty(&WorkQueue->WorkerQueue.EntryListHead)) &&
        (WorkQueue->WorkerQueue.CurrentCount <
         WorkQueue->WorkerQueue.MaximumCount) &&
        (WorkQueue->DynamicThreadCount < 16))
    {
        /* Let the balance manager know about it */
        DPRINT1("Requesting a new thread. CurrentCount: %lu. MaxCount: %lu\n",
                WorkQueue->WorkerQueue.CurrentCount,
                WorkQueue->WorkerQueue.MaximumCount);
        KeSetEvent(&ExpThreadSetManagerEvent, 0, FALSE);
    }
}

Referenced by _Function_class_(), _IRQL_requires_max_(), add_thread_job(), CcpDereferenceCache(), CcPostWorkQueue(), CcScheduleReadAhead(), CdAddToWorkque(), CTEScheduleEvent(), DriverEntry(), ExpWorkerThreadBalanceManager(), Ext2FloppyFlushDpc(), Ext2QueueCloseRequest(), Ext2QueueRequest(), FatAddToWorkque(), FatCleanVolumeDpc(), FatDeferredFlushDpc(), FatPagingFileErrorHandler(), FltpFastIoDetachDevice(), ForwardedIoCompletionRoutine(), HalAllocateAdapterChannel(), IopDoLoadUnloadDriver(), IopLogDpcRoutine(), IoQueueWorkItem(), IoReportTargetDeviceChangeAsynchronous(), IoWriteErrorLogEntry(), KdpTimeSlipDpcRoutine(), KsGenerateEvent(), KsInstallBusEnumInterface(), KspBusDpcRoutine(), KsQueueWorkItem(), KsRemoveBusEnumInterface(), NdisScheduleWorkItem(), NtfsQueueRequest(), NtSetSystemPowerState(), ObpDeferObjectDeletion(), PiQueueDeviceAction(), PostOnlineNotification(), RxAddToWorkque(), SendOnlineNotificationWorker(), UDFPostRequest(), UDFQueueDelayedClose(), UDFStartEjectWaiter(), UDFVVerify(), USBPORT_TimerDpc(), USBSTOR_QueueWorkItem(), USBSTOR_TimerRoutine(), and VfatQueueRequest().

◆ ExSwapinWorkerThreads()

VOID NTAPI ExSwapinWorkerThreads ( IN BOOLEAN AllowSwap )

Definition at line 637 of file work.c.

{
    KEVENT Event;
    PETHREAD CurrentThread = PsGetCurrentThread(), Thread;
    PEPROCESS Process = PsInitialSystemProcess;
    KAPC Apc;
    PAGED_CODE();
 
    /* Initialize an event so we know when we're done */
    KeInitializeEvent(&Event, NotificationEvent, FALSE);
 
    /* Lock this routine */
    ExAcquireFastMutex(&ExpWorkerSwapinMutex);
 
    /* New threads cannot swap anymore */
    ExpWorkersCanSwap = AllowSwap;
 
    /* Loop all threads in the system process */
    Thread = PsGetNextProcessThread(Process, NULL);
    while (Thread)
    {
        /* Skip threads with explicit permission to do this */
        if (Thread->ExWorkerCanWaitUser) goto Next;
 
        /* Check if we reached ourselves */
        if (Thread == CurrentThread)
        {
            /* Do it inline */
            KeSetKernelStackSwapEnable(AllowSwap);
        }
        else
        {
            /* Queue an APC */
            KeInitializeApc(&Apc,
                            &Thread->Tcb,
                            InsertApcEnvironment,
                            ExpSetSwappingKernelApc,
                            NULL,
                            NULL,
                            KernelMode,
                            &AllowSwap);
            if (KeInsertQueueApc(&Apc, &Event, NULL, 3))
            {
                /* Wait for the APC to run */
                KeWaitForSingleObject(&Event, Executive, KernelMode, FALSE, NULL);
                KeClearEvent(&Event);
            }
        }
 
        /* Next thread */
Next:
        Thread = PsGetNextProcessThread(Process, Thread);
    }
 
    /* Release the lock */
    ExReleaseFastMutex(&ExpWorkerSwapinMutex);
}

Referenced by NtSetSystemPowerState().

Variable Documentation

◆ ExCriticalWorkerThreads

ULONG ExCriticalWorkerThreads

Definition at line 34 of file work.c.

Referenced by CcInitializeCacheManager(), and ExpInitializeWorkerThreads().

◆ ExDelayedWorkerThreads

ULONG ExDelayedWorkerThreads

Definition at line 35 of file work.c.

Referenced by ExpInitializeWorkerThreads().

◆ ExpAdditionalCriticalWorkerThreads

ULONG ExpAdditionalCriticalWorkerThreads

Definition at line 36 of file work.c.

Referenced by ExpInitializeWorkerThreads().

◆ ExpAdditionalDelayedWorkerThreads

ULONG ExpAdditionalDelayedWorkerThreads

Definition at line 37 of file work.c.

Referenced by ExpInitializeWorkerThreads().

◆ ExpLastWorkerThread

PETHREAD ExpLastWorkerThread

Definition at line 50 of file work.c.

Referenced by ExpWorkerThreadBalanceManager().

◆ ExpThreadSetManagerEvent

KEVENT ExpThreadSetManagerEvent

Definition at line 45 of file work.c.

Referenced by ExpInitializeWorkerThreads(), ExpWorkerThreadBalanceManager(), and ExQueueWorkItem().

◆ ExpThreadSetManagerShutdownEvent

KEVENT ExpThreadSetManagerShutdownEvent

Definition at line 46 of file work.c.

Referenced by ExpInitializeWorkerThreads(), and ExpWorkerThreadBalanceManager().

◆ ExpWorkerListHead

LIST_ENTRY ExpWorkerListHead

Definition at line 41 of file work.c.

Referenced by ExpInitializeWorkerThreads().

◆ ExpWorkersCanSwap

BOOLEAN ExpWorkersCanSwap

Definition at line 40 of file work.c.

Referenced by ExpInitializeWorkerThreads(), ExpWorkerThreadEntryPoint(), and ExSwapinWorkerThreads().

◆ ExpWorkerSwapinMutex

FAST_MUTEX ExpWorkerSwapinMutex

Definition at line 42 of file work.c.

Referenced by ExpInitializeWorkerThreads(), and ExSwapinWorkerThreads().

◆ ExpWorkerThreadBalanceManagerPtr

PETHREAD ExpWorkerThreadBalanceManagerPtr

Definition at line 49 of file work.c.

Referenced by ExpInitializeWorkerThreads().

◆ ExWorkerQueue

EX_WORK_QUEUE ExWorkerQueue[MaximumWorkQueue]

Definition at line 31 of file work.c.

Referenced by ExpCheckDynamicThreadCount(), ExpCreateWorkerThread(), ExpDetectWorkerThreadDeadlock(), ExpInitializeWorkerThreads(), ExpWorkerThreadEntryPoint(), ExQueueWorkItem(), and KeTerminateThread().

Variables
EX_WORK_QUEUE	ExWorkerQueue [MaximumWorkQueue]

ULONG	ExCriticalWorkerThreads

ULONG	ExDelayedWorkerThreads

ULONG	ExpAdditionalCriticalWorkerThreads

ULONG	ExpAdditionalDelayedWorkerThreads

BOOLEAN	ExpWorkersCanSwap

LIST_ENTRY	ExpWorkerListHead

FAST_MUTEX	ExpWorkerSwapinMutex

KEVENT	ExpThreadSetManagerEvent

KEVENT	ExpThreadSetManagerShutdownEvent

PETHREAD	ExpWorkerThreadBalanceManagerPtr

PETHREAD	ExpLastWorkerThread

Macros

Functions

Variables

Macro Definition Documentation

◆ EX_CRITICAL_QUEUE_PRIORITY_INCREMENT

◆ EX_CRITICAL_WORK_THREADS

◆ EX_DELAYED_QUEUE_PRIORITY_INCREMENT

◆ EX_DELAYED_WORK_THREADS

◆ EX_DYNAMIC_WORK_THREAD

◆ EX_HYPERCRITICAL_QUEUE_PRIORITY_INCREMENT

◆ EX_HYPERCRITICAL_WORK_THREADS

◆ NDEBUG

Function Documentation

◆ ExpCheckDynamicThreadCount()

◆ ExpCreateWorkerThread()

◆ ExpDetectWorkerThreadDeadlock()

◆ ExpInitializeWorkerThreads()

◆ ExpSetSwappingKernelApc()

◆ ExpWorkerThreadBalanceManager()

◆ ExpWorkerThreadEntryPoint()

◆ ExQueueWorkItem()

◆ ExSwapinWorkerThreads()

Variable Documentation

◆ ExCriticalWorkerThreads

◆ ExDelayedWorkerThreads

◆ ExpAdditionalCriticalWorkerThreads

◆ ExpAdditionalDelayedWorkerThreads

◆ ExpLastWorkerThread

◆ ExpThreadSetManagerEvent

◆ ExpThreadSetManagerShutdownEvent

◆ ExpWorkerListHead

◆ ExpWorkersCanSwap

◆ ExpWorkerSwapinMutex

◆ ExpWorkerThreadBalanceManagerPtr

◆ ExWorkerQueue