intr.cpp File Reference

#include "xhcidrv.h"
#include <debug.h>

Include dependency graph for intr.cpp:

Go to the source code of this file.

Classes
struct	_XHCI_INTERRUPT_CONTEXT

Macros
#define	NDEBUG
#define	XHCI_SECONDARY_MAX   16
#define	XHCI_SEGMENTS_DEFAULT   4
#define	XHCI_SEGMENTS_RING_FULL   8
#define	XHCI_SEGMENTS_LIMITED   2
#define	XHCI_BUDGET_CHECK_EVENTS   64
#define	XHCI_HANDBACK_EVENTS   128
#define	XHCI_REASON_EVENT_RING_FULL   0x1011
#define	XHCI_DRAIN_WAIT_MS   60000

Typedefs
typedef XhciInterrupters::Interrupter	XHCI_INTERRUPTER
typedef XhciInterrupters::Interrupter *	PXHCI_INTERRUPTER
typedef struct _XHCI_INTERRUPT_CONTEXT	XHCI_INTERRUPT_CONTEXT
typedef struct _XHCI_INTERRUPT_CONTEXT *	PXHCI_INTERRUPT_CONTEXT
typedef ULONG(NTAPI *	PFN_XHCI_ACTIVE_COUNT) (_In_ USHORT GroupNumber)
typedef ULONG(NTAPI *	PFN_XHCI_INDEX_FROM_NUMBER) (_In_ PPROCESSOR_NUMBER Number)
typedef ULONG(NTAPI *	PFN_XHCI_CURRENT_NUMBER) (_Out_opt_ PPROCESSOR_NUMBER Number)
typedef NTSTATUS(NTAPI *	PFN_XHCI_DPC_WATCHDOG) (_Out_ PKDPC_WATCHDOG_INFORMATION Information)

Functions
	_IRQL_requires_ (PASSIVE_LEVEL) static PVOID NTAPI XhciFindKernelRoutine(_In_ PCWSTR Name)
	Sends a synchronous IOCTL to the specified device object.
static ULONG NTAPI	XhciActiveProcessors (_In_ USHORT Group)
static ULONG NTAPI	XhciProcessorIndex (_In_ USHORT Group, _In_ ULONG Number)
static ULONG NTAPI	XhciCurrentProcessorIndex (VOID)
static ULONG NTAPI	XhciReadInterrupter (_In_ PXHCI_INTERRUPTER Entry, _In_ ULONG Offset)
static VOID NTAPI	XhciWriteInterrupter (_In_ PXHCI_INTERRUPTER Entry, _In_ ULONG Offset, _In_ ULONG Value)
static VOID NTAPI	XhciWriteInterrupter64 (_In_ PXHCI_INTERRUPTER Entry, _In_ ULONG Offset, _In_ ULONG64 Value)
static VOID NTAPI	XhciWriteDequeue (_In_ PXHCI_INTERRUPTER Entry, _In_ BOOLEAN ClearBusy)
static VOID NTAPI	XhciResetEventRing (_Inout_ PXHCI_INTERRUPTER Entry)
static VOID NTAPI	XhciProgramEventRing (_In_ PXHCI_INTERRUPTER Entry)
static VOID NTAPI	XhciUnmaskInterrupter (_In_ PXHCI_INTERRUPTER Entry)
static BOOLEAN NTAPI	XhciEvtInterruptIsr (_In_ WDFINTERRUPT Interrupt, _In_ ULONG MessageID)
static NTSTATUS NTAPI	XhciEvtInterruptEnable (_In_ WDFINTERRUPT Interrupt, _In_ WDFDEVICE AssociatedDevice)
static NTSTATUS NTAPI	XhciEvtInterruptDisable (_In_ WDFINTERRUPT Interrupt, _In_ WDFDEVICE AssociatedDevice)
static BOOLEAN NTAPI	XhciDpcBudgetSpent (_Inout_ PBOOLEAN LimitsKnown, _Inout_ PULONG TimeLimit, _Inout_ PULONG WatchdogLimit)
static PXHCI_TRB NTAPI	XhciDequeueTrb (_In_ PXHCI_INTERRUPTER Entry)
static BOOLEAN NTAPI	XhciEventPending (_In_ PXHCI_INTERRUPTER Entry, _In_ PXHCI_TRB Trb)
static VOID NTAPI	XhciAdvanceDequeue (_Inout_ PXHCI_INTERRUPTER Entry)
static VOID NTAPI	XhciRouteTransferEvent (_In_ PXHCI_INTERRUPTER Entry, _Inout_ PXHCI_TRB Event)
static VOID NTAPI	XhciDispatchEvent (_In_ PXHCI_INTERRUPTER Entry, _Inout_ PXHCI_TRB Event)
static VOID NTAPI	XhciRequeueEvents (_Inout_ PXHCI_INTERRUPTER Entry)
static VOID NTAPI	XhciEvtRequeueWorkItem (_In_ WDFWORKITEM WorkItem)
static VOID NTAPI	XhciEvtInterruptDpc (_In_ WDFINTERRUPT Interrupt, _In_ WDFOBJECT AssociatedObject)
static BOOLEAN NTAPI	XhciIsMessageRequirement (_In_ PIO_RESOURCE_DESCRIPTOR Descriptor)
static BOOLEAN NTAPI	XhciIsMessageResource (_In_ PCM_PARTIAL_RESOURCE_DESCRIPTOR Descriptor)
	for (Current=0;Current< Lists;Current++)
	while (Index< WdfIoResourceListGetCount(List))

Variables
static PFN_XHCI_ACTIVE_COUNT	XhciActiveCountRoutine
static PFN_XHCI_INDEX_FROM_NUMBER	XhciIndexFromNumberRoutine
static PFN_XHCI_CURRENT_NUMBER	XhciCurrentNumberRoutine
static PFN_XHCI_DPC_WATCHDOG	XhciDpcWatchdogRoutine
static EVT_WDF_INTERRUPT_ISR	XhciEvtInterruptIsr
static EVT_WDF_INTERRUPT_DPC	XhciEvtInterruptDpc
static EVT_WDF_INTERRUPT_ENABLE	XhciEvtInterruptEnable
static EVT_WDF_INTERRUPT_DISABLE	XhciEvtInterruptDisable
static EVT_WDF_WORKITEM	XhciEvtRequeueWorkItem
_Out_ PULONG	ListIndex = 0
_Out_ PULONG _Out_ PULONG	Messages
PIO_RESOURCE_DESCRIPTOR	Descriptor
ULONG	Lists = WdfIoResourceRequirementsListGetCount(Requirements)
ULONG	Current
ULONG	Index = 0
ULONG	Count
ULONG	Descriptors
BOOLEAN	MsiX
BOOLEAN	BestMsiX = FALSE
_In_ ULONG	Wanted
IO_RESOURCE_DESCRIPTOR	Trimmed
ULONG	Offered
ULONG	Claim

Macro Definition Documentation

NDEBUG

#define NDEBUG

Definition at line 10 of file intr.cpp.

XHCI_BUDGET_CHECK_EVENTS

#define XHCI_BUDGET_CHECK_EVENTS   64

Definition at line 27 of file intr.cpp.

XHCI_DRAIN_WAIT_MS

#define XHCI_DRAIN_WAIT_MS   60000

Definition at line 30 of file intr.cpp.

XHCI_HANDBACK_EVENTS

#define XHCI_HANDBACK_EVENTS   128

Definition at line 28 of file intr.cpp.

XHCI_REASON_EVENT_RING_FULL

#define XHCI_REASON_EVENT_RING_FULL   0x1011

Definition at line 29 of file intr.cpp.

XHCI_SECONDARY_MAX

#define XHCI_SECONDARY_MAX   16

Definition at line 23 of file intr.cpp.

XHCI_SEGMENTS_DEFAULT

#define XHCI_SEGMENTS_DEFAULT   4

Definition at line 24 of file intr.cpp.

XHCI_SEGMENTS_LIMITED

#define XHCI_SEGMENTS_LIMITED   2

Definition at line 26 of file intr.cpp.

XHCI_SEGMENTS_RING_FULL

#define XHCI_SEGMENTS_RING_FULL   8

Definition at line 25 of file intr.cpp.

Typedef Documentation

PFN_XHCI_ACTIVE_COUNT

typedef ULONG(NTAPI * PFN_XHCI_ACTIVE_COUNT) (_In_ USHORT GroupNumber)

Definition at line 33 of file intr.cpp.

PFN_XHCI_CURRENT_NUMBER

typedef ULONG(NTAPI * PFN_XHCI_CURRENT_NUMBER) (_Out_opt_ PPROCESSOR_NUMBER Number)

Definition at line 35 of file intr.cpp.

PFN_XHCI_DPC_WATCHDOG

typedef NTSTATUS(NTAPI * PFN_XHCI_DPC_WATCHDOG) (_Out_ PKDPC_WATCHDOG_INFORMATION Information)

Definition at line 36 of file intr.cpp.

PFN_XHCI_INDEX_FROM_NUMBER

typedef ULONG(NTAPI * PFN_XHCI_INDEX_FROM_NUMBER) (_In_ PPROCESSOR_NUMBER Number)

Definition at line 34 of file intr.cpp.

PXHCI_INTERRUPT_CONTEXT

typedef struct _XHCI_INTERRUPT_CONTEXT * PXHCI_INTERRUPT_CONTEXT

PXHCI_INTERRUPTER

typedef XhciInterrupters::Interrupter * PXHCI_INTERRUPTER

Definition at line 13 of file intr.cpp.

XHCI_INTERRUPT_CONTEXT

typedef struct _XHCI_INTERRUPT_CONTEXT XHCI_INTERRUPT_CONTEXT

XHCI_INTERRUPTER

typedef XhciInterrupters::Interrupter XHCI_INTERRUPTER

Definition at line 13 of file intr.cpp.

Function Documentation

_IRQL_requires_()

_IRQL_requires_

(

PASSIVE_LEVEL

)

Sends a synchronous IOCTL to the specified device object.

Drops every MSI descriptor and keeps only lists that still offer a line interrupt.

Picks the list with the best MSI offer: MSI-X first, then the most messages.

Keeps message descriptors covering Wanted messages and removes the others.

Definition at line 51 of file intr.cpp.

{
    UNICODE_STRING RoutineName;
 
    RtlInitUnicodeString(&RoutineName, Name);
    return MmGetSystemRoutineAddress(&RoutineName);
}

for()

for

(

)

Definition at line 739 of file intr.cpp.

    {
        List = WdfIoResourceRequirementsListGetIoResList(Requirements, Current);
        Count = 0;
        Descriptors = 0;
 
        for (Index = 0; Index < WdfIoResourceListGetCount(List); Index++)
        {
            Descriptor = WdfIoResourceListGetDescriptor(List, Index);
            if (!XhciIsMessageRequirement(Descriptor) || (Descriptor->Option & IO_RESOURCE_ALTERNATIVE))
                continue;
 
            Descriptors++;
            Count += Descriptor->u.Interrupt.MaximumVector - Descriptor->u.Interrupt.MinimumVector + 1;
        }
 
        /* MSI-X lists carry one descriptor per message, MSI a single ranged one */
        MsiX = (Descriptors > 1);
 
        if (MsiX && (!BestMsiX || Count > *Messages))
        {
            BestMsiX = TRUE;
            *ListIndex = Current;
            *Messages = Count;
        }
        else if (!MsiX && !BestMsiX && Count > *Messages)
        {
            *ListIndex = Current;
            *Messages = Count;
        }
    }

while()

while

(

)

Definition at line 789 of file intr.cpp.

    {
        Descriptor = WdfIoResourceListGetDescriptor(List, Index);
        if (!XhciIsMessageRequirement(Descriptor))
        {
            Index++;
            continue;
        }
 
        if (Wanted == 0)
        {
            WdfIoResourceListRemove(List, Index);
            continue;
        }
 
        Offered = Descriptor->u.Interrupt.MaximumVector - Descriptor->u.Interrupt.MinimumVector + 1;
        if (Offered <= Wanted)
        {
            Wanted -= Offered;
            Index++;
            continue;
        }
 
        /* Multi message MSI grants powers of two only */
        Claim = 1;
        while (Claim < Wanted)
            Claim <<= 1;
 
        Trimmed = *Descriptor;
        Trimmed.u.Interrupt.MaximumVector = CM_RESOURCE_INTERRUPT_MESSAGE_TOKEN;
        Trimmed.u.Interrupt.MinimumVector = CM_RESOURCE_INTERRUPT_MESSAGE_TOKEN - Claim + 1;
        WdfIoResourceListUpdateDescriptor(List, &Trimmed, Index);
 
        DPRINT("Message descriptor %lu trimmed to %lu messages\n", Index, Claim);
        Wanted = 0;
        Index++;
    }

XhciActiveProcessors()

static ULONG NTAPI XhciActiveProcessors ( _In_ USHORT  Group )

static

Definition at line 67 of file intr.cpp.

{
    if (XhciActiveCountRoutine != NULL)
        return XhciActiveCountRoutine(Group);
 
    return KeQueryActiveProcessorCount(NULL);
}

Referenced by XhciInterrupters::AffinitizeResourceRequirements(), XhciInterrupters::BuildLookup(), and XhciInterrupters::SecondaryCountFor().

XhciAdvanceDequeue()

static VOID NTAPI XhciAdvanceDequeue ( _Inout_ PXHCI_INTERRUPTER  Entry )

static

Definition at line 366 of file intr.cpp.

{
    Entry->DequeueIndex++;
    if (Entry->DequeueIndex < Entry->TrbsPerSegment)
        return;
 
    Entry->DequeueIndex = 0;
    Entry->DequeueSegment++;
 
    if (Entry->DequeueSegment < Entry->SegmentCount)
    {
        Entry->Segment = CONTAINING_RECORD(Entry->Segment->Link.Flink, XhciDmaBuffer, Link);
        return;
    }
 
    /* Wrapped past the last ERST entry: the producer toggled its cycle state too */
    Entry->DequeueSegment = 0;
    Entry->CycleState ^= XHCI_TRB_CYCLE;
    Entry->Segment = CONTAINING_RECORD(Entry->SegmentList.Flink, XhciDmaBuffer, Link);
}

Referenced by XhciEvtInterruptDpc().

XhciCurrentProcessorIndex()

static ULONG NTAPI XhciCurrentProcessorIndex ( VOID  )

static

Definition at line 108 of file intr.cpp.

{
    if (XhciCurrentNumberRoutine != NULL)
        return XhciCurrentNumberRoutine(NULL);
 
    /* Single group kernels: the number is the index */
    return KeGetCurrentProcessorNumber();
}

Referenced by XhciInterrupters::TargetForCurrentProcessor().

XhciDequeueTrb()

static PXHCI_TRB NTAPI XhciDequeueTrb ( _In_ PXHCI_INTERRUPTER  Entry )

static

Definition at line 344 of file intr.cpp.

{
    return (PXHCI_TRB)Entry->Segment->VirtualAddress + Entry->DequeueIndex;
}

Referenced by XhciEvtInterruptDpc().

XhciDispatchEvent()

static VOID NTAPI XhciDispatchEvent ( _In_ PXHCI_INTERRUPTER  Entry, _Inout_ PXHCI_TRB  Event  )

static

Definition at line 434 of file intr.cpp.

{
    XhciController* Controller = Entry->Controller;
    ULONG Type = XhciTrbType(Event);
    ULONG Code = XhciTrbCompletionCode(Event);
 
    if (Code == static_cast<ULONG>(XhciCompletionCode::VendorQuirk199) &&
        Controller->HasErrata(XhciErrata::EvtDropVendorCode199))
    {
        DPRINT1("Dropping event type %lu with vendor completion code 199\n", Type);
        return;
    }
 
    if (Type == static_cast<ULONG>(XhciTrbType::TransferEvent))
    {
        XhciRouteTransferEvent(Entry, Event);
        return;
    }
 
    if (Type == static_cast<ULONG>(XhciTrbType::HostControllerEvent) &&
        Code == static_cast<ULONG>(XhciCompletionCode::EventRingFull))
    {
        Entry->RingFullCount++;
        DPRINT1("Interrupter %lu event ring full (%lu times)\n", Entry->Index, Entry->RingFullCount);
 
        if (Controller->HasErrata(XhciErrata::EvtResetOnRingOverflow))
            Controller->RaiseControllerFault(XhciRecovery::ResetHost, XHCI_REASON_EVENT_RING_FULL);
        return;
    }
 
    /* Only transfer events are targeted at secondary interrupters */
    if (Entry->Index != 0)
    {
        DPRINT1("Event type %lu on secondary interrupter %lu, dropped\n", Type, Entry->Index);
        return;
    }
 
    switch (Type)
    {
        case static_cast<ULONG>(XhciTrbType::CommandCompletionEvent):
        case static_cast<ULONG>(XhciTrbType::VendorCommandCompletionEvent):
            Controller->m_Commands.OnCompletionEvent(Event);
            break;
 
        case static_cast<ULONG>(XhciTrbType::PortStatusChangeEvent):
            Controller->m_RootHub.OnPortStatusChangeEvent(Event);
            break;
 
        case static_cast<ULONG>(XhciTrbType::DeviceNotificationEvent):
            if (Controller->m_Slots.LookupSlot(XhciTrbSlotId(Event)) != NULL)
                Controller->m_Slots.OnDeviceNotificationEvent(Event);
            break;
 
        case static_cast<ULONG>(XhciTrbType::HostControllerEvent):
            DPRINT1("Host controller event with completion code %lu\n", Code);
            break;
 
        case static_cast<ULONG>(XhciTrbType::MfindexWrapEvent):
            /* USBCMD.EWE is never set, so the controller should not send these */
            DPRINT1("Unexpected MFINDEX wrap event\n");
            break;
 
        default:
            DPRINT1("Unknown event type %lu (code %lu), dropped\n", Type, Code);
            break;
    }
}

Referenced by XhciEvtInterruptDpc().

XhciDpcBudgetSpent()

static BOOLEAN NTAPI XhciDpcBudgetSpent ( _Inout_ PBOOLEAN  LimitsKnown, _Inout_ PULONG  TimeLimit, _Inout_ PULONG  WatchdogLimit  )

static

Remaining DPC budget check. Limits are computed on the first successful query.

Definition at line 313 of file intr.cpp.

{
    KDPC_WATCHDOG_INFORMATION Watchdog;
 
    /* ReactOS may not export it; the DPC then never yields */
    if (XhciDpcWatchdogRoutine == NULL)
        return FALSE;
 
    if (!NT_SUCCESS(XhciDpcWatchdogRoutine(&Watchdog)))
        return FALSE;
 
    if (!*LimitsKnown)
    {
        *TimeLimit = (Watchdog.DpcTimeLimit * 95) / 100;
        *WatchdogLimit = (Watchdog.DpcWatchdogLimit * 25) / 100;
        *LimitsKnown = TRUE;
    }
 
    /* The counts are ticks left, so this yields early on purpose */
    if (Watchdog.DpcTimeCount < *TimeLimit)
        return TRUE;
 
    return (Watchdog.DpcWatchdogCount < *WatchdogLimit);
}

Referenced by XhciEvtInterruptDpc().

XhciEventPending()

static BOOLEAN NTAPI XhciEventPending ( _In_ PXHCI_INTERRUPTER  Entry, _In_ PXHCI_TRB  Trb  )

static

Definition at line 353 of file intr.cpp.

{
    ULONG Control;
 
    Control = *(volatile ULONG*)&Trb->Dword[3];
    return (Control & XHCI_TRB_CYCLE) == Entry->CycleState;
}

Referenced by XhciEvtInterruptDpc().

XhciEvtInterruptDisable()

static NTSTATUS NTAPI XhciEvtInterruptDisable ( _In_ WDFINTERRUPT  Interrupt, _In_ WDFDEVICE  AssociatedDevice  )

static

Definition at line 287 of file intr.cpp.

{
    PXHCI_INTERRUPTER Entry;
    ULONG Iman;
 
    UNREFERENCED_PARAMETER(AssociatedDevice);
 
    Entry = XhciInterruptContext(Interrupt)->Entry;
    if (Entry == NULL)
        return STATUS_SUCCESS;
 
    Entry->Enabled = FALSE;
 
    Iman = XhciReadInterrupter(Entry, XHCI_IMAN);
    XhciWriteInterrupter(Entry, XHCI_IMAN, Iman & ~XHCI_IMAN_ENABLE);
    return STATUS_SUCCESS;
}

XhciEvtInterruptDpc()

static VOID NTAPI XhciEvtInterruptDpc ( _In_ WDFINTERRUPT  Interrupt, _In_ WDFOBJECT  AssociatedObject  )

static

Definition at line 563 of file intr.cpp.

{
    PXHCI_INTERRUPTER Entry;
    PXHCI_TRB Trb;
    XHCI_TRB Event;
    ULONG Processed;
    BOOLEAN LimitsKnown = FALSE;
    ULONG TimeLimit = 0;
    ULONG WatchdogLimit = 0;
 
    UNREFERENCED_PARAMETER(AssociatedObject);
 
    Entry = XhciInterruptContext(Interrupt)->Entry;
    if (Entry == NULL)
        return;
 
    /* A second DPC or a pending requeue already owns the ring */
    KeAcquireSpinLockAtDpcLevel(&Entry->Lock);
    if (Entry->DpcRunning)
    {
        KeReleaseSpinLockFromDpcLevel(&Entry->Lock);
        return;
    }
    Entry->DpcRunning = TRUE;
    KeReleaseSpinLockFromDpcLevel(&Entry->Lock);
 
    Trb = XhciDequeueTrb(Entry);
    for (Processed = 0; XhciEventPending(Entry, Trb); Processed++)
    {
        if ((Processed % XHCI_BUDGET_CHECK_EVENTS) == 0 &&
            XhciDpcBudgetSpent(&LimitsKnown, &TimeLimit, &WatchdogLimit))
        {
            XhciWriteDequeue(Entry, FALSE);
            XhciRequeueEvents(Entry);
            return;
        }
 
        /* Copy only after the cycle bit said the TRB is complete */
        RtlCopyMemory(&Event, Trb, sizeof(Event));
        XhciAdvanceDequeue(Entry);
        XhciDispatchEvent(Entry, &Event);
 
        Trb = XhciDequeueTrb(Entry);
 
        /* Give consumed TRBs back early on long bursts; EHB stays set */
        if (Processed != 0 &&
            (Processed % XHCI_HANDBACK_EVENTS) == 0 &&
            XhciEventPending(Entry, Trb))
        {
            XhciWriteDequeue(Entry, FALSE);
        }
    }
 
    /* Written even when nothing was consumed, so a spurious interrupt still clears EHB */
    KeAcquireSpinLockAtDpcLevel(&Entry->Lock);
    Entry->DpcRunning = FALSE;
    XhciWriteDequeue(Entry, TRUE);
    KeReleaseSpinLockFromDpcLevel(&Entry->Lock);
}

XhciEvtInterruptEnable()

static NTSTATUS NTAPI XhciEvtInterruptEnable ( _In_ WDFINTERRUPT  Interrupt, _In_ WDFDEVICE  AssociatedDevice  )

static

Definition at line 262 of file intr.cpp.

{
    PXHCI_INTERRUPTER Entry;
 
    UNREFERENCED_PARAMETER(AssociatedDevice);
 
    Entry = XhciInterruptContext(Interrupt)->Entry;
    if (Entry == NULL)
        return STATUS_SUCCESS;
 
    Entry->Enabled = TRUE;
    Entry->PendingDisable = FALSE;
 
    /* Drop a stale DpcRunning left by a requeue skipped at D0 exit */
    Entry->DpcRunning = FALSE;
 
    XhciUnmaskInterrupter(Entry);
    return STATUS_SUCCESS;
}

XhciEvtInterruptIsr()

static BOOLEAN NTAPI XhciEvtInterruptIsr ( _In_ WDFINTERRUPT  Interrupt, _In_ ULONG  MessageID  )

static

Definition at line 234 of file intr.cpp.

{
    PXHCI_INTERRUPTER Entry;
    ULONG Iman;
 
    Entry = XhciInterruptContext(Interrupt)->Entry;
    if (Entry == NULL)
        return FALSE;
 
    /* A shared line needs IMAN to tell whether this controller raised it */
    if (MessageID == 0 && !Entry->MessageSignaled)
    {
        Iman = XhciReadInterrupter(Entry, XHCI_IMAN);
        if ((Iman & XHCI_IMAN_PENDING) == 0)
            return FALSE;
 
        XhciWriteInterrupter(Entry, XHCI_IMAN, Iman);
    }
 
    WdfInterruptQueueDpcForIsr(Interrupt);
    return TRUE;
}

XhciEvtRequeueWorkItem()

static VOID NTAPI XhciEvtRequeueWorkItem ( _In_ WDFWORKITEM  WorkItem )

static

Definition at line 542 of file intr.cpp.

{
    WDFINTERRUPT Interrupt;
    PXHCI_INTERRUPTER Entry;
    KIRQL OldIrql;
 
    Interrupt = (WDFINTERRUPT)WdfWorkItemGetParentObject(WorkItem);
    Entry = XhciInterruptContext(Interrupt)->Entry;
 
    KeAcquireSpinLock(&Entry->Lock, &OldIrql);
    Entry->DpcRunning = FALSE;
    KeSetEvent(&Entry->RequeueIdle, IO_NO_INCREMENT, FALSE);
    KeReleaseSpinLock(&Entry->Lock, OldIrql);
 
    WdfInterruptQueueDpcForIsr(Interrupt);
}

XhciIsMessageRequirement()

static BOOLEAN NTAPI XhciIsMessageRequirement ( _In_ PIO_RESOURCE_DESCRIPTOR  Descriptor )

static

Definition at line 630 of file intr.cpp.

{
    const USHORT MessageFlags = CM_RESOURCE_INTERRUPT_LATCHED | CM_RESOURCE_INTERRUPT_MESSAGE;
 
    return Descriptor->Type == CmResourceTypeInterrupt &&
           (Descriptor->Flags & MessageFlags) == MessageFlags;
}

Referenced by XhciInterrupters::AffinitizeResourceRequirements(), for(), and while().

XhciIsMessageResource()

static BOOLEAN NTAPI XhciIsMessageResource ( _In_ PCM_PARTIAL_RESOURCE_DESCRIPTOR  Descriptor )

static

Definition at line 642 of file intr.cpp.

{
    const USHORT MessageFlags = CM_RESOURCE_INTERRUPT_LATCHED | CM_RESOURCE_INTERRUPT_MESSAGE;
 
    return Descriptor->Type == CmResourceTypeInterrupt &&
           (Descriptor->Flags & MessageFlags) == MessageFlags;
}

Referenced by XhciInterrupters::Prepare().

XhciProcessorIndex()

static ULONG NTAPI XhciProcessorIndex ( _In_ USHORT  Group, _In_ ULONG  Number  )

static

Definition at line 79 of file intr.cpp.

{
    PROCESSOR_NUMBER Processor;
 
    if (XhciIndexFromNumberRoutine != NULL)
    {
        RtlZeroMemory(&Processor, sizeof(Processor));
        Processor.Group = Group;
        Processor.Number = (UCHAR)Number;
        return XhciIndexFromNumberRoutine(&Processor);
    }
 
    /* Without group support only group 0 exists and the index is the number */
    return (Group == 0) ? Number : MAXULONG;
}

Referenced by XhciInterrupters::BuildLookup().

XhciProgramEventRing()

static VOID NTAPI XhciProgramEventRing ( _In_ PXHCI_INTERRUPTER  Entry )

static

ERSTSZ, ERDP, then ERSTBA, which arms the event ring (xHCI 4.9.4).

Definition at line 206 of file intr.cpp.

{
    XhciWriteInterrupter(Entry, XHCI_ERSTSZ, Entry->SegmentCount);
    XhciWriteDequeue(Entry, TRUE);
    XhciWriteInterrupter64(Entry, XHCI_ERSTBA, (ULONG64)Entry->Erst->LogicalAddress.QuadPart);
}

Referenced by XhciInterrupters::D0Entry(), and XhciInterrupters::PostReset().

XhciReadInterrupter()

static ULONG NTAPI XhciReadInterrupter ( _In_ PXHCI_INTERRUPTER  Entry, _In_ ULONG  Offset  )

static

Definition at line 128 of file intr.cpp.

{
    return Entry->Controller->m_Registers.ReadRuntime32(Entry->RegisterOffset + Offset);
}

Referenced by XhciEvtInterruptDisable(), XhciEvtInterruptIsr(), and XhciUnmaskInterrupter().

XhciRequeueEvents()

static VOID NTAPI XhciRequeueEvents ( _Inout_ PXHCI_INTERRUPTER  Entry )

static

Hands the rest of the ring to a later DPC. EHB stays set, so no interrupt arrives meanwhile.

Definition at line 508 of file intr.cpp.

{
    KeAcquireSpinLockAtDpcLevel(&Entry->Lock);
 
    /* D0 exit is draining: leave the rest for the next D0 entry */
    if (Entry->PendingDisable)
    {
        KeReleaseSpinLockFromDpcLevel(&Entry->Lock);
        DPRINT("Interrupter %lu requeue skipped, disable pending\n", Entry->Index);
        return;
    }
 
    KeClearEvent(&Entry->RequeueIdle);
    KeReleaseSpinLockFromDpcLevel(&Entry->Lock);
 
    /* Requeue is NULL if creation failed or it was already torn down */
    if (Entry->Requeue != NULL)
    {
        WdfWorkItemEnqueue(Entry->Requeue);
        return;
    }
 
    KeAcquireSpinLockAtDpcLevel(&Entry->Lock);
    Entry->DpcRunning = FALSE;
    KeSetEvent(&Entry->RequeueIdle, IO_NO_INCREMENT, FALSE);
    KeReleaseSpinLockFromDpcLevel(&Entry->Lock);
 
    WdfInterruptQueueDpcForIsr(Entry->Interrupt);
}

Referenced by XhciEvtInterruptDpc().

XhciResetEventRing()

static VOID NTAPI XhciResetEventRing ( _Inout_ PXHCI_INTERRUPTER  Entry )

static

Zeroes the event ring and puts the consumer back at the start (xHC halted).

Definition at line 181 of file intr.cpp.

{
    PLIST_ENTRY Link;
    XhciDmaBuffer* Buffer;
 
    for (Link = Entry->SegmentList.Flink; Link != &Entry->SegmentList; Link = Link->Flink)
    {
        Buffer = CONTAINING_RECORD(Link, XhciDmaBuffer, Link);
        RtlZeroMemory(Buffer->VirtualAddress, Entry->SegmentSize);
    }
 
    Entry->Segment = CONTAINING_RECORD(Entry->SegmentList.Flink, XhciDmaBuffer, Link);
    Entry->CycleState = XHCI_TRB_CYCLE;
    Entry->DequeueIndex = 0;
    Entry->DequeueSegment = 0;
 
    /* A requeue skipped at D0 exit can leave this set */
    Entry->DpcRunning = FALSE;
}

Referenced by XhciInterrupters::D0Entry(), and XhciInterrupters::PostReset().

XhciRouteTransferEvent()

static VOID NTAPI XhciRouteTransferEvent ( _In_ PXHCI_INTERRUPTER  Entry, _Inout_ PXHCI_TRB  Event  )

static

Definition at line 391 of file intr.cpp.

{
    XhciController* Controller = Entry->Controller;
    ULONG Code = XhciTrbCompletionCode(Event);
    BOOLEAN Isoch;
    BOOLEAN Direct;
 
    if ((Event->Dword[3] & XHCI_TRANSFER_EVENT_ED) &&
        Controller->HasErrata(XhciErrata::IsochStarveEventNoEd) &&
        (Code == static_cast<ULONG>(XhciCompletionCode::RingUnderrun) ||
         Code == static_cast<ULONG>(XhciCompletionCode::RingOverrun)))
    {
        /* These parts report ring empty with ED set and a zero pointer */
        Event->Dword[3] &= ~XHCI_TRANSFER_EVENT_ED;
    }
 
    Isoch = ((Event->Dword[0] & XHCI_EVENT_DATA_TYPE_MASK) == USB_ENDPOINT_TYPE_ISOCHRONOUS);
    Direct = (Event->Dword[3] & XHCI_TRANSFER_EVENT_ED) &&
             !Isoch &&
             !Controller->HasErrata(XhciErrata::EvtCheckTrbAddress);
 
    /* The transfer ring takes it straight away unless it has to go through the endpoint */
    if (Direct && XhciTransferRing::OnUntargetedTransferEvent(Controller, Event))
        return;
 
    if (Controller->m_Slots.LookupSlot(XhciTrbSlotId(Event)) == NULL)
    {
        if (!Isoch)
        {
            DPRINT1("Transfer event for slot %lu endpoint %lu has no device, dropped\n",
                    XhciTrbSlotId(Event), (Event->Dword[3] & XHCI_TRB_ENDPOINT_MASK) >> XHCI_TRB_ENDPOINT_SHIFT);
        }
        return;
    }
 
    Controller->m_Slots.OnTransferEvent(Event);
}

Referenced by XhciDispatchEvent().

XhciUnmaskInterrupter()

static VOID NTAPI XhciUnmaskInterrupter ( _In_ PXHCI_INTERRUPTER  Entry )

static

Definition at line 217 of file intr.cpp.

{
    ULONG Iman;
 
    XhciWriteInterrupter(Entry, XHCI_IMOD, XHCI_IMOD_INTERVAL_50US);
 
    /* IP is write one to clear, so writing back what was read also drops a stale IP */
    Iman = XhciReadInterrupter(Entry, XHCI_IMAN);
    XhciWriteInterrupter(Entry, XHCI_IMAN, Iman | XHCI_IMAN_ENABLE);
}

Referenced by XhciInterrupters::PostReset(), and XhciEvtInterruptEnable().

XhciWriteDequeue()

static VOID NTAPI XhciWriteDequeue ( _In_ PXHCI_INTERRUPTER  Entry, _In_ BOOLEAN  ClearBusy  )

static

Writes ERDP from the software dequeue position; ClearBusy also writes EHB to clear it.

Definition at line 161 of file intr.cpp.

{
    ULONG64 Value;
 
    Value = (ULONG64)Entry->Segment->LogicalAddress.QuadPart +
            (ULONG64)Entry->DequeueIndex * sizeof(XHCI_TRB);
    Value |= Entry->DequeueSegment & XHCI_ERDP_SEGMENT_MASK;
 
    if (ClearBusy)
        Value |= XHCI_ERDP_BUSY;
 
    XhciWriteInterrupter64(Entry, XHCI_ERDP, Value);
}

Referenced by XhciEvtInterruptDpc(), and XhciProgramEventRing().

XhciWriteInterrupter()

static VOID NTAPI XhciWriteInterrupter ( _In_ PXHCI_INTERRUPTER  Entry, _In_ ULONG  Offset, _In_ ULONG  Value  )

static

Definition at line 138 of file intr.cpp.

{
    Entry->Controller->m_Registers.WriteRuntime32(Entry->RegisterOffset + Offset, Value);
}

Referenced by XhciInterrupters::PrepareOne(), XhciInterrupters::ReleaseOne(), XhciEvtInterruptDisable(), XhciEvtInterruptIsr(), XhciProgramEventRing(), and XhciUnmaskInterrupter().

XhciWriteInterrupter64()

static VOID NTAPI XhciWriteInterrupter64 ( _In_ PXHCI_INTERRUPTER  Entry, _In_ ULONG  Offset, _In_ ULONG64  Value  )

static

Definition at line 149 of file intr.cpp.

{
    Entry->Controller->m_Registers.WriteRuntime64(Entry->RegisterOffset + Offset, Value);
}

Referenced by XhciInterrupters::ReleaseOne(), XhciProgramEventRing(), and XhciWriteDequeue().

Variable Documentation

BestMsiX

return BestMsiX = FALSE

Definition at line 733 of file intr.cpp.

Referenced by for().

Claim

ULONG Claim

Definition at line 787 of file intr.cpp.

Referenced by while().

Count

ULONG Count

Definition at line 730 of file intr.cpp.

Referenced by for().

Current

ULONG Current

Definition at line 728 of file intr.cpp.

Referenced by for().

Descriptor

PIO_RESOURCE_DESCRIPTOR Descriptor

Definition at line 726 of file intr.cpp.

Referenced by while().

Descriptors

ULONG Descriptors

Definition at line 731 of file intr.cpp.

Referenced by AtaReqScsiInquiryDeviceIdentifiers(), ClasspDeviceGetLBAStatusWorker(), DeviceProcessDsmTrimRequest(), for(), HalpQueryAcpiResourceRequirements(), KspPinPropertyHandler(), KspValidateConnectRequest(), PciAddResizableBarRequirements(), PciIdeXPdoQueryResourceRequirements(), PdoQueryResourceRequirements(), and USBPORT_RootHubCreateDevice().

Index

ULONG Index = 0

Definition at line 729 of file intr.cpp.

ListIndex

* ListIndex = 0

Definition at line 722 of file intr.cpp.

Referenced by XhciInterrupters::AffinitizeResourceRequirements(), XhciInterrupters::FilterResourceRequirements(), for(), and UpdatePagefileEntry().

Lists

Lists = WdfIoResourceRequirementsListGetCount(Requirements)

Definition at line 727 of file intr.cpp.

Messages

* Messages

Initial value:

{
    WDFIORESLIST List

Definition at line 723 of file intr.cpp.

Referenced by XhciInterrupters::AffinitizeResourceRequirements(), XhciInterrupters::FilterResourceRequirements(), for(), XhciInterrupters::SecondaryCountFor(), and WndProc().

MsiX

BOOLEAN MsiX

Definition at line 732 of file intr.cpp.

Referenced by for().

Offered

ULONG Offered

Definition at line 786 of file intr.cpp.

Referenced by while().

Trimmed

IO_RESOURCE_DESCRIPTOR Trimmed

Definition at line 784 of file intr.cpp.

Referenced by MmWorkingSetManager(), and while().

Wanted

_In_ ULONG Wanted

Initial value:

{
    PIO_RESOURCE_DESCRIPTOR Descriptor

Definition at line 781 of file intr.cpp.

Referenced by Ext2ExpandBlock(), Ext2TruncateExtent(), Ext2TruncateIndirect(), FieldMatches(), HubIoGetConnectionInformationV2(), HubLinkChange(), and while().

XhciActiveCountRoutine

PFN_XHCI_ACTIVE_COUNT XhciActiveCountRoutine

static

Definition at line 38 of file intr.cpp.

Referenced by XhciInterrupters::Create(), and XhciActiveProcessors().

XhciCurrentNumberRoutine

PFN_XHCI_CURRENT_NUMBER XhciCurrentNumberRoutine

static

Definition at line 40 of file intr.cpp.

Referenced by XhciInterrupters::Create(), and XhciCurrentProcessorIndex().

XhciDpcWatchdogRoutine

PFN_XHCI_DPC_WATCHDOG XhciDpcWatchdogRoutine

static

Definition at line 41 of file intr.cpp.

Referenced by XhciInterrupters::Create(), and XhciDpcBudgetSpent().

XhciEvtInterruptDisable

EVT_WDF_INTERRUPT_DISABLE XhciEvtInterruptDisable

static

Definition at line 46 of file intr.cpp.

Referenced by XhciInterrupters::CreateInterrupt().

XhciEvtInterruptDpc

EVT_WDF_INTERRUPT_DPC XhciEvtInterruptDpc

static

Definition at line 44 of file intr.cpp.

Referenced by XhciInterrupters::CreateInterrupt().

XhciEvtInterruptEnable

EVT_WDF_INTERRUPT_ENABLE XhciEvtInterruptEnable

static

Definition at line 45 of file intr.cpp.

Referenced by XhciInterrupters::CreateInterrupt().

XhciEvtInterruptIsr

EVT_WDF_INTERRUPT_ISR XhciEvtInterruptIsr

static

Definition at line 43 of file intr.cpp.

Referenced by XhciInterrupters::CreateInterrupt().

XhciEvtRequeueWorkItem

EVT_WDF_WORKITEM XhciEvtRequeueWorkItem

static

Definition at line 47 of file intr.cpp.

Referenced by XhciInterrupters::PrepareOne().

XhciIndexFromNumberRoutine

PFN_XHCI_INDEX_FROM_NUMBER XhciIndexFromNumberRoutine

static

Definition at line 39 of file intr.cpp.

Referenced by XhciInterrupters::Create(), and XhciProcessorIndex().