XhciControlRing Class Reference

#include <ctrlxfer.h>

Collaboration diagram for XhciControlRing:

Static Public Member Functions
static NTSTATUS	Initialize (_In_ XhciTransferRing *Ring)
static VOID	Enable (_In_ XhciTransferRing *Ring)
static VOID	Disable (_In_ XhciTransferRing *Ring)
static NTSTATUS	EnableForwardProgress (_In_ XhciTransferRing *Ring, _In_ ULONG MaxTransferSize)
static VOID	Cleanup (_In_ XhciTransferRing *Ring)
static VOID	ResumeRingFill (_In_ XhciTransferRing *Ring)
static VOID	SuspendRingFill (_In_ XhciTransferRing *Ring)
static VOID	AllowReclaimOnCancel (_In_ XhciTransferRing *Ring)
static VOID	ConsumePendingEvents (_In_ XhciTransferRing *Ring)
static VOID	RecoverTransfers (_In_ XhciTransferRing *Ring)
static BOOLEAN	DoorbellRungSinceFill (_In_ const XhciTransferRing *Ring)
static BOOLEAN	HasQueuedWork (_In_ const XhciTransferRing *Ring)
static BOOLEAN	OnTransferEvent (_In_opt_ XhciTransferRing *Ring, _In_ XhciController *Controller, _In_ const XHCI_TRB *Event)

Static Public Attributes
static EVT_WDF_IO_QUEUE_IO_DEFAULT	EvtIoDefault
static EVT_WDF_IO_QUEUE_IO_CANCELED_ON_QUEUE	EvtIoCanceledOnQueue
static EVT_WDF_DPC	EvtCompletionDpc

Private Member Functions
struct	DECLSPEC_ALIGN (8) EventTag
VOID	InitTransfer (_Out_ XhciControlTransfer *Transfer, _In_ WDFREQUEST Request)
NTSTATUS	PrepareTransfer (_Inout_ XhciControlTransfer *Transfer)
VOID	BuildControlTd (_Inout_ XhciControlTransfer *Transfer)
VOID	WriteTd (_Inout_ XhciControlTransfer *Transfer)
VOID	ReleaseResources (_Inout_ XhciControlTransfer *Transfer)
	_Requires_lock_held_ (m_Ring->m_Lock) NTSTATUS CompleteHeld()
	_Requires_lock_held_ (m_Ring->m_Lock) VOID Finish(_Inout_ XhciControlTransfer *Transfer)
VOID	Abort (_Inout_ XhciControlTransfer *Transfer, _In_ BOOLEAN FromTimer)
VOID	QueueCompletion ()
	_Requires_lock_held_ (m_Ring->m_Lock) VOID CopyIn(_Inout_ XhciControlTransfer *Transfer
BOOLEAN	OnEventDataEvent (_In_ EventTag *Tag, _In_ const XHCI_TRB *Event)
BOOLEAN	OnTrbEvent (_In_ const XHCI_TRB *Event)
BOOLEAN	LocateTrb (_In_ const XhciControlTransfer *Transfer, _In_ const XHCI_TRB *Event, _Out_ PULONG Bytes, _Out_ PULONG SkippedTds) const

Static Private Member Functions
static EventTag *	TagFromEvent (_In_opt_ XhciTransferRing *Ring, _In_ const XHCI_TRB *Event)

Private Attributes
_In_ ULONG	Bytes
EventTag	m_Tags [TagCount]
XhciTransferRing *	m_Ring
XhciControlTransfer *	m_Current
LIST_ENTRY	m_CanceledOnQueue
WDFDPC	m_CanceledOnQueueDpc
WDFTIMER	m_Timer
ULONG	m_NextTag
LONG	m_Outstanding
volatile LONG	m_ReservedMdlBusy
BOOLEAN	m_ImmediateAllowed
BOOLEAN	m_ReclaimAndAcknowledge
BOOLEAN	m_AcknowledgeExpectedEvents
BOOLEAN	m_ReclaimOnCancelAllowed
BOOLEAN	m_CanceledDpcRunning

Static Private Attributes
static const ULONG	TagCount = 4
static EVT_WDF_TIMER	EvtTimeout
static EVT_WDF_DPC	EvtCanceledOnQueueDpc
static EVT_WDF_REQUEST_CANCEL	EvtRequestCancel
static DRIVER_LIST_CONTROL	EvtDmaReady

Detailed Description

Control ring state and the entry points the transfer ring calls for it.

Definition at line 15 of file ctrlxfer.h.

Member Function Documentation

_Requires_lock_held_() [1/3]

XhciControlRing::_Requires_lock_held_ ( m_Ring->  m_Lock )

private

_Requires_lock_held_() [2/3]

XhciControlRing::_Requires_lock_held_ ( m_Ring->  m_Lock )

private

_Requires_lock_held_() [3/3]

XhciControlRing::_Requires_lock_held_ ( m_Ring->  m_Lock )

private

Abort()

VOID XhciControlRing::Abort ( _Inout_ XhciControlTransfer *  Transfer, _In_ BOOLEAN  FromTimer  )

private

Shared by the cancel callback and the timeout timer; both act like a cancel.

Definition at line 984 of file ctrlxfer.cpp.

{
    XhciTransferRing* Ring = m_Ring;
    BOOLEAN Reported = FALSE;
    BOOLEAN Queue = FALSE;
    BOOLEAN Pending;
    BOOLEAN Lost;
    KIRQL OldIrql;
 
    KeAcquireSpinLock(&Ring->m_Lock, &OldIrql);
    if (FromTimer)
        Pending = (Transfer->TimeoutState == XhciControlTimeout::Armed);
    else
        Pending = (Transfer->CancelState == XhciControlCancel::Armed);
 
    if (Pending)
    {
        if (m_ReclaimOnCancelAllowed)
            Queue = TRUE;
        else
            Reported = TRUE;
    }
    KeReleaseSpinLock(&Ring->m_Lock, OldIrql);
 
    if (Reported)
        Ring->m_Endpoint->OnTransferCanceled();
 
    KeAcquireSpinLock(&Ring->m_Lock, &OldIrql);
    if (FromTimer)
        Lost = (Transfer->TimeoutState == XhciControlTimeout::TimerRoutinePending);
    else
        Lost = (Transfer->CancelState == XhciControlCancel::CancelRoutinePending);
 
    /* A completion left this request to us. After a reported cancel it waits for reclaim, or that permission would complete the next request */
    if (Lost && (!Reported || m_ReclaimAndAcknowledge || m_ReclaimOnCancelAllowed))
        Queue = TRUE;
 
    /* Last, so nothing completes the request while this callback still uses it */
    if (FromTimer)
        Transfer->TimeoutState = XhciControlTimeout::Expired;
    else
        Transfer->CancelState = XhciControlCancel::Done;
    KeReleaseSpinLock(&Ring->m_Lock, OldIrql);
 
    if (Queue)
        QueueCompletion();
}

AllowReclaimOnCancel()

VOID XhciControlRing::AllowReclaimOnCancel ( _In_ XhciTransferRing *  Ring )

static

Definition at line 1221 of file ctrlxfer.cpp.

{
    XhciControlRing* Self = XhciCtlOf(Ring);
    BOOLEAN Queue;
    KIRQL OldIrql;
 
    KeAcquireSpinLock(&Ring->m_Lock, &OldIrql);
    Queue = (Self->m_Current != NULL &&
             (Self->m_Current->CancelState == XhciControlCancel::Done ||
              Self->m_Current->TimeoutState == XhciControlTimeout::Expired));
    Self->m_ReclaimOnCancelAllowed = TRUE;
    KeReleaseSpinLock(&Ring->m_Lock, OldIrql);
 
    if (Queue)
        Self->QueueCompletion();
}

Referenced by XhciTransferRing::AllowReclaimOnCancel().

BuildControlTd()

VOID XhciControlRing::BuildControlTd ( _Inout_ XhciControlTransfer *  Transfer )

private

Definition at line 530 of file ctrlxfer.cpp.

{
    XhciTransferRing* Ring = m_Ring;
    PURB Urb = Transfer->Urb;
    PDMA_ADAPTER Adapter;
    BOOLEAN Acknowledge = FALSE;
    NTSTATUS Status;
    KIRQL OldIrql;
 
    Status = PrepareTransfer(Transfer);
    if (NT_SUCCESS(Status))
    {
        if (Urb->UrbHeader.Function == URB_FUNCTION_CONTROL_TRANSFER_EX &&
            Urb->UrbControlTransferEx.Timeout != 0)
        {
            KeAcquireSpinLock(&Ring->m_Lock, &OldIrql);
            if (Transfer->TimeoutState == XhciControlTimeout::Idle ||
                Transfer->TimeoutState == XhciControlTimeout::Armed)
            {
                Transfer->TimeoutState = XhciControlTimeout::Armed;
                WdfTimerStart(m_Timer, WDF_REL_TIMEOUT_IN_MS(Urb->UrbControlTransferEx.Timeout));
            }
            KeReleaseSpinLock(&Ring->m_Lock, OldIrql);
        }
 
        Ring->UpdateInterrupterTarget(XhciTransferRing::UrbProcessorIndex(Urb));
 
        if (Transfer->Mechanism != XhciControlMechanism::Dma)
        {
            WriteTd(Transfer);
            return;
        }
 
        Adapter = Ring->m_Controller->m_Buffers.DmaAdapter();
        KeRaiseIrql(DISPATCH_LEVEL, &OldIrql);
        Status = Adapter->DmaOperations->GetScatterGatherList(Adapter,
                                                              Ring->m_Controller->m_WdmDevice,
                                                              Transfer->Mdl,
                                                              MmGetMdlVirtualAddress(Transfer->Mdl),
                                                              Transfer->BytesTotal,
                                                              EvtDmaReady,
                                                              Transfer,
                                                              !Transfer->DataIn);
        KeLowerIrql(OldIrql);
 
        if (NT_SUCCESS(Status))
            return;
 
        DPRINT1("Control request %p: scatter gather list not built, status 0x%lx\n",
                Transfer->Request, Status);
    }
 
    KeAcquireSpinLock(&Ring->m_Lock, &OldIrql);
 
    Transfer->Status = Status;
 
    if (XhciCtlMapState(Ring) == XhciMapState::Filling)
    {
        XhciCtlSetMapState(Ring, XhciMapState::PausedFill);
        if (m_Current == Transfer)
            CompleteHeld();
    }
    else if (XhciCtlMapState(Ring) == XhciMapState::Halting)
    {
        /* The request stays held; reclaim completes it with the stored status */
        XhciCtlSetMapState(Ring, XhciMapState::Halted);
        Acknowledge = TRUE;
    }
 
    KeReleaseSpinLock(&Ring->m_Lock, OldIrql);
 
    if (Acknowledge)
        Ring->m_Endpoint->OnMappingStopped();
}

Referenced by ResumeRingFill().

Cleanup()

VOID XhciControlRing::Cleanup ( _In_ XhciTransferRing *  Ring )

static

Definition at line 286 of file ctrlxfer.cpp.

{
    XhciControlRing* Self = XhciCtlOf(Ring);
    ULONG Index;
 
    if (Self->m_Current != NULL)
        DPRINT1("Control ring %p cleaned up while it holds request %p\n", Ring, Self->m_Current->Request);
 
    /* A late fast path event must not take freed ring memory for a live tag */
    for (Index = 0; Index < TagCount; Index++)
    {
        Self->m_Tags[Index].Signature = 0;
        Self->m_Tags[Index].Transfer = NULL;
    }
}

Referenced by XhciTransferRing::Cleanup().

ConsumePendingEvents()

VOID XhciControlRing::ConsumePendingEvents ( _In_ XhciTransferRing *  Ring )

static

Definition at line 1240 of file ctrlxfer.cpp.

{
    XhciControlRing* Self = XhciCtlOf(Ring);
    BOOLEAN Drained;
    KIRQL OldIrql;
 
    KeAcquireSpinLock(&Ring->m_Lock, &OldIrql);
    Drained = (Self->m_Outstanding == 0);
    if (!Drained)
        Self->m_AcknowledgeExpectedEvents = TRUE;
    KeReleaseSpinLock(&Ring->m_Lock, OldIrql);
 
    if (Drained)
        Ring->m_Endpoint->OnExpectedEventsProcessed();
}

Referenced by XhciTransferRing::ConsumePendingEvents().

DECLSPEC_ALIGN()

struct XhciControlRing::DECLSPEC_ALIGN ( 8  )

inlineprivate

What an Event Data TRB points at. Tags live in the ring, not in the request, so a late or duplicate event never reads freed request memory.

Definition at line 63 of file ctrlxfer.h.

    {
        ULONG Signature;
        ULONG Index;
        XhciTransferRing* Ring;
        XhciControlTransfer* Transfer;
    };

Disable()

VOID XhciControlRing::Disable ( _In_ XhciTransferRing *  Ring )

static

Definition at line 237 of file ctrlxfer.cpp.

{
    XhciControlRing* Self = XhciCtlOf(Ring);
    XhciMapState State;
    KIRQL OldIrql;
 
    /* The common ring already waited, with a limit, for the last mapping context to unwind */
    KeAcquireSpinLock(&Ring->m_Lock, &OldIrql);
 
    State = XhciCtlMapState(Ring);
    if (State == XhciMapState::PausedFill)
        XhciCtlSetMapState(Ring, XhciMapState::Halted);
    else if (State == XhciMapState::Filling || State == XhciMapState::Halting)
        DPRINT1("Control ring %p disabled in map state %ld\n", Ring, static_cast<LONG>(State));
 
    if (Self->m_Current != NULL)
        DPRINT1("Control ring %p disabled while it holds request %p\n", Ring, Self->m_Current->Request);
 
    KeReleaseSpinLock(&Ring->m_Lock, OldIrql);
}

DoorbellRungSinceFill()

BOOLEAN XhciControlRing::DoorbellRungSinceFill ( _In_ const XhciTransferRing *  Ring )

static

Definition at line 1278 of file ctrlxfer.cpp.

{
    XhciTransferRing* Writable = const_cast<XhciTransferRing*>(Ring);
    BOOLEAN Rung;
    KIRQL OldIrql;
 
    ASSERT(XhciCtlMapState(Ring) == XhciMapState::Halted);
 
    KeAcquireSpinLock(&Writable->m_Lock, &OldIrql);
    Rung = Writable->m_DoorbellRung;
    KeReleaseSpinLock(&Writable->m_Lock, OldIrql);
 
    return Rung;
}

Referenced by XhciTransferRing::DoorbellRungSinceFill().

Enable()

VOID XhciControlRing::Enable ( _In_ XhciTransferRing *  Ring )

static

Definition at line 230 of file ctrlxfer.cpp.

{
    UNREFERENCED_PARAMETER(Ring);
}

Referenced by _IRQL_requires_max_().

EnableForwardProgress()

NTSTATUS XhciControlRing::EnableForwardProgress ( _In_ XhciTransferRing *  Ring, _In_ ULONG  MaxTransferSize  )

static

Definition at line 260 of file ctrlxfer.cpp.

{
    XhciControlRing* Self = XhciCtlOf(Ring);
    ULONG Pages;
    ULONG Trbs;
    KIRQL OldIrql;
 
    PAGED_CODE();
 
    /* Worst case page span of the buffer, then setup, two Event Data, status and a link */
    Pages = (PAGE_SIZE - 1 + MaxTransferSize + PAGE_SIZE - 1) / PAGE_SIZE;
    Trbs = Pages + 5;
    if (Trbs > Ring->m_LastIndex + 1)
        DPRINT1("Control ring %p: %lu TRBs for a forward progress transfer exceed the segment\n", Ring, Trbs);
 
    KeAcquireSpinLock(&Ring->m_Lock, &OldIrql);
    Ring->m_DoubleBufferSize = 0;
    Self->m_ImmediateAllowed = FALSE;
    KeReleaseSpinLock(&Ring->m_Lock, OldIrql);
 
    return STATUS_SUCCESS;
}

HasQueuedWork()

BOOLEAN XhciControlRing::HasQueuedWork ( _In_ const XhciTransferRing *  Ring )

static

Definition at line 1295 of file ctrlxfer.cpp.

{
    return Ring->m_Type.Control.m_Current != NULL;
}

Referenced by XhciTransferRing::HasQueuedWork().

Initialize()

NTSTATUS XhciControlRing::Initialize ( _In_ XhciTransferRing *  Ring )

static

Definition at line 178 of file ctrlxfer.cpp.

{
    XhciControlRing* Self = XhciCtlOf(Ring);
    WDF_TIMER_CONFIG TimerConfig;
    WDF_DPC_CONFIG DpcConfig;
    WDF_OBJECT_ATTRIBUTES Attributes;
    NTSTATUS Status;
    ULONG Index;
 
    PAGED_CODE();
 
    RtlZeroMemory(Self, sizeof(*Self));
    Self->m_Ring = Ring;
    InitializeListHead(&Self->m_CanceledOnQueue);
 
    for (Index = 0; Index < TagCount; Index++)
    {
        Self->m_Tags[Index].Signature = XhciCtlTagSignature;
        Self->m_Tags[Index].Index = Index;
        Self->m_Tags[Index].Ring = Ring;
    }
 
    /* xHCI 6.4.1.1: immediate data needs a max packet size of at least 8 */
    Self->m_ImmediateAllowed = !Ring->m_Controller->HasErrata(XhciErrata::XferNoInlineData) &&
                               Ring->m_Endpoint->MaxPacketSize() >= XhciCtlImmediateBytes;
 
    WDF_TIMER_CONFIG_INIT(&TimerConfig, EvtTimeout);
    WDF_OBJECT_ATTRIBUTES_INIT(&Attributes);
    Attributes.ParentObject = Ring->m_Queue;
    Status = WdfTimerCreate(&TimerConfig, &Attributes, &Self->m_Timer);
    if (!NT_SUCCESS(Status))
    {
        DPRINT1("Control ring %p: no timeout timer, status 0x%lx\n", Ring, Status);
        return Status;
    }
 
    WDF_DPC_CONFIG_INIT(&DpcConfig, EvtCanceledOnQueueDpc);
    WDF_OBJECT_ATTRIBUTES_INIT(&Attributes);
    Attributes.ParentObject = Ring->m_Queue;
    Status = WdfDpcCreate(&DpcConfig, &Attributes, &Self->m_CanceledOnQueueDpc);
    if (!NT_SUCCESS(Status))
    {
        DPRINT1("Control ring %p: no DPC for requests canceled in the queue, status 0x%lx\n",
                Ring, Status);
        return Status;
    }
 
    return STATUS_SUCCESS;
}

Referenced by _IRQL_requires_().

InitTransfer()

VOID XhciControlRing::InitTransfer ( _Out_ XhciControlTransfer *  Transfer, _In_ WDFREQUEST  Request  )

private

Definition at line 306 of file ctrlxfer.cpp.

{
    WDF_REQUEST_PARAMETERS Parameters;
    PURB Urb;
 
    WDF_REQUEST_PARAMETERS_INIT(&Parameters);
    WdfRequestGetParameters(Request, &Parameters);
    Urb = static_cast<PURB>(Parameters.Parameters.Others.Arg1);
 
    RtlZeroMemory(Transfer, sizeof(*Transfer));
    Transfer->Request = Request;
    Transfer->Urb = Urb;
    Transfer->Ring = m_Ring;
    Transfer->Mechanism = XhciControlMechanism::NoData;
    Transfer->BytesTotal = Urb->UrbControlTransfer.TransferBufferLength;
    Transfer->DataIn = (Urb->UrbControlTransfer.TransferFlags & USBD_TRANSFER_DIRECTION_IN) != 0;
    Transfer->CancelState = XhciControlCancel::Idle;
    Transfer->TimeoutState = XhciControlTimeout::Idle;
    Transfer->Status = STATUS_PENDING;
    Transfer->CompletionCode = static_cast<ULONG>(XhciCompletionCode::Invalid);
    Transfer->FirstIndex = m_Ring->m_EnqueueIndex;
    Transfer->EndIndex = m_Ring->m_EnqueueIndex;
    Transfer->Initialized = TRUE;
 
    /* QUIRK: the length reads 0 until completion, as it did on the USB 2.0 stack */
    Urb->UrbControlTransfer.TransferBufferLength = 0;
}

LocateTrb()

BOOLEAN XhciControlRing::LocateTrb ( _In_ const XhciControlTransfer *  Transfer, _In_ const XHCI_TRB *  Event, _Out_ PULONG  Bytes, _Out_ PULONG  SkippedTds  ) const

private

Definition at line 1484 of file ctrlxfer.cpp.

{
    const XhciTransferRing* Ring = m_Ring;
    ULONG64 Pointer = XhciTrbPointer(Event);
    ULONG64 Base = Ring->SegmentBase();
    ULONG EventLength = Event->Dword[2] & XHCI_TRANSFER_EVENT_LENGTH_MASK;
    XhciCompletionCode Code = static_cast<XhciCompletionCode>(XhciTrbCompletionCode(Event));
    const XHCI_TRB* Trb;
    ULONG Index = Transfer->FirstIndex;
    ULONG Visited;
    ULONG Length;
    ULONG Type;
    BOOLEAN Match;
 
    *Bytes = 0;
    *SkippedTds = 0;
 
    if (Pointer == 0)
        return FALSE;
 
    /* Stopped on an empty ring: every TD of the request already ran */
    if (Pointer == Ring->EnqueuePointer())
    {
        *SkippedTds = Transfer->TdCount;
        return TRUE;
    }
 
    /* Walk the TD from its first slot to its end slot, wrapping at the Link slot */
    for (Visited = 0; Visited <= Ring->m_LastIndex; Visited++)
    {
        Trb = &Ring->m_Trbs[Index];
        Type = XhciTrbType(Trb);
        Length = Trb->Dword[2] & XHCI_TRB_LENGTH_MASK;
        Match = (Base + static_cast<ULONG64>(Index) * sizeof(*Trb) == Pointer);
 
        /* A link written in mid-segment jumps to the start, skipping the stale slots after it */
        if (Type == static_cast<ULONG>(XhciTrbType::Link) && !Match && Index != Transfer->EndIndex)
        {
            Index = 0;
            continue;
        }
 
        if (Type == static_cast<ULONG>(XhciTrbType::Normal) || Type == static_cast<ULONG>(XhciTrbType::DataStage))
        {
            if (!Match)
            {
                *Bytes += Length;
            }
            else if (Code == XhciCompletionCode::StoppedShortPacket)
            {
                *Bytes = EventLength;
            }
            else if (Code == XhciCompletionCode::Stopped ||
                     Code == XhciCompletionCode::DataBufferError ||
                     Code == XhciCompletionCode::BabbleDetected ||
                     Code == XhciCompletionCode::StallError ||
                     Code == XhciCompletionCode::UsbTransactionError ||
                     Code == XhciCompletionCode::SplitTransactionError)
            {
                /* The event carries the residual of this TRB */
                if (EventLength <= Length)
                    *Bytes += Length - EventLength;
            }
        }
        else if (Type == static_cast<ULONG>(XhciTrbType::EventData) && !Match)
        {
            /* That TD finished and reported its own byte count */
            *Bytes = 0;
            (*SkippedTds)++;
        }
 
        if (Match)
            return TRUE;
 
        if (Index == Transfer->EndIndex)
            break;
 
        Index = (Index >= Ring->m_LastIndex) ? 0 : Index + 1;
    }
 
    return FALSE;
}

Referenced by OnTrbEvent().

OnEventDataEvent()

BOOLEAN XhciControlRing::OnEventDataEvent ( _In_ EventTag *  Tag, _In_ const XHCI_TRB *  Event  )

private

Definition at line 1375 of file ctrlxfer.cpp.

{
    XhciTransferRing* Ring = m_Ring;
    XhciControlTransfer* Transfer;
    ULONG Code = XhciTrbCompletionCode(Event);
    ULONG Bytes = Event->Dword[2] & XHCI_TRANSFER_EVENT_LENGTH_MASK;
    ULONG EndpointId = (Event->Dword[3] & XHCI_TRB_ENDPOINT_MASK) >> XHCI_TRB_ENDPOINT_SHIFT;
    BOOLEAN StatusStage = (XhciTrbPointer(Event) & XhciCtlStatusStageFlag) != 0;
    BOOLEAN Halted = FALSE;
    BOOLEAN Stopped = FALSE;
    BOOLEAN Drained = FALSE;
    KIRQL OldIrql;
 
    KeAcquireSpinLock(&Ring->m_Lock, &OldIrql);
 
    Transfer = Tag->Transfer;
    if (Transfer == NULL || Transfer != m_Current)
    {
        KeReleaseSpinLock(&Ring->m_Lock, OldIrql);
 
        DPRINT1("Control ring %p: stale Event Data event, code %lu, dropped\n", Ring, Code);
        return TRUE;
    }
 
    if (EndpointId != Ring->m_Endpoint->Dci() || XhciTrbSlotId(Event) != Ring->m_Device->SlotId())
    {
        DPRINT1("Control ring %p: event for slot %lu endpoint %lu, ring is slot %lu endpoint %lu\n",
                Ring, XhciTrbSlotId(Event), EndpointId, Ring->m_Device->SlotId(), Ring->m_Endpoint->Dci());
    }
 
    if (Bytes > Transfer->BytesTotal)
    {
        DPRINT1("Control ring %p: %lu bytes reported for a %lu byte request\n",
                Ring, Bytes, Transfer->BytesTotal);
        Bytes = 0;
    }
 
    if (!StatusStage)
    {
        Transfer->BytesTransferred = Bytes;
        CopyIn(Transfer, Bytes);
    }
 
    if (XhciCtlIsHaltCode(Code))
    {
        /* The endpoint resets the pipe, then reclaim completes the request */
        Transfer->CompletionCode = Code;
        Halted = TRUE;
    }
    else if (XhciCtlIsStopCode(Code))
    {
        Transfer->EventsReceived++;
 
        if (StatusStage)
        {
            if (Transfer->BytesTransferred == Transfer->BytesTotal)
                Transfer->CompletionCode = static_cast<ULONG>(XhciCompletionCode::Success);
            else if (Code == static_cast<ULONG>(XhciCompletionCode::StoppedShortPacket))
                Transfer->CompletionCode = Code;
 
            m_Outstanding = max(static_cast<LONG>(Transfer->TdCount) - static_cast<LONG>(Transfer->EventsReceived), 0L);
        }
        else
        {
            /* QUIRK: a stop on the data stage expects no further events */
            m_Outstanding = 0;
        }
 
        Stopped = TRUE;
    }
    else
    {
        Transfer->EventsReceived++;
 
        if (StatusStage)
        {
            Transfer->CompletionCode = Code;
            if (Transfer->CancelState != XhciControlCancel::Done &&
                Transfer->TimeoutState != XhciControlTimeout::Expired)
            {
                CompleteHeld();
            }
        }
 
        if (m_Outstanding > 0)
        {
            m_Outstanding--;
            Drained = (m_Outstanding == 0 && m_AcknowledgeExpectedEvents);
        }
    }
 
    KeReleaseSpinLock(&Ring->m_Lock, OldIrql);
 
    if (Halted)
        Ring->m_Endpoint->OnHaltedCompletionCode(Code, TRUE);
 
    /* QUIRK: the drop stopped event flag is only consulted for events without Event Data */
    if (Stopped)
        Ring->m_Endpoint->OnStoppedEvent();
 
    if (Drained)
        Ring->m_Endpoint->OnExpectedEventsProcessed();
 
    return TRUE;
}

Referenced by OnTransferEvent().

OnTransferEvent()

BOOLEAN XhciControlRing::OnTransferEvent ( _In_opt_ XhciTransferRing *  Ring, _In_ XhciController *  Controller, _In_ const XHCI_TRB *  Event  )

static

The ring is always known; the NULL ring form is kept only for callers without one.

Definition at line 1338 of file ctrlxfer.cpp.

{
    EventTag* Tag;
 
    UNREFERENCED_PARAMETER(Controller);
 
    if (!(Event->Dword[3] & XHCI_TRANSFER_EVENT_ED))
    {
        /* Events without Event Data only come through the endpoint, with the ring known */
        if (Ring == NULL)
            return FALSE;
 
        return XhciCtlOf(Ring)->OnTrbEvent(Event);
    }
 
    Tag = TagFromEvent(Ring, Event);
    if (Tag == NULL)
    {
        if (Ring == NULL)
        {
            DPRINT1("Control Event Data value 0x%I64x names no live transfer, dropped\n",
                    XhciTrbPointer(Event));
            return FALSE;
        }
 
        DPRINT1("Control ring %p: Event Data value 0x%I64x is not one of its own, dropped\n",
                Ring, XhciTrbPointer(Event));
        return FALSE;
    }
 
    return XhciCtlOf(Tag->Ring)->OnEventDataEvent(Tag, Event);
}

Referenced by XhciTransferRing::OnTransferEvent().

OnTrbEvent()

BOOLEAN XhciControlRing::OnTrbEvent ( _In_ const XHCI_TRB *  Event )

private

Definition at line 1572 of file ctrlxfer.cpp.

{
    XhciTransferRing* Ring = m_Ring;
    XhciController* Controller = Ring->m_Controller;
    XhciControlTransfer* Transfer;
    ULONG Code = XhciTrbCompletionCode(Event);
    ULONG64 Pointer = XhciTrbPointer(Event);
    ULONG64 Base;
    ULONG Bytes;
    ULONG Skipped;
    LONG Outstanding;
    KIRQL OldIrql;
 
    if (XhciCtlIsStopCode(Code) && Ring->m_Endpoint->ShouldDropStoppedEvent())
        return TRUE;
 
    KeAcquireSpinLock(&Ring->m_Lock, &OldIrql);
 
    Transfer = m_Current;
    if (Transfer == NULL)
    {
        KeReleaseSpinLock(&Ring->m_Lock, OldIrql);
        DPRINT("Control ring %p: event code %lu with no request held\n", Ring, Code);
        return FALSE;
    }
 
    if (!LocateTrb(Transfer, Event, &Bytes, &Skipped))
    {
        KeReleaseSpinLock(&Ring->m_Lock, OldIrql);
 
        Base = Ring->SegmentBase();
        if (Pointer >= Base && Pointer < Base + Ring->m_SegmentSize)
        {
            if (Controller->HasErrata(XhciErrata::EvtDiscardRepeatedEd0))
            {
                DPRINT1("Control ring %p: TRB 0x%I64x is outside the request, likely a duplicate, dropped\n",
                        Ring, Pointer);
            }
            else
            {
                DPRINT1("Control ring %p: TRB 0x%I64x is outside the request, likely a duplicate, resetting\n",
                        Ring, Pointer);
                Controller->RaiseControllerFault(XhciRecovery::ResetHost, XhciCtlReasonDuplicateEvent);
            }
        }
        else
        {
            DPRINT1("Control ring %p: TRB 0x%I64x is not on the ring, resetting\n", Ring, Pointer);
            Controller->RaiseControllerFault(XhciRecovery::ResetHost, XhciCtlReasonBadTrbPointer);
        }
        return FALSE;
    }
 
    if (Bytes > Transfer->BytesTotal)
    {
        DPRINT1("Control ring %p: %lu bytes found for a %lu byte request\n", Ring, Bytes, Transfer->BytesTotal);
        Bytes = 0;
    }
 
    /* Zero keeps a count an earlier data stage event already reported */
    if (Bytes != 0)
    {
        Transfer->BytesTransferred = Bytes;
        CopyIn(Transfer, Bytes);
    }
 
    if (XhciCtlIsHaltCode(Code))
    {
        Transfer->CompletionCode = Code;
        KeReleaseSpinLock(&Ring->m_Lock, OldIrql);
 
        Ring->m_Endpoint->OnHaltedCompletionCode(Code, TRUE);
        return TRUE;
    }
 
    if (XhciCtlIsStopCode(Code))
    {
        if (Transfer->BytesTransferred == Transfer->BytesTotal)
            Transfer->CompletionCode = static_cast<ULONG>(XhciCompletionCode::Success);
        else if (Code == static_cast<ULONG>(XhciCompletionCode::StoppedShortPacket))
            Transfer->CompletionCode = Code;
 
        Transfer->EventsReceived++;
        Outstanding = static_cast<LONG>(Skipped + 1) - static_cast<LONG>(Transfer->EventsReceived);
        m_Outstanding = max(Outstanding, 0L);
        KeReleaseSpinLock(&Ring->m_Lock, OldIrql);
 
        Ring->m_Endpoint->OnStoppedEvent();
        return TRUE;
    }
 
    KeReleaseSpinLock(&Ring->m_Lock, OldIrql);
 
    DPRINT1("Control ring %p: unexpected code %lu for TRB 0x%I64x\n", Ring, Code, Pointer);
    return FALSE;
}

Referenced by OnTransferEvent().

PrepareTransfer()

NTSTATUS XhciControlRing::PrepareTransfer ( _Inout_ XhciControlTransfer *  Transfer )

private

Definition at line 425 of file ctrlxfer.cpp.

{
    XhciTransferRing* Ring = m_Ring;
    struct _URB_CONTROL_TRANSFER* Urb = &Transfer->Urb->UrbControlTransfer;
    PMDL UrbMdl = Urb->TransferBufferMDL;
    ULONG Bytes = Transfer->BytesTotal;
    XhciControlMechanism Mechanism;
    PMDL Mdl;
 
    /* A transfer mapped again after a stop starts over */
    ReleaseResources(Transfer);
 
    if (Bytes == 0)
        return STATUS_SUCCESS;
 
    if (m_ImmediateAllowed && Bytes <= XhciCtlImmediateBytes && !Transfer->DataIn)
    {
        Mechanism = XhciControlMechanism::Immediate;
    }
    else
    {
        Mechanism = XhciControlMechanism::Dma;
 
        if ((UrbMdl == NULL || UrbMdl->Next == NULL) && Bytes <= Ring->m_DoubleBufferSize)
        {
            Transfer->DoubleBuffer = Ring->BorrowBounceBuffer();
            if (Transfer->DoubleBuffer != NULL)
                Mechanism = XhciControlMechanism::DoubleBuffer;
        }
    }
 
    Transfer->Mechanism = Mechanism;
 
    if (Mechanism != XhciControlMechanism::Dma)
    {
        Transfer->SystemBuffer = Urb->TransferBuffer;
        if (Transfer->SystemBuffer == NULL && UrbMdl != NULL)
            Transfer->SystemBuffer = MmGetSystemAddressForMdlSafe(UrbMdl, NormalPagePriority);
 
        if (Transfer->SystemBuffer == NULL)
        {
            DPRINT1("Control request %p: no system address for its buffer\n", Transfer->Request);
            return STATUS_INSUFFICIENT_RESOURCES;
        }
    }
    else if (UrbMdl != NULL)
    {
        Transfer->Mdl = UrbMdl;
    }
    else
    {
        Mdl = IoAllocateMdl(Urb->TransferBuffer, Bytes, FALSE, FALSE, NULL);
        if (Mdl == NULL)
        {
            /* The reserve MDL serves one request at a time */
            if (!(Urb->TransferFlags & USB3_URB_RESERVED_RESOURCES) ||
                Ring->m_ReservedMdl == NULL ||
                InterlockedCompareExchange(&m_ReservedMdlBusy, 1, 0) != 0)
            {
                DPRINT1("Control request %p: no MDL for %lu bytes\n", Transfer->Request, Bytes);
                return STATUS_INSUFFICIENT_RESOURCES;
            }
 
            Mdl = Ring->m_ReservedMdl;
            MmInitializeMdl(Mdl, Urb->TransferBuffer, Bytes);
        }
 
        MmBuildMdlForNonPagedPool(Mdl);
        Transfer->Mdl = Mdl;
    }
 
    /* QUIRK: one TD per request, never split */
    if (Bytes > XhciCtlMaxTransfer)
    {
        DPRINT1("Control request %p: %lu bytes is more than 64 KB\n", Transfer->Request, Bytes);
        return STATUS_NOT_SUPPORTED;
    }
 
    if (Bytes > Ring->m_MaxStageSize)
    {
        DPRINT1("Control request %p: %lu bytes is more than the DMA stage size %lu\n",
                Transfer->Request, Bytes, Ring->m_MaxStageSize);
        return STATUS_INSUFFICIENT_RESOURCES;
    }
 
    /* QUIRK: the system buffer is NULL for Dma, so this never fires there */
    if (Bytes == Ring->m_MaxStageSize &&
        (reinterpret_cast<ULONG_PTR>(Transfer->SystemBuffer) & (PAGE_SIZE - 1)) != 0)
    {
        DPRINT1("Control request %p: stage sized buffer %p is not page aligned\n",
                Transfer->Request, Transfer->SystemBuffer);
        return STATUS_INSUFFICIENT_RESOURCES;
    }
 
    if (Mechanism == XhciControlMechanism::Dma && Transfer->Mdl->Next != NULL)
    {
        DPRINT1("Control request %p: chained MDLs are not supported\n", Transfer->Request);
        return STATUS_NOT_SUPPORTED;
    }
 
    return STATUS_SUCCESS;
}

Referenced by BuildControlTd().

QueueCompletion()

VOID XhciControlRing::QueueCompletion ( )

private

Definition at line 876 of file ctrlxfer.cpp.

{
    WdfDpcEnqueue(m_Ring->m_CompletionDpc);
}

Referenced by Abort(), AllowReclaimOnCancel(), RecoverTransfers(), and ResumeRingFill().

RecoverTransfers()

VOID XhciControlRing::RecoverTransfers ( _In_ XhciTransferRing *  Ring )

static

Definition at line 1258 of file ctrlxfer.cpp.

{
    XhciControlRing* Self = XhciCtlOf(Ring);
    BOOLEAN Held;
    KIRQL OldIrql;
 
    KeAcquireSpinLock(&Ring->m_Lock, &OldIrql);
    Held = (Self->m_Current != NULL);
    if (Held)
        Self->m_ReclaimAndAcknowledge = TRUE;
    KeReleaseSpinLock(&Ring->m_Lock, OldIrql);
 
    if (Held)
        Self->QueueCompletion();
    else
        Ring->m_Endpoint->OnTransfersReclaimed();
}

Referenced by XhciTransferRing::RecoverTransfers().

ReleaseResources()

VOID XhciControlRing::ReleaseResources ( _Inout_ XhciControlTransfer *  Transfer )

private

Definition at line 388 of file ctrlxfer.cpp.

{
    XhciTransferRing* Ring = m_Ring;
    PDMA_ADAPTER Adapter;
    KIRQL OldIrql;
 
    if (Transfer->DoubleBuffer != NULL)
    {
        Ring->ReturnBounceBuffer(Transfer->DoubleBuffer);
        Transfer->DoubleBuffer = NULL;
    }
 
    if (Transfer->SgList != NULL)
    {
        Adapter = Ring->m_Controller->m_Buffers.DmaAdapter();
        KeRaiseIrql(DISPATCH_LEVEL, &OldIrql);
        Adapter->DmaOperations->PutScatterGatherList(Adapter, Transfer->SgList, !Transfer->DataIn);
        KeLowerIrql(OldIrql);
        Transfer->SgList = NULL;
    }
 
    if (Transfer->Mdl != NULL && Transfer->Mdl == Ring->m_ReservedMdl)
    {
        InterlockedExchange(&m_ReservedMdlBusy, 0);
    }
    else if (Transfer->Mdl != NULL && Transfer->Mdl != Transfer->Urb->UrbControlTransfer.TransferBufferMDL)
    {
        IoFreeMdl(Transfer->Mdl);
    }
 
    Transfer->Mdl = NULL;
    Transfer->SystemBuffer = NULL;
    Transfer->Mechanism = XhciControlMechanism::NoData;
}

Referenced by PrepareTransfer().

ResumeRingFill()

VOID XhciControlRing::ResumeRingFill ( _In_ XhciTransferRing *  Ring )

static

Definition at line 1145 of file ctrlxfer.cpp.

{
    XhciControlRing* Self = XhciCtlOf(Ring);
    XhciControlTransfer* Transfer;
    BOOLEAN Queue = FALSE;
    BOOLEAN Map = FALSE;
    KIRQL OldIrql;
 
    KeAcquireSpinLock(&Ring->m_Lock, &OldIrql);
 
    Ring->m_DoorbellRung = FALSE;
    Self->m_ReclaimAndAcknowledge = FALSE;
    Self->m_AcknowledgeExpectedEvents = FALSE;
    Self->m_ReclaimOnCancelAllowed = FALSE;
    Self->m_Outstanding = 0;
 
    if (XhciCtlMapState(Ring) != XhciMapState::Halted)
        DPRINT1("Control ring %p: mapping started in map state %ld\n", Ring, Ring->m_MapState);
 
    Transfer = Self->m_Current;
    if (Transfer == NULL)
    {
        XhciCtlSetMapState(Ring, XhciMapState::PausedFill);
    }
    else if (Transfer->CancelState == XhciControlCancel::Done ||
             Transfer->TimeoutState == XhciControlTimeout::Expired)
    {
        /* Its cancel or timeout was reported before this start and the endpoint dropped that report */
        XhciCtlSetMapState(Ring, XhciMapState::PausedFill);
        Queue = TRUE;
    }
    else
    {
        XhciCtlSetMapState(Ring, XhciMapState::Filling);
        Map = TRUE;
    }
 
    KeReleaseSpinLock(&Ring->m_Lock, OldIrql);
 
    if (Queue)
        Self->QueueCompletion();
 
    /* QUIRK: a held request is mapped from scratch, which only happens if it never reached the hardware */
    if (Map)
        Self->BuildControlTd(Transfer);
}

Referenced by XhciTransferRing::ResumeRingFill().

SuspendRingFill()

VOID XhciControlRing::SuspendRingFill ( _In_ XhciTransferRing *  Ring )

static

Definition at line 1194 of file ctrlxfer.cpp.

{
    BOOLEAN Acknowledge = FALSE;
    KIRQL OldIrql;
 
    KeAcquireSpinLock(&Ring->m_Lock, &OldIrql);
 
    if (XhciCtlMapState(Ring) == XhciMapState::Filling)
    {
        /* The mapping context acknowledges once it is done */
        XhciCtlSetMapState(Ring, XhciMapState::Halting);
    }
    else
    {
        ASSERT(XhciCtlMapState(Ring) == XhciMapState::PausedFill);
        XhciCtlSetMapState(Ring, XhciMapState::Halted);
        Acknowledge = TRUE;
    }
 
    KeReleaseSpinLock(&Ring->m_Lock, OldIrql);
 
    if (Acknowledge)
        Ring->m_Endpoint->OnMappingStopped();
}

Referenced by XhciTransferRing::SuspendRingFill().

TagFromEvent()

XhciControlRing::EventTag * XhciControlRing::TagFromEvent ( _In_opt_ XhciTransferRing *  Ring, _In_ const XHCI_TRB *  Event  )

staticprivate

The tag an Event Data value names, or NULL when it is not a live tag (of Ring, when given).

Definition at line 1304 of file ctrlxfer.cpp.

{
    ULONG_PTR Address = static_cast<ULONG_PTR>(XhciTrbPointer(Event)) & ~XhciCtlValueFlagBits;
    EventTag* Tag = reinterpret_cast<EventTag*>(Address);
    XhciControlRing* Owner;
    ULONG Index;
 
    if (Address == 0)
        return NULL;
 
    /* With the ring known, only compare addresses; nothing is read through the value */
    if (Ring != NULL)
    {
        Owner = XhciCtlOf(Ring);
        for (Index = 0; Index < TagCount; Index++)
        {
            if (Tag == &Owner->m_Tags[Index])
                return Tag;
        }
        return NULL;
    }
 
    if (Tag->Signature != XhciCtlTagSignature || Tag->Index >= TagCount || Tag->Ring == NULL)
        return NULL;
 
    if (&XhciCtlOf(Tag->Ring)->m_Tags[Tag->Index] != Tag)
        return NULL;
 
    return Tag;
}

Referenced by OnTransferEvent().

WriteTd()

VOID XhciControlRing::WriteTd ( _Inout_ XhciControlTransfer *  Transfer )

private

Definition at line 624 of file ctrlxfer.cpp.

{
    XhciTransferRing* Ring = m_Ring;
    XhciController* Controller = Ring->m_Controller;
    struct _URB_CONTROL_TRANSFER* Urb = &Transfer->Urb->UrbControlTransfer;
    PSCATTER_GATHER_LIST SgList = Transfer->SgList;
    BOOLEAN Short512 = Controller->HasErrata(XhciErrata::CtlSplitDataAt512);
    XhciCtlStep Next = XhciCtlStep::Setup;
    XhciCtlStep Step;
    EventTag* Tag;
    ULONG_PTR TagValue;
    ULONG64 Address;
    XHCI_TRB Trb;
    ULONG Target;
    ULONG Packets;
    ULONG Mapped = 0;
    ULONG Element = 0;
    ULONG Offset = 0;
    ULONG Chunk;
    ULONG Limit;
    ULONG Remaining;
    ULONG TdCount = 0;
    ULONG FirstSlot;
    ULONG Trt;
    BOOLEAN First = TRUE;
    BOOLEAN DirOnNormal = FALSE;
    BOOLEAN LastData;
    BOOLEAN Acknowledge = FALSE;
    volatile UCHAR* CycleByte;
    KIRQL OldIrql;
 
    Packets = Ring->PacketCount(Transfer->BytesTotal);
    Target = (Ring->m_InterrupterTarget & XHCI_TRB_INTERRUPTER_MASK) << XHCI_TRB_INTERRUPTER_SHIFT;
 
    Tag = &m_Tags[m_NextTag % TagCount];
    m_NextTag++;
    ASSERT((reinterpret_cast<ULONG_PTR>(Tag) & XhciCtlValueFlagBits) == 0);
    TagValue = reinterpret_cast<ULONG_PTR>(Tag) | USB_ENDPOINT_TYPE_CONTROL;
 
    FirstSlot = Ring->m_EnqueueIndex;
 
    /* Keep the whole TD inside the segment when Link TRBs inside a TD are not allowed */
    if (Controller->HasErrata(XhciErrata::RingKeepTdInOneSegment) &&
        Ring->m_EnqueueIndex + XhciCtlWorstCaseTd > Ring->m_LastIndex)
    {
        XhciCtlWrap(Ring, TRUE, FALSE);
        First = FALSE;
    }
 
    while (Next != XhciCtlStep::Done)
    {
        Step = Next;
 
        if (Ring->m_EnqueueIndex >= Ring->m_LastIndex)
        {
            XhciCtlWrap(Ring,
                        First,
                        Step == XhciCtlStep::MoreData ||
                        Step == XhciCtlStep::DataEventData ||
                        Step == XhciCtlStep::StatusEventData);
            First = FALSE;
            continue;
        }
 
        RtlZeroMemory(&Trb, sizeof(Trb));
        Trb.Dword[2] = Target;
 
        switch (Step)
        {
            case XhciCtlStep::Setup:
            {
                if (Transfer->BytesTotal == 0)
                    Trt = XHCI_TRB_TRT_NO_DATA;
                else
                    Trt = Transfer->DataIn ? XHCI_TRB_TRT_IN : XHCI_TRB_TRT_OUT;
 
                RtlCopyMemory(&Trb.Dword[0], Urb->SetupPacket, XhciCtlSetupBytes);
                Trb.Dword[2] |= XhciCtlSetupBytes;
                Trb.Dword[3] = XhciCtlTrbType(XhciTrbType::SetupStage) | XHCI_TRB_IDT |
                               (Trt << XHCI_TRB_TRT_SHIFT);
                Next = Transfer->BytesTotal ? XhciCtlStep::FirstData : XhciCtlStep::Status;
                break;
            }
 
            case XhciCtlStep::FirstData:
            case XhciCtlStep::MoreData:
            {
                if (Step == XhciCtlStep::FirstData)
                {
                    Trb.Dword[3] = XhciCtlTrbType(XhciTrbType::DataStage);
                    if (Transfer->DataIn)
                        Trb.Dword[3] |= XHCI_TRB_DIR_IN;
                }
                else
                {
                    Trb.Dword[3] = XhciCtlTrbType(XhciTrbType::Normal);
                    if (DirOnNormal)
                        Trb.Dword[3] |= XHCI_TRB_DIR_IN;
                }
                Trb.Dword[3] |= XHCI_TRB_CHAIN;
 
                if (Transfer->Mechanism == XhciControlMechanism::Immediate)
                {
                    RtlCopyMemory(&Trb.Dword[0], Transfer->SystemBuffer, Transfer->BytesTotal);
                    Trb.Dword[3] |= XHCI_TRB_IDT;
                    Chunk = Transfer->BytesTotal;
                    LastData = TRUE;
                }
                else if (Transfer->Mechanism == XhciControlMechanism::DoubleBuffer)
                {
                    if (!Transfer->DataIn)
                    {
                        RtlCopyMemory(Transfer->DoubleBuffer->VirtualAddress,
                                      Transfer->SystemBuffer,
                                      Transfer->BytesTotal);
                    }
 
                    Address = Transfer->DoubleBuffer->LogicalAddress.QuadPart;
                    Trb.Dword[0] = static_cast<ULONG>(Address);
                    Trb.Dword[1] = static_cast<ULONG>(Address >> 32);
                    Chunk = Transfer->BytesTotal;
                    LastData = TRUE;
                }
                else
                {
                    Remaining = SgList->Elements[Element].Length - Offset;
                    Limit = XhciCtlMaxTrbLength;
 
                    /* These parts want a short first Data Stage TRB; IN marks what follows */
                    if (Short512 && Step == XhciCtlStep::FirstData && Remaining > XhciCtlShortDataStageTrb)
                    {
                        Limit = XhciCtlShortDataStageTrb;
                        DirOnNormal = Transfer->DataIn;
                    }
 
                    Address = SgList->Elements[Element].Address.QuadPart + Offset;
 
                    /* A TRB buffer may not cross a 64 KB boundary (xHCI 6.4.1.1) */
                    Limit = min(Limit, 0x10000 - static_cast<ULONG>(Address & 0xFFFF));
 
                    Chunk = min(Remaining, Limit);
                    Trb.Dword[0] = static_cast<ULONG>(Address);
                    Trb.Dword[1] = static_cast<ULONG>(Address >> 32);
 
                    Offset += Chunk;
                    if (Offset >= SgList->Elements[Element].Length)
                    {
                        Element++;
                        Offset = 0;
                    }
                    LastData = (Element >= SgList->NumberOfElements);
                }
 
                Mapped += Chunk;
                Trb.Dword[2] |= (Chunk & XHCI_TRB_LENGTH_MASK) |
                                (Ring->TdSize(Packets, Mapped, LastData) << XHCI_TRB_TD_SIZE_SHIFT);
 
                if (LastData)
                {
                    Trb.Dword[3] |= XHCI_TRB_ENT;
                    Next = XhciCtlStep::DataEventData;
                }
                else
                {
                    Next = XhciCtlStep::MoreData;
                }
                break;
            }
 
            case XhciCtlStep::DataEventData:
            case XhciCtlStep::StatusEventData:
            {
                Address = TagValue;
                if (Step == XhciCtlStep::StatusEventData)
                    Address |= XhciCtlStatusStageFlag;
 
                Trb.Dword[0] = static_cast<ULONG>(Address);
                Trb.Dword[1] = static_cast<ULONG>(Address >> 32);
                Trb.Dword[3] = XhciCtlTrbType(XhciTrbType::EventData) | XHCI_TRB_IOC;
                TdCount++;
                Next = (Step == XhciCtlStep::DataEventData) ? XhciCtlStep::Status : XhciCtlStep::Done;
                break;
            }
 
            case XhciCtlStep::Status:
            {
                /* QUIRK: the direction comes from the flags alone, even with no data stage */
                Trb.Dword[3] = XhciCtlTrbType(XhciTrbType::StatusStage) | XHCI_TRB_CHAIN | XHCI_TRB_ENT;
                if (!Transfer->DataIn)
                    Trb.Dword[3] |= XHCI_TRB_DIR_IN;
                Next = XhciCtlStep::StatusEventData;
                break;
            }
 
            default:
            {
                Next = XhciCtlStep::Done;
                continue;
            }
        }
 
        XhciCtlPut(Ring, &Trb, First);
        First = FALSE;
    }
 
    Transfer->FirstIndex = FirstSlot;
    Transfer->EndIndex = Ring->m_EnqueueIndex;
 
    /* Invalid TRB after the TD so the controller stops there */
    RtlZeroMemory(&Trb, sizeof(Trb));
    Trb.Dword[3] = (Ring->m_Cycle & XHCI_TRB_CYCLE) ^ XHCI_TRB_CYCLE;
    Ring->m_Trbs[Ring->m_EnqueueIndex] = Trb;
 
    KeAcquireSpinLock(&Ring->m_Lock, &OldIrql);
 
    if (Transfer->TagArmed)
        m_Tags[Transfer->TagIndex].Transfer = NULL;
    Tag->Transfer = Transfer;
    Transfer->TagIndex = Tag->Index;
    Transfer->TagArmed = TRUE;
 
    Transfer->TdCount = TdCount;
    Ring->m_DoorbellRung = TRUE;
 
    if (XhciCtlMapState(Ring) == XhciMapState::Filling)
    {
        XhciCtlSetMapState(Ring, XhciMapState::PausedFill);
    }
    else if (XhciCtlMapState(Ring) == XhciMapState::Halting)
    {
        XhciCtlSetMapState(Ring, XhciMapState::Halted);
        Acknowledge = TRUE;
    }
 
    KeReleaseSpinLock(&Ring->m_Lock, OldIrql);
 
    /* The byte write is the barrier that publishes the TD before its first TRB turns valid */
    KeMemoryBarrier();
    CycleByte = reinterpret_cast<volatile UCHAR*>(&Ring->m_Trbs[FirstSlot]) + 12;
    WRITE_REGISTER_UCHAR(const_cast<PUCHAR>(CycleByte), *CycleByte ^ XHCI_TRB_CYCLE);
    KeMemoryBarrier();
 
    Ring->RingDoorbell();
 
    if (Acknowledge)
        Ring->m_Endpoint->OnMappingStopped();
}

Referenced by BuildControlTd().

Member Data Documentation

Bytes

_In_ ULONG XhciControlRing::Bytes

private

Definition at line 120 of file ctrlxfer.h.

Referenced by LocateTrb(), OnEventDataEvent(), OnTrbEvent(), and PrepareTransfer().

EvtCanceledOnQueueDpc

VOID XhciControlRing::EvtCanceledOnQueueDpc

staticprivate

Definition at line 84 of file ctrlxfer.h.

Referenced by Initialize().

EvtCompletionDpc

VOID XhciControlRing::EvtCompletionDpc

static

Definition at line 37 of file ctrlxfer.h.

Referenced by _IRQL_requires_().

EvtDmaReady

VOID NTAPI XhciControlRing::EvtDmaReady

staticprivate

Definition at line 86 of file ctrlxfer.h.

Referenced by BuildControlTd().

EvtIoCanceledOnQueue

VOID XhciControlRing::EvtIoCanceledOnQueue

static

Definition at line 36 of file ctrlxfer.h.

Referenced by _IRQL_requires_().

EvtIoDefault

VOID XhciControlRing::EvtIoDefault

static

Definition at line 35 of file ctrlxfer.h.

Referenced by _IRQL_requires_().

EvtRequestCancel

VOID XhciControlRing::EvtRequestCancel

staticprivate

Definition at line 85 of file ctrlxfer.h.

EvtTimeout

VOID XhciControlRing::EvtTimeout

staticprivate

Definition at line 83 of file ctrlxfer.h.

Referenced by Initialize().

m_AcknowledgeExpectedEvents

BOOLEAN XhciControlRing::m_AcknowledgeExpectedEvents

private

Definition at line 151 of file ctrlxfer.h.

Referenced by ConsumePendingEvents(), OnEventDataEvent(), and ResumeRingFill().

m_CanceledDpcRunning

BOOLEAN XhciControlRing::m_CanceledDpcRunning

private

Definition at line 153 of file ctrlxfer.h.

m_CanceledOnQueue

LIST_ENTRY XhciControlRing::m_CanceledOnQueue

private

Definition at line 143 of file ctrlxfer.h.

Referenced by Initialize().

m_CanceledOnQueueDpc

WDFDPC XhciControlRing::m_CanceledOnQueueDpc

private

Definition at line 144 of file ctrlxfer.h.

Referenced by Initialize().

m_Current

XhciControlTransfer* XhciControlRing::m_Current

private

The one request on this ring, or NULL

Definition at line 142 of file ctrlxfer.h.

Referenced by AllowReclaimOnCancel(), BuildControlTd(), Cleanup(), Disable(), OnEventDataEvent(), OnTrbEvent(), RecoverTransfers(), and ResumeRingFill().

m_ImmediateAllowed

BOOLEAN XhciControlRing::m_ImmediateAllowed

private

Definition at line 149 of file ctrlxfer.h.

Referenced by EnableForwardProgress(), Initialize(), and PrepareTransfer().

m_NextTag

ULONG XhciControlRing::m_NextTag

private

Definition at line 146 of file ctrlxfer.h.

Referenced by WriteTd().

m_Outstanding

LONG XhciControlRing::m_Outstanding

private

Events still expected after a stop

Definition at line 147 of file ctrlxfer.h.

Referenced by ConsumePendingEvents(), OnEventDataEvent(), OnTrbEvent(), and ResumeRingFill().

m_ReclaimAndAcknowledge

BOOLEAN XhciControlRing::m_ReclaimAndAcknowledge

private

Definition at line 150 of file ctrlxfer.h.

Referenced by Abort(), RecoverTransfers(), and ResumeRingFill().

m_ReclaimOnCancelAllowed

BOOLEAN XhciControlRing::m_ReclaimOnCancelAllowed

private

Definition at line 152 of file ctrlxfer.h.

Referenced by Abort(), AllowReclaimOnCancel(), and ResumeRingFill().

m_ReservedMdlBusy

volatile LONG XhciControlRing::m_ReservedMdlBusy

private

Definition at line 148 of file ctrlxfer.h.

Referenced by PrepareTransfer(), and ReleaseResources().

m_Ring

XhciTransferRing* XhciControlRing::m_Ring

private

Definition at line 141 of file ctrlxfer.h.

Referenced by Abort(), BuildControlTd(), Initialize(), InitTransfer(), LocateTrb(), OnEventDataEvent(), OnTrbEvent(), PrepareTransfer(), QueueCompletion(), ReleaseResources(), and WriteTd().

m_Tags

EventTag XhciControlRing::m_Tags[TagCount]

private

Definition at line 140 of file ctrlxfer.h.

Referenced by Cleanup(), Initialize(), TagFromEvent(), and WriteTd().

m_Timer

WDFTIMER XhciControlRing::m_Timer

private

Definition at line 145 of file ctrlxfer.h.

Referenced by BuildControlTd(), and Initialize().

TagCount

const ULONG XhciControlRing::TagCount = 4

staticprivate

Definition at line 81 of file ctrlxfer.h.

Referenced by Cleanup(), Initialize(), TagFromEvent(), and WriteTd().

---

The documentation for this class was generated from the following files:

• drivers/usb/usbxhci/ctrlxfer.h
• drivers/usb/usbxhci/ctrlxfer.cpp