#include <xferring.h>

Collaboration diagram for XhciTransferRing:

[legend]

Public Member Functions
WDFQUEUE	Queue () const

	_IRQL_requires_max_ (DISPATCH_LEVEL) NTSTATUS Enable()

	_IRQL_requires_max_ (DISPATCH_LEVEL) VOID Disable(_In_ BOOLEAN FreeResources)

	_IRQL_requires_ (PASSIVE_LEVEL) NTSTATUS EnableForwardProgress(_In_ ULONG MaxTransferSize)

VOID	InitializeRing ()

ULONG64	SegmentBase () const

ULONG64	EnqueuePointer () const

ULONG64	DequeuePointerValue () const

VOID	ResumeRingFill ()

VOID	SuspendRingFill ()

VOID	OnPipeHalted ()

VOID	OnClientPipeReset ()

VOID	StoppedEventReceived ()

VOID	AllowReclaimOnCancel ()

VOID	ConsumePendingEvents ()

VOID	RecoverTransfers ()

BOOLEAN	DoorbellRungSinceFill () const

BOOLEAN	HasQueuedWork () const

BOOLEAN	OnTransferEvent (_In_ const XHCI_TRB *Event)

BOOLEAN	IsLikelyDuplicate (_In_ const XHCI_TRB *Event) const

NTSTATUS	EnsureFreeSegments (_In_ ULONG Count, _In_ BOOLEAN Async)

NTSTATUS	TakeSegments (_In_ ULONG Count, _Inout_ PLIST_ENTRY List)

VOID	ReturnSegments (_Inout_ PLIST_ENTRY List, _In_ BOOLEAN ToHead)

XhciDmaBuffer *	BorrowBounceBuffer ()

VOID	ReturnBounceBuffer (_In_ XhciDmaBuffer *Buffer)

XhciMapState	ChangeMapState (_In_ BOOLEAN FromAny, _In_ XhciMapState From, _In_ XhciMapState To)

VOID	BuildLinkTrb (_Out_ PXHCI_TRB Trb, _In_ BOOLEAN FirstOfTd, _In_ BOOLEAN InDataStage) const

ULONG	PacketCount (_In_ ULONG Bytes) const

ULONG	TdSize (_In_ ULONG PacketCount, _In_ ULONG BytesMappedSoFar, _In_ BOOLEAN LastTrbOfTd) const

VOID	UpdateInterrupterTarget ()

VOID	UpdateInterrupterTarget (_In_ ULONG ProcessorIndex)

VOID	RingDoorbell ()

VOID	Cleanup ()

	_IRQL_requires_ (PASSIVE_LEVEL) VOID GrowSegments()

BOOLEAN	IsControl () const

BOOLEAN	IsBulkOrInterrupt () const

BOOLEAN	IsIsoch () const

	_IRQL_requires_max_ (DISPATCH_LEVEL) VOID WaitForMappingIdle()

VOID	MakeLive ()

VOID	RetireLive ()

VOID	DereferenceLive ()

Static Public Member Functions
static NTSTATUS	Create (_In_ XhciEndpoint Endpoint, _In_ WDFOBJECT Parent, _In_ ULONG StreamId, _Out_ XhciTransferRing *Ring)

static XhciTransferRing *	FromQueue (_In_ WDFQUEUE Queue)

static BOOLEAN	OnUntargetedTransferEvent (_In_ XhciController Controller, _In_ const XHCI_TRB Event)

static ULONG	UrbProcessorIndex (_In_ PURB Urb)

static USBD_STATUS	UsbdStatusFromCompletionCode (_In_ ULONG CompletionCode)

static NTSTATUS	NtStatusFromUsbdStatus (_In_ USBD_STATUS UsbdStatus)

static XhciTransferRing *	ReferenceLiveRing (_In_ XhciController Controller, _In_ const XhciUsbDevice Device, _In_ ULONG Dci)

Public Attributes
XhciController *	m_Controller

XhciUsbDevice *	m_Device

XhciEndpoint *	m_Endpoint

ULONG	m_StreamId

ULONG	m_TransferType

WDFQUEUE	m_Queue

WDFDPC	m_CompletionDpc

KSPIN_LOCK	m_Lock

volatile LONG	m_MapState

ULONG	m_InterrupterTarget

BOOLEAN	m_DoorbellRung

PMDL	m_ReservedMdl

ULONG	m_SegmentSize

ULONG	m_DoubleBufferSize

ULONG	m_MaxStageSize

ULONG	m_MapRegisterCount

XhciDmaBuffer *	m_Segment

PXHCI_TRB	m_Trbs

ULONG	m_EnqueueIndex

ULONG	m_LastIndex

ULONG	m_Cycle

LIST_ENTRY	m_FreeSegments

LIST_ENTRY	m_DoubleBuffers

union {
XhciControlRing Control

XhciBulkRing Bulk

XhciIsochRing Isoch

}	m_Type

LIST_ENTRY	m_LiveLink

volatile LONG	m_EventUsers

BOOLEAN	m_Live

BOOLEAN	m_TypeReady

BOOLEAN	m_GrowthQueued

BOOLEAN	m_ForwardProgressQueue

ULONG	m_GrowthWanted

WDFWORKITEM	m_GrowthItem

Detailed Description

One transfer ring, stored as its WDFQUEUE context; the type specific state follows in m_Type.

Definition at line 28 of file xferring.h.

Member Function Documentation

◆ _IRQL_requires_() [1/2]

XhciTransferRing::_IRQL_requires_ ( PASSIVE_LEVEL )

◆ _IRQL_requires_() [2/2]

XhciTransferRing::_IRQL_requires_ ( PASSIVE_LEVEL )

◆ _IRQL_requires_max_() [1/3]

XhciTransferRing::_IRQL_requires_max_ ( DISPATCH_LEVEL )

◆ _IRQL_requires_max_() [2/3]

XhciTransferRing::_IRQL_requires_max_ ( DISPATCH_LEVEL )

◆ _IRQL_requires_max_() [3/3]

XhciTransferRing::_IRQL_requires_max_ ( DISPATCH_LEVEL )

◆ AllowReclaimOnCancel()

VOID XhciTransferRing::AllowReclaimOnCancel ( )

Definition at line 730 of file xferring.cpp.

{
    if (IsControl())
        XhciControlRing::AllowReclaimOnCancel(this);
    else if (IsIsoch())
        XhciIsochRing::AllowReclaimOnCancel(this);
    else
        XhciBulkRing::AllowReclaimOnCancel(this);
}

Referenced by XhciEndpointMachine::NotifyRingsReclaimOnCancel().

◆ BorrowBounceBuffer()

XhciDmaBuffer * XhciTransferRing::BorrowBounceBuffer ( )

0x200 byte buffer from the double buffer cache, or a new one; NULL on failure.

Definition at line 1014 of file xferring.cpp.

{
    XhciDmaBuffer* Buffer = NULL;
 
    /* Forward progress rings never double buffer */
    if (m_DoubleBufferSize == 0)
        return NULL;
 
    {
        XhciSpinLockGuard Guard(&m_Lock);
 
        if (!IsListEmpty(&m_DoubleBuffers))
        {
            Buffer = CONTAINING_RECORD(RemoveHeadList(&m_DoubleBuffers), XhciDmaBuffer, Link);
            XhciClearListEntry(&Buffer->Link);
        }
    }
 
    if (Buffer == NULL)
        Buffer = m_Controller->m_Buffers.Acquire(m_DoubleBufferSize);
 
    return Buffer;
}

Referenced by XhciControlRing::PrepareTransfer().

◆ BuildLinkTrb()

VOID XhciTransferRing::BuildLinkTrb	(	_Out_ PXHCI_TRB	Trb,
		_In_ BOOLEAN	FirstOfTd,
		_In_ BOOLEAN	InDataStage
	)		const

Zeroed link TRB with type, cycle, chain and interrupter target set; the caller adds pointer and TC.

Definition at line 1090 of file xferring.cpp.

{
    ULONG Control;
 
    RtlZeroMemory(Trb, sizeof(*Trb));
 
    /* A TD's first TRB stays invalid until the whole TD is written */
    Control = static_cast<ULONG>(XhciTrbType::Link) << XHCI_TRB_TYPE_SHIFT;
    Control |= FirstOfTd ? (m_Cycle ^ 1) & XHCI_TRB_CYCLE : m_Cycle & XHCI_TRB_CYCLE;
 
    if (m_Controller->HasErrata(XhciErrata::RingLinkTrbChain))
        Control |= XHCI_TRB_CHAIN;
    else if (!FirstOfTd && InDataStage)
        Control |= XHCI_TRB_CHAIN;
 
    Trb->Dword[2] = (m_InterrupterTarget & XHCI_TRB_INTERRUPTER_MASK) << XHCI_TRB_INTERRUPTER_SHIFT;
    Trb->Dword[3] = Control;
}

◆ ChangeMapState()

XhciMapState XhciTransferRing::ChangeMapState	(	_In_ BOOLEAN	FromAny,
		_In_ XhciMapState	From,
		_In_ XhciMapState	To
	)

Interlocked map state change, unconditional when FromAny; returns the old state.

Definition at line 1048 of file xferring.cpp.

{
    LONG Old;
    BOOLEAN Changed;
 
    if (FromAny)
    {
        Old = InterlockedExchange(&m_MapState, static_cast<LONG>(To));
        Changed = TRUE;
    }
    else
    {
        Old = InterlockedCompareExchange(&m_MapState, static_cast<LONG>(To), static_cast<LONG>(From));
        Changed = (Old == static_cast<LONG>(From));
    }
 
    if (Changed)
    {
        if (To == XhciMapState::Halted)
            m_Endpoint->OnMappingStopped();
    }
    else if (From == XhciMapState::Filling)
    {
        /* A stop arrived while this context was mapping; it acknowledges the stop itself */
        ASSERT(Old == static_cast<LONG>(XhciMapState::Halting));
 
        if (InterlockedCompareExchange(&m_MapState,
                                       static_cast<LONG>(XhciMapState::Halted),
                                       static_cast<LONG>(XhciMapState::Halting)) ==
            static_cast<LONG>(XhciMapState::Halting))
        {
            m_Endpoint->OnMappingStopped();
        }
    }
 
    return static_cast<XhciMapState>(Old);
}

◆ Cleanup()

VOID XhciTransferRing::Cleanup ( )

Definition at line 340 of file xferring.cpp.

{
    LIST_ENTRY Leftover;
 
    /* The queue failed before the ring was filled in */
    if (m_Controller == NULL)
        return;
 
    RetireLive();
 
    InitializeListHead(&Leftover);
    {
        XhciSpinLockGuard Guard(&m_Lock);
 
        XhciRingAppendList(&Leftover, &m_FreeSegments);
        XhciRingAppendList(&Leftover, &m_DoubleBuffers);
    }
 
    if (!IsListEmpty(&Leftover))
    {
        DPRINT1("Ring for DCI %lu stream %lu still pooled %lu buffers at cleanup\n",
                m_Endpoint->Dci(), m_StreamId, XhciRingCountList(&Leftover));
        m_Controller->m_Buffers.FreeList(&Leftover);
    }
 
    if (m_Segment != NULL)
    {
        m_Controller->m_Buffers.Free(m_Segment);
        m_Segment = NULL;
        m_Trbs = NULL;
    }
 
    if (m_ReservedMdl != NULL)
    {
        IoFreeMdl(m_ReservedMdl);
        m_ReservedMdl = NULL;
    }
 
    if (!m_TypeReady)
        return;
 
    m_TypeReady = FALSE;
    if (IsControl())
        XhciControlRing::Cleanup(this);
    else if (IsIsoch())
        XhciIsochRing::Cleanup(this);
    else
        XhciBulkRing::Cleanup(this);
}

◆ ConsumePendingEvents()

VOID XhciTransferRing::ConsumePendingEvents ( )

Definition at line 741 of file xferring.cpp.

{
    if (IsControl())
        XhciControlRing::ConsumePendingEvents(this);
    else if (IsIsoch())
        XhciIsochRing::ConsumePendingEvents(this);
    else
        XhciBulkRing::ConsumePendingEvents(this);
}

Referenced by XhciEndpointMachine::NotifyRingsDrainEvents().

◆ Create()

static NTSTATUS XhciTransferRing::Create	(	_In_ XhciEndpoint *	Endpoint,
		_In_ WDFOBJECT	Parent,
		_In_ ULONG	StreamId,
		_Out_ XhciTransferRing **	Ring
	)

static

PASSIVE_LEVEL

Referenced by XhciEndpoint::Initialize().

◆ DequeuePointerValue()

ULONG64 XhciTransferRing::DequeuePointerValue ( ) const

Enqueue pointer with DCS, plus SCT 1 for a stream ring (context and Set TR Dequeue value).

Definition at line 670 of file xferring.cpp.

{
    ULONG64 Value = EnqueuePointer();
 
    if (m_Cycle & 1)
        Value |= XHCI_DEQUEUE_CYCLE;
 
    if (m_StreamId != 0)
        Value |= (ULONG64)XHCI_SCT_PRIMARY_RING << XHCI_DEQUEUE_SCT_SHIFT;
 
    return Value;
}

Referenced by XhciEndpoint::CloseStreams(), XhciEndpoint::ControlResetDone(), XhciEndpoint::DequeueAfterReset(), XhciEndpoint::DequeueAfterStop(), XhciEndpoint::EndpointDequeuePointer(), XhciEndpoint::OpenStreams(), XhciEndpointMachine::ReconfigureForReset(), and XhciEndpoint::ResetDropDone().

◆ DereferenceLive()

VOID XhciTransferRing::DereferenceLive ( )

Definition at line 453 of file xferring.cpp.

{
    InterlockedDecrement(&m_EventUsers);
}

Referenced by OnUntargetedTransferEvent().

◆ DoorbellRungSinceFill()

BOOLEAN XhciTransferRing::DoorbellRungSinceFill ( ) const

Definition at line 763 of file xferring.cpp.

{
    if (IsControl())
        return XhciControlRing::DoorbellRungSinceFill(this);
 
    if (IsIsoch())
        return XhciIsochRing::DoorbellRungSinceFill(this);
 
    return XhciBulkRing::DoorbellRungSinceFill(this);
}

Referenced by XhciEndpointMachine::DoorbellRungSinceFill().

◆ EnqueuePointer()

ULONG64 XhciTransferRing::EnqueuePointer ( ) const

Definition at line 664 of file xferring.cpp.

{
    return SegmentBase() + (ULONG64)m_EnqueueIndex * sizeof(XHCI_TRB);
}

Referenced by DequeuePointerValue(), XhciControlRing::LocateTrb(), and XhciEndpoint::StreamRingEmpty().

◆ EnsureFreeSegments()

NTSTATUS XhciTransferRing::EnsureFreeSegments	(	_In_ ULONG	Count,
		_In_ BOOLEAN	Async
	)

Makes sure the free pool holds Count segments; Async may return STATUS_PENDING.

Definition at line 853 of file xferring.cpp.

{
    LIST_ENTRY Taken;
    ULONG Have;
    ULONG Missing;
    BOOLEAN Enqueue = FALSE;
    NTSTATUS Status;
 
    {
        XhciSpinLockGuard Guard(&m_Lock);
 
        Have = XhciRingCountList(&m_FreeSegments);
    }
 
    if (Have >= Count)
        return STATUS_SUCCESS;
 
    Missing = Count - Have;
    InitializeListHead(&Taken);
 
    Status = m_Controller->m_Buffers.AcquireList(m_SegmentSize, Missing, &Taken);
    if (NT_SUCCESS(Status))
    {
        XhciSpinLockGuard Guard(&m_Lock);
 
        XhciRingAppendList(&m_FreeSegments, &Taken);
        return STATUS_SUCCESS;
    }
 
    /* At PASSIVE_LEVEL the pool already grew synchronously, so the failure is final */
    if (Async && m_GrowthItem != NULL && KeGetCurrentIrql() > PASSIVE_LEVEL)
    {
        {
            XhciSpinLockGuard Guard(&m_Lock);
 
            if (m_GrowthWanted < Missing)
                m_GrowthWanted = Missing;
 
            if (!m_GrowthQueued)
            {
                m_GrowthQueued = TRUE;
                Enqueue = TRUE;
            }
        }
 
        if (Enqueue)
            WdfWorkItemEnqueue(m_GrowthItem);
 
        return STATUS_PENDING;
    }
 
    DPRINT1("Ring for DCI %lu stream %lu could not get %lu segments, 0x%08lx\n",
            m_Endpoint->Dci(), m_StreamId, Missing, Status);
    return Status;
}

Referenced by _IRQL_requires_(), and TakeSegments().

◆ FromQueue()

XhciTransferRing * XhciTransferRing::FromQueue ( _In_ WDFQUEUE Queue )

static

Definition at line 308 of file xferring.cpp.

{
    return XhciRingContext(Queue);
}

Referenced by XhciRingGrowthWorkItem().

◆ HasQueuedWork()

BOOLEAN XhciTransferRing::HasQueuedWork ( ) const

Definition at line 775 of file xferring.cpp.

{
    if (IsControl())
        return XhciControlRing::HasQueuedWork(this);
 
    if (IsIsoch())
        return XhciIsochRing::HasQueuedWork(this);
 
    return XhciBulkRing::HasQueuedWork(this);
}

◆ InitializeRing()

VOID XhciTransferRing::InitializeRing ( )

Zeroes the current segment and resets enqueue and cycle state.

Definition at line 648 of file xferring.cpp.

{
    RtlZeroMemory(m_Segment->VirtualAddress, m_Segment->Size);
    m_Trbs = (PXHCI_TRB)m_Segment->VirtualAddress;
    m_EnqueueIndex = 0;
    m_LastIndex = m_SegmentSize / sizeof(XHCI_TRB) - 1;
    m_Cycle = 1;
}

Referenced by _IRQL_requires_max_(), XhciEndpoint::ControlResetDone(), XhciEndpoint::DequeueAfterStop(), XhciEndpointMachine::ReconfigureForReset(), and XhciEndpoint::ResetDropDone().

◆ IsBulkOrInterrupt()

BOOLEAN XhciTransferRing::IsBulkOrInterrupt ( ) const

Definition at line 327 of file xferring.cpp.

{
    return m_TransferType == USB_ENDPOINT_TYPE_BULK ||
           m_TransferType == USB_ENDPOINT_TYPE_INTERRUPT;
}

Referenced by IsLikelyDuplicate(), OnPipeHalted(), OnTransferEvent(), OnUntargetedTransferEvent(), and StoppedEventReceived().

◆ IsControl()

BOOLEAN XhciTransferRing::IsControl ( ) const

Definition at line 321 of file xferring.cpp.

{
    return m_TransferType == USB_ENDPOINT_TYPE_CONTROL;
}

Referenced by _IRQL_requires_(), _IRQL_requires_max_(), AllowReclaimOnCancel(), Cleanup(), ConsumePendingEvents(), DoorbellRungSinceFill(), HasQueuedWork(), OnTransferEvent(), OnUntargetedTransferEvent(), RecoverTransfers(), ResumeRingFill(), and SuspendRingFill().

◆ IsIsoch()

BOOLEAN XhciTransferRing::IsIsoch ( ) const

Definition at line 334 of file xferring.cpp.

{
    return m_TransferType == USB_ENDPOINT_TYPE_ISOCHRONOUS;
}

Referenced by _IRQL_requires_(), _IRQL_requires_max_(), AllowReclaimOnCancel(), Cleanup(), ConsumePendingEvents(), DoorbellRungSinceFill(), HasQueuedWork(), OnClientPipeReset(), OnTransferEvent(), RecoverTransfers(), ResumeRingFill(), and SuspendRingFill().

◆ IsLikelyDuplicate()

BOOLEAN XhciTransferRing::IsLikelyDuplicate ( _In_ const XHCI_TRB * Event ) const

Stream duplicate heuristic for ED = 0 events nobody claimed (bulk only).

Definition at line 843 of file xferring.cpp.

{
    if (IsBulkOrInterrupt())
        return XhciBulkRing::IsLikelyDuplicate(this, Event);
 
    return FALSE;
}

Referenced by XhciEndpoint::OnTransferEvent().

◆ MakeLive()

VOID XhciTransferRing::MakeLive ( )

Definition at line 391 of file xferring.cpp.

{
    XhciSpinLockGuard Guard(&XhciLiveRingLock);
 
    InsertTailList(&XhciLiveRings, &m_LiveLink);
    m_Live = TRUE;
}

Referenced by _IRQL_requires_().

◆ NtStatusFromUsbdStatus()

NTSTATUS XhciTransferRing::NtStatusFromUsbdStatus ( _In_ USBD_STATUS UsbdStatus )

static

Definition at line 1237 of file xferring.cpp.

{
    switch (UsbdStatus)
    {
        case USBD_STATUS_SUCCESS:
        case USBD_STATUS_PORT_OPERATION_PENDING:
            return STATUS_SUCCESS;
 
        case USBD_STATUS_INSUFFICIENT_RESOURCES:
            return STATUS_INSUFFICIENT_RESOURCES;
 
        case USBD_STATUS_INVALID_PARAMETER:
        case USBD_STATUS_INVALID_URB_FUNCTION:
        case USBD_STATUS_INVALID_PIPE_HANDLE:
        case USBD_STATUS_BAD_START_FRAME:
            return STATUS_INVALID_PARAMETER;
 
        case USBD_STATUS_NOT_SUPPORTED:
            return STATUS_NOT_SUPPORTED;
 
        case USBD_STATUS_DEVICE_GONE:
            return STATUS_NO_SUCH_DEVICE;
 
        case USBD_STATUS_CANCELED:
            return STATUS_CANCELLED;
 
        default:
            return STATUS_UNSUCCESSFUL;
    }
}

Referenced by XhciIsochRing::Finalize().

◆ OnClientPipeReset()

VOID XhciTransferRing::OnClientPipeReset ( )

Definition at line 714 of file xferring.cpp.

{
    /* Only an isoch ring cares: its next ASAP URB starts a new stream */
    if (IsIsoch())
        XhciIsochRing::OnClientPipeReset(this);
}

Referenced by XhciEndpointMachine::NotifyRingsClientReset().

◆ OnPipeHalted()

VOID XhciTransferRing::OnPipeHalted ( )

Definition at line 706 of file xferring.cpp.

{
    /* A halted control ring waits for its reset and an isoch ring never reports a halt */
    if (IsBulkOrInterrupt())
        XhciBulkRing::OnPipeHalted(this);
}

Referenced by XhciEndpointMachine::NotifyRingsHalted().

◆ OnTransferEvent()

BOOLEAN XhciTransferRing::OnTransferEvent ( _In_ const XHCI_TRB * Event )

TRUE when this ring recognized the event (the endpoint gave it to this ring).

Definition at line 787 of file xferring.cpp.

{
    if (IsControl())
        return XhciControlRing::OnTransferEvent(this, m_Controller, Event);
 
    if (IsBulkOrInterrupt())
        return XhciBulkRing::OnTransferEvent(this, m_Controller, Event);
 
    if (IsIsoch())
        return XhciIsochRing::OnTransferEvent(this, Event);
 
    return FALSE;
}

Referenced by XhciEndpoint::OnTransferEvent(), and OnUntargetedTransferEvent().

◆ OnUntargetedTransferEvent()

BOOLEAN XhciTransferRing::OnUntargetedTransferEvent	(	_In_ XhciController *	Controller,
		_In_ const XHCI_TRB *	Event
	)

static

Interrupter fast path: finds the ring by slot and DCI; the type module then checks the Event Data value.

Definition at line 803 of file xferring.cpp.

{
    ULONG EndpointType = Event->Dword[0] & XHCI_EVENT_DATA_TYPE_MASK;
    ULONG SlotId = XhciTrbSlotId(Event);
    ULONG Dci = (Event->Dword[3] & XHCI_TRB_ENDPOINT_MASK) >> XHCI_TRB_ENDPOINT_SHIFT;
    XhciTransferRing* Ring;
    BOOLEAN SameType;
 
    if (EndpointType == USB_ENDPOINT_TYPE_ISOCHRONOUS)
        return FALSE;
 
    /* Find the ring by slot and endpoint id so the type handler can check the pointer before using it */
    Ring = ReferenceLiveRing(Controller,
                             static_cast<const XhciUsbDevice*>(Controller->m_Slots.LookupSlot(SlotId)),
                             Dci);
    if (Ring == NULL)
        return FALSE;
 
    if (EndpointType == USB_ENDPOINT_TYPE_CONTROL)
        SameType = Ring->IsControl();
    else
        SameType = Ring->IsBulkOrInterrupt();
 
    if (SameType)
    {
        Ring->OnTransferEvent(Event);
    }
    else
    {
        DPRINT1("Stale Event Data 0x%I64x for slot %lu DCI %lu type %lu, dropped\n",
                XhciTrbPointer(Event), SlotId, Dci, Ring->m_TransferType);
    }
 
    Ring->DereferenceLive();
    return TRUE;
}

Referenced by XhciRouteTransferEvent().

◆ PacketCount()

ULONG XhciTransferRing::PacketCount ( _In_ ULONG Bytes ) const

Definition at line 1113 of file xferring.cpp.

{
    ULONG MaxPacket = m_Endpoint->MaxPacketSize();
 
    /* xHCI 0.96 parts take no TD Size, so the count is not needed */
    if (m_Controller->m_Registers.VersionMajor() < 1 || MaxPacket == 0)
        return 0;
 
    return (Bytes + MaxPacket - 1) / MaxPacket;
}

Referenced by TdSize(), and XhciControlRing::WriteTd().

◆ Queue()

WDFQUEUE XhciTransferRing::Queue ( ) const

Definition at line 315 of file xferring.cpp.

{
    return m_Queue;
}

Referenced by _IRQL_requires_(), XhciEndpoint::Initialize(), XhciEndpoint::OpenStreams(), XhciEndpointMachine::PurgeStreamQueues(), XhciEndpointMachine::RestartStreamQueues(), XhciEndpoint::StreamsDisableDone(), and XhciEndpoint::StreamsEnableDone().

◆ RecoverTransfers()

VOID XhciTransferRing::RecoverTransfers ( )

Definition at line 752 of file xferring.cpp.

{
    if (IsControl())
        XhciControlRing::RecoverTransfers(this);
    else if (IsIsoch())
        XhciIsochRing::RecoverTransfers(this);
    else
        XhciBulkRing::RecoverTransfers(this);
}

Referenced by XhciEndpointMachine::NotifyRingsReclaim().

◆ ReferenceLiveRing()

XhciTransferRing * XhciTransferRing::ReferenceLiveRing	(	_In_ XhciController *	Controller,
		_In_ const XhciUsbDevice *	Device,
		_In_ ULONG	Dci
	)

static

Finds the non stream ring of Device and Dci and takes an event reference on it.

Definition at line 419 of file xferring.cpp.

{
    XhciSpinLockGuard Guard(&XhciLiveRingLock);
    XhciTransferRing* Found = NULL;
    PLIST_ENTRY Entry;
 
    for (Entry = XhciLiveRings.Flink; Entry != &XhciLiveRings; Entry = Entry->Flink)
    {
        XhciTransferRing* Ring = CONTAINING_RECORD(Entry, XhciTransferRing, m_LiveLink);
 
        if (Ring->m_Controller != Controller ||
            Ring->m_Device != Device ||
            Ring->m_Endpoint->Dci() != Dci)
        {
            continue;
        }
 
        /* The event does not say which stream it belongs to; the endpoint sorts that out */
        if (Ring->m_StreamId != 0)
            return NULL;
 
        Found = Ring;
    }
 
    if (Found != NULL)
        InterlockedIncrement(&Found->m_EventUsers);
 
    return Found;
}

Referenced by OnUntargetedTransferEvent().

◆ ResumeRingFill()

VOID XhciTransferRing::ResumeRingFill ( )

Definition at line 684 of file xferring.cpp.

{
    if (IsControl())
        XhciControlRing::ResumeRingFill(this);
    else if (IsIsoch())
        XhciIsochRing::ResumeRingFill(this);
    else
        XhciBulkRing::ResumeRingFill(this);
}

Referenced by XhciEndpointMachine::ResumeRingFill().

◆ RetireLive()

VOID XhciTransferRing::RetireLive ( )

Definition at line 400 of file xferring.cpp.

{
    {
        XhciSpinLockGuard Guard(&XhciLiveRingLock);
 
        if (!m_Live)
            return;
 
        RemoveEntryList(&m_LiveLink);
        XhciClearListEntry(&m_LiveLink);
        m_Live = FALSE;
    }
 
    /* Nobody can find the ring anymore; let an event handler already inside it finish */
    while (InterlockedCompareExchange(&m_EventUsers, 0, 0) != 0)
        YieldProcessor();
}

Referenced by Cleanup().

◆ ReturnBounceBuffer()

VOID XhciTransferRing::ReturnBounceBuffer ( _In_ XhciDmaBuffer * Buffer )

Definition at line 1039 of file xferring.cpp.

{
    XhciSpinLockGuard Guard(&m_Lock);
 
    InsertTailList(&m_DoubleBuffers, &Buffer->Link);
}

Referenced by XhciControlRing::ReleaseResources().

◆ ReturnSegments()

VOID XhciTransferRing::ReturnSegments	(	_Inout_ PLIST_ENTRY	List,
		_In_ BOOLEAN	ToHead
	)

Returns segments to the pool; the caller holds m_Lock.

Definition at line 989 of file xferring.cpp.

{
    if (IsListEmpty(List))
        return;
 
    if (m_Controller->HasErrata(XhciErrata::RingClearNewSegments))
    {
        PLIST_ENTRY Entry;
 
        for (Entry = List->Flink; Entry != List; Entry = Entry->Flink)
        {
            XhciDmaBuffer* Segment = CONTAINING_RECORD(Entry, XhciDmaBuffer, Link);
            RtlZeroMemory(Segment->VirtualAddress, Segment->Size);
        }
    }
 
    if (ToHead)
        XhciRingPrependList(&m_FreeSegments, List);
    else
        XhciRingAppendList(&m_FreeSegments, List);
}

◆ RingDoorbell()

VOID XhciTransferRing::RingDoorbell ( )

Rings this ring's doorbell (DCI and stream id) and records that it was rung.

Definition at line 1175 of file xferring.cpp.

{
    /* The cycle bit flip must be visible before the controller is told to look */
    KeMemoryBarrier();
    m_Device->RingDoorbell(m_Endpoint->Dci(), m_StreamId);
}

Referenced by XhciControlRing::WriteTd().

◆ SegmentBase()

ULONG64 XhciTransferRing::SegmentBase ( ) const

Definition at line 658 of file xferring.cpp.

{
    return (ULONG64)m_Segment->LogicalAddress.QuadPart;
}

Referenced by EnqueuePointer(), XhciControlRing::LocateTrb(), XhciControlRing::OnTrbEvent(), and XhciEndpoint::StreamRingEmpty().

◆ StoppedEventReceived()

VOID XhciTransferRing::StoppedEventReceived ( )

Definition at line 722 of file xferring.cpp.

{
    /* An isoch ring already tracks the stop through its own events */
    if (IsBulkOrInterrupt())
        XhciBulkRing::StoppedEventReceived(this);
}

Referenced by XhciEndpointMachine::NotifyRingsStoppedEvent().

◆ SuspendRingFill()

VOID XhciTransferRing::SuspendRingFill ( )

Definition at line 695 of file xferring.cpp.

{
    if (IsControl())
        XhciControlRing::SuspendRingFill(this);
    else if (IsIsoch())
        XhciIsochRing::SuspendRingFill(this);
    else
        XhciBulkRing::SuspendRingFill(this);
}

Referenced by XhciEndpointMachine::SuspendRingFill().

◆ TakeSegments()

NTSTATUS XhciTransferRing::TakeSegments	(	_In_ ULONG	Count,
		_Inout_ PLIST_ENTRY	List
	)

Moves Count pool segments to the head of List. Same results as EnsureFreeSegments.

Definition at line 950 of file xferring.cpp.

{
    LIST_ENTRY Moved;
    ULONG Index;
    NTSTATUS Status;
 
    Status = EnsureFreeSegments(Count, TRUE);
    if (Status != STATUS_SUCCESS)
        return Status;
 
    InitializeListHead(&Moved);
    {
        XhciSpinLockGuard Guard(&m_Lock);
 
        if (XhciRingCountList(&m_FreeSegments) < Count)
        {
            Status = STATUS_INSUFFICIENT_RESOURCES;
        }
        else
        {
            for (Index = 0; Index < Count; Index++)
                InsertTailList(&Moved, RemoveHeadList(&m_FreeSegments));
 
            XhciRingPrependList(List, &Moved);
        }
    }
 
    if (!NT_SUCCESS(Status))
    {
        DPRINT1("Ring for DCI %lu stream %lu lost its reserved segments\n",
                m_Endpoint->Dci(), m_StreamId);
    }
 
    return Status;
}

◆ TdSize()

ULONG XhciTransferRing::TdSize	(	_In_ ULONG	PacketCount,
		_In_ ULONG	BytesMappedSoFar,
		_In_ BOOLEAN	LastTrbOfTd
	)		const

TD Size field (xHCI 4.11.2.4) for a TRB, given bytes mapped so far including it.

Definition at line 1126 of file xferring.cpp.

{
    ULONG MaxPacket = m_Endpoint->MaxPacketSize();
    ULONG Sent;
    ULONG Remaining;
 
    if (LastTrbOfTd || MaxPacket == 0 || m_Controller->m_Registers.VersionMajor() < 1)
        return 0;
 
    Sent = BytesMappedSoFar / MaxPacket;
    if (Sent >= PacketCount)
        return 0;
 
    Remaining = PacketCount - Sent;
    return min(Remaining, 31);
}

Referenced by XhciControlRing::WriteTd().

◆ UpdateInterrupterTarget() [1/2]

VOID XhciTransferRing::UpdateInterrupterTarget ( )

Recomputes the 10 bit interrupter target for work issued on this processor.

Definition at line 1147 of file xferring.cpp.

{
    m_InterrupterTarget = m_Controller->m_Interrupters.TargetForCurrentProcessor() &
                          XHCI_TRB_INTERRUPTER_MASK;
}

Referenced by XhciControlRing::BuildControlTd(), and UpdateInterrupterTarget().

◆ UpdateInterrupterTarget() [2/2]

VOID XhciTransferRing::UpdateInterrupterTarget ( _In_ ULONG ProcessorIndex )

Same, for the processor index UCX stamped into the URB; XHCI_URB_NO_PROCESSOR means the current one.

Definition at line 1154 of file xferring.cpp.

{
    if (ProcessorIndex == XHCI_URB_NO_PROCESSOR)
    {
        UpdateInterrupterTarget();
        return;
    }
 
    m_InterrupterTarget = m_Controller->m_Interrupters.TargetForProcessor(ProcessorIndex) &
                          XHCI_TRB_INTERRUPTER_MASK;
}

◆ UrbProcessorIndex()

ULONG XhciTransferRing::UrbProcessorIndex ( _In_ PURB Urb )

static

Processor index UCX stamped into the first ULONG of the URB's HCD area.

Definition at line 1168 of file xferring.cpp.

{
    return *(PULONG)&Urb->UrbControlTransfer.hca;
}

Referenced by XhciControlRing::BuildControlTd(), XhciBulkRing::NextTdSlot(), and XhciIsochRing::NextTdSlot().

◆ UsbdStatusFromCompletionCode()

USBD_STATUS XhciTransferRing::UsbdStatusFromCompletionCode ( _In_ ULONG CompletionCode )

static

Code 0 (no hardware result) gives the "not set" value 0xFFFFFFFF for the caller to replace.

Definition at line 1184 of file xferring.cpp.

{
    switch (static_cast<XhciCompletionCode>(CompletionCode))
    {
        case XhciCompletionCode::Invalid:
            return XhciRingUsbdNotSet;
 
        /* QUIRK: short packets always succeed, USBD_SHORT_TRANSFER_OK is not consulted */
        case XhciCompletionCode::Success:
        case XhciCompletionCode::ShortPacket:
        case XhciCompletionCode::Stopped:
        case XhciCompletionCode::StoppedShortPacket:
            return USBD_STATUS_SUCCESS;
 
        case XhciCompletionCode::DataBufferError:
            return USBD_STATUS_DATA_BUFFER_ERROR;
 
        case XhciCompletionCode::BabbleDetected:
        case XhciCompletionCode::IsochBufferOverrun:
            return USBD_STATUS_BABBLE_DETECTED;
 
        case XhciCompletionCode::StallError:
            return USBD_STATUS_STALL_PID;
 
        case XhciCompletionCode::UsbTransactionError:
        case XhciCompletionCode::SplitTransactionError:
            return USBD_STATUS_XACT_ERROR;
 
        case XhciCompletionCode::MissedService:
            return USBD_STATUS_ISO_TD_ERROR;
 
        case XhciCompletionCode::NoPingResponse:
            return USBD_STATUS_NO_PING_RESPONSE;
 
        case XhciCompletionCode::StoppedLengthInvalid:
            return USBD_STATUS_ISO_NOT_ACCESSED_BY_HW;
 
        case XhciCompletionCode::InvalidStreamType:
            return USBD_STATUS_INVALID_STREAM_TYPE;
 
        case XhciCompletionCode::InvalidStreamId:
            return USBD_STATUS_INVALID_STREAM_ID;
 
        default:
            /* QUIRK: any other code is reported as a transaction error */
            DPRINT1("Unexpected completion code %lu reported as a transaction error\n",
                    CompletionCode);
            return USBD_STATUS_XACT_ERROR;
    }
}

Referenced by XhciIsochRing::CompleteTd().

Member Data Documentation

◆ Bulk

XhciBulkRing XhciTransferRing::Bulk

Definition at line 199 of file xferring.h.

◆ Control

XhciControlRing XhciTransferRing::Control

Definition at line 198 of file xferring.h.

Referenced by BuildLinkTrb().

◆ Isoch

XhciIsochRing XhciTransferRing::Isoch

Definition at line 200 of file xferring.h.

◆ m_CompletionDpc

WDFDPC XhciTransferRing::m_CompletionDpc

Definition at line 176 of file xferring.h.

Referenced by _IRQL_requires_(), and XhciControlRing::QueueCompletion().

◆ m_Controller

XhciController* XhciTransferRing::m_Controller

Definition at line 170 of file xferring.h.

Referenced by _IRQL_requires_(), BorrowBounceBuffer(), XhciControlRing::BuildControlTd(), BuildLinkTrb(), Cleanup(), EnsureFreeSegments(), OnTransferEvent(), XhciControlRing::OnTrbEvent(), PacketCount(), ReferenceLiveRing(), XhciControlRing::ReleaseResources(), ReturnSegments(), TdSize(), UpdateInterrupterTarget(), and XhciControlRing::WriteTd().

◆ m_Cycle

ULONG XhciTransferRing::m_Cycle

Producer cycle state

Definition at line 192 of file xferring.h.

Referenced by _IRQL_requires_(), BuildLinkTrb(), DequeuePointerValue(), InitializeRing(), and XhciControlRing::WriteTd().

◆ m_Device

XhciUsbDevice* XhciTransferRing::m_Device

Definition at line 171 of file xferring.h.

Referenced by _IRQL_requires_(), XhciControlRing::OnEventDataEvent(), ReferenceLiveRing(), and RingDoorbell().

◆ m_DoorbellRung

BOOLEAN XhciTransferRing::m_DoorbellRung

Since the last ResumeRingFill

Definition at line 180 of file xferring.h.

Referenced by XhciControlRing::WriteTd().

◆ m_DoubleBuffers

LIST_ENTRY XhciTransferRing::m_DoubleBuffers

Definition at line 194 of file xferring.h.

Referenced by _IRQL_requires_(), BorrowBounceBuffer(), Cleanup(), and ReturnBounceBuffer().

◆ m_DoubleBufferSize

ULONG XhciTransferRing::m_DoubleBufferSize

Definition at line 184 of file xferring.h.

Referenced by _IRQL_requires_(), and BorrowBounceBuffer().

◆ m_Endpoint

XhciEndpoint* XhciTransferRing::m_Endpoint

Definition at line 172 of file xferring.h.

Referenced by _IRQL_requires_(), _IRQL_requires_max_(), XhciControlRing::Abort(), XhciControlRing::BuildControlTd(), ChangeMapState(), Cleanup(), EnsureFreeSegments(), XhciControlRing::OnEventDataEvent(), XhciControlRing::OnTrbEvent(), PacketCount(), ReferenceLiveRing(), RingDoorbell(), TakeSegments(), TdSize(), and XhciControlRing::WriteTd().

◆ m_EnqueueIndex

ULONG XhciTransferRing::m_EnqueueIndex

Definition at line 190 of file xferring.h.

Referenced by EnqueuePointer(), InitializeRing(), XhciControlRing::InitTransfer(), and XhciControlRing::WriteTd().

◆ m_EventUsers

volatile LONG XhciTransferRing::m_EventUsers

Definition at line 227 of file xferring.h.

Referenced by DereferenceLive(), and RetireLive().

◆ m_ForwardProgressQueue

BOOLEAN XhciTransferRing::m_ForwardProgressQueue

Definition at line 231 of file xferring.h.

◆ m_FreeSegments

LIST_ENTRY XhciTransferRing::m_FreeSegments

Definition at line 193 of file xferring.h.

Referenced by _IRQL_requires_(), Cleanup(), EnsureFreeSegments(), ReturnSegments(), and TakeSegments().

◆ m_GrowthItem

WDFWORKITEM XhciTransferRing::m_GrowthItem

Definition at line 233 of file xferring.h.

Referenced by _IRQL_requires_(), and EnsureFreeSegments().

◆ m_GrowthQueued

BOOLEAN XhciTransferRing::m_GrowthQueued

Definition at line 230 of file xferring.h.

Referenced by EnsureFreeSegments().

◆ m_GrowthWanted

ULONG XhciTransferRing::m_GrowthWanted

Definition at line 232 of file xferring.h.

Referenced by EnsureFreeSegments().

◆ m_InterrupterTarget

ULONG XhciTransferRing::m_InterrupterTarget

Definition at line 179 of file xferring.h.

Referenced by BuildLinkTrb(), UpdateInterrupterTarget(), and XhciControlRing::WriteTd().

◆ m_LastIndex

ULONG XhciTransferRing::m_LastIndex

Segment size / 16 - 1

Definition at line 191 of file xferring.h.

Referenced by _IRQL_requires_(), InitializeRing(), XhciControlRing::LocateTrb(), and XhciControlRing::WriteTd().

◆ m_Live

BOOLEAN XhciTransferRing::m_Live

Definition at line 228 of file xferring.h.

Referenced by MakeLive(), and RetireLive().

◆ m_LiveLink

LIST_ENTRY XhciTransferRing::m_LiveLink

Definition at line 226 of file xferring.h.

Referenced by _IRQL_requires_(), MakeLive(), ReferenceLiveRing(), and RetireLive().

◆ m_Lock

KSPIN_LOCK XhciTransferRing::m_Lock

Guards the segment fields, pools and type state

Definition at line 177 of file xferring.h.

Referenced by _IRQL_requires_(), XhciControlRing::Abort(), BorrowBounceBuffer(), XhciControlRing::BuildControlTd(), Cleanup(), EnsureFreeSegments(), XhciControlRing::OnEventDataEvent(), XhciControlRing::OnTrbEvent(), ReturnBounceBuffer(), TakeSegments(), and XhciControlRing::WriteTd().

◆ m_MapRegisterCount

ULONG XhciTransferRing::m_MapRegisterCount

Definition at line 186 of file xferring.h.

Referenced by _IRQL_requires_().

◆ m_MapState

volatile LONG XhciTransferRing::m_MapState

XhciMapState

Definition at line 178 of file xferring.h.

Referenced by _IRQL_requires_(), _IRQL_requires_max_(), and ChangeMapState().

◆ m_MaxStageSize

ULONG XhciTransferRing::m_MaxStageSize

Definition at line 185 of file xferring.h.

Referenced by _IRQL_requires_(), and XhciControlRing::PrepareTransfer().

◆ m_Queue

WDFQUEUE XhciTransferRing::m_Queue

Definition at line 175 of file xferring.h.

Referenced by _IRQL_requires_(), and Queue().

◆ m_ReservedMdl

PMDL XhciTransferRing::m_ReservedMdl

Definition at line 181 of file xferring.h.

Referenced by Cleanup(), XhciControlRing::PrepareTransfer(), and XhciControlRing::ReleaseResources().

◆ m_Segment

XhciDmaBuffer* XhciTransferRing::m_Segment

Current segment

Definition at line 188 of file xferring.h.

Referenced by _IRQL_requires_(), _IRQL_requires_max_(), Cleanup(), InitializeRing(), and SegmentBase().

◆ m_SegmentSize

ULONG XhciTransferRing::m_SegmentSize

Definition at line 183 of file xferring.h.

Referenced by _IRQL_requires_(), EnsureFreeSegments(), and InitializeRing().

◆ m_StreamId

ULONG XhciTransferRing::m_StreamId

Definition at line 173 of file xferring.h.

Referenced by _IRQL_requires_(), _IRQL_requires_max_(), Cleanup(), DequeuePointerValue(), EnsureFreeSegments(), ReferenceLiveRing(), RingDoorbell(), and TakeSegments().

◆ m_TransferType

ULONG XhciTransferRing::m_TransferType

bmAttributes & 3

Definition at line 174 of file xferring.h.

Referenced by _IRQL_requires_(), IsBulkOrInterrupt(), IsControl(), IsIsoch(), and OnUntargetedTransferEvent().

◆ m_Trbs

PXHCI_TRB XhciTransferRing::m_Trbs

Definition at line 189 of file xferring.h.

Referenced by _IRQL_requires_(), Cleanup(), InitializeRing(), XhciControlRing::LocateTrb(), and XhciControlRing::WriteTd().

◆

union { ... } XhciTransferRing::m_Type

◆ m_TypeReady

BOOLEAN XhciTransferRing::m_TypeReady

Definition at line 229 of file xferring.h.

Referenced by _IRQL_requires_(), and Cleanup().

The documentation for this class was generated from the following files:

drivers/usb/usbxhci/xferring.h
drivers/usb/usbxhci/xferring.cpp

Public Member Functions

Static Public Member Functions

Public Attributes

Detailed Description

Member Function Documentation

◆ _IRQL_requires_() [1/2]

◆ _IRQL_requires_() [2/2]

◆ _IRQL_requires_max_() [1/3]

◆ _IRQL_requires_max_() [2/3]

◆ _IRQL_requires_max_() [3/3]

◆ AllowReclaimOnCancel()

◆ BorrowBounceBuffer()

◆ BuildLinkTrb()

◆ ChangeMapState()

◆ Cleanup()

◆ ConsumePendingEvents()

◆ Create()

◆ DequeuePointerValue()

◆ DereferenceLive()

◆ DoorbellRungSinceFill()

◆ EnqueuePointer()

◆ EnsureFreeSegments()

◆ FromQueue()

◆ HasQueuedWork()

◆ InitializeRing()

◆ IsBulkOrInterrupt()

◆ IsControl()

◆ IsIsoch()

◆ IsLikelyDuplicate()

◆ MakeLive()

◆ NtStatusFromUsbdStatus()

◆ OnClientPipeReset()

◆ OnPipeHalted()

◆ OnTransferEvent()

◆ OnUntargetedTransferEvent()

◆ PacketCount()

◆ Queue()

◆ RecoverTransfers()

◆ ReferenceLiveRing()

◆ ResumeRingFill()

◆ RetireLive()

◆ ReturnBounceBuffer()

◆ ReturnSegments()

◆ RingDoorbell()

◆ SegmentBase()

◆ StoppedEventReceived()

◆ SuspendRingFill()

◆ TakeSegments()

◆ TdSize()

◆ UpdateInterrupterTarget() [1/2]

◆ UpdateInterrupterTarget() [2/2]

◆ UrbProcessorIndex()

◆ UsbdStatusFromCompletionCode()

Member Data Documentation

◆ Bulk

◆ Control

◆ Isoch

◆ m_CompletionDpc

◆ m_Controller

◆ m_Cycle

◆ m_Device

◆ m_DoorbellRung

◆ m_DoubleBuffers

◆ m_DoubleBufferSize

◆ m_Endpoint

◆ m_EnqueueIndex

◆ m_EventUsers

◆ m_ForwardProgressQueue

◆ m_FreeSegments

◆ m_GrowthItem

◆ m_GrowthQueued

◆ m_GrowthWanted

◆ m_InterrupterTarget

◆ m_LastIndex

◆ m_Live

◆ m_LiveLink

◆ m_Lock

◆ m_MapRegisterCount

◆ m_MapState

◆ m_MaxStageSize