XhciIsochRing Class Reference

#include <isoxfer.h>

Collaboration diagram for XhciIsochRing:

Static Public Member Functions
static NTSTATUS	Initialize (_In_ XhciTransferRing *Ring)
static VOID	Enable (_In_ XhciTransferRing *Ring)
static VOID	Disable (_In_ XhciTransferRing *Ring)
static VOID	ReleaseParkedMapping (_In_ XhciTransferRing *Ring)
static VOID	Cleanup (_In_ XhciTransferRing *Ring)
static VOID	ResumeRingFill (_In_ XhciTransferRing *Ring)
static VOID	SuspendRingFill (_In_ XhciTransferRing *Ring)
static VOID	OnClientPipeReset (_In_ XhciTransferRing *Ring)
static VOID	AllowReclaimOnCancel (_In_ XhciTransferRing *Ring)
static VOID	ConsumePendingEvents (_In_ XhciTransferRing *Ring)
static VOID	RecoverTransfers (_In_ XhciTransferRing *Ring)
static BOOLEAN	DoorbellRungSinceFill (_In_ const XhciTransferRing *Ring)
static BOOLEAN	HasQueuedWork (_In_ const XhciTransferRing *Ring)
static BOOLEAN	OnTransferEvent (_In_ XhciTransferRing *Ring, _In_ const XHCI_TRB *Event)
static VOID	SegmentsArrived (_In_ XhciTransferRing *Ring, _In_ NTSTATUS Status)

Static Public Attributes
static EVT_WDF_IO_QUEUE_STATE	ReadyNotification
static EVT_WDF_IO_QUEUE_IO_CANCELED_ON_QUEUE	EvtIoCanceledOnQueue
static EVT_WDF_DPC	EvtCompletionDpc

Private Types
enum class	Outcome : ULONG {   Ready , Completed , WaitSegments , WaitDma ,   Gap , Late , Failed }

Static Private Member Functions
static VOID	BuildTdsFromQueue (_In_ XhciTransferRing *Ring)
static VOID	FillLoop (_In_ XhciTransferRing *Ring)
static VOID	AwaitRingEmpty (_In_ XhciTransferRing *Ring)
static BOOLEAN	LeaveRingEmptyWait (_In_ XhciTransferRing *Ring)
static VOID	KickAfterStage (_In_ XhciTransferRing *Ring, _In_ ULONG InFlight)
static BOOLEAN	ParkOnSegments (_In_ XhciTransferRing *Ring)
static XhciIsochStage *	NextTdSlot (_In_ XhciTransferRing *Ring)
static BOOLEAN	TakeRequest (_In_ XhciTransferRing *Ring, _In_ WDFREQUEST Request, _Out_ XhciIsochTransfer **Taken)
static BOOLEAN	PacketsValid (_In_ const XhciIsochTransfer *Transfer)
static NTSTATUS	ScheduleStartFrame (_In_ XhciTransferRing *Ring, _Inout_ XhciIsochTransfer *Transfer)
static VOID	FlagMissedPackets (_In_ XhciTransferRing *Ring, _Inout_ XhciIsochTransfer *Transfer)
static VOID	SkipUnmapped (_Inout_ XhciIsochTransfer *Transfer)
static XhciIsochStage *	AcquireStage (_In_ XhciTransferRing *Ring, _Inout_ XhciIsochTransfer *Transfer)
static VOID	FreeStage (_In_ XhciTransferRing *Ring, _Inout_ XhciIsochStage *Stage)
static VOID	PutScatterGather (_In_ XhciTransferRing *Ring, _In_opt_ PSCATTER_GATHER_LIST SgList)
static BOOLEAN	TransferDone (_In_ const XhciIsochTransfer *Transfer)
static Outcome	PrimeStage (_In_ XhciTransferRing *Ring, _Inout_ XhciIsochStage *Stage)
static NTSTATUS	AcquireStageMdl (_In_ XhciTransferRing *Ring, _Inout_ XhciIsochStage *Stage)
static BOOLEAN	SizeStage (_In_ XhciTransferRing *Ring, _Inout_ XhciIsochStage *Stage)
static ULONG	EstimateSegments (_In_ XhciTransferRing *Ring, _In_ const XhciIsochStage *Stage)
static Outcome	FailStage (_In_ XhciTransferRing *Ring, _Inout_ XhciIsochStage *Stage)
static Outcome	EncodeStage (_In_ XhciTransferRing *Ring, _Inout_ XhciIsochStage *Stage)
static VOID	WriteStage (_In_ XhciTransferRing *Ring, _Inout_ XhciIsochStage *Stage)
static ULONG	LayoutStage (_In_ XhciTransferRing *Ring, _Inout_ XhciIsochStage *Stage, _In_ BOOLEAN Commit)
static VOID	InsertLink (_In_ XhciTransferRing *Ring, _Inout_ XhciIsochStage *Stage, _In_ BOOLEAN FirstOfTd)
static VOID	CompleteTd (_In_ XhciTransferRing *Ring, _In_ XhciIsochTransfer *Transfer, _In_ XhciIsochStage *Stage, _In_ ULONG Sequence, _In_ ULONG Code, _In_ ULONG Bytes, _In_ BOOLEAN CloseStage)
static VOID	FinishExpiredIsoch (_In_ XhciTransferRing *Ring, _In_ const XhciIsochTransfer *Newest, _In_ ULONG Sequence)
static VOID	RetireExpiredStages (_In_ XhciTransferRing *Ring, _In_ XhciIsochTransfer *Transfer, _In_ XhciIsochStage *Stage, _In_ ULONG Sequence)
static ULONG	ReleaseAllStages (_In_ XhciTransferRing *Ring, _Inout_ XhciIsochTransfer *Transfer)
static VOID	Finalize (_In_ XhciTransferRing *Ring, _Inout_ XhciIsochTransfer *Transfer, _In_ USBD_STATUS PacketStatus, _In_ USBD_STATUS HeaderStatus)
static VOID	CompletePlain (_In_ XhciTransferRing *Ring, _Inout_ XhciIsochTransfer *Transfer, _In_ USBD_STATUS PacketStatus, _In_ USBD_STATUS HeaderStatus, _In_ BOOLEAN Async)
static VOID	CompleteAfterUncancel (_In_ XhciTransferRing *Ring, _Inout_ XhciIsochTransfer *Transfer, _In_ USBD_STATUS PacketStatus, _In_ USBD_STATUS HeaderStatus, _In_ BOOLEAN FinishCanceled, _In_ BOOLEAN Async)
static VOID	CompleteBatch (_In_ XhciTransferRing *Ring, _Inout_ PLIST_ENTRY Batch)
static BOOLEAN	OwnsRequests (_In_ const XhciTransferRing *Ring)
static VOID	Ed1Event (_In_ XhciTransferRing *Ring, _In_ const XHCI_TRB *Event)
static VOID	Ed0Event (_In_ XhciTransferRing *Ring, _In_ const XHCI_TRB *Event)
static BOOLEAN	FindTrb (_In_ XhciTransferRing *Ring, _In_ const XHCI_TRB *Event, _Outptr_result_maybenull_ XhciIsochStage **Found, _Out_ PULONG Bytes)
static VOID	OnRingStarved (_In_ XhciTransferRing *Ring)
static BOOLEAN	FinishLostStages (_In_ XhciTransferRing *Ring)

Private Attributes
BOOLEAN	m_FreshStream
BOOLEAN	m_DoorbellAfterFirstTd
BOOLEAN	m_DpcRunning
ULONG	m_NextStartFrame
ULONG	m_LastScheduledFrame
ULONG	m_PacketsPerFrame
ULONG	m_PendingStages
ULONG	m_TrbsPerTd
ULONG	m_BufferRetries
ULONG	m_NextSequence
volatile LONG	m_Flags
volatile LONG	m_FillWanted
volatile LONG	m_RingEmptySeen
volatile LONG	m_FillAgain
volatile LONG	m_LateStage
volatile LONG	m_SegmentWait
XhciIsochStage *	m_CurrentStage
LIST_ENTRY	m_Pending
LIST_ENTRY	m_WaitingForCancel
LIST_ENTRY	m_Completion
WDFTIMER	m_Watchdog
ULONG	m_Transfers
ULONG	m_FailedTransfers
ULONG64	m_Bytes
ULONG	m_Tds
ULONG	m_TdFailures
ULONG	m_MissedServiceErrors
ULONG	m_RingEmptyEvents

Static Private Attributes
static EVT_WDF_REQUEST_CANCEL	EvtRequestCancel
static EVT_WDF_TIMER	EvtWatchdog
static DRIVER_LIST_CONTROL	DmaCallback

Detailed Description

Isochronous ring state and the entry points the transfer ring module calls.

Definition at line 79 of file isoxfer.h.

Member Enumeration Documentation

Outcome

enum class XhciIsochRing::Outcome : ULONG

strongprivate

Enumerator
Ready
Completed
WaitSegments
WaitDma
Gap
Late
Failed

Definition at line 142 of file isoxfer.h.

    {
        Ready,
        Completed,
        WaitSegments,
        WaitDma,
        Gap,
        Late,
        Failed
    };

Member Function Documentation

AcquireStage()

static XhciIsochStage * XhciIsochRing::AcquireStage ( _In_ XhciTransferRing *  Ring, _Inout_ XhciIsochTransfer *  Transfer  )

staticprivate

Referenced by NextTdSlot().

AcquireStageMdl()

NTSTATUS XhciIsochRing::AcquireStageMdl ( _In_ XhciTransferRing *  Ring, _Inout_ XhciIsochStage *  Stage  )

staticprivate

Definition at line 1465 of file isoxfer.cpp.

{
    XhciIsochTransfer* Transfer = Stage->Transfer;
    PMDL Mdl = Transfer->Mdl;
    ULONG Offset = Transfer->BytesQueued + Transfer->BytesDropped;
    PUCHAR Start;
    ULONG Length;
    PMDL Partial;
 
    while (Mdl != NULL && Offset >= MmGetMdlByteCount(Mdl))
    {
        Offset -= MmGetMdlByteCount(Mdl);
        Mdl = Mdl->Next;
    }
 
    if (Mdl == NULL)
    {
        DPRINT1("Isoch DCI %lu: MDL chain shorter than the transfer\n", Ring->m_Endpoint->Dci());
        return STATUS_INSUFFICIENT_RESOURCES;
    }
 
    if (Offset == 0)
    {
        Stage->Mdl = Mdl;
        return STATUS_SUCCESS;
    }
 
    Start = static_cast<PUCHAR>(MmGetMdlVirtualAddress(Mdl)) + Offset;
    Length = MmGetMdlByteCount(Mdl) - Offset;
 
    Partial = IoAllocateMdl(Start, Length, FALSE, FALSE, NULL);
    if (Partial == NULL)
    {
        DPRINT1("Isoch DCI %lu: partial MDL allocation failed\n", Ring->m_Endpoint->Dci());
        return STATUS_INSUFFICIENT_RESOURCES;
    }
 
    IoBuildPartialMdl(Mdl, Partial, Start, Length);
    Partial->Next = Mdl->Next;
    Stage->Mdl = Partial;
    Stage->FreeMdl = TRUE;
    return STATUS_SUCCESS;
}

Referenced by PrimeStage().

AllowReclaimOnCancel()

VOID XhciIsochRing::AllowReclaimOnCancel ( _In_ XhciTransferRing *  Ring )

static

Definition at line 702 of file isoxfer.cpp.

{
    XhciIsochRing* Self = &Ring->m_Type.Isoch;
    BOOLEAN Queue = FALSE;
    KIRQL OldIrql;
 
    KeAcquireSpinLock(&Ring->m_Lock, &OldIrql);
    if (Self->m_Flags & XhciIsochCanceled)
    {
        XhciIsochMoveList(&Self->m_Completion, &Self->m_Pending);
        Queue = TRUE;
    }
    else
    {
        InterlockedOr(&Self->m_Flags, XhciIsochOkToReclaim);
    }
    KeReleaseSpinLock(&Ring->m_Lock, OldIrql);
 
    if (Queue)
        WdfDpcEnqueue(Ring->m_CompletionDpc);
}

Referenced by XhciTransferRing::AllowReclaimOnCancel().

AwaitRingEmpty()

VOID XhciIsochRing::AwaitRingEmpty ( _In_ XhciTransferRing *  Ring )

staticprivate

Definition at line 908 of file isoxfer.cpp.

{
    if (Ring->ChangeMapState(FALSE, XhciMapState::Filling, XhciMapState::AwaitingStarve) == XhciMapState::Filling)
        WdfTimerStart(Ring->m_Type.Isoch.m_Watchdog, WDF_REL_TIMEOUT_IN_MS(XhciIsochWatchdogMs));
}

Referenced by FillLoop().

BuildTdsFromQueue()

VOID XhciIsochRing::BuildTdsFromQueue ( _In_ XhciTransferRing *  Ring )

staticprivate

Definition at line 824 of file isoxfer.cpp.

{
    XhciIsochRing* Self = &Ring->m_Type.Isoch;
    LONG State;
    KIRQL OldIrql;
 
    KeRaiseIrql(DISPATCH_LEVEL, &OldIrql);
 
    for (;;)
    {
        FillLoop(Ring);
 
        State = InterlockedCompareExchange(&Ring->m_MapState, 0, 0);
 
        if (State == static_cast<LONG>(XhciMapState::PausedFill) && Self->m_FillWanted != 0)
        {
            if (Ring->ChangeMapState(FALSE, XhciMapState::PausedFill, XhciMapState::Filling) == XhciMapState::PausedFill)
                continue;
        }
        else if (State == static_cast<LONG>(XhciMapState::AwaitingStarve) && Self->m_RingEmptySeen != 0)
        {
            if (LeaveRingEmptyWait(Ring))
                continue;
        }
 
        break;
    }
 
    KeLowerIrql(OldIrql);
}

Referenced by KickAfterStage(), OnRingStarved(), ResumeRingFill(), and SegmentsArrived().

Cleanup()

VOID XhciIsochRing::Cleanup ( _In_ XhciTransferRing *  Ring )

static

Definition at line 518 of file isoxfer.cpp.

{
    BOOLEAN Owned;
    KIRQL OldIrql;
 
    KeAcquireSpinLock(&Ring->m_Lock, &OldIrql);
    Owned = OwnsRequests(Ring);
    KeReleaseSpinLock(&Ring->m_Lock, OldIrql);
 
    if (Owned)
    {
        DPRINT1("Isoch DCI %lu cleaned up with requests still owned\n", Ring->m_Endpoint->Dci());
        NT_ASSERT(FALSE);
    }
}

Referenced by XhciTransferRing::Cleanup().

CompleteAfterUncancel()

static VOID XhciIsochRing::CompleteAfterUncancel ( _In_ XhciTransferRing *  Ring, _Inout_ XhciIsochTransfer *  Transfer, _In_ USBD_STATUS  PacketStatus, _In_ USBD_STATUS  HeaderStatus, _In_ BOOLEAN  FinishCanceled, _In_ BOOLEAN  Async  )

staticprivate

Referenced by CompleteBatch(), CompleteTd(), FailStage(), FinishExpiredIsoch(), and NextTdSlot().

CompleteBatch()

VOID XhciIsochRing::CompleteBatch ( _In_ XhciTransferRing *  Ring, _Inout_ PLIST_ENTRY  Batch  )

staticprivate

Definition at line 2477 of file isoxfer.cpp.

{
    XhciIsochTransfer* Transfer;
    KIRQL OldIrql;
 
    while (!IsListEmpty(Batch))
    {
        Transfer = CONTAINING_RECORD(RemoveHeadList(Batch), XhciIsochTransfer, Link);
        XhciClearListEntry(&Transfer->Link);
 
        if (Transfer->Status != STATUS_PENDING)
        {
            WdfRequestComplete(Transfer->Request, Transfer->Status);
            continue;
        }
 
        ReleaseAllStages(Ring, Transfer);
 
        /* QUIRK: everything reclaimed reports canceled, canceled or not */
        KeAcquireSpinLock(&Ring->m_Lock, &OldIrql);
        CompleteAfterUncancel(Ring, Transfer, XhciIsochNotSet, USBD_STATUS_CANCELED, TRUE, FALSE);
        KeReleaseSpinLock(&Ring->m_Lock, OldIrql);
    }
}

CompletePlain()

static VOID XhciIsochRing::CompletePlain ( _In_ XhciTransferRing *  Ring, _Inout_ XhciIsochTransfer *  Transfer, _In_ USBD_STATUS  PacketStatus, _In_ USBD_STATUS  HeaderStatus, _In_ BOOLEAN  Async  )

staticprivate

Referenced by TakeRequest().

CompleteTd()

VOID XhciIsochRing::CompleteTd ( _In_ XhciTransferRing *  Ring, _In_ XhciIsochTransfer *  Transfer, _In_ XhciIsochStage *  Stage, _In_ ULONG  Sequence, _In_ ULONG  Code, _In_ ULONG  Bytes, _In_ BOOLEAN  CloseStage  )

staticprivate

Accounts one TD event; closing the stage's last TD frees the stage.

Definition at line 2091 of file isoxfer.cpp.

{
    XhciIsochRing* Self = &Ring->m_Type.Isoch;
    struct _URB_ISOCH_TRANSFER* Urb;
    PSCATTER_GATHER_LIST SgList;
    ULONG Packet;
    ULONG Index;
    ULONG InFlight;
    KIRQL OldIrql;
 
    KeAcquireSpinLock(&Ring->m_Lock, &OldIrql);
 
    /* A reclaim or a stale pass may have taken the stage meanwhile */
    if (!XhciIsochIsActive(&Self->m_Pending, Transfer, Sequence) ||
        !Stage->InUse ||
        Stage->Transfer != Transfer)
    {
        KeReleaseSpinLock(&Ring->m_Lock, OldIrql);
        return;
    }
 
    Urb = Transfer->Urb;
    Packet = Stage->NextPacket - 1;
    if (Packet >= Transfer->PacketsInUrb)
    {
        DPRINT1("Isoch DCI %lu: event for packet %lu of %lu\n",
                Ring->m_Endpoint->Dci(), Packet, Transfer->PacketsInUrb);
        KeReleaseSpinLock(&Ring->m_Lock, OldIrql);
        return;
    }
 
    /* TDs the controller passed without a Missed Service event */
    for (Index = Packet; Index > 0 && Urb->IsoPacket[Index - 1].Status == XhciIsochNotSet; Index--)
    {
        DPRINT1("Isoch DCI %lu: packet %lu was skipped by the controller\n",
                Ring->m_Endpoint->Dci(), Index - 1);
        Urb->IsoPacket[Index - 1].Status = USBD_STATUS_ISO_TD_ERROR;
    }
 
    if (Urb->IsoPacket[Packet].Status == XhciIsochNotSet && (!XhciIsochStoppedCode(Code) || Bytes != 0))
    {
        Urb->IsoPacket[Packet].Status = XhciTransferRing::UsbdStatusFromCompletionCode(Code);
        Transfer->BytesTransferred += Bytes;
        if (XhciIsochEndpointIn(Ring))
            Urb->IsoPacket[Packet].Length = Bytes;
    }
 
    if (Packet != Stage->LastPacket || !CloseStage)
    {
        KeReleaseSpinLock(&Ring->m_Lock, OldIrql);
        return;
    }
 
    if (!Stage->OnHardware)
    {
        DPRINT1("Isoch DCI %lu: end of a stage still being written was reported\n", Ring->m_Endpoint->Dci());
        KeReleaseSpinLock(&Ring->m_Lock, OldIrql);
        return;
    }
 
    SgList = Stage->SgList;
    Stage->SgList = NULL;
    KeReleaseSpinLock(&Ring->m_Lock, OldIrql);
 
    PutScatterGather(Ring, SgList);
 
    KeAcquireSpinLock(&Ring->m_Lock, &OldIrql);
    if (!XhciIsochIsActive(&Self->m_Pending, Transfer, Sequence) || !Stage->InUse)
    {
        KeReleaseSpinLock(&Ring->m_Lock, OldIrql);
        return;
    }
 
    FreeStage(Ring, Stage);
    Transfer->TdsRetired++;
    if (Self->m_PendingStages != 0)
        Self->m_PendingStages--;
    InFlight = Self->m_PendingStages;
 
    /* Complete only once no stage is in use and every packet was queued or dropped */
    if (TransferDone(Transfer))
        CompleteAfterUncancel(Ring, Transfer, XhciIsochNotSet, XhciIsochNotSet, FALSE, FALSE);
 
    KeReleaseSpinLock(&Ring->m_Lock, OldIrql);
 
    KickAfterStage(Ring, InFlight);
}

Referenced by Ed0Event(), Ed1Event(), FinishLostStages(), and RetireExpiredStages().

ConsumePendingEvents()

VOID XhciIsochRing::ConsumePendingEvents ( _In_ XhciTransferRing *  Ring )

static

Definition at line 726 of file isoxfer.cpp.

{
    /* Lost Missed Service events mean there is nothing reliable to wait for; late ones are dropped */
    Ring->m_Endpoint->OnExpectedEventsProcessed();
}

Referenced by XhciTransferRing::ConsumePendingEvents().

Disable()

VOID XhciIsochRing::Disable ( _In_ XhciTransferRing *  Ring )

static

Definition at line 479 of file isoxfer.cpp.

{
    XhciIsochRing* Self = &Ring->m_Type.Isoch;
    KIRQL OldIrql;
 
    ReleaseParkedMapping(Ring);
    WdfTimerStop(Self->m_Watchdog, FALSE);
 
    KeAcquireSpinLock(&Ring->m_Lock, &OldIrql);
    if (OwnsRequests(Ring))
        DPRINT1("Isoch DCI %lu disabled while it still owns requests\n", Ring->m_Endpoint->Dci());
    KeReleaseSpinLock(&Ring->m_Lock, OldIrql);
}

DoorbellRungSinceFill()

BOOLEAN XhciIsochRing::DoorbellRungSinceFill ( _In_ const XhciTransferRing *  Ring )

static

Definition at line 750 of file isoxfer.cpp.

{
    return (Ring->m_Type.Isoch.m_Flags & XhciIsochDoorbellRung) != 0;
}

Referenced by XhciTransferRing::DoorbellRungSinceFill().

Ed0Event()

VOID XhciIsochRing::Ed0Event ( _In_ XhciTransferRing *  Ring, _In_ const XHCI_TRB *  Event  )

staticprivate

Definition at line 2661 of file isoxfer.cpp.

{
    XhciIsochRing* Self = &Ring->m_Type.Isoch;
    ULONG Code = XhciTrbCompletionCode(Event);
    ULONG64 Pointer = XhciTrbPointer(Event);
    XhciIsochTransfer* Transfer = NULL;
    XhciIsochStage* Stage;
    ULONG Sequence = 0;
    ULONG Bytes;
    BOOLEAN Matched;
    KIRQL OldIrql;
 
    switch (static_cast<XhciCompletionCode>(Code))
    {
        case XhciCompletionCode::RingUnderrun:
        case XhciCompletionCode::RingOverrun:
            OnRingStarved(Ring);
            return;
 
        case XhciCompletionCode::MissedService:
            Self->m_MissedServiceErrors++;
 
            /* A Missed Service event without a TRB pointer has nothing to complete */
            if (Pointer == 0)
                return;
            break;
 
        case XhciCompletionCode::NoPingResponse:
            UcxEndpointNoPingResponseError(Ring->m_Endpoint->Handle());
 
            /* The length is invalid without a TRB pointer */
            if (Pointer == 0)
                return;
            break;
 
        case XhciCompletionCode::Stopped:
        case XhciCompletionCode::StoppedLengthInvalid:
        case XhciCompletionCode::StoppedShortPacket:
        case XhciCompletionCode::DataBufferError:
        case XhciCompletionCode::BabbleDetected:
        case XhciCompletionCode::IsochBufferOverrun:
        case XhciCompletionCode::UsbTransactionError:
        case XhciCompletionCode::StallError:
        case XhciCompletionCode::EventLost:
        case XhciCompletionCode::SplitTransactionError:
            break;
 
        default:
            DPRINT1("Isoch DCI %lu: unexpected code %lu without Event Data\n", Ring->m_Endpoint->Dci(), Code);
            break;
    }
 
    KeAcquireSpinLock(&Ring->m_Lock, &OldIrql);
 
    if (Self->m_Flags & XhciIsochStoppedSeen)
    {
        KeReleaseSpinLock(&Ring->m_Lock, OldIrql);
        return;
    }
 
    Matched = FindTrb(Ring, Event, &Stage, &Bytes);
    if (Stage != NULL)
    {
        Transfer = Stage->Transfer;
        Sequence = Transfer->Sequence;
    }
 
    if (XhciIsochStoppedCode(Code))
    {
        /* Also covers a stop at the enqueue pointer or on a Link TRB after the last stage */
        InterlockedOr(&Self->m_Flags, XhciIsochStoppedSeen);
        KeReleaseSpinLock(&Ring->m_Lock, OldIrql);
 
        if (Stage != NULL)
            CompleteTd(Ring, Transfer, Stage, Sequence, Code, Bytes, FALSE);
 
        Ring->m_Endpoint->OnStoppedEvent();
        return;
    }
 
    KeReleaseSpinLock(&Ring->m_Lock, OldIrql);
 
    if (Stage == NULL)
    {
        if (!Matched)
        {
            DPRINT("Isoch DCI %lu: event at 0x%I64x code %lu matched no stage, dropped\n",
                   Ring->m_Endpoint->Dci(), Pointer, Code);
        }
        return;
    }
 
    /* The following Event Data event or a later stale pass closes the stage */
    FinishExpiredIsoch(Ring, Transfer, Sequence);
    RetireExpiredStages(Ring, Transfer, Stage, Sequence);
    CompleteTd(Ring, Transfer, Stage, Sequence, Code, Bytes, FALSE);
}

Referenced by OnTransferEvent().

Ed1Event()

VOID XhciIsochRing::Ed1Event ( _In_ XhciTransferRing *  Ring, _In_ const XHCI_TRB *  Event  )

staticprivate

Definition at line 2564 of file isoxfer.cpp.

{
    XhciIsochRing* Self = &Ring->m_Type.Isoch;
    ULONG64 Value = XhciTrbPointer(Event);
    ULONG Code = XhciTrbCompletionCode(Event);
    ULONG Bytes = Event->Dword[2] & XHCI_TRANSFER_EVENT_LENGTH_MASK;
    ULONG Packet = (ULONG)(Value >> 16) & 0xFFFF;
    ULONG Frame = (ULONG)(Value >> 32);
    XhciIsochTransfer* Transfer = NULL;
    XhciIsochStage* Stage = NULL;
    ULONG Sequence = 0;
    PLIST_ENTRY Entry;
    ULONG Index;
    KIRQL OldIrql;
 
    if ((Value & XHCI_EVENT_DATA_TYPE_MASK) != USB_ENDPOINT_TYPE_ISOCHRONOUS)
    {
        DPRINT1("Isoch DCI %lu: Event Data 0x%I64x is not an isoch tag, resetting\n",
                Ring->m_Endpoint->Dci(), Value);
        Ring->m_Controller->RaiseControllerFault(XhciRecovery::ResetHost, XhciIsochReasonBadEventData);
        return;
    }
 
    KeAcquireSpinLock(&Ring->m_Lock, &OldIrql);
 
    if (Self->m_Flags & XhciIsochStoppedSeen)
    {
        KeReleaseSpinLock(&Ring->m_Lock, OldIrql);
        return;
    }
 
    /* QUIRK: found by start frame, so of two URBs with one start frame the older one matches */
    for (Entry = Self->m_Pending.Flink; Entry != &Self->m_Pending; Entry = Entry->Flink)
    {
        XhciIsochTransfer* Candidate = CONTAINING_RECORD(Entry, XhciIsochTransfer, Link);
 
        if (Candidate->StartFrame == Frame)
        {
            Transfer = Candidate;
            break;
        }
    }
 
    if (Transfer != NULL)
    {
        Sequence = Transfer->Sequence;
 
        for (Index = 0; Index < Transfer->StageCount; Index++)
        {
            XhciIsochStage* Candidate = XhciIsochStageAt(Transfer, Index);
 
            if (Packet <= Candidate->LastPacket)
            {
                Stage = Candidate;
                break;
            }
        }
 
        if (Stage == NULL)
        {
            DPRINT1("Isoch DCI %lu: no stage holds packet %lu of request %p\n",
                    Ring->m_Endpoint->Dci(), Packet, Transfer->Request);
        }
        else if (Packet == 0 || Packet > Stage->NextPacket - 1)
        {
            Stage->NextPacket = Packet + 1;
            Transfer->PacketsRetired = Packet + 1;
        }
    }
 
    if (XhciIsochStoppedCode(Code))
    {
        /* Some controllers report the stop on the Event Data TRB */
        InterlockedOr(&Self->m_Flags, XhciIsochStoppedSeen);
        KeReleaseSpinLock(&Ring->m_Lock, OldIrql);
 
        if (Stage != NULL)
            CompleteTd(Ring, Transfer, Stage, Sequence, Code, Bytes, FALSE);
 
        Ring->m_Endpoint->OnStoppedEvent();
        return;
    }
 
    KeReleaseSpinLock(&Ring->m_Lock, OldIrql);
 
    /* A late event of a request that is gone already */
    if (Stage == NULL)
        return;
 
    FinishExpiredIsoch(Ring, Transfer, Sequence);
    RetireExpiredStages(Ring, Transfer, Stage, Sequence);
    CompleteTd(Ring, Transfer, Stage, Sequence, Code, Bytes, TRUE);
}

Referenced by OnTransferEvent().

Enable()

VOID XhciIsochRing::Enable ( _In_ XhciTransferRing *  Ring )

static

Definition at line 468 of file isoxfer.cpp.

{
    KIRQL OldIrql;
 
    KeAcquireSpinLock(&Ring->m_Lock, &OldIrql);
    Ring->m_Type.Isoch.m_FreshStream = TRUE;
    KeReleaseSpinLock(&Ring->m_Lock, OldIrql);
}

Referenced by _IRQL_requires_max_().

EncodeStage()

XhciIsochRing::Outcome XhciIsochRing::EncodeStage ( _In_ XhciTransferRing *  Ring, _Inout_ XhciIsochStage *  Stage  )

staticprivate

Definition at line 1675 of file isoxfer.cpp.

{
    XhciIsochRing* Self = &Ring->m_Type.Isoch;
    PDMA_ADAPTER Adapter = Ring->m_Controller->m_Buffers.DmaAdapter();
    NTSTATUS Status;
    KIRQL OldIrql;
 
    InterlockedExchange(&Self->m_LateStage, 0);
 
    KeRaiseIrql(DISPATCH_LEVEL, &OldIrql);
    Status = Adapter->DmaOperations->GetScatterGatherList(Adapter,
                                                          WdfDeviceWdmGetDeviceObject(Ring->m_Controller->m_Device),
                                                          Stage->Mdl,
                                                          MmGetMdlVirtualAddress(Stage->Mdl),
                                                          Stage->Size,
                                                          DmaCallback,
                                                          Stage,
                                                          !XhciIsochEndpointIn(Ring));
    KeLowerIrql(OldIrql);
 
    if (!NT_SUCCESS(Status))
    {
        DPRINT1("Isoch DCI %lu: GetScatterGatherList for %lu bytes failed, 0x%08lx\n",
                Ring->m_Endpoint->Dci(), Stage->Size, Status);
        (VOID)FailStage(Ring, Stage);
        return Outcome::Ready;
    }
 
    /* Whichever of this context and the callback gets here second keeps mapping */
    if (InterlockedXor(&Self->m_FillAgain, 1) == 1)
        return (Self->m_LateStage != 0) ? Outcome::Late : Outcome::Ready;
 
    return Outcome::WaitDma;
}

Referenced by FillLoop().

EstimateSegments()

ULONG XhciIsochRing::EstimateSegments ( _In_ XhciTransferRing *  Ring, _In_ const XhciIsochStage *  Stage  )

staticprivate

Definition at line 1601 of file isoxfer.cpp.

{
    XhciIsochRing* Self = &Ring->m_Type.Isoch;
    BOOLEAN Guard = XhciIsochNeedsLinkGuard(Ring);
    ULONG PerTd;
    ULONG Needed;
    ULONG Available;
    ULONG Usable;
    ULONG Segments = 0;
 
    PerTd = 1 + ((XhciIsochEsitPayload(Ring) + 2 * PAGE_SIZE - 2) >> PAGE_SHIFT);
    if (Guard)
        PerTd += 1;
    if (MmGetMdlByteCount(Stage->Mdl) < Stage->Size)
        PerTd += 2;
 
    /* A TD never needs more than one segment */
    if (PerTd > Ring->m_LastIndex)
        PerTd = Ring->m_LastIndex;
 
    Self->m_TrbsPerTd = PerTd;
 
    Needed = PerTd * (Stage->LastPacket - Stage->FirstPacket + 1);
    Available = Ring->m_LastIndex - Ring->m_EnqueueIndex;
    Available -= Available % PerTd;
 
    if (Needed > Available)
    {
        Usable = Ring->m_LastIndex - (Ring->m_LastIndex % PerTd);
        Segments = (Needed - Available + Usable - 1) / Usable;
    }
 
    if (Guard)
        Segments++;
 
    return Segments;
}

Referenced by PrimeStage().

FailStage()

XhciIsochRing::Outcome XhciIsochRing::FailStage ( _In_ XhciTransferRing *  Ring, _Inout_ XhciIsochStage *  Stage  )

staticprivate

Drops a stage that cannot be mapped; completes its request when nothing of it is on the ring.

Definition at line 1643 of file isoxfer.cpp.

{
    XhciIsochTransfer* Transfer = Stage->Transfer;
    PSCATTER_GATHER_LIST SgList = Stage->SgList;
    Outcome Result;
    KIRQL OldIrql;
 
    Stage->SgList = NULL;
    PutScatterGather(Ring, SgList);
 
    KeAcquireSpinLock(&Ring->m_Lock, &OldIrql);
    FreeStage(Ring, Stage);
 
    if (Transfer->TdsRetired == Transfer->TdsQueued)
    {
        CompleteAfterUncancel(Ring, Transfer, XhciIsochNotSet, USBD_STATUS_INSUFFICIENT_RESOURCES, TRUE, FALSE);
        Result = Outcome::Completed;
    }
    else
    {
        /* The request completes when its last mapped stage comes back */
        SkipUnmapped(Transfer);
        Result = Outcome::Failed;
    }
 
    KeReleaseSpinLock(&Ring->m_Lock, OldIrql);
    return Result;
}

Referenced by EncodeStage(), PrimeStage(), and WriteStage().

FillLoop()

VOID XhciIsochRing::FillLoop ( _In_ XhciTransferRing *  Ring )

staticprivate

Definition at line 857 of file isoxfer.cpp.

{
    XhciIsochRing* Self = &Ring->m_Type.Isoch;
    XhciIsochStage* Stage;
    Outcome Result;
 
    for (;;)
    {
        InterlockedExchange(&Self->m_FillWanted, 0);
        InterlockedExchange(&Self->m_RingEmptySeen, 0);
 
        Stage = NextTdSlot(Ring);
        if (Stage == NULL)
        {
            Ring->ChangeMapState(FALSE, XhciMapState::Filling, XhciMapState::PausedFill);
            return;
        }
 
        Result = PrimeStage(Ring, Stage);
        if (Result == Outcome::Completed)
            continue;
        if (Result == Outcome::WaitSegments)
            return;
        if (Result == Outcome::Gap)
        {
            AwaitRingEmpty(Ring);
            return;
        }
        if (Result == Outcome::Failed)
        {
            Ring->ChangeMapState(FALSE, XhciMapState::Filling, XhciMapState::PausedFill);
            return;
        }
 
        Result = EncodeStage(Ring, Stage);
        if (Result == Outcome::WaitDma)
            return;
        if (Result == Outcome::Late)
        {
            AwaitRingEmpty(Ring);
            return;
        }
 
        /* Also completes a stop that raced in */
        if (Ring->ChangeMapState(FALSE, XhciMapState::Filling, XhciMapState::Filling) != XhciMapState::Filling)
            return;
    }
}

Referenced by BuildTdsFromQueue().

Finalize()

VOID XhciIsochRing::Finalize ( _In_ XhciTransferRing *  Ring, _Inout_ XhciIsochTransfer *  Transfer, _In_ USBD_STATUS  PacketStatus, _In_ USBD_STATUS  HeaderStatus  )

staticprivate

Fills in the URB from the transfer record and frees what intake built.

Definition at line 2312 of file isoxfer.cpp.

{
    XhciIsochRing* Self = &Ring->m_Type.Isoch;
    struct _URB_ISOCH_TRANSFER* Urb = Transfer->Urb;
    USBD_STATUS Status;
    ULONG Index;
 
    NT_ASSERT(Transfer->StageCount == 0);
 
    Urb->TransferBufferLength = Transfer->BytesTransferred;
 
    for (Index = 0; Index < Transfer->PacketsInUrb; Index++)
    {
        if (PacketStatus != XhciIsochNotSet)
            Urb->IsoPacket[Index].Status = PacketStatus;
        else if (Urb->IsoPacket[Index].Status == XhciIsochNotSet)
            Urb->IsoPacket[Index].Status = USBD_STATUS_ISO_NOT_ACCESSED_BY_HW;
 
        /* QUIRK: added to whatever the URB held, as the USB 2.0 stack does */
        if (Urb->IsoPacket[Index].Status != USBD_STATUS_SUCCESS)
        {
            Urb->ErrorCount++;
            Self->m_TdFailures++;
        }
        Self->m_Tds++;
    }
 
    if (HeaderStatus != XhciIsochNotSet)
        Status = HeaderStatus;
    else if (Urb->ErrorCount == Transfer->PacketsInUrb)
        Status = USBD_STATUS_ISOCH_REQUEST_FAILED;
    else
        Status = USBD_STATUS_SUCCESS;
 
    Urb->Hdr.Status = Status;
    Transfer->Status = XhciTransferRing::NtStatusFromUsbdStatus(Status);
 
    Self->m_Transfers++;
    Self->m_Bytes += Transfer->BytesTransferred;
    if (!NT_SUCCESS(Transfer->Status))
    {
        Self->m_FailedTransfers++;
 
        if (Transfer->Status != STATUS_CANCELLED)
        {
            DPRINT1("Isoch DCI %lu request %p failed, 0x%08lx USBD 0x%08lx, %lu of %lu packets in error\n",
                    Ring->m_Endpoint->Dci(), Transfer->Request, Transfer->Status, Status,
                    Urb->ErrorCount, Transfer->PacketsInUrb);
        }
    }
 
    if (Transfer->OwnMdl)
    {
        IoFreeMdl(Transfer->Mdl);
        Transfer->OwnMdl = FALSE;
    }
    Transfer->Mdl = NULL;
    Transfer->Initialized = 0;
}

FindTrb()

static BOOLEAN XhciIsochRing::FindTrb ( _In_ XhciTransferRing *  Ring, _In_ const XHCI_TRB *  Event, _Outptr_result_maybenull_ XhciIsochStage **  Found, _Out_ PULONG  Bytes  )

staticprivate

Referenced by Ed0Event().

FinishExpiredIsoch()

VOID XhciIsochRing::FinishExpiredIsoch ( _In_ XhciTransferRing *  Ring, _In_ const XhciIsochTransfer *  Newest, _In_ ULONG  Sequence  )

staticprivate

Completes every active transfer older than Newest; their events will not come.

Definition at line 2188 of file isoxfer.cpp.

{
    XhciIsochRing* Self = &Ring->m_Type.Isoch;
    XhciIsochTransfer* Transfer;
    LIST_ENTRY Stale;
    KIRQL OldIrql;
 
    InitializeListHead(&Stale);
 
    KeAcquireSpinLock(&Ring->m_Lock, &OldIrql);
    if (XhciIsochIsActive(&Self->m_Pending, Newest, Sequence))
    {
        while (Self->m_Pending.Flink != &Newest->Link)
            InsertTailList(&Stale, RemoveHeadList(&Self->m_Pending));
    }
    KeReleaseSpinLock(&Ring->m_Lock, OldIrql);
 
    while (!IsListEmpty(&Stale))
    {
        Transfer = CONTAINING_RECORD(RemoveHeadList(&Stale), XhciIsochTransfer, Link);
        XhciClearListEntry(&Transfer->Link);
 
        DPRINT("Isoch DCI %lu: completing stale request %p\n", Ring->m_Endpoint->Dci(), Transfer->Request);
        ReleaseAllStages(Ring, Transfer);
 
        KeAcquireSpinLock(&Ring->m_Lock, &OldIrql);
        CompleteAfterUncancel(Ring, Transfer, XhciIsochNotSet, XhciIsochNotSet, TRUE, FALSE);
        KeReleaseSpinLock(&Ring->m_Lock, OldIrql);
    }
}

Referenced by Ed0Event(), and Ed1Event().

FinishLostStages()

BOOLEAN XhciIsochRing::FinishLostStages ( _In_ XhciTransferRing *  Ring )

staticprivate

Closes in flight stages as missed once every frame they were scheduled for has passed.

Definition at line 2960 of file isoxfer.cpp.

{
    XhciIsochRing* Self = &Ring->m_Type.Isoch;
    ULONG Now = Ring->m_Controller->GetFrameNumber(XhciIsochBiasNow);
    XhciIsochTransfer* Transfer;
    XhciIsochStage* Stage;
    PLIST_ENTRY Entry;
    ULONG Sequence;
    ULONG Index;
    BOOLEAN Stuck;
    KIRQL OldIrql;
 
    KeAcquireSpinLock(&Ring->m_Lock, &OldIrql);
    if ((Self->m_Flags & XhciIsochLastFrameValid) && (LONG)(Now - Self->m_LastScheduledFrame) <= 0)
    {
        KeReleaseSpinLock(&Ring->m_Lock, OldIrql);
        return FALSE;
    }
    KeReleaseSpinLock(&Ring->m_Lock, OldIrql);
 
    for (;;)
    {
        Stage = NULL;
        Transfer = NULL;
        Sequence = 0;
 
        KeAcquireSpinLock(&Ring->m_Lock, &OldIrql);
        for (Entry = Self->m_Pending.Flink; Entry != &Self->m_Pending && Stage == NULL; Entry = Entry->Flink)
        {
            Transfer = CONTAINING_RECORD(Entry, XhciIsochTransfer, Link);
 
            for (Index = 0; Index < Transfer->StageCount; Index++)
            {
                if (XhciIsochStageAt(Transfer, Index)->OnHardware)
                {
                    Stage = XhciIsochStageAt(Transfer, Index);
                    Stage->NextPacket = Stage->LastPacket + 1;
                    Sequence = Transfer->Sequence;
                    break;
                }
            }
        }
        KeReleaseSpinLock(&Ring->m_Lock, OldIrql);
 
        if (Stage == NULL)
            break;
 
        DPRINT1("Isoch DCI %lu: closing a stage whose events never came\n", Ring->m_Endpoint->Dci());
        CompleteTd(Ring, Transfer, Stage, Sequence, static_cast<ULONG>(XhciCompletionCode::MissedService), 0, TRUE);
 
        KeAcquireSpinLock(&Ring->m_Lock, &OldIrql);
        Stuck = Stage->InUse && Stage->OnHardware && Stage->Transfer == Transfer;
        KeReleaseSpinLock(&Ring->m_Lock, OldIrql);
 
        if (Stuck)
            break;
    }
 
    KeAcquireSpinLock(&Ring->m_Lock, &OldIrql);
    InterlockedAnd(&Self->m_Flags, ~XhciIsochEmptyExpected);
    if (Self->m_PendingStages == 0)
        InterlockedAnd(&Self->m_Flags, ~XhciIsochLastFrameValid);
    KeReleaseSpinLock(&Ring->m_Lock, OldIrql);
 
    return TRUE;
}

FlagMissedPackets()

VOID XhciIsochRing::FlagMissedPackets ( _In_ XhciTransferRing *  Ring, _Inout_ XhciIsochTransfer *  Transfer  )

staticprivate

Marks the unmapped frames that are already due as late.

Definition at line 1253 of file isoxfer.cpp.

{
    ULONG PerFrame = Ring->m_Type.Isoch.m_PacketsPerFrame;
    ULONG First = Transfer->PacketsQueued + Transfer->PacketsDropped;
    ULONG Frame = Transfer->StartFrame + XhciIsochFrames(First, PerFrame);
    ULONG End = Frame + XhciIsochFrames(Transfer->PacketsInUrb - First, PerFrame);
    ULONG Now = Ring->m_Controller->GetFrameNumber(XhciIsochBiasLate);
    ULONG Packet;
    ULONG Index;
 
    /* QUIRK: unsigned compares without wrap handling */
    while (Frame < End && Frame <= Now)
    {
        Packet = (Frame - Transfer->StartFrame) * PerFrame;
 
        for (Index = 0; Index < PerFrame && Packet + Index < Transfer->PacketsInUrb; Index++)
        {
            Transfer->Urb->IsoPacket[Packet + Index].Status = USBD_STATUS_ISO_NOT_ACCESSED_LATE;
            Transfer->BytesDropped += XhciIsochPacketLength(Transfer, Packet + Index);
            Transfer->PacketsDropped++;
        }
 
        Frame++;
    }
}

Referenced by NextTdSlot(), and TakeRequest().

FreeStage()

static VOID XhciIsochRing::FreeStage ( _In_ XhciTransferRing *  Ring, _Inout_ XhciIsochStage *  Stage  )

staticprivate

Referenced by CompleteTd(), FailStage(), PrimeStage(), ReleaseAllStages(), and WriteStage().

HasQueuedWork()

BOOLEAN XhciIsochRing::HasQueuedWork ( _In_ const XhciTransferRing *  Ring )

static

Definition at line 757 of file isoxfer.cpp.

{
    PKSPIN_LOCK Lock = const_cast<PKSPIN_LOCK>(&Ring->m_Lock);
    BOOLEAN Pending;
    KIRQL OldIrql;
 
    KeAcquireSpinLock(Lock, &OldIrql);
    Pending = !IsListEmpty(&Ring->m_Type.Isoch.m_Pending);
    KeReleaseSpinLock(Lock, OldIrql);
 
    return Pending;
}

Referenced by XhciTransferRing::HasQueuedWork().

Initialize()

NTSTATUS XhciIsochRing::Initialize ( _In_ XhciTransferRing *  Ring )

static

Definition at line 422 of file isoxfer.cpp.

{
    XhciIsochRing* Self = &Ring->m_Type.Isoch;
    USB_DEVICE_SPEED Speed = Ring->m_Device->Speed();
    ULONG Interval = Ring->m_Endpoint->Descriptor()->bInterval;
    WDF_TIMER_CONFIG TimerConfig;
    WDF_OBJECT_ATTRIBUTES Attributes;
    ULONG Period;
    NTSTATUS Status;
 
    RtlZeroMemory(Self, sizeof(*Self));
    InitializeListHead(&Self->m_Pending);
    InitializeListHead(&Self->m_WaitingForCancel);
    InitializeListHead(&Self->m_Completion);
 
    if (Speed >= UsbHighSpeed)
    {
        /* QUIRK: the raw bInterval is used, and 0 behaves as an 8 microframe period */
        Period = (Interval == 0 || Interval > 4) ? 8 : (1UL << (Interval - 1));
        Self->m_PacketsPerFrame = 8 / Period;
    }
    else
    {
        NT_ASSERT(Speed == UsbFullSpeed);
        Self->m_PacketsPerFrame = 1;
    }
 
    WDF_TIMER_CONFIG_INIT(&TimerConfig, EvtWatchdog);
    TimerConfig.TolerableDelay = 0;
    WDF_OBJECT_ATTRIBUTES_INIT(&Attributes);
    Attributes.ParentObject = Ring->m_Queue;
 
    Status = WdfTimerCreate(&TimerConfig, &Attributes, &Self->m_Watchdog);
    if (!NT_SUCCESS(Status))
    {
        DPRINT1("Isoch DCI %lu: ring empty watchdog not created, 0x%08lx\n",
                Ring->m_Endpoint->Dci(), Status);
        return Status;
    }
 
    DPRINT("Isoch DCI %lu: %lu packets per frame\n", Ring->m_Endpoint->Dci(), Self->m_PacketsPerFrame);
    return STATUS_SUCCESS;
}

Referenced by _IRQL_requires_().

InsertLink()

VOID XhciIsochRing::InsertLink ( _In_ XhciTransferRing *  Ring, _Inout_ XhciIsochStage *  Stage, _In_ BOOLEAN  FirstOfTd  )

staticprivate

Ends the current segment with a Link TRB into the stage's next reserved segment.

Definition at line 2056 of file isoxfer.cpp.

{
    XhciDmaBuffer* Next;
    XHCI_TRB Trb;
    KIRQL OldIrql;
 
    /* The layout pass made sure a reserved segment is there */
    NT_ASSERT(!IsListEmpty(&Stage->FreeSegments));
 
    Next = CONTAINING_RECORD(RemoveHeadList(&Stage->FreeSegments), XhciDmaBuffer, Link);
    XhciClearListEntry(&Next->Link);
 
    Ring->BuildLinkTrb(&Trb, FirstOfTd, !FirstOfTd);
    Trb.Dword[0] = Next->LogicalAddress.LowPart;
    Trb.Dword[1] = (ULONG)Next->LogicalAddress.HighPart;
    Ring->m_Trbs[Ring->m_EnqueueIndex] = Trb;
 
    /* Never Toggle Cycle: the ring is an endless forward chain of fresh segments */
    KeAcquireSpinLock(&Ring->m_Lock, &OldIrql);
    InsertTailList(&Stage->UsedSegments, &Ring->m_Segment->Link);
    Ring->m_Segment = Next;
    Ring->m_Trbs = static_cast<PXHCI_TRB>(Next->VirtualAddress);
    Ring->m_EnqueueIndex = 0;
    Stage->LastSegment = Next;
    Stage->EndIndex = 0;
    KeReleaseSpinLock(&Ring->m_Lock, OldIrql);
}

Referenced by LayoutStage().

KickAfterStage()

VOID XhciIsochRing::KickAfterStage ( _In_ XhciTransferRing *  Ring, _In_ ULONG  InFlight  )

staticprivate

A stage left the ring: map more, or leave the ring empty wait once nothing is in flight.

Definition at line 931 of file isoxfer.cpp.

{
    InterlockedExchange(&Ring->m_Type.Isoch.m_FillWanted, 1);
 
    if (Ring->ChangeMapState(FALSE, XhciMapState::PausedFill, XhciMapState::Filling) == XhciMapState::PausedFill)
    {
        BuildTdsFromQueue(Ring);
        return;
    }
 
    if (InFlight == 0 && LeaveRingEmptyWait(Ring))
        BuildTdsFromQueue(Ring);
}

Referenced by CompleteTd().

LayoutStage()

ULONG XhciIsochRing::LayoutStage ( _In_ XhciTransferRing *  Ring, _Inout_ XhciIsochStage *  Stage, _In_ BOOLEAN  Commit  )

staticprivate

Lays the stage's TDs out on the ring. Without Commit it only counts how many TDs fit the current segment and the reserved ones, the last of them still able to end the stage.

Definition at line 1823 of file isoxfer.cpp.

{
    XhciIsochRing* Self = &Ring->m_Type.Isoch;
    XhciIsochTransfer* Transfer = Stage->Transfer;
    const SCATTER_GATHER_LIST* SgList = Stage->SgList;
    ULONG LastSlot = Ring->m_LastIndex;
    ULONG LastPacket = Stage->LastPacket;
    ULONG PerFrame = Self->m_PacketsPerFrame;
    BOOLEAN Guard = XhciIsochNeedsLinkGuard(Ring);
    BOOLEAN AllowBei = !Ring->m_Controller->HasErrata(XhciErrata::IsochInterruptEveryTd);
    ULONG Hcc2 = Ring->m_Controller->m_Registers.Hccparams2();
    BOOLEAN ExtendedTbc = (Hcc2 & XHCI_HCC2_LEC) && (Hcc2 & XHCI_HCC2_ETC);
    ULONG Esit = XhciIsochEsitPayload(Ring);
    ULONG MaxPacket = Ring->m_Endpoint->MaxPacketSize();
    ULONG BurstSize = Ring->m_Endpoint->MaxBurst() + 1;
    ULONG Target = (Ring->m_InterrupterTarget & XHCI_TRB_INTERRUPTER_MASK) << XHCI_TRB_INTERRUPTER_SHIFT;
    ULONG Spares = XhciIsochCountList(&Stage->FreeSegments);
    ULONG Index = Ring->m_EnqueueIndex;
    ULONG Element = 0;
    ULONG Offset = 0;
    ULONG Placed = 0;
    ULONG Fit = 0;
    ULONG Packet;
    KIRQL OldIrql;
 
    if (Commit)
    {
        KeAcquireSpinLock(&Ring->m_Lock, &OldIrql);
        Stage->FirstSegment = Ring->m_Segment;
        Stage->LastSegment = Ring->m_Segment;
        Stage->ScanSegment = Ring->m_Segment;
        Stage->FirstIndex = Ring->m_EnqueueIndex;
        Stage->EndIndex = Ring->m_EnqueueIndex;
        Stage->ScanIndex = Ring->m_EnqueueIndex;
        KeReleaseSpinLock(&Ring->m_Lock, OldIrql);
    }
 
    for (Packet = Stage->FirstPacket; Packet <= LastPacket; Packet++)
    {
        ULONG Length = XhciIsochPacketLength(Transfer, Packet);
        ULONG Carried = min(Length, Esit);
        ULONG Pieces = XhciIsochCountPieces(SgList, Element, Offset, Carried);
        BOOLEAN IsLast = (Packet == LastPacket);
        ULONG Frame = Transfer->StartFrame + Packet / PerFrame;
        PXHCI_TRB First = NULL;
        volatile UCHAR* CycleByte;
        BOOLEAN Tail;
        BOOLEAN Bei;
        XHCI_TRB Trb;
        ULONG Tdpc;
        ULONG Tbc;
        ULONG Tlbpc;
        ULONG Mapped = 0;
        ULONG Piece;
        ULONG64 Value;
 
        if (Commit)
            Index = Ring->m_EnqueueIndex;
 
        /* The data TRBs and the Event Data TRB of a TD stay in front of the Link slot */
        if (Index + Pieces + 1 > LastSlot)
        {
            if (!Commit)
            {
                if (Spares == 0 || Pieces + 1 > LastSlot)
                    break;
                Spares--;
            }
            else
            {
                First = &Ring->m_Trbs[Ring->m_EnqueueIndex];
                InsertLink(Ring, Stage, TRUE);
            }
            Index = 0;
        }
 
        /* A full speed TD behind a TT ends in a new segment instead of meeting a Link first */
        Tail = Guard && (IsLast || LastSlot - (Index + Pieces) < Self->m_TrbsPerTd);
 
        if (!Commit)
        {
            if (!Guard || Spares != 0)
                Fit = Placed + 1;
 
            if (Tail)
            {
                if (Spares == 0)
                    break;
                Spares--;
                Index = 1;
            }
            else
            {
                Index += Pieces + 1;
            }
 
            Placed++;
            XhciIsochAdvance(SgList, &Element, &Offset, Length);
            continue;
        }
 
        /* Burst fields (xHCI 4.11.2.3) */
        Tdpc = (MaxPacket != 0) ? (Carried + MaxPacket - 1) / MaxPacket : 1;
        if (Tdpc == 0)
            Tdpc = 1;
        Tbc = (Tdpc + BurstSize - 1) / BurstSize - 1;
        Tlbpc = (Tdpc % BurstSize) ? (Tdpc % BurstSize) - 1 : BurstSize - 1;
 
        if (First == NULL)
            First = &Ring->m_Trbs[Ring->m_EnqueueIndex];
 
        for (Piece = 0; Piece < Pieces; Piece++)
        {
            BOOLEAN LastData = (Piece + 1 == Pieces);
            PXHCI_TRB Slot = &Ring->m_Trbs[Ring->m_EnqueueIndex];
            ULONG64 Address;
            ULONG Chunk;
            ULONG SizeField;
 
            /* QUIRK: bytes beyond the max ESIT payload of a packet are never sent */
            XhciIsochTakePiece(SgList, &Element, &Offset, Carried - Mapped, &Address, &Chunk);
            Mapped += Chunk;
 
            RtlZeroMemory(&Trb, sizeof(Trb));
            Trb.Dword[0] = (ULONG)Address;
            Trb.Dword[1] = (ULONG)(Address >> 32);
 
            if (Piece == 0 && ExtendedTbc)
                SizeField = Tbc;
            else
                SizeField = Ring->TdSize(Ring->PacketCount(Carried), Mapped, LastData);
 
            Trb.Dword[2] = Chunk | (SizeField << XHCI_TRB_TD_SIZE_SHIFT) | Target;
            Trb.Dword[3] = XHCI_TRB_CHAIN | (LastData ? XHCI_TRB_ENT : 0);
 
            if (Piece == 0)
            {
                /* QUIRK: every TD carries its Frame ID, SIA is never used */
                Trb.Dword[3] |= (static_cast<ULONG>(XhciTrbType::Isoch) << XHCI_TRB_TYPE_SHIFT) |
                                (Tlbpc << XHCI_ISOCH_TLBPC_SHIFT) |
                                ((Frame & XHCI_ISOCH_FRAME_ID_MASK) << XHCI_ISOCH_FRAME_ID_SHIFT);
                if (!ExtendedTbc)
                    Trb.Dword[3] |= Tbc << XHCI_ISOCH_TBC_SHIFT;
            }
            else
            {
                Trb.Dword[3] |= static_cast<ULONG>(XhciTrbType::Normal) << XHCI_TRB_TYPE_SHIFT;
            }
 
            /* The first slot of the TD stays invalid until the whole TD is written */
            Trb.Dword[3] |= (Slot == First) ? (Ring->m_Cycle ^ XHCI_TRB_CYCLE) : Ring->m_Cycle;
 
            *Slot = Trb;
            Ring->m_EnqueueIndex++;
        }
 
        XhciIsochAdvance(SgList, &Element, &Offset, Length - Mapped);
 
        if (Tail)
        {
            RtlZeroMemory(&Trb, sizeof(Trb));
            Trb.Dword[3] = (static_cast<ULONG>(XhciTrbType::Normal) << XHCI_TRB_TYPE_SHIFT) |
                           XHCI_TRB_CHAIN | XHCI_TRB_ENT | Ring->m_Cycle;
            Ring->m_Trbs[Ring->m_EnqueueIndex] = Trb;
            Ring->m_EnqueueIndex++;
 
            InsertLink(Ring, Stage, FALSE);
        }
 
        /* An interrupt at least every 128 packets and at every stage end */
        Bei = AllowBei &&
              !IsLast &&
              Packet + 1 != Transfer->PacketsInUrb &&
              ((Packet + 1) & XhciIsochInterruptEvery) != 0;
 
        Value = USB_ENDPOINT_TYPE_ISOCHRONOUS | ((ULONG64)Packet << 16) | ((ULONG64)Transfer->StartFrame << 32);
        RtlZeroMemory(&Trb, sizeof(Trb));
        Trb.Dword[0] = (ULONG)Value;
        Trb.Dword[1] = (ULONG)(Value >> 32);
        Trb.Dword[2] = Target;
        Trb.Dword[3] = (static_cast<ULONG>(XhciTrbType::EventData) << XHCI_TRB_TYPE_SHIFT) |
                       XHCI_TRB_IOC | (Bei ? XHCI_TRB_BEI : 0) | Ring->m_Cycle;
        Ring->m_Trbs[Ring->m_EnqueueIndex] = Trb;
        Ring->m_EnqueueIndex++;
 
        /* Recycled segments are not cleared, and a stale TRB could look valid */
        Stage->EndIndex = Ring->m_EnqueueIndex;
        RtlZeroMemory(&Ring->m_Trbs[Ring->m_EnqueueIndex], sizeof(XHCI_TRB));
 
        if (IsLast || Self->m_DoorbellAfterFirstTd)
        {
            KeAcquireSpinLock(&Ring->m_Lock, &OldIrql);
            if (IsLast)
            {
                Transfer->PacketsQueued = Packet + 1 - Transfer->PacketsDropped;
                Transfer->TdsQueued++;
                Transfer->BytesQueued += Stage->Size;
                Stage->OnHardware = TRUE;
                Self->m_PendingStages++;
                Self->m_LastScheduledFrame = Frame;
                InterlockedOr(&Self->m_Flags, XhciIsochLastFrameValid);
 
                /* A cut stage leaves the rest of the URB to skip */
                if (Stage->Truncated)
                    SkipUnmapped(Transfer);
            }
            InterlockedOr(&Self->m_Flags, XhciIsochEmptyExpected | XhciIsochDoorbellRung);
            KeReleaseSpinLock(&Ring->m_Lock, OldIrql);
        }
 
        /* A one byte register write publishes the TD after every store above */
        KeMemoryBarrier();
        CycleByte = reinterpret_cast<volatile UCHAR*>(&First->Dword[3]);
        WRITE_REGISTER_UCHAR(const_cast<PUCHAR>(CycleByte), (UCHAR)(*CycleByte ^ XHCI_TRB_CYCLE));
        KeMemoryBarrier();
 
        if (Self->m_DoorbellAfterFirstTd)
        {
            Self->m_DoorbellAfterFirstTd = FALSE;
            Ring->RingDoorbell();
        }
 
        Placed++;
    }
 
    return Commit ? Placed : Fit;
}

Referenced by WriteStage().

LeaveRingEmptyWait()

BOOLEAN XhciIsochRing::LeaveRingEmptyWait ( _In_ XhciTransferRing *  Ring )

staticprivate

Definition at line 916 of file isoxfer.cpp.

{
    if (Ring->ChangeMapState(FALSE, XhciMapState::AwaitingStarve, XhciMapState::Filling) !=
        XhciMapState::AwaitingStarve)
    {
        return FALSE;
    }
 
    WdfTimerStop(Ring->m_Type.Isoch.m_Watchdog, FALSE);
    return TRUE;
}

Referenced by BuildTdsFromQueue(), KickAfterStage(), and OnRingStarved().

NextTdSlot()

XhciIsochStage * XhciIsochRing::NextTdSlot ( _In_ XhciTransferRing *  Ring )

staticprivate

Definition at line 966 of file isoxfer.cpp.

{
    XhciIsochRing* Self = &Ring->m_Type.Isoch;
    XhciIsochTransfer* Transfer;
    XhciIsochStage* Stage = NULL;
    WDFREQUEST Request;
    BOOLEAN Idle;
    NTSTATUS Status;
    KIRQL OldIrql;
 
    KeAcquireSpinLock(&Ring->m_Lock, &OldIrql);
    Self->m_CurrentStage = NULL;
 
    if (Self->m_PendingStages >= XhciIsochMaxInFlight)
    {
        KeReleaseSpinLock(&Ring->m_Lock, OldIrql);
        return NULL;
    }
 
    if (!IsListEmpty(&Self->m_Pending))
    {
        Transfer = CONTAINING_RECORD(Self->m_Pending.Blink, XhciIsochTransfer, Link);
 
        if (Self->m_BufferRetries == XhciIsochBufferRetryLimit)
        {
            /* QUIRK: nothing counts the retries up, so this is never reached */
            DPRINT1("Isoch DCI %lu gave up waiting for segments\n", Ring->m_Endpoint->Dci());
 
            if (Transfer->TdsRetired != Transfer->TdsQueued)
            {
                SkipUnmapped(Transfer);
                KeReleaseSpinLock(&Ring->m_Lock, OldIrql);
                return NULL;
            }
 
            CompleteAfterUncancel(Ring, Transfer, XhciIsochNotSet, USBD_STATUS_INSUFFICIENT_RESOURCES, TRUE, FALSE);
        }
        else if (Transfer->PacketsQueued + Transfer->PacketsDropped < Transfer->PacketsInUrb)
        {
            FlagMissedPackets(Ring, Transfer);
 
            if (Transfer->Urb->IsoPacket[Transfer->PacketsInUrb - 1].Status != USBD_STATUS_ISO_NOT_ACCESSED_LATE)
            {
                Stage = AcquireStage(Ring, Transfer);
                Self->m_CurrentStage = Stage;
                KeReleaseSpinLock(&Ring->m_Lock, OldIrql);
                return Stage;
            }
 
            if (Transfer->TdsRetired != Transfer->TdsQueued)
            {
                SkipUnmapped(Transfer);
                KeReleaseSpinLock(&Ring->m_Lock, OldIrql);
                return NULL;
            }
 
            /* QUIRK: like the USB 2.0 stack, a URB that ran late reports success, even when canceled */
            CompleteAfterUncancel(Ring, Transfer, USBD_STATUS_SUCCESS, USBD_STATUS_SUCCESS, TRUE, TRUE);
        }
    }
 
    KeReleaseSpinLock(&Ring->m_Lock, OldIrql);
 
    for (;;)
    {
        Status = WdfIoQueueRetrieveNextRequest(Ring->m_Queue, &Request);
        if (!NT_SUCCESS(Status))
        {
            if (Self->m_PendingStages == 0)
                DPRINT("Isoch DCI %lu has nothing queued, the stream may starve\n", Ring->m_Endpoint->Dci());
            return NULL;
        }
 
        if (TakeRequest(Ring, Request, &Transfer))
            break;
    }
 
    KeAcquireSpinLock(&Ring->m_Lock, &OldIrql);
    Idle = (Self->m_PendingStages == 0);
    KeReleaseSpinLock(&Ring->m_Lock, OldIrql);
 
    /* Every TD on the ring reports to one event ring so its events stay in order */
    if (Idle)
        Ring->UpdateInterrupterTarget(XhciTransferRing::UrbProcessorIndex((PURB)Transfer->Urb));
 
    KeAcquireSpinLock(&Ring->m_Lock, &OldIrql);
    Stage = AcquireStage(Ring, Transfer);
    InsertTailList(&Self->m_Pending, &Transfer->Link);
    Self->m_CurrentStage = Stage;
    KeReleaseSpinLock(&Ring->m_Lock, OldIrql);
 
    return Stage;
}

Referenced by FillLoop().

OnClientPipeReset()

VOID XhciIsochRing::OnClientPipeReset ( _In_ XhciTransferRing *  Ring )

static

Definition at line 691 of file isoxfer.cpp.

{
    KIRQL OldIrql;
 
    KeAcquireSpinLock(&Ring->m_Lock, &OldIrql);
    Ring->m_Type.Isoch.m_FreshStream = TRUE;
    KeReleaseSpinLock(&Ring->m_Lock, OldIrql);
}

Referenced by XhciTransferRing::OnClientPipeReset().

OnRingStarved()

VOID XhciIsochRing::OnRingStarved ( _In_ XhciTransferRing *  Ring )

staticprivate

Definition at line 2924 of file isoxfer.cpp.

{
    XhciIsochRing* Self = &Ring->m_Type.Isoch;
    ULONG Now = Ring->m_Controller->GetFrameNumber(XhciIsochBiasNow);
    BOOLEAN Abort;
    KIRQL OldIrql;
 
    KeAcquireSpinLock(&Ring->m_Lock, &OldIrql);
    Self->m_RingEmptyEvents++;
    InterlockedAnd(&Self->m_Flags, ~XhciIsochEmptyExpected);
    if (Self->m_PendingStages == 0)
        InterlockedAnd(&Self->m_Flags, ~XhciIsochLastFrameValid);
 
    Abort = Ring->m_Controller->HasErrata(XhciErrata::IsochMissedServiceSilent) &&
            Self->m_PendingStages != 0 &&
            (Self->m_Flags & XhciIsochLastFrameValid) &&
            (LONG)(Now - Self->m_LastScheduledFrame) >= 0;
    KeReleaseSpinLock(&Ring->m_Lock, OldIrql);
 
    if (Abort)
    {
        /* The controller ran past TDs without a word. Their events may still sit
         * on a secondary event ring, so the endpoint stops and reclaims instead. */
        DPRINT1("Isoch DCI %lu: ring empty with stages unreported, reclaiming\n", Ring->m_Endpoint->Dci());
        Ring->m_Endpoint->OnTransferCanceled();
        return;
    }
 
    InterlockedExchange(&Self->m_RingEmptySeen, 1);
    if (LeaveRingEmptyWait(Ring))
        BuildTdsFromQueue(Ring);
}

Referenced by Ed0Event().

OnTransferEvent()

BOOLEAN XhciIsochRing::OnTransferEvent ( _In_ XhciTransferRing *  Ring, _In_ const XHCI_TRB *  Event  )

static

Definition at line 772 of file isoxfer.cpp.

{
    if (Event->Dword[3] & XHCI_TRANSFER_EVENT_ED)
        Ed1Event(Ring, Event);
    else
        Ed0Event(Ring, Event);
 
    return TRUE;
}

Referenced by XhciTransferRing::OnTransferEvent().

OwnsRequests()

static BOOLEAN XhciIsochRing::OwnsRequests ( _In_ const XhciTransferRing *  Ring )

staticprivate

Referenced by Cleanup(), and Disable().

PacketsValid()

BOOLEAN XhciIsochRing::PacketsValid ( _In_ const XhciIsochTransfer *  Transfer )

staticprivate

Definition at line 1170 of file isoxfer.cpp.

{
    const struct _URB_ISOCH_TRANSFER* Urb = Transfer->Urb;
    ULONG Index;
 
    if (Transfer->PacketsInUrb == 0)
        return FALSE;
 
    for (Index = 0; Index + 1 < Transfer->PacketsInUrb; Index++)
    {
        if (Urb->IsoPacket[Index].Offset > Urb->IsoPacket[Index + 1].Offset)
            return FALSE;
    }
 
    return Urb->IsoPacket[Transfer->PacketsInUrb - 1].Offset <= Transfer->BytesTotal;
}

Referenced by TakeRequest().

ParkOnSegments()

BOOLEAN XhciIsochRing::ParkOnSegments ( _In_ XhciTransferRing *  Ring )

staticprivate

Definition at line 948 of file isoxfer.cpp.

{
    XhciIsochRing* Self = &Ring->m_Type.Isoch;
 
    if (InterlockedCompareExchange(&Self->m_SegmentWait, XhciIsochWaitParked, XhciIsochWaitIdle) ==
        XhciIsochWaitIdle)
    {
        return TRUE;
    }
 
    InterlockedExchange(&Self->m_SegmentWait, XhciIsochWaitIdle);
    return FALSE;
}

Referenced by PrimeStage().

PrimeStage()

XhciIsochRing::Outcome XhciIsochRing::PrimeStage ( _In_ XhciTransferRing *  Ring, _Inout_ XhciIsochStage *  Stage  )

staticprivate

Definition at line 1393 of file isoxfer.cpp.

{
    XhciIsochRing* Self = &Ring->m_Type.Isoch;
    XhciIsochTransfer* Transfer = Stage->Transfer;
    ULONG Frame;
    ULONG Segments;
    BOOLEAN Gap = FALSE;
    NTSTATUS Status;
    KIRQL OldIrql;
 
    Stage->FirstPacket = Transfer->PacketsQueued + Transfer->PacketsDropped;
    Frame = Transfer->StartFrame + XhciIsochFrames(Stage->FirstPacket, Self->m_PacketsPerFrame);
 
    /* Without contiguous Frame IDs the controller must run dry before a jump (xHCI 4.11.2.5) */
    if (!Ring->m_Controller->m_Registers.ContiguousFrameId())
    {
        KeAcquireSpinLock(&Ring->m_Lock, &OldIrql);
        if ((Self->m_Flags & XhciIsochLastFrameValid) &&
            Frame != Self->m_LastScheduledFrame + 1 &&
            ((Self->m_Flags & XhciIsochEmptyExpected) || Self->m_PendingStages != 0))
        {
            FreeStage(Ring, Stage);
            Gap = TRUE;
        }
        KeReleaseSpinLock(&Ring->m_Lock, OldIrql);
 
        if (Gap)
        {
            DPRINT("Isoch DCI %lu: frame 0x%lx does not follow 0x%lx, waiting for the ring to drain\n",
                   Ring->m_Endpoint->Dci(), Frame, Self->m_LastScheduledFrame);
            return Outcome::Gap;
        }
    }
 
    if (!NT_SUCCESS(AcquireStageMdl(Ring, Stage)))
        return FailStage(Ring, Stage);
 
    if (!SizeStage(Ring, Stage))
        return FailStage(Ring, Stage);
 
    Segments = EstimateSegments(Ring, Stage);
    if (Segments == 0)
        return Outcome::Ready;
 
    Status = Ring->TakeSegments(Segments, &Stage->FreeSegments);
    if (Status == STATUS_PENDING)
    {
        KeAcquireSpinLock(&Ring->m_Lock, &OldIrql);
        FreeStage(Ring, Stage);
        KeReleaseSpinLock(&Ring->m_Lock, OldIrql);
 
        if (ParkOnSegments(Ring))
            return Outcome::WaitSegments;
 
        /* The segments already arrived; take the stage again from the top */
        return Outcome::Completed;
    }
 
    if (!NT_SUCCESS(Status))
    {
        DPRINT1("Isoch DCI %lu: %lu segments not available, 0x%08lx\n",
                Ring->m_Endpoint->Dci(), Segments, Status);
        return FailStage(Ring, Stage);
    }
 
    Self->m_BufferRetries = 0;
    return Outcome::Ready;
}

Referenced by FillLoop().

PutScatterGather()

static VOID XhciIsochRing::PutScatterGather ( _In_ XhciTransferRing *  Ring, _In_opt_ PSCATTER_GATHER_LIST  SgList  )

staticprivate

Referenced by CompleteTd(), FailStage(), ReleaseAllStages(), and WriteStage().

RecoverTransfers()

VOID XhciIsochRing::RecoverTransfers ( _In_ XhciTransferRing *  Ring )

static

Definition at line 734 of file isoxfer.cpp.

{
    XhciIsochRing* Self = &Ring->m_Type.Isoch;
    KIRQL OldIrql;
 
    KeAcquireSpinLock(&Ring->m_Lock, &OldIrql);
    InterlockedOr(&Self->m_Flags, XhciIsochReclaimAck);
    XhciIsochMoveList(&Self->m_Completion, &Self->m_Pending);
    KeReleaseSpinLock(&Ring->m_Lock, OldIrql);
 
    /* Only the DPC can tell that nothing is owned anymore */
    WdfDpcEnqueue(Ring->m_CompletionDpc);
}

Referenced by XhciTransferRing::RecoverTransfers().

ReleaseAllStages()

ULONG XhciIsochRing::ReleaseAllStages ( _In_ XhciTransferRing *  Ring, _Inout_ XhciIsochTransfer *  Transfer  )

staticprivate

Returns the lists and frees every stage of a transfer nobody else can reach anymore.

Definition at line 2269 of file isoxfer.cpp.

{
    XhciIsochRing* Self = &Ring->m_Type.Isoch;
    XhciIsochStage* Stage;
    PSCATTER_GATHER_LIST SgList;
    ULONG Freed = 0;
    KIRQL OldIrql;
 
    for (;;)
    {
        KeAcquireSpinLock(&Ring->m_Lock, &OldIrql);
        if (Transfer->StageCount == 0)
        {
            KeReleaseSpinLock(&Ring->m_Lock, OldIrql);
            break;
        }
 
        Stage = XhciIsochStageAt(Transfer, 0);
        SgList = Stage->SgList;
        Stage->SgList = NULL;
        KeReleaseSpinLock(&Ring->m_Lock, OldIrql);
 
        PutScatterGather(Ring, SgList);
 
        KeAcquireSpinLock(&Ring->m_Lock, &OldIrql);
        if (Stage->OnHardware)
        {
            Freed++;
            Transfer->TdsRetired++;
            if (Self->m_PendingStages != 0)
                Self->m_PendingStages--;
        }
        FreeStage(Ring, Stage);
        KeReleaseSpinLock(&Ring->m_Lock, OldIrql);
    }
 
    return Freed;
}

Referenced by CompleteBatch(), and FinishExpiredIsoch().

ReleaseParkedMapping()

VOID XhciIsochRing::ReleaseParkedMapping ( _In_ XhciTransferRing *  Ring )

static

Lets a mapping pass parked on a segment wait go, so a disable never waits on it.

Definition at line 495 of file isoxfer.cpp.

{
    XhciIsochRing* Self = &Ring->m_Type.Isoch;
 
    /* A pass parked on a segment wait only resumes from a PASSIVE work item; let it go */
    if (InterlockedCompareExchange(&Self->m_SegmentWait, XhciIsochWaitAbandoned, XhciIsochWaitParked) !=
        XhciIsochWaitParked)
    {
        return;
    }
 
    DPRINT1("Isoch DCI %lu disabled while waiting for segments\n", Ring->m_Endpoint->Dci());
 
    if (Ring->ChangeMapState(FALSE, XhciMapState::Halting, XhciMapState::Halted) != XhciMapState::Halting)
    {
        InterlockedCompareExchange(&Ring->m_MapState,
                                   static_cast<LONG>(XhciMapState::PausedFill),
                                   static_cast<LONG>(XhciMapState::Filling));
    }
}

Referenced by Disable().

ResumeRingFill()

VOID XhciIsochRing::ResumeRingFill ( _In_ XhciTransferRing *  Ring )

static

Definition at line 636 of file isoxfer.cpp.

{
    XhciIsochRing* Self = &Ring->m_Type.Isoch;
    XhciMapState Previous;
    KIRQL OldIrql;
 
    InterlockedExchange(&Self->m_Flags, 0);
 
    KeAcquireSpinLock(&Ring->m_Lock, &OldIrql);
    Self->m_PendingStages = 0;
    KeReleaseSpinLock(&Ring->m_Lock, OldIrql);
 
    Previous = Ring->ChangeMapState(TRUE, XhciMapState::Halted, XhciMapState::Filling);
    if (Previous == XhciMapState::Halted)
    {
        BuildTdsFromQueue(Ring);
        return;
    }
 
    DPRINT1("Isoch DCI %lu started mapping from map state %ld\n",
            Ring->m_Endpoint->Dci(), static_cast<LONG>(Previous));
}

Referenced by XhciTransferRing::ResumeRingFill().

RetireExpiredStages()

VOID XhciIsochRing::RetireExpiredStages ( _In_ XhciTransferRing *  Ring, _In_ XhciIsochTransfer *  Transfer, _In_ XhciIsochStage *  Stage, _In_ ULONG  Sequence  )

staticprivate

Closes the stages of Transfer older than Stage as missed.

Definition at line 2224 of file isoxfer.cpp.

{
    XhciIsochRing* Self = &Ring->m_Type.Isoch;
    XhciIsochStage* Oldest;
    BOOLEAN Progress;
    KIRQL OldIrql;
 
    for (;;)
    {
        KeAcquireSpinLock(&Ring->m_Lock, &OldIrql);
        if (!XhciIsochIsActive(&Self->m_Pending, Transfer, Sequence) ||
            !Stage->InUse ||
            Transfer->StageCount == 0)
        {
            KeReleaseSpinLock(&Ring->m_Lock, OldIrql);
            return;
        }
 
        Oldest = XhciIsochStageAt(Transfer, 0);
        if (Oldest == Stage)
        {
            KeReleaseSpinLock(&Ring->m_Lock, OldIrql);
            return;
        }
 
        Oldest->NextPacket = Oldest->LastPacket + 1;
        KeReleaseSpinLock(&Ring->m_Lock, OldIrql);
 
        CompleteTd(Ring, Transfer, Oldest, Sequence, static_cast<ULONG>(XhciCompletionCode::MissedService), 0, TRUE);
 
        KeAcquireSpinLock(&Ring->m_Lock, &OldIrql);
        Progress = !Oldest->InUse || XhciIsochStageAt(Transfer, 0) != Oldest;
        KeReleaseSpinLock(&Ring->m_Lock, OldIrql);
 
        if (!Progress)
            return;
    }
}

Referenced by Ed0Event(), and Ed1Event().

ScheduleStartFrame()

NTSTATUS XhciIsochRing::ScheduleStartFrame ( _In_ XhciTransferRing *  Ring, _Inout_ XhciIsochTransfer *  Transfer  )

staticprivate

Definition at line 1189 of file isoxfer.cpp.

{
    XhciIsochRing* Self = &Ring->m_Type.Isoch;
    struct _URB_ISOCH_TRANSFER* Urb = Transfer->Urb;
    ULONG Threshold;
    ULONG StartFrame;
    ULONG Now = 0;
    ULONG Index;
    LONG Distance;
    KIRQL OldIrql;
 
    Threshold = (Ring->m_Device->Speed() >= UsbHighSpeed) ? XhciIsochRestartFast : XhciIsochRestartSlow;
 
    if (Urb->TransferFlags & USBD_START_ISO_TRANSFER_ASAP)
        Now = Ring->m_Controller->GetFrameNumber(XhciIsochBiasNow);
 
    KeAcquireSpinLock(&Ring->m_Lock, &OldIrql);
 
    if (Urb->TransferFlags & USBD_START_ISO_TRANSFER_ASAP)
    {
        StartFrame = Self->m_NextStartFrame;
 
        /* QUIRK: unsigned and without wrap handling, and the threshold is really a packet count */
        if (Self->m_FreshStream || (Now > StartFrame && Now - StartFrame > Threshold))
        {
            StartFrame = Now + XhciIsochAsapDelay;
            Self->m_DoorbellAfterFirstTd = TRUE;
        }
 
        Urb->StartFrame = StartFrame;
    }
    else
    {
        StartFrame = Urb->StartFrame;
    }
 
    Self->m_NextStartFrame = StartFrame + XhciIsochFrames(Transfer->PacketsInUrb, Self->m_PacketsPerFrame);
    KeReleaseSpinLock(&Ring->m_Lock, OldIrql);
 
    Transfer->StartFrame = StartFrame;
    Urb->TransferBufferLength = 0;
    Urb->ErrorCount = 0;
    for (Index = 0; Index < Transfer->PacketsInUrb; Index++)
        Urb->IsoPacket[Index].Status = XhciIsochNotSet;
 
    Distance = (LONG)(StartFrame - Ring->m_Controller->GetFrameNumber(XhciIsochBiasLate));
    if (Distance > XhciIsochFrameWindow || Distance < -XhciIsochFrameWindow)
    {
        DPRINT1("Isoch DCI %lu: start frame 0x%lx is %ld frames away\n",
                Ring->m_Endpoint->Dci(), StartFrame, Distance);
        return STATUS_UNSUCCESSFUL;
    }
 
    KeAcquireSpinLock(&Ring->m_Lock, &OldIrql);
    Self->m_FreshStream = FALSE;
    KeReleaseSpinLock(&Ring->m_Lock, OldIrql);
 
    return STATUS_SUCCESS;
}

Referenced by TakeRequest().

SegmentsArrived()

VOID XhciIsochRing::SegmentsArrived ( _In_ XhciTransferRing *  Ring, _In_ NTSTATUS  Status  )

static

Common buffer callback for asynchronous segment growth.

Definition at line 785 of file isoxfer.cpp.

{
    XhciIsochRing* Self = &Ring->m_Type.Isoch;
 
    if (!NT_SUCCESS(Status))
        DPRINT1("Isoch DCI %lu segment growth failed, 0x%08lx\n", Ring->m_Endpoint->Dci(), Status);
 
    Self->m_BufferRetries = 0;
 
    for (;;)
    {
        /* QUIRK: the pass runs even when a stop came in meanwhile, so one more stage can go out */
        if (InterlockedCompareExchange(&Self->m_SegmentWait, XhciIsochWaitIdle, XhciIsochWaitParked) ==
            XhciIsochWaitParked)
        {
            BuildTdsFromQueue(Ring);
            return;
        }
 
        if (InterlockedCompareExchange(&Self->m_SegmentWait, XhciIsochWaitIdle, XhciIsochWaitAbandoned) ==
            XhciIsochWaitAbandoned)
        {
            return;
        }
 
        /* The pass has not left yet; it sees this and retries the stage itself */
        if (InterlockedCompareExchange(&Self->m_SegmentWait, XhciIsochWaitArrived, XhciIsochWaitIdle) ==
            XhciIsochWaitIdle)
        {
            return;
        }
    }
}

SizeStage()

BOOLEAN XhciIsochRing::SizeStage ( _In_ XhciTransferRing *  Ring, _Inout_ XhciIsochStage *  Stage  )

staticprivate

Picks the last packet and byte count of a stage; FALSE when not even one frame fits.

Definition at line 1513 of file isoxfer.cpp.

{
    XhciIsochTransfer* Transfer = Stage->Transfer;
    ULONG PerFrame = Ring->m_Type.Isoch.m_PacketsPerFrame;
    ULONG Used = Transfer->BytesQueued + Transfer->BytesDropped;
    ULONG Remaining = (Transfer->BytesTotal > Used) ? Transfer->BytesTotal - Used : 0;
    ULONG Limit = Ring->m_MaxStageSize;
    ULONG Registers = 0;
    PDMA_ADAPTER Adapter;
    DMA_TRANSFER_INFO Info;
    ULONG Packet;
    ULONG Total = 0;
    PMDL Mdl;
    ULONG Left;
 
    Adapter = Ring->m_Controller->m_Buffers.DmaAdapter();
    if (Adapter->Version >= DEVICE_DESCRIPTION_VERSION3 &&
        Adapter->DmaOperations->Size > FIELD_OFFSET(DMA_OPERATIONS, GetDmaTransferInfo) &&
        Adapter->DmaOperations->GetDmaTransferInfo != NULL)
    {
        RtlZeroMemory(&Info, sizeof(Info));
        Info.Version = DMA_TRANSFER_INFO_VERSION1;
        if (NT_SUCCESS(Adapter->DmaOperations->GetDmaTransferInfo(Adapter,
                                                                  Stage->Mdl,
                                                                  0,
                                                                  Remaining,
                                                                  !XhciIsochEndpointIn(Ring),
                                                                  &Info)))
        {
            Registers = Info.V1.MapRegisterCount;
        }
    }
 
    if (Registers == 0)
    {
        /* Without the HAL query count the pages the chain spans */
        for (Mdl = Stage->Mdl, Left = Remaining; Mdl != NULL && Left != 0; Mdl = Mdl->Next)
        {
            ULONG Chunk = min(Left, MmGetMdlByteCount(Mdl));
 
            Registers += ADDRESS_AND_SIZE_TO_SPAN_PAGES(MmGetMdlVirtualAddress(Mdl), Chunk);
            Left -= Chunk;
        }
    }
 
    if (Registers > Ring->m_MapRegisterCount && Ring->m_MapRegisterCount > 1)
        Limit = min(Limit, (Ring->m_MapRegisterCount - 1) * PAGE_SIZE);
 
    if (Remaining <= Limit)
    {
        Stage->LastPacket = Transfer->PacketsInUrb - 1;
        Stage->Size = Remaining;
        return TRUE;
    }
 
    /* Whole frames only */
    Packet = Stage->FirstPacket;
    while (Packet < Transfer->PacketsInUrb)
    {
        ULONG FrameBytes = 0;
        ULONG Count;
 
        for (Count = 0; Count < PerFrame && Packet + Count < Transfer->PacketsInUrb; Count++)
            FrameBytes += XhciIsochPacketLength(Transfer, Packet + Count);
 
        if (Total + FrameBytes > Limit)
            break;
 
        Total += FrameBytes;
        Packet += Count;
    }
 
    /* Fail instead of building an empty stage when one frame exceeds the limit */
    if (Packet == Stage->FirstPacket)
    {
        DPRINT1("Isoch DCI %lu: one frame needs more than %lu bytes of mapping\n",
                Ring->m_Endpoint->Dci(), Limit);
        return FALSE;
    }
 
    Stage->LastPacket = Packet - 1;
    Stage->Size = Total;
    return TRUE;
}

Referenced by PrimeStage().

SkipUnmapped()

VOID XhciIsochRing::SkipUnmapped ( _Inout_ XhciIsochTransfer *  Transfer )

staticprivate

Definition at line 1282 of file isoxfer.cpp.

{
    Transfer->PacketsDropped = Transfer->PacketsInUrb - Transfer->PacketsQueued;
    Transfer->BytesDropped = (Transfer->BytesTotal > Transfer->BytesQueued) ?
                             Transfer->BytesTotal - Transfer->BytesQueued : 0;
}

Referenced by FailStage(), LayoutStage(), and NextTdSlot().

SuspendRingFill()

VOID XhciIsochRing::SuspendRingFill ( _In_ XhciTransferRing *  Ring )

static

Definition at line 661 of file isoxfer.cpp.

{
    XhciMapState Previous;
 
    Previous = Ring->ChangeMapState(TRUE, XhciMapState::Halted, XhciMapState::Halting);
 
    switch (Previous)
    {
        case XhciMapState::AwaitingStarve:
            WdfTimerStop(Ring->m_Type.Isoch.m_Watchdog, FALSE);
            Ring->ChangeMapState(FALSE, XhciMapState::Halting, XhciMapState::Halted);
            break;
 
        case XhciMapState::PausedFill:
            Ring->ChangeMapState(FALSE, XhciMapState::Halting, XhciMapState::Halted);
            break;
 
        case XhciMapState::Filling:
            /* The context that owns mapping acknowledges the stop */
            break;
 
        default:
            DPRINT1("Isoch DCI %lu stopped mapping from map state %ld\n",
                    Ring->m_Endpoint->Dci(), static_cast<LONG>(Previous));
            break;
    }
}

Referenced by XhciTransferRing::SuspendRingFill().

TakeRequest()

BOOLEAN XhciIsochRing::TakeRequest ( _In_ XhciTransferRing *  Ring, _In_ WDFREQUEST  Request, _Out_ XhciIsochTransfer **  Taken  )

staticprivate

Sets up a request taken from the queue; FALSE when it was completed instead.

Definition at line 1063 of file isoxfer.cpp.

{
    XhciIsochRing* Self = &Ring->m_Type.Isoch;
    XhciRequestData* Data = XhciGetRequestData(Request);
    struct _URB_ISOCH_TRANSFER* Urb;
    XhciIsochTransfer* Transfer;
    NTSTATUS Status;
 
    *Taken = NULL;
 
    if (Data == NULL)
    {
        DPRINT1("Isoch request %p has no driver context\n", Request);
        WdfRequestComplete(Request, STATUS_INVALID_DEVICE_STATE);
        return FALSE;
    }
 
    Urb = XhciIsochUrbOf(Request);
    Transfer = &Data->Isoch;
 
    /* UCX reuses requests, so the record starts from zero every time */
    RtlZeroMemory(Transfer, sizeof(*Transfer));
    XhciClearListEntry(&Transfer->Link);
    Transfer->Initialized = XhciIsochTransferTag;
    Transfer->Sequence = ++Self->m_NextSequence;
    Transfer->Request = Request;
    Transfer->Urb = Urb;
    Transfer->Ring = Ring;
    Transfer->CancelState = XhciIsochCancel::Idle;
    Transfer->Status = STATUS_PENDING;
    Transfer->BytesTotal = Urb->TransferBufferLength;
    Transfer->PacketsInUrb = (Urb->NumberOfPackets <= XhciIsochMaxPackets) ? Urb->NumberOfPackets : 0;
 
    if (Urb->TransferBufferMDL != NULL)
    {
        Transfer->Mdl = Urb->TransferBufferMDL;
    }
    else if (Transfer->BytesTotal != 0)
    {
        if (Urb->TransferBuffer != NULL)
            Transfer->Mdl = IoAllocateMdl(Urb->TransferBuffer, Transfer->BytesTotal, FALSE, FALSE, NULL);
 
        if (Transfer->Mdl == NULL)
        {
            DPRINT1("Isoch DCI %lu request %p: no MDL for a %lu byte buffer\n",
                    Ring->m_Endpoint->Dci(), Request, Transfer->BytesTotal);
 
            /* QUIRK: the caller's packet statuses and error count are kept */
            CompletePlain(Ring, Transfer, XhciIsochNotSet, USBD_STATUS_INSUFFICIENT_RESOURCES, FALSE);
            return FALSE;
        }
 
        Transfer->OwnMdl = TRUE;
        MmBuildMdlForNonPagedPool(Transfer->Mdl);
    }
 
    /* Packet offsets must lie inside the buffer before the MDL chain is walked */
    if (!PacketsValid(Transfer))
    {
        DPRINT1("Isoch DCI %lu request %p: %lu packets with bad offsets for %lu bytes\n",
                Ring->m_Endpoint->Dci(), Request, Urb->NumberOfPackets, Transfer->BytesTotal);
        Urb->ErrorCount = 0;
        CompletePlain(Ring, Transfer, USBD_STATUS_ISO_NOT_ACCESSED_BY_HW, USBD_STATUS_INVALID_PARAMETER, FALSE);
        return FALSE;
    }
 
    Status = ScheduleStartFrame(Ring, Transfer);
    if (!NT_SUCCESS(Status))
    {
        CompletePlain(Ring, Transfer, USBD_STATUS_ISO_NOT_ACCESSED_LATE, USBD_STATUS_BAD_START_FRAME, FALSE);
        return FALSE;
    }
 
    FlagMissedPackets(Ring, Transfer);
    if (Urb->IsoPacket[Transfer->PacketsInUrb - 1].Status == USBD_STATUS_ISO_NOT_ACCESSED_LATE)
    {
        DPRINT("Isoch DCI %lu request %p is late in full\n", Ring->m_Endpoint->Dci(), Request);
        CompletePlain(Ring, Transfer, USBD_STATUS_SUCCESS, USBD_STATUS_SUCCESS, TRUE);
        return FALSE;
    }
 
    if (Transfer->BytesTotal == 0)
    {
        DPRINT("Isoch DCI %lu request %p has no data\n", Ring->m_Endpoint->Dci(), Request);
        CompletePlain(Ring, Transfer, XhciIsochNotSet, XhciIsochNotSet, TRUE);
        return FALSE;
    }
 
    Transfer->CancelState = XhciIsochCancel::Armed;
    Status = WdfRequestMarkCancelableEx(Request, EvtRequestCancel);
    if (!NT_SUCCESS(Status))
    {
        DPRINT1("Isoch DCI %lu request %p canceled before mapping, 0x%08lx\n",
                Ring->m_Endpoint->Dci(), Request, Status);
        Transfer->CancelState = XhciIsochCancel::Done;
        CompletePlain(Ring, Transfer, XhciIsochNotSet, USBD_STATUS_CANCELED, FALSE);
        return FALSE;
    }
 
    *Taken = Transfer;
    return TRUE;
}

Referenced by NextTdSlot().

TransferDone()

BOOLEAN XhciIsochRing::TransferDone ( _In_ const XhciIsochTransfer *  Transfer )

staticprivate

Every stage is back and every packet is mapped or skipped.

Definition at line 1382 of file isoxfer.cpp.

{
    return Transfer->StageCount == 0 &&
           Transfer->TdsRetired == Transfer->TdsQueued &&
           Transfer->PacketsQueued + Transfer->PacketsDropped >= Transfer->PacketsInUrb;
}

WriteStage()

VOID XhciIsochRing::WriteStage ( _In_ XhciTransferRing *  Ring, _Inout_ XhciIsochStage *  Stage  )

staticprivate

Writes a mapped stage into the ring, or drops it when it is late or nothing fits.

Definition at line 1747 of file isoxfer.cpp.

{
    XhciIsochRing* Self = &Ring->m_Type.Isoch;
    XhciIsochTransfer* Transfer = Stage->Transfer;
    PSCATTER_GATHER_LIST SgList;
    ULONG Frame;
    ULONG Now;
    ULONG Fit;
    ULONG Packet;
    ULONG Size = 0;
    KIRQL OldIrql;
 
    if (Ring->m_Controller->HasErrata(XhciErrata::IsochSkipPastFrames))
    {
        Frame = Transfer->StartFrame + Stage->FirstPacket / Self->m_PacketsPerFrame;
        Now = Ring->m_Controller->GetFrameNumber(XhciIsochBiasLate);
 
        if ((LONG)(Now - Frame) >= 0)
        {
            DPRINT1("Isoch DCI %lu: stage for frame 0x%lx is late at 0x%lx\n",
                    Ring->m_Endpoint->Dci(), Frame, Now);
 
            SgList = Stage->SgList;
            Stage->SgList = NULL;
            PutScatterGather(Ring, SgList);
 
            KeAcquireSpinLock(&Ring->m_Lock, &OldIrql);
            FreeStage(Ring, Stage);
            InterlockedOr(&Self->m_Flags, XhciIsochEmptyExpected);
            KeReleaseSpinLock(&Ring->m_Lock, OldIrql);
 
            /* The controller runs off the end and reports the ring empty */
            Ring->RingDoorbell();
            InterlockedExchange(&Self->m_LateStage, 1);
            return;
        }
    }
 
    Fit = LayoutStage(Ring, Stage, FALSE);
    if (Fit == 0)
    {
        DPRINT1("Isoch DCI %lu: packet %lu does not fit the reserved segments\n",
                Ring->m_Endpoint->Dci(), Stage->FirstPacket);
        (VOID)FailStage(Ring, Stage);
        return;
    }
 
    /* End the stage at the last TD that fits in the segment */
    if (Stage->FirstPacket + Fit - 1 != Stage->LastPacket)
    {
        DPRINT1("Isoch DCI %lu: stage cut after %lu of %lu packets\n",
                Ring->m_Endpoint->Dci(), Fit, Stage->LastPacket - Stage->FirstPacket + 1);
 
        for (Packet = Stage->FirstPacket; Packet < Stage->FirstPacket + Fit; Packet++)
            Size += XhciIsochPacketLength(Transfer, Packet);
 
        KeAcquireSpinLock(&Ring->m_Lock, &OldIrql);
        Stage->LastPacket = Stage->FirstPacket + Fit - 1;
        Stage->Size = Size;
        Stage->Truncated = TRUE;
        KeReleaseSpinLock(&Ring->m_Lock, OldIrql);
    }
 
    (VOID)LayoutStage(Ring, Stage, TRUE);
 
    /* The stage may be gone already; only the ring is used from here */
    Ring->RingDoorbell();
}

Member Data Documentation

DmaCallback

VOID NTAPI XhciIsochRing::DmaCallback

staticprivate

Definition at line 155 of file isoxfer.h.

Referenced by EncodeStage().

EvtCompletionDpc

VOID NTAPI XhciIsochRing::EvtCompletionDpc

static

Definition at line 103 of file isoxfer.h.

Referenced by _IRQL_requires_().

EvtIoCanceledOnQueue

VOID NTAPI XhciIsochRing::EvtIoCanceledOnQueue

static

Definition at line 102 of file isoxfer.h.

Referenced by _IRQL_requires_().

EvtRequestCancel

VOID NTAPI XhciIsochRing::EvtRequestCancel

staticprivate

Definition at line 153 of file isoxfer.h.

EvtWatchdog

VOID NTAPI XhciIsochRing::EvtWatchdog

staticprivate

Definition at line 154 of file isoxfer.h.

Referenced by Initialize().

m_BufferRetries

ULONG XhciIsochRing::m_BufferRetries

private

Definition at line 342 of file isoxfer.h.

Referenced by NextTdSlot(), PrimeStage(), and SegmentsArrived().

m_Bytes

ULONG64 XhciIsochRing::m_Bytes

private

Definition at line 359 of file isoxfer.h.

Referenced by Finalize().

m_Completion

LIST_ENTRY XhciIsochRing::m_Completion

private

Definition at line 353 of file isoxfer.h.

Referenced by AllowReclaimOnCancel(), Initialize(), and RecoverTransfers().

m_CurrentStage

XhciIsochStage* XhciIsochRing::m_CurrentStage

private

Definition at line 350 of file isoxfer.h.

Referenced by NextTdSlot().

m_DoorbellAfterFirstTd

BOOLEAN XhciIsochRing::m_DoorbellAfterFirstTd

private

Mapping context only

Definition at line 335 of file isoxfer.h.

Referenced by LayoutStage(), and ScheduleStartFrame().

m_DpcRunning

BOOLEAN XhciIsochRing::m_DpcRunning

private

Definition at line 336 of file isoxfer.h.

m_FailedTransfers

ULONG XhciIsochRing::m_FailedTransfers

private

Definition at line 358 of file isoxfer.h.

Referenced by Finalize().

m_FillAgain

volatile LONG XhciIsochRing::m_FillAgain

private

Interlocked DMA callback handoff

Definition at line 347 of file isoxfer.h.

Referenced by EncodeStage().

m_FillWanted

volatile LONG XhciIsochRing::m_FillWanted

private

Interlocked

Definition at line 345 of file isoxfer.h.

Referenced by BuildTdsFromQueue(), and FillLoop().

m_Flags

volatile LONG XhciIsochRing::m_Flags

private

Interlocked

Definition at line 344 of file isoxfer.h.

Referenced by AllowReclaimOnCancel(), Ed0Event(), Ed1Event(), FinishLostStages(), LayoutStage(), OnRingStarved(), PrimeStage(), RecoverTransfers(), ResumeRingFill(), and WriteStage().

m_FreshStream

BOOLEAN XhciIsochRing::m_FreshStream

private

Definition at line 334 of file isoxfer.h.

Referenced by ScheduleStartFrame().

m_LastScheduledFrame

ULONG XhciIsochRing::m_LastScheduledFrame

private

Definition at line 338 of file isoxfer.h.

Referenced by FinishLostStages(), LayoutStage(), OnRingStarved(), and PrimeStage().

m_LateStage

volatile LONG XhciIsochRing::m_LateStage

private

Interlocked

Definition at line 348 of file isoxfer.h.

Referenced by EncodeStage(), and WriteStage().

m_MissedServiceErrors

ULONG XhciIsochRing::m_MissedServiceErrors

private

Definition at line 362 of file isoxfer.h.

Referenced by Ed0Event().

m_NextSequence

ULONG XhciIsochRing::m_NextSequence

private

Definition at line 343 of file isoxfer.h.

Referenced by TakeRequest().

m_NextStartFrame

ULONG XhciIsochRing::m_NextStartFrame

private

Definition at line 337 of file isoxfer.h.

Referenced by ScheduleStartFrame().

m_PacketsPerFrame

ULONG XhciIsochRing::m_PacketsPerFrame

private

Definition at line 339 of file isoxfer.h.

Referenced by Initialize(), LayoutStage(), PrimeStage(), ScheduleStartFrame(), and WriteStage().

m_Pending

LIST_ENTRY XhciIsochRing::m_Pending

private

Definition at line 351 of file isoxfer.h.

Referenced by AllowReclaimOnCancel(), CompleteTd(), Ed1Event(), FinishExpiredIsoch(), FinishLostStages(), Initialize(), NextTdSlot(), RecoverTransfers(), and RetireExpiredStages().

m_PendingStages

ULONG XhciIsochRing::m_PendingStages

private

Definition at line 340 of file isoxfer.h.

Referenced by CompleteTd(), FinishLostStages(), LayoutStage(), NextTdSlot(), OnRingStarved(), PrimeStage(), ReleaseAllStages(), and ResumeRingFill().

m_RingEmptyEvents

ULONG XhciIsochRing::m_RingEmptyEvents

private

Definition at line 363 of file isoxfer.h.

Referenced by OnRingStarved().

m_RingEmptySeen

volatile LONG XhciIsochRing::m_RingEmptySeen

private

Interlocked

Definition at line 346 of file isoxfer.h.

Referenced by BuildTdsFromQueue(), FillLoop(), and OnRingStarved().

m_SegmentWait

volatile LONG XhciIsochRing::m_SegmentWait

private

Interlocked segment callback handoff

Definition at line 349 of file isoxfer.h.

Referenced by ParkOnSegments(), ReleaseParkedMapping(), and SegmentsArrived().

m_TdFailures

ULONG XhciIsochRing::m_TdFailures

private

Definition at line 361 of file isoxfer.h.

Referenced by Finalize().

m_Tds

ULONG XhciIsochRing::m_Tds

private

Definition at line 360 of file isoxfer.h.

Referenced by Finalize().

m_Transfers

ULONG XhciIsochRing::m_Transfers

private

Definition at line 357 of file isoxfer.h.

Referenced by Finalize().

m_TrbsPerTd

ULONG XhciIsochRing::m_TrbsPerTd

private

Definition at line 341 of file isoxfer.h.

Referenced by EstimateSegments(), and LayoutStage().

m_WaitingForCancel

LIST_ENTRY XhciIsochRing::m_WaitingForCancel

private

Definition at line 352 of file isoxfer.h.

Referenced by Initialize().

m_Watchdog

WDFTIMER XhciIsochRing::m_Watchdog

private

Definition at line 354 of file isoxfer.h.

Referenced by Disable(), and Initialize().

ReadyNotification

VOID NTAPI XhciIsochRing::ReadyNotification

static

Definition at line 101 of file isoxfer.h.

Referenced by _IRQL_requires_().

---

The documentation for this class was generated from the following files:

• drivers/usb/usbxhci/isoxfer.h
• drivers/usb/usbxhci/isoxfer.cpp