XhciBulkRing Class Reference

#include <bulkio.h>

Collaboration diagram for XhciBulkRing:

Static Public Member Functions
static NTSTATUS	Initialize (_In_ XhciTransferRing *Ring)
static VOID	Enable (_In_ XhciTransferRing *Ring)
static VOID	Disable (_In_ XhciTransferRing *Ring)
static VOID	ReleaseParkedMapping (_In_ XhciTransferRing *Ring)
static NTSTATUS	EnableForwardProgress (_In_ XhciTransferRing *Ring, _In_ ULONG MaxTransferSize)
static VOID	Cleanup (_In_ XhciTransferRing *Ring)
static VOID	ResumeRingFill (_In_ XhciTransferRing *Ring)
static VOID	SuspendRingFill (_In_ XhciTransferRing *Ring)
static VOID	OnPipeHalted (_In_ XhciTransferRing *Ring)
static VOID	StoppedEventReceived (_In_ XhciTransferRing *Ring)
static VOID	AllowReclaimOnCancel (_In_ XhciTransferRing *Ring)
static VOID	ConsumePendingEvents (_In_ XhciTransferRing *Ring)
static VOID	RecoverTransfers (_In_ XhciTransferRing *Ring)
static BOOLEAN	DoorbellRungSinceFill (_In_ const XhciTransferRing *Ring)
static BOOLEAN	HasQueuedWork (_In_ const XhciTransferRing *Ring)
static BOOLEAN	IsLikelyDuplicate (_In_ const XhciTransferRing *Ring, _In_ const XHCI_TRB *Event)
static BOOLEAN	OnTransferEvent (_In_opt_ XhciTransferRing *Ring, _In_ XhciController *Controller, _In_ const XHCI_TRB *Event)
static VOID	SegmentsArrived (_In_ XhciTransferRing *Ring, _In_ NTSTATUS Status)

Static Public Attributes
static EVT_WDF_IO_QUEUE_STATE	ReadyNotification
static EVT_WDF_IO_QUEUE_IO_CANCELED_ON_QUEUE	EvtIoCanceledOnQueue
static EVT_WDF_DPC	EvtCompletionDpc

Private Types
enum class	Outcome : ULONG {   Ready , Completed , WaitSegments , WaitDma ,   Failed }

Static Private Member Functions
static VOID	BuildTdsFromQueue (_In_ XhciTransferRing *Ring)
static VOID	FillLoop (_In_ XhciTransferRing *Ring)
static VOID	KickFill (_In_ XhciTransferRing *Ring)
static XhciBulkStage *	NextTdSlot (_In_ XhciTransferRing *Ring)
static XhciBulkStage *	AcquireStage (_In_ XhciTransferRing *Ring, _Inout_ XhciBulkTransfer *Transfer)
static VOID	ReleaseStage (_In_ XhciTransferRing *Ring, _Inout_ XhciBulkStage *Stage)
static VOID	PutScatterGather (_In_ XhciTransferRing *Ring, _Inout_ XhciBulkStage *Stage)
static VOID	InitTransfer (_In_ XhciTransferRing *Ring, _Inout_ XhciBulkTransfer *Transfer, _In_ WDFREQUEST Request)
static VOID	ChooseMechanism (_In_ XhciTransferRing *Ring, _Inout_ XhciBulkTransfer *Transfer)
static NTSTATUS	ArrangeBuffer (_In_ XhciTransferRing *Ring, _Inout_ XhciBulkTransfer *Transfer)
static VOID	DropMdl (_In_ XhciTransferRing *Ring, _Inout_ XhciBulkTransfer *Transfer)
static VOID	FreeTransferResources (_In_ XhciTransferRing *Ring, _Inout_ XhciBulkTransfer *Transfer)
static Outcome	PrimeStage (_In_ XhciTransferRing *Ring, _Inout_ XhciBulkStage *Stage)
static NTSTATUS	AcquireStageMdl (_In_ XhciTransferRing *Ring, _Inout_ XhciBulkStage *Stage)
static VOID	SizeStage (_In_ XhciTransferRing *Ring, _Inout_ XhciBulkStage *Stage)
static VOID	EstimateTrbs (_In_ XhciTransferRing *Ring, _Inout_ XhciBulkStage *Stage)
static BOOLEAN	EstimateSegments (_In_ XhciTransferRing *Ring, _In_ const XhciBulkStage *Stage, _Out_ PULONG Segments)
static BOOLEAN	ParkOnSegments (_In_ XhciTransferRing *Ring)
static Outcome	FailStage (_In_ XhciTransferRing *Ring, _Inout_ XhciBulkStage *Stage)
static Outcome	EncodeStage (_In_ XhciTransferRing *Ring, _Inout_ XhciBulkStage *Stage)
static BOOLEAN	BuildTd (_In_ XhciTransferRing *Ring, _Inout_ XhciBulkStage *Stage)
static BOOLEAN	InsertLink (_In_ XhciTransferRing *Ring, _Inout_ XhciBulkStage *Stage, _In_ BOOLEAN FirstOfTd)
static VOID	UndoTd (_In_ XhciTransferRing *Ring, _Inout_ XhciBulkStage *Stage)
static VOID	CompleteAfterUncancel (_In_ XhciTransferRing *Ring, _Inout_ XhciBulkTransfer *Transfer, _In_ USBD_STATUS Default, _In_ BOOLEAN FinishCanceled)
static VOID	Complete (_In_ XhciTransferRing *Ring, _Inout_ XhciBulkTransfer *Transfer, _In_ USBD_STATUS Default)
static VOID	CompleteBatch (_In_ XhciTransferRing *Ring, _Inout_ PLIST_ENTRY Batch)
static VOID	QueueCompletionDpc (_In_ XhciTransferRing *Ring)
static BOOLEAN	OwnsRequests (_In_ const XhciTransferRing *Ring)
static BOOLEAN	Ed1Event (_In_ XhciTransferRing *Ring, _In_ const XHCI_TRB *Event)
static BOOLEAN	Ed0Event (_In_ XhciTransferRing *Ring, _In_ const XHCI_TRB *Event)
static BOOLEAN	IsLiveStage (_In_ const XhciTransferRing *Ring, _In_ const XhciBulkStage *Stage, _In_ ULONG64 EventData)
static BOOLEAN	LookupStage (_In_ const XhciTransferRing *Ring, _In_ ULONG64 Address, _Outptr_result_maybenull_ XhciBulkStage **Stage, _Out_ PULONG Skipped)
static VOID	HaltedEvent (_In_ XhciTransferRing *Ring, _Inout_ XhciBulkStage *Stage, _In_ ULONG Code, _In_ ULONG Bytes)
static VOID	StoppedEvent (_In_ XhciTransferRing *Ring, _Inout_opt_ XhciBulkStage *Stage, _In_ ULONG Code, _In_ ULONG Bytes, _In_ ULONG Skipped)
static BOOLEAN	LikelyDuplicate (_In_ const XhciTransferRing *Ring, _In_ ULONG64 Address, _Out_ PBOOLEAN PointsToNoOp)
static VOID	Ed0Mismatch (_In_ XhciTransferRing *Ring, _In_ const XHCI_TRB *Event)
static VOID	StreamStopOrHalt (_In_ XhciTransferRing *Ring)
static VOID	StreamStopOrHaltOnce (_In_ XhciTransferRing *Ring, _In_ ULONG Flag)

Private Attributes
BOOLEAN	m_ImmediateData
BOOLEAN	m_DpcRunning
ULONG	m_Flags
ULONG	m_MaxPendingStages
ULONG	m_PendingStages
ULONG	m_OutstandingEvents
ULONG	m_SegmentWaits
volatile LONG	m_FillAgain
volatile LONG	m_FillWanted
volatile LONG	m_SegmentWait
volatile LONG	m_ForwardProgressBusy
LIST_ENTRY	m_Pending
LIST_ENTRY	m_WaitingForCancel
LIST_ENTRY	m_Completion
XhciBulkStage	m_Stages [StageSlots]

Static Private Attributes
static const ULONG	StageSlots = 6
static EVT_WDF_REQUEST_CANCEL	EvtRequestCancel
static DRIVER_LIST_CONTROL	DmaCallback

Detailed Description

Bulk and interrupt ring state and the entry points the transfer ring calls for it.

Definition at line 87 of file bulkio.h.

Member Enumeration Documentation

Outcome

enum class XhciBulkRing::Outcome : ULONG

strongprivate

Enumerator
Ready
Completed
WaitSegments
WaitDma
Failed

Definition at line 166 of file bulkio.h.

    {
        Ready,
        Completed,
        WaitSegments,
        WaitDma,
        Failed
    };

Member Function Documentation

AcquireStage()

static XhciBulkStage * XhciBulkRing::AcquireStage ( _In_ XhciTransferRing *  Ring, _Inout_ XhciBulkTransfer *  Transfer  )

staticprivate

Referenced by NextTdSlot().

AcquireStageMdl()

NTSTATUS XhciBulkRing::AcquireStageMdl ( _In_ XhciTransferRing *  Ring, _Inout_ XhciBulkStage *  Stage  )

staticprivate

Definition at line 1397 of file bulkio.cpp.

{
    XhciBulkTransfer* Transfer = Stage->Transfer;
    PMDL Mdl = Transfer->Mdl;
    ULONG Offset = Transfer->BytesQueued;
    PUCHAR Start;
    ULONG Length;
    PMDL Partial;
 
    while (Mdl != NULL && Offset >= MmGetMdlByteCount(Mdl))
    {
        Offset -= MmGetMdlByteCount(Mdl);
        Mdl = Mdl->Next;
    }
 
    if (Mdl == NULL)
    {
        DPRINT1("Bulk DCI %lu stream %lu: MDL chain shorter than the transfer\n",
                Ring->m_Endpoint->Dci(), Ring->m_StreamId);
        return STATUS_INSUFFICIENT_RESOURCES;
    }
 
    if (Offset == 0)
    {
        Stage->Mdl = Mdl;
        return STATUS_SUCCESS;
    }
 
    Start = static_cast<PUCHAR>(MmGetMdlVirtualAddress(Mdl)) + Offset;
    Length = MmGetMdlByteCount(Mdl) - Offset;
 
    Partial = IoAllocateMdl(Start, Length, FALSE, FALSE, NULL);
    if (Partial == NULL)
    {
        DPRINT1("Bulk DCI %lu stream %lu: partial MDL allocation failed\n",
                Ring->m_Endpoint->Dci(), Ring->m_StreamId);
        return STATUS_INSUFFICIENT_RESOURCES;
    }
 
    IoBuildPartialMdl(Mdl, Partial, Start, Length);
    Partial->Next = Mdl->Next;
    Stage->Mdl = Partial;
    Stage->FreeMdl = TRUE;
    return STATUS_SUCCESS;
}

Referenced by PrimeStage().

AllowReclaimOnCancel()

VOID XhciBulkRing::AllowReclaimOnCancel ( _In_ XhciTransferRing *  Ring )

static

Definition at line 680 of file bulkio.cpp.

{
    XhciBulkRing* Bulk = &Ring->m_Type.Bulk;
    BOOLEAN Queue = FALSE;
    KIRQL OldIrql;
 
    KeAcquireSpinLock(&Ring->m_Lock, &OldIrql);
    if (Bulk->m_Flags & XhciBulkCanceled)
    {
        /* QUIRK: the "OK to reclaim" flag stays clear when a cancel already happened */
        XhciBulkMoveList(&Bulk->m_Completion, &Bulk->m_Pending);
        Queue = TRUE;
    }
    else
    {
        Bulk->m_Flags |= XhciBulkOkToReclaim;
    }
    KeReleaseSpinLock(&Ring->m_Lock, OldIrql);
 
    if (Queue)
        QueueCompletionDpc(Ring);
}

Referenced by XhciTransferRing::AllowReclaimOnCancel().

ArrangeBuffer()

NTSTATUS XhciBulkRing::ArrangeBuffer ( _In_ XhciTransferRing *  Ring, _Inout_ XhciBulkTransfer *  Transfer  )

staticprivate

Definition at line 1207 of file bulkio.cpp.

{
    XhciBulkRing* Bulk = &Ring->m_Type.Bulk;
    struct _URB_BULK_OR_INTERRUPT_TRANSFER* Urb = Transfer->Urb;
 
    switch (Transfer->Mechanism)
    {
        case XhciBulkMechanism::Immediate:
        case XhciBulkMechanism::DoubleBuffer:
            if (Urb->TransferBuffer != NULL)
            {
                Transfer->Buffer = static_cast<PUCHAR>(Urb->TransferBuffer);
            }
            else if (Urb->TransferBufferMDL != NULL)
            {
                Transfer->Buffer = static_cast<PUCHAR>(
                    MmGetSystemAddressForMdlSafe(Urb->TransferBufferMDL, NormalPagePriority));
            }
 
            if (Transfer->Buffer == NULL)
            {
                DPRINT1("Bulk DCI %lu stream %lu: no system address for the buffer\n",
                        Ring->m_Endpoint->Dci(), Ring->m_StreamId);
                return STATUS_INSUFFICIENT_RESOURCES;
            }
            return STATUS_SUCCESS;
 
        case XhciBulkMechanism::Dma:
            if (Urb->TransferBufferMDL != NULL)
            {
                Transfer->Mdl = Urb->TransferBufferMDL;
                return STATUS_SUCCESS;
            }
 
            Transfer->Mdl = IoAllocateMdl(Urb->TransferBuffer, Transfer->BytesTotal, FALSE, FALSE, NULL);
            if (Transfer->Mdl == NULL)
            {
                if (!(Urb->TransferFlags & USB3_URB_RESERVED_RESOURCES) ||
                    Ring->m_ReservedMdl == NULL)
                {
                    DPRINT1("Bulk DCI %lu stream %lu: MDL allocation failed\n",
                            Ring->m_Endpoint->Dci(), Ring->m_StreamId);
                    return STATUS_INSUFFICIENT_RESOURCES;
                }
 
                /* The reserved MDL serves one request at a time */
                if (InterlockedCompareExchange(&Bulk->m_ForwardProgressBusy, 1, 0) != 0)
                {
                    DPRINT1("Bulk DCI %lu stream %lu: forward progress MDL already in use\n",
                            Ring->m_Endpoint->Dci(), Ring->m_StreamId);
                    return STATUS_INSUFFICIENT_RESOURCES;
                }
 
                Transfer->Mdl = Ring->m_ReservedMdl;
                Transfer->UsesReservedMdl = TRUE;
                MmInitializeMdl(Transfer->Mdl, Urb->TransferBuffer, Transfer->BytesTotal);
            }
 
            MmBuildMdlForNonPagedPool(Transfer->Mdl);
            return STATUS_SUCCESS;
 
        default:
            return STATUS_SUCCESS;
    }
}

Referenced by NextTdSlot().

BuildTd()

BOOLEAN XhciBulkRing::BuildTd ( _In_ XhciTransferRing *  Ring, _Inout_ XhciBulkStage *  Stage  )

staticprivate

Writes the TD of a stage and hands it to the controller. On failure the ring is put back as it was and the caller fails the stage.

Definition at line 1736 of file bulkio.cpp.

{
    XhciBulkRing* Bulk = &Ring->m_Type.Bulk;
    XhciBulkTransfer* Transfer = Stage->Transfer;
    XhciDmaBuffer* PadBuffer = Ring->m_Controller->m_SplitPadBuffer;
    PSCATTER_GATHER_LIST SgList = Stage->SgList;
    ULONG PadBytes = XhciBulkPadSize(Ring, Stage->Size);
    BOOLEAN NoOps = XhciBulkNeedsNoOps(Ring);
    ULONG Packets = Ring->PacketCount(Stage->Size);
    ULONG Pending = Stage->BurstTrbBudget;
    ULONG Mapped = 0;
    ULONG Element = 0;
    ULONG64 ElementAddress = 0;
    ULONG ElementLeft = 0;
    XhciBulkStep Step = XhciBulkStep::Normal;
    PXHCI_TRB First = &Ring->m_Trbs[Ring->m_EnqueueIndex];
    volatile UCHAR* CycleByte;
    XHCI_TRB Trb;
    ULONG64 Value;
    ULONG Chunk;
    ULONG Index;
    BOOLEAN IsFirst;
    KIRQL OldIrql;
 
    Stage->TrbsWritten = 0;
 
    if (Ring->m_Controller->HasErrata(XhciErrata::RingKeepTdInOneSegment) && !InsertLink(Ring, Stage, TRUE))
    {
        UndoTd(Ring, Stage);
        return FALSE;
    }
 
    while (Step != XhciBulkStep::Done)
    {
        IsFirst = (&Ring->m_Trbs[Ring->m_EnqueueIndex] == First);
 
        /* Keep each burst sized piece of the TD inside one segment (xHCI 4.11.7.1) */
        if (Pending + Ring->m_EnqueueIndex > Ring->m_LastIndex ||
            (NoOps && Step == XhciBulkStep::EventData &&
             Ring->m_EnqueueIndex + XhciBulkNoOpCount + 1 > Ring->m_LastIndex))
        {
            /* A failed insert fails the TD instead of retrying */
            if (!InsertLink(Ring, Stage, IsFirst))
            {
                UndoTd(Ring, Stage);
                return FALSE;
            }
            continue;
        }
 
        RtlZeroMemory(&Trb, sizeof(Trb));
 
        if (Step == XhciBulkStep::Normal)
        {
            Trb.Dword[3] = static_cast<ULONG>(XhciTrbType::Normal) << XHCI_TRB_TYPE_SHIFT;
 
            switch (Transfer->Mechanism)
            {
                case XhciBulkMechanism::Immediate:
                    RtlCopyMemory(&Trb.Dword[0], Stage->Buffer, Stage->Size);
                    Trb.Dword[3] |= XHCI_TRB_IDT;
                    Chunk = Stage->Size;
                    Step = XhciBulkStep::EventData;
                    break;
 
                case XhciBulkMechanism::DoubleBuffer:
                    if (!XhciBulkTransferIn(Transfer))
                        RtlCopyMemory(Stage->DoubleBuffer->VirtualAddress, Stage->Buffer, Stage->Size);
                    Trb.Dword[0] = Stage->DoubleBuffer->LogicalAddress.LowPart;
                    Trb.Dword[1] = (ULONG)Stage->DoubleBuffer->LogicalAddress.HighPart;
                    Chunk = Stage->Size;
                    Step = XhciBulkStep::EventData;
                    break;
 
                case XhciBulkMechanism::Dma:
                    if (ElementLeft == 0)
                    {
                        ElementAddress = (ULONG64)SgList->Elements[Element].Address.QuadPart;
                        ElementLeft = SgList->Elements[Element].Length;
                        Element++;
                    }
 
                    /* A TRB buffer may not cross a 64 KB boundary (xHCI 6.4.1.1) */
                    Chunk = min(ElementLeft, XhciBulkTrbBytesMax - (ULONG)(ElementAddress & 0xFFFF));
                    Trb.Dword[0] = (ULONG)ElementAddress;
                    Trb.Dword[1] = (ULONG)(ElementAddress >> 32);
                    ElementAddress += Chunk;
                    ElementLeft -= Chunk;
 
                    if (ElementLeft == 0 && Element >= SgList->NumberOfElements)
                        Step = XhciBulkStep::EventData;
                    break;
 
                default:
                    Chunk = 0;
                    Step = XhciBulkStep::EventData;
                    break;
            }
 
            Mapped += Chunk;
            Trb.Dword[2] = Chunk |
                           (Ring->TdSize(Packets, Mapped, Step == XhciBulkStep::EventData) << XHCI_TRB_TD_SIZE_SHIFT);
            Trb.Dword[3] |= XHCI_TRB_CHAIN;
            Pending--;
        }
        else if (Step == XhciBulkStep::Pad)
        {
            Trb.Dword[0] = PadBuffer->LogicalAddress.LowPart;
            Trb.Dword[1] = (ULONG)PadBuffer->LogicalAddress.HighPart;
            Trb.Dword[2] = PadBytes;
            Trb.Dword[3] = (static_cast<ULONG>(XhciTrbType::Normal) << XHCI_TRB_TYPE_SHIFT) | XHCI_TRB_CHAIN;
            Pending--;
            PadBytes = 0;
            Step = XhciBulkStep::EventData;
        }
        else
        {
            Value = XhciBulkEventData(Ring, Stage);
            Trb.Dword[0] = (ULONG)Value;
            Trb.Dword[1] = (ULONG)(Value >> 32);
            Trb.Dword[3] = (static_cast<ULONG>(XhciTrbType::EventData) << XHCI_TRB_TYPE_SHIFT) | XHCI_TRB_IOC;
            Step = XhciBulkStep::Done;
        }
 
        Stage->TrbsWritten++;
 
        if (Step == XhciBulkStep::EventData)
        {
            if (PadBytes != 0 && PadBuffer != NULL)
            {
                Trb.Dword[2] += 1 << XHCI_TRB_TD_SIZE_SHIFT;
                Step = XhciBulkStep::Pad;
 
                /* The pad still needs a slot when the burst is used up */
                if (Pending == 0)
                    Pending = 1;
            }
            else
            {
                /* xHCI 4.12.3: let a flow controlled ring reach the Event Data TRB */
                Trb.Dword[3] |= XHCI_TRB_ENT;
                Pending = 1;
            }
        }
        else if (Step == XhciBulkStep::Normal && Pending == 0)
        {
            Pending = (Stage->TrbsReserved > Stage->TrbsWritten) ?
                      min(Stage->BurstTrbBudget, Stage->TrbsReserved - Stage->TrbsWritten) :
                      Stage->BurstTrbBudget;
        }
 
        /* The first slot stays invalid until the whole TD is written */
        Trb.Dword[3] |= IsFirst ? (Ring->m_Cycle ^ XHCI_TRB_CYCLE) : Ring->m_Cycle;
        Trb.Dword[2] |= Ring->m_InterrupterTarget << XHCI_TRB_INTERRUPTER_SHIFT;
 
        Ring->m_Trbs[Ring->m_EnqueueIndex] = Trb;
        Ring->m_EnqueueIndex++;
 
        if (Step == XhciBulkStep::Done && NoOps)
        {
            for (Index = 0; Index < XhciBulkNoOpCount; Index++)
            {
                RtlZeroMemory(&Trb, sizeof(Trb));
                Trb.Dword[3] = (static_cast<ULONG>(XhciTrbType::NoOp) << XHCI_TRB_TYPE_SHIFT) | Ring->m_Cycle;
                Ring->m_Trbs[Ring->m_EnqueueIndex] = Trb;
                Ring->m_EnqueueIndex++;
            }
        }
    }
 
    NT_ASSERT(Stage->TrbsWritten <= Stage->TrbsReserved);
 
    /* Recycled segments are not cleared, so the slot after the TD must be */
    Stage->LastSegment = Ring->m_Segment;
    Stage->EndIndex = Ring->m_EnqueueIndex;
    RtlZeroMemory(&Ring->m_Trbs[Ring->m_EnqueueIndex], sizeof(XHCI_TRB));
 
    KeAcquireSpinLock(&Ring->m_Lock, &OldIrql);
    Transfer->TdsQueued++;
    Transfer->BytesQueued += Stage->Size;
    Transfer->MappedOnce = TRUE;
    Bulk->m_PendingStages++;
    Stage->OnHardware = TRUE;
    Ring->m_DoorbellRung = TRUE;
    KeReleaseSpinLock(&Ring->m_Lock, OldIrql);
 
    /* A one byte register write publishes the TD after every store above */
    KeMemoryBarrier();
    CycleByte = reinterpret_cast<volatile UCHAR*>(&First->Dword[3]);
    WRITE_REGISTER_UCHAR(const_cast<PUCHAR>(CycleByte), (UCHAR)(*CycleByte ^ XHCI_TRB_CYCLE));
    KeMemoryBarrier();
 
    Ring->RingDoorbell();
    return TRUE;
}

Referenced by EncodeStage().

BuildTdsFromQueue()

VOID XhciBulkRing::BuildTdsFromQueue ( _In_ XhciTransferRing *  Ring )

staticprivate

Definition at line 842 of file bulkio.cpp.

{
    XhciBulkRing* Bulk = &Ring->m_Type.Bulk;
    KIRQL OldIrql;
 
    KeRaiseIrql(DISPATCH_LEVEL, &OldIrql);
 
    for (;;)
    {
        FillLoop(Ring);
 
        if (Ring->m_MapState != static_cast<LONG>(XhciMapState::PausedFill) ||
            Bulk->m_FillWanted == 0)
        {
            break;
        }
 
        if (Ring->ChangeMapState(FALSE, XhciMapState::PausedFill, XhciMapState::Filling) != XhciMapState::PausedFill)
            break;
    }
 
    KeLowerIrql(OldIrql);
}

Referenced by KickFill(), ResumeRingFill(), and SegmentsArrived().

ChooseMechanism()

VOID XhciBulkRing::ChooseMechanism ( _In_ XhciTransferRing *  Ring, _Inout_ XhciBulkTransfer *  Transfer  )

staticprivate

Definition at line 1177 of file bulkio.cpp.

{
    if (Transfer->BytesTotal == 0)
    {
        Transfer->Mechanism = XhciBulkMechanism::NoData;
        return;
    }
 
    if (Ring->m_Type.Bulk.m_ImmediateData && Transfer->BytesTotal <= XhciBulkImmediateBytes)
    {
        Transfer->Mechanism = XhciBulkMechanism::Immediate;
        return;
    }
 
    if (!XhciBulkChainedMdl(Transfer) && Transfer->BytesTotal <= Ring->m_DoubleBufferSize)
    {
        Transfer->DoubleBuffer = Ring->BorrowBounceBuffer();
        if (Transfer->DoubleBuffer != NULL)
        {
            Transfer->Mechanism = XhciBulkMechanism::DoubleBuffer;
            return;
        }
    }
 
    Transfer->Mechanism = XhciBulkMechanism::Dma;
}

Referenced by NextTdSlot().

Cleanup()

VOID XhciBulkRing::Cleanup ( _In_ XhciTransferRing *  Ring )

static

Definition at line 502 of file bulkio.cpp.

{
    BOOLEAN Owned;
    KIRQL OldIrql;
 
    KeAcquireSpinLock(&Ring->m_Lock, &OldIrql);
    Owned = OwnsRequests(Ring);
    KeReleaseSpinLock(&Ring->m_Lock, OldIrql);
 
    if (Owned)
    {
        DPRINT1("Bulk DCI %lu stream %lu cleaned up with requests still owned\n",
                Ring->m_Endpoint->Dci(), Ring->m_StreamId);
        NT_ASSERT(FALSE);
    }
}

Referenced by XhciTransferRing::Cleanup().

Complete()

static VOID XhciBulkRing::Complete ( _In_ XhciTransferRing *  Ring, _Inout_ XhciBulkTransfer *  Transfer, _In_ USBD_STATUS  Default  )

staticprivate

Referenced by CompleteBatch(), and NextTdSlot().

CompleteAfterUncancel()

static VOID XhciBulkRing::CompleteAfterUncancel ( _In_ XhciTransferRing *  Ring, _Inout_ XhciBulkTransfer *  Transfer, _In_ USBD_STATUS  Default, _In_ BOOLEAN  FinishCanceled  )

staticprivate

Referenced by Ed1Event(), FailStage(), HaltedEvent(), and NextTdSlot().

CompleteBatch()

VOID XhciBulkRing::CompleteBatch ( _In_ XhciTransferRing *  Ring, _Inout_ PLIST_ENTRY  Batch  )

staticprivate

Definition at line 2096 of file bulkio.cpp.

{
    XhciBulkRing* Bulk = &Ring->m_Type.Bulk;
    XhciBulkTransfer* Transfer;
    XhciBulkStage* Stage;
    LIST_ENTRY Canceled;
    LIST_ENTRY Finished;
    LIST_ENTRY Requeue;
    BOOLEAN OnHardware;
    NTSTATUS Status;
    KIRQL OldIrql;
 
    InitializeListHead(&Canceled);
    InitializeListHead(&Finished);
    InitializeListHead(&Requeue);
 
    KeAcquireSpinLock(&Ring->m_Lock, &OldIrql);
 
    while (!IsListEmpty(Batch))
    {
        Transfer = CONTAINING_RECORD(RemoveHeadList(Batch), XhciBulkTransfer, Link);
        XhciClearListEntry(&Transfer->Link);
 
        Stage = Transfer->Stage;
        if (Stage != NULL)
        {
            KeReleaseSpinLock(&Ring->m_Lock, OldIrql);
            PutScatterGather(Ring, Stage);
            KeAcquireSpinLock(&Ring->m_Lock, &OldIrql);
 
            OnHardware = Stage->OnHardware;
            ReleaseStage(Ring, Stage);
            if (OnHardware)
            {
                Transfer->TdsRetired++;
                Bulk->m_PendingStages--;
            }
        }
 
        if (Transfer->CancelState == XhciBulkCancel::Done)
        {
            InsertTailList(&Canceled, &Transfer->Link);
            continue;
        }
 
        if (Transfer->CancelState == XhciBulkCancel::Armed)
        {
            Status = WdfRequestUnmarkCancelable(Transfer->Request);
            if (!NT_SUCCESS(Status))
            {
                Transfer->CancelState = XhciBulkCancel::CancelRoutinePending;
                InsertTailList(&Bulk->m_WaitingForCancel, &Transfer->Link);
                continue;
            }
            Transfer->CancelState = XhciBulkCancel::Idle;
        }
 
        if (Transfer->CompletionCode == static_cast<ULONG>(XhciCompletionCode::Success) ||
            Transfer->CompletionCode == static_cast<ULONG>(XhciCompletionCode::StoppedShortPacket))
        {
            InsertTailList(&Finished, &Transfer->Link);
        }
        else
        {
            InsertTailList(&Requeue, &Transfer->Link);
        }
    }
 
    KeReleaseSpinLock(&Ring->m_Lock, OldIrql);
 
    while (!IsListEmpty(&Canceled))
    {
        Transfer = CONTAINING_RECORD(RemoveHeadList(&Canceled), XhciBulkTransfer, Link);
        XhciClearListEntry(&Transfer->Link);
 
        if (Ring->m_StreamId != 0 && Transfer->CompletionCode == 0)
            Transfer->CompletionCode = Ring->m_Endpoint->StreamHaltedCode();
 
        Complete(Ring, Transfer, USBD_STATUS_CANCELED);
    }
 
    while (!IsListEmpty(&Finished))
    {
        Transfer = CONTAINING_RECORD(RemoveHeadList(&Finished), XhciBulkTransfer, Link);
        XhciClearListEntry(&Transfer->Link);
        Complete(Ring, Transfer, XhciBulkStatusNotSet);
    }
 
    /* From the tail, so the oldest request ends up at the head of the queue */
    while (!IsListEmpty(&Requeue))
    {
        Transfer = CONTAINING_RECORD(RemoveTailList(&Requeue), XhciBulkTransfer, Link);
        XhciClearListEntry(&Transfer->Link);
 
        /* It resumes where the hardware stopped */
        Transfer->BytesQueued = Transfer->BytesTransferred;
 
        if (Transfer->DoubleBuffer != NULL)
        {
            Ring->ReturnBounceBuffer(Transfer->DoubleBuffer);
            Transfer->DoubleBuffer = NULL;
        }
        DropMdl(Ring, Transfer);
 
        Status = WdfRequestRequeue(Transfer->Request);
        if (!NT_SUCCESS(Status))
        {
            DPRINT1("Bulk DCI %lu stream %lu could not requeue request %p, 0x%08lx\n",
                    Ring->m_Endpoint->Dci(), Ring->m_StreamId, Transfer->Request, Status);
            Complete(Ring, Transfer, USBD_STATUS_CANCELED);
        }
    }
}

ConsumePendingEvents()

VOID XhciBulkRing::ConsumePendingEvents ( _In_ XhciTransferRing *  Ring )

static

Definition at line 705 of file bulkio.cpp.

{
    XhciBulkRing* Bulk = &Ring->m_Type.Bulk;
    BOOLEAN Acknowledge = FALSE;
    KIRQL OldIrql;
 
    KeAcquireSpinLock(&Ring->m_Lock, &OldIrql);
    if (Bulk->m_OutstandingEvents == 0)
        Acknowledge = TRUE;
    else
        Bulk->m_Flags |= XhciBulkAckExpected;
    KeReleaseSpinLock(&Ring->m_Lock, OldIrql);
 
    if (Acknowledge)
        Ring->m_Endpoint->OnExpectedEventsProcessed();
}

Referenced by XhciTransferRing::ConsumePendingEvents().

Disable()

VOID XhciBulkRing::Disable ( _In_ XhciTransferRing *  Ring )

static

Definition at line 422 of file bulkio.cpp.

{
    KIRQL OldIrql;
 
    ReleaseParkedMapping(Ring);
 
    InterlockedCompareExchange(&Ring->m_MapState,
                               static_cast<LONG>(XhciMapState::Halted),
                               static_cast<LONG>(XhciMapState::PausedFill));
 
    KeAcquireSpinLock(&Ring->m_Lock, &OldIrql);
    if (OwnsRequests(Ring))
    {
        DPRINT1("Bulk DCI %lu stream %lu disabled while it still owns requests\n",
                Ring->m_Endpoint->Dci(), Ring->m_StreamId);
    }
    KeReleaseSpinLock(&Ring->m_Lock, OldIrql);
}

DoorbellRungSinceFill()

BOOLEAN XhciBulkRing::DoorbellRungSinceFill ( _In_ const XhciTransferRing *  Ring )

static

Definition at line 740 of file bulkio.cpp.

{
    return Ring->m_DoorbellRung;
}

Referenced by XhciTransferRing::DoorbellRungSinceFill().

DropMdl()

VOID XhciBulkRing::DropMdl ( _In_ XhciTransferRing *  Ring, _Inout_ XhciBulkTransfer *  Transfer  )

staticprivate

Definition at line 1276 of file bulkio.cpp.

{
    if (Transfer->Mdl == NULL)
        return;
 
    if (Transfer->UsesReservedMdl)
    {
        Transfer->UsesReservedMdl = FALSE;
        InterlockedExchange(&Ring->m_Type.Bulk.m_ForwardProgressBusy, 0);
    }
    else if (Transfer->Mdl != Transfer->Urb->TransferBufferMDL)
    {
        IoFreeMdl(Transfer->Mdl);
    }
 
    Transfer->Mdl = NULL;
}

Referenced by CompleteBatch(), and FreeTransferResources().

Ed0Event()

BOOLEAN XhciBulkRing::Ed0Event ( _In_ XhciTransferRing *  Ring, _In_ const XHCI_TRB *  Event  )

staticprivate

Definition at line 2431 of file bulkio.cpp.

{
    XhciBulkRing* Bulk = &Ring->m_Type.Bulk;
    ULONG Code = XhciTrbCompletionCode(Event);
    XhciBulkStage* Stage;
    ULONG Skipped;
    ULONG Bytes = 0;
    KIRQL OldIrql;
 
    if (XhciBulkIsStoppedCode(Code) && Ring->m_Endpoint->ShouldDropStoppedEvent())
        return TRUE;
 
    KeAcquireSpinLock(&Ring->m_Lock, &OldIrql);
 
    if (!LookupStage(Ring, XhciTrbPointer(Event), &Stage, &Skipped))
    {
        if (Ring->m_StreamId == 0)
            Ed0Mismatch(Ring, Event);
        KeReleaseSpinLock(&Ring->m_Lock, OldIrql);
        return FALSE;
    }
 
    if (Stage != NULL)
    {
        Bytes = XhciBulkStageBytes(Stage, Event);
        if (Bytes > Stage->Size)
        {
            DPRINT1("Bulk DCI %lu stream %lu: %lu bytes found in a %lu byte stage\n",
                    Ring->m_Endpoint->Dci(), Ring->m_StreamId, Bytes, Stage->Size);
            Bytes = 0;
        }
    }
 
    if (XhciBulkIsHaltedCode(Code))
    {
        KeReleaseSpinLock(&Ring->m_Lock, OldIrql);
 
        /* QUIRK: old controllers report a halt one past the last TRB; that event is dropped */
        if (Stage != NULL)
            HaltedEvent(Ring, Stage, Code, Bytes);
        else
            DPRINT("Bulk DCI %lu: halt reported at the enqueue pointer, dropped\n", Ring->m_Endpoint->Dci());
 
        return TRUE;
    }
 
    if (XhciBulkIsStoppedCode(Code))
    {
        StoppedEvent(Ring, Stage, Code, Bytes, Skipped);
        Bulk->m_Flags |= XhciBulkStoppedSeen;
        KeReleaseSpinLock(&Ring->m_Lock, OldIrql);
 
        Ring->m_Endpoint->OnStoppedEvent();
        return TRUE;
    }
 
    KeReleaseSpinLock(&Ring->m_Lock, OldIrql);
    DPRINT1("Bulk DCI %lu stream %lu: unexpected code %lu without Event Data\n",
            Ring->m_Endpoint->Dci(), Ring->m_StreamId, Code);
    return FALSE;
}

Referenced by OnTransferEvent().

Ed0Mismatch()

static VOID XhciBulkRing::Ed0Mismatch ( _In_ XhciTransferRing *  Ring, _In_ const XHCI_TRB *  Event  )

staticprivate

Referenced by Ed0Event().

Ed1Event()

BOOLEAN XhciBulkRing::Ed1Event ( _In_ XhciTransferRing *  Ring, _In_ const XHCI_TRB *  Event  )

staticprivate

Definition at line 2317 of file bulkio.cpp.

{
    XhciBulkRing* Bulk = &Ring->m_Type.Bulk;
    ULONG64 Value = XhciTrbPointer(Event);
    XhciBulkStage* Stage = XhciBulkStageFromEventData(Value);
    XhciBulkTransfer* Transfer;
    XhciBulkStage* Match;
    ULONG Code = XhciTrbCompletionCode(Event);
    ULONG Bytes = Event->Dword[2] & XHCI_TRANSFER_EVENT_LENGTH_MASK;
    ULONG EndpointId = (Event->Dword[3] & XHCI_TRB_ENDPOINT_MASK) >> XHCI_TRB_ENDPOINT_SHIFT;
    BOOLEAN Acknowledge = FALSE;
    BOOLEAN Live;
    ULONG64 Address;
    ULONG Skipped;
    KIRQL OldIrql;
 
    KeAcquireSpinLock(&Ring->m_Lock, &OldIrql);
    Live = IsLiveStage(Ring, Stage, Value);
    KeReleaseSpinLock(&Ring->m_Lock, OldIrql);
 
    if (!Live)
    {
        /* A stale or unknown Event Data pointer is dropped, never dereferenced */
        if (Ring->m_StreamId == 0)
        {
            DPRINT1("Bulk DCI %lu: stale or unknown Event Data 0x%I64x code %lu, dropped\n",
                    Ring->m_Endpoint->Dci(), Value, Code);
        }
 
        KickFill(Ring);
        return (Ring->m_StreamId == 0);
    }
 
    Transfer = Stage->Transfer;
 
    if (EndpointId != Ring->m_Endpoint->Dci() || XhciTrbSlotId(Event) != Ring->m_Device->SlotId())
    {
        DPRINT1("Bulk event for slot %lu DCI %lu reached slot %lu DCI %lu\n",
                XhciTrbSlotId(Event), EndpointId, Ring->m_Device->SlotId(), Ring->m_Endpoint->Dci());
    }
 
    if (Bytes > Stage->Size)
    {
        DPRINT1("Bulk DCI %lu stream %lu: %lu bytes reported for a %lu byte stage\n",
                Ring->m_Endpoint->Dci(), Ring->m_StreamId, Bytes, Stage->Size);
        Bytes = 0;
    }
 
    if (XhciBulkIsHaltedCode(Code))
    {
        DPRINT1("Bulk DCI %lu stream %lu: halted code %lu reported through Event Data\n",
                Ring->m_Endpoint->Dci(), Ring->m_StreamId, Code);
        HaltedEvent(Ring, Stage, Code, Bytes);
    }
    else if (XhciBulkIsStoppedCode(Code))
    {
        /* QUIRK: the drop stopped event flag is not consulted for Event Data events */
        Address = (ULONG64)Stage->FirstSegment->LogicalAddress.QuadPart +
                  (ULONG64)Stage->FirstIndex * sizeof(XHCI_TRB);
 
        KeAcquireSpinLock(&Ring->m_Lock, &OldIrql);
        if (LookupStage(Ring, Address, &Match, &Skipped) && Match == Stage)
            StoppedEvent(Ring, Stage, Code, Bytes, Skipped);
        else
            NT_ASSERT(FALSE);
        Bulk->m_Flags |= XhciBulkStoppedSeen;
        KeReleaseSpinLock(&Ring->m_Lock, OldIrql);
 
        Ring->m_Endpoint->OnStoppedEvent();
    }
    else
    {
        /* Only a short packet or a fully queued request records the code, so a reclaim cannot finish a partial one as Success */
        if (Code == static_cast<ULONG>(XhciCompletionCode::ShortPacket) ||
            Transfer->BytesQueued == Transfer->BytesTotal)
        {
            Transfer->CompletionCode = Code;
        }
 
        Transfer->BytesTransferred += Bytes;
        XhciBulkCopyIn(Transfer, Stage, Bytes);
        PutScatterGather(Ring, Stage);
 
        KeAcquireSpinLock(&Ring->m_Lock, &OldIrql);
        ReleaseStage(Ring, Stage);
        Transfer->TdsRetired++;
 
        if (Code == static_cast<ULONG>(XhciCompletionCode::ShortPacket) ||
            Transfer->BytesQueued == Transfer->BytesTotal)
        {
            CompleteAfterUncancel(Ring, Transfer, XhciBulkStatusNotSet, FALSE);
        }
 
        Bulk->m_PendingStages--;
        if (Bulk->m_OutstandingEvents != 0)
        {
            Bulk->m_OutstandingEvents--;
            if (Bulk->m_OutstandingEvents == 0 && (Bulk->m_Flags & XhciBulkAckExpected))
                Acknowledge = TRUE;
        }
        KeReleaseSpinLock(&Ring->m_Lock, OldIrql);
 
        if (Acknowledge)
            Ring->m_Endpoint->OnExpectedEventsProcessed();
    }
 
    /* A slot on the ring freed up */
    KickFill(Ring);
    return TRUE;
}

Referenced by OnTransferEvent().

Enable()

VOID XhciBulkRing::Enable ( _In_ XhciTransferRing *  Ring )

static

Definition at line 401 of file bulkio.cpp.

{
    ULONG Burst;
 
    NT_ASSERT(Ring->m_MaxStageSize >= XhciBulkMinStageSize);
 
    if (Ring->m_Device->Speed() != UsbSuperSpeed)
        return;
 
    Burst = Ring->m_Endpoint->MaxPacketSize() * (Ring->m_Endpoint->MaxBurst() + 1);
    if (Burst == 0 || (Ring->m_MaxStageSize % Burst) == 0)
        return;
 
    Ring->m_MaxStageSize -= Ring->m_MaxStageSize % Burst;
    Ring->m_MapRegisterCount = (Ring->m_MaxStageSize + PAGE_SIZE - 1) / PAGE_SIZE;
    DPRINT("Bulk DCI %lu stream %lu stage size cut to %lu\n",
           Ring->m_Endpoint->Dci(), Ring->m_StreamId, Ring->m_MaxStageSize);
}

Referenced by _IRQL_requires_max_().

EnableForwardProgress()

NTSTATUS XhciBulkRing::EnableForwardProgress ( _In_ XhciTransferRing *  Ring, _In_ ULONG  MaxTransferSize  )

static

Definition at line 466 of file bulkio.cpp.

{
    ULONG Needed = XhciBulkWorstPages(MaxTransferSize) + 1;
    ULONG PerBurst = XhciBulkWorstPages(Ring->m_Endpoint->MaxEsitPayload()) + 1;
    ULONG PerSegment = Ring->m_SegmentSize / sizeof(XHCI_TRB) - 1;
    ULONG Usable = PerSegment - (PerSegment % PerBurst);
    ULONG Segments;
    NTSTATUS Status;
 
    if (Usable == 0)
    {
        DPRINT1("Bulk DCI %lu: a burst of %lu TRBs does not fit a segment\n",
                Ring->m_Endpoint->Dci(), PerBurst);
        return STATUS_INSUFFICIENT_RESOURCES;
    }
 
    Segments = (Needed + Usable - 1) / Usable;
    if (Segments == 0)
        Segments = 1;
 
    Status = Ring->EnsureFreeSegments(Segments, FALSE);
    if (!NT_SUCCESS(Status))
    {
        DPRINT1("Bulk DCI %lu: %lu forward progress segments not available, 0x%08lx\n",
                Ring->m_Endpoint->Dci(), Segments, Status);
        return Status;
    }
 
    Ring->m_DoubleBufferSize = 0;
    Ring->m_Type.Bulk.m_ImmediateData = FALSE;
    return STATUS_SUCCESS;
}

EncodeStage()

XhciBulkRing::Outcome XhciBulkRing::EncodeStage ( _In_ XhciTransferRing *  Ring, _Inout_ XhciBulkStage *  Stage  )

staticprivate

Definition at line 1661 of file bulkio.cpp.

{
    XhciBulkRing* Bulk = &Ring->m_Type.Bulk;
    PDMA_ADAPTER Adapter;
    NTSTATUS Status;
    KIRQL OldIrql;
 
    if (Stage->Transfer->Mechanism != XhciBulkMechanism::Dma)
    {
        if (!BuildTd(Ring, Stage))
            return FailStage(Ring, Stage);
        return Outcome::Ready;
    }
 
    Adapter = Ring->m_Controller->m_Buffers.DmaAdapter();
 
    KeRaiseIrql(DISPATCH_LEVEL, &OldIrql);
    Status = Adapter->DmaOperations->GetScatterGatherList(Adapter,
                                                          WdfDeviceWdmGetDeviceObject(Ring->m_Controller->m_Device),
                                                          Stage->Mdl,
                                                          MmGetMdlVirtualAddress(Stage->Mdl),
                                                          Stage->Size,
                                                          DmaCallback,
                                                          Stage,
                                                          !XhciBulkTransferIn(Stage->Transfer));
    KeLowerIrql(OldIrql);
 
    if (!NT_SUCCESS(Status))
    {
        DPRINT1("Bulk DCI %lu stream %lu: GetScatterGatherList failed, 0x%08lx\n",
                Ring->m_Endpoint->Dci(), Ring->m_StreamId, Status);
        return FailStage(Ring, Stage);
    }
 
    /* Whichever of this context and the callback gets here second keeps mapping */
    if (InterlockedXor(&Bulk->m_FillAgain, 1) == 1)
        return Outcome::Ready;
 
    return Outcome::WaitDma;
}

Referenced by FillLoop().

EstimateSegments()

BOOLEAN XhciBulkRing::EstimateSegments ( _In_ XhciTransferRing *  Ring, _In_ const XhciBulkStage *  Stage, _Out_ PULONG  Segments  )

staticprivate

Definition at line 1588 of file bulkio.cpp.

{
    ULONG Available;
    ULONG Usable;
 
    *Segments = 0;
 
    if (Ring->m_Controller->HasErrata(XhciErrata::RingKeepTdInOneSegment))
    {
        /* The TD always starts in a fresh segment */
        *Segments = 1;
    }
    else
    {
        Available = Ring->m_LastIndex - Ring->m_EnqueueIndex;
        Available -= Available % Stage->BurstTrbBudget;
 
        if (Stage->TrbsReserved > Available)
        {
            Usable = Ring->m_LastIndex - (Ring->m_LastIndex % Stage->BurstTrbBudget);
 
            /* One burst can need more TRBs than a segment holds */
            if (Usable == 0)
            {
                DPRINT1("Bulk DCI %lu stream %lu: a %lu TRB burst does not fit a segment\n",
                        Ring->m_Endpoint->Dci(), Ring->m_StreamId, Stage->BurstTrbBudget);
                return FALSE;
            }
 
            *Segments = (Stage->TrbsReserved - Available + Usable - 1) / Usable;
        }
    }
 
    if (XhciBulkNeedsNoOps(Ring))
        *Segments += 1;
 
    return TRUE;
}

Referenced by PrimeStage().

EstimateTrbs()

VOID XhciBulkRing::EstimateTrbs ( _In_ XhciTransferRing *  Ring, _Inout_ XhciBulkStage *  Stage  )

staticprivate

Definition at line 1502 of file bulkio.cpp.

{
    XhciBulkTransfer* Transfer = Stage->Transfer;
    ULONG Burst;
    ULONG Carry = 0;
    ULONG CarryTrbs = 0;
    ULONG Left;
    PMDL Mdl;
 
    if (Transfer->Mechanism != XhciBulkMechanism::Dma)
    {
        /* One Normal and one Event Data TRB */
        Stage->TrbsReserved = 2;
        Stage->BurstTrbBudget = 2;
    }
    else
    {
        Burst = min(Ring->m_Endpoint->MaxEsitPayload(), Stage->Size);
        if (Burst == 0)
            Burst = 1;
 
        Stage->BurstTrbBudget = XhciBulkWorstPages(Burst) + 1;
        Stage->TrbsReserved = 0;
 
        if (!XhciBulkChainedMdl(Transfer))
        {
            Stage->TrbsReserved = ADDRESS_AND_SIZE_TO_SPAN_PAGES(MmGetMdlVirtualAddress(Stage->Mdl),
                                                                 Stage->Size);
        }
        else
        {
            /* QUIRK: per burst sized fragment estimate; a carried fragment is counted twice */
            for (Mdl = Stage->Mdl, Left = Stage->Size; Mdl != NULL && Left != 0; Mdl = Mdl->Next)
            {
                ULONG Chunk = min(Left, MmGetMdlByteCount(Mdl));
                ULONG Offset = 0;
 
                Left -= Chunk;
                while (Offset < Chunk)
                {
                    PUCHAR Va = static_cast<PUCHAR>(MmGetMdlVirtualAddress(Mdl)) + Offset;
                    ULONG Piece = Chunk - Offset;
                    ULONG Span;
 
                    if (Piece + Carry >= Burst)
                    {
                        Span = ADDRESS_AND_SIZE_TO_SPAN_PAGES(Va, Burst - Carry) + CarryTrbs;
                        Offset += Burst - Carry;
                        Carry = 0;
                        CarryTrbs = 0;
                    }
                    else
                    {
                        Span = ADDRESS_AND_SIZE_TO_SPAN_PAGES(Va, Piece);
                        Carry = Piece;
                        CarryTrbs = Span;
                        Offset = Chunk;
                    }
 
                    Stage->TrbsReserved += Span;
 
                    if (Span >= Ring->m_LastIndex)
                    {
                        DPRINT1("Bulk DCI %lu stream %lu: a %lu TRB fragment needs a whole segment\n",
                                Ring->m_Endpoint->Dci(), Ring->m_StreamId, Span);
                    }
 
                    if (Span > Stage->BurstTrbBudget)
                        Stage->BurstTrbBudget = Span;
                }
            }
        }
 
        Stage->TrbsReserved += 1;
    }
 
    if (XhciBulkPadSize(Ring, Stage->Size) != 0)
    {
        Stage->BurstTrbBudget += 1;
        Stage->TrbsReserved += 1;
    }
}

Referenced by PrimeStage().

FailStage()

XhciBulkRing::Outcome XhciBulkRing::FailStage ( _In_ XhciTransferRing *  Ring, _Inout_ XhciBulkStage *  Stage  )

staticprivate

Drops a stage that cannot be mapped; completes its request when nothing of it is on the ring.

Definition at line 1632 of file bulkio.cpp.

{
    XhciBulkTransfer* Transfer = Stage->Transfer;
    Outcome Result;
    KIRQL OldIrql;
 
    PutScatterGather(Ring, Stage);
 
    KeAcquireSpinLock(&Ring->m_Lock, &OldIrql);
    ReleaseStage(Ring, Stage);
 
    if (Transfer->TdsQueued == Transfer->TdsRetired)
    {
        CompleteAfterUncancel(Ring, Transfer, USBD_STATUS_INSUFFICIENT_RESOURCES, TRUE);
        Result = Outcome::Completed;
    }
    else
    {
        Transfer->BytesQueued = Transfer->BytesTotal;
        Result = Outcome::Failed;
    }
 
    KeReleaseSpinLock(&Ring->m_Lock, OldIrql);
    return Result;
}

Referenced by EncodeStage(), and PrimeStage().

FillLoop()

VOID XhciBulkRing::FillLoop ( _In_ XhciTransferRing *  Ring )

staticprivate

Definition at line 868 of file bulkio.cpp.

{
    XhciBulkRing* Bulk = &Ring->m_Type.Bulk;
    XhciBulkStage* Stage;
    Outcome Result;
    BOOLEAN Pause = FALSE;
 
    for (;;)
    {
        InterlockedExchange(&Bulk->m_FillWanted, 0);
 
        Stage = NextTdSlot(Ring);
        if (Stage == NULL)
        {
            Pause = TRUE;
            break;
        }
 
        Result = PrimeStage(Ring, Stage);
        if (Result == Outcome::Completed)
            continue;
        if (Result == Outcome::WaitSegments)
            break;
        if (Result == Outcome::Failed)
        {
            Pause = TRUE;
            break;
        }
 
        Result = EncodeStage(Ring, Stage);
        if (Result == Outcome::Completed)
            continue;
        if (Result == Outcome::WaitDma)
            break;
        if (Result == Outcome::Failed)
        {
            Pause = TRUE;
            break;
        }
 
        /* Also completes a stop that raced in */
        if (Ring->ChangeMapState(FALSE, XhciMapState::Filling, XhciMapState::Filling) != XhciMapState::Filling)
            break;
    }
 
    if (Pause)
        Ring->ChangeMapState(FALSE, XhciMapState::Filling, XhciMapState::PausedFill);
}

Referenced by BuildTdsFromQueue().

FreeTransferResources()

VOID XhciBulkRing::FreeTransferResources ( _In_ XhciTransferRing *  Ring, _Inout_ XhciBulkTransfer *  Transfer  )

staticprivate

Definition at line 1297 of file bulkio.cpp.

{
    NT_ASSERT(Transfer->Stage == NULL);
 
    if (Transfer->DoubleBuffer != NULL)
    {
        Ring->ReturnBounceBuffer(Transfer->DoubleBuffer);
        Transfer->DoubleBuffer = NULL;
    }
 
    DropMdl(Ring, Transfer);
    Transfer->Buffer = NULL;
    Transfer->Initialized = 0;
}

HaltedEvent()

VOID XhciBulkRing::HaltedEvent ( _In_ XhciTransferRing *  Ring, _Inout_ XhciBulkStage *  Stage, _In_ ULONG  Code, _In_ ULONG  Bytes  )

staticprivate

Definition at line 2541 of file bulkio.cpp.

{
    XhciBulkRing* Bulk = &Ring->m_Type.Bulk;
    XhciBulkTransfer* Transfer = Stage->Transfer;
    BOOLEAN Acknowledge = FALSE;
    KIRQL OldIrql;
 
    KeAcquireSpinLock(&Ring->m_Lock, &OldIrql);
    Bulk->m_Flags |= XhciBulkHalted;
    Bulk->m_OutstandingEvents = 1;
    KeReleaseSpinLock(&Ring->m_Lock, OldIrql);
 
    /* The machine must see the halt before the client sees the failed request */
    Ring->m_Endpoint->OnHaltedCompletionCode(Code, TRUE);
 
    Transfer->CompletionCode = Code;
    Transfer->BytesTransferred += Bytes;
    XhciBulkCopyIn(Transfer, Stage, Bytes);
    PutScatterGather(Ring, Stage);
 
    KeAcquireSpinLock(&Ring->m_Lock, &OldIrql);
    ReleaseStage(Ring, Stage);
    Transfer->TdsRetired++;
    CompleteAfterUncancel(Ring, Transfer, XhciBulkStatusNotSet, FALSE);
    Bulk->m_PendingStages--;
    if (Bulk->m_OutstandingEvents != 0)
        Bulk->m_OutstandingEvents--;
    if (Bulk->m_Flags & XhciBulkAckExpected)
        Acknowledge = TRUE;
    KeReleaseSpinLock(&Ring->m_Lock, OldIrql);
 
    if (Acknowledge)
        Ring->m_Endpoint->OnExpectedEventsProcessed();
}

Referenced by Ed0Event(), and Ed1Event().

HasQueuedWork()

BOOLEAN XhciBulkRing::HasQueuedWork ( _In_ const XhciTransferRing *  Ring )

static

Definition at line 747 of file bulkio.cpp.

{
    PKSPIN_LOCK Lock = const_cast<PKSPIN_LOCK>(&Ring->m_Lock);
    BOOLEAN Pending;
    KIRQL OldIrql;
 
    KeAcquireSpinLock(Lock, &OldIrql);
    Pending = !IsListEmpty(&Ring->m_Type.Bulk.m_Pending);
    KeReleaseSpinLock(Lock, OldIrql);
 
    return Pending;
}

Referenced by XhciTransferRing::HasQueuedWork().

Initialize()

NTSTATUS XhciBulkRing::Initialize ( _In_ XhciTransferRing *  Ring )

static

Definition at line 357 of file bulkio.cpp.

{
    XhciBulkRing* Bulk = &Ring->m_Type.Bulk;
    XhciController* Controller = Ring->m_Controller;
    ULONG Index;
 
    RtlZeroMemory(Bulk, sizeof(*Bulk));
    InitializeListHead(&Bulk->m_Pending);
    InitializeListHead(&Bulk->m_WaitingForCancel);
    InitializeListHead(&Bulk->m_Completion);
 
    for (Index = 0; Index < StageSlots; Index++)
    {
        XhciBulkStage* Stage = &Bulk->m_Stages[Index];
 
        Stage->Signature = XhciBulkStageTag;
        Stage->Ring = Ring;
        InitializeListHead(&Stage->FreeSegments);
        InitializeListHead(&Stage->UsedSegments);
    }
 
    Bulk->m_ImmediateData = !Controller->HasErrata(XhciErrata::XferNoInlineData) &&
                            Ring->m_Endpoint->MaxPacketSize() >= XhciBulkImmediateBytes &&
                            !XhciBulkEndpointIn(Ring);
 
    if (Controller->HasErrata(XhciErrata::XferOneAsyncTdAtATime) ||
        Controller->HasErrata(XhciErrata::RingKeepTdInOneSegment))
    {
        Bulk->m_MaxPendingStages = 1;
    }
    else
    {
        Bulk->m_MaxPendingStages = XhciBulkPendingStagesMax;
    }
 
    /* A TD has to fit one segment, less the Link, Event Data and pad slots */
    if (Controller->HasErrata(XhciErrata::RingKeepTdInOneSegment))
        Ring->m_MaxStageSize = (Ring->m_SegmentSize / sizeof(XHCI_TRB) - 4) * PAGE_SIZE;
 
    return STATUS_SUCCESS;
}

Referenced by _IRQL_requires_().

InitTransfer()

VOID XhciBulkRing::InitTransfer ( _In_ XhciTransferRing *  Ring, _Inout_ XhciBulkTransfer *  Transfer, _In_ WDFREQUEST  Request  )

staticprivate

Definition at line 1139 of file bulkio.cpp.

{
    WDF_REQUEST_PARAMETERS Parameters;
    struct _URB_BULK_OR_INTERRUPT_TRANSFER* Urb;
 
    /* A requeued request is already set up; keep its progress instead of starting over */
    if (Transfer->Initialized == XhciBulkTransferTag)
    {
        NT_ASSERT(Transfer->CancelState == XhciBulkCancel::Idle);
        NT_ASSERT(Transfer->Status == STATUS_PENDING);
        NT_ASSERT(!XhciIsListEntryLinked(&Transfer->Link));
        NT_ASSERT(Transfer->TdsQueued == Transfer->TdsRetired);
        return;
    }
 
    WDF_REQUEST_PARAMETERS_INIT(&Parameters);
    WdfRequestGetParameters(Request, &Parameters);
    Urb = static_cast<struct _URB_BULK_OR_INTERRUPT_TRANSFER*>(Parameters.Parameters.Others.Arg1);
 
    RtlZeroMemory(Transfer, sizeof(*Transfer));
    XhciClearListEntry(&Transfer->Link);
    Transfer->Initialized = XhciBulkTransferTag;
    Transfer->Request = Request;
    Transfer->Urb = Urb;
    Transfer->Ring = Ring;
    Transfer->CancelState = XhciBulkCancel::Idle;
    Transfer->Status = STATUS_PENDING;
    Transfer->Mechanism = XhciBulkMechanism::NoData;
    Transfer->BytesTotal = Urb->TransferBufferLength;
 
    /* QUIRK: the length reads 0 until completion, as the USB 2.0 stack does */
    Urb->TransferBufferLength = 0;
}

Referenced by NextTdSlot().

InsertLink()

BOOLEAN XhciBulkRing::InsertLink ( _In_ XhciTransferRing *  Ring, _Inout_ XhciBulkStage *  Stage, _In_ BOOLEAN  FirstOfTd  )

staticprivate

Ends the current segment with a Link TRB into the next reserved one.

Definition at line 1936 of file bulkio.cpp.

{
    XhciDmaBuffer* Next;
    XHCI_TRB Trb;
    KIRQL OldIrql;
 
    if (IsListEmpty(&Stage->FreeSegments))
    {
        DPRINT1("Bulk DCI %lu stream %lu: no reserved segment left for a Link TRB\n",
                Ring->m_Endpoint->Dci(), Ring->m_StreamId);
        return FALSE;
    }
 
    Next = CONTAINING_RECORD(RemoveHeadList(&Stage->FreeSegments), XhciDmaBuffer, Link);
    XhciClearListEntry(&Next->Link);
 
    /* Chained even ahead of the pad TRB */
    Ring->BuildLinkTrb(&Trb, FirstOfTd, TRUE);
    Trb.Dword[0] = Next->LogicalAddress.LowPart;
    Trb.Dword[1] = (ULONG)Next->LogicalAddress.HighPart;
    Ring->m_Trbs[Ring->m_EnqueueIndex] = Trb;
 
    /* The ring never toggles its cycle state: it is an endless forward chain of segments */
    KeAcquireSpinLock(&Ring->m_Lock, &OldIrql);
    InsertTailList(&Stage->UsedSegments, &Ring->m_Segment->Link);
    Ring->m_Segment = Next;
    Ring->m_Trbs = static_cast<PXHCI_TRB>(Next->VirtualAddress);
    Ring->m_EnqueueIndex = 0;
    Stage->LastSegment = Next;
    KeReleaseSpinLock(&Ring->m_Lock, OldIrql);
 
    return TRUE;
}

Referenced by BuildTd().

IsLikelyDuplicate()

BOOLEAN XhciBulkRing::IsLikelyDuplicate ( _In_ const XhciTransferRing *  Ring, _In_ const XHCI_TRB *  Event  )

static

Definition at line 762 of file bulkio.cpp.

{
    PKSPIN_LOCK Lock = const_cast<PKSPIN_LOCK>(&Ring->m_Lock);
    BOOLEAN NoOp;
    BOOLEAN Likely;
    KIRQL OldIrql;
 
    KeAcquireSpinLock(Lock, &OldIrql);
    Likely = LikelyDuplicate(Ring, XhciTrbPointer(Event), &NoOp);
    KeReleaseSpinLock(Lock, OldIrql);
 
    return Likely;
}

Referenced by XhciTransferRing::IsLikelyDuplicate().

IsLiveStage()

static BOOLEAN XhciBulkRing::IsLiveStage ( _In_ const XhciTransferRing *  Ring, _In_ const XhciBulkStage *  Stage, _In_ ULONG64  EventData  )

staticprivate

Referenced by Ed1Event().

KickFill()

VOID XhciBulkRing::KickFill ( _In_ XhciTransferRing *  Ring )

staticprivate

Definition at line 919 of file bulkio.cpp.

{
    InterlockedExchange(&Ring->m_Type.Bulk.m_FillWanted, 1);
 
    if (Ring->ChangeMapState(FALSE, XhciMapState::PausedFill, XhciMapState::Filling) == XhciMapState::PausedFill)
        BuildTdsFromQueue(Ring);
}

Referenced by Ed1Event().

LikelyDuplicate()

static BOOLEAN XhciBulkRing::LikelyDuplicate ( _In_ const XhciTransferRing *  Ring, _In_ ULONG64  Address, _Out_ PBOOLEAN  PointsToNoOp  )

staticprivate

Referenced by IsLikelyDuplicate().

LookupStage()

static BOOLEAN XhciBulkRing::LookupStage ( _In_ const XhciTransferRing *  Ring, _In_ ULONG64  Address, _Outptr_result_maybenull_ XhciBulkStage **  Stage, _Out_ PULONG  Skipped  )

staticprivate

Referenced by Ed0Event(), and Ed1Event().

NextTdSlot()

XhciBulkStage * XhciBulkRing::NextTdSlot ( _In_ XhciTransferRing *  Ring )

staticprivate

Definition at line 929 of file bulkio.cpp.

{
    XhciBulkRing* Bulk = &Ring->m_Type.Bulk;
    XhciBulkTransfer* Transfer;
    XhciBulkStage* Stage;
    WDFREQUEST Request;
    BOOLEAN Idle;
    NTSTATUS Status;
    KIRQL OldIrql;
 
    KeAcquireSpinLock(&Ring->m_Lock, &OldIrql);
 
    if (Bulk->m_PendingStages >= Bulk->m_MaxPendingStages)
    {
        KeReleaseSpinLock(&Ring->m_Lock, OldIrql);
        return NULL;
    }
 
    if (!IsListEmpty(&Bulk->m_Pending))
    {
        Transfer = CONTAINING_RECORD(Bulk->m_Pending.Blink, XhciBulkTransfer, Link);
 
        /* A zero length request has not mapped yet even though its byte count is met */
        if (Transfer->BytesQueued < Transfer->BytesTotal || !Transfer->MappedOnce)
        {
            if (Bulk->m_SegmentWaits != XhciBulkSegmentWaitLimit)
            {
                Stage = AcquireStage(Ring, Transfer);
                KeReleaseSpinLock(&Ring->m_Lock, OldIrql);
                return Stage;
            }
 
            if (Transfer->TdsQueued != Transfer->TdsRetired)
            {
                /* QUIRK: completes later with the partial length and the hardware status */
                Transfer->BytesQueued = Transfer->BytesTotal;
                KeReleaseSpinLock(&Ring->m_Lock, OldIrql);
                return NULL;
            }
 
            DPRINT1("Bulk DCI %lu stream %lu gave up waiting for segments\n",
                    Ring->m_Endpoint->Dci(), Ring->m_StreamId);
            CompleteAfterUncancel(Ring, Transfer, USBD_STATUS_INSUFFICIENT_RESOURCES, TRUE);
 
            /* Reset the count so the next split request waits again */
            Bulk->m_SegmentWaits = 0;
        }
    }
 
    KeReleaseSpinLock(&Ring->m_Lock, OldIrql);
 
    for (;;)
    {
        Status = WdfIoQueueRetrieveNextRequest(Ring->m_Queue, &Request);
        if (!NT_SUCCESS(Status))
            return NULL;
 
        if (XhciGetRequestData(Request) == NULL)
        {
            DPRINT1("Bulk request %p has no driver context\n", Request);
            WdfRequestComplete(Request, STATUS_INVALID_DEVICE_STATE);
            continue;
        }
 
        Transfer = &XhciGetRequestData(Request)->Bulk;
        InitTransfer(Ring, Transfer, Request);
        Transfer->MappedOnce = FALSE;
        ChooseMechanism(Ring, Transfer);
 
        Status = ArrangeBuffer(Ring, Transfer);
        if (!NT_SUCCESS(Status))
        {
            DPRINT1("Bulk DCI %lu stream %lu request %p has no usable buffer, 0x%08lx\n",
                    Ring->m_Endpoint->Dci(), Ring->m_StreamId, Request, Status);
            Complete(Ring, Transfer, USBD_STATUS_INSUFFICIENT_RESOURCES);
            continue;
        }
 
        Transfer->CancelState = XhciBulkCancel::Armed;
        Status = WdfRequestMarkCancelableEx(Request, EvtRequestCancel);
        if (!NT_SUCCESS(Status))
        {
            DPRINT1("Bulk DCI %lu stream %lu request %p canceled before mapping, 0x%08lx\n",
                    Ring->m_Endpoint->Dci(), Ring->m_StreamId, Request, Status);
            Transfer->CancelState = XhciBulkCancel::Done;
            Complete(Ring, Transfer, USBD_STATUS_CANCELED);
            continue;
        }
 
        break;
    }
 
    KeAcquireSpinLock(&Ring->m_Lock, &OldIrql);
    Idle = (Bulk->m_PendingStages == 0);
    KeReleaseSpinLock(&Ring->m_Lock, OldIrql);
 
    /* Every stage on the ring shares one event ring so its events stay in order */
    if (Idle)
        Ring->UpdateInterrupterTarget(XhciTransferRing::UrbProcessorIndex((PURB)Transfer->Urb));
 
    KeAcquireSpinLock(&Ring->m_Lock, &OldIrql);
    Stage = AcquireStage(Ring, Transfer);
    InsertTailList(&Bulk->m_Pending, &Transfer->Link);
    KeReleaseSpinLock(&Ring->m_Lock, OldIrql);
 
    return Stage;
}

Referenced by FillLoop().

OnPipeHalted()

VOID XhciBulkRing::OnPipeHalted ( _In_ XhciTransferRing *  Ring )

static

Definition at line 666 of file bulkio.cpp.

{
    StreamStopOrHaltOnce(Ring, XhciBulkHalted);
}

Referenced by XhciTransferRing::OnPipeHalted().

OnTransferEvent()

BOOLEAN XhciBulkRing::OnTransferEvent ( _In_opt_ XhciTransferRing *  Ring, _In_ XhciController *  Controller, _In_ const XHCI_TRB *  Event  )

static

The ring is always known; the NULL ring form is kept only for callers without one.

Definition at line 779 of file bulkio.cpp.

{
    UNREFERENCED_PARAMETER(Controller);
 
    if (Ring == NULL)
    {
        DPRINT1("Bulk transfer event 0x%I64x without a ring, dropped\n", XhciTrbPointer(Event));
        return FALSE;
    }
 
    if (Event->Dword[3] & XHCI_TRANSFER_EVENT_ED)
        return Ed1Event(Ring, Event);
 
    return Ed0Event(Ring, Event);
}

Referenced by XhciTransferRing::OnTransferEvent().

OwnsRequests()

static BOOLEAN XhciBulkRing::OwnsRequests ( _In_ const XhciTransferRing *  Ring )

staticprivate

Referenced by Cleanup(), and Disable().

ParkOnSegments()

BOOLEAN XhciBulkRing::ParkOnSegments ( _In_ XhciTransferRing *  Ring )

staticprivate

Definition at line 1382 of file bulkio.cpp.

{
    XhciBulkRing* Bulk = &Ring->m_Type.Bulk;
    LONG Previous;
 
    Previous = InterlockedCompareExchange(&Bulk->m_SegmentWait, XhciBulkWaitParked, XhciBulkWaitIdle);
    if (Previous == XhciBulkWaitIdle)
        return TRUE;
 
    InterlockedExchange(&Bulk->m_SegmentWait, XhciBulkWaitIdle);
    return FALSE;
}

Referenced by PrimeStage().

PrimeStage()

XhciBulkRing::Outcome XhciBulkRing::PrimeStage ( _In_ XhciTransferRing *  Ring, _Inout_ XhciBulkStage *  Stage  )

staticprivate

Definition at line 1317 of file bulkio.cpp.

{
    XhciBulkRing* Bulk = &Ring->m_Type.Bulk;
    XhciBulkTransfer* Transfer = Stage->Transfer;
    ULONG Segments;
    NTSTATUS Status;
    KIRQL OldIrql;
 
    switch (Transfer->Mechanism)
    {
        case XhciBulkMechanism::Immediate:
            Stage->Buffer = Transfer->Buffer + Transfer->BytesQueued;
            break;
 
        case XhciBulkMechanism::DoubleBuffer:
            Stage->Buffer = Transfer->Buffer + Transfer->BytesQueued;
            Stage->DoubleBuffer = Transfer->DoubleBuffer;
            break;
 
        case XhciBulkMechanism::Dma:
            if (!NT_SUCCESS(AcquireStageMdl(Ring, Stage)))
                return FailStage(Ring, Stage);
            break;
 
        default:
            break;
    }
 
    SizeStage(Ring, Stage);
    EstimateTrbs(Ring, Stage);
 
    if (!EstimateSegments(Ring, Stage, &Segments))
        return FailStage(Ring, Stage);
 
    if (Segments == 0)
        return Outcome::Ready;
 
    Status = Ring->TakeSegments(Segments, &Stage->FreeSegments);
    if (Status == STATUS_PENDING)
    {
        KeAcquireSpinLock(&Ring->m_Lock, &OldIrql);
        ReleaseStage(Ring, Stage);
        KeReleaseSpinLock(&Ring->m_Lock, OldIrql);
 
        if (ParkOnSegments(Ring))
            return Outcome::WaitSegments;
 
        /* The segments already arrived; map the request again from the top */
        return Outcome::Completed;
    }
 
    if (!NT_SUCCESS(Status))
    {
        DPRINT1("Bulk DCI %lu stream %lu: %lu segments not available, 0x%08lx\n",
                Ring->m_Endpoint->Dci(), Ring->m_StreamId, Segments, Status);
        return FailStage(Ring, Stage);
    }
 
    Bulk->m_SegmentWaits = 0;
    return Outcome::Ready;
}

Referenced by FillLoop().

PutScatterGather()

static VOID XhciBulkRing::PutScatterGather ( _In_ XhciTransferRing *  Ring, _Inout_ XhciBulkStage *  Stage  )

staticprivate

Referenced by CompleteBatch(), Ed1Event(), FailStage(), and HaltedEvent().

QueueCompletionDpc()

VOID XhciBulkRing::QueueCompletionDpc ( _In_ XhciTransferRing *  Ring )

staticprivate

Definition at line 2213 of file bulkio.cpp.

{
    WdfDpcEnqueue(Ring->m_CompletionDpc);
}

Referenced by AllowReclaimOnCancel(), and RecoverTransfers().

RecoverTransfers()

VOID XhciBulkRing::RecoverTransfers ( _In_ XhciTransferRing *  Ring )

static

Definition at line 724 of file bulkio.cpp.

{
    XhciBulkRing* Bulk = &Ring->m_Type.Bulk;
    KIRQL OldIrql;
 
    KeAcquireSpinLock(&Ring->m_Lock, &OldIrql);
    XhciBulkMoveList(&Bulk->m_Completion, &Bulk->m_Pending);
    Bulk->m_Flags |= XhciBulkReclaimAck;
    KeReleaseSpinLock(&Ring->m_Lock, OldIrql);
 
    /* Only the DPC can tell that nothing is owned anymore */
    QueueCompletionDpc(Ring);
}

Referenced by XhciTransferRing::RecoverTransfers().

ReleaseParkedMapping()

VOID XhciBulkRing::ReleaseParkedMapping ( _In_ XhciTransferRing *  Ring )

static

Lets a mapping loop parked on a segment wait go, so a disable never waits on it.

Definition at line 443 of file bulkio.cpp.

{
    XhciBulkRing* Bulk = &Ring->m_Type.Bulk;
 
    /* A loop parked on a segment wait only resumes from a PASSIVE work item; let it go */
    if (InterlockedCompareExchange(&Bulk->m_SegmentWait, XhciBulkWaitAbandoned, XhciBulkWaitParked) ==
        XhciBulkWaitParked)
    {
        DPRINT1("Bulk DCI %lu stream %lu disabled while waiting for segments\n",
                Ring->m_Endpoint->Dci(), Ring->m_StreamId);
 
        if (Ring->ChangeMapState(FALSE, XhciMapState::Halting, XhciMapState::Halted) !=
            XhciMapState::Halting)
        {
            InterlockedCompareExchange(&Ring->m_MapState,
                                       static_cast<LONG>(XhciMapState::PausedFill),
                                       static_cast<LONG>(XhciMapState::Filling));
        }
    }
}

Referenced by Disable().

ReleaseStage()

static VOID XhciBulkRing::ReleaseStage ( _In_ XhciTransferRing *  Ring, _Inout_ XhciBulkStage *  Stage  )

staticprivate

Referenced by CompleteBatch(), Ed1Event(), FailStage(), HaltedEvent(), and PrimeStage().

ResumeRingFill()

VOID XhciBulkRing::ResumeRingFill ( _In_ XhciTransferRing *  Ring )

static

Definition at line 606 of file bulkio.cpp.

{
    XhciBulkRing* Bulk = &Ring->m_Type.Bulk;
    XhciMapState Previous;
    KIRQL OldIrql;
 
    KeAcquireSpinLock(&Ring->m_Lock, &OldIrql);
    Bulk->m_Flags = 0;
    Bulk->m_OutstandingEvents = 0;
    Ring->m_DoorbellRung = FALSE;
    NT_ASSERT(Bulk->m_PendingStages == 0);
    KeReleaseSpinLock(&Ring->m_Lock, OldIrql);
 
    Previous = Ring->ChangeMapState(TRUE, XhciMapState::Halted, XhciMapState::Filling);
    if (Previous == XhciMapState::Halted)
    {
        BuildTdsFromQueue(Ring);
        return;
    }
 
    DPRINT1("Bulk DCI %lu stream %lu started mapping from map state %ld\n",
            Ring->m_Endpoint->Dci(), Ring->m_StreamId, static_cast<LONG>(Previous));
}

Referenced by XhciTransferRing::ResumeRingFill().

SegmentsArrived()

VOID XhciBulkRing::SegmentsArrived ( _In_ XhciTransferRing *  Ring, _In_ NTSTATUS  Status  )

static

Common buffer callback for asynchronous segment growth.

Definition at line 799 of file bulkio.cpp.

{
    XhciBulkRing* Bulk = &Ring->m_Type.Bulk;
 
    if (!NT_SUCCESS(Status))
    {
        DPRINT1("Bulk DCI %lu stream %lu segment growth failed, 0x%08lx\n",
                Ring->m_Endpoint->Dci(), Ring->m_StreamId, Status);
    }
 
    for (;;)
    {
        /* The loop already left: resume it here, whatever the map state */
        if (InterlockedCompareExchange(&Bulk->m_SegmentWait, XhciBulkWaitIdle, XhciBulkWaitParked) ==
            XhciBulkWaitParked)
        {
            Bulk->m_SegmentWaits++;
            BuildTdsFromQueue(Ring);
            return;
        }
 
        /* The ring was disabled while waiting */
        if (InterlockedCompareExchange(&Bulk->m_SegmentWait, XhciBulkWaitIdle, XhciBulkWaitAbandoned) ==
            XhciBulkWaitAbandoned)
        {
            return;
        }
 
        /* The loop has not left yet; it sees this and retries the stage itself */
        if (InterlockedCompareExchange(&Bulk->m_SegmentWait, XhciBulkWaitArrived, XhciBulkWaitIdle) ==
            XhciBulkWaitIdle)
        {
            Bulk->m_SegmentWaits++;
            return;
        }
    }
}

SizeStage()

VOID XhciBulkRing::SizeStage ( _In_ XhciTransferRing *  Ring, _Inout_ XhciBulkStage *  Stage  )

staticprivate

Definition at line 1446 of file bulkio.cpp.

{
    XhciBulkTransfer* Transfer = Stage->Transfer;
    ULONG Remaining = Transfer->BytesTotal - Transfer->BytesQueued;
    ULONG Limit = Ring->m_MaxStageSize;
    ULONG Registers = 0;
    PDMA_ADAPTER Adapter;
    DMA_TRANSFER_INFO Info;
    PMDL Mdl;
    ULONG Left;
 
    if (Transfer->Mechanism != XhciBulkMechanism::Dma)
    {
        Stage->Size = Remaining;
        return;
    }
 
    Adapter = Ring->m_Controller->m_Buffers.DmaAdapter();
    if (Adapter->Version >= DEVICE_DESCRIPTION_VERSION3 &&
        Adapter->DmaOperations->Size > FIELD_OFFSET(DMA_OPERATIONS, GetDmaTransferInfo) &&
        Adapter->DmaOperations->GetDmaTransferInfo != NULL)
    {
        RtlZeroMemory(&Info, sizeof(Info));
        Info.Version = DMA_TRANSFER_INFO_VERSION1;
        if (NT_SUCCESS(Adapter->DmaOperations->GetDmaTransferInfo(Adapter,
                                                                  Stage->Mdl,
                                                                  0,
                                                                  Remaining,
                                                                  !XhciBulkTransferIn(Transfer),
                                                                  &Info)))
        {
            Registers = Info.V1.MapRegisterCount;
        }
    }
 
    if (Registers == 0)
    {
        /* Without the HAL query count the pages the chain spans */
        for (Mdl = Stage->Mdl, Left = Remaining; Mdl != NULL && Left != 0; Mdl = Mdl->Next)
        {
            ULONG Chunk = min(Left, MmGetMdlByteCount(Mdl));
 
            Registers += ADDRESS_AND_SIZE_TO_SPAN_PAGES(MmGetMdlVirtualAddress(Mdl), Chunk);
            Left -= Chunk;
        }
    }
 
    if (Registers > Ring->m_MapRegisterCount && Ring->m_MapRegisterCount > 1)
        Limit = min(Limit, (Ring->m_MapRegisterCount - 1) * PAGE_SIZE);
 
    Stage->Size = min(Remaining, Limit);
}

Referenced by PrimeStage().

StoppedEvent()

static VOID XhciBulkRing::StoppedEvent ( _In_ XhciTransferRing *  Ring, _Inout_opt_ XhciBulkStage *  Stage, _In_ ULONG  Code, _In_ ULONG  Bytes, _In_ ULONG  Skipped  )

staticprivate

Referenced by Ed0Event(), and Ed1Event().

StoppedEventReceived()

VOID XhciBulkRing::StoppedEventReceived ( _In_ XhciTransferRing *  Ring )

static

Definition at line 673 of file bulkio.cpp.

{
    StreamStopOrHaltOnce(Ring, XhciBulkStoppedSeen);
}

Referenced by XhciTransferRing::StoppedEventReceived().

StreamStopOrHalt()

static VOID XhciBulkRing::StreamStopOrHalt ( _In_ XhciTransferRing *  Ring )

staticprivate

Referenced by StreamStopOrHaltOnce().

StreamStopOrHaltOnce()

VOID XhciBulkRing::StreamStopOrHaltOnce ( _In_ XhciTransferRing *  Ring, _In_ ULONG  Flag  )

staticprivate

Definition at line 646 of file bulkio.cpp.

{
    XhciBulkRing* Bulk = &Ring->m_Type.Bulk;
    KIRQL OldIrql;
 
    if (Ring->m_StreamId == 0)
        return;
 
    KeAcquireSpinLock(&Ring->m_Lock, &OldIrql);
    if (!(Bulk->m_Flags & Flag))
    {
        StreamStopOrHalt(Ring);
        Bulk->m_Flags |= Flag;
    }
    KeReleaseSpinLock(&Ring->m_Lock, OldIrql);
}

Referenced by OnPipeHalted(), and StoppedEventReceived().

SuspendRingFill()

VOID XhciBulkRing::SuspendRingFill ( _In_ XhciTransferRing *  Ring )

static

Definition at line 632 of file bulkio.cpp.

{
    XhciMapState Previous;
 
    Previous = Ring->ChangeMapState(TRUE, XhciMapState::Halted, XhciMapState::Halting);
 
    /* QUIRK: a loop parked on a segment wait acknowledges only when the segments arrive */
    if (Previous == XhciMapState::PausedFill)
        Ring->ChangeMapState(FALSE, XhciMapState::Halting, XhciMapState::Halted);
}

Referenced by XhciTransferRing::SuspendRingFill().

UndoTd()

VOID XhciBulkRing::UndoTd ( _In_ XhciTransferRing *  Ring, _Inout_ XhciBulkStage *  Stage  )

staticprivate

Rewinds the ring to where a TD that could not be finished started.

Definition at line 1975 of file bulkio.cpp.

{
    KIRQL OldIrql;
 
    KeAcquireSpinLock(&Ring->m_Lock, &OldIrql);
 
    if (Ring->m_Segment != Stage->FirstSegment)
    {
        RemoveEntryList(&Stage->FirstSegment->Link);
        XhciClearListEntry(&Stage->FirstSegment->Link);
        InsertTailList(&Stage->FreeSegments, &Ring->m_Segment->Link);
        Ring->m_Segment = Stage->FirstSegment;
        Ring->m_Trbs = static_cast<PXHCI_TRB>(Stage->FirstSegment->VirtualAddress);
    }
 
    /* The first slot never became valid, so the controller cannot be past it */
    Ring->m_EnqueueIndex = Stage->FirstIndex;
    RtlZeroMemory(&Ring->m_Trbs[Stage->FirstIndex], sizeof(XHCI_TRB));
    Stage->LastSegment = Stage->FirstSegment;
    Stage->EndIndex = Stage->FirstIndex;
 
    KeReleaseSpinLock(&Ring->m_Lock, OldIrql);
}

Referenced by BuildTd().

Member Data Documentation

DmaCallback

VOID NTAPI XhciBulkRing::DmaCallback

staticprivate

Definition at line 176 of file bulkio.h.

Referenced by EncodeStage().

EvtCompletionDpc

VOID NTAPI XhciBulkRing::EvtCompletionDpc

static

Definition at line 115 of file bulkio.h.

Referenced by _IRQL_requires_().

EvtIoCanceledOnQueue

VOID NTAPI XhciBulkRing::EvtIoCanceledOnQueue

static

Definition at line 114 of file bulkio.h.

Referenced by _IRQL_requires_().

EvtRequestCancel

VOID NTAPI XhciBulkRing::EvtRequestCancel

staticprivate

Definition at line 175 of file bulkio.h.

m_Completion

LIST_ENTRY XhciBulkRing::m_Completion

private

Definition at line 356 of file bulkio.h.

m_DpcRunning

BOOLEAN XhciBulkRing::m_DpcRunning

private

Definition at line 344 of file bulkio.h.

m_FillAgain

volatile LONG XhciBulkRing::m_FillAgain

private

Interlocked DMA callback handoff

Definition at line 350 of file bulkio.h.

m_FillWanted

volatile LONG XhciBulkRing::m_FillWanted

private

Interlocked

Definition at line 351 of file bulkio.h.

m_Flags

ULONG XhciBulkRing::m_Flags

private

Definition at line 345 of file bulkio.h.

m_ForwardProgressBusy

volatile LONG XhciBulkRing::m_ForwardProgressBusy

private

Definition at line 353 of file bulkio.h.

m_ImmediateData

BOOLEAN XhciBulkRing::m_ImmediateData

private

Set at initialization and forward progress

Definition at line 343 of file bulkio.h.

m_MaxPendingStages

ULONG XhciBulkRing::m_MaxPendingStages

private

Definition at line 346 of file bulkio.h.

m_OutstandingEvents

ULONG XhciBulkRing::m_OutstandingEvents

private

Definition at line 348 of file bulkio.h.

m_Pending

LIST_ENTRY XhciBulkRing::m_Pending

private

Definition at line 354 of file bulkio.h.

m_PendingStages

ULONG XhciBulkRing::m_PendingStages

private

Definition at line 347 of file bulkio.h.

m_SegmentWait

volatile LONG XhciBulkRing::m_SegmentWait

private

Interlocked segment callback handoff

Definition at line 352 of file bulkio.h.

m_SegmentWaits

ULONG XhciBulkRing::m_SegmentWaits

private

Mapping context only

Definition at line 349 of file bulkio.h.

m_Stages

XhciBulkStage XhciBulkRing::m_Stages[StageSlots]

private

Definition at line 357 of file bulkio.h.

m_WaitingForCancel

LIST_ENTRY XhciBulkRing::m_WaitingForCancel

private

Definition at line 355 of file bulkio.h.

ReadyNotification

VOID NTAPI XhciBulkRing::ReadyNotification

static

Definition at line 113 of file bulkio.h.

Referenced by _IRQL_requires_().

StageSlots

const ULONG XhciBulkRing::StageSlots = 6

staticprivate

Five stages on the hardware plus the one being prepared.

Definition at line 164 of file bulkio.h.

Referenced by Initialize().

---

The documentation for this class was generated from the following files:

• drivers/usb/usbxhci/bulkio.h
• drivers/usb/usbxhci/bulkio.cpp