bulkio.cpp File Reference

#include "xhcidrv.h"
#include <debug.h>

Include dependency graph for bulkio.cpp:

Go to the source code of this file.

Macros
#define	NDEBUG

Enumerations
enum class	XhciBulkStep : ULONG { Normal , Pad , EventData , Done }

Functions
	C_ASSERT ((sizeof(XhciBulkStage) % 8)==0)
	C_ASSERT (alignof(XhciBulkStage) > XHCI_EVENT_DATA_TYPE_MASK)
static BOOLEAN NTAPI	XhciBulkIsHaltedCode (_In_ ULONG Code)
static BOOLEAN NTAPI	XhciBulkIsStoppedCode (_In_ ULONG Code)
static BOOLEAN NTAPI	XhciBulkEndpointIn (_In_ const XhciTransferRing *Ring)
static BOOLEAN NTAPI	XhciBulkTransferIn (_In_ const XhciBulkTransfer *Transfer)
static BOOLEAN NTAPI	XhciBulkChainedMdl (_In_ const XhciBulkTransfer *Transfer)
static ULONG NTAPI	XhciBulkWorstPages (_In_ ULONG Bytes)
static BOOLEAN NTAPI	XhciBulkSegmentHas (_In_ const XhciDmaBuffer *Segment, _In_ ULONG Start, _In_ ULONG OnePastEnd, _In_ ULONG64 Address)
static BOOLEAN NTAPI	XhciBulkNeedsNoOps (_In_ const XhciTransferRing *Ring)
static ULONG NTAPI	XhciBulkPadSize (_In_ const XhciTransferRing *Ring, _In_ ULONG Size)
static VOID NTAPI	XhciBulkCopyIn (_In_ const XhciBulkTransfer *Transfer, _In_ const XhciBulkStage *Stage, _In_ ULONG Bytes)
static VOID NTAPI	XhciBulkMoveList (_Inout_ PLIST_ENTRY Destination, _Inout_ PLIST_ENTRY Source)
static BOOLEAN NTAPI	XhciBulkScanSegment (_In_ const XhciDmaBuffer *Segment, _In_ ULONG Start, _In_ const XHCI_TRB *Event, _Inout_ PULONG Bytes)
static ULONG NTAPI	XhciBulkStageBytes (_In_ const XhciBulkStage *Stage, _In_ const XHCI_TRB *Event)
static BOOLEAN NTAPI	XhciBulkStageContains (_In_ const XhciBulkStage *Stage, _In_ ULONG64 Address)
static ULONG64 NTAPI	XhciBulkEventData (_In_ const XhciTransferRing *Ring, _In_ const XhciBulkStage *Stage)
static XhciBulkStage *NTAPI	XhciBulkStageFromEventData (_In_ ULONG64 Value)
	_Requires_lock_held_ (Ring->m_Lock) XhciBulkStage *XhciBulkRing
	_Requires_lock_not_held_ (Ring->m_Lock) VOID XhciBulkRing

Variables
static const ULONG	XhciBulkStageTag = 'gtSB'
static const ULONG	XhciBulkTransferTag = 'rTkB'
static const ULONG	XhciBulkPendingStagesMax = 5
static const ULONG	XhciBulkSegmentWaitLimit = 3
static const ULONG	XhciBulkTrbBytesMax = 0x10000
static const ULONG	XhciBulkImmediateBytes = 8
static const ULONG	XhciBulkMinStageSize = 0x10000
static const ULONG	XhciBulkNoOpCount = 8
static const USBD_STATUS	XhciBulkStatusNotSet = (USBD_STATUS)0xFFFFFFFF
static const ULONG	XhciBulkReasonEd0Invalid = 0x101F
static const ULONG	XhciBulkReasonEd0Duplicate = 0x1020
static const ULONG	XhciBulkAckExpected = 0x01
static const ULONG	XhciBulkHalted = 0x02
static const ULONG	XhciBulkStoppedSeen = 0x04
static const ULONG	XhciBulkCanceled = 0x08
static const ULONG	XhciBulkOkToReclaim = 0x10
static const ULONG	XhciBulkReclaimAck = 0x20
static const LONG	XhciBulkWaitIdle = 0
static const LONG	XhciBulkWaitParked = 1
static const LONG	XhciBulkWaitArrived = 2
static const LONG	XhciBulkWaitAbandoned = 3

Macro Definition Documentation

NDEBUG

#define NDEBUG

Definition at line 10 of file bulkio.cpp.

Enumeration Type Documentation

XhciBulkStep

enum class XhciBulkStep : ULONG

strong

Enumerator
Normal
Pad
EventData
Done

Definition at line 42 of file bulkio.cpp.

{
    Normal,
    Pad,
    EventData,
    Done
};

Function Documentation

_Requires_lock_held_()

_Requires_lock_held_

(

Ring->

m_Lock

)

Finds the stage holding a TRB address; Skipped counts the stages before it.

Stream ring accounting when another stream saw the halt or the stopped event.

Definition at line 1038 of file bulkio.cpp.

{
    XhciBulkRing* Bulk = &Ring->m_Type.Bulk;
    XhciBulkStage* Stage;
    ULONG Index;
 
    if (Transfer->Stage != NULL)
        return NULL;
 
    for (Index = 0; Index < StageSlots; Index++)
    {
        Stage = &Bulk->m_Stages[Index];
        if (Stage->InUse)
            continue;
 
        Stage->InUse = TRUE;
        Stage->OnHardware = FALSE;
        Stage->FreeMdl = FALSE;
        Stage->Generation++;
        Stage->Transfer = Transfer;
        InitializeListHead(&Stage->FreeSegments);
        InitializeListHead(&Stage->UsedSegments);
        Stage->Size = 0;
        Stage->Mdl = NULL;
        Stage->SgList = NULL;
        Stage->Buffer = NULL;
        Stage->DoubleBuffer = NULL;
        Stage->BurstTrbBudget = 0;
        Stage->TrbsReserved = 0;
        Stage->TrbsWritten = 0;
        Stage->FirstSegment = Ring->m_Segment;
        Stage->FirstIndex = Ring->m_EnqueueIndex;
        Stage->LastSegment = Ring->m_Segment;
        Stage->EndIndex = Ring->m_EnqueueIndex;
 
        Transfer->Stage = Stage;
        return Stage;
    }
 
    DPRINT1("Bulk DCI %lu stream %lu ran out of stage records\n",
            Ring->m_Endpoint->Dci(), Ring->m_StreamId);
    return NULL;
}

_Requires_lock_not_held_()

_Requires_lock_not_held_

(

Ring->

m_Lock

)

Definition at line 1114 of file bulkio.cpp.

{
    PDMA_ADAPTER Adapter;
    KIRQL OldIrql;
 
    if (Stage->SgList == NULL)
        return;
 
    /* Never under the ring lock: the HAL may run another DMA callback of this ring */
    Adapter = Ring->m_Controller->m_Buffers.DmaAdapter();
    KeRaiseIrql(DISPATCH_LEVEL, &OldIrql);
    Adapter->DmaOperations->PutScatterGatherList(Adapter,
                                                 Stage->SgList,
                                                 !XhciBulkTransferIn(Stage->Transfer));
    KeLowerIrql(OldIrql);
    Stage->SgList = NULL;
}

C_ASSERT() [1/2]

C_ASSERT

(

(sizeof(XhciBulkStage) % 8)

= =0

)

C_ASSERT() [2/2]

C_ASSERT	(	alignof(XhciBulkStage)	,
		XHCI_EVENT_DATA_TYPE_MASK
	)

XhciBulkChainedMdl()

static BOOLEAN NTAPI XhciBulkChainedMdl ( _In_ const XhciBulkTransfer *  Transfer )

static

Definition at line 111 of file bulkio.cpp.

{
    return Transfer->Urb->TransferBufferMDL != NULL &&
           Transfer->Urb->TransferBufferMDL->Next != NULL;
}

Referenced by XhciBulkRing::ChooseMechanism(), and XhciBulkRing::EstimateTrbs().

XhciBulkCopyIn()

static VOID NTAPI XhciBulkCopyIn ( _In_ const XhciBulkTransfer *  Transfer, _In_ const XhciBulkStage *  Stage, _In_ ULONG  Bytes  )

static

Definition at line 188 of file bulkio.cpp.

{
    if (Bytes != 0 &&
        Transfer->Mechanism == XhciBulkMechanism::DoubleBuffer &&
        XhciBulkTransferIn(Transfer))
    {
        RtlCopyMemory(Stage->Buffer, Stage->DoubleBuffer->VirtualAddress, Bytes);
    }
}

Referenced by XhciBulkRing::Ed1Event(), and XhciBulkRing::HaltedEvent().

XhciBulkEndpointIn()

static BOOLEAN NTAPI XhciBulkEndpointIn ( _In_ const XhciTransferRing *  Ring )

static

Definition at line 93 of file bulkio.cpp.

{
    return (Ring->m_Endpoint->Descriptor()->bEndpointAddress & USB_ENDPOINT_DIRECTION_MASK) != 0;
}

Referenced by XhciBulkRing::Initialize(), XhciBulkNeedsNoOps(), and XhciBulkPadSize().

XhciBulkEventData()

static ULONG64 NTAPI XhciBulkEventData ( _In_ const XhciTransferRing *  Ring, _In_ const XhciBulkStage *  Stage  )

static

Event Data value of a stage: its address, the endpoint type, and on 32 bit its generation.

Definition at line 333 of file bulkio.cpp.

{
    ULONG64 Value = (ULONG_PTR)Stage | (Ring->m_TransferType & XHCI_EVENT_DATA_TYPE_MASK);
 
#ifndef _WIN64
    Value |= (ULONG64)Stage->Generation << 32;
#endif
    return Value;
}

Referenced by XhciBulkRing::BuildTd().

XhciBulkIsHaltedCode()

static BOOLEAN NTAPI XhciBulkIsHaltedCode ( _In_ ULONG  Code )

static

Definition at line 60 of file bulkio.cpp.

{
    switch (static_cast<XhciCompletionCode>(Code))
    {
        case XhciCompletionCode::DataBufferError:
        case XhciCompletionCode::BabbleDetected:
        case XhciCompletionCode::UsbTransactionError:
        case XhciCompletionCode::StallError:
        case XhciCompletionCode::InvalidStreamType:
        case XhciCompletionCode::EventLost:
        case XhciCompletionCode::InvalidStreamId:
        case XhciCompletionCode::SplitTransactionError:
            return TRUE;
        default:
            return FALSE;
    }
}

Referenced by XhciBulkRing::Ed0Event(), and XhciBulkRing::Ed1Event().

XhciBulkIsStoppedCode()

static BOOLEAN NTAPI XhciBulkIsStoppedCode ( _In_ ULONG  Code )

static

Definition at line 82 of file bulkio.cpp.

{
    return Code == static_cast<ULONG>(XhciCompletionCode::Stopped) ||
           Code == static_cast<ULONG>(XhciCompletionCode::StoppedLengthInvalid) ||
           Code == static_cast<ULONG>(XhciCompletionCode::StoppedShortPacket);
}

Referenced by XhciBulkRing::Ed0Event(), and XhciBulkRing::Ed1Event().

XhciBulkMoveList()

static VOID NTAPI XhciBulkMoveList ( _Inout_ PLIST_ENTRY  Destination, _Inout_ PLIST_ENTRY  Source  )

static

Definition at line 204 of file bulkio.cpp.

{
    while (!IsListEmpty(Source))
        InsertTailList(Destination, RemoveHeadList(Source));
}

Referenced by XhciBulkRing::AllowReclaimOnCancel(), and XhciBulkRing::RecoverTransfers().

XhciBulkNeedsNoOps()

static BOOLEAN NTAPI XhciBulkNeedsNoOps ( _In_ const XhciTransferRing *  Ring )

static

Definition at line 148 of file bulkio.cpp.

{
    return Ring->m_Controller->HasErrata(XhciErrata::XferPadSsBulkInTail) &&
           Ring->m_Device->Speed() == UsbSuperSpeed &&
           Ring->m_TransferType == USB_ENDPOINT_TYPE_BULK &&
           XhciBulkEndpointIn(Ring);
}

Referenced by XhciBulkRing::BuildTd(), and XhciBulkRing::EstimateSegments().

XhciBulkPadSize()

static ULONG NTAPI XhciBulkPadSize ( _In_ const XhciTransferRing *  Ring, _In_ ULONG  Size  )

static

Bytes of the split IN rounding pad for a stage of Size bytes, 0 when none applies.

Definition at line 161 of file bulkio.cpp.

{
    const XHCI_SLOT_CONTEXT* Slot;
    USB_DEVICE_SPEED Speed = Ring->m_Device->Speed();
    ULONG MaxPacket = Ring->m_Endpoint->MaxPacketSize();
 
    if (!Ring->m_Controller->HasErrata(XhciErrata::XferPacketAlignChunks) ||
        !XhciBulkEndpointIn(Ring) ||
        (Speed != UsbLowSpeed && Speed != UsbFullSpeed) ||
        MaxPacket == 0 ||
        (Size % MaxPacket) == 0)
    {
        return 0;
    }
 
    Slot = static_cast<const XHCI_SLOT_CONTEXT*>(Ring->m_Device->OutputContext(0));
    if (Slot == NULL || Slot->TTHubSlotId == 0)
        return 0;
 
    return MaxPacket - (Size % MaxPacket);
}

Referenced by XhciBulkRing::BuildTd(), and XhciBulkRing::EstimateTrbs().

XhciBulkScanSegment()

static BOOLEAN NTAPI XhciBulkScanSegment ( _In_ const XhciDmaBuffer *  Segment, _In_ ULONG  Start, _In_ const XHCI_TRB *  Event, _Inout_ PULONG  Bytes  )

static

Scans one segment of a stage from Start for the TRB the event names, adding the bytes the stage moved before it. Returns TRUE when the TRB was found.

Definition at line 219 of file bulkio.cpp.

{
    const XHCI_TRB* Trbs = static_cast<const XHCI_TRB*>(Segment->VirtualAddress);
    ULONG Slots = Segment->Size / sizeof(XHCI_TRB);
    ULONG64 Pointer = XhciTrbPointer(Event);
    ULONG Residual = Event->Dword[2] & XHCI_TRANSFER_EVENT_LENGTH_MASK;
    ULONG Code = XhciTrbCompletionCode(Event);
    ULONG64 Address = (ULONG64)Segment->LogicalAddress.QuadPart + (ULONG64)Start * sizeof(XHCI_TRB);
    ULONG Index;
 
    for (Index = Start; Index < Slots; Index++, Address += sizeof(XHCI_TRB))
    {
        BOOLEAN Match = (Address == Pointer);
        ULONG Length = Trbs[Index].Dword[2] & XHCI_TRB_LENGTH_MASK;
 
        switch (XhciTrbType(&Trbs[Index]))
        {
            case static_cast<ULONG>(XhciTrbType::Normal):
                if (!Match)
                {
                    *Bytes += Length;
                    break;
                }
 
                switch (static_cast<XhciCompletionCode>(Code))
                {
                    case XhciCompletionCode::Stopped:
                    case XhciCompletionCode::DataBufferError:
                    case XhciCompletionCode::BabbleDetected:
                    case XhciCompletionCode::StallError:
                    case XhciCompletionCode::UsbTransactionError:
                    case XhciCompletionCode::SplitTransactionError:
                        if (Residual <= Length)
                            *Bytes += Length - Residual;
                        break;
                    case XhciCompletionCode::StoppedShortPacket:
                        *Bytes = Residual;
                        break;
                    default:
                        break;
                }
                break;
 
            case static_cast<ULONG>(XhciTrbType::EventData):
                if (!Match)
                    *Bytes = 0;
                break;
 
            case static_cast<ULONG>(XhciTrbType::Link):
                return Match;
 
            default:
                break;
        }
 
        if (Match)
            return TRUE;
    }
 
    return FALSE;
}

Referenced by XhciBulkStageBytes().

XhciBulkSegmentHas()

static BOOLEAN NTAPI XhciBulkSegmentHas ( _In_ const XhciDmaBuffer *  Segment, _In_ ULONG  Start, _In_ ULONG  OnePastEnd, _In_ ULONG64  Address  )

static

Definition at line 131 of file bulkio.cpp.

{
    ULONG64 Base = (ULONG64)Segment->LogicalAddress.QuadPart;
    ULONG Slots = Segment->Size / sizeof(XHCI_TRB);
    ULONG64 Limit;
 
    Limit = (OnePastEnd >= Slots) ? Segment->Size : (ULONG64)OnePastEnd * sizeof(XHCI_TRB);
    return Address >= Base + (ULONG64)Start * sizeof(XHCI_TRB) && Address < Base + Limit;
}

Referenced by XhciBulkStageContains().

XhciBulkStageBytes()

static ULONG NTAPI XhciBulkStageBytes ( _In_ const XhciBulkStage *  Stage, _In_ const XHCI_TRB *  Event  )

static

Definition at line 288 of file bulkio.cpp.

{
    const LIST_ENTRY* Entry;
    ULONG Bytes = 0;
    ULONG Start = Stage->FirstIndex;
 
    for (Entry = Stage->UsedSegments.Flink; Entry != &Stage->UsedSegments; Entry = Entry->Flink)
    {
        if (XhciBulkScanSegment(CONTAINING_RECORD(Entry, XhciDmaBuffer, Link), Start, Event, &Bytes))
            return Bytes;
        Start = 0;
    }
 
    if (!XhciBulkScanSegment(Stage->LastSegment, Start, Event, &Bytes))
        DPRINT1("Event TRB 0x%I64x not found while sizing its stage\n", XhciTrbPointer(Event));
 
    return Bytes;
}

Referenced by XhciBulkRing::Ed0Event().

XhciBulkStageContains()

static BOOLEAN NTAPI XhciBulkStageContains ( _In_ const XhciBulkStage *  Stage, _In_ ULONG64  Address  )

static

Definition at line 312 of file bulkio.cpp.

{
    const LIST_ENTRY* Entry;
    ULONG Start = Stage->FirstIndex;
 
    for (Entry = Stage->UsedSegments.Flink; Entry != &Stage->UsedSegments; Entry = Entry->Flink)
    {
        if (XhciBulkSegmentHas(CONTAINING_RECORD(Entry, XhciDmaBuffer, Link), Start, MAXULONG, Address))
            return TRUE;
        Start = 0;
    }
 
    return XhciBulkSegmentHas(Stage->LastSegment, Start, Stage->EndIndex, Address);
}

XhciBulkStageFromEventData()

static XhciBulkStage *NTAPI XhciBulkStageFromEventData ( _In_ ULONG64  Value )

static

Definition at line 348 of file bulkio.cpp.

{
    return reinterpret_cast<XhciBulkStage*>((ULONG_PTR)Value & ~(ULONG_PTR)XHCI_EVENT_DATA_TYPE_MASK);
}

Referenced by XhciBulkRing::Ed1Event().

XhciBulkTransferIn()

static BOOLEAN NTAPI XhciBulkTransferIn ( _In_ const XhciBulkTransfer *  Transfer )

static

Definition at line 102 of file bulkio.cpp.

{
    return (Transfer->Urb->TransferFlags & USBD_TRANSFER_DIRECTION_IN) != 0;
}

Referenced by _Requires_lock_not_held_(), XhciBulkRing::BuildTd(), XhciBulkRing::EncodeStage(), XhciBulkRing::SizeStage(), and XhciBulkCopyIn().

XhciBulkWorstPages()

static ULONG NTAPI XhciBulkWorstPages ( _In_ ULONG  Bytes )

static

Worst case number of pages Bytes can touch at any alignment.

Definition at line 122 of file bulkio.cpp.

{
    return (Bytes + 2 * (PAGE_SIZE - 1)) / PAGE_SIZE;
}

Referenced by XhciBulkRing::EnableForwardProgress(), and XhciBulkRing::EstimateTrbs().

Variable Documentation

XhciBulkAckExpected

const ULONG XhciBulkAckExpected = 0x01

static

Definition at line 29 of file bulkio.cpp.

Referenced by XhciBulkRing::ConsumePendingEvents(), XhciBulkRing::Ed1Event(), and XhciBulkRing::HaltedEvent().

XhciBulkCanceled

const ULONG XhciBulkCanceled = 0x08

static

Definition at line 32 of file bulkio.cpp.

Referenced by XhciBulkRing::AllowReclaimOnCancel().

XhciBulkHalted

const ULONG XhciBulkHalted = 0x02

static

Definition at line 30 of file bulkio.cpp.

Referenced by XhciBulkRing::HaltedEvent(), and XhciBulkRing::OnPipeHalted().

XhciBulkImmediateBytes

const ULONG XhciBulkImmediateBytes = 8

static

Definition at line 19 of file bulkio.cpp.

Referenced by XhciBulkRing::ChooseMechanism(), and XhciBulkRing::Initialize().

XhciBulkMinStageSize

const ULONG XhciBulkMinStageSize = 0x10000

static

Definition at line 20 of file bulkio.cpp.

Referenced by XhciBulkRing::Enable().

XhciBulkNoOpCount

const ULONG XhciBulkNoOpCount = 8

static

Definition at line 21 of file bulkio.cpp.

Referenced by XhciBulkRing::BuildTd().

XhciBulkOkToReclaim

const ULONG XhciBulkOkToReclaim = 0x10

static

Definition at line 33 of file bulkio.cpp.

Referenced by XhciBulkRing::AllowReclaimOnCancel().

XhciBulkPendingStagesMax

const ULONG XhciBulkPendingStagesMax = 5

static

Definition at line 16 of file bulkio.cpp.

Referenced by XhciBulkRing::Initialize().

XhciBulkReasonEd0Duplicate

const ULONG XhciBulkReasonEd0Duplicate = 0x1020

static

Definition at line 26 of file bulkio.cpp.

XhciBulkReasonEd0Invalid

const ULONG XhciBulkReasonEd0Invalid = 0x101F

static

Definition at line 25 of file bulkio.cpp.

XhciBulkReclaimAck

const ULONG XhciBulkReclaimAck = 0x20

static

Definition at line 34 of file bulkio.cpp.

Referenced by XhciBulkRing::RecoverTransfers().

XhciBulkSegmentWaitLimit

const ULONG XhciBulkSegmentWaitLimit = 3

static

Definition at line 17 of file bulkio.cpp.

Referenced by XhciBulkRing::NextTdSlot().

XhciBulkStageTag

const ULONG XhciBulkStageTag = 'gtSB'

static

Definition at line 13 of file bulkio.cpp.

Referenced by XhciBulkRing::Initialize().

XhciBulkStatusNotSet

const USBD_STATUS XhciBulkStatusNotSet = (USBD_STATUS)0xFFFFFFFF

static

Definition at line 22 of file bulkio.cpp.

Referenced by XhciBulkRing::CompleteBatch(), XhciBulkRing::Ed1Event(), and XhciBulkRing::HaltedEvent().

XhciBulkStoppedSeen

const ULONG XhciBulkStoppedSeen = 0x04

static

Definition at line 31 of file bulkio.cpp.

Referenced by XhciBulkRing::Ed0Event(), XhciBulkRing::Ed1Event(), and XhciBulkRing::StoppedEventReceived().

XhciBulkTransferTag

const ULONG XhciBulkTransferTag = 'rTkB'

static

Definition at line 14 of file bulkio.cpp.

Referenced by XhciBulkRing::InitTransfer().

XhciBulkTrbBytesMax

const ULONG XhciBulkTrbBytesMax = 0x10000

static

Definition at line 18 of file bulkio.cpp.

Referenced by XhciBulkRing::BuildTd().

XhciBulkWaitAbandoned

const LONG XhciBulkWaitAbandoned = 3

static

Definition at line 40 of file bulkio.cpp.

Referenced by XhciBulkRing::ReleaseParkedMapping(), and XhciBulkRing::SegmentsArrived().

XhciBulkWaitArrived

const LONG XhciBulkWaitArrived = 2

static

Definition at line 39 of file bulkio.cpp.

Referenced by XhciBulkRing::SegmentsArrived().

XhciBulkWaitIdle

const LONG XhciBulkWaitIdle = 0

static

Definition at line 37 of file bulkio.cpp.

Referenced by XhciBulkRing::ParkOnSegments(), and XhciBulkRing::SegmentsArrived().

XhciBulkWaitParked

const LONG XhciBulkWaitParked = 1

static

Definition at line 38 of file bulkio.cpp.

Referenced by XhciBulkRing::ParkOnSegments(), XhciBulkRing::ReleaseParkedMapping(), and XhciBulkRing::SegmentsArrived().