ReactOS 0.4.15-dev-5884-gab5aff5
xferpkt.c File Reference
#include "classp.h"
#include "debug.h"
Include dependency graph for xferpkt.c:

Go to the source code of this file.

Functions

NTSTATUS InitializeTransferPackets (PDEVICE_OBJECT Fdo)
 
VOID DestroyAllTransferPackets (PDEVICE_OBJECT Fdo)
 
 __drv_allocatesMem (Mem)
 
VOID DestroyTransferPacket (_In_ __drv_freesMem(mem) PTRANSFER_PACKET Pkt)
 
VOID EnqueueFreeTransferPacket (PDEVICE_OBJECT Fdo, __drv_aliasesMem PTRANSFER_PACKET Pkt)
 
PTRANSFER_PACKET DequeueFreeTransferPacket (PDEVICE_OBJECT Fdo, BOOLEAN AllocIfNeeded)
 
PTRANSFER_PACKET DequeueFreeTransferPacketEx (_In_ PDEVICE_OBJECT Fdo, _In_ BOOLEAN AllocIfNeeded, _In_ ULONG Node)
 
VOID SetupReadWriteTransferPacket (PTRANSFER_PACKET Pkt, PVOID Buf, ULONG Len, LARGE_INTEGER DiskLocation, PIRP OriginalIrp)
 
NTSTATUS SubmitTransferPacket (PTRANSFER_PACKET Pkt)
 
NTSTATUS NTAPI TransferPktComplete (IN PDEVICE_OBJECT NullFdo, IN PIRP Irp, IN PVOID Context)
 
VOID SetupEjectionTransferPacket (TRANSFER_PACKET *Pkt, BOOLEAN PreventMediaRemoval, PKEVENT SyncEventPtr, PIRP OriginalIrp)
 
VOID SetupModeSenseTransferPacket (TRANSFER_PACKET *Pkt, PKEVENT SyncEventPtr, PVOID ModeSenseBuffer, UCHAR ModeSenseBufferLen, UCHAR PageMode, UCHAR SubPage, PIRP OriginalIrp, UCHAR PageControl)
 
VOID SetupModeSelectTransferPacket (TRANSFER_PACKET *Pkt, PKEVENT SyncEventPtr, PVOID ModeSelectBuffer, UCHAR ModeSelectBufferLen, BOOLEAN SavePages, PIRP OriginalIrp)
 
VOID SetupDriveCapacityTransferPacket (TRANSFER_PACKET *Pkt, PVOID ReadCapacityBuffer, ULONG ReadCapacityBufferLen, PKEVENT SyncEventPtr, PIRP OriginalIrp, BOOLEAN Use16ByteCdb)
 
VOID NTAPI CleanupTransferPacketToWorkingSetSizeWorker (_In_ PVOID Fdo, _In_opt_ PVOID Context, _In_ PIO_WORKITEM IoWorkItem)
 
VOID CleanupTransferPacketToWorkingSetSize (_In_ PDEVICE_OBJECT Fdo, _In_ BOOLEAN LimitNumPktToDelete, _In_ ULONG Node)
 
 _IRQL_requires_max_ (APC_LEVEL)
 

Function Documentation

◆ __drv_allocatesMem()

__drv_allocatesMem ( Mem  )

Definition at line 322 of file xferpkt.c.

324 :28195) // This function may not allocate memory in some error cases.
325#endif
326PTRANSFER_PACKET NewTransferPacket(PDEVICE_OBJECT Fdo)
327{
328 PFUNCTIONAL_DEVICE_EXTENSION fdoExt = Fdo->DeviceExtension;
329 PCLASS_PRIVATE_FDO_DATA fdoData = fdoExt->PrivateFdoData;
330 PTRANSFER_PACKET newPkt = NULL;
331 ULONG transferLength = (ULONG)-1;
333
334 if (NT_SUCCESS(status)) {
335 status = RtlULongAdd(fdoData->HwMaxXferLen, PAGE_SIZE, &transferLength);
336 if (!NT_SUCCESS(status)) {
337
338 TracePrint((TRACE_LEVEL_ERROR, TRACE_FLAG_RW, "Integer overflow in calculating transfer packet size."));
340 }
341 }
342
343 /*
344 * Allocate the actual packet.
345 */
346 if (NT_SUCCESS(status)) {
347 newPkt = ExAllocatePoolWithTag(NonPagedPoolNx, sizeof(TRANSFER_PACKET), 'pnPC');
348 if (newPkt == NULL) {
349 TracePrint((TRACE_LEVEL_WARNING, TRACE_FLAG_RW, "Failed to allocate transfer packet."));
351 } else {
352 RtlZeroMemory(newPkt, sizeof(TRANSFER_PACKET));
354 if (fdoExt->AdapterDescriptor->SrbType == SRB_TYPE_STORAGE_REQUEST_BLOCK) {
355#if (NTDDI_VERSION >= NTDDI_WINBLUE)
356 if ((fdoExt->MiniportDescriptor != NULL) &&
357 (fdoExt->MiniportDescriptor->Size >= RTL_SIZEOF_THROUGH_FIELD(STORAGE_MINIPORT_DESCRIPTOR, ExtraIoInfoSupported)) &&
358 (fdoExt->MiniportDescriptor->ExtraIoInfoSupported == TRUE)) {
360 fdoExt->AdapterDescriptor->AddressType,
362 NULL,
363 2,
366 );
367 } else {
369 fdoExt->AdapterDescriptor->AddressType,
371 NULL,
372 1,
374 );
375 }
376#else
378 fdoExt->AdapterDescriptor->AddressType,
380 NULL,
381 1,
383 );
384#endif
385 } else {
386#ifdef _MSC_VER
387#pragma prefast(suppress:6014, "The allocated memory that Pkt->Srb points to will be freed in DestroyTransferPacket().")
388#endif
389 newPkt->Srb = ExAllocatePoolWithTag(NonPagedPoolNx, sizeof(SCSI_REQUEST_BLOCK), '-brs');
390 if (newPkt->Srb == NULL) {
392 }
393
394 }
395
396 if (status != STATUS_SUCCESS)
397 {
398 TracePrint((TRACE_LEVEL_WARNING, TRACE_FLAG_RW, "Failed to allocate SRB."));
399 FREE_POOL(newPkt);
400 }
401 }
402 }
403
404 /*
405 * Allocate Irp for the packet.
406 */
407 if (NT_SUCCESS(status) && newPkt != NULL) {
408 newPkt->Irp = IoAllocateIrp(Fdo->StackSize, FALSE);
409 if (newPkt->Irp == NULL) {
410 TracePrint((TRACE_LEVEL_WARNING, TRACE_FLAG_RW, "Failed to allocate IRP for transfer packet."));
412 }
413 }
414
415 /*
416 * Allocate a MDL. Add one page to the length to insure an extra page
417 * entry is allocated if the buffer does not start on page boundaries.
418 */
419 if (NT_SUCCESS(status) && newPkt != NULL) {
420
421 NT_ASSERT(transferLength != (ULONG)-1);
422
423 newPkt->PartialMdl = IoAllocateMdl(NULL,
424 transferLength,
425 FALSE,
426 FALSE,
427 NULL);
428 if (newPkt->PartialMdl == NULL) {
429 TracePrint((TRACE_LEVEL_WARNING, TRACE_FLAG_RW, "Failed to allocate MDL for transfer packet."));
431 } else {
432 NT_ASSERT(newPkt->PartialMdl->Size >= (CSHORT)(sizeof(MDL) + BYTES_TO_PAGES(fdoData->HwMaxXferLen) * sizeof(PFN_NUMBER)));
433 }
434
435 }
436
437 /*
438 * Allocate per-packet retry history, if required
439 */
440 if (NT_SUCCESS(status) &&
441 (fdoData->InterpretSenseInfo != NULL) &&
442 (newPkt != NULL)
443 ) {
444 // attempt to allocate also the history
445 ULONG historyByteCount = 0;
446
447 // SAL annotation and ClassInitializeEx() should both catch this case
448 NT_ASSERT(fdoData->InterpretSenseInfo->HistoryCount != 0);
449 _Analysis_assume_(fdoData->InterpretSenseInfo->HistoryCount != 0);
450
451 historyByteCount = sizeof(SRB_HISTORY_ITEM) * fdoData->InterpretSenseInfo->HistoryCount;
452 historyByteCount += sizeof(SRB_HISTORY) - sizeof(SRB_HISTORY_ITEM);
453
454 newPkt->RetryHistory = (PSRB_HISTORY)ExAllocatePoolWithTag(NonPagedPoolNx, historyByteCount, 'hrPC');
455
456 if (newPkt->RetryHistory == NULL) {
457 TracePrint((TRACE_LEVEL_WARNING, TRACE_FLAG_RW, "Failed to allocate MDL for transfer packet."));
459 } else {
460 // call this routine directly once since it's the first initialization of
461 // the structure and the internal maximum count field is not yet setup.
462 HistoryInitializeRetryLogs(newPkt->RetryHistory, fdoData->InterpretSenseInfo->HistoryCount);
463 }
464 }
465
466 /*
467 * Enqueue the packet in our static AllTransferPacketsList
468 * (just so we can find it during debugging if its stuck somewhere).
469 */
470 if (NT_SUCCESS(status) && newPkt != NULL)
471 {
472 KIRQL oldIrql;
473 newPkt->Fdo = Fdo;
474#if DBG
475 newPkt->DbgPktId = InterlockedIncrement((volatile LONG *)&fdoData->DbgMaxPktId);
476#endif
477 KeAcquireSpinLock(&fdoData->SpinLock, &oldIrql);
479 KeReleaseSpinLock(&fdoData->SpinLock, oldIrql);
480
481 } else {
482 // free any resources acquired above (in reverse order)
483 if (newPkt != NULL) {
484 FREE_POOL(newPkt->RetryHistory);
485 if (newPkt->PartialMdl != NULL) { IoFreeMdl(newPkt->PartialMdl); }
486 if (newPkt->Irp != NULL) { IoFreeIrp(newPkt->Irp); }
487 if (newPkt->Srb != NULL) { FREE_POOL(newPkt->Srb); }
488 FREE_POOL(newPkt);
489 }
490 }
491
492 return newPkt;
493}
#define InterlockedIncrement
Definition: armddk.h:53
LONG NTSTATUS
Definition: precomp.h:26
#define FREE_POOL(_PoolPtr)
Definition: cdrom.h:782
struct _SRB_HISTORY * PSRB_HISTORY
struct _SRB_HISTORY_ITEM SRB_HISTORY_ITEM
NTSTATUS CreateStorageRequestBlock(_Inout_ PSTORAGE_REQUEST_BLOCK *Srb, _In_ USHORT AddressType, _In_opt_ PSRB_ALLOCATE_ROUTINE AllocateRoutine, _Inout_opt_ ULONG *ByteSize, _In_ ULONG NumSrbExData,...)
Definition: srblib.c:249
VOID HistoryInitializeRetryLogs(_Out_ PSRB_HISTORY History, ULONG HistoryCount)
Definition: history.c:36
PVOID DefaultStorageRequestBlockAllocateRoutine(_In_ CLONG ByteSize)
Definition: srblib.c:28
#define NULL
Definition: types.h:112
#define TRUE
Definition: types.h:120
#define FALSE
Definition: types.h:117
#define NT_SUCCESS(StatCode)
Definition: apphelp.c:32
#define InsertTailList(ListHead, Entry)
#define ExAllocatePoolWithTag(hernya, size, tag)
Definition: env_spec_w32.h:350
UCHAR KIRQL
Definition: env_spec_w32.h:591
#define KeReleaseSpinLock(sl, irql)
Definition: env_spec_w32.h:627
#define PAGE_SIZE
Definition: env_spec_w32.h:49
#define KeAcquireSpinLock(sl, irql)
Definition: env_spec_w32.h:609
#define IoFreeMdl
Definition: fxmdl.h:89
#define IoAllocateMdl
Definition: fxmdl.h:88
NTKRNLVISTAAPI USHORT NTAPI KeGetCurrentNodeNumber()
Definition: ke.c:40
#define _Analysis_assume_(expr)
Definition: ms_sal.h:2901
#define RTL_SIZEOF_THROUGH_FIELD(type, field)
Definition: ntbasedef.h:672
PIRP NTAPI IoAllocateIrp(IN CCHAR StackSize, IN BOOLEAN ChargeQuota)
Definition: irp.c:615
VOID NTAPI IoFreeIrp(IN PIRP Irp)
Definition: irp.c:1666
#define STATUS_INTEGER_OVERFLOW
Definition: ntstatus.h:385
long LONG
Definition: pedump.c:60
#define SRB_TYPE_STORAGE_REQUEST_BLOCK
Definition: srb.h:655
* PSTORAGE_REQUEST_BLOCK
Definition: srb.h:652
@ SrbExDataTypeScsiCdb16
Definition: srb.h:450
@ SrbExDataTypeIoInfo
Definition: srb.h:456
ULONG PFN_NUMBER
Definition: ke.h:9
#define STATUS_SUCCESS
Definition: shellext.h:65
#define TRACE_LEVEL_WARNING
Definition: storswtr.h:28
#define TRACE_LEVEL_ERROR
Definition: storswtr.h:27
PCLASS_INTERPRET_SENSE_INFO2 InterpretSenseInfo
Definition: classp.h:958
KSPIN_LOCK SpinLock
Definition: classp.h:795
LIST_ENTRY AllTransferPacketsList
Definition: classp.h:771
PSTORAGE_ADAPTER_DESCRIPTOR AdapterDescriptor
Definition: classpnp.h:877
PSTORAGE_REQUEST_BLOCK_HEADER Srb
Definition: classp.h:580
PMDL PartialMdl
Definition: classp.h:596
PSRB_HISTORY RetryHistory
Definition: classp.h:598
PDEVICE_OBJECT Fdo
Definition: classp.h:503
ULONG AllocateNode
Definition: classp.h:611
LIST_ENTRY AllPktsListEntry
Definition: classp.h:499
Definition: ps.c:97
#define RtlZeroMemory(Destination, Length)
Definition: typedefs.h:262
uint32_t ULONG
Definition: typedefs.h:59
#define STATUS_INSUFFICIENT_RESOURCES
Definition: udferr_usr.h:158
short CSHORT
Definition: umtypes.h:127
_Must_inspect_result_ _In_ WDFDEVICE Fdo
Definition: wdffdo.h:461
#define BYTES_TO_PAGES(Size)
MDL
Definition: mmtypes.h:117
#define NT_ASSERT
Definition: rtlfuncs.h:3310

◆ _IRQL_requires_max_()

_IRQL_requires_max_ ( APC_LEVEL  )

Definition at line 1693 of file xferpkt.c.

1728{
1731 PCDB pCdb;
1732 ULONG srbLength;
1733
1734 PAGED_CODE();
1735
1736 TracePrint((TRACE_LEVEL_VERBOSE,
1737 TRACE_FLAG_IOCTL,
1738 "ClasspSetupPopulateTokenTransferPacket (%p): Entering function. Irp %p\n",
1739 Pkt->Fdo,
1740 OriginalIrp));
1741
1742 fdoExt = Pkt->Fdo->DeviceExtension;
1743 fdoData = fdoExt->PrivateFdoData;
1744
1745 if (fdoExt->AdapterDescriptor->SrbType == SRB_TYPE_STORAGE_REQUEST_BLOCK) {
1746 srbLength = ((PSTORAGE_REQUEST_BLOCK) fdoData->SrbTemplate)->SrbLength;
1747 NT_ASSERT(((PSTORAGE_REQUEST_BLOCK) Pkt->Srb)->SrbLength >= srbLength);
1748 } else {
1749 srbLength = fdoData->SrbTemplate->Length;
1750 }
1751 RtlCopyMemory(Pkt->Srb, fdoData->SrbTemplate, srbLength); // copies _contents_ of SRB blocks
1752
1754 SrbSetCdbLength(Pkt->Srb, 16);
1759 SrbSetDataBuffer(Pkt->Srb, PopulateTokenBuffer);
1761
1763
1764 pCdb = SrbGetCdb(Pkt->Srb);
1765 if (pCdb) {
1766 pCdb->TOKEN_OPERATION.OperationCode = SCSIOP_POPULATE_TOKEN;
1767 pCdb->TOKEN_OPERATION.ServiceAction = SERVICE_ACTION_POPULATE_TOKEN;
1768
1769 REVERSE_BYTES(&pCdb->TOKEN_OPERATION.ListIdentifier, &ListIdentifier);
1770 REVERSE_BYTES(&pCdb->TOKEN_OPERATION.ParameterListLength, &Length);
1771 }
1772
1773 Pkt->BufPtrCopy = PopulateTokenBuffer;
1775
1779
1781 Pkt->ContinuationContext = OffloadReadContext;
1782
1783 TracePrint((TRACE_LEVEL_VERBOSE,
1784 TRACE_FLAG_IOCTL,
1785 "ClasspSetupPopulateTokenTransferPacket (%p): Exiting function with Irp %p\n",
1786 Pkt->Fdo,
1787 OriginalIrp));
1788
1789 return;
1790}
#define PAGED_CODE()
_In_ PTRANSFER_PACKET _In_ ULONG _In_ PIRP OriginalIrp
Definition: classp.h:1757
_In_ PTRANSFER_PACKET _In_ ULONG _In_ PIRP _In_ ULONG ListIdentifier
Definition: classp.h:1759
_In_ PTRANSFER_PACKET Pkt
Definition: classp.h:1754
#define NUM_POPULATE_TOKEN_RETRIES
Definition: classp.h:204
VOID ClasspPopulateTokenTransferPacketDone(_In_ PVOID Context)
Definition: class.c:14166
#define SRB_FLAGS_DATA_OUT
Definition: srb.h:401
#define SRB_SIMPLE_TAG_REQUEST
Definition: srb.h:423
#define SRB_FLAGS_DISABLE_SYNCH_TRANSFER
Definition: srb.h:397
#define SRB_FLAGS_NO_QUEUE_FREEZE
Definition: srb.h:404
_In_ ULONG _In_ ULONG _In_ ULONG Length
Definition: ntddpcm.h:102
#define SERVICE_ACTION_POPULATE_TOKEN
Definition: scsi.h:442
#define SCSIOP_POPULATE_TOKEN
Definition: scsi.h:340
#define REVERSE_BYTES(Destination, Source)
Definition: scsi.h:3465
FORCEINLINE VOID SrbSetRequestAttribute(_In_ PVOID Srb, _In_ UCHAR RequestAttribute)
Definition: srbhelper.h:1131
FORCEINLINE VOID SrbAssignSrbFlags(_In_ PVOID Srb, _In_ ULONG Flags)
Definition: srbhelper.h:946
FORCEINLINE VOID SrbSetDataTransferLength(_In_ PVOID Srb, _In_ ULONG DataTransferLength)
Definition: srbhelper.h:784
FORCEINLINE VOID SrbSetOriginalRequest(_In_ PVOID Srb, _In_opt_ PVOID OriginalRequest)
Definition: srbhelper.h:710
FORCEINLINE VOID SrbSetSenseInfoBufferLength(_In_ PVOID Srb, _In_ UCHAR SenseInfoBufferLength)
Definition: srbhelper.h:675
FORCEINLINE VOID SrbSetSenseInfoBuffer(_In_ PVOID Srb, _In_opt_ PVOID SenseInfoBuffer)
Definition: srbhelper.h:657
FORCEINLINE VOID SrbSetDataBuffer(_In_ PVOID Srb, _In_opt_ __drv_aliasesMem PVOID DataBuffer)
Definition: srbhelper.h:747
FORCEINLINE VOID SrbSetCdbLength(_In_ PVOID Srb, _In_ UCHAR CdbLength)
Definition: srbhelper.h:1093
FORCEINLINE VOID SrbSetTimeOutValue(_In_ PVOID Srb, _In_ ULONG TimeOutValue)
Definition: srbhelper.h:821
#define TRACE_LEVEL_VERBOSE
Definition: storswtr.h:30
PSTORAGE_REQUEST_BLOCK_HEADER SrbTemplate
Definition: classp.h:790
PVOID DeviceExtension
Definition: env_spec_w32.h:418
UCHAR NumRetries
Definition: classp.h:516
SENSE_DATA_EX SrbErrorSenseData
Definition: classp.h:570
ULONG BufLenCopy
Definition: classp.h:563
PIRP OriginalIrp
Definition: classp.h:509
BOOLEAN CompleteOriginalIrpWhenLastPacketCompletes
Definition: classp.h:510
PVOID ContinuationContext
Definition: classp.h:609
PUCHAR BufPtrCopy
Definition: classp.h:562
PCONTINUATION_ROUTINE ContinuationRoutine
Definition: classp.h:608
#define RtlCopyMemory(Destination, Source, Length)
Definition: typedefs.h:263
Definition: cdrw_hw.h:28
struct _CDB::_TOKEN_OPERATION TOKEN_OPERATION
#define SrbGetCdb(srb)
Definition: usbstor.h:17

◆ CleanupTransferPacketToWorkingSetSize()

VOID CleanupTransferPacketToWorkingSetSize ( _In_ PDEVICE_OBJECT  Fdo,
_In_ BOOLEAN  LimitNumPktToDelete,
_In_ ULONG  Node 
)

Definition at line 1615 of file xferpkt.c.

1634{
1635 PFUNCTIONAL_DEVICE_EXTENSION fdoExt = Fdo->DeviceExtension;
1636 PCLASS_PRIVATE_FDO_DATA fdoData = fdoExt->PrivateFdoData;
1637 KIRQL oldIrql;
1638 SINGLE_LIST_ENTRY pktList;
1639 PSINGLE_LIST_ENTRY slistEntry;
1640 PTRANSFER_PACKET pktToDelete;
1641 ULONG requiredNumPktToDelete = fdoData->FreeTransferPacketsLists[Node].NumTotalTransferPackets -
1643
1644 if (LimitNumPktToDelete) {
1645 requiredNumPktToDelete = MIN(requiredNumPktToDelete, MAX_CLEANUP_TRANSFER_PACKETS_AT_ONCE);
1646 }
1647
1648 TracePrint((TRACE_LEVEL_INFORMATION, TRACE_FLAG_RW, "CleanupTransferPacketToWorkingSetSize (%p): Exiting stress, block freeing %d packets.", Fdo, requiredNumPktToDelete));
1649
1650 /*
1651 * Check the counter again with lock held. This eliminates a race condition
1652 * while still allowing us to not grab the spinlock in the common codepath.
1653 *
1654 * Note that the spinlock does not synchronize with threads dequeuing free
1655 * packets to send (DequeueFreeTransferPacket does that with a lightweight
1656 * interlocked exchange); the spinlock prevents multiple threads in this function
1657 * from deciding to free too many extra packets at once.
1658 */
1659 SimpleInitSlistHdr(&pktList);
1660 KeAcquireSpinLock(&fdoData->SpinLock, &oldIrql);
1663 (requiredNumPktToDelete--)){
1664
1665 pktToDelete = DequeueFreeTransferPacketEx(Fdo, FALSE, Node);
1666 if (pktToDelete){
1667 SimplePushSlist(&pktList,
1668 (PSINGLE_LIST_ENTRY)&pktToDelete->SlistEntry);
1670 } else {
1671 TracePrint((TRACE_LEVEL_INFORMATION, TRACE_FLAG_RW,
1672 "Extremely unlikely condition (non-fatal): %d packets dequeued at once for Fdo %p. NumTotalTransferPackets=%d (1). Node=%d",
1674 Fdo,
1676 Node));
1677 break;
1678 }
1679 }
1680 KeReleaseSpinLock(&fdoData->SpinLock, oldIrql);
1681
1682 slistEntry = SimplePopSlist(&pktList);
1683 while (slistEntry) {
1684 pktToDelete = CONTAINING_RECORD(slistEntry, TRANSFER_PACKET, SlistEntry);
1685 DestroyTransferPacket(pktToDelete);
1686 slistEntry = SimplePopSlist(&pktList);
1687 }
1688
1689 return;
1690}
#define InterlockedDecrement
Definition: armddk.h:52
#define MIN(x, y)
Definition: rdesktop.h:171
FORCEINLINE VOID SimpleInitSlistHdr(SINGLE_LIST_ENTRY *SListHdr)
Definition: classp.h:1211
#define MAX_CLEANUP_TRANSFER_PACKETS_AT_ONCE
Definition: classp.h:639
FORCEINLINE SINGLE_LIST_ENTRY * SimplePopSlist(SINGLE_LIST_ENTRY *SListHdr)
Definition: classp.h:1220
FORCEINLINE VOID SimplePushSlist(SINGLE_LIST_ENTRY *SListHdr, SINGLE_LIST_ENTRY *SListEntry)
Definition: classp.h:1215
union node Node
Definition: types.h:1255
#define TRACE_LEVEL_INFORMATION
Definition: storswtr.h:29
ULONG LocalMaxWorkingSetTransferPackets
Definition: classp.h:712
PPNL_SLIST_HEADER FreeTransferPacketsLists
Definition: classp.h:772
DECLSPEC_CACHEALIGN ULONG NumFreeTransferPackets
Definition: classp.h:645
ULONG NumTotalTransferPackets
Definition: classp.h:646
Definition: ntbasedef.h:628
SLIST_ENTRY SlistEntry
Definition: classp.h:500
#define CONTAINING_RECORD(address, type, field)
Definition: typedefs.h:260
Definition: dlist.c:348
VOID DestroyTransferPacket(_In_ __drv_freesMem(mem) PTRANSFER_PACKET Pkt)
Definition: xferpkt.c:500
PTRANSFER_PACKET DequeueFreeTransferPacketEx(_In_ PDEVICE_OBJECT Fdo, _In_ BOOLEAN AllocIfNeeded, _In_ ULONG Node)
Definition: xferpkt.c:661

Referenced by CleanupTransferPacketToWorkingSetSizeWorker(), and EnqueueFreeTransferPacket().

◆ CleanupTransferPacketToWorkingSetSizeWorker()

VOID NTAPI CleanupTransferPacketToWorkingSetSizeWorker ( _In_ PVOID  Fdo,
_In_opt_ PVOID  Context,
_In_ PIO_WORKITEM  IoWorkItem 
)

Definition at line 1591 of file xferpkt.c.

1596{
1598
1599 PAGED_CODE();
1600
1602
1603 //
1604 // Release the remove lock acquired in EnqueueFreeTransferPacket
1605 //
1607
1608 if (IoWorkItem != NULL) {
1610 }
1611}
VOID NTAPI ClassReleaseRemoveLock(_In_ PDEVICE_OBJECT DeviceObject, _In_opt_ PIRP Tag)
Definition: lock.c:251
VOID NTAPI IoFreeWorkItem(IN PIO_WORKITEM IoWorkItem)
Definition: iowork.c:64
uint32_t ULONG_PTR
Definition: typedefs.h:65
_In_opt_ PVOID _In_ PIO_WORKITEM IoWorkItem
Definition: iotypes.h:521
VOID CleanupTransferPacketToWorkingSetSize(_In_ PDEVICE_OBJECT Fdo, _In_ BOOLEAN LimitNumPktToDelete, _In_ ULONG Node)
Definition: xferpkt.c:1615

◆ DequeueFreeTransferPacket()

◆ DequeueFreeTransferPacketEx()

PTRANSFER_PACKET DequeueFreeTransferPacketEx ( _In_ PDEVICE_OBJECT  Fdo,
_In_ BOOLEAN  AllocIfNeeded,
_In_ ULONG  Node 
)

Definition at line 661 of file xferpkt.c.

665{
666 PFUNCTIONAL_DEVICE_EXTENSION fdoExt = Fdo->DeviceExtension;
667 PCLASS_PRIVATE_FDO_DATA fdoData = fdoExt->PrivateFdoData;
669 PSLIST_ENTRY slistEntry;
670
672
673 if (slistEntry) {
674 slistEntry->Next = NULL;
675 pkt = CONTAINING_RECORD(slistEntry, TRANSFER_PACKET, SlistEntry);
677
678 // when dequeuing the packet, also reset the history data
680
681 } else {
682 if (AllocIfNeeded) {
683 /*
684 * We are in stress and have run out of lookaside packets.
685 * In order to service the current transfer,
686 * allocate an extra packet.
687 * We will free it lazily when we are out of stress.
688 */
689 pkt = NewTransferPacket(Fdo);
690 if (pkt) {
695 } else {
696 TracePrint((TRACE_LEVEL_WARNING, TRACE_FLAG_RW, "DequeueFreeTransferPacket: packet allocation failed"));
697 }
698 } else {
699 pkt = NULL;
700 }
701 }
702
703 return pkt;
704}
#define HISTORYINITIALIZERETRYLOGS(_packet)
Definition: classp.h:2330
ULONG DbgPeakNumTransferPackets
Definition: classp.h:647
DECLSPEC_CACHEALIGN SLIST_HEADER SListHeader
Definition: classp.h:644
#define max(a, b)
Definition: svc.c:63
#define InterlockedPopEntrySList(SListHead)
Definition: rtlfuncs.h:3392
#define PSLIST_ENTRY
Definition: rtltypes.h:134

Referenced by CleanupTransferPacketToWorkingSetSize(), DequeueFreeTransferPacket(), DestroyAllTransferPackets(), and EnqueueFreeTransferPacket().

◆ DestroyAllTransferPackets()

VOID DestroyAllTransferPackets ( PDEVICE_OBJECT  Fdo)

Definition at line 288 of file xferpkt.c.

289{
290 PFUNCTIONAL_DEVICE_EXTENSION fdoExt = Fdo->DeviceExtension;
291 PCLASS_PRIVATE_FDO_DATA fdoData = fdoExt->PrivateFdoData;
292 TRANSFER_PACKET *pkt;
293 ULONG index;
294 ULONG arraySize;
295
296 PAGED_CODE();
297
298 //
299 // fdoData->FreeTransferPacketsLists could be NULL if
300 // there was an error during start device.
301 //
302 if (fdoData->FreeTransferPacketsLists != NULL) {
303
305
306 arraySize = KeQueryHighestNodeNumber() + 1;
307 for (index = 0; index < arraySize; index++) {
309 while (pkt) {
313 }
314
316 }
317 }
318
319 FREE_POOL(fdoData->SrbTemplate);
320}
#define index(s, c)
Definition: various.h:29
#define IsListEmpty(ListHead)
Definition: env_spec_w32.h:954
GLuint index
Definition: glext.h:6031
NTKRNLVISTAAPI USHORT NTAPI KeQueryHighestNodeNumber()
Definition: ke.c:32
LIST_ENTRY DeferredClientIrpList
Definition: classp.h:777

◆ DestroyTransferPacket()

VOID DestroyTransferPacket ( _In_ __drv_freesMem(mem) PTRANSFER_PACKET  Pkt)

Definition at line 500 of file xferpkt.c.

501{
503 PCLASS_PRIVATE_FDO_DATA fdoData = fdoExt->PrivateFdoData;
504 KIRQL oldIrql;
505
506 NT_ASSERT(!Pkt->SlistEntry.Next);
507// NT_ASSERT(!Pkt->OriginalIrp);
508
509 KeAcquireSpinLock(&fdoData->SpinLock, &oldIrql);
510
511 /*
512 * Delete the packet from our all-packets queue.
513 */
518
519 KeReleaseSpinLock(&fdoData->SpinLock, oldIrql);
520
522 IoFreeIrp(Pkt->Irp);
524 FREE_POOL(Pkt->Srb);
525 FREE_POOL(Pkt);
526}
#define RemoveEntryList(Entry)
Definition: env_spec_w32.h:986
#define InitializeListHead(ListHead)
Definition: env_spec_w32.h:944

Referenced by CleanupTransferPacketToWorkingSetSize(), DestroyAllTransferPackets(), and EnqueueFreeTransferPacket().

◆ EnqueueFreeTransferPacket()

VOID EnqueueFreeTransferPacket ( PDEVICE_OBJECT  Fdo,
__drv_aliasesMem PTRANSFER_PACKET  Pkt 
)

Definition at line 529 of file xferpkt.c.

530{
531 PFUNCTIONAL_DEVICE_EXTENSION fdoExt = Fdo->DeviceExtension;
532 PCLASS_PRIVATE_FDO_DATA fdoData = fdoExt->PrivateFdoData;
533 ULONG allocateNode;
534 KIRQL oldIrql;
535
536 NT_ASSERT(!Pkt->SlistEntry.Next);
537
538 allocateNode = Pkt->AllocateNode;
540 InterlockedIncrement((volatile LONG *)&(fdoData->FreeTransferPacketsLists[allocateNode].NumFreeTransferPackets));
541
542 /*
543 * If the total number of packets is larger than LocalMinWorkingSetTransferPackets,
544 * that means that we've been in stress. If all those packets are now
545 * free, then we are now out of stress and can free the extra packets.
546 * Attempt to free down to LocalMaxWorkingSetTransferPackets immediately, and
547 * down to LocalMinWorkingSetTransferPackets lazily (one at a time).
548 * However, since we're at DPC, do this is a work item. If the device is removed
549 * or we are unable to allocate the work item, do NOT free more than
550 * MAX_CLEANUP_TRANSFER_PACKETS_AT_ONCE. Subsequent IO completions will end up freeing
551 * up the rest, even if it is MAX_CLEANUP_TRANSFER_PACKETS_AT_ONCE at a time.
552 */
553 if (fdoData->FreeTransferPacketsLists[allocateNode].NumFreeTransferPackets >=
554 fdoData->FreeTransferPacketsLists[allocateNode].NumTotalTransferPackets) {
555
556 /*
557 * 1. Immediately snap down to our UPPER threshold.
558 */
559 if (fdoData->FreeTransferPacketsLists[allocateNode].NumTotalTransferPackets >
561
562 ULONG isRemoved;
563 PIO_WORKITEM workItem = NULL;
564
565 workItem = IoAllocateWorkItem(Fdo);
566
567 //
568 // Acquire a remove lock in order to make sure the device object and its
569 // private data structures will exist when the workitem fires.
570 // The remove lock will be released by the workitem (CleanupTransferPacketToWorkingSetSize).
571 //
572 isRemoved = ClassAcquireRemoveLock(Fdo, (PIRP)workItem);
573
574 if (workItem && !isRemoved) {
575
576 TracePrint((TRACE_LEVEL_INFORMATION,
577 TRACE_FLAG_GENERAL,
578 "EnqueueFreeTransferPacket: Device (%p), queuing work item to clean up free transfer packets.\n",
579 Fdo));
580
581 //
582 // Queue a work item to trim down the total number of transfer packets to with the
583 // working size.
584 //
586
587 } else {
588
589 if (workItem) {
590 IoFreeWorkItem(workItem);
591 }
592
593 if (isRemoved != REMOVE_COMPLETE) {
594 ClassReleaseRemoveLock(Fdo, (PIRP)workItem);
595 }
596
597 TracePrint((TRACE_LEVEL_ERROR,
598 TRACE_FLAG_GENERAL,
599 "EnqueueFreeTransferPacket: Device (%p), Failed to allocate memory for the work item.\n",
600 Fdo));
601
603 }
604 }
605
606 /*
607 * 2. Lazily work down to our LOWER threshold (by only freeing one packet at a time).
608 */
609 if (fdoData->FreeTransferPacketsLists[allocateNode].NumTotalTransferPackets >
611 /*
612 * Check the counter again with lock held. This eliminates a race condition
613 * while still allowing us to not grab the spinlock in the common codepath.
614 *
615 * Note that the spinlock does not synchronize with threads dequeuing free
616 * packets to send (DequeueFreeTransferPacket does that with a lightweight
617 * interlocked exchange); the spinlock prevents multiple threads in this function
618 * from deciding to free too many extra packets at once.
619 */
620 PTRANSFER_PACKET pktToDelete = NULL;
621
622 TracePrint((TRACE_LEVEL_INFORMATION, TRACE_FLAG_RW, "Exiting stress, lazily freeing one of %d/%d packets from node %d.",
625 allocateNode));
626
627 KeAcquireSpinLock(&fdoData->SpinLock, &oldIrql);
628 if ((fdoData->FreeTransferPacketsLists[allocateNode].NumFreeTransferPackets >=
629 fdoData->FreeTransferPacketsLists[allocateNode].NumTotalTransferPackets) &&
630 (fdoData->FreeTransferPacketsLists[allocateNode].NumTotalTransferPackets >
632
633 pktToDelete = DequeueFreeTransferPacketEx(Fdo, FALSE, allocateNode);
634 if (pktToDelete) {
635 InterlockedDecrement((volatile LONG *)&(fdoData->FreeTransferPacketsLists[allocateNode].NumTotalTransferPackets));
636 } else {
637 TracePrint((TRACE_LEVEL_INFORMATION, TRACE_FLAG_RW,
638 "Extremely unlikely condition (non-fatal): %d packets dequeued at once for Fdo %p. NumTotalTransferPackets=%d (2). Node=%d",
640 Fdo,
642 allocateNode));
643 }
644 }
645 KeReleaseSpinLock(&fdoData->SpinLock, oldIrql);
646
647 if (pktToDelete) {
648 DestroyTransferPacket(pktToDelete);
649 }
650 }
651
652 }
653
654}
IO_WORKITEM_ROUTINE_EX CleanupTransferPacketToWorkingSetSizeWorker
Definition: classp.h:1745
#define ClassAcquireRemoveLock(devobj, tag)
Definition: classpnp.h:100
#define REMOVE_COMPLETE
Definition: classpnp.h:98
PIO_WORKITEM NTAPI IoAllocateWorkItem(IN PDEVICE_OBJECT DeviceObject)
Definition: iowork.c:75
NTKRNLVISTAAPI VOID NTAPI IoQueueWorkItemEx(_Inout_ PIO_WORKITEM IoWorkItem, _In_ PIO_WORKITEM_ROUTINE_EX WorkerRoutine, _In_ WORK_QUEUE_TYPE QueueType, _In_opt_ __drv_aliasesMem PVOID Context)
Definition: io.c:55
ULONG LocalMinWorkingSetTransferPackets
Definition: classp.h:711
@ DelayedWorkQueue
Definition: extypes.h:190
#define InterlockedPushEntrySList(SListHead, SListEntry)
Definition: rtlfuncs.h:3389

Referenced by InitializeTransferPackets(), ServiceTransferRequest(), and TransferPktComplete().

◆ InitializeTransferPackets()

NTSTATUS InitializeTransferPackets ( PDEVICE_OBJECT  Fdo)

Definition at line 45 of file xferpkt.c.

46{
47 PCOMMON_DEVICE_EXTENSION commonExt = Fdo->DeviceExtension;
48 PFUNCTIONAL_DEVICE_EXTENSION fdoExt = Fdo->DeviceExtension;
49 PCLASS_PRIVATE_FDO_DATA fdoData = fdoExt->PrivateFdoData;
50 PSTORAGE_ADAPTER_DESCRIPTOR adapterDesc = commonExt->PartitionZeroExtension->AdapterDescriptor;
52 STORAGE_PROPERTY_ID propertyId;
53 OSVERSIONINFOEXW osVersionInfo;
54 ULONG hwMaxPages;
55 ULONG arraySize;
57 ULONG maxOutstandingIOPerLUN;
58 ULONG minWorkingSetTransferPackets;
59 ULONG maxWorkingSetTransferPackets;
60
62
63 PAGED_CODE();
64
65 //
66 // Precompute the maximum transfer length
67 //
68 NT_ASSERT(adapterDesc->MaximumTransferLength);
69
70 hwMaxPages = adapterDesc->MaximumPhysicalPages ? adapterDesc->MaximumPhysicalPages-1 : 0;
71
72 fdoData->HwMaxXferLen = MIN(adapterDesc->MaximumTransferLength, hwMaxPages << PAGE_SHIFT);
73 fdoData->HwMaxXferLen = MAX(fdoData->HwMaxXferLen, PAGE_SIZE);
74
75 //
76 // Allocate per-node free packet lists
77 //
78 arraySize = KeQueryHighestNodeNumber() + 1;
80 ExAllocatePoolWithTag(NonPagedPoolNxCacheAligned,
81 sizeof(PNL_SLIST_HEADER) * arraySize,
83
84 if (fdoData->FreeTransferPacketsLists == NULL) {
86 return status;
87 }
88
89 for (index = 0; index < arraySize; index++) {
93 }
94
96
97 //
98 // Set the packet threshold numbers based on the Windows Client or Server SKU.
99 //
100
101 osVersionInfo.dwOSVersionInfoSize = sizeof(OSVERSIONINFOEXW);
102 status = RtlGetVersion((POSVERSIONINFOW) &osVersionInfo);
103
105
106 //
107 // Retrieve info on IO capability supported by port drivers
108 //
109
111 status = ClassGetDescriptor(fdoExt->CommonExtension.LowerDeviceObject,
112 &propertyId,
113 (PVOID *)&devIoCapabilityDesc);
114
115 if (NT_SUCCESS(status) && (devIoCapabilityDesc != NULL)) {
116 maxOutstandingIOPerLUN = devIoCapabilityDesc->LunMaxIoCount;
117 FREE_POOL(devIoCapabilityDesc);
118
119#if DBG
120 fdoData->MaxOutstandingIOPerLUN = maxOutstandingIOPerLUN;
121#endif
122
123 } else {
124 maxOutstandingIOPerLUN = MAX_OUTSTANDING_IO_PER_LUN_DEFAULT;
125
126#if DBG
127 fdoData->MaxOutstandingIOPerLUN = 0;
128#endif
129
130 }
131
132 //
133 // StorageDeviceIoCapabilityProperty support is optional so
134 // ignore any failures.
135 //
136
138
139
140 if ((osVersionInfo.wProductType != VER_NT_DOMAIN_CONTROLLER) &&
141 (osVersionInfo.wProductType != VER_NT_SERVER)) {
142
143 // this is Client SKU
144
145 minWorkingSetTransferPackets = MIN_WORKINGSET_TRANSFER_PACKETS_Client;
146
147 // Note: the reason we use max here is to guarantee a reasonable large max number
148 // in the case where the port driver may return a very small supported outstanding
149 // IOs. For example, even EMMC drive only reports 1 outstanding IO supported, we
150 // may still want to set this value to be at least
151 // MAX_WORKINGSET_TRANSFER_PACKETS_Client.
152 maxWorkingSetTransferPackets = max(MAX_WORKINGSET_TRANSFER_PACKETS_Client,
153 2 * maxOutstandingIOPerLUN);
154
155 } else {
156
157 // this is Server SKU
158 // Note: the addition max here to make sure we set the min to be at least
159 // MIN_WORKINGSET_TRANSFER_PACKETS_Server_LowerBound no matter what maxOutstandingIOPerLUN
160 // reported. We shouldn't set this value to be smaller than client system.
161 // In other words, the minWorkingSetTransferPackets for server will always between
162 // MIN_WORKINGSET_TRANSFER_PACKETS_Server_LowerBound and MIN_WORKINGSET_TRANSFER_PACKETS_Server_UpperBound
163
164 minWorkingSetTransferPackets =
167 maxOutstandingIOPerLUN));
168
169 maxWorkingSetTransferPackets = max(MAX_WORKINGSET_TRANSFER_PACKETS_Server,
170 2 * maxOutstandingIOPerLUN);
171 }
172
173
174 fdoData->LocalMinWorkingSetTransferPackets = minWorkingSetTransferPackets;
175 fdoData->LocalMaxWorkingSetTransferPackets = maxWorkingSetTransferPackets;
176
177 //
178 // Allow class driver to override the settings
179 //
180 if (commonExt->DriverExtension->WorkingSet != NULL) {
181 PCLASS_WORKING_SET workingSet = commonExt->DriverExtension->WorkingSet;
182
183 // override only if non-zero
184 if (workingSet->XferPacketsWorkingSetMinimum != 0)
185 {
186 fdoData->LocalMinWorkingSetTransferPackets = workingSet->XferPacketsWorkingSetMinimum;
187 // adjust maximum upwards if needed
189 {
191 }
192 }
193 // override only if non-zero
194 if (workingSet->XferPacketsWorkingSetMaximum != 0)
195 {
196 fdoData->LocalMaxWorkingSetTransferPackets = workingSet->XferPacketsWorkingSetMaximum;
197 // adjust minimum downwards if needed
199 {
201 }
202 }
203 // that's all the adjustments required/allowed
204 } // end working set size special code
205
206 for (index = 0; index < arraySize; index++) {
208 PTRANSFER_PACKET pkt = NewTransferPacket(Fdo);
209 if (pkt) {
211 pkt->AllocateNode = index;
213 } else {
215 break;
216 }
217 }
219 }
220
221 //
222 // Pre-initialize our SCSI_REQUEST_BLOCK template with all
223 // the constant fields. This will save a little time for each xfer.
224 // NOTE: a CdbLength field of 10 may not always be appropriate
225 //
226
227 if (NT_SUCCESS(status)) {
228 if (fdoExt->AdapterDescriptor->SrbType == SRB_TYPE_STORAGE_REQUEST_BLOCK) {
229 ULONG ByteSize = 0;
230
231 #if (NTDDI_VERSION >= NTDDI_WINBLUE)
232 if ((fdoExt->MiniportDescriptor != NULL) &&
233 (fdoExt->MiniportDescriptor->Size >= RTL_SIZEOF_THROUGH_FIELD(STORAGE_MINIPORT_DESCRIPTOR, ExtraIoInfoSupported)) &&
234 (fdoExt->MiniportDescriptor->ExtraIoInfoSupported == TRUE)) {
236 fdoExt->AdapterDescriptor->AddressType,
238 &ByteSize,
239 2,
242 );
243 } else {
245 fdoExt->AdapterDescriptor->AddressType,
247 &ByteSize,
248 1,
250 );
251 }
252 #else
254 fdoExt->AdapterDescriptor->AddressType,
256 &ByteSize,
257 1,
259 );
260 #endif
261 if (NT_SUCCESS(status)) {
263 } else {
265 }
266 } else {
267 fdoData->SrbTemplate = ExAllocatePoolWithTag(NonPagedPoolNx, sizeof(SCSI_REQUEST_BLOCK), '-brs');
268 if (fdoData->SrbTemplate == NULL) {
270 } else {
272 fdoData->SrbTemplate->Length = sizeof(SCSI_REQUEST_BLOCK);
273 fdoData->SrbTemplate->Function = SRB_FUNCTION_EXECUTE_SCSI;
274 }
275 }
276 }
277
278 if (status == STATUS_SUCCESS) {
281 SrbSetCdbLength(fdoData->SrbTemplate, 10);
282 }
283
284 return status;
285}
#define MAX(x, y)
Definition: rdesktop.h:175
#define MAX_OUTSTANDING_IO_PER_LUN_DEFAULT
Definition: classp.h:638
#define MAX_WORKINGSET_TRANSFER_PACKETS_Client
Definition: classp.h:632
#define MIN_WORKINGSET_TRANSFER_PACKETS_Server_LowerBound
Definition: classp.h:634
#define MIN_WORKINGSET_TRANSFER_PACKETS_Server_UpperBound
Definition: classp.h:633
#define MIN_INITIAL_TRANSFER_PACKETS
Definition: classp.h:630
#define CLASS_TAG_PRIVATE_DATA
Definition: classp.h:189
#define MAX_WORKINGSET_TRANSFER_PACKETS_Server
Definition: classp.h:635
#define MIN_WORKINGSET_TRANSFER_PACKETS_Client
Definition: classp.h:631
NTSTATUS NTAPI RtlGetVersion(IN OUT PRTL_OSVERSIONINFOW lpVersionInformation)
Definition: version.c:158
struct _SCSI_REQUEST_BLOCK SCSI_REQUEST_BLOCK
#define SRB_FUNCTION_EXECUTE_SCSI
Definition: srb.h:315
#define PAGE_SHIFT
Definition: env_spec_w32.h:45
#define min(a, b)
Definition: monoChain.cc:55
enum _STORAGE_PROPERTY_ID STORAGE_PROPERTY_ID
@ StorageDeviceIoCapabilityProperty
Definition: ntddstor.h:534
* PSTORAGE_DEVICE_IO_CAPABILITY_DESCRIPTOR
Definition: ntddstor.h:695
* PSTORAGE_ADAPTER_DESCRIPTOR
Definition: ntddstor.h:599
#define SENSE_BUFFER_SIZE_EX
Definition: scsi.h:596
PCLASS_WORKING_SET WorkingSet
Definition: classpnp.h:588
PCLASS_DRIVER_EXTENSION DriverExtension
Definition: classpnp.h:600
PDEVICE_OBJECT LowerDeviceObject
Definition: classpnp.h:598
struct _FUNCTIONAL_DEVICE_EXTENSION * PartitionZeroExtension
Definition: classpnp.h:599
COMMON_DEVICE_EXTENSION CommonExtension
Definition: classpnp.h:873
ULONG dwOSVersionInfoSize
Definition: rtltypes.h:269
UCHAR wProductType
Definition: rtltypes.h:278
FORCEINLINE VOID InitializeSListHead(_Out_ PSLIST_HEADER SListHead)
Definition: rtlfuncs.h:3351
#define VER_NT_SERVER
#define VER_NT_DOMAIN_CONTROLLER
_IRQL_requires_same_ _In_ CLONG ByteSize
Definition: rtltypes.h:393
struct _OSVERSIONINFOEXW OSVERSIONINFOEXW
VOID EnqueueFreeTransferPacket(PDEVICE_OBJECT Fdo, __drv_aliasesMem PTRANSFER_PACKET Pkt)
Definition: xferpkt.c:529

Referenced by ClassPnpStartDevice().

◆ SetupDriveCapacityTransferPacket()

VOID SetupDriveCapacityTransferPacket ( TRANSFER_PACKET Pkt,
PVOID  ReadCapacityBuffer,
ULONG  ReadCapacityBufferLen,
PKEVENT  SyncEventPtr,
PIRP  OriginalIrp,
BOOLEAN  Use16ByteCdb 
)

Definition at line 1480 of file xferpkt.c.

1486{
1488 PCLASS_PRIVATE_FDO_DATA fdoData = fdoExt->PrivateFdoData;
1489 PCDB pCdb;
1490 ULONG srbLength;
1491 ULONG timeoutValue = fdoExt->TimeOutValue;
1492
1493 if (fdoExt->AdapterDescriptor->SrbType == SRB_TYPE_STORAGE_REQUEST_BLOCK) {
1494 srbLength = ((PSTORAGE_REQUEST_BLOCK) fdoData->SrbTemplate)->SrbLength;
1495 NT_ASSERT(((PSTORAGE_REQUEST_BLOCK) Pkt->Srb)->SrbLength >= srbLength);
1496 } else {
1497 srbLength = fdoData->SrbTemplate->Length;
1498 }
1499 RtlCopyMemory(Pkt->Srb, fdoData->SrbTemplate, srbLength); // copies _contents_ of SRB blocks
1500
1505
1506
1507 SrbSetTimeOutValue(Pkt->Srb, timeoutValue);
1509 SrbSetDataTransferLength(Pkt->Srb, ReadCapacityBufferLen);
1510
1512
1513 pCdb = SrbGetCdb(Pkt->Srb);
1514 if (pCdb) {
1515 if (Use16ByteCdb == TRUE) {
1516 NT_ASSERT(ReadCapacityBufferLen >= sizeof(READ_CAPACITY_DATA_EX));
1517 SrbSetCdbLength(Pkt->Srb, 16);
1518 pCdb->CDB16.OperationCode = SCSIOP_READ_CAPACITY16;
1519 REVERSE_BYTES(&pCdb->CDB16.TransferLength, &ReadCapacityBufferLen);
1520 pCdb->AsByte[1] = 0x10; // Service Action
1521 } else {
1522 SrbSetCdbLength(Pkt->Srb, 10);
1523 pCdb->CDB10.OperationCode = SCSIOP_READ_CAPACITY;
1524 }
1525 }
1526
1528 Pkt->BufLenCopy = ReadCapacityBufferLen;
1529
1532#if !defined(__REACTOS__) && NTDDI_VERSION >= NTDDI_WINBLUE
1534#endif
1535
1536 Pkt->SyncEventPtr = SyncEventPtr;
1538}
_In_ PREAD_CAPACITY_DATA ReadCapacityBuffer
Definition: cdrom.h:1103
#define SCSIOP_READ_CAPACITY
Definition: cdrw_hw.h:904
#define NUM_DRIVECAPACITY_RETRIES
Definition: classp.h:161
#define SRB_FLAGS_DATA_IN
Definition: srb.h:400
#define SCSIOP_READ_CAPACITY16
Definition: scsi.h:364
UCHAR NumIoTimeoutRetries
Definition: classp.h:518
PKEVENT SyncEventPtr
Definition: classp.h:535
struct _CDB::_CDB10 CDB10
UCHAR AsByte[16]
Definition: scsi.h:1988
struct _CDB::_CDB16 CDB16

Referenced by ClassReadDriveCapacity().

◆ SetupEjectionTransferPacket()

VOID SetupEjectionTransferPacket ( TRANSFER_PACKET Pkt,
BOOLEAN  PreventMediaRemoval,
PKEVENT  SyncEventPtr,
PIRP  OriginalIrp 
)

Definition at line 1315 of file xferpkt.c.

1319{
1321 PCLASS_PRIVATE_FDO_DATA fdoData = fdoExt->PrivateFdoData;
1322 PCDB pCdb;
1323 ULONG srbLength;
1324
1325 PAGED_CODE();
1326
1327 if (fdoExt->AdapterDescriptor->SrbType == SRB_TYPE_STORAGE_REQUEST_BLOCK) {
1328 srbLength = ((PSTORAGE_REQUEST_BLOCK) fdoData->SrbTemplate)->SrbLength;
1329 NT_ASSERT(((PSTORAGE_REQUEST_BLOCK) Pkt->Srb)->SrbLength >= srbLength);
1330 } else {
1331 srbLength = fdoData->SrbTemplate->Length;
1332 }
1333 RtlCopyMemory(Pkt->Srb, fdoData->SrbTemplate, srbLength); // copies _contents_ of SRB blocks
1334
1336 SrbSetCdbLength(Pkt->Srb, 6);
1341
1343
1344 pCdb = SrbGetCdb(Pkt->Srb);
1345 if (pCdb) {
1346 pCdb->MEDIA_REMOVAL.OperationCode = SCSIOP_MEDIUM_REMOVAL;
1347 pCdb->MEDIA_REMOVAL.Prevent = PreventMediaRemoval;
1348 }
1349
1350 Pkt->BufPtrCopy = NULL;
1351 Pkt->BufLenCopy = 0;
1352
1355 Pkt->SyncEventPtr = SyncEventPtr;
1357}
#define SCSIOP_MEDIUM_REMOVAL
Definition: cdrw_hw.h:902
#define NUM_LOCKMEDIAREMOVAL_RETRIES
Definition: classp.h:158
struct _CDB::_MEDIA_REMOVAL MEDIA_REMOVAL

◆ SetupModeSelectTransferPacket()

VOID SetupModeSelectTransferPacket ( TRANSFER_PACKET Pkt,
PKEVENT  SyncEventPtr,
PVOID  ModeSelectBuffer,
UCHAR  ModeSelectBufferLen,
BOOLEAN  SavePages,
PIRP  OriginalIrp 
)

Definition at line 1425 of file xferpkt.c.

1432{
1434 PCLASS_PRIVATE_FDO_DATA fdoData = fdoExt->PrivateFdoData;
1435 PCDB pCdb;
1436 ULONG srbLength;
1437
1438 if (fdoExt->AdapterDescriptor->SrbType == SRB_TYPE_STORAGE_REQUEST_BLOCK) {
1439 srbLength = ((PSTORAGE_REQUEST_BLOCK) fdoData->SrbTemplate)->SrbLength;
1440 NT_ASSERT(((PSTORAGE_REQUEST_BLOCK) Pkt->Srb)->SrbLength >= srbLength);
1441 } else {
1442 srbLength = fdoData->SrbTemplate->Length;
1443 }
1444 RtlCopyMemory(Pkt->Srb, fdoData->SrbTemplate, srbLength); // copies _contents_ of SRB blocks
1445
1447 SrbSetCdbLength(Pkt->Srb, 6);
1452 SrbSetDataBuffer(Pkt->Srb, ModeSelectBuffer);
1453 SrbSetDataTransferLength(Pkt->Srb, ModeSelectBufferLen);
1454
1456
1457 pCdb = SrbGetCdb(Pkt->Srb);
1458 if (pCdb) {
1459 pCdb->MODE_SELECT.OperationCode = SCSIOP_MODE_SELECT;
1460 pCdb->MODE_SELECT.SPBit = SavePages;
1461 pCdb->MODE_SELECT.PFBit = 1;
1462 pCdb->MODE_SELECT.ParameterListLength = (UCHAR)ModeSelectBufferLen;
1463 }
1464
1465 Pkt->BufPtrCopy = ModeSelectBuffer;
1466 Pkt->BufLenCopy = ModeSelectBufferLen;
1467
1470 Pkt->SyncEventPtr = SyncEventPtr;
1472}
#define SCSIOP_MODE_SELECT
Definition: cdrw_hw.h:891
#define NUM_MODESELECT_RETRIES
Definition: classp.h:160
struct _CDB::_MODE_SELECT MODE_SELECT
unsigned char UCHAR
Definition: xmlstorage.h:181

Referenced by ClasspModeSelect().

◆ SetupModeSenseTransferPacket()

VOID SetupModeSenseTransferPacket ( TRANSFER_PACKET Pkt,
PKEVENT  SyncEventPtr,
PVOID  ModeSenseBuffer,
UCHAR  ModeSenseBufferLen,
UCHAR  PageMode,
UCHAR  SubPage,
PIRP  OriginalIrp,
UCHAR  PageControl 
)

Definition at line 1365 of file xferpkt.c.

1374{
1376 PCLASS_PRIVATE_FDO_DATA fdoData = fdoExt->PrivateFdoData;
1377 PCDB pCdb;
1378 ULONG srbLength;
1379
1380 PAGED_CODE();
1381
1382 if (fdoExt->AdapterDescriptor->SrbType == SRB_TYPE_STORAGE_REQUEST_BLOCK) {
1383 srbLength = ((PSTORAGE_REQUEST_BLOCK) fdoData->SrbTemplate)->SrbLength;
1384 NT_ASSERT(((PSTORAGE_REQUEST_BLOCK) Pkt->Srb)->SrbLength >= srbLength);
1385 } else {
1386 srbLength = fdoData->SrbTemplate->Length;
1387 }
1388 RtlCopyMemory(Pkt->Srb, fdoData->SrbTemplate, srbLength); // copies _contents_ of SRB blocks
1389
1391 SrbSetCdbLength(Pkt->Srb, 6);
1396 SrbSetDataBuffer(Pkt->Srb, ModeSenseBuffer);
1397 SrbSetDataTransferLength(Pkt->Srb, ModeSenseBufferLen);
1398
1399
1401
1402 pCdb = SrbGetCdb(Pkt->Srb);
1403 if (pCdb) {
1404 pCdb->MODE_SENSE.OperationCode = SCSIOP_MODE_SENSE;
1405 pCdb->MODE_SENSE.PageCode = PageMode;
1406 pCdb->MODE_SENSE.SubPageCode = SubPage;
1407 pCdb->MODE_SENSE.Pc = PageControl;
1408 pCdb->MODE_SENSE.AllocationLength = (UCHAR)ModeSenseBufferLen;
1409 }
1410
1411 Pkt->BufPtrCopy = ModeSenseBuffer;
1412 Pkt->BufLenCopy = ModeSenseBufferLen;
1413
1416 Pkt->SyncEventPtr = SyncEventPtr;
1418}
_In_ size_t _In_ UCHAR PageMode
Definition: cdrom.h:1326
_In_ ULONG _In_ UCHAR _In_ UCHAR PageControl
Definition: cdrom.h:1319
#define SCSIOP_MODE_SENSE
Definition: cdrw_hw.h:896
#define NUM_MODESENSE_RETRIES
Definition: classp.h:159
struct _CDB::_MODE_SENSE MODE_SENSE

Referenced by ClasspModeSense().

◆ SetupReadWriteTransferPacket()

VOID SetupReadWriteTransferPacket ( PTRANSFER_PACKET  Pkt,
PVOID  Buf,
ULONG  Len,
LARGE_INTEGER  DiskLocation,
PIRP  OriginalIrp 
)

Definition at line 718 of file xferpkt.c.

723{
726 PCLASS_PRIVATE_FDO_DATA fdoData = fdoExt->PrivateFdoData;
728 UCHAR majorFunc = origCurSp->MajorFunction;
729 LARGE_INTEGER logicalBlockAddr;
730 ULONG numTransferBlocks;
731 PCDB pCdb;
732 ULONG srbLength;
733 ULONG timeoutValue = fdoExt->TimeOutValue;
734
735 logicalBlockAddr.QuadPart = Int64ShrlMod32(DiskLocation.QuadPart, fdoExt->SectorShift);
736 numTransferBlocks = Len >> fdoExt->SectorShift;
737
738 /*
739 * This field is useful when debugging, since low-memory conditions are
740 * handled differently for CDROM (which is the only driver using StartIO)
741 */
743
744 /*
745 * Slap the constant SRB fields in from our pre-initialized template.
746 * We'll then only have to fill in the unique fields for this transfer.
747 * Tell lower drivers to sort the SRBs by the logical block address
748 * so that disk seeks are minimized.
749 */
750 if (fdoExt->AdapterDescriptor->SrbType == SRB_TYPE_STORAGE_REQUEST_BLOCK) {
751 srbLength = ((PSTORAGE_REQUEST_BLOCK) fdoData->SrbTemplate)->SrbLength;
752 NT_ASSERT(((PSTORAGE_REQUEST_BLOCK) Pkt->Srb)->SrbLength >= srbLength);
753 } else {
754 srbLength = fdoData->SrbTemplate->Length;
755 }
756 RtlCopyMemory(Pkt->Srb, fdoData->SrbTemplate, srbLength); // copies _contents_ of SRB blocks
757 SrbSetDataBuffer(Pkt->Srb, Buf);
759 SrbSetQueueSortKey(Pkt->Srb, logicalBlockAddr.LowPart);
760 if (logicalBlockAddr.QuadPart > 0xFFFFFFFF) {
761 //
762 // If the requested LBA is more than max ULONG set the
763 // QueueSortKey to the maximum value, so that these
764 // requests can be added towards the end of the queue.
765 //
766
767 SrbSetQueueSortKey(Pkt->Srb, 0xFFFFFFFF);
768 }
771
772
773 SrbSetTimeOutValue(Pkt->Srb, timeoutValue);
774
775 /*
776 * Arrange values in CDB in big-endian format.
777 */
778 pCdb = SrbGetCdb(Pkt->Srb);
779 if (pCdb) {
781 REVERSE_BYTES_QUAD(&pCdb->CDB16.LogicalBlock, &logicalBlockAddr);
782 REVERSE_BYTES(&pCdb->CDB16.TransferLength, &numTransferBlocks);
783 pCdb->CDB16.OperationCode = (majorFunc==IRP_MJ_READ) ? SCSIOP_READ16 : SCSIOP_WRITE16;
784 SrbSetCdbLength(Pkt->Srb, 16);
785 } else {
786 pCdb->CDB10.LogicalBlockByte0 = ((PFOUR_BYTE)&logicalBlockAddr.LowPart)->Byte3;
787 pCdb->CDB10.LogicalBlockByte1 = ((PFOUR_BYTE)&logicalBlockAddr.LowPart)->Byte2;
788 pCdb->CDB10.LogicalBlockByte2 = ((PFOUR_BYTE)&logicalBlockAddr.LowPart)->Byte1;
789 pCdb->CDB10.LogicalBlockByte3 = ((PFOUR_BYTE)&logicalBlockAddr.LowPart)->Byte0;
790 pCdb->CDB10.TransferBlocksMsb = ((PFOUR_BYTE)&numTransferBlocks)->Byte1;
791 pCdb->CDB10.TransferBlocksLsb = ((PFOUR_BYTE)&numTransferBlocks)->Byte0;
792 pCdb->CDB10.OperationCode = (majorFunc==IRP_MJ_READ) ? SCSIOP_READ : SCSIOP_WRITE;
793 }
794 }
795
796 /*
797 * Set SRB and IRP flags
798 */
800 if (TEST_FLAG(OriginalIrp->Flags, IRP_PAGING_IO) ||
803 }
805
806 /*
807 * Allow caching only if this is not a write-through request.
808 * If write-through and caching is enabled on the device, force
809 * media access.
810 * Ignore SL_WRITE_THROUGH for reads; it's only set because the file handle was opened with WRITE_THROUGH.
811 */
812 if ((majorFunc == IRP_MJ_WRITE) && TEST_FLAG(origCurSp->Flags, SL_WRITE_THROUGH) && pCdb) {
813 pCdb->CDB10.ForceUnitAccess = fdoExt->CdbForceUnitAccess;
814 } else {
816 }
817
818 /*
819 * Remember the buf and len in the SRB because miniports
820 * can overwrite SRB.DataTransferLength and we may need it again
821 * for the retry.
822 */
823 Pkt->BufPtrCopy = Buf;
824 Pkt->BufLenCopy = Len;
825 Pkt->TargetLocationCopy = DiskLocation;
826
831#if !defined(__REACTOS__) && NTDDI_VERSION >= NTDDI_WINBLUE
834#endif
835
836
837 if (pCdb) {
838 DBGLOGFLUSHINFO(fdoData, TRUE, (BOOLEAN)(pCdb->CDB10.ForceUnitAccess), FALSE);
839 } else {
840 DBGLOGFLUSHINFO(fdoData, TRUE, FALSE, FALSE);
841 }
842}
unsigned char BOOLEAN
#define SRB_CLASS_FLAGS_PAGING
Definition: cdrom.h:165
#define TEST_FLAG(Flags, Bit)
Definition: cdrom.h:1495
#define SCSIOP_WRITE
Definition: cdrw_hw.h:906
#define SCSIOP_READ
Definition: cdrw_hw.h:905
#define DEV_USE_16BYTE_CDB
Definition: classpnp.h:183
#define Len
Definition: deflate.h:82
#define DBGLOGFLUSHINFO(_fdoData, _isIO, _isFUA, _isFlush)
Definition: debug.h:129
#define SCSIOP_READ16
Definition: scsi.h:914
#define SCSIOP_WRITE16
Definition: scsi.h:915
#define SRB_FLAGS_ADAPTER_CACHE_ENABLE
Definition: srb.h:405
struct _FOUR_BYTE * PFOUR_BYTE
#define Int64ShrlMod32(a, b)
#define IRP_MJ_READ
Definition: rdpdr.c:46
#define IRP_MJ_WRITE
Definition: rdpdr.c:47
#define REVERSE_BYTES_QUAD(Destination, Source)
Definition: scsi.h:3452
FORCEINLINE VOID SrbSetSrbFlags(_In_ PVOID Srb, _In_ ULONG Flags)
Definition: srbhelper.h:964
FORCEINLINE VOID SrbSetQueueSortKey(_In_ PVOID Srb, _In_ ULONG QueueSortKey)
Definition: srbhelper.h:839
CLASS_INIT_DATA InitData
Definition: classpnp.h:577
PDRIVER_STARTIO ClassStartIo
Definition: classpnp.h:543
UCHAR MaxNumberOfIoRetries
Definition: classp.h:976
LARGE_INTEGER TargetLocationCopy
Definition: classp.h:564
UCHAR NumThinProvisioningRetries
Definition: classp.h:517
BOOLEAN DriverUsesStartIO
Definition: classp.h:543
LONGLONG QuadPart
Definition: typedefs.h:114
ULONG LowPart
Definition: typedefs.h:106
__drv_aliasesMem FORCEINLINE PIO_STACK_LOCATION IoGetCurrentIrpStackLocation(_In_ PIRP Irp)
Definition: iofuncs.h:2793
#define IRP_PAGING_IO
#define SL_WRITE_THROUGH
Definition: iotypes.h:1824
#define IRP_SYNCHRONOUS_PAGING_IO

Referenced by ServiceTransferRequest(), and StepLowMemRetry().

◆ SubmitTransferPacket()

NTSTATUS SubmitTransferPacket ( PTRANSFER_PACKET  Pkt)

Definition at line 850 of file xferpkt.c.

851{
853 PDEVICE_OBJECT nextDevObj = commonExtension->LowerDeviceObject;
855 PCLASS_PRIVATE_FDO_DATA fdoData = fdoExtension->PrivateFdoData;
856 BOOLEAN idleRequest = FALSE;
857 PIO_STACK_LOCATION nextSp;
858
859 NT_ASSERT(Pkt->Irp->CurrentLocation == Pkt->Irp->StackCount+1);
860
861 /*
862 * Attach the SRB to the IRP.
863 * The reused IRP's stack location has to be rewritten for each retry
864 * call because IoCompleteRequest clears the stack locations.
865 */
867
868
870 nextSp->MajorFunction = IRP_MJ_SCSI;
871 nextSp->Parameters.Scsi.Srb = (PSCSI_REQUEST_BLOCK)Pkt->Srb;
872
874 Pkt->Srb->SrbStatus = 0;
876
878 /*
879 * Only dereference the "original IRP"'s stack location
880 * if its a real client irp (as opposed to a static irp
881 * we're using just for result status for one of the non-IO scsi commands).
882 *
883 * For read/write, propagate the storage-specific IRP stack location flags
884 * (e.g. SL_OVERRIDE_VERIFY_VOLUME, SL_WRITE_THROUGH).
885 */
887 nextSp->Flags = origCurSp->Flags;
888 }
889
890 //
891 // If the request is not split, we can use the original IRP MDL. If the
892 // request needs to be split, we need to use a partial MDL. The partial MDL
893 // is needed because more than one driver might be mapping the same MDL
894 // and this causes problems.
895 //
896 if (Pkt->UsePartialMdl == FALSE) {
897 Pkt->Irp->MdlAddress = Pkt->OriginalIrp->MdlAddress;
898 } else {
900 Pkt->Irp->MdlAddress = Pkt->PartialMdl;
901 }
902
903
906
907 //
908 // Set the original irp here for SFIO.
909 //
911
912 //
913 // No need to lock for IdlePrioritySupported, since it will
914 // be modified only at initialization time.
915 //
916 if (fdoData->IdlePrioritySupported == TRUE) {
917 idleRequest = ClasspIsIdleRequest(Pkt->OriginalIrp);
918 if (idleRequest) {
920 } else {
922 }
923 }
924
926 return IoCallDriver(nextDevObj, Pkt->Irp);
927}
IO_COMPLETION_ROUTINE TransferPktComplete
Definition: classp.h:1724
FORCEINLINE BOOLEAN ClasspIsIdleRequest(PIRP Irp)
Definition: classp.h:1268
#define HISTORYLOGSENDPACKET(_packet)
Definition: classp.h:2345
FORCEINLINE VOID ClasspSrbSetOriginalIrp(_In_ PSTORAGE_REQUEST_BLOCK_HEADER Srb, _In_ PIRP Irp)
Definition: classp.h:2530
#define DBGLOGSENDPACKET(_pkt)
Definition: debug.h:127
struct _SCSI_REQUEST_BLOCK * PSCSI_REQUEST_BLOCK
VOID NTAPI IoBuildPartialMdl(IN PMDL SourceMdl, IN PMDL TargetMdl, IN PVOID VirtualAddress, IN ULONG Length)
Definition: iomdl.c:96
#define IoSetCompletionRoutine(_Irp, _CompletionRoutine, _Context, _InvokeOnSuccess, _InvokeOnError, _InvokeOnCancel)
Definition: irp.cpp:498
VOID NTAPI IoReuseIrp(IN OUT PIRP Irp, IN NTSTATUS Status)
Definition: irp.c:1971
#define IoCallDriver
Definition: irp.c:1225
#define STATUS_NOT_SUPPORTED
Definition: ntstatus.h:423
FORCEINLINE VOID SrbSetScsiStatus(_In_ PVOID Srb, _In_ UCHAR ScsiStatus)
Definition: srbhelper.h:1056
FORCEINLINE PVOID SrbGetDataBuffer(_In_ PVOID Srb)
Definition: srbhelper.h:728
FORCEINLINE ULONG SrbGetDataTransferLength(_In_ PVOID Srb)
Definition: srbhelper.h:765
BOOLEAN IdlePrioritySupported
Definition: classp.h:808
struct _NAMED_PIPE_CREATE_PARAMETERS * Parameters
Definition: iotypes.h:3128
BOOLEAN UsePartialMdl
Definition: classp.h:595
__drv_aliasesMem FORCEINLINE PIO_STACK_LOCATION IoGetNextIrpStackLocation(_In_ PIRP Irp)
Definition: iofuncs.h:2695
#define IRP_MJ_SCSI

Referenced by ClasspContinueOffloadWrite(), ClasspModeSelect(), ClasspModeSense(), ClasspReceivePopulateTokenInformation(), ClasspReceiveWriteUsingTokenInformation(), ClassReadDriveCapacity(), ServiceTransferRequest(), and TransferPacketRetryTimerDpc().

◆ TransferPktComplete()

NTSTATUS NTAPI TransferPktComplete ( IN PDEVICE_OBJECT  NullFdo,
IN PIRP  Irp,
IN PVOID  Context 
)

Definition at line 932 of file xferpkt.c.

933{
936 PCLASS_PRIVATE_FDO_DATA fdoData = fdoExt->PrivateFdoData;
937 BOOLEAN packetDone = FALSE;
938 BOOLEAN idleRequest = FALSE;
939 ULONG transferLength;
940 LARGE_INTEGER completionTime;
941 ULONGLONG lastIoCompletionTime;
942
943 UNREFERENCED_PARAMETER(NullFdo);
944
945 /*
946 * Put all the assertions and spew in here so we don't have to look at them.
947 */
951
952
953 completionTime = ClasspGetCurrentTime();
954
955 //
956 // Record the time at which the last IO completed while snapping the old
957 // value to be used later. This can occur on multiple threads and hence
958 // could be overwritten with an older value. This is OK because this value
959 // is maintained as a heuristic.
960 //
961
962#ifdef _WIN64
963#ifndef __REACTOS__
964 lastIoCompletionTime = ReadULong64NoFence((volatile ULONG64*)&fdoData->LastIoCompletionTime.QuadPart);
965 WriteULong64NoFence((volatile ULONG64*)&fdoData->LastIoCompletionTime.QuadPart,
966 completionTime.QuadPart);
967#else
968 lastIoCompletionTime = *(volatile ULONG64*)&fdoData->LastIoCompletionTime.QuadPart;
969 *((volatile ULONG64*)&fdoData->LastIoCompletionTime.QuadPart) = completionTime.QuadPart;
970#endif
971#else
972 lastIoCompletionTime = InterlockedExchangeNoFence64((volatile LONG64*)&fdoData->LastIoCompletionTime.QuadPart,
973 completionTime.QuadPart);
974#endif
975
976 if (fdoData->IdlePrioritySupported == TRUE) {
977 idleRequest = ClasspIsIdleRequest(pkt->OriginalIrp);
978 if (idleRequest) {
980 NT_ASSERT(fdoData->ActiveIdleIoCount >= 0);
981 } else {
982 fdoData->LastNonIdleIoTime = completionTime;
984 NT_ASSERT(fdoData->ActiveIoCount >= 0);
985 }
986 }
987
988 //
989 // If partial MDL was used, unmap the pages. When the packet is retried, the
990 // MDL will be recreated. If the packet is done, the MDL will be ready to be reused.
991 //
992 if (pkt->UsePartialMdl) {
994 }
995
996 if (SRB_STATUS(pkt->Srb->SrbStatus) == SRB_STATUS_SUCCESS) {
997
998 NT_ASSERT(NT_SUCCESS(Irp->IoStatus.Status));
999
1000 transferLength = SrbGetDataTransferLength(pkt->Srb);
1001
1003
1004 /*
1005 * The port driver should not have allocated a sense buffer
1006 * if the SRB succeeded.
1007 */
1008 NT_ASSERT(!PORT_ALLOCATED_SENSE_EX(fdoExt, pkt->Srb));
1009
1010 /*
1011 * Add this packet's transferred length to the original IRP's.
1012 */
1014 (LONG)transferLength);
1015
1016
1017 if ((pkt->InLowMemRetry) ||
1019 packetDone = StepLowMemRetry(pkt);
1020 } else {
1021 packetDone = TRUE;
1022 }
1023
1024 }
1025 else {
1026 /*
1027 * The packet failed. We may retry it if possible.
1028 */
1029 BOOLEAN shouldRetry;
1030
1031 /*
1032 * Make sure IRP status matches SRB error status (since we propagate it).
1033 */
1034 if (NT_SUCCESS(Irp->IoStatus.Status)){
1035 Irp->IoStatus.Status = STATUS_UNSUCCESSFUL;
1036 }
1037
1038 /*
1039 * The packet failed.
1040 * So when sending the packet down we either saw either an error or STATUS_PENDING,
1041 * and so we returned STATUS_PENDING for the original IRP.
1042 * So now we must mark the original irp pending to match that, _regardless_ of
1043 * whether we actually switch threads here by retrying.
1044 * (We also have to mark the irp pending if the lower driver marked the irp pending;
1045 * that is dealt with farther down).
1046 */
1049 }
1050
1051 /*
1052 * Interpret the SRB error (to a meaningful IRP status)
1053 * and determine if we should retry this packet.
1054 * This call looks at the returned SENSE info to figure out what to do.
1055 */
1056 shouldRetry = InterpretTransferPacketError(pkt);
1057
1058 /*
1059 * If the SRB queue is locked-up, release it.
1060 * Do this after calling the error handler.
1061 */
1062 if (pkt->Srb->SrbStatus & SRB_STATUS_QUEUE_FROZEN){
1063 ClassReleaseQueue(pkt->Fdo);
1064 }
1065
1066
1067 if (NT_SUCCESS(Irp->IoStatus.Status)){
1068 /*
1069 * The error was recovered above in the InterpretTransferPacketError() call.
1070 */
1071
1072 NT_ASSERT(!shouldRetry);
1073
1074 /*
1075 * In the case of a recovered error,
1076 * add the transfer length to the original Irp as we would in the success case.
1077 */
1080
1081 if ((pkt->InLowMemRetry) ||
1083 packetDone = StepLowMemRetry(pkt);
1084 } else {
1085 packetDone = TRUE;
1086 }
1087 } else {
1088 if (shouldRetry && (pkt->NumRetries > 0)){
1089 packetDone = RetryTransferPacket(pkt);
1090 } else if (shouldRetry && (pkt->RetryHistory != NULL)){
1091 // don't limit retries if class driver has custom interpretation routines
1092 packetDone = RetryTransferPacket(pkt);
1093 } else {
1094 packetDone = TRUE;
1095 }
1096 }
1097 }
1098
1099 /*
1100 * If the packet is completed, put it back in the free list.
1101 * If it is the last packet servicing the original request, complete the original irp.
1102 */
1103 if (packetDone){
1104 LONG numPacketsRemaining;
1105 PIRP deferredIrp;
1106 PDEVICE_OBJECT Fdo = pkt->Fdo;
1107 UCHAR uniqueAddr = 0;
1108
1109 /*
1110 * In case a remove is pending, bump the lock count so we don't get freed
1111 * right after we complete the original irp.
1112 */
1113 ClassAcquireRemoveLock(Fdo, (PVOID)&uniqueAddr);
1114
1115
1116 /*
1117 * Sometimes the port driver can allocates a new 'sense' buffer
1118 * to report transfer errors, e.g. when the default sense buffer
1119 * is too small. If so, it is up to us to free it.
1120 * Now that we're done using the sense info, free it if appropriate.
1121 * Then clear the sense buffer so it doesn't pollute future errors returned in this packet.
1122 */
1123 if (PORT_ALLOCATED_SENSE_EX(fdoExt, pkt->Srb)) {
1124 TracePrint((TRACE_LEVEL_INFORMATION, TRACE_FLAG_RW, "Freeing port-allocated sense buffer for pkt %ph.", pkt));
1128 } else {
1131 }
1132
1134
1135 /*
1136 * Call IoSetMasterIrpStatus to set appropriate status
1137 * for the Master IRP.
1138 */
1140
1141 if (!NT_SUCCESS(Irp->IoStatus.Status)){
1142 /*
1143 * If the original I/O originated in user space (i.e. it is thread-queued),
1144 * and the error is user-correctable (e.g. media is missing, for removable media),
1145 * alert the user.
1146 * Since this is only one of possibly several packets completing for the original IRP,
1147 * we may do this more than once for a single request. That's ok; this allows
1148 * us to test each returned status with IoIsErrorUserInduced().
1149 */
1150 if (IoIsErrorUserInduced(Irp->IoStatus.Status) &&
1152 pkt->OriginalIrp->Tail.Overlay.Thread){
1153
1155 }
1156 }
1157
1158 /*
1159 * We use a field in the original IRP to count
1160 * down the transfer pieces as they complete.
1161 */
1162 numPacketsRemaining = InterlockedDecrement(
1163 (PLONG)&pkt->OriginalIrp->Tail.Overlay.DriverContext[0]);
1164
1165 if (numPacketsRemaining > 0){
1166 /*
1167 * More transfer pieces remain for the original request.
1168 * Wait for them to complete before completing the original irp.
1169 */
1170 } else {
1171
1172 /*
1173 * All the transfer pieces are done.
1174 * Complete the original irp if appropriate.
1175 */
1176 NT_ASSERT(numPacketsRemaining == 0);
1178
1180 KIRQL oldIrql;
1181
1185 }
1187
1188 /*
1189 * We submitted all the downward irps, including this last one, on the thread
1190 * that the OriginalIrp came in on. So the OriginalIrp is completing on a
1191 * different thread iff this last downward irp is completing on a different thread.
1192 * If BlkCache is loaded, for example, it will often complete
1193 * requests out of the cache on the same thread, therefore not marking the downward
1194 * irp pending and not requiring us to do so here. If the downward request is completing
1195 * on the same thread, then by not marking the OriginalIrp pending we can save an APC
1196 * and get extra perf benefit out of BlkCache.
1197 * Note that if the packet ever cycled due to retry or LowMemRetry,
1198 * we set the pending bit in those codepaths.
1199 */
1200 if (pkt->Irp->PendingReturned){
1202 }
1203
1204
1206
1207 //
1208 // Drop the count only after completing the request, to give
1209 // Mm some amount of time to issue its next critical request
1210 //
1211
1213 {
1214 KeAcquireSpinLock(&fdoData->SpinLock, &oldIrql);
1215
1217 {
1218 fdoData->MaxInterleavedNormalIo = 0;
1219 } else {
1221 }
1222
1223 fdoData->NumHighPriorityPagingIo--;
1224
1225 if (fdoData->NumHighPriorityPagingIo == 0)
1226 {
1227 LARGE_INTEGER period;
1228
1229 //
1230 // Exiting throttle mode
1231 //
1232
1234
1235 period.QuadPart = fdoData->ThrottleStopTime.QuadPart - fdoData->ThrottleStartTime.QuadPart;
1237 }
1238
1239 KeReleaseSpinLock(&fdoData->SpinLock, oldIrql);
1240 }
1241
1242 if (idleRequest) {
1244 }
1245
1246 /*
1247 * We may have been called by one of the class drivers (e.g. cdrom)
1248 * via the legacy API ClassSplitRequest.
1249 * This is the only case for which the packet engine is called for an FDO
1250 * with a StartIo routine; in that case, we have to call IoStartNextPacket
1251 * now that the original irp has been completed.
1252 */
1255 TracePrint((TRACE_LEVEL_INFORMATION, TRACE_FLAG_RW, "SRB_FLAGS_DONT_START_NEXT_PACKET should never be set here (?)"));
1256 } else {
1257 KeRaiseIrql(DISPATCH_LEVEL, &oldIrql);
1258 IoStartNextPacket(Fdo, TRUE); // yes, some IO is now cancellable
1259 KeLowerIrql(oldIrql);
1260 }
1261 }
1262 }
1263 }
1264
1265 /*
1266 * If the packet was synchronous, write the final result back to the issuer's status buffer
1267 * and signal his event.
1268 */
1269 if (pkt->SyncEventPtr){
1270 KeSetEvent(pkt->SyncEventPtr, 0, FALSE);
1271 pkt->SyncEventPtr = NULL;
1272 }
1273
1274 /*
1275 * If the operation isn't a normal read/write, but needs to do more
1276 * operation-specific processing, call the operation's continuation
1277 * routine. The operation may create and queue another transfer packet
1278 * within this routine, but pkt is still freed after returning from the
1279 * continuation routine.
1280 */
1281 if (pkt->ContinuationRoutine != NULL){
1284 }
1285
1286 /*
1287 * Free the completed packet.
1288 */
1289 pkt->UsePartialMdl = FALSE;
1290// pkt->OriginalIrp = NULL;
1291 pkt->InLowMemRetry = FALSE;
1293
1294 /*
1295 * Now that we have freed some resources,
1296 * try again to send one of the previously deferred irps.
1297 */
1298 deferredIrp = DequeueDeferredClientIrp(Fdo);
1299 if (deferredIrp){
1300 ServiceTransferRequest(Fdo, deferredIrp, TRUE);
1301 }
1302
1303 ClassReleaseRemoveLock(Fdo, (PVOID)&uniqueAddr);
1304 }
1305
1307}
BOOLEAN StepLowMemRetry(PTRANSFER_PACKET Pkt)
Definition: retry.c:706
#define HISTORYLOGRETURNEDPACKET(_packet)
Definition: classp.h:2357
PIRP DequeueDeferredClientIrp(PDEVICE_OBJECT Fdo)
Definition: clntirp.c:113
FORCEINLINE LARGE_INTEGER ClasspGetCurrentTime(VOID)
Definition: classp.h:1280
VOID ClasspPerfIncrementSuccessfulIo(IN PFUNCTIONAL_DEVICE_EXTENSION FdoExtension)
Definition: utils.c:486
BOOLEAN RetryTransferPacket(PTRANSFER_PACKET Pkt)
Definition: retry.c:453
struct _TRANSFER_PACKET * PTRANSFER_PACKET
VOID ClasspCompleteIdleRequest(PFUNCTIONAL_DEVICE_EXTENSION FdoExtension)
Definition: clntirp.c:713
FORCEINLINE BOOLEAN PORT_ALLOCATED_SENSE_EX(_In_ PFUNCTIONAL_DEVICE_EXTENSION FdoExtension, _In_ PSTORAGE_REQUEST_BLOCK_HEADER Srb)
Definition: classp.h:2547
BOOLEAN InterpretTransferPacketError(PTRANSFER_PACKET Pkt)
Definition: retry.c:40
FORCEINLINE VOID FREE_PORT_ALLOCATED_SENSE_BUFFER_EX(_In_ PFUNCTIONAL_DEVICE_EXTENSION FdoExtension, _In_ PSTORAGE_REQUEST_BLOCK_HEADER Srb)
Definition: classp.h:2560
_In_ PIRP Irp
Definition: csq.h:116
ULONG ClassMaxInterleavePerCriticalIo
Definition: class.c:88
NTSTATUS ServiceTransferRequest(PDEVICE_OBJECT Fdo, PIRP Irp, BOOLEAN PostToDpc)
Definition: class.c:3341
VOID NTAPI ClassReleaseQueue(_In_ PDEVICE_OBJECT Fdo)
Definition: class.c:11589
#define DBGCHECKRETURNEDPKT(_pkt)
Definition: debug.h:126
#define DBGLOGRETURNPACKET(_pkt)
Definition: debug.h:128
VOID NTAPI ClassCompleteRequest(_In_ PDEVICE_OBJECT DeviceObject, _In_ PIRP Irp, _In_ CCHAR PriorityBoost)
Definition: lock.c:401
#define SRB_FLAGS_DONT_START_NEXT_PACKET
Definition: srb.h:415
#define SRB_STATUS(Status)
Definition: srb.h:389
#define SRB_STATUS_QUEUE_FROZEN
Definition: srb.h:386
#define SRB_STATUS_SUCCESS
Definition: srb.h:341
#define KeRaiseIrql(irql, oldIrql)
Definition: env_spec_w32.h:597
#define KeLowerIrql(oldIrql)
Definition: env_spec_w32.h:602
#define KeSetEvent(pEvt, foo, foo2)
Definition: env_spec_w32.h:476
#define KeQuerySystemTime(t)
Definition: env_spec_w32.h:570
#define DISPATCH_LEVEL
Definition: env_spec_w32.h:696
#define InterlockedExchangeAdd
Definition: interlocked.h:181
#define InterlockedExchangeNoFence64
Definition: interlocked.h:174
IoMarkIrpPending(Irp)
unsigned __int64 ULONG64
Definition: imports.h:198
static int priority
Definition: timer.c:163
#define UNREFERENCED_PARAMETER(P)
Definition: ntbasedef.h:317
VOID NTAPI IoStartNextPacket(IN PDEVICE_OBJECT DeviceObject, IN BOOLEAN Cancelable)
Definition: device.c:1847
IO_PAGING_PRIORITY FASTCALL IoGetPagingIoPriority(IN PIRP Irp)
Definition: irp.c:1748
VOID NTAPI IoSetHardErrorOrVerifyDevice(IN PIRP Irp, IN PDEVICE_OBJECT DeviceObject)
Definition: util.c:316
NTKRNLVISTAAPI VOID IoSetMasterIrpStatus(_Inout_ PIRP MasterIrp, _In_ NTSTATUS Status)
Definition: io.c:131
#define STATUS_MORE_PROCESSING_REQUIRED
Definition: shellext.h:68
FORCEINLINE ULONG SrbGetSrbFlags(_In_ PVOID Srb)
Definition: srbhelper.h:927
FORCEINLINE PVOID SrbGetSenseInfoBuffer(_In_ PVOID Srb)
Definition: srbhelper.h:619
FORCEINLINE UCHAR SrbGetSenseInfoBufferLength(_In_ PVOID Srb)
Definition: srbhelper.h:638
LARGE_INTEGER LastIoCompletionTime
Definition: classp.h:940
ULONG MaxInterleavedNormalIo
Definition: classp.h:831
LARGE_INTEGER LongestThrottlePeriod
Definition: classp.h:846
LARGE_INTEGER ThrottleStartTime
Definition: classp.h:836
ULONG NumHighPriorityPagingIo
Definition: classp.h:826
LARGE_INTEGER LastNonIdleIoTime
Definition: classp.h:935
LARGE_INTEGER ThrottleStopTime
Definition: classp.h:841
BOOLEAN LoggedTURFailureSinceLastIO
Definition: classp.h:753
IO_STATUS_BLOCK IoStatus
ULONG LowMemRetry_remainingBufLen
Definition: classp.h:546
BOOLEAN InLowMemRetry
Definition: classp.h:544
int64_t LONG64
Definition: typedefs.h:68
int32_t * PLONG
Definition: typedefs.h:58
uint64_t ULONGLONG
Definition: typedefs.h:67
#define STATUS_UNSUCCESSFUL
Definition: udferr_usr.h:132
#define IoIsErrorUserInduced(Status)
Definition: iofuncs.h:2817
enum _IO_PAGING_PRIORITY IO_PAGING_PRIORITY
@ IoPagingPriorityHigh
Definition: iotypes.h:2822
@ IoPagingPriorityInvalid
Definition: iotypes.h:2820
#define IO_DISK_INCREMENT
Definition: iotypes.h:600
#define MmPrepareMdlForReuse(_Mdl)