VirtIORing-Packed.c File Reference

#include "osdep.h"
#include "virtio_pci.h"
#include "VirtIO.h"
#include "kdebugprint.h"
#include "virtio_ring.h"
#include "windows/virtio_ring_allocation.h"
#include <pshpack1.h>
#include <poppack.h>

Include dependency graph for VirtIORing-Packed.c:

Go to the source code of this file.

Classes
struct	vring_packed_desc_event
struct	vring_packed_desc
struct	vring_desc_state_packed
struct	virtqueue_packed

Macros
#define	BUG_ON(condition)   { if (condition) { KeBugCheck(0xE0E1E2E3); }}
#define	BAD_RING(vq, fmt, ...)   DPrintf(0, "%s: queue %d: " fmt, __FUNCTION__, vq->vq.index, __VA_ARGS__); BUG_ON(true)
#define	VRING_DESC_F_NEXT   1
#define	VRING_DESC_F_WRITE   2
#define	VRING_DESC_F_INDIRECT   4
#define	VRING_PACKED_DESC_F_AVAIL   7
#define	VRING_PACKED_DESC_F_USED   15
#define	VRING_PACKED_EVENT_FLAG_ENABLE   0x0
#define	VRING_PACKED_EVENT_FLAG_DISABLE   0x1
#define	VRING_PACKED_EVENT_FLAG_DESC   0x2
#define	VRING_PACKED_EVENT_F_WRAP_CTR   15
#define	packedvq(vq)   ((struct virtqueue_packed *)vq)

Functions
static bool	vring_need_event (__u16 event_idx, __u16 new_idx, __u16 old)
unsigned int	vring_control_block_size_packed (u16 qsize)
unsigned long	vring_size_packed (unsigned int num, unsigned long align)
static int	virtqueue_add_buf_packed (struct virtqueue *_vq, struct scatterlist sg[], unsigned int out, unsigned int in, void *opaque, void *va_indirect, ULONGLONG phys_indirect)
static void	detach_buf_packed (struct virtqueue_packed *vq, unsigned int id)
static void *	virtqueue_detach_unused_buf_packed (struct virtqueue *_vq)
static void	virtqueue_disable_cb_packed (struct virtqueue *_vq)
static bool	is_used_desc_packed (const struct virtqueue_packed *vq, u16 idx, bool used_wrap_counter)
static bool	virtqueue_poll_packed (struct virtqueue_packed *vq, u16 off_wrap)
static unsigned	virtqueue_enable_cb_prepare_packed (struct virtqueue_packed *vq)
static bool	virtqueue_enable_cb_packed (struct virtqueue *_vq)
static bool	virtqueue_enable_cb_delayed_packed (struct virtqueue *_vq)
static BOOLEAN	virtqueue_is_interrupt_enabled_packed (struct virtqueue *_vq)
static void	virtqueue_shutdown_packed (struct virtqueue *_vq)
static bool	more_used_packed (const struct virtqueue_packed *vq)
static void *	virtqueue_get_buf_packed (struct virtqueue *_vq, unsigned int *len)
static BOOLEAN	virtqueue_has_buf_packed (struct virtqueue *_vq)
static bool	virtqueue_kick_prepare_packed (struct virtqueue *_vq)
static void	virtqueue_kick_always_packed (struct virtqueue *_vq)
struct virtqueue *	vring_new_virtqueue_packed (unsigned int index, unsigned int num, unsigned int vring_align, VirtIODevice *vdev, void *pages, void(*notify)(struct virtqueue *), void *control)

Macro Definition Documentation

BAD_RING

#define BAD_RING	(		vq,
			fmt,
			...
	)		DPrintf(0, "%s: queue %d: " fmt, __FUNCTION__, vq->vq.index, __VA_ARGS__); BUG_ON(true)

Definition at line 63 of file VirtIORing-Packed.c.

BUG_ON

#define BUG_ON

(

condition

)

{ if (condition) { KeBugCheck(0xE0E1E2E3); }}

Definition at line 62 of file VirtIORing-Packed.c.

packedvq

#define packedvq

(

vq

)

((struct virtqueue_packed *)vq)

Definition at line 155 of file VirtIORing-Packed.c.

VRING_DESC_F_INDIRECT

#define VRING_DESC_F_INDIRECT   4

Definition at line 70 of file VirtIORing-Packed.c.

VRING_DESC_F_NEXT

#define VRING_DESC_F_NEXT   1

Definition at line 66 of file VirtIORing-Packed.c.

VRING_DESC_F_WRITE

#define VRING_DESC_F_WRITE   2

Definition at line 68 of file VirtIORing-Packed.c.

VRING_PACKED_DESC_F_AVAIL

#define VRING_PACKED_DESC_F_AVAIL   7

Definition at line 76 of file VirtIORing-Packed.c.

VRING_PACKED_DESC_F_USED

#define VRING_PACKED_DESC_F_USED   15

Definition at line 77 of file VirtIORing-Packed.c.

VRING_PACKED_EVENT_F_WRAP_CTR

#define VRING_PACKED_EVENT_F_WRAP_CTR   15

Definition at line 94 of file VirtIORing-Packed.c.

VRING_PACKED_EVENT_FLAG_DESC

#define VRING_PACKED_EVENT_FLAG_DESC   0x2

Definition at line 89 of file VirtIORing-Packed.c.

VRING_PACKED_EVENT_FLAG_DISABLE

#define VRING_PACKED_EVENT_FLAG_DISABLE   0x1

Definition at line 82 of file VirtIORing-Packed.c.

VRING_PACKED_EVENT_FLAG_ENABLE

#define VRING_PACKED_EVENT_FLAG_ENABLE   0x0

Definition at line 80 of file VirtIORing-Packed.c.

Function Documentation

detach_buf_packed()

static void detach_buf_packed ( struct virtqueue_packed *  vq, unsigned int  id  )

static

Definition at line 294 of file VirtIORing-Packed.c.

{
    struct vring_desc_state_packed *state = &vq->packed.desc_state[id];
 
    /* Clear data ptr. */
    state->data = NULL;
 
    vq->packed.desc_state[state->last].next = (u16)vq->free_head;
    vq->free_head = id;
    vq->num_free += state->num;
}

Referenced by virtqueue_detach_unused_buf_packed(), and virtqueue_get_buf_packed().

is_used_desc_packed()

static bool is_used_desc_packed ( const struct virtqueue_packed *  vq, u16  idx, bool  used_wrap_counter  )

inlinestatic

Definition at line 336 of file VirtIORing-Packed.c.

{
    bool avail, used;
    u16 flags;
 
    flags = vq->packed.vring.desc[idx].flags;
    avail = !!(flags & (1 << VRING_PACKED_DESC_F_AVAIL));
    used = !!(flags & (1 << VRING_PACKED_DESC_F_USED));
 
    return avail == used && used == used_wrap_counter;
}

Referenced by more_used_packed(), virtqueue_enable_cb_delayed_packed(), and virtqueue_poll_packed().

more_used_packed()

static bool more_used_packed ( const struct virtqueue_packed *  vq )

inlinestatic

Definition at line 480 of file VirtIORing-Packed.c.

{
    return is_used_desc_packed(vq, vq->last_used_idx,
        vq->packed.used_wrap_counter);
}

Referenced by virtqueue_get_buf_packed(), and virtqueue_has_buf_packed().

virtqueue_add_buf_packed()

static int virtqueue_add_buf_packed ( struct virtqueue *  _vq, struct scatterlist  sg[], unsigned int  out, unsigned int  in, void *  opaque, void *  va_indirect, ULONGLONG  phys_indirect  )

static

Definition at line 171 of file VirtIORing-Packed.c.

{
    struct virtqueue_packed *vq = packedvq(_vq);
    unsigned int descs_used;
    struct vring_packed_desc *desc;
    u16 head, id, i;
 
    descs_used = out + in;
    head = vq->packed.next_avail_idx;
    id = (u16)vq->free_head;
 
    BUG_ON(descs_used == 0);
    BUG_ON(id >= vq->packed.vring.num);
 
    if (va_indirect && vq->num_free > 0) {
        desc = va_indirect;
        for (i = 0; i < descs_used; i++) {
            desc[i].flags = i < out ? 0 : VRING_DESC_F_WRITE;
            desc[i].addr = sg[i].physAddr.QuadPart;
            desc[i].len = sg[i].length;
        }
        vq->packed.vring.desc[head].addr = phys_indirect;
        vq->packed.vring.desc[head].len = descs_used * sizeof(struct vring_packed_desc);
        vq->packed.vring.desc[head].id = id;
 
        KeMemoryBarrier();
        vq->packed.vring.desc[head].flags = VRING_DESC_F_INDIRECT | vq->avail_used_flags;
 
        DPrintf(5, "Added buffer head %i to Q%d\n", head, vq->vq.index);
        head++;
        if (head >= vq->packed.vring.num) {
            head = 0;
            vq->packed.avail_wrap_counter ^= 1;
            vq->avail_used_flags ^=
                1 << VRING_PACKED_DESC_F_AVAIL |
                1 << VRING_PACKED_DESC_F_USED;
        }
        vq->packed.next_avail_idx = head;
        /* We're using some buffers from the free list. */
        vq->num_free -= 1;
        vq->num_added += 1;
 
        vq->free_head = vq->packed.desc_state[id].next;
 
        /* Store token and indirect buffer state. */
        vq->packed.desc_state[id].num = 1;
        vq->packed.desc_state[id].data = opaque;
        vq->packed.desc_state[id].last = id;
 
    } else {
        unsigned int n;
        u16 curr, prev, head_flags;
        if (vq->num_free < descs_used) {
            DPrintf(6, "Can't add buffer to Q%d\n", vq->vq.index);
            return -ENOSPC;
        }
        desc = vq->packed.vring.desc;
        i = head;
        curr = id;
        for (n = 0; n < descs_used; n++) {
            u16 flags = vq->avail_used_flags;
            flags |= n < out ? 0 : VRING_DESC_F_WRITE;
            if (n != descs_used - 1) {
                flags |= VRING_DESC_F_NEXT;
            }
            desc[i].addr = sg[n].physAddr.QuadPart;
            desc[i].len = sg[n].length;
            desc[i].id = id;
            if (n == 0) {
                head_flags = flags;
            }
            else {
                desc[i].flags = flags;
            }
 
            prev = curr;
            curr = vq->packed.desc_state[curr].next;
 
            if (++i >= vq->packed.vring.num) {
                i = 0;
                vq->avail_used_flags ^=
                    1 << VRING_PACKED_DESC_F_AVAIL |
                    1 << VRING_PACKED_DESC_F_USED;
            }
        }
 
        if (i < head)
            vq->packed.avail_wrap_counter ^= 1;
 
        /* We're using some buffers from the free list. */
        vq->num_free -= descs_used;
 
        /* Update free pointer */
        vq->packed.next_avail_idx = i;
        vq->free_head = curr;
 
        /* Store token. */
        vq->packed.desc_state[id].num = (u16)descs_used;
        vq->packed.desc_state[id].data = opaque;
        vq->packed.desc_state[id].last = prev;
 
        /*
         * A driver MUST NOT make the first descriptor in the list
         * available before all subsequent descriptors comprising
         * the list are made available.
         */
        KeMemoryBarrier();
        vq->packed.vring.desc[head].flags = head_flags;
        vq->num_added += descs_used;
 
        DPrintf(5, "Added buffer head @%i+%d to Q%d\n", head, descs_used, vq->vq.index);
    }
 
    return 0;
}

Referenced by vring_new_virtqueue_packed().

virtqueue_detach_unused_buf_packed()

static void * virtqueue_detach_unused_buf_packed ( struct virtqueue *  _vq )

static

Definition at line 306 of file VirtIORing-Packed.c.

{
    struct virtqueue_packed *vq = packedvq(_vq);
    unsigned int i;
    void *buf;
 
    for (i = 0; i < vq->packed.vring.num; i++) {
        if (!vq->packed.desc_state[i].data)
            continue;
        /* detach_buf clears data, so grab it now. */
        buf = vq->packed.desc_state[i].data;
        detach_buf_packed(vq, i);
        return buf;
    }
    /* That should have freed everything. */
    BUG_ON(vq->num_free != vq->packed.vring.num);
 
    return NULL;
}

Referenced by vring_new_virtqueue_packed().

virtqueue_disable_cb_packed()

static void virtqueue_disable_cb_packed ( struct virtqueue *  _vq )

static

Definition at line 326 of file VirtIORing-Packed.c.

{
    struct virtqueue_packed *vq = packedvq(_vq);
 
    if (vq->packed.event_flags_shadow != VRING_PACKED_EVENT_FLAG_DISABLE) {
        vq->packed.event_flags_shadow = VRING_PACKED_EVENT_FLAG_DISABLE;
        vq->packed.vring.driver->flags = vq->packed.event_flags_shadow;
    }
}

Referenced by vring_new_virtqueue_packed().

virtqueue_enable_cb_delayed_packed()

static bool virtqueue_enable_cb_delayed_packed ( struct virtqueue *  _vq )

static

Definition at line 401 of file VirtIORing-Packed.c.

{
    struct virtqueue_packed *vq = packedvq(_vq);
    bool event_suppression_enabled = vq->vq.vdev->event_suppression_enabled;
    u16 used_idx, wrap_counter;
    u16 bufs;
 
    /*
     * We optimistically turn back on interrupts, then check if there was
     * more to do.
     */
 
    if (event_suppression_enabled) {
        /* TODO: tune this threshold */
        bufs = (vq->packed.vring.num - vq->num_free) * 3 / 4;
        wrap_counter = vq->packed.used_wrap_counter;
 
        used_idx = vq->last_used_idx + bufs;
        if (used_idx >= vq->packed.vring.num) {
            used_idx -= (u16)vq->packed.vring.num;
            wrap_counter ^= 1;
        }
 
        vq->packed.vring.driver->off_wrap = used_idx |
            (wrap_counter << VRING_PACKED_EVENT_F_WRAP_CTR);
 
        /*
         * We need to update event offset and event wrap
         * counter first before updating event flags.
         */
        KeMemoryBarrier();
    }
 
    if (vq->packed.event_flags_shadow == VRING_PACKED_EVENT_FLAG_DISABLE) {
        vq->packed.event_flags_shadow = event_suppression_enabled ?
            VRING_PACKED_EVENT_FLAG_DESC :
            VRING_PACKED_EVENT_FLAG_ENABLE;
        vq->packed.vring.driver->flags = vq->packed.event_flags_shadow;
    }
 
    /*
     * We need to update event suppression structure first
     * before re-checking for more used buffers.
     */
    KeMemoryBarrier();
 
    if (is_used_desc_packed(vq,
        vq->last_used_idx,
        vq->packed.used_wrap_counter)) {
        return false;
    }
 
    return true;
}

Referenced by vring_new_virtqueue_packed().

virtqueue_enable_cb_packed()

static bool virtqueue_enable_cb_packed ( struct virtqueue *  _vq )

static

Definition at line 393 of file VirtIORing-Packed.c.

{
    struct virtqueue_packed *vq = packedvq(_vq);
    unsigned last_used_idx = virtqueue_enable_cb_prepare_packed(vq);
 
    return !virtqueue_poll_packed(vq, (u16)last_used_idx);
}

Referenced by vring_new_virtqueue_packed().

virtqueue_enable_cb_prepare_packed()

static unsigned virtqueue_enable_cb_prepare_packed ( struct virtqueue_packed *  vq )

inlinestatic

Definition at line 362 of file VirtIORing-Packed.c.

{
    bool event_suppression_enabled = vq->vq.vdev->event_suppression_enabled;
    /*
     * We optimistically turn back on interrupts, then check if there was
     * more to do.
     */
 
    if (event_suppression_enabled) {
        vq->packed.vring.driver->off_wrap =
            vq->last_used_idx |
            (vq->packed.used_wrap_counter <<
                VRING_PACKED_EVENT_F_WRAP_CTR);
        /*
         * We need to update event offset and event wrap
         * counter first before updating event flags.
         */
        KeMemoryBarrier();
    }
 
    if (vq->packed.event_flags_shadow == VRING_PACKED_EVENT_FLAG_DISABLE) {
        vq->packed.event_flags_shadow = event_suppression_enabled ?
            VRING_PACKED_EVENT_FLAG_DESC :
            VRING_PACKED_EVENT_FLAG_ENABLE;
        vq->packed.vring.driver->flags = vq->packed.event_flags_shadow;
    }
 
    return vq->last_used_idx | ((u16)vq->packed.used_wrap_counter <<
        VRING_PACKED_EVENT_F_WRAP_CTR);
}

Referenced by virtqueue_enable_cb_packed().

virtqueue_get_buf_packed()

static void * virtqueue_get_buf_packed ( struct virtqueue *  _vq, unsigned int *  len  )

static

Definition at line 486 of file VirtIORing-Packed.c.

{
    struct virtqueue_packed *vq = packedvq(_vq);
    u16 last_used, id;
    void *ret;
 
    if (!more_used_packed(vq)) {
        DPrintf(6, "%s: No more buffers in queue\n", __FUNCTION__);
        return NULL;
    }
 
    /* Only get used elements after they have been exposed by host. */
    KeMemoryBarrier();
 
    last_used = vq->last_used_idx;
    id = vq->packed.vring.desc[last_used].id;
    *len = vq->packed.vring.desc[last_used].len;
 
    if (id >= vq->packed.vring.num) {
        BAD_RING(vq, "id %u out of range\n", id);
        return NULL;
    }
    if (!vq->packed.desc_state[id].data) {
        BAD_RING(vq, "id %u is not a head!\n", id);
        return NULL;
    }
 
    /* detach_buf_packed clears data, so grab it now. */
    ret = vq->packed.desc_state[id].data;
    detach_buf_packed(vq, id);
 
    vq->last_used_idx += vq->packed.desc_state[id].num;
    if (vq->last_used_idx >= vq->packed.vring.num) {
        vq->last_used_idx -= (u16)vq->packed.vring.num;
        vq->packed.used_wrap_counter ^= 1;
    }
 
    /*
     * If we expect an interrupt for the next entry, tell host
     * by writing event index and flush out the write before
     * the read in the next get_buf call.
     */
    if (vq->packed.event_flags_shadow == VRING_PACKED_EVENT_FLAG_DESC) {
        vq->packed.vring.driver->off_wrap = vq->last_used_idx |
            ((u16)vq->packed.used_wrap_counter <<
                VRING_PACKED_EVENT_F_WRAP_CTR);
        KeMemoryBarrier();
    }
 
    return ret;
}

Referenced by vring_new_virtqueue_packed().

virtqueue_has_buf_packed()

static BOOLEAN virtqueue_has_buf_packed ( struct virtqueue *  _vq )

static

Definition at line 540 of file VirtIORing-Packed.c.

{
    struct virtqueue_packed *vq = packedvq(_vq);
    return more_used_packed(vq);
}

Referenced by vring_new_virtqueue_packed().

virtqueue_is_interrupt_enabled_packed()

static BOOLEAN virtqueue_is_interrupt_enabled_packed ( struct virtqueue *  _vq )

static

Definition at line 456 of file VirtIORing-Packed.c.

{
    struct virtqueue_packed *vq = packedvq(_vq);
    return vq->packed.event_flags_shadow & VRING_PACKED_EVENT_FLAG_DISABLE;
}

Referenced by vring_new_virtqueue_packed().

virtqueue_kick_always_packed()

static void virtqueue_kick_always_packed ( struct virtqueue *  _vq )

static

Definition at line 589 of file VirtIORing-Packed.c.

{
    struct virtqueue_packed *vq = packedvq(_vq);
    KeMemoryBarrier();
    vq->num_added = 0;
    virtqueue_notify(_vq);
}

Referenced by vring_new_virtqueue_packed().

virtqueue_kick_prepare_packed()

static bool virtqueue_kick_prepare_packed ( struct virtqueue *  _vq )

static

Definition at line 546 of file VirtIORing-Packed.c.

{
    struct virtqueue_packed *vq = packedvq(_vq);
    u16 new, old, off_wrap, flags, wrap_counter, event_idx;
    bool needs_kick;
    union {
        struct {
            __le16 off_wrap;
            __le16 flags;
        };
        u32 value32;
    } snapshot;
 
    /*
     * We need to expose the new flags value before checking notification
     * suppressions.
     */
    KeMemoryBarrier();
 
    old = vq->packed.next_avail_idx - vq->num_added;
    new = vq->packed.next_avail_idx;
    vq->num_added = 0;
 
    snapshot.value32 = *(u32 *)vq->packed.vring.device;
    flags = snapshot.flags;
 
    if (flags != VRING_PACKED_EVENT_FLAG_DESC) {
        needs_kick = (flags != VRING_PACKED_EVENT_FLAG_DISABLE);
        goto out;
    }
 
    off_wrap = snapshot.off_wrap;
 
    wrap_counter = off_wrap >> VRING_PACKED_EVENT_F_WRAP_CTR;
    event_idx = off_wrap & ~(1 << VRING_PACKED_EVENT_F_WRAP_CTR);
    if (wrap_counter != vq->packed.avail_wrap_counter)
        event_idx -= (u16)vq->packed.vring.num;
 
    needs_kick = vring_need_event(event_idx, new, old);
out:
    return needs_kick;
}

Referenced by vring_new_virtqueue_packed().

virtqueue_poll_packed()

static bool virtqueue_poll_packed ( struct virtqueue_packed *  vq, u16  off_wrap  )

inlinestatic

Definition at line 349 of file VirtIORing-Packed.c.

{
    bool wrap_counter;
    u16 used_idx;
    KeMemoryBarrier();
 
    wrap_counter = off_wrap >> VRING_PACKED_EVENT_F_WRAP_CTR;
    used_idx = off_wrap & ~(1 << VRING_PACKED_EVENT_F_WRAP_CTR);
 
    return is_used_desc_packed(vq, used_idx, wrap_counter);
 
}

Referenced by virtqueue_enable_cb_packed().

virtqueue_shutdown_packed()

static void virtqueue_shutdown_packed ( struct virtqueue *  _vq )

static

Definition at line 462 of file VirtIORing-Packed.c.

{
    struct virtqueue_packed *vq = packedvq(_vq);
    unsigned int num = vq->packed.vring.num;
    void *pages = vq->packed.vring.desc;
    unsigned int vring_align = _vq->vdev->addr ? PAGE_SIZE : SMP_CACHE_BYTES;
 
    RtlZeroMemory(pages, vring_size_packed(num, vring_align));
    vring_new_virtqueue_packed(
        _vq->index,
        num,
        vring_align,
        _vq->vdev,
        pages,
        _vq->notification_cb,
        _vq);
}

Referenced by vring_new_virtqueue_packed().

vring_control_block_size_packed()

unsigned int vring_control_block_size_packed

(

u16

qsize

)

Definition at line 157 of file VirtIORing-Packed.c.

{
    return sizeof(struct virtqueue_packed) + sizeof(struct vring_desc_state_packed) * qsize;
}

Referenced by vring_control_block_size().

vring_need_event()

static bool vring_need_event ( __u16  event_idx, __u16  new_idx, __u16  old  )

inlinestatic

Definition at line 101 of file VirtIORing-Packed.c.

{
    /* Note: Xen has similar logic for notification hold-off
    * in include/xen/interface/io/ring.h with req_event and req_prod
    * corresponding to event_idx + 1 and new_idx respectively.
    * Note also that req_event and req_prod in Xen start at 1,
    * event indexes in virtio start at 0. */
    return (__u16)(new_idx - event_idx - 1) < (__u16)(new_idx - old);
}

Referenced by virtqueue_kick_prepare_packed().

vring_new_virtqueue_packed()

struct virtqueue * vring_new_virtqueue_packed	(	unsigned int	index,
		unsigned int	num,
		unsigned int	vring_align,
		VirtIODevice *	vdev,
		void *	pages,
		void(*)(struct virtqueue *)	notify,
		void *	control
	)

Definition at line 598 of file VirtIORing-Packed.c.

{
    struct virtqueue_packed *vq = packedvq(control);
    unsigned int i;
 
    vq->vq.vdev = vdev;
    vq->vq.notification_cb = notify;
    vq->vq.index = index;
 
    vq->vq.avail_va = (u8 *)pages + num * sizeof(struct vring_packed_desc);
    vq->vq.used_va = (u8 *)vq->vq.avail_va + sizeof(struct vring_packed_desc_event);
 
    /* initialize the ring */
    vq->packed.vring.num = num;
    vq->packed.vring.desc = pages;
    vq->packed.vring.driver = vq->vq.avail_va;
    vq->packed.vring.device = vq->vq.used_va;
 
    vq->num_free = num;
    vq->free_head = 0;
    vq->num_added = 0;
    vq->packed.avail_wrap_counter = 1;
    vq->packed.used_wrap_counter = 1;
    vq->last_used_idx = 0;
    vq->avail_used_flags = 1 << VRING_PACKED_DESC_F_AVAIL;
    vq->packed.next_avail_idx = 0;
    vq->packed.event_flags_shadow = 0;
    vq->packed.desc_state = vq->desc_states;
 
    RtlZeroMemory(vq->packed.desc_state, num * sizeof(*vq->packed.desc_state));
    for (i = 0; i < num - 1; i++) {
        vq->packed.desc_state[i].next = i + 1;
    }
 
    vq->vq.add_buf = virtqueue_add_buf_packed;
    vq->vq.detach_unused_buf = virtqueue_detach_unused_buf_packed;
    vq->vq.disable_cb = virtqueue_disable_cb_packed;
    vq->vq.enable_cb = virtqueue_enable_cb_packed;
    vq->vq.enable_cb_delayed = virtqueue_enable_cb_delayed_packed;
    vq->vq.get_buf = virtqueue_get_buf_packed;
    vq->vq.has_buf = virtqueue_has_buf_packed;
    vq->vq.is_interrupt_enabled = virtqueue_is_interrupt_enabled_packed;
    vq->vq.kick_always = virtqueue_kick_always_packed;
    vq->vq.kick_prepare = virtqueue_kick_prepare_packed;
    vq->vq.shutdown = virtqueue_shutdown_packed;
    return &vq->vq;
}

Referenced by vio_modern_setup_vq(), and virtqueue_shutdown_packed().

vring_size_packed()

unsigned long vring_size_packed	(	unsigned int	num,
		unsigned long	align
	)

Definition at line 162 of file VirtIORing-Packed.c.

{
    /* array of descriptors */
    unsigned long res = num * sizeof(struct vring_packed_desc);
    /* driver and device event */
    res += 2 * sizeof(struct vring_packed_desc_event);
    return res;
}

Referenced by virtqueue_shutdown_packed(), and vring_size().