VirtIORing.c File Reference

#include "osdep.h"
#include "virtio_pci.h"
#include "VirtIO.h"
#include "kdebugprint.h"
#include "virtio_ring.h"
#include "windows/virtio_ring_allocation.h"
#include <pshpack1.h>
#include <poppack.h>

Include dependency graph for VirtIORing.c:

Go to the source code of this file.

Classes
struct	vring_desc
struct	vring_avail
struct	vring_used_elem
struct	vring_used
struct	vring
struct	virtqueue_split

Macros
#define	DESC_INDEX(num, i)   ((i) & ((num) - 1))
#define	VIRTQ_DESC_F_NEXT   1
#define	VIRTQ_DESC_F_WRITE   2
#define	VIRTQ_DESC_F_INDIRECT   4
#define	VIRTQ_USED_F_NO_NOTIFY   1
#define	VIRTQ_AVAIL_F_NO_INTERRUPT   1
#define	VRING_AVAIL_ALIGN_SIZE   2
#define	VRING_USED_ALIGN_SIZE   4
#define	VRING_DESC_ALIGN_SIZE   16
#define	vring_used_event(vr)   ((vr)->avail->ring[(vr)->num])
#define	vring_avail_event(vr)   (*(__virtio16 *)&(vr)->used->ring[(vr)->num])
#define	splitvq(vq)   ((struct virtqueue_split *)vq)

Functions
static void	vring_init (struct vring *vr, unsigned int num, void *p, unsigned long align)
static unsigned	vring_size_split (unsigned int num, unsigned long align)
static int	vring_need_event (__u16 event_idx, __u16 new_idx, __u16 old)
static u16	get_unused_desc (struct virtqueue_split *vq)
static void	put_unused_desc_chain (struct virtqueue_split *vq, u16 idx)
static int	virtqueue_add_buf_split (struct virtqueue *_vq, struct scatterlist sg[], unsigned int out, unsigned int in, void *opaque, void *va_indirect, ULONGLONG phys_indirect)
static void *	virtqueue_get_buf_split (struct virtqueue *_vq, unsigned int *len)
static BOOLEAN	virtqueue_has_buf_split (struct virtqueue *_vq)
static bool	virtqueue_kick_prepare_split (struct virtqueue *_vq)
static void	virtqueue_kick_always_split (struct virtqueue *_vq)
static bool	virtqueue_enable_cb_split (struct virtqueue *_vq)
static bool	virtqueue_enable_cb_delayed_split (struct virtqueue *_vq)
static void	virtqueue_disable_cb_split (struct virtqueue *_vq)
static BOOLEAN	virtqueue_is_interrupt_enabled_split (struct virtqueue *_vq)
static void	virtqueue_shutdown_split (struct virtqueue *_vq)
static void *	virtqueue_detach_unused_buf_split (struct virtqueue *_vq)
unsigned int	vring_control_block_size (u16 qsize, bool packed)
struct virtqueue *	vring_new_virtqueue_split (unsigned int index, unsigned int num, unsigned int vring_align, VirtIODevice *vdev, void *pages, void(*notify)(struct virtqueue *), void *control)
void	vring_transport_features (VirtIODevice *vdev, u64 *features)
u32	virtio_get_indirect_page_capacity ()
unsigned long	vring_size (unsigned int num, unsigned long align, bool packed)

Macro Definition Documentation

DESC_INDEX

#define DESC_INDEX	(		num,
			i
	)		((i) & ((num) - 1))

Definition at line 39 of file VirtIORing.c.

splitvq

#define splitvq

(

vq

)

((struct virtqueue_split *)vq)

Definition at line 191 of file VirtIORing.c.

VIRTQ_AVAIL_F_NO_INTERRUPT

#define VIRTQ_AVAIL_F_NO_INTERRUPT   1

Definition at line 55 of file VirtIORing.c.

VIRTQ_DESC_F_INDIRECT

#define VIRTQ_DESC_F_INDIRECT   4

Definition at line 46 of file VirtIORing.c.

VIRTQ_DESC_F_NEXT

#define VIRTQ_DESC_F_NEXT   1

Definition at line 42 of file VirtIORing.c.

VIRTQ_DESC_F_WRITE

#define VIRTQ_DESC_F_WRITE   2

Definition at line 44 of file VirtIORing.c.

VIRTQ_USED_F_NO_NOTIFY

#define VIRTQ_USED_F_NO_NOTIFY   1

Definition at line 51 of file VirtIORing.c.

VRING_AVAIL_ALIGN_SIZE

#define VRING_AVAIL_ALIGN_SIZE   2

Definition at line 99 of file VirtIORing.c.

vring_avail_event

#define vring_avail_event

(

vr

)

(*(__virtio16 *)&(vr)->used->ring[(vr)->num])

Definition at line 141 of file VirtIORing.c.

VRING_DESC_ALIGN_SIZE

#define VRING_DESC_ALIGN_SIZE   16

Definition at line 101 of file VirtIORing.c.

VRING_USED_ALIGN_SIZE

#define VRING_USED_ALIGN_SIZE   4

Definition at line 100 of file VirtIORing.c.

vring_used_event

#define vring_used_event

(

vr

)

((vr)->avail->ring[(vr)->num])

Definition at line 140 of file VirtIORing.c.

Function Documentation

get_unused_desc()

static u16 get_unused_desc ( struct virtqueue_split *  vq )

inlinestatic

Definition at line 196 of file VirtIORing.c.

{
    u16 idx = vq->first_unused;
    ASSERT(vq->num_unused > 0);
 
    vq->first_unused = vq->vring.desc[idx].next;
    vq->num_unused--;
    return idx;
}

Referenced by virtqueue_add_buf_split().

put_unused_desc_chain()

static void put_unused_desc_chain ( struct virtqueue_split *  vq, u16  idx  )

inlinestatic

Definition at line 207 of file VirtIORing.c.

{
    u16 start = idx;
 
    vq->opaque[idx] = NULL;
    while (vq->vring.desc[idx].flags & VIRTQ_DESC_F_NEXT) {
        idx = vq->vring.desc[idx].next;
        vq->num_unused++;
    }
 
    vq->vring.desc[idx].flags = VIRTQ_DESC_F_NEXT;
    vq->vring.desc[idx].next = vq->first_unused;
    vq->num_unused++;
 
    vq->first_unused = start;
}

Referenced by virtqueue_detach_unused_buf_split(), and virtqueue_get_buf_split().

virtio_get_indirect_page_capacity()

u32 virtio_get_indirect_page_capacity

(

)

Definition at line 550 of file VirtIORing.c.

{
    return PAGE_SIZE / sizeof(struct vring_desc);
}

virtqueue_add_buf_split()

static int virtqueue_add_buf_split ( struct virtqueue *  _vq, struct scatterlist  sg[], unsigned int  out, unsigned int  in, void *  opaque, void *  va_indirect, ULONGLONG  phys_indirect  )

static

Definition at line 225 of file VirtIORing.c.

{
    struct virtqueue_split *vq = splitvq(_vq);
    struct vring *vring = &vq->vring;
    unsigned int i;
    u16 idx;
 
    if (va_indirect && (out + in) > 1 && vq->num_unused > 0) {
        /* Use one indirect descriptor */
        struct vring_desc *desc = (struct vring_desc *)va_indirect;
 
        for (i = 0; i < out + in; i++) {
            desc[i].flags = (i < out ? 0 : VIRTQ_DESC_F_WRITE);
            desc[i].flags |= VIRTQ_DESC_F_NEXT;
            desc[i].addr = sg[i].physAddr.QuadPart;
            desc[i].len = sg[i].length;
            desc[i].next = (u16)i + 1;
        }
        desc[i - 1].flags &= ~VIRTQ_DESC_F_NEXT;
 
        idx = get_unused_desc(vq);
        vq->vring.desc[idx].flags = VIRTQ_DESC_F_INDIRECT;
        vq->vring.desc[idx].addr = phys_indirect;
        vq->vring.desc[idx].len = i * sizeof(struct vring_desc);
 
        vq->opaque[idx] = opaque;
    } else {
        u16 last_idx;
 
        /* Use out + in regular descriptors */
        if (out + in > vq->num_unused) {
            return -ENOSPC;
        }
 
        /* First descriptor */
        idx = last_idx = get_unused_desc(vq);
        vq->opaque[idx] = opaque;
 
        vring->desc[idx].addr = sg[0].physAddr.QuadPart;
        vring->desc[idx].len = sg[0].length;
        vring->desc[idx].flags = VIRTQ_DESC_F_NEXT;
        if (out == 0) {
            vring->desc[idx].flags |= VIRTQ_DESC_F_WRITE;
        }
        vring->desc[idx].next = vq->first_unused;
 
        /* The rest of descriptors */
        for (i = 1; i < out + in; i++) {
            last_idx = get_unused_desc(vq);
 
            vring->desc[last_idx].addr = sg[i].physAddr.QuadPart;
            vring->desc[last_idx].len = sg[i].length;
            vring->desc[last_idx].flags = VIRTQ_DESC_F_NEXT;
            if (i >= out) {
                vring->desc[last_idx].flags |= VIRTQ_DESC_F_WRITE;
            }
            vring->desc[last_idx].next = vq->first_unused;
        }
        vring->desc[last_idx].flags &= ~VIRTQ_DESC_F_NEXT;
    }
 
    /* Write the first descriptor into the available ring */
    vring->avail->ring[DESC_INDEX(vring->num, vq->master_vring_avail.idx)] = idx;
    KeMemoryBarrier();
    vring->avail->idx = ++vq->master_vring_avail.idx;
    vq->num_added_since_kick++;
 
    return 0;
}

Referenced by vring_new_virtqueue_split().

virtqueue_detach_unused_buf_split()

static void * virtqueue_detach_unused_buf_split ( struct virtqueue *  _vq )

static

Definition at line 455 of file VirtIORing.c.

{
    struct virtqueue_split *vq = splitvq(_vq);
    u16 idx;
    void *opaque = NULL;
 
    for (idx = 0; idx < (u16)vq->vring.num; idx++) {
        opaque = vq->opaque[idx];
        if (opaque) {
            put_unused_desc_chain(vq, idx);
            vq->vring.avail->idx = --vq->master_vring_avail.idx;
            break;
        }
    }
    return opaque;
}

Referenced by vring_new_virtqueue_split().

virtqueue_disable_cb_split()

static void virtqueue_disable_cb_split ( struct virtqueue *  _vq )

static

Definition at line 415 of file VirtIORing.c.

{
    struct virtqueue_split *vq = splitvq(_vq);
    if (virtqueue_is_interrupt_enabled(_vq)) {
        vq->master_vring_avail.flags |= VIRTQ_AVAIL_F_NO_INTERRUPT;
        if (!_vq->vdev->event_suppression_enabled)
        {
            vq->vring.avail->flags = vq->master_vring_avail.flags;
        }
    }
}

Referenced by vring_new_virtqueue_split().

virtqueue_enable_cb_delayed_split()

static bool virtqueue_enable_cb_delayed_split ( struct virtqueue *  _vq )

static

Definition at line 394 of file VirtIORing.c.

{
    struct virtqueue_split *vq = splitvq(_vq);
    u16 bufs;
 
    if (!virtqueue_is_interrupt_enabled(_vq)) {
        vq->master_vring_avail.flags &= ~VIRTQ_AVAIL_F_NO_INTERRUPT;
        if (!_vq->vdev->event_suppression_enabled)
        {
            vq->vring.avail->flags = vq->master_vring_avail.flags;
        }
    }
 
    /* Note that 3/4 is an arbitrary threshold */
    bufs = (u16)(vq->master_vring_avail.idx - vq->last_used) * 3 / 4;
    vring_used_event(&vq->vring) = vq->last_used + bufs;
    KeMemoryBarrier();
    return ((vq->vring.used->idx - vq->last_used) <= bufs);
}

Referenced by vring_new_virtqueue_split().

virtqueue_enable_cb_split()

static bool virtqueue_enable_cb_split ( struct virtqueue *  _vq )

static

Definition at line 376 of file VirtIORing.c.

{
    struct virtqueue_split *vq = splitvq(_vq);
    if (!virtqueue_is_interrupt_enabled(_vq)) {
        vq->master_vring_avail.flags &= ~VIRTQ_AVAIL_F_NO_INTERRUPT;
        if (!_vq->vdev->event_suppression_enabled)
        {
            vq->vring.avail->flags = vq->master_vring_avail.flags;
        }
    }
 
    vring_used_event(&vq->vring) = vq->last_used;
    KeMemoryBarrier();
    return (vq->last_used == vq->vring.used->idx);
}

Referenced by vring_new_virtqueue_split().

virtqueue_get_buf_split()

static void * virtqueue_get_buf_split ( struct virtqueue *  _vq, unsigned int *  len  )

static

Definition at line 303 of file VirtIORing.c.

{
    struct virtqueue_split *vq = splitvq(_vq);
    void *opaque;
    u16 idx;
 
    if (vq->last_used == (int)vq->vring.used->idx) {
        /* No descriptor index in the used ring */
        return NULL;
    }
    KeMemoryBarrier();
 
    idx = DESC_INDEX(vq->vring.num, vq->last_used);
    *len = vq->vring.used->ring[idx].len;
 
    /* Get the first used descriptor */
    idx = (u16)vq->vring.used->ring[idx].id;
    opaque = vq->opaque[idx];
 
    /* Put all descriptors back to the free list */
    put_unused_desc_chain(vq, idx);
 
    vq->last_used++;
    if (_vq->vdev->event_suppression_enabled && virtqueue_is_interrupt_enabled(_vq)) {
        vring_used_event(&vq->vring) = vq->last_used;
        KeMemoryBarrier();
    }
 
    ASSERT(opaque != NULL);
    return opaque;
}

Referenced by vring_new_virtqueue_split().

virtqueue_has_buf_split()

static BOOLEAN virtqueue_has_buf_split ( struct virtqueue *  _vq )

static

Definition at line 338 of file VirtIORing.c.

{
    struct virtqueue_split *vq = splitvq(_vq);
    return (vq->last_used != vq->vring.used->idx);
}

Referenced by vring_new_virtqueue_split().

virtqueue_is_interrupt_enabled_split()

static BOOLEAN virtqueue_is_interrupt_enabled_split ( struct virtqueue *  _vq )

static

Definition at line 428 of file VirtIORing.c.

{
    struct virtqueue_split *vq = splitvq(_vq);
    return !(vq->master_vring_avail.flags & VIRTQ_AVAIL_F_NO_INTERRUPT);
}

Referenced by vring_new_virtqueue_split().

virtqueue_kick_always_split()

static void virtqueue_kick_always_split ( struct virtqueue *  _vq )

static

Definition at line 366 of file VirtIORing.c.

{
    struct virtqueue_split *vq = splitvq(_vq);
    KeMemoryBarrier();
    vq->num_added_since_kick = 0;
    virtqueue_notify(_vq);
}

Referenced by vring_new_virtqueue_split().

virtqueue_kick_prepare_split()

static bool virtqueue_kick_prepare_split ( struct virtqueue *  _vq )

static

Definition at line 345 of file VirtIORing.c.

{
    struct virtqueue_split *vq = splitvq(_vq);
    bool wrap_around;
    u16 old, new;
    KeMemoryBarrier();
 
    wrap_around = (vq->num_added_since_kick >= (1 << 16));
 
    old = (u16)(vq->master_vring_avail.idx - vq->num_added_since_kick);
    new = vq->master_vring_avail.idx;
    vq->num_added_since_kick = 0;
 
    if (_vq->vdev->event_suppression_enabled) {
        return wrap_around || (bool)vring_need_event(vring_avail_event(&vq->vring), new, old);
    } else {
        return !(vq->vring.used->flags & VIRTQ_USED_F_NO_NOTIFY);
    }
}

Referenced by vring_new_virtqueue_split().

virtqueue_shutdown_split()

static void virtqueue_shutdown_split ( struct virtqueue *  _vq )

static

Definition at line 435 of file VirtIORing.c.

{
    struct virtqueue_split *vq = splitvq(_vq);
    unsigned int num = vq->vring.num;
    void *pages = vq->vring.desc;
    unsigned int vring_align = _vq->vdev->addr ? PAGE_SIZE : SMP_CACHE_BYTES;
 
    RtlZeroMemory(pages, vring_size_split(num, vring_align));
    (void)vring_new_virtqueue_split(
        _vq->index,
        vq->vring.num,
        vring_align,
        _vq->vdev,
        pages,
        _vq->notification_cb,
        vq);
}

Referenced by vring_new_virtqueue_split().

vring_control_block_size()

unsigned int vring_control_block_size	(	u16	qsize,
		bool	packed
	)

Definition at line 474 of file VirtIORing.c.

{
    unsigned int res;
    if (packed) {
        return vring_control_block_size_packed(qsize);
    }
    res = sizeof(struct virtqueue_split);
    res += sizeof(void *) * qsize;
    return res;
}

Referenced by vio_legacy_query_vq_alloc(), and vio_modern_query_vq_alloc().

vring_init()

static void vring_init ( struct vring *  vr, unsigned int  num, void *  p, unsigned long  align  )

inlinestatic

Definition at line 143 of file VirtIORing.c.

{
    vr->num = num;
    vr->desc = (struct vring_desc *)p;
    vr->avail = (struct vring_avail *)((__u8 *)p + num * sizeof(struct vring_desc));
    vr->used = (struct vring_used *)(((ULONG_PTR)&vr->avail->ring[num] + sizeof(__virtio16)
        + align - 1) & ~((ULONG_PTR)align - 1));
}

Referenced by vring_new_virtqueue_split().

vring_need_event()

static int vring_need_event ( __u16  event_idx, __u16  new_idx, __u16  old  )

inlinestatic

Definition at line 167 of file VirtIORing.c.

{
    /* Note: Xen has similar logic for notification hold-off
    * in include/xen/interface/io/ring.h with req_event and req_prod
    * corresponding to event_idx + 1 and new_idx respectively.
    * Note also that req_event and req_prod in Xen start at 1,
    * event indexes in virtio start at 0. */
    return (__u16)(new_idx - event_idx - 1) < (__u16)(new_idx - old);
}

Referenced by virtqueue_kick_prepare_split().

vring_new_virtqueue_split()

struct virtqueue * vring_new_virtqueue_split	(	unsigned int	index,
		unsigned int	num,
		unsigned int	vring_align,
		VirtIODevice *	vdev,
		void *	pages,
		void(*)(struct virtqueue *)	notify,
		void *	control
	)

Definition at line 486 of file VirtIORing.c.

{
    struct virtqueue_split *vq = splitvq(control);
    u16 i;
 
    if (DESC_INDEX(num, num) != 0) {
        DPrintf(0, "Virtqueue length %u is not a power of 2\n", num);
        return NULL;
    }
 
    RtlZeroMemory(vq, sizeof(*vq) + num * sizeof(void *));
 
    vring_init(&vq->vring, num, pages, vring_align);
    vq->vq.vdev = vdev;
    vq->vq.notification_cb = notify;
    vq->vq.index = index;
 
    /* Build a linked list of unused descriptors */
    vq->num_unused = num;
    vq->first_unused = 0;
    for (i = 0; i < num - 1; i++) {
        vq->vring.desc[i].flags = VIRTQ_DESC_F_NEXT;
        vq->vring.desc[i].next = i + 1;
    }
    vq->vq.avail_va = vq->vring.avail;
    vq->vq.used_va = vq->vring.used;
    vq->vq.add_buf = virtqueue_add_buf_split;
    vq->vq.detach_unused_buf = virtqueue_detach_unused_buf_split;
    vq->vq.disable_cb = virtqueue_disable_cb_split;
    vq->vq.enable_cb = virtqueue_enable_cb_split;
    vq->vq.enable_cb_delayed = virtqueue_enable_cb_delayed_split;
    vq->vq.get_buf = virtqueue_get_buf_split;
    vq->vq.has_buf = virtqueue_has_buf_split;
    vq->vq.is_interrupt_enabled = virtqueue_is_interrupt_enabled_split;
    vq->vq.kick_always = virtqueue_kick_always_split;
    vq->vq.kick_prepare = virtqueue_kick_prepare_split;
    vq->vq.shutdown = virtqueue_shutdown_split;
    return &vq->vq;
}

Referenced by vio_legacy_setup_vq(), vio_modern_setup_vq(), and virtqueue_shutdown_split().

vring_size()

unsigned long vring_size	(	unsigned int	num,
		unsigned long	align,
		bool	packed
	)

Definition at line 555 of file VirtIORing.c.

{
    if (packed) {
        return vring_size_packed(num, align);
    } else {
        return vring_size_split(num, align);
    }
}

Referenced by vio_legacy_query_vq_alloc(), vio_legacy_setup_vq(), and vring_pci_size().

vring_size_split()

static unsigned vring_size_split ( unsigned int  num, unsigned long  align  )

inlinestatic

Definition at line 153 of file VirtIORing.c.

{
#pragma warning (push)
#pragma warning (disable:4319)
    return ((sizeof(struct vring_desc) * num + sizeof(__virtio16) * (3 + num)
        + align - 1) & ~(align - 1))
        + sizeof(__virtio16) * 3 + sizeof(struct vring_used_elem) * num;
#pragma warning(pop)
}

Referenced by virtqueue_shutdown_split(), and vring_size().

vring_transport_features()

void vring_transport_features	(	VirtIODevice *	vdev,
		u64 *	features
	)

Definition at line 534 of file VirtIORing.c.

{
    unsigned int i;
 
    for (i = VIRTIO_TRANSPORT_F_START; i < VIRTIO_TRANSPORT_F_END; i++) {
        if (i != VIRTIO_RING_F_INDIRECT_DESC &&
            i != VIRTIO_RING_F_EVENT_IDX &&
            i != VIRTIO_F_VERSION_1) {
            virtio_feature_disable(*features, i);
        }
    }
}

Referenced by vio_legacy_set_features(), and vio_modern_set_features().