62#define BUG_ON(condition) { if (condition) { KeBugCheck(0xE0E1E2E3); }}
63#define BAD_RING(vq, fmt, ...) DPrintf(0, "%s: queue %d: " fmt, __FUNCTION__, vq->vq.index, __VA_ARGS__); BUG_ON(true)
66#define VRING_DESC_F_NEXT 1
68#define VRING_DESC_F_WRITE 2
70#define VRING_DESC_F_INDIRECT 4
76#define VRING_PACKED_DESC_F_AVAIL 7
77#define VRING_PACKED_DESC_F_USED 15
80#define VRING_PACKED_EVENT_FLAG_ENABLE 0x0
82#define VRING_PACKED_EVENT_FLAG_DISABLE 0x1
89#define VRING_PACKED_EVENT_FLAG_DESC 0x2
94#define VRING_PACKED_EVENT_F_WRAP_CTR 15
108 return (
__u16)(new_idx - event_idx - 1) < (
__u16)(new_idx - old);
155#define packedvq(vq) ((struct virtqueue_packed *)vq)
181 unsigned int descs_used;
185 descs_used =
out +
in;
192 if (va_indirect && vq->
num_free > 0) {
194 for (
i = 0;
i < descs_used;
i++) {
196 desc[
i].addr = sg[
i].physAddr.QuadPart;
197 desc[
i].len = sg[
i].length;
199 vq->
packed.vring.desc[
head].addr = phys_indirect;
206 DPrintf(5,
"Added buffer head %i to Q%d\n",
head, vq->
vq.index);
210 vq->
packed.avail_wrap_counter ^= 1;
224 vq->
packed.desc_state[
id].data = opaque;
229 u16 curr, prev, head_flags;
231 DPrintf(6,
"Can't add buffer to Q%d\n", vq->
vq.index);
237 for (
n = 0;
n < descs_used;
n++) {
240 if (
n != descs_used - 1) {
243 desc[
i].addr = sg[
n].physAddr.QuadPart;
244 desc[
i].len = sg[
n].length;
254 curr = vq->
packed.desc_state[curr].next;
256 if (++
i >= vq->
packed.vring.num) {
265 vq->
packed.avail_wrap_counter ^= 1;
276 vq->
packed.desc_state[
id].data = opaque;
277 vq->
packed.desc_state[
id].last = prev;
285 vq->
packed.vring.desc[
head].flags = head_flags;
288 DPrintf(5,
"Added buffer head @%i+%d to Q%d\n",
head, descs_used, vq->
vq.index);
312 for (
i = 0;
i <
vq->packed.vring.num;
i++) {
313 if (!
vq->packed.desc_state[
i].data)
316 buf =
vq->packed.desc_state[
i].data;
332 vq->packed.vring.driver->flags =
vq->packed.event_flags_shadow;
364 bool event_suppression_enabled =
vq->vq.vdev->event_suppression_enabled;
370 if (event_suppression_enabled) {
371 vq->packed.vring.driver->off_wrap =
373 (
vq->packed.used_wrap_counter <<
383 vq->packed.event_flags_shadow = event_suppression_enabled ?
386 vq->packed.vring.driver->flags =
vq->packed.event_flags_shadow;
389 return vq->last_used_idx | ((
u16)
vq->packed.used_wrap_counter <<
404 bool event_suppression_enabled =
vq->vq.vdev->event_suppression_enabled;
405 u16 used_idx, wrap_counter;
413 if (event_suppression_enabled) {
415 bufs = (
vq->packed.vring.num -
vq->num_free) * 3 / 4;
416 wrap_counter =
vq->packed.used_wrap_counter;
418 used_idx =
vq->last_used_idx +
bufs;
419 if (used_idx >=
vq->packed.vring.num) {
420 used_idx -= (
u16)
vq->packed.vring.num;
424 vq->packed.vring.driver->off_wrap = used_idx |
435 vq->packed.event_flags_shadow = event_suppression_enabled ?
438 vq->packed.vring.driver->flags =
vq->packed.event_flags_shadow;
449 vq->packed.used_wrap_counter)) {
465 unsigned int num =
vq->packed.vring.num;
466 void *pages =
vq->packed.vring.desc;
483 vq->packed.used_wrap_counter);
502 last_used =
vq->last_used_idx;
503 id =
vq->packed.vring.desc[last_used].id;
504 *
len =
vq->packed.vring.desc[last_used].len;
506 if (
id >=
vq->packed.vring.num) {
510 if (!
vq->packed.desc_state[
id].data) {
516 ret =
vq->packed.desc_state[
id].data;
519 vq->last_used_idx +=
vq->packed.desc_state[
id].num;
520 if (
vq->last_used_idx >=
vq->packed.vring.num) {
521 vq->last_used_idx -= (
u16)
vq->packed.vring.num;
522 vq->packed.used_wrap_counter ^= 1;
531 vq->packed.vring.driver->off_wrap =
vq->last_used_idx |
532 ((
u16)
vq->packed.used_wrap_counter <<
549 u16 new, old, off_wrap,
flags, wrap_counter, event_idx;
566 new = vq->
packed.next_avail_idx;
581 if (wrap_counter != vq->
packed.avail_wrap_counter)
601 unsigned int vring_align,
618 vq->packed.vring.num =
num;
619 vq->packed.vring.desc = pages;
620 vq->packed.vring.driver =
vq->vq.avail_va;
621 vq->packed.vring.device =
vq->vq.used_va;
626 vq->packed.avail_wrap_counter = 1;
627 vq->packed.used_wrap_counter = 1;
628 vq->last_used_idx = 0;
630 vq->packed.next_avail_idx = 0;
631 vq->packed.event_flags_shadow = 0;
632 vq->packed.desc_state =
vq->desc_states;
635 for (
i = 0;
i <
num - 1;
i++) {
636 vq->packed.desc_state[
i].next =
i + 1;
#define VRING_PACKED_EVENT_FLAG_DISABLE
static void detach_buf_packed(struct virtqueue_packed *vq, unsigned int id)
struct virtqueue * vring_new_virtqueue_packed(unsigned int index, unsigned int num, unsigned int vring_align, VirtIODevice *vdev, void *pages, void(*notify)(struct virtqueue *), void *control)
#define VRING_PACKED_EVENT_FLAG_DESC
static bool more_used_packed(const struct virtqueue_packed *vq)
static bool vring_need_event(__u16 event_idx, __u16 new_idx, __u16 old)
static void virtqueue_kick_always_packed(struct virtqueue *_vq)
static bool virtqueue_enable_cb_delayed_packed(struct virtqueue *_vq)
static bool virtqueue_kick_prepare_packed(struct virtqueue *_vq)
static BOOLEAN virtqueue_is_interrupt_enabled_packed(struct virtqueue *_vq)
static unsigned virtqueue_enable_cb_prepare_packed(struct virtqueue_packed *vq)
#define VRING_DESC_F_WRITE
#define VRING_PACKED_EVENT_F_WRAP_CTR
static bool virtqueue_poll_packed(struct virtqueue_packed *vq, u16 off_wrap)
#define BAD_RING(vq, fmt,...)
static void * virtqueue_get_buf_packed(struct virtqueue *_vq, unsigned int *len)
static int virtqueue_add_buf_packed(struct virtqueue *_vq, struct scatterlist sg[], unsigned int out, unsigned int in, void *opaque, void *va_indirect, ULONGLONG phys_indirect)
#define BUG_ON(condition)
static BOOLEAN virtqueue_has_buf_packed(struct virtqueue *_vq)
#define VRING_PACKED_EVENT_FLAG_ENABLE
static bool virtqueue_enable_cb_packed(struct virtqueue *_vq)
static bool is_used_desc_packed(const struct virtqueue_packed *vq, u16 idx, bool used_wrap_counter)
static void * virtqueue_detach_unused_buf_packed(struct virtqueue *_vq)
#define VRING_DESC_F_NEXT
static void virtqueue_disable_cb_packed(struct virtqueue *_vq)
unsigned long vring_size_packed(unsigned int num, unsigned long align)
static void virtqueue_shutdown_packed(struct virtqueue *_vq)
unsigned int vring_control_block_size_packed(u16 qsize)
#define VRING_PACKED_DESC_F_USED
#define VRING_DESC_F_INDIRECT
#define VRING_PACKED_DESC_F_AVAIL
void virtqueue_notify(struct virtqueue *vq)
struct outqueuenode * head
#define DPrintf(Level, Fmt)
int align(int length, int align)
GLenum GLuint GLenum GLsizei const GLchar * buf
GLsizei GLenum const GLvoid GLsizei GLenum GLbyte GLbyte GLbyte GLdouble GLdouble GLdouble GLfloat GLfloat GLfloat GLint GLint GLint GLshort GLshort GLshort GLubyte GLubyte GLubyte GLuint GLuint GLuint GLushort GLushort GLushort GLbyte GLbyte GLbyte GLbyte GLdouble GLdouble GLdouble GLdouble GLfloat GLfloat GLfloat GLfloat GLint GLint GLint GLint GLshort GLshort GLshort GLshort GLubyte GLubyte GLubyte GLubyte GLuint GLuint GLuint GLuint GLushort GLushort GLushort GLushort GLboolean const GLdouble const GLfloat const GLint const GLshort const GLbyte const GLdouble const GLfloat const GLint const GLshort const GLdouble const GLfloat const GLint const GLshort const GLdouble const GLfloat const GLint const GLshort const GLdouble const GLfloat const GLint const GLshort const GLdouble const GLdouble const GLfloat const GLfloat const GLint const GLint const GLshort const GLshort const GLdouble const GLfloat const GLint const GLshort const GLdouble const GLfloat const GLint const GLshort const GLdouble const GLfloat const GLint const GLshort const GLdouble const GLfloat const GLint const GLshort const GLdouble const GLfloat const GLint const GLshort const GLdouble const GLfloat const GLint const GLshort const GLdouble const GLfloat const GLint const GLshort GLenum GLenum GLenum GLfloat GLenum GLint GLenum GLenum GLenum GLfloat GLenum GLenum GLint GLenum GLfloat GLenum GLint GLint GLushort GLenum GLenum GLfloat GLenum GLenum GLint GLfloat const GLubyte GLenum GLenum GLenum const GLfloat GLenum GLenum const GLint GLenum GLint GLint GLsizei GLsizei GLint GLenum GLenum const GLvoid GLenum GLenum const GLfloat GLenum GLenum const GLint GLenum GLenum const GLdouble GLenum GLenum const GLfloat GLenum GLenum const GLint GLsizei GLuint GLfloat GLuint GLbitfield GLfloat GLint GLuint GLboolean GLenum GLfloat GLenum GLbitfield GLenum GLfloat GLfloat GLint GLint const GLfloat GLenum GLfloat GLfloat GLint GLint GLfloat GLfloat GLint GLint const GLfloat GLint GLfloat GLfloat GLint GLfloat GLfloat GLint GLfloat GLfloat const GLdouble const GLfloat const GLdouble const GLfloat GLint i
static const WCHAR desc[]
FORCEINLINE VOID KeMemoryBarrier(VOID)
struct vring_desc_state_packed * desc_state
struct vring_packed_desc * desc
struct virtqueue_packed::@4337 packed
struct vring_packed_desc_event * device
struct vring_desc_state_packed desc_states[]
struct virtqueue_packed::@4337::@4338 vring
struct vring_packed_desc_event * driver
void(* notification_cb)(struct virtqueue *vq)
#define RtlZeroMemory(Destination, Length)
__u64 __bitwise__ __virtio64
ActualNumberDriverObjects * sizeof(PDRIVER_OBJECT)) PDRIVER_OBJECT *DriverObjectList