ReactOS  0.4.15-dev-985-gd905dd5
VirtIORing.c File Reference
#include "osdep.h"
#include "virtio_pci.h"
#include "VirtIO.h"
#include "kdebugprint.h"
#include "virtio_ring.h"
#include "windows/virtio_ring_allocation.h"
#include <pshpack1.h>
#include <poppack.h>
Include dependency graph for VirtIORing.c:

Go to the source code of this file.

Classes

struct  vring_desc
 
struct  vring_avail
 
struct  vring_used_elem
 
struct  vring_used
 
struct  vring
 
struct  virtqueue_split
 

Macros

#define DESC_INDEX(num, i)   ((i) & ((num) - 1))
 
#define VIRTQ_DESC_F_NEXT   1
 
#define VIRTQ_DESC_F_WRITE   2
 
#define VIRTQ_DESC_F_INDIRECT   4
 
#define VIRTQ_USED_F_NO_NOTIFY   1
 
#define VIRTQ_AVAIL_F_NO_INTERRUPT   1
 
#define VRING_AVAIL_ALIGN_SIZE   2
 
#define VRING_USED_ALIGN_SIZE   4
 
#define VRING_DESC_ALIGN_SIZE   16
 
#define vring_used_event(vr)   ((vr)->avail->ring[(vr)->num])
 
#define vring_avail_event(vr)   (*(__virtio16 *)&(vr)->used->ring[(vr)->num])
 
#define splitvq(vq)   ((struct virtqueue_split *)vq)
 

Functions

static void vring_init (struct vring *vr, unsigned int num, void *p, unsigned long align)
 
static unsigned vring_size_split (unsigned int num, unsigned long align)
 
static int vring_need_event (__u16 event_idx, __u16 new_idx, __u16 old)
 
static u16 get_unused_desc (struct virtqueue_split *vq)
 
static void put_unused_desc_chain (struct virtqueue_split *vq, u16 idx)
 
static int virtqueue_add_buf_split (struct virtqueue *_vq, struct scatterlist sg[], unsigned int out, unsigned int in, void *opaque, void *va_indirect, ULONGLONG phys_indirect)
 
static voidvirtqueue_get_buf_split (struct virtqueue *_vq, unsigned int *len)
 
static BOOLEAN virtqueue_has_buf_split (struct virtqueue *_vq)
 
static bool virtqueue_kick_prepare_split (struct virtqueue *_vq)
 
static void virtqueue_kick_always_split (struct virtqueue *_vq)
 
static bool virtqueue_enable_cb_split (struct virtqueue *_vq)
 
static bool virtqueue_enable_cb_delayed_split (struct virtqueue *_vq)
 
static void virtqueue_disable_cb_split (struct virtqueue *_vq)
 
static BOOLEAN virtqueue_is_interrupt_enabled_split (struct virtqueue *_vq)
 
static void virtqueue_shutdown_split (struct virtqueue *_vq)
 
static voidvirtqueue_detach_unused_buf_split (struct virtqueue *_vq)
 
unsigned int vring_control_block_size (u16 qsize, bool packed)
 
struct virtqueuevring_new_virtqueue_split (unsigned int index, unsigned int num, unsigned int vring_align, VirtIODevice *vdev, void *pages, void(*notify)(struct virtqueue *), void *control)
 
void vring_transport_features (VirtIODevice *vdev, u64 *features)
 
u32 virtio_get_indirect_page_capacity ()
 
unsigned long vring_size (unsigned int num, unsigned long align, bool packed)
 

Macro Definition Documentation

◆ DESC_INDEX

#define DESC_INDEX (   num,
  i 
)    ((i) & ((num) - 1))

Definition at line 39 of file VirtIORing.c.

◆ splitvq

#define splitvq (   vq)    ((struct virtqueue_split *)vq)

Definition at line 191 of file VirtIORing.c.

◆ VIRTQ_AVAIL_F_NO_INTERRUPT

#define VIRTQ_AVAIL_F_NO_INTERRUPT   1

Definition at line 55 of file VirtIORing.c.

◆ VIRTQ_DESC_F_INDIRECT

#define VIRTQ_DESC_F_INDIRECT   4

Definition at line 46 of file VirtIORing.c.

◆ VIRTQ_DESC_F_NEXT

#define VIRTQ_DESC_F_NEXT   1

Definition at line 42 of file VirtIORing.c.

◆ VIRTQ_DESC_F_WRITE

#define VIRTQ_DESC_F_WRITE   2

Definition at line 44 of file VirtIORing.c.

◆ VIRTQ_USED_F_NO_NOTIFY

#define VIRTQ_USED_F_NO_NOTIFY   1

Definition at line 51 of file VirtIORing.c.

◆ VRING_AVAIL_ALIGN_SIZE

#define VRING_AVAIL_ALIGN_SIZE   2

Definition at line 99 of file VirtIORing.c.

◆ vring_avail_event

#define vring_avail_event (   vr)    (*(__virtio16 *)&(vr)->used->ring[(vr)->num])

Definition at line 141 of file VirtIORing.c.

◆ VRING_DESC_ALIGN_SIZE

#define VRING_DESC_ALIGN_SIZE   16

Definition at line 101 of file VirtIORing.c.

◆ VRING_USED_ALIGN_SIZE

#define VRING_USED_ALIGN_SIZE   4

Definition at line 100 of file VirtIORing.c.

◆ vring_used_event

#define vring_used_event (   vr)    ((vr)->avail->ring[(vr)->num])

Definition at line 140 of file VirtIORing.c.

Function Documentation

◆ get_unused_desc()

static u16 get_unused_desc ( struct virtqueue_split vq)
inlinestatic

Definition at line 196 of file VirtIORing.c.

197 {
198  u16 idx = vq->first_unused;
199  ASSERT(vq->num_unused > 0);
200 
201  vq->first_unused = vq->vring.desc[idx].next;
202  vq->num_unused--;
203  return idx;
204 }
unsigned int idx
Definition: utils.c:41
unsigned int num_unused
Definition: VirtIORing.c:184
ASSERT((InvokeOnSuccess||InvokeOnError||InvokeOnCancel) ?(CompletionRoutine !=NULL) :TRUE)
struct vring vring
Definition: VirtIORing.c:179
USHORT u16
Definition: btrfs.h:13

Referenced by virtqueue_add_buf_split().

◆ put_unused_desc_chain()

static void put_unused_desc_chain ( struct virtqueue_split vq,
u16  idx 
)
inlinestatic

Definition at line 207 of file VirtIORing.c.

208 {
209  u16 start = idx;
210 
211  vq->opaque[idx] = NULL;
212  while (vq->vring.desc[idx].flags & VIRTQ_DESC_F_NEXT) {
213  idx = vq->vring.desc[idx].next;
214  vq->num_unused++;
215  }
216 
217  vq->vring.desc[idx].flags = VIRTQ_DESC_F_NEXT;
218  vq->vring.desc[idx].next = vq->first_unused;
219  vq->num_unused++;
220 
221  vq->first_unused = start;
222 }
void * opaque[]
Definition: VirtIORing.c:188
unsigned int idx
Definition: utils.c:41
smooth NULL
Definition: ftsmooth.c:416
unsigned int num_unused
Definition: VirtIORing.c:184
#define VIRTQ_DESC_F_NEXT
Definition: VirtIORing.c:42
GLuint start
Definition: gl.h:1545
struct vring vring
Definition: VirtIORing.c:179
USHORT u16
Definition: btrfs.h:13

Referenced by virtqueue_detach_unused_buf_split(), and virtqueue_get_buf_split().

◆ virtio_get_indirect_page_capacity()

u32 virtio_get_indirect_page_capacity ( )

Definition at line 550 of file VirtIORing.c.

551 {
552  return PAGE_SIZE / sizeof(struct vring_desc);
553 }
#define PAGE_SIZE
Definition: env_spec_w32.h:49

◆ virtqueue_add_buf_split()

static int virtqueue_add_buf_split ( struct virtqueue _vq,
struct scatterlist  sg[],
unsigned int  out,
unsigned int  in,
void opaque,
void va_indirect,
ULONGLONG  phys_indirect 
)
static

Definition at line 225 of file VirtIORing.c.

233 {
234  struct virtqueue_split *vq = splitvq(_vq);
235  struct vring *vring = &vq->vring;
236  unsigned int i;
237  u16 idx;
238 
239  if (va_indirect && (out + in) > 1 && vq->num_unused > 0) {
240  /* Use one indirect descriptor */
241  struct vring_desc *desc = (struct vring_desc *)va_indirect;
242 
243  for (i = 0; i < out + in; i++) {
244  desc[i].flags = (i < out ? 0 : VIRTQ_DESC_F_WRITE);
245  desc[i].flags |= VIRTQ_DESC_F_NEXT;
246  desc[i].addr = sg[i].physAddr.QuadPart;
247  desc[i].len = sg[i].length;
248  desc[i].next = (u16)i + 1;
249  }
250  desc[i - 1].flags &= ~VIRTQ_DESC_F_NEXT;
251 
252  idx = get_unused_desc(vq);
253  vq->vring.desc[idx].flags = VIRTQ_DESC_F_INDIRECT;
254  vq->vring.desc[idx].addr = phys_indirect;
255  vq->vring.desc[idx].len = i * sizeof(struct vring_desc);
256 
257  vq->opaque[idx] = opaque;
258  } else {
259  u16 last_idx;
260 
261  /* Use out + in regular descriptors */
262  if (out + in > vq->num_unused) {
263  return -ENOSPC;
264  }
265 
266  /* First descriptor */
267  idx = last_idx = get_unused_desc(vq);
268  vq->opaque[idx] = opaque;
269 
270  vring->desc[idx].addr = sg[0].physAddr.QuadPart;
271  vring->desc[idx].len = sg[0].length;
272  vring->desc[idx].flags = VIRTQ_DESC_F_NEXT;
273  if (out == 0) {
274  vring->desc[idx].flags |= VIRTQ_DESC_F_WRITE;
275  }
276  vring->desc[idx].next = vq->first_unused;
277 
278  /* The rest of descriptors */
279  for (i = 1; i < out + in; i++) {
280  last_idx = get_unused_desc(vq);
281 
282  vring->desc[last_idx].addr = sg[i].physAddr.QuadPart;
283  vring->desc[last_idx].len = sg[i].length;
284  vring->desc[last_idx].flags = VIRTQ_DESC_F_NEXT;
285  if (i >= out) {
286  vring->desc[last_idx].flags |= VIRTQ_DESC_F_WRITE;
287  }
288  vring->desc[last_idx].next = vq->first_unused;
289  }
290  vring->desc[last_idx].flags &= ~VIRTQ_DESC_F_NEXT;
291  }
292 
293  /* Write the first descriptor into the available ring */
294  vring->avail->ring[DESC_INDEX(vring->num, vq->master_vring_avail.idx)] = idx;
295  KeMemoryBarrier();
296  vring->avail->idx = ++vq->master_vring_avail.idx;
297  vq->num_added_since_kick++;
298 
299  return 0;
300 }
static u16 get_unused_desc(struct virtqueue_split *vq)
Definition: VirtIORing.c:196
void * opaque[]
Definition: VirtIORing.c:188
struct vring_avail * avail
Definition: VirtIORing.c:135
#define splitvq(vq)
Definition: VirtIORing.c:191
#define DESC_INDEX(num, i)
Definition: VirtIORing.c:39
struct virtqueue vq
Definition: VirtIORing.c:178
GLsizei GLenum const GLvoid GLsizei GLenum GLbyte GLbyte GLbyte GLdouble GLdouble GLdouble GLfloat GLfloat GLfloat GLint GLint GLint GLshort GLshort GLshort GLubyte GLubyte GLubyte GLuint GLuint GLuint GLushort GLushort GLushort GLbyte GLbyte GLbyte GLbyte GLdouble GLdouble GLdouble GLdouble GLfloat GLfloat GLfloat GLfloat GLint GLint GLint GLint GLshort GLshort GLshort GLshort GLubyte GLubyte GLubyte GLubyte GLuint GLuint GLuint GLuint GLushort GLushort GLushort GLushort GLboolean const GLdouble const GLfloat const GLint const GLshort const GLbyte const GLdouble const GLfloat const GLint const GLshort const GLdouble const GLfloat const GLint const GLshort const GLdouble const GLfloat const GLint const GLshort const GLdouble const GLfloat const GLint const GLshort const GLdouble const GLdouble const GLfloat const GLfloat const GLint const GLint const GLshort const GLshort const GLdouble const GLfloat const GLint const GLshort const GLdouble const GLfloat const GLint const GLshort const GLdouble const GLfloat const GLint const GLshort const GLdouble const GLfloat const GLint const GLshort const GLdouble const GLfloat const GLint const GLshort const GLdouble const GLfloat const GLint const GLshort const GLdouble const GLfloat const GLint const GLshort GLenum GLenum GLenum GLfloat GLenum GLint GLenum GLenum GLenum GLfloat GLenum GLenum GLint GLenum GLfloat GLenum GLint GLint GLushort GLenum GLenum GLfloat GLenum GLenum GLint GLfloat const GLubyte GLenum GLenum GLenum const GLfloat GLenum GLenum const GLint GLenum GLint GLint GLsizei GLsizei GLint GLenum GLenum const GLvoid GLenum GLenum const GLfloat GLenum GLenum const GLint GLenum GLenum const GLdouble GLenum GLenum const GLfloat GLenum GLenum const GLint GLsizei GLuint GLfloat GLuint GLbitfield GLfloat GLint GLuint GLboolean GLenum GLfloat GLenum GLbitfield GLenum GLfloat GLfloat GLint GLint const GLfloat GLenum GLfloat GLfloat GLint GLint GLfloat GLfloat GLint GLint const GLfloat GLint GLfloat GLfloat GLint GLfloat GLfloat GLint GLfloat GLfloat const GLdouble const GLfloat const GLdouble const GLfloat GLint i
Definition: glfuncs.h:248
static const WCHAR desc[]
Definition: protectdata.c:36
unsigned int num_added_since_kick
Definition: VirtIORing.c:185
unsigned int idx
Definition: utils.c:41
Definition: arc.h:49
#define VIRTQ_DESC_F_WRITE
Definition: VirtIORing.c:44
static FILE * out
Definition: regtests2xml.c:44
#define u16
Definition: types.h:8
unsigned int num_unused
Definition: VirtIORing.c:184
unsigned int num
Definition: VirtIORing.c:131
#define VIRTQ_DESC_F_NEXT
Definition: VirtIORing.c:42
#define VIRTQ_DESC_F_INDIRECT
Definition: VirtIORing.c:46
struct vring_desc * desc
Definition: VirtIORing.c:133
GLuint in
Definition: glext.h:9616
struct vring vring
Definition: VirtIORing.c:179
struct virtqueue_split::@1003 master_vring_avail
FORCEINLINE VOID KeMemoryBarrier(VOID)
Definition: ke.h:58
USHORT u16
Definition: btrfs.h:13

Referenced by vring_new_virtqueue_split().

◆ virtqueue_detach_unused_buf_split()

static void* virtqueue_detach_unused_buf_split ( struct virtqueue _vq)
static

Definition at line 455 of file VirtIORing.c.

456 {
457  struct virtqueue_split *vq = splitvq(_vq);
458  u16 idx;
459  void *opaque = NULL;
460 
461  for (idx = 0; idx < (u16)vq->vring.num; idx++) {
462  opaque = vq->opaque[idx];
463  if (opaque) {
465  vq->vring.avail->idx = --vq->master_vring_avail.idx;
466  break;
467  }
468  }
469  return opaque;
470 }
void * opaque[]
Definition: VirtIORing.c:188
#define splitvq(vq)
Definition: VirtIORing.c:191
struct virtqueue vq
Definition: VirtIORing.c:178
static void put_unused_desc_chain(struct virtqueue_split *vq, u16 idx)
Definition: VirtIORing.c:207
unsigned int idx
Definition: utils.c:41
smooth NULL
Definition: ftsmooth.c:416
#define u16
Definition: types.h:8
USHORT u16
Definition: btrfs.h:13

Referenced by vring_new_virtqueue_split().

◆ virtqueue_disable_cb_split()

static void virtqueue_disable_cb_split ( struct virtqueue _vq)
static

Definition at line 415 of file VirtIORing.c.

416 {
417  struct virtqueue_split *vq = splitvq(_vq);
419  vq->master_vring_avail.flags |= VIRTQ_AVAIL_F_NO_INTERRUPT;
420  if (!_vq->vdev->event_suppression_enabled)
421  {
422  vq->vring.avail->flags = vq->master_vring_avail.flags;
423  }
424  }
425 }
#define splitvq(vq)
Definition: VirtIORing.c:191
struct virtqueue vq
Definition: VirtIORing.c:178
#define VIRTQ_AVAIL_F_NO_INTERRUPT
Definition: VirtIORing.c:55
static BOOLEAN virtqueue_is_interrupt_enabled(struct virtqueue *vq)
Definition: VirtIO.h:110
bool event_suppression_enabled
Definition: virtio_pci.h:245
VirtIODevice * vdev
Definition: VirtIO.h:44

Referenced by vring_new_virtqueue_split().

◆ virtqueue_enable_cb_delayed_split()

static bool virtqueue_enable_cb_delayed_split ( struct virtqueue _vq)
static

Definition at line 394 of file VirtIORing.c.

395 {
396  struct virtqueue_split *vq = splitvq(_vq);
397  u16 bufs;
398 
399  if (!virtqueue_is_interrupt_enabled(_vq)) {
400  vq->master_vring_avail.flags &= ~VIRTQ_AVAIL_F_NO_INTERRUPT;
401  if (!_vq->vdev->event_suppression_enabled)
402  {
403  vq->vring.avail->flags = vq->master_vring_avail.flags;
404  }
405  }
406 
407  /* Note that 3/4 is an arbitrary threshold */
408  bufs = (u16)(vq->master_vring_avail.idx - vq->last_used) * 3 / 4;
409  vring_used_event(&vq->vring) = vq->last_used + bufs;
410  KeMemoryBarrier();
411  return ((vq->vring.used->idx - vq->last_used) <= bufs);
412 }
const GLenum * bufs
Definition: glext.h:6026
#define splitvq(vq)
Definition: VirtIORing.c:191
struct virtqueue vq
Definition: VirtIORing.c:178
#define VIRTQ_AVAIL_F_NO_INTERRUPT
Definition: VirtIORing.c:55
static BOOLEAN virtqueue_is_interrupt_enabled(struct virtqueue *vq)
Definition: VirtIO.h:110
#define u16
Definition: types.h:8
#define vring_used_event(vr)
Definition: VirtIORing.c:140
bool event_suppression_enabled
Definition: virtio_pci.h:245
VirtIODevice * vdev
Definition: VirtIO.h:44
FORCEINLINE VOID KeMemoryBarrier(VOID)
Definition: ke.h:58
USHORT u16
Definition: btrfs.h:13

Referenced by vring_new_virtqueue_split().

◆ virtqueue_enable_cb_split()

static bool virtqueue_enable_cb_split ( struct virtqueue _vq)
static

Definition at line 376 of file VirtIORing.c.

377 {
378  struct virtqueue_split *vq = splitvq(_vq);
379  if (!virtqueue_is_interrupt_enabled(_vq)) {
380  vq->master_vring_avail.flags &= ~VIRTQ_AVAIL_F_NO_INTERRUPT;
381  if (!_vq->vdev->event_suppression_enabled)
382  {
383  vq->vring.avail->flags = vq->master_vring_avail.flags;
384  }
385  }
386 
387  vring_used_event(&vq->vring) = vq->last_used;
388  KeMemoryBarrier();
389  return (vq->last_used == vq->vring.used->idx);
390 }
#define splitvq(vq)
Definition: VirtIORing.c:191
struct virtqueue vq
Definition: VirtIORing.c:178
#define VIRTQ_AVAIL_F_NO_INTERRUPT
Definition: VirtIORing.c:55
static BOOLEAN virtqueue_is_interrupt_enabled(struct virtqueue *vq)
Definition: VirtIO.h:110
#define vring_used_event(vr)
Definition: VirtIORing.c:140
bool event_suppression_enabled
Definition: virtio_pci.h:245
VirtIODevice * vdev
Definition: VirtIO.h:44
FORCEINLINE VOID KeMemoryBarrier(VOID)
Definition: ke.h:58

Referenced by vring_new_virtqueue_split().

◆ virtqueue_get_buf_split()

static void* virtqueue_get_buf_split ( struct virtqueue _vq,
unsigned int len 
)
static

Definition at line 303 of file VirtIORing.c.

306 {
307  struct virtqueue_split *vq = splitvq(_vq);
308  void *opaque;
309  u16 idx;
310 
311  if (vq->last_used == (int)vq->vring.used->idx) {
312  /* No descriptor index in the used ring */
313  return NULL;
314  }
315  KeMemoryBarrier();
316 
317  idx = DESC_INDEX(vq->vring.num, vq->last_used);
318  *len = vq->vring.used->ring[idx].len;
319 
320  /* Get the first used descriptor */
321  idx = (u16)vq->vring.used->ring[idx].id;
322  opaque = vq->opaque[idx];
323 
324  /* Put all descriptors back to the free list */
326 
327  vq->last_used++;
329  vring_used_event(&vq->vring) = vq->last_used;
330  KeMemoryBarrier();
331  }
332 
333  ASSERT(opaque != NULL);
334  return opaque;
335 }
void * opaque[]
Definition: VirtIORing.c:188
#define splitvq(vq)
Definition: VirtIORing.c:191
#define DESC_INDEX(num, i)
Definition: VirtIORing.c:39
struct virtqueue vq
Definition: VirtIORing.c:178
static void put_unused_desc_chain(struct virtqueue_split *vq, u16 idx)
Definition: VirtIORing.c:207
static BOOLEAN virtqueue_is_interrupt_enabled(struct virtqueue *vq)
Definition: VirtIO.h:110
unsigned int idx
Definition: utils.c:41
smooth NULL
Definition: ftsmooth.c:416
#define u16
Definition: types.h:8
ASSERT((InvokeOnSuccess||InvokeOnError||InvokeOnCancel) ?(CompletionRoutine !=NULL) :TRUE)
GLenum GLsizei len
Definition: glext.h:6722
#define vring_used_event(vr)
Definition: VirtIORing.c:140
bool event_suppression_enabled
Definition: virtio_pci.h:245
VirtIODevice * vdev
Definition: VirtIO.h:44
FORCEINLINE VOID KeMemoryBarrier(VOID)
Definition: ke.h:58
USHORT u16
Definition: btrfs.h:13

Referenced by vring_new_virtqueue_split().

◆ virtqueue_has_buf_split()

static BOOLEAN virtqueue_has_buf_split ( struct virtqueue _vq)
static

Definition at line 338 of file VirtIORing.c.

339 {
340  struct virtqueue_split *vq = splitvq(_vq);
341  return (vq->last_used != vq->vring.used->idx);
342 }
#define splitvq(vq)
Definition: VirtIORing.c:191
struct virtqueue vq
Definition: VirtIORing.c:178

Referenced by vring_new_virtqueue_split().

◆ virtqueue_is_interrupt_enabled_split()

static BOOLEAN virtqueue_is_interrupt_enabled_split ( struct virtqueue _vq)
static

Definition at line 428 of file VirtIORing.c.

429 {
430  struct virtqueue_split *vq = splitvq(_vq);
431  return !(vq->master_vring_avail.flags & VIRTQ_AVAIL_F_NO_INTERRUPT);
432 }
#define splitvq(vq)
Definition: VirtIORing.c:191
struct virtqueue vq
Definition: VirtIORing.c:178
#define VIRTQ_AVAIL_F_NO_INTERRUPT
Definition: VirtIORing.c:55

Referenced by vring_new_virtqueue_split().

◆ virtqueue_kick_always_split()

static void virtqueue_kick_always_split ( struct virtqueue _vq)
static

Definition at line 366 of file VirtIORing.c.

367 {
368  struct virtqueue_split *vq = splitvq(_vq);
369  KeMemoryBarrier();
370  vq->num_added_since_kick = 0;
371  virtqueue_notify(_vq);
372 }
#define splitvq(vq)
Definition: VirtIORing.c:191
struct virtqueue vq
Definition: VirtIORing.c:178
void virtqueue_notify(struct virtqueue *vq)
FORCEINLINE VOID KeMemoryBarrier(VOID)
Definition: ke.h:58

Referenced by vring_new_virtqueue_split().

◆ virtqueue_kick_prepare_split()

static bool virtqueue_kick_prepare_split ( struct virtqueue _vq)
static

Definition at line 345 of file VirtIORing.c.

346 {
347  struct virtqueue_split *vq = splitvq(_vq);
348  bool wrap_around;
349  u16 old, new;
350  KeMemoryBarrier();
351 
352  wrap_around = (vq->num_added_since_kick >= (1 << 16));
353 
354  old = (u16)(vq->master_vring_avail.idx - vq->num_added_since_kick);
355  new = vq->master_vring_avail.idx;
356  vq->num_added_since_kick = 0;
357 
358  if (_vq->vdev->event_suppression_enabled) {
359  return wrap_around || (bool)vring_need_event(vring_avail_event(&vq->vring), new, old);
360  } else {
361  return !(vq->vring.used->flags & VIRTQ_USED_F_NO_NOTIFY);
362  }
363 }
#define splitvq(vq)
Definition: VirtIORing.c:191
struct virtqueue vq
Definition: VirtIORing.c:178
static int vring_need_event(__u16 event_idx, __u16 new_idx, __u16 old)
Definition: VirtIORing.c:167
#define VIRTQ_USED_F_NO_NOTIFY
Definition: VirtIORing.c:51
typedef bool(CARDLIBPROC *pCanDragProc)(CardRegion &stackobj
#define u16
Definition: types.h:8
#define vring_avail_event(vr)
Definition: VirtIORing.c:141
bool event_suppression_enabled
Definition: virtio_pci.h:245
VirtIODevice * vdev
Definition: VirtIO.h:44
FORCEINLINE VOID KeMemoryBarrier(VOID)
Definition: ke.h:58
USHORT u16
Definition: btrfs.h:13

Referenced by vring_new_virtqueue_split().

◆ virtqueue_shutdown_split()

static void virtqueue_shutdown_split ( struct virtqueue _vq)
static

Definition at line 435 of file VirtIORing.c.

436 {
437  struct virtqueue_split *vq = splitvq(_vq);
438  unsigned int num = vq->vring.num;
439  void *pages = vq->vring.desc;
440  unsigned int vring_align = _vq->vdev->addr ? PAGE_SIZE : SMP_CACHE_BYTES;
441 
442  RtlZeroMemory(pages, vring_size_split(num, vring_align));
444  _vq->index,
445  vq->vring.num,
446  vring_align,
447  _vq->vdev,
448  pages,
449  _vq->notification_cb,
450  vq);
451 }
static unsigned vring_size_split(unsigned int num, unsigned long align)
Definition: VirtIORing.c:153
struct png_info_def **typedef void(__cdecl typeof(png_destroy_read_struct))(struct png_struct_def **
Definition: typeof.h:49
void(* notification_cb)(struct virtqueue *vq)
Definition: VirtIO.h:46
#define SMP_CACHE_BYTES
Definition: osdep.h:39
#define splitvq(vq)
Definition: VirtIORing.c:191
unsigned int index
Definition: VirtIO.h:45
struct virtqueue vq
Definition: VirtIORing.c:178
GLuint GLuint num
Definition: glext.h:9618
#define PAGE_SIZE
Definition: env_spec_w32.h:49
VirtIODevice * vdev
Definition: VirtIO.h:44
#define RtlZeroMemory(Destination, Length)
Definition: typedefs.h:262
ULONG_PTR addr
Definition: virtio_pci.h:239
struct virtqueue * vring_new_virtqueue_split(unsigned int index, unsigned int num, unsigned int vring_align, VirtIODevice *vdev, void *pages, void(*notify)(struct virtqueue *), void *control)
Definition: VirtIORing.c:486

Referenced by vring_new_virtqueue_split().

◆ vring_control_block_size()

unsigned int vring_control_block_size ( u16  qsize,
bool  packed 
)

Definition at line 474 of file VirtIORing.c.

475 {
476  unsigned int res;
477  if (packed) {
478  return vring_control_block_size_packed(qsize);
479  }
480  res = sizeof(struct virtqueue_split);
481  res += sizeof(void *) * qsize;
482  return res;
483 }
GLuint GLenum GLsizei GLsizei GLint GLint GLboolean packed
Definition: glext.h:9271
GLuint res
Definition: glext.h:9613
unsigned int vring_control_block_size_packed(u16 qsize)

Referenced by vio_legacy_query_vq_alloc(), and vio_modern_query_vq_alloc().

◆ vring_init()

static void vring_init ( struct vring vr,
unsigned int  num,
void p,
unsigned long  align 
)
inlinestatic

Definition at line 143 of file VirtIORing.c.

145 {
146  vr->num = num;
147  vr->desc = (struct vring_desc *)p;
148  vr->avail = (struct vring_avail *)((__u8 *)p + num * sizeof(struct vring_desc));
149  vr->used = (struct vring_used *)(((ULONG_PTR)&vr->avail->ring[num] + sizeof(__virtio16)
150  + align - 1) & ~((ULONG_PTR)align - 1));
151 }
unsigned char __u8
Definition: compat.h:88
int align(int length, int align)
Definition: dsound8.c:36
uint32_t ULONG_PTR
Definition: typedefs.h:65
VRET vr
Definition: symbols.c:94
__u16 __bitwise__ __virtio16
Definition: virtio_types.h:43
GLuint GLuint num
Definition: glext.h:9618
#define ULONG_PTR
Definition: config.h:101
GLfloat GLfloat p
Definition: glext.h:8902

Referenced by vring_new_virtqueue_split().

◆ vring_need_event()

static int vring_need_event ( __u16  event_idx,
__u16  new_idx,
__u16  old 
)
inlinestatic

Definition at line 167 of file VirtIORing.c.

168 {
169  /* Note: Xen has similar logic for notification hold-off
170  * in include/xen/interface/io/ring.h with req_event and req_prod
171  * corresponding to event_idx + 1 and new_idx respectively.
172  * Note also that req_event and req_prod in Xen start at 1,
173  * event indexes in virtio start at 0. */
174  return (__u16)(new_idx - event_idx - 1) < (__u16)(new_idx - old);
175 }
unsigned short __u16
Definition: compat.h:89

Referenced by virtqueue_kick_prepare_split().

◆ vring_new_virtqueue_split()

struct virtqueue* vring_new_virtqueue_split ( unsigned int  index,
unsigned int  num,
unsigned int  vring_align,
VirtIODevice vdev,
void pages,
void(*)(struct virtqueue *)  notify,
void control 
)

Definition at line 486 of file VirtIORing.c.

494 {
495  struct virtqueue_split *vq = splitvq(control);
496  u16 i;
497 
498  if (DESC_INDEX(num, num) != 0) {
499  DPrintf(0, "Virtqueue length %u is not a power of 2\n", num);
500  return NULL;
501  }
502 
503  RtlZeroMemory(vq, sizeof(*vq) + num * sizeof(void *));
504 
505  vring_init(&vq->vring, num, pages, vring_align);
506  vq->vq.vdev = vdev;
507  vq->vq.notification_cb = notify;
508  vq->vq.index = index;
509 
510  /* Build a linked list of unused descriptors */
511  vq->num_unused = num;
512  vq->first_unused = 0;
513  for (i = 0; i < num - 1; i++) {
514  vq->vring.desc[i].flags = VIRTQ_DESC_F_NEXT;
515  vq->vring.desc[i].next = i + 1;
516  }
517  vq->vq.avail_va = vq->vring.avail;
518  vq->vq.used_va = vq->vring.used;
519  vq->vq.add_buf = virtqueue_add_buf_split;
520  vq->vq.detach_unused_buf = virtqueue_detach_unused_buf_split;
521  vq->vq.disable_cb = virtqueue_disable_cb_split;
522  vq->vq.enable_cb = virtqueue_enable_cb_split;
523  vq->vq.enable_cb_delayed = virtqueue_enable_cb_delayed_split;
524  vq->vq.get_buf = virtqueue_get_buf_split;
525  vq->vq.has_buf = virtqueue_has_buf_split;
526  vq->vq.is_interrupt_enabled = virtqueue_is_interrupt_enabled_split;
527  vq->vq.kick_always = virtqueue_kick_always_split;
528  vq->vq.kick_prepare = virtqueue_kick_prepare_split;
529  vq->vq.shutdown = virtqueue_shutdown_split;
530  return &vq->vq;
531 }
static BOOLEAN virtqueue_is_interrupt_enabled_split(struct virtqueue *_vq)
Definition: VirtIORing.c:428
static void vring_init(struct vring *vr, unsigned int num, void *p, unsigned long align)
Definition: VirtIORing.c:143
int notify
Definition: msacm.c:1365
static bool virtqueue_kick_prepare_split(struct virtqueue *_vq)
Definition: VirtIORing.c:345
#define splitvq(vq)
Definition: VirtIORing.c:191
static void * virtqueue_get_buf_split(struct virtqueue *_vq, unsigned int *len)
Definition: VirtIORing.c:303
#define DESC_INDEX(num, i)
Definition: VirtIORing.c:39
static void virtqueue_kick_always_split(struct virtqueue *_vq)
Definition: VirtIORing.c:366
struct virtqueue vq
Definition: VirtIORing.c:178
GLsizei GLenum const GLvoid GLsizei GLenum GLbyte GLbyte GLbyte GLdouble GLdouble GLdouble GLfloat GLfloat GLfloat GLint GLint GLint GLshort GLshort GLshort GLubyte GLubyte GLubyte GLuint GLuint GLuint GLushort GLushort GLushort GLbyte GLbyte GLbyte GLbyte GLdouble GLdouble GLdouble GLdouble GLfloat GLfloat GLfloat GLfloat GLint GLint GLint GLint GLshort GLshort GLshort GLshort GLubyte GLubyte GLubyte GLubyte GLuint GLuint GLuint GLuint GLushort GLushort GLushort GLushort GLboolean const GLdouble const GLfloat const GLint const GLshort const GLbyte const GLdouble const GLfloat const GLint const GLshort const GLdouble const GLfloat const GLint const GLshort const GLdouble const GLfloat const GLint const GLshort const GLdouble const GLfloat const GLint const GLshort const GLdouble const GLdouble const GLfloat const GLfloat const GLint const GLint const GLshort const GLshort const GLdouble const GLfloat const GLint const GLshort const GLdouble const GLfloat const GLint const GLshort const GLdouble const GLfloat const GLint const GLshort const GLdouble const GLfloat const GLint const GLshort const GLdouble const GLfloat const GLint const GLshort const GLdouble const GLfloat const GLint const GLshort const GLdouble const GLfloat const GLint const GLshort GLenum GLenum GLenum GLfloat GLenum GLint GLenum GLenum GLenum GLfloat GLenum GLenum GLint GLenum GLfloat GLenum GLint GLint GLushort GLenum GLenum GLfloat GLenum GLenum GLint GLfloat const GLubyte GLenum GLenum GLenum const GLfloat GLenum GLenum const GLint GLenum GLint GLint GLsizei GLsizei GLint GLenum GLenum const GLvoid GLenum GLenum const GLfloat GLenum GLenum const GLint GLenum GLenum const GLdouble GLenum GLenum const GLfloat GLenum GLenum const GLint GLsizei GLuint GLfloat GLuint GLbitfield GLfloat GLint GLuint GLboolean GLenum GLfloat GLenum GLbitfield GLenum GLfloat GLfloat GLint GLint const GLfloat GLenum GLfloat GLfloat GLint GLint GLfloat GLfloat GLint GLint const GLfloat GLint GLfloat GLfloat GLint GLfloat GLfloat GLint GLfloat GLfloat const GLdouble const GLfloat const GLdouble const GLfloat GLint i
Definition: glfuncs.h:248
#define DPrintf(Level, Fmt)
Definition: kdebugprint.h:61
static void virtqueue_disable_cb_split(struct virtqueue *_vq)
Definition: VirtIORing.c:415
smooth NULL
Definition: ftsmooth.c:416
static void virtqueue_shutdown_split(struct virtqueue *_vq)
Definition: VirtIORing.c:435
GLuint GLuint num
Definition: glext.h:9618
#define index(s, c)
Definition: various.h:29
static BOOLEAN virtqueue_has_buf_split(struct virtqueue *_vq)
Definition: VirtIORing.c:338
static bool virtqueue_enable_cb_delayed_split(struct virtqueue *_vq)
Definition: VirtIORing.c:394
#define VIRTQ_DESC_F_NEXT
Definition: VirtIORing.c:42
static void * virtqueue_detach_unused_buf_split(struct virtqueue *_vq)
Definition: VirtIORing.c:455
#define RtlZeroMemory(Destination, Length)
Definition: typedefs.h:262
static bool virtqueue_enable_cb_split(struct virtqueue *_vq)
Definition: VirtIORing.c:376
static int virtqueue_add_buf_split(struct virtqueue *_vq, struct scatterlist sg[], unsigned int out, unsigned int in, void *opaque, void *va_indirect, ULONGLONG phys_indirect)
Definition: VirtIORing.c:225
USHORT u16
Definition: btrfs.h:13

Referenced by vio_legacy_setup_vq(), vio_modern_setup_vq(), and virtqueue_shutdown_split().

◆ vring_size()

unsigned long vring_size ( unsigned int  num,
unsigned long  align,
bool  packed 
)

Definition at line 555 of file VirtIORing.c.

556 {
557  if (packed) {
558  return vring_size_packed(num, align);
559  } else {
560  return vring_size_split(num, align);
561  }
562 }
static unsigned vring_size_split(unsigned int num, unsigned long align)
Definition: VirtIORing.c:153
unsigned long vring_size_packed(unsigned int num, unsigned long align)
GLuint GLenum GLsizei GLsizei GLint GLint GLboolean packed
Definition: glext.h:9271
int align(int length, int align)
Definition: dsound8.c:36
GLuint GLuint num
Definition: glext.h:9618

Referenced by vio_legacy_query_vq_alloc(), vio_legacy_setup_vq(), and vring_pci_size().

◆ vring_size_split()

static unsigned vring_size_split ( unsigned int  num,
unsigned long  align 
)
inlinestatic

Definition at line 153 of file VirtIORing.c.

154 {
155 #pragma warning (push)
156 #pragma warning (disable:4319)
157  return ((sizeof(struct vring_desc) * num + sizeof(__virtio16) * (3 + num)
158  + align - 1) & ~(align - 1))
159  + sizeof(__virtio16) * 3 + sizeof(struct vring_used_elem) * num;
160 #pragma warning(pop)
161 }
int align(int length, int align)
Definition: dsound8.c:36
__u16 __bitwise__ __virtio16
Definition: virtio_types.h:43
GLuint GLuint num
Definition: glext.h:9618

Referenced by virtqueue_shutdown_split(), and vring_size().

◆ vring_transport_features()

void vring_transport_features ( VirtIODevice vdev,
u64 features 
)

Definition at line 534 of file VirtIORing.c.

537 {
538  unsigned int i;
539 
543  i != VIRTIO_F_VERSION_1) {
544  virtio_feature_disable(*features, i);
545  }
546  }
547 }
#define virtio_feature_disable(FeaturesList, Feature)
Definition: virtio_pci.h:313
#define VIRTIO_TRANSPORT_F_END
Definition: virtio_config.h:53
GLsizei GLenum const GLvoid GLsizei GLenum GLbyte GLbyte GLbyte GLdouble GLdouble GLdouble GLfloat GLfloat GLfloat GLint GLint GLint GLshort GLshort GLshort GLubyte GLubyte GLubyte GLuint GLuint GLuint GLushort GLushort GLushort GLbyte GLbyte GLbyte GLbyte GLdouble GLdouble GLdouble GLdouble GLfloat GLfloat GLfloat GLfloat GLint GLint GLint GLint GLshort GLshort GLshort GLshort GLubyte GLubyte GLubyte GLubyte GLuint GLuint GLuint GLuint GLushort GLushort GLushort GLushort GLboolean const GLdouble const GLfloat const GLint const GLshort const GLbyte const GLdouble const GLfloat const GLint const GLshort const GLdouble const GLfloat const GLint const GLshort const GLdouble const GLfloat const GLint const GLshort const GLdouble const GLfloat const GLint const GLshort const GLdouble const GLdouble const GLfloat const GLfloat const GLint const GLint const GLshort const GLshort const GLdouble const GLfloat const GLint const GLshort const GLdouble const GLfloat const GLint const GLshort const GLdouble const GLfloat const GLint const GLshort const GLdouble const GLfloat const GLint const GLshort const GLdouble const GLfloat const GLint const GLshort const GLdouble const GLfloat const GLint const GLshort const GLdouble const GLfloat const GLint const GLshort GLenum GLenum GLenum GLfloat GLenum GLint GLenum GLenum GLenum GLfloat GLenum GLenum GLint GLenum GLfloat GLenum GLint GLint GLushort GLenum GLenum GLfloat GLenum GLenum GLint GLfloat const GLubyte GLenum GLenum GLenum const GLfloat GLenum GLenum const GLint GLenum GLint GLint GLsizei GLsizei GLint GLenum GLenum const GLvoid GLenum GLenum const GLfloat GLenum GLenum const GLint GLenum GLenum const GLdouble GLenum GLenum const GLfloat GLenum GLenum const GLint GLsizei GLuint GLfloat GLuint GLbitfield GLfloat GLint GLuint GLboolean GLenum GLfloat GLenum GLbitfield GLenum GLfloat GLfloat GLint GLint const GLfloat GLenum GLfloat GLfloat GLint GLint GLfloat GLfloat GLint GLint const GLfloat GLint GLfloat GLfloat GLint GLfloat GLfloat GLint GLfloat GLfloat const GLdouble const GLfloat const GLdouble const GLfloat GLint i
Definition: glfuncs.h:248
#define VIRTIO_RING_F_EVENT_IDX
Definition: virtio_ring.h:45
#define VIRTIO_TRANSPORT_F_START
Definition: virtio_config.h:52
#define VIRTIO_RING_F_INDIRECT_DESC
Definition: virtio_ring.h:39
#define VIRTIO_F_VERSION_1
Definition: virtio_config.h:63

Referenced by vio_legacy_set_features(), and vio_modern_set_features().