FD.io VPP  v19.08.3-2-gbabecb413
Vector Packet Processing
pci.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2018 Cisco and/or its affiliates.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at:
6  *
7  * http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 
16 #include <fcntl.h>
17 #include <sys/ioctl.h>
18 
19 #include <vppinfra/types.h>
20 #include <vlib/vlib.h>
21 #include <vlib/pci/pci.h>
22 #include <vnet/ethernet/ethernet.h>
23 #include <vnet/ip/ip4_packet.h>
24 #include <vnet/ip/ip6_packet.h>
27 
28 #define PCI_VENDOR_ID_VIRTIO 0x1af4
29 #define PCI_DEVICE_ID_VIRTIO_NIC 0x1000
30 /* Doesn't support modern device */
31 #define PCI_DEVICE_ID_VIRTIO_NIC_MODERN 0x1041
32 
33 #define PCI_CAPABILITY_LIST 0x34
34 #define PCI_CAP_ID_VNDR 0x09
35 #define PCI_CAP_ID_MSIX 0x11
36 
37 #define PCI_MSIX_ENABLE 0x8000
38 
39 #define PCI_CONFIG_SIZE(vif) ((vif->msix_enabled == VIRTIO_MSIX_ENABLED) ? \
40  24 : 20)
41 
42 static pci_device_id_t virtio_pci_device_ids[] = {
43  {
45  .device_id = PCI_DEVICE_ID_VIRTIO_NIC},
46  {
47  .vendor_id = PCI_VENDOR_ID_VIRTIO,
48  .device_id = PCI_DEVICE_ID_VIRTIO_NIC_MODERN},
49  {0},
50 };
51 
52 static void
54  int len, u32 addr)
55 {
56  u32 size = 0;
58 
59  while (len > 0)
60  {
61  if (len >= 4)
62  {
63  size = 4;
64  vlib_pci_read_io_u32 (vm, h, PCI_CONFIG_SIZE (vif) + addr, dst);
65  }
66  else if (len >= 2)
67  {
68  size = 2;
69  vlib_pci_read_io_u16 (vm, h, PCI_CONFIG_SIZE (vif) + addr, dst);
70  }
71  else
72  {
73  size = 1;
74  vlib_pci_read_io_u8 (vm, h, PCI_CONFIG_SIZE (vif) + addr, dst);
75  }
76  dst = (u8 *) dst + size;
77  addr += size;
78  len -= size;
79  }
80 }
81 
82 static void
84  void *src, int len, u32 addr)
85 {
86  u32 size = 0;
88 
89  while (len > 0)
90  {
91  if (len >= 4)
92  {
93  size = 4;
94  vlib_pci_write_io_u32 (vm, h, PCI_CONFIG_SIZE (vif) + addr, src);
95  }
96  else if (len >= 2)
97  {
98  size = 2;
99  vlib_pci_write_io_u16 (vm, h, PCI_CONFIG_SIZE (vif) + addr, src);
100  }
101  else
102  {
103  size = 1;
104  vlib_pci_write_io_u8 (vm, h, PCI_CONFIG_SIZE (vif) + addr, src);
105  }
106  src = (u8 *) src + size;
107  addr += size;
108  len -= size;
109  }
110 }
111 
112 static u64
114 {
115  u32 features;
116  vlib_pci_read_io_u32 (vm, vif->pci_dev_handle, VIRTIO_PCI_HOST_FEATURES,
117  &features);
118  return features;
119 }
120 
121 static u32
123  u64 features)
124 {
125  if ((features >> 32) != 0)
126  {
127  clib_warning ("only 32 bit features are allowed for legacy virtio!");
128  }
129  u32 feature = 0, guest_features = (u32) features;
130  vlib_pci_write_io_u32 (vm, vif->pci_dev_handle, VIRTIO_PCI_GUEST_FEATURES,
131  &guest_features);
132  vlib_pci_read_io_u32 (vm, vif->pci_dev_handle, VIRTIO_PCI_GUEST_FEATURES,
133  &feature);
134  return feature;
135 }
136 
137 static u8
139 {
140  u8 status = 0;
141  vlib_pci_read_io_u8 (vm, vif->pci_dev_handle, VIRTIO_PCI_STATUS, &status);
142  return status;
143 }
144 
145 static void
147 {
148  if (status != VIRTIO_CONFIG_STATUS_RESET)
149  status |= virtio_pci_legacy_get_status (vm, vif);
150  vlib_pci_write_io_u8 (vm, vif->pci_dev_handle, VIRTIO_PCI_STATUS, &status);
151 }
152 
153 static u8
155 {
156  virtio_pci_legacy_set_status (vm, vif, VIRTIO_CONFIG_STATUS_RESET);
157  return virtio_pci_legacy_get_status (vm, vif);
158 }
159 
160 static u8
162 {
163  u8 isr = 0;
164  vlib_pci_read_io_u8 (vm, vif->pci_dev_handle, VIRTIO_PCI_ISR, &isr);
165  return isr;
166 }
167 
168 static u16
170  u16 queue_id)
171 {
172  u16 queue_num = 0;
173  vlib_pci_write_io_u16 (vm, vif->pci_dev_handle, VIRTIO_PCI_QUEUE_SEL,
174  &queue_id);
175  vlib_pci_read_io_u16 (vm, vif->pci_dev_handle, VIRTIO_PCI_QUEUE_NUM,
176  &queue_num);
177  return queue_num;
178 }
179 
180 static int
182  u16 queue_id, void *p)
183 {
185  u32 addr2 = 0;
186  vlib_pci_write_io_u16 (vm, vif->pci_dev_handle, VIRTIO_PCI_QUEUE_SEL,
187  &queue_id);
188  vlib_pci_write_io_u32 (vm, vif->pci_dev_handle, VIRTIO_PCI_QUEUE_PFN,
189  (u32 *) & addr);
190  vlib_pci_read_io_u32 (vm, vif->pci_dev_handle, VIRTIO_PCI_QUEUE_PFN,
191  &addr2);
192  if ((u32) addr == addr2)
193  return 0;
194  return 1;
195 }
196 
197 static void
199  u16 queue_id)
200 {
201  u32 src = 0;
202  vlib_pci_write_io_u16 (vm, vif->pci_dev_handle, VIRTIO_PCI_QUEUE_SEL,
203  &queue_id);
204  vlib_pci_write_io_u32 (vm, vif->pci_dev_handle, VIRTIO_PCI_QUEUE_PFN, &src);
205 }
206 
207 inline void
209  u16 queue_id)
210 {
211  vlib_pci_write_io_u16 (vm, vif->pci_dev_handle, VIRTIO_PCI_QUEUE_NOTIFY,
212  &queue_id);
213 }
214 
215 /* Enable one vector (0) for Link State Intrerrupt */
216 static u16
218  u16 vec)
219 {
220  vlib_pci_write_io_u16 (vm, vif->pci_dev_handle, VIRTIO_MSI_CONFIG_VECTOR,
221  &vec);
222  vlib_pci_read_io_u16 (vm, vif->pci_dev_handle, VIRTIO_MSI_CONFIG_VECTOR,
223  &vec);
224  return vec;
225 }
226 
227 static u16
229  u16 queue_id)
230 {
231  vlib_pci_write_io_u16 (vm, vif->pci_dev_handle, VIRTIO_PCI_QUEUE_SEL,
232  &queue_id);
233  vlib_pci_write_io_u16 (vm, vif->pci_dev_handle, VIRTIO_MSI_QUEUE_VECTOR,
234  &vec);
235  vlib_pci_read_io_u16 (vm, vif->pci_dev_handle, VIRTIO_MSI_QUEUE_VECTOR,
236  &vec);
237  return vec;
238 }
239 
240 static u32
242  u32 flags)
243 {
244  return 0;
245 }
246 
247 static clib_error_t *
249 {
250  virtio_main_t *vim = &virtio_main;
251  virtio_net_config_t config;
252  clib_error_t *error = 0;
253  u16 max_queue_pairs = 1;
254 
255  if (vif->features & VIRTIO_FEATURE (VIRTIO_NET_F_MQ))
256  {
258  sizeof (config.max_virtqueue_pairs),
260  max_virtqueue_pairs));
261  max_queue_pairs = config.max_virtqueue_pairs;
262  }
263 
264  virtio_log_debug (vim, vif, "max queue pair is %x", max_queue_pairs);
265  if (max_queue_pairs < 1 || max_queue_pairs > 0x8000)
266  return clib_error_return (error, "max queue pair is %x", max_queue_pairs);
267 
268  vif->max_queue_pairs = max_queue_pairs;
269  return error;
270 }
271 
272 static void
274 {
276  sizeof (vif->mac_addr), 0);
277 }
278 
279 static u32
281 {
282  if (vif->features & VIRTIO_FEATURE (VIRTIO_NET_F_MAC))
283  {
285  sizeof (vif->mac_addr), 0);
286  return 0;
287  }
288  return 1;
289 }
290 
291 static u16
293 {
294  /*
295  * Minimal driver: assumes link is up
296  */
297  u16 status = 1;
298  if (vif->features & VIRTIO_FEATURE (VIRTIO_NET_F_STATUS))
299  virtio_pci_legacy_read_config (vm, vif, &status, sizeof (status), /* mac */
301  status));
302  return status;
303 }
304 
305 static void
307 {
308  vnet_main_t *vnm = vnet_get_main ();
309  virtio_main_t *vim = &virtio_main;
310  uword pd = vlib_pci_get_private_data (vm, h);
311  virtio_if_t *vif = pool_elt_at_index (vim->interfaces, pd);
312  u16 qid = line;
313 
315 }
316 
317 static void
319 {
320  vnet_main_t *vnm = vnet_get_main ();
321  virtio_main_t *vim = &virtio_main;
322  uword pd = vlib_pci_get_private_data (vm, h);
323  virtio_if_t *vif = pool_elt_at_index (vim->interfaces, pd);
324 
325  if (virtio_pci_is_link_up (vm, vif) & VIRTIO_NET_S_LINK_UP)
326  {
327  vif->flags |= VIRTIO_IF_FLAG_ADMIN_UP;
330  }
331  else
332  {
333  vif->flags &= ~VIRTIO_IF_FLAG_ADMIN_UP;
335  }
336 }
337 
338 static void
340 {
341  virtio_main_t *vim = &virtio_main;
342  uword pd = vlib_pci_get_private_data (vm, h);
343  virtio_if_t *vif = pool_elt_at_index (vim->interfaces, pd);
344  u8 isr = 0;
345  u16 line = 0;
346 
347  isr = virtio_pci_legacy_get_isr (vm, vif);
348 
349  /*
350  * If the lower bit is set: look through the used rings of
351  * all virtqueues for the device, to see if any progress has
352  * been made by the device which requires servicing.
353  */
354  if (isr & VIRTIO_PCI_ISR_INTR)
355  virtio_pci_irq_0_handler (vm, h, line);
356 
357  if (isr & VIRTIO_PCI_ISR_CONFIG)
358  virtio_pci_irq_1_handler (vm, h, line);
359 }
360 
361 inline void
363 {
364  struct status_struct
365  {
366  u8 bit;
367  char *str;
368  };
369  struct status_struct *status_entry;
370  static struct status_struct status_array[] = {
371 #define _(s,b) { .str = #s, .bit = b, },
373 #undef _
374  {.str = NULL}
375  };
376 
377  vlib_cli_output (vm, " status 0x%x", vif->status);
378 
379  status_entry = (struct status_struct *) &status_array;
380  while (status_entry->str)
381  {
382  if (vif->status & status_entry->bit)
383  vlib_cli_output (vm, " %s (%x)", status_entry->str,
384  status_entry->bit);
385  status_entry++;
386  }
387 }
388 
389 inline void
391 {
392  u32 data_u32;
393  u16 data_u16;
394  u8 data_u8;
395  vlib_pci_read_io_u32 (vm, vif->pci_dev_handle, VIRTIO_PCI_HOST_FEATURES,
396  &data_u32);
397  vlib_cli_output (vm, "remote features 0x%lx", data_u32);
398  vlib_pci_read_io_u32 (vm, vif->pci_dev_handle, VIRTIO_PCI_GUEST_FEATURES,
399  &data_u32);
400  vlib_cli_output (vm, "guest features 0x%lx", data_u32);
401  vlib_pci_read_io_u32 (vm, vif->pci_dev_handle, VIRTIO_PCI_QUEUE_PFN,
402  &data_u32);
403  vlib_cli_output (vm, "queue address 0x%lx", data_u32);
404  vlib_pci_read_io_u16 (vm, vif->pci_dev_handle, VIRTIO_PCI_QUEUE_NUM,
405  &data_u16);
406  vlib_cli_output (vm, "queue size 0x%x", data_u16);
407  vlib_pci_read_io_u16 (vm, vif->pci_dev_handle, VIRTIO_PCI_QUEUE_SEL,
408  &data_u16);
409  vlib_cli_output (vm, "queue select 0x%x", data_u16);
410  vlib_pci_read_io_u16 (vm, vif->pci_dev_handle, VIRTIO_PCI_QUEUE_NOTIFY,
411  &data_u16);
412  vlib_cli_output (vm, "queue notify 0x%x", data_u16);
413  vlib_pci_read_io_u8 (vm, vif->pci_dev_handle, VIRTIO_PCI_STATUS, &data_u8);
414  vlib_cli_output (vm, "status 0x%x", data_u8);
415  vlib_pci_read_io_u8 (vm, vif->pci_dev_handle, VIRTIO_PCI_ISR, &data_u8);
416  vlib_cli_output (vm, "isr 0x%x", data_u8);
417 
418  if (vif->msix_enabled == VIRTIO_MSIX_ENABLED)
419  {
420  vlib_pci_read_io_u16 (vm, vif->pci_dev_handle, VIRTIO_MSI_CONFIG_VECTOR,
421  &data_u16);
422  vlib_cli_output (vm, "config vector 0x%x", data_u16);
423  u16 queue_id = 0;
424  vlib_pci_write_io_u16 (vm, vif->pci_dev_handle, VIRTIO_PCI_QUEUE_SEL,
425  &queue_id);
426  vlib_pci_read_io_u16 (vm, vif->pci_dev_handle, VIRTIO_MSI_QUEUE_VECTOR,
427  &data_u16);
428  vlib_cli_output (vm, "queue vector for queue (0) 0x%x", data_u16);
429  }
430 
431  u8 mac[6];
432  virtio_pci_legacy_read_config (vm, vif, mac, sizeof (mac), 0);
433  vlib_cli_output (vm, "mac %U", format_ethernet_address, mac);
434  virtio_pci_legacy_read_config (vm, vif, &data_u16, sizeof (u16), /* offset to status */
435  6);
436  vlib_cli_output (vm, "link up/down status 0x%x", data_u16);
437  virtio_pci_legacy_read_config (vm, vif, &data_u16, sizeof (u16),
438  /* offset to max_virtqueue */ 8);
439  vlib_cli_output (vm, "num of virtqueue 0x%x", data_u16);
440  virtio_pci_legacy_read_config (vm, vif, &data_u16, sizeof (u16), /* offset to mtu */
441  10);
442  vlib_cli_output (vm, "mtu 0x%x", data_u16);
443 
444  u32 i = PCI_CONFIG_SIZE (vif) + 12, a = 4;
445  i += a;
446  i &= ~a;
447  for (; i < 64; i += 4)
448  {
449  u32 data = 0;
450  vlib_pci_read_io_u32 (vm, vif->pci_dev_handle, i, &data);
451  vlib_cli_output (vm, "0x%lx", data);
452  }
453 }
454 
456 {
457  struct virtio_net_ctrl_hdr ctrl;
458  virtio_net_ctrl_ack status;
459  u8 data[1024];
460 };
461 
462 static int
464  struct virtio_ctrl_msg *data, u32 len)
465 {
466  virtio_main_t *vim = &virtio_main;
467  virtio_vring_t *vring = vif->cxq_vring;
468  virtio_net_ctrl_ack status = VIRTIO_NET_ERR;
469  struct virtio_ctrl_msg result;
470  u32 buffer_index;
471  vlib_buffer_t *b;
472  u16 used, next, avail;
473  u16 sz = vring->size;
474  u16 mask = sz - 1;
475 
476  used = vring->desc_in_use;
477  next = vring->desc_next;
478  avail = vring->avail->idx;
479  struct vring_desc *d = &vring->desc[next];
480 
481  if (vlib_buffer_alloc (vm, &buffer_index, 1))
482  b = vlib_get_buffer (vm, buffer_index);
483  else
484  return VIRTIO_NET_ERR;
485  /*
486  * current_data may not be initialized with 0 and may contain
487  * previous offset.
488  */
489  b->current_data = 0;
491  sizeof (struct virtio_ctrl_msg));
492  d->flags = VRING_DESC_F_NEXT;
493  d->addr = vlib_buffer_get_current_pa (vm, b);
494  d->len = sizeof (struct virtio_net_ctrl_hdr);
495  vring->avail->ring[avail & mask] = next;
496  avail++;
497  next = (next + 1) & mask;
498  d->next = next;
499  used++;
500 
501  d = &vring->desc[next];
502  d->flags = VRING_DESC_F_NEXT;
503  d->addr = vlib_buffer_get_current_pa (vm, b) +
504  STRUCT_OFFSET_OF (struct virtio_ctrl_msg, data);
505  d->len = len;
506  next = (next + 1) & mask;
507  d->next = next;
508  used++;
509 
510  d = &vring->desc[next];
511  d->flags = VRING_DESC_F_WRITE;
512  d->addr = vlib_buffer_get_current_pa (vm, b) +
513  STRUCT_OFFSET_OF (struct virtio_ctrl_msg, status);
514  d->len = sizeof (data->status);
515  next = (next + 1) & mask;
516  used++;
517 
519  vring->avail->idx = avail;
520  vring->desc_next = next;
521  vring->desc_in_use = used;
522 
523  if ((vring->used->flags & VIRTIO_RING_FLAG_MASK_INT) == 0)
524  {
525  virtio_kick (vm, vring, vif);
526  }
527 
528  u16 last = vring->last_used_idx, n_left = 0;
529  n_left = vring->used->idx - last;
530 
531  while (n_left)
532  {
533  struct vring_used_elem *e = &vring->used->ring[last & mask];
534  u16 slot = e->id;
535 
536  d = &vring->desc[slot];
537  while (d->flags & VRING_DESC_F_NEXT)
538  {
539  used--;
540  slot = d->next;
541  d = &vring->desc[slot];
542  }
543  used--;
544  last++;
545  n_left--;
546  }
547  vring->desc_in_use = used;
548  vring->last_used_idx = last;
549 
551  clib_memcpy (&result, vlib_buffer_get_current (b),
552  sizeof (struct virtio_ctrl_msg));
553  virtio_log_debug (vim, vif, "ctrl-queue: status %u", result.status);
554  status = result.status;
555  vlib_buffer_free (vm, &buffer_index, 1);
556  return status;
557 }
558 
559 static int
561 {
562  virtio_main_t *vim = &virtio_main;
563  struct virtio_ctrl_msg gso_hdr;
564  virtio_net_ctrl_ack status = VIRTIO_NET_ERR;
565 
566  gso_hdr.ctrl.class = VIRTIO_NET_CTRL_GUEST_OFFLOADS;
568  gso_hdr.status = VIRTIO_NET_ERR;
569  u64 offloads = VIRTIO_FEATURE (VIRTIO_NET_F_GUEST_CSUM)
570  | VIRTIO_FEATURE (VIRTIO_NET_F_GUEST_TSO4)
571  | VIRTIO_FEATURE (VIRTIO_NET_F_GUEST_TSO6)
572  | VIRTIO_FEATURE (VIRTIO_NET_F_GUEST_UFO);
573  clib_memcpy (gso_hdr.data, &offloads, sizeof (offloads));
574 
575  status = virtio_pci_send_ctrl_msg (vm, vif, &gso_hdr, sizeof (offloads));
576  virtio_log_debug (vim, vif, "enable gso");
577  return status;
578 }
579 
580 static int
582  u16 num_queues)
583 {
584  virtio_main_t *vim = &virtio_main;
585  struct virtio_ctrl_msg mq_hdr;
586  virtio_net_ctrl_ack status = VIRTIO_NET_ERR;
587 
588  mq_hdr.ctrl.class = VIRTIO_NET_CTRL_MQ;
590  mq_hdr.status = VIRTIO_NET_ERR;
591  clib_memcpy (mq_hdr.data, &num_queues, sizeof (num_queues));
592 
593  status = virtio_pci_send_ctrl_msg (vm, vif, &mq_hdr, sizeof (num_queues));
594  virtio_log_debug (vim, vif, "multi-queue enable %u queues", num_queues);
595  return status;
596 }
597 
598 static u8
600 {
601  if (qsz < 64 || qsz > 4096)
602  return 0;
603  if ((qsz % 64) != 0)
604  return 0;
605  return 1;
606 }
607 
608 clib_error_t *
610  u16 queue_num)
611 {
612  clib_error_t *error = 0;
613  virtio_main_t *vim = &virtio_main;
614  u16 queue_size = 0;
615  virtio_vring_t *vring;
616  struct vring vr;
617  u32 i = 0;
618  void *ptr = NULL;
619 
620  queue_size = virtio_pci_legacy_get_queue_num (vm, vif, queue_num);
621  if (!virtio_pci_queue_size_valid (queue_size))
622  clib_warning ("queue size is not valid");
623 
624  if (!is_pow2 (queue_size))
625  return clib_error_return (0, "ring size must be power of 2");
626 
627  if (queue_size > 32768)
628  return clib_error_return (0, "ring size must be 32768 or lower");
629 
630  if (queue_size == 0)
631  queue_size = 256;
632 
634  vring = vec_elt_at_index (vif->cxq_vring, 0);
635  i = vring_size (queue_size, VIRTIO_PCI_VRING_ALIGN);
637  ptr =
639  vif->numa_node);
640  if (!ptr)
641  return vlib_physmem_last_error (vm);
642  clib_memset (ptr, 0, i);
643  vring_init (&vr, queue_size, ptr, VIRTIO_PCI_VRING_ALIGN);
644  vring->desc = vr.desc;
645  vring->avail = vr.avail;
646  vring->used = vr.used;
647  vring->queue_id = queue_num;
648  vring->avail->flags = VIRTIO_RING_FLAG_MASK_INT;
649 
650  ASSERT (vring->buffers == 0);
651 
652  vring->size = queue_size;
653  virtio_log_debug (vim, vif, "control-queue: number %u, size %u", queue_num,
654  queue_size);
655  virtio_pci_legacy_setup_queue (vm, vif, queue_num, ptr);
656  vring->kick_fd = -1;
657 
658  return error;
659 }
660 
661 clib_error_t *
663 {
664  clib_error_t *error = 0;
665  virtio_main_t *vim = &virtio_main;
667  u16 queue_size = 0;
668  virtio_vring_t *vring;
669  struct vring vr;
670  u32 i = 0;
671  void *ptr = NULL;
672 
673  queue_size = virtio_pci_legacy_get_queue_num (vm, vif, queue_num);
674  if (!virtio_pci_queue_size_valid (queue_size))
675  clib_warning ("queue size is not valid");
676 
677  if (!is_pow2 (queue_size))
678  return clib_error_return (0, "ring size must be power of 2");
679 
680  if (queue_size > 32768)
681  return clib_error_return (0, "ring size must be 32768 or lower");
682 
683  if (queue_size == 0)
684  queue_size = 256;
685 
686  if (queue_num % 2)
687  {
688  if (TX_QUEUE_ACCESS (queue_num) > vtm->n_vlib_mains)
689  return error;
692  vring = vec_elt_at_index (vif->txq_vrings, TX_QUEUE_ACCESS (queue_num));
693  clib_spinlock_init (&vring->lockp);
694  }
695  else
696  {
699  vring = vec_elt_at_index (vif->rxq_vrings, RX_QUEUE_ACCESS (queue_num));
700  }
701  i = vring_size (queue_size, VIRTIO_PCI_VRING_ALIGN);
703  ptr =
705  vif->numa_node);
706  if (!ptr)
707  return vlib_physmem_last_error (vm);
708  clib_memset (ptr, 0, i);
709  vring_init (&vr, queue_size, ptr, VIRTIO_PCI_VRING_ALIGN);
710  vring->desc = vr.desc;
711  vring->avail = vr.avail;
712  vring->used = vr.used;
713  vring->queue_id = queue_num;
714  vring->avail->flags = VIRTIO_RING_FLAG_MASK_INT;
715 
716  ASSERT (vring->buffers == 0);
717  vec_validate_aligned (vring->buffers, queue_size, CLIB_CACHE_LINE_BYTES);
718  if (queue_num % 2)
719  {
720  virtio_log_debug (vim, vif, "tx-queue: number %u, size %u", queue_num,
721  queue_size);
722  clib_memset_u32 (vring->buffers, ~0, queue_size);
723  }
724  else
725  {
726  virtio_log_debug (vim, vif, "rx-queue: number %u, size %u", queue_num,
727  queue_size);
728  }
729  vring->size = queue_size;
730  if (virtio_pci_legacy_setup_queue (vm, vif, queue_num, ptr))
731  return clib_error_return (0, "error in queue address setup");
732 
733  vring->kick_fd = -1;
734  return error;
735 }
736 
737 static void
739  u64 req_features)
740 {
741  /*
742  * if features are not requested
743  * default: all supported features
744  */
745  u64 supported_features = VIRTIO_FEATURE (VIRTIO_NET_F_CSUM)
746  | VIRTIO_FEATURE (VIRTIO_NET_F_GUEST_CSUM)
749  | VIRTIO_FEATURE (VIRTIO_NET_F_MAC)
750  | VIRTIO_FEATURE (VIRTIO_NET_F_GSO)
751  | VIRTIO_FEATURE (VIRTIO_NET_F_GUEST_TSO4)
752  | VIRTIO_FEATURE (VIRTIO_NET_F_GUEST_TSO6)
753  | VIRTIO_FEATURE (VIRTIO_NET_F_GUEST_UFO)
754  | VIRTIO_FEATURE (VIRTIO_NET_F_HOST_TSO4)
755  | VIRTIO_FEATURE (VIRTIO_NET_F_HOST_TSO6)
756  | VIRTIO_FEATURE (VIRTIO_NET_F_HOST_UFO)
757  | VIRTIO_FEATURE (VIRTIO_NET_F_MRG_RXBUF)
758  | VIRTIO_FEATURE (VIRTIO_NET_F_STATUS)
759  | VIRTIO_FEATURE (VIRTIO_NET_F_CTRL_VQ)
760  | VIRTIO_FEATURE (VIRTIO_NET_F_MQ)
761  | VIRTIO_FEATURE (VIRTIO_F_NOTIFY_ON_EMPTY)
762  | VIRTIO_FEATURE (VIRTIO_F_ANY_LAYOUT)
763  | VIRTIO_FEATURE (VIRTIO_RING_F_INDIRECT_DESC);
764 
765  if (req_features == 0)
766  {
767  req_features = supported_features;
768  }
769 
770  vif->features = req_features & vif->remote_features & supported_features;
771 
773  {
774  virtio_net_config_t config;
775  virtio_pci_legacy_read_config (vm, vif, &config.mtu,
776  sizeof (config.mtu),
778  mtu));
779  if (config.mtu < 64)
781  }
782 
783  vif->features = virtio_pci_legacy_set_features (vm, vif, vif->features);
784 }
785 
786 void
788 {
790 }
791 
792 int
794 {
795  u8 status = 0;
796 
797  /*
798  * Reset the device
799  */
800  status = virtio_pci_legacy_reset (vm, vif);
801 
802  /*
803  * Set the Acknowledge status bit
804  */
805  virtio_pci_legacy_set_status (vm, vif, VIRTIO_CONFIG_STATUS_ACK);
806 
807  /*
808  * Set the Driver status bit
809  */
810  virtio_pci_legacy_set_status (vm, vif, VIRTIO_CONFIG_STATUS_DRIVER);
811 
812  /*
813  * Read the status and verify it
814  */
815  status = virtio_pci_legacy_get_status (vm, vif);
816  if (!
817  ((status & VIRTIO_CONFIG_STATUS_ACK)
818  && (status & VIRTIO_CONFIG_STATUS_DRIVER)))
819  return -1;
820  vif->status = status;
821 
822  return 0;
823 }
824 
825 clib_error_t *
827 {
828  clib_error_t *error = 0;
829  virtio_main_t *vim = &virtio_main;
830  struct virtio_pci_cap cap;
831  u8 pos, common_cfg = 0, notify_base = 0, dev_cfg = 0, isr = 0, pci_cfg = 0;
833 
834  if ((error = vlib_pci_read_config_u8 (vm, h, PCI_CAPABILITY_LIST, &pos)))
835  {
836  virtio_log_error (vim, vif, "error in reading capabilty list position");
837  clib_error_return (error, "error in reading capabilty list position");
838  }
839  while (pos)
840  {
841  if ((error =
842  vlib_pci_read_write_config (vm, h, VLIB_READ, pos, &cap,
843  sizeof (cap))))
844  {
845  virtio_log_error (vim, vif, "%s [%2x]",
846  "error in reading the capability at", pos);
847  clib_error_return (error,
848  "error in reading the capability at [%2x]", pos);
849  }
850 
851  if (cap.cap_vndr == PCI_CAP_ID_MSIX)
852  {
853  u16 flags, table_size, table_size_mask = 0x07FF;
854 
855  if ((error =
856  vlib_pci_read_write_config (vm, h, VLIB_READ, pos + 2, &flags,
857  sizeof (flags))))
858  clib_error_return (error,
859  "error in reading the capability at [%2x]",
860  pos + 2);
861 
862  table_size = flags & table_size_mask;
863  virtio_log_debug (vim, vif, "flags:0x%x %s 0x%x", flags,
864  "msix interrupt vector table-size", table_size);
865 
866  if (flags & PCI_MSIX_ENABLE)
867  {
868  virtio_log_debug (vim, vif, "msix interrupt enabled");
870  }
871  else
872  {
873  virtio_log_debug (vim, vif, "msix interrupt disabled");
875  }
876  }
877 
878  if (cap.cap_vndr != PCI_CAP_ID_VNDR)
879  {
880  virtio_log_debug (vim, vif, "[%2x] %s %2x ", pos,
881  "skipping non VNDR cap id:", cap.cap_vndr);
882  goto next;
883  }
884 
885  virtio_log_debug (vim, vif,
886  "[%4x] cfg type: %u, bar: %u, offset: %04x, len: %u",
887  pos, cap.cfg_type, cap.bar, cap.offset, cap.length);
888  switch (cap.cfg_type)
889  {
891  common_cfg = 1;
892  break;
894  notify_base = 1;
895  break;
897  dev_cfg = 1;
898  break;
900  isr = 1;
901  break;
903  if (cap.bar == 0)
904  pci_cfg = 1;
905  break;
906  }
907  next:
908  pos = cap.cap_next;
909  }
910 
911  if (common_cfg == 0 || notify_base == 0 || dev_cfg == 0 || isr == 0)
912  {
913  virtio_log_debug (vim, vif, "legacy virtio pci device found");
914  return error;
915  }
916 
917  if (!pci_cfg)
918  clib_error_return (error, "modern virtio pci device found");
919 
920  virtio_log_debug (vim, vif, "transitional virtio pci device found");
921  return error;
922 }
923 
924 static clib_error_t *
927 {
928  clib_error_t *error = 0;
929  virtio_main_t *vim = &virtio_main;
930  u8 status = 0;
931 
932  if ((error = virtio_pci_read_caps (vm, vif)))
933  clib_error_return (error, "Device is not supported");
934 
935  if (virtio_pci_reset_device (vm, vif) < 0)
936  {
937  virtio_log_error (vim, vif, "Failed to reset the device");
938  clib_error_return (error, "Failed to reset the device");
939  }
940  /*
941  * read device features and negotiate (user) requested features
942  */
944  virtio_negotiate_features (vm, vif, args->features);
945 
946  /*
947  * After FEATURE_OK, driver should not accept new feature bits
948  */
949  virtio_pci_legacy_set_status (vm, vif, VIRTIO_CONFIG_STATUS_FEATURES_OK);
950  status = virtio_pci_legacy_get_status (vm, vif);
951  if (!(status & VIRTIO_CONFIG_STATUS_FEATURES_OK))
952  {
953  virtio_log_error (vim, vif,
954  "error encountered: Device doesn't support requested features");
955  clib_error_return (error, "Device doesn't support requested features");
956  }
957  vif->status = status;
958 
959  /*
960  * get or set the mac address
961  */
962  if (virtio_pci_get_mac (vm, vif))
963  {
964  f64 now = vlib_time_now (vm);
965  u32 rnd;
966  rnd = (u32) (now * 1e6);
967  rnd = random_u32 (&rnd);
968 
969  memcpy (vif->mac_addr + 2, &rnd, sizeof (rnd));
970  vif->mac_addr[0] = 2;
971  vif->mac_addr[1] = 0xfe;
972  virtio_pci_set_mac (vm, vif);
973  }
974 
976 
977  /*
978  * Initialize the virtqueues
979  */
980  if ((error = virtio_pci_get_max_virtqueue_pairs (vm, vif)))
981  goto err;
982 
983  for (int i = 0; i < vif->max_queue_pairs; i++)
984  {
985  if ((error = virtio_pci_vring_init (vm, vif, RX_QUEUE (i))))
986  {
987  virtio_log_warning (vim, vif, "%s (%u) %s", "error in rxq-queue",
988  RX_QUEUE (i), "initialization");
989  }
990  else
991  {
992  vif->num_rxqs++;
993  }
994 
995  if ((error = virtio_pci_vring_init (vm, vif, TX_QUEUE (i))))
996  {
997  virtio_log_warning (vim, vif, "%s (%u) %s", "error in txq-queue",
998  TX_QUEUE (i), "initialization");
999  }
1000  else
1001  {
1002  vif->num_txqs++;
1003  }
1004  }
1005 
1006  if (vif->features & VIRTIO_FEATURE (VIRTIO_NET_F_CTRL_VQ))
1007  {
1008  if ((error =
1009  virtio_pci_control_vring_init (vm, vif, vif->max_queue_pairs * 2)))
1010  {
1011  virtio_log_warning (vim, vif, "%s (%u) %s",
1012  "error in control-queue",
1013  vif->max_queue_pairs * 2, "initialization");
1014  if (vif->features & VIRTIO_FEATURE (VIRTIO_NET_F_MQ))
1015  vif->features &= ~VIRTIO_FEATURE (VIRTIO_NET_F_MQ);
1016  }
1017  }
1018  else
1019  {
1020  virtio_log_debug (vim, vif, "control queue is not available");
1021  vif->cxq_vring = NULL;
1022  }
1023 
1024  /*
1025  * set the msix interrupts
1026  */
1027  if (vif->msix_enabled == VIRTIO_MSIX_ENABLED)
1028  {
1029  if (virtio_pci_legacy_set_config_irq (vm, vif, 1) ==
1030  VIRTIO_MSI_NO_VECTOR)
1031  virtio_log_warning (vim, vif, "config vector 1 is not set");
1032  if (virtio_pci_legacy_set_queue_irq (vm, vif, 0, 0) ==
1033  VIRTIO_MSI_NO_VECTOR)
1034  virtio_log_warning (vim, vif, "queue vector 0 is not set");
1035  }
1036 
1037  /*
1038  * set the driver status OK
1039  */
1040  virtio_pci_legacy_set_status (vm, vif, VIRTIO_CONFIG_STATUS_DRIVER_OK);
1041  vif->status = virtio_pci_legacy_get_status (vm, vif);
1042 err:
1043  return error;
1044 }
1045 
1046 void
1048 {
1049  vnet_main_t *vnm = vnet_get_main ();
1050  virtio_main_t *vim = &virtio_main;
1051  virtio_if_t *vif;
1053  clib_error_t *error = 0;
1054 
1055  /* *INDENT-OFF* */
1056  pool_foreach (vif, vim->interfaces, ({
1057  if (vif->pci_addr.as_u32 == args->addr)
1058  {
1059  args->rv = VNET_API_ERROR_INVALID_VALUE;
1060  args->error =
1061  clib_error_return (error, "PCI address in use");
1062  vlib_log (VLIB_LOG_LEVEL_ERR, vim->log_default, "%U: %s",
1063  format_vlib_pci_addr, &args->addr,
1064  " PCI address in use");
1065  return;
1066  }
1067  }));
1068  /* *INDENT-ON* */
1069 
1070  pool_get (vim->interfaces, vif);
1071  vif->dev_instance = vif - vim->interfaces;
1072  vif->per_interface_next_index = ~0;
1073  vif->pci_addr.as_u32 = args->addr;
1074 
1075  if ((error =
1076  vlib_pci_device_open (vm, (vlib_pci_addr_t *) & vif->pci_addr,
1077  virtio_pci_device_ids, &h)))
1078  {
1079  args->rv = VNET_API_ERROR_INVALID_INTERFACE;
1080  args->error =
1081  clib_error_return (error, "pci-addr %U", format_vlib_pci_addr,
1082  &vif->pci_addr);
1083  vlib_log (VLIB_LOG_LEVEL_ERR, vim->log_default, "%U: %s",
1084  format_vlib_pci_addr, &vif->pci_addr,
1085  "error encountered on pci device open");
1086  pool_put (vim->interfaces, vif);
1087  return;
1088  }
1089  vif->pci_dev_handle = h;
1090  vlib_pci_set_private_data (vm, h, vif->dev_instance);
1091  vif->numa_node = vlib_pci_get_numa_node (vm, h);
1092 
1093  if ((error = vlib_pci_bus_master_enable (vm, h)))
1094  {
1095  virtio_log_error (vim, vif,
1096  "error encountered on pci bus master enable");
1097  goto error;
1098  }
1099 
1100  if ((error = vlib_pci_io_region (vm, h, 0)))
1101  {
1102  virtio_log_error (vim, vif, "error encountered on pci io region");
1103  goto error;
1104  }
1105 
1107  {
1108  if ((error = vlib_pci_register_msix_handler (vm, h, 0, 1,
1110  {
1111  virtio_log_error (vim, vif,
1112  "error encountered on pci register msix handler 0");
1113  goto error;
1114  }
1115  if ((error = vlib_pci_register_msix_handler (vm, h, 1, 1,
1117  {
1118  virtio_log_error (vim, vif,
1119  "error encountered on pci register msix handler 1");
1120  goto error;
1121  }
1122 
1123  if ((error = vlib_pci_enable_msix_irq (vm, h, 0, 2)))
1124  {
1125  virtio_log_error (vim, vif,
1126  "error encountered on pci enable msix irq");
1127  goto error;
1128  }
1129  vif->support_int_mode = 1;
1130  virtio_log_debug (vim, vif, "device supports msix interrupts");
1131  }
1132  else if (vlib_pci_get_num_msix_interrupts (vm, h) == 1)
1133  {
1134  /*
1135  * if msix table-size is 1, fall back to intX.
1136  */
1137  if ((error =
1139  {
1140  virtio_log_error (vim, vif,
1141  "error encountered on pci register interrupt handler");
1142  goto error;
1143  }
1144  vif->support_int_mode = 1;
1145  virtio_log_debug (vim, vif, "pci register interrupt handler");
1146  }
1147  else
1148  {
1149  /*
1150  * WARN: intX is showing some weird behaviour.
1151  * Please don't use interrupt mode with UIO driver.
1152  */
1153  vif->support_int_mode = 0;
1154  virtio_log_debug (vim, vif, "driver is configured in poll mode only");
1155  }
1156 
1157  if ((error = vlib_pci_intr_enable (vm, h)))
1158  {
1159  virtio_log_error (vim, vif,
1160  "error encountered on pci interrupt enable");
1161  goto error;
1162  }
1163 
1164  if ((error = virtio_pci_device_init (vm, vif, args)))
1165  {
1166  virtio_log_error (vim, vif, "error encountered on device init");
1167  goto error;
1168  }
1169 
1170  vif->type = VIRTIO_IF_TYPE_PCI;
1171  /* create interface */
1173  vif->dev_instance, vif->mac_addr,
1174  &vif->hw_if_index,
1176 
1177  if (error)
1178  {
1179  virtio_log_error (vim, vif,
1180  "error encountered on ethernet register interface");
1181  goto error;
1182  }
1183 
1184  vnet_sw_interface_t *sw = vnet_get_hw_sw_interface (vnm, vif->hw_if_index);
1185  vif->sw_if_index = sw->sw_if_index;
1186  args->sw_if_index = sw->sw_if_index;
1187 
1188  vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, vif->hw_if_index);
1190  vnet_hw_interface_set_input_node (vnm, vif->hw_if_index,
1191  virtio_input_node.index);
1192  u32 i = 0;
1193  vec_foreach_index (i, vif->rxq_vrings)
1194  {
1195  vnet_hw_interface_assign_rx_thread (vnm, vif->hw_if_index, i, ~0);
1197  /* Set default rx mode to POLLING */
1198  vnet_hw_interface_set_rx_mode (vnm, vif->hw_if_index, i,
1200  }
1201  if (virtio_pci_is_link_up (vm, vif) & VIRTIO_NET_S_LINK_UP)
1202  {
1203  vif->flags |= VIRTIO_IF_FLAG_ADMIN_UP;
1204  vnet_hw_interface_set_flags (vnm, vif->hw_if_index,
1206  }
1207  else
1208  vnet_hw_interface_set_flags (vnm, vif->hw_if_index, 0);
1209 
1210  if (vif->features & VIRTIO_FEATURE (VIRTIO_NET_F_CTRL_VQ))
1211  {
1212  if (vif->features & VIRTIO_FEATURE (VIRTIO_NET_F_CTRL_GUEST_OFFLOADS) &&
1213  args->gso_enabled)
1214  {
1215  if (virtio_pci_enable_gso (vm, vif))
1216  {
1217  virtio_log_warning (vim, vif, "gso is not enabled");
1218  }
1219  else
1220  {
1221  vif->gso_enabled = 1;
1223  vnm->interface_main.gso_interface_count++;
1224  }
1225  }
1226  if (vif->features & VIRTIO_FEATURE (VIRTIO_NET_F_MQ))
1227  {
1228  if (virtio_pci_enable_multiqueue (vm, vif, vif->max_queue_pairs))
1229  virtio_log_warning (vim, vif, "multiqueue is not set");
1230  }
1231  }
1232  return;
1233 
1234 
1235 error:
1236  virtio_pci_delete_if (vm, vif);
1237  args->rv = VNET_API_ERROR_INVALID_INTERFACE;
1238  args->error = error;
1239 }
1240 
1241 int
1243 {
1244  vnet_main_t *vnm = vnet_get_main ();
1245  virtio_main_t *vim = &virtio_main;
1246  u32 i = 0;
1247 
1248  if (vif->type != VIRTIO_IF_TYPE_PCI)
1249  return VNET_API_ERROR_INVALID_INTERFACE;
1250 
1252 
1253  for (i = 0; i < vif->max_queue_pairs; i++)
1254  {
1255  virtio_pci_legacy_del_queue (vm, vif, RX_QUEUE (i));
1256  virtio_pci_legacy_del_queue (vm, vif, TX_QUEUE (i));
1257  }
1258 
1259  if (vif->features & VIRTIO_FEATURE (VIRTIO_NET_F_CTRL_VQ))
1260  virtio_pci_legacy_del_queue (vm, vif, vif->max_queue_pairs * 2);
1261 
1262  virtio_pci_legacy_reset (vm, vif);
1263 
1264  if (vif->gso_enabled)
1266 
1267  if (vif->hw_if_index)
1268  {
1270  vec_foreach_index (i, vif->rxq_vrings)
1271  {
1273  }
1275  }
1276 
1278 
1279  vec_foreach_index (i, vif->rxq_vrings)
1280  {
1281  virtio_vring_t *vring = vec_elt_at_index (vif->rxq_vrings, i);
1282  if (vring->kick_fd != -1)
1283  close (vring->kick_fd);
1284  if (vring->used)
1285  {
1286  virtio_free_rx_buffers (vm, vring);
1287  }
1288  vec_free (vring->buffers);
1289  vlib_physmem_free (vm, vring->desc);
1290  }
1291 
1292  vec_foreach_index (i, vif->txq_vrings)
1293  {
1294  virtio_vring_t *vring = vec_elt_at_index (vif->txq_vrings, i);
1295  if (vring->kick_fd != -1)
1296  close (vring->kick_fd);
1297  if (vring->used)
1298  {
1299  virtio_free_used_desc (vm, vring);
1300  }
1301  vec_free (vring->buffers);
1302  clib_spinlock_free (&vring->lockp);
1303  vlib_physmem_free (vm, vring->desc);
1304  }
1305 
1306  if (vif->cxq_vring != NULL)
1307  {
1308  u16 last = vif->cxq_vring->last_used_idx;
1309  u16 n_left = vif->cxq_vring->used->idx - last;
1310  while (n_left)
1311  {
1312  last++;
1313  n_left--;
1314  }
1315 
1316  vif->cxq_vring->last_used_idx = last;
1317  vlib_physmem_free (vm, vif->cxq_vring->desc);
1318  }
1319 
1320  vec_free (vif->rxq_vrings);
1321  vec_free (vif->txq_vrings);
1322  vec_free (vif->cxq_vring);
1323 
1324  if (vif->fd != -1)
1325  vif->fd = -1;
1326  if (vif->tap_fd != -1)
1327  vif->tap_fd = -1;
1328  clib_error_free (vif->error);
1329  memset (vif, 0, sizeof (*vif));
1330  pool_put (vim->interfaces, vif);
1331 
1332  return 0;
1333 }
1334 
1335 /*
1336  * fd.io coding-style-patch-verification: ON
1337  *
1338  * Local Variables:
1339  * eval: (c-set-style "gnu")
1340  * End:
1341  */
static u8 virtio_pci_legacy_get_isr(vlib_main_t *vm, virtio_if_t *vif)
Definition: pci.c:161
struct vring_used * used
Definition: virtio.h:105
static uword vlib_buffer_get_current_pa(vlib_main_t *vm, vlib_buffer_t *b)
Definition: buffer_funcs.h:427
#define vec_foreach_index(var, v)
Iterate over vector indices.
static clib_error_t * vlib_pci_intr_enable(vlib_main_t *vm, vlib_pci_dev_handle_t h)
Definition: pci.h:239
vlib_node_registration_t virtio_input_node
(constructor) VLIB_REGISTER_NODE (virtio_input_node)
Definition: node.c:398
void virtio_set_net_hdr_size(virtio_if_t *vif)
Definition: virtio.c:263
virtio_if_t * interfaces
Definition: virtio.h:193
u32 flags
Definition: vhost_user.h:141
clib_error_t * virtio_pci_vring_init(vlib_main_t *vm, virtio_if_t *vif, u16 queue_num)
Definition: pci.c:662
static int virtio_pci_legacy_setup_queue(vlib_main_t *vm, virtio_if_t *vif, u16 queue_id, void *p)
Definition: pci.c:181
static u32 virtio_pci_flag_change(vnet_main_t *vnm, vnet_hw_interface_t *hw, u32 flags)
Definition: pci.c:241
static void virtio_pci_legacy_del_queue(vlib_main_t *vm, virtio_if_t *vif, u16 queue_id)
Definition: pci.c:198
static clib_error_t * vlib_pci_bus_master_enable(vlib_main_t *vm, vlib_pci_dev_handle_t h)
Definition: pci.h:271
static void * vlib_physmem_alloc_aligned_on_numa(vlib_main_t *vm, uword n_bytes, uword alignment, u32 numa_node)
Definition: physmem_funcs.h:63
#define VIRTIO_PCI_CAP_ISR_CFG
Definition: pci.h:125
vl_api_mac_address_t mac
Definition: l2.api:490
a
Definition: bitmap.h:538
static void vlib_buffer_free(vlib_main_t *vm, u32 *buffers, u32 n_buffers)
Free buffers Frees the entire buffer chain for each buffer.
Definition: buffer_funcs.h:865
u16 max_virtqueue_pairs
Definition: virtio.h:95
void ethernet_delete_interface(vnet_main_t *vnm, u32 hw_if_index)
Definition: interface.c:324
vnet_main_t * vnet_get_main(void)
Definition: misc.c:46
static u16 virtio_pci_legacy_get_queue_num(vlib_main_t *vm, virtio_if_t *vif, u16 queue_id)
Definition: pci.c:169
vnet_interface_main_t interface_main
Definition: vnet.h:56
#define VIRTIO_MSI_QUEUE_VECTOR
Definition: pci.h:36
i16 current_data
signed offset in data[], pre_data[] that we are currently processing.
Definition: buffer.h:110
unsigned long u64
Definition: types.h:89
int gso_enabled
Definition: virtio.h:183
void vlib_pci_device_close(vlib_main_t *vm, vlib_pci_dev_handle_t h)
Definition: pci.c:1286
#define VIRTIO_PCI_STATUS
Definition: pci.h:31
#define CLIB_MEMORY_STORE_BARRIER()
Definition: clib.h:119
#define VIRTIO_PCI_QUEUE_PFN
Definition: pci.h:27
clib_memset(h->entries, 0, sizeof(h->entries[0]) *entries)
static u8 virtio_pci_legacy_get_status(vlib_main_t *vm, virtio_if_t *vif)
Definition: pci.c:138
static f64 vlib_time_now(vlib_main_t *vm)
Definition: main.h:279
u32 vlib_pci_get_num_msix_interrupts(vlib_main_t *vm, vlib_pci_dev_handle_t h)
Definition: pci.c:177
static clib_error_t * vlib_physmem_last_error(struct vlib_main_t *vm)
static u32 virtio_pci_legacy_set_features(vlib_main_t *vm, virtio_if_t *vif, u64 features)
Definition: pci.c:122
static clib_error_t * vlib_pci_intr_disable(vlib_main_t *vm, vlib_pci_dev_handle_t h)
Definition: pci.h:255
static vnet_hw_interface_t * vnet_get_hw_interface(vnet_main_t *vnm, u32 hw_if_index)
vnet_device_class_t virtio_device_class
static clib_error_t * virtio_pci_device_init(vlib_main_t *vm, virtio_if_t *vif, virtio_pci_create_if_args_t *args)
Definition: pci.c:925
vl_api_address_t src
Definition: gre.api:51
static heap_elt_t * last(heap_header_t *h)
Definition: heap.c:53
void virtio_vring_set_numa_node(vlib_main_t *vm, virtio_if_t *vif, u32 idx)
Definition: virtio.c:247
#define STRUCT_OFFSET_OF(t, f)
Definition: clib.h:66
clib_error_t * vlib_pci_enable_msix_irq(vlib_main_t *vm, vlib_pci_dev_handle_t h, u16 start, u16 count)
Definition: pci.c:894
#define VIRTIO_MSI_CONFIG_VECTOR
Definition: pci.h:35
u8 data[128]
Definition: ipsec.api:251
#define vec_validate_aligned(V, I, A)
Make sure vector is long enough for given index (no header, specified alignment)
Definition: vec.h:450
#define pool_get(P, E)
Allocate an object E from a pool P (unspecified alignment).
Definition: pool.h:236
vhost_vring_addr_t addr
Definition: vhost_user.h:147
#define VIRTIO_PCI_HOST_FEATURES
Definition: pci.h:25
static int virtio_pci_enable_gso(vlib_main_t *vm, virtio_if_t *vif)
Definition: pci.c:560
#define VIRTIO_NET_CTRL_GUEST_OFFLOADS
Definition: pci.h:117
static void virtio_pci_set_mac(vlib_main_t *vm, virtio_if_t *vif)
Definition: pci.c:273
static int virtio_pci_send_ctrl_msg(vlib_main_t *vm, virtio_if_t *vif, struct virtio_ctrl_msg *data, u32 len)
Definition: pci.c:463
unsigned char u8
Definition: types.h:56
void virtio_pci_create_if(vlib_main_t *vm, virtio_pci_create_if_args_t *args)
Definition: pci.c:1047
double f64
Definition: types.h:142
static void clib_spinlock_free(clib_spinlock_t *p)
Definition: lock.h:70
static vnet_sw_interface_t * vnet_get_hw_sw_interface(vnet_main_t *vnm, u32 hw_if_index)
#define clib_memcpy(d, s, n)
Definition: string.h:180
static void virtio_pci_irq_handler(vlib_main_t *vm, vlib_pci_dev_handle_t h)
Definition: pci.c:339
clib_error_t * vlib_pci_read_write_config(vlib_main_t *vm, vlib_pci_dev_handle_t h, vlib_read_or_write_t read_or_write, uword address, void *data, u32 n_bytes)
Definition: pci.c:1028
u32 msix_enabled
Definition: virtio.h:155
#define VIRTIO_PCI_CAP_COMMON_CFG
Definition: pci.h:121
struct vring_avail * avail
Definition: virtio.h:106
u64 features
Definition: virtio.h:164
int virtio_pci_reset_device(vlib_main_t *vm, virtio_if_t *vif)
Definition: pci.c:793
u32 hw_if_index
Definition: virtio.h:141
u8 * format_ethernet_address(u8 *s, va_list *args)
Definition: format.c:44
static u8 virtio_pci_queue_size_valid(u16 qsz)
Definition: pci.c:599
#define pool_foreach(VAR, POOL, BODY)
Iterate through pool.
Definition: pool.h:493
static u16 virtio_pci_legacy_set_queue_irq(vlib_main_t *vm, virtio_if_t *vif, u16 vec, u16 queue_id)
Definition: pci.c:228
#define PCI_DEVICE_ID_VIRTIO_NIC
Definition: pci.c:29
#define TX_QUEUE_ACCESS(X)
Definition: virtio.h:80
void device_status(vlib_main_t *vm, virtio_if_t *vif)
Definition: pci.c:362
static_always_inline void vnet_device_input_set_interrupt_pending(vnet_main_t *vnm, u32 hw_if_index, u16 queue_id)
Definition: devices.h:136
static void virtio_negotiate_features(vlib_main_t *vm, virtio_if_t *vif, u64 req_features)
Definition: pci.c:738
vnet_hw_interface_flags_t flags
Definition: interface.h:506
void virtio_free_rx_buffers(vlib_main_t *vm, virtio_vring_t *vring)
Definition: virtio.c:161
#define vec_elt_at_index(v, i)
Get vector value at index i checking that i is in bounds.
clib_spinlock_t lockp
Definition: virtio.h:107
#define clib_error_return(e, args...)
Definition: error.h:99
#define VIRTIO_PCI_CAP_NOTIFY_CFG
Definition: pci.h:123
unsigned int u32
Definition: types.h:88
static void virtio_pci_legacy_read_config(vlib_main_t *vm, virtio_if_t *vif, void *dst, int len, u32 addr)
Definition: pci.c:53
#define VIRTIO_NET_CTRL_GUEST_OFFLOADS_SET
Definition: pci.h:118
#define virtio_log_debug(vim, vif, f,...)
Definition: pci.h:135
u16 queue_id
Definition: virtio.h:114
static void clib_spinlock_init(clib_spinlock_t *p)
Definition: lock.h:63
u16 num_txqs
Definition: virtio.h:171
u32 pci_dev_handle
Definition: virtio.h:160
#define TX_QUEUE(X)
Definition: virtio.h:78
static void virtio_pci_legacy_set_status(vlib_main_t *vm, virtio_if_t *vif, u8 status)
Definition: pci.c:146
#define pool_elt_at_index(p, i)
Returns pointer to element at given index.
Definition: pool.h:514
uword size
#define VIRTIO_PCI_GUEST_FEATURES
Definition: pci.h:26
#define VIRTIO_NET_F_MTU
Definition: pci.h:105
int tap_fd
Definition: virtio.h:159
static void virtio_pci_irq_1_handler(vlib_main_t *vm, vlib_pci_dev_handle_t h, u16 line)
Definition: pci.c:318
void vlib_pci_set_private_data(vlib_main_t *vm, vlib_pci_dev_handle_t h, uword private_data)
Definition: pci.c:155
unsigned short u16
Definition: types.h:57
#define VIRTIO_PCI_ISR_CONFIG
Definition: pci.h:48
struct virtio_net_ctrl_hdr ctrl
Definition: pci.c:457
static void * vlib_buffer_get_current(vlib_buffer_t *b)
Get pointer to current data to process.
Definition: buffer.h:229
#define pool_put(P, E)
Free an object E in pool P.
Definition: pool.h:286
clib_error_t * vlib_pci_register_msix_handler(vlib_main_t *vm, vlib_pci_dev_handle_t h, u32 start, u32 count, pci_msix_handler_function_t *msix_handler)
Definition: pci.c:837
static u64 virtio_pci_legacy_get_features(vlib_main_t *vm, virtio_if_t *vif)
Definition: pci.c:113
#define PCI_MSIX_ENABLE
Definition: pci.c:37
uword vlib_pci_get_private_data(vlib_main_t *vm, vlib_pci_dev_handle_t h)
Definition: pci.c:148
u32 vlib_pci_dev_handle_t
Definition: pci.h:97
static u16 virtio_pci_legacy_set_config_irq(vlib_main_t *vm, virtio_if_t *vif, u16 vec)
Definition: pci.c:217
void virtio_pci_read_device_feature(vlib_main_t *vm, virtio_if_t *vif)
Definition: pci.c:787
vl_api_address_t dst
Definition: gre.api:52
clib_error_t * virtio_pci_read_caps(vlib_main_t *vm, virtio_if_t *vif)
Definition: pci.c:826
static clib_error_t * virtio_pci_get_max_virtqueue_pairs(vlib_main_t *vm, virtio_if_t *vif)
Definition: pci.c:248
#define PCI_CAPABILITY_LIST
Definition: pci.c:33
u8 status
Definition: virtio.h:172
u8 len
Definition: ip_types.api:90
u16 desc_next
Definition: virtio.h:109
u8 * format_vlib_pci_addr(u8 *s, va_list *va)
Definition: pci.c:141
#define VIRTIO_PCI_VRING_ALIGN
Definition: pci.h:133
u16 num_rxqs
Definition: virtio.h:170
virtio_vring_t * rxq_vrings
Definition: virtio.h:162
u32 vlib_pci_get_numa_node(vlib_main_t *vm, vlib_pci_dev_handle_t h)
Definition: pci.c:170
u16 last_used_idx
Definition: virtio.h:119
#define VIRTIO_PCI_QUEUE_NOTIFY
Definition: pci.h:30
vlib_main_t * vm
Definition: buffer.c:323
#define VIRTIO_PCI_ISR_INTR
Definition: pci.h:46
static void vlib_physmem_free(vlib_main_t *vm, void *p)
Definition: physmem_funcs.h:89
#define VIRTIO_RING_FLAG_MASK_INT
Definition: virtio.h:99
#define vec_free(V)
Free vector&#39;s memory (no header).
Definition: vec.h:341
#define clib_warning(format, args...)
Definition: error.h:59
static u16 virtio_pci_is_link_up(vlib_main_t *vm, virtio_if_t *vif)
Definition: pci.c:292
void debug_device_config_space(vlib_main_t *vm, virtio_if_t *vif)
Definition: pci.c:390
void vlib_log(vlib_log_level_t level, vlib_log_class_t class, char *fmt,...)
Definition: log.c:87
u8 mac_addr[6]
Definition: virtio.h:173
static void virtio_pci_legacy_write_config(vlib_main_t *vm, virtio_if_t *vif, void *src, int len, u32 addr)
Definition: pci.c:83
u32 flags
Definition: virtio.h:138
clib_error_t * virtio_pci_control_vring_init(vlib_main_t *vm, virtio_if_t *vif, u16 queue_num)
Definition: pci.c:609
clib_error_t * error
Definition: virtio.h:167
static uword round_pow2(uword x, uword pow2)
Definition: clib.h:248
void virtio_pci_legacy_notify_queue(vlib_main_t *vm, virtio_if_t *vif, u16 queue_id)
Definition: pci.c:208
int virtio_pci_delete_if(vlib_main_t *vm, virtio_if_t *vif)
Definition: pci.c:1242
u32 numa_node
Definition: virtio.h:143
virtio_if_type_t type
Definition: virtio.h:145
void virtio_free_used_desc(vlib_main_t *vm, virtio_vring_t *vring)
Definition: virtio.c:198
#define PCI_DEVICE_ID_VIRTIO_NIC_MODERN
Definition: pci.c:31
#define PCI_CONFIG_SIZE(vif)
Definition: pci.c:39
#define ASSERT(truth)
void vnet_hw_interface_assign_rx_thread(vnet_main_t *vnm, u32 hw_if_index, u16 queue_id, uword thread_index)
Definition: devices.c:139
u64 remote_features
Definition: virtio.h:164
#define VIRTIO_FEATURE(X)
Definition: virtio.h:76
clib_error_t * vlib_pci_register_intx_handler(vlib_main_t *vm, vlib_pci_dev_handle_t h, pci_intx_handler_function_t *intx_handler)
Definition: pci.c:786
virtio_main_t virtio_main
Definition: virtio.c:37
static uword is_pow2(uword x)
Definition: clib.h:236
#define VIRTIO_PCI_QUEUE_SEL
Definition: pci.h:29
Definition: defs.h:56
static u64 vlib_physmem_get_pa(vlib_main_t *vm, void *mem)
#define RX_QUEUE_ACCESS(X)
Definition: virtio.h:81
virtio_net_ctrl_ack status
Definition: pci.c:458
#define VIRTIO_PCI_CAP_DEVICE_CFG
Definition: pci.h:127
#define VIRTIO_NET_F_CTRL_GUEST_OFFLOADS
Definition: pci.h:104
#define PCI_CAP_ID_VNDR
Definition: pci.c:34
clib_error_t * ethernet_register_interface(vnet_main_t *vnm, u32 dev_class_index, u32 dev_instance, const u8 *address, u32 *hw_if_index_return, ethernet_flag_change_function_t flag_change)
Definition: interface.c:278
clib_error_t * vnet_hw_interface_set_flags(vnet_main_t *vnm, u32 hw_if_index, vnet_hw_interface_flags_t flags)
Definition: interface.c:492
VLIB buffer representation.
Definition: buffer.h:102
u8 data[1024]
Definition: pci.c:459
u64 uword
Definition: types.h:112
int fd
Definition: virtio.h:154
static void virtio_pci_irq_0_handler(vlib_main_t *vm, vlib_pci_dev_handle_t h, u16 line)
Definition: pci.c:306
#define virtio_log_warning(vim, vif, f,...)
Definition: pci.h:142
#define virtio_log_error(vim, vif, f,...)
Definition: pci.h:149
#define clib_error_free(e)
Definition: error.h:86
clib_error_t * vlib_pci_io_region(vlib_main_t *vm, vlib_pci_dev_handle_t h, u32 resource)
Definition: pci.c:1171
u32 * buffers
Definition: virtio.h:118
int vnet_hw_interface_unassign_rx_thread(vnet_main_t *vnm, u32 hw_if_index, u16 queue_id)
Definition: devices.c:188
static u8 virtio_pci_legacy_reset(vlib_main_t *vm, virtio_if_t *vif)
Definition: pci.c:154
virtio_vring_t * cxq_vring
Definition: virtio.h:185
static u32 random_u32(u32 *seed)
32-bit random number generator
Definition: random.h:69
static vlib_thread_main_t * vlib_get_thread_main()
Definition: global_funcs.h:32
struct vring_desc * desc
Definition: virtio.h:104
#define CLIB_MEMORY_BARRIER()
Definition: clib.h:116
int vnet_hw_interface_set_rx_mode(vnet_main_t *vnm, u32 hw_if_index, u16 queue_id, vnet_hw_interface_rx_mode mode)
Definition: devices.c:253
#define VIRTIO_PCI_CAP_PCI_CFG
Definition: pci.h:129
u16 max_queue_pairs
Definition: virtio.h:169
#define CLIB_CACHE_LINE_BYTES
Definition: cache.h:59
#define PCI_VENDOR_ID_VIRTIO
Definition: pci.c:28
static u32 vlib_buffer_alloc(vlib_main_t *vm, u32 *buffers, u32 n_buffers)
Allocate buffers into supplied array.
Definition: buffer_funcs.h:612
#define PCI_CAP_ID_MSIX
Definition: pci.c:35
void vlib_cli_output(vlib_main_t *vm, char *fmt,...)
Definition: cli.c:772
clib_error_t * vlib_pci_device_open(vlib_main_t *vm, vlib_pci_addr_t *addr, pci_device_id_t ids[], vlib_pci_dev_handle_t *handle)
Definition: pci.c:1226
#define RX_QUEUE(X)
Definition: virtio.h:79
static u32 virtio_pci_get_mac(vlib_main_t *vm, virtio_if_t *vif)
Definition: pci.c:280
static vlib_buffer_t * vlib_get_buffer(vlib_main_t *vm, u32 buffer_index)
Translate buffer index into buffer pointer.
Definition: buffer_funcs.h:85
static void vnet_hw_interface_set_input_node(vnet_main_t *vnm, u32 hw_if_index, u32 node_index)
Definition: devices.h:79
#define VIRTIO_NET_CTRL_MQ
Definition: vhost_user.h:40
#define VIRTIO_PCI_QUEUE_NUM
Definition: pci.h:28
u16 vendor_id
Definition: pci.h:127
static int virtio_pci_enable_multiqueue(vlib_main_t *vm, virtio_if_t *vif, u16 num_queues)
Definition: pci.c:581
#define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET
Definition: vhost_user.h:41
u16 desc_in_use
Definition: virtio.h:108
virtio_vring_t * txq_vrings
Definition: virtio.h:163
#define VIRTIO_PCI_QUEUE_ADDR_SHIFT
Definition: pci.h:131
static_always_inline void virtio_kick(vlib_main_t *vm, virtio_vring_t *vring, virtio_if_t *vif)
Definition: virtio.h:218
#define VIRTIO_PCI_ISR
Definition: pci.h:32
static_always_inline void clib_memset_u32(void *p, u32 val, uword count)
Definition: string.h:332