FD.io VPP  v19.08-27-gf4dcae4
Vector Packet Processing
pci.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2018 Cisco and/or its affiliates.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at:
6  *
7  * http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 
16 #include <fcntl.h>
17 #include <sys/ioctl.h>
18 
19 #include <vppinfra/types.h>
20 #include <vlib/vlib.h>
21 #include <vlib/pci/pci.h>
22 #include <vnet/ethernet/ethernet.h>
23 #include <vnet/ip/ip4_packet.h>
24 #include <vnet/ip/ip6_packet.h>
27 
28 #define PCI_VENDOR_ID_VIRTIO 0x1af4
29 #define PCI_DEVICE_ID_VIRTIO_NIC 0x1000
30 /* Doesn't support modern device */
31 #define PCI_DEVICE_ID_VIRTIO_NIC_MODERN 0x1041
32 
33 #define PCI_CAPABILITY_LIST 0x34
34 #define PCI_CAP_ID_VNDR 0x09
35 #define PCI_CAP_ID_MSIX 0x11
36 
37 #define PCI_MSIX_ENABLE 0x8000
38 
39 #define PCI_CONFIG_SIZE(vif) ((vif->msix_enabled == VIRTIO_MSIX_ENABLED) ? \
40  24 : 20)
41 
42 static pci_device_id_t virtio_pci_device_ids[] = {
43  {
45  .device_id = PCI_DEVICE_ID_VIRTIO_NIC},
46  {
47  .vendor_id = PCI_VENDOR_ID_VIRTIO,
48  .device_id = PCI_DEVICE_ID_VIRTIO_NIC_MODERN},
49  {0},
50 };
51 
52 static void
54  int len, u32 addr)
55 {
56  u32 size = 0;
58 
59  while (len > 0)
60  {
61  if (len >= 4)
62  {
63  size = 4;
64  vlib_pci_read_io_u32 (vm, h, PCI_CONFIG_SIZE (vif) + addr, dst);
65  }
66  else if (len >= 2)
67  {
68  size = 2;
69  vlib_pci_read_io_u16 (vm, h, PCI_CONFIG_SIZE (vif) + addr, dst);
70  }
71  else
72  {
73  size = 1;
74  vlib_pci_read_io_u8 (vm, h, PCI_CONFIG_SIZE (vif) + addr, dst);
75  }
76  dst = (u8 *) dst + size;
77  addr += size;
78  len -= size;
79  }
80 }
81 
82 static void
84  void *src, int len, u32 addr)
85 {
86  u32 size = 0;
88 
89  while (len > 0)
90  {
91  if (len >= 4)
92  {
93  size = 4;
94  vlib_pci_write_io_u32 (vm, h, PCI_CONFIG_SIZE (vif) + addr, src);
95  }
96  else if (len >= 2)
97  {
98  size = 2;
99  vlib_pci_write_io_u16 (vm, h, PCI_CONFIG_SIZE (vif) + addr, src);
100  }
101  else
102  {
103  size = 1;
104  vlib_pci_write_io_u8 (vm, h, PCI_CONFIG_SIZE (vif) + addr, src);
105  }
106  src = (u8 *) src + size;
107  addr += size;
108  len -= size;
109  }
110 }
111 
112 static u64
114 {
115  u32 features;
116  vlib_pci_read_io_u32 (vm, vif->pci_dev_handle, VIRTIO_PCI_HOST_FEATURES,
117  &features);
118  return features;
119 }
120 
121 static u32
123  u64 features)
124 {
125  if ((features >> 32) != 0)
126  {
127  clib_warning ("only 32 bit features are allowed for legacy virtio!");
128  }
129  u32 feature = 0, guest_features = (u32) features;
130  vlib_pci_write_io_u32 (vm, vif->pci_dev_handle, VIRTIO_PCI_GUEST_FEATURES,
131  &guest_features);
132  vlib_pci_read_io_u32 (vm, vif->pci_dev_handle, VIRTIO_PCI_GUEST_FEATURES,
133  &feature);
134  return feature;
135 }
136 
137 static u8
139 {
140  u8 status = 0;
141  vlib_pci_read_io_u8 (vm, vif->pci_dev_handle, VIRTIO_PCI_STATUS, &status);
142  return status;
143 }
144 
145 static void
147 {
148  if (status != VIRTIO_CONFIG_STATUS_RESET)
149  status |= virtio_pci_legacy_get_status (vm, vif);
150  vlib_pci_write_io_u8 (vm, vif->pci_dev_handle, VIRTIO_PCI_STATUS, &status);
151 }
152 
153 static u8
155 {
156  virtio_pci_legacy_set_status (vm, vif, VIRTIO_CONFIG_STATUS_RESET);
157  return virtio_pci_legacy_get_status (vm, vif);
158 }
159 
160 static u8
162 {
163  u8 isr = 0;
164  vlib_pci_read_io_u8 (vm, vif->pci_dev_handle, VIRTIO_PCI_ISR, &isr);
165  return isr;
166 }
167 
168 static u16
170  u16 queue_id)
171 {
172  u16 queue_num = 0;
173  vlib_pci_write_io_u16 (vm, vif->pci_dev_handle, VIRTIO_PCI_QUEUE_SEL,
174  &queue_id);
175  vlib_pci_read_io_u16 (vm, vif->pci_dev_handle, VIRTIO_PCI_QUEUE_NUM,
176  &queue_num);
177  return queue_num;
178 }
179 
180 static int
182  u16 queue_id, void *p)
183 {
185  u32 addr2 = 0;
186  vlib_pci_write_io_u16 (vm, vif->pci_dev_handle, VIRTIO_PCI_QUEUE_SEL,
187  &queue_id);
188  vlib_pci_write_io_u32 (vm, vif->pci_dev_handle, VIRTIO_PCI_QUEUE_PFN,
189  (u32 *) & addr);
190  vlib_pci_read_io_u32 (vm, vif->pci_dev_handle, VIRTIO_PCI_QUEUE_PFN,
191  &addr2);
192  if ((u32) addr == addr2)
193  return 0;
194  return 1;
195 }
196 
197 static void
199  u16 queue_id)
200 {
201  u32 src = 0;
202  vlib_pci_write_io_u16 (vm, vif->pci_dev_handle, VIRTIO_PCI_QUEUE_SEL,
203  &queue_id);
204  vlib_pci_write_io_u32 (vm, vif->pci_dev_handle, VIRTIO_PCI_QUEUE_PFN, &src);
205 }
206 
207 inline void
209  u16 queue_id)
210 {
211  vlib_pci_write_io_u16 (vm, vif->pci_dev_handle, VIRTIO_PCI_QUEUE_NOTIFY,
212  &queue_id);
213 }
214 
215 /* Enable one vector (0) for Link State Intrerrupt */
216 static u16
218  u16 vec)
219 {
220  vlib_pci_write_io_u16 (vm, vif->pci_dev_handle, VIRTIO_MSI_CONFIG_VECTOR,
221  &vec);
222  vlib_pci_read_io_u16 (vm, vif->pci_dev_handle, VIRTIO_MSI_CONFIG_VECTOR,
223  &vec);
224  return vec;
225 }
226 
227 static u16
229  u16 queue_id)
230 {
231  vlib_pci_write_io_u16 (vm, vif->pci_dev_handle, VIRTIO_PCI_QUEUE_SEL,
232  &queue_id);
233  vlib_pci_write_io_u16 (vm, vif->pci_dev_handle, VIRTIO_MSI_QUEUE_VECTOR,
234  &vec);
235  vlib_pci_read_io_u16 (vm, vif->pci_dev_handle, VIRTIO_MSI_QUEUE_VECTOR,
236  &vec);
237  return vec;
238 }
239 
240 static u32
242  u32 flags)
243 {
244  return 0;
245 }
246 
247 static clib_error_t *
249 {
250  virtio_main_t *vim = &virtio_main;
251  virtio_net_config_t config;
252  clib_error_t *error = 0;
253  u16 max_queue_pairs = 1;
254 
255  if (vif->features & VIRTIO_FEATURE (VIRTIO_NET_F_MQ))
256  {
258  sizeof (config.max_virtqueue_pairs),
260  max_virtqueue_pairs));
261  max_queue_pairs = config.max_virtqueue_pairs;
262  }
263 
264  virtio_log_debug (vim, vif, "max queue pair is %x", max_queue_pairs);
265  if (max_queue_pairs < 1 || max_queue_pairs > 0x8000)
266  return clib_error_return (error, "max queue pair is %x", max_queue_pairs);
267 
268  vif->max_queue_pairs = max_queue_pairs;
269  return error;
270 }
271 
272 static void
274 {
276  sizeof (vif->mac_addr), 0);
277 }
278 
279 static u32
281 {
282  if (vif->features & VIRTIO_FEATURE (VIRTIO_NET_F_MAC))
283  {
285  sizeof (vif->mac_addr), 0);
286  return 0;
287  }
288  return 1;
289 }
290 
291 static u16
293 {
294  /*
295  * Minimal driver: assumes link is up
296  */
297  u16 status = 1;
298  if (vif->features & VIRTIO_FEATURE (VIRTIO_NET_F_STATUS))
299  virtio_pci_legacy_read_config (vm, vif, &status, sizeof (status), /* mac */
301  status));
302  return status;
303 }
304 
305 static void
307 {
308  vnet_main_t *vnm = vnet_get_main ();
309  virtio_main_t *vim = &virtio_main;
310  uword pd = vlib_pci_get_private_data (vm, h);
311  virtio_if_t *vif = pool_elt_at_index (vim->interfaces, pd);
312  u16 qid = line;
313 
315 }
316 
317 static void
319 {
320  vnet_main_t *vnm = vnet_get_main ();
321  virtio_main_t *vim = &virtio_main;
322  uword pd = vlib_pci_get_private_data (vm, h);
323  virtio_if_t *vif = pool_elt_at_index (vim->interfaces, pd);
324 
325  if (virtio_pci_is_link_up (vm, vif) & VIRTIO_NET_S_LINK_UP)
326  {
327  vif->flags |= VIRTIO_IF_FLAG_ADMIN_UP;
330  }
331  else
332  {
333  vif->flags &= ~VIRTIO_IF_FLAG_ADMIN_UP;
335  }
336 }
337 
338 static void
340 {
341  virtio_main_t *vim = &virtio_main;
342  uword pd = vlib_pci_get_private_data (vm, h);
343  virtio_if_t *vif = pool_elt_at_index (vim->interfaces, pd);
344  u8 isr = 0;
345  u16 line = 0;
346 
347  isr = virtio_pci_legacy_get_isr (vm, vif);
348 
349  /*
350  * If the lower bit is set: look through the used rings of
351  * all virtqueues for the device, to see if any progress has
352  * been made by the device which requires servicing.
353  */
354  if (isr & VIRTIO_PCI_ISR_INTR)
355  virtio_pci_irq_0_handler (vm, h, line);
356 
357  if (isr & VIRTIO_PCI_ISR_CONFIG)
358  virtio_pci_irq_1_handler (vm, h, line);
359 }
360 
361 inline void
363 {
364  struct status_struct
365  {
366  u8 bit;
367  char *str;
368  };
369  struct status_struct *status_entry;
370  static struct status_struct status_array[] = {
371 #define _(s,b) { .str = #s, .bit = b, },
373 #undef _
374  {.str = NULL}
375  };
376 
377  vlib_cli_output (vm, " status 0x%x", vif->status);
378 
379  status_entry = (struct status_struct *) &status_array;
380  while (status_entry->str)
381  {
382  if (vif->status & status_entry->bit)
383  vlib_cli_output (vm, " %s (%x)", status_entry->str,
384  status_entry->bit);
385  status_entry++;
386  }
387 }
388 
389 inline void
391 {
392  u32 data_u32;
393  u16 data_u16;
394  u8 data_u8;
395  vlib_pci_read_io_u32 (vm, vif->pci_dev_handle, VIRTIO_PCI_HOST_FEATURES,
396  &data_u32);
397  vlib_cli_output (vm, "remote features 0x%lx", data_u32);
398  vlib_pci_read_io_u32 (vm, vif->pci_dev_handle, VIRTIO_PCI_GUEST_FEATURES,
399  &data_u32);
400  vlib_cli_output (vm, "guest features 0x%lx", data_u32);
401  vlib_pci_read_io_u32 (vm, vif->pci_dev_handle, VIRTIO_PCI_QUEUE_PFN,
402  &data_u32);
403  vlib_cli_output (vm, "queue address 0x%lx", data_u32);
404  vlib_pci_read_io_u16 (vm, vif->pci_dev_handle, VIRTIO_PCI_QUEUE_NUM,
405  &data_u16);
406  vlib_cli_output (vm, "queue size 0x%x", data_u16);
407  vlib_pci_read_io_u16 (vm, vif->pci_dev_handle, VIRTIO_PCI_QUEUE_SEL,
408  &data_u16);
409  vlib_cli_output (vm, "queue select 0x%x", data_u16);
410  vlib_pci_read_io_u16 (vm, vif->pci_dev_handle, VIRTIO_PCI_QUEUE_NOTIFY,
411  &data_u16);
412  vlib_cli_output (vm, "queue notify 0x%x", data_u16);
413  vlib_pci_read_io_u8 (vm, vif->pci_dev_handle, VIRTIO_PCI_STATUS, &data_u8);
414  vlib_cli_output (vm, "status 0x%x", data_u8);
415  vlib_pci_read_io_u8 (vm, vif->pci_dev_handle, VIRTIO_PCI_ISR, &data_u8);
416  vlib_cli_output (vm, "isr 0x%x", data_u8);
417 
418  if (vif->msix_enabled == VIRTIO_MSIX_ENABLED)
419  {
420  vlib_pci_read_io_u16 (vm, vif->pci_dev_handle, VIRTIO_MSI_CONFIG_VECTOR,
421  &data_u16);
422  vlib_cli_output (vm, "config vector 0x%x", data_u16);
423  u16 queue_id = 0;
424  vlib_pci_write_io_u16 (vm, vif->pci_dev_handle, VIRTIO_PCI_QUEUE_SEL,
425  &queue_id);
426  vlib_pci_read_io_u16 (vm, vif->pci_dev_handle, VIRTIO_MSI_QUEUE_VECTOR,
427  &data_u16);
428  vlib_cli_output (vm, "queue vector for queue (0) 0x%x", data_u16);
429  }
430 
431  u8 mac[6];
432  virtio_pci_legacy_read_config (vm, vif, mac, sizeof (mac), 0);
433  vlib_cli_output (vm, "mac %U", format_ethernet_address, mac);
434  virtio_pci_legacy_read_config (vm, vif, &data_u16, sizeof (u16), /* offset to status */
435  6);
436  vlib_cli_output (vm, "link up/down status 0x%x", data_u16);
437  virtio_pci_legacy_read_config (vm, vif, &data_u16, sizeof (u16),
438  /* offset to max_virtqueue */ 8);
439  vlib_cli_output (vm, "num of virtqueue 0x%x", data_u16);
440  virtio_pci_legacy_read_config (vm, vif, &data_u16, sizeof (u16), /* offset to mtu */
441  10);
442  vlib_cli_output (vm, "mtu 0x%x", data_u16);
443 
444  u32 i = PCI_CONFIG_SIZE (vif) + 12, a = 4;
445  i += a;
446  i &= ~a;
447  for (; i < 64; i += 4)
448  {
449  u32 data = 0;
450  vlib_pci_read_io_u32 (vm, vif->pci_dev_handle, i, &data);
451  vlib_cli_output (vm, "0x%lx", data);
452  }
453 }
454 
456 {
457  struct virtio_net_ctrl_hdr ctrl;
458  virtio_net_ctrl_ack status;
459  u8 data[1024];
460 };
461 
462 static int
464  struct virtio_ctrl_msg *data, u32 len)
465 {
466  virtio_main_t *vim = &virtio_main;
467  virtio_vring_t *vring = vif->cxq_vring;
468  virtio_net_ctrl_ack status = VIRTIO_NET_ERR;
469  struct virtio_ctrl_msg result;
470  u32 buffer_index;
471  vlib_buffer_t *b;
472  u16 used, next, avail;
473  u16 sz = vring->size;
474  u16 mask = sz - 1;
475 
476  used = vring->desc_in_use;
477  next = vring->desc_next;
478  avail = vring->avail->idx;
479  struct vring_desc *d = &vring->desc[next];
480 
481  if (vlib_buffer_alloc (vm, &buffer_index, 1))
482  b = vlib_get_buffer (vm, buffer_index);
483  else
484  return VIRTIO_NET_ERR;
485  /*
486  * current_data may not be initialized with 0 and may contain
487  * previous offset.
488  */
489  b->current_data = 0;
491  sizeof (struct virtio_ctrl_msg));
492  d->flags = VRING_DESC_F_NEXT;
493  d->addr = vlib_buffer_get_current_pa (vm, b);
494  d->len = sizeof (struct virtio_net_ctrl_hdr);
495  vring->avail->ring[avail & mask] = next;
496  avail++;
497  next = (next + 1) & mask;
498  d->next = next;
499  used++;
500 
501  d = &vring->desc[next];
502  d->flags = VRING_DESC_F_NEXT;
503  d->addr = vlib_buffer_get_current_pa (vm, b) +
504  STRUCT_OFFSET_OF (struct virtio_ctrl_msg, data);
505  d->len = len;
506  next = (next + 1) & mask;
507  d->next = next;
508  used++;
509 
510  d = &vring->desc[next];
511  d->flags = VRING_DESC_F_WRITE;
512  d->addr = vlib_buffer_get_current_pa (vm, b) +
513  STRUCT_OFFSET_OF (struct virtio_ctrl_msg, status);
514  d->len = sizeof (data->status);
515  next = (next + 1) & mask;
516  used++;
517 
519  vring->avail->idx = avail;
520  vring->desc_next = next;
521  vring->desc_in_use = used;
522 
523  if ((vring->used->flags & VIRTIO_RING_FLAG_MASK_INT) == 0)
524  {
525  virtio_kick (vm, vring, vif);
526  }
527 
528  u16 last = vring->last_used_idx, n_left = 0;
529  n_left = vring->used->idx - last;
530 
531  while (n_left)
532  {
533  struct vring_used_elem *e = &vring->used->ring[last & mask];
534  u16 slot = e->id;
535 
536  d = &vring->desc[slot];
537  while (d->flags & VRING_DESC_F_NEXT)
538  {
539  used--;
540  slot = d->next;
541  d = &vring->desc[slot];
542  }
543  used--;
544  last++;
545  n_left--;
546  }
547  vring->desc_in_use = used;
548  vring->last_used_idx = last;
549 
551  clib_memcpy (&result, vlib_buffer_get_current (b),
552  sizeof (struct virtio_ctrl_msg));
553  virtio_log_debug (vim, vif, "ctrl-queue: status %u", result.status);
554  status = result.status;
555  vlib_buffer_free (vm, &buffer_index, 1);
556  return status;
557 }
558 
559 static int
561 {
562  virtio_main_t *vim = &virtio_main;
563  struct virtio_ctrl_msg gso_hdr;
564  virtio_net_ctrl_ack status = VIRTIO_NET_ERR;
565 
566  gso_hdr.ctrl.class = VIRTIO_NET_CTRL_GUEST_OFFLOADS;
568  gso_hdr.status = VIRTIO_NET_ERR;
569  u64 offloads = VIRTIO_FEATURE (VIRTIO_NET_F_GUEST_CSUM)
570  | VIRTIO_FEATURE (VIRTIO_NET_F_GUEST_TSO4)
571  | VIRTIO_FEATURE (VIRTIO_NET_F_GUEST_TSO6)
572  | VIRTIO_FEATURE (VIRTIO_NET_F_GUEST_UFO);
573  clib_memcpy (gso_hdr.data, &offloads, sizeof (offloads));
574 
575  status = virtio_pci_send_ctrl_msg (vm, vif, &gso_hdr, sizeof (offloads));
576  virtio_log_debug (vim, vif, "enable gso");
577  return status;
578 }
579 
580 static int
582  u16 num_queues)
583 {
584  virtio_main_t *vim = &virtio_main;
585  struct virtio_ctrl_msg mq_hdr;
586  virtio_net_ctrl_ack status = VIRTIO_NET_ERR;
587 
588  mq_hdr.ctrl.class = VIRTIO_NET_CTRL_MQ;
590  mq_hdr.status = VIRTIO_NET_ERR;
591  clib_memcpy (mq_hdr.data, &num_queues, sizeof (num_queues));
592 
593  status = virtio_pci_send_ctrl_msg (vm, vif, &mq_hdr, sizeof (num_queues));
594  virtio_log_debug (vim, vif, "multi-queue enable %u queues", num_queues);
595  return status;
596 }
597 
598 static u8
600 {
601  if (qsz < 64 || qsz > 4096)
602  return 0;
603  if ((qsz % 64) != 0)
604  return 0;
605  return 1;
606 }
607 
608 clib_error_t *
610  u16 queue_num)
611 {
612  clib_error_t *error = 0;
613  virtio_main_t *vim = &virtio_main;
614  u16 queue_size = 0;
615  virtio_vring_t *vring;
616  struct vring vr;
617  u32 i = 0;
618  void *ptr = NULL;
619 
620  queue_size = virtio_pci_legacy_get_queue_num (vm, vif, queue_num);
621  if (!virtio_pci_queue_size_valid (queue_size))
622  clib_warning ("queue size is not valid");
623 
624  if (!is_pow2 (queue_size))
625  return clib_error_return (0, "ring size must be power of 2");
626 
627  if (queue_size > 32768)
628  return clib_error_return (0, "ring size must be 32768 or lower");
629 
630  if (queue_size == 0)
631  queue_size = 256;
632 
634  vring = vec_elt_at_index (vif->cxq_vring, 0);
635  i = vring_size (queue_size, VIRTIO_PCI_VRING_ALIGN);
637  ptr =
639  vif->numa_node);
640  if (!ptr)
641  return vlib_physmem_last_error (vm);
642  clib_memset (ptr, 0, i);
643  vring_init (&vr, queue_size, ptr, VIRTIO_PCI_VRING_ALIGN);
644  vring->desc = vr.desc;
645  vring->avail = vr.avail;
646  vring->used = vr.used;
647  vring->queue_id = queue_num;
648  vring->avail->flags = VIRTIO_RING_FLAG_MASK_INT;
649 
650  ASSERT (vring->buffers == 0);
651 
652  vring->size = queue_size;
653  virtio_log_debug (vim, vif, "control-queue: number %u, size %u", queue_num,
654  queue_size);
655  virtio_pci_legacy_setup_queue (vm, vif, queue_num, ptr);
656  vring->kick_fd = -1;
657 
658  return error;
659 }
660 
661 clib_error_t *
663 {
664  clib_error_t *error = 0;
665  virtio_main_t *vim = &virtio_main;
667  u16 queue_size = 0;
668  virtio_vring_t *vring;
669  struct vring vr;
670  u32 i = 0;
671  void *ptr = NULL;
672 
673  queue_size = virtio_pci_legacy_get_queue_num (vm, vif, queue_num);
674  if (!virtio_pci_queue_size_valid (queue_size))
675  clib_warning ("queue size is not valid");
676 
677  if (!is_pow2 (queue_size))
678  return clib_error_return (0, "ring size must be power of 2");
679 
680  if (queue_size > 32768)
681  return clib_error_return (0, "ring size must be 32768 or lower");
682 
683  if (queue_size == 0)
684  queue_size = 256;
685 
686  if (queue_num % 2)
687  {
688  if (TX_QUEUE_ACCESS (queue_num) > vtm->n_vlib_mains)
689  return error;
692  vring = vec_elt_at_index (vif->txq_vrings, TX_QUEUE_ACCESS (queue_num));
693  clib_spinlock_init (&vring->lockp);
694  }
695  else
696  {
699  vring = vec_elt_at_index (vif->rxq_vrings, RX_QUEUE_ACCESS (queue_num));
700  }
701  i = vring_size (queue_size, VIRTIO_PCI_VRING_ALIGN);
703  ptr =
705  vif->numa_node);
706  if (!ptr)
707  return vlib_physmem_last_error (vm);
708  clib_memset (ptr, 0, i);
709  vring_init (&vr, queue_size, ptr, VIRTIO_PCI_VRING_ALIGN);
710  vring->desc = vr.desc;
711  vring->avail = vr.avail;
712  vring->used = vr.used;
713  vring->queue_id = queue_num;
714  vring->avail->flags = VIRTIO_RING_FLAG_MASK_INT;
715 
716  ASSERT (vring->buffers == 0);
717  vec_validate_aligned (vring->buffers, queue_size, CLIB_CACHE_LINE_BYTES);
718  if (queue_num % 2)
719  {
720  virtio_log_debug (vim, vif, "tx-queue: number %u, size %u", queue_num,
721  queue_size);
722  }
723  else
724  {
725  virtio_log_debug (vim, vif, "rx-queue: number %u, size %u", queue_num,
726  queue_size);
727  }
728  vring->size = queue_size;
729  if (virtio_pci_legacy_setup_queue (vm, vif, queue_num, ptr))
730  return clib_error_return (0, "error in queue address setup");
731 
732  vring->kick_fd = -1;
733  return error;
734 }
735 
736 static void
738  u64 req_features)
739 {
740  /*
741  * if features are not requested
742  * default: all supported features
743  */
744  u64 supported_features = VIRTIO_FEATURE (VIRTIO_NET_F_CSUM)
745  | VIRTIO_FEATURE (VIRTIO_NET_F_GUEST_CSUM)
748  | VIRTIO_FEATURE (VIRTIO_NET_F_MAC)
749  | VIRTIO_FEATURE (VIRTIO_NET_F_GSO)
750  | VIRTIO_FEATURE (VIRTIO_NET_F_GUEST_TSO4)
751  | VIRTIO_FEATURE (VIRTIO_NET_F_GUEST_TSO6)
752  | VIRTIO_FEATURE (VIRTIO_NET_F_GUEST_UFO)
753  | VIRTIO_FEATURE (VIRTIO_NET_F_HOST_TSO4)
754  | VIRTIO_FEATURE (VIRTIO_NET_F_HOST_TSO6)
755  | VIRTIO_FEATURE (VIRTIO_NET_F_HOST_UFO)
756  | VIRTIO_FEATURE (VIRTIO_NET_F_MRG_RXBUF)
757  | VIRTIO_FEATURE (VIRTIO_NET_F_STATUS)
758  | VIRTIO_FEATURE (VIRTIO_NET_F_CTRL_VQ)
759  | VIRTIO_FEATURE (VIRTIO_NET_F_MQ)
760  | VIRTIO_FEATURE (VIRTIO_F_NOTIFY_ON_EMPTY)
761  | VIRTIO_FEATURE (VIRTIO_F_ANY_LAYOUT)
762  | VIRTIO_FEATURE (VIRTIO_RING_F_INDIRECT_DESC);
763 
764  if (req_features == 0)
765  {
766  req_features = supported_features;
767  }
768 
769  vif->features = req_features & vif->remote_features & supported_features;
770 
772  {
773  virtio_net_config_t config;
774  virtio_pci_legacy_read_config (vm, vif, &config.mtu,
775  sizeof (config.mtu),
777  mtu));
778  if (config.mtu < 64)
780  }
781 
782  vif->features = virtio_pci_legacy_set_features (vm, vif, vif->features);
783 }
784 
785 void
787 {
789 }
790 
791 int
793 {
794  u8 status = 0;
795 
796  /*
797  * Reset the device
798  */
799  status = virtio_pci_legacy_reset (vm, vif);
800 
801  /*
802  * Set the Acknowledge status bit
803  */
804  virtio_pci_legacy_set_status (vm, vif, VIRTIO_CONFIG_STATUS_ACK);
805 
806  /*
807  * Set the Driver status bit
808  */
809  virtio_pci_legacy_set_status (vm, vif, VIRTIO_CONFIG_STATUS_DRIVER);
810 
811  /*
812  * Read the status and verify it
813  */
814  status = virtio_pci_legacy_get_status (vm, vif);
815  if (!
816  ((status & VIRTIO_CONFIG_STATUS_ACK)
817  && (status & VIRTIO_CONFIG_STATUS_DRIVER)))
818  return -1;
819  vif->status = status;
820 
821  return 0;
822 }
823 
824 clib_error_t *
826 {
827  clib_error_t *error = 0;
828  virtio_main_t *vim = &virtio_main;
829  struct virtio_pci_cap cap;
830  u8 pos, common_cfg = 0, notify_base = 0, dev_cfg = 0, isr = 0, pci_cfg = 0;
832 
833  if ((error = vlib_pci_read_config_u8 (vm, h, PCI_CAPABILITY_LIST, &pos)))
834  {
835  virtio_log_error (vim, vif, "error in reading capabilty list position");
836  clib_error_return (error, "error in reading capabilty list position");
837  }
838  while (pos)
839  {
840  if ((error =
841  vlib_pci_read_write_config (vm, h, VLIB_READ, pos, &cap,
842  sizeof (cap))))
843  {
844  virtio_log_error (vim, vif, "%s [%2x]",
845  "error in reading the capability at", pos);
846  clib_error_return (error,
847  "error in reading the capability at [%2x]", pos);
848  }
849 
850  if (cap.cap_vndr == PCI_CAP_ID_MSIX)
851  {
852  u16 flags, table_size, table_size_mask = 0x07FF;
853 
854  if ((error =
855  vlib_pci_read_write_config (vm, h, VLIB_READ, pos + 2, &flags,
856  sizeof (flags))))
857  clib_error_return (error,
858  "error in reading the capability at [%2x]",
859  pos + 2);
860 
861  table_size = flags & table_size_mask;
862  virtio_log_debug (vim, vif, "flags:0x%x %s 0x%x", flags,
863  "msix interrupt vector table-size", table_size);
864 
865  if (flags & PCI_MSIX_ENABLE)
866  {
867  virtio_log_debug (vim, vif, "msix interrupt enabled");
869  }
870  else
871  {
872  virtio_log_debug (vim, vif, "msix interrupt disabled");
874  }
875  }
876 
877  if (cap.cap_vndr != PCI_CAP_ID_VNDR)
878  {
879  virtio_log_debug (vim, vif, "[%2x] %s %2x ", pos,
880  "skipping non VNDR cap id:", cap.cap_vndr);
881  goto next;
882  }
883 
884  virtio_log_debug (vim, vif,
885  "[%4x] cfg type: %u, bar: %u, offset: %04x, len: %u",
886  pos, cap.cfg_type, cap.bar, cap.offset, cap.length);
887  switch (cap.cfg_type)
888  {
890  common_cfg = 1;
891  break;
893  notify_base = 1;
894  break;
896  dev_cfg = 1;
897  break;
899  isr = 1;
900  break;
902  if (cap.bar == 0)
903  pci_cfg = 1;
904  break;
905  }
906  next:
907  pos = cap.cap_next;
908  }
909 
910  if (common_cfg == 0 || notify_base == 0 || dev_cfg == 0 || isr == 0)
911  {
912  virtio_log_debug (vim, vif, "legacy virtio pci device found");
913  return error;
914  }
915 
916  if (!pci_cfg)
917  clib_error_return (error, "modern virtio pci device found");
918 
919  virtio_log_debug (vim, vif, "transitional virtio pci device found");
920  return error;
921 }
922 
923 static clib_error_t *
926 {
927  clib_error_t *error = 0;
928  virtio_main_t *vim = &virtio_main;
929  u8 status = 0;
930 
931  if ((error = virtio_pci_read_caps (vm, vif)))
932  clib_error_return (error, "Device is not supported");
933 
934  if (virtio_pci_reset_device (vm, vif) < 0)
935  {
936  virtio_log_error (vim, vif, "Failed to reset the device");
937  clib_error_return (error, "Failed to reset the device");
938  }
939  /*
940  * read device features and negotiate (user) requested features
941  */
943  virtio_negotiate_features (vm, vif, args->features);
944 
945  /*
946  * After FEATURE_OK, driver should not accept new feature bits
947  */
948  virtio_pci_legacy_set_status (vm, vif, VIRTIO_CONFIG_STATUS_FEATURES_OK);
949  status = virtio_pci_legacy_get_status (vm, vif);
950  if (!(status & VIRTIO_CONFIG_STATUS_FEATURES_OK))
951  {
952  virtio_log_error (vim, vif,
953  "error encountered: Device doesn't support requested features");
954  clib_error_return (error, "Device doesn't support requested features");
955  }
956  vif->status = status;
957 
958  /*
959  * get or set the mac address
960  */
961  if (virtio_pci_get_mac (vm, vif))
962  {
963  f64 now = vlib_time_now (vm);
964  u32 rnd;
965  rnd = (u32) (now * 1e6);
966  rnd = random_u32 (&rnd);
967 
968  memcpy (vif->mac_addr + 2, &rnd, sizeof (rnd));
969  vif->mac_addr[0] = 2;
970  vif->mac_addr[1] = 0xfe;
971  virtio_pci_set_mac (vm, vif);
972  }
973 
975 
976  /*
977  * Initialize the virtqueues
978  */
979  if ((error = virtio_pci_get_max_virtqueue_pairs (vm, vif)))
980  goto err;
981 
982  for (int i = 0; i < vif->max_queue_pairs; i++)
983  {
984  if ((error = virtio_pci_vring_init (vm, vif, RX_QUEUE (i))))
985  {
986  virtio_log_warning (vim, vif, "%s (%u) %s", "error in rxq-queue",
987  RX_QUEUE (i), "initialization");
988  }
989  else
990  {
991  vif->num_rxqs++;
992  }
993 
994  if ((error = virtio_pci_vring_init (vm, vif, TX_QUEUE (i))))
995  {
996  virtio_log_warning (vim, vif, "%s (%u) %s", "error in txq-queue",
997  TX_QUEUE (i), "initialization");
998  }
999  else
1000  {
1001  vif->num_txqs++;
1002  }
1003  }
1004 
1005  if (vif->features & VIRTIO_FEATURE (VIRTIO_NET_F_CTRL_VQ))
1006  {
1007  if ((error =
1008  virtio_pci_control_vring_init (vm, vif, vif->max_queue_pairs * 2)))
1009  {
1010  virtio_log_warning (vim, vif, "%s (%u) %s",
1011  "error in control-queue",
1012  vif->max_queue_pairs * 2, "initialization");
1013  if (vif->features & VIRTIO_FEATURE (VIRTIO_NET_F_MQ))
1014  vif->features &= ~VIRTIO_FEATURE (VIRTIO_NET_F_MQ);
1015  }
1016  }
1017  else
1018  {
1019  virtio_log_debug (vim, vif, "control queue is not available");
1020  vif->cxq_vring = NULL;
1021  }
1022 
1023  /*
1024  * set the msix interrupts
1025  */
1026  if (vif->msix_enabled == VIRTIO_MSIX_ENABLED)
1027  {
1028  if (virtio_pci_legacy_set_config_irq (vm, vif, 1) ==
1029  VIRTIO_MSI_NO_VECTOR)
1030  virtio_log_warning (vim, vif, "config vector 1 is not set");
1031  if (virtio_pci_legacy_set_queue_irq (vm, vif, 0, 0) ==
1032  VIRTIO_MSI_NO_VECTOR)
1033  virtio_log_warning (vim, vif, "queue vector 0 is not set");
1034  }
1035 
1036  /*
1037  * set the driver status OK
1038  */
1039  virtio_pci_legacy_set_status (vm, vif, VIRTIO_CONFIG_STATUS_DRIVER_OK);
1040  vif->status = virtio_pci_legacy_get_status (vm, vif);
1041 err:
1042  return error;
1043 }
1044 
1045 void
1047 {
1048  vnet_main_t *vnm = vnet_get_main ();
1049  virtio_main_t *vim = &virtio_main;
1050  virtio_if_t *vif;
1052  clib_error_t *error = 0;
1053 
1054  /* *INDENT-OFF* */
1055  pool_foreach (vif, vim->interfaces, ({
1056  if (vif->pci_addr.as_u32 == args->addr)
1057  {
1058  args->rv = VNET_API_ERROR_INVALID_VALUE;
1059  args->error =
1060  clib_error_return (error, "PCI address in use");
1061  vlib_log (VLIB_LOG_LEVEL_ERR, vim->log_default, "%U: %s",
1062  format_vlib_pci_addr, &args->addr,
1063  " PCI address in use");
1064  return;
1065  }
1066  }));
1067  /* *INDENT-ON* */
1068 
1069  pool_get (vim->interfaces, vif);
1070  vif->dev_instance = vif - vim->interfaces;
1071  vif->per_interface_next_index = ~0;
1072  vif->pci_addr.as_u32 = args->addr;
1073 
1074  if ((error =
1075  vlib_pci_device_open (vm, (vlib_pci_addr_t *) & vif->pci_addr,
1076  virtio_pci_device_ids, &h)))
1077  {
1078  pool_put (vim->interfaces, vif);
1079  args->rv = VNET_API_ERROR_INVALID_INTERFACE;
1080  args->error =
1081  clib_error_return (error, "pci-addr %U", format_vlib_pci_addr,
1082  &vif->pci_addr);
1083  vlib_log (VLIB_LOG_LEVEL_ERR, vim->log_default, "%U: %s",
1084  format_vlib_pci_addr, &vif->pci_addr,
1085  "error encountered on pci device open");
1086  return;
1087  }
1088  vif->pci_dev_handle = h;
1089  vlib_pci_set_private_data (vm, h, vif->dev_instance);
1090  vif->numa_node = vlib_pci_get_numa_node (vm, h);
1091 
1092  if ((error = vlib_pci_bus_master_enable (vm, h)))
1093  {
1094  virtio_log_error (vim, vif,
1095  "error encountered on pci bus master enable");
1096  goto error;
1097  }
1098 
1099  if ((error = vlib_pci_io_region (vm, h, 0)))
1100  {
1101  virtio_log_error (vim, vif, "error encountered on pci io region");
1102  goto error;
1103  }
1104 
1106  {
1107  if ((error = vlib_pci_register_msix_handler (vm, h, 0, 1,
1109  {
1110  virtio_log_error (vim, vif,
1111  "error encountered on pci register msix handler 0");
1112  goto error;
1113  }
1114  if ((error = vlib_pci_register_msix_handler (vm, h, 1, 1,
1116  {
1117  virtio_log_error (vim, vif,
1118  "error encountered on pci register msix handler 1");
1119  goto error;
1120  }
1121 
1122  if ((error = vlib_pci_enable_msix_irq (vm, h, 0, 2)))
1123  {
1124  virtio_log_error (vim, vif,
1125  "error encountered on pci enable msix irq");
1126  goto error;
1127  }
1128  vif->support_int_mode = 1;
1129  virtio_log_debug (vim, vif, "device supports msix interrupts");
1130  }
1131  else if (vlib_pci_get_num_msix_interrupts (vm, h) == 1)
1132  {
1133  /*
1134  * if msix table-size is 1, fall back to intX.
1135  */
1136  if ((error =
1138  {
1139  virtio_log_error (vim, vif,
1140  "error encountered on pci register interrupt handler");
1141  goto error;
1142  }
1143  vif->support_int_mode = 1;
1144  virtio_log_debug (vim, vif, "pci register interrupt handler");
1145  }
1146  else
1147  {
1148  /*
1149  * WARN: intX is showing some weird behaviour.
1150  * Please don't use interrupt mode with UIO driver.
1151  */
1152  vif->support_int_mode = 0;
1153  virtio_log_debug (vim, vif, "driver is configured in poll mode only");
1154  }
1155 
1156  if ((error = vlib_pci_intr_enable (vm, h)))
1157  {
1158  virtio_log_error (vim, vif,
1159  "error encountered on pci interrupt enable");
1160  goto error;
1161  }
1162 
1163  if ((error = virtio_pci_device_init (vm, vif, args)))
1164  {
1165  virtio_log_error (vim, vif, "error encountered on device init");
1166  goto error;
1167  }
1168 
1169  vif->type = VIRTIO_IF_TYPE_PCI;
1170  /* create interface */
1172  vif->dev_instance, vif->mac_addr,
1173  &vif->hw_if_index,
1175 
1176  if (error)
1177  {
1178  virtio_log_error (vim, vif,
1179  "error encountered on ethernet register interface");
1180  goto error;
1181  }
1182 
1183  vnet_sw_interface_t *sw = vnet_get_hw_sw_interface (vnm, vif->hw_if_index);
1184  vif->sw_if_index = sw->sw_if_index;
1185  args->sw_if_index = sw->sw_if_index;
1186 
1187  vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, vif->hw_if_index);
1189  vnet_hw_interface_set_input_node (vnm, vif->hw_if_index,
1190  virtio_input_node.index);
1191  u32 i = 0;
1192  vec_foreach_index (i, vif->rxq_vrings)
1193  {
1194  vnet_hw_interface_assign_rx_thread (vnm, vif->hw_if_index, i, ~0);
1196  /* Set default rx mode to POLLING */
1197  vnet_hw_interface_set_rx_mode (vnm, vif->hw_if_index, i,
1199  }
1200  if (virtio_pci_is_link_up (vm, vif) & VIRTIO_NET_S_LINK_UP)
1201  {
1202  vif->flags |= VIRTIO_IF_FLAG_ADMIN_UP;
1203  vnet_hw_interface_set_flags (vnm, vif->hw_if_index,
1205  }
1206  else
1207  vnet_hw_interface_set_flags (vnm, vif->hw_if_index, 0);
1208 
1209  if (vif->features & VIRTIO_FEATURE (VIRTIO_NET_F_CTRL_VQ))
1210  {
1211  if (vif->features & VIRTIO_FEATURE (VIRTIO_NET_F_CTRL_GUEST_OFFLOADS) &&
1212  args->gso_enabled)
1213  {
1214  if (virtio_pci_enable_gso (vm, vif))
1215  {
1216  virtio_log_warning (vim, vif, "gso is not enabled");
1217  }
1218  else
1219  {
1220  vif->gso_enabled = 1;
1222  vnm->interface_main.gso_interface_count++;
1223  }
1224  }
1225  if (vif->features & VIRTIO_FEATURE (VIRTIO_NET_F_MQ))
1226  {
1227  if (virtio_pci_enable_multiqueue (vm, vif, vif->max_queue_pairs))
1228  virtio_log_warning (vim, vif, "multiqueue is not set");
1229  }
1230  }
1231  return;
1232 
1233 
1234 error:
1235  virtio_pci_delete_if (vm, vif);
1236  args->rv = VNET_API_ERROR_INVALID_INTERFACE;
1237  args->error = error;
1238 }
1239 
1240 int
1242 {
1243  vnet_main_t *vnm = vnet_get_main ();
1244  virtio_main_t *vim = &virtio_main;
1245  u32 i = 0;
1246 
1247  if (vif->type != VIRTIO_IF_TYPE_PCI)
1248  return VNET_API_ERROR_INVALID_INTERFACE;
1249 
1251 
1252  for (i = 0; i < vif->max_queue_pairs; i++)
1253  {
1254  virtio_pci_legacy_del_queue (vm, vif, RX_QUEUE (i));
1255  virtio_pci_legacy_del_queue (vm, vif, TX_QUEUE (i));
1256  }
1257 
1258  if (vif->features & VIRTIO_FEATURE (VIRTIO_NET_F_CTRL_VQ))
1259  virtio_pci_legacy_del_queue (vm, vif, vif->max_queue_pairs * 2);
1260 
1261  virtio_pci_legacy_reset (vm, vif);
1262 
1263  if (vif->gso_enabled)
1265 
1266  if (vif->hw_if_index)
1267  {
1269  vec_foreach_index (i, vif->rxq_vrings)
1270  {
1272  }
1274  }
1275 
1277 
1278  vec_foreach_index (i, vif->rxq_vrings)
1279  {
1280  virtio_vring_t *vring = vec_elt_at_index (vif->rxq_vrings, i);
1281  if (vring->kick_fd != -1)
1282  close (vring->kick_fd);
1283  if (vring->used)
1284  {
1285  virtio_free_rx_buffers (vm, vring);
1286  }
1287  vec_free (vring->buffers);
1288  vlib_physmem_free (vm, vring->desc);
1289  }
1290 
1291  vec_foreach_index (i, vif->txq_vrings)
1292  {
1293  virtio_vring_t *vring = vec_elt_at_index (vif->txq_vrings, i);
1294  if (vring->kick_fd != -1)
1295  close (vring->kick_fd);
1296  if (vring->used)
1297  {
1298  virtio_free_used_desc (vm, vring);
1299  }
1300  vec_free (vring->buffers);
1301  clib_spinlock_free (&vring->lockp);
1302  vlib_physmem_free (vm, vring->desc);
1303  }
1304 
1305  if (vif->cxq_vring != NULL)
1306  {
1307  u16 last = vif->cxq_vring->last_used_idx;
1308  u16 n_left = vif->cxq_vring->used->idx - last;
1309  while (n_left)
1310  {
1311  last++;
1312  n_left--;
1313  }
1314 
1315  vif->cxq_vring->last_used_idx = last;
1316  vlib_physmem_free (vm, vif->cxq_vring->desc);
1317  }
1318 
1319  vec_free (vif->rxq_vrings);
1320  vec_free (vif->txq_vrings);
1321  vec_free (vif->cxq_vring);
1322 
1323  if (vif->fd != -1)
1324  vif->fd = -1;
1325  if (vif->tap_fd != -1)
1326  vif->tap_fd = -1;
1327  clib_error_free (vif->error);
1328  memset (vif, 0, sizeof (*vif));
1329  pool_put (vim->interfaces, vif);
1330 
1331  return 0;
1332 }
1333 
1334 /*
1335  * fd.io coding-style-patch-verification: ON
1336  *
1337  * Local Variables:
1338  * eval: (c-set-style "gnu")
1339  * End:
1340  */
static u8 virtio_pci_legacy_get_isr(vlib_main_t *vm, virtio_if_t *vif)
Definition: pci.c:161
struct vring_used * used
Definition: virtio.h:105
static uword vlib_buffer_get_current_pa(vlib_main_t *vm, vlib_buffer_t *b)
Definition: buffer_funcs.h:427
#define vec_foreach_index(var, v)
Iterate over vector indices.
static clib_error_t * vlib_pci_intr_enable(vlib_main_t *vm, vlib_pci_dev_handle_t h)
Definition: pci.h:239
vlib_node_registration_t virtio_input_node
(constructor) VLIB_REGISTER_NODE (virtio_input_node)
Definition: node.c:397
void virtio_set_net_hdr_size(virtio_if_t *vif)
Definition: virtio.c:257
virtio_if_t * interfaces
Definition: virtio.h:192
u32 flags
Definition: vhost_user.h:141
clib_error_t * virtio_pci_vring_init(vlib_main_t *vm, virtio_if_t *vif, u16 queue_num)
Definition: pci.c:662
static int virtio_pci_legacy_setup_queue(vlib_main_t *vm, virtio_if_t *vif, u16 queue_id, void *p)
Definition: pci.c:181
static u32 virtio_pci_flag_change(vnet_main_t *vnm, vnet_hw_interface_t *hw, u32 flags)
Definition: pci.c:241
static void virtio_pci_legacy_del_queue(vlib_main_t *vm, virtio_if_t *vif, u16 queue_id)
Definition: pci.c:198
static clib_error_t * vlib_pci_bus_master_enable(vlib_main_t *vm, vlib_pci_dev_handle_t h)
Definition: pci.h:271
static void * vlib_physmem_alloc_aligned_on_numa(vlib_main_t *vm, uword n_bytes, uword alignment, u32 numa_node)
Definition: physmem_funcs.h:63
#define VIRTIO_PCI_CAP_ISR_CFG
Definition: pci.h:125
vl_api_mac_address_t mac
Definition: l2.api:490
a
Definition: bitmap.h:538
static void vlib_buffer_free(vlib_main_t *vm, u32 *buffers, u32 n_buffers)
Free buffers Frees the entire buffer chain for each buffer.
Definition: buffer_funcs.h:865
u16 max_virtqueue_pairs
Definition: virtio.h:95
void ethernet_delete_interface(vnet_main_t *vnm, u32 hw_if_index)
Definition: interface.c:324
vnet_main_t * vnet_get_main(void)
Definition: misc.c:46
static u16 virtio_pci_legacy_get_queue_num(vlib_main_t *vm, virtio_if_t *vif, u16 queue_id)
Definition: pci.c:169
vnet_interface_main_t interface_main
Definition: vnet.h:56
#define VIRTIO_MSI_QUEUE_VECTOR
Definition: pci.h:36
i16 current_data
signed offset in data[], pre_data[] that we are currently processing.
Definition: buffer.h:110
unsigned long u64
Definition: types.h:89
int gso_enabled
Definition: virtio.h:182
void vlib_pci_device_close(vlib_main_t *vm, vlib_pci_dev_handle_t h)
Definition: pci.c:1275
#define VIRTIO_PCI_STATUS
Definition: pci.h:31
#define CLIB_MEMORY_STORE_BARRIER()
Definition: clib.h:118
#define VIRTIO_PCI_QUEUE_PFN
Definition: pci.h:27
#define NULL
Definition: clib.h:58
static u8 virtio_pci_legacy_get_status(vlib_main_t *vm, virtio_if_t *vif)
Definition: pci.c:138
static f64 vlib_time_now(vlib_main_t *vm)
Definition: main.h:258
u32 vlib_pci_get_num_msix_interrupts(vlib_main_t *vm, vlib_pci_dev_handle_t h)
Definition: pci.c:177
static clib_error_t * vlib_physmem_last_error(struct vlib_main_t *vm)
static u32 virtio_pci_legacy_set_features(vlib_main_t *vm, virtio_if_t *vif, u64 features)
Definition: pci.c:122
static clib_error_t * vlib_pci_intr_disable(vlib_main_t *vm, vlib_pci_dev_handle_t h)
Definition: pci.h:255
static vnet_hw_interface_t * vnet_get_hw_interface(vnet_main_t *vnm, u32 hw_if_index)
vnet_device_class_t virtio_device_class
static clib_error_t * virtio_pci_device_init(vlib_main_t *vm, virtio_if_t *vif, virtio_pci_create_if_args_t *args)
Definition: pci.c:924
vl_api_address_t src
Definition: gre.api:51
static heap_elt_t * last(heap_header_t *h)
Definition: heap.c:53
void virtio_vring_set_numa_node(vlib_main_t *vm, virtio_if_t *vif, u32 idx)
Definition: virtio.c:241
clib_memset(h->entries, 0, sizeof(h->entries[0])*entries)
#define STRUCT_OFFSET_OF(t, f)
Definition: clib.h:65
clib_error_t * vlib_pci_enable_msix_irq(vlib_main_t *vm, vlib_pci_dev_handle_t h, u16 start, u16 count)
Definition: pci.c:883
#define VIRTIO_MSI_CONFIG_VECTOR
Definition: pci.h:35
u8 data[128]
Definition: ipsec.api:249
#define vec_validate_aligned(V, I, A)
Make sure vector is long enough for given index (no header, specified alignment)
Definition: vec.h:450
#define pool_get(P, E)
Allocate an object E from a pool P (unspecified alignment).
Definition: pool.h:236
vhost_vring_addr_t addr
Definition: vhost_user.h:147
#define VIRTIO_PCI_HOST_FEATURES
Definition: pci.h:25
static int virtio_pci_enable_gso(vlib_main_t *vm, virtio_if_t *vif)
Definition: pci.c:560
#define VIRTIO_NET_CTRL_GUEST_OFFLOADS
Definition: pci.h:117
static void virtio_pci_set_mac(vlib_main_t *vm, virtio_if_t *vif)
Definition: pci.c:273
static int virtio_pci_send_ctrl_msg(vlib_main_t *vm, virtio_if_t *vif, struct virtio_ctrl_msg *data, u32 len)
Definition: pci.c:463
unsigned char u8
Definition: types.h:56
void virtio_pci_create_if(vlib_main_t *vm, virtio_pci_create_if_args_t *args)
Definition: pci.c:1046
double f64
Definition: types.h:142
static void clib_spinlock_free(clib_spinlock_t *p)
Definition: lock.h:68
static vnet_sw_interface_t * vnet_get_hw_sw_interface(vnet_main_t *vnm, u32 hw_if_index)
#define clib_memcpy(d, s, n)
Definition: string.h:180
static void virtio_pci_irq_handler(vlib_main_t *vm, vlib_pci_dev_handle_t h)
Definition: pci.c:339
clib_error_t * vlib_pci_read_write_config(vlib_main_t *vm, vlib_pci_dev_handle_t h, vlib_read_or_write_t read_or_write, uword address, void *data, u32 n_bytes)
Definition: pci.c:1017
u32 msix_enabled
Definition: virtio.h:154
#define VIRTIO_PCI_CAP_COMMON_CFG
Definition: pci.h:121
struct vring_avail * avail
Definition: virtio.h:106
u64 features
Definition: virtio.h:163
int virtio_pci_reset_device(vlib_main_t *vm, virtio_if_t *vif)
Definition: pci.c:792
u32 hw_if_index
Definition: virtio.h:140
u8 * format_ethernet_address(u8 *s, va_list *args)
Definition: format.c:44
static u8 virtio_pci_queue_size_valid(u16 qsz)
Definition: pci.c:599
#define pool_foreach(VAR, POOL, BODY)
Iterate through pool.
Definition: pool.h:493
static u16 virtio_pci_legacy_set_queue_irq(vlib_main_t *vm, virtio_if_t *vif, u16 vec, u16 queue_id)
Definition: pci.c:228
#define PCI_DEVICE_ID_VIRTIO_NIC
Definition: pci.c:29
#define TX_QUEUE_ACCESS(X)
Definition: virtio.h:80
void device_status(vlib_main_t *vm, virtio_if_t *vif)
Definition: pci.c:362
static_always_inline void vnet_device_input_set_interrupt_pending(vnet_main_t *vnm, u32 hw_if_index, u16 queue_id)
Definition: devices.h:136
static void virtio_negotiate_features(vlib_main_t *vm, virtio_if_t *vif, u64 req_features)
Definition: pci.c:737
vnet_hw_interface_flags_t flags
Definition: interface.h:505
void virtio_free_rx_buffers(vlib_main_t *vm, virtio_vring_t *vring)
Definition: virtio.c:155
#define vec_elt_at_index(v, i)
Get vector value at index i checking that i is in bounds.
clib_spinlock_t lockp
Definition: virtio.h:107
#define clib_error_return(e, args...)
Definition: error.h:99
#define VIRTIO_PCI_CAP_NOTIFY_CFG
Definition: pci.h:123
unsigned int u32
Definition: types.h:88
static void virtio_pci_legacy_read_config(vlib_main_t *vm, virtio_if_t *vif, void *dst, int len, u32 addr)
Definition: pci.c:53
#define VIRTIO_NET_CTRL_GUEST_OFFLOADS_SET
Definition: pci.h:118
#define virtio_log_debug(vim, vif, f,...)
Definition: pci.h:135
u16 queue_id
Definition: virtio.h:114
static void clib_spinlock_init(clib_spinlock_t *p)
Definition: lock.h:61
u16 num_txqs
Definition: virtio.h:170
u32 pci_dev_handle
Definition: virtio.h:159
#define TX_QUEUE(X)
Definition: virtio.h:78
static void virtio_pci_legacy_set_status(vlib_main_t *vm, virtio_if_t *vif, u8 status)
Definition: pci.c:146
#define pool_elt_at_index(p, i)
Returns pointer to element at given index.
Definition: pool.h:514
uword size
#define VIRTIO_PCI_GUEST_FEATURES
Definition: pci.h:26
#define VIRTIO_NET_F_MTU
Definition: pci.h:105
int tap_fd
Definition: virtio.h:158
static void virtio_pci_irq_1_handler(vlib_main_t *vm, vlib_pci_dev_handle_t h, u16 line)
Definition: pci.c:318
void vlib_pci_set_private_data(vlib_main_t *vm, vlib_pci_dev_handle_t h, uword private_data)
Definition: pci.c:155
unsigned short u16
Definition: types.h:57
#define VIRTIO_PCI_ISR_CONFIG
Definition: pci.h:48
struct virtio_net_ctrl_hdr ctrl
Definition: pci.c:457
static void * vlib_buffer_get_current(vlib_buffer_t *b)
Get pointer to current data to process.
Definition: buffer.h:229
#define pool_put(P, E)
Free an object E in pool P.
Definition: pool.h:286
clib_error_t * vlib_pci_register_msix_handler(vlib_main_t *vm, vlib_pci_dev_handle_t h, u32 start, u32 count, pci_msix_handler_function_t *msix_handler)
Definition: pci.c:826
static u64 virtio_pci_legacy_get_features(vlib_main_t *vm, virtio_if_t *vif)
Definition: pci.c:113
#define PCI_MSIX_ENABLE
Definition: pci.c:37
uword vlib_pci_get_private_data(vlib_main_t *vm, vlib_pci_dev_handle_t h)
Definition: pci.c:148
u32 vlib_pci_dev_handle_t
Definition: pci.h:97
static u16 virtio_pci_legacy_set_config_irq(vlib_main_t *vm, virtio_if_t *vif, u16 vec)
Definition: pci.c:217
void virtio_pci_read_device_feature(vlib_main_t *vm, virtio_if_t *vif)
Definition: pci.c:786
vl_api_address_t dst
Definition: gre.api:52
clib_error_t * virtio_pci_read_caps(vlib_main_t *vm, virtio_if_t *vif)
Definition: pci.c:825
static clib_error_t * virtio_pci_get_max_virtqueue_pairs(vlib_main_t *vm, virtio_if_t *vif)
Definition: pci.c:248
#define PCI_CAPABILITY_LIST
Definition: pci.c:33
u8 status
Definition: virtio.h:171
u8 len
Definition: ip_types.api:90
u16 desc_next
Definition: virtio.h:109
u8 * format_vlib_pci_addr(u8 *s, va_list *va)
Definition: pci.c:141
#define VIRTIO_PCI_VRING_ALIGN
Definition: pci.h:133
u16 num_rxqs
Definition: virtio.h:169
virtio_vring_t * rxq_vrings
Definition: virtio.h:161
u32 vlib_pci_get_numa_node(vlib_main_t *vm, vlib_pci_dev_handle_t h)
Definition: pci.c:170
u16 last_used_idx
Definition: virtio.h:118
#define VIRTIO_PCI_QUEUE_NOTIFY
Definition: pci.h:30
vlib_main_t * vm
Definition: buffer.c:312
#define VIRTIO_PCI_ISR_INTR
Definition: pci.h:46
static void vlib_physmem_free(vlib_main_t *vm, void *p)
Definition: physmem_funcs.h:89
#define VIRTIO_RING_FLAG_MASK_INT
Definition: virtio.h:99
#define vec_free(V)
Free vector&#39;s memory (no header).
Definition: vec.h:341
#define clib_warning(format, args...)
Definition: error.h:59
static u16 virtio_pci_is_link_up(vlib_main_t *vm, virtio_if_t *vif)
Definition: pci.c:292
void debug_device_config_space(vlib_main_t *vm, virtio_if_t *vif)
Definition: pci.c:390
void vlib_log(vlib_log_level_t level, vlib_log_class_t class, char *fmt,...)
Definition: log.c:87
u8 mac_addr[6]
Definition: virtio.h:172
static void virtio_pci_legacy_write_config(vlib_main_t *vm, virtio_if_t *vif, void *src, int len, u32 addr)
Definition: pci.c:83
u32 flags
Definition: virtio.h:137
clib_error_t * virtio_pci_control_vring_init(vlib_main_t *vm, virtio_if_t *vif, u16 queue_num)
Definition: pci.c:609
clib_error_t * error
Definition: virtio.h:166
static uword round_pow2(uword x, uword pow2)
Definition: clib.h:241
void virtio_pci_legacy_notify_queue(vlib_main_t *vm, virtio_if_t *vif, u16 queue_id)
Definition: pci.c:208
int virtio_pci_delete_if(vlib_main_t *vm, virtio_if_t *vif)
Definition: pci.c:1241
u32 numa_node
Definition: virtio.h:142
virtio_if_type_t type
Definition: virtio.h:144
void virtio_free_used_desc(vlib_main_t *vm, virtio_vring_t *vring)
Definition: virtio.c:192
#define PCI_DEVICE_ID_VIRTIO_NIC_MODERN
Definition: pci.c:31
#define PCI_CONFIG_SIZE(vif)
Definition: pci.c:39
#define ASSERT(truth)
void vnet_hw_interface_assign_rx_thread(vnet_main_t *vnm, u32 hw_if_index, u16 queue_id, uword thread_index)
Definition: devices.c:139
u64 remote_features
Definition: virtio.h:163
#define VIRTIO_FEATURE(X)
Definition: virtio.h:76
clib_error_t * vlib_pci_register_intx_handler(vlib_main_t *vm, vlib_pci_dev_handle_t h, pci_intx_handler_function_t *intx_handler)
Definition: pci.c:775
virtio_main_t virtio_main
Definition: virtio.c:37
static uword is_pow2(uword x)
Definition: clib.h:235
#define VIRTIO_PCI_QUEUE_SEL
Definition: pci.h:29
Definition: defs.h:56
static u64 vlib_physmem_get_pa(vlib_main_t *vm, void *mem)
#define RX_QUEUE_ACCESS(X)
Definition: virtio.h:81
virtio_net_ctrl_ack status
Definition: pci.c:458
#define VIRTIO_PCI_CAP_DEVICE_CFG
Definition: pci.h:127
#define VIRTIO_NET_F_CTRL_GUEST_OFFLOADS
Definition: pci.h:104
#define PCI_CAP_ID_VNDR
Definition: pci.c:34
clib_error_t * ethernet_register_interface(vnet_main_t *vnm, u32 dev_class_index, u32 dev_instance, const u8 *address, u32 *hw_if_index_return, ethernet_flag_change_function_t flag_change)
Definition: interface.c:278
clib_error_t * vnet_hw_interface_set_flags(vnet_main_t *vnm, u32 hw_if_index, vnet_hw_interface_flags_t flags)
Definition: interface.c:501
VLIB buffer representation.
Definition: buffer.h:102
u8 data[1024]
Definition: pci.c:459
u64 uword
Definition: types.h:112
int fd
Definition: virtio.h:153
static void virtio_pci_irq_0_handler(vlib_main_t *vm, vlib_pci_dev_handle_t h, u16 line)
Definition: pci.c:306
#define virtio_log_warning(vim, vif, f,...)
Definition: pci.h:142
#define virtio_log_error(vim, vif, f,...)
Definition: pci.h:149
#define clib_error_free(e)
Definition: error.h:86
clib_error_t * vlib_pci_io_region(vlib_main_t *vm, vlib_pci_dev_handle_t h, u32 resource)
Definition: pci.c:1160
u32 * buffers
Definition: virtio.h:117
int vnet_hw_interface_unassign_rx_thread(vnet_main_t *vnm, u32 hw_if_index, u16 queue_id)
Definition: devices.c:188
static u8 virtio_pci_legacy_reset(vlib_main_t *vm, virtio_if_t *vif)
Definition: pci.c:154
virtio_vring_t * cxq_vring
Definition: virtio.h:184
static u32 random_u32(u32 *seed)
32-bit random number generator
Definition: random.h:69
static vlib_thread_main_t * vlib_get_thread_main()
Definition: global_funcs.h:32
struct vring_desc * desc
Definition: virtio.h:104
#define CLIB_MEMORY_BARRIER()
Definition: clib.h:115
int vnet_hw_interface_set_rx_mode(vnet_main_t *vnm, u32 hw_if_index, u16 queue_id, vnet_hw_interface_rx_mode mode)
Definition: devices.c:253
#define VIRTIO_PCI_CAP_PCI_CFG
Definition: pci.h:129
u16 max_queue_pairs
Definition: virtio.h:168
#define CLIB_CACHE_LINE_BYTES
Definition: cache.h:59
#define PCI_VENDOR_ID_VIRTIO
Definition: pci.c:28
static u32 vlib_buffer_alloc(vlib_main_t *vm, u32 *buffers, u32 n_buffers)
Allocate buffers into supplied array.
Definition: buffer_funcs.h:612
#define PCI_CAP_ID_MSIX
Definition: pci.c:35
void vlib_cli_output(vlib_main_t *vm, char *fmt,...)
Definition: cli.c:768
clib_error_t * vlib_pci_device_open(vlib_main_t *vm, vlib_pci_addr_t *addr, pci_device_id_t ids[], vlib_pci_dev_handle_t *handle)
Definition: pci.c:1215
#define RX_QUEUE(X)
Definition: virtio.h:79
static u32 virtio_pci_get_mac(vlib_main_t *vm, virtio_if_t *vif)
Definition: pci.c:280
static vlib_buffer_t * vlib_get_buffer(vlib_main_t *vm, u32 buffer_index)
Translate buffer index into buffer pointer.
Definition: buffer_funcs.h:85
static void vnet_hw_interface_set_input_node(vnet_main_t *vnm, u32 hw_if_index, u32 node_index)
Definition: devices.h:79
#define VIRTIO_NET_CTRL_MQ
Definition: vhost_user.h:40
#define VIRTIO_PCI_QUEUE_NUM
Definition: pci.h:28
u16 vendor_id
Definition: pci.h:127
static int virtio_pci_enable_multiqueue(vlib_main_t *vm, virtio_if_t *vif, u16 num_queues)
Definition: pci.c:581
#define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET
Definition: vhost_user.h:41
u16 desc_in_use
Definition: virtio.h:108
virtio_vring_t * txq_vrings
Definition: virtio.h:162
#define VIRTIO_PCI_QUEUE_ADDR_SHIFT
Definition: pci.h:131
static_always_inline void virtio_kick(vlib_main_t *vm, virtio_vring_t *vring, virtio_if_t *vif)
Definition: virtio.h:217
#define VIRTIO_PCI_ISR
Definition: pci.h:32