FD.io VPP  v17.01.1-3-gc6833f8
Vector Packet Processing
init.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2015 Cisco and/or its affiliates.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at:
6  *
7  * http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 #include <vnet/vnet.h>
16 #include <vppinfra/vec.h>
17 #include <vppinfra/error.h>
18 #include <vppinfra/format.h>
19 #include <vppinfra/bitmap.h>
20 
21 #include <vnet/ethernet/ethernet.h>
22 #include <vnet/devices/dpdk/dpdk.h>
23 #include <vlib/unix/physmem.h>
24 #include <vlib/pci/pci.h>
25 
26 #include <stdio.h>
27 #include <stdlib.h>
28 #include <unistd.h>
29 #include <sys/stat.h>
30 #include <sys/mount.h>
31 #include <string.h>
32 #include <fcntl.h>
33 
34 #include "dpdk_priv.h"
35 
37 
38 /* force linker to link functions used by vlib and declared weak */
40  &rte_pktmbuf_init,
41  &rte_pktmbuf_pool_init,
42 };
43 
44 #define LINK_STATE_ELOGS 0
45 
46 #define DEFAULT_HUGE_DIR "/run/vpp/hugepages"
47 #define VPP_RUN_DIR "/run/vpp"
48 
49 /* Port configuration, mildly modified Intel app values */
50 
51 static struct rte_eth_conf port_conf_template = {
52  .rxmode = {
53  .split_hdr_size = 0,
54  .header_split = 0, /**< Header Split disabled */
55  .hw_ip_checksum = 0, /**< IP checksum offload disabled */
56  .hw_vlan_filter = 0, /**< VLAN filtering disabled */
57  .hw_strip_crc = 0, /**< CRC stripped by hardware */
58  },
59  .txmode = {
60  .mq_mode = ETH_MQ_TX_NONE,
61  },
62 };
63 
66 {
67  vlib_main_t *vm = vlib_get_main ();
69  int rv;
70  int j;
71 
72  ASSERT (os_get_cpu_number () == 0);
73 
75  {
77  rte_eth_dev_stop (xd->device_index);
78  }
79 
80  rv = rte_eth_dev_configure (xd->device_index, xd->rx_q_used,
81  xd->tx_q_used, &xd->port_conf);
82 
83  if (rv < 0)
84  return clib_error_return (0, "rte_eth_dev_configure[%d]: err %d",
85  xd->device_index, rv);
86 
87  /* Set up one TX-queue per worker thread */
88  for (j = 0; j < xd->tx_q_used; j++)
89  {
90  rv = rte_eth_tx_queue_setup (xd->device_index, j, xd->nb_tx_desc,
91  xd->cpu_socket, &xd->tx_conf);
92 
93  /* retry with any other CPU socket */
94  if (rv < 0)
95  rv = rte_eth_tx_queue_setup (xd->device_index, j, xd->nb_tx_desc,
96  SOCKET_ID_ANY, &xd->tx_conf);
97  if (rv < 0)
98  break;
99  }
100 
101  if (rv < 0)
102  return clib_error_return (0, "rte_eth_tx_queue_setup[%d]: err %d",
103  xd->device_index, rv);
104 
105  for (j = 0; j < xd->rx_q_used; j++)
106  {
107 
108  rv = rte_eth_rx_queue_setup (xd->device_index, j, xd->nb_rx_desc,
109  xd->cpu_socket, 0,
110  bm->
111  pktmbuf_pools[xd->cpu_socket_id_by_queue
112  [j]]);
113 
114  /* retry with any other CPU socket */
115  if (rv < 0)
116  rv = rte_eth_rx_queue_setup (xd->device_index, j, xd->nb_rx_desc,
117  SOCKET_ID_ANY, 0,
118  bm->
119  pktmbuf_pools[xd->cpu_socket_id_by_queue
120  [j]]);
121  if (rv < 0)
122  return clib_error_return (0, "rte_eth_rx_queue_setup[%d]: err %d",
123  xd->device_index, rv);
124  }
125 
127  {
128  int rv;
129  rv = rte_eth_dev_start (xd->device_index);
130  if (rv < 0)
131  clib_warning ("rte_eth_dev_start %d returned %d",
132  xd->device_index, rv);
133  }
134  return 0;
135 }
136 
137 static u32
139 {
140  dpdk_main_t *dm = &dpdk_main;
142  u32 old = 0;
143 
145  {
146  old = (xd->flags & DPDK_DEVICE_FLAG_PROMISC) != 0;
147 
150  else
152 
154  {
156  rte_eth_promiscuous_enable (xd->device_index);
157  else
158  rte_eth_promiscuous_disable (xd->device_index);
159  }
160  }
161  else if (ETHERNET_INTERFACE_FLAG_CONFIG_MTU (flags))
162  {
163  /*
164  * DAW-FIXME: The Cisco VIC firmware does not provide an api for a
165  * driver to dynamically change the mtu. If/when the
166  * VIC firmware gets fixed, then this should be removed.
167  */
168  if (xd->pmd == VNET_DPDK_PMD_ENIC)
169  {
170  struct rte_eth_dev_info dev_info;
171 
172  /*
173  * Restore mtu to what has been set by CIMC in the firmware cfg.
174  */
175  rte_eth_dev_info_get (xd->device_index, &dev_info);
176  hi->max_packet_bytes = dev_info.max_rx_pktlen;
177 
179  "Cisco VIC mtu can only be changed "
180  "using CIMC then rebooting the server!");
181  }
182  else
183  {
184  int rv;
185 
186  xd->port_conf.rxmode.max_rx_pkt_len = hi->max_packet_bytes;
187 
189  rte_eth_dev_stop (xd->device_index);
190 
191  rv = rte_eth_dev_configure
192  (xd->device_index, xd->rx_q_used, xd->tx_q_used, &xd->port_conf);
193 
194  if (rv < 0)
196  "rte_eth_dev_configure[%d]: err %d",
197  xd->device_index, rv);
198 
199  rte_eth_dev_set_mtu (xd->device_index, hi->max_packet_bytes);
200 
202  {
203  int rv = rte_eth_dev_start (xd->device_index);
204  if (rv < 0)
205  clib_warning ("rte_eth_dev_start %d returned %d",
206  xd->device_index, rv);
207  }
208  }
209  }
210  return old;
211 }
212 
213 void
215 {
216  int q;
217  vec_validate (xd->lockp, xd->tx_q_used - 1);
218  for (q = 0; q < xd->tx_q_used; q++)
219  {
222  memset ((void *) xd->lockp[q], 0, CLIB_CACHE_LINE_BYTES);
223  }
224 }
225 
226 void
228 {
229  int q;
230 
231  for (q = 0; q < vec_len (xd->lockp); q++)
232  clib_mem_free ((void *) xd->lockp[q]);
233  vec_free (xd->lockp);
234  xd->lockp = 0;
235 }
236 
237 static clib_error_t *
239 {
240  u32 nports;
241  u32 nb_desc = 0;
242  int i;
243  clib_error_t *error;
244  vlib_main_t *vm = vlib_get_main ();
248  dpdk_device_t *xd;
249  vlib_pci_addr_t last_pci_addr;
250  u32 last_pci_addr_port = 0;
251  vlib_thread_registration_t *tr, *tr_hqos;
252  uword *p, *p_hqos;
253 
254  u32 next_cpu = 0, next_hqos_cpu = 0;
255  u8 af_packet_port_id = 0;
256  last_pci_addr.as_u32 = ~0;
257 
258  dm->input_cpu_first_index = 0;
259  dm->input_cpu_count = 1;
260 
261  /* find out which cpus will be used for input */
262  p = hash_get_mem (tm->thread_registrations_by_name, "workers");
263  tr = p ? (vlib_thread_registration_t *) p[0] : 0;
264 
265  if (tr && tr->count > 0)
266  {
268  dm->input_cpu_count = tr->count;
269  }
270 
273 
276 
277  dm->hqos_cpu_first_index = 0;
278  dm->hqos_cpu_count = 0;
279 
280  /* find out which cpus will be used for I/O TX */
281  p_hqos = hash_get_mem (tm->thread_registrations_by_name, "hqos-threads");
282  tr_hqos = p_hqos ? (vlib_thread_registration_t *) p_hqos[0] : 0;
283 
284  if (tr_hqos && tr_hqos->count > 0)
285  {
286  dm->hqos_cpu_first_index = tr_hqos->first_index;
287  dm->hqos_cpu_count = tr_hqos->count;
288  }
289 
292 
295 
296 #ifdef NETMAP
297  if (rte_netmap_probe () < 0)
298  return clib_error_return (0, "rte netmap probe failed");
299 #endif
300 
301  nports = rte_eth_dev_count ();
302  if (nports < 1)
303  {
304  clib_warning ("DPDK drivers found no ports...");
305  }
306 
307  if (CLIB_DEBUG > 0)
308  clib_warning ("DPDK drivers found %d ports...", nports);
309 
310  /*
311  * All buffers are all allocated from the same rte_mempool.
312  * Thus they all have the same number of data bytes.
313  */
317  "dpdk rx");
318 
319  if (dm->conf->enable_tcp_udp_checksum)
322 
323  for (i = 0; i < nports; i++)
324  {
325  u8 addr[6];
326  u8 vlan_strip = 0;
327  int j;
328  struct rte_eth_dev_info dev_info;
329  clib_error_t *rv;
330  struct rte_eth_link l;
331  dpdk_device_config_t *devconf = 0;
332  vlib_pci_addr_t pci_addr;
333  uword *p = 0;
334 
335  rte_eth_dev_info_get (i, &dev_info);
336  if (dev_info.pci_dev) /* bonded interface has no pci info */
337  {
338  pci_addr.domain = dev_info.pci_dev->addr.domain;
339  pci_addr.bus = dev_info.pci_dev->addr.bus;
340  pci_addr.slot = dev_info.pci_dev->addr.devid;
341  pci_addr.function = dev_info.pci_dev->addr.function;
342  p =
344  pci_addr.as_u32);
345  }
346 
347  if (p)
348  devconf = pool_elt_at_index (dm->conf->dev_confs, p[0]);
349  else
350  devconf = &dm->conf->default_devconf;
351 
352  /* Create vnet interface */
356  xd->cpu_socket = (i8) rte_eth_dev_socket_id (i);
357 
358  /* Handle interface naming for devices with multiple ports sharing same PCI ID */
359  if (dev_info.pci_dev)
360  {
361  struct rte_eth_dev_info di = { 0 };
362  rte_eth_dev_info_get (i + 1, &di);
363  if (di.pci_dev && pci_addr.as_u32 != last_pci_addr.as_u32 &&
364  memcmp (&dev_info.pci_dev->addr, &di.pci_dev->addr,
365  sizeof (struct rte_pci_addr)) == 0)
366  {
367  xd->interface_name_suffix = format (0, "0");
368  last_pci_addr.as_u32 = pci_addr.as_u32;
369  last_pci_addr_port = i;
370  }
371  else if (pci_addr.as_u32 == last_pci_addr.as_u32)
372  {
374  format (0, "%u", i - last_pci_addr_port);
375  }
376  else
377  {
378  last_pci_addr.as_u32 = ~0;
379  }
380  }
381  else
382  last_pci_addr.as_u32 = ~0;
383 
384  clib_memcpy (&xd->tx_conf, &dev_info.default_txconf,
385  sizeof (struct rte_eth_txconf));
386  if (dm->conf->no_multi_seg)
387  {
388  xd->tx_conf.txq_flags |= ETH_TXQ_FLAGS_NOMULTSEGS;
389  port_conf_template.rxmode.jumbo_frame = 0;
390  }
391  else
392  {
393  xd->tx_conf.txq_flags &= ~ETH_TXQ_FLAGS_NOMULTSEGS;
394  port_conf_template.rxmode.jumbo_frame = 1;
396  }
397 
399  sizeof (struct rte_eth_conf));
400 
401  xd->tx_q_used = clib_min (dev_info.max_tx_queues, tm->n_vlib_mains);
402 
403  if (devconf->num_tx_queues > 0
404  && devconf->num_tx_queues < xd->tx_q_used)
405  xd->tx_q_used = clib_min (xd->tx_q_used, devconf->num_tx_queues);
406 
407  if (devconf->num_rx_queues > 1 && dm->use_rss == 0)
408  {
409  dm->use_rss = 1;
410  }
411 
412  if (devconf->num_rx_queues > 1
413  && dev_info.max_rx_queues >= devconf->num_rx_queues)
414  {
415  xd->rx_q_used = devconf->num_rx_queues;
416  xd->port_conf.rxmode.mq_mode = ETH_MQ_RX_RSS;
417  if (devconf->rss_fn == 0)
418  xd->port_conf.rx_adv_conf.rss_conf.rss_hf =
419  ETH_RSS_IP | ETH_RSS_UDP | ETH_RSS_TCP;
420  else
421  xd->port_conf.rx_adv_conf.rss_conf.rss_hf = devconf->rss_fn;
422  }
423  else
424  xd->rx_q_used = 1;
425 
427 
428  /* workaround for drivers not setting driver_name */
429  if ((!dev_info.driver_name) && (dev_info.pci_dev))
430 #if RTE_VERSION < RTE_VERSION_NUM(16, 11, 0, 0)
431  dev_info.driver_name = dev_info.pci_dev->driver->name;
432 #else
433  dev_info.driver_name = dev_info.pci_dev->driver->driver.name;
434 #endif
435  ASSERT (dev_info.driver_name);
436 
437  if (!xd->pmd)
438  {
439 
440 
441 #define _(s,f) else if (dev_info.driver_name && \
442  !strcmp(dev_info.driver_name, s)) \
443  xd->pmd = VNET_DPDK_PMD_##f;
444  if (0)
445  ;
447 #undef _
448  else
450 
451 
452  switch (xd->pmd)
453  {
454  /* 1G adapters */
455  case VNET_DPDK_PMD_E1000EM:
456  case VNET_DPDK_PMD_IGB:
457  case VNET_DPDK_PMD_IGBVF:
459  break;
460 
461  /* 10G adapters */
462  case VNET_DPDK_PMD_IXGBE:
463  case VNET_DPDK_PMD_IXGBEVF:
464  case VNET_DPDK_PMD_THUNDERX:
468  break;
469  case VNET_DPDK_PMD_DPAA2:
471  break;
472 
473  /* Cisco VIC */
474  case VNET_DPDK_PMD_ENIC:
475  rte_eth_link_get_nowait (i, &l);
478  if (l.link_speed == 40000)
479  {
482  }
483  else
484  {
487  }
488  break;
489 
490  /* Intel Fortville */
491  case VNET_DPDK_PMD_I40E:
492  case VNET_DPDK_PMD_I40EVF:
497 
498  switch (dev_info.pci_dev->id.device_id)
499  {
503  break;
504  case I40E_DEV_ID_QSFP_A:
505  case I40E_DEV_ID_QSFP_B:
506  case I40E_DEV_ID_QSFP_C:
508  break;
509  case I40E_DEV_ID_VF:
510  rte_eth_link_get_nowait (i, &l);
511  xd->port_type = l.link_speed == 10000 ?
513  break;
514  default:
516  }
517  break;
518 
519  case VNET_DPDK_PMD_CXGBE:
520  switch (dev_info.pci_dev->id.device_id)
521  {
522  case 0x540d: /* T580-CR */
523  case 0x5410: /* T580-LP-cr */
527  break;
528  case 0x5403: /* T540-CR */
532  break;
533  default:
537  }
538  break;
539 
540  /* Intel Red Rock Canyon */
541  case VNET_DPDK_PMD_FM10K:
545  break;
546 
547  /* virtio */
548  case VNET_DPDK_PMD_VIRTIO:
552  break;
553 
554  /* vmxnet3 */
555  case VNET_DPDK_PMD_VMXNET3:
557  xd->tx_conf.txq_flags |= ETH_TXQ_FLAGS_NOMULTSEGS;
558  break;
559 
560  case VNET_DPDK_PMD_AF_PACKET:
562  xd->af_packet_port_id = af_packet_port_id++;
563  break;
564 
565  case VNET_DPDK_PMD_BOND:
568  break;
569 
570  default:
572  }
573 
574  if (devconf->num_rx_desc)
575  xd->nb_rx_desc = devconf->num_rx_desc;
576 
577  if (devconf->num_tx_desc)
578  xd->nb_tx_desc = devconf->num_tx_desc;
579  }
580 
581  /*
582  * Ensure default mtu is not > the mtu read from the hardware.
583  * Otherwise rte_eth_dev_configure() will fail and the port will
584  * not be available.
585  */
586  if (ETHERNET_MAX_PACKET_BYTES > dev_info.max_rx_pktlen)
587  {
588  /*
589  * This device does not support the platforms's max frame
590  * size. Use it's advertised mru instead.
591  */
592  xd->port_conf.rxmode.max_rx_pkt_len = dev_info.max_rx_pktlen;
593  }
594  else
595  {
596  xd->port_conf.rxmode.max_rx_pkt_len = ETHERNET_MAX_PACKET_BYTES;
597 
598  /*
599  * Some platforms do not account for Ethernet FCS (4 bytes) in
600  * MTU calculations. To interop with them increase mru but only
601  * if the device's settings can support it.
602  */
603  if ((dev_info.max_rx_pktlen >= (ETHERNET_MAX_PACKET_BYTES + 4)) &&
604  xd->port_conf.rxmode.hw_strip_crc)
605  {
606  /*
607  * Allow additional 4 bytes (for Ethernet FCS). These bytes are
608  * stripped by h/w and so will not consume any buffer memory.
609  */
610  xd->port_conf.rxmode.max_rx_pkt_len += 4;
611  }
612  }
613 
614  if (xd->pmd == VNET_DPDK_PMD_AF_PACKET)
615  {
616  f64 now = vlib_time_now (vm);
617  u32 rnd;
618  rnd = (u32) (now * 1e6);
619  rnd = random_u32 (&rnd);
620  clib_memcpy (addr + 2, &rnd, sizeof (rnd));
621  addr[0] = 2;
622  addr[1] = 0xfe;
623  }
624  else
625  rte_eth_macaddr_get (i, (struct ether_addr *) addr);
626 
627  if (xd->tx_q_used < tm->n_vlib_mains)
629 
630  xd->device_index = xd - dm->devices;
631  ASSERT (i == xd->device_index);
632  xd->per_interface_next_index = ~0;
633 
634  /* assign interface to input thread */
636  int q;
637 
638  if (devconf->workers)
639  {
640  int i;
641  q = 0;
642  /* *INDENT-OFF* */
643  clib_bitmap_foreach (i, devconf->workers, ({
644  int cpu = dm->input_cpu_first_index + i;
645  unsigned lcore = vlib_worker_threads[cpu].lcore_id;
646  vec_validate(xd->cpu_socket_id_by_queue, q);
647  xd->cpu_socket_id_by_queue[q] = rte_lcore_to_socket_id(lcore);
648  vec_add2(dm->devices_by_cpu[cpu], dq, 1);
649  dq->device = xd->device_index;
650  dq->queue_id = q++;
651  }));
652  /* *INDENT-ON* */
653  }
654  else
655  for (q = 0; q < xd->rx_q_used; q++)
656  {
657  int cpu = dm->input_cpu_first_index + next_cpu;
658  unsigned lcore = vlib_worker_threads[cpu].lcore_id;
659 
660  /*
661  * numa node for worker thread handling this queue
662  * needed for taking buffers from the right mempool
663  */
665  xd->cpu_socket_id_by_queue[q] = rte_lcore_to_socket_id (lcore);
666 
667  /*
668  * construct vector of (device,queue) pairs for each worker thread
669  */
670  vec_add2 (dm->devices_by_cpu[cpu], dq, 1);
671  dq->device = xd->device_index;
672  dq->queue_id = q;
673 
674  next_cpu++;
675  if (next_cpu == dm->input_cpu_count)
676  next_cpu = 0;
677  }
678 
679 
680  if (devconf->hqos_enabled)
681  {
683 
684  if (devconf->hqos.hqos_thread_valid)
685  {
686  int cpu = dm->hqos_cpu_first_index + devconf->hqos.hqos_thread;
687 
688  if (devconf->hqos.hqos_thread >= dm->hqos_cpu_count)
689  return clib_error_return (0, "invalid HQoS thread index");
690 
691  vec_add2 (dm->devices_by_hqos_cpu[cpu], dq, 1);
692  dq->device = xd->device_index;
693  dq->queue_id = 0;
694  }
695  else
696  {
697  int cpu = dm->hqos_cpu_first_index + next_hqos_cpu;
698 
699  if (dm->hqos_cpu_count == 0)
700  return clib_error_return (0, "no HQoS threads available");
701 
702  vec_add2 (dm->devices_by_hqos_cpu[cpu], dq, 1);
703  dq->device = xd->device_index;
704  dq->queue_id = 0;
705 
706  next_hqos_cpu++;
707  if (next_hqos_cpu == dm->hqos_cpu_count)
708  next_hqos_cpu = 0;
709 
710  devconf->hqos.hqos_thread_valid = 1;
711  devconf->hqos.hqos_thread = cpu;
712  }
713  }
714 
717  for (j = 0; j < tm->n_vlib_mains; j++)
718  {
721  vec_reset_length (xd->tx_vectors[j]);
722  }
723 
726  for (j = 0; j < xd->rx_q_used; j++)
727  {
730  vec_reset_length (xd->rx_vectors[j]);
731  }
732 
735 
736  rv = dpdk_port_setup (dm, xd);
737 
738  if (rv)
739  return rv;
740 
741  if (devconf->hqos_enabled)
742  {
743  rv = dpdk_port_setup_hqos (xd, &devconf->hqos);
744  if (rv)
745  return rv;
746  }
747 
748  /* count the number of descriptors used for this device */
749  nb_desc += xd->nb_rx_desc + xd->nb_tx_desc * xd->tx_q_used;
750 
752  (dm->vnet_main, dpdk_device_class.index, xd->device_index,
753  /* ethernet address */ addr,
755  if (error)
756  return error;
757 
759  xd->vlib_sw_if_index = sw->sw_if_index;
761 
762  /*
763  * DAW-FIXME: The Cisco VIC firmware does not provide an api for a
764  * driver to dynamically change the mtu. If/when the
765  * VIC firmware gets fixed, then this should be removed.
766  */
767  if (xd->pmd == VNET_DPDK_PMD_ENIC)
768  {
769  /*
770  * Initialize mtu to what has been set by CIMC in the firmware cfg.
771  */
772  hi->max_packet_bytes = dev_info.max_rx_pktlen;
773  if (devconf->vlan_strip_offload != DPDK_DEVICE_VLAN_STRIP_OFF)
774  vlan_strip = 1; /* remove vlan tag from VIC port by default */
775  else
776  clib_warning ("VLAN strip disabled for interface\n");
777  }
778  else if (devconf->vlan_strip_offload == DPDK_DEVICE_VLAN_STRIP_ON)
779  vlan_strip = 1;
780 
781  if (vlan_strip)
782  {
783  int vlan_off;
784  vlan_off = rte_eth_dev_get_vlan_offload (xd->device_index);
785  vlan_off |= ETH_VLAN_STRIP_OFFLOAD;
786  xd->port_conf.rxmode.hw_vlan_strip = vlan_off;
787  if (rte_eth_dev_set_vlan_offload (xd->device_index, vlan_off) == 0)
788  clib_warning ("VLAN strip enabled for interface\n");
789  else
790  clib_warning ("VLAN strip cannot be supported by interface\n");
791  }
792 
794  xd->port_conf.rxmode.max_rx_pkt_len - sizeof (ethernet_header_t);
795 
796  rte_eth_dev_set_mtu (xd->device_index, hi->max_packet_bytes);
797  }
798 
799  if (nb_desc > dm->conf->num_mbufs)
800  clib_warning ("%d mbufs allocated but total rx/tx ring size is %d\n",
801  dm->conf->num_mbufs, nb_desc);
802 
803  return 0;
804 }
805 
806 static void
808 {
809  vlib_pci_main_t *pm = &pci_main;
810  clib_error_t *error;
812  u8 *pci_addr = 0;
813  int num_whitelisted = vec_len (conf->dev_confs);
814 
815  /* *INDENT-OFF* */
816  pool_foreach (d, pm->pci_devs, ({
817  dpdk_device_config_t * devconf = 0;
818  vec_reset_length (pci_addr);
819  pci_addr = format (pci_addr, "%U%c", format_vlib_pci_addr, &d->bus_address, 0);
820 
821  if (d->device_class != PCI_CLASS_NETWORK_ETHERNET)
822  continue;
823 
824  if (num_whitelisted)
825  {
826  uword * p = hash_get (conf->device_config_index_by_pci_addr, d->bus_address.as_u32);
827 
828  if (!p)
829  continue;
830 
831  devconf = pool_elt_at_index (conf->dev_confs, p[0]);
832  }
833 
834  /* virtio */
835  if (d->vendor_id == 0x1af4 && d->device_id == 0x1000)
836  ;
837  /* vmxnet3 */
838  else if (d->vendor_id == 0x15ad && d->device_id == 0x07b0)
839  ;
840  /* all Intel devices */
841  else if (d->vendor_id == 0x8086)
842  ;
843  /* Cisco VIC */
844  else if (d->vendor_id == 0x1137 && d->device_id == 0x0043)
845  ;
846  /* Chelsio T4/T5 */
847  else if (d->vendor_id == 0x1425 && (d->device_id & 0xe000) == 0x4000)
848  ;
849  else
850  {
851  clib_warning ("Unsupported Ethernet PCI device 0x%04x:0x%04x found "
852  "at PCI address %s\n", (u16) d->vendor_id, (u16) d->device_id,
853  pci_addr);
854  continue;
855  }
856 
857  error = vlib_pci_bind_to_uio (d, (char *) conf->uio_driver_name);
858 
859  if (error)
860  {
861  if (devconf == 0)
862  {
863  pool_get (conf->dev_confs, devconf);
865  devconf - conf->dev_confs);
866  devconf->pci_addr.as_u32 = d->bus_address.as_u32;
867  }
868  devconf->is_blacklisted = 1;
869  clib_error_report (error);
870  }
871  }));
872  /* *INDENT-ON* */
873  vec_free (pci_addr);
874 }
875 
876 static clib_error_t *
877 dpdk_device_config (dpdk_config_main_t * conf, vlib_pci_addr_t pci_addr,
878  unformat_input_t * input, u8 is_default)
879 {
880  clib_error_t *error = 0;
881  uword *p;
882  dpdk_device_config_t *devconf;
883  unformat_input_t sub_input;
884 
885  if (is_default)
886  {
887  devconf = &conf->default_devconf;
888  }
889  else
890  {
891  p = hash_get (conf->device_config_index_by_pci_addr, pci_addr.as_u32);
892 
893  if (!p)
894  {
895  pool_get (conf->dev_confs, devconf);
896  hash_set (conf->device_config_index_by_pci_addr, pci_addr.as_u32,
897  devconf - conf->dev_confs);
898  }
899  else
900  return clib_error_return (0,
901  "duplicate configuration for PCI address %U",
902  format_vlib_pci_addr, &pci_addr);
903  }
904 
905  devconf->pci_addr.as_u32 = pci_addr.as_u32;
906  devconf->hqos_enabled = 0;
908 
909  if (!input)
910  return 0;
911 
914  {
915  if (unformat (input, "num-rx-queues %u", &devconf->num_rx_queues))
916  ;
917  else if (unformat (input, "num-tx-queues %u", &devconf->num_tx_queues))
918  ;
919  else if (unformat (input, "num-rx-desc %u", &devconf->num_rx_desc))
920  ;
921  else if (unformat (input, "num-tx-desc %u", &devconf->num_tx_desc))
922  ;
923  else if (unformat (input, "workers %U", unformat_bitmap_list,
924  &devconf->workers))
925  ;
926  else
927  if (unformat
928  (input, "rss %U", unformat_vlib_cli_sub_input, &sub_input))
929  {
930  error = unformat_rss_fn (&sub_input, &devconf->rss_fn);
931  if (error)
932  break;
933  }
934  else if (unformat (input, "vlan-strip-offload off"))
936  else if (unformat (input, "vlan-strip-offload on"))
938  else
939  if (unformat
940  (input, "hqos %U", unformat_vlib_cli_sub_input, &sub_input))
941  {
942  devconf->hqos_enabled = 1;
943  error = unformat_hqos (&sub_input, &devconf->hqos);
944  if (error)
945  break;
946  }
947  else if (unformat (input, "hqos"))
948  {
949  devconf->hqos_enabled = 1;
950  }
951  else
952  {
953  error = clib_error_return (0, "unknown input `%U'",
954  format_unformat_error, input);
955  break;
956  }
957  }
958 
959  if (error)
960  return error;
961 
962  if (devconf->workers && devconf->num_rx_queues == 0)
963  devconf->num_rx_queues = clib_bitmap_count_set_bits (devconf->workers);
964  else if (devconf->workers &&
965  clib_bitmap_count_set_bits (devconf->workers) !=
966  devconf->num_rx_queues)
967  error =
969  "%U: number of worker threadds must be "
970  "equal to number of rx queues", format_vlib_pci_addr,
971  &pci_addr);
972 
973  return error;
974 }
975 
976 static clib_error_t *
978 {
979  clib_error_t *error = 0;
980  dpdk_main_t *dm = &dpdk_main;
983  dpdk_device_config_t *devconf;
984  vlib_pci_addr_t pci_addr;
985  unformat_input_t sub_input;
986  u8 *s, *tmp = 0;
987  u8 *rte_cmd = 0, *ethname = 0;
988  u32 log_level;
989  int ret, i;
990  int num_whitelisted = 0;
991  u8 no_pci = 0;
992  u8 no_huge = 0;
993  u8 huge_dir = 0;
994  u8 file_prefix = 0;
995  u8 *socket_mem = 0;
996 
997  conf->device_config_index_by_pci_addr = hash_create (0, sizeof (uword));
998 
1000  {
1001  /* Prime the pump */
1002  if (unformat (input, "no-hugetlb"))
1003  {
1004  vec_add1 (conf->eal_init_args, (u8 *) "no-huge");
1005  no_huge = 1;
1006  }
1007 
1008  else if (unformat (input, "enable-tcp-udp-checksum"))
1009  conf->enable_tcp_udp_checksum = 1;
1010 
1011  else if (unformat (input, "decimal-interface-names"))
1013 
1014  else if (unformat (input, "no-multi-seg"))
1015  conf->no_multi_seg = 1;
1016 
1017  else if (unformat (input, "dev default %U", unformat_vlib_cli_sub_input,
1018  &sub_input))
1019  {
1020  error =
1021  dpdk_device_config (conf, (vlib_pci_addr_t) (u32) ~ 1, &sub_input,
1022  1);
1023 
1024  if (error)
1025  return error;
1026  }
1027  else
1028  if (unformat
1029  (input, "dev %U %U", unformat_vlib_pci_addr, &pci_addr,
1030  unformat_vlib_cli_sub_input, &sub_input))
1031  {
1032  error = dpdk_device_config (conf, pci_addr, &sub_input, 0);
1033 
1034  if (error)
1035  return error;
1036 
1037  num_whitelisted++;
1038  }
1039  else if (unformat (input, "dev %U", unformat_vlib_pci_addr, &pci_addr))
1040  {
1041  error = dpdk_device_config (conf, pci_addr, 0, 0);
1042 
1043  if (error)
1044  return error;
1045 
1046  num_whitelisted++;
1047  }
1048  else if (unformat (input, "num-mbufs %d", &conf->num_mbufs))
1049  ;
1050  else if (unformat (input, "kni %d", &conf->num_kni))
1051  ;
1052  else if (unformat (input, "uio-driver %s", &conf->uio_driver_name))
1053  ;
1054  else if (unformat (input, "socket-mem %s", &socket_mem))
1055  ;
1056  else if (unformat (input, "no-pci"))
1057  {
1058  no_pci = 1;
1059  tmp = format (0, "--no-pci%c", 0);
1060  vec_add1 (conf->eal_init_args, tmp);
1061  }
1062  else if (unformat (input, "poll-sleep %d", &dm->poll_sleep))
1063  ;
1064 
1065 #define _(a) \
1066  else if (unformat(input, #a)) \
1067  { \
1068  tmp = format (0, "--%s%c", #a, 0); \
1069  vec_add1 (conf->eal_init_args, tmp); \
1070  }
1072 #undef _
1073 #define _(a) \
1074  else if (unformat(input, #a " %s", &s)) \
1075  { \
1076  if (!strncmp(#a, "huge-dir", 8)) \
1077  huge_dir = 1; \
1078  else if (!strncmp(#a, "file-prefix", 11)) \
1079  file_prefix = 1; \
1080  tmp = format (0, "--%s%c", #a, 0); \
1081  vec_add1 (conf->eal_init_args, tmp); \
1082  vec_add1 (s, 0); \
1083  vec_add1 (conf->eal_init_args, s); \
1084  }
1086 #undef _
1087 #define _(a,b) \
1088  else if (unformat(input, #a " %s", &s)) \
1089  { \
1090  tmp = format (0, "-%s%c", #b, 0); \
1091  vec_add1 (conf->eal_init_args, tmp); \
1092  vec_add1 (s, 0); \
1093  vec_add1 (conf->eal_init_args, s); \
1094  }
1096 #undef _
1097 #define _(a,b) \
1098  else if (unformat(input, #a " %s", &s)) \
1099  { \
1100  tmp = format (0, "-%s%c", #b, 0); \
1101  vec_add1 (conf->eal_init_args, tmp); \
1102  vec_add1 (s, 0); \
1103  vec_add1 (conf->eal_init_args, s); \
1104  conf->a##_set_manually = 1; \
1105  }
1107 #undef _
1108  else if (unformat (input, "default"))
1109  ;
1110 
1111  else if (unformat_skip_white_space (input))
1112  ;
1113  else
1114  {
1115  error = clib_error_return (0, "unknown input `%U'",
1116  format_unformat_error, input);
1117  goto done;
1118  }
1119  }
1120 
1121  if (!conf->uio_driver_name)
1122  conf->uio_driver_name = format (0, "igb_uio%c", 0);
1123 
1124  /*
1125  * Use 1G huge pages if available.
1126  */
1127  if (!no_huge && !huge_dir)
1128  {
1129  u32 x, *mem_by_socket = 0;
1130  uword c = 0;
1131  u8 use_1g = 1;
1132  u8 use_2m = 1;
1133  u8 less_than_1g = 1;
1134  int rv;
1135 
1136  umount (DEFAULT_HUGE_DIR);
1137 
1138  /* Process "socket-mem" parameter value */
1139  if (vec_len (socket_mem))
1140  {
1141  unformat_input_t in;
1142  unformat_init_vector (&in, socket_mem);
1144  {
1145  if (unformat (&in, "%u,", &x))
1146  ;
1147  else if (unformat (&in, "%u", &x))
1148  ;
1149  else if (unformat (&in, ","))
1150  x = 0;
1151  else
1152  break;
1153 
1154  vec_add1 (mem_by_socket, x);
1155 
1156  if (x > 1023)
1157  less_than_1g = 0;
1158  }
1159  /* Note: unformat_free vec_frees(in.buffer), aka socket_mem... */
1160  unformat_free (&in);
1161  socket_mem = 0;
1162  }
1163  else
1164  {
1165  /* *INDENT-OFF* */
1167  {
1168  vec_validate(mem_by_socket, c);
1169  mem_by_socket[c] = 256; /* default per-socket mem */
1170  }
1171  ));
1172  /* *INDENT-ON* */
1173  }
1174 
1175  /* check if available enough 1GB pages for each socket */
1176  /* *INDENT-OFF* */
1178  {
1179  int pages_avail, page_size, mem;
1180 
1181  vec_validate(mem_by_socket, c);
1182  mem = mem_by_socket[c];
1183 
1184  page_size = 1024;
1185  pages_avail = vlib_sysfs_get_free_hugepages(c, page_size * 1024);
1186 
1187  if (pages_avail < 0 || page_size * pages_avail < mem)
1188  use_1g = 0;
1189 
1190  page_size = 2;
1191  pages_avail = vlib_sysfs_get_free_hugepages(c, page_size * 1024);
1192 
1193  if (pages_avail < 0 || page_size * pages_avail < mem)
1194  use_2m = 0;
1195  }));
1196  /* *INDENT-ON* */
1197 
1198  if (mem_by_socket == 0)
1199  {
1200  error = clib_error_return (0, "mem_by_socket NULL");
1201  goto done;
1202  }
1203  _vec_len (mem_by_socket) = c + 1;
1204 
1205  /* regenerate socket_mem string */
1206  vec_foreach_index (x, mem_by_socket)
1207  socket_mem = format (socket_mem, "%s%u",
1208  socket_mem ? "," : "", mem_by_socket[x]);
1209  socket_mem = format (socket_mem, "%c", 0);
1210 
1211  vec_free (mem_by_socket);
1212 
1213  rv = mkdir (VPP_RUN_DIR, 0755);
1214  if (rv && errno != EEXIST)
1215  {
1216  error = clib_error_return (0, "mkdir '%s' failed errno %d",
1217  VPP_RUN_DIR, errno);
1218  goto done;
1219  }
1220 
1221  rv = mkdir (DEFAULT_HUGE_DIR, 0755);
1222  if (rv && errno != EEXIST)
1223  {
1224  error = clib_error_return (0, "mkdir '%s' failed errno %d",
1225  DEFAULT_HUGE_DIR, errno);
1226  goto done;
1227  }
1228 
1229  if (use_1g && !(less_than_1g && use_2m))
1230  {
1231  rv =
1232  mount ("none", DEFAULT_HUGE_DIR, "hugetlbfs", 0, "pagesize=1G");
1233  }
1234  else if (use_2m)
1235  {
1236  rv = mount ("none", DEFAULT_HUGE_DIR, "hugetlbfs", 0, NULL);
1237  }
1238  else
1239  {
1240  return clib_error_return (0, "not enough free huge pages");
1241  }
1242 
1243  if (rv)
1244  {
1245  error = clib_error_return (0, "mount failed %d", errno);
1246  goto done;
1247  }
1248 
1249  tmp = format (0, "--huge-dir%c", 0);
1250  vec_add1 (conf->eal_init_args, tmp);
1251  tmp = format (0, "%s%c", DEFAULT_HUGE_DIR, 0);
1252  vec_add1 (conf->eal_init_args, tmp);
1253  if (!file_prefix)
1254  {
1255  tmp = format (0, "--file-prefix%c", 0);
1256  vec_add1 (conf->eal_init_args, tmp);
1257  tmp = format (0, "vpp%c", 0);
1258  vec_add1 (conf->eal_init_args, tmp);
1259  }
1260  }
1261 
1262  vec_free (rte_cmd);
1263  vec_free (ethname);
1264 
1265  if (error)
1266  return error;
1267 
1268  /* I'll bet that -c and -n must be the first and second args... */
1269  if (!conf->coremask_set_manually)
1270  {
1272  uword *coremask = 0;
1273  int i;
1274 
1275  /* main thread core */
1276  coremask = clib_bitmap_set (coremask, tm->main_lcore, 1);
1277 
1278  for (i = 0; i < vec_len (tm->registrations); i++)
1279  {
1280  tr = tm->registrations[i];
1281  coremask = clib_bitmap_or (coremask, tr->coremask);
1282  }
1283 
1284  vec_insert (conf->eal_init_args, 2, 1);
1285  conf->eal_init_args[1] = (u8 *) "-c";
1286  tmp = format (0, "%U%c", format_bitmap_hex, coremask, 0);
1287  conf->eal_init_args[2] = tmp;
1288  clib_bitmap_free (coremask);
1289  }
1290 
1291  if (!conf->nchannels_set_manually)
1292  {
1293  vec_insert (conf->eal_init_args, 2, 3);
1294  conf->eal_init_args[3] = (u8 *) "-n";
1295  tmp = format (0, "%d", conf->nchannels);
1296  conf->eal_init_args[4] = tmp;
1297  }
1298 
1299  if (no_pci == 0 && geteuid () == 0)
1300  dpdk_bind_devices_to_uio (conf);
1301 
1302 #define _(x) \
1303  if (devconf->x == 0 && conf->default_devconf.x > 0) \
1304  devconf->x = conf->default_devconf.x ;
1305 
1306  /* *INDENT-OFF* */
1307  pool_foreach (devconf, conf->dev_confs, ({
1308 
1309  /* default per-device config items */
1310  foreach_dpdk_device_config_item
1311 
1312  /* add DPDK EAL whitelist/blacklist entry */
1313  if (num_whitelisted > 0 && devconf->is_blacklisted == 0)
1314  {
1315  tmp = format (0, "-w%c", 0);
1316  vec_add1 (conf->eal_init_args, tmp);
1317  tmp = format (0, "%U%c", format_vlib_pci_addr, &devconf->pci_addr, 0);
1318  vec_add1 (conf->eal_init_args, tmp);
1319  }
1320  else if (num_whitelisted == 0 && devconf->is_blacklisted != 0)
1321  {
1322  tmp = format (0, "-b%c", 0);
1323  vec_add1 (conf->eal_init_args, tmp);
1324  tmp = format (0, "%U%c", format_vlib_pci_addr, &devconf->pci_addr, 0);
1325  vec_add1 (conf->eal_init_args, tmp);
1326  }
1327  }));
1328  /* *INDENT-ON* */
1329 
1330 #undef _
1331 
1332  /* set master-lcore */
1333  tmp = format (0, "--master-lcore%c", 0);
1334  vec_add1 (conf->eal_init_args, tmp);
1335  tmp = format (0, "%u%c", tm->main_lcore, 0);
1336  vec_add1 (conf->eal_init_args, tmp);
1337 
1338  /* set socket-mem */
1339  tmp = format (0, "--socket-mem%c", 0);
1340  vec_add1 (conf->eal_init_args, tmp);
1341  tmp = format (0, "%s%c", socket_mem, 0);
1342  vec_add1 (conf->eal_init_args, tmp);
1343 
1344  /* NULL terminate the "argv" vector, in case of stupidity */
1345  vec_add1 (conf->eal_init_args, 0);
1346  _vec_len (conf->eal_init_args) -= 1;
1347 
1348  /* Set up DPDK eal and packet mbuf pool early. */
1349 
1350  log_level = (CLIB_DEBUG > 0) ? RTE_LOG_DEBUG : RTE_LOG_NOTICE;
1351 
1352  rte_set_log_level (log_level);
1353 
1354  vm = vlib_get_main ();
1355 
1356  /* make copy of args as rte_eal_init tends to mess up with arg array */
1357  for (i = 1; i < vec_len (conf->eal_init_args); i++)
1358  conf->eal_init_args_str = format (conf->eal_init_args_str, "%s ",
1359  conf->eal_init_args[i]);
1360 
1361  ret =
1362  rte_eal_init (vec_len (conf->eal_init_args),
1363  (char **) conf->eal_init_args);
1364 
1365  /* lazy umount hugepages */
1366  umount2 (DEFAULT_HUGE_DIR, MNT_DETACH);
1367 
1368  if (ret < 0)
1369  return clib_error_return (0, "rte_eal_init returned %d", ret);
1370 
1371  /* Dump the physical memory layout prior to creating the mbuf_pool */
1372  fprintf (stdout, "DPDK physical memory layout:\n");
1373  rte_dump_physmem_layout (stdout);
1374 
1375  /* main thread 1st */
1376  error = vlib_buffer_pool_create (vm, conf->num_mbufs, rte_socket_id ());
1377  if (error)
1378  return error;
1379 
1380  for (i = 0; i < RTE_MAX_LCORE; i++)
1381  {
1382  error = vlib_buffer_pool_create (vm, conf->num_mbufs,
1383  rte_lcore_to_socket_id (i));
1384  if (error)
1385  return error;
1386  }
1387 
1388 done:
1389  return error;
1390 }
1391 
1393 
1394 void
1396 {
1397  vnet_main_t *vnm = vnet_get_main ();
1398  struct rte_eth_link prev_link = xd->link;
1399  u32 hw_flags = 0;
1400  u8 hw_flags_chg = 0;
1401 
1402  /* only update link state for PMD interfaces */
1403  if ((xd->flags & DPDK_DEVICE_FLAG_PMD) == 0)
1404  return;
1405 
1406  xd->time_last_link_update = now ? now : xd->time_last_link_update;
1407  memset (&xd->link, 0, sizeof (xd->link));
1408  rte_eth_link_get_nowait (xd->device_index, &xd->link);
1409 
1410  if (LINK_STATE_ELOGS)
1411  {
1412  vlib_main_t *vm = vlib_get_main ();
1413  ELOG_TYPE_DECLARE (e) =
1414  {
1415  .format =
1416  "update-link-state: sw_if_index %d, admin_up %d,"
1417  "old link_state %d new link_state %d",.format_args = "i4i1i1i1",};
1418 
1419  struct
1420  {
1421  u32 sw_if_index;
1422  u8 admin_up;
1423  u8 old_link_state;
1424  u8 new_link_state;
1425  } *ed;
1426  ed = ELOG_DATA (&vm->elog_main, e);
1427  ed->sw_if_index = xd->vlib_sw_if_index;
1428  ed->admin_up = (xd->flags & DPDK_DEVICE_FLAG_ADMIN_UP) != 0;
1429  ed->old_link_state = (u8)
1431  ed->new_link_state = (u8) xd->link.link_status;
1432  }
1433 
1434  if ((xd->flags & DPDK_DEVICE_FLAG_ADMIN_UP) &&
1435  ((xd->link.link_status != 0) ^
1437  {
1438  hw_flags_chg = 1;
1439  hw_flags |= (xd->link.link_status ? VNET_HW_INTERFACE_FLAG_LINK_UP : 0);
1440  }
1441 
1442  if (hw_flags_chg || (xd->link.link_duplex != prev_link.link_duplex))
1443  {
1444  hw_flags_chg = 1;
1445  switch (xd->link.link_duplex)
1446  {
1447  case ETH_LINK_HALF_DUPLEX:
1449  break;
1450  case ETH_LINK_FULL_DUPLEX:
1452  break;
1453  default:
1454  break;
1455  }
1456  }
1457  if (hw_flags_chg || (xd->link.link_speed != prev_link.link_speed))
1458  {
1459  hw_flags_chg = 1;
1460  switch (xd->link.link_speed)
1461  {
1462  case ETH_SPEED_NUM_10M:
1464  break;
1465  case ETH_SPEED_NUM_100M:
1467  break;
1468  case ETH_SPEED_NUM_1G:
1469  hw_flags |= VNET_HW_INTERFACE_FLAG_SPEED_1G;
1470  break;
1471  case ETH_SPEED_NUM_10G:
1473  break;
1474  case ETH_SPEED_NUM_40G:
1476  break;
1477  case 0:
1478  break;
1479  default:
1480  clib_warning ("unknown link speed %d", xd->link.link_speed);
1481  break;
1482  }
1483  }
1484  if (hw_flags_chg)
1485  {
1486  if (LINK_STATE_ELOGS)
1487  {
1488  vlib_main_t *vm = vlib_get_main ();
1489 
1490  ELOG_TYPE_DECLARE (e) =
1491  {
1492  .format =
1493  "update-link-state: sw_if_index %d, new flags %d",.format_args
1494  = "i4i4",};
1495 
1496  struct
1497  {
1498  u32 sw_if_index;
1499  u32 flags;
1500  } *ed;
1501  ed = ELOG_DATA (&vm->elog_main, e);
1502  ed->sw_if_index = xd->vlib_sw_if_index;
1503  ed->flags = hw_flags;
1504  }
1505  vnet_hw_interface_set_flags (vnm, xd->vlib_hw_if_index, hw_flags);
1506  }
1507 }
1508 
1509 static uword
1511 {
1512  clib_error_t *error;
1513  vnet_main_t *vnm = vnet_get_main ();
1514  dpdk_main_t *dm = &dpdk_main;
1516  dpdk_device_t *xd;
1518  int i;
1519 
1520  error = dpdk_lib_init (dm);
1521 
1522  /*
1523  * Turn on the input node if we found some devices to drive
1524  * and we're not running worker threads or i/o threads
1525  */
1526 
1527  if (error == 0 && vec_len (dm->devices) > 0)
1528  {
1529  if (tm->n_vlib_mains == 1)
1531  VLIB_NODE_STATE_POLLING);
1532  else
1533  for (i = 0; i < tm->n_vlib_mains; i++)
1534  if (vec_len (dm->devices_by_cpu[i]) > 0)
1536  VLIB_NODE_STATE_POLLING);
1537  }
1538 
1539  if (error)
1540  clib_error_report (error);
1541 
1542  tm->worker_thread_release = 1;
1543 
1544  f64 now = vlib_time_now (vm);
1545  vec_foreach (xd, dm->devices)
1546  {
1547  dpdk_update_link_state (xd, now);
1548  }
1549 
1550  {
1551  /*
1552  * Extra set up for bond interfaces:
1553  * 1. Setup MACs for bond interfaces and their slave links which was set
1554  * in dpdk_port_setup() but needs to be done again here to take effect.
1555  * 2. Set up info for bond interface related CLI support.
1556  */
1557  int nports = rte_eth_dev_count ();
1558  if (nports > 0)
1559  {
1560  for (i = 0; i < nports; i++)
1561  {
1562  struct rte_eth_dev_info dev_info;
1563  rte_eth_dev_info_get (i, &dev_info);
1564  if (!dev_info.driver_name)
1565 #if RTE_VERSION < RTE_VERSION_NUM(16, 11, 0, 0)
1566  dev_info.driver_name = dev_info.pci_dev->driver->name;
1567 #else
1568  dev_info.driver_name = dev_info.pci_dev->driver->driver.name;
1569 #endif
1570  ASSERT (dev_info.driver_name);
1571  if (strncmp (dev_info.driver_name, "rte_bond_pmd", 12) == 0)
1572  {
1573  u8 addr[6];
1574  u8 slink[16];
1575  int nlink = rte_eth_bond_slaves_get (i, slink, 16);
1576  if (nlink > 0)
1577  {
1578  vnet_hw_interface_t *bhi;
1579  ethernet_interface_t *bei;
1580  int rv;
1581 
1582  /* Get MAC of 1st slave link */
1583  rte_eth_macaddr_get (slink[0],
1584  (struct ether_addr *) addr);
1585  /* Set MAC of bounded interface to that of 1st slave link */
1586  rv =
1587  rte_eth_bond_mac_address_set (i,
1588  (struct ether_addr *)
1589  addr);
1590  if (rv < 0)
1591  clib_warning ("Failed to set MAC address");
1592 
1593  /* Populate MAC of bonded interface in VPP hw tables */
1594  bhi =
1595  vnet_get_hw_interface (vnm,
1596  dm->devices[i].vlib_hw_if_index);
1597  bei =
1599  clib_memcpy (bhi->hw_address, addr, 6);
1600  clib_memcpy (bei->address, addr, 6);
1601  /* Init l3 packet size allowed on bonded interface */
1606  while (nlink >= 1)
1607  { /* for all slave links */
1608  int slave = slink[--nlink];
1609  dpdk_device_t *sdev = &dm->devices[slave];
1610  vnet_hw_interface_t *shi;
1611  vnet_sw_interface_t *ssi;
1612  /* Add MAC to all slave links except the first one */
1613  if (nlink)
1614  rte_eth_dev_mac_addr_add (slave,
1615  (struct ether_addr *)
1616  addr, 0);
1617  /* Set slaves bitmap for bonded interface */
1618  bhi->bond_info =
1619  clib_bitmap_set (bhi->bond_info,
1620  sdev->vlib_hw_if_index, 1);
1621  /* Set slave link flags on slave interface */
1622  shi =
1624  ssi =
1628 
1629  /* Set l3 packet size allowed as the lowest of slave */
1630  if (bhi->max_l3_packet_bytes[VLIB_RX] >
1635 
1636  /* Set max packet size allowed as the lowest of slave */
1637  if (bhi->max_packet_bytes > shi->max_packet_bytes)
1638  bhi->max_packet_bytes = shi->max_packet_bytes;
1639  }
1640  }
1641  }
1642  }
1643  }
1644  }
1645 
1646  while (1)
1647  {
1648  /*
1649  * check each time through the loop in case intervals are changed
1650  */
1651  f64 min_wait = dm->link_state_poll_interval < dm->stat_poll_interval ?
1653 
1654  vlib_process_wait_for_event_or_clock (vm, min_wait);
1655 
1656  if (dm->admin_up_down_in_progress)
1657  /* skip the poll if an admin up down is in progress (on any interface) */
1658  continue;
1659 
1660  vec_foreach (xd, dm->devices)
1661  {
1662  f64 now = vlib_time_now (vm);
1663  if ((now - xd->time_last_stats_update) >= dm->stat_poll_interval)
1664  dpdk_update_counters (xd, now);
1665  if ((now - xd->time_last_link_update) >= dm->link_state_poll_interval)
1666  dpdk_update_link_state (xd, now);
1667 
1668  }
1669  }
1670 
1671  return 0;
1672 }
1673 
1674 /* *INDENT-OFF* */
1676  .function = dpdk_process,
1677  .type = VLIB_NODE_TYPE_PROCESS,
1678  .name = "dpdk-process",
1679  .process_log2_n_stack_bytes = 17,
1680 };
1681 /* *INDENT-ON* */
1682 
1683 int
1685 {
1686  if (interval < DPDK_MIN_STATS_POLL_INTERVAL)
1687  return (VNET_API_ERROR_INVALID_VALUE);
1688 
1689  dpdk_main.stat_poll_interval = interval;
1690 
1691  return 0;
1692 }
1693 
1694 int
1696 {
1697  if (interval < DPDK_MIN_LINK_POLL_INTERVAL)
1698  return (VNET_API_ERROR_INVALID_VALUE);
1699 
1700  dpdk_main.link_state_poll_interval = interval;
1701 
1702  return 0;
1703 }
1704 
1705 clib_error_t *
1707 {
1708  dpdk_main_t *dm = &dpdk_main;
1709  vlib_node_t *ei;
1710  clib_error_t *error = 0;
1712 
1713  /* verify that structs are cacheline aligned */
1714  STATIC_ASSERT (offsetof (dpdk_device_t, cacheline0) == 0,
1715  "Cache line marker must be 1st element in dpdk_device_t");
1716  STATIC_ASSERT (offsetof (dpdk_device_t, cacheline1) ==
1718  "Data in cache line 0 is bigger than cache line size");
1719  STATIC_ASSERT (offsetof (dpdk_worker_t, cacheline0) == 0,
1720  "Cache line marker must be 1st element in dpdk_worker_t");
1721  STATIC_ASSERT (offsetof (frame_queue_trace_t, cacheline0) == 0,
1722  "Cache line marker must be 1st element in frame_queue_trace_t");
1723 
1724  dm->vlib_main = vm;
1725  dm->vnet_main = vnet_get_main ();
1726  dm->conf = &dpdk_config_main;
1727 
1728  ei = vlib_get_node_by_name (vm, (u8 *) "ethernet-input");
1729  if (ei == 0)
1730  return clib_error_return (0, "ethernet-input node AWOL");
1731 
1732  dm->ethernet_input_node_index = ei->index;
1733 
1734  dm->conf->nchannels = 4;
1735  dm->conf->num_mbufs = dm->conf->num_mbufs ? dm->conf->num_mbufs : NB_MBUF;
1736  vec_add1 (dm->conf->eal_init_args, (u8 *) "vnet");
1737 
1738  dm->dpdk_device_by_kni_port_id = hash_create (0, sizeof (uword));
1739  dm->vu_sw_if_index_by_listener_fd = hash_create (0, sizeof (uword));
1740  dm->vu_sw_if_index_by_sock_fd = hash_create (0, sizeof (uword));
1741 
1742  /* $$$ use n_thread_stacks since it's known-good at this point */
1743  vec_validate (dm->recycle, tm->n_thread_stacks - 1);
1744 
1745  /* Default vlib_buffer_t flags, DISABLES tcp/udp checksumming... */
1746  dm->buffer_flags_template =
1749 
1752 
1753  /* init CLI */
1754  if ((error = vlib_call_init_function (vm, dpdk_cli_init)))
1755  return error;
1756 
1757  return error;
1758 }
1759 
1761 
1762 
1763 /*
1764  * fd.io coding-style-patch-verification: ON
1765  *
1766  * Local Variables:
1767  * eval: (c-set-style "gnu")
1768  * End:
1769  */
u32 ** d_trace_buffers
Definition: dpdk.h:186
#define vec_validate(V, I)
Make sure vector is long enough for given index (no header, unspecified alignment) ...
Definition: vec.h:396
#define DPDK_DEVICE_FLAG_PROMISC
Definition: dpdk.h:193
#define DPDK_NB_TX_DESC_10GE
Definition: dpdk_priv.h:21
f64 time_last_link_update
Definition: dpdk.h:224
vmrglw vmrglh hi
void unformat_init_vector(unformat_input_t *input, u8 *vector_string)
Definition: unformat.c:1025
static u8 * format_bitmap_hex(u8 *s, va_list *args)
Format a bitmap as a string of hex bytes.
Definition: bitmap.h:744
format_function_t format_vlib_pci_addr
Definition: pci.h:237
#define vec_foreach_index(var, v)
Iterate over vector indices.
#define hash_set(h, key, value)
Definition: hash.h:254
sll srl srl sll sra u16x4 i
Definition: vector_sse2.h:343
#define clib_min(x, y)
Definition: clib.h:326
uword unformat(unformat_input_t *i, char *fmt,...)
Definition: unformat.c:966
u32 vlib_buffer_get_or_create_free_list(vlib_main_t *vm, u32 n_data_bytes, char *fmt,...)
#define VNET_HW_INTERFACE_FLAG_SPEED_1G
Definition: interface.h:390
clib_error_t * vnet_hw_interface_set_flags(vnet_main_t *vnm, u32 hw_if_index, u32 flags)
Definition: interface.c:531
static f64 vlib_process_wait_for_event_or_clock(vlib_main_t *vm, f64 dt)
Suspend a cooperative multi-tasking thread Waits for an event, or for the indicated number of seconds...
Definition: node_funcs.h:684
u8 interface_name_format_decimal
Definition: dpdk.h:349
static vlib_main_t * vlib_get_main(void)
Definition: global_funcs.h:23
#define NB_MBUF
Definition: dpdk.h:63
static clib_error_t * dpdk_lib_init(dpdk_main_t *dm)
Definition: init.c:238
u8 use_rss
Definition: dpdk.h:405
vnet_device_class_t dpdk_device_class
static u32 dpdk_flag_change(vnet_main_t *vnm, vnet_hw_interface_t *hi, u32 flags)
Definition: init.c:138
#define DPDK_DEVICE_VLAN_STRIP_OFF
Definition: dpdk.h:316
#define UNFORMAT_END_OF_INPUT
Definition: format.h:143
#define I40E_DEV_ID_QSFP_B
Definition: dpdk_priv.h:28
#define NULL
Definition: clib.h:55
u32 index
Definition: node.h:237
clib_error_t * vlib_pci_bind_to_uio(vlib_pci_device_t *d, char *uio_driver_name)
Definition: linux_pci.c:95
static f64 vlib_time_now(vlib_main_t *vm)
Definition: main.h:182
#define vec_add2_aligned(V, P, N, A)
Add N elements to end of vector V, return pointer to new elements in P.
Definition: vec.h:533
static uword * clib_bitmap_or(uword *ai, uword *bi)
Logical operator across two bitmaps.
uword unformat_vlib_cli_sub_input(unformat_input_t *i, va_list *args)
Definition: cli.c:150
u16 flags
Definition: dpdk.h:191
static vnet_hw_interface_t * vnet_get_hw_interface(vnet_main_t *vnm, u32 hw_if_index)
#define vec_add1(V, E)
Add 1 element to end of vector (unspecified alignment).
Definition: vec.h:482
unsigned rte_socket_id()
dpdk_device_and_queue_t ** devices_by_hqos_cpu
Definition: dpdk.h:366
#define DPDK_NB_RX_DESC_VIRTIO
Definition: dpdk_priv.h:18
#define vec_add2(V, P, N)
Add N elements to end of vector V, return pointer to new elements in P.
Definition: vec.h:521
#define DPDK_DEVICE_FLAG_HQOS
Definition: dpdk.h:198
vlib_buffer_main_t * buffer_main
Definition: main.h:104
u32 per_interface_next_index
Definition: dpdk.h:179
u8 enable_tcp_udp_checksum
Definition: dpdk.h:335
static uword * clib_bitmap_set(uword *ai, uword i, uword value)
Sets the ith bit of a bitmap to new_value Removes trailing zeros from the bitmap. ...
Definition: bitmap.h:167
#define DPDK_DEVICE_VLAN_STRIP_ON
Definition: dpdk.h:317
static vnet_sw_interface_t * vnet_get_sw_interface(vnet_main_t *vnm, u32 sw_if_index)
static clib_error_t * dpdk_config(vlib_main_t *vm, unformat_input_t *input)
Definition: init.c:977
#define DPDK_NB_TX_DESC_DEFAULT
Definition: dpdk_priv.h:17
#define clib_error_report(e)
Definition: error.h:125
#define foreach_eal_double_hyphen_predicate_arg
Definition: dpdk_priv.h:34
unformat_function_t unformat_vlib_pci_addr
Definition: pci.h:236
#define VNET_HW_INTERFACE_FLAG_LINK_UP
Definition: interface.h:377
#define vec_validate_aligned(V, I, A)
Make sure vector is long enough for given index (no header, specified alignment)
Definition: vec.h:407
u16 device_id
Definition: pci.h:80
dpdk_device_config_hqos_t hqos
Definition: dpdk.h:324
dpdk_main_t dpdk_main
Definition: init.c:36
#define pool_get(P, E)
Allocate an object E from a pool P (unspecified alignment).
Definition: pool.h:200
vlib_pci_addr_t bus_address
Definition: pci.h:58
#define vec_reset_length(v)
Reset vector length to zero NULL-pointer tolerant.
#define DPDK_DEVICE_FLAG_PMD
Definition: dpdk.h:194
clib_error_t * vlib_buffer_pool_create(vlib_main_t *vm, unsigned num_mbufs, unsigned socket_id)
#define I40E_DEV_ID_QSFP_A
Definition: dpdk_priv.h:27
static vnet_sw_interface_t * vnet_get_hw_sw_interface(vnet_main_t *vnm, u32 hw_if_index)
vnet_main_t * vnet_get_main(void)
Definition: misc.c:46
#define I40E_DEV_ID_SFP_XL710
Definition: dpdk_priv.h:26
#define LINK_STATE_ELOGS
Definition: init.c:44
struct rte_mbuf *** tx_vectors
Definition: dpdk.h:182
foreach_dpdk_device_config_item clib_bitmap_t * workers
Definition: dpdk.h:322
#define pool_foreach(VAR, POOL, BODY)
Iterate through pool.
Definition: pool.h:348
#define VLIB_INIT_FUNCTION(x)
Definition: init.h:111
dpdk_config_main_t dpdk_config_main
Definition: dpdk.h:358
#define I40E_DEV_ID_QSFP_C
Definition: dpdk_priv.h:29
#define VPP_RUN_DIR
Definition: init.c:47
vlib_node_registration_t dpdk_input_node
(constructor) VLIB_REGISTER_NODE (dpdk_input_node)
Definition: node.c:662
dpdk_device_config_t default_devconf
Definition: dpdk.h:352
f64 stat_poll_interval
Definition: dpdk.h:417
#define IP_BUFFER_L4_CHECKSUM_CORRECT
Definition: buffer.h:50
int input_cpu_first_index
Definition: dpdk.h:408
char i8
Definition: types.h:45
#define vec_elt_at_index(v, i)
Get vector value at index i checking that i is in bounds.
u16 rx_q_used
Definition: dpdk.h:210
static void unformat_free(unformat_input_t *i)
Definition: format.h:161
#define clib_warning(format, args...)
Definition: error.h:59
#define DPDK_DEVICE_FLAG_MAYBE_MULTISEG
Definition: dpdk.h:196
u16 vendor_id
Definition: pci.h:79
#define vlib_call_init_function(vm, x)
Definition: init.h:161
clib_error_t * unformat_rss_fn(unformat_input_t *input, uword *rss_fn)
Definition: format.c:707
#define DPDK_NB_TX_DESC_VIRTIO
Definition: dpdk_priv.h:19
u32 device_index
Definition: dpdk.h:173
struct rte_eth_conf port_conf
Definition: dpdk.h:213
dpdk_worker_t * workers
Definition: dpdk.h:378
#define DPDK_NB_TX_DESC_40GE
Definition: dpdk_priv.h:23
#define I40E_DEV_ID_10G_BASE_T
Definition: dpdk_priv.h:30
f64 time_last_stats_update
Definition: dpdk.h:231
ethernet_main_t ethernet_main
Definition: init.c:45
dpdk_hqos_thread_t * hqos_threads
Definition: dpdk.h:381
static struct rte_eth_conf port_conf_template
Definition: init.c:51
clib_error_t * dpdk_init(vlib_main_t *vm)
Definition: init.c:1706
u32 vlib_sw_if_index
Definition: dpdk.h:176
struct rte_eth_txconf tx_conf
Definition: dpdk.h:214
vlib_worker_thread_t * vlib_worker_threads
Definition: threads.c:54
#define hash_get(h, key)
Definition: hash.h:248
#define clib_bitmap_foreach(i, ai, body)
Macro to iterate across set bits in a bitmap.
Definition: bitmap.h:361
#define pool_elt_at_index(p, i)
Returns pointer to element at given index.
Definition: pool.h:369
#define vec_insert(V, N, M)
Insert N vector elements starting at element M, initialize new elements to zero (no header...
Definition: vec.h:646
vlib_pci_addr_t pci_addr
Definition: dpdk.h:312
#define foreach_eal_double_hyphen_arg
Definition: dpdk_priv.h:50
#define VLIB_BUFFER_DEFAULT_FREE_LIST_BYTES
Definition: buffer.h:311
dpdk_device_and_queue_t ** devices_by_cpu
Definition: dpdk.h:365
#define ETHERNET_INTERFACE_FLAG_CONFIG_MTU(flags)
Definition: ethernet.h:119
u8 ** eal_init_args
Definition: dpdk.h:331
#define VNET_HW_INTERFACE_FLAG_SPEED_10M
Definition: interface.h:388
u32 vlib_hw_if_index
Definition: dpdk.h:175
uword unformat_skip_white_space(unformat_input_t *input)
Definition: unformat.c:809
#define VNET_SW_INTERFACE_FLAG_BOND_SLAVE
Definition: interface.h:537
#define foreach_eal_single_hyphen_mandatory_arg
Definition: dpdk_priv.h:40
vlib_pci_device_t * pci_devs
Definition: pci.h:116
int dpdk_set_link_state_poll_interval(f64 interval)
Definition: init.c:1695
#define VNET_HW_INTERFACE_FLAG_HALF_DUPLEX
Definition: interface.h:380
uword os_get_cpu_number(void)
Definition: unix-misc.c:224
static vlib_node_registration_t dpdk_process_node
(constructor) VLIB_REGISTER_NODE (dpdk_process_node)
Definition: init.c:1675
#define foreach_dpdk_pmd
Definition: dpdk.h:70
#define ELOG_DATA(em, f)
Definition: elog.h:392
dpdk_port_type_t port_type
Definition: dpdk.h:232
static uword dpdk_process(vlib_main_t *vm, vlib_node_runtime_t *rt, vlib_frame_t *f)
Definition: init.c:1510
#define VLIB_CONFIG_FUNCTION(x, n,...)
Definition: init.h:118
#define DPDK_MIN_STATS_POLL_INTERVAL
Definition: dpdk.h:236
int input_cpu_count
Definition: dpdk.h:409
#define VLIB_FRAME_SIZE
Definition: node.h:328
void * vlib_weakly_linked_functions[]
Definition: init.c:39
u16 tx_q_used
Definition: dpdk.h:209
u16 nb_rx_desc
Definition: dpdk.h:211
void vlib_cli_output(vlib_main_t *vm, char *fmt,...)
Definition: cli.c:576
static clib_error_t * dpdk_device_config(dpdk_config_main_t *conf, vlib_pci_addr_t pci_addr, unformat_input_t *input, u8 is_default)
Definition: init.c:877
#define DPDK_DEVICE_FLAG_ADMIN_UP
Definition: dpdk.h:192
u32 ** recycle
Definition: dpdk.h:369
#define VNET_HW_INTERFACE_BOND_INFO_SLAVE
Definition: interface.h:463
#define foreach_eal_single_hyphen_arg
Definition: dpdk_priv.h:44
int dpdk_set_stat_poll_interval(f64 interval)
Definition: init.c:1684
#define DPDK_NB_RX_DESC_DEFAULT
Definition: dpdk_priv.h:16
svmdb_client_t * c
dpdk_device_t * devices
Definition: dpdk.h:364
static void dpdk_update_counters(dpdk_device_t *xd, f64 now)
Definition: dpdk_priv.h:80
u8 nchannels_set_manually
Definition: dpdk.h:339
u16 * cpu_socket_id_by_queue
Definition: dpdk.h:212
vlib_pci_main_t pci_main
Definition: pci.c:53
#define vec_free(V)
Free vector&#39;s memory (no header).
Definition: vec.h:300
volatile u32 ** lockp
Definition: dpdk.h:170
dpdk_device_config_t * dev_confs
Definition: dpdk.h:353
static vlib_thread_main_t * vlib_get_thread_main()
Definition: global_funcs.h:32
struct rte_mbuf *** rx_vectors
Definition: dpdk.h:183
void dpdk_device_lock_free(dpdk_device_t *xd)
Definition: init.c:227
#define clib_memcpy(a, b, c)
Definition: string.h:69
#define DPDK_NB_RX_DESC_ENIC
Definition: dpdk_priv.h:24
dpdk_pmd_t pmd
Definition: dpdk.h:188
#define VLIB_BUFFER_TOTAL_LENGTH_VALID
Definition: buffer.h:99
elog_main_t elog_main
Definition: main.h:141
static void dpdk_bind_devices_to_uio(dpdk_config_main_t *conf)
Definition: init.c:807
#define ETHERNET_INTERFACE_FLAG_ACCEPT_ALL
Definition: ethernet.h:113
u8 coremask_set_manually
Definition: dpdk.h:338
#define ELOG_TYPE_DECLARE(f)
Definition: elog.h:350
#define DEFAULT_HUGE_DIR
Definition: init.c:46
clib_error_t * dpdk_port_setup(dpdk_main_t *dm, dpdk_device_t *xd)
Definition: init.c:65
#define VNET_HW_INTERFACE_FLAG_SPEED_10G
Definition: interface.h:391
#define vec_validate_ha(V, I, H, A)
Make sure vector is long enough for given index (general version).
Definition: vec.h:376
u8 * interface_name_suffix
Definition: dpdk.h:203
void dpdk_update_link_state(dpdk_device_t *xd, f64 now)
Definition: init.c:1395
#define hash_create(elts, value_bytes)
Definition: hash.h:658
#define VNET_HW_INTERFACE_FLAG_FULL_DUPLEX
Definition: interface.h:381
u32 max_l3_packet_bytes[VLIB_N_RX_TX]
Definition: interface.h:449
#define ASSERT(truth)
void dpdk_device_config_hqos_default(dpdk_device_config_hqos_t *hqos)
Definition: hqos.c:208
#define DPDK_DEVICE_FLAG_PMD_SUPPORTS_PTYPE
Definition: dpdk.h:195
unsigned int u32
Definition: types.h:88
int hqos_cpu_count
Definition: dpdk.h:413
u8 * format_unformat_error(u8 *s, va_list *va)
Definition: unformat.c:91
u32 poll_sleep
Definition: dpdk.h:420
u8 af_packet_port_id
Definition: dpdk.h:221
#define I40E_DEV_ID_VF
Definition: dpdk_priv.h:31
static void clib_mem_free(void *p)
Definition: mem.h:176
Bitmaps built as vectors of machine words.
clib_error_t * ethernet_register_interface(vnet_main_t *vnm, u32 dev_class_index, u32 dev_instance, u8 *address, u32 *hw_if_index_return, ethernet_flag_change_function_t flag_change)
Definition: interface.c:226
#define clib_bitmap_free(v)
Free a bitmap.
Definition: bitmap.h:92
static void vlib_node_set_state(vlib_main_t *vm, u32 node_index, vlib_node_state_t new_state)
Set node dispatch state.
Definition: node_funcs.h:146
#define DPDK_LINK_POLL_INTERVAL
Definition: dpdk.h:238
uword * thread_registrations_by_name
Definition: threads.h:274
struct rte_eth_link link
Definition: dpdk.h:223
#define DPDK_NB_RX_DESC_10GE
Definition: dpdk_priv.h:20
clib_error_t * dpdk_port_setup_hqos(dpdk_device_t *xd, dpdk_device_config_hqos_t *hqos)
Definition: hqos.c:250
u64 uword
Definition: types.h:112
static uword clib_bitmap_count_set_bits(uword *ai)
Return the number of set bits in a bitmap.
Definition: bitmap.h:441
#define IP_BUFFER_L4_CHECKSUM_COMPUTED
Definition: buffer.h:49
vlib_node_t * vlib_get_node_by_name(vlib_main_t *vm, u8 *name)
Definition: node.c:45
Definition: defs.h:47
#define DPDK_STATS_POLL_INTERVAL
Definition: dpdk.h:235
#define VNET_HW_INTERFACE_FLAG_SPEED_100M
Definition: interface.h:389
u32 ethernet_input_node_index
Definition: dpdk.h:384
#define vec_len(v)
Number of elements in vector (rvalue-only, NULL tolerant)
double f64
Definition: types.h:142
unsigned char u8
Definition: types.h:56
u8 admin_up_down_in_progress
Definition: dpdk.h:403
#define STATIC_ASSERT(truth,...)
static uword unformat_bitmap_list(unformat_input_t *input, va_list *va)
unformat a list of bit ranges into a bitmap (eg "0-3,5-7,11" )
Definition: bitmap.h:693
vlib_main_t ** vlib_mains
void dpdk_device_lock_init(dpdk_device_t *xd)
Definition: init.c:214
#define DPDK_MIN_LINK_POLL_INTERVAL
Definition: dpdk.h:239
#define hash_get_mem(h, key)
Definition: hash.h:268
u32 buffer_flags_template
Definition: dpdk.h:372
static void * clib_mem_alloc_aligned(uword size, uword align)
Definition: mem.h:117
static uword unformat_check_input(unformat_input_t *i)
Definition: format.h:169
static u32 random_u32(u32 *seed)
32-bit random number generator
Definition: random.h:69
#define VNET_HW_INTERFACE_FLAG_SPEED_40G
Definition: interface.h:392
u32 vlib_buffer_free_list_index
Definition: dpdk.h:375
#define VLIB_REGISTER_NODE(x,...)
Definition: node.h:143
u8 * format(u8 *s, const char *fmt,...)
Definition: format.c:418
#define VNET_BUFFER_RTE_MBUF_VALID
Definition: buffer.h:65
int hqos_cpu_first_index
Definition: dpdk.h:412
#define ETHERNET_MAX_PACKET_BYTES
Definition: ethernet.h:106
#define vec_foreach(var, vec)
Vector iterator.
i8 cpu_socket
Definition: dpdk.h:189
#define ETHERNET_INTERFACE_FLAG_CONFIG_PROMISC(flags)
Definition: ethernet.h:114
uword * cpu_socket_bitmap
Definition: threads.h:309
vhost_vring_addr_t addr
Definition: vhost-user.h:81
#define clib_error_return(e, args...)
Definition: error.h:111
struct _unformat_input_t unformat_input_t
u8 * uio_driver_name
Definition: dpdk.h:333
vlib_thread_registration_t ** registrations
Definition: threads.h:272
u32 flags
Definition: vhost-user.h:75
#define CLIB_CACHE_LINE_BYTES
Definition: cache.h:67
ethernet_interface_t * interfaces
Definition: ethernet.h:243
vnet_main_t * vnet_main
Definition: dpdk.h:424
u16 nb_tx_desc
Definition: dpdk.h:200
clib_error_t * unformat_hqos(unformat_input_t *input, dpdk_device_config_hqos_t *hqos)
Definition: format.c:730
uword * device_config_index_by_pci_addr
Definition: dpdk.h:354
static uword vnet_hw_interface_is_link_up(vnet_main_t *vnm, u32 hw_if_index)
volatile u32 worker_thread_release
Definition: threads.h:315
#define DPDK_NB_RX_DESC_40GE
Definition: dpdk_priv.h:22
uword * dpdk_device_by_kni_port_id
Definition: dpdk.h:394
Definition: defs.h:46
uword * vu_sw_if_index_by_listener_fd
Definition: dpdk.h:395
f64 link_state_poll_interval
Definition: dpdk.h:416
CLIB vectors are ubiquitous dynamically resized arrays with by user defined "headers".
clib_error_t * dpdk_cli_init(vlib_main_t *vm)
Definition: cli.c:1283
dpdk_config_main_t * conf
Definition: dpdk.h:425
uword * vu_sw_if_index_by_sock_fd
Definition: dpdk.h:396
vlib_main_t * vlib_main
Definition: dpdk.h:423