FD.io VPP  v20.09-64-g4f7b92f0a
Vector Packet Processing
device.c
Go to the documentation of this file.
1 /*
2  *------------------------------------------------------------------
3  * Copyright (c) 2018 Cisco and/or its affiliates.
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at:
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  *------------------------------------------------------------------
16  */
17 
18 #include <unistd.h>
19 #include <fcntl.h>
20 #include <net/if.h>
21 #include <linux/if_link.h>
22 #include <linux/if_ether.h>
23 
24 #include <vppinfra/linux/sysfs.h>
25 #include <vlib/vlib.h>
26 #include <vlib/unix/unix.h>
27 #include <vlib/pci/pci.h>
28 #include <vnet/ethernet/ethernet.h>
29 
30 #include <rdma/rdma.h>
31 
32 /* Default RSS hash key (from DPDK MLX driver) */
33 static u8 rdma_rss_hash_key[] = {
34  0x2c, 0xc6, 0x81, 0xd1,
35  0x5b, 0xdb, 0xf4, 0xf7,
36  0xfc, 0xa2, 0x83, 0x19,
37  0xdb, 0x1a, 0x3e, 0x94,
38  0x6b, 0x9e, 0x38, 0xd9,
39  0x2c, 0x9c, 0x03, 0xd1,
40  0xad, 0x99, 0x44, 0xa7,
41  0xd9, 0x56, 0x3d, 0x59,
42  0x06, 0x3c, 0x25, 0xf3,
43  0xfc, 0x1f, 0xdc, 0x2a,
44 };
45 
47 
48 #define rdma_log__(lvl, dev, f, ...) \
49  do { \
50  vlib_log((lvl), rdma_main.log_class, "%s: " f, \
51  &(dev)->name, ##__VA_ARGS__); \
52  } while (0)
53 
54 #define rdma_log(lvl, dev, f, ...) \
55  rdma_log__((lvl), (dev), "%s (%d): " f, strerror(errno), errno, ##__VA_ARGS__)
56 
57 static struct ibv_flow *
58 rdma_rxq_init_flow (const rdma_device_t * rd, struct ibv_qp *qp,
59  const mac_address_t * mac, const mac_address_t * mask,
60  u16 ether_type, u32 flags)
61 {
62  struct ibv_flow *flow;
63  struct raw_eth_flow_attr
64  {
65  struct ibv_flow_attr attr;
66  struct ibv_flow_spec_eth spec_eth;
67  } __attribute__ ((packed)) fa;
68 
69  memset (&fa, 0, sizeof (fa));
70  fa.attr.num_of_specs = 1;
71  fa.attr.port = 1;
72  fa.attr.flags = flags;
73  fa.spec_eth.type = IBV_FLOW_SPEC_ETH;
74  fa.spec_eth.size = sizeof (struct ibv_flow_spec_eth);
75 
76  memcpy (fa.spec_eth.val.dst_mac, mac, sizeof (fa.spec_eth.val.dst_mac));
77  memcpy (fa.spec_eth.mask.dst_mac, mask, sizeof (fa.spec_eth.mask.dst_mac));
78 
79  if (ether_type)
80  {
81  fa.spec_eth.val.ether_type = ether_type;
82  fa.spec_eth.mask.ether_type = 0xffff;
83  }
84 
85  flow = ibv_create_flow (qp, &fa.attr);
86  if (!flow)
87  rdma_log (VLIB_LOG_LEVEL_ERR, rd, "ibv_create_flow() failed");
88  return flow;
89 }
90 
91 static u32
92 rdma_rxq_destroy_flow (const rdma_device_t * rd, struct ibv_flow **flow)
93 {
94  if (!*flow)
95  return 0;
96 
97  if (ibv_destroy_flow (*flow))
98  {
99  rdma_log (VLIB_LOG_LEVEL_ERR, rd, "ibv_destroy_flow() failed");
100  return ~0;
101  }
102 
103  *flow = 0;
104  return 0;
105 }
106 
107 static u32
109 {
110  const mac_address_t all = {.bytes = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0} };
111  int err;
112 
113  err = rdma_rxq_destroy_flow (rd, &rd->flow_mcast6);
114  err |= rdma_rxq_destroy_flow (rd, &rd->flow_ucast6);
115  err |= rdma_rxq_destroy_flow (rd, &rd->flow_mcast4);
116  err |= rdma_rxq_destroy_flow (rd, &rd->flow_ucast4);
117  if (err)
118  return ~0;
119 
120  rd->flow_ucast6 =
121  rdma_rxq_init_flow (rd, rd->rx_qp6, &all, &all, ntohs (ETH_P_IPV6), 0);
122  rd->flow_ucast4 = rdma_rxq_init_flow (rd, rd->rx_qp4, &all, &all, 0, 0);
123  if (!rd->flow_ucast6 || !rd->flow_ucast4)
124  return ~0;
125 
126  rd->flags |= RDMA_DEVICE_F_PROMISC;
127  return 0;
128 }
129 
130 static u32
132 {
133  const mac_address_t ucast = {.bytes = {0xff, 0xff, 0xff, 0xff, 0xff, 0xff}
134  };
135  const mac_address_t mcast = {.bytes = {0x1, 0x0, 0x0, 0x0, 0x0, 0x0} };
136  int err;
137 
138  err = rdma_rxq_destroy_flow (rd, &rd->flow_mcast6);
139  err |= rdma_rxq_destroy_flow (rd, &rd->flow_ucast6);
140  err |= rdma_rxq_destroy_flow (rd, &rd->flow_mcast4);
141  err |= rdma_rxq_destroy_flow (rd, &rd->flow_ucast4);
142  if (err)
143  return ~0;
144 
145  rd->flow_ucast6 =
146  rdma_rxq_init_flow (rd, rd->rx_qp6, &rd->hwaddr, &ucast,
147  ntohs (ETH_P_IPV6), 0);
148  rd->flow_mcast6 =
149  rdma_rxq_init_flow (rd, rd->rx_qp6, &mcast, &mcast, ntohs (ETH_P_IPV6),
150  IBV_FLOW_ATTR_FLAGS_DONT_TRAP
151  /* let others receive mcast packet too (eg. Linux) */
152  );
153  rd->flow_ucast4 =
154  rdma_rxq_init_flow (rd, rd->rx_qp4, &rd->hwaddr, &ucast, 0, 0);
155  rd->flow_mcast4 =
156  rdma_rxq_init_flow (rd, rd->rx_qp4, &mcast, &mcast, 0,
157  IBV_FLOW_ATTR_FLAGS_DONT_TRAP
158  /* let others receive mcast packet too (eg. Linux) */
159  );
160  if (!rd->flow_ucast6 || !rd->flow_mcast6 || !rd->flow_ucast4
161  || !rd->flow_mcast4)
162  return ~0;
163 
164  rd->flags &= ~RDMA_DEVICE_F_PROMISC;
165  return 0;
166 }
167 
168 static clib_error_t *
169 rdma_mac_change (vnet_hw_interface_t * hw, const u8 * old, const u8 * new)
170 {
171  rdma_main_t *rm = &rdma_main;
173  mac_address_from_bytes (&rd->hwaddr, new);
174  if (!(rd->flags & RDMA_DEVICE_F_PROMISC) && rdma_dev_set_ucast (rd))
175  {
176  mac_address_from_bytes (&rd->hwaddr, old);
177  return clib_error_return_unix (0, "MAC update failed");
178  }
179  return 0;
180 }
181 
182 static u32
184 {
185  rdma_log__ (VLIB_LOG_LEVEL_ERR, rd, "MTU change not supported");
186  return ~0;
187 }
188 
189 static u32
191 {
192  rdma_main_t *rm = &rdma_main;
194 
195  switch (flags)
196  {
198  return rdma_dev_set_ucast (rd);
200  return rdma_dev_set_promisc (rd);
202  return rdma_dev_change_mtu (rd);
203  }
204 
205  rdma_log__ (VLIB_LOG_LEVEL_ERR, rd, "unknown flag %x requested", flags);
206  return ~0;
207 }
208 
209 static void
211 {
212  struct ibv_port_attr attr;
213  u32 width = 0;
214  u32 speed = 0;
215 
216  if (ibv_query_port (rd->ctx, port, &attr))
217  {
220  return;
221  }
222 
223  /* update state */
224  switch (attr.state)
225  {
226  case IBV_PORT_ACTIVE: /* fallthrough */
227  case IBV_PORT_ACTIVE_DEFER:
228  rd->flags |= RDMA_DEVICE_F_LINK_UP;
231  break;
232  default:
233  rd->flags &= ~RDMA_DEVICE_F_LINK_UP;
235  break;
236  }
237 
238  /* update speed */
239  switch (attr.active_width)
240  {
241  case 1:
242  width = 1;
243  break;
244  case 2:
245  width = 4;
246  break;
247  case 4:
248  width = 8;
249  break;
250  case 8:
251  width = 12;
252  break;
253  }
254  switch (attr.active_speed)
255  {
256  case 1:
257  speed = 2500000;
258  break;
259  case 2:
260  speed = 5000000;
261  break;
262  case 4: /* fallthrough */
263  case 8:
264  speed = 10000000;
265  break;
266  case 16:
267  speed = 14000000;
268  break;
269  case 32:
270  speed = 25000000;
271  break;
272  }
273  vnet_hw_interface_set_link_speed (vnm, rd->hw_if_index, width * speed);
274 }
275 
276 static clib_error_t *
278 {
279  rdma_main_t *rm = &rdma_main;
281  return clib_error_return (0, "RDMA: %s: async event error", rd->name);
282 }
283 
284 static clib_error_t *
286 {
287  vnet_main_t *vnm = vnet_get_main ();
288  rdma_main_t *rm = &rdma_main;
290  int ret;
291  struct ibv_async_event event;
292  ret = ibv_get_async_event (rd->ctx, &event);
293  if (ret < 0)
294  return clib_error_return_unix (0, "ibv_get_async_event() failed");
295 
296  switch (event.event_type)
297  {
298  case IBV_EVENT_PORT_ACTIVE:
299  rdma_update_state (vnm, rd, event.element.port_num);
300  break;
301  case IBV_EVENT_PORT_ERR:
302  rdma_update_state (vnm, rd, event.element.port_num);
303  break;
304  case IBV_EVENT_DEVICE_FATAL:
305  rd->flags &= ~RDMA_DEVICE_F_LINK_UP;
307  vlib_log_emerg (rm->log_class, "%s: fatal error", rd->name);
308  break;
309  default:
310  rdma_log__ (VLIB_LOG_LEVEL_ERR, rd, "unhandeld RDMA async event %i",
311  event.event_type);
312  break;
313  }
314 
315  ibv_ack_async_event (&event);
316  return 0;
317 }
318 
319 static clib_error_t *
321 {
322  clib_file_t t = { 0 };
323  int ret;
324 
325  /* make RDMA async event fd non-blocking */
326  ret = fcntl (rd->ctx->async_fd, F_GETFL);
327  if (ret < 0)
328  return clib_error_return_unix (0, "fcntl(F_GETFL) failed");
329 
330  ret = fcntl (rd->ctx->async_fd, F_SETFL, ret | O_NONBLOCK);
331  if (ret < 0)
332  return clib_error_return_unix (0, "fcntl(F_SETFL, O_NONBLOCK) failed");
333 
334  /* register RDMA async event fd */
336  t.file_descriptor = rd->ctx->async_fd;
338  t.private_data = rd->dev_instance;
339  t.description = format (0, "%v async event", rd->name);
340 
342  return 0;
343 }
344 
345 static void
347 {
349 }
350 
351 static clib_error_t *
353 {
354  clib_error_t *err =
356  rd->dev_instance, rd->hwaddr.bytes,
358 
359  /* Indicate ability to support L3 DMAC filtering and
360  * initialize interface to L3 non-promisc mode */
365  return err;
366 }
367 
368 static void
370 {
374 }
375 
376 static void
378 {
379  rdma_main_t *rm = &rdma_main;
380  rdma_rxq_t *rxq;
381  rdma_txq_t *txq;
382 
383 #define _(fn, arg) if (arg) \
384  { \
385  int rv; \
386  if ((rv = fn (arg))) \
387  rdma_log (VLIB_LOG_LEVEL_DEBUG, rd, #fn "() failed (rv = %d)", rv); \
388  }
389 
390  _(ibv_destroy_flow, rd->flow_mcast6);
391  _(ibv_destroy_flow, rd->flow_ucast6);
392  _(ibv_destroy_flow, rd->flow_mcast4);
393  _(ibv_destroy_flow, rd->flow_ucast4);
394  _(ibv_dereg_mr, rd->mr);
395  vec_foreach (txq, rd->txqs)
396  {
397  _(ibv_destroy_qp, txq->qp);
398  _(ibv_destroy_cq, txq->cq);
399  }
400  vec_foreach (rxq, rd->rxqs)
401  {
402  _(ibv_destroy_wq, rxq->wq);
403  _(ibv_destroy_cq, rxq->cq);
404  }
405  _(ibv_destroy_rwq_ind_table, rd->rx_rwq_ind_tbl);
406  _(ibv_destroy_qp, rd->rx_qp6);
407  _(ibv_destroy_qp, rd->rx_qp4);
408  _(ibv_dealloc_pd, rd->pd);
409  _(ibv_close_device, rd->ctx);
410 #undef _
411 
412  clib_error_free (rd->error);
413 
414  vec_free (rd->rxqs);
415  vec_free (rd->txqs);
416  vec_free (rd->name);
418  pool_put (rm->devices, rd);
419 }
420 
421 static clib_error_t *
423 {
424  rdma_rxq_t *rxq;
425  struct ibv_wq_init_attr wqia;
426  struct ibv_cq_init_attr_ex cqa = { };
427  struct ibv_wq_attr wqa;
428  struct ibv_cq_ex *cqex;
429 
431  rxq = vec_elt_at_index (rd->rxqs, qid);
432  rxq->size = n_desc;
433  vec_validate_aligned (rxq->bufs, n_desc - 1, CLIB_CACHE_LINE_BYTES);
434 
435  cqa.cqe = n_desc;
436  if (rd->flags & RDMA_DEVICE_F_MLX5DV)
437  {
438  struct mlx5dv_cq_init_attr dvcq = { };
439  dvcq.comp_mask = MLX5DV_CQ_INIT_ATTR_MASK_COMPRESSED_CQE;
440  dvcq.cqe_comp_res_format = MLX5DV_CQE_RES_FORMAT_HASH;
441 
442  if ((cqex = mlx5dv_create_cq (rd->ctx, &cqa, &dvcq)) == 0)
443  return clib_error_return_unix (0, "Create mlx5dv rx CQ Failed");
444  }
445  else
446  {
447  if ((cqex = ibv_create_cq_ex (rd->ctx, &cqa)) == 0)
448  return clib_error_return_unix (0, "Create CQ Failed");
449  }
450 
451  rxq->cq = ibv_cq_ex_to_cq (cqex);
452 
453  memset (&wqia, 0, sizeof (wqia));
454  wqia.wq_type = IBV_WQT_RQ;
455  wqia.max_wr = n_desc;
456  wqia.max_sge = 1;
457  wqia.pd = rd->pd;
458  wqia.cq = rxq->cq;
459  if ((rxq->wq = ibv_create_wq (rd->ctx, &wqia)) == 0)
460  return clib_error_return_unix (0, "Create WQ Failed");
461 
462  memset (&wqa, 0, sizeof (wqa));
463  wqa.attr_mask = IBV_WQ_ATTR_STATE;
464  wqa.wq_state = IBV_WQS_RDY;
465  if (ibv_modify_wq (rxq->wq, &wqa) != 0)
466  return clib_error_return_unix (0, "Modify WQ (RDY) Failed");
467 
468  if (rd->flags & RDMA_DEVICE_F_MLX5DV)
469  {
470  struct mlx5dv_obj obj = { };
471  struct mlx5dv_cq dv_cq;
472  struct mlx5dv_rwq dv_rwq;
473  u64 qw0;
474 
475  obj.cq.in = rxq->cq;
476  obj.cq.out = &dv_cq;
477  obj.rwq.in = rxq->wq;
478  obj.rwq.out = &dv_rwq;
479 
480  if ((mlx5dv_init_obj (&obj, MLX5DV_OBJ_CQ | MLX5DV_OBJ_RWQ)))
481  return clib_error_return_unix (0, "mlx5dv: failed to init rx obj");
482 
483  if (dv_cq.cqe_size != sizeof (mlx5dv_cqe_t))
484  return clib_error_return_unix (0, "mlx5dv: incompatible rx CQE size");
485 
486  rxq->log2_cq_size = max_log2 (dv_cq.cqe_cnt);
487  rxq->cqes = (mlx5dv_cqe_t *) dv_cq.buf;
488  rxq->cq_db = (volatile u32 *) dv_cq.dbrec;
489  rxq->cqn = dv_cq.cqn;
490 
491  rxq->wqes = (mlx5dv_rwq_t *) dv_rwq.buf;
492  rxq->wq_db = (volatile u32 *) dv_rwq.dbrec;
493  rxq->wq_stride = dv_rwq.stride;
494  rxq->wqe_cnt = dv_rwq.wqe_cnt;
495 
496  qw0 = clib_host_to_net_u32 (vlib_buffer_get_default_data_size (vm));
497  qw0 |= (u64) clib_host_to_net_u32 (rd->lkey) << 32;
498 
499  for (int i = 0; i < rxq->size; i++)
500  rxq->wqes[i].dsz_and_lkey = qw0;
501 
502  for (int i = 0; i < (1 << rxq->log2_cq_size); i++)
503  rxq->cqes[i].opcode_cqefmt_se_owner = 0xff;
504  }
505 
506  return 0;
507 }
508 
509 static clib_error_t *
511 {
512  struct ibv_rwq_ind_table_init_attr rwqia;
513  struct ibv_qp_init_attr_ex qpia;
514  struct ibv_wq **ind_tbl;
515  u32 i;
516 
517  ASSERT (is_pow2 (vec_len (rd->rxqs))
518  && "rxq number should be a power of 2");
519 
520  ind_tbl = vec_new (struct ibv_wq *, vec_len (rd->rxqs));
521  vec_foreach_index (i, rd->rxqs)
522  ind_tbl[i] = vec_elt_at_index (rd->rxqs, i)->wq;
523  memset (&rwqia, 0, sizeof (rwqia));
524  rwqia.log_ind_tbl_size = min_log2 (vec_len (ind_tbl));
525  rwqia.ind_tbl = ind_tbl;
526  if ((rd->rx_rwq_ind_tbl = ibv_create_rwq_ind_table (rd->ctx, &rwqia)) == 0)
527  return clib_error_return_unix (0, "RWQ indirection table create failed");
528  vec_free (ind_tbl);
529 
530  memset (&qpia, 0, sizeof (qpia));
531  qpia.qp_type = IBV_QPT_RAW_PACKET;
532  qpia.comp_mask =
533  IBV_QP_INIT_ATTR_PD | IBV_QP_INIT_ATTR_IND_TABLE |
534  IBV_QP_INIT_ATTR_RX_HASH;
535  qpia.pd = rd->pd;
536  qpia.rwq_ind_tbl = rd->rx_rwq_ind_tbl;
538  qpia.rx_hash_conf.rx_hash_key_len = sizeof (rdma_rss_hash_key);
539  qpia.rx_hash_conf.rx_hash_key = rdma_rss_hash_key;
540  qpia.rx_hash_conf.rx_hash_function = IBV_RX_HASH_FUNC_TOEPLITZ;
541 
542  qpia.rx_hash_conf.rx_hash_fields_mask =
543  IBV_RX_HASH_SRC_IPV4 | IBV_RX_HASH_DST_IPV4 | IBV_RX_HASH_SRC_PORT_TCP |
544  IBV_RX_HASH_DST_PORT_TCP;
545  if ((rd->rx_qp4 = ibv_create_qp_ex (rd->ctx, &qpia)) == 0)
546  return clib_error_return_unix (0, "IPv4 Queue Pair create failed");
547 
548  qpia.rx_hash_conf.rx_hash_fields_mask =
549  IBV_RX_HASH_SRC_IPV6 | IBV_RX_HASH_DST_IPV6 | IBV_RX_HASH_SRC_PORT_TCP |
550  IBV_RX_HASH_DST_PORT_TCP;
551  if ((rd->rx_qp6 = ibv_create_qp_ex (rd->ctx, &qpia)) == 0)
552  return clib_error_return_unix (0, "IPv6 Queue Pair create failed");
553 
554  if (rdma_dev_set_ucast (rd))
555  return clib_error_return_unix (0, "Set unicast mode failed");
556 
557  return 0;
558 }
559 
560 static clib_error_t *
562 {
563  rdma_txq_t *txq;
564  struct ibv_qp_init_attr qpia;
565  struct ibv_qp_attr qpa;
566  int qp_flags;
567 
569  txq = vec_elt_at_index (rd->txqs, qid);
570  ASSERT (is_pow2 (n_desc));
571  txq->bufs_log2sz = min_log2 (n_desc);
572  vec_validate_aligned (txq->bufs, n_desc - 1, CLIB_CACHE_LINE_BYTES);
573 
574  if ((txq->cq = ibv_create_cq (rd->ctx, n_desc, NULL, NULL, 0)) == 0)
575  return clib_error_return_unix (0, "Create CQ Failed");
576 
577  memset (&qpia, 0, sizeof (qpia));
578  qpia.send_cq = txq->cq;
579  qpia.recv_cq = txq->cq;
580  qpia.cap.max_send_wr = n_desc;
581  qpia.cap.max_send_sge = 1;
582  qpia.qp_type = IBV_QPT_RAW_PACKET;
583 
584  if ((txq->qp = ibv_create_qp (rd->pd, &qpia)) == 0)
585  return clib_error_return_unix (0, "Queue Pair create failed");
586 
587  memset (&qpa, 0, sizeof (qpa));
588  qp_flags = IBV_QP_STATE | IBV_QP_PORT;
589  qpa.qp_state = IBV_QPS_INIT;
590  qpa.port_num = 1;
591  if (ibv_modify_qp (txq->qp, &qpa, qp_flags) != 0)
592  return clib_error_return_unix (0, "Modify QP (init) Failed");
593 
594  memset (&qpa, 0, sizeof (qpa));
595  qp_flags = IBV_QP_STATE;
596  qpa.qp_state = IBV_QPS_RTR;
597  if (ibv_modify_qp (txq->qp, &qpa, qp_flags) != 0)
598  return clib_error_return_unix (0, "Modify QP (receive) Failed");
599 
600  memset (&qpa, 0, sizeof (qpa));
601  qp_flags = IBV_QP_STATE;
602  qpa.qp_state = IBV_QPS_RTS;
603  if (ibv_modify_qp (txq->qp, &qpa, qp_flags) != 0)
604  return clib_error_return_unix (0, "Modify QP (send) Failed");
605 
606  txq->ibv_cq = txq->cq;
607  txq->ibv_qp = txq->qp;
608 
609  if (rd->flags & RDMA_DEVICE_F_MLX5DV)
610  {
611  rdma_mlx5_wqe_t *tmpl = (void *) txq->dv_wqe_tmpl;
612  struct mlx5dv_cq dv_cq;
613  struct mlx5dv_qp dv_qp;
614  struct mlx5dv_obj obj = { };
615 
616  obj.cq.in = txq->cq;
617  obj.cq.out = &dv_cq;
618  obj.qp.in = txq->qp;
619  obj.qp.out = &dv_qp;
620 
621  if (mlx5dv_init_obj (&obj, MLX5DV_OBJ_CQ | MLX5DV_OBJ_QP))
622  return clib_error_return_unix (0, "DV init obj failed");
623 
624  if (RDMA_TXQ_BUF_SZ (txq) > dv_qp.sq.wqe_cnt
625  || !is_pow2 (dv_qp.sq.wqe_cnt)
626  || sizeof (rdma_mlx5_wqe_t) != dv_qp.sq.stride
627  || (uword) dv_qp.sq.buf % sizeof (rdma_mlx5_wqe_t))
628  return clib_error_return (0, "Unsupported DV SQ parameters");
629 
630  if (RDMA_TXQ_BUF_SZ (txq) > dv_cq.cqe_cnt
631  || !is_pow2 (dv_cq.cqe_cnt)
632  || sizeof (struct mlx5_cqe64) != dv_cq.cqe_size
633  || (uword) dv_cq.buf % sizeof (struct mlx5_cqe64))
634  return clib_error_return (0, "Unsupported DV CQ parameters");
635 
636  /* get SQ and doorbell addresses */
637  txq->dv_sq_wqes = dv_qp.sq.buf;
638  txq->dv_sq_dbrec = dv_qp.dbrec;
639  txq->dv_sq_db = dv_qp.bf.reg;
640  txq->dv_sq_log2sz = min_log2 (dv_qp.sq.wqe_cnt);
641 
642  /* get CQ and doorbell addresses */
643  txq->dv_cq_cqes = dv_cq.buf;
644  txq->dv_cq_dbrec = dv_cq.dbrec;
645  txq->dv_cq_log2sz = min_log2 (dv_cq.cqe_cnt);
646 
647  /* init tx desc template */
648  STATIC_ASSERT_SIZEOF (txq->dv_wqe_tmpl, sizeof (*tmpl));
649  mlx5dv_set_ctrl_seg (&tmpl->ctrl, 0, MLX5_OPCODE_SEND, 0,
650  txq->qp->qp_num, 0, RDMA_MLX5_WQE_DS, 0,
652  tmpl->eseg.inline_hdr_sz = htobe16 (MLX5_ETH_L2_INLINE_HEADER_SIZE);
653  mlx5dv_set_data_seg (&tmpl->dseg, 0, rd->lkey, 0);
654  }
655 
656  return 0;
657 }
658 
659 static clib_error_t *
661  u32 txq_size, u32 rxq_num)
662 {
663  clib_error_t *err;
666  u32 i;
667 
668  if (rd->ctx == 0)
669  return clib_error_return_unix (0, "Device Open Failed");
670 
671  if ((rd->pd = ibv_alloc_pd (rd->ctx)) == 0)
672  return clib_error_return_unix (0, "PD Alloc Failed");
673 
674  if ((rd->mr = ibv_reg_mr (rd->pd, (void *) bm->buffer_mem_start,
675  bm->buffer_mem_size,
676  IBV_ACCESS_LOCAL_WRITE)) == 0)
677  return clib_error_return_unix (0, "Register MR Failed");
678 
679  rd->lkey = rd->mr->lkey; /* avoid indirection in datapath */
680 
682 
683  if ((rd->mr = ibv_reg_mr (rd->pd, (void *) bm->buffer_mem_start,
684  bm->buffer_mem_size,
685  IBV_ACCESS_LOCAL_WRITE)) == 0)
686  return clib_error_return_unix (0, "Register MR Failed");
687  rd->lkey = rd->mr->lkey; /* avoid indirection in datapath */
688 
689  /*
690  * /!\ WARNING /!\ creation order is important
691  * We *must* create TX queues *before* RX queues, otherwise we will receive
692  * the broacast packets we sent
693  */
694  for (i = 0; i < tm->n_vlib_mains; i++)
695  if ((err = rdma_txq_init (vm, rd, i, txq_size)))
696  return err;
697 
698  for (i = 0; i < rxq_num; i++)
699  if ((err = rdma_rxq_init (vm, rd, i, rxq_size)))
700  return err;
701  if ((err = rdma_rxq_finalize (vm, rd)))
702  return err;
703 
704  return 0;
705 }
706 
707 static uword
708 sysfs_path_to_pci_addr (char *path, vlib_pci_addr_t * addr)
709 {
710  uword rv;
711  unformat_input_t in;
712  u8 *s;
713 
714  s = clib_sysfs_link_to_name (path);
715  if (!s)
716  return 0;
717 
718  unformat_init_string (&in, (char *) s, strlen ((char *) s));
719  rv = unformat (&in, "%U", unformat_vlib_pci_addr, addr);
720  unformat_free (&in);
721  vec_free (s);
722  return rv;
723 }
724 
725 void
727 {
728  vnet_main_t *vnm = vnet_get_main ();
729  rdma_main_t *rm = &rdma_main;
730  rdma_device_t *rd;
731  vlib_pci_addr_t pci_addr;
732  struct ibv_device **dev_list;
733  int n_devs;
734  u8 *s;
735  u16 qid;
736  int i;
737 
738  args->rxq_size = args->rxq_size ? args->rxq_size : 1024;
739  args->txq_size = args->txq_size ? args->txq_size : 1024;
740  args->rxq_num = args->rxq_num ? args->rxq_num : 1;
741 
742  if (!is_pow2 (args->rxq_num))
743  {
744  args->rv = VNET_API_ERROR_INVALID_VALUE;
745  args->error =
746  clib_error_return (0, "rx queue number must be a power of two");
747  goto err0;
748  }
749 
750  if (args->rxq_size < VLIB_FRAME_SIZE || args->txq_size < VLIB_FRAME_SIZE ||
751  args->rxq_size > 65535 || args->txq_size > 65535 ||
752  !is_pow2 (args->rxq_size) || !is_pow2 (args->txq_size))
753  {
754  args->rv = VNET_API_ERROR_INVALID_VALUE;
755  args->error = clib_error_return (0, "queue size must be a power of two "
756  "between %i and 65535",
758  goto err0;
759  }
760 
761  dev_list = ibv_get_device_list (&n_devs);
762  if (n_devs == 0)
763  {
764  args->error =
766  "no RDMA devices available. Is the ib_uverbs module loaded?");
767  goto err0;
768  }
769 
770  /* get PCI address */
771  s = format (0, "/sys/class/net/%s/device%c", args->ifname, 0);
772  if (sysfs_path_to_pci_addr ((char *) s, &pci_addr) == 0)
773  {
774  args->error =
775  clib_error_return (0, "cannot find PCI address for device ");
776  goto err1;
777  }
778 
779  pool_get_zero (rm->devices, rd);
780  rd->dev_instance = rd - rm->devices;
782  rd->linux_ifname = format (0, "%s", args->ifname);
783 
784  if (!args->name || 0 == args->name[0])
785  rd->name = format (0, "%s/%d", args->ifname, rd->dev_instance);
786  else
787  rd->name = format (0, "%s", args->name);
788 
789  rd->pci = vlib_pci_get_device_info (vm, &pci_addr, &args->error);
790  if (!rd->pci)
791  goto err2;
792 
793  /* if we failed to parse NUMA node, default to 0 */
794  if (-1 == rd->pci->numa_node)
795  rd->pci->numa_node = 0;
796 
798 
799  if (strncmp ((char *) rd->pci->driver_name, "mlx5_core", 9))
800  {
801  args->error =
803  "invalid interface (only mlx5 supported for now)");
804  goto err2;
805  }
806 
807  for (i = 0; i < n_devs; i++)
808  {
809  vlib_pci_addr_t addr;
810 
811  vec_reset_length (s);
812  s = format (s, "%s/device%c", dev_list[i]->dev_path, 0);
813 
814  if (sysfs_path_to_pci_addr ((char *) s, &addr) == 0)
815  continue;
816 
817  if (addr.as_u32 != rd->pci->addr.as_u32)
818  continue;
819 
820  if ((rd->ctx = ibv_open_device (dev_list[i])))
821  break;
822  }
823 
824  if (args->mode != RDMA_MODE_IBV)
825  {
826  struct mlx5dv_context mlx5dv_attrs = { };
827 
828  if (mlx5dv_query_device (rd->ctx, &mlx5dv_attrs) == 0)
829  {
830  if ((mlx5dv_attrs.flags & MLX5DV_CONTEXT_FLAGS_CQE_V1))
831  rd->flags |= RDMA_DEVICE_F_MLX5DV;
832  }
833  else
834  {
835  if (args->mode == RDMA_MODE_DV)
836  {
837  args->error = clib_error_return (0, "Direct Verbs mode not "
838  "supported on this interface");
839  goto err2;
840  }
841  }
842  }
843 
844  if ((args->error = rdma_dev_init (vm, rd, args->rxq_size, args->txq_size,
845  args->rxq_num)))
846  goto err2;
847 
848  if ((args->error = rdma_register_interface (vnm, rd)))
849  goto err2;
850 
851  if ((args->error = rdma_async_event_init (rd)))
852  goto err3;
853 
854  rdma_update_state (vnm, rd, 1);
855 
857  args->sw_if_index = rd->sw_if_index = sw->sw_if_index;
858  /*
859  * FIXME: add support for interrupt mode
860  * vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, rd->hw_if_index);
861  * hw->flags |= VNET_HW_INTERFACE_FLAG_SUPPORTS_INT_MODE;
862  */
864  rdma_input_node.index);
865  vec_foreach_index (qid, rd->rxqs)
867 
868  vec_free (s);
869  return;
870 
871 err3:
872  rdma_unregister_interface (vnm, rd);
873 err2:
874  rdma_dev_cleanup (rd);
875 err1:
876  ibv_free_device_list (dev_list);
877  vec_free (s);
878  args->rv = VNET_API_ERROR_INVALID_INTERFACE;
879 err0:
880  vlib_log_err (rm->log_class, "%U", format_clib_error, args->error);
881 }
882 
883 void
885 {
888  rdma_dev_cleanup (rd);
889 }
890 
891 static clib_error_t *
893 {
894  vnet_hw_interface_t *hi = vnet_get_hw_interface (vnm, hw_if_index);
895  rdma_main_t *rm = &rdma_main;
897  uword is_up = (flags & VNET_SW_INTERFACE_FLAG_ADMIN_UP) != 0;
898 
899  if (rd->flags & RDMA_DEVICE_F_ERROR)
900  return clib_error_return (0, "device is in error state");
901 
902  if (is_up)
903  {
906  rd->flags |= RDMA_DEVICE_F_ADMIN_UP;
907  }
908  else
909  {
911  rd->flags &= ~RDMA_DEVICE_F_ADMIN_UP;
912  }
913  return 0;
914 }
915 
916 static void
918  u32 node_index)
919 {
920  rdma_main_t *rm = &rdma_main;
921  vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, hw_if_index);
924  ~0 ==
926  vlib_node_add_next (vlib_get_main (), rdma_input_node.index, node_index);
927 }
928 
929 static char *rdma_tx_func_error_strings[] = {
930 #define _(n,s) s,
932 #undef _
933 };
934 
935 /* *INDENT-OFF* */
937 {
938  .name = "RDMA interface",
939  .format_device = format_rdma_device,
940  .format_device_name = format_rdma_device_name,
941  .admin_up_down_function = rdma_interface_admin_up_down,
942  .rx_redirect_to_node = rdma_set_interface_next_node,
943  .tx_function_n_errors = RDMA_TX_N_ERROR,
944  .tx_function_error_strings = rdma_tx_func_error_strings,
945  .mac_addr_change_function = rdma_mac_change,
946 };
947 /* *INDENT-ON* */
948 
949 clib_error_t *
951 {
952  rdma_main_t *rm = &rdma_main;
954 
955  rm->log_class = vlib_log_register_class ("rdma", 0);
956 
957  /* vlib_buffer_t template */
960 
961  for (int i = 0; i < tm->n_vlib_mains; i++)
962  {
964  clib_memset (&ptd->buffer_template, 0, sizeof (vlib_buffer_t));
965  ptd->buffer_template.flags = VLIB_BUFFER_TOTAL_LENGTH_VALID;
966  ptd->buffer_template.ref_count = 1;
967  vnet_buffer (&ptd->buffer_template)->sw_if_index[VLIB_TX] = (u32) ~ 0;
968  }
969 
970  return 0;
971 }
972 
973 /* *INDENT-OFF* */
975 {
976  .runs_after = VLIB_INITS ("pci_bus_init"),
977 };
978 /* *INDENT-OFF* */
979 
980 /*
981  * fd.io coding-style-patch-verification: ON
982  *
983  * Local Variables:
984  * eval: (c-set-style "gnu")
985  * End:
986  */
vlib_log_class_t vlib_log_register_class(char *class, char *subclass)
Definition: log.c:209
u32 flags
buffer flags: VLIB_BUFFER_FREE_LIST_INDEX_MASK: bits used to store free list index, VLIB_BUFFER_IS_TRACED: trace this buffer.
Definition: buffer.h:124
struct ibv_mr * mr
Definition: rdma.h:174
volatile u32 * dv_sq_dbrec
Definition: rdma.h:111
struct mlx5_cqe64 * dv_cq_cqes
Definition: rdma.h:113
#define vec_foreach_index(var, v)
Iterate over vector indices.
u8 * format_clib_error(u8 *s, va_list *va)
Definition: error.c:191
static u32 rdma_dev_set_ucast(rdma_device_t *rd)
Definition: device.c:131
u8 * linux_ifname
Definition: rdma.h:167
static clib_error_t * rdma_rxq_init(vlib_main_t *vm, rdma_device_t *rd, u16 qid, u32 n_desc)
Definition: device.c:422
vl_api_mac_address_t mac
Definition: l2.api:502
rdma_mlx5_wqe_t * dv_sq_wqes
Definition: rdma.h:110
vl_api_wireguard_peer_flags_t flags
Definition: wireguard.api:103
u32 wq_stride
Definition: rdma.h:89
void ethernet_delete_interface(vnet_main_t *vnm, u32 hw_if_index)
Definition: interface.c:378
struct ibv_flow * flow_mcast6
Definition: rdma.h:181
#define ntohs(x)
Definition: af_xdp.bpf.c:29
static u32 rdma_rxq_destroy_flow(const rdma_device_t *rd, struct ibv_flow **flow)
Definition: device.c:92
vnet_main_t * vnet_get_main(void)
Definition: misc.c:46
u64 dsz_and_lkey
Definition: rdma_mlx5dv.h:69
#define pool_get_zero(P, E)
Allocate an object E from a pool P and zero it.
Definition: pool.h:255
volatile u32 * cq_db
Definition: rdma.h:86
u32 cqn
Definition: rdma.h:87
#define rdma_log(lvl, dev, f,...)
Definition: device.c:54
format_function_t format_rdma_device
Definition: rdma.h:236
unsigned long u64
Definition: types.h:89
u32 size
Definition: rdma.h:75
static u32 rdma_dev_set_promisc(rdma_device_t *rd)
Definition: device.c:108
vlib_pci_device_info_t * pci
Definition: rdma.h:165
clib_memset(h->entries, 0, sizeof(h->entries[0]) *entries)
u32 dev_instance
Definition: rdma.h:170
static clib_error_t * rdma_rxq_finalize(vlib_main_t *vm, rdma_device_t *rd)
Definition: device.c:510
u8 opcode_cqefmt_se_owner
Definition: rdma_mlx5dv.h:51
mlx5dv_rwq_t * wqes
Definition: rdma.h:84
static vnet_hw_interface_t * vnet_get_hw_interface(vnet_main_t *vnm, u32 hw_if_index)
#define RDMA_TXQ_DV_INVALID_ID
Definition: rdma.h:141
u32 file_descriptor
Definition: file.h:54
struct ibv_wq * wq
Definition: rdma.h:73
volatile u32 * dv_cq_dbrec
Definition: rdma.h:114
u32 per_interface_next_index
Definition: rdma.h:158
static void vlib_pci_free_device_info(vlib_pci_device_info_t *di)
Definition: pci.h:114
for(i=1;i<=collision_buckets;i++)
vlib_buffer_main_t * buffer_main
Definition: main.h:183
rdma_main_t rdma_main
Definition: device.c:46
vlib_main_t * vm
Definition: in2out_ed.c:1582
u8 * format(u8 *s, const char *fmt,...)
Definition: format.c:424
vl_api_fib_path_t path
Definition: mfib_types.api:34
#define vec_validate_aligned(V, I, A)
Make sure vector is long enough for given index (no header, specified alignment)
Definition: vec.h:520
#define ETHERNET_INTERFACE_FLAG_DEFAULT_L3
Definition: ethernet.h:149
u16 mask
Definition: flow_types.api:52
vhost_vring_addr_t addr
Definition: vhost_user.h:111
mac_address_t hwaddr
Definition: rdma.h:168
static uword vlib_node_add_next(vlib_main_t *vm, uword node, uword next_node)
Definition: node_funcs.h:1173
unsigned char u8
Definition: types.h:56
static uword min_log2(uword x)
Definition: clib.h:161
#define vec_reset_length(v)
Reset vector length to zero NULL-pointer tolerant.
clib_file_function_t * read_function
Definition: file.h:67
static vnet_sw_interface_t * vnet_get_hw_sw_interface(vnet_main_t *vnm, u32 hw_if_index)
vlib_log_class_t log_class
Definition: rdma.h:202
static void rdma_async_event_cleanup(rdma_device_t *rd)
Definition: device.c:346
struct ibv_flow * flow_ucast6
Definition: rdma.h:180
#define rdma_log__(lvl, dev, f,...)
Definition: device.c:48
rdma_per_thread_data_t * per_thread_data
Definition: rdma.h:200
VNET_DEVICE_CLASS(af_xdp_device_class)
struct ibv_flow * flow_mcast4
Definition: rdma.h:179
#define VLIB_INIT_FUNCTION(x)
Definition: init.h:173
static void rdma_set_interface_next_node(vnet_main_t *vnm, u32 hw_if_index, u32 node_index)
Definition: device.c:917
struct ibv_pd * pd
Definition: rdma.h:173
static uword sysfs_path_to_pci_addr(char *path, vlib_pci_addr_t *addr)
Definition: device.c:708
#define vec_new(T, N)
Create new vector of given type and length (unspecified alignment, no header).
Definition: vec.h:350
rdma_device_t * devices
Definition: rdma.h:201
vnet_hw_interface_flags_t flags
Definition: interface.h:537
uword buffer_mem_size
Definition: buffer.h:453
#define vec_elt_at_index(v, i)
Get vector value at index i checking that i is in bounds.
struct ibv_cq * cq
Definition: rdma.h:134
#define clib_error_return(e, args...)
Definition: error.h:99
static void rdma_dev_cleanup(rdma_device_t *rd)
Definition: device.c:377
clib_file_main_t file_main
Definition: main.c:63
#define vlib_log_emerg(...)
Definition: log.h:102
unsigned int u32
Definition: types.h:88
#define VLIB_FRAME_SIZE
Definition: node.h:377
void unformat_init_string(unformat_input_t *input, char *string, int string_len)
Definition: unformat.c:1029
u32 flags
Definition: rdma.h:157
static clib_error_t * rdma_txq_init(vlib_main_t *vm, rdma_device_t *rd, u16 qid, u32 n_desc)
Definition: device.c:561
u32 * bufs
Definition: rdma.h:74
vlib_pci_device_info_t * vlib_pci_get_device_info(vlib_main_t *vm, vlib_pci_addr_t *addr, clib_error_t **error)
Definition: pci.c:202
u8 * description
Definition: file.h:70
#define pool_elt_at_index(p, i)
Returns pointer to element at given index.
Definition: pool.h:534
static_always_inline void mac_address_from_bytes(mac_address_t *mac, const u8 *bytes)
Definition: mac_address.h:92
clib_error_t * rdma_init(vlib_main_t *vm)
Definition: device.c:950
struct _unformat_input_t unformat_input_t
unsigned short u16
Definition: types.h:57
static clib_error_t * rdma_interface_admin_up_down(vnet_main_t *vnm, u32 hw_if_index, u32 flags)
Definition: device.c:892
vlib_node_registration_t rdma_input_node
(constructor) VLIB_REGISTER_NODE (rdma_input_node)
Definition: input.c:655
#define clib_error_return_unix(e, args...)
Definition: error.h:102
static u32 rdma_dev_change_mtu(rdma_device_t *rd)
Definition: device.c:183
struct ibv_cq * cq
Definition: rdma.h:72
#define pool_put(P, E)
Free an object E in pool P.
Definition: pool.h:302
unformat_function_t unformat_vlib_pci_addr
Definition: pci.h:323
struct ibv_cq * ibv_cq
Definition: rdma.h:104
struct ibv_qp * qp
Definition: rdma.h:135
static u8 rdma_rss_hash_key[]
Definition: device.c:33
static_always_inline u32 vlib_buffer_get_default_data_size(vlib_main_t *vm)
Definition: buffer_funcs.h:96
vlib_buffer_t buffer_template
Definition: rdma.h:195
u32 hw_if_index
Definition: rdma.h:160
u32 wqe_cnt
Definition: rdma.h:88
struct ibv_rwq_ind_table * rx_rwq_ind_tbl
Definition: rdma.h:177
clib_error_t * error
Definition: rdma.h:183
clib_error_t * error
Definition: rdma.h:227
rdma_mode_t mode
Definition: rdma.h:222
static void rdma_unregister_interface(vnet_main_t *vnm, rdma_device_t *rd)
Definition: device.c:369
struct ibv_flow * flow_ucast4
Definition: rdma.h:178
sll srl srl sll sra u16x4 i
Definition: vector_sse42.h:317
#define vec_free(V)
Free vector&#39;s memory (no header).
Definition: vec.h:380
u8 * driver_name
Definition: pci.h:82
u32 lkey
Definition: rdma.h:161
#define ETHERNET_INTERFACE_FLAG_MTU
Definition: ethernet.h:155
#define ETHERNET_INTERFACE_FLAG_ACCEPT_ALL
Definition: ethernet.h:152
#define RDMA_TXQ_BUF_SZ(txq)
Definition: rdma.h:143
u32 sw_if_index
Definition: rdma.h:159
u16 log2_cq_size
Definition: rdma.h:79
#define ASSERT(truth)
format_function_t format_rdma_device_name
Definition: rdma.h:237
void vnet_hw_interface_assign_rx_thread(vnet_main_t *vnm, u32 hw_if_index, u16 queue_id, uword thread_index)
Definition: devices.c:139
u8 bufs_log2sz
Definition: rdma.h:122
static uword clib_file_add(clib_file_main_t *um, clib_file_t *template)
Definition: file.h:96
static void clib_file_del_by_index(clib_file_main_t *um, uword index)
Definition: file.h:119
u8 * name
Definition: rdma.h:166
volatile u64 * dv_sq_db
Definition: rdma.h:112
rdma_txq_t * txqs
Definition: rdma.h:156
u8 dv_cq_log2sz
Definition: rdma.h:124
u8 dv_sq_log2sz
Definition: rdma.h:123
static u32 rdma_flag_change(vnet_main_t *vnm, vnet_hw_interface_t *hw, u32 flags)
Definition: device.c:190
struct ibv_qp * rx_qp4
Definition: rdma.h:175
vlib_pci_addr_t addr
Definition: pci.h:66
rdma_rxq_t * rxqs
Definition: rdma.h:155
static vlib_main_t * vlib_get_main(void)
Definition: global_funcs.h:23
vl_api_ip4_address_t hi
Definition: arp.api:37
vnet_device_class_t rdma_device_class
static uword is_pow2(uword x)
Definition: clib.h:252
u8 dv_wqe_tmpl[64]
Definition: rdma.h:128
vl_api_flow_t flow
Definition: flow_types.api:240
Definition: defs.h:47
static clib_error_t * rdma_dev_init(vlib_main_t *vm, rdma_device_t *rd, u32 rxq_size, u32 txq_size, u32 rxq_num)
Definition: device.c:660
mlx5dv_cqe_t * cqes
Definition: rdma.h:83
u32 async_event_clib_file_index
Definition: rdma.h:169
#define vec_len(v)
Number of elements in vector (rvalue-only, NULL tolerant)
static void ethernet_mac_address_generate(u8 *mac)
Definition: mac_address.h:74
clib_error_t * ethernet_register_interface(vnet_main_t *vnm, u32 dev_class_index, u32 dev_instance, const u8 *address, u32 *hw_if_index_return, ethernet_flag_change_function_t flag_change)
Definition: interface.c:331
clib_error_t * vnet_hw_interface_set_flags(vnet_main_t *vnm, u32 hw_if_index, vnet_hw_interface_flags_t flags)
Definition: interface.c:498
static uword max_log2(uword x)
Definition: clib.h:208
VLIB buffer representation.
Definition: buffer.h:102
u64 uword
Definition: types.h:112
static void unformat_free(unformat_input_t *i)
Definition: format.h:163
static struct ibv_flow * rdma_rxq_init_flow(const rdma_device_t *rd, struct ibv_qp *qp, const mac_address_t *mac, const mac_address_t *mask, u16 ether_type, u32 flags)
Definition: device.c:58
static void rdma_update_state(vnet_main_t *vnm, rdma_device_t *rd, int port)
Definition: device.c:210
uword buffer_mem_start
Definition: buffer.h:452
u32 * bufs
Definition: rdma.h:118
#define foreach_rdma_tx_func_error
Definition: rdma.h:249
volatile u32 * wq_db
Definition: rdma.h:85
#define clib_error_free(e)
Definition: error.h:86
static char * rdma_tx_func_error_strings[]
Definition: device.c:929
u16 port
Definition: lb_types.api:72
clib_file_function_t * error_function
Definition: file.h:67
int vnet_hw_interface_unassign_rx_thread(vnet_main_t *vnm, u32 hw_if_index, u16 queue_id)
Definition: devices.c:188
static clib_error_t * rdma_mac_change(vnet_hw_interface_t *hw, const u8 *old, const u8 *new)
Definition: device.c:169
#define vnet_buffer(b)
Definition: buffer.h:417
static clib_error_t * rdma_register_interface(vnet_main_t *vnm, rdma_device_t *rd)
Definition: device.c:352
static clib_error_t * rdma_async_event_init(rdma_device_t *rd)
Definition: device.c:320
static vlib_thread_main_t * vlib_get_thread_main()
Definition: global_funcs.h:32
#define vec_foreach(var, vec)
Vector iterator.
static clib_error_t * rdma_async_event_error_ready(clib_file_t *f)
Definition: device.c:277
uword private_data
Definition: file.h:64
#define MLX5_ETH_L2_INLINE_HEADER_SIZE
Definition: rdma.h:43
#define RDMA_MLX5_WQE_DS
Definition: rdma.h:64
#define vlib_log_err(...)
Definition: log.h:105
Definition: file.h:51
static clib_error_t * rdma_async_event_read_ready(clib_file_t *f)
Definition: device.c:285
void rdma_delete_if(vlib_main_t *vm, rdma_device_t *rd)
Definition: device.c:884
#define CLIB_CACHE_LINE_BYTES
Definition: cache.h:59
void rdma_create_if(vlib_main_t *vm, rdma_create_if_args_t *args)
Definition: device.c:726
#define STATIC_ASSERT_SIZEOF(d, s)
static u8 vlib_buffer_pool_get_default_for_numa(vlib_main_t *vm, u32 numa_node)
Definition: buffer_funcs.h:199
volatile u8 ref_count
Reference count for this buffer.
Definition: buffer.h:130
#define VLIB_INITS(...)
Definition: init.h:357
static void vnet_hw_interface_set_link_speed(vnet_main_t *vnm, u32 hw_if_index, u32 link_speed)
static void vnet_hw_interface_set_input_node(vnet_main_t *vnm, u32 hw_if_index, u32 node_index)
Definition: devices.h:79
struct ibv_context * ctx
Definition: rdma.h:172
uword unformat(unformat_input_t *i, const char *fmt,...)
Definition: unformat.c:978
u8 * clib_sysfs_link_to_name(char *link)
Definition: sysfs.c:90
u32 ethernet_set_flags(vnet_main_t *vnm, u32 hw_if_index, u32 flags)
Definition: interface.c:426
struct ibv_qp * rx_qp6
Definition: rdma.h:176
struct ibv_qp * ibv_qp
Definition: rdma.h:105