19 #include <rte_config.h> 21 #include <rte_ethdev.h> 28 "VLIB_BUFFER_PRE_DATA_SIZE must be equal to RTE_PKTMBUF_HEADROOM");
30 #ifndef CLIB_MARCH_VARIANT 38 struct rte_mempool *mp, *nmp;
39 struct rte_pktmbuf_pool_private priv;
40 enum rte_iova_mode iova_mode;
45 sizeof (
struct rte_mbuf) + sizeof (vlib_buffer_t) + bp->data_size;
54 name =
format (name,
"vpp pool %u%c", bp->index, 0);
55 mp = rte_mempool_create_empty ((
char *) name,
vec_len (bp->buffers),
56 elt_size, 512, sizeof (priv),
61 name =
format (name,
"vpp pool %u (no cache)%c", bp->index, 0);
62 nmp = rte_mempool_create_empty ((
char *) name,
vec_len (bp->buffers),
63 elt_size, 0, sizeof (priv),
70 mp->pool_id = nmp->pool_id = bp->index;
72 rte_mempool_set_ops_byname (mp,
"vpp",
NULL);
73 rte_mempool_set_ops_byname (nmp,
"vpp-no-cache",
NULL);
79 rte_pktmbuf_pool_init (mp, &priv);
80 rte_pktmbuf_pool_init (nmp, &priv);
82 iova_mode = rte_eal_iova_mode ();
88 struct rte_mempool_objhdr *hdr;
91 hdr = (
struct rte_mempool_objhdr *) RTE_PTR_SUB (mb,
sizeof (*hdr));
93 hdr->iova = (iova_mode == RTE_IOVA_VA) ?
95 STAILQ_INSERT_TAIL (&mp->elt_list, hdr, next);
96 STAILQ_INSERT_TAIL (&nmp->elt_list, hdr, next);
98 nmp->populated_size++;
103 rte_mempool_obj_iter (mp, rte_pktmbuf_init, 0);
115 if (rte_eth_dev_count_avail ())
125 for (i = 0; i < pm->
n_pages; i++)
127 char *va = ((
char *) pm->
base) + i * page_sz;
128 uword pa = (iova_mode == RTE_IOVA_VA) ?
135 struct rte_mempool_memhdr *memhdr;
140 memhdr->len = page_sz;
144 STAILQ_INSERT_TAIL (&mp->mem_list, memhdr, next);
172 struct rte_mbuf *mb = obj;
181 void *
const *obj_table,
unsigned n)
183 const int batch_size = 32;
186 u8 buffer_pool_index = mp->pool_id;
188 u32 bufs[batch_size];
190 void *
const *obj = obj_table;
211 while (n >= batch_size)
215 sizeof (
struct rte_mbuf));
218 obj_table += batch_size;
224 n,
sizeof (
struct rte_mbuf));
235 struct rte_mempool *
new,
void *obj,
238 struct rte_mbuf *mb = obj;
253 void *
const *obj_table,
258 struct rte_mempool *mp;
260 u8 buffer_pool_index = cmp->pool_id;
288 void **obj_table,
unsigned n)
290 const int batch_size = 32;
292 u32 bufs[batch_size], total = 0, n_alloc = 0;
293 u8 buffer_pool_index = mp->pool_id;
294 void **obj = obj_table;
296 while (n >= batch_size)
300 if (n_alloc != batch_size)
304 -(
i32)
sizeof (
struct rte_mbuf));
318 -(
i32)
sizeof (
struct rte_mbuf));
332 sizeof (
struct rte_mbuf));
343 #ifndef CLIB_MARCH_VARIANT 363 struct rte_mempool *cmp;
374 struct rte_mempool_ops ops = { };
376 strncpy (ops.name,
"vpp", 4);
382 rte_mempool_register_ops (&ops);
384 strncpy (ops.name,
"vpp-no-cache", 13);
388 rte_mempool_register_ops (&ops);
399 sizeof (
struct rte_mbuf));
#define vlib_buffer_from_rte_mbuf(x)
#define CLIB_MARCH_FN_POINTER(fn)
#define clib_error(format, args...)
static int dpdk_ops_vpp_alloc(struct rte_mempool *mp)
static_always_inline vlib_buffer_pool_t * vlib_get_buffer_pool(vlib_main_t *vm, u8 buffer_pool_index)
#define VLIB_BUFFER_PRE_DATA_SIZE
vlib_buffer_main_t * buffer_main
VLIB_BUFFER_SET_EXT_HDR_SIZE(sizeof(struct rte_mempool_objhdr)+sizeof(struct rte_mbuf))
static_always_inline void vlib_get_buffers_with_offset(vlib_main_t *vm, u32 *bi, void **b, int count, i32 offset)
Translate array of buffer indices into buffer pointers with offset.
vlib_physmem_map_t * vlib_physmem_get_map(vlib_main_t *vm, u32 index)
#define vec_validate_aligned(V, I, A)
Make sure vector is long enough for given index (no header, specified alignment)
u8 buffer_pool_index
index of buffer pool this buffer belongs.
#define vec_reset_length(v)
Reset vector length to zero NULL-pointer tolerant.
#define static_always_inline
clib_error_t * dpdk_buffer_pool_init(vlib_main_t *vm, vlib_buffer_pool_t *bp)
static void dpdk_ops_vpp_free(struct rte_mempool *mp)
static u32 vlib_get_buffer_index(vlib_main_t *vm, void *p)
Translate buffer pointer into buffer index.
static_always_inline void dpdk_ops_vpp_enqueue_one(vlib_buffer_t *bt, void *obj)
static_always_inline void vlib_buffer_copy_template(vlib_buffer_t *b, vlib_buffer_t *bt)
#define rte_mbuf_from_vlib_buffer(x)
vlib_buffer_t buffer_template
vlib_buffer_pool_t * buffer_pools
static_always_inline void vlib_get_buffer_indices_with_offset(vlib_main_t *vm, void **b, u32 *bi, uword count, i32 offset)
Translate array of buffer pointers into buffer indices with offset.
#define clib_atomic_sub_fetch(a, b)
static_always_inline u32 vlib_buffer_get_default_data_size(vlib_main_t *vm)
STATIC_ASSERT(VLIB_BUFFER_PRE_DATA_SIZE==RTE_PKTMBUF_HEADROOM,"VLIB_BUFFER_PRE_DATA_SIZE must be equal to RTE_PKTMBUF_HEADROOM")
static_always_inline void vlib_buffer_pool_put(vlib_main_t *vm, u8 buffer_pool_index, u32 *buffers, u32 n_buffers)
#define vec_free(V)
Free vector's memory (no header).
#define clib_warning(format, args...)
int CLIB_MULTIARCH_FN() dpdk_ops_vpp_enqueue_no_cache(struct rte_mempool *cmp, void *const *obj_table, unsigned n)
int CLIB_MULTIARCH_FN() dpdk_ops_vpp_enqueue(struct rte_mempool *mp, void *const *obj_table, unsigned n)
CLIB_MARCH_FN_REGISTRATION(dpdk_ops_vpp_enqueue)
#define VLIB_BUFFER_HDR_SIZE
static u32 vlib_buffer_alloc_from_pool(vlib_main_t *vm, u32 *buffers, u32 n_buffers, u8 buffer_pool_index)
Allocate buffers from specific pool into supplied array.
static_always_inline void dpdk_ops_vpp_enqueue_no_cache_one(vlib_main_t *vm, struct rte_mempool *old, struct rte_mempool *new, void *obj, vlib_buffer_t *bt)
static void * vlib_buffer_ptr_from_index(uword buffer_mem_start, u32 buffer_index, uword offset)
static void * clib_mem_alloc(uword size)
static uword pointer_to_uword(const void *p)
static vlib_main_t * vlib_get_main(void)
static u64 vlib_physmem_get_pa(vlib_main_t *vm, void *mem)
clib_error_t * dpdk_buffer_pools_create(vlib_main_t *vm)
#define vec_len(v)
Number of elements in vector (rvalue-only, NULL tolerant)
struct rte_mempool ** dpdk_no_cache_mempool_by_buffer_pool_index
VLIB buffer representation.
static unsigned dpdk_ops_vpp_get_count_no_cache(const struct rte_mempool *mp)
static unsigned dpdk_ops_vpp_get_count(const struct rte_mempool *mp)
#define vec_foreach(var, vec)
Vector iterator.
static int dpdk_ops_vpp_dequeue_no_cache(struct rte_mempool *mp, void **obj_table, unsigned n)
struct rte_mempool ** dpdk_mempool_by_buffer_pool_index
#define CLIB_CACHE_LINE_BYTES
int CLIB_MULTIARCH_FN() dpdk_ops_vpp_dequeue(struct rte_mempool *mp, void **obj_table, unsigned n)
volatile u8 ref_count
Reference count for this buffer.
static vlib_buffer_t * vlib_get_buffer(vlib_main_t *vm, u32 buffer_index)
Translate buffer index into buffer pointer.
#define CLIB_MULTIARCH_FN(fn)