20 #include <linux/if_link.h> 21 #include <bpf/libbpf.h> 72 af_xdp_log (VLIB_LOG_LEVEL_ERR, ad,
"set unicast not supported yet");
76 "set promiscuous not supported yet");
79 af_xdp_log (VLIB_LOG_LEVEL_ERR, ad,
"set mtu not supported yet");
83 af_xdp_log (VLIB_LOG_LEVEL_ERR, ad,
"unknown flag %x requested", flags);
92 struct xsk_socket **xsk;
93 struct xsk_umem **umem;
113 bpf_object__unload (ad->
bpf_obj);
133 args->
rv = VNET_API_ERROR_INVALID_VALUE;
140 if (bpf_prog_load (args->
prog, BPF_PROG_TYPE_XDP, &ad->
bpf_obj, &fd))
142 args->
rv = VNET_API_ERROR_SYSCALL_ERROR_5;
148 #ifndef XDP_FLAGS_REPLACE 149 #define XDP_FLAGS_REPLACE 0 153 args->
rv = VNET_API_ERROR_SYSCALL_ERROR_6;
163 bpf_object__unload (ad->
bpf_obj);
178 struct xsk_umem_config umem_config;
179 struct xsk_socket_config sock_config;
180 struct xdp_options opt;
186 struct xsk_ring_cons *rx = qid < rxq_num ? &rxq->
rx : 0;
187 struct xsk_ring_prod *fq = &rxq->
fq;
188 struct xsk_ring_prod *tx = qid < txq_num ? &txq->
tx : 0;
189 struct xsk_ring_cons *cq = &txq->
cq;
192 memset (&umem_config, 0,
sizeof (umem_config));
193 umem_config.fill_size = args->
rxq_size;
194 umem_config.comp_size = args->
txq_size;
195 umem_config.frame_size =
197 umem_config.flags = XDP_UMEM_UNALIGNED_CHUNK_FLAG;
202 args->
rv = VNET_API_ERROR_SYSCALL_ERROR_1;
207 memset (&sock_config, 0,
sizeof (sock_config));
208 sock_config.rx_size = args->
rxq_size;
209 sock_config.tx_size = args->
txq_size;
210 sock_config.bind_flags = XDP_USE_NEED_WAKEUP;
216 sock_config.bind_flags |= XDP_COPY;
219 sock_config.bind_flags |= XDP_ZEROCOPY;
222 if (xsk_socket__create
223 (xsk, ad->
linux_ifname, qid, *umem, rx, tx, &sock_config))
225 args->
rv = VNET_API_ERROR_SYSCALL_ERROR_2;
228 "xsk_socket__create() failed (is linux netdev %s up?)",
233 fd = xsk_socket__fd (*xsk);
234 optlen =
sizeof (opt);
235 if (getsockopt (fd, SOL_XDP, XDP_OPTIONS, &opt, &optlen))
237 args->
rv = VNET_API_ERROR_SYSCALL_ERROR_3;
242 if (opt.flags & XDP_OPTIONS_ZEROCOPY)
243 ad->
flags |= AF_XDP_DEVICE_F_ZEROCOPY;
245 rxq->
xsk_fd = qid < rxq_num ? fd : -1;
246 txq->
xsk_fd = qid < txq_num ? fd : -1;
251 xsk_socket__delete (*xsk);
253 xsk_umem__delete (*umem);
267 s = (
char *)
format (0,
"/sys/class/net/%s/device/numa_node%c", ifname, 0);
268 fptr = fopen (s,
"rb");
274 if (fscanf (fptr,
"%d\n", &numa) != 1)
302 int rxq_num, txq_num, q_num;
312 args->
rv = VNET_API_ERROR_INVALID_VALUE;
321 args->
rv = VNET_API_ERROR_INVALID_VALUE;
324 "queue size must be a power of two between %i and 65535",
337 q_num =
clib_max (rxq_num, txq_num);
343 for (i = 0; i < q_num; i++)
395 args->
rv = VNET_API_ERROR_INVALID_INTERFACE;
409 for (i = 0; i < rxq_num; i++)
449 if (ad->
flags & AF_XDP_DEVICE_F_ERROR)
456 ad->
flags |= AF_XDP_DEVICE_F_ADMIN_UP;
461 ad->
flags &= ~AF_XDP_DEVICE_F_ADMIN_UP;
475 if (node_index == ~0)
503 .name =
"AF_XDP interface",
vlib_log_class_t vlib_log_register_class(char *class, char *subclass)
#define vec_validate(V, I)
Make sure vector is long enough for given index (no header, unspecified alignment) ...
u32 flags
buffer flags: VLIB_BUFFER_FREE_LIST_INDEX_MASK: bits used to store free list index, VLIB_BUFFER_IS_TRACED: trace this buffer.
#define UNIX_FILE_EVENT_EDGE_TRIGGERED
u8 * format_clib_error(u8 *s, va_list *va)
#define vec_set_len(v, l)
Set vector length to a user-defined value.
vl_api_wireguard_peer_flags_t flags
af_xdp_main_t af_xdp_main
void ethernet_delete_interface(vnet_main_t *vnm, u32 hw_if_index)
vnet_device_class_t af_xdp_device_class
vnet_main_t * vnet_get_main(void)
#define pool_get_zero(P, E)
Allocate an object E from a pool P and zero it.
u32 per_interface_next_index
format_function_t format_af_xdp_device_name
static vnet_hw_interface_t * vnet_get_hw_interface(vnet_main_t *vnm, u32 hw_if_index)
vlib_buffer_main_t * buffer_main
#define XDP_FLAGS_REPLACE
void af_xdp_create_if(vlib_main_t *vm, af_xdp_create_if_args_t *args)
#define vec_validate_aligned(V, I, A)
Make sure vector is long enough for given index (no header, specified alignment)
static uword vlib_node_add_next(vlib_main_t *vm, uword node, uword next_node)
af_xdp_device_t * devices
u8 buffer_pool_index
index of buffer pool this buffer belongs.
static int af_xdp_load_program(af_xdp_create_if_args_t *args, af_xdp_device_t *ad)
static void clib_spinlock_free(clib_spinlock_t *p)
static vnet_sw_interface_t * vnet_get_hw_sw_interface(vnet_main_t *vnm, u32 hw_if_index)
vlib_log_class_t log_class
vlib_buffer_t * buffer_template
gdb_af_xdp_pair_t gdb_af_xdp_get_prod(const struct xsk_ring_prod *prod)
VNET_DEVICE_CLASS(af_xdp_device_class)
#define VLIB_INIT_FUNCTION(x)
static_always_inline void vnet_device_input_set_interrupt_pending(vnet_main_t *vnm, u32 hw_if_index, u16 queue_id)
vnet_hw_interface_flags_t flags
#define vec_elt_at_index(v, i)
Get vector value at index i checking that i is in bounds.
#define clib_error_return(e, args...)
clib_file_main_t file_main
static void clib_spinlock_init(clib_spinlock_t *p)
#define pool_elt_at_index(p, i)
Returns pointer to element at given index.
#define clib_error_return_unix(e, args...)
#define pool_put(P, E)
Free an object E in pool P.
gdb_af_xdp_pair_t gdb_af_xdp_get_cons(const struct xsk_ring_cons *cons)
static void af_xdp_clear(u32 dev_instance)
static clib_error_t * af_xdp_mac_change(vnet_hw_interface_t *hw, const u8 *old, const u8 *new)
static_always_inline u32 vlib_buffer_get_default_data_size(vlib_main_t *vm)
static char * af_xdp_tx_func_error_strings[]
static void af_xdp_set_interface_next_node(vnet_main_t *vnm, u32 hw_if_index, u32 node_index)
sll srl srl sll sra u16x4 i
#define vec_free(V)
Free vector's memory (no header).
#define ETHERNET_INTERFACE_FLAG_MTU
#define ETHERNET_INTERFACE_FLAG_ACCEPT_ALL
static u32 af_xdp_flag_change(vnet_main_t *vnm, vnet_hw_interface_t *hw, u32 flags)
format_function_t format_af_xdp_device
#define uword_to_pointer(u, type)
void vnet_hw_interface_assign_rx_thread(vnet_main_t *vnm, u32 hw_if_index, u16 queue_id, uword thread_index)
clib_error_t * af_xdp_init(vlib_main_t *vm)
static uword clib_file_add(clib_file_main_t *um, clib_file_t *template)
static void clib_file_del_by_index(clib_file_main_t *um, uword index)
unsigned int if_nametoindex(const char *ifname)
static vlib_main_t * vlib_get_main(void)
static uword is_pow2(uword x)
#define foreach_af_xdp_tx_func_error
static clib_error_t * af_xdp_interface_admin_up_down(vnet_main_t *vnm, u32 hw_if_index, u32 flags)
errno_t memcpy_s(void *__restrict__ dest, rsize_t dmax, const void *__restrict__ src, rsize_t n)
copy src to dest, at most n bytes, up to dmax
static void ethernet_mac_address_generate(u8 *mac)
clib_error_t * ethernet_register_interface(vnet_main_t *vnm, u32 dev_class_index, u32 dev_instance, const u8 *address, u32 *hw_if_index_return, ethernet_flag_change_function_t flag_change)
clib_error_t * vnet_hw_interface_set_flags(vnet_main_t *vnm, u32 hw_if_index, vnet_hw_interface_flags_t flags)
VLIB buffer representation.
#define clib_error_free(e)
int vnet_hw_interface_unassign_rx_thread(vnet_main_t *vnm, u32 hw_if_index, u16 queue_id)
static int af_xdp_create_queue(vlib_main_t *vm, af_xdp_create_if_args_t *args, af_xdp_device_t *ad, int qid, int rxq_num, int txq_num)
static vlib_thread_main_t * vlib_get_thread_main()
#define clib_error_return_code(e, code, flags, args...)
vlib_node_registration_t af_xdp_input_node
(constructor) VLIB_REGISTER_NODE (af_xdp_input_node)
#define vec_foreach(var, vec)
Vector iterator.
#define vlib_log_err(...)
#define CLIB_CACHE_LINE_BYTES
static int af_xdp_get_numa(const char *ifname)
static u8 vlib_buffer_pool_get_default_for_numa(vlib_main_t *vm, u32 numa_node)
#define af_xdp_log(lvl, dev, f,...)
void af_xdp_delete_if(vlib_main_t *vm, af_xdp_device_t *ad)
volatile u8 ref_count
Reference count for this buffer.
static void vnet_hw_interface_set_input_node(vnet_main_t *vnm, u32 hw_if_index, u32 node_index)
struct bpf_object * bpf_obj
static clib_error_t * af_xdp_device_rxq_read_ready(clib_file_t *f)