FD.io VPP  v19.01.2-3-gf61a1a8
Vector Packet Processing
output.c
Go to the documentation of this file.
1 /*
2  *------------------------------------------------------------------
3  * Copyright (c) 2018 Cisco and/or its affiliates.
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at:
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  *------------------------------------------------------------------
16  */
17 
18 #include <vlib/vlib.h>
19 #include <vlib/unix/unix.h>
20 #include <vlib/pci/pci.h>
21 #include <vnet/ethernet/ethernet.h>
22 #include <vnet/devices/devices.h>
23 
24 #include <vmxnet3/vmxnet3.h>
25 
28 {
29  vmxnet3_tx_comp_ring *comp_ring = &txq->tx_comp_ring;
30 
31  comp_ring->next++;
32  if (PREDICT_FALSE (comp_ring->next == txq->size))
33  {
34  comp_ring->next = 0;
35  comp_ring->gen ^= VMXNET3_TXCF_GEN;
36  }
37 }
38 
41 {
42  txq->tx_ring.produce++;
43  if (PREDICT_FALSE (txq->tx_ring.produce == txq->size))
44  {
45  txq->tx_ring.produce = 0;
46  txq->tx_ring.gen ^= VMXNET3_TXF_GEN;
47  }
48 }
49 
52 {
53  txq->tx_ring.consume++;
54  txq->tx_ring.consume &= txq->size - 1;
55 }
56 
59  vmxnet3_txq_t * txq)
60 {
61  vmxnet3_tx_comp *tx_comp;
62  vmxnet3_tx_comp_ring *comp_ring;
63 
64  comp_ring = &txq->tx_comp_ring;
65  tx_comp = &txq->tx_comp[comp_ring->next];
66 
67  while ((tx_comp->flags & VMXNET3_TXCF_GEN) == comp_ring->gen)
68  {
69  u16 eop_idx = tx_comp->index & VMXNET3_TXC_INDEX;
70  u32 bi0 = txq->tx_ring.bufs[txq->tx_ring.consume];
71 
72  vlib_buffer_free_one (vm, bi0);
73  while (txq->tx_ring.consume != eop_idx)
74  {
76  }
78 
80  tx_comp = &txq->tx_comp[comp_ring->next];
81  }
82 }
83 
86 {
87  u16 count;
88 
89  count = (txq->tx_ring.consume - txq->tx_ring.produce - 1);
90  /* Wrapped? */
91  if (txq->tx_ring.produce >= txq->tx_ring.consume)
92  count += txq->size;
93  return count;
94 }
95 
97  vlib_node_runtime_t * node,
98  vlib_frame_t * frame)
99 {
100  vmxnet3_main_t *vmxm = &vmxnet3_main;
101  vnet_interface_output_runtime_t *rd = (void *) node->runtime_data;
103  u32 *buffers = vlib_frame_vector_args (frame);
104  u32 bi0;
105  vlib_buffer_t *b0;
106  vmxnet3_tx_desc *txd = 0;
107  u32 desc_idx, generation, first_idx;
108  u16 space_left;
109  u16 n_left = frame->n_vectors;
110  vmxnet3_txq_t *txq;
111  u32 thread_index = vm->thread_index;
112  u16 qid = thread_index, produce;
113 
114  if (PREDICT_FALSE (!(vd->flags & VMXNET3_DEVICE_F_LINK_UP)))
115  {
116  vlib_buffer_free (vm, buffers, n_left);
117  vlib_error_count (vm, node->node_index, VMXNET3_TX_ERROR_LINK_DOWN,
118  n_left);
119  return (0);
120  }
121 
122  txq = vec_elt_at_index (vd->txqs, qid % vd->num_tx_queues);
124 
125  vmxnet3_txq_release (vm, vd, txq);
126 
127  produce = txq->tx_ring.produce;
128  while (PREDICT_TRUE (n_left))
129  {
130  u16 space_needed = 1, i;
131  vlib_buffer_t *b;
132 
133  bi0 = buffers[0];
134  b0 = vlib_get_buffer (vm, bi0);
135  b = b0;
136 
137  space_left = vmxnet3_tx_ring_space_left (txq);
138  while (b->flags & VLIB_BUFFER_NEXT_PRESENT)
139  {
140  u32 next_buffer = b->next_buffer;
141 
142  b = vlib_get_buffer (vm, next_buffer);
143  space_needed++;
144  }
145  if (PREDICT_FALSE (space_left < space_needed))
146  {
147  vmxnet3_txq_release (vm, vd, txq);
148  space_left = vmxnet3_tx_ring_space_left (txq);
149 
150  if (PREDICT_FALSE (space_left < space_needed))
151  {
152  vlib_buffer_free_one (vm, bi0);
153  vlib_error_count (vm, node->node_index,
154  VMXNET3_TX_ERROR_NO_FREE_SLOTS, 1);
155  buffers++;
156  n_left--;
157  /*
158  * Drop this packet. But we may have enough room for the next
159  * packet
160  */
161  continue;
162  }
163  }
164 
165  /*
166  * Toggle the generation bit for SOP fragment to avoid device starts
167  * reading incomplete packet
168  */
169  generation = txq->tx_ring.gen ^ VMXNET3_TXF_GEN;
170  first_idx = txq->tx_ring.produce;
171  for (i = 0; i < space_needed; i++)
172  {
173  b0 = vlib_get_buffer (vm, bi0);
175 
176  desc_idx = txq->tx_ring.produce;
177 
179  txq->tx_ring.bufs[desc_idx] = bi0;
180 
181  txd = &txq->tx_desc[desc_idx];
182  txd->address = vlib_buffer_get_current_pa (vm, b0);
183 
184  txd->flags[0] = generation | b0->current_length;
185 
186  generation = txq->tx_ring.gen;
187 
188  txd->flags[1] = 0;
189  bi0 = b0->next_buffer;
190  }
191 
192  txd->flags[1] = VMXNET3_TXF_CQ | VMXNET3_TXF_EOP;
193  asm volatile ("":::"memory");
194  /*
195  * Now toggle back the generation bit for the first segment.
196  * Device can start reading the packet
197  */
198  txq->tx_desc[first_idx].flags[0] ^= VMXNET3_TXF_GEN;
199 
200  buffers++;
201  n_left--;
202  }
203 
204  if (PREDICT_TRUE (produce != txq->tx_ring.produce))
206  txq->tx_ring.produce);
207 
209 
210  return (frame->n_vectors - n_left);
211 }
212 
213 /*
214  * fd.io coding-style-patch-verification: ON
215  *
216  * Local Variables:
217  * eval: (c-set-style "gnu")
218  * End:
219  */
static_always_inline void vmxnet3_tx_ring_advance_consume(vmxnet3_txq_t *txq)
Definition: output.c:51
static uword vlib_buffer_get_current_pa(vlib_main_t *vm, vlib_buffer_t *b)
Definition: buffer_funcs.h:328
static void vlib_buffer_free(vlib_main_t *vm, u32 *buffers, u32 n_buffers)
Free buffers Frees the entire buffer chain for each buffer.
Definition: buffer_funcs.h:529
#define VMXNET3_REG_TXPROD
Definition: vmxnet3.h:62
#define PREDICT_TRUE(x)
Definition: clib.h:112
static void vlib_error_count(vlib_main_t *vm, uword node_index, uword counter, uword increment)
Definition: error_funcs.h:57
clib_spinlock_t lock
Definition: vmxnet3.h:448
static_always_inline void clib_spinlock_unlock_if_init(clib_spinlock_t *p)
Definition: lock.h:98
u32 thread_index
Definition: main.h:179
vmxnet3_main_t vmxnet3_main
Definition: vmxnet3.c:28
int i
VNET_DEVICE_CLASS_TX_FN() vmxnet3_device_class(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
Definition: output.c:96
#define static_always_inline
Definition: clib.h:99
#define vec_elt_at_index(v, i)
Get vector value at index i checking that i is in bounds.
#define VNET_DEVICE_CLASS_TX_FN(devclass)
Definition: interface.h:287
unsigned int u32
Definition: types.h:88
#define pool_elt_at_index(p, i)
Returns pointer to element at given index.
Definition: pool.h:511
#define VMXNET3_TXC_INDEX
Definition: vmxnet3.h:93
u16 current_length
Nbytes between current data and the end of this buffer.
Definition: buffer.h:114
static_always_inline u16 vmxnet3_tx_ring_space_left(vmxnet3_txq_t *txq)
Definition: output.c:85
unsigned short u16
Definition: types.h:57
static_always_inline void vmxnet3_tx_ring_advance_produce(vmxnet3_txq_t *txq)
Definition: output.c:40
vmxnet3_tx_comp_ring tx_comp_ring
Definition: vmxnet3.h:453
vmxnet3_tx_comp * tx_comp
Definition: vmxnet3.h:451
#define PREDICT_FALSE(x)
Definition: clib.h:111
#define VMXNET3_TXF_CQ
Definition: vmxnet3.h:91
#define VMXNET3_TXF_GEN
Definition: vmxnet3.h:89
vlib_main_t * vm
Definition: buffer.c:301
static_always_inline void vmxnet3_txq_release(vlib_main_t *vm, vmxnet3_device_t *vd, vmxnet3_txq_t *txq)
Definition: output.c:58
u32 next_buffer
Next buffer for this linked-list of buffers.
Definition: buffer.h:130
#define VMXNET3_TXCF_GEN
Definition: vmxnet3.h:92
size_t count
Definition: vapi.c:47
#define VMXNET3_TXF_EOP
Definition: vmxnet3.h:90
static_always_inline void vmxnet3_tx_comp_ring_advance_next(vmxnet3_txq_t *txq)
Definition: output.c:27
#define VLIB_BUFFER_TRACE_TRAJECTORY_INIT(b)
Definition: buffer.h:495
vmxnet3_tx_desc * tx_desc
Definition: vmxnet3.h:450
static void * vlib_frame_vector_args(vlib_frame_t *f)
Get pointer to frame vector data.
Definition: node_funcs.h:274
vmxnet3_device_t * devices
Definition: vmxnet3.h:498
static void vlib_buffer_free_one(vlib_main_t *vm, u32 buffer_index)
Free one buffer Shorthand to free a single buffer chain.
Definition: buffer_funcs.h:570
u32 flags
buffer flags: VLIB_BUFFER_FREE_LIST_INDEX_MASK: bits used to store free list index, VLIB_BUFFER_IS_TRACED: trace this buffer.
Definition: buffer.h:117
vmxnet3_tx_ring tx_ring
Definition: vmxnet3.h:452
static_always_inline void clib_spinlock_lock_if_init(clib_spinlock_t *p)
Definition: lock.h:82
static vlib_buffer_t * vlib_get_buffer(vlib_main_t *vm, u32 buffer_index)
Translate buffer index into buffer pointer.
Definition: buffer_funcs.h:62
static_always_inline void vmxnet3_reg_write_inline(vmxnet3_device_t *vd, u8 bar, u32 addr, u32 val)
Definition: vmxnet3.h:548