FD.io VPP  v21.01.1
Vector Packet Processing
buffer.h
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2015 Cisco and/or its affiliates.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at:
6  *
7  * http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 /*
16  * buffer.h: VLIB buffers
17  *
18  * Copyright (c) 2008 Eliot Dresselhaus
19  *
20  * Permission is hereby granted, free of charge, to any person obtaining
21  * a copy of this software and associated documentation files (the
22  * "Software"), to deal in the Software without restriction, including
23  * without limitation the rights to use, copy, modify, merge, publish,
24  * distribute, sublicense, and/or sell copies of the Software, and to
25  * permit persons to whom the Software is furnished to do so, subject to
26  * the following conditions:
27  *
28  * The above copyright notice and this permission notice shall be
29  * included in all copies or substantial portions of the Software.
30  *
31  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
32  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
33  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
34  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
35  * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
36  * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
37  * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
38  */
39 
40 #ifndef included_vlib_buffer_h
41 #define included_vlib_buffer_h
42 
43 #include <vppinfra/types.h>
44 #include <vppinfra/cache.h>
45 #include <vppinfra/serialize.h>
46 #include <vppinfra/vector.h>
47 #include <vppinfra/lock.h>
48 #include <vlib/error.h> /* for vlib_error_t */
49 
50 #include <vlib/config.h> /* for __PRE_DATA_SIZE */
51 #define VLIB_BUFFER_PRE_DATA_SIZE __PRE_DATA_SIZE
52 
53 #define VLIB_BUFFER_DEFAULT_DATA_SIZE (2048)
54 
55 /* Minimum buffer chain segment size. Does not apply to last buffer in chain.
56  Dataplane code can safely asume that specified amount of data is not split
57  into 2 chained buffers */
58 #define VLIB_BUFFER_MIN_CHAIN_SEG_SIZE (128)
59 
60 /* Amount of head buffer data copied to each replica head buffer */
61 #define VLIB_BUFFER_CLONE_HEAD_SIZE (256)
62 
63 /** \file
64  vlib buffer structure definition and a few select
65  access methods. This structure and the buffer allocation
66  mechanism should perhaps live in vnet, but it would take a lot
67  of typing to make it so.
68 */
69 
70 /**
71  * Buffer Flags
72  */
73 #define foreach_vlib_buffer_flag \
74  _( 0, IS_TRACED, 0) \
75  _( 1, NEXT_PRESENT, "next-present") \
76  _( 2, TOTAL_LENGTH_VALID, 0) \
77  _( 3, EXT_HDR_VALID, "ext-hdr-valid")
78 
79 /* NOTE: only buffer generic flags should be defined here, please consider
80  using user flags. i.e. src/vnet/buffer.h */
81 
82 enum
83 {
84 #define _(bit, name, v) VLIB_BUFFER_##name = (1 << (bit)),
86 #undef _
87 };
88 
89 enum
90 {
91 #define _(bit, name, v) VLIB_BUFFER_LOG2_##name = (bit),
93 #undef _
94 };
95 
96  /* User defined buffer flags. */
97 #define LOG2_VLIB_BUFFER_FLAG_USER(n) (32 - (n))
98 #define VLIB_BUFFER_FLAG_USER(n) (1 << LOG2_VLIB_BUFFER_FLAG_USER(n))
99 #define VLIB_BUFFER_FLAGS_ALL (0x0f)
100 
101 /** VLIB buffer representation. */
102 typedef union
103 {
104  struct
105  {
106  CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
107 
108  /** signed offset in data[], pre_data[] that we are currently
109  * processing. If negative current header points into predata area. */
111 
112  /** Nbytes between current data and the end of this buffer. */
114 
115  /** buffer flags:
116  <br> VLIB_BUFFER_FREE_LIST_INDEX_MASK: bits used to store free list index,
117  <br> VLIB_BUFFER_IS_TRACED: trace this buffer.
118  <br> VLIB_BUFFER_NEXT_PRESENT: this is a multi-chunk buffer.
119  <br> VLIB_BUFFER_TOTAL_LENGTH_VALID: as it says
120  <br> VLIB_BUFFER_EXT_HDR_VALID: buffer contains valid external buffer manager header,
121  set to avoid adding it to a flow report
122  <br> VLIB_BUFFER_FLAG_USER(n): user-defined bit N
123  */
125 
126  /** Generic flow identifier */
128 
129  /** Reference count for this buffer. */
130  volatile u8 ref_count;
131 
132  /** index of buffer pool this buffer belongs. */
134 
135  /** Error code for buffers to be enqueued to error handler. */
137 
138  /** Next buffer for this linked-list of buffers. Only valid if
139  * VLIB_BUFFER_NEXT_PRESENT flag is set. */
141 
142  /** The following fields can be in a union because once a packet enters
143  * the punt path, it is no longer on a feature arc */
144  union
145  {
146  /** Used by feature subgraph arcs to visit enabled feature nodes */
148  /* the reason the packet once punted */
150  };
151 
152  /** Opaque data used by sub-graphs for their own purposes. */
153  u32 opaque[10];
154 
155  /** part of buffer metadata which is initialized on alloc ends here. */
156  STRUCT_MARK (template_end);
157 
158  /** start of 2nd half (2nd cacheline on systems where cacheline size is 64) */
159  CLIB_ALIGN_MARK (second_half, 64);
160 
161  /** Specifies trace buffer handle if VLIB_PACKET_IS_TRACED flag is
162  * set. */
164 
165  /** Only valid for first buffer in chain. Current length plus total length
166  * given here give total number of bytes in buffer chain. */
168 
169  /**< More opaque data, see ../vnet/vnet/buffer.h */
170  u32 opaque2[14];
171 
172  /** start of buffer headroom */
173  CLIB_ALIGN_MARK (headroom, 64);
174 
175  /** Space for inserting data before buffer start. Packet rewrite string
176  * will be rewritten backwards and may extend back before
177  * buffer->data[0]. Must come directly before packet data. */
179 
180  /** Packet data */
181  u8 data[];
182  };
183 #ifdef CLIB_HAVE_VEC128
184  u8x16 as_u8x16[4];
185 #endif
186 #ifdef CLIB_HAVE_VEC256
187  u8x32 as_u8x32[2];
188 #endif
189 #ifdef CLIB_HAVE_VEC512
190  u8x64 as_u8x64[1];
191 #endif
192 } vlib_buffer_t;
193 
196  "VLIB_BUFFER_PRE_DATA_SIZE must be divisible by cache line size");
197 
198 #define VLIB_BUFFER_HDR_SIZE (sizeof(vlib_buffer_t) - VLIB_BUFFER_PRE_DATA_SIZE)
199 
200 /** \brief Prefetch buffer metadata.
201  The first 64 bytes of buffer contains most header information
202 
203  @param b - (vlib_buffer_t *) pointer to the buffer
204  @param type - LOAD, STORE. In most cases, STORE is the right answer
205 */
206 
207 #define vlib_prefetch_buffer_header(b,type) CLIB_PREFETCH (b, 64, type)
208 #define vlib_prefetch_buffer_data(b,type) \
209  CLIB_PREFETCH (vlib_buffer_get_current(b), CLIB_CACHE_LINE_BYTES, type)
210 
211 always_inline void
213 {
214  ASSERT (sizeof (b[0]) % 64 == 0);
215 
216  /* Rewrite data must be before and contiguous with packet data. */
218 }
219 
222 {
223  return pointer_to_uword (b->data);
224 }
225 
226 /** \brief Get pointer to current data to process
227 
228  @param b - (vlib_buffer_t *) pointer to the buffer
229  @return - (void *) (b->data + b->current_data)
230 */
231 
232 always_inline void *
234 {
235  /* Check bounds. */
236  ASSERT ((signed) b->current_data >= (signed) -VLIB_BUFFER_PRE_DATA_SIZE);
237  return b->data + b->current_data;
238 }
239 
242 {
243  return vlib_buffer_get_va (b) + b->current_data;
244 }
245 
246 /** \brief Advance current data pointer by the supplied (signed!) amount
247 
248  @param b - (vlib_buffer_t *) pointer to the buffer
249  @param l - (word) signed increment
250 */
251 always_inline void
253 {
254  ASSERT (b->current_length >= l);
255  b->current_data += l;
256  b->current_length -= l;
257 
258  ASSERT ((b->flags & VLIB_BUFFER_NEXT_PRESENT) == 0 ||
260 }
261 
262 /** \brief Check if there is enough space in buffer to advance
263 
264  @param b - (vlib_buffer_t *) pointer to the buffer
265  @param l - (word) size to check
266  @return - 0 if there is less space than 'l' in buffer
267 */
270 {
271  return b->current_length >= l;
272 }
273 
274 /** \brief Reset current header & length to state they were in when
275  packet was received.
276 
277  @param b - (vlib_buffer_t *) pointer to the buffer
278 */
279 
280 always_inline void
282 {
283  b->current_length += clib_max (b->current_data, 0);
284  b->current_data = 0;
285 }
286 
287 /** \brief Get pointer to buffer's opaque data array
288 
289  @param b - (vlib_buffer_t *) pointer to the buffer
290  @return - (void *) b->opaque
291 */
292 always_inline void *
294 {
295  return (void *) b->opaque;
296 }
297 
298 /** \brief Get pointer to buffer's opaque2 data array
299 
300  @param b - (vlib_buffer_t *) pointer to the buffer
301  @return - (void *) b->opaque2
302 */
303 always_inline void *
305 {
306  return (void *) b->opaque2;
307 }
308 
309 /** \brief Get pointer to the end of buffer's data
310  * @param b pointer to the buffer
311  * @return pointer to tail of packet's data
312  */
315 {
316  return b->data + b->current_data + b->current_length;
317 }
318 
319 /** \brief Append uninitialized data to buffer
320  * @param b pointer to the buffer
321  * @param size number of uninitialized bytes
322  * @return pointer to beginning of uninitialized data
323  */
324 always_inline void *
326 {
327  void *p = vlib_buffer_get_tail (b);
328  /* XXX make sure there's enough space */
329  b->current_length += size;
330  return p;
331 }
332 
333 /** \brief Prepend uninitialized data to buffer
334  * @param b pointer to the buffer
335  * @param size number of uninitialized bytes
336  * @return pointer to beginning of uninitialized data
337  */
338 always_inline void *
340 {
342  b->current_data -= size;
343  b->current_length += size;
344 
345  return vlib_buffer_get_current (b);
346 }
347 
348 /** \brief Make head room, typically for packet headers
349  * @param b pointer to the buffer
350  * @param size number of head room bytes
351  * @return pointer to start of buffer (current data)
352  */
353 always_inline void *
355 {
356  b->current_data += size;
357  return vlib_buffer_get_current (b);
358 }
359 
360 /** \brief Construct a trace handle from thread and pool index
361  * @param thread Thread id
362  * @param pool_index Pool index
363  * @return trace handle
364  */
367 {
368  u32 rv;
369  ASSERT (thread < 0xff);
370  ASSERT (pool_index < 0x00FFFFFF);
371  rv = (thread << 24) | (pool_index & 0x00FFFFFF);
372  return rv;
373 }
374 
375 /** \brief Extract the thread id from a trace handle
376  * @param trace_handle the trace handle
377  * @return the thread id
378  */
381 {
382  u32 trace_handle = b->trace_handle;
383 
384  return trace_handle >> 24;
385 }
386 
387 /** \brief Extract the trace (pool) index from a trace handle
388  * @param trace_handle the trace handle
389  * @return the trace index
390  */
393 {
394  u32 trace_handle = b->trace_handle;
395  return trace_handle & 0x00FFFFFF;
396 }
397 
398 /** \brief Retrieve bytes from buffer head
399  * @param b pointer to the buffer
400  * @param size number of bytes to pull
401  * @return pointer to start of buffer (current data)
402  */
403 always_inline void *
405 {
407  return 0;
408 
409  void *data = vlib_buffer_get_current (b);
410  vlib_buffer_advance (b, size);
411  return data;
412 }
413 
414 /* Forward declaration. */
415 struct vlib_main_t;
416 
417 #define VLIB_BUFFER_POOL_PER_THREAD_CACHE_SZ 512
418 
419 typedef struct
420 {
421  CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
425 
426 typedef struct
427 {
428  CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
441 
442  /* per-thread data */
444 
445  /* buffer metadata template */
448 
449 #define VLIB_BUFFER_MAX_NUMA_NODES 32
450 
451 typedef struct
452 {
453  CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
454  /* Virtual memory address and size of buffer memory, used for calculating
455  buffer index */
459 
460  /* Hash table mapping buffer index into number
461  0 => allocated but free, 1 => allocated and not-free.
462  If buffer index is not in hash table then this buffer
463  has never been allocated. */
466  u8 default_buffer_pool_index_for_numa[VLIB_BUFFER_MAX_NUMA_NODES];
467 
468  /* config */
473 
474  /* logging */
477 
479 
480 /*
481  */
482 
483 /** \brief Compile time buffer trajectory tracing option
484  Turn this on if you run into "bad monkey" contexts,
485  and you want to know exactly which nodes they've visited...
486  See vlib/main.c...
487 */
488 #define VLIB_BUFFER_TRACE_TRAJECTORY 0
489 
490 #if VLIB_BUFFER_TRACE_TRAJECTORY > 0
491 extern void (*vlib_buffer_trace_trajectory_cb) (vlib_buffer_t * b, u32 index);
492 extern void (*vlib_buffer_trace_trajectory_init_cb) (vlib_buffer_t * b);
493 extern void vlib_buffer_trace_trajectory_init (vlib_buffer_t * b);
494 #define VLIB_BUFFER_TRACE_TRAJECTORY_INIT(b) \
495  vlib_buffer_trace_trajectory_init (b);
496 #else
497 #define VLIB_BUFFER_TRACE_TRAJECTORY_INIT(b)
498 #endif /* VLIB_BUFFER_TRACE_TRAJECTORY */
499 
500 extern u16 __vlib_buffer_external_hdr_size;
501 #define VLIB_BUFFER_SET_EXT_HDR_SIZE(x) \
502 static void __clib_constructor \
503 vnet_buffer_set_ext_hdr_size() \
504 { \
505  if (__vlib_buffer_external_hdr_size) \
506  clib_error ("buffer external header space already set"); \
507  __vlib_buffer_external_hdr_size = CLIB_CACHE_LINE_ROUND (x); \
508 }
509 
510 #endif /* included_vlib_buffer_h */
511 
512 /*
513  * fd.io coding-style-patch-verification: ON
514  *
515  * Local Variables:
516  * eval: (c-set-style "gnu")
517  * End:
518  */
u32 opaque2[14]
Definition: buffer.h:170
u32 flags
buffer flags: VLIB_BUFFER_FREE_LIST_INDEX_MASK: bits used to store free list index, VLIB_BUFFER_IS_TRACED: trace this buffer.
Definition: buffer.h:124
#define CLIB_CACHE_LINE_ALIGN_MARK(mark)
Definition: cache.h:60
static u8 * vlib_buffer_get_tail(vlib_buffer_t *b)
Get pointer to the end of buffer&#39;s data.
Definition: buffer.h:314
static uword vlib_buffer_get_current_va(vlib_buffer_t *b)
Definition: buffer.h:241
static u32 vlib_buffer_make_trace_handle(u32 thread, u32 pool_index)
Construct a trace handle from thread and pool index.
Definition: buffer.h:366
static void vlib_buffer_reset(vlib_buffer_t *b)
Reset current header & length to state they were in when packet was received.
Definition: buffer.h:281
static u32 vlib_buffer_get_trace_index(vlib_buffer_t *b)
Extract the trace (pool) index from a trace handle.
Definition: buffer.h:392
i16 current_data
signed offset in data[], pre_data[] that we are currently processing.
Definition: buffer.h:110
#define VLIB_BUFFER_PRE_DATA_SIZE
Definition: buffer.h:51
u32 physmem_map_index
Definition: buffer.h:434
u16 current_length
Nbytes between current data and the end of this buffer.
Definition: buffer.h:113
u16 vlib_error_t
Definition: error.h:43
#define STRUCT_MARK(mark)
Definition: clib.h:75
vlib_main_t * vm
Definition: in2out_ed.c:1580
uword log2_page_size
Definition: buffer.h:431
unsigned char u8
Definition: types.h:56
u8 data[128]
Definition: ipsec_types.api:90
u8 buffer_pool_index
index of buffer pool this buffer belongs.
Definition: buffer.h:133
vlib_buffer_pool_thread_t * threads
Definition: buffer.h:443
u32 vlib_log_class_t
Definition: vlib.h:51
i64 word
Definition: types.h:111
STATIC_ASSERT_SIZEOF(vlib_buffer_t, 128+VLIB_BUFFER_PRE_DATA_SIZE)
uword buffer_mem_size
Definition: buffer.h:457
const cJSON *const b
Definition: cJSON.h:255
pthread_t thread[MAX_CONNS]
Definition: main.c:142
unsigned int u32
Definition: types.h:88
vlib_error_t error
Error code for buffers to be enqueued to error handler.
Definition: buffer.h:136
static void * vlib_buffer_make_headroom(vlib_buffer_t *b, u8 size)
Make head room, typically for packet headers.
Definition: buffer.h:354
unsigned short u16
Definition: types.h:57
clib_mem_page_sz_t
Definition: mem.h:57
u32 size
Definition: vhost_user.h:106
vlib_buffer_t buffer_template
Definition: buffer.h:446
static void * vlib_buffer_get_current(vlib_buffer_t *b)
Get pointer to current data to process.
Definition: buffer.h:233
vlib_buffer_pool_t * buffer_pools
Definition: buffer.h:458
u32 trace_handle
Specifies trace buffer handle if VLIB_PACKET_IS_TRACED flag is set.
Definition: buffer.h:163
#define always_inline
Definition: ipsec.h:28
vlib_log_class_t log_default
Definition: buffer.h:475
u32 punt_reason
Definition: buffer.h:149
clib_mem_page_sz_t log2_page_size
Definition: buffer.h:472
clib_spinlock_t buffer_known_hash_lockp
Definition: buffer.h:465
u32 flow_id
Generic flow identifier.
Definition: buffer.h:127
#define VLIB_BUFFER_MAX_NUMA_NODES
Definition: buffer.h:449
u8 data[]
Packet data.
Definition: buffer.h:181
static u32 vlib_buffer_get_trace_thread(vlib_buffer_t *b)
Extract the thread id from a trace handle.
Definition: buffer.h:380
clib_spinlock_t lock
Definition: buffer.h:440
static void * vlib_get_buffer_opaque2(vlib_buffer_t *b)
Get pointer to buffer&#39;s opaque2 data array.
Definition: buffer.h:304
u32 current_config_index
Used by feature subgraph arcs to visit enabled feature nodes.
Definition: buffer.h:147
u32 buffers_per_numa
Definition: buffer.h:469
#define ASSERT(truth)
u8 pre_data[VLIB_BUFFER_PRE_DATA_SIZE]
Space for inserting data before buffer start.
Definition: buffer.h:178
static void vlib_buffer_advance(vlib_buffer_t *b, word l)
Advance current data pointer by the supplied (signed!) amount.
Definition: buffer.h:252
static uword pointer_to_uword(const void *p)
Definition: types.h:131
#define clib_max(x, y)
Definition: clib.h:321
static void * vlib_buffer_push_uninit(vlib_buffer_t *b, u8 size)
Prepend uninitialized data to buffer.
Definition: buffer.h:339
static u8 vlib_buffer_has_space(vlib_buffer_t *b, word l)
Check if there is enough space in buffer to advance.
Definition: buffer.h:269
#define CLIB_ALIGN_MARK(name, alignment)
Definition: clib.h:90
#define VLIB_BUFFER_POOL_PER_THREAD_CACHE_SZ
Definition: buffer.h:417
static void * vlib_get_buffer_opaque(vlib_buffer_t *b)
Get pointer to buffer&#39;s opaque data array.
Definition: buffer.h:293
u32 next_buffer
Next buffer for this linked-list of buffers.
Definition: buffer.h:140
VLIB buffer representation.
Definition: buffer.h:102
u64 uword
Definition: types.h:112
uword buffer_mem_start
Definition: buffer.h:456
uword * buffer_known_hash
Definition: buffer.h:464
u32 default_data_size
Definition: buffer.h:471
u32 index
Definition: flow_types.api:221
#define VLIB_BUFFER_MIN_CHAIN_SEG_SIZE
Definition: buffer.h:58
clib_error_t * vlib_buffer_main_init(struct vlib_main_t *vm)
Definition: buffer.c:852
static uword vlib_buffer_get_va(vlib_buffer_t *b)
Definition: buffer.h:221
static void * vlib_buffer_pull(vlib_buffer_t *b, u8 size)
Retrieve bytes from buffer head.
Definition: buffer.h:404
#define foreach_vlib_buffer_flag
Buffer Flags.
Definition: buffer.h:73
static void * vlib_buffer_put_uninit(vlib_buffer_t *b, u16 size)
Append uninitialized data to buffer.
Definition: buffer.h:325
#define CLIB_CACHE_LINE_BYTES
Definition: cache.h:59
u32 total_length_not_including_first_buffer
Only valid for first buffer in chain.
Definition: buffer.h:167
static void vlib_buffer_struct_is_sane(vlib_buffer_t *b)
Definition: buffer.h:212
volatile u8 ref_count
Reference count for this buffer.
Definition: buffer.h:130
u32 opaque[10]
Opaque data used by sub-graphs for their own purposes.
Definition: buffer.h:153
STATIC_ASSERT(((VNET_BUFFER_FLAGS_ALL_AVAIL &VLIB_BUFFER_FLAGS_ALL)==0), "VLIB / VNET buffer flags overlap")
signed short i16
Definition: types.h:46