FD.io VPP  v18.11-rc0-18-g2a3fb1a
Vector Packet Processing
mem.h
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2015 Cisco and/or its affiliates.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at:
6  *
7  * http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 /*
16  Copyright (c) 2001, 2002, 2003 Eliot Dresselhaus
17 
18  Permission is hereby granted, free of charge, to any person obtaining
19  a copy of this software and associated documentation files (the
20  "Software"), to deal in the Software without restriction, including
21  without limitation the rights to use, copy, modify, merge, publish,
22  distribute, sublicense, and/or sell copies of the Software, and to
23  permit persons to whom the Software is furnished to do so, subject to
24  the following conditions:
25 
26  The above copyright notice and this permission notice shall be
27  included in all copies or substantial portions of the Software.
28 
29  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
30  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
31  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
32  NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
33  LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
34  OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
35  WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
36 */
37 
38 #ifndef _included_clib_mem_h
39 #define _included_clib_mem_h
40 
41 #include <stdarg.h>
42 #include <unistd.h>
43 #include <sys/mman.h>
44 
45 #include <vppinfra/clib.h> /* uword, etc */
46 #include <vppinfra/clib_error.h>
48 #include <vppinfra/os.h>
49 #include <vppinfra/string.h> /* memcpy, memset */
50 #include <vppinfra/valgrind.h>
51 
52 #define CLIB_MAX_MHEAPS 256
53 
54 /* Per CPU heaps. */
56 
57 always_inline void *
59 {
60  int cpu = os_get_thread_index ();
61  return clib_per_cpu_mheaps[cpu];
62 }
63 
64 always_inline void *
66 {
67  int cpu = os_get_thread_index ();
68  void *old = clib_per_cpu_mheaps[cpu];
69  clib_per_cpu_mheaps[cpu] = new_heap;
70  return old;
71 }
72 
73 /* Memory allocator which may call os_out_of_memory() if it fails */
74 always_inline void *
76  int os_out_of_memory_on_failure)
77 {
78  void *heap, *p;
79  uword offset, cpu;
80 
81  if (align_offset > align)
82  {
83  if (align > 0)
84  align_offset %= align;
85  else
86  align_offset = align;
87  }
88 
89  cpu = os_get_thread_index ();
90  heap = clib_per_cpu_mheaps[cpu];
91  heap = mheap_get_aligned (heap, size, align, align_offset, &offset);
92  clib_per_cpu_mheaps[cpu] = heap;
93 
94  if (offset != ~0)
95  {
96  p = heap + offset;
97 #if CLIB_DEBUG > 0
98  VALGRIND_MALLOCLIKE_BLOCK (p, mheap_data_bytes (heap, offset), 0, 0);
99 #endif
100  return p;
101  }
102  else
103  {
104  if (os_out_of_memory_on_failure)
105  os_out_of_memory ();
106  return 0;
107  }
108 }
109 
110 /* Memory allocator which calls os_out_of_memory() when it fails */
111 always_inline void *
113 {
114  return clib_mem_alloc_aligned_at_offset (size, /* align */ 1,
115  /* align_offset */ 0,
116  /* os_out_of_memory */ 1);
117 }
118 
119 always_inline void *
121 {
122  return clib_mem_alloc_aligned_at_offset (size, align, /* align_offset */ 0,
123  /* os_out_of_memory */ 1);
124 }
125 
126 /* Memory allocator which calls os_out_of_memory() when it fails */
127 always_inline void *
129 {
130  return clib_mem_alloc_aligned_at_offset (size, /* align */ 1,
131  /* align_offset */ 0,
132  /* os_out_of_memory */ 0);
133 }
134 
135 always_inline void *
137 {
138  return clib_mem_alloc_aligned_at_offset (size, align, /* align_offset */ 0,
139  /* os_out_of_memory */ 0);
140 }
141 
142 
143 
144 /* Memory allocator which panics when it fails.
145  Use macro so that clib_panic macro can expand __FUNCTION__ and __LINE__. */
146 #define clib_mem_alloc_aligned_no_fail(size,align) \
147 ({ \
148  uword _clib_mem_alloc_size = (size); \
149  void * _clib_mem_alloc_p; \
150  _clib_mem_alloc_p = clib_mem_alloc_aligned (_clib_mem_alloc_size, (align)); \
151  if (! _clib_mem_alloc_p) \
152  clib_panic ("failed to allocate %d bytes", _clib_mem_alloc_size); \
153  _clib_mem_alloc_p; \
154 })
155 
156 #define clib_mem_alloc_no_fail(size) clib_mem_alloc_aligned_no_fail(size,1)
157 
158 /* Alias to stack allocator for naming consistency. */
159 #define clib_mem_alloc_stack(bytes) __builtin_alloca(bytes)
160 
163 {
164  void *heap = clib_mem_get_per_cpu_heap ();
165  uword offset = (uword) p - (uword) heap;
166  mheap_elt_t *e, *n;
167 
168  if (offset >= vec_len (heap))
169  return 0;
170 
171  e = mheap_elt_at_uoffset (heap, offset);
172  n = mheap_next_elt (e);
173 
174  /* Check that heap forward and reverse pointers agree. */
175  return e->n_user_data == n->prev_n_user_data;
176 }
177 
178 always_inline void
179 clib_mem_free (void *p)
180 {
181  u8 *heap = clib_mem_get_per_cpu_heap ();
182 
183  /* Make sure object is in the correct heap. */
185 
186  mheap_put (heap, (u8 *) p - heap);
187 
188 #if CLIB_DEBUG > 0
190 #endif
191 }
192 
193 always_inline void *
194 clib_mem_realloc (void *p, uword new_size, uword old_size)
195 {
196  /* By default use alloc, copy and free to emulate realloc. */
197  void *q = clib_mem_alloc (new_size);
198  if (q)
199  {
200  uword copy_size;
201  if (old_size < new_size)
202  copy_size = old_size;
203  else
204  copy_size = new_size;
205  clib_memcpy (q, p, copy_size);
206  clib_mem_free (p);
207  }
208  return q;
209 }
210 
212 clib_mem_size (void *p)
213 {
216  return mheap_elt_data_bytes (e);
217 }
218 
219 always_inline void *
221 {
222  return clib_mem_get_per_cpu_heap ();
223 }
224 
225 always_inline void *
226 clib_mem_set_heap (void *heap)
227 {
228  return clib_mem_set_per_cpu_heap (heap);
229 }
230 
231 void *clib_mem_init (void *heap, uword size);
232 
233 void clib_mem_exit (void);
234 
236 
237 void clib_mem_validate (void);
238 
239 void clib_mem_trace (int enable);
240 
241 typedef struct
242 {
243  /* Total number of objects allocated. */
245 
246  /* Total allocated bytes. Bytes used and free.
247  used + free = total */
248  uword bytes_total, bytes_used, bytes_free;
249 
250  /* Number of bytes used by mheap data structure overhead
251  (e.g. free lists, mheap header). */
253 
254  /* Amount of free space returned to operating system. */
256 
257  /* For malloc which puts small objects in sbrk region and
258  large objects in mmap'ed regions. */
261 
262  /* Max. number of bytes in this heap. */
265 
267 
268 u8 *format_clib_mem_usage (u8 * s, va_list * args);
269 
270 /* Allocate virtual address space. */
271 always_inline void *
273 {
274  void *mmap_addr;
275  uword flags = MAP_PRIVATE;
276 
277 #ifdef MAP_ANONYMOUS
278  flags |= MAP_ANONYMOUS;
279 #endif
280 
281  mmap_addr = mmap (0, size, PROT_READ | PROT_WRITE, flags, -1, 0);
282  if (mmap_addr == (void *) -1)
283  mmap_addr = 0;
284 
285  return mmap_addr;
286 }
287 
288 always_inline void
290 {
291  munmap (addr, size);
292 }
293 
294 always_inline void *
296 {
297  void *mmap_addr;
298  uword flags = MAP_PRIVATE | MAP_FIXED;
299 
300  /* To unmap we "map" with no protection. If we actually called
301  munmap then other callers could steal the address space. By
302  changing to PROT_NONE the kernel can free up the pages which is
303  really what we want "unmap" to mean. */
304  mmap_addr = mmap (addr, size, PROT_NONE, flags, -1, 0);
305  if (mmap_addr == (void *) -1)
306  mmap_addr = 0;
307 
308  return mmap_addr;
309 }
310 
311 always_inline void *
313 {
314  void *mmap_addr;
315  uword flags = MAP_PRIVATE | MAP_FIXED;
316 
317  mmap_addr = mmap (addr, size, (PROT_READ | PROT_WRITE), flags, -1, 0);
318  if (mmap_addr == (void *) -1)
319  mmap_addr = 0;
320 
321  return mmap_addr;
322 }
323 
324 typedef struct
325 {
326 #define CLIB_MEM_VM_F_SHARED (1 << 0)
327 #define CLIB_MEM_VM_F_HUGETLB (1 << 1)
328 #define CLIB_MEM_VM_F_NUMA_PREFER (1 << 2)
329 #define CLIB_MEM_VM_F_NUMA_FORCE (1 << 3)
330 #define CLIB_MEM_VM_F_HUGETLB_PREALLOC (1 << 4)
331 #define CLIB_MEM_VM_F_LOCKED (1 << 5)
332  u32 flags; /**< vm allocation flags:
333  <br> CLIB_MEM_VM_F_SHARED: request shared memory, file
334  descriptor will be provided on successful allocation.
335  <br> CLIB_MEM_VM_F_HUGETLB: request hugepages.
336  <br> CLIB_MEM_VM_F_NUMA_PREFER: numa_node field contains valid
337  numa node preference.
338  <br> CLIB_MEM_VM_F_NUMA_FORCE: fail if setting numa policy fails.
339  <br> CLIB_MEM_VM_F_HUGETLB_PREALLOC: pre-allocate hugepages if
340  number of available pages is not sufficient.
341  <br> CLIB_MEM_VM_F_LOCKED: request locked memory.
342  */
343  char *name; /**< Name for memory allocation, set by caller. */
344  uword size; /**< Allocation size, set by caller. */
345  int numa_node; /**< numa node preference. Valid if CLIB_MEM_VM_F_NUMA_PREFER set. */
346  void *addr; /**< Pointer to allocated memory, set on successful allocation. */
347  int fd; /**< File descriptor, set on successful allocation if CLIB_MEM_VM_F_SHARED is set. */
348  int log2_page_size; /* Page size in log2 format, set on successful allocation. */
349  int n_pages; /* Number of pages. */
350  uword requested_va; /**< Request fixed position mapping */
352 
355 int clib_mem_vm_get_log2_page_size (int fd);
356 u64 *clib_mem_vm_get_paddr (void *mem, int log2_page_size, int n_pages);
357 
358 typedef struct
359 {
360  uword size; /**< Map size */
361  int fd; /**< File descriptor to be mapped */
362  uword requested_va; /**< Request fixed position mapping */
363  void *addr; /**< Pointer to mapped memory, if successful */
365 
367 void clib_mem_vm_randomize_va (uword * requested_va, u32 log2_page_size);
368 
369 #include <vppinfra/error.h> /* clib_panic */
370 
371 #endif /* _included_clib_mem_h */
372 
373 /*
374  * fd.io coding-style-patch-verification: ON
375  *
376  * Local Variables:
377  * eval: (c-set-style "gnu")
378  * End:
379  */
uword bytes_overhead
Definition: mem.h:252
void clib_mem_validate(void)
Definition: mem_mheap.c:142
void * clib_per_cpu_mheaps[CLIB_MAX_MHEAPS]
Definition: mem_mheap.c:46
static void * clib_mem_alloc_aligned_at_offset(uword size, uword align, uword align_offset, int os_out_of_memory_on_failure)
Definition: mem.h:75
a
Definition: bitmap.h:538
uword bytes_free_reclaimed
Definition: mem.h:255
clib_error_t * clib_mem_vm_ext_alloc(clib_mem_vm_alloc_t *a)
Definition: mem.c:80
unsigned long u64
Definition: types.h:89
void * addr
Pointer to allocated memory, set on successful allocation.
Definition: mem.h:346
uword requested_va
Request fixed position mapping.
Definition: mem.h:350
static void * clib_mem_realloc(void *p, uword new_size, uword old_size)
Definition: mem.h:194
void os_out_of_memory(void)
Definition: unix-misc.c:219
int numa_node
numa node preference.
Definition: mem.h:345
static void usage(void)
Definition: health_check.c:14
uword bytes_used_sbrk
Definition: mem.h:259
clib_error_t * clib_mem_vm_ext_map(clib_mem_vm_map_t *a)
Definition: mem.c:295
vhost_vring_addr_t addr
Definition: vhost_user.h:116
#define CLIB_MAX_MHEAPS
Definition: mem.h:52
uword bytes_used
Definition: mem.h:248
unsigned char u8
Definition: types.h:56
u64 clib_mem_vm_get_page_size(int fd)
Definition: mem.c:50
uword object_count
Definition: mem.h:244
uword requested_va
Request fixed position mapping.
Definition: mem.h:362
#define always_inline
Definition: clib.h:92
char * name
Name for memory allocation, set by caller.
Definition: mem.h:343
static uword mheap_elt_data_bytes(mheap_elt_t *e)
uword size
Allocation size, set by caller.
Definition: mem.h:344
unsigned int u32
Definition: types.h:88
uword size
int clib_mem_vm_get_log2_page_size(int fd)
Definition: mem.c:59
int fd
File descriptor, set on successful allocation if CLIB_MEM_VM_F_SHARED is set.
Definition: mem.h:347
static void * clib_mem_get_per_cpu_heap(void)
Definition: mem.h:58
u8 * format_clib_mem_usage(u8 *s, va_list *args)
Definition: mem_mheap.c:126
uword bytes_used_mmap
Definition: mem.h:260
u64 * clib_mem_vm_get_paddr(void *mem, int log2_page_size, int n_pages)
Definition: mem.c:256
u32 flags
Definition: vhost_user.h:110
void * clib_mem_init(void *heap, uword size)
Definition: mem_mheap.c:60
#define VALGRIND_MALLOCLIKE_BLOCK(addr, sizeB, rzB, is_zeroed)
Definition: valgrind.h:3894
static void * clib_mem_alloc_or_null(uword size)
Definition: mem.h:128
void * mheap_get_aligned(void *v, uword n_user_data_bytes, uword align, uword align_offset, uword *offset_return)
Definition: mheap.c:643
uword bytes_max
Definition: mem.h:263
static void * clib_mem_set_heap(void *heap)
Definition: mem.h:226
#define clib_memcpy(a, b, c)
Definition: string.h:75
static uword clib_mem_size(void *p)
Definition: mem.h:212
u32 flags
vm allocation flags: CLIB_MEM_VM_F_SHARED: request shared memory, file descriptor will be provided ...
Definition: mem.h:332
static void * clib_mem_get_heap(void)
Definition: mem.h:220
static void * clib_mem_alloc_aligned_or_null(uword size, uword align)
Definition: mem.h:136
#define ASSERT(truth)
uword size
Map size.
Definition: mem.h:360
static void clib_mem_free(void *p)
Definition: mem.h:179
static uword clib_mem_is_heap_object(void *p)
Definition: mem.h:162
static void * clib_mem_alloc(uword size)
Definition: mem.h:112
static void * clib_mem_set_per_cpu_heap(u8 *new_heap)
Definition: mem.h:65
static mheap_elt_t * mheap_elt_at_uoffset(void *v, uword uo)
template key/value backing page structure
Definition: bihash_doc.h:44
static void clib_mem_vm_free(void *addr, uword size)
Definition: mem.h:289
void mheap_put(void *v, uword uoffset)
Definition: mheap.c:771
#define vec_len(v)
Number of elements in vector (rvalue-only, NULL tolerant)
void clib_mem_usage(clib_mem_usage_t *usage)
Definition: mem_mheap.c:133
u64 uword
Definition: types.h:112
static_always_inline uword os_get_thread_index(void)
Definition: os.h:62
static mheap_elt_t * mheap_next_elt(mheap_elt_t *e)
struct clib_bihash_value offset
template key/value backing page structure
void * mem
static void * clib_mem_alloc_aligned(uword size, uword align)
Definition: mem.h:120
int fd
File descriptor to be mapped.
Definition: mem.h:361
static void * clib_mem_vm_unmap(void *addr, uword size)
Definition: mem.h:295
void clib_mem_vm_randomize_va(uword *requested_va, u32 log2_page_size)
Definition: mem.c:65
#define VALGRIND_FREELIKE_BLOCK(addr, rzB)
Definition: valgrind.h:3905
static uword mheap_data_bytes(void *v, uword uo)
int log2_page_size
Definition: mem.h:348
void clib_mem_trace(int enable)
Definition: mem_mheap.c:154
uword clib_mem_get_page_size(void)
Definition: mem_mheap.c:110
static void * clib_mem_vm_alloc(uword size)
Definition: mem.h:272
void clib_mem_exit(void)
Definition: mem_mheap.c:49
static mheap_elt_t * mheap_user_pointer_to_elt(void *v)
void * addr
Pointer to mapped memory, if successful.
Definition: mem.h:363
static void * clib_mem_vm_map(void *addr, uword size)
Definition: mem.h:312