53 #ifndef CLIB_MARCH_VARIANT 60 u32 normalized_start = (seg->
start + f->nitems - f->tail) % f->nitems;
61 s =
format (s,
"[%u, %u], len %u, next %d, prev %d", normalized_start,
62 (normalized_start + seg->
length) % f->nitems, seg->
length,
115 for (i = 0; i <
vec_len (data); i++)
118 for (i = 0; i < trace_len; i++)
121 if (trace[i].action == 1)
124 s =
format (s,
"adding [%u, %u]:", trace[i].offset,
126 trace[i].
len) % dummy_fifo->nitems);
128 trace[i].len, &data[offset]);
130 else if (trace[i].action == 2)
133 s =
format (s,
"adding [%u, %u]:", 0, trace[i].
len);
139 s =
format (s,
"read: %u", trace[i].
len);
155 u32 indent = va_arg (*args,
u32);
156 u32 ooo_segment_index = f->ooos_list_head;
164 ooo_segment_index = seg->
next;
174 int verbose = va_arg (*args,
int);
181 s =
format (s,
"cursize %u nitems %u has_event %d\n",
182 f->cursize, f->nitems, f->has_event);
184 indent, f->head, f->tail, f->segment_manager);
187 s =
format (s,
"%Uvpp session %d thread %d app session %d thread %d\n",
189 f->master_thread_index, f->client_session_index,
190 f->client_thread_index);
194 s =
format (s,
"%Uooo pool %d active elts newest %u\n",
208 u32 rounded_data_size;
211 rounded_data_size = (1 << (
max_log2 (data_size_in_bytes)));
218 f->nitems = data_size_in_bytes;
231 if (--f->refcnt == 0)
263 next->prev = cur->
prev;
273 f->ooos_list_head = cur->
next;
287 u32 new_index, s_end_pos, s_index;
288 u32 normalized_position, normalized_end_position;
291 normalized_position = (f->tail +
offset) % f->nitems;
292 normalized_end_position = (f->tail + offset + length) % f->nitems;
299 f->ooos_list_head = s - f->ooo_segments;
300 f->ooos_newest = f->ooos_list_head;
324 s_index = s - f->ooo_segments;
331 new_index = new_s - f->ooo_segments;
339 prev->
next = new_index;
344 f->ooos_list_head = new_index;
347 new_s->
next = s_index;
349 f->ooos_newest = new_index;
353 else if (
position_gt (f, normalized_position, s_end_pos))
356 new_index = new_s - f->ooo_segments;
364 new_s->
prev = s_index;
366 f->ooos_newest = new_index;
378 s->
start = normalized_position;
380 f->ooos_newest = s - f->ooo_segments;
386 if (
position_gt (f, normalized_end_position, s_end_pos))
393 normalized_end_position))
407 f->ooos_newest = s - f->ooo_segments;
419 u32 index, bytes = 0;
425 ASSERT (diff != n_bytes_enqueued);
427 if (diff > n_bytes_enqueued)
431 while (0 <= diff && diff < n_bytes_enqueued)
433 index = s - f->ooo_segments;
440 f->tail %= f->nitems;
460 ASSERT (bytes <= f->nitems);
465 const u8 * copy_from_here)
467 u32 total_copy_bytes, first_copy_bytes, second_copy_bytes;
480 total_copy_bytes = (nitems - cursize) < max_bytes ?
481 (nitems - cursize) : max_bytes;
486 first_copy_bytes = ((nitems - f->tail) < total_copy_bytes)
487 ? (nitems - f->tail) : total_copy_bytes;
490 f->tail += first_copy_bytes;
491 f->tail = (f->tail == nitems) ? 0 : f->tail;
494 second_copy_bytes = total_copy_bytes - first_copy_bytes;
495 if (second_copy_bytes)
498 copy_from_here + first_copy_bytes,
500 f->tail += second_copy_bytes;
501 f->tail = (f->tail == nitems) ? 0 : f->tail;
509 ASSERT (max_bytes <= (nitems - cursize));
510 f->tail += max_bytes;
511 f->tail = f->tail % nitems;
512 total_copy_bytes = max_bytes;
522 ASSERT (cursize + total_copy_bytes <= nitems);
525 return (total_copy_bytes);
528 #ifndef CLIB_MARCH_VARIANT 531 const u8 * copy_from_here)
548 u32 total_copy_bytes, first_copy_bytes, second_copy_bytes;
549 u32 cursize, nitems, normalized_offset;
557 ASSERT (required_bytes < nitems);
559 normalized_offset = (f->tail +
offset) % nitems;
562 if ((required_bytes + offset) > (nitems - cursize))
570 total_copy_bytes = required_bytes;
573 first_copy_bytes = ((nitems - normalized_offset) < total_copy_bytes)
574 ? (nitems - normalized_offset) : total_copy_bytes;
580 second_copy_bytes = total_copy_bytes - first_copy_bytes;
581 if (second_copy_bytes)
583 normalized_offset += first_copy_bytes;
584 normalized_offset %= nitems;
586 ASSERT (normalized_offset == 0);
589 copy_from_here + first_copy_bytes, second_copy_bytes);
595 #ifndef CLIB_MARCH_VARIANT 610 first_chunk = f->nitems - f->head;
611 ASSERT (len <= f->nitems);
612 if (len <= first_chunk)
625 u32 total_copy_bytes, first_copy_bytes, second_copy_bytes;
636 total_copy_bytes = (cursize < max_bytes) ? cursize : max_bytes;
641 first_copy_bytes = ((nitems - f->head) < total_copy_bytes)
642 ? (nitems - f->head) : total_copy_bytes;
644 f->head += first_copy_bytes;
645 f->head = (f->head == nitems) ? 0 : f->head;
648 second_copy_bytes = total_copy_bytes - first_copy_bytes;
649 if (second_copy_bytes)
652 &f->data[f->head], second_copy_bytes);
653 f->head += second_copy_bytes;
654 f->head = (f->head == nitems) ? 0 : f->head;
661 ASSERT (max_bytes <= cursize);
662 f->head += max_bytes;
663 f->head = f->head % nitems;
664 cursize -= max_bytes;
665 total_copy_bytes = max_bytes;
668 ASSERT (f->head <= nitems);
669 ASSERT (cursize >= total_copy_bytes);
672 return (total_copy_bytes);
675 #ifndef CLIB_MARCH_VARIANT 686 u32 max_bytes,
u8 * copy_here)
688 u32 total_copy_bytes, first_copy_bytes, second_copy_bytes;
689 u32 cursize, nitems, real_head;
697 real_head = f->head + relative_offset;
698 real_head = real_head >= nitems ? real_head - nitems : real_head;
701 total_copy_bytes = (cursize - relative_offset < max_bytes) ?
702 cursize - relative_offset : max_bytes;
708 ((nitems - real_head) < total_copy_bytes) ?
709 (nitems - real_head) : total_copy_bytes;
713 second_copy_bytes = total_copy_bytes - first_copy_bytes;
714 if (second_copy_bytes)
720 return total_copy_bytes;
723 #ifndef CLIB_MARCH_VARIANT 736 u32 total_drop_bytes, first_drop_bytes, second_drop_bytes;
747 total_drop_bytes = (cursize < max_bytes) ? cursize : max_bytes;
753 ((nitems - f->head) < total_drop_bytes) ?
754 (nitems - f->head) : total_drop_bytes;
755 f->head += first_drop_bytes;
756 f->head = (f->head == nitems) ? 0 : f->head;
759 second_drop_bytes = total_drop_bytes - first_drop_bytes;
760 if (second_drop_bytes)
762 f->head += second_drop_bytes;
763 f->head = (f->head == nitems) ? 0 : f->head;
766 ASSERT (f->head <= nitems);
767 ASSERT (cursize >= total_drop_bytes);
770 return total_drop_bytes;
792 fs[0].
len = ((nitems - f->head) < cursize) ? (nitems - f->head) : cursize;
793 fs[0].
data = f->data + f->head;
795 if (fs[0].
len < cursize)
797 fs[1].
len = cursize - fs[0].
len;
798 fs[1].
data = f->data;
811 u32 total_drop_bytes;
817 total_drop_bytes = fs[0].
len + fs[1].
len;
821 f->head = (f->head + fs[0].
len) % f->nitems;
822 total_drop_bytes = fs[0].
len;
845 f->head = f->tail = pointer % f->nitems;
#define vec_validate(V, I)
Make sure vector is long enough for given index (no header, unspecified alignment) ...
static vlib_cli_command_t trace
(constructor) VLIB_CLI_COMMAND (trace)
void svm_fifo_init_pointers(svm_fifo_t *f, u32 pointer)
Set fifo pointers to requested offset.
static u32 position_diff(svm_fifo_t *f, u32 posa, u32 posb)
int svm_fifo_segments(svm_fifo_t *f, svm_fifo_segment_t *fs)
static u8 svm_fifo_has_ooo_data(svm_fifo_t *f)
void svm_fifo_free(svm_fifo_t *f)
#define clib_memcpy_fast(a, b, c)
u32 prev
Previous linked-list element pool index.
#define CLIB_MARCH_FN_SELECT(fn)
void svm_fifo_overwrite_head(svm_fifo_t *f, u8 *data, u32 len)
clib_memset(h->entries, 0, sizeof(h->entries[0])*entries)
static int ooo_segment_try_collect(svm_fifo_t *f, u32 n_bytes_enqueued)
Removes segments that can now be enqueued because the fifo's tail has advanced.
ooo_segment_t * svm_fifo_first_ooo_segment(svm_fifo_t *f)
void svm_fifo_dequeue_drop_all(svm_fifo_t *f)
#define SVM_FIFO_INVALID_INDEX
#define pool_get(P, E)
Allocate an object E from a pool P (unspecified alignment).
static u32 ooo_segment_distance_from_tail(svm_fifo_t *f, u32 pos)
struct _svm_fifo svm_fifo_t
static void ooo_segment_add(svm_fifo_t *f, u32 offset, u32 length)
Add segment to fifo's out-of-order segment list.
static u32 svm_fifo_max_dequeue(svm_fifo_t *f)
u8 * format_ooo_list(u8 *s, va_list *args)
int svm_fifo_enqueue_nowait(svm_fifo_t *f, u32 max_bytes, const u8 *copy_from_here)
#define pool_elt_at_index(p, i)
Returns pointer to element at given index.
static u8 position_gt(svm_fifo_t *f, u32 a, u32 b)
u32 svm_fifo_number_ooo_segments(svm_fifo_t *f)
#define pool_put(P, E)
Free an object E in pool P.
static ooo_segment_t * ooo_segment_next(svm_fifo_t *f, ooo_segment_t *s)
#define svm_fifo_trace_add(_f, _s, _l, _t)
#define clib_atomic_fetch_add_rel(a, b)
u8 * svm_fifo_replay(u8 *s, svm_fifo_t *f, u8 no_read, u8 verbose)
#define pool_free(p)
Free a pool.
static u8 position_leq(svm_fifo_t *f, u32 a, u32 b)
static ooo_segment_t * ooo_segment_get_prev(svm_fifo_t *f, ooo_segment_t *s)
static void ooo_segment_del(svm_fifo_t *f, u32 index)
CLIB_MARCH_FN(svm_fifo_enqueue_nowait, int, svm_fifo_t *f, u32 max_bytes, const u8 *copy_from_here)
#define OOO_SEGMENT_INVALID_INDEX
u8 * format_ooo_segment(u8 *s, va_list *args)
static void * clib_mem_alloc_aligned_or_null(uword size, uword align)
u8 * format_svm_fifo(u8 *s, va_list *args)
static u8 position_lt(svm_fifo_t *f, u32 a, u32 b)
static void clib_mem_free(void *p)
int svm_fifo_enqueue_with_offset(svm_fifo_t *f, u32 offset, u32 required_bytes, u8 *copy_from_here)
u32 length
Length of segment.
#define clib_atomic_fetch_sub_rel(a, b)
u8 * svm_fifo_dump_trace(u8 *s, svm_fifo_t *f)
u32 next
Next linked-list element pool index.
template key/value backing page structure
void svm_fifo_segments_free(svm_fifo_t *f, svm_fifo_segment_t *fs)
int svm_fifo_dequeue_drop(svm_fifo_t *f, u32 max_bytes)
static u32 ooo_segment_end_pos(svm_fifo_t *f, ooo_segment_t *s)
#define vec_len(v)
Number of elements in vector (rvalue-only, NULL tolerant)
static ooo_segment_t * ooo_segment_new(svm_fifo_t *f, u32 start, u32 length)
static uword max_log2(uword x)
struct clib_bihash_value offset
template key/value backing page structure
#define vec_foreach(var, vec)
Vector iterator.
#define SVM_FIFO_INVALID_SESSION_INDEX
#define CLIB_CACHE_LINE_BYTES
int svm_fifo_peek(svm_fifo_t *f, u32 relative_offset, u32 max_bytes, u8 *copy_here)
int svm_fifo_dequeue_nowait(svm_fifo_t *f, u32 max_bytes, u8 *copy_here)
u32 start
Start of segment, normalized.
static u32 ooo_segment_distance_to_tail(svm_fifo_t *f, u32 pos)
svm_fifo_t * svm_fifo_create(u32 data_size_in_bytes)
create an svm fifo, in the current heap.
static uword pool_elts(void *v)
Number of active elements in a pool.