40 #include <sys/queue.h> 43 #include <rte_config.h> 51 #define RTE_TAILQ_RING_NAME "RTE_RING" 53 enum rte_ring_queue_behavior {
54 RTE_RING_QUEUE_FIXED = 0,
55 RTE_RING_QUEUE_VARIABLE
58 #define RTE_RING_MZ_PREFIX "RG_" 60 #define RTE_RING_NAMESIZE (RTE_MEMZONE_NAMESIZE - \ 61 sizeof(RTE_RING_MZ_PREFIX) + 1) 65 #if RTE_CACHE_LINE_SIZE < 128 66 #define PROD_ALIGN (RTE_CACHE_LINE_SIZE * 2) 67 #define CONS_ALIGN (RTE_CACHE_LINE_SIZE * 2) 69 #define PROD_ALIGN RTE_CACHE_LINE_SIZE 70 #define CONS_ALIGN RTE_CACHE_LINE_SIZE 74 struct rte_ring_headtail {
75 volatile uint32_t head;
76 volatile uint32_t tail;
111 #define RING_F_SP_ENQ 0x0001 112 #define RING_F_SC_DEQ 0x0002 121 #define RING_F_EXACT_SZ 0x0004 122 #define RTE_RING_SZ_MASK (0x7fffffffU) 223 int socket_id,
unsigned flags);
245 #define ENQUEUE_PTRS(r, ring_start, prod_head, obj_table, n, obj_type) do { \ 247 const uint32_t size = (r)->size; \ 248 uint32_t idx = prod_head & (r)->mask; \ 249 obj_type *ring = (obj_type *)ring_start; \ 250 if (likely(idx + n < size)) { \ 251 for (i = 0; i < (n & ((~(unsigned)0x3))); i+=4, idx+=4) { \ 252 ring[idx] = obj_table[i]; \ 253 ring[idx+1] = obj_table[i+1]; \ 254 ring[idx+2] = obj_table[i+2]; \ 255 ring[idx+3] = obj_table[i+3]; \ 259 ring[idx++] = obj_table[i++]; \ 261 ring[idx++] = obj_table[i++]; \ 263 ring[idx++] = obj_table[i++]; \ 266 for (i = 0; idx < size; i++, idx++)\ 267 ring[idx] = obj_table[i]; \ 268 for (idx = 0; i < n; i++, idx++) \ 269 ring[idx] = obj_table[i]; \ 276 #define DEQUEUE_PTRS(r, ring_start, cons_head, obj_table, n, obj_type) do { \ 278 uint32_t idx = cons_head & (r)->mask; \ 279 const uint32_t size = (r)->size; \ 280 obj_type *ring = (obj_type *)ring_start; \ 281 if (likely(idx + n < size)) { \ 282 for (i = 0; i < (n & (~(unsigned)0x3)); i+=4, idx+=4) {\ 283 obj_table[i] = ring[idx]; \ 284 obj_table[i+1] = ring[idx+1]; \ 285 obj_table[i+2] = ring[idx+2]; \ 286 obj_table[i+3] = ring[idx+3]; \ 290 obj_table[i++] = ring[idx++]; \ 292 obj_table[i++] = ring[idx++]; \ 294 obj_table[i++] = ring[idx++]; \ 297 for (i = 0; idx < size; i++, idx++) \ 298 obj_table[i] = ring[idx]; \ 299 for (idx = 0; i < n; i++, idx++) \ 300 obj_table[i] = ring[idx]; \ 313 #ifdef RTE_RING_USE_C11_MEM_MODEL 314 #include "rte_ring_c11_mem.h" 316 #include "rte_ring_generic.h" 340 __rte_ring_do_enqueue(
struct rte_ring *r,
void *
const *obj_table,
341 unsigned int n,
enum rte_ring_queue_behavior behavior,
342 int is_sp,
unsigned int *free_space)
344 uint32_t prod_head, prod_next;
345 uint32_t free_entries;
347 n = __rte_ring_move_prod_head(r, is_sp, n, behavior,
348 &prod_head, &prod_next, &free_entries);
352 ENQUEUE_PTRS(r, &r[1], prod_head, obj_table, n,
void *);
354 update_tail(&r->prod, prod_head, prod_next, is_sp, 1);
356 if (free_space != NULL)
357 *free_space = free_entries - n;
382 __rte_ring_do_dequeue(
struct rte_ring *r,
void **obj_table,
383 unsigned int n,
enum rte_ring_queue_behavior behavior,
384 int is_sc,
unsigned int *available)
386 uint32_t cons_head, cons_next;
389 n = __rte_ring_move_cons_head(r, is_sc, n, behavior,
390 &cons_head, &cons_next, &entries);
394 DEQUEUE_PTRS(r, &r[1], cons_head, obj_table, n,
void *);
396 update_tail(&r->cons, cons_head, cons_next, is_sc, 0);
399 if (available != NULL)
400 *available = entries - n;
424 unsigned int n,
unsigned int *free_space)
426 return __rte_ring_do_enqueue(r, obj_table, n, RTE_RING_QUEUE_FIXED,
427 __IS_MP, free_space);
447 unsigned int n,
unsigned int *free_space)
449 return __rte_ring_do_enqueue(r, obj_table, n, RTE_RING_QUEUE_FIXED,
450 __IS_SP, free_space);
474 unsigned int n,
unsigned int *free_space)
476 return __rte_ring_do_enqueue(r, obj_table, n, RTE_RING_QUEUE_FIXED,
477 r->prod.single, free_space);
558 unsigned int n,
unsigned int *available)
560 return __rte_ring_do_dequeue(r, obj_table, n, RTE_RING_QUEUE_FIXED,
582 unsigned int n,
unsigned int *available)
584 return __rte_ring_do_dequeue(r, obj_table, n, RTE_RING_QUEUE_FIXED,
609 unsigned int *available)
611 return __rte_ring_do_dequeue(r, obj_table, n, RTE_RING_QUEUE_FIXED,
612 r->cons.single, available);
684 static inline unsigned 687 uint32_t prod_tail = r->prod.tail;
688 uint32_t cons_tail = r->cons.tail;
689 uint32_t count = (prod_tail - cons_tail) & r->
mask;
701 static inline unsigned 747 static inline unsigned int 761 static inline unsigned int 807 unsigned int n,
unsigned int *free_space)
809 return __rte_ring_do_enqueue(r, obj_table, n,
810 RTE_RING_QUEUE_VARIABLE, __IS_MP, free_space);
830 unsigned int n,
unsigned int *free_space)
832 return __rte_ring_do_enqueue(r, obj_table, n,
833 RTE_RING_QUEUE_VARIABLE, __IS_SP, free_space);
857 unsigned int n,
unsigned int *free_space)
859 return __rte_ring_do_enqueue(r, obj_table, n, RTE_RING_QUEUE_VARIABLE,
860 r->prod.single, free_space);
885 unsigned int n,
unsigned int *available)
887 return __rte_ring_do_dequeue(r, obj_table, n,
888 RTE_RING_QUEUE_VARIABLE, __IS_MC, available);
910 unsigned int n,
unsigned int *available)
912 return __rte_ring_do_dequeue(r, obj_table, n,
913 RTE_RING_QUEUE_VARIABLE, __IS_SC, available);
937 unsigned int n,
unsigned int *available)
939 return __rte_ring_do_dequeue(r, obj_table, n,
940 RTE_RING_QUEUE_VARIABLE,
941 r->cons.single, available);
#define __rte_always_inline
const struct rte_memzone * memzone
static __rte_always_inline unsigned int rte_ring_enqueue_bulk(struct rte_ring *r, void *const *obj_table, unsigned int n, unsigned int *free_space)
char name [RTE_MEMZONE_NAMESIZE] __rte_cache_aligned
static __rte_always_inline int rte_ring_dequeue(struct rte_ring *r, void **obj_p)
static __rte_always_inline unsigned rte_ring_mc_dequeue_burst(struct rte_ring *r, void **obj_table, unsigned int n, unsigned int *available)
struct rte_ring_headtail prod __rte_aligned(PROD_ALIGN)
static int rte_ring_empty(const struct rte_ring *r)
static __rte_always_inline unsigned rte_ring_mp_enqueue_burst(struct rte_ring *r, void *const *obj_table, unsigned int n, unsigned int *free_space)
static __rte_always_inline int rte_ring_mc_dequeue(struct rte_ring *r, void **obj_p)
void rte_ring_list_dump(FILE *f)
static __rte_always_inline unsigned int rte_ring_mp_enqueue_bulk(struct rte_ring *r, void *const *obj_table, unsigned int n, unsigned int *free_space)
static __rte_always_inline int rte_ring_sp_enqueue(struct rte_ring *r, void *obj)
static __rte_always_inline int rte_ring_mp_enqueue(struct rte_ring *r, void *obj)
static unsigned int rte_ring_get_capacity(const struct rte_ring *r)
static unsigned int rte_ring_get_size(const struct rte_ring *r)
static __rte_always_inline int rte_ring_sc_dequeue(struct rte_ring *r, void **obj_p)
void rte_ring_free(struct rte_ring *r)
static __rte_always_inline unsigned rte_ring_sp_enqueue_burst(struct rte_ring *r, void *const *obj_table, unsigned int n, unsigned int *free_space)
static __rte_always_inline unsigned rte_ring_sc_dequeue_burst(struct rte_ring *r, void **obj_table, unsigned int n, unsigned int *available)
static __rte_always_inline unsigned rte_ring_dequeue_burst(struct rte_ring *r, void **obj_table, unsigned int n, unsigned int *available)
static __rte_always_inline unsigned int rte_ring_sp_enqueue_bulk(struct rte_ring *r, void *const *obj_table, unsigned int n, unsigned int *free_space)
void rte_ring_dump(FILE *f, const struct rte_ring *r)
static unsigned rte_ring_count(const struct rte_ring *r)
struct rte_ring * rte_ring_create(const char *name, unsigned count, int socket_id, unsigned flags)
static __rte_always_inline unsigned int rte_ring_dequeue_bulk(struct rte_ring *r, void **obj_table, unsigned int n, unsigned int *available)
static __rte_always_inline unsigned int rte_ring_sc_dequeue_bulk(struct rte_ring *r, void **obj_table, unsigned int n, unsigned int *available)
struct rte_ring * rte_ring_lookup(const char *name)
static unsigned rte_ring_free_count(const struct rte_ring *r)
static __rte_always_inline int rte_ring_enqueue(struct rte_ring *r, void *obj)
static __rte_always_inline unsigned rte_ring_enqueue_burst(struct rte_ring *r, void *const *obj_table, unsigned int n, unsigned int *free_space)
static __rte_always_inline unsigned int rte_ring_mc_dequeue_bulk(struct rte_ring *r, void **obj_table, unsigned int n, unsigned int *available)
int rte_ring_init(struct rte_ring *r, const char *name, unsigned count, unsigned flags)
static int rte_ring_full(const struct rte_ring *r)
ssize_t rte_ring_get_memsize(unsigned count)
#define RTE_MEMZONE_NAMESIZE