EM-ODP 4.4.0
Event Machine on ODP
Loading...
Searching...
No Matches
em_pool.c
1/*
2 * Copyright (c) 2015-2026, Nokia Solutions and Networks
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 *
9 * * Redistributions of source code must retain the above copyright
10 * notice, this list of conditions and the following disclaimer.
11 * * Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
14 * * Neither the name of the copyright holder nor the names of its
15 * contributors may be used to endorse or promote products derived
16 * from this software without specific prior written permission.
17 *
18 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29 */
30
31#ifndef _GNU_SOURCE
32#define _GNU_SOURCE
33#endif
34
35#ifdef HAVE_CONFIG_H
36#include "config.h"
37#endif
38
39#include <stdbool.h>
40#include <stdint.h>
41#include <stdio.h>
42#include <string.h>
43
44#include <odp_api.h>
45
46#include <event_machine.h>
47#include <event_machine/platform/env/environment.h>
48
49#include "em_chaining.h"
50#include "em_error.h"
51#include "em_event.h"
52#include "em_event_inline.h"
53#include "em_event_state.h"
54#include "em_event_types.h"
55#include "em_libconfig.h"
56#include "em_mem.h"
57#include "em_pool.h"
58#include "em_pool_inline.h"
59#include "em_pool_types.h"
60#include "em_queue.h"
61#include "em_queue_inline.h"
62#include "em_queue_types.h"
63#include "misc/list.h"
64#include "misc/objpool.h"
65
66#ifndef __clang__
67COMPILE_TIME_ASSERT(EM_POOL_DEFAULT > (em_pool_t)0 &&
68 EM_POOL_DEFAULT < (em_pool_t)EM_CONFIG_POOLS,
69 EM_ODP_EM_DEFAULT_POOL_ERROR);
70COMPILE_TIME_ASSERT(EM_POOL_UNDEF != EM_POOL_DEFAULT,
71 EM_ODP_EM_POOL_UNDEF_ERROR);
72#endif
73COMPILE_TIME_ASSERT(EM_EVENT_USER_AREA_MAX_SIZE < UINT16_MAX,
74 EM_ODP_EM_EVENT_USER_AREA_MAX_SIZE_ERROR);
75
76/**
77 * @def ALIGN_OFFSET_MAX
78 *
79 * Max supported value for the config file option 'pool.align_offset'.
80 */
81#define ALIGN_OFFSET_MAX ((int)(16))
82
83/* ALIGN_OFFSET_MAX <= 2^bits - 1, must fit into event_hdr_t::align_offset */
84COMPILE_TIME_ASSERT(ALIGN_OFFSET_MAX <=
85 ((1 << (sizeof_field(event_hdr_t, align_offset) * 8)) - 1),
86 ALIGN_OFFSET_MAX__TOO_LARGE);
87
88/*
89 * The payload placement math 'base + 32 - align_offset' (see event_alloc_buf()
90 * and the buf.size adjustment in create_subpools()) relies on a 32-byte base
91 * alignment. ALIGN_OFFSET_MAX must stay below 32 to keep '32 - align_offset'
92 * positive and avoid unsigned underflow.
93 */
94COMPILE_TIME_ASSERT(ALIGN_OFFSET_MAX < 32, ALIGN_OFFSET_MAX__BREAKS_32B_BASE);
95
96/**
97 * @brief Undef value for a pool_subpool_t
98 * pool_subpool_undef = {.pool = EM_POOL_UNDEF, .subpool = 0};
99 */
100const pool_subpool_t pool_subpool_undef = {.pool = (uint32_t)(uintptr_t)EM_POOL_UNDEF,
101 .subpool = 0};
102
103static inline mpool_elem_t *
104mpool_poolelem2pool(objpool_elem_t *const objpool_elem)
105{
106 return (mpool_elem_t *)((uintptr_t)objpool_elem -
107 offsetof(mpool_elem_t, objpool_elem));
108}
109
110uint32_t align_offset_get(const em_pool_cfg_t *pool_cfg)
111{
112 /* Pool-specific param overrides config file 'align_offset' value */
113 if (pool_cfg->event_type == EM_EVENT_TYPE_SW ||
114 pool_cfg->event_type == EM_EVENT_TYPE_PACKET) {
115 if (pool_cfg->align_offset.in_use)
116 return pool_cfg->align_offset.value; /* pool cfg */
117 else
118 return em_shm->opt.pool.align_offset; /* cfg file */
119 }
120
121 return 0;
122}
123
124uint32_t odp_align_get(const em_pool_cfg_t *pool_cfg)
125{
126 const odp_pool_capability_t *capa =
127 &em_shm->mpool_tbl.odp_pool_capability;
128
129 uint32_t align = ODP_CACHE_LINE_SIZE;
130
131 /* Set subpool minimum alignment */
132 if (pool_cfg->event_type == EM_EVENT_TYPE_PACKET) {
133 if (align > capa->pkt.max_align)
134 return capa->pkt.max_align;
135 } else {
136 if (align > capa->buf.max_align)
137 return capa->buf.max_align;
138 }
139
140 return align;
141}
142
143static em_pool_t
144pool_alloc(em_pool_t pool)
145{
146 mpool_elem_t *mpool_elem;
147
148 if (pool == EM_POOL_UNDEF) {
149 objpool_elem_t *objpool_elem =
150 objpool_rem(&em_shm->mpool_pool.objpool, em_core_id());
151
152 if (unlikely(objpool_elem == NULL))
153 return EM_POOL_UNDEF;
154
155 mpool_elem = mpool_poolelem2pool(objpool_elem);
156 } else {
157 int ret;
158
159 mpool_elem = pool_elem_get(pool);
160 if (unlikely(mpool_elem == NULL))
161 return EM_POOL_UNDEF;
162
163 ret = objpool_rem_elem(&em_shm->mpool_pool.objpool,
164 &mpool_elem->objpool_elem);
165 if (unlikely(ret != 0))
166 return EM_POOL_UNDEF;
167 }
168
169 odp_atomic_inc_u32(&em_shm->pool_count);
170 return mpool_elem->em_pool;
171}
172
173static em_status_t
174pool_free(em_pool_t pool)
175{
176 mpool_elem_t *mpool_elem = pool_elem_get(pool);
177
178 if (unlikely(mpool_elem == NULL))
179 return EM_ERR_BAD_ID;
180
181 objpool_add(&em_shm->mpool_pool.objpool,
182 mpool_elem->objpool_elem.subpool_idx,
183 &mpool_elem->objpool_elem);
184
185 odp_atomic_dec_u32(&em_shm->pool_count);
186 return EM_OK;
187}
188
189static int event_type_from_string(const char *str, em_event_type_t *event_type /*out*/)
190{
191 if (strstr(str, "EM_EVENT_TYPE_SW")) {
192 *event_type = EM_EVENT_TYPE_SW;
193 } else if (strstr(str, "EM_EVENT_TYPE_PACKET")) {
194 *event_type = EM_EVENT_TYPE_PACKET;
195 } else if (strstr(str, "EM_EVENT_TYPE_VECTOR")) {
196 *event_type = EM_EVENT_TYPE_VECTOR;
197 } else {
198 EM_LOG(EM_LOG_ERR, "Event type %s not supported.\n", str);
199 return -1;
200 }
201
202 return 0;
203}
204
205/* Read option: startup_pools.conf[i].pool_cfg.subpools[j] from the EM config file */
206static inline int read_config_subpool(const libconfig_list_t *subpool, int index,
207 const char *pool_cfg_str, em_pool_cfg_t *cfg/*out*/)
208{
209 int ret;
210 /* Option: subpools[index].size */
211 ret = em_libconfig_list_lookup_int(subpool, index, "size",
212 (int *)&cfg->subpool[index].size);
213 if (unlikely(ret != 1)) {
214 EM_LOG(EM_LOG_ERR,
215 "Option '%s.subpools[%d].size' not found or wrong type.\n",
216 pool_cfg_str, index);
217 return -1;
218 }
219
220 if (cfg->subpool[index].size <= 0) {
221 EM_LOG(EM_LOG_ERR, "Invalid '%s.subpools[%d].size'.\n",
222 pool_cfg_str, index);
223 return -1;
224 }
225
226 /* Option: subpools[index].num */
227 ret = em_libconfig_list_lookup_int(subpool, index, "num",
228 (int *)&cfg->subpool[index].num);
229 if (unlikely(ret != 1)) {
230 EM_LOG(EM_LOG_ERR,
231 "Option '%s.subpools[%d].num' not found or wrong type.\n",
232 pool_cfg_str, index);
233 return -1;
234 }
235
236 if (cfg->subpool[index].num <= 0) {
237 EM_LOG(EM_LOG_ERR, "Invalid '%s.subpools[%d].num'.\n",
238 pool_cfg_str, index);
239 return -1;
240 }
241
242 /*
243 * Option: subpools[index].cache_size
244 * Not mandatory
245 */
246 ret = em_libconfig_list_lookup_int(subpool, index, "cache_size",
247 (int *)&cfg->subpool[index].cache_size);
248
249 /* If cache_size is given, check if it is valid */
250 if (ret == 1) {
251 uint32_t min_cache_size = 0;
252 const odp_pool_capability_t *capa;
253
254 capa = &em_shm->mpool_tbl.odp_pool_capability;
255
256 switch (cfg->event_type) {
257 case EM_EVENT_TYPE_SW:
258 min_cache_size = capa->buf.min_cache_size;
259 break;
260 case EM_EVENT_TYPE_PACKET:
261 min_cache_size = capa->pkt.min_cache_size;
262 break;
263 case EM_EVENT_TYPE_VECTOR:
264 if (em_shm->opt.vector.backend == EM_VECTOR_BACKEND_PACKET)
265 min_cache_size = capa->vector.min_cache_size;
266 else /* EM_VECTOR_BACKEND_EVENT */
267 min_cache_size = capa->event_vector.min_cache_size;
268
269 break;
270 default:
271 EM_LOG(EM_LOG_ERR,
272 "'%s.subpools[%d].cache_size' for pool type:%u not available.\n",
273 pool_cfg_str, index, cfg->event_type);
274 return -1;
275 }
276
277 if (unlikely(cfg->subpool[index].cache_size < min_cache_size)) {
278 EM_LOG(EM_LOG_ERR,
279 "'%s.subpools[%d].cache_size' too small.\n",
280 pool_cfg_str, index);
281 return -1;
282 }
283 } else if (ret == 0) {/*cache_size is given but with wrong data type*/
284 EM_LOG(EM_LOG_ERR,
285 "'%s.subpools[%d].cache_size' wrong data type.\n",
286 pool_cfg_str, index);
287 return -1;
288 }
289
290 /* No need to return fail -1 when cache_size not given (ret == -1) */
291 return 0;
292}
293
294static int is_pool_type_supported(em_event_type_t type,
295 const char **err_str/*out*/)
296{
297 const odp_pool_capability_t *capa = &em_shm->mpool_tbl.odp_pool_capability;
298
299 if (type == EM_EVENT_TYPE_SW) {
300 if (capa->buf.max_pools == 0) {
301 *err_str = "SW (buf) pool type unsupported";
302 return -1;
303 }
304 } else if (type == EM_EVENT_TYPE_PACKET) {
305 if (capa->pkt.max_pools == 0) {
306 *err_str = "PACKET pool type unsupported";
307 return -1;
308 }
309 } else if (type == EM_EVENT_TYPE_VECTOR) {
310 if (em_shm->opt.vector.backend == EM_VECTOR_BACKEND_PACKET) {
311 if (capa->vector.max_pools == 0) {
312 *err_str = "VECTOR (pkt) pool type unsupported";
313 return -1;
314 }
315 } else {
316 /* EM_VECTOR_BACKEND_EVENT */
317 if (capa->event_vector.max_pools == 0) {
318 *err_str = "EVENT_VECTOR pool type unsupported";
319 return -1;
320 }
321 }
322 } else {
323 *err_str = "Pool type unsupported, use _SW, _PACKET or _VECTOR";
324 return -1;
325 }
326
327 return 0;
328}
329
330/*
331 * pool_create() helper: determine payload alignment.
332 */
333static inline int is_align_offset_valid(const em_pool_cfg_t *pool_cfg)
334{
335 uint32_t offset = align_offset_get(pool_cfg);
336 uint32_t align = odp_align_get(pool_cfg);
337
338 /* verify alignment requirements */
339 if (!POWEROF2(offset)) {
340 EM_LOG(EM_LOG_ERR, "align_offset %d must be power of 2\n",
341 offset);
342 return -1;
343 }
344
345 if (!POWEROF2(align)) {
346 EM_LOG(EM_LOG_ERR, "Invalid max_align value: %d\n"
347 "max_align must be power of 2\n",
348 align);
349 return -2;
350 }
351
352 if (offset > ALIGN_OFFSET_MAX) {
353 EM_LOG(EM_LOG_ERR,
354 "align_offset %d must be less or equal than max align_offset %d\n",
355 offset, ALIGN_OFFSET_MAX);
356 return -3;
357 }
358
359 if (offset >= align) {
360 EM_LOG(EM_LOG_ERR,
361 "align offset %d must be less than configuration max_align %d\n",
362 offset, align);
363 return -4;
364 }
365
366 return 0;
367}
368
369static inline int is_user_area_valid(const em_pool_cfg_t *pool_cfg,
370 const odp_pool_capability_t *capa,
371 const char **err_str/*out*/)
372{
373 /* No need to check when pool specific value is not used */
374 if (!pool_cfg->user_area.in_use)
375 return 0;
376
377 if (pool_cfg->user_area.size > EM_EVENT_USER_AREA_MAX_SIZE) {
378 *err_str = "Event user area too large";
379 return -1;
380 }
381
382 if (pool_cfg->event_type == EM_EVENT_TYPE_PACKET) {
383 size_t req_odp_uarea_sz = pool_cfg->user_area.size +
384 sizeof(event_hdr_t);
385 if (req_odp_uarea_sz > capa->pkt.max_uarea_size) {
386 *err_str = "ODP pkt max uarea not large enough";
387 return -1;
388 }
389 }
390 if (pool_cfg->event_type == EM_EVENT_TYPE_VECTOR) {
391 size_t req_odp_uarea_sz = pool_cfg->user_area.size + sizeof(event_hdr_t);
392
393 if (em_shm->opt.vector.backend == EM_VECTOR_BACKEND_PACKET) {
394 if (req_odp_uarea_sz > capa->vector.max_uarea_size) {
395 *err_str = "ODP pkt-vector max uarea not large enough";
396 return -1;
397 }
398 } else {
399 /* EM_VECTOR_BACKEND_EVENT */
400 if (req_odp_uarea_sz > capa->event_vector.max_uarea_size) {
401 *err_str = "ODP event-vector max uarea not large enough";
402 return -1;
403 }
404 }
405 }
406
407 return 0;
408}
409
410/* Read option: startup_pools.conf[index].pool_cfg.align_offset from the EM config file */
411static inline int read_config_align_offset(const libconfig_group_t *align_offset,
412 const char *pool_cfg_str,
413 em_pool_cfg_t *cfg/*out*/)
414{
415 int ret;
416
417 /* Option: startup_pools.conf[index].pool_cfg.align_offset.in_use */
418 ret = em_libconfig_group_lookup_bool(align_offset, "in_use",
419 &cfg->align_offset.in_use);
420 if (unlikely(!ret)) {
421 EM_LOG(EM_LOG_ERR,
422 "'%s.align_offset.in_use' not found or wrong type\n",
423 pool_cfg_str);
424 return -1;
425 }
426
427 /* Option: startup_pools.conf[index].pool_cfg.align_offset.value */
428 ret = em_libconfig_group_lookup_int(align_offset, "value",
429 (int *)&cfg->align_offset.value);
430 if (unlikely(!ret)) {
431 EM_LOG(EM_LOG_ERR,
432 "'%s.align_offset.value' not found or wrong type\n",
433 pool_cfg_str);
434 return -1;
435 }
436
437 /* Check whether the given value is valid or not */
438 ret = is_align_offset_valid(cfg);
439 if (unlikely(ret)) {
440 EM_LOG(EM_LOG_ERR, "Failure detected when checking '%s.align_offset.value': %d\n",
441 pool_cfg_str, cfg->align_offset.value);
442 return ret;
443 }
444
445 return 0;
446}
447
448/* Read option: startup_pools.conf[index].pool_cfg.user_area from the EM config file */
449static inline int read_config_user_area(const libconfig_group_t *user_area,
450 const char *pool_cfg_str,
451 em_pool_cfg_t *cfg/*out*/)
452{
453 int ret;
454 const odp_pool_capability_t *capa;
455 const char *err_str = "";
456
457 /* Option: startup_pools.conf[index].pool_cfg.user_area.in_use */
458 ret = em_libconfig_group_lookup_bool(user_area, "in_use",
459 &cfg->user_area.in_use);
460 if (unlikely(!ret)) {
461 EM_LOG(EM_LOG_ERR,
462 "'%s.user_area.in_use' not found or wrong type\n",
463 pool_cfg_str);
464 return -1;
465 }
466
467 /* Option: startup_pools.conf[index].pool_cfg.user_area.size */
468 ret = em_libconfig_group_lookup_int(user_area, "size",
469 (int *)&cfg->user_area.size);
470 if (unlikely(!ret)) {
471 EM_LOG(EM_LOG_ERR,
472 "'%s.user_area.size' not found or wrong type\n",
473 pool_cfg_str);
474 return -1;
475 }
476
477 capa = &em_shm->mpool_tbl.odp_pool_capability;
478 /* Check whether the given value is valid or not */
479 if (is_user_area_valid(cfg, capa, &err_str) < 0) {
480 EM_LOG(EM_LOG_ERR, "%s: %ld\n", err_str, cfg->user_area.size);
481 return -1;
482 }
483
484 return 0;
485}
486
487/* Read option: startup_pools.conf[index].pool_cfg.pkt.headroom from the EM config file */
488static inline int read_config_pkt_headroom(const libconfig_group_t *pkt_headroom,
489 const char *pool_cfg_str,
490 em_pool_cfg_t *cfg/*out*/)
491{
492 int ret;
493 const odp_pool_capability_t *capa;
494
495 /*Option: startup_pools.conf[index].pool_cfg.pkt.headroom.in_use*/
496 ret = em_libconfig_group_lookup_bool(pkt_headroom, "in_use",
497 &cfg->pkt.headroom.in_use);
498 if (unlikely(!ret)) {
499 EM_LOG(EM_LOG_ERR,
500 "'%s.pkt.headroom.in_use' not found or wrong type\n",
501 pool_cfg_str);
502 return -1;
503 }
504
505 /*Option: startup_pools.conf[index].pool_cfg.pkt.headroom.value*/
506 ret = em_libconfig_group_lookup_int(pkt_headroom, "value",
507 (int *)&cfg->pkt.headroom.value);
508 if (unlikely(!ret)) {
509 EM_LOG(EM_LOG_ERR,
510 "'%s.pkt.headroom.value' not found or wrong type\n",
511 pool_cfg_str);
512 return -1;
513 }
514
515 /* Check whether the given value is valid or not */
516 capa = &em_shm->mpool_tbl.odp_pool_capability;
517 if (cfg->pkt.headroom.in_use &&
518 cfg->pkt.headroom.value > capa->pkt.max_headroom) {
519 EM_LOG(EM_LOG_ERR,
520 "'%s.pkt.headroom.value' %d too large (max=%d)\n",
521 pool_cfg_str, cfg->pkt.headroom.value,
522 capa->pkt.max_headroom);
523 return -1;
524 }
525
526 return 0;
527}
528
529/* Read option: startup_pools.conf[index] from the EM config file */
530static int read_config_startup_pools_conf(const libconfig_list_t *list, int index)
531{
532 int ret;
533 int pool;
534 int ret_pool;
535 int num_subpools;
536 const char *pool_name;
537 const char *event_type;
538 char pool_cfg_str[40];
539 libconfig_group_t *pool_cfg;
540 const libconfig_list_t *subpool;
541 const libconfig_group_t *headroom;
542 const libconfig_group_t *user_area;
543 const libconfig_group_t *align_offset;
544 em_startup_pool_conf_t *conf = &em_shm->opt.startup_pools.conf[index];
545 em_pool_cfg_t *cfg = &conf->cfg;
546 const char *err_str = "";
547
548 snprintf(pool_cfg_str, sizeof(pool_cfg_str),
549 "startup_pools.conf[%d].pool_cfg", index);
550
551 pool_cfg = em_libconfig_list_lookup_group(list, index, "pool_cfg");
552 if (!pool_cfg) {
553 EM_LOG(EM_LOG_ERR, "Conf option '%s' not found\n", pool_cfg_str);
554 return -1;
555 }
556
557 em_pool_cfg_init(cfg);
558
559 /*
560 * Read mandatory fields first, in case they are not provided, no need
561 * to proceed to read optional fields.
562 */
563
564 /* Option: startup_pools.conf[index].pool_cfg.event_type */
565 ret = em_libconfig_group_lookup_string(pool_cfg, "event_type", &event_type);
566 if (unlikely(!ret)) {
567 EM_LOG(EM_LOG_ERR, "'%s.event_type' not found.\n", pool_cfg_str);
568 return -1;
569 }
570
571 ret = event_type_from_string(event_type, &cfg->event_type/*out*/);
572 if (unlikely(ret < 0))
573 return -1;
574
575 ret = is_pool_type_supported(cfg->event_type, &err_str/*out*/);
576 if (unlikely(ret)) {
577 EM_LOG(EM_LOG_ERR, "%s", err_str);
578 return -1;
579 }
580
581 /* Option: startup_pools.conf[index].pool_cfg.num_subpools */
582 ret = em_libconfig_group_lookup_int(pool_cfg, "num_subpools",
583 &cfg->num_subpools);
584 if (unlikely(!ret)) {
585 EM_LOG(EM_LOG_ERR, "'%s.num_subpools' not found.\n", pool_cfg_str);
586 return -1;
587 }
588
589 if (cfg->num_subpools <= 0 || cfg->num_subpools > EM_MAX_SUBPOOLS) {
590 EM_LOG(EM_LOG_ERR, "Invalid '%s.num_subpools'\n"
591 "Valid value range is [1, %d]\n", pool_cfg_str,
592 EM_MAX_SUBPOOLS);
593 return -1;
594 }
595
596 /* Option: startup_pools.conf[index].pool_cfg.subpools */
597 subpool = em_libconfig_group_lookup_list(pool_cfg, "subpools");
598 if (unlikely(!subpool)) {
599 EM_LOG(EM_LOG_ERR, "'%s.subpools' not found.\n", pool_cfg_str);
600 return -1;
601 }
602
603 num_subpools = em_libconfig_list_length(subpool);
604 if (unlikely(num_subpools != cfg->num_subpools)) {
605 EM_LOG(EM_LOG_ERR, "The number of subpool configuration given\n"
606 "in '%s.subpools' does not match '%s.num_subpools'.\n",
607 pool_cfg_str, pool_cfg_str);
608 return -1;
609 }
610
611 for (int j = 0; j < num_subpools; j++) {
612 ret = read_config_subpool(subpool, j, pool_cfg_str, cfg);
613
614 if (unlikely(ret < 0))
615 return -1;
616 }
617
618 /* Following are optional configurations */
619
620 /* Option: startup_pools.conf[index].pool */
621 ret_pool = em_libconfig_list_lookup_int(list, index, "pool", &pool);
622 if (unlikely(ret_pool == 0)) {
623 EM_LOG(EM_LOG_ERR,
624 "'startup_pools.conf[%d].pool' has wrong data type(expect int)\n",
625 index);
626 return -1;
627 }
628
629 /* startup_pools.conf[index].pool is provided */
630 if (ret_pool == 1) {
631 if (pool < 0 || pool > EM_CONFIG_POOLS) {
632 EM_LOG(EM_LOG_ERR, "Invalid pool ID %d, valid IDs are within [0, %d]\n",
633 pool, EM_CONFIG_POOLS);
634 return -1;
635 }
636
637 conf->pool = (em_pool_t)(uintptr_t)pool;
638 }
639
640 /* Option: startup_pools.conf[index].name */
641 ret = em_libconfig_list_lookup_string(list, index, "name", &pool_name);
642 if (unlikely(ret == 0)) {
643 EM_LOG(EM_LOG_ERR,
644 "'startup_pools.conf[%d].name' has wrong data type(expect string)\n",
645 index);
646 return -1;
647 }
648
649 if (ret_pool == 1 && ret == 1) { /*Both pool and name have been given*/
650 const char *is_default_name = strstr(pool_name, EM_POOL_DEFAULT_NAME);
651 bool is_default_id = (conf->pool == EM_POOL_DEFAULT);
652
653 if (is_default_name && !is_default_id) {
654 EM_LOG(EM_LOG_ERR,
655 "Default name \"%s\" with non-default ID %d\n",
656 EM_POOL_DEFAULT_NAME, (int)(uintptr_t)conf->pool);
657 return -1;
658 }
659
660 if (is_default_id && !is_default_name) {
661 EM_LOG(EM_LOG_ERR,
662 "Default pool ID 1 with non-default name \"%s\"\n",
663 pool_name);
664 return -1;
665 }
666 }
667
668 if (ret == 1) { /* Pool name is given and no conflict with pool ID */
669 strncpy(conf->name, pool_name, EM_POOL_NAME_LEN - 1);
670 conf->name[EM_POOL_NAME_LEN - 1] = '\0';
671 }
672
673 align_offset = em_libconfig_group_lookup_group(pool_cfg, "align_offset");
674 /*align_offset is provided*/
675 if (align_offset && read_config_align_offset(align_offset, pool_cfg_str, cfg))
676 return -1;
677
678 user_area = em_libconfig_group_lookup_group(pool_cfg, "user_area");
679 if (user_area && read_config_user_area(user_area, pool_cfg_str, cfg))
680 return -1;
681
682 headroom = em_libconfig_group_lookup_group(pool_cfg, "pkt.headroom");
683 if (headroom) {
684 if (read_config_pkt_headroom(headroom, pool_cfg_str, cfg))
685 return -1;
686
687 /* Ignore the given pkt.headroom for non packet event type */
688 if (conf->cfg.event_type != EM_EVENT_TYPE_PACKET)
689 EM_PRINT("pkt.headroom will be ignored for non packet type!\n");
690 }
691
692 return 0;
693}
694
695/* Print option: startup_pools from the EM config file */
696static void print_config_startup_pools(void)
697{
698 em_startup_pool_conf_t *conf;
699 char str_conf[32];
700 const char *str = "";
701
702 EM_PRINT(" startup_pools.num: %u\n", em_shm->opt.startup_pools.num);
703
704 for (uint32_t i = 0; i < em_shm->opt.startup_pools.num; i++) {
705 conf = &em_shm->opt.startup_pools.conf[i];
706
707 snprintf(str_conf, sizeof(str_conf), " startup_pools.conf[%d]", i);
708
709 if (conf->name[0])
710 EM_PRINT("%s.name: %s\n", str_conf, conf->name);
711
712 if (conf->pool)
713 EM_PRINT("%s.pool: %d\n", str_conf, (int)(uintptr_t)conf->pool);
714
715 /*event type*/
716 if (conf->cfg.event_type == EM_EVENT_TYPE_SW)
717 str = "EM_EVENT_TYPE_SW";
718 else if (conf->cfg.event_type == EM_EVENT_TYPE_PACKET)
719 str = "EM_EVENT_TYPE_PACKET";
720 else if (conf->cfg.event_type == EM_EVENT_TYPE_VECTOR)
721 str = "EM_EVENT_TYPE_VECTOR";
722 EM_PRINT("%s.pool_cfg.event_type: %s\n", str_conf, str);
723
724 /*align_offset*/
725 str = conf->cfg.align_offset.in_use ? "true" : "false";
726 EM_PRINT("%s.pool_cfg.align_offset.in_use: %s\n", str_conf, str);
727 EM_PRINT("%s.pool_cfg.align_offset.value: %d\n", str_conf,
728 conf->cfg.align_offset.value);
729
730 /*user area*/
731 str = conf->cfg.user_area.in_use ? "true" : "false";
732 EM_PRINT("%s.pool_cfg.user_area.in_use: %s\n", str_conf, str);
733 EM_PRINT("%s.pool_cfg.user_area.size: %ld\n", str_conf,
734 conf->cfg.user_area.size);
735
736 /*pkt headroom*/
737 str = conf->cfg.pkt.headroom.in_use ? "true" : "false";
738 EM_PRINT("%s.pool_cfg.pkt.headroom.in_use: %s\n", str_conf, str);
739 EM_PRINT("%s.pool_cfg.pkt.headroom.value: %d\n", str_conf,
740 conf->cfg.pkt.headroom.value);
741
742 /*number of subpools*/
743 EM_PRINT("%s.pool_cfg.num_subpools: %u\n", str_conf,
744 conf->cfg.num_subpools);
745
746 /*subpools*/
747 for (int j = 0; j < conf->cfg.num_subpools; j++) {
748 EM_PRINT("%s.pool_cfg.subpools[%d].size: %u\n", str_conf,
749 j, conf->cfg.subpool[j].size);
750
751 EM_PRINT("%s.pool_cfg.subpools[%d].num: %u\n", str_conf,
752 j, conf->cfg.subpool[j].num);
753
754 EM_PRINT("%s.pool_cfg.subpools[%d].cache_size: %u\n",
755 str_conf, j, conf->cfg.subpool[j].cache_size);
756 }
757 }
758}
759
760/* Read option: startup_pools from the EM config file */
761static int read_config_startup_pools(void)
762{
763 int ret;
764 int list_len;
765 int num_startup_pools;
766 const libconfig_list_t *conf_list;
767 libconfig_setting_t *default_setting;
768 libconfig_setting_t *runtime_setting;
769 libconfig_setting_t *startup_pools_setting;
770
771 em_libconfig_lookup(&em_shm->libconfig, "startup_pools",
772 &default_setting, &runtime_setting);
773
774 /*
775 * Option: startup_pools
776 *
777 * Optional. Thus, when runtime configuration is provided, and option
778 * "startup_pools" is given, use it. However, when option "startup_pools"
779 * is not specified in the given runtime configuration file, returns
780 * without giving error, which means no startup pools will be created.
781 * Note that it does not fall back to use the option "startup_pools"
782 * specified in the default configuration file.
783 */
784 if (em_shm->libconfig.has_cfg_runtime) {
785 if (runtime_setting)
786 startup_pools_setting = runtime_setting;
787 else
788 return 0;
789 } else {
790 if (default_setting)
791 startup_pools_setting = default_setting;
792 else
793 return 0;
794 }
795
796 EM_PRINT("EM-startup_pools config:\n");
797 /*
798 * Option: startup_pools.num
799 * Mandatory when startup_pools option is given
800 */
801 ret = em_libconfig_setting_lookup_int(startup_pools_setting, "num",
802 &num_startup_pools);
803 if (unlikely(!ret)) {
804 EM_LOG(EM_LOG_ERR, "Option 'startup_pools.num' not found\n");
805 return -1;
806 }
807
808 if (num_startup_pools <= 0 || num_startup_pools > EM_CONFIG_POOLS - 1) {
809 EM_LOG(EM_LOG_ERR,
810 "Number of startup_pools %d is too large or too small\n"
811 "Valid value range is [1, %d]\n",
812 num_startup_pools, EM_CONFIG_POOLS - 1);
813 return -1;
814 }
815
816 conf_list = em_libconfig_setting_get_list(startup_pools_setting, "conf");
817 if (!conf_list) {
818 EM_LOG(EM_LOG_ERR, "Conf option 'startup_pools.conf' not found\n");
819 return -1;
820 }
821
822 list_len = em_libconfig_list_length(conf_list);
823 if (list_len != num_startup_pools) {
824 EM_LOG(EM_LOG_ERR,
825 "The number of pool configuration(s) given in\n"
826 "'startup_pools.conf':%d does not match number of\n"
827 "startup_pools specified in 'startup_pools.num': %d\n",
828 list_len, num_startup_pools);
829 return -1;
830 }
831
832 for (int i = 0; i < list_len; i++) {
833 if (read_config_startup_pools_conf(conf_list, i) < 0)
834 return -1;
835 }
836
837 em_shm->opt.startup_pools.num = num_startup_pools;
838
839 print_config_startup_pools();
840 return 0;
841}
842
843/* Read option: pool from the EM config file */
844static int read_config_pool(void)
845{
846 const char *conf_str;
847 bool val_bool = false;
848 int val = 0;
849 int ret;
850
851 const odp_pool_capability_t *capa =
852 &em_shm->mpool_tbl.odp_pool_capability;
853
854 EM_PRINT("EM pool config:\n");
855
856 /*
857 * Option: pool.statistics.available
858 */
859 conf_str = "pool.statistics.available";
860 ret = em_libconfig_lookup_bool(&em_shm->libconfig, conf_str, &val_bool);
861 if (unlikely(!ret)) {
862 EM_LOG(EM_LOG_ERR, "Config option '%s' not found\n", conf_str);
863 return -1;
864 }
865 em_shm->opt.pool.statistics.available = (int)val_bool;
866 EM_PRINT(" %s: %s(%d)\n", conf_str, val_bool ? "true" : "false", val_bool);
867
868 /*
869 * Option: pool.statistics.alloc_ops
870 */
871 conf_str = "pool.statistics.alloc_ops";
872 ret = em_libconfig_lookup_bool(&em_shm->libconfig, conf_str, &val_bool);
873 if (unlikely(!ret)) {
874 EM_LOG(EM_LOG_ERR, "Config option '%s' not found\n", conf_str);
875 return -1;
876 }
877 em_shm->opt.pool.statistics.alloc_ops = (int)val_bool;
878 EM_PRINT(" %s: %s(%d)\n", conf_str, val_bool ? "true" : "false", val_bool);
879
880 /*
881 * Option: pool.statistics.alloc_fails
882 */
883 conf_str = "pool.statistics.alloc_fails";
884 ret = em_libconfig_lookup_bool(&em_shm->libconfig, conf_str, &val_bool);
885 if (unlikely(!ret)) {
886 EM_LOG(EM_LOG_ERR, "Config option '%s' not found\n", conf_str);
887 return -1;
888 }
889 em_shm->opt.pool.statistics.alloc_fails = (int)val_bool;
890 EM_PRINT(" %s: %s(%d)\n", conf_str, val_bool ? "true" : "false", val_bool);
891
892 /*
893 * Option: pool.statistics.free_ops
894 */
895 conf_str = "pool.statistics.free_ops";
896 ret = em_libconfig_lookup_bool(&em_shm->libconfig, conf_str, &val_bool);
897 if (unlikely(!ret)) {
898 EM_LOG(EM_LOG_ERR, "Config option '%s' not found\n", conf_str);
899 return -1;
900 }
901 em_shm->opt.pool.statistics.free_ops = (int)val_bool;
902 EM_PRINT(" %s: %s(%d)\n", conf_str, val_bool ? "true" : "false", val_bool);
903
904 /*
905 * Option: pool.statistics.total_ops
906 */
907 conf_str = "pool.statistics.total_ops";
908 ret = em_libconfig_lookup_bool(&em_shm->libconfig, conf_str, &val_bool);
909 if (unlikely(!ret)) {
910 EM_LOG(EM_LOG_ERR, "Config option '%s' not found\n", conf_str);
911 return -1;
912 }
913 em_shm->opt.pool.statistics.total_ops = (int)val_bool;
914 EM_PRINT(" %s: %s(%d)\n", conf_str, val_bool ? "true" : "false", val_bool);
915
916 /*
917 * Option: pool.statistics.cache_available
918 */
919 conf_str = "pool.statistics.cache_available";
920 ret = em_libconfig_lookup_bool(&em_shm->libconfig, conf_str, &val_bool);
921 if (unlikely(!ret)) {
922 EM_LOG(EM_LOG_ERR, "Config option '%s' not found\n", conf_str);
923 return -1;
924 }
925 em_shm->opt.pool.statistics.cache_available = (int)val_bool;
926 EM_PRINT(" %s: %s(%d)\n", conf_str, val_bool ? "true" : "false", val_bool);
927
928 /*
929 * Option: pool.statistics.cache_alloc_ops
930 */
931 conf_str = "pool.statistics.cache_alloc_ops";
932 ret = em_libconfig_lookup_bool(&em_shm->libconfig, conf_str, &val_bool);
933 if (unlikely(!ret)) {
934 EM_LOG(EM_LOG_ERR, "Config option '%s' not found\n", conf_str);
935 return -1;
936 }
937 em_shm->opt.pool.statistics.cache_alloc_ops = (int)val_bool;
938 EM_PRINT(" %s: %s(%d)\n", conf_str, val_bool ? "true" : "false", val_bool);
939
940 /*
941 * Option: pool.statistics.cache_free_ops
942 */
943 conf_str = "pool.statistics.cache_free_ops";
944 ret = em_libconfig_lookup_bool(&em_shm->libconfig, conf_str, &val_bool);
945 if (unlikely(!ret)) {
946 EM_LOG(EM_LOG_ERR, "Config option '%s' not found\n", conf_str);
947 return -1;
948 }
949 em_shm->opt.pool.statistics.cache_free_ops = (int)val_bool;
950 EM_PRINT(" %s: %s(%d)\n", conf_str, val_bool ? "true" : "false", val_bool);
951
952 /*
953 * Option: pool.statistics.core_cache_available
954 */
955 conf_str = "pool.statistics.core_cache_available";
956 ret = em_libconfig_lookup_bool(&em_shm->libconfig, conf_str, &val_bool);
957 if (unlikely(!ret)) {
958 EM_LOG(EM_LOG_ERR, "Config option '%s' not found\n", conf_str);
959 return -1;
960 }
961 em_shm->opt.pool.statistics.core_cache_available = (int)val_bool;
962 EM_PRINT(" %s: %s(%d)\n", conf_str, val_bool ? "true" : "false", val_bool);
963
964 /*
965 * Option: pool.align_offset
966 */
967 conf_str = "pool.align_offset";
968 ret = em_libconfig_lookup_int(&em_shm->libconfig, conf_str, &val);
969 if (unlikely(!ret)) {
970 EM_LOG(EM_LOG_ERR, "Config option '%s' not found.\n", conf_str);
971 return -1;
972 }
973 if (val < 0 || val > ALIGN_OFFSET_MAX || !POWEROF2(val)) {
974 EM_LOG(EM_LOG_ERR,
975 "Bad config value '%s = %d' (max: %d and value must be power of 2)\n",
976 conf_str, val, ALIGN_OFFSET_MAX);
977 return -1;
978 }
979 /* store & print the value */
980 em_shm->opt.pool.align_offset = val;
981 EM_PRINT(" %s (default): %d (max: %d)\n",
982 conf_str, val, ALIGN_OFFSET_MAX);
983
984 /*
985 * Option: pool.user_area_size
986 */
987 conf_str = "pool.user_area_size";
988 ret = em_libconfig_lookup_int(&em_shm->libconfig, conf_str, &val);
989 if (unlikely(!ret)) {
990 EM_LOG(EM_LOG_ERR, "Config option '%s' not found.\n", conf_str);
991 return -1;
992 }
993 if (val < 0 || (unsigned int)val > capa->pkt.max_uarea_size ||
994 val > EM_EVENT_USER_AREA_MAX_SIZE) {
995 EM_LOG(EM_LOG_ERR, "Bad config value '%s = %d'\n",
996 conf_str, val);
997 return -1;
998 }
999 /* store & print the value */
1000 em_shm->opt.pool.user_area_size = val;
1001 EM_PRINT(" %s (default): %d (max: %d)\n",
1002 conf_str, val,
1003 MIN(EM_EVENT_USER_AREA_MAX_SIZE, capa->pkt.max_uarea_size));
1004
1005 /*
1006 * Option: pool.pkt_headroom
1007 */
1008 conf_str = "pool.pkt_headroom";
1009 ret = em_libconfig_lookup_int(&em_shm->libconfig, conf_str, &val);
1010 if (unlikely(!ret)) {
1011 EM_LOG(EM_LOG_ERR, "Config option '%s' not found.\n", conf_str);
1012 return -1;
1013 }
1014
1015 if (val < 0 || (unsigned int)val > capa->pkt.max_headroom) {
1016 EM_LOG(EM_LOG_ERR, "Bad config value '%s = %d'\n",
1017 conf_str, val);
1018 return -1;
1019 }
1020 /* store & print the value */
1021 em_shm->opt.pool.pkt_headroom = val;
1022 EM_PRINT(" %s (default): %d (max: %u)\n",
1023 conf_str, val, capa->pkt.max_headroom);
1024
1025 return 0;
1026}
1027
1028static int
1029read_config_file(void)
1030{
1031 /* Option: pool */
1032 if (read_config_pool() < 0)
1033 return -1;
1034
1035 /* Option: startup_pools */
1036 if (read_config_startup_pools() < 0)
1037 return -1;
1038
1039 return 0;
1040}
1041
1042/* We use following static asserts and function check_em_pool_subpool_stats()
1043 * to verify at both compile time and runtime that, em_pool_subpool_stats_t is
1044 * exactly the same as odp_pool_stats_t except the last struct member, namely,
1045 * 'em_pool_subpool_stats_t::__internal_use', whose size must also be bigger
1046 * than that of 'odp_pool_stats_t::thread'. This allows us to avoid exposing ODP
1047 * type in EM-ODP API (at event_machine_pool.h in this case) and allows us to
1048 * type cast 'em_pool_subpool_stats_t' to 'odp_pool_stats_t', ensuring high
1049 * performance (see em_pool_stats() and em_pool_subpool_stats()).
1050 */
1051
1052ODP_STATIC_ASSERT(sizeof(odp_pool_stats_t) <= sizeof(em_pool_subpool_stats_t),
1053 "Size of odp_pool_stats_t must be smaller than that of em_pool_subpool_stats_t");
1054
1055ODP_STATIC_ASSERT(offsetof(odp_pool_stats_t, available) ==
1056 offsetof(em_pool_subpool_stats_t, available) &&
1057 sizeof_field(odp_pool_stats_t, available) ==
1058 sizeof_field(em_pool_subpool_stats_t, available),
1059 "em_pool_subpool_stats_t.available differs from odp_pool_stats_t.available!");
1060
1061ODP_STATIC_ASSERT(offsetof(odp_pool_stats_t, alloc_ops) ==
1062 offsetof(em_pool_subpool_stats_t, alloc_ops) &&
1063 sizeof_field(odp_pool_stats_t, alloc_ops) ==
1064 sizeof_field(em_pool_subpool_stats_t, alloc_ops),
1065 "em_pool_subpool_stats_t.alloc_ops differs from odp_pool_stats_t.alloc_ops!");
1066
1067ODP_STATIC_ASSERT(offsetof(odp_pool_stats_t, alloc_fails) ==
1068 offsetof(em_pool_subpool_stats_t, alloc_fails) &&
1069 sizeof_field(odp_pool_stats_t, alloc_fails) ==
1070 sizeof_field(em_pool_subpool_stats_t, alloc_fails),
1071 "em_pool_subpool_stats_t.alloc_fails differs from odp_pool_stats_t.alloc_fails!");
1072
1073ODP_STATIC_ASSERT(offsetof(odp_pool_stats_t, free_ops) ==
1074 offsetof(em_pool_subpool_stats_t, free_ops) &&
1075 sizeof_field(odp_pool_stats_t, free_ops) ==
1076 sizeof_field(em_pool_subpool_stats_t, free_ops),
1077 "em_pool_subpool_stats_t.free_ops differs from odp_pool_stats_t.free_ops!");
1078
1079ODP_STATIC_ASSERT(offsetof(odp_pool_stats_t, total_ops) ==
1080 offsetof(em_pool_subpool_stats_t, total_ops) &&
1081 sizeof_field(odp_pool_stats_t, total_ops) ==
1082 sizeof_field(em_pool_subpool_stats_t, total_ops),
1083 "em_pool_subpool_stats_t.total_ops differs from odp_pool_stats_t.total_ops!");
1084
1085ODP_STATIC_ASSERT(offsetof(odp_pool_stats_t, cache_available) ==
1086 offsetof(em_pool_subpool_stats_t, cache_available) &&
1087 sizeof_field(odp_pool_stats_t, cache_available) ==
1088 sizeof_field(em_pool_subpool_stats_t, cache_available),
1089 "em_pool_subpool_stats_t.cache_available differs from that of odp_pool_stats_t!");
1090
1091ODP_STATIC_ASSERT(offsetof(odp_pool_stats_t, cache_alloc_ops) ==
1092 offsetof(em_pool_subpool_stats_t, cache_alloc_ops) &&
1093 sizeof_field(odp_pool_stats_t, cache_alloc_ops) ==
1094 sizeof_field(em_pool_subpool_stats_t, cache_alloc_ops),
1095 "em_pool_subpool_stats_t.cache_alloc_ops differs from that of odp_pool_stats_t!");
1096
1097ODP_STATIC_ASSERT(offsetof(odp_pool_stats_t, cache_free_ops) ==
1098 offsetof(em_pool_subpool_stats_t, cache_free_ops) &&
1099 sizeof_field(odp_pool_stats_t, cache_free_ops) ==
1100 sizeof_field(em_pool_subpool_stats_t, cache_free_ops),
1101 "em_pool_subpool_stats_t.cache_free_ops differs from that of odp_pool_stats_t!");
1102
1103ODP_STATIC_ASSERT(offsetof(odp_pool_stats_t, thread) ==
1104 offsetof(em_pool_subpool_stats_t, __internal_use) &&
1105 sizeof_field(odp_pool_stats_t, thread) <=
1106 sizeof_field(em_pool_subpool_stats_t, __internal_use),
1107 "em_pool_subpool_stats_t.__internal_use differs from odp_pool_stats_t.thread");
1108
1109#define STRUCT_ERR_STR \
1110"em_pool_subpool_stats_t.%s differs from odp_pool_stats_t.%s either in size or in offset!\n"
1111
1112static int check_em_pool_subpool_stats(void)
1113{
1114 if (sizeof(odp_pool_stats_t) > sizeof(em_pool_subpool_stats_t)) {
1115 EM_LOG(EM_LOG_ERR,
1116 "Size of odp_pool_stats_t bigger than that of em_pool_subpool_stats_t\n");
1117 return -1;
1118 }
1119
1120 if (offsetof(odp_pool_stats_t, available) !=
1121 offsetof(em_pool_subpool_stats_t, available) ||
1122 sizeof_field(odp_pool_stats_t, available) !=
1123 sizeof_field(em_pool_subpool_stats_t, available)) {
1124 EM_LOG(EM_LOG_ERR, STRUCT_ERR_STR, "available", "available");
1125 return -1;
1126 }
1127
1128 if (offsetof(odp_pool_stats_t, alloc_ops) !=
1129 offsetof(em_pool_subpool_stats_t, alloc_ops) ||
1130 sizeof_field(odp_pool_stats_t, alloc_ops) !=
1131 sizeof_field(em_pool_subpool_stats_t, alloc_ops)) {
1132 EM_LOG(EM_LOG_ERR, STRUCT_ERR_STR, "alloc_ops", "alloc_ops");
1133 return -1;
1134 }
1135
1136 if (offsetof(odp_pool_stats_t, alloc_fails) !=
1137 offsetof(em_pool_subpool_stats_t, alloc_fails) ||
1138 sizeof_field(odp_pool_stats_t, alloc_fails) !=
1139 sizeof_field(em_pool_subpool_stats_t, alloc_fails)) {
1140 EM_LOG(EM_LOG_ERR, STRUCT_ERR_STR, "alloc_fails", "alloc_fails");
1141 return -1;
1142 }
1143
1144 if (offsetof(odp_pool_stats_t, free_ops) !=
1145 offsetof(em_pool_subpool_stats_t, free_ops) ||
1146 sizeof_field(odp_pool_stats_t, free_ops) !=
1147 sizeof_field(em_pool_subpool_stats_t, free_ops)) {
1148 EM_LOG(EM_LOG_ERR, STRUCT_ERR_STR, "free_ops", "free_ops");
1149 return -1;
1150 }
1151
1152 if (offsetof(odp_pool_stats_t, total_ops) !=
1153 offsetof(em_pool_subpool_stats_t, total_ops) ||
1154 sizeof_field(odp_pool_stats_t, total_ops) !=
1155 sizeof_field(em_pool_subpool_stats_t, total_ops)) {
1156 EM_LOG(EM_LOG_ERR, STRUCT_ERR_STR, "total_ops", "total_ops");
1157 return -1;
1158 }
1159
1160 if (offsetof(odp_pool_stats_t, cache_available) !=
1161 offsetof(em_pool_subpool_stats_t, cache_available) ||
1162 sizeof_field(odp_pool_stats_t, cache_available) !=
1163 sizeof_field(em_pool_subpool_stats_t, cache_available)) {
1164 EM_LOG(EM_LOG_ERR, STRUCT_ERR_STR, "cache_available", "cache_available");
1165 return -1;
1166 }
1167
1168 if (offsetof(odp_pool_stats_t, cache_alloc_ops) !=
1169 offsetof(em_pool_subpool_stats_t, cache_alloc_ops) ||
1170 sizeof_field(odp_pool_stats_t, cache_alloc_ops) !=
1171 sizeof_field(em_pool_subpool_stats_t, cache_alloc_ops)) {
1172 EM_LOG(EM_LOG_ERR, STRUCT_ERR_STR, "cache_alloc_ops", "cache_alloc_ops");
1173 return -1;
1174 }
1175
1176 if (offsetof(odp_pool_stats_t, cache_free_ops) !=
1177 offsetof(em_pool_subpool_stats_t, cache_free_ops) ||
1178 sizeof_field(odp_pool_stats_t, cache_free_ops) !=
1179 sizeof_field(em_pool_subpool_stats_t, cache_free_ops)) {
1180 EM_LOG(EM_LOG_ERR, STRUCT_ERR_STR, "cache_free_ops", "cache_free_ops");
1181 return -1;
1182 }
1183
1184 if (offsetof(odp_pool_stats_t, thread) !=
1185 offsetof(em_pool_subpool_stats_t, __internal_use) ||
1186 sizeof_field(odp_pool_stats_t, thread) >
1187 sizeof_field(em_pool_subpool_stats_t, __internal_use)) {
1188 EM_LOG(EM_LOG_ERR, STRUCT_ERR_STR, "__internal_use", "thread");
1189 return -1;
1190 }
1191
1192 return 0;
1193}
1194
1195/* We use following static asserts and function check_em_pool_subpool_stats_selected()
1196 * to verify at both compile time and runtime that, em_pool_subpool_stats_selected_t
1197 * is exactly the same as odp_pool_stats_selected_t This allows us to avoid exposing
1198 * ODP type in EM-ODP API (at event_machine_pool.h in this case) and allows us to
1199 * type cast 'em_pool_subpool_stats_selected_t' to 'odp_pool_stats_selected_t', ensuring
1200 * high performance (see em_pool_stats_selected() and em_pool_subpool_stats_selected()).
1201 */
1202
1203#define SIZE_NOT_EQUAL_ERR_STR \
1204"Size of odp_pool_stats_selected_t must equal to that of em_pool_subpool_stats_selected_t\n"
1205
1206ODP_STATIC_ASSERT(sizeof(odp_pool_stats_selected_t) == sizeof(em_pool_subpool_stats_selected_t),
1207 SIZE_NOT_EQUAL_ERR_STR);
1208
1209ODP_STATIC_ASSERT(offsetof(odp_pool_stats_selected_t, available) ==
1210 offsetof(em_pool_subpool_stats_selected_t, available) &&
1211 sizeof_field(odp_pool_stats_selected_t, available) ==
1212 sizeof_field(em_pool_subpool_stats_selected_t, available),
1213 "available in em_pool_subpool_stats_selected_t and odp_pool_stats_selected_t differs!");
1214
1215ODP_STATIC_ASSERT(offsetof(odp_pool_stats_selected_t, alloc_ops) ==
1216 offsetof(em_pool_subpool_stats_selected_t, alloc_ops) &&
1217 sizeof_field(odp_pool_stats_selected_t, alloc_ops) ==
1218 sizeof_field(em_pool_subpool_stats_selected_t, alloc_ops),
1219 "em_pool_subpool_stats_selected_t.alloc_ops differs from odp_pool_stats_selected_t.alloc_ops!");
1220
1221ODP_STATIC_ASSERT(offsetof(odp_pool_stats_selected_t, alloc_fails) ==
1222 offsetof(em_pool_subpool_stats_t, alloc_fails) &&
1223 sizeof_field(odp_pool_stats_selected_t, alloc_fails) ==
1224 sizeof_field(em_pool_subpool_stats_t, alloc_fails),
1225 "em_pool_subpool_stats_selected_t.alloc_fails differs from odp_pool_stats_selected_t.alloc_fails!");
1226
1227ODP_STATIC_ASSERT(offsetof(odp_pool_stats_selected_t, free_ops) ==
1228 offsetof(em_pool_subpool_stats_selected_t, free_ops) &&
1229 sizeof_field(odp_pool_stats_selected_t, free_ops) ==
1230 sizeof_field(em_pool_subpool_stats_selected_t, free_ops),
1231 "em_pool_subpool_stats_selected_t.free_ops differs from odp_pool_stats_selected_t.free_ops!");
1232
1233ODP_STATIC_ASSERT(offsetof(odp_pool_stats_selected_t, total_ops) ==
1234 offsetof(em_pool_subpool_stats_selected_t, total_ops) &&
1235 sizeof_field(odp_pool_stats_selected_t, total_ops) ==
1236 sizeof_field(em_pool_subpool_stats_selected_t, total_ops),
1237 "em_pool_subpool_stats_selected_t.total_ops differs from odp_pool_stats_selected_t.total_ops!");
1238
1239ODP_STATIC_ASSERT(offsetof(odp_pool_stats_selected_t, cache_available) ==
1240 offsetof(em_pool_subpool_stats_selected_t, cache_available) &&
1241 sizeof_field(odp_pool_stats_selected_t, cache_available) ==
1242 sizeof_field(em_pool_subpool_stats_selected_t, cache_available),
1243 "em_pool_subpool_stats_selected_t.cache_available differs from that of odp_pool_stats_selected_t!");
1244
1245ODP_STATIC_ASSERT(offsetof(odp_pool_stats_selected_t, cache_alloc_ops) ==
1246 offsetof(em_pool_subpool_stats_selected_t, cache_alloc_ops) &&
1247 sizeof_field(odp_pool_stats_selected_t, cache_alloc_ops) ==
1248 sizeof_field(em_pool_subpool_stats_selected_t, cache_alloc_ops),
1249 "em_pool_subpool_stats_selected_t.cache_alloc_ops differs from that of odp_pool_stats_selected_t!");
1250
1251ODP_STATIC_ASSERT(offsetof(odp_pool_stats_selected_t, cache_free_ops) ==
1252 offsetof(em_pool_subpool_stats_selected_t, cache_free_ops) &&
1253 sizeof_field(odp_pool_stats_selected_t, cache_free_ops) ==
1254 sizeof_field(em_pool_subpool_stats_selected_t, cache_free_ops),
1255 "em_pool_subpool_stats_selected_t.cache_free_ops differs from that of odp_pool_stats_selected_t!");
1256
1257#define SELECTED_TYPE_ERR_FMT \
1258"em_pool_subpool_stats_selected_t.%s differs from odp_pool_stats_selected_t.%s\n"
1259
1260static int check_em_pool_subpool_stats_selected(void)
1261{
1262 if (sizeof(odp_pool_stats_selected_t) != sizeof(em_pool_subpool_stats_selected_t)) {
1263 EM_LOG(EM_LOG_ERR,
1264 "odp_pool_stats_selected_t vs em_pool_subpool_stats_selected_t size diff\n");
1265 return -1;
1266 }
1267
1268 if (offsetof(odp_pool_stats_selected_t, available) !=
1269 offsetof(em_pool_subpool_stats_selected_t, available) ||
1270 sizeof_field(odp_pool_stats_selected_t, available) !=
1271 sizeof_field(em_pool_subpool_stats_selected_t, available)) {
1272 EM_LOG(EM_LOG_ERR, SELECTED_TYPE_ERR_FMT, "available", "available");
1273 return -1;
1274 }
1275
1276 if (offsetof(odp_pool_stats_selected_t, alloc_ops) !=
1277 offsetof(em_pool_subpool_stats_selected_t, alloc_ops) ||
1278 sizeof_field(odp_pool_stats_selected_t, alloc_ops) !=
1279 sizeof_field(em_pool_subpool_stats_selected_t, alloc_ops)) {
1280 EM_LOG(EM_LOG_ERR, SELECTED_TYPE_ERR_FMT, "alloc_ops", "alloc_ops");
1281 return -1;
1282 }
1283
1284 if (offsetof(odp_pool_stats_selected_t, alloc_fails) !=
1285 offsetof(em_pool_subpool_stats_selected_t, alloc_fails) ||
1286 sizeof_field(odp_pool_stats_selected_t, alloc_fails) !=
1287 sizeof_field(em_pool_subpool_stats_selected_t, alloc_fails)) {
1288 EM_LOG(EM_LOG_ERR, SELECTED_TYPE_ERR_FMT, "alloc_fails", "alloc_fails");
1289 return -1;
1290 }
1291
1292 if (offsetof(odp_pool_stats_selected_t, free_ops) !=
1293 offsetof(em_pool_subpool_stats_selected_t, free_ops) ||
1294 sizeof_field(odp_pool_stats_selected_t, free_ops) !=
1295 sizeof_field(em_pool_subpool_stats_selected_t, free_ops)) {
1296 EM_LOG(EM_LOG_ERR, SELECTED_TYPE_ERR_FMT, "free_ops", "free_ops");
1297 return -1;
1298 }
1299
1300 if (offsetof(odp_pool_stats_selected_t, total_ops) !=
1301 offsetof(em_pool_subpool_stats_selected_t, total_ops) ||
1302 sizeof_field(odp_pool_stats_selected_t, total_ops) !=
1303 sizeof_field(em_pool_subpool_stats_selected_t, total_ops)) {
1304 EM_LOG(EM_LOG_ERR, SELECTED_TYPE_ERR_FMT, "total_ops", "total_ops");
1305 return -1;
1306 }
1307
1308 if (offsetof(odp_pool_stats_selected_t, cache_available) !=
1309 offsetof(em_pool_subpool_stats_selected_t, cache_available) ||
1310 sizeof_field(odp_pool_stats_selected_t, cache_available) !=
1311 sizeof_field(em_pool_subpool_stats_selected_t, cache_available)) {
1312 EM_LOG(EM_LOG_ERR, SELECTED_TYPE_ERR_FMT, "cache_available", "cache_available");
1313 return -1;
1314 }
1315
1316 if (offsetof(odp_pool_stats_selected_t, cache_alloc_ops) !=
1317 offsetof(em_pool_subpool_stats_selected_t, cache_alloc_ops) ||
1318 sizeof_field(odp_pool_stats_selected_t, cache_alloc_ops) !=
1319 sizeof_field(em_pool_subpool_stats_selected_t, cache_alloc_ops)) {
1320 EM_LOG(EM_LOG_ERR, SELECTED_TYPE_ERR_FMT, "cache_alloc_ops", "cache_alloc_ops");
1321 return -1;
1322 }
1323
1324 if (offsetof(odp_pool_stats_selected_t, cache_free_ops) !=
1325 offsetof(em_pool_subpool_stats_selected_t, cache_free_ops) ||
1326 sizeof_field(odp_pool_stats_selected_t, cache_free_ops) !=
1327 sizeof_field(em_pool_subpool_stats_selected_t, cache_free_ops)) {
1328 EM_LOG(EM_LOG_ERR, SELECTED_TYPE_ERR_FMT, "cache_free_ops", "cache_free_ops");
1329 return -1;
1330 }
1331
1332 return 0;
1333}
1334
1335ODP_STATIC_ASSERT(sizeof(odp_pool_stats_opt_t) == sizeof(em_pool_stats_opt_t),
1336 "Size of odp_pool_stats_opt_t differs from that of em_pool_stats_opt_t\n");
1337
1338em_status_t
1339pool_init(mpool_tbl_t *const mpool_tbl, mpool_pool_t *const mpool_pool,
1340 const em_pool_cfg_t *default_pool_cfg)
1341{
1342 int ret;
1343 em_pool_t pool;
1344 em_pool_t pool_default;
1345 em_startup_pool_conf_t *startup_pool_conf;
1346 bool default_pool_set = false;
1347 const uint32_t objpool_subpools = MIN(4, OBJSUBPOOLS_MAX);
1348
1349 /* Return error if em_pool_subpool_stats_t differs from odp_pool_stats_t */
1350 if (check_em_pool_subpool_stats())
1351 return EM_ERR;
1352
1353 /*Return error if em_pool_subpool_stats_selected_t differs from odp_pool_stats_selected_t*/
1354 if (check_em_pool_subpool_stats_selected())
1355 return EM_ERR;
1356
1357 memset(mpool_tbl, 0, sizeof(mpool_tbl_t));
1358 memset(mpool_pool, 0, sizeof(mpool_pool_t));
1359 odp_atomic_init_u32(&em_shm->pool_count, 0);
1360
1361 ret = objpool_init(&mpool_pool->objpool, objpool_subpools);
1362 if (ret != 0)
1363 return EM_ERR_OPERATION_FAILED;
1364
1365 for (uint32_t i = 0; i < EM_CONFIG_POOLS; i++) {
1366 pool = pool_idx2hdl(i);
1367 mpool_elem_t *mpool_elem = pool_elem_get(pool);
1368
1369 if (unlikely(!mpool_elem))
1370 return EM_ERR_BAD_POINTER;
1371
1372 mpool_elem->em_pool = pool;
1373 mpool_elem->event_type = EM_EVENT_TYPE_UNDEF;
1374 for (int j = 0; j < EM_MAX_SUBPOOLS; j++) {
1375 mpool_elem->odp_pool[j] = ODP_POOL_INVALID;
1376 mpool_elem->size[j] = 0;
1377 }
1378
1379 objpool_add(&mpool_pool->objpool, i % objpool_subpools,
1380 &mpool_elem->objpool_elem);
1381 }
1382
1383 /* Init the mapping tbl from odp-pool(=subpool) index to em-pool */
1384 if (odp_pool_max_index() >= POOL_ODP2EM_TBL_LEN)
1385 return EM_ERR_TOO_LARGE;
1386 for (int i = 0; i < POOL_ODP2EM_TBL_LEN; i++)
1387 mpool_tbl->pool_subpool_odp2em[i].both = pool_subpool_undef.both;
1388
1389 /* Store common ODP pool capabilities in the mpool_tbl for easy access*/
1390 if (odp_pool_capability(&mpool_tbl->odp_pool_capability) != 0)
1391 return EM_ERR_LIB_FAILED;
1392
1393 /* Read EM-pool and EM-startup_pools related runtime config options */
1394 if (read_config_file())
1395 return EM_ERR_LIB_FAILED;
1396
1397 /*
1398 * Create default and startup pools.
1399 *
1400 * If default pool configuration is given through 'startup_pools.conf'
1401 * in em-odp.conf, use that instead. Otherwise use default_pool_cfg.
1402 *
1403 * Allocate/reserve default pool first here so when creating startup
1404 * pools whose configuration does not provide pool handle, default pool
1405 * handle EM_POOL_DEFAULT(1) won't be allocated to them.
1406 */
1407 pool_default = pool_alloc(EM_POOL_DEFAULT);
1408
1409 if (unlikely(pool_default == EM_POOL_UNDEF ||
1410 pool_default != EM_POOL_DEFAULT))
1411 return EM_ERR_ALLOC_FAILED;
1412
1413 /* Create startup pools whose configuration is provided by the EM config file */
1414 for (uint32_t i = 0; i < em_shm->opt.startup_pools.num; i++) {
1415 startup_pool_conf = &em_shm->opt.startup_pools.conf[i];
1416
1417 /* Default pool is provided by the EM config file */
1418 if (strstr(startup_pool_conf->name, EM_POOL_DEFAULT_NAME) ||
1419 startup_pool_conf->pool == EM_POOL_DEFAULT) {
1420 default_pool_set = true;
1421 pool_free(EM_POOL_DEFAULT);
1422 pool = em_pool_create(EM_POOL_DEFAULT_NAME,
1423 EM_POOL_DEFAULT,
1424 &startup_pool_conf->cfg);
1425 } else {
1426 pool = em_pool_create(startup_pool_conf->name,
1427 startup_pool_conf->pool,
1428 &startup_pool_conf->cfg);
1429 }
1430
1431 if (pool == EM_POOL_UNDEF)
1432 return EM_ERR_ALLOC_FAILED;
1433 }
1434
1435 /* Create the default pool if it is not provided by the EM config file */
1436 if (!default_pool_set) {
1437 pool_free(EM_POOL_DEFAULT);
1438 pool = em_pool_create(EM_POOL_DEFAULT_NAME, EM_POOL_DEFAULT,
1439 default_pool_cfg);
1440 if (pool == EM_POOL_UNDEF || pool != EM_POOL_DEFAULT)
1441 return EM_ERR_ALLOC_FAILED;
1442 }
1443
1444 return EM_OK;
1445}
1446
1447em_status_t
1448pool_term(const mpool_tbl_t *mpool_tbl)
1449{
1450 em_status_t stat = EM_OK;
1451
1452 (void)mpool_tbl;
1453
1454 EM_PRINT("\n"
1455 "Status before delete:\n");
1456 em_pool_info_print_all();
1457
1458 for (int i = 0; i < EM_CONFIG_POOLS; i++) {
1459 em_pool_t pool = pool_idx2hdl(i);
1460 const mpool_elem_t *mpool_elem = pool_elem_get(pool);
1461 em_status_t ret;
1462
1463 if (mpool_elem && pool_allocated(mpool_elem)) {
1464 ret = pool_delete(pool);
1465 if (ret != EM_OK)
1466 stat = ret; /* save last error as return val */
1467 }
1468 }
1469
1470 return stat;
1471}
1472
1473/* Helper func to invalid_pool_cfg() */
1474static int invalid_pool_cache_cfg(const em_pool_cfg_t *pool_cfg,
1475 const char **err_str/*out*/)
1476{
1477 const odp_pool_capability_t *capa =
1478 &em_shm->mpool_tbl.odp_pool_capability;
1479 uint32_t min_cache_size;
1480 uint32_t cache_size;
1481
1482 if (pool_cfg->event_type == EM_EVENT_TYPE_SW)
1483 min_cache_size = capa->buf.min_cache_size;
1484 else if (pool_cfg->event_type == EM_EVENT_TYPE_PACKET)
1485 min_cache_size = capa->pkt.min_cache_size;
1486 else if (pool_cfg->event_type == EM_EVENT_TYPE_VECTOR)
1487 if (em_shm->opt.vector.backend == EM_VECTOR_BACKEND_PACKET)
1488 min_cache_size = capa->vector.min_cache_size;
1489 else /* EM_VECTOR_BACKEND_EVENT */
1490 min_cache_size = capa->event_vector.min_cache_size;
1491 else
1492 return -9;
1493
1494 for (int i = 0; i < pool_cfg->num_subpools; i++) {
1495 if (pool_cfg->subpool[i].size <= 0 ||
1496 pool_cfg->subpool[i].num <= 0) {
1497 *err_str = "Invalid subpool size/num";
1498 return -(1 * 10 + i); /* -10, -11, ... */
1499 }
1500
1501 cache_size = pool_cfg->subpool[i].cache_size;
1502 if (unlikely(cache_size < min_cache_size)) {
1503 *err_str = "Requested cache size too small";
1504 return -(2 * 10 + i); /* -20, -21, ... */
1505 }
1506 /*
1507 * If the given cache size is larger than odp-max,
1508 * then use odp-max:
1509 * if (cache_size > max_cache_size)
1510 * cache_size = max_cache_size;
1511 * This is done later in pool_create();
1512 */
1513 }
1514
1515 return 0;
1516}
1517
1518int invalid_pool_cfg(const em_pool_cfg_t *pool_cfg, const char **err_str/*out*/)
1519{
1520 int ret = 0;
1521 const odp_pool_capability_t *capa = &em_shm->mpool_tbl.odp_pool_capability;
1522
1523 if (!pool_cfg) {
1524 *err_str = "Pool config NULL";
1525 return -1;
1526 }
1527 if (pool_cfg->__internal_check != EM_CHECK_INIT_CALLED) {
1528 *err_str = "Not initialized: em_pool_cfg_init(pool_cfg) not called";
1529 return -1;
1530 }
1531
1532 if (pool_cfg->num_subpools <= 0 ||
1533 pool_cfg->num_subpools > EM_MAX_SUBPOOLS) {
1534 *err_str = "Invalid number of subpools";
1535 return -1;
1536 }
1537
1538 ret = is_pool_type_supported(pool_cfg->event_type, err_str/*out*/);
1539 if (ret)
1540 return ret;
1541
1542 ret = is_align_offset_valid(pool_cfg);
1543 if (ret) {
1544 *err_str = "Invalid align offset";
1545 return ret;
1546 }
1547
1548 ret = is_user_area_valid(pool_cfg, capa, err_str/*out*/);
1549 if (ret)
1550 return ret;
1551
1552 if (pool_cfg->event_type == EM_EVENT_TYPE_PACKET &&
1553 pool_cfg->pkt.headroom.in_use &&
1554 pool_cfg->pkt.headroom.value > capa->pkt.max_headroom) {
1555 *err_str = "Requested pkt headroom size too large";
1556 return -1;
1557 }
1558
1559 ret = invalid_pool_cache_cfg(pool_cfg, err_str/*out*/);
1560
1561 return ret; /* 0: success, <0: error */
1562}
1563
1564int check_pool_uarea_persistence(const em_pool_cfg_t *pool_cfg, const char **err_str/*out*/)
1565{
1566 bool has_uarea_persistence;
1567 const odp_pool_capability_t *capa = &em_shm->mpool_tbl.odp_pool_capability;
1568
1569 switch (pool_cfg->event_type) {
1570 case EM_EVENT_TYPE_SW:
1571 has_uarea_persistence = capa->buf.uarea_persistence ? true : false;
1572 *err_str = "buf-pool (EM_EVENT_TYPE_SW)";
1573 break;
1574 case EM_EVENT_TYPE_PACKET:
1575 has_uarea_persistence = capa->pkt.uarea_persistence ? true : false;
1576 *err_str = "pkt-pool (EM_EVENT_TYPE_PACKET)";
1577 break;
1578 case EM_EVENT_TYPE_VECTOR:
1579 if (em_shm->opt.vector.backend == EM_VECTOR_BACKEND_PACKET)
1580 has_uarea_persistence = capa->vector.uarea_persistence ? true : false;
1581 else /* EM_VECTOR_BACKEND_EVENT */
1582 has_uarea_persistence = capa->event_vector.uarea_persistence ? true : false;
1583 *err_str = "vector-pool (EM_EVENT_TYPE_VECTOR)";
1584 break;
1585 default:
1586 has_uarea_persistence = false;
1587 *err_str = "unknown pool-type";
1588 break;
1589 }
1590
1591 return has_uarea_persistence ? 0 : -1; /* 0: success, <0: not supported */
1592}
1593
1594/*
1595 * Helper to pool_create() - preallocate all events in the pool for ESV to
1596 * maintain event state over multiple alloc- and free-operations.
1597 */
1598static void
1599pool_prealloc(const mpool_elem_t *pool_elem)
1600{
1601 event_prealloc_hdr_t *prealloc_hdr = NULL;
1602 uint64_t num_tot = 0;
1603 uint64_t num = 0;
1604 uint64_t num_free = 0;
1605 const uint32_t size = pool_elem->pool_cfg.subpool[0].size;
1606 list_node_t evlist;
1607 list_node_t *node;
1608
1609 list_init(&evlist);
1610
1611 for (int i = 0; i < pool_elem->num_subpools; i++)
1612 num_tot += pool_elem->pool_cfg.subpool[i].num;
1613
1614 do {
1615 prealloc_hdr = event_prealloc(pool_elem, size);
1616 if (likely(prealloc_hdr)) {
1617 list_add(&evlist, &prealloc_hdr->list_node);
1618 num++;
1619 }
1620 } while (prealloc_hdr);
1621
1622 if (unlikely(num < num_tot))
1623 INTERNAL_ERROR(EM_FATAL(EM_ERR_TOO_SMALL),
1624 EM_ESCOPE_POOL_CREATE,
1625 "alloc: events expected:%" PRIu64 " actual:%" PRIu64 "",
1626 num_tot, num);
1627
1628 while (!list_is_empty(&evlist)) {
1629 node = list_rem_first(&evlist);
1630 prealloc_hdr = list_node_to_prealloc_hdr(node);
1631 em_free(prealloc_hdr->ev_hdr.event);
1632 num_free++;
1633 }
1634
1635 if (unlikely(num_free > num))
1636 INTERNAL_ERROR(EM_FATAL(EM_ERR_TOO_LARGE),
1637 EM_ESCOPE_POOL_CREATE,
1638 "free: events expected:%" PRIu64 " actual:%" PRIu64 "",
1639 num, num_free);
1640}
1641
1642/*
1643 * pool_create() helper: sort subpool cfg in ascending order based on buf size
1644 */
1645static void
1646sort_pool_cfg(const em_pool_cfg_t *pool_cfg, em_pool_cfg_t *sorted_cfg /*out*/)
1647{
1648 const int num_subpools = pool_cfg->num_subpools;
1649
1650 *sorted_cfg = *pool_cfg;
1651
1652 for (int i = 0; i < num_subpools - 1; i++) {
1653 int idx = i; /* array index containing smallest size */
1654
1655 for (int j = i + 1; j < num_subpools; j++) {
1656 if (sorted_cfg->subpool[j].size <
1657 sorted_cfg->subpool[idx].size)
1658 idx = j; /* store idx to smallest */
1659 }
1660
1661 /* min size at [idx], swap with [i] */
1662 if (idx != i) {
1663 uint32_t size = sorted_cfg->subpool[i].size;
1664 uint32_t num = sorted_cfg->subpool[i].num;
1665 uint32_t cache_size = sorted_cfg->subpool[i].cache_size;
1666
1667 sorted_cfg->subpool[i] = sorted_cfg->subpool[idx];
1668
1669 sorted_cfg->subpool[idx].size = size;
1670 sorted_cfg->subpool[idx].num = num;
1671 sorted_cfg->subpool[idx].cache_size = cache_size;
1672 }
1673 }
1674}
1675
1676/*
1677 * pool_create() helper: set pool event-cache size.
1678 *
1679 * Set the requested subpool cache-size based on user provided value and
1680 * limit set by odp-pool-capability.
1681 * Requested value can be larger than odp-max, use odp--max in this
1682 * case.
1683 * Verification against odp-min value done in invalid_pool_cfg().
1684 */
1685static void
1686set_poolcache_size(em_pool_cfg_t *pool_cfg)
1687{
1688 const odp_pool_capability_t *capa =
1689 &em_shm->mpool_tbl.odp_pool_capability;
1690 int num_subpools = pool_cfg->num_subpools;
1691 uint32_t max_cache_size;
1692
1693 if (pool_cfg->event_type == EM_EVENT_TYPE_SW) {
1694 max_cache_size = capa->buf.max_cache_size;
1695 } else if (pool_cfg->event_type == EM_EVENT_TYPE_PACKET) {
1696 max_cache_size = capa->pkt.max_cache_size;
1697 } else {
1698 /* EM_EVENT_TYPE_VECTOR */
1699 if (em_shm->opt.vector.backend == EM_VECTOR_BACKEND_PACKET)
1700 max_cache_size = capa->vector.max_cache_size;
1701 else /* EM_VECTOR_BACKEND_EVENT */
1702 max_cache_size = capa->event_vector.max_cache_size;
1703 }
1704
1705 for (int i = 0; i < num_subpools; i++) {
1706 if (max_cache_size < pool_cfg->subpool[i].cache_size)
1707 pool_cfg->subpool[i].cache_size = max_cache_size;
1708 }
1709}
1710
1711/*
1712 * pool_create() helper: determine user area size.
1713 */
1714static int
1715set_uarea_size(const em_pool_cfg_t *pool_cfg, size_t *uarea_size/*out*/)
1716{
1717 size_t size = 0;
1718 size_t max_size = 0;
1719 const odp_pool_capability_t *capa =
1720 &em_shm->mpool_tbl.odp_pool_capability;
1721
1722 if (pool_cfg->user_area.in_use) /* use pool-cfg */
1723 size = pool_cfg->user_area.size;
1724 else /* use cfg-file */
1725 size = em_shm->opt.pool.user_area_size;
1726
1727 if (pool_cfg->event_type == EM_EVENT_TYPE_PACKET) {
1728 max_size = MIN(capa->pkt.max_uarea_size, EM_EVENT_USER_AREA_MAX_SIZE);
1729 } else if (pool_cfg->event_type == EM_EVENT_TYPE_VECTOR) {
1730 if (em_shm->opt.vector.backend == EM_VECTOR_BACKEND_PACKET) {
1731 max_size = MIN(capa->vector.max_uarea_size,
1732 EM_EVENT_USER_AREA_MAX_SIZE);
1733 } else { /* EM_VECTOR_BACKEND_EVENT */
1734 max_size = MIN(capa->event_vector.max_uarea_size,
1735 EM_EVENT_USER_AREA_MAX_SIZE);
1736 }
1737 } else if (size > 0) {
1738 /* EM_EVENT_TYPE_SW: bufs */
1739 max_size = MIN(capa->buf.max_uarea_size, EM_EVENT_USER_AREA_MAX_SIZE);
1740 }
1741
1742 if (size > max_size)
1743 return -1;
1744
1745 *uarea_size = size;
1746
1747 return 0;
1748}
1749
1750/*
1751 * pool_create() helper: set the pkt headroom
1752 */
1753static int
1754set_pkt_headroom(const em_pool_cfg_t *pool_cfg,
1755 uint32_t *pkt_headroom /*out*/,
1756 uint32_t *max_headroom /*out, for err print only*/)
1757{
1758 const odp_pool_capability_t *capa =
1759 &em_shm->mpool_tbl.odp_pool_capability;
1760 /* default value from cfg file */
1761 uint32_t headroom = em_shm->opt.pool.pkt_headroom;
1762
1763 /* Pool-specific param overrides config file value */
1764 if (pool_cfg->pkt.headroom.in_use)
1765 headroom = pool_cfg->pkt.headroom.value;
1766
1767 *pkt_headroom = headroom;
1768 *max_headroom = capa->pkt.max_headroom;
1769
1770 if (unlikely(headroom > capa->pkt.max_headroom))
1771 return -1;
1772
1773 return 0;
1774}
1775
1776/** Helper to create_subpools() */
1777static void set_pool_params_stats(odp_pool_stats_opt_t *param_stats /*out*/,
1778 const odp_pool_stats_opt_t *capa_stats,
1779 const em_pool_stats_opt_t *stats_opt)
1780{
1781 param_stats->all = 0;
1782
1783 if (capa_stats->bit.available)
1784 param_stats->bit.available = stats_opt->available;
1785
1786 if (capa_stats->bit.alloc_ops)
1787 param_stats->bit.alloc_ops = stats_opt->alloc_ops;
1788
1789 if (capa_stats->bit.alloc_fails)
1790 param_stats->bit.alloc_fails = stats_opt->alloc_fails;
1791
1792 if (capa_stats->bit.free_ops)
1793 param_stats->bit.free_ops = stats_opt->free_ops;
1794
1795 if (capa_stats->bit.total_ops)
1796 param_stats->bit.total_ops = stats_opt->total_ops;
1797
1798 if (capa_stats->bit.cache_alloc_ops)
1799 param_stats->bit.cache_alloc_ops = stats_opt->cache_alloc_ops;
1800
1801 if (capa_stats->bit.cache_available)
1802 param_stats->bit.cache_available = stats_opt->cache_available;
1803
1804 if (capa_stats->bit.cache_free_ops)
1805 param_stats->bit.cache_free_ops = stats_opt->cache_free_ops;
1806
1807 if (capa_stats->bit.thread_cache_available)
1808 param_stats->bit.thread_cache_available = stats_opt->core_cache_available;
1809}
1810
1811/** Helper to create_subpools() */
1812static void set_pool_params_pkt(odp_pool_param_t *pool_params /* out */,
1813 const em_pool_cfg_t *pool_cfg,
1814 uint32_t size, uint32_t num, uint32_t cache_size,
1815 uint32_t align_offset, uint32_t odp_align,
1816 uint32_t uarea_size, uint32_t pkt_headroom)
1817{
1818 const odp_pool_capability_t *capa = &em_shm->mpool_tbl.odp_pool_capability;
1819
1820 odp_pool_param_init(pool_params);
1821
1822 pool_params->type = ODP_POOL_PACKET;
1823 /* num == max_num, helps pool-info stats calculation */
1824 pool_params->pkt.num = num;
1825 pool_params->pkt.max_num = num;
1826
1827 if (size > align_offset)
1828 size = size - align_offset;
1829 else
1830 size = 1; /* 0:default, can be big => use 1 */
1831 /* len == max_len */
1832 pool_params->pkt.len = size;
1833 pool_params->pkt.max_len = size;
1834 pool_params->pkt.seg_len = size;
1835 pool_params->pkt.align = odp_align;
1836 /*
1837 * Reserve space for the event header in each packet's
1838 * ODP-user-area:
1839 */
1840 pool_params->pkt.uarea_size = sizeof(event_hdr_t) + uarea_size;
1841 /*
1842 * Set the pkt headroom.
1843 * Make sure the alloc-alignment fits into the headroom.
1844 */
1845 pool_params->pkt.headroom = pkt_headroom;
1846 if (pkt_headroom < align_offset)
1847 pool_params->pkt.headroom = align_offset;
1848
1849 pool_params->pkt.cache_size = cache_size;
1850
1851 /* Pkt pool statistics */
1852 if (pool_cfg->stats_opt.in_use) {
1853 set_pool_params_stats(&pool_params->stats, &capa->pkt.stats,
1854 &pool_cfg->stats_opt.opt);
1855 } else {
1856 set_pool_params_stats(&pool_params->stats, &capa->pkt.stats,
1857 &em_shm->opt.pool.statistics);/*from cnf file*/
1858 }
1859}
1860
1861static void set_pool_params_vector(odp_pool_param_t *pool_params /* out */,
1862 const em_pool_cfg_t *pool_cfg,
1863 uint32_t size, uint32_t num,
1864 uint32_t cache_size, uint32_t uarea_size)
1865{
1866 const odp_pool_capability_t *capa = &em_shm->mpool_tbl.odp_pool_capability;
1867 const odp_pool_stats_opt_t *capa_stats_opt;
1868
1869 odp_pool_param_init(pool_params);
1870
1871 if (em_shm->opt.vector.backend == EM_VECTOR_BACKEND_PACKET) {
1872 /* ODP packet vector pool */
1873 pool_params->type = ODP_POOL_VECTOR;
1874 pool_params->vector.num = num;
1875 pool_params->vector.max_size = size;
1876 /* Reserve space for the EM event header in the vector's ODP-user-area */
1877 pool_params->vector.uarea_size = sizeof(event_hdr_t) + uarea_size;
1878 pool_params->vector.cache_size = cache_size;
1879 capa_stats_opt = &capa->vector.stats;
1880 } else { /* EM_VECTOR_BACKEND_EVENT */
1881 /* ODP event vector pool */
1882 pool_params->type = ODP_POOL_EVENT_VECTOR;
1883 pool_params->event_vector.num = num;
1884 pool_params->event_vector.max_size = size;
1885 /* Reserve space for the EM event header in the vector's ODP-user-area */
1886 pool_params->event_vector.uarea_size = sizeof(event_hdr_t) + uarea_size;
1887 pool_params->event_vector.cache_size = cache_size;
1888 capa_stats_opt = &capa->event_vector.stats;
1889 }
1890
1891 /* Vector pool statistics */
1892 if (pool_cfg->stats_opt.in_use)
1893 set_pool_params_stats(&pool_params->stats, capa_stats_opt,
1894 &pool_cfg->stats_opt.opt);
1895 else
1896 set_pool_params_stats(&pool_params->stats, capa_stats_opt,
1897 &em_shm->opt.pool.statistics);
1898}
1899
1900/** Helper to create_subpools() */
1901static void set_pool_params_buf(odp_pool_param_t *pool_params /* out */,
1902 const em_pool_cfg_t *pool_cfg,
1903 uint32_t size, uint32_t num, uint32_t cache_size,
1904 uint32_t align_offset, uint32_t odp_align,
1905 uint32_t uarea_size)
1906{
1907 const odp_pool_capability_t *capa = &em_shm->mpool_tbl.odp_pool_capability;
1908
1909 odp_pool_param_init(pool_params);
1910
1911 pool_params->type = ODP_POOL_BUFFER;
1912 pool_params->buf.num = num;
1913 pool_params->buf.size = size;
1914 if (align_offset)
1915 pool_params->buf.size += 32 - align_offset;
1916 pool_params->buf.align = odp_align;
1917 pool_params->buf.uarea_size = sizeof(event_hdr_t) + uarea_size;
1918 pool_params->buf.cache_size = cache_size;
1919
1920 /* Buf pool statistics */
1921 if (pool_cfg->stats_opt.in_use)
1922 set_pool_params_stats(&pool_params->stats, &capa->buf.stats,
1923 &pool_cfg->stats_opt.opt);
1924 else
1925 set_pool_params_stats(&pool_params->stats, &capa->buf.stats,
1926 &em_shm->opt.pool.statistics);
1927}
1928
1929static int
1930create_subpools(const em_pool_cfg_t *pool_cfg,
1931 uint32_t align_offset, uint32_t odp_align,
1932 uint32_t uarea_size, uint32_t pkt_headroom,
1933 mpool_elem_t *mpool_elem /*out*/)
1934{
1935 const int num_subpools = pool_cfg->num_subpools;
1936 mpool_tbl_t *const mpool_tbl = &em_shm->mpool_tbl;
1937
1938 for (int i = 0; i < num_subpools; i++) {
1939 char pool_name[ODP_POOL_NAME_LEN];
1940 odp_pool_param_t pool_params;
1941 uint32_t size = pool_cfg->subpool[i].size;
1942 uint32_t num = pool_cfg->subpool[i].num;
1943 uint32_t cache_size = pool_cfg->subpool[i].cache_size;
1944
1945 if (pool_cfg->event_type == EM_EVENT_TYPE_PACKET) {
1946 set_pool_params_pkt(&pool_params /* out */, pool_cfg,
1947 size, num, cache_size,
1948 align_offset, odp_align,
1949 uarea_size, pkt_headroom);
1950 } else if (pool_cfg->event_type == EM_EVENT_TYPE_VECTOR) {
1951 set_pool_params_vector(&pool_params /* out */, pool_cfg,
1952 size, num, cache_size,
1953 uarea_size);
1954 } else { /* pool_cfg->event_type == EM_EVENT_TYPE_SW */
1955 set_pool_params_buf(&pool_params /* out */, pool_cfg,
1956 size, num, cache_size,
1957 align_offset, odp_align, uarea_size);
1958 }
1959
1960 mpool_elem->size[i] = pool_cfg->subpool[i].size;
1961 mpool_elem->stats_opt = pool_params.stats;
1962
1963 snprintf(pool_name, sizeof(pool_name), "%" PRI_POOL ":%d-%s",
1964 mpool_elem->em_pool, i, mpool_elem->name);
1965 pool_name[sizeof(pool_name) - 1] = '\0';
1966
1967 odp_pool_t odp_pool = odp_pool_create(pool_name, &pool_params);
1968
1969 if (unlikely(odp_pool == ODP_POOL_INVALID))
1970 return -1;
1971
1972 mpool_elem->odp_pool[i] = odp_pool;
1973 mpool_elem->num_subpools++; /* created subpools for delete */
1974
1975 int odp_pool_idx = odp_pool_index(odp_pool);
1976
1977 if (unlikely(odp_pool_idx < 0))
1978 return -2;
1979
1980 /* Store mapping from odp-pool (idx) to em-pool & subpool */
1981 mpool_tbl->pool_subpool_odp2em[odp_pool_idx].pool =
1982 (uint32_t)(uintptr_t)mpool_elem->em_pool;
1983 mpool_tbl->pool_subpool_odp2em[odp_pool_idx].subpool = i;
1984
1985 /* odp_pool_print(odp_pool); */
1986 }
1987
1988 return 0;
1989}
1990
1991em_pool_t
1992pool_create(const char *name, em_pool_t req_pool, const em_pool_cfg_t *pool_cfg)
1993{
1994 const em_event_type_t pool_evtype = pool_cfg->event_type;
1995 int err = 0;
1996
1997 /* Allocate a free EM pool */
1998 const em_pool_t pool = pool_alloc(req_pool/* requested or undef*/);
1999
2000 if (unlikely(pool == EM_POOL_UNDEF))
2001 return EM_POOL_UNDEF;
2002
2003 mpool_elem_t *mpool_elem = pool_elem_get(pool);
2004
2005 /* Sanity check */
2006 if (!mpool_elem || mpool_elem->em_pool != pool)
2007 return EM_POOL_UNDEF;
2008
2009 mpool_elem->event_type = pool_evtype;
2010 /* Store successfully created subpools later */
2011 mpool_elem->num_subpools = 0;
2012 /* Store the event pool name, if given */
2013 if (name && *name) {
2014 strncpy(mpool_elem->name, name, sizeof(mpool_elem->name) - 1);
2015 mpool_elem->name[sizeof(mpool_elem->name) - 1] = '\0';
2016 } else {
2017 mpool_elem->name[0] = '\0';
2018 }
2019
2020 em_pool_cfg_t sorted_cfg;
2021
2022 /*
2023 * Sort the subpool cfg in ascending order based on the buffer size
2024 */
2025 sort_pool_cfg(pool_cfg, &sorted_cfg/*out*/);
2026 /* Use sorted_cfg instead of pool_cfg from here on */
2027
2028 /*
2029 * Set the cache-size of each subpool in the EM-pool
2030 */
2031 set_poolcache_size(&sorted_cfg);
2032
2033 /* Store the sorted config */
2034 mpool_elem->pool_cfg = sorted_cfg;
2035
2036 /*
2037 * Event payload alignment requirement for the pool
2038 */
2039 /* align only valid for bufs and pkts */
2040 /* align_offset and odp_align must be checked beforehand in invalid_pool_cfg(...)*/
2041 uint32_t align_offset = align_offset_get(&sorted_cfg);
2042 uint32_t odp_align = odp_align_get(&sorted_cfg);
2043
2044 /* store the align offset, needed in pkt-alloc */
2045 mpool_elem->align_offset = align_offset;
2046
2047 /*
2048 * Event user area size.
2049 * Pool-specific param overrides config file 'user_area_size' value
2050 */
2051 size_t uarea_size = 0;
2052
2053 err = set_uarea_size(&sorted_cfg, &uarea_size/*out*/);
2054 if (unlikely(err)) {
2055 INTERNAL_ERROR(EM_ERR_TOO_LARGE, EM_ESCOPE_POOL_CREATE,
2056 "EM-pool:\"%s\" invalid uarea config: req.size:%zu",
2057 name, uarea_size);
2058 goto error;
2059 }
2060
2061 /* store the user_area sizes, needed in alloc */
2062 mpool_elem->user_area.size = uarea_size & UINT16_MAX;
2063
2064 /*
2065 * Set the headroom for events in EM packet pools
2066 */
2067 uint32_t pkt_headroom = 0;
2068 uint32_t max_headroom = 0;
2069
2070 if (pool_evtype == EM_EVENT_TYPE_PACKET) {
2071 err = set_pkt_headroom(&sorted_cfg, &pkt_headroom/*out*/,
2072 &max_headroom/*out*/);
2073 if (unlikely(err)) {
2074 INTERNAL_ERROR(EM_ERR_TOO_LARGE, EM_ESCOPE_POOL_CREATE,
2075 "EM-pool:\"%s\" invalid pkt headroom:\n"
2076 "headroom:%u vs. max:headroom:%u",
2077 name, pkt_headroom, max_headroom);
2078 goto error;
2079 }
2080 }
2081
2082 /*
2083 * Create the subpools for the EM event-pool.
2084 * Each EM subpool is an ODP pool.
2085 */
2086 err = create_subpools(&sorted_cfg, align_offset, odp_align,
2087 (uint32_t)uarea_size, pkt_headroom, mpool_elem /*out*/);
2088 if (unlikely(err)) {
2089 INTERNAL_ERROR(EM_FATAL(EM_ERR_ALLOC_FAILED),
2090 EM_ESCOPE_POOL_CREATE,
2091 "EM-pool:\"%s\" create fails:%d\n"
2092 "subpools req:%d vs. subpools created:%d",
2093 name, err, sorted_cfg.num_subpools,
2094 mpool_elem->num_subpools);
2095 goto error;
2096 }
2097
2098 /*
2099 * ESV: preallocate all events in the pool
2100 */
2101 if (esv_enabled() && em_shm->opt.esv.prealloc_pools)
2102 pool_prealloc(mpool_elem);
2103
2104 /* Success! */
2105 return mpool_elem->em_pool;
2106
2107error:
2108 (void)pool_delete(pool);
2109 return EM_POOL_UNDEF;
2110}
2111
2112em_status_t
2113pool_delete(em_pool_t pool)
2114{
2115 mpool_tbl_t *const mpool_tbl = &em_shm->mpool_tbl;
2116 mpool_elem_t *const mpool_elem = pool_elem_get(pool);
2117
2118 if (unlikely(mpool_elem == NULL || !pool_allocated(mpool_elem)))
2119 return EM_ERR_BAD_ARG;
2120
2121 for (int i = 0; i < mpool_elem->num_subpools; i++) {
2122 odp_pool_t odp_pool = mpool_elem->odp_pool[i];
2123 int odp_pool_idx;
2124 int ret;
2125
2126 if (odp_pool == ODP_POOL_INVALID)
2127 return EM_ERR_NOT_FOUND;
2128
2129 odp_pool_idx = odp_pool_index(odp_pool);
2130
2131 ret = odp_pool_destroy(odp_pool);
2132 if (unlikely(ret))
2133 return EM_ERR_LIB_FAILED;
2134
2135 mpool_elem->odp_pool[i] = ODP_POOL_INVALID;
2136 mpool_elem->size[i] = 0;
2137
2138 /* Clear mapping from odp-pool (idx) to em-pool & subpool */
2139 if (unlikely(odp_pool_idx < 0))
2140 return EM_ERR_BAD_ID;
2141 mpool_tbl->pool_subpool_odp2em[odp_pool_idx].both = pool_subpool_undef.both;
2142 }
2143
2144 mpool_elem->name[0] = '\0';
2145 mpool_elem->event_type = EM_EVENT_TYPE_UNDEF;
2146 mpool_elem->num_subpools = 0;
2147
2148 return pool_free(pool);
2149}
2150
2151em_pool_t
2152pool_find(const char *name)
2153{
2154 if (name && *name) {
2155 for (int i = 0; i < EM_CONFIG_POOLS; i++) {
2156 const mpool_elem_t *mpool_elem =
2157 &em_shm->mpool_tbl.pool[i];
2158
2159 if (pool_allocated(mpool_elem) &&
2160 !strncmp(name, mpool_elem->name, EM_POOL_NAME_LEN))
2161 return mpool_elem->em_pool;
2162 }
2163 }
2164
2165 return EM_POOL_UNDEF;
2166}
2167
2168unsigned int
2169pool_count(void)
2170{
2171 return odp_atomic_load_u32(&em_shm->pool_count);
2172}
2173
2174void pool_stats_opt_odp2em(em_pool_stats_opt_t *em_stats_opt /*out*/,
2175 const odp_pool_stats_opt_t *odp_stats_opt)
2176{
2177 memset(em_stats_opt, 0, sizeof(*em_stats_opt));
2178
2179 em_stats_opt->available = odp_stats_opt->bit.available;
2180 em_stats_opt->alloc_ops = odp_stats_opt->bit.alloc_ops;
2181 em_stats_opt->alloc_fails = odp_stats_opt->bit.alloc_fails;
2182 em_stats_opt->free_ops = odp_stats_opt->bit.free_ops;
2183 em_stats_opt->total_ops = odp_stats_opt->bit.total_ops;
2184 em_stats_opt->cache_available = odp_stats_opt->bit.cache_available;
2185 em_stats_opt->cache_alloc_ops = odp_stats_opt->bit.cache_alloc_ops;
2186 em_stats_opt->cache_free_ops = odp_stats_opt->bit.cache_free_ops;
2187 em_stats_opt->core_cache_available = odp_stats_opt->bit.thread_cache_available;
2188}
2189
2190#define POOL_INFO_HDR_STR \
2191" id name type offset uarea sizes [size count(used/free) cache]\n"
2192
2193#define POOL_INFO_SUBSTR_FMT \
2194"%d:[sz=%" PRIu32 " n=%" PRIu32 "(%" PRIu32 "/%" PRIu32 ") $=%" PRIu32 "]"
2195
2196#define POOL_INFO_SUBSTR_NO_STATS_FMT \
2197"%d:[sz=%" PRIu32 " n=%" PRIu32 "(-/-) cache=%" PRIu32 "]"
2198
2199void pool_info_print_hdr(unsigned int num_pools)
2200{
2201 if (num_pools == 1) {
2202 EM_PRINT("EM Event Pool\n"
2203 "-------------\n"
2204 POOL_INFO_HDR_STR);
2205 } else {
2206 EM_PRINT("EM Event Pools:%2u\n"
2207 "-----------------\n"
2208 POOL_INFO_HDR_STR, num_pools);
2209 }
2210}
2211
2212void pool_info_print(em_pool_t pool)
2213{
2214 em_pool_info_t pool_info;
2215 em_status_t stat;
2216 const char *pool_type;
2217
2218 stat = em_pool_info(pool, &pool_info/*out*/);
2219 if (unlikely(stat != EM_OK)) {
2220 EM_PRINT(" %-6" PRI_POOL " %-16s n/a n/a n/a n/a [n/a]\n",
2221 pool, "err:n/a");
2222 return;
2223 }
2224
2225 if (pool_info.event_type == EM_EVENT_TYPE_VECTOR)
2226 pool_type = "vec";
2227 else if (pool_info.event_type == EM_EVENT_TYPE_PACKET)
2228 pool_type = "pkt";
2229 else
2230 pool_type = "buf";
2231
2232 EM_PRINT(" %-6" PRI_POOL " %-16s %4s %02u %02zu %02u ",
2233 pool, pool_info.name, pool_type,
2234 pool_info.align_offset, pool_info.user_area_size,
2235 pool_info.num_subpools);
2236
2237 for (int i = 0; i < pool_info.num_subpools; i++) {
2238 char subpool_str[42];
2239
2240 if (pool_info.subpool[i].used || pool_info.subpool[i].free) {
2241 snprintf(subpool_str, sizeof(subpool_str),
2242 POOL_INFO_SUBSTR_FMT, i,
2243 pool_info.subpool[i].size,
2244 pool_info.subpool[i].num,
2245 pool_info.subpool[i].used,
2246 pool_info.subpool[i].free,
2247 pool_info.subpool[i].cache_size);
2248 } else {
2249 snprintf(subpool_str, sizeof(subpool_str),
2250 POOL_INFO_SUBSTR_NO_STATS_FMT, i,
2251 pool_info.subpool[i].size,
2252 pool_info.subpool[i].num,
2253 pool_info.subpool[i].cache_size);
2254 }
2255 subpool_str[sizeof(subpool_str) - 1] = '\0';
2256 EM_PRINT(" %-42s", subpool_str);
2257 }
2258
2259 EM_PRINT("\n");
2260}
2261
2262#define POOL_STATS_OPT_HDR_STR \
2263"Pool Available Alloc_ops Alloc_fails Free_ops Total_ops Cache_available" \
2264" Cache_alloc_ops Cache_free_ops Core_cache_available\n"\
2265
2266void pool_stats_opt_print_hdr(void)
2267{
2268 EM_PRINT("EM pool statistic counter options: 1(enable) 0(disable)\n"
2269 "-------------------------------------------------------\n"
2270 POOL_STATS_OPT_HDR_STR);
2271}
2272
2273void pool_stats_opt_print(em_pool_t pool)
2274{
2275 em_status_t stat;
2276 em_pool_stats_opt_t pool_stats_opt;
2277
2278 stat = em_pool_stats_opt(pool, &pool_stats_opt/*out*/);
2279 if (unlikely(stat != EM_OK)) {
2280 EM_PRINT("Fail to fetch pool statistics counters options for pool %" PRI_POOL "\n",
2281 pool);
2282 return;
2283 }
2284
2285 EM_PRINT("%-8" PRI_POOL "%-10u%-10u%-12u%-9u%-10u%-16u%-16u%-15u%-21u\n",
2286 pool, pool_stats_opt.available,
2287 pool_stats_opt.alloc_ops,
2288 pool_stats_opt.alloc_fails,
2289 pool_stats_opt.free_ops,
2290 pool_stats_opt.total_ops,
2291 pool_stats_opt.cache_available,
2292 pool_stats_opt.cache_alloc_ops,
2293 pool_stats_opt.cache_free_ops,
2294 pool_stats_opt.core_cache_available);
2295}
2296
2297#define POOL_STATS_HDR_STR \
2298"EM pool statistics for pool %" PRI_POOL ":\n\n"\
2299"Subpool Available Alloc_ops Alloc_fails Free_ops Total_ops Cache_available" \
2300" Cache_alloc_ops Cache_free_ops\n"\
2301"--------------------------------------------------------------------------" \
2302"-------------------------------\n%s"
2303
2304#define POOL_STATS_LEN 107
2305#define POOL_STATS_FMT "%-8u%-10lu%-10lu%-12lu%-9lu%-10lu%-16lu%-16lu%-15lu\n"
2306
2307void pool_stats_print(em_pool_t pool)
2308{
2309 em_status_t stat;
2310 em_pool_stats_t pool_stats;
2311 const em_pool_subpool_stats_t *subpool_stats;
2312 int len = 0;
2313 int n_print = 0;
2314 const mpool_elem_t *pool_elem = pool_elem_get(pool);
2315 const int stats_str_len = EM_MAX_SUBPOOLS * POOL_STATS_LEN + 1;
2316 char stats_str[stats_str_len];
2317
2318 if (pool_elem == NULL || !pool_allocated(pool_elem)) {
2319 EM_LOG(EM_LOG_ERR, "EM-pool:%" PRI_POOL " invalid\n", pool);
2320 return;
2321 }
2322
2323 stat = em_pool_stats(pool, &pool_stats);
2324 if (unlikely(stat != EM_OK)) {
2325 EM_PRINT("Failed to fetch EM pool statistics\n");
2326 return;
2327 }
2328
2329 for (uint32_t i = 0; i < pool_stats.num_subpools; i++) {
2330 subpool_stats = &pool_stats.subpool_stats[i];
2331 n_print = snprintf(stats_str + len, stats_str_len - len,
2332 POOL_STATS_FMT,
2333 i, subpool_stats->available,
2334 subpool_stats->alloc_ops,
2335 subpool_stats->alloc_fails,
2336 subpool_stats->free_ops,
2337 subpool_stats->total_ops,
2338 subpool_stats->cache_available,
2339 subpool_stats->cache_alloc_ops,
2340 subpool_stats->cache_free_ops);
2341
2342 /* Not enough space to hold more subpool stats */
2343 if (n_print >= stats_str_len - len)
2344 break;
2345
2346 len += n_print;
2347 }
2348
2349 stats_str[len] = '\0';
2350 EM_PRINT(POOL_STATS_HDR_STR, pool, stats_str);
2351}
2352
2353#define POOL_STATS_SELECTED_HDR_STR \
2354"Selected EM pool statistics for pool %" PRI_POOL ":\n\n"\
2355"Selected statistic counters: %s\n\n"\
2356"Subpool Available Alloc_ops Alloc_fails Free_ops Total_ops Cache_available" \
2357" Cache_alloc_ops Cache_free_ops\n"\
2358"--------------------------------------------------------------------------" \
2359"-------------------------------\n%s"
2360
2361#define OPT_STR_LEN 150
2362
2363static void fill_opt_str(char *opt_str, const em_pool_stats_opt_t *opt)
2364{
2365 int n_print;
2366 int len = 0;
2367
2368 if (opt->available) {
2369 n_print = snprintf(opt_str + len, 12, "%s", "available");
2370 len += n_print;
2371 }
2372
2373 if (opt->alloc_ops) {
2374 n_print = snprintf(opt_str + len, 12, "%s", len ? ", alloc_ops" : "alloc_ops");
2375 len += n_print;
2376 }
2377
2378 if (opt->alloc_fails) {
2379 n_print = snprintf(opt_str + len, 14, "%s", len ? ", alloc_fails" : "alloc_fails");
2380 len += n_print;
2381 }
2382
2383 if (opt->free_ops) {
2384 n_print = snprintf(opt_str + len, 11, "%s", len ? ", free_ops" : "free_ops");
2385 len += n_print;
2386 }
2387
2388 if (opt->total_ops) {
2389 n_print = snprintf(opt_str + len, 12, "%s", len ? ", total_ops" : "total_ops");
2390 len += n_print;
2391 }
2392
2393 if (opt->cache_available) {
2394 n_print = snprintf(opt_str + len, 18, "%s",
2395 len ? ", cache_available" : "cache_available");
2396 len += n_print;
2397 }
2398
2399 if (opt->cache_alloc_ops) {
2400 n_print = snprintf(opt_str + len, 18, "%s",
2401 len ? ", cache_alloc_ops" : "cache_alloc_ops");
2402 len += n_print;
2403 }
2404
2405 if (opt->cache_free_ops)
2406 snprintf(opt_str + len, 17, "%s", len ? ", cache_free_ops" : "cache_free_ops");
2407}
2408
2409void pool_stats_selected_print(em_pool_t pool, const em_pool_stats_opt_t *opt)
2410{
2411 em_status_t stat;
2412 em_pool_stats_selected_t pool_stats = {0};
2413 const em_pool_subpool_stats_selected_t *subpool_stats;
2414 int len = 0;
2415 int n_print = 0;
2416 const mpool_elem_t *pool_elem = pool_elem_get(pool);
2417 char opt_str[OPT_STR_LEN];
2418 const int stats_str_len = EM_MAX_SUBPOOLS * POOL_STATS_LEN + 1;
2419 char stats_str[stats_str_len];
2420
2421 if (pool_elem == NULL || !pool_allocated(pool_elem)) {
2422 EM_LOG(EM_LOG_ERR, "EM-pool:%" PRI_POOL " invalid\n", pool);
2423 return;
2424 }
2425
2426 stat = em_pool_stats_selected(pool, &pool_stats, opt);
2427 if (unlikely(stat != EM_OK)) {
2428 EM_PRINT("Failed to fetch EM selected pool statistics\n");
2429 return;
2430 }
2431
2432 for (uint32_t i = 0; i < pool_stats.num_subpools; i++) {
2433 subpool_stats = &pool_stats.subpool_stats[i];
2434
2435 n_print = snprintf(stats_str + len, stats_str_len - len,
2436 POOL_STATS_FMT,
2437 i,
2438 subpool_stats->available,
2439 subpool_stats->alloc_ops,
2440 subpool_stats->alloc_fails,
2441 subpool_stats->free_ops,
2442 subpool_stats->total_ops,
2443 subpool_stats->cache_available,
2444 subpool_stats->cache_alloc_ops,
2445 subpool_stats->cache_free_ops);
2446
2447 /* Not enough space to hold more subpool stats */
2448 if (n_print >= stats_str_len - len)
2449 break;
2450
2451 len += n_print;
2452 }
2453 stats_str[len] = '\0';
2454
2455 /* Fill selected statistic counters */
2456 fill_opt_str(opt_str, opt);
2457
2458 EM_PRINT(POOL_STATS_SELECTED_HDR_STR, pool, opt_str, stats_str);
2459}
2460
2461#define SUBPOOL_STATS_HDR_STR \
2462"EM subpool statistics for pool %" PRI_POOL ":\n\n"\
2463"Subpool Available Alloc_ops Alloc_fails Free_ops Total_ops Cache_available" \
2464" Cache_alloc_ops Cache_free_ops\n"\
2465"--------------------------------------------------------------------------" \
2466"-------------------------------\n%s"
2467
2468void subpools_stats_print(em_pool_t pool, const int subpools[], int num_subpools)
2469{
2470 int num_stats;
2471 em_pool_subpool_stats_t stats[num_subpools];
2472 int len = 0;
2473 int n_print = 0;
2474 const mpool_elem_t *pool_elem = pool_elem_get(pool);
2475 const int stats_str_len = num_subpools * POOL_STATS_LEN + 1;
2476 char stats_str[stats_str_len];
2477
2478 if (pool_elem == NULL || !pool_allocated(pool_elem)) {
2479 EM_LOG(EM_LOG_ERR, "EM-pool:%" PRI_POOL " invalid\n", pool);
2480 return;
2481 }
2482
2483 num_stats = em_pool_subpool_stats(pool, subpools, num_subpools, stats);
2484 if (unlikely(!num_stats || num_stats > num_subpools)) {
2485 EM_LOG(EM_LOG_ERR, "Failed to fetch subpool statistics\n");
2486 return;
2487 }
2488
2489 /* Print subpool stats */
2490 for (int i = 0; i < num_stats; i++) {
2491 n_print = snprintf(stats_str + len, stats_str_len - len,
2492 POOL_STATS_FMT,
2493 subpools[i], stats[i].available, stats[i].alloc_ops,
2494 stats[i].alloc_fails, stats[i].free_ops,
2495 stats[i].total_ops, stats[i].cache_available,
2496 stats[i].cache_alloc_ops, stats[i].cache_free_ops);
2497
2498 /* Not enough space to hold more subpool stats */
2499 if (n_print >= stats_str_len - len)
2500 break;
2501
2502 len += n_print;
2503 }
2504
2505 stats_str[len] = '\0';
2506 EM_PRINT(SUBPOOL_STATS_HDR_STR, pool, stats_str);
2507}
2508
2509#define SUBPOOL_STATS_SELECTED_HDR_STR \
2510"Selected EM subpool statistics for pool %" PRI_POOL ":\n\n"\
2511"Selected statistic counters: %s\n\n"\
2512"Subpool Available Alloc_ops Alloc_fails Free_ops Total_ops Cache_available" \
2513" Cache_alloc_ops Cache_free_ops\n"\
2514"--------------------------------------------------------------------------" \
2515"-------------------------------\n%s"
2516
2517void subpools_stats_selected_print(em_pool_t pool, const int subpools[],
2518 int num_subpools, const em_pool_stats_opt_t *opt)
2519{
2520 int num_stats;
2521 char opt_str[OPT_STR_LEN];
2522 em_pool_subpool_stats_selected_t stats[num_subpools];
2523 int len = 0;
2524 int n_print = 0;
2525 const mpool_elem_t *pool_elem = pool_elem_get(pool);
2526 const int stats_str_len = num_subpools * POOL_STATS_LEN + 1;
2527 char stats_str[stats_str_len];
2528
2529 if (pool_elem == NULL || !pool_allocated(pool_elem)) {
2530 EM_LOG(EM_LOG_ERR, "EM-pool:%" PRI_POOL " invalid\n", pool);
2531 return;
2532 }
2533
2534 memset(stats, 0, sizeof(stats));
2535 num_stats = em_pool_subpool_stats_selected(pool, subpools, num_subpools, stats, opt);
2536 if (unlikely(!num_stats || num_stats > num_subpools)) {
2537 EM_LOG(EM_LOG_ERR, "Failed to fetch selected subpool statistics\n");
2538 return;
2539 }
2540
2541 /* Print subpool stats */
2542 for (int i = 0; i < num_stats; i++) {
2543 n_print = snprintf(stats_str + len, stats_str_len - len,
2544 POOL_STATS_FMT,
2545 subpools[i], stats[i].available, stats[i].alloc_ops,
2546 stats[i].alloc_fails, stats[i].free_ops,
2547 stats[i].total_ops, stats[i].cache_available,
2548 stats[i].cache_alloc_ops, stats[i].cache_free_ops);
2549
2550 /* Not enough space to hold more subpool stats */
2551 if (n_print >= stats_str_len - len)
2552 break;
2553
2554 len += n_print;
2555 }
2556 stats_str[len] = '\0';
2557
2558 /* Fill selected statistic counters */
2559 fill_opt_str(opt_str, opt);
2560 EM_PRINT(SUBPOOL_STATS_SELECTED_HDR_STR, pool, opt_str, stats_str);
2561}
2562
2563void print_pool_elem_info(void)
2564{
2565 EM_PRINT("\n"
2566 "pool-elem size: %zu B\n",
2567 sizeof(mpool_elem_t));
2568
2569 DBG_PRINT("\t\toffset\tsize\n"
2570 "\t\t------\t-----\n"
2571 "event_type:\t%3zu B\t%3zu B\n"
2572 "align_offset:\t%3zu B\t%3zu B\n"
2573 "user_area info:\t%3zu B\t%3zu B\n"
2574 "num_subpools:\t%3zu B\t%3zu B\n"
2575 "size[]:\t\t%3zu B\t%3zu B\n"
2576 "odp_pool[]:\t%3zu B\t%3zu B\n"
2577 "em_pool:\t%3zu B\t%3zu B\n"
2578 "objpool_elem:\t%3zu B\t%3zu B\n"
2579 "stats_opt:\t%3zu B\t%3zu B\n"
2580 "pool_cfg:\t%3zu B\t%3zu B\n"
2581 "name[]:\t\t%3zu B\t%3zu B\n",
2582 offsetof(mpool_elem_t, event_type), sizeof_field(mpool_elem_t, event_type),
2583 offsetof(mpool_elem_t, align_offset), sizeof_field(mpool_elem_t, align_offset),
2584 offsetof(mpool_elem_t, user_area), sizeof_field(mpool_elem_t, user_area),
2585 offsetof(mpool_elem_t, num_subpools), sizeof_field(mpool_elem_t, num_subpools),
2586 offsetof(mpool_elem_t, size), sizeof_field(mpool_elem_t, size),
2587 offsetof(mpool_elem_t, odp_pool), sizeof_field(mpool_elem_t, odp_pool),
2588 /* NOLINTNEXTLINE(bugprone-sizeof-expression) */
2589 offsetof(mpool_elem_t, em_pool), sizeof_field(mpool_elem_t, em_pool),
2590 offsetof(mpool_elem_t, objpool_elem), sizeof_field(mpool_elem_t, objpool_elem),
2591 offsetof(mpool_elem_t, stats_opt), sizeof_field(mpool_elem_t, stats_opt),
2592 offsetof(mpool_elem_t, pool_cfg), sizeof_field(mpool_elem_t, pool_cfg),
2593 offsetof(mpool_elem_t, name), sizeof_field(mpool_elem_t, name));
2594
2595 EM_PRINT("\n");
2596}
em_chaining.h
em_error.h
INTERNAL_ERROR
#define INTERNAL_ERROR(error, escope, fmt,...)
Definition em_error.h:58
em_event.h
em_event_inline.h
em_event_state.h
em_event_types.h
event_hdr_t
struct event_hdr event_hdr_t
EM_CHECK_INIT_CALLED
#define EM_CHECK_INIT_CALLED
Definition em_init_types.h:56
em_mem.h
em_shm
em_shm_t * em_shm
Definition event_machine_init.c:131
em_pool.h
em_pool_inline.h
em_pool_types.h
POOL_ODP2EM_TBL_LEN
#define POOL_ODP2EM_TBL_LEN
Definition em_pool_types.h:95
em_queue.h
em_queue_inline.h
em_queue_types.h
event_machine.h
EM_CONFIG_POOLS
#define EM_CONFIG_POOLS
Definition event_machine_config.h:256
EM_EVENT_USER_AREA_MAX_SIZE
#define EM_EVENT_USER_AREA_MAX_SIZE
Definition event_machine_config.h:295
EM_POOL_DEFAULT_NAME
#define EM_POOL_DEFAULT_NAME
Definition event_machine_config.h:282
EM_POOL_DEFAULT
#define EM_POOL_DEFAULT
Definition event_machine_config.h:262
EM_MAX_SUBPOOLS
#define EM_MAX_SUBPOOLS
The maximum number of subpools in each EM pool. The subpool is a pool with buffers of only one size.
Definition event_machine_config.h:276
EM_POOL_NAME_LEN
#define EM_POOL_NAME_LEN
Definition event_machine_config.h:268
em_event_type_t
uint32_t em_event_type_t
Definition event_machine_types.h:221
PRI_POOL
#define PRI_POOL
Definition event_machine_types.h:126
EM_POOL_UNDEF
#define EM_POOL_UNDEF
Definition event_machine_types.h:124
em_core_id
int em_core_id(void)
Definition event_machine_core.c:56
EM_OK
#define EM_OK
Definition event_machine_error.h:104
EM_FATAL
#define EM_FATAL(error)
Definition event_machine_error.h:83
em_status_t
uint32_t em_status_t
Definition event_machine_error.h:96
EM_ERR_NOT_FOUND
@ EM_ERR_NOT_FOUND
Definition event_machine_error.h:136
EM_ERR_OPERATION_FAILED
@ EM_ERR_OPERATION_FAILED
Definition event_machine_error.h:147
EM_ERR_TOO_LARGE
@ EM_ERR_TOO_LARGE
Definition event_machine_error.h:152
EM_ERR_BAD_ID
@ EM_ERR_BAD_ID
Definition event_machine_error.h:123
EM_ERR_TOO_SMALL
@ EM_ERR_TOO_SMALL
Definition event_machine_error.h:154
EM_ERR_ALLOC_FAILED
@ EM_ERR_ALLOC_FAILED
Definition event_machine_error.h:145
EM_ERR_BAD_ARG
@ EM_ERR_BAD_ARG
Definition event_machine_error.h:119
EM_ERR_LIB_FAILED
@ EM_ERR_LIB_FAILED
Definition event_machine_error.h:149
EM_ERR
@ EM_ERR
Definition event_machine_error.h:172
EM_ERR_BAD_POINTER
@ EM_ERR_BAD_POINTER
Definition event_machine_error.h:129
em_free
void em_free(em_event_t event)
Definition event_machine_event.c:336
EM_EVENT_TYPE_SW
@ EM_EVENT_TYPE_SW
Definition event_machine_event.h:196
EM_EVENT_TYPE_UNDEF
@ EM_EVENT_TYPE_UNDEF
Definition event_machine_event.h:193
EM_EVENT_TYPE_PACKET
@ EM_EVENT_TYPE_PACKET
Definition event_machine_event.h:199
EM_EVENT_TYPE_VECTOR
@ EM_EVENT_TYPE_VECTOR
Definition event_machine_event.h:208
em_pool_create
em_pool_t em_pool_create(const char *name, em_pool_t pool, const em_pool_cfg_t *pool_cfg)
Definition event_machine_pool.c:98
em_pool_cfg_init
void em_pool_cfg_init(em_pool_cfg_t *const pool_cfg)
Definition event_machine_pool.c:64
em_pool_subpool_stats_selected
int em_pool_subpool_stats_selected(em_pool_t pool, const int subpools[], int num_subpools, em_pool_subpool_stats_selected_t subpool_stats[], const em_pool_stats_opt_t *opt)
Retrieve selected statistics about subpool(s) of an EM pool.
Definition event_machine_pool.c:619
em_pool_stats_selected
em_status_t em_pool_stats_selected(em_pool_t pool, em_pool_stats_selected_t *pool_stats, const em_pool_stats_opt_t *opt)
Retrieve selected statistics about an EM pool.
Definition event_machine_pool.c:566
em_pool_info
em_status_t em_pool_info(em_pool_t pool, em_pool_info_t *pool_info)
Definition event_machine_pool.c:235
em_pool_stats_opt
em_status_t em_pool_stats_opt(em_pool_t pool, em_pool_stats_opt_t *pool_stats_opt)
Definition event_machine_pool.c:352
em_pool_subpool_stats
int em_pool_subpool_stats(em_pool_t pool, const int subpools[], int num_subpools, em_pool_subpool_stats_t subpool_stats[])
Retrieve statistics about subpool(s) of an EM pool.
Definition event_machine_pool.c:464
em_pool_stats
em_status_t em_pool_stats(em_pool_t pool, em_pool_stats_t *pool_stats)
Retrieve statistics about an EM pool.
Definition event_machine_pool.c:392
em_pool_info_print_all
void em_pool_info_print_all(void)
Definition event_machine_pool.c:337
OBJSUBPOOLS_MAX
#define OBJSUBPOOLS_MAX
Definition objpool.h:62
em_pool_cfg_t
Definition event_machine_pool.h:141
em_pool_cfg_t::num_subpools
int num_subpools
Definition event_machine_pool.h:264
em_pool_cfg_t::num
uint32_t num
Definition event_machine_pool.h:290
em_pool_cfg_t::value
uint32_t value
Definition event_machine_pool.h:194
em_pool_cfg_t::subpool
struct em_pool_cfg_t::@23 subpool[EM_MAX_SUBPOOLS]
em_pool_cfg_t::__internal_check
uint32_t __internal_check
Definition event_machine_pool.h:331
em_pool_cfg_t::user_area
struct em_pool_cfg_t::@21 user_area
em_pool_cfg_t::cache_size
uint32_t cache_size
Definition event_machine_pool.h:303
em_pool_cfg_t::headroom
struct em_pool_cfg_t::@22::@25 headroom
em_pool_cfg_t::event_type
em_event_type_t event_type
Definition event_machine_pool.h:156
em_pool_cfg_t::size
size_t size
Definition event_machine_pool.h:224
em_pool_cfg_t::in_use
bool in_use
Definition event_machine_pool.h:185
em_pool_cfg_t::pkt
struct em_pool_cfg_t::@22 pkt
em_pool_cfg_t::align_offset
struct em_pool_cfg_t::@20 align_offset
em_pool_cfg_t::stats_opt
struct em_pool_cfg_t::@24 stats_opt
em_pool_info_t
Definition event_machine_pool.h:337
em_pool_info_t::free
uint32_t free
Definition event_machine_pool.h:374
em_pool_info_t::event_type
em_event_type_t event_type
Definition event_machine_pool.h:343
em_pool_info_t::size
uint32_t size
Definition event_machine_pool.h:354
em_pool_info_t::align_offset
uint32_t align_offset
Definition event_machine_pool.h:345
em_pool_info_t::user_area_size
size_t user_area_size
Definition event_machine_pool.h:347
em_pool_info_t::num_subpools
int num_subpools
Definition event_machine_pool.h:351
em_pool_info_t::cache_size
uint32_t cache_size
Definition event_machine_pool.h:358
em_pool_info_t::num
uint32_t num
Definition event_machine_pool.h:356
em_pool_info_t::used
uint32_t used
Definition event_machine_pool.h:366
em_pool_stats_selected_t
Definition event_machine_pool.h:455
em_pool_stats_t
Definition event_machine_pool.h:413
em_pool_subpool_stats_selected_t
Definition event_machine_pool.h:423
em_pool_subpool_stats_selected_t::available
uint64_t available
Definition event_machine_pool.h:425
em_pool_subpool_stats_selected_t::free_ops
uint64_t free_ops
Definition event_machine_pool.h:436
em_pool_subpool_stats_selected_t::alloc_ops
uint64_t alloc_ops
Definition event_machine_pool.h:430
em_pool_subpool_stats_selected_t::total_ops
uint64_t total_ops
Definition event_machine_pool.h:441
em_pool_subpool_stats_selected_t::cache_alloc_ops
uint64_t cache_alloc_ops
Definition event_machine_pool.h:449
em_pool_subpool_stats_selected_t::alloc_fails
uint64_t alloc_fails
Definition event_machine_pool.h:433
em_pool_subpool_stats_selected_t::cache_available
uint64_t cache_available
Definition event_machine_pool.h:444
em_pool_subpool_stats_selected_t::cache_free_ops
uint64_t cache_free_ops
Definition event_machine_pool.h:452
em_pool_subpool_stats_t
Definition event_machine_pool.h:378
em_pool_subpool_stats_t::cache_free_ops
uint64_t cache_free_ops
Definition event_machine_pool.h:407
em_pool_subpool_stats_t::available
uint64_t available
Definition event_machine_pool.h:380
em_pool_subpool_stats_t::cache_available
uint64_t cache_available
Definition event_machine_pool.h:399
em_pool_subpool_stats_t::total_ops
uint64_t total_ops
Definition event_machine_pool.h:396
em_pool_subpool_stats_t::free_ops
uint64_t free_ops
Definition event_machine_pool.h:391
em_pool_subpool_stats_t::alloc_ops
uint64_t alloc_ops
Definition event_machine_pool.h:385
em_pool_subpool_stats_t::alloc_fails
uint64_t alloc_fails
Definition event_machine_pool.h:388
em_pool_subpool_stats_t::cache_alloc_ops
uint64_t cache_alloc_ops
Definition event_machine_pool.h:404
em_shm_t::pool_count
odp_atomic_u32_t pool_count
Definition em_mem.h:183
em_shm_t::opt
em_cfgfile_opts_t opt
Definition em_mem.h:99
em_startup_pool_conf_t
Definition event_machine_init.h:595
event_hdr
Definition em_event_types.h:194
event_hdr::event
em_event_t event
Definition em_event_types.h:268
list_node_t
Definition list.h:44
mpool_elem_t
Definition em_pool_types.h:56
mpool_elem_t::user_area
struct mpool_elem_t::@54 user_area
mpool_elem_t::pool_cfg
em_pool_cfg_t pool_cfg
Definition em_pool_types.h:86
mpool_elem_t::stats_opt
odp_pool_stats_opt_t stats_opt
Definition em_pool_types.h:84
mpool_elem_t::align_offset
uint32_t align_offset
Definition em_pool_types.h:60
mpool_elem_t::objpool_elem
objpool_elem_t objpool_elem
Definition em_pool_types.h:82
mpool_elem_t::em_pool
em_pool_t em_pool
Definition em_pool_types.h:80
mpool_elem_t::odp_pool
odp_pool_t odp_pool[EM_MAX_SUBPOOLS]
Definition em_pool_types.h:78
mpool_elem_t::size
uint16_t size
Definition em_pool_types.h:64
mpool_elem_t::event_type
em_event_type_t event_type
Definition em_pool_types.h:58
mpool_elem_t::num_subpools
int num_subpools
Definition em_pool_types.h:74
mpool_pool_t
Definition em_pool_types.h:145
mpool_tbl_t
Definition em_pool_types.h:128
mpool_tbl_t::pool_subpool_odp2em
pool_subpool_t pool_subpool_odp2em[POOL_ODP2EM_TBL_LEN]
Definition em_pool_types.h:136
objpool_elem_t
Definition objpool.h:70
objpool_elem_t::subpool_idx
uint32_t subpool_idx
Definition objpool.h:74
em_pool_stats_opt_t
Definition event_machine_pool.h:98
em_pool_stats_opt_t::cache_alloc_ops
uint64_t cache_alloc_ops
Definition event_machine_pool.h:120
em_pool_stats_opt_t::alloc_ops
uint64_t alloc_ops
Definition event_machine_pool.h:105
em_pool_stats_opt_t::free_ops
uint64_t free_ops
Definition event_machine_pool.h:111
em_pool_stats_opt_t::cache_available
uint64_t cache_available
Definition event_machine_pool.h:117
em_pool_stats_opt_t::core_cache_available
uint64_t core_cache_available
Definition event_machine_pool.h:130
em_pool_stats_opt_t::cache_free_ops
uint64_t cache_free_ops
Definition event_machine_pool.h:123
em_pool_stats_opt_t::available
uint64_t available
Definition event_machine_pool.h:102
em_pool_stats_opt_t::alloc_fails
uint64_t alloc_fails
Definition event_machine_pool.h:108
em_pool_stats_opt_t::total_ops
uint64_t total_ops
Definition event_machine_pool.h:114
event_prealloc_hdr
Definition em_event_types.h:333
event_prealloc_hdr::list_node
list_node_t list_node
Definition em_event_types.h:343
pool_subpool_t
Definition em_pool_types.h:109