EM-ODP  3.7.0
Event Machine on ODP
em_pool.c
1 /*
2  * Copyright (c) 2015-2023, Nokia Solutions and Networks
3  * All rights reserved.
4  *
5  * Redistribution and use in source and binary forms, with or without
6  * modification, are permitted provided that the following conditions
7  * are met:
8  *
9  * * Redistributions of source code must retain the above copyright
10  * notice, this list of conditions and the following disclaimer.
11  * * Redistributions in binary form must reproduce the above copyright
12  * notice, this list of conditions and the following disclaimer in the
13  * documentation and/or other materials provided with the distribution.
14  * * Neither the name of the copyright holder nor the names of its
15  * contributors may be used to endorse or promote products derived
16  * from this software without specific prior written permission.
17  *
18  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22  * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29  */
30 
31 #include "em_include.h"
32 
33 #ifndef __clang__
34 COMPILE_TIME_ASSERT(EM_POOL_DEFAULT > (em_pool_t)0 &&
35  EM_POOL_DEFAULT < (em_pool_t)EM_CONFIG_POOLS,
36  EM_ODP_EM_DEFAULT_POOL_ERROR);
37 COMPILE_TIME_ASSERT(EM_POOL_UNDEF != EM_POOL_DEFAULT,
38  EM_ODP_EM_POOL_UNDEF_ERROR);
39 #endif
40 COMPILE_TIME_ASSERT(EM_EVENT_USER_AREA_MAX_SIZE < UINT16_MAX,
41  EM_ODP_EM_EVENT_USER_AREA_MAX_SIZE_ERROR);
42 
43 /**
44  * @def ALIGN_OFFSET_MAX
45  *
46  * Max supported value for the config file option 'pool.align_offset'.
47  */
48 #define ALIGN_OFFSET_MAX ((int)(16))
49 
50 /* ALIGN_OFFSET_MAX <= 2^bits - 1, must fit into event_hdr_t::align_offset */
51 COMPILE_TIME_ASSERT(ALIGN_OFFSET_MAX <=
52  ((1 << (sizeof_field(event_hdr_t, align_offset) * 8)) - 1),
53  ALIGN_OFFSET_MAX__TOO_LARGE);
54 
55 /**
56  * @brief Undef value for a pool_subpool_t
57  * pool_subpool_undef = {.pool = EM_POOL_UNDEF, .subpool = 0};
58  */
59 const pool_subpool_t pool_subpool_undef = {.pool = (uint32_t)(uintptr_t)EM_POOL_UNDEF,
60  .subpool = 0};
61 
62 static inline mpool_elem_t *
63 mpool_poolelem2pool(objpool_elem_t *const objpool_elem)
64 {
65  return (mpool_elem_t *)((uintptr_t)objpool_elem -
66  offsetof(mpool_elem_t, objpool_elem));
67 }
68 
69 static em_pool_t
70 pool_alloc(em_pool_t pool)
71 {
72  mpool_elem_t *mpool_elem;
73 
74  if (pool == EM_POOL_UNDEF) {
75  objpool_elem_t *objpool_elem =
76  objpool_rem(&em_shm->mpool_pool.objpool, em_core_id());
77 
78  if (unlikely(objpool_elem == NULL))
79  return EM_POOL_UNDEF;
80 
81  mpool_elem = mpool_poolelem2pool(objpool_elem);
82  } else {
83  int ret;
84 
85  mpool_elem = pool_elem_get(pool);
86  if (unlikely(mpool_elem == NULL))
87  return EM_POOL_UNDEF;
88 
89  ret = objpool_rem_elem(&em_shm->mpool_pool.objpool,
90  &mpool_elem->objpool_elem);
91  if (unlikely(ret != 0))
92  return EM_POOL_UNDEF;
93  }
94 
95  env_atomic32_inc(&em_shm->pool_count);
96  return mpool_elem->em_pool;
97 }
98 
99 static em_status_t
100 pool_free(em_pool_t pool)
101 {
102  mpool_elem_t *mpool_elem = pool_elem_get(pool);
103 
104  if (unlikely(mpool_elem == NULL))
105  return EM_ERR_BAD_ID;
106 
107  objpool_add(&em_shm->mpool_pool.objpool,
108  mpool_elem->objpool_elem.subpool_idx,
109  &mpool_elem->objpool_elem);
110 
111  env_atomic32_dec(&em_shm->pool_count);
112  return EM_OK;
113 }
114 
115 static int event_type_from_string(const char *str, em_event_type_t *event_type /*out*/)
116 {
117  if (strstr(str, "EM_EVENT_TYPE_SW")) {
118  *event_type = EM_EVENT_TYPE_SW;
119  } else if (strstr(str, "EM_EVENT_TYPE_PACKET")) {
120  *event_type = EM_EVENT_TYPE_PACKET;
121  } else if (strstr(str, "EM_EVENT_TYPE_VECTOR")) {
122  *event_type = EM_EVENT_TYPE_VECTOR;
123  } else {
124  EM_LOG(EM_LOG_ERR, "Event type %s not supported.\n", str);
125  return -1;
126  }
127 
128  return 0;
129 }
130 
131 /* Read option: startup_pools.conf[i].pool_cfg.subpools[j] from the EM config file */
132 static inline int read_config_subpool(const libconfig_list_t *subpool, int index,
133  const char *pool_cfg_str, em_pool_cfg_t *cfg/*out*/)
134 {
135  int ret;
136  /* Option: subpools[index].size */
137  ret = em_libconfig_list_lookup_int(subpool, index, "size",
138  (int *)&cfg->subpool[index].size);
139  if (unlikely(ret != 1)) {
140  EM_LOG(EM_LOG_ERR,
141  "Option '%s.subpools[%d].size' not found or wrong type.\n",
142  pool_cfg_str, index);
143  return -1;
144  }
145 
146  if (cfg->subpool[index].size <= 0) {
147  EM_LOG(EM_LOG_ERR, "Invalid '%s.subpools[%d].size'.\n",
148  pool_cfg_str, index);
149  return -1;
150  }
151 
152  /* Option: subpools[index].num */
153  ret = em_libconfig_list_lookup_int(subpool, index, "num",
154  (int *)&cfg->subpool[index].num);
155  if (unlikely(ret != 1)) {
156  EM_LOG(EM_LOG_ERR,
157  "Option '%s.subpools[%d].num' not found or wrong type.\n",
158  pool_cfg_str, index);
159  return -1;
160  }
161 
162  if (cfg->subpool[index].num <= 0) {
163  EM_LOG(EM_LOG_ERR, "Invalid '%s.subpools[%d].num'.\n",
164  pool_cfg_str, index);
165  return -1;
166  }
167 
168  /*
169  * Option: subpools[index].cache_size
170  * Not mandatory
171  */
172  ret = em_libconfig_list_lookup_int(subpool, index, "cache_size",
173  (int *)&cfg->subpool[index].cache_size);
174 
175  /* If cache_size is given, check if it is valid */
176  if (ret == 1) {
177  uint32_t min_cache_size;
178  const odp_pool_capability_t *capa;
179 
180  capa = &em_shm->mpool_tbl.odp_pool_capability;
181 
182  min_cache_size = (cfg->event_type == EM_EVENT_TYPE_SW) ?
183  capa->buf.min_cache_size : capa->pkt.min_cache_size;
184 
185  if (unlikely(cfg->subpool[index].cache_size < min_cache_size)) {
186  EM_LOG(EM_LOG_ERR,
187  "'%s.subpools[%d].cache_size' too small.\n",
188  pool_cfg_str, index);
189  return -1;
190  }
191  } else if (ret == 0) {/*cache_size is given but with wrong data type*/
192  EM_LOG(EM_LOG_ERR,
193  "'%s.subpools[%d].cache_size' wrong data type.\n",
194  pool_cfg_str, index);
195  return -1;
196  }
197 
198  /* No need to return fail -1 when cache_size not given (ret == -1) */
199  return 0;
200 }
201 
202 static int is_pool_type_supported(em_event_type_t type,
203  const char **err_str/*out*/)
204 {
205  const odp_pool_capability_t *capa = &em_shm->mpool_tbl.odp_pool_capability;
206 
207  if (type == EM_EVENT_TYPE_SW) {
208  if (capa->buf.max_pools == 0) {
209  *err_str = "SW (buf) pool type unsupported";
210  return -1;
211  }
212  } else if (type == EM_EVENT_TYPE_PACKET) {
213  if (capa->pkt.max_pools == 0) {
214  *err_str = "PACKET pool type unsupported";
215  return -1;
216  }
217  } else if (type == EM_EVENT_TYPE_VECTOR) {
218  if (capa->vector.max_pools == 0) {
219  *err_str = "VECTOR pool type unsupported";
220  return -1;
221  }
222  } else {
223  *err_str = "Pool type unsupported, use _SW, _PACKET or _VECTOR";
224  return -1;
225  }
226 
227  return 0;
228 }
229 
230 static inline bool is_align_offset_valid(const em_pool_cfg_t *pool_cfg)
231 {
232  if (pool_cfg->align_offset.in_use &&
233  (pool_cfg->align_offset.value > ALIGN_OFFSET_MAX ||
234  !POWEROF2(pool_cfg->align_offset.value))) {
235  return false;
236  }
237 
238  return true;
239 }
240 
241 static inline int is_user_area_valid(const em_pool_cfg_t *pool_cfg,
242  const odp_pool_capability_t *capa,
243  const char **err_str/*out*/)
244 {
245  /* No need to check when pool specific value is not used */
246  if (!pool_cfg->user_area.in_use)
247  return 0;
248 
249  if (pool_cfg->user_area.size > EM_EVENT_USER_AREA_MAX_SIZE) {
250  *err_str = "Event user area too large";
251  return -1;
252  }
253 
254  if (pool_cfg->event_type == EM_EVENT_TYPE_PACKET) {
255  size_t req_odp_uarea_sz = pool_cfg->user_area.size +
256  sizeof(event_hdr_t);
257  if (req_odp_uarea_sz > capa->pkt.max_uarea_size) {
258  *err_str = "ODP pkt max uarea not large enough";
259  return -1;
260  }
261  }
262  if (pool_cfg->event_type == EM_EVENT_TYPE_VECTOR) {
263  size_t req_odp_uarea_sz = pool_cfg->user_area.size +
264  sizeof(event_hdr_t);
265  if (req_odp_uarea_sz > capa->vector.max_uarea_size) {
266  *err_str = "ODP pkt-vector max uarea not large enough";
267  return -1;
268  }
269  }
270 
271  return 0;
272 }
273 
274 /* Read option: startup_pools.conf[index].pool_cfg.align_offset from the EM config file */
275 static inline int read_config_align_offset(const libconfig_group_t *align_offset,
276  const char *pool_cfg_str,
277  em_pool_cfg_t *cfg/*out*/)
278 {
279  int ret;
280 
281  /* Option: startup_pools.conf[index].pool_cfg.align_offset.in_use */
282  ret = em_libconfig_group_lookup_bool(align_offset, "in_use",
283  &cfg->align_offset.in_use);
284  if (unlikely(!ret)) {
285  EM_LOG(EM_LOG_ERR,
286  "'%s.align_offset.in_use' not found or wrong type\n",
287  pool_cfg_str);
288  return -1;
289  }
290 
291  /* Option: startup_pools.conf[index].pool_cfg.align_offset.value */
292  ret = em_libconfig_group_lookup_int(align_offset, "value",
293  (int *)&cfg->align_offset.value);
294  if (unlikely(!ret)) {
295  EM_LOG(EM_LOG_ERR,
296  "'%s.align_offset.value' not found or wrong type\n",
297  pool_cfg_str);
298  return -1;
299  }
300 
301  /* Check whether the given value is valid or not */
302  if (!is_align_offset_valid(cfg)) {
303  EM_LOG(EM_LOG_ERR, "Invalid '%s.align_offset.value': %d\n"
304  "Max align_offset is %d and it must be power of 2\n",
305  pool_cfg_str, cfg->align_offset.value, ALIGN_OFFSET_MAX);
306  return -1;
307  }
308 
309  return 0;
310 }
311 
312 /* Read option: startup_pools.conf[index].pool_cfg.user_area from the EM config file */
313 static inline int read_config_user_area(const libconfig_group_t *user_area,
314  const char *pool_cfg_str,
315  em_pool_cfg_t *cfg/*out*/)
316 {
317  int ret;
318  const odp_pool_capability_t *capa;
319  const char *err_str = "";
320 
321  /* Option: startup_pools.conf[index].pool_cfg.user_area.in_use */
322  ret = em_libconfig_group_lookup_bool(user_area, "in_use",
323  &cfg->user_area.in_use);
324  if (unlikely(!ret)) {
325  EM_LOG(EM_LOG_ERR,
326  "'%s.user_area.in_use' not found or wrong type\n",
327  pool_cfg_str);
328  return -1;
329  }
330 
331  /* Option: startup_pools.conf[index].pool_cfg.user_area.size */
332  ret = em_libconfig_group_lookup_int(user_area, "size",
333  (int *)&cfg->user_area.size);
334  if (unlikely(!ret)) {
335  EM_LOG(EM_LOG_ERR,
336  "'%s.user_area.size' not found or wrong type\n",
337  pool_cfg_str);
338  return -1;
339  }
340 
341  capa = &em_shm->mpool_tbl.odp_pool_capability;
342  /* Check whether the given value is valid or not */
343  if (is_user_area_valid(cfg, capa, &err_str) < 0) {
344  EM_LOG(EM_LOG_ERR, "%s: %ld\n", err_str, cfg->user_area.size);
345  return -1;
346  }
347 
348  return 0;
349 }
350 
351 /* Read option: startup_pools.conf[index].pool_cfg.pkt.headroom from the EM config file */
352 static inline int read_config_pkt_headroom(const libconfig_group_t *pkt_headroom,
353  const char *pool_cfg_str,
354  em_pool_cfg_t *cfg/*out*/)
355 {
356  int ret;
357  const odp_pool_capability_t *capa;
358 
359  /*Option: startup_pools.conf[index].pool_cfg.pkt.headroom.in_use*/
360  ret = em_libconfig_group_lookup_bool(pkt_headroom, "in_use",
361  &cfg->pkt.headroom.in_use);
362  if (unlikely(!ret)) {
363  EM_LOG(EM_LOG_ERR,
364  "'%s.pkt.headroom.in_use' not found or wrong type\n",
365  pool_cfg_str);
366  return -1;
367  }
368 
369  /*Option: startup_pools.conf[index].pool_cfg.pkt.headroom.value*/
370  ret = em_libconfig_group_lookup_int(pkt_headroom, "value",
371  (int *)&cfg->pkt.headroom.value);
372  if (unlikely(!ret)) {
373  EM_LOG(EM_LOG_ERR,
374  "'%s.pkt.headroom.value' not found or wrong type\n",
375  pool_cfg_str);
376  return -1;
377  }
378 
379  /* Check whether the given value is valid or not */
380  capa = &em_shm->mpool_tbl.odp_pool_capability;
381  if (cfg->pkt.headroom.in_use &&
382  cfg->pkt.headroom.value > capa->pkt.max_headroom) {
383  EM_LOG(EM_LOG_ERR,
384  "'%s.pkt.headroom.value' %d too large (max=%d)\n",
385  pool_cfg_str, cfg->pkt.headroom.value,
386  capa->pkt.max_headroom);
387  return -1;
388  }
389 
390  return 0;
391 }
392 
393 /* Read option: startup_pools.conf[index] from the EM config file */
394 static int read_config_startup_pools_conf(const libconfig_list_t *list, int index)
395 {
396  int ret;
397  int pool;
398  int ret_pool;
399  int num_subpools;
400  const char *pool_name;
401  const char *event_type;
402  char pool_cfg_str[40];
403  libconfig_group_t *pool_cfg;
404  const libconfig_list_t *subpool;
405  const libconfig_group_t *headroom;
406  const libconfig_group_t *user_area;
407  const libconfig_group_t *align_offset;
408  startup_pool_conf_t *conf = &em_shm->opt.startup_pools.conf[index];
409  em_pool_cfg_t *cfg = &conf->cfg;
410  const char *err_str = "";
411 
412  snprintf(pool_cfg_str, sizeof(pool_cfg_str),
413  "startup_pools.conf[%d].pool_cfg", index);
414 
415  pool_cfg = em_libconfig_list_lookup_group(list, index, "pool_cfg");
416  if (!pool_cfg) {
417  EM_LOG(EM_LOG_ERR, "Conf option '%s' not found\n", pool_cfg_str);
418  return -1;
419  }
420 
421  em_pool_cfg_init(cfg);
422 
423  /*
424  * Read mandatory fields first, in case they are not provided, no need
425  * to proceed to read optional fields.
426  */
427 
428  /* Option: startup_pools.conf[index].pool_cfg.event_type */
429  ret = em_libconfig_group_lookup_string(pool_cfg, "event_type", &event_type);
430  if (unlikely(!ret)) {
431  EM_LOG(EM_LOG_ERR, "'%s.event_type' not found.\n", pool_cfg_str);
432  return -1;
433  }
434 
435  ret = event_type_from_string(event_type, &cfg->event_type/*out*/);
436  if (unlikely(ret < 0))
437  return -1;
438 
439  ret = is_pool_type_supported(cfg->event_type, &err_str/*out*/);
440  if (unlikely(ret)) {
441  EM_LOG(EM_LOG_ERR, "%s", err_str);
442  return -1;
443  }
444 
445  /* Option: startup_pools.conf[index].pool_cfg.num_subpools */
446  ret = em_libconfig_group_lookup_int(pool_cfg, "num_subpools",
447  &cfg->num_subpools);
448  if (unlikely(!ret)) {
449  EM_LOG(EM_LOG_ERR, "'%s.num_subpools' not found.\n", pool_cfg_str);
450  return -1;
451  }
452 
453  if (cfg->num_subpools <= 0 || cfg->num_subpools > EM_MAX_SUBPOOLS) {
454  EM_LOG(EM_LOG_ERR, "Invalid '%s.num_subpools'\n"
455  "Valid value range is [1, %d]\n", pool_cfg_str,
456  EM_MAX_SUBPOOLS);
457  return -1;
458  }
459 
460  /* Option: startup_pools.conf[index].pool_cfg.subpools */
461  subpool = em_libconfig_group_lookup_list(pool_cfg, "subpools");
462  if (unlikely(!subpool)) {
463  EM_LOG(EM_LOG_ERR, "'%s.subpools' not found.\n", pool_cfg_str);
464  return -1;
465  }
466 
467  num_subpools = em_libconfig_list_length(subpool);
468  if (unlikely(num_subpools != cfg->num_subpools)) {
469  EM_LOG(EM_LOG_ERR, "The number of subpool configuration given\n"
470  "in '%s.subpools' does not match '%s.num_subpools'.\n",
471  pool_cfg_str, pool_cfg_str);
472  return -1;
473  }
474 
475  for (int j = 0; j < num_subpools; j++) {
476  ret = read_config_subpool(subpool, j, pool_cfg_str, cfg);
477 
478  if (unlikely(ret < 0))
479  return -1;
480  }
481 
482  /* Following are optional configurations */
483 
484  /* Option: startup_pools.conf[index].pool */
485  ret_pool = em_libconfig_list_lookup_int(list, index, "pool", &pool);
486  if (unlikely(ret_pool == 0)) {
487  EM_LOG(EM_LOG_ERR,
488  "'startup_pools.conf[%d].pool' has wrong data type(expect int)\n",
489  index);
490  return -1;
491  }
492 
493  /* startup_pools.conf[index].pool is provided */
494  if (ret_pool == 1) {
495  if (pool < 0 || pool > EM_CONFIG_POOLS) {
496  EM_LOG(EM_LOG_ERR, "Invalid pool ID %d, valid IDs are within [0, %d]\n",
497  pool, EM_CONFIG_POOLS);
498  return -1;
499  }
500 
501  conf->pool = (em_pool_t)(uintptr_t)pool;
502  }
503 
504  /* Option: startup_pools.conf[index].name */
505  ret = em_libconfig_list_lookup_string(list, index, "name", &pool_name);
506  if (unlikely(ret == 0)) {
507  EM_LOG(EM_LOG_ERR,
508  "'startup_pools.conf[%d].name' has wrong data type(expect string)\n",
509  index);
510  return -1;
511  }
512 
513  if (ret_pool == 1 && ret == 1) { /*Both pool and name have been given*/
514  const char *is_default_name = strstr(pool_name, EM_POOL_DEFAULT_NAME);
515  bool is_default_id = (conf->pool == EM_POOL_DEFAULT);
516 
517  if (is_default_name && !is_default_id) {
518  EM_LOG(EM_LOG_ERR,
519  "Default name \"%s\" with non-default ID %d\n",
520  EM_POOL_DEFAULT_NAME, (int)(uintptr_t)conf->pool);
521  return -1;
522  }
523 
524  if (is_default_id && !is_default_name) {
525  EM_LOG(EM_LOG_ERR,
526  "Default pool ID 1 with non-default name \"%s\"\n",
527  pool_name);
528  return -1;
529  }
530  }
531 
532  if (ret == 1) { /* Pool name is given and no conflict with pool ID */
533  strncpy(conf->name, pool_name, EM_POOL_NAME_LEN - 1);
534  conf->name[EM_POOL_NAME_LEN - 1] = '\0';
535  }
536 
537  align_offset = em_libconfig_group_lookup_group(pool_cfg, "align_offset");
538  /*align_offset is provided*/
539  if (align_offset && read_config_align_offset(align_offset, pool_cfg_str, cfg))
540  return -1;
541 
542  user_area = em_libconfig_group_lookup_group(pool_cfg, "user_area");
543  if (user_area && read_config_user_area(user_area, pool_cfg_str, cfg))
544  return -1;
545 
546  headroom = em_libconfig_group_lookup_group(pool_cfg, "pkt.headroom");
547  if (headroom) {
548  if (read_config_pkt_headroom(headroom, pool_cfg_str, cfg))
549  return -1;
550 
551  /* Ignore the given pkt.headroom for non packet event type */
552  if (conf->cfg.event_type != EM_EVENT_TYPE_PACKET)
553  EM_PRINT("pkt.headroom will be ignored for non packet type!\n");
554  }
555 
556  return 0;
557 }
558 
559 /* Print option: startup_pools from the EM config file */
560 static void print_config_startup_pools(void)
561 {
562  startup_pool_conf_t *conf;
563  char str_conf[32];
564  const char *str = "";
565 
566  EM_PRINT(" startup_pools.num: %u\n", em_shm->opt.startup_pools.num);
567 
568  for (uint32_t i = 0; i < em_shm->opt.startup_pools.num; i++) {
569  conf = &em_shm->opt.startup_pools.conf[i];
570 
571  snprintf(str_conf, sizeof(str_conf), " startup_pools.conf[%d]", i);
572 
573  if (*conf->name)
574  EM_PRINT("%s.name: %s\n", str_conf, conf->name);
575 
576  if (conf->pool)
577  EM_PRINT("%s.pool: %d\n", str_conf, (int)(uintptr_t)conf->pool);
578 
579  /*event type*/
580  if (conf->cfg.event_type == EM_EVENT_TYPE_SW)
581  str = "EM_EVENT_TYPE_SW";
582  else if (conf->cfg.event_type == EM_EVENT_TYPE_PACKET)
583  str = "EM_EVENT_TYPE_PACKET";
584  else if (conf->cfg.event_type == EM_EVENT_TYPE_VECTOR)
585  str = "EM_EVENT_TYPE_VECTOR";
586  EM_PRINT("%s.pool_cfg.event_type: %s\n", str_conf, str);
587 
588  /*align_offset*/
589  str = conf->cfg.align_offset.in_use ? "true" : "false";
590  EM_PRINT("%s.pool_cfg.align_offset.in_use: %s\n", str_conf, str);
591  EM_PRINT("%s.pool_cfg.align_offset.value: %d\n", str_conf,
592  conf->cfg.align_offset.value);
593 
594  /*user area*/
595  str = conf->cfg.user_area.in_use ? "true" : "false";
596  EM_PRINT("%s.pool_cfg.user_area.in_use: %s\n", str_conf, str);
597  EM_PRINT("%s.pool_cfg.user_area.size: %ld\n", str_conf,
598  conf->cfg.user_area.size);
599 
600  /*pkt headroom*/
601  str = conf->cfg.pkt.headroom.in_use ? "true" : "false";
602  EM_PRINT("%s.pool_cfg.pkt.headroom.in_use: %s\n", str_conf, str);
603  EM_PRINT("%s.pool_cfg.pkt.headroom.value: %d\n", str_conf,
604  conf->cfg.pkt.headroom.value);
605 
606  /*number of subpools*/
607  EM_PRINT("%s.pool_cfg.num_subpools: %u\n", str_conf,
608  conf->cfg.num_subpools);
609 
610  /*subpools*/
611  for (int j = 0; j < conf->cfg.num_subpools; j++) {
612  EM_PRINT("%s.pool_cfg.subpools[%d].size: %u\n", str_conf,
613  j, conf->cfg.subpool[j].size);
614 
615  EM_PRINT("%s.pool_cfg.subpools[%d].num: %u\n", str_conf,
616  j, conf->cfg.subpool[j].num);
617 
618  EM_PRINT("%s.pool_cfg.subpools[%d].cache_size: %u\n",
619  str_conf, j, conf->cfg.subpool[j].cache_size);
620  }
621  }
622 }
623 
624 /* Read option: startup_pools from the EM config file */
625 static int read_config_startup_pools(void)
626 {
627  int ret;
628  int list_len;
629  int num_startup_pools;
630  const libconfig_list_t *conf_list;
631  libconfig_setting_t *default_setting;
632  libconfig_setting_t *runtime_setting;
633  libconfig_setting_t *startup_pools_setting;
634 
635  em_libconfig_lookup(&em_shm->libconfig, "startup_pools",
636  &default_setting, &runtime_setting);
637 
638  /*
639  * Option: startup_pools
640  *
641  * Optional. Thus, when runtime configuration is provided, and option
642  * "startup_pools" is given, use it. However, when option "startup_pools"
643  * is not specified in the given runtime configuration file, returns
644  * without giving error, which means no startup pools will be created.
645  * Note that it does not fall back to use the option "startup_pools"
646  * specified in the default configuration file.
647  */
648  if (em_shm->libconfig.has_cfg_runtime) {
649  if (runtime_setting)
650  startup_pools_setting = runtime_setting;
651  else
652  return 0;
653  } else {
654  if (default_setting)
655  startup_pools_setting = default_setting;
656  else
657  return 0;
658  }
659 
660  EM_PRINT("EM-startup_pools config:\n");
661  /*
662  * Option: startup_pools.num
663  * Mandatory when startup_pools option is given
664  */
665  ret = em_libconfig_setting_lookup_int(startup_pools_setting, "num",
666  &num_startup_pools);
667  if (unlikely(!ret)) {
668  EM_LOG(EM_LOG_ERR, "Option 'startup_pools.num' not found\n");
669  return -1;
670  }
671 
672  if (num_startup_pools <= 0 || num_startup_pools > EM_CONFIG_POOLS - 1) {
673  EM_LOG(EM_LOG_ERR,
674  "Number of startup_pools %d is too large or too small\n"
675  "Valid value range is [1, %d]\n",
676  num_startup_pools, EM_CONFIG_POOLS - 1);
677  return -1;
678  }
679 
680  conf_list = em_libconfig_setting_get_list(startup_pools_setting, "conf");
681  if (!conf_list) {
682  EM_LOG(EM_LOG_ERR, "Conf option 'startup_pools.conf' not found\n");
683  return -1;
684  }
685 
686  list_len = em_libconfig_list_length(conf_list);
687  if (list_len != num_startup_pools) {
688  EM_LOG(EM_LOG_ERR,
689  "The number of pool configuration(s) given in\n"
690  "'startup_pools.conf':%d does not match number of\n"
691  "startup_pools specified in 'startup_pools.num': %d\n",
692  list_len, num_startup_pools);
693  return -1;
694  }
695 
696  for (int i = 0; i < list_len; i++) {
697  if (read_config_startup_pools_conf(conf_list, i) < 0)
698  return -1;
699  }
700 
701  em_shm->opt.startup_pools.num = num_startup_pools;
702 
703  print_config_startup_pools();
704  return 0;
705 }
706 
707 /* Read option: pool from the EM config file */
708 static int read_config_pool(void)
709 {
710  const char *conf_str;
711  bool val_bool = false;
712  int val = 0;
713  int ret;
714 
715  const odp_pool_capability_t *capa =
716  &em_shm->mpool_tbl.odp_pool_capability;
717 
718  EM_PRINT("EM-pool config:\n");
719 
720  /*
721  * Option: pool.statistics.available
722  */
723  conf_str = "pool.statistics.available";
724  ret = em_libconfig_lookup_bool(&em_shm->libconfig, conf_str, &val_bool);
725  if (unlikely(!ret)) {
726  EM_LOG(EM_LOG_ERR, "Config option '%s' not found\n", conf_str);
727  return -1;
728  }
729  em_shm->opt.pool.statistics.available = (int)val_bool;
730  EM_PRINT(" %s: %s(%d)\n", conf_str, val_bool ? "true" : "false", val_bool);
731 
732  /*
733  * Option: pool.statistics.alloc_ops
734  */
735  conf_str = "pool.statistics.alloc_ops";
736  ret = em_libconfig_lookup_bool(&em_shm->libconfig, conf_str, &val_bool);
737  if (unlikely(!ret)) {
738  EM_LOG(EM_LOG_ERR, "Config option '%s' not found\n", conf_str);
739  return -1;
740  }
741  em_shm->opt.pool.statistics.alloc_ops = (int)val_bool;
742  EM_PRINT(" %s: %s(%d)\n", conf_str, val_bool ? "true" : "false", val_bool);
743 
744  /*
745  * Option: pool.statistics.alloc_fails
746  */
747  conf_str = "pool.statistics.alloc_fails";
748  ret = em_libconfig_lookup_bool(&em_shm->libconfig, conf_str, &val_bool);
749  if (unlikely(!ret)) {
750  EM_LOG(EM_LOG_ERR, "Config option '%s' not found\n", conf_str);
751  return -1;
752  }
753  em_shm->opt.pool.statistics.alloc_fails = (int)val_bool;
754  EM_PRINT(" %s: %s(%d)\n", conf_str, val_bool ? "true" : "false", val_bool);
755 
756  /*
757  * Option: pool.statistics.free_ops
758  */
759  conf_str = "pool.statistics.free_ops";
760  ret = em_libconfig_lookup_bool(&em_shm->libconfig, conf_str, &val_bool);
761  if (unlikely(!ret)) {
762  EM_LOG(EM_LOG_ERR, "Config option '%s' not found\n", conf_str);
763  return -1;
764  }
765  em_shm->opt.pool.statistics.free_ops = (int)val_bool;
766  EM_PRINT(" %s: %s(%d)\n", conf_str, val_bool ? "true" : "false", val_bool);
767 
768  /*
769  * Option: pool.statistics.total_ops
770  */
771  conf_str = "pool.statistics.total_ops";
772  ret = em_libconfig_lookup_bool(&em_shm->libconfig, conf_str, &val_bool);
773  if (unlikely(!ret)) {
774  EM_LOG(EM_LOG_ERR, "Config option '%s' not found\n", conf_str);
775  return -1;
776  }
777  em_shm->opt.pool.statistics.total_ops = (int)val_bool;
778  EM_PRINT(" %s: %s(%d)\n", conf_str, val_bool ? "true" : "false", val_bool);
779 
780  /*
781  * Option: pool.statistics.cache_available
782  */
783  conf_str = "pool.statistics.cache_available";
784  ret = em_libconfig_lookup_bool(&em_shm->libconfig, conf_str, &val_bool);
785  if (unlikely(!ret)) {
786  EM_LOG(EM_LOG_ERR, "Config option '%s' not found\n", conf_str);
787  return -1;
788  }
789  em_shm->opt.pool.statistics.cache_available = (int)val_bool;
790  EM_PRINT(" %s: %s(%d)\n", conf_str, val_bool ? "true" : "false", val_bool);
791 
792  /*
793  * Option: pool.statistics.cache_alloc_ops
794  */
795  conf_str = "pool.statistics.cache_alloc_ops";
796  ret = em_libconfig_lookup_bool(&em_shm->libconfig, conf_str, &val_bool);
797  if (unlikely(!ret)) {
798  EM_LOG(EM_LOG_ERR, "Config option '%s' not found\n", conf_str);
799  return -1;
800  }
801  em_shm->opt.pool.statistics.cache_alloc_ops = (int)val_bool;
802  EM_PRINT(" %s: %s(%d)\n", conf_str, val_bool ? "true" : "false", val_bool);
803 
804  /*
805  * Option: pool.statistics.cache_free_ops
806  */
807  conf_str = "pool.statistics.cache_free_ops";
808  ret = em_libconfig_lookup_bool(&em_shm->libconfig, conf_str, &val_bool);
809  if (unlikely(!ret)) {
810  EM_LOG(EM_LOG_ERR, "Config option '%s' not found\n", conf_str);
811  return -1;
812  }
813  em_shm->opt.pool.statistics.cache_free_ops = (int)val_bool;
814  EM_PRINT(" %s: %s(%d)\n", conf_str, val_bool ? "true" : "false", val_bool);
815 
816  /*
817  * Option: pool.statistics.core_cache_available
818  */
819  conf_str = "pool.statistics.core_cache_available";
820  ret = em_libconfig_lookup_bool(&em_shm->libconfig, conf_str, &val_bool);
821  if (unlikely(!ret)) {
822  EM_LOG(EM_LOG_ERR, "Config option '%s' not found\n", conf_str);
823  return -1;
824  }
825  em_shm->opt.pool.statistics.core_cache_available = (int)val_bool;
826  EM_PRINT(" %s: %s(%d)\n", conf_str, val_bool ? "true" : "false", val_bool);
827 
828  /*
829  * Option: pool.align_offset
830  */
831  conf_str = "pool.align_offset";
832  ret = em_libconfig_lookup_int(&em_shm->libconfig, conf_str, &val);
833  if (unlikely(!ret)) {
834  EM_LOG(EM_LOG_ERR, "Config option '%s' not found.\n", conf_str);
835  return -1;
836  }
837  if (val < 0 || val > ALIGN_OFFSET_MAX || !POWEROF2(val)) {
838  EM_LOG(EM_LOG_ERR,
839  "Bad config value '%s = %d' (max: %d and value must be power of 2)\n",
840  conf_str, val, ALIGN_OFFSET_MAX);
841  return -1;
842  }
843  /* store & print the value */
844  em_shm->opt.pool.align_offset = val;
845  EM_PRINT(" %s (default): %d (max: %d)\n",
846  conf_str, val, ALIGN_OFFSET_MAX);
847 
848  /*
849  * Option: pool.user_area_size
850  */
851  conf_str = "pool.user_area_size";
852  ret = em_libconfig_lookup_int(&em_shm->libconfig, conf_str, &val);
853  if (unlikely(!ret)) {
854  EM_LOG(EM_LOG_ERR, "Config option '%s' not found.\n", conf_str);
855  return -1;
856  }
857  if (val < 0 || (unsigned int)val > capa->pkt.max_uarea_size ||
858  val > EM_EVENT_USER_AREA_MAX_SIZE) {
859  EM_LOG(EM_LOG_ERR, "Bad config value '%s = %d'\n",
860  conf_str, val);
861  return -1;
862  }
863  /* store & print the value */
864  em_shm->opt.pool.user_area_size = val;
865  EM_PRINT(" %s (default): %d (max: %d)\n",
866  conf_str, val,
867  MIN(EM_EVENT_USER_AREA_MAX_SIZE, capa->pkt.max_uarea_size));
868 
869  /*
870  * Option: pool.pkt_headroom
871  */
872  conf_str = "pool.pkt_headroom";
873  ret = em_libconfig_lookup_int(&em_shm->libconfig, conf_str, &val);
874  if (unlikely(!ret)) {
875  EM_LOG(EM_LOG_ERR, "Config option '%s' not found.\n", conf_str);
876  return -1;
877  }
878 
879  if (val < 0 || (unsigned int)val > capa->pkt.max_headroom) {
880  EM_LOG(EM_LOG_ERR, "Bad config value '%s = %d'\n",
881  conf_str, val);
882  return -1;
883  }
884  /* store & print the value */
885  em_shm->opt.pool.pkt_headroom = val;
886  EM_PRINT(" %s (default): %d (max: %u)\n",
887  conf_str, val, capa->pkt.max_headroom);
888 
889  return 0;
890 }
891 
892 static int
893 read_config_file(void)
894 {
895  /* Option: pool */
896  if (read_config_pool() < 0)
897  return -1;
898 
899  /* Option: startup_pools */
900  if (read_config_startup_pools() < 0)
901  return -1;
902 
903  return 0;
904 }
905 
906 /* We use following static asserts and function check_em_pool_subpool_stats()
907  * to verify at both compile time and runtime that, em_pool_subpool_stats_t is
908  * exactly the same as odp_pool_stats_t except the last struct member, namely,
909  * 'em_pool_subpool_stats_t::__internal_use', whose size must also be bigger
910  * than that of 'odp_pool_stats_t::thread'. This allows us to avoid exposing ODP
911  * type in EM-ODP API (at event_machine_pool.h in this case) and allows us to
912  * type cast 'em_pool_subpool_stats_t' to 'odp_pool_stats_t', ensuring high
913  * performance (see em_pool_stats() and em_pool_subpool_stats()).
914  */
915 
916 ODP_STATIC_ASSERT(sizeof(odp_pool_stats_t) <= sizeof(em_pool_subpool_stats_t),
917  "Size of odp_pool_stats_t must be smaller than that of em_pool_subpool_stats_t");
918 
919 ODP_STATIC_ASSERT(offsetof(odp_pool_stats_t, available) ==
920  offsetof(em_pool_subpool_stats_t, available) &&
921  sizeof_field(odp_pool_stats_t, available) ==
922  sizeof_field(em_pool_subpool_stats_t, available),
923  "em_pool_subpool_stats_t.available differs from odp_pool_stats_t.available!");
924 
925 ODP_STATIC_ASSERT(offsetof(odp_pool_stats_t, alloc_ops) ==
926  offsetof(em_pool_subpool_stats_t, alloc_ops) &&
927  sizeof_field(odp_pool_stats_t, alloc_ops) ==
928  sizeof_field(em_pool_subpool_stats_t, alloc_ops),
929  "em_pool_subpool_stats_t.alloc_ops differs from odp_pool_stats_t.alloc_ops!");
930 
931 ODP_STATIC_ASSERT(offsetof(odp_pool_stats_t, alloc_fails) ==
932  offsetof(em_pool_subpool_stats_t, alloc_fails) &&
933  sizeof_field(odp_pool_stats_t, alloc_fails) ==
934  sizeof_field(em_pool_subpool_stats_t, alloc_fails),
935  "em_pool_subpool_stats_t.alloc_fails differs from odp_pool_stats_t.alloc_fails!");
936 
937 ODP_STATIC_ASSERT(offsetof(odp_pool_stats_t, free_ops) ==
938  offsetof(em_pool_subpool_stats_t, free_ops) &&
939  sizeof_field(odp_pool_stats_t, free_ops) ==
940  sizeof_field(em_pool_subpool_stats_t, free_ops),
941  "em_pool_subpool_stats_t.free_ops differs from odp_pool_stats_t.free_ops!");
942 
943 ODP_STATIC_ASSERT(offsetof(odp_pool_stats_t, total_ops) ==
944  offsetof(em_pool_subpool_stats_t, total_ops) &&
945  sizeof_field(odp_pool_stats_t, total_ops) ==
946  sizeof_field(em_pool_subpool_stats_t, total_ops),
947  "em_pool_subpool_stats_t.total_ops differs from odp_pool_stats_t.total_ops!");
948 
949 ODP_STATIC_ASSERT(offsetof(odp_pool_stats_t, cache_available) ==
950  offsetof(em_pool_subpool_stats_t, cache_available) &&
951  sizeof_field(odp_pool_stats_t, cache_available) ==
952  sizeof_field(em_pool_subpool_stats_t, cache_available),
953  "em_pool_subpool_stats_t.cache_available differs from that of odp_pool_stats_t!");
954 
955 ODP_STATIC_ASSERT(offsetof(odp_pool_stats_t, cache_alloc_ops) ==
956  offsetof(em_pool_subpool_stats_t, cache_alloc_ops) &&
957  sizeof_field(odp_pool_stats_t, cache_alloc_ops) ==
958  sizeof_field(em_pool_subpool_stats_t, cache_alloc_ops),
959  "em_pool_subpool_stats_t.cache_alloc_ops differs from that of odp_pool_stats_t!");
960 
961 ODP_STATIC_ASSERT(offsetof(odp_pool_stats_t, cache_free_ops) ==
962  offsetof(em_pool_subpool_stats_t, cache_free_ops) &&
963  sizeof_field(odp_pool_stats_t, cache_free_ops) ==
964  sizeof_field(em_pool_subpool_stats_t, cache_free_ops),
965  "em_pool_subpool_stats_t.cache_free_ops differs from that of odp_pool_stats_t!");
966 
967 ODP_STATIC_ASSERT(offsetof(odp_pool_stats_t, thread) ==
968  offsetof(em_pool_subpool_stats_t, __internal_use) &&
969  sizeof_field(odp_pool_stats_t, thread) <=
970  sizeof_field(em_pool_subpool_stats_t, __internal_use),
971  "em_pool_subpool_stats_t.__internal_use differs from odp_pool_stats_t.thread");
972 
973 #define STRUCT_ERR_STR \
974 "em_pool_subpool_stats_t.%s differs from odp_pool_stats_t.%s either in size or in offset!\n"
975 
976 static int check_em_pool_subpool_stats(void)
977 {
978  if (sizeof(odp_pool_stats_t) > sizeof(em_pool_subpool_stats_t)) {
979  EM_LOG(EM_LOG_ERR,
980  "Size of odp_pool_stats_t bigger than that of em_pool_subpool_stats_t\n");
981  return -1;
982  }
983 
984  if (offsetof(odp_pool_stats_t, available) !=
985  offsetof(em_pool_subpool_stats_t, available) ||
986  sizeof_field(odp_pool_stats_t, available) !=
987  sizeof_field(em_pool_subpool_stats_t, available)) {
988  EM_LOG(EM_LOG_ERR, STRUCT_ERR_STR, "available", "available");
989  return -1;
990  }
991 
992  if (offsetof(odp_pool_stats_t, alloc_ops) !=
993  offsetof(em_pool_subpool_stats_t, alloc_ops) ||
994  sizeof_field(odp_pool_stats_t, alloc_ops) !=
995  sizeof_field(em_pool_subpool_stats_t, alloc_ops)) {
996  EM_LOG(EM_LOG_ERR, STRUCT_ERR_STR, "alloc_ops", "alloc_ops");
997  return -1;
998  }
999 
1000  if (offsetof(odp_pool_stats_t, alloc_fails) !=
1001  offsetof(em_pool_subpool_stats_t, alloc_fails) ||
1002  sizeof_field(odp_pool_stats_t, alloc_fails) !=
1003  sizeof_field(em_pool_subpool_stats_t, alloc_fails)) {
1004  EM_LOG(EM_LOG_ERR, STRUCT_ERR_STR, "alloc_fails", "alloc_fails");
1005  return -1;
1006  }
1007 
1008  if (offsetof(odp_pool_stats_t, free_ops) !=
1009  offsetof(em_pool_subpool_stats_t, free_ops) ||
1010  sizeof_field(odp_pool_stats_t, free_ops) !=
1011  sizeof_field(em_pool_subpool_stats_t, free_ops)) {
1012  EM_LOG(EM_LOG_ERR, STRUCT_ERR_STR, "free_ops", "free_ops");
1013  return -1;
1014  }
1015 
1016  if (offsetof(odp_pool_stats_t, total_ops) !=
1017  offsetof(em_pool_subpool_stats_t, total_ops) ||
1018  sizeof_field(odp_pool_stats_t, total_ops) !=
1019  sizeof_field(em_pool_subpool_stats_t, total_ops)) {
1020  EM_LOG(EM_LOG_ERR, STRUCT_ERR_STR, "total_ops", "total_ops");
1021  return -1;
1022  }
1023 
1024  if (offsetof(odp_pool_stats_t, cache_available) !=
1025  offsetof(em_pool_subpool_stats_t, cache_available) ||
1026  sizeof_field(odp_pool_stats_t, cache_available) !=
1027  sizeof_field(em_pool_subpool_stats_t, cache_available)) {
1028  EM_LOG(EM_LOG_ERR, STRUCT_ERR_STR, "cache_available", "cache_available");
1029  return -1;
1030  }
1031 
1032  if (offsetof(odp_pool_stats_t, cache_alloc_ops) !=
1033  offsetof(em_pool_subpool_stats_t, cache_alloc_ops) ||
1034  sizeof_field(odp_pool_stats_t, cache_alloc_ops) !=
1035  sizeof_field(em_pool_subpool_stats_t, cache_alloc_ops)) {
1036  EM_LOG(EM_LOG_ERR, STRUCT_ERR_STR, "cache_alloc_ops", "cache_alloc_ops");
1037  return -1;
1038  }
1039 
1040  if (offsetof(odp_pool_stats_t, cache_free_ops) !=
1041  offsetof(em_pool_subpool_stats_t, cache_free_ops) ||
1042  sizeof_field(odp_pool_stats_t, cache_free_ops) !=
1043  sizeof_field(em_pool_subpool_stats_t, cache_free_ops)) {
1044  EM_LOG(EM_LOG_ERR, STRUCT_ERR_STR, "cache_free_ops", "cache_free_ops");
1045  return -1;
1046  }
1047 
1048  if (offsetof(odp_pool_stats_t, thread) !=
1049  offsetof(em_pool_subpool_stats_t, __internal_use) ||
1050  sizeof_field(odp_pool_stats_t, thread) >
1051  sizeof_field(em_pool_subpool_stats_t, __internal_use)) {
1052  EM_LOG(EM_LOG_ERR, STRUCT_ERR_STR, "__internal_use", "thread");
1053  return -1;
1054  }
1055 
1056  return 0;
1057 }
1058 
1059 /* We use following static asserts and function check_em_pool_subpool_stats_selected()
1060  * to verify at both compile time and runtime that, em_pool_subpool_stats_selected_t
1061  * is exactly the same as odp_pool_stats_selected_t This allows us to avoid exposing
1062  * ODP type in EM-ODP API (at event_machine_pool.h in this case) and allows us to
1063  * type cast 'em_pool_subpool_stats_selected_t' to 'odp_pool_stats_selected_t', ensuring
1064  * high performance (see em_pool_stats_selected() and em_pool_subpool_stats_selected()).
1065  */
1066 
1067 #define SIZE_NOT_EQUAL_ERR_STR \
1068 "Size of odp_pool_stats_selected_t must equal to that of em_pool_subpool_stats_selected_t\n"
1069 
1070 ODP_STATIC_ASSERT(sizeof(odp_pool_stats_selected_t) == sizeof(em_pool_subpool_stats_selected_t),
1071  SIZE_NOT_EQUAL_ERR_STR);
1072 
1073 ODP_STATIC_ASSERT(offsetof(odp_pool_stats_selected_t, available) ==
1074  offsetof(em_pool_subpool_stats_selected_t, available) &&
1075  sizeof_field(odp_pool_stats_selected_t, available) ==
1076  sizeof_field(em_pool_subpool_stats_selected_t, available),
1077  "available in em_pool_subpool_stats_selected_t and odp_pool_stats_selected_t differs!");
1078 
1079 ODP_STATIC_ASSERT(offsetof(odp_pool_stats_selected_t, alloc_ops) ==
1080  offsetof(em_pool_subpool_stats_selected_t, alloc_ops) &&
1081  sizeof_field(odp_pool_stats_selected_t, alloc_ops) ==
1082  sizeof_field(em_pool_subpool_stats_selected_t, alloc_ops),
1083  "em_pool_subpool_stats_selected_t.alloc_ops differs from odp_pool_stats_selected_t.alloc_ops!");
1084 
1085 ODP_STATIC_ASSERT(offsetof(odp_pool_stats_selected_t, alloc_fails) ==
1086  offsetof(em_pool_subpool_stats_t, alloc_fails) &&
1087  sizeof_field(odp_pool_stats_selected_t, alloc_fails) ==
1088  sizeof_field(em_pool_subpool_stats_t, alloc_fails),
1089  "em_pool_subpool_stats_selected_t.alloc_fails differs from odp_pool_stats_selected_t.alloc_fails!");
1090 
1091 ODP_STATIC_ASSERT(offsetof(odp_pool_stats_selected_t, free_ops) ==
1092  offsetof(em_pool_subpool_stats_selected_t, free_ops) &&
1093  sizeof_field(odp_pool_stats_selected_t, free_ops) ==
1094  sizeof_field(em_pool_subpool_stats_selected_t, free_ops),
1095  "em_pool_subpool_stats_selected_t.free_ops differs from odp_pool_stats_selected_t.free_ops!");
1096 
1097 ODP_STATIC_ASSERT(offsetof(odp_pool_stats_selected_t, total_ops) ==
1098  offsetof(em_pool_subpool_stats_selected_t, total_ops) &&
1099  sizeof_field(odp_pool_stats_selected_t, total_ops) ==
1100  sizeof_field(em_pool_subpool_stats_selected_t, total_ops),
1101  "em_pool_subpool_stats_selected_t.total_ops differs from odp_pool_stats_selected_t.total_ops!");
1102 
1103 ODP_STATIC_ASSERT(offsetof(odp_pool_stats_selected_t, cache_available) ==
1104  offsetof(em_pool_subpool_stats_selected_t, cache_available) &&
1105  sizeof_field(odp_pool_stats_selected_t, cache_available) ==
1106  sizeof_field(em_pool_subpool_stats_selected_t, cache_available),
1107  "em_pool_subpool_stats_selected_t.cache_available differs from that of odp_pool_stats_selected_t!");
1108 
1109 ODP_STATIC_ASSERT(offsetof(odp_pool_stats_selected_t, cache_alloc_ops) ==
1110  offsetof(em_pool_subpool_stats_selected_t, cache_alloc_ops) &&
1111  sizeof_field(odp_pool_stats_selected_t, cache_alloc_ops) ==
1112  sizeof_field(em_pool_subpool_stats_selected_t, cache_alloc_ops),
1113  "em_pool_subpool_stats_selected_t.cache_alloc_ops differs from that of odp_pool_stats_selected_t!");
1114 
1115 ODP_STATIC_ASSERT(offsetof(odp_pool_stats_selected_t, cache_free_ops) ==
1116  offsetof(em_pool_subpool_stats_selected_t, cache_free_ops) &&
1117  sizeof_field(odp_pool_stats_selected_t, cache_free_ops) ==
1118  sizeof_field(em_pool_subpool_stats_selected_t, cache_free_ops),
1119  "em_pool_subpool_stats_selected_t.cache_free_ops differs from that of odp_pool_stats_selected_t!");
1120 
1121 #define SELECTED_TYPE_ERR_FMT \
1122 "em_pool_subpool_stats_selected_t.%s differs from odp_pool_stats_selected_t.%s\n"
1123 
1124 static int check_em_pool_subpool_stats_selected(void)
1125 {
1126  if (sizeof(odp_pool_stats_selected_t) != sizeof(em_pool_subpool_stats_selected_t)) {
1127  EM_LOG(EM_LOG_ERR,
1128  "odp_pool_stats_selected_t vs em_pool_subpool_stats_selected_t size diff\n");
1129  return -1;
1130  }
1131 
1132  if (offsetof(odp_pool_stats_selected_t, available) !=
1133  offsetof(em_pool_subpool_stats_selected_t, available) ||
1134  sizeof_field(odp_pool_stats_selected_t, available) !=
1135  sizeof_field(em_pool_subpool_stats_selected_t, available)) {
1136  EM_LOG(EM_LOG_ERR, SELECTED_TYPE_ERR_FMT, "available", "available");
1137  return -1;
1138  }
1139 
1140  if (offsetof(odp_pool_stats_selected_t, alloc_ops) !=
1141  offsetof(em_pool_subpool_stats_selected_t, alloc_ops) ||
1142  sizeof_field(odp_pool_stats_selected_t, alloc_ops) !=
1143  sizeof_field(em_pool_subpool_stats_selected_t, alloc_ops)) {
1144  EM_LOG(EM_LOG_ERR, SELECTED_TYPE_ERR_FMT, "alloc_ops", "alloc_ops");
1145  return -1;
1146  }
1147 
1148  if (offsetof(odp_pool_stats_selected_t, alloc_fails) !=
1149  offsetof(em_pool_subpool_stats_selected_t, alloc_fails) ||
1150  sizeof_field(odp_pool_stats_selected_t, alloc_fails) !=
1151  sizeof_field(em_pool_subpool_stats_selected_t, alloc_fails)) {
1152  EM_LOG(EM_LOG_ERR, SELECTED_TYPE_ERR_FMT, "alloc_fails", "alloc_fails");
1153  return -1;
1154  }
1155 
1156  if (offsetof(odp_pool_stats_selected_t, free_ops) !=
1157  offsetof(em_pool_subpool_stats_selected_t, free_ops) ||
1158  sizeof_field(odp_pool_stats_selected_t, free_ops) !=
1159  sizeof_field(em_pool_subpool_stats_selected_t, free_ops)) {
1160  EM_LOG(EM_LOG_ERR, SELECTED_TYPE_ERR_FMT, "free_ops", "free_ops");
1161  return -1;
1162  }
1163 
1164  if (offsetof(odp_pool_stats_selected_t, total_ops) !=
1165  offsetof(em_pool_subpool_stats_selected_t, total_ops) ||
1166  sizeof_field(odp_pool_stats_selected_t, total_ops) !=
1167  sizeof_field(em_pool_subpool_stats_selected_t, total_ops)) {
1168  EM_LOG(EM_LOG_ERR, SELECTED_TYPE_ERR_FMT, "total_ops", "total_ops");
1169  return -1;
1170  }
1171 
1172  if (offsetof(odp_pool_stats_selected_t, cache_available) !=
1173  offsetof(em_pool_subpool_stats_selected_t, cache_available) ||
1174  sizeof_field(odp_pool_stats_selected_t, cache_available) !=
1175  sizeof_field(em_pool_subpool_stats_selected_t, cache_available)) {
1176  EM_LOG(EM_LOG_ERR, SELECTED_TYPE_ERR_FMT, "cache_available", "cache_available");
1177  return -1;
1178  }
1179 
1180  if (offsetof(odp_pool_stats_selected_t, cache_alloc_ops) !=
1181  offsetof(em_pool_subpool_stats_selected_t, cache_alloc_ops) ||
1182  sizeof_field(odp_pool_stats_selected_t, cache_alloc_ops) !=
1183  sizeof_field(em_pool_subpool_stats_selected_t, cache_alloc_ops)) {
1184  EM_LOG(EM_LOG_ERR, SELECTED_TYPE_ERR_FMT, "cache_alloc_ops", "cache_alloc_ops");
1185  return -1;
1186  }
1187 
1188  if (offsetof(odp_pool_stats_selected_t, cache_free_ops) !=
1189  offsetof(em_pool_subpool_stats_selected_t, cache_free_ops) ||
1190  sizeof_field(odp_pool_stats_selected_t, cache_free_ops) !=
1191  sizeof_field(em_pool_subpool_stats_selected_t, cache_free_ops)) {
1192  EM_LOG(EM_LOG_ERR, SELECTED_TYPE_ERR_FMT, "cache_free_ops", "cache_free_ops");
1193  return -1;
1194  }
1195 
1196  return 0;
1197 }
1198 
1199 ODP_STATIC_ASSERT(sizeof(odp_pool_stats_opt_t) == sizeof(em_pool_stats_opt_t),
1200  "Size of odp_pool_stats_opt_t differs from that of em_pool_stats_opt_t\n");
1201 
1202 em_status_t
1203 pool_init(mpool_tbl_t *const mpool_tbl, mpool_pool_t *const mpool_pool,
1204  const em_pool_cfg_t *default_pool_cfg)
1205 {
1206  int ret;
1207  em_pool_t pool;
1208  em_pool_t pool_default;
1209  startup_pool_conf_t *startup_pool_conf;
1210  bool default_pool_set = false;
1211  const uint32_t objpool_subpools = MIN(4, OBJSUBPOOLS_MAX);
1212 
1213  /* Return error if em_pool_subpool_stats_t differs from odp_pool_stats_t */
1214  if (check_em_pool_subpool_stats())
1215  return EM_ERR;
1216 
1217  /*Return error if em_pool_subpool_stats_selected_t differs from odp_pool_stats_selected_t*/
1218  if (check_em_pool_subpool_stats_selected())
1219  return EM_ERR;
1220 
1221  memset(mpool_tbl, 0, sizeof(mpool_tbl_t));
1222  memset(mpool_pool, 0, sizeof(mpool_pool_t));
1223  env_atomic32_init(&em_shm->pool_count);
1224 
1225  ret = objpool_init(&mpool_pool->objpool, objpool_subpools);
1226  if (ret != 0)
1227  return EM_ERR_OPERATION_FAILED;
1228 
1229  for (uint32_t i = 0; i < EM_CONFIG_POOLS; i++) {
1230  pool = pool_idx2hdl(i);
1231  mpool_elem_t *mpool_elem = pool_elem_get(pool);
1232 
1233  if (unlikely(!mpool_elem))
1234  return EM_ERR_BAD_POINTER;
1235 
1236  mpool_elem->em_pool = pool;
1237  mpool_elem->event_type = EM_EVENT_TYPE_UNDEF;
1238  for (int j = 0; j < EM_MAX_SUBPOOLS; j++) {
1239  mpool_elem->odp_pool[j] = ODP_POOL_INVALID;
1240  mpool_elem->size[j] = 0;
1241  }
1242 
1243  objpool_add(&mpool_pool->objpool, i % objpool_subpools,
1244  &mpool_elem->objpool_elem);
1245  }
1246 
1247  /* Init the mapping tbl from odp-pool(=subpool) index to em-pool */
1248  if (odp_pool_max_index() >= POOL_ODP2EM_TBL_LEN)
1249  return EM_ERR_TOO_LARGE;
1250  for (int i = 0; i < POOL_ODP2EM_TBL_LEN; i++)
1251  mpool_tbl->pool_subpool_odp2em[i].both = pool_subpool_undef.both;
1252 
1253  /* Store common ODP pool capabilities in the mpool_tbl for easy access*/
1254  if (odp_pool_capability(&mpool_tbl->odp_pool_capability) != 0)
1255  return EM_ERR_LIB_FAILED;
1256 
1257  /* Read EM-pool and EM-startup_pools related runtime config options */
1258  if (read_config_file())
1259  return EM_ERR_LIB_FAILED;
1260 
1261  /*
1262  * Create default and startup pools.
1263  *
1264  * If default pool configuration is given through 'startup_pools.conf'
1265  * in em-odp.conf, use that instead. Otherwise use default_pool_cfg.
1266  *
1267  * Allocate/reserve default pool first here so when creating startup
1268  * pools whose configuration does not provide pool handle, default pool
1269  * handle EM_POOL_DEFAULT(1) won't be allocated to them.
1270  */
1271  pool_default = pool_alloc(EM_POOL_DEFAULT);
1272 
1273  if (unlikely(pool_default == EM_POOL_UNDEF ||
1274  pool_default != EM_POOL_DEFAULT))
1275  return EM_ERR_ALLOC_FAILED;
1276 
1277  /* Create startup pools whose configuration is provided by the EM config file */
1278  for (uint32_t i = 0; i < em_shm->opt.startup_pools.num; i++) {
1279  startup_pool_conf = &em_shm->opt.startup_pools.conf[i];
1280 
1281  /* Default pool is provided by the EM config file */
1282  if (strstr(startup_pool_conf->name, EM_POOL_DEFAULT_NAME) ||
1283  startup_pool_conf->pool == EM_POOL_DEFAULT) {
1284  default_pool_set = true;
1285  pool_free(EM_POOL_DEFAULT);
1286  pool = em_pool_create(EM_POOL_DEFAULT_NAME,
1287  EM_POOL_DEFAULT,
1288  &startup_pool_conf->cfg);
1289  } else {
1290  pool = em_pool_create(startup_pool_conf->name,
1291  startup_pool_conf->pool,
1292  &startup_pool_conf->cfg);
1293  }
1294 
1295  if (pool == EM_POOL_UNDEF)
1296  return EM_ERR_ALLOC_FAILED;
1297  }
1298 
1299  /* Create the default pool if it is not provided by the EM config file */
1300  if (!default_pool_set) {
1301  pool_free(EM_POOL_DEFAULT);
1302  pool = em_pool_create(EM_POOL_DEFAULT_NAME, EM_POOL_DEFAULT,
1303  default_pool_cfg);
1304  if (pool == EM_POOL_UNDEF || pool != EM_POOL_DEFAULT)
1305  return EM_ERR_ALLOC_FAILED;
1306  }
1307 
1308  return EM_OK;
1309 }
1310 
1311 em_status_t
1312 pool_term(const mpool_tbl_t *mpool_tbl)
1313 {
1314  em_status_t stat = EM_OK;
1315 
1316  (void)mpool_tbl;
1317 
1318  EM_PRINT("\n"
1319  "Status before delete:\n");
1320  em_pool_info_print_all();
1321 
1322  for (int i = 0; i < EM_CONFIG_POOLS; i++) {
1323  em_pool_t pool = pool_idx2hdl(i);
1324  const mpool_elem_t *mpool_elem = pool_elem_get(pool);
1325  em_status_t ret;
1326 
1327  if (mpool_elem && pool_allocated(mpool_elem)) {
1328  ret = pool_delete(pool);
1329  if (ret != EM_OK)
1330  stat = ret; /* save last error as return val */
1331  }
1332  }
1333 
1334  return stat;
1335 }
1336 
1337 /* Helper func to invalid_pool_cfg() */
1338 static int invalid_pool_cache_cfg(const em_pool_cfg_t *pool_cfg,
1339  const char **err_str/*out*/)
1340 {
1341  const odp_pool_capability_t *capa =
1342  &em_shm->mpool_tbl.odp_pool_capability;
1343  uint32_t min_cache_size;
1344  uint32_t cache_size;
1345 
1346  if (pool_cfg->event_type == EM_EVENT_TYPE_SW)
1347  min_cache_size = capa->buf.min_cache_size;
1348  else if (pool_cfg->event_type == EM_EVENT_TYPE_PACKET)
1349  min_cache_size = capa->pkt.min_cache_size;
1350  else if (pool_cfg->event_type == EM_EVENT_TYPE_VECTOR)
1351  min_cache_size = capa->vector.min_cache_size;
1352  else
1353  return -9;
1354 
1355  for (int i = 0; i < pool_cfg->num_subpools; i++) {
1356  if (pool_cfg->subpool[i].size <= 0 ||
1357  pool_cfg->subpool[i].num <= 0) {
1358  *err_str = "Invalid subpool size/num";
1359  return -(1 * 10 + i); /* -10, -11, ... */
1360  }
1361 
1362  cache_size = pool_cfg->subpool[i].cache_size;
1363  if (unlikely(cache_size < min_cache_size)) {
1364  *err_str = "Requested cache size too small";
1365  return -(2 * 10 + i); /* -20, -21, ... */
1366  }
1367  /*
1368  * If the given cache size is larger than odp-max,
1369  * then use odp-max:
1370  * if (cache_size > max_cache_size)
1371  * cache_size = max_cache_size;
1372  * This is done later in pool_create();
1373  */
1374  }
1375 
1376  return 0;
1377 }
1378 
1379 int invalid_pool_cfg(const em_pool_cfg_t *pool_cfg, const char **err_str/*out*/)
1380 {
1381  int ret = 0;
1382  const odp_pool_capability_t *capa = &em_shm->mpool_tbl.odp_pool_capability;
1383 
1384  if (!pool_cfg) {
1385  *err_str = "Pool config NULL";
1386  return -1;
1387  }
1388  if (pool_cfg->__internal_check != EM_CHECK_INIT_CALLED) {
1389  *err_str = "Pool config not initialized";
1390  return -1;
1391  }
1392 
1393  if (pool_cfg->num_subpools <= 0 ||
1394  pool_cfg->num_subpools > EM_MAX_SUBPOOLS) {
1395  *err_str = "Invalid number of subpools";
1396  return -1;
1397  }
1398 
1399  ret = is_pool_type_supported(pool_cfg->event_type, err_str/*out*/);
1400  if (ret)
1401  return ret;
1402 
1403  if (!is_align_offset_valid(pool_cfg)) {
1404  *err_str = "Invalid align offset";
1405  return -1;
1406  }
1407 
1408  ret = is_user_area_valid(pool_cfg, capa, err_str/*out*/);
1409  if (ret)
1410  return ret;
1411 
1412  if (pool_cfg->event_type == EM_EVENT_TYPE_PACKET &&
1413  pool_cfg->pkt.headroom.in_use &&
1414  pool_cfg->pkt.headroom.value > capa->pkt.max_headroom) {
1415  *err_str = "Requested pkt headroom size too large";
1416  return -1;
1417  }
1418 
1419  ret = invalid_pool_cache_cfg(pool_cfg, err_str/*out*/);
1420 
1421  return ret; /* 0: success, <0: error */
1422 }
1423 
1424 int check_pool_uarea_persistence(const em_pool_cfg_t *pool_cfg, const char **err_str/*out*/)
1425 {
1426 #if ODP_VERSION_API_NUM(1, 42, 0) <= ODP_VERSION_API
1427  bool has_uarea_persistence;
1428  const odp_pool_capability_t *capa = &em_shm->mpool_tbl.odp_pool_capability;
1429 
1430  switch (pool_cfg->event_type) {
1431  case EM_EVENT_TYPE_SW:
1432  has_uarea_persistence = capa->buf.uarea_persistence ? true : false;
1433  *err_str = "buf-pool (EM_EVENT_TYPE_SW)";
1434  break;
1435  case EM_EVENT_TYPE_PACKET:
1436  has_uarea_persistence = capa->pkt.uarea_persistence ? true : false;
1437  *err_str = "pkt-pool (EM_EVENT_TYPE_PACKET)";
1438  break;
1439  case EM_EVENT_TYPE_VECTOR:
1440  has_uarea_persistence = capa->vector.uarea_persistence ? true : false;
1441  *err_str = "vector-pool (EM_EVENT_TYPE_VECTOR)";
1442  break;
1443  default:
1444  has_uarea_persistence = false;
1445  *err_str = "unknown pool-type";
1446  break;
1447  }
1448 
1449  return has_uarea_persistence ? 0 : -1; /* 0: success, <0: not supported */
1450 #else
1451  return 0;
1452 #endif
1453 }
1454 
1455 /*
1456  * Helper to pool_create() - preallocate all events in the pool for ESV to
1457  * maintain event state over multiple alloc- and free-operations.
1458  */
1459 static void
1460 pool_prealloc(const mpool_elem_t *pool_elem)
1461 {
1462  event_prealloc_hdr_t *prealloc_hdr = NULL;
1463  uint64_t num_tot = 0;
1464  uint64_t num = 0;
1465  uint64_t num_free = 0;
1466  const uint32_t size = pool_elem->pool_cfg.subpool[0].size;
1467  list_node_t evlist;
1468  list_node_t *node;
1469 
1470  list_init(&evlist);
1471 
1472  for (int i = 0; i < pool_elem->num_subpools; i++)
1473  num_tot += pool_elem->pool_cfg.subpool[i].num;
1474 
1475  do {
1476  prealloc_hdr = event_prealloc(pool_elem, size);
1477  if (likely(prealloc_hdr)) {
1478  list_add(&evlist, &prealloc_hdr->list_node);
1479  num++;
1480  }
1481  } while (prealloc_hdr);
1482 
1483  if (unlikely(num < num_tot))
1484  INTERNAL_ERROR(EM_FATAL(EM_ERR_TOO_SMALL),
1485  EM_ESCOPE_POOL_CREATE,
1486  "alloc: events expected:%" PRIu64 " actual:%" PRIu64 "",
1487  num_tot, num);
1488 
1489  while (!list_is_empty(&evlist)) {
1490  node = list_rem_first(&evlist);
1491  prealloc_hdr = list_node_to_prealloc_hdr(node);
1492  em_free(prealloc_hdr->ev_hdr.event);
1493  num_free++;
1494  }
1495 
1496  if (unlikely(num_free > num))
1497  INTERNAL_ERROR(EM_FATAL(EM_ERR_TOO_LARGE),
1498  EM_ESCOPE_POOL_CREATE,
1499  "free: events expected:%" PRIu64 " actual:%" PRIu64 "",
1500  num, num_free);
1501 }
1502 
1503 /*
1504  * pool_create() helper: sort subpool cfg in ascending order based on buf size
1505  */
1506 static void
1507 sort_pool_cfg(const em_pool_cfg_t *pool_cfg, em_pool_cfg_t *sorted_cfg /*out*/)
1508 {
1509  const int num_subpools = pool_cfg->num_subpools;
1510 
1511  *sorted_cfg = *pool_cfg;
1512 
1513  for (int i = 0; i < num_subpools - 1; i++) {
1514  int idx = i; /* array index containing smallest size */
1515 
1516  for (int j = i + 1; j < num_subpools; j++) {
1517  if (sorted_cfg->subpool[j].size <
1518  sorted_cfg->subpool[idx].size)
1519  idx = j; /* store idx to smallest */
1520  }
1521 
1522  /* min size at [idx], swap with [i] */
1523  if (idx != i) {
1524  uint32_t size = sorted_cfg->subpool[i].size;
1525  uint32_t num = sorted_cfg->subpool[i].num;
1526  uint32_t cache_size = sorted_cfg->subpool[i].cache_size;
1527 
1528  sorted_cfg->subpool[i] = sorted_cfg->subpool[idx];
1529 
1530  sorted_cfg->subpool[idx].size = size;
1531  sorted_cfg->subpool[idx].num = num;
1532  sorted_cfg->subpool[idx].cache_size = cache_size;
1533  }
1534  }
1535 }
1536 
1537 /*
1538  * pool_create() helper: set pool event-cache size.
1539  *
1540  * Set the requested subpool cache-size based on user provided value and
1541  * limit set by odp-pool-capability.
1542  * Requested value can be larger than odp-max, use odp--max in this
1543  * case.
1544  * Verification against odp-min value done in invalid_pool_cfg().
1545  */
1546 static void
1547 set_poolcache_size(em_pool_cfg_t *pool_cfg)
1548 {
1549  const odp_pool_capability_t *capa =
1550  &em_shm->mpool_tbl.odp_pool_capability;
1551  int num_subpools = pool_cfg->num_subpools;
1552  uint32_t max_cache_size;
1553 
1554  if (pool_cfg->event_type == EM_EVENT_TYPE_SW)
1555  max_cache_size = capa->buf.max_cache_size;
1556  else if (pool_cfg->event_type == EM_EVENT_TYPE_PACKET)
1557  max_cache_size = capa->pkt.max_cache_size;
1558  else /* EM_EVENT_TYPE_VECTOR */
1559  max_cache_size = capa->vector.max_cache_size;
1560 
1561  for (int i = 0; i < num_subpools; i++) {
1562  if (max_cache_size < pool_cfg->subpool[i].cache_size)
1563  pool_cfg->subpool[i].cache_size = max_cache_size;
1564  }
1565 }
1566 
1567 /*
1568  * pool_create() helper: determine payload alignment.
1569  */
1570 static int
1571 set_align(const em_pool_cfg_t *pool_cfg,
1572  uint32_t *align_offset /*out*/, uint32_t *odp_align /*out*/)
1573 {
1574  const odp_pool_capability_t *capa =
1575  &em_shm->mpool_tbl.odp_pool_capability;
1576  uint32_t offset = 0;
1577  uint32_t align = ODP_CACHE_LINE_SIZE;
1578 
1579  /* Pool-specific param overrides config file 'align_offset' value */
1580  if (pool_cfg->align_offset.in_use)
1581  offset = pool_cfg->align_offset.value; /* pool cfg */
1582  else
1583  offset = em_shm->opt.pool.align_offset; /* cfg file */
1584 
1585  /* Set subpool minimum alignment */
1586  if (pool_cfg->event_type == EM_EVENT_TYPE_PACKET) {
1587  if (align > capa->pkt.max_align)
1588  align = capa->pkt.max_align;
1589  } else {
1590  if (align > capa->buf.max_align)
1591  align = capa->buf.max_align;
1592  }
1593 
1594  *align_offset = offset;
1595  *odp_align = align;
1596 
1597  /* verify alignment requirements */
1598  if (!POWEROF2(align) || align <= offset)
1599  return -1;
1600 
1601  return 0;
1602 }
1603 
1604 /*
1605  * pool_create() helper: determine user area size.
1606  */
1607 static int
1608 set_uarea_size(const em_pool_cfg_t *pool_cfg, size_t *uarea_size/*out*/)
1609 {
1610  size_t size = 0;
1611  size_t max_size = 0;
1612  const odp_pool_capability_t *capa =
1613  &em_shm->mpool_tbl.odp_pool_capability;
1614 
1615  if (pool_cfg->user_area.in_use) /* use pool-cfg */
1616  size = pool_cfg->user_area.size;
1617  else /* use cfg-file */
1618  size = em_shm->opt.pool.user_area_size;
1619 
1620  if (pool_cfg->event_type == EM_EVENT_TYPE_PACKET)
1621  max_size = MIN(capa->pkt.max_uarea_size, EM_EVENT_USER_AREA_MAX_SIZE);
1622  else if (pool_cfg->event_type == EM_EVENT_TYPE_VECTOR)
1623  max_size = MIN(capa->vector.max_uarea_size, EM_EVENT_USER_AREA_MAX_SIZE);
1624  else if (size > 0) /* EM_EVENT_TYPE_SW: bufs */
1625  max_size = MIN(capa->buf.max_uarea_size, EM_EVENT_USER_AREA_MAX_SIZE);
1626 
1627  if (size > max_size)
1628  return -1;
1629 
1630  *uarea_size = size;
1631 
1632  return 0;
1633 }
1634 
1635 /*
1636  * pool_create() helper: set the pkt headroom
1637  */
1638 static int
1639 set_pkt_headroom(const em_pool_cfg_t *pool_cfg,
1640  uint32_t *pkt_headroom /*out*/,
1641  uint32_t *max_headroom /*out, for err print only*/)
1642 {
1643  const odp_pool_capability_t *capa =
1644  &em_shm->mpool_tbl.odp_pool_capability;
1645  /* default value from cfg file */
1646  uint32_t headroom = em_shm->opt.pool.pkt_headroom;
1647 
1648  /* Pool-specific param overrides config file value */
1649  if (pool_cfg->pkt.headroom.in_use)
1650  headroom = pool_cfg->pkt.headroom.value;
1651 
1652  *pkt_headroom = headroom;
1653  *max_headroom = capa->pkt.max_headroom;
1654 
1655  if (unlikely(headroom > capa->pkt.max_headroom))
1656  return -1;
1657 
1658  return 0;
1659 }
1660 
1661 /** Helper to create_subpools() */
1662 static void set_pool_params_stats(odp_pool_stats_opt_t *param_stats /*out*/,
1663  const odp_pool_stats_opt_t *capa_stats,
1664  const em_pool_stats_opt_t *stats_opt)
1665 {
1666  param_stats->all = 0;
1667 
1668  if (capa_stats->bit.available)
1669  param_stats->bit.available = stats_opt->available;
1670 
1671  if (capa_stats->bit.alloc_ops)
1672  param_stats->bit.alloc_ops = stats_opt->alloc_ops;
1673 
1674  if (capa_stats->bit.alloc_fails)
1675  param_stats->bit.alloc_fails = stats_opt->alloc_fails;
1676 
1677  if (capa_stats->bit.free_ops)
1678  param_stats->bit.free_ops = stats_opt->free_ops;
1679 
1680  if (capa_stats->bit.total_ops)
1681  param_stats->bit.total_ops = stats_opt->total_ops;
1682 
1683  if (capa_stats->bit.cache_alloc_ops)
1684  param_stats->bit.cache_alloc_ops = stats_opt->cache_alloc_ops;
1685 
1686  if (capa_stats->bit.cache_available)
1687  param_stats->bit.cache_available = stats_opt->cache_available;
1688 
1689  if (capa_stats->bit.cache_free_ops)
1690  param_stats->bit.cache_free_ops = stats_opt->cache_free_ops;
1691 
1692  if (capa_stats->bit.thread_cache_available)
1693  param_stats->bit.thread_cache_available = stats_opt->core_cache_available;
1694 }
1695 
1696 /** Helper to create_subpools() */
1697 static void set_pool_params_pkt(odp_pool_param_t *pool_params /* out */,
1698  const em_pool_cfg_t *pool_cfg,
1699  uint32_t size, uint32_t num, uint32_t cache_size,
1700  uint32_t align_offset, uint32_t odp_align,
1701  uint32_t uarea_size, uint32_t pkt_headroom)
1702 {
1703  const odp_pool_capability_t *capa = &em_shm->mpool_tbl.odp_pool_capability;
1704 
1705  odp_pool_param_init(pool_params);
1706 
1707  pool_params->type = ODP_POOL_PACKET;
1708  /* num == max_num, helps pool-info stats calculation */
1709  pool_params->pkt.num = num;
1710  pool_params->pkt.max_num = num;
1711 
1712  if (size > align_offset)
1713  size = size - align_offset;
1714  else
1715  size = 1; /* 0:default, can be big => use 1 */
1716  /* len == max_len */
1717  pool_params->pkt.len = size;
1718  pool_params->pkt.max_len = size;
1719  pool_params->pkt.seg_len = size;
1720  pool_params->pkt.align = odp_align;
1721  /*
1722  * Reserve space for the event header in each packet's
1723  * ODP-user-area:
1724  */
1725  pool_params->pkt.uarea_size = sizeof(event_hdr_t) + uarea_size;
1726  /*
1727  * Set the pkt headroom.
1728  * Make sure the alloc-alignment fits into the headroom.
1729  */
1730  pool_params->pkt.headroom = pkt_headroom;
1731  if (pkt_headroom < align_offset)
1732  pool_params->pkt.headroom = align_offset;
1733 
1734  pool_params->pkt.cache_size = cache_size;
1735 
1736  /* Pkt pool statistics */
1737  if (pool_cfg->stats_opt.in_use) {
1738  set_pool_params_stats(&pool_params->stats, &capa->pkt.stats,
1739  &pool_cfg->stats_opt.opt);
1740  } else {
1741  set_pool_params_stats(&pool_params->stats, &capa->pkt.stats,
1742  &em_shm->opt.pool.statistics);/*from cnf file*/
1743  }
1744 }
1745 
1746 static void set_pool_params_vector(odp_pool_param_t *pool_params /* out */,
1747  const em_pool_cfg_t *pool_cfg,
1748  uint32_t size, uint32_t num,
1749  uint32_t cache_size, uint32_t uarea_size)
1750 {
1751  const odp_pool_capability_t *capa = &em_shm->mpool_tbl.odp_pool_capability;
1752 
1753  odp_pool_param_init(pool_params);
1754 
1755  pool_params->type = ODP_POOL_VECTOR;
1756  pool_params->vector.num = num;
1757  pool_params->vector.max_size = size;
1758  /* Reserve space for the EM event header in the vector's ODP-user-area */
1759  pool_params->vector.uarea_size = sizeof(event_hdr_t) + uarea_size;
1760  pool_params->vector.cache_size = cache_size;
1761 
1762  /* Vector pool statistics */
1763  if (pool_cfg->stats_opt.in_use)
1764  set_pool_params_stats(&pool_params->stats, &capa->vector.stats,
1765  &pool_cfg->stats_opt.opt);
1766  else
1767  set_pool_params_stats(&pool_params->stats, &capa->vector.stats,
1768  &em_shm->opt.pool.statistics);
1769 }
1770 
1771 /** Helper to create_subpools() */
1772 static void set_pool_params_buf(odp_pool_param_t *pool_params /* out */,
1773  const em_pool_cfg_t *pool_cfg,
1774  uint32_t size, uint32_t num, uint32_t cache_size,
1775  uint32_t align_offset, uint32_t odp_align,
1776  uint32_t uarea_size)
1777 {
1778  const odp_pool_capability_t *capa = &em_shm->mpool_tbl.odp_pool_capability;
1779 
1780  odp_pool_param_init(pool_params);
1781 
1782  pool_params->type = ODP_POOL_BUFFER;
1783  pool_params->buf.num = num;
1784  pool_params->buf.size = size;
1785  if (align_offset)
1786  pool_params->buf.size += 32 - align_offset;
1787  pool_params->buf.align = odp_align;
1788  pool_params->buf.uarea_size = sizeof(event_hdr_t) + uarea_size;
1789  pool_params->buf.cache_size = cache_size;
1790 
1791  /* Buf pool statistics */
1792  if (pool_cfg->stats_opt.in_use)
1793  set_pool_params_stats(&pool_params->stats, &capa->buf.stats,
1794  &pool_cfg->stats_opt.opt);
1795  else
1796  set_pool_params_stats(&pool_params->stats, &capa->buf.stats,
1797  &em_shm->opt.pool.statistics);
1798 }
1799 
1800 static int
1801 create_subpools(const em_pool_cfg_t *pool_cfg,
1802  uint32_t align_offset, uint32_t odp_align,
1803  uint32_t uarea_size, uint32_t pkt_headroom,
1804  mpool_elem_t *mpool_elem /*out*/)
1805 {
1806  const int num_subpools = pool_cfg->num_subpools;
1807  mpool_tbl_t *const mpool_tbl = &em_shm->mpool_tbl;
1808 
1809  for (int i = 0; i < num_subpools; i++) {
1810  char pool_name[ODP_POOL_NAME_LEN];
1811  odp_pool_param_t pool_params;
1812  uint32_t size = pool_cfg->subpool[i].size;
1813  uint32_t num = pool_cfg->subpool[i].num;
1814  uint32_t cache_size = pool_cfg->subpool[i].cache_size;
1815 
1816  if (pool_cfg->event_type == EM_EVENT_TYPE_PACKET) {
1817  set_pool_params_pkt(&pool_params /* out */, pool_cfg,
1818  size, num, cache_size,
1819  align_offset, odp_align,
1820  uarea_size, pkt_headroom);
1821  } else if (pool_cfg->event_type == EM_EVENT_TYPE_VECTOR) {
1822  set_pool_params_vector(&pool_params /* out */, pool_cfg,
1823  size, num, cache_size,
1824  uarea_size);
1825  } else { /* pool_cfg->event_type == EM_EVENT_TYPE_SW */
1826  set_pool_params_buf(&pool_params /* out */, pool_cfg,
1827  size, num, cache_size,
1828  align_offset, odp_align, uarea_size);
1829  }
1830 
1831  mpool_elem->size[i] = pool_cfg->subpool[i].size;
1832  mpool_elem->stats_opt = pool_params.stats;
1833 
1834  snprintf(pool_name, sizeof(pool_name), "%" PRI_POOL ":%d-%s",
1835  mpool_elem->em_pool, i, mpool_elem->name);
1836  pool_name[sizeof(pool_name) - 1] = '\0';
1837 
1838  odp_pool_t odp_pool = odp_pool_create(pool_name, &pool_params);
1839 
1840  if (unlikely(odp_pool == ODP_POOL_INVALID))
1841  return -1;
1842 
1843  mpool_elem->odp_pool[i] = odp_pool;
1844  mpool_elem->num_subpools++; /* created subpools for delete */
1845 
1846  int odp_pool_idx = odp_pool_index(odp_pool);
1847 
1848  if (unlikely(odp_pool_idx < 0))
1849  return -2;
1850 
1851  /* Store mapping from odp-pool (idx) to em-pool & subpool */
1852  mpool_tbl->pool_subpool_odp2em[odp_pool_idx].pool =
1853  (uint32_t)(uintptr_t)mpool_elem->em_pool;
1854  mpool_tbl->pool_subpool_odp2em[odp_pool_idx].subpool = i;
1855 
1856  /* odp_pool_print(odp_pool); */
1857  }
1858 
1859  return 0;
1860 }
1861 
1862 em_pool_t
1863 pool_create(const char *name, em_pool_t req_pool, const em_pool_cfg_t *pool_cfg)
1864 {
1865  const em_event_type_t pool_evtype = pool_cfg->event_type;
1866  int err = 0;
1867 
1868  /* Allocate a free EM pool */
1869  const em_pool_t pool = pool_alloc(req_pool/* requested or undef*/);
1870 
1871  if (unlikely(pool == EM_POOL_UNDEF))
1872  return EM_POOL_UNDEF;
1873 
1874  mpool_elem_t *mpool_elem = pool_elem_get(pool);
1875 
1876  /* Sanity check */
1877  if (!mpool_elem || mpool_elem->em_pool != pool)
1878  return EM_POOL_UNDEF;
1879 
1880  mpool_elem->event_type = pool_evtype;
1881  /* Store successfully created subpools later */
1882  mpool_elem->num_subpools = 0;
1883  /* Store the event pool name, if given */
1884  if (name && *name) {
1885  strncpy(mpool_elem->name, name, sizeof(mpool_elem->name));
1886  mpool_elem->name[sizeof(mpool_elem->name) - 1] = '\0';
1887  } else {
1888  mpool_elem->name[0] = '\0';
1889  }
1890 
1891  em_pool_cfg_t sorted_cfg;
1892 
1893  /*
1894  * Sort the subpool cfg in ascending order based on the buffer size
1895  */
1896  sort_pool_cfg(pool_cfg, &sorted_cfg/*out*/);
1897  /* Use sorted_cfg instead of pool_cfg from here on */
1898 
1899  /*
1900  * Set the cache-size of each subpool in the EM-pool
1901  */
1902  set_poolcache_size(&sorted_cfg);
1903 
1904  /* Store the sorted config */
1905  mpool_elem->pool_cfg = sorted_cfg;
1906 
1907  /*
1908  * Event payload alignment requirement for the pool
1909  */
1910  uint32_t align_offset = 0;
1911  uint32_t odp_align = 0;
1912 
1913  /* align only valid for bufs and pkts */
1914  if (pool_evtype == EM_EVENT_TYPE_SW ||
1915  pool_evtype == EM_EVENT_TYPE_PACKET) {
1916  err = set_align(&sorted_cfg, &align_offset/*out*/,
1917  &odp_align/*out*/);
1918  if (unlikely(err)) {
1919  INTERNAL_ERROR(EM_ERR_TOO_LARGE, EM_ESCOPE_POOL_CREATE,
1920  "EM-pool:\"%s\" align mismatch:\n"
1921  "align:%u cfg:align_offset:%u",
1922  name, odp_align, align_offset);
1923  goto error;
1924  }
1925  }
1926  /* store the align offset, needed in pkt-alloc */
1927  mpool_elem->align_offset = align_offset;
1928 
1929  /*
1930  * Event user area size.
1931  * Pool-specific param overrides config file 'user_area_size' value
1932  */
1933  size_t uarea_size = 0;
1934 
1935  err = set_uarea_size(&sorted_cfg, &uarea_size/*out*/);
1936  if (unlikely(err)) {
1937  INTERNAL_ERROR(EM_ERR_TOO_LARGE, EM_ESCOPE_POOL_CREATE,
1938  "EM-pool:\"%s\" invalid uarea config: req.size:%zu",
1939  name, uarea_size);
1940  goto error;
1941  }
1942 
1943  /* store the user_area sizes, needed in alloc */
1944  mpool_elem->user_area.size = uarea_size & UINT16_MAX;
1945 
1946  /*
1947  * Set the headroom for events in EM packet pools
1948  */
1949  uint32_t pkt_headroom = 0;
1950  uint32_t max_headroom = 0;
1951 
1952  if (pool_evtype == EM_EVENT_TYPE_PACKET) {
1953  err = set_pkt_headroom(&sorted_cfg, &pkt_headroom/*out*/,
1954  &max_headroom/*out*/);
1955  if (unlikely(err)) {
1956  INTERNAL_ERROR(EM_ERR_TOO_LARGE, EM_ESCOPE_POOL_CREATE,
1957  "EM-pool:\"%s\" invalid pkt headroom:\n"
1958  "headroom:%u vs. max:headroom:%u",
1959  name, pkt_headroom, max_headroom);
1960  goto error;
1961  }
1962  }
1963 
1964  /*
1965  * Create the subpools for the EM event-pool.
1966  * Each EM subpool is an ODP pool.
1967  */
1968  err = create_subpools(&sorted_cfg, align_offset, odp_align,
1969  (uint32_t)uarea_size, pkt_headroom, mpool_elem /*out*/);
1970  if (unlikely(err)) {
1971  INTERNAL_ERROR(EM_FATAL(EM_ERR_ALLOC_FAILED),
1972  EM_ESCOPE_POOL_CREATE,
1973  "EM-pool:\"%s\" create fails:%d\n"
1974  "subpools req:%d vs. subpools created:%d",
1975  name, err, sorted_cfg.num_subpools,
1976  mpool_elem->num_subpools);
1977  goto error;
1978  }
1979 
1980  /*
1981  * ESV: preallocate all events in the pool
1982  */
1983  if (esv_enabled() && em_shm->opt.esv.prealloc_pools)
1984  pool_prealloc(mpool_elem);
1985 
1986  /* Success! */
1987  return mpool_elem->em_pool;
1988 
1989 error:
1990  (void)pool_delete(pool);
1991  return EM_POOL_UNDEF;
1992 }
1993 
1994 em_status_t
1995 pool_delete(em_pool_t pool)
1996 {
1997  mpool_tbl_t *const mpool_tbl = &em_shm->mpool_tbl;
1998  mpool_elem_t *const mpool_elem = pool_elem_get(pool);
1999 
2000  if (unlikely(mpool_elem == NULL || !pool_allocated(mpool_elem)))
2001  return EM_ERR_BAD_ARG;
2002 
2003  for (int i = 0; i < mpool_elem->num_subpools; i++) {
2004  odp_pool_t odp_pool = mpool_elem->odp_pool[i];
2005  int odp_pool_idx;
2006  int ret;
2007 
2008  if (odp_pool == ODP_POOL_INVALID)
2009  return EM_ERR_NOT_FOUND;
2010 
2011  odp_pool_idx = odp_pool_index(odp_pool);
2012 
2013  ret = odp_pool_destroy(odp_pool);
2014  if (unlikely(ret))
2015  return EM_ERR_LIB_FAILED;
2016 
2017  mpool_elem->odp_pool[i] = ODP_POOL_INVALID;
2018  mpool_elem->size[i] = 0;
2019 
2020  /* Clear mapping from odp-pool (idx) to em-pool & subpool */
2021  if (unlikely(odp_pool_idx < 0))
2022  return EM_ERR_BAD_ID;
2023  mpool_tbl->pool_subpool_odp2em[odp_pool_idx].both = pool_subpool_undef.both;
2024  }
2025 
2026  mpool_elem->name[0] = '\0';
2027  mpool_elem->event_type = EM_EVENT_TYPE_UNDEF;
2028  mpool_elem->num_subpools = 0;
2029 
2030  return pool_free(pool);
2031 }
2032 
2033 em_pool_t
2034 pool_find(const char *name)
2035 {
2036  if (name && *name) {
2037  for (int i = 0; i < EM_CONFIG_POOLS; i++) {
2038  const mpool_elem_t *mpool_elem =
2039  &em_shm->mpool_tbl.pool[i];
2040 
2041  if (pool_allocated(mpool_elem) &&
2042  !strncmp(name, mpool_elem->name, EM_POOL_NAME_LEN))
2043  return mpool_elem->em_pool;
2044  }
2045  }
2046 
2047  return EM_POOL_UNDEF;
2048 }
2049 
2050 unsigned int
2051 pool_count(void)
2052 {
2053  return env_atomic32_get(&em_shm->pool_count);
2054 }
2055 
2056 #define POOL_INFO_HDR_STR \
2057 " id name type offset uarea sizes [size count(used/free) cache]\n"
2058 
2059 #define POOL_INFO_SUBSTR_FMT \
2060 "%d:[sz=%" PRIu32 " n=%" PRIu32 "(%" PRIu32 "/%" PRIu32 ") $=%" PRIu32 "]"
2061 
2062 #define POOL_INFO_SUBSTR_NO_STATS_FMT \
2063 "%d:[sz=%" PRIu32 " n=%" PRIu32 "(-/-) cache=%" PRIu32 "]"
2064 
2065 void pool_info_print_hdr(unsigned int num_pools)
2066 {
2067  if (num_pools == 1) {
2068  EM_PRINT("EM Event Pool\n"
2069  "-------------\n"
2070  POOL_INFO_HDR_STR);
2071  } else {
2072  EM_PRINT("EM Event Pools:%2u\n"
2073  "-----------------\n"
2074  POOL_INFO_HDR_STR, num_pools);
2075  }
2076 }
2077 
2078 void pool_info_print(em_pool_t pool)
2079 {
2080  em_pool_info_t pool_info;
2081  em_status_t stat;
2082  const char *pool_type;
2083 
2084  stat = em_pool_info(pool, &pool_info/*out*/);
2085  if (unlikely(stat != EM_OK)) {
2086  EM_PRINT(" %-6" PRI_POOL " %-16s n/a n/a n/a n/a [n/a]\n",
2087  pool, "err:n/a");
2088  return;
2089  }
2090 
2091  if (pool_info.event_type == EM_EVENT_TYPE_VECTOR)
2092  pool_type = "vec";
2093  else if (pool_info.event_type == EM_EVENT_TYPE_PACKET)
2094  pool_type = "pkt";
2095  else
2096  pool_type = "buf";
2097 
2098  EM_PRINT(" %-6" PRI_POOL " %-16s %4s %02u %02zu %02u ",
2099  pool, pool_info.name, pool_type,
2100  pool_info.align_offset, pool_info.user_area_size,
2101  pool_info.num_subpools);
2102 
2103  for (int i = 0; i < pool_info.num_subpools; i++) {
2104  char subpool_str[42];
2105 
2106  if (pool_info.subpool[i].used || pool_info.subpool[i].free) {
2107  snprintf(subpool_str, sizeof(subpool_str),
2108  POOL_INFO_SUBSTR_FMT, i,
2109  pool_info.subpool[i].size,
2110  pool_info.subpool[i].num,
2111  pool_info.subpool[i].used,
2112  pool_info.subpool[i].free,
2113  pool_info.subpool[i].cache_size);
2114  } else {
2115  snprintf(subpool_str, sizeof(subpool_str),
2116  POOL_INFO_SUBSTR_NO_STATS_FMT, i,
2117  pool_info.subpool[i].size,
2118  pool_info.subpool[i].num,
2119  pool_info.subpool[i].cache_size);
2120  }
2121  subpool_str[sizeof(subpool_str) - 1] = '\0';
2122  EM_PRINT(" %-42s", subpool_str);
2123  }
2124 
2125  EM_PRINT("\n");
2126 }
2127 
2128 #define POOL_STATS_HDR_STR \
2129 "EM pool statistics for pool %" PRI_POOL ":\n\n"\
2130 "Subpool Available Alloc_ops Alloc_fails Free_ops Total_ops Cache_available" \
2131 " Cache_alloc_ops Cache_free_ops\n"\
2132 "--------------------------------------------------------------------------" \
2133 "-------------------------------\n%s"
2134 
2135 #define POOL_STATS_LEN 107
2136 #define POOL_STATS_FMT "%-8u%-10lu%-10lu%-12lu%-9lu%-10lu%-16lu%-16lu%-15lu\n"
2137 
2138 void pool_stats_print(em_pool_t pool)
2139 {
2140  em_status_t stat;
2141  em_pool_stats_t pool_stats;
2142  const em_pool_subpool_stats_t *subpool_stats;
2143  int len = 0;
2144  int n_print = 0;
2145  const mpool_elem_t *pool_elem = pool_elem_get(pool);
2146  const int stats_str_len = EM_MAX_SUBPOOLS * POOL_STATS_LEN + 1;
2147  char stats_str[stats_str_len];
2148 
2149  if (pool_elem == NULL || !pool_allocated(pool_elem)) {
2150  EM_LOG(EM_LOG_ERR, "EM-pool:%" PRI_POOL " invalid\n", pool);
2151  return;
2152  }
2153 
2154  stat = em_pool_stats(pool, &pool_stats);
2155  if (unlikely(stat != EM_OK)) {
2156  EM_PRINT("Failed to fetch EM pool statistics\n");
2157  return;
2158  }
2159 
2160  for (uint32_t i = 0; i < pool_stats.num_subpools; i++) {
2161  subpool_stats = &pool_stats.subpool_stats[i];
2162  n_print = snprintf(stats_str + len, stats_str_len - len,
2163  POOL_STATS_FMT,
2164  i, subpool_stats->available,
2165  subpool_stats->alloc_ops,
2166  subpool_stats->alloc_fails,
2167  subpool_stats->free_ops,
2168  subpool_stats->total_ops,
2169  subpool_stats->cache_available,
2170  subpool_stats->cache_alloc_ops,
2171  subpool_stats->cache_free_ops);
2172 
2173  /* Not enough space to hold more subpool stats */
2174  if (n_print >= stats_str_len - len)
2175  break;
2176 
2177  len += n_print;
2178  }
2179 
2180  stats_str[len] = '\0';
2181  EM_PRINT(POOL_STATS_HDR_STR, pool, stats_str);
2182 }
2183 
2184 #define POOL_STATS_SELECTED_HDR_STR \
2185 "Selected EM pool statistics for pool %" PRI_POOL ":\n\n"\
2186 "Selected statistic counters: %s\n\n"\
2187 "Subpool Available Alloc_ops Alloc_fails Free_ops Total_ops Cache_available" \
2188 " Cache_alloc_ops Cache_free_ops\n"\
2189 "--------------------------------------------------------------------------" \
2190 "-------------------------------\n%s"
2191 
2192 #define OPT_STR_LEN 150
2193 
2194 static void fill_opt_str(char *opt_str, const em_pool_stats_opt_t *opt)
2195 {
2196  int n_print;
2197  int len = 0;
2198 
2199  if (opt->available) {
2200  n_print = snprintf(opt_str + len, 12, "%s", "available");
2201  len += n_print;
2202  }
2203 
2204  if (opt->alloc_ops) {
2205  n_print = snprintf(opt_str + len, 12, "%s", len ? ", alloc_ops" : "alloc_ops");
2206  len += n_print;
2207  }
2208 
2209  if (opt->alloc_fails) {
2210  n_print = snprintf(opt_str + len, 14, "%s", len ? ", alloc_fails" : "alloc_fails");
2211  len += n_print;
2212  }
2213 
2214  if (opt->free_ops) {
2215  n_print = snprintf(opt_str + len, 11, "%s", len ? ", free_ops" : "free_ops");
2216  len += n_print;
2217  }
2218 
2219  if (opt->total_ops) {
2220  n_print = snprintf(opt_str + len, 12, "%s", len ? ", total_ops" : "total_ops");
2221  len += n_print;
2222  }
2223 
2224  if (opt->cache_available) {
2225  n_print = snprintf(opt_str + len, 18, "%s",
2226  len ? ", cache_available" : "cache_available");
2227  len += n_print;
2228  }
2229 
2230  if (opt->cache_alloc_ops) {
2231  n_print = snprintf(opt_str + len, 18, "%s",
2232  len ? ", cache_alloc_ops" : "cache_alloc_ops");
2233  len += n_print;
2234  }
2235 
2236  if (opt->cache_free_ops)
2237  snprintf(opt_str + len, 17, "%s", len ? ", cache_free_ops" : "cache_free_ops");
2238 }
2239 
2240 void pool_stats_selected_print(em_pool_t pool, const em_pool_stats_opt_t *opt)
2241 {
2242  em_status_t stat;
2243  em_pool_stats_selected_t pool_stats = {0};
2244  const em_pool_subpool_stats_selected_t *subpool_stats;
2245  int len = 0;
2246  int n_print = 0;
2247  const mpool_elem_t *pool_elem = pool_elem_get(pool);
2248  char opt_str[OPT_STR_LEN];
2249  const int stats_str_len = EM_MAX_SUBPOOLS * POOL_STATS_LEN + 1;
2250  char stats_str[stats_str_len];
2251 
2252  if (pool_elem == NULL || !pool_allocated(pool_elem)) {
2253  EM_LOG(EM_LOG_ERR, "EM-pool:%" PRI_POOL " invalid\n", pool);
2254  return;
2255  }
2256 
2257  stat = em_pool_stats_selected(pool, &pool_stats, opt);
2258  if (unlikely(stat != EM_OK)) {
2259  EM_PRINT("Failed to fetch EM selected pool statistics\n");
2260  return;
2261  }
2262 
2263  for (uint32_t i = 0; i < pool_stats.num_subpools; i++) {
2264  subpool_stats = &pool_stats.subpool_stats[i];
2265 
2266  n_print = snprintf(stats_str + len, stats_str_len - len,
2267  POOL_STATS_FMT,
2268  i,
2269  subpool_stats->available,
2270  subpool_stats->alloc_ops,
2271  subpool_stats->alloc_fails,
2272  subpool_stats->free_ops,
2273  subpool_stats->total_ops,
2274  subpool_stats->cache_available,
2275  subpool_stats->cache_alloc_ops,
2276  subpool_stats->cache_free_ops);
2277 
2278  /* Not enough space to hold more subpool stats */
2279  if (n_print >= stats_str_len - len)
2280  break;
2281 
2282  len += n_print;
2283  }
2284  stats_str[len] = '\0';
2285 
2286  /* Fill selected statistic counters */
2287  fill_opt_str(opt_str, opt);
2288 
2289  EM_PRINT(POOL_STATS_SELECTED_HDR_STR, pool, opt_str, stats_str);
2290 }
2291 
2292 #define SUBPOOL_STATS_HDR_STR \
2293 "EM subpool statistics for pool %" PRI_POOL ":\n\n"\
2294 "Subpool Available Alloc_ops Alloc_fails Free_ops Total_ops Cache_available" \
2295 " Cache_alloc_ops Cache_free_ops\n"\
2296 "--------------------------------------------------------------------------" \
2297 "-------------------------------\n%s"
2298 
2299 void subpools_stats_print(em_pool_t pool, const int subpools[], int num_subpools)
2300 {
2301  int num_stats;
2302  em_pool_subpool_stats_t stats[num_subpools];
2303  int len = 0;
2304  int n_print = 0;
2305  const mpool_elem_t *pool_elem = pool_elem_get(pool);
2306  const int stats_str_len = num_subpools * POOL_STATS_LEN + 1;
2307  char stats_str[stats_str_len];
2308 
2309  if (pool_elem == NULL || !pool_allocated(pool_elem)) {
2310  EM_LOG(EM_LOG_ERR, "EM-pool:%" PRI_POOL " invalid\n", pool);
2311  return;
2312  }
2313 
2314  num_stats = em_pool_subpool_stats(pool, subpools, num_subpools, stats);
2315  if (unlikely(!num_stats || num_stats > num_subpools)) {
2316  EM_LOG(EM_LOG_ERR, "Failed to fetch subpool statistics\n");
2317  return;
2318  }
2319 
2320  /* Print subpool stats */
2321  for (int i = 0; i < num_stats; i++) {
2322  n_print = snprintf(stats_str + len, stats_str_len - len,
2323  POOL_STATS_FMT,
2324  subpools[i], stats[i].available, stats[i].alloc_ops,
2325  stats[i].alloc_fails, stats[i].free_ops,
2326  stats[i].total_ops, stats[i].cache_available,
2327  stats[i].cache_alloc_ops, stats[i].cache_free_ops);
2328 
2329  /* Not enough space to hold more subpool stats */
2330  if (n_print >= stats_str_len - len)
2331  break;
2332 
2333  len += n_print;
2334  }
2335 
2336  stats_str[len] = '\0';
2337  EM_PRINT(SUBPOOL_STATS_HDR_STR, pool, stats_str);
2338 }
2339 
2340 #define SUBPOOL_STATS_SELECTED_HDR_STR \
2341 "Selected EM subpool statistics for pool %" PRI_POOL ":\n\n"\
2342 "Selected statistic counters: %s\n\n"\
2343 "Subpool Available Alloc_ops Alloc_fails Free_ops Total_ops Cache_available" \
2344 " Cache_alloc_ops Cache_free_ops\n"\
2345 "--------------------------------------------------------------------------" \
2346 "-------------------------------\n%s"
2347 
2348 void subpools_stats_selected_print(em_pool_t pool, const int subpools[],
2349  int num_subpools, const em_pool_stats_opt_t *opt)
2350 {
2351  int num_stats;
2352  char opt_str[OPT_STR_LEN];
2353  em_pool_subpool_stats_selected_t stats[num_subpools];
2354  int len = 0;
2355  int n_print = 0;
2356  const mpool_elem_t *pool_elem = pool_elem_get(pool);
2357  const int stats_str_len = num_subpools * POOL_STATS_LEN + 1;
2358  char stats_str[stats_str_len];
2359 
2360  if (pool_elem == NULL || !pool_allocated(pool_elem)) {
2361  EM_LOG(EM_LOG_ERR, "EM-pool:%" PRI_POOL " invalid\n", pool);
2362  return;
2363  }
2364 
2365  memset(stats, 0, sizeof(stats));
2366  num_stats = em_pool_subpool_stats_selected(pool, subpools, num_subpools, stats, opt);
2367  if (unlikely(!num_stats || num_stats > num_subpools)) {
2368  EM_LOG(EM_LOG_ERR, "Failed to fetch selected subpool statistics\n");
2369  return;
2370  }
2371 
2372  /* Print subpool stats */
2373  for (int i = 0; i < num_stats; i++) {
2374  n_print = snprintf(stats_str + len, stats_str_len - len,
2375  POOL_STATS_FMT,
2376  subpools[i], stats[i].available, stats[i].alloc_ops,
2377  stats[i].alloc_fails, stats[i].free_ops,
2378  stats[i].total_ops, stats[i].cache_available,
2379  stats[i].cache_alloc_ops, stats[i].cache_free_ops);
2380 
2381  /* Not enough space to hold more subpool stats */
2382  if (n_print >= stats_str_len - len)
2383  break;
2384 
2385  len += n_print;
2386  }
2387  stats_str[len] = '\0';
2388 
2389  /* Fill selected statistic counters */
2390  fill_opt_str(opt_str, opt);
2391  EM_PRINT(SUBPOOL_STATS_SELECTED_HDR_STR, pool, opt_str, stats_str);
2392 }
2393 
2394 void print_pool_elem_info(void)
2395 {
2396  EM_PRINT("\n"
2397  "pool-elem size: %zu B\n",
2398  sizeof(mpool_elem_t));
2399 
2400  EM_DBG("\t\toffset\tsize\n"
2401  "\t\t------\t-----\n"
2402  "event_type:\t%3zu B\t%3zu B\n"
2403  "align_offset:\t%3zu B\t%3zu B\n"
2404  "user_area info:\t%3zu B\t%3zu B\n"
2405  "num_subpools:\t%3zu B\t%3zu B\n"
2406  "size[]:\t\t%3zu B\t%3zu B\n"
2407  "odp_pool[]:\t%3zu B\t%3zu B\n"
2408  "em_pool:\t%3zu B\t%3zu B\n"
2409  "objpool_elem:\t%3zu B\t%3zu B\n"
2410  "stats_opt:\t%3zu B\t%3zu B\n"
2411  "pool_cfg:\t%3zu B\t%3zu B\n"
2412  "name[]:\t\t%3zu B\t%3zu B\n",
2413  offsetof(mpool_elem_t, event_type), sizeof_field(mpool_elem_t, event_type),
2414  offsetof(mpool_elem_t, align_offset), sizeof_field(mpool_elem_t, align_offset),
2415  offsetof(mpool_elem_t, user_area), sizeof_field(mpool_elem_t, user_area),
2416  offsetof(mpool_elem_t, num_subpools), sizeof_field(mpool_elem_t, num_subpools),
2417  offsetof(mpool_elem_t, size), sizeof_field(mpool_elem_t, size),
2418  offsetof(mpool_elem_t, odp_pool), sizeof_field(mpool_elem_t, odp_pool),
2419  offsetof(mpool_elem_t, em_pool), sizeof_field(mpool_elem_t, em_pool),
2420  offsetof(mpool_elem_t, objpool_elem), sizeof_field(mpool_elem_t, objpool_elem),
2421  offsetof(mpool_elem_t, stats_opt), sizeof_field(mpool_elem_t, stats_opt),
2422  offsetof(mpool_elem_t, pool_cfg), sizeof_field(mpool_elem_t, pool_cfg),
2423  offsetof(mpool_elem_t, name), sizeof_field(mpool_elem_t, name));
2424 
2425  EM_PRINT("\n");
2426 }
mpool_elem_t::size
uint16_t size
Definition: em_pool_types.h:55
EM_EVENT_TYPE_VECTOR
@ EM_EVENT_TYPE_VECTOR
Definition: event_machine_hw_types.h:84
mpool_elem_t::em_pool
em_pool_t em_pool
Definition: em_pool_types.h:71
em_pool_cfg_t::align_offset
struct em_pool_cfg_t::@8 align_offset
em_pool_stats_opt_t::available
uint64_t available
Definition: event_machine_pool.h:102
em_pool_subpool_stats
int em_pool_subpool_stats(em_pool_t pool, const int subpools[], int num_subpools, em_pool_subpool_stats_t subpool_stats[])
Retrieve statistics about subpool(s) of an EM pool.
Definition: event_machine_pool.c:399
EM_OK
#define EM_OK
Definition: event_machine_types.h:329
EM_ERR
@ EM_ERR
Definition: event_machine_hw_types.h:312
EM_EVENT_TYPE_SW
@ EM_EVENT_TYPE_SW
Definition: event_machine_hw_types.h:72
em_shm_t::pool_count
env_atomic32_t pool_count
Definition: em_mem.h:144
pool_subpool_undef
const pool_subpool_t pool_subpool_undef
Undef value for a pool_subpool_t pool_subpool_undef = {.pool = EM_POOL_UNDEF, .subpool = 0};.
Definition: em_pool.c:59
mpool_elem_t::user_area
struct mpool_elem_t::@54 user_area
em_pool_info_t::align_offset
uint32_t align_offset
Definition: event_machine_pool.h:333
event_hdr_t
struct event_hdr event_hdr_t
pool_subpool_t
Definition: em_pool_types.h:100
em_pool_info_t::size
uint32_t size
Definition: event_machine_pool.h:340
em_pool_stats_opt_t::alloc_ops
uint64_t alloc_ops
Definition: event_machine_pool.h:105
EM_POOL_DEFAULT
#define EM_POOL_DEFAULT
Definition: event_machine_hw_config.h:191
em_pool_cfg_t::user_area
struct em_pool_cfg_t::@9 user_area
EM_ESCOPE_POOL_CREATE
#define EM_ESCOPE_POOL_CREATE
Definition: event_machine_hw_types.h:489
em_pool_stats_opt_t::cache_alloc_ops
uint64_t cache_alloc_ops
Definition: event_machine_pool.h:120
mpool_elem_t::odp_pool
odp_pool_t odp_pool[EM_MAX_SUBPOOLS]
Definition: em_pool_types.h:69
em_pool_stats_opt_t::core_cache_available
uint64_t core_cache_available
Definition: event_machine_pool.h:130
EM_ERR_TOO_SMALL
@ EM_ERR_TOO_SMALL
Definition: event_machine_hw_types.h:296
event_prealloc_hdr::list_node
list_node_t list_node
Definition: em_event_types.h:311
PRI_POOL
#define PRI_POOL
Definition: event_machine_hw_types.h:62
em_free
void em_free(em_event_t event)
Definition: event_machine_event.c:261
em_pool_info_print_all
void em_pool_info_print_all(void)
Definition: event_machine_pool.c:313
em_pool_info_t::num
uint32_t num
Definition: event_machine_pool.h:342
em_pool_subpool_stats_t::cache_free_ops
uint64_t cache_free_ops
Definition: event_machine_pool.h:393
em_pool_info_t::used
uint32_t used
Definition: event_machine_pool.h:352
em_pool_cfg_t::value
uint32_t value
Definition: event_machine_pool.h:194
mpool_elem_t::objpool_elem
objpool_elem_t objpool_elem
Definition: em_pool_types.h:73
em_pool_info_t::num_subpools
int num_subpools
Definition: event_machine_pool.h:337
em_pool_stats_opt_t::free_ops
uint64_t free_ops
Definition: event_machine_pool.h:111
em_pool_subpool_stats_selected_t::free_ops
uint64_t free_ops
Definition: event_machine_pool.h:422
EM_ERR_ALLOC_FAILED
@ EM_ERR_ALLOC_FAILED
Definition: event_machine_hw_types.h:287
EM_ERR_LIB_FAILED
@ EM_ERR_LIB_FAILED
Definition: event_machine_hw_types.h:291
em_pool_cfg_t::in_use
bool in_use
Definition: event_machine_pool.h:185
em_pool_subpool_stats_selected_t::available
uint64_t available
Definition: event_machine_pool.h:411
em_pool_info_t::user_area_size
size_t user_area_size
Definition: event_machine_pool.h:335
em_pool_create
em_pool_t em_pool_create(const char *name, em_pool_t pool, const em_pool_cfg_t *pool_cfg)
Definition: event_machine_pool.c:76
list_node_t
Definition: list.h:42
em_pool_subpool_stats_t::alloc_fails
uint64_t alloc_fails
Definition: event_machine_pool.h:374
em_pool_stats_opt_t::cache_free_ops
uint64_t cache_free_ops
Definition: event_machine_pool.h:123
em_pool_stats_selected_t
Definition: event_machine_pool.h:441
event_hdr::event
em_event_t event
Definition: em_event_types.h:246
EM_ERR_TOO_LARGE
@ EM_ERR_TOO_LARGE
Definition: event_machine_hw_types.h:294
em_pool_subpool_stats_selected_t::cache_available
uint64_t cache_available
Definition: event_machine_pool.h:430
event_prealloc_hdr
Definition: em_event_types.h:301
event_hdr
Definition: em_event_types.h:184
EM_EVENT_USER_AREA_MAX_SIZE
#define EM_EVENT_USER_AREA_MAX_SIZE
Definition: event_machine_hw_config.h:216
em_shm_t::libconfig
libconfig_t libconfig
Definition: em_mem.h:146
em_pool_cfg_t
Definition: event_machine_pool.h:141
em_pool_info_t::cache_size
uint32_t cache_size
Definition: event_machine_pool.h:344
mpool_elem_t
Definition: em_pool_types.h:47
em_pool_subpool_stats_t
Definition: event_machine_pool.h:364
em_pool_subpool_stats_selected_t::total_ops
uint64_t total_ops
Definition: event_machine_pool.h:427
em_pool_subpool_stats_t::free_ops
uint64_t free_ops
Definition: event_machine_pool.h:377
em_pool_subpool_stats_selected_t::alloc_fails
uint64_t alloc_fails
Definition: event_machine_pool.h:419
mpool_tbl_t::pool_subpool_odp2em
pool_subpool_t pool_subpool_odp2em[POOL_ODP2EM_TBL_LEN]
Definition: em_pool_types.h:127
EM_ERR_BAD_ID
@ EM_ERR_BAD_ID
Definition: event_machine_hw_types.h:265
em_pool_stats_opt_t::total_ops
uint64_t total_ops
Definition: event_machine_pool.h:114
INTERNAL_ERROR
#define INTERNAL_ERROR(error, escope, fmt,...)
Definition: em_error.h:43
em_pool_cfg_init
void em_pool_cfg_init(em_pool_cfg_t *const pool_cfg)
Definition: event_machine_pool.c:43
mpool_elem_t::event_type
em_event_type_t event_type
Definition: em_pool_types.h:49
mpool_elem_t::stats_opt
odp_pool_stats_opt_t stats_opt
Definition: em_pool_types.h:75
mpool_elem_t::pool_cfg
em_pool_cfg_t pool_cfg
Definition: em_pool_types.h:77
mpool_elem_t::align_offset
uint32_t align_offset
Definition: em_pool_types.h:51
em_pool_cfg_t::num
uint32_t num
Definition: event_machine_pool.h:281
em_pool_stats_opt_t::cache_available
uint64_t cache_available
Definition: event_machine_pool.h:117
mpool_elem_t::num_subpools
int num_subpools
Definition: em_pool_types.h:65
em_pool_cfg_t::__internal_check
uint32_t __internal_check
Definition: event_machine_pool.h:319
mpool_pool_t
Definition: em_pool_types.h:136
em_pool_cfg_t::cache_size
uint32_t cache_size
Definition: event_machine_pool.h:294
em_pool_cfg_t::num_subpools
int num_subpools
Definition: event_machine_pool.h:264
em_pool_info_t::event_type
em_event_type_t event_type
Definition: event_machine_pool.h:331
startup_pool_conf_t
Definition: em_init.h:47
em_pool_info
em_status_t em_pool_info(em_pool_t pool, em_pool_info_t *pool_info)
Definition: event_machine_pool.c:216
em_pool_stats_opt_t::alloc_fails
uint64_t alloc_fails
Definition: event_machine_pool.h:108
EM_POOL_DEFAULT_NAME
#define EM_POOL_DEFAULT_NAME
Definition: event_machine_hw_config.h:202
em_status_t
uint32_t em_status_t
Definition: event_machine_types.h:321
mpool_tbl_t
Definition: em_pool_types.h:119
EM_ERR_OPERATION_FAILED
@ EM_ERR_OPERATION_FAILED
Definition: event_machine_hw_types.h:289
em_pool_subpool_stats_t::alloc_ops
uint64_t alloc_ops
Definition: event_machine_pool.h:371
objpool_elem_t
Definition: objpool.h:48
em_pool_subpool_stats_t::available
uint64_t available
Definition: event_machine_pool.h:366
em_pool_stats_t
Definition: event_machine_pool.h:399
em_event_type_t
uint32_t em_event_type_t
Definition: event_machine_types.h:85
em_shm
em_shm_t * em_shm
Definition: event_machine_init.c:41
em_pool_cfg_t::pkt
struct em_pool_cfg_t::@10 pkt
em_include.h
em_pool_stats_opt_t
Definition: event_machine_pool.h:98
em_pool_subpool_stats_selected_t::cache_alloc_ops
uint64_t cache_alloc_ops
Definition: event_machine_pool.h:435
em_pool_subpool_stats_t::cache_available
uint64_t cache_available
Definition: event_machine_pool.h:385
em_core_id
int em_core_id(void)
Definition: event_machine_core.c:34
EM_EVENT_TYPE_UNDEF
@ EM_EVENT_TYPE_UNDEF
Definition: event_machine_hw_types.h:69
em_pool_subpool_stats_selected
int em_pool_subpool_stats_selected(em_pool_t pool, const int subpools[], int num_subpools, em_pool_subpool_stats_selected_t subpool_stats[], const em_pool_stats_opt_t *opt)
Retrieve selected statistics about subpool(s) of an EM pool.
Definition: event_machine_pool.c:554
EM_POOL_NAME_LEN
#define EM_POOL_NAME_LEN
Definition: event_machine_hw_config.h:196
libconfig::has_cfg_runtime
bool has_cfg_runtime
Definition: em_libconfig_types.h:52
EM_ERR_NOT_FOUND
@ EM_ERR_NOT_FOUND
Definition: event_machine_hw_types.h:278
EM_POOL_UNDEF
#define EM_POOL_UNDEF
Definition: event_machine_hw_types.h:60
EM_CHECK_INIT_CALLED
#define EM_CHECK_INIT_CALLED
Definition: em_include.h:69
EM_EVENT_TYPE_PACKET
@ EM_EVENT_TYPE_PACKET
Definition: event_machine_hw_types.h:75
EM_ERR_BAD_POINTER
@ EM_ERR_BAD_POINTER
Definition: event_machine_hw_types.h:271
em_pool_cfg_t::subpool
struct em_pool_cfg_t::@11 subpool[EM_MAX_SUBPOOLS]
em_pool_subpool_stats_selected_t::cache_free_ops
uint64_t cache_free_ops
Definition: event_machine_pool.h:438
EM_ERR_BAD_ARG
@ EM_ERR_BAD_ARG
Definition: event_machine_hw_types.h:261
EM_CONFIG_POOLS
#define EM_CONFIG_POOLS
Definition: event_machine_config.h:119
em_pool_cfg_t::size
size_t size
Definition: event_machine_pool.h:224
em_pool_stats_selected
em_status_t em_pool_stats_selected(em_pool_t pool, em_pool_stats_selected_t *pool_stats, const em_pool_stats_opt_t *opt)
Retrieve selected statistics about an EM pool.
Definition: event_machine_pool.c:501
em_pool_cfg_t::event_type
em_event_type_t event_type
Definition: event_machine_pool.h:156
EM_MAX_SUBPOOLS
#define EM_MAX_SUBPOOLS
The number of subpools in each EM pool. The subpool is a pool with buffers of only one size.
Definition: event_machine_hw_types.h:254
em_pool_cfg_t::stats_opt
struct em_pool_cfg_t::@12 stats_opt
em_pool_subpool_stats_t::cache_alloc_ops
uint64_t cache_alloc_ops
Definition: event_machine_pool.h:390
em_pool_info_t::free
uint32_t free
Definition: event_machine_pool.h:360
em_pool_info_t
Definition: event_machine_pool.h:325
em_pool_subpool_stats_selected_t
Definition: event_machine_pool.h:409
POOL_ODP2EM_TBL_LEN
#define POOL_ODP2EM_TBL_LEN
Definition: em_pool_types.h:86
em_pool_stats
em_status_t em_pool_stats(em_pool_t pool, em_pool_stats_t *pool_stats)
Retrieve statistics about an EM pool.
Definition: event_machine_pool.c:327
em_pool_subpool_stats_selected_t::alloc_ops
uint64_t alloc_ops
Definition: event_machine_pool.h:416
em_pool_subpool_stats_t::total_ops
uint64_t total_ops
Definition: event_machine_pool.h:382
em_pool_cfg_t::headroom
struct em_pool_cfg_t::@10::@13 headroom