EM-ODP  3.7.0
Event Machine on ODP
event_machine_event.c
1 /*
2  * Copyright (c) 2015-2023, Nokia Solutions and Networks
3  * All rights reserved.
4  *
5  * Redistribution and use in source and binary forms, with or without
6  * modification, are permitted provided that the following conditions
7  * are met:
8  *
9  * * Redistributions of source code must retain the above copyright
10  * notice, this list of conditions and the following disclaimer.
11  * * Redistributions in binary form must reproduce the above copyright
12  * notice, this list of conditions and the following disclaimer in the
13  * documentation and/or other materials provided with the distribution.
14  * * Neither the name of the copyright holder nor the names of its
15  * contributors may be used to endorse or promote products derived
16  * from this software without specific prior written permission.
17  *
18  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22  * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29  */
30 
31 #include "em_include.h"
32 
33 em_event_t em_alloc(uint32_t size, em_event_type_t type, em_pool_t pool)
34 {
35  const mpool_elem_t *const pool_elem = pool_elem_get(pool);
36  em_event_type_t major_type = em_event_type_major(type);
37 
38  if (EM_CHECK_LEVEL > 0 &&
39  unlikely(size == 0 || !pool_elem || major_type == EM_EVENT_TYPE_TIMER_IND)) {
40  INTERNAL_ERROR(EM_ERR_BAD_ARG, EM_ESCOPE_ALLOC,
41  "Invalid args: size:%u type:%u pool:%" PRI_POOL "",
42  size, type, pool);
43  return EM_EVENT_UNDEF;
44  }
45  if (EM_CHECK_LEVEL >= 2 && unlikely(!pool_allocated(pool_elem)))
46  INTERNAL_ERROR(EM_ERR_NOT_CREATED, EM_ESCOPE_ALLOC,
47  "Invalid pool:%" PRI_POOL ", pool not created", pool);
48 
49  /*
50  * EM event pools created with type=SW can not support pkt events.
51  */
52  if (EM_CHECK_LEVEL >= 1 &&
53  unlikely(pool_elem->event_type == EM_EVENT_TYPE_SW &&
54  major_type == EM_EVENT_TYPE_PACKET)) {
55  INTERNAL_ERROR(EM_ERR_NOT_IMPLEMENTED, EM_ESCOPE_ALLOC,
56  "EM-pool:%s(%" PRI_POOL "):\n"
57  "Invalid event type:0x%x for buf",
58  pool_elem->name, pool_elem->em_pool, type);
59  return EM_EVENT_UNDEF;
60  }
61  if (EM_CHECK_LEVEL >= 1 &&
62  unlikely(pool_elem->event_type == EM_EVENT_TYPE_VECTOR &&
63  major_type != EM_EVENT_TYPE_VECTOR)) {
64  INTERNAL_ERROR(EM_ERR_NOT_IMPLEMENTED, EM_ESCOPE_ALLOC,
65  "EM-pool:%s(%" PRI_POOL "):\n"
66  "Invalid event type:0x%x for vector",
67  pool_elem->name, pool_elem->em_pool, type);
68  return EM_EVENT_UNDEF;
69  }
70 
71  const em_event_t event = event_alloc(pool_elem, size, type, EVSTATE__ALLOC);
72 
73  if (EM_CHECK_LEVEL > 0 && unlikely(event == EM_EVENT_UNDEF)) {
74  em_status_t err =
75  INTERNAL_ERROR(EM_ERR_ALLOC_FAILED, EM_ESCOPE_ALLOC,
76  "EM-pool:'%s': sz:%u type:0x%x pool:%" PRI_POOL "",
77  pool_elem->name, size, type, pool);
78  if (EM_DEBUG_PRINT && err != EM_OK &&
79  (pool_elem->stats_opt.bit.available ||
80  pool_elem->stats_opt.bit.cache_available)) {
81  em_pool_info_print(pool);
82  }
83  return EM_EVENT_UNDEF;
84  }
85 
86  if (EM_API_HOOKS_ENABLE && event != EM_EVENT_UNDEF)
87  call_api_hooks_alloc(&event, 1, 1, size, type, pool);
88 
89  return event;
90 }
91 
92 int em_alloc_multi(em_event_t events[/*out*/], int num,
93  uint32_t size, em_event_type_t type, em_pool_t pool)
94 {
95  if (unlikely(num == 0))
96  return 0;
97 
98  const mpool_elem_t *const pool_elem = pool_elem_get(pool);
99  int ret = 0;
100 
101  if (EM_CHECK_LEVEL > 0 &&
102  unlikely(!events || num < 0 || size == 0 || !pool_elem ||
103  em_event_type_major(type) == EM_EVENT_TYPE_TIMER_IND)) {
104  INTERNAL_ERROR(EM_ERR_BAD_ARG, EM_ESCOPE_ALLOC_MULTI,
105  "Invalid args: events:%p num:%d size:%u type:%u pool:%" PRI_POOL "",
106  events, num, size, type, pool);
107  return 0;
108  }
109  if (EM_CHECK_LEVEL >= 2 && unlikely(!pool_allocated(pool_elem)))
110  INTERNAL_ERROR(EM_ERR_BAD_ID, EM_ESCOPE_ALLOC_MULTI,
111  "Invalid pool:%" PRI_POOL ", pool not created", pool);
112 
113  if (pool_elem->event_type == EM_EVENT_TYPE_PACKET) {
114  /*
115  * EM event pools created with type=PKT can support SW events
116  * as well as pkt events.
117  */
118  ret = event_alloc_pkt_multi(events, num, pool_elem, size, type);
119  } else if (pool_elem->event_type == EM_EVENT_TYPE_SW) {
120  /*
121  * EM event pools created with type=SW can not support
122  * pkt events.
123  */
124  if (EM_CHECK_LEVEL >= 1 &&
125  unlikely(em_event_type_major(type) == EM_EVENT_TYPE_PACKET)) {
126  INTERNAL_ERROR(EM_ERR_NOT_IMPLEMENTED, EM_ESCOPE_ALLOC_MULTI,
127  "EM-pool:%s(%" PRI_POOL "): Invalid event type:0x%x for buf",
128  pool_elem->name, pool, type);
129  return 0;
130  }
131  ret = event_alloc_buf_multi(events, num, pool_elem, size, type);
132  } else if (pool_elem->event_type == EM_EVENT_TYPE_VECTOR) {
133  if (EM_CHECK_LEVEL >= 1 &&
134  unlikely(em_event_type_major(type) != EM_EVENT_TYPE_VECTOR)) {
135  INTERNAL_ERROR(EM_ERR_NOT_IMPLEMENTED, EM_ESCOPE_ALLOC,
136  "EM-pool:%s(%" PRI_POOL "): Inv. event type:0x%x for vector",
137  pool_elem->name, pool, type);
138  return 0;
139  }
140  ret = event_alloc_vector_multi(events, num, pool_elem, size, type);
141  }
142 
143  if (unlikely(EM_CHECK_LEVEL > 0 && ret != num)) {
144  em_status_t err =
145  INTERNAL_ERROR(EM_ERR_ALLOC_FAILED, EM_ESCOPE_ALLOC_MULTI,
146  "Requested num:%d events, allocated:%d\n"
147  "EM-pool:'%s': sz:%u type:0x%x pool:%" PRI_POOL "",
148  num, ret,
149  pool_elem->name, size, type, pool);
150  if (EM_DEBUG_PRINT && err != EM_OK &&
151  (pool_elem->stats_opt.bit.available ||
152  pool_elem->stats_opt.bit.cache_available)) {
153  em_pool_info_print(pool);
154  }
155  }
156 
157  if (EM_API_HOOKS_ENABLE && ret > 0)
158  call_api_hooks_alloc(events, ret, num, size, type, pool);
159 
160  return ret;
161 }
162 
163 /**
164  * @brief Helper to check if the event is a vector
165  *
166  * @param vector_event Event handle
167  * @return true the event is a vector
168  * @return false the event is NOT a vector
169  */
170 static inline bool is_vector_type(em_event_t vector_event)
171 {
172  odp_event_t odp_event = event_em2odp(vector_event);
173  odp_event_type_t odp_etype = odp_event_type(odp_event);
174 
175  if (odp_etype == ODP_EVENT_PACKET_VECTOR)
176  return true;
177 
178  return false;
179 }
180 
181 /**
182  * @brief Helper to check if the event is a vector, if not report an error
183  *
184  * @param vector_event Event handle
185  * @param escope Error scope to use if reporting an error
186  * @return true the event is a vector
187  * @return false the event is NOT a vector, reports an error
188  */
189 static inline bool is_vector_type_or_error(em_event_t vector_event,
190  em_escope_t escope)
191 {
192  bool is_vec = is_vector_type(vector_event);
193 
194  if (likely(is_vec))
195  return true;
196 
197  INTERNAL_ERROR(EM_ERR_BAD_TYPE, escope, "Event not a vector");
198  return false;
199 }
200 
201 /**
202  * @brief Handle ESV state for 'em_free' for the event-table of a vector event
203  *
204  * @param event Vector event handle
205  */
206 static void event_vector_prepare_free_full(em_event_t event, const uint16_t api_op)
207 {
208  /* em_free() frees the vector as well as all the events it contains */
209  em_event_t *ev_tbl;
210  uint32_t sz = event_vector_tbl(event, &ev_tbl);
211 
212  if (sz) {
213  event_hdr_t *ev_hdrs[sz];
214 
215  /* same as event_to_hdr_multi(), removes gcc-12 LTO error in haswell */
216  for (uint32_t i = 0; i < sz; i++)
217  ev_hdrs[i] = event_to_hdr(ev_tbl[i]);
218 
219  evstate_free_multi(ev_tbl, ev_hdrs, sz, api_op);
220 
221  /* drop ESV generation from event handles */
222  (void)events_em2pkt_inplace(ev_tbl, sz);
223  }
224 }
225 
226 /**
227  * @brief Handle ESV state for 'em_event_unmark_free/_multi' for the event-table
228  * of a vector event.
229  *
230  * @param event Vector event handle
231  */
232 static void event_vector_prepare_free_full__revert(em_event_t event, const uint16_t api_op)
233 {
234  /* em_free() frees the vector as well as all the events it contains */
235  em_event_t *ev_tbl;
236  uint32_t sz = event_vector_tbl(event, &ev_tbl);
237 
238  if (sz) {
239  event_hdr_t *ev_hdrs[sz];
240 
241  event_to_hdr_multi(ev_tbl, ev_hdrs, sz);
242  evstate_unmark_free_multi(ev_tbl, ev_hdrs, sz, api_op);
243 
244  /* restore dropped ESV generation to event handles, unmodified in header */
245  for (unsigned int i = 0; i < sz; i++)
246  ev_tbl[i] = ev_hdrs[i]->event;
247  }
248 }
249 
250 /**
251  * Helper to em_free() and em_free_multi() to determine whether timeout events
252  * from periodic timer rings can exist and if free() needs to check for them.
253  * Active tmo events from ring timers must never be freed (user error), only
254  * inactive (last tmo event after cancel) can be freed.
255  */
256 static inline bool timer_rings_used(void)
257 {
258  return em_shm->timers.num_ring_create_calls > 0 ? true : false;
259 }
260 
261 void em_free(em_event_t event)
262 {
263  if (EM_CHECK_LEVEL > 0 && unlikely(event == EM_EVENT_UNDEF)) {
264  INTERNAL_ERROR(EM_ERR_BAD_ARG, EM_ESCOPE_FREE,
265  "event undefined!");
266  return;
267  }
268 
269  odp_event_t odp_event = event_em2odp(event);
270  const bool esv_ena = esv_enabled();
271  /* Is a check for an active periodic tmo event from a timer ring needed? */
272  const bool check_tmos = EM_CHECK_LEVEL >= 2 && timer_rings_used();
273 
274  if (unlikely(check_tmos && odp_event_type(odp_event) == ODP_EVENT_TIMEOUT)) {
275  event_hdr_t *ev_hdr = event_to_hdr(event);
276 
277  if (unlikely(ev_hdr->flags.tmo_type != EM_TMO_TYPE_NONE)) {
278  INTERNAL_ERROR(EM_ERR_BAD_STATE, EM_ESCOPE_FREE,
279  "Can't free active TIMER event");
280  return;
281  }
282  if (esv_ena)
283  evstate_free(event, ev_hdr, EVSTATE__FREE);
284  } else if (esv_ena) {
285  event_hdr_t *ev_hdr = event_to_hdr(event);
286 
287  evstate_free(event, ev_hdr, EVSTATE__FREE);
288  if (is_vector_type(event))
289  event_vector_prepare_free_full(event, EVSTATE__FREE);
290  }
291 
293  call_api_hooks_free(&event, 1);
294 
295  odp_event_free(odp_event);
296 }
297 
298 /**
299  * Helper to em_free_multi() to remove active periodic tmo events
300  * (from ring timers) from the free list.
301  *
302  * Active tmo events from ring timers must never be freed (user error), only
303  * inactive (last tmo event after cancel) can be freed. Thus remove the active
304  * ones if the user incorrectly tries to free them.
305  */
306 static inline int
307 rem_active_ring_timer_tmos(const int num, odp_event_t odp_evtbl[/*in/out*/],
308  event_hdr_t *ev_hdr_tbl[/*in/out*/],
309  em_event_t ev_tbl[/*in/out*/])
310 {
311  int first_tmo_idx = -1;
312 
313  /* find first active tmo-event */
314  for (int i = 0; i < num; i++) {
315  if (unlikely(odp_event_type(odp_evtbl[i]) == ODP_EVENT_TIMEOUT &&
316  ev_hdr_tbl[i]->flags.tmo_type != EM_TMO_TYPE_NONE)) {
317  first_tmo_idx = i;
318  break;
319  }
320  }
321 
322  /*
323  * No active tmo events found - all OK, return.
324  * This is the normal, no-error, scenario.
325  */
326  if (likely(first_tmo_idx == -1))
327  return num;
328 
329  /*
330  * Error: Active tmo events found - remove them from the arrays
331  */
332 
333  /* last event is tmo, no need to move/copy anything, just drop last */
334  if (first_tmo_idx == num - 1)
335  return num - 1;
336 
337  /*
338  * Store indexes of "normal events" (i.e. events other than active
339  * tmo events) to copy from 'first_tmo_idx + 1' onwards.
340  */
341  int num_cpy = 0;
342  int cpy_idx[num - first_tmo_idx - 1];
343 
344  for (int i = first_tmo_idx + 1; i < num; i++) {
345  if (likely(!(odp_event_type(odp_evtbl[i]) == ODP_EVENT_TIMEOUT &&
346  ev_hdr_tbl[i]->flags.tmo_type != EM_TMO_TYPE_NONE)))
347  cpy_idx[num_cpy++] = i;
348  }
349 
350  /* all further events were active tmo events, drop them */
351  if (num_cpy == 0)
352  return first_tmo_idx;
353 
354  /*
355  * Remove all active tmo events from the arrays by copying the "normal"
356  * events into the slots occupied by the active tmo events.
357  */
358  for (int i = 0; i < num_cpy; i++) {
359  int src_idx = cpy_idx[i];
360  int dst_idx = first_tmo_idx + i;
361 
362  odp_evtbl[dst_idx] = odp_evtbl[src_idx];
363  ev_hdr_tbl[dst_idx] = ev_hdr_tbl[src_idx];
364  ev_tbl[dst_idx] = ev_tbl[src_idx];
365  }
366 
367  return first_tmo_idx + num_cpy;
368 }
369 
370 void em_free_multi(em_event_t events[], int num)
371 {
372  if (EM_CHECK_LEVEL > 0 && unlikely(!events || num < 0)) {
373  INTERNAL_ERROR(EM_ERR_BAD_ARG, EM_ESCOPE_FREE_MULTI,
374  "Inv.args: events[]:%p num:%d", events, num);
375  return;
376  }
377  if (unlikely(num == 0))
378  return;
379 
380  if (EM_CHECK_LEVEL >= 3) {
381  int i;
382 
383  for (i = 0; i < num && events[i] != EM_EVENT_UNDEF; i++)
384  ;
385  if (unlikely(i != num)) {
386  INTERNAL_ERROR(EM_ERR_BAD_ARG, EM_ESCOPE_FREE_MULTI,
387  "events[%d] undefined!", i);
388  return;
389  }
390  }
391 
392  int num_free = num;
393  const bool esv_ena = esv_enabled();
394  odp_event_t odp_events[num];
395 
396  events_em2odp(events, odp_events/*out*/, num);
397 
398  /* Is a check for active periodic tmo events from timer rings needed? */
399  const bool check_tmos = EM_CHECK_LEVEL >= 2 && timer_rings_used();
400 
401  if (check_tmos || esv_ena) {
402  event_hdr_t *ev_hdrs[num];
403 
404  event_to_hdr_multi(events, ev_hdrs, num);
405 
406  if (check_tmos) {
407  num_free = rem_active_ring_timer_tmos(num, odp_events, ev_hdrs, events);
408  if (unlikely(num_free != num))
409  INTERNAL_ERROR(EM_ERR_BAD_STATE, EM_ESCOPE_FREE_MULTI,
410  "Can't free active TIMER events: %d of %d ignored",
411  num_free, num);
412  }
413 
414  if (esv_ena) {
415  evstate_free_multi(events, ev_hdrs, num_free, EVSTATE__FREE_MULTI);
416 
417  for (int i = 0; i < num_free; i++) {
418  if (is_vector_type(events[i]))
419  event_vector_prepare_free_full(events[i],
420  EVSTATE__FREE_MULTI);
421  }
422  }
423  }
424 
426  call_api_hooks_free(events, num_free);
427 
428  odp_event_free_multi(odp_events, num_free);
429 }
430 
431 /**
432  * Helper to em_send().
433  * Send out of EM via event-chaining and a user-provided function
434  * 'event_send_device()' to another device
435  */
436 static inline em_status_t
437 send_external(em_event_t event, em_queue_t queue)
438 {
440  call_api_hooks_send(&event, 1, queue, EM_EVENT_GROUP_UNDEF);
441 
442  em_status_t stat = send_chaining(event, queue);
443 
444  if (EM_CHECK_LEVEL == 0)
445  return stat;
446 
447  RETURN_ERROR_IF(stat != EM_OK, stat, EM_ESCOPE_SEND,
448  "send out-of-EM via event-chaining failed: Q:%" PRI_QUEUE "", queue);
449  return EM_OK;
450 }
451 
452 /**
453  * Helper to em_send_multi().
454  * Send out of EM via event-chaining and a user-provided function
455  * 'event_send_device()' to another device
456  */
457 static inline int
458 send_external_multi(const em_event_t events[], int num, em_queue_t queue)
459 {
461  call_api_hooks_send(events, num, queue, EM_EVENT_GROUP_UNDEF);
462 
463  int num_sent = send_chaining_multi(events, num, queue);
464 
465  if (EM_CHECK_LEVEL > 0 && unlikely(num_sent != num)) {
466  INTERNAL_ERROR(EM_ERR_OPERATION_FAILED, EM_ESCOPE_SEND_MULTI,
467  "send_chaining_multi: req:%d, sent:%d",
468  num, num_sent);
469  }
470 
471  return num_sent;
472 }
473 
474 /**
475  * Helper to em_send().
476  * Send to an EM internal queue.
477  */
478 static inline em_status_t
479 send_internal(em_event_t event, event_hdr_t *ev_hdr, em_queue_t queue)
480 {
481  queue_elem_t *q_elem = queue_elem_get(queue);
482  em_status_t stat;
483 
484  RETURN_ERROR_IF(EM_CHECK_LEVEL > 0 && !q_elem,
485  EM_ERR_BAD_ARG, EM_ESCOPE_SEND,
486  "Invalid queue:%" PRI_QUEUE "", queue);
487  RETURN_ERROR_IF(EM_CHECK_LEVEL >= 2 && !queue_allocated(q_elem),
488  EM_ERR_BAD_STATE, EM_ESCOPE_SEND,
489  "Invalid queue:%" PRI_QUEUE "", queue);
490 
491  /* Buffer events sent from EO-start to scheduled queues */
492  if (unlikely(em_locm.start_eo_elem && q_elem->flags.scheduled)) {
493  /*
494  * em_send() called from within an EO-start function:
495  * all events sent to scheduled queues will be buffered
496  * and sent when the EO-start operation completes.
497  */
498  if (esv_enabled())
499  evstate_usr2em(event, ev_hdr, EVSTATE__SEND);
500 
501  int num_sent = eo_start_buffer_events(&event, 1, queue);
502 
503  if (unlikely(num_sent != 1)) {
505  goto error_return;
506  }
507 
508  return EM_OK; /* Success */
509  }
510 
512  call_api_hooks_send(&event, 1, queue, EM_EVENT_GROUP_UNDEF);
513 
514  if (q_elem->type == EM_QUEUE_TYPE_OUTPUT) {
515  /*
516  * Send out of EM via an EM output-queue and a user provided
517  * function of type em_output_func_t
518  */
519  stat = send_output(event, q_elem);
520 
521  if (unlikely(stat != EM_OK))
522  goto error_return_noesv;
523 
524  return EM_OK; /* Success */
525  }
526 
527  /*
528  * Normal send to a queue on this device
529  */
530  if (esv_enabled())
531  evstate_usr2em(event, ev_hdr, EVSTATE__SEND);
532 
533  switch (q_elem->type) {
537  stat = send_event(event, q_elem);
538  break;
540  stat = queue_unsched_enqueue(event, q_elem);
541  break;
542  case EM_QUEUE_TYPE_LOCAL:
543  stat = send_local(event, q_elem);
544  break;
545  default:
546  stat = EM_ERR_NOT_FOUND;
547  break;
548  }
549 
550  if (likely(stat == EM_OK))
551  return EM_OK; /* Success */
552 
553 error_return:
554  if (esv_enabled())
555  evstate_usr2em_revert(event, ev_hdr, EVSTATE__SEND__FAIL);
556 error_return_noesv:
557  if (EM_CHECK_LEVEL == 0)
558  return stat;
559  stat = INTERNAL_ERROR(stat, EM_ESCOPE_SEND,
560  "send failed: Q:%" PRI_QUEUE " type:%" PRI_QTYPE "",
561  queue, q_elem->type);
562  return stat;
563 }
564 
565 /**
566  * Helper to em_send_multi().
567  * Send to an EM internal queue.
568  */
569 static inline int
570 send_internal_multi(const em_event_t events[], event_hdr_t *ev_hdrs[],
571  int num, em_queue_t queue)
572 {
573  queue_elem_t *q_elem = queue_elem_get(queue);
574  int num_sent;
575 
576  if (EM_CHECK_LEVEL > 0 && unlikely(!q_elem)) {
577  INTERNAL_ERROR(EM_ERR_BAD_ARG, EM_ESCOPE_SEND_MULTI,
578  "Invalid queue:%" PRI_QUEUE "", queue);
579  return 0;
580  }
581  if (EM_CHECK_LEVEL >= 2 && unlikely(!queue_allocated(q_elem))) {
582  INTERNAL_ERROR(EM_ERR_BAD_STATE, EM_ESCOPE_SEND_MULTI,
583  "Invalid queue:%" PRI_QUEUE "", queue);
584  return 0;
585  }
586 
587  /* Buffer events sent from EO-start to scheduled queues */
588  if (unlikely(em_locm.start_eo_elem && q_elem->flags.scheduled)) {
589  /*
590  * em_send_multi() called from within an EO-start function:
591  * all events sent to scheduled queues will be buffered
592  * and sent when the EO-start operation completes.
593  */
594  if (esv_enabled())
595  evstate_usr2em_multi(events, ev_hdrs, num,
596  EVSTATE__SEND_MULTI);
597  num_sent = eo_start_buffer_events(events, num, queue);
598 
599  if (unlikely(num_sent != num))
600  goto error_return;
601 
602  return num_sent; /* Success */
603  }
604 
606  call_api_hooks_send(events, num, queue, EM_EVENT_GROUP_UNDEF);
607 
608  if (q_elem->type == EM_QUEUE_TYPE_OUTPUT) {
609  /*
610  * Send out of EM via an EM output-queue and a user provided
611  * function of type em_output_func_t
612  */
613  num_sent = send_output_multi(events, num, q_elem);
614 
615  if (unlikely(num_sent != num))
616  goto error_return_noesv;
617 
618  return num_sent; /* Success */
619  }
620 
621  /*
622  * Normal send to a queue on this device
623  */
624  if (esv_enabled())
625  evstate_usr2em_multi(events, ev_hdrs, num, EVSTATE__SEND_MULTI);
626 
627  switch (q_elem->type) {
631  num_sent = send_event_multi(events, num, q_elem);
632  break;
634  num_sent = queue_unsched_enqueue_multi(events, num, q_elem);
635  break;
636  case EM_QUEUE_TYPE_LOCAL:
637  num_sent = send_local_multi(events, num, q_elem);
638  break;
639  default:
640  num_sent = 0;
641  break;
642  }
643 
644  if (likely(num_sent == num))
645  return num_sent; /* Success */
646 
647 error_return:
648  if (esv_enabled())
649  evstate_usr2em_revert_multi(&events[num_sent],
650  &ev_hdrs[num_sent],
651  num - num_sent,
652  EVSTATE__SEND_MULTI__FAIL);
653 error_return_noesv:
654  if (EM_CHECK_LEVEL > 0)
655  INTERNAL_ERROR(EM_ERR_LIB_FAILED, EM_ESCOPE_SEND_MULTI,
656  "send-multi failed: req:%d, sent:%d",
657  num, num_sent);
658  return num_sent;
659 }
660 
661 em_status_t em_send(em_event_t event, em_queue_t queue)
662 {
663  const bool is_external = queue_external(queue);
664 
666  EM_ERR_BAD_ARG, EM_ESCOPE_SEND, "Invalid event");
667 
668  event_hdr_t *ev_hdr = event_to_hdr(event);
669 
671  EM_ERR_BAD_ARG, EM_ESCOPE_SEND, "Timer-ring event can't be sent");
672 
673  /* avoid unnecessary writing 'undef' in case event is a ref */
674  if (ev_hdr->egrp != EM_EVENT_GROUP_UNDEF)
675  ev_hdr->egrp = EM_EVENT_GROUP_UNDEF;
676 
677  /*
678  * External queue belongs to another EM instance, send out via EMC/BIP
679  */
680  if (is_external)
681  return send_external(event, queue);
682 
683  /*
684  * Queue belongs to this EM instance
685  */
686  return send_internal(event, ev_hdr, queue);
687 }
688 
689 /*
690  * em_send_group_multi() helper: check events
691  */
692 static inline em_status_t
693 send_multi_check_events(const em_event_t events[], int num)
694 {
695  if (EM_CHECK_LEVEL > 0 && unlikely(!events || num <= 0))
696  return EM_ERR_BAD_ARG;
697 
698  if (EM_CHECK_LEVEL >= 3) {
699  int i;
700 
701  for (i = 0; i < num && events[i] != EM_EVENT_UNDEF; i++)
702  ;
703  if (unlikely(i != num))
704  return EM_ERR_BAD_POINTER;
705  }
706 
707  return EM_OK;
708 }
709 
710 int em_send_multi(const em_event_t events[], int num, em_queue_t queue)
711 {
712  const bool is_external = queue_external(queue);
713  event_hdr_t *ev_hdrs[num];
714 
715  /* Check events */
716  em_status_t err = send_multi_check_events(events, num);
717 
718  if (unlikely(err != EM_OK)) {
719  INTERNAL_ERROR(err, EM_ESCOPE_SEND_MULTI,
720  "Invalid events:%p num:%d", events, num);
721  return 0;
722  }
723 
724  event_to_hdr_multi(events, ev_hdrs, num);
725 
726  for (int i = 0; i < num; i++) {
727  if (EM_CHECK_LEVEL > 0 &&
728  unlikely(ev_hdrs[i]->event_type == EM_EVENT_TYPE_TIMER_IND)) {
729  INTERNAL_ERROR(EM_ERR_BAD_ARG, EM_ESCOPE_SEND_MULTI,
730  "Timer-ring event[%d] can't be sent", i);
731  return 0;
732  }
733  /* avoid unnecessary writing 'undef' in case event is a ref */
734  if (ev_hdrs[i]->egrp != EM_EVENT_GROUP_UNDEF)
735  ev_hdrs[i]->egrp = EM_EVENT_GROUP_UNDEF;
736  }
737 
738  /*
739  * External queue belongs to another EM instance, send out via EMC/BIP
740  */
741  if (is_external)
742  return send_external_multi(events, num, queue);
743 
744  /*
745  * Queue belongs to this EM instance
746  */
747  return send_internal_multi(events, ev_hdrs, num, queue);
748 }
749 
750 void *em_event_pointer(em_event_t event)
751 {
752  if (EM_CHECK_LEVEL > 0 && unlikely(event == EM_EVENT_UNDEF)) {
753  INTERNAL_ERROR(EM_ERR_BAD_ARG, EM_ESCOPE_EVENT_POINTER,
754  "event undefined!");
755  return NULL;
756  }
757 
758  void *ev_ptr = event_pointer(event);
759 
760  if (EM_CHECK_LEVEL > 0 && unlikely(!ev_ptr))
761  INTERNAL_ERROR(EM_ERR_BAD_POINTER, EM_ESCOPE_EVENT_POINTER,
762  "Event pointer NULL (unsupported event type)");
763 
764  return ev_ptr;
765 }
766 
767 void *em_event_pointer_and_size(em_event_t event, uint32_t *size /*out*/)
768 {
769  if (EM_CHECK_LEVEL > 0 && unlikely(event == EM_EVENT_UNDEF)) {
770  INTERNAL_ERROR(EM_ERR_BAD_ARG, EM_ESCOPE_EVENT_POINTER_AND_SIZE,
771  "event undefined!");
772  return NULL;
773  }
774 
775  if (!size) {
776  /* User not interested in 'size',
777  * fall back to em_event_pointer() functionality
778  */
779  void *ev_ptr = event_pointer(event);
780 
781  if (EM_CHECK_LEVEL > 0 && unlikely(!ev_ptr))
782  INTERNAL_ERROR(EM_ERR_BAD_POINTER, EM_ESCOPE_EVENT_POINTER_AND_SIZE,
783  "Event pointer NULL (unsupported event type)");
784  return ev_ptr;
785  }
786 
787  const odp_event_t odp_event = event_em2odp(event);
788  const odp_event_type_t odp_etype = odp_event_type(odp_event);
789  uint32_t event_size = 0;
790  void *ev_ptr = NULL; /* return value */
791 
792  if (odp_etype == ODP_EVENT_PACKET) {
793  const odp_packet_t odp_pkt = odp_packet_from_event(odp_event);
794 
795  ev_ptr = odp_packet_data_seg_len(odp_pkt, &event_size);
796  } else if (odp_etype == ODP_EVENT_BUFFER) {
797  const odp_buffer_t odp_buf = odp_buffer_from_event(odp_event);
798  const event_hdr_t *ev_hdr = odp_buffer_user_area(odp_buf);
799  const uint32_t align_offset = ev_hdr->align_offset;
800 
801  ev_ptr = odp_buffer_addr(odp_buf);
802  if (align_offset)
803  ev_ptr = (void *)((uintptr_t)ev_ptr + 32 - align_offset);
804  event_size = ev_hdr->event_size;
805  }
806 
807  if (EM_CHECK_LEVEL > 0 && unlikely(!ev_ptr)) {
808  INTERNAL_ERROR(EM_ERR_BAD_TYPE, EM_ESCOPE_EVENT_POINTER_AND_SIZE,
809  "Event pointer NULL (odp event type:%u)", odp_etype);
810  /* NULL for unrecognized odp_etype, also for vectors and timer ring tmos */
811  return NULL;
812  }
813 
814  *size = event_size;
815  return ev_ptr;
816 }
817 
818 uint32_t em_event_get_size(em_event_t event)
819 {
820  if (unlikely(EM_CHECK_LEVEL > 0 && event == EM_EVENT_UNDEF)) {
821  INTERNAL_ERROR(EM_ERR_BAD_ARG, EM_ESCOPE_EVENT_GET_SIZE,
822  "event undefined!");
823  return 0;
824  }
825 
826  const odp_event_t odp_event = event_em2odp(event);
827  const odp_event_type_t odp_etype = odp_event_type(odp_event);
828 
829  if (odp_etype == ODP_EVENT_PACKET) {
830  odp_packet_t odp_pkt = odp_packet_from_event(odp_event);
831 
832  return odp_packet_seg_len(odp_pkt);
833  } else if (odp_etype == ODP_EVENT_BUFFER) {
834  odp_buffer_t odp_buf = odp_buffer_from_event(odp_event);
835  const event_hdr_t *ev_hdr = odp_buffer_user_area(odp_buf);
836 
837  return ev_hdr->event_size;
838  } else if (odp_etype == ODP_EVENT_TIMEOUT) {
839  return 0;
840  }
841 
842  if (EM_CHECK_LEVEL > 0)
843  INTERNAL_ERROR(EM_ERR_NOT_FOUND, EM_ESCOPE_EVENT_GET_SIZE,
844  "Unexpected odp event type:%u", odp_etype);
845  return 0;
846 }
847 
848 static inline odp_pool_t event_get_odp_pool(em_event_t event)
849 {
850  odp_event_t odp_event = event_em2odp(event);
851 
852 #if ODP_VERSION_API_NUM(1, 43, 0) <= ODP_VERSION_API
853  return odp_event_pool(odp_event);
854 #else
855  odp_event_type_t type = odp_event_type(odp_event);
856  odp_pool_t odp_pool = ODP_POOL_INVALID;
857 
858  if (type == ODP_EVENT_PACKET) {
859  odp_packet_t pkt = odp_packet_from_event(odp_event);
860 
861  odp_pool = odp_packet_pool(pkt);
862  } else if (type == ODP_EVENT_BUFFER) {
863  odp_buffer_t buf = odp_buffer_from_event(odp_event);
864 
865  odp_pool = odp_buffer_pool(buf);
866  } else if (type == ODP_EVENT_PACKET_VECTOR) {
867  odp_packet_vector_t pktvec = odp_packet_vector_from_event(odp_event);
868 
869  odp_pool = odp_packet_vector_pool(pktvec);
870  }
871 
872  return odp_pool;
873 #endif
874 }
875 
876 em_pool_t em_event_get_pool(em_event_t event)
877 {
878  if (EM_CHECK_LEVEL > 0 && unlikely(event == EM_EVENT_UNDEF)) {
879  INTERNAL_ERROR(EM_ERR_BAD_ARG, EM_ESCOPE_EVENT_GET_POOL,
880  "event undefined!");
881  return EM_POOL_UNDEF;
882  }
883 
884  odp_pool_t odp_pool = event_get_odp_pool(event);
885 
886  if (unlikely(odp_pool == ODP_POOL_INVALID))
887  return EM_POOL_UNDEF;
888 
889  em_pool_t pool = pool_odp2em(odp_pool);
890 
891  /*
892  * Don't report an error if 'pool == EM_POOL_UNDEF' since that might
893  * happen if the event is e.g. input from pktio that is using external
894  * (to EM) odp pools.
895  */
896  return pool;
897 }
898 
899 em_pool_t em_event_get_pool_subpool(em_event_t event, int *subpool /*out*/)
900 {
901  if (EM_CHECK_LEVEL > 0 && unlikely(event == EM_EVENT_UNDEF)) {
902  INTERNAL_ERROR(EM_ERR_BAD_ARG, EM_ESCOPE_EVENT_GET_POOL_SUBPOOL,
903  "event undefined!");
904  return EM_POOL_UNDEF;
905  }
906 
907  odp_pool_t odp_pool = event_get_odp_pool(event);
908 
909  if (unlikely(odp_pool == ODP_POOL_INVALID))
910  return EM_POOL_UNDEF;
911 
912  pool_subpool_t pool_subpool = pool_subpool_odp2em(odp_pool);
913 
914  if (unlikely(pool_subpool.pool == (uint32_t)(uintptr_t)EM_POOL_UNDEF))
915  return EM_POOL_UNDEF;
916 
917  if (subpool)
918  *subpool = pool_subpool.subpool;
919 
920  return (em_pool_t)(uintptr_t)pool_subpool.pool;
921 }
922 
924 {
925  if (EM_CHECK_LEVEL > 0)
927  EM_ESCOPE_EVENT_SET_TYPE, "event undefined!");
928 
929  /* similar to 'ev_hdr = event_to_hdr(event)', slightly extended: */
930  odp_event_t odp_event = event_em2odp(event);
931  odp_event_type_t evtype = odp_event_type(odp_event);
932  event_hdr_t *ev_hdr;
933 
934  switch (evtype) {
935  case ODP_EVENT_PACKET: {
936  odp_packet_t odp_pkt = odp_packet_from_event(odp_event);
937 
938  ev_hdr = odp_packet_user_area(odp_pkt);
939  break;
940  }
941  case ODP_EVENT_BUFFER: {
942  odp_buffer_t odp_buf = odp_buffer_from_event(odp_event);
943 
944  ev_hdr = odp_buffer_user_area(odp_buf);
945  break;
946  }
947  case ODP_EVENT_PACKET_VECTOR: {
948  odp_packet_vector_t odp_pktvec = odp_packet_vector_from_event(odp_event);
949  em_event_type_t new_major = em_event_type_major(newtype);
950 
951  if (EM_CHECK_LEVEL >= 1)
953  EM_ERR_BAD_ARG, EM_ESCOPE_EVENT_SET_TYPE,
954  "Event type:0x%x not suitable for a vector", newtype);
955  ev_hdr = odp_packet_vector_user_area(odp_pktvec);
956  break;
957  }
958  default:
959  return INTERNAL_ERROR(EM_ERR_NOT_IMPLEMENTED, EM_ESCOPE_EVENT_SET_TYPE,
960  "Unsupported odp event type:%u", evtype);
961  }
962 
963  ev_hdr->event_type = newtype;
964 
965  return EM_OK;
966 }
967 
969 {
970  const event_hdr_t *ev_hdr;
971 
972  if (EM_CHECK_LEVEL > 0 && unlikely(event == EM_EVENT_UNDEF)) {
973  INTERNAL_ERROR(EM_ERR_BAD_ARG, EM_ESCOPE_EVENT_GET_TYPE,
974  "event undefined!");
975  return EM_EVENT_TYPE_UNDEF;
976  }
977 
978  ev_hdr = event_to_hdr(event);
979 
980  if (EM_CHECK_LEVEL >= 3 && unlikely(ev_hdr == NULL)) {
981  INTERNAL_ERROR(EM_ERR_BAD_POINTER, EM_ESCOPE_EVENT_GET_TYPE,
982  "ev_hdr == NULL");
983  return EM_EVENT_TYPE_UNDEF;
984  }
985 
986  return ev_hdr->event_type;
987 }
988 
989 int em_event_get_type_multi(const em_event_t events[], int num,
990  em_event_type_t types[/*out:num*/])
991 {
992  int i;
993 
994  /* Check all args */
995  if (EM_CHECK_LEVEL > 0) {
996  if (unlikely(!events || num < 0 || !types)) {
998  EM_ESCOPE_EVENT_GET_TYPE_MULTI,
999  "Inv.args: events:%p num:%d types:%p",
1000  events, num, types);
1001  return 0;
1002  }
1003  if (unlikely(!num))
1004  return 0;
1005  }
1006 
1007  if (EM_CHECK_LEVEL >= 3) {
1008  for (i = 0; i < num && events[i] != EM_EVENT_UNDEF; i++)
1009  ;
1010  if (unlikely(i != num)) {
1012  EM_ESCOPE_EVENT_GET_TYPE_MULTI,
1013  "events[%d] undefined!", i);
1014  return 0;
1015  }
1016  }
1017 
1018  event_hdr_t *ev_hdrs[num];
1019 
1020  event_to_hdr_multi(events, ev_hdrs, num);
1021 
1022  for (i = 0; i < num; i++)
1023  types[i] = ev_hdrs[i]->event_type;
1024 
1025  return num;
1026 }
1027 
1028 int em_event_same_type_multi(const em_event_t events[], int num,
1029  em_event_type_t *same_type /*out*/)
1030 {
1031  /* Check all args */
1032  if (EM_CHECK_LEVEL > 0) {
1033  if (unlikely(!events || num < 0 || !same_type)) {
1035  EM_ESCOPE_EVENT_SAME_TYPE_MULTI,
1036  "Inv.args: events:%p num:%d same_type:%p",
1037  events, num, same_type);
1038  return 0;
1039  }
1040  if (unlikely(!num))
1041  return 0;
1042  }
1043 
1044  if (EM_CHECK_LEVEL >= 3) {
1045  int i;
1046 
1047  for (i = 0; i < num && events[i] != EM_EVENT_UNDEF; i++)
1048  ;
1049  if (unlikely(i != num)) {
1051  EM_ESCOPE_EVENT_SAME_TYPE_MULTI,
1052  "events[%d] undefined!", i);
1053  return 0;
1054  }
1055  }
1056 
1057  const em_event_type_t type = event_to_hdr(events[0])->event_type;
1058  int same = 1;
1059 
1060  for (; same < num && type == event_to_hdr(events[same])->event_type;
1061  same++)
1062  ;
1063 
1064  *same_type = type;
1065  return same;
1066 }
1067 
1068 em_status_t em_event_mark_send(em_event_t event, em_queue_t queue)
1069 {
1070  if (!esv_enabled())
1071  return EM_OK;
1072 
1073  /* Check all args */
1074  if (EM_CHECK_LEVEL > 0)
1076  EM_ERR_BAD_ARG, EM_ESCOPE_EVENT_MARK_SEND,
1077  "Inv.args: event:%" PRI_EVENT "", event);
1078  if (EM_CHECK_LEVEL >= 3) {
1079  const queue_elem_t *const q_elem = queue_elem_get(queue);
1080 
1081  RETURN_ERROR_IF(!q_elem, EM_ERR_BAD_ARG, EM_ESCOPE_EVENT_MARK_SEND,
1082  "Inv.args: Q:%" PRI_QUEUE "", queue);
1083  RETURN_ERROR_IF(!queue_allocated(q_elem) || !q_elem->flags.scheduled,
1084  EM_ERR_BAD_STATE, EM_ESCOPE_EVENT_MARK_SEND,
1085  "Inv.queue:%" PRI_QUEUE " type:%" PRI_QTYPE "",
1086  queue, q_elem->type);
1087  }
1088 
1089  event_hdr_t *ev_hdr = event_to_hdr(event);
1090 
1092  EM_ERR_BAD_ARG, EM_ESCOPE_EVENT_MARK_SEND, "Timer-ring event not allowed");
1093 
1094  /* avoid unnecessary writing 'undef' in case event is a ref */
1095  if (ev_hdr->egrp != EM_EVENT_GROUP_UNDEF)
1096  ev_hdr->egrp = EM_EVENT_GROUP_UNDEF;
1097 
1098  evstate_usr2em(event, ev_hdr, EVSTATE__MARK_SEND);
1099 
1100  /*
1101  * Data memory barrier, we are bypassing em_send(), odp_queue_enq()
1102  * and need to guarantee memory sync before the event ends up into an
1103  * EM queue again.
1104  */
1105  odp_mb_full();
1106 
1107  return EM_OK;
1108 }
1109 
1111 {
1112  if (!esv_enabled())
1113  return EM_OK;
1114 
1115  /* Check all args */
1116  if (EM_CHECK_LEVEL > 0)
1118  EM_ERR_BAD_ARG, EM_ESCOPE_EVENT_UNMARK_SEND,
1119  "Inv.args: event:%" PRI_EVENT "", event);
1120 
1121  event_hdr_t *ev_hdr = event_to_hdr(event);
1122 
1124  EM_ERR_BAD_ARG, EM_ESCOPE_EVENT_UNMARK_SEND,
1125  "Timer-ring event not allowed");
1126 
1127  evstate_unmark_send(event, ev_hdr);
1128 
1129  return EM_OK;
1130 }
1131 
1132 void em_event_mark_free(em_event_t event)
1133 {
1134  if (!esv_enabled())
1135  return;
1136 
1137  if (EM_CHECK_LEVEL > 0 && unlikely(event == EM_EVENT_UNDEF)) {
1138  INTERNAL_ERROR(EM_ERR_BAD_ARG, EM_ESCOPE_EVENT_MARK_FREE,
1139  "Event undefined!");
1140  return;
1141  }
1142 
1143  event_hdr_t *const ev_hdr = event_to_hdr(event);
1144 
1145  if (EM_CHECK_LEVEL > 0 && unlikely(ev_hdr->event_type == EM_EVENT_TYPE_TIMER_IND)) {
1146  INTERNAL_ERROR(EM_ERR_BAD_ARG, EM_ESCOPE_EVENT_MARK_FREE,
1147  "Timer-ring event not allowed");
1148  return;
1149  }
1150 
1151  evstate_free(event, ev_hdr, EVSTATE__MARK_FREE);
1152 
1153  if (is_vector_type(event))
1154  event_vector_prepare_free_full(event, EVSTATE__MARK_FREE);
1155 }
1156 
1157 void em_event_unmark_free(em_event_t event)
1158 {
1159  if (!esv_enabled())
1160  return;
1161 
1162  if (EM_CHECK_LEVEL > 0 && unlikely(event == EM_EVENT_UNDEF)) {
1163  INTERNAL_ERROR(EM_ERR_BAD_ARG, EM_ESCOPE_EVENT_UNMARK_FREE,
1164  "Event undefined!");
1165  return;
1166  }
1167 
1168  event_hdr_t *const ev_hdr = event_to_hdr(event);
1169 
1170  if (EM_CHECK_LEVEL > 0 && unlikely(ev_hdr->event_type == EM_EVENT_TYPE_TIMER_IND)) {
1171  INTERNAL_ERROR(EM_ERR_BAD_ARG, EM_ESCOPE_EVENT_UNMARK_FREE,
1172  "Timer-ring event not allowed");
1173  return;
1174  }
1175 
1176  evstate_unmark_free(event, ev_hdr, EVSTATE__UNMARK_FREE);
1177  if (is_vector_type(event))
1178  event_vector_prepare_free_full__revert(event, EVSTATE__UNMARK_FREE);
1179 }
1180 
1181 void em_event_mark_free_multi(const em_event_t events[], int num)
1182 {
1183  if (!esv_enabled())
1184  return;
1185 
1186  if (EM_CHECK_LEVEL > 0 && unlikely(!events || num < 0)) {
1187  INTERNAL_ERROR(EM_ERR_BAD_ARG, EM_ESCOPE_EVENT_MARK_FREE_MULTI,
1188  "Inv.args: events[]:%p num:%d", events, num);
1189  return;
1190  }
1191  if (unlikely(num == 0))
1192  return;
1193 
1194  if (EM_CHECK_LEVEL >= 3) {
1195  int i;
1196 
1197  for (i = 0; i < num && events[i] != EM_EVENT_UNDEF; i++)
1198  ;
1199  if (unlikely(i != num)) {
1201  EM_ESCOPE_EVENT_MARK_FREE_MULTI,
1202  "events[%d] undefined!", i);
1203  return;
1204  }
1205  }
1206 
1207  event_hdr_t *ev_hdrs[num];
1208 
1209  event_to_hdr_multi(events, ev_hdrs, num);
1210 
1211  for (int i = 0; i < num; i++) {
1212  if (EM_CHECK_LEVEL > 0 &&
1213  unlikely(ev_hdrs[i]->event_type == EM_EVENT_TYPE_TIMER_IND)) {
1214  INTERNAL_ERROR(EM_ERR_BAD_ARG, EM_ESCOPE_EVENT_MARK_FREE_MULTI,
1215  "Timer-ring event[%d] not allowed", i);
1216  continue;
1217  }
1218 
1219  evstate_free(events[i], ev_hdrs[i], EVSTATE__MARK_FREE_MULTI);
1220  if (is_vector_type(events[i]))
1221  event_vector_prepare_free_full(events[i], EVSTATE__MARK_FREE_MULTI);
1222  }
1223 }
1224 
1225 void em_event_unmark_free_multi(const em_event_t events[], int num)
1226 {
1227  if (!esv_enabled())
1228  return;
1229 
1230  if (unlikely(!events || num < 0)) {
1231  INTERNAL_ERROR(EM_ERR_BAD_ARG, EM_ESCOPE_EVENT_UNMARK_FREE_MULTI,
1232  "Inv.args: events[]:%p num:%d", events, num);
1233  return;
1234  }
1235  if (unlikely(num == 0))
1236  return;
1237 
1238  if (EM_CHECK_LEVEL >= 3) {
1239  int i;
1240 
1241  for (i = 0; i < num && events[i] != EM_EVENT_UNDEF; i++)
1242  ;
1243  if (unlikely(i != num)) {
1245  EM_ESCOPE_EVENT_UNMARK_FREE_MULTI,
1246  "events[%d] undefined!", i);
1247  return;
1248  }
1249  }
1250 
1251  event_hdr_t *ev_hdrs[num];
1252 
1253  event_to_hdr_multi(events, ev_hdrs, num);
1254 
1255  for (int i = 0; i < num; i++) {
1256  if (EM_CHECK_LEVEL > 0 &&
1257  unlikely(ev_hdrs[i]->event_type == EM_EVENT_TYPE_TIMER_IND)) {
1258  INTERNAL_ERROR(EM_ERR_BAD_ARG, EM_ESCOPE_EVENT_UNMARK_FREE_MULTI,
1259  "Timer-ring event[%d] not allowed", i);
1260  continue;
1261  }
1262 
1263  evstate_unmark_free(events[i], ev_hdrs[i], EVSTATE__UNMARK_FREE_MULTI);
1264  if (is_vector_type(events[i]))
1265  event_vector_prepare_free_full__revert(events[i],
1266  EVSTATE__UNMARK_FREE_MULTI);
1267  }
1268 }
1269 
1270 static em_event_t event_clone_part(em_event_t event, em_pool_t pool/*or EM_POOL_UNDEF*/,
1271  uint32_t offset, uint32_t len, bool clone_uarea,
1272  em_escope_t escope)
1273 {
1274  const mpool_elem_t *pool_elem = pool_elem_get(pool);
1275  /* use escope to distinguish between em_event_clone() and em_event_clone_part() */
1276  const bool is_clone_part = escope == EM_ESCOPE_EVENT_CLONE_PART ? true : false;
1277 
1278  /* Check all args */
1279  if (EM_CHECK_LEVEL > 0 &&
1280  unlikely(event == EM_EVENT_UNDEF ||
1281  (pool != EM_POOL_UNDEF && !pool_elem))) {
1283  "Inv.args: event:%" PRI_EVENT " pool:%" PRI_POOL "",
1284  event, pool);
1285  return EM_EVENT_UNDEF;
1286  }
1287 
1288  if (EM_CHECK_LEVEL >= 2 &&
1289  unlikely(pool_elem && !pool_allocated(pool_elem))) {
1291  "Inv.args: pool:%" PRI_POOL " not created", pool);
1292  return EM_EVENT_UNDEF;
1293  }
1294 
1295  odp_event_t odp_event = event_em2odp(event);
1296  odp_event_type_t odp_evtype = odp_event_type(odp_event);
1297  odp_pool_t odp_pool = ODP_POOL_INVALID;
1298  odp_packet_t pkt = ODP_PACKET_INVALID;
1299  odp_buffer_t buf = ODP_BUFFER_INVALID;
1300 
1301  if (unlikely(odp_evtype != ODP_EVENT_PACKET &&
1302  odp_evtype != ODP_EVENT_BUFFER)) {
1303  INTERNAL_ERROR(EM_ERR_BAD_ID, escope,
1304  "Inv. odp-event-type:%d", odp_evtype);
1305  return EM_EVENT_UNDEF;
1306  }
1307 
1308  /* Obtain the event-hdr, event-size and the pool to use */
1309  const event_hdr_t *ev_hdr;
1310  uint32_t size;
1311  em_event_type_t type;
1312  em_pool_t em_pool = pool;
1313  event_hdr_t *clone_hdr;
1314  em_event_t clone_event; /* return value */
1315 
1316  if (odp_evtype == ODP_EVENT_PACKET) {
1317  pkt = odp_packet_from_event(odp_event);
1318  ev_hdr = odp_packet_user_area(pkt);
1319  size = odp_packet_seg_len(pkt);
1320  if (pool == EM_POOL_UNDEF) {
1321  odp_pool = odp_packet_pool(pkt);
1322  em_pool = pool_odp2em(odp_pool);
1323  }
1324  } else /* ODP_EVENT_BUFFER */ {
1325  buf = odp_buffer_from_event(odp_event);
1326  ev_hdr = odp_buffer_user_area(buf);
1327  size = ev_hdr->event_size;
1328  if (pool == EM_POOL_UNDEF) {
1329  odp_pool = odp_buffer_pool(buf);
1330  em_pool = pool_odp2em(odp_pool);
1331  }
1332  }
1333 
1334  if (is_clone_part) {
1335  if (EM_CHECK_LEVEL >= 1) {
1336  uint64_t offset64 = offset;
1337  uint64_t len64 = len;
1338  uint64_t size64 = size;
1339 
1340  if (unlikely(len == 0 || offset64 + len64 > size64)) {
1342  "Inv.args: offset=%u len=%u (0 < offset+len <= %u)",
1343  offset, len, size);
1344  return EM_EVENT_UNDEF;
1345  }
1346  }
1347  if (len < size)
1348  size = len;
1349  }
1350 
1351  /* No EM-pool found */
1352  if (em_pool == EM_POOL_UNDEF) {
1353  if (unlikely(odp_evtype == ODP_EVENT_BUFFER)) {
1355  "No suitable event-pool found");
1356  return EM_EVENT_UNDEF;
1357  }
1358  /* odp_evtype == ODP_EVENT_PACKET:
1359  * Not an EM-pool, e.g. event from external pktio odp-pool.
1360  * Allocate and clone pkt via ODP directly.
1361  */
1362  clone_event = pkt_clone_odp(pkt, odp_pool, offset, size, is_clone_part);
1363  if (unlikely(clone_event == EM_EVENT_UNDEF)) {
1365  "Cloning from ext odp-pool:%" PRIu64 " failed",
1366  odp_pool_to_u64(odp_pool));
1367  }
1368  return clone_event;
1369  }
1370 
1371  /*
1372  * Clone the event from an EM-pool:
1373  */
1374  if (em_pool != pool)
1375  pool_elem = pool_elem_get(em_pool);
1376  type = ev_hdr->event_type;
1377 
1378  /* EM event pools created with type=SW can not support pkt events */
1379  if (unlikely(EM_CHECK_LEVEL > 0 &&
1380  pool_elem->event_type == EM_EVENT_TYPE_SW &&
1381  em_event_type_major(type) == EM_EVENT_TYPE_PACKET)) {
1383  "EM-pool:%s(%" PRI_POOL "):\n"
1384  "Invalid event type:0x%x for buf",
1385  pool_elem->name, em_pool, type);
1386  return EM_EVENT_UNDEF;
1387  }
1388 
1389  if (EM_CHECK_LEVEL > 0 &&
1390  unlikely(clone_uarea && ev_hdr->user_area.isinit &&
1391  pool_elem->user_area.size < ev_hdr->user_area.size)) {
1393  "EM-pool:%s(%" PRI_POOL "):\n"
1394  "Available user-area too small, clone uarea %u < needed uarea %u",
1395  pool_elem->name, em_pool, pool_elem->user_area.size,
1396  ev_hdr->user_area.size);
1397  return EM_EVENT_UNDEF;
1398  }
1399 
1400  if (pool_elem->event_type == EM_EVENT_TYPE_PACKET)
1401  clone_hdr = event_alloc_pkt(pool_elem, size);
1402  else /* EM_EVENT_TYPE_SW */
1403  clone_hdr = event_alloc_buf(pool_elem, size);
1404 
1405  if (unlikely(!clone_hdr)) {
1407  "EM-pool:'%s': sz:%u type:0x%x pool:%" PRI_POOL "",
1408  pool_elem->name, size, type, em_pool);
1409  if (EM_DEBUG_PRINT && err != EM_OK &&
1410  (pool_elem->stats_opt.bit.available ||
1411  pool_elem->stats_opt.bit.cache_available))
1412  em_pool_info_print(em_pool);
1413  return EM_EVENT_UNDEF;
1414  }
1415 
1416  /* Update event ESV state for alloc/clone */
1417  if (esv_enabled())
1418  (void)evstate_alloc(clone_hdr->event, clone_hdr, EVSTATE__EVENT_CLONE);
1419 
1420  clone_hdr->flags.all = 0; /* clear only after evstate_alloc() */
1421  clone_hdr->event_type = type; /* store the event type */
1422  clone_hdr->event_size = size; /* store requested size */
1423  clone_hdr->egrp = EM_EVENT_GROUP_UNDEF;
1424  clone_hdr->user_area.all = ev_hdr->user_area.all;
1425  clone_hdr->user_area.size = pool_elem->user_area.size; /* uarea size comes from pool */
1426  clone_hdr->user_area.isinit = 1;
1427 
1428  /* Copy the event uarea content if used */
1429  if (clone_uarea &&
1430  ev_hdr->user_area.isinit && ev_hdr->user_area.size > 0) {
1431  const void *uarea_ptr = (void *)((uintptr_t)ev_hdr + sizeof(event_hdr_t));
1432  void *clone_uarea_ptr = (void *)((uintptr_t)clone_hdr + sizeof(event_hdr_t));
1433  size_t sz = MIN(pool_elem->user_area.size, ev_hdr->user_area.size);
1434 
1435  memcpy(clone_uarea_ptr, uarea_ptr, sz);
1436  }
1437 
1438  clone_event = clone_hdr->event;
1439 
1440  /* Copy event payload from the parent event into the clone event */
1441  uintptr_t src_addr = (uintptr_t)event_pointer(event) + offset;
1442  const void *src = (void *)src_addr;
1443  void *dst = event_pointer(clone_event);
1444 
1445  memcpy(dst, src, size);
1446 
1447  /* Call the 'alloc' API hook function also for event-clone */
1448  if (EM_API_HOOKS_ENABLE && clone_event != EM_EVENT_UNDEF)
1449  call_api_hooks_alloc(&clone_event, 1, 1, size, type, pool);
1450 
1451  return clone_event;
1452 }
1453 
1454 em_event_t em_event_clone(em_event_t event, em_pool_t pool/*or EM_POOL_UNDEF*/)
1455 {
1456  return event_clone_part(event, pool, 0, 0, true, EM_ESCOPE_EVENT_CLONE);
1457 }
1458 
1459 em_event_t em_event_clone_part(em_event_t event, em_pool_t pool/*or EM_POOL_UNDEF*/,
1460  uint32_t offset, uint32_t len, bool clone_uarea)
1461 {
1462  return event_clone_part(event, pool, offset, len, clone_uarea,
1463  EM_ESCOPE_EVENT_CLONE_PART);
1464 }
1465 
1466 static inline int
1467 event_uarea_init(em_event_t event, event_hdr_t **ev_hdr/*out*/)
1468 {
1469  const odp_event_t odp_event = event_em2odp(event);
1470  const odp_event_type_t odp_evtype = odp_event_type(odp_event);
1471  odp_pool_t odp_pool = ODP_POOL_INVALID;
1472  odp_packet_t odp_pkt;
1473  odp_buffer_t odp_buf;
1474  odp_packet_vector_t odp_pktvec;
1475  event_hdr_t *hdr;
1476  bool is_init;
1477 
1478  switch (odp_evtype) {
1479  case ODP_EVENT_PACKET:
1480  odp_pkt = odp_packet_from_event(odp_event);
1481  hdr = odp_packet_user_area(odp_pkt);
1482  is_init = hdr->user_area.isinit;
1483  if (!is_init)
1484  odp_pool = odp_packet_pool(odp_pkt);
1485  break;
1486  case ODP_EVENT_BUFFER:
1487  odp_buf = odp_buffer_from_event(odp_event);
1488  hdr = odp_buffer_user_area(odp_buf);
1489  is_init = hdr->user_area.isinit;
1490  if (!is_init)
1491  odp_pool = odp_buffer_pool(odp_buf);
1492  break;
1493  case ODP_EVENT_PACKET_VECTOR:
1494  odp_pktvec = odp_packet_vector_from_event(odp_event);
1495  hdr = odp_packet_vector_user_area(odp_pktvec);
1496  is_init = hdr->user_area.isinit;
1497  if (!is_init)
1498  odp_pool = odp_packet_vector_pool(odp_pktvec);
1499  break;
1500  default:
1501  return -1;
1502  }
1503 
1504  *ev_hdr = hdr;
1505 
1506  if (!is_init) {
1507  /*
1508  * Event user area metadata is not initialized in
1509  * the event header - initialize it:
1510  */
1511  hdr->user_area.all = 0; /* user_area.{} = all zero (.sizes=0) */
1512  hdr->user_area.isinit = 1;
1513 
1514  em_pool_t pool = pool_odp2em(odp_pool);
1515 
1516  if (pool == EM_POOL_UNDEF)
1517  return 0; /* ext ODP pool: OK, no user area, sz=0 */
1518 
1519  /* Event from an EM event pool, can init event user area */
1520  const mpool_elem_t *pool_elem = pool_elem_get(pool);
1521 
1522  if (unlikely(!pool_elem))
1523  return -2; /* invalid pool_elem */
1524 
1525  hdr->user_area.size = pool_elem->user_area.size;
1526  }
1527 
1528  return 0;
1529 }
1530 
1531 void *em_event_uarea_get(em_event_t event, size_t *size /*out, if given*/)
1532 {
1533  /* Check args */
1534  if (EM_CHECK_LEVEL > 0 && unlikely(event == EM_EVENT_UNDEF)) {
1535  INTERNAL_ERROR(EM_ERR_BAD_ARG, EM_ESCOPE_EVENT_UAREA_GET,
1536  "Inv.arg: event undef");
1537  goto no_uarea;
1538  }
1539 
1540  event_hdr_t *ev_hdr = NULL;
1541  int err = event_uarea_init(event, &ev_hdr/*out*/);
1542 
1543  if (EM_CHECK_LEVEL > 0 && unlikely(err)) {
1544  INTERNAL_ERROR(EM_ERR_OPERATION_FAILED, EM_ESCOPE_EVENT_UAREA_GET,
1545  "Cannot init event user area: %d", err);
1546  goto no_uarea;
1547  }
1548 
1549  if (ev_hdr->user_area.size == 0)
1550  goto no_uarea;
1551 
1552  /*
1553  * Event has user area configured, return pointer and size
1554  */
1555  void *uarea_ptr = (void *)((uintptr_t)ev_hdr + sizeof(event_hdr_t));
1556 
1557  if (size)
1558  *size = ev_hdr->user_area.size;
1559 
1560  return uarea_ptr;
1561 
1562 no_uarea:
1563  if (size)
1564  *size = 0;
1565  return NULL;
1566 }
1567 
1568 em_status_t em_event_uarea_id_set(em_event_t event, uint16_t id)
1569 {
1570  /* Check args */
1571  if (EM_CHECK_LEVEL > 0)
1573  EM_ERR_BAD_ARG, EM_ESCOPE_EVENT_UAREA_ID_SET,
1574  "Inv.arg: event undef");
1575 
1576  event_hdr_t *ev_hdr = NULL;
1577  int err = event_uarea_init(event, &ev_hdr/*out*/);
1578 
1579  if (EM_CHECK_LEVEL > 0)
1581  EM_ESCOPE_EVENT_UAREA_ID_SET,
1582  "Cannot init event user area: %d", err);
1583 
1584  ev_hdr->user_area.id = id;
1585  ev_hdr->user_area.isset_id = 1;
1586 
1587  return EM_OK;
1588 }
1589 
1590 em_status_t em_event_uarea_id_get(em_event_t event, bool *isset /*out*/,
1591  uint16_t *id /*out*/)
1592 {
1593  bool id_set = false;
1594  em_status_t status = EM_OK;
1595 
1596  /* Check args, either 'isset' or 'id' ptrs must be provided (or both) */
1597  if (EM_CHECK_LEVEL > 0 &&
1598  (event == EM_EVENT_UNDEF || !(id || isset))) {
1599  status = INTERNAL_ERROR(EM_ERR_BAD_ARG, EM_ESCOPE_EVENT_UAREA_ID_GET,
1600  "Inv.args: event:%" PRI_EVENT " isset:%p id:%p",
1601  event, isset, id);
1602  goto id_isset;
1603  }
1604 
1605  event_hdr_t *ev_hdr = NULL;
1606  int err = event_uarea_init(event, &ev_hdr/*out*/);
1607 
1608  if (EM_CHECK_LEVEL > 0 && unlikely(err)) {
1610  EM_ESCOPE_EVENT_UAREA_ID_GET,
1611  "Cannot init event user area: %d", err);
1612  goto id_isset;
1613  }
1614 
1615  if (ev_hdr->user_area.isset_id) {
1616  /* user-area-id has been set */
1617  id_set = true;
1618  if (id)
1619  *id = ev_hdr->user_area.id; /*out*/
1620  }
1621 
1622 id_isset:
1623  if (isset)
1624  *isset = id_set; /*out*/
1625  return status;
1626 }
1627 
1629  em_event_uarea_info_t *uarea_info /*out*/)
1630 {
1631  em_status_t status = EM_ERROR;
1632 
1633  /* Check args */
1634  if (EM_CHECK_LEVEL > 0 &&
1635  unlikely(event == EM_EVENT_UNDEF || !uarea_info)) {
1636  status = INTERNAL_ERROR(EM_ERR_BAD_ARG, EM_ESCOPE_EVENT_UAREA_INFO,
1637  "Inv.args: event:%" PRI_EVENT " uarea_info:%p",
1638  event, uarea_info);
1639  goto err_uarea;
1640  }
1641 
1642  event_hdr_t *ev_hdr = NULL;
1643  int err = event_uarea_init(event, &ev_hdr/*out*/);
1644 
1645  if (EM_CHECK_LEVEL > 0 && unlikely(err)) {
1647  EM_ESCOPE_EVENT_UAREA_INFO,
1648  "Cannot init event user area: %d", err);
1649  goto err_uarea;
1650  }
1651 
1652  if (ev_hdr->user_area.size == 0) {
1653  uarea_info->uarea = NULL;
1654  uarea_info->size = 0;
1655  } else {
1656  uarea_info->uarea = (void *)((uintptr_t)ev_hdr +
1657  sizeof(event_hdr_t));
1658  uarea_info->size = ev_hdr->user_area.size;
1659  }
1660 
1661  if (ev_hdr->user_area.isset_id) {
1662  uarea_info->id.isset = true;
1663  uarea_info->id.value = ev_hdr->user_area.id;
1664  } else {
1665  uarea_info->id.isset = false;
1666  uarea_info->id.value = 0;
1667  }
1668 
1669  return EM_OK;
1670 
1671 err_uarea:
1672  if (uarea_info) {
1673  uarea_info->uarea = NULL;
1674  uarea_info->size = 0;
1675  uarea_info->id.isset = false;
1676  uarea_info->id.value = 0;
1677  }
1678  return status;
1679 }
1680 
1681 em_event_t em_event_ref(em_event_t event)
1682 {
1683  /* Check args */
1684  if (unlikely(EM_CHECK_LEVEL > 0 && event == EM_EVENT_UNDEF)) {
1685  INTERNAL_ERROR(EM_ERR_BAD_ARG, EM_ESCOPE_EVENT_REF,
1686  "Invalid arg: event:%" PRI_EVENT "", event);
1687  return EM_EVENT_UNDEF;
1688  }
1689 
1690  odp_event_t odp_event = event_em2odp(event);
1691  odp_event_type_t odp_etype = odp_event_type(odp_event);
1692 
1693  if (EM_CHECK_LEVEL > 0 && unlikely(odp_etype != ODP_EVENT_PACKET)) {
1694  INTERNAL_ERROR(EM_ERR_NOT_IMPLEMENTED, EM_ESCOPE_EVENT_REF,
1695  "Event not a packet! Refs not supported for odp-events of type:%d",
1696  odp_etype);
1697  return EM_EVENT_UNDEF;
1698  }
1699 
1700  odp_packet_t odp_pkt = odp_packet_from_event(odp_event);
1701  odp_packet_t pkt_ref = odp_packet_ref_static(odp_pkt);
1702  event_hdr_t *ev_hdr = odp_packet_user_area(odp_pkt);
1703 
1704  if (EM_CHECK_LEVEL > 0 && unlikely(pkt_ref == ODP_PACKET_INVALID)) {
1705  INTERNAL_ERROR(EM_ERR_LIB_FAILED, EM_ESCOPE_EVENT_REF,
1706  "ODP failure in odp_packet_ref_static()");
1707  return EM_EVENT_UNDEF;
1708  }
1709 
1710  if (unlikely(EM_CHECK_LEVEL >= 2 && odp_pkt != pkt_ref)) {
1711  INTERNAL_ERROR(EM_FATAL(EM_ERR_NOT_IMPLEMENTED), EM_ESCOPE_EVENT_REF,
1712  "EM assumes all refs use the same handle");
1713  odp_packet_free(odp_pkt);
1714  return EM_EVENT_UNDEF;
1715  }
1716 
1717  /*
1718  * Indicate that this event has references and some of the ESV checks
1719  * must be omitted (evgen) - 'refs_used' will be set for the whole
1720  * lifetime of this event, i.e. until the event is freed back into the
1721  * pool. Important only for the first call of em_event_ref(), subsequent
1722  * calls write same value.
1723  */
1724  ev_hdr->flags.refs_used = 1;
1725 
1726  em_event_t ref = event;
1727 
1728  if (esv_enabled())
1729  ref = evstate_ref(event, ev_hdr);
1730 
1731  return ref;
1732 }
1733 
1734 bool em_event_has_ref(em_event_t event)
1735 {
1736  /* Check args */
1737  if (unlikely(EM_CHECK_LEVEL > 0 && event == EM_EVENT_UNDEF)) {
1738  INTERNAL_ERROR(EM_ERR_BAD_ARG, EM_ESCOPE_EVENT_HAS_REF,
1739  "Invalid arg: event:%" PRI_EVENT "", event);
1740  return false;
1741  }
1742 
1743  return event_has_ref(event);
1744 }
1745 
1746 void em_event_vector_free(em_event_t vector_event)
1747 {
1748  if (EM_CHECK_LEVEL > 0 &&
1749  unlikely(vector_event == EM_EVENT_UNDEF)) {
1750  INTERNAL_ERROR(EM_ERR_BAD_ARG, EM_ESCOPE_EVENT_VECTOR_FREE,
1751  "Invalid args: vector_event:%" PRI_EVENT "",
1752  vector_event);
1753  return;
1754  }
1755 
1756  if (EM_CHECK_LEVEL > 2 &&
1757  unlikely(!is_vector_type_or_error(vector_event, EM_ESCOPE_EVENT_VECTOR_FREE))) {
1758  return;
1759  }
1760 
1761  if (EM_API_HOOKS_ENABLE)
1762  call_api_hooks_free(&vector_event, 1);
1763 
1764  if (esv_enabled()) {
1765  event_hdr_t *const ev_hdr = eventvec_to_hdr(vector_event);
1766 
1767  evstate_free(vector_event, ev_hdr, EVSTATE__EVENT_VECTOR_FREE);
1768  }
1769 
1770  odp_event_t odp_event = event_em2odp(vector_event);
1771  odp_packet_vector_t pkt_vec = odp_packet_vector_from_event(odp_event);
1772 
1773  odp_packet_vector_free(pkt_vec);
1774 }
1775 
1776 uint32_t em_event_vector_tbl(em_event_t vector_event,
1777  em_event_t **event_tbl/*out*/)
1778 {
1779  if (EM_CHECK_LEVEL > 0 &&
1780  unlikely(vector_event == EM_EVENT_UNDEF || !event_tbl)) {
1781  INTERNAL_ERROR(EM_ERR_BAD_ARG, EM_ESCOPE_EVENT_VECTOR_TBL,
1782  "Invalid args: vector_event:%" PRI_EVENT " event_tbl:%p",
1783  vector_event, event_tbl);
1784  return 0;
1785  }
1786 
1787  if (EM_CHECK_LEVEL > 2 &&
1788  unlikely(!is_vector_type_or_error(vector_event, EM_ESCOPE_EVENT_VECTOR_TBL))) {
1789  *event_tbl = NULL;
1790  return 0;
1791  }
1792 
1793  return event_vector_tbl(vector_event, event_tbl /*out*/);
1794 }
1795 
1796 uint32_t em_event_vector_size(em_event_t vector_event)
1797 {
1798  if (EM_CHECK_LEVEL > 0 &&
1799  unlikely(vector_event == EM_EVENT_UNDEF)) {
1800  INTERNAL_ERROR(EM_ERR_BAD_ARG, EM_ESCOPE_EVENT_VECTOR_SIZE,
1801  "Invalid arg, vector_event undefined!", vector_event);
1802  return 0;
1803  }
1804 
1805  if (EM_CHECK_LEVEL > 2 &&
1806  unlikely(!is_vector_type_or_error(vector_event, EM_ESCOPE_EVENT_VECTOR_SIZE)))
1807  return 0;
1808 
1809  odp_event_t odp_event = event_em2odp(vector_event);
1810  odp_packet_vector_t pkt_vec = odp_packet_vector_from_event(odp_event);
1811 
1812  return odp_packet_vector_size(pkt_vec);
1813 }
1814 
1815 void em_event_vector_size_set(em_event_t vector_event, uint32_t size)
1816 {
1817  if (EM_CHECK_LEVEL > 0 &&
1818  unlikely(vector_event == EM_EVENT_UNDEF)) {
1819  INTERNAL_ERROR(EM_ERR_BAD_ARG, EM_ESCOPE_EVENT_VECTOR_SIZE_SET,
1820  "Invalid arg, vector_event undefined!", vector_event);
1821  return;
1822  }
1823 
1824  if (EM_CHECK_LEVEL > 2 &&
1825  unlikely(!is_vector_type_or_error(vector_event, EM_ESCOPE_EVENT_VECTOR_SIZE_SET)))
1826  return;
1827 
1828  odp_event_t odp_event = event_em2odp(vector_event);
1829  odp_packet_vector_t pkt_vec = odp_packet_vector_from_event(odp_event);
1830 
1831  odp_packet_vector_size_set(pkt_vec, size);
1832 }
1833 
1834 uint32_t em_event_vector_max_size(em_event_t vector_event)
1835 {
1836  if (EM_CHECK_LEVEL > 0 &&
1837  unlikely(vector_event == EM_EVENT_UNDEF)) {
1838  INTERNAL_ERROR(EM_ERR_BAD_ARG, EM_ESCOPE_EVENT_VECTOR_MAX_SIZE,
1839  "Invalid arg, vector_event undefined!", vector_event);
1840  return 0;
1841  }
1842 
1843  if (EM_CHECK_LEVEL > 2 &&
1844  unlikely(!is_vector_type_or_error(vector_event, EM_ESCOPE_EVENT_VECTOR_MAX_SIZE)))
1845  return 0;
1846 
1847  uint32_t max_size = 0;
1848  em_status_t err = event_vector_max_size(vector_event, &max_size,
1849  EM_ESCOPE_EVENT_VECTOR_MAX_SIZE);
1850  if (unlikely(err != EM_OK))
1851  return 0;
1852 
1853  return max_size;
1854 }
1855 
1856 em_status_t em_event_vector_info(em_event_t vector_event,
1857  em_event_vector_info_t *vector_info /*out*/)
1858 {
1859  em_status_t status = EM_ERROR;
1860 
1861  /* Check args */
1862  if (EM_CHECK_LEVEL > 0 &&
1863  unlikely(vector_event == EM_EVENT_UNDEF || !vector_info)) {
1864  status = INTERNAL_ERROR(EM_ERR_BAD_ARG, EM_ESCOPE_EVENT_VECTOR_INFO,
1865  "Invalid args: vector_event:%" PRI_EVENT " vector_info:%p",
1866  vector_event, vector_info);
1867  goto err_vecinfo;
1868  }
1869 
1870  if (EM_CHECK_LEVEL > 2 &&
1871  unlikely(!is_vector_type_or_error(vector_event, EM_ESCOPE_EVENT_VECTOR_INFO))) {
1872  status = EM_ERR_BAD_TYPE;
1873  goto err_vecinfo;
1874  }
1875 
1876  /* Get the max size */
1877  status = event_vector_max_size(vector_event, &vector_info->max_size,
1878  EM_ESCOPE_EVENT_VECTOR_INFO);
1879  if (unlikely(status != EM_OK))
1880  goto err_vecinfo;
1881 
1882  /* Get vector size and the event-table */
1883  vector_info->size = event_vector_tbl(vector_event, &vector_info->event_tbl/*out*/);
1884 
1885  return EM_OK;
1886 
1887 err_vecinfo:
1888  if (vector_info) {
1889  vector_info->event_tbl = NULL;
1890  vector_info->size = 0;
1891  vector_info->max_size = 0;
1892  }
1893  return status;
1894 }
1895 
1896 uint64_t em_event_to_u64(em_event_t event)
1897 {
1898  return (uint64_t)event;
1899 }
EM_TMO_TYPE_NONE
@ EM_TMO_TYPE_NONE
Definition: api/event_machine_timer.h:251
mpool_elem_t::size
uint16_t size
Definition: em_pool_types.h:55
em_event_mark_send
em_status_t em_event_mark_send(em_event_t event, em_queue_t queue)
Definition: event_machine_event.c:1068
EM_EVENT_TYPE_VECTOR
@ EM_EVENT_TYPE_VECTOR
Definition: event_machine_hw_types.h:84
mpool_elem_t::em_pool
em_pool_t em_pool
Definition: em_pool_types.h:71
em_event_uarea_info_t::size
size_t size
Definition: event_machine_event.h:885
EM_OK
#define EM_OK
Definition: event_machine_types.h:329
em_event_vector_info_t::event_tbl
em_event_t * event_tbl
Definition: event_machine_event.h:1134
evstate_unmark_free
void evstate_unmark_free(const em_event_t event, event_hdr_t *const ev_hdr, const uint16_t api_op)
Definition: em_event_state.c:1094
event_hdr::event_size
uint32_t event_size
Definition: em_event_types.h:255
EM_EVENT_TYPE_SW
@ EM_EVENT_TYPE_SW
Definition: event_machine_hw_types.h:72
mpool_elem_t::user_area
struct mpool_elem_t::@54 user_area
ev_hdr_user_area_t::isset_id
uint32_t isset_id
Definition: em_event_types.h:166
event_hdr_t
struct event_hdr event_hdr_t
em_alloc_multi
int em_alloc_multi(em_event_t events[], int num, uint32_t size, em_event_type_t type, em_pool_t pool)
Definition: event_machine_event.c:92
pool_subpool_t
Definition: em_pool_types.h:100
EM_ERR_NOT_CREATED
@ EM_ERR_NOT_CREATED
Definition: event_machine_hw_types.h:274
em_event_clone
em_event_t em_event_clone(em_event_t event, em_pool_t pool)
Clone an event.
Definition: event_machine_event.c:1454
EM_EVENT_UNDEF
#define EM_EVENT_UNDEF
Definition: event_machine_types.h:62
evstate_free_multi
void evstate_free_multi(const em_event_t ev_tbl[], event_hdr_t *const ev_hdr_tbl[], const int num, const uint16_t api_op)
Definition: em_event_state.c:932
em_event_uarea_info_t::value
uint16_t value
Definition: event_machine_event.h:895
EM_API_HOOKS_ENABLE
#define EM_API_HOOKS_ENABLE
Definition: event_machine_config.h:182
EM_QUEUE_TYPE_PARALLEL
@ EM_QUEUE_TYPE_PARALLEL
Definition: event_machine_hw_types.h:117
evstate_usr2em_revert_multi
void evstate_usr2em_revert_multi(const em_event_t ev_tbl[], event_hdr_t *const ev_hdr_tbl[], const int num, const uint16_t api_op)
Definition: em_event_state.c:1018
em_event_uarea_id_set
em_status_t em_event_uarea_id_set(em_event_t event, uint16_t id)
Set the event user area ID.
Definition: event_machine_event.c:1568
EM_ERR_TOO_SMALL
@ EM_ERR_TOO_SMALL
Definition: event_machine_hw_types.h:296
event_hdr::event_type
em_event_type_t event_type
Definition: em_event_types.h:241
PRI_POOL
#define PRI_POOL
Definition: event_machine_hw_types.h:62
queue_elem_t::type
uint8_t type
Definition: em_queue_types.h:216
em_event_vector_info
em_status_t em_event_vector_info(em_event_t vector_event, em_event_vector_info_t *vector_info)
Retrieve information about the given vector event.
Definition: event_machine_event.c:1856
em_locm
ENV_LOCAL em_locm_t em_locm
em_event_vector_info_t
Vector event information filled by em_event_vector_info()
Definition: event_machine_event.h:1132
em_free
void em_free(em_event_t event)
Definition: event_machine_event.c:261
evstate_usr2em_revert
void evstate_usr2em_revert(em_event_t event, event_hdr_t *const ev_hdr, const uint16_t api_op)
Definition: em_event_state.c:999
em_event_mark_free_multi
void em_event_mark_free_multi(const em_event_t events[], int num)
Mark multiple events as "free".
Definition: event_machine_event.c:1181
em_send
em_status_t em_send(em_event_t event, em_queue_t queue)
Definition: event_machine_event.c:661
EM_ERR_ALLOC_FAILED
@ EM_ERR_ALLOC_FAILED
Definition: event_machine_hw_types.h:287
EM_ERR_LIB_FAILED
@ EM_ERR_LIB_FAILED
Definition: event_machine_hw_types.h:291
event_hdr::refs_used
uint8_t refs_used
Definition: em_event_types.h:220
em_event_uarea_get
void * em_event_uarea_get(em_event_t event, size_t *size)
Get a pointer to the event user area, optionally along with its size.
Definition: event_machine_event.c:1531
em_event_unmark_free_multi
void em_event_unmark_free_multi(const em_event_t events[], int num)
Unmark multiple events previously marked as "free".
Definition: event_machine_event.c:1225
em_event_uarea_info_t
Event user area information filled by em_event_uarea_info()
Definition: event_machine_event.h:881
EM_QUEUE_TYPE_ATOMIC
@ EM_QUEUE_TYPE_ATOMIC
Definition: event_machine_hw_types.h:112
evstate_free
void evstate_free(em_event_t event, event_hdr_t *const ev_hdr, const uint16_t api_op)
Definition: em_event_state.c:914
eo_start_buffer_events
int eo_start_buffer_events(const em_event_t events[], int num, em_queue_t queue)
Definition: em_eo.c:412
event_hdr::flags
union event_hdr::@34 flags
PRI_EVENT
#define PRI_EVENT
Definition: event_machine_types.h:64
em_event_vector_free
void em_event_vector_free(em_event_t vector_event)
Free the vector event only, not the events it contains.
Definition: event_machine_event.c:1746
pkt_clone_odp
em_event_t pkt_clone_odp(odp_packet_t pkt, odp_pool_t pkt_pool, uint32_t offset, uint32_t size, bool is_clone_part)
Definition: em_event.c:105
event_hdr::event
em_event_t event
Definition: em_event_types.h:246
em_event_unmark_free
void em_event_unmark_free(em_event_t event)
Unmark an event previously marked as "free" (i.e mark as "allocated" again).
Definition: event_machine_event.c:1157
EM_QUEUE_TYPE_UNSCHEDULED
@ EM_QUEUE_TYPE_UNSCHEDULED
Definition: event_machine_hw_types.h:127
event_hdr
Definition: em_event_types.h:184
em_event_uarea_id_get
em_status_t em_event_uarea_id_get(em_event_t event, bool *isset, uint16_t *id)
Get the event user area ID along with information if it has been set.
Definition: event_machine_event.c:1590
em_event_clone_part
em_event_t em_event_clone_part(em_event_t event, em_pool_t pool, uint32_t offset, uint32_t len, bool clone_uarea)
Partially clone an event.
Definition: event_machine_event.c:1459
em_event_uarea_info_t::uarea
void * uarea
Definition: event_machine_event.h:883
em_event_uarea_info_t::isset
bool isset
Definition: event_machine_event.h:890
mpool_elem_t
Definition: em_pool_types.h:47
em_event_get_pool_subpool
em_pool_t em_event_get_pool_subpool(em_event_t event, int *subpool)
Returns the EM event-pool and subpool the event was allocated from.
Definition: event_machine_event.c:899
em_event_vector_max_size
uint32_t em_event_vector_max_size(em_event_t vector_event)
Maximum number of event handles that can be stored in a vector.
Definition: event_machine_event.c:1834
em_event_get_type
em_event_type_t em_event_get_type(em_event_t event)
Definition: event_machine_event.c:968
RETURN_ERROR_IF
#define RETURN_ERROR_IF(cond, error, escope, fmt,...)
Definition: em_error.h:50
em_event_vector_size_set
void em_event_vector_size_set(em_event_t vector_event, uint32_t size)
Set the number of event handles stored in a vector.
Definition: event_machine_event.c:1815
evstate_alloc
em_event_t evstate_alloc(const em_event_t event, event_hdr_t *const ev_hdr, const uint16_t api_op)
Definition: em_event_state.c:738
em_alloc
em_event_t em_alloc(uint32_t size, em_event_type_t type, em_pool_t pool)
Definition: event_machine_event.c:33
EM_ERR_BAD_ID
@ EM_ERR_BAD_ID
Definition: event_machine_hw_types.h:265
INTERNAL_ERROR
#define INTERNAL_ERROR(error, escope, fmt,...)
Definition: em_error.h:43
event_hdr::align_offset
uint8_t align_offset
Definition: em_event_types.h:236
mpool_elem_t::event_type
em_event_type_t event_type
Definition: em_pool_types.h:49
mpool_elem_t::stats_opt
odp_pool_stats_opt_t stats_opt
Definition: em_pool_types.h:75
em_event_pointer_and_size
void * em_event_pointer_and_size(em_event_t event, uint32_t *size)
Get a pointer to the event structure/data as well as the event size.
Definition: event_machine_event.c:767
event_hdr::user_area
ev_hdr_user_area_t user_area
Definition: em_event_types.h:277
em_escope_t
uint32_t em_escope_t
Definition: event_machine_types.h:348
queue_elem_t::scheduled
uint8_t scheduled
Definition: em_queue_types.h:199
em_status_t
uint32_t em_status_t
Definition: event_machine_types.h:321
em_event_vector_tbl
uint32_t em_event_vector_tbl(em_event_t vector_event, em_event_t **event_tbl)
Get the event vector table from an event of (major) type EM_EVENT_TYPE_VECTOR.
Definition: event_machine_event.c:1776
EM_QUEUE_TYPE_LOCAL
@ EM_QUEUE_TYPE_LOCAL
Definition: event_machine_hw_types.h:131
PRI_QUEUE
#define PRI_QUEUE
Definition: event_machine_types.h:109
em_event_vector_size
uint32_t em_event_vector_size(em_event_t vector_event)
Number of event handles available (set) in a vector.
Definition: event_machine_event.c:1796
EM_CHECK_LEVEL
#define EM_CHECK_LEVEL
Definition: event_machine_config.h:253
EM_ERR_OPERATION_FAILED
@ EM_ERR_OPERATION_FAILED
Definition: event_machine_hw_types.h:289
event_hdr::egrp
em_event_group_t egrp
Definition: em_event_types.h:265
em_event_type_t
uint32_t em_event_type_t
Definition: event_machine_types.h:85
em_event_get_size
uint32_t em_event_get_size(em_event_t event)
Definition: event_machine_event.c:818
em_shm
em_shm_t * em_shm
Definition: event_machine_init.c:41
em_pool_info_print
void em_pool_info_print(em_pool_t pool)
Definition: event_machine_pool.c:287
em_event_unmark_send
em_status_t em_event_unmark_send(em_event_t event)
Definition: event_machine_event.c:1110
evstate_usr2em_multi
void evstate_usr2em_multi(const em_event_t ev_tbl[], event_hdr_t *const ev_hdr_tbl[], const int num, const uint16_t api_op)
Definition: em_event_state.c:1008
EM_DEBUG_PRINT
#define EM_DEBUG_PRINT
Definition: event_machine_config.h:292
EM_EVENT_TYPE_TIMER_IND
@ EM_EVENT_TYPE_TIMER_IND
Definition: event_machine_hw_types.h:91
em_include.h
em_event_to_u64
uint64_t em_event_to_u64(em_event_t event)
Definition: event_machine_event.c:1896
EM_QUEUE_TYPE_PARALLEL_ORDERED
@ EM_QUEUE_TYPE_PARALLEL_ORDERED
Definition: event_machine_hw_types.h:122
evstate_unmark_free_multi
void evstate_unmark_free_multi(const em_event_t ev_tbl[], event_hdr_t *const ev_hdr_tbl[], const int num, const uint16_t api_op)
Definition: em_event_state.c:1110
EM_QUEUE_TYPE_OUTPUT
@ EM_QUEUE_TYPE_OUTPUT
Definition: event_machine_hw_types.h:137
em_event_mark_free
void em_event_mark_free(em_event_t event)
Mark the event as "free".
Definition: event_machine_event.c:1132
EM_ERR_BAD_TYPE
@ EM_ERR_BAD_TYPE
Definition: event_machine_hw_types.h:267
EM_EVENT_TYPE_UNDEF
@ EM_EVENT_TYPE_UNDEF
Definition: event_machine_hw_types.h:69
em_event_same_type_multi
int em_event_same_type_multi(const em_event_t events[], int num, em_event_type_t *same_type)
Definition: event_machine_event.c:1028
EM_ERR_NOT_FOUND
@ EM_ERR_NOT_FOUND
Definition: event_machine_hw_types.h:278
em_event_get_type_multi
int em_event_get_type_multi(const em_event_t events[], int num, em_event_type_t types[])
Definition: event_machine_event.c:989
EM_POOL_UNDEF
#define EM_POOL_UNDEF
Definition: event_machine_hw_types.h:60
EM_EVENT_TYPE_PACKET
@ EM_EVENT_TYPE_PACKET
Definition: event_machine_hw_types.h:75
event_hdr::tmo_type
uint8_t tmo_type
Definition: em_event_types.h:225
em_event_ref
em_event_t em_event_ref(em_event_t event)
Definition: event_machine_event.c:1681
EM_ERR_BAD_POINTER
@ EM_ERR_BAD_POINTER
Definition: event_machine_hw_types.h:271
em_event_has_ref
bool em_event_has_ref(em_event_t event)
Definition: event_machine_event.c:1734
evstate_ref
em_event_t evstate_ref(const em_event_t event, event_hdr_t *const ev_hdr)
Definition: em_event_state.c:780
em_event_set_type
em_status_t em_event_set_type(em_event_t event, em_event_type_t newtype)
Definition: event_machine_event.c:923
em_event_vector_info_t::max_size
uint32_t max_size
Definition: event_machine_event.h:1141
EM_ERR_BAD_ARG
@ EM_ERR_BAD_ARG
Definition: event_machine_hw_types.h:261
EM_ERR_NOT_IMPLEMENTED
@ EM_ERR_NOT_IMPLEMENTED
Definition: event_machine_hw_types.h:282
em_free_multi
void em_free_multi(em_event_t events[], int num)
Definition: event_machine_event.c:370
evstate_usr2em
void evstate_usr2em(em_event_t event, event_hdr_t *const ev_hdr, const uint16_t api_op)
Definition: em_event_state.c:990
em_send_multi
int em_send_multi(const em_event_t events[], int num, em_queue_t queue)
Definition: event_machine_event.c:710
em_event_get_pool
em_pool_t em_event_get_pool(em_event_t event)
Returns the EM event-pool the event was allocated from.
Definition: event_machine_event.c:876
ev_hdr_user_area_t::size
uint32_t size
Definition: em_event_types.h:170
EM_ERR_BAD_STATE
@ EM_ERR_BAD_STATE
Definition: event_machine_hw_types.h:263
EM_EVENT_GROUP_UNDEF
#define EM_EVENT_GROUP_UNDEF
Definition: event_machine_types.h:141
em_event_uarea_info_t::id
struct em_event_uarea_info_t::@1 id
evstate_unmark_send
void evstate_unmark_send(const em_event_t event, event_hdr_t *const ev_hdr)
Definition: em_event_state.c:1079
queue_elem_t
Definition: em_queue_types.h:180
em_event_pointer
void * em_event_pointer(em_event_t event)
Definition: event_machine_event.c:750
EM_ERROR
#define EM_ERROR
Definition: event_machine_types.h:337
ev_hdr_user_area_t::id
uint32_t id
Definition: em_event_types.h:172
em_locm_t::start_eo_elem
eo_elem_t * start_eo_elem
Definition: em_mem.h:225
ev_hdr_user_area_t::isinit
uint32_t isinit
Definition: em_event_types.h:168
em_event_vector_info_t::size
uint32_t size
Definition: event_machine_event.h:1136
em_event_uarea_info
em_status_t em_event_uarea_info(em_event_t event, em_event_uarea_info_t *uarea_info)
Get the event user area information for a given event.
Definition: event_machine_event.c:1628