EM-ODP  3.7.0
Event Machine on ODP
em_event_state.c
1 /*
2  * Copyright (c) 2020-2022, Nokia Solutions and Networks
3  * All rights reserved.
4  *
5  * Redistribution and use in source and binary forms, with or without
6  * modification, are permitted provided that the following conditions
7  * are met:
8  *
9  * * Redistributions of source code must retain the above copyright
10  * notice, this list of conditions and the following disclaimer.
11  * * Redistributions in binary form must reproduce the above copyright
12  * notice, this list of conditions and the following disclaimer in the
13  * documentation and/or other materials provided with the distribution.
14  * * Neither the name of the copyright holder nor the names of its
15  * contributors may be used to endorse or promote products derived
16  * from this software without specific prior written permission.
17  *
18  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22  * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29  */
30 
31 #include "em_include.h"
32 
33 static int read_config_file(void);
34 
35 /**
36  * Initial counter values set during an alloc-operation: ref=1, send=0
37  * (em_alloc/_multi(), em_event_clone())
38  */
39 static const evstate_cnt_t init_cnt_alloc = {.evgen = EVGEN_INIT,
40  .rsvd = 0,
41  .ref_cnt = REF_CNT_INIT - 1,
42  .send_cnt = 0 + SEND_CNT_INIT};
43 /**
44  * Initial counter values for external events entering into EM
45  * (event not allocated by EM): ref=1, send=1
46  */
47 static const evstate_cnt_t init_cnt_extev = {.evgen = EVGEN_INIT,
48  .rsvd = 0,
49  .ref_cnt = REF_CNT_INIT - 1,
50  .send_cnt = 1 + SEND_CNT_INIT};
51 
52 /**
53  * Information about an event-state update location
54  */
55 typedef struct {
56  const char *str;
57  em_escope_t escope;
58 } evstate_info_t;
59 
60 /**
61  * Constant table containing event-state update location information.
62  * Only accessed when an erroneous event state has been detected and is being
63  * reported to the error handler.
64  */
65 static const evstate_info_t evstate_info_tbl[] = {
66  [EVSTATE__UNDEF] = {.str = "undefined",
67  .escope = (EM_ESCOPE_INTERNAL_MASK | 0)},
68  [EVSTATE__PREALLOC] = {.str = "pool-create(prealloc-events)",
69  .escope = EM_ESCOPE_POOL_CREATE},
70  [EVSTATE__ALLOC] = {.str = "em_alloc()",
71  .escope = EM_ESCOPE_ALLOC},
72  [EVSTATE__ALLOC_MULTI] = {.str = "em_alloc_multi()",
73  .escope = EM_ESCOPE_ALLOC_MULTI},
74  [EVSTATE__EVENT_CLONE] = {.str = "em_event_clone()",
75  .escope = EM_ESCOPE_EVENT_CLONE},
76  [EVSTATE__EVENT_REF] = {.str = "em_event_ref()",
77  .escope = EM_ESCOPE_EVENT_REF},
78  [EVSTATE__FREE] = {.str = "em_free()",
79  .escope = EM_ESCOPE_FREE},
80  [EVSTATE__FREE_MULTI] = {.str = "em_free_multi()",
81  .escope = EM_ESCOPE_FREE_MULTI},
82  [EVSTATE__EVENT_VECTOR_FREE] = {.str = "em_event_vector_free()",
83  .escope = EM_ESCOPE_EVENT_VECTOR_FREE},
84  [EVSTATE__INIT] = {.str = "init-event",
85  .escope = EM_ESCOPE_ODP_EXT},
86  [EVSTATE__INIT_MULTI] = {.str = "init-events",
87  .escope = EM_ESCOPE_ODP_EXT},
88  [EVSTATE__INIT_EXTEV] = {.str = "dispatch(init-ext-event)",
89  .escope = EM_ESCOPE_DISPATCH},
90  [EVSTATE__INIT_EXTEV_MULTI] = {.str = "dispatch(init-ext-events)",
91  .escope = EM_ESCOPE_DISPATCH},
92  [EVSTATE__UPDATE_EXTEV] = {.str = "dispatch(update-ext-event)",
93  .escope = EM_ESCOPE_DISPATCH},
94  [EVSTATE__SEND] = {.str = "em_send()",
95  .escope = EM_ESCOPE_SEND},
96  [EVSTATE__SEND__FAIL] = {.str = "em_send(fail)",
97  .escope = EM_ESCOPE_SEND},
98  [EVSTATE__SEND_EGRP] = {.str = "em_send_group()",
99  .escope = EM_ESCOPE_SEND_GROUP},
100  [EVSTATE__SEND_EGRP__FAIL] = {.str = "em_send_group(fail)",
101  .escope = EM_ESCOPE_SEND_GROUP},
102  [EVSTATE__SEND_MULTI] = {.str = "em_send_multi()",
103  .escope = EM_ESCOPE_SEND_MULTI},
104  [EVSTATE__SEND_MULTI__FAIL] = {.str = "em_send_multi(fail)",
105  .escope = EM_ESCOPE_SEND_MULTI},
106  [EVSTATE__SEND_EGRP_MULTI] = {.str = "em_send_group_multi()",
107  .escope = EM_ESCOPE_SEND_GROUP_MULTI},
108  [EVSTATE__SEND_EGRP_MULTI__FAIL] = {.str = "em_send_group_multi(fail)",
109  .escope = EM_ESCOPE_SEND_GROUP_MULTI},
110  [EVSTATE__EO_START_SEND_BUFFERED] = {.str = "eo-start:send-buffered-events()",
111  .escope = EM_ESCOPE_SEND_MULTI},
112  [EVSTATE__MARK_SEND] = {.str = "em_event_mark_send()",
113  .escope = EM_ESCOPE_EVENT_MARK_SEND},
114  [EVSTATE__UNMARK_SEND] = {.str = "em_event_unmark_send()",
115  .escope = EM_ESCOPE_EVENT_UNMARK_SEND},
116  [EVSTATE__MARK_FREE] = {.str = "em_event_mark_free()",
117  .escope = EM_ESCOPE_EVENT_MARK_FREE},
118  [EVSTATE__UNMARK_FREE] = {.str = "em_event_unmark_free()",
119  .escope = EM_ESCOPE_EVENT_UNMARK_FREE},
120  [EVSTATE__MARK_FREE_MULTI] = {.str = "em_event_mark_free_multi()",
121  .escope = EM_ESCOPE_EVENT_MARK_FREE_MULTI},
122  [EVSTATE__UNMARK_FREE_MULTI] = {.str = "em_event_unmark_free_multi()",
123  .escope = EM_ESCOPE_EVENT_UNMARK_FREE_MULTI},
124  [EVSTATE__DISPATCH] = {.str = "em_dispatch(single-event)",
125  .escope = EM_ESCOPE_DISPATCH},
126  [EVSTATE__DISPATCH_MULTI] = {.str = "em_dispatch(multiple-events)",
127  .escope = EM_ESCOPE_DISPATCH},
128  [EVSTATE__DISPATCH_SCHED__FAIL] = {.str = "em_dispatch(drop sched-events)",
129  .escope = EM_ESCOPE_DISPATCH},
130  [EVSTATE__DISPATCH_LOCAL__FAIL] = {.str = "em_dispatch(drop local-events)",
131  .escope = EM_ESCOPE_DISPATCH},
132  [EVSTATE__DEQUEUE] = {.str = "em_queue_dequeue()",
133  .escope = EM_ESCOPE_QUEUE_DEQUEUE},
134  [EVSTATE__DEQUEUE_MULTI] = {.str = "em_queue_dequeue_multi()",
135  .escope = EM_ESCOPE_QUEUE_DEQUEUE_MULTI},
136  [EVSTATE__TMO_SET_ABS] = {.str = "em_tmo_set_abs()",
137  .escope = EM_ESCOPE_TMO_SET_ABS},
138  [EVSTATE__TMO_SET_ABS__FAIL] = {.str = "em_tmo_set_abs(fail)",
139  .escope = EM_ESCOPE_TMO_SET_ABS},
140  [EVSTATE__TMO_SET_REL] = {.str = "em_tmo_set_rel()",
141  .escope = EM_ESCOPE_TMO_SET_REL},
142  [EVSTATE__TMO_SET_REL__FAIL] = {.str = "em_tmo_set_rel(fail)",
143  .escope = EM_ESCOPE_TMO_SET_REL},
144  [EVSTATE__TMO_SET_PERIODIC] = {.str = "em_tmo_set_periodic()",
145  .escope = EM_ESCOPE_TMO_SET_PERIODIC},
146  [EVSTATE__TMO_SET_PERIODIC__FAIL] = {.str = "em_tmo_set_periodic(fail)",
147  .escope = EM_ESCOPE_TMO_SET_PERIODIC},
148  [EVSTATE__TMO_CANCEL] = {.str = "em_tmo_cancel()",
149  .escope = EM_ESCOPE_TMO_CANCEL},
150  [EVSTATE__TMO_ACK] = {.str = "em_tmo_ack()",
151  .escope = EM_ESCOPE_TMO_ACK},
152  [EVSTATE__TMO_ACK__NOSKIP] = {.str = "em_tmo_ack(noskip)",
153  .escope = EM_ESCOPE_TMO_ACK},
154  [EVSTATE__TMO_ACK__FAIL] = {.str = "em_tmo_ack(fail)",
155  .escope = EM_ESCOPE_TMO_ACK},
156  [EVSTATE__TMO_CREATE] = {.str = "em_tmo_create()",
157  .escope = EM_ESCOPE_TMO_CREATE},
158  [EVSTATE__TMO_DELETE] = {.str = "em_tmo_delete()",
159  .escope = EM_ESCOPE_TMO_DELETE},
160  [EVSTATE__AG_DELETE] = {.str = "em_atomic_group_delete(flush)",
161  .escope = EM_ESCOPE_ATOMIC_GROUP_DELETE},
162  [EVSTATE__TERM_CORE__QUEUE_LOCAL] = {.str = "em_term_core(local-queue)",
163  .escope = EM_ESCOPE_TERM_CORE},
164  [EVSTATE__TERM] = {.str = "em_term()",
165  .escope = EM_ESCOPE_TERM},
166  /* Last: */
167  [EVSTATE__LAST] = {.str = "last",
168  .escope = (EM_ESCOPE_INTERNAL_MASK | 0)}
169 };
170 
171 static const char *const help_str_em2usr =
172 "OK: 'send < ref, both >=0'. Err otherwise";
173 static const char *const help_str_usr2em =
174 "OK: 'send <= ref, both >=0' AND 'hdl evgen == evgen'. Err otherwise";
175 static const char *const help_str_usr2em_ref =
176 "OK: 'send <= ref, both >=0'. Err otherwise";
177 
178 static inline void
179 esv_update_state(ev_hdr_state_t *const evstate, const uint16_t api_op,
180  const void *const ev_ptr)
181 {
182  const em_locm_t *const locm = &em_locm;
183  const uint32_t *const pl_u32 = ev_ptr;
184  const queue_elem_t *const q_elem = locm->current.q_elem;
185 
186  if (ev_ptr)
187  evstate->payload_first = *pl_u32;
188 
189  if (!q_elem) {
190  evstate->eo_idx = (int16_t)eo_hdl2idx(EM_EO_UNDEF); /* -1 is fine */
191  evstate->queue_idx = (int16_t)queue_hdl2idx(EM_QUEUE_UNDEF); /* -1 is fine */
192  } else {
193  evstate->eo_idx = (int16_t)eo_hdl2idx((em_eo_t)(uintptr_t)q_elem->eo);
194  evstate->queue_idx = (int16_t)queue_hdl2idx((em_queue_t)(uintptr_t)q_elem->queue);
195  }
196  evstate->api_op = (uint8_t)api_op; /* no trucation */
197  evstate->core = locm->core_id;
198 }
199 
200 static inline void
201 evhdr_update_state(event_hdr_t *const ev_hdr, const uint16_t api_op)
202 {
203  if (!em_shm->opt.esv.store_state)
204  return; /* don't store updated state */
205 
206  const void *ev_ptr = NULL;
207 
208  if (em_shm->opt.esv.store_first_u32)
209  ev_ptr = event_pointer(ev_hdr->event);
210 
211  esv_update_state(&ev_hdr->state, api_op, ev_ptr);
212 }
213 
214 /* "Normal" ESV Error format */
215 #define EVSTATE_ERROR_FMT \
216 "ESV: Event:%" PRI_EVENT " state error -- counts:\t" \
217 "send:%" PRIi16 " ref:%" PRIi16 " evgen:%" PRIu16 "(%" PRIu16 ")\n" \
218 " Help: %s\n" \
219 " prev-state:%s core:%02u:\t" \
220 " EO:%" PRI_EO "-\"%s\" Q:%" PRI_QUEUE "-\"%s\" u32[0]:%s\n" \
221 "=> err-state:%s core:%02u:\t" \
222 " EO:%" PRI_EO "-\"%s\" Q:%" PRI_QUEUE "-\"%s\" u32[0]:%s\n" \
223 " event:0x%016" PRIx64 ": ptr:0x%" PRIx64 ""
224 
225 /* ESV Error format for references */
226 #define EVSTATE_REF_ERROR_FMT \
227 "ESV: RefEvent:%" PRI_EVENT " state error -- counts:\t" \
228 "send:%" PRIi16 " ref:%" PRIi16 " (evgen:%" PRIu16 " ignored for refs)\n" \
229 " Help: %s\n" \
230 " prev-state:n/a (not valid for event references)\n" \
231 "=> err-state:%s core:%02u:\t" \
232 " EO:%" PRI_EO "-\"%s\" Q:%" PRI_QUEUE "-\"%s\" u32[0]:%s\n" \
233 " event:0x%016" PRIx64 ": ptr:0x%" PRIx64 ""
234 
235 /* ESV Error format for em_event_unmark_send/free/_multi() */
236 #define EVSTATE_UNMARK_ERROR_FMT \
237 "ESV: Event:%" PRI_EVENT " state error - Invalid 'unmark'-API use\n"\
238 " prev-state:%s core:%02u:\t" \
239 " EO:%" PRI_EO "-\"%s\" Q:%" PRI_QUEUE "-\"%s\" u32[0]:%s\n" \
240 "=> err-state:%s core:%02u:\t" \
241 " EO:%" PRI_EO "-\"%s\" Q:%" PRI_QUEUE "-\"%s\" u32[0]:%s\n"
242 
243 /* ESV Error format when esv.store_state = false */
244 #define EVSTATE__NO_PREV_STATE__ERROR_FMT \
245 "ESV: Event:%" PRI_EVENT " state error -- counts:\t" \
246 "send:%" PRIi16 " ref:%" PRIi16 " evgen:%" PRIu16 "(%" PRIu16 ")\n" \
247 " Help: %s\n" \
248 " prev-state:n/a (disabled in conf)\n" \
249 "=> err-state:%s core:%02u:\t" \
250 " EO:%" PRI_EO "-\"%s\" Q:%" PRI_QUEUE "-\"%s\" u32[0]:%s\n" \
251 " event:0x%016" PRIx64 ": ptr:0x%" PRIx64 ""
252 
253 /* ESV Error format for em_event_unmark_send/free/_multi() when esv.store_state = false */
254 #define EVSTATE__NO_PREV_STATE__UNMARK_ERROR_FMT \
255 "ESV: Event:%" PRI_EVENT " state error - Invalid 'unmark'-API use\n"\
256 " prev-state:n/a (disabled in conf)\n" \
257 "=> err-state:%s core:%02u:\t" \
258 " EO:%" PRI_EO "-\"%s\" Q:%" PRI_QUEUE "-\"%s\" u32[0]:%s\n"
259 
260 /**
261  * ESV Error reporting
262  */
263 static inline void
264 esv_error(const evstate_cnt_t cnt,
265  evhdl_t evhdl, const event_hdr_t *const ev_hdr,
266  const uint16_t api_op, bool is_unmark_error,
267  const char *const help_str)
268 {
269  uint16_t prev_op = ev_hdr->state.api_op;
270  ev_hdr_state_t prev_state = ev_hdr->state; /* store prev good state */
271  ev_hdr_state_t err_state = {0}; /* store current invalid/error state */
272  const em_event_t event = event_hdr_to_event(ev_hdr);
273  const void *ev_ptr = NULL;
274 
275  if (unlikely(prev_op > EVSTATE__LAST))
276  prev_op = EVSTATE__UNDEF;
277 
278  const evstate_info_t *err_info = &evstate_info_tbl[api_op];
279  const evstate_info_t *prev_info = &evstate_info_tbl[prev_op];
280 
281  char curr_eoname[EM_EO_NAME_LEN] = "(noname)";
282  char prev_eoname[EM_EO_NAME_LEN] = "(noname)";
283  char curr_qname[EM_QUEUE_NAME_LEN] = "(noname)";
284  char prev_qname[EM_QUEUE_NAME_LEN] = "(noname)";
285  char curr_payload[sizeof("0x12345678 ")] = "(n/a)";
286  char prev_payload[sizeof("0x12345678 ")] = "(n/a)";
287 
288  const eo_elem_t *eo_elem;
289  const queue_elem_t *q_elem;
290 
291  /* Check event!=undef to avoid error in event_pointer() */
292  if (likely(event != EM_EVENT_UNDEF))
293  ev_ptr = event_pointer(event);
294  /* Store the new _invalid_ event-state info into a separate struct */
295  esv_update_state(&err_state, api_op, ev_ptr);
296 
297  /*
298  * Print the first 32bits of the event payload on failure,
299  * the option 'esv.store_payload_first_u32' affects storing during valid
300  * state transitions.
301  */
302  if (ev_ptr) {
303  snprintf(curr_payload, sizeof(curr_payload),
304  "0x%08" PRIx32 "", err_state.payload_first);
305  curr_payload[sizeof(curr_payload) - 1] = '\0';
306  }
307 
308  em_eo_t curr_eo = eo_idx2hdl(err_state.eo_idx);
309  em_queue_t curr_queue = queue_idx2hdl(err_state.queue_idx);
310 
311  /* current EO-name: */
312  eo_elem = eo_elem_get(curr_eo);
313  if (eo_elem != NULL)
314  eo_get_name(eo_elem, curr_eoname, sizeof(curr_eoname));
315  /* current queue-name: */
316  q_elem = queue_elem_get(curr_queue);
317  if (q_elem != NULL)
318  queue_get_name(q_elem, curr_qname, sizeof(curr_qname));
319 
320  const int16_t send_cnt = cnt.send_cnt - SEND_CNT_INIT;
321  uint16_t evgen_cnt = cnt.evgen - EVGEN_INIT;
322  const uint16_t evgen_hdl = evhdl.evgen - EVGEN_INIT;
323  const int16_t ref_cnt = REF_CNT_INIT - cnt.ref_cnt;
324 
325  /* Read the previous event state only if it has been stored */
326  if (em_shm->opt.esv.store_state) {
327  /*
328  * Print the first 32 bits of the event payload for the previous
329  * valid state transition, if enabled in the EM config file:
330  * 'esv.store_payload_first_u32 = true', otherwise not stored.
331  */
332  if (em_shm->opt.esv.store_first_u32) {
333  snprintf(prev_payload, sizeof(prev_payload),
334  "0x%08" PRIx32 "", prev_state.payload_first);
335  prev_payload[sizeof(prev_payload) - 1] = '\0';
336  }
337 
338  em_eo_t prev_eo = eo_idx2hdl(prev_state.eo_idx);
339  em_queue_t prev_queue = queue_idx2hdl(prev_state.queue_idx);
340 
341  /* previous EO-name: */
342  eo_elem = eo_elem_get(prev_eo);
343  if (eo_elem != NULL)
344  eo_get_name(eo_elem, prev_eoname, sizeof(prev_eoname));
345  /* previous queue-name: */
346  q_elem = queue_elem_get(prev_queue);
347  if (q_elem != NULL)
348  queue_get_name(q_elem, prev_qname, sizeof(prev_qname));
349 
350  if (ev_hdr->flags.refs_used) {
351  /* Reference ESV Error, prev state available */
352  INTERNAL_ERROR(EM_FATAL(EM_ERR_EVENT_STATE),
353  err_info->escope, EVSTATE_REF_ERROR_FMT,
354  event, send_cnt, ref_cnt, evgen_cnt, help_str,
355  err_info->str, err_state.core,
356  curr_eo, curr_eoname, curr_queue, curr_qname,
357  curr_payload, evhdl.event, evhdl.evptr);
358  } else if (!is_unmark_error) {
359  /* "Normal" ESV Error, prev state available */
360  INTERNAL_ERROR(EM_FATAL(EM_ERR_EVENT_STATE),
361  err_info->escope, EVSTATE_ERROR_FMT,
362  event, send_cnt, ref_cnt, evgen_hdl, evgen_cnt, help_str,
363  prev_info->str, prev_state.core, prev_eo, prev_eoname,
364  prev_queue, prev_qname, prev_payload,
365  err_info->str, err_state.core, curr_eo, curr_eoname,
366  curr_queue, curr_qname, curr_payload,
367  evhdl.event, evhdl.evptr);
368  } else {
369  /*
370  * ESV Error from em_event_unmark_send/free/_multi(),
371  * prev state available.
372  */
373  INTERNAL_ERROR(EM_FATAL(EM_ERR_EVENT_STATE),
374  err_info->escope, EVSTATE_UNMARK_ERROR_FMT,
375  event,
376  prev_info->str, prev_state.core,
377  prev_eo, prev_eoname,
378  prev_queue, prev_qname, prev_payload,
379  err_info->str, err_state.core,
380  curr_eo, curr_eoname,
381  curr_queue, curr_qname, curr_payload);
382  }
383  } else { /* em_shm->opt.esv.store_state == false */
384  /* No previous state stored by EM at runtime */
385  if (!is_unmark_error) {
386  /* "Normal" ESV Error, prev state not stored */
387  INTERNAL_ERROR(EM_FATAL(EM_ERR_EVENT_STATE),
388  err_info->escope, EVSTATE__NO_PREV_STATE__ERROR_FMT,
389  event, send_cnt, ref_cnt, evgen_hdl, evgen_cnt, help_str,
390  err_info->str, err_state.core, curr_eo, curr_eoname,
391  curr_queue, curr_qname, curr_payload,
392  evhdl.event, evhdl.evptr);
393  } else {
394  /*
395  * ESV Error from em_event_unmark_send/free/_multi(),
396  * prev state not stored.
397  */
398  INTERNAL_ERROR(EM_FATAL(EM_ERR_EVENT_STATE),
399  err_info->escope, EVSTATE__NO_PREV_STATE__UNMARK_ERROR_FMT,
400  event,
401  err_info->str, err_state.core, curr_eo, curr_eoname,
402  curr_queue, curr_qname, curr_payload);
403  }
404  }
405 }
406 
407 static void
408 evstate_error(const evstate_cnt_t cnt, evhdl_t evhdl,
409  const event_hdr_t *const ev_hdr, const uint16_t api_op,
410  const char *const help_str)
411 {
412  /* "Normal" ESV Error */
413  esv_error(cnt, evhdl, ev_hdr, api_op, false, help_str);
414 }
415 
416 /**
417  * ESV Error reporting for invalid em_event_unmark...() API use
418  */
419 static void
420 evstate_unmark_error(const event_hdr_t *const ev_hdr, const uint16_t api_op)
421 {
422  evstate_cnt_t dont_care = {.u64 = 0};
423  evhdl_t dont_care_hdl = {.event = EM_EVENT_UNDEF};
424 
425  /* ESV Error from em_event_unmark_send/free/_multi() */
426  esv_error(dont_care, dont_care_hdl, ev_hdr, api_op, true, "n/a");
427 }
428 
429 static inline em_event_t
430 esv_evinit(const em_event_t event, event_hdr_t *const ev_hdr,
431  const evstate_cnt_t init_cnt, const uint16_t api_op)
432 {
433  evhdl_t evhdl = {.event = event};
434 
435  evhdl.evgen = EVGEN_INIT;
436  ev_hdr->event = evhdl.event;
437 
438  /* Set initial counters (atomic) */
439  __atomic_store_n(&ev_hdr->state_cnt.u64, init_cnt.u64,
440  __ATOMIC_RELAXED);
441  /* Set initial state information (non-atomic) */
442  evhdr_update_state(ev_hdr, api_op);
443 
444  return evhdl.event;
445 }
446 
447 static inline void
448 esv_evinit_multi(em_event_t ev_tbl[/*in/out*/],
449  event_hdr_t *const ev_hdr_tbl[], const int num,
450  const evstate_cnt_t init_cnt, const uint16_t api_op)
451 {
452  evhdl_t *const evhdl_tbl = (evhdl_t *)ev_tbl;
453 
454  for (int i = 0; i < num; i++) {
455  evhdl_tbl[i].evgen = EVGEN_INIT;
456  ev_hdr_tbl[i]->event = evhdl_tbl[i].event;
457 
458  /* Set initial counters for ext-events (atomic) */
459  __atomic_store_n(&ev_hdr_tbl[i]->state_cnt.u64,
460  init_cnt.u64, __ATOMIC_RELAXED);
461  /* Set initial state information (non-atomic) */
462  evhdr_update_state(ev_hdr_tbl[i], api_op);
463  }
464 }
465 
466 static inline em_event_t
467 esv_evinit_ext(const em_event_t event, event_hdr_t *const ev_hdr,
468  const uint16_t api_op)
469 {
470  /*
471  * Combination of:
472  * event = esv_evinit(..., init_cnt_extev, ...)
473  * return evstate_em2usr(event, ...);
474  */
475  evhdl_t evhdl = {.event = event};
476  const evstate_cnt_t init = init_cnt_extev;
477  const evstate_cnt_t sub = {.evgen = 0, .rsvd = 0,
478  .ref_cnt = 0, .send_cnt = 1};
479  const evstate_cnt_t cnt = {.u64 = init.u64 - sub.u64};
480 
481  evhdl.evgen = cnt.evgen;
482  ev_hdr->event = evhdl.event;
483 
484  /* Set initial counters (atomic) */
485  __atomic_store_n(&ev_hdr->state_cnt.u64, cnt.u64,
486  __ATOMIC_RELAXED);
487 
488  /* Set initial state information (non-atomic) */
489  evhdr_update_state(ev_hdr, api_op);
490 
491  return evhdl.event;
492 }
493 
494 static inline em_event_t
495 esv_em2usr(const em_event_t event, event_hdr_t *const ev_hdr,
496  const evstate_cnt_t cnt, const uint16_t api_op, const bool is_revert)
497 {
498  const bool refs_used = ev_hdr->flags.refs_used;
499  evhdl_t evhdl = {.event = event};
500  evstate_cnt_t new_cnt;
501 
502  /* Update state-count and return value of all counters (atomic) */
503  if (unlikely(is_revert)) {
504  /* Revert previous em2usr counter update on failed operation */
505  new_cnt.u64 = __atomic_add_fetch(&ev_hdr->state_cnt.u64,
506  cnt.u64, __ATOMIC_RELAXED);
507  } else {
508  /* Normal em2usr counter update */
509  new_cnt.u64 = __atomic_sub_fetch(&ev_hdr->state_cnt.u64,
510  cnt.u64, __ATOMIC_RELAXED);
511  }
512 
513  if (!refs_used) {
514  evhdl.evgen = new_cnt.evgen;
515  ev_hdr->event = evhdl.event;
516  }
517 
518  const int16_t ref_cnt = REF_CNT_INIT - new_cnt.ref_cnt;
519  const int16_t send_cnt = new_cnt.send_cnt - SEND_CNT_INIT;
520 
521  /*
522  * Check state count:
523  * OK: send_cnt < ref_cnt and both >=0. Error otherwise.
524  */
525  if (unlikely(send_cnt >= ref_cnt || send_cnt < 0)) {
526  /* report fatal event-state error, never return */
527  evstate_error(new_cnt, evhdl, ev_hdr, api_op, help_str_em2usr);
528  /* never reached */
529  }
530 
531  /*
532  * Valid state transition, update state (non-atomic)
533  */
534  if (!refs_used)
535  evhdr_update_state(ev_hdr, api_op);
536 
537  return evhdl.event;
538 }
539 
540 static inline void
541 esv_em2usr_multi(em_event_t ev_tbl[/*in/out*/],
542  event_hdr_t *const ev_hdr_tbl[], const int num,
543  const evstate_cnt_t cnt, const uint16_t api_op,
544  const bool is_revert)
545 {
546  evhdl_t *const evhdl_tbl = (evhdl_t *)ev_tbl;
547  evstate_cnt_t new_cnt;
548 
549  for (int i = 0; i < num; i++) {
550  const bool refs_used = ev_hdr_tbl[i]->flags.refs_used;
551 
552  /* Update state-count and return value of all counters (atomic) */
553  if (unlikely(is_revert)) {
554  /* Revert em2usr counter update on failed operation */
555  new_cnt.u64 =
556  __atomic_add_fetch(&ev_hdr_tbl[i]->state_cnt.u64,
557  cnt.u64, __ATOMIC_RELAXED);
558  } else {
559  /* Normal em2usr counter update */
560  new_cnt.u64 =
561  __atomic_sub_fetch(&ev_hdr_tbl[i]->state_cnt.u64,
562  cnt.u64, __ATOMIC_RELAXED);
563  }
564 
565  if (!refs_used) {
566  evhdl_tbl[i].evgen = new_cnt.evgen;
567  ev_hdr_tbl[i]->event = evhdl_tbl[i].event;
568  }
569 
570  const int16_t ref_cnt = REF_CNT_INIT - new_cnt.ref_cnt;
571  const int16_t send_cnt = new_cnt.send_cnt - SEND_CNT_INIT;
572 
573  /*
574  * Check state count:
575  * OK: send_cnt < ref_cnt and both >=0. Error otherwise.
576  */
577  if (unlikely(send_cnt >= ref_cnt || send_cnt < 0)) {
578  /* report fatal event-state error, never return */
579  evstate_error(new_cnt, evhdl_tbl[i], ev_hdr_tbl[i],
580  api_op, help_str_em2usr);
581  /* never reached */
582  }
583 
584  /*
585  * Valid state transition, update state (non-atomic)
586  */
587  if (!refs_used)
588  evhdr_update_state(ev_hdr_tbl[i], api_op);
589  }
590 }
591 
592 static inline void
593 esv_usr2em(const em_event_t event, event_hdr_t *const ev_hdr,
594  const evstate_cnt_t cnt, const uint16_t api_op, const bool is_revert)
595 {
596  const bool refs_used = ev_hdr->flags.refs_used;
597  evhdl_t evhdl = {.event = event};
598  evstate_cnt_t new_cnt;
599 
600  /* Update state-count and return value of all counters (atomic) */
601  if (unlikely(is_revert)) {
602  /* Revert previous usr2em counter update on failed operation */
603  new_cnt.u64 = __atomic_sub_fetch(&ev_hdr->state_cnt.u64,
604  cnt.u64, __ATOMIC_RELAXED);
605 
606  if (unlikely(new_cnt.evgen == EVGEN_INIT - 1)) {
607  /* Avoid .evgen counter wrap */
608  const evstate_cnt_t add = {.evgen = EVGEN_MAX - EVGEN_INIT,
609  .rsvd = 0, .ref_cnt = 0, .send_cnt = 0};
610  new_cnt.u64 = __atomic_add_fetch(&ev_hdr->state_cnt.u64,
611  add.u64, __ATOMIC_RELAXED);
612  }
613  } else {
614  /* Normal usr2em counter update */
615  new_cnt.u64 = __atomic_add_fetch(&ev_hdr->state_cnt.u64,
616  cnt.u64, __ATOMIC_RELAXED);
617 
618  if (unlikely(new_cnt.evgen == EVGEN_MAX)) {
619  /* Avoid .evgen counter wrap */
620  const evstate_cnt_t sub = {.evgen = EVGEN_MAX - EVGEN_INIT,
621  .rsvd = 0, .ref_cnt = 0, .send_cnt = 0};
622  __atomic_fetch_sub(&ev_hdr->state_cnt.u64, sub.u64,
623  __ATOMIC_RELAXED);
624  }
625  /* cmp new_cnt.evgen vs evhdl.evgen of previous gen, thus -1 */
626  new_cnt.evgen -= 1;
627  }
628 
629  const int16_t ref_cnt = REF_CNT_INIT - new_cnt.ref_cnt;
630  const int16_t send_cnt = new_cnt.send_cnt - SEND_CNT_INIT;
631 
632  /*
633  * Check state count:
634  * OK: send_cnt <= ref_cnt and both >=0.
635  * AND
636  * OK: event handle evgen == evgen count (not checked for references)
637  * Error otherwise.
638  *
639  * Check evgen only for events that never had references.
640  * Reference usage mixes up the evgen since the same event can be
641  * sent and freed multiple times.
642  */
643  if (unlikely((send_cnt > ref_cnt || send_cnt < 0) ||
644  (!refs_used && evhdl.evgen != new_cnt.evgen))) {
645  const char *const help_str = refs_used ? help_str_usr2em_ref : help_str_usr2em;
646 
647  /* report fatal event-state error, never return */
648  evstate_error(new_cnt, evhdl, ev_hdr, api_op, help_str);
649  /* never reached */
650  }
651 
652  /*
653  * Valid state transition, update state (non-atomic)
654  */
655  if (!refs_used)
656  evhdr_update_state(ev_hdr, api_op);
657 }
658 
659 static inline void
660 esv_usr2em_multi(const em_event_t ev_tbl[],
661  event_hdr_t *const ev_hdr_tbl[], const int num,
662  const evstate_cnt_t cnt, const uint16_t api_op,
663  const bool is_revert)
664 {
665  const evhdl_t *const evhdl_tbl = (const evhdl_t *)ev_tbl;
666  evstate_cnt_t new_cnt;
667 
668  for (int i = 0; i < num; i++) {
669  const bool refs_used = ev_hdr_tbl[i]->flags.refs_used;
670 
671  /* Update state-count and return value of all counters (atomic) */
672  if (unlikely(is_revert)) {
673  /* Revert usr2em counter update on failed operation */
674  new_cnt.u64 =
675  __atomic_sub_fetch(&ev_hdr_tbl[i]->state_cnt.u64,
676  cnt.u64, __ATOMIC_RELAXED);
677 
678  if (unlikely(new_cnt.evgen == EVGEN_INIT - 1)) {
679  /* Avoid .evgen counter wrap */
680  const evstate_cnt_t add = {.evgen = EVGEN_MAX - EVGEN_INIT,
681  .rsvd = 0, .ref_cnt = 0, .send_cnt = 0};
682  new_cnt.u64 =
683  __atomic_add_fetch(&ev_hdr_tbl[i]->state_cnt.u64,
684  add.u64, __ATOMIC_RELAXED);
685  }
686  } else {
687  /* Normal usr2em counter update */
688  new_cnt.u64 =
689  __atomic_add_fetch(&ev_hdr_tbl[i]->state_cnt.u64,
690  cnt.u64, __ATOMIC_RELAXED);
691 
692  if (unlikely(new_cnt.evgen == EVGEN_MAX)) {
693  /* Avoid .evgen counter wrap */
694  const evstate_cnt_t sub = {.evgen = EVGEN_MAX - EVGEN_INIT,
695  .rsvd = 0, .ref_cnt = 0, .send_cnt = 0};
696  __atomic_fetch_sub(&ev_hdr_tbl[i]->state_cnt.u64, sub.u64,
697  __ATOMIC_RELAXED);
698  }
699 
700  new_cnt.evgen -= 1;
701  }
702 
703  const int16_t ref_cnt = REF_CNT_INIT - new_cnt.ref_cnt;
704  const int16_t send_cnt = new_cnt.send_cnt - SEND_CNT_INIT;
705 
706  /*
707  * Check state count:
708  * OK: send_cnt <= ref_cnt and both >=0.
709  * AND
710  * OK: event handle evgen == evgen count (not checked for references)
711  * Error otherwise.
712  *
713  * Check evgen only for events that never had references.
714  * Reference usage mixes up the evgen since the same event can be
715  * sent and freed multiple times.
716  */
717  if (unlikely((send_cnt > ref_cnt || send_cnt < 0) ||
718  (!refs_used && evhdl_tbl[i].evgen != new_cnt.evgen))) {
719  /* report fatal event-state error, never return */
720  evstate_error(new_cnt, evhdl_tbl[i], ev_hdr_tbl[i],
721  api_op, help_str_usr2em);
722  /* never reached */
723  }
724 
725  /*
726  * Valid state transition, update state (non-atomic)
727  */
728  if (!refs_used)
729  evhdr_update_state(ev_hdr_tbl[i], api_op);
730  }
731 }
732 
733 em_event_t evstate_prealloc(const em_event_t event, event_hdr_t *const ev_hdr)
734 {
735  return esv_evinit(event, ev_hdr, init_cnt_alloc, EVSTATE__PREALLOC);
736 }
737 
738 em_event_t evstate_alloc(const em_event_t event, event_hdr_t *const ev_hdr,
739  const uint16_t api_op)
740 {
741  if (!em_shm->opt.esv.prealloc_pools || ev_hdr->flags.refs_used)
742  return esv_evinit(event, ev_hdr, init_cnt_alloc, api_op);
743 
744  const evstate_cnt_t sub = {.evgen = 0, .rsvd = 0,
745  .ref_cnt = 1, .send_cnt = 0};
746 
747  return esv_em2usr(event, ev_hdr, sub, api_op, false);
748 }
749 
750 em_event_t evstate_alloc_tmo(const em_event_t event, event_hdr_t *const ev_hdr)
751 {
752  return esv_evinit(event, ev_hdr, init_cnt_alloc, EVSTATE__TMO_CREATE);
753 }
754 
755 void evstate_alloc_multi(em_event_t ev_tbl[/*in/out*/],
756  event_hdr_t *const ev_hdr_tbl[], const int num)
757 {
758  if (!em_shm->opt.esv.prealloc_pools) {
759  esv_evinit_multi(ev_tbl/*in/out*/, ev_hdr_tbl, num,
760  init_cnt_alloc, EVSTATE__ALLOC_MULTI);
761  return;
762  }
763 
764  /* em_shm->opt.esv.prealloc_pools: */
765  const evstate_cnt_t sub = {.evgen = 0, .rsvd = 0,
766  .ref_cnt = 1, .send_cnt = 0};
767 
768  for (int i = 0; i < num; i++) {
769  if (ev_hdr_tbl[i]->flags.refs_used) {
770  ev_tbl[i] = esv_evinit(ev_tbl[i], ev_hdr_tbl[i],
771  init_cnt_alloc,
772  EVSTATE__ALLOC_MULTI);
773  } else {
774  ev_tbl[i] = esv_em2usr(ev_tbl[i], ev_hdr_tbl[i], sub,
775  EVSTATE__ALLOC_MULTI, false);
776  }
777  }
778 }
779 
780 em_event_t evstate_ref(const em_event_t event, event_hdr_t *const ev_hdr)
781 {
782  const evstate_cnt_t sub = {.evgen = 0, .rsvd = 0,
783  .ref_cnt = 1, .send_cnt = 0};
784 
785  return esv_em2usr(event, ev_hdr, sub, EVSTATE__EVENT_REF, false);
786 }
787 
788 em_event_t evstate_init(const em_event_t event, event_hdr_t *const ev_hdr,
789  bool is_extev)
790 {
791  if (is_extev)
792  return esv_evinit_ext(event, ev_hdr, EVSTATE__INIT_EXTEV);
793  else
794  return esv_evinit(event, ev_hdr, init_cnt_alloc, EVSTATE__INIT);
795 }
796 
797 void evstate_init_multi(em_event_t ev_tbl[/*in/out*/],
798  event_hdr_t *const ev_hdr_tbl[], const int num,
799  bool is_extev)
800 {
801  uint16_t api_op;
802  evstate_cnt_t init_cnt;
803 
804  if (is_extev) {
805  api_op = EVSTATE__INIT_EXTEV_MULTI;
806  init_cnt = init_cnt_extev;
807  } else {
808  api_op = EVSTATE__INIT_MULTI;
809  init_cnt = init_cnt_alloc;
810  }
811 
812  esv_evinit_multi(ev_tbl/*in/out*/, ev_hdr_tbl, num,
813  init_cnt, api_op);
814 }
815 
816 /**
817  * This is a combined calculation of the following three separate
818  * calculations:
819  *
820  * mark allocated:
821  * const evstate_cnt_t sub = {.evgen = 0, .rsvd = 0,
822  * .ref_cnt = 1, .send_cnt = 0};
823  * event = esv_em2usr(event, ev_hdr, sub, api_op, false);
824  *
825  * mark sent:
826  * const evstate_cnt_t add = {.evgen = 1, .rsvd = 0,
827  * .ref_cnt = 0, .send_cnt = 1};
828  * esv_usr2em(event, ev_hdr, add, api_op, false);
829  *
830  * mark em2usr for dispatch to user EO:
831  * const evstate_cnt_t sub2 = {.evgen = 0, .rsvd = 0,
832  * .ref_cnt = 0, .send_cnt = 1};
833  * event = esv_em2usr(event, ev_hdr, sub2, api_op, false);
834  *
835  * combined = add - sub - sub2
836  * add: {.evgen = 1, .rsvd = 0, .ref_cnt = 0, .send_cnt = 1}
837  * sub: - {.evgen = 0, .rsvd = 0, .ref_cnt = 1, .send_cnt = 0}
838  * sub2: - {.evgen = 0, .rsvd = 0, .ref_cnt = 0, .send_cnt = 1}
839  * -------------------------------------------------------
840  * cmb = {.evgen = 1, .rsvd = 0, .ref_cnt =-1, .send_cnt = 0}
841  */
842 static inline em_event_t
843 esv_update_ext(const em_event_t event, event_hdr_t *const ev_hdr,
844  const uint16_t api_op)
845 {
846  const evstate_cnt_t sub = {.evgen = 0, .rsvd = 0,
847  .ref_cnt = 1, .send_cnt = 0};
848  const evstate_cnt_t add = {.evgen = 1, .rsvd = 0,
849  .ref_cnt = 0, .send_cnt = 0};
850  const evstate_cnt_t cmb = {.u64 = add.u64 - sub.u64}; /* combined, wraps */
851 
852  const bool refs_used = ev_hdr->flags.refs_used;
853  evhdl_t evhdl = {.event = event};
854  evstate_cnt_t new_cnt;
855 
856  /* Update state-count and return value of all counters (atomic) */
857  new_cnt.u64 = __atomic_add_fetch(&ev_hdr->state_cnt.u64,
858  cmb.u64, __ATOMIC_RELAXED);
859 
860  if (unlikely(new_cnt.evgen == EVGEN_MAX)) {
861  /* Avoid .evgen counter wrap */
862  const evstate_cnt_t wrap = {.evgen = EVGEN_MAX - EVGEN_INIT,
863  .rsvd = 0, .ref_cnt = 0, .send_cnt = 0};
864  new_cnt.u64 = __atomic_sub_fetch(&ev_hdr->state_cnt.u64, wrap.u64,
865  __ATOMIC_RELAXED);
866  }
867 
868  if (!refs_used) {
869  evhdl.evgen = new_cnt.evgen;
870  ev_hdr->event = evhdl.event;
871  }
872 
873  const int16_t ref_cnt = REF_CNT_INIT - new_cnt.ref_cnt;
874  const int16_t send_cnt = new_cnt.send_cnt - SEND_CNT_INIT;
875 
876  /*
877  * Check state count:
878  * OK: send_cnt < ref_cnt and both >=0. Error otherwise.
879  */
880  if (unlikely(send_cnt >= ref_cnt || send_cnt < 0)) {
881  /* report fatal event-state error, never return */
882  evstate_error(new_cnt, evhdl, ev_hdr, api_op, help_str_em2usr);
883  /* never reached */
884  }
885 
886  /*
887  * Valid state transition, update state (non-atomic)
888  */
889  if (!refs_used)
890  evhdr_update_state(ev_hdr, api_op);
891 
892  return evhdl.event;
893 }
894 
895 em_event_t evstate_update(const em_event_t event, event_hdr_t *const ev_hdr,
896  bool is_extev)
897 {
898  em_event_t ret_event;
899 
900  if (is_extev) {
901  /* combined mark allocated & mark sent */
902  ret_event = esv_update_ext(event, ev_hdr, EVSTATE__UPDATE_EXTEV);
903  } else {
904  /* mark allocated */
905  const evstate_cnt_t sub = {.evgen = 0, .rsvd = 0,
906  .ref_cnt = 1, .send_cnt = 0};
907 
908  ret_event = esv_em2usr(event, ev_hdr, sub, EVSTATE__UPDATE_EXTEV, false);
909  }
910 
911  return ret_event;
912 }
913 
914 void evstate_free(em_event_t event, event_hdr_t *const ev_hdr,
915  const uint16_t api_op)
916 {
917  const evstate_cnt_t add = {.evgen = 1, .rsvd = 0,
918  .ref_cnt = 1, .send_cnt = 0};
919 
920  esv_usr2em(event, ev_hdr, add, api_op, false);
921 }
922 
923 void evstate_free_revert(em_event_t event, event_hdr_t *const ev_hdr,
924  const uint16_t api_op)
925 {
926  const evstate_cnt_t sub = {.evgen = 1, .rsvd = 0,
927  .ref_cnt = 1, .send_cnt = 0};
928 
929  esv_usr2em(event, ev_hdr, sub, api_op, true /*revert*/);
930 }
931 
932 void evstate_free_multi(const em_event_t ev_tbl[],
933  event_hdr_t *const ev_hdr_tbl[], const int num,
934  const uint16_t api_op)
935 {
936  const evstate_cnt_t add = {.evgen = 1, .rsvd = 0,
937  .ref_cnt = 1, .send_cnt = 0};
938 
939  esv_usr2em_multi(ev_tbl, ev_hdr_tbl, num, add, api_op, false);
940 }
941 
942 void evstate_free_revert_multi(const em_event_t ev_tbl[],
943  event_hdr_t *const ev_hdr_tbl[], const int num,
944  const uint16_t api_op)
945 {
946  const evstate_cnt_t sub = {.evgen = 1, .rsvd = 0,
947  .ref_cnt = 1, .send_cnt = 0};
948 
949  esv_usr2em_multi(ev_tbl, ev_hdr_tbl, num, sub, api_op, true /*revert*/);
950 }
951 
952 em_event_t evstate_em2usr(const em_event_t event, event_hdr_t *const ev_hdr,
953  const uint16_t api_op)
954 {
955  const evstate_cnt_t sub = {.evgen = 0, .rsvd = 0,
956  .ref_cnt = 0, .send_cnt = 1};
957 
958  return esv_em2usr(event, ev_hdr, sub, api_op, false);
959 }
960 
961 em_event_t evstate_em2usr_revert(const em_event_t event, event_hdr_t *const ev_hdr,
962  const uint16_t api_op)
963 {
964  const evstate_cnt_t add = {.evgen = 0, .rsvd = 0,
965  .ref_cnt = 0, .send_cnt = 1};
966 
967  return esv_em2usr(event, ev_hdr, add, api_op, true /*revert*/);
968 }
969 
970 void evstate_em2usr_multi(em_event_t ev_tbl[/*in/out*/],
971  event_hdr_t *const ev_hdr_tbl[], const int num,
972  const uint16_t api_op)
973 {
974  const evstate_cnt_t sub = {.evgen = 0, .rsvd = 0,
975  .ref_cnt = 0, .send_cnt = 1};
976 
977  esv_em2usr_multi(ev_tbl/*in/out*/, ev_hdr_tbl, num, sub, api_op, false);
978 }
979 
980 void evstate_em2usr_revert_multi(em_event_t ev_tbl[/*in/out*/],
981  event_hdr_t *const ev_hdr_tbl[], const int num,
982  const uint16_t api_op)
983 {
984  const evstate_cnt_t add = {.evgen = 0, .rsvd = 0,
985  .ref_cnt = 0, .send_cnt = 1};
986 
987  esv_em2usr_multi(ev_tbl/*in/out*/, ev_hdr_tbl, num, add, api_op, true /*revert*/);
988 }
989 
990 void evstate_usr2em(const em_event_t event, event_hdr_t *const ev_hdr,
991  const uint16_t api_op)
992 {
993  const evstate_cnt_t add = {.evgen = 1, .rsvd = 0,
994  .ref_cnt = 0, .send_cnt = 1};
995 
996  esv_usr2em(event, ev_hdr, add, api_op, false);
997 }
998 
999 void evstate_usr2em_revert(const em_event_t event, event_hdr_t *const ev_hdr,
1000  const uint16_t api_op)
1001 {
1002  const evstate_cnt_t sub = {.evgen = 1, .rsvd = 0,
1003  .ref_cnt = 0, .send_cnt = 1};
1004 
1005  esv_usr2em(event, ev_hdr, sub, api_op, true /*revert*/);
1006 }
1007 
1008 void evstate_usr2em_multi(const em_event_t ev_tbl[],
1009  event_hdr_t *const ev_hdr_tbl[], const int num,
1010  const uint16_t api_op)
1011 {
1012  const evstate_cnt_t add = {.evgen = 1, .rsvd = 0,
1013  .ref_cnt = 0, .send_cnt = 1};
1014 
1015  esv_usr2em_multi(ev_tbl, ev_hdr_tbl, num, add, api_op, false);
1016 }
1017 
1018 void evstate_usr2em_revert_multi(const em_event_t ev_tbl[],
1019  event_hdr_t *const ev_hdr_tbl[], const int num,
1020  const uint16_t api_op)
1021 {
1022  const evstate_cnt_t sub = {.evgen = 1, .rsvd = 0,
1023  .ref_cnt = 0, .send_cnt = 1};
1024 
1025  esv_usr2em_multi(ev_tbl, ev_hdr_tbl, num, sub, api_op, true /*revert*/);
1026 }
1027 
1028 /*
1029  * Ensure that em_event_unmark_...() is only called after
1030  * em_event_mark_...() (not after normal em_send/free() etc).
1031  */
1032 static inline void
1033 check_valid_unmark(const event_hdr_t *ev_hdr, uint16_t api_op,
1034  const uint16_t expected_ops[], const int num_ops)
1035 {
1036  /* event refs: can't rely on prev api_op */
1037  if (ev_hdr->flags.refs_used)
1038  return;
1039 
1040  uint16_t prev_op = ev_hdr->state.api_op;
1041 
1042  for (int i = 0; i < num_ops; i++) {
1043  if (prev_op == expected_ops[i])
1044  return; /* success */
1045  }
1046 
1047  /* previous API was NOT em_event_mark_..., report FATAL error! */
1048  evstate_unmark_error(ev_hdr, api_op);
1049 }
1050 
1051 static inline void
1052 check_valid_unmark_multi(event_hdr_t *const ev_hdr_tbl[], const int num_evs,
1053  uint16_t api_op, const uint16_t expected_ops[], const int num_ops)
1054 {
1055  uint16_t prev_op;
1056  bool is_valid;
1057 
1058  for (int i = 0; i < num_evs; i++) {
1059  /* event refs: can't rely on prev api_op */
1060  if (ev_hdr_tbl[i]->flags.refs_used)
1061  continue;
1062 
1063  prev_op = ev_hdr_tbl[i]->state.api_op;
1064  is_valid = false;
1065 
1066  for (int j = 0; j < num_ops; j++) {
1067  if (prev_op == expected_ops[j]) {
1068  is_valid = true;
1069  break; /* success */
1070  }
1071  }
1072 
1073  /* previous API was NOT em_event_mark_..., report FATAL error!*/
1074  if (unlikely(!is_valid))
1075  evstate_unmark_error(ev_hdr_tbl[i], api_op);
1076  }
1077 }
1078 
1079 void evstate_unmark_send(const em_event_t event, event_hdr_t *const ev_hdr)
1080 {
1081  if (em_shm->opt.esv.store_state) {
1082  uint16_t expected_prev_ops[1] = {EVSTATE__MARK_SEND};
1083  /*
1084  * Ensure that em_event_unmark_send() is only called after
1085  * em_event_mark_send/_multi() (not after em_send() etc).
1086  */
1087  check_valid_unmark(ev_hdr, EVSTATE__UNMARK_SEND,
1088  expected_prev_ops, 1);
1089  }
1090 
1091  evstate_usr2em_revert(event, ev_hdr, EVSTATE__UNMARK_SEND);
1092 }
1093 
1094 void evstate_unmark_free(const em_event_t event, event_hdr_t *const ev_hdr,
1095  const uint16_t api_op)
1096 {
1097  if (em_shm->opt.esv.store_state) {
1098  uint16_t expected_prev_ops[2] = {EVSTATE__MARK_FREE,
1099  EVSTATE__MARK_FREE_MULTI};
1100  /*
1101  * Ensure that em_event_unmark_free() is only called
1102  * after em_event_mark_free() (not after em_free() etc).
1103  */
1104  check_valid_unmark(ev_hdr, api_op, expected_prev_ops, 2);
1105  }
1106 
1107  evstate_free_revert(event, ev_hdr, api_op);
1108 }
1109 
1110 void evstate_unmark_free_multi(const em_event_t ev_tbl[],
1111  event_hdr_t *const ev_hdr_tbl[], const int num,
1112  const uint16_t api_op)
1113 {
1114  if (em_shm->opt.esv.store_state) {
1115  uint16_t expected_prev_ops[2] = {EVSTATE__MARK_FREE_MULTI,
1116  EVSTATE__MARK_FREE};
1117  /*
1118  * Ensure that em_event_unmark_free_multi() is only
1119  * called after em_event_mark_free_multi()
1120  * (not after em_free/_multi() etc).
1121  */
1122  check_valid_unmark_multi(ev_hdr_tbl, num, api_op,
1123  expected_prev_ops, 2);
1124  }
1125 
1126  evstate_free_revert_multi(ev_tbl, ev_hdr_tbl, num, api_op);
1127 }
1128 
1129 static int read_config_file(void)
1130 {
1131  const char *conf_str;
1132  bool val_bool = false;
1133  int ret;
1134 
1135  EM_PRINT("EM ESV config: (EM_ESV_ENABLE=%d)\n", EM_ESV_ENABLE);
1136 
1137  /*
1138  * Option: esv.enable - runtime enable/disable
1139  */
1140  conf_str = "esv.enable";
1141  ret = em_libconfig_lookup_bool(&em_shm->libconfig, conf_str, &val_bool);
1142  if (unlikely(!ret)) {
1143  EM_LOG(EM_LOG_ERR, "Config option '%s' not found\n", conf_str);
1144  return -1;
1145  }
1146  /* store & print the value */
1147  em_shm->opt.esv.enable = (int)val_bool;
1148  EM_PRINT(" %s: %s(%d)\n", conf_str, val_bool ? "true" : "false",
1149  val_bool);
1150 
1151  if (!em_shm->opt.esv.enable) {
1152  /* Read no more options if ESV is disabled */
1153  memset(&em_shm->opt.esv, 0, sizeof(em_shm->opt.esv));
1154  return 0;
1155  }
1156 
1157  /*
1158  * Option: esv.store_state
1159  */
1160  conf_str = "esv.store_state";
1161  ret = em_libconfig_lookup_bool(&em_shm->libconfig, conf_str, &val_bool);
1162  if (unlikely(!ret)) {
1163  EM_LOG(EM_LOG_ERR, "Config option '%s' not found\n", conf_str);
1164  return -1;
1165  }
1166  /* store & print the value */
1167  em_shm->opt.esv.store_state = (int)val_bool;
1168  EM_PRINT(" %s: %s(%d)\n", conf_str, val_bool ? "true" : "false",
1169  val_bool);
1170 
1171  /*
1172  * Option: esv.store_payload_first_u32
1173  */
1174  conf_str = "esv.store_payload_first_u32";
1175  ret = em_libconfig_lookup_bool(&em_shm->libconfig, conf_str, &val_bool);
1176  if (unlikely(!ret)) {
1177  EM_LOG(EM_LOG_ERR, "Config option '%s' not found\n", conf_str);
1178  return -1;
1179  }
1180  /* store & print the value */
1181  em_shm->opt.esv.store_first_u32 = (int)val_bool;
1182  EM_PRINT(" %s: %s(%d)\n", conf_str, val_bool ? "true" : "false",
1183  val_bool);
1184 
1185  /*
1186  * Option: esv.prealloc_pools
1187  */
1188  conf_str = "esv.prealloc_pools";
1189  ret = em_libconfig_lookup_bool(&em_shm->libconfig, conf_str, &val_bool);
1190  if (unlikely(!ret)) {
1191  EM_LOG(EM_LOG_ERR, "Config option '%s' not found\n", conf_str);
1192  return -1;
1193  }
1194  /* store & print the value */
1195  em_shm->opt.esv.prealloc_pools = (int)val_bool;
1196  EM_PRINT(" %s: %s(%d)\n", conf_str, val_bool ? "true" : "false",
1197  val_bool);
1198 
1199  return 0;
1200 }
1201 
1202 em_status_t esv_init(void)
1203 {
1204  if (read_config_file())
1205  return EM_ERR_LIB_FAILED;
1206 
1207  return EM_OK;
1208 }
1209 
1210 void esv_disabled_warn_config(void)
1211 {
1212  const char *conf_str = "esv.enable";
1213  bool val_bool = false;
1214  int ret;
1215 
1216  EM_PRINT("EM ESV config: (EM_ESV_ENABLE=%d)\n", EM_ESV_ENABLE);
1217  EM_PRINT(" ESV disabled\n");
1218 
1219  ret = em_libconfig_lookup_bool(&em_shm->libconfig, conf_str, &val_bool);
1220  if (unlikely(!ret))
1221  return; /* ESV state option not found in runtime, no warning */
1222 
1223  EM_PRINT(" %s: %s(%d)\n", conf_str, val_bool ? "true" : "false", val_bool);
1224 
1225  if (unlikely(val_bool))
1226  EM_PRINT(" WARNING: ESV disabled (build-time) - config file option IGNORED!\n");
1227 }
EM_OK
#define EM_OK
Definition: event_machine_types.h:329
evstate_unmark_free
void evstate_unmark_free(const em_event_t event, event_hdr_t *const ev_hdr, const uint16_t api_op)
Definition: em_event_state.c:1094
evstate_em2usr_revert_multi
void evstate_em2usr_revert_multi(em_event_t ev_tbl[], event_hdr_t *const ev_hdr_tbl[], const int num, const uint16_t api_op)
Definition: em_event_state.c:980
EVGEN_MAX
#define EVGEN_MAX
Definition: em_event_state.h:113
REF_CNT_INIT
#define REF_CNT_INIT
Definition: em_event_state.h:117
ODP_PACKED
Definition: em_event_types.h:129
EM_EVENT_UNDEF
#define EM_EVENT_UNDEF
Definition: event_machine_types.h:62
evstate_free_multi
void evstate_free_multi(const em_event_t ev_tbl[], event_hdr_t *const ev_hdr_tbl[], const int num, const uint16_t api_op)
Definition: em_event_state.c:932
EM_ESCOPE_POOL_CREATE
#define EM_ESCOPE_POOL_CREATE
Definition: event_machine_hw_types.h:489
EM_ESCOPE_TERM
#define EM_ESCOPE_TERM
Definition: event_machine_hw_types.h:473
evstate_usr2em_revert_multi
void evstate_usr2em_revert_multi(const em_event_t ev_tbl[], event_hdr_t *const ev_hdr_tbl[], const int num, const uint16_t api_op)
Definition: em_event_state.c:1018
esv_disabled_warn_config
void esv_disabled_warn_config(void)
Definition: em_event_state.c:1210
em_locm
ENV_LOCAL em_locm_t em_locm
evstate_usr2em_revert
void evstate_usr2em_revert(em_event_t event, event_hdr_t *const ev_hdr, const uint16_t api_op)
Definition: em_event_state.c:999
em_locm_t::current
em_locm_current_t current
Definition: em_mem.h:190
evstate_init_multi
void evstate_init_multi(em_event_t ev_tbl[], event_hdr_t *const ev_hdr_tbl[], const int num, bool is_extev)
Definition: em_event_state.c:797
EM_ERR_LIB_FAILED
@ EM_ERR_LIB_FAILED
Definition: event_machine_hw_types.h:291
event_hdr::refs_used
uint8_t refs_used
Definition: em_event_types.h:220
EM_EO_UNDEF
#define EM_EO_UNDEF
Definition: event_machine_types.h:95
ODP_PACKED::payload_first
uint32_t payload_first
Definition: em_event_types.h:134
evstate_free
void evstate_free(em_event_t event, event_hdr_t *const ev_hdr, const uint16_t api_op)
Definition: em_event_state.c:914
evstate_init
em_event_t evstate_init(const em_event_t event, event_hdr_t *const ev_hdr, bool is_extev)
Definition: em_event_state.c:788
evhdl_t
Definition: em_event_types.h:67
event_hdr::flags
union event_hdr::@34 flags
evstate_free_revert
void evstate_free_revert(em_event_t event, event_hdr_t *const ev_hdr, const uint16_t api_op)
Definition: em_event_state.c:923
evstate_info_t
Definition: em_event_state.c:55
event_hdr::event
em_event_t event
Definition: em_event_types.h:246
EM_QUEUE_NAME_LEN
#define EM_QUEUE_NAME_LEN
Definition: event_machine_config.h:125
event_hdr
Definition: em_event_types.h:184
eo_elem_t
Definition: em_eo_types.h:47
ODP_PACKED::queue_idx
int16_t queue_idx
Definition: em_event_types.h:148
evstate_update
em_event_t evstate_update(const em_event_t event, event_hdr_t *const ev_hdr, bool is_extev)
Definition: em_event_state.c:895
em_shm_t::libconfig
libconfig_t libconfig
Definition: em_mem.h:146
queue_elem_t::queue
uint32_t queue
Definition: em_queue_types.h:225
ODP_PACKED::api_op
uint8_t api_op
Definition: em_event_types.h:154
evstate_em2usr_revert
em_event_t evstate_em2usr_revert(em_event_t event, event_hdr_t *const ev_hdr, const uint16_t api_op)
Definition: em_event_state.c:961
evstate_alloc
em_event_t evstate_alloc(const em_event_t event, event_hdr_t *const ev_hdr, const uint16_t api_op)
Definition: em_event_state.c:738
EM_EO_NAME_LEN
#define EM_EO_NAME_LEN
Definition: event_machine_config.h:155
INTERNAL_ERROR
#define INTERNAL_ERROR(error, escope, fmt,...)
Definition: em_error.h:43
em_locm_t::core_id
int core_id
Definition: em_mem.h:196
em_escope_t
uint32_t em_escope_t
Definition: event_machine_types.h:348
event_hdr::state
ev_hdr_state_t state
Definition: em_event_types.h:207
evstate_alloc_tmo
em_event_t evstate_alloc_tmo(const em_event_t event, event_hdr_t *const ev_hdr)
Definition: em_event_state.c:750
em_status_t
uint32_t em_status_t
Definition: event_machine_types.h:321
ODP_PACKED::q_elem
queue_elem_t * q_elem
Definition: em_mem.h:174
evstate_free_revert_multi
void evstate_free_revert_multi(const em_event_t ev_tbl[], event_hdr_t *const ev_hdr_tbl[], const int num, const uint16_t api_op)
Definition: em_event_state.c:942
event_hdr::state_cnt
evstate_cnt_t state_cnt
Definition: em_event_types.h:195
ODP_PACKED::core
uint8_t core
Definition: em_event_types.h:156
evstate_em2usr
em_event_t evstate_em2usr(em_event_t event, event_hdr_t *const ev_hdr, const uint16_t api_op)
Definition: em_event_state.c:952
evstate_prealloc
em_event_t evstate_prealloc(const em_event_t event, event_hdr_t *const ev_hdr)
Definition: em_event_state.c:733
em_shm
em_shm_t * em_shm
Definition: event_machine_init.c:41
EM_QUEUE_UNDEF
#define EM_QUEUE_UNDEF
Definition: event_machine_types.h:107
evstate_usr2em_multi
void evstate_usr2em_multi(const em_event_t ev_tbl[], event_hdr_t *const ev_hdr_tbl[], const int num, const uint16_t api_op)
Definition: em_event_state.c:1008
em_include.h
EM_ESV_ENABLE
#define EM_ESV_ENABLE
Definition: event_machine_config.h:275
evstate_alloc_multi
void evstate_alloc_multi(em_event_t ev_tbl[], event_hdr_t *const ev_hdr_tbl[], const int num)
Definition: em_event_state.c:755
evstate_unmark_free_multi
void evstate_unmark_free_multi(const em_event_t ev_tbl[], event_hdr_t *const ev_hdr_tbl[], const int num, const uint16_t api_op)
Definition: em_event_state.c:1110
SEND_CNT_INIT
#define SEND_CNT_INIT
Definition: em_event_state.h:115
queue_elem_t::eo
uint16_t eo
Definition: em_queue_types.h:222
ODP_PACKED::eo_idx
int16_t eo_idx
Definition: em_event_types.h:141
EVGEN_INIT
#define EVGEN_INIT
Definition: em_event_state.h:111
EM_ERR_EVENT_STATE
@ EM_ERR_EVENT_STATE
Definition: event_machine_hw_types.h:299
evstate_em2usr_multi
void evstate_em2usr_multi(em_event_t ev_tbl[], event_hdr_t *const ev_hdr_tbl[], const int num, const uint16_t api_op)
Definition: em_event_state.c:970
evstate_ref
em_event_t evstate_ref(const em_event_t event, event_hdr_t *const ev_hdr)
Definition: em_event_state.c:780
esv_init
em_status_t esv_init(void)
Definition: em_event_state.c:1202
EM_ESCOPE_TERM_CORE
#define EM_ESCOPE_TERM_CORE
Definition: event_machine_hw_types.h:478
em_locm_t
Definition: em_mem.h:188
evstate_usr2em
void evstate_usr2em(em_event_t event, event_hdr_t *const ev_hdr, const uint16_t api_op)
Definition: em_event_state.c:990
EM_ESCOPE_ODP_EXT
#define EM_ESCOPE_ODP_EXT
Definition: event_machine_hw_types.h:678
evstate_unmark_send
void evstate_unmark_send(const em_event_t event, event_hdr_t *const ev_hdr)
Definition: em_event_state.c:1079
queue_elem_t
Definition: em_queue_types.h:180
EM_ESCOPE_INTERNAL_MASK
#define EM_ESCOPE_INTERNAL_MASK
Definition: event_machine_hw_types.h:445