EM-ODP 4.4.0
Event Machine on ODP
Loading...
Searching...
No Matches
em_event_state.c
1/*
2 * Copyright (c) 2020-2022, Nokia Solutions and Networks
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 *
9 * * Redistributions of source code must retain the above copyright
10 * notice, this list of conditions and the following disclaimer.
11 * * Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
14 * * Neither the name of the copyright holder nor the names of its
15 * contributors may be used to endorse or promote products derived
16 * from this software without specific prior written permission.
17 *
18 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29 */
30
31#ifndef _GNU_SOURCE
32#define _GNU_SOURCE
33#endif
34
35#ifdef HAVE_CONFIG_H
36#include "config.h"
37#endif
38
39#include <stdbool.h>
40#include <stdint.h>
41#include <stdio.h>
42#include <string.h>
43
44#include <odp_api.h>
45
46#include <event_machine.h>
47#include <event_machine/platform/env/environment.h>
48
49#include "em_eo.h"
50#include "em_error.h"
51#include "em_event.h"
52#include "em_event_inline.h"
53#include "em_event_state.h"
54#include "em_libconfig.h"
55#include "em_queue.h"
56#include "em_event_types.h"
57#include "em_mem.h"
58#include "em_pool.h"
59#include "em_pool_types.h"
60#include "em_queue_inline.h"
61#include "em_queue_types.h"
62
63static int read_config_file(void);
64
65/**
66 * Initial counter values set during an alloc-operation: ref=1, send=0
67 * (em_alloc/_multi(), em_event_clone())
68 */
69static const evstate_cnt_t init_cnt_alloc = {.evgen = EVGEN_INIT,
70 .rsvd = 0,
71 .ref_cnt = REF_CNT_INIT - 1,
72 .send_cnt = 0 + SEND_CNT_INIT};
73/**
74 * Initial counter values for external events entering into EM
75 * (event not allocated by EM): ref=1, send=1
76 */
77static const evstate_cnt_t init_cnt_extev = {.evgen = EVGEN_INIT,
78 .rsvd = 0,
79 .ref_cnt = REF_CNT_INIT - 1,
80 .send_cnt = 1 + SEND_CNT_INIT};
81
82/**
83 * Information about an event-state update location
84 */
85typedef struct {
86 const char *str;
87 em_escope_t escope;
88} evstate_info_t;
89
90/**
91 * Constant table containing event-state update location information.
92 * Only accessed when an erroneous event state has been detected and is being
93 * reported to the error handler.
94 */
95static const evstate_info_t evstate_info_tbl[] = {
96 [EVSTATE__UNDEF] = {.str = "undefined",
97 .escope = (EM_ESCOPE_INTERNAL_MASK | 0)},
98 [EVSTATE__PREALLOC] = {.str = "pool-create(prealloc-events)",
99 .escope = EM_ESCOPE_POOL_CREATE},
100 [EVSTATE__ALLOC] = {.str = "em_alloc()",
101 .escope = EM_ESCOPE_ALLOC},
102 [EVSTATE__ALLOC_MULTI] = {.str = "em_alloc_multi()",
103 .escope = EM_ESCOPE_ALLOC_MULTI},
104 [EVSTATE__EVENT_CLONE] = {.str = "em_event_clone()",
105 .escope = EM_ESCOPE_EVENT_CLONE},
106 [EVSTATE__EVENT_REF] = {.str = "em_event_ref()",
107 .escope = EM_ESCOPE_EVENT_REF},
108 [EVSTATE__FREE] = {.str = "em_free()",
109 .escope = EM_ESCOPE_FREE},
110 [EVSTATE__FREE_MULTI] = {.str = "em_free_multi()",
111 .escope = EM_ESCOPE_FREE_MULTI},
112 [EVSTATE__EVENT_VECTOR_FREE] = {.str = "em_event_vector_free()",
113 .escope = EM_ESCOPE_EVENT_VECTOR_FREE},
114 [EVSTATE__INIT] = {.str = "init-event",
115 .escope = EM_ESCOPE_ODP_EXT},
116 [EVSTATE__INIT_MULTI] = {.str = "init-events",
117 .escope = EM_ESCOPE_ODP_EXT},
118 [EVSTATE__INIT_EXTEV] = {.str = "dispatch(init-ext-event)",
119 .escope = EM_ESCOPE_DISPATCH},
120 [EVSTATE__INIT_EXTEV_MULTI] = {.str = "dispatch(init-ext-events)",
121 .escope = EM_ESCOPE_DISPATCH},
122 [EVSTATE__UPDATE_EXTEV] = {.str = "dispatch(update-ext-event)",
123 .escope = EM_ESCOPE_DISPATCH},
124 [EVSTATE__SEND] = {.str = "em_send()",
125 .escope = EM_ESCOPE_SEND},
126 [EVSTATE__SEND__FAIL] = {.str = "em_send(fail)",
127 .escope = EM_ESCOPE_SEND},
128 [EVSTATE__SEND_EGRP] = {.str = "em_send_group()",
129 .escope = EM_ESCOPE_SEND_GROUP},
130 [EVSTATE__SEND_EGRP__FAIL] = {.str = "em_send_group(fail)",
131 .escope = EM_ESCOPE_SEND_GROUP},
132 [EVSTATE__SEND_MULTI] = {.str = "em_send_multi()",
133 .escope = EM_ESCOPE_SEND_MULTI},
134 [EVSTATE__SEND_MULTI__FAIL] = {.str = "em_send_multi(fail)",
135 .escope = EM_ESCOPE_SEND_MULTI},
136 [EVSTATE__SEND_EGRP_MULTI] = {.str = "em_send_group_multi()",
137 .escope = EM_ESCOPE_SEND_GROUP_MULTI},
138 [EVSTATE__SEND_EGRP_MULTI__FAIL] = {.str = "em_send_group_multi(fail)",
139 .escope = EM_ESCOPE_SEND_GROUP_MULTI},
140 [EVSTATE__EO_START_SEND_BUFFERED] = {.str = "eo-start:send-buffered-events()",
141 .escope = EM_ESCOPE_SEND_MULTI},
142 [EVSTATE__MARK_SEND] = {.str = "em_event_mark_send()",
143 .escope = EM_ESCOPE_EVENT_MARK_SEND},
144 [EVSTATE__UNMARK_SEND] = {.str = "em_event_unmark_send()",
145 .escope = EM_ESCOPE_EVENT_UNMARK_SEND},
146 [EVSTATE__MARK_FREE] = {.str = "em_event_mark_free()",
147 .escope = EM_ESCOPE_EVENT_MARK_FREE},
148 [EVSTATE__UNMARK_FREE] = {.str = "em_event_unmark_free()",
149 .escope = EM_ESCOPE_EVENT_UNMARK_FREE},
150 [EVSTATE__MARK_FREE_MULTI] = {.str = "em_event_mark_free_multi()",
151 .escope = EM_ESCOPE_EVENT_MARK_FREE_MULTI},
152 [EVSTATE__UNMARK_FREE_MULTI] = {.str = "em_event_unmark_free_multi()",
153 .escope = EM_ESCOPE_EVENT_UNMARK_FREE_MULTI},
154 [EVSTATE__DISPATCH] = {.str = "em_dispatch(single-event)",
155 .escope = EM_ESCOPE_DISPATCH},
156 [EVSTATE__DISPATCH_MULTI] = {.str = "em_dispatch(multiple-events)",
157 .escope = EM_ESCOPE_DISPATCH},
158 [EVSTATE__DISPATCH_AGGR] = {.str = "em_dispatch(aggr-vec inner-event)",
159 .escope = EM_ESCOPE_DISPATCH},
160 [EVSTATE__DISPATCH_SCHED__FAIL] = {.str = "em_dispatch(drop sched-events)",
161 .escope = EM_ESCOPE_DISPATCH},
162 [EVSTATE__DISPATCH_LOCAL__FAIL] = {.str = "em_dispatch(drop local-events)",
163 .escope = EM_ESCOPE_DISPATCH},
164 [EVSTATE__DEQUEUE] = {.str = "em_queue_dequeue()",
165 .escope = EM_ESCOPE_QUEUE_DEQUEUE},
166 [EVSTATE__DEQUEUE_MULTI] = {.str = "em_queue_dequeue_multi()",
167 .escope = EM_ESCOPE_QUEUE_DEQUEUE_MULTI},
168 [EVSTATE__TMO_SET_ABS] = {.str = "em_tmo_set_abs()",
169 .escope = EM_ESCOPE_TMO_SET_ABS},
170 [EVSTATE__TMO_SET_ABS__FAIL] = {.str = "em_tmo_set_abs(fail)",
171 .escope = EM_ESCOPE_TMO_SET_ABS},
172 [EVSTATE__TMO_SET_REL] = {.str = "em_tmo_set_rel()",
173 .escope = EM_ESCOPE_TMO_SET_REL},
174 [EVSTATE__TMO_SET_REL__FAIL] = {.str = "em_tmo_set_rel(fail)",
175 .escope = EM_ESCOPE_TMO_SET_REL},
176 [EVSTATE__TMO_SET_PERIODIC] = {.str = "em_tmo_set_periodic()",
177 .escope = EM_ESCOPE_TMO_SET_PERIODIC},
178 [EVSTATE__TMO_SET_PERIODIC__FAIL] = {.str = "em_tmo_set_periodic(fail)",
179 .escope = EM_ESCOPE_TMO_SET_PERIODIC},
180 [EVSTATE__TMO_CANCEL] = {.str = "em_tmo_cancel()",
181 .escope = EM_ESCOPE_TMO_CANCEL},
182 [EVSTATE__TMO_ACK] = {.str = "em_tmo_ack()",
183 .escope = EM_ESCOPE_TMO_ACK},
184 [EVSTATE__TMO_ACK__NOSKIP] = {.str = "em_tmo_ack(noskip)",
185 .escope = EM_ESCOPE_TMO_ACK},
186 [EVSTATE__TMO_ACK__FAIL] = {.str = "em_tmo_ack(fail)",
187 .escope = EM_ESCOPE_TMO_ACK},
188 [EVSTATE__TMO_CREATE] = {.str = "em_tmo_create()",
189 .escope = EM_ESCOPE_TMO_CREATE},
190 [EVSTATE__TMO_DELETE] = {.str = "em_tmo_delete()",
191 .escope = EM_ESCOPE_TMO_DELETE},
192 [EVSTATE__AG_DELETE] = {.str = "em_atomic_group_delete(flush)",
193 .escope = EM_ESCOPE_ATOMIC_GROUP_DELETE},
194 [EVSTATE__TERM_CORE__QUEUE_LOCAL] = {.str = "em_term_core(local-queue)",
195 .escope = EM_ESCOPE_TERM_CORE},
196 [EVSTATE__TERM] = {.str = "em_term()",
197 .escope = EM_ESCOPE_TERM},
198 /* Last: */
199 [EVSTATE__LAST] = {.str = "last",
200 .escope = (EM_ESCOPE_INTERNAL_MASK | 0)}
201};
202
203static const char *const help_str_em2usr =
204"OK: 'send < ref, both >=0'. Err otherwise";
205static const char *const help_str_usr2em =
206"OK: 'send <= ref, both >=0' AND 'hdl evgen == evgen'. Err otherwise";
207static const char *const help_str_usr2em_ref =
208"OK: 'send <= ref, both >=0'. Err otherwise";
209
210static inline void
211esv_update_state(ev_hdr_state_t *const evstate, const uint16_t api_op,
212 const void *const ev_ptr)
213{
214 const em_locm_t *const locm = &em_locm;
215 const uint32_t *const pl_u32 = ev_ptr;
216 const queue_elem_t *const q_elem = locm->current.q_elem;
217
218 if (ev_ptr)
219 evstate->payload_first = *pl_u32;
220
221 if (!q_elem) {
222 evstate->eo_idx = (int16_t)eo_hdl2idx(EM_EO_UNDEF); /* -1 is fine */
223 evstate->queue_idx = (int16_t)queue_hdl2idx(EM_QUEUE_UNDEF); /* -1 is fine */
224 } else {
225 evstate->eo_idx = (int16_t)eo_hdl2idx((em_eo_t)(uintptr_t)q_elem->eo);
226 evstate->queue_idx = (int16_t)queue_hdl2idx((em_queue_t)(uintptr_t)q_elem->queue);
227 }
228 evstate->api_op = (uint8_t)api_op; /* no truncation */
229 evstate->core = locm->core_id;
230}
231
232static inline void
233evhdr_update_state(event_hdr_t *const ev_hdr, const uint16_t api_op)
234{
235 if (!em_shm->opt.esv.store_state)
236 return; /* don't store updated state */
237
238 const void *ev_ptr = NULL;
239
240 if (em_shm->opt.esv.store_first_u32)
241 ev_ptr = event_pointer(ev_hdr->event);
242
243 esv_update_state(&ev_hdr->state, api_op, ev_ptr);
244}
245
246/* "Normal" ESV Error format */
247#define EVSTATE_ERROR_FMT \
248"ESV: Event:%" PRI_EVENT " state error -- counts:\t" \
249"send:%" PRIi16 " ref:%" PRIi16 " evgen:%" PRIu16 "(%" PRIu16 ")\n" \
250" Help: %s\n" \
251" prev-state:%s core:%02u:\t" \
252" EO:%" PRI_EO "-\"%s\" Q:%" PRI_QUEUE "-\"%s\" u32[0]:%s\n" \
253"=> err-state:%s core:%02u:\t" \
254" EO:%" PRI_EO "-\"%s\" Q:%" PRI_QUEUE "-\"%s\" u32[0]:%s\n" \
255" event:0x%016" PRIx64 ": ptr:0x%" PRIx64 ""
256
257/* ESV Error format for references */
258#define EVSTATE_REF_ERROR_FMT \
259"ESV: RefEvent:%" PRI_EVENT " state error -- counts:\t" \
260"send:%" PRIi16 " ref:%" PRIi16 " (evgen:%" PRIu16 " ignored for refs)\n" \
261" Help: %s\n" \
262" prev-state:n/a (not valid for event references)\n" \
263"=> err-state:%s core:%02u:\t" \
264" EO:%" PRI_EO "-\"%s\" Q:%" PRI_QUEUE "-\"%s\" u32[0]:%s\n" \
265" event:0x%016" PRIx64 ": ptr:0x%" PRIx64 ""
266
267/* ESV Error format for em_event_unmark_send/free/_multi() */
268#define EVSTATE_UNMARK_ERROR_FMT \
269"ESV: Event:%" PRI_EVENT " state error - Invalid 'unmark'-API use\n"\
270" prev-state:%s core:%02u:\t" \
271" EO:%" PRI_EO "-\"%s\" Q:%" PRI_QUEUE "-\"%s\" u32[0]:%s\n" \
272"=> err-state:%s core:%02u:\t" \
273" EO:%" PRI_EO "-\"%s\" Q:%" PRI_QUEUE "-\"%s\" u32[0]:%s\n"
274
275/* ESV Error format when esv.store_state = false */
276#define EVSTATE__NO_PREV_STATE__ERROR_FMT \
277"ESV: Event:%" PRI_EVENT " state error -- counts:\t" \
278"send:%" PRIi16 " ref:%" PRIi16 " evgen:%" PRIu16 "(%" PRIu16 ")\n" \
279" Help: %s\n" \
280" prev-state:n/a (disabled in conf)\n" \
281"=> err-state:%s core:%02u:\t" \
282" EO:%" PRI_EO "-\"%s\" Q:%" PRI_QUEUE "-\"%s\" u32[0]:%s\n" \
283" event:0x%016" PRIx64 ": ptr:0x%" PRIx64 ""
284
285/* ESV Error format for em_event_unmark_send/free/_multi() when esv.store_state = false */
286#define EVSTATE__NO_PREV_STATE__UNMARK_ERROR_FMT \
287"ESV: Event:%" PRI_EVENT " state error - Invalid 'unmark'-API use\n"\
288" prev-state:n/a (disabled in conf)\n" \
289"=> err-state:%s core:%02u:\t" \
290" EO:%" PRI_EO "-\"%s\" Q:%" PRI_QUEUE "-\"%s\" u32[0]:%s\n"
291
292/**
293 * ESV Error reporting
294 */
295static inline void
296esv_error(const evstate_cnt_t cnt,
297 evhdl_t evhdl, const event_hdr_t *const ev_hdr,
298 const uint16_t api_op, bool is_unmark_error,
299 const char *const help_str)
300{
301 uint16_t prev_op = ev_hdr->state.api_op;
302 ev_hdr_state_t prev_state = ev_hdr->state; /* store prev good state */
303 ev_hdr_state_t err_state = {0}; /* store current invalid/error state */
304 const em_event_t event = event_hdr_to_event(ev_hdr);
305 const void *ev_ptr = NULL;
306
307 if (unlikely(prev_op > EVSTATE__LAST))
308 prev_op = EVSTATE__UNDEF;
309
310 const evstate_info_t *err_info = &evstate_info_tbl[api_op];
311 const evstate_info_t *prev_info = &evstate_info_tbl[prev_op];
312
313 char curr_eoname[EM_EO_NAME_LEN] = "(noname)";
314 char prev_eoname[EM_EO_NAME_LEN] = "(noname)";
315 char curr_qname[EM_QUEUE_NAME_LEN] = "(noname)";
316 char prev_qname[EM_QUEUE_NAME_LEN] = "(noname)";
317 char curr_payload[sizeof("0x12345678 ")] = "(n/a)";
318 char prev_payload[sizeof("0x12345678 ")] = "(n/a)";
319
320 const eo_elem_t *eo_elem;
321 const queue_elem_t *q_elem;
322
323 /* Check event!=undef to avoid error in event_pointer() */
324 if (likely(event != EM_EVENT_UNDEF))
325 ev_ptr = event_pointer(event);
326 /* Store the new _invalid_ event-state info into a separate struct */
327 esv_update_state(&err_state, api_op, ev_ptr);
328
329 /*
330 * Print the first 32bits of the event payload on failure,
331 * the option 'esv.store_payload_first_u32' affects storing during valid
332 * state transitions.
333 */
334 if (ev_ptr) {
335 snprintf(curr_payload, sizeof(curr_payload),
336 "0x%08" PRIx32 "", err_state.payload_first);
337 curr_payload[sizeof(curr_payload) - 1] = '\0';
338 }
339
340 em_eo_t curr_eo = eo_idx2hdl(err_state.eo_idx);
341 em_queue_t curr_queue = queue_idx2hdl(err_state.queue_idx);
342
343 /* current EO-name: */
344 eo_elem = eo_elem_get(curr_eo);
345 if (eo_elem != NULL)
346 eo_name(eo_elem, curr_eoname, sizeof(curr_eoname));
347 /* current queue-name: */
348 q_elem = queue_elem_get(curr_queue);
349 if (q_elem != NULL)
350 queue_name(q_elem, curr_qname, sizeof(curr_qname));
351
352 const int16_t send_cnt = cnt.send_cnt - SEND_CNT_INIT;
353 uint16_t evgen_cnt = cnt.evgen - EVGEN_INIT;
354 const uint16_t evgen_hdl = evhdl.evgen - EVGEN_INIT;
355 const int16_t ref_cnt = REF_CNT_INIT - cnt.ref_cnt;
356
357 /* Read the previous event state only if it has been stored */
358 if (em_shm->opt.esv.store_state) {
359 /*
360 * Print the first 32 bits of the event payload for the previous
361 * valid state transition, if enabled in the EM config file:
362 * 'esv.store_payload_first_u32 = true', otherwise not stored.
363 */
364 if (em_shm->opt.esv.store_first_u32) {
365 snprintf(prev_payload, sizeof(prev_payload),
366 "0x%08" PRIx32 "", prev_state.payload_first);
367 prev_payload[sizeof(prev_payload) - 1] = '\0';
368 }
369
370 em_eo_t prev_eo = eo_idx2hdl(prev_state.eo_idx);
371 em_queue_t prev_queue = queue_idx2hdl(prev_state.queue_idx);
372
373 /* previous EO-name: */
374 eo_elem = eo_elem_get(prev_eo);
375 if (eo_elem != NULL)
376 eo_name(eo_elem, prev_eoname, sizeof(prev_eoname));
377 /* previous queue-name: */
378 q_elem = queue_elem_get(prev_queue);
379 if (q_elem != NULL)
380 queue_name(q_elem, prev_qname, sizeof(prev_qname));
381
382 if (ev_hdr->flags.refs_used) {
383 /* Reference ESV Error, prev state available */
384 INTERNAL_ERROR(EM_FATAL(EM_ERR_EVENT_STATE),
385 err_info->escope, EVSTATE_REF_ERROR_FMT,
386 event, send_cnt, ref_cnt, evgen_cnt, help_str,
387 err_info->str, err_state.core,
388 curr_eo, curr_eoname, curr_queue, curr_qname,
389 curr_payload, evhdl.event, evhdl.evptr);
390 } else if (!is_unmark_error) {
391 /* "Normal" ESV Error, prev state available */
392 INTERNAL_ERROR(EM_FATAL(EM_ERR_EVENT_STATE),
393 err_info->escope, EVSTATE_ERROR_FMT,
394 event, send_cnt, ref_cnt, evgen_hdl, evgen_cnt, help_str,
395 prev_info->str, prev_state.core, prev_eo, prev_eoname,
396 prev_queue, prev_qname, prev_payload,
397 err_info->str, err_state.core, curr_eo, curr_eoname,
398 curr_queue, curr_qname, curr_payload,
399 evhdl.event, evhdl.evptr);
400 } else {
401 /*
402 * ESV Error from em_event_unmark_send/free/_multi(),
403 * prev state available.
404 */
405 INTERNAL_ERROR(EM_FATAL(EM_ERR_EVENT_STATE),
406 err_info->escope, EVSTATE_UNMARK_ERROR_FMT,
407 event,
408 prev_info->str, prev_state.core,
409 prev_eo, prev_eoname,
410 prev_queue, prev_qname, prev_payload,
411 err_info->str, err_state.core,
412 curr_eo, curr_eoname,
413 curr_queue, curr_qname, curr_payload);
414 }
415 } else { /* em_shm->opt.esv.store_state == false */
416 /* No previous state stored by EM at runtime */
417 if (!is_unmark_error) {
418 /* "Normal" ESV Error, prev state not stored */
419 INTERNAL_ERROR(EM_FATAL(EM_ERR_EVENT_STATE),
420 err_info->escope, EVSTATE__NO_PREV_STATE__ERROR_FMT,
421 event, send_cnt, ref_cnt, evgen_hdl, evgen_cnt, help_str,
422 err_info->str, err_state.core, curr_eo, curr_eoname,
423 curr_queue, curr_qname, curr_payload,
424 evhdl.event, evhdl.evptr);
425 } else {
426 /*
427 * ESV Error from em_event_unmark_send/free/_multi(),
428 * prev state not stored.
429 */
430 INTERNAL_ERROR(EM_FATAL(EM_ERR_EVENT_STATE),
431 err_info->escope, EVSTATE__NO_PREV_STATE__UNMARK_ERROR_FMT,
432 event,
433 err_info->str, err_state.core, curr_eo, curr_eoname,
434 curr_queue, curr_qname, curr_payload);
435 }
436 }
437}
438
439static void
440evstate_error(const evstate_cnt_t cnt, evhdl_t evhdl,
441 const event_hdr_t *const ev_hdr, const uint16_t api_op,
442 const char *const help_str)
443{
444 /* "Normal" ESV Error */
445 esv_error(cnt, evhdl, ev_hdr, api_op, false, help_str);
446}
447
448/**
449 * ESV Error reporting for invalid em_event_unmark...() API use
450 */
451static void
452evstate_unmark_error(const event_hdr_t *const ev_hdr, const uint16_t api_op)
453{
454 evstate_cnt_t dont_care = {.u64 = 0};
455 evhdl_t dont_care_hdl = {.event = EM_EVENT_UNDEF};
456
457 /* ESV Error from em_event_unmark_send/free/_multi() */
458 esv_error(dont_care, dont_care_hdl, ev_hdr, api_op, true, "n/a");
459}
460
461static inline em_event_t
462esv_evinit(const em_event_t event, event_hdr_t *const ev_hdr,
463 const evstate_cnt_t init_cnt, const uint16_t api_op)
464{
465 evhdl_t evhdl = {.event = event};
466
467 evhdl.evgen = EVGEN_INIT;
468 ev_hdr->event = evhdl.event;
469
470 /* Set initial counters (atomic) */
471 __atomic_store_n(&ev_hdr->state_cnt.u64, init_cnt.u64,
472 __ATOMIC_RELAXED);
473 /* Set initial state information (non-atomic) */
474 evhdr_update_state(ev_hdr, api_op);
475
476 return evhdl.event;
477}
478
479static inline void
480esv_evinit_multi(em_event_t ev_tbl[/*in/out*/],
481 event_hdr_t *const ev_hdr_tbl[], const int num,
482 const evstate_cnt_t init_cnt, const uint16_t api_op)
483{
484 evhdl_t *const evhdl_tbl = (evhdl_t *)ev_tbl;
485
486 for (int i = 0; i < num; i++) {
487 evhdl_tbl[i].evgen = EVGEN_INIT;
488 ev_hdr_tbl[i]->event = evhdl_tbl[i].event;
489
490 /* Set initial counters for ext-events (atomic) */
491 __atomic_store_n(&ev_hdr_tbl[i]->state_cnt.u64,
492 init_cnt.u64, __ATOMIC_RELAXED);
493 /* Set initial state information (non-atomic) */
494 evhdr_update_state(ev_hdr_tbl[i], api_op);
495 }
496}
497
498static inline em_event_t
499esv_evinit_ext(const em_event_t event, event_hdr_t *const ev_hdr,
500 const uint16_t api_op)
501{
502 /*
503 * Combination of:
504 * event = esv_evinit(..., init_cnt_extev, ...)
505 * return evstate_em2usr(event, ...);
506 */
507 evhdl_t evhdl = {.event = event};
508 const evstate_cnt_t init = init_cnt_extev;
509 const evstate_cnt_t sub = {.evgen = 0, .rsvd = 0,
510 .ref_cnt = 0, .send_cnt = 1};
511 const evstate_cnt_t cnt = {.u64 = init.u64 - sub.u64};
512
513 evhdl.evgen = cnt.evgen;
514 ev_hdr->event = evhdl.event;
515
516 /* Set initial counters (atomic) */
517 __atomic_store_n(&ev_hdr->state_cnt.u64, cnt.u64,
518 __ATOMIC_RELAXED);
519
520 /* Set initial state information (non-atomic) */
521 evhdr_update_state(ev_hdr, api_op);
522
523 return evhdl.event;
524}
525
526static inline em_event_t
527esv_em2usr(const em_event_t event, event_hdr_t *const ev_hdr,
528 const evstate_cnt_t cnt, const uint16_t api_op, const bool is_revert)
529{
530 const bool refs_used = ev_hdr->flags.refs_used;
531 evhdl_t evhdl = {.event = event};
532 evstate_cnt_t new_cnt;
533
534 /* Update state-count and return value of all counters (atomic) */
535 if (unlikely(is_revert)) {
536 /* Revert previous em2usr counter update on failed operation */
537 new_cnt.u64 = __atomic_add_fetch(&ev_hdr->state_cnt.u64,
538 cnt.u64, __ATOMIC_RELAXED);
539 } else {
540 /* Normal em2usr counter update */
541 new_cnt.u64 = __atomic_sub_fetch(&ev_hdr->state_cnt.u64,
542 cnt.u64, __ATOMIC_RELAXED);
543 }
544
545 if (!refs_used) {
546 evhdl.evgen = new_cnt.evgen;
547 ev_hdr->event = evhdl.event;
548 }
549
550 const int16_t ref_cnt = REF_CNT_INIT - new_cnt.ref_cnt;
551 const int16_t send_cnt = new_cnt.send_cnt - SEND_CNT_INIT;
552
553 /*
554 * Check state count:
555 * OK: send_cnt < ref_cnt and both >=0. Error otherwise.
556 */
557 if (unlikely(send_cnt >= ref_cnt || send_cnt < 0)) {
558 /* report fatal event-state error, never return */
559 evstate_error(new_cnt, evhdl, ev_hdr, api_op, help_str_em2usr);
560 /* never reached */
561 }
562
563 /*
564 * Valid state transition, update state (non-atomic)
565 */
566 if (!refs_used)
567 evhdr_update_state(ev_hdr, api_op);
568
569 return evhdl.event;
570}
571
572static inline void
573esv_em2usr_multi(em_event_t ev_tbl[/*in/out*/],
574 event_hdr_t *const ev_hdr_tbl[], const int num,
575 const evstate_cnt_t cnt, const uint16_t api_op,
576 const bool is_revert)
577{
578 evhdl_t *const evhdl_tbl = (evhdl_t *)ev_tbl;
579 evstate_cnt_t new_cnt;
580
581 for (int i = 0; i < num; i++) {
582 if (unlikely(ev_hdr_tbl[i]->event_type == EM_EVENT_TYPE_ODP))
583 continue; /* skip ODP events */
584
585 const bool refs_used = ev_hdr_tbl[i]->flags.refs_used;
586
587 /* Update state-count and return value of all counters (atomic) */
588 if (unlikely(is_revert)) {
589 /* Revert em2usr counter update on failed operation */
590 new_cnt.u64 =
591 __atomic_add_fetch(&ev_hdr_tbl[i]->state_cnt.u64,
592 cnt.u64, __ATOMIC_RELAXED);
593 } else {
594 /* Normal em2usr counter update */
595 new_cnt.u64 =
596 __atomic_sub_fetch(&ev_hdr_tbl[i]->state_cnt.u64,
597 cnt.u64, __ATOMIC_RELAXED);
598 }
599
600 if (!refs_used) {
601 evhdl_tbl[i].evgen = new_cnt.evgen;
602 ev_hdr_tbl[i]->event = evhdl_tbl[i].event;
603 }
604
605 const int16_t ref_cnt = REF_CNT_INIT - new_cnt.ref_cnt;
606 const int16_t send_cnt = new_cnt.send_cnt - SEND_CNT_INIT;
607
608 /*
609 * Check state count:
610 * OK: send_cnt < ref_cnt and both >=0. Error otherwise.
611 */
612 if (unlikely(send_cnt >= ref_cnt || send_cnt < 0)) {
613 /* report fatal event-state error, never return */
614 evstate_error(new_cnt, evhdl_tbl[i], ev_hdr_tbl[i],
615 api_op, help_str_em2usr);
616 /* never reached */
617 }
618
619 /*
620 * Valid state transition, update state (non-atomic)
621 */
622 if (!refs_used)
623 evhdr_update_state(ev_hdr_tbl[i], api_op);
624 }
625}
626
627static inline void
628esv_usr2em(const em_event_t event, event_hdr_t *const ev_hdr,
629 const evstate_cnt_t cnt, const uint16_t api_op, const bool is_revert)
630{
631 const bool refs_used = ev_hdr->flags.refs_used;
632 evhdl_t evhdl = {.event = event};
633 evstate_cnt_t new_cnt;
634
635 /* Update state-count and return value of all counters (atomic) */
636 if (unlikely(is_revert)) {
637 /* Revert previous usr2em counter update on failed operation */
638 new_cnt.u64 = __atomic_sub_fetch(&ev_hdr->state_cnt.u64,
639 cnt.u64, __ATOMIC_RELAXED);
640
641 if (unlikely(new_cnt.evgen == EVGEN_INIT - 1)) {
642 /* Avoid .evgen counter wrap */
643 const evstate_cnt_t add = {.evgen = EVGEN_MAX - EVGEN_INIT,
644 .rsvd = 0, .ref_cnt = 0, .send_cnt = 0};
645 new_cnt.u64 = __atomic_add_fetch(&ev_hdr->state_cnt.u64,
646 add.u64, __ATOMIC_RELAXED);
647 }
648 } else {
649 /* Normal usr2em counter update */
650 new_cnt.u64 = __atomic_add_fetch(&ev_hdr->state_cnt.u64,
651 cnt.u64, __ATOMIC_RELAXED);
652
653 if (unlikely(new_cnt.evgen == EVGEN_MAX)) {
654 /* Avoid .evgen counter wrap */
655 const evstate_cnt_t sub = {.evgen = EVGEN_MAX - EVGEN_INIT,
656 .rsvd = 0, .ref_cnt = 0, .send_cnt = 0};
657 __atomic_fetch_sub(&ev_hdr->state_cnt.u64, sub.u64,
658 __ATOMIC_RELAXED);
659 }
660 /* cmp new_cnt.evgen vs evhdl.evgen of previous gen, thus -1 */
661 new_cnt.evgen -= 1;
662 }
663
664 const int16_t ref_cnt = REF_CNT_INIT - new_cnt.ref_cnt;
665 const int16_t send_cnt = new_cnt.send_cnt - SEND_CNT_INIT;
666
667 /*
668 * Check state count:
669 * OK: send_cnt <= ref_cnt and both >=0.
670 * AND
671 * OK: event handle evgen == evgen count (not checked for references)
672 * Error otherwise.
673 *
674 * Check evgen only for events that never had references.
675 * Reference usage mixes up the evgen since the same event can be
676 * sent and freed multiple times.
677 */
678 if (unlikely((send_cnt > ref_cnt || send_cnt < 0) ||
679 (!refs_used && evhdl.evgen != new_cnt.evgen))) {
680 const char *const help_str = refs_used ? help_str_usr2em_ref : help_str_usr2em;
681
682 /* report fatal event-state error, never return */
683 evstate_error(new_cnt, evhdl, ev_hdr, api_op, help_str);
684 /* never reached */
685 }
686
687 /*
688 * Valid state transition, update state (non-atomic)
689 */
690 if (!refs_used)
691 evhdr_update_state(ev_hdr, api_op);
692}
693
694static inline void
695esv_usr2em_multi(const em_event_t ev_tbl[],
696 event_hdr_t *const ev_hdr_tbl[], const int num,
697 const evstate_cnt_t cnt, const uint16_t api_op,
698 const bool is_revert)
699{
700 const evhdl_t *const evhdl_tbl = (const evhdl_t *)ev_tbl;
701 evstate_cnt_t new_cnt;
702
703 for (int i = 0; i < num; i++) {
704 if (unlikely(ev_hdr_tbl[i]->event_type == EM_EVENT_TYPE_ODP))
705 continue; /* skip ODP events */
706
707 const bool refs_used = ev_hdr_tbl[i]->flags.refs_used;
708
709 /* Update state-count and return value of all counters (atomic) */
710 if (unlikely(is_revert)) {
711 /* Revert usr2em counter update on failed operation */
712 new_cnt.u64 =
713 __atomic_sub_fetch(&ev_hdr_tbl[i]->state_cnt.u64,
714 cnt.u64, __ATOMIC_RELAXED);
715
716 if (unlikely(new_cnt.evgen == EVGEN_INIT - 1)) {
717 /* Avoid .evgen counter wrap */
718 const evstate_cnt_t add = {.evgen = EVGEN_MAX - EVGEN_INIT,
719 .rsvd = 0, .ref_cnt = 0, .send_cnt = 0};
720 new_cnt.u64 =
721 __atomic_add_fetch(&ev_hdr_tbl[i]->state_cnt.u64,
722 add.u64, __ATOMIC_RELAXED);
723 }
724 } else {
725 /* Normal usr2em counter update */
726 new_cnt.u64 =
727 __atomic_add_fetch(&ev_hdr_tbl[i]->state_cnt.u64,
728 cnt.u64, __ATOMIC_RELAXED);
729
730 if (unlikely(new_cnt.evgen == EVGEN_MAX)) {
731 /* Avoid .evgen counter wrap */
732 const evstate_cnt_t sub = {.evgen = EVGEN_MAX - EVGEN_INIT,
733 .rsvd = 0, .ref_cnt = 0, .send_cnt = 0};
734 __atomic_fetch_sub(&ev_hdr_tbl[i]->state_cnt.u64, sub.u64,
735 __ATOMIC_RELAXED);
736 }
737
738 new_cnt.evgen -= 1;
739 }
740
741 const int16_t ref_cnt = REF_CNT_INIT - new_cnt.ref_cnt;
742 const int16_t send_cnt = new_cnt.send_cnt - SEND_CNT_INIT;
743
744 /*
745 * Check state count:
746 * OK: send_cnt <= ref_cnt and both >=0.
747 * AND
748 * OK: event handle evgen == evgen count (not checked for references)
749 * Error otherwise.
750 *
751 * Check evgen only for events that never had references.
752 * Reference usage mixes up the evgen since the same event can be
753 * sent and freed multiple times.
754 */
755 if (unlikely((send_cnt > ref_cnt || send_cnt < 0) ||
756 (!refs_used && evhdl_tbl[i].evgen != new_cnt.evgen))) {
757 /* report fatal event-state error, never return */
758 evstate_error(new_cnt, evhdl_tbl[i], ev_hdr_tbl[i],
759 api_op, help_str_usr2em);
760 /* never reached */
761 }
762
763 /*
764 * Valid state transition, update state (non-atomic)
765 */
766 if (!refs_used)
767 evhdr_update_state(ev_hdr_tbl[i], api_op);
768 }
769}
770
771em_event_t evstate_prealloc(const em_event_t event, event_hdr_t *const ev_hdr)
772{
773 return esv_evinit(event, ev_hdr, init_cnt_alloc, EVSTATE__PREALLOC);
774}
775
776em_event_t evstate_alloc(const em_event_t event, event_hdr_t *const ev_hdr,
777 const uint16_t api_op)
778{
779 if (!em_shm->opt.esv.prealloc_pools || ev_hdr->flags.refs_used)
780 return esv_evinit(event, ev_hdr, init_cnt_alloc, api_op);
781
782 const evstate_cnt_t sub = {.evgen = 0, .rsvd = 0,
783 .ref_cnt = 1, .send_cnt = 0};
784
785 return esv_em2usr(event, ev_hdr, sub, api_op, false);
786}
787
788em_event_t evstate_alloc_tmo(const em_event_t event, event_hdr_t *const ev_hdr)
789{
790 return esv_evinit(event, ev_hdr, init_cnt_alloc, EVSTATE__TMO_CREATE);
791}
792
793void evstate_alloc_multi(em_event_t ev_tbl[/*in/out*/],
794 event_hdr_t *const ev_hdr_tbl[], const int num)
795{
796 if (!em_shm->opt.esv.prealloc_pools) {
797 esv_evinit_multi(ev_tbl/*in/out*/, ev_hdr_tbl, num,
798 init_cnt_alloc, EVSTATE__ALLOC_MULTI);
799 return;
800 }
801
802 /* em_shm->opt.esv.prealloc_pools: */
803 const evstate_cnt_t sub = {.evgen = 0, .rsvd = 0,
804 .ref_cnt = 1, .send_cnt = 0};
805
806 for (int i = 0; i < num; i++) {
807 if (ev_hdr_tbl[i]->flags.refs_used) {
808 ev_tbl[i] = esv_evinit(ev_tbl[i], ev_hdr_tbl[i],
809 init_cnt_alloc,
810 EVSTATE__ALLOC_MULTI);
811 } else {
812 ev_tbl[i] = esv_em2usr(ev_tbl[i], ev_hdr_tbl[i], sub,
813 EVSTATE__ALLOC_MULTI, false);
814 }
815 }
816}
817
818em_event_t evstate_ref(const em_event_t event, event_hdr_t *const ev_hdr)
819{
820 const evstate_cnt_t sub = {.evgen = 0, .rsvd = 0,
821 .ref_cnt = 1, .send_cnt = 0};
822
823 return esv_em2usr(event, ev_hdr, sub, EVSTATE__EVENT_REF, false);
824}
825
826em_event_t evstate_init(const em_event_t event, event_hdr_t *const ev_hdr,
827 bool is_extev)
828{
829 if (is_extev)
830 return esv_evinit_ext(event, ev_hdr, EVSTATE__INIT_EXTEV);
831 else
832 return esv_evinit(event, ev_hdr, init_cnt_alloc, EVSTATE__INIT);
833}
834
835void evstate_init_multi(em_event_t ev_tbl[/*in/out*/],
836 event_hdr_t *const ev_hdr_tbl[], const int num,
837 bool is_extev)
838{
839 uint16_t api_op;
840 evstate_cnt_t init_cnt;
841
842 if (is_extev) {
843 api_op = EVSTATE__INIT_EXTEV_MULTI;
844 init_cnt = init_cnt_extev;
845 } else {
846 api_op = EVSTATE__INIT_MULTI;
847 init_cnt = init_cnt_alloc;
848 }
849
850 esv_evinit_multi(ev_tbl/*in/out*/, ev_hdr_tbl, num,
851 init_cnt, api_op);
852}
853
854/**
855 * This is a combined calculation of the following three separate
856 * calculations:
857 *
858 * mark allocated:
859 * const evstate_cnt_t sub = {.evgen = 0, .rsvd = 0,
860 * .ref_cnt = 1, .send_cnt = 0};
861 * event = esv_em2usr(event, ev_hdr, sub, api_op, false);
862 *
863 * mark sent:
864 * const evstate_cnt_t add = {.evgen = 1, .rsvd = 0,
865 * .ref_cnt = 0, .send_cnt = 1};
866 * esv_usr2em(event, ev_hdr, add, api_op, false);
867 *
868 * mark em2usr for dispatch to user EO:
869 * const evstate_cnt_t sub2 = {.evgen = 0, .rsvd = 0,
870 * .ref_cnt = 0, .send_cnt = 1};
871 * event = esv_em2usr(event, ev_hdr, sub2, api_op, false);
872 *
873 * combined = add - sub - sub2
874 * add: {.evgen = 1, .rsvd = 0, .ref_cnt = 0, .send_cnt = 1}
875 * sub: - {.evgen = 0, .rsvd = 0, .ref_cnt = 1, .send_cnt = 0}
876 * sub2: - {.evgen = 0, .rsvd = 0, .ref_cnt = 0, .send_cnt = 1}
877 * -------------------------------------------------------
878 * cmb = {.evgen = 1, .rsvd = 0, .ref_cnt =-1, .send_cnt = 0}
879 */
880static inline em_event_t
881esv_update_ext(const em_event_t event, event_hdr_t *const ev_hdr,
882 const uint16_t api_op)
883{
884 const evstate_cnt_t sub = {.evgen = 0, .rsvd = 0,
885 .ref_cnt = 1, .send_cnt = 0};
886 const evstate_cnt_t add = {.evgen = 1, .rsvd = 0,
887 .ref_cnt = 0, .send_cnt = 0};
888 const evstate_cnt_t cmb = {.u64 = add.u64 - sub.u64}; /* combined, wraps */
889
890 const bool refs_used = ev_hdr->flags.refs_used;
891 evhdl_t evhdl = {.event = event};
892 evstate_cnt_t new_cnt;
893
894 /* Update state-count and return value of all counters (atomic) */
895 new_cnt.u64 = __atomic_add_fetch(&ev_hdr->state_cnt.u64,
896 cmb.u64, __ATOMIC_RELAXED);
897
898 if (unlikely(new_cnt.evgen == EVGEN_MAX)) {
899 /* Avoid .evgen counter wrap */
900 const evstate_cnt_t wrap = {.evgen = EVGEN_MAX - EVGEN_INIT,
901 .rsvd = 0, .ref_cnt = 0, .send_cnt = 0};
902 new_cnt.u64 = __atomic_sub_fetch(&ev_hdr->state_cnt.u64, wrap.u64,
903 __ATOMIC_RELAXED);
904 }
905
906 if (!refs_used) {
907 evhdl.evgen = new_cnt.evgen;
908 ev_hdr->event = evhdl.event;
909 }
910
911 const int16_t ref_cnt = REF_CNT_INIT - new_cnt.ref_cnt;
912 const int16_t send_cnt = new_cnt.send_cnt - SEND_CNT_INIT;
913
914 /*
915 * Check state count:
916 * OK: send_cnt < ref_cnt and both >=0. Error otherwise.
917 */
918 if (unlikely(send_cnt >= ref_cnt || send_cnt < 0)) {
919 /* report fatal event-state error, never return */
920 evstate_error(new_cnt, evhdl, ev_hdr, api_op, help_str_em2usr);
921 /* never reached */
922 }
923
924 /*
925 * Valid state transition, update state (non-atomic)
926 */
927 if (!refs_used)
928 evhdr_update_state(ev_hdr, api_op);
929
930 return evhdl.event;
931}
932
933em_event_t evstate_update(const em_event_t event, event_hdr_t *const ev_hdr,
934 bool is_extev)
935{
936 em_event_t ret_event;
937
938 if (is_extev) {
939 /* combined mark allocated & mark sent */
940 ret_event = esv_update_ext(event, ev_hdr, EVSTATE__UPDATE_EXTEV);
941 } else {
942 /* mark allocated */
943 const evstate_cnt_t sub = {.evgen = 0, .rsvd = 0,
944 .ref_cnt = 1, .send_cnt = 0};
945
946 ret_event = esv_em2usr(event, ev_hdr, sub, EVSTATE__UPDATE_EXTEV, false);
947 }
948
949 return ret_event;
950}
951
952void evstate_free(const em_event_t event, event_hdr_t *const ev_hdr,
953 const uint16_t api_op)
954{
955 const evstate_cnt_t add = {.evgen = 1, .rsvd = 0,
956 .ref_cnt = 1, .send_cnt = 0};
957
958 esv_usr2em(event, ev_hdr, add, api_op, false);
959}
960
961void evstate_free_revert(const em_event_t event, event_hdr_t *const ev_hdr,
962 const uint16_t api_op)
963{
964 const evstate_cnt_t sub = {.evgen = 1, .rsvd = 0,
965 .ref_cnt = 1, .send_cnt = 0};
966
967 esv_usr2em(event, ev_hdr, sub, api_op, true /*revert*/);
968}
969
970void evstate_free_multi(const em_event_t ev_tbl[],
971 event_hdr_t *const ev_hdr_tbl[], const int num,
972 const uint16_t api_op)
973{
974 const evstate_cnt_t add = {.evgen = 1, .rsvd = 0,
975 .ref_cnt = 1, .send_cnt = 0};
976
977 esv_usr2em_multi(ev_tbl, ev_hdr_tbl, num, add, api_op, false);
978}
979
980void evstate_free_revert_multi(const em_event_t ev_tbl[],
981 event_hdr_t *const ev_hdr_tbl[], const int num,
982 const uint16_t api_op)
983{
984 const evstate_cnt_t sub = {.evgen = 1, .rsvd = 0,
985 .ref_cnt = 1, .send_cnt = 0};
986
987 esv_usr2em_multi(ev_tbl, ev_hdr_tbl, num, sub, api_op, true /*revert*/);
988}
989
990em_event_t evstate_em2usr(const em_event_t event, event_hdr_t *const ev_hdr,
991 const uint16_t api_op)
992{
993 if (unlikely(ev_hdr->event_type == EM_EVENT_TYPE_ODP))
994 return event;
995
996 const evstate_cnt_t sub = {.evgen = 0, .rsvd = 0,
997 .ref_cnt = 0, .send_cnt = 1};
998
999 return esv_em2usr(event, ev_hdr, sub, api_op, false);
1000}
1001
1002em_event_t evstate_em2usr_revert(const em_event_t event, event_hdr_t *const ev_hdr,
1003 const uint16_t api_op)
1004{
1005 if (unlikely(ev_hdr->event_type == EM_EVENT_TYPE_ODP))
1006 return event;
1007
1008 const evstate_cnt_t add = {.evgen = 0, .rsvd = 0,
1009 .ref_cnt = 0, .send_cnt = 1};
1010
1011 return esv_em2usr(event, ev_hdr, add, api_op, true /*revert*/);
1012}
1013
1014void evstate_em2usr_multi(em_event_t ev_tbl[/*in/out*/],
1015 event_hdr_t *const ev_hdr_tbl[], const int num,
1016 const uint16_t api_op)
1017{
1018 const evstate_cnt_t sub = {.evgen = 0, .rsvd = 0,
1019 .ref_cnt = 0, .send_cnt = 1};
1020
1021 esv_em2usr_multi(ev_tbl/*in/out*/, ev_hdr_tbl, num, sub, api_op, false);
1022}
1023
1024void evstate_em2usr_revert_multi(em_event_t ev_tbl[/*in/out*/],
1025 event_hdr_t *const ev_hdr_tbl[], const int num,
1026 const uint16_t api_op)
1027{
1028 const evstate_cnt_t add = {.evgen = 0, .rsvd = 0,
1029 .ref_cnt = 0, .send_cnt = 1};
1030
1031 esv_em2usr_multi(ev_tbl/*in/out*/, ev_hdr_tbl, num, add, api_op, true /*revert*/);
1032}
1033
1034void evstate_usr2em(const em_event_t event, event_hdr_t *const ev_hdr,
1035 const uint16_t api_op)
1036{
1037 if (unlikely(ev_hdr->event_type == EM_EVENT_TYPE_ODP))
1038 return;
1039
1040 const evstate_cnt_t add = {.evgen = 1, .rsvd = 0,
1041 .ref_cnt = 0, .send_cnt = 1};
1042
1043 esv_usr2em(event, ev_hdr, add, api_op, false);
1044}
1045
1046void evstate_usr2em_revert(const em_event_t event, event_hdr_t *const ev_hdr,
1047 const uint16_t api_op)
1048{
1049 if (unlikely(ev_hdr->event_type == EM_EVENT_TYPE_ODP))
1050 return;
1051
1052 const evstate_cnt_t sub = {.evgen = 1, .rsvd = 0,
1053 .ref_cnt = 0, .send_cnt = 1};
1054
1055 esv_usr2em(event, ev_hdr, sub, api_op, true /*revert*/);
1056}
1057
1058void evstate_usr2em_multi(const em_event_t ev_tbl[],
1059 event_hdr_t *const ev_hdr_tbl[], const int num,
1060 const uint16_t api_op)
1061{
1062 const evstate_cnt_t add = {.evgen = 1, .rsvd = 0,
1063 .ref_cnt = 0, .send_cnt = 1};
1064
1065 esv_usr2em_multi(ev_tbl, ev_hdr_tbl, num, add, api_op, false);
1066}
1067
1068void evstate_usr2em_revert_multi(const em_event_t ev_tbl[],
1069 event_hdr_t *const ev_hdr_tbl[], const int num,
1070 const uint16_t api_op)
1071{
1072 const evstate_cnt_t sub = {.evgen = 1, .rsvd = 0,
1073 .ref_cnt = 0, .send_cnt = 1};
1074
1075 esv_usr2em_multi(ev_tbl, ev_hdr_tbl, num, sub, api_op, true /*revert*/);
1076}
1077
1078/*
1079 * Ensure that em_event_unmark_...() is only called after
1080 * em_event_mark_...() (not after normal em_send/free() etc).
1081 */
1082static inline void
1083check_valid_unmark(const event_hdr_t *ev_hdr, uint16_t api_op,
1084 const uint16_t expected_ops[], const int num_ops)
1085{
1086 /* event refs: can't rely on prev api_op */
1087 if (ev_hdr->flags.refs_used)
1088 return;
1089
1090 uint16_t prev_op = ev_hdr->state.api_op;
1091
1092 for (int i = 0; i < num_ops; i++) {
1093 if (prev_op == expected_ops[i])
1094 return; /* success */
1095 }
1096
1097 /* previous API was NOT em_event_mark_..., report FATAL error! */
1098 evstate_unmark_error(ev_hdr, api_op);
1099}
1100
1101static inline void
1102check_valid_unmark_multi(event_hdr_t *const ev_hdr_tbl[], const int num_evs,
1103 uint16_t api_op, const uint16_t expected_ops[], const int num_ops)
1104{
1105 uint16_t prev_op;
1106 bool is_valid;
1107
1108 for (int i = 0; i < num_evs; i++) {
1109 /* event refs: can't rely on prev api_op */
1110 if (ev_hdr_tbl[i]->flags.refs_used)
1111 continue;
1112
1113 prev_op = ev_hdr_tbl[i]->state.api_op;
1114 is_valid = false;
1115
1116 for (int j = 0; j < num_ops; j++) {
1117 if (prev_op == expected_ops[j]) {
1118 is_valid = true;
1119 break; /* success */
1120 }
1121 }
1122
1123 /* previous API was NOT em_event_mark_..., report FATAL error!*/
1124 if (unlikely(!is_valid))
1125 evstate_unmark_error(ev_hdr_tbl[i], api_op);
1126 }
1127}
1128
1129void evstate_unmark_send(const em_event_t event, event_hdr_t *const ev_hdr)
1130{
1131 if (em_shm->opt.esv.store_state) {
1132 uint16_t expected_prev_ops[1] = {EVSTATE__MARK_SEND};
1133 /*
1134 * Ensure that em_event_unmark_send() is only called after
1135 * em_event_mark_send/_multi() (not after em_send() etc).
1136 */
1137 check_valid_unmark(ev_hdr, EVSTATE__UNMARK_SEND,
1138 expected_prev_ops, 1);
1139 }
1140
1141 evstate_usr2em_revert(event, ev_hdr, EVSTATE__UNMARK_SEND);
1142}
1143
1144void evstate_unmark_free(const em_event_t event, event_hdr_t *const ev_hdr,
1145 const uint16_t api_op)
1146{
1147 if (em_shm->opt.esv.store_state) {
1148 uint16_t expected_prev_ops[2] = {EVSTATE__MARK_FREE,
1149 EVSTATE__MARK_FREE_MULTI};
1150 /*
1151 * Ensure that em_event_unmark_free() is only called
1152 * after em_event_mark_free() (not after em_free() etc).
1153 */
1154 check_valid_unmark(ev_hdr, api_op, expected_prev_ops, 2);
1155 }
1156
1157 evstate_free_revert(event, ev_hdr, api_op);
1158}
1159
1160void evstate_unmark_free_multi(const em_event_t ev_tbl[],
1161 event_hdr_t *const ev_hdr_tbl[], const int num,
1162 const uint16_t api_op)
1163{
1164 if (em_shm->opt.esv.store_state) {
1165 uint16_t expected_prev_ops[2] = {EVSTATE__MARK_FREE_MULTI,
1166 EVSTATE__MARK_FREE};
1167 /*
1168 * Ensure that em_event_unmark_free_multi() is only
1169 * called after em_event_mark_free_multi()
1170 * (not after em_free/_multi() etc).
1171 */
1172 check_valid_unmark_multi(ev_hdr_tbl, num, api_op,
1173 expected_prev_ops, 2);
1174 }
1175
1176 evstate_free_revert_multi(ev_tbl, ev_hdr_tbl, num, api_op);
1177}
1178
1179static int read_config_file(void)
1180{
1181 const char *conf_str;
1182 bool val_bool = false;
1183 int ret;
1184
1185 EM_PRINT("EM ESV config:\n");
1186
1187 if (unlikely(!EM_ESV_ENABLE)) {
1188 EM_PRINT(" ESV is DISABLED (build time)\n");
1189 /* ESV disabled at compile time, skip reading config */
1190 memset(&em_shm->opt.esv, 0, sizeof(em_shm->opt.esv));
1191 return 0;
1192 }
1193
1194 /*
1195 * Option: esv.enable - runtime enable/disable
1196 */
1197 conf_str = "esv.enable";
1198 ret = em_libconfig_lookup_bool(&em_shm->libconfig, conf_str, &val_bool);
1199 if (unlikely(!ret)) {
1200 EM_LOG(EM_LOG_ERR, "Config option '%s' not found\n", conf_str);
1201 return -1;
1202 }
1203
1204 /* store & print the value */
1205 em_shm->opt.esv.enable = (int)val_bool;
1206 EM_PRINT(" %s: %s(%d)\n", conf_str, val_bool ? "true" : "false", val_bool);
1207
1208 if (!em_shm->opt.esv.enable) {
1209 /* Read no more options if ESV is disabled */
1210 memset(&em_shm->opt.esv, 0, sizeof(em_shm->opt.esv));
1211 return 0;
1212 }
1213
1214 /*
1215 * Option: esv.store_state
1216 */
1217 conf_str = "esv.store_state";
1218 ret = em_libconfig_lookup_bool(&em_shm->libconfig, conf_str, &val_bool);
1219 if (unlikely(!ret)) {
1220 EM_LOG(EM_LOG_ERR, "Config option '%s' not found\n", conf_str);
1221 return -1;
1222 }
1223 /* store & print the value */
1224 em_shm->opt.esv.store_state = (int)val_bool;
1225 EM_PRINT(" %s: %s(%d)\n", conf_str, val_bool ? "true" : "false",
1226 val_bool);
1227
1228 /*
1229 * Option: esv.store_payload_first_u32
1230 */
1231 conf_str = "esv.store_payload_first_u32";
1232 ret = em_libconfig_lookup_bool(&em_shm->libconfig, conf_str, &val_bool);
1233 if (unlikely(!ret)) {
1234 EM_LOG(EM_LOG_ERR, "Config option '%s' not found\n", conf_str);
1235 return -1;
1236 }
1237 /* store & print the value */
1238 em_shm->opt.esv.store_first_u32 = (int)val_bool;
1239 EM_PRINT(" %s: %s(%d)\n", conf_str, val_bool ? "true" : "false",
1240 val_bool);
1241
1242 /*
1243 * Option: esv.prealloc_pools
1244 */
1245 conf_str = "esv.prealloc_pools";
1246 ret = em_libconfig_lookup_bool(&em_shm->libconfig, conf_str, &val_bool);
1247 if (unlikely(!ret)) {
1248 EM_LOG(EM_LOG_ERR, "Config option '%s' not found\n", conf_str);
1249 return -1;
1250 }
1251 /* store & print the value */
1252 em_shm->opt.esv.prealloc_pools = (int)val_bool;
1253 EM_PRINT(" %s: %s(%d)\n", conf_str, val_bool ? "true" : "false",
1254 val_bool);
1255
1256 return 0;
1257}
1258
1259em_status_t esv_init(void)
1260{
1261 if (read_config_file())
1262 return EM_ERR_LIB_FAILED;
1263
1264 return EM_OK;
1265}
em_eo.h
em_error.h
INTERNAL_ERROR
#define INTERNAL_ERROR(error, escope, fmt,...)
Definition em_error.h:58
em_event.h
em_event_inline.h
em_event_state.h
SEND_CNT_INIT
#define SEND_CNT_INIT
Definition em_event_state.h:124
EVGEN_INIT
#define EVGEN_INIT
Definition em_event_state.h:120
EVGEN_MAX
#define EVGEN_MAX
Definition em_event_state.h:122
REF_CNT_INIT
#define REF_CNT_INIT
Definition em_event_state.h:126
em_event_types.h
em_mem.h
em_locm
ENV_LOCAL em_locm_t em_locm
em_shm
em_shm_t * em_shm
Definition event_machine_init.c:131
em_pool.h
em_pool_types.h
em_queue.h
em_queue_inline.h
em_queue_types.h
event_machine.h
EM_ESV_ENABLE
#define EM_ESV_ENABLE
Definition event_machine_config.h:456
EM_EO_NAME_LEN
#define EM_EO_NAME_LEN
Definition event_machine_config.h:133
EM_QUEUE_NAME_LEN
#define EM_QUEUE_NAME_LEN
Definition event_machine_config.h:154
EM_EVENT_UNDEF
#define EM_EVENT_UNDEF
Definition event_machine_types.h:110
EM_QUEUE_UNDEF
#define EM_QUEUE_UNDEF
Definition event_machine_types.h:148
EM_EO_UNDEF
#define EM_EO_UNDEF
Definition event_machine_types.h:136
EM_OK
#define EM_OK
Definition event_machine_error.h:104
em_escope_t
uint32_t em_escope_t
Definition event_machine_error.h:184
EM_FATAL
#define EM_FATAL(error)
Definition event_machine_error.h:83
EM_ESCOPE_INTERNAL_MASK
#define EM_ESCOPE_INTERNAL_MASK
Definition event_machine_error.h:331
EM_ESCOPE_ODP_EXT
#define EM_ESCOPE_ODP_EXT
Definition event_machine_error.h:724
em_status_t
uint32_t em_status_t
Definition event_machine_error.h:96
EM_ERR_EVENT_STATE
@ EM_ERR_EVENT_STATE
Definition event_machine_error.h:157
EM_ERR_LIB_FAILED
@ EM_ERR_LIB_FAILED
Definition event_machine_error.h:149
EM_EVENT_TYPE_ODP
@ EM_EVENT_TYPE_ODP
Definition event_machine_event.h:218
ODP_PACKED
Definition em_event_types.h:139
ODP_PACKED::q_elem
queue_elem_t * q_elem
Definition em_mem.h:214
ODP_PACKED::api_op
uint8_t api_op
Definition em_event_types.h:164
ODP_PACKED::eo_idx
int16_t eo_idx
Definition em_event_types.h:151
ODP_PACKED::core
uint8_t core
Definition em_event_types.h:166
ODP_PACKED::payload_first
uint32_t payload_first
Definition em_event_types.h:144
ODP_PACKED::queue_idx
int16_t queue_idx
Definition em_event_types.h:158
em_locm_t
Definition em_mem.h:226
em_locm_t::current
em_locm_current_t current
Definition em_mem.h:228
em_locm_t::core_id
int core_id
Definition em_mem.h:239
em_shm_t::opt
em_cfgfile_opts_t opt
Definition em_mem.h:99
eo_elem_t
Definition em_eo_types.h:55
event_hdr
Definition em_event_types.h:194
event_hdr::refs_used
uint8_t refs_used
Definition em_event_types.h:230
event_hdr::state
ev_hdr_state_t state
Definition em_event_types.h:217
event_hdr::event
em_event_t event
Definition em_event_types.h:268
event_hdr::state_cnt
evstate_cnt_t state_cnt
Definition em_event_types.h:205
event_hdr::flags
union event_hdr::@43 flags
event_hdr::event_type
em_event_type_t event_type
Definition em_event_types.h:263
evstate_info_t
Definition em_event_state.c:85
queue_elem_t
Definition em_queue_types.h:236
queue_elem_t::queue
uint32_t queue
Definition em_queue_types.h:268
queue_elem_t::eo
uint16_t eo
Definition em_queue_types.h:265
evhdl_t
Definition em_event_types.h:77