em_queue_types.h

Go to the documentation of this file.

/*
 *   Copyright (c) 2015-2026, Nokia Solutions and Networks
 *   All rights reserved.
 *
 *   Redistribution and use in source and binary forms, with or without
 *   modification, are permitted provided that the following conditions
 *   are met:
 *
 *     * Redistributions of source code must retain the above copyright
 *       notice, this list of conditions and the following disclaimer.
 *     * Redistributions in binary form must reproduce the above copyright
 *       notice, this list of conditions and the following disclaimer in the
 *       documentation and/or other materials provided with the distribution.
 *     * Neither the name of the copyright holder nor the names of its
 *       contributors may be used to endorse or promote products derived
 *       from this software without specific prior written permission.
 *
 *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */
 
 /**
  * @file
  * EM internal queue types & definitions
  *
  */
 
#ifndef EM_QUEUE_TYPES_H_
#define EM_QUEUE_TYPES_H_
 
#include <stdbool.h>
#include <stdint.h>
 
#include <odp_api.h>
 
#include <event_machine.h>
#include <event_machine/platform/env/environment.h>
 
#include "misc/list.h"
#include "misc/objpool.h"
 
#ifdef __cplusplus
extern "C" {
#endif
 
/*
 * Forward declaration — queue_elem_t holds an eo_elem_t* (pointer only).
 * Using a forward declaration instead of #include "em_eo_types.h" keeps
 * the *_types.h layer free of inter-includes, preventing circular
 * dependencies if a reverse pointer is ever added.
 */
struct eo_elem_t;
typedef struct eo_elem_t eo_elem_t;
 
/*
 * EM internal queue ids - local part of the queue only, i.e missing the
 * device-id.
 * Note that the EM queue handle range is determined by 'EM_QUEUE_RANGE_OFFSET'
 */
#define MAX_INTERNAL_QUEUES (EM_MAX_CORES + EM_MAX_EXTTHRS + 1/*shared internal queue*/)
 
COMPILE_TIME_ASSERT(MAX_INTERNAL_QUEUES - 1 >= EM_MAX_CORES, TOO_FEW_INTERNAL_QUEUES_ERROR);
 
/* Priority for the EM-internal queues */
#define INTERNAL_QUEUE_PRIORITY (EM_QUEUE_PRIO_HIGHEST)
 
#define QUEUE_ELEM_VALID ((uint16_t)0xCAFE)
 
#define INVALID_QUEUE_ID ((uint16_t)0xFFFF)
 
/* Verify that the byte order is defined for 'internal_queue_t' */
#if \
(__BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__) && \
(__BYTE_ORDER__ != __ORDER_BIG_ENDIAN__)
#error __BYTE_ORDER__ not defined!
#endif
 
/**
 * Internal representation of the EM queue handle
 * The EM queue handle contains a 16-bit queue-id and a 16-bit device-id.
 */
typedef union {
        em_queue_t queue;
        struct {
#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
                uint16_t queue_id;
                uint16_t device_id;
#ifdef EM_64_BIT
                uint32_t unused;
#endif
#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
#ifdef EM_64_BIT
                uint32_t unused;
#endif
                uint16_t device_id;
                uint16_t queue_id;
#endif
        };
} internal_queue_t;
 
/* Verify size of struct, i.e. accept no padding */
COMPILE_TIME_ASSERT(sizeof(internal_queue_t) == sizeof(em_queue_t),
                    INTERNAL_QUEUE_T_SIZE_ERROR);
 
/**
 * Queue state
 */
typedef enum queue_state {
        /** Invalid queue state, queue not created/allocated */
        EM_QUEUE_STATE_INVALID = 0,
 
        /*
         * Scheduled queue (ATOMIC, PARALLEL, ORDERED) states:
         * (keep state values consecutive: ...n-1,n,n+1...)
         */
        /** Queue initialization, allocated and being set up */
        EM_QUEUE_STATE_INIT = 1,
        /** Queue added/bound to an EO, but EO-start not yet complete */
        EM_QUEUE_STATE_BIND = 2,
        /** Queue ready, related EO started  */
        EM_QUEUE_STATE_READY = 3,
 
        /*
         * Non-scheduled queue (UNSCHED, OUTPUT) state use the UNSCHEDULED-state.
         */
        /* Use separate value for unscheduled queues to catch illegal usage */
        EM_QUEUE_STATE_UNSCHEDULED = 255
} queue_state_e;
/**
 * Queue state, packed into uint8_t.
 *
 * The 'enum queue_state' or queue_state_e will always fit into an uint8_t.
 * Save space in the queue_elem_t by using this instead.
 */
typedef uint8_t queue_state_t;
 
/**
 * Atomic-group queue specific part of the queue element
 */
typedef struct q_elem_atomic_group_ {
        /** The atomic group handle (if any) of this queue */
        em_atomic_group_t atomic_group;
        /** List node for linking queue elems belonging to an atomic group */
        list_node_t agrp_node;
} q_elem_atomic_group_t;
 
/**
 * Output queue specific part of the queue element
 */
typedef struct q_elem_output_ {
        /** Output Queue config, incl. output_fn(..., output_fn_args) */
        em_output_queue_conf_t output_conf;
        /** Copied output_fn_args content of length 'args_len' stored in event */
        em_event_t output_fn_args_event;
        /** Output queue index used for output queue tracking, not same as queue index */
        uint32_t idx;
        /** Lock for output queues during an ordered-context */
        odp_ticketlock_t lock;
} q_elem_output_t;
 
/**
 * Parent queue aggregator data.
 *
 * Stored in the parent (scheduled or unscheduled) queue element when the
 * queue was created with child aggregator queues (em_queue_param_t::num_aggr).
 */
typedef struct q_elem_aggr_parent_ {
        /** Number of aggregator child queues */
        uint32_t num_aggr;
        /** Array of aggregator child queue handles (as uint32_t's) */
        uint32_t aggr_queues[EM_QUEUE_MAX_AGGR];
} q_elem_aggr_parent_t;
 
/**
 * Child aggregator queue data.
 *
 * Stored in aggregator (EM_QUEUE_TYPE_AGGR) queue elements.
 * Aggregator queues are never added to an EO or queue group, so this
 * data shares space (union) with q_elem_atomic_group_t and q_elem_output_t.
 */
typedef struct q_elem_aggr_ {
        /** Back-reference to the parent queue handle */
        em_queue_t parent_queue;
 
        /** Event vector pool (from em_queue_aggr_conf_t) */
        em_pool_t pool;
 
        /** Maximum time to wait to fill a vector (from em_queue_aggr_conf_t) */
        uint64_t max_tmo_ns;
 
        /** Maximum number of events in vector (from em_queue_aggr_conf_t) */
        uint32_t max_size;
 
        /** Aggregation and vector event type (from em_queue_aggr_conf_t) */
        em_event_type_t event_type;
} q_elem_aggr_t;
 
/**
 * Internal EM queue flags.
 *
 * Contains per-queue boolean flags packed into a single byte, which describe
 * aspects of the queue's nature and use. Intended for internal bookkeeping
 * within EM.
 */
typedef union queue_elem_flags_ {
        /** Access all flags as an 8-bit unsigned integer */
        uint8_t all;
        struct {
                /** true(1):receive_multi_func(), false(0):receive_func() */
                uint8_t use_multi_rcv   : 1;
                /** Set if queue is scheduled, i.e. atomic, parallel or ordered */
                uint8_t scheduled       : 1;
                /** Does this queue belong to an EM Atomic Group (true/false)? */
                uint8_t in_atomic_group : 1;
                /** Is this an ODP pktin event queue (true/false)? */
                uint8_t is_pktin        : 1;
                /** Does this queue have aggregator child queues (true/false)? */
                uint8_t has_aggr        : 1;
                /** Reserved bits */
                uint8_t rsvd            : 3;
        };
} queue_elem_flags_t;
 
/**
 * EM queue element
 */
typedef struct queue_elem_t {
        /**
         * Check that contents is an EM queue elem.
         *
         * EM will verify that the ODP queue context actually points to an
         * EM queue elem and not to something else:
         *     queue_elem_t *q_elem = odp_queue_context(odp_queue);
         *     if (!q_elem || q_elem->valid_check != QUEUE_ELEM_VALID)
         *             EM_ERROR(...);
         * Keep first.
         */
        uint16_t valid_check;
 
        /** Queue flags */
        queue_elem_flags_t flags;
 
        /** Queue state */
        queue_state_t state; /* queue_state_e */
 
        /** Queue priority */
        uint8_t priority; /* em_queue_prio_t */
 
        /** Atomic, parallel, ordered, unscheduled, local, output, aggregator */
        uint8_t type; /* em_queue_type_t */
 
        /** Max number of events passed to the EO's multi-event receive function */
        uint16_t max_events; /* only used if flags.use_multi_rcv == true */
 
        /** EM EO that this queue belongs to */
        uint16_t eo; /* em_eo_t */
 
        /** Queue handle */
        uint32_t queue; /* em_queue_t */
 
        /** Associated ODP queue handle */
        odp_queue_t odp_queue;
 
        /** User defined queue context (can be NULL) */
        void *context;
 
        union {
                /** Copy of the event receive function for better performance */
                em_receive_func_t receive_func;
                /** Copy of the multi-event receive function for better performance */
                em_receive_multi_func_t receive_multi_func;
        };
 
        /** Copy of the user defined eo context (or NULL) for performance */
        void *eo_ctx;
 
        /**
         * Atomic group specific data.
         * Only used if the queue belongs to an atomic group. Cannot be placed
         * in the union below (with the other q_elem_*_t's) since a queue might
         * both belong to an atomic group and have aggregator child queues (i.e.
         * .agrp and .aggr_parent might be simultaneously in use).
         */
        q_elem_atomic_group_t agrp;
 
        /**
         * Type-specific queue data (union).
         *
         * Only one variant is active, determined by the queue type:
         * - aggr_parent: scheduled or unscheduled queue with child aggregator queues,
         *                only used when aggregators have been configured and set up.
         * - aggr:        EM_QUEUE_TYPE_AGGR (aggregator child queue)
         * - output:      EM_QUEUE_TYPE_OUTPUT
         */
        union {
                q_elem_aggr_parent_t aggr_parent;
                q_elem_aggr_t aggr;
                q_elem_output_t output;
        };
 
        /** Associated eo element */
        eo_elem_t *eo_elem;
 
        /** Queue group handle of this queue */
        em_queue_group_t queue_group;
 
        /** List node for linking queue elems belonging to an EO */
        list_node_t eo_queue_node;
        /** List node for linking queue elems belonging to a queue group */
        list_node_t qgrp_node;
 
        /** Queue pool elem for linking free queues for queue_alloc() */
        objpool_elem_t queue_pool_elem;
 
        /**
         * End of queue element data,
         * for offsetof(queue_elem_t, end_data)
         */
        uint8_t end_data[0];
 
        /*
         * ! EMPTY SPACE !
         */
 
        /** Guarantee that size is a multiple of cache line size */
        void *end[0] ENV_CACHE_LINE_ALIGNED;
} queue_elem_t ENV_CACHE_LINE_ALIGNED;
 
COMPILE_TIME_ASSERT(sizeof(queue_elem_t) % ENV_CACHE_LINE_SIZE == 0,
                    QUEUE_ELEM_T__SIZE_ERROR);
/* Catch unintentional size increase */
COMPILE_TIME_ASSERT(sizeof(queue_elem_t) <= 3 * ENV_CACHE_LINE_SIZE,
                    QUEUE_ELEM_T__SIZE_TOO_LARGE);
 
/**
 * EM queue element table
 */
typedef struct queue_tbl_t {
        /** ODP queue capabilities common for all queues */
        odp_queue_capability_t odp_queue_capability;
        /** ODP schedule capabilities related to queues */
        odp_schedule_capability_t odp_schedule_capability;
        /** Current number of allocated output queues */
        odp_atomic_u32_t output_queue_count ENV_CACHE_LINE_ALIGNED;
        /** Free output queue indexes */
        bool output_queue_idx_free[EM_MAX_OUTPUT_QUEUES];
        /** Lock for output queue (de-)allocations */
        odp_ticketlock_t output_queue_lock;
 
        /** First static queue ID */
        uint16_t first_static_queue_id;
        /** Last static queue ID */
        uint16_t last_static_queue_id;
 
        /** First internal queue ID */
        uint16_t first_internal_queue_id;
        /** Last internal queue ID */
        uint16_t last_internal_queue_id;
        /** Shared internal queue ID */
        uint16_t shared_internal_queue_id;
 
        /** First dynamic queue ID */
        uint16_t first_dyn_queue_id;
        /** Last dynamic queue ID */
        uint16_t last_dyn_queue_id;
 
        /** First aggregator queue ID */
        uint16_t first_aggr_queue_id;
        /** Last aggregator queue ID */
        uint16_t last_aggr_queue_id;
 
        /** Maximum number of EM queues */
        uint16_t max_queue_num;
 
        /**
         * Dynamically allocated memory starts here. The elements
         * are in a single continuous memory block. The amount of memory
         * reserved depends on the max number of EM queues given in
         * EM config file.
         */
        /** Queue element table, format: queue_elem[em_queue_max_num()] */
        queue_elem_t *queue_elem ENV_CACHE_LINE_ALIGNED;
        /** Queue name table, format: name[em_queue_max_num()][EM_QUEUE_NAME_LEN] */
        char (*name)[EM_QUEUE_NAME_LEN];
} queue_tbl_t;
 
/**
 * Pool of free queues
 */
typedef struct queue_pool_t {
        objpool_t objpool;
} queue_pool_t;
 
/**
 * Local queues, i.e. core-local storage for events to local queues
 */
typedef struct local_queues_t {
        int empty;
        struct {
                int empty_prio;
                odp_stash_t stash;
        } prio[EM_QUEUE_PRIO_NUM];
} local_queues_t;
 
/**
 * Track output-queues used during a dispatch round (burst)
 */
typedef struct output_queue_track_t {
        unsigned int idx_cnt;
        uint16_t idx[EM_MAX_OUTPUT_QUEUES];
        queue_elem_t *used_queues[EM_MAX_OUTPUT_QUEUES];
} output_queue_track_t;
 
#ifdef __cplusplus
}
#endif
 
#endif /* EM_QUEUE_TYPES_H_ */