From: Brian Brooks <brian.bro...@arm.com> Signed-off-by: Brian Brooks <brian.bro...@arm.com> Reviewed-by: Ola Liljedahl <ola.liljed...@arm.com> --- /** Email created from pull request 166 (brbrooks:cloud-dev) ** https://github.com/Linaro/odp/pull/166 ** Patch: https://github.com/Linaro/odp/pull/166.patch ** Base sha: 344fe21a06ef4273f34c4441edd0caef876517d9 ** Merge commit sha: 45584e110dae85e9ed20736b254a7fdd044e6a76 **/ .../linux-dpdk/include/odp_packet_io_internal.h | 3 + .../linux-generic/include/odp_packet_io_internal.h | 3 + .../linux-generic/include/odp_queue_internal.h | 5 ++ platform/linux-generic/include/odp_schedule_if.h | 65 ++++++---------------- platform/linux-generic/odp_packet_io.c | 4 +- platform/linux-generic/odp_queue.c | 8 +-- platform/linux-generic/schedule/generic.c | 22 +++----- platform/linux-generic/schedule/iquery.c | 20 +++---- platform/linux-generic/schedule/scalable.c | 10 ++-- platform/linux-generic/schedule/sp.c | 18 +++--- 10 files changed, 67 insertions(+), 91 deletions(-)
diff --git a/platform/linux-dpdk/include/odp_packet_io_internal.h b/platform/linux-dpdk/include/odp_packet_io_internal.h index 8e158dc9..14b6a8d3 100644 --- a/platform/linux-dpdk/include/odp_packet_io_internal.h +++ b/platform/linux-dpdk/include/odp_packet_io_internal.h @@ -207,6 +207,9 @@ extern const pktio_if_ops_t loopback_pktio_ops; extern const pktio_if_ops_t dpdk_pktio_ops; extern const pktio_if_ops_t * const pktio_if_ops[]; +int pktin_poll(int pktio_index, int num_queue, int index[]); +void pktio_stop_finalize(int pktio_index); + #ifdef __cplusplus } #endif diff --git a/platform/linux-generic/include/odp_packet_io_internal.h b/platform/linux-generic/include/odp_packet_io_internal.h index 1a4e345f..dc049f46 100644 --- a/platform/linux-generic/include/odp_packet_io_internal.h +++ b/platform/linux-generic/include/odp_packet_io_internal.h @@ -271,6 +271,9 @@ int sock_stats_fd(pktio_entry_t *pktio_entry, int fd); int sock_stats_reset_fd(pktio_entry_t *pktio_entry, int fd); +int pktin_poll(int pktio_index, int num_queue, int index[]); +void pktio_stop_finalize(int pktio_index); + #ifdef __cplusplus } #endif diff --git a/platform/linux-generic/include/odp_queue_internal.h b/platform/linux-generic/include/odp_queue_internal.h index dd846d59..a1b2a73d 100644 --- a/platform/linux-generic/include/odp_queue_internal.h +++ b/platform/linux-generic/include/odp_queue_internal.h @@ -76,6 +76,11 @@ static inline queue_t qentry_to_int(queue_entry_t *qentry) return (queue_t)(qentry); } +odp_queue_t queue_handle(uint32_t queue_index); +void queue_destroy_finalize(uint32_t queue_index); +int queue_idx_deq_multi(uint32_t queue_index, odp_event_t ev[], int num); +int queue_empty(uint32_t queue_index); + #ifdef __cplusplus } #endif diff --git a/platform/linux-generic/include/odp_schedule_if.h b/platform/linux-generic/include/odp_schedule_if.h index 2bcf23f2..8f39eec1 100644 --- a/platform/linux-generic/include/odp_schedule_if.h +++ b/platform/linux-generic/include/odp_schedule_if.h @@ -7,64 +7,31 @@ #ifndef ODP_SCHEDULE_IF_H_ #define ODP_SCHEDULE_IF_H_ -#ifdef __cplusplus -extern "C" { -#endif - #include <odp/api/queue.h> #include <odp_queue_if.h> #include <odp/api/schedule.h> -typedef void (*schedule_pktio_start_fn_t)(int pktio_index, int num_in_queue, - int in_queue_idx[]); -typedef int (*schedule_thr_add_fn_t)(odp_schedule_group_t group, int thr); -typedef int (*schedule_thr_rem_fn_t)(odp_schedule_group_t group, int thr); -typedef int (*schedule_num_grps_fn_t)(void); -typedef int (*schedule_init_queue_fn_t)( - uint32_t queue_index, const odp_schedule_param_t *sched_param); -typedef void (*schedule_destroy_queue_fn_t)(uint32_t queue_index); -typedef int (*schedule_sched_queue_fn_t)(uint32_t queue_index); -typedef int (*schedule_unsched_queue_fn_t)(uint32_t queue_index); -typedef int (*schedule_ord_enq_multi_fn_t)(queue_t q_int, - void *buf_hdr[], int num, int *ret); -typedef void (*schedule_order_lock_fn_t)(void); -typedef void (*schedule_order_unlock_fn_t)(void); -typedef unsigned (*schedule_max_ordered_locks_fn_t)(void); -typedef void (*schedule_save_context_fn_t)(uint32_t queue_index); - typedef struct schedule_fn_t { - int status_sync; - schedule_pktio_start_fn_t pktio_start; - schedule_thr_add_fn_t thr_add; - schedule_thr_rem_fn_t thr_rem; - schedule_num_grps_fn_t num_grps; - schedule_init_queue_fn_t init_queue; - schedule_destroy_queue_fn_t destroy_queue; - schedule_sched_queue_fn_t sched_queue; - schedule_ord_enq_multi_fn_t ord_enq_multi; - schedule_order_lock_fn_t order_lock; - schedule_order_unlock_fn_t order_unlock; - schedule_max_ordered_locks_fn_t max_ordered_locks; + int status_sync; + void (*pktio_start)(int pktio_index, int num_in_queue, + int in_queue_idx[]); + int (*thr_add)(odp_schedule_group_t group, int thr); + int (*thr_rem)(odp_schedule_group_t group, int thr); + int (*num_grps)(void); + int (*init_queue)(uint32_t queue_index, + const odp_schedule_param_t *sched_param); + void (*destroy_queue)(uint32_t queue_index); + int (*sched_queue)(uint32_t queue_index); + int (*ord_enq_multi)(queue_t q_int, void *buf_hdr[], int num, int *ret); + void (*order_lock)(void); + void (*order_unlock)(void); + unsigned (*max_ordered_locks)(void); /* Called only when status_sync is set */ - schedule_unsched_queue_fn_t unsched_queue; - schedule_save_context_fn_t save_context; - + int (*unsched_queue)(uint32_t queue_index); + void (*save_context)(uint32_t queue_index); } schedule_fn_t; -/* Interface towards the scheduler */ extern const schedule_fn_t *sched_fn; -/* Interface for the scheduler */ -int sched_cb_pktin_poll(int pktio_index, int num_queue, int index[]); -void sched_cb_pktio_stop_finalize(int pktio_index); -odp_queue_t sched_cb_queue_handle(uint32_t queue_index); -void sched_cb_queue_destroy_finalize(uint32_t queue_index); -int sched_cb_queue_deq_multi(uint32_t queue_index, odp_event_t ev[], int num); -int sched_cb_queue_empty(uint32_t queue_index); - -#ifdef __cplusplus -} -#endif - #endif diff --git a/platform/linux-generic/odp_packet_io.c b/platform/linux-generic/odp_packet_io.c index 066f89dc..e78c712e 100644 --- a/platform/linux-generic/odp_packet_io.c +++ b/platform/linux-generic/odp_packet_io.c @@ -665,7 +665,7 @@ static int pktin_deq_multi(queue_t q_int, odp_buffer_hdr_t *buf_hdr[], int num) return nbr; } -int sched_cb_pktin_poll(int pktio_index, int num_queue, int index[]) +int pktin_poll(int pktio_index, int num_queue, int index[]) { odp_buffer_hdr_t *hdr_tbl[QUEUE_MULTI_MAX]; int num, idx; @@ -703,7 +703,7 @@ int sched_cb_pktin_poll(int pktio_index, int num_queue, int index[]) return 0; } -void sched_cb_pktio_stop_finalize(int pktio_index) +void pktio_stop_finalize(int pktio_index) { int state; pktio_entry_t *entry = pktio_entry_by_index(pktio_index); diff --git a/platform/linux-generic/odp_queue.c b/platform/linux-generic/odp_queue.c index 7ad346ad..46197e00 100644 --- a/platform/linux-generic/odp_queue.c +++ b/platform/linux-generic/odp_queue.c @@ -234,7 +234,7 @@ static odp_queue_t queue_create(const char *name, return handle; } -void sched_cb_queue_destroy_finalize(uint32_t queue_index) +void queue_destroy_finalize(uint32_t queue_index) { queue_entry_t *queue = get_qentry(queue_index); @@ -643,19 +643,19 @@ static int queue_info(odp_queue_t handle, odp_queue_info_t *info) return 0; } -odp_queue_t sched_cb_queue_handle(uint32_t queue_index) +odp_queue_t queue_handle(uint32_t queue_index) { return queue_from_id(queue_index); } -int sched_cb_queue_deq_multi(uint32_t queue_index, odp_event_t ev[], int num) +int queue_idx_deq_multi(uint32_t queue_index, odp_event_t ev[], int num) { queue_entry_t *qe = get_qentry(queue_index); return deq_multi(qe, (odp_buffer_hdr_t **)ev, num); } -int sched_cb_queue_empty(uint32_t queue_index) +int queue_empty(uint32_t queue_index) { queue_entry_t *queue = get_qentry(queue_index); int ret = 0; diff --git a/platform/linux-generic/schedule/generic.c b/platform/linux-generic/schedule/generic.c index 24ab8e06..36ed857b 100644 --- a/platform/linux-generic/schedule/generic.c +++ b/platform/linux-generic/schedule/generic.c @@ -24,6 +24,7 @@ #include <odp_ring_internal.h> #include <odp_timer_internal.h> #include <odp_schedule_subsystem.h> +#include <odp_packet_io_internal.h> /* Should remove this dependency */ #include <odp_queue_internal.h> @@ -368,11 +369,6 @@ static int schedule_init_global(void) return 0; } -static inline void queue_destroy_finalize(uint32_t qi) -{ - sched_cb_queue_destroy_finalize(qi); -} - static int schedule_term_global(void) { int ret = 0; @@ -858,13 +854,13 @@ static inline int do_schedule_grp(odp_queue_t *out_queue, odp_event_t out_ev[], if (ordered && max_num < MAX_DEQ) max_deq = max_num; - num = sched_cb_queue_deq_multi(qi, sched_local.ev_stash, - max_deq); + num = queue_idx_deq_multi(qi, sched_local.ev_stash, + max_deq); if (num < 0) { /* Destroyed queue. Continue scheduling the same * priority queue. */ - sched_cb_queue_destroy_finalize(qi); + queue_destroy_finalize(qi); continue; } @@ -874,7 +870,7 @@ static inline int do_schedule_grp(odp_queue_t *out_queue, odp_event_t out_ev[], continue; } - handle = sched_cb_queue_handle(qi); + handle = queue_handle(qi); sched_local.num = num; sched_local.index = 0; sched_local.queue = handle; @@ -1003,15 +999,15 @@ static inline int do_schedule(odp_queue_t *out_queue, odp_event_t out_ev[], cmd = &sched->pktio_cmd[cmd_index]; /* Poll packet input */ - if (odp_unlikely(sched_cb_pktin_poll(cmd->pktio_index, - cmd->num_pktin, - cmd->pktin))){ + if (odp_unlikely(pktin_poll(cmd->pktio_index, + cmd->num_pktin, + cmd->pktin))){ /* Pktio stopped or closed. Remove poll command and call * stop_finalize when all commands of the pktio has * been removed. */ if (schedule_pktio_stop(cmd->pktio_index, cmd->pktin[0]) == 0) - sched_cb_pktio_stop_finalize(cmd->pktio_index); + pktio_stop_finalize(cmd->pktio_index); free_pktio_cmd(cmd); } else { diff --git a/platform/linux-generic/schedule/iquery.c b/platform/linux-generic/schedule/iquery.c index f03014b8..ac0d0981 100644 --- a/platform/linux-generic/schedule/iquery.c +++ b/platform/linux-generic/schedule/iquery.c @@ -24,6 +24,7 @@ #include <odp_config_internal.h> #include <odp_timer_internal.h> #include <odp_schedule_subsystem.h> +#include <odp_packet_io_internal.h> /* Should remove this dependency */ #include <odp_queue_internal.h> @@ -293,10 +294,10 @@ static int schedule_term_global(void) odp_event_t events[1]; if (sched->availables[i]) - count = sched_cb_queue_deq_multi(i, events, 1); + count = queue_idx_deq_multi(i, events, 1); if (count < 0) - sched_cb_queue_destroy_finalize(i); + queue_destroy_finalize(i); else if (count > 0) ODP_ERR("Queue (%d) not empty\n", i); } @@ -673,16 +674,16 @@ static inline void pktio_poll_input(void) cmd = &sched->pktio_poll.commands[index]; /* Poll packet input */ - if (odp_unlikely(sched_cb_pktin_poll(cmd->pktio, - cmd->count, - cmd->pktin))) { + if (odp_unlikely(pktin_poll(cmd->pktio, + cmd->count, + cmd->pktin))) { /* Pktio stopped or closed. Remove poll * command and call stop_finalize when all * commands of the pktio has been removed. */ if (schedule_pktio_stop(cmd->pktio, cmd->pktin[0]) == 0) - sched_cb_pktio_stop_finalize(cmd->pktio); + pktio_stop_finalize(cmd->pktio); free_pktio_cmd(cmd); } else { @@ -1514,12 +1515,11 @@ static inline int consume_queue(int prio, unsigned int queue_index) if (is_ordered_queue(queue_index)) max = 1; - count = sched_cb_queue_deq_multi( - queue_index, cache->stash, max); + count = queue_idx_deq_multi(queue_index, cache->stash, max); if (count < 0) { DO_SCHED_UNLOCK(); - sched_cb_queue_destroy_finalize(queue_index); + queue_destroy_finalize(queue_index); DO_SCHED_LOCK(); return 0; } @@ -1529,7 +1529,7 @@ static inline int consume_queue(int prio, unsigned int queue_index) cache->top = &cache->stash[0]; cache->count = count; - cache->queue = sched_cb_queue_handle(queue_index); + cache->queue = queue_handle(queue_index); return count; } diff --git a/platform/linux-generic/schedule/scalable.c b/platform/linux-generic/schedule/scalable.c index 787553ca..0fba38a7 100644 --- a/platform/linux-generic/schedule/scalable.c +++ b/platform/linux-generic/schedule/scalable.c @@ -693,7 +693,7 @@ static inline void _schedule_release_ordered(sched_scalable_thread_state_t *ts) ts->rctx = NULL; } -static void pktin_poll(sched_scalable_thread_state_t *ts) +static void poll_pktin(sched_scalable_thread_state_t *ts) { uint32_t i, tag, hi, npolls = 0; int pktio_index, queue_index; @@ -715,8 +715,8 @@ static void pktin_poll(sched_scalable_thread_state_t *ts) /* Tag grabbed */ pktio_index = TAG_2_PKTIO(tag); queue_index = TAG_2_QUEUE(tag); - if (odp_unlikely(sched_cb_pktin_poll(pktio_index, - 1, &queue_index))) { + if (odp_unlikely(pktin_poll(pktio_index, + 1, &queue_index))) { /* Pktio stopped or closed * Remove tag from pktin_tags */ @@ -729,7 +729,7 @@ static void pktin_poll(sched_scalable_thread_state_t *ts) */ if (__atomic_sub_fetch(&pktin_count[pktio_index], 1, __ATOMIC_RELAXED) == 0) - sched_cb_pktio_stop_finalize(pktio_index); + pktio_stop_finalize(pktio_index); } else { /* We don't know whether any packets were found and enqueued * Write back original tag value to release pktin queue @@ -984,7 +984,7 @@ static int _schedule(odp_queue_t *from, odp_event_t ev[], int num_evts) } } - pktin_poll(ts); + poll_pktin(ts); return 0; } diff --git a/platform/linux-generic/schedule/sp.c b/platform/linux-generic/schedule/sp.c index 19700f90..a40d42d8 100644 --- a/platform/linux-generic/schedule/sp.c +++ b/platform/linux-generic/schedule/sp.c @@ -17,6 +17,8 @@ #include <odp_ring_internal.h> #include <odp_timer_internal.h> #include <odp_schedule_subsystem.h> +#include <odp_packet_io_internal.h> +#include <odp_queue_internal.h> #define NUM_THREAD ODP_THREAD_COUNT_MAX #define NUM_QUEUE ODP_CONFIG_QUEUES @@ -224,7 +226,7 @@ static int term_global(void) for (qi = 0; qi < NUM_QUEUE; qi++) { if (sched_global->queue_cmd[qi].s.init) { /* todo: dequeue until empty ? */ - sched_cb_queue_destroy_finalize(qi); + queue_destroy_finalize(qi); } } @@ -500,7 +502,7 @@ static int schedule_multi(odp_queue_t *from, uint64_t wait, if (sched_local.cmd) { /* Continue scheduling if queue is not empty */ - if (sched_cb_queue_empty(sched_local.cmd->s.index) == 0) + if (queue_empty(sched_local.cmd->s.index) == 0) add_tail(sched_local.cmd); sched_local.cmd = NULL; @@ -519,10 +521,10 @@ static int schedule_multi(odp_queue_t *from, uint64_t wait, cmd = sched_cmd(); if (cmd && cmd->s.type == CMD_PKTIO) { - if (sched_cb_pktin_poll(cmd->s.index, cmd->s.num_pktin, - cmd->s.pktin_idx)) { + if (pktin_poll(cmd->s.index, cmd->s.num_pktin, + cmd->s.pktin_idx)) { /* Pktio stopped or closed. */ - sched_cb_pktio_stop_finalize(cmd->s.index); + pktio_stop_finalize(cmd->s.index); } else { /* Continue polling pktio. */ add_tail(cmd); @@ -554,20 +556,20 @@ static int schedule_multi(odp_queue_t *from, uint64_t wait, } qi = cmd->s.index; - num = sched_cb_queue_deq_multi(qi, events, 1); + num = queue_idx_deq_multi(qi, events, 1); if (num > 0) { sched_local.cmd = cmd; if (from) - *from = sched_cb_queue_handle(qi); + *from = queue_handle(qi); return num; } if (num < 0) { /* Destroyed queue */ - sched_cb_queue_destroy_finalize(qi); + queue_destroy_finalize(qi); continue; }