Signed-off-by: Maxim Uvarov <maxim.uva...@linaro.org> --- v2: fixed according to Mikes comments.
ipc.dox | 228 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 228 insertions(+) create mode 100644 ipc.dox diff --git a/ipc.dox b/ipc.dox new file mode 100644 index 0000000..fd8e71d --- /dev/null +++ b/ipc.dox @@ -0,0 +1,228 @@ +/* Copyright (c) 2014, Linaro Limited + * All rights reserved + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +/** +@page ipc_design Inter Process Communication (IPC) API + +@tableofcontents + +@section ipc_intro Introduction + This document defines the two different ODP application modes + multithreading and multiprocessing with respect to their impact on IPC + +@subsection odp_modes Application Thread/Process modes: + ODP applications can use following programming models for multi core support: + -# Single application with ODP worker Threads. + -# Multi process application with single packet I/O pool and common initialization. + -# Different processed communicated thought IPC API. + +@todo - add diagram about IPC modes. + +@subsubsection odp_mode_threads Thread mode + The initialization sequence for thread mode is following: + +@verbatim + main() { + /* Init ODP before calling anything else. */ + odp_init_global(NULL, NULL); + + /* Init this thread. */ + odp_init_local(); + + /* Allocate memory for packets pool. That memory will be visible for all threads.*/ + shm = odp_shm_reserve("shm_packet_pool", + SHM_PKT_POOL_SIZE, ODP_CACHE_LINE_SIZE, 0); + pool_base = odp_shm_addr(shm); + + /* Create pool instance with reserved shm. */ + pool = odp_buffer_pool_create("packet_pool", pool_base, + SHM_PKT_POOL_SIZE, + SHM_PKT_POOL_BUF_SIZE, + ODP_CACHE_LINE_SIZE, + ODP_BUFFER_TYPE_PACKET); + + /* Create worker threads. */ + odph_linux_pthread_create(&thread_tbl[i], 1, core, thr_run_func, + &args); + } + + /* thread function */ + thr_run_func () { + /* Lookup the packet pool */ + pkt_pool = odp_buffer_pool_lookup("packet_pool"); + + /* Open a packet IO instance for this thread */ + pktio = odp_pktio_open("eth0", pkt_pool); + + for (;;) { + /* read buffer */ + buf = odp_schedule(NULL, ODP_SCHED_WAIT); + ... do something ... + } + } +@endverbatim + +@subsubsection odp_mode_processes Processes mode with shared memory + Initialization sequence in processes mode with shared memory is following: + +@verbatim + main() { + /* Init ODP before calling anything else. In process mode odp_init_global + * function called only once in main run process. + */ + odp_init_global(NULL, NULL); + + /* Init this thread. */ + odp_init_local(); + + /* Allocate memory for packets pool. That memory will be visible for all threads.*/ + shm = odp_shm_reserve("shm_packet_pool", + SHM_PKT_POOL_SIZE, ODP_CACHE_LINE_SIZE, 0); + pool_base = odp_shm_addr(shm); + + /* Create pool instance with reserved shm. */ + pool = odp_buffer_pool_create("packet_pool", pool_base, + SHM_PKT_POOL_SIZE, + SHM_PKT_POOL_BUF_SIZE, + ODP_CACHE_LINE_SIZE, + ODP_BUFFER_TYPE_PACKET); + + /* Call odph_linux_process_fork_n which will fork() current process to + * different processes. + */ + odph_linux_process_fork_n(proc, num_workers, first_core); + + /* Run same function as thread uses */ + thr_run_func(); + } + + /* thread function */ + thr_run_func () { + /* Lookup the packet pool */ + pkt_pool = odp_buffer_pool_lookup("packet_pool"); + + /* Open a packet IO instance for this thread */ + pktio = odp_pktio_open("eth0", pkt_pool); + + for (;;) { + /* read buffer */ + buf = odp_schedule(NULL, ODP_SCHED_WAIT); + ... do something ... + } + } +@endverbatim + +@subsubsection odp_mode_sep_processes Separate Processes mode + This mode differs from mode with common shared memory. Each execution thread is completely independent process which calls + odp_init_global() and do other initialization process then opens IPC pktio interface and does packets exchange between processes + to communicate between these independent processes. IPC pktio interface may be used to exchange packets. + For the base implementation (linux-generic) shared memory is used as the IPC mechanism to make it easy to reuse for different use + cases. The use cases are process that may be spread amongst different VMs, bare metal or regular Linux user space, in fact any + process that can share memory. + + In hardware implementations IPC pktio can be offloaded to HW SoC packets functions. + The initialization sequence in the separate thread mode model is same as it is process, both using shared memory but with following + difference: + +@subsubsection odp_mode_sep_processes_cons Separate Processes Sender (linux-generic) + -# Each process calls odp_init_global(), pool creation and etc. + + -# ODP_SHM_PROC flag provided to be able to map that memory from different process. + +@verbatim + shm = odp_shm_reserve("shm_packet_pool", + SHM_PKT_POOL_SIZE, + ODP_CACHE_LINE_SIZE, + ODP_SHM_PROC); + + pool_base = odp_shm_addr(shm); +@endverbatim + + -# Worker thread (or process) creates IPC pktio, and sends buffers to it: + + A) + odp_pktio_t ipc_pktio = odp_pktio_open("ipc_pktio", 0); + odp_pktio_send(ipc_pktio, pkt_tbl, pkts); + + B) instead of using packet io queue can be used in following way: + +@verbatim + odp_queue_t ipcq = odp_pktio_outq_getdef(ipc_pktio); + /* Then enqueue the packet for output queue */ + odp_queue_enq(ipcq, buf); +@endverbatim + +@subsubsection odp_mode_sep_processes_recv Separate Processes Receiver (linux-generic) + On the other end process also creates IPC packet I/O and receives packets + from it. + +@verbatim + /* Create packet pool visible by only second process. We will copy + * packets to that queue from IPC shared memory. + */ + shm = odp_shm_reserve("local_packet_pool", + SHM_PKT_POOL_SIZE, ODP_CACHE_LINE_SIZE, 0); + + pool_base = odp_shm_addr(shm); + pool = odp_buffer_pool_create("ipc_packet_pool", pool_base, + SHM_PKT_POOL_SIZE, + SHM_PKT_POOL_BUF_SIZE, + ODP_CACHE_LINE_SIZE, + ODP_BUFFER_TYPE_PACKET); + + pool_base = NULL; + /* Loop to find remote shared pool */ + while (1) { + shm = odp_shm_reserve("shm_packet_pool", + SHM_PKT_POOL_SIZE, + ODP_CACHE_LINE_SIZE, + ODP_SHM_PROC_NOCREAT); <- ODP_SHM_PROC_NOCREAT flag provided to + not create shared memory object, do only lookup. + pool_base = odp_shm_addr(shm); + if (pool_base != NULL) { + break; + } else { + ODP_DBG("looking up for shm_packet_pool\n"); + sleep(1); + } + } + + + /* Do lookup packet I/O in IPC shared memory, + * and link it to local pool. */ + while (1) { + pktio = odp_pktio_lookup("ipc_pktio", pool, pool_base); + if (pktio == ODP_PKTIO_INVALID) { + sleep(1); + printf("pid %d: looking for ipc_pktio\n", getpid()); + continue; + } + break; + } + + /* Get packets from the IPC */ + for (;;) { + pkts = odp_pktio_recv(pktio, pkt_tbl, MAX_PKT_BURST); + ... + } +@endverbatim + +@subsubsection odp_mode_sep_processes_hw Separate Processes Hardware optimized + Hardware SoC implementation of IPC exchange can differ. It can use a shared pool + or it can rely on the hardware for packet transmission. But the API interface remains the + + + Hardware SoC implementation of IPC exchange can differ. It can use share pool + or can relay on hardware for packet transmission. But the API interface remains the + same: + + odp_pktio_open(), odp_pktio_lookup() + +@todo - Bug 825 odp_buffer_pool_create() API will change to allocate memory for pool inside it. + So odp_shm_reserve() for remote pool memory and odp_pktio_lookup() can go inside + odp_buffer_pool_create(). + +*/ -- 1.8.5.1.163.gd7aced9 _______________________________________________ lng-odp mailing list lng-odp@lists.linaro.org http://lists.linaro.org/mailman/listinfo/lng-odp