Signed-off-by: Maxim Uvarov <maxim.uva...@linaro.org>
---
v2: fixed according to Mikes comments.
ipc.dox | 228 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 228 insertions(+)
create mode 100644 ipc.dox
diff --git a/ipc.dox b/ipc.dox
new file mode 100644
index 0000000..fd8e71d
--- /dev/null
+++ b/ipc.dox
@@ -0,0 +1,228 @@
+/* Copyright (c) 2014, Linaro Limited
+ * All rights reserved
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+/**
+@page ipc_design Inter Process Communication (IPC) API
+
+@tableofcontents
+
+@section ipc_intro Introduction
+ This document defines the two different ODP application modes
+ multithreading and multiprocessing with respect to their impact on IPC
+
+@subsection odp_modes Application Thread/Process modes:
+ ODP applications can use following programming models for multi core
support:
+ -# Single application with ODP worker Threads.
+ -# Multi process application with single packet I/O pool and common
initialization.
+ -# Different processed communicated thought IPC API.
+
+@todo - add diagram about IPC modes.
+
+@subsubsection odp_mode_threads Thread mode
+ The initialization sequence for thread mode is following:
+
+@verbatim
+ main() {
+ /* Init ODP before calling anything else. */
+ odp_init_global(NULL, NULL);
+
+ /* Init this thread. */
+ odp_init_local();
+
+ /* Allocate memory for packets pool. That memory will be
visible for all threads.*/
+ shm = odp_shm_reserve("shm_packet_pool",
+ SHM_PKT_POOL_SIZE, ODP_CACHE_LINE_SIZE,
0);
+ pool_base = odp_shm_addr(shm);
+
+ /* Create pool instance with reserved shm. */
+ pool = odp_buffer_pool_create("packet_pool", pool_base,
+ SHM_PKT_POOL_SIZE,
+ SHM_PKT_POOL_BUF_SIZE,
+ ODP_CACHE_LINE_SIZE,
+ ODP_BUFFER_TYPE_PACKET);
+
+ /* Create worker threads. */
+ odph_linux_pthread_create(&thread_tbl[i], 1, core, thr_run_func,
+ &args);
+ }
+
+ /* thread function */
+ thr_run_func () {
+ /* Lookup the packet pool */
+ pkt_pool = odp_buffer_pool_lookup("packet_pool");
+
+ /* Open a packet IO instance for this thread */
+ pktio = odp_pktio_open("eth0", pkt_pool);
+
+ for (;;) {
+ /* read buffer */
+ buf = odp_schedule(NULL, ODP_SCHED_WAIT);
+ ... do something ...
+ }
+ }
+@endverbatim
+
+@subsubsection odp_mode_processes Processes mode with shared memory
+ Initialization sequence in processes mode with shared memory is
following:
+
+@verbatim
+ main() {
+ /* Init ODP before calling anything else. In process mode
odp_init_global
+ * function called only once in main run process.
+ */
+ odp_init_global(NULL, NULL);
+
+ /* Init this thread. */
+ odp_init_local();
+
+ /* Allocate memory for packets pool. That memory will be
visible for all threads.*/
+ shm = odp_shm_reserve("shm_packet_pool",
+ SHM_PKT_POOL_SIZE, ODP_CACHE_LINE_SIZE, 0);
+ pool_base = odp_shm_addr(shm);
+
+ /* Create pool instance with reserved shm. */
+ pool = odp_buffer_pool_create("packet_pool", pool_base,
+ SHM_PKT_POOL_SIZE,
+ SHM_PKT_POOL_BUF_SIZE,
+ ODP_CACHE_LINE_SIZE,
+ ODP_BUFFER_TYPE_PACKET);
+
+ /* Call odph_linux_process_fork_n which will fork() current
process to
+ * different processes.
+ */
+ odph_linux_process_fork_n(proc, num_workers, first_core);
+
+ /* Run same function as thread uses */
+ thr_run_func();
+ }
+
+ /* thread function */
+ thr_run_func () {
+ /* Lookup the packet pool */
+ pkt_pool = odp_buffer_pool_lookup("packet_pool");
+
+ /* Open a packet IO instance for this thread */
+ pktio = odp_pktio_open("eth0", pkt_pool);
+
+ for (;;) {
+ /* read buffer */
+ buf = odp_schedule(NULL, ODP_SCHED_WAIT);
+ ... do something ...
+ }
+ }
+@endverbatim
+
+@subsubsection odp_mode_sep_processes Separate Processes mode
+ This mode differs from mode with common shared memory. Each execution
thread is completely independent process which calls
+ odp_init_global() and do other initialization process then opens IPC
pktio interface and does packets exchange between processes
+ to communicate between these independent processes. IPC pktio interface
may be used to exchange packets.
+ For the base implementation (linux-generic) shared memory is used as
the IPC mechanism to make it easy to reuse for different use
+ cases. The use cases are process that may be spread amongst different
VMs, bare metal or regular Linux user space, in fact any
+ process that can share memory.
+
+ In hardware implementations IPC pktio can be offloaded to HW SoC
packets functions.
+ The initialization sequence in the separate thread mode model is same
as it is process, both using shared memory but with following
+ difference:
+
+@subsubsection odp_mode_sep_processes_cons Separate Processes Sender
(linux-generic)
+ -# Each process calls odp_init_global(), pool creation and etc.
+
+ -# ODP_SHM_PROC flag provided to be able to map that memory from
different process.
+
+@verbatim
+ shm = odp_shm_reserve("shm_packet_pool",
+ SHM_PKT_POOL_SIZE,
+ ODP_CACHE_LINE_SIZE,
+ ODP_SHM_PROC);
+
+ pool_base = odp_shm_addr(shm);
+@endverbatim
+
+ -# Worker thread (or process) creates IPC pktio, and sends buffers to
it:
+
+ A)
+ odp_pktio_t ipc_pktio = odp_pktio_open("ipc_pktio", 0);
+ odp_pktio_send(ipc_pktio, pkt_tbl, pkts);
+
+ B) instead of using packet io queue can be used in following way:
+
+@verbatim
+ odp_queue_t ipcq = odp_pktio_outq_getdef(ipc_pktio);
+ /* Then enqueue the packet for output queue */
+ odp_queue_enq(ipcq, buf);
+@endverbatim
+
+@subsubsection odp_mode_sep_processes_recv Separate Processes Receiver
(linux-generic)
+ On the other end process also creates IPC packet I/O and receives
packets
+ from it.
+
+@verbatim
+ /* Create packet pool visible by only second process. We will copy
+ * packets to that queue from IPC shared memory.
+ */
+ shm = odp_shm_reserve("local_packet_pool",
+ SHM_PKT_POOL_SIZE, ODP_CACHE_LINE_SIZE, 0);
+
+ pool_base = odp_shm_addr(shm);
+ pool = odp_buffer_pool_create("ipc_packet_pool", pool_base,
+ SHM_PKT_POOL_SIZE,
+ SHM_PKT_POOL_BUF_SIZE,
+ ODP_CACHE_LINE_SIZE,
+ ODP_BUFFER_TYPE_PACKET);
+
+ pool_base = NULL;
+ /* Loop to find remote shared pool */
+ while (1) {
+ shm = odp_shm_reserve("shm_packet_pool",
+ SHM_PKT_POOL_SIZE,
+ ODP_CACHE_LINE_SIZE,
+ ODP_SHM_PROC_NOCREAT); <- ODP_SHM_PROC_NOCREAT
flag provided to
+ not create shared memory
object, do only lookup.
+ pool_base = odp_shm_addr(shm);
+ if (pool_base != NULL) {
+ break;
+ } else {
+ ODP_DBG("looking up for shm_packet_pool\n");
+ sleep(1);
+ }
+ }
+
+
+ /* Do lookup packet I/O in IPC shared memory,
+ * and link it to local pool. */
+ while (1) {
+ pktio = odp_pktio_lookup("ipc_pktio", pool, pool_base);
+ if (pktio == ODP_PKTIO_INVALID) {
+ sleep(1);
+ printf("pid %d: looking for ipc_pktio\n", getpid());
+ continue;
+ }
+ break;
+ }
+
+ /* Get packets from the IPC */
+ for (;;) {
+ pkts = odp_pktio_recv(pktio, pkt_tbl, MAX_PKT_BURST);
+ ...
+ }
+@endverbatim
+
+@subsubsection odp_mode_sep_processes_hw Separate Processes Hardware optimized
+ Hardware SoC implementation of IPC exchange can differ. It can use a
shared pool
+ or it can rely on the hardware for packet transmission. But the API
interface remains the
+
+
+ Hardware SoC implementation of IPC exchange can differ. It can use
share pool
+ or can relay on hardware for packet transmission. But the API interface
remains the
+ same:
+
+ odp_pktio_open(), odp_pktio_lookup()
+
+@todo - Bug 825 odp_buffer_pool_create() API will change to allocate memory
for pool inside it.
+ So odp_shm_reserve() for remote pool memory and odp_pktio_lookup() can
go inside
+ odp_buffer_pool_create().
+
+*/
--
1.8.5.1.163.gd7aced9
_______________________________________________
lng-odp mailing list
lng-odp@lists.linaro.org
http://lists.linaro.org/mailman/listinfo/lng-odp
