This is an automated email from the ASF dual-hosted git repository. xiaoxiang781216 pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/nuttx.git
commit cc6b9388ac5fa24aca40f408b14cb07752956d05 Author: Yongrong Wang <[email protected]> AuthorDate: Wed Jun 25 14:48:08 2025 +0800 drivers/rpmsg/rpmsg_virtio: dispatch rx in the interrupt, run RT endpoints there The rx interrupt only woke the rx thread, which then walked the used ring and called every endpoint callback. An endpoint that needs a low latency answer therefore always paid a thread switch. Walk the ring in the rx interrupt instead. Messages for an RPMSG_PRIO_RT endpoint that has finished its name service handshake are delivered right there; the others are put on a ready list and handled by the rx thread as before. An RT endpoint still waiting for NS_CREATE_ACK keeps going through the thread, so its first data message cannot overtake the ACK and leave ns_bound_cb() uncalled. The ready list nodes are allocated before the vdev is initialized, so an interrupt that arrives during initialization finds them in place. The check that drops a message whose length does not fit its buffer moves with the ring walk into the interrupt. Signed-off-by: Yongrong Wang <[email protected]> Signed-off-by: yintao <[email protected]> Signed-off-by: Bowen Wang <[email protected]> --- drivers/rpmsg/rpmsg_virtio.c | 354 ++++++++++++++++++++++++++++--------------- 1 file changed, 231 insertions(+), 123 deletions(-) diff --git a/drivers/rpmsg/rpmsg_virtio.c b/drivers/rpmsg/rpmsg_virtio.c index 9f20b0ad0f3..df83d862b83 100644 --- a/drivers/rpmsg/rpmsg_virtio.c +++ b/drivers/rpmsg/rpmsg_virtio.c @@ -62,22 +62,31 @@ * Private Types ****************************************************************************/ +struct rpmsg_virtio_node_s +{ + FAR struct rpmsg_hdr *hdr; + struct metal_list node; +}; + struct rpmsg_virtio_priv_s { - struct rpmsg_s rpmsg; - struct rpmsg_virtio_device rvdev; - struct rpmsg_virtio_shm_pool pool[2]; - struct virtio_device *vdev; - sem_t semrx; - sem_t semtx; - pid_t tid; - vq_notify notifytx; - uint16_t headrx; - uint16_t headtx; + struct rpmsg_s rpmsg; + struct rpmsg_virtio_device rvdev; + struct rpmsg_virtio_shm_pool pool[2]; + FAR struct virtio_device *vdev; + sem_t semrx; + sem_t semtx; + pid_t tid; + vq_notify notifytx; + uint16_t headrx; + uint16_t headtx; + FAR struct rpmsg_virtio_node_s *noderx; + struct metal_list freerx; + struct metal_list readyrx; #ifdef CONFIG_RPMSG_VIRTIO_PM - spinlock_t lock; - struct pm_wakelock_s wakelock; - struct wdog_s wdog; + spinlock_t lock; + struct pm_wakelock_s wakelock; + struct wdog_s wdog; #endif }; @@ -85,8 +94,8 @@ struct rpmsg_virtio_priv_s * Private Function Prototypes ****************************************************************************/ -static void rpmsg_virtio_wakeup_rx(FAR struct rpmsg_virtio_priv_s *priv); -static void rpmsg_virtio_wakeup_tx(FAR struct rpmsg_virtio_priv_s *priv); +static void rpmsg_virtio_rx_wakeup(FAR struct rpmsg_virtio_priv_s *priv); +static void rpmsg_virtio_tx_wakeup(FAR struct rpmsg_virtio_priv_s *priv); static int rpmsg_virtio_wait(FAR struct rpmsg_s *rpmsg, FAR sem_t *sem); static int rpmsg_virtio_post(FAR struct rpmsg_s *rpmsg, FAR sem_t *sem); @@ -211,29 +220,8 @@ rpmsg_virtio_pm_action(FAR struct rpmsg_virtio_priv_s *priv, bool stay) spin_unlock_irqrestore_nopreempt(&priv->lock, flags); } -/**************************************************************************** - * Name: rpmsg_virtio_available_rx - ****************************************************************************/ - -static inline bool -rpmsg_virtio_available_rx(FAR struct rpmsg_virtio_priv_s *priv) -{ - FAR struct rpmsg_virtio_device *rvdev = &priv->rvdev; - FAR struct virtqueue *rvq = rvdev->rvq; - - if (rpmsg_virtio_get_role(rvdev) == RPMSG_HOST) - { - return priv->headrx != rvq->vq_used_cons_idx; - } - else - { - return priv->headrx != rvq->vq_available_idx; - } -} - #else # define rpmsg_virtio_pm_action(priv, stay) -# define rpmsg_virtio_available_rx(priv) true #endif /**************************************************************************** @@ -245,6 +233,41 @@ static bool rpmsg_virtio_is_recursive(FAR struct rpmsg_virtio_priv_s *priv) return nxsched_gettid() == priv->tid; } +/**************************************************************************** + * Name: rpmsg_virtio_process_rx_buffer + * + * Note: this function must be called with the rdev->lock held + ****************************************************************************/ + +static int rpmsg_virtio_process_rx_buffer(FAR struct rpmsg_device *rdev, + FAR struct rpmsg_endpoint *ept, + FAR struct rpmsg_hdr *hdr) +{ + int status; + + if (ept->dest_addr == RPMSG_ADDR_ANY) + { + /* First message received from the remote side, + * update channel destination address + */ + + ept->dest_addr = hdr->src; + } + + rpmsg_ept_incref(ept); + metal_mutex_release(&rdev->lock); + + status = ept->cb(ept, RPMSG_LOCATE_DATA(hdr), hdr->len, hdr->src, + ept->priv); + + RPMSG_ASSERT(status >= 0 || status == RPMSG_SUCCESS_BUFFER_RELEASED, + "unexpected callback status\r\n"); + + metal_mutex_acquire(&rdev->lock); + rpmsg_ept_decref(ept); + return status; +} + /**************************************************************************** * Name: rpmsg_virtio_rx_worker ****************************************************************************/ @@ -254,72 +277,35 @@ static void rpmsg_virtio_rx_worker(FAR struct rpmsg_virtio_priv_s *priv) FAR struct virtio_device *vdev = priv->vdev; FAR struct rpmsg_virtio_device *rvdev = vdev->priv; FAR struct rpmsg_device *rdev = &rvdev->rdev; + FAR struct rpmsg_virtio_node_s *node; FAR struct rpmsg_endpoint *ept; - FAR struct rpmsg_hdr *rp_hdr; + FAR struct metal_list *tmp; + FAR struct rpmsg_hdr *hdr; bool release = false; - bool last = false; - uint32_t len; - uint16_t idx; int status; - if (!rpmsg_virtio_available_rx(priv)) - { - return; - } - metal_mutex_acquire(&rdev->lock); - while (!last) + while ((tmp = metal_list_first(&priv->readyrx)) != NULL) { - /* Process the received data from remote node */ - - rp_hdr = rpmsg_virtio_get_rx_buffer(rvdev, &len, &idx, &last); - if (!rp_hdr) - { - break; - } - - rp_hdr->reserved = idx; - - /* Get the channel node from the remote device channels list; drop a - * message whose length does not fit its buffer. - */ - - ept = len >= sizeof(*rp_hdr) && - rp_hdr->len <= len - sizeof(*rp_hdr) ? - rpmsg_get_ept_from_addr(rdev, rp_hdr->dst) : NULL; - rpmsg_ept_incref(ept); - RPMSG_BUF_HELD_INC(rp_hdr); - metal_mutex_release(&rdev->lock); + metal_list_del(tmp); + node = metal_container_of(tmp, struct rpmsg_virtio_node_s, node); + hdr = node->hdr; + metal_list_add_tail(&priv->freerx, &node->node); + ept = rpmsg_get_ept_from_addr(rdev, hdr->dst); if (ept) { - if (ept->dest_addr == RPMSG_ADDR_ANY) - { - /* First message received from the remote side, - * update channel destination address - */ - - ept->dest_addr = rp_hdr->src; - } - - status = ept->cb(ept, RPMSG_LOCATE_DATA(rp_hdr), - rp_hdr->len, rp_hdr->src, ept->priv); - - RPMSG_ASSERT(status >= 0 || - status == RPMSG_SUCCESS_BUFFER_RELEASED, - "unexpected callback status\r\n"); + status = rpmsg_virtio_process_rx_buffer(rdev, ept, hdr); } else { status = RPMSG_SUCCESS; } - metal_mutex_acquire(&rdev->lock); - rpmsg_ept_decref(ept); if (status != RPMSG_SUCCESS_BUFFER_RELEASED && - rpmsg_virtio_buf_held_dec_test(rp_hdr)) + rpmsg_virtio_buf_held_dec_test(hdr)) { - rpmsg_virtio_release_rx_buffer_nolock(rvdev, rp_hdr); + rpmsg_virtio_release_rx_buffer_nolock(rvdev, hdr); release = true; } } @@ -335,10 +321,10 @@ static void rpmsg_virtio_rx_worker(FAR struct rpmsg_virtio_priv_s *priv) } /**************************************************************************** - * Name: rpmsg_virtio_wakeup_rx + * Name: rpmsg_virtio_rx_wakeup ****************************************************************************/ -static void rpmsg_virtio_wakeup_rx(FAR struct rpmsg_virtio_priv_s *priv) +static void rpmsg_virtio_rx_wakeup(FAR struct rpmsg_virtio_priv_s *priv) { int semcount; @@ -350,10 +336,10 @@ static void rpmsg_virtio_wakeup_rx(FAR struct rpmsg_virtio_priv_s *priv) } /**************************************************************************** - * Name: rpmsg_virtio_wakeup_tx + * Name: rpmsg_virtio_tx_wakeup ****************************************************************************/ -static void rpmsg_virtio_wakeup_tx(FAR struct rpmsg_virtio_priv_s *priv) +static void rpmsg_virtio_tx_wakeup(FAR struct rpmsg_virtio_priv_s *priv) { int semcount; @@ -410,7 +396,7 @@ static int rpmsg_virtio_post(FAR struct rpmsg_s *rpmsg, FAR sem_t *sem) if (priv && semcount >= 0) { - rpmsg_virtio_wakeup_tx(priv); + rpmsg_virtio_tx_wakeup(priv); } return ret; @@ -513,10 +499,10 @@ static void rpmsg_virtio_dump(FAR struct rpmsg_s *rpmsg) } /**************************************************************************** - * Name: rpmsg_virtio_update_rx + * Name: rpmsg_virtio_rx_update ****************************************************************************/ -static void rpmsg_virtio_update_rx(FAR struct rpmsg_virtio_priv_s *priv, +static void rpmsg_virtio_rx_update(FAR struct rpmsg_virtio_priv_s *priv, FAR struct virtqueue *rvq) { FAR struct rpmsg_virtio_device *rvdev = &priv->rvdev; @@ -550,6 +536,103 @@ static void rpmsg_virtio_update_tx(FAR struct rpmsg_virtio_priv_s *priv) } } +/**************************************************************************** + * Name: rpmsg_virtio_rx_enqueue + ****************************************************************************/ + +static void rpmsg_virtio_rx_enqueue(FAR struct rpmsg_virtio_priv_s *priv, + FAR struct rpmsg_hdr *hdr) +{ + FAR struct rpmsg_virtio_node_s *node; + FAR struct metal_list *tmp; + + tmp = metal_list_first(&priv->freerx); + DEBUGASSERT(tmp != NULL); + metal_list_del(tmp); + node = metal_container_of(tmp, struct rpmsg_virtio_node_s, node); + node->hdr = hdr; + metal_list_add_tail(&priv->readyrx, &node->node); +} + +/**************************************************************************** + * Name: rpmsg_virtio_rx_dispatch + ****************************************************************************/ + +static void rpmsg_virtio_rx_dispatch(FAR struct rpmsg_virtio_priv_s *priv) +{ + FAR struct rpmsg_virtio_device *rvdev = &priv->rvdev; + FAR struct rpmsg_device *rdev = &rvdev->rdev; + FAR struct rpmsg_endpoint *ept; + FAR struct rpmsg_hdr *hdr; + bool release = false; + bool last = false; + uint32_t len; + uint16_t idx; + int status; + + metal_mutex_acquire(&rdev->lock); + while (!last) + { + /* Process the received data from remote node */ + + hdr = rpmsg_virtio_get_rx_buffer(rvdev, &len, &idx, &last); + if (hdr == NULL) + { + break; + } + + hdr->reserved = idx; + + /* Drop a message whose length does not fit its buffer */ + + ept = len >= sizeof(*hdr) && hdr->len <= len - sizeof(*hdr) ? + rpmsg_get_ept_from_addr(rdev, hdr->dst) : NULL; + RPMSG_BUF_HELD_INC(hdr); + + if (ept) + { + if (ept->priority == RPMSG_PRIO_RT && + ept->dest_addr != RPMSG_ADDR_ANY) + { + status = rpmsg_virtio_process_rx_buffer(rdev, ept, hdr); + } + else + { + rpmsg_virtio_rx_enqueue(priv, hdr); + status = RPMSG_SUCCESS_BUFFER_RELEASED; + } + } + else + { + status = RPMSG_SUCCESS; + } + + if (status != RPMSG_SUCCESS_BUFFER_RELEASED && + rpmsg_virtio_buf_held_dec_test(hdr)) + { + rpmsg_virtio_release_rx_buffer_nolock(rvdev, hdr); + release = true; + } + } + + if (release) + { + /* Tell peer we return some rx buffer */ + + virtqueue_kick(rvdev->rvq); + } + + if (!metal_list_is_empty(&priv->readyrx)) + { + metal_mutex_release(&rdev->lock); + rpmsg_virtio_rx_wakeup(priv); + } + else + { + metal_mutex_release(&rdev->lock); + } +} + /**************************************************************************** * Name: rpmsg_virtio_rx_callback ****************************************************************************/ @@ -559,8 +642,8 @@ static void rpmsg_virtio_rx_callback(FAR struct virtqueue *vq) FAR struct rpmsg_virtio_priv_s *priv = metal_container_of(vq->vq_dev->priv, struct rpmsg_virtio_priv_s, rvdev); - rpmsg_virtio_update_rx(priv, vq); - rpmsg_virtio_wakeup_rx(priv); + rpmsg_virtio_rx_update(priv, vq); + rpmsg_virtio_rx_dispatch(priv); } /**************************************************************************** @@ -572,7 +655,7 @@ static void rpmsg_virtio_tx_callback(FAR struct virtqueue *vq) FAR struct rpmsg_virtio_priv_s *priv = metal_container_of(vq->vq_dev->priv, struct rpmsg_virtio_priv_s, rvdev); - rpmsg_virtio_wakeup_tx(priv); + rpmsg_virtio_tx_wakeup(priv); /* rpmsg_virtio_tx_callback() called normally means the tx buffer has been * returned by peer, so call rpmsg_virtio_pm_action(false) to enter to @@ -640,6 +723,8 @@ static int rpmsg_virtio_start(FAR struct rpmsg_virtio_priv_s *priv) rpmsg_virtio_tx_callback, }; + uint16_t nrx; + uint16_t i; int ret; if (virtio_has_feature(vdev, VIRTIO_RPMSG_F_BUFSZ)) @@ -650,35 +735,56 @@ static int rpmsg_virtio_start(FAR struct rpmsg_virtio_priv_s *priv) &config.r2h_buf_size); } - if (vdev->role == VIRTIO_DEV_DRIVER && - virtio_has_feature(vdev, VIRTIO_RPMSG_F_BUFADDR)) + if (vdev->role == VIRTIO_DEV_DRIVER) { - FAR void *shmbuf_va0; - FAR void *shmbuf_va1; - uint64_t shmbuf_pa0; - uint64_t shmbuf_pa1; - - /* In OpenAMP, priv->pool[0] is the RX share memory pool, should use - * r2h_buf_addr and r2h_buf_size - */ - - virtio_read_config_member(vdev, struct fw_rsc_config, r2h_buf_addr, - &shmbuf_pa0); - shmbuf_va0 = up_addrenv_pa_to_va((uintptr_t)shmbuf_pa0); - rpmsg_virtio_init_shm_pool(&priv->pool[0], shmbuf_va0, - config.r2h_buf_size * vdev->vrings_info[0].info.num_descs); + if (virtio_has_feature(vdev, VIRTIO_RPMSG_F_BUFADDR)) + { + FAR void *shmbuf_va0; + FAR void *shmbuf_va1; + uint64_t shmbuf_pa0; + uint64_t shmbuf_pa1; + + /* In OpenAMP, priv->pool[0] is the RX share memory pool, + * should use r2h_buf_addr and r2h_buf_size + */ + + virtio_read_config_member(vdev, struct fw_rsc_config, r2h_buf_addr, + &shmbuf_pa0); + shmbuf_va0 = up_addrenv_pa_to_va((uintptr_t)shmbuf_pa0); + rpmsg_virtio_init_shm_pool(&priv->pool[0], shmbuf_va0, + config.r2h_buf_size * vdev->vrings_info[0].info.num_descs); + + /* In OpenAMP, priv->pool[1] is the TX share memory pool, + * should use h2r_buf_addr and h2r_buf_size + */ + + virtio_read_config_member(vdev, struct fw_rsc_config, h2r_buf_addr, + &shmbuf_pa1); + shmbuf_va1 = up_addrenv_pa_to_va((uintptr_t)shmbuf_pa1); + rpmsg_virtio_init_shm_pool(&priv->pool[1], shmbuf_va1, + config.h2r_buf_size * vdev->vrings_info[1].info.num_descs); + + config.split_shpool = true; + } - /* In OpenAMP, priv->pool[1] is the TX share memory pool, should use - * h2r_buf_addr and h2r_buf_size - */ + nrx = vdev->vrings_info[0].info.num_descs; + } + else + { + nrx = vdev->vrings_info[1].info.num_descs; + } - virtio_read_config_member(vdev, struct fw_rsc_config, h2r_buf_addr, - &shmbuf_pa1); - shmbuf_va1 = up_addrenv_pa_to_va((uintptr_t)shmbuf_pa1); - rpmsg_virtio_init_shm_pool(&priv->pool[1], shmbuf_va1, - config.h2r_buf_size * vdev->vrings_info[1].info.num_descs); + priv->noderx = kmm_malloc(nrx * sizeof(*priv->noderx)); + if (priv->noderx == NULL) + { + return -ENOMEM; + } - config.split_shpool = true; + metal_list_init(&priv->readyrx); + metal_list_init(&priv->freerx); + for (i = 0; i < nrx; i++) + { + metal_list_add_tail(&priv->freerx, &priv->noderx[i].node); } ret = rpmsg_init_vdev_with_config(&priv->rvdev, vdev, rpmsg_ns_bind, @@ -687,6 +793,7 @@ static int rpmsg_virtio_start(FAR struct rpmsg_virtio_priv_s *priv) if (ret < 0) { rpmsgerr("rpmsg_init_vdev failed, ret=%d\n", ret); + kmm_free(priv->noderx); return ret; } @@ -697,8 +804,8 @@ static int rpmsg_virtio_start(FAR struct rpmsg_virtio_priv_s *priv) /* Wake up the rx thread to process message */ - rpmsg_virtio_update_rx(priv, priv->rvdev.rvq); - rpmsg_virtio_wakeup_rx(priv); + rpmsg_virtio_rx_update(priv, priv->rvdev.rvq); + rpmsg_virtio_rx_dispatch(priv); /* Open tx buffer return callback before the services created below can * wait for a tx buffer. @@ -902,9 +1009,10 @@ void rpmsg_virtio_remove(FAR struct virtio_device *vdev) rpmsg_deinit_vdev(&priv->rvdev); - /* Delete the kthread */ + /* Delete the kthread before freeing noderx: it walks readyrx/freerx */ kthread_delete(priv->tid); + kmm_free(priv->noderx); /* Free the private data */
