Enabling Rx timestamp offload on the AVX2/AVX512 vector paths
introduced several issues:
- PTP packets were no longer flagged. Applications that start the
port with Rx timestamp offload and enable timesync afterwards
(e.g. ptpclient) get a vector Rx path, which never set
RTE_MBUF_F_RX_IEEE1588_PTP, mbuf timesync or the queue time_high.
ptpclient therefore dropped all PTP frames and printed no T1-T4.
- The dynamic timestamp flag was merged through a 32-bit SIMD lane,
so it was lost when the registered flag bit is 32 or higher.
- For scattered packets the first segment kept the timestamp of the
first descriptor, as reassembly does not copy the dynfield, and
non-EOP descriptors were fed into the rollover tracking.
- The 32b to 64b conversion ran before the burst was trimmed for
alignment, updating the state with a packet not yet returned.
Move the post-loop processing into a common helper which runs after
the trim, sets the timestamp flag on ol_flags, converts only EOP
descriptors and stores the result on the first segment, and flags
PTP packets like the scalar path. time_high is updated only for PTP
packets so the timesync read returns the PTP packet timestamp.
Fixes: 332eb4ec8ff0 ("net/ice: support Rx timestamp offload on vector path")
Signed-off-by: Soumyadeep Hore <[email protected]>
---
drivers/net/intel/ice/ice_rxtx_vec_avx2.c | 57 +++----------------
drivers/net/intel/ice/ice_rxtx_vec_avx512.c | 57 +++----------------
drivers/net/intel/ice/ice_rxtx_vec_common.h | 61 +++++++++++++++++++++
3 files changed, 79 insertions(+), 96 deletions(-)
diff --git a/drivers/net/intel/ice/ice_rxtx_vec_avx2.c
b/drivers/net/intel/ice/ice_rxtx_vec_avx2.c
index d1b31bb452..1e468d819b 100644
--- a/drivers/net/intel/ice/ice_rxtx_vec_avx2.c
+++ b/drivers/net/intel/ice/ice_rxtx_vec_avx2.c
@@ -46,6 +46,9 @@ _ice_recv_raw_pkts_vec_avx2(struct ci_rx_queue *rxq, struct
rte_mbuf **rx_pkts,
struct ci_rx_entry *sw_ring = &rxq->sw_ring[rxq->rx_tail];
volatile union ci_rx_flex_desc *rxdp = rxq->rx_flex_ring + rxq->rx_tail;
const int avx_aligned = ((rxq->rx_tail & 1) == 0);
+#ifndef RTE_NET_INTEL_USE_16BYTE_DESC
+ const uint8_t *split_flags = split_packet;
+#endif
rte_prefetch0(rxdp);
@@ -559,9 +562,6 @@ _ice_recv_raw_pkts_vec_avx2(struct ci_rx_queue *rxq, struct
rte_mbuf **rx_pkts,
*RTE_MBUF_DYNFIELD(rx_pkts[i + 7],
rxq->ts_offset, uint32_t *) =
_mm256_extract_epi32(raw_desc_bh6_7, 7);
-
- mbuf_flags = _mm256_or_si256(mbuf_flags,
-
_mm256_set1_epi32((int)rxq->ts_flag));
} /* if() on Rx timestamp parsing */
} /* if() on RSS hash or Rx timestamp parsing */
#endif
@@ -698,51 +698,6 @@ _ice_recv_raw_pkts_vec_avx2(struct ci_rx_queue *rxq,
struct rte_mbuf **rx_pkts,
break;
}
-#ifndef RTE_NET_INTEL_USE_16BYTE_DESC
- /**
- * Convert the stashed 32b Rx timestamps to 64b for the packets that
- * were actually received, tracking the register rollover. This mirrors
- * the scalar Rx path and is only done over valid (received) packets, so
- * timestamps of non-DD descriptors never corrupt the rollover state.
- */
- if (offload && received > 0 &&
- (rxq->offloads & RTE_ETH_RX_OFFLOAD_TIMESTAMP)) {
- struct ice_vsi *vsi = rxq->ice_vsi;
- struct ice_hw *hw = ICE_VSI_TO_HW(vsi);
- struct ice_adapter *ad = vsi->adapter;
- uint64_t ts_ns;
- bool is_tsinit = false;
- uint64_t sw_cur_time =
- rte_get_timer_cycles() / (rte_get_timer_hz() / 1000);
-
- if (unlikely(sw_cur_time - rxq->hw_time_update > 4))
- is_tsinit = true;
-
- for (uint16_t k = 0; k < received; k++) {
- uint32_t ts_high = *RTE_MBUF_DYNFIELD(rx_pkts[k],
- rxq->ts_offset, uint32_t *);
-
- rxq->time_high = ts_high;
- if (unlikely(is_tsinit)) {
- ts_ns = ice_tstamp_convert_32b_64b(hw, ad, 1,
- ts_high);
- rxq->hw_time_low = (uint32_t)ts_ns;
- rxq->hw_time_high = (uint32_t)(ts_ns >> 32);
- is_tsinit = false;
- } else {
- if (ts_high < rxq->hw_time_low)
- rxq->hw_time_high += 1;
- ts_ns = (uint64_t)rxq->hw_time_high << 32 |
ts_high;
- rxq->hw_time_low = ts_high;
- }
- *RTE_MBUF_DYNFIELD(rx_pkts[k], rxq->ts_offset,
- rte_mbuf_timestamp_t *) = ts_ns;
- }
- rxq->hw_time_update = rte_get_timer_cycles() /
- (rte_get_timer_hz() / 1000);
- }
-#endif
-
/* update tail pointers */
rxq->rx_tail += received;
rxq->rx_tail &= (rxq->nb_rx_desc - 1);
@@ -751,6 +706,12 @@ _ice_recv_raw_pkts_vec_avx2(struct ci_rx_queue *rxq,
struct rte_mbuf **rx_pkts,
received--;
}
rxq->rxrearm_nb += received;
+
+#ifndef RTE_NET_INTEL_USE_16BYTE_DESC
+ if (offload && received > 0 &&
+ (rxq->offloads & RTE_ETH_RX_OFFLOAD_TIMESTAMP))
+ ice_rx_vec_ts_finalize(rxq, rx_pkts, received, split_flags);
+#endif
return received;
}
diff --git a/drivers/net/intel/ice/ice_rxtx_vec_avx512.c
b/drivers/net/intel/ice/ice_rxtx_vec_avx512.c
index 81eaa47842..ef189803d6 100644
--- a/drivers/net/intel/ice/ice_rxtx_vec_avx512.c
+++ b/drivers/net/intel/ice/ice_rxtx_vec_avx512.c
@@ -47,6 +47,9 @@ _ice_recv_raw_pkts_vec_avx512(struct ci_rx_queue *rxq,
0, rxq->mbuf_initializer);
struct ci_rx_entry *sw_ring = &rxq->sw_ring[rxq->rx_tail];
volatile union ci_rx_flex_desc *rxdp = rxq->rx_flex_ring + rxq->rx_tail;
+#ifndef RTE_NET_INTEL_USE_16BYTE_DESC
+ const uint8_t *split_flags = split_packet;
+#endif
rte_prefetch0(rxdp);
@@ -581,9 +584,6 @@ _ice_recv_raw_pkts_vec_avx512(struct ci_rx_queue *rxq,
*RTE_MBUF_DYNFIELD(rx_pkts[i + 7],
rxq->ts_offset, uint32_t *) =
_mm256_extract_epi32(raw_desc_bh6_7, 7);
-
- mbuf_flags = _mm256_or_si256(mbuf_flags,
-
_mm256_set1_epi32((int)rxq->ts_flag));
} /* if() on Rx timestamp parsing */
} /* if() on RSS hash or Rx timestamp parsing */
#endif
@@ -724,51 +724,6 @@ _ice_recv_raw_pkts_vec_avx512(struct ci_rx_queue *rxq,
break;
}
-#ifndef RTE_NET_INTEL_USE_16BYTE_DESC
- /**
- * Convert the stashed 32b Rx timestamps to 64b for the packets that
- * were actually received, tracking the register rollover. This mirrors
- * the scalar Rx path and is only done over valid (received) packets, so
- * timestamps of non-DD descriptors never corrupt the rollover state.
- */
- if (do_offload && received > 0 &&
- (rxq->offloads & RTE_ETH_RX_OFFLOAD_TIMESTAMP)) {
- struct ice_vsi *vsi = rxq->ice_vsi;
- struct ice_hw *hw = ICE_VSI_TO_HW(vsi);
- struct ice_adapter *ad = vsi->adapter;
- uint64_t ts_ns;
- bool is_tsinit = false;
- uint64_t sw_cur_time =
- rte_get_timer_cycles() / (rte_get_timer_hz() / 1000);
-
- if (unlikely(sw_cur_time - rxq->hw_time_update > 4))
- is_tsinit = true;
-
- for (uint16_t k = 0; k < received; k++) {
- uint32_t ts_high = *RTE_MBUF_DYNFIELD(rx_pkts[k],
- rxq->ts_offset, uint32_t *);
-
- rxq->time_high = ts_high;
- if (unlikely(is_tsinit)) {
- ts_ns = ice_tstamp_convert_32b_64b(hw, ad, 1,
- ts_high);
- rxq->hw_time_low = (uint32_t)ts_ns;
- rxq->hw_time_high = (uint32_t)(ts_ns >> 32);
- is_tsinit = false;
- } else {
- if (ts_high < rxq->hw_time_low)
- rxq->hw_time_high += 1;
- ts_ns = (uint64_t)rxq->hw_time_high << 32 |
ts_high;
- rxq->hw_time_low = ts_high;
- }
- *RTE_MBUF_DYNFIELD(rx_pkts[k], rxq->ts_offset,
- rte_mbuf_timestamp_t *) = ts_ns;
- }
- rxq->hw_time_update = rte_get_timer_cycles() /
- (rte_get_timer_hz() / 1000);
- }
-#endif
-
/* update tail pointers */
rxq->rx_tail += received;
rxq->rx_tail &= (rxq->nb_rx_desc - 1);
@@ -777,6 +732,12 @@ _ice_recv_raw_pkts_vec_avx512(struct ci_rx_queue *rxq,
received--;
}
rxq->rxrearm_nb += received;
+
+#ifndef RTE_NET_INTEL_USE_16BYTE_DESC
+ if (do_offload && received > 0 &&
+ (rxq->offloads & RTE_ETH_RX_OFFLOAD_TIMESTAMP))
+ ice_rx_vec_ts_finalize(rxq, rx_pkts, received, split_flags);
+#endif
return received;
}
diff --git a/drivers/net/intel/ice/ice_rxtx_vec_common.h
b/drivers/net/intel/ice/ice_rxtx_vec_common.h
index d5ef213e08..54413eebd3 100644
--- a/drivers/net/intel/ice/ice_rxtx_vec_common.h
+++ b/drivers/net/intel/ice/ice_rxtx_vec_common.h
@@ -5,9 +5,70 @@
#ifndef _ICE_RXTX_VEC_COMMON_H_
#define _ICE_RXTX_VEC_COMMON_H_
+#include <rte_cycles.h>
+#include <rte_mbuf_dyn.h>
+
#include "../common/rx.h"
#include "ice_rxtx.h"
+#ifndef RTE_NET_INTEL_USE_16BYTE_DESC
+/*
+ * Extend the 32b Rx timestamps stashed in the mbuf dynfield by the vector
+ * loop to 64b, set the timestamp flag and mark PTP packets. Only EOP
+ * descriptors carry the timestamp; it is stored on the packet's first segment.
+ */
+static inline void
+ice_rx_vec_ts_finalize(struct ci_rx_queue *rxq, struct rte_mbuf **rx_pkts,
+ uint16_t nb_bufs, const uint8_t *split_flags)
+{
+ struct ice_vsi *vsi = rxq->ice_vsi;
+ struct ice_adapter *ad = vsi->adapter;
+ struct ice_hw *hw = ICE_VSI_TO_HW(vsi);
+ const uint64_t ms_hz = rte_get_timer_hz() / 1000;
+ bool is_tsinit = rte_get_timer_cycles() / ms_hz - rxq->hw_time_update >
4;
+ /* first segment of a packet that started in a previous burst */
+ struct rte_mbuf *head = split_flags != NULL ? rxq->pkt_first_seg : NULL;
+ uint16_t k;
+
+ for (k = 0; k < nb_bufs; k++) {
+ struct rte_mbuf *mb = rx_pkts[k];
+ uint32_t ts_high;
+ uint64_t ts_ns;
+
+ if (head == NULL)
+ head = mb;
+ if (split_flags != NULL && split_flags[k] != 0)
+ continue;
+
+ ts_high = *RTE_MBUF_DYNFIELD(mb, rxq->ts_offset, uint32_t *);
+ if (unlikely(is_tsinit)) {
+ ts_ns = ice_tstamp_convert_32b_64b(hw, ad, 1, ts_high);
+ rxq->hw_time_low = (uint32_t)ts_ns;
+ rxq->hw_time_high = (uint32_t)(ts_ns >> 32);
+ is_tsinit = false;
+ } else {
+ if (ts_high < rxq->hw_time_low)
+ rxq->hw_time_high += 1;
+ ts_ns = (uint64_t)rxq->hw_time_high << 32 | ts_high;
+ rxq->hw_time_low = ts_high;
+ }
+ *RTE_MBUF_DYNFIELD(head, rxq->ts_offset, rte_mbuf_timestamp_t
*) = ts_ns;
+ /* reassembly copies ol_flags from the EOP segment to the head
*/
+ mb->ol_flags |= rxq->ts_flag;
+
+ if (ad->ptp_ena && (mb->packet_type & RTE_PTYPE_L2_MASK) ==
+ RTE_PTYPE_L2_ETHER_TIMESYNC) {
+ rxq->time_high = ts_high;
+ head->timesync = rxq->queue_id;
+ mb->timesync = rxq->queue_id;
+ mb->ol_flags |= RTE_MBUF_F_RX_IEEE1588_PTP;
+ }
+ head = NULL;
+ }
+ rxq->hw_time_update = rte_get_timer_cycles() / ms_hz;
+}
+#endif
+
static inline int
ice_tx_desc_done(struct ci_tx_queue *txq, uint16_t idx)
{
--
2.43.0