Hardware checksum offload needs the pseudo-header checksum precomputed by the driver. Compute the inner/outer pseudo-header checksums, including walking IPv6 extension headers, in the Tx prepare path.
Signed-off-by: Jiacheng Ye <[email protected]> --- drivers/net/hinic3/hinic3_ethdev.c | 1 + drivers/net/hinic3/hinic3_ethdev.h | 1 + drivers/net/hinic3/hinic3_tx.c | 179 ++++++++++++++++++++++++++++- drivers/net/hinic3/hinic3_tx.h | 11 ++ 4 files changed, 186 insertions(+), 6 deletions(-) diff --git a/drivers/net/hinic3/hinic3_ethdev.c b/drivers/net/hinic3/hinic3_ethdev.c index ffbd080ede..0f7a653594 100644 --- a/drivers/net/hinic3/hinic3_ethdev.c +++ b/drivers/net/hinic3/hinic3_ethdev.c @@ -871,6 +871,7 @@ hinic3_dev_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *info) RTE_ETH_TX_OFFLOAD_TCP_CKSUM | RTE_ETH_TX_OFFLOAD_SCTP_CKSUM | RTE_ETH_TX_OFFLOAD_OUTER_IPV4_CKSUM | + RTE_ETH_TX_OFFLOAD_OUTER_UDP_CKSUM | RTE_ETH_TX_OFFLOAD_VXLAN_TNL_TSO | RTE_ETH_TX_OFFLOAD_TCP_TSO | RTE_ETH_TX_OFFLOAD_MULTI_SEGS | diff --git a/drivers/net/hinic3/hinic3_ethdev.h b/drivers/net/hinic3/hinic3_ethdev.h index 399c9c9342..1afd7cd77a 100644 --- a/drivers/net/hinic3/hinic3_ethdev.h +++ b/drivers/net/hinic3/hinic3_ethdev.h @@ -49,6 +49,7 @@ #define HINIC3_PKT_TX_OUTER_IP_CKSUM RTE_MBUF_F_TX_OUTER_IP_CKSUM #define HINIC3_PKT_TX_OUTER_UDP_CKSUM RTE_MBUF_F_TX_OUTER_UDP_CKSUM #define HINIC3_PKT_TX_OUTER_IPV6 RTE_MBUF_F_TX_OUTER_IPV6 +#define HINIC3_PKT_TX_OUTER_IPV4 RTE_MBUF_F_TX_OUTER_IPV4 #define HINIC3_PKT_RX_LRO RTE_MBUF_F_RX_LRO #define HINIC3_PKT_TX_L4_NO_CKSUM RTE_MBUF_F_TX_L4_NO_CKSUM diff --git a/drivers/net/hinic3/hinic3_tx.c b/drivers/net/hinic3/hinic3_tx.c index c3f053f869..bf79b1bd28 100644 --- a/drivers/net/hinic3/hinic3_tx.c +++ b/drivers/net/hinic3/hinic3_tx.c @@ -31,7 +31,7 @@ (HINIC3_PKT_TX_IP_CKSUM | HINIC3_PKT_TX_TCP_CKSUM | \ HINIC3_PKT_TX_UDP_CKSUM | HINIC3_PKT_TX_SCTP_CKSUM | \ HINIC3_PKT_TX_OUTER_IP_CKSUM | HINIC3_PKT_TX_OUTER_UDP_CKSUM | \ - HINIC3_PKT_TX_TCP_SEG) + HINIC3_PKT_TX_TCP_SEG | HINIC3_PKT_TX_IPV6) static inline uint16_t hinic3_get_sq_free_wqebbs(struct hinic3_txq *sq) @@ -366,6 +366,68 @@ hinic3_tx_done_cleanup(void *txq, uint32_t free_cnt) return hinic3_xmit_mbuf_cleanup(tx_queue, try_free_cnt); } +static void hinic3_get_ipv6_len_proto(const void *hdr, uint16_t *hdr_len, uint8_t *proto) +{ + const struct hinic3_ipv6_ext_hdr *xh = NULL; + uint8_t i; + + *proto = ((const struct rte_ipv6_hdr *)hdr)->proto; + *hdr_len = sizeof(struct rte_ipv6_hdr); + + /* + * Consistent with ucode's IPV6_MAX_EXT_HDRS, + * maximum parsing up to IPV6_MAX_EXT_HDRS layer. + */ + for (i = 0; i < IPV6_MAX_EXT_HDRS; i++) { + xh = (const struct hinic3_ipv6_ext_hdr *)((const uint8_t *)hdr + *hdr_len); + switch (*proto) { + case IPPROTO_HOPOPTS: + case IPPROTO_ROUTING: + case IPPROTO_DSTOPTS: + /* hdr len is a multiple of 8, excluding the first 8 bytes */ + *hdr_len += FIXED_EXT_HDR_LEN + xh->len * UNIT_BYTES_U; + *proto = xh->next_hdr; + break; + case IPPROTO_AH: + /* hdr len is a multiple of 4, excluding the first 8 bytes */ + *hdr_len += FIXED_EXT_HDR_LEN + xh->len * UNIT_BYTES_AH; + *proto = xh->next_hdr; + break; + case IPPROTO_FRAGMENT: + /* hdr len is fixed 8 bytes */ + *hdr_len += FIXED_EXT_HDR_LEN; + *proto = xh->next_hdr; + break; + default: + break; + } + } +} + +static uint16_t +hinic3_ipv6_phdr_cksum(const struct rte_ipv6_hdr *ipv6_hdr, uint64_t ol_flags) +{ + uint32_t sum; + uint8_t proto; + uint16_t l3_len; + rte_be32_t l4_len; + rte_be32_t l4_proto; + + hinic3_get_ipv6_len_proto((const void *)ipv6_hdr, &l3_len, &proto); + l4_proto = rte_cpu_to_be_16(proto); + if (ol_flags & HINIC3_PKT_TX_TCP_SEG) + l4_len = 0; + else + l4_len = rte_cpu_to_be_16(rte_be_to_cpu_16(ipv6_hdr->payload_len) - + l3_len + sizeof(*ipv6_hdr)); + + sum = __rte_raw_cksum(ipv6_hdr->src_addr.a, + sizeof(ipv6_hdr->src_addr.a) + sizeof(ipv6_hdr->dst_addr.a), 0); + sum = __rte_raw_cksum(&l4_len, sizeof(l4_len), sum); + sum = __rte_raw_cksum(&l4_proto, sizeof(l4_proto), sum); + + return __rte_raw_cksum_reduce(sum); +} /** * Prepare the data packet to be sent and calculate the internal L3 offset. * @@ -378,6 +440,90 @@ hinic3_tx_done_cleanup(void *txq, uint32_t free_cnt) * @return * 0 as success, -EINVAL as failure. */ +static inline uint16_t +hinic3_calculate_l4_cksum(struct rte_mbuf *mbuf, uint16_t inner_l3_offset) +{ + struct rte_ipv4_hdr *ipv4_hdr; + struct rte_ipv6_hdr *ipv6_hdr; + uint64_t ol_flags = mbuf->ol_flags; + + if (ol_flags & HINIC3_PKT_TX_IPV4) { + ipv4_hdr = rte_pktmbuf_mtod_offset(mbuf, struct rte_ipv4_hdr *, + inner_l3_offset); + + if (ol_flags & HINIC3_PKT_TX_IP_CKSUM) + ipv4_hdr->hdr_checksum = 0; + + return rte_ipv4_phdr_cksum(ipv4_hdr, ol_flags); + } + + ipv6_hdr = rte_pktmbuf_mtod_offset(mbuf, struct rte_ipv6_hdr *, + inner_l3_offset); + + return hinic3_ipv6_phdr_cksum(ipv6_hdr, ol_flags); +} + +static inline void hinic3_calculate_outer_udp_checksum(struct rte_mbuf *mbuf, + uint16_t outer_l3_offset) +{ + struct rte_ipv4_hdr *ipv4_hdr; + struct rte_ipv6_hdr *ipv6_hdr; + struct rte_udp_hdr *udp_hdr; + uint64_t ol_flags = mbuf->ol_flags; + + if (ol_flags & HINIC3_PKT_TX_OUTER_IPV4) { + ipv4_hdr = rte_pktmbuf_mtod_offset(mbuf, struct rte_ipv4_hdr *, outer_l3_offset); + if (ol_flags & HINIC3_PKT_TX_OUTER_IP_CKSUM) + ipv4_hdr->hdr_checksum = 0; + udp_hdr = (struct rte_udp_hdr *)((char *)ipv4_hdr + mbuf->outer_l3_len); + if (udp_hdr->dgram_cksum != 0x0) + udp_hdr->dgram_cksum = rte_ipv4_phdr_cksum(ipv4_hdr, ol_flags); + } else if (ol_flags & HINIC3_PKT_TX_OUTER_IPV6) { + ipv6_hdr = rte_pktmbuf_mtod_offset(mbuf, struct rte_ipv6_hdr *, outer_l3_offset); + udp_hdr = rte_pktmbuf_mtod_offset(mbuf, struct rte_udp_hdr *, + (outer_l3_offset + mbuf->outer_l3_len)); + udp_hdr->dgram_cksum = hinic3_ipv6_phdr_cksum(ipv6_hdr, ol_flags); + } +} + +static inline void hinic3_calculate_outer_phdr_checksum(struct rte_mbuf *mbuf, + uint16_t outer_l3_offset) +{ + uint64_t ol_flags = mbuf->ol_flags; + + if (ol_flags & HINIC3_PKT_TX_OUTER_UDP_CKSUM) + hinic3_calculate_outer_udp_checksum(mbuf, outer_l3_offset); +} + +static inline void hinic3_calculate_phdr_checksum(struct rte_mbuf *mbuf, + uint16_t inner_l3_offset) +{ + uint64_t ol_flags = mbuf->ol_flags; + struct rte_tcp_hdr *tcp_hdr; + struct rte_udp_hdr *udp_hdr; + uint16_t l4_offset = inner_l3_offset + mbuf->l3_len; + + switch (ol_flags & HINIC3_PKT_TX_L4_MASK) { + case HINIC3_PKT_TX_UDP_CKSUM: + udp_hdr = rte_pktmbuf_mtod_offset(mbuf, struct rte_udp_hdr *, l4_offset); + udp_hdr->dgram_cksum = hinic3_calculate_l4_cksum(mbuf, inner_l3_offset); + break; + case HINIC3_PKT_TX_TCP_CKSUM: + tcp_hdr = rte_pktmbuf_mtod_offset(mbuf, struct rte_tcp_hdr *, l4_offset); + tcp_hdr->cksum = hinic3_calculate_l4_cksum(mbuf, inner_l3_offset); + break; + case HINIC3_PKT_TX_SCTP_CKSUM: + /* Sctp csum no need to calculate pseudo-header */ + break; + default: + if (ol_flags & HINIC3_PKT_TX_TCP_SEG) { + tcp_hdr = rte_pktmbuf_mtod_offset(mbuf, struct rte_tcp_hdr *, l4_offset); + tcp_hdr->cksum = hinic3_calculate_l4_cksum(mbuf, inner_l3_offset); + } + break; + } +} + static int hinic3_tx_offload_pkt_prepare(struct hinic3_nic_dev *nic_dev, struct rte_mbuf *mbuf, uint16_t *inner_l3_offset) @@ -389,7 +535,7 @@ hinic3_tx_offload_pkt_prepare(struct hinic3_nic_dev *nic_dev, struct rte_mbuf *m HINIC3_SUPPORT_VXLAN_OFFLOAD(nic_dev)) || ((ol_flags & HINIC3_PKT_TX_TUNNEL_GENEVE) && HINIC3_SUPPORT_GENEVE_OFFLOAD(nic_dev)) || - ((ol_flags & HINIC3_PKT_TX_TUNNEL_IPIP) && + (((ol_flags & HINIC3_PKT_TX_TUNNEL_IPIP) == HINIC3_PKT_TX_TUNNEL_IPIP) && HINIC3_SUPPORT_IPXIP_OFFLOAD(nic_dev)))) return -EINVAL; } @@ -400,15 +546,33 @@ hinic3_tx_offload_pkt_prepare(struct hinic3_nic_dev *nic_dev, struct rte_mbuf *m #endif /* Support tunnel. */ if ((ol_flags & HINIC3_PKT_TX_TUNNEL_MASK)) { - if ((ol_flags & HINIC3_PKT_TX_OUTER_IP_CKSUM) || - (ol_flags & HINIC3_PKT_TX_OUTER_IPV6) || - (ol_flags & HINIC3_PKT_TX_TCP_SEG)) { + /* + * For IPv6-in-IPv6 packets, ol_flags cannot be used to determine + * whether the delivered port offload value contains UDP and IP + * hardware offload flags, so it is impossible to determine how + * l2_len is populated. Here, the property that l2_len is 0 for + * IP-in-IP is used to distinguish population modes. If + * mbuf->l2_len == 0, the upper layer has enabled UDP and IP + * hardware offload flags; the inner L3 offset is then + * outer_l2_len + outer_l3_len. + */ + if ((ol_flags & HINIC3_PKT_TX_TUNNEL_IPIP) == HINIC3_PKT_TX_TUNNEL_IPIP) { + if (mbuf->l2_len == 0) + *inner_l3_offset = mbuf->outer_l2_len + mbuf->outer_l3_len; + else + *inner_l3_offset = mbuf->l2_len; + } else if ((ol_flags & HINIC3_PKT_TX_OUTER_IP_CKSUM) || + (ol_flags & HINIC3_PKT_TX_OUTER_UDP_CKSUM) || + (ol_flags & HINIC3_PKT_TX_TCP_SEG)) { /* * For this senmatic, l2_len of mbuf means - * len(out_udp + vxlan + in_eth). + * len(out_udp + vxlan/geneve + in_eth). */ *inner_l3_offset = mbuf->l2_len + mbuf->outer_l2_len + mbuf->outer_l3_len; + if (!HINIC3_IS_SP230_NIC(nic_dev)) + /* Process the outer UDP pseudo-header checksum */ + hinic3_calculate_outer_phdr_checksum(mbuf, mbuf->outer_l2_len); } else { /* * For this senmatic, l2_len of mbuf means @@ -420,6 +584,9 @@ hinic3_tx_offload_pkt_prepare(struct hinic3_nic_dev *nic_dev, struct rte_mbuf *m /* For non-tunnel type pkts. */ *inner_l3_offset = mbuf->l2_len; } + if (!HINIC3_IS_SP230_NIC(nic_dev)) + /* Process the pseudo-header checksum */ + hinic3_calculate_phdr_checksum(mbuf, *inner_l3_offset); return 0; } diff --git a/drivers/net/hinic3/hinic3_tx.h b/drivers/net/hinic3/hinic3_tx.h index 10ea174ea2..2fafb400b5 100644 --- a/drivers/net/hinic3/hinic3_tx.h +++ b/drivers/net/hinic3/hinic3_tx.h @@ -408,6 +408,17 @@ struct __rte_cache_aligned hinic3_txq { uint64_t prof_tx_end_tsc; #endif }; +#define IPV4_VERSION 4 +#define IPV6_VERSION 6 +#define FIXED_EXT_HDR_LEN 8 +#define UNIT_BYTES_U 8 +#define UNIT_BYTES_AH 4 +#define IPV6_MAX_EXT_HDRS 9 + +struct hinic3_ipv6_ext_hdr { + uint8_t next_hdr; + uint8_t len; +}; void hinic3_flush_txqs(struct hinic3_nic_dev *nic_dev); void hinic3_free_txq_mbufs(struct hinic3_txq *txq); -- 2.33.0
