This commit adds the AVX512 implementation of the pop_vlan action. The implementation here is auto-validated by the miniflow extract autovalidator, hence its correctness can be easily tested and verified.
Signed-off-by: Emma Finn <emma.f...@intel.com> --- lib/odp-execute-avx512.c | 77 ++++++++++++++++++++++++++++++++++++++- lib/odp-execute-private.c | 2 +- lib/odp-execute-private.h | 2 +- 3 files changed, 78 insertions(+), 3 deletions(-) diff --git a/lib/odp-execute-avx512.c b/lib/odp-execute-avx512.c index 0765b8e3d..7a21a60b1 100644 --- a/lib/odp-execute-avx512.c +++ b/lib/odp-execute-avx512.c @@ -14,6 +14,11 @@ * limitations under the License. */ +#ifdef __x86_64__ +/* Sparse cannot handle the AVX512 instructions. */ +#if !defined(__CHECKER__) + + #include <config.h> #include <errno.h> @@ -25,6 +30,71 @@ #include "immintrin.h" +VLOG_DEFINE_THIS_MODULE(odp_execute_avx512); +BUILD_ASSERT_DECL(offsetof(struct dp_packet, l2_5_ofs) + + MEMBER_SIZEOF(struct dp_packet, l2_5_ofs) == + offsetof(struct dp_packet, l3_ofs)); + +BUILD_ASSERT_DECL(offsetof(struct dp_packet, l3_ofs) + + MEMBER_SIZEOF(struct dp_packet, l3_ofs) == + offsetof(struct dp_packet, l4_ofs)); + +static inline void ALWAYS_INLINE +avx512_dp_packet_resize_l2(struct dp_packet *b, int increment) +{ + /* update packet size/data pointers */ + dp_packet_set_data(b, (char *) dp_packet_data(b) - increment); + dp_packet_set_size(b, dp_packet_size(b) + increment); + + /* Increment u16 packet offset values */ + const __m128i v_zeros = _mm_setzero_si128(); + const __m128i v_u16_max = _mm_cmpeq_epi16(v_zeros, v_zeros); + + /* Only these lanes can be incremented for push-VLAN action. */ + const uint8_t k_lanes = 0b1110; + __m128i v_offset = _mm_set1_epi16(VLAN_HEADER_LEN); + + /* Load packet and compare with UINT16_MAX */ + void *adjust_ptr = &b->l2_pad_size; + __m128i v_adjust_src = _mm_loadu_si128(adjust_ptr); + __mmask8 k_cmp = _mm_mask_cmpneq_epu16_mask(k_lanes, v_adjust_src, + v_u16_max); + + /* Add VLAN_HEADER_LEN using compare mask, store results. */ + __m128i v_adjust_wip = _mm_mask_sub_epi16(v_adjust_src, k_cmp, + v_adjust_src, v_offset); + _mm_storeu_si128(adjust_ptr, v_adjust_wip); + +} + +static inline void ALWAYS_INLINE +avx512_eth_pop_vlan(struct dp_packet *packet) +{ + struct vlan_eth_header *veh = dp_packet_eth(packet); + + if (veh && dp_packet_size(packet) >= sizeof *veh && + eth_type_vlan(veh->veth_type)) { + + __m128i v_ether = _mm_loadu_si128((void *) veh); + __m128i v_realign = _mm_alignr_epi8(v_ether, _mm_setzero_si128(), + 16 - VLAN_HEADER_LEN); + _mm_storeu_si128((void *) veh, v_realign); + avx512_dp_packet_resize_l2(packet, -VLAN_HEADER_LEN); + + } +} + +static void +action_avx512_pop_vlan(void *dp OVS_UNUSED, struct dp_packet_batch *batch, + const struct nlattr *a OVS_UNUSED, + bool should_steal OVS_UNUSED) +{ + struct dp_packet *packet; + + DP_PACKET_BATCH_FOR_EACH (i, packet, batch) { + avx512_eth_pop_vlan(packet); + } +} /* Probe functions to check ISA requirements. */ static int32_t @@ -62,8 +132,13 @@ action_avx512_probe(void) int32_t -action_avx512_init(void) +action_avx512_init(struct odp_execute_action_impl *self) { avx512_isa_probe(0); + self->funcs[OVS_ACTION_ATTR_POP_VLAN] = action_avx512_pop_vlan; + return 0; } + +#endif +#endif diff --git a/lib/odp-execute-private.c b/lib/odp-execute-private.c index 19a04f518..7c58d90d2 100644 --- a/lib/odp-execute-private.c +++ b/lib/odp-execute-private.c @@ -52,7 +52,7 @@ static struct odp_execute_action_impl action_impls[] = { .available = 1, .name = "avx512", .probe = action_avx512_probe, - .init_func = NULL, + .init_func = action_avx512_init, }, #endif }; diff --git a/lib/odp-execute-private.h b/lib/odp-execute-private.h index 4c09bee63..5ba2868bf 100644 --- a/lib/odp-execute-private.h +++ b/lib/odp-execute-private.h @@ -102,7 +102,7 @@ int32_t odp_execute_action_set(const char *name, int32_t odp_action_scalar_init(struct odp_execute_action_impl *self); /* Init function for the optimized with AVX512 actions. */ -int32_t action_avx512_init(void); +int32_t action_avx512_init(struct odp_execute_action_impl *self); /* Probe function to check ISA requirements. */ int32_t action_avx512_probe(void); -- 2.25.1 _______________________________________________ dev mailing list d...@openvswitch.org https://mail.openvswitch.org/mailman/listinfo/ovs-dev