From: Dmitry Eremin-Solenikov <dmitry.ereminsoleni...@linaro.org> To ease adding IPv6/IPcomp/etc modularize IPsec implementation, refactoring out functions handling ESP/AH and header parsing/tunneling.
Signed-off-by: Dmitry Eremin-Solenikov <dmitry.ereminsoleni...@linaro.org> --- /** Email created from pull request 304 (lumag:ipsec-ipv6-2) ** https://github.com/Linaro/odp/pull/304 ** Patch: https://github.com/Linaro/odp/pull/304.patch ** Base sha: 4cb02e1caccb9179575e95448fd46979e17d0905 ** Merge commit sha: 6f6906b608f0626d119b255b3733a4a0863aa8d1 **/ platform/linux-generic/odp_ipsec.c | 1089 ++++++++++++++++++++---------------- 1 file changed, 597 insertions(+), 492 deletions(-) diff --git a/platform/linux-generic/odp_ipsec.c b/platform/linux-generic/odp_ipsec.c index b17e4cd7b..6ce5bc781 100644 --- a/platform/linux-generic/odp_ipsec.c +++ b/platform/linux-generic/odp_ipsec.c @@ -123,8 +123,8 @@ static inline int _odp_ipv4_csum(odp_packet_t pkt, return 0; } -/** @internal Checksum offset in IPv4 header */ -#define _ODP_IPV4HDR_CSUM_OFFSET 10 +#define _ODP_IPV4HDR_CSUM_OFFSET ODP_OFFSETOF(_odp_ipv4hdr_t, chksum) +#define _ODP_IPV4HDR_PROTO_OFFSET ODP_OFFSETOF(_odp_ipv4hdr_t, proto) /** * Calculate and fill in IPv4 checksum @@ -158,7 +158,7 @@ static inline int _odp_ipv4_csum_update(odp_packet_t pkt) 2, &chksum); } -#define ipv4_hdr_len(ip) (_ODP_IPV4HDR_IHL(ip->ver_ihl) * 4) +#define ipv4_hdr_len(ip) (_ODP_IPV4HDR_IHL((ip)->ver_ihl) * 4) static inline void ipv4_adjust_len(_odp_ipv4hdr_t *ip, int adj) { @@ -218,200 +218,310 @@ static inline odp_pktio_parser_layer_t parse_layer(odp_ipsec_proto_layer_t l) return ODP_PKTIO_PARSER_LAYER_NONE; } -static ipsec_sa_t *ipsec_in_single(odp_packet_t pkt, - odp_ipsec_sa_t sa, - odp_packet_t *pkt_out, - odp_ipsec_op_status_t *status) -{ - ipsec_sa_t *ipsec_sa = NULL; - uint32_t ip_offset = odp_packet_l3_offset(pkt); - _odp_ipv4hdr_t *ip = odp_packet_l3_ptr(pkt, NULL); - uint16_t ip_hdr_len = ipv4_hdr_len(ip); - odp_crypto_packet_op_param_t param; - int rc; +typedef struct { + _odp_ipv4hdr_t *ip; unsigned stats_length; - uint16_t ipsec_offset; /**< Offset of IPsec header from - buffer start */ - uint8_t iv[IPSEC_MAX_IV_LEN]; /**< ESP IV storage */ - ipsec_aad_t aad; /**< AAD, note ESN is not fully supported */ - unsigned hdr_len; /**< Length of IPsec headers */ - unsigned trl_len; /**< Length of IPsec trailers */ - uint8_t ip_tos; /**< Saved IP TOS value */ - uint8_t ip_ttl; /**< Saved IP TTL value */ - uint16_t ip_frag_offset; /**< Saved IP flags value */ - odp_crypto_packet_result_t crypto; /**< Crypto operation result */ - odp_packet_hdr_t *pkt_hdr; + uint16_t ip_offset; + uint16_t ip_hdr_len; + uint16_t ip_tot_len; + union { + struct { + uint16_t ip_df; + uint8_t ip_tos; + } out_tunnel; + struct { + uint16_t hdr_len; + uint16_t trl_len; + } in; + }; + union { + struct { + uint8_t tos; + uint8_t ttl; + uint16_t frag_offset; + } ah_ipv4; + }; + ipsec_aad_t aad; + uint8_t iv[IPSEC_MAX_IV_LEN]; +} ipsec_state_t; + +static int ipsec_parse_ipv4(ipsec_state_t *state) +{ + if (_ODP_IPV4HDR_IS_FRAGMENT(odp_be_to_cpu_16(state->ip->frag_offset))) + return -1; - ODP_ASSERT(ODP_PACKET_OFFSET_INVALID != ip_offset); - ODP_ASSERT(NULL != ip); + state->ip_hdr_len = ipv4_hdr_len(state->ip); + state->ip_tot_len = odp_be_to_cpu_16(state->ip->tot_len); - ip_tos = 0; - ip_ttl = 0; - ip_frag_offset = 0; + return 0; +} - /* Initialize parameters block */ - memset(¶m, 0, sizeof(param)); +static inline ipsec_sa_t *ipsec_get_sa(odp_ipsec_sa_t sa, + odp_ipsec_protocol_t proto, + uint32_t spi, + void *dst_addr, + odp_ipsec_op_status_t *status) +{ + ipsec_sa_t *ipsec_sa; + + if (ODP_IPSEC_SA_INVALID == sa) { + ipsec_sa_lookup_t lookup; + + lookup.proto = proto; + lookup.spi = spi; + lookup.dst_addr = dst_addr; + + ipsec_sa = _odp_ipsec_sa_lookup(&lookup); + if (NULL == ipsec_sa) { + status->error.sa_lookup = 1; + return NULL; + } + } else { + ipsec_sa = _odp_ipsec_sa_use(sa); + ODP_ASSERT(NULL != ipsec_sa); + if (ipsec_sa->proto != proto || + ipsec_sa->spi != spi) { + status->error.proto = 1; + return ipsec_sa; + } + } + + return ipsec_sa; +} - ipsec_offset = ip_offset + ip_hdr_len; +static int ipsec_in_iv(odp_packet_t pkt, + ipsec_state_t *state, + ipsec_sa_t *ipsec_sa, + uint16_t iv_offset) +{ + memcpy(state->iv, ipsec_sa->salt, ipsec_sa->salt_length); + if (odp_packet_copy_to_mem(pkt, + iv_offset, + ipsec_sa->esp_iv_len, + state->iv + ipsec_sa->salt_length) < 0) + return -1; - if (odp_be_to_cpu_16(ip->tot_len) + ip_offset > odp_packet_len(pkt)) { + if (ipsec_sa->aes_ctr_iv) { + state->iv[12] = 0; + state->iv[13] = 0; + state->iv[14] = 0; + state->iv[15] = 1; + } + + return 0; +} + +static int ipsec_in_esp(odp_packet_t *pkt, + ipsec_state_t *state, + ipsec_sa_t **_ipsec_sa, + odp_ipsec_sa_t sa, + odp_crypto_packet_op_param_t *param, + odp_ipsec_op_status_t *status) +{ + _odp_esphdr_t esp; + uint16_t ipsec_offset; + ipsec_sa_t *ipsec_sa; + + ipsec_offset = state->ip_offset + state->ip_hdr_len; + + if (odp_packet_copy_to_mem(*pkt, ipsec_offset, + sizeof(esp), &esp) < 0) { status->error.alg = 1; - goto err; + return -1; } - if (_ODP_IPV4HDR_IS_FRAGMENT(odp_be_to_cpu_16(ip->frag_offset))) { - status->error.proto = 1; - goto err; + ipsec_sa = ipsec_get_sa(sa, ODP_IPSEC_ESP, + odp_be_to_cpu_32(esp.spi), + &state->ip->dst_addr, status); + *_ipsec_sa = ipsec_sa; + if (status->error.all) + return -1; + + if (ipsec_in_iv(*pkt, state, ipsec_sa, + ipsec_offset + _ODP_ESPHDR_LEN) < 0) { + status->error.alg = 1; + return -1; } - /* Check IP header for IPSec protocols and look it up */ - if (_ODP_IPPROTO_ESP == ip->proto) { - _odp_esphdr_t esp; + state->in.hdr_len = _ODP_ESPHDR_LEN + ipsec_sa->esp_iv_len; + state->in.trl_len = _ODP_ESPTRL_LEN + ipsec_sa->icv_len; - if (odp_packet_copy_to_mem(pkt, ipsec_offset, - sizeof(esp), &esp) < 0) { - status->error.alg = 1; - goto err; - } + param->cipher_range.offset = ipsec_offset + state->in.hdr_len; + param->cipher_range.length = state->ip_tot_len - + state->ip_hdr_len - + state->in.hdr_len - + ipsec_sa->icv_len; + param->override_iv_ptr = state->iv; - if (ODP_IPSEC_SA_INVALID == sa) { - ipsec_sa_lookup_t lookup; + state->aad.spi = esp.spi; + state->aad.seq_no = esp.seq_no; - lookup.proto = ODP_IPSEC_ESP; - lookup.spi = odp_be_to_cpu_32(esp.spi); - lookup.dst_addr = &ip->dst_addr; + param->aad.ptr = (uint8_t *)&state->aad; - ipsec_sa = _odp_ipsec_sa_lookup(&lookup); - if (NULL == ipsec_sa) { - status->error.sa_lookup = 1; - goto err; - } - } else { - ipsec_sa = _odp_ipsec_sa_use(sa); - ODP_ASSERT(NULL != ipsec_sa); - if (ipsec_sa->proto != ODP_IPSEC_ESP || - ipsec_sa->spi != odp_be_to_cpu_32(esp.spi)) { - status->error.proto = 1; - goto err; - } - } + param->auth_range.offset = ipsec_offset; + param->auth_range.length = state->ip_tot_len - + state->ip_hdr_len - + ipsec_sa->icv_len; + param->hash_result_offset = state->ip_offset + + state->ip_tot_len - + ipsec_sa->icv_len; - memcpy(iv, ipsec_sa->salt, ipsec_sa->salt_length); - if (odp_packet_copy_to_mem(pkt, - ipsec_offset + _ODP_ESPHDR_LEN, - ipsec_sa->esp_iv_len, - iv + ipsec_sa->salt_length) < 0) { - status->error.alg = 1; - goto err; - } + state->stats_length = param->cipher_range.length; - if (ipsec_sa->aes_ctr_iv) { - iv[12] = 0; - iv[13] = 0; - iv[14] = 0; - iv[15] = 1; - } + return 0; +} - hdr_len = _ODP_ESPHDR_LEN + ipsec_sa->esp_iv_len; - trl_len = _ODP_ESPTRL_LEN + ipsec_sa->icv_len; +static int ipsec_in_esp_post(odp_packet_t pkt, + ipsec_state_t *state) +{ + _odp_esptrl_t esptrl; + uint32_t esptrl_offset = state->ip_offset + + state->ip_tot_len - + state->in.trl_len; + + if (odp_packet_copy_to_mem(pkt, esptrl_offset, + sizeof(esptrl), &esptrl) < 0 || + state->ip_offset + esptrl.pad_len > esptrl_offset || + _odp_packet_cmp_data(pkt, esptrl_offset - esptrl.pad_len, + ipsec_padding, esptrl.pad_len) != 0) + return -1; - param.cipher_range.offset = ipsec_offset + hdr_len; - param.cipher_range.length = odp_be_to_cpu_16(ip->tot_len) - - ip_hdr_len - - hdr_len - - ipsec_sa->icv_len; - param.override_iv_ptr = iv; + state->ip->proto = esptrl.next_header; + state->in.trl_len += esptrl.pad_len; - aad.spi = esp.spi; - aad.seq_no = esp.seq_no; + return 0; +} - param.aad.ptr = (uint8_t *)&aad; +static int ipsec_in_ah(odp_packet_t *pkt, + ipsec_state_t *state, + ipsec_sa_t **_ipsec_sa, + odp_ipsec_sa_t sa, + odp_crypto_packet_op_param_t *param, + odp_ipsec_op_status_t *status) +{ + _odp_ahhdr_t ah; + uint16_t ipsec_offset; + ipsec_sa_t *ipsec_sa; - param.auth_range.offset = ipsec_offset; - param.auth_range.length = odp_be_to_cpu_16(ip->tot_len) - - ip_hdr_len - - ipsec_sa->icv_len; - param.hash_result_offset = ip_offset + - odp_be_to_cpu_16(ip->tot_len) - - ipsec_sa->icv_len; + ipsec_offset = state->ip_offset + state->ip_hdr_len; - stats_length = param.cipher_range.length; - } else if (_ODP_IPPROTO_AH == ip->proto) { - _odp_ahhdr_t ah; + if (odp_packet_copy_to_mem(*pkt, ipsec_offset, + sizeof(ah), &ah) < 0) { + status->error.alg = 1; + return -1; + } - if (odp_packet_copy_to_mem(pkt, ipsec_offset, - sizeof(ah), &ah) < 0) { - status->error.alg = 1; - goto err; - } + ipsec_sa = ipsec_get_sa(sa, ODP_IPSEC_AH, + odp_be_to_cpu_32(ah.spi), + &state->ip->dst_addr, status); + *_ipsec_sa = ipsec_sa; + if (status->error.all) + return -1; - if (ODP_IPSEC_SA_INVALID == sa) { - ipsec_sa_lookup_t lookup; + if (ipsec_in_iv(*pkt, state, ipsec_sa, + ipsec_offset + _ODP_AHHDR_LEN) < 0) { + status->error.alg = 1; + return -1; + } - lookup.proto = ODP_IPSEC_AH; - lookup.spi = odp_be_to_cpu_32(ah.spi); - lookup.dst_addr = &ip->dst_addr; + param->override_iv_ptr = state->iv; - ipsec_sa = _odp_ipsec_sa_lookup(&lookup); - if (NULL == ipsec_sa) { - status->error.sa_lookup = 1; - goto err; - } - } else { - ipsec_sa = _odp_ipsec_sa_use(sa); - ODP_ASSERT(NULL != ipsec_sa); - if (ipsec_sa->proto != ODP_IPSEC_AH || - ipsec_sa->spi != odp_be_to_cpu_32(ah.spi)) { - status->error.proto = 1; - goto err; - } - } + state->in.hdr_len = (ah.ah_len + 2) * 4; + state->in.trl_len = 0; - memcpy(iv, ipsec_sa->salt, ipsec_sa->salt_length); - if (odp_packet_copy_to_mem(pkt, - ipsec_offset + _ODP_AHHDR_LEN, - ipsec_sa->esp_iv_len, - iv + ipsec_sa->salt_length) < 0) { - status->error.alg = 1; - goto err; - } - param.override_iv_ptr = iv; + /* Save everything to context */ + state->ah_ipv4.tos = state->ip->tos; + state->ah_ipv4.frag_offset = state->ip->frag_offset; + state->ah_ipv4.ttl = state->ip->ttl; + + /* FIXME: zero copy of header, passing it to crypto! */ + /* + * If authenticating, zero the mutable fields build the request + */ + state->ip->chksum = 0; + state->ip->tos = 0; + state->ip->frag_offset = 0; + state->ip->ttl = 0; - hdr_len = (ah.ah_len + 2) * 4; - trl_len = 0; + state->aad.spi = ah.spi; + state->aad.seq_no = ah.seq_no; - /* Save everything to context */ - ip_tos = ip->tos; - ip_frag_offset = odp_be_to_cpu_16(ip->frag_offset); - ip_ttl = ip->ttl; + param->aad.ptr = (uint8_t *)&state->aad; - /* FIXME: zero copy of header, passing it to crypto! */ - /* - * If authenticating, zero the mutable fields build the request - */ - ip->chksum = 0; - ip->tos = 0; - ip->frag_offset = 0; - ip->ttl = 0; + param->auth_range.offset = state->ip_offset; + param->auth_range.length = state->ip_tot_len; + param->hash_result_offset = ipsec_offset + _ODP_AHHDR_LEN + + ipsec_sa->esp_iv_len; - aad.spi = ah.spi; - aad.seq_no = ah.seq_no; + state->stats_length = param->auth_range.length; - param.aad.ptr = (uint8_t *)&aad; + return 0; +} + +static int ipsec_in_ah_post(odp_packet_t pkt, + ipsec_state_t *state) +{ + _odp_ahhdr_t ah; + uint16_t ipsec_offset; - param.auth_range.offset = ip_offset; - param.auth_range.length = odp_be_to_cpu_16(ip->tot_len); - param.hash_result_offset = ipsec_offset + _ODP_AHHDR_LEN + - ipsec_sa->esp_iv_len; + ipsec_offset = state->ip_offset + state->ip_hdr_len; - stats_length = param.auth_range.length; + if (odp_packet_copy_to_mem(pkt, ipsec_offset, + sizeof(ah), &ah) < 0) + return -1; + + state->ip->proto = ah.next_header; + + /* Restore mutable fields */ + state->ip->ttl = state->ah_ipv4.ttl; + state->ip->tos = state->ah_ipv4.tos; + state->ip->frag_offset = state->ah_ipv4.frag_offset; + + return 0; +} + +static ipsec_sa_t *ipsec_in_single(odp_packet_t pkt, + odp_ipsec_sa_t sa, + odp_packet_t *pkt_out, + odp_ipsec_op_status_t *status) +{ + ipsec_state_t state; + ipsec_sa_t *ipsec_sa = NULL; + odp_crypto_packet_op_param_t param; + int rc; + odp_crypto_packet_result_t crypto; /**< Crypto operation result */ + odp_packet_hdr_t *pkt_hdr; + + state.ip_offset = odp_packet_l3_offset(pkt); + ODP_ASSERT(ODP_PACKET_OFFSET_INVALID != state.ip_offset); + + state.ip = odp_packet_l3_ptr(pkt, NULL); + ODP_ASSERT(NULL != state.ip); + + /* Initialize parameters block */ + memset(¶m, 0, sizeof(param)); + + rc = ipsec_parse_ipv4(&state); + if (rc < 0 || + state.ip_tot_len + state.ip_offset > odp_packet_len(pkt)) { + status->error.alg = 1; + goto err; + } + + /* Check IP header for IPSec protocols and look it up */ + if (_ODP_IPPROTO_ESP == state.ip->proto) { + rc = ipsec_in_esp(&pkt, &state, &ipsec_sa, sa, ¶m, status); + } else if (_ODP_IPPROTO_AH == state.ip->proto) { + rc = ipsec_in_ah(&pkt, &state, &ipsec_sa, sa, ¶m, status); } else { status->error.proto = 1; goto err; } + if (rc < 0) + goto err; if (_odp_ipsec_sa_replay_precheck(ipsec_sa, - odp_be_to_cpu_32(aad.seq_no), + odp_be_to_cpu_32(state.aad.seq_no), status) < 0) goto err; @@ -450,70 +560,30 @@ static ipsec_sa_t *ipsec_in_single(odp_packet_t pkt, goto err; } - if (_odp_ipsec_sa_stats_update(ipsec_sa, stats_length, status) < 0) + if (_odp_ipsec_sa_stats_update(ipsec_sa, + state.stats_length, + status) < 0) goto err; if (_odp_ipsec_sa_replay_update(ipsec_sa, - odp_be_to_cpu_32(aad.seq_no), + odp_be_to_cpu_32(state.aad.seq_no), status) < 0) goto err; - ip_offset = odp_packet_l3_offset(pkt); - ip = odp_packet_l3_ptr(pkt, NULL); - ip_hdr_len = ipv4_hdr_len(ip); - - if (_ODP_IPPROTO_ESP == ip->proto) { - /* - * Finish cipher by finding ESP trailer and processing - */ - _odp_esptrl_t esptrl; - uint32_t esptrl_offset = ip_offset + - odp_be_to_cpu_16(ip->tot_len) - - trl_len; - - if (odp_packet_copy_to_mem(pkt, esptrl_offset, - sizeof(esptrl), &esptrl) < 0) { - status->error.proto = 1; - goto err; - } - - if (ip_offset + esptrl.pad_len > esptrl_offset) { - status->error.proto = 1; - goto err; - } - - if (_odp_packet_cmp_data(pkt, esptrl_offset - esptrl.pad_len, - ipsec_padding, esptrl.pad_len) != 0) { - status->error.proto = 1; - goto err; - } + state.ip = odp_packet_l3_ptr(pkt, NULL); - ip->proto = esptrl.next_header; - trl_len += esptrl.pad_len; - } else if (_ODP_IPPROTO_AH == ip->proto) { - /* - * Finish auth - */ - _odp_ahhdr_t ah; - - if (odp_packet_copy_to_mem(pkt, ipsec_offset, - sizeof(ah), &ah) < 0) { - status->error.alg = 1; - goto err; - } - - ip->proto = ah.next_header; - - /* Restore mutable fields */ - ip->ttl = ip_ttl; - ip->tos = ip_tos; - ip->frag_offset = odp_cpu_to_be_16(ip_frag_offset); - } else { + if (ODP_IPSEC_ESP == ipsec_sa->proto) + rc = ipsec_in_esp_post(pkt, &state); + else if (ODP_IPSEC_AH == ipsec_sa->proto) + rc = ipsec_in_ah_post(pkt, &state); + else + rc = -1; + if (rc < 0) { status->error.proto = 1; goto err; } - if (odp_packet_trunc_tail(&pkt, trl_len, NULL, NULL) < 0) { + if (odp_packet_trunc_tail(&pkt, state.in.trl_len, NULL, NULL) < 0) { status->error.alg = 1; goto err; } @@ -521,32 +591,36 @@ static ipsec_sa_t *ipsec_in_single(odp_packet_t pkt, if (ODP_IPSEC_MODE_TUNNEL == ipsec_sa->mode) { /* We have a tunneled IPv4 packet, strip outer and IPsec * headers */ - odp_packet_move_data(pkt, ip_hdr_len + hdr_len, 0, - ip_offset); - if (odp_packet_trunc_head(&pkt, ip_hdr_len + hdr_len, + odp_packet_move_data(pkt, state.ip_hdr_len + state.in.hdr_len, + 0, + state.ip_offset); + if (odp_packet_trunc_head(&pkt, state.ip_hdr_len + + state.in.hdr_len, NULL, NULL) < 0) { status->error.alg = 1; goto err; } + + if (odp_packet_len(pkt) > sizeof(*state.ip)) { + state.ip = odp_packet_l3_ptr(pkt, NULL); + state.ip->ttl -= ipsec_sa->dec_ttl; + _odp_ipv4_csum_update(pkt); + } } else { - odp_packet_move_data(pkt, hdr_len, 0, - ip_offset + ip_hdr_len); - if (odp_packet_trunc_head(&pkt, hdr_len, + odp_packet_move_data(pkt, state.in.hdr_len, 0, + state.ip_offset + state.ip_hdr_len); + if (odp_packet_trunc_head(&pkt, state.in.hdr_len, NULL, NULL) < 0) { status->error.alg = 1; goto err; } - } - - /* Finalize the IPv4 header */ - if (odp_packet_len(pkt) > sizeof(*ip)) { - ip = odp_packet_l3_ptr(pkt, NULL); - - if (ODP_IPSEC_MODE_TRANSPORT == ipsec_sa->mode) - ipv4_adjust_len(ip, -(hdr_len + trl_len)); - ip->ttl -= ipsec_sa->dec_ttl; - _odp_ipv4_csum_update(pkt); + if (odp_packet_len(pkt) > sizeof(*state.ip)) { + state.ip = odp_packet_l3_ptr(pkt, NULL); + ipv4_adjust_len(state.ip, + -(state.in.hdr_len + state.in.trl_len)); + _odp_ipv4_csum_update(pkt); + } } pkt_hdr = odp_packet_hdr(pkt); @@ -554,7 +628,7 @@ static ipsec_sa_t *ipsec_in_single(odp_packet_t pkt, packet_parse_reset(pkt_hdr); packet_parse_l3_l4(pkt_hdr, parse_layer(ipsec_config.inbound.parse), - ip_offset, _ODP_ETHTYPE_IPV4); + state.ip_offset, _ODP_ETHTYPE_IPV4); *pkt_out = pkt; @@ -577,317 +651,353 @@ uint32_t ipsec_seq_no(ipsec_sa_t *ipsec_sa) } /* Helper for calculating encode length using data length and block size */ -#define ESP_ENCODE_LEN(x, b) ((((x) + ((b) - 1)) / (b)) * (b)) +#define IPSEC_PAD_LEN(x, b) ((((x) + ((b) - 1)) / (b)) * (b)) -static ipsec_sa_t *ipsec_out_single(odp_packet_t pkt, - odp_ipsec_sa_t sa, - odp_packet_t *pkt_out, - odp_ipsec_out_opt_t *opt ODP_UNUSED, - odp_ipsec_op_status_t *status) +static int ipsec_out_tunnel_parse_ipv4(ipsec_state_t *state, + ipsec_sa_t *ipsec_sa) { - ipsec_sa_t *ipsec_sa = NULL; - uint32_t ip_offset = odp_packet_l3_offset(pkt); - _odp_ipv4hdr_t *ip = odp_packet_l3_ptr(pkt, NULL); - uint16_t ip_hdr_len = ipv4_hdr_len(ip); - odp_crypto_packet_op_param_t param; - unsigned stats_length; - int rc; - uint16_t ipsec_offset; /**< Offset of IPsec header from - buffer start */ - uint8_t iv[IPSEC_MAX_IV_LEN]; /**< ESP IV storage */ - ipsec_aad_t aad; /**< AAD, note ESN is not fully supported */ - unsigned hdr_len; /**< Length of IPsec headers */ - unsigned trl_len; /**< Length of IPsec trailers */ - uint8_t ip_tos; /**< Saved IP TOS value */ - uint8_t ip_ttl; /**< Saved IP TTL value */ - uint16_t ip_frag_offset; /**< Saved IP flags value */ - odp_crypto_packet_result_t crypto; /**< Crypto operation result */ - odp_packet_hdr_t *pkt_hdr; + _odp_ipv4hdr_t *ipv4hdr = state->ip; + uint16_t flags = odp_be_to_cpu_16(ipv4hdr->frag_offset); - ODP_ASSERT(ODP_PACKET_OFFSET_INVALID != ip_offset); - ODP_ASSERT(NULL != ip); + ipv4hdr->ttl -= ipsec_sa->dec_ttl; + state->out_tunnel.ip_tos = ipv4hdr->tos; + state->out_tunnel.ip_df = _ODP_IPV4HDR_FLAGS_DONT_FRAG(flags); - ip_tos = 0; - ip_ttl = 0; - ip_frag_offset = 0; + return 0; +} - ipsec_sa = _odp_ipsec_sa_use(sa); - ODP_ASSERT(NULL != ipsec_sa); +static int ipsec_out_tunnel_ipv4(odp_packet_t *pkt, + ipsec_state_t *state, + ipsec_sa_t *ipsec_sa) +{ + _odp_ipv4hdr_t out_ip; + uint16_t flags; + + out_ip.ver_ihl = 0x45; + if (ipsec_sa->copy_dscp) + out_ip.tos = state->out_tunnel.ip_tos; + else + out_ip.tos = (state->out_tunnel.ip_tos & + ~_ODP_IP_TOS_DSCP_MASK) | + (ipsec_sa->out.tun_dscp << + _ODP_IP_TOS_DSCP_SHIFT); + state->ip_tot_len = odp_packet_len(*pkt) - state->ip_offset; + state->ip_tot_len += _ODP_IPV4HDR_LEN; + + out_ip.tot_len = odp_cpu_to_be_16(state->ip_tot_len); + /* No need to convert to BE: ID just should not be duplicated */ + out_ip.id = odp_atomic_fetch_add_u32(&ipsec_sa->out.tun_hdr_id, + 1); + if (ipsec_sa->copy_df) + flags = state->out_tunnel.ip_df; + else + flags = ((uint16_t)ipsec_sa->out.tun_df) << 14; + out_ip.frag_offset = odp_cpu_to_be_16(flags); + out_ip.ttl = ipsec_sa->out.tun_ttl; + out_ip.proto = _ODP_IPPROTO_IPIP; + /* Will be filled later by packet checksum update */ + out_ip.chksum = 0; + out_ip.src_addr = ipsec_sa->out.tun_src_ip; + out_ip.dst_addr = ipsec_sa->out.tun_dst_ip; + + if (odp_packet_extend_head(pkt, _ODP_IPV4HDR_LEN, + NULL, NULL) < 0) + return -1; - /* Initialize parameters block */ - memset(¶m, 0, sizeof(param)); + odp_packet_move_data(*pkt, 0, _ODP_IPV4HDR_LEN, state->ip_offset); - if (ODP_IPSEC_MODE_TRANSPORT == ipsec_sa->mode && - _ODP_IPV4HDR_IS_FRAGMENT(odp_be_to_cpu_16(ip->frag_offset))) { - status->error.alg = 1; - goto err; - } + odp_packet_copy_from_mem(*pkt, state->ip_offset, + _ODP_IPV4HDR_LEN, &out_ip); - if (odp_be_to_cpu_16(ip->tot_len) + ip_offset > odp_packet_len(pkt)) { - status->error.alg = 1; - goto err; - } + odp_packet_l4_offset_set(*pkt, state->ip_offset + _ODP_IPV4HDR_LEN); - if (ODP_IPSEC_MODE_TUNNEL == ipsec_sa->mode) { - _odp_ipv4hdr_t out_ip; - uint16_t tot_len; + state->ip = odp_packet_l3_ptr(*pkt, NULL); + state->ip_hdr_len = _ODP_IPV4HDR_LEN; - ip->ttl -= ipsec_sa->dec_ttl; + return 0; +} - out_ip.ver_ihl = 0x45; - if (ipsec_sa->copy_dscp) - out_ip.tos = ip->tos; - else - out_ip.tos = (ip->tos & ~_ODP_IP_TOS_DSCP_MASK) | - (ipsec_sa->out.tun_dscp << - _ODP_IP_TOS_DSCP_SHIFT); - tot_len = odp_be_to_cpu_16(ip->tot_len) + _ODP_IPV4HDR_LEN; - out_ip.tot_len = odp_cpu_to_be_16(tot_len); - /* No need to convert to BE: ID just should not be duplicated */ - out_ip.id = odp_atomic_fetch_add_u32(&ipsec_sa->out.tun_hdr_id, - 1); - if (ipsec_sa->copy_df) - out_ip.frag_offset = ip->frag_offset & 0x4000; - else - out_ip.frag_offset = - ((uint16_t)ipsec_sa->out.tun_df) << 14; - out_ip.ttl = ipsec_sa->out.tun_ttl; - out_ip.proto = _ODP_IPV4; - /* Will be filled later by packet checksum update */ - out_ip.chksum = 0; - out_ip.src_addr = ipsec_sa->out.tun_src_ip; - out_ip.dst_addr = ipsec_sa->out.tun_dst_ip; - - if (odp_packet_extend_head(&pkt, _ODP_IPV4HDR_LEN, - NULL, NULL) < 0) { - status->error.alg = 1; - goto err; +static int ipsec_out_iv(ipsec_state_t *state, + ipsec_sa_t *ipsec_sa) +{ + if (ipsec_sa->use_counter_iv) { + uint64_t ctr; + + /* Both GCM and CTR use 8-bit counters */ + ODP_ASSERT(sizeof(ctr) == ipsec_sa->esp_iv_len); + + ctr = odp_atomic_fetch_add_u64(&ipsec_sa->out.counter, + 1); + /* Check for overrun */ + if (ctr == 0) + return -1; + + memcpy(state->iv, ipsec_sa->salt, ipsec_sa->salt_length); + memcpy(state->iv + ipsec_sa->salt_length, &ctr, + ipsec_sa->esp_iv_len); + + if (ipsec_sa->aes_ctr_iv) { + state->iv[12] = 0; + state->iv[13] = 0; + state->iv[14] = 0; + state->iv[15] = 1; } + } else if (ipsec_sa->esp_iv_len) { + uint32_t len; - odp_packet_move_data(pkt, 0, _ODP_IPV4HDR_LEN, ip_offset); + len = odp_random_data(state->iv, ipsec_sa->esp_iv_len, + ODP_RANDOM_CRYPTO); - odp_packet_copy_from_mem(pkt, ip_offset, - _ODP_IPV4HDR_LEN, &out_ip); + if (len != ipsec_sa->esp_iv_len) + return -1; + } - odp_packet_l4_offset_set(pkt, ip_offset + _ODP_IPV4HDR_LEN); + return 0; +} - ip = odp_packet_l3_ptr(pkt, NULL); - ip_hdr_len = _ODP_IPV4HDR_LEN; +static int ipsec_out_esp(odp_packet_t *pkt, + ipsec_state_t *state, + ipsec_sa_t *ipsec_sa, + odp_crypto_packet_op_param_t *param, + odp_ipsec_op_status_t *status) +{ + _odp_esphdr_t esp; + _odp_esptrl_t esptrl; + uint32_t encrypt_len; + uint16_t ip_data_len = state->ip_tot_len - + state->ip_hdr_len; + uint32_t pad_block = ipsec_sa->esp_block_len; + uint16_t ipsec_offset = state->ip_offset + state->ip_hdr_len; + unsigned hdr_len; + unsigned trl_len; + + /* ESP trailer should be 32-bit right aligned */ + if (pad_block < 4) + pad_block = 4; + + encrypt_len = IPSEC_PAD_LEN(ip_data_len + _ODP_ESPTRL_LEN, + pad_block); + + hdr_len = _ODP_ESPHDR_LEN + ipsec_sa->esp_iv_len; + trl_len = encrypt_len - + ip_data_len + + ipsec_sa->icv_len; + + if (ipsec_out_iv(state, ipsec_sa) < 0) { + status->error.alg = 1; + return -1; } - ipsec_offset = ip_offset + ip_hdr_len; + param->override_iv_ptr = state->iv; - if (ipsec_sa->proto == ODP_IPSEC_ESP) { - _odp_esphdr_t esp; - _odp_esptrl_t esptrl; - uint32_t encrypt_len; - uint16_t ip_data_len = odp_be_to_cpu_16(ip->tot_len) - - ip_hdr_len; - uint32_t pad_block = ipsec_sa->esp_block_len; + if (odp_packet_extend_tail(pkt, trl_len, NULL, NULL) < 0 || + odp_packet_extend_head(pkt, hdr_len, NULL, NULL) < 0) { + status->error.alg = 1; + return -1; + } - /* ESP trailer should be 32-bit right aligned */ - if (pad_block < 4) - pad_block = 4; + odp_packet_move_data(*pkt, 0, hdr_len, ipsec_offset); + + state->ip = odp_packet_l3_ptr(*pkt, NULL); + + /* Set IPv4 length before authentication */ + ipv4_adjust_len(state->ip, hdr_len + trl_len); + state->ip_tot_len += hdr_len + trl_len; + + uint32_t esptrl_offset = state->ip_offset + + state->ip_hdr_len + + hdr_len + + encrypt_len - + _ODP_ESPTRL_LEN; + + memset(&esp, 0, sizeof(esp)); + esp.spi = odp_cpu_to_be_32(ipsec_sa->spi); + esp.seq_no = odp_cpu_to_be_32(ipsec_seq_no(ipsec_sa)); + + state->aad.spi = esp.spi; + state->aad.seq_no = esp.seq_no; + + param->aad.ptr = (uint8_t *)&state->aad; + + memset(&esptrl, 0, sizeof(esptrl)); + esptrl.pad_len = encrypt_len - ip_data_len - _ODP_ESPTRL_LEN; + esptrl.next_header = state->ip->proto; + state->ip->proto = _ODP_IPPROTO_ESP; + + odp_packet_copy_from_mem(*pkt, + ipsec_offset, _ODP_ESPHDR_LEN, + &esp); + odp_packet_copy_from_mem(*pkt, + ipsec_offset + _ODP_ESPHDR_LEN, + ipsec_sa->esp_iv_len, + state->iv + ipsec_sa->salt_length); + odp_packet_copy_from_mem(*pkt, + esptrl_offset - esptrl.pad_len, + esptrl.pad_len, ipsec_padding); + odp_packet_copy_from_mem(*pkt, + esptrl_offset, _ODP_ESPTRL_LEN, + &esptrl); + + param->cipher_range.offset = ipsec_offset + hdr_len; + param->cipher_range.length = state->ip_tot_len - + state->ip_hdr_len - + hdr_len - + ipsec_sa->icv_len; + + param->auth_range.offset = ipsec_offset; + param->auth_range.length = state->ip_tot_len - + state->ip_hdr_len - + ipsec_sa->icv_len; + param->hash_result_offset = state->ip_offset + + state->ip_tot_len - + ipsec_sa->icv_len; + + state->stats_length = param->cipher_range.length; - encrypt_len = ESP_ENCODE_LEN(ip_data_len + _ODP_ESPTRL_LEN, - pad_block); + return 0; +} - hdr_len = _ODP_ESPHDR_LEN + ipsec_sa->esp_iv_len; - trl_len = encrypt_len - - ip_data_len + - ipsec_sa->icv_len; +static int ipsec_out_ah(odp_packet_t *pkt, + ipsec_state_t *state, + ipsec_sa_t *ipsec_sa, + odp_crypto_packet_op_param_t *param, + odp_ipsec_op_status_t *status) +{ + _odp_ahhdr_t ah; + unsigned hdr_len = _ODP_AHHDR_LEN + ipsec_sa->esp_iv_len + + ipsec_sa->icv_len; + uint16_t ipsec_offset = state->ip_offset + state->ip_hdr_len; - if (ipsec_sa->use_counter_iv) { - uint64_t ctr; + /* Save IPv4 stuff */ + state->ah_ipv4.tos = state->ip->tos; + state->ah_ipv4.frag_offset = state->ip->frag_offset; + state->ah_ipv4.ttl = state->ip->ttl; - /* Both GCM and CTR use 8-bit counters */ - ODP_ASSERT(sizeof(ctr) == ipsec_sa->esp_iv_len); + if (odp_packet_extend_head(pkt, hdr_len, NULL, NULL) < 0) { + status->error.alg = 1; + return -1; + } - ctr = odp_atomic_fetch_add_u64(&ipsec_sa->out.counter, - 1); - /* Check for overrun */ - if (ctr == 0) - goto err; + odp_packet_move_data(*pkt, 0, hdr_len, ipsec_offset); - memcpy(iv, ipsec_sa->salt, ipsec_sa->salt_length); - memcpy(iv + ipsec_sa->salt_length, &ctr, - ipsec_sa->esp_iv_len); + state->ip = odp_packet_l3_ptr(*pkt, NULL); - if (ipsec_sa->aes_ctr_iv) { - iv[12] = 0; - iv[13] = 0; - iv[14] = 0; - iv[15] = 1; - } - } else if (ipsec_sa->esp_iv_len) { - uint32_t len; + /* Set IPv4 length before authentication */ + ipv4_adjust_len(state->ip, hdr_len); + state->ip_tot_len += hdr_len; - len = odp_random_data(iv, ipsec_sa->esp_iv_len, - ODP_RANDOM_CRYPTO); + memset(&ah, 0, sizeof(ah)); + ah.spi = odp_cpu_to_be_32(ipsec_sa->spi); + ah.ah_len = 1 + (ipsec_sa->esp_iv_len + ipsec_sa->icv_len) / 4; + ah.seq_no = odp_cpu_to_be_32(ipsec_seq_no(ipsec_sa)); + ah.next_header = state->ip->proto; + state->ip->proto = _ODP_IPPROTO_AH; - if (len != ipsec_sa->esp_iv_len) { - status->error.alg = 1; - goto err; - } - } + state->aad.spi = ah.spi; + state->aad.seq_no = ah.seq_no; - param.override_iv_ptr = iv; + param->aad.ptr = (uint8_t *)&state->aad; - if (odp_packet_extend_tail(&pkt, trl_len, NULL, NULL) < 0) { - status->error.alg = 1; - goto err; - } + /* For GMAC */ + if (ipsec_out_iv(state, ipsec_sa) < 0) { + status->error.alg = 1; + return -1; + } - if (odp_packet_extend_head(&pkt, hdr_len, NULL, NULL) < 0) { - status->error.alg = 1; - goto err; - } + param->override_iv_ptr = state->iv; - odp_packet_move_data(pkt, 0, hdr_len, ipsec_offset); - - ip = odp_packet_l3_ptr(pkt, NULL); - - /* Set IPv4 length before authentication */ - ipv4_adjust_len(ip, hdr_len + trl_len); - - uint32_t esptrl_offset = ip_offset + - ip_hdr_len + - hdr_len + - encrypt_len - - _ODP_ESPTRL_LEN; - - memset(&esp, 0, sizeof(esp)); - esp.spi = odp_cpu_to_be_32(ipsec_sa->spi); - esp.seq_no = odp_cpu_to_be_32(ipsec_seq_no(ipsec_sa)); - - aad.spi = esp.spi; - aad.seq_no = esp.seq_no; - - param.aad.ptr = (uint8_t *)&aad; - - memset(&esptrl, 0, sizeof(esptrl)); - esptrl.pad_len = encrypt_len - ip_data_len - _ODP_ESPTRL_LEN; - esptrl.next_header = ip->proto; - ip->proto = _ODP_IPPROTO_ESP; - - odp_packet_copy_from_mem(pkt, - ipsec_offset, _ODP_ESPHDR_LEN, - &esp); - odp_packet_copy_from_mem(pkt, - ipsec_offset + _ODP_ESPHDR_LEN, - ipsec_sa->esp_iv_len, - iv + ipsec_sa->salt_length); - odp_packet_copy_from_mem(pkt, - esptrl_offset - esptrl.pad_len, - esptrl.pad_len, ipsec_padding); - odp_packet_copy_from_mem(pkt, - esptrl_offset, _ODP_ESPTRL_LEN, - &esptrl); - - param.cipher_range.offset = ipsec_offset + hdr_len; - param.cipher_range.length = odp_be_to_cpu_16(ip->tot_len) - - ip_hdr_len - - hdr_len - - ipsec_sa->icv_len; - - param.auth_range.offset = ipsec_offset; - param.auth_range.length = odp_be_to_cpu_16(ip->tot_len) - - ip_hdr_len - - ipsec_sa->icv_len; - param.hash_result_offset = ip_offset + - odp_be_to_cpu_16(ip->tot_len) - - ipsec_sa->icv_len; - - stats_length = param.cipher_range.length; - } else if (ipsec_sa->proto == ODP_IPSEC_AH) { - _odp_ahhdr_t ah; - - hdr_len = _ODP_AHHDR_LEN + ipsec_sa->esp_iv_len + - ipsec_sa->icv_len; - trl_len = 0; - - /* Save IPv4 stuff */ - ip_tos = ip->tos; - ip_frag_offset = odp_be_to_cpu_16(ip->frag_offset); - ip_ttl = ip->ttl; - - if (odp_packet_extend_tail(&pkt, trl_len, NULL, NULL) < 0) { - status->error.alg = 1; - goto err; - } + odp_packet_copy_from_mem(*pkt, + ipsec_offset, _ODP_AHHDR_LEN, + &ah); + odp_packet_copy_from_mem(*pkt, + ipsec_offset + _ODP_AHHDR_LEN, + ipsec_sa->esp_iv_len, + state->iv + ipsec_sa->salt_length); + _odp_packet_set_data(*pkt, + ipsec_offset + _ODP_AHHDR_LEN + + ipsec_sa->esp_iv_len, + 0, ipsec_sa->icv_len); - if (odp_packet_extend_head(&pkt, hdr_len, NULL, NULL) < 0) { - status->error.alg = 1; - goto err; - } + state->ip->chksum = 0; + state->ip->tos = 0; + state->ip->frag_offset = 0; + state->ip->ttl = 0; - odp_packet_move_data(pkt, 0, hdr_len, ipsec_offset); + param->auth_range.offset = state->ip_offset; + param->auth_range.length = state->ip_tot_len; + param->hash_result_offset = ipsec_offset + _ODP_AHHDR_LEN + + ipsec_sa->esp_iv_len; - ip = odp_packet_l3_ptr(pkt, NULL); + state->stats_length = param->auth_range.length; - /* Set IPv4 length before authentication */ - ipv4_adjust_len(ip, hdr_len + trl_len); + return 0; +} - memset(&ah, 0, sizeof(ah)); - ah.spi = odp_cpu_to_be_32(ipsec_sa->spi); - ah.ah_len = 1 + (ipsec_sa->esp_iv_len + ipsec_sa->icv_len) / 4; - ah.seq_no = odp_cpu_to_be_32(ipsec_seq_no(ipsec_sa)); - ah.next_header = ip->proto; - ip->proto = _ODP_IPPROTO_AH; +static void ipsec_out_ah_post(ipsec_state_t *state) +{ + state->ip->ttl = state->ah_ipv4.ttl; + state->ip->tos = state->ah_ipv4.tos; + state->ip->frag_offset = state->ah_ipv4.frag_offset; +} - aad.spi = ah.spi; - aad.seq_no = ah.seq_no; +static ipsec_sa_t *ipsec_out_single(odp_packet_t pkt, + odp_ipsec_sa_t sa, + odp_packet_t *pkt_out, + odp_ipsec_out_opt_t *opt ODP_UNUSED, + odp_ipsec_op_status_t *status) +{ + ipsec_state_t state; + ipsec_sa_t *ipsec_sa; + odp_crypto_packet_op_param_t param; + int rc; + odp_crypto_packet_result_t crypto; /**< Crypto operation result */ + odp_packet_hdr_t *pkt_hdr; - param.aad.ptr = (uint8_t *)&aad; + state.ip_offset = odp_packet_l3_offset(pkt); + ODP_ASSERT(ODP_PACKET_OFFSET_INVALID != state.ip_offset); - /* For GMAC */ - if (ipsec_sa->use_counter_iv) { - uint64_t ctr; + state.ip = odp_packet_l3_ptr(pkt, NULL); + ODP_ASSERT(NULL != state.ip); - ODP_ASSERT(sizeof(ctr) == ipsec_sa->esp_iv_len); + ipsec_sa = _odp_ipsec_sa_use(sa); + ODP_ASSERT(NULL != ipsec_sa); - ctr = odp_atomic_fetch_add_u64(&ipsec_sa->out.counter, - 1); - /* Check for overrun */ - if (ctr == 0) - goto err; + /* Initialize parameters block */ + memset(¶m, 0, sizeof(param)); - memcpy(iv, ipsec_sa->salt, ipsec_sa->salt_length); - memcpy(iv + ipsec_sa->salt_length, &ctr, - ipsec_sa->esp_iv_len); - param.override_iv_ptr = iv; + if (ODP_IPSEC_MODE_TRANSPORT == ipsec_sa->mode) { + rc = ipsec_parse_ipv4(&state); + if (state.ip_tot_len + state.ip_offset != odp_packet_len(pkt)) + rc = -1; + } else { + rc = ipsec_out_tunnel_parse_ipv4(&state, ipsec_sa); + if (rc < 0) { + status->error.alg = 1; + goto err; } - odp_packet_copy_from_mem(pkt, - ipsec_offset, _ODP_AHHDR_LEN, - &ah); - odp_packet_copy_from_mem(pkt, - ipsec_offset + _ODP_AHHDR_LEN, - ipsec_sa->esp_iv_len, - iv + ipsec_sa->salt_length); - _odp_packet_set_data(pkt, - ipsec_offset + _ODP_AHHDR_LEN + - ipsec_sa->esp_iv_len, - 0, ipsec_sa->icv_len); - - ip->chksum = 0; - ip->tos = 0; - ip->frag_offset = 0; - ip->ttl = 0; - - param.auth_range.offset = ip_offset; - param.auth_range.length = odp_be_to_cpu_16(ip->tot_len); - param.hash_result_offset = ipsec_offset + _ODP_AHHDR_LEN + - ipsec_sa->esp_iv_len; - - stats_length = param.auth_range.length; + rc = ipsec_out_tunnel_ipv4(&pkt, &state, ipsec_sa); + } + if (rc < 0) { + status->error.alg = 1; + goto err; + } + + if (ODP_IPSEC_ESP == ipsec_sa->proto) { + rc = ipsec_out_esp(&pkt, &state, ipsec_sa, ¶m, status); + } else if (ODP_IPSEC_AH == ipsec_sa->proto) { + rc = ipsec_out_ah(&pkt, &state, ipsec_sa, ¶m, status); } else { status->error.alg = 1; goto err; } + if (rc < 0) + goto err; /* No need to run precheck here, we know that packet is authentic */ - if (_odp_ipsec_sa_stats_update(ipsec_sa, stats_length, status) < 0) + if (_odp_ipsec_sa_stats_update(ipsec_sa, + state.stats_length, + status) < 0) goto err; param.session = ipsec_sa->session; @@ -922,14 +1032,9 @@ static ipsec_sa_t *ipsec_out_single(odp_packet_t pkt, goto err; } - ip = odp_packet_l3_ptr(pkt, NULL); - /* Finalize the IPv4 header */ - if (ip->proto == _ODP_IPPROTO_AH) { - ip->ttl = ip_ttl; - ip->tos = ip_tos; - ip->frag_offset = odp_cpu_to_be_16(ip_frag_offset); - } + if (ODP_IPSEC_AH == ipsec_sa->proto) + ipsec_out_ah_post(&state); _odp_ipv4_csum_update(pkt);
