From: Dmitry Eremin-Solenikov <[email protected]> Per idea of Janne Peltonen, do not allocate/free crypto contexts for each operation, providing significant speed increase. Each thread on startup allocates hmac+cipher contexts pair for each crypto session. Then they are initialized on demand, when session is first executed using this thread.
Signed-off-by: Dmitry Eremin-Solenikov <[email protected]> --- /** Email created from pull request 342 (lumag:openssl-ctx) ** https://github.com/Linaro/odp/pull/342 ** Patch: https://github.com/Linaro/odp/pull/342.patch ** Base sha: 6b5cdc77eb9759a2349b10372a964648559bc92c ** Merge commit sha: 4efcbed0936263184a699b1bedcf9e0bf15c5f73 **/ platform/linux-generic/include/odp_internal.h | 2 + platform/linux-generic/odp_crypto.c | 199 +++++++++++++++++--------- platform/linux-generic/odp_init.c | 13 ++ 3 files changed, 145 insertions(+), 69 deletions(-) diff --git a/platform/linux-generic/include/odp_internal.h b/platform/linux-generic/include/odp_internal.h index fc69cd0c5..94aa16ccd 100644 --- a/platform/linux-generic/include/odp_internal.h +++ b/platform/linux-generic/include/odp_internal.h @@ -108,6 +108,8 @@ int odp_queue_term_global(void); int odp_crypto_init_global(void); int odp_crypto_term_global(void); +int _odp_crypto_init_local(void); +int _odp_crypto_term_local(void); int odp_timer_init_global(void); int odp_timer_term_global(void); diff --git a/platform/linux-generic/odp_crypto.c b/platform/linux-generic/odp_crypto.c index 9b5d94034..855bf2eba 100644 --- a/platform/linux-generic/odp_crypto.c +++ b/platform/linux-generic/odp_crypto.c @@ -85,6 +85,9 @@ typedef odp_crypto_alg_err_t (*crypto_func_t)(odp_crypto_op_param_t *param, odp_crypto_generic_session_t *session); +typedef uint32_t crypto_valid; +#define CV_SIZE sizeof(crypto_valid) + /** * Per crypto session data structure */ @@ -112,8 +115,20 @@ struct odp_crypto_generic_session_t { const EVP_MD *evp_md; crypto_func_t func; } auth; + + crypto_valid cipher_valid[(ODP_THREAD_COUNT_MAX + CV_SIZE - 1) / + CV_SIZE]; + crypto_valid hmac_valid[(ODP_THREAD_COUNT_MAX + CV_SIZE - 1) / + CV_SIZE]; + unsigned idx; }; +#define IS_VALID(session, type, id) \ + (session->type ## _valid[id / CV_SIZE] & (1 << (id % CV_SIZE))) + +#define SET_VALID(session, type, id) \ + session->type ## _valid[id / CV_SIZE] |= (1 << (id % CV_SIZE)) + typedef struct odp_crypto_global_s odp_crypto_global_t; struct odp_crypto_global_s { @@ -125,6 +140,16 @@ struct odp_crypto_global_s { static odp_crypto_global_t *global; +typedef struct crypto_local_t { + struct { + HMAC_CTX *hmac; + EVP_CIPHER_CTX *cipher; + } ctx[MAX_SESSIONS]; + int id; +} crypto_local_t; + +static __thread crypto_local_t local; + static odp_crypto_generic_op_result_t *get_op_result_from_event(odp_event_t ev) { @@ -146,6 +171,8 @@ odp_crypto_generic_session_t *alloc_session(void) } odp_spinlock_unlock(&global->lock); + session->idx = (session - global->sessions) / sizeof(*session); + return session; } @@ -165,24 +192,51 @@ null_crypto_routine(odp_crypto_op_param_t *param ODP_UNUSED, return ODP_CRYPTO_ALG_ERR_NONE; } +/* Mimic new OpenSSL 1.1.y API */ +#if OPENSSL_VERSION_NUMBER < 0x10100000L +static HMAC_CTX *HMAC_CTX_new(void) +{ + HMAC_CTX *ctx = malloc(sizeof(*ctx)); + + HMAC_CTX_init(ctx); + return ctx; +} + +static void HMAC_CTX_free(HMAC_CTX *ctx) +{ + HMAC_CTX_cleanup(ctx); + free(ctx); +} +#endif + static -void packet_hmac_calculate(HMAC_CTX *ctx, - odp_crypto_op_param_t *param, - odp_crypto_generic_session_t *session, - uint8_t *hash) +void packet_hmac(odp_crypto_op_param_t *param, + odp_crypto_generic_session_t *session, + uint8_t *hash) { + HMAC_CTX *ctx = local.ctx[session->idx].hmac; odp_packet_t pkt = param->out_pkt; uint32_t offset = param->auth_range.offset; uint32_t len = param->auth_range.length; ODP_ASSERT(offset + len <= odp_packet_len(pkt)); - HMAC_Init_ex(ctx, - session->auth.key, - session->auth.key_length, - session->auth.evp_md, - NULL); + if (!IS_VALID(session, hmac, local.id)) { + SET_VALID(session, hmac, local.id); + HMAC_Init_ex(ctx, + session->auth.key, + session->auth.key_length, + session->auth.evp_md, + NULL); + } else { + HMAC_Init_ex(ctx, + NULL, + session->auth.key_length, + NULL, + NULL); + } + /* Hash it */ while (len > 0) { uint32_t seglen = 0; /* GCC */ void *mapaddr = odp_packet_offset(pkt, offset, &seglen, NULL); @@ -196,34 +250,6 @@ void packet_hmac_calculate(HMAC_CTX *ctx, HMAC_Final(ctx, hash, NULL); } -#if OPENSSL_VERSION_NUMBER < 0x10100000L -static -void packet_hmac(odp_crypto_op_param_t *param, - odp_crypto_generic_session_t *session, - uint8_t *hash) -{ - HMAC_CTX ctx; - - /* Hash it */ - HMAC_CTX_init(&ctx); - packet_hmac_calculate(&ctx, param, session, hash); - HMAC_CTX_cleanup(&ctx); -} -#else -static -void packet_hmac(odp_crypto_op_param_t *param, - odp_crypto_generic_session_t *session, - uint8_t *hash) -{ - HMAC_CTX *ctx; - - /* Hash it */ - ctx = HMAC_CTX_new(); - packet_hmac_calculate(ctx, param, session, hash); - HMAC_CTX_free(ctx); -} -#endif - static odp_crypto_alg_err_t auth_gen(odp_crypto_op_param_t *param, odp_crypto_generic_session_t *session) @@ -386,7 +412,7 @@ static odp_crypto_alg_err_t cipher_encrypt(odp_crypto_op_param_t *param, odp_crypto_generic_session_t *session) { - EVP_CIPHER_CTX *ctx; + EVP_CIPHER_CTX *ctx = local.ctx[session->idx].cipher; void *iv_ptr; int ret; @@ -398,16 +424,16 @@ odp_crypto_alg_err_t cipher_encrypt(odp_crypto_op_param_t *param, return ODP_CRYPTO_ALG_ERR_IV_INVALID; /* Encrypt it */ - ctx = EVP_CIPHER_CTX_new(); - EVP_EncryptInit_ex(ctx, session->cipher.evp_cipher, NULL, - session->cipher.key_data, NULL); + if (!IS_VALID(session, cipher, local.id)) { + SET_VALID(session, cipher, local.id); + EVP_EncryptInit_ex(ctx, session->cipher.evp_cipher, NULL, + session->cipher.key_data, NULL); + EVP_CIPHER_CTX_set_padding(ctx, 0); + } EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, iv_ptr); - EVP_CIPHER_CTX_set_padding(ctx, 0); ret = internal_encrypt(ctx, param); - EVP_CIPHER_CTX_free(ctx); - return ret <= 0 ? ODP_CRYPTO_ALG_ERR_DATA_SIZE : ODP_CRYPTO_ALG_ERR_NONE; } @@ -416,7 +442,7 @@ static odp_crypto_alg_err_t cipher_decrypt(odp_crypto_op_param_t *param, odp_crypto_generic_session_t *session) { - EVP_CIPHER_CTX *ctx; + EVP_CIPHER_CTX *ctx = local.ctx[session->idx].cipher; void *iv_ptr; int ret; @@ -428,16 +454,16 @@ odp_crypto_alg_err_t cipher_decrypt(odp_crypto_op_param_t *param, return ODP_CRYPTO_ALG_ERR_IV_INVALID; /* Decrypt it */ - ctx = EVP_CIPHER_CTX_new(); - EVP_DecryptInit_ex(ctx, session->cipher.evp_cipher, NULL, - session->cipher.key_data, NULL); + if (!IS_VALID(session, cipher, local.id)) { + SET_VALID(session, cipher, local.id); + EVP_DecryptInit_ex(ctx, session->cipher.evp_cipher, NULL, + session->cipher.key_data, NULL); + EVP_CIPHER_CTX_set_padding(ctx, 0); + } EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, iv_ptr); - EVP_CIPHER_CTX_set_padding(ctx, 0); ret = internal_decrypt(ctx, param); - EVP_CIPHER_CTX_free(ctx); - return ret <= 0 ? ODP_CRYPTO_ALG_ERR_DATA_SIZE : ODP_CRYPTO_ALG_ERR_NONE; } @@ -473,7 +499,7 @@ static odp_crypto_alg_err_t aes_gcm_encrypt(odp_crypto_op_param_t *param, odp_crypto_generic_session_t *session) { - EVP_CIPHER_CTX *ctx; + EVP_CIPHER_CTX *ctx = local.ctx[session->idx].cipher; const uint8_t *aad_head = param->aad.ptr; uint32_t aad_len = param->aad.length; void *iv_ptr; @@ -489,13 +515,15 @@ odp_crypto_alg_err_t aes_gcm_encrypt(odp_crypto_op_param_t *param, return ODP_CRYPTO_ALG_ERR_IV_INVALID; /* Encrypt it */ - ctx = EVP_CIPHER_CTX_new(); - EVP_EncryptInit_ex(ctx, session->cipher.evp_cipher, NULL, - session->cipher.key_data, NULL); - EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_IVLEN, - session->p.iv.length, NULL); + if (!IS_VALID(session, cipher, local.id)) { + SET_VALID(session, cipher, local.id); + EVP_EncryptInit_ex(ctx, session->cipher.evp_cipher, NULL, + session->cipher.key_data, NULL); + EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_IVLEN, + session->p.iv.length, NULL); + EVP_CIPHER_CTX_set_padding(ctx, 0); + } EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, iv_ptr); - EVP_CIPHER_CTX_set_padding(ctx, 0); /* Authenticate header data (if any) without encrypting them */ if (aad_len > 0) @@ -509,8 +537,6 @@ odp_crypto_alg_err_t aes_gcm_encrypt(odp_crypto_op_param_t *param, odp_packet_copy_from_mem(param->out_pkt, param->hash_result_offset, session->p.auth_digest_len, block); - EVP_CIPHER_CTX_free(ctx); - return ret <= 0 ? ODP_CRYPTO_ALG_ERR_DATA_SIZE : ODP_CRYPTO_ALG_ERR_NONE; } @@ -519,7 +545,7 @@ static odp_crypto_alg_err_t aes_gcm_decrypt(odp_crypto_op_param_t *param, odp_crypto_generic_session_t *session) { - EVP_CIPHER_CTX *ctx; + EVP_CIPHER_CTX *ctx = local.ctx[session->idx].cipher; const uint8_t *aad_head = param->aad.ptr; uint32_t aad_len = param->aad.length; int dummy_len = 0; @@ -535,13 +561,15 @@ odp_crypto_alg_err_t aes_gcm_decrypt(odp_crypto_op_param_t *param, return ODP_CRYPTO_ALG_ERR_IV_INVALID; /* Decrypt it */ - ctx = EVP_CIPHER_CTX_new(); - EVP_DecryptInit_ex(ctx, session->cipher.evp_cipher, NULL, - session->cipher.key_data, NULL); - EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_IVLEN, - session->p.iv.length, NULL); + if (!IS_VALID(session, cipher, local.id)) { + SET_VALID(session, cipher, local.id); + EVP_DecryptInit_ex(ctx, session->cipher.evp_cipher, NULL, + session->cipher.key_data, NULL); + EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_IVLEN, + session->p.iv.length, NULL); + EVP_CIPHER_CTX_set_padding(ctx, 0); + } EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, iv_ptr); - EVP_CIPHER_CTX_set_padding(ctx, 0); odp_packet_copy_to_mem(param->out_pkt, param->hash_result_offset, session->p.auth_digest_len, block); @@ -555,8 +583,6 @@ odp_crypto_alg_err_t aes_gcm_decrypt(odp_crypto_op_param_t *param, ret = internal_decrypt(ctx, param); - EVP_CIPHER_CTX_free(ctx); - return ret <= 0 ? ODP_CRYPTO_ALG_ERR_ICV_CHECK : ODP_CRYPTO_ALG_ERR_NONE; } @@ -1060,6 +1086,41 @@ int odp_crypto_term_global(void) return rc; } +int _odp_crypto_init_local(void) +{ + unsigned i; + + memset(&local, 0, sizeof(local)); + + for (i = 0; i < MAX_SESSIONS; i++) { + local.ctx[i].hmac = HMAC_CTX_new(); + local.ctx[i].cipher = EVP_CIPHER_CTX_new(); + + if (local.ctx[i].hmac == NULL || local.ctx[i].cipher == NULL) { + _odp_crypto_term_local(); + return -1; + } + } + + local.id = odp_thread_id(); + + return 0; +} + +int _odp_crypto_term_local(void) +{ + unsigned i; + + for (i = 0; i < MAX_SESSIONS; i++) { + if (local.ctx[i].hmac != NULL) + HMAC_CTX_free(local.ctx[i].hmac); + if (local.ctx[i].cipher != NULL) + EVP_CIPHER_CTX_free(local.ctx[i].cipher); + } + + return 0; +} + odp_random_kind_t odp_random_max_kind(void) { return ODP_RANDOM_CRYPTO; diff --git a/platform/linux-generic/odp_init.c b/platform/linux-generic/odp_init.c index fe03709b7..e5de95025 100644 --- a/platform/linux-generic/odp_init.c +++ b/platform/linux-generic/odp_init.c @@ -300,6 +300,12 @@ int odp_init_local(odp_instance_t instance, odp_thread_type_t thr_type) } stage = PKTIO_INIT; + if (_odp_crypto_init_local()) { + ODP_ERR("ODP crypto local init failed.\n"); + goto init_fail; + } + stage = CRYPTO_INIT; + if (odp_pool_init_local()) { ODP_ERR("ODP pool local init failed.\n"); goto init_fail; @@ -352,6 +358,13 @@ int _odp_term_local(enum init_stage stage) } /* Fall through */ + case CRYPTO_INIT: + if (_odp_crypto_term_local()) { + ODP_ERR("ODP crypto local term failed.\n"); + rc = -1; + } + /* Fall through */ + case POOL_INIT: if (odp_pool_term_local()) { ODP_ERR("ODP buffer pool local term failed.\n");
