Author: dsahlberg Date: Tue Jul 1 08:52:03 2025 New Revision: 1926894 URL: http://svn.apache.org/viewvc?rev=1926894&view=rev Log: On the PR-8 branch:
Applied all commits from the PR to allow for easier review and testing. Full commit message with comments will follow when merging to trunk. Modified: serf/branches/PR-8/CMakeLists.txt serf/branches/PR-8/SConstruct serf/branches/PR-8/buckets/ssl_buckets.c serf/branches/PR-8/serf_bucket_types.h serf/branches/PR-8/test/serf_get.c serf/branches/PR-8/test/test_ssl.c Modified: serf/branches/PR-8/CMakeLists.txt URL: http://svn.apache.org/viewvc/serf/branches/PR-8/CMakeLists.txt?rev=1926894&r1=1926893&r2=1926894&view=diff ============================================================================== --- serf/branches/PR-8/CMakeLists.txt (original) +++ serf/branches/PR-8/CMakeLists.txt Tue Jul 1 08:52:03 2025 @@ -305,6 +305,11 @@ CheckNotFunction("X509_STORE_CTX_get0_ch CheckNotFunction("ASN1_STRING_get0_data" "NULL" "SERF_NO_SSL_ASN1_STRING_GET0_DATA" "openssl/asn1.h" "${OPENSSL_INCLUDE_DIR}" ${OPENSSL_LIBRARIES} ${SERF_STANDARD_LIBRARIES}) +CheckFunction("OSSL_STORE_open_ex" + "NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL" + "SERF_HAVE_OSSL_STORE_OPEN_EX" "openssl/store.h" + "${OPENSSL_INCLUDE_DIR}" ${OPENSSL_LIBRARIES} + ${SERF_STANDARD_LIBRARIES}) CheckFunction("CRYPTO_set_locking_callback" "NULL" "SERF_HAVE_SSL_LOCKING_CALLBACKS" "openssl/crypto.h" "${OPENSSL_INCLUDE_DIR}" ${OPENSSL_LIBRARIES} ${SERF_STANDARD_LIBRARIES}) Modified: serf/branches/PR-8/SConstruct URL: http://svn.apache.org/viewvc/serf/branches/PR-8/SConstruct?rev=1926894&r1=1926893&r2=1926894&view=diff ============================================================================== --- serf/branches/PR-8/SConstruct (original) +++ serf/branches/PR-8/SConstruct Tue Jul 1 08:52:03 2025 @@ -612,6 +612,9 @@ if conf.CheckFunc('OpenSSL_version_num', env.Append(CPPDEFINES=['SERF_HAVE_OPENSSL_VERSION_NUM']) if conf.CheckFunc('SSL_set_alpn_protos', ssl_includes, 'C', 'NULL, NULL, 0'): env.Append(CPPDEFINES=['SERF_HAVE_OPENSSL_ALPN']) +if conf.CheckFunc('OSSL_STORE_open_ex', ssl_includes, 'C', + 'NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL'): + env.Append(CPPDEFINES=['SERF_HAVE_OSSL_STORE_OPEN_EX']) if conf.CheckType('OSSL_HANDSHAKE_STATE', ssl_includes): env.Append(CPPDEFINES=['SERF_HAVE_OSSL_HANDSHAKE_STATE']) env = conf.Finish() Modified: serf/branches/PR-8/buckets/ssl_buckets.c URL: http://svn.apache.org/viewvc/serf/branches/PR-8/buckets/ssl_buckets.c?rev=1926894&r1=1926893&r2=1926894&view=diff ============================================================================== --- serf/branches/PR-8/buckets/ssl_buckets.c (original) +++ serf/branches/PR-8/buckets/ssl_buckets.c Tue Jul 1 08:52:03 2025 @@ -44,6 +44,15 @@ #ifndef OPENSSL_NO_OCSP /* requires openssl 0.9.7 or later */ #include <openssl/ocsp.h> #endif +#if defined(SERF_HAVE_OSSL_STORE_OPEN_EX) +#include <openssl/store.h> +#include <openssl/evp.h> +#include <openssl/safestack.h> +#include <openssl/ui.h> +#ifndef sk_EVP_PKEY_new_null +DEFINE_STACK_OF(EVP_PKEY) +#endif +#endif #ifndef APR_ARRAY_PUSH #define APR_ARRAY_PUSH(ary,type) (*((type *)apr_array_push(ary))) @@ -117,6 +126,8 @@ * */ +static int ssl_x509_ex_data_idx = -1; + typedef struct bucket_list { serf_bucket_t *bucket; struct bucket_list *next; @@ -177,12 +188,20 @@ struct serf_ssl_context_t { apr_pool_t *cert_pw_cache_pool; const char *cert_pw_success; + /* Cert uri callbacks */ + serf_ssl_need_cert_uri_t cert_uri_callback; + void *cert_uri_userdata; + apr_pool_t *cert_uri_cache_pool; + const char *cert_uri_success; + /* Server cert callbacks */ serf_ssl_need_server_cert_t server_cert_callback; serf_ssl_server_cert_chain_cb_t server_cert_chain_callback; void *server_cert_userdata; const char *cert_path; + const char *cert_uri; + const char *cert_pw; X509 *cached_cert; EVP_PKEY *cached_cert_pw; @@ -1502,6 +1521,12 @@ static apr_status_t do_init_libraries(vo OpenSSL_add_all_algorithms(); #endif +#if defined(SERF_HAVE_OSSL_STORE_OPEN_EX) + if (ssl_x509_ex_data_idx < 0) { + ssl_x509_ex_data_idx = X509_get_ex_new_index(0, NULL, NULL, NULL, NULL); + } +#endif + #if APR_HAS_THREADS && defined(SERF_HAVE_SSL_LOCKING_CALLBACKS) numlocks = CRYPTO_num_locks(); apr_pool_create(&ssl_pool, NULL); @@ -1533,10 +1558,50 @@ static apr_status_t init_ssl_libraries(v return serf__init_once(&init_ctx, do_init_libraries, NULL); } +#if defined(SERF_HAVE_OSSL_STORE_OPEN_EX) + +static int ssl_pass_cb(UI *ui, UI_STRING *uis) +{ + serf_ssl_context_t *ctx = UI_get0_user_data(ui); + + const char *password; + apr_status_t status; + + if (ctx->cert_pw_success) { + status = APR_SUCCESS; + password = ctx->cert_pw_success; + ctx->cert_pw_success = NULL; + } + else if (ctx->cert_pw_callback) { + status = ctx->cert_pw_callback(ctx->cert_pw_userdata, + ctx->cert_uri, + &password); + } + else { + return 0; + } + + UI_set_result(ui, uis, password); + + ctx->cert_pw = apr_pstrdup(ctx->pool, password); + + return 1; +} + +#endif + static int ssl_need_client_cert(SSL *ssl, X509 **cert, EVP_PKEY **pkey) { serf_ssl_context_t *ctx = SSL_get_app_data(ssl); +#if defined(SERF_HAVE_OSSL_STORE_OPEN_EX) + STACK_OF(X509) *leaves; + STACK_OF(X509) *intermediates; + STACK_OF(EVP_PKEY) *keys; + X509_STORE *requests; + UI_METHOD *ui_method; +#endif apr_status_t status; + int retrying_success = 0; serf__log(LOGLVL_DEBUG, LOGCOMP_SSL, __FILE__, ctx->config, "Server requests a client certificate.\n"); @@ -1547,6 +1612,212 @@ static int ssl_need_client_cert(SSL *ssl return 1; } +#if defined(SERF_HAVE_OSSL_STORE_OPEN_EX) + + /* until further notice */ + *cert = NULL; + *pkey = NULL; + + leaves = sk_X509_new_null(); + intermediates = sk_X509_new_null(); + keys = sk_EVP_PKEY_new_null(); + requests = X509_STORE_new(); + + ui_method = UI_create_method("passphrase"); + UI_method_set_reader(ui_method, ssl_pass_cb); + + while (ctx->cert_uri_callback) { + const char *cert_uri = NULL; + OSSL_STORE_CTX *store = NULL; + OSSL_STORE_INFO *info; + X509 *c; + STACK_OF(X509_NAME) *requested; + int type; + + retrying_success = 0; + + if (ctx->cert_uri_success) { + status = APR_SUCCESS; + cert_uri = ctx->cert_uri_success; + ctx->cert_uri_success = NULL; + retrying_success = 1; + } else { + status = ctx->cert_uri_callback(ctx->cert_uri_userdata, &cert_uri); + } + + if (status || !cert_uri) { + break; + } + + ctx->cert_uri = cert_uri; + + /* server side request some certs? this list may be empty */ + requested = SSL_get_client_CA_list(ssl); + + store = OSSL_STORE_open_ex(cert_uri, NULL, NULL, ui_method, ctx, NULL, + NULL, NULL); + if (!store) { + int err = ERR_get_error(); + serf__log(LOGLVL_ERROR, LOGCOMP_SSL, __FILE__, ctx->config, + "OpenSSL store error (%s): %d %d\n", cert_uri, + ERR_GET_LIB(err), ERR_GET_REASON(err)); + break; + } + + /* walk the store, what are we working with */ + + while (!OSSL_STORE_eof(store)) { + info = OSSL_STORE_load(store); + + if (!info) { + break; + } + + type = OSSL_STORE_INFO_get_type(info); + if (type == OSSL_STORE_INFO_CERT) { + X509 *c = OSSL_STORE_INFO_get1_CERT(info); + + int n, i; + + int is_ca = X509_check_ca(c); + + /* split into leaves and intermediate certs */ + if (is_ca) { + sk_X509_push(intermediates, c); + } + else { + sk_X509_push(leaves, c); + } + + /* any cert with an issuer matching our requested CAs is also + * added to the requests list, except for leaf certs which are + * marked as requested with a flag so we can skip the chain + * check later. */ + n = sk_X509_NAME_num(requested); + for (i = 0; i < n; ++i) { + X509_NAME *name = sk_X509_NAME_value(requested, i); + if (X509_NAME_cmp(name, X509_get_issuer_name(c)) == 0) { + if (is_ca) { + X509_STORE_add_cert(requests, c); + } + else { + X509_set_ex_data(c, ssl_x509_ex_data_idx, + (void *)1); + } + } + } + + } else if (type == OSSL_STORE_INFO_PKEY) { + EVP_PKEY *k = OSSL_STORE_INFO_get1_PKEY(info); + + sk_EVP_PKEY_push(keys, k); + } + + OSSL_STORE_INFO_free(info); + } + + /* FIXME: openssl error checking goes here */ + + OSSL_STORE_close(store); + + /* walk the leaf certificates, choose the best one */ + + while ((c = sk_X509_pop(leaves))) { + + EVP_PKEY *k = NULL; + int i, n, found = 0; + + /* no key, skip */ + n = sk_EVP_PKEY_num(keys); + for (i = 0; i < n; ++i) { + k = sk_EVP_PKEY_value(keys, i); + if (X509_check_private_key(c, k)) { + found = 1; + break; + } + } + if (!found) { + continue; + } + + /* CAs requested? if so, skip non matches, if not, accept all */ + if (sk_X509_NAME_num(requested) && + !X509_get_ex_data(c, ssl_x509_ex_data_idx)) { + STACK_OF(X509) *chain; + + chain = X509_build_chain(c, intermediates, requests, 0, NULL, + NULL); + + if (!chain) { + continue; + } + + sk_X509_pop_free(chain, X509_free); + } + + /* no best candidate yet? we're in first place */ + if (!*cert) { + EVP_PKEY_up_ref(k); + *cert = c; /* don't dup, we're returning this */ + *pkey = k; + continue; + } + + /* were we issued after the previous best? */ + if (ASN1_TIME_compare(X509_get0_notBefore(*cert), + X509_get0_notBefore(c)) < 0) { + X509_free(*cert); + EVP_PKEY_free(*pkey); + EVP_PKEY_up_ref(k); + *cert = c; /* don't dup, we're returning this */ + *pkey = k; + continue; + } + + X509_free(c); + } + + break; + } + + sk_X509_pop_free(leaves, X509_free); + sk_X509_pop_free(intermediates, X509_free); + sk_EVP_PKEY_pop_free(keys, EVP_PKEY_free); + X509_STORE_free(requests); + UI_destroy_method(ui_method); + + /* we settled on a cert and key, cache it for later */ + + if (*cert && *pkey) { + + ctx->cached_cert = *cert; + ctx->cached_cert_pw = *pkey; + if (!retrying_success && ctx->cert_cache_pool) { + const char *c; + + c = apr_pstrdup(ctx->cert_cache_pool, ctx->cert_uri); + + apr_pool_userdata_setn(c, "serf:ssl:cert", + apr_pool_cleanup_null, + ctx->cert_cache_pool); + } + + if (!retrying_success && ctx->cert_pw_cache_pool && ctx->cert_pw) { + const char *pw; + + pw = apr_pstrdup(ctx->cert_pw_cache_pool, + ctx->cert_pw); + + apr_pool_userdata_setn(pw, "serf:ssl:certpw", + apr_pool_cleanup_null, + ctx->cert_pw_cache_pool); + } + + return 1; + } + +#endif + while (ctx->cert_callback) { const char *cert_path; apr_file_t *cert_file; @@ -1554,7 +1825,7 @@ static int ssl_need_client_cert(SSL *ssl BIO_METHOD *biom; PKCS12 *p12; int i; - int retrying_success = 0; + retrying_success = 0; if (ctx->cert_file_success) { status = APR_SUCCESS; @@ -1704,6 +1975,22 @@ void serf_ssl_client_cert_password_set( } +void serf_ssl_cert_uri_set( + serf_ssl_context_t *context, + serf_ssl_need_cert_uri_t callback, + void *data, + void *cache_pool) +{ + context->cert_uri_callback = callback; + context->cert_uri_userdata = data; + context->cert_cache_pool = cache_pool; + if (context->cert_cache_pool) { + apr_pool_userdata_get((void**)&context->cert_uri_success, + "serf:ssl:certuri", cache_pool); + } +} + + void serf_ssl_server_cert_callback_set( serf_ssl_context_t *context, serf_ssl_need_server_cert_t callback, @@ -1780,6 +2067,7 @@ static serf_ssl_context_t *ssl_init_cont ssl_ctx->cert_callback = NULL; ssl_ctx->cert_pw_callback = NULL; + ssl_ctx->cert_uri_callback = NULL; ssl_ctx->server_cert_callback = NULL; ssl_ctx->server_cert_chain_callback = NULL; Modified: serf/branches/PR-8/serf_bucket_types.h URL: http://svn.apache.org/viewvc/serf/branches/PR-8/serf_bucket_types.h?rev=1926894&r1=1926893&r2=1926894&view=diff ============================================================================== --- serf/branches/PR-8/serf_bucket_types.h (original) +++ serf/branches/PR-8/serf_bucket_types.h Tue Jul 1 08:52:03 2025 @@ -600,6 +600,10 @@ typedef apr_status_t (*serf_ssl_need_cer const char *cert_path, const char **password); +typedef apr_status_t (*serf_ssl_need_cert_uri_t)( + void *data, + const char **cert_uri); + /** * Callback type for server certificate status info and OCSP responses. * Note that CERT can be NULL in case its called from the OCSP callback. @@ -616,18 +620,53 @@ typedef apr_status_t (*serf_ssl_server_c const serf_ssl_certificate_t * const * certs, apr_size_t certs_len); +/** + * Set a callback to provide a filesystem path to a PKCS12 file. + * + * This has been replaced by serf_ssl_cert_uri_set(). On Unix + * platforms the same path from serf_ssl_client_cert_provider_set() + * can be passed to serf_ssl_cert_uri_set(). On Windows the drive + * letter will be interpreted by serf_ssl_cert_uri_set() as a scheme, + * so the same path will not work, and will need to be escaped as + * a file URL instead. + */ void serf_ssl_client_cert_provider_set( serf_ssl_context_t *context, serf_ssl_need_client_cert_t callback, void *data, void *cache_pool); +/** + * Set a callback to provide the password corresponding to the URL of + * the client certificate store. + * + * If the serf_ssl_client_cert_provider_set callback is set, this + * password will also be used to decode the PKCS12 file. + */ void serf_ssl_client_cert_password_set( serf_ssl_context_t *context, serf_ssl_need_cert_password_t callback, void *data, void *cache_pool); +/** + * Set a callback to provide the URL of the client certificate store. + * + * In the absence of a scheme the default scheme is file:, and the file + * can point to PKCS12, PEM or other supported certificates and keys. + * + * With the correct OpenSSL provider configured, URLs can be provided + * for pkcs11, tpm2, and other certificate stores. + * + * On Windows, file paths must be escaped as file: URLs to prevent the + * drive letter being intepreted as a scheme. + */ +void serf_ssl_cert_uri_set( + serf_ssl_context_t *context, + serf_ssl_need_cert_uri_t callback, + void *data, + void *cache_pool); + /** * Set a callback to override the default SSL server certificate validation * algorithm. Modified: serf/branches/PR-8/test/serf_get.c URL: http://svn.apache.org/viewvc/serf/branches/PR-8/test/serf_get.c?rev=1926894&r1=1926893&r2=1926894&view=diff ============================================================================== --- serf/branches/PR-8/test/serf_get.c (original) +++ serf/branches/PR-8/test/serf_get.c Tue Jul 1 08:52:03 2025 @@ -41,6 +41,7 @@ typedef struct app_baton_t { int use_h2direct; const char *pem_path; const char *pem_pwd; + const char *cert_uri; serf_bucket_alloc_t *bkt_alloc; serf_context_t *serf_ctx; } app_baton_t; @@ -80,7 +81,8 @@ static apr_status_t client_cert_pw_cb(vo { app_baton_t *ctx = data; - if (strcmp(cert_path, ctx->pem_path) == 0) + if ((ctx->cert_uri && !strcmp(cert_path, ctx->cert_uri)) || + (ctx->pem_path && !strcmp(cert_path, ctx->pem_path))) { *password = ctx->pem_pwd; return APR_SUCCESS; @@ -89,6 +91,15 @@ static apr_status_t client_cert_pw_cb(vo return APR_EGENERAL; } +static apr_status_t client_cert_uri_cb(void *data, const char **cert_uri) +{ + app_baton_t *ctx = data; + + *cert_uri = ctx->cert_uri; + + return APR_SUCCESS; +} + static void print_ssl_cert_errors(int failures) { if (failures) { @@ -235,6 +246,13 @@ static apr_status_t conn_setup(apr_socke pool); } + if (ctx->cert_uri) { + serf_ssl_cert_uri_set(conn_ctx->ssl_ctx, + client_cert_uri_cb, + ctx, + pool); + } + if (ctx->pem_pwd) { serf_ssl_client_cert_password_set(conn_ctx->ssl_ctx, client_cert_pw_cb, @@ -494,6 +512,7 @@ credentials_callback(char **username, #define CERTPWD 257 #define HTTP2FLAG 258 #define H2DIRECT 259 +#define CERTURI 260 static const apr_getopt_option_t options[] = { @@ -510,6 +529,7 @@ static const apr_getopt_option_t options {NULL, 'f', 1, "<file> Use the <file> as the request body"}, {NULL, 'p', 1, "<hostname:port> Use the <host:port> as proxy server"}, {"cert", CERTFILE, 1, "<file> Use SSL client certificate <file>"}, + {"certuri", CERTURI, 1, "<uri> Use SSL client certificate <uri>"}, {"certpwd", CERTPWD, 1, "<password> Password for the SSL client certificate"}, {NULL, 'r', 1, "<header:value> Use <header:value> as request header"}, {"debug", 'd', 0, "Enable debugging"}, @@ -564,7 +584,7 @@ int main(int argc, const char **argv) int print_headers, debug, negotiate_http2, use_h2direct; const char *username = NULL; const char *password = ""; - const char *pem_path = NULL, *pem_pwd = NULL; + const char *pem_path = NULL, *pem_pwd = NULL, *cert_uri = NULL; apr_getopt_t *opt; int opt_c; const char *opt_arg; @@ -677,6 +697,9 @@ int main(int argc, const char **argv) case CERTFILE: pem_path = opt_arg; break; + case CERTURI: + cert_uri = opt_arg; + break; case CERTPWD: pem_pwd = opt_arg; break; @@ -731,6 +754,7 @@ int main(int argc, const char **argv) app_ctx.hostname = url.hostname; app_ctx.pem_path = pem_path; app_ctx.pem_pwd = pem_pwd; + app_ctx.cert_uri = cert_uri; context = serf_context_create(pool); app_ctx.serf_ctx = context; Modified: serf/branches/PR-8/test/test_ssl.c URL: http://svn.apache.org/viewvc/serf/branches/PR-8/test/test_ssl.c?rev=1926894&r1=1926893&r2=1926894&view=diff ============================================================================== --- serf/branches/PR-8/test/test_ssl.c (original) +++ serf/branches/PR-8/test/test_ssl.c Tue Jul 1 08:52:03 2025 @@ -1173,6 +1173,76 @@ static void test_ssl_client_certificate( EndVerify } +static apr_status_t +client_cert_uri_conn_setup(apr_socket_t *skt, + serf_bucket_t **input_bkt, + serf_bucket_t **output_bkt, + void *setup_baton, + apr_pool_t *pool) +{ + test_baton_t *tb = setup_baton; + apr_status_t status; + + status = https_set_root_ca_conn_setup(skt, input_bkt, output_bkt, + setup_baton, pool); + if (status) + return status; + + serf_ssl_cert_uri_set(tb->ssl_context, + client_cert_cb, + tb, + pool); + + serf_ssl_client_cert_password_set(tb->ssl_context, + client_cert_pw_cb, + tb, + pool); + + return APR_SUCCESS; +} + +static void test_ssl_client_certificate_uri(CuTest *tc) +{ +#if defined(SERF_HAVE_OSSL_STORE_OPEN_EX) + test_baton_t *tb = tc->testBaton; + handler_baton_t handler_ctx[1]; + const int num_requests = sizeof(handler_ctx)/sizeof(handler_ctx[0]); + apr_status_t status; + + + /* Set up a test context and a https server */ + /* The SSL server uses the complete certificate chain to validate the client + certificate. */ + setup_test_mock_https_server(tb, server_key, + all_server_certs, + test_clientcert_optional); + status = setup_test_client_https_context(tb, + client_cert_uri_conn_setup, + NULL, /* No server cert callback */ + tb->pool); + CuAssertIntEquals(tc, APR_SUCCESS, status); + + Given(tb->mh) + ConnectionSetup(ClientCertificateIsValid, + ClientCertificateCNEqualTo("Serf Client")) + + GETRequest(URLEqualTo("/"), ChunkedBodyEqualTo("1"), + HeaderEqualTo("Host", tb->serv_host)) + Respond(WithCode(200), WithChunkedBody("")) + EndGiven + + create_new_request(tb, &handler_ctx[0], "GET", "/", 1); + + status = run_client_and_mock_servers_loops(tb, num_requests, handler_ctx, + tb->pool); + CuAssertTrue(tc, tb->result_flags & TEST_RESULT_CLIENT_CERTCB_CALLED); + CuAssertTrue(tc, tb->result_flags & TEST_RESULT_CLIENT_CERTPWCB_CALLED); + Verify(tb->mh) + CuAssert(tc, ErrorMessage, VerifyConnectionSetupOk); + EndVerify +#endif +} + /* Validate that the expired certificate is reported as failure in the callback. */ static void test_ssl_expired_server_cert(CuTest *tc) @@ -2752,6 +2822,7 @@ CuSuite *test_ssl(void) SUITE_ADD_TEST(suite, test_ssl_large_response); SUITE_ADD_TEST(suite, test_ssl_large_request); SUITE_ADD_TEST(suite, test_ssl_client_certificate); + SUITE_ADD_TEST(suite, test_ssl_client_certificate_uri); SUITE_ADD_TEST(suite, test_ssl_expired_server_cert); SUITE_ADD_TEST(suite, test_ssl_future_server_cert); SUITE_ADD_TEST(suite, test_ssl_revoked_server_cert);