I fixed the problem. I'm listing it here in case someone else runs into
this:
In the loadEngine() I call ENGINE_set_default(). I have to the opposite
when unloading my engine.
Unfortunately, I could not find any "ENGINE_unset_default()" or
"ENGINE_unregister_all()" so I had to explicitly unregister my method
pointers. So my new unloadEngine looks like the following:
void unloadEngine(ENGINE *e)
{
ENGINE_unregister_pkey_asn1_meths(e);
ENGINE_unregister_pkey_meths(e);
ENGINE_unregister_RSA(e);
ENGINE_remove(e);
/* Release the functional reference from ENGINE_init() */
ENGINE_finish(e);
/* Release the structural reference from ENGINE_by_id() */
ENGINE_free(e);
}
On 10-08-13 09:23 AM, Jeff Saremi wrote:
> I'm trying to use my custom engine however I cannot get it to clean up
> nicely.
> For the initialization i used the sample in openssl ENGINE(3) documentation.
>
> Here's how it goes:
>
> ENGINE *loadEngine()
> {
> ENGINE *e;
> e = ENGINE_by_id(MY_ENGINE_ID);
> if(!e)
> ENGINE_load_my();
> e = ENGINE_by_id(MY_ENGINE_ID); /* try again */
> if(!e)
> /* the engine isn't available */
> return NULL;
> if(!ENGINE_init(e)) {
> /* the engine couldn't initialise, release 'e' */
> ENGINE_free(e);
> return NULL;
> }
> ENGINE_set_default(e, ENGINE_METHOD_ALL);
> return e;
> }
> void unloadEngine(ENGINE *e)
> {
> /* Release the functional reference from ENGINE_init() */
> ENGINE_finish(e);
> /* Release the structural reference from ENGINE_by_id() */
> ENGINE_free(e);
> /* ENGINE_cleanup(); my engine_finish method does not get
> called unless I include this line */
> }
> void testKeyGen()
> {
> /* some openssl initialization code such as loading ciphers,
> algorithms, existing engines, establishing dynamic locks and so on */
>
> ENGINE *e = loadEngine();
> ASSERT(e, "could not create the engine\n");
> EVP_PKEY *pkey = NULL;
> genPKey(e, &pkey); /* some calls to EVP_PKEY_CTX_new_id,
> EVP_PKEY_keygen and so on */
> ASSERT(pkey, "could not generate PKEY\n");
>
> /* so far so good */
> if(pkey)
> EVP_PKEY_free(pkey);
>
> unloadEngine(e);
> /* at the end of this I have unfreed memory (the dynamic lock I
> created in my engine because my_finish was not called */
> }
>
> And here's my engine initialization and deinit routines. I stepped
> through the code in ENGINE_finish() and realized that the following line
> (to_return = e->finish(e); ) does not get executed because apparently
> the ref count is not zero:
>
> ********* file openssl/crypto/engine/eng_init.c **************
> int engine_unlocked_finish(ENGINE *e, int unlock_for_handlers)
> {
> if((e->funct_ref == 0) && e->finish)
> {
> if(unlock_for_handlers)
> CRYPTO_w_unlock(CRYPTO_LOCK_ENGINE);
> to_return = e->finish(e);
> if(unlock_for_handlers)
> CRYPTO_w_lock(CRYPTO_LOCK_ENGINE);
> if(!to_return)
> return 0;
> ...
> }
>
> ********* continuation of my code -- my engine init routines *********
>
> static int my_init(ENGINE *e)
> {
> ERR_load_MY_strings();
> my_lock_id = CRYPTO_get_new_dynlockid();
> ...
> return 1;
> }
> static int my_finish(ENGINE *e)
> {
> ERR_unload_MY_strings();
> CRYPTO_destroy_dynlockid(my_lock_id);
> return 1;
> }
> static int bind_helper(ENGINE *e)
> {
> ...
> if (!ENGINE_set_id(e, MY_ENGINE_ID)
> || !ENGINE_set_name(e, MY_ENGINE_NAME)
> || !ENGINE_set_destroy_function(e, my_destroy)
> || !ENGINE_set_init_function(e, my_init)
> || !ENGINE_set_finish_function(e, my_finish)
> ...
> return 0;
> return 1;
> }
> static ENGINE *engine_my(void)
> {
> ENGINE *ret = ENGINE_new();
> if (!ret)
> return NULL;
> if (!bind_helper(ret))
> {
> ENGINE_free(ret);
> return NULL;
> }
> return ret;
> }
> void ENGINE_load_my(void)
> {
> ENGINE *toadd = engine_my();
> if (!toadd)
> return;
> ENGINE_add(toadd);
> ENGINE_free(toadd);
> ERR_clear_error();
> }
>
>
>
>
>
> This email contains Morega Systems Inc. Privileged and Confidential
> information.
This email contains Morega Systems Inc. Privileged and Confidential information.
