My custom engine_finish method does not get called through ENGINE_finish

[Jeff Saremi]

 I'm trying to use my custom engine however I cannot get it to clean up
nicely.
For the initialization i used the sample in openssl ENGINE(3) documentation.

Here's how it goes:

    ENGINE *loadEngine()
    {
        ENGINE *e;
        e = ENGINE_by_id(MY_ENGINE_ID);
        if(!e)
            ENGINE_load_my();
        e = ENGINE_by_id(MY_ENGINE_ID); /* try again */
        if(!e)
            /* the engine isn't available */
            return NULL;
        if(!ENGINE_init(e)) {
            /* the engine couldn't initialise, release 'e' */
            ENGINE_free(e);
            return NULL;
        }
        ENGINE_set_default(e, ENGINE_METHOD_ALL);
        return e;
    }
    void unloadEngine(ENGINE *e)
    {
        /* Release the functional reference from ENGINE_init() */
        ENGINE_finish(e);
        /* Release the structural reference from ENGINE_by_id() */
        ENGINE_free(e);
        /* ENGINE_cleanup();  my engine_finish method does not get
    called unless I include this line */
    }
    void testKeyGen()
    {
       /* some openssl initialization code such as loading ciphers,
    algorithms, existing engines, establishing dynamic locks and so on */

        ENGINE *e = loadEngine();
        ASSERT(e, "could not create the engine\n");
        EVP_PKEY *pkey = NULL;
        genPKey(e, &pkey); /* some calls to EVP_PKEY_CTX_new_id,
    EVP_PKEY_keygen and so on */
        ASSERT(pkey, "could not generate PKEY\n");

        /* so far so good */
        if(pkey)
            EVP_PKEY_free(pkey);

        unloadEngine(e);
       /* at the end of this I have unfreed memory (the dynamic lock I
    created in my engine because my_finish was not called */
    }

And here's my engine initialization and deinit routines. I stepped
through the code in ENGINE_finish() and realized that the following line
(to_return = e->finish(e); ) does not get executed because apparently
the ref count is not zero:

********* file openssl/crypto/engine/eng_init.c  **************
int engine_unlocked_finish(ENGINE *e, int unlock_for_handlers)
    {
    if((e->funct_ref == 0) && e->finish)
        {
        if(unlock_for_handlers)
            CRYPTO_w_unlock(CRYPTO_LOCK_ENGINE);
        to_return = e->finish(e);
        if(unlock_for_handlers)
            CRYPTO_w_lock(CRYPTO_LOCK_ENGINE);
        if(!to_return)
            return 0;
...
    }

********* continuation of my code -- my engine init routines  *********

    static int my_init(ENGINE *e)
    {
        ERR_load_MY_strings();
        my_lock_id = CRYPTO_get_new_dynlockid();
        ...
        return 1;
    }
    static int my_finish(ENGINE *e)
    {
        ERR_unload_MY_strings();
        CRYPTO_destroy_dynlockid(my_lock_id);
        return 1;
    }
    static int bind_helper(ENGINE *e)
    {
    ...
        if (!ENGINE_set_id(e, MY_ENGINE_ID)
                || !ENGINE_set_name(e, MY_ENGINE_NAME)
                || !ENGINE_set_destroy_function(e, my_destroy)
                || !ENGINE_set_init_function(e, my_init)
                || !ENGINE_set_finish_function(e, my_finish)
    ...
            return 0;
        return 1;
    }
    static ENGINE *engine_my(void)
    {
        ENGINE *ret = ENGINE_new();
        if (!ret)
            return NULL;
        if (!bind_helper(ret))
        {
            ENGINE_free(ret);
            return NULL;
        }
        return ret;
    }
    void ENGINE_load_my(void)
    {
        ENGINE *toadd = engine_my();
        if (!toadd)
            return;
        ENGINE_add(toadd);
        ENGINE_free(toadd);
        ERR_clear_error();
    }





This email contains Morega Systems Inc. Privileged and Confidential information.