Hang in BN_MONT_CTX_set

[Ton Hospel]

openssl 0.9.8 on linux:

BN_MONT_CTX_set hangs if the argument is 0

Demonstration program:

#include <stdio.h>
#include <stdlib.h>
#include <openssl/bn.h>

void error(int line) {
    fprintf(stderr, "Error on line %d\n", line);
    exit(1);
}

int main(void) {
    BIGNUM m;
    BN_MONT_CTX mont;
    BN_CTX *ctx;

    BN_init(&m);
    if (!BN_zero(&m)) error(__LINE__);
    ctx = BN_CTX_new();
    if (!ctx) error(__LINE__);
    BN_MONT_CTX_init(&mont);

    fprintf(stderr, "Now on line %d\n", __LINE__);
    if (!BN_MONT_CTX_set(&mont, &m, ctx)) error(__LINE__);
    fprintf(stderr, "Now on line %d\n", __LINE__);

    BN_MONT_CTX_free(&mont);
    BN_CTX_free(ctx);
    BN_free(&m);
    return 0;
}

====================================

The essence of the hang can be reduced to:
where BN_mod_inverse doesn't seem to like the quick and dirty constructed
tmod:

int test_set(BN_CTX *ctx) {
    int ret = 0;
    BN_ULONG buf[2];
    BIGNUM *R, *Ri;
    BIGNUM tmod;

    BN_CTX_start(ctx);
    if((Ri = BN_CTX_get(ctx)) == NULL) goto err;
    R= BN_new();                                        /* grab RR as a temp */
    if (!R) goto err;
    BN_zero(R);
    if (!(BN_set_bit(R,BN_BITS2))) goto err;    /* R */

    buf[0]=0;
    buf[1]=0;
    tmod.d=buf;
    tmod.top=1;
    tmod.dmax=2;
    tmod.neg=0;
    fprintf(stderr, "File %s, line %d\n", __FILE__, __LINE__);
    BN_mod_inverse(Ri,R,&tmod,ctx);
    fprintf(stderr, "File %s, line %d\n", __FILE__, __LINE__);
  err:
    return 1;
}
______________________________________________________________________
OpenSSL Project                                 http://www.openssl.org
Development Mailing List                       openssl-dev@openssl.org
Automated List Manager                           [EMAIL PROTECTED]