[openssl.org #3392] EVP_SignFinal smashes the stack with RSA key. RSA key provides n,e,d,p,q.

[noloa...@gmail.com via RT]

EVP_SignFinal smashes the stack with RSA key. RSA key provides
n,e,d,p,q. RSA_check_key OK.

p and q were solved from n,e,d offline because the key check failed without it.

*****

(gdb) r
Starting program: /home/jwalton/openssl-test.exe
Signature: 78f2c9af23b9a2a42e3b57dec454fa43ea6627992f48d40a33da6a7c93f98b4
*** stack smashing detected ***: /home/jwalton/openssl-test.exe terminated

Program received signal SIGABRT, Aborted.
0x00007ffff7156f79 in __GI_raise (sig=sig@entry=6)
    at ../nptl/sysdeps/unix/sysv/linux/raise.c:56
56    ../nptl/sysdeps/unix/sysv/linux/raise.c: No such file or directory.

*****

Ubuntu 14.04 x64 fully patched. It should be the security-equivalent
of 1.0.1h. I'm not sure what other improvements Ubuntu has provided.

$ uname -a
Linux ubuntu 3.13.0-29-generic #53-Ubuntu SMP Wed Jun 4 21:00:20 UTC
2014 x86_64 x86_64 x86_64 GNU/Linux
$ openssl version
OpenSSL 1.0.1f 6 Jan 2014


// g++ -Wall -Wextra -Wconversion -g3 -std=c++11 openssl-test.cpp -o openssl-test.exe -lssl -lcrypto

#include <iostream>
using std::cout;
using std::cerr;
using std::endl;

#include <string>
using std::string;

#include <memory>
using std::unique_ptr;

#include <openssl/bn.h>
#include <openssl/rsa.h>
#include <openssl/evp.h>
#include <openssl/err.h>

const char nz[] =
	"20446702916744654562596343388758805860065209639960173505037453331270270518732245"
	"08977372301204320323609709562340204469011575537734525469644875960570778896584888"
	"95017468362112062706438336639499925362469853626937363871851454247879222415857219"
	"92924045675229348655595626434390043002821512765630397723028023792577935108185822"
	"75369257422156693093780503115582009714681996492027000881132703628678639279359312"
	"17624250488602118597634417704467037220158572506211078553986931332640811506974231"
	"88751482418465308470313958250757758547155699749157985955379381294962058862159085"
	"915015369381046959790476428631998204940879604226680285601";

const char ez[] = "65537";

const char dz[] =
	"23583109899396195101799862623499368829246520235662137651186064319555667005065389"
	"11356936879137503597382515919515633242482643314423192704128296593672966061810149"
	"31632061789402182278402640746140338406535182197235078430096761014345948432406842"
	"76746396884059179774424728049430754391920261073195321175575450790865379829879825"
	"22396626690057355718157403493216553255260857777965627529169195827622139772389760"
	"13057175483467867984218114225248961766503010944557397801270779301059273764049922"
	"00150833924259148778478404572782464027609558833769999511998277062853834711506435"
	"61410605789710883438795588594095047409018233862167884701";

const char pz[] =
	"15737705590244743839558616502896029191493197327877753279847020015603526753735923"
	"90718294084119093232085749598005372477289597182368848096852332845373492076546615"
	"30801859889389455120932077199406250387226339056140578989122526711937239401762061"
	"949364440402067108084155200696015505170135950332209194782224750221639";

const char qz[] =
	"12992175256740635899099334754006444501823007340248226099417932857332386190837921"
	"12746269565434716649972371852989646481333243433270528522640603220881224011247812"
	"49085873464824282666514908127141915943024862618996371026577302203267804867959037"
	"802770797169483022132210859867700312376409633383772189122488119155159";

using BN_ptr = std::unique_ptr<BIGNUM, decltype(&::BN_free)>;
using RSA_ptr = std::unique_ptr<RSA, decltype(&::RSA_free)>;
using EVP_PKEY_ptr = std::unique_ptr<EVP_PKEY, decltype(&::EVP_PKEY_free)>;
using EVP_MD_CTX_ptr = std::unique_ptr<EVP_MD_CTX, decltype(&::EVP_MD_CTX_destroy)>;

#define UNUSED(x) ((void)x)

int main(int argc, char* argv[])
{
	UNUSED(argc); UNUSED(argv);

	int rc;
	long err;

	RSA_ptr rsa(RSA_new(), ::RSA_free);
	BIGNUM *n = NULL, *e = NULL, *d = NULL, *p = NULL, *q = NULL;

	rc = BN_dec2bn(&n, nz);
	if(rc == 0 || n == NULL) {
		cerr << "BN_dec2bn failed for n" << endl;
		exit(1);
	}
	rsa->n = n;

	rc = BN_dec2bn(&e, ez);
	if(rc == 0 || e == NULL) {
		cerr << "BN_dec2bn failed for e" << endl;
		exit(1);
	}
	rsa->e = e;

	rc = BN_dec2bn(&d, dz);
	if(rc == 0 || d == NULL) {
		cerr << "BN_dec2bn failed for d" << endl;
		exit(1);
	}
	rsa->d = d;

	rc = BN_dec2bn(&p, pz);
	if(rc == 0 || p == NULL) {
		cerr << "BN_dec2bn failed for p" << endl;
		exit(1);
	}
	rsa->p = p;

	rc = BN_dec2bn(&q, qz);
	if(rc == 0 || q == NULL) {
		cerr << "BN_dec2bn failed for q" << endl;
		exit(1);
	}
	rsa->q = q;

	rc = RSA_check_key(rsa.get());
	err = ERR_get_error();
	if(rc != 1) {
		cerr << "RSA_check_key failed, error 0x" << std::hex << err << endl;
		exit(1);
	}

	EVP_PKEY_ptr pkey(EVP_PKEY_new(), ::EVP_PKEY_free);
	
	rc = EVP_PKEY_set1_RSA(pkey.get(), rsa.get());
	err = ERR_get_error();
	if(rc != 1) {
		cerr << "EVP_PKEY_set1_RSA failed, error 0x" << std::hex << err << endl;
		exit(1);
	}

	EVP_MD_CTX_ptr ctx(EVP_MD_CTX_create(), ::EVP_MD_CTX_destroy);
	EVP_MD_CTX_init(ctx.get());

	const EVP_MD* md = EVP_sha256();
	rc = EVP_SignInit(ctx.get(), md);
	err = ERR_get_error();
	if(rc != 1) {
		cerr << "EVP_SignInit_ex failed, error 0x" << std::hex << err << endl;
		exit(1);
	}

	const char message[] = "Now is the time for all good men...";
	rc = EVP_SignUpdate(ctx.get(), message, (unsigned int)sizeof(message));
	err = ERR_get_error();
	if(rc != 1) {
		cerr << "EVP_SignUpdate failed, error 0x" << std::hex << err << endl;
		exit(1);
	}

	unsigned char signature[EVP_MAX_MD_SIZE];
	unsigned int length = (unsigned int)sizeof(signature);
	rc = EVP_SignFinal(ctx.get(), signature, &length, pkey.get());
	err = ERR_get_error();
	if(rc != 1) {
		cerr << "EVP_SignFinal failed, error 0x" << std::hex << err << endl;
		exit(1);
	}

	unsigned int len = std::min(EVP_MD_size(md), EVP_PKEY_size(pkey.get()));
	cout << "Signature: ";

	for(unsigned i = 0; i < len; i++)
		cout << std::hex << (signature[i] & 0xFF);
	cout << endl;

	return 0;
}