Problems encrypting with RSA and RSA_NO_PADDING

[Patrick Mackinlay]

I am having problem encrytping and then decrypting messages using RSA 
withthe padding set to RSA_NO_PADDING. Most messages work, but some of 
them don't. By don't I mean the encryption/decryption process works fine 
except that the decrypted message is nothnig like the original! I have 
attached a C source file that demonstrates the problem.
My system is linux 2.4 and I have tried compiling the code with the 
redhat openssl-devel-0.9.6-3 rpm installed and with the lastest openssl 
tarball (dated 9-7-2001, openssl-0.9.6b.tar.gz).
Am I doing something wrong or is there a bug?

regards,
Patrick

-- 
Patrick Mackinlay                                [EMAIL PROTECTED]
ICQ: 59277981                                        tel: +44 7050699851
                                                      fax: +44 7050699852
SpaceSurfer Limited                          http://www.spacesurfer.com/

#include <openssl/ssl.h>

char *rsaMakeKeyString(char privateFlag, RSA *pKey)
 {
 BUF_MEM *pBuffer;
 char *pKeyString;
 BIO *pStringBIO = BIO_new(BIO_s_mem());

 if (pStringBIO==NULL)
  return NULL;
    
 if (privateFlag)
  PEM_write_bio_RSAPrivateKey(pStringBIO, pKey, NULL, NULL, 0, NULL, NULL);
 else
  PEM_write_bio_RSAPublicKey(pStringBIO, pKey);
    
 BIO_flush(pStringBIO);
 BIO_get_mem_ptr(pStringBIO, &pBuffer);
 /* --PIM-- Why set to noclose when it is no longer used when the function returns?
 BIO_set_close(pStringBIO, BIO_NOCLOSE); // So BIO_free() leaves BUF_MEM alone
  --/PIM-- */
 BIO_set_close(pStringBIO, BIO_CLOSE);
 pKeyString = malloc(pBuffer->length + 1);
 memcpy(pKeyString, pBuffer->data, pBuffer->length + 1);
 BIO_free(pStringBIO);

 return pKeyString;
 }

RSA* rsaLoadKey(char privateFlag, char *pKeyString)
 {
 RSA *pKey;
 BUF_MEM *pBuffer;
 BIO *pStringBio = BIO_new(BIO_s_mem());

 if (pStringBio==NULL)
  return NULL;
    
 pBuffer = BUF_MEM_new();
 BUF_MEM_grow(pBuffer, strlen(pKeyString) + 1);
 memcpy(pBuffer->data, pKeyString, strlen(pKeyString) + 1);
 BIO_set_mem_buf(pStringBio, pBuffer, BIO_CLOSE);

 if(privateFlag)
  pKey = PEM_read_bio_RSAPrivateKey(pStringBio, NULL, NULL, NULL);
 else
  pKey = PEM_read_bio_RSAPublicKey(pStringBio,NULL, NULL, NULL);

 BIO_free(pStringBio);
 if (pKey == NULL)
  return NULL;

 return pKey;
 }

char 
BuggyMessage[]="f687c3b86fe97c5cf69956c49bcbeef383849d757ad801cdfd5af57ef88c6a85416c7708a3cb727371b6b97d45bfc62abc42577865cf00e31de2402c8501f78fa36da4c4c46e6fd057a5ed978f008b897c858d6700cdb78c6a5da4bf87caf57f0ac2510e8d7f56607d9f5e3ed79d4df474426096680ae1f31548aae5fd032e03";

char 
WorkingMessage[]="6b334b5f8da27aff934f08f1df8e50b29828b896c99b3e85a7e6256d4a8c9f9dfea36a1af789c27d8a3cd50d8d0af8a1c269d525a116a347b251e7185130490edd01660a9a963dd6efb2a63af906616813c12800f86485761bbfda1c50fb7f22927b52143ef59f58368e9840cf014e5b5f03a667fea1b345353902e79ede07a2";

char *DecodeBytes(char *pInput, int length)
 {
 char *pOutput=malloc(length);
 int n;

 for(n=0; n<length; n++)
  {
  unsigned char *pV=&pOutput[n];
  *pV=(unsigned char)pInput[2*n] >= (unsigned char)'a' ? (unsigned 
char)pInput[2*n]-(unsigned char)'a'+10 : (unsigned char)pInput[2*n]-(unsigned char)'0';
  *pV=*pV<<4;
  *pV+=(unsigned char)pInput[2*n+1] >= (unsigned char)'a' ? (unsigned 
char)pInput[2*n+1]-(unsigned char)'a'+10 : (unsigned char)pInput[2*n+1]-(unsigned 
char)'0';
  }

 return pOutput;
 }


int main(int argc, char* argv[])
 {
 RSA *rsa, *rsa_pub, *rsa_priv;

 int orig_length, enc_length, dec_length;
 int i, j;
  
 unsigned char *original_message;
 unsigned char encrypted_message[255];
 unsigned char decrypted_message[255];
 char error[120];
 char *privateString;
 char *publicString;
    
 BIO *bio_out;

 // int pad = RSA_PKCS1_OAEP_PADDING;
 int pad = RSA_NO_PADDING;

 SSLeay_add_ssl_algorithms();
 SSL_load_error_strings();

 rsa = RSA_generate_key(1024, 65535, NULL, NULL);

 // printf("public modulus: %s\n", BN_bn2dec(rsa->n));
 // printf("public exponent: %s\n", BN_bn2dec(rsa->e));
 // printf("private exponent: %s\n", BN_bn2dec(rsa->d));
 // printf("p: %s\n", BN_bn2dec(rsa->p));
 // printf("q: %s\n", BN_bn2dec(rsa->q));

 printf ("RSA size: %d\n", RSA_size(rsa));

 publicString = rsaMakeKeyString(0, rsa);
 privateString = rsaMakeKeyString(1, rsa);

 printf("%s", publicString);
 printf("%s", privateString);

 rsa_pub = rsaLoadKey(0, publicString);
 rsa_priv = rsaLoadKey(1, privateString);
    /*  RSA_print_fp(stdout, rsa, 0); */
    
 // orig_length = strlen(original_message)+1;
 orig_length=128;
 original_message=DecodeBytes(BuggyMessage, orig_length);
    
 printf("Original message:\n");
 for (j = 0; j < orig_length;)
  {
  printf("%2.2x", original_message[j]);
  j++;
  if (j%32==0)
    printf("\n");
  }
 printf("\n");

 for (i= 0; i< 1; i++)
  {
  enc_length = RSA_public_encrypt(orig_length, original_message, encrypted_message, 
rsa_pub, pad);

  printf("Enclength: %d\n", enc_length);
        
  printf("Encrypted message:\n");
  for (j = 0; j < enc_length;)
   {
   printf("%2.2x", encrypted_message[j]);
   j++;
   if (j%32==0)
     printf("\n");
   }
  printf("\n");
  }
    
 ERR_print_errors_fp(stdout);
 dec_length = RSA_private_decrypt(enc_length, encrypted_message, decrypted_message, 
rsa_priv, pad);
 printf("Declength: %d\n", dec_length);
 printf("Decrypted message:\n");
 for (j = 0; j < dec_length;)
  {
  printf("%2.2x", decrypted_message[j]);
  j++;
  if (j%32==0)
    printf("\n");
  }
 printf("\n");

 if (strncmp(decrypted_message, original_message, orig_length)==0)
  printf("It worked\n");
 else
  printf("It failed\n");
 return 0;
 }