copensslrsalibcryptolibssl

C code encrypted file cannot be decrypted using openssl

This is the code that encrypts the file test.txt to test.enc. Key pair key.pem and key.pub was generated using openssl:

openssl genrsa -out key.pem
openssl rsa -in key.pem -out key.pub -pubout

#include <stdio.h>
#include <stdlib.h>

#include <limits.h>
#include <dirent.h>

#include <openssl/bio.h>
#include <openssl/evp.h>
#include <openssl/pem.h>

#define BUFFER_SIZE 1024

static int evp_aes_encrypt(char *in_path, char *out_path, EVP_PKEY *pkey)
{
    FILE *in_file = fopen(in_path, "rb");
    if (!in_file)
        return -1;

    FILE *out_file = fopen(out_path, "wb");
    if (!out_file)
    {
        fclose(in_file);
        return -1;
    }

    EVP_CIPHER_CTX* ctx = EVP_CIPHER_CTX_new();
    if (!ctx)
    {
        fclose(in_file);
        fclose(out_file);
        return -1;
    }

    int len;
    unsigned char iv[EVP_MAX_IV_LENGTH];
    int i;
    unsigned char* ek = NULL;

    if (EVP_SealInit(ctx, EVP_aes_256_cbc(), &ek, &len, iv, &pkey, 1) != 1)
    {
        printf("1\n");
        EVP_CIPHER_CTX_free(ctx);
        fclose(in_file);
        fclose(out_file);
        return -1;
    }

    unsigned char in_buffer[BUFFER_SIZE];
    unsigned char out_buffer[BUFFER_SIZE + EVP_MAX_IV_LENGTH];

    int bytes_read, bytes_written;
    while ((bytes_read = fread(in_buffer, 1, BUFFER_SIZE, in_file)) > 0)
    {
        if (EVP_SealUpdate(ctx, out_buffer, &bytes_written, in_buffer, bytes_read) != 1) {
            EVP_CIPHER_CTX_free(ctx);
            fclose(in_file);
            fclose(out_file);
            return -1;
        }
        fwrite(out_buffer, 1, bytes_written, out_file);
    }

    if (EVP_SealFinal(ctx, out_buffer, &bytes_written) != 1) {
        EVP_CIPHER_CTX_free(ctx);
        fclose(in_file);
        fclose(out_file);
        return -1;
    }

    fwrite(out_buffer, 1, bytes_written, out_file);

    EVP_CIPHER_CTX_free(ctx);
    fclose(in_file);
    fclose(out_file);

    return 0;
}

int main(void)
{
    FILE *pub = fopen("key.pub", "rb");
    EVP_PKEY *pkey = PEM_read_PUBKEY(pub, NULL, NULL, NULL);
    evp_aes_encrypt("test.txt", "test.enc", pkey);
    return 0;
}

Encrypted file then is being decrypted using openssl command:

openssl rsautl -in test.enc -out test.dec -inkey key.pem -decrypt

Then this error appears:

RSA operation error
407D290301000000:error:0200009F:rsa routines:RSA_padding_check_PKCS1_type_2:pkcs decoding error:crypto/rsa/rsa_pk1.c:269:
407D290301000000:error:02000072:rsa routines:rsa_ossl_private_decrypt:padding check failed:crypto/rsa/rsa_ossl.c:499:

Seems like there is a problem with padding, but I have no idea how to fix it. Thanks in advance.

Solution

  • "Seems like there is a problem with padding," -- actually the problem is you did only half the job.

    EVP_Seal/Open does hybrid encryption, also called envelope (or enveloped) encryption. This is described almost correctly in wikipedia but I will rephrase to match the OpenSSL implementation and my preferences.

    Encryption consists of three steps and two different encryptions:

    1. generate a nonce key for the symmetric algorithm (called DEK)
    2. encrypt DEK using RSA with recipient's public key (which you must already have, so this might be considered step -1, but it doesn't need to be repeated for each message)
    3. encrypt the data using a symmetric cipher (in your example AES-256-CBC) using DEK and depending on the mode sometimes random IV/nonce (CBC uses an IV); also depending on the mode this may additionally produce an authentication tag (CBC does not)

    Encryption steps 1 and 2 can be done in either order, but I believe this order is more common and it is definitely the one used in OpenSSL (SealInit does 0 and 1, SealUpdate and SealFinal together do 2).

    To decrypt you need all 2-4 data items produced by steps 1 and 2: RSA-encrypted-DEK, IV if applicable, encrypted data, and tag if applicable. In real systems there are data structures that package up these data items, plus additional metadata: wikipedia mentions PKCS7 (nowadays replaced by CMS, and its variant SMIME) and PGP; there are also JOSE/JWE, XMLenc, and more. Decryption similarly requires two steps:

    1. decrypt transmitted encrypted-DEK using RSA and locally-known private key
    2. decrypt transmitted encrypted-data using same symmetric algorithm, decrypted DEK, if applicable transmitted IV/nonce, and if applicable transmitted tag

    (and this time you must do them in this order).

    However, to decrypt with OpenSSL commandline you need these items in separate files (except the tag for an authenticating mode, which commandline enc can't handle at all). Thus I modified your program to write encrypted-DEK, IV, and ciphertext to 3 files, as well as refactoring slightly to reduce the clutter, changing filenames to avoid conflict, and adding minimal error handling:

    // SO76696165
#include <stdio.h>
#include <stdlib.h>

//#include <openssl/bio.h>
#include <openssl/evp.h>
#include <openssl/pem.h>
#include <openssl/err.h>

#define BUFFER_SIZE 1024

static int evp_aes_encrypt(FILE *fin, FILE *fek, FILE *fiv, FILE *fenc, EVP_PKEY *pkey)
{
    EVP_CIPHER_CTX* ctx = EVP_CIPHER_CTX_new();
    if (!ctx) return -1;

    unsigned char iv[EVP_MAX_IV_LENGTH];
    unsigned char* ek = malloc(EVP_PKEY_get_size(pkey));
    int ekl, ivl;

    if (EVP_SealInit(ctx, EVP_aes_256_cbc(), &ek, &ekl, iv, &pkey, 1) != 1)
    { EVP_CIPHER_CTX_free(ctx); free(ek); return -2; }
    ivl = 16; // or EVP_CIPHER_CTX_get_iv_length(EVP_aes_256_cbc());
    fwrite (ek,ekl,1,fek); fwrite (iv,ivl,1,fiv);

    unsigned char in_buffer[BUFFER_SIZE];
    unsigned char out_buffer[BUFFER_SIZE + EVP_MAX_IV_LENGTH];
    // actually this only needs + cipher_blocksize, but bytes are cheap now

    int bytes_read, bytes_written;
    while ((bytes_read = fread(in_buffer, 1, BUFFER_SIZE, fin)) > 0)
    {
        if (EVP_SealUpdate(ctx, out_buffer, &bytes_written, in_buffer, bytes_read) != 1) {
            EVP_CIPHER_CTX_free(ctx); free(ek); return -3; }
        fwrite(out_buffer, 1, bytes_written, fenc);
    }

    if (EVP_SealFinal(ctx, out_buffer, &bytes_written) != 1) {
        EVP_CIPHER_CTX_free(ctx); free(ek); return -4; }

    fwrite(out_buffer, 1, bytes_written, fenc);
    EVP_CIPHER_CTX_free(ctx); free(ek); 
    return 0;
}

int main(void)
{
    FILE *pub = fopen("76696165.pub", "rb");
    EVP_PKEY *pkey = PEM_read_PUBKEY(pub, NULL, NULL, NULL);
    FILE *fin = fopen("76696165.in", "rb");
    FILE *fek = fopen("76696165.ek", "wb");
    FILE *fiv = fopen("76696165.iv", "wb");
    FILE *fenc = fopen("76696165.enc", "wb");
    int err = evp_aes_encrypt(fin, fek, fiv, fenc, pkey);
    if( err ){ printf ("error in step %d\n", err);
        ERR_print_errors_fp(stdout); return 1; }
    // fclose not needed when exiting normally
    return 0;
}

    After running this I have:

    $ ls -rt1 76696165*
76696165.pub
76696165.prv
76696165.c
76696165.exe
76696165.in
76696165.iv
76696165.enc
76696165.ek
$ # STEP 1
$ openssl rsautl -decrypt -inkey 76696165.prv <76696165.ek >76696165.dek
$ xxd 76696165.dek; xxd 76696165.iv
0000000: a46e 5084 e28b 8f9c 7fe3 a465 fbeb 4ed7  .nP........e..N.
0000010: 7c4b 38f5 3174 f905 3034 52df d992 da3d  |K8.1t..04R....=
0000000: 959b a611 3fbb 5486 b6c6 2056 f9ee 6f03  ....?.T... V..o.
$ # STEP 2
$ keyhex=$(xxd -p -c32 <76696165.dek) ivhex=$(xxd -p <76696165.iv) 
$ openssl enc -aes-256-cbc -d -K $keyhex -iv $ivhex <76696165.enc
Once upon a midnight dreary, while I pondered, weak and weary,
Over many a quaint and curious volume of forgotten lore—
While I nodded, nearly napping, suddenly there came a tapping,
As of some one gently rapping, rapping at my chamber door.
"'Tis some visiter," I muttered, "tapping at my chamber door—
            Only this and nothing more."
-- Edgar Allan Poe