How to create EVP_PKEY from public key string in C

I am trying to perform string data encryption in C using OpenSSL with a public key shared with me. I have searched online but I haven't had much success finding helpful information.

My encryption should use RSA PKCS1 Padding with SHA1 OAEP hash.

I created this function to encrypt

int rsaEvpEncrypt(EVP_PKEY* pkey, const char * data){
    EVP_PKEY_CTX * ctx = NULL;
    //EVP_PKEY * pkey = NULL;
    unsigned char * encrypted = NULL;
    size_t outlen = 0;
    const size_t inlen = strlen(data);

    tappa_printf("Print %d", 1);
    ctx = EVP_PKEY_CTX_new(pkey, NULL);
    tappa_printf("Print %d", 2);
    if (!ctx) {
        cleanup(ctx, NULL, NULL, NULL);
        return 1000;
    }

    // Init encryption context.
    if (EVP_PKEY_encrypt_init(ctx) <= 0) { // <-SEGFAULT LINE
        cleanup(ctx, pkey, encrypted, NULL);
        return 5;
    }

    tappa_printf("Print %d", 3);
    // Defines padding
    if (EVP_PKEY_CTX_set_rsa_padding(ctx, RSA_PKCS1_OAEP_PADDING) <= 0) {
        cleanup(ctx, pkey, encrypted, NULL);
        return 6;
    }

    // Evaluating size for encrypted buffer and encrypting
    if (EVP_PKEY_encrypt(ctx, NULL, &outlen, (const unsigned char *) data, inlen) <= 0) {
        cleanup(ctx, pkey, encrypted, NULL);
        return 7;
    }

    encrypted = (unsigned char *)OPENSSL_malloc(outlen);
    tappa_printf("Print %d", 5);
    if (!encrypted) {
        cleanup(ctx, pkey, encrypted, NULL);
        return 8;
    }

    tappa_printf("Print %d", 6);
    if (EVP_PKEY_encrypt(ctx, encrypted, &outlen, (const unsigned char *)data, inlen) <= 0) {
        cleanup(ctx, pkey, encrypted, NULL);
        return 9;
    }

    tappa_print("\"ENCRYPTED: %s", (char *)encrypted);
}

and this function to generate a random key

int rsaEvpKeyGen(EVP_PKEY **outPK){
    EVP_PKEY_CTX * ctx = NULL;
    EVP_PKEY * pkey = NULL;
    EVP_PKEY * rkey = NULL;

    // const size_t inlen = strlen(argv[1]);
    size_t outlen = 0;
    size_t doutlen = 0;

    unsigned char * encrypted = NULL;
    unsigned char * decrypted = NULL;

    // Create context
     ctx = EVP_PKEY_CTX_new_id(EVP_PKEY_RSA, NULL);
     if (!ctx) {
         cleanup(ctx, pkey, encrypted, decrypted);
         return 1;
     }
     // Init key generator
     if (EVP_PKEY_keygen_init(ctx) <= 0) {
         cleanup(ctx, pkey, encrypted, decrypted);
         return 2;
     }
     int KEY_BITS = 2048;
     // Configure key generation
     if (EVP_PKEY_CTX_set_rsa_keygen_bits(ctx, KEY_BITS) <= 0) {
         cleanup(ctx, pkey, encrypted, decrypted);
         return 3;
     }
 
     // Create keys
     if (EVP_PKEY_keygen(ctx, &pkey) <= 0) {
         cleanup(ctx, pkey, encrypted, decrypted);
         return 4;
     }
     
    *outPK =pkey;
    return 0;
}

The code works as expected and encrypts the data as expected. However for my application to work, I need to be able to use a Public key generated externally.

In the following code, I attempt to create an EVP_PKEY from pub_key_str using EVP_PKEY_new_raw_public_key but it returns NULL

int rsaEvpKeyGen(EVP_PKEY **outPK){
    EVP_PKEY_CTX * ctx = NULL;
    EVP_PKEY * pkey = NULL;
    EVP_PKEY * rkey = NULL;

    // const size_t inlen = strlen(argv[1]);
    size_t outlen = 0;
    size_t doutlen = 0;

    unsigned char * encrypted = NULL;
    unsigned char * decrypted = NULL;
    const char *pub_key_str = "-----BEGIN PUBLIC KEY-----\n"
                              "MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA1WRkqlsZbmaNWbOZr/M4\n"
                              "ddwXKjmHUJ8zn9hy0WE15A8T4xjNuzJ/ATKbz20/2wVgFEGTWPNCUjkUvFzr3mhN\n"
                              "v/YJXo5iXQDP3M6rVv/w3SXaiR9C8yhDxWUz7WX/clmj7vDvk5Iydgq07gVScx+\n"
                              "5DBVbGAx10jHSDSa7f/NXMn9yNHad4hN0i1l+tsTB39CFsNeVFr349y4R8HA5Zma\n"
                              "EKseLc8iKzkwEQcuyMn4znaFpnOL0CmSrYB5K1E9zmmtDhMvDs540ZotcH/xpJiV\n"
                              "XEdQOWGFN6rryeAKP8y+sMRfQLxoiTg37YfsxTscf0gzKBizoamTF3TjRUNM55aE\n"
                              "BwIDAQAB\n"
                              "-----END PUBLIC KEY-----";

    pkey = EVP_PKEY_new_raw_public_key(EVP_PKEY_RSA,NULL, (const unsigned char *) pub_key_str,
                                       strlen(pub_key_str) );

    tappa_printf("PKEY is null %d",pkey == NULL);
  
    *outPK =pkey;
    
    return 0;
}

The posted public key seems to be invalid. Check for a copy/paste error or compare the key with the other side.

EVP_PKEY_new_raw_public_key() is intended for the import of raw keys as they are used e.g. in the context of Ed25519 or X25519.

A public PEM key can be imported e.g. as follows:

const char* spkiPem = "-----BEGIN PUBLIC KEY-----\n"
    "MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAunF5aDa6HCfLMMI/MZLT\n"
    "5hDk304CU+ypFMFiBjowQdUMQKYHZ+fklB7GpLxCatxYJ/hZ7rjfHH3Klq20/Y1E\n"
    "bYDRopyTSfkrTzPzwsX4Ur/l25CtdQldhHCTMgwf/Ev/buBNobfzdZE+Dhdv5lQw\n"
    "KtjI43lDKvAi5kEet2TFwfJcJrBiRJeEcLfVgWTXGRQn7gngWKykUu5rS83eAU1x\n"
    "H9FLojQfyia89/EykiOO7/3UWwd+MATZ9HLjSx2/Lf3g2jr81eifEmYDlri/OZp4\n"
    "OhZu+0Bo1LXloCTe+vmIQ2YCX7EatUOuyQMt2Vwx4uV+d/A3DP6PtMGBKpF8St4i\n"
    "GwIDAQAB\n"
    "-----END PUBLIC KEY-----";

const unsigned char* key = (const unsigned char*)spkiPem;
size_t keylen = strlen(spkiPem);

OSSL_DECODER_CTX* dctx;
EVP_PKEY* pkey = NULL;

dctx = OSSL_DECODER_CTX_new_for_pkey(&pkey, "PEM", NULL, "RSA", OSSL_KEYMGMT_SELECT_PUBLIC_KEY, NULL, NULL);
if (dctx == NULL) {
    // Error handling
}

if (!OSSL_DECODER_from_data(dctx, &key, &keylen)) {
    // Error handling
}

// Use pkey
...

The functions used here are available as of OpenSSL 3.0, for more details see OSSL_DECODER_CTX_new_for_pkey and OSSL_DECODER_from_data().

For the sake of completeness: To import a private RSA key, use OSSL_KEYMGMT_SELECT_KEYPAIR (instead of OSSL_KEYMGMT_SELECT_PUBLIC_KEY) as selector. For DER encoded keys, the input type "DER" (instead of "PEM") must be applied. Examples can be found e.g. in the documentation for OSSL_DECODER_from_data().