I wrote unit test to check if rsa encrypt/decrypt works properly but it fails when i check data after decryption. It appears that padding is not being removed by library.
My encrypt/decrypt functions:
int secure_crypto::rsa_decrypt(const std::vector<unsigned char> &data, EVP_PKEY *privateKey,
std::vector<unsigned char> &decrypted) {
try {
//create the context for verifying using unique_ptr
EVP_PKEY_CTX_free_ptr ctx(EVP_PKEY_CTX_new(privateKey, nullptr), ::EVP_PKEY_CTX_free);
if (ctx == nullptr) {
return -1;
}
//init the context for encryption
if (EVP_PKEY_decrypt_init(ctx.get()) <= 0) {
return -2;
}
//set the padding
if (EVP_PKEY_CTX_set_rsa_padding(ctx.get(), RSA_PKCS1_OAEP_PADDING) <= 0) {
return -3;
}
//set the digest
if (EVP_PKEY_CTX_set_rsa_oaep_md(ctx.get(), EVP_sha256()) <= 0) {
return -4;
}
//get the size of the encrypted data
size_t decryptedLen;
if (EVP_PKEY_decrypt(ctx.get(), nullptr, &decryptedLen, data.data(), data.size()) <= 0) {
return -5;
}
//resize the encrypted vector
decrypted.resize(decryptedLen);
//decrypt the data
if (EVP_PKEY_decrypt(ctx.get(), decrypted.data(), &decryptedLen, data.data(), data.size()) <= 0) {
return -6;
}
return 0;
} catch (const std::exception &e) {
#if LOG_CRYPTO_ERRORS
dbg_println("[crypto] Error while rsa decrypting: {}", e.what());
#endif
return -1;
}
}
int secure_crypto::rsa_encrypt(const std::vector<unsigned char> &data, EVP_PKEY *pubKey,
std::vector<unsigned char> &encrypted) {
try {
EVP_PKEY_CTX_free_ptr ctx(EVP_PKEY_CTX_new(pubKey, nullptr), ::EVP_PKEY_CTX_free);
if (ctx == nullptr) {
return -1;
}
//init the context for encryption
if (EVP_PKEY_encrypt_init(ctx.get()) <= 0) {
return -2;
}
//set the padding
if (EVP_PKEY_CTX_set_rsa_padding(ctx.get(), RSA_PKCS1_OAEP_PADDING) <= 0) {
return -3;
}
//set the digeste
if (EVP_PKEY_CTX_set_rsa_oaep_md(ctx.get(), EVP_sha256()) <= 0) {
return -4;
}
//check if data does not exceed the key size
if (data.empty() || data.size() > EVP_PKEY_get_size(pubKey)) {
return -7;
}
//get the size of the encrypted data
size_t encryptedLen;
if (EVP_PKEY_encrypt(ctx.get(), nullptr, &encryptedLen, data.data(), data.size()) <= 0) {
return -5;
}
//resize the encrypted vector
encrypted.resize(encryptedLen);
//encrypt the data
if (EVP_PKEY_encrypt(ctx.get(), encrypted.data(), &encryptedLen, data.data(), data.size()) <= 0) {
return -6;
}
return 0;
} catch (const std::exception &e) {
#if LOG_CRYPTO_ERRORS
dbg_println("[crypto] Error while rsa encrypting: {}", e.what());
#endif
return -1;
}
}
Unit test code
TEST(CryptoTests, TestRsaEncryptDecrypt) {
secure_string publicKey = "-----BEGIN PUBLIC KEY-----\n"
"MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAyZ+pUSCfGR5GNhpMgtPO\n"
"3TKW8NPiSvnsG3bLgVUwMCNbN1mfPamB3kGfQPtlFOqUaEv1bjsrcBAXROhjFCMS\n"
"ScxlW7rN996eBX8nVC3oMz3KXOnviTItjKOpN0CymL/GmNQwKz5Xmt2h7+0xgmPJ\n"
"uBSxo3fPwG1jbevqAIieJU/ELCHuAaRY54Oga215tuPZV8O0zsMR9ZXTW7TEpntR\n"
"CxsdICDxCIDd78Up2MRXnV2H6ugfdACjx/KvyhXeQO52l9lq0AkRVl2O5VC3s1hp\n"
"x+oRCUv23n9XZSdCL6oZjnt2PK2eT2sWW6pTMB5kmAGWvVpeMRicc/k7Q2mgaqpE\n"
"LwIDAQAB\n"
"-----END PUBLIC KEY-----";
secure_string privateKey = "-----BEGIN PRIVATE KEY-----\n"
"MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQDJn6lRIJ8ZHkY2\n"
"GkyC087dMpbw0+JK+ewbdsuBVTAwI1s3WZ89qYHeQZ9A+2UU6pRoS/VuOytwEBdE\n"
"6GMUIxJJzGVbus333p4FfydULegzPcpc6e+JMi2Mo6k3QLKYv8aY1DArPlea3aHv\n"
"7TGCY8m4FLGjd8/AbWNt6+oAiJ4lT8QsIe4BpFjng6BrbXm249lXw7TOwxH1ldNb\n"
"tMSme1ELGx0gIPEIgN3vxSnYxFedXYfq6B90AKPH8q/KFd5A7naX2WrQCRFWXY7l\n"
"ULezWGnH6hEJS/bef1dlJ0IvqhmOe3Y8rZ5PaxZbqlMwHmSYAZa9Wl4xGJxz+TtD\n"
"aaBqqkQvAgMBAAECggEAAOMTc2VG/69wXJ/hVyFQKQa9//JefaeUwnWHp7fPEKkg\n"
"Pvb03IM6UlIYITEhctx0P62Pv1MkQOqGiiaMqaF/jCkmBCrNAyWiHqj1tS+xLu9w\n"
"50LaL2Ulr5XFesJaoBM8dM4JxyA1lMbvjbGNt8+6Adr4BOZVkb7hjV+OU2wA6dyb\n"
"bE9mqCkHTjszQIaFH0KRQxVoTHhGLBcfZjaG2rnex87ZsYWRwdm6b7WRU/kGd09E\n"
"jilF1JTMx7YBAURQVliJ5AE1hcowkPkylu6OyYKc8Z7b1PepN4g4ojunv+Kybdj+\n"
"HKhiQCTBksB35kSTvUGkJCgD2sbXEnt7FkPHQp7HIQKBgQD7IPd4aRNhJdQsU3PW\n"
"d3Z3EYRBFa/Lh590tVr6aJvRrbpHdeHWLsqvYf3ZcDgsBP+ed15FrcSoytS264Iv\n"
"2coEPwHgAsdh114nAoA7V44HQLreEVPFq80006gpY+O9o0Ylh9UUVr5BoU5iakVP\n"
"ENafiidLG70Ayy7uWZECNh6G5wKBgQDNiN3a8WT1m4zE3K1k2gWjllTqWZ3D8k3y\n"
"WJD3XaeG0cTJq92donYFM49dgpX5notel0t2ycCdQDrpRYrTLLpYsky54iRXu/UH\n"
"hvHnyfC4b5g893SCgUk3NiXiSRZdA4I2+MjvI+QncYlQUbaBJP8mOu5mkQDIJtHD\n"
"nRdYnfxXeQKBgAGDelpcb8R3IOc/fYQ+ZhVjoqNtyQh2Ac0HRswZk07ZODU0Da9n\n"
"ysrx0UDdP0lsoknHOGc9G+FUj6Gmm7tEGcaaxp0eByucy7F6f2FcLrv2dOnfutyH\n"
"I4589BeDxZyf/k09m1grCayFl15ae551YIXHpGL8gNAfaWIAYjJcNyTjAoGATWtW\n"
"lhgDfXhgJ+8v6SMuqJYB2lxzTtHAhp7RzDuc1O3DAGwaiDwf1Im9YO3FbEdKHBFF\n"
"PbN1OEZvalrllfHtJ2fR+tqx8dI8098C09p/MBno8F4NpvLoD0H4NypsoBCoReik\n"
"V1/fmvPrUzPUWQUEjvP3xO6tqrbkoJhVqO2plZECgYBaxcjCbFVWdoCtYtCa0LWR\n"
"dXLBjDojE3pkaS6zFfJ2VJ1zZKRQSVuJ/kOH5wCC0ihQfjJQLqJENytL74Jyc0Zm\n"
"Mli7dgaehAI2wkSvQIrtFGI/OwDMAVEDPnu/Cto3RiBRccmYYPEMvom6yM7Ssgsr\n"
"MUu2JiK8+YNdsj4rthOyMQ==\n"
"-----END PRIVATE KEY-----";
secure_string data = "test";
secure_string encrypted;
secure_string decrypted;
EVP_PKEY *key = nullptr;
ASSERT_EQ(secure_crypto::read_public_key(publicKey, &key), 0);
ASSERT_TRUE(key);
EVP_PKEY *privKey = nullptr;
ASSERT_EQ(secure_crypto::read_private_key(privateKey, &privKey), 0);
ASSERT_TRUE(privKey);
secure_string encryptedData;
ASSERT_EQ(secure_crypto::rsa_encrypt(data, key, encryptedData), 0);
ASSERT_FALSE(encryptedData.empty());
ASSERT_EQ(encryptedData.size(), 256);
secure_string decryptedData;
ASSERT_EQ(secure_crypto::rsa_decrypt(encryptedData, privKey, decryptedData), 0);
ASSERT_EQ(decryptedData, data);
}
Result:
Expected equality of these values:
decryptedData
Which is: { 't' (116, 0x74), 'e' (101, 0x65), 's' (115, 0x73), 't' (116, 0x74), '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', ... }
data
Which is: { 't' (116, 0x74), 'e' (101, 0x65), 's' (115, 0x73), 't' (116, 0x74) }
What may be the problem here? I tried to find any solution which may resolve the issue, but found nothing. I can manually remove padding but i read that openssl should do it by itself.
From the documentation of EVP_PKEY_decrypt(EVP_PKEY_CTX *ctx, unsigned char *out, size_t *outlen, const unsigned char *in, size_t inlen):
The EVP_PKEY_decrypt() function performs a public key decryption operation using ctx. The data to be decrypted is specified using the in and inlen parameters. If out is NULL then the minimum required size of the output buffer is written to the *outlen parameter.
If out is not NULL then before the call the *outlen parameter must contain the length of the out buffer. If the call is successful the decrypted data is written to out and the amount of the decrypted data written to *outlen, otherwise an error is returned.
This means that in your first call of EVP_PKEY_decrypt() only the minimum required size for the output buffer is written into encryptedLen (for your test data this is 256 bytes which correponds to the size of the RSA key / modulus).
In your second call of EVP_PKEY_decrypt() the decrypted data is written to encrypted.data() and only now the amount of data is written to encryptedLen (which for your plaintext test is 4 bytes).
However, this last length information is not taken into account in the current code, which causes the issue. To solve the problem, it is enough to simply add a decrypted.resize(decryptedLen) after the second EVP_PKEY_decrypt() call.