I haven't find solution by myself and need help.
I'm writting simple client-server encrypted chat programm using OpenSSL 3 library.
Client's and server's socket parts both inherited via 'abstract_secure_socket' pure virtual class, where SSL RSA keypairs generation are encapsulated (public key for encryption, private for decryption).
Firstly, in first launch, keys generated via EVP_RSA_gen() to files:
#define _RSA_KEY_LEN 2136 // 267 length of encr. message (256 usefull symbols (0-255) + 11 padding)
typedef unsigned char uchar;
bool abstract_secure_socket::generateKeys()
{
int bytes = 0;
std::string __err = _ERR "abstract_secSoc_generateKeys()" ;
EVP_PKEY* temp_RSAkeypair = EVP_RSA_gen(_RSA_KEY_LEN);
if(nullptr == temp_RSAkeypair){
//fprintf(stderr,"error: rsa pub gen\n");
std::cerr << __err + "RSA keypair gen error" << std::endl;
return false;
}
FILE*fp = nullptr;
bool __res = false;
do{
// PUBLIC KEY
fp = fopen(encr_pub_key_path_.c_str(),"wt");
if(nullptr == fp){
std::cerr << __err + "pub key file write error" << std::endl;
// perror("pub key file write error");
// return false;
break;
}
bytes = PEM_write_PUBKEY(fp,temp_RSAkeypair);
fclose(fp);
std::cout << "PEM_write_PUBKEY() result::" << bytes << std::endl;
if(!bytes){
std::cerr << __err + "PEM_write public" << std::endl;
break;
}
// PRIVATE KEY
fp = fopen(encr_priv_key_path_.c_str(),"wt");
if(nullptr == fp){
std::cerr << __err + "priv key file write error" << std::endl;
// perror("priv key file write error");
// return false;
break;
}
bytes = PEM_write_PrivateKey(fp,temp_RSAkeypair,NULL,NULL,0,NULL,NULL);
fclose(fp);
std::cout << "PEM_write_PrivateKey() result::" << bytes << std::endl;
if(!bytes){
std::cerr << __err + "PEM_write private" << std::endl;
break;
}
__res = true;
}while(false);
// temp memory alloc free
EVP_PKEY_free(temp_RSAkeypair);
return /*true*/__res;
}
After that i'm write keys to RAM (OpenSSL's 'BIO' struct) by BIO_write() and PEM_read_bio_PUBKEY()/PEM_read_bio_PrivateKey():
bool abstract_secure_socket::initInternalKeys()
{
std::string __err = _ERR "initInternalKeys()";
if ( !fs::exists( encr_priv_key_path_) || !fs::exists(encr_pub_key_path_) )
{
std::cout << "Missing server key/s file/s!! Let's gen new keypairs.." << std::endl;
if(!generateKeys()){
std::cerr << __err + "->generateKeys()" << std::endl;
return false;
}
std::cout << _SUC "New keys ready.";
}
BIO* __temp_bio = BIO_new(BIO_s_mem());
if(nullptr == __temp_bio){
std::cerr << __err << std::endl;
return false;
}
bool __res = false;
do{
std::string __temp_key_str = CFileRead(encr_priv_key_path_);
if(trim_copy(__temp_key_str).empty()){
//std::cerr << __err + "file: '" << encr_priv_key_path_ << "' is empty." << std::endl;
std::cerr << __err + "->CFileRead() private key file is empty." << std::endl;
}
//__s_key = CFileRead(encr_priv_key_path_);
if(!str2privKey(__temp_key_str, &encr_priv_key_, __temp_bio)){
std::cerr << _D "abstract_secSoc:: str2privKey FAILED" << std::endl;
break;
}
/*__s_key*/ encr_pub_key_str_ = CFileRead(encr_pub_key_path_);
if(trim_copy(/*__s_key*/encr_pub_key_str_).empty()){
//std::cerr << __err + "file: '" << encr_pub_key_path_ << "' is empty." << std::endl;
std::cerr << __err + "->CFileRead() public key file is empty." << std::endl;
}
if(!str2pubKey(/*__s_key*/encr_pub_key_str_,&encr_pub_key_, __temp_bio)){
std::cerr << _D "abstract_secSoc:: str2pubKey FAILED" << std::endl;
break;
}
__res = true;
}while(false);
BIO_free(__temp_bio);
return __res;
}
bool abstract_secure_socket::str2privKey(std::string const& privCipher__, EVP_PKEY**privKeyPtr, BIO* SSL_mem_bio__)
{
std::string __err = _ERR "str2privKey()";
bool __res = false;
// BIO* __temp_bio = BIO_new(BIO_s_mem());
if(nullptr == /*__temp_bio*/SSL_mem_bio__){
std::cerr << __err << std::endl;
return false;
}
do{
int __bio_data_written = BIO_write(SSL_mem_bio__/*__temp_bio*/, privCipher__.c_str(), privCipher__.length());
if(__bio_data_written <= 0){
std::cout << __err + "bio_write" << std::endl;
break;
}
std::cout << "str2privKey bytes::" << __bio_data_written << std::endl;
// EVP_PKEY* __temp_ptr = PEM_read_bio_PrivateKey(SSL_mem_bio__/*__temp_bio*/, NULL, 0, 0);
EVP_PKEY* __temp_ptr = nullptr;
PEM_read_bio_PrivateKey(SSL_mem_bio__,&__temp_ptr,0,0);
if(nullptr == __temp_ptr){
std::cerr << __err + "PEM_read_bio_Priv" << std::endl;
break;
}
*privKeyPtr = __temp_ptr;
__res = true;
}while(false);
//clear temp alloc mem
// BIO_free(__temp_bio);
return __res;
}
bool abstract_secure_socket::str2pubKey(std::string const& pubCipher__, EVP_PKEY**__pubKeyPtr, BIO* SSL_mem_bio__)
{
std::string __err = _ERR "str2pubKey()";
int __bio_data_written = BIO_write(SSL_mem_bio__, pubCipher__.c_str(), pubCipher__.length());
std::cout << "str2pubKey bytes::" << __bio_data_written << std::endl;
//EVP_PKEY* __temp_ptr = PEM_read_bio_PUBKEY(SSL_mem_bio__, NULL, 0, 0);
EVP_PKEY* __temp_ptr = nullptr;
PEM_read_bio_PUBKEY(SSL_mem_bio__,&__temp_ptr,0,0);
if(nullptr == __temp_ptr){
std::cerr << __err + "PEM_read_bio_PUB" << std::endl;
return false;
}
*__pubKeyPtr = __temp_ptr;
return true;
}
Encryption and decryption of random strings (600) test passed successfully each launch without errors in both server and client:
void abstract_secure_socket::test_keys(int nwords2test__, int mes_length__)
{
for(auto i=0;i!=nwords2test__;++i){
std::string word = get_random_word(mes_length__);
std::cout << "WORD:->" << word << ';' << std::endl;
bool __errFlag = false;
std::string encr_word = encrypt(word,encr_pub_key_,&__errFlag);
std::cout << "ENCRYPTED RESULT:->" << encr_word << ';' << std::endl;
if(__errFlag)
throw std::runtime_error("ENCRYPT_ERROR OCCURRED");
__errFlag = false;
std::string decr_word = decrypt(encr_word,&__errFlag);
std::cout << "DECRYPTED RESULT:->" << decr_word << ';' << std::endl;
if(__errFlag)
throw std::runtime_error("DECRYPT_ERROR OCCURRED");
}
}
bool abstract_secure_socket::init_keys()
{
if(!initInternalKeys()){
std::cerr << _ERR "init_socket()_internal_keys" << std::endl;
return false;
}
pub_bio_ = BIO_new(BIO_s_mem());
if(nullptr == pub_bio_){
std::cerr << _ERR "init_socket()_bio" << std::endl;
return false;
}
int encr_decr_errors_amount = 0;
try {
test_keys(600,30);
} catch (std::runtime_error& e) {
std::cerr << e.what() << std::endl;
encr_decr_errors_amount++;
}
std::cout << "TEST ERRORS AMOUNT::" << encr_decr_errors_amount << std::endl;
std::this_thread::sleep_for(std::chrono::milliseconds(5000));
return true;
}
Encrypt and decrypt methods:
std::string abstract_secure_socket::encrypt(std::string const& str__, EVP_PKEY* SSLPubKey__, bool * __errFlagPtr) const
{
std::string __err_prefix = _ERR "SecSoc_encr";
EVP_PKEY_CTX* ctx = nullptr;
uchar* dst = nullptr;
bool __tempErrFlag = false;
std::string __outStr = "";
// 'goto' for free openSSL structs allocs
do{
ctx = EVP_PKEY_CTX_new(SSLPubKey__,NULL);
if(!ctx){
std::cerr << __err_prefix + "ctx_new alloc" << std::endl;
__tempErrFlag = true;
break;
}
int ctx_init_res = EVP_PKEY_encrypt_init(ctx);
if(ctx_init_res <= 0){
std::cerr << __err_prefix + "ctx_init" << std::endl;
__tempErrFlag = true;
break;
}
// EVP_PKEY_CTX_set_rsa_padding(ctx, RSA_PKCS1_OAEP_PADDING);
uchar const* __src = reinterpret_cast<uchar const*>(str__.c_str());
//std::basic_string<uchar > const __src = reinterpret_cast<uchar const*>(str__.c_str());
auto __strsiz = str__.size();
size_t outl=0;
// First, we need to determine buffer length
if(EVP_PKEY_encrypt(ctx, NULL , &outl, __src, __strsiz) <= 0){
std::cerr << __err_prefix + "BIO ENCRerror: cannot determine buffer length" << std::endl;
__tempErrFlag = true;
// return "";
break;
}
std::cout << _D "ENCRYPT OUTLEN::" << outl << std::endl;
dst = (uchar*)OPENSSL_malloc(outl);
if(!dst){
std::cerr << __err_prefix + "BIO ENCRerror: OPENSSL_malloc failed" << std::endl;
__tempErrFlag = true;
break;
}
if(EVP_PKEY_encrypt(ctx,dst,&outl, __src/*__src.data()*/,__strsiz) <= 0){
std::cerr << __err_prefix + "BIO ENCRerror: encrypt" << std::endl;
__tempErrFlag = true;
break;
}
__outStr = std::string(reinterpret_cast<char*>(dst),outl);
}while(false);
// send local error flag to input bool flag pointer
if(__errFlagPtr)
*__errFlagPtr = __tempErrFlag;
//if uchar buffer has been allocated successfully
if(dst){
// free temp uchar string after copying to std::string
OPENSSL_free(dst);
}
if(ctx){
// free alloc of CTX struct
EVP_PKEY_CTX_free(ctx);
}
return __outStr;
}
std::string abstract_secure_socket::decrypt(std::string const& str__, EVP_PKEY* SSLPrivateKey__, bool * __errFlagPtr) const
{
std::string __err_prefix = _ERR "SecSoc_decr";
EVP_PKEY_CTX* ctx = nullptr;
bool __tempErrFlag = false;
std::string __outStr = "";
// wchar(uchar) buffer
uchar* dst = nullptr;
// 'goto' for openSSL structs allocs free
do{
ctx = EVP_PKEY_CTX_new(SSLPrivateKey__,NULL);
if(!ctx){
std::cerr << __err_prefix + "EVP_PKEY_CTX alloc error" << std::endl;
__tempErrFlag = true;
break;
}
int ctx_init_res = EVP_PKEY_decrypt_init(ctx);
if(ctx_init_res <= 0){
std::cerr << __err_prefix + "ctx_init" << std::endl;
__tempErrFlag = true;
break;
}
// EVP_PKEY_CTX_set_rsa_padding(ctx, RSA_PKCS1_OAEP_PADDING);
uchar const* __src = reinterpret_cast<uchar const*>(str__.c_str());
//std::basic_string<uchar > const __src = reinterpret_cast<uchar const*>(str__.c_str());
auto __strsiz = str__.size();
size_t outl=0;
// First, we need to determine buffer length
if(EVP_PKEY_decrypt(ctx, NULL , &outl, __src, __strsiz) <= 0){
std::cerr << __err_prefix + "BIO DECRerror: cannot determine buffer length" << std::endl;
__tempErrFlag = true;
break;
}
std::cout << _D "DECRYPT OUTLEN::" << outl << std::endl;
dst = (uchar*)OPENSSL_malloc(outl);
if(!dst){
std::cerr << __err_prefix + "BIO DECRerror: OPENSSL_malloc failed" << std::endl;
__tempErrFlag = true;
break;
}
if(EVP_PKEY_decrypt(ctx,dst,&outl,__src,__strsiz) <= 0){
std::cerr << __err_prefix + "BIO DECRerror: decrypt" << std::endl;
__tempErrFlag = true;
break;
}
std::cout << "Temp result of decryption: " << dst << ';' << std::endl;
// temp uchar string->std::string
__outStr = std::string(reinterpret_cast<char*>(dst), outl);
}while(false);
// send local error flag to input bool flag pointer
if(__errFlagPtr)
*__errFlagPtr = __tempErrFlag;
//if uchar buffer has been allocated successfully
if(dst){
//free pointer of wchar(uchar) buffer
OPENSSL_free(dst);
}
if(ctx){
// free alloc of CTX struct
EVP_PKEY_CTX_free(ctx);
}
return __outStr;
}
PROBLEM: However, i'm stuck on 'key exchange' handling part. After i'm sending client public key to server socket, server successfully initialize a EVP_PKEY* public key pointer with recvested key string in user's struct without errors, decrypts some test message via this client's key and sending it back to client, this message encryption by user fails randomly (from my observations, in ~5/6 cases). Identical situation when client's encrypts his message via server's public key, and server tries to decrypt this message after recv().
First client connection, message "Y-!АФОРМАТИВ-X" (success decrypt):
SERVER ANSWER:-> ��2�-%M�C�������� L,H}���G�g^"s�-#G1�B�Z���s��y��>E~��ң���W�3�FI��-�1��D7H�t'.X)<�x���o�C��B��| ���>eO }|�Q��D�NICL2�:�OP��)��] ���Y4��8$P�ҺV������&~/���q�g��.s�+�
&���6�Njj �go���ߖ�P!Fs�A��S�*���W�gz}���@�`Z�%n�r�; DECRYPTED ANSWER:->[DEBUG]::DECRYPT OUTLEN::267 Temp result of decryption: Y-!АФОРМАТИВ-X;
Y-!АФОРМАТИВ-X;
Second connection (and most cases) with failed decrypt:
SERVER ANSWER:-> �"���apf���V���|�B�4�lrA���b�3�s�Њ�iݨ�%���nv�5��r�-�JI-Q9�������|�b�Y����?�s�7��3�Dۡ��^ܓQ�2����$`(>a�s�l��l�sH*� 9M7� T��4/s7�n�)�s] P; DECRYPTED ANSWER:->[DEBUG]::DECRYPT OUTLEN::267 [ERROR]::SecSoc_decrBIO DECRerror: decrypt ;
Program error at second EVP_PKEY_decrypt(ctx,dst,&outl,__src,__strsiz) call, after determine buffer length.
WHAT I'M TRIED: I'm tried to use BIO_new_mem_buf(privCipher__.c_str(),-1); instead of BIO_new(BIO_s_mem()) in str2pubKey() function for public key init without success; also tried to use local temporary BIO* object instead of class member 'pub_bio'.
I would be very appreciate for any help!
P.S. Server uses poll(..) multiplexing I/O for read/write to client's fds, client uses listening thread for listening another clients/server. Without encryption server/client logic works fine.
P.S.S. OpenSSL 3.0.8 7 Feb 2023; cmake version 3.18.4; OS Debian 11.
SOLVED: i found that because data after EVP_PKEY encryption stores in binary form (without encoding), before send() call i'm must to represent my binary encrypted message in text bytes encoded form (and decode it back to binary after recv() and before decrypt operation). Solution that i found on an Internet in C-code and adopted to C++ uses base64 encoding-decoding scheme:
#include <openssl/rsa.h>
#include <openssl/pem.h>
#include <openssl/evp.h>
#define _RSA_KEY_LEN 2136 // 267 length of encr. message (256 usefull symbols (0-255) + 11 padding)
typedef unsigned char uchar;
std::string base64Encode(std::string const & message__) const;
int calcDecodeLength(std::string const & b64input__) const;
std::string base64Decode(std::string const & b64message) const;
std::string abstract_secure_socket::base64Encode(/*const unsigned char *message*/std::string const & message__) const
{
std::string __err_prefix = _ERR "base64Encode():->";
size_t strSiz = message__.size();
uchar const * rawString = reinterpret_cast<uchar const*>(message__.data());
BIO *bio = nullptr;
BIO *b64 = nullptr;
FILE* stream = nullptr;
int encodedSize = 4*ceil((double)strSiz/3);
std::string __resStr = "";
std::vector<char> buffer(encodedSize+1);
// do-while(false) 'goto' for C-interfaces deallocs
do{
stream = fmemopen(buffer.data(), encodedSize+1, "w");
if(nullptr == stream){
std::cerr << __err_prefix + "fmemopen()" << std::endl;
break;
}
bio = BIO_new_fp(stream, BIO_NOCLOSE);
if(nullptr == bio){
std::cerr << __err_prefix + "BIO file stream alloc" << std::endl;
break;
}
b64 = BIO_new(BIO_f_base64());
if(nullptr == b64){
std::cerr << __err_prefix + "BIO base64 alloc" << std::endl;
break;
}
bio = BIO_push(b64, bio);
BIO_set_flags(bio, BIO_FLAGS_BASE64_NO_NL);
int __nbytes_written = BIO_write(bio, rawString, strSiz);
if(__nbytes_written <= 0 ){
std::cerr << __err_prefix + "BIO_write()" << std::endl;
break;
}
(void)BIO_flush(bio);
__resStr = std::string(buffer.data(),/*__nbytes_written*/encodedSize);
std::cout << _D "base64Encode()::__resStr::" << __resStr << ';' << std::endl;
}while(false);
if(nullptr != bio){
BIO_free_all(bio);
}
if(nullptr != stream){
fclose(stream);
}
return __resStr;
}
std::string abstract_secure_socket::base64Decode(std::string const & b64message) const
{
std::string __err_prefix = _ERR "base64Decode():->";
size_t strSiz = b64message.size();
BIO *bio = nullptr;
BIO *b64 = nullptr;
int decodedLength = calcDecodeLength(b64message);
std::vector<uchar> __base64outBuffer_decode(decodedLength+1);
FILE* stream = nullptr;
std::string __resultStr = "";
do{
stream = fmemopen(const_cast<char*>(b64message.data()), strSiz, "r");
if(nullptr == stream){
std::cerr << __err_prefix + "fmemopen()" << std::endl;
break;
}
bio = BIO_new_fp(stream, BIO_NOCLOSE);
if(nullptr == bio){
std::cerr << __err_prefix + "BIO file stream alloc" << std::endl;
break;
}
b64 = BIO_new(BIO_f_base64());
if(nullptr == b64){
std::cerr << __err_prefix + "BIO base64 alloc" << std::endl;
break;
}
bio = BIO_push(b64, bio);
BIO_set_flags(bio, BIO_FLAGS_BASE64_NO_NL);
decodedLength = BIO_read(bio, __base64outBuffer_decode.data(), strSiz);
if(decodedLength <= 0 ){
std::cerr << __err_prefix + "BIO_read()" << std::endl;
break;
}
(__base64outBuffer_decode.data())[decodedLength] = '\0';
__resultStr = std::string(reinterpret_cast<char *>(__base64outBuffer_decode.data()), decodedLength);
}while(false);
if(nullptr != bio){
BIO_free_all(bio);
}
if(nullptr != stream){
fclose(stream);
}
return __resultStr;
}
int abstract_secure_socket::calcDecodeLength(std::string const & b64input__) const
{
int padding = 0;
auto __size = b64input__.size();
// Check for trailing '=''s as padding
if(b64input__[__size-1] == '=' && b64input__[__size-2] == '=')
padding = 2;
else if (b64input__[__size-1] == '=')
padding = 1;
return (int)__size*0.75 - padding;
}
For receiving encrypted message i uses this "one-liner" (returns the original decrypted message in std::string):
virtual inline std::string get_encr_message4rom_socket(size_t size_lim__, int sock_descr__) const
{
return abstract_secure_socket::decrypt(abstract_secure_socket::base64Decode(abstract_socket::get_message4rom_socket(size_lim__,sock_descr__)));
}
, where 'sock_descr__' is a endpoint socket file descriptor.
And for sending i uses this (return amount (size_t) of successfully sent bytes to target socket via send() ):
virtual inline size_t send_encr_message2socket(std::string const& str__, int sock_descr__, EVP_PKEY* SSLpub_key__) const
{
return abstract_socket::send_message2socket(abstract_secure_socket::base64Encode(abstract_secure_socket::encrypt(str__,SSLpub_key__)),sock_descr__);
}
, where 'SSLpub_key__' is a SSL RSA public key of target client/server.