I write a function that executes a Python string in C++ using the Python C API. The function works perfectly. When I run it, it executes the Python string.
The problem occurs when I make a .so file and then call the same function from the Python side using C++.
It gives a weird error. free() invalid pointer on Py_Initialize(); this line.
When I comment down the line, then a segmentation fault occurs.
My runner.cpp code is:
#include <vector>
#include <dlfcn.h>
#include <iostream>
#include <sstream>
#include <cstdlib>
#include <cstring>
typedef void (*Py_Initialize_t)();
typedef int (*PyRun_SimpleString_t)(const char *);
typedef void (*Py_Finalize_t)();
std::string execute_command(const char* command) {
FILE* pipe = popen(command, "r");
if (!pipe) {
return "Error executing command";
}
std::ostringstream output;
char buffer[128];
while (!feof(pipe)) {
if (fgets(buffer, 128, pipe) != nullptr) {
output << buffer;
}
}
// Close the pipe and return the output
pclose(pipe);
return output.str();
}
std::vector<std::string> split_string(const std::string& input, char delimiter) {
std::vector<std::string> tokens;
std::istringstream iss(input);
std::string token;
while (std::getline(iss, token, delimiter)) {
tokens.push_back(token);
}
return tokens;
}
void execute_Prog730(std::string program) {
std::cout << "Running Engine" << std::endl;
std::string python_version_output = execute_command("python3 -c \"import sys; print(sys.version)\"");
size_t start_pos = python_version_output.find_first_of("0123456789");
size_t end_pos = python_version_output.find_first_not_of("0123456789.", start_pos);
std::string python_version = python_version_output.substr(start_pos, end_pos - start_pos);
std::vector<std::string> parts = split_string(python_version, '.');
std::ostringstream lib_name_stream;
lib_name_stream << "libpython" << parts[0] << '.' << parts[1] << ".so";
std::string lib_name = lib_name_stream.str();
std::cout << lib_name << std::endl;
void* handle = dlopen(lib_name.c_str(), RTLD_LAZY);
if(!handle && lib_name.back() == 'o') {
std::string lib_name_so1 = lib_name + ".1";
handle = dlopen(lib_name_so1.c_str(), RTLD_LAZY);
}
if (!handle) {
std::cerr << "Failed to load Python library" << std::endl;
//return 1;
}
auto Py_Initialize = (Py_Initialize_t)dlsym(handle, "Py_Initialize");
auto PyRun_SimpleString = (PyRun_SimpleString_t)dlsym(handle, "PyRun_SimpleString");
auto Py_Finalize = (Py_Finalize_t)dlsym(handle, "Py_Finalize");
if (!Py_Initialize || !PyRun_SimpleString || !Py_Finalize) {
std::cerr << "Failed to resolve Python functions" << std::endl;
dlclose(handle);
}
std::cout << "Debug" << std::endl;
Py_Initialize();
std::cout << "Debug" << std::endl;
PyRun_SimpleString(program.c_str());
Py_Finalize();
dlclose(handle);
}
My runner.h file is:
#include <iostream>
#include "runner.cpp"
void execute_Prog730(std::string program);
My main.cpp file is:
#include <iostream>
#include <cstring>
#include <string.h>
#include <runner.h>
extern "C" void run(const char* program) {
std::string pr(program);
std::cout << pr << std::endl;
execute_Prog730(pr);
}
The command I use to compile:
g++ -o main.so --shared -fPIC main.cpp -I/home/lakshit/Desktop/temp/include
Finally my main.py file is:
import ctypes
module = ctypes.CDLL('/home/lakshit/Desktop/temp/main.so')
func = module.run
func.argtypes = [ctypes.c_char_p]
func(b"print('hello')")
Note: I am using Ubuntu 22.04 (Jammy Jellyfish). Some versions require to add -ldl command to compile the .so file successfully.
To summarize the comment discussion:
Py_Initialize and Py_FinalizeRTLD_NOLOADctypes.CDLL drops the GIL when calling C code.If you are calling a C function that you know will use the Python C API you should instead use ctypes.PyDLL which doesn't drop the GIL, and you won't need to re-acquire it from the C++ side.
The following code solves the issue on my system.
#include <vector>
#include <dlfcn.h>
#include <iostream>
#include <sstream>
#include <cstdlib>
#include <cstring>
namespace {
enum PyGILState_STATE
{
locked, unlocked
};
using PyRun_SimpleString_t = int (*)(const char *);
using PyGILState_Ensure_t = PyGILState_STATE (*)(void);
using PyGILState_Release_t = void (*)(PyGILState_STATE);
}
std::string execute_command(const char* command) {
FILE* pipe = popen(command, "r");
if (!pipe) {
return "Error executing command";
}
std::ostringstream output;
char buffer[128];
while (!feof(pipe)) {
if (fgets(buffer, 128, pipe) != nullptr) {
output << buffer;
}
}
// Close the pipe and return the output
pclose(pipe);
return output.str();
}
std::vector<std::string> split_string(const std::string& input, char delimiter) {
std::vector<std::string> tokens;
std::istringstream iss(input);
std::string token;
while (std::getline(iss, token, delimiter)) {
tokens.push_back(token);
}
return tokens;
}
void execute_Prog730(std::string program) {
std::cout << "Running Engine" << std::endl;
std::string python_version_output = execute_command("python3 -c \"import sys; print(sys.version)\"");
size_t start_pos = python_version_output.find_first_of("0123456789");
size_t end_pos = python_version_output.find_first_not_of("0123456789.", start_pos);
std::string python_version = python_version_output.substr(start_pos, end_pos - start_pos);
std::vector<std::string> parts = split_string(python_version, '.');
std::ostringstream lib_name_stream;
lib_name_stream << "libpython" << parts[0] << '.' << parts[1] << ".so";
std::string lib_name = lib_name_stream.str();
std::cout << lib_name << std::endl;
void* handle = dlopen(lib_name.c_str(), RTLD_NOLOAD); // Use the already-loaded Python interpreter
if(!handle && lib_name.back() == 'o') {
std::string lib_name_so1 = lib_name + ".1";
handle = dlopen(lib_name_so1.c_str(), RTLD_NOLOAD);
}
if (!handle) {
std::cerr << "Failed to load Python library" << std::endl;
//return 1;
}
auto PyRun_SimpleString = (PyRun_SimpleString_t)dlsym(handle, "PyRun_SimpleString");
auto PyGILState_Ensure = (PyGILState_Ensure_t)dlsym(handle, "PyGILState_Ensure");
auto PyGILState_Release = (PyGILState_Release_t)dlsym(handle, "PyGILState_Release");
if (!PyRun_SimpleString || !PyGILState_Ensure || !PyGILState_Release) {
std::cerr << "Failed to resolve Python functions" << std::endl;
if (handle)
{
dlclose(handle);
}
}
std::cout << "Debug" << std::endl;
auto state = PyGILState_Ensure(); // Acquire GIL
PyRun_SimpleString(program.c_str());
PyGILState_Release(state); // Release GIL
if (handle)
{ dlclose(handle); }
}
print('hello')
Running Engine
libpython3.10.so
Failed to load Python library
Debug
hello
The Failed to load Python library happens because the main Python executable doesn't load libpython3, and instead statically links it, so those functions are loaded from the executable, not the shared library. If another version of Python dynamically links libpython3 then it will succeed.