pybind11 shared_ptr to Python-implemented class calls pure virtual instead of override

I have a ScriptFactory system in which I have an object that creates instances of custom objects created in Python, which are derived from an abstract base class in C++. The objects are created correctly, but when I try to use the override python function, it calls the base C++ pure virtual function.

This is the simplest example with the structure I need:

Here is the C++ full code (compiled into MyTest.pyd)

#include <pybind11/pybind11.h>
#include <pybind11/stl.h>
#include <memory>
#include <string>
#include <iostream>

// Abstract class to be implemented in Python (the problematic one)
struct Script {
    virtual ~Script() = default;
    virtual void init() = 0;
};


// Class that generates scripts, (abstract, should be implemented in Python)
struct ScriptFactory {
    virtual ~ScriptFactory() = default;
    virtual std::shared_ptr<Script> getScript(const std::string& type) = 0;
};


// In this example, the class that generates the script using the ScriptFactory and attempts to use the generated script
class Container {
private:
    std::shared_ptr<Script> script;
    std::shared_ptr<ScriptFactory> factory;
public:
    Container(std::shared_ptr<ScriptFactory> fact) : factory(fact) {};
    void doSth(const std::string& name) {
        script = factory->getScript(name);
        script->init(); // Calls Script::init (pure virtual) instead of Pythons implementation
    }
};

// ------------------- Wrappers -------------------

namespace py = pybind11;

struct PyScript : public Script {
    using Script::Script;

    void init() override {
        PYBIND11_OVERRIDE_PURE(
            void,   
            Script,
            init
        );
    }
};

struct PyScriptFactory : ScriptFactory {
    using ScriptFactory::ScriptFactory;

    std::shared_ptr<Script> getScript(const std::string& type) override {
        PYBIND11_OVERRIDE_PURE(
            std::shared_ptr<Script>,
            ScriptFactory,
            getScript,
            type
        );
    }
};

// ------------------- Bindings -------------------

PYBIND11_MODULE(MyTest, m) {
    py::class_<Script, PyScript, std::shared_ptr<Script>>(m, "Script")
        .def(py::init<>())
        .def("init", &Script::init);

    py::class_<ScriptFactory, PyScriptFactory, std::shared_ptr<ScriptFactory>>(m, "ScriptFactory")
        .def(py::init<>())
        .def("getScript", &ScriptFactory::getScript);

    py::class_<Container>(m, "Container")
        .def(py::init<std::shared_ptr<ScriptFactory>>())
        .def("doSth", &Container::doSth);
}

And here the Python code that uses it:

import MyTest as t

class MyFactory(t.ScriptFactory):
    def __init__(self):
        super().__init__()
        self.scripts:dict = {}

    def registerScript(self, script_name, func):
        self.scripts[script_name] = func
    
    def getScript(self, name):
        return self.scripts[name]()

class MyScript(t.Script):
    def __init__(self):
        super().__init__()
    
    def init(self):
        print("Hello, world!")


# Creates the factory and registers the script
my_factory = MyFactory()
my_factory.registerScript("MyScript", lambda: MyScript())

# Creates the container and calls its function
my_cont = t.Container(my_factory)
my_cont.doSth("MyScript")

# Expected: 'Hello, world!'
# Real result: 'RuntimeError: Tried to call pure virtual function "Script::init"'

From all the tests I have been doing, it looks like the C++ side loses the link to the Python override, but I’m not completely sure.

when you use std::shared_ptr as the holder for python objects, the python and C++ objects have unrelated lifetime, the python object keeps the C++ object alive, but the opposite is not true, when MyScript is destroyed because there are no references to it you get this "pure virtual call".

as of pybind11v3 (which was released a month ago), you can use py::smart_holder instead of std::shared_ptr and it will make sure the python object lives as long as the C++ object, and it can be cast to shared_ptr, but you'll need the trampoline objects to inherit py::trampoline_self_life_support, see Combining virtual functions and inheritance

struct PyScript : public Script, py::trampoline_self_life_support {
    using Script::Script;

    void init() override {
        PYBIND11_OVERRIDE_PURE(
            void,   
            Script,
            init
        );
    }
};

struct PyScriptFactory : public ScriptFactory, py::trampoline_self_life_support {
    using ScriptFactory::ScriptFactory;

    std::shared_ptr<Script> getScript(const std::string& type) override {
        PYBIND11_OVERRIDE_PURE(
            std::shared_ptr<Script>,
            ScriptFactory,
            getScript,
            type
        );
    }
};

// ------------------- Bindings -------------------

PYBIND11_MODULE(MyTest, m) {
    py::class_<Script, PyScript, py::smart_holder>(m, "Script")
        .def(py::init<>())
        .def("init", &Script::init);

    py::class_<ScriptFactory, PyScriptFactory, py::smart_holder>(m, "ScriptFactory")
        .def(py::init<>())
        .def("getScript", &ScriptFactory::getScript);

    py::class_<Container>(m, "Container")
        .def(py::init<std::shared_ptr<ScriptFactory>>())
        .def("doSth", &Container::doSth);
}