How to bind functions returning references with pybind11?

When binding C++ with pybind11, I ran into an issue regarding a couple of class members that return (const or non-const) references; considering the following snippet:

struct Data {
    double value1;
    double value2;
};

class Element {
 public:
    Element() = default;
    Element(Data values) : data(values) { }
    Element(const Element& other) : data(other.data) { printf("copying from %p\n", &other); }

    Data data;
};

class Container {
 public:
    Container() : data(10, Element(Data{0.1, 0.2})) {};

    Element& operator[](size_t idx) { return data[idx]; }
    const Element& operator[](size_t idx) const { return data[idx]; }

 protected:
    std::vector< Element > data;
};

which is bound to a Python module with:

py::class_< Data > (module, "Data")
    .def(py::init< double, double >(), "Constructs a new instance", "v1"_a, "v2"_a)
    .def_readwrite("value1", &Data::value1)
    .def_readwrite("value2", &Data::value2);

py::class_< Element > (module, "Element")
    .def(py::init< Data >(), "Constructs a new instance", "values"_a)
    .def_readwrite("data", &Element::data)
    .def("__repr__", [](const Element& e){ return std::to_string(e.data.value1); });

py::class_< Container > (module, "Container")
    .def(py::init< >(), "Constructs a new instance")
    .def("__getitem__", [](Container& c, size_t idx) { return c[idx]; }, "idx"_a)
    .def("__setitem__", [](Container& c, size_t idx, Element e) { c[idx] = e; }, "idx"_a, "val"_a);

I am having trouble getting the [] operator to work on Container class

print("-------------")
foo = module.Data(0.9, 0.8)
print(foo.value2)
foo.value2 = 0.7    # works
print(foo.value2)

print("-------------")
e = module.Element(motion.Data(0.3, 0.2))
print(e.data.value1)
e.data.value2 = 0.6    # works
print(e.data.value2)
e.data = foo           # works
print(e.data.value2)

print("-------------")
c = motion.Container()
print(c[0].data.value1)
c[0] = e                   # works
print(c[0].data.value1)
c[0].data = foo            # does not work (!)
print(c[0].data.value1)
c[0].data.value1 = 0.0     # does not work (!)
print(c[0].data.value1)

While the __getitem__ ([]) function does seem to be working as intended, it seems to fail when accessing members on the returned object; instead, a temporary copy is created from the returned reference, and any changes to that instance are not applied. I've tried 1) declaring a std::shared_ptr<Element> holder type when binding the Element class; 2) defining specific return value policy py::return_value_policy::reference and py::return_value_policy::reference_internal on __getitem__; and 3) defining specific call policies py::keep_alive<0,1>() and py::keep_alive<1,0>(); but none of these solutions worked.

Any hints on what how solve this issue?

C++ deduces the return type of your lambda to be Element not Element& thus making a copy so explicitly define the return type, and make sure you also take items by reference to avoid copying.

    py::class_< Container > (module, "Container")
    .def(py::init< >(), "Constructs a new instance")
    .def("__getitem__", [](Container& c, size_t idx) ->Element& { return c[idx]; }, "idx"_a, py::return_value_policy::reference_internal)
    .def("__setitem__", [](Container& c, size_t idx, const Element& e) { c[idx] = e; }, "idx"_a, "val"_a);

you can define the return type to be ->auto& if you are using C++14 or higher.