Using Python ctypes and numpy library, I pass data to a shared library and encounter a very weird behavior
C function :
#include <stdio.h>
typedef struct {
double *a;
double *b;
} s_gate;
void printdouble(s_gate*, int);
void printdouble(s_gate *gate, int n) {
for (int i =0; i < n; i++) {
printf("gate->a[%d] = %f\n", i, gate->a[i]);
}
for (int i =0; i < n; i++) {
printf("gate->b[%d] = %f\n", i, gate->b[i]);
}
}
Python code :
import ctypes
import numpy as np
class s_gate(ctypes.Structure):
_fields_ = [('a', ctypes.POINTER(ctypes.c_double)),
('b', ctypes.POINTER(ctypes.c_double))]
def __init__(self, mydict:dict):
mask = [True, False, True, True, True, True, False, False, False, True]
a = np.ascontiguousarray(mydict['a'], dtype=np.double)
b = np.ascontiguousarray(mydict['b'], dtype=np.double)
setattr(self, 'a', a[0,:].ctypes.data_as(ctypes.POINTER(ctypes.c_double)))
setattr(self, 'b', b[0,:].ctypes.data_as(ctypes.POINTER(ctypes.c_double)))
self.size = 10
if __name__ == "__main__":
a = np.array([[1,2,3,4,5,6,7,8,9,10], [10,9,8,7,6,5,4,3,2,1]], dtype=np.double).T
b = a + 100
data = {'a': a, 'b': b}
mylib = ctypes.CDLL('./mwe.so')
mylib.printdouble.argstype = [ctypes.POINTER(s_gate), ctypes.c_int]
mylib.printdouble.restype = ctypes.c_void_p
print(f'Sending \n{a} and \n{b}')
gate = s_gate(data)
mylib.printdouble(ctypes.byref(gate), gate.size)
Running this code, I got the expected result, which is :
[[ 1. 2. 3. 4. 5. 6. 7. 8. 9. 10.]
[10. 9. 8. 7. 6. 5. 4. 3. 2. 1.]] and
[[101. 102. 103. 104. 105. 106. 107. 108. 109. 110.]
[110. 109. 108. 107. 106. 105. 104. 103. 102. 101.]]
gate->a[0] = 1.000000
gate->a[1] = 2.000000
gate->a[2] = 3.000000
gate->a[3] = 4.000000
gate->a[4] = 5.000000
gate->a[5] = 6.000000
...
gate->b[0] = 101.000000
gate->b[1] = 102.000000
gate->b[2] = 103.000000
gate->b[3] = 104.000000
gate->b[4] = 105.000000
gate->b[5] = 106.000000
...
Now, let's use the mask variable in the __init__ method of the s_gate class. So let's replace lines 11 and 12 with :
setattr(self, 'a', a[0,mask].ctypes.data_as(ctypes.POINTER(ctypes.c_double)))
setattr(self, 'b', b[0,mask].ctypes.data_as(ctypes.POINTER(ctypes.c_double)))
self.size = sum(mask)
The results is now :
gate->a[0] = 0.000000
gate->a[1] = 0.000000
gate->a[2] = 0.000000
gate->a[3] = 0.000000
gate->a[4] = 0.000000
gate->a[5] = 0.000000
gate->b[0] = 101.000000
gate->b[1] = 103.000000
gate->b[2] = 104.000000
gate->b[3] = 105.000000
gate->b[4] = 106.000000
gate->b[5] = 110.000000
All gate.a data is zeroed ! The expected result is of course [1, 3, 4, 5, 6, 10] for gate.a
What I tried so far :
Use np.require, np.ascontiguousarray to ensure C contiguous array, perform integer indexing instead of logical indexing, perform 2D logical indexing (array[slices_xy] instead of array[slice_x, slice_y]).
Use various copy, deepcopy methods and create an intermediate data with the first slicing...
Nothing worked, as soon as the slice that maskdescribe in this code appears (instead of :), this behavior appears. What goes wrong ?
The memory is being freed for the masked arrays creating undefined behavior. Likely the a pointer's memory is reused but the b pointer happens to still be the same. Both are freed though.
Create the masked arrays and hold a reference to them in the s_gate object, then it works:
import ctypes as ct
import numpy as np
PDOUBLE = ct.POINTER(ct.c_double)
class s_gate(ct.Structure):
_fields_ = [('a', PDOUBLE),
('b', PDOUBLE)]
def __init__(self, a, b):
mask = [True, False, True, True, True, True, False, False, False, True]
self.tmpa = a[0,mask] # hold a reference to the arrays
self.tmpb = b[0,mask]
self.a = self.tmpa.ctypes.data_as(PDOUBLE) # get pointers to the arrays
self.b = self.tmpb.ctypes.data_as(PDOUBLE)
self.size = sum(mask)
a = np.array([[1,2,3,4,5,6,7,8,9,10], [10,9,8,7,6,5,4,3,2,1]], dtype=np.double)
b = a + 100
mylib = ct.CDLL('./test')
mylib.printdouble.argstype = ct.POINTER(s_gate), ct.c_int
mylib.printdouble.restype = ct.c_void_p
print(f'Sending \n{a} and \n{b}')
gate = s_gate(a, b)
mylib.printdouble(ct.byref(gate), gate.size)
Output:
[[ 1. 2. 3. 4. 5. 6. 7. 8. 9. 10.]
[10. 9. 8. 7. 6. 5. 4. 3. 2. 1.]] and
[[101. 102. 103. 104. 105. 106. 107. 108. 109. 110.]
[110. 109. 108. 107. 106. 105. 104. 103. 102. 101.]]
gate->a[0] = 1.000000
gate->a[1] = 3.000000
gate->a[2] = 4.000000
gate->a[3] = 5.000000
gate->a[4] = 6.000000
gate->a[5] = 10.000000
gate->b[0] = 101.000000
gate->b[1] = 103.000000
gate->b[2] = 104.000000
gate->b[3] = 105.000000
gate->b[4] = 106.000000
gate->b[5] = 110.000000