I am trying to use the Numpy C-API to extend random numbers through BitGenerator as in the numpy example in the documentation, and I am running into a PyObject cannot be convert to bitgen_t error.
I have copied the first code into a file named extension_rand.pyx as follows:
from libc.stdint cimport uint32_t
from cpython.pycapsule import PyCapsule_IsValid, PyCapsule_GetPointer
import numpy as np
cimport numpy as np
cimport cython
from numpy.random cimport bitgen_t
from numpy.random import PCG64
np.import_array()
@cython.boundscheck(False)
@cython.wraparound(False)
def uniforms(Py_ssize_t n):
"""
Create an array of `n` uniformly distributed doubles.
A 'real' distribution would want to process the values into
some non-uniform distribution
"""
cdef Py_ssize_t i
cdef bitgen_t *rng
cdef const char *capsule_name = "BitGenerator"
cdef double[::1] random_values
cdef np.ndarray randoms
x = PCG64()
capsule = x.capsule
# Optional check that the capsule if from a BitGenerator
if not PyCapsule_IsValid(capsule, capsule_name):
raise ValueError("Invalid pointer to anon_func_state")
# Cast the pointer
rng = PyCapsule_GetPointer(capsule, capsule_name)
random_values = np.empty(n, dtype='float64')
with x.lock, nogil:
for i in range(n):
# Call the function
random_values[i] = rng.next_double(rng.state)
randoms = np.asarray(random_values)
return randoms
My setup.py file is as follows:
# setup.py
from distutils.core import setup
from Cython.Build import cythonize
from setuptools.extension import Extension
import numpy as np
import os
lib_path = os.path.join(np.get_include(), '..', '..', 'random', 'lib')
extending = Extension(name="extension_rand",
sources=['extension_rand.pyx'],
include_dirs=[np.get_include()],
library_dirs=[lib_path],
libraries=['npyrandom']
)
setup(
ext_modules=cythonize(
extending, language_level="3", annotate="True"
)
)
When I run
python setup.py build_ext --inplace
I get the following error:
Compiling extension_rand.pyx because it changed.
[1/1] Cythonizing extension_rand.pyx
Error compiling Cython file:
------------------------------------------------------------
...
capsule = x.capsule
# Optional check that the capsule if from a BitGenerator
if not PyCapsule_IsValid(capsule, capsule_name):
raise ValueError("Invalid pointer to anon_func_state")
# Cast the pointer
rng = PyCapsule_GetPointer(capsule, capsule_name)
^
------------------------------------------------------------
extension_rand.pyx:33:30: Cannot convert Python object to 'bitgen_t *'
Error compiling Cython file:
------------------------------------------------------------
...
capsule = x.capsule
# Optional check that the capsule if from a BitGenerator
if not PyCapsule_IsValid(capsule, capsule_name):
raise ValueError("Invalid pointer to anon_func_state")
# Cast the pointer
rng = PyCapsule_GetPointer(capsule, capsule_name)
^
------------------------------------------------------------
extension_rand.pyx:33:30: Storing unsafe C derivative of temporary Python reference
Traceback (most recent call last):
File "/Users/sayantanghosh/work/cython_test/setup.py", line 21, in <module>
ext_modules=cythonize(
^^^^^^^^^^
File "/Users/sayantanghosh/anaconda3/envs/nonlin/lib/python3.11/site-packages/Cython/Build/Dependencies.py", line 1154, in cythonize
cythonize_one(*args)
File "/Users/sayantanghosh/anaconda3/envs/nonlin/lib/python3.11/site-packages/Cython/Build/Dependencies.py", line 1321, in cythonize_one
raise CompileError(None, pyx_file)
Cython.Compiler.Errors.CompileError: extension_rand.pyx
The output of my import numpy as np; np.show_runtime() is
[{'numpy_version': '1.26.4',
'python': '3.11.10 | packaged by conda-forge | (main, Sep 30 2024, 17:57:16) '
'[Clang 17.0.6 ]',
'uname': uname_result(system='Darwin', node='calypso.local', release='22.6.0', version='Darwin Kernel Version 22.6.0: Thu Sep 5 20:48:48 PDT 2024; root:xnu-8796.141.3.708.1~1/RELEASE_X86_64', machine='x86_64')},
{'simd_extensions': {'baseline': ['SSE', 'SSE2', 'SSE3', 'SSSE3'],
'found': ['SSE41',
'POPCNT',
'SSE42',
'AVX',
'F16C',
'FMA3',
'AVX2'],
'not_found': ['AVX512F',
'AVX512CD',
'AVX512_KNL',
'AVX512_SKX',
'AVX512_CLX',
'AVX512_CNL',
'AVX512_ICL']}},
{'architecture': 'Haswell',
'filepath': '/Users/sayantanghosh/anaconda3/envs/nonlin/lib/libopenblasp-r0.3.21.dylib',
'internal_api': 'openblas',
'num_threads': 2,
'prefix': 'libopenblas',
'threading_layer': 'pthreads',
'user_api': 'blas',
'version': '0.3.21'}]
My setuptools version is '75.1.0', while cython version is '3.0.10'.
I am not sure if this is a cython or numpy C-API related issue.
I will be grateful for any help.
I think there's two issues:
from cpython.pycapsule import PyCapsule_IsValid, PyCapsule_GetPointer needs to be from cpython.pycapsule cimport PyCapsule_IsValid, PyCapsule_GetPointer. (cimport rather than import so the type is known at Cython compile time. Otherwise it assumes that the functions are arbitrary Python functions returning arbitrary Python objects).
PyCapsule_GetPointer returns a void* and you need to cast that to a bitgen_t* (i.e. rng = <bitgen_t*>PyCapsule_GetPointer(capsule, capsule_name))