Consider simplified example using multiprocessing inside a class that use cupy for simulation. this part
obj = FUNC(par)
obj.simulate()
done in with cupy, then data copy to CPU. multiprocessing does not touch data on GPU but still, I get the following error:
import tqdm
import cupy as cp
import numpy as np
from multiprocessing import Pool
class FUNC:
def __init__(self, par) -> None:
self.x = par['x']
def simulate(self):
x = self.x
xs = []
for t in tqdm.trange(nt):
dx = cp.random.rand(nn, ns) * 0.001
x += dx
xs.append(x.get())
self.xs = np.asarray(xs)
def func(self, i):
x = self.xs[:, :, i]
return [np.mean(x)]
def stats(self):
with Pool(processes=2) as pool:
data = (pool.map(self.func, range(ns)))
return cp.asnumpy(data)
if __name__ == "__main__":
nn = 10
ns = 10
nt = 2500
par = {
"x": cp.random.randn(nn, ns).astype('f'),
}
obj = FUNC(par)
obj.simulate()
data = obj.stats()
print(data.shape)
Process ForkPoolWorker-1:
Traceback (most recent call last):
File "/home/ziaee/anaconda3/envs/sbinmms/lib/python3.9/multiprocessing/process.py", line 315, in _bootstrap
self.run()
File "/home/ziaee/anaconda3/envs/sbinmms/lib/python3.9/multiprocessing/process.py", line 108, in run
self._target(*self._args, **self._kwargs)
File "/home/ziaee/anaconda3/envs/sbinmms/lib/python3.9/multiprocessing/pool.py", line 114, in worker
task = get()
File "/home/ziaee/anaconda3/envs/sbinmms/lib/python3.9/multiprocessing/queues.py", line 368, in get
return _ForkingPickler.loads(res)
File "cupy/_core/core.pyx", line 2250, in cupy._core.core.array
File "cupy/_core/core.pyx", line 2271, in cupy._core.core.array
File "cupy/_core/core.pyx", line 2403, in cupy._core.core._array_default
File "cupy/_core/core.pyx", line 171, in cupy._core.core.ndarray.__init__
File "cupy/cuda/memory.pyx", line 698, in cupy.cuda.memory.alloc
File "cupy/cuda/memory.pyx", line 1375, in cupy.cuda.memory.MemoryPool.malloc
File "cupy/cuda/memory.pyx", line 1395, in cupy.cuda.memory.MemoryPool.malloc
File "cupy/cuda/device.pyx", line 48, in cupy.cuda.device.get_device_id
File "cupy_backends/cuda/api/runtime.pyx", line 159, in cupy_backends.cuda.api.runtime.getDevice
File "cupy_backends/cuda/api/runtime.pyx", line 132, in cupy_backends.cuda.api.runtime.check_status
cupy_backends.cuda.api.runtime.CUDARuntimeError: cudaErrorInitializationError: initialization error
Process ForkPoolWorker-2:
Traceback (most recent call last):
File "/home/ziaee/anaconda3/envs/sbinmms/lib/python3.9/multiprocessing/process.py", line 315, in _bootstrap
self.run()
File "/home/ziaee/anaconda3/envs/sbinmms/lib/python3.9/multiprocessing/process.py", line 108, in run
self._target(*self._args, **self._kwargs)
File "/home/ziaee/anaconda3/envs/sbinmms/lib/python3.9/multiprocessing/pool.py", line 114, in worker
task = get()
File "/home/ziaee/anaconda3/envs/sbinmms/lib/python3.9/multiprocessing/queues.py", line 368, in get
return _ForkingPickler.loads(res)
File "cupy/_core/core.pyx", line 2250, in cupy._core.core.array
File "cupy/_core/core.pyx", line 2271, in cupy._core.core.array
File "cupy/_core/core.pyx", line 2403, in cupy._core.core._array_default
File "cupy/_core/core.pyx", line 171, in cupy._core.core.ndarray.__init__
File "cupy/cuda/memory.pyx", line 698, in cupy.cuda.memory.alloc
File "cupy/cuda/memory.pyx", line 1375, in cupy.cuda.memory.MemoryPool.malloc
File "cupy/cuda/memory.pyx", line 1395, in cupy.cuda.memory.MemoryPool.malloc
File "cupy/cuda/device.pyx", line 48, in cupy.cuda.device.get_device_id
File "cupy_backends/cuda/api/runtime.pyx", line 159, in cupy_backends.cuda.api.runtime.getDevice
File "cupy_backends/cuda/api/runtime.pyx", line 132, in cupy_backends.cuda.api.runtime.check_status
cupy_backends.cuda.api.runtime.CUDARuntimeError: cudaErrorInitializationError: initialization error
Process ForkPoolWorker-3:
Traceback (most recent call last):
File "/home/ziaee/anaconda3/envs/sbinmms/lib/python3.9/multiprocessing/process.py", line 315, in _bootstrap
self.run()
File "/home/ziaee/anaconda3/envs/sbinmms/lib/python3.9/multiprocessing/process.py", line 108, in run
self._target(*self._args, **self._kwargs)
File "/home/ziaee/anaconda3/envs/sbinmms/lib/python3.9/multiprocessing/pool.py", line 114, in worker
task = get()
File "/home/ziaee/anaconda3/envs/sbinmms/lib/python3.9/multiprocessing/queues.py", line 368, in get
return _ForkingPickler.loads(res)
File "cupy/_core/core.pyx", line 2250, in cupy._core.core.array
File "cupy/_core/core.pyx", line 2271, in cupy._core.core.array
File "cupy/_core/core.pyx", line 2403, in cupy._core.core._array_default
File "cupy/_core/core.pyx", line 171, in cupy._core.core.ndarray.__init__
File "cupy/cuda/memory.pyx", line 698, in cupy.cuda.memory.alloc
File "cupy/cuda/memory.pyx", line 1375, in cupy.cuda.memory.MemoryPool.malloc
File "cupy/cuda/memory.pyx", line 1395, in cupy.cuda.memory.MemoryPool.malloc
File "cupy/cuda/device.pyx", line 48, in cupy.cuda.device.get_device_id
File "cupy_backends/cuda/api/runtime.pyx", line 159, in cupy_backends.cuda.api.runtime.getDevice
File "cupy_backends/cuda/api/runtime.pyx", line 132, in cupy_backends.cuda.api.runtime.check_status
cupy_backends.cuda.api.runtime.CUDARuntimeError: cudaErrorInitializationError: initialization error
Process ForkPoolWorker-4:
Traceback (most recent call last):
File "/home/ziaee/anaconda3/envs/sbinmms/lib/python3.9/multiprocessing/process.py", line 315, in _bootstrap
self.run()
File "/home/ziaee/anaconda3/envs/sbinmms/lib/python3.9/multiprocessing/process.py", line 108, in run
self._target(*self._args, **self._kwargs)
File "/home/ziaee/anaconda3/envs/sbinmms/lib/python3.9/multiprocessing/pool.py", line 114, in worker
task = get()
File "/home/ziaee/anaconda3/envs/sbinmms/lib/python3.9/multiprocessing/queues.py", line 368, in get
return _ForkingPickler.loads(res)
File "cupy/_core/core.pyx", line 2250, in cupy._core.core.array
File "cupy/_core/core.pyx", line 2271, in cupy._core.core.array
File "cupy/_core/core.pyx", line 2403, in cupy._core.core._array_default
File "cupy/_core/core.pyx", line 171, in cupy._core.core.ndarray.__init__
File "cupy/cuda/memory.pyx", line 698, in cupy.cuda.memory.alloc
File "cupy/cuda/memory.pyx", line 1375, in cupy.cuda.memory.MemoryPool.malloc
File "cupy/cuda/memory.pyx", line 1395, in cupy.cuda.memory.MemoryPool.malloc
File "cupy/cuda/device.pyx", line 48, in cupy.cuda.device.get_device_id
File "cupy_backends/cuda/api/runtime.pyx", line 159, in cupy_backends.cuda.api.runtime.getDevice
File "cupy_backends/cuda/api/runtime.pyx", line 132, in cupy_backends.cuda.api.runtime.check_status
cupy_backends.cuda.api.runtime.CUDARuntimeError: cudaErrorInitializationError: initialization error
Process ForkPoolWorker-5:
Traceback (most recent call last):
File "/home/ziaee/anaconda3/envs/sbinmms/lib/python3.9/multiprocessing/process.py", line 315, in _bootstrap
self.run()
File "/home/ziaee/anaconda3/envs/sbinmms/lib/python3.9/multiprocessing/process.py", line 108, in run
self._target(*self._args, **self._kwargs)
File "/home/ziaee/anaconda3/envs/sbinmms/lib/python3.9/multiprocessing/pool.py", line 114, in worker
task = get()
File "/home/ziaee/anaconda3/envs/sbinmms/lib/python3.9/multiprocessing/queues.py", line 368, in get
return _ForkingPickler.loads(res)
File "cupy/_core/core.pyx", line 2250, in cupy._core.core.array
File "cupy/_core/core.pyx", line 2271, in cupy._core.core.array
File "cupy/_core/core.pyx", line 2403, in cupy._core.core._array_default
File "cupy/_core/core.pyx", line 171, in cupy._core.core.ndarray.__init__
File "cupy/cuda/memory.pyx", line 698, in cupy.cuda.memory.alloc
File "cupy/cuda/memory.pyx", line 1375, in cupy.cuda.memory.MemoryPool.malloc
File "cupy/cuda/memory.pyx", line 1395, in cupy.cuda.memory.MemoryPool.malloc
File "cupy/cuda/device.pyx", line 48, in cupy.cuda.device.get_device_id
File "cupy_backends/cuda/api/runtime.pyx", line 159, in cupy_backends.cuda.api.runtime.getDevice
File "cupy_backends/cuda/api/runtime.pyx", line 132, in cupy_backends.cuda.api.runtime.check_status
cupy_backends.cuda.api.runtime.CUDARuntimeError: cudaErrorInitializationError: initialization error
where am I doing wrong?
The issue has to do with the default start method not working with CUDA Multiprocessing. By explicitly setting the start method to spawn with multiprocessing.set_start_method('spawn', force=True) this issue is resolved.