I'm trying to compute optical flow between two frames and then warp the previous frame using the computed optical flow. I found cv2 has Farneback Optical FLow and so I'm using that to compute Flow. I took the default parameters from the cv2 tutorial and I'm warping the frame using the code given in this answer. But when I see the warped frame, it is exactly as previous frame and no change (arrays are equal).
With further debugging, I found that the computed flow values are too low. Why is this happening? Am I doing something wrong?
Code:
def get_optical_flow(prev_frame: numpy.ndarray, next_frame: numpy.ndarray) -> numpy.ndarray:
prev_gray = skimage.color.rgb2gray(prev_frame)
next_gray = skimage.color.rgb2gray(next_frame)
flow = cv2.calcOpticalFlowFarneback(prev_gray, next_gray, None, 0.5, 3, 15, 3, 5, 1.2, 0)
return flow
def warp_frame(prev_frame: numpy.ndarray, flow: numpy.ndarray):
h, w = flow.shape[:2]
flow = -flow
flow[:,:,0] += numpy.arange(w)
flow[:,:,1] += numpy.arange(h)[:,numpy.newaxis]
# res = cv2.remap(img, flow, None, cv2.INTER_LINEAR)
next_frame = cv2.remap(prev_frame, flow, None, cv2.INTER_LINEAR)
return next_frame
def demo1():
prev_frame_path = Path('./frame025.png')
next_frame_path = Path('./frame027.png')
prev_frame = skimage.io.imread(prev_frame_path.as_posix())
next_frame = skimage.io.imread(next_frame_path.as_posix())
flow = get_optical_flow(prev_frame, next_frame)
print(f'Flow: max:{flow.max()}, min:{flow.min()}, mean:{flow.__abs__().mean()}')
warped_frame = warp_frame(prev_frame, flow)
print(numpy.array_equal(prev_frame, warped_frame))
pyplot.subplot(1,3,1)
pyplot.imshow(prev_frame)
pyplot.subplot(1,3,2)
pyplot.imshow(next_frame)
pyplot.subplot(1,3,3)
pyplot.imshow(warped_frame)
pyplot.show()
return
Output:
Warped Image is exactly the same as prev image, while it should look like next image.
Any help is appreciated!
The issue is with converting the rgb frames to gray. skimage.color.rgb2gray()
changes the intensity range from [0,255]
to [0,1]
. Changing it back to [0,255]
worked!
def get_optical_flow(prev_frame: numpy.ndarray, next_frame: numpy.ndarray) -> numpy.ndarray:
prev_gray = (skimage.color.rgb2gray(prev_frame) * 255).astype('uint8')
next_gray = (skimage.color.rgb2gray(next_frame) * 255).astype('uint8')
flow = cv2.calcOpticalFlowFarneback(prev_gray, next_gray, None, 0.5, 3, 15, 3, 5, 1.2, 0)
return flow