I am figuring with an application with pyqt5. I use QMediaPlayer and QVideoWidget to play videos and what I need to do first is to get the current video frame when I pause the video. I have read the answer PyQt5 Access Frames with QmediaPlayer, and I have implemented the QAbstractVideoSurface and overridden the those methods. Finally I am able to get video frame and the QAbstractVideoSurface code shows as follows:
class VideoFrameGrabber(QAbstractVideoSurface):
frame_available = pyqtSignal(QImage)
def __init__(self, widget: QWidget, parent=None):
super().__init__(parent)
self.widget = widget
self.current_frame = None
self.image_format = QImage.Format_Invalid
self.target_rect = None
self.source_rect = None
self.image_size = None
def supportedPixelFormats(self, type):
print("supportedPixelFormats() called")
print("supportedPixelFormats() finished")
return [QVideoFrame.Format_ARGB32, QVideoFrame.Format_ARGB32_Premultiplied,
QVideoFrame.Format_RGB32, QVideoFrame.Format_RGB24, QVideoFrame.Format_RGB565,
QVideoFrame.Format_RGB555, QVideoFrame.Format_ARGB8565_Premultiplied,
QVideoFrame.Format_BGRA32, QVideoFrame.Format_BGRA32_Premultiplied, QVideoFrame.Format_BGR32,
QVideoFrame.Format_BGR24, QVideoFrame.Format_BGR565, QVideoFrame.Format_BGR555,
QVideoFrame.Format_BGRA5658_Premultiplied, QVideoFrame.Format_AYUV444,
QVideoFrame.Format_AYUV444_Premultiplied, QVideoFrame.Format_YUV444,
QVideoFrame.Format_YUV420P, QVideoFrame.Format_YV12, QVideoFrame.Format_UYVY,
QVideoFrame.Format_YUYV, QVideoFrame.Format_NV12, QVideoFrame.Format_NV21,
QVideoFrame.Format_IMC1, QVideoFrame.Format_IMC2, QVideoFrame.Format_IMC3,
QVideoFrame.Format_IMC4, QVideoFrame.Format_Y8, QVideoFrame.Format_Y16,
QVideoFrame.Format_Jpeg, QVideoFrame.Format_CameraRaw, QVideoFrame.Format_AdobeDng]
def isFormatSupported(self, format):
print("isFormatSupported() called")
image_format = QVideoFrame.imageFormatFromPixelFormat(format.pixelFormat())
size = format.frameSize()
print("isFormatSupported() finished")
return image_format != QVideoFrame.Format_Invalid and not size.isEmpty() and \
format.handleType() == QAbstractVideoBuffer.NoHandle
def start(self, format):
print("start() called")
image_format = QVideoFrame.imageFormatFromPixelFormat(format.pixelFormat())
size = format.frameSize()
if image_format != QImage.Format_Invalid and not size.isEmpty():
self.image_format = image_format
self.image_size = size
self.source_rect = format.viewport()
super().start(format)
# self.widget.updateGeometry()
# self.update_video_rect()
print("start() finished")
return True
else:
print("start() finished")
return False
def stop(self):
print("stop() called")
self.current_frame = QVideoFrame()
self.target_rect = QRect()
super().stop()
print("stop() finished")
def present(self, frame: QVideoFrame):
print("present called")
if frame.isValid():
clone_frame = QVideoFrame(frame)
clone_frame.map(QAbstractVideoBuffer.ReadOnly)
image = QImage(clone_frame.bits(), frame.width(), frame.height(), frame.bytesPerLine(), \
QVideoFrame.imageFormatFromPixelFormat(frame.pixelFormat()))
clone_frame.unmap()
# image.save("frame.jpg")
self.frame_available.emit(image)
if self.surfaceFormat().pixelFormat() != frame.pixelFormat() or \
self.surfaceFormat().frameSize() != frame.size():
self.setError(QAbstractVideoSurface.IncorrectFormatError)
self.stop()
print("present finished: Return False")
return False
else:
self.current_frame = frame
# self.widget.repaint(self.target_rect)
print("present finished: Return True")
return True
def update_video_rect(self):
print("update_video_rect() called")
size = self.surfaceFormat().sizeHint()
size.scale(self.widget.size().boundedTo(size), Qt.KeepAspectRatio)
self.target_rect = QRect(QPoint(0, 0), size)
self.target_rect.moveCenter(self.widget.rect().center())
print("update_video_rect() finished")
def paint(self, painter):
print("paint() called")
if self.current_frame.map(QAbstractVideoBuffer.ReadOnly):
old_transform = painter.transform()
if self.surfaceFormat().scanLineDirection() == QVideoSurfaceFormat.BottomToTop:
self.painter.scale(1, -1)
self.painter.translate(0, -self.widget.height())
image = QImage(self.current_frame.bits(), self.current_frame.width(), self.current_frame.height(),
self.current_frame.bytesPerLine(), self.image_format)
self.painter.drawImage(self.target_rect, image, self.source_rect)
self.painter.setTransform(old_transform)
self.current_frame.unmap()
print("paint() finished")
And because I use QVideoWidget to play videos, so I need to change the output of QMediaPlayer to my QAbstractVideoSurface to get video frames like this.
# definition here
class PlayerMainWindow(Ui_MainWindow, QMainWindow):
def __init__(self, parent=None):
self.player = QMediaPlayer(self)
self.player.setVideoOutput(self.video_widget)
self.auto_video_grabber = VideoFrameGrabber(self)
# what I need to do next is to detect faces in the frame
self.auto_video_grabber.frame_available.connect(self.auto_detect)
def auto_detect_clicked(self):
# the detector_widget is a widget contains serveral labels to show process information
# use signal and slot to update those lables
if self.detector_widget.is_detecting():
return
self.detector_widget.connect_face_available(self.on_face_available)
self.stackedWidget.setCurrentIndex(1) # the detector widget
self.player.setVideoOutput(self.auto_video_grabber)
self.player.pause()
# Receive frame(QImage) in auto_detect()
def auto_detect(self, frame: QImage):
self.player.setVideoOutput(self.video_widget)
self.player.pause()
if frame is None:
return
frame.save("auto_capture.jpg")
self.detector_widget.start_detector(frame)
But now the problem is that sometimes the frame I get in auto_detect() is not the current frame when I pause the video(the frame had been played) but sometimes it's correct. Also I tried to save the frame in present() method but it's the same as I received in auto_detect() method.
Oh my god, finally I solved the problem. The code shown above is ok, what I do is to change the video decoder from LAV to K-Lite.