pythonpyqtpyqt5eye-tracking

How to draw eye gaze data from eye tracker by PyQt while a video is playing


I am working bulid a eye gaze visualization tool just like this by PyQt5, and I also checked this post. here is the code by modifing the above links. turn out it's worked, but the video always get stuck sometime (the audio is normal, the the frames stuck, both video content and ellipse,just like this video),anyone can help?

import os
import time

from PyQt5 import QtCore, QtGui, QtWidgets, QtMultimedia, QtMultimediaWidgets
import tobii_research as tr
import numpy as np

class Widget(QtWidgets.QWidget):

    def __init__(self, parent=None):

        super(Widget, self).__init__(parent)

        #first window,just have a single button for play the video
        self.resize(256, 256)
        self.btn_play = QtWidgets.QPushButton(self)
        self.btn_play.setGeometry(QtCore.QRect(100, 100, 28, 28))
        self.btn_play.setObjectName("btn_open")
        self.btn_play.setText("Play")
        self.btn_play.clicked.connect(self.Play_video)#click to play video
        #

        self._scene = QtWidgets.QGraphicsScene(self)
        self._gv = QtWidgets.QGraphicsView(self._scene)
        #construct a videoitem for showing the video
        self._videoitem = QtMultimediaWidgets.QGraphicsVideoItem()
        #add it into the scene
        self._scene.addItem(self._videoitem)

        # assign _ellipse_item is the gaze data, and embed it into videoitem,so it can show above the video.
        self._ellipse_item = QtWidgets.QGraphicsEllipseItem(QtCore.QRectF(0, 0, 40, 40), self._videoitem)
        self._ellipse_item.setBrush(QtGui.QBrush(QtCore.Qt.black))
        self._ellipse_item.setPen(QtGui.QPen(QtCore.Qt.red))
        #self._scene.addItem(self._ellipse_item)
        self._gv.fitInView(self._videoitem)

        self._player = QtMultimedia.QMediaPlayer(self, QtMultimedia.QMediaPlayer.VideoSurface)
        self._player.setVideoOutput(self._videoitem)
        file = os.path.join(os.path.dirname(__file__), "video.mp4")#video.mp4 is under the same dirctory
        self._player.setMedia(QtMultimedia.QMediaContent(QtCore.QUrl.fromLocalFile(file)))
        print(f"self._videoitem::{self._videoitem.size()}")

        #get eye tracker
        self.eyetrackers = tr.find_all_eyetrackers()
        self.my_eyetracker = self.eyetrackers[0]

    def gaze_data_callback(self, gaze_data_):

        #for now, I don't know the coordinate system,just randomly assign the gaze data to test the functionality
        self._ellipse_item.setPos(float(np.random.choice(range(0, 300))), float(np.random.choice(range(0, 240))))
        print("time.time()::{}".format(time.time()))


    def Play_video(self):
        self.my_eyetracker.subscribe_to(tr.EYETRACKER_GAZE_DATA, self.gaze_data_callback, as_dictionary=True)
        #size = QtCore.QSizeF(1920.0, 1080.0)#I hope it can fullscreen the video
        #self._videoitem.setSize(size)
        #self._gv.showFullScreen()
        self._gv.resize(720,720)
        self._gv.show()
        self._player.play()


if __name__ == '__main__':
    import sys
    app = QtWidgets.QApplication(sys.argv)
    w = Widget()
    w.show()
    sys.exit(app.exec_())

cmd printout information here


Solution

  • According to the warning message:

    QObject::startTimer: Timers can only be used with threads started with QThread
    

    it can be deduced that the callback associated with "my_eyetracker" is executed in a secondary thread, so the position of the item from a different thread to the main thread would be updated, which could generate the problem described by the OP.

    The solution is to send the callback information to the guide through signals.

    import os
    import time
    
    from PyQt5 import QtCore, QtGui, QtWidgets, QtMultimedia, QtMultimediaWidgets
    import tobii_research as tr
    import numpy as np
    
    
    class EyeTracker(QtCore.QObject):
        positionChanged = QtCore.pyqtSignal(float, float)
    
        def __init__(self, tracker, parent=None):
            super(EyeTracker, self).__init__(parent)
            self._tracker = tracker
    
        @property
        def tracker(self):
            return self._tracker
    
        def start(self):
            self.tracker.subscribe_to(
                tr.EYETRACKER_GAZE_DATA, self._callback, as_dictionary=True
            )
    
        def _callback(self, gaze_data_):
            self.positionChanged.emit(
                float(np.random.choice(range(0, 300))),
                float(np.random.choice(range(0, 240))),
            )
            print("time.time()::{}".format(time.time()))
    
    
    class Widget(QtWidgets.QWidget):
        def __init__(self, parent=None):
    
            super(Widget, self).__init__(parent)
    
            # first window,just have a single button for play the video
            self.resize(256, 256)
            self.btn_play = QtWidgets.QPushButton(self)
            self.btn_play.setGeometry(QtCore.QRect(100, 100, 28, 28))
            self.btn_play.setObjectName("btn_open")
            self.btn_play.setText("Play")
            self.btn_play.clicked.connect(self.Play_video)  # click to play video
            #
    
            self._scene = QtWidgets.QGraphicsScene(self)
            self._gv = QtWidgets.QGraphicsView(self._scene)
            # construct a videoitem for showing the video
            self._videoitem = QtMultimediaWidgets.QGraphicsVideoItem()
            # add it into the scene
            self._scene.addItem(self._videoitem)
    
            # assign _ellipse_item is the gaze data, and embed it into videoitem,so it can show above the video.
            self._ellipse_item = QtWidgets.QGraphicsEllipseItem(
                QtCore.QRectF(0, 0, 40, 40), self._videoitem
            )
            self._ellipse_item.setBrush(QtGui.QBrush(QtCore.Qt.black))
            self._ellipse_item.setPen(QtGui.QPen(QtCore.Qt.red))
            # self._scene.addItem(self._ellipse_item)
            self._gv.fitInView(self._videoitem)
    
            self._player = QtMultimedia.QMediaPlayer(
                self, QtMultimedia.QMediaPlayer.VideoSurface
            )
            self._player.setVideoOutput(self._videoitem)
            file = os.path.join(
                os.path.dirname(__file__), "video.mp4"
            )  # video.mp4 is under the same dirctory
            self._player.setMedia(
                QtMultimedia.QMediaContent(QtCore.QUrl.fromLocalFile(file))
            )
            print(f"self._videoitem::{self._videoitem.size()}")
    
            # get eye tracker
            eyetrackers = tr.find_all_eyetrackers()
            self.tracker = EyeTracker(eyetrackers[0])
            self.tracker.positionChanged.connect(self._ellipse_item.setPos)
    
        def Play_video(self):
            self.tracker.start()
            # size = QtCore.QSizeF(1920.0, 1080.0)#I hope it can fullscreen the video
            # self._videoitem.setSize(size)
            # self._gv.showFullScreen()
            self._gv.resize(720, 720)
            self._gv.show()
            self._player.play()
    
    
    if __name__ == "__main__":
        import sys
    
        app = QtWidgets.QApplication(sys.argv)
        w = Widget()
        w.show()
        sys.exit(app.exec_())