pythonmatplotlibpyqtpyside6matplotlib-animation

Animated graph slows down progressively


I am trying to put an animated graph made with matplotlib funcAnimation into a PySide6 widget. It takes data in from a serial port, plots it, and records it into a csv. My issue is that after a set period of time, it starts to slow down significantly. While data is coming in every second, after about 20 minutes or so, it begins to only update every 2 seconds.

I tried changing the interval time and while that does make it a last a little longer, ultimately, they all still end up slowing down undesirably. I implemented a code which checks for execution time and noticed that it gradually increases until it is past over a second. I then tried taking it out of the widget but the same result ensues. Some other ideas I had were using blitting or turning it into a list/dict instead of a csv but just wanted to get some input before implementing. I saw a post on reddit with the same issue and in which blitting didn't work and he uses a csv so I'm not sure about my previous ideas: [Reddit Post](https://www.reddit.com/r/AskProgramming/comments/owde4a/matplotlib_animation_exponnetially_slower_the/ In this post they got a few suggestions which seemed to work for them but only this one applied to my code: Why are ax.set_xlabel(), ax.grid, etc. in your animate() function for each of the axes? These should be set one time when you create the axes, not on every call to animate(). So I changed that and put it into setup, this meant that I had to delete my ax.cla() in the animate loop. Now, it doesnt just show the last 10 values anymore so I need to fix that too.

My code:

`
import sys
from PySide6 import QtGui, QtCore
from PySide6.QtGui import QScreen, QPixmap
from pathlib import Path
from PySide6.QtWidgets import QWidget, QApplication, QPushButton, QVBoxLayout, QMainWindow, QHBoxLayout, QLabel
from matplotlib.backends.backend_qtagg import (
    FigureCanvas, NavigationToolbar2QT as NavigationToolbar)
from matplotlib.backends.backend_qtagg import NavigationToolbar2QT as NavigationToolbar
from matplotlib.figure import Figure
import matplotlib.pyplot as plt
from PySide6.QtCore import Qt
from matplotlib.animation import FuncAnimation
import pandas as pd
import serial
import matplotlib.image as image
import csv
import subprocess
import time
import openpyxl
import cProfile

#Initialize Serial Port
ser = serial.Serial()
ser.baudrate = 28800
ser.port = 'COM3'
timeout = 1
parity = serial.PARITY_ODD
bytesize = serial.EIGHTBITS
ser.open()

#Image Widget
class Widget(QWidget):
    def __init__(self):
        super().__init__()

        #Initialize value that will select one image if the value is reached and another if not
        self.int4_value = 0

        #Creating the pixmap
        self.image_label = QLabel()
        self.original_pixmap = QPixmap("C:/Users/mlee/Downloads/mid_green_background (1).png")

        self.scaled_pixmap = self.original_pixmap.scaled(20, 20)
        self.image_label.setPixmap(self.scaled_pixmap)
        #Layout
        v_layout = QVBoxLayout()
        v_layout.addWidget(self.image_label, alignment=Qt.AlignTop)

        self.setLayout(v_layout)

    def update_int4_value(self, new_value):
        #Updating value to change image
        self.int4_value = new_value
        if self.int4_value == 1365:
            self.original_pixmap = QPixmap("C:/Users/mlee/Downloads/mid_green_background (1).png")
        else:
            self.original_pixmap = QPixmap("C:/Users/mlee/Downloads/solid_red_background (1).jpg")
        self.scaled_pixmap = self.original_pixmap.scaled(20, 20)
        self.image_label.setPixmap(self.scaled_pixmap)


class Window(QMainWindow):
    def __init__(self):
        #Counts the number of screenshots
        self.screenshot_counter = self.load_screenshot_counter()
        super().__init__()
        #Import widget
        self.widget1 = Widget()
        self.app = app
        self.setWindowTitle("Custom")
        #Add menu bar
        menu_bar = self.menuBar()
        file_menu = menu_bar.addMenu("&File")
        quit_action = file_menu.addAction("Quit")
        quit_action.triggered.connect(self.quit)
        save_menu = menu_bar.addMenu("&Save")
        Screenshot = save_menu.addAction("Screenshot")
        Screenshot.triggered.connect(self.screenshot)
        #Set up graph as widget
        self._main = QWidget()
        self.setCentralWidget(self._main)
        layout = QHBoxLayout(self._main)
        self.fig = Figure(figsize=(5, 3))
        self.canvas = FigureCanvas(self.fig)
        layout.addWidget(self.canvas, stretch=24)
        layout.addWidget(self.widget1, stretch=1)

        #Set up toolbar
        self.addToolBar(NavigationToolbar(self.canvas, self))

        #Creating csv file to store incoming data and adding headers
        with open('csv_graph.csv', 'w', newline='') as csvfile:
            fieldnames = ['Value1', 'Value2', 'Value3', 'Value4', 'Value5', 'Value6']
            writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
            writer.writeheader()

        #Call the loop
        self.setup()

    def setup(self):
            #At 500ms interval, failed after 1400 frames
            #At 250 ms interval, failed after 1600 frames
            #At 1ms interval, failed after 1800 frames
        #Create the subplot
        self.ax = self.fig.add_subplot(111)
        # Legend and labels
        self.ax.legend(loc='upper left')
        self.ax.set_xlabel('Time(seconds)')
        self.ax.set_ylabel('Value')
        self.ax.set_title('Random Data')
        #Animation function
        self.ani = FuncAnimation(self.canvas.figure, self.animate, interval=1, cache_frame_data=False)

    def animate(self, i):
        #Timer to check execution time
        start_time = time.time()

        #Reading serial port data and only going htrough the loop if length is larger than 2
        # since the first input is normally all 0
        bytestoread = ser.inWaiting()
        new_value = ser.read(bytestoread)
        if len(new_value) >= 2:
            #Turning all the bytes into ints
            byte1 = new_value[2:4]  # Extract the first byte
            int1 = int.from_bytes(byte1, byteorder='little')
            print(int1)

            byte2 = new_value[4:6]  # Extract the second byte
            int2 = int.from_bytes(byte2, byteorder='little')
            print(int2)

            byte3 = new_value[6:8]  # Extract the third byte
            int3 = int.from_bytes(byte3, byteorder='little')
            print(int3)

            byte4 = new_value[8:10]
            int4 = int.from_bytes(byte4, byteorder='little')
            print(int4)

            #Pass int4 to the image widget
            self.widget1.update_int4_value(int4)

            byte5 = new_value[10:12]  # Day & Hour
            int5 = int.from_bytes(byte5, byteorder='little')
            # print(int5)

            byte6 = new_value[12:14]  # Minutes & Seconds
            int6 = int.from_bytes(byte6, byteorder='little')
            print(int6)


            #Write the data into the csv
            with open('csv_graph.csv', 'a', newline='') as csvfile:
                csv_writer = csv.writer(csvfile)
                csv_writer.writerow([int1, int2, int3, int4, int5, int6])

            #Read from that csv and then take the last 10 rows
            data = pd.read_csv('csv_graph.csv')
            last_x_rows = data.tail(10)

            #Assigning the values to variables
            x = last_x_rows['Value6']
            y1 = last_x_rows['Value1']
            y2 = last_x_rows['Value2']
            y3 = last_x_rows['Value3']
            y4 = last_x_rows['Value4']

            # Plotting
            # self.ax.cla()
            self.ax.plot(x, y1, color='Red', label='Red')
            self.ax.plot(x, y2, color='Blue', label='Blue')
            self.ax.plot(x, y3, color='Purple', label='Purple')
            self.ax.plot(x, y4, color='Green', label='Green')


            #Opening the csv in an excel
            with pd.ExcelWriter("C:/Users/mlee/Documents/Excel_CSV/New_Excel.xlsx", mode='a', engine='openpyxl',
                                if_sheet_exists='replace') as writer:
                data.to_excel(writer, sheet_name='Sheet_1')

            #Execution time check
            end_time = time.time()
            self.execution_time = end_time - start_time
            print(f"Frame {i}: Execution Time = {self.execution_time:.2f} seconds")

            #Was trying to use blitting but didn't do it right
            return self.ax

    #Functions for menu bar
    def quit(self):
        self.app.quit()

    def load_screenshot_counter(self):
        counter_file = Path("screenshot_counter.txt")
        if counter_file.exists():
            with counter_file.open("r") as file:
                return int(file.read())
        else:
            return 1

    def save_screenshot_counter(self):
        counter_file = Path("C:/Users/mlee/Downloads/screenshot_counter.txt")
        with counter_file.open("w") as file:
            file.write(str(self.screenshot_counter))

    def screenshot(self):
        # Get the primary screen
        primary_screen = QApplication.primaryScreen()

        # Get the available size of the screen (excluding reserved areas)
        available_size = primary_screen.availableSize()
        print(available_size)

        shot = QScreen.grabWindow(QApplication.primaryScreen(), 0, 90, 95, 1410, 700)
        file_path = f"C:/Users/mlee/Downloads/Screenshot_{self.screenshot_counter}.png"
        shot.save(file_path, "PNG")
        self.screenshot_counter += 1
        self.save_screenshot_counter()


app = QApplication(sys.argv)
widget = Window()

widget.show()
with cProfile.Profile() as pr:
    sys.exit(app.exec())
    pr.print_stats(sort='cumtime')
`

Solution

  • Well firstly, you are reading the .csv file inside of your animation function which is not so good because the file is getting bigger each time it stores the data inside of it and therefore when it reads the data it takes a lot of time, and re-plotting all the data each frame can be inefficient. Instead, updating the existing plot data is typically more efficient. First solve the issue of reading data from .csv you can store the data more efficiently too with the "collections" module called deque. Deque has fast appends so when you're continuously adding new data points and need to maintain only a fixed number of the most recent ones, a deque with a set maxlen automatically discards the oldest items when the new ones are added. This behaevior is ideal for your case.

    from collections import deque
    

    then you can make the buffer for the last 10 data points from deque

        self.data_buffer = deque(maxlen=10)  # Buffer to store the last 10 data points
        self.csv_data = []  # Make a list for storing the data
    

    and if you are going to write the information inside of the .csv then I would suggest that you write the data in batches instead on every frame.Make a variable something like this.

        self.csv_batch_size = 50  # Adjust this accordingly 
    

    then in your animate function you can make an append so you store the data

        new_data = [int1, int2, int3, int4, int5, int6]
        self.data_buffer.append(new_data)
        self.csv_data.append(new_data)
    

    then you can make an if statement for checking the size

        if len(self.csv_data) >= self.csv_batch_size:
                self.write_to_csv()
                self.csv_data = []
    

    then you update the plot

        df = pd.DataFrame(list(self.data_buffer), columns=['Value1', 'Value2', 'Value3', 'Value4', 'Value5', 'Value6'])
        x = range(len(df))
        self.ax.clear()
        self.ax.plot(x, df['Value1'], color='Red')
        self.ax.plot(x, df['Value2'], color='Blue')
    

    then you can make a function that is called inside of the if statement to save the data in the csv.

    def write_to_csv(self):
        with open('csv_graph.csv', 'a', newline='') as csvfile:
            csv_writer = csv.writer(csvfile)
            csv_writer.writerows(self.csv_data)
    

    that is the first step I would take, and not to mention that you are writing in the excel every frame that is costly too so I would do that on demand or less frequently.