pythonopencvpygtkpyopenglpygobject

How to render OpenCV frames in OpenGL 3.2 GtkGLArea in Python?


How do I render images captured by OpenCV in OpenGL 3.2 GtkGLArea in Python? Many of the examples online are 10 years old, outdated and are using OpenGL 2.1 or 1.1 examples.

The only way I can do this is through shaders. The only working example to get anything to draw in a GtkGLArea is this one:
Using Gtk GLArea in Python GTK3

I found this example in C++ and translated it into Python but I cannot get it to work or output anything.
How to Display Image using OpenGL

Here is my code, it does not display anything. What am I doing wrong?

Write Display Function called by OpenCV capture thread:

import os
import gi
gi.require_version("Gtk", "3.0")
from gi.repository import Gtk, GdkPixbuf, GLib

USE_OPENGL = True

def writeDisplay(uiBuilder, frame, imageDisplay):
    # Write Frame
    frame = cv.cvtColor(frame, cv.COLOR_BGR2RGB)

    if USE_OPENGL:
        # Render frame using OpenGL
        GLib.idle_add(imageDisplay.render, frame)

OpenGL Renderer GTK Widget

import gi
gi.require_version('Gtk', '3.0')
from gi.repository import Gtk
from OpenGL.GL import *
from OpenGL.GL import shaders
import numpy as np

VERTEX_SOURCE = '''
#version 330
layout (location=0) in vec3 position;
layout (location=1) in vec3 color;
layout (location=2) in vec2 texCoord;
out vec3 ourColor;
out vec2 TexCoord;
void main()
{
gl_Position = vec4(position,1.0);
ourColor = color;
TexCoord= vec2(texCoord.x,1.0-texCoord.y);
}'''

FRAGMENT_SOURCE ='''
#version 330
in vec3 ourColor;
in vec2 TexCoord;
out vec4 color;
uniform sampler2D ourTexture;
void main(){
color = texture(ourTexture , TexCoord);
};'''

recVertices = np.array([
    # Positions           Colors           Texture Coords
    0.5,  0.5, 0.0,   1.0, 0.0, 0.0,    1.0, 1.0,   # Top Right
    0.5, -0.5, 0.0,   0.0, 1.0, 0.0,    1.0, 0.0,   # Bottom Right
    -0.5, -0.5, 0.0,   0.0, 0.0, 1.0,   0.0, 0.0,   # Bottom Left
    -0.5,  0.5, 0.0,   1.0, 1.0, 0.0,   0.0, 1.0    # Top Left
], dtype=np.float32)

indices = np.array([
    0, 1, 3, # First Triangle
    1, 2, 3  # Second Triangle
])

def checkGlError(op: str):
    error = glGetError()
    if error is not None and error != 0:
        print("after %s() glError (0x%x)", op, error)

# Based on examples:
# https://stackoverflow.com/questions/42153819/how-to-load-and-display-an-image-in-opengl-es-3-0-using-c
# https://stackoverflow.com/questions/47565884/use-of-the-gtk-glarea-in-pygobject-gtk3
class OpenGLRenderer(Gtk.GLArea):
    def __init__(self):
        Gtk.GLArea.__init__(self)
        self.connect("realize", self.onRealize)
        self.connect("render", self.onRender)
        self.ctx = None
        self.frame = None
        self.area = None
        self.shaderProgram = None
        self.positionHandle = None
        self.textureId = None
        self.vao = None

    def onRealize(self, area):

        error = area.get_error()
        if error != None:
            print("your graphics card is probably too old : ", error)
        else:
            print(area, "realize... fine so far")

        self.ctx = self.get_context()
        self.ctx.make_current()

        print("OpenGL realized", self.ctx)

    def onRender(self, area, ctx):
        
        self.render(self.frame)
        return True

    def setupGraphics(self, width, height):

        if self.shaderProgram is None:
            # Load Shaders, Create program, Setup Graphics
            vertexShader = glCreateShader(GL_VERTEX_SHADER)
            glShaderSource(vertexShader, VERTEX_SOURCE)
            glCompileShader(vertexShader)

            pixelShader = glCreateShader(GL_FRAGMENT_SHADER)
            glShaderSource(pixelShader, FRAGMENT_SOURCE)
            glCompileShader(pixelShader)

            self.shaderProgram = glCreateProgram()
            glAttachShader(self.shaderProgram, vertexShader)
            glAttachShader(self.shaderProgram, pixelShader)
            glLinkProgram(self.shaderProgram)
            self.positionHandle = glGetAttribLocation(self.shaderProgram, "position")

        glViewport(0, 0, width, height)
    
    def initBuffers(self):
        # Initialize an buffer to store all the verticles and transfer them to the GPU
        self.vao = glGenVertexArrays(1) # Generate VAO
        vbos = glGenBuffers(1) # Generate VBO
        ebo = glGenBuffers(1) # Generate EPBO
        glBindVertexArray(self.vao) # Bind the Vertex Array

        glBindBuffer(GL_ARRAY_BUFFER, vbos) # Bind verticles array for OpenGL to use
        glBufferData(GL_ARRAY_BUFFER, len(recVertices), recVertices, GL_STATIC_DRAW)

        glBindBuffer(GL_ELEMENT_ARRAY_BUFFER, ebo) # Bind the indices for information about drawing sequence
        glBufferData(GL_ELEMENT_ARRAY_BUFFER, len(indices), indices, GL_STATIC_DRAW)
        
        # 1. set the vertex attributes pointers
        # Position Attribute
        glVertexAttribPointer(0, 3, GL_FLOAT, GL_FALSE, 8 * sizeof(GLfloat), ctypes.c_void_p(0))
        glEnableVertexAttribArray(0)
        # Color Attribute
        glVertexAttribPointer(1, 3, GL_FLOAT, GL_FALSE, 8 * sizeof(GLfloat), ctypes.c_void_p(3 * sizeof(GLfloat)))
        glEnableVertexAttribArray(1)
        # Texture Coordinate Attribute
        glVertexAttribPointer(2, 2, GL_FLOAT, GL_FALSE, 8 * sizeof(GLfloat), ctypes.c_void_p(6 * sizeof(GLfloat)))
        glEnableVertexAttribArray(2)

        glBindVertexArray(0) # 3. Unbind VAO
    
    def generateTexture(self, frame):
        # Update Frame
        self.frame = frame

        # If we have a frame to display
        if frame is not None:
            # extract array from Image
            h, w, d = frame.shape

            # Generate Texture
            self.textureId = glGenTextures(1)
            glBindTexture(GL_TEXTURE_2D, self.textureId) # Bind our 2D texture so that following set up will be applied

            # Set texture wrapping parameter
            glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_MIRRORED_REPEAT)
            glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_MIRRORED_REPEAT)

            # Set texture Filtering parameter
            glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST)
            glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST)

            glTexImage2D(GL_TEXTURE_2D, 0, GL_RGB, w, h, 0, GL_RGB, GL_UNSIGNED_BYTE, frame)
            glGenerateMipmap(GL_TEXTURE_2D)
            glBindTexture(GL_TEXTURE_2D, 0) # Unbind 2D textures

    def render(self, frame):

        # Set OpenGL Render Context
        if self.ctx is not None and frame is not None:
            self.ctx.make_current()

            # extract array from Image
            h, w, d = frame.shape

            # Initialize Graphics
            self.setupGraphics(w, h)

            # Generate Texture
            self.generateTexture(frame)

            # Clear Screen
            glClearColor(0, 0, 1, 1)
            glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT)

            # Render Frame
            glUseProgram(self.shaderProgram)
            # checkGlError("glUseProgram")

            # glVertexAttribPointer(self.positionHandle, 2, GL_FLOAT, GL_FALSE, 0, recVertices)
            # checkGlError("glVertexAttribPointer")
            # glEnableVertexAttribArray(self.positionHandle)
            # checkGlError("glEnableVertexAttribArray")
            # glDrawArrays(GL_TRIANGLE_FAN, 0, 4)
            # checkGlError("glDrawArrays")
            glActiveTexture(GL_TEXTURE0)
            checkGlError("glActiveTexture")
            glBindTexture(GL_TEXTURE_2D, self.textureId)
            checkGlError("glBindTexture")
            mlocation = glGetUniformLocation(self.shaderProgram, "ourTexture")
            checkGlError("glGetUniformLocation")
            glUniform1i(mlocation, 0)
            checkGlError("glUniform1i")
            self.initBuffers()
            glBindVertexArray(self.vao)
            checkGlError("glBindVertexArray")
            glDrawElements(GL_TRIANGLES, 6, GL_UNSIGNED_INT, 0)
            
            # Queue Draw
            glFlush()
            self.queue_draw()

Solution

  • Here is the answer, this is tested with OpenCV frames which are a 3 dimentional array [width][height][RGB Color] which need to be flattened into a linear array. OpenGL will take the width and height of the image and traverse it properly. This guide was immensely helpful in getting this to work: https://open.gl/drawing

    import gi
    gi.require_version('Gtk', '3.0')
    from gi.repository import Gtk
    from OpenGL.GL import *
    import numpy as np
    
    VERTEX_SOURCE = '''
    #version 330
    layout (location=0) in vec3 position;
    layout (location=1) in vec3 color;
    layout (location=2) in vec2 texCoord;
    out vec3 ourColor;
    out vec2 TexCoord;
    void main()
    {
    gl_Position = vec4(position,1.0);
    ourColor = color;
    TexCoord= vec2(texCoord.x,1.0-texCoord.y);
    }'''
    
    FRAGMENT_SOURCE ='''
    #version 330
    in vec3 ourColor;
    in vec2 TexCoord;
    out vec4 color;
    uniform sampler2D ourTexture;
    void main(){
    color = texture(ourTexture, TexCoord);
    };'''
    
    recVertices = np.array([
        # Positions           Colors           Texture Coords
        1.0,  1.0, 0.0,   1.0, 0.0, 0.0,    1.0, 1.0,   # Top Right    0
        1.0, -1.0, 0.0,   0.0, 1.0, 0.0,    1.0, 0.0,   # Bottom Right 1
        -1.0, -1.0, 0.0,   0.0, 0.0, 1.0,   0.0, 0.0,   # Bottom Left  2
        -1.0,  1.0, 0.0,   1.0, 1.0, 0.0,   0.0, 1.0,   # Top Left     3
        1.0,  1.0, 0.0,   1.0, 0.0, 0.0,    1.0, 1.0,   # Top Right    4
    ], dtype=np.float32)
    
    def checkGlError(op: str):
        error = glGetError()
        if error is not None and error != 0:
            print("after %s() glError (0x%x)", op, error)
    
    # Based on examples:
    # https://stackoverflow.com/questions/42153819/how-to-load-and-display-an-image-in-opengl-es-3-0-using-c
    # https://stackoverflow.com/questions/47565884/use-of-the-gtk-glarea-in-pygobject-gtk3
    class OpenGLRenderer(Gtk.GLArea):
        def __init__(self):
            Gtk.GLArea.__init__(self)
            self.connect("realize", self.onRealize)
            self.connect("render", self.onRender)
            self.ctx = None
            self.frame = None
            self.area = None
            self.shaderProgram = None
            self.positionHandle = None
            self.textureId = None
            self.vao = None
            self.vbos = None
    
        def onRealize(self, area):
    
            error = area.get_error()
            if error != None:
                print("your graphics card is probably too old : ", error)
            else:
                print(area, "realize... fine so far")
    
            self.ctx = self.get_context()
            self.ctx.make_current()
    
            print("OpenGL realized", self.ctx)
    
        def onRender(self, area, ctx):
            
            self.render(self.frame)
            return True
    
        def setupGraphics(self):
    
            if self.shaderProgram is None:
                # Load Shaders, Create program, Setup Graphics
                vertexShader = glCreateShader(GL_VERTEX_SHADER)
                glShaderSource(vertexShader, VERTEX_SOURCE)
                glCompileShader(vertexShader)
                status = glGetShaderiv(vertexShader, GL_COMPILE_STATUS)
                print("Compile vertexShader status: " + str(status == GL_TRUE))
    
                pixelShader = glCreateShader(GL_FRAGMENT_SHADER)
                glShaderSource(pixelShader, FRAGMENT_SOURCE)
                glCompileShader(pixelShader)
                status = glGetShaderiv(pixelShader, GL_COMPILE_STATUS)
                print("Compile vertexShader status: " + str(status == GL_TRUE))
    
                self.shaderProgram = glCreateProgram()
                glAttachShader(self.shaderProgram, vertexShader)
                glAttachShader(self.shaderProgram, pixelShader)
                glLinkProgram(self.shaderProgram)
                glBindFragDataLocation(self.shaderProgram, 0, "color")
                self.positionHandle = glGetAttribLocation(self.shaderProgram, "position")
    
                # Initalize Vertex Buffers
                self.initBuffers()
        
        def initBuffers(self):
            # Initialize an buffer to store all the verticles and transfer them to the GPU
            self.vao = glGenVertexArrays(1) # Generate VAO
            self.vbos = glGenBuffers(1) # Generate VBO
            glBindVertexArray(self.vao) # Bind the Vertex Array
    
            glBindBuffer(GL_ARRAY_BUFFER, self.vbos) # Bind verticles array for OpenGL to use
            glBufferData(GL_ARRAY_BUFFER, sizeof(GLfloat) * len(recVertices), recVertices, GL_STATIC_DRAW)
            
            # 1. set the vertex attributes pointers
            # Position Attribute
            glVertexAttribPointer(0, 3, GL_FLOAT, GL_FALSE, 8 * sizeof(GLfloat), ctypes.c_void_p(0))
            glEnableVertexAttribArray(0)
            # Color Attribute
            glVertexAttribPointer(1, 3, GL_FLOAT, GL_FALSE, 8 * sizeof(GLfloat), ctypes.c_void_p(3 * sizeof(GLfloat)))
            glEnableVertexAttribArray(1)
            # Texture Coordinate Attribute
            glVertexAttribPointer(2, 2, GL_FLOAT, GL_FALSE, 8 * sizeof(GLfloat), ctypes.c_void_p(6 * sizeof(GLfloat)))
            glEnableVertexAttribArray(2)
    
            glBindVertexArray(0) # 3. Unbind VAO
        
        def generateTexture(self, frame):
            # Update Frame
            self.frame = frame
    
            # Delete previous textures to avoid memory leak
            if self.textureId is not None:
                glDeleteTextures(1, [self.textureId])
    
            # If we have a frame to display
            if frame is not None:
                # extract array from Image
                h, w, d = frame.shape
    
                # Frame is a 3 dimentional array where shape eg. (1920, 1080, 3)
                # Where it is w, h, and 3 values for color
                # https://www.educba.com/numpy-flatten/
                pixels = frame.flatten(order = 'C')
    
                # Generate Texture
                self.textureId = glGenTextures(1)
                glBindTexture(GL_TEXTURE_2D, self.textureId) # Bind our 2D texture so that following set up will be applied
    
                # Set texture wrapping parameter
                glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE)
                glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE)
    
                # Set texture Filtering parameter
                glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR)
                glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR)
    
                glTexImage2D(GL_TEXTURE_2D, 0, GL_RGB, w, h, 0, GL_RGB, GL_UNSIGNED_BYTE, pixels)
                glGenerateMipmap(GL_TEXTURE_2D)
                glBindTexture(GL_TEXTURE_2D, 0) # Unbind 2D textures
    
        def render(self, frame):
    
            # Set OpenGL Render Context
            if self.ctx is not None and frame is not None:
                self.ctx.make_current()
    
                # Initialize Graphics
                self.setupGraphics()
    
                # Generate Texture
                self.generateTexture(frame)
    
                # Clear Screen
                glClearColor(0, 0, 1, 1)
                glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT)
    
                # Use Shader Program, Bind Vertex Array and Texture
                glUseProgram(self.shaderProgram)
                checkGlError("glUseProgram")
                glActiveTexture(GL_TEXTURE0)
                checkGlError("glActiveTexture")
                glBindTexture(GL_TEXTURE_2D, self.textureId)
                checkGlError("glBindTexture")
                mlocation = glGetUniformLocation(self.shaderProgram, "ourTexture")
                checkGlError("glGetUniformLocation")
                glUniform1i(mlocation, 0)
                checkGlError("glUniform1i")
                glBindVertexArray(self.vao)
                checkGlError("glBindVertexArray")
    
                # Render Frame
                glDrawArrays(GL_TRIANGLES, 0, 3)
                glDrawArrays(GL_TRIANGLES, 2, 3)
                
                # Queue Draw
                glFlush()
                self.queue_draw()
    

    Rendering Textures in GtkGLArea in Python Full Example

    import gi
    
    gi.require_version("Gtk", "3.0")
    from gi.repository import Gtk
    import numpy as np
    from OpenGL.GL import *
    from OpenGL.GL import shaders
    
    FRAGMENT_SOURCE ='''
    #version 330
    in vec3 Color;
    in vec2 Texcoord;
    out vec4 outColor;
    uniform sampler2D imageTexture;
    void main()
    {
        outColor = texture(imageTexture, Texcoord);
    }'''
    
    VERTEX_SOURCE = '''
    #version 330
    layout (location=0) in vec3 position;
    layout (location=1) in vec3 color;
    layout (location=2) in vec2 texcoord;
    out vec3 Color;
    out vec2 Texcoord;
    void main()
    {
        Color = color;
        Texcoord = texcoord;
        gl_Position = vec4(position, 1.0);
    }'''
    
    def on_realize(self, area):        
        # We need to make the context current if we want to
        # call GL API
        area.make_current()
    
    def on_render(area, context):
        print("%s\n", glGetString(GL_VERSION))
        area.make_current()
    
        ############################################
        # Init Shaders
        ############################################
        glClearColor(0, 0, 0, 1)
        glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT)
        VERTEX_SHADER_PROG = shaders.compileShader(VERTEX_SOURCE, GL_VERTEX_SHADER)
        FRAGMENT_SHADER_PROG = shaders.compileShader(FRAGMENT_SOURCE, GL_FRAGMENT_SHADER)
        shaderProgram = shaders.compileProgram(VERTEX_SHADER_PROG, FRAGMENT_SHADER_PROG)
        
        ############################################
        # Init Buffers
        ############################################
        # Create a new VAO (Vertex Array Object) and bind it
        vao = glGenVertexArrays(1)
        glBindVertexArray(vao)
        # Generate buffers to hold our vertices
        vertex_buffer = glGenBuffers(1)
        glBindBuffer(GL_ARRAY_BUFFER, vertex_buffer)
        # Send the data over to the buffer
        vertices = np.array([
            # Positions      Color           Texchords 
            1.0,  1.0, 0.0,  0.0, 1.0, 0.0,  1.0, 0.0, # Top Right       0
            1.0, -1.0, 0.0,  0.0, 0.0, 1.0,  1.0, 1.0, # Bottom Right    1
            -1.0, -1.0, 0.0, 1.0, 1.0, 1.0,  0.0, 1.0, # Bottom Left     2
            -1.0,  1.0, 0.0, 1.0, 0.0, 0.0,  0.0, 0.0, # Top Left        3
            1.0,  1.0, 0.0,  0.0, 1.0, 0.0,  1.0, 0.0, # Top Right       4
        ], dtype=np.float32)
        
        size = sizeof(GLfloat) * len(vertices)
        glBufferData(GL_ARRAY_BUFFER, size, vertices, GL_STATIC_DRAW)
    
        # Specify the layout of the vertex data
        posAttrib = glGetAttribLocation(shaderProgram, "position")
        glEnableVertexAttribArray(posAttrib)
        glVertexAttribPointer(posAttrib, 3, GL_FLOAT, GL_FALSE, 8 * sizeof(GLfloat), ctypes.c_void_p(0))
    
        colAttrib = glGetAttribLocation(shaderProgram, "color")
        glEnableVertexAttribArray(colAttrib)
        glVertexAttribPointer(colAttrib, 3, GL_FLOAT, GL_FALSE, 8 * sizeof(GLfloat), ctypes.c_void_p(3 * sizeof(GLfloat)))
    
        texAttrib = glGetAttribLocation(shaderProgram, "texcoord")
        glEnableVertexAttribArray(texAttrib)
        glVertexAttribPointer(texAttrib, 2, GL_FLOAT, GL_FALSE, 8 * sizeof(GLfloat), ctypes.c_void_p(6 * sizeof(GLfloat)))
    
        # Unbind the VAO first (Important)
        glBindVertexArray(0)
    
        ############################################
        # Render
        ############################################
        glBindBuffer(GL_ARRAY_BUFFER, 0)
        glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT)
        glUseProgram(shaderProgram)
        glBindVertexArray(vao)
    
        # Load Textures
        width = 2
        height = 2
        textureId = glGenTextures(1)
        glActiveTexture(GL_TEXTURE0)
        glBindTexture(GL_TEXTURE_2D, textureId)
    
        # Black/white checkerboard
        pixels = [
            0.0, 0.0, 0.0,   1.0, 1.0, 1.0,
            1.0, 1.0, 1.0,   0.0, 0.0, 0.0
        ]
    
        glTexImage2D(GL_TEXTURE_2D, 0, GL_RGB, width, height, 0, GL_RGB, GL_FLOAT, pixels)
        glUniform1i(glGetUniformLocation(shaderProgram, "imageTexture"), 0)
        glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE)
        glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE)
        glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST)
        glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST)
    
        glDrawArrays(GL_TRIANGLES, 0, 3)
        glDrawArrays(GL_TRIANGLES, 2, 3)
        glBindVertexArray(0)
        glUseProgram(0)
    
        # we completed our drawing; the draw commands will be
        # flushed at the end of the signal emission chain, and
        # the buffers will be drawn on the window
        return True
    
    win = Gtk.Window()
    area = Gtk.GLArea()
    #area.set_required_version(2, 1)
    #major, minor = area.get_required_version()
    #print("Version " + str(major) + "." + str(minor))
    area.connect('render', on_render)
    area.connect('realize', on_realize)
    win.connect("destroy", Gtk.main_quit)
    win.add(area)
    win.show_all()
    Gtk.main()