How do I render images captured by OpenCV in OpenGL 3.2 GtkGLArea in Python? Many of the examples online are 10 years old, outdated and are using OpenGL 2.1 or 1.1 examples.
The only way I can do this is through shaders. The only working example to get anything to draw in a GtkGLArea is this one:
Using Gtk GLArea in Python GTK3
I found this example in C++ and translated it into Python but I cannot get it to work or output anything.
How to Display Image using OpenGL
Here is my code, it does not display anything. What am I doing wrong?
Write Display Function called by OpenCV capture thread:
import os
import gi
gi.require_version("Gtk", "3.0")
from gi.repository import Gtk, GdkPixbuf, GLib
USE_OPENGL = True
def writeDisplay(uiBuilder, frame, imageDisplay):
# Write Frame
frame = cv.cvtColor(frame, cv.COLOR_BGR2RGB)
if USE_OPENGL:
# Render frame using OpenGL
GLib.idle_add(imageDisplay.render, frame)
OpenGL Renderer GTK Widget
import gi
gi.require_version('Gtk', '3.0')
from gi.repository import Gtk
from OpenGL.GL import *
from OpenGL.GL import shaders
import numpy as np
VERTEX_SOURCE = '''
#version 330
layout (location=0) in vec3 position;
layout (location=1) in vec3 color;
layout (location=2) in vec2 texCoord;
out vec3 ourColor;
out vec2 TexCoord;
void main()
{
gl_Position = vec4(position,1.0);
ourColor = color;
TexCoord= vec2(texCoord.x,1.0-texCoord.y);
}'''
FRAGMENT_SOURCE ='''
#version 330
in vec3 ourColor;
in vec2 TexCoord;
out vec4 color;
uniform sampler2D ourTexture;
void main(){
color = texture(ourTexture , TexCoord);
};'''
recVertices = np.array([
# Positions Colors Texture Coords
0.5, 0.5, 0.0, 1.0, 0.0, 0.0, 1.0, 1.0, # Top Right
0.5, -0.5, 0.0, 0.0, 1.0, 0.0, 1.0, 0.0, # Bottom Right
-0.5, -0.5, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, # Bottom Left
-0.5, 0.5, 0.0, 1.0, 1.0, 0.0, 0.0, 1.0 # Top Left
], dtype=np.float32)
indices = np.array([
0, 1, 3, # First Triangle
1, 2, 3 # Second Triangle
])
def checkGlError(op: str):
error = glGetError()
if error is not None and error != 0:
print("after %s() glError (0x%x)", op, error)
# Based on examples:
# https://stackoverflow.com/questions/42153819/how-to-load-and-display-an-image-in-opengl-es-3-0-using-c
# https://stackoverflow.com/questions/47565884/use-of-the-gtk-glarea-in-pygobject-gtk3
class OpenGLRenderer(Gtk.GLArea):
def __init__(self):
Gtk.GLArea.__init__(self)
self.connect("realize", self.onRealize)
self.connect("render", self.onRender)
self.ctx = None
self.frame = None
self.area = None
self.shaderProgram = None
self.positionHandle = None
self.textureId = None
self.vao = None
def onRealize(self, area):
error = area.get_error()
if error != None:
print("your graphics card is probably too old : ", error)
else:
print(area, "realize... fine so far")
self.ctx = self.get_context()
self.ctx.make_current()
print("OpenGL realized", self.ctx)
def onRender(self, area, ctx):
self.render(self.frame)
return True
def setupGraphics(self, width, height):
if self.shaderProgram is None:
# Load Shaders, Create program, Setup Graphics
vertexShader = glCreateShader(GL_VERTEX_SHADER)
glShaderSource(vertexShader, VERTEX_SOURCE)
glCompileShader(vertexShader)
pixelShader = glCreateShader(GL_FRAGMENT_SHADER)
glShaderSource(pixelShader, FRAGMENT_SOURCE)
glCompileShader(pixelShader)
self.shaderProgram = glCreateProgram()
glAttachShader(self.shaderProgram, vertexShader)
glAttachShader(self.shaderProgram, pixelShader)
glLinkProgram(self.shaderProgram)
self.positionHandle = glGetAttribLocation(self.shaderProgram, "position")
glViewport(0, 0, width, height)
def initBuffers(self):
# Initialize an buffer to store all the verticles and transfer them to the GPU
self.vao = glGenVertexArrays(1) # Generate VAO
vbos = glGenBuffers(1) # Generate VBO
ebo = glGenBuffers(1) # Generate EPBO
glBindVertexArray(self.vao) # Bind the Vertex Array
glBindBuffer(GL_ARRAY_BUFFER, vbos) # Bind verticles array for OpenGL to use
glBufferData(GL_ARRAY_BUFFER, len(recVertices), recVertices, GL_STATIC_DRAW)
glBindBuffer(GL_ELEMENT_ARRAY_BUFFER, ebo) # Bind the indices for information about drawing sequence
glBufferData(GL_ELEMENT_ARRAY_BUFFER, len(indices), indices, GL_STATIC_DRAW)
# 1. set the vertex attributes pointers
# Position Attribute
glVertexAttribPointer(0, 3, GL_FLOAT, GL_FALSE, 8 * sizeof(GLfloat), ctypes.c_void_p(0))
glEnableVertexAttribArray(0)
# Color Attribute
glVertexAttribPointer(1, 3, GL_FLOAT, GL_FALSE, 8 * sizeof(GLfloat), ctypes.c_void_p(3 * sizeof(GLfloat)))
glEnableVertexAttribArray(1)
# Texture Coordinate Attribute
glVertexAttribPointer(2, 2, GL_FLOAT, GL_FALSE, 8 * sizeof(GLfloat), ctypes.c_void_p(6 * sizeof(GLfloat)))
glEnableVertexAttribArray(2)
glBindVertexArray(0) # 3. Unbind VAO
def generateTexture(self, frame):
# Update Frame
self.frame = frame
# If we have a frame to display
if frame is not None:
# extract array from Image
h, w, d = frame.shape
# Generate Texture
self.textureId = glGenTextures(1)
glBindTexture(GL_TEXTURE_2D, self.textureId) # Bind our 2D texture so that following set up will be applied
# Set texture wrapping parameter
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_MIRRORED_REPEAT)
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_MIRRORED_REPEAT)
# Set texture Filtering parameter
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST)
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST)
glTexImage2D(GL_TEXTURE_2D, 0, GL_RGB, w, h, 0, GL_RGB, GL_UNSIGNED_BYTE, frame)
glGenerateMipmap(GL_TEXTURE_2D)
glBindTexture(GL_TEXTURE_2D, 0) # Unbind 2D textures
def render(self, frame):
# Set OpenGL Render Context
if self.ctx is not None and frame is not None:
self.ctx.make_current()
# extract array from Image
h, w, d = frame.shape
# Initialize Graphics
self.setupGraphics(w, h)
# Generate Texture
self.generateTexture(frame)
# Clear Screen
glClearColor(0, 0, 1, 1)
glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT)
# Render Frame
glUseProgram(self.shaderProgram)
# checkGlError("glUseProgram")
# glVertexAttribPointer(self.positionHandle, 2, GL_FLOAT, GL_FALSE, 0, recVertices)
# checkGlError("glVertexAttribPointer")
# glEnableVertexAttribArray(self.positionHandle)
# checkGlError("glEnableVertexAttribArray")
# glDrawArrays(GL_TRIANGLE_FAN, 0, 4)
# checkGlError("glDrawArrays")
glActiveTexture(GL_TEXTURE0)
checkGlError("glActiveTexture")
glBindTexture(GL_TEXTURE_2D, self.textureId)
checkGlError("glBindTexture")
mlocation = glGetUniformLocation(self.shaderProgram, "ourTexture")
checkGlError("glGetUniformLocation")
glUniform1i(mlocation, 0)
checkGlError("glUniform1i")
self.initBuffers()
glBindVertexArray(self.vao)
checkGlError("glBindVertexArray")
glDrawElements(GL_TRIANGLES, 6, GL_UNSIGNED_INT, 0)
# Queue Draw
glFlush()
self.queue_draw()
Here is the answer, this is tested with OpenCV frames which are a 3 dimentional array [width][height][RGB Color] which need to be flattened into a linear array. OpenGL will take the width and height of the image and traverse it properly. This guide was immensely helpful in getting this to work: https://open.gl/drawing
import gi
gi.require_version('Gtk', '3.0')
from gi.repository import Gtk
from OpenGL.GL import *
import numpy as np
VERTEX_SOURCE = '''
#version 330
layout (location=0) in vec3 position;
layout (location=1) in vec3 color;
layout (location=2) in vec2 texCoord;
out vec3 ourColor;
out vec2 TexCoord;
void main()
{
gl_Position = vec4(position,1.0);
ourColor = color;
TexCoord= vec2(texCoord.x,1.0-texCoord.y);
}'''
FRAGMENT_SOURCE ='''
#version 330
in vec3 ourColor;
in vec2 TexCoord;
out vec4 color;
uniform sampler2D ourTexture;
void main(){
color = texture(ourTexture, TexCoord);
};'''
recVertices = np.array([
# Positions Colors Texture Coords
1.0, 1.0, 0.0, 1.0, 0.0, 0.0, 1.0, 1.0, # Top Right 0
1.0, -1.0, 0.0, 0.0, 1.0, 0.0, 1.0, 0.0, # Bottom Right 1
-1.0, -1.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, # Bottom Left 2
-1.0, 1.0, 0.0, 1.0, 1.0, 0.0, 0.0, 1.0, # Top Left 3
1.0, 1.0, 0.0, 1.0, 0.0, 0.0, 1.0, 1.0, # Top Right 4
], dtype=np.float32)
def checkGlError(op: str):
error = glGetError()
if error is not None and error != 0:
print("after %s() glError (0x%x)", op, error)
# Based on examples:
# https://stackoverflow.com/questions/42153819/how-to-load-and-display-an-image-in-opengl-es-3-0-using-c
# https://stackoverflow.com/questions/47565884/use-of-the-gtk-glarea-in-pygobject-gtk3
class OpenGLRenderer(Gtk.GLArea):
def __init__(self):
Gtk.GLArea.__init__(self)
self.connect("realize", self.onRealize)
self.connect("render", self.onRender)
self.ctx = None
self.frame = None
self.area = None
self.shaderProgram = None
self.positionHandle = None
self.textureId = None
self.vao = None
self.vbos = None
def onRealize(self, area):
error = area.get_error()
if error != None:
print("your graphics card is probably too old : ", error)
else:
print(area, "realize... fine so far")
self.ctx = self.get_context()
self.ctx.make_current()
print("OpenGL realized", self.ctx)
def onRender(self, area, ctx):
self.render(self.frame)
return True
def setupGraphics(self):
if self.shaderProgram is None:
# Load Shaders, Create program, Setup Graphics
vertexShader = glCreateShader(GL_VERTEX_SHADER)
glShaderSource(vertexShader, VERTEX_SOURCE)
glCompileShader(vertexShader)
status = glGetShaderiv(vertexShader, GL_COMPILE_STATUS)
print("Compile vertexShader status: " + str(status == GL_TRUE))
pixelShader = glCreateShader(GL_FRAGMENT_SHADER)
glShaderSource(pixelShader, FRAGMENT_SOURCE)
glCompileShader(pixelShader)
status = glGetShaderiv(pixelShader, GL_COMPILE_STATUS)
print("Compile vertexShader status: " + str(status == GL_TRUE))
self.shaderProgram = glCreateProgram()
glAttachShader(self.shaderProgram, vertexShader)
glAttachShader(self.shaderProgram, pixelShader)
glLinkProgram(self.shaderProgram)
glBindFragDataLocation(self.shaderProgram, 0, "color")
self.positionHandle = glGetAttribLocation(self.shaderProgram, "position")
# Initalize Vertex Buffers
self.initBuffers()
def initBuffers(self):
# Initialize an buffer to store all the verticles and transfer them to the GPU
self.vao = glGenVertexArrays(1) # Generate VAO
self.vbos = glGenBuffers(1) # Generate VBO
glBindVertexArray(self.vao) # Bind the Vertex Array
glBindBuffer(GL_ARRAY_BUFFER, self.vbos) # Bind verticles array for OpenGL to use
glBufferData(GL_ARRAY_BUFFER, sizeof(GLfloat) * len(recVertices), recVertices, GL_STATIC_DRAW)
# 1. set the vertex attributes pointers
# Position Attribute
glVertexAttribPointer(0, 3, GL_FLOAT, GL_FALSE, 8 * sizeof(GLfloat), ctypes.c_void_p(0))
glEnableVertexAttribArray(0)
# Color Attribute
glVertexAttribPointer(1, 3, GL_FLOAT, GL_FALSE, 8 * sizeof(GLfloat), ctypes.c_void_p(3 * sizeof(GLfloat)))
glEnableVertexAttribArray(1)
# Texture Coordinate Attribute
glVertexAttribPointer(2, 2, GL_FLOAT, GL_FALSE, 8 * sizeof(GLfloat), ctypes.c_void_p(6 * sizeof(GLfloat)))
glEnableVertexAttribArray(2)
glBindVertexArray(0) # 3. Unbind VAO
def generateTexture(self, frame):
# Update Frame
self.frame = frame
# Delete previous textures to avoid memory leak
if self.textureId is not None:
glDeleteTextures(1, [self.textureId])
# If we have a frame to display
if frame is not None:
# extract array from Image
h, w, d = frame.shape
# Frame is a 3 dimentional array where shape eg. (1920, 1080, 3)
# Where it is w, h, and 3 values for color
# https://www.educba.com/numpy-flatten/
pixels = frame.flatten(order = 'C')
# Generate Texture
self.textureId = glGenTextures(1)
glBindTexture(GL_TEXTURE_2D, self.textureId) # Bind our 2D texture so that following set up will be applied
# Set texture wrapping parameter
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE)
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE)
# Set texture Filtering parameter
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR)
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR)
glTexImage2D(GL_TEXTURE_2D, 0, GL_RGB, w, h, 0, GL_RGB, GL_UNSIGNED_BYTE, pixels)
glGenerateMipmap(GL_TEXTURE_2D)
glBindTexture(GL_TEXTURE_2D, 0) # Unbind 2D textures
def render(self, frame):
# Set OpenGL Render Context
if self.ctx is not None and frame is not None:
self.ctx.make_current()
# Initialize Graphics
self.setupGraphics()
# Generate Texture
self.generateTexture(frame)
# Clear Screen
glClearColor(0, 0, 1, 1)
glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT)
# Use Shader Program, Bind Vertex Array and Texture
glUseProgram(self.shaderProgram)
checkGlError("glUseProgram")
glActiveTexture(GL_TEXTURE0)
checkGlError("glActiveTexture")
glBindTexture(GL_TEXTURE_2D, self.textureId)
checkGlError("glBindTexture")
mlocation = glGetUniformLocation(self.shaderProgram, "ourTexture")
checkGlError("glGetUniformLocation")
glUniform1i(mlocation, 0)
checkGlError("glUniform1i")
glBindVertexArray(self.vao)
checkGlError("glBindVertexArray")
# Render Frame
glDrawArrays(GL_TRIANGLES, 0, 3)
glDrawArrays(GL_TRIANGLES, 2, 3)
# Queue Draw
glFlush()
self.queue_draw()
Rendering Textures in GtkGLArea in Python Full Example
import gi
gi.require_version("Gtk", "3.0")
from gi.repository import Gtk
import numpy as np
from OpenGL.GL import *
from OpenGL.GL import shaders
FRAGMENT_SOURCE ='''
#version 330
in vec3 Color;
in vec2 Texcoord;
out vec4 outColor;
uniform sampler2D imageTexture;
void main()
{
outColor = texture(imageTexture, Texcoord);
}'''
VERTEX_SOURCE = '''
#version 330
layout (location=0) in vec3 position;
layout (location=1) in vec3 color;
layout (location=2) in vec2 texcoord;
out vec3 Color;
out vec2 Texcoord;
void main()
{
Color = color;
Texcoord = texcoord;
gl_Position = vec4(position, 1.0);
}'''
def on_realize(self, area):
# We need to make the context current if we want to
# call GL API
area.make_current()
def on_render(area, context):
print("%s\n", glGetString(GL_VERSION))
area.make_current()
############################################
# Init Shaders
############################################
glClearColor(0, 0, 0, 1)
glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT)
VERTEX_SHADER_PROG = shaders.compileShader(VERTEX_SOURCE, GL_VERTEX_SHADER)
FRAGMENT_SHADER_PROG = shaders.compileShader(FRAGMENT_SOURCE, GL_FRAGMENT_SHADER)
shaderProgram = shaders.compileProgram(VERTEX_SHADER_PROG, FRAGMENT_SHADER_PROG)
############################################
# Init Buffers
############################################
# Create a new VAO (Vertex Array Object) and bind it
vao = glGenVertexArrays(1)
glBindVertexArray(vao)
# Generate buffers to hold our vertices
vertex_buffer = glGenBuffers(1)
glBindBuffer(GL_ARRAY_BUFFER, vertex_buffer)
# Send the data over to the buffer
vertices = np.array([
# Positions Color Texchords
1.0, 1.0, 0.0, 0.0, 1.0, 0.0, 1.0, 0.0, # Top Right 0
1.0, -1.0, 0.0, 0.0, 0.0, 1.0, 1.0, 1.0, # Bottom Right 1
-1.0, -1.0, 0.0, 1.0, 1.0, 1.0, 0.0, 1.0, # Bottom Left 2
-1.0, 1.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, # Top Left 3
1.0, 1.0, 0.0, 0.0, 1.0, 0.0, 1.0, 0.0, # Top Right 4
], dtype=np.float32)
size = sizeof(GLfloat) * len(vertices)
glBufferData(GL_ARRAY_BUFFER, size, vertices, GL_STATIC_DRAW)
# Specify the layout of the vertex data
posAttrib = glGetAttribLocation(shaderProgram, "position")
glEnableVertexAttribArray(posAttrib)
glVertexAttribPointer(posAttrib, 3, GL_FLOAT, GL_FALSE, 8 * sizeof(GLfloat), ctypes.c_void_p(0))
colAttrib = glGetAttribLocation(shaderProgram, "color")
glEnableVertexAttribArray(colAttrib)
glVertexAttribPointer(colAttrib, 3, GL_FLOAT, GL_FALSE, 8 * sizeof(GLfloat), ctypes.c_void_p(3 * sizeof(GLfloat)))
texAttrib = glGetAttribLocation(shaderProgram, "texcoord")
glEnableVertexAttribArray(texAttrib)
glVertexAttribPointer(texAttrib, 2, GL_FLOAT, GL_FALSE, 8 * sizeof(GLfloat), ctypes.c_void_p(6 * sizeof(GLfloat)))
# Unbind the VAO first (Important)
glBindVertexArray(0)
############################################
# Render
############################################
glBindBuffer(GL_ARRAY_BUFFER, 0)
glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT)
glUseProgram(shaderProgram)
glBindVertexArray(vao)
# Load Textures
width = 2
height = 2
textureId = glGenTextures(1)
glActiveTexture(GL_TEXTURE0)
glBindTexture(GL_TEXTURE_2D, textureId)
# Black/white checkerboard
pixels = [
0.0, 0.0, 0.0, 1.0, 1.0, 1.0,
1.0, 1.0, 1.0, 0.0, 0.0, 0.0
]
glTexImage2D(GL_TEXTURE_2D, 0, GL_RGB, width, height, 0, GL_RGB, GL_FLOAT, pixels)
glUniform1i(glGetUniformLocation(shaderProgram, "imageTexture"), 0)
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE)
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE)
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST)
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST)
glDrawArrays(GL_TRIANGLES, 0, 3)
glDrawArrays(GL_TRIANGLES, 2, 3)
glBindVertexArray(0)
glUseProgram(0)
# we completed our drawing; the draw commands will be
# flushed at the end of the signal emission chain, and
# the buffers will be drawn on the window
return True
win = Gtk.Window()
area = Gtk.GLArea()
#area.set_required_version(2, 1)
#major, minor = area.get_required_version()
#print("Version " + str(major) + "." + str(minor))
area.connect('render', on_render)
area.connect('realize', on_realize)
win.connect("destroy", Gtk.main_quit)
win.add(area)
win.show_all()
Gtk.main()