如何在OpenGL 3.2 GtkGLArea中渲染OpenCV帧？

如何在OpenGL 3.2 GtkGLArea中渲染OpenCV捕获的图像？网上的许多示例都是10年前的，过时的，使用的是OpenGL 2.1或1.1示例。
我能做到这一点的唯一方法是通过着色器。在GtkGLArea中绘制任何内容的唯一工作示例是：
Using Gtk GLArea in Python GTK3
我在C++中找到了这个例子，并将其翻译成Python，但我无法让它工作或输出任何东西。
How to Display Image using OpenGL
这是我的代码，它没有显示任何东西。我做错了什么？

写入OpenCV捕获线程调用的显示函数：

import os
import gi
gi.require_version("Gtk", "3.0")
from gi.repository import Gtk, GdkPixbuf, GLib

USE_OPENGL = True

def writeDisplay(uiBuilder, frame, imageDisplay):
    # Write Frame
    frame = cv.cvtColor(frame, cv.COLOR_BGR2RGB)

    if USE_OPENGL:
        # Render frame using OpenGL
        GLib.idle_add(imageDisplay.render, frame)

OpenGL渲染器GTK小工具

import gi
gi.require_version('Gtk', '3.0')
from gi.repository import Gtk
from OpenGL.GL import *
from OpenGL.GL import shaders
import numpy as np

VERTEX_SOURCE = '''
#version 330
layout (location=0) in vec3 position;
layout (location=1) in vec3 color;
layout (location=2) in vec2 texCoord;
out vec3 ourColor;
out vec2 TexCoord;
void main()
{
gl_Position = vec4(position,1.0);
ourColor = color;
TexCoord= vec2(texCoord.x,1.0-texCoord.y);
}'''

FRAGMENT_SOURCE ='''
#version 330
in vec3 ourColor;
in vec2 TexCoord;
out vec4 color;
uniform sampler2D ourTexture;
void main(){
color = texture(ourTexture , TexCoord);
};'''

recVertices = np.array([
    # Positions           Colors           Texture Coords
    0.5,  0.5, 0.0,   1.0, 0.0, 0.0,    1.0, 1.0,   # Top Right
    0.5, -0.5, 0.0,   0.0, 1.0, 0.0,    1.0, 0.0,   # Bottom Right
    -0.5, -0.5, 0.0,   0.0, 0.0, 1.0,   0.0, 0.0,   # Bottom Left
    -0.5,  0.5, 0.0,   1.0, 1.0, 0.0,   0.0, 1.0    # Top Left
], dtype=np.float32)

indices = np.array([
    0, 1, 3, # First Triangle
    1, 2, 3  # Second Triangle
])

def checkGlError(op: str):
    error = glGetError()
    if error is not None and error != 0:
        print("after %s() glError (0x%x)", op, error)

# Based on examples:
# https://stackoverflow.com/questions/42153819/how-to-load-and-display-an-image-in-opengl-es-3-0-using-c
# https://stackoverflow.com/questions/47565884/use-of-the-gtk-glarea-in-pygobject-gtk3
class OpenGLRenderer(Gtk.GLArea):
    def __init__(self):
        Gtk.GLArea.__init__(self)
        self.connect("realize", self.onRealize)
        self.connect("render", self.onRender)
        self.ctx = None
        self.frame = None
        self.area = None
        self.shaderProgram = None
        self.positionHandle = None
        self.textureId = None
        self.vao = None

    def onRealize(self, area):

        error = area.get_error()
        if error != None:
            print("your graphics card is probably too old : ", error)
        else:
            print(area, "realize... fine so far")

        self.ctx = self.get_context()
        self.ctx.make_current()

        print("OpenGL realized", self.ctx)

    def onRender(self, area, ctx):
        
        self.render(self.frame)
        return True

    def setupGraphics(self, width, height):

        if self.shaderProgram is None:
            # Load Shaders, Create program, Setup Graphics
            vertexShader = glCreateShader(GL_VERTEX_SHADER)
            glShaderSource(vertexShader, VERTEX_SOURCE)
            glCompileShader(vertexShader)

            pixelShader = glCreateShader(GL_FRAGMENT_SHADER)
            glShaderSource(pixelShader, FRAGMENT_SOURCE)
            glCompileShader(pixelShader)

            self.shaderProgram = glCreateProgram()
            glAttachShader(self.shaderProgram, vertexShader)
            glAttachShader(self.shaderProgram, pixelShader)
            glLinkProgram(self.shaderProgram)
            self.positionHandle = glGetAttribLocation(self.shaderProgram, "position")

        glViewport(0, 0, width, height)
    
    def initBuffers(self):
        # Initialize an buffer to store all the verticles and transfer them to the GPU
        self.vao = glGenVertexArrays(1) # Generate VAO
        vbos = glGenBuffers(1) # Generate VBO
        ebo = glGenBuffers(1) # Generate EPBO
        glBindVertexArray(self.vao) # Bind the Vertex Array

        glBindBuffer(GL_ARRAY_BUFFER, vbos) # Bind verticles array for OpenGL to use
        glBufferData(GL_ARRAY_BUFFER, len(recVertices), recVertices, GL_STATIC_DRAW)

        glBindBuffer(GL_ELEMENT_ARRAY_BUFFER, ebo) # Bind the indices for information about drawing sequence
        glBufferData(GL_ELEMENT_ARRAY_BUFFER, len(indices), indices, GL_STATIC_DRAW)
        
        # 1. set the vertex attributes pointers
        # Position Attribute
        glVertexAttribPointer(0, 3, GL_FLOAT, GL_FALSE, 8 * sizeof(GLfloat), ctypes.c_void_p(0))
        glEnableVertexAttribArray(0)
        # Color Attribute
        glVertexAttribPointer(1, 3, GL_FLOAT, GL_FALSE, 8 * sizeof(GLfloat), ctypes.c_void_p(3 * sizeof(GLfloat)))
        glEnableVertexAttribArray(1)
        # Texture Coordinate Attribute
        glVertexAttribPointer(2, 2, GL_FLOAT, GL_FALSE, 8 * sizeof(GLfloat), ctypes.c_void_p(6 * sizeof(GLfloat)))
        glEnableVertexAttribArray(2)

        glBindVertexArray(0) # 3. Unbind VAO
    
    def generateTexture(self, frame):
        # Update Frame
        self.frame = frame

        # If we have a frame to display
        if frame is not None:
            # extract array from Image
            h, w, d = frame.shape

            # Generate Texture
            self.textureId = glGenTextures(1)
            glBindTexture(GL_TEXTURE_2D, self.textureId) # Bind our 2D texture so that following set up will be applied

            # Set texture wrapping parameter
            glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_MIRRORED_REPEAT)
            glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_MIRRORED_REPEAT)

            # Set texture Filtering parameter
            glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST)
            glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST)

            glTexImage2D(GL_TEXTURE_2D, 0, GL_RGB, w, h, 0, GL_RGB, GL_UNSIGNED_BYTE, frame)
            glGenerateMipmap(GL_TEXTURE_2D)
            glBindTexture(GL_TEXTURE_2D, 0) # Unbind 2D textures

    def render(self, frame):

        # Set OpenGL Render Context
        if self.ctx is not None and frame is not None:
            self.ctx.make_current()

            # extract array from Image
            h, w, d = frame.shape

            # Initialize Graphics
            self.setupGraphics(w, h)

            # Generate Texture
            self.generateTexture(frame)

            # Clear Screen
            glClearColor(0, 0, 1, 1)
            glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT)

            # Render Frame
            glUseProgram(self.shaderProgram)
            # checkGlError("glUseProgram")

            # glVertexAttribPointer(self.positionHandle, 2, GL_FLOAT, GL_FALSE, 0, recVertices)
            # checkGlError("glVertexAttribPointer")
            # glEnableVertexAttribArray(self.positionHandle)
            # checkGlError("glEnableVertexAttribArray")
            # glDrawArrays(GL_TRIANGLE_FAN, 0, 4)
            # checkGlError("glDrawArrays")
            glActiveTexture(GL_TEXTURE0)
            checkGlError("glActiveTexture")
            glBindTexture(GL_TEXTURE_2D, self.textureId)
            checkGlError("glBindTexture")
            mlocation = glGetUniformLocation(self.shaderProgram, "ourTexture")
            checkGlError("glGetUniformLocation")
            glUniform1i(mlocation, 0)
            checkGlError("glUniform1i")
            self.initBuffers()
            glBindVertexArray(self.vao)
            checkGlError("glBindVertexArray")
            glDrawElements(GL_TRIANGLES, 6, GL_UNSIGNED_INT, 0)
            
            # Queue Draw
            glFlush()
            self.queue_draw()

答案是这样的，我们用OpenCV帧进行了测试，OpenCV帧是一个三维数组[宽度][高度][RGB颜色]，需要将其扁平化为线性数组。OpenGL将获取图像的宽度和高度，并正确地遍历它。本指南对实现这一点非常有帮助：https://open.gl/drawing

import gi
gi.require_version('Gtk', '3.0')
from gi.repository import Gtk
from OpenGL.GL import *
import numpy as np

VERTEX_SOURCE = '''
#version 330
layout (location=0) in vec3 position;
layout (location=1) in vec3 color;
layout (location=2) in vec2 texCoord;
out vec3 ourColor;
out vec2 TexCoord;
void main()
{
gl_Position = vec4(position,1.0);
ourColor = color;
TexCoord= vec2(texCoord.x,1.0-texCoord.y);
}'''

FRAGMENT_SOURCE ='''
#version 330
in vec3 ourColor;
in vec2 TexCoord;
out vec4 color;
uniform sampler2D ourTexture;
void main(){
color = texture(ourTexture, TexCoord);
};'''

recVertices = np.array([
    # Positions           Colors           Texture Coords
    1.0,  1.0, 0.0,   1.0, 0.0, 0.0,    1.0, 1.0,   # Top Right    0
    1.0, -1.0, 0.0,   0.0, 1.0, 0.0,    1.0, 0.0,   # Bottom Right 1
    -1.0, -1.0, 0.0,   0.0, 0.0, 1.0,   0.0, 0.0,   # Bottom Left  2
    -1.0,  1.0, 0.0,   1.0, 1.0, 0.0,   0.0, 1.0,   # Top Left     3
    1.0,  1.0, 0.0,   1.0, 0.0, 0.0,    1.0, 1.0,   # Top Right    4
], dtype=np.float32)

def checkGlError(op: str):
    error = glGetError()
    if error is not None and error != 0:
        print("after %s() glError (0x%x)", op, error)

# Based on examples:
# https://stackoverflow.com/questions/42153819/how-to-load-and-display-an-image-in-opengl-es-3-0-using-c
# https://stackoverflow.com/questions/47565884/use-of-the-gtk-glarea-in-pygobject-gtk3
class OpenGLRenderer(Gtk.GLArea):
    def __init__(self):
        Gtk.GLArea.__init__(self)
        self.connect("realize", self.onRealize)
        self.connect("render", self.onRender)
        self.ctx = None
        self.frame = None
        self.area = None
        self.shaderProgram = None
        self.positionHandle = None
        self.textureId = None
        self.vao = None
        self.vbos = None

    def onRealize(self, area):

        error = area.get_error()
        if error != None:
            print("your graphics card is probably too old : ", error)
        else:
            print(area, "realize... fine so far")

        self.ctx = self.get_context()
        self.ctx.make_current()

        print("OpenGL realized", self.ctx)

    def onRender(self, area, ctx):
        
        self.render(self.frame)
        return True

    def setupGraphics(self):

        if self.shaderProgram is None:
            # Load Shaders, Create program, Setup Graphics
            vertexShader = glCreateShader(GL_VERTEX_SHADER)
            glShaderSource(vertexShader, VERTEX_SOURCE)
            glCompileShader(vertexShader)
            status = glGetShaderiv(vertexShader, GL_COMPILE_STATUS)
            print("Compile vertexShader status: " + str(status == GL_TRUE))

            pixelShader = glCreateShader(GL_FRAGMENT_SHADER)
            glShaderSource(pixelShader, FRAGMENT_SOURCE)
            glCompileShader(pixelShader)
            status = glGetShaderiv(pixelShader, GL_COMPILE_STATUS)
            print("Compile vertexShader status: " + str(status == GL_TRUE))

            self.shaderProgram = glCreateProgram()
            glAttachShader(self.shaderProgram, vertexShader)
            glAttachShader(self.shaderProgram, pixelShader)
            glLinkProgram(self.shaderProgram)
            glBindFragDataLocation(self.shaderProgram, 0, "color")
            self.positionHandle = glGetAttribLocation(self.shaderProgram, "position")

            # Initalize Vertex Buffers
            self.initBuffers()
    
    def initBuffers(self):
        # Initialize an buffer to store all the verticles and transfer them to the GPU
        self.vao = glGenVertexArrays(1) # Generate VAO
        self.vbos = glGenBuffers(1) # Generate VBO
        glBindVertexArray(self.vao) # Bind the Vertex Array

        glBindBuffer(GL_ARRAY_BUFFER, self.vbos) # Bind verticles array for OpenGL to use
        glBufferData(GL_ARRAY_BUFFER, sizeof(GLfloat) * len(recVertices), recVertices, GL_STATIC_DRAW)
        
        # 1. set the vertex attributes pointers
        # Position Attribute
        glVertexAttribPointer(0, 3, GL_FLOAT, GL_FALSE, 8 * sizeof(GLfloat), ctypes.c_void_p(0))
        glEnableVertexAttribArray(0)
        # Color Attribute
        glVertexAttribPointer(1, 3, GL_FLOAT, GL_FALSE, 8 * sizeof(GLfloat), ctypes.c_void_p(3 * sizeof(GLfloat)))
        glEnableVertexAttribArray(1)
        # Texture Coordinate Attribute
        glVertexAttribPointer(2, 2, GL_FLOAT, GL_FALSE, 8 * sizeof(GLfloat), ctypes.c_void_p(6 * sizeof(GLfloat)))
        glEnableVertexAttribArray(2)

        glBindVertexArray(0) # 3. Unbind VAO
    
    def generateTexture(self, frame):
        # Update Frame
        self.frame = frame

        # Delete previous textures to avoid memory leak
        if self.textureId is not None:
            glDeleteTextures(1, [self.textureId])

        # If we have a frame to display
        if frame is not None:
            # extract array from Image
            h, w, d = frame.shape

            # Frame is a 3 dimentional array where shape eg. (1920, 1080, 3)
            # Where it is w, h, and 3 values for color
            # https://www.educba.com/numpy-flatten/
            pixels = frame.flatten(order = 'C')

            # Generate Texture
            self.textureId = glGenTextures(1)
            glBindTexture(GL_TEXTURE_2D, self.textureId) # Bind our 2D texture so that following set up will be applied

            # Set texture wrapping parameter
            glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE)
            glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE)

            # Set texture Filtering parameter
            glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR)
            glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR)

            glTexImage2D(GL_TEXTURE_2D, 0, GL_RGB, w, h, 0, GL_RGB, GL_UNSIGNED_BYTE, pixels)
            glGenerateMipmap(GL_TEXTURE_2D)
            glBindTexture(GL_TEXTURE_2D, 0) # Unbind 2D textures

    def render(self, frame):

        # Set OpenGL Render Context
        if self.ctx is not None and frame is not None:
            self.ctx.make_current()

            # Initialize Graphics
            self.setupGraphics()

            # Generate Texture
            self.generateTexture(frame)

            # Clear Screen
            glClearColor(0, 0, 1, 1)
            glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT)

            # Use Shader Program, Bind Vertex Array and Texture
            glUseProgram(self.shaderProgram)
            checkGlError("glUseProgram")
            glActiveTexture(GL_TEXTURE0)
            checkGlError("glActiveTexture")
            glBindTexture(GL_TEXTURE_2D, self.textureId)
            checkGlError("glBindTexture")
            mlocation = glGetUniformLocation(self.shaderProgram, "ourTexture")
            checkGlError("glGetUniformLocation")
            glUniform1i(mlocation, 0)
            checkGlError("glUniform1i")
            glBindVertexArray(self.vao)
            checkGlError("glBindVertexArray")

            # Render Frame
            glDrawArrays(GL_TRIANGLES, 0, 3)
            glDrawArrays(GL_TRIANGLES, 2, 3)
            
            # Queue Draw
            glFlush()
            self.queue_draw()

在Python中渲染GtkGLArea中的纹理完整示例

import gi

gi.require_version("Gtk", "3.0")
from gi.repository import Gtk
import numpy as np
from OpenGL.GL import *
from OpenGL.GL import shaders

FRAGMENT_SOURCE ='''
#version 330
in vec3 Color;
in vec2 Texcoord;
out vec4 outColor;
uniform sampler2D imageTexture;
void main()
{
    outColor = texture(imageTexture, Texcoord);
}'''

VERTEX_SOURCE = '''
#version 330
layout (location=0) in vec3 position;
layout (location=1) in vec3 color;
layout (location=2) in vec2 texcoord;
out vec3 Color;
out vec2 Texcoord;
void main()
{
    Color = color;
    Texcoord = texcoord;
    gl_Position = vec4(position, 1.0);
}'''

def on_realize(self, area):        
    # We need to make the context current if we want to
    # call GL API
    area.make_current()

def on_render(area, context):
    print("%s\n", glGetString(GL_VERSION))
    area.make_current()

    ############################################
    # Init Shaders
    ############################################
    glClearColor(0, 0, 0, 1)
    glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT)
    VERTEX_SHADER_PROG = shaders.compileShader(VERTEX_SOURCE, GL_VERTEX_SHADER)
    FRAGMENT_SHADER_PROG = shaders.compileShader(FRAGMENT_SOURCE, GL_FRAGMENT_SHADER)
    shaderProgram = shaders.compileProgram(VERTEX_SHADER_PROG, FRAGMENT_SHADER_PROG)
    
    ############################################
    # Init Buffers
    ############################################
    # Create a new VAO (Vertex Array Object) and bind it
    vao = glGenVertexArrays(1)
    glBindVertexArray(vao)
    # Generate buffers to hold our vertices
    vertex_buffer = glGenBuffers(1)
    glBindBuffer(GL_ARRAY_BUFFER, vertex_buffer)
    # Send the data over to the buffer
    vertices = np.array([
        # Positions      Color           Texchords 
        1.0,  1.0, 0.0,  0.0, 1.0, 0.0,  1.0, 0.0, # Top Right       0
        1.0, -1.0, 0.0,  0.0, 0.0, 1.0,  1.0, 1.0, # Bottom Right    1
        -1.0, -1.0, 0.0, 1.0, 1.0, 1.0,  0.0, 1.0, # Bottom Left     2
        -1.0,  1.0, 0.0, 1.0, 0.0, 0.0,  0.0, 0.0, # Top Left        3
        1.0,  1.0, 0.0,  0.0, 1.0, 0.0,  1.0, 0.0, # Top Right       4
    ], dtype=np.float32)
    
    size = sizeof(GLfloat) * len(vertices)
    glBufferData(GL_ARRAY_BUFFER, size, vertices, GL_STATIC_DRAW)

    # Specify the layout of the vertex data
    posAttrib = glGetAttribLocation(shaderProgram, "position")
    glEnableVertexAttribArray(posAttrib)
    glVertexAttribPointer(posAttrib, 3, GL_FLOAT, GL_FALSE, 8 * sizeof(GLfloat), ctypes.c_void_p(0))

    colAttrib = glGetAttribLocation(shaderProgram, "color")
    glEnableVertexAttribArray(colAttrib)
    glVertexAttribPointer(colAttrib, 3, GL_FLOAT, GL_FALSE, 8 * sizeof(GLfloat), ctypes.c_void_p(3 * sizeof(GLfloat)))

    texAttrib = glGetAttribLocation(shaderProgram, "texcoord")
    glEnableVertexAttribArray(texAttrib)
    glVertexAttribPointer(texAttrib, 2, GL_FLOAT, GL_FALSE, 8 * sizeof(GLfloat), ctypes.c_void_p(6 * sizeof(GLfloat)))

    # Unbind the VAO first (Important)
    glBindVertexArray(0)

    ############################################
    # Render
    ############################################
    glBindBuffer(GL_ARRAY_BUFFER, 0)
    glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT)
    glUseProgram(shaderProgram)
    glBindVertexArray(vao)

    # Load Textures
    width = 2
    height = 2
    textureId = glGenTextures(1)
    glActiveTexture(GL_TEXTURE0)
    glBindTexture(GL_TEXTURE_2D, textureId)

    # Black/white checkerboard
    pixels = [
        0.0, 0.0, 0.0,   1.0, 1.0, 1.0,
        1.0, 1.0, 1.0,   0.0, 0.0, 0.0
    ]

    glTexImage2D(GL_TEXTURE_2D, 0, GL_RGB, width, height, 0, GL_RGB, GL_FLOAT, pixels)
    glUniform1i(glGetUniformLocation(shaderProgram, "imageTexture"), 0)
    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE)
    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE)
    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST)
    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST)

    glDrawArrays(GL_TRIANGLES, 0, 3)
    glDrawArrays(GL_TRIANGLES, 2, 3)
    glBindVertexArray(0)
    glUseProgram(0)

    # we completed our drawing; the draw commands will be
    # flushed at the end of the signal emission chain, and
    # the buffers will be drawn on the window
    return True

win = Gtk.Window()
area = Gtk.GLArea()
#area.set_required_version(2, 1)
#major, minor = area.get_required_version()
#print("Version " + str(major) + "." + str(minor))
area.connect('render', on_render)
area.connect('realize', on_realize)
win.connect("destroy", Gtk.main_quit)
win.add(area)
win.show_all()
Gtk.main()

如何在OpenGL 3.2 GtkGLArea中渲染OpenCV帧？

1条答案

相关问题

热门标签

最新问答