[{"id":12330,"web_url":"https://patchwork.libcamera.org/comment/12330/","msgid":"<20200906005819.GC25630@pendragon.ideasonboard.com>","date":"2020-09-06T00:58:19","subject":"Re: [libcamera-devel] [PATCH v4 2/3] qcam: add OpenGL renderer","submitter":{"id":2,"url":"https://patchwork.libcamera.org/api/people/2/","name":"Laurent Pinchart","email":"laurent.pinchart@ideasonboard.com"},"content":"Hi Show,\n\nOn Fri, Sep 04, 2020 at 04:43:13PM +0800, Show Liu wrote:\n> qcam: add OpenGL renderer\n> \n> Signed-off-by: Show Liu \n\nDo I assume correctly that this was a stray file that shouldn't have\nbeen sent as part of this series ?\n\n> ---\n> src/qcam/meson.build | 2 +\n> src/qcam/renderer.cpp | 346 ++++++++++++++++++++++++++++++++++++++++++\n> src/qcam/renderer.h | 81 ++++++++++\n> 3 files changed, 429 insertions(+)\n> create mode 100644 src/qcam/renderer.cpp\n> create mode 100644 src/qcam/renderer.h\n> \n> diff --git a/src/qcam/meson.build b/src/qcam/meson.build\n> index e0c6f26..8c9032f 100644\n> --- a/src/qcam/meson.build\n> +++ b/src/qcam/meson.build\n> @@ -7,11 +7,13 @@ qcam_sources = files([\n> 'main.cpp',\n> 'main_window.cpp',\n> 'viewfinder.cpp',\n> + 'renderer.cpp'\n> ])\n> \n> qcam_moc_headers = files([\n> 'main_window.h',\n> 'viewfinder.h',\n> + 'renderer.h'\n> ])\n> \n> qcam_resources = files([\n> diff --git a/src/qcam/renderer.cpp b/src/qcam/renderer.cpp\n> new file mode 100644\n> index 0000000..23e8fa8\n> --- /dev/null\n> +++ b/src/qcam/renderer.cpp\n> @@ -0,0 +1,346 @@\n> +/* SPDX-License-Identifier: GPL-2.0-or-later */\n> +/*\n> + * Copyright (C) 2020, Linaro\n> + *\n> + * renderer.cpp - Render YUV format frame by OpenGL shader\n> + */\n> +\n> +#include \"renderer.h\"\n> +\n> +Renderer::Renderer()\n> +\t: fbo_(nullptr),\n> +\t vbo_(QOpenGLBuffer::VertexBuffer),\n> +\t pFShader_(nullptr),\n> +\t pVShader_(nullptr),\n> +\t textureU_(QOpenGLTexture::Target2D),\n> +\t textureV_(QOpenGLTexture::Target2D),\n> +\t textureY_(QOpenGLTexture::Target2D)\n> +{\n> +\t/* offscreen format setup */\n> +\tsetFormat(requestedFormat());\n> +\tcreate();\n> +\n> +\t/* create OpenGL context */\n> +\tif (ctx_.create()) {\n> +\t\tctx_.makeCurrent(this);\n> +\t\tinitializeOpenGLFunctions();\n> +\t} else {\n> +\t\tqWarning() << \"[Renderer]: \"\n> +\t\t\t << \"OpenGL renderer is not available.\";\n> +\t}\n> +}\n> +\n> +Renderer::~Renderer()\n> +{\n> +\tif (vbo_.isCreated()) {\n> +\t\tvbo_.release();\n> +\t\tvbo_.destroy();\n> +\t}\n> +\n> +\tif (fbo_) {\n> +\t\tfbo_->release();\n> +\t\tdelete fbo_;\n> +\t}\n> +\n> +\tremoveShader();\n> +\n> +\tif (textureY_.isCreated())\n> +\t\ttextureY_.destroy();\n> +\n> +\tif (textureU_.isCreated())\n> +\t\ttextureU_.destroy();\n> +\n> +\tif (textureV_.isCreated())\n> +\t\ttextureV_.destroy();\n> +\n> +\tctx_.doneCurrent();\n> +}\n> +\n> +void Renderer::initializeGL()\n> +{\n> +\tglEnable(GL_TEXTURE_2D);\n> +\tglDisable(GL_DEPTH_TEST);\n> +\n> +\tstatic const GLfloat vertices[]{\n> +\t\t-1.0f, -1.0f, -1.0f, +1.0f,\n> +\t\t+1.0f, +1.0f, +1.0f, -1.0f,\n> +\t\t0.0f, 1.0f, 0.0f, 0.0f,\n> +\t\t1.0f, 0.0f, 1.0f, 1.0f\n> +\t};\n> +\n> +\tvbo_.create();\n> +\tvbo_.bind();\n> +\tvbo_.allocate(vertices, sizeof(vertices));\n> +\n> +\tglClearColor(1.0f, 1.0f, 1.0f, 0.0f);\n> +}\n> +\n> +bool Renderer::selectShader(const libcamera::PixelFormat &format)\n> +{\n> +\tbool ret = true;\n> +\tswitch (format) {\n> +\tcase libcamera::formats::NV12:\n> +\t\thorzSubSample_ = 2;\n> +\t\tvertSubSample_ = 2;\n> +\t\tvsrc_ = \":NV_vertex_shader.glsl\";\n> +\t\tfsrc_ = \":NV_2_planes_UV_f.glsl\";\n> +\t\tbreak;\n> +\tcase libcamera::formats::NV21:\n> +\t\thorzSubSample_ = 2;\n> +\t\tvertSubSample_ = 2;\n> +\t\tvsrc_ = \":NV_vertex_shader.glsl\";\n> +\t\tfsrc_ = \":NV_2_planes_VU_f.glsl\";\n> +\t\tbreak;\n> +\tcase libcamera::formats::NV16:\n> +\t\thorzSubSample_ = 2;\n> +\t\tvertSubSample_ = 1;\n> +\t\tvsrc_ = \":NV_vertex_shader.glsl\";\n> +\t\tfsrc_ = \":NV_2_planes_UV_f.glsl\";\n> +\t\tbreak;\n> +\tcase libcamera::formats::NV61:\n> +\t\thorzSubSample_ = 2;\n> +\t\tvertSubSample_ = 1;\n> +\t\tvsrc_ = \":NV_vertex_shader.glsl\";\n> +\t\tfsrc_ = \":NV_2_planes_VU_f.glsl\";\n> +\t\tbreak;\n> +\tcase libcamera::formats::NV24:\n> +\t\thorzSubSample_ = 1;\n> +\t\tvertSubSample_ = 1;\n> +\t\tvsrc_ = \":NV_vertex_shader.glsl\";\n> +\t\tfsrc_ = \":NV_2_planes_UV_f.glsl\";\n> +\t\tbreak;\n> +\tcase libcamera::formats::NV42:\n> +\t\thorzSubSample_ = 1;\n> +\t\tvertSubSample_ = 1;\n> +\t\tvsrc_ = \":NV_vertex_shader.glsl\";\n> +\t\tfsrc_ = \":NV_2_planes_VU_f.glsl\";\n> +\t\tbreak;\n> +\tcase libcamera::formats::YUV420:\n> +\t\thorzSubSample_ = 2;\n> +\t\tvertSubSample_ = 2;\n> +\t\tvsrc_ = \":NV_vertex_shader.glsl\";\n> +\t\tfsrc_ = \":NV_3_planes_UV_f.glsl\";\n> +\t\tbreak;\n> +\tdefault:\n> +\t\tret = false;\n> +\t\tqWarning() << \"[Renderer]: \"\n> +\t\t\t << \"format not support yet.\";\n> +\t\tbreak;\n> +\t};\n> +\n> +\treturn ret;\n> +}\n> +\n> +void Renderer::removeShader()\n> +{\n> +\tif (shaderProgram_.isLinked()) {\n> +\t\tshaderProgram_.release();\n> +\t\tshaderProgram_.removeAllShaders();\n> +\t}\n> +\n> +\tif (pFShader_)\n> +\t\tdelete pFShader_;\n> +\n> +\tif (pVShader_)\n> +\t\tdelete pVShader_;\n> +}\n> +\n> +bool Renderer::createShader()\n> +{\n> +\tbool bCompile;\n> +\n> +\t/* Create Vertex Shader */\n> +\tpVShader_ = new QOpenGLShader(QOpenGLShader::Vertex, this);\n> +\n> +\tbCompile = pVShader_->compileSourceFile(vsrc_);\n> +\tif (!bCompile) {\n> +\t\tqWarning() << \"[Renderer]: \" << pVShader_->log();\n> +\t\treturn bCompile;\n> +\t}\n> +\n> +\tshaderProgram_.addShader(pVShader_);\n> +\n> +\t/* Create Fragment Shader */\n> +\tpFShader_ = new QOpenGLShader(QOpenGLShader::Fragment, this);\n> +\n> +\tbCompile = pFShader_->compileSourceFile(fsrc_);\n> +\tif (!bCompile) {\n> +\t\tqWarning() << \"[Renderer]: \" << pFShader_->log();\n> +\t\treturn bCompile;\n> +\t}\n> +\n> +\tshaderProgram_.addShader(pFShader_);\n> +\n> +\t// Link shader pipeline\n> +\tif (!shaderProgram_.link()) {\n> +\t\tqWarning() << \"[Renderer]: \" << shaderProgram_.log();\n> +\t\treturn false;\n> +\t}\n> +\n> +\t// Bind shader pipeline for use\n> +\tif (!shaderProgram_.bind()) {\n> +\t\tqWarning() << \"[Renderer]: \" << shaderProgram_.log();\n> +\t\treturn false;\n> +\t}\n> +\treturn true;\n> +}\n> +\n> +bool Renderer::configure(const libcamera::PixelFormat &format, const QSize &size)\n> +{\n> +\tbool ret = true;\n> +\n> +\tif (selectShader(format)) {\n> +\t\tret = createShader();\n> +\t\tif (!ret)\n> +\t\t\treturn ret;\n> +\n> +\t\tshaderProgram_.enableAttributeArray(ATTRIB_VERTEX);\n> +\t\tshaderProgram_.enableAttributeArray(ATTRIB_TEXTURE);\n> +\n> +\t\tshaderProgram_.setAttributeBuffer(ATTRIB_VERTEX,\n> +\t\t\t\t\t\t GL_FLOAT,\n> +\t\t\t\t\t\t 0,\n> +\t\t\t\t\t\t 2,\n> +\t\t\t\t\t\t 2 * sizeof(GLfloat));\n> +\t\tshaderProgram_.setAttributeBuffer(ATTRIB_TEXTURE,\n> +\t\t\t\t\t\t GL_FLOAT,\n> +\t\t\t\t\t\t 8 * sizeof(GLfloat),\n> +\t\t\t\t\t\t 2,\n> +\t\t\t\t\t\t 2 * sizeof(GLfloat));\n> +\n> +\t\ttextureUniformY_ = shaderProgram_.uniformLocation(\"tex_y\");\n> +\t\ttextureUniformU_ = shaderProgram_.uniformLocation(\"tex_u\");\n> +\t\ttextureUniformV_ = shaderProgram_.uniformLocation(\"tex_v\");\n> +\n> +\t\tif (!textureY_.isCreated())\n> +\t\t\ttextureY_.create();\n> +\n> +\t\tif (!textureU_.isCreated())\n> +\t\t\ttextureU_.create();\n> +\n> +\t\tif (!textureV_.isCreated())\n> +\t\t\ttextureV_.create();\n> +\n> +\t\tid_y_ = textureY_.textureId();\n> +\t\tid_u_ = textureU_.textureId();\n> +\t\tid_v_ = textureV_.textureId();\n> +\n> +\t\tfbo_ = new QOpenGLFramebufferObject(size.width(),\n> +\t\t\t\t\t\t size.height(),\n> +\t\t\t\t\t\t GL_TEXTURE_2D);\n> +\t\tfbo_->bind();\n> +\t\tglViewport(0, 0, size.width(), size.height());\n> +\n> +\t\tformat_ = format;\n> +\t\tsize_ = size;\n> +\t} else {\n> +\t\tret = false;\n> +\t}\n> +\treturn ret;\n> +}\n> +\n> +void Renderer::configureTexture(unsigned int id)\n> +{\n> +\tglBindTexture(GL_TEXTURE_2D, id);\n> +\tglTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);\n> +\tglTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);\n> +\tglTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);\n> +\tglTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);\n> +}\n> +\n> +void Renderer::render(unsigned char *buffer)\n> +{\n> +\tQMutexLocker locker(&mutex_);\n> +\n> +\tglClearColor(0.0, 0.0, 0.0, 1.0);\n> +\tglClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);\n> +\n> +\tswitch (format_) {\n> +\tcase libcamera::formats::NV12:\n> +\tcase libcamera::formats::NV21:\n> +\tcase libcamera::formats::NV16:\n> +\tcase libcamera::formats::NV61:\n> +\tcase libcamera::formats::NV24:\n> +\tcase libcamera::formats::NV42:\n> +\t\t/* activate texture 0 */\n> +\t\tglActiveTexture(GL_TEXTURE0);\n> +\t\tconfigureTexture(id_y_);\n> +\t\tglTexImage2D(GL_TEXTURE_2D,\n> +\t\t\t 0,\n> +\t\t\t GL_RED,\n> +\t\t\t size_.width(),\n> +\t\t\t size_.height(),\n> +\t\t\t 0,\n> +\t\t\t GL_RED,\n> +\t\t\t GL_UNSIGNED_BYTE,\n> +\t\t\t buffer);\n> +\t\tglUniform1i(textureUniformY_, 0);\n> +\n> +\t\t/* activate texture 1 */\n> +\t\tglActiveTexture(GL_TEXTURE1);\n> +\t\tconfigureTexture(id_u_);\n> +\t\tglTexImage2D(GL_TEXTURE_2D,\n> +\t\t\t 0,\n> +\t\t\t GL_RG,\n> +\t\t\t size_.width() / horzSubSample_,\n> +\t\t\t size_.height() / vertSubSample_,\n> +\t\t\t 0,\n> +\t\t\t GL_RG,\n> +\t\t\t GL_UNSIGNED_BYTE,\n> +\t\t\t buffer + size_.width() * size_.height());\n> +\t\tglUniform1i(textureUniformU_, 1);\n> +\t\tbreak;\n> +\tcase libcamera::formats::YUV420:\n> +\t\t/* activate texture 0 */\n> +\t\tglActiveTexture(GL_TEXTURE0);\n> +\t\tconfigureTexture(id_y_);\n> +\t\tglTexImage2D(GL_TEXTURE_2D,\n> +\t\t\t 0,\n> +\t\t\t GL_RED,\n> +\t\t\t size_.width(),\n> +\t\t\t size_.height(),\n> +\t\t\t 0,\n> +\t\t\t GL_RED,\n> +\t\t\t GL_UNSIGNED_BYTE,\n> +\t\t\t buffer);\n> +\t\tglUniform1i(textureUniformY_, 0);\n> +\n> +\t\t/* activate texture 1 */\n> +\t\tglActiveTexture(GL_TEXTURE1);\n> +\t\tconfigureTexture(id_u_);\n> +\t\tglTexImage2D(GL_TEXTURE_2D,\n> +\t\t\t 0,\n> +\t\t\t GL_RG,\n> +\t\t\t size_.width() / horzSubSample_,\n> +\t\t\t size_.height() / vertSubSample_,\n> +\t\t\t 0,\n> +\t\t\t GL_RG,\n> +\t\t\t GL_UNSIGNED_BYTE,\n> +\t\t\t buffer + size_.width() * size_.height());\n> +\t\tglUniform1i(textureUniformU_, 1);\n> +\n> +\t\t/* activate texture 2 */\n> +\t\tglActiveTexture(GL_TEXTURE2);\n> +\t\tconfigureTexture(id_v_);\n> +\t\tglTexImage2D(GL_TEXTURE_2D,\n> +\t\t\t 0,\n> +\t\t\t GL_RG,\n> +\t\t\t size_.width() / horzSubSample_,\n> +\t\t\t size_.height() / vertSubSample_,\n> +\t\t\t 0, GL_RG,\n> +\t\t\t GL_UNSIGNED_BYTE,\n> +\t\t\t buffer + size_.width() * size_.height() * 5 / 4);\n> +\t\tglUniform1i(textureUniformV_, 2);\n> +\t\tbreak;\n> +\tdefault:\n> +\t\tbreak;\n> +\t};\n> +\n> +\tglDrawArrays(GL_TRIANGLE_FAN, 0, 4);\n> +}\n> +\n> +QImage Renderer::toImage()\n> +{\n> +\tQMutexLocker locker(&mutex_);\n> +\treturn (fbo_->toImage(true));\n> +}\n> diff --git a/src/qcam/renderer.h b/src/qcam/renderer.h\n> new file mode 100644\n> index 0000000..1ea0c48\n> --- /dev/null\n> +++ b/src/qcam/renderer.h\n> @@ -0,0 +1,81 @@\n> +/* SPDX-License-Identifier: GPL-2.0-or-later */\n> +/*\n> + * Copyright (C) 2020, Linaro\n> + *\n> + * renderer.h - Render YUV format frame by OpenGL shader\n> + */\n> +#ifndef __QCAM_RENDERER_H__\n> +#define __QCAM_RENDERER_H__\n> +\n> +#include \n> +#include \n> +#include \n> +#include \n> +#include \n> +#include \n> +#include \n> +#include \n> +#include \n> +#include \n> +#include \n> +#include \n> +\n> +#include \n> +\n> +#define ATTRIB_VERTEX 0\n> +#define ATTRIB_TEXTURE 1\n> +\n> +class Renderer : public QOffscreenSurface, protected QOpenGLFunctions\n> +{\n> +\tQ_OBJECT\n> +\n> +public:\n> +\tRenderer();\n> +\t~Renderer();\n> +\n> +\tvoid initializeGL();\n> +\tbool configure(const libcamera::PixelFormat &format, const QSize &size);\n> +\tvoid render(unsigned char *buffer);\n> +\tQImage toImage();\n> +\n> +private:\n> +\tbool createShader();\n> +\tvoid configureTexture(unsigned int id);\n> +\tvoid removeShader();\n> +\tbool selectShader(const libcamera::PixelFormat &format);\n> +\n> +\t/* OpenGL renderer components */\n> +\tQOpenGLContext ctx_;\n> +\tQOpenGLFramebufferObject *fbo_;\n> +\tQOpenGLBuffer vbo_;\n> +\tQOpenGLShader *pFShader_;\n> +\tQOpenGLShader *pVShader_;\n> +\tQOpenGLShaderProgram shaderProgram_;\n> +\tQSurfaceFormat surfaceFormat_;\n> +\n> +\t/* Fragment and Vertex shader file */\n> +\tQString fsrc_;\n> +\tQString vsrc_;\n> +\n> +\t/* YUV frame size and format */\n> +\tlibcamera::PixelFormat format_;\n> +\tQSize size_;\n> +\n> +\t/* YUV texture planars and parameters*/\n> +\tGLuint id_u_;\n> +\tGLuint id_v_;\n> +\tGLuint id_y_;\n> +\tGLuint textureUniformU_;\n> +\tGLuint textureUniformV_;\n> +\tGLuint textureUniformY_;\n> +\tQOpenGLTexture textureU_;\n> +\tQOpenGLTexture textureV_;\n> +\tQOpenGLTexture textureY_;\n> +\tunsigned int horzSubSample_;\n> +\tunsigned int vertSubSample_;\n> +\n> +\tQImage image_;\n> +\tQMutex mutex_; /* Prevent concurrent access to image_ */\n> +};\n> +\n> +#endif /* __QCAM_RENDERER_H__ */","headers":{"Return-Path":"","X-Original-To":"parsemail@patchwork.libcamera.org","Delivered-To":"parsemail@patchwork.libcamera.org","Received":["from lancelot.ideasonboard.com (lancelot.ideasonboard.com\n\t[92.243.16.209])\n\tby patchwork.libcamera.org (Postfix) with ESMTPS id 3FD3EBDB1D\n\tfor ;\n\tSun, 6 Sep 2020 00:58:45 +0000 (UTC)","from lancelot.ideasonboard.com (localhost [IPv6:::1])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTP id 0AC6D62B5D;\n\tSun, 6 Sep 2020 02:58:45 +0200 (CEST)","from perceval.ideasonboard.com (perceval.ideasonboard.com\n\t[IPv6:2001:4b98:dc2:55:216:3eff:fef7:d647])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id DD6FF61EA0\n\tfor ;\n\tSun, 6 Sep 2020 02:58:43 +0200 (CEST)","from pendragon.ideasonboard.com (62-78-145-57.bb.dnainternet.fi\n\t[62.78.145.57])\n\tby perceval.ideasonboard.com (Postfix) with ESMTPSA id CBC17277;\n\tSun, 6 Sep 2020 02:58:42 +0200 (CEST)"],"Authentication-Results":"lancelot.ideasonboard.com;\n\tdkim=fail reason=\"signature verification failed\" (1024-bit key;\n\tunprotected) header.d=ideasonboard.com header.i=@ideasonboard.com\n\theader.b=\"gt2GYcRC\"; dkim-atps=neutral","DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/simple; d=ideasonboard.com;\n\ts=mail; t=1599353923;\n\tbh=Jn5Urc8sIz+vU7JihJUlPU4ddSNe5u0Xw5x1a2a3Lps=;\n\th=Date:From:To:Cc:Subject:References:In-Reply-To:From;\n\tb=gt2GYcRCkrXgp4MvvLYEB4GCj3B3XoH8QAdX3B9ySFVmdYCvEPqu9zANHVK07is0J\n\t6nf5XdNDyOz6tpuZcvQ5m7AxjsgiUVihJI47F/YAsJuPTUOkSOoWDxvkjHEx5ktcfS\n\tYgcI+hHc/qWrk2t0zY8ejotX8WT8rU34KfA0at64=","Date":"Sun, 6 Sep 2020 03:58:19 +0300","From":"Laurent Pinchart ","To":"Show Liu ","Message-ID":"<20200906005819.GC25630@pendragon.ideasonboard.com>","References":"<20200904084316.7319-1-show.liu@linaro.org>\n\t<20200904084316.7319-3-show.liu@linaro.org>","MIME-Version":"1.0","Content-Disposition":"inline","In-Reply-To":"<20200904084316.7319-3-show.liu@linaro.org>","Subject":"Re: [libcamera-devel] [PATCH v4 2/3] qcam: add OpenGL renderer","X-BeenThere":"libcamera-devel@lists.libcamera.org","X-Mailman-Version":"2.1.29","Precedence":"list","List-Id":"","List-Unsubscribe":",\n\t","List-Archive":"","List-Post":"","List-Help":"","List-Subscribe":",\n\t","Cc":"libcamera-devel@lists.libcamera.org","Content-Type":"text/plain; charset=\"us-ascii\"","Content-Transfer-Encoding":"7bit","Errors-To":"libcamera-devel-bounces@lists.libcamera.org","Sender":"\"libcamera-devel\" "}},{"id":12360,"web_url":"https://patchwork.libcamera.org/comment/12360/","msgid":"","date":"2020-09-08T01:52:53","subject":"Re: [libcamera-devel] [PATCH v4 2/3] qcam: add OpenGL renderer","submitter":{"id":24,"url":"https://patchwork.libcamera.org/api/people/24/","name":"Show Liu","email":"show.liu@linaro.org"},"content":"Hi Laurent,\n\nOn Sun, Sep 6, 2020 at 8:58 AM Laurent Pinchart <\nlaurent.pinchart@ideasonboard.com> wrote:\n\n> Hi Show,\n>\n> On Fri, Sep 04, 2020 at 04:43:13PM +0800, Show Liu wrote:\n> > qcam: add OpenGL renderer\n> >\n> > Signed-off-by: Show Liu \n>\n> Do I assume correctly that this was a stray file that shouldn't have\n> been sent as part of this series ?\n>\n\nYes. Please skip this.\nSorry for the confusion.\n\nBest Regards,\nShow Liu\n\n>\n> > ---\n> > src/qcam/meson.build | 2 +\n> > src/qcam/renderer.cpp | 346 ++++++++++++++++++++++++++++++++++++++++++\n> > src/qcam/renderer.h | 81 ++++++++++\n> > 3 files changed, 429 insertions(+)\n> > create mode 100644 src/qcam/renderer.cpp\n> > create mode 100644 src/qcam/renderer.h\n> >\n> > diff --git a/src/qcam/meson.build b/src/qcam/meson.build\n> > index e0c6f26..8c9032f 100644\n> > --- a/src/qcam/meson.build\n> > +++ b/src/qcam/meson.build\n> > @@ -7,11 +7,13 @@ qcam_sources = files([\n> > 'main.cpp',\n> > 'main_window.cpp',\n> > 'viewfinder.cpp',\n> > + 'renderer.cpp'\n> > ])\n> >\n> > qcam_moc_headers = files([\n> > 'main_window.h',\n> > 'viewfinder.h',\n> > + 'renderer.h'\n> > ])\n> >\n> > qcam_resources = files([\n> > diff --git a/src/qcam/renderer.cpp b/src/qcam/renderer.cpp\n> > new file mode 100644\n> > index 0000000..23e8fa8\n> > --- /dev/null\n> > +++ b/src/qcam/renderer.cpp\n> > @@ -0,0 +1,346 @@\n> > +/* SPDX-License-Identifier: GPL-2.0-or-later */\n> > +/*\n> > + * Copyright (C) 2020, Linaro\n> > + *\n> > + * renderer.cpp - Render YUV format frame by OpenGL shader\n> > + */\n> > +\n> > +#include \"renderer.h\"\n> > +\n> > +Renderer::Renderer()\n> > + : fbo_(nullptr),\n> > + vbo_(QOpenGLBuffer::VertexBuffer),\n> > + pFShader_(nullptr),\n> > + pVShader_(nullptr),\n> > + textureU_(QOpenGLTexture::Target2D),\n> > + textureV_(QOpenGLTexture::Target2D),\n> > + textureY_(QOpenGLTexture::Target2D)\n> > +{\n> > + /* offscreen format setup */\n> > + setFormat(requestedFormat());\n> > + create();\n> > +\n> > + /* create OpenGL context */\n> > + if (ctx_.create()) {\n> > + ctx_.makeCurrent(this);\n> > + initializeOpenGLFunctions();\n> > + } else {\n> > + qWarning() << \"[Renderer]: \"\n> > + << \"OpenGL renderer is not available.\";\n> > + }\n> > +}\n> > +\n> > +Renderer::~Renderer()\n> > +{\n> > + if (vbo_.isCreated()) {\n> > + vbo_.release();\n> > + vbo_.destroy();\n> > + }\n> > +\n> > + if (fbo_) {\n> > + fbo_->release();\n> > + delete fbo_;\n> > + }\n> > +\n> > + removeShader();\n> > +\n> > + if (textureY_.isCreated())\n> > + textureY_.destroy();\n> > +\n> > + if (textureU_.isCreated())\n> > + textureU_.destroy();\n> > +\n> > + if (textureV_.isCreated())\n> > + textureV_.destroy();\n> > +\n> > + ctx_.doneCurrent();\n> > +}\n> > +\n> > +void Renderer::initializeGL()\n> > +{\n> > + glEnable(GL_TEXTURE_2D);\n> > + glDisable(GL_DEPTH_TEST);\n> > +\n> > + static const GLfloat vertices[]{\n> > + -1.0f, -1.0f, -1.0f, +1.0f,\n> > + +1.0f, +1.0f, +1.0f, -1.0f,\n> > + 0.0f, 1.0f, 0.0f, 0.0f,\n> > + 1.0f, 0.0f, 1.0f, 1.0f\n> > + };\n> > +\n> > + vbo_.create();\n> > + vbo_.bind();\n> > + vbo_.allocate(vertices, sizeof(vertices));\n> > +\n> > + glClearColor(1.0f, 1.0f, 1.0f, 0.0f);\n> > +}\n> > +\n> > +bool Renderer::selectShader(const libcamera::PixelFormat &format)\n> > +{\n> > + bool ret = true;\n> > + switch (format) {\n> > + case libcamera::formats::NV12:\n> > + horzSubSample_ = 2;\n> > + vertSubSample_ = 2;\n> > + vsrc_ = \":NV_vertex_shader.glsl\";\n> > + fsrc_ = \":NV_2_planes_UV_f.glsl\";\n> > + break;\n> > + case libcamera::formats::NV21:\n> > + horzSubSample_ = 2;\n> > + vertSubSample_ = 2;\n> > + vsrc_ = \":NV_vertex_shader.glsl\";\n> > + fsrc_ = \":NV_2_planes_VU_f.glsl\";\n> > + break;\n> > + case libcamera::formats::NV16:\n> > + horzSubSample_ = 2;\n> > + vertSubSample_ = 1;\n> > + vsrc_ = \":NV_vertex_shader.glsl\";\n> > + fsrc_ = \":NV_2_planes_UV_f.glsl\";\n> > + break;\n> > + case libcamera::formats::NV61:\n> > + horzSubSample_ = 2;\n> > + vertSubSample_ = 1;\n> > + vsrc_ = \":NV_vertex_shader.glsl\";\n> > + fsrc_ = \":NV_2_planes_VU_f.glsl\";\n> > + break;\n> > + case libcamera::formats::NV24:\n> > + horzSubSample_ = 1;\n> > + vertSubSample_ = 1;\n> > + vsrc_ = \":NV_vertex_shader.glsl\";\n> > + fsrc_ = \":NV_2_planes_UV_f.glsl\";\n> > + break;\n> > + case libcamera::formats::NV42:\n> > + horzSubSample_ = 1;\n> > + vertSubSample_ = 1;\n> > + vsrc_ = \":NV_vertex_shader.glsl\";\n> > + fsrc_ = \":NV_2_planes_VU_f.glsl\";\n> > + break;\n> > + case libcamera::formats::YUV420:\n> > + horzSubSample_ = 2;\n> > + vertSubSample_ = 2;\n> > + vsrc_ = \":NV_vertex_shader.glsl\";\n> > + fsrc_ = \":NV_3_planes_UV_f.glsl\";\n> > + break;\n> > + default:\n> > + ret = false;\n> > + qWarning() << \"[Renderer]: \"\n> > + << \"format not support yet.\";\n> > + break;\n> > + };\n> > +\n> > + return ret;\n> > +}\n> > +\n> > +void Renderer::removeShader()\n> > +{\n> > + if (shaderProgram_.isLinked()) {\n> > + shaderProgram_.release();\n> > + shaderProgram_.removeAllShaders();\n> > + }\n> > +\n> > + if (pFShader_)\n> > + delete pFShader_;\n> > +\n> > + if (pVShader_)\n> > + delete pVShader_;\n> > +}\n> > +\n> > +bool Renderer::createShader()\n> > +{\n> > + bool bCompile;\n> > +\n> > + /* Create Vertex Shader */\n> > + pVShader_ = new QOpenGLShader(QOpenGLShader::Vertex, this);\n> > +\n> > + bCompile = pVShader_->compileSourceFile(vsrc_);\n> > + if (!bCompile) {\n> > + qWarning() << \"[Renderer]: \" << pVShader_->log();\n> > + return bCompile;\n> > + }\n> > +\n> > + shaderProgram_.addShader(pVShader_);\n> > +\n> > + /* Create Fragment Shader */\n> > + pFShader_ = new QOpenGLShader(QOpenGLShader::Fragment, this);\n> > +\n> > + bCompile = pFShader_->compileSourceFile(fsrc_);\n> > + if (!bCompile) {\n> > + qWarning() << \"[Renderer]: \" << pFShader_->log();\n> > + return bCompile;\n> > + }\n> > +\n> > + shaderProgram_.addShader(pFShader_);\n> > +\n> > + // Link shader pipeline\n> > + if (!shaderProgram_.link()) {\n> > + qWarning() << \"[Renderer]: \" << shaderProgram_.log();\n> > + return false;\n> > + }\n> > +\n> > + // Bind shader pipeline for use\n> > + if (!shaderProgram_.bind()) {\n> > + qWarning() << \"[Renderer]: \" << shaderProgram_.log();\n> > + return false;\n> > + }\n> > + return true;\n> > +}\n> > +\n> > +bool Renderer::configure(const libcamera::PixelFormat &format, const\n> QSize &size)\n> > +{\n> > + bool ret = true;\n> > +\n> > + if (selectShader(format)) {\n> > + ret = createShader();\n> > + if (!ret)\n> > + return ret;\n> > +\n> > + shaderProgram_.enableAttributeArray(ATTRIB_VERTEX);\n> > + shaderProgram_.enableAttributeArray(ATTRIB_TEXTURE);\n> > +\n> > + shaderProgram_.setAttributeBuffer(ATTRIB_VERTEX,\n> > + GL_FLOAT,\n> > + 0,\n> > + 2,\n> > + 2 * sizeof(GLfloat));\n> > + shaderProgram_.setAttributeBuffer(ATTRIB_TEXTURE,\n> > + GL_FLOAT,\n> > + 8 * sizeof(GLfloat),\n> > + 2,\n> > + 2 * sizeof(GLfloat));\n> > +\n> > + textureUniformY_ = shaderProgram_.uniformLocation(\"tex_y\");\n> > + textureUniformU_ = shaderProgram_.uniformLocation(\"tex_u\");\n> > + textureUniformV_ = shaderProgram_.uniformLocation(\"tex_v\");\n> > +\n> > + if (!textureY_.isCreated())\n> > + textureY_.create();\n> > +\n> > + if (!textureU_.isCreated())\n> > + textureU_.create();\n> > +\n> > + if (!textureV_.isCreated())\n> > + textureV_.create();\n> > +\n> > + id_y_ = textureY_.textureId();\n> > + id_u_ = textureU_.textureId();\n> > + id_v_ = textureV_.textureId();\n> > +\n> > + fbo_ = new QOpenGLFramebufferObject(size.width(),\n> > + size.height(),\n> > + GL_TEXTURE_2D);\n> > + fbo_->bind();\n> > + glViewport(0, 0, size.width(), size.height());\n> > +\n> > + format_ = format;\n> > + size_ = size;\n> > + } else {\n> > + ret = false;\n> > + }\n> > + return ret;\n> > +}\n> > +\n> > +void Renderer::configureTexture(unsigned int id)\n> > +{\n> > + glBindTexture(GL_TEXTURE_2D, id);\n> > + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);\n> > + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);\n> > + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S,\n> GL_CLAMP_TO_EDGE);\n> > + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T,\n> GL_CLAMP_TO_EDGE);\n> > +}\n> > +\n> > +void Renderer::render(unsigned char *buffer)\n> > +{\n> > + QMutexLocker locker(&mutex_);\n> > +\n> > + glClearColor(0.0, 0.0, 0.0, 1.0);\n> > + glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);\n> > +\n> > + switch (format_) {\n> > + case libcamera::formats::NV12:\n> > + case libcamera::formats::NV21:\n> > + case libcamera::formats::NV16:\n> > + case libcamera::formats::NV61:\n> > + case libcamera::formats::NV24:\n> > + case libcamera::formats::NV42:\n> > + /* activate texture 0 */\n> > + glActiveTexture(GL_TEXTURE0);\n> > + configureTexture(id_y_);\n> > + glTexImage2D(GL_TEXTURE_2D,\n> > + 0,\n> > + GL_RED,\n> > + size_.width(),\n> > + size_.height(),\n> > + 0,\n> > + GL_RED,\n> > + GL_UNSIGNED_BYTE,\n> > + buffer);\n> > + glUniform1i(textureUniformY_, 0);\n> > +\n> > + /* activate texture 1 */\n> > + glActiveTexture(GL_TEXTURE1);\n> > + configureTexture(id_u_);\n> > + glTexImage2D(GL_TEXTURE_2D,\n> > + 0,\n> > + GL_RG,\n> > + size_.width() / horzSubSample_,\n> > + size_.height() / vertSubSample_,\n> > + 0,\n> > + GL_RG,\n> > + GL_UNSIGNED_BYTE,\n> > + buffer + size_.width() * size_.height());\n> > + glUniform1i(textureUniformU_, 1);\n> > + break;\n> > + case libcamera::formats::YUV420:\n> > + /* activate texture 0 */\n> > + glActiveTexture(GL_TEXTURE0);\n> > + configureTexture(id_y_);\n> > + glTexImage2D(GL_TEXTURE_2D,\n> > + 0,\n> > + GL_RED,\n> > + size_.width(),\n> > + size_.height(),\n> > + 0,\n> > + GL_RED,\n> > + GL_UNSIGNED_BYTE,\n> > + buffer);\n> > + glUniform1i(textureUniformY_, 0);\n> > +\n> > + /* activate texture 1 */\n> > + glActiveTexture(GL_TEXTURE1);\n> > + configureTexture(id_u_);\n> > + glTexImage2D(GL_TEXTURE_2D,\n> > + 0,\n> > + GL_RG,\n> > + size_.width() / horzSubSample_,\n> > + size_.height() / vertSubSample_,\n> > + 0,\n> > + GL_RG,\n> > + GL_UNSIGNED_BYTE,\n> > + buffer + size_.width() * size_.height());\n> > + glUniform1i(textureUniformU_, 1);\n> > +\n> > + /* activate texture 2 */\n> > + glActiveTexture(GL_TEXTURE2);\n> > + configureTexture(id_v_);\n> > + glTexImage2D(GL_TEXTURE_2D,\n> > + 0,\n> > + GL_RG,\n> > + size_.width() / horzSubSample_,\n> > + size_.height() / vertSubSample_,\n> > + 0, GL_RG,\n> > + GL_UNSIGNED_BYTE,\n> > + buffer + size_.width() * size_.height() * 5 /\n> 4);\n> > + glUniform1i(textureUniformV_, 2);\n> > + break;\n> > + default:\n> > + break;\n> > + };\n> > +\n> > + glDrawArrays(GL_TRIANGLE_FAN, 0, 4);\n> > +}\n> > +\n> > +QImage Renderer::toImage()\n> > +{\n> > + QMutexLocker locker(&mutex_);\n> > + return (fbo_->toImage(true));\n> > +}\n> > diff --git a/src/qcam/renderer.h b/src/qcam/renderer.h\n> > new file mode 100644\n> > index 0000000..1ea0c48\n> > --- /dev/null\n> > +++ b/src/qcam/renderer.h\n> > @@ -0,0 +1,81 @@\n> > +/* SPDX-License-Identifier: GPL-2.0-or-later */\n> > +/*\n> > + * Copyright (C) 2020, Linaro\n> > + *\n> > + * renderer.h - Render YUV format frame by OpenGL shader\n> > + */\n> > +#ifndef __QCAM_RENDERER_H__\n> > +#define __QCAM_RENDERER_H__\n> > +\n> > +#include \n> > +#include \n> > +#include \n> > +#include \n> > +#include \n> > +#include \n> > +#include \n> > +#include \n> > +#include \n> > +#include \n> > +#include \n> > +#include \n> > +\n> > +#include \n> > +\n> > +#define ATTRIB_VERTEX 0\n> > +#define ATTRIB_TEXTURE 1\n> > +\n> > +class Renderer : public QOffscreenSurface, protected QOpenGLFunctions\n> > +{\n> > + Q_OBJECT\n> > +\n> > +public:\n> > + Renderer();\n> > + ~Renderer();\n> > +\n> > + void initializeGL();\n> > + bool configure(const libcamera::PixelFormat &format, const QSize\n> &size);\n> > + void render(unsigned char *buffer);\n> > + QImage toImage();\n> > +\n> > +private:\n> > + bool createShader();\n> > + void configureTexture(unsigned int id);\n> > + void removeShader();\n> > + bool selectShader(const libcamera::PixelFormat &format);\n> > +\n> > + /* OpenGL renderer components */\n> > + QOpenGLContext ctx_;\n> > + QOpenGLFramebufferObject *fbo_;\n> > + QOpenGLBuffer vbo_;\n> > + QOpenGLShader *pFShader_;\n> > + QOpenGLShader *pVShader_;\n> > + QOpenGLShaderProgram shaderProgram_;\n> > + QSurfaceFormat surfaceFormat_;\n> > +\n> > + /* Fragment and Vertex shader file */\n> > + QString fsrc_;\n> > + QString vsrc_;\n> > +\n> > + /* YUV frame size and format */\n> > + libcamera::PixelFormat format_;\n> > + QSize size_;\n> > +\n> > + /* YUV texture planars and parameters*/\n> > + GLuint id_u_;\n> > + GLuint id_v_;\n> > + GLuint id_y_;\n> > + GLuint textureUniformU_;\n> > + GLuint textureUniformV_;\n> > + GLuint textureUniformY_;\n> > + QOpenGLTexture textureU_;\n> > + QOpenGLTexture textureV_;\n> > + QOpenGLTexture textureY_;\n> > + unsigned int horzSubSample_;\n> > + unsigned int vertSubSample_;\n> > +\n> > + QImage image_;\n> > + QMutex mutex_; /* Prevent concurrent access to image_ */\n> > +};\n> > +\n> > +#endif /* __QCAM_RENDERER_H__ */\n>\n> --\n> Regards,\n>\n> Laurent Pinchart\n>","headers":{"Return-Path":"","X-Original-To":"parsemail@patchwork.libcamera.org","Delivered-To":"parsemail@patchwork.libcamera.org","Received":["from lancelot.ideasonboard.com (lancelot.ideasonboard.com\n\t[92.243.16.209])\n\tby patchwork.libcamera.org (Postfix) with ESMTPS id A07C2BDB1D\n\tfor ;\n\tTue, 8 Sep 2020 01:53:09 +0000 (UTC)","from lancelot.ideasonboard.com (localhost [IPv6:::1])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTP id E353662BCE;\n\tTue, 8 Sep 2020 03:53:08 +0200 (CEST)","from mail-pj1-x1041.google.com (mail-pj1-x1041.google.com\n\t[IPv6:2607:f8b0:4864:20::1041])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id A8F3560370\n\tfor ;\n\tTue, 8 Sep 2020 03:53:06 +0200 (CEST)","by mail-pj1-x1041.google.com with SMTP id s2so7192527pjr.4\n\tfor ;\n\tMon, 07 Sep 2020 18:53:06 -0700 (PDT)"],"Authentication-Results":"lancelot.ideasonboard.com;\n\tdkim=fail reason=\"signature verification failed\" (2048-bit key;\n\tunprotected) header.d=linaro.org header.i=@linaro.org\n\theader.b=\"ywATliol\"; dkim-atps=neutral","DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/relaxed; d=linaro.org; s=google;\n\th=mime-version:references:in-reply-to:from:date:message-id:subject:to\n\t:cc; bh=3iQ7OI5efD/V5sxSUQmS0vScOoH1xKGjOrrqojIVao4=;\n\tb=ywATliolquJWHG8DHjX/rd4FpZKcvUOgEjs9nFYI2ATBxcV9TDwhCvrFAb1AqgO+GF\n\tpylwQFBUBEOXT9CMUP1e/S3ILJMH90PPmzIU70kSsY3prqy/ohQOIses5BzWz8amRaNv\n\tzb66lfM/78tZk65sscS4gg7IbFy31V9eXKKoHwcFSCs4ZrabdRTpd8bsDFyvHFIeRTQc\n\t2WLsZkQxvcdSwfjezafQR2gvRAMsDW2tCuo3ScIuNZiPArmBZbY1y0ZNfLCw2XsWFXVz\n\texIRssa4+pJvWI4hS6J/hUIUSAH5DpJZ4WWsalmLG2mMf1k8K2uuUW0fdMYPdEkuzZdw\n\t98qQ==","X-Google-DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/relaxed;\n\td=1e100.net; s=20161025;\n\th=x-gm-message-state:mime-version:references:in-reply-to:from:date\n\t:message-id:subject:to:cc;\n\tbh=3iQ7OI5efD/V5sxSUQmS0vScOoH1xKGjOrrqojIVao4=;\n\tb=tb1y08zzs6sIO1tTl1ITmc1n2XcsRdu8EDTDig0ZYunzf+9UGx+AIX28gSEofZi7zE\n\thNgYmxgFUBu6rwdvKKe73z8B3UXr6ld06y3Yxzw96ZiLkenoB0jxPh3lDM+54tqoh5bv\n\txWb4623PK0l0EnELPCbrNxGSz3iQ6s1YcZywiLkGUjG9wnfT/lfLyXGujP3kgPJiWbEN\n\t5PpMdwJIHno4cFHICHuFUmUg2VPq1/6q88rcMj5lx+RxZvNmr12JVdTbA4umzTt/ISAr\n\tO7dCbpqmB7rtYHaxG5rFtu7D1nklU9j/0jSAtyCQqZFAfDKh+g7oL8CJFkdit/jrdfIZ\n\tC15g==","X-Gm-Message-State":"AOAM530YBm7/rYTn/N7cpB2xHdH0tmuF4td8zLIQDZrMFi11ApEGHcG9\n\tXh4tpxZRIY5wFWDqUQZ4koTfnT3ltQAByjs1zX9sK/fwR9HdHw==","X-Google-Smtp-Source":"ABdhPJy5KQmKcmvf6s/lYpwEODQfRtP5z5md1C/8bkALEKVzLonCL8Dcq8E4SmlfsCBDDfu+0K2haC5LFH/vroqfCFU=","X-Received":"by 2002:a17:90a:a40d:: with SMTP id\n\ty13mr1701811pjp.183.1599529984713; \n\tMon, 07 Sep 2020 18:53:04 -0700 (PDT)","MIME-Version":"1.0","References":"<20200904084316.7319-1-show.liu@linaro.org>\n\t<20200904084316.7319-3-show.liu@linaro.org>\n\t<20200906005819.GC25630@pendragon.ideasonboard.com>","In-Reply-To":"<20200906005819.GC25630@pendragon.ideasonboard.com>","From":"Show Liu ","Date":"Tue, 8 Sep 2020 09:52:53 +0800","Message-ID":"","To":"Laurent Pinchart ","Subject":"Re: [libcamera-devel] [PATCH v4 2/3] qcam: add OpenGL renderer","X-BeenThere":"libcamera-devel@lists.libcamera.org","X-Mailman-Version":"2.1.29","Precedence":"list","List-Id":"","List-Unsubscribe":",\n\t","List-Archive":"","List-Post":"","List-Help":"","List-Subscribe":",\n\t","Cc":"libcamera-devel@lists.libcamera.org","Content-Type":"multipart/mixed;\n\tboundary=\"===============6996628176684848553==\"","Errors-To":"libcamera-devel-bounces@lists.libcamera.org","Sender":"\"libcamera-devel\" "}}]