From patchwork Tue Sep 15 02:46:22 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Laurent Pinchart X-Patchwork-Id: 9609 Return-Path: X-Original-To: parsemail@patchwork.libcamera.org Delivered-To: parsemail@patchwork.libcamera.org Received: from lancelot.ideasonboard.com (lancelot.ideasonboard.com [92.243.16.209]) by patchwork.libcamera.org (Postfix) with ESMTPS id E31D2C3B5B for ; Tue, 15 Sep 2020 02:46:59 +0000 (UTC) Received: from lancelot.ideasonboard.com (localhost [IPv6:::1]) by lancelot.ideasonboard.com (Postfix) with ESMTP id AE2BC62DF5; Tue, 15 Sep 2020 04:46:59 +0200 (CEST) Authentication-Results: lancelot.ideasonboard.com; dkim=fail reason="signature verification failed" (1024-bit key; unprotected) header.d=ideasonboard.com header.i=@ideasonboard.com header.b="nUXdmaKZ"; dkim-atps=neutral Received: from perceval.ideasonboard.com (perceval.ideasonboard.com [IPv6:2001:4b98:dc2:55:216:3eff:fef7:d647]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id 4854062D5B for ; Tue, 15 Sep 2020 04:46:56 +0200 (CEST) Received: from pendragon.lan (62-78-145-57.bb.dnainternet.fi [62.78.145.57]) by perceval.ideasonboard.com (Postfix) with ESMTPSA id AA084FD8; Tue, 15 Sep 2020 04:46:55 +0200 (CEST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=ideasonboard.com; s=mail; t=1600138015; bh=x8IM0UmVSibea9vem8aAXgtvGFZk+36n7Zx/8uFvat0=; h=From:To:Subject:Date:In-Reply-To:References:From; b=nUXdmaKZ1bRkAOfmzVZC7yzfOuE+6ntf7O0336wvvxjeQNALQxBinKBngOgaq8nhh VgbTTI3ckkG89vWYwIEhV8UjPVw05L0jBC0AKlcrOmd905r8KxrvGM37zpzy4tDV9U 7JqMMc68JpHIDZMeozotqWVMWLtrfzhogN84XghU= From: Laurent Pinchart To: libcamera-devel@lists.libcamera.org, Show Liu Date: Tue, 15 Sep 2020 05:46:22 +0300 Message-Id: <20200915024623.30667-4-laurent.pinchart@ideasonboard.com> X-Mailer: git-send-email 2.27.0 In-Reply-To: <20200915024623.30667-1-laurent.pinchart@ideasonboard.com> References: <20200915024623.30667-1-laurent.pinchart@ideasonboard.com> MIME-Version: 1.0 Subject: [libcamera-devel] [PATCH v7 3/4] qcam: Add ViewFinderGL class to accelerate the format conversion X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: libcamera-devel-bounces@lists.libcamera.org Sender: "libcamera-devel" From: Show Liu The viewfinderGL accelerates the format conversion by using OpenGL ES shader. The minimum Qt version is bumped to v5.4, as QOpenGLWidget wasn't available before that. Signed-off-by: Show Liu Reviewed-by: Laurent Pinchart Signed-off-by: Laurent Pinchart --- meson.build | 1 + src/qcam/meson.build | 17 +- src/qcam/viewfinder_gl.cpp | 451 +++++++++++++++++++++++++++++++++++++ src/qcam/viewfinder_gl.h | 96 ++++++++ 4 files changed, 563 insertions(+), 2 deletions(-) create mode 100644 src/qcam/viewfinder_gl.cpp create mode 100644 src/qcam/viewfinder_gl.h diff --git a/meson.build b/meson.build index 1ea35e92ed67..c58d458ca0ff 100644 --- a/meson.build +++ b/meson.build @@ -26,6 +26,7 @@ libcamera_version = libcamera_git_version.split('+')[0] # Configure the build environment. cc = meson.get_compiler('c') +cxx = meson.get_compiler('cpp') config_h = configuration_data() if cc.has_header_symbol('execinfo.h', 'backtrace') diff --git a/src/qcam/meson.build b/src/qcam/meson.build index a4bad0a0659e..9bb48c0d06c5 100644 --- a/src/qcam/meson.build +++ b/src/qcam/meson.build @@ -16,14 +16,14 @@ qcam_moc_headers = files([ qcam_resources = files([ 'assets/feathericons/feathericons.qrc', - 'assets/shader/shaders.qrc' ]) qt5 = import('qt5') qt5_dep = dependency('qt5', method : 'pkg-config', modules : ['Core', 'Gui', 'Widgets'], - required : get_option('qcam')) + required : get_option('qcam'), + version : '>=5.4') if qt5_dep.found() qcam_deps = [ @@ -42,6 +42,19 @@ if qt5_dep.found() ]) endif + if cxx.has_header_symbol('QOpenGLWidget', 'QOpenGLWidget', + dependencies : qt5_dep, args : '-fPIC') + qcam_sources += files([ + 'viewfinder_gl.cpp', + ]) + qcam_moc_headers += files([ + 'viewfinder_gl.h', + ]) + qcam_resources += files([ + 'assets/shader/shaders.qrc' + ]) + endif + # gcc 9 introduced a deprecated-copy warning that is triggered by Qt until # Qt 5.13. clang 10 introduced the same warning, but detects more issues # that are not fixed in Qt yet. Disable the warning manually in both cases. diff --git a/src/qcam/viewfinder_gl.cpp b/src/qcam/viewfinder_gl.cpp new file mode 100644 index 000000000000..fbe21dcf1ad2 --- /dev/null +++ b/src/qcam/viewfinder_gl.cpp @@ -0,0 +1,451 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2020, Linaro + * + * viewfinderGL.cpp - OpenGL Viewfinder for rendering by OpenGL shader + */ + +#include "viewfinder_gl.h" + +#include + +#include + +static const QList supportedFormats{ + libcamera::formats::NV12, + libcamera::formats::NV21, + libcamera::formats::NV16, + libcamera::formats::NV61, + libcamera::formats::NV24, + libcamera::formats::NV42, + libcamera::formats::YUV420, + libcamera::formats::YVU420, +}; + +ViewFinderGL::ViewFinderGL(QWidget *parent) + : QOpenGLWidget(parent), buffer_(nullptr), yuvData_(nullptr), + fragmentShader_(nullptr), vertexShader_(nullptr), + vertexBuffer_(QOpenGLBuffer::VertexBuffer), + textureU_(QOpenGLTexture::Target2D), + textureV_(QOpenGLTexture::Target2D), + textureY_(QOpenGLTexture::Target2D) +{ +} + +ViewFinderGL::~ViewFinderGL() +{ + removeShader(); +} + +const QList &ViewFinderGL::nativeFormats() const +{ + return supportedFormats; +} + +int ViewFinderGL::setFormat(const libcamera::PixelFormat &format, + const QSize &size) +{ + /* If the fragment is created remove it and create a new one. */ + if (fragmentShader_) { + if (shaderProgram_.isLinked()) { + shaderProgram_.release(); + shaderProgram_.removeShader(fragmentShader_); + delete fragmentShader_; + } + } + + if (!selectFormat(format)) + return -1; + + format_ = format; + size_ = size; + + updateGeometry(); + return 0; +} + +void ViewFinderGL::stop() +{ + if (buffer_) { + renderComplete(buffer_); + buffer_ = nullptr; + } +} + +QImage ViewFinderGL::getCurrentImage() +{ + QMutexLocker locker(&mutex_); + + return grabFramebuffer(); +} + +void ViewFinderGL::render(libcamera::FrameBuffer *buffer, MappedBuffer *map) +{ + if (buffer->planes().size() != 1) { + qWarning() << "Multi-planar buffers are not supported"; + return; + } + + if (buffer_) + renderComplete(buffer_); + + yuvData_ = static_cast(map->memory); + update(); + buffer_ = buffer; +} + +bool ViewFinderGL::selectFormat(const libcamera::PixelFormat &format) +{ + bool ret = true; + switch (format) { + case libcamera::formats::NV12: + horzSubSample_ = 2; + vertSubSample_ = 2; + vertexShaderSrc_ = ":NV_vertex_shader.glsl"; + fragmentShaderSrc_ = ":NV_2_planes_UV_f.glsl"; + break; + case libcamera::formats::NV21: + horzSubSample_ = 2; + vertSubSample_ = 2; + vertexShaderSrc_ = ":NV_vertex_shader.glsl"; + fragmentShaderSrc_ = ":NV_2_planes_VU_f.glsl"; + break; + case libcamera::formats::NV16: + horzSubSample_ = 2; + vertSubSample_ = 1; + vertexShaderSrc_ = ":NV_vertex_shader.glsl"; + fragmentShaderSrc_ = ":NV_2_planes_UV_f.glsl"; + break; + case libcamera::formats::NV61: + horzSubSample_ = 2; + vertSubSample_ = 1; + vertexShaderSrc_ = ":NV_vertex_shader.glsl"; + fragmentShaderSrc_ = ":NV_2_planes_VU_f.glsl"; + break; + case libcamera::formats::NV24: + horzSubSample_ = 1; + vertSubSample_ = 1; + vertexShaderSrc_ = ":NV_vertex_shader.glsl"; + fragmentShaderSrc_ = ":NV_2_planes_UV_f.glsl"; + break; + case libcamera::formats::NV42: + horzSubSample_ = 1; + vertSubSample_ = 1; + vertexShaderSrc_ = ":NV_vertex_shader.glsl"; + fragmentShaderSrc_ = ":NV_2_planes_VU_f.glsl"; + break; + case libcamera::formats::YUV420: + horzSubSample_ = 2; + vertSubSample_ = 2; + vertexShaderSrc_ = ":NV_vertex_shader.glsl"; + fragmentShaderSrc_ = ":NV_3_planes_f.glsl"; + break; + case libcamera::formats::YVU420: + horzSubSample_ = 2; + vertSubSample_ = 2; + vertexShaderSrc_ = ":NV_vertex_shader.glsl"; + fragmentShaderSrc_ = ":NV_3_planes_f.glsl"; + break; + default: + ret = false; + qWarning() << "[ViewFinderGL]:" + << "format not supported."; + break; + }; + + return ret; +} + +bool ViewFinderGL::createVertexShader() +{ + /* Create Vertex Shader */ + vertexShader_ = new QOpenGLShader(QOpenGLShader::Vertex, this); + + /* Compile the vertex shader */ + if (!vertexShader_->compileSourceFile(vertexShaderSrc_)) { + qWarning() << "[ViewFinderGL]:" << vertexShader_->log(); + return false; + } + + shaderProgram_.addShader(vertexShader_); + return true; +} + +bool ViewFinderGL::createFragmentShader() +{ + int attributeVertex; + int attributeTexture; + + /* Create Fragment Shader */ + fragmentShader_ = new QOpenGLShader(QOpenGLShader::Fragment, this); + + /* Compile the fragment shader */ + if (!fragmentShader_->compileSourceFile(fragmentShaderSrc_)) { + qWarning() << "[ViewFinderGL]:" << fragmentShader_->log(); + return false; + } + + shaderProgram_.addShader(fragmentShader_); + + /* Link shader pipeline */ + if (!shaderProgram_.link()) { + qWarning() << "[ViewFinderGL]:" << shaderProgram_.log(); + close(); + } + + /* Bind shader pipeline for use */ + if (!shaderProgram_.bind()) { + qWarning() << "[ViewFinderGL]:" << shaderProgram_.log(); + close(); + } + + attributeVertex = shaderProgram_.attributeLocation("vertexIn"); + attributeTexture = shaderProgram_.attributeLocation("textureIn"); + + shaderProgram_.enableAttributeArray(attributeVertex); + shaderProgram_.setAttributeBuffer(attributeVertex, + GL_FLOAT, + 0, + 2, + 2 * sizeof(GLfloat)); + + shaderProgram_.enableAttributeArray(attributeTexture); + shaderProgram_.setAttributeBuffer(attributeTexture, + GL_FLOAT, + 8 * sizeof(GLfloat), + 2, + 2 * sizeof(GLfloat)); + + textureUniformY_ = shaderProgram_.uniformLocation("tex_y"); + textureUniformU_ = shaderProgram_.uniformLocation("tex_u"); + textureUniformV_ = shaderProgram_.uniformLocation("tex_v"); + + if (!textureY_.isCreated()) + textureY_.create(); + + if (!textureU_.isCreated()) + textureU_.create(); + + if (!textureV_.isCreated()) + textureV_.create(); + + id_y_ = textureY_.textureId(); + id_u_ = textureU_.textureId(); + id_v_ = textureV_.textureId(); + return true; +} + +void ViewFinderGL::configureTexture(unsigned int id) +{ + glBindTexture(GL_TEXTURE_2D, id); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); +} + +void ViewFinderGL::removeShader() +{ + if (shaderProgram_.isLinked()) { + shaderProgram_.release(); + shaderProgram_.removeAllShaders(); + } + + if (fragmentShader_) + delete fragmentShader_; + + if (vertexShader_) + delete vertexShader_; +} + +void ViewFinderGL::initializeGL() +{ + initializeOpenGLFunctions(); + glEnable(GL_TEXTURE_2D); + glDisable(GL_DEPTH_TEST); + + static const GLfloat coordinates[2][4][2]{ + { + /* Vertex coordinates */ + { -1.0f, -1.0f }, + { -1.0f, +1.0f }, + { +1.0f, +1.0f }, + { +1.0f, -1.0f }, + }, + { + /* Texture coordinates */ + { 0.0f, 1.0f }, + { 0.0f, 0.0f }, + { 1.0f, 0.0f }, + { 1.0f, 1.0f }, + }, + }; + + vertexBuffer_.create(); + vertexBuffer_.bind(); + vertexBuffer_.allocate(coordinates, sizeof(coordinates)); + + /* Create Vertex Shader */ + if (!createVertexShader()) + qWarning() << "[ViewFinderGL]: create vertex shader failed."; + + glClearColor(1.0f, 1.0f, 1.0f, 0.0f); +} + +void ViewFinderGL::doRender() +{ + switch (format_) { + case libcamera::formats::NV12: + case libcamera::formats::NV21: + case libcamera::formats::NV16: + case libcamera::formats::NV61: + case libcamera::formats::NV24: + case libcamera::formats::NV42: + /* Activate texture Y */ + glActiveTexture(GL_TEXTURE0); + configureTexture(id_y_); + glTexImage2D(GL_TEXTURE_2D, + 0, + GL_RED, + size_.width(), + size_.height(), + 0, + GL_RED, + GL_UNSIGNED_BYTE, + yuvData_); + shaderProgram_.setUniformValue(textureUniformY_, 0); + + /* Activate texture UV/VU */ + glActiveTexture(GL_TEXTURE1); + configureTexture(id_u_); + glTexImage2D(GL_TEXTURE_2D, + 0, + GL_RG, + size_.width() / horzSubSample_, + size_.height() / vertSubSample_, + 0, + GL_RG, + GL_UNSIGNED_BYTE, + (char *)yuvData_ + size_.width() * size_.height()); + shaderProgram_.setUniformValue(textureUniformU_, 1); + break; + + case libcamera::formats::YUV420: + /* Activate texture Y */ + glActiveTexture(GL_TEXTURE0); + configureTexture(id_y_); + glTexImage2D(GL_TEXTURE_2D, + 0, + GL_RED, + size_.width(), + size_.height(), + 0, + GL_RED, + GL_UNSIGNED_BYTE, + yuvData_); + shaderProgram_.setUniformValue(textureUniformY_, 0); + + /* Activate texture U */ + glActiveTexture(GL_TEXTURE1); + configureTexture(id_u_); + glTexImage2D(GL_TEXTURE_2D, + 0, + GL_RED, + size_.width() / horzSubSample_, + size_.height() / vertSubSample_, + 0, + GL_RED, + GL_UNSIGNED_BYTE, + (char *)yuvData_ + size_.width() * size_.height()); + shaderProgram_.setUniformValue(textureUniformU_, 1); + + /* Activate texture V */ + glActiveTexture(GL_TEXTURE2); + configureTexture(id_v_); + glTexImage2D(GL_TEXTURE_2D, + 0, + GL_RED, + size_.width() / horzSubSample_, + size_.height() / vertSubSample_, + 0, + GL_RED, + GL_UNSIGNED_BYTE, + (char *)yuvData_ + size_.width() * size_.height() * 5 / 4); + shaderProgram_.setUniformValue(textureUniformV_, 2); + break; + + case libcamera::formats::YVU420: + /* Activate texture Y */ + glActiveTexture(GL_TEXTURE0); + configureTexture(id_y_); + glTexImage2D(GL_TEXTURE_2D, + 0, + GL_RED, + size_.width(), + size_.height(), + 0, + GL_RED, + GL_UNSIGNED_BYTE, + yuvData_); + shaderProgram_.setUniformValue(textureUniformY_, 0); + + /* Activate texture V */ + glActiveTexture(GL_TEXTURE2); + configureTexture(id_v_); + glTexImage2D(GL_TEXTURE_2D, + 0, + GL_RED, + size_.width() / horzSubSample_, + size_.height() / vertSubSample_, + 0, + GL_RED, + GL_UNSIGNED_BYTE, + (char *)yuvData_ + size_.width() * size_.height()); + shaderProgram_.setUniformValue(textureUniformV_, 2); + + /* Activate texture U */ + glActiveTexture(GL_TEXTURE1); + configureTexture(id_u_); + glTexImage2D(GL_TEXTURE_2D, + 0, + GL_RED, + size_.width() / horzSubSample_, + size_.height() / vertSubSample_, + 0, + GL_RED, + GL_UNSIGNED_BYTE, + (char *)yuvData_ + size_.width() * size_.height() * 5 / 4); + shaderProgram_.setUniformValue(textureUniformU_, 1); + break; + + default: + break; + }; +} + +void ViewFinderGL::paintGL() +{ + if (!fragmentShader_) + if (!createFragmentShader()) { + qWarning() << "[ViewFinderGL]:" + << "create fragment shader failed."; + } + + if (yuvData_) { + glClearColor(0.0, 0.0, 0.0, 1.0); + glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT); + + doRender(); + glDrawArrays(GL_TRIANGLE_FAN, 0, 4); + } +} + +void ViewFinderGL::resizeGL(int w, int h) +{ + glViewport(0, 0, w, h); +} + +QSize ViewFinderGL::sizeHint() const +{ + return size_.isValid() ? size_ : QSize(640, 480); +} diff --git a/src/qcam/viewfinder_gl.h b/src/qcam/viewfinder_gl.h new file mode 100644 index 000000000000..69502b7a543e --- /dev/null +++ b/src/qcam/viewfinder_gl.h @@ -0,0 +1,96 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2020, Linaro + * + * viewfinder_GL.h - OpenGL Viewfinder for rendering by OpenGL shader + * + */ +#ifndef __VIEWFINDER_GL_H__ +#define __VIEWFINDER_GL_H__ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "viewfinder.h" + +class ViewFinderGL : public QOpenGLWidget, + public ViewFinder, + protected QOpenGLFunctions +{ + Q_OBJECT + +public: + ViewFinderGL(QWidget *parent = nullptr); + ~ViewFinderGL(); + + const QList &nativeFormats() const override; + + int setFormat(const libcamera::PixelFormat &format, const QSize &size) override; + void render(libcamera::FrameBuffer *buffer, MappedBuffer *map) override; + void stop() override; + + QImage getCurrentImage() override; + +Q_SIGNALS: + void renderComplete(libcamera::FrameBuffer *buffer); + +protected: + void initializeGL() override; + void paintGL() override; + void resizeGL(int w, int h) override; + QSize sizeHint() const override; + +private: + bool selectFormat(const libcamera::PixelFormat &format); + + void configureTexture(unsigned int id); + bool createFragmentShader(); + bool createVertexShader(); + void removeShader(); + void doRender(); + + /* Captured image size, format and buffer */ + libcamera::FrameBuffer *buffer_; + libcamera::PixelFormat format_; + QSize size_; + unsigned char *yuvData_; + + /* OpenGL components for rendering */ + QOpenGLShader *fragmentShader_; + QOpenGLShader *vertexShader_; + QOpenGLShaderProgram shaderProgram_; + + /* Vertex buffer */ + QOpenGLBuffer vertexBuffer_; + + /* Fragment and Vertex shader file name */ + QString fragmentShaderSrc_; + QString vertexShaderSrc_; + + /* YUV texture planars and parameters */ + GLuint id_u_; + GLuint id_v_; + GLuint id_y_; + GLuint textureUniformU_; + GLuint textureUniformV_; + GLuint textureUniformY_; + QOpenGLTexture textureU_; + QOpenGLTexture textureV_; + QOpenGLTexture textureY_; + unsigned int horzSubSample_; + unsigned int vertSubSample_; + + QMutex mutex_; /* Prevent concurrent access to image_ */ +}; + +#endif /* __VIEWFINDER_GL_H__ */