From patchwork Fri Aug 21 16:16:00 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Show Liu X-Patchwork-Id: 9355 Return-Path: X-Original-To: parsemail@patchwork.libcamera.org Delivered-To: parsemail@patchwork.libcamera.org Received: from lancelot.ideasonboard.com (lancelot.ideasonboard.com [92.243.16.209]) by patchwork.libcamera.org (Postfix) with ESMTPS id 753ADBD87C for ; Fri, 21 Aug 2020 16:16:28 +0000 (UTC) Received: from lancelot.ideasonboard.com (localhost [IPv6:::1]) by lancelot.ideasonboard.com (Postfix) with ESMTP id 3D71C620D6; Fri, 21 Aug 2020 18:16:28 +0200 (CEST) Authentication-Results: lancelot.ideasonboard.com; dkim=fail reason="signature verification failed" (2048-bit key; unprotected) header.d=linaro.org header.i=@linaro.org header.b="aI57YFWP"; dkim-atps=neutral Received: from mail-pf1-x444.google.com (mail-pf1-x444.google.com [IPv6:2607:f8b0:4864:20::444]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id 460C260383 for ; Fri, 21 Aug 2020 18:16:27 +0200 (CEST) Received: by mail-pf1-x444.google.com with SMTP id k18so1294674pfp.7 for ; Fri, 21 Aug 2020 09:16:27 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=linaro.org; s=google; h=from:to:cc:subject:date:message-id:in-reply-to:references :mime-version:content-transfer-encoding; bh=f1tzrQ5pmtkLWnf1GsbIuWMKx2XqQVu5ilMV9XPrYRM=; b=aI57YFWPV54sghwCBjC3cf1S/tR6QXeFF68O84ik+jGWquSPOSO1tDnsO9acmUmcGb QzhwS0v19SPkKR29HaSdGgizBENIitW/gMbP/yazFLPG8LNIY2Ku8GSpLGn9dCfbL164 1ups/hc2t5HUP/iOmojvJQVU11l7Okr+tCrTFkCSqso+ndXUIQX9Cr50cljiywGvdulK q6ZVkQB8lIxMdyPzfvmwTgQUSKVjbMRRmm/wNeQHl2E0Vb/wAHWXoacaxDItLuDODj6I M0e7kSD6LPMX93gpdxPBf6YcctiNLgx9k2mg7BdnVlmEBFCaEg8ntHIbymwGaEFR6y0D 5BHg== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20161025; h=x-gm-message-state:from:to:cc:subject:date:message-id:in-reply-to :references:mime-version:content-transfer-encoding; bh=f1tzrQ5pmtkLWnf1GsbIuWMKx2XqQVu5ilMV9XPrYRM=; b=XCyu4qXtNurHtFwG43IZ8oLliMEJj+at6n11uh2Nh22Gh//UkXh7UNSbBZJ6cHBrOc mB698jl64RPG1jzX7UsNztwYdZSmkk2d0rYotOfD44gjj+nvc5bbeyW1zOw6kKfmLwSJ +tNYrYQd66bi7QkTYuK0ZeIf9lThb7BG6ahQ1KIvwLa1jhIk6bjiximNaSHKhavYJbh/ lKmBtbt+hMugQRWVa2HhfqOdIZ4XNW3TNHhtKjPREcfXx7E7ZH931gK/ht/S0AARZ275 jn6dgoZvAodRfcDQTUzYJfRqR0y9V56FKH8Ca4QYEuNM56jX31ifkaTpMnmaQTLRB/AX Aa2g== X-Gm-Message-State: AOAM533onw4Pvvt1EaJPGXvHGIQ4/055ryccpOjUBfjWVrOXr0WWrJXc 5tPtrm5gFEbSNI+P0dnICccwdEcf4jYELA== X-Google-Smtp-Source: ABdhPJzrgnNhuk3VMnHpgdM/docmo1MgDaD3ehGJszVk9PNfCwTuIjRklm3mKNBKlF+3UREJxG0j9w== X-Received: by 2002:a63:f30b:: with SMTP id l11mr2828202pgh.445.1598026585543; Fri, 21 Aug 2020 09:16:25 -0700 (PDT) Received: from localhost.localdomain (2001-b011-200c-3405-5522-d8a8-1333-f09d.dynamic-ip6.hinet.net. [2001:b011:200c:3405:5522:d8a8:1333:f09d]) by smtp.gmail.com with ESMTPSA id h15sm2524184pjf.54.2020.08.21.09.16.24 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Fri, 21 Aug 2020 09:16:25 -0700 (PDT) From: Show Liu To: libcamera-devel@lists.libcamera.org Date: Sat, 22 Aug 2020 00:16:00 +0800 Message-Id: <20200821161602.5093-2-show.liu@linaro.org> X-Mailer: git-send-email 2.20.1 In-Reply-To: <20200821161602.5093-1-show.liu@linaro.org> References: <20200821161602.5093-1-show.liu@linaro.org> MIME-Version: 1.0 Subject: [libcamera-devel] [PATCH v4 1/3] qcam: add OpenGL shader code as QT resource X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: libcamera-devel-bounces@lists.libcamera.org Sender: "libcamera-devel" qcam: add OpenGL shader code as QT resource Signed-off-by: Show Liu --- src/qcam/assets/shader/NV_2_planes_UV_f.glsl | 32 +++++++++++++++++++ src/qcam/assets/shader/NV_2_planes_VU_f.glsl | 32 +++++++++++++++++++ src/qcam/assets/shader/NV_3_planes_UV_f.glsl | 33 ++++++++++++++++++++ src/qcam/assets/shader/NV_3_planes_VU_f.glsl | 33 ++++++++++++++++++++ src/qcam/assets/shader/NV_vertex_shader.glsl | 16 ++++++++++ src/qcam/assets/shader/shaders.qrc | 10 ++++++ src/qcam/meson.build | 1 + 7 files changed, 157 insertions(+) create mode 100644 src/qcam/assets/shader/NV_2_planes_UV_f.glsl create mode 100644 src/qcam/assets/shader/NV_2_planes_VU_f.glsl create mode 100644 src/qcam/assets/shader/NV_3_planes_UV_f.glsl create mode 100644 src/qcam/assets/shader/NV_3_planes_VU_f.glsl create mode 100644 src/qcam/assets/shader/NV_vertex_shader.glsl create mode 100644 src/qcam/assets/shader/shaders.qrc diff --git a/src/qcam/assets/shader/NV_2_planes_UV_f.glsl b/src/qcam/assets/shader/NV_2_planes_UV_f.glsl new file mode 100644 index 0000000..32c6e90 --- /dev/null +++ b/src/qcam/assets/shader/NV_2_planes_UV_f.glsl @@ -0,0 +1,32 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (C) 2020, Linaro + * + * NV_2_planes_UV_f.glsl - Fragment shader code for NV12, NV16 and NV24 formats + */ + +#ifdef GL_ES +precision highp float; +#endif + +varying vec2 textureOut; +uniform sampler2D tex_y; +uniform sampler2D tex_u; + +void main(void) +{ + vec3 yuv; + vec3 rgb; + mat3 convert_mat = mat3( + vec3(1.1640625, 1.1640625, 1.1640625), + vec3(0.0, -0.390625, 2.015625), + vec3(1.5975625, -0.8125, 0.0) + ); + + yuv.x = texture2D(tex_y, textureOut).r - 0.0625; + yuv.y = texture2D(tex_u, textureOut).r - 0.5; + yuv.z = texture2D(tex_u, textureOut).g - 0.5; + + rgb = convert_mat * yuv; + gl_FragColor = vec4(rgb, 1.0); +} diff --git a/src/qcam/assets/shader/NV_2_planes_VU_f.glsl b/src/qcam/assets/shader/NV_2_planes_VU_f.glsl new file mode 100644 index 0000000..aae12de --- /dev/null +++ b/src/qcam/assets/shader/NV_2_planes_VU_f.glsl @@ -0,0 +1,32 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (C) 2020, Linaro + * + * NV_2_planes_VU_f.glsl - Fragment shader code for NV21, NV61 and NV42 formats + */ + +#ifdef GL_ES +precision highp float; +#endif + +varying vec2 textureOut; +uniform sampler2D tex_y; +uniform sampler2D tex_u; + +void main(void) +{ + vec3 yuv; + vec3 rgb; + mat3 convert_mat = mat3( + vec3(1.1640625, 1.1640625, 1.1640625), + vec3(0.0, -0.390625, 2.015625), + vec3(1.5975625, -0.8125, 0.0) + ); + + yuv.x = texture2D(tex_y, textureOut).r - 0.0625; + yuv.y = texture2D(tex_u, textureOut).g - 0.5; + yuv.z = texture2D(tex_u, textureOut).r - 0.5; + + rgb = convert_mat * yuv; + gl_FragColor = vec4(rgb, 1.0); +} diff --git a/src/qcam/assets/shader/NV_3_planes_UV_f.glsl b/src/qcam/assets/shader/NV_3_planes_UV_f.glsl new file mode 100644 index 0000000..21fff3a --- /dev/null +++ b/src/qcam/assets/shader/NV_3_planes_UV_f.glsl @@ -0,0 +1,33 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (C) 2020, Linaro + * + * NV_3_planes_UV_f.glsl - Fragment shader code for YUV420 format + */ + +#ifdef GL_ES +precision highp float; +#endif + +varying vec2 textureOut; +uniform sampler2D tex_y; +uniform sampler2D tex_u; +uniform sampler2D tex_v; + +void main(void) +{ + vec3 yuv; + vec3 rgb; + mat3 convert_mat = mat3( + vec3(1.1640625, 1.1640625, 1.1640625), + vec3(0.0, -0.390625, 2.015625), + vec3(1.5975625, -0.8125, 0.0) + ); + + yuv.x = texture2D(tex_y, textureOut).r - 0.0625; + yuv.y = texture2D(tex_u, textureOut).r - 0.5; + yuv.z = texture2D(tex_v, textureOut).g - 0.5; + + rgb = convert_mat * yuv; + gl_FragColor = vec4(rgb, 1.0); +} diff --git a/src/qcam/assets/shader/NV_3_planes_VU_f.glsl b/src/qcam/assets/shader/NV_3_planes_VU_f.glsl new file mode 100644 index 0000000..df00170 --- /dev/null +++ b/src/qcam/assets/shader/NV_3_planes_VU_f.glsl @@ -0,0 +1,33 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (C) 2020, Linaro + * + * NV_3_planes_VU_f.glsl - Fragment shader code for YVU420 format + */ + +#ifdef GL_ES +precision highp float; +#endif + +varying vec2 textureOut; +uniform sampler2D tex_y; +uniform sampler2D tex_u; +uniform sampler2D tex_v; + +void main(void) +{ + vec3 yuv; + vec3 rgb; + mat3 convert_mat = mat3( + vec3(1.1640625, 1.1640625, 1.1640625), + vec3(0.0, -0.390625, 2.015625), + vec3(1.5975625, -0.8125, 0.0) + ); + + yuv.x = texture2D(tex_y, textureOut).r - 0.0625; + yuv.y = texture2D(tex_u, textureOut).g - 0.5; + yuv.z = texture2D(tex_v, textureOut).r - 0.5; + + rgb = convert_mat * yuv; + gl_FragColor = vec4(rgb, 1.0); +} diff --git a/src/qcam/assets/shader/NV_vertex_shader.glsl b/src/qcam/assets/shader/NV_vertex_shader.glsl new file mode 100644 index 0000000..403b791 --- /dev/null +++ b/src/qcam/assets/shader/NV_vertex_shader.glsl @@ -0,0 +1,16 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (C) 2020, Linaro + * + * NV_vertex_shader.glsl - Vertex shader code for NV family + */ + +attribute vec4 vertexIn; +attribute vec2 textureIn; +varying vec2 textureOut; + +void main(void) +{ + gl_Position = vertexIn; + textureOut = textureIn; +} diff --git a/src/qcam/assets/shader/shaders.qrc b/src/qcam/assets/shader/shaders.qrc new file mode 100644 index 0000000..6fe4c7f --- /dev/null +++ b/src/qcam/assets/shader/shaders.qrc @@ -0,0 +1,10 @@ + + + +./NV_vertex_shader.glsl +./NV_2_planes_UV_f.glsl +./NV_2_planes_VU_f.glsl +./NV_3_planes_UV_f.glsl +./NV_3_planes_VU_f.glsl + + diff --git a/src/qcam/meson.build b/src/qcam/meson.build index 6ea886a..e0c6f26 100644 --- a/src/qcam/meson.build +++ b/src/qcam/meson.build @@ -16,6 +16,7 @@ qcam_moc_headers = files([ qcam_resources = files([ 'assets/feathericons/feathericons.qrc', + 'assets/shader/shaders.qrc' ]) qt5 = import('qt5') From patchwork Fri Aug 21 16:16:01 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Show Liu X-Patchwork-Id: 9356 Return-Path: X-Original-To: parsemail@patchwork.libcamera.org Delivered-To: parsemail@patchwork.libcamera.org Received: from lancelot.ideasonboard.com (lancelot.ideasonboard.com [92.243.16.209]) by patchwork.libcamera.org (Postfix) with ESMTPS id C46B8BD87C for ; Fri, 21 Aug 2020 16:16:33 +0000 (UTC) Received: from lancelot.ideasonboard.com (localhost [IPv6:::1]) by lancelot.ideasonboard.com (Postfix) with ESMTP id 91E1E61FE7; Fri, 21 Aug 2020 18:16:33 +0200 (CEST) Authentication-Results: lancelot.ideasonboard.com; dkim=fail reason="signature verification failed" (2048-bit key; unprotected) header.d=linaro.org header.i=@linaro.org header.b="nojltH6r"; dkim-atps=neutral Received: from mail-pj1-x1032.google.com (mail-pj1-x1032.google.com [IPv6:2607:f8b0:4864:20::1032]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id 7229860383 for ; Fri, 21 Aug 2020 18:16:32 +0200 (CEST) Received: by mail-pj1-x1032.google.com with SMTP id 2so1023832pjx.5 for ; Fri, 21 Aug 2020 09:16:32 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=linaro.org; s=google; h=from:to:cc:subject:date:message-id:in-reply-to:references :mime-version:content-transfer-encoding; bh=pd0uFP/AovgRjHxc9WoEZwRE9JT0940ZP/Y05SE1T2o=; b=nojltH6rJmTN+YqTEu+tgVGKyX0yPVFVSpUGM2QfU0uEFSo/yvdj45EG7xQN4XwWSK HD4OEiPvd5pNscibiXJxhNMZ7uCryItVQ6vTmADdWw2WTkYk3nvBOyQ543RrGlrX+M80 qyNqV55NwVLjQKYpbsP7vh1z/5n3BTlqPon7opyodVKtf5ky0BtT1CrGRXCys92DcpcW pCRC9RXcdCzVtuYmcWqIRamc488A2EySnPElzvGQq9tBl7/MYDVdlSajwj829ySw/ARg 6UvX3VL/EMwDUav27tBuHXiJNIbHkUKq37vv9G9YwwqNot3YClOkfti5Nal3xHQ1Xh8k zLpg== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20161025; h=x-gm-message-state:from:to:cc:subject:date:message-id:in-reply-to :references:mime-version:content-transfer-encoding; bh=pd0uFP/AovgRjHxc9WoEZwRE9JT0940ZP/Y05SE1T2o=; b=KxJBB7jZJi5br6jD5PqyCAukcdhvT71FQjVK/GLxpaBGF7y3dSEAw//bc3Jy9Eit2B DaCo/4g6dqFR0ubR2Wke4F25T8rf7aBbe9yVoNkbgm9tO59Cut5fV39Mmm0u8Poez++E TtwMayEHmhxFfS6wSeQTCFaq3BFPWyBAg6JKCyhbR/P7/xV2bX0HFnzcqLAifCpB8WeF r5WmS8OnBwhDgMzjaGq5Je9WAIrklbAXwG/VdqzDvkTieYgzQCxvxo9edXFXV2FLtXS2 hCNZtgsoAqLKkB72fy6kCjE/ONZ7qV4aOgAjs/qJdCuUblpazBLvk7g2TntXahfHgb9t W2iw== X-Gm-Message-State: AOAM530LdYCn2mJsb+Y+arYDc0KMFcpCt1HwGhqntht6G2xKMxhToTmr ULiXpzGtojtA1Gbp/muSb/Q1wQLRKEpoFA== X-Google-Smtp-Source: ABdhPJx5/yXJ86co8VMU8KNFSTN6miRDb9oM9+1CFrClcULNuSnX9gYnYqCOBMHAQYcQ7dKgg2xXyA== X-Received: by 2002:a17:90b:1287:: with SMTP id fw7mr2932712pjb.218.1598026590205; Fri, 21 Aug 2020 09:16:30 -0700 (PDT) Received: from localhost.localdomain (2001-b011-200c-3405-5522-d8a8-1333-f09d.dynamic-ip6.hinet.net. [2001:b011:200c:3405:5522:d8a8:1333:f09d]) by smtp.gmail.com with ESMTPSA id h15sm2524184pjf.54.2020.08.21.09.16.29 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Fri, 21 Aug 2020 09:16:29 -0700 (PDT) From: Show Liu To: libcamera-devel@lists.libcamera.org Date: Sat, 22 Aug 2020 00:16:01 +0800 Message-Id: <20200821161602.5093-3-show.liu@linaro.org> X-Mailer: git-send-email 2.20.1 In-Reply-To: <20200821161602.5093-1-show.liu@linaro.org> References: <20200821161602.5093-1-show.liu@linaro.org> MIME-Version: 1.0 Subject: [libcamera-devel] [PATCH v4 2/3] qcam: add OpenGL renderer X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: libcamera-devel-bounces@lists.libcamera.org Sender: "libcamera-devel" qcam: add OpenGL renderer Signed-off-by: Show Liu --- src/qcam/meson.build | 2 + src/qcam/renderer.cpp | 346 ++++++++++++++++++++++++++++++++++++++++++ src/qcam/renderer.h | 81 ++++++++++ 3 files changed, 429 insertions(+) create mode 100644 src/qcam/renderer.cpp create mode 100644 src/qcam/renderer.h diff --git a/src/qcam/meson.build b/src/qcam/meson.build index e0c6f26..8c9032f 100644 --- a/src/qcam/meson.build +++ b/src/qcam/meson.build @@ -7,11 +7,13 @@ qcam_sources = files([ 'main.cpp', 'main_window.cpp', 'viewfinder.cpp', + 'renderer.cpp' ]) qcam_moc_headers = files([ 'main_window.h', 'viewfinder.h', + 'renderer.h' ]) qcam_resources = files([ diff --git a/src/qcam/renderer.cpp b/src/qcam/renderer.cpp new file mode 100644 index 0000000..23e8fa8 --- /dev/null +++ b/src/qcam/renderer.cpp @@ -0,0 +1,346 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (C) 2020, Linaro + * + * renderer.cpp - Render YUV format frame by OpenGL shader + */ + +#include "renderer.h" + +Renderer::Renderer() + : fbo_(nullptr), + vbo_(QOpenGLBuffer::VertexBuffer), + pFShader_(nullptr), + pVShader_(nullptr), + textureU_(QOpenGLTexture::Target2D), + textureV_(QOpenGLTexture::Target2D), + textureY_(QOpenGLTexture::Target2D) +{ + /* offscreen format setup */ + setFormat(requestedFormat()); + create(); + + /* create OpenGL context */ + if (ctx_.create()) { + ctx_.makeCurrent(this); + initializeOpenGLFunctions(); + } else { + qWarning() << "[Renderer]: " + << "OpenGL renderer is not available."; + } +} + +Renderer::~Renderer() +{ + if (vbo_.isCreated()) { + vbo_.release(); + vbo_.destroy(); + } + + if (fbo_) { + fbo_->release(); + delete fbo_; + } + + removeShader(); + + if (textureY_.isCreated()) + textureY_.destroy(); + + if (textureU_.isCreated()) + textureU_.destroy(); + + if (textureV_.isCreated()) + textureV_.destroy(); + + ctx_.doneCurrent(); +} + +void Renderer::initializeGL() +{ + glEnable(GL_TEXTURE_2D); + glDisable(GL_DEPTH_TEST); + + static const GLfloat vertices[]{ + -1.0f, -1.0f, -1.0f, +1.0f, + +1.0f, +1.0f, +1.0f, -1.0f, + 0.0f, 1.0f, 0.0f, 0.0f, + 1.0f, 0.0f, 1.0f, 1.0f + }; + + vbo_.create(); + vbo_.bind(); + vbo_.allocate(vertices, sizeof(vertices)); + + glClearColor(1.0f, 1.0f, 1.0f, 0.0f); +} + +bool Renderer::selectShader(const libcamera::PixelFormat &format) +{ + bool ret = true; + switch (format) { + case libcamera::formats::NV12: + horzSubSample_ = 2; + vertSubSample_ = 2; + vsrc_ = ":NV_vertex_shader.glsl"; + fsrc_ = ":NV_2_planes_UV_f.glsl"; + break; + case libcamera::formats::NV21: + horzSubSample_ = 2; + vertSubSample_ = 2; + vsrc_ = ":NV_vertex_shader.glsl"; + fsrc_ = ":NV_2_planes_VU_f.glsl"; + break; + case libcamera::formats::NV16: + horzSubSample_ = 2; + vertSubSample_ = 1; + vsrc_ = ":NV_vertex_shader.glsl"; + fsrc_ = ":NV_2_planes_UV_f.glsl"; + break; + case libcamera::formats::NV61: + horzSubSample_ = 2; + vertSubSample_ = 1; + vsrc_ = ":NV_vertex_shader.glsl"; + fsrc_ = ":NV_2_planes_VU_f.glsl"; + break; + case libcamera::formats::NV24: + horzSubSample_ = 1; + vertSubSample_ = 1; + vsrc_ = ":NV_vertex_shader.glsl"; + fsrc_ = ":NV_2_planes_UV_f.glsl"; + break; + case libcamera::formats::NV42: + horzSubSample_ = 1; + vertSubSample_ = 1; + vsrc_ = ":NV_vertex_shader.glsl"; + fsrc_ = ":NV_2_planes_VU_f.glsl"; + break; + case libcamera::formats::YUV420: + horzSubSample_ = 2; + vertSubSample_ = 2; + vsrc_ = ":NV_vertex_shader.glsl"; + fsrc_ = ":NV_3_planes_UV_f.glsl"; + break; + default: + ret = false; + qWarning() << "[Renderer]: " + << "format not support yet."; + break; + }; + + return ret; +} + +void Renderer::removeShader() +{ + if (shaderProgram_.isLinked()) { + shaderProgram_.release(); + shaderProgram_.removeAllShaders(); + } + + if (pFShader_) + delete pFShader_; + + if (pVShader_) + delete pVShader_; +} + +bool Renderer::createShader() +{ + bool bCompile; + + /* Create Vertex Shader */ + pVShader_ = new QOpenGLShader(QOpenGLShader::Vertex, this); + + bCompile = pVShader_->compileSourceFile(vsrc_); + if (!bCompile) { + qWarning() << "[Renderer]: " << pVShader_->log(); + return bCompile; + } + + shaderProgram_.addShader(pVShader_); + + /* Create Fragment Shader */ + pFShader_ = new QOpenGLShader(QOpenGLShader::Fragment, this); + + bCompile = pFShader_->compileSourceFile(fsrc_); + if (!bCompile) { + qWarning() << "[Renderer]: " << pFShader_->log(); + return bCompile; + } + + shaderProgram_.addShader(pFShader_); + + // Link shader pipeline + if (!shaderProgram_.link()) { + qWarning() << "[Renderer]: " << shaderProgram_.log(); + return false; + } + + // Bind shader pipeline for use + if (!shaderProgram_.bind()) { + qWarning() << "[Renderer]: " << shaderProgram_.log(); + return false; + } + return true; +} + +bool Renderer::configure(const libcamera::PixelFormat &format, const QSize &size) +{ + bool ret = true; + + if (selectShader(format)) { + ret = createShader(); + if (!ret) + return ret; + + shaderProgram_.enableAttributeArray(ATTRIB_VERTEX); + shaderProgram_.enableAttributeArray(ATTRIB_TEXTURE); + + shaderProgram_.setAttributeBuffer(ATTRIB_VERTEX, + GL_FLOAT, + 0, + 2, + 2 * sizeof(GLfloat)); + shaderProgram_.setAttributeBuffer(ATTRIB_TEXTURE, + GL_FLOAT, + 8 * sizeof(GLfloat), + 2, + 2 * sizeof(GLfloat)); + + textureUniformY_ = shaderProgram_.uniformLocation("tex_y"); + textureUniformU_ = shaderProgram_.uniformLocation("tex_u"); + textureUniformV_ = shaderProgram_.uniformLocation("tex_v"); + + if (!textureY_.isCreated()) + textureY_.create(); + + if (!textureU_.isCreated()) + textureU_.create(); + + if (!textureV_.isCreated()) + textureV_.create(); + + id_y_ = textureY_.textureId(); + id_u_ = textureU_.textureId(); + id_v_ = textureV_.textureId(); + + fbo_ = new QOpenGLFramebufferObject(size.width(), + size.height(), + GL_TEXTURE_2D); + fbo_->bind(); + glViewport(0, 0, size.width(), size.height()); + + format_ = format; + size_ = size; + } else { + ret = false; + } + return ret; +} + +void Renderer::configureTexture(unsigned int id) +{ + glBindTexture(GL_TEXTURE_2D, id); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); +} + +void Renderer::render(unsigned char *buffer) +{ + QMutexLocker locker(&mutex_); + + glClearColor(0.0, 0.0, 0.0, 1.0); + glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT); + + switch (format_) { + case libcamera::formats::NV12: + case libcamera::formats::NV21: + case libcamera::formats::NV16: + case libcamera::formats::NV61: + case libcamera::formats::NV24: + case libcamera::formats::NV42: + /* activate texture 0 */ + glActiveTexture(GL_TEXTURE0); + configureTexture(id_y_); + glTexImage2D(GL_TEXTURE_2D, + 0, + GL_RED, + size_.width(), + size_.height(), + 0, + GL_RED, + GL_UNSIGNED_BYTE, + buffer); + glUniform1i(textureUniformY_, 0); + + /* activate texture 1 */ + glActiveTexture(GL_TEXTURE1); + configureTexture(id_u_); + glTexImage2D(GL_TEXTURE_2D, + 0, + GL_RG, + size_.width() / horzSubSample_, + size_.height() / vertSubSample_, + 0, + GL_RG, + GL_UNSIGNED_BYTE, + buffer + size_.width() * size_.height()); + glUniform1i(textureUniformU_, 1); + break; + case libcamera::formats::YUV420: + /* activate texture 0 */ + glActiveTexture(GL_TEXTURE0); + configureTexture(id_y_); + glTexImage2D(GL_TEXTURE_2D, + 0, + GL_RED, + size_.width(), + size_.height(), + 0, + GL_RED, + GL_UNSIGNED_BYTE, + buffer); + glUniform1i(textureUniformY_, 0); + + /* activate texture 1 */ + glActiveTexture(GL_TEXTURE1); + configureTexture(id_u_); + glTexImage2D(GL_TEXTURE_2D, + 0, + GL_RG, + size_.width() / horzSubSample_, + size_.height() / vertSubSample_, + 0, + GL_RG, + GL_UNSIGNED_BYTE, + buffer + size_.width() * size_.height()); + glUniform1i(textureUniformU_, 1); + + /* activate texture 2 */ + glActiveTexture(GL_TEXTURE2); + configureTexture(id_v_); + glTexImage2D(GL_TEXTURE_2D, + 0, + GL_RG, + size_.width() / horzSubSample_, + size_.height() / vertSubSample_, + 0, GL_RG, + GL_UNSIGNED_BYTE, + buffer + size_.width() * size_.height() * 5 / 4); + glUniform1i(textureUniformV_, 2); + break; + default: + break; + }; + + glDrawArrays(GL_TRIANGLE_FAN, 0, 4); +} + +QImage Renderer::toImage() +{ + QMutexLocker locker(&mutex_); + return (fbo_->toImage(true)); +} diff --git a/src/qcam/renderer.h b/src/qcam/renderer.h new file mode 100644 index 0000000..1ea0c48 --- /dev/null +++ b/src/qcam/renderer.h @@ -0,0 +1,81 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (C) 2020, Linaro + * + * renderer.h - Render YUV format frame by OpenGL shader + */ +#ifndef __QCAM_RENDERER_H__ +#define __QCAM_RENDERER_H__ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#define ATTRIB_VERTEX 0 +#define ATTRIB_TEXTURE 1 + +class Renderer : public QOffscreenSurface, protected QOpenGLFunctions +{ + Q_OBJECT + +public: + Renderer(); + ~Renderer(); + + void initializeGL(); + bool configure(const libcamera::PixelFormat &format, const QSize &size); + void render(unsigned char *buffer); + QImage toImage(); + +private: + bool createShader(); + void configureTexture(unsigned int id); + void removeShader(); + bool selectShader(const libcamera::PixelFormat &format); + + /* OpenGL renderer components */ + QOpenGLContext ctx_; + QOpenGLFramebufferObject *fbo_; + QOpenGLBuffer vbo_; + QOpenGLShader *pFShader_; + QOpenGLShader *pVShader_; + QOpenGLShaderProgram shaderProgram_; + QSurfaceFormat surfaceFormat_; + + /* Fragment and Vertex shader file */ + QString fsrc_; + QString vsrc_; + + /* YUV frame size and format */ + libcamera::PixelFormat format_; + QSize size_; + + /* YUV texture planars and parameters*/ + GLuint id_u_; + GLuint id_v_; + GLuint id_y_; + GLuint textureUniformU_; + GLuint textureUniformV_; + GLuint textureUniformY_; + QOpenGLTexture textureU_; + QOpenGLTexture textureV_; + QOpenGLTexture textureY_; + unsigned int horzSubSample_; + unsigned int vertSubSample_; + + QImage image_; + QMutex mutex_; /* Prevent concurrent access to image_ */ +}; + +#endif /* __QCAM_RENDERER_H__ */ From patchwork Fri Aug 21 16:16:02 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Show Liu X-Patchwork-Id: 9357 Return-Path: X-Original-To: parsemail@patchwork.libcamera.org Delivered-To: parsemail@patchwork.libcamera.org Received: from lancelot.ideasonboard.com (lancelot.ideasonboard.com [92.243.16.209]) by patchwork.libcamera.org (Postfix) with ESMTPS id 2B4C5BD87C for ; Fri, 21 Aug 2020 16:16:37 +0000 (UTC) Received: from lancelot.ideasonboard.com (localhost [IPv6:::1]) by lancelot.ideasonboard.com (Postfix) with ESMTP id EAC1261FE7; Fri, 21 Aug 2020 18:16:36 +0200 (CEST) Authentication-Results: lancelot.ideasonboard.com; dkim=fail reason="signature verification failed" (2048-bit key; unprotected) header.d=linaro.org header.i=@linaro.org header.b="KSrFtMls"; dkim-atps=neutral Received: from mail-pj1-x1042.google.com (mail-pj1-x1042.google.com [IPv6:2607:f8b0:4864:20::1042]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id 12E9F60383 for ; Fri, 21 Aug 2020 18:16:35 +0200 (CEST) Received: by mail-pj1-x1042.google.com with SMTP id t6so1028315pjr.0 for ; Fri, 21 Aug 2020 09:16:35 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=linaro.org; s=google; h=from:to:cc:subject:date:message-id:in-reply-to:references :mime-version:content-transfer-encoding; bh=AABaVwCLWJKMc3x6nrrpXEKPsBtmSlwX7fcsA/Kyxpg=; b=KSrFtMlsKdInzHkSnuHHxn0VbrRL6d6UFxUdHo9C1cZ/7j/vczunbIJEAxfKYr+aLr OuFxQoWUxKnpgzbReSWXxESzjUygVrA2tjAPne30Z++oBZWdCE/PM6joFDoyUZqrLK+g ed1yIRIa+EV1eN2M/i4bzeUfLmdRmRgQLSlC4LSCdvmf41hCASUzI4A1wnsp6VPQxLcS 7SOi2iZjWBCz95rQ++mMdgLkQoweykVrsvB1pE0yp0+hsP7PymJnIxlDPrknA50ZW6c2 8+IdjnmwgZ8/Y9EMKuwKvx0HPSeSq7K7a8Ll61kXUYeeDqOeNIbYgm0icWFlEqCdKYqX DOGw== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20161025; h=x-gm-message-state:from:to:cc:subject:date:message-id:in-reply-to :references:mime-version:content-transfer-encoding; bh=AABaVwCLWJKMc3x6nrrpXEKPsBtmSlwX7fcsA/Kyxpg=; b=YxTJ2M+bFAFsDBFkHjJXiP199urZ9HvbcFkO+RXbNQH8g3w59JzoHZKLnIMSphRB1k tRVz+rg6MkSOnyDOWGnO2Lh5iFvvNinMAAlZB71CURnuONPmmhfcB55nP7eQYaY0wsAn X0dK60vVYr1zTm61ILy5jsoSL8daYUEaEBcNjLLx7uTRgyb4WT5vP/oMto8+76UKHW83 UPw+CuSFDJAI0ZVoiWX2hXOEGOZQg1YvxlhtSiKVEI79WKT/YExYhCq11jF5f7FviZbj M2n/p7912F5WKzc3nBM72/Qum6LVR3IEDPtvHw1TUPzjgaEDyJxJlPFyC6HI69+2SS8G kp2A== X-Gm-Message-State: AOAM533V4HZg1Ki96KfEtvELHCYZQxND1RQtXj5TXnm6fMAOTEokg9/i 9iCxLShIYFrLiHG2Xt50nrdTbzHR2OPaOA== X-Google-Smtp-Source: ABdhPJw/0ms84Nj8v3JhE3XSPQP8sAsRpMREyjWEjDQFAY5sSl0S8DiAsRZvHjyJQyANxzvWVbi8fA== X-Received: by 2002:a17:90a:db96:: with SMTP id h22mr3089527pjv.28.1598026593458; Fri, 21 Aug 2020 09:16:33 -0700 (PDT) Received: from localhost.localdomain (2001-b011-200c-3405-5522-d8a8-1333-f09d.dynamic-ip6.hinet.net. [2001:b011:200c:3405:5522:d8a8:1333:f09d]) by smtp.gmail.com with ESMTPSA id h15sm2524184pjf.54.2020.08.21.09.16.32 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Fri, 21 Aug 2020 09:16:33 -0700 (PDT) From: Show Liu To: libcamera-devel@lists.libcamera.org Date: Sat, 22 Aug 2020 00:16:02 +0800 Message-Id: <20200821161602.5093-4-show.liu@linaro.org> X-Mailer: git-send-email 2.20.1 In-Reply-To: <20200821161602.5093-1-show.liu@linaro.org> References: <20200821161602.5093-1-show.liu@linaro.org> MIME-Version: 1.0 Subject: [libcamera-devel] [PATCH v4 3/3] qcam: use the OpenGL renderer as NV family YUV format converter X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: libcamera-devel-bounces@lists.libcamera.org Sender: "libcamera-devel" qcam: use the OpenGL renderer as NV family YUV format converter Signed-off-by: Show Liu --- src/qcam/main.cpp | 3 +++ src/qcam/main_window.cpp | 2 ++ src/qcam/main_window.h | 1 + src/qcam/viewfinder.cpp | 46 +++++++++++++++++++++++++++++++++------- src/qcam/viewfinder.h | 10 +++++++++ 5 files changed, 54 insertions(+), 8 deletions(-) diff --git a/src/qcam/main.cpp b/src/qcam/main.cpp index b3468cb..a336486 100644 --- a/src/qcam/main.cpp +++ b/src/qcam/main.cpp @@ -35,6 +35,9 @@ OptionsParser::Options parseOptions(int argc, char *argv[]) "help"); parser.addOption(OptStream, &streamKeyValue, "Set configuration of a camera stream", "stream", true); + parser.addOption(OptRender, OptionString, + "Choose the render type use qt|gles", "render", + ArgumentRequired, "render"); OptionsParser::Options options = parser.parse(argc, argv); if (options.isSet(OptHelp)) diff --git a/src/qcam/main_window.cpp b/src/qcam/main_window.cpp index 7503537..26e8a69 100644 --- a/src/qcam/main_window.cpp +++ b/src/qcam/main_window.cpp @@ -106,6 +106,8 @@ MainWindow::MainWindow(CameraManager *cm, const OptionsParser::Options &options) connect(&titleTimer_, SIGNAL(timeout()), this, SLOT(updateTitle())); viewfinder_ = new ViewFinder(this); + if (options_.isSet(OptRender)) + viewfinder_->setRender(options_[OptRender].toString().c_str()); connect(viewfinder_, &ViewFinder::renderComplete, this, &MainWindow::queueRequest); setCentralWidget(viewfinder_); diff --git a/src/qcam/main_window.h b/src/qcam/main_window.h index 3d21779..ecc9938 100644 --- a/src/qcam/main_window.h +++ b/src/qcam/main_window.h @@ -38,6 +38,7 @@ enum { OptCamera = 'c', OptHelp = 'h', OptStream = 's', + OptRender = 'r', }; class CaptureRequest diff --git a/src/qcam/viewfinder.cpp b/src/qcam/viewfinder.cpp index dcf8a15..7e2b7a7 100644 --- a/src/qcam/viewfinder.cpp +++ b/src/qcam/viewfinder.cpp @@ -20,6 +20,7 @@ #include #include "format_converter.h" +#include "renderer.h" static const QMap nativeFormats { @@ -34,13 +35,15 @@ static const QMap nativeFormats }; ViewFinder::ViewFinder(QWidget *parent) - : QWidget(parent), buffer_(nullptr) + : QWidget(parent), buffer_(nullptr), renderer_(nullptr) { icon_ = QIcon(":camera-off.svg"); } ViewFinder::~ViewFinder() { + if (renderer_) + delete renderer_; } const QList &ViewFinder::nativeFormats() const @@ -59,14 +62,26 @@ int ViewFinder::setFormat(const libcamera::PixelFormat &format, * the destination image. */ if (!::nativeFormats.contains(format)) { - int ret = converter_.configure(format, size); - if (ret < 0) - return ret; + if ((renderType_ == RenderGLES) && + renderer_->configure(format, size)) { + qInfo() << "Using OpenGL shader format conversion from " + << format.toString().c_str(); + } else { + int ret = converter_.configure(format, size); + if (ret < 0) + return ret; + + image_ = QImage(size, QImage::Format_RGB32); - image_ = QImage(size, QImage::Format_RGB32); + qInfo() << "Using software format conversion from" + << format.toString().c_str(); - qInfo() << "Using software format conversion from" - << format.toString().c_str(); + if (renderType_ == RenderGLES) { + renderType_ = RenderQT; + qInfo() << "Using QT rendering due to " + << "OpenGL shader configure failed."; + } + } } else { qInfo() << "Zero-copy enabled"; } @@ -111,7 +126,11 @@ void ViewFinder::render(libcamera::FrameBuffer *buffer, MappedBuffer *map) * Otherwise, convert the format and release the frame * buffer immediately. */ - converter_.convert(memory, size, &image_); + if (renderType_ == RenderGLES) { + renderer_->render(memory); + image_ = QImage(renderer_->toImage()); + } else + converter_.convert(memory, size, &image_); } } @@ -179,3 +198,14 @@ QSize ViewFinder::sizeHint() const { return size_.isValid() ? size_ : QSize(640, 480); } + +void ViewFinder::setRender(std::string render_type) +{ + if (render_type == "gles") { + renderer_ = new Renderer(); + renderer_->initializeGL(); + renderType_ = RenderGLES; + } else { + renderType_ = RenderQT; + } +} diff --git a/src/qcam/viewfinder.h b/src/qcam/viewfinder.h index 26a1320..ed09c1e 100644 --- a/src/qcam/viewfinder.h +++ b/src/qcam/viewfinder.h @@ -20,6 +20,7 @@ #include #include "format_converter.h" +#include "renderer.h" class QImage; @@ -28,6 +29,11 @@ struct MappedBuffer { size_t size; }; +enum RenderType { + RenderGLES, + RenderQT, +}; + class ViewFinder : public QWidget { Q_OBJECT @@ -43,6 +49,7 @@ public: void stop(); QImage getCurrentImage(); + void setRender(std::string render_type); Q_SIGNALS: void renderComplete(libcamera::FrameBuffer *buffer); @@ -66,6 +73,9 @@ private: libcamera::FrameBuffer *buffer_; QImage image_; QMutex mutex_; /* Prevent concurrent access to image_ */ + + Renderer *renderer_; /* OpenGL renderer */ + RenderType renderType_; }; #endif /* __QCAM_VIEWFINDER__ */