From patchwork Mon Aug 29 10:04:14 2022 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Laurent Pinchart X-Patchwork-Id: 17225 Return-Path: X-Original-To: parsemail@patchwork.libcamera.org Delivered-To: parsemail@patchwork.libcamera.org Received: from lancelot.ideasonboard.com (lancelot.ideasonboard.com [92.243.16.209]) by patchwork.libcamera.org (Postfix) with ESMTPS id 95D13C3272 for ; Mon, 29 Aug 2022 10:04:32 +0000 (UTC) Received: from lancelot.ideasonboard.com (localhost [IPv6:::1]) by lancelot.ideasonboard.com (Postfix) with ESMTP id 4F45E61FBE; Mon, 29 Aug 2022 12:04:32 +0200 (CEST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=libcamera.org; s=mail; t=1661767472; bh=6sbzgQdntHRRZ4KlQmYYZ41mnQ+VTWs7kF/CEKL6a4M=; h=To:Date:In-Reply-To:References:Subject:List-Id:List-Unsubscribe: List-Archive:List-Post:List-Help:List-Subscribe:From:Reply-To: From; b=lM+U05l0oCHRDZQQySTXmgYFB+eyn3IANuouVhgkKuAx+GlZrhvZp2oJsJEIctte6 SqiMPuqZdLZdves8ANKopulZNB8abA/Et8tG+LZELES+7BeGGmu8AAasm1Nu9gKwKl bNWKVzfqmLe51aHaIl3eKaKORwp7fE5CJL4ckyGr5daeOnsJ/yBxALWiWigC9ivULf PpM18T6ZT/Vma7KD+/mX0u7N+iPSYeM2WSxFntC5lXYdh5WEo90dP28AblOnWromPT A2mZRpIZoNeBwIefByoCxns8zGsHlDM+Poa4hS6ZRHmh0IpSBWl0ScYi0xq64D4w7m wvd/BPiNJLpxg== Received: from perceval.ideasonboard.com (perceval.ideasonboard.com [213.167.242.64]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id 932A461FBE for ; Mon, 29 Aug 2022 12:04:30 +0200 (CEST) Authentication-Results: lancelot.ideasonboard.com; dkim=pass (1024-bit key; unprotected) header.d=ideasonboard.com header.i=@ideasonboard.com header.b="W4abnQlc"; dkim-atps=neutral Received: from pendragon.ideasonboard.com (62-78-145-57.bb.dnainternet.fi [62.78.145.57]) by perceval.ideasonboard.com (Postfix) with ESMTPSA id B82B58BD for ; Mon, 29 Aug 2022 12:04:29 +0200 (CEST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=ideasonboard.com; s=mail; t=1661767470; bh=6sbzgQdntHRRZ4KlQmYYZ41mnQ+VTWs7kF/CEKL6a4M=; h=From:To:Subject:Date:In-Reply-To:References:From; b=W4abnQlcu1Vx9iMAE6DwwdLFq8oN7u3tkZ8LWHf8jKzLrJ5VDHKpCZQp7Vc2/WtUa Vl2NWlBXO9+NQJmCL467tpPsFdrxfuaYYOAZ0mE/Ea3QTEE2HKaQF/gp0gf7IMoCvg O7ms//fWgJNrriag66Of5f22k7J21lzX0PwrK3ek= To: libcamera-devel@lists.libcamera.org Date: Mon, 29 Aug 2022 13:04:14 +0300 Message-Id: <20220829100414.28404-4-laurent.pinchart@ideasonboard.com> X-Mailer: git-send-email 2.35.1 In-Reply-To: <20220829100414.28404-1-laurent.pinchart@ideasonboard.com> References: <20220829100414.28404-1-laurent.pinchart@ideasonboard.com> MIME-Version: 1.0 Subject: [libcamera-devel] [PATCH 3/3] qcam: viewfinder_gl: Take color space into account for YUV rendering X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-Patchwork-Original-From: Laurent Pinchart via libcamera-devel From: Laurent Pinchart Reply-To: Laurent Pinchart Errors-To: libcamera-devel-bounces@lists.libcamera.org Sender: "libcamera-devel" Update the YUV shaders and the viewfinder_gl to correctly take the Y'CbCr encoding and the quantization range into account when rendering YUV formats to RGB. Support for the primaries and transfer function will be added in a subsequent step. Signed-off-by: Laurent Pinchart Reviewed-by: Umang Jain Reviewed-by: Kunal Agarwal --- src/qcam/assets/shader/YUV_2_planes.frag | 27 ++++---- src/qcam/assets/shader/YUV_3_planes.frag | 23 ++++--- src/qcam/assets/shader/YUV_packed.frag | 17 ++--- src/qcam/viewfinder_gl.cpp | 79 +++++++++++++++++++++++- src/qcam/viewfinder_gl.h | 2 + 5 files changed, 115 insertions(+), 33 deletions(-) diff --git a/src/qcam/assets/shader/YUV_2_planes.frag b/src/qcam/assets/shader/YUV_2_planes.frag index 254463c05cac..da8dbcc5f801 100644 --- a/src/qcam/assets/shader/YUV_2_planes.frag +++ b/src/qcam/assets/shader/YUV_2_planes.frag @@ -13,27 +13,30 @@ varying vec2 textureOut; uniform sampler2D tex_y; uniform sampler2D tex_u; +const mat3 yuv2rgb_matrix = mat3( + YUV2RGB_MATRIX +); + +const vec3 yuv2rgb_offset = vec3( + YUV2RGB_Y_OFFSET / 255.0, 128.0 / 255.0, 128.0 / 255.0 +); + void main(void) { vec3 yuv; - vec3 rgb; - mat3 yuv2rgb_bt601_mat = mat3( - vec3(1.164, 1.164, 1.164), - vec3(0.000, -0.392, 2.017), - vec3(1.596, -0.813, 0.000) - ); - yuv.x = texture2D(tex_y, textureOut).r - 0.063; + yuv.x = texture2D(tex_y, textureOut).r; #if defined(YUV_PATTERN_UV) - yuv.y = texture2D(tex_u, textureOut).r - 0.500; - yuv.z = texture2D(tex_u, textureOut).a - 0.500; + yuv.y = texture2D(tex_u, textureOut).r; + yuv.z = texture2D(tex_u, textureOut).a; #elif defined(YUV_PATTERN_VU) - yuv.y = texture2D(tex_u, textureOut).a - 0.500; - yuv.z = texture2D(tex_u, textureOut).r - 0.500; + yuv.y = texture2D(tex_u, textureOut).a; + yuv.z = texture2D(tex_u, textureOut).r; #else #error Invalid pattern #endif - rgb = yuv2rgb_bt601_mat * yuv; + vec3 rgb = yuv2rgb_matrix * (vec3(y, uv) - yuv2rgb_offset); + gl_FragColor = vec4(rgb, 1.0); } diff --git a/src/qcam/assets/shader/YUV_3_planes.frag b/src/qcam/assets/shader/YUV_3_planes.frag index 2be74b5d2a9d..e754129d74d1 100644 --- a/src/qcam/assets/shader/YUV_3_planes.frag +++ b/src/qcam/assets/shader/YUV_3_planes.frag @@ -14,20 +14,23 @@ uniform sampler2D tex_y; uniform sampler2D tex_u; uniform sampler2D tex_v; +const mat3 yuv2rgb_matrix = mat3( + YUV2RGB_MATRIX +); + +const vec3 yuv2rgb_offset = vec3( + YUV2RGB_Y_OFFSET / 255.0, 128.0 / 255.0, 128.0 / 255.0 +); + void main(void) { vec3 yuv; - vec3 rgb; - mat3 yuv2rgb_bt601_mat = mat3( - vec3(1.164, 1.164, 1.164), - vec3(0.000, -0.392, 2.017), - vec3(1.596, -0.813, 0.000) - ); - yuv.x = texture2D(tex_y, textureOut).r - 0.063; - yuv.y = texture2D(tex_u, textureOut).r - 0.500; - yuv.z = texture2D(tex_v, textureOut).r - 0.500; + yuv.x = texture2D(tex_y, textureOut).r; + yuv.y = texture2D(tex_u, textureOut).r; + yuv.z = texture2D(tex_v, textureOut).r; + + vec3 rgb = yuv2rgb_matrix * (vec3(y, uv) - yuv2rgb_offset); - rgb = yuv2rgb_bt601_mat * yuv; gl_FragColor = vec4(rgb, 1.0); } diff --git a/src/qcam/assets/shader/YUV_packed.frag b/src/qcam/assets/shader/YUV_packed.frag index d6efd4ce92a9..b9ef9d41beae 100644 --- a/src/qcam/assets/shader/YUV_packed.frag +++ b/src/qcam/assets/shader/YUV_packed.frag @@ -14,15 +14,16 @@ varying vec2 textureOut; uniform sampler2D tex_y; uniform vec2 tex_step; +const mat3 yuv2rgb_matrix = mat3( + YUV2RGB_MATRIX +); + +const vec3 yuv2rgb_offset = vec3( + YUV2RGB_Y_OFFSET / 255.0, 128.0 / 255.0, 128.0 / 255.0 +); + void main(void) { - mat3 yuv2rgb_bt601_mat = mat3( - vec3(1.164, 1.164, 1.164), - vec3(0.000, -0.392, 2.017), - vec3(1.596, -0.813, 0.000) - ); - vec3 yuv2rgb_bt601_offset = vec3(0.063, 0.500, 0.500); - /* * The sampler won't interpolate the texture correctly along the X axis, * as each RGBA pixel effectively stores two pixels. We thus need to @@ -76,7 +77,7 @@ void main(void) float y = mix(y_left, y_right, step(0.5, f_x)); - vec3 rgb = yuv2rgb_bt601_mat * (vec3(y, uv) - yuv2rgb_bt601_offset); + vec3 rgb = yuv2rgb_matrix * (vec3(y, uv) - yuv2rgb_offset); gl_FragColor = vec4(rgb, 1.0); } diff --git a/src/qcam/viewfinder_gl.cpp b/src/qcam/viewfinder_gl.cpp index ec295b6de0dd..e2aa24703ff0 100644 --- a/src/qcam/viewfinder_gl.cpp +++ b/src/qcam/viewfinder_gl.cpp @@ -7,9 +7,12 @@ #include "viewfinder_gl.h" +#include + #include #include #include +#include #include @@ -56,7 +59,8 @@ static const QList supportedFormats{ }; ViewFinderGL::ViewFinderGL(QWidget *parent) - : QOpenGLWidget(parent), buffer_(nullptr), image_(nullptr), + : QOpenGLWidget(parent), buffer_(nullptr), + colorSpace_(libcamera::ColorSpace::Raw), image_(nullptr), vertexBuffer_(QOpenGLBuffer::VertexBuffer) { } @@ -72,10 +76,10 @@ const QList &ViewFinderGL::nativeFormats() const } int ViewFinderGL::setFormat(const libcamera::PixelFormat &format, const QSize &size, - [[maybe_unused]] const libcamera::ColorSpace &colorSpace, + const libcamera::ColorSpace &colorSpace, unsigned int stride) { - if (format != format_) { + if (format != format_ || colorSpace != colorSpace_) { /* * If the fragment already exists, remove it and create a new * one for the new format. @@ -89,7 +93,10 @@ int ViewFinderGL::setFormat(const libcamera::PixelFormat &format, const QSize &s if (!selectFormat(format)) return -1; + selectColorSpace(colorSpace); + format_ = format; + colorSpace_ = colorSpace; } size_ = size; @@ -318,6 +325,72 @@ bool ViewFinderGL::selectFormat(const libcamera::PixelFormat &format) return ret; } +void ViewFinderGL::selectColorSpace(const libcamera::ColorSpace &colorSpace) +{ + std::array yuv2rgb; + + /* OpenGL stores arrays in column-major order. */ + switch (colorSpace.ycbcrEncoding) { + case libcamera::ColorSpace::YcbcrEncoding::None: + yuv2rgb = { + 1.0000, 0.0000, 0.0000, + 0.0000, 1.0000, 0.0000, + 0.0000, 0.0000, 1.0000, + }; + break; + + case libcamera::ColorSpace::YcbcrEncoding::Rec601: + yuv2rgb = { + 1.0000, 1.0000, 1.0000, + 0.0000, -0.3441, 1.7720, + 1.4020, -0.7141, 0.0000, + }; + break; + + case libcamera::ColorSpace::YcbcrEncoding::Rec709: + yuv2rgb = { + 1.0000, 1.0000, 1.0000, + 0.0000, -0.1873, 1.8856, + 1.5748, -0.4681, 0.0000, + }; + break; + + case libcamera::ColorSpace::YcbcrEncoding::Rec2020: + yuv2rgb = { + 1.0000, 1.0000, 1.0000, + 0.0000, -0.1646, 1.8814, + 1.4746, -0.5714, 0.0000, + }; + break; + } + + double offset; + + switch (colorSpace.range) { + case libcamera::ColorSpace::Range::Full: + offset = 0.0; + break; + + case libcamera::ColorSpace::Range::Limited: + offset = 16.0; + + for (unsigned int i = 0; i < 3; ++i) + yuv2rgb[i] *= 255.0 / 219.0; + for (unsigned int i = 4; i < 9; ++i) + yuv2rgb[i] *= 255.0 / 224.0; + break; + } + + QStringList matrix; + + for (double coeff : yuv2rgb) + matrix.append(QString::number(coeff, 'f')); + + fragmentShaderDefines_.append("#define YUV2RGB_MATRIX " + matrix.join(", ")); + fragmentShaderDefines_.append(QString("#define YUV2RGB_Y_OFFSET %1") + .arg(offset, 0, 'f', 1)); +} + bool ViewFinderGL::createVertexShader() { /* Create Vertex Shader */ diff --git a/src/qcam/viewfinder_gl.h b/src/qcam/viewfinder_gl.h index 798830a31cd2..68c2912df12f 100644 --- a/src/qcam/viewfinder_gl.h +++ b/src/qcam/viewfinder_gl.h @@ -57,6 +57,7 @@ protected: private: bool selectFormat(const libcamera::PixelFormat &format); + void selectColorSpace(const libcamera::ColorSpace &colorSpace); void configureTexture(QOpenGLTexture &texture); bool createFragmentShader(); @@ -67,6 +68,7 @@ private: /* Captured image size, format and buffer */ libcamera::FrameBuffer *buffer_; libcamera::PixelFormat format_; + libcamera::ColorSpace colorSpace_; QSize size_; unsigned int stride_; Image *image_;