Patch Detail
Show a patch.
GET /api/1.1/patches/17225/?format=api
{ "id": 17225, "url": "https://patchwork.libcamera.org/api/1.1/patches/17225/?format=api", "web_url": "https://patchwork.libcamera.org/patch/17225/", "project": { "id": 1, "url": "https://patchwork.libcamera.org/api/1.1/projects/1/?format=api", "name": "libcamera", "link_name": "libcamera", "list_id": "libcamera_core", "list_email": "libcamera-devel@lists.libcamera.org", "web_url": "", "scm_url": "", "webscm_url": "" }, "msgid": "<20220829100414.28404-4-laurent.pinchart@ideasonboard.com>", "date": "2022-08-29T10:04:14", "name": "[libcamera-devel,3/3] qcam: viewfinder_gl: Take color space into account for YUV rendering", "commit_ref": null, "pull_url": null, "state": "accepted", "archived": false, "hash": "d72a7b9f4058faa38d89cb4f80469e973cdc4ab2", "submitter": { "id": 2, "url": "https://patchwork.libcamera.org/api/1.1/people/2/?format=api", "name": "Laurent Pinchart", "email": "laurent.pinchart@ideasonboard.com" }, "delegate": null, "mbox": "https://patchwork.libcamera.org/patch/17225/mbox/", "series": [ { "id": 3445, "url": "https://patchwork.libcamera.org/api/1.1/series/3445/?format=api", "web_url": "https://patchwork.libcamera.org/project/libcamera/list/?series=3445", "date": "2022-08-29T10:04:11", "name": "Misc color space plumbing improvements", "version": 1, "mbox": "https://patchwork.libcamera.org/series/3445/mbox/" } ], "comments": "https://patchwork.libcamera.org/api/patches/17225/comments/", "check": "pending", "checks": "https://patchwork.libcamera.org/api/patches/17225/checks/", "tags": {}, "headers": { "Return-Path": "<libcamera-devel-bounces@lists.libcamera.org>", "X-Original-To": "parsemail@patchwork.libcamera.org", "Delivered-To": "parsemail@patchwork.libcamera.org", "Received": [ "from lancelot.ideasonboard.com (lancelot.ideasonboard.com\n\t[92.243.16.209])\n\tby patchwork.libcamera.org (Postfix) with ESMTPS id 95D13C3272\n\tfor <parsemail@patchwork.libcamera.org>;\n\tMon, 29 Aug 2022 10:04:32 +0000 (UTC)", "from lancelot.ideasonboard.com (localhost [IPv6:::1])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTP id 4F45E61FBE;\n\tMon, 29 Aug 2022 12:04:32 +0200 (CEST)", "from perceval.ideasonboard.com (perceval.ideasonboard.com\n\t[213.167.242.64])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id 932A461FBE\n\tfor <libcamera-devel@lists.libcamera.org>;\n\tMon, 29 Aug 2022 12:04:30 +0200 (CEST)", "from pendragon.ideasonboard.com (62-78-145-57.bb.dnainternet.fi\n\t[62.78.145.57])\n\tby perceval.ideasonboard.com (Postfix) with ESMTPSA id B82B58BD\n\tfor <libcamera-devel@lists.libcamera.org>;\n\tMon, 29 Aug 2022 12:04:29 +0200 (CEST)" ], "DKIM-Signature": [ "v=1; a=rsa-sha256; c=relaxed/simple; d=libcamera.org;\n\ts=mail; t=1661767472;\n\tbh=6sbzgQdntHRRZ4KlQmYYZ41mnQ+VTWs7kF/CEKL6a4M=;\n\th=To:Date:In-Reply-To:References:Subject:List-Id:List-Unsubscribe:\n\tList-Archive:List-Post:List-Help:List-Subscribe:From:Reply-To:\n\tFrom;\n\tb=lM+U05l0oCHRDZQQySTXmgYFB+eyn3IANuouVhgkKuAx+GlZrhvZp2oJsJEIctte6\n\tSqiMPuqZdLZdves8ANKopulZNB8abA/Et8tG+LZELES+7BeGGmu8AAasm1Nu9gKwKl\n\tbNWKVzfqmLe51aHaIl3eKaKORwp7fE5CJL4ckyGr5daeOnsJ/yBxALWiWigC9ivULf\n\tPpM18T6ZT/Vma7KD+/mX0u7N+iPSYeM2WSxFntC5lXYdh5WEo90dP28AblOnWromPT\n\tA2mZRpIZoNeBwIefByoCxns8zGsHlDM+Poa4hS6ZRHmh0IpSBWl0ScYi0xq64D4w7m\n\twvd/BPiNJLpxg==", "v=1; a=rsa-sha256; c=relaxed/simple; d=ideasonboard.com;\n\ts=mail; t=1661767470;\n\tbh=6sbzgQdntHRRZ4KlQmYYZ41mnQ+VTWs7kF/CEKL6a4M=;\n\th=From:To:Subject:Date:In-Reply-To:References:From;\n\tb=W4abnQlcu1Vx9iMAE6DwwdLFq8oN7u3tkZ8LWHf8jKzLrJ5VDHKpCZQp7Vc2/WtUa\n\tVl2NWlBXO9+NQJmCL467tpPsFdrxfuaYYOAZ0mE/Ea3QTEE2HKaQF/gp0gf7IMoCvg\n\tO7ms//fWgJNrriag66Of5f22k7J21lzX0PwrK3ek=" ], "Authentication-Results": "lancelot.ideasonboard.com; dkim=pass (1024-bit key; \n\tunprotected) header.d=ideasonboard.com\n\theader.i=@ideasonboard.com\n\theader.b=\"W4abnQlc\"; dkim-atps=neutral", "To": "libcamera-devel@lists.libcamera.org", "Date": "Mon, 29 Aug 2022 13:04:14 +0300", "Message-Id": "<20220829100414.28404-4-laurent.pinchart@ideasonboard.com>", "X-Mailer": "git-send-email 2.35.1", "In-Reply-To": "<20220829100414.28404-1-laurent.pinchart@ideasonboard.com>", "References": "<20220829100414.28404-1-laurent.pinchart@ideasonboard.com>", "MIME-Version": "1.0", "Content-Transfer-Encoding": "8bit", "Subject": "[libcamera-devel] [PATCH 3/3] qcam: viewfinder_gl: Take color space\n\tinto account for YUV rendering", "X-BeenThere": "libcamera-devel@lists.libcamera.org", "X-Mailman-Version": "2.1.29", "Precedence": "list", "List-Id": "<libcamera-devel.lists.libcamera.org>", "List-Unsubscribe": "<https://lists.libcamera.org/options/libcamera-devel>,\n\t<mailto:libcamera-devel-request@lists.libcamera.org?subject=unsubscribe>", "List-Archive": "<https://lists.libcamera.org/pipermail/libcamera-devel/>", "List-Post": "<mailto:libcamera-devel@lists.libcamera.org>", "List-Help": "<mailto:libcamera-devel-request@lists.libcamera.org?subject=help>", "List-Subscribe": "<https://lists.libcamera.org/listinfo/libcamera-devel>,\n\t<mailto:libcamera-devel-request@lists.libcamera.org?subject=subscribe>", "From": "Laurent Pinchart via libcamera-devel\n\t<libcamera-devel@lists.libcamera.org>", "Reply-To": "Laurent Pinchart <laurent.pinchart@ideasonboard.com>", "Errors-To": "libcamera-devel-bounces@lists.libcamera.org", "Sender": "\"libcamera-devel\" <libcamera-devel-bounces@lists.libcamera.org>" }, "content": "Update the YUV shaders and the viewfinder_gl to correctly take the\nY'CbCr encoding and the quantization range into account when rendering\nYUV formats to RGB. Support for the primaries and transfer function will\nbe added in a subsequent step.\n\nSigned-off-by: Laurent Pinchart <laurent.pinchart@ideasonboard.com>\n---\n src/qcam/assets/shader/YUV_2_planes.frag | 27 ++++----\n src/qcam/assets/shader/YUV_3_planes.frag | 23 ++++---\n src/qcam/assets/shader/YUV_packed.frag | 17 ++---\n src/qcam/viewfinder_gl.cpp | 79 +++++++++++++++++++++++-\n src/qcam/viewfinder_gl.h | 2 +\n 5 files changed, 115 insertions(+), 33 deletions(-)", "diff": "diff --git a/src/qcam/assets/shader/YUV_2_planes.frag b/src/qcam/assets/shader/YUV_2_planes.frag\nindex 254463c05cac..da8dbcc5f801 100644\n--- a/src/qcam/assets/shader/YUV_2_planes.frag\n+++ b/src/qcam/assets/shader/YUV_2_planes.frag\n@@ -13,27 +13,30 @@ varying vec2 textureOut;\n uniform sampler2D tex_y;\n uniform sampler2D tex_u;\n \n+const mat3 yuv2rgb_matrix = mat3(\n+\tYUV2RGB_MATRIX\n+);\n+\n+const vec3 yuv2rgb_offset = vec3(\n+\tYUV2RGB_Y_OFFSET / 255.0, 128.0 / 255.0, 128.0 / 255.0\n+);\n+\n void main(void)\n {\n \tvec3 yuv;\n-\tvec3 rgb;\n-\tmat3 yuv2rgb_bt601_mat = mat3(\n-\t\tvec3(1.164, 1.164, 1.164),\n-\t\tvec3(0.000, -0.392, 2.017),\n-\t\tvec3(1.596, -0.813, 0.000)\n-\t);\n \n-\tyuv.x = texture2D(tex_y, textureOut).r - 0.063;\n+\tyuv.x = texture2D(tex_y, textureOut).r;\n #if defined(YUV_PATTERN_UV)\n-\tyuv.y = texture2D(tex_u, textureOut).r - 0.500;\n-\tyuv.z = texture2D(tex_u, textureOut).a - 0.500;\n+\tyuv.y = texture2D(tex_u, textureOut).r;\n+\tyuv.z = texture2D(tex_u, textureOut).a;\n #elif defined(YUV_PATTERN_VU)\n-\tyuv.y = texture2D(tex_u, textureOut).a - 0.500;\n-\tyuv.z = texture2D(tex_u, textureOut).r - 0.500;\n+\tyuv.y = texture2D(tex_u, textureOut).a;\n+\tyuv.z = texture2D(tex_u, textureOut).r;\n #else\n #error Invalid pattern\n #endif\n \n-\trgb = yuv2rgb_bt601_mat * yuv;\n+\tvec3 rgb = yuv2rgb_matrix * (vec3(y, uv) - yuv2rgb_offset);\n+\n \tgl_FragColor = vec4(rgb, 1.0);\n }\ndiff --git a/src/qcam/assets/shader/YUV_3_planes.frag b/src/qcam/assets/shader/YUV_3_planes.frag\nindex 2be74b5d2a9d..e754129d74d1 100644\n--- a/src/qcam/assets/shader/YUV_3_planes.frag\n+++ b/src/qcam/assets/shader/YUV_3_planes.frag\n@@ -14,20 +14,23 @@ uniform sampler2D tex_y;\n uniform sampler2D tex_u;\n uniform sampler2D tex_v;\n \n+const mat3 yuv2rgb_matrix = mat3(\n+\tYUV2RGB_MATRIX\n+);\n+\n+const vec3 yuv2rgb_offset = vec3(\n+\tYUV2RGB_Y_OFFSET / 255.0, 128.0 / 255.0, 128.0 / 255.0\n+);\n+\n void main(void)\n {\n \tvec3 yuv;\n-\tvec3 rgb;\n-\tmat3 yuv2rgb_bt601_mat = mat3(\n-\t\tvec3(1.164, 1.164, 1.164),\n-\t\tvec3(0.000, -0.392, 2.017),\n-\t\tvec3(1.596, -0.813, 0.000)\n-\t);\n \n-\tyuv.x = texture2D(tex_y, textureOut).r - 0.063;\n-\tyuv.y = texture2D(tex_u, textureOut).r - 0.500;\n-\tyuv.z = texture2D(tex_v, textureOut).r - 0.500;\n+\tyuv.x = texture2D(tex_y, textureOut).r;\n+\tyuv.y = texture2D(tex_u, textureOut).r;\n+\tyuv.z = texture2D(tex_v, textureOut).r;\n+\n+\tvec3 rgb = yuv2rgb_matrix * (vec3(y, uv) - yuv2rgb_offset);\n \n-\trgb = yuv2rgb_bt601_mat * yuv;\n \tgl_FragColor = vec4(rgb, 1.0);\n }\ndiff --git a/src/qcam/assets/shader/YUV_packed.frag b/src/qcam/assets/shader/YUV_packed.frag\nindex d6efd4ce92a9..b9ef9d41beae 100644\n--- a/src/qcam/assets/shader/YUV_packed.frag\n+++ b/src/qcam/assets/shader/YUV_packed.frag\n@@ -14,15 +14,16 @@ varying vec2 textureOut;\n uniform sampler2D tex_y;\n uniform vec2 tex_step;\n \n+const mat3 yuv2rgb_matrix = mat3(\n+\tYUV2RGB_MATRIX\n+);\n+\n+const vec3 yuv2rgb_offset = vec3(\n+\tYUV2RGB_Y_OFFSET / 255.0, 128.0 / 255.0, 128.0 / 255.0\n+);\n+\n void main(void)\n {\n-\tmat3 yuv2rgb_bt601_mat = mat3(\n-\t\tvec3(1.164, 1.164, 1.164),\n-\t\tvec3(0.000, -0.392, 2.017),\n-\t\tvec3(1.596, -0.813, 0.000)\n-\t);\n-\tvec3 yuv2rgb_bt601_offset = vec3(0.063, 0.500, 0.500);\n-\n \t/*\n \t * The sampler won't interpolate the texture correctly along the X axis,\n \t * as each RGBA pixel effectively stores two pixels. We thus need to\n@@ -76,7 +77,7 @@ void main(void)\n \n \tfloat y = mix(y_left, y_right, step(0.5, f_x));\n \n-\tvec3 rgb = yuv2rgb_bt601_mat * (vec3(y, uv) - yuv2rgb_bt601_offset);\n+\tvec3 rgb = yuv2rgb_matrix * (vec3(y, uv) - yuv2rgb_offset);\n \n \tgl_FragColor = vec4(rgb, 1.0);\n }\ndiff --git a/src/qcam/viewfinder_gl.cpp b/src/qcam/viewfinder_gl.cpp\nindex ec295b6de0dd..e2aa24703ff0 100644\n--- a/src/qcam/viewfinder_gl.cpp\n+++ b/src/qcam/viewfinder_gl.cpp\n@@ -7,9 +7,12 @@\n \n #include \"viewfinder_gl.h\"\n \n+#include <array>\n+\n #include <QByteArray>\n #include <QFile>\n #include <QImage>\n+#include <QStringList>\n \n #include <libcamera/formats.h>\n \n@@ -56,7 +59,8 @@ static const QList<libcamera::PixelFormat> supportedFormats{\n };\n \n ViewFinderGL::ViewFinderGL(QWidget *parent)\n-\t: QOpenGLWidget(parent), buffer_(nullptr), image_(nullptr),\n+\t: QOpenGLWidget(parent), buffer_(nullptr),\n+\t colorSpace_(libcamera::ColorSpace::Raw), image_(nullptr),\n \t vertexBuffer_(QOpenGLBuffer::VertexBuffer)\n {\n }\n@@ -72,10 +76,10 @@ const QList<libcamera::PixelFormat> &ViewFinderGL::nativeFormats() const\n }\n \n int ViewFinderGL::setFormat(const libcamera::PixelFormat &format, const QSize &size,\n-\t\t\t [[maybe_unused]] const libcamera::ColorSpace &colorSpace,\n+\t\t\t const libcamera::ColorSpace &colorSpace,\n \t\t\t unsigned int stride)\n {\n-\tif (format != format_) {\n+\tif (format != format_ || colorSpace != colorSpace_) {\n \t\t/*\n \t\t * If the fragment already exists, remove it and create a new\n \t\t * one for the new format.\n@@ -89,7 +93,10 @@ int ViewFinderGL::setFormat(const libcamera::PixelFormat &format, const QSize &s\n \t\tif (!selectFormat(format))\n \t\t\treturn -1;\n \n+\t\tselectColorSpace(colorSpace);\n+\n \t\tformat_ = format;\n+\t\tcolorSpace_ = colorSpace;\n \t}\n \n \tsize_ = size;\n@@ -318,6 +325,72 @@ bool ViewFinderGL::selectFormat(const libcamera::PixelFormat &format)\n \treturn ret;\n }\n \n+void ViewFinderGL::selectColorSpace(const libcamera::ColorSpace &colorSpace)\n+{\n+\tstd::array<double, 9> yuv2rgb;\n+\n+\t/* OpenGL stores arrays in column-major order. */\n+\tswitch (colorSpace.ycbcrEncoding) {\n+\tcase libcamera::ColorSpace::YcbcrEncoding::None:\n+\t\tyuv2rgb = {\n+\t\t\t1.0000, 0.0000, 0.0000,\n+\t\t\t0.0000, 1.0000, 0.0000,\n+\t\t\t0.0000, 0.0000, 1.0000,\n+\t\t};\n+\t\tbreak;\n+\n+\tcase libcamera::ColorSpace::YcbcrEncoding::Rec601:\n+\t\tyuv2rgb = {\n+\t\t\t1.0000, 1.0000, 1.0000,\n+\t\t\t0.0000, -0.3441, 1.7720,\n+\t\t\t1.4020, -0.7141, 0.0000,\n+\t\t};\n+\t\tbreak;\n+\n+\tcase libcamera::ColorSpace::YcbcrEncoding::Rec709:\n+\t\tyuv2rgb = {\n+\t\t\t1.0000, 1.0000, 1.0000,\n+\t\t\t0.0000, -0.1873, 1.8856,\n+\t\t\t1.5748, -0.4681, 0.0000,\n+\t\t};\n+\t\tbreak;\n+\n+\tcase libcamera::ColorSpace::YcbcrEncoding::Rec2020:\n+\t\tyuv2rgb = {\n+\t\t\t1.0000, 1.0000, 1.0000,\n+\t\t\t0.0000, -0.1646, 1.8814,\n+\t\t\t1.4746, -0.5714, 0.0000,\n+\t\t};\n+\t\tbreak;\n+\t}\n+\n+\tdouble offset;\n+\n+\tswitch (colorSpace.range) {\n+\tcase libcamera::ColorSpace::Range::Full:\n+\t\toffset = 0.0;\n+\t\tbreak;\n+\n+\tcase libcamera::ColorSpace::Range::Limited:\n+\t\toffset = 16.0;\n+\n+\t\tfor (unsigned int i = 0; i < 3; ++i)\n+\t\t\tyuv2rgb[i] *= 255.0 / 219.0;\n+\t\tfor (unsigned int i = 4; i < 9; ++i)\n+\t\t\tyuv2rgb[i] *= 255.0 / 224.0;\n+\t\tbreak;\n+\t}\n+\n+\tQStringList matrix;\n+\n+\tfor (double coeff : yuv2rgb)\n+\t\tmatrix.append(QString::number(coeff, 'f'));\n+\n+\tfragmentShaderDefines_.append(\"#define YUV2RGB_MATRIX \" + matrix.join(\", \"));\n+\tfragmentShaderDefines_.append(QString(\"#define YUV2RGB_Y_OFFSET %1\")\n+\t\t.arg(offset, 0, 'f', 1));\n+}\n+\n bool ViewFinderGL::createVertexShader()\n {\n \t/* Create Vertex Shader */\ndiff --git a/src/qcam/viewfinder_gl.h b/src/qcam/viewfinder_gl.h\nindex 798830a31cd2..68c2912df12f 100644\n--- a/src/qcam/viewfinder_gl.h\n+++ b/src/qcam/viewfinder_gl.h\n@@ -57,6 +57,7 @@ protected:\n \n private:\n \tbool selectFormat(const libcamera::PixelFormat &format);\n+\tvoid selectColorSpace(const libcamera::ColorSpace &colorSpace);\n \n \tvoid configureTexture(QOpenGLTexture &texture);\n \tbool createFragmentShader();\n@@ -67,6 +68,7 @@ private:\n \t/* Captured image size, format and buffer */\n \tlibcamera::FrameBuffer *buffer_;\n \tlibcamera::PixelFormat format_;\n+\tlibcamera::ColorSpace colorSpace_;\n \tQSize size_;\n \tunsigned int stride_;\n \tImage *image_;\n", "prefixes": [ "libcamera-devel", "3/3" ] }