From patchwork Thu Feb 25 18:36:22 2021 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Andrey Konovalov X-Patchwork-Id: 11388 Return-Path: X-Original-To: parsemail@patchwork.libcamera.org Delivered-To: parsemail@patchwork.libcamera.org Received: from lancelot.ideasonboard.com (lancelot.ideasonboard.com [92.243.16.209]) by patchwork.libcamera.org (Postfix) with ESMTPS id B8B89BD1F1 for ; Thu, 25 Feb 2021 18:37:00 +0000 (UTC) Received: from lancelot.ideasonboard.com (localhost [IPv6:::1]) by lancelot.ideasonboard.com (Postfix) with ESMTP id 8843168A5D; Thu, 25 Feb 2021 19:37:00 +0100 (CET) Authentication-Results: lancelot.ideasonboard.com; dkim=fail reason="signature verification failed" (2048-bit key; unprotected) header.d=linaro.org header.i=@linaro.org header.b="MQETSXHw"; dkim-atps=neutral Received: from mail-lf1-x130.google.com (mail-lf1-x130.google.com [IPv6:2a00:1450:4864:20::130]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id 16AC168A59 for ; Thu, 25 Feb 2021 19:36:59 +0100 (CET) Received: by mail-lf1-x130.google.com with SMTP id u4so10095226lfs.0 for ; Thu, 25 Feb 2021 10:36:59 -0800 (PST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=linaro.org; s=google; h=from:to:cc:subject:date:message-id:in-reply-to:references; bh=olzp5+f2p8vLdXof0aF2IDvl6Y/vw19jdOA72WR6mWo=; b=MQETSXHwHgzMdIBAksHaiRXvf+ePh+8OtSux7W+IePaX1ODcMIkJpD8qWfMCUGQSLn k92jwqDck+G0ewKL+A09W/Fa7WKOq9UhQu8nQnwLBcTEUk9friDlE4PqaUZaBFHhRCvs wjuiqGFE8dFyYMC2B4cGFbbfB6YF2k/MbcHHqEFi0LpqBJITzkTRjb4spMTTTyR1goT9 eSokwU+jInttsHK+H18ohpqOqc09oFIqpxs11ThuD9xVCbiu6DLnLcGV9OZuzp3yAj6k ofs1NDepR/OSRBde2UYX7lgON2nIIK8hEUR+ZteiAm+QZThiruaBUOy+y4wV7oiHiT/O rnhA== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20161025; h=x-gm-message-state:from:to:cc:subject:date:message-id:in-reply-to :references; bh=olzp5+f2p8vLdXof0aF2IDvl6Y/vw19jdOA72WR6mWo=; b=KFknzOhpi/HhKdzvuKotXrfuiMagOWzNoBEO/FlExuYiXB4Dnt3bzxu0WDxH+8N5kh cN0El0MvZoQQOmu8sF4lclq6NITKOZApzW+cRou12iCibF5tL64NtqbpMGYMCYGmOFZr wYeWprzX4YpZhW1j+fQgWmz67TeCl1yNo29Vi9xxVy+9olNdechIjadOXeGBAWV7TAQx sOyQhesj6u+JUM1t+LBYPCJZM0dqL//R4YWQVUURngpiqspQjkLf4dv6JvcarAArf5g8 Yw1qL3oVxPiXemIjgBd4RUhUH06VHqahC1xOYNqKJVj2MwCBZZ8EnRdVp8LLYhkeI0jQ AeEQ== X-Gm-Message-State: AOAM532jYoIZ6WhYIF59bh/FVW3OIytrDSiw8GoTuXoY23vTK9DYn2Pa DY0Lqlr2I+B295TBnuxV3zG4Ha8c8cSEbg== X-Google-Smtp-Source: ABdhPJwY/OHs+uajDSmRYX5jny9OQ7rltrRc7q6pII2I9Tlqs/hlUA/m5cwT8J3wN13BWuFkRpQmJQ== X-Received: by 2002:a05:6512:34c8:: with SMTP id w8mr2714772lfr.537.1614278218615; Thu, 25 Feb 2021 10:36:58 -0800 (PST) Received: from localhost.localdomain ([85.249.43.69]) by smtp.googlemail.com with ESMTPSA id x1sm1136772ljh.62.2021.02.25.10.36.57 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Thu, 25 Feb 2021 10:36:58 -0800 (PST) From: Andrey Konovalov To: libcamera-devel@lists.libcamera.org, laurent.pinchart@ideasonboard.com Date: Thu, 25 Feb 2021 21:36:22 +0300 Message-Id: <20210225183626.28288-2-andrey.konovalov@linaro.org> X-Mailer: git-send-email 2.17.1 In-Reply-To: <20210225183626.28288-1-andrey.konovalov@linaro.org> References: <20210225183626.28288-1-andrey.konovalov@linaro.org> Subject: [libcamera-devel] [PATCH 1/5] qcam: viewfinder_gl: change uniform float tex_stepx to vec2 tex_step X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Cc: morgan@casual-effects.com MIME-Version: 1.0 Errors-To: libcamera-devel-bounces@lists.libcamera.org Sender: "libcamera-devel" In preparation to extend the supported formats, extend the tex_stepx uniform to cover the steps between texels in both horizontal and vertical directions. Signed-off-by: Andrey Konovalov --- src/qcam/assets/shader/YUV_packed.frag | 8 ++++---- src/qcam/viewfinder_gl.cpp | 7 ++++--- src/qcam/viewfinder_gl.h | 2 +- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/src/qcam/assets/shader/YUV_packed.frag b/src/qcam/assets/shader/YUV_packed.frag index 224dfafe..d6efd4ce 100644 --- a/src/qcam/assets/shader/YUV_packed.frag +++ b/src/qcam/assets/shader/YUV_packed.frag @@ -12,7 +12,7 @@ precision mediump float; varying vec2 textureOut; uniform sampler2D tex_y; -uniform float tex_stepx; +uniform vec2 tex_step; void main(void) { @@ -49,10 +49,10 @@ void main(void) * a = fract(x) * 2 - 1 if fract(x) >= 0.5 */ vec2 pos = textureOut; - float f_x = fract(pos.x / tex_stepx); + float f_x = fract(pos.x / tex_step.x); - vec4 left = texture2D(tex_y, vec2(pos.x - f_x * tex_stepx, pos.y)); - vec4 right = texture2D(tex_y, vec2(pos.x + (1.0 - f_x) * tex_stepx , pos.y)); + vec4 left = texture2D(tex_y, vec2(pos.x - f_x * tex_step.x, pos.y)); + vec4 right = texture2D(tex_y, vec2(pos.x + (1.0 - f_x) * tex_step.x , pos.y)); #if defined(YUV_PATTERN_UYVY) float y_left = mix(left.g, left.a, f_x * 2.0); diff --git a/src/qcam/viewfinder_gl.cpp b/src/qcam/viewfinder_gl.cpp index 5d9b442e..ff719418 100644 --- a/src/qcam/viewfinder_gl.cpp +++ b/src/qcam/viewfinder_gl.cpp @@ -289,7 +289,7 @@ bool ViewFinderGL::createFragmentShader() textureUniformY_ = shaderProgram_.uniformLocation("tex_y"); textureUniformU_ = shaderProgram_.uniformLocation("tex_u"); textureUniformV_ = shaderProgram_.uniformLocation("tex_v"); - textureUniformStepX_ = shaderProgram_.uniformLocation("tex_stepx"); + textureUniformStep_ = shaderProgram_.uniformLocation("tex_step"); /* Create the textures. */ for (std::unique_ptr &texture : textures_) { @@ -508,8 +508,9 @@ void ViewFinderGL::doRender() * ([0, 1]). There are exactly width - 1 steps between the * leftmost and rightmost texels. */ - shaderProgram_.setUniformValue(textureUniformStepX_, - 1.0f / (size_.width() / 2 - 1)); + shaderProgram_.setUniformValue(textureUniformStep_, + 1.0f / (size_.width() / 2 - 1), + 1.0f /* not used */); break; case libcamera::formats::ABGR8888: diff --git a/src/qcam/viewfinder_gl.h b/src/qcam/viewfinder_gl.h index 150fa4ae..1b1faa91 100644 --- a/src/qcam/viewfinder_gl.h +++ b/src/qcam/viewfinder_gl.h @@ -85,7 +85,7 @@ private: GLuint textureUniformU_; GLuint textureUniformV_; GLuint textureUniformY_; - GLuint textureUniformStepX_; + GLuint textureUniformStep_; unsigned int horzSubSample_; unsigned int vertSubSample_; From patchwork Thu Feb 25 18:36:23 2021 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Andrey Konovalov X-Patchwork-Id: 11389 Return-Path: X-Original-To: parsemail@patchwork.libcamera.org Delivered-To: parsemail@patchwork.libcamera.org Received: from lancelot.ideasonboard.com (lancelot.ideasonboard.com [92.243.16.209]) by patchwork.libcamera.org (Postfix) with ESMTPS id 20AB1BD1F1 for ; Thu, 25 Feb 2021 18:37:05 +0000 (UTC) Received: from lancelot.ideasonboard.com (localhost [IPv6:::1]) by lancelot.ideasonboard.com (Postfix) with ESMTP id E4E2368A5F; Thu, 25 Feb 2021 19:37:04 +0100 (CET) Authentication-Results: lancelot.ideasonboard.com; dkim=fail reason="signature verification failed" (2048-bit key; unprotected) header.d=linaro.org header.i=@linaro.org header.b="ueqoIbQ2"; dkim-atps=neutral Received: from mail-lf1-x136.google.com (mail-lf1-x136.google.com [IPv6:2a00:1450:4864:20::136]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id 0DA3468A59 for ; Thu, 25 Feb 2021 19:37:03 +0100 (CET) Received: by mail-lf1-x136.google.com with SMTP id u4so10095535lfs.0 for ; Thu, 25 Feb 2021 10:37:03 -0800 (PST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=linaro.org; s=google; h=from:to:cc:subject:date:message-id:in-reply-to:references; bh=BpmDhIQhpX7hcjrbtD7qJnZrzb7EHJzsq72n+mHUsG4=; b=ueqoIbQ2AMKbtqCJ089iiAqVzSoiEaoTtrTKHpYrmUqMN1os/SzheY54aZf0/maMvX PpYSzAbF68g8BDiJm87RdTWlnPK1dcgvQis6+DvMtFOCXRXMTWWHSC6DVmTFQYLs2Q84 3/8HPujT6eOBwcFf0XOgMkBF/bdfaplXL8ul8EHMFIY+Tf80UkoEXsGAQ07PutFqAeD2 H8lHiJZQk3c1PJEJvFzQAa4vAheIbAIdhL/MdP1C45APaIUB3lICjjoSyqS4HgMk7H1h KET3UjFcLUpImJRJ3P/8vCTFvBQcdJgtbzWAedcMxVhmBtGgMUPjQSWM1+EkyHU57u5M 90lg== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20161025; h=x-gm-message-state:from:to:cc:subject:date:message-id:in-reply-to :references; bh=BpmDhIQhpX7hcjrbtD7qJnZrzb7EHJzsq72n+mHUsG4=; b=hz9Feh/uj/Z2yes9dR75Fvj0p81G++35dtotMvQ2aontLRueqfQwN6zMU6sywDjV8p k7S+Co8BpqyKI5Rd6U7nwrj+h7N/EersyWsIElvRHiZu5+ZCquZi8Y7jKUKZtWbkHnIk yAKvMIxtXTey/1kh1k1+Qvk407mUjaNmicA42L+kLG1CCa1VVQqjUqTjf2byhatMvDrv 2g6WvEGRygHsCkWdoXDj0HH/7U7n5KqgqHDWT2DkE+E6t8dMr2ubw6I3Fek4kb5q93Xf MSjS0c99QiIjfUtQuwhcNk8BqIqzd3MLpjvKlR/Rhugb/o9hRAWzbj18YFdsRJu1GkfF Gcrg== X-Gm-Message-State: AOAM531dERXJ0TDOaeCqDqeAyBNlGzgcWyK3qEaCEByjSn1zE7B3/Hlm KcyAuFevAe9F7LjpLIXnXtp5yZ2dHyWopw== X-Google-Smtp-Source: ABdhPJw4m3d+V0IOYhCRKvClEf4gvYqeM6pHlACbUmq6na51c93zxsWfTy97r4QG+5tBZSndVMWJzQ== X-Received: by 2002:a19:e216:: with SMTP id z22mr2534061lfg.27.1614278222412; Thu, 25 Feb 2021 10:37:02 -0800 (PST) Received: from localhost.localdomain ([85.249.43.69]) by smtp.googlemail.com with ESMTPSA id x1sm1136772ljh.62.2021.02.25.10.37.00 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Thu, 25 Feb 2021 10:37:01 -0800 (PST) From: Andrey Konovalov To: libcamera-devel@lists.libcamera.org, laurent.pinchart@ideasonboard.com Date: Thu, 25 Feb 2021 21:36:23 +0300 Message-Id: <20210225183626.28288-3-andrey.konovalov@linaro.org> X-Mailer: git-send-email 2.17.1 In-Reply-To: <20210225183626.28288-1-andrey.konovalov@linaro.org> References: <20210225183626.28288-1-andrey.konovalov@linaro.org> Subject: [libcamera-devel] [PATCH 2/5] qcam: viewfinder_gl: Add shader to render packed RAW10 formats X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Cc: morgan@casual-effects.com MIME-Version: 1.0 Errors-To: libcamera-devel-bounces@lists.libcamera.org Sender: "libcamera-devel" The shader supports all 4 packed RAW10 variants. Simple bi-linear filtering is implemented. The 2 LS bits of the 10-bit colour values are dropped as the RGBA format we convert into has only 8 bits per colour. The texture coordinates passed to the fragment shader are ajusted to point to the nearest pixel in the image. This prevents artifacts when the image is scaled from the frame resolution to the window size. Signed-off-by: Andrey Konovalov --- src/qcam/assets/shader/bayer_1x_packed.frag | 174 ++++++++++++++++++++ src/qcam/assets/shader/shaders.qrc | 1 + src/qcam/viewfinder_gl.cpp | 77 ++++++++- src/qcam/viewfinder_gl.h | 6 + 4 files changed, 256 insertions(+), 2 deletions(-) create mode 100644 src/qcam/assets/shader/bayer_1x_packed.frag diff --git a/src/qcam/assets/shader/bayer_1x_packed.frag b/src/qcam/assets/shader/bayer_1x_packed.frag new file mode 100644 index 00000000..0a87c6db --- /dev/null +++ b/src/qcam/assets/shader/bayer_1x_packed.frag @@ -0,0 +1,174 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Based on the code from http://jgt.akpeters.com/papers/McGuire08/ + * + * Efficient, High-Quality Bayer Demosaic Filtering on GPUs + * + * Morgan McGuire + * + * This paper appears in issue Volume 13, Number 4. + * --------------------------------------------------------- + * Copyright (c) 2008, Morgan McGuire. All rights reserved. + * + * + * Modified by Linaro Ltd for 10/12-bit packed vs 8-bit raw Bayer format, + * and for simpler demosaic algorithm. + * Copyright (C) 2020, Linaro + * + * bayer_1x_packed.frag - Fragment shader code for raw Bayer 10-bit and 12-bit + * packed formats + */ + +#ifdef GL_ES +precision mediump float; +#endif + +varying vec2 textureOut; + +/* the texture size: tex_size.xy is in bytes, tex_size.zw is in pixels */ +uniform vec4 tex_size; +uniform vec2 tex_step; +uniform vec2 tex_bayer_first_red; + +uniform sampler2D tex_raw; + +void main(void) +{ + vec3 rgb; + + /* + * center.xy holds the coordinates of the pixel being sampled + * on the [0, 1] range. + * center.zw holds the coordinates of the pixel being sampled + * on the [0, width/height-1] range. + */ + vec4 center; + /* + * x-positions of the adjacent pixels on the [0, 1] range. + */ + vec2 xcoords; + /* + * y-positions of the adjacent pixels on the [0, 1] range. + */ + vec2 ycoords; + + /* + * The coordinates passed to the shader in textureOut may point + * to a place in between the pixels if the viewfinder window is scaled + * from the original captured frame size. Align them to the nearest + * pixel. + */ + center.zw = floor(textureOut * tex_size.zw); + center.y = center.w; + /* + * Add a small number (a few mantissa's LSBs) to avoid float + * representation issues. Maybe paranoic. + */ + center.x = BPP_X * center.z + 0.02; + + const float threshold_l = 0.127 /* fract(BPP_X) * 0.5 + 0.02 */; + const float threshold_h = 0.625 /* 1.0 - fract(BPP_X) * 1.5 */; + + float fract_x = fract(center.x); + /* + * The below floor() call ensures that center.x points + * at one of the bytes representing the 8 higher bits of + * the pixel value, not at the byte containing the LS bits + * of the group of the pixels. + */ + center.x = floor(center.x); + center.xy *= tex_step; + + xcoords = center.x + vec2(-tex_step.x, tex_step.x); + ycoords = center.y + vec2(-tex_step.y, tex_step.y); + /* + * If xcoords[0] points at the byte containing the LS bits + * of the previous group of the pixels, move xcoords[0] one + * byte back. + */ + xcoords[0] += (fract_x < threshold_l) ? -tex_step.x : 0.0; + /* + * If xcoords[1] points at the byte containing the LS bits + * of the current group of the pixels, move xcoords[1] one + * byte forward. + */ + xcoords[1] += (fract_x > threshold_h) ? tex_step.x : 0.0; + + vec2 alternate = mod(center.zw + tex_bayer_first_red, 2.0); + bool even_col = alternate.x < 1.0; + bool even_raw = alternate.y < 1.0; + + /* + * We need to sample the central pixel and the ones with offset + * of -1 to +1 pixel in both X and Y directions. Let's name these + * pixels as below, where C is the central pixel: + * +----+----+----+----+ + * | \ x| | | | + * |y \ | -1 | 0 | +1 | + * +----+----+----+----+ + * | +1 | D2 | A1 | D3 | + * +----+----+----+----+ + * | 0 | B0 | C | B1 | + * +----+----+----+----+ + * | -1 | D0 | A0 | D1 | + * +----+----+----+----+ + * In the below equations (0,-1).r means "r component of the texel + * shifted by -tex_step.y from the center.xy one" etc. + * In the even raw / even column (EE) case the colour values are: + * R = C = (0,0).r, + * G = (A0 + A1 + B0 + B1) / 4.0 = + * ( (0,-1).r + (0,1).r + (-1,0).r + (1,0).r ) / 4.0, + * B = (D0 + D1 + D2 + D3) / 4.0 = + * ( (-1,-1).r + (1,-1).r + (-1,1).r + (1,1).r ) / 4.0 + * For even raw / odd column (EO): + * R = (B0 + B1) / 2.0 = ( (-1,0).r + (1,0).r ) / 2.0, + * G = C = (0,0).r, + * B = (A0 + A1) / 2.0 = ( (0,-1).r + (0,1).r ) / 2.0 + * For odd raw / even column (OE): + * R = (A0 + A1) / 2.0 = ( (0,-1).r + (0,1).r ) / 2.0, + * G = C = (0,0).r, + * B = (B0 + B1) / 2.0 = ( (-1,0).r + (1,0).r ) / 2.0 + * For odd raw / odd column (OO): + * R = (D0 + D1 + D2 + D3) / 4.0 = + * ( (-1,-1).r + (1,-1).r + (-1,1).r + (1,1).r ) / 4.0, + * G = (A0 + A1 + B0 + B1) / 4.0 = + * ( (0,-1).r + (0,1).r + (-1,0).r + (1,0).r ) / 4.0, + * B = C = (0,0).r + */ + + /* + * Fetch the values and precalculate the terms: + * patterns.x = (A0 + A1) / 2.0 + * patterns.y = (B0 + B1) / 2.0 + * patterns.z = (A0 + A1 + B0 + B1) / 4.0 + * patterns.w = (D0 + D1 + D2 + D3) / 4.0 + */ + #define fetch(x, y) texture2D(tex_raw, vec2(x, y)).r + + float C = texture2D(tex_raw, center.xy).r; + vec4 patterns = vec4( + fetch(center.x, ycoords[0]), /* A0: (0,-1) */ + fetch(xcoords[0], center.y), /* B0: (-1,0) */ + fetch(xcoords[0], ycoords[0]), /* D0: (-1,-1) */ + fetch(xcoords[1], ycoords[0])); /* D1: (1,-1) */ + vec4 temp = vec4( + fetch(center.x, ycoords[1]), /* A1: (0,1) */ + fetch(xcoords[1], center.y), /* B1: (1,0) */ + fetch(xcoords[1], ycoords[1]), /* D3: (1,1) */ + fetch(xcoords[0], ycoords[1])); /* D2: (-1,1) */ + patterns = (patterns + temp) * 0.5; + /* .x = (A0 + A1) / 2.0, .y = (B0 + B1) / 2.0 */ + /* .z = (D0 + D3) / 2.0, .w = (D1 + D2) / 2.0 */ + patterns.w = (patterns.z + patterns.w) * 0.5; + patterns.z = (patterns.x + patterns.y) * 0.5; + + rgb = (even_col) ? + ((even_raw) ? + vec3(C, patterns.zw) : + vec3(patterns.x, C, patterns.y)) : + ((even_raw) ? + vec3(patterns.y, C, patterns.x) : + vec3(patterns.wz, C)); + + gl_FragColor = vec4(rgb, 1.0); +} diff --git a/src/qcam/assets/shader/shaders.qrc b/src/qcam/assets/shader/shaders.qrc index 8a8f9de1..d76d65c5 100644 --- a/src/qcam/assets/shader/shaders.qrc +++ b/src/qcam/assets/shader/shaders.qrc @@ -5,6 +5,7 @@ YUV_2_planes.frag YUV_3_planes.frag YUV_packed.frag + bayer_1x_packed.frag identity.vert diff --git a/src/qcam/viewfinder_gl.cpp b/src/qcam/viewfinder_gl.cpp index ff719418..b324d77f 100644 --- a/src/qcam/viewfinder_gl.cpp +++ b/src/qcam/viewfinder_gl.cpp @@ -36,6 +36,11 @@ static const QList supportedFormats{ libcamera::formats::RGBA8888, libcamera::formats::BGR888, libcamera::formats::RGB888, + /* Raw Bayer 10-bit packed */ + libcamera::formats::SBGGR10_CSI2P, + libcamera::formats::SGBRG10_CSI2P, + libcamera::formats::SGRBG10_CSI2P, + libcamera::formats::SRGGB10_CSI2P, }; ViewFinderGL::ViewFinderGL(QWidget *parent) @@ -114,6 +119,9 @@ bool ViewFinderGL::selectFormat(const libcamera::PixelFormat &format) { bool ret = true; + /* Set min/mag filters to GL_LINEAR by default. */ + textureMinMagFilters_ = GL_LINEAR; + fragmentShaderDefines_.clear(); switch (format) { @@ -203,6 +211,34 @@ bool ViewFinderGL::selectFormat(const libcamera::PixelFormat &format) fragmentShaderDefines_.append("#define RGB_PATTERN bgr"); fragmentShaderFile_ = ":RGB.frag"; break; + case libcamera::formats::SBGGR10_CSI2P: + firstRed_.setX(1.0); + firstRed_.setY(1.0); + fragmentShaderDefines_.append("#define BPP_X 1.25"); + fragmentShaderFile_ = ":bayer_1x_packed.frag"; + textureMinMagFilters_ = GL_NEAREST; + break; + case libcamera::formats::SGBRG10_CSI2P: + firstRed_.setX(0.0); + firstRed_.setY(1.0); + fragmentShaderDefines_.append("#define BPP_X 1.25"); + fragmentShaderFile_ = ":bayer_1x_packed.frag"; + textureMinMagFilters_ = GL_NEAREST; + break; + case libcamera::formats::SGRBG10_CSI2P: + firstRed_.setX(1.0); + firstRed_.setY(0.0); + fragmentShaderDefines_.append("#define BPP_X 1.25"); + fragmentShaderFile_ = ":bayer_1x_packed.frag"; + textureMinMagFilters_ = GL_NEAREST; + break; + case libcamera::formats::SRGGB10_CSI2P: + firstRed_.setX(0.0); + firstRed_.setY(0.0); + fragmentShaderDefines_.append("#define BPP_X 1.25"); + fragmentShaderFile_ = ":bayer_1x_packed.frag"; + textureMinMagFilters_ = GL_NEAREST; + break; default: ret = false; qWarning() << "[ViewFinderGL]:" @@ -290,6 +326,8 @@ bool ViewFinderGL::createFragmentShader() textureUniformU_ = shaderProgram_.uniformLocation("tex_u"); textureUniformV_ = shaderProgram_.uniformLocation("tex_v"); textureUniformStep_ = shaderProgram_.uniformLocation("tex_step"); + textureUniformSize_ = shaderProgram_.uniformLocation("tex_size"); + textureUniformBayerFirstRed_ = shaderProgram_.uniformLocation("tex_bayer_first_red"); /* Create the textures. */ for (std::unique_ptr &texture : textures_) { @@ -306,8 +344,10 @@ bool ViewFinderGL::createFragmentShader() void ViewFinderGL::configureTexture(QOpenGLTexture &texture) { glBindTexture(GL_TEXTURE_2D, texture.textureId()); - glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR); - glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, + textureMinMagFilters_); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, + textureMinMagFilters_); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); } @@ -547,6 +587,39 @@ void ViewFinderGL::doRender() shaderProgram_.setUniformValue(textureUniformY_, 0); break; + case libcamera::formats::SBGGR10_CSI2P: + case libcamera::formats::SGBRG10_CSI2P: + case libcamera::formats::SGRBG10_CSI2P: + case libcamera::formats::SRGGB10_CSI2P: + /* + * Packed raw Bayer 10-bit formats are stored in GL_RED texture. + * The texture width is 10/8 of the image width. + * TODO: account for padding bytes at the end of the line. + */ + glActiveTexture(GL_TEXTURE0); + configureTexture(*textures_[0]); + glTexImage2D(GL_TEXTURE_2D, + 0, + GL_RED, + size_.width() * 5 / 4, + size_.height(), + 0, + GL_RED, + GL_UNSIGNED_BYTE, + data_); + shaderProgram_.setUniformValue(textureUniformY_, 0); + shaderProgram_.setUniformValue(textureUniformBayerFirstRed_, + firstRed_); + shaderProgram_.setUniformValue(textureUniformSize_, + size_.width() * 5 / 4, + size_.height(), + size_.width(), + size_.height()); + shaderProgram_.setUniformValue(textureUniformStep_, + 1.0f / (size_.width() * 5 / 4 - 1), + 1.0f / (size_.height() - 1)); + break; + default: break; }; diff --git a/src/qcam/viewfinder_gl.h b/src/qcam/viewfinder_gl.h index 1b1faa91..337718e3 100644 --- a/src/qcam/viewfinder_gl.h +++ b/src/qcam/viewfinder_gl.h @@ -81,13 +81,19 @@ private: /* Textures */ std::array, 3> textures_; + /* Common texture parameters */ + GLuint textureMinMagFilters_; /* YUV texture parameters */ GLuint textureUniformU_; GLuint textureUniformV_; GLuint textureUniformY_; + GLuint textureUniformSize_; GLuint textureUniformStep_; unsigned int horzSubSample_; unsigned int vertSubSample_; + /* Raw Bayer texture parameters */ + GLuint textureUniformBayerFirstRed_; + QPointF firstRed_; QMutex mutex_; /* Prevent concurrent access to image_ */ }; From patchwork Thu Feb 25 18:36:24 2021 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Andrey Konovalov X-Patchwork-Id: 11390 Return-Path: X-Original-To: parsemail@patchwork.libcamera.org Delivered-To: parsemail@patchwork.libcamera.org Received: from lancelot.ideasonboard.com (lancelot.ideasonboard.com [92.243.16.209]) by patchwork.libcamera.org (Postfix) with ESMTPS id 8B99CBD1F1 for ; Thu, 25 Feb 2021 18:37:06 +0000 (UTC) Received: from lancelot.ideasonboard.com (localhost [IPv6:::1]) by lancelot.ideasonboard.com (Postfix) with ESMTP id 5630C68A64; Thu, 25 Feb 2021 19:37:06 +0100 (CET) Authentication-Results: lancelot.ideasonboard.com; dkim=fail reason="signature verification failed" (2048-bit key; unprotected) header.d=linaro.org header.i=@linaro.org header.b="hzigwtkI"; dkim-atps=neutral Received: from mail-lj1-x230.google.com (mail-lj1-x230.google.com [IPv6:2a00:1450:4864:20::230]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id A849F68A59 for ; Thu, 25 Feb 2021 19:37:05 +0100 (CET) Received: by mail-lj1-x230.google.com with SMTP id q23so7630775lji.8 for ; Thu, 25 Feb 2021 10:37:05 -0800 (PST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=linaro.org; s=google; h=from:to:cc:subject:date:message-id:in-reply-to:references; bh=veWN98fZcbakbj7dt4jyzAprw0las+6G3CgLk/aoTdY=; b=hzigwtkIor+w/Zn9D8e4f8twrCfppK2sv4k4ZXs9c9D8kZjcYh1TUhQeQIgOqsca1a GPjUlCTwkKh7d8AqAksxe1j9GUMd2cebf18oJY3wFxJYwGcmLDuOpVo38qp58+CEpEke WkEgx2E/Wr6m7pbI0NdG59387sukdkGFSmqAExXwY/lvAdn+49BRP05/YGSmCiLc6gbi 644CZz6X1DvyV+t7Nlu+cfZk9l/Ih/TsResEK85d+FL1MwBBchh0cYti/t2pVKVu7ATN /0/MrxCr5Mp4OcSiN8uaVOjIIosZj9cayvPaCMDqqVdl1cyRQw4YtcPcor6W84E9tsQD qXWg== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20161025; h=x-gm-message-state:from:to:cc:subject:date:message-id:in-reply-to :references; bh=veWN98fZcbakbj7dt4jyzAprw0las+6G3CgLk/aoTdY=; b=HtgTKOM/DgtWVx/eoYHYbmdaida6fNA34E1ljZblWYPipg0Sg1Snw86PGmne/zmgR1 A6rmApSrCrZazGGP0Wd0wh1NpQ4Dxo4gynQmhQibS6pJXoYFYluznQRpKUg/CIijRQ4m hoFAr+MKQghdXOzJ04brCJZW7RTin2ekSiBXcajMhY0PPX1SJL66pyuBXP6ZIwCBelI0 rbYHpBfM6q6xAfOQOfY2qMw1c8gFbFbjlGWz4/wT6tie/seqexUPNUuRk5JE2BQ8Pzyn JsLKK/Xjfy0qESBJxI/t8YxSnARE996uQudcbRreNLgJbteTdoUV4qNhQ+p0u0U7TyKg Jq0Q== X-Gm-Message-State: AOAM530a6cH088gwtoWcwDWaR1boUG4YdXFuBv+84XekyYd1b/ONMBVA yZCequ2kfYWdiENqSI/L74vyrJOZnxesXA== X-Google-Smtp-Source: ABdhPJy1CUwpfAic1BBQLg1WCsB4jLk4vC5TILTsrUue4WGXjpac+qz4j0j6BAeBlWil4UV7jmOywA== X-Received: by 2002:a2e:b4f9:: with SMTP id s25mr2275648ljm.49.1614278225055; Thu, 25 Feb 2021 10:37:05 -0800 (PST) Received: from localhost.localdomain ([85.249.43.69]) by smtp.googlemail.com with ESMTPSA id x1sm1136772ljh.62.2021.02.25.10.37.03 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Thu, 25 Feb 2021 10:37:04 -0800 (PST) From: Andrey Konovalov To: libcamera-devel@lists.libcamera.org, laurent.pinchart@ideasonboard.com Date: Thu, 25 Feb 2021 21:36:24 +0300 Message-Id: <20210225183626.28288-4-andrey.konovalov@linaro.org> X-Mailer: git-send-email 2.17.1 In-Reply-To: <20210225183626.28288-1-andrey.konovalov@linaro.org> References: <20210225183626.28288-1-andrey.konovalov@linaro.org> Subject: [libcamera-devel] [PATCH 3/5] qcam: viewfinder_gl: RAW10P: handle the padding bytes properly X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Cc: morgan@casual-effects.com MIME-Version: 1.0 Errors-To: libcamera-devel-bounces@lists.libcamera.org Sender: "libcamera-devel" The texture size must account for the padding bytes which may be present at the end of the lines in the frame. Signed-off-by: Andrey Konovalov --- src/qcam/viewfinder_gl.cpp | 11 +++++++---- src/qcam/viewfinder_gl.h | 1 + 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/qcam/viewfinder_gl.cpp b/src/qcam/viewfinder_gl.cpp index b324d77f..9db536a6 100644 --- a/src/qcam/viewfinder_gl.cpp +++ b/src/qcam/viewfinder_gl.cpp @@ -111,6 +111,10 @@ void ViewFinderGL::render(libcamera::FrameBuffer *buffer, MappedBuffer *map) renderComplete(buffer_); data_ = static_cast(map->memory); + /* + * \todo Get the stride from the buffer instead of computing it naively + */ + stride_ = buffer->metadata().planes[0].bytesused / size_.height(); update(); buffer_ = buffer; } @@ -594,14 +598,13 @@ void ViewFinderGL::doRender() /* * Packed raw Bayer 10-bit formats are stored in GL_RED texture. * The texture width is 10/8 of the image width. - * TODO: account for padding bytes at the end of the line. */ glActiveTexture(GL_TEXTURE0); configureTexture(*textures_[0]); glTexImage2D(GL_TEXTURE_2D, 0, GL_RED, - size_.width() * 5 / 4, + stride_, size_.height(), 0, GL_RED, @@ -611,12 +614,12 @@ void ViewFinderGL::doRender() shaderProgram_.setUniformValue(textureUniformBayerFirstRed_, firstRed_); shaderProgram_.setUniformValue(textureUniformSize_, - size_.width() * 5 / 4, + stride_, size_.height(), size_.width(), size_.height()); shaderProgram_.setUniformValue(textureUniformStep_, - 1.0f / (size_.width() * 5 / 4 - 1), + 1.0f / (stride_ - 1), 1.0f / (size_.height() - 1)); break; diff --git a/src/qcam/viewfinder_gl.h b/src/qcam/viewfinder_gl.h index 337718e3..49f8364a 100644 --- a/src/qcam/viewfinder_gl.h +++ b/src/qcam/viewfinder_gl.h @@ -66,6 +66,7 @@ private: libcamera::FrameBuffer *buffer_; libcamera::PixelFormat format_; QSize size_; + int stride_; unsigned char *data_; /* Shaders */ From patchwork Thu Feb 25 18:36:25 2021 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Andrey Konovalov X-Patchwork-Id: 11391 Return-Path: X-Original-To: parsemail@patchwork.libcamera.org Delivered-To: parsemail@patchwork.libcamera.org Received: from lancelot.ideasonboard.com (lancelot.ideasonboard.com [92.243.16.209]) by patchwork.libcamera.org (Postfix) with ESMTPS id 3B7BDBD1F1 for ; Thu, 25 Feb 2021 18:37:09 +0000 (UTC) Received: from lancelot.ideasonboard.com (localhost [IPv6:::1]) by lancelot.ideasonboard.com (Postfix) with ESMTP id 087AC68A5F; Thu, 25 Feb 2021 19:37:09 +0100 (CET) Authentication-Results: lancelot.ideasonboard.com; dkim=fail reason="signature verification failed" (2048-bit key; unprotected) header.d=linaro.org header.i=@linaro.org header.b="dJX/UQaj"; dkim-atps=neutral Received: from mail-lj1-x234.google.com (mail-lj1-x234.google.com [IPv6:2a00:1450:4864:20::234]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id 59D4768A5D for ; Thu, 25 Feb 2021 19:37:07 +0100 (CET) Received: by mail-lj1-x234.google.com with SMTP id q23so7630889lji.8 for ; Thu, 25 Feb 2021 10:37:07 -0800 (PST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=linaro.org; s=google; h=from:to:cc:subject:date:message-id:in-reply-to:references; bh=vfbBToVN87uQ5A/29HWO7lvbJuMk/InA3sQYu3SoI5Q=; b=dJX/UQajwz+N6fY3+nBa2OnEoMXSgkpeDSSII0FeVapTDJ06EGGMfXCEkjxWFi4DMr sMnS2SR2wMI4up2406GTH9Y/7t2V0DIIN/KwQNypKsiuBjrW3YprX79hqgalK61efIt2 NbL7L16TFSs56bmpHZSDUYeCeaBk7buS9MfQll40Z3EA0yrWApsFO+a5ISyvCyIQVpw8 VX3gMihRxHK9rLMi6muyyu1ReWFzZsnJOlwHw2qPYd8lQkHYj+6VS+t49cg9m8Ja1Ofw A1Nt3tSnedBMUGXO+1x0JOMlBAnza5DrM2X+HQJFH5uKIOcMW5jFVP5Prjz7PgDuFJJG eI7A== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20161025; h=x-gm-message-state:from:to:cc:subject:date:message-id:in-reply-to :references; bh=vfbBToVN87uQ5A/29HWO7lvbJuMk/InA3sQYu3SoI5Q=; b=RkNNhK6BT5bM5aGn3P+Mx51Zb4o8ZRP4g6bhrfoaemnnILRImksoRZPF96SbCohO2t X1Bh+sAyDzA8gT+AcX2M/AGltvSDyse1ZkbASlPFw+mPscr4T7sQ0fdcNQAhC7CGGOAX 0/F84p31dvdZikI21HwmRFp8LuVZad3UwADXLOd+sUQ22GC5hzR4fOxBQKzg1HYNPV4I dRvZVsox08ndvyPTC2MwNIr276hKC2annMMjc8IALi2zttwk9+YHOgH+5fbuAeFFL4En tchKnf45fZ8luqy0cM1vBH9jlJO2oMElHDHJLTM5oA+fyJ5p8cloihdQ0Q3zvp4iJ0Wb XMkw== X-Gm-Message-State: AOAM532sZUsBjUcGoxeW9+6ZO++Hh1xyf+gNUckLajMJR2RMq6SE08J0 dHDkPXzbEbB9MsYHmY+1UyF0TVM6Hsjdjg== X-Google-Smtp-Source: ABdhPJyu7BtnhdMXIW3ApS3p5Of6oneus3Yzy95h8l1YUNF0hLNAkwVsRDtrSswXza+9NKUDXjZd1w== X-Received: by 2002:a2e:8899:: with SMTP id k25mr2310397lji.502.1614278226839; Thu, 25 Feb 2021 10:37:06 -0800 (PST) Received: from localhost.localdomain ([85.249.43.69]) by smtp.googlemail.com with ESMTPSA id x1sm1136772ljh.62.2021.02.25.10.37.05 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Thu, 25 Feb 2021 10:37:06 -0800 (PST) From: Andrey Konovalov To: libcamera-devel@lists.libcamera.org, laurent.pinchart@ideasonboard.com Date: Thu, 25 Feb 2021 21:36:25 +0300 Message-Id: <20210225183626.28288-5-andrey.konovalov@linaro.org> X-Mailer: git-send-email 2.17.1 In-Reply-To: <20210225183626.28288-1-andrey.konovalov@linaro.org> References: <20210225183626.28288-1-andrey.konovalov@linaro.org> Subject: [libcamera-devel] [PATCH 4/5] qcam: viewfinder_gl: Add support for RAW12 packed formats X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Cc: morgan@casual-effects.com MIME-Version: 1.0 Errors-To: libcamera-devel-bounces@lists.libcamera.org Sender: "libcamera-devel" All the four Bayer orders are supported. The 4 LS bits of the 12-bit colour values are dropped as the RGBA format we convert into has only 8 bits per colour. Signed-off-by: Andrey Konovalov --- src/qcam/assets/shader/bayer_1x_packed.frag | 43 ++++++++++++++--- src/qcam/viewfinder_gl.cpp | 53 +++++++++++++++++---- 2 files changed, 82 insertions(+), 14 deletions(-) diff --git a/src/qcam/assets/shader/bayer_1x_packed.frag b/src/qcam/assets/shader/bayer_1x_packed.frag index 0a87c6db..d09c6fce 100644 --- a/src/qcam/assets/shader/bayer_1x_packed.frag +++ b/src/qcam/assets/shader/bayer_1x_packed.frag @@ -23,6 +23,40 @@ precision mediump float; #endif +/* + * These constants are used to select the bytes containing the HS part of + * the pixel value: + * BPP - bytes per pixel, + * THRESHOLD_L = fract(BPP) * 0.5 + 0.02 + * THRESHOLD_H = 1.0 - fract(BPP) * 1.5 + 0.02 + * Let X is the x coordinate in the texture measured in bytes (so that the + * range is from 0 to (stride_-1)) aligned on the nearest pixel. + * E.g. for RAW10P: + * -------------+-------------------+-------------------+-- + * pixel No | 0 1 2 3 | 4 5 6 7 | ... + * -------------+-------------------+-------------------+-- + * byte offset | 0 1 2 3 4 | 5 6 7 8 9 | ... + * -------------+-------------------+-------------------+-- + * X | 0.0 1.25 2.5 3.75 | 5.0 6.25 7.5 8.75 | ... + * -------------+-------------------+-------------------+-- + * If fract(X) < THRESHOLD_L then the previous byte contains the LS + * bits of the pixel values and needs to be skipped. + * If fract(X) > THRESHOLD_H then the next byte contains the LS bits + * of the pixel values and needs to be skipped. + */ +#if defined(RAW10P) +#define BPP 1.25 +#define THRESHOLD_L 0.14 +#define THRESHOLD_H 0.64 +#elif defined(RAW12P) +#define BPP 1.5 +#define THRESHOLD_L 0.27 +#define THRESHOLD_H 0.27 +#else +#error Invalid raw format +#endif + + varying vec2 textureOut; /* the texture size: tex_size.xy is in bytes, tex_size.zw is in pixels */ @@ -64,10 +98,7 @@ void main(void) * Add a small number (a few mantissa's LSBs) to avoid float * representation issues. Maybe paranoic. */ - center.x = BPP_X * center.z + 0.02; - - const float threshold_l = 0.127 /* fract(BPP_X) * 0.5 + 0.02 */; - const float threshold_h = 0.625 /* 1.0 - fract(BPP_X) * 1.5 */; + center.x = BPP * center.z + 0.02; float fract_x = fract(center.x); /* @@ -86,13 +117,13 @@ void main(void) * of the previous group of the pixels, move xcoords[0] one * byte back. */ - xcoords[0] += (fract_x < threshold_l) ? -tex_step.x : 0.0; + xcoords[0] += (fract_x < THRESHOLD_L) ? -tex_step.x : 0.0; /* * If xcoords[1] points at the byte containing the LS bits * of the current group of the pixels, move xcoords[1] one * byte forward. */ - xcoords[1] += (fract_x > threshold_h) ? tex_step.x : 0.0; + xcoords[1] += (fract_x > THRESHOLD_H) ? tex_step.x : 0.0; vec2 alternate = mod(center.zw + tex_bayer_first_red, 2.0); bool even_col = alternate.x < 1.0; diff --git a/src/qcam/viewfinder_gl.cpp b/src/qcam/viewfinder_gl.cpp index 9db536a6..13ab31aa 100644 --- a/src/qcam/viewfinder_gl.cpp +++ b/src/qcam/viewfinder_gl.cpp @@ -41,6 +41,11 @@ static const QList supportedFormats{ libcamera::formats::SGBRG10_CSI2P, libcamera::formats::SGRBG10_CSI2P, libcamera::formats::SRGGB10_CSI2P, + /* Raw Bayer 12-bit packed */ + libcamera::formats::SBGGR12_CSI2P, + libcamera::formats::SGBRG12_CSI2P, + libcamera::formats::SGRBG12_CSI2P, + libcamera::formats::SRGGB12_CSI2P, }; ViewFinderGL::ViewFinderGL(QWidget *parent) @@ -218,28 +223,56 @@ bool ViewFinderGL::selectFormat(const libcamera::PixelFormat &format) case libcamera::formats::SBGGR10_CSI2P: firstRed_.setX(1.0); firstRed_.setY(1.0); - fragmentShaderDefines_.append("#define BPP_X 1.25"); + fragmentShaderDefines_.append("#define RAW10P"); fragmentShaderFile_ = ":bayer_1x_packed.frag"; textureMinMagFilters_ = GL_NEAREST; break; case libcamera::formats::SGBRG10_CSI2P: firstRed_.setX(0.0); firstRed_.setY(1.0); - fragmentShaderDefines_.append("#define BPP_X 1.25"); + fragmentShaderDefines_.append("#define RAW10P"); fragmentShaderFile_ = ":bayer_1x_packed.frag"; textureMinMagFilters_ = GL_NEAREST; break; case libcamera::formats::SGRBG10_CSI2P: firstRed_.setX(1.0); firstRed_.setY(0.0); - fragmentShaderDefines_.append("#define BPP_X 1.25"); + fragmentShaderDefines_.append("#define RAW10P"); fragmentShaderFile_ = ":bayer_1x_packed.frag"; textureMinMagFilters_ = GL_NEAREST; break; case libcamera::formats::SRGGB10_CSI2P: firstRed_.setX(0.0); firstRed_.setY(0.0); - fragmentShaderDefines_.append("#define BPP_X 1.25"); + fragmentShaderDefines_.append("#define RAW10P"); + fragmentShaderFile_ = ":bayer_1x_packed.frag"; + textureMinMagFilters_ = GL_NEAREST; + break; + case libcamera::formats::SBGGR12_CSI2P: + firstRed_.setX(1.0); + firstRed_.setY(1.0); + fragmentShaderDefines_.append("#define RAW12P"); + fragmentShaderFile_ = ":bayer_1x_packed.frag"; + textureMinMagFilters_ = GL_NEAREST; + break; + case libcamera::formats::SGBRG12_CSI2P: + firstRed_.setX(0.0); + firstRed_.setY(1.0); + fragmentShaderDefines_.append("#define RAW12P"); + fragmentShaderFile_ = ":bayer_1x_packed.frag"; + textureMinMagFilters_ = GL_NEAREST; + break; + case libcamera::formats::SGRBG12_CSI2P: + firstRed_.setX(1.0); + firstRed_.setY(0.0); + fragmentShaderDefines_.append("#define RAW12P"); + fragmentShaderFile_ = ":bayer_1x_packed.frag"; + textureMinMagFilters_ = GL_NEAREST; + break; + case libcamera::formats::SRGGB12_CSI2P: + firstRed_.setX(0.0); + firstRed_.setY(0.0); + fragmentShaderDefines_.append("#define RAW12P"); fragmentShaderFile_ = ":bayer_1x_packed.frag"; textureMinMagFilters_ = GL_NEAREST; break; @@ -595,9 +628,13 @@ void ViewFinderGL::doRender() case libcamera::formats::SGBRG10_CSI2P: case libcamera::formats::SGRBG10_CSI2P: case libcamera::formats::SRGGB10_CSI2P: + case libcamera::formats::SBGGR12_CSI2P: + case libcamera::formats::SGBRG12_CSI2P: + case libcamera::formats::SGRBG12_CSI2P: + case libcamera::formats::SRGGB12_CSI2P: /* - * Packed raw Bayer 10-bit formats are stored in GL_RED texture. - * The texture width is 10/8 of the image width. + * Packed raw Bayer 10-bit and 12-bit formats are stored in + * GL_RED texture. */ glActiveTexture(GL_TEXTURE0); configureTexture(*textures_[0]); @@ -614,9 +651,9 @@ void ViewFinderGL::doRender() shaderProgram_.setUniformValue(textureUniformBayerFirstRed_, firstRed_); shaderProgram_.setUniformValue(textureUniformSize_, - stride_, + stride_, /* width in bytes */ size_.height(), - size_.width(), + size_.width(), /* in pixels */ size_.height()); shaderProgram_.setUniformValue(textureUniformStep_, 1.0f / (stride_ - 1), From patchwork Thu Feb 25 18:36:26 2021 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Andrey Konovalov X-Patchwork-Id: 11392 Return-Path: X-Original-To: parsemail@patchwork.libcamera.org Delivered-To: parsemail@patchwork.libcamera.org Received: from lancelot.ideasonboard.com (lancelot.ideasonboard.com [92.243.16.209]) by patchwork.libcamera.org (Postfix) with ESMTPS id A2F47BD1F1 for ; Thu, 25 Feb 2021 18:37:10 +0000 (UTC) Received: from lancelot.ideasonboard.com (localhost [IPv6:::1]) by lancelot.ideasonboard.com (Postfix) with ESMTP id 6FD4368A60; Thu, 25 Feb 2021 19:37:10 +0100 (CET) Authentication-Results: lancelot.ideasonboard.com; dkim=fail reason="signature verification failed" (2048-bit key; unprotected) header.d=linaro.org header.i=@linaro.org header.b="nhYhKAhJ"; dkim-atps=neutral Received: from mail-lf1-x12e.google.com (mail-lf1-x12e.google.com [IPv6:2a00:1450:4864:20::12e]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id A988468A51 for ; Thu, 25 Feb 2021 19:37:09 +0100 (CET) Received: by mail-lf1-x12e.google.com with SMTP id m22so10075477lfg.5 for ; Thu, 25 Feb 2021 10:37:09 -0800 (PST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=linaro.org; s=google; h=from:to:cc:subject:date:message-id:in-reply-to:references; bh=kzBnu6nrYfOM4w8lU7MNMsAhbObCzbJss9j9gDPnlqg=; b=nhYhKAhJLFTHS+Kd5vR7/2X114aAnO5n5R4qfzOfLtlaAMGqrVFTT26GulR2P7VbrK WQHg5qrl7Fz0Dbav1XmzKx2YByAe1SE35oO8xRRpM/hUudf4KSVh9w4YZkYS67Qtzq17 gx32C4s6p8nb75L1jIIOzW4/pLRy8aWmP7t0zJaejnIhGtrNbGbt10iSPUfF9mY4OUFE aU5dC1EzebBOclfQIl5DNd6ncTd76y2C7sv6dLcCxT/Zo2uR8FaHHIrimbFH03ODHLTI RakcQAG5lZa1A/0ilWtZk9Kl2VD+83h/evxF2svcsfBH4X/Qh4Zcz880eUQknoscbLbi s4Ng== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20161025; h=x-gm-message-state:from:to:cc:subject:date:message-id:in-reply-to :references; bh=kzBnu6nrYfOM4w8lU7MNMsAhbObCzbJss9j9gDPnlqg=; b=Kxy+nNtpZeSpJbCPduWnV1I42KRXPunhHSMfFbDnzBZ89W79EjuyNoYhtH7Lmij6ym wDG0H07AJgrGdyBSSri2KmBii3zM5mGwhQJkxJrC3BhAH6ED7x2rtwcx0UsxKrtI5yqW kdrRIiQuhFaNBHtSN1Zejvh9kN7ISijCIK+DrWaMol4anS+vCIAg6XWtv+X8URinvCd5 Z7WWYlIodnYL4GKtsf5Jp0nxdk3xgoqjpL/HBVQmEwAy6YHxD8CQgnW488t1wDP8gBEk 8RZkawhxmWSsWf0E4T2lou0lYniB241anE65qXSxFupPJBOvZnS5JKKOQ5suTSfSo2fG 21fw== X-Gm-Message-State: AOAM533PIPUqcTaNSfVS5K6tzYpOWe1RTVw5+WV2Va3gmOruD2U+9T+W eIqzu3R6GxBXmDIgJPyYCEdSwov4baNkZg== X-Google-Smtp-Source: ABdhPJwF553lIAuUC2XT+mtqN0pC2KDHpN1VMD8R8zPhUyk+OkNVDRw/kxrUsL6+K2ekRDaTIwHTQQ== X-Received: by 2002:a19:c14d:: with SMTP id r74mr479133lff.456.1614278229049; Thu, 25 Feb 2021 10:37:09 -0800 (PST) Received: from localhost.localdomain ([85.249.43.69]) by smtp.googlemail.com with ESMTPSA id x1sm1136772ljh.62.2021.02.25.10.37.07 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Thu, 25 Feb 2021 10:37:08 -0800 (PST) From: Andrey Konovalov To: libcamera-devel@lists.libcamera.org, laurent.pinchart@ideasonboard.com Date: Thu, 25 Feb 2021 21:36:26 +0300 Message-Id: <20210225183626.28288-6-andrey.konovalov@linaro.org> X-Mailer: git-send-email 2.17.1 In-Reply-To: <20210225183626.28288-1-andrey.konovalov@linaro.org> References: <20210225183626.28288-1-andrey.konovalov@linaro.org> Subject: [libcamera-devel] [PATCH 5/5] qcam: viewfinder_gl: Add support for RAW8 Bayer formats X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Cc: morgan@casual-effects.com MIME-Version: 1.0 Errors-To: libcamera-devel-bounces@lists.libcamera.org Sender: "libcamera-devel" All the four Bayer orders are supported. The texture coordinates passed to the fragment shader are ajusted to point to the nearest pixel in the image. This prevents artifacts when the image is scaled from the frame resolution to the window size. Signed-off-by: Andrey Konovalov --- src/qcam/assets/shader/bayer_8.frag | 136 ++++++++++++++++++++++++++++ src/qcam/assets/shader/bayer_8.vert | 72 +++++++++++++++ src/qcam/assets/shader/shaders.qrc | 1 + src/qcam/viewfinder_gl.cpp | 37 +++++++- 4 files changed, 244 insertions(+), 2 deletions(-) create mode 100644 src/qcam/assets/shader/bayer_8.frag create mode 100644 src/qcam/assets/shader/bayer_8.vert diff --git a/src/qcam/assets/shader/bayer_8.frag b/src/qcam/assets/shader/bayer_8.frag new file mode 100644 index 00000000..f308baff --- /dev/null +++ b/src/qcam/assets/shader/bayer_8.frag @@ -0,0 +1,136 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* +From http://jgt.akpeters.com/papers/McGuire08/ + +Efficient, High-Quality Bayer Demosaic Filtering on GPUs + +Morgan McGuire + +This paper appears in issue Volume 13, Number 4. +--------------------------------------------------------- +Copyright (c) 2008, Morgan McGuire. All rights reserved. + + +Modified by Linaro Ltd to integrate it into libcamera, and to +fix the artifacts due to pixel coordinates interpolation. +Copyright (C) 2021, Linaro +*/ + +//Pixel Shader + +varying vec2 textureOut; + +/* The texture size: tex_size.xy is in bytes, tex_size.zw is in pixels */ +uniform vec4 tex_size; +uniform vec2 tex_step; + +/** Pixel position of the first red pixel in the */ +/** Bayer pattern. [{0,1}, {0, 1}]*/ +uniform vec2 tex_bayer_first_red; + +/** Monochrome RGBA or GL_LUMINANCE Bayer encoded texture.*/ +uniform sampler2D tex_raw; + +void main(void) { + #define fetch(x, y) texture2D(tex_raw, vec2(x, y)).r + + /** .xy = Pixel being sampled in the fragment shader on the range [0, 1] + .zw = ...on the range [0, sourceSize], offset by firstRed */ + vec4 center; + + /** center.x + (-2/w, -1/w, 1/w, 2/w); These are the x-positions */ + /** of the adjacent pixels.*/ + vec4 xCoord; + + /** center.y + (-2/h, -1/h, 1/h, 2/h); These are the y-positions */ + /** of the adjacent pixels.*/ + vec4 yCoord; + + /* Align the center coordinates to the nearest pixel */ + center.zw = floor(textureOut * tex_size.zw); + center.xy = center.zw * tex_step; + center.zw += tex_bayer_first_red; + + xCoord = center.x + vec4(-2.0 * tex_step.x, + -tex_step.x, tex_step.x, 2.0 * tex_step.x); + yCoord = center.y + vec4(-2.0 * tex_step.y, + -tex_step.y, tex_step.y, 2.0 * tex_step.y); + + float C = texture2D(tex_raw, center.xy).r; // ( 0, 0) + const vec4 kC = vec4( 4.0, 6.0, 5.0, 5.0) / 8.0; + + // Determine which of four types of pixels we are on. + vec2 alternate = mod(floor(center.zw), 2.0); + + vec4 Dvec = vec4( + fetch(xCoord[1], yCoord[1]), // (-1,-1) + fetch(xCoord[1], yCoord[2]), // (-1, 1) + fetch(xCoord[2], yCoord[1]), // ( 1,-1) + fetch(xCoord[2], yCoord[2])); // ( 1, 1) + + vec4 PATTERN = (kC.xyz * C).xyzz; + + // Can also be a dot product with (1,1,1,1) on hardware where that is + // specially optimized. + // Equivalent to: D = Dvec[0] + Dvec[1] + Dvec[2] + Dvec[3]; + Dvec.xy += Dvec.zw; + Dvec.x += Dvec.y; + + vec4 value = vec4( + fetch(center.x, yCoord[0]), // ( 0,-2) + fetch(center.x, yCoord[1]), // ( 0,-1) + fetch(xCoord[0], center.y), // (-1, 0) + fetch(xCoord[1], center.y)); // (-2, 0) + + vec4 temp = vec4( + fetch(center.x, yCoord[3]), // ( 0, 2) + fetch(center.x, yCoord[2]), // ( 0, 1) + fetch(xCoord[3], center.y), // ( 2, 0) + fetch(xCoord[2], center.y)); // ( 1, 0) + + // Even the simplest compilers should be able to constant-fold these to + // avoid the division. + // Note that on scalar processors these constants force computation of some + // identical products twice. + const vec4 kA = vec4(-1.0, -1.5, 0.5, -1.0) / 8.0; + const vec4 kB = vec4( 2.0, 0.0, 0.0, 4.0) / 8.0; + const vec4 kD = vec4( 0.0, 2.0, -1.0, -1.0) / 8.0; + + // Conserve constant registers and take advantage of free swizzle on load + #define kE (kA.xywz) + #define kF (kB.xywz) + + value += temp; + + // There are five filter patterns (identity, cross, checker, + // theta, phi). Precompute the terms from all of them and then + // use swizzles to assign to color channels. + // + // Channel Matches + // x cross (e.g., EE G) + // y checker (e.g., EE B) + // z theta (e.g., EO R) + // w phi (e.g., EO R) + #define A (value[0]) + #define B (value[1]) + #define D (Dvec.x) + #define E (value[2]) + #define F (value[3]) + + // Avoid zero elements. On a scalar processor this saves two MADDs + // and it has no effect on a vector processor. + PATTERN.yzw += (kD.yz * D).xyy; + + PATTERN += (kA.xyz * A).xyzx + (kE.xyw * E).xyxz; + PATTERN.xw += kB.xw * B; + PATTERN.xz += kF.xz * F; + + vec3 rgb = (alternate.y == 0.0) ? + ((alternate.x == 0.0) ? + vec3(C, PATTERN.xy) : + vec3(PATTERN.z, C, PATTERN.w)) : + ((alternate.x == 0.0) ? + vec3(PATTERN.w, C, PATTERN.z) : + vec3(PATTERN.yx, C)); + gl_FragColor = vec4(rgb, 1.0); +} diff --git a/src/qcam/assets/shader/bayer_8.vert b/src/qcam/assets/shader/bayer_8.vert new file mode 100644 index 00000000..26525db7 --- /dev/null +++ b/src/qcam/assets/shader/bayer_8.vert @@ -0,0 +1,72 @@ +/* +From http://jgt.akpeters.com/papers/McGuire08/ + +Efficient, High-Quality Bayer Demosaic Filtering on GPUs + +Morgan McGuire + +This paper appears in issue Volume 13, Number 4. +--------------------------------------------------------- +Copyright (c) 2008, Morgan McGuire. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +//Vertex Shader + +attribute vec4 vertexIn; +attribute vec2 textureIn; + +/* The texture size: tex_size.xy is in bytes, tex_size.zw is in pixels */ +uniform vec4 tex_size; +uniform vec2 tex_step; + +/** Pixel position of the first red pixel in the */ +/** Bayer pattern. [{0,1}, {0, 1}]*/ +uniform vec2 tex_bayer_first_red; + +/** .xy = Pixel being sampled in the fragment shader on the range [0, 1] + .zw = ...on the range [0, sourceSize], offset by firstRed */ +varying vec4 center; + +/** center.x + (-2/w, -1/w, 1/w, 2/w); These are the x-positions */ +/** of the adjacent pixels.*/ +varying vec4 xCoord; + +/** center.y + (-2/h, -1/h, 1/h, 2/h); These are the y-positions */ +/** of the adjacent pixels.*/ +varying vec4 yCoord; + +void main(void) { + center.xy = textureIn; + center.zw = textureIn * tex_size.zw + tex_bayer_first_red; + + xCoord = center.x + vec4(-2.0 * tex_step.x, + -tex_step.x, tex_step.x, 2.0 * tex_step.x); + yCoord = center.y + vec4(-2.0 * tex_step.y, + -tex_step.y, tex_step.y, 2.0 * tex_step.y); + + gl_Position = vertexIn; +} diff --git a/src/qcam/assets/shader/shaders.qrc b/src/qcam/assets/shader/shaders.qrc index d76d65c5..79f44a30 100644 --- a/src/qcam/assets/shader/shaders.qrc +++ b/src/qcam/assets/shader/shaders.qrc @@ -5,6 +5,7 @@ YUV_2_planes.frag YUV_3_planes.frag YUV_packed.frag + bayer_8.frag bayer_1x_packed.frag identity.vert diff --git a/src/qcam/viewfinder_gl.cpp b/src/qcam/viewfinder_gl.cpp index 13ab31aa..ef85e93d 100644 --- a/src/qcam/viewfinder_gl.cpp +++ b/src/qcam/viewfinder_gl.cpp @@ -36,6 +36,11 @@ static const QList supportedFormats{ libcamera::formats::RGBA8888, libcamera::formats::BGR888, libcamera::formats::RGB888, + /* Raw Bayer 8-bit */ + libcamera::formats::SBGGR8, + libcamera::formats::SGBRG8, + libcamera::formats::SGRBG8, + libcamera::formats::SRGGB8, /* Raw Bayer 10-bit packed */ libcamera::formats::SBGGR10_CSI2P, libcamera::formats::SGBRG10_CSI2P, @@ -220,6 +225,30 @@ bool ViewFinderGL::selectFormat(const libcamera::PixelFormat &format) fragmentShaderDefines_.append("#define RGB_PATTERN bgr"); fragmentShaderFile_ = ":RGB.frag"; break; + case libcamera::formats::SBGGR8: + firstRed_.setX(1.0); + firstRed_.setY(1.0); + fragmentShaderFile_ = ":bayer_8.frag"; + textureMinMagFilters_ = GL_NEAREST; + break; + case libcamera::formats::SGBRG8: + firstRed_.setX(0.0); + firstRed_.setY(1.0); + fragmentShaderFile_ = ":bayer_8.frag"; + textureMinMagFilters_ = GL_NEAREST; + break; + case libcamera::formats::SGRBG8: + firstRed_.setX(1.0); + firstRed_.setY(0.0); + fragmentShaderFile_ = ":bayer_8.frag"; + textureMinMagFilters_ = GL_NEAREST; + break; + case libcamera::formats::SRGGB8: + firstRed_.setX(0.0); + firstRed_.setY(0.0); + fragmentShaderFile_ = ":bayer_8.frag"; + textureMinMagFilters_ = GL_NEAREST; + break; case libcamera::formats::SBGGR10_CSI2P: firstRed_.setX(1.0); firstRed_.setY(1.0); @@ -624,6 +653,10 @@ void ViewFinderGL::doRender() shaderProgram_.setUniformValue(textureUniformY_, 0); break; + case libcamera::formats::SBGGR8: + case libcamera::formats::SGBRG8: + case libcamera::formats::SGRBG8: + case libcamera::formats::SRGGB8: case libcamera::formats::SBGGR10_CSI2P: case libcamera::formats::SGBRG10_CSI2P: case libcamera::formats::SGRBG10_CSI2P: @@ -633,8 +666,8 @@ void ViewFinderGL::doRender() case libcamera::formats::SGRBG12_CSI2P: case libcamera::formats::SRGGB12_CSI2P: /* - * Packed raw Bayer 10-bit and 12-bit formats are stored in - * GL_RED texture. + * Raw Bayer 8-bit, and packed raw Bayer 10-bit/12-bit formats + * are stored in GL_RED texture. */ glActiveTexture(GL_TEXTURE0); configureTexture(*textures_[0]);