From patchwork Fri Jun 11 16:27:23 2021 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Andrey Konovalov X-Patchwork-Id: 12573 Return-Path: X-Original-To: parsemail@patchwork.libcamera.org Delivered-To: parsemail@patchwork.libcamera.org Received: from lancelot.ideasonboard.com (lancelot.ideasonboard.com [92.243.16.209]) by patchwork.libcamera.org (Postfix) with ESMTPS id B4AAEC320B for ; Fri, 11 Jun 2021 16:27:44 +0000 (UTC) Received: from lancelot.ideasonboard.com (localhost [IPv6:::1]) by lancelot.ideasonboard.com (Postfix) with ESMTP id 74AFF6892B; Fri, 11 Jun 2021 18:27:44 +0200 (CEST) Authentication-Results: lancelot.ideasonboard.com; dkim=fail reason="signature verification failed" (2048-bit key; unprotected) header.d=linaro.org header.i=@linaro.org header.b="ZoXwmaoX"; dkim-atps=neutral Received: from mail-lf1-x132.google.com (mail-lf1-x132.google.com [IPv6:2a00:1450:4864:20::132]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id 90A346029E for ; Fri, 11 Jun 2021 18:27:42 +0200 (CEST) Received: by mail-lf1-x132.google.com with SMTP id k40so9391536lfv.7 for ; Fri, 11 Jun 2021 09:27:42 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=linaro.org; s=google; h=from:to:cc:subject:date:message-id:in-reply-to:references :mime-version:content-transfer-encoding; bh=mv2rnIZBXYiZJVAJqcAZOvGmMppMCyKTu4OnVWYQArQ=; b=ZoXwmaoXJkGWewFylxgg/7CBUyLiE2pp8ZGpBtQymJWcvMgLfyzm7hcAsNMG7dA1Qi /Yiw/nq3j+x9Tapk6xOr5AVbW7xkvdUd2iwXe724u6Ck5XcUaQML3dBdSfz4vCLZFC93 X8rVB4V6gEsVGjfwXLdNuh4m6Uw9wjU4bR7gcjYtC0420movYWy+DhTIvGQp/UL8fmXI Y7qFIe2Zg+JTXUEGim4MAF/FxlwUCvb2AhxIbqIRlDm0qNqOvdoa22So7QTC3nFuo9iD yd/QPVsyYkjPcoRJ8y1/hebrGvNSUjKysAx/Zl4WIeuNGcd1SAJhDjBykDSEb2HRTmP6 xQEA== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20161025; h=x-gm-message-state:from:to:cc:subject:date:message-id:in-reply-to :references:mime-version:content-transfer-encoding; bh=mv2rnIZBXYiZJVAJqcAZOvGmMppMCyKTu4OnVWYQArQ=; b=LTYutRQvAMsM2ZzhKn1MIhbSXirS1ZKCPC1I5Ac4OLaGmMbPNjYhVDytSqiSKt8VVX odYxBTLvjBOGR9Sf2IteC1sanqhY5B490qNLeVa3/Rl4xGoalboQ/t+wAe/2FPtmqsD7 z+nBxY86gZ3yYGu+4JF1NCVjtxNRhSbPEX1J/jUmyKp/erOoD0YccoyE5AQB7Jxesf3b qt5Jmhw/YWeGschQQJV9Eu76mSFY409OF1kCckVSsED6HDfNHgJMaSwEJ8DcdCfSXF7C PgBQBmmFzd1SxVHg4u3dezWIT3FkRxy8XoPdSN2mtZ+a9CkCCDfnqmZaLE5fYo9cEKOE cnuw== X-Gm-Message-State: AOAM5305sRLHFzvX9dYkxxkwkB1n948931VfpmAa0lZg5YDfFGwtGdXE 5q6dThydSwatp6zRd2IN5uE6yeyBhuUSPQ== X-Google-Smtp-Source: ABdhPJwet9CBnhvKAjuWLdrYkB6HPBBL8ZD2TkthBQIOyR3tbssx2X2wDRIccT0xDnX/ErlDJiJ9zQ== X-Received: by 2002:a19:c347:: with SMTP id t68mr3100033lff.653.1623428862060; Fri, 11 Jun 2021 09:27:42 -0700 (PDT) Received: from localhost.localdomain ([85.249.44.185]) by smtp.googlemail.com with ESMTPSA id l2sm777773lji.70.2021.06.11.09.27.41 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Fri, 11 Jun 2021 09:27:41 -0700 (PDT) From: Andrey Konovalov To: libcamera-devel@lists.libcamera.org, laurent.pinchart@ideasonboard.com Date: Fri, 11 Jun 2021 19:27:23 +0300 Message-Id: <20210611162726.824789-2-andrey.konovalov@linaro.org> X-Mailer: git-send-email 2.25.1 In-Reply-To: <20210611162726.824789-1-andrey.konovalov@linaro.org> References: <20210611162726.824789-1-andrey.konovalov@linaro.org> MIME-Version: 1.0 Subject: [libcamera-devel] [PATCH v3 1/4] qcam: viewfinder_gl: change uniform float tex_stepx to vec2 tex_step X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Cc: morgan@casual-effects.com Errors-To: libcamera-devel-bounces@lists.libcamera.org Sender: "libcamera-devel" In preparation to extend the supported formats, extend the tex_stepx uniform to cover the steps between texels in both horizontal and vertical directions. Signed-off-by: Andrey Konovalov Reviewed-by: Laurent Pinchart --- src/qcam/assets/shader/YUV_packed.frag | 8 ++++---- src/qcam/viewfinder_gl.cpp | 7 ++++--- src/qcam/viewfinder_gl.h | 2 +- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/src/qcam/assets/shader/YUV_packed.frag b/src/qcam/assets/shader/YUV_packed.frag index 224dfafe..d6efd4ce 100644 --- a/src/qcam/assets/shader/YUV_packed.frag +++ b/src/qcam/assets/shader/YUV_packed.frag @@ -12,7 +12,7 @@ precision mediump float; varying vec2 textureOut; uniform sampler2D tex_y; -uniform float tex_stepx; +uniform vec2 tex_step; void main(void) { @@ -49,10 +49,10 @@ void main(void) * a = fract(x) * 2 - 1 if fract(x) >= 0.5 */ vec2 pos = textureOut; - float f_x = fract(pos.x / tex_stepx); + float f_x = fract(pos.x / tex_step.x); - vec4 left = texture2D(tex_y, vec2(pos.x - f_x * tex_stepx, pos.y)); - vec4 right = texture2D(tex_y, vec2(pos.x + (1.0 - f_x) * tex_stepx , pos.y)); + vec4 left = texture2D(tex_y, vec2(pos.x - f_x * tex_step.x, pos.y)); + vec4 right = texture2D(tex_y, vec2(pos.x + (1.0 - f_x) * tex_step.x , pos.y)); #if defined(YUV_PATTERN_UYVY) float y_left = mix(left.g, left.a, f_x * 2.0); diff --git a/src/qcam/viewfinder_gl.cpp b/src/qcam/viewfinder_gl.cpp index 5d9b442e..ff719418 100644 --- a/src/qcam/viewfinder_gl.cpp +++ b/src/qcam/viewfinder_gl.cpp @@ -289,7 +289,7 @@ bool ViewFinderGL::createFragmentShader() textureUniformY_ = shaderProgram_.uniformLocation("tex_y"); textureUniformU_ = shaderProgram_.uniformLocation("tex_u"); textureUniformV_ = shaderProgram_.uniformLocation("tex_v"); - textureUniformStepX_ = shaderProgram_.uniformLocation("tex_stepx"); + textureUniformStep_ = shaderProgram_.uniformLocation("tex_step"); /* Create the textures. */ for (std::unique_ptr &texture : textures_) { @@ -508,8 +508,9 @@ void ViewFinderGL::doRender() * ([0, 1]). There are exactly width - 1 steps between the * leftmost and rightmost texels. */ - shaderProgram_.setUniformValue(textureUniformStepX_, - 1.0f / (size_.width() / 2 - 1)); + shaderProgram_.setUniformValue(textureUniformStep_, + 1.0f / (size_.width() / 2 - 1), + 1.0f /* not used */); break; case libcamera::formats::ABGR8888: diff --git a/src/qcam/viewfinder_gl.h b/src/qcam/viewfinder_gl.h index 150fa4ae..1b1faa91 100644 --- a/src/qcam/viewfinder_gl.h +++ b/src/qcam/viewfinder_gl.h @@ -85,7 +85,7 @@ private: GLuint textureUniformU_; GLuint textureUniformV_; GLuint textureUniformY_; - GLuint textureUniformStepX_; + GLuint textureUniformStep_; unsigned int horzSubSample_; unsigned int vertSubSample_; From patchwork Fri Jun 11 16:27:24 2021 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Andrey Konovalov X-Patchwork-Id: 12574 Return-Path: X-Original-To: parsemail@patchwork.libcamera.org Delivered-To: parsemail@patchwork.libcamera.org Received: from lancelot.ideasonboard.com (lancelot.ideasonboard.com [92.243.16.209]) by patchwork.libcamera.org (Postfix) with ESMTPS id 20370C320B for ; Fri, 11 Jun 2021 16:27:49 +0000 (UTC) Received: from lancelot.ideasonboard.com (localhost [IPv6:::1]) by lancelot.ideasonboard.com (Postfix) with ESMTP id D4A2568931; Fri, 11 Jun 2021 18:27:48 +0200 (CEST) Authentication-Results: lancelot.ideasonboard.com; dkim=fail reason="signature verification failed" (2048-bit key; unprotected) header.d=linaro.org header.i=@linaro.org header.b="OFCvitVP"; dkim-atps=neutral Received: from mail-lf1-x12c.google.com (mail-lf1-x12c.google.com [IPv6:2a00:1450:4864:20::12c]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id 288796029E for ; Fri, 11 Jun 2021 18:27:47 +0200 (CEST) Received: by mail-lf1-x12c.google.com with SMTP id a1so9347679lfr.12 for ; Fri, 11 Jun 2021 09:27:47 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=linaro.org; s=google; h=from:to:cc:subject:date:message-id:in-reply-to:references :mime-version:content-transfer-encoding; bh=zkWWuiVh73mI6akPOsw43GT58R59OWPHZph29DbR1m8=; b=OFCvitVPKVK6MWORetRsEvegGEiR2PnUVPdQutg2s0rFfVaaA06KvXvP9xtMRezli/ ZovKWsws619kWQcS5NqiLufx9pV2jcwNDIEnV3Y83VMOOdl2YIx8aJu752xNxv6gE906 c5rgMiXqqEpDx+Fejby/xiaxaWyE+BtXbZMibgYJmCAZG1mxgDlyP/Z30c14rfp0qvKA Ifz+F/UNSST9o0eAZ8iVRvPWa956kSXY1ZhNq9G1UxjG3OpRXut2MYjQeywEqaklkkys tawYMWBAwu+OFrJlpYBTYzLWLUpcWQjI/aEHPAH3lFNHmv3phIZwxw4gjN/xghwCIscl TmBQ== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20161025; h=x-gm-message-state:from:to:cc:subject:date:message-id:in-reply-to :references:mime-version:content-transfer-encoding; bh=zkWWuiVh73mI6akPOsw43GT58R59OWPHZph29DbR1m8=; b=T/94N4w/EuseD6dZ3NUiQHqXVHZN4kn4QTsVdxLx75m/p2Tz34P47qkFoK0TX8zHbg le+a4ivBIBcJfGdbicw+0UBTVkzTbV7Fb90j36gko5qrb8cYNPHyz5JY/FX9tYNLHbyK NByLS/D4idHIQW4Q53ZWq33XBiahZcnnC3bSUtuQfojyLUuLQjm1CJKcjV+k8Z3CGpHI 37yPXeHHeTpLuPdt7/LgiEI8DZNTcKifE2fjyvcCL+ikfoIAxJ/JWTPokLNPsZrovvSW bwj1o1dmpEk01Im506sZN7X111uRw4mmd8/1ysDuh2zCa8lag5c0VOhdYKlUlsLgHDr1 I7Ig== X-Gm-Message-State: AOAM531Q5I8u49kbgNuK71kVmXPsdFz7Wb2hWD76e8+9CF5+Z9FJ7bCB /JtGMp1riKCa3uv1zcnqTcI/XZ+HobQoVg== X-Google-Smtp-Source: ABdhPJx8+sthl3iLaVAtb8PVWq2o6UM3Y9Q9XdfZuyajMWb2HYn7T09qPJmOmDeWw8jbOhegNK3HKg== X-Received: by 2002:a05:6512:3891:: with SMTP id n17mr3253442lft.333.1623428866488; Fri, 11 Jun 2021 09:27:46 -0700 (PDT) Received: from localhost.localdomain ([85.249.44.185]) by smtp.googlemail.com with ESMTPSA id l2sm777773lji.70.2021.06.11.09.27.45 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Fri, 11 Jun 2021 09:27:45 -0700 (PDT) From: Andrey Konovalov To: libcamera-devel@lists.libcamera.org, laurent.pinchart@ideasonboard.com Date: Fri, 11 Jun 2021 19:27:24 +0300 Message-Id: <20210611162726.824789-3-andrey.konovalov@linaro.org> X-Mailer: git-send-email 2.25.1 In-Reply-To: <20210611162726.824789-1-andrey.konovalov@linaro.org> References: <20210611162726.824789-1-andrey.konovalov@linaro.org> MIME-Version: 1.0 Subject: [libcamera-devel] [PATCH v3 2/4] qcam: viewfinder_gl: Add shader to render packed RAW10 formats X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Cc: morgan@casual-effects.com Errors-To: libcamera-devel-bounces@lists.libcamera.org Sender: "libcamera-devel" The shader supports all 4 packed RAW10 variants. Simple bi-linear filtering is implemented. The 2 LS bits of the 10-bit colour values are dropped as the RGBA format we convert into has only 8 bits per colour. The texture coordinates passed to the fragment shader are ajusted to point to the nearest pixel in the image. This prevents artifacts when the image is scaled from the frame resolution to the window size. Signed-off-by: Andrey Konovalov --- src/qcam/assets/shader/bayer_1x_packed.frag | 174 ++++++++++++++++++++ src/qcam/assets/shader/shaders.qrc | 1 + src/qcam/viewfinder_gl.cpp | 80 ++++++++- src/qcam/viewfinder_gl.h | 7 + 4 files changed, 260 insertions(+), 2 deletions(-) create mode 100644 src/qcam/assets/shader/bayer_1x_packed.frag diff --git a/src/qcam/assets/shader/bayer_1x_packed.frag b/src/qcam/assets/shader/bayer_1x_packed.frag new file mode 100644 index 00000000..0a87c6db --- /dev/null +++ b/src/qcam/assets/shader/bayer_1x_packed.frag @@ -0,0 +1,174 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Based on the code from http://jgt.akpeters.com/papers/McGuire08/ + * + * Efficient, High-Quality Bayer Demosaic Filtering on GPUs + * + * Morgan McGuire + * + * This paper appears in issue Volume 13, Number 4. + * --------------------------------------------------------- + * Copyright (c) 2008, Morgan McGuire. All rights reserved. + * + * + * Modified by Linaro Ltd for 10/12-bit packed vs 8-bit raw Bayer format, + * and for simpler demosaic algorithm. + * Copyright (C) 2020, Linaro + * + * bayer_1x_packed.frag - Fragment shader code for raw Bayer 10-bit and 12-bit + * packed formats + */ + +#ifdef GL_ES +precision mediump float; +#endif + +varying vec2 textureOut; + +/* the texture size: tex_size.xy is in bytes, tex_size.zw is in pixels */ +uniform vec4 tex_size; +uniform vec2 tex_step; +uniform vec2 tex_bayer_first_red; + +uniform sampler2D tex_raw; + +void main(void) +{ + vec3 rgb; + + /* + * center.xy holds the coordinates of the pixel being sampled + * on the [0, 1] range. + * center.zw holds the coordinates of the pixel being sampled + * on the [0, width/height-1] range. + */ + vec4 center; + /* + * x-positions of the adjacent pixels on the [0, 1] range. + */ + vec2 xcoords; + /* + * y-positions of the adjacent pixels on the [0, 1] range. + */ + vec2 ycoords; + + /* + * The coordinates passed to the shader in textureOut may point + * to a place in between the pixels if the viewfinder window is scaled + * from the original captured frame size. Align them to the nearest + * pixel. + */ + center.zw = floor(textureOut * tex_size.zw); + center.y = center.w; + /* + * Add a small number (a few mantissa's LSBs) to avoid float + * representation issues. Maybe paranoic. + */ + center.x = BPP_X * center.z + 0.02; + + const float threshold_l = 0.127 /* fract(BPP_X) * 0.5 + 0.02 */; + const float threshold_h = 0.625 /* 1.0 - fract(BPP_X) * 1.5 */; + + float fract_x = fract(center.x); + /* + * The below floor() call ensures that center.x points + * at one of the bytes representing the 8 higher bits of + * the pixel value, not at the byte containing the LS bits + * of the group of the pixels. + */ + center.x = floor(center.x); + center.xy *= tex_step; + + xcoords = center.x + vec2(-tex_step.x, tex_step.x); + ycoords = center.y + vec2(-tex_step.y, tex_step.y); + /* + * If xcoords[0] points at the byte containing the LS bits + * of the previous group of the pixels, move xcoords[0] one + * byte back. + */ + xcoords[0] += (fract_x < threshold_l) ? -tex_step.x : 0.0; + /* + * If xcoords[1] points at the byte containing the LS bits + * of the current group of the pixels, move xcoords[1] one + * byte forward. + */ + xcoords[1] += (fract_x > threshold_h) ? tex_step.x : 0.0; + + vec2 alternate = mod(center.zw + tex_bayer_first_red, 2.0); + bool even_col = alternate.x < 1.0; + bool even_raw = alternate.y < 1.0; + + /* + * We need to sample the central pixel and the ones with offset + * of -1 to +1 pixel in both X and Y directions. Let's name these + * pixels as below, where C is the central pixel: + * +----+----+----+----+ + * | \ x| | | | + * |y \ | -1 | 0 | +1 | + * +----+----+----+----+ + * | +1 | D2 | A1 | D3 | + * +----+----+----+----+ + * | 0 | B0 | C | B1 | + * +----+----+----+----+ + * | -1 | D0 | A0 | D1 | + * +----+----+----+----+ + * In the below equations (0,-1).r means "r component of the texel + * shifted by -tex_step.y from the center.xy one" etc. + * In the even raw / even column (EE) case the colour values are: + * R = C = (0,0).r, + * G = (A0 + A1 + B0 + B1) / 4.0 = + * ( (0,-1).r + (0,1).r + (-1,0).r + (1,0).r ) / 4.0, + * B = (D0 + D1 + D2 + D3) / 4.0 = + * ( (-1,-1).r + (1,-1).r + (-1,1).r + (1,1).r ) / 4.0 + * For even raw / odd column (EO): + * R = (B0 + B1) / 2.0 = ( (-1,0).r + (1,0).r ) / 2.0, + * G = C = (0,0).r, + * B = (A0 + A1) / 2.0 = ( (0,-1).r + (0,1).r ) / 2.0 + * For odd raw / even column (OE): + * R = (A0 + A1) / 2.0 = ( (0,-1).r + (0,1).r ) / 2.0, + * G = C = (0,0).r, + * B = (B0 + B1) / 2.0 = ( (-1,0).r + (1,0).r ) / 2.0 + * For odd raw / odd column (OO): + * R = (D0 + D1 + D2 + D3) / 4.0 = + * ( (-1,-1).r + (1,-1).r + (-1,1).r + (1,1).r ) / 4.0, + * G = (A0 + A1 + B0 + B1) / 4.0 = + * ( (0,-1).r + (0,1).r + (-1,0).r + (1,0).r ) / 4.0, + * B = C = (0,0).r + */ + + /* + * Fetch the values and precalculate the terms: + * patterns.x = (A0 + A1) / 2.0 + * patterns.y = (B0 + B1) / 2.0 + * patterns.z = (A0 + A1 + B0 + B1) / 4.0 + * patterns.w = (D0 + D1 + D2 + D3) / 4.0 + */ + #define fetch(x, y) texture2D(tex_raw, vec2(x, y)).r + + float C = texture2D(tex_raw, center.xy).r; + vec4 patterns = vec4( + fetch(center.x, ycoords[0]), /* A0: (0,-1) */ + fetch(xcoords[0], center.y), /* B0: (-1,0) */ + fetch(xcoords[0], ycoords[0]), /* D0: (-1,-1) */ + fetch(xcoords[1], ycoords[0])); /* D1: (1,-1) */ + vec4 temp = vec4( + fetch(center.x, ycoords[1]), /* A1: (0,1) */ + fetch(xcoords[1], center.y), /* B1: (1,0) */ + fetch(xcoords[1], ycoords[1]), /* D3: (1,1) */ + fetch(xcoords[0], ycoords[1])); /* D2: (-1,1) */ + patterns = (patterns + temp) * 0.5; + /* .x = (A0 + A1) / 2.0, .y = (B0 + B1) / 2.0 */ + /* .z = (D0 + D3) / 2.0, .w = (D1 + D2) / 2.0 */ + patterns.w = (patterns.z + patterns.w) * 0.5; + patterns.z = (patterns.x + patterns.y) * 0.5; + + rgb = (even_col) ? + ((even_raw) ? + vec3(C, patterns.zw) : + vec3(patterns.x, C, patterns.y)) : + ((even_raw) ? + vec3(patterns.y, C, patterns.x) : + vec3(patterns.wz, C)); + + gl_FragColor = vec4(rgb, 1.0); +} diff --git a/src/qcam/assets/shader/shaders.qrc b/src/qcam/assets/shader/shaders.qrc index 8a8f9de1..d76d65c5 100644 --- a/src/qcam/assets/shader/shaders.qrc +++ b/src/qcam/assets/shader/shaders.qrc @@ -5,6 +5,7 @@ YUV_2_planes.frag YUV_3_planes.frag YUV_packed.frag + bayer_1x_packed.frag identity.vert diff --git a/src/qcam/viewfinder_gl.cpp b/src/qcam/viewfinder_gl.cpp index ff719418..44e410c9 100644 --- a/src/qcam/viewfinder_gl.cpp +++ b/src/qcam/viewfinder_gl.cpp @@ -36,6 +36,11 @@ static const QList supportedFormats{ libcamera::formats::RGBA8888, libcamera::formats::BGR888, libcamera::formats::RGB888, + /* Raw Bayer 10-bit packed */ + libcamera::formats::SBGGR10_CSI2P, + libcamera::formats::SGBRG10_CSI2P, + libcamera::formats::SGRBG10_CSI2P, + libcamera::formats::SRGGB10_CSI2P, }; ViewFinderGL::ViewFinderGL(QWidget *parent) @@ -106,6 +111,10 @@ void ViewFinderGL::render(libcamera::FrameBuffer *buffer, MappedBuffer *map) renderComplete(buffer_); data_ = static_cast(map->memory); + /* + * \todo Get the stride from the buffer instead of computing it naively + */ + stride_ = buffer->metadata().planes[0].bytesused / size_.height(); update(); buffer_ = buffer; } @@ -114,6 +123,9 @@ bool ViewFinderGL::selectFormat(const libcamera::PixelFormat &format) { bool ret = true; + /* Set min/mag filters to GL_LINEAR by default. */ + textureMinMagFilters_ = GL_LINEAR; + fragmentShaderDefines_.clear(); switch (format) { @@ -203,6 +215,34 @@ bool ViewFinderGL::selectFormat(const libcamera::PixelFormat &format) fragmentShaderDefines_.append("#define RGB_PATTERN bgr"); fragmentShaderFile_ = ":RGB.frag"; break; + case libcamera::formats::SBGGR10_CSI2P: + firstRed_.setX(1.0); + firstRed_.setY(1.0); + fragmentShaderDefines_.append("#define BPP_X 1.25"); + fragmentShaderFile_ = ":bayer_1x_packed.frag"; + textureMinMagFilters_ = GL_NEAREST; + break; + case libcamera::formats::SGBRG10_CSI2P: + firstRed_.setX(0.0); + firstRed_.setY(1.0); + fragmentShaderDefines_.append("#define BPP_X 1.25"); + fragmentShaderFile_ = ":bayer_1x_packed.frag"; + textureMinMagFilters_ = GL_NEAREST; + break; + case libcamera::formats::SGRBG10_CSI2P: + firstRed_.setX(1.0); + firstRed_.setY(0.0); + fragmentShaderDefines_.append("#define BPP_X 1.25"); + fragmentShaderFile_ = ":bayer_1x_packed.frag"; + textureMinMagFilters_ = GL_NEAREST; + break; + case libcamera::formats::SRGGB10_CSI2P: + firstRed_.setX(0.0); + firstRed_.setY(0.0); + fragmentShaderDefines_.append("#define BPP_X 1.25"); + fragmentShaderFile_ = ":bayer_1x_packed.frag"; + textureMinMagFilters_ = GL_NEAREST; + break; default: ret = false; qWarning() << "[ViewFinderGL]:" @@ -290,6 +330,8 @@ bool ViewFinderGL::createFragmentShader() textureUniformU_ = shaderProgram_.uniformLocation("tex_u"); textureUniformV_ = shaderProgram_.uniformLocation("tex_v"); textureUniformStep_ = shaderProgram_.uniformLocation("tex_step"); + textureUniformSize_ = shaderProgram_.uniformLocation("tex_size"); + textureUniformBayerFirstRed_ = shaderProgram_.uniformLocation("tex_bayer_first_red"); /* Create the textures. */ for (std::unique_ptr &texture : textures_) { @@ -306,8 +348,10 @@ bool ViewFinderGL::createFragmentShader() void ViewFinderGL::configureTexture(QOpenGLTexture &texture) { glBindTexture(GL_TEXTURE_2D, texture.textureId()); - glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR); - glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, + textureMinMagFilters_); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, + textureMinMagFilters_); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); } @@ -547,6 +591,38 @@ void ViewFinderGL::doRender() shaderProgram_.setUniformValue(textureUniformY_, 0); break; + case libcamera::formats::SBGGR10_CSI2P: + case libcamera::formats::SGBRG10_CSI2P: + case libcamera::formats::SGRBG10_CSI2P: + case libcamera::formats::SRGGB10_CSI2P: + /* + * Packed raw Bayer 10-bit formats are stored in GL_RED texture. + * The texture width is equal to the stride. + */ + glActiveTexture(GL_TEXTURE0); + configureTexture(*textures_[0]); + glTexImage2D(GL_TEXTURE_2D, + 0, + GL_RED, + stride_, + size_.height(), + 0, + GL_RED, + GL_UNSIGNED_BYTE, + data_); + shaderProgram_.setUniformValue(textureUniformY_, 0); + shaderProgram_.setUniformValue(textureUniformBayerFirstRed_, + firstRed_); + shaderProgram_.setUniformValue(textureUniformSize_, + stride_, /* width in bytes */ + size_.height(), + size_.width(), /* in pixels */ + size_.height()); + shaderProgram_.setUniformValue(textureUniformStep_, + 1.0f / (stride_ - 1), + 1.0f / (size_.height() - 1)); + break; + default: break; }; diff --git a/src/qcam/viewfinder_gl.h b/src/qcam/viewfinder_gl.h index 1b1faa91..0171885a 100644 --- a/src/qcam/viewfinder_gl.h +++ b/src/qcam/viewfinder_gl.h @@ -66,6 +66,7 @@ private: libcamera::FrameBuffer *buffer_; libcamera::PixelFormat format_; QSize size_; + unsigned int stride_; unsigned char *data_; /* Shaders */ @@ -81,13 +82,19 @@ private: /* Textures */ std::array, 3> textures_; + /* Common texture parameters */ + GLuint textureMinMagFilters_; /* YUV texture parameters */ GLuint textureUniformU_; GLuint textureUniformV_; GLuint textureUniformY_; + GLuint textureUniformSize_; GLuint textureUniformStep_; unsigned int horzSubSample_; unsigned int vertSubSample_; + /* Raw Bayer texture parameters */ + GLuint textureUniformBayerFirstRed_; + QPointF firstRed_; QMutex mutex_; /* Prevent concurrent access to image_ */ }; From patchwork Fri Jun 11 16:27:25 2021 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Andrey Konovalov X-Patchwork-Id: 12575 Return-Path: X-Original-To: parsemail@patchwork.libcamera.org Delivered-To: parsemail@patchwork.libcamera.org Received: from lancelot.ideasonboard.com (lancelot.ideasonboard.com [92.243.16.209]) by patchwork.libcamera.org (Postfix) with ESMTPS id B2F4CC320B for ; Fri, 11 Jun 2021 16:27:50 +0000 (UTC) Received: from lancelot.ideasonboard.com (localhost [IPv6:::1]) by lancelot.ideasonboard.com (Postfix) with ESMTP id 746AA68933; Fri, 11 Jun 2021 18:27:50 +0200 (CEST) Authentication-Results: lancelot.ideasonboard.com; dkim=fail reason="signature verification failed" (2048-bit key; unprotected) header.d=linaro.org header.i=@linaro.org header.b="k1zRoXGd"; dkim-atps=neutral Received: from mail-lf1-x133.google.com (mail-lf1-x133.google.com [IPv6:2a00:1450:4864:20::133]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id F353268935 for ; Fri, 11 Jun 2021 18:27:48 +0200 (CEST) Received: by mail-lf1-x133.google.com with SMTP id r5so9412689lfr.5 for ; Fri, 11 Jun 2021 09:27:48 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=linaro.org; s=google; h=from:to:cc:subject:date:message-id:in-reply-to:references :mime-version:content-transfer-encoding; bh=dPIsWz/6G4lpzuNBj+oiRUoT7VO2YgCg8GWk0qJ0Dsw=; b=k1zRoXGdqfKUQNnNt65KzOdrIz4zSOgL42x4Lc/Fw1gpGdPjIrPV/eRBipcTHdhV7i 8ZJkkNdJ3/mBAQ3phqDeTLPt6qbV5ZX5GMhnMMAH3xfLXh4FAqk15VulU9a8+STaottd 7ho+/P7qr99eaXM7aVyS+3oacHTWPolLfnKGCj/8P+SfMwyXUosYfZ9bGsNUeW0TcbAd Qg3hMX8H3EOrs/3o18X+4kWNVdrjRORs/ym8pz6viHccuqtjC2x8JoSn5MUyu2SA6zuK qK2NdsJrUYXSOTd/A6Ha3vHegNj6A0M6hvz7zLeru+pEAAtAi16M/k0Xaga6qpXcNg6k n/PA== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20161025; h=x-gm-message-state:from:to:cc:subject:date:message-id:in-reply-to :references:mime-version:content-transfer-encoding; bh=dPIsWz/6G4lpzuNBj+oiRUoT7VO2YgCg8GWk0qJ0Dsw=; b=kE1qltbhrsLL2KYvvkMGITIqtjhhabWM4BynpYprl8vxO2CJG01EWvfgozVpLL1t+k 2RLwxJsmcyxbt2WwXyZyL3kVzEas7k7QMbQYyX8/uoOMdkKPbit2cdZEhcrqy0W052Jg ZfKFQVCdNHjtUQpU6nTnom4k4qQYi1eFhi5dOWH/SyCXYwkXst4JVwkXYTzVRq7uTmi3 xFekm31WIVlDPCDiNNJoB2VohqMosF0Wpe32lLjSVgTe5+2+aSyHNbAR5BS+9hGQuTq6 BzFrr1oyUYNB2u/zLDRwVmh5o9s1cl7pr6+Lx0b90TKSiygu2fek8dkeVveXRUEvaLhd YMhA== X-Gm-Message-State: AOAM530geVsI3x4IgN+HkQkwsLuxSYFO1yvKZtcwM1Hh6ibh+JrH79tN yLykSZB+mQDcEo9IDQb8mXXAzhGXa6PWPA== X-Google-Smtp-Source: ABdhPJzAI6J5GiuxzDgxOBKFuhLLfDMGZB7Q8zavC7fXUmPZnMsAcvtO9CzwVqH0pOI0OSq10RJivA== X-Received: by 2002:a19:5e18:: with SMTP id s24mr3186753lfb.545.1623428868376; Fri, 11 Jun 2021 09:27:48 -0700 (PDT) Received: from localhost.localdomain ([85.249.44.185]) by smtp.googlemail.com with ESMTPSA id l2sm777773lji.70.2021.06.11.09.27.47 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Fri, 11 Jun 2021 09:27:47 -0700 (PDT) From: Andrey Konovalov To: libcamera-devel@lists.libcamera.org, laurent.pinchart@ideasonboard.com Date: Fri, 11 Jun 2021 19:27:25 +0300 Message-Id: <20210611162726.824789-4-andrey.konovalov@linaro.org> X-Mailer: git-send-email 2.25.1 In-Reply-To: <20210611162726.824789-1-andrey.konovalov@linaro.org> References: <20210611162726.824789-1-andrey.konovalov@linaro.org> MIME-Version: 1.0 Subject: [libcamera-devel] [PATCH v3 3/4] qcam: viewfinder_gl: Add support for RAW12 packed formats X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Cc: morgan@casual-effects.com Errors-To: libcamera-devel-bounces@lists.libcamera.org Sender: "libcamera-devel" All the four Bayer orders are supported. The 4 LS bits of the 12-bit colour values are dropped as the RGBA format we convert into has only 8 bits per colour. Signed-off-by: Andrey Konovalov Reviewed-by: Laurent Pinchart --- src/qcam/assets/shader/bayer_1x_packed.frag | 43 +++++++++++++++--- src/qcam/viewfinder_gl.cpp | 48 ++++++++++++++++++--- 2 files changed, 80 insertions(+), 11 deletions(-) diff --git a/src/qcam/assets/shader/bayer_1x_packed.frag b/src/qcam/assets/shader/bayer_1x_packed.frag index 0a87c6db..d09c6fce 100644 --- a/src/qcam/assets/shader/bayer_1x_packed.frag +++ b/src/qcam/assets/shader/bayer_1x_packed.frag @@ -23,6 +23,40 @@ precision mediump float; #endif +/* + * These constants are used to select the bytes containing the HS part of + * the pixel value: + * BPP - bytes per pixel, + * THRESHOLD_L = fract(BPP) * 0.5 + 0.02 + * THRESHOLD_H = 1.0 - fract(BPP) * 1.5 + 0.02 + * Let X is the x coordinate in the texture measured in bytes (so that the + * range is from 0 to (stride_-1)) aligned on the nearest pixel. + * E.g. for RAW10P: + * -------------+-------------------+-------------------+-- + * pixel No | 0 1 2 3 | 4 5 6 7 | ... + * -------------+-------------------+-------------------+-- + * byte offset | 0 1 2 3 4 | 5 6 7 8 9 | ... + * -------------+-------------------+-------------------+-- + * X | 0.0 1.25 2.5 3.75 | 5.0 6.25 7.5 8.75 | ... + * -------------+-------------------+-------------------+-- + * If fract(X) < THRESHOLD_L then the previous byte contains the LS + * bits of the pixel values and needs to be skipped. + * If fract(X) > THRESHOLD_H then the next byte contains the LS bits + * of the pixel values and needs to be skipped. + */ +#if defined(RAW10P) +#define BPP 1.25 +#define THRESHOLD_L 0.14 +#define THRESHOLD_H 0.64 +#elif defined(RAW12P) +#define BPP 1.5 +#define THRESHOLD_L 0.27 +#define THRESHOLD_H 0.27 +#else +#error Invalid raw format +#endif + + varying vec2 textureOut; /* the texture size: tex_size.xy is in bytes, tex_size.zw is in pixels */ @@ -64,10 +98,7 @@ void main(void) * Add a small number (a few mantissa's LSBs) to avoid float * representation issues. Maybe paranoic. */ - center.x = BPP_X * center.z + 0.02; - - const float threshold_l = 0.127 /* fract(BPP_X) * 0.5 + 0.02 */; - const float threshold_h = 0.625 /* 1.0 - fract(BPP_X) * 1.5 */; + center.x = BPP * center.z + 0.02; float fract_x = fract(center.x); /* @@ -86,13 +117,13 @@ void main(void) * of the previous group of the pixels, move xcoords[0] one * byte back. */ - xcoords[0] += (fract_x < threshold_l) ? -tex_step.x : 0.0; + xcoords[0] += (fract_x < THRESHOLD_L) ? -tex_step.x : 0.0; /* * If xcoords[1] points at the byte containing the LS bits * of the current group of the pixels, move xcoords[1] one * byte forward. */ - xcoords[1] += (fract_x > threshold_h) ? tex_step.x : 0.0; + xcoords[1] += (fract_x > THRESHOLD_H) ? tex_step.x : 0.0; vec2 alternate = mod(center.zw + tex_bayer_first_red, 2.0); bool even_col = alternate.x < 1.0; diff --git a/src/qcam/viewfinder_gl.cpp b/src/qcam/viewfinder_gl.cpp index 44e410c9..dcfaf973 100644 --- a/src/qcam/viewfinder_gl.cpp +++ b/src/qcam/viewfinder_gl.cpp @@ -41,6 +41,11 @@ static const QList supportedFormats{ libcamera::formats::SGBRG10_CSI2P, libcamera::formats::SGRBG10_CSI2P, libcamera::formats::SRGGB10_CSI2P, + /* Raw Bayer 12-bit packed */ + libcamera::formats::SBGGR12_CSI2P, + libcamera::formats::SGBRG12_CSI2P, + libcamera::formats::SGRBG12_CSI2P, + libcamera::formats::SRGGB12_CSI2P, }; ViewFinderGL::ViewFinderGL(QWidget *parent) @@ -218,28 +223,56 @@ bool ViewFinderGL::selectFormat(const libcamera::PixelFormat &format) case libcamera::formats::SBGGR10_CSI2P: firstRed_.setX(1.0); firstRed_.setY(1.0); - fragmentShaderDefines_.append("#define BPP_X 1.25"); + fragmentShaderDefines_.append("#define RAW10P"); fragmentShaderFile_ = ":bayer_1x_packed.frag"; textureMinMagFilters_ = GL_NEAREST; break; case libcamera::formats::SGBRG10_CSI2P: firstRed_.setX(0.0); firstRed_.setY(1.0); - fragmentShaderDefines_.append("#define BPP_X 1.25"); + fragmentShaderDefines_.append("#define RAW10P"); fragmentShaderFile_ = ":bayer_1x_packed.frag"; textureMinMagFilters_ = GL_NEAREST; break; case libcamera::formats::SGRBG10_CSI2P: firstRed_.setX(1.0); firstRed_.setY(0.0); - fragmentShaderDefines_.append("#define BPP_X 1.25"); + fragmentShaderDefines_.append("#define RAW10P"); fragmentShaderFile_ = ":bayer_1x_packed.frag"; textureMinMagFilters_ = GL_NEAREST; break; case libcamera::formats::SRGGB10_CSI2P: firstRed_.setX(0.0); firstRed_.setY(0.0); - fragmentShaderDefines_.append("#define BPP_X 1.25"); + fragmentShaderDefines_.append("#define RAW10P"); + fragmentShaderFile_ = ":bayer_1x_packed.frag"; + textureMinMagFilters_ = GL_NEAREST; + break; + case libcamera::formats::SBGGR12_CSI2P: + firstRed_.setX(1.0); + firstRed_.setY(1.0); + fragmentShaderDefines_.append("#define RAW12P"); + fragmentShaderFile_ = ":bayer_1x_packed.frag"; + textureMinMagFilters_ = GL_NEAREST; + break; + case libcamera::formats::SGBRG12_CSI2P: + firstRed_.setX(0.0); + firstRed_.setY(1.0); + fragmentShaderDefines_.append("#define RAW12P"); + fragmentShaderFile_ = ":bayer_1x_packed.frag"; + textureMinMagFilters_ = GL_NEAREST; + break; + case libcamera::formats::SGRBG12_CSI2P: + firstRed_.setX(1.0); + firstRed_.setY(0.0); + fragmentShaderDefines_.append("#define RAW12P"); + fragmentShaderFile_ = ":bayer_1x_packed.frag"; + textureMinMagFilters_ = GL_NEAREST; + break; + case libcamera::formats::SRGGB12_CSI2P: + firstRed_.setX(0.0); + firstRed_.setY(0.0); + fragmentShaderDefines_.append("#define RAW12P"); fragmentShaderFile_ = ":bayer_1x_packed.frag"; textureMinMagFilters_ = GL_NEAREST; break; @@ -595,8 +628,13 @@ void ViewFinderGL::doRender() case libcamera::formats::SGBRG10_CSI2P: case libcamera::formats::SGRBG10_CSI2P: case libcamera::formats::SRGGB10_CSI2P: + case libcamera::formats::SBGGR12_CSI2P: + case libcamera::formats::SGBRG12_CSI2P: + case libcamera::formats::SGRBG12_CSI2P: + case libcamera::formats::SRGGB12_CSI2P: /* - * Packed raw Bayer 10-bit formats are stored in GL_RED texture. + * Packed raw Bayer 10-bit and 12-bit formats are stored in + * GL_RED texture. * The texture width is equal to the stride. */ glActiveTexture(GL_TEXTURE0); From patchwork Fri Jun 11 16:27:26 2021 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Andrey Konovalov X-Patchwork-Id: 12576 Return-Path: X-Original-To: parsemail@patchwork.libcamera.org Delivered-To: parsemail@patchwork.libcamera.org Received: from lancelot.ideasonboard.com (lancelot.ideasonboard.com [92.243.16.209]) by patchwork.libcamera.org (Postfix) with ESMTPS id 209A0C320B for ; Fri, 11 Jun 2021 16:27:53 +0000 (UTC) Received: from lancelot.ideasonboard.com (localhost [IPv6:::1]) by lancelot.ideasonboard.com (Postfix) with ESMTP id D229E6892E; Fri, 11 Jun 2021 18:27:52 +0200 (CEST) Authentication-Results: lancelot.ideasonboard.com; dkim=fail reason="signature verification failed" (2048-bit key; unprotected) header.d=linaro.org header.i=@linaro.org header.b="s02B/qfo"; dkim-atps=neutral Received: from mail-lf1-x134.google.com (mail-lf1-x134.google.com [IPv6:2a00:1450:4864:20::134]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id A77CA68939 for ; Fri, 11 Jun 2021 18:27:50 +0200 (CEST) Received: by mail-lf1-x134.google.com with SMTP id bp38so9458063lfb.0 for ; Fri, 11 Jun 2021 09:27:50 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=linaro.org; s=google; h=from:to:cc:subject:date:message-id:in-reply-to:references :mime-version:content-transfer-encoding; bh=v1yLVlRyd9h1kPxiadXi+BkUjG64afCsCJ0wPzLg6w8=; b=s02B/qfotlj8wbHWdX0T5qb7X13do/eE6IhBPkm09oRJKbOOf6BdliEv+wPWGlNNnb /uvS0Ldi7X+BrpD8UnEz2tUORm3aaH3nCGqOkujW0BqfxlcLjr1O1ceIKQPW8JvlxwTa DxKc08lEsR49mmVbYfefRv7J4WxSsBPwYA590s686nhKF1hgGIoGLsXmuzOhrvg93KdQ WFPmXKjUv5lmPZv/YG/6nptbvb6n6xpxMlfC0MfVClxWivXDYBkHDJVjzWfFewppA0I0 4xif7sgllLHoEAvUzyuNdNBnmiOTg7iv0lzsYU6fkJzEwHLffV2yaYx74DpFAoSDjbcc 8qfg== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20161025; h=x-gm-message-state:from:to:cc:subject:date:message-id:in-reply-to :references:mime-version:content-transfer-encoding; bh=v1yLVlRyd9h1kPxiadXi+BkUjG64afCsCJ0wPzLg6w8=; b=SfhqMpeq+fFBP24j6EDSTs4ouP1qWS3Mtl7NlYWLFek4IiKE5vYE3qer4REpCc29Kp 2PErXfRzsFpOQHtR9EjMu7WyO2+kBv0laczdtH6BoaNaMVUIecKR/wPKrk3Y24dYA3KM n9+pEKQZwJxe+EjA0Ccv8z76r+5bB9s59yPvCqbGDpeoGuC1PMUMT37lkS+3S6aD9FL0 4OKspBmgblvr089oow6E6qG5lG4u7lFNjssVKa4e/3j9M2Djz4rD9htd0mcx3RKUWx25 0buQDX5iW6KXte5p+K8FMxwZNTUFoCWqRWb7QSLtlrwjgkXPEq0otRiIta2kl6TZgbk0 /Phw== X-Gm-Message-State: AOAM5310b1gp/HeWnpFHy/myJi6Rl/FcxlpsLS0Q99/t/MHWfgSd6Lcb wpaSGvozDhRsGfaW+iFXZoFOzCV7dUpaJA== X-Google-Smtp-Source: ABdhPJwwZzq9qqH1Awjl/hoV8m8GBwGcAuc6cvIaGdfZ/xB40qnxwAD/h7g7+DJ/QlPFZXFuQWORVA== X-Received: by 2002:ac2:544a:: with SMTP id d10mr3213024lfn.656.1623428870062; Fri, 11 Jun 2021 09:27:50 -0700 (PDT) Received: from localhost.localdomain ([85.249.44.185]) by smtp.googlemail.com with ESMTPSA id l2sm777773lji.70.2021.06.11.09.27.49 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Fri, 11 Jun 2021 09:27:49 -0700 (PDT) From: Andrey Konovalov To: libcamera-devel@lists.libcamera.org, laurent.pinchart@ideasonboard.com Date: Fri, 11 Jun 2021 19:27:26 +0300 Message-Id: <20210611162726.824789-5-andrey.konovalov@linaro.org> X-Mailer: git-send-email 2.25.1 In-Reply-To: <20210611162726.824789-1-andrey.konovalov@linaro.org> References: <20210611162726.824789-1-andrey.konovalov@linaro.org> MIME-Version: 1.0 Subject: [libcamera-devel] [PATCH v3 4/4] qcam: viewfinder_gl: Add support for RAW8 Bayer formats X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Cc: morgan@casual-effects.com Errors-To: libcamera-devel-bounces@lists.libcamera.org Sender: "libcamera-devel" All the four Bayer orders are supported. The texture coordinates passed to the fragment shader are ajusted to point to the nearest pixel in the image. This prevents artifacts when the image is scaled from the frame resolution to the window size. Signed-off-by: Andrey Konovalov --- src/qcam/assets/shader/bayer_8.frag | 136 ++++++++++++++++++++++++++++ src/qcam/assets/shader/shaders.qrc | 1 + src/qcam/viewfinder_gl.cpp | 37 +++++++- 3 files changed, 172 insertions(+), 2 deletions(-) create mode 100644 src/qcam/assets/shader/bayer_8.frag diff --git a/src/qcam/assets/shader/bayer_8.frag b/src/qcam/assets/shader/bayer_8.frag new file mode 100644 index 00000000..d93ef1da --- /dev/null +++ b/src/qcam/assets/shader/bayer_8.frag @@ -0,0 +1,136 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* +From http://jgt.akpeters.com/papers/McGuire08/ + +Efficient, High-Quality Bayer Demosaic Filtering on GPUs + +Morgan McGuire + +This paper appears in issue Volume 13, Number 4. +--------------------------------------------------------- +Copyright (c) 2008, Morgan McGuire. All rights reserved. + + +Modified by Linaro Ltd to integrate it into libcamera, and to +fix the artifacts due to pixel coordinates interpolation. +Copyright (C) 2021, Linaro +*/ + +//Pixel Shader + +varying vec2 textureOut; + +/* The texture size: tex_size.xy is in bytes, tex_size.zw is in pixels */ +uniform vec4 tex_size; +uniform vec2 tex_step; + +/** Pixel position of the first red pixel in the */ +/** Bayer pattern. [{0,1}, {0, 1}]*/ +uniform vec2 tex_bayer_first_red; + +/** Monochrome RGBA or GL_LUMINANCE Bayer encoded texture.*/ +uniform sampler2D tex_raw; + +void main(void) { + #define fetch(x, y) texture2D(tex_raw, vec2(x, y)).r + + /** .xy = Pixel being sampled in the fragment shader on the range [0, 1] + .zw = ...on the range [0, sourceSize], offset by firstRed */ + vec4 center; + + /** center.x + (-2/w, -1/w, 1/w, 2/w); These are the x-positions */ + /** of the adjacent pixels.*/ + vec4 xCoord; + + /** center.y + (-2/h, -1/h, 1/h, 2/h); These are the y-positions */ + /** of the adjacent pixels.*/ + vec4 yCoord; + + /* Align the center coordinates to the nearest pixel */ + center.zw = floor(textureOut * tex_size.zw); + center.xy = center.zw * tex_step; + center.zw += tex_bayer_first_red; + + xCoord = center.x + vec4(-2.0 * tex_step.x, + -tex_step.x, tex_step.x, 2.0 * tex_step.x); + yCoord = center.y + vec4(-2.0 * tex_step.y, + -tex_step.y, tex_step.y, 2.0 * tex_step.y); + + float C = texture2D(tex_raw, center.xy).r; // ( 0, 0) + const vec4 kC = vec4( 4.0, 6.0, 5.0, 5.0) / 8.0; + + // Determine which of four types of pixels we are on. + vec2 alternate = mod(floor(center.zw), 2.0); + + vec4 Dvec = vec4( + fetch(xCoord[1], yCoord[1]), // (-1,-1) + fetch(xCoord[1], yCoord[2]), // (-1, 1) + fetch(xCoord[2], yCoord[1]), // ( 1,-1) + fetch(xCoord[2], yCoord[2])); // ( 1, 1) + + vec4 PATTERN = (kC.xyz * C).xyzz; + + // Can also be a dot product with (1,1,1,1) on hardware where that is + // specially optimized. + // Equivalent to: D = Dvec[0] + Dvec[1] + Dvec[2] + Dvec[3]; + Dvec.xy += Dvec.zw; + Dvec.x += Dvec.y; + + vec4 value = vec4( + fetch(center.x, yCoord[0]), // ( 0,-2) + fetch(center.x, yCoord[1]), // ( 0,-1) + fetch(xCoord[0], center.y), // (-2, 0) + fetch(xCoord[1], center.y)); // (-1, 0) + + vec4 temp = vec4( + fetch(center.x, yCoord[3]), // ( 0, 2) + fetch(center.x, yCoord[2]), // ( 0, 1) + fetch(xCoord[3], center.y), // ( 2, 0) + fetch(xCoord[2], center.y)); // ( 1, 0) + + // Even the simplest compilers should be able to constant-fold these to + // avoid the division. + // Note that on scalar processors these constants force computation of some + // identical products twice. + const vec4 kA = vec4(-1.0, -1.5, 0.5, -1.0) / 8.0; + const vec4 kB = vec4( 2.0, 0.0, 0.0, 4.0) / 8.0; + const vec4 kD = vec4( 0.0, 2.0, -1.0, -1.0) / 8.0; + + // Conserve constant registers and take advantage of free swizzle on load + #define kE (kA.xywz) + #define kF (kB.xywz) + + value += temp; + + // There are five filter patterns (identity, cross, checker, + // theta, phi). Precompute the terms from all of them and then + // use swizzles to assign to color channels. + // + // Channel Matches + // x cross (e.g., EE G) + // y checker (e.g., EE B) + // z theta (e.g., EO R) + // w phi (e.g., EO R) + #define A (value[0]) + #define B (value[1]) + #define D (Dvec.x) + #define E (value[2]) + #define F (value[3]) + + // Avoid zero elements. On a scalar processor this saves two MADDs + // and it has no effect on a vector processor. + PATTERN.yzw += (kD.yz * D).xyy; + + PATTERN += (kA.xyz * A).xyzx + (kE.xyw * E).xyxz; + PATTERN.xw += kB.xw * B; + PATTERN.xz += kF.xz * F; + + vec3 rgb = (alternate.y == 0.0) ? + ((alternate.x == 0.0) ? + vec3(C, PATTERN.xy) : + vec3(PATTERN.z, C, PATTERN.w)) : + ((alternate.x == 0.0) ? + vec3(PATTERN.w, C, PATTERN.z) : + vec3(PATTERN.yx, C)); + gl_FragColor = vec4(rgb, 1.0); +} diff --git a/src/qcam/assets/shader/shaders.qrc b/src/qcam/assets/shader/shaders.qrc index d76d65c5..79f44a30 100644 --- a/src/qcam/assets/shader/shaders.qrc +++ b/src/qcam/assets/shader/shaders.qrc @@ -5,6 +5,7 @@ YUV_2_planes.frag YUV_3_planes.frag YUV_packed.frag + bayer_8.frag bayer_1x_packed.frag identity.vert diff --git a/src/qcam/viewfinder_gl.cpp b/src/qcam/viewfinder_gl.cpp index dcfaf973..98b6b39d 100644 --- a/src/qcam/viewfinder_gl.cpp +++ b/src/qcam/viewfinder_gl.cpp @@ -36,6 +36,11 @@ static const QList supportedFormats{ libcamera::formats::RGBA8888, libcamera::formats::BGR888, libcamera::formats::RGB888, + /* Raw Bayer 8-bit */ + libcamera::formats::SBGGR8, + libcamera::formats::SGBRG8, + libcamera::formats::SGRBG8, + libcamera::formats::SRGGB8, /* Raw Bayer 10-bit packed */ libcamera::formats::SBGGR10_CSI2P, libcamera::formats::SGBRG10_CSI2P, @@ -220,6 +225,30 @@ bool ViewFinderGL::selectFormat(const libcamera::PixelFormat &format) fragmentShaderDefines_.append("#define RGB_PATTERN bgr"); fragmentShaderFile_ = ":RGB.frag"; break; + case libcamera::formats::SBGGR8: + firstRed_.setX(1.0); + firstRed_.setY(1.0); + fragmentShaderFile_ = ":bayer_8.frag"; + textureMinMagFilters_ = GL_NEAREST; + break; + case libcamera::formats::SGBRG8: + firstRed_.setX(0.0); + firstRed_.setY(1.0); + fragmentShaderFile_ = ":bayer_8.frag"; + textureMinMagFilters_ = GL_NEAREST; + break; + case libcamera::formats::SGRBG8: + firstRed_.setX(1.0); + firstRed_.setY(0.0); + fragmentShaderFile_ = ":bayer_8.frag"; + textureMinMagFilters_ = GL_NEAREST; + break; + case libcamera::formats::SRGGB8: + firstRed_.setX(0.0); + firstRed_.setY(0.0); + fragmentShaderFile_ = ":bayer_8.frag"; + textureMinMagFilters_ = GL_NEAREST; + break; case libcamera::formats::SBGGR10_CSI2P: firstRed_.setX(1.0); firstRed_.setY(1.0); @@ -624,6 +653,10 @@ void ViewFinderGL::doRender() shaderProgram_.setUniformValue(textureUniformY_, 0); break; + case libcamera::formats::SBGGR8: + case libcamera::formats::SGBRG8: + case libcamera::formats::SGRBG8: + case libcamera::formats::SRGGB8: case libcamera::formats::SBGGR10_CSI2P: case libcamera::formats::SGBRG10_CSI2P: case libcamera::formats::SGRBG10_CSI2P: @@ -633,8 +666,8 @@ void ViewFinderGL::doRender() case libcamera::formats::SGRBG12_CSI2P: case libcamera::formats::SRGGB12_CSI2P: /* - * Packed raw Bayer 10-bit and 12-bit formats are stored in - * GL_RED texture. + * Raw Bayer 8-bit, and packed raw Bayer 10-bit/12-bit formats + * are stored in GL_RED texture. * The texture width is equal to the stride. */ glActiveTexture(GL_TEXTURE0);