From patchwork Thu Jun 17 12:39:39 2021 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Andrey Konovalov X-Patchwork-Id: 12630 Return-Path: X-Original-To: parsemail@patchwork.libcamera.org Delivered-To: parsemail@patchwork.libcamera.org Received: from lancelot.ideasonboard.com (lancelot.ideasonboard.com [92.243.16.209]) by patchwork.libcamera.org (Postfix) with ESMTPS id F1CACC3218 for ; Thu, 17 Jun 2021 12:40:11 +0000 (UTC) Received: from lancelot.ideasonboard.com (localhost [IPv6:::1]) by lancelot.ideasonboard.com (Postfix) with ESMTP id B5BE46892D; Thu, 17 Jun 2021 14:40:11 +0200 (CEST) Authentication-Results: lancelot.ideasonboard.com; dkim=fail reason="signature verification failed" (2048-bit key; unprotected) header.d=linaro.org header.i=@linaro.org header.b="Qn3qEsyY"; dkim-atps=neutral Received: from mail-lf1-x12f.google.com (mail-lf1-x12f.google.com [IPv6:2a00:1450:4864:20::12f]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id 73B0F6050C for ; Thu, 17 Jun 2021 14:40:10 +0200 (CEST) Received: by mail-lf1-x12f.google.com with SMTP id a1so10070749lfr.12 for ; Thu, 17 Jun 2021 05:40:10 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=linaro.org; s=google; h=from:to:cc:subject:date:message-id:in-reply-to:references :mime-version:content-transfer-encoding; bh=mv2rnIZBXYiZJVAJqcAZOvGmMppMCyKTu4OnVWYQArQ=; b=Qn3qEsyYiHvvtIfhMLxPcjQBPVxiAoxmyTcqHf3xdGnq4cJAYQi3AX7BsRpK6wsRae 3NX76FwfNYmiNk06d0kPPK3x0G4ELsGhqiaELM9cqITkHPbzzboiidGiyzumgrtitN9W eeZEV/sDddwaRSjd5vkYIqp6rVPclgK//XrI85GglKt2v/AT+2Xl+Ubpox1EPWxYKbIA do3F3Wia2jeTmmbiRe+W32RzVlFaai8B12F3S8BTH8E4zQ95fkbUI+G4g48AQC2SYf1N OQ+2DAGyrhFLRfNT9ZgWKcK3qZDk/ZL3dzp01988qVG9wYXebm9bD73lQQ6Uiaj4GKq6 vCdQ== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20161025; h=x-gm-message-state:from:to:cc:subject:date:message-id:in-reply-to :references:mime-version:content-transfer-encoding; bh=mv2rnIZBXYiZJVAJqcAZOvGmMppMCyKTu4OnVWYQArQ=; b=oBXF6/9UzdZcdroDpuWuiV02A/PThpwahgp8BptsMVkYvr5qAPoMTfCTMopLmttFiF UVxLBIzFMhsyrZS2Yt/n0EavKV9ATECuWfzJ4cf27cckXSNKfHOnyDBaVI/x/c8+yCBB 0ZyFYJJIT4FvUCRxVN6G2+iUKtqo0350SqXVcHMvRui6BAgJA4xAwVGHZtMKwr4feRuI vJ+SKozBYv0/qpfllLc3YBk+QEv97AXoF5ZOBX2hhO5UHB3cS7JX5Wvk4xzsGHeYc2cg pgmz491dl7jHbRnFVfKMwoSJjfZ1wWGrG3V1VPVXy7wleSSVDleqD1jh3mCVYIAQwFId wBdA== X-Gm-Message-State: AOAM533RdYjcrQQXKZOh3nP32UdB9LT2ssiUXMp6PbV+v1bYypVr+gHN TFbwEZsBsBiHfvJUs5T4RcY/qKVGzaEAKA== X-Google-Smtp-Source: ABdhPJxpVlSfCa5tpu8Ss6yVcYhTg6RH7HBLQSnIw7EewiiuyKtrYXUBYkcdDjIvJBlDuk0EVBvt6w== X-Received: by 2002:a05:6512:2192:: with SMTP id b18mr3919926lft.422.1623933609002; Thu, 17 Jun 2021 05:40:09 -0700 (PDT) Received: from localhost.localdomain ([85.249.44.185]) by smtp.googlemail.com with ESMTPSA id t17sm268800ljt.1.2021.06.17.05.40.08 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Thu, 17 Jun 2021 05:40:08 -0700 (PDT) From: Andrey Konovalov To: libcamera-devel@lists.libcamera.org, laurent.pinchart@ideasonboard.com Date: Thu, 17 Jun 2021 15:39:39 +0300 Message-Id: <20210617123941.1095481-2-andrey.konovalov@linaro.org> X-Mailer: git-send-email 2.25.1 In-Reply-To: <20210617123941.1095481-1-andrey.konovalov@linaro.org> References: <20210617123941.1095481-1-andrey.konovalov@linaro.org> MIME-Version: 1.0 Subject: [libcamera-devel] [PATCH v4 1/3] qcam: viewfinder_gl: change uniform float tex_stepx to vec2 tex_step X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Cc: morgan@casual-effects.com Errors-To: libcamera-devel-bounces@lists.libcamera.org Sender: "libcamera-devel" In preparation to extend the supported formats, extend the tex_stepx uniform to cover the steps between texels in both horizontal and vertical directions. Signed-off-by: Andrey Konovalov Reviewed-by: Laurent Pinchart --- src/qcam/assets/shader/YUV_packed.frag | 8 ++++---- src/qcam/viewfinder_gl.cpp | 7 ++++--- src/qcam/viewfinder_gl.h | 2 +- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/src/qcam/assets/shader/YUV_packed.frag b/src/qcam/assets/shader/YUV_packed.frag index 224dfafe..d6efd4ce 100644 --- a/src/qcam/assets/shader/YUV_packed.frag +++ b/src/qcam/assets/shader/YUV_packed.frag @@ -12,7 +12,7 @@ precision mediump float; varying vec2 textureOut; uniform sampler2D tex_y; -uniform float tex_stepx; +uniform vec2 tex_step; void main(void) { @@ -49,10 +49,10 @@ void main(void) * a = fract(x) * 2 - 1 if fract(x) >= 0.5 */ vec2 pos = textureOut; - float f_x = fract(pos.x / tex_stepx); + float f_x = fract(pos.x / tex_step.x); - vec4 left = texture2D(tex_y, vec2(pos.x - f_x * tex_stepx, pos.y)); - vec4 right = texture2D(tex_y, vec2(pos.x + (1.0 - f_x) * tex_stepx , pos.y)); + vec4 left = texture2D(tex_y, vec2(pos.x - f_x * tex_step.x, pos.y)); + vec4 right = texture2D(tex_y, vec2(pos.x + (1.0 - f_x) * tex_step.x , pos.y)); #if defined(YUV_PATTERN_UYVY) float y_left = mix(left.g, left.a, f_x * 2.0); diff --git a/src/qcam/viewfinder_gl.cpp b/src/qcam/viewfinder_gl.cpp index 5d9b442e..ff719418 100644 --- a/src/qcam/viewfinder_gl.cpp +++ b/src/qcam/viewfinder_gl.cpp @@ -289,7 +289,7 @@ bool ViewFinderGL::createFragmentShader() textureUniformY_ = shaderProgram_.uniformLocation("tex_y"); textureUniformU_ = shaderProgram_.uniformLocation("tex_u"); textureUniformV_ = shaderProgram_.uniformLocation("tex_v"); - textureUniformStepX_ = shaderProgram_.uniformLocation("tex_stepx"); + textureUniformStep_ = shaderProgram_.uniformLocation("tex_step"); /* Create the textures. */ for (std::unique_ptr &texture : textures_) { @@ -508,8 +508,9 @@ void ViewFinderGL::doRender() * ([0, 1]). There are exactly width - 1 steps between the * leftmost and rightmost texels. */ - shaderProgram_.setUniformValue(textureUniformStepX_, - 1.0f / (size_.width() / 2 - 1)); + shaderProgram_.setUniformValue(textureUniformStep_, + 1.0f / (size_.width() / 2 - 1), + 1.0f /* not used */); break; case libcamera::formats::ABGR8888: diff --git a/src/qcam/viewfinder_gl.h b/src/qcam/viewfinder_gl.h index 150fa4ae..1b1faa91 100644 --- a/src/qcam/viewfinder_gl.h +++ b/src/qcam/viewfinder_gl.h @@ -85,7 +85,7 @@ private: GLuint textureUniformU_; GLuint textureUniformV_; GLuint textureUniformY_; - GLuint textureUniformStepX_; + GLuint textureUniformStep_; unsigned int horzSubSample_; unsigned int vertSubSample_; From patchwork Thu Jun 17 12:39:40 2021 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Andrey Konovalov X-Patchwork-Id: 12631 Return-Path: X-Original-To: parsemail@patchwork.libcamera.org Delivered-To: parsemail@patchwork.libcamera.org Received: from lancelot.ideasonboard.com (lancelot.ideasonboard.com [92.243.16.209]) by patchwork.libcamera.org (Postfix) with ESMTPS id 5456BC3218 for ; Thu, 17 Jun 2021 12:40:14 +0000 (UTC) Received: from lancelot.ideasonboard.com (localhost [IPv6:::1]) by lancelot.ideasonboard.com (Postfix) with ESMTP id 1BE6F68948; Thu, 17 Jun 2021 14:40:14 +0200 (CEST) Authentication-Results: lancelot.ideasonboard.com; dkim=fail reason="signature verification failed" (2048-bit key; unprotected) header.d=linaro.org header.i=@linaro.org header.b="rZurB7rB"; dkim-atps=neutral Received: from mail-lf1-x133.google.com (mail-lf1-x133.google.com [IPv6:2a00:1450:4864:20::133]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id 57EF068942 for ; Thu, 17 Jun 2021 14:40:12 +0200 (CEST) Received: by mail-lf1-x133.google.com with SMTP id r198so10233007lff.11 for ; Thu, 17 Jun 2021 05:40:12 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=linaro.org; s=google; h=from:to:cc:subject:date:message-id:in-reply-to:references :mime-version:content-transfer-encoding; bh=foglHs1dol6Vf01V9LnjbFUkDz3xND1z0WG78ltjV/Q=; b=rZurB7rB/P/0csxHPB2Z9c4W8T/sovO4ipLFIBBwizBLTjQ2CRCsA3KT/GQ2uksTbF BS8/2f8gEM1W5WuunjqYApE3DRRgR/GnSwaMtXkgE6yTCkXznV6KiQ/kGZFyoICKosPY jjqrC3OYkZA4UeVwhg2DNjGjvsIDNyKqw6EAhA7S2dXfYqtdjotL7ebkYC5NxuTl0MIf SRPpT6jxJHEmGfNcK8I0tur33QUL5VozYq6mWNEKfUqz8pq6rxa4GGedO9hDQrW1n0Iz EzcxDjw5oLnoLD8wPvQHaAxTKWo3mcuKcVYmX9UCGcxkqh7akpIfYdmHd/w3+uXyQ+RI Tylg== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20161025; h=x-gm-message-state:from:to:cc:subject:date:message-id:in-reply-to :references:mime-version:content-transfer-encoding; bh=foglHs1dol6Vf01V9LnjbFUkDz3xND1z0WG78ltjV/Q=; b=Up6AzclLrakc95w+wVx1+3+wg50LGSwkf4YItReW6tO05wVHdGOL/tiAP4Xe8DSPFq 17h7ulxI6HinWslLzk4ciX+ye8aIskzhca+jdBgmaeG6rJXBPJoUc1b5UI5wFC4SVYzC GXEmQua9ihHRmguq0mebmNMrErSgadZgfMTjNH0oQsOjwXTRqDTjuzVGdMlLilAOKNff 0mMtMYB5U5hcD6mWXIgQk1ndqoIyNv+Nac8hzerln15t9JR51fSk9c8QIQEyKe9a6WcG 3QddkUmDQL2F8BinG7WNVSsGxGoQuX+/StdBUIaac6WL7CEpgezjeZX3thohpgIgLjg0 Urxg== X-Gm-Message-State: AOAM533pZ24qX0NM2VT9z2DNi8j9VRDIc1zkL2D/0KQVxU0i3qUaeirI /MSnSSqVFGJ5NMSS98/3VgqDK0P0bx5TBg== X-Google-Smtp-Source: ABdhPJw8KAEK4eDYbdxvId2/yuyAUCHBUfXQJRLtCHfRxO8RIiHyf5Wr/ElVPJtBOcgR+N7qi5mjSw== X-Received: by 2002:ac2:4c9c:: with SMTP id d28mr3926264lfl.260.1623933611720; Thu, 17 Jun 2021 05:40:11 -0700 (PDT) Received: from localhost.localdomain ([85.249.44.185]) by smtp.googlemail.com with ESMTPSA id t17sm268800ljt.1.2021.06.17.05.40.10 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Thu, 17 Jun 2021 05:40:11 -0700 (PDT) From: Andrey Konovalov To: libcamera-devel@lists.libcamera.org, laurent.pinchart@ideasonboard.com Date: Thu, 17 Jun 2021 15:39:40 +0300 Message-Id: <20210617123941.1095481-3-andrey.konovalov@linaro.org> X-Mailer: git-send-email 2.25.1 In-Reply-To: <20210617123941.1095481-1-andrey.konovalov@linaro.org> References: <20210617123941.1095481-1-andrey.konovalov@linaro.org> MIME-Version: 1.0 Subject: [libcamera-devel] [PATCH v4 2/3] qcam: viewfinder_gl: Add shader to render packed RAW10 formats X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Cc: morgan@casual-effects.com Errors-To: libcamera-devel-bounces@lists.libcamera.org Sender: "libcamera-devel" The shader supports all 4 packed RAW10 variants. Simple bi-linear Bayer interpolation of nearest pixels is implemented. The 2 LS bits of the 10-bit colour values are dropped as the RGBA format we convert into has only 8 bits per colour. The texture coordinates passed to the fragment shader are ajusted to point to the nearest pixel in the image. This prevents artifacts when the image is scaled from the frame resolution to the window size. Signed-off-by: Andrey Konovalov --- src/qcam/assets/shader/bayer_1x_packed.frag | 185 ++++++++++++++++++++ src/qcam/assets/shader/shaders.qrc | 1 + src/qcam/viewfinder_gl.cpp | 78 ++++++++- src/qcam/viewfinder_gl.h | 9 + 4 files changed, 271 insertions(+), 2 deletions(-) create mode 100644 src/qcam/assets/shader/bayer_1x_packed.frag diff --git a/src/qcam/assets/shader/bayer_1x_packed.frag b/src/qcam/assets/shader/bayer_1x_packed.frag new file mode 100644 index 00000000..1df83849 --- /dev/null +++ b/src/qcam/assets/shader/bayer_1x_packed.frag @@ -0,0 +1,185 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Based on the code from http://jgt.akpeters.com/papers/McGuire08/ + * + * Efficient, High-Quality Bayer Demosaic Filtering on GPUs + * + * Morgan McGuire + * + * This paper appears in issue Volume 13, Number 4. + * --------------------------------------------------------- + * Copyright (c) 2008, Morgan McGuire. All rights reserved. + * + * + * Modified by Linaro Ltd for 10/12-bit packed vs 8-bit raw Bayer format, + * and for simpler demosaic algorithm. + * Copyright (C) 2020, Linaro + * + * bayer_1x_packed.frag - Fragment shader code for raw Bayer 10-bit and 12-bit + * packed formats + */ + +#ifdef GL_ES +precision mediump float; +#endif + +varying vec2 textureOut; + +/* the texture size in pixels */ +uniform vec2 tex_size; +uniform vec2 tex_step; +uniform vec2 tex_bayer_first_red; + +uniform sampler2D tex_raw; + +void main(void) +{ + vec3 rgb; + + /* + * center_bytes holds the coordinates of the MS byte of the pixel + * being sampled on the [0, stride-1/height-1] range. + * center_pixel holds the coordinates of the pixel being sampled + * on the [0, width/height-1] range. + */ + vec2 center_bytes; + vec2 center_pixel; + + /* + * x- and y-positions of the adjacent pixels on the [0, 1] range. + */ + vec2 xcoords; + vec2 ycoords; + + /* + * The coordinates passed to the shader in textureOut may point + * to a place in between the pixels if the texture format doesn't + * match the image format. In particulr, MIPI packed raw Bayer + * formats don't have a matching texture format. + * In this case align the coordinates to the left nearest pixel + * by hand. + */ + center_pixel = floor(textureOut * tex_size); + center_bytes.y = center_pixel.y; + + /* + * Add a small number (a few mantissa's LSBs) to avoid float + * representation issues. Maybe paranoic. + */ + center_bytes.x = BPP_X * center_pixel.x + 0.02; + + const float threshold_l = 0.127 /* fract(BPP_X) * 0.5 + 0.02 */; + const float threshold_h = 0.625 /* 1.0 - fract(BPP_X) * 1.5 */; + + float fract_x = fract(center_bytes.x); + + /* + * The below floor() call ensures that center_bytes.x points + * at one of the bytes representing the 8 higher bits of + * the pixel value, not at the byte containing the LS bits + * of the group of the pixels. + */ + center_bytes.x = floor(center_bytes.x); + center_bytes *= tex_step; + + xcoords = center_bytes.x + vec2(-tex_step.x, tex_step.x); + ycoords = center_bytes.y + vec2(-tex_step.y, tex_step.y); + + /* + * If xcoords[0] points at the byte containing the LS bits + * of the previous group of the pixels, move xcoords[0] one + * byte back. + */ + xcoords[0] += (fract_x < threshold_l) ? -tex_step.x : 0.0; + + /* + * If xcoords[1] points at the byte containing the LS bits + * of the current group of the pixels, move xcoords[1] one + * byte forward. + */ + xcoords[1] += (fract_x > threshold_h) ? tex_step.x : 0.0; + + vec2 alternate = mod(center_pixel.xy + tex_bayer_first_red, 2.0); + bool even_col = alternate.x < 1.0; + bool even_row = alternate.y < 1.0; + + /* + * We need to sample the central pixel and the ones with offset + * of -1 to +1 pixel in both X and Y directions. Let's name these + * pixels as below, where C is the central pixel: + * + * +----+----+----+----+ + * | \ x| | | | + * |y \ | -1 | 0 | +1 | + * +----+----+----+----+ + * | +1 | D2 | A1 | D3 | + * +----+----+----+----+ + * | 0 | B0 | C | B1 | + * +----+----+----+----+ + * | -1 | D0 | A0 | D1 | + * +----+----+----+----+ + * + * In the below equations (0,-1).r means "r component of the texel + * shifted by -tex_step.y from the center_bytes one" etc. + * + * In the even row / even column (EE) case the colour values are: + * R = C = (0,0).r, + * G = (A0 + A1 + B0 + B1) / 4.0 = + * ( (0,-1).r + (0,1).r + (-1,0).r + (1,0).r ) / 4.0, + * B = (D0 + D1 + D2 + D3) / 4.0 = + * ( (-1,-1).r + (1,-1).r + (-1,1).r + (1,1).r ) / 4.0 + * + * For even row / odd column (EO): + * R = (B0 + B1) / 2.0 = ( (-1,0).r + (1,0).r ) / 2.0, + * G = C = (0,0).r, + * B = (A0 + A1) / 2.0 = ( (0,-1).r + (0,1).r ) / 2.0 + * + * For odd row / even column (OE): + * R = (A0 + A1) / 2.0 = ( (0,-1).r + (0,1).r ) / 2.0, + * G = C = (0,0).r, + * B = (B0 + B1) / 2.0 = ( (-1,0).r + (1,0).r ) / 2.0 + * + * For odd row / odd column (OO): + * R = (D0 + D1 + D2 + D3) / 4.0 = + * ( (-1,-1).r + (1,-1).r + (-1,1).r + (1,1).r ) / 4.0, + * G = (A0 + A1 + B0 + B1) / 4.0 = + * ( (0,-1).r + (0,1).r + (-1,0).r + (1,0).r ) / 4.0, + * B = C = (0,0).r + */ + + /* + * Fetch the values and precalculate the terms: + * patterns.x = (A0 + A1) / 2.0 + * patterns.y = (B0 + B1) / 2.0 + * patterns.z = (A0 + A1 + B0 + B1) / 4.0 + * patterns.w = (D0 + D1 + D2 + D3) / 4.0 + */ + #define fetch(x, y) texture2D(tex_raw, vec2(x, y)).r + + float C = texture2D(tex_raw, center_bytes).r; + vec4 patterns = vec4( + fetch(center_bytes.x, ycoords[0]), /* A0: (0,-1) */ + fetch(xcoords[0], center_bytes.y), /* B0: (-1,0) */ + fetch(xcoords[0], ycoords[0]), /* D0: (-1,-1) */ + fetch(xcoords[1], ycoords[0])); /* D1: (1,-1) */ + vec4 temp = vec4( + fetch(center_bytes.x, ycoords[1]), /* A1: (0,1) */ + fetch(xcoords[1], center_bytes.y), /* B1: (1,0) */ + fetch(xcoords[1], ycoords[1]), /* D3: (1,1) */ + fetch(xcoords[0], ycoords[1])); /* D2: (-1,1) */ + patterns = (patterns + temp) * 0.5; + /* .x = (A0 + A1) / 2.0, .y = (B0 + B1) / 2.0 */ + /* .z = (D0 + D3) / 2.0, .w = (D1 + D2) / 2.0 */ + patterns.w = (patterns.z + patterns.w) * 0.5; + patterns.z = (patterns.x + patterns.y) * 0.5; + + rgb = even_col ? + (even_row ? + vec3(C, patterns.zw) : + vec3(patterns.x, C, patterns.y)) : + (even_row ? + vec3(patterns.y, C, patterns.x) : + vec3(patterns.wz, C)); + + gl_FragColor = vec4(rgb, 1.0); +} diff --git a/src/qcam/assets/shader/shaders.qrc b/src/qcam/assets/shader/shaders.qrc index 8a8f9de1..d76d65c5 100644 --- a/src/qcam/assets/shader/shaders.qrc +++ b/src/qcam/assets/shader/shaders.qrc @@ -5,6 +5,7 @@ YUV_2_planes.frag YUV_3_planes.frag YUV_packed.frag + bayer_1x_packed.frag identity.vert diff --git a/src/qcam/viewfinder_gl.cpp b/src/qcam/viewfinder_gl.cpp index ff719418..ffbbc6c5 100644 --- a/src/qcam/viewfinder_gl.cpp +++ b/src/qcam/viewfinder_gl.cpp @@ -36,6 +36,11 @@ static const QList supportedFormats{ libcamera::formats::RGBA8888, libcamera::formats::BGR888, libcamera::formats::RGB888, + /* Raw Bayer 10-bit packed */ + libcamera::formats::SBGGR10_CSI2P, + libcamera::formats::SGBRG10_CSI2P, + libcamera::formats::SGRBG10_CSI2P, + libcamera::formats::SRGGB10_CSI2P, }; ViewFinderGL::ViewFinderGL(QWidget *parent) @@ -106,6 +111,10 @@ void ViewFinderGL::render(libcamera::FrameBuffer *buffer, MappedBuffer *map) renderComplete(buffer_); data_ = static_cast(map->memory); + /* + * \todo Get the stride from the buffer instead of computing it naively + */ + stride_ = buffer->metadata().planes[0].bytesused / size_.height(); update(); buffer_ = buffer; } @@ -114,6 +123,9 @@ bool ViewFinderGL::selectFormat(const libcamera::PixelFormat &format) { bool ret = true; + /* Set min/mag filters to GL_LINEAR by default. */ + textureMinMagFilters_ = GL_LINEAR; + fragmentShaderDefines_.clear(); switch (format) { @@ -203,6 +215,34 @@ bool ViewFinderGL::selectFormat(const libcamera::PixelFormat &format) fragmentShaderDefines_.append("#define RGB_PATTERN bgr"); fragmentShaderFile_ = ":RGB.frag"; break; + case libcamera::formats::SBGGR10_CSI2P: + firstRed_.setX(1.0); + firstRed_.setY(1.0); + fragmentShaderDefines_.append("#define BPP_X 1.25"); + fragmentShaderFile_ = ":bayer_1x_packed.frag"; + textureMinMagFilters_ = GL_NEAREST; + break; + case libcamera::formats::SGBRG10_CSI2P: + firstRed_.setX(0.0); + firstRed_.setY(1.0); + fragmentShaderDefines_.append("#define BPP_X 1.25"); + fragmentShaderFile_ = ":bayer_1x_packed.frag"; + textureMinMagFilters_ = GL_NEAREST; + break; + case libcamera::formats::SGRBG10_CSI2P: + firstRed_.setX(1.0); + firstRed_.setY(0.0); + fragmentShaderDefines_.append("#define BPP_X 1.25"); + fragmentShaderFile_ = ":bayer_1x_packed.frag"; + textureMinMagFilters_ = GL_NEAREST; + break; + case libcamera::formats::SRGGB10_CSI2P: + firstRed_.setX(0.0); + firstRed_.setY(0.0); + fragmentShaderDefines_.append("#define BPP_X 1.25"); + fragmentShaderFile_ = ":bayer_1x_packed.frag"; + textureMinMagFilters_ = GL_NEAREST; + break; default: ret = false; qWarning() << "[ViewFinderGL]:" @@ -290,6 +330,8 @@ bool ViewFinderGL::createFragmentShader() textureUniformU_ = shaderProgram_.uniformLocation("tex_u"); textureUniformV_ = shaderProgram_.uniformLocation("tex_v"); textureUniformStep_ = shaderProgram_.uniformLocation("tex_step"); + textureUniformSize_ = shaderProgram_.uniformLocation("tex_size"); + textureUniformBayerFirstRed_ = shaderProgram_.uniformLocation("tex_bayer_first_red"); /* Create the textures. */ for (std::unique_ptr &texture : textures_) { @@ -306,8 +348,10 @@ bool ViewFinderGL::createFragmentShader() void ViewFinderGL::configureTexture(QOpenGLTexture &texture) { glBindTexture(GL_TEXTURE_2D, texture.textureId()); - glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR); - glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, + textureMinMagFilters_); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, + textureMinMagFilters_); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); } @@ -547,6 +591,36 @@ void ViewFinderGL::doRender() shaderProgram_.setUniformValue(textureUniformY_, 0); break; + case libcamera::formats::SBGGR10_CSI2P: + case libcamera::formats::SGBRG10_CSI2P: + case libcamera::formats::SGRBG10_CSI2P: + case libcamera::formats::SRGGB10_CSI2P: + /* + * Packed raw Bayer 10-bit formats are stored in GL_RED texture. + * The texture width is equal to the stride. + */ + glActiveTexture(GL_TEXTURE0); + configureTexture(*textures_[0]); + glTexImage2D(GL_TEXTURE_2D, + 0, + GL_RED, + stride_, + size_.height(), + 0, + GL_RED, + GL_UNSIGNED_BYTE, + data_); + shaderProgram_.setUniformValue(textureUniformY_, 0); + shaderProgram_.setUniformValue(textureUniformBayerFirstRed_, + firstRed_); + shaderProgram_.setUniformValue(textureUniformSize_, + size_.width(), /* in pixels */ + size_.height()); + shaderProgram_.setUniformValue(textureUniformStep_, + 1.0f / (stride_ - 1), + 1.0f / (size_.height() - 1)); + break; + default: break; }; diff --git a/src/qcam/viewfinder_gl.h b/src/qcam/viewfinder_gl.h index 1b1faa91..508155b1 100644 --- a/src/qcam/viewfinder_gl.h +++ b/src/qcam/viewfinder_gl.h @@ -66,6 +66,7 @@ private: libcamera::FrameBuffer *buffer_; libcamera::PixelFormat format_; QSize size_; + unsigned int stride_; unsigned char *data_; /* Shaders */ @@ -81,6 +82,9 @@ private: /* Textures */ std::array, 3> textures_; + /* Common texture parameters */ + GLuint textureMinMagFilters_; + /* YUV texture parameters */ GLuint textureUniformU_; GLuint textureUniformV_; @@ -89,6 +93,11 @@ private: unsigned int horzSubSample_; unsigned int vertSubSample_; + /* Raw Bayer texture parameters */ + GLuint textureUniformSize_; + GLuint textureUniformBayerFirstRed_; + QPointF firstRed_; + QMutex mutex_; /* Prevent concurrent access to image_ */ }; From patchwork Thu Jun 17 12:39:41 2021 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Andrey Konovalov X-Patchwork-Id: 12632 Return-Path: X-Original-To: parsemail@patchwork.libcamera.org Delivered-To: parsemail@patchwork.libcamera.org Received: from lancelot.ideasonboard.com (lancelot.ideasonboard.com [92.243.16.209]) by patchwork.libcamera.org (Postfix) with ESMTPS id 2E66AC3218 for ; Thu, 17 Jun 2021 12:40:20 +0000 (UTC) Received: from lancelot.ideasonboard.com (localhost [IPv6:::1]) by lancelot.ideasonboard.com (Postfix) with ESMTP id C8DF668944; Thu, 17 Jun 2021 14:40:19 +0200 (CEST) Authentication-Results: lancelot.ideasonboard.com; dkim=fail reason="signature verification failed" (2048-bit key; unprotected) header.d=linaro.org header.i=@linaro.org header.b="s6+PCWVZ"; dkim-atps=neutral Received: from mail-lf1-x135.google.com (mail-lf1-x135.google.com [IPv6:2a00:1450:4864:20::135]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id 85D306050C for ; Thu, 17 Jun 2021 14:40:18 +0200 (CEST) Received: by mail-lf1-x135.google.com with SMTP id d16so2849343lfn.3 for ; Thu, 17 Jun 2021 05:40:18 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=linaro.org; s=google; h=from:to:cc:subject:date:message-id:in-reply-to:references :mime-version:content-transfer-encoding; bh=fQXv1ymDgOZC6pJ6+ca2d6fUUKAU03cUbgX/J6RCUHI=; b=s6+PCWVZijNAXo6uKrigb0X9QTYg1TQNXLWmhcbgdUUJm1HeHezw3C7uKaJ/NinhzO i3kALAxor/MlhXYtZWVTMGuMzsZ1t0Y1Ziulay7f94qJpk6ryiCiI1M8RVXfOwLgE8ea hRMetmkb7JHLtirYrsNvtMU2bRrzRx/E1LgxprUt4VCUq1ndpmLw/HIk/jRwuzgYCEQK J8OI4WPdO3I+oeWVRsM/vTAl3Y5h300fEgxCHQd/MUIWm50PS1F4Nd0vTfAifp/BCbmW 6Wii2yqEYKAm+SDkGekzamlvEdqURWHVj+0bWTo78MELctLqlEieDo6kw43DXCIfThl7 bkLw== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20161025; h=x-gm-message-state:from:to:cc:subject:date:message-id:in-reply-to :references:mime-version:content-transfer-encoding; bh=fQXv1ymDgOZC6pJ6+ca2d6fUUKAU03cUbgX/J6RCUHI=; b=CEZTyaRucpK+PKjbAkTWPEPnT+vQEA7LULpHF1LN534NQuICo6xDCeUZFLiySxCV/V Loj+4IsjhHHQr8++8GQimExhs4I26M199B1d65iAS9O3wSwEcv8m8ppoIrwZ2Wi3Cm52 HIzeSRswJ2L5Ufwd3TYtyv2FvD+KO+uM7O8sx2CRsWMgTYo4WJYAEz+ZH+VKAcI98nfG zVRL5iEOhF73pyMc7ixIjZABSsTa6iVDT5NnZPlIxbnDTJBdvu0Vx7Ya2z+TU6Jd++Jm z+UXDP/QgNhJBplByPeeS3wHxNUke/RSXydEKVFja/3S92p1R/MBYeCQJxxAafUku8uo Pn6g== X-Gm-Message-State: AOAM532gziY7snUk1KEizg1+ri4KoEuvAelJVgGAtIr1APONmwWUPA2y SLZfmNjXyQlmFf6dWH8BS3WJIc8Ja8OTIA== X-Google-Smtp-Source: ABdhPJyJBXo4G8YxQ8uelQOs1d5lqs2H2ZUnPbF4sB7fcFmDpv9XXVL95+qSzJ/CjjtDpokzzXUH4Q== X-Received: by 2002:a19:ac45:: with SMTP id r5mr4088367lfc.449.1623933617943; Thu, 17 Jun 2021 05:40:17 -0700 (PDT) Received: from localhost.localdomain ([85.249.44.185]) by smtp.googlemail.com with ESMTPSA id t17sm268800ljt.1.2021.06.17.05.40.17 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Thu, 17 Jun 2021 05:40:17 -0700 (PDT) From: Andrey Konovalov To: libcamera-devel@lists.libcamera.org, laurent.pinchart@ideasonboard.com Date: Thu, 17 Jun 2021 15:39:41 +0300 Message-Id: <20210617123941.1095481-4-andrey.konovalov@linaro.org> X-Mailer: git-send-email 2.25.1 In-Reply-To: <20210617123941.1095481-1-andrey.konovalov@linaro.org> References: <20210617123941.1095481-1-andrey.konovalov@linaro.org> MIME-Version: 1.0 Subject: [libcamera-devel] [PATCH v4 3/3] qcam: viewfinder_gl: Add support for RAW12 packed formats X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Cc: morgan@casual-effects.com Errors-To: libcamera-devel-bounces@lists.libcamera.org Sender: "libcamera-devel" All the four Bayer orders are supported. The 4 LS bits of the 12-bit colour values are dropped as the RGBA format we convert into has only 8 bits per colour. Signed-off-by: Andrey Konovalov Reviewed-by: Laurent Pinchart --- src/qcam/assets/shader/bayer_1x_packed.frag | 43 +++++++++++++++--- src/qcam/viewfinder_gl.cpp | 48 ++++++++++++++++++--- 2 files changed, 80 insertions(+), 11 deletions(-) diff --git a/src/qcam/assets/shader/bayer_1x_packed.frag b/src/qcam/assets/shader/bayer_1x_packed.frag index 1df83849..b2aa329e 100644 --- a/src/qcam/assets/shader/bayer_1x_packed.frag +++ b/src/qcam/assets/shader/bayer_1x_packed.frag @@ -23,6 +23,40 @@ precision mediump float; #endif +/* + * These constants are used to select the bytes containing the HS part of + * the pixel value: + * BPP - bytes per pixel, + * THRESHOLD_L = fract(BPP) * 0.5 + 0.02 + * THRESHOLD_H = 1.0 - fract(BPP) * 1.5 + 0.02 + * Let X is the x coordinate in the texture measured in bytes (so that the + * range is from 0 to (stride_-1)) aligned on the nearest pixel. + * E.g. for RAW10P: + * -------------+-------------------+-------------------+-- + * pixel No | 0 1 2 3 | 4 5 6 7 | ... + * -------------+-------------------+-------------------+-- + * byte offset | 0 1 2 3 4 | 5 6 7 8 9 | ... + * -------------+-------------------+-------------------+-- + * X | 0.0 1.25 2.5 3.75 | 5.0 6.25 7.5 8.75 | ... + * -------------+-------------------+-------------------+-- + * If fract(X) < THRESHOLD_L then the previous byte contains the LS + * bits of the pixel values and needs to be skipped. + * If fract(X) > THRESHOLD_H then the next byte contains the LS bits + * of the pixel values and needs to be skipped. + */ +#if defined(RAW10P) +#define BPP 1.25 +#define THRESHOLD_L 0.14 +#define THRESHOLD_H 0.64 +#elif defined(RAW12P) +#define BPP 1.5 +#define THRESHOLD_L 0.27 +#define THRESHOLD_H 0.27 +#else +#error Invalid raw format +#endif + + varying vec2 textureOut; /* the texture size in pixels */ @@ -66,10 +100,7 @@ void main(void) * Add a small number (a few mantissa's LSBs) to avoid float * representation issues. Maybe paranoic. */ - center_bytes.x = BPP_X * center_pixel.x + 0.02; - - const float threshold_l = 0.127 /* fract(BPP_X) * 0.5 + 0.02 */; - const float threshold_h = 0.625 /* 1.0 - fract(BPP_X) * 1.5 */; + center_bytes.x = BPP * center_pixel.x + 0.02; float fract_x = fract(center_bytes.x); @@ -90,14 +121,14 @@ void main(void) * of the previous group of the pixels, move xcoords[0] one * byte back. */ - xcoords[0] += (fract_x < threshold_l) ? -tex_step.x : 0.0; + xcoords[0] += (fract_x < THRESHOLD_L) ? -tex_step.x : 0.0; /* * If xcoords[1] points at the byte containing the LS bits * of the current group of the pixels, move xcoords[1] one * byte forward. */ - xcoords[1] += (fract_x > threshold_h) ? tex_step.x : 0.0; + xcoords[1] += (fract_x > THRESHOLD_H) ? tex_step.x : 0.0; vec2 alternate = mod(center_pixel.xy + tex_bayer_first_red, 2.0); bool even_col = alternate.x < 1.0; diff --git a/src/qcam/viewfinder_gl.cpp b/src/qcam/viewfinder_gl.cpp index ffbbc6c5..a6aa2b44 100644 --- a/src/qcam/viewfinder_gl.cpp +++ b/src/qcam/viewfinder_gl.cpp @@ -41,6 +41,11 @@ static const QList supportedFormats{ libcamera::formats::SGBRG10_CSI2P, libcamera::formats::SGRBG10_CSI2P, libcamera::formats::SRGGB10_CSI2P, + /* Raw Bayer 12-bit packed */ + libcamera::formats::SBGGR12_CSI2P, + libcamera::formats::SGBRG12_CSI2P, + libcamera::formats::SGRBG12_CSI2P, + libcamera::formats::SRGGB12_CSI2P, }; ViewFinderGL::ViewFinderGL(QWidget *parent) @@ -218,28 +223,56 @@ bool ViewFinderGL::selectFormat(const libcamera::PixelFormat &format) case libcamera::formats::SBGGR10_CSI2P: firstRed_.setX(1.0); firstRed_.setY(1.0); - fragmentShaderDefines_.append("#define BPP_X 1.25"); + fragmentShaderDefines_.append("#define RAW10P"); fragmentShaderFile_ = ":bayer_1x_packed.frag"; textureMinMagFilters_ = GL_NEAREST; break; case libcamera::formats::SGBRG10_CSI2P: firstRed_.setX(0.0); firstRed_.setY(1.0); - fragmentShaderDefines_.append("#define BPP_X 1.25"); + fragmentShaderDefines_.append("#define RAW10P"); fragmentShaderFile_ = ":bayer_1x_packed.frag"; textureMinMagFilters_ = GL_NEAREST; break; case libcamera::formats::SGRBG10_CSI2P: firstRed_.setX(1.0); firstRed_.setY(0.0); - fragmentShaderDefines_.append("#define BPP_X 1.25"); + fragmentShaderDefines_.append("#define RAW10P"); fragmentShaderFile_ = ":bayer_1x_packed.frag"; textureMinMagFilters_ = GL_NEAREST; break; case libcamera::formats::SRGGB10_CSI2P: firstRed_.setX(0.0); firstRed_.setY(0.0); - fragmentShaderDefines_.append("#define BPP_X 1.25"); + fragmentShaderDefines_.append("#define RAW10P"); + fragmentShaderFile_ = ":bayer_1x_packed.frag"; + textureMinMagFilters_ = GL_NEAREST; + break; + case libcamera::formats::SBGGR12_CSI2P: + firstRed_.setX(1.0); + firstRed_.setY(1.0); + fragmentShaderDefines_.append("#define RAW12P"); + fragmentShaderFile_ = ":bayer_1x_packed.frag"; + textureMinMagFilters_ = GL_NEAREST; + break; + case libcamera::formats::SGBRG12_CSI2P: + firstRed_.setX(0.0); + firstRed_.setY(1.0); + fragmentShaderDefines_.append("#define RAW12P"); + fragmentShaderFile_ = ":bayer_1x_packed.frag"; + textureMinMagFilters_ = GL_NEAREST; + break; + case libcamera::formats::SGRBG12_CSI2P: + firstRed_.setX(1.0); + firstRed_.setY(0.0); + fragmentShaderDefines_.append("#define RAW12P"); + fragmentShaderFile_ = ":bayer_1x_packed.frag"; + textureMinMagFilters_ = GL_NEAREST; + break; + case libcamera::formats::SRGGB12_CSI2P: + firstRed_.setX(0.0); + firstRed_.setY(0.0); + fragmentShaderDefines_.append("#define RAW12P"); fragmentShaderFile_ = ":bayer_1x_packed.frag"; textureMinMagFilters_ = GL_NEAREST; break; @@ -595,8 +628,13 @@ void ViewFinderGL::doRender() case libcamera::formats::SGBRG10_CSI2P: case libcamera::formats::SGRBG10_CSI2P: case libcamera::formats::SRGGB10_CSI2P: + case libcamera::formats::SBGGR12_CSI2P: + case libcamera::formats::SGBRG12_CSI2P: + case libcamera::formats::SGRBG12_CSI2P: + case libcamera::formats::SRGGB12_CSI2P: /* - * Packed raw Bayer 10-bit formats are stored in GL_RED texture. + * Packed raw Bayer 10-bit and 12-bit formats are stored in + * GL_RED texture. * The texture width is equal to the stride. */ glActiveTexture(GL_TEXTURE0);