From patchwork Thu May 27 10:55:07 2021 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Andrey Konovalov X-Patchwork-Id: 12448 Return-Path: X-Original-To: parsemail@patchwork.libcamera.org Delivered-To: parsemail@patchwork.libcamera.org Received: from lancelot.ideasonboard.com (lancelot.ideasonboard.com [92.243.16.209]) by patchwork.libcamera.org (Postfix) with ESMTPS id E8DF2C3203 for ; Thu, 27 May 2021 10:55:30 +0000 (UTC) Received: from lancelot.ideasonboard.com (localhost [IPv6:::1]) by lancelot.ideasonboard.com (Postfix) with ESMTP id 2F493602AD; Thu, 27 May 2021 12:55:29 +0200 (CEST) Authentication-Results: lancelot.ideasonboard.com; dkim=fail reason="signature verification failed" (2048-bit key; unprotected) header.d=linaro.org header.i=@linaro.org header.b="G1KipA81"; dkim-atps=neutral Received: from mail-lf1-x134.google.com (mail-lf1-x134.google.com [IPv6:2a00:1450:4864:20::134]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id 4C5F1602AA for ; Thu, 27 May 2021 12:55:26 +0200 (CEST) Received: by mail-lf1-x134.google.com with SMTP id a2so7328950lfc.9 for ; Thu, 27 May 2021 03:55:26 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=linaro.org; s=google; h=from:to:cc:subject:date:message-id:in-reply-to:references :mime-version:content-transfer-encoding; bh=1pJOzZy0cA+5PFPsdwnYX3JXmgdw7xElcszVRkfrj+Q=; b=G1KipA815odUSHkrusrLl7MC5/MvtmFW6OiIkYmHq5xOPp0f03i+2v8NfgLLbQEiRT 79o/DYQBl/bCXrBUvDcSH4DcrlQFUXKNkVQ72YFjziFbY//ZznoVksTfVwGDE8CLTNIZ tQi5BlUwARPfXOqkjMfOx37xy5iQwTIl0l8lVBo/14qkpZ04ewVqkB9iODVKZ6+uXm3r YCD79f05foO6y5A3Enx9Ujtbv0ZBI5NS3CJSGQ4yxAK9t8qw/HhPFnCqMzQl+0MDx3Cz 02+/ALg907acSVq9LCG98ZhzggOfCjNsomgjvhVyEyOR1J6U2yXWZfDynW5cLAsAq3sz dxSQ== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20161025; h=x-gm-message-state:from:to:cc:subject:date:message-id:in-reply-to :references:mime-version:content-transfer-encoding; bh=1pJOzZy0cA+5PFPsdwnYX3JXmgdw7xElcszVRkfrj+Q=; b=fLzsiaafRs38jy+9YQRXdc4pDbta9Blhmfdo622trZJk1koZ/eBXIStT8otWkVPeqM /4yC5ICGxMEncjY8rDofjFbz3cLPvjs32PAAPYYx+HgAAXfwAEN865aU1VlIJCAYOEoO 3cpL+GHF0L8C32/rfm8WR9T5c4eHqf3y95S7s6PPNCNE5jg/h3YKkF+8HsLuQVu0S1An JfIodHwXK5OJqJF4dbVHDSL67V1DSssI6wYna1cveHQ3TM8EsGqhqYGg9LyZiDefy9DH bAl5LDetccWfC9TKkkZgMX4WEpR64XGR6oLZGxoOXjvE2BQ4Hyphid/1gfNsD4Ff9B8p oiOA== X-Gm-Message-State: AOAM5308/x0qWRpRU/qGzYfEn1pL63IZouqsLD6iCNoXWJsrdYwzxxh6 nChaavnWCrLCCPf8PAJo2NlrEUPOPHJgcg== X-Google-Smtp-Source: ABdhPJx/Bl56fm8igrtOJI5+jHzYQ3gbAOoQtG2UNEsnYctsOTiFNhrv47p0ThZdEJtgglXeLec7aw== X-Received: by 2002:ac2:5a45:: with SMTP id r5mr2095818lfn.342.1622112925584; Thu, 27 May 2021 03:55:25 -0700 (PDT) Received: from localhost.localdomain ([85.249.41.56]) by smtp.googlemail.com with ESMTPSA id h23sm170148lfc.52.2021.05.27.03.55.24 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Thu, 27 May 2021 03:55:25 -0700 (PDT) From: Andrey Konovalov To: libcamera-devel@lists.libcamera.org, laurent.pinchart@ideasonboard.com Date: Thu, 27 May 2021 13:55:07 +0300 Message-Id: <20210527105511.447089-2-andrey.konovalov@linaro.org> X-Mailer: git-send-email 2.25.1 In-Reply-To: <20210527105511.447089-1-andrey.konovalov@linaro.org> References: <20210527105511.447089-1-andrey.konovalov@linaro.org> MIME-Version: 1.0 Subject: [libcamera-devel] [PATCH v2 1/5] qcam: viewfinder_gl: change uniform float tex_stepx to vec2 tex_step X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Cc: morgan@casual-effects.com Errors-To: libcamera-devel-bounces@lists.libcamera.org Sender: "libcamera-devel" In preparation to extend the supported formats, extend the tex_stepx uniform to cover the steps between texels in both horizontal and vertical directions. Signed-off-by: Andrey Konovalov Reviewed-by: Laurent Pinchart --- src/qcam/assets/shader/YUV_packed.frag | 8 ++++---- src/qcam/viewfinder_gl.cpp | 7 ++++--- src/qcam/viewfinder_gl.h | 2 +- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/src/qcam/assets/shader/YUV_packed.frag b/src/qcam/assets/shader/YUV_packed.frag index 224dfafe..d6efd4ce 100644 --- a/src/qcam/assets/shader/YUV_packed.frag +++ b/src/qcam/assets/shader/YUV_packed.frag @@ -12,7 +12,7 @@ precision mediump float; varying vec2 textureOut; uniform sampler2D tex_y; -uniform float tex_stepx; +uniform vec2 tex_step; void main(void) { @@ -49,10 +49,10 @@ void main(void) * a = fract(x) * 2 - 1 if fract(x) >= 0.5 */ vec2 pos = textureOut; - float f_x = fract(pos.x / tex_stepx); + float f_x = fract(pos.x / tex_step.x); - vec4 left = texture2D(tex_y, vec2(pos.x - f_x * tex_stepx, pos.y)); - vec4 right = texture2D(tex_y, vec2(pos.x + (1.0 - f_x) * tex_stepx , pos.y)); + vec4 left = texture2D(tex_y, vec2(pos.x - f_x * tex_step.x, pos.y)); + vec4 right = texture2D(tex_y, vec2(pos.x + (1.0 - f_x) * tex_step.x , pos.y)); #if defined(YUV_PATTERN_UYVY) float y_left = mix(left.g, left.a, f_x * 2.0); diff --git a/src/qcam/viewfinder_gl.cpp b/src/qcam/viewfinder_gl.cpp index 5d9b442e..ff719418 100644 --- a/src/qcam/viewfinder_gl.cpp +++ b/src/qcam/viewfinder_gl.cpp @@ -289,7 +289,7 @@ bool ViewFinderGL::createFragmentShader() textureUniformY_ = shaderProgram_.uniformLocation("tex_y"); textureUniformU_ = shaderProgram_.uniformLocation("tex_u"); textureUniformV_ = shaderProgram_.uniformLocation("tex_v"); - textureUniformStepX_ = shaderProgram_.uniformLocation("tex_stepx"); + textureUniformStep_ = shaderProgram_.uniformLocation("tex_step"); /* Create the textures. */ for (std::unique_ptr &texture : textures_) { @@ -508,8 +508,9 @@ void ViewFinderGL::doRender() * ([0, 1]). There are exactly width - 1 steps between the * leftmost and rightmost texels. */ - shaderProgram_.setUniformValue(textureUniformStepX_, - 1.0f / (size_.width() / 2 - 1)); + shaderProgram_.setUniformValue(textureUniformStep_, + 1.0f / (size_.width() / 2 - 1), + 1.0f /* not used */); break; case libcamera::formats::ABGR8888: diff --git a/src/qcam/viewfinder_gl.h b/src/qcam/viewfinder_gl.h index 150fa4ae..1b1faa91 100644 --- a/src/qcam/viewfinder_gl.h +++ b/src/qcam/viewfinder_gl.h @@ -85,7 +85,7 @@ private: GLuint textureUniformU_; GLuint textureUniformV_; GLuint textureUniformY_; - GLuint textureUniformStepX_; + GLuint textureUniformStep_; unsigned int horzSubSample_; unsigned int vertSubSample_; From patchwork Thu May 27 10:55:08 2021 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Andrey Konovalov X-Patchwork-Id: 12449 Return-Path: X-Original-To: parsemail@patchwork.libcamera.org Delivered-To: parsemail@patchwork.libcamera.org Received: from lancelot.ideasonboard.com (lancelot.ideasonboard.com [92.243.16.209]) by patchwork.libcamera.org (Postfix) with ESMTPS id 12985C3203 for ; Thu, 27 May 2021 10:55:32 +0000 (UTC) Received: from lancelot.ideasonboard.com (localhost [IPv6:::1]) by lancelot.ideasonboard.com (Postfix) with ESMTP id 7625768923; Thu, 27 May 2021 12:55:31 +0200 (CEST) Authentication-Results: lancelot.ideasonboard.com; dkim=fail reason="signature verification failed" (2048-bit key; unprotected) header.d=linaro.org header.i=@linaro.org header.b="Y59n6c8N"; dkim-atps=neutral Received: from mail-lj1-x233.google.com (mail-lj1-x233.google.com [IPv6:2a00:1450:4864:20::233]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id 26905602AD for ; Thu, 27 May 2021 12:55:27 +0200 (CEST) Received: by mail-lj1-x233.google.com with SMTP id 131so217265ljj.3 for ; Thu, 27 May 2021 03:55:27 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=linaro.org; s=google; h=from:to:cc:subject:date:message-id:in-reply-to:references :mime-version:content-transfer-encoding; bh=Jg8BbPza8DoY2Yeb1WDkLpj593lnv5Q+IdDQRo3+qyQ=; b=Y59n6c8Nne/NGGzG3xOdKTp44lZ+Hiso67f1QFelly+prPLlu5dB4d1Ad33121mpuO KQ9jgL9uo53a3wSCZ+FDycpODDR2TVF2HzD61vZJQAtAvKZLEP3O8n3J2u0U2OcpvOHw iNe4wiLC3/IoknOHAL/HwfRQTFTaD/yeLM3T/HbZ/h1an7MTAmnx1XEleNfwtsCIq782 Pondw6hX5ruBk4HZBCnHSI+BHZUCzsIn4/LxpeXIL3uyUxG+Y0iZ3QV09r/cERGZg9a5 BciRiWrhi7k4hcELw9bSyjkTa9e1ArZAuyuVc1aT8P6EhYy24kT00PfDqpPN6LpJpPj9 aqKg== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20161025; h=x-gm-message-state:from:to:cc:subject:date:message-id:in-reply-to :references:mime-version:content-transfer-encoding; bh=Jg8BbPza8DoY2Yeb1WDkLpj593lnv5Q+IdDQRo3+qyQ=; b=s0XT2XnRo8rNQG8xlV5BZQHA+6LTsYRd6dvpSSkE9E7/zNeBXJEjVmFTW0fGpcKuKR fCEtTCs5b0QsgQq0pWB5XrYvcOzkjCm8H1cTXDtLtBTRcJPfGU3fIBmWZUoZ4hiMMAJy X39xpbtpNJFrvNLCZBq0VZYezPIheD+b0g6xy3FaKQByhvGruQCYJEW8nWmc1Ygdm6eW VhJk8wP9P9LarTq9LvohCgTQNwNvMffuxIFz5FDLKZ9N5jPXT/F5Bw4S2BjI9LkfmYLt iaHCfO0c+SVkOYE0KmL6I1BxmVpJsJBnnbyRKyM0GS52/w1wj3eGpL6rh7pMVnZ5lPD0 ngEA== X-Gm-Message-State: AOAM532DrZv81Q5nq70lNkd/Ie0v1yrrTkDgi439Tjzug7N8yYnwciK2 dCUGLegf1I1HJ2hbvrRpWKn+36VZPGEHvg== X-Google-Smtp-Source: ABdhPJwM4pblIC6ICg+dCxHvyM09IufpsdpgQ3UeOzjKZ6sNJW8GGkplfoWhiA55OQvzar+Y3zckOw== X-Received: by 2002:a2e:8e21:: with SMTP id r1mr2093055ljk.166.1622112926492; Thu, 27 May 2021 03:55:26 -0700 (PDT) Received: from localhost.localdomain ([85.249.41.56]) by smtp.googlemail.com with ESMTPSA id h23sm170148lfc.52.2021.05.27.03.55.25 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Thu, 27 May 2021 03:55:26 -0700 (PDT) From: Andrey Konovalov To: libcamera-devel@lists.libcamera.org, laurent.pinchart@ideasonboard.com Date: Thu, 27 May 2021 13:55:08 +0300 Message-Id: <20210527105511.447089-3-andrey.konovalov@linaro.org> X-Mailer: git-send-email 2.25.1 In-Reply-To: <20210527105511.447089-1-andrey.konovalov@linaro.org> References: <20210527105511.447089-1-andrey.konovalov@linaro.org> MIME-Version: 1.0 Subject: [libcamera-devel] [PATCH v2 2/5] qcam: viewfinder_gl: Add shader to render packed RAW10 formats X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Cc: morgan@casual-effects.com Errors-To: libcamera-devel-bounces@lists.libcamera.org Sender: "libcamera-devel" The shader supports all 4 packed RAW10 variants. Simple bi-linear filtering is implemented. The 2 LS bits of the 10-bit colour values are dropped as the RGBA format we convert into has only 8 bits per colour. The texture coordinates passed to the fragment shader are ajusted to point to the nearest pixel in the image. This prevents artifacts when the image is scaled from the frame resolution to the window size. Signed-off-by: Andrey Konovalov --- src/qcam/assets/shader/bayer_1x_packed.frag | 174 ++++++++++++++++++++ src/qcam/assets/shader/shaders.qrc | 1 + src/qcam/viewfinder_gl.cpp | 77 ++++++++- src/qcam/viewfinder_gl.h | 6 + 4 files changed, 256 insertions(+), 2 deletions(-) create mode 100644 src/qcam/assets/shader/bayer_1x_packed.frag diff --git a/src/qcam/assets/shader/bayer_1x_packed.frag b/src/qcam/assets/shader/bayer_1x_packed.frag new file mode 100644 index 00000000..0a87c6db --- /dev/null +++ b/src/qcam/assets/shader/bayer_1x_packed.frag @@ -0,0 +1,174 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Based on the code from http://jgt.akpeters.com/papers/McGuire08/ + * + * Efficient, High-Quality Bayer Demosaic Filtering on GPUs + * + * Morgan McGuire + * + * This paper appears in issue Volume 13, Number 4. + * --------------------------------------------------------- + * Copyright (c) 2008, Morgan McGuire. All rights reserved. + * + * + * Modified by Linaro Ltd for 10/12-bit packed vs 8-bit raw Bayer format, + * and for simpler demosaic algorithm. + * Copyright (C) 2020, Linaro + * + * bayer_1x_packed.frag - Fragment shader code for raw Bayer 10-bit and 12-bit + * packed formats + */ + +#ifdef GL_ES +precision mediump float; +#endif + +varying vec2 textureOut; + +/* the texture size: tex_size.xy is in bytes, tex_size.zw is in pixels */ +uniform vec4 tex_size; +uniform vec2 tex_step; +uniform vec2 tex_bayer_first_red; + +uniform sampler2D tex_raw; + +void main(void) +{ + vec3 rgb; + + /* + * center.xy holds the coordinates of the pixel being sampled + * on the [0, 1] range. + * center.zw holds the coordinates of the pixel being sampled + * on the [0, width/height-1] range. + */ + vec4 center; + /* + * x-positions of the adjacent pixels on the [0, 1] range. + */ + vec2 xcoords; + /* + * y-positions of the adjacent pixels on the [0, 1] range. + */ + vec2 ycoords; + + /* + * The coordinates passed to the shader in textureOut may point + * to a place in between the pixels if the viewfinder window is scaled + * from the original captured frame size. Align them to the nearest + * pixel. + */ + center.zw = floor(textureOut * tex_size.zw); + center.y = center.w; + /* + * Add a small number (a few mantissa's LSBs) to avoid float + * representation issues. Maybe paranoic. + */ + center.x = BPP_X * center.z + 0.02; + + const float threshold_l = 0.127 /* fract(BPP_X) * 0.5 + 0.02 */; + const float threshold_h = 0.625 /* 1.0 - fract(BPP_X) * 1.5 */; + + float fract_x = fract(center.x); + /* + * The below floor() call ensures that center.x points + * at one of the bytes representing the 8 higher bits of + * the pixel value, not at the byte containing the LS bits + * of the group of the pixels. + */ + center.x = floor(center.x); + center.xy *= tex_step; + + xcoords = center.x + vec2(-tex_step.x, tex_step.x); + ycoords = center.y + vec2(-tex_step.y, tex_step.y); + /* + * If xcoords[0] points at the byte containing the LS bits + * of the previous group of the pixels, move xcoords[0] one + * byte back. + */ + xcoords[0] += (fract_x < threshold_l) ? -tex_step.x : 0.0; + /* + * If xcoords[1] points at the byte containing the LS bits + * of the current group of the pixels, move xcoords[1] one + * byte forward. + */ + xcoords[1] += (fract_x > threshold_h) ? tex_step.x : 0.0; + + vec2 alternate = mod(center.zw + tex_bayer_first_red, 2.0); + bool even_col = alternate.x < 1.0; + bool even_raw = alternate.y < 1.0; + + /* + * We need to sample the central pixel and the ones with offset + * of -1 to +1 pixel in both X and Y directions. Let's name these + * pixels as below, where C is the central pixel: + * +----+----+----+----+ + * | \ x| | | | + * |y \ | -1 | 0 | +1 | + * +----+----+----+----+ + * | +1 | D2 | A1 | D3 | + * +----+----+----+----+ + * | 0 | B0 | C | B1 | + * +----+----+----+----+ + * | -1 | D0 | A0 | D1 | + * +----+----+----+----+ + * In the below equations (0,-1).r means "r component of the texel + * shifted by -tex_step.y from the center.xy one" etc. + * In the even raw / even column (EE) case the colour values are: + * R = C = (0,0).r, + * G = (A0 + A1 + B0 + B1) / 4.0 = + * ( (0,-1).r + (0,1).r + (-1,0).r + (1,0).r ) / 4.0, + * B = (D0 + D1 + D2 + D3) / 4.0 = + * ( (-1,-1).r + (1,-1).r + (-1,1).r + (1,1).r ) / 4.0 + * For even raw / odd column (EO): + * R = (B0 + B1) / 2.0 = ( (-1,0).r + (1,0).r ) / 2.0, + * G = C = (0,0).r, + * B = (A0 + A1) / 2.0 = ( (0,-1).r + (0,1).r ) / 2.0 + * For odd raw / even column (OE): + * R = (A0 + A1) / 2.0 = ( (0,-1).r + (0,1).r ) / 2.0, + * G = C = (0,0).r, + * B = (B0 + B1) / 2.0 = ( (-1,0).r + (1,0).r ) / 2.0 + * For odd raw / odd column (OO): + * R = (D0 + D1 + D2 + D3) / 4.0 = + * ( (-1,-1).r + (1,-1).r + (-1,1).r + (1,1).r ) / 4.0, + * G = (A0 + A1 + B0 + B1) / 4.0 = + * ( (0,-1).r + (0,1).r + (-1,0).r + (1,0).r ) / 4.0, + * B = C = (0,0).r + */ + + /* + * Fetch the values and precalculate the terms: + * patterns.x = (A0 + A1) / 2.0 + * patterns.y = (B0 + B1) / 2.0 + * patterns.z = (A0 + A1 + B0 + B1) / 4.0 + * patterns.w = (D0 + D1 + D2 + D3) / 4.0 + */ + #define fetch(x, y) texture2D(tex_raw, vec2(x, y)).r + + float C = texture2D(tex_raw, center.xy).r; + vec4 patterns = vec4( + fetch(center.x, ycoords[0]), /* A0: (0,-1) */ + fetch(xcoords[0], center.y), /* B0: (-1,0) */ + fetch(xcoords[0], ycoords[0]), /* D0: (-1,-1) */ + fetch(xcoords[1], ycoords[0])); /* D1: (1,-1) */ + vec4 temp = vec4( + fetch(center.x, ycoords[1]), /* A1: (0,1) */ + fetch(xcoords[1], center.y), /* B1: (1,0) */ + fetch(xcoords[1], ycoords[1]), /* D3: (1,1) */ + fetch(xcoords[0], ycoords[1])); /* D2: (-1,1) */ + patterns = (patterns + temp) * 0.5; + /* .x = (A0 + A1) / 2.0, .y = (B0 + B1) / 2.0 */ + /* .z = (D0 + D3) / 2.0, .w = (D1 + D2) / 2.0 */ + patterns.w = (patterns.z + patterns.w) * 0.5; + patterns.z = (patterns.x + patterns.y) * 0.5; + + rgb = (even_col) ? + ((even_raw) ? + vec3(C, patterns.zw) : + vec3(patterns.x, C, patterns.y)) : + ((even_raw) ? + vec3(patterns.y, C, patterns.x) : + vec3(patterns.wz, C)); + + gl_FragColor = vec4(rgb, 1.0); +} diff --git a/src/qcam/assets/shader/shaders.qrc b/src/qcam/assets/shader/shaders.qrc index 8a8f9de1..d76d65c5 100644 --- a/src/qcam/assets/shader/shaders.qrc +++ b/src/qcam/assets/shader/shaders.qrc @@ -5,6 +5,7 @@ YUV_2_planes.frag YUV_3_planes.frag YUV_packed.frag + bayer_1x_packed.frag identity.vert diff --git a/src/qcam/viewfinder_gl.cpp b/src/qcam/viewfinder_gl.cpp index ff719418..b324d77f 100644 --- a/src/qcam/viewfinder_gl.cpp +++ b/src/qcam/viewfinder_gl.cpp @@ -36,6 +36,11 @@ static const QList supportedFormats{ libcamera::formats::RGBA8888, libcamera::formats::BGR888, libcamera::formats::RGB888, + /* Raw Bayer 10-bit packed */ + libcamera::formats::SBGGR10_CSI2P, + libcamera::formats::SGBRG10_CSI2P, + libcamera::formats::SGRBG10_CSI2P, + libcamera::formats::SRGGB10_CSI2P, }; ViewFinderGL::ViewFinderGL(QWidget *parent) @@ -114,6 +119,9 @@ bool ViewFinderGL::selectFormat(const libcamera::PixelFormat &format) { bool ret = true; + /* Set min/mag filters to GL_LINEAR by default. */ + textureMinMagFilters_ = GL_LINEAR; + fragmentShaderDefines_.clear(); switch (format) { @@ -203,6 +211,34 @@ bool ViewFinderGL::selectFormat(const libcamera::PixelFormat &format) fragmentShaderDefines_.append("#define RGB_PATTERN bgr"); fragmentShaderFile_ = ":RGB.frag"; break; + case libcamera::formats::SBGGR10_CSI2P: + firstRed_.setX(1.0); + firstRed_.setY(1.0); + fragmentShaderDefines_.append("#define BPP_X 1.25"); + fragmentShaderFile_ = ":bayer_1x_packed.frag"; + textureMinMagFilters_ = GL_NEAREST; + break; + case libcamera::formats::SGBRG10_CSI2P: + firstRed_.setX(0.0); + firstRed_.setY(1.0); + fragmentShaderDefines_.append("#define BPP_X 1.25"); + fragmentShaderFile_ = ":bayer_1x_packed.frag"; + textureMinMagFilters_ = GL_NEAREST; + break; + case libcamera::formats::SGRBG10_CSI2P: + firstRed_.setX(1.0); + firstRed_.setY(0.0); + fragmentShaderDefines_.append("#define BPP_X 1.25"); + fragmentShaderFile_ = ":bayer_1x_packed.frag"; + textureMinMagFilters_ = GL_NEAREST; + break; + case libcamera::formats::SRGGB10_CSI2P: + firstRed_.setX(0.0); + firstRed_.setY(0.0); + fragmentShaderDefines_.append("#define BPP_X 1.25"); + fragmentShaderFile_ = ":bayer_1x_packed.frag"; + textureMinMagFilters_ = GL_NEAREST; + break; default: ret = false; qWarning() << "[ViewFinderGL]:" @@ -290,6 +326,8 @@ bool ViewFinderGL::createFragmentShader() textureUniformU_ = shaderProgram_.uniformLocation("tex_u"); textureUniformV_ = shaderProgram_.uniformLocation("tex_v"); textureUniformStep_ = shaderProgram_.uniformLocation("tex_step"); + textureUniformSize_ = shaderProgram_.uniformLocation("tex_size"); + textureUniformBayerFirstRed_ = shaderProgram_.uniformLocation("tex_bayer_first_red"); /* Create the textures. */ for (std::unique_ptr &texture : textures_) { @@ -306,8 +344,10 @@ bool ViewFinderGL::createFragmentShader() void ViewFinderGL::configureTexture(QOpenGLTexture &texture) { glBindTexture(GL_TEXTURE_2D, texture.textureId()); - glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR); - glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, + textureMinMagFilters_); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, + textureMinMagFilters_); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); } @@ -547,6 +587,39 @@ void ViewFinderGL::doRender() shaderProgram_.setUniformValue(textureUniformY_, 0); break; + case libcamera::formats::SBGGR10_CSI2P: + case libcamera::formats::SGBRG10_CSI2P: + case libcamera::formats::SGRBG10_CSI2P: + case libcamera::formats::SRGGB10_CSI2P: + /* + * Packed raw Bayer 10-bit formats are stored in GL_RED texture. + * The texture width is 10/8 of the image width. + * TODO: account for padding bytes at the end of the line. + */ + glActiveTexture(GL_TEXTURE0); + configureTexture(*textures_[0]); + glTexImage2D(GL_TEXTURE_2D, + 0, + GL_RED, + size_.width() * 5 / 4, + size_.height(), + 0, + GL_RED, + GL_UNSIGNED_BYTE, + data_); + shaderProgram_.setUniformValue(textureUniformY_, 0); + shaderProgram_.setUniformValue(textureUniformBayerFirstRed_, + firstRed_); + shaderProgram_.setUniformValue(textureUniformSize_, + size_.width() * 5 / 4, + size_.height(), + size_.width(), + size_.height()); + shaderProgram_.setUniformValue(textureUniformStep_, + 1.0f / (size_.width() * 5 / 4 - 1), + 1.0f / (size_.height() - 1)); + break; + default: break; }; diff --git a/src/qcam/viewfinder_gl.h b/src/qcam/viewfinder_gl.h index 1b1faa91..337718e3 100644 --- a/src/qcam/viewfinder_gl.h +++ b/src/qcam/viewfinder_gl.h @@ -81,13 +81,19 @@ private: /* Textures */ std::array, 3> textures_; + /* Common texture parameters */ + GLuint textureMinMagFilters_; /* YUV texture parameters */ GLuint textureUniformU_; GLuint textureUniformV_; GLuint textureUniformY_; + GLuint textureUniformSize_; GLuint textureUniformStep_; unsigned int horzSubSample_; unsigned int vertSubSample_; + /* Raw Bayer texture parameters */ + GLuint textureUniformBayerFirstRed_; + QPointF firstRed_; QMutex mutex_; /* Prevent concurrent access to image_ */ }; From patchwork Thu May 27 10:55:09 2021 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Andrey Konovalov X-Patchwork-Id: 12450 Return-Path: X-Original-To: parsemail@patchwork.libcamera.org Delivered-To: parsemail@patchwork.libcamera.org Received: from lancelot.ideasonboard.com (lancelot.ideasonboard.com [92.243.16.209]) by patchwork.libcamera.org (Postfix) with ESMTPS id C9249C3203 for ; Thu, 27 May 2021 10:55:32 +0000 (UTC) Received: from lancelot.ideasonboard.com (localhost [IPv6:::1]) by lancelot.ideasonboard.com (Postfix) with ESMTP id 0447568930; Thu, 27 May 2021 12:55:32 +0200 (CEST) Authentication-Results: lancelot.ideasonboard.com; dkim=fail reason="signature verification failed" (2048-bit key; unprotected) header.d=linaro.org header.i=@linaro.org header.b="r83vRxX1"; dkim-atps=neutral Received: from mail-lj1-x231.google.com (mail-lj1-x231.google.com [IPv6:2a00:1450:4864:20::231]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id 7F83768923 for ; Thu, 27 May 2021 12:55:28 +0200 (CEST) Received: by mail-lj1-x231.google.com with SMTP id o8so268822ljp.0 for ; Thu, 27 May 2021 03:55:28 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=linaro.org; s=google; h=from:to:cc:subject:date:message-id:in-reply-to:references :mime-version:content-transfer-encoding; bh=qAip27BP+CSOHYaYAUIBsQQCEYbNM1TjHaCu7Zmcni4=; b=r83vRxX1CYVmXaxL5WRtsgGfEFnQqMmvgIwVBEz35NmD3g/frGwVGkPY/DMbvq01gk LMJzQU0c8m5P3nE5xMVqu3cdh8Vjo0cmwhPqZRwFXpZBUkMuX74t/bOqLUFjIBxEuKes 32JNNufNePFM05tHLmTQrWkbOIMtoFgrJmlreZB7dishoVsaHtfO9kHohsyi7cTgxODb z5PpU61P6IfSA+wx8qc8oBK0FIjQNOvurZJoT+74vZTI9wxaUOQCUD8Gas5gIHI2Jvtr uPuMMU26Oy5BFHtxg53YHdJekDVHMpUdH1gVWKVUvz14BThriAl9mYz6uqI9ei5ajD0j mcjw== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20161025; h=x-gm-message-state:from:to:cc:subject:date:message-id:in-reply-to :references:mime-version:content-transfer-encoding; bh=qAip27BP+CSOHYaYAUIBsQQCEYbNM1TjHaCu7Zmcni4=; b=ILtIi45yJC9WGsaUaSNs7gGn+OECKFRxmwYVrb2cBteVyCljhOGwu/yEAiYwbLqBEs 9L+DCuAaaXP+2F/UF5mcZn21aolJmXTze6Z+OgFQ2GvNqC9fXHcpXPB+QPpzJ4J4kx51 fXU5tg8mdAXRsXpzmGm3vfIzA3VNfiFQF/kNbFmnDHqnsKWQNMvqiw2mvBuoODyNZAR4 xpsYH8pYgKRHV+M06T0JGxjq9hpd65Ytu5hmUHyPSgZn8RA27EznoqLHsDAcp3Gw4B30 mESwzUemGTmocQrqAOo6eqH/Nh7Qm2W6yl7avTbgvRlQXk01reYKM1Osir3lhxC0+Lr1 BXNw== X-Gm-Message-State: AOAM531NFi7/W5Ym7LsWQE4sOaqvP8xw5ENgQgSqOMOV0rIZy0Uu79YO P/HASFS3D74GXsjKQCN5xIjgUGD9orw75A== X-Google-Smtp-Source: ABdhPJzF2B4pbHcYAiaDMIkLdFIQi6pZMf320MSbq/CShFlGs5NaZOi+8fkgbH+f2db7wbOGSbQRIA== X-Received: by 2002:a05:651c:3db:: with SMTP id f27mr2026396ljp.241.1622112927387; Thu, 27 May 2021 03:55:27 -0700 (PDT) Received: from localhost.localdomain ([85.249.41.56]) by smtp.googlemail.com with ESMTPSA id h23sm170148lfc.52.2021.05.27.03.55.26 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Thu, 27 May 2021 03:55:26 -0700 (PDT) From: Andrey Konovalov To: libcamera-devel@lists.libcamera.org, laurent.pinchart@ideasonboard.com Date: Thu, 27 May 2021 13:55:09 +0300 Message-Id: <20210527105511.447089-4-andrey.konovalov@linaro.org> X-Mailer: git-send-email 2.25.1 In-Reply-To: <20210527105511.447089-1-andrey.konovalov@linaro.org> References: <20210527105511.447089-1-andrey.konovalov@linaro.org> MIME-Version: 1.0 Subject: [libcamera-devel] [PATCH v2 3/5] qcam: viewfinder_gl: RAW10P: handle the padding bytes properly X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Cc: morgan@casual-effects.com Errors-To: libcamera-devel-bounces@lists.libcamera.org Sender: "libcamera-devel" The texture size must account for the padding bytes which may be present at the end of the lines in the frame. Signed-off-by: Andrey Konovalov --- src/qcam/viewfinder_gl.cpp | 11 +++++++---- src/qcam/viewfinder_gl.h | 1 + 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/qcam/viewfinder_gl.cpp b/src/qcam/viewfinder_gl.cpp index b324d77f..9db536a6 100644 --- a/src/qcam/viewfinder_gl.cpp +++ b/src/qcam/viewfinder_gl.cpp @@ -111,6 +111,10 @@ void ViewFinderGL::render(libcamera::FrameBuffer *buffer, MappedBuffer *map) renderComplete(buffer_); data_ = static_cast(map->memory); + /* + * \todo Get the stride from the buffer instead of computing it naively + */ + stride_ = buffer->metadata().planes[0].bytesused / size_.height(); update(); buffer_ = buffer; } @@ -594,14 +598,13 @@ void ViewFinderGL::doRender() /* * Packed raw Bayer 10-bit formats are stored in GL_RED texture. * The texture width is 10/8 of the image width. - * TODO: account for padding bytes at the end of the line. */ glActiveTexture(GL_TEXTURE0); configureTexture(*textures_[0]); glTexImage2D(GL_TEXTURE_2D, 0, GL_RED, - size_.width() * 5 / 4, + stride_, size_.height(), 0, GL_RED, @@ -611,12 +614,12 @@ void ViewFinderGL::doRender() shaderProgram_.setUniformValue(textureUniformBayerFirstRed_, firstRed_); shaderProgram_.setUniformValue(textureUniformSize_, - size_.width() * 5 / 4, + stride_, size_.height(), size_.width(), size_.height()); shaderProgram_.setUniformValue(textureUniformStep_, - 1.0f / (size_.width() * 5 / 4 - 1), + 1.0f / (stride_ - 1), 1.0f / (size_.height() - 1)); break; diff --git a/src/qcam/viewfinder_gl.h b/src/qcam/viewfinder_gl.h index 337718e3..49f8364a 100644 --- a/src/qcam/viewfinder_gl.h +++ b/src/qcam/viewfinder_gl.h @@ -66,6 +66,7 @@ private: libcamera::FrameBuffer *buffer_; libcamera::PixelFormat format_; QSize size_; + int stride_; unsigned char *data_; /* Shaders */ From patchwork Thu May 27 10:55:10 2021 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Andrey Konovalov X-Patchwork-Id: 12451 Return-Path: X-Original-To: parsemail@patchwork.libcamera.org Delivered-To: parsemail@patchwork.libcamera.org Received: from lancelot.ideasonboard.com (lancelot.ideasonboard.com [92.243.16.209]) by patchwork.libcamera.org (Postfix) with ESMTPS id 08735C3203 for ; Thu, 27 May 2021 10:55:34 +0000 (UTC) Received: from lancelot.ideasonboard.com (localhost [IPv6:::1]) by lancelot.ideasonboard.com (Postfix) with ESMTP id 7D7B46892A; Thu, 27 May 2021 12:55:33 +0200 (CEST) Authentication-Results: lancelot.ideasonboard.com; dkim=fail reason="signature verification failed" (2048-bit key; unprotected) header.d=linaro.org header.i=@linaro.org header.b="ZP4ywRct"; dkim-atps=neutral Received: from mail-lj1-x234.google.com (mail-lj1-x234.google.com [IPv6:2a00:1450:4864:20::234]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id 68986602AE for ; Thu, 27 May 2021 12:55:29 +0200 (CEST) Received: by mail-lj1-x234.google.com with SMTP id c15so194066ljr.7 for ; Thu, 27 May 2021 03:55:29 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=linaro.org; s=google; h=from:to:cc:subject:date:message-id:in-reply-to:references :mime-version:content-transfer-encoding; bh=5A0z3TEDOT6/S5LTn+Sn0yVOZBNjVvE2904e61JpYMA=; b=ZP4ywRctetpTOYIHSkpU2v2ksmXukgDAiagjil5MKvyLcX/waGRl8KDjxuS+nr7k1m mnNiYNVnHLIkCGkApAEa9aex/5DRYR2GXVE0UI3BZsjox/fZDSDwU0ehbtOqjUEBzdik tQ6gUUZJFmPY66SVBa7Af0TjY8wlFQK9pO88uPh6T2M10/30vnbtReSICph+qqZPElXl 5ugW0uW09h9E237GjzC6klV84fLl5PIh6tIhsY3G3J6FFQzRGzbiQValbwQVbZbrsQPX BjlSYbYaycXb5zmMUjNjAkzhaNiD1jdLgGKmXKBt578+e1k6ijd7VzNa5Iq2FLrYtByg TLDQ== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20161025; h=x-gm-message-state:from:to:cc:subject:date:message-id:in-reply-to :references:mime-version:content-transfer-encoding; bh=5A0z3TEDOT6/S5LTn+Sn0yVOZBNjVvE2904e61JpYMA=; b=OTd++N4t49YldHBf4muJvA02ix6unNU4WNS3ePELJ0EhtTW999do8j6t5CyuawST00 ALnKdUJB6i9eB6NngzRiw7h38LsIPbxXQZH1QGVgoM7umbkXZ7KD5e6L5GDWxOnG0A3x vi5rPdUlswKr/DeHPT0ylUFRZISOFtIoDdVmPbnRzMXRb2Uwo+aboUvmev/a9f9vRo8Z 9lu/bjhfcMzkm0HLfI+U41QmoWsRt7K1PNEBcgME4p1hjzz5P0jCchMuVgM1iZrXYzOS GevURwQ/0xYTF2hmPxvhGnL/nlPywkVvKqnqYSgVpcrsB40BAE5EVpfhrXSy6b+PIm8T vHRQ== X-Gm-Message-State: AOAM533uKeupUXa2e4Pdmof1NO4NJVDgl1ZA4sWlkmLoKAPIFoKR32bE j8+6NvX5lQghFHswQ2lo3XKTrEwUt5iJqQ== X-Google-Smtp-Source: ABdhPJxMPJj5btsLXO0J2+9kgbc3dqxUjHZ/fBwWd9fRDymPm0URGVa+gYx7fZaKdSwISc9srTD4HA== X-Received: by 2002:a2e:a553:: with SMTP id e19mr2085799ljn.280.1622112928535; Thu, 27 May 2021 03:55:28 -0700 (PDT) Received: from localhost.localdomain ([85.249.41.56]) by smtp.googlemail.com with ESMTPSA id h23sm170148lfc.52.2021.05.27.03.55.27 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Thu, 27 May 2021 03:55:27 -0700 (PDT) From: Andrey Konovalov To: libcamera-devel@lists.libcamera.org, laurent.pinchart@ideasonboard.com Date: Thu, 27 May 2021 13:55:10 +0300 Message-Id: <20210527105511.447089-5-andrey.konovalov@linaro.org> X-Mailer: git-send-email 2.25.1 In-Reply-To: <20210527105511.447089-1-andrey.konovalov@linaro.org> References: <20210527105511.447089-1-andrey.konovalov@linaro.org> MIME-Version: 1.0 Subject: [libcamera-devel] [PATCH v2 4/5] qcam: viewfinder_gl: Add support for RAW12 packed formats X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Cc: morgan@casual-effects.com Errors-To: libcamera-devel-bounces@lists.libcamera.org Sender: "libcamera-devel" All the four Bayer orders are supported. The 4 LS bits of the 12-bit colour values are dropped as the RGBA format we convert into has only 8 bits per colour. Signed-off-by: Andrey Konovalov Reviewed-by: Laurent Pinchart --- src/qcam/assets/shader/bayer_1x_packed.frag | 43 ++++++++++++++--- src/qcam/viewfinder_gl.cpp | 53 +++++++++++++++++---- 2 files changed, 82 insertions(+), 14 deletions(-) diff --git a/src/qcam/assets/shader/bayer_1x_packed.frag b/src/qcam/assets/shader/bayer_1x_packed.frag index 0a87c6db..d09c6fce 100644 --- a/src/qcam/assets/shader/bayer_1x_packed.frag +++ b/src/qcam/assets/shader/bayer_1x_packed.frag @@ -23,6 +23,40 @@ precision mediump float; #endif +/* + * These constants are used to select the bytes containing the HS part of + * the pixel value: + * BPP - bytes per pixel, + * THRESHOLD_L = fract(BPP) * 0.5 + 0.02 + * THRESHOLD_H = 1.0 - fract(BPP) * 1.5 + 0.02 + * Let X is the x coordinate in the texture measured in bytes (so that the + * range is from 0 to (stride_-1)) aligned on the nearest pixel. + * E.g. for RAW10P: + * -------------+-------------------+-------------------+-- + * pixel No | 0 1 2 3 | 4 5 6 7 | ... + * -------------+-------------------+-------------------+-- + * byte offset | 0 1 2 3 4 | 5 6 7 8 9 | ... + * -------------+-------------------+-------------------+-- + * X | 0.0 1.25 2.5 3.75 | 5.0 6.25 7.5 8.75 | ... + * -------------+-------------------+-------------------+-- + * If fract(X) < THRESHOLD_L then the previous byte contains the LS + * bits of the pixel values and needs to be skipped. + * If fract(X) > THRESHOLD_H then the next byte contains the LS bits + * of the pixel values and needs to be skipped. + */ +#if defined(RAW10P) +#define BPP 1.25 +#define THRESHOLD_L 0.14 +#define THRESHOLD_H 0.64 +#elif defined(RAW12P) +#define BPP 1.5 +#define THRESHOLD_L 0.27 +#define THRESHOLD_H 0.27 +#else +#error Invalid raw format +#endif + + varying vec2 textureOut; /* the texture size: tex_size.xy is in bytes, tex_size.zw is in pixels */ @@ -64,10 +98,7 @@ void main(void) * Add a small number (a few mantissa's LSBs) to avoid float * representation issues. Maybe paranoic. */ - center.x = BPP_X * center.z + 0.02; - - const float threshold_l = 0.127 /* fract(BPP_X) * 0.5 + 0.02 */; - const float threshold_h = 0.625 /* 1.0 - fract(BPP_X) * 1.5 */; + center.x = BPP * center.z + 0.02; float fract_x = fract(center.x); /* @@ -86,13 +117,13 @@ void main(void) * of the previous group of the pixels, move xcoords[0] one * byte back. */ - xcoords[0] += (fract_x < threshold_l) ? -tex_step.x : 0.0; + xcoords[0] += (fract_x < THRESHOLD_L) ? -tex_step.x : 0.0; /* * If xcoords[1] points at the byte containing the LS bits * of the current group of the pixels, move xcoords[1] one * byte forward. */ - xcoords[1] += (fract_x > threshold_h) ? tex_step.x : 0.0; + xcoords[1] += (fract_x > THRESHOLD_H) ? tex_step.x : 0.0; vec2 alternate = mod(center.zw + tex_bayer_first_red, 2.0); bool even_col = alternate.x < 1.0; diff --git a/src/qcam/viewfinder_gl.cpp b/src/qcam/viewfinder_gl.cpp index 9db536a6..13ab31aa 100644 --- a/src/qcam/viewfinder_gl.cpp +++ b/src/qcam/viewfinder_gl.cpp @@ -41,6 +41,11 @@ static const QList supportedFormats{ libcamera::formats::SGBRG10_CSI2P, libcamera::formats::SGRBG10_CSI2P, libcamera::formats::SRGGB10_CSI2P, + /* Raw Bayer 12-bit packed */ + libcamera::formats::SBGGR12_CSI2P, + libcamera::formats::SGBRG12_CSI2P, + libcamera::formats::SGRBG12_CSI2P, + libcamera::formats::SRGGB12_CSI2P, }; ViewFinderGL::ViewFinderGL(QWidget *parent) @@ -218,28 +223,56 @@ bool ViewFinderGL::selectFormat(const libcamera::PixelFormat &format) case libcamera::formats::SBGGR10_CSI2P: firstRed_.setX(1.0); firstRed_.setY(1.0); - fragmentShaderDefines_.append("#define BPP_X 1.25"); + fragmentShaderDefines_.append("#define RAW10P"); fragmentShaderFile_ = ":bayer_1x_packed.frag"; textureMinMagFilters_ = GL_NEAREST; break; case libcamera::formats::SGBRG10_CSI2P: firstRed_.setX(0.0); firstRed_.setY(1.0); - fragmentShaderDefines_.append("#define BPP_X 1.25"); + fragmentShaderDefines_.append("#define RAW10P"); fragmentShaderFile_ = ":bayer_1x_packed.frag"; textureMinMagFilters_ = GL_NEAREST; break; case libcamera::formats::SGRBG10_CSI2P: firstRed_.setX(1.0); firstRed_.setY(0.0); - fragmentShaderDefines_.append("#define BPP_X 1.25"); + fragmentShaderDefines_.append("#define RAW10P"); fragmentShaderFile_ = ":bayer_1x_packed.frag"; textureMinMagFilters_ = GL_NEAREST; break; case libcamera::formats::SRGGB10_CSI2P: firstRed_.setX(0.0); firstRed_.setY(0.0); - fragmentShaderDefines_.append("#define BPP_X 1.25"); + fragmentShaderDefines_.append("#define RAW10P"); + fragmentShaderFile_ = ":bayer_1x_packed.frag"; + textureMinMagFilters_ = GL_NEAREST; + break; + case libcamera::formats::SBGGR12_CSI2P: + firstRed_.setX(1.0); + firstRed_.setY(1.0); + fragmentShaderDefines_.append("#define RAW12P"); + fragmentShaderFile_ = ":bayer_1x_packed.frag"; + textureMinMagFilters_ = GL_NEAREST; + break; + case libcamera::formats::SGBRG12_CSI2P: + firstRed_.setX(0.0); + firstRed_.setY(1.0); + fragmentShaderDefines_.append("#define RAW12P"); + fragmentShaderFile_ = ":bayer_1x_packed.frag"; + textureMinMagFilters_ = GL_NEAREST; + break; + case libcamera::formats::SGRBG12_CSI2P: + firstRed_.setX(1.0); + firstRed_.setY(0.0); + fragmentShaderDefines_.append("#define RAW12P"); + fragmentShaderFile_ = ":bayer_1x_packed.frag"; + textureMinMagFilters_ = GL_NEAREST; + break; + case libcamera::formats::SRGGB12_CSI2P: + firstRed_.setX(0.0); + firstRed_.setY(0.0); + fragmentShaderDefines_.append("#define RAW12P"); fragmentShaderFile_ = ":bayer_1x_packed.frag"; textureMinMagFilters_ = GL_NEAREST; break; @@ -595,9 +628,13 @@ void ViewFinderGL::doRender() case libcamera::formats::SGBRG10_CSI2P: case libcamera::formats::SGRBG10_CSI2P: case libcamera::formats::SRGGB10_CSI2P: + case libcamera::formats::SBGGR12_CSI2P: + case libcamera::formats::SGBRG12_CSI2P: + case libcamera::formats::SGRBG12_CSI2P: + case libcamera::formats::SRGGB12_CSI2P: /* - * Packed raw Bayer 10-bit formats are stored in GL_RED texture. - * The texture width is 10/8 of the image width. + * Packed raw Bayer 10-bit and 12-bit formats are stored in + * GL_RED texture. */ glActiveTexture(GL_TEXTURE0); configureTexture(*textures_[0]); @@ -614,9 +651,9 @@ void ViewFinderGL::doRender() shaderProgram_.setUniformValue(textureUniformBayerFirstRed_, firstRed_); shaderProgram_.setUniformValue(textureUniformSize_, - stride_, + stride_, /* width in bytes */ size_.height(), - size_.width(), + size_.width(), /* in pixels */ size_.height()); shaderProgram_.setUniformValue(textureUniformStep_, 1.0f / (stride_ - 1), From patchwork Thu May 27 10:55:11 2021 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Andrey Konovalov X-Patchwork-Id: 12452 Return-Path: X-Original-To: parsemail@patchwork.libcamera.org Delivered-To: parsemail@patchwork.libcamera.org Received: from lancelot.ideasonboard.com (lancelot.ideasonboard.com [92.243.16.209]) by patchwork.libcamera.org (Postfix) with ESMTPS id 0A04BC3203 for ; Thu, 27 May 2021 10:55:37 +0000 (UTC) Received: from lancelot.ideasonboard.com (localhost [IPv6:::1]) by lancelot.ideasonboard.com (Postfix) with ESMTP id C0459602AF; Thu, 27 May 2021 12:55:36 +0200 (CEST) Authentication-Results: lancelot.ideasonboard.com; dkim=fail reason="signature verification failed" (2048-bit key; unprotected) header.d=linaro.org header.i=@linaro.org header.b="k7epmvCC"; dkim-atps=neutral Received: from mail-lj1-x231.google.com (mail-lj1-x231.google.com [IPv6:2a00:1450:4864:20::231]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id 6CCEC602AB for ; Thu, 27 May 2021 12:55:30 +0200 (CEST) Received: by mail-lj1-x231.google.com with SMTP id p20so188729ljj.8 for ; Thu, 27 May 2021 03:55:30 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=linaro.org; s=google; h=from:to:cc:subject:date:message-id:in-reply-to:references :mime-version:content-transfer-encoding; bh=lV47W5pBA/MVL8Hi2QxGJiTUjZ70nJNpg0cFlxhb3TU=; b=k7epmvCCbyy6JABrmX9oev+cCxwVonACdqfbTzLTVUp5ZKEuLPkgiNQC+h46r1CLtv xkOm8rWfpt58iZddQ0Qo9aWNDllxnrgo7tw6zTLjQ+rzIUcMr2o8a24kCnUKUUdwJRcc nsQlJWIxPFFUv18B7zJ8qA+3qZohyVkuKntMNE4DupHc2gpWvFekovauaIOwLbEeHNQh qgUeK77KHUXWNdDG+EaIYTytNNBIoegEXBthdo/Vn2NVBsGfFsIunFyWwpZ+AeyfyWst 4AEEEV0wdL6Hrg0dOv9StBvQYrgHkwqM1MCU+QGb5LjXn6WsobR1vyd7xNUKGAH23P5p +ZRw== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20161025; h=x-gm-message-state:from:to:cc:subject:date:message-id:in-reply-to :references:mime-version:content-transfer-encoding; bh=lV47W5pBA/MVL8Hi2QxGJiTUjZ70nJNpg0cFlxhb3TU=; b=QW2mZd1KGiKmmTNC6qC7s2c9rX9LCOMfWwpcq8RkbE3KzGplewDOUIl3Y6ATSnmY79 fBiwpjgfVoMHG4K3Fj7UbJ3O1MA+Ol/vzoAouxSqOFzU2E0Zw3o5kd1BM/P+zyo/OHls 4YB58ARXDyTzHbwKD8B3vHZwJ1EAuScgLdNj3AbAEyoLhNeFMvFT4/5y+lvnDRLjWmA5 uinjWnzsLI7LqsT+uKAoJnKDmVb8wv3KdevJqyILQY6mUxAdLJQdI7G4pMrTlM69jbD/ /+ikfT5HfiaNeMzJGxlQd4FQZSgRBLP1hCg15UugLWUHZkVhcS5IhM/MR7KqphAI4bFu aRAw== X-Gm-Message-State: AOAM5313CYIfiGCgmy60JuYD+eigXdVdUwbxUN58dvvt2s8Lm+G5LrY3 MDT9A76s4H3U8cAt/ReiDBT0S3KZ6X9Qiw== X-Google-Smtp-Source: ABdhPJxc4uarbFcFAcfn9Qi93ttCO8ac+0RDj+e2Fs68QxdlZnHcbmemA0hlpLI2l1hJX8tiGkd/iQ== X-Received: by 2002:a2e:a78c:: with SMTP id c12mr2027444ljf.421.1622112929690; Thu, 27 May 2021 03:55:29 -0700 (PDT) Received: from localhost.localdomain ([85.249.41.56]) by smtp.googlemail.com with ESMTPSA id h23sm170148lfc.52.2021.05.27.03.55.28 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Thu, 27 May 2021 03:55:28 -0700 (PDT) From: Andrey Konovalov To: libcamera-devel@lists.libcamera.org, laurent.pinchart@ideasonboard.com Date: Thu, 27 May 2021 13:55:11 +0300 Message-Id: <20210527105511.447089-6-andrey.konovalov@linaro.org> X-Mailer: git-send-email 2.25.1 In-Reply-To: <20210527105511.447089-1-andrey.konovalov@linaro.org> References: <20210527105511.447089-1-andrey.konovalov@linaro.org> MIME-Version: 1.0 Subject: [libcamera-devel] [PATCH v2 5/5] qcam: viewfinder_gl: Add support for RAW8 Bayer formats X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Cc: morgan@casual-effects.com Errors-To: libcamera-devel-bounces@lists.libcamera.org Sender: "libcamera-devel" All the four Bayer orders are supported. The texture coordinates passed to the fragment shader are ajusted to point to the nearest pixel in the image. This prevents artifacts when the image is scaled from the frame resolution to the window size. Signed-off-by: Andrey Konovalov --- src/qcam/assets/shader/bayer_8.frag | 136 ++++++++++++++++++++++++++++ src/qcam/assets/shader/bayer_8.vert | 72 +++++++++++++++ src/qcam/assets/shader/shaders.qrc | 1 + src/qcam/viewfinder_gl.cpp | 37 +++++++- 4 files changed, 244 insertions(+), 2 deletions(-) create mode 100644 src/qcam/assets/shader/bayer_8.frag create mode 100644 src/qcam/assets/shader/bayer_8.vert diff --git a/src/qcam/assets/shader/bayer_8.frag b/src/qcam/assets/shader/bayer_8.frag new file mode 100644 index 00000000..d93ef1da --- /dev/null +++ b/src/qcam/assets/shader/bayer_8.frag @@ -0,0 +1,136 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* +From http://jgt.akpeters.com/papers/McGuire08/ + +Efficient, High-Quality Bayer Demosaic Filtering on GPUs + +Morgan McGuire + +This paper appears in issue Volume 13, Number 4. +--------------------------------------------------------- +Copyright (c) 2008, Morgan McGuire. All rights reserved. + + +Modified by Linaro Ltd to integrate it into libcamera, and to +fix the artifacts due to pixel coordinates interpolation. +Copyright (C) 2021, Linaro +*/ + +//Pixel Shader + +varying vec2 textureOut; + +/* The texture size: tex_size.xy is in bytes, tex_size.zw is in pixels */ +uniform vec4 tex_size; +uniform vec2 tex_step; + +/** Pixel position of the first red pixel in the */ +/** Bayer pattern. [{0,1}, {0, 1}]*/ +uniform vec2 tex_bayer_first_red; + +/** Monochrome RGBA or GL_LUMINANCE Bayer encoded texture.*/ +uniform sampler2D tex_raw; + +void main(void) { + #define fetch(x, y) texture2D(tex_raw, vec2(x, y)).r + + /** .xy = Pixel being sampled in the fragment shader on the range [0, 1] + .zw = ...on the range [0, sourceSize], offset by firstRed */ + vec4 center; + + /** center.x + (-2/w, -1/w, 1/w, 2/w); These are the x-positions */ + /** of the adjacent pixels.*/ + vec4 xCoord; + + /** center.y + (-2/h, -1/h, 1/h, 2/h); These are the y-positions */ + /** of the adjacent pixels.*/ + vec4 yCoord; + + /* Align the center coordinates to the nearest pixel */ + center.zw = floor(textureOut * tex_size.zw); + center.xy = center.zw * tex_step; + center.zw += tex_bayer_first_red; + + xCoord = center.x + vec4(-2.0 * tex_step.x, + -tex_step.x, tex_step.x, 2.0 * tex_step.x); + yCoord = center.y + vec4(-2.0 * tex_step.y, + -tex_step.y, tex_step.y, 2.0 * tex_step.y); + + float C = texture2D(tex_raw, center.xy).r; // ( 0, 0) + const vec4 kC = vec4( 4.0, 6.0, 5.0, 5.0) / 8.0; + + // Determine which of four types of pixels we are on. + vec2 alternate = mod(floor(center.zw), 2.0); + + vec4 Dvec = vec4( + fetch(xCoord[1], yCoord[1]), // (-1,-1) + fetch(xCoord[1], yCoord[2]), // (-1, 1) + fetch(xCoord[2], yCoord[1]), // ( 1,-1) + fetch(xCoord[2], yCoord[2])); // ( 1, 1) + + vec4 PATTERN = (kC.xyz * C).xyzz; + + // Can also be a dot product with (1,1,1,1) on hardware where that is + // specially optimized. + // Equivalent to: D = Dvec[0] + Dvec[1] + Dvec[2] + Dvec[3]; + Dvec.xy += Dvec.zw; + Dvec.x += Dvec.y; + + vec4 value = vec4( + fetch(center.x, yCoord[0]), // ( 0,-2) + fetch(center.x, yCoord[1]), // ( 0,-1) + fetch(xCoord[0], center.y), // (-2, 0) + fetch(xCoord[1], center.y)); // (-1, 0) + + vec4 temp = vec4( + fetch(center.x, yCoord[3]), // ( 0, 2) + fetch(center.x, yCoord[2]), // ( 0, 1) + fetch(xCoord[3], center.y), // ( 2, 0) + fetch(xCoord[2], center.y)); // ( 1, 0) + + // Even the simplest compilers should be able to constant-fold these to + // avoid the division. + // Note that on scalar processors these constants force computation of some + // identical products twice. + const vec4 kA = vec4(-1.0, -1.5, 0.5, -1.0) / 8.0; + const vec4 kB = vec4( 2.0, 0.0, 0.0, 4.0) / 8.0; + const vec4 kD = vec4( 0.0, 2.0, -1.0, -1.0) / 8.0; + + // Conserve constant registers and take advantage of free swizzle on load + #define kE (kA.xywz) + #define kF (kB.xywz) + + value += temp; + + // There are five filter patterns (identity, cross, checker, + // theta, phi). Precompute the terms from all of them and then + // use swizzles to assign to color channels. + // + // Channel Matches + // x cross (e.g., EE G) + // y checker (e.g., EE B) + // z theta (e.g., EO R) + // w phi (e.g., EO R) + #define A (value[0]) + #define B (value[1]) + #define D (Dvec.x) + #define E (value[2]) + #define F (value[3]) + + // Avoid zero elements. On a scalar processor this saves two MADDs + // and it has no effect on a vector processor. + PATTERN.yzw += (kD.yz * D).xyy; + + PATTERN += (kA.xyz * A).xyzx + (kE.xyw * E).xyxz; + PATTERN.xw += kB.xw * B; + PATTERN.xz += kF.xz * F; + + vec3 rgb = (alternate.y == 0.0) ? + ((alternate.x == 0.0) ? + vec3(C, PATTERN.xy) : + vec3(PATTERN.z, C, PATTERN.w)) : + ((alternate.x == 0.0) ? + vec3(PATTERN.w, C, PATTERN.z) : + vec3(PATTERN.yx, C)); + gl_FragColor = vec4(rgb, 1.0); +} diff --git a/src/qcam/assets/shader/bayer_8.vert b/src/qcam/assets/shader/bayer_8.vert new file mode 100644 index 00000000..26525db7 --- /dev/null +++ b/src/qcam/assets/shader/bayer_8.vert @@ -0,0 +1,72 @@ +/* +From http://jgt.akpeters.com/papers/McGuire08/ + +Efficient, High-Quality Bayer Demosaic Filtering on GPUs + +Morgan McGuire + +This paper appears in issue Volume 13, Number 4. +--------------------------------------------------------- +Copyright (c) 2008, Morgan McGuire. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +//Vertex Shader + +attribute vec4 vertexIn; +attribute vec2 textureIn; + +/* The texture size: tex_size.xy is in bytes, tex_size.zw is in pixels */ +uniform vec4 tex_size; +uniform vec2 tex_step; + +/** Pixel position of the first red pixel in the */ +/** Bayer pattern. [{0,1}, {0, 1}]*/ +uniform vec2 tex_bayer_first_red; + +/** .xy = Pixel being sampled in the fragment shader on the range [0, 1] + .zw = ...on the range [0, sourceSize], offset by firstRed */ +varying vec4 center; + +/** center.x + (-2/w, -1/w, 1/w, 2/w); These are the x-positions */ +/** of the adjacent pixels.*/ +varying vec4 xCoord; + +/** center.y + (-2/h, -1/h, 1/h, 2/h); These are the y-positions */ +/** of the adjacent pixels.*/ +varying vec4 yCoord; + +void main(void) { + center.xy = textureIn; + center.zw = textureIn * tex_size.zw + tex_bayer_first_red; + + xCoord = center.x + vec4(-2.0 * tex_step.x, + -tex_step.x, tex_step.x, 2.0 * tex_step.x); + yCoord = center.y + vec4(-2.0 * tex_step.y, + -tex_step.y, tex_step.y, 2.0 * tex_step.y); + + gl_Position = vertexIn; +} diff --git a/src/qcam/assets/shader/shaders.qrc b/src/qcam/assets/shader/shaders.qrc index d76d65c5..79f44a30 100644 --- a/src/qcam/assets/shader/shaders.qrc +++ b/src/qcam/assets/shader/shaders.qrc @@ -5,6 +5,7 @@ YUV_2_planes.frag YUV_3_planes.frag YUV_packed.frag + bayer_8.frag bayer_1x_packed.frag identity.vert diff --git a/src/qcam/viewfinder_gl.cpp b/src/qcam/viewfinder_gl.cpp index 13ab31aa..ef85e93d 100644 --- a/src/qcam/viewfinder_gl.cpp +++ b/src/qcam/viewfinder_gl.cpp @@ -36,6 +36,11 @@ static const QList supportedFormats{ libcamera::formats::RGBA8888, libcamera::formats::BGR888, libcamera::formats::RGB888, + /* Raw Bayer 8-bit */ + libcamera::formats::SBGGR8, + libcamera::formats::SGBRG8, + libcamera::formats::SGRBG8, + libcamera::formats::SRGGB8, /* Raw Bayer 10-bit packed */ libcamera::formats::SBGGR10_CSI2P, libcamera::formats::SGBRG10_CSI2P, @@ -220,6 +225,30 @@ bool ViewFinderGL::selectFormat(const libcamera::PixelFormat &format) fragmentShaderDefines_.append("#define RGB_PATTERN bgr"); fragmentShaderFile_ = ":RGB.frag"; break; + case libcamera::formats::SBGGR8: + firstRed_.setX(1.0); + firstRed_.setY(1.0); + fragmentShaderFile_ = ":bayer_8.frag"; + textureMinMagFilters_ = GL_NEAREST; + break; + case libcamera::formats::SGBRG8: + firstRed_.setX(0.0); + firstRed_.setY(1.0); + fragmentShaderFile_ = ":bayer_8.frag"; + textureMinMagFilters_ = GL_NEAREST; + break; + case libcamera::formats::SGRBG8: + firstRed_.setX(1.0); + firstRed_.setY(0.0); + fragmentShaderFile_ = ":bayer_8.frag"; + textureMinMagFilters_ = GL_NEAREST; + break; + case libcamera::formats::SRGGB8: + firstRed_.setX(0.0); + firstRed_.setY(0.0); + fragmentShaderFile_ = ":bayer_8.frag"; + textureMinMagFilters_ = GL_NEAREST; + break; case libcamera::formats::SBGGR10_CSI2P: firstRed_.setX(1.0); firstRed_.setY(1.0); @@ -624,6 +653,10 @@ void ViewFinderGL::doRender() shaderProgram_.setUniformValue(textureUniformY_, 0); break; + case libcamera::formats::SBGGR8: + case libcamera::formats::SGBRG8: + case libcamera::formats::SGRBG8: + case libcamera::formats::SRGGB8: case libcamera::formats::SBGGR10_CSI2P: case libcamera::formats::SGBRG10_CSI2P: case libcamera::formats::SGRBG10_CSI2P: @@ -633,8 +666,8 @@ void ViewFinderGL::doRender() case libcamera::formats::SGRBG12_CSI2P: case libcamera::formats::SRGGB12_CSI2P: /* - * Packed raw Bayer 10-bit and 12-bit formats are stored in - * GL_RED texture. + * Raw Bayer 8-bit, and packed raw Bayer 10-bit/12-bit formats + * are stored in GL_RED texture. */ glActiveTexture(GL_TEXTURE0); configureTexture(*textures_[0]);