From patchwork Wed Jun 11 01:32:36 2025 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Bryan O'Donoghue X-Patchwork-Id: 23532 Return-Path: X-Original-To: parsemail@patchwork.libcamera.org Delivered-To: parsemail@patchwork.libcamera.org Received: from lancelot.ideasonboard.com (lancelot.ideasonboard.com [92.243.16.209]) by patchwork.libcamera.org (Postfix) with ESMTPS id E8FEBC3331 for ; Wed, 11 Jun 2025 01:34:17 +0000 (UTC) Received: from lancelot.ideasonboard.com (localhost [IPv6:::1]) by lancelot.ideasonboard.com (Postfix) with ESMTP id D043C68DF2; Wed, 11 Jun 2025 03:34:16 +0200 (CEST) Authentication-Results: lancelot.ideasonboard.com; dkim=pass (2048-bit key; unprotected) header.d=linaro.org header.i=@linaro.org header.b="g4T9lbaZ"; dkim-atps=neutral Received: from mail-wm1-x32a.google.com (mail-wm1-x32a.google.com [IPv6:2a00:1450:4864:20::32a]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id 6C0B768DEF for ; Wed, 11 Jun 2025 03:34:13 +0200 (CEST) Received: by mail-wm1-x32a.google.com with SMTP id 5b1f17b1804b1-450cf214200so56603065e9.1 for ; Tue, 10 Jun 2025 18:34:13 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=linaro.org; s=google; t=1749605653; x=1750210453; darn=lists.libcamera.org; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:from:to:cc:subject:date :message-id:reply-to; bh=kou1M46O9tnzkm45TzkfsVRpNX7itRxzavLfzZx/hkQ=; b=g4T9lbaZ/vYURQCL1Gn1cjvRNhEChTRH/J0vZE6IWCvR5V9LiNAXcijAf7CaQUeGtK H+yJtC0jOUnBEx5ies/IewD4Uo3Jnpd0uA1cpi5uhLL1eN25g1bwwrbS8IxL3LIk6vtq AHFGwpqKmoW9eh1BAIlgNDXWRlX7MC8MFObBUIV2TR/12o874GCGkfFzEi24QGjoYmmG sZj9Ax7QUeNbBrb6sNngh4ZMOsYiigyJEfzk25L+uaHGTrQWFO2zmwIUbJXGePnf5lBB XoMsg2t/V32suGYGhApbJ54cE6NrUQEfeVDJK/ULIlLC67vZgBt0yczMSdbdimt1r3h9 nG1g== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20230601; t=1749605653; x=1750210453; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:x-gm-message-state:from:to:cc :subject:date:message-id:reply-to; bh=kou1M46O9tnzkm45TzkfsVRpNX7itRxzavLfzZx/hkQ=; b=KnZSljZighBmtTy2kKhGZ3/TPpvgZZoh2K4m4dmFj0Q6Y/SPEIJjien1L8slGn1C5m YGcrriUYodocga2Va2WT6u8E+2tjK3bFKQa+yftsLQOajDqJ3x3LP5h9nXqaElmpoVYJ 07eefPu6dsIeoG1BQ6+QcJ17olAED9Xjq7GXxbzUI/wzoCn+KNvmNT/NNfuM6mmpXVtq sprEgC7Gg6mdsgrUFQuOo784/34jSBU4sBVR2xPkWpv4e1r9/EuIfF32QnLkKVWdOrfb L111nkF8/Ir0aZSJ9D6mBa4IiHIKDpsPvrVuw1e9+bgHAMLdXpIM55EU/OH8eNfeqlFs LOow== X-Gm-Message-State: AOJu0Yx1GfCKZywA3NEztvtrOdPEkcX/PrmATYRIElAO2f3t/PRc189Z aXlL1BTZIXG2UTO0LJE4j2ZuUlxDEN4RT9oEKdjSlBOzfV3l5YPyide9vDG99SWb0/lTiQq2g7+ vrJyOdxo= X-Gm-Gg: ASbGnct1+V4c0WGGYKJBS66O+y2oQvmrBXKodyV2KcfANQD1jg/WKib0U2VbYjA/lk2 Z8r6GfxE4fbcjPMFea/uK0VNrNiHGVhwXUJvEaUTAopNLbyl7BwP9LZdoXuk7bOvg4Hm/06E8Wh 6RibNcYVwjtbFgMWHWc4ItM1My/Vp44CdA2kgYTJN0IrAbp5Dp/QUYkxCVZo3NX4ER6n2tpoSwG Nc4tiSUPO8Q0hb5XtuPdDsB/yWUpKDod6HpJK41Gbx2wI9hiFMLIceC6VbWJaxs65nT/kbV8Xoi ZDJStHm2dBCFxoQaCSy3/T/eAaO8VEpjKlNmE6dj9lzYYDQVdJfTO1fsez3pF2DntGToiCIwHr8 J8/kcO/KZUkPcFSTYATxErv8Kmncj2sVDq2sLAIH/aA== X-Google-Smtp-Source: AGHT+IE4aFfPCQTf035R8DBlYuxane9r27Xpk0diUUUxTvhvjyo/QIKBTm1B37uCXcX3SvRLnuDDlA== X-Received: by 2002:a05:600c:8b13:b0:450:d3b9:4bac with SMTP id 5b1f17b1804b1-45324f4f377mr4999125e9.28.1749605652599; Tue, 10 Jun 2025 18:34:12 -0700 (PDT) Received: from inspiron14p-linux.ht.home (188-141-3-146.dynamic.upc.ie. [188.141.3.146]) by smtp.gmail.com with ESMTPSA id 5b1f17b1804b1-4532514138asm5680625e9.3.2025.06.10.18.34.10 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Tue, 10 Jun 2025 18:34:12 -0700 (PDT) From: Bryan O'Donoghue To: libcamera-devel@lists.libcamera.org Cc: Bryan O'Donoghue Subject: [PATCH 26/35] libcamera: shaders: Extend debayer shaders to apply RGB gain values on output Date: Wed, 11 Jun 2025 02:32:36 +0100 Message-ID: <20250611013245.133785-27-bryan.odonoghue@linaro.org> X-Mailer: git-send-email 2.49.0 In-Reply-To: <20250611013245.133785-1-bryan.odonoghue@linaro.org> References: <20250611013245.133785-1-bryan.odonoghue@linaro.org> MIME-Version: 1.0 X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: libcamera-devel-bounces@lists.libcamera.org Sender: "libcamera-devel" Extend out the bayer fragment shaders to take 3 x 256 byte inputs as textures from the CPU. We then use an index to the table to recover the colour-corrected values provided by the SoftIPA thread. Signed-off-by: Bryan O'Donoghue --- .../internal/shaders/bayer_1x_packed.frag | 56 ++++++++++++++++ .../libcamera/internal/shaders/bayer_8.frag | 62 ++++++++++++++++- src/libcamera/software_isp/debayer_egl.cpp | 67 +++++++++++++------ src/libcamera/software_isp/debayer_egl.h | 7 +- 4 files changed, 169 insertions(+), 23 deletions(-) diff --git a/include/libcamera/internal/shaders/bayer_1x_packed.frag b/include/libcamera/internal/shaders/bayer_1x_packed.frag index 19b13ad0..90bd6457 100644 --- a/include/libcamera/internal/shaders/bayer_1x_packed.frag +++ b/include/libcamera/internal/shaders/bayer_1x_packed.frag @@ -65,6 +65,10 @@ uniform vec2 tex_step; uniform vec2 tex_bayer_first_red; uniform sampler2D tex_y; +uniform sampler2D red_param; +uniform sampler2D green_param; +uniform sampler2D blue_param; +uniform mat3 ccm; void main(void) { @@ -212,5 +216,57 @@ void main(void) vec3(patterns.y, C, patterns.x) : vec3(patterns.wz, C)); +#if defined(APPLY_CCM_PARAMETERS) + /* + * CCM is a 3x3 in the format + * + * +--------------+----------------+---------------+ + * | RedRedGain | RedGreenGain | RedBlueGain | + * +--------------+----------------+---------------+ + * | GreenRedGain | GreenGreenGain | GreenBlueGain | + * +--------------+----------------+---------------+ + * | BlueRedGain | BlueGreenGain | BlueBlueGain | + * +--------------+----------------+---------------+ + * + * Rout = RedRedGain * Rin + RedGreenGain * Gin + RedBlueGain * Bin + * Gout = GreenRedGain * Rin + GreenGreenGain * Gin + GreenBlueGain * Bin + * Bout = BlueRedGain * Rin + BlueGreenGain * Gin + BlueBlueGain * Bin + * + * We upload to the GPU without transposition glUniformMatrix3f(.., .., GL_FALSE, ccm); + * + * CPU + * float ccm [] = { + * RedRedGain, RedGreenGain, RedBlueGain, + * GreenRedGain, GreenGreenGain, GreenBlueGain, + * BlueRedGain, BlueGreenGain, BlueBlueGain, + * }; + * + * GPU + * ccm = { + * RedRedGain, GreenRedGain, BlueRedGain, + * RedGreenGain, GreenGreenGain, BlueGreenGain, + * RedBlueGain, GreenBlueGain, BlueBlueGain, + * } + * + * However the indexing for the mat data-type is column major hence + * ccm[0][0] = RedRedGain, ccm[0][1] = RedGreenGain, ccm[0][2] = RedBlueGain + * + */ + float rin, gin, bin; + rin = rgb.r; + gin = rgb.g; + bin = rgb.b; + + rgb.r = (rin * ccm[0][0]) + (gin * ccm[0][1]) + (bin * ccm[0][2]); + rgb.g = (rin * ccm[1][0]) + (gin * ccm[1][1]) + (bin * ccm[1][2]); + rgb.b = (rin * ccm[2][0]) + (gin * ccm[2][1]) + (bin * ccm[2][2]); + +#elif defined(APPLY_RGB_PARAMETERS) + /* Apply bayer params */ + rgb.r = texture2D(red_param, vec2(rgb.r, 0.5)).r; + rgb.g = texture2D(green_param, vec2(rgb.g, 0.5)).g; + rgb.b = texture2D(blue_param, vec2(rgb.b, 0.5)).b; +#endif + gl_FragColor = vec4(rgb, 1.0); } diff --git a/include/libcamera/internal/shaders/bayer_8.frag b/include/libcamera/internal/shaders/bayer_8.frag index aa7a1b00..5955c2ea 100644 --- a/include/libcamera/internal/shaders/bayer_8.frag +++ b/include/libcamera/internal/shaders/bayer_8.frag @@ -21,11 +21,17 @@ precision highp float; /** Monochrome RGBA or GL_LUMINANCE Bayer encoded texture.*/ uniform sampler2D tex_y; +uniform sampler2D red_param; +uniform sampler2D green_param; +uniform sampler2D blue_param; varying vec4 center; varying vec4 yCoord; varying vec4 xCoord; +uniform mat3 ccm; void main(void) { + vec3 rgb; + #define fetch(x, y) texture2D(tex_y, vec2(x, y)).r float C = texture2D(tex_y, center.xy).r; // ( 0, 0) @@ -97,11 +103,65 @@ void main(void) { PATTERN.xw += kB.xw * B; PATTERN.xz += kF.xz * F; - gl_FragColor.rgb = (alternate.y == 0.0) ? + rgb = (alternate.y == 0.0) ? ((alternate.x == 0.0) ? vec3(C, PATTERN.xy) : vec3(PATTERN.z, C, PATTERN.w)) : ((alternate.x == 0.0) ? vec3(PATTERN.w, C, PATTERN.z) : vec3(PATTERN.yx, C)); + +#if defined(APPLY_CCM_PARAMETERS) + /* + * CCM is a 3x3 in the format + * + * +--------------+----------------+---------------+ + * | RedRedGain | RedGreenGain | RedBlueGain | + * +--------------+----------------+---------------+ + * | GreenRedGain | GreenGreenGain | GreenBlueGain | + * +--------------+----------------+---------------+ + * | BlueRedGain | BlueGreenGain | BlueBlueGain | + * +--------------+----------------+---------------+ + * + * Rout = RedRedGain * Rin + RedGreenGain * Gin + RedBlueGain * Bin + * Gout = GreenRedGain * Rin + GreenGreenGain * Gin + GreenBlueGain * Bin + * Bout = BlueRedGain * Rin + BlueGreenGain * Gin + BlueBlueGain * Bin + * + * We upload to the GPU without transposition glUniformMatrix3f(.., .., GL_FALSE, ccm); + * + * CPU + * float ccm [] = { + * RedRedGain, RedGreenGain, RedBlueGain, + * GreenRedGain, GreenGreenGain, GreenBlueGain, + * BlueRedGain, BlueGreenGain, BlueBlueGain, + * }; + * + * GPU + * ccm = { + * RedRedGain, GreenRedGain, BlueRedGain, + * RedGreenGain, GreenGreenGain, BlueGreenGain, + * RedBlueGain, GreenBlueGain, BlueBlueGain, + * } + * + * However the indexing for the mat data-type is column major hence + * ccm[0][0] = RedRedGain, ccm[0][1] = RedGreenGain, ccm[0][2] = RedBlueGain + * + */ + float rin, gin, bin; + rin = rgb.r; + gin = rgb.g; + bin = rgb.b; + + rgb.r = (rin * ccm[0][0]) + (gin * ccm[0][1]) + (bin * ccm[0][2]); + rgb.g = (rin * ccm[1][0]) + (gin * ccm[1][1]) + (bin * ccm[1][2]); + rgb.b = (rin * ccm[2][0]) + (gin * ccm[2][1]) + (bin * ccm[2][2]); + +#elif defined(APPLY_RGB_PARAMETERS) + /* Apply bayer params */ + rgb.r = texture2D(red_param, vec2(rgb.r, 0.5)).r; + rgb.g = texture2D(red_param, vec2(rgb.g, 0.5)).g; + rgb.b = texture2D(red_param, vec2(rgb.b, 0.5)).b; +#endif + + gl_FragColor.rgb = rgb; } diff --git a/src/libcamera/software_isp/debayer_egl.cpp b/src/libcamera/software_isp/debayer_egl.cpp index 3fb15511..824cd6d3 100644 --- a/src/libcamera/software_isp/debayer_egl.cpp +++ b/src/libcamera/software_isp/debayer_egl.cpp @@ -99,6 +99,7 @@ int DebayerEGL::getShaderVariableLocations(void) textureUniformRedLookupDataIn_ = glGetUniformLocation(programId_, "red_param"); textureUniformGreenLookupDataIn_ = glGetUniformLocation(programId_, "green_param"); textureUniformBlueLookupDataIn_ = glGetUniformLocation(programId_, "blue_param"); + ccmUniformDataIn_ = glGetUniformLocation(programId_, "ccm"); textureUniformStep_ = glGetUniformLocation(programId_, "tex_step"); textureUniformSize_ = glGetUniformLocation(programId_, "tex_size"); @@ -111,6 +112,7 @@ int DebayerEGL::getShaderVariableLocations(void) << " red_param " << textureUniformRedLookupDataIn_ << " green_param " << textureUniformGreenLookupDataIn_ << " blue_param " << textureUniformBlueLookupDataIn_ + << " ccm " << ccmUniformDataIn_ << " tex_step " << textureUniformStep_ << " tex_size " << textureUniformSize_ << " stride_factor " << textureUniformStrideFactor_ @@ -215,8 +217,13 @@ int DebayerEGL::initBayerShaders(PixelFormat inputFormat, PixelFormat outputForm break; }; - // Flag to shaders that we have parameter gain tables - egl_.pushEnv(shaderEnv, "#define APPLY_RGB_PARAMETERS"); + if (ccmEnabled_) { + // Run the CCM if available + egl_.pushEnv(shaderEnv, "#define APPLY_CCM_PARAMETERS"); + } else { + // Flag to shaders that we have parameter gain tables + egl_.pushEnv(shaderEnv, "#define APPLY_RGB_PARAMETERS"); + } if (egl_.compileVertexShader(vertexShaderId_, vertexShaderData, vertexShaderDataLen, shaderEnv)) goto compile_fail; @@ -266,6 +273,8 @@ int DebayerEGL::configure(const StreamConfiguration &inputCfg, const std::vector> &outputCfgs, bool ccmEnabled) { + GLint maxTextureImageUnits; + if (getInputConfig(inputCfg.pixelFormat, inputConfig_) != 0) return -EINVAL; @@ -284,7 +293,7 @@ int DebayerEGL::configure(const StreamConfiguration &inputCfg, inputConfig_.stride = inputCfg.stride; width_ = inputCfg.size.width; height_ = inputCfg.size.height; - ccmEnabled_ = ccmEnabled = false; + ccmEnabled_ = ccmEnabled = true; if (outputCfgs.size() != 1) { LOG(Debayer, Error) @@ -304,30 +313,35 @@ int DebayerEGL::configure(const StreamConfiguration &inputCfg, glGetIntegerv(GL_MAX_TEXTURE_IMAGE_UNITS, &maxTextureImageUnits); LOG(Debayer, Debug) << "Fragment shader maximum texture units " << maxTextureImageUnits; - if (maxTextureImageUnits < DEBAYER_EGL_MIN_SIMPLE_RGB_GAIN_TEXTURE_UNITS) { + if (!ccmEnabled && maxTextureImageUnits < DEBAYER_EGL_MIN_SIMPLE_RGB_GAIN_TEXTURE_UNITS) { LOG(Debayer, Error) << "Fragment shader texture unit count " << maxTextureImageUnits << " required minimum for RGB gain table lookup " << DEBAYER_EGL_MIN_SIMPLE_RGB_GAIN_TEXTURE_UNITS << " try using an identity CCM "; return -ENODEV; } + // Raw bayer input as texture eglImageBayerIn_ = new eGLImage(width_, height_, 32, GL_TEXTURE0, 0); if (!eglImageBayerIn_) return -ENOMEM; - /// RGB correction tables as 2d textures - // eGL doesn't support glTexImage1D so we do a little hack with 2D to compensate - eglImageRedLookup_ = new eGLImage(DebayerParams::kRGBLookupSize, 1, 32, GL_TEXTURE1, 1); - if (!eglImageRedLookup_) - return -ENOMEM; + // Only do the RGB lookup table textures if CCM is disabled + if (!ccmEnabled_) { - eglImageGreenLookup_ = new eGLImage(DebayerParams::kRGBLookupSize, 1, 32, GL_TEXTURE2, 2); - if (!eglImageGreenLookup_) - return -ENOMEM; + /// RGB correction tables as 2d textures + // eGL doesn't support glTexImage1D so we do a little hack with 2D to compensate + eglImageRedLookup_ = new eGLImage(DebayerParams::kRGBLookupSize, 1, 32, GL_TEXTURE1, 1); + if (!eglImageRedLookup_) + return -ENOMEM; - eglImageBlueLookup_ = new eGLImage(DebayerParams::kRGBLookupSize, 1, 32, GL_TEXTURE3, 3); - if (!eglImageBlueLookup_) - return -ENOMEM; + eglImageGreenLookup_ = new eGLImage(DebayerParams::kRGBLookupSize, 1, 32, GL_TEXTURE2, 2); + if (!eglImageGreenLookup_) + return -ENOMEM; + + eglImageBlueLookup_ = new eGLImage(DebayerParams::kRGBLookupSize, 1, 32, GL_TEXTURE3, 3); + if (!eglImageBlueLookup_) + return -ENOMEM; + } // Create a single BO (calling gbm_surface_lock_front_buffer() again before gbm_surface_release_buffer() would create another BO) if (gbmSurface_.mapSurface()) @@ -440,9 +454,11 @@ void DebayerEGL::setShaderVariableValues(void) // To simultaneously sample multiple textures we need to use multiple // texture units glUniform1i(textureUniformBayerDataIn_, eglImageBayerIn_->texture_unit_uniform_id_); - glUniform1i(textureUniformRedLookupDataIn_, eglImageRedLookup_->texture_unit_uniform_id_); - glUniform1i(textureUniformGreenLookupDataIn_, eglImageGreenLookup_->texture_unit_uniform_id_); - glUniform1i(textureUniformBlueLookupDataIn_, eglImageBlueLookup_->texture_unit_uniform_id_); + if (!ccmEnabled_) { + glUniform1i(textureUniformRedLookupDataIn_, eglImageRedLookup_->texture_unit_uniform_id_); + glUniform1i(textureUniformGreenLookupDataIn_, eglImageGreenLookup_->texture_unit_uniform_id_); + glUniform1i(textureUniformBlueLookupDataIn_, eglImageBlueLookup_->texture_unit_uniform_id_); + } // These values are: // firstRed = tex_bayer_first_red - bayer_8.vert @@ -494,9 +510,18 @@ void DebayerEGL::debayerGPU(MappedFrameBuffer &in, MappedFrameBuffer &out, Debay egl_.createTexture2D(eglImageBayerIn_, inputConfig_.stride, height_, in.planes()[0].data()); // Populate bayer parameters - egl_.createTexture2D(eglImageRedLookup_, DebayerParams::kRGBLookupSize, 1, ¶ms.red); - egl_.createTexture2D(eglImageGreenLookup_, DebayerParams::kRGBLookupSize, 1, ¶ms.green); - egl_.createTexture2D(eglImageBlueLookup_, DebayerParams::kRGBLookupSize, 1, ¶ms.blue); + if (ccmEnabled_) { + GLfloat ccm[] = { + 1, 0, 0, + 0, 1, 0, + 0, 0, 1, + }; + glUniformMatrix3fv(ccmUniformDataIn_, 1, GL_FALSE, ccm); + } else { + egl_.createTexture2D(eglImageRedLookup_, DebayerParams::kRGBLookupSize, 1, ¶ms.red); + egl_.createTexture2D(eglImageGreenLookup_, DebayerParams::kRGBLookupSize, 1, ¶ms.green); + egl_.createTexture2D(eglImageBlueLookup_, DebayerParams::kRGBLookupSize, 1, ¶ms.blue); + } // Setup the scene setShaderVariableValues(); diff --git a/src/libcamera/software_isp/debayer_egl.h b/src/libcamera/software_isp/debayer_egl.h index c0fc220f..94bc6fc4 100644 --- a/src/libcamera/software_isp/debayer_egl.h +++ b/src/libcamera/software_isp/debayer_egl.h @@ -134,17 +134,22 @@ private: GLint textureUniformProjMatrix_; GLint textureUniformBayerDataIn_; + + // These textures will either point to simple RGB gains or to CCM lookup tables GLint textureUniformRedLookupDataIn_; GLint textureUniformGreenLookupDataIn_; GLint textureUniformBlueLookupDataIn_; + // Represent per-frame CCM as a uniform vector of floats 3 x 3 + GLint ccmUniformDataIn_; + bool ccmEnabled_; + Rectangle window_; std::unique_ptr stats_; eGL egl_; GBM gbmSurface_; uint32_t width_; uint32_t height_; - bool ccmEnabled_; GLfloat vcoordinates[DEBAYER_OPENGL_COORDS][2] = { { -1.0f, -1.0f },