From patchwork Thu Jun 13 13:26:00 2024 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Dan Scally X-Patchwork-Id: 20297 Return-Path: X-Original-To: parsemail@patchwork.libcamera.org Delivered-To: parsemail@patchwork.libcamera.org Received: from lancelot.ideasonboard.com (lancelot.ideasonboard.com [92.243.16.209]) by patchwork.libcamera.org (Postfix) with ESMTPS id 47BC1C32D0 for ; Thu, 13 Jun 2024 13:26:51 +0000 (UTC) Received: from lancelot.ideasonboard.com (localhost [IPv6:::1]) by lancelot.ideasonboard.com (Postfix) with ESMTP id AB321654AA; Thu, 13 Jun 2024 15:26:47 +0200 (CEST) Authentication-Results: lancelot.ideasonboard.com; dkim=pass (1024-bit key; unprotected) header.d=ideasonboard.com header.i=@ideasonboard.com header.b="Xjd49yAF"; dkim-atps=neutral Received: from perceval.ideasonboard.com (perceval.ideasonboard.com [IPv6:2001:4b98:dc2:55:216:3eff:fef7:d647]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id 175766549E for ; Thu, 13 Jun 2024 15:26:31 +0200 (CEST) Received: from mail.ideasonboard.com (cpc141996-chfd3-2-0-cust928.12-3.cable.virginm.net [86.13.91.161]) by perceval.ideasonboard.com (Postfix) with ESMTPSA id EFEBCE39; Thu, 13 Jun 2024 15:26:16 +0200 (CEST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=ideasonboard.com; s=mail; t=1718285177; bh=vw4/8NKlaP0apaFryUMqZb/vlDjetP8o3aT6zDKuM1Q=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=Xjd49yAFe+8nKn1wxNhiYJKy/GbVuhPo1xO+HFULr2udd/g7Rt3us1RPTkPFvuIXV LQBttIXNIWkZFGfsdsWdwNav4Rt+G70/oz7Kx9l5H5BHLDtIi1GaR4L5ZfsVL+TMX2 dY+hHTrPExSOqmjtn3Ko1Y/17eY5zrG/08XQ2utg= From: Daniel Scally To: libcamera-devel@lists.libcamera.org Cc: dan.scally@ideasonboard.com, nayden.kanchev@arm.com, jacopo.mondi@ideasonboard.com Subject: [PATCH 08/10] ipa: mali-c55: Add AWB Algorithm Date: Thu, 13 Jun 2024 14:26:00 +0100 Message-Id: <20240613132602.1021721-9-dan.scally@ideasonboard.com> X-Mailer: git-send-email 2.34.1 In-Reply-To: <20240613132602.1021721-1-dan.scally@ideasonboard.com> References: <20240613132602.1021721-1-dan.scally@ideasonboard.com> MIME-Version: 1.0 X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: libcamera-devel-bounces@lists.libcamera.org Sender: "libcamera-devel" Add a simple grey-world auto white balance algorithm to the mali-c55 IPA. Acked-by: Nayden Kanchev Co-developed-by: Jacopo Mondi Signed-off-by: Jacopo Mondi Signed-off-by: Daniel Scally --- src/ipa/mali-c55/algorithms/awb.cpp | 231 ++++++++++++++++++++++++ src/ipa/mali-c55/algorithms/awb.h | 38 ++++ src/ipa/mali-c55/algorithms/meson.build | 1 + src/ipa/mali-c55/ipa_context.h | 10 + 4 files changed, 280 insertions(+) create mode 100644 src/ipa/mali-c55/algorithms/awb.cpp create mode 100644 src/ipa/mali-c55/algorithms/awb.h diff --git a/src/ipa/mali-c55/algorithms/awb.cpp b/src/ipa/mali-c55/algorithms/awb.cpp new file mode 100644 index 00000000..5ab3dcba --- /dev/null +++ b/src/ipa/mali-c55/algorithms/awb.cpp @@ -0,0 +1,231 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2024, Ideas On Board Oy + * + * awb.cpp - Mali C55 grey world auto white balance algorithm + */ + +#include "awb.h" + +#include + +#include + +#include + +namespace libcamera { + +namespace ipa::mali_c55::algorithms { + +LOG_DEFINE_CATEGORY(MaliC55Awb) + +/* Number of frames at which we should run AWB at full speed */ +static constexpr uint32_t kNumStartupFrames = 4; + +Awb::Awb() +{ +} + +int Awb::configure([[maybe_unused]] IPAContext &context, + [[maybe_unused]] const IPACameraSensorInfo &configInfo) +{ + /* + * Initially we have no idea what the colour balance will be like, so + * for the first frame we will make no assumptions and leave the R/B + * channels unmodified. + */ + context.activeState.awb.rGain = 1.0; + context.activeState.awb.bGain = 1.0; + + return 0; +} + +void Awb::fillGainsParamBlock(mali_c55_params_block_header *block, IPAContext &context, + IPAFrameContext &frameContext) +{ + struct mali_c55_params_awb_gains *gains = + reinterpret_cast(block); + + gains->header.type = MALI_C55_PARAM_BLOCK_AWB_GAINS; + gains->header.enabled = true; + gains->header.size = sizeof(struct mali_c55_params_awb_gains); + + double rGain = context.activeState.awb.rGain; + double bGain = context.activeState.awb.bGain; + + /* + * The gains here map as follows: + * gain00 = R + * gain01 = Gr + * gain10 = Gb + * gain11 = B + * + * This holds true regardless of the bayer order of the input data, as + * the mapping is done internally in the ISP. + */ + gains->gain00 = int(rGain * pow(2, 8)); + gains->gain01 = 256; /* Otherwise known as 1.0 */ + gains->gain10 = 256; + gains->gain11 = int(bGain * pow(2, 8)); + + frameContext.awb.rGain = rGain; + frameContext.awb.bGain = bGain; +} + +void Awb::fillConfigParamBlock(mali_c55_params_block_header *block) +{ + struct mali_c55_params_awb_config *config = + reinterpret_cast(block); + + config->header.type = MALI_C55_PARAM_BLOCK_AWB_CONFIG; + config->header.enabled = true; + config->header.size = sizeof(struct mali_c55_params_awb_config); + + /* Tap the stats after the purple fringe block */ + config->tap_point = MALI_C55_AWB_STATS_TAP_PF; + + /* Get R/G and B/G ratios as statistics */ + config->stats_mode = MALI_C55_AWB_MODE_RGBG; + + /* Default white level */ + config->white_level = 1023; + + /* Default black level */ + config->black_level = 0; + + /* + * By default pixels are included who's colour ratios are bounded in a + * region (on a cr ratio x cb ratio graph) defined by four points: + * (0.25, 0.25) + * (0.25, 1.99609375) + * (1.99609375, 1.99609375) + * (1.99609375, 0.25) + * + * The ratios themselves are stored in Q4.8 format. + * + * \todo should these perhaps be tunable? + */ + config->cr_max = 511; + config->cr_min = 64; + config->cb_max = 511; + config->cb_min = 64; + + /* We use the full 15x15 zoning scheme */ + config->nodes_used_horiz = 15; + config->nodes_used_vert = 15; + + /* + * We set the trimming boundaries equivalent to the main boundaries. In + * other words; no trimming. + */ + config->cr_high = 511; + config->cr_low = 64; + config->cb_high = 511; + config->cb_low = 64; + +} + +void Awb::prepare(IPAContext &context, const uint32_t frame, + IPAFrameContext &frameContext, mali_c55_params_block_header *block) +{ + fillGainsParamBlock(block, context, frameContext); + + if (frame > 0) + return; + + char *params = reinterpret_cast(block); + + block = reinterpret_cast + (params + sizeof(struct mali_c55_params_awb_gains)); + + fillConfigParamBlock(block); +} + +void Awb::process(IPAContext &context, const uint32_t frame, + IPAFrameContext &frameContext, const mali_c55_stats_buffer *stats, + [[maybe_unused]] ControlList &metadata) +{ + const struct mali_c55_awb_average_ratios *awb_ratios = stats->awb_ratios; + + /* + * The ISP produces average R:G and B:G ratios for zones. We take the + * average of all the zones with data and simply invert them to provide + * gain figures that we can apply to approximate a grey world. + */ + unsigned int counted_zones = 0; + double rgSum = 0, bgSum = 0; + + for (unsigned int i = 0; i < 225; i++) { + if (!awb_ratios[i].num_pixels) + continue; + + /* + * The statistics are in Q4.8 format, so we convert to double + * here. + */ + rgSum += (awb_ratios[i].avg_rg_gr * pow(2, -8)); + bgSum += (awb_ratios[i].avg_bg_br * pow(2, -8)); + counted_zones++; + } + + /* + * Sometimes the first frame's statistics have no valid pixels, in which + * case we'll just assume a grey world until they say otherwise. + */ + double rgAvg, bgAvg; + if (!counted_zones) { + rgAvg = 1.0; + bgAvg = 1.0; + } else { + rgAvg = rgSum / counted_zones; + bgAvg = bgSum / counted_zones; + } + + /* + * The statistics are generated _after_ white balancing is performed in + * the ISP. To get the true ratio we therefore have to adjust the stats + * figure by the gains that were applied when the statistics for this + * frame were generated. + */ + double rRatio = rgAvg / frameContext.awb.rGain; + double bRatio = bgAvg / frameContext.awb.bGain; + + /* + * And then we can simply invert the ratio to find the gain we should + * apply. + */ + double rGain = 1 / rRatio; + double bGain = 1 / bRatio; + + /* + * Running at full speed, this algorithm results in oscillations in the + * colour balance. To remove those we dampen the speed at which it makes + * changes in gain, unless we're in the startup phase in which case we + * want to fix the miscolouring as quickly as possible. + */ + double speed = frame < kNumStartupFrames ? 1.0 : 0.2; + rGain = speed * rGain + context.activeState.awb.rGain * (1.0 - speed); + bGain = speed * bGain + context.activeState.awb.bGain * (1.0 - speed); + + context.activeState.awb.rGain = rGain; + context.activeState.awb.bGain = bGain; + + metadata.set(controls::ColourGains, { + static_cast(frameContext.awb.rGain), + static_cast(frameContext.awb.bGain), + }); + + LOG(MaliC55Awb, Debug) << "For frame number " << frame << ": " + << "Average R/G Ratio: " << rgAvg + << ", Average B/G Ratio: " << bgAvg + << "\nrGain applied to this frame: " << frameContext.awb.rGain + << ", bGain applied to this frame: " << frameContext.awb.bGain + << "\nrGain to apply: " << context.activeState.awb.rGain + << ", bGain to apply: " << context.activeState.awb.bGain; +} + +REGISTER_IPA_ALGORITHM(Awb, "Awb") + +} /* namespace ipa::mali_c55::algorithms */ + +} /* namespace libcamera */ diff --git a/src/ipa/mali-c55/algorithms/awb.h b/src/ipa/mali-c55/algorithms/awb.h new file mode 100644 index 00000000..4836585d --- /dev/null +++ b/src/ipa/mali-c55/algorithms/awb.h @@ -0,0 +1,38 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2024, Ideas on Board Oy + * + * awb.h - Mali C55 grey world auto white balance algorithm + */ + +#include "algorithm.h" +#include "ipa_context.h" + +namespace libcamera { + +namespace ipa::mali_c55::algorithms { + +class Awb : public Algorithm +{ +public: + Awb(); + ~Awb() = default; + + int configure(IPAContext &context, + const IPACameraSensorInfo &configInfo) override; + void prepare(IPAContext &context, const uint32_t frame, + IPAFrameContext &frameContext, + mali_c55_params_block_header *block) override; + void process(IPAContext &context, const uint32_t frame, + IPAFrameContext &frameContext, + const mali_c55_stats_buffer *stats, + ControlList &metadata) override; +private: + void fillGainsParamBlock(mali_c55_params_block_header *block, + IPAContext &context, IPAFrameContext &frameContext); + void fillConfigParamBlock(mali_c55_params_block_header *block); +}; + +} /* namespace ipa::mali_c55::algorithms */ + +} /* namespace libcamera */ diff --git a/src/ipa/mali-c55/algorithms/meson.build b/src/ipa/mali-c55/algorithms/meson.build index 96808431..f11791aa 100644 --- a/src/ipa/mali-c55/algorithms/meson.build +++ b/src/ipa/mali-c55/algorithms/meson.build @@ -2,5 +2,6 @@ mali_c55_ipa_algorithms = files([ 'agc.cpp', + 'awb.cpp', 'blc.cpp', ]) diff --git a/src/ipa/mali-c55/ipa_context.h b/src/ipa/mali-c55/ipa_context.h index 73a7cd78..105e5776 100644 --- a/src/ipa/mali-c55/ipa_context.h +++ b/src/ipa/mali-c55/ipa_context.h @@ -50,6 +50,11 @@ struct IPAActiveState { uint32_t exposureMode; uint32_t temperatureK; } agc; + + struct { + double rGain; + double bGain; + } awb; }; struct IPAFrameContext : public FrameContext { @@ -58,6 +63,11 @@ struct IPAFrameContext : public FrameContext { double sensorGain; double ispGain; } agc; + + struct { + double rGain; + double bGain; + } awb; }; struct IPAContext {