Patch Detail
Show a patch.
GET /api/1.1/patches/21857/?format=api
{ "id": 21857, "url": "https://patchwork.libcamera.org/api/1.1/patches/21857/?format=api", "web_url": "https://patchwork.libcamera.org/patch/21857/", "project": { "id": 1, "url": "https://patchwork.libcamera.org/api/1.1/projects/1/?format=api", "name": "libcamera", "link_name": "libcamera", "list_id": "libcamera_core", "list_email": "libcamera-devel@lists.libcamera.org", "web_url": "", "scm_url": "", "webscm_url": "" }, "msgid": "<20241107114819.57599-10-dan.scally@ideasonboard.com>", "date": "2024-11-07T11:48:17", "name": "[v3,09/11] ipa: mali-c55: Add AWB Algorithm", "commit_ref": null, "pull_url": null, "state": "superseded", "archived": false, "hash": "8e59ef99a44ebab4af58f977e0dc55f48720e911", "submitter": { "id": 156, "url": "https://patchwork.libcamera.org/api/1.1/people/156/?format=api", "name": "Dan Scally", "email": "dan.scally@ideasonboard.com" }, "delegate": null, "mbox": "https://patchwork.libcamera.org/patch/21857/mbox/", "series": [ { "id": 4777, "url": "https://patchwork.libcamera.org/api/1.1/series/4777/?format=api", "web_url": "https://patchwork.libcamera.org/project/libcamera/list/?series=4777", "date": "2024-11-07T11:48:08", "name": "Add Mali-C55 IPA Module and Algorithms", "version": 3, "mbox": "https://patchwork.libcamera.org/series/4777/mbox/" } ], "comments": "https://patchwork.libcamera.org/api/patches/21857/comments/", "check": "pending", "checks": "https://patchwork.libcamera.org/api/patches/21857/checks/", "tags": {}, "headers": { "Return-Path": "<libcamera-devel-bounces@lists.libcamera.org>", "X-Original-To": "parsemail@patchwork.libcamera.org", "Delivered-To": "parsemail@patchwork.libcamera.org", "Received": [ "from lancelot.ideasonboard.com (lancelot.ideasonboard.com\n\t[92.243.16.209])\n\tby patchwork.libcamera.org (Postfix) with ESMTPS id 08603C323E\n\tfor <parsemail@patchwork.libcamera.org>;\n\tThu, 7 Nov 2024 11:48:46 +0000 (UTC)", "from lancelot.ideasonboard.com (localhost [IPv6:::1])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTP id B1A9C65497;\n\tThu, 7 Nov 2024 12:48:44 +0100 (CET)", "from perceval.ideasonboard.com (perceval.ideasonboard.com\n\t[IPv6:2001:4b98:dc2:55:216:3eff:fef7:d647])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id 44CC46547D\n\tfor <libcamera-devel@lists.libcamera.org>;\n\tThu, 7 Nov 2024 12:48:30 +0100 (CET)", "from mail.ideasonboard.com\n\t(cpc141996-chfd3-2-0-cust928.12-3.cable.virginm.net [86.13.91.161])\n\tby perceval.ideasonboard.com (Postfix) with ESMTPSA id 453851858;\n\tThu, 7 Nov 2024 12:48:21 +0100 (CET)" ], "Authentication-Results": "lancelot.ideasonboard.com; dkim=pass (1024-bit key;\n\tunprotected) header.d=ideasonboard.com header.i=@ideasonboard.com\n\theader.b=\"CGQ8EGKy\"; dkim-atps=neutral", "DKIM-Signature": "v=1; a=rsa-sha256; c=relaxed/simple; d=ideasonboard.com;\n\ts=mail; t=1730980101;\n\tbh=rF79IPcQgmazGPs4CMmH2QfCxbIKXE4SzmJzxQqiLjc=;\n\th=From:To:Cc:Subject:Date:In-Reply-To:References:From;\n\tb=CGQ8EGKykWoWvImoiKteWxELPi0yow76juxNG+1NoB3G/DnwawSrxGFBvckn9uCr9\n\te/VVoL46XhghMnoHKf4BRFaIEi/1tRi9cnAd7/pG3A4DdJEOu5ZadyzxwcbYUD6UGD\n\taW/p3LAASbLF/XokWOqjpE7euLXTH+k14ssvLTKs=", "From": "Daniel Scally <dan.scally@ideasonboard.com>", "To": "libcamera-devel@lists.libcamera.org", "Cc": "Anthony.McGivern@arm.com, Daniel Scally <dan.scally@ideasonboard.com>,\n\tKieran Bingham <kieran.bingham@ideasonboard.com>,\n\tNayden Kanchev <nayden.kanchev@arm.com>,\n\tJacopo Mondi <jacopo.mondi@ideasonboard.com>", "Subject": "[PATCH v3 09/11] ipa: mali-c55: Add AWB Algorithm", "Date": "Thu, 7 Nov 2024 11:48:17 +0000", "Message-Id": "<20241107114819.57599-10-dan.scally@ideasonboard.com>", "X-Mailer": "git-send-email 2.34.1", "In-Reply-To": "<20241107114819.57599-1-dan.scally@ideasonboard.com>", "References": "<20241107114819.57599-1-dan.scally@ideasonboard.com>", "MIME-Version": "1.0", "Content-Transfer-Encoding": "8bit", "X-BeenThere": "libcamera-devel@lists.libcamera.org", "X-Mailman-Version": "2.1.29", "Precedence": "list", "List-Id": "<libcamera-devel.lists.libcamera.org>", "List-Unsubscribe": "<https://lists.libcamera.org/options/libcamera-devel>,\n\t<mailto:libcamera-devel-request@lists.libcamera.org?subject=unsubscribe>", "List-Archive": "<https://lists.libcamera.org/pipermail/libcamera-devel/>", "List-Post": "<mailto:libcamera-devel@lists.libcamera.org>", "List-Help": "<mailto:libcamera-devel-request@lists.libcamera.org?subject=help>", "List-Subscribe": "<https://lists.libcamera.org/listinfo/libcamera-devel>,\n\t<mailto:libcamera-devel-request@lists.libcamera.org?subject=subscribe>", "Errors-To": "libcamera-devel-bounces@lists.libcamera.org", "Sender": "\"libcamera-devel\" <libcamera-devel-bounces@lists.libcamera.org>" }, "content": "Add a simple grey-world auto white balance algorithm to the mali-c55\nIPA.\n\nReviewed-by: Kieran Bingham <kieran.bingham@ideasonboard.com>\nAcked-by: Nayden Kanchev <nayden.kanchev@arm.com>\nCo-developed-by: Jacopo Mondi <jacopo.mondi@ideasonboard.com>\nSigned-off-by: Jacopo Mondi <jacopo.mondi@ideasonboard.com>\nSigned-off-by: Daniel Scally <dan.scally@ideasonboard.com>\n---\nChanges in v3:\n\n\t- Used the Q format helpers\n\nChanges in v2:\n\n\t- Use the union rather than reinterpret_cast<>() to abstract the block\n\n src/ipa/mali-c55/algorithms/awb.cpp | 230 ++++++++++++++++++++++++\n src/ipa/mali-c55/algorithms/awb.h | 40 +++++\n src/ipa/mali-c55/algorithms/meson.build | 1 +\n src/ipa/mali-c55/ipa_context.h | 10 ++\n 4 files changed, 281 insertions(+)\n create mode 100644 src/ipa/mali-c55/algorithms/awb.cpp\n create mode 100644 src/ipa/mali-c55/algorithms/awb.h", "diff": "diff --git a/src/ipa/mali-c55/algorithms/awb.cpp b/src/ipa/mali-c55/algorithms/awb.cpp\nnew file mode 100644\nindex 00000000..c956757f\n--- /dev/null\n+++ b/src/ipa/mali-c55/algorithms/awb.cpp\n@@ -0,0 +1,230 @@\n+/* SPDX-License-Identifier: LGPL-2.1-or-later */\n+/*\n+ * Copyright (C) 2024, Ideas On Board Oy\n+ *\n+ * awb.cpp - Mali C55 grey world auto white balance algorithm\n+ */\n+\n+#include \"awb.h\"\n+\n+#include <cmath>\n+\n+#include <libcamera/base/log.h>\n+#include <libcamera/base/utils.h>\n+\n+#include <libcamera/control_ids.h>\n+\n+#include \"libipa/helpers.h\"\n+\n+namespace libcamera {\n+\n+namespace ipa::mali_c55::algorithms {\n+\n+LOG_DEFINE_CATEGORY(MaliC55Awb)\n+\n+/* Number of frames at which we should run AWB at full speed */\n+static constexpr uint32_t kNumStartupFrames = 4;\n+\n+Awb::Awb()\n+{\n+}\n+\n+int Awb::configure([[maybe_unused]] IPAContext &context,\n+\t\t [[maybe_unused]] const IPACameraSensorInfo &configInfo)\n+{\n+\t/*\n+\t * Initially we have no idea what the colour balance will be like, so\n+\t * for the first frame we will make no assumptions and leave the R/B\n+\t * channels unmodified.\n+\t */\n+\tcontext.activeState.awb.rGain = 1.0;\n+\tcontext.activeState.awb.bGain = 1.0;\n+\n+\treturn 0;\n+}\n+\n+size_t Awb::fillGainsParamBlock(mali_c55_params_block block, IPAContext &context,\n+\t\t\t\tIPAFrameContext &frameContext)\n+{\n+\tblock.header->type = MALI_C55_PARAM_BLOCK_AWB_GAINS;\n+\tblock.header->flags = MALI_C55_PARAM_BLOCK_FL_NONE;\n+\tblock.header->size = sizeof(struct mali_c55_params_awb_gains);\n+\n+\tdouble rGain = context.activeState.awb.rGain;\n+\tdouble bGain = context.activeState.awb.bGain;\n+\n+\t/*\n+\t * The gains here map as follows:\n+\t *\tgain00 = R\n+\t *\tgain01 = Gr\n+\t *\tgain10 = Gb\n+\t *\tgain11 = B\n+\t *\n+\t * This holds true regardless of the bayer order of the input data, as\n+\t * the mapping is done internally in the ISP.\n+\t */\n+\tblock.awb_gains->gain00 = ipa::toUnsignedQFormat(rGain, 4, 8);\n+\tblock.awb_gains->gain01 = ipa::toUnsignedQFormat(1.0, 4, 8);\n+\tblock.awb_gains->gain10 = ipa::toUnsignedQFormat(1.0, 4, 8);\n+\tblock.awb_gains->gain11 = ipa::toUnsignedQFormat(bGain, 4, 8);\n+\n+\tframeContext.awb.rGain = rGain;\n+\tframeContext.awb.bGain = bGain;\n+\n+\treturn sizeof(struct mali_c55_params_awb_gains);\n+}\n+\n+size_t Awb::fillConfigParamBlock(mali_c55_params_block block)\n+{\n+\tblock.header->type = MALI_C55_PARAM_BLOCK_AWB_CONFIG;\n+\tblock.header->flags = MALI_C55_PARAM_BLOCK_FL_NONE;\n+\tblock.header->size = sizeof(struct mali_c55_params_awb_config);\n+\n+\t/* Tap the stats after the purple fringe block */\n+\tblock.awb_config->tap_point = MALI_C55_AWB_STATS_TAP_PF;\n+\n+\t/* Get R/G and B/G ratios as statistics */\n+\tblock.awb_config->stats_mode = MALI_C55_AWB_MODE_RGBG;\n+\n+\t/* Default white level */\n+\tblock.awb_config->white_level = 1023;\n+\n+\t/* Default black level */\n+\tblock.awb_config->black_level = 0;\n+\n+\t/*\n+\t * By default pixels are included who's colour ratios are bounded in a\n+\t * region (on a cr ratio x cb ratio graph) defined by four points:\n+\t *\t(0.25, 0.25)\n+\t *\t(0.25, 1.99609375)\n+\t *\t(1.99609375, 1.99609375)\n+\t *\t(1.99609375, 0.25)\n+\t *\n+\t * The ratios themselves are stored in Q4.8 format.\n+\t *\n+\t * \\todo should these perhaps be tunable?\n+\t */\n+\tblock.awb_config->cr_max = 511;\n+\tblock.awb_config->cr_min = 64;\n+\tblock.awb_config->cb_max = 511;\n+\tblock.awb_config->cb_min = 64;\n+\n+\t/* We use the full 15x15 zoning scheme */\n+\tblock.awb_config->nodes_used_horiz = 15;\n+\tblock.awb_config->nodes_used_vert = 15;\n+\n+\t/*\n+\t * We set the trimming boundaries equivalent to the main boundaries. In\n+\t * other words; no trimming.\n+\t */\n+\tblock.awb_config->cr_high = 511;\n+\tblock.awb_config->cr_low = 64;\n+\tblock.awb_config->cb_high = 511;\n+\tblock.awb_config->cb_low = 64;\n+\n+\treturn sizeof(struct mali_c55_params_awb_config);\n+}\n+\n+void Awb::prepare(IPAContext &context, const uint32_t frame,\n+\t\t IPAFrameContext &frameContext, mali_c55_params_buffer *params)\n+{\n+\tmali_c55_params_block block;\n+\tblock.data = ¶ms->data[params->total_size];\n+\n+\tparams->total_size += fillGainsParamBlock(block, context, frameContext);\n+\n+\tif (frame > 0)\n+\t\treturn;\n+\n+\tblock.data = ¶ms->data[params->total_size];\n+\tparams->total_size += fillConfigParamBlock(block);\n+}\n+\n+void Awb::process(IPAContext &context, const uint32_t frame,\n+\t\t IPAFrameContext &frameContext, const mali_c55_stats_buffer *stats,\n+\t\t [[maybe_unused]] ControlList &metadata)\n+{\n+\tconst struct mali_c55_awb_average_ratios *awb_ratios = stats->awb_ratios;\n+\n+\t/*\n+\t * The ISP produces average R:G and B:G ratios for zones. We take the\n+\t * average of all the zones with data and simply invert them to provide\n+\t * gain figures that we can apply to approximate a grey world.\n+\t */\n+\tunsigned int counted_zones = 0;\n+\tdouble rgSum = 0, bgSum = 0;\n+\n+\tfor (unsigned int i = 0; i < 225; i++) {\n+\t\tif (!awb_ratios[i].num_pixels)\n+\t\t\tcontinue;\n+\n+\t\t/*\n+\t\t * The statistics are in Q4.8 format, so we convert to double\n+\t\t * here.\n+\t\t */\n+\t\trgSum += ipa::fromUnsignedQFormat(awb_ratios[i].avg_rg_gr, 8);\n+\t\tbgSum += ipa::fromUnsignedQFormat(awb_ratios[i].avg_bg_br, 8);\n+\t\tcounted_zones++;\n+\t}\n+\n+\t/*\n+\t * Sometimes the first frame's statistics have no valid pixels, in which\n+\t * case we'll just assume a grey world until they say otherwise.\n+\t */\n+\tdouble rgAvg, bgAvg;\n+\tif (!counted_zones) {\n+\t\trgAvg = 1.0;\n+\t\tbgAvg = 1.0;\n+\t} else {\n+\t\trgAvg = rgSum / counted_zones;\n+\t\tbgAvg = bgSum / counted_zones;\n+\t}\n+\n+\t/*\n+\t * The statistics are generated _after_ white balancing is performed in\n+\t * the ISP. To get the true ratio we therefore have to adjust the stats\n+\t * figure by the gains that were applied when the statistics for this\n+\t * frame were generated.\n+\t */\n+\tdouble rRatio = rgAvg / frameContext.awb.rGain;\n+\tdouble bRatio = bgAvg / frameContext.awb.bGain;\n+\n+\t/*\n+\t * And then we can simply invert the ratio to find the gain we should\n+\t * apply.\n+\t */\n+\tdouble rGain = 1 / rRatio;\n+\tdouble bGain = 1 / bRatio;\n+\n+\t/*\n+\t * Running at full speed, this algorithm results in oscillations in the\n+\t * colour balance. To remove those we dampen the speed at which it makes\n+\t * changes in gain, unless we're in the startup phase in which case we\n+\t * want to fix the miscolouring as quickly as possible.\n+\t */\n+\tdouble speed = frame < kNumStartupFrames ? 1.0 : 0.2;\n+\trGain = speed * rGain + context.activeState.awb.rGain * (1.0 - speed);\n+\tbGain = speed * bGain + context.activeState.awb.bGain * (1.0 - speed);\n+\n+\tcontext.activeState.awb.rGain = rGain;\n+\tcontext.activeState.awb.bGain = bGain;\n+\n+\tmetadata.set(controls::ColourGains, {\n+\t\tstatic_cast<float>(frameContext.awb.rGain),\n+\t\tstatic_cast<float>(frameContext.awb.bGain),\n+\t});\n+\n+\tLOG(MaliC55Awb, Debug) << \"For frame number \" << frame << \": \"\n+\t\t<< \"Average R/G Ratio: \" << rgAvg\n+\t\t<< \", Average B/G Ratio: \" << bgAvg\n+\t\t<< \"\\nrGain applied to this frame: \" << frameContext.awb.rGain\n+\t\t<< \", bGain applied to this frame: \" << frameContext.awb.bGain\n+\t\t<< \"\\nrGain to apply: \" << context.activeState.awb.rGain\n+\t\t<< \", bGain to apply: \" << context.activeState.awb.bGain;\n+}\n+\n+REGISTER_IPA_ALGORITHM(Awb, \"Awb\")\n+\n+} /* namespace ipa::mali_c55::algorithms */\n+\n+} /* namespace libcamera */\ndiff --git a/src/ipa/mali-c55/algorithms/awb.h b/src/ipa/mali-c55/algorithms/awb.h\nnew file mode 100644\nindex 00000000..800c2e83\n--- /dev/null\n+++ b/src/ipa/mali-c55/algorithms/awb.h\n@@ -0,0 +1,40 @@\n+/* SPDX-License-Identifier: LGPL-2.1-or-later */\n+/*\n+ * Copyright (C) 2024, Ideas on Board Oy\n+ *\n+ * awb.h - Mali C55 grey world auto white balance algorithm\n+ */\n+\n+#include \"algorithm.h\"\n+#include \"ipa_context.h\"\n+\n+namespace libcamera {\n+\n+namespace ipa::mali_c55::algorithms {\n+\n+class Awb : public Algorithm\n+{\n+public:\n+\tAwb();\n+\t~Awb() = default;\n+\n+\tint configure(IPAContext &context,\n+\t\t const IPACameraSensorInfo &configInfo) override;\n+\tvoid prepare(IPAContext &context, const uint32_t frame,\n+\t\t IPAFrameContext &frameContext,\n+\t\t mali_c55_params_buffer *params) override;\n+\tvoid process(IPAContext &context, const uint32_t frame,\n+\t\t IPAFrameContext &frameContext,\n+\t\t const mali_c55_stats_buffer *stats,\n+\t\t ControlList &metadata) override;\n+\n+private:\n+\tsize_t fillGainsParamBlock(mali_c55_params_block block,\n+\t\t\t\t IPAContext &context,\n+\t\t\t\t IPAFrameContext &frameContext);\n+\tsize_t fillConfigParamBlock(mali_c55_params_block block);\n+};\n+\n+} /* namespace ipa::mali_c55::algorithms */\n+\n+} /* namespace libcamera */\ndiff --git a/src/ipa/mali-c55/algorithms/meson.build b/src/ipa/mali-c55/algorithms/meson.build\nindex 96808431..f11791aa 100644\n--- a/src/ipa/mali-c55/algorithms/meson.build\n+++ b/src/ipa/mali-c55/algorithms/meson.build\n@@ -2,5 +2,6 @@\n \n mali_c55_ipa_algorithms = files([\n 'agc.cpp',\n+ 'awb.cpp',\n 'blc.cpp',\n ])\ndiff --git a/src/ipa/mali-c55/ipa_context.h b/src/ipa/mali-c55/ipa_context.h\nindex 766acc42..fa5c9451 100644\n--- a/src/ipa/mali-c55/ipa_context.h\n+++ b/src/ipa/mali-c55/ipa_context.h\n@@ -51,6 +51,11 @@ struct IPAActiveState {\n \t\tuint32_t exposureMode;\n \t\tuint32_t temperatureK;\n \t} agc;\n+\n+\tstruct {\n+\t\tdouble rGain;\n+\t\tdouble bGain;\n+\t} awb;\n };\n \n struct IPAFrameContext : public FrameContext {\n@@ -59,6 +64,11 @@ struct IPAFrameContext : public FrameContext {\n \t\tdouble sensorGain;\n \t\tdouble ispGain;\n \t} agc;\n+\n+\tstruct {\n+\t\tdouble rGain;\n+\t\tdouble bGain;\n+\t} awb;\n };\n \n struct IPAContext {\n", "prefixes": [ "v3", "09/11" ] }