Patch Detail
Show a patch.
GET /api/patches/22888/?format=api
{ "id": 22888, "url": "https://patchwork.libcamera.org/api/patches/22888/?format=api", "web_url": "https://patchwork.libcamera.org/patch/22888/", "project": { "id": 1, "url": "https://patchwork.libcamera.org/api/projects/1/?format=api", "name": "libcamera", "link_name": "libcamera", "list_id": "libcamera_core", "list_email": "libcamera-devel@lists.libcamera.org", "web_url": "", "scm_url": "", "webscm_url": "" }, "msgid": "<20250227105733.187611-6-keke.li@amlogic.com>", "date": "2025-02-27T10:57:27", "name": "[v3,05/11] ipa: c3-isp: Add Agc algorithm", "commit_ref": null, "pull_url": null, "state": "new", "archived": false, "hash": "cc2c5f0959151fc77046f0615b460e65e5563bd7", "submitter": { "id": 217, "url": "https://patchwork.libcamera.org/api/people/217/?format=api", "name": "Keke Li", "email": "keke.li@amlogic.com" }, "delegate": null, "mbox": "https://patchwork.libcamera.org/patch/22888/mbox/", "series": [ { "id": 5027, "url": "https://patchwork.libcamera.org/api/series/5027/?format=api", "web_url": "https://patchwork.libcamera.org/project/libcamera/list/?series=5027", "date": "2025-02-27T10:57:22", "name": "Add Amlogic C3 ISP pipeline handler and IPA", "version": 3, "mbox": "https://patchwork.libcamera.org/series/5027/mbox/" } ], "comments": "https://patchwork.libcamera.org/api/patches/22888/comments/", "check": "pending", "checks": "https://patchwork.libcamera.org/api/patches/22888/checks/", "tags": {}, "headers": { "Return-Path": "<libcamera-devel-bounces@lists.libcamera.org>", "X-Original-To": "parsemail@patchwork.libcamera.org", "Delivered-To": "parsemail@patchwork.libcamera.org", "Received": [ "from lancelot.ideasonboard.com (lancelot.ideasonboard.com\n\t[92.243.16.209])\n\tby patchwork.libcamera.org (Postfix) with ESMTPS id DC144BF415\n\tfor <parsemail@patchwork.libcamera.org>;\n\tThu, 27 Feb 2025 10:58:52 +0000 (UTC)", "from lancelot.ideasonboard.com (localhost [IPv6:::1])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTP id A95B168753;\n\tThu, 27 Feb 2025 11:58:52 +0100 (CET)", "from mail-sh.amlogic.com (unknown [58.32.228.46])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id 8D17668749\n\tfor <libcamera-devel@lists.libcamera.org>;\n\tThu, 27 Feb 2025 11:58:50 +0100 (CET)", "from droid10.amlogic.com (10.18.11.213) by mail-sh.amlogic.com\n\t(10.18.11.5) with Microsoft SMTP Server id 15.1.2507.39;\n\tThu, 27 Feb 2025 18:58:48 +0800" ], "From": "Keke Li <keke.li@amlogic.com>", "To": "<libcamera-devel@lists.libcamera.org>", "CC": "<kieran.bingham@ideasonboard.com>, <laurent.pinchart@ideasonboard.com>, \n\t<dan.scally@ideasonboard.com>, Keke Li <keke.li@amlogic.com>", "Subject": "[PATCH v3 05/11] ipa: c3-isp: Add Agc algorithm", "Date": "Thu, 27 Feb 2025 18:57:27 +0800", "Message-ID": "<20250227105733.187611-6-keke.li@amlogic.com>", "X-Mailer": "git-send-email 2.29.0", "In-Reply-To": "<20250227105733.187611-1-keke.li@amlogic.com>", "References": "<20250227105733.187611-1-keke.li@amlogic.com>", "MIME-Version": "1.0", "Content-Transfer-Encoding": "8bit", "Content-Type": "text/plain", "X-Originating-IP": "[10.18.11.213]", "X-BeenThere": "libcamera-devel@lists.libcamera.org", "X-Mailman-Version": "2.1.29", "Precedence": "list", "List-Id": "<libcamera-devel.lists.libcamera.org>", "List-Unsubscribe": "<https://lists.libcamera.org/options/libcamera-devel>,\n\t<mailto:libcamera-devel-request@lists.libcamera.org?subject=unsubscribe>", "List-Archive": "<https://lists.libcamera.org/pipermail/libcamera-devel/>", "List-Post": "<mailto:libcamera-devel@lists.libcamera.org>", "List-Help": "<mailto:libcamera-devel-request@lists.libcamera.org?subject=help>", "List-Subscribe": "<https://lists.libcamera.org/listinfo/libcamera-devel>,\n\t<mailto:libcamera-devel-request@lists.libcamera.org?subject=subscribe>", "Errors-To": "libcamera-devel-bounces@lists.libcamera.org", "Sender": "\"libcamera-devel\" <libcamera-devel-bounces@lists.libcamera.org>" }, "content": "Add a new Agc algorithm for the C3 ISP. The tuning files for\nuncalibrated sensors is extended to enable the algorithm.\n\nSigned-off-by: Keke Li <keke.li@amlogic.com>\n---\n src/ipa/c3-isp/algorithms/agc.cpp | 310 ++++++++++++++++++++++++++\n src/ipa/c3-isp/algorithms/agc.h | 51 +++++\n src/ipa/c3-isp/algorithms/meson.build | 2 +-\n src/ipa/c3-isp/data/meson.build | 5 +-\n src/ipa/c3-isp/data/uncalibrated.yaml | 7 +\n 5 files changed, 371 insertions(+), 4 deletions(-)\n create mode 100644 src/ipa/c3-isp/algorithms/agc.cpp\n create mode 100644 src/ipa/c3-isp/algorithms/agc.h\n create mode 100644 src/ipa/c3-isp/data/uncalibrated.yaml", "diff": "diff --git a/src/ipa/c3-isp/algorithms/agc.cpp b/src/ipa/c3-isp/algorithms/agc.cpp\nnew file mode 100644\nindex 00000000..b9357cbe\n--- /dev/null\n+++ b/src/ipa/c3-isp/algorithms/agc.cpp\n@@ -0,0 +1,310 @@\n+/* SPDX-License-Identifier: LGPL-2.1-or-later */\n+/*\n+ * Copyright (C) 2024, Amlogic\n+ *\n+ * C3ISP AGC/AEC mean-based control algorithm\n+ */\n+\n+#include \"agc.h\"\n+\n+#include <cmath>\n+\n+#include <libcamera/base/log.h>\n+#include <libcamera/base/utils.h>\n+\n+#include <libcamera/control_ids.h>\n+#include <libcamera/property_ids.h>\n+\n+#include \"libipa/histogram.h\"\n+\n+/**\n+ * \\file agc.h\n+ */\n+\n+namespace libcamera {\n+\n+using namespace std::literals::chrono_literals;\n+\n+namespace ipa::c3isp::algorithms {\n+\n+/**\n+ * \\class Agc\n+ * \\brief A mean-based auto-exposure algorithm\n+ */\n+\n+LOG_DEFINE_CATEGORY(C3ISPAgc)\n+\n+Agc::Agc()\n+{\n+}\n+\n+/**\n+ * \\brief Initialise the AGC algorithm from tuning files\n+ * \\param[in] context The shared IPA context\n+ * \\param[in] tuningData The YamlObject containing Agc tuning data\n+ *\n+ * This function calls the base class' tuningData parsers to discover which\n+ * control values are supported.\n+ *\n+ * \\return 0 on success or errors from the base class\n+ */\n+int Agc::init(IPAContext &context, const YamlObject &tuningData)\n+{\n+\tint ret;\n+\n+\tret = parseTuningData(tuningData);\n+\tif (ret)\n+\t\treturn ret;\n+\n+\tcontext.ctrlMap[&controls::AeEnable] = ControlInfo(false, true);\n+\n+\tcontext.ctrlMap.merge(controls());\n+\n+\treturn 0;\n+}\n+\n+/**\n+ * \\brief Configure the AGC given a configInfo\n+ * \\param[in] context The shared IPA context\n+ * \\param[in] configInfo The IPA configuration data\n+ *\n+ * \\return 0\n+ */\n+int Agc::configure(IPAContext &context,\n+\t\t [[maybe_unused]] const IPACameraSensorInfo &configInfo)\n+{\n+\tconst IPASessionConfiguration &configuration = context.configuration;\n+\tIPAActiveState &activeState = context.activeState;\n+\n+\t/* Configure the default gain and exposure */\n+\tactiveState.agc.autoEnabled = true;\n+\tactiveState.agc.automatic.sensorGain = configuration.agc.minAnalogueGain;\n+\tactiveState.agc.automatic.exposure = configuration.agc.defaultExposure;\n+\tactiveState.agc.manual.sensorGain = configuration.agc.minAnalogueGain;\n+\tactiveState.agc.manual.exposure = configuration.agc.defaultExposure;\n+\tactiveState.agc.constraintMode = constraintModes().begin()->first;\n+\tactiveState.agc.exposureMode = exposureModeHelpers().begin()->first;\n+\n+\tsetLimits(configuration.agc.minShutterSpeed,\n+\t\t configuration.agc.maxShutterSpeed,\n+\t\t configuration.agc.minAnalogueGain,\n+\t\t configuration.agc.maxAnalogueGain);\n+\n+\tresetFrameCount();\n+\n+\treturn 0;\n+}\n+\n+/**\n+ * \\copydoc libcamera::ipa::Algorithm::queueRequest\n+ */\n+void Agc::queueRequest(IPAContext &context, const uint32_t frame,\n+\t\t [[maybe_unused]] IPAFrameContext &frameContext,\n+\t\t const ControlList &controls)\n+{\n+\tauto &agc = context.activeState.agc;\n+\n+\tconst auto &constraintMode = controls.get(controls::AeConstraintMode);\n+\tagc.constraintMode = constraintMode.value_or(agc.constraintMode);\n+\n+\tconst auto &exposureMode = controls.get(controls::AeExposureMode);\n+\tagc.exposureMode = exposureMode.value_or(agc.exposureMode);\n+\n+\tconst auto &agcEnable = controls.get(controls::AeEnable);\n+\tif (agcEnable && *agcEnable != agc.autoEnabled) {\n+\t\tagc.autoEnabled = *agcEnable;\n+\n+\t\tLOG(C3ISPAgc, Info)\n+\t\t\t<< (agc.autoEnabled ? \"Enabling\" : \"Disablin\")\n+\t\t\t<< \" AGC\";\n+\t}\n+\n+\tif (agc.autoEnabled)\n+\t\treturn;\n+\n+\tconst auto &exposure = controls.get(controls::ExposureTime);\n+\tif (exposure) {\n+\t\tagc.manual.exposure = *exposure * 1.0us /\n+\t\t\t\t context.configuration.sensor.lineDuration;\n+\n+\t\tLOG(C3ISPAgc, Debug)\n+\t\t\t<< \"Exposure set to \" << agc.manual.exposure\n+\t\t\t<< \" on request sequence \" << frame;\n+\t}\n+\n+\tconst auto &analogueGain = controls.get(controls::AnalogueGain);\n+\tif (analogueGain) {\n+\t\tagc.manual.sensorGain = *analogueGain;\n+\n+\t\tLOG(C3ISPAgc, Debug)\n+\t\t\t<< \"Analogue gain set to \" << agc.manual.sensorGain\n+\t\t\t<< \" on request sequence \" << frame;\n+\t}\n+}\n+\n+/**\n+ * \\copydoc libcamera::ipa::Algorithm::prepare\n+ */\n+void Agc::prepare(IPAContext &context, const uint32_t frame,\n+\t\t [[maybe_unused]] IPAFrameContext &frameContext,\n+\t\t C3ISPParams *params)\n+{\n+\tif (frame > 0)\n+\t\treturn;\n+\n+\tauto AeCfg = params->block<BlockType::AEConfig>();\n+\tAeCfg.setEnabled(C3_ISP_PARAMS_BLOCK_FL_ENABLE);\n+\n+\tAeCfg->tap_point = C3_ISP_AE_STATS_TAP_MLS;\n+\n+\t/* A 17x15 grid */\n+\tAeCfg->horiz_zones_num = 17;\n+\tAeCfg->vert_zones_num = 15;\n+\n+\tSpan<uint8_t> weights{ AeCfg->zone_weight, C3_ISP_AE_MAX_ZONES };\n+\tstd::fill(weights.begin(), weights.end(), 1);\n+\n+\tSize sensorSize = context.configuration.sensor.size;\n+\tuint8_t maxPointNum =\n+\t\tstd::max(AeCfg->horiz_zones_num, AeCfg->vert_zones_num) + 1;\n+\n+\tfor (unsigned int i = 0; i < maxPointNum; i++) {\n+\t\tuint16_t hidx = i * sensorSize.width / AeCfg->horiz_zones_num;\n+\n+\t\t/* Aligned with 2 */\n+\t\thidx = hidx / 2 * 2;\n+\t\tAeCfg->horiz_coord[i] =\n+\t\t\tstd::min(hidx, (uint16_t)sensorSize.width);\n+\n+\t\tuint16_t vidx = i * sensorSize.height / AeCfg->vert_zones_num;\n+\n+\t\t/* Aligned with 2 */\n+\t\tvidx = vidx / 2 * 2;\n+\t\tAeCfg->vert_coord[i] =\n+\t\t\tstd::min(vidx, (uint16_t)sensorSize.height);\n+\t}\n+}\n+\n+Histogram Agc::parseStatistics(const c3_isp_stats_info *stats)\n+{\n+\tconst struct c3_isp_ae_stats *info = &stats->ae;\n+\tstd::vector<uint8_t> means(C3_ISP_AE_MAX_ZONES, 0);\n+\n+\t/*\n+\t * Each zone has a 5-bin histogram and the total sum is normalized to\n+\t * 65535. For the convenience of calculation, we use the centre of the\n+\t * 5-bin as target area. So it can be assumed that:\n+\t * hist1 represents the number of brightness 16,\n+\t * hist2 represents the number of brightness 72,\n+\t * hist3 represents the number of brightness 128,\n+\t *\n+\t * Finally, the average brightness of a zone can be calculated.\n+\t */\n+\tfor (unsigned int i = 0; i < C3_ISP_AE_MAX_ZONES; i++) {\n+\t\tuint16_t hist2 = 65535 - info->stats[i].hist0 -\n+\t\t\t\t info->stats[i].hist1 -\n+\t\t\t\t info->stats[i].hist3 -\n+\t\t\t\t info->stats[i].hist4;\n+\n+\t\tuint32_t lumaSum = info->stats[i].hist1 * 16 +\n+\t\t\t\t hist2 * 72 +\n+\t\t\t\t info->stats[i].hist3 * 128;\n+\n+\t\tuint32_t histSum = info->stats[i].hist1 +\n+\t\t\t\t hist2 +\n+\t\t\t\t info->stats[i].hist3;\n+\n+\t\tmeans[i] = lumaSum / histSum;\n+\t}\n+\n+\tlumaMeans_ = means;\n+\n+\t/* This is a 1024-bin histogram */\n+\tuint32_t *hist = const_cast<uint32_t *>(info->hist);\n+\n+\treturn Histogram(Span<uint32_t>(hist, std::size(info->hist)));\n+}\n+\n+/**\n+ * \\brief Estimate the relative luminance of the frame with a given gain\n+ * \\param[in] gain The gain to apply in estimating luminance\n+ *\n+ * The estimation is based on the average value of the zone. Each\n+ * average value is multiplied by the gain, and then saturated to\n+ * to approximate the sensor behaviour at high brightness values.\n+ * The approximation is quite rough, as it doesn't take into account\n+ * non-linearities when approaching saturation.\n+ *\n+ * The values are normalized to the [0.0, 1.0] range, where 1.0 corresponds\n+ * to a theoretical perfect reflector of 100% reference white.\n+ *\n+ * More detailed information can be found in:\n+ * https://en.wikipedia.org/wiki/Relative_luminance\n+ *\n+ * \\return The relative luminance of the frame\n+ */\n+double Agc::estimateLuminance(double gain) const\n+{\n+\tdouble sum = 0.0;\n+\n+\tfor (unsigned int i = 0; i < lumaMeans_.size(); i++)\n+\t\tsum += std::min(lumaMeans_[i] * gain, 128.0);\n+\n+\treturn sum / lumaMeans_.size() / 128;\n+}\n+\n+/**\n+ * \\brief Process C3 ISP statistics, and run AGC operations\n+ * \\param[in] context The shared IPA context\n+ * \\param[in] frame The current frame sequence number\n+ * \\param[in] frameContext The current frame context\n+ * \\param[in] stats The C3 ISP statistics and ISP results\n+ * \\param[out] metadata Metadata for the frame, to be filled by the algorithm\n+ *\n+ * Identify the current image brightness, and use that to estimate the optimal\n+ * new exposure and gain for the scene.\n+ */\n+void Agc::process(IPAContext &context, [[maybe_unused]] const uint32_t frame,\n+\t\t IPAFrameContext &frameContext,\n+\t\t const c3_isp_stats_info *stats,\n+\t\t ControlList &metadata)\n+{\n+\tHistogram hist = parseStatistics(stats);\n+\n+\tutils::Duration shutterTime;\n+\tdouble aGain, dGain;\n+\n+\t/*\n+\t * The Agc algorithm needs to know the effective exposure value that was\n+\t * applied to the sensor when the statistics were collected.\n+\t */\n+\tutils::Duration exposureTime = context.configuration.sensor.lineDuration *\n+\t\t\t\t frameContext.agc.exposure;\n+\tdouble analogueGain = frameContext.agc.sensorGain;\n+\tutils::Duration effectiveExposureValue = exposureTime * analogueGain;\n+\n+\tstd::tie(shutterTime, aGain, dGain) =\n+\t\tcalculateNewEv(context.activeState.agc.constraintMode,\n+\t\t\t context.activeState.agc.exposureMode, hist,\n+\t\t\t effectiveExposureValue);\n+\n+\tLOG(C3ISPAgc, Debug)\n+\t\t<< \"Shutter time, analogue gain and digital gain are \"\n+\t\t<< shutterTime << \", \" << aGain << \" and \" << dGain;\n+\n+\tIPAActiveState &activeState = context.activeState;\n+\n+\tactiveState.agc.automatic.exposure = shutterTime / context.configuration.sensor.lineDuration;\n+\tactiveState.agc.automatic.sensorGain = aGain;\n+\n+\tmetadata.set(controls::AnalogueGain, frameContext.agc.sensorGain);\n+\tmetadata.set(controls::ExposureTime, exposureTime.get<std::micro>());\n+\n+\tlumaMeans_ = {};\n+}\n+\n+REGISTER_IPA_ALGORITHM(Agc, \"Agc\")\n+\n+} /* namespace ipa::c3isp::algorithms */\n+\n+} /* namespace libcamera */\ndiff --git a/src/ipa/c3-isp/algorithms/agc.h b/src/ipa/c3-isp/algorithms/agc.h\nnew file mode 100644\nindex 00000000..e7395903\n--- /dev/null\n+++ b/src/ipa/c3-isp/algorithms/agc.h\n@@ -0,0 +1,51 @@\n+/* SPDX-License-Identifier: LGPL-2.1-or-later */\n+/*\n+ * Copyright (C) 2024, Amlogic\n+ *\n+ * C3ISP AGC/AEC mean-based control algorithm\n+ */\n+\n+#pragma once\n+\n+#include <linux/c3-isp-config.h>\n+\n+#include <libcamera/base/span.h>\n+#include <libcamera/base/utils.h>\n+\n+#include <libcamera/geometry.h>\n+\n+#include \"libipa/agc_mean_luminance.h\"\n+#include \"algorithm.h\"\n+\n+namespace libcamera {\n+\n+namespace ipa::c3isp::algorithms {\n+\n+class Agc : public Algorithm, public AgcMeanLuminance\n+{\n+public:\n+\tAgc();\n+\t~Agc() = default;\n+\n+\tint init(IPAContext &context, const YamlObject &tuningData) override;\n+\tint configure(IPAContext &context, const IPACameraSensorInfo &configInfo) override;\n+\tvoid queueRequest(IPAContext &context, const uint32_t frame,\n+\t\t\t IPAFrameContext &frameContext,\n+\t\t\t const ControlList &controls) override;\n+\tvoid prepare(IPAContext &context, const uint32_t frame,\n+\t\t IPAFrameContext &frameContext,\n+\t\t C3ISPParams *params) override;\n+\tvoid process(IPAContext &context, const uint32_t frame,\n+\t\t IPAFrameContext &frameContext,\n+\t\t const c3_isp_stats_info *stats,\n+\t\t ControlList &metadata) override;\n+private:\n+\tHistogram parseStatistics(const c3_isp_stats_info *stats);\n+\tdouble estimateLuminance(double gain) const override;\n+\n+\tstd::vector<uint8_t> lumaMeans_;\n+};\n+\n+} /* namespace ipa::c3isp::algorithms */\n+\n+} /* namespace libcamera */\ndiff --git a/src/ipa/c3-isp/algorithms/meson.build b/src/ipa/c3-isp/algorithms/meson.build\nindex 1e00af33..3e44fe91 100644\n--- a/src/ipa/c3-isp/algorithms/meson.build\n+++ b/src/ipa/c3-isp/algorithms/meson.build\n@@ -1,5 +1,5 @@\n # SPDX-License-Identifier: CC0-1.0\n \n c3isp_ipa_algorithms = files([\n-\n+ 'agc.cpp',\n ])\ndiff --git a/src/ipa/c3-isp/data/meson.build b/src/ipa/c3-isp/data/meson.build\nindex 75e44bac..1e98cbcf 100644\n--- a/src/ipa/c3-isp/data/meson.build\n+++ b/src/ipa/c3-isp/data/meson.build\n@@ -1,9 +1,8 @@\n # SPDX-License-Identifier: CC0-1.0\n \n conf_files = files([\n-\n+ 'uncalibrated.yaml'\n ])\n \n install_data(conf_files,\n- install_dir : ipa_data_dir / 'c3isp',\n- install_tag : 'runtime')\n+ install_dir : ipa_data_dir / 'c3isp')\ndiff --git a/src/ipa/c3-isp/data/uncalibrated.yaml b/src/ipa/c3-isp/data/uncalibrated.yaml\nnew file mode 100644\nindex 00000000..6dcc0295\n--- /dev/null\n+++ b/src/ipa/c3-isp/data/uncalibrated.yaml\n@@ -0,0 +1,7 @@\n+# SPDX-License-Identifier: CC0-1.0\n+%YAML 1.1\n+---\n+version: 1\n+algorithms:\n+ - Agc:\n+...\n", "prefixes": [ "v3", "05/11" ] }