Patch Detail
Show a patch.
GET /api/patches/11961/?format=api
{ "id": 11961, "url": "https://patchwork.libcamera.org/api/patches/11961/?format=api", "web_url": "https://patchwork.libcamera.org/patch/11961/", "project": { "id": 1, "url": "https://patchwork.libcamera.org/api/projects/1/?format=api", "name": "libcamera", "link_name": "libcamera", "list_id": "libcamera_core", "list_email": "libcamera-devel@lists.libcamera.org", "web_url": "", "scm_url": "", "webscm_url": "" }, "msgid": "<20210416074909.24218-5-jeanmichel.hautbois@ideasonboard.com>", "date": "2021-04-16T07:49:09", "name": "[libcamera-devel,v5,4/4] ipa: ipu3: Add support for IPU3 AEC/AGC algorithm", "commit_ref": null, "pull_url": null, "state": "accepted", "archived": false, "hash": "9fa62b7ddbbc353ba06eb2267400e860c1a4548b", "submitter": { "id": 75, "url": "https://patchwork.libcamera.org/api/people/75/?format=api", "name": "Jean-Michel Hautbois", "email": "jeanmichel.hautbois@ideasonboard.com" }, "delegate": null, "mbox": "https://patchwork.libcamera.org/patch/11961/mbox/", "series": [ { "id": 1938, "url": "https://patchwork.libcamera.org/api/series/1938/?format=api", "web_url": "https://patchwork.libcamera.org/project/libcamera/list/?series=1938", "date": "2021-04-16T07:49:05", "name": "Implement IPA algorithms and demo with IPU3", "version": 5, "mbox": "https://patchwork.libcamera.org/series/1938/mbox/" } ], "comments": "https://patchwork.libcamera.org/api/patches/11961/comments/", "check": "pending", "checks": "https://patchwork.libcamera.org/api/patches/11961/checks/", "tags": {}, "headers": { "Return-Path": "<libcamera-devel-bounces@lists.libcamera.org>", "X-Original-To": "parsemail@patchwork.libcamera.org", "Delivered-To": "parsemail@patchwork.libcamera.org", "Received": [ "from lancelot.ideasonboard.com (lancelot.ideasonboard.com\n\t[92.243.16.209])\n\tby patchwork.libcamera.org (Postfix) with ESMTPS id D91ACBD235\n\tfor <parsemail@patchwork.libcamera.org>;\n\tFri, 16 Apr 2021 07:49:17 +0000 (UTC)", "from lancelot.ideasonboard.com (localhost [IPv6:::1])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTP id 4809E68826;\n\tFri, 16 Apr 2021 09:49:17 +0200 (CEST)", "from perceval.ideasonboard.com (perceval.ideasonboard.com\n\t[IPv6:2001:4b98:dc2:55:216:3eff:fef7:d647])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id 04D3E68819\n\tfor <libcamera-devel@lists.libcamera.org>;\n\tFri, 16 Apr 2021 09:49:15 +0200 (CEST)", "from localhost.localdomain (unknown\n\t[IPv6:2a01:e0a:169:7140:5b63:445c:7960:347a])\n\tby perceval.ideasonboard.com (Postfix) with ESMTPSA id A5FEA8A4;\n\tFri, 16 Apr 2021 09:49:14 +0200 (CEST)" ], "Authentication-Results": "lancelot.ideasonboard.com;\n\tdkim=fail reason=\"signature verification failed\" (1024-bit key;\n\tunprotected) header.d=ideasonboard.com header.i=@ideasonboard.com\n\theader.b=\"sCz4FC6n\"; dkim-atps=neutral", "DKIM-Signature": "v=1; a=rsa-sha256; c=relaxed/simple; d=ideasonboard.com;\n\ts=mail; t=1618559354;\n\tbh=tN07RqwqjZR+1gBYFioom902LBDXcplNMq+SrhrWRng=;\n\th=From:To:Cc:Subject:Date:In-Reply-To:References:From;\n\tb=sCz4FC6nE1f1aZgti3DvXdP1xYRrZIjXhuXuJVTF1q/My6K1bp49Qah0pg338mlp7\n\ta7VCxK0/pmky9ydskns0J//UG1D/33VjmBj63GdVCLs3HhKpDPBh/th5tMQD1N1uZZ\n\t3VCZ7mG5u8/U2DguxaCobRmL7+2qbjzkmnQHLi+I=", "From": "Jean-Michel Hautbois <jeanmichel.hautbois@ideasonboard.com>", "To": "libcamera-devel@lists.libcamera.org", "Date": "Fri, 16 Apr 2021 09:49:09 +0200", "Message-Id": "<20210416074909.24218-5-jeanmichel.hautbois@ideasonboard.com>", "X-Mailer": "git-send-email 2.27.0", "In-Reply-To": "<20210416074909.24218-1-jeanmichel.hautbois@ideasonboard.com>", "References": "<20210416074909.24218-1-jeanmichel.hautbois@ideasonboard.com>", "MIME-Version": "1.0", "Subject": "[libcamera-devel] [PATCH v5 4/4] ipa: ipu3: Add support for IPU3\n\tAEC/AGC algorithm", "X-BeenThere": "libcamera-devel@lists.libcamera.org", "X-Mailman-Version": "2.1.29", "Precedence": "list", "List-Id": "<libcamera-devel.lists.libcamera.org>", "List-Unsubscribe": "<https://lists.libcamera.org/options/libcamera-devel>,\n\t<mailto:libcamera-devel-request@lists.libcamera.org?subject=unsubscribe>", "List-Archive": "<https://lists.libcamera.org/pipermail/libcamera-devel/>", "List-Post": "<mailto:libcamera-devel@lists.libcamera.org>", "List-Help": "<mailto:libcamera-devel-request@lists.libcamera.org?subject=help>", "List-Subscribe": "<https://lists.libcamera.org/listinfo/libcamera-devel>,\n\t<mailto:libcamera-devel-request@lists.libcamera.org?subject=subscribe>", "Content-Type": "text/plain; charset=\"us-ascii\"", "Content-Transfer-Encoding": "7bit", "Errors-To": "libcamera-devel-bounces@lists.libcamera.org", "Sender": "\"libcamera-devel\" <libcamera-devel-bounces@lists.libcamera.org>" }, "content": "Implement basic auto-exposure (AE) and auto-gain (AG) correction functions.\nThe functions computeTargetExposure() and computeGain() are adapted from\nthe Raspberry Pi AGC implementation to suit the IPU3 structures, and\nfiltering is added to reduce visible stepsize when there are large\nexposure changes.\n\nSigned-off-by: Jean-Michel Hautbois <jeanmichel.hautbois@ideasonboard.com>\n---\n src/ipa/ipu3/ipu3.cpp | 16 ++-\n src/ipa/ipu3/ipu3_agc.cpp | 206 ++++++++++++++++++++++++++++++++++++++\n src/ipa/ipu3/ipu3_agc.h | 62 ++++++++++++\n src/ipa/ipu3/meson.build | 1 +\n 4 files changed, 282 insertions(+), 3 deletions(-)\n create mode 100644 src/ipa/ipu3/ipu3_agc.cpp\n create mode 100644 src/ipa/ipu3/ipu3_agc.h", "diff": "diff --git a/src/ipa/ipu3/ipu3.cpp b/src/ipa/ipu3/ipu3.cpp\nindex ab052a8a..48dc28c1 100644\n--- a/src/ipa/ipu3/ipu3.cpp\n+++ b/src/ipa/ipu3/ipu3.cpp\n@@ -21,10 +21,11 @@\n #include \"libcamera/internal/buffer.h\"\n #include \"libcamera/internal/log.h\"\n \n+#include \"ipu3_agc.h\"\n #include \"ipu3_awb.h\"\n \n static constexpr uint32_t kMaxCellWidthPerSet = 160;\n-static constexpr uint32_t kMaxCellHeightPerSet = 80;\n+static constexpr uint32_t kMaxCellHeightPerSet = 56;\n \n namespace libcamera {\n \n@@ -70,6 +71,8 @@ private:\n \n \t/* Interface to the AWB algorithm */\n \tstd::unique_ptr<ipa::IPU3Awb> awbAlgo_;\n+\t/* Interface to the AEC/AGC algorithm */\n+\tstd::unique_ptr<ipa::IPU3Agc> agcAlgo_;\n \n \t/* Local parameter storage */\n \tstruct ipu3_uapi_params params_;\n@@ -168,6 +171,9 @@ void IPAIPU3::configure(const std::map<uint32_t, ControlInfoMap> &entityControls\n \n \tawbAlgo_ = std::make_unique<ipa::IPU3Awb>();\n \tawbAlgo_->initialise(params_, bdsOutputSize, bdsGrid_);\n+\n+\tagcAlgo_ = std::make_unique<ipa::IPU3Agc>();\n+\tagcAlgo_->initialise(bdsGrid_);\n }\n \n void IPAIPU3::mapBuffers(const std::vector<IPABuffer> &buffers)\n@@ -239,8 +245,8 @@ void IPAIPU3::processControls([[maybe_unused]] unsigned int frame,\n \n void IPAIPU3::fillParams(unsigned int frame, ipu3_uapi_params *params)\n {\n-\t/* Pass a default gamma of 1.0 (default linear correction) */\n-\tawbAlgo_->updateWbParameters(params_, 1.0);\n+\tif (agcAlgo_->updateControls())\n+\t\tawbAlgo_->updateWbParameters(params_, agcAlgo_->gamma());\n \n \t*params = params_;\n \n@@ -255,8 +261,12 @@ void IPAIPU3::parseStatistics(unsigned int frame,\n {\n \tControlList ctrls(controls::controls);\n \n+\tagcAlgo_->process(stats, exposure_, gain_);\n \tawbAlgo_->calculateWBGains(stats);\n \n+\tif (agcAlgo_->updateControls())\n+\t\tsetControls(frame);\n+\n \tipa::ipu3::IPU3Action op;\n \top.op = ipa::ipu3::ActionMetadataReady;\n \top.controls = ctrls;\ndiff --git a/src/ipa/ipu3/ipu3_agc.cpp b/src/ipa/ipu3/ipu3_agc.cpp\nnew file mode 100644\nindex 00000000..04ab55a1\n--- /dev/null\n+++ b/src/ipa/ipu3/ipu3_agc.cpp\n@@ -0,0 +1,206 @@\n+/* SPDX-License-Identifier: LGPL-2.1-or-later */\n+/*\n+ * Copyright (C) 2021, Ideas On Board\n+ *\n+ * ipu3_agc.cpp - AGC/AEC control algorithm\n+ */\n+\n+#include \"ipu3_agc.h\"\n+\n+#include <algorithm>\n+#include <cmath>\n+#include <numeric>\n+\n+#include \"libcamera/internal/log.h\"\n+\n+#include \"libipa/histogram.h\"\n+\n+namespace libcamera {\n+\n+namespace ipa {\n+\n+LOG_DEFINE_CATEGORY(IPU3Agc)\n+\n+/* Number of frames to wait before calculating stats on minimum exposure */\n+static constexpr uint32_t kInitialFrameMinAECount = 4;\n+/* Number of frames to wait between new gain/exposure estimations */\n+static constexpr uint32_t kFrameSkipCount = 6;\n+\n+/* Maximum ISO value for analogue gain */\n+static constexpr uint32_t kMinISO = 100;\n+static constexpr uint32_t kMaxISO = 1500;\n+\n+/* Maximum analogue gain value\n+ * \\todo grab it from a camera helper */\n+static constexpr uint32_t kMinGain = kMinISO / 100;\n+static constexpr uint32_t kMaxGain = kMaxISO / 100;\n+\n+/* \\todo use calculated value based on sensor */\n+static constexpr uint32_t kMinExposure = 1;\n+static constexpr uint32_t kMaxExposure = 1976;\n+\n+/* \\todo those should be get from pipeline handler ! */\n+/* line duration in microseconds */\n+static constexpr double kLineDuration = 16.8;\n+static constexpr double kMaxExposureTime = kMaxExposure * kLineDuration;\n+\n+/* Histogram constants */\n+static constexpr uint32_t knumHistogramBins = 256;\n+static constexpr double kEvGainTarget = 0.5;\n+\n+IPU3Agc::IPU3Agc()\n+\t: frameCount_(0), lastFrame_(0), converged_(false),\n+\t updateControls_(false), iqMean_(0.0), gamma_(1.0),\n+\t prevExposure_(0.0), prevExposureNoDg_(0.0),\n+\t currentExposure_(0.0), currentExposureNoDg_(0.0)\n+{\n+}\n+\n+void IPU3Agc::initialise(struct ipu3_uapi_grid_config &bdsGrid)\n+{\n+\taeGrid_ = bdsGrid;\n+}\n+\n+void IPU3Agc::processBrightness(const ipu3_uapi_stats_3a *stats)\n+{\n+\tconst struct ipu3_uapi_grid_config statsAeGrid = stats->stats_4a_config.awb_config.grid;\n+\tRectangle aeRegion = { statsAeGrid.x_start,\n+\t\t\t statsAeGrid.y_start,\n+\t\t\t static_cast<unsigned int>(statsAeGrid.x_end - statsAeGrid.x_start) + 1,\n+\t\t\t static_cast<unsigned int>(statsAeGrid.y_end - statsAeGrid.y_start) + 1 };\n+\tPoint topleft = aeRegion.topLeft();\n+\tint topleftX = topleft.x >> aeGrid_.block_width_log2;\n+\tint topleftY = topleft.y >> aeGrid_.block_height_log2;\n+\n+\tuint32_t startY = topleftY * aeGrid_.width << aeGrid_.block_width_log2;\n+\tuint32_t startX = topleftX << aeGrid_.block_width_log2;\n+\tuint32_t endX = (startX + (aeRegion.size().width >> aeGrid_.block_width_log2)) << aeGrid_.block_width_log2;\n+\tuint32_t i, j;\n+\tuint32_t count = 0;\n+\n+\tuint32_t hist[knumHistogramBins] = { 0 };\n+\tfor (j = topleftY;\n+\t j < topleftY + (aeRegion.size().height >> aeGrid_.block_height_log2);\n+\t j++) {\n+\t\tfor (i = startX + startY; i < endX + startY; i += 8) {\n+\t\t\t/**\n+\t\t\t * The grid width (and maybe height) is not reliable.\n+\t\t\t * We observed a bit shift which makes the value 160 to be 32 in the stats grid.\n+\t\t\t * Use the one passed at init time.\n+\t\t\t */\n+\t\t\tif (stats->awb_raw_buffer.meta_data[i + 4 + j * aeGrid_.width] == 0) {\n+\t\t\t\tuint8_t Gr = stats->awb_raw_buffer.meta_data[i + j * aeGrid_.width];\n+\t\t\t\tuint8_t Gb = stats->awb_raw_buffer.meta_data[i + 3 + j * aeGrid_.width];\n+\t\t\t\thist[(Gr + Gb) / 2]++;\n+\t\t\t\tcount++;\n+\t\t\t}\n+\t\t}\n+\t}\n+\n+\t/* Limit the gamma effect for now */\n+\tgamma_ = 1.1;\n+\n+\t/* Estimate the quantile mean of the top 2% of the histogram */\n+\tiqMean_ = Histogram(Span<uint32_t>(hist)).interQuantileMean(0.98, 1.0);\n+}\n+\n+void IPU3Agc::filterExposure(bool desaturate)\n+{\n+\tdouble speed = 0.2;\n+\tif (prevExposure_ == 0.0) {\n+\t\t/* DG stands for digital gain.*/\n+\t\tprevExposure_ = currentExposure_;\n+\t\tprevExposureNoDg_ = currentExposureNoDg_;\n+\t} else {\n+\t\t/**\n+\t\t * If we are close to the desired result, go faster to avoid making\n+\t\t * multiple micro-adjustments.\n+\t\t * \\ todo: Make this customisable?\n+\t\t */\n+\t\tif (prevExposure_ < 1.2 * currentExposure_ &&\n+\t\t prevExposure_ > 0.8 * currentExposure_)\n+\t\t\tspeed = sqrt(speed);\n+\t\tprevExposure_ = speed * currentExposure_ +\n+\t\t\t\tprevExposure_ * (1.0 - speed);\n+\t\t/**\n+\t\t * When desaturing, take a big jump down in exposure_no_dg,\n+\t\t * which we'll hide with digital gain.\n+\t\t */\n+\t\tif (desaturate)\n+\t\t\tprevExposureNoDg_ =\n+\t\t\t\tcurrentExposureNoDg_;\n+\t\telse\n+\t\t\tprevExposureNoDg_ =\n+\t\t\t\tspeed * currentExposureNoDg_ +\n+\t\t\t\tprevExposureNoDg_ * (1.0 - speed);\n+\t}\n+\t/**\n+\t * We can't let the no_dg exposure deviate too far below the\n+\t * total exposure, as there might not be enough digital gain available\n+\t * in the ISP to hide it (which will cause nasty oscillation).\n+\t */\n+\tdouble fastReduceThreshold = 0.4;\n+\tif (prevExposureNoDg_ <\n+\t prevExposure_ * fastReduceThreshold)\n+\t\tprevExposureNoDg_ = prevExposure_ * fastReduceThreshold;\n+\tLOG(IPU3Agc, Debug) << \"After filtering, total_exposure \" << prevExposure_;\n+}\n+\n+void IPU3Agc::lockExposureGain(uint32_t &exposure, uint32_t &gain)\n+{\n+\tupdateControls_ = false;\n+\n+\t/* Algorithm initialization should wait for first valid frames */\n+\t/* \\todo - have a number of frames given by DelayedControls ?\n+\t * - implement a function for IIR */\n+\tif ((frameCount_ < kInitialFrameMinAECount) || (frameCount_ - lastFrame_ < kFrameSkipCount))\n+\t\treturn;\n+\n+\t/* Are we correctly exposed ? */\n+\tif (std::abs(iqMean_ - kEvGainTarget * knumHistogramBins) <= 1) {\n+\t\tLOG(IPU3Agc, Debug) << \"!!! Good exposure with iqMean = \" << iqMean_;\n+\t\tconverged_ = true;\n+\t} else {\n+\t\tdouble newGain = kEvGainTarget * knumHistogramBins / iqMean_;\n+\n+\t\t/* extracted from Rpi::Agc::computeTargetExposure */\n+\t\tdouble currentShutter = exposure * kLineDuration;\n+\t\tcurrentExposureNoDg_ = currentShutter * gain;\n+\t\tLOG(IPU3Agc, Debug) << \"Actual total exposure \" << currentExposureNoDg_\n+\t\t\t\t << \" Shutter speed \" << currentShutter\n+\t\t\t\t << \" Gain \" << gain;\n+\t\tcurrentExposure_ = currentExposureNoDg_ * newGain;\n+\t\tdouble maxTotalExposure = kMaxExposureTime * kMaxGain;\n+\t\tcurrentExposure_ = std::min(currentExposure_, maxTotalExposure);\n+\t\tLOG(IPU3Agc, Debug) << \"Target total exposure \" << currentExposure_;\n+\n+\t\t/* \\todo: estimate if we need to desaturate */\n+\t\tfilterExposure(false);\n+\n+\t\tdouble newExposure = 0.0;\n+\t\tif (currentShutter < kMaxExposureTime) {\n+\t\t\texposure = std::clamp(static_cast<uint32_t>(exposure * currentExposure_ / currentExposureNoDg_), kMinExposure, kMaxExposure);\n+\t\t\tnewExposure = currentExposure_ / exposure;\n+\t\t\tgain = std::clamp(static_cast<uint32_t>(gain * currentExposure_ / newExposure), kMinGain, kMaxGain);\n+\t\t\tupdateControls_ = true;\n+\t\t} else if (currentShutter >= kMaxExposureTime) {\n+\t\t\tgain = std::clamp(static_cast<uint32_t>(gain * currentExposure_ / currentExposureNoDg_), kMinGain, kMaxGain);\n+\t\t\tnewExposure = currentExposure_ / gain;\n+\t\t\texposure = std::clamp(static_cast<uint32_t>(exposure * currentExposure_ / newExposure), kMinExposure, kMaxExposure);\n+\t\t\tupdateControls_ = true;\n+\t\t}\n+\t\tLOG(IPU3Agc, Debug) << \"Adjust exposure \" << exposure * kLineDuration << \" and gain \" << gain;\n+\t}\n+\tlastFrame_ = frameCount_;\n+}\n+\n+void IPU3Agc::process(const ipu3_uapi_stats_3a *stats, uint32_t &exposure, uint32_t &gain)\n+{\n+\tprocessBrightness(stats);\n+\tlockExposureGain(exposure, gain);\n+\tframeCount_++;\n+}\n+\n+} /* namespace ipa */\n+\n+} /* namespace libcamera */\ndiff --git a/src/ipa/ipu3/ipu3_agc.h b/src/ipa/ipu3/ipu3_agc.h\nnew file mode 100644\nindex 00000000..f59bc420\n--- /dev/null\n+++ b/src/ipa/ipu3/ipu3_agc.h\n@@ -0,0 +1,62 @@\n+/* SPDX-License-Identifier: LGPL-2.1-or-later */\n+/*\n+ * Copyright (C) 2021, Ideas On Board\n+ *\n+ * ipu3_agc.h - IPU3 AGC/AEC control algorithm\n+ */\n+#ifndef __LIBCAMERA_IPU3_AGC_H__\n+#define __LIBCAMERA_IPU3_AGC_H__\n+\n+#include <array>\n+#include <unordered_map>\n+\n+#include <linux/intel-ipu3.h>\n+\n+#include <libcamera/geometry.h>\n+\n+#include \"libipa/algorithm.h\"\n+\n+namespace libcamera {\n+\n+namespace ipa {\n+\n+class IPU3Agc : public Algorithm\n+{\n+public:\n+\tIPU3Agc();\n+\t~IPU3Agc() = default;\n+\n+\tvoid initialise(struct ipu3_uapi_grid_config &bdsGrid);\n+\tvoid process(const ipu3_uapi_stats_3a *stats, uint32_t &exposure, uint32_t &gain);\n+\tbool converged() { return converged_; }\n+\tbool updateControls() { return updateControls_; }\n+\t/* \\todo Use a metadata exchange between IPAs */\n+\tdouble gamma() { return gamma_; }\n+\n+private:\n+\tvoid processBrightness(const ipu3_uapi_stats_3a *stats);\n+\tvoid filterExposure(bool desaturate);\n+\tvoid lockExposureGain(uint32_t &exposure, uint32_t &gain);\n+\n+\tstruct ipu3_uapi_grid_config aeGrid_;\n+\n+\tuint64_t frameCount_;\n+\tuint64_t lastFrame_;\n+\n+\tbool converged_;\n+\tbool updateControls_;\n+\n+\tdouble iqMean_;\n+\tdouble gamma_;\n+\n+\tdouble prevExposure_;\n+\tdouble prevExposureNoDg_;\n+\tdouble currentExposure_;\n+\tdouble currentExposureNoDg_;\n+};\n+\n+} /* namespace ipa */\n+\n+} /* namespace libcamera */\n+\n+#endif /* __LIBCAMERA_IPU3_AGC_H__ */\ndiff --git a/src/ipa/ipu3/meson.build b/src/ipa/ipu3/meson.build\nindex 1040698e..0d843846 100644\n--- a/src/ipa/ipu3/meson.build\n+++ b/src/ipa/ipu3/meson.build\n@@ -4,6 +4,7 @@ ipa_name = 'ipa_ipu3'\n \n ipu3_ipa_sources = files([\n 'ipu3.cpp',\n+ 'ipu3_agc.cpp',\n 'ipu3_awb.cpp',\n ])\n \n", "prefixes": [ "libcamera-devel", "v5", "4/4" ] }