Patch Detail
Show a patch.
GET /api/1.1/patches/11369/?format=api
{ "id": 11369, "url": "https://patchwork.libcamera.org/api/1.1/patches/11369/?format=api", "web_url": "https://patchwork.libcamera.org/patch/11369/", "project": { "id": 1, "url": "https://patchwork.libcamera.org/api/1.1/projects/1/?format=api", "name": "libcamera", "link_name": "libcamera", "list_id": "libcamera_core", "list_email": "libcamera-devel@lists.libcamera.org", "web_url": "", "scm_url": "", "webscm_url": "" }, "msgid": "<20210223164041.49932-4-jeanmichel.hautbois@ideasonboard.com>", "date": "2021-02-23T16:40:41", "name": "[libcamera-devel,RFC,v2,3/3] WIP: ipa: ipu3: Add support for IPU3 AEC/AGC algorithm", "commit_ref": null, "pull_url": null, "state": "superseded", "archived": false, "hash": "7d2cecc409e5097f2e9a9601d41243a76764edd8", "submitter": { "id": 75, "url": "https://patchwork.libcamera.org/api/1.1/people/75/?format=api", "name": "Jean-Michel Hautbois", "email": "jeanmichel.hautbois@ideasonboard.com" }, "delegate": null, "mbox": "https://patchwork.libcamera.org/patch/11369/mbox/", "series": [ { "id": 1722, "url": "https://patchwork.libcamera.org/api/1.1/series/1722/?format=api", "web_url": "https://patchwork.libcamera.org/project/libcamera/list/?series=1722", "date": "2021-02-23T16:40:38", "name": "Implement IPA algorithms and demo with IPU3", "version": 2, "mbox": "https://patchwork.libcamera.org/series/1722/mbox/" } ], "comments": "https://patchwork.libcamera.org/api/patches/11369/comments/", "check": "pending", "checks": "https://patchwork.libcamera.org/api/patches/11369/checks/", "tags": {}, "headers": { "Return-Path": "<libcamera-devel-bounces@lists.libcamera.org>", "X-Original-To": "parsemail@patchwork.libcamera.org", "Delivered-To": "parsemail@patchwork.libcamera.org", "Received": [ "from lancelot.ideasonboard.com (lancelot.ideasonboard.com\n\t[92.243.16.209])\n\tby patchwork.libcamera.org (Postfix) with ESMTPS id 65734BD1F1\n\tfor <parsemail@patchwork.libcamera.org>;\n\tTue, 23 Feb 2021 16:40:49 +0000 (UTC)", "from lancelot.ideasonboard.com (localhost [IPv6:::1])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTP id 715F768A35;\n\tTue, 23 Feb 2021 17:40:48 +0100 (CET)", "from perceval.ideasonboard.com (perceval.ideasonboard.com\n\t[IPv6:2001:4b98:dc2:55:216:3eff:fef7:d647])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id 2490E68A31\n\tfor <libcamera-devel@lists.libcamera.org>;\n\tTue, 23 Feb 2021 17:40:44 +0100 (CET)", "from localhost.localdomain (unknown\n\t[IPv6:2a01:e0a:169:7140:d7c9:eb3:c460:c24a])\n\tby perceval.ideasonboard.com (Postfix) with ESMTPSA id C6F8B9E2;\n\tTue, 23 Feb 2021 17:40:43 +0100 (CET)" ], "Authentication-Results": "lancelot.ideasonboard.com;\n\tdkim=fail reason=\"signature verification failed\" (1024-bit key;\n\tunprotected) header.d=ideasonboard.com header.i=@ideasonboard.com\n\theader.b=\"ORzSSGu/\"; dkim-atps=neutral", "DKIM-Signature": "v=1; a=rsa-sha256; c=relaxed/simple; d=ideasonboard.com;\n\ts=mail; t=1614098443;\n\tbh=2KXdTw7glBCyYisb7cqKMHtmm379qQ3rGt3x/iJ+ORY=;\n\th=From:To:Cc:Subject:Date:In-Reply-To:References:From;\n\tb=ORzSSGu//UST0P04CwlyHc2ti/ORM0zxzIqSIAcwxqzyZKZX9j1CXFvg9iQcYrSaR\n\tmE0x3JdqsZ+diPMYjVwyhJ/lsyqWwPDIDR0Orxvb975j4fXlrPd37nbfl0IVW1eoxZ\n\t4BYGotz4b9aJWE6QAbt2OLtIMYWuyYxAiMxT2BSw=", "From": "Jean-Michel Hautbois <jeanmichel.hautbois@ideasonboard.com>", "To": "libcamera-devel@lists.libcamera.org", "Date": "Tue, 23 Feb 2021 17:40:41 +0100", "Message-Id": "<20210223164041.49932-4-jeanmichel.hautbois@ideasonboard.com>", "X-Mailer": "git-send-email 2.27.0", "In-Reply-To": "<20210223164041.49932-1-jeanmichel.hautbois@ideasonboard.com>", "References": "<20210223164041.49932-1-jeanmichel.hautbois@ideasonboard.com>", "MIME-Version": "1.0", "Subject": "[libcamera-devel] [RFC PATCH v2 3/3] WIP: ipa: ipu3: Add support\n\tfor IPU3 AEC/AGC algorithm", "X-BeenThere": "libcamera-devel@lists.libcamera.org", "X-Mailman-Version": "2.1.29", "Precedence": "list", "List-Id": "<libcamera-devel.lists.libcamera.org>", "List-Unsubscribe": "<https://lists.libcamera.org/options/libcamera-devel>,\n\t<mailto:libcamera-devel-request@lists.libcamera.org?subject=unsubscribe>", "List-Archive": "<https://lists.libcamera.org/pipermail/libcamera-devel/>", "List-Post": "<mailto:libcamera-devel@lists.libcamera.org>", "List-Help": "<mailto:libcamera-devel-request@lists.libcamera.org?subject=help>", "List-Subscribe": "<https://lists.libcamera.org/listinfo/libcamera-devel>,\n\t<mailto:libcamera-devel-request@lists.libcamera.org?subject=subscribe>", "Content-Type": "text/plain; charset=\"us-ascii\"", "Content-Transfer-Encoding": "7bit", "Errors-To": "libcamera-devel-bounces@lists.libcamera.org", "Sender": "\"libcamera-devel\" <libcamera-devel-bounces@lists.libcamera.org>" }, "content": "Inherit from the Algorithm class to implement basic auto-exposure and\nauto-gain functions.\n\nWhile exposure is not locked, AWB is not calculated and corrected.\nImplement a basic \"skewness-based\" for demonstration purpose.\n\nSigned-off-by: Jean-Michel Hautbois <jeanmichel.hautbois@ideasonboard.com>\n---\n src/ipa/ipu3/ipu3.cpp | 10 ++-\n src/ipa/ipu3/ipu3_agc.cpp | 171 ++++++++++++++++++++++++++++++++++++++\n src/ipa/ipu3/ipu3_agc.h | 62 ++++++++++++++\n src/ipa/ipu3/meson.build | 1 +\n 4 files changed, 243 insertions(+), 1 deletion(-)\n create mode 100644 src/ipa/ipu3/ipu3_agc.cpp\n create mode 100644 src/ipa/ipu3/ipu3_agc.h", "diff": "diff --git a/src/ipa/ipu3/ipu3.cpp b/src/ipa/ipu3/ipu3.cpp\nindex 6fae5160..cabd0c71 100644\n--- a/src/ipa/ipu3/ipu3.cpp\n+++ b/src/ipa/ipu3/ipu3.cpp\n@@ -21,6 +21,7 @@\n #include \"libcamera/internal/buffer.h\"\n #include \"libcamera/internal/log.h\"\n \n+#include \"ipu3_agc.h\"\n #include \"ipu3_awb.h\"\n \n namespace libcamera {\n@@ -65,6 +66,8 @@ private:\n \n \t/* Interface to the AWB algorithm */\n \tipa::IPU3Awb *awbAlgo_;\n+\t/* Interface to the AEC/AGC algorithm */\n+\tipa::IPU3Agc *agcAlgo_;\n \t/* Local parameter storage */\n \tipu3_uapi_params params_;\n };\n@@ -101,6 +104,8 @@ void IPAIPU3::configure(const std::map<uint32_t, ControlInfoMap> &entityControls\n \tawbAlgo_ = new ipa::IPU3Awb();\n \tawbAlgo_->initialise(params_);\n \n+\tagcAlgo_ = new ipa::IPU3Agc();\n+\n \tsetControls(0);\n }\n \n@@ -187,7 +192,10 @@ void IPAIPU3::parseStatistics(unsigned int frame,\n {\n \tControlList ctrls(controls::controls);\n \n-\tawbAlgo_->calculateWBGains(Rectangle(250, 160, 800, 400), stats);\n+\tagcAlgo_->process(stats, exposure_, gain_);\n+\tif (agcAlgo_->converged())\n+\t\tawbAlgo_->calculateWBGains(Rectangle(250, 160, 800, 400), stats);\n+\n \tsetControls(frame);\n \n \tipa::ipu3::IPU3Action op;\ndiff --git a/src/ipa/ipu3/ipu3_agc.cpp b/src/ipa/ipu3/ipu3_agc.cpp\nnew file mode 100644\nindex 00000000..2636e2ea\n--- /dev/null\n+++ b/src/ipa/ipu3/ipu3_agc.cpp\n@@ -0,0 +1,171 @@\n+/* SPDX-License-Identifier: BSD-2-Clause */\n+/*\n+ * Copyright (C) 2019, Raspberry Pi (Trading) Limited\n+ *\n+ * ipu3_agc.cpp - AGC/AEC control algorithm\n+ */\n+\n+#include \"ipu3_agc.h\"\n+\n+#include <numeric>\n+\n+#include \"libcamera/internal/log.h\"\n+\n+namespace libcamera {\n+\n+namespace ipa {\n+\n+LOG_DEFINE_CATEGORY(IPU3Agc)\n+\n+/* Number of frames to wait before calculating stats on minimum exposure */\n+static const uint32_t kInitialFrameMinAECount = 4;\n+/* Number of frames to wait before calculating stats on maximum exposure */\n+static const uint32_t kInitialFrameMaxAECount = 8;\n+/* Number of frames to wait before calculating stats and estimate gain/exposure */\n+static const uint32_t kInitialFrameSkipCount = 12;\n+\n+/* Number of frames to wait between new gain/exposure estimations */\n+static const uint32_t kFrameSkipCount = 4;\n+\n+IPU3Agc::IPU3Agc()\n+\t: frameCount_(0), converged_(false)\n+{\n+}\n+\n+IPU3Agc::~IPU3Agc()\n+{\n+}\n+\n+void IPU3Agc::initialise()\n+{\n+}\n+\n+void IPU3Agc::process()\n+{\n+}\n+\n+/*\n+ * \\todo This function is taken from numerical recipes and calculates all\n+ * moments. It needs to be rewritten properly and maybe in a \"math\" class ?\n+ */\n+void IPU3Agc::moments(std::unordered_map<uint32_t, uint32_t> &data, int n)\n+{\n+\tint j;\n+\tdouble ep = 0.0, s, p;\n+\tdouble ave, adev, sdev;\n+\tdouble var, skew, curt;\n+\n+\ts = 0.0;\n+\tfor (j = 1; j <= n; j++)\n+\t\ts += data[j];\n+\n+\tave = s / n;\n+\tadev = var = skew = curt = 0.0;\n+\n+\tfor (j = 1; j <= n; j++) {\n+\t\tadev += s = data[j] - (ave);\n+\t\tep += s;\n+\t\tvar += (p = s * s);\n+\t\tskew += (p *= s);\n+\t\tcurt += (p *= s);\n+\t}\n+\n+\tadev /= n;\n+\tvar = (var - ep * ep / n) / (n - 1);\n+\tsdev = std::sqrt(var);\n+\n+\tif (var) {\n+\t\tskew /= n * var * sdev;\n+\t\tcurt = curt / (n * var * var) - 3.0;\n+\t}\n+\tskew_ = skew;\n+}\n+\n+void IPU3Agc::processBrightness(const ipu3_uapi_stats_3a *stats)\n+{\n+\tcellsBrightness_.clear();\n+\n+\t/*\\todo Replace constant values with real BDS configuration */\n+\tfor (uint32_t j = 0; j < 45; j++) {\n+\t\tfor (uint32_t i = 0; i < 160 * 45 * 8; i += 8) {\n+\t\t\tuint8_t Gr = stats->awb_raw_buffer.meta_data[i];\n+\t\t\tuint8_t R = stats->awb_raw_buffer.meta_data[i + 1];\n+\t\t\tuint8_t B = stats->awb_raw_buffer.meta_data[i + 2];\n+\t\t\tuint8_t Gb = stats->awb_raw_buffer.meta_data[i + 3];\n+\t\t\tcellsBrightness_.push_back(static_cast<uint32_t>(0.299 * R + 0.587 * (Gr + Gb) / 2 + 0.114 * B));\n+\t\t}\n+\t}\n+\tstd::sort(cellsBrightness_.begin(), cellsBrightness_.end());\n+\n+\t/* \\todo create a class to generate histograms ! */\n+\tstd::unordered_map<uint32_t, uint32_t> hist;\n+\tfor (uint32_t const &val : cellsBrightness_)\n+\t\thist[val]++;\n+\tmoments(hist, 256);\n+}\n+\n+/* \\todo make this function a math one ? */\n+uint32_t IPU3Agc::rootApproximation()\n+{\n+\treturn (currentExposure_ * prevSkew_ + prevExposure_ * currentSkew_) / (prevSkew_ + currentSkew_);\n+}\n+\n+void IPU3Agc::lockExposure(uint32_t &exposure, uint32_t &gain)\n+{\n+\t/* Algorithm initialization wait for first valid frames */\n+\t/* \\todo - have a number of frames given by DelayedControls ?\n+\t * - implement a function for IIR */\n+\tif (frameCount_ == kInitialFrameMinAECount) {\n+\t\tprevExposure_ = exposure;\n+\n+\t\tprevSkew_ = skew_;\n+\t\t/* \\todo use configured values */\n+\t\texposure = 800;\n+\t\tgain = 8;\n+\t\tcurrentExposure_ = exposure;\n+\t} else if (frameCount_ == kInitialFrameMaxAECount) {\n+\t\tcurrentSkew_ = skew_;\n+\t\texposure = rootApproximation();\n+\t\tnextExposure_ = exposure;\n+\t\tlastFrame_ = frameCount_;\n+\t} else if ((frameCount_ >= kInitialFrameSkipCount) && (frameCount_ - lastFrame_ >= kFrameSkipCount)) {\n+\t\tcurrentSkew_ = skew_;\n+\t\t/* \\todo properly calculate a gain */\n+\t\tif (frameCount_ == kInitialFrameSkipCount)\n+\t\t\tgain = ((8 * prevSkew_) + (1 * currentSkew_)) / (prevSkew_ + currentSkew_);\n+\n+\t\tif (currentSkew_ - prevSkew_ > 1) {\n+\t\t\t/* under exposed */\n+\t\t\tprevExposure_ = nextExposure_;\n+\t\t\texposure = rootApproximation();\n+\t\t\tnextExposure_ = exposure;\n+\t\t} else if (currentSkew_ - prevSkew_ < -1) {\n+\t\t\t/* over exposed */\n+\t\t\tcurrentExposure_ = nextExposure_;\n+\t\t\texposure = rootApproximation();\n+\t\t\tnextExposure_ = exposure;\n+\t\t} else {\n+\t\t\t/* we have converged */\n+\t\t\tconverged_ = true;\n+\t\t}\n+\t\tlastFrame_ = frameCount_;\n+\t\tprevSkew_ = currentSkew_;\n+\t}\n+}\n+\n+void IPU3Agc::process(const ipu3_uapi_stats_3a *stats, uint32_t &exposure, uint32_t &gain)\n+{\n+\tprocessBrightness(stats);\n+\tif (!converged_)\n+\t\tlockExposure(exposure, gain);\n+\telse {\n+\t\t/* Are we still well exposed ? */\n+\t\tif ((skew_ < 2) || (skew_ > 4))\n+\t\t\tconverged_ = false;\n+\t}\n+\tframeCount_++;\n+}\n+\n+} /* namespace ipa */\n+\n+} /* namespace libcamera */\ndiff --git a/src/ipa/ipu3/ipu3_agc.h b/src/ipa/ipu3/ipu3_agc.h\nnew file mode 100644\nindex 00000000..b14a2a2f\n--- /dev/null\n+++ b/src/ipa/ipu3/ipu3_agc.h\n@@ -0,0 +1,62 @@\n+/* SPDX-License-Identifier: BSD-2-Clause */\n+/*\n+ * Copyright (C) 2019, Raspberry Pi (Trading) Limited\n+ *\n+ * ipu3_agc.h - IPU3 AGC/AEC control algorithm\n+ */\n+#ifndef __LIBCAMERA_IPU3_AGC_H__\n+#define __LIBCAMERA_IPU3_AGC_H__\n+\n+#include <unordered_map>\n+#include <vector>\n+\n+#include <linux/intel-ipu3.h>\n+\n+#include <libcamera/geometry.h>\n+\n+#include \"libipa/algorithm.h\"\n+\n+namespace libcamera {\n+\n+namespace ipa {\n+\n+class IPU3Agc : public Algorithm\n+{\n+public:\n+\tIPU3Agc();\n+\t~IPU3Agc();\n+\n+\tvoid initialise() override;\n+\tvoid process() override;\n+\n+\tvoid process(const ipu3_uapi_stats_3a *stats, uint32_t &exposure, uint32_t &gain);\n+\tbool converged() { return converged_; }\n+\n+private:\n+\tvoid moments(std::unordered_map<uint32_t, uint32_t> &data, int n);\n+\tvoid processBrightness(const ipu3_uapi_stats_3a *stats);\n+\tuint32_t rootApproximation();\n+\tvoid lockExposure(uint32_t &exposure, uint32_t &gain);\n+\n+\tuint64_t frameCount_;\n+\tuint64_t lastFrame_;\n+\n+\t/* Vector of calculated brightness for each cell */\n+\tstd::vector<uint32_t> cellsBrightness_;\n+\n+\t/* Values for filtering */\n+\tuint32_t prevExposure_;\n+\tuint32_t currentExposure_;\n+\tuint32_t nextExposure_;\n+\n+\tdouble skew_;\n+\tdouble prevSkew_;\n+\tdouble currentSkew_;\n+\tbool converged_;\n+};\n+\n+} /* namespace ipa */\n+\n+} /* namespace libcamera */\n+\n+#endif /* __LIBCAMERA_IPU3_AGC_H__ */\ndiff --git a/src/ipa/ipu3/meson.build b/src/ipa/ipu3/meson.build\nindex 07a864c8..43ad0e0d 100644\n--- a/src/ipa/ipu3/meson.build\n+++ b/src/ipa/ipu3/meson.build\n@@ -4,6 +4,7 @@ ipa_name = 'ipa_ipu3'\n \n ipu3_ipa_sources = files([\n 'ipu3.cpp',\n+ 'ipu3_agc.cpp',\n 'ipu3_awb.cpp',\n ])\n \n", "prefixes": [ "libcamera-devel", "RFC", "v2", "3/3" ] }