Patch Detail
Show a patch.
GET /api/patches/13538/?format=api
{ "id": 13538, "url": "https://patchwork.libcamera.org/api/patches/13538/?format=api", "web_url": "https://patchwork.libcamera.org/patch/13538/", "project": { "id": 1, "url": "https://patchwork.libcamera.org/api/projects/1/?format=api", "name": "libcamera", "link_name": "libcamera", "list_id": "libcamera_core", "list_email": "libcamera-devel@lists.libcamera.org", "web_url": "", "scm_url": "", "webscm_url": "" }, "msgid": "<20210827080227.26370-4-jeanmichel.hautbois@ideasonboard.com>", "date": "2021-08-27T08:02:26", "name": "[libcamera-devel,v2,3/4] ipa: ipu3: Document AGC mean-based algorithm", "commit_ref": null, "pull_url": null, "state": "superseded", "archived": false, "hash": "b50cf21e2a4dd59a2e127bdc55e726e77a2c7ba8", "submitter": { "id": 75, "url": "https://patchwork.libcamera.org/api/people/75/?format=api", "name": "Jean-Michel Hautbois", "email": "jeanmichel.hautbois@ideasonboard.com" }, "delegate": null, "mbox": "https://patchwork.libcamera.org/patch/13538/mbox/", "series": [ { "id": 2407, "url": "https://patchwork.libcamera.org/api/series/2407/?format=api", "web_url": "https://patchwork.libcamera.org/project/libcamera/list/?series=2407", "date": "2021-08-27T08:02:23", "name": "IPU3: AWB and AGC improvements", "version": 2, "mbox": "https://patchwork.libcamera.org/series/2407/mbox/" } ], "comments": "https://patchwork.libcamera.org/api/patches/13538/comments/", "check": "pending", "checks": "https://patchwork.libcamera.org/api/patches/13538/checks/", "tags": {}, "headers": { "Return-Path": "<libcamera-devel-bounces@lists.libcamera.org>", "X-Original-To": "parsemail@patchwork.libcamera.org", "Delivered-To": "parsemail@patchwork.libcamera.org", "Received": [ "from lancelot.ideasonboard.com (lancelot.ideasonboard.com\n\t[92.243.16.209])\n\tby patchwork.libcamera.org (Postfix) with ESMTPS id 8A11EBD87C\n\tfor <parsemail@patchwork.libcamera.org>;\n\tFri, 27 Aug 2021 08:02:38 +0000 (UTC)", "from lancelot.ideasonboard.com (localhost [IPv6:::1])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTP id ED3A56893E;\n\tFri, 27 Aug 2021 10:02:36 +0200 (CEST)", "from perceval.ideasonboard.com (perceval.ideasonboard.com\n\t[213.167.242.64])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id 096106893B\n\tfor <libcamera-devel@lists.libcamera.org>;\n\tFri, 27 Aug 2021 10:02:34 +0200 (CEST)", "from tatooine.ideasonboard.com (unknown\n\t[IPv6:2a01:e0a:169:7140:ccf5:c267:eba8:cbb5])\n\tby perceval.ideasonboard.com (Postfix) with ESMTPSA id C00155A1;\n\tFri, 27 Aug 2021 10:02:33 +0200 (CEST)" ], "Authentication-Results": "lancelot.ideasonboard.com;\n\tdkim=fail reason=\"signature verification failed\" (1024-bit key;\n\tunprotected) header.d=ideasonboard.com header.i=@ideasonboard.com\n\theader.b=\"UTphcs4Q\"; dkim-atps=neutral", "DKIM-Signature": "v=1; a=rsa-sha256; c=relaxed/simple; d=ideasonboard.com;\n\ts=mail; t=1630051353;\n\tbh=jo/UG3hHfzS0A3qGlAUTIz02Qi2QLXk8m8kR5U0LNh4=;\n\th=From:To:Cc:Subject:Date:In-Reply-To:References:From;\n\tb=UTphcs4QHxaqctbAVTsJJqrtzP/iRwIjMiibLATqu06VV6bjEN+z21VSkptAH/Xeh\n\tvDL12Tihcdl/GGjCv54ObOFqSsnGljP3SYdgxj6wOcEPZWutjciyd2zWmdJ3VTGs0x\n\tlGSal18tyYk3RltjQ+NG9Fm15H7Rti31ckvVuRkA=", "From": "Jean-Michel Hautbois <jeanmichel.hautbois@ideasonboard.com>", "To": "libcamera-devel@lists.libcamera.org", "Date": "Fri, 27 Aug 2021 10:02:26 +0200", "Message-Id": "<20210827080227.26370-4-jeanmichel.hautbois@ideasonboard.com>", "X-Mailer": "git-send-email 2.30.2", "In-Reply-To": "<20210827080227.26370-1-jeanmichel.hautbois@ideasonboard.com>", "References": "<20210827080227.26370-1-jeanmichel.hautbois@ideasonboard.com>", "MIME-Version": "1.0", "Content-Transfer-Encoding": "8bit", "Subject": "[libcamera-devel] [PATCH v2 3/4] ipa: ipu3: Document AGC mean-based\n\talgorithm", "X-BeenThere": "libcamera-devel@lists.libcamera.org", "X-Mailman-Version": "2.1.29", "Precedence": "list", "List-Id": "<libcamera-devel.lists.libcamera.org>", "List-Unsubscribe": "<https://lists.libcamera.org/options/libcamera-devel>,\n\t<mailto:libcamera-devel-request@lists.libcamera.org?subject=unsubscribe>", "List-Archive": "<https://lists.libcamera.org/pipermail/libcamera-devel/>", "List-Post": "<mailto:libcamera-devel@lists.libcamera.org>", "List-Help": "<mailto:libcamera-devel-request@lists.libcamera.org?subject=help>", "List-Subscribe": "<https://lists.libcamera.org/listinfo/libcamera-devel>,\n\t<mailto:libcamera-devel-request@lists.libcamera.org?subject=subscribe>", "Errors-To": "libcamera-devel-bounces@lists.libcamera.org", "Sender": "\"libcamera-devel\" <libcamera-devel-bounces@lists.libcamera.org>" }, "content": "The AGC mean-based algorithm is partially documented. Improve the\ndocumentation to help understand how it works and mark some \\todo for\nfuture improvements.\n\nSigned-off-by: Jean-Michel Hautbois <jeanmichel.hautbois@ideasonboard.com>\n---\n src/ipa/ipu3/algorithms/agc_mean.cpp | 91 +++++++++++++++++++++++++---\n src/ipa/ipu3/algorithms/agc_mean.h | 2 +-\n 2 files changed, 84 insertions(+), 9 deletions(-)", "diff": "diff --git a/src/ipa/ipu3/algorithms/agc_mean.cpp b/src/ipa/ipu3/algorithms/agc_mean.cpp\nindex 193f6e9a..b535e14b 100644\n--- a/src/ipa/ipu3/algorithms/agc_mean.cpp\n+++ b/src/ipa/ipu3/algorithms/agc_mean.cpp\n@@ -2,7 +2,7 @@\n /*\n * Copyright (C) 2021, Ideas On Board\n *\n- * agc_mean.cpp - AGC/AEC control algorithm\n+ * agc_mean.cpp - AGC/AEC mean-based control algorithm\n */\n \n #include \"agc_mean.h\"\n@@ -17,12 +17,26 @@\n \n #include \"libipa/histogram.h\"\n \n+/**\n+ * \\file agc_mean.h\n+ */\n+\n namespace libcamera {\n \n using namespace std::literals::chrono_literals;\n \n namespace ipa::ipu3::algorithms {\n \n+/**\n+ * \\class AgcMean\n+ * \\brief The class to use the mean-based auto-exposure algorithm\n+ *\n+ * The mean-based algorithm is calculating an exposure and gain value such as\n+ * a given quantity of pixels lie in the top 2% of the histogram. The AWB gains\n+ * are not used here, and all cells in the grid have the same weight, like an\n+ * average-metering case.\n+ */\n+\n LOG_DEFINE_CATEGORY(IPU3AgcMean)\n \n /* Number of frames to wait before calculating stats on minimum exposure */\n@@ -30,13 +44,20 @@ static constexpr uint32_t kInitialFrameMinAECount = 4;\n /* Number of frames to wait between new gain/exposure estimations */\n static constexpr uint32_t kFrameSkipCount = 6;\n \n-/* Maximum ISO value for analogue gain */\n+/* Minimum ISO value for analogue gain (no digital gain supported) */\n static constexpr uint32_t kMinISO = 100;\n+/* Maximum ISO value for analogue gain (no digital gain supported) */\n static constexpr uint32_t kMaxISO = 1500;\n \n-/* Maximum analogue gain value\n- * \\todo grab it from a camera helper */\n+/*\n+ * Minimum analogue gain value\n+ * \\todo grab it from a camera helper\n+ */\n static constexpr uint32_t kMinGain = kMinISO / 100;\n+/*\n+ * Maximum analogue gain value\n+ * \\todo grab it from a camera helper\n+ */\n static constexpr uint32_t kMaxGain = kMaxISO / 100;\n \n /* \\todo use calculated value based on sensor */\n@@ -45,6 +66,7 @@ static constexpr uint32_t kMaxExposure = 1976;\n \n /* Histogram constants */\n static constexpr uint32_t knumHistogramBins = 256;\n+/* Target value to reach for the top 2% of the histogram */\n static constexpr double kEvGainTarget = 0.5;\n \n /* A cell is 8 bytes and contains averages for RGB values and saturation ratio */\n@@ -57,8 +79,18 @@ AgcMean::AgcMean()\n {\n }\n \n+/**\n+ * \\brief Configure the AGC given a configInfo\n+ * \\param[in] context The shared IPA context (\\todo not used yet)\n+ * \\param[in] configInfo The IPA configuration data, received from the pipeline\n+ * handler\n+ *\n+ * \\return 0\n+ */\n int AgcMean::configure([[maybe_unused]] IPAContext &context,\n-\t\t const IPAConfigInfo &configInfo){\n+\t\t const IPAConfigInfo &configInfo)\n+{\n+\t/* \\todo use the configInfo fields and IPAContext to store the limits */\n \tlineDuration_ = configInfo.sensorInfo.lineLength * 1.0s\n \t\t / configInfo.sensorInfo.pixelRate;\n \tmaxExposureTime_ = kMaxExposure * lineDuration_;\n@@ -66,9 +98,22 @@ int AgcMean::configure([[maybe_unused]] IPAContext &context,\n \treturn 0;\n }\n \n+/**\n+ * \\brief Estimate the mean quantile of the top 2% of the histogram\n+ * \\param[in] stats The statistics buffer coming from the pipeline handler\n+ * \\param[in] grid The grid used to store the statistics in the IPU3\n+ */\n void AgcMean::processBrightness(const ipu3_uapi_stats_3a *stats,\n \t\t\t const ipu3_uapi_grid_config &grid)\n {\n+\t/*\n+\t * Get the applied grid from the statistics buffer. When the kernel\n+\t * receives a grid from the parameters buffer, it will check and align\n+\t * all the values. For instance, it will automatically fill the x_end\n+\t * value based on x_start, grid width and log2 width.\n+\t * \\todo Use the grid calculated in configure as there is a bug in IPU3\n+\t * causing the width (maybe height) to be bit-shifted.\n+\t */\n \tconst struct ipu3_uapi_grid_config statsAeGrid = stats->stats_4a_config.awb_config.grid;\n \tRectangle aeRegion = { statsAeGrid.x_start,\n \t\t\t statsAeGrid.y_start,\n@@ -85,6 +130,7 @@ void AgcMean::processBrightness(const ipu3_uapi_stats_3a *stats,\n \tuint32_t i, j;\n \tuint32_t count = 0;\n \n+\t/* Initialise the histogram array */\n \tuint32_t hist[knumHistogramBins] = { 0 };\n \tfor (j = topleftY;\n \t j < topleftY + (aeRegion.size().height >> grid.block_height_log2);\n@@ -92,12 +138,18 @@ void AgcMean::processBrightness(const ipu3_uapi_stats_3a *stats,\n \t\tfor (i = startX + startY; i < endX + startY; i += kCellSize) {\n \t\t\t/*\n \t\t\t * The grid width (and maybe height) is not reliable.\n-\t\t\t * We observed a bit shift which makes the value 160 to be 32 in the stats grid.\n-\t\t\t * Use the one passed at init time.\n+\t\t\t * We observed a bit shift which makes the value 160 to\n+\t\t\t * be 32 in the stats grid. Use the one from configure.\n \t\t\t */\n \t\t\tif (stats->awb_raw_buffer.meta_data[i + 4 + j * grid.width] == 0) {\n \t\t\t\tuint8_t Gr = stats->awb_raw_buffer.meta_data[i + 0 + j * grid.width];\n \t\t\t\tuint8_t Gb = stats->awb_raw_buffer.meta_data[i + 3 + j * grid.width];\n+\t\t\t\t/*\n+\t\t\t\t * Store the average green value to estimate the\n+\t\t\t\t * brightness. Even the over exposed pixels are\n+\t\t\t\t * taken into account.\n+\t\t\t\t * \\todo remove count which is not used.\n+\t\t\t\t */\n \t\t\t\thist[(Gr + Gb) / 2]++;\n \t\t\t\tcount++;\n \t\t\t}\n@@ -108,11 +160,14 @@ void AgcMean::processBrightness(const ipu3_uapi_stats_3a *stats,\n \tiqMean_ = Histogram(Span<uint32_t>(hist)).interQuantileMean(0.98, 1.0);\n }\n \n+/**\n+ * \\brief Apply a filter on the exposure value to limit the speed of changes\n+ */\n void AgcMean::filterExposure()\n {\n \tdouble speed = 0.2;\n \tif (prevExposure_ == 0s) {\n-\t\t/* DG stands for digital gain.*/\n+\t\t/* DG stands for digital gain, which is always 1.0 for now. */\n \t\tprevExposure_ = currentExposure_;\n \t\tprevExposureNoDg_ = currentExposureNoDg_;\n \t} else {\n@@ -134,6 +189,7 @@ void AgcMean::filterExposure()\n \t * We can't let the no_dg exposure deviate too far below the\n \t * total exposure, as there might not be enough digital gain available\n \t * in the ISP to hide it (which will cause nasty oscillation).\n+\t * \\todo add the digital gain usage\n \t */\n \tdouble fastReduceThreshold = 0.4;\n \tif (prevExposureNoDg_ <\n@@ -142,6 +198,11 @@ void AgcMean::filterExposure()\n \tLOG(IPU3AgcMean, Debug) << \"After filtering, total_exposure \" << prevExposure_;\n }\n \n+/**\n+ * \\brief Estimate the new exposure and gain values\n+ * \\param[in] exposure The exposure value reference as a number of lines\n+ * \\param[in] gain The gain reference to be updated\n+ */\n void AgcMean::lockExposureGain(uint32_t &exposure, double &gain)\n {\n \t/* Algorithm initialization should wait for first valid frames */\n@@ -154,15 +215,20 @@ void AgcMean::lockExposureGain(uint32_t &exposure, double &gain)\n \tif (std::abs(iqMean_ - kEvGainTarget * knumHistogramBins) <= 1) {\n \t\tLOG(IPU3AgcMean, Debug) << \"!!! Good exposure with iqMean = \" << iqMean_;\n \t} else {\n+\t\t/* Estimate the gain needed to have the proportion wanted */\n \t\tdouble newGain = kEvGainTarget * knumHistogramBins / iqMean_;\n \n \t\t/* extracted from Rpi::Agc::computeTargetExposure */\n+\t\t/* Calculate the shutter time in seconds */\n \t\tlibcamera::utils::Duration currentShutter = exposure * lineDuration_;\n+\t\t/* Ev = shutter_time * gain */\n \t\tcurrentExposureNoDg_ = currentShutter * gain;\n \t\tLOG(IPU3AgcMean, Debug) << \"Actual total exposure \" << currentExposureNoDg_\n \t\t\t\t << \" Shutter speed \" << currentShutter\n \t\t\t\t << \" Gain \" << gain;\n+\t\t/* Apply the gain calculated to the current exposure value */\n \t\tcurrentExposure_ = currentExposureNoDg_ * newGain;\n+\t\t/* Clamp the exposure value to the min and max authorized */\n \t\tlibcamera::utils::Duration maxTotalExposure = maxExposureTime_ * kMaxGain;\n \t\tcurrentExposure_ = std::min(currentExposure_, maxTotalExposure);\n \t\tLOG(IPU3AgcMean, Debug) << \"Target total exposure \" << currentExposure_;\n@@ -170,6 +236,7 @@ void AgcMean::lockExposureGain(uint32_t &exposure, double &gain)\n \t\t/* \\todo: estimate if we need to desaturate */\n \t\tfilterExposure();\n \n+\t\t/* Divide the exposure value as new exposure and gain values */\n \t\tlibcamera::utils::Duration newExposure = 0.0s;\n \t\tif (currentShutter < maxExposureTime_) {\n \t\t\texposure = std::clamp(static_cast<uint32_t>(exposure * currentExposure_ / currentExposureNoDg_), kMinExposure, kMaxExposure);\n@@ -185,11 +252,19 @@ void AgcMean::lockExposureGain(uint32_t &exposure, double &gain)\n \tlastFrame_ = frameCount_;\n }\n \n+/**\n+ * \\brief Process IPU3 statistics, and run AGC operations\n+ * \\param[in] context The shared IPA context\n+ * \\param[in] stats The IPU3 statistics and ISP results\n+ */\n void AgcMean::process(IPAContext &context, const ipu3_uapi_stats_3a *stats)\n {\n+\t/* Get the latest exposure and gain applied */\n \tuint32_t &exposure = context.frameContext.agc.exposure;\n \tdouble &gain = context.frameContext.agc.gain;\n+\t/* Calculate the current brightness */\n \tprocessBrightness(stats, context.configuration.grid.bdsGrid);\n+\t/* Update the exposure and gain values */\n \tlockExposureGain(exposure, gain);\n \tframeCount_++;\n }\ndiff --git a/src/ipa/ipu3/algorithms/agc_mean.h b/src/ipa/ipu3/algorithms/agc_mean.h\nindex 97114121..6232597d 100644\n--- a/src/ipa/ipu3/algorithms/agc_mean.h\n+++ b/src/ipa/ipu3/algorithms/agc_mean.h\n@@ -2,7 +2,7 @@\n /*\n * Copyright (C) 2021, Ideas On Board\n *\n- * agc_mean.h - IPU3 AGC/AEC control algorithm\n+ * agc_mean.h - AGC/AEC mean-based control algorithm\n */\n #ifndef __LIBCAMERA_IPU3_ALGORITHMS_AGC_H__\n #define __LIBCAMERA_IPU3_ALGORITHMS_AGC_H__\n", "prefixes": [ "libcamera-devel", "v2", "3/4" ] }