Patch Detail
Show a patch.
GET /api/patches/14578/?format=api
{ "id": 14578, "url": "https://patchwork.libcamera.org/api/patches/14578/?format=api", "web_url": "https://patchwork.libcamera.org/patch/14578/", "project": { "id": 1, "url": "https://patchwork.libcamera.org/api/projects/1/?format=api", "name": "libcamera", "link_name": "libcamera", "list_id": "libcamera_core", "list_email": "libcamera-devel@lists.libcamera.org", "web_url": "", "scm_url": "", "webscm_url": "" }, "msgid": "<20211111140928.136111-8-jeanmichel.hautbois@ideasonboard.com>", "date": "2021-11-11T14:09:21", "name": "[libcamera-devel,v4,07/14] ipa: ipu3: agc: Improve gain calculation", "commit_ref": null, "pull_url": null, "state": "accepted", "archived": false, "hash": "19e6edd3dfd03188c4f84fce107637eb04f52106", "submitter": { "id": 75, "url": "https://patchwork.libcamera.org/api/people/75/?format=api", "name": "Jean-Michel Hautbois", "email": "jeanmichel.hautbois@ideasonboard.com" }, "delegate": null, "mbox": "https://patchwork.libcamera.org/patch/14578/mbox/", "series": [ { "id": 2714, "url": "https://patchwork.libcamera.org/api/series/2714/?format=api", "web_url": "https://patchwork.libcamera.org/project/libcamera/list/?series=2714", "date": "2021-11-11T14:09:14", "name": "IPA: IPU3: Introduce per-frame controls", "version": 4, "mbox": "https://patchwork.libcamera.org/series/2714/mbox/" } ], "comments": "https://patchwork.libcamera.org/api/patches/14578/comments/", "check": "pending", "checks": "https://patchwork.libcamera.org/api/patches/14578/checks/", "tags": {}, "headers": { "Return-Path": "<libcamera-devel-bounces@lists.libcamera.org>", "X-Original-To": "parsemail@patchwork.libcamera.org", "Delivered-To": "parsemail@patchwork.libcamera.org", "Received": [ "from lancelot.ideasonboard.com (lancelot.ideasonboard.com\n\t[92.243.16.209])\n\tby patchwork.libcamera.org (Postfix) with ESMTPS id A5345C3250\n\tfor <parsemail@patchwork.libcamera.org>;\n\tThu, 11 Nov 2021 14:09:42 +0000 (UTC)", "from lancelot.ideasonboard.com (localhost [IPv6:::1])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTP id 0F7ED60441;\n\tThu, 11 Nov 2021 15:09:42 +0100 (CET)", "from perceval.ideasonboard.com (perceval.ideasonboard.com\n\t[213.167.242.64])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id 8D6CC6036C\n\tfor <libcamera-devel@lists.libcamera.org>;\n\tThu, 11 Nov 2021 15:09:32 +0100 (CET)", "from tatooine.ideasonboard.com (unknown\n\t[IPv6:2a01:e0a:169:7140:e627:8337:a781:d98])\n\tby perceval.ideasonboard.com (Postfix) with ESMTPSA id 463C61543;\n\tThu, 11 Nov 2021 15:09:32 +0100 (CET)" ], "Authentication-Results": "lancelot.ideasonboard.com;\n\tdkim=fail reason=\"signature verification failed\" (1024-bit key;\n\tunprotected) header.d=ideasonboard.com header.i=@ideasonboard.com\n\theader.b=\"BYm6HyZP\"; dkim-atps=neutral", "DKIM-Signature": "v=1; a=rsa-sha256; c=relaxed/simple; d=ideasonboard.com;\n\ts=mail; t=1636639772;\n\tbh=BeJz93F6GMcTUgsLx2/ns6lOLKAugtTQPDs0TL8H4uc=;\n\th=From:To:Cc:Subject:Date:In-Reply-To:References:From;\n\tb=BYm6HyZPjf0JqgyAyG4KV1Dw8ZUE3NsRj/X5MM6xPyJubXY62/uVZrDuRPfjQYTmA\n\t0PyAWnYI/Wk0AhJIuXunEuFwLshfU0N3PJ/S+tFQRY8O6Fj6i14K3l8+wKzBFyw18G\n\t0qGz6L+6KIGSOo1fdDhh7XSHKL+l9a4W0NRXMG4U=", "From": "Jean-Michel Hautbois <jeanmichel.hautbois@ideasonboard.com>", "To": "libcamera-devel@lists.libcamera.org", "Date": "Thu, 11 Nov 2021 15:09:21 +0100", "Message-Id": "<20211111140928.136111-8-jeanmichel.hautbois@ideasonboard.com>", "X-Mailer": "git-send-email 2.32.0", "In-Reply-To": "<20211111140928.136111-1-jeanmichel.hautbois@ideasonboard.com>", "References": "<20211111140928.136111-1-jeanmichel.hautbois@ideasonboard.com>", "MIME-Version": "1.0", "Content-Type": "text/plain; charset=UTF-8", "Content-Transfer-Encoding": "8bit", "Subject": "[libcamera-devel] [PATCH v4 07/14] ipa: ipu3: agc: Improve gain\n\tcalculation", "X-BeenThere": "libcamera-devel@lists.libcamera.org", "X-Mailman-Version": "2.1.29", "Precedence": "list", "List-Id": "<libcamera-devel.lists.libcamera.org>", "List-Unsubscribe": "<https://lists.libcamera.org/options/libcamera-devel>,\n\t<mailto:libcamera-devel-request@lists.libcamera.org?subject=unsubscribe>", "List-Archive": "<https://lists.libcamera.org/pipermail/libcamera-devel/>", "List-Post": "<mailto:libcamera-devel@lists.libcamera.org>", "List-Help": "<mailto:libcamera-devel-request@lists.libcamera.org?subject=help>", "List-Subscribe": "<https://lists.libcamera.org/listinfo/libcamera-devel>,\n\t<mailto:libcamera-devel-request@lists.libcamera.org?subject=subscribe>", "Errors-To": "libcamera-devel-bounces@lists.libcamera.org", "Sender": "\"libcamera-devel\" <libcamera-devel-bounces@lists.libcamera.org>" }, "content": "When an image is partially saturated, its brightness is not increasing\nlinearly when the shutter time or gain increases. It is a big issue with\na backlight as the algorithm is fading to darkness right now.\n\nIntroduce a function to estimate the brightness of the frame, based on\nthe current exposure/gain and loop on it several times to estimate it\nagain and approach the non linear function.\n\nInspired-by: 7de5506c30b3 (\"libcamera: src: ipa: raspberrypi: agc: Improve gain update calculation for partly saturated images\")\nSigned-off-by: Jean-Michel Hautbois <jeanmichel.hautbois@ideasonboard.com>\nReviewed-by: Kieran Bingham <kieran.bingham@ideasonboard.com>\n---\n src/ipa/ipu3/algorithms/agc.cpp | 104 +++++++++++++++++++++++++++++++-\n src/ipa/ipu3/algorithms/agc.h | 6 +-\n 2 files changed, 106 insertions(+), 4 deletions(-)", "diff": "diff --git a/src/ipa/ipu3/algorithms/agc.cpp b/src/ipa/ipu3/algorithms/agc.cpp\nindex 61ca8b3f..a34f070e 100644\n--- a/src/ipa/ipu3/algorithms/agc.cpp\n+++ b/src/ipa/ipu3/algorithms/agc.cpp\n@@ -67,6 +67,17 @@ static constexpr uint32_t kMinCellsPerZoneRatio = 255 * 20 / 100;\n /* Number of frames to wait before calculating stats on minimum exposure */\n static constexpr uint32_t kNumStartupFrames = 10;\n \n+/* Maximum luminance used for brightness normalization */\n+static constexpr uint32_t kMaxLuminance = 255;\n+\n+/*\n+ * Normalized luma value target.\n+ *\n+ * It's a number that's chosen so that, when the camera points at a grey\n+ * target, the resulting image brightness is considered right.\n+ */\n+static constexpr double kNormalizedLumaTarget = 0.16;\n+\n Agc::Agc()\n \t: frameCount_(0), iqMean_(0.0), lineDuration_(0s), minExposureLines_(0),\n \t maxExposureLines_(0), filteredExposure_(0s), currentExposure_(0s),\n@@ -185,14 +196,20 @@ void Agc::filterExposure()\n /**\n * \\brief Estimate the new exposure and gain values\n * \\param[inout] frameContext The shared IPA frame Context\n+ * \\param[in] currentYGain The gain calculated on the current brightness level\n */\n-void Agc::computeExposure(IPAFrameContext &frameContext)\n+void Agc::computeExposure(IPAFrameContext &frameContext, double currentYGain)\n {\n \t/* Get the effective exposure and gain applied on the sensor. */\n \tuint32_t exposure = frameContext.sensor.exposure;\n \tdouble analogueGain = frameContext.sensor.gain;\n \n-\t/* Estimate the gain needed to have the proportion wanted */\n+\t/*\n+\t * Estimate the gain needed to have the proportion of pixels in a given\n+\t * range wanted. iqMean_ returns the mean value of the top 2% of the\n+\t * cumulative histogram, and we want it to be as close as possible to a\n+\t * configured target.\n+\t */\n \tdouble evGain = kEvGainTarget * knumHistogramBins / iqMean_;\n \n \tif (std::abs(evGain - 1.0) < 0.01) {\n@@ -202,6 +219,7 @@ void Agc::computeExposure(IPAFrameContext &frameContext)\n \t}\n \n \t/* extracted from Rpi::Agc::computeTargetExposure */\n+\n \t/* Calculate the shutter time in seconds */\n \tutils::Duration currentShutter = exposure * lineDuration_;\n \tLOG(IPU3Agc, Debug) << \"Actual total exposure \" << currentShutter * analogueGain\n@@ -209,6 +227,14 @@ void Agc::computeExposure(IPAFrameContext &frameContext)\n \t\t\t << \" Gain \" << analogueGain\n \t\t\t << \" Needed ev gain \" << evGain;\n \n+\tif (evGain < currentYGain)\n+\t\tevGain = currentYGain;\n+\n+\t/* Consider within 1% of the target as correctly exposed */\n+\tif (std::abs(evGain - 1.0) < 0.01)\n+\t\tLOG(IPU3Agc, Debug) << \"We are well exposed (iqMean = \"\n+\t\t\t\t << iqMean_ << \")\";\n+\n \t/*\n \t * Calculate the current exposure value for the scene as the latest\n \t * exposure value applied multiplied by the new estimated gain.\n@@ -257,6 +283,56 @@ void Agc::computeExposure(IPAFrameContext &frameContext)\n \tprevExposureValue_ = shutterTime * analogueGain;\n }\n \n+/**\n+ * \\brief Estimate the average brightness of the frame\n+ * \\param[in] frameContext The shared IPA frame context\n+ * \\param[in] grid The grid used to store the statistics in the IPU3\n+ * \\param[in] stats The IPU3 statistics and ISP results\n+ * \\param[in] currentYGain The gain calculated on the current brightness level\n+ * \\return The normalized luma\n+ *\n+ * Luma is the weighted sum of gamma-compressed R′G′B′ components of a color\n+ * video. The luma values are normalized as 0.0 to 1.0, with 1.0 being a\n+ * theoretical perfect reflector of 100% reference white. We use the Rec. 601\n+ * luma here.\n+ *\n+ * More detailed information can be found in:\n+ * https://en.wikipedia.org/wiki/Luma_(video)\n+ */\n+double Agc::computeInitialY(IPAFrameContext &frameContext,\n+\t\t\t const ipu3_uapi_grid_config &grid,\n+\t\t\t const ipu3_uapi_stats_3a *stats,\n+\t\t\t double currentYGain)\n+{\n+\tdouble redSum = 0, greenSum = 0, blueSum = 0;\n+\n+\tfor (unsigned int cellY = 0; cellY < grid.height; cellY++) {\n+\t\tfor (unsigned int cellX = 0; cellX < grid.width; cellX++) {\n+\t\t\tuint32_t cellPosition = cellY * stride_ + cellX;\n+\n+\t\t\tconst ipu3_uapi_awb_set_item *cell =\n+\t\t\t\treinterpret_cast<const ipu3_uapi_awb_set_item *>(\n+\t\t\t\t\t&stats->awb_raw_buffer.meta_data[cellPosition]\n+\t\t\t\t);\n+\n+\t\t\tredSum += cell->R_avg * currentYGain;\n+\t\t\tgreenSum += (cell->Gr_avg + cell->Gb_avg) / 2 * currentYGain;\n+\t\t\tblueSum += cell->B_avg * currentYGain;\n+\t\t}\n+\t}\n+\n+\t/*\n+\t * Estimate the sum of the brightness values, weighted with the gains\n+\t * applied on the channels in AWB as the Rec. 601 luma.\n+\t */\n+\tdouble Y_sum = redSum * frameContext.awb.gains.red * .299 +\n+\t\t greenSum * frameContext.awb.gains.green * .587 +\n+\t\t blueSum * frameContext.awb.gains.blue * .114;\n+\n+\t/* Return the normalized relative luminance. */\n+\treturn Y_sum / (grid.height * grid.width) / kMaxLuminance;\n+}\n+\n /**\n * \\brief Process IPU3 statistics, and run AGC operations\n * \\param[in] context The shared IPA context\n@@ -268,7 +344,29 @@ void Agc::computeExposure(IPAFrameContext &frameContext)\n void Agc::process(IPAContext &context, const ipu3_uapi_stats_3a *stats)\n {\n \tmeasureBrightness(stats, context.configuration.grid.bdsGrid);\n-\tcomputeExposure(context.frameContext);\n+\n+\tdouble currentYGain = 1.0;\n+\tdouble targetY = kNormalizedLumaTarget;\n+\n+\t/*\n+\t * Do this calculation a few times as brightness increase can be\n+\t * non-linear when there are saturated regions.\n+\t */\n+\tfor (int i = 0; i < 8; i++) {\n+\t\tdouble initialY = computeInitialY(context.frameContext,\n+\t\t\t\t\t\t context.configuration.grid.bdsGrid,\n+\t\t\t\t\t\t stats, currentYGain);\n+\t\tdouble extra_gain = std::min(10.0, targetY / (initialY + .001));\n+\n+\t\tcurrentYGain *= extra_gain;\n+\t\tLOG(IPU3Agc, Debug) << \"Initial Y \" << initialY\n+\t\t\t\t << \" target \" << targetY\n+\t\t\t\t << \" gives gain \" << currentYGain;\n+\t\tif (extra_gain < 1.01)\n+\t\t\tbreak;\n+\t}\n+\n+\tcomputeExposure(context.frameContext, currentYGain);\n \tframeCount_++;\n }\n \ndiff --git a/src/ipa/ipu3/algorithms/agc.h b/src/ipa/ipu3/algorithms/agc.h\nindex f0db25ee..79736283 100644\n--- a/src/ipa/ipu3/algorithms/agc.h\n+++ b/src/ipa/ipu3/algorithms/agc.h\n@@ -34,7 +34,11 @@ private:\n \tvoid measureBrightness(const ipu3_uapi_stats_3a *stats,\n \t\t\t const ipu3_uapi_grid_config &grid);\n \tvoid filterExposure();\n-\tvoid computeExposure(IPAFrameContext &frameContext);\n+\tvoid computeExposure(IPAFrameContext &frameContext, double currentYGain);\n+\tdouble computeInitialY(IPAFrameContext &frameContext,\n+\t\t\t const ipu3_uapi_grid_config &grid,\n+\t\t\t const ipu3_uapi_stats_3a *stats,\n+\t\t\t double currentYGain);\n \n \tuint64_t frameCount_;\n \tuint64_t lastFrame_;\n", "prefixes": [ "libcamera-devel", "v4", "07/14" ] }