[{"id":20946,"web_url":"https://patchwork.libcamera.org/comment/20946/","msgid":"<81a8f4c3-0532-ac25-2e1c-257451f82d8d@ideasonboard.com>","date":"2021-11-15T07:54:40","subject":"Re: [libcamera-devel] [PATCH v5 07/14] ipa: ipu3: agc: Improve gain\n\tcalculation","submitter":{"id":86,"url":"https://patchwork.libcamera.org/api/people/86/","name":"Umang Jain","email":"umang.jain@ideasonboard.com"},"content":"Hi JM,\n\nOn 11/13/21 2:19 PM, Jean-Michel Hautbois wrote:\n> When an image is partially saturated, its brightness is not increasing\n> linearly when the shutter time or gain increases. It is a big issue with\n> a backlight as the algorithm is fading to darkness right now.\n>\n> Introduce a function to estimate the brightness of the frame, based on\n> the current exposure/gain and loop on it several times to estimate it\n> again and approach the non linear function.\n>\n> Inspired-by: 7de5506c30b3 (\"libcamera: src: ipa: raspberrypi: agc: Improve gain update calculation for partly saturated images\")\n> Signed-off-by: Jean-Michel Hautbois \n> Reviewed-by: Kieran Bingham \n> Reviewed-by: Paul Elder \n> ---\n> src/ipa/ipu3/algorithms/agc.cpp | 104 ++++++++++++++++++++++++++++++--\n> src/ipa/ipu3/algorithms/agc.h | 6 +-\n> 2 files changed, 103 insertions(+), 7 deletions(-)\n>\n> diff --git a/src/ipa/ipu3/algorithms/agc.cpp b/src/ipa/ipu3/algorithms/agc.cpp\n> index 61ca8b3f..a9f9244b 100644\n> --- a/src/ipa/ipu3/algorithms/agc.cpp\n> +++ b/src/ipa/ipu3/algorithms/agc.cpp\n> @@ -67,6 +67,17 @@ static constexpr uint32_t kMinCellsPerZoneRatio = 255 * 20 / 100;\n> /* Number of frames to wait before calculating stats on minimum exposure */\n> static constexpr uint32_t kNumStartupFrames = 10;\n> \n> +/* Maximum luminance used for brightness normalization */\n> +static constexpr uint32_t kMaxLuminance = 255;\n> +\n> +/*\n> + * Normalized luma value target.\n> + *\n> + * It's a number that's chosen so that, when the camera points at a grey\n> + * target, the resulting image brightness is considered right.\n> + */\n> +static constexpr double kNormalizedLumaTarget = 0.16;\n> +\n> Agc::Agc()\n> \t: frameCount_(0), iqMean_(0.0), lineDuration_(0s), minExposureLines_(0),\n> \t maxExposureLines_(0), filteredExposure_(0s), currentExposure_(0s),\n> @@ -185,23 +196,32 @@ void Agc::filterExposure()\n> /**\n> * \\brief Estimate the new exposure and gain values\n> * \\param[inout] frameContext The shared IPA frame Context\n> + * \\param[in] currentYGain The gain calculated on the current brightness level\n> */\n> -void Agc::computeExposure(IPAFrameContext &frameContext)\n> +void Agc::computeExposure(IPAFrameContext &frameContext, double currentYGain)\n> {\n> \t/* Get the effective exposure and gain applied on the sensor. */\n> \tuint32_t exposure = frameContext.sensor.exposure;\n> \tdouble analogueGain = frameContext.sensor.gain;\n> \n> -\t/* Estimate the gain needed to have the proportion wanted */\n> +\t/*\n> +\t * Estimate the gain needed to have the proportion of pixels in a given\n> +\t * range wanted. iqMean_ returns the mean value of the top 2% of the\n\ns/given range wanted./given desired range./\n\nReviewed-by: Umang Jain \n\n> +\t * cumulative histogram, and we want it to be as close as possible to a\n> +\t * configured target.\n> +\t */\n> \tdouble evGain = kEvGainTarget * knumHistogramBins / iqMean_;\n> \n> -\tif (std::abs(evGain - 1.0) < 0.01) {\n> +\tif (evGain < currentYGain)\n> +\t\tevGain = currentYGain;\n> +\n> +\t/* Consider within 1% of the target as correctly exposed */\n> +\tif (std::abs(evGain - 1.0) < 0.01)\n> \t\tLOG(IPU3Agc, Debug) << \"We are well exposed (iqMean = \"\n> \t\t\t\t << iqMean_ << \")\";\n> -\t\treturn;\n> -\t}\n> \n> \t/* extracted from Rpi::Agc::computeTargetExposure */\n> +\n> \t/* Calculate the shutter time in seconds */\n> \tutils::Duration currentShutter = exposure * lineDuration_;\n> \tLOG(IPU3Agc, Debug) << \"Actual total exposure \" << currentShutter * analogueGain\n> @@ -257,6 +277,56 @@ void Agc::computeExposure(IPAFrameContext &frameContext)\n> \tprevExposureValue_ = shutterTime * analogueGain;\n> }\n> \n> +/**\n> + * \\brief Estimate the average brightness of the frame\n> + * \\param[in] frameContext The shared IPA frame context\n> + * \\param[in] grid The grid used to store the statistics in the IPU3\n> + * \\param[in] stats The IPU3 statistics and ISP results\n> + * \\param[in] currentYGain The gain calculated on the current brightness level\n> + * \\return The normalized luma\n> + *\n> + * Luma is the weighted sum of gamma-compressed R′G′B′ components of a color\n> + * video. The luma values are normalized as 0.0 to 1.0, with 1.0 being a\n> + * theoretical perfect reflector of 100% reference white. We use the Rec. 601\n> + * luma here.\n> + *\n> + * More detailed information can be found in:\n> + * https://en.wikipedia.org/wiki/Luma_(video)\n> + */\n> +double Agc::computeInitialY(IPAFrameContext &frameContext,\n> +\t\t\t const ipu3_uapi_grid_config &grid,\n> +\t\t\t const ipu3_uapi_stats_3a *stats,\n> +\t\t\t double currentYGain)\n> +{\n> +\tdouble redSum = 0, greenSum = 0, blueSum = 0;\n> +\n> +\tfor (unsigned int cellY = 0; cellY < grid.height; cellY++) {\n> +\t\tfor (unsigned int cellX = 0; cellX < grid.width; cellX++) {\n> +\t\t\tuint32_t cellPosition = cellY * stride_ + cellX;\n> +\n> +\t\t\tconst ipu3_uapi_awb_set_item *cell =\n> +\t\t\t\treinterpret_cast(\n> +\t\t\t\t\t&stats->awb_raw_buffer.meta_data[cellPosition]\n> +\t\t\t\t);\n> +\n> +\t\t\tredSum += cell->R_avg * currentYGain;\n> +\t\t\tgreenSum += (cell->Gr_avg + cell->Gb_avg) / 2 * currentYGain;\n> +\t\t\tblueSum += cell->B_avg * currentYGain;\n> +\t\t}\n> +\t}\n> +\n> +\t/*\n> +\t * Estimate the sum of the brightness values, weighted with the gains\n> +\t * applied on the channels in AWB as the Rec. 601 luma.\n> +\t */\n> +\tdouble Y_sum = redSum * frameContext.awb.gains.red * .299 +\n> +\t\t greenSum * frameContext.awb.gains.green * .587 +\n> +\t\t blueSum * frameContext.awb.gains.blue * .114;\n> +\n> +\t/* Return the normalized relative luminance. */\n> +\treturn Y_sum / (grid.height * grid.width) / kMaxLuminance;\n> +}\n> +\n> /**\n> * \\brief Process IPU3 statistics, and run AGC operations\n> * \\param[in] context The shared IPA context\n> @@ -268,7 +338,29 @@ void Agc::computeExposure(IPAFrameContext &frameContext)\n> void Agc::process(IPAContext &context, const ipu3_uapi_stats_3a *stats)\n> {\n> \tmeasureBrightness(stats, context.configuration.grid.bdsGrid);\n> -\tcomputeExposure(context.frameContext);\n> +\n> +\tdouble currentYGain = 1.0;\n> +\tdouble targetY = kNormalizedLumaTarget;\n> +\n> +\t/*\n> +\t * Do this calculation a few times as brightness increase can be\n> +\t * non-linear when there are saturated regions.\n> +\t */\n> +\tfor (int i = 0; i < 8; i++) {\n> +\t\tdouble initialY = computeInitialY(context.frameContext,\n> +\t\t\t\t\t\t context.configuration.grid.bdsGrid,\n> +\t\t\t\t\t\t stats, currentYGain);\n> +\t\tdouble extra_gain = std::min(10.0, targetY / (initialY + .001));\n> +\n> +\t\tcurrentYGain *= extra_gain;\n> +\t\tLOG(IPU3Agc, Debug) << \"Initial Y \" << initialY\n> +\t\t\t\t << \" target \" << targetY\n> +\t\t\t\t << \" gives gain \" << currentYGain;\n> +\t\tif (extra_gain < 1.01)\n> +\t\t\tbreak;\n> +\t}\n> +\n\n\nFeels like this can be consolidated in a helper function and then called \nhere. Up to you,\n\nReviewed-by: Umang Jain \n\n> +\tcomputeExposure(context.frameContext, currentYGain);\n> \tframeCount_++;\n> }\n> \n> diff --git a/src/ipa/ipu3/algorithms/agc.h b/src/ipa/ipu3/algorithms/agc.h\n> index f0db25ee..79736283 100644\n> --- a/src/ipa/ipu3/algorithms/agc.h\n> +++ b/src/ipa/ipu3/algorithms/agc.h\n> @@ -34,7 +34,11 @@ private:\n> \tvoid measureBrightness(const ipu3_uapi_stats_3a *stats,\n> \t\t\t const ipu3_uapi_grid_config &grid);\n> \tvoid filterExposure();\n> -\tvoid computeExposure(IPAFrameContext &frameContext);\n> +\tvoid computeExposure(IPAFrameContext &frameContext, double currentYGain);\n> +\tdouble computeInitialY(IPAFrameContext &frameContext,\n> +\t\t\t const ipu3_uapi_grid_config &grid,\n> +\t\t\t const ipu3_uapi_stats_3a *stats,\n> +\t\t\t double currentYGain);\n> \n> \tuint64_t frameCount_;\n> \tuint64_t lastFrame_;","headers":{"Return-Path":"","X-Original-To":"parsemail@patchwork.libcamera.org","Delivered-To":"parsemail@patchwork.libcamera.org","Received":["from lancelot.ideasonboard.com (lancelot.ideasonboard.com\n\t[92.243.16.209])\n\tby patchwork.libcamera.org (Postfix) with ESMTPS id 7D2A5BF415\n\tfor ;\n\tMon, 15 Nov 2021 07:54:48 +0000 (UTC)","from lancelot.ideasonboard.com (localhost [IPv6:::1])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTP id AEA2F6036B;\n\tMon, 15 Nov 2021 08:54:47 +0100 (CET)","from perceval.ideasonboard.com (perceval.ideasonboard.com\n\t[IPv6:2001:4b98:dc2:55:216:3eff:fef7:d647])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id 3425C60120\n\tfor ;\n\tMon, 15 Nov 2021 08:54:46 +0100 (CET)","from [192.168.1.106] (unknown [103.251.226.33])\n\tby perceval.ideasonboard.com (Postfix) with ESMTPSA id 37A32E7;\n\tMon, 15 Nov 2021 08:54:45 +0100 (CET)"],"Authentication-Results":"lancelot.ideasonboard.com; dkim=pass (1024-bit key;\n\tunprotected) header.d=ideasonboard.com header.i=@ideasonboard.com\n\theader.b=\"rQ6VtWbE\"; dkim-atps=neutral","DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/simple; d=ideasonboard.com;\n\ts=mail; t=1636962885;\n\tbh=qa0EmB6Z/XoJkVUs+xDcUCwAn+hpozyMkSAUHy4DawQ=;\n\th=Subject:To:References:From:Date:In-Reply-To:From;\n\tb=rQ6VtWbENkZiAdhHc2aVR7kn8004TWGY/Veo1NcBMjKd9niHQgjk5/ln/P4dNm6YQ\n\tJDsGIdRPIq//0y6nqmueNnVKst1Hnif3gRZGcOSzAMdGjd4Z08P7Ye7z0Xxpvlg/yN\n\tEsQWYLTBvnkvYIbwbtXNpXCQahUCxLZFs40jKGL8=","To":"Jean-Michel Hautbois ,\n\tlibcamera-devel@lists.libcamera.org","References":"<20211113084947.21892-1-jeanmichel.hautbois@ideasonboard.com>\n\t<20211113084947.21892-8-jeanmichel.hautbois@ideasonboard.com>","From":"Umang Jain ","Message-ID":"<81a8f4c3-0532-ac25-2e1c-257451f82d8d@ideasonboard.com>","Date":"Mon, 15 Nov 2021 13:24:40 +0530","User-Agent":"Mozilla/5.0 (X11; Linux x86_64; rv:78.0) Gecko/20100101\n\tThunderbird/78.10.2","MIME-Version":"1.0","In-Reply-To":"<20211113084947.21892-8-jeanmichel.hautbois@ideasonboard.com>","Content-Type":"text/plain; charset=utf-8; format=flowed","Content-Transfer-Encoding":"8bit","Content-Language":"en-US","Subject":"Re: [libcamera-devel] [PATCH v5 07/14] ipa: ipu3: agc: Improve gain\n\tcalculation","X-BeenThere":"libcamera-devel@lists.libcamera.org","X-Mailman-Version":"2.1.29","Precedence":"list","List-Id":"","List-Unsubscribe":",\n\t","List-Archive":"","List-Post":"","List-Help":"","List-Subscribe":",\n\t","Errors-To":"libcamera-devel-bounces@lists.libcamera.org","Sender":"\"libcamera-devel\" "}}]