[{"id":20718,"web_url":"https://patchwork.libcamera.org/comment/20718/","msgid":"<163638320648.275423.2754813441588763685@Monstersaurus>","date":"2021-11-08T14:53:26","subject":"Re: [libcamera-devel] [PATCH 07/22] ipa: ipu3: agc: Improve gain\n\tcalculation","submitter":{"id":4,"url":"https://patchwork.libcamera.org/api/people/4/","name":"Kieran Bingham","email":"kieran.bingham@ideasonboard.com"},"content":"Quoting Jean-Michel Hautbois (2021-11-08 13:13:35)\n> When an image is partially saturated, its brightness is not increasing\n> linearly when the shutter time or gain increases. It is a big issue with\n> a backlight as the algorithm is fading to darkness right now.\n> \n> Introduce a function to estimate the brightness of the frame, based on\n> the current exposure/gain and loop on it several times to estimate it\n> again and approach the non linear function.\n> \n> Inspired-by: 7de5506c30b3 (\"libcamera: src: ipa: raspberrypi: agc: Improve gain update calculation for partly saturated images\")\n> Signed-off-by: Jean-Michel Hautbois \n> ---\n> src/ipa/ipu3/algorithms/agc.cpp | 77 ++++++++++++++++++++++++++++++++-\n> src/ipa/ipu3/algorithms/agc.h | 6 ++-\n> 2 files changed, 80 insertions(+), 3 deletions(-)\n> \n> diff --git a/src/ipa/ipu3/algorithms/agc.cpp b/src/ipa/ipu3/algorithms/agc.cpp\n> index 119a7938..e4048d40 100644\n> --- a/src/ipa/ipu3/algorithms/agc.cpp\n> +++ b/src/ipa/ipu3/algorithms/agc.cpp\n> @@ -186,8 +186,9 @@ void Agc::filterExposure()\n> * \\brief Estimate the new exposure and gain values\n> * \\param[inout] exposure The exposure value reference as a number of lines\n> * \\param[inout] gain The gain reference to be updated\n> + * \\param[in] currentYGain The gain calculated on the current brightness level\n> */\n> -void Agc::computeExposure(uint32_t &exposure, double &analogueGain)\n> +void Agc::computeExposure(uint32_t &exposure, double &analogueGain, double currentYGain)\n> {\n> /* Estimate the gain needed to have the proportion wanted */\n> double evGain = kEvGainTarget * knumHistogramBins / iqMean_;\n> @@ -206,6 +207,14 @@ void Agc::computeExposure(uint32_t &exposure, double &analogueGain)\n> << \" Gain \" << analogueGain\n> << \" Needed ev gain \" << evGain;\n> \n> + if (evGain < currentYGain)\n> + evGain = currentYGain;\n> +\n> + if (std::abs(evGain - 1.0) < 0.01) {\n> + LOG(IPU3Agc, Debug) << \"We are well exposed (iqMean = \"\n> + << iqMean_ << \")\";\n> + }\n\n{ } are not needed for a single line statement.\n\n\n> +\n> /*\n> * Calculate the current exposure value for the scene as the latest\n> * exposure value applied multiplied by the new estimated gain.\n> @@ -253,6 +262,48 @@ void Agc::computeExposure(uint32_t &exposure, double &analogueGain)\n> prevExposureValue_ = shutterTime * analogueGain;\n> }\n> \n> +/**\n> + * \\brief Estimate the average brightness of the frame\n> + * \\param[in] context The shared IPA context\n> + * \\param[in] grid The grid used to store the statistics in the IPU3\n> + * \\param[in] stats The IPU3 statistics and ISP results\n> + * \\param[in] currentYGain The gain calculated on the current brightness level\n> + */\n> +double Agc::computeInitialY(IPAFrameContext &frameContext,\n> + const ipu3_uapi_grid_config &grid,\n> + const ipu3_uapi_stats_3a *stats,\n> + double currentYGain)\n> +{\n> + double redSum = 0, greenSum = 0, blueSum = 0;\n> +\n> + for (unsigned int cellY = 0; cellY < grid.height; cellY++) {\n> + for (unsigned int cellX = 0; cellX < grid.width; cellX++) {\n> + uint32_t cellPosition = cellY * stride_ + cellX;\n> +\n> + const ipu3_uapi_awb_set_item *cell =\n> + reinterpret_cast(\n> + &stats->awb_raw_buffer.meta_data[cellPosition]\n> + );\n> +\n> + redSum += cell->R_avg * currentYGain;\n> + greenSum += (cell->Gr_avg + cell->Gb_avg) / 2 * currentYGain;\n> + blueSum += cell->B_avg * currentYGain;\n> + }\n> + }\n> +\n> + /*\n> + * Estimate the sum of the brightness values, weighted with the gains\n> + * applied on the channels in AWB.\n> + */\n> + double Y_sum = redSum * frameContext.awb.gains.red * .299 +\n> + greenSum * frameContext.awb.gains.green * .587 +\n> + blueSum * frameContext.awb.gains.blue * .114;\n\nIs this some RGB to HSL/YUV calcuation? Can we make it clearer somehow?\n\nIt currently looks like 'magic'.\n\n> +\n> + /* And return the average brightness */\n> + return Y_sum / (grid.height * grid.width);\n> +}\n> +\n> +\n> /**\n> * \\brief Process IPU3 statistics, and run AGC operations\n> * \\param[in] context The shared IPA context\n> @@ -267,7 +318,29 @@ void Agc::process(IPAContext &context, const ipu3_uapi_stats_3a *stats)\n> uint32_t &exposure = context.frameContext.agc.exposure;\n> double &analogueGain = context.frameContext.agc.gain;\n> measureBrightness(stats, context.configuration.grid.bdsGrid);\n> - computeExposure(exposure, analogueGain);\n> +\n> + double currentYGain = 1.0;\n> + /* \\todo: the target Y needs to be grabbed from a configuration */\n\nCan we quantify what '60' means at all yet? Is it 60%? 60/255? something\nelse? Does it have units or a meaning at all?\n\n> + double targetY = 60;\n\nline break here.\n\n> + /*\n> + * Do this calculation a few times as brightness increase can be\n> + * non-linear when there are saturated regions.\n> + */\n> + for (int i = 0; i < 8; i++) {\n> + double initialY = computeInitialY(context.frameContext,\n> + context.configuration.grid.bdsGrid,\n> + stats, currentYGain);\n\ndoes this value change? Should it be outside of the loop - that's a lot\nof iterations over every cell.\n\nHrm ... in fact, it looks like it is dependant upon the currentYGain\nvalue ... so maybe this is required.\n\n\n> + double extra_gain = std::min(10.0, targetY / (initialY + .001));\n> +\n> + currentYGain *= extra_gain;\n> + LOG(IPU3Agc, Debug) << \"Initial Y \" << initialY\n> + << \" target \" << targetY\n> + << \" gives gain \" << currentYGain;\n> + if (extra_gain < 1.01)\n> + break;\n> + }\n> +\n> + computeExposure(exposure, analogueGain, currentYGain);\n> frameCount_++;\n> }\n> \n> diff --git a/src/ipa/ipu3/algorithms/agc.h b/src/ipa/ipu3/algorithms/agc.h\n> index 69e0b831..0a9152a9 100644\n> --- a/src/ipa/ipu3/algorithms/agc.h\n> +++ b/src/ipa/ipu3/algorithms/agc.h\n> @@ -34,7 +34,11 @@ private:\n> void measureBrightness(const ipu3_uapi_stats_3a *stats,\n> const ipu3_uapi_grid_config &grid);\n> void filterExposure();\n> - void computeExposure(uint32_t &exposure, double &gain);\n> + void computeExposure(uint32_t &exposure, double &gain, double currentYGain);\n> + double computeInitialY(IPAFrameContext &frameContext,\n> + const ipu3_uapi_grid_config &grid,\n> + const ipu3_uapi_stats_3a *stats,\n> + double currentYGain);\n> \n> uint64_t frameCount_;\n> uint64_t lastFrame_;\n> -- \n> 2.32.0\n>","headers":{"Return-Path":"","X-Original-To":"parsemail@patchwork.libcamera.org","Delivered-To":"parsemail@patchwork.libcamera.org","Received":["from lancelot.ideasonboard.com (lancelot.ideasonboard.com\n\t[92.243.16.209])\n\tby patchwork.libcamera.org (Postfix) with ESMTPS id 0F95ABF415\n\tfor ;\n\tMon, 8 Nov 2021 14:53:31 +0000 (UTC)","from lancelot.ideasonboard.com (localhost [IPv6:::1])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTP id 546AD6034A;\n\tMon, 8 Nov 2021 15:53:30 +0100 (CET)","from perceval.ideasonboard.com (perceval.ideasonboard.com\n\t[IPv6:2001:4b98:dc2:55:216:3eff:fef7:d647])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id C736C6032C\n\tfor ;\n\tMon, 8 Nov 2021 15:53:28 +0100 (CET)","from pendragon.ideasonboard.com\n\t(cpc89244-aztw30-2-0-cust3082.18-1.cable.virginm.net [86.31.172.11])\n\tby perceval.ideasonboard.com (Postfix) with ESMTPSA id 670519FF;\n\tMon, 8 Nov 2021 15:53:28 +0100 (CET)"],"Authentication-Results":"lancelot.ideasonboard.com; dkim=pass (1024-bit key;\n\tunprotected) header.d=ideasonboard.com header.i=@ideasonboard.com\n\theader.b=\"hHtfif9O\"; dkim-atps=neutral","DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/simple; d=ideasonboard.com;\n\ts=mail; t=1636383208;\n\tbh=iy5IvqrD6/4VAo7WZcriWVApargaa49xgMt93BJh1cU=;\n\th=In-Reply-To:References:Subject:From:To:Date:From;\n\tb=hHtfif9OszmSfvWFASBY3QMaBspFDf0fwPEty4VtBlB+i1mO+YGumI9Qam8sCnShx\n\t6VpuMGUtvLonVozZWUfnrfqS8pZibv8pWVMQ9N3MpuOX7lOxIuK3kjDpZSrbmBW9SS\n\t+K++uszloGRgvUREWaRy2F7GV8gnGP7tQ2ke6rXE=","Content-Type":"text/plain; charset=\"utf-8\"","MIME-Version":"1.0","Content-Transfer-Encoding":"quoted-printable","In-Reply-To":"<20211108131350.130665-8-jeanmichel.hautbois@ideasonboard.com>","References":"<20211108131350.130665-1-jeanmichel.hautbois@ideasonboard.com>\n\t<20211108131350.130665-8-jeanmichel.hautbois@ideasonboard.com>","From":"Kieran Bingham ","To":"Jean-Michel Hautbois ,\n\tlibcamera-devel@lists.libcamera.org","Date":"Mon, 08 Nov 2021 14:53:26 +0000","Message-ID":"<163638320648.275423.2754813441588763685@Monstersaurus>","User-Agent":"alot/0.9.1","Subject":"Re: [libcamera-devel] [PATCH 07/22] ipa: ipu3: agc: Improve gain\n\tcalculation","X-BeenThere":"libcamera-devel@lists.libcamera.org","X-Mailman-Version":"2.1.29","Precedence":"list","List-Id":"","List-Unsubscribe":",\n\t","List-Archive":"","List-Post":"","List-Help":"","List-Subscribe":",\n\t","Errors-To":"libcamera-devel-bounces@lists.libcamera.org","Sender":"\"libcamera-devel\" "}}]