Patch Detail
Show a patch.
GET /api/1.1/patches/15374/?format=api
{ "id": 15374, "url": "https://patchwork.libcamera.org/api/1.1/patches/15374/?format=api", "web_url": "https://patchwork.libcamera.org/patch/15374/", "project": { "id": 1, "url": "https://patchwork.libcamera.org/api/1.1/projects/1/?format=api", "name": "libcamera", "link_name": "libcamera", "list_id": "libcamera_core", "list_email": "libcamera-devel@lists.libcamera.org", "web_url": "", "scm_url": "", "webscm_url": "" }, "msgid": "<20220223104824.25723-4-jeanmichel.hautbois@ideasonboard.com>", "date": "2022-02-23T10:48:23", "name": "[libcamera-devel,v2,3/4] ipa: rkisp1: agc: Introduce histogram calculation", "commit_ref": null, "pull_url": null, "state": "superseded", "archived": false, "hash": "660d5e93f5365f1bbd949009d95f09caae0acb6f", "submitter": { "id": 75, "url": "https://patchwork.libcamera.org/api/1.1/people/75/?format=api", "name": "Jean-Michel Hautbois", "email": "jeanmichel.hautbois@ideasonboard.com" }, "delegate": null, "mbox": "https://patchwork.libcamera.org/patch/15374/mbox/", "series": [ { "id": 2936, "url": "https://patchwork.libcamera.org/api/1.1/series/2936/?format=api", "web_url": "https://patchwork.libcamera.org/project/libcamera/list/?series=2936", "date": "2022-02-23T10:48:21", "name": "IPA RkISP1 awb and misc improvements", "version": 2, "mbox": "https://patchwork.libcamera.org/series/2936/mbox/" } ], "comments": "https://patchwork.libcamera.org/api/patches/15374/comments/", "check": "pending", "checks": "https://patchwork.libcamera.org/api/patches/15374/checks/", "tags": {}, "headers": { "Return-Path": "<libcamera-devel-bounces@lists.libcamera.org>", "X-Original-To": "parsemail@patchwork.libcamera.org", "Delivered-To": "parsemail@patchwork.libcamera.org", "Received": [ "from lancelot.ideasonboard.com (lancelot.ideasonboard.com\n\t[92.243.16.209])\n\tby patchwork.libcamera.org (Postfix) with ESMTPS id 38D6CC3261\n\tfor <parsemail@patchwork.libcamera.org>;\n\tWed, 23 Feb 2022 10:48:34 +0000 (UTC)", "from lancelot.ideasonboard.com (localhost [IPv6:::1])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTP id 8C69E61183;\n\tWed, 23 Feb 2022 11:48:31 +0100 (CET)", "from perceval.ideasonboard.com (perceval.ideasonboard.com\n\t[213.167.242.64])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id CA03261101\n\tfor <libcamera-devel@lists.libcamera.org>;\n\tWed, 23 Feb 2022 11:48:28 +0100 (CET)", "from tatooine.ideasonboard.com (unknown\n\t[IPv6:2a01:e0a:169:7140:7ebc:9e38:c4df:c57])\n\tby perceval.ideasonboard.com (Postfix) with ESMTPSA id 8360A11BF;\n\tWed, 23 Feb 2022 11:48:28 +0100 (CET)" ], "Authentication-Results": "lancelot.ideasonboard.com;\n\tdkim=fail reason=\"signature verification failed\" (1024-bit key;\n\tunprotected) header.d=ideasonboard.com header.i=@ideasonboard.com\n\theader.b=\"oF18WPIb\"; dkim-atps=neutral", "DKIM-Signature": "v=1; a=rsa-sha256; c=relaxed/simple; d=ideasonboard.com;\n\ts=mail; t=1645613308;\n\tbh=jxU9hk1cYKxbO9o549ZcQQLvcDjxXbHlDRELGKmSvnc=;\n\th=From:To:Cc:Subject:Date:In-Reply-To:References:From;\n\tb=oF18WPIbM+mPX8EmtYXRhHzEPxjfQB63gQT890BiRW4FeOszW44dBmbjT9pN/qI7j\n\tJsB2/QW8fmseP12ldYA9X7ktVqAXlFx3AUB6Zr/8fpVB58wn4UcvxwRozKX6KvE/eh\n\t46zA1AWDTOl/16y3AM+nfvBe7aarWg/u/Ovb2/So=", "From": "Jean-Michel Hautbois <jeanmichel.hautbois@ideasonboard.com>", "To": "libcamera-devel@lists.libcamera.org", "Date": "Wed, 23 Feb 2022 11:48:23 +0100", "Message-Id": "<20220223104824.25723-4-jeanmichel.hautbois@ideasonboard.com>", "X-Mailer": "git-send-email 2.32.0", "In-Reply-To": "<20220223104824.25723-1-jeanmichel.hautbois@ideasonboard.com>", "References": "<20220223104824.25723-1-jeanmichel.hautbois@ideasonboard.com>", "MIME-Version": "1.0", "Content-Transfer-Encoding": "8bit", "Subject": "[libcamera-devel] [PATCH v2 3/4] ipa: rkisp1: agc: Introduce\n\thistogram calculation", "X-BeenThere": "libcamera-devel@lists.libcamera.org", "X-Mailman-Version": "2.1.29", "Precedence": "list", "List-Id": "<libcamera-devel.lists.libcamera.org>", "List-Unsubscribe": "<https://lists.libcamera.org/options/libcamera-devel>,\n\t<mailto:libcamera-devel-request@lists.libcamera.org?subject=unsubscribe>", "List-Archive": "<https://lists.libcamera.org/pipermail/libcamera-devel/>", "List-Post": "<mailto:libcamera-devel@lists.libcamera.org>", "List-Help": "<mailto:libcamera-devel-request@lists.libcamera.org?subject=help>", "List-Subscribe": "<https://lists.libcamera.org/listinfo/libcamera-devel>,\n\t<mailto:libcamera-devel-request@lists.libcamera.org?subject=subscribe>", "Errors-To": "libcamera-devel-bounces@lists.libcamera.org", "Sender": "\"libcamera-devel\" <libcamera-devel-bounces@lists.libcamera.org>" }, "content": "As for the IPU3, we can estimate the histogram of the luminance. The\nRkISP1 can estimate multiple ones, the R, G and B ones, the Y only one\nand a combination of RGB. The one we are interested by in AGC is the Y\nhistogram.\n\nUse the hardware revision to determine the number of bins of the\nproduced histogram, and use it to populate a vector passed down to the\nlibipa::Histogram class.\n\nAs the sensor deGamma and AWB are applied, we also need to get back to\nthe relative luminance value of 0.16, as for the IPU3.\n\nSigned-off-by: Jean-Michel Hautbois <jeanmichel.hautbois@ideasonboard.com>\nTested-by: Peter Griffin <peter.griffin@linaro.org>\n---\n src/ipa/rkisp1/algorithms/agc.cpp | 97 ++++++++++++++++++++++++++-----\n src/ipa/rkisp1/algorithms/agc.h | 4 +-\n src/ipa/rkisp1/ipa_context.cpp | 3 +\n src/ipa/rkisp1/ipa_context.h | 1 +\n 4 files changed, 89 insertions(+), 16 deletions(-)", "diff": "diff --git a/src/ipa/rkisp1/algorithms/agc.cpp b/src/ipa/rkisp1/algorithms/agc.cpp\nindex 50980835..0311ecf4 100644\n--- a/src/ipa/rkisp1/algorithms/agc.cpp\n+++ b/src/ipa/rkisp1/algorithms/agc.cpp\n@@ -16,6 +16,8 @@\n \n #include <libcamera/ipa/core_ipa_interface.h>\n \n+#include \"libipa/histogram.h\"\n+\n /**\n * \\file agc.h\n */\n@@ -43,6 +45,9 @@ static constexpr utils::Duration kMaxShutterSpeed = 60ms;\n /* Number of frames to wait before calculating stats on minimum exposure */\n static constexpr uint32_t kNumStartupFrames = 10;\n \n+/* Target value to reach for the top 2% of the histogram */\n+static constexpr double kEvGainTarget = 0.5;\n+\n /*\n * Relative luminance target.\n *\n@@ -51,10 +56,10 @@ static constexpr uint32_t kNumStartupFrames = 10;\n *\n * \\todo Why is the value different between IPU3 and RkISP1 ?\n */\n-static constexpr double kRelativeLuminanceTarget = 0.4;\n+static constexpr double kRelativeLuminanceTarget = 0.16;\n \n Agc::Agc()\n-\t: frameCount_(0), numCells_(0), filteredExposure_(0s)\n+\t: frameCount_(0), numCells_(0), numHistBins_(0), filteredExposure_(0s)\n {\n }\n \n@@ -65,8 +70,7 @@ Agc::Agc()\n *\n * \\return 0\n */\n-int Agc::configure(IPAContext &context,\n-\t\t [[maybe_unused]] const IPACameraSensorInfo &configInfo)\n+int Agc::configure(IPAContext &context, const IPACameraSensorInfo &configInfo)\n {\n \t/* Configure the default exposure and gain. */\n \tcontext.frameContext.agc.gain = std::max(context.configuration.agc.minAnalogueGain, kMinAnalogueGain);\n@@ -77,10 +81,19 @@ int Agc::configure(IPAContext &context,\n \t * - versions < V12 have RKISP1_CIF_ISP_AE_MEAN_MAX_V10 entries,\n \t * - versions >= V12 have RKISP1_CIF_ISP_AE_MEAN_MAX_V12 entries.\n \t */\n-\tif (context.configuration.hw.revision < RKISP1_V12)\n+\tif (context.configuration.hw.revision < RKISP1_V12) {\n \t\tnumCells_ = RKISP1_CIF_ISP_AE_MEAN_MAX_V10;\n-\telse\n+\t\tnumHistBins_ = RKISP1_CIF_ISP_HIST_BIN_N_MAX_V10;\n+\t} else {\n \t\tnumCells_ = RKISP1_CIF_ISP_AE_MEAN_MAX_V12;\n+\t\tnumHistBins_ = RKISP1_CIF_ISP_HIST_BIN_N_MAX_V12;\n+\t}\n+\n+\t/* Define the measurement window for AGC. */\n+\tcontext.configuration.agc.measureWindow.h_offs = configInfo.outputSize.width / 8;\n+\tcontext.configuration.agc.measureWindow.v_offs = configInfo.outputSize.height / 8;\n+\tcontext.configuration.agc.measureWindow.h_size = 3 * configInfo.outputSize.width / 4;\n+\tcontext.configuration.agc.measureWindow.v_size = 3 * configInfo.outputSize.height / 4;\n \n \treturn 0;\n }\n@@ -125,7 +138,7 @@ utils::Duration Agc::filterExposure(utils::Duration exposureValue)\n * \\param[inout] frameContext The shared IPA frame Context\n * \\param[in] yGain The gain calculated on the current brightness level\n */\n-void Agc::computeExposure(IPAContext &context, double yGain)\n+void Agc::computeExposure(IPAContext &context, double yGain, double iqMeanGain)\n {\n \tIPASessionConfiguration &configuration = context.configuration;\n \tIPAFrameContext &frameContext = context.frameContext;\n@@ -134,6 +147,9 @@ void Agc::computeExposure(IPAContext &context, double yGain)\n \tuint32_t exposure = frameContext.sensor.exposure;\n \tdouble analogueGain = frameContext.sensor.gain;\n \n+\t/* Use the highest of the two gain estimates. */\n+\tdouble evGain = std::max(yGain, iqMeanGain);\n+\n \tutils::Duration minShutterSpeed = configuration.agc.minShutterSpeed;\n \tutils::Duration maxShutterSpeed = std::min(configuration.agc.maxShutterSpeed,\n \t\t\t\t\t\t kMaxShutterSpeed);\n@@ -144,7 +160,7 @@ void Agc::computeExposure(IPAContext &context, double yGain)\n \t\t\t\t\t kMaxAnalogueGain);\n \n \t/* Consider within 1% of the target as correctly exposed. */\n-\tif (utils::abs_diff(yGain, 1.0) < 0.01)\n+\tif (utils::abs_diff(evGain, 1.0) < 0.01)\n \t\treturn;\n \n \t/* extracted from Rpi::Agc::computeTargetExposure. */\n@@ -161,13 +177,13 @@ void Agc::computeExposure(IPAContext &context, double yGain)\n \tLOG(RkISP1Agc, Debug) << \"Actual total exposure \" << currentShutter * analogueGain\n \t\t\t << \" Shutter speed \" << currentShutter\n \t\t\t << \" Gain \" << analogueGain\n-\t\t\t << \" Needed ev gain \" << yGain;\n+\t\t\t << \" Needed ev gain \" << evGain;\n \n \t/*\n \t * Calculate the current exposure value for the scene as the latest\n \t * exposure value applied multiplied by the new estimated gain.\n \t */\n-\tutils::Duration exposureValue = effectiveExposureValue * yGain;\n+\tutils::Duration exposureValue = effectiveExposureValue * evGain;\n \n \t/* Clamp the exposure value to the min and max authorized. */\n \tutils::Duration maxTotalExposure = maxShutterSpeed * maxAnalogueGain;\n@@ -238,6 +254,23 @@ double Agc::estimateLuminance(const rkisp1_cif_isp_ae_stat *ae,\n \treturn ySum / numCells_ / 255;\n }\n \n+/**\n+ * \\brief Estimate the mean value of the top 2% of the histogram\n+ * \\param[in] hist The histogram statistics computed by the ImgU\n+ * \\return The mean value of the top 2% of the histogram\n+ */\n+double Agc::measureBrightness(const rkisp1_cif_isp_hist_stat *hist) const\n+{\n+\tstd::vector<uint32_t> measHist(numHistBins_);\n+\n+\t/* Initialise the histogram array using the maximum available size */\n+\tfor (unsigned int histBin = 0; histBin < numHistBins_; histBin++)\n+\t\tmeasHist.push_back(hist->hist_bins[histBin]);\n+\n+\t/* Estimate the quantile mean of the top 2% of the histogram. */\n+\treturn Histogram(Span<uint32_t>(measHist)).interQuantileMean(0.98, 1.0);\n+}\n+\n /**\n * \\brief Process RkISP1 statistics, and run AGC operations\n * \\param[in] context The shared IPA context\n@@ -252,9 +285,13 @@ void Agc::process(IPAContext &context, const rkisp1_stat_buffer *stats)\n \tASSERT(stats->meas_type & RKISP1_CIF_ISP_STAT_AUTOEXP);\n \n \tconst rkisp1_cif_isp_ae_stat *ae = ¶ms->ae;\n+\tconst rkisp1_cif_isp_hist_stat *hist = ¶ms->hist;\n \n \tframeCount_ = context.frameContext.frameId;\n \n+\tdouble iqMean = measureBrightness(hist);\n+\tdouble iqMeanGain = kEvGainTarget * numHistBins_ / iqMean;\n+\n \t/*\n \t * Estimate the gain needed to achieve a relative luminance target. To\n \t * account for non-linearity caused by saturation, the value needs to be\n@@ -277,14 +314,44 @@ void Agc::process(IPAContext &context, const rkisp1_stat_buffer *stats)\n \t\t\tbreak;\n \t}\n \n-\tcomputeExposure(context, yGain);\n+\tcomputeExposure(context, yGain, iqMeanGain);\n }\n \n-void Agc::prepare([[maybe_unused]] IPAContext &context,\n-\t\t rkisp1_params_cfg *params)\n+/**\n+ * \\copydoc libcamera::ipa::Algorithm::prepare\n+ */\n+void Agc::prepare(IPAContext &context, rkisp1_params_cfg *params)\n {\n-\tparams->module_ens |= RKISP1_CIF_ISP_MODULE_AEC;\n-\tparams->module_en_update |= RKISP1_CIF_ISP_MODULE_AEC;\n+\t/* Configure the measurement window. */\n+\tparams->meas.aec_config.meas_window = context.configuration.agc.measureWindow;\n+\t/* Use a continuous methode for measure. */\n+\tparams->meas.aec_config.autostop = RKISP1_CIF_ISP_EXP_CTRL_AUTOSTOP_0;\n+\t/* Estimate Y as (R + G + B) x (85/256). */\n+\tparams->meas.aec_config.mode = RKISP1_CIF_ISP_EXP_MEASURING_MODE_1;\n+\n+\tif (context.frameContext.frameId == 0) {\n+\t\tparams->module_cfg_update |= RKISP1_CIF_ISP_MODULE_AEC;\n+\t\tparams->module_ens |= RKISP1_CIF_ISP_MODULE_AEC;\n+\t\tparams->module_en_update |= RKISP1_CIF_ISP_MODULE_AEC;\n+\t}\n+\n+\t/* Configure histogram. */\n+\tparams->meas.hst_config.meas_window = context.configuration.agc.measureWindow;\n+\t/* Produce the luminance histogram. */\n+\tparams->meas.hst_config.mode = RKISP1_CIF_ISP_HISTOGRAM_MODE_Y_HISTOGRAM;\n+\t/* Set an average weighted histogram. */\n+\tfor (unsigned int histBin = 0; histBin < numHistBins_; histBin++)\n+\t\tparams->meas.hst_config.hist_weight[histBin] = 1;\n+\t/* Step size can't be less than 3. */\n+\tparams->meas.hst_config.histogram_predivider = 4;\n+\n+\tif (context.frameContext.frameId == 0) {\n+\t\t/* Update the configuration for histogram. */\n+\t\tparams->module_cfg_update |= RKISP1_CIF_ISP_MODULE_HST;\n+\t\t/* Enable the histogram measure unit. */\n+\t\tparams->module_ens |= RKISP1_CIF_ISP_MODULE_HST;\n+\t\tparams->module_en_update |= RKISP1_CIF_ISP_MODULE_HST;\n+\t}\n }\n \n } /* namespace ipa::rkisp1::algorithms */\ndiff --git a/src/ipa/rkisp1/algorithms/agc.h b/src/ipa/rkisp1/algorithms/agc.h\nindex 942c9d7a..872776d0 100644\n--- a/src/ipa/rkisp1/algorithms/agc.h\n+++ b/src/ipa/rkisp1/algorithms/agc.h\n@@ -32,13 +32,15 @@ public:\n \tvoid process(IPAContext &context, const rkisp1_stat_buffer *stats) override;\n \n private:\n-\tvoid computeExposure(IPAContext &Context, double yGain);\n+\tvoid computeExposure(IPAContext &Context, double yGain, double iqMeanGain);\n \tutils::Duration filterExposure(utils::Duration exposureValue);\n \tdouble estimateLuminance(const rkisp1_cif_isp_ae_stat *ae, double gain);\n+\tdouble measureBrightness(const rkisp1_cif_isp_hist_stat *hist) const;\n \n \tuint64_t frameCount_;\n \n \tuint32_t numCells_;\n+\tuint32_t numHistBins_;\n \n \tutils::Duration filteredExposure_;\n };\ndiff --git a/src/ipa/rkisp1/ipa_context.cpp b/src/ipa/rkisp1/ipa_context.cpp\nindex 992c9225..39acef47 100644\n--- a/src/ipa/rkisp1/ipa_context.cpp\n+++ b/src/ipa/rkisp1/ipa_context.cpp\n@@ -71,6 +71,9 @@ namespace libcamera::ipa::rkisp1 {\n * \\var IPASessionConfiguration::agc.maxAnalogueGain\n * \\brief Maximum analogue gain supported with the configured sensor\n *\n+ * \\var IPASessionConfiguration::agc.measureWindow\n+ * \\brief AGC measure window\n+ *\n * \\var IPASessionConfiguration::hw\n * \\brief RkISP1-specific hardware information\n *\ndiff --git a/src/ipa/rkisp1/ipa_context.h b/src/ipa/rkisp1/ipa_context.h\nindex c447369f..9ac3b40c 100644\n--- a/src/ipa/rkisp1/ipa_context.h\n+++ b/src/ipa/rkisp1/ipa_context.h\n@@ -22,6 +22,7 @@ struct IPASessionConfiguration {\n \t\tutils::Duration maxShutterSpeed;\n \t\tdouble minAnalogueGain;\n \t\tdouble maxAnalogueGain;\n+\t\tstruct rkisp1_cif_isp_window measureWindow;\n \t} agc;\n \n \tstruct {\n", "prefixes": [ "libcamera-devel", "v2", "3/4" ] }