Patch Detail
Show a patch.
GET /api/patches/15380/?format=api
{ "id": 15380, "url": "https://patchwork.libcamera.org/api/patches/15380/?format=api", "web_url": "https://patchwork.libcamera.org/patch/15380/", "project": { "id": 1, "url": "https://patchwork.libcamera.org/api/projects/1/?format=api", "name": "libcamera", "link_name": "libcamera", "list_id": "libcamera_core", "list_email": "libcamera-devel@lists.libcamera.org", "web_url": "", "scm_url": "", "webscm_url": "" }, "msgid": "<20220224113347.80001-5-jeanmichel.hautbois@ideasonboard.com>", "date": "2022-02-24T11:33:46", "name": "[libcamera-devel,v3,4/5] ipa: rkisp1: agc: Add a histogram-based gain", "commit_ref": null, "pull_url": null, "state": "accepted", "archived": false, "hash": "a65f135cdb6f9fb3847cd6a181d0253689b05159", "submitter": { "id": 75, "url": "https://patchwork.libcamera.org/api/people/75/?format=api", "name": "Jean-Michel Hautbois", "email": "jeanmichel.hautbois@ideasonboard.com" }, "delegate": null, "mbox": "https://patchwork.libcamera.org/patch/15380/mbox/", "series": [ { "id": 2937, "url": "https://patchwork.libcamera.org/api/series/2937/?format=api", "web_url": "https://patchwork.libcamera.org/project/libcamera/list/?series=2937", "date": "2022-02-24T11:33:42", "name": "IPA RkISP1 awb and misc improvements", "version": 3, "mbox": "https://patchwork.libcamera.org/series/2937/mbox/" } ], "comments": "https://patchwork.libcamera.org/api/patches/15380/comments/", "check": "pending", "checks": "https://patchwork.libcamera.org/api/patches/15380/checks/", "tags": {}, "headers": { "Return-Path": "<libcamera-devel-bounces@lists.libcamera.org>", "X-Original-To": "parsemail@patchwork.libcamera.org", "Delivered-To": "parsemail@patchwork.libcamera.org", "Received": [ "from lancelot.ideasonboard.com (lancelot.ideasonboard.com\n\t[92.243.16.209])\n\tby patchwork.libcamera.org (Postfix) with ESMTPS id 83CDDC3261\n\tfor <parsemail@patchwork.libcamera.org>;\n\tThu, 24 Feb 2022 11:34:00 +0000 (UTC)", "from lancelot.ideasonboard.com (localhost [IPv6:::1])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTP id 473BB61168;\n\tThu, 24 Feb 2022 12:33:59 +0100 (CET)", "from perceval.ideasonboard.com (perceval.ideasonboard.com\n\t[IPv6:2001:4b98:dc2:55:216:3eff:fef7:d647])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id 9526261154\n\tfor <libcamera-devel@lists.libcamera.org>;\n\tThu, 24 Feb 2022 12:33:53 +0100 (CET)", "from tatooine.ideasonboard.com (unknown\n\t[IPv6:2a01:e0a:169:7140:ce74:6df2:4b76:b230])\n\tby perceval.ideasonboard.com (Postfix) with ESMTPSA id 4204B104C;\n\tThu, 24 Feb 2022 12:33:53 +0100 (CET)" ], "Authentication-Results": "lancelot.ideasonboard.com;\n\tdkim=fail reason=\"signature verification failed\" (1024-bit key;\n\tunprotected) header.d=ideasonboard.com header.i=@ideasonboard.com\n\theader.b=\"iJCjt3zc\"; dkim-atps=neutral", "DKIM-Signature": "v=1; a=rsa-sha256; c=relaxed/simple; d=ideasonboard.com;\n\ts=mail; t=1645702433;\n\tbh=wBoNw/EYK+s+bs9nw/IQgAzt+pdWyALtzBcldHYP5qg=;\n\th=From:To:Cc:Subject:Date:In-Reply-To:References:From;\n\tb=iJCjt3zcwzFlndWuthZwSG/xbkb4MRRkZsytWY9sjD/DbjHfxpoyWWkB4IKZ1F/1C\n\tYXhvRMZu3lU9XIfYZHpgKFqb/Ot9xW8U/XSwZ7wGKXTtPwMY6m0BUYBUJoHNo+7pRZ\n\tZUpkRoYG+Evem9+1p4Xkz/iZmcM1Md7vuXXYzhxw=", "From": "Jean-Michel Hautbois <jeanmichel.hautbois@ideasonboard.com>", "To": "libcamera-devel@lists.libcamera.org", "Date": "Thu, 24 Feb 2022 12:33:46 +0100", "Message-Id": "<20220224113347.80001-5-jeanmichel.hautbois@ideasonboard.com>", "X-Mailer": "git-send-email 2.32.0", "In-Reply-To": "<20220224113347.80001-1-jeanmichel.hautbois@ideasonboard.com>", "References": "<20220224113347.80001-1-jeanmichel.hautbois@ideasonboard.com>", "MIME-Version": "1.0", "Content-Transfer-Encoding": "8bit", "Subject": "[libcamera-devel] [PATCH v3 4/5] ipa: rkisp1: agc: Add a\n\thistogram-based gain", "X-BeenThere": "libcamera-devel@lists.libcamera.org", "X-Mailman-Version": "2.1.29", "Precedence": "list", "List-Id": "<libcamera-devel.lists.libcamera.org>", "List-Unsubscribe": "<https://lists.libcamera.org/options/libcamera-devel>,\n\t<mailto:libcamera-devel-request@lists.libcamera.org?subject=unsubscribe>", "List-Archive": "<https://lists.libcamera.org/pipermail/libcamera-devel/>", "List-Post": "<mailto:libcamera-devel@lists.libcamera.org>", "List-Help": "<mailto:libcamera-devel-request@lists.libcamera.org?subject=help>", "List-Subscribe": "<https://lists.libcamera.org/listinfo/libcamera-devel>,\n\t<mailto:libcamera-devel-request@lists.libcamera.org?subject=subscribe>", "Errors-To": "libcamera-devel-bounces@lists.libcamera.org", "Sender": "\"libcamera-devel\" <libcamera-devel-bounces@lists.libcamera.org>" }, "content": "As for the IPU3, we can estimate the histogram of the luminance. The\nRkISP1 can estimate multiple ones, the R, G and B ones, the Y only one\nand a combination of RGB. The one we are interested by in AGC is the Y\nhistogram.\n\nUse the hardware revision to determine the number of bins of the\nproduced histogram, and use it to populate a vector passed down to the\nlibipa::Histogram class.\n\nSigned-off-by: Jean-Michel Hautbois <jeanmichel.hautbois@ideasonboard.com>\nTested-by: Peter Griffin <peter.griffin@linaro.org>\n---\nv3:\n - Don't copy histogram values, pass it to the Histogram class\n directly\n - Change the date of the copyright\n---\n src/ipa/rkisp1/algorithms/agc.cpp | 94 ++++++++++++++++++++++++++-----\n src/ipa/rkisp1/algorithms/agc.h | 6 +-\n src/ipa/rkisp1/ipa_context.cpp | 3 +\n src/ipa/rkisp1/ipa_context.h | 1 +\n 4 files changed, 88 insertions(+), 16 deletions(-)", "diff": "diff --git a/src/ipa/rkisp1/algorithms/agc.cpp b/src/ipa/rkisp1/algorithms/agc.cpp\nindex dd97afc0..2e95ee05 100644\n--- a/src/ipa/rkisp1/algorithms/agc.cpp\n+++ b/src/ipa/rkisp1/algorithms/agc.cpp\n@@ -1,6 +1,6 @@\n /* SPDX-License-Identifier: LGPL-2.1-or-later */\n /*\n- * Copyright (C) 2021, Ideas On Board\n+ * Copyright (C) 2021-2022, Ideas On Board\n *\n * agc.cpp - AGC/AEC mean-based control algorithm\n */\n@@ -16,6 +16,8 @@\n \n #include <libcamera/ipa/core_ipa_interface.h>\n \n+#include \"libipa/histogram.h\"\n+\n /**\n * \\file agc.h\n */\n@@ -43,18 +45,22 @@ static constexpr utils::Duration kMaxShutterSpeed = 60ms;\n /* Number of frames to wait before calculating stats on minimum exposure */\n static constexpr uint32_t kNumStartupFrames = 10;\n \n+/* Target value to reach for the top 2% of the histogram */\n+static constexpr double kEvGainTarget = 0.5;\n+\n /*\n * Relative luminance target.\n *\n * It's a number that's chosen so that, when the camera points at a grey\n * target, the resulting image brightness is considered right.\n *\n- * \\todo Why is the value different between IPU3 and RkISP1 ?\n+ * \\todo The value is not the same for IPU3 and RkIsp1 because we don't have\n+ * sensor degamma yet.\n */\n static constexpr double kRelativeLuminanceTarget = 0.4;\n \n Agc::Agc()\n-\t: frameCount_(0), numCells_(0), filteredExposure_(0s)\n+\t: frameCount_(0), numCells_(0), numHistBins_(0), filteredExposure_(0s)\n {\n }\n \n@@ -65,8 +71,7 @@ Agc::Agc()\n *\n * \\return 0\n */\n-int Agc::configure(IPAContext &context,\n-\t\t [[maybe_unused]] const IPACameraSensorInfo &configInfo)\n+int Agc::configure(IPAContext &context, const IPACameraSensorInfo &configInfo)\n {\n \t/* Configure the default exposure and gain. */\n \tcontext.frameContext.agc.gain = std::max(context.configuration.agc.minAnalogueGain, kMinAnalogueGain);\n@@ -77,10 +82,22 @@ int Agc::configure(IPAContext &context,\n \t * - versions < V12 have RKISP1_CIF_ISP_AE_MEAN_MAX_V10 entries,\n \t * - versions >= V12 have RKISP1_CIF_ISP_AE_MEAN_MAX_V12 entries.\n \t */\n-\tif (context.configuration.hw.revision < RKISP1_V12)\n+\tif (context.configuration.hw.revision < RKISP1_V12) {\n \t\tnumCells_ = RKISP1_CIF_ISP_AE_MEAN_MAX_V10;\n-\telse\n+\t\tnumHistBins_ = RKISP1_CIF_ISP_HIST_BIN_N_MAX_V10;\n+\t} else {\n \t\tnumCells_ = RKISP1_CIF_ISP_AE_MEAN_MAX_V12;\n+\t\tnumHistBins_ = RKISP1_CIF_ISP_HIST_BIN_N_MAX_V12;\n+\t}\n+\n+\t/*\n+\t * Define the measurement window for AGC as a centered rectangle\n+\t * covering 3/4 of the image width and height.\n+\t */\n+\tcontext.configuration.agc.measureWindow.h_offs = configInfo.outputSize.width / 8;\n+\tcontext.configuration.agc.measureWindow.v_offs = configInfo.outputSize.height / 8;\n+\tcontext.configuration.agc.measureWindow.h_size = 3 * configInfo.outputSize.width / 4;\n+\tcontext.configuration.agc.measureWindow.v_size = 3 * configInfo.outputSize.height / 4;\n \n \t/* \\todo Use actual frame index by populating it in the frameContext. */\n \tframeCount_ = 0;\n@@ -126,8 +143,9 @@ utils::Duration Agc::filterExposure(utils::Duration exposureValue)\n * \\brief Estimate the new exposure and gain values\n * \\param[inout] frameContext The shared IPA frame Context\n * \\param[in] yGain The gain calculated on the current brightness level\n+ * \\param[in] iqMeanGain The gain calculated based on the relative luminance target\n */\n-void Agc::computeExposure(IPAContext &context, double yGain)\n+void Agc::computeExposure(IPAContext &context, double yGain, double iqMeanGain)\n {\n \tIPASessionConfiguration &configuration = context.configuration;\n \tIPAFrameContext &frameContext = context.frameContext;\n@@ -136,6 +154,9 @@ void Agc::computeExposure(IPAContext &context, double yGain)\n \tuint32_t exposure = frameContext.sensor.exposure;\n \tdouble analogueGain = frameContext.sensor.gain;\n \n+\t/* Use the highest of the two gain estimates. */\n+\tdouble evGain = std::max(yGain, iqMeanGain);\n+\n \tutils::Duration minShutterSpeed = configuration.agc.minShutterSpeed;\n \tutils::Duration maxShutterSpeed = std::min(configuration.agc.maxShutterSpeed,\n \t\t\t\t\t\t kMaxShutterSpeed);\n@@ -146,7 +167,7 @@ void Agc::computeExposure(IPAContext &context, double yGain)\n \t\t\t\t\t kMaxAnalogueGain);\n \n \t/* Consider within 1% of the target as correctly exposed. */\n-\tif (utils::abs_diff(yGain, 1.0) < 0.01)\n+\tif (utils::abs_diff(evGain, 1.0) < 0.01)\n \t\treturn;\n \n \t/* extracted from Rpi::Agc::computeTargetExposure. */\n@@ -163,13 +184,13 @@ void Agc::computeExposure(IPAContext &context, double yGain)\n \tLOG(RkISP1Agc, Debug) << \"Actual total exposure \" << currentShutter * analogueGain\n \t\t\t << \" Shutter speed \" << currentShutter\n \t\t\t << \" Gain \" << analogueGain\n-\t\t\t << \" Needed ev gain \" << yGain;\n+\t\t\t << \" Needed ev gain \" << evGain;\n \n \t/*\n \t * Calculate the current exposure value for the scene as the latest\n \t * exposure value applied multiplied by the new estimated gain.\n \t */\n-\tutils::Duration exposureValue = effectiveExposureValue * yGain;\n+\tutils::Duration exposureValue = effectiveExposureValue * evGain;\n \n \t/* Clamp the exposure value to the min and max authorized. */\n \tutils::Duration maxTotalExposure = maxShutterSpeed * maxAnalogueGain;\n@@ -240,6 +261,18 @@ double Agc::estimateLuminance(const rkisp1_cif_isp_ae_stat *ae,\n \treturn ySum / numCells_ / 255;\n }\n \n+/**\n+ * \\brief Estimate the mean value of the top 2% of the histogram\n+ * \\param[in] hist The histogram statistics computed by the ImgU\n+ * \\return The mean value of the top 2% of the histogram\n+ */\n+double Agc::measureBrightness(const rkisp1_cif_isp_hist_stat *hist) const\n+{\n+\tHistogram histogram{ Span<const uint32_t>(hist->hist_bins, numHistBins_) };\n+\t/* Estimate the quantile mean of the top 2% of the histogram. */\n+\treturn histogram.interQuantileMean(0.98, 1.0);\n+}\n+\n /**\n * \\brief Process RkISP1 statistics, and run AGC operations\n * \\param[in] context The shared IPA context\n@@ -254,6 +287,10 @@ void Agc::process(IPAContext &context, const rkisp1_stat_buffer *stats)\n \tASSERT(stats->meas_type & RKISP1_CIF_ISP_STAT_AUTOEXP);\n \n \tconst rkisp1_cif_isp_ae_stat *ae = ¶ms->ae;\n+\tconst rkisp1_cif_isp_hist_stat *hist = ¶ms->hist;\n+\n+\tdouble iqMean = measureBrightness(hist);\n+\tdouble iqMeanGain = kEvGainTarget * numHistBins_ / iqMean;\n \n \t/*\n \t * Estimate the gain needed to achieve a relative luminance target. To\n@@ -277,15 +314,44 @@ void Agc::process(IPAContext &context, const rkisp1_stat_buffer *stats)\n \t\t\tbreak;\n \t}\n \n-\tcomputeExposure(context, yGain);\n+\tcomputeExposure(context, yGain, iqMeanGain);\n \tframeCount_++;\n }\n \n-void Agc::prepare([[maybe_unused]] IPAContext &context,\n-\t\t rkisp1_params_cfg *params)\n+/**\n+ * \\copydoc libcamera::ipa::Algorithm::prepare\n+ */\n+void Agc::prepare(IPAContext &context, rkisp1_params_cfg *params)\n {\n+\tif (context.frameContext.frameCount > 0)\n+\t\treturn;\n+\n+\t/* Configure the measurement window. */\n+\tparams->meas.aec_config.meas_window = context.configuration.agc.measureWindow;\n+\t/* Use a continuous method for measure. */\n+\tparams->meas.aec_config.autostop = RKISP1_CIF_ISP_EXP_CTRL_AUTOSTOP_0;\n+\t/* Estimate Y as (R + G + B) x (85/256). */\n+\tparams->meas.aec_config.mode = RKISP1_CIF_ISP_EXP_MEASURING_MODE_1;\n+\n+\tparams->module_cfg_update |= RKISP1_CIF_ISP_MODULE_AEC;\n \tparams->module_ens |= RKISP1_CIF_ISP_MODULE_AEC;\n \tparams->module_en_update |= RKISP1_CIF_ISP_MODULE_AEC;\n+\n+\t/* Configure histogram. */\n+\tparams->meas.hst_config.meas_window = context.configuration.agc.measureWindow;\n+\t/* Produce the luminance histogram. */\n+\tparams->meas.hst_config.mode = RKISP1_CIF_ISP_HISTOGRAM_MODE_Y_HISTOGRAM;\n+\t/* Set an average weighted histogram. */\n+\tfor (unsigned int histBin = 0; histBin < numHistBins_; histBin++)\n+\t\tparams->meas.hst_config.hist_weight[histBin] = 1;\n+\t/* Step size can't be less than 3. */\n+\tparams->meas.hst_config.histogram_predivider = 4;\n+\n+\t/* Update the configuration for histogram. */\n+\tparams->module_cfg_update |= RKISP1_CIF_ISP_MODULE_HST;\n+\t/* Enable the histogram measure unit. */\n+\tparams->module_ens |= RKISP1_CIF_ISP_MODULE_HST;\n+\tparams->module_en_update |= RKISP1_CIF_ISP_MODULE_HST;\n }\n \n } /* namespace ipa::rkisp1::algorithms */\ndiff --git a/src/ipa/rkisp1/algorithms/agc.h b/src/ipa/rkisp1/algorithms/agc.h\nindex 942c9d7a..ce1adf27 100644\n--- a/src/ipa/rkisp1/algorithms/agc.h\n+++ b/src/ipa/rkisp1/algorithms/agc.h\n@@ -1,6 +1,6 @@\n /* SPDX-License-Identifier: LGPL-2.1-or-later */\n /*\n- * Copyright (C) 2021, Ideas On Board\n+ * Copyright (C) 2021-2022, Ideas On Board\n *\n * agc.h - RkISP1 AGC/AEC mean-based control algorithm\n */\n@@ -32,13 +32,15 @@ public:\n \tvoid process(IPAContext &context, const rkisp1_stat_buffer *stats) override;\n \n private:\n-\tvoid computeExposure(IPAContext &Context, double yGain);\n+\tvoid computeExposure(IPAContext &Context, double yGain, double iqMeanGain);\n \tutils::Duration filterExposure(utils::Duration exposureValue);\n \tdouble estimateLuminance(const rkisp1_cif_isp_ae_stat *ae, double gain);\n+\tdouble measureBrightness(const rkisp1_cif_isp_hist_stat *hist) const;\n \n \tuint64_t frameCount_;\n \n \tuint32_t numCells_;\n+\tuint32_t numHistBins_;\n \n \tutils::Duration filteredExposure_;\n };\ndiff --git a/src/ipa/rkisp1/ipa_context.cpp b/src/ipa/rkisp1/ipa_context.cpp\nindex 664f572f..790e159b 100644\n--- a/src/ipa/rkisp1/ipa_context.cpp\n+++ b/src/ipa/rkisp1/ipa_context.cpp\n@@ -71,6 +71,9 @@ namespace libcamera::ipa::rkisp1 {\n * \\var IPASessionConfiguration::agc.maxAnalogueGain\n * \\brief Maximum analogue gain supported with the configured sensor\n *\n+ * \\var IPASessionConfiguration::agc.measureWindow\n+ * \\brief AGC measure window\n+ *\n * \\var IPASessionConfiguration::hw\n * \\brief RkISP1-specific hardware information\n *\ndiff --git a/src/ipa/rkisp1/ipa_context.h b/src/ipa/rkisp1/ipa_context.h\nindex 212fa052..35e9b8e5 100644\n--- a/src/ipa/rkisp1/ipa_context.h\n+++ b/src/ipa/rkisp1/ipa_context.h\n@@ -22,6 +22,7 @@ struct IPASessionConfiguration {\n \t\tutils::Duration maxShutterSpeed;\n \t\tdouble minAnalogueGain;\n \t\tdouble maxAnalogueGain;\n+\t\tstruct rkisp1_cif_isp_window measureWindow;\n \t} agc;\n \n \tstruct {\n", "prefixes": [ "libcamera-devel", "v3", "4/5" ] }