[{"id":18961,"web_url":"https://patchwork.libcamera.org/comment/18961/","msgid":"","date":"2021-08-20T01:08:46","subject":"Re: [libcamera-devel] [PATCH v4 6/9] ipa: ipu3: convert AWB to the\n\tnew algorithm interface","submitter":{"id":2,"url":"https://patchwork.libcamera.org/api/people/2/","name":"Laurent Pinchart","email":"laurent.pinchart@ideasonboard.com"},"content":"Hi Jean-Michel,\n\nThank you for the patch.\n\nOn Thu, Aug 19, 2021 at 04:57:56PM +0200, Jean-Michel Hautbois wrote:\n> When the stats are received, pass them with the context to the existing\n> AWB algorithm. IPAFrameContext now has a new structure to store the\n> gains calculated by the AWB algorithm.\n> \n> When an EventFillParams event is received, call prepare() and set the new\n> gains accordingly in the params structure.\n> There is no more a need for the IPU3Awb::initialise() function, as the\n> params are always set in prepare().\n> \n> Signed-off-by: Jean-Michel Hautbois \n> Reviewed-by: Kieran Bingham \n\nReviewed-by: Laurent Pinchart \n\n> ---\n> src/ipa/ipu3/ipa_context.h | 8 +++\n> src/ipa/ipu3/ipu3.cpp | 22 ++++++--\n> src/ipa/ipu3/ipu3_awb.cpp | 107 ++++++++++++++++++-------------------\n> src/ipa/ipu3/ipu3_awb.h | 6 +--\n> 4 files changed, 82 insertions(+), 61 deletions(-)\n> \n> diff --git a/src/ipa/ipu3/ipa_context.h b/src/ipa/ipu3/ipa_context.h\n> index 4b12f129..24dd8bf7 100644\n> --- a/src/ipa/ipu3/ipa_context.h\n> +++ b/src/ipa/ipu3/ipa_context.h\n> @@ -24,6 +24,14 @@ struct IPASessionConfiguration {\n> };\n> \n> struct IPAFrameContext {\n> +\tstruct {\n> +\t\tstruct {\n> +\t\t\tdouble red;\n> +\t\t\tdouble green;\n> +\t\t\tdouble blue;\n> +\t\t} gains;\n> +\t} awb;\n> +\n> \tstruct {\n> \t\tstruct ipu3_uapi_gamma_corr_lut gammaCorrection;\n> \t} toneMapping;\n> diff --git a/src/ipa/ipu3/ipu3.cpp b/src/ipa/ipu3/ipu3.cpp\n> index edc2d911..cede897e 100644\n> --- a/src/ipa/ipu3/ipu3.cpp\n> +++ b/src/ipa/ipu3/ipu3.cpp\n> @@ -95,6 +95,23 @@ static constexpr uint32_t kMaxCellHeightPerSet = 56;\n> * \\brief BDS output size configured by the pipeline handler\n> */\n> \n> +/**\n> + * \\struct IPAFrameContext::awb\n> + * \\brief Context for the Automatic White Balance algorithm\n> + *\n> + * \\struct IPAFrameContext::awb::gains\n> + * \\brief White balance gains\n> + *\n> + * \\var IPAFrameContext::awb::gains::red\n> + * \\brief White balance gain for R channel\n> + *\n> + * \\var IPAFrameContext::awb::gains::green\n> + * \\brief White balance gain for G channel\n> + *\n> + * \\var IPAFrameContext::awb::gains::blue\n> + * \\brief White balance gain for B channel\n> + */\n> +\n> /**\n> * \\struct IPAFrameContext::toneMapping\n> * \\brief Context for ToneMapping and Gamma control\n> @@ -357,7 +374,6 @@ int IPAIPU3::configure(const IPAConfigInfo &configInfo)\n> \t}\n> \n> \tawbAlgo_ = std::make_unique();\n> -\tawbAlgo_->initialise(params_, context_.configuration.grid.bdsOutputSize, context_.configuration.grid.bdsGrid);\n> \tagcAlgo_ = std::make_unique();\n> \tagcAlgo_->initialise(context_.configuration.grid.bdsGrid, sensorInfo_);\n> \n> @@ -437,7 +453,7 @@ void IPAIPU3::fillParams(unsigned int frame, ipu3_uapi_params *params)\n> \t\talgo->prepare(context_, params_);\n> \n> \tif (agcAlgo_->updateControls())\n> -\t\tawbAlgo_->updateWbParameters(params_);\n> +\t\tawbAlgo_->prepare(context_, params_);\n> \n> \t*params = params_;\n> \n> @@ -460,7 +476,7 @@ void IPAIPU3::parseStatistics(unsigned int frame,\n> \tagcAlgo_->process(stats, exposure_, gain);\n> \tgain_ = camHelper_->gainCode(gain);\n> \n> -\tawbAlgo_->calculateWBGains(stats);\n> +\tawbAlgo_->process(context_, stats);\n> \n> \tif (agcAlgo_->updateControls())\n> \t\tsetControls(frame);\n> diff --git a/src/ipa/ipu3/ipu3_awb.cpp b/src/ipa/ipu3/ipu3_awb.cpp\n> index c2d9e0c1..490bf514 100644\n> --- a/src/ipa/ipu3/ipu3_awb.cpp\n> +++ b/src/ipa/ipu3/ipu3_awb.cpp\n> @@ -107,25 +107,6 @@ static const struct ipu3_uapi_bnr_static_config imguCssBnrDefaults = {\n> \t.opt_center_sqr = { 419904, 133956 },\n> };\n> \n> -/* Default settings for Auto White Balance replicated from the Kernel*/\n> -static const struct ipu3_uapi_awb_config_s imguCssAwbDefaults = {\n> -\t.rgbs_thr_gr = 8191,\n> -\t.rgbs_thr_r = 8191,\n> -\t.rgbs_thr_gb = 8191,\n> -\t.rgbs_thr_b = 8191 | IPU3_UAPI_AWB_RGBS_THR_B_EN | IPU3_UAPI_AWB_RGBS_THR_B_INCL_SAT,\n> -\t.grid = {\n> -\t\t.width = 160,\n> -\t\t.height = 36,\n> -\t\t.block_width_log2 = 3,\n> -\t\t.block_height_log2 = 4,\n> -\t\t.height_per_slice = 1, /* Overridden by kernel. */\n> -\t\t.x_start = 0,\n> -\t\t.y_start = 0,\n> -\t\t.x_end = 0,\n> -\t\t.y_end = 0,\n> -\t},\n> -};\n> -\n> /* Default color correction matrix defined as an identity matrix */\n> static const struct ipu3_uapi_ccm_mat_config imguCssCcmDefault = {\n> \t8191, 0, 0, 0,\n> @@ -140,41 +121,12 @@ IPU3Awb::IPU3Awb()\n> \tasyncResults_.greenGain = 1.0;\n> \tasyncResults_.redGain = 1.0;\n> \tasyncResults_.temperatureK = 4500;\n> -}\n> -\n> -IPU3Awb::~IPU3Awb()\n> -{\n> -}\n> -\n> -void IPU3Awb::initialise(ipu3_uapi_params ¶ms, const Size &bdsOutputSize, struct ipu3_uapi_grid_config &bdsGrid)\n> -{\n> -\tparams.use.acc_awb = 1;\n> -\tparams.acc_param.awb.config = imguCssAwbDefaults;\n> -\n> -\tawbGrid_ = bdsGrid;\n> -\tparams.acc_param.awb.config.grid = awbGrid_;\n> -\n> -\tparams.use.acc_bnr = 1;\n> -\tparams.acc_param.bnr = imguCssBnrDefaults;\n> -\t/**\n> -\t * Optical center is column (respectively row) startminus X (respectively Y) center.\n> -\t * For the moment use BDS as a first approximation, but it should\n> -\t * be calculated based on Shading (SHD) parameters.\n> -\t */\n> -\tparams.acc_param.bnr.column_size = bdsOutputSize.width;\n> -\tparams.acc_param.bnr.opt_center.x_reset = awbGrid_.x_start - (bdsOutputSize.width / 2);\n> -\tparams.acc_param.bnr.opt_center.y_reset = awbGrid_.y_start - (bdsOutputSize.height / 2);\n> -\tparams.acc_param.bnr.opt_center_sqr.x_sqr_reset = params.acc_param.bnr.opt_center.x_reset\n> -\t\t\t\t\t\t\t* params.acc_param.bnr.opt_center.x_reset;\n> -\tparams.acc_param.bnr.opt_center_sqr.y_sqr_reset = params.acc_param.bnr.opt_center.y_reset\n> -\t\t\t\t\t\t\t* params.acc_param.bnr.opt_center.y_reset;\n> -\n> -\tparams.use.acc_ccm = 1;\n> -\tparams.acc_param.ccm = imguCssCcmDefault;\n> \n> \tzones_.reserve(kAwbStatsSizeX * kAwbStatsSizeY);\n> }\n> \n> +IPU3Awb::~IPU3Awb() = default;\n> +\n> /**\n> * The function estimates the correlated color temperature using\n> * from RGB color space input.\n> @@ -321,22 +273,67 @@ void IPU3Awb::calculateWBGains(const ipu3_uapi_stats_3a *stats)\n> \t}\n> }\n> \n> -void IPU3Awb::updateWbParameters(ipu3_uapi_params ¶ms)\n> +void IPU3Awb::process(IPAContext &context, const ipu3_uapi_stats_3a *&stats)\n> +{\n> +\tcalculateWBGains(stats);\n> +\n> +\t/*\n> +\t * Gains are only recalculated if enough zones were detected.\n> +\t * The results are cached, so if no results were calculated, we set the\n> +\t * cached values from asyncResults_ here.\n> +\t */\n> +\tcontext.frameContext.awb.gains.blue = asyncResults_.blueGain;\n> +\tcontext.frameContext.awb.gains.green = asyncResults_.greenGain;\n> +\tcontext.frameContext.awb.gains.red = asyncResults_.redGain;\n> +}\n> +\n> +void IPU3Awb::prepare(IPAContext &context, ipu3_uapi_params ¶ms)\n> {\n> +\tparams.acc_param.awb.config.rgbs_thr_gr = 8191;\n> +\tparams.acc_param.awb.config.rgbs_thr_r = 8191;\n> +\tparams.acc_param.awb.config.rgbs_thr_gb = 8191;\n> +\tparams.acc_param.awb.config.rgbs_thr_b = IPU3_UAPI_AWB_RGBS_THR_B_INCL_SAT\n> +\t\t\t\t\t | IPU3_UAPI_AWB_RGBS_THR_B_EN\n> +\t\t\t\t\t | 8191;\n> +\n> +\tawbGrid_ = context.configuration.grid.bdsGrid;\n> +\n> +\tparams.acc_param.awb.config.grid = context.configuration.grid.bdsGrid;\n> +\n> +\t/*\n> +\t * Optical center is column start (respectively row start) of the\n> +\t * region of interest minus its X center (respectively Y center).\n> +\t *\n> +\t * For the moment use BDS as a first approximation, but it should\n> +\t * be calculated based on Shading (SHD) parameters.\n> +\t */\n> +\tparams.acc_param.bnr = imguCssBnrDefaults;\n> +\tSize &bdsOutputSize = context.configuration.grid.bdsOutputSize;\n> +\tparams.acc_param.bnr.column_size = bdsOutputSize.width;\n> +\tparams.acc_param.bnr.opt_center.x_reset = awbGrid_.x_start - (bdsOutputSize.width / 2);\n> +\tparams.acc_param.bnr.opt_center.y_reset = awbGrid_.y_start - (bdsOutputSize.height / 2);\n> +\tparams.acc_param.bnr.opt_center_sqr.x_sqr_reset = params.acc_param.bnr.opt_center.x_reset\n> +\t\t\t\t\t\t\t* params.acc_param.bnr.opt_center.x_reset;\n> +\tparams.acc_param.bnr.opt_center_sqr.y_sqr_reset = params.acc_param.bnr.opt_center.y_reset\n> +\t\t\t\t\t\t\t* params.acc_param.bnr.opt_center.y_reset;\n> \t/*\n> \t * Green gains should not be touched and considered 1.\n> \t * Default is 16, so do not change it at all.\n> \t * 4096 is the value for a gain of 1.0\n> \t */\n> -\tparams.acc_param.bnr.wb_gains.gr = 16;\n> -\tparams.acc_param.bnr.wb_gains.r = 4096 * asyncResults_.redGain;\n> -\tparams.acc_param.bnr.wb_gains.b = 4096 * asyncResults_.blueGain;\n> -\tparams.acc_param.bnr.wb_gains.gb = 16;\n> +\tparams.acc_param.bnr.wb_gains.gr = 16 * context.frameContext.awb.gains.green;\n> +\tparams.acc_param.bnr.wb_gains.r = 4096 * context.frameContext.awb.gains.red;\n> +\tparams.acc_param.bnr.wb_gains.b = 4096 * context.frameContext.awb.gains.blue;\n> +\tparams.acc_param.bnr.wb_gains.gb = 16 * context.frameContext.awb.gains.green;\n> \n> \tLOG(IPU3Awb, Debug) << \"Color temperature estimated: \" << asyncResults_.temperatureK;\n> \n> \t/* The CCM matrix may change when color temperature will be used */\n> \tparams.acc_param.ccm = imguCssCcmDefault;\n> +\n> +\tparams.use.acc_awb = 1;\n> +\tparams.use.acc_bnr = 1;\n> +\tparams.use.acc_ccm = 1;\n> }\n> \n> } /* namespace ipa::ipu3 */\n> diff --git a/src/ipa/ipu3/ipu3_awb.h b/src/ipa/ipu3/ipu3_awb.h\n> index eeb2920b..81fe03b0 100644\n> --- a/src/ipa/ipu3/ipu3_awb.h\n> +++ b/src/ipa/ipu3/ipu3_awb.h\n> @@ -29,9 +29,8 @@ public:\n> \tIPU3Awb();\n> \t~IPU3Awb();\n> \n> -\tvoid initialise(ipu3_uapi_params ¶ms, const Size &bdsOutputSize, struct ipu3_uapi_grid_config &bdsGrid);\n> -\tvoid calculateWBGains(const ipu3_uapi_stats_3a *stats);\n> -\tvoid updateWbParameters(ipu3_uapi_params ¶ms);\n> +\tvoid prepare(IPAContext &context, ipu3_uapi_params ¶ms) override;\n> +\tvoid process(IPAContext &context, const ipu3_uapi_stats_3a *&stats) override;\n> \n> \tstruct Ipu3AwbCell {\n> \t\tunsigned char greenRedAvg;\n> @@ -72,6 +71,7 @@ public:\n> \t};\n> \n> private:\n> +\tvoid calculateWBGains(const ipu3_uapi_stats_3a *stats);\n> \tvoid generateZones(std::vector &zones);\n> \tvoid generateAwbStats(const ipu3_uapi_stats_3a *stats);\n> \tvoid clearAwbStats();","headers":{"Return-Path":"","X-Original-To":"parsemail@patchwork.libcamera.org","Delivered-To":"parsemail@patchwork.libcamera.org","Received":["from lancelot.ideasonboard.com (lancelot.ideasonboard.com\n\t[92.243.16.209])\n\tby patchwork.libcamera.org (Postfix) with ESMTPS id 47FC8BD87C\n\tfor ;\n\tFri, 20 Aug 2021 01:08:57 +0000 (UTC)","from lancelot.ideasonboard.com (localhost [IPv6:::1])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTP id B5F2868895;\n\tFri, 20 Aug 2021 03:08:56 +0200 (CEST)","from perceval.ideasonboard.com (perceval.ideasonboard.com\n\t[IPv6:2001:4b98:dc2:55:216:3eff:fef7:d647])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id 2757668886\n\tfor ;\n\tFri, 20 Aug 2021 03:08:55 +0200 (CEST)","from pendragon.ideasonboard.com (62-78-145-57.bb.dnainternet.fi\n\t[62.78.145.57])\n\tby perceval.ideasonboard.com (Postfix) with ESMTPSA id 92C658F;\n\tFri, 20 Aug 2021 03:08:54 +0200 (CEST)"],"Authentication-Results":"lancelot.ideasonboard.com; dkim=pass (1024-bit key;\n\tunprotected) header.d=ideasonboard.com header.i=@ideasonboard.com\n\theader.b=\"usGoVild\"; dkim-atps=neutral","DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/simple; d=ideasonboard.com;\n\ts=mail; t=1629421734;\n\tbh=R8K/77fOdRyTMSSeDbBBYaDX1kK/e6dqfEn+tkHd+dw=;\n\th=Date:From:To:Cc:Subject:References:In-Reply-To:From;\n\tb=usGoVildhuHVRdCAYGsXsYx8lwg7Zf8ofl1ecJVALYleVrvdq+3zp12xivcb68u3M\n\tat4uGvqEbHhnJhfzhxIY0EH11XlJRcwGstNvIfHfFWc/gj20qlgc4NZIQZly1c0vcy\n\tu6x1qbblUTmkrDrl88DSCXyJGMNibmRGS6Uwo1SE=","Date":"Fri, 20 Aug 2021 04:08:46 +0300","From":"Laurent Pinchart ","To":"Jean-Michel Hautbois ","Message-ID":"","References":"<20210819145759.184192-1-jeanmichel.hautbois@ideasonboard.com>\n\t<20210819145759.184192-7-jeanmichel.hautbois@ideasonboard.com>","MIME-Version":"1.0","Content-Type":"text/plain; charset=utf-8","Content-Disposition":"inline","In-Reply-To":"<20210819145759.184192-7-jeanmichel.hautbois@ideasonboard.com>","Subject":"Re: [libcamera-devel] [PATCH v4 6/9] ipa: ipu3: convert AWB to the\n\tnew algorithm interface","X-BeenThere":"libcamera-devel@lists.libcamera.org","X-Mailman-Version":"2.1.29","Precedence":"list","List-Id":"","List-Unsubscribe":",\n\t","List-Archive":"","List-Post":"","List-Help":"","List-Subscribe":",\n\t","Cc":"libcamera-devel@lists.libcamera.org","Errors-To":"libcamera-devel-bounces@lists.libcamera.org","Sender":"\"libcamera-devel\" "}}]