[{"id":22464,"web_url":"https://patchwork.libcamera.org/comment/22464/","msgid":"","date":"2022-03-27T20:43:29","subject":"Re: [libcamera-devel] [PATCH v3.2 5/5] ipa: rkisp1: Introduce AWB","submitter":{"id":2,"url":"https://patchwork.libcamera.org/api/people/2/","name":"Laurent Pinchart","email":"laurent.pinchart@ideasonboard.com"},"content":"Hi Jnea-Michel,\n\nThank you for the patch.\n\nOn Thu, Feb 24, 2022 at 03:39:50PM +0100, Jean-Michel Hautbois wrote:\n> The RkISP1 ISP calculates a mean value for Y, Cr and Cb at each frame.\n> There is a RGB mode which could theoretically give us the values for R,\n> G and B directly, but it seems to be failing right now.\n> \n> Convert those values into R, G and B and estimate the gain to apply in a\n> grey world.\n> \n> Signed-off-by: Jean-Michel Hautbois \n> Tested-by: Peter Griffin \n> \n> ---\n> v3.2: - Change the clamping for the awb gains\n> v3: - Change the date of the copyright\n> - Fix the YCbCr to RGB formula\n> ---\n> src/ipa/rkisp1/algorithms/awb.cpp | 179 ++++++++++++++++++++++++++\n> src/ipa/rkisp1/algorithms/awb.h | 33 +++++\n> src/ipa/rkisp1/algorithms/meson.build | 1 +\n> src/ipa/rkisp1/ipa_context.cpp | 28 ++++\n> src/ipa/rkisp1/ipa_context.h | 16 +++\n> src/ipa/rkisp1/rkisp1.cpp | 2 +\n> 6 files changed, 259 insertions(+)\n> create mode 100644 src/ipa/rkisp1/algorithms/awb.cpp\n> create mode 100644 src/ipa/rkisp1/algorithms/awb.h\n> \n> diff --git a/src/ipa/rkisp1/algorithms/awb.cpp b/src/ipa/rkisp1/algorithms/awb.cpp\n> new file mode 100644\n> index 00000000..a34fac2a\n> --- /dev/null\n> +++ b/src/ipa/rkisp1/algorithms/awb.cpp\n> @@ -0,0 +1,179 @@\n> +/* SPDX-License-Identifier: LGPL-2.1-or-later */\n> +/*\n> + * Copyright (C) 2021-2022, Ideas On Board\n> + *\n> + * awb.cpp - AWB control algorithm\n> + */\n> +\n> +#include \"awb.h\"\n> +\n> +#include \n> +#include \n> +\n> +#include \n> +\n> +#include \n> +\n> +/**\n> + * \\file awb.h\n> + */\n> +\n> +namespace libcamera {\n> +\n> +namespace ipa::rkisp1::algorithms {\n> +\n> +/**\n> + * \\class Awb\n> + * \\brief A Grey world white balance correction algorithm\n> + */\n> +\n> +LOG_DEFINE_CATEGORY(RkISP1Awb)\n> +\n> +/**\n> + * \\copydoc libcamera::ipa::Algorithm::configure\n> + */\n> +int Awb::configure(IPAContext &context,\n> +\t\t const IPACameraSensorInfo &configInfo)\n> +{\n> +\tcontext.frameContext.awb.gains.red = 1.0;\n> +\tcontext.frameContext.awb.gains.blue = 1.0;\n> +\tcontext.frameContext.awb.gains.green = 1.0;\n> +\n> +\t/*\n> +\t * Define the measurement window for AWB as a centered rectangle\n> +\t * covering 3/4 of the image width and height.\n> +\t */\n> +\tcontext.configuration.awb.measureWindow.h_offs = configInfo.outputSize.width / 8;\n> +\tcontext.configuration.awb.measureWindow.v_offs = configInfo.outputSize.height / 8;\n> +\tcontext.configuration.awb.measureWindow.h_size = 3 * configInfo.outputSize.width / 4;\n> +\tcontext.configuration.awb.measureWindow.v_size = 3 * configInfo.outputSize.height / 4;\n> +\n> +\treturn 0;\n> +}\n> +\n> +uint32_t Awb::estimateCCT(double red, double green, double blue)\n> +{\n> +\t/* Convert the RGB values to CIE tristimulus values (XYZ) */\n> +\tdouble X = (-0.14282) * (red) + (1.54924) * (green) + (-0.95641) * (blue);\n> +\tdouble Y = (-0.32466) * (red) + (1.57837) * (green) + (-0.73191) * (blue);\n> +\tdouble Z = (-0.68202) * (red) + (0.77073) * (green) + (0.56332) * (blue);\n> +\n> +\t/* Calculate the normalized chromaticity values */\n> +\tdouble x = X / (X + Y + Z);\n> +\tdouble y = Y / (X + Y + Z);\n> +\n> +\t/* Calculate CCT */\n> +\tdouble n = (x - 0.3320) / (0.1858 - y);\n> +\treturn 449 * n * n * n + 3525 * n * n + 6823.3 * n + 5520.33;\n> +}\n> +\n> +/**\n> + * \\copydoc libcamera::ipa::Algorithm::prepare\n> + */\n> +void Awb::prepare(IPAContext &context, rkisp1_params_cfg *params)\n> +{\n> +\tparams->others.awb_gain_config.gain_green_b = 256 * context.frameContext.awb.gains.green;\n> +\tparams->others.awb_gain_config.gain_blue = 256 * context.frameContext.awb.gains.blue;\n> +\tparams->others.awb_gain_config.gain_red = 256 * context.frameContext.awb.gains.red;\n> +\tparams->others.awb_gain_config.gain_green_r = 256 * context.frameContext.awb.gains.green;\n> +\n> +\t/* Update the gains. */\n> +\tparams->module_cfg_update |= RKISP1_CIF_ISP_MODULE_AWB_GAIN;\n> +\n> +\t/* If we already have configured the gains and window, return. */\n> +\tif (context.frameContext.frameCount > 0)\n> +\t\treturn;\n> +\n> +\t/* Configure the gains to apply. */\n> +\tparams->module_en_update |= RKISP1_CIF_ISP_MODULE_AWB_GAIN;\n> +\t/* Update the ISP to apply the gains configured. */\n> +\tparams->module_ens |= RKISP1_CIF_ISP_MODULE_AWB_GAIN;\n> +\n> +\t/* Configure the measure window for AWB. */\n> +\tparams->meas.awb_meas_config.awb_wnd = context.configuration.awb.measureWindow;\n> +\t/*\n> +\t * Measure Y, Cr and Cb means.\n> +\t * \\todo RGB is not working, the kernel seems to not configure it ?\n> +\t */\n> +\tparams->meas.awb_meas_config.awb_mode = RKISP1_CIF_ISP_AWB_MODE_YCBCR;\n> +\t/* Reference Cr and Cb. */\n> +\tparams->meas.awb_meas_config.awb_ref_cb = 128;\n> +\tparams->meas.awb_meas_config.awb_ref_cr = 128;\n> +\t/* Y values to include are between min_y and max_y only. */\n> +\tparams->meas.awb_meas_config.min_y = 16;\n> +\tparams->meas.awb_meas_config.max_y = 250;\n> +\t/* Maximum Cr+Cb value to take into account for awb. */\n> +\tparams->meas.awb_meas_config.max_csum = 250;\n> +\t/* Minimum Cr and Cb values to take into account. */\n> +\tparams->meas.awb_meas_config.min_c = 16;\n> +\t/* Number of frames to use to estimate the mean (0 means 1 frame). */\n> +\tparams->meas.awb_meas_config.frames = 0;\n> +\n> +\t/* Update AWB measurement unit configuration. */\n> +\tparams->module_cfg_update |= RKISP1_CIF_ISP_MODULE_AWB;\n> +\t/* Make sure the ISP is measuring the means for the next frame. */\n> +\tparams->module_en_update |= RKISP1_CIF_ISP_MODULE_AWB;\n> +\tparams->module_ens |= RKISP1_CIF_ISP_MODULE_AWB;\n> +}\n> +\n> +/**\n> + * \\copydoc libcamera::ipa::Algorithm::process\n> + */\n> +void Awb::process([[maybe_unused]] IPAContext &context, const rkisp1_stat_buffer *stats)\n> +{\n> +\tconst rkisp1_cif_isp_stat *params = &stats->params;\n> +\tconst rkisp1_cif_isp_awb_stat *awb = ¶ms->awb;\n> +\n> +\t/* Get the YCbCr mean values */\n> +\tdouble yMean = awb->awb_mean[0].mean_y_or_g;\n> +\tdouble crMean = awb->awb_mean[0].mean_cr_or_r;\n> +\tdouble cbMean = awb->awb_mean[0].mean_cb_or_b;\n> +\n> +\t/*\n> +\t * Convert from YCbCr to RGB.\n> +\t * The hardware uses the following formulas:\n> +\t * Y = 16 + 0.2500 R + 0.5000 G + 0.1094 B\n> +\t * Cb = 128 - 0.1406 R - 0.2969 G + 0.4375 B\n> +\t * Cr = 128 + 0.4375 R - 0.3750 G - 0.0625 B\n> +\t *\n> +\t * The inverse matrix is thus:\n> +\t * [[1,1636, -0,0623, 1,6008]\n> +\t * [1,1636, -0,4045, -0,7949]\n> +\t * [1,1636, 1,9912, -0,0250]]\n> +\t */\n> +\tyMean -= 16;\n> +\tcbMean -= 128;\n> +\tcrMean -= 128;\n> +\tdouble redMean = 1.1636 * yMean - 0.0623 * cbMean + 1.6008 * crMean;\n> +\tdouble blueMean = 1.1636 * yMean - 0.4045 * cbMean - 0.7949 * crMean;\n> +\tdouble greenMean = 1.1636 * yMean + 1.9912 * cbMean - 0.0250 * crMean;\n\nYou've swapped blue and green.\n\n> +\n> +\t/* Estimate the red and blue gains to apply in a grey world. */\n> +\tdouble redGain = greenMean / (redMean + 1);\n> +\tdouble blueGain = greenMean / (blueMean + 1);\n> +\n> +\t/*\n> +\t * Gain values are unsigned integer value, range 0 to 4 with 8 bit\n> +\t * fractional part.\n> +\t */\n> +\tredGain = std::clamp(redGain, 0.0, 1023.0 / 256);\n> +\tblueGain = std::clamp(blueGain, 0.0, 1023.0 / 256);\n\nShouldn't you saturate after filtering, not before, to avoid possibly\nslowing down convergence ? That's probably a theoretical concern though,\nas we shouldn't reach saturation.\n\nReviewed-by: Laurent Pinchart \n\n> +\n> +\t/* Filter the values to avoid oscillations. */\n> +\tIPAFrameContext &frameContext = context.frameContext;\n> +\n> +\tframeContext.awb.temperatureK = estimateCCT(redMean, greenMean, blueMean);\n> +\tframeContext.awb.gains.red = 0.2 * redGain +\n> +\t\t\t\t 0.8 * frameContext.awb.gains.red;\n> +\tframeContext.awb.gains.blue = 0.2 * blueGain +\n> +\t\t\t\t 0.8 * frameContext.awb.gains.blue;\n> +\t/* Hardcode the green gain to 1.0. */\n> +\tframeContext.awb.gains.green = 1.0;\n> +\n> +\tLOG(RkISP1Awb, Debug) << \"Gain found for red: \" << context.frameContext.awb.gains.red\n> +\t\t\t << \" and for blue: \" << context.frameContext.awb.gains.blue;\n> +}\n> +\n> +} /* namespace ipa::rkisp1::algorithms */\n> +\n> +} /* namespace libcamera */\n> diff --git a/src/ipa/rkisp1/algorithms/awb.h b/src/ipa/rkisp1/algorithms/awb.h\n> new file mode 100644\n> index 00000000..11946643\n> --- /dev/null\n> +++ b/src/ipa/rkisp1/algorithms/awb.h\n> @@ -0,0 +1,33 @@\n> +/* SPDX-License-Identifier: LGPL-2.1-or-later */\n> +/*\n> + * Copyright (C) 2021-2022, Ideas On Board\n> + *\n> + * awb.h - AWB control algorithm\n> + */\n> +\n> +#pragma once\n> +\n> +#include \n> +\n> +#include \"algorithm.h\"\n> +\n> +namespace libcamera {\n> +\n> +namespace ipa::rkisp1::algorithms {\n> +\n> +class Awb : public Algorithm\n> +{\n> +public:\n> +\tAwb() = default;\n> +\t~Awb() = default;\n> +\n> +\tint configure(IPAContext &context, const IPACameraSensorInfo &configInfo) override;\n> +\tvoid prepare(IPAContext &context, rkisp1_params_cfg *params) override;\n> +\tvoid process(IPAContext &context, const rkisp1_stat_buffer *stats) override;\n> +\n> +private:\n> +\tuint32_t estimateCCT(double red, double green, double blue);\n> +};\n> +\n> +} /* namespace ipa::rkisp1::algorithms */\n> +} /* namespace libcamera */\n> diff --git a/src/ipa/rkisp1/algorithms/meson.build b/src/ipa/rkisp1/algorithms/meson.build\n> index 27c97731..7ec53d89 100644\n> --- a/src/ipa/rkisp1/algorithms/meson.build\n> +++ b/src/ipa/rkisp1/algorithms/meson.build\n> @@ -2,5 +2,6 @@\n> \n> rkisp1_ipa_algorithms = files([\n> 'agc.cpp',\n> + 'awb.cpp',\n> 'blc.cpp',\n> ])\n> diff --git a/src/ipa/rkisp1/ipa_context.cpp b/src/ipa/rkisp1/ipa_context.cpp\n> index 790e159b..d6d8456c 100644\n> --- a/src/ipa/rkisp1/ipa_context.cpp\n> +++ b/src/ipa/rkisp1/ipa_context.cpp\n> @@ -81,6 +81,14 @@ namespace libcamera::ipa::rkisp1 {\n> * \\brief Hardware revision of the ISP\n> */\n> \n> +/**\n> + * \\var IPASessionConfiguration::awb\n> + * \\brief AWB parameters configuration of the IPA\n> + *\n> + * \\var IPASessionConfiguration::awb.measureWindow\n> + * \\brief AWB measure window\n> + */\n> +\n> /**\n> * \\var IPASessionConfiguration::sensor\n> * \\brief Sensor-specific configuration of the IPA\n> @@ -105,6 +113,26 @@ namespace libcamera::ipa::rkisp1 {\n> * The gain should be adapted to the sensor specific gain code before applying.\n> */\n> \n> +/**\n> + * \\var IPAFrameContext::awb\n> + * \\brief Context for the Automatic White Balance algorithm\n> + *\n> + * \\struct IPAFrameContext::awb.gains\n> + * \\brief White balance gains\n> + *\n> + * \\var IPAFrameContext::awb.gains.red\n> + * \\brief White balance gain for R channel\n> + *\n> + * \\var IPAFrameContext::awb.gains.green\n> + * \\brief White balance gain for G channel\n> + *\n> + * \\var IPAFrameContext::awb.gains.blue\n> + * \\brief White balance gain for B channel\n> + *\n> + * \\var IPAFrameContext::awb.temperatureK\n> + * \\brief Estimated color temperature\n> + */\n> +\n> /**\n> * \\var IPAFrameContext::sensor\n> * \\brief Effective sensor values\n> diff --git a/src/ipa/rkisp1/ipa_context.h b/src/ipa/rkisp1/ipa_context.h\n> index 35e9b8e5..f387cace 100644\n> --- a/src/ipa/rkisp1/ipa_context.h\n> +++ b/src/ipa/rkisp1/ipa_context.h\n> @@ -12,6 +12,8 @@\n> \n> #include \n> \n> +#include \n> +\n> namespace libcamera {\n> \n> namespace ipa::rkisp1 {\n> @@ -25,6 +27,10 @@ struct IPASessionConfiguration {\n> \t\tstruct rkisp1_cif_isp_window measureWindow;\n> \t} agc;\n> \n> +\tstruct {\n> +\t\tstruct rkisp1_cif_isp_window measureWindow;\n> +\t} awb;\n> +\n> \tstruct {\n> \t\tutils::Duration lineDuration;\n> \t} sensor;\n> @@ -40,6 +46,16 @@ struct IPAFrameContext {\n> \t\tdouble gain;\n> \t} agc;\n> \n> +\tstruct {\n> +\t\tstruct {\n> +\t\t\tdouble red;\n> +\t\t\tdouble green;\n> +\t\t\tdouble blue;\n> +\t\t} gains;\n> +\n> +\t\tdouble temperatureK;\n> +\t} awb;\n> +\n> \tstruct {\n> \t\tuint32_t exposure;\n> \t\tdouble gain;\n> diff --git a/src/ipa/rkisp1/rkisp1.cpp b/src/ipa/rkisp1/rkisp1.cpp\n> index bb3deb93..f4db3a21 100644\n> --- a/src/ipa/rkisp1/rkisp1.cpp\n> +++ b/src/ipa/rkisp1/rkisp1.cpp\n> @@ -27,6 +27,7 @@\n> \n> #include \"algorithms/agc.h\"\n> #include \"algorithms/algorithm.h\"\n> +#include \"algorithms/awb.h\"\n> #include \"algorithms/blc.h\"\n> #include \"libipa/camera_sensor_helper.h\"\n> \n> @@ -127,6 +128,7 @@ int IPARkISP1::init(const IPASettings &settings, unsigned int hwRevision)\n> \n> \t/* Construct our Algorithms */\n> \talgorithms_.push_back(std::make_unique());\n> +\talgorithms_.push_back(std::make_unique());\n> \talgorithms_.push_back(std::make_unique());\n> \n> \treturn 0;\n> -- \n> 2.32.0\n>","headers":{"Return-Path":"","X-Original-To":"parsemail@patchwork.libcamera.org","Delivered-To":"parsemail@patchwork.libcamera.org","Received":["from lancelot.ideasonboard.com (lancelot.ideasonboard.com\n\t[92.243.16.209])\n\tby patchwork.libcamera.org (Postfix) with ESMTPS id 6F19AC0F1B\n\tfor ;\n\tSun, 27 Mar 2022 20:43:34 +0000 (UTC)","from lancelot.ideasonboard.com (localhost [IPv6:::1])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTP id C609565633;\n\tSun, 27 Mar 2022 22:43:33 +0200 (CEST)","from perceval.ideasonboard.com (perceval.ideasonboard.com\n\t[213.167.242.64])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id 82DD2600AB\n\tfor ;\n\tSun, 27 Mar 2022 22:43:31 +0200 (CEST)","from pendragon.ideasonboard.com (62-78-145-57.bb.dnainternet.fi\n\t[62.78.145.57])\n\tby perceval.ideasonboard.com (Postfix) with ESMTPSA id A96E92F7;\n\tSun, 27 Mar 2022 22:43:30 +0200 (CEST)"],"DKIM-Signature":["v=1; a=rsa-sha256; c=relaxed/simple; d=libcamera.org;\n\ts=mail; t=1648413813;\n\tbh=HI/DOuk78rztiPUvz/wcAuPDVJVuX8WWBQIxLvUT4Vw=;\n\th=Date:To:References:In-Reply-To:Subject:List-Id:List-Unsubscribe:\n\tList-Archive:List-Post:List-Help:List-Subscribe:From:Reply-To:Cc:\n\tFrom;\n\tb=piD3OZ53O8NFSlkckIXTmTC5eW5h+PAqVtgbsKMIzPMLTYhY+d6Q0HrwZU43ToVWS\n\tQKO6HU8srbni27YFFEu+ce3xM+tuu/rWunY/djuobzG282s5nJwD5jpAZ8uqQe9gFk\n\tyB2JkeDMDxknShQklt6GvsapyFt45OETFUA7yZNNBmVr70goHCVZOVWBYZslwVwTZP\n\tqTgJgfUd7fb7nltK6ceyVXuiF3hVidXjZBAg6yJ7/M1XHa9XOHFAe14VQKnAqw6X/+\n\t1Gi/QGU0AvBsXO7lF/0Yr2eczkV32igDup8l7TUg3X2gPChr5CviQ2j73hXm97zOhX\n\tSA6AxV8b5+4/g==","v=1; a=rsa-sha256; c=relaxed/simple; d=ideasonboard.com;\n\ts=mail; t=1648413811;\n\tbh=HI/DOuk78rztiPUvz/wcAuPDVJVuX8WWBQIxLvUT4Vw=;\n\th=Date:From:To:Cc:Subject:References:In-Reply-To:From;\n\tb=TAgSWKGUhc9rFa+pCrnLwQFo/QXie7gSr24pnLj9FRK/BFsNKtez+rSrMenTN5eRs\n\tOinG73FdMmtqOAtiCeTO1empDdAEdQt1bi9NafaV7EUyHO8bPOzMD94m0KxVGK99L/\n\t/8Z4gegTbYYOIR3eFkff8JFqQdNBPP4urFPFQ5zI="],"Authentication-Results":"lancelot.ideasonboard.com; dkim=pass (1024-bit key; \n\tunprotected) header.d=ideasonboard.com\n\theader.i=@ideasonboard.com\n\theader.b=\"TAgSWKGU\"; dkim-atps=neutral","Date":"Sun, 27 Mar 2022 23:43:29 +0300","To":"Jean-Michel Hautbois ","Message-ID":"","References":"<20220224143747.96540-1-jeanmichel.hautbois@ideasonboard.com>\n\t<20220224143950.98661-1-jeanmichel.hautbois@ideasonboard.com>","MIME-Version":"1.0","Content-Type":"text/plain; charset=utf-8","Content-Disposition":"inline","In-Reply-To":"<20220224143950.98661-1-jeanmichel.hautbois@ideasonboard.com>","Subject":"Re: [libcamera-devel] [PATCH v3.2 5/5] ipa: rkisp1: Introduce AWB","X-BeenThere":"libcamera-devel@lists.libcamera.org","X-Mailman-Version":"2.1.29","Precedence":"list","List-Id":"","List-Unsubscribe":",\n\t","List-Archive":"","List-Post":"","List-Help":"","List-Subscribe":",\n\t","From":"Laurent Pinchart via libcamera-devel\n\t","Reply-To":"Laurent Pinchart ","Cc":"libcamera-devel@lists.libcamera.org","Errors-To":"libcamera-devel-bounces@lists.libcamera.org","Sender":"\"libcamera-devel\" "}},{"id":22465,"web_url":"https://patchwork.libcamera.org/comment/22465/","msgid":"","date":"2022-03-27T21:07:50","subject":"Re: [libcamera-devel] [PATCH v3.2 5/5] ipa: rkisp1: Introduce AWB","submitter":{"id":2,"url":"https://patchwork.libcamera.org/api/people/2/","name":"Laurent Pinchart","email":"laurent.pinchart@ideasonboard.com"},"content":"On Sun, Mar 27, 2022 at 11:43:29PM +0300, Laurent Pinchart via libcamera-devel wrote:\n> On Thu, Feb 24, 2022 at 03:39:50PM +0100, Jean-Michel Hautbois wrote:\n> > The RkISP1 ISP calculates a mean value for Y, Cr and Cb at each frame.\n> > There is a RGB mode which could theoretically give us the values for R,\n> > G and B directly, but it seems to be failing right now.\n> > \n> > Convert those values into R, G and B and estimate the gain to apply in a\n> > grey world.\n> > \n> > Signed-off-by: Jean-Michel Hautbois \n> > Tested-by: Peter Griffin \n> > \n> > ---\n> > v3.2: - Change the clamping for the awb gains\n> > v3: - Change the date of the copyright\n> > - Fix the YCbCr to RGB formula\n> > ---\n> > src/ipa/rkisp1/algorithms/awb.cpp | 179 ++++++++++++++++++++++++++\n> > src/ipa/rkisp1/algorithms/awb.h | 33 +++++\n> > src/ipa/rkisp1/algorithms/meson.build | 1 +\n> > src/ipa/rkisp1/ipa_context.cpp | 28 ++++\n> > src/ipa/rkisp1/ipa_context.h | 16 +++\n> > src/ipa/rkisp1/rkisp1.cpp | 2 +\n> > 6 files changed, 259 insertions(+)\n> > create mode 100644 src/ipa/rkisp1/algorithms/awb.cpp\n> > create mode 100644 src/ipa/rkisp1/algorithms/awb.h\n> > \n> > diff --git a/src/ipa/rkisp1/algorithms/awb.cpp b/src/ipa/rkisp1/algorithms/awb.cpp\n> > new file mode 100644\n> > index 00000000..a34fac2a\n> > --- /dev/null\n> > +++ b/src/ipa/rkisp1/algorithms/awb.cpp\n> > @@ -0,0 +1,179 @@\n> > +/* SPDX-License-Identifier: LGPL-2.1-or-later */\n> > +/*\n> > + * Copyright (C) 2021-2022, Ideas On Board\n> > + *\n> > + * awb.cpp - AWB control algorithm\n> > + */\n> > +\n> > +#include \"awb.h\"\n> > +\n> > +#include \n> > +#include \n> > +\n> > +#include \n> > +\n> > +#include \n> > +\n> > +/**\n> > + * \\file awb.h\n> > + */\n> > +\n> > +namespace libcamera {\n> > +\n> > +namespace ipa::rkisp1::algorithms {\n> > +\n> > +/**\n> > + * \\class Awb\n> > + * \\brief A Grey world white balance correction algorithm\n> > + */\n> > +\n> > +LOG_DEFINE_CATEGORY(RkISP1Awb)\n> > +\n> > +/**\n> > + * \\copydoc libcamera::ipa::Algorithm::configure\n> > + */\n> > +int Awb::configure(IPAContext &context,\n> > +\t\t const IPACameraSensorInfo &configInfo)\n> > +{\n> > +\tcontext.frameContext.awb.gains.red = 1.0;\n> > +\tcontext.frameContext.awb.gains.blue = 1.0;\n> > +\tcontext.frameContext.awb.gains.green = 1.0;\n> > +\n> > +\t/*\n> > +\t * Define the measurement window for AWB as a centered rectangle\n> > +\t * covering 3/4 of the image width and height.\n> > +\t */\n> > +\tcontext.configuration.awb.measureWindow.h_offs = configInfo.outputSize.width / 8;\n> > +\tcontext.configuration.awb.measureWindow.v_offs = configInfo.outputSize.height / 8;\n> > +\tcontext.configuration.awb.measureWindow.h_size = 3 * configInfo.outputSize.width / 4;\n> > +\tcontext.configuration.awb.measureWindow.v_size = 3 * configInfo.outputSize.height / 4;\n> > +\n> > +\treturn 0;\n> > +}\n> > +\n> > +uint32_t Awb::estimateCCT(double red, double green, double blue)\n> > +{\n> > +\t/* Convert the RGB values to CIE tristimulus values (XYZ) */\n> > +\tdouble X = (-0.14282) * (red) + (1.54924) * (green) + (-0.95641) * (blue);\n> > +\tdouble Y = (-0.32466) * (red) + (1.57837) * (green) + (-0.73191) * (blue);\n> > +\tdouble Z = (-0.68202) * (red) + (0.77073) * (green) + (0.56332) * (blue);\n> > +\n> > +\t/* Calculate the normalized chromaticity values */\n> > +\tdouble x = X / (X + Y + Z);\n> > +\tdouble y = Y / (X + Y + Z);\n> > +\n> > +\t/* Calculate CCT */\n> > +\tdouble n = (x - 0.3320) / (0.1858 - y);\n> > +\treturn 449 * n * n * n + 3525 * n * n + 6823.3 * n + 5520.33;\n> > +}\n> > +\n> > +/**\n> > + * \\copydoc libcamera::ipa::Algorithm::prepare\n> > + */\n> > +void Awb::prepare(IPAContext &context, rkisp1_params_cfg *params)\n> > +{\n> > +\tparams->others.awb_gain_config.gain_green_b = 256 * context.frameContext.awb.gains.green;\n> > +\tparams->others.awb_gain_config.gain_blue = 256 * context.frameContext.awb.gains.blue;\n> > +\tparams->others.awb_gain_config.gain_red = 256 * context.frameContext.awb.gains.red;\n> > +\tparams->others.awb_gain_config.gain_green_r = 256 * context.frameContext.awb.gains.green;\n> > +\n> > +\t/* Update the gains. */\n> > +\tparams->module_cfg_update |= RKISP1_CIF_ISP_MODULE_AWB_GAIN;\n> > +\n> > +\t/* If we already have configured the gains and window, return. */\n> > +\tif (context.frameContext.frameCount > 0)\n> > +\t\treturn;\n> > +\n> > +\t/* Configure the gains to apply. */\n> > +\tparams->module_en_update |= RKISP1_CIF_ISP_MODULE_AWB_GAIN;\n> > +\t/* Update the ISP to apply the gains configured. */\n> > +\tparams->module_ens |= RKISP1_CIF_ISP_MODULE_AWB_GAIN;\n> > +\n> > +\t/* Configure the measure window for AWB. */\n> > +\tparams->meas.awb_meas_config.awb_wnd = context.configuration.awb.measureWindow;\n> > +\t/*\n> > +\t * Measure Y, Cr and Cb means.\n> > +\t * \\todo RGB is not working, the kernel seems to not configure it ?\n> > +\t */\n> > +\tparams->meas.awb_meas_config.awb_mode = RKISP1_CIF_ISP_AWB_MODE_YCBCR;\n> > +\t/* Reference Cr and Cb. */\n> > +\tparams->meas.awb_meas_config.awb_ref_cb = 128;\n> > +\tparams->meas.awb_meas_config.awb_ref_cr = 128;\n> > +\t/* Y values to include are between min_y and max_y only. */\n> > +\tparams->meas.awb_meas_config.min_y = 16;\n> > +\tparams->meas.awb_meas_config.max_y = 250;\n> > +\t/* Maximum Cr+Cb value to take into account for awb. */\n> > +\tparams->meas.awb_meas_config.max_csum = 250;\n> > +\t/* Minimum Cr and Cb values to take into account. */\n> > +\tparams->meas.awb_meas_config.min_c = 16;\n> > +\t/* Number of frames to use to estimate the mean (0 means 1 frame). */\n> > +\tparams->meas.awb_meas_config.frames = 0;\n> > +\n> > +\t/* Update AWB measurement unit configuration. */\n> > +\tparams->module_cfg_update |= RKISP1_CIF_ISP_MODULE_AWB;\n> > +\t/* Make sure the ISP is measuring the means for the next frame. */\n> > +\tparams->module_en_update |= RKISP1_CIF_ISP_MODULE_AWB;\n> > +\tparams->module_ens |= RKISP1_CIF_ISP_MODULE_AWB;\n> > +}\n> > +\n> > +/**\n> > + * \\copydoc libcamera::ipa::Algorithm::process\n> > + */\n> > +void Awb::process([[maybe_unused]] IPAContext &context, const rkisp1_stat_buffer *stats)\n> > +{\n> > +\tconst rkisp1_cif_isp_stat *params = &stats->params;\n> > +\tconst rkisp1_cif_isp_awb_stat *awb = ¶ms->awb;\n> > +\n> > +\t/* Get the YCbCr mean values */\n> > +\tdouble yMean = awb->awb_mean[0].mean_y_or_g;\n> > +\tdouble crMean = awb->awb_mean[0].mean_cr_or_r;\n> > +\tdouble cbMean = awb->awb_mean[0].mean_cb_or_b;\n> > +\n> > +\t/*\n> > +\t * Convert from YCbCr to RGB.\n> > +\t * The hardware uses the following formulas:\n> > +\t * Y = 16 + 0.2500 R + 0.5000 G + 0.1094 B\n> > +\t * Cb = 128 - 0.1406 R - 0.2969 G + 0.4375 B\n> > +\t * Cr = 128 + 0.4375 R - 0.3750 G - 0.0625 B\n> > +\t *\n> > +\t * The inverse matrix is thus:\n> > +\t * [[1,1636, -0,0623, 1,6008]\n> > +\t * [1,1636, -0,4045, -0,7949]\n> > +\t * [1,1636, 1,9912, -0,0250]]\n> > +\t */\n> > +\tyMean -= 16;\n> > +\tcbMean -= 128;\n> > +\tcrMean -= 128;\n> > +\tdouble redMean = 1.1636 * yMean - 0.0623 * cbMean + 1.6008 * crMean;\n> > +\tdouble blueMean = 1.1636 * yMean - 0.4045 * cbMean - 0.7949 * crMean;\n> > +\tdouble greenMean = 1.1636 * yMean + 1.9912 * cbMean - 0.0250 * crMean;\n> \n> You've swapped blue and green.\n> \n> > +\n> > +\t/* Estimate the red and blue gains to apply in a grey world. */\n> > +\tdouble redGain = greenMean / (redMean + 1);\n> > +\tdouble blueGain = greenMean / (blueMean + 1);\n> > +\n> > +\t/*\n> > +\t * Gain values are unsigned integer value, range 0 to 4 with 8 bit\n> > +\t * fractional part.\n> > +\t */\n> > +\tredGain = std::clamp(redGain, 0.0, 1023.0 / 256);\n> > +\tblueGain = std::clamp(blueGain, 0.0, 1023.0 / 256);\n> \n> Shouldn't you saturate after filtering, not before, to avoid possibly\n> slowing down convergence ? That's probably a theoretical concern though,\n> as we shouldn't reach saturation.\n>\n> Reviewed-by: Laurent Pinchart \n> \n> > +\n> > +\t/* Filter the values to avoid oscillations. */\n> > +\tIPAFrameContext &frameContext = context.frameContext;\n> > +\n> > +\tframeContext.awb.temperatureK = estimateCCT(redMean, greenMean, blueMean);\n> > +\tframeContext.awb.gains.red = 0.2 * redGain +\n> > +\t\t\t\t 0.8 * frameContext.awb.gains.red;\n> > +\tframeContext.awb.gains.blue = 0.2 * blueGain +\n> > +\t\t\t\t 0.8 * frameContext.awb.gains.blue;\n\nIt could also be nice to store the speed in a variable for clarity.\nCombined with the previous comment, this is what I've tested, without\nany noticeable regression:\n\n\tIPAFrameContext &frameContext = context.frameContext;\n\n\t/* Filter the values to avoid oscillations. */\n\tdouble speed = 0.2;\n\tredGain = speed * redGain + (1 - speed) * frameContext.awb.gains.red;\n\tblueGain = speed * blueGain + (1 - speed) * frameContext.awb.gains.blue;\n\n\t/*\n\t * Gain values are unsigned integer value, range 0 to 4 with 8 bit\n\t * fractional part.\n\t */\n\tframeContext.awb.gains.red = std::clamp(redGain, 0.0, 1023.0 / 256);\n\tframeContext.awb.gains.blue = std::clamp(blueGain, 0.0, 1023.0 / 256);\n\t/* Hardcode the green gain to 1.0. */\n\tframeContext.awb.gains.green = 1.0;\n\n\tframeContext.awb.temperatureK = estimateCCT(redMean, greenMean, blueMean);\n\n> > +\t/* Hardcode the green gain to 1.0. */\n> > +\tframeContext.awb.gains.green = 1.0;\n> > +\n> > +\tLOG(RkISP1Awb, Debug) << \"Gain found for red: \" << context.frameContext.awb.gains.red\n> > +\t\t\t << \" and for blue: \" << context.frameContext.awb.gains.blue;\n> > +}\n> > +\n> > +} /* namespace ipa::rkisp1::algorithms */\n> > +\n> > +} /* namespace libcamera */\n> > diff --git a/src/ipa/rkisp1/algorithms/awb.h b/src/ipa/rkisp1/algorithms/awb.h\n> > new file mode 100644\n> > index 00000000..11946643\n> > --- /dev/null\n> > +++ b/src/ipa/rkisp1/algorithms/awb.h\n> > @@ -0,0 +1,33 @@\n> > +/* SPDX-License-Identifier: LGPL-2.1-or-later */\n> > +/*\n> > + * Copyright (C) 2021-2022, Ideas On Board\n> > + *\n> > + * awb.h - AWB control algorithm\n> > + */\n> > +\n> > +#pragma once\n> > +\n> > +#include \n> > +\n> > +#include \"algorithm.h\"\n> > +\n> > +namespace libcamera {\n> > +\n> > +namespace ipa::rkisp1::algorithms {\n> > +\n> > +class Awb : public Algorithm\n> > +{\n> > +public:\n> > +\tAwb() = default;\n> > +\t~Awb() = default;\n> > +\n> > +\tint configure(IPAContext &context, const IPACameraSensorInfo &configInfo) override;\n> > +\tvoid prepare(IPAContext &context, rkisp1_params_cfg *params) override;\n> > +\tvoid process(IPAContext &context, const rkisp1_stat_buffer *stats) override;\n> > +\n> > +private:\n> > +\tuint32_t estimateCCT(double red, double green, double blue);\n> > +};\n> > +\n> > +} /* namespace ipa::rkisp1::algorithms */\n> > +} /* namespace libcamera */\n> > diff --git a/src/ipa/rkisp1/algorithms/meson.build b/src/ipa/rkisp1/algorithms/meson.build\n> > index 27c97731..7ec53d89 100644\n> > --- a/src/ipa/rkisp1/algorithms/meson.build\n> > +++ b/src/ipa/rkisp1/algorithms/meson.build\n> > @@ -2,5 +2,6 @@\n> > \n> > rkisp1_ipa_algorithms = files([\n> > 'agc.cpp',\n> > + 'awb.cpp',\n> > 'blc.cpp',\n> > ])\n> > diff --git a/src/ipa/rkisp1/ipa_context.cpp b/src/ipa/rkisp1/ipa_context.cpp\n> > index 790e159b..d6d8456c 100644\n> > --- a/src/ipa/rkisp1/ipa_context.cpp\n> > +++ b/src/ipa/rkisp1/ipa_context.cpp\n> > @@ -81,6 +81,14 @@ namespace libcamera::ipa::rkisp1 {\n> > * \\brief Hardware revision of the ISP\n> > */\n> > \n> > +/**\n> > + * \\var IPASessionConfiguration::awb\n> > + * \\brief AWB parameters configuration of the IPA\n> > + *\n> > + * \\var IPASessionConfiguration::awb.measureWindow\n> > + * \\brief AWB measure window\n> > + */\n> > +\n> > /**\n> > * \\var IPASessionConfiguration::sensor\n> > * \\brief Sensor-specific configuration of the IPA\n> > @@ -105,6 +113,26 @@ namespace libcamera::ipa::rkisp1 {\n> > * The gain should be adapted to the sensor specific gain code before applying.\n> > */\n> > \n> > +/**\n> > + * \\var IPAFrameContext::awb\n> > + * \\brief Context for the Automatic White Balance algorithm\n> > + *\n> > + * \\struct IPAFrameContext::awb.gains\n> > + * \\brief White balance gains\n> > + *\n> > + * \\var IPAFrameContext::awb.gains.red\n> > + * \\brief White balance gain for R channel\n> > + *\n> > + * \\var IPAFrameContext::awb.gains.green\n> > + * \\brief White balance gain for G channel\n> > + *\n> > + * \\var IPAFrameContext::awb.gains.blue\n> > + * \\brief White balance gain for B channel\n> > + *\n> > + * \\var IPAFrameContext::awb.temperatureK\n> > + * \\brief Estimated color temperature\n> > + */\n> > +\n> > /**\n> > * \\var IPAFrameContext::sensor\n> > * \\brief Effective sensor values\n> > diff --git a/src/ipa/rkisp1/ipa_context.h b/src/ipa/rkisp1/ipa_context.h\n> > index 35e9b8e5..f387cace 100644\n> > --- a/src/ipa/rkisp1/ipa_context.h\n> > +++ b/src/ipa/rkisp1/ipa_context.h\n> > @@ -12,6 +12,8 @@\n> > \n> > #include \n> > \n> > +#include \n> > +\n> > namespace libcamera {\n> > \n> > namespace ipa::rkisp1 {\n> > @@ -25,6 +27,10 @@ struct IPASessionConfiguration {\n> > \t\tstruct rkisp1_cif_isp_window measureWindow;\n> > \t} agc;\n> > \n> > +\tstruct {\n> > +\t\tstruct rkisp1_cif_isp_window measureWindow;\n> > +\t} awb;\n> > +\n> > \tstruct {\n> > \t\tutils::Duration lineDuration;\n> > \t} sensor;\n> > @@ -40,6 +46,16 @@ struct IPAFrameContext {\n> > \t\tdouble gain;\n> > \t} agc;\n> > \n> > +\tstruct {\n> > +\t\tstruct {\n> > +\t\t\tdouble red;\n> > +\t\t\tdouble green;\n> > +\t\t\tdouble blue;\n> > +\t\t} gains;\n> > +\n> > +\t\tdouble temperatureK;\n> > +\t} awb;\n> > +\n> > \tstruct {\n> > \t\tuint32_t exposure;\n> > \t\tdouble gain;\n> > diff --git a/src/ipa/rkisp1/rkisp1.cpp b/src/ipa/rkisp1/rkisp1.cpp\n> > index bb3deb93..f4db3a21 100644\n> > --- a/src/ipa/rkisp1/rkisp1.cpp\n> > +++ b/src/ipa/rkisp1/rkisp1.cpp\n> > @@ -27,6 +27,7 @@\n> > \n> > #include \"algorithms/agc.h\"\n> > #include \"algorithms/algorithm.h\"\n> > +#include \"algorithms/awb.h\"\n> > #include \"algorithms/blc.h\"\n> > #include \"libipa/camera_sensor_helper.h\"\n> > \n> > @@ -127,6 +128,7 @@ int IPARkISP1::init(const IPASettings &settings, unsigned int hwRevision)\n> > \n> > \t/* Construct our Algorithms */\n> > \talgorithms_.push_back(std::make_unique());\n> > +\talgorithms_.push_back(std::make_unique());\n> > \talgorithms_.push_back(std::make_unique());\n> > \n> > \treturn 0;","headers":{"Return-Path":"","X-Original-To":"parsemail@patchwork.libcamera.org","Delivered-To":"parsemail@patchwork.libcamera.org","Received":["from lancelot.ideasonboard.com (lancelot.ideasonboard.com\n\t[92.243.16.209])\n\tby patchwork.libcamera.org (Postfix) with ESMTPS id 3DF81C3256\n\tfor ;\n\tSun, 27 Mar 2022 21:07:56 +0000 (UTC)","from lancelot.ideasonboard.com (localhost [IPv6:::1])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTP id 95332600B0;\n\tSun, 27 Mar 2022 23:07:55 +0200 (CEST)","from perceval.ideasonboard.com (perceval.ideasonboard.com\n\t[IPv6:2001:4b98:dc2:55:216:3eff:fef7:d647])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id DD29B600AB\n\tfor ;\n\tSun, 27 Mar 2022 23:07:53 +0200 (CEST)","from pendragon.ideasonboard.com (62-78-145-57.bb.dnainternet.fi\n\t[62.78.145.57])\n\tby perceval.ideasonboard.com (Postfix) with ESMTPSA id 522C52F7;\n\tSun, 27 Mar 2022 23:07:53 +0200 (CEST)"],"DKIM-Signature":["v=1; a=rsa-sha256; c=relaxed/simple; d=libcamera.org;\n\ts=mail; t=1648415275;\n\tbh=BHlgw3ifrjvCuM4DKiyOSA+J8ohp9pF6uPAA8fq7Zwc=;\n\th=Date:To:References:In-Reply-To:Subject:List-Id:List-Unsubscribe:\n\tList-Archive:List-Post:List-Help:List-Subscribe:From:Reply-To:\n\tFrom;\n\tb=eh4YxK9tzIjCaygTNKXMfCriRjAiNbanVJRANPIbNlHmuJq4b4HvEcSsq87uE60jU\n\tvQt+T+EVc8FEhYZil+8M4GU+BMrOz17hCcIOz11r1nhE4ouo4X1P8MCOFmWT5W7YtC\n\tI5KLgZIYaOvgJ0SRDXbrCtOmdg0Rep7MGz7gT52XfTVkGJHLhnCb7NCJ6tmEhc5di1\n\tw+A6dCAtPYRw+vbYbY1kqE9jnCK+Wxe7Ogsx4b/1JD0tJBnvRZcAKS0s7qxI7HUle3\n\tMA9OVXvFRIruW5Nn7dXknmMZRHbqHG/1hSgbM8FAwjiYvum7ZlX39C/FfCrkswZvAX\n\tywyA0euxidxcg==","v=1; a=rsa-sha256; c=relaxed/simple; d=ideasonboard.com;\n\ts=mail; t=1648415273;\n\tbh=BHlgw3ifrjvCuM4DKiyOSA+J8ohp9pF6uPAA8fq7Zwc=;\n\th=Date:From:To:Subject:References:In-Reply-To:From;\n\tb=Ofj1Q8voqZz4JORiMUDLrpEt+ABDpUGyFN5HGWQFzLYdKcaSnnrhppTM+CK9I2fZQ\n\t+IB1i1xnz3KNGKNLOMzVsWLaew3tjIORuH69gYMZhUynxWRzL4NI/2Bs165yxAuI1p\n\tpDUU6lqWsxkRPu67OLfnkc5+2YZup0uBwgyzNV68="],"Authentication-Results":"lancelot.ideasonboard.com; dkim=pass (1024-bit key; \n\tunprotected) header.d=ideasonboard.com\n\theader.i=@ideasonboard.com\n\theader.b=\"Ofj1Q8vo\"; dkim-atps=neutral","Date":"Mon, 28 Mar 2022 00:07:50 +0300","To":"Jean-Michel Hautbois ,\n\tlibcamera-devel@lists.libcamera.org","Message-ID":"","References":"<20220224143747.96540-1-jeanmichel.hautbois@ideasonboard.com>\n\t<20220224143950.98661-1-jeanmichel.hautbois@ideasonboard.com>\n\t","MIME-Version":"1.0","Content-Type":"text/plain; charset=utf-8","Content-Disposition":"inline","In-Reply-To":"","Subject":"Re: [libcamera-devel] [PATCH v3.2 5/5] ipa: rkisp1: Introduce AWB","X-BeenThere":"libcamera-devel@lists.libcamera.org","X-Mailman-Version":"2.1.29","Precedence":"list","List-Id":"","List-Unsubscribe":",\n\t","List-Archive":"","List-Post":"","List-Help":"","List-Subscribe":",\n\t","From":"Laurent Pinchart via libcamera-devel\n\t","Reply-To":"Laurent Pinchart ","Errors-To":"libcamera-devel-bounces@lists.libcamera.org","Sender":"\"libcamera-devel\" "}}]