| Message ID | 20220223104824.25723-5-jeanmichel.hautbois@ideasonboard.com |
|---|---|
| State | Superseded |
| Headers | show |
| Series |
|
| Related | show |
Hi Jean-Michel, Thank you for the patch. On Wed, Feb 23, 2022 at 11:48:24AM +0100, Jean-Michel Hautbois wrote: > The RkISP1 ISP calculates a mean value for Y, Cr and Cb at each frame. > There is a RGB mode which could theoretically give us the values for R, > G and B directly, but it seems to be failing right now. > > Convert those values into R, G and B and estimate the gain to apply in a > grey world. > > Signed-off-by: Jean-Michel Hautbois <jeanmichel.hautbois@ideasonboard.com> > Tested-by: Peter Griffin <peter.griffin@linaro.org> > --- > src/ipa/rkisp1/algorithms/awb.cpp | 155 ++++++++++++++++++++++++++ > src/ipa/rkisp1/algorithms/awb.h | 33 ++++++ > src/ipa/rkisp1/algorithms/meson.build | 1 + > src/ipa/rkisp1/ipa_context.cpp | 28 +++++ > src/ipa/rkisp1/ipa_context.h | 16 +++ > src/ipa/rkisp1/rkisp1.cpp | 2 + > 6 files changed, 235 insertions(+) > create mode 100644 src/ipa/rkisp1/algorithms/awb.cpp > create mode 100644 src/ipa/rkisp1/algorithms/awb.h > > diff --git a/src/ipa/rkisp1/algorithms/awb.cpp b/src/ipa/rkisp1/algorithms/awb.cpp > new file mode 100644 > index 00000000..8fa8c2f7 > --- /dev/null > +++ b/src/ipa/rkisp1/algorithms/awb.cpp > @@ -0,0 +1,155 @@ > +/* SPDX-License-Identifier: LGPL-2.1-or-later */ > +/* > + * Copyright (C) 2021, Ideas On Board 2021-2022 Time flies :-) > + * > + * awb.cpp - AWB control algorithm > + */ > + > +#include "awb.h" > + > +#include <algorithm> > +#include <cmath> > + > +#include <libcamera/base/log.h> > + > +#include <libcamera/ipa/core_ipa_interface.h> > + > +/** > + * \file awb.h > + */ > + > +namespace libcamera { > + > +namespace ipa::rkisp1::algorithms { > + > +/** > + * \class Awb > + * \brief A Grey world white balance correction algorithm > + */ > + > +LOG_DEFINE_CATEGORY(RkISP1Awb) > + > +/** > + * \copydoc libcamera::ipa::Algorithm::configure > + */ > +int Awb::configure(IPAContext &context, > + const IPACameraSensorInfo &configInfo) > +{ > + context.frameContext.awb.gains.red = 1.0; > + context.frameContext.awb.gains.blue = 1.0; > + context.frameContext.awb.gains.green = 1.0; > + > + /* Define the measurement window for AGC. */ Same comment as for patch 3/4. > + context.configuration.awb.measureWindow.h_offs = configInfo.outputSize.width / 8; > + context.configuration.awb.measureWindow.v_offs = configInfo.outputSize.height / 8; > + context.configuration.awb.measureWindow.h_size = 3 * configInfo.outputSize.width / 4; > + context.configuration.awb.measureWindow.v_size = 3 * configInfo.outputSize.height / 4; > + > + return 0; > +} > + > +uint32_t Awb::estimateCCT(double red, double green, double blue) > +{ > + /* Convert the RGB values to CIE tristimulus values (XYZ) */ > + double X = (-0.14282) * (red) + (1.54924) * (green) + (-0.95641) * (blue); > + double Y = (-0.32466) * (red) + (1.57837) * (green) + (-0.73191) * (blue); > + double Z = (-0.68202) * (red) + (0.77073) * (green) + (0.56332) * (blue); > + > + /* Calculate the normalized chromaticity values */ > + double x = X / (X + Y + Z); > + double y = Y / (X + Y + Z); > + > + /* Calculate CCT */ > + double n = (x - 0.3320) / (0.1858 - y); > + return 449 * n * n * n + 3525 * n * n + 6823.3 * n + 5520.33; > +} > + > +/** > + * \copydoc libcamera::ipa::Algorithm::process > + */ > +void Awb::process([[maybe_unused]] IPAContext &context, const rkisp1_stat_buffer *stats) Please move this after prepare() to match the order of the .h file. > +{ > + const rkisp1_cif_isp_stat *params = &stats->params; > + const rkisp1_cif_isp_awb_stat *awb = ¶ms->awb; > + > + /* Get the YCbCr mean values */ > + double yMean = awb->awb_mean[0].mean_y_or_g; > + double crMean = awb->awb_mean[0].mean_cr_or_r; > + double cbMean = awb->awb_mean[0].mean_cb_or_b; > + > + /* Convert from YCbCr to RGB. */ > + double redMean = yMean + 1.402 * (crMean - 128); > + double blueMean = yMean + 1.772 * (cbMean - 128); > + double greenMean = yMean - 0.34414 * (cbMean - 128) - 0.71414 * (crMean - 128); Where do those values come from ? Unless I'm mistaken, the hardware uses the following formulas to convert from RGB to YCbCr. Y = 16 + 0.2500 R + 0.5000 G + 0.1094 B Cb = 128 - 0.1406 R - 0.2969 G + 0.4375 B Cr = 128 + 0.4375 R - 0.3750 G - 0.0625 B > + > + /* Estimate the red and blue gains to apply in a grey world. */ > + double redGain = greenMean / (redMean + 1); > + double blueGain = greenMean / (blueMean + 1); > + > + /* Filter the values to avoid oscillations. */ > + IPAFrameContext &frameContext = context.frameContext; > + > + frameContext.awb.temperatureK = estimateCCT(redMean, greenMean, blueMean); > + frameContext.awb.gains.red = 0.2 * redGain + > + 0.8 * frameContext.awb.gains.red; > + frameContext.awb.gains.blue = 0.2 * blueGain + > + 0.8 * frameContext.awb.gains.blue; Don't you need to saturate the gains, either here, or in prepare() ? > + /* Hardcode the green gain to 1.0. */ > + frameContext.awb.gains.green = 1.0; > + > + LOG(RkISP1Awb, Debug) << "Gain found for red: " << context.frameContext.awb.gains.red > + << " and for blue: " << context.frameContext.awb.gains.blue; > +} > + > +/** > + * \copydoc libcamera::ipa::Algorithm::prepare > + */ > +void Awb::prepare(IPAContext &context, > + rkisp1_params_cfg *params) This holds on a single line. > +{ > + params->others.awb_gain_config.gain_green_b = 256 * context.frameContext.awb.gains.green; > + params->others.awb_gain_config.gain_blue = 256 * context.frameContext.awb.gains.blue; > + params->others.awb_gain_config.gain_red = 256 * context.frameContext.awb.gains.red; > + params->others.awb_gain_config.gain_green_r = 256 * context.frameContext.awb.gains.green; > + > + if (context.frameContext.frameId == 0) { > + /* Configure the gains to apply. */ > + params->module_en_update |= RKISP1_CIF_ISP_MODULE_AWB_GAIN; > + /* Update the ISP to apply the gains configured. */ > + params->module_ens |= RKISP1_CIF_ISP_MODULE_AWB_GAIN; > + } > + /* Update the gains. */ > + params->module_cfg_update |= RKISP1_CIF_ISP_MODULE_AWB_GAIN; I'd move this line just after setting the gains above. > + > + /* Configure the measure window for AWB. */ > + params->meas.awb_meas_config.awb_wnd = context.configuration.awb.measureWindow; > + /* > + * Measure Y, Cr and Cb means. > + * \todo RGB is not working, the kernel seems to not configure it ? Does this mean that if RGB worked, you would prefer using that ? > + */ > + params->meas.awb_meas_config.awb_mode = RKISP1_CIF_ISP_AWB_MODE_YCBCR; > + /* Reference Cr and Cb. */ > + params->meas.awb_meas_config.awb_ref_cb = 128; > + params->meas.awb_meas_config.awb_ref_cr = 128; > + /* Y values to include are between min_y and max_y only. */ > + params->meas.awb_meas_config.min_y = 16; > + params->meas.awb_meas_config.max_y = 250; > + /* Maximum Cr+Cb value to take into account for awb. */ > + params->meas.awb_meas_config.max_csum = 250; > + /* Minimum Cr and Cb values to take into account. */ > + params->meas.awb_meas_config.min_c = 16; > + /* Number of frames to use to estimate the mean (0 means 1 frame). */ > + params->meas.awb_meas_config.frames = 0; All this can move inside the if () below. > + > + if (context.frameContext.frameId == 0) { > + /* Update AWB measurement unit configuration. */ > + params->module_cfg_update |= RKISP1_CIF_ISP_MODULE_AWB; > + /* Make sure the ISP is measuring the means for the next frame. */ > + params->module_en_update |= RKISP1_CIF_ISP_MODULE_AWB; > + params->module_ens |= RKISP1_CIF_ISP_MODULE_AWB; > + } > +} > + > +} /* namespace ipa::rkisp1::algorithms */ > + > +} /* namespace libcamera */ > diff --git a/src/ipa/rkisp1/algorithms/awb.h b/src/ipa/rkisp1/algorithms/awb.h > new file mode 100644 > index 00000000..0a9fb82c > --- /dev/null > +++ b/src/ipa/rkisp1/algorithms/awb.h > @@ -0,0 +1,33 @@ > +/* SPDX-License-Identifier: LGPL-2.1-or-later */ > +/* > + * Copyright (C) 2021, Ideas On Board 2021-2022 > + * > + * awb.h - AWB control algorithm > + */ > + > +#pragma once > + > +#include <linux/rkisp1-config.h> > + > +#include "algorithm.h" > + > +namespace libcamera { > + > +namespace ipa::rkisp1::algorithms { > + > +class Awb : public Algorithm > +{ > +public: > + Awb() = default; > + ~Awb() = default; > + > + int configure(IPAContext &context, const IPACameraSensorInfo &configInfo) override; > + void prepare(IPAContext &context, rkisp1_params_cfg *params) override; > + void process(IPAContext &context, const rkisp1_stat_buffer *stats) override; > + > +private: > + uint32_t estimateCCT(double red, double green, double blue); > +}; > + > +} /* namespace ipa::rkisp1::algorithms */ > +} /* namespace libcamera */ > diff --git a/src/ipa/rkisp1/algorithms/meson.build b/src/ipa/rkisp1/algorithms/meson.build > index 27c97731..7ec53d89 100644 > --- a/src/ipa/rkisp1/algorithms/meson.build > +++ b/src/ipa/rkisp1/algorithms/meson.build > @@ -2,5 +2,6 @@ > > rkisp1_ipa_algorithms = files([ > 'agc.cpp', > + 'awb.cpp', > 'blc.cpp', > ]) > diff --git a/src/ipa/rkisp1/ipa_context.cpp b/src/ipa/rkisp1/ipa_context.cpp > index 39acef47..c25f44ec 100644 > --- a/src/ipa/rkisp1/ipa_context.cpp > +++ b/src/ipa/rkisp1/ipa_context.cpp > @@ -81,6 +81,14 @@ namespace libcamera::ipa::rkisp1 { > * \brief Hardware revision of the ISP > */ > > +/** > + * \var IPASessionConfiguration::awb > + * \brief AWB parameters configuration of the IPA > + * > + * \var IPASessionConfiguration::awb.measureWindow > + * \brief AWB measure window > + */ > + > /** > * \var IPASessionConfiguration::sensor > * \brief Sensor-specific configuration of the IPA > @@ -105,6 +113,26 @@ namespace libcamera::ipa::rkisp1 { > * The gain should be adapted to the sensor specific gain code before applying. > */ > > +/** > + * \var IPAFrameContext::awb > + * \brief Context for the Automatic White Balance algorithm > + * > + * \struct IPAFrameContext::awb.gains > + * \brief White balance gains > + * > + * \var IPAFrameContext::awb.gains.red > + * \brief White balance gain for R channel > + * > + * \var IPAFrameContext::awb.gains.green > + * \brief White balance gain for G channel > + * > + * \var IPAFrameContext::awb.gains.blue > + * \brief White balance gain for B channel > + * > + * \var IPAFrameContext::awb.temperatureK > + * \brief Estimated color temperature > + */ > + > /** > * \var IPAFrameContext::sensor > * \brief Effective sensor values > diff --git a/src/ipa/rkisp1/ipa_context.h b/src/ipa/rkisp1/ipa_context.h > index 9ac3b40c..51eae8b6 100644 > --- a/src/ipa/rkisp1/ipa_context.h > +++ b/src/ipa/rkisp1/ipa_context.h > @@ -12,6 +12,8 @@ > > #include <libcamera/base/utils.h> > > +#include <libcamera/geometry.h> > + > namespace libcamera { > > namespace ipa::rkisp1 { > @@ -25,6 +27,10 @@ struct IPASessionConfiguration { > struct rkisp1_cif_isp_window measureWindow; > } agc; > > + struct { > + struct rkisp1_cif_isp_window measureWindow; > + } awb; > + > struct { > utils::Duration lineDuration; > } sensor; > @@ -40,6 +46,16 @@ struct IPAFrameContext { > double gain; > } agc; > > + struct { > + struct { > + double red; > + double green; > + double blue; > + } gains; > + > + double temperatureK; > + } awb; > + > struct { > uint32_t exposure; > double gain; > diff --git a/src/ipa/rkisp1/rkisp1.cpp b/src/ipa/rkisp1/rkisp1.cpp > index f82b7cb3..381fc0e0 100644 > --- a/src/ipa/rkisp1/rkisp1.cpp > +++ b/src/ipa/rkisp1/rkisp1.cpp > @@ -27,6 +27,7 @@ > > #include "algorithms/agc.h" > #include "algorithms/algorithm.h" > +#include "algorithms/awb.h" > #include "algorithms/blc.h" > #include "libipa/camera_sensor_helper.h" > > @@ -126,6 +127,7 @@ int IPARkISP1::init(const IPASettings &settings, unsigned int hwRevision) > > /* Construct our Algorithms */ > algorithms_.push_back(std::make_unique<algorithms::Agc>()); > + algorithms_.push_back(std::make_unique<algorithms::Awb>()); > algorithms_.push_back(std::make_unique<algorithms::BlackLevelCorrection>()); > > return 0;
Hi Laurent, On 24/02/2022 08:13, Laurent Pinchart wrote: > Hi Jean-Michel, > > Thank you for the patch. > > On Wed, Feb 23, 2022 at 11:48:24AM +0100, Jean-Michel Hautbois wrote: >> The RkISP1 ISP calculates a mean value for Y, Cr and Cb at each frame. >> There is a RGB mode which could theoretically give us the values for R, >> G and B directly, but it seems to be failing right now. >> >> Convert those values into R, G and B and estimate the gain to apply in a >> grey world. >> >> Signed-off-by: Jean-Michel Hautbois <jeanmichel.hautbois@ideasonboard.com> >> Tested-by: Peter Griffin <peter.griffin@linaro.org> >> --- >> src/ipa/rkisp1/algorithms/awb.cpp | 155 ++++++++++++++++++++++++++ >> src/ipa/rkisp1/algorithms/awb.h | 33 ++++++ >> src/ipa/rkisp1/algorithms/meson.build | 1 + >> src/ipa/rkisp1/ipa_context.cpp | 28 +++++ >> src/ipa/rkisp1/ipa_context.h | 16 +++ >> src/ipa/rkisp1/rkisp1.cpp | 2 + >> 6 files changed, 235 insertions(+) >> create mode 100644 src/ipa/rkisp1/algorithms/awb.cpp >> create mode 100644 src/ipa/rkisp1/algorithms/awb.h >> >> diff --git a/src/ipa/rkisp1/algorithms/awb.cpp b/src/ipa/rkisp1/algorithms/awb.cpp >> new file mode 100644 >> index 00000000..8fa8c2f7 >> --- /dev/null >> +++ b/src/ipa/rkisp1/algorithms/awb.cpp >> @@ -0,0 +1,155 @@ >> +/* SPDX-License-Identifier: LGPL-2.1-or-later */ >> +/* >> + * Copyright (C) 2021, Ideas On Board > > 2021-2022 > > Time flies :-) Thanks, I will apply it on all the other files too then ;-). > >> + * >> + * awb.cpp - AWB control algorithm >> + */ >> + >> +#include "awb.h" >> + >> +#include <algorithm> >> +#include <cmath> >> + >> +#include <libcamera/base/log.h> >> + >> +#include <libcamera/ipa/core_ipa_interface.h> >> + >> +/** >> + * \file awb.h >> + */ >> + >> +namespace libcamera { >> + >> +namespace ipa::rkisp1::algorithms { >> + >> +/** >> + * \class Awb >> + * \brief A Grey world white balance correction algorithm >> + */ >> + >> +LOG_DEFINE_CATEGORY(RkISP1Awb) >> + >> +/** >> + * \copydoc libcamera::ipa::Algorithm::configure >> + */ >> +int Awb::configure(IPAContext &context, >> + const IPACameraSensorInfo &configInfo) >> +{ >> + context.frameContext.awb.gains.red = 1.0; >> + context.frameContext.awb.gains.blue = 1.0; >> + context.frameContext.awb.gains.green = 1.0; >> + >> + /* Define the measurement window for AGC. */ > > Same comment as for patch 3/4. > >> + context.configuration.awb.measureWindow.h_offs = configInfo.outputSize.width / 8; >> + context.configuration.awb.measureWindow.v_offs = configInfo.outputSize.height / 8; >> + context.configuration.awb.measureWindow.h_size = 3 * configInfo.outputSize.width / 4; >> + context.configuration.awb.measureWindow.v_size = 3 * configInfo.outputSize.height / 4; >> + >> + return 0; >> +} >> + >> +uint32_t Awb::estimateCCT(double red, double green, double blue) >> +{ >> + /* Convert the RGB values to CIE tristimulus values (XYZ) */ >> + double X = (-0.14282) * (red) + (1.54924) * (green) + (-0.95641) * (blue); >> + double Y = (-0.32466) * (red) + (1.57837) * (green) + (-0.73191) * (blue); >> + double Z = (-0.68202) * (red) + (0.77073) * (green) + (0.56332) * (blue); >> + >> + /* Calculate the normalized chromaticity values */ >> + double x = X / (X + Y + Z); >> + double y = Y / (X + Y + Z); >> + >> + /* Calculate CCT */ >> + double n = (x - 0.3320) / (0.1858 - y); >> + return 449 * n * n * n + 3525 * n * n + 6823.3 * n + 5520.33; >> +} >> + >> +/** >> + * \copydoc libcamera::ipa::Algorithm::process >> + */ >> +void Awb::process([[maybe_unused]] IPAContext &context, const rkisp1_stat_buffer *stats) > > Please move this after prepare() to match the order of the .h file. > >> +{ >> + const rkisp1_cif_isp_stat *params = &stats->params; >> + const rkisp1_cif_isp_awb_stat *awb = ¶ms->awb; >> + >> + /* Get the YCbCr mean values */ >> + double yMean = awb->awb_mean[0].mean_y_or_g; >> + double crMean = awb->awb_mean[0].mean_cr_or_r; >> + double cbMean = awb->awb_mean[0].mean_cb_or_b; >> + >> + /* Convert from YCbCr to RGB. */ >> + double redMean = yMean + 1.402 * (crMean - 128); >> + double blueMean = yMean + 1.772 * (cbMean - 128); >> + double greenMean = yMean - 0.34414 * (cbMean - 128) - 0.71414 * (crMean - 128); > > Where do those values come from ? Unless I'm mistaken, the hardware uses > the following formulas to convert from RGB to YCbCr. > > Y = 16 + 0.2500 R + 0.5000 G + 0.1094 B > Cb = 128 - 0.1406 R - 0.2969 G + 0.4375 B > Cr = 128 + 0.4375 R - 0.3750 G - 0.0625 B > I took the JPEG File Interchange Format conversion formula, and not the inverse matrix of the one the HW is using indeed. I will need to calculate it then (keeping in mind that I convert from YCrCb because the RGB mode is not working -yet- )... >> + >> + /* Estimate the red and blue gains to apply in a grey world. */ >> + double redGain = greenMean / (redMean + 1); >> + double blueGain = greenMean / (blueMean + 1); >> + >> + /* Filter the values to avoid oscillations. */ >> + IPAFrameContext &frameContext = context.frameContext; >> + >> + frameContext.awb.temperatureK = estimateCCT(redMean, greenMean, blueMean); >> + frameContext.awb.gains.red = 0.2 * redGain + >> + 0.8 * frameContext.awb.gains.red; >> + frameContext.awb.gains.blue = 0.2 * blueGain + >> + 0.8 * frameContext.awb.gains.blue; > > Don't you need to saturate the gains, either here, or in prepare() ? > >> + /* Hardcode the green gain to 1.0. */ >> + frameContext.awb.gains.green = 1.0; >> + >> + LOG(RkISP1Awb, Debug) << "Gain found for red: " << context.frameContext.awb.gains.red >> + << " and for blue: " << context.frameContext.awb.gains.blue; >> +} >> + >> +/** >> + * \copydoc libcamera::ipa::Algorithm::prepare >> + */ >> +void Awb::prepare(IPAContext &context, >> + rkisp1_params_cfg *params) > > This holds on a single line. > >> +{ >> + params->others.awb_gain_config.gain_green_b = 256 * context.frameContext.awb.gains.green; >> + params->others.awb_gain_config.gain_blue = 256 * context.frameContext.awb.gains.blue; >> + params->others.awb_gain_config.gain_red = 256 * context.frameContext.awb.gains.red; >> + params->others.awb_gain_config.gain_green_r = 256 * context.frameContext.awb.gains.green; >> + >> + if (context.frameContext.frameId == 0) { >> + /* Configure the gains to apply. */ >> + params->module_en_update |= RKISP1_CIF_ISP_MODULE_AWB_GAIN; >> + /* Update the ISP to apply the gains configured. */ >> + params->module_ens |= RKISP1_CIF_ISP_MODULE_AWB_GAIN; >> + } >> + /* Update the gains. */ >> + params->module_cfg_update |= RKISP1_CIF_ISP_MODULE_AWB_GAIN; > > I'd move this line just after setting the gains above. > >> + >> + /* Configure the measure window for AWB. */ >> + params->meas.awb_meas_config.awb_wnd = context.configuration.awb.measureWindow; >> + /* >> + * Measure Y, Cr and Cb means. >> + * \todo RGB is not working, the kernel seems to not configure it ? > > Does this mean that if RGB worked, you would prefer using that ? Yes, as it would remove the need for the YCrCb to RGB conversion. > >> + */ >> + params->meas.awb_meas_config.awb_mode = RKISP1_CIF_ISP_AWB_MODE_YCBCR; >> + /* Reference Cr and Cb. */ >> + params->meas.awb_meas_config.awb_ref_cb = 128; >> + params->meas.awb_meas_config.awb_ref_cr = 128; >> + /* Y values to include are between min_y and max_y only. */ >> + params->meas.awb_meas_config.min_y = 16; >> + params->meas.awb_meas_config.max_y = 250; >> + /* Maximum Cr+Cb value to take into account for awb. */ >> + params->meas.awb_meas_config.max_csum = 250; >> + /* Minimum Cr and Cb values to take into account. */ >> + params->meas.awb_meas_config.min_c = 16; >> + /* Number of frames to use to estimate the mean (0 means 1 frame). */ >> + params->meas.awb_meas_config.frames = 0; > > All this can move inside the if () below. > >> + >> + if (context.frameContext.frameId == 0) { >> + /* Update AWB measurement unit configuration. */ >> + params->module_cfg_update |= RKISP1_CIF_ISP_MODULE_AWB; >> + /* Make sure the ISP is measuring the means for the next frame. */ >> + params->module_en_update |= RKISP1_CIF_ISP_MODULE_AWB; >> + params->module_ens |= RKISP1_CIF_ISP_MODULE_AWB; >> + } >> +} >> + >> +} /* namespace ipa::rkisp1::algorithms */ >> + >> +} /* namespace libcamera */ >> diff --git a/src/ipa/rkisp1/algorithms/awb.h b/src/ipa/rkisp1/algorithms/awb.h >> new file mode 100644 >> index 00000000..0a9fb82c >> --- /dev/null >> +++ b/src/ipa/rkisp1/algorithms/awb.h >> @@ -0,0 +1,33 @@ >> +/* SPDX-License-Identifier: LGPL-2.1-or-later */ >> +/* >> + * Copyright (C) 2021, Ideas On Board > > 2021-2022 > >> + * >> + * awb.h - AWB control algorithm >> + */ >> + >> +#pragma once >> + >> +#include <linux/rkisp1-config.h> >> + >> +#include "algorithm.h" >> + >> +namespace libcamera { >> + >> +namespace ipa::rkisp1::algorithms { >> + >> +class Awb : public Algorithm >> +{ >> +public: >> + Awb() = default; >> + ~Awb() = default; >> + >> + int configure(IPAContext &context, const IPACameraSensorInfo &configInfo) override; >> + void prepare(IPAContext &context, rkisp1_params_cfg *params) override; >> + void process(IPAContext &context, const rkisp1_stat_buffer *stats) override; >> + >> +private: >> + uint32_t estimateCCT(double red, double green, double blue); >> +}; >> + >> +} /* namespace ipa::rkisp1::algorithms */ >> +} /* namespace libcamera */ >> diff --git a/src/ipa/rkisp1/algorithms/meson.build b/src/ipa/rkisp1/algorithms/meson.build >> index 27c97731..7ec53d89 100644 >> --- a/src/ipa/rkisp1/algorithms/meson.build >> +++ b/src/ipa/rkisp1/algorithms/meson.build >> @@ -2,5 +2,6 @@ >> >> rkisp1_ipa_algorithms = files([ >> 'agc.cpp', >> + 'awb.cpp', >> 'blc.cpp', >> ]) >> diff --git a/src/ipa/rkisp1/ipa_context.cpp b/src/ipa/rkisp1/ipa_context.cpp >> index 39acef47..c25f44ec 100644 >> --- a/src/ipa/rkisp1/ipa_context.cpp >> +++ b/src/ipa/rkisp1/ipa_context.cpp >> @@ -81,6 +81,14 @@ namespace libcamera::ipa::rkisp1 { >> * \brief Hardware revision of the ISP >> */ >> >> +/** >> + * \var IPASessionConfiguration::awb >> + * \brief AWB parameters configuration of the IPA >> + * >> + * \var IPASessionConfiguration::awb.measureWindow >> + * \brief AWB measure window >> + */ >> + >> /** >> * \var IPASessionConfiguration::sensor >> * \brief Sensor-specific configuration of the IPA >> @@ -105,6 +113,26 @@ namespace libcamera::ipa::rkisp1 { >> * The gain should be adapted to the sensor specific gain code before applying. >> */ >> >> +/** >> + * \var IPAFrameContext::awb >> + * \brief Context for the Automatic White Balance algorithm >> + * >> + * \struct IPAFrameContext::awb.gains >> + * \brief White balance gains >> + * >> + * \var IPAFrameContext::awb.gains.red >> + * \brief White balance gain for R channel >> + * >> + * \var IPAFrameContext::awb.gains.green >> + * \brief White balance gain for G channel >> + * >> + * \var IPAFrameContext::awb.gains.blue >> + * \brief White balance gain for B channel >> + * >> + * \var IPAFrameContext::awb.temperatureK >> + * \brief Estimated color temperature >> + */ >> + >> /** >> * \var IPAFrameContext::sensor >> * \brief Effective sensor values >> diff --git a/src/ipa/rkisp1/ipa_context.h b/src/ipa/rkisp1/ipa_context.h >> index 9ac3b40c..51eae8b6 100644 >> --- a/src/ipa/rkisp1/ipa_context.h >> +++ b/src/ipa/rkisp1/ipa_context.h >> @@ -12,6 +12,8 @@ >> >> #include <libcamera/base/utils.h> >> >> +#include <libcamera/geometry.h> >> + >> namespace libcamera { >> >> namespace ipa::rkisp1 { >> @@ -25,6 +27,10 @@ struct IPASessionConfiguration { >> struct rkisp1_cif_isp_window measureWindow; >> } agc; >> >> + struct { >> + struct rkisp1_cif_isp_window measureWindow; >> + } awb; >> + >> struct { >> utils::Duration lineDuration; >> } sensor; >> @@ -40,6 +46,16 @@ struct IPAFrameContext { >> double gain; >> } agc; >> >> + struct { >> + struct { >> + double red; >> + double green; >> + double blue; >> + } gains; >> + >> + double temperatureK; >> + } awb; >> + >> struct { >> uint32_t exposure; >> double gain; >> diff --git a/src/ipa/rkisp1/rkisp1.cpp b/src/ipa/rkisp1/rkisp1.cpp >> index f82b7cb3..381fc0e0 100644 >> --- a/src/ipa/rkisp1/rkisp1.cpp >> +++ b/src/ipa/rkisp1/rkisp1.cpp >> @@ -27,6 +27,7 @@ >> >> #include "algorithms/agc.h" >> #include "algorithms/algorithm.h" >> +#include "algorithms/awb.h" >> #include "algorithms/blc.h" >> #include "libipa/camera_sensor_helper.h" >> >> @@ -126,6 +127,7 @@ int IPARkISP1::init(const IPASettings &settings, unsigned int hwRevision) >> >> /* Construct our Algorithms */ >> algorithms_.push_back(std::make_unique<algorithms::Agc>()); >> + algorithms_.push_back(std::make_unique<algorithms::Awb>()); >> algorithms_.push_back(std::make_unique<algorithms::BlackLevelCorrection>()); >> >> return 0; >
Hi Jean-Michel, On Thu, Feb 24, 2022 at 09:11:14AM +0100, Jean-Michel Hautbois wrote: > On 24/02/2022 08:13, Laurent Pinchart wrote: > > On Wed, Feb 23, 2022 at 11:48:24AM +0100, Jean-Michel Hautbois wrote: > >> The RkISP1 ISP calculates a mean value for Y, Cr and Cb at each frame. > >> There is a RGB mode which could theoretically give us the values for R, > >> G and B directly, but it seems to be failing right now. > >> > >> Convert those values into R, G and B and estimate the gain to apply in a > >> grey world. > >> > >> Signed-off-by: Jean-Michel Hautbois <jeanmichel.hautbois@ideasonboard.com> > >> Tested-by: Peter Griffin <peter.griffin@linaro.org> > >> --- > >> src/ipa/rkisp1/algorithms/awb.cpp | 155 ++++++++++++++++++++++++++ > >> src/ipa/rkisp1/algorithms/awb.h | 33 ++++++ > >> src/ipa/rkisp1/algorithms/meson.build | 1 + > >> src/ipa/rkisp1/ipa_context.cpp | 28 +++++ > >> src/ipa/rkisp1/ipa_context.h | 16 +++ > >> src/ipa/rkisp1/rkisp1.cpp | 2 + > >> 6 files changed, 235 insertions(+) > >> create mode 100644 src/ipa/rkisp1/algorithms/awb.cpp > >> create mode 100644 src/ipa/rkisp1/algorithms/awb.h > >> > >> diff --git a/src/ipa/rkisp1/algorithms/awb.cpp b/src/ipa/rkisp1/algorithms/awb.cpp > >> new file mode 100644 > >> index 00000000..8fa8c2f7 > >> --- /dev/null > >> +++ b/src/ipa/rkisp1/algorithms/awb.cpp > >> @@ -0,0 +1,155 @@ > >> +/* SPDX-License-Identifier: LGPL-2.1-or-later */ > >> +/* > >> + * Copyright (C) 2021, Ideas On Board > > > > 2021-2022 > > > > Time flies :-) > > Thanks, I will apply it on all the other files too then ;-). > > >> + * > >> + * awb.cpp - AWB control algorithm > >> + */ > >> + > >> +#include "awb.h" > >> + > >> +#include <algorithm> > >> +#include <cmath> > >> + > >> +#include <libcamera/base/log.h> > >> + > >> +#include <libcamera/ipa/core_ipa_interface.h> > >> + > >> +/** > >> + * \file awb.h > >> + */ > >> + > >> +namespace libcamera { > >> + > >> +namespace ipa::rkisp1::algorithms { > >> + > >> +/** > >> + * \class Awb > >> + * \brief A Grey world white balance correction algorithm > >> + */ > >> + > >> +LOG_DEFINE_CATEGORY(RkISP1Awb) > >> + > >> +/** > >> + * \copydoc libcamera::ipa::Algorithm::configure > >> + */ > >> +int Awb::configure(IPAContext &context, > >> + const IPACameraSensorInfo &configInfo) > >> +{ > >> + context.frameContext.awb.gains.red = 1.0; > >> + context.frameContext.awb.gains.blue = 1.0; > >> + context.frameContext.awb.gains.green = 1.0; > >> + > >> + /* Define the measurement window for AGC. */ > > > > Same comment as for patch 3/4. > > > >> + context.configuration.awb.measureWindow.h_offs = configInfo.outputSize.width / 8; > >> + context.configuration.awb.measureWindow.v_offs = configInfo.outputSize.height / 8; > >> + context.configuration.awb.measureWindow.h_size = 3 * configInfo.outputSize.width / 4; > >> + context.configuration.awb.measureWindow.v_size = 3 * configInfo.outputSize.height / 4; > >> + > >> + return 0; > >> +} > >> + > >> +uint32_t Awb::estimateCCT(double red, double green, double blue) > >> +{ > >> + /* Convert the RGB values to CIE tristimulus values (XYZ) */ > >> + double X = (-0.14282) * (red) + (1.54924) * (green) + (-0.95641) * (blue); > >> + double Y = (-0.32466) * (red) + (1.57837) * (green) + (-0.73191) * (blue); > >> + double Z = (-0.68202) * (red) + (0.77073) * (green) + (0.56332) * (blue); > >> + > >> + /* Calculate the normalized chromaticity values */ > >> + double x = X / (X + Y + Z); > >> + double y = Y / (X + Y + Z); > >> + > >> + /* Calculate CCT */ > >> + double n = (x - 0.3320) / (0.1858 - y); > >> + return 449 * n * n * n + 3525 * n * n + 6823.3 * n + 5520.33; > >> +} > >> + > >> +/** > >> + * \copydoc libcamera::ipa::Algorithm::process > >> + */ > >> +void Awb::process([[maybe_unused]] IPAContext &context, const rkisp1_stat_buffer *stats) > > > > Please move this after prepare() to match the order of the .h file. > > > >> +{ > >> + const rkisp1_cif_isp_stat *params = &stats->params; > >> + const rkisp1_cif_isp_awb_stat *awb = ¶ms->awb; > >> + > >> + /* Get the YCbCr mean values */ > >> + double yMean = awb->awb_mean[0].mean_y_or_g; > >> + double crMean = awb->awb_mean[0].mean_cr_or_r; > >> + double cbMean = awb->awb_mean[0].mean_cb_or_b; > >> + > >> + /* Convert from YCbCr to RGB. */ > >> + double redMean = yMean + 1.402 * (crMean - 128); > >> + double blueMean = yMean + 1.772 * (cbMean - 128); > >> + double greenMean = yMean - 0.34414 * (cbMean - 128) - 0.71414 * (crMean - 128); > > > > Where do those values come from ? Unless I'm mistaken, the hardware uses > > the following formulas to convert from RGB to YCbCr. > > > > Y = 16 + 0.2500 R + 0.5000 G + 0.1094 B > > Cb = 128 - 0.1406 R - 0.2969 G + 0.4375 B > > Cr = 128 + 0.4375 R - 0.3750 G - 0.0625 B > > I took the JPEG File Interchange Format conversion formula, and not the > inverse matrix of the one the HW is using indeed. > I will need to calculate it then (keeping in mind that I convert from > YCrCb because the RGB mode is not working -yet- )... numpy to the rescue. import numpy as np print(np.linalg.inv(np.array([[0.2500, 0.5000, 0.1094], [-0.1406, -0.2969, 0.4375], [0.4375, -0.3750, -0.0625]]))) [[ 1.16360251 -0.06228394 1.60078229] [ 1.16360251 -0.404527 -0.79491914] [ 1.16360251 1.99117443 -0.02500915]] > >> + > >> + /* Estimate the red and blue gains to apply in a grey world. */ > >> + double redGain = greenMean / (redMean + 1); > >> + double blueGain = greenMean / (blueMean + 1); > >> + > >> + /* Filter the values to avoid oscillations. */ > >> + IPAFrameContext &frameContext = context.frameContext; > >> + > >> + frameContext.awb.temperatureK = estimateCCT(redMean, greenMean, blueMean); > >> + frameContext.awb.gains.red = 0.2 * redGain + > >> + 0.8 * frameContext.awb.gains.red; > >> + frameContext.awb.gains.blue = 0.2 * blueGain + > >> + 0.8 * frameContext.awb.gains.blue; > > > > Don't you need to saturate the gains, either here, or in prepare() ? > > > >> + /* Hardcode the green gain to 1.0. */ > >> + frameContext.awb.gains.green = 1.0; > >> + > >> + LOG(RkISP1Awb, Debug) << "Gain found for red: " << context.frameContext.awb.gains.red > >> + << " and for blue: " << context.frameContext.awb.gains.blue; > >> +} > >> + > >> +/** > >> + * \copydoc libcamera::ipa::Algorithm::prepare > >> + */ > >> +void Awb::prepare(IPAContext &context, > >> + rkisp1_params_cfg *params) > > > > This holds on a single line. > > > >> +{ > >> + params->others.awb_gain_config.gain_green_b = 256 * context.frameContext.awb.gains.green; > >> + params->others.awb_gain_config.gain_blue = 256 * context.frameContext.awb.gains.blue; > >> + params->others.awb_gain_config.gain_red = 256 * context.frameContext.awb.gains.red; > >> + params->others.awb_gain_config.gain_green_r = 256 * context.frameContext.awb.gains.green; > >> + > >> + if (context.frameContext.frameId == 0) { > >> + /* Configure the gains to apply. */ > >> + params->module_en_update |= RKISP1_CIF_ISP_MODULE_AWB_GAIN; > >> + /* Update the ISP to apply the gains configured. */ > >> + params->module_ens |= RKISP1_CIF_ISP_MODULE_AWB_GAIN; > >> + } > >> + /* Update the gains. */ > >> + params->module_cfg_update |= RKISP1_CIF_ISP_MODULE_AWB_GAIN; > > > > I'd move this line just after setting the gains above. > > > >> + > >> + /* Configure the measure window for AWB. */ > >> + params->meas.awb_meas_config.awb_wnd = context.configuration.awb.measureWindow; > >> + /* > >> + * Measure Y, Cr and Cb means. > >> + * \todo RGB is not working, the kernel seems to not configure it ? > > > > Does this mean that if RGB worked, you would prefer using that ? > > Yes, as it would remove the need for the YCrCb to RGB conversion. > > >> + */ > >> + params->meas.awb_meas_config.awb_mode = RKISP1_CIF_ISP_AWB_MODE_YCBCR; > >> + /* Reference Cr and Cb. */ > >> + params->meas.awb_meas_config.awb_ref_cb = 128; > >> + params->meas.awb_meas_config.awb_ref_cr = 128; > >> + /* Y values to include are between min_y and max_y only. */ > >> + params->meas.awb_meas_config.min_y = 16; > >> + params->meas.awb_meas_config.max_y = 250; > >> + /* Maximum Cr+Cb value to take into account for awb. */ > >> + params->meas.awb_meas_config.max_csum = 250; > >> + /* Minimum Cr and Cb values to take into account. */ > >> + params->meas.awb_meas_config.min_c = 16; > >> + /* Number of frames to use to estimate the mean (0 means 1 frame). */ > >> + params->meas.awb_meas_config.frames = 0; > > > > All this can move inside the if () below. > > > >> + > >> + if (context.frameContext.frameId == 0) { > >> + /* Update AWB measurement unit configuration. */ > >> + params->module_cfg_update |= RKISP1_CIF_ISP_MODULE_AWB; > >> + /* Make sure the ISP is measuring the means for the next frame. */ > >> + params->module_en_update |= RKISP1_CIF_ISP_MODULE_AWB; > >> + params->module_ens |= RKISP1_CIF_ISP_MODULE_AWB; > >> + } > >> +} > >> + > >> +} /* namespace ipa::rkisp1::algorithms */ > >> + > >> +} /* namespace libcamera */ > >> diff --git a/src/ipa/rkisp1/algorithms/awb.h b/src/ipa/rkisp1/algorithms/awb.h > >> new file mode 100644 > >> index 00000000..0a9fb82c > >> --- /dev/null > >> +++ b/src/ipa/rkisp1/algorithms/awb.h > >> @@ -0,0 +1,33 @@ > >> +/* SPDX-License-Identifier: LGPL-2.1-or-later */ > >> +/* > >> + * Copyright (C) 2021, Ideas On Board > > > > 2021-2022 > > > >> + * > >> + * awb.h - AWB control algorithm > >> + */ > >> + > >> +#pragma once > >> + > >> +#include <linux/rkisp1-config.h> > >> + > >> +#include "algorithm.h" > >> + > >> +namespace libcamera { > >> + > >> +namespace ipa::rkisp1::algorithms { > >> + > >> +class Awb : public Algorithm > >> +{ > >> +public: > >> + Awb() = default; > >> + ~Awb() = default; > >> + > >> + int configure(IPAContext &context, const IPACameraSensorInfo &configInfo) override; > >> + void prepare(IPAContext &context, rkisp1_params_cfg *params) override; > >> + void process(IPAContext &context, const rkisp1_stat_buffer *stats) override; > >> + > >> +private: > >> + uint32_t estimateCCT(double red, double green, double blue); > >> +}; > >> + > >> +} /* namespace ipa::rkisp1::algorithms */ > >> +} /* namespace libcamera */ > >> diff --git a/src/ipa/rkisp1/algorithms/meson.build b/src/ipa/rkisp1/algorithms/meson.build > >> index 27c97731..7ec53d89 100644 > >> --- a/src/ipa/rkisp1/algorithms/meson.build > >> +++ b/src/ipa/rkisp1/algorithms/meson.build > >> @@ -2,5 +2,6 @@ > >> > >> rkisp1_ipa_algorithms = files([ > >> 'agc.cpp', > >> + 'awb.cpp', > >> 'blc.cpp', > >> ]) > >> diff --git a/src/ipa/rkisp1/ipa_context.cpp b/src/ipa/rkisp1/ipa_context.cpp > >> index 39acef47..c25f44ec 100644 > >> --- a/src/ipa/rkisp1/ipa_context.cpp > >> +++ b/src/ipa/rkisp1/ipa_context.cpp > >> @@ -81,6 +81,14 @@ namespace libcamera::ipa::rkisp1 { > >> * \brief Hardware revision of the ISP > >> */ > >> > >> +/** > >> + * \var IPASessionConfiguration::awb > >> + * \brief AWB parameters configuration of the IPA > >> + * > >> + * \var IPASessionConfiguration::awb.measureWindow > >> + * \brief AWB measure window > >> + */ > >> + > >> /** > >> * \var IPASessionConfiguration::sensor > >> * \brief Sensor-specific configuration of the IPA > >> @@ -105,6 +113,26 @@ namespace libcamera::ipa::rkisp1 { > >> * The gain should be adapted to the sensor specific gain code before applying. > >> */ > >> > >> +/** > >> + * \var IPAFrameContext::awb > >> + * \brief Context for the Automatic White Balance algorithm > >> + * > >> + * \struct IPAFrameContext::awb.gains > >> + * \brief White balance gains > >> + * > >> + * \var IPAFrameContext::awb.gains.red > >> + * \brief White balance gain for R channel > >> + * > >> + * \var IPAFrameContext::awb.gains.green > >> + * \brief White balance gain for G channel > >> + * > >> + * \var IPAFrameContext::awb.gains.blue > >> + * \brief White balance gain for B channel > >> + * > >> + * \var IPAFrameContext::awb.temperatureK > >> + * \brief Estimated color temperature > >> + */ > >> + > >> /** > >> * \var IPAFrameContext::sensor > >> * \brief Effective sensor values > >> diff --git a/src/ipa/rkisp1/ipa_context.h b/src/ipa/rkisp1/ipa_context.h > >> index 9ac3b40c..51eae8b6 100644 > >> --- a/src/ipa/rkisp1/ipa_context.h > >> +++ b/src/ipa/rkisp1/ipa_context.h > >> @@ -12,6 +12,8 @@ > >> > >> #include <libcamera/base/utils.h> > >> > >> +#include <libcamera/geometry.h> > >> + > >> namespace libcamera { > >> > >> namespace ipa::rkisp1 { > >> @@ -25,6 +27,10 @@ struct IPASessionConfiguration { > >> struct rkisp1_cif_isp_window measureWindow; > >> } agc; > >> > >> + struct { > >> + struct rkisp1_cif_isp_window measureWindow; > >> + } awb; > >> + > >> struct { > >> utils::Duration lineDuration; > >> } sensor; > >> @@ -40,6 +46,16 @@ struct IPAFrameContext { > >> double gain; > >> } agc; > >> > >> + struct { > >> + struct { > >> + double red; > >> + double green; > >> + double blue; > >> + } gains; > >> + > >> + double temperatureK; > >> + } awb; > >> + > >> struct { > >> uint32_t exposure; > >> double gain; > >> diff --git a/src/ipa/rkisp1/rkisp1.cpp b/src/ipa/rkisp1/rkisp1.cpp > >> index f82b7cb3..381fc0e0 100644 > >> --- a/src/ipa/rkisp1/rkisp1.cpp > >> +++ b/src/ipa/rkisp1/rkisp1.cpp > >> @@ -27,6 +27,7 @@ > >> > >> #include "algorithms/agc.h" > >> #include "algorithms/algorithm.h" > >> +#include "algorithms/awb.h" > >> #include "algorithms/blc.h" > >> #include "libipa/camera_sensor_helper.h" > >> > >> @@ -126,6 +127,7 @@ int IPARkISP1::init(const IPASettings &settings, unsigned int hwRevision) > >> > >> /* Construct our Algorithms */ > >> algorithms_.push_back(std::make_unique<algorithms::Agc>()); > >> + algorithms_.push_back(std::make_unique<algorithms::Awb>()); > >> algorithms_.push_back(std::make_unique<algorithms::BlackLevelCorrection>()); > >> > >> return 0;
diff --git a/src/ipa/rkisp1/algorithms/awb.cpp b/src/ipa/rkisp1/algorithms/awb.cpp new file mode 100644 index 00000000..8fa8c2f7 --- /dev/null +++ b/src/ipa/rkisp1/algorithms/awb.cpp @@ -0,0 +1,155 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2021, Ideas On Board + * + * awb.cpp - AWB control algorithm + */ + +#include "awb.h" + +#include <algorithm> +#include <cmath> + +#include <libcamera/base/log.h> + +#include <libcamera/ipa/core_ipa_interface.h> + +/** + * \file awb.h + */ + +namespace libcamera { + +namespace ipa::rkisp1::algorithms { + +/** + * \class Awb + * \brief A Grey world white balance correction algorithm + */ + +LOG_DEFINE_CATEGORY(RkISP1Awb) + +/** + * \copydoc libcamera::ipa::Algorithm::configure + */ +int Awb::configure(IPAContext &context, + const IPACameraSensorInfo &configInfo) +{ + context.frameContext.awb.gains.red = 1.0; + context.frameContext.awb.gains.blue = 1.0; + context.frameContext.awb.gains.green = 1.0; + + /* Define the measurement window for AGC. */ + context.configuration.awb.measureWindow.h_offs = configInfo.outputSize.width / 8; + context.configuration.awb.measureWindow.v_offs = configInfo.outputSize.height / 8; + context.configuration.awb.measureWindow.h_size = 3 * configInfo.outputSize.width / 4; + context.configuration.awb.measureWindow.v_size = 3 * configInfo.outputSize.height / 4; + + return 0; +} + +uint32_t Awb::estimateCCT(double red, double green, double blue) +{ + /* Convert the RGB values to CIE tristimulus values (XYZ) */ + double X = (-0.14282) * (red) + (1.54924) * (green) + (-0.95641) * (blue); + double Y = (-0.32466) * (red) + (1.57837) * (green) + (-0.73191) * (blue); + double Z = (-0.68202) * (red) + (0.77073) * (green) + (0.56332) * (blue); + + /* Calculate the normalized chromaticity values */ + double x = X / (X + Y + Z); + double y = Y / (X + Y + Z); + + /* Calculate CCT */ + double n = (x - 0.3320) / (0.1858 - y); + return 449 * n * n * n + 3525 * n * n + 6823.3 * n + 5520.33; +} + +/** + * \copydoc libcamera::ipa::Algorithm::process + */ +void Awb::process([[maybe_unused]] IPAContext &context, const rkisp1_stat_buffer *stats) +{ + const rkisp1_cif_isp_stat *params = &stats->params; + const rkisp1_cif_isp_awb_stat *awb = ¶ms->awb; + + /* Get the YCbCr mean values */ + double yMean = awb->awb_mean[0].mean_y_or_g; + double crMean = awb->awb_mean[0].mean_cr_or_r; + double cbMean = awb->awb_mean[0].mean_cb_or_b; + + /* Convert from YCbCr to RGB. */ + double redMean = yMean + 1.402 * (crMean - 128); + double blueMean = yMean + 1.772 * (cbMean - 128); + double greenMean = yMean - 0.34414 * (cbMean - 128) - 0.71414 * (crMean - 128); + + /* Estimate the red and blue gains to apply in a grey world. */ + double redGain = greenMean / (redMean + 1); + double blueGain = greenMean / (blueMean + 1); + + /* Filter the values to avoid oscillations. */ + IPAFrameContext &frameContext = context.frameContext; + + frameContext.awb.temperatureK = estimateCCT(redMean, greenMean, blueMean); + frameContext.awb.gains.red = 0.2 * redGain + + 0.8 * frameContext.awb.gains.red; + frameContext.awb.gains.blue = 0.2 * blueGain + + 0.8 * frameContext.awb.gains.blue; + /* Hardcode the green gain to 1.0. */ + frameContext.awb.gains.green = 1.0; + + LOG(RkISP1Awb, Debug) << "Gain found for red: " << context.frameContext.awb.gains.red + << " and for blue: " << context.frameContext.awb.gains.blue; +} + +/** + * \copydoc libcamera::ipa::Algorithm::prepare + */ +void Awb::prepare(IPAContext &context, + rkisp1_params_cfg *params) +{ + params->others.awb_gain_config.gain_green_b = 256 * context.frameContext.awb.gains.green; + params->others.awb_gain_config.gain_blue = 256 * context.frameContext.awb.gains.blue; + params->others.awb_gain_config.gain_red = 256 * context.frameContext.awb.gains.red; + params->others.awb_gain_config.gain_green_r = 256 * context.frameContext.awb.gains.green; + + if (context.frameContext.frameId == 0) { + /* Configure the gains to apply. */ + params->module_en_update |= RKISP1_CIF_ISP_MODULE_AWB_GAIN; + /* Update the ISP to apply the gains configured. */ + params->module_ens |= RKISP1_CIF_ISP_MODULE_AWB_GAIN; + } + /* Update the gains. */ + params->module_cfg_update |= RKISP1_CIF_ISP_MODULE_AWB_GAIN; + + /* Configure the measure window for AWB. */ + params->meas.awb_meas_config.awb_wnd = context.configuration.awb.measureWindow; + /* + * Measure Y, Cr and Cb means. + * \todo RGB is not working, the kernel seems to not configure it ? + */ + params->meas.awb_meas_config.awb_mode = RKISP1_CIF_ISP_AWB_MODE_YCBCR; + /* Reference Cr and Cb. */ + params->meas.awb_meas_config.awb_ref_cb = 128; + params->meas.awb_meas_config.awb_ref_cr = 128; + /* Y values to include are between min_y and max_y only. */ + params->meas.awb_meas_config.min_y = 16; + params->meas.awb_meas_config.max_y = 250; + /* Maximum Cr+Cb value to take into account for awb. */ + params->meas.awb_meas_config.max_csum = 250; + /* Minimum Cr and Cb values to take into account. */ + params->meas.awb_meas_config.min_c = 16; + /* Number of frames to use to estimate the mean (0 means 1 frame). */ + params->meas.awb_meas_config.frames = 0; + + if (context.frameContext.frameId == 0) { + /* Update AWB measurement unit configuration. */ + params->module_cfg_update |= RKISP1_CIF_ISP_MODULE_AWB; + /* Make sure the ISP is measuring the means for the next frame. */ + params->module_en_update |= RKISP1_CIF_ISP_MODULE_AWB; + params->module_ens |= RKISP1_CIF_ISP_MODULE_AWB; + } +} + +} /* namespace ipa::rkisp1::algorithms */ + +} /* namespace libcamera */ diff --git a/src/ipa/rkisp1/algorithms/awb.h b/src/ipa/rkisp1/algorithms/awb.h new file mode 100644 index 00000000..0a9fb82c --- /dev/null +++ b/src/ipa/rkisp1/algorithms/awb.h @@ -0,0 +1,33 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2021, Ideas On Board + * + * awb.h - AWB control algorithm + */ + +#pragma once + +#include <linux/rkisp1-config.h> + +#include "algorithm.h" + +namespace libcamera { + +namespace ipa::rkisp1::algorithms { + +class Awb : public Algorithm +{ +public: + Awb() = default; + ~Awb() = default; + + int configure(IPAContext &context, const IPACameraSensorInfo &configInfo) override; + void prepare(IPAContext &context, rkisp1_params_cfg *params) override; + void process(IPAContext &context, const rkisp1_stat_buffer *stats) override; + +private: + uint32_t estimateCCT(double red, double green, double blue); +}; + +} /* namespace ipa::rkisp1::algorithms */ +} /* namespace libcamera */ diff --git a/src/ipa/rkisp1/algorithms/meson.build b/src/ipa/rkisp1/algorithms/meson.build index 27c97731..7ec53d89 100644 --- a/src/ipa/rkisp1/algorithms/meson.build +++ b/src/ipa/rkisp1/algorithms/meson.build @@ -2,5 +2,6 @@ rkisp1_ipa_algorithms = files([ 'agc.cpp', + 'awb.cpp', 'blc.cpp', ]) diff --git a/src/ipa/rkisp1/ipa_context.cpp b/src/ipa/rkisp1/ipa_context.cpp index 39acef47..c25f44ec 100644 --- a/src/ipa/rkisp1/ipa_context.cpp +++ b/src/ipa/rkisp1/ipa_context.cpp @@ -81,6 +81,14 @@ namespace libcamera::ipa::rkisp1 { * \brief Hardware revision of the ISP */ +/** + * \var IPASessionConfiguration::awb + * \brief AWB parameters configuration of the IPA + * + * \var IPASessionConfiguration::awb.measureWindow + * \brief AWB measure window + */ + /** * \var IPASessionConfiguration::sensor * \brief Sensor-specific configuration of the IPA @@ -105,6 +113,26 @@ namespace libcamera::ipa::rkisp1 { * The gain should be adapted to the sensor specific gain code before applying. */ +/** + * \var IPAFrameContext::awb + * \brief Context for the Automatic White Balance algorithm + * + * \struct IPAFrameContext::awb.gains + * \brief White balance gains + * + * \var IPAFrameContext::awb.gains.red + * \brief White balance gain for R channel + * + * \var IPAFrameContext::awb.gains.green + * \brief White balance gain for G channel + * + * \var IPAFrameContext::awb.gains.blue + * \brief White balance gain for B channel + * + * \var IPAFrameContext::awb.temperatureK + * \brief Estimated color temperature + */ + /** * \var IPAFrameContext::sensor * \brief Effective sensor values diff --git a/src/ipa/rkisp1/ipa_context.h b/src/ipa/rkisp1/ipa_context.h index 9ac3b40c..51eae8b6 100644 --- a/src/ipa/rkisp1/ipa_context.h +++ b/src/ipa/rkisp1/ipa_context.h @@ -12,6 +12,8 @@ #include <libcamera/base/utils.h> +#include <libcamera/geometry.h> + namespace libcamera { namespace ipa::rkisp1 { @@ -25,6 +27,10 @@ struct IPASessionConfiguration { struct rkisp1_cif_isp_window measureWindow; } agc; + struct { + struct rkisp1_cif_isp_window measureWindow; + } awb; + struct { utils::Duration lineDuration; } sensor; @@ -40,6 +46,16 @@ struct IPAFrameContext { double gain; } agc; + struct { + struct { + double red; + double green; + double blue; + } gains; + + double temperatureK; + } awb; + struct { uint32_t exposure; double gain; diff --git a/src/ipa/rkisp1/rkisp1.cpp b/src/ipa/rkisp1/rkisp1.cpp index f82b7cb3..381fc0e0 100644 --- a/src/ipa/rkisp1/rkisp1.cpp +++ b/src/ipa/rkisp1/rkisp1.cpp @@ -27,6 +27,7 @@ #include "algorithms/agc.h" #include "algorithms/algorithm.h" +#include "algorithms/awb.h" #include "algorithms/blc.h" #include "libipa/camera_sensor_helper.h" @@ -126,6 +127,7 @@ int IPARkISP1::init(const IPASettings &settings, unsigned int hwRevision) /* Construct our Algorithms */ algorithms_.push_back(std::make_unique<algorithms::Agc>()); + algorithms_.push_back(std::make_unique<algorithms::Awb>()); algorithms_.push_back(std::make_unique<algorithms::BlackLevelCorrection>()); return 0;