diff --git a/src/ipa/ipu3/algorithms/agc.cpp b/src/ipa/ipu3/algorithms/agc.cpp index b0f535b9902fa54d0ec3d689467d75613473cbcd..b30c40d93eb0f2aae6aec04a0f7fefa92978e8af 100644 --- a/src/ipa/ipu3/algorithms/agc.cpp +++ b/src/ipa/ipu3/algorithms/agc.cpp @@ -217,17 +217,15 @@ void Agc::process(IPAContext &context, [[maybe_unused]] const uint32_t frame, ControlList &metadata) { Histogram hist = parseStatistics(stats, context.configuration.grid.bdsGrid); + ipa::awb::Context awbContext; + + if (context.activeState.awb.autoEnabled) + awbContext = context.activeState.awb.automatic; + else + awbContext = context.activeState.awb.manual; agc_.process(context.configuration.agc, context.activeState.agc, frameContext.agc, {{ - .traits = AgcTraits{ - rgbTriples_, - {{ - context.activeState.awb.gains.red, - context.activeState.awb.gains.blue, - context.activeState.awb.gains.green, - }}, - bdsGrid_, - }, + .traits = AgcTraits{rgbTriples_, awbContext.gains, bdsGrid_}, .yHist = hist, .exposure = frameContext.sensor.exposure, .gain = frameContext.sensor.gain, diff --git a/src/ipa/ipu3/algorithms/awb.cpp b/src/ipa/ipu3/algorithms/awb.cpp index 613bdbbf51cd127e03906571813260a6857067dc..84d9f62e1c5836380776df785339ef847354583a 100644 --- a/src/ipa/ipu3/algorithms/awb.cpp +++ b/src/ipa/ipu3/algorithms/awb.cpp @@ -13,8 +13,6 @@ #include -#include "libipa/colours.h" - /** * \file awb.h */ @@ -26,11 +24,77 @@ namespace ipa::ipu3::algorithms { LOG_DEFINE_CATEGORY(IPU3Awb) /* - * When zones are used for the grey world algorithm, they are only considered if - * their average green value is at least 16/255 (after black level subtraction) - * to exclude zones that are too dark and don't provide relevant colour - * information (on the opposite side of the spectrum, saturated regions are - * excluded by the ImgU statistics engine). + * \todo IPU3 doesn't support the Lux algorithm. + */ +static constexpr unsigned int kDefaultLux = 500; + +/** + * \brief The IPU3 specific implementation of AwbStats + */ +class Ipu3AwbStats final : public AwbStats +{ +public: + Ipu3AwbStats() = default; + + /** + * \brief Constructs an Ipu3AwbStats object with the given RGB means + * + * \param rgbMeans A vector of doubles representing the RGB mean values + */ + Ipu3AwbStats(const RGB &rgbMeans) + : rgbMeans_(rgbMeans) + { + rg_ = rgbMeans_.r() / rgbMeans_.g(); + bg_ = rgbMeans_.b() / rgbMeans_.g(); + } + + double computeColourError(const RGB &gains) const override; + bool valid() const override; + RGB rgbMeans() const override; + +private: + RGB rgbMeans_; + double rg_; + double bg_; +}; + +double Ipu3AwbStats::computeColourError(const RGB &gains) const +{ + /* + * Compute the sum of the squared colour error (non-greyness) as + * it appears in the log likelihood equation. + */ + double deltaR = gains.r() * rg_ - 1.0; + double deltaB = gains.b() * bg_ - 1.0; + double delta2 = deltaR * deltaR + deltaB * deltaB; + + return delta2; +} + +bool Ipu3AwbStats::valid() const +{ + /* + * This validity assessment is designed to allow libipa to decide + * whether there's enough information in the statistics for a frame to + * be useful. The IPU3 implementation already drops any statistics zone + * with an average value below a threshold though so we don't need to do + * it in libipa. Report all stats as valid. + */ + + return true; +} + +RGB Ipu3AwbStats::rgbMeans() const +{ + return rgbMeans_; +} + +/* + * Zones are only considered if their average green value is at least + * kMinGreenLevelInZone/255 (after black level subtraction) to exclude zones + * that are too dark and don't provide relevant colour information (on the + * opposite side of the spectrum, saturated regions are excluded by the ImgU + * statistics engine). */ static constexpr uint32_t kMinGreenLevelInZone = 16; @@ -74,26 +138,6 @@ static constexpr uint32_t kMinCellsPerZoneRatio = 255 * 90 / 100; * \brief Sum of the average blue values of each unsaturated cell in the zone */ -/** - * \struct Awb::AwbStatus - * \brief AWB parameters calculated - * - * The AwbStatus structure is intended to store the AWB - * parameters calculated by the algorithm - * - * \var AwbStatus::temperatureK - * \brief Color temperature calculated - * - * \var AwbStatus::redGain - * \brief Gain calculated for the red channel - * - * \var AwbStatus::greenGain - * \brief Gain calculated for the green channel - * - * \var AwbStatus::blueGain - * \brief Gain calculated for the blue channel - */ - /* Default settings for Bayer noise reduction replicated from the Kernel */ static const struct ipu3_uapi_bnr_static_config imguCssBnrDefaults = { .wb_gains = { 16, 16, 16, 16 }, @@ -114,11 +158,7 @@ static const struct ipu3_uapi_bnr_static_config imguCssBnrDefaults = { /** * \class Awb - * \brief A Grey world white balance correction algorithm - * - * The Grey World algorithm assumes that the scene, in average, is neutral grey. - * Reference: Lam, Edmund & Fung, George. (2008). Automatic White Balancing in - * Digital Photography. 10.1201/9781420054538.ch10. + * \brief The IPU3 white balance correction algorithm implementation * * The IPU3 generates statistics from the Bayer Down Scaler output into a grid * defined in the ipu3_uapi_awb_config_s structure. @@ -168,26 +208,26 @@ static const struct ipu3_uapi_bnr_static_config imguCssBnrDefaults = { * cells are ignored. The grid configuration is computed by * IPAIPU3::calculateBdsGrid(). * - * Before calculating the gains, the algorithm aggregates the cell averages for - * each zone in generateAwbStats(). Cells that have a too high ratio of - * saturated pixels are ignored, and only zones that contain enough - * non-saturated cells are then used by the algorithm. - * - * The Grey World algorithm will then estimate the red and blue gains to apply, and - * store the results in the metadata. The green gain is always set to 1. + * Before running the AWB algorithm, we aggregate the cell averages for each + * zone in generateAwbStats(). Cells that have a too high ratio of saturated + * pixels are ignored, and only zones that contain enough non-saturated cells + * are then used by the algorithm. */ Awb::Awb() : Algorithm() { - asyncResults_.blueGain = 1.0; - asyncResults_.greenGain = 1.0; - asyncResults_.redGain = 1.0; - asyncResults_.temperatureK = 4500; - zones_.reserve(kAwbStatsSizeX * kAwbStatsSizeY); } +/** + * \copydoc libcamera::ipa::Algorithm::init + */ +int Awb::init(IPAContext &context, const ValueNode &tuningData) +{ + return awbAlgo_.init(tuningData, context.ctrlMap); +} + /** * \copydoc libcamera::ipa::Algorithm::configure */ @@ -197,12 +237,14 @@ int Awb::configure(IPAContext &context, const ipu3_uapi_grid_config &grid = context.configuration.grid.bdsGrid; stride_ = context.configuration.grid.stride; + awbAlgo_.configure(context.activeState.awb); + cellsPerZoneX_ = std::round(grid.width / static_cast(kAwbStatsSizeX)); cellsPerZoneY_ = std::round(grid.height / static_cast(kAwbStatsSizeY)); /* * Configure the minimum proportion of cells counted within a zone - * for it to be relevant for the grey world algorithm. + * for it to be used. * \todo This proportion could be configured. */ cellsPerZoneThreshold_ = cellsPerZoneX_ * cellsPerZoneY_ * kMaxCellSaturationRatio; @@ -211,6 +253,17 @@ int Awb::configure(IPAContext &context, return 0; } +/** + * \copydoc libcamera::ipa::Algorithm::queueRequest + */ +void Awb::queueRequest(IPAContext &context, const uint32_t frame, + IPAFrameContext &frameContext, + const ControlList &controls) +{ + awbAlgo_.queueRequest(context.activeState.awb, frame, frameContext.awb, + controls); +} + constexpr uint16_t Awb::threshold(float value) { /* AWB thresholds are in the range [0, 8191] */ @@ -237,11 +290,12 @@ constexpr uint16_t Awb::gainValue(double gain) /** * \copydoc libcamera::ipa::Algorithm::prepare */ -void Awb::prepare(IPAContext &context, - [[maybe_unused]] const uint32_t frame, - [[maybe_unused]] IPAFrameContext &frameContext, - ipu3_uapi_params *params) +void Awb::prepare(IPAContext &context, [[maybe_unused]] const uint32_t frame, + IPAFrameContext &frameContext, + [[maybe_unused]] ipu3_uapi_params *params) { + awbAlgo_.prepare(context.activeState.awb, frameContext.awb); + /* * Green saturation thresholds are reduced because we are using the * green channel only in the exposure computation. @@ -279,13 +333,11 @@ void Awb::prepare(IPAContext &context, params->acc_param.bnr.opt_center_sqr.y_sqr_reset = params->acc_param.bnr.opt_center.y_reset * params->acc_param.bnr.opt_center.y_reset; - params->acc_param.bnr.wb_gains.gr = gainValue(context.activeState.awb.gains.green); - params->acc_param.bnr.wb_gains.r = gainValue(context.activeState.awb.gains.red); - params->acc_param.bnr.wb_gains.b = gainValue(context.activeState.awb.gains.blue); - params->acc_param.bnr.wb_gains.gb = gainValue(context.activeState.awb.gains.green); - - LOG(IPU3Awb, Debug) << "Color temperature estimated: " << asyncResults_.temperatureK; + params->acc_param.bnr.wb_gains.gr = gainValue(frameContext.awb.gains.g()); + params->acc_param.bnr.wb_gains.r = gainValue(frameContext.awb.gains.r()); + params->acc_param.bnr.wb_gains.b = gainValue(frameContext.awb.gains.b()); + params->acc_param.bnr.wb_gains.gb = gainValue(frameContext.awb.gains.g()); params->use.acc_awb = 1; params->use.acc_bnr = 1; @@ -366,9 +418,17 @@ void Awb::clearAwbStats() } } -void Awb::awbGreyWorld() +Ipu3AwbStats Awb::calculateRgbMeans(const ipu3_uapi_stats_3a *stats) { - LOG(IPU3Awb, Debug) << "Grey world AWB"; + ASSERT(stats->stats_3a_status.awb_en); + + clearAwbStats(); + generateAwbStats(stats); + generateZones(); + + if (zones_.size() <= 10) + return {}; + /* * Make a separate list of the derivatives for each of red and blue, so * that we can sort them to exclude the extreme gains. We could @@ -399,66 +459,21 @@ void Awb::awbGreyWorld() double redGain = sumRed.g() / (sumRed.r() + 1), blueGain = sumBlue.g() / (sumBlue.b() + 1); - /* Color temperature is not relevant in Grey world but still useful to estimate it :-) */ - asyncResults_.temperatureK = estimateCCT({{ sumRed.r(), sumRed.g(), sumBlue.b() }}); - - /* - * Gain values are unsigned integer value ranging [0, 8) with 13 bit - * fractional part. - */ - redGain = std::clamp(redGain, 0.0, 65535.0 / 8192); - blueGain = std::clamp(blueGain, 0.0, 65535.0 / 8192); - - asyncResults_.redGain = redGain; - /* Hardcode the green gain to 1.0. */ - asyncResults_.greenGain = 1.0; - asyncResults_.blueGain = blueGain; -} - -void Awb::calculateWBGains(const ipu3_uapi_stats_3a *stats) -{ - ASSERT(stats->stats_3a_status.awb_en); - - clearAwbStats(); - generateAwbStats(stats); - generateZones(); - - LOG(IPU3Awb, Debug) << "Valid zones: " << zones_.size(); - - if (zones_.size() > 10) { - awbGreyWorld(); - LOG(IPU3Awb, Debug) << "Gain found for red: " << asyncResults_.redGain - << " and for blue: " << asyncResults_.blueGain; - } + return Ipu3AwbStats({ { 1.0 / redGain, 1.0, 1.0 / blueGain } }); } /** * \copydoc libcamera::ipa::Algorithm::process */ void Awb::process(IPAContext &context, [[maybe_unused]] const uint32_t frame, - [[maybe_unused]] IPAFrameContext &frameContext, - const ipu3_uapi_stats_3a *stats, - [[maybe_unused]] ControlList &metadata) + IPAFrameContext &frameContext, + [[maybe_unused]] const ipu3_uapi_stats_3a *stats, + ControlList &metadata) { - calculateWBGains(stats); + Ipu3AwbStats awbStats = calculateRgbMeans(stats); - /* - * Gains are only recalculated if enough zones were detected. - * The results are cached, so if no results were calculated, we set the - * cached values from asyncResults_ here. - */ - context.activeState.awb.gains.blue = asyncResults_.blueGain; - context.activeState.awb.gains.green = asyncResults_.greenGain; - context.activeState.awb.gains.red = asyncResults_.redGain; - context.activeState.awb.temperatureK = asyncResults_.temperatureK; - - metadata.set(controls::AwbEnable, true); - metadata.set(controls::ColourGains, { - static_cast(context.activeState.awb.gains.red), - static_cast(context.activeState.awb.gains.blue) - }); - metadata.set(controls::ColourTemperature, - context.activeState.awb.temperatureK); + awbAlgo_.process(context.activeState.awb, frameContext.awb, awbStats, + kDefaultLux, metadata); } REGISTER_IPA_ALGORITHM(Awb, "Awb") diff --git a/src/ipa/ipu3/algorithms/awb.h b/src/ipa/ipu3/algorithms/awb.h index 7ec322318dab54ae7c8a647a67a0cf5815a36eb6..e1a7b2139e5dafa748b908230aa3189f33b09ced 100644 --- a/src/ipa/ipu3/algorithms/awb.h +++ b/src/ipa/ipu3/algorithms/awb.h @@ -13,7 +13,8 @@ #include -#include "libcamera/internal/vector.h" +#include "libipa/awb.h" +#include "libipa/fixedpoint.h" #include "algorithm.h" @@ -21,6 +22,8 @@ namespace libcamera { namespace ipa::ipu3::algorithms { +class Ipu3AwbStats; + /* Region size for the statistics generation algorithm */ static constexpr uint32_t kAwbStatsSizeX = 16; static constexpr uint32_t kAwbStatsSizeY = 12; @@ -39,7 +42,11 @@ class Awb : public Algorithm public: Awb(); + int init(IPAContext &context, const ValueNode &tuningData) override; int configure(IPAContext &context, const IPAConfigInfo &configInfo) override; + void queueRequest(IPAContext &context, const uint32_t frame, + IPAFrameContext &frameContext, + const ControlList &controls) override; void prepare(IPAContext &context, const uint32_t frame, IPAFrameContext &frameContext, ipu3_uapi_params *params) override; @@ -49,15 +56,7 @@ public: ControlList &metadata) override; private: - struct AwbStatus { - double temperatureK; - double redGain; - double greenGain; - double blueGain; - }; - -private: - void calculateWBGains(const ipu3_uapi_stats_3a *stats); + Ipu3AwbStats calculateRgbMeans(const ipu3_uapi_stats_3a *stats); void generateZones(); void generateAwbStats(const ipu3_uapi_stats_3a *stats); void clearAwbStats(); @@ -67,7 +66,7 @@ private: std::vector> zones_; Accumulator awbStats_[kAwbStatsSizeX * kAwbStatsSizeY]; - AwbStatus asyncResults_; + AwbAlgorithm> awbAlgo_; uint32_t stride_; uint32_t cellsPerZoneX_; diff --git a/src/ipa/ipu3/ipa_context.cpp b/src/ipa/ipu3/ipa_context.cpp index c358b099810477c5f8264d3024f99b5bf2467f06..5cd8fedf27870474eb35745520206de885006b4a 100644 --- a/src/ipa/ipu3/ipa_context.cpp +++ b/src/ipa/ipu3/ipa_context.cpp @@ -113,22 +113,7 @@ namespace libcamera::ipa::ipu3 { /** * \var IPAActiveState::awb - * \brief Context for the Automatic White Balance algorithm - * - * \var IPAActiveState::awb.gains - * \brief White balance gains - * - * \var IPAActiveState::awb.gains.red - * \brief White balance gain for R channel - * - * \var IPAActiveState::awb.gains.green - * \brief White balance gain for G channel - * - * \var IPAActiveState::awb.gains.blue - * \brief White balance gain for B channel - * - * \var IPAActiveState::awb.temperatureK - * \brief Estimated color temperature + * \brief Active auto-white balance parameters for the IPA */ /** @@ -162,4 +147,9 @@ namespace libcamera::ipa::ipu3 { * \brief Per-frame state for the AGC algorithm */ +/** + * \var IPAFrameContext::awb + * \brief Per-frame auto-white balance parameters for the IPA + */ + } /* namespace libcamera::ipa::ipu3 */ diff --git a/src/ipa/ipu3/ipa_context.h b/src/ipa/ipu3/ipa_context.h index e8f27bc4299bbc22dc987f68cff8388ad80d9322..d2673b42bebb36dde06a795b6c1fbed6e9052535 100644 --- a/src/ipa/ipu3/ipa_context.h +++ b/src/ipa/ipu3/ipa_context.h @@ -18,6 +18,7 @@ #include #include +#include #include #include @@ -47,16 +48,7 @@ struct IPAActiveState { } af; agc::ActiveState agc; - - struct { - struct { - double red; - double green; - double blue; - } gains; - - double temperatureK; - } awb; + ipa::awb::ActiveState awb; struct { double gamma; @@ -71,6 +63,7 @@ struct IPAFrameContext : public FrameContext { } sensor; agc::FrameContext agc; + ipa::awb::FrameContext awb; }; struct IPAContext { diff --git a/src/ipa/libipa/awb.cpp b/src/ipa/libipa/awb.cpp index da835bec018d55b13200008bc24ac11db25b7bff..0e312690b8d2becccbb1405d93cac9c04b2d61a4 100644 --- a/src/ipa/libipa/awb.cpp +++ b/src/ipa/libipa/awb.cpp @@ -75,11 +75,6 @@ namespace awb { * hardware-specific statistics data. */ -/** - * AwbStat::~AwbStat - * \brief Virtual class destructor - */ - /** * \fn AwbStats::computeColourError() * \brief Compute an error value for when the given gains would be applied