diff --git a/src/ipa/libipa/awb.cpp b/src/ipa/libipa/awb.cpp index 29a78069c52c..95291840d8fb 100644 --- a/src/ipa/libipa/awb.cpp +++ b/src/ipa/libipa/awb.cpp @@ -2,7 +2,7 @@ /* * Copyright (C) 2024 Ideas on Board Oy * - * Generic AWB algorithms + * libIPA Awb algorithms */ #include "awb.h" @@ -11,9 +11,16 @@ #include +#include "awb_bayes.h" +#include "awb_grey.h" + +constexpr int32_t kMinColourTemperature = 2500; +constexpr int32_t kMaxColourTemperature = 10000; +constexpr int32_t kDefaultColourTemperature = 5000; + /** * \file awb.h - * \brief Base classes for AWB algorithms + * \brief libipa awb implementation */ namespace libcamera { @@ -22,30 +29,60 @@ LOG_DEFINE_CATEGORY(Awb) namespace ipa { +namespace awb { + /** - * \class AwbResult - * \brief The result of an AWB calculation + * \struct Context + * \brief Awb gains and colour temperature + * + * \var Context::gains + * \brief The white balance gains * - * This class holds the result of an auto white balance calculation. + * \var Context::temperatureK + * \brief The colour temperature, in Kelvin */ /** - * \var AwbResult::gains - * \brief The calculated white balance gains + * \struct ActiveState + * \brief Active awb state shared across frames + * + * \var ActiveState::manual + * \brief The most recent manually requested awb state + * + * \var ActiveState::automatic + * \brief The most recent automatically calculated awb state + * + * \var ActiveState::autoEnabled + * \brief True when automatic awb is selected */ /** - * \var AwbResult::colourTemperature - * \brief The calculated colour temperature in Kelvin + * \struct FrameContext + * \brief Per-frame awb state + * + * \var FrameContext::autoEnabled + * \brief True when automatic awb is in use */ +} /* namespace awb */ + /** * \class AwbStats - * \brief An abstraction class wrapping hardware-specific AWB statistics + * \brief An abstraction class wrapping hardware-specific awb statistics * - * IPA modules using an AWB algorithm based on the AwbAlgorithm class need to - * implement this class to give the algorithm access to the hardware-specific - * statistics data. + * IPA modules shall provide a derived implementation of this class to give the + * awb algorithm access to the hardware-specific statistics data in a common + * format. + */ + +/** + * \fn AwbStats::AwbStats() + * \brief Construct an empty AwbStat + */ + +/** + * AwbStat::~AwbStat + * \brief Virtual class destructor */ /** @@ -57,7 +94,7 @@ namespace ipa { * applied. To keep the actual implementations computationally inexpensive, * the squared colour error shall be returned. * - * If the AWB statistics provide multiple zones, the average of the individual + * If the awb statistics provide multiple zones, the average of the individual * squared errors shall be returned. Averaging/normalizing is necessary so that * the numeric dimensions are the same on all hardware platforms. * @@ -65,79 +102,328 @@ namespace ipa { */ /** - * \fn AwbStats::rgbMeans() - * \brief Get RGB means of the statistics + * \fn AwbStats::valid() + * \brief Retrieve if the awb statistics are valid * - * Fetch the RGB means from the statistics. The values of each channel are - * dimensionless and only the ratios are used for further calculations. This is - * used by the simple grey world model to calculate the gains to apply. + * If the colour mean values are too small, the AwbAlgorithm class doesn't have + * enough information to meaningfully calculate white-balance gains. Freeze the + * algorithm in that case. + * + * \return True if the awb statistics are valid, false otherwise + */ + +/** + * \fn AwbStats::rgbMeans + * \brief Retrieve the RGB means values + * + * Fetch the RGB means from the statistics in the [0, 1] range. * * \return The RGB means */ /** - * \class AwbAlgorithm - * \brief A base class for auto white balance algorithms + * \var AwbStats::rgbMeans_ + * \brief Mean values of colour channels + */ + +/** + * \var AwbStats::rg_ + * \brief Red/Green ratio + */ + +/** + * \var AwbStats::bg_ + * \brief Blue/Green ratio + */ + +/** + * \class AwbImplementation + * \brief Pure virtual base class for awb algorithms implementations + * + * The AwbImplementation class defines the interface for the awb algorithm + * implementations. * - * This class is a base class for auto white balance algorithms. It defines an - * interface for the algorithms to implement, and is used by the IPAs to - * interact with the concrete implementation. + * It is currently implemented by the AwbGrey and AwbBayes classes. + * + * The interface defines an init() function to initialize the algorithm with the + * content of the tuning file and two functions to compute colour gains + * according to the algorithm operating mode (auto or manual) in use. + * + * The calculateAwb() function calculates colour gains given a set of statistics + * provided by the IPA module. It is used when the algorithm operates in auto + * mode and gains are dynamically computed given a new set of statistics from + * the awb engine. + * + * The gainsFromColourTemperature() function instead computes the white balance + * gains from a colour temperature. This function does not take any statistics + * into account. It is used to compute the colour gains when the user manually + * specifies a colour temperature. */ /** - * \fn AwbAlgorithm::init() - * \brief Initialize the algorithm with the given tuning data - * \param[in] tuningData The tuning data to use for the algorithm + * \struct AwbImplementation::Result + * \brief The result of an awb computation * - * \return 0 on success, a negative error code otherwise + * \var AwbImplementation::Result::gains + * \brief The computed white balance gains + * + * \var AwbImplementation::Result::temperatureK + * \brief The computed colour temperature, in Kelvin */ /** - * \fn AwbAlgorithm::calculateAwb() - * \brief Calculate AWB data from the given statistics - * \param[in] stats The statistics to use for the calculation - * \param[in] lux The lux value of the scene + * \fn AwbImplementation::~AwbImplementation + * \brief Virtual class destructor + */ + +/** + * \fn AwbImplementation::init() + * \param[in] tuningData + * \brief Initialize the algorithm by parsing \a tuningData + */ + +/** + * \fn AwbImplementation::calculateAwb() + * \param[in] stats The awb statistics + * \param[in] lux The estimated lux level + * \param[in] range The colour temperature search limits * - * Calculate an AwbResult object from the given statistics and lux value. A \a - * lux value of 0 means it is unknown or invalid and the algorithm shall ignore - * it. + * Calculate a new set of colour gains and a colour temperature given a new + * set of statistics \a stats, an estimated luminance \a lux and a range of + * colour temperature to limit the search. * - * \return The AWB result + * \return A Result computed using the new statistics + */ +/** + * \fn AwbImplementation::gainsFromColourTemperature() + * \param[in] temperatureK Colour temperature in Kelvin + * + * Calculate a new set of colour gains for a new manually specified colour + * temperature \a temperatureK. + * + * \return The RGB gain values adjusted to \a temperatureK or std::nullopt + */ + +/** + * \class AwbAlgorithmBase + * \brief Base class for AwbAlgorithm for non-templated functions implementation + * + * Base class for AwbAlgorithm where non-templated functions are implemented. + * IPA implementations shall use AwbAlgorithm and not this class. */ /** - * \fn AwbAlgorithm::gainsFromColourTemperature() - * \brief Compute white balance gains from a colour temperature - * \param[in] colourTemperature The colour temperature in Kelvin + * \brief Initialize the algorithm with the given tuning data + * \param[in] tuningData The tuning data to use for the algorithm * - * Compute the white balance gains from a \a colourTemperature. This function - * does not take any statistics into account. It is used to compute the colour - * gains when the user manually specifies a colour temperature. + * Parse \a tuningData to initialize the awb algorithm and register controls. + * IPA modules are expected to call this function as part of their + * implementation of Algorithm::init(). * - * \return The colour gains or std::nullopt if the conversion is not possible + * \return 0 on success, a negative error code otherwise + */ +int AwbAlgorithmBase::init(const ValueNode &tuningData) +{ + bayes_ = false; + + if (!tuningData.contains("algorithm")) + LOG(Awb, Info) << "No Awb algorithm specified, using grey world"; + + auto mode = tuningData["algorithm"].get("grey"); + if (mode == "grey") { + impl_ = std::make_unique(); + } else if (mode == "bayes") { + impl_ = std::make_unique(); + bayes_ = true; + } else { + LOG(Awb, Error) << "Unknown Awb algorithm: " << mode; + return -EINVAL; + } + + LOG(Awb, Debug) << "Using Awb algorithm: " << mode; + + int ret = impl_->init(tuningData); + if (ret) + return ret; + + controls_[&controls::ColourTemperature] = + ControlInfo(kMinColourTemperature, kMaxColourTemperature, + kDefaultColourTemperature); + controls_[&controls::AwbEnable] = ControlInfo(false, true); + + return parseModeConfigs(tuningData, controls::AwbAuto); +} + +/** + * \brief Configure the awb algorithm + * \param[in] state The awb active state + * \return 0 if successful, an error code otherwise */ +int AwbAlgorithmBase::configure(awb::ActiveState &state) +{ + state.manual.gains = RGB{ 1.0 }; + auto gains = impl_->gainsFromColourTemperature(kDefaultColourTemperature); + if (gains) + state.automatic.gains = *gains; + else + state.automatic.gains = RGB{ 1.0 }; + + state.autoEnabled = true; + state.manual.temperatureK = kDefaultColourTemperature; + state.automatic.temperatureK = kDefaultColourTemperature; + + return 0; +} /** - * \fn AwbAlgorithm::controls() - * \brief Get the controls info map for this algorithm + * \brief Queue a Request to the awb algorithm + * \param[in] state The awb active state + * \param[in] frame The frame number + * \param[in] frameContext The awb frame context + * \param[in] controls The list of controls part of the Request * - * \return The controls info map + * Queue a new Request to the awb algorithm and modify its behaviour according + * to the provided controls. + * + * The currently handled controls are: + * - controls::AwbEnable + * - controls::AwbMode + * - controls::ColourGains + * - controls::ColourTemperature */ +void AwbAlgorithmBase::queueRequest(awb::ActiveState &state, + [[maybe_unused]] const uint32_t frame, + awb::FrameContext &frameContext, + const ControlList &controls) +{ + const auto &awbEnable = controls.get(controls::AwbEnable); + if (awbEnable && *awbEnable != state.autoEnabled) { + state.autoEnabled = *awbEnable; + + LOG(Awb, Debug) + << (*awbEnable ? "Enabling" : "Disabling") << " Awb"; + } + + auto mode = controls.get(controls::AwbMode); + if (mode) { + auto it = modes_.find(static_cast(*mode)); + if (it == modes_.end()) { + LOG(Awb, Error) << "Unsupported Awb mode " << *mode; + return; + } + + currentMode_ = &it->second; + } + + frameContext.autoEnabled = state.autoEnabled; + + if (frameContext.autoEnabled) + return; + + const auto &colourGains = controls.get(controls::ColourGains); + const auto &colourTemperature = controls.get(controls::ColourTemperature); + bool update = false; + if (colourGains) { + state.manual.gains.r() = (*colourGains)[0]; + state.manual.gains.b() = (*colourGains)[1]; + /* + * \todo Colour temperature reported in metadata is now + * incorrect, as we can't deduce the temperature from the gains. + * This will be fixed with the bayes AWB algorithm. + */ + update = true; + } else if (colourTemperature) { + state.manual.temperatureK = *colourTemperature; + const auto &gains = impl_->gainsFromColourTemperature(*colourTemperature); + if (gains) { + state.manual.gains.r() = gains->r(); + state.manual.gains.b() = gains->b(); + update = true; + } + } + + if (update) + LOG(Awb, Debug) + << "Set colour gains to " << state.manual.gains; + + frameContext.gains = state.manual.gains; + frameContext.temperatureK = state.manual.temperatureK; +} /** - * \fn AwbAlgorithm::handleControls() - * \param[in] controls The controls to handle - * \brief Handle the controls supplied in a request + * \brief Set the gains and colour temperature values in \a frameContext + * \param[in] state The awb active state + * \param[in] frameContext The awb frame context + * + * If auto mode is enabled, take the most recently computed gains and use them + * for the current frame. Otherwise, if in manual mode, gains and colour + * temperature for a frame are set at queueRequest() time. */ +void AwbAlgorithmBase::prepare(awb::ActiveState &state, + awb::FrameContext &frameContext) +{ + if (frameContext.autoEnabled) { + frameContext.gains = state.automatic.gains; + frameContext.temperatureK = state.automatic.temperatureK; + } +} /** + * \brief Process awb statistics to calculate gains and populate metadata + * \param[in] state The awb active state + * \param[in] frameContext The awb frame context + * \param[in] stats The awb statistics + * \param[in] lux The lux value as estimated by the IPA module + * \param[out] metadata The metadata list + * + * Process \a stats to calculate new gains and colour temperature and populate + * \a metadata with the results. + */ +void AwbAlgorithmBase::process(awb::ActiveState &state, + awb::FrameContext &frameContext, + const AwbStats &stats, unsigned int lux, + ControlList &metadata) +{ + if (!stats.valid()) + return; + + auto awbResult = impl_->calculateAwb(stats, lux, { currentMode_->ctLo, + currentMode_->ctHi }); + + /* + * Clamp the gain values to the hardware, according to the gainMin_ + * and gainMax_ values. + */ + awbResult.gains = awbResult.gains.clamp(gainMin_, gainMax_); + + /* Smooth color gains adjustments. */ + double speed = 0.2; + double ct = awbResult.temperatureK; + ct = ct * speed + state.automatic.temperatureK * (1 - speed); + + state.automatic.temperatureK = awbResult.temperatureK; + state.automatic.gains = awbResult.gains * speed + + state.automatic.gains * (1 - speed); + + /* Populate metadata. */ + metadata.set(controls::AwbEnable, frameContext.autoEnabled); + metadata.set(controls::ColourGains, { static_cast(frameContext.gains.r()), + static_cast(frameContext.gains.b()) }); + metadata.set(controls::ColourTemperature, frameContext.temperatureK); + + LOG(Awb, Debug) << std::showpoint << "Means " << stats.rgbMeans() + << ", gains " << state.automatic.gains + << ", temp " << state.automatic.temperatureK << "K"; +} + +/* * \brief Parse the mode configurations from the tuning data * \param[in] tuningData the ValueNode representing the tuning data * \param[in] def The default value for the AwbMode control * * Utility function to parse the tuning data for an AwbMode entry and read all - * provided modes. It adds controls::AwbMode to AwbAlgorithm::controls_ and - * populates AwbAlgorithm::modes_. For a list of possible modes see \ref + * provided modes. It adds controls::AwbMode to AwbAlgorithmBase::controls_ and + * populates AwbAlgorithmBase::modes_. For a list of possible modes see \ref * controls::AwbModeEnum. * * Each mode entry must contain a "lo" and "hi" key to specify the lower and @@ -156,15 +442,23 @@ namespace ipa { * ... * \endcode * - * If \a def is supplied but not contained in the the \a tuningData, -EINVAL is + * If \a def is supplied but not contained in the \a tuningData, -EINVAL is * returned. * + * AwbModes are only used by the AwbBayes implementation. + * * \sa controls::AwbModeEnum * \return Zero on success, negative error code otherwise */ -int AwbAlgorithm::parseModeConfigs(const ValueNode &tuningData, - const ControlValue &def) +int AwbAlgorithmBase::parseModeConfigs(const ValueNode &tuningData, + const ControlValue &def) { + if (!bayes_) { + /* AwbGrey does not support and does not use modes. */ + currentMode_ = &AwbGreyMode; + return 0; + } + std::vector availableModes; const ValueNode &modes = tuningData[controls::AwbMode.name()]; @@ -227,37 +521,83 @@ int AwbAlgorithm::parseModeConfigs(const ValueNode &tuningData, } controls_[&controls::AwbMode] = ControlInfo(availableModes, def); + currentMode_ = &modes_[controls::AwbAuto]; return 0; } /** - * \class AwbAlgorithm::ModeConfig - * \brief Holds the configuration of a single AWB mode + * \var AwbAlgorithmBase::controls_ + * \brief Controls info map for the controls registered by the awb algorithm + */ + +/** + * \var AwbAlgorithmBase::gainMin_ + * \brief The minimum supported gain value + * + * Minimum gain value used to clamp the awb algorithm calculation results in the + * range supported by the platform awb engine. * - * AWB modes limit the regulation of the AWB algorithm to a specific range of - * colour temperatures. + * The min and max gain values are initialized by AwbAlgorithm::init(). */ /** - * \var AwbAlgorithm::ModeConfig::ctLo + * \var AwbAlgorithmBase::gainMax_ + * \brief The maximum supported gain value + * + * Maximum gain value used to clamp the awb algorithm calculation results in the + * range supported by the platform awb engine. + * + * The min and max gain values are initialized by AwbAlgorithm::init(). + */ + +/* + * \class AwbAlgorithmBase::ModeConfig + * \brief Holds the configuration of a single awb mode + * + * AWB modes limit the regulation of the awb algorithm to a specific range of + * colour temperatures. Use by AwbBayes only. + */ + +/* + * \var AwbAlgorithmBase::ModeConfig::ctLo * \brief The lowest valid colour temperature of that mode */ -/** - * \var AwbAlgorithm::ModeConfig::ctHi +/* + * \var AwbAlgorithmBase::ModeConfig::ctHi * \brief The highest valid colour temperature of that mode */ +/* + * \var AwbAlgorithmBase::modes_ + * \brief Map of all configured modes + * \sa AwbAlgorithmBase::parseModeConfigs + */ + /** - * \var AwbAlgorithm::controls_ - * \brief Controls info map for the controls provided by the algorithm + * \class AwbAlgorithm + * \brief The libipa awb algorithm + * \tparam Q The fixedpoint register representation of gain values + * + * Implement the awb algorithm for libipa. + * + * The AwbAlgorithm class implements an interface similar in spirit to the one + * of the Algorithm class. IPA modules are expected to store an instance of + * AwbAlgorithm as class member, template it with the awb engine gain register + * representation and call its function in their implementations of the + * Algorithm interface. + * + * The AwbAlgorithm instantiates an AwbImplementation implementation (AwbGrey or + * AwbBayes) at init() time by parsing the tuning data and uses it to compute + * the RGB gains and estimate a colour temperature given a set of statistics + * in the form of a AwbStats derived implementation. */ /** - * \var AwbAlgorithm::modes_ - * \brief Map of all configured modes - * \sa AwbAlgorithm::parseModeConfigs + * \fn AwbAlgorithm::init() + * \param[in] controls The info map of the IPA controls + * \copydoc AwbAlgorithmBase::init() */ } /* namespace ipa */ diff --git a/src/ipa/libipa/awb.h b/src/ipa/libipa/awb.h index 09c00e47d604..72f46367a136 100644 --- a/src/ipa/libipa/awb.h +++ b/src/ipa/libipa/awb.h @@ -2,11 +2,12 @@ /* * Copyright (C) 2024 Ideas on Board Oy * - * Generic AWB algorithms + * libIPA AWB algorithms */ #pragma once +#include #include #include @@ -20,46 +21,118 @@ namespace libcamera { namespace ipa { -struct AwbResult { +namespace awb { + +struct Context { RGB gains; - double colourTemperature; + unsigned int temperatureK; }; +struct ActiveState { + Context manual; + Context automatic; + + bool autoEnabled; +}; + +struct FrameContext : public Context { + bool autoEnabled; +}; + +} /* namespace awb */ + struct AwbStats { + AwbStats() = default; + virtual ~AwbStats() = default; + virtual double computeColourError(const RGB &gains) const = 0; - virtual RGB rgbMeans() const = 0; + virtual bool valid() const = 0; + RGB rgbMeans() const { return rgbMeans_; } protected: - ~AwbStats() = default; + RGB rgbMeans_; + double rg_; + double bg_; }; -class AwbAlgorithm +class AwbImplementation { public: - virtual ~AwbAlgorithm() = default; + struct Result { + RGB gains; + unsigned int temperatureK; + }; + virtual ~AwbImplementation() = default; virtual int init(const ValueNode &tuningData) = 0; - virtual AwbResult calculateAwb(const AwbStats &stats, unsigned int lux) = 0; - virtual std::optional> gainsFromColourTemperature(double colourTemperature) = 0; + virtual Result calculateAwb(const AwbStats &stats, unsigned int lux, + std::array range) = 0; + virtual std::optional> + gainsFromColourTemperature(double temperatureK) = 0; +}; - const ControlInfoMap::Map &controls() const - { - return controls_; - } +class AwbAlgorithmBase +{ +public: + int init(const ValueNode &tuningData); - virtual void handleControls([[maybe_unused]] const ControlList &controls) {} + int configure(awb::ActiveState &state); + + void queueRequest(awb::ActiveState &state, + const uint32_t frame, + awb::FrameContext &frameContext, + const ControlList &controls); + + void prepare(awb::ActiveState &state, awb::FrameContext &frameContext); + + void process(awb::ActiveState &state, awb::FrameContext &frameContext, + const AwbStats &stats, unsigned int lux, + ControlList &metadata); protected: - int parseModeConfigs(const ValueNode &tuningData, - const ControlValue &def = {}); + AwbAlgorithmBase() = default; + + ControlInfoMap::Map controls_; + float gainMin_; + float gainMax_; +private: struct ModeConfig { double ctHi; double ctLo; }; - ControlInfoMap::Map controls_; - std::map modes_; + /* AwbGrey does not support modes; */ + static constexpr ModeConfig AwbGreyMode = { 0.0, 0.0 }; + + int parseModeConfigs(const ValueNode &tuningData, + const ControlValue &def = {}); + + std::map modes_; + const ModeConfig *currentMode_ = nullptr; + std::unique_ptr impl_; + bool bayes_ = false; +}; + +template +class AwbAlgorithm : public AwbAlgorithmBase +{ +public: + int init(const ValueNode &tuningData, ControlInfoMap::Map &controls) + { + AwbAlgorithmBase::init(tuningData); + + gainMin_ = std::max(Q::TraitsType::min, 1.0f); + gainMax_ = Q::TraitsType::max; + + controls_[&controls::ColourGains] = + ControlInfo(gainMin_, gainMax_, + Span{ { 1.0f, 1.0f } }); + + controls.insert(controls_.begin(), controls_.end()); + + return 0; + } }; } /* namespace ipa */ diff --git a/src/ipa/libipa/awb_bayes.cpp b/src/ipa/libipa/awb_bayes.cpp index 9fd85e5a4505..2c10f9c4c82d 100644 --- a/src/ipa/libipa/awb_bayes.cpp +++ b/src/ipa/libipa/awb_bayes.cpp @@ -140,11 +140,6 @@ void Interpolator::interpolate(const Pwl &a, const Pwl &b, Pwl &dest, doubl * \brief How far to wander off CT curve towards "more green" */ -/** - * \var AwbBayes::currentMode_ - * \brief The currently selected mode - */ - int AwbBayes::init(const ValueNode &tuningData) { int ret = colourGainCurve_.readYaml(tuningData["colourGains"], "ct", "gains"); @@ -172,14 +167,6 @@ int AwbBayes::init(const ValueNode &tuningData) return ret; } - ret = parseModeConfigs(tuningData, controls::AwbAuto); - if (ret) { - LOG(Awb, Error) - << "Failed to parse mode parameter from tuning file"; - return ret; - } - currentMode_ = &modes_[controls::AwbAuto]; - transversePos_ = tuningData["transversePos"].get(0.01); transverseNeg_ = tuningData["transverseNeg"].get(0.01); if (transversePos_ <= 0 || transverseNeg_ <= 0) { @@ -260,18 +247,16 @@ int AwbBayes::readPriors(const ValueNode &tuningData) return 0; } -void AwbBayes::handleControls(const ControlList &controls) -{ - auto mode = controls.get(controls::AwbMode); - if (mode) { - auto it = modes_.find(static_cast(*mode)); - if (it != modes_.end()) - currentMode_ = &it->second; - else - LOG(Awb, Error) << "Unsupported AWB mode " << *mode; - } -} - +/** + * \brief Compute white balance gains from a colour temperature + * \param[in] colourTemperature The colour temperature in Kelvin + * + * Compute the white balance gains from a \a colourTemperature. This function + * does not take any statistics into account. It simply interpolates the colour + * gains curve on a new colour temperature. + * + * \return The RGB gain values adjusted to \a temperatureK + */ std::optional> AwbBayes::gainsFromColourTemperature(double colourTemperature) { /* @@ -283,7 +268,9 @@ std::optional> AwbBayes::gainsFromColourTemperature(double colourTem return RGB{ { gains[0], 1.0, gains[1] } }; } -AwbResult AwbBayes::calculateAwb(const AwbStats &stats, unsigned int lux) +AwbImplementation::Result +AwbBayes::calculateAwb(const AwbStats &stats, unsigned int lux, + std::array range) { ipa::Pwl prior; if (lux > 0) { @@ -295,7 +282,7 @@ AwbResult AwbBayes::calculateAwb(const AwbStats &stats, unsigned int lux) prior.append(0, 1.0); } - double t = coarseSearch(prior, stats); + double t = coarseSearch(prior, stats, range); double r = ctR_.eval(t); double b = ctB_.eval(t); LOG(Awb, Debug) @@ -316,14 +303,15 @@ AwbResult AwbBayes::calculateAwb(const AwbStats &stats, unsigned int lux) << "After fine search: r " << r << " b " << b << " (gains r " << 1 / r << " b " << 1 / b << ")"; - return { { { 1.0 / r, 1.0, 1.0 / b } }, t }; + return { { { 1.0 / r, 1.0, 1.0 / b } }, static_cast(t) }; } -double AwbBayes::coarseSearch(const ipa::Pwl &prior, const AwbStats &stats) const +double AwbBayes::coarseSearch(const ipa::Pwl &prior, const AwbStats &stats, + Span range) const { std::vector points; size_t bestPoint = 0; - double t = currentMode_->ctLo; + double t = range[0]; int spanR = -1; int spanB = -1; LimitsRecorder errorLimits; @@ -345,14 +333,14 @@ double AwbBayes::coarseSearch(const ipa::Pwl &prior, const AwbStats &stats) cons if (points.back().y() < points[bestPoint].y()) bestPoint = points.size() - 1; - if (t == currentMode_->ctHi) + if (t == range[1]) break; /* * Ensure even steps along the r/b curve by scaling them by the * current t. */ - t = std::min(t + t / 10 * kSearchStep, currentMode_->ctHi); + t = std::min(t + t / 10 * kSearchStep, range[1]); } t = points[bestPoint].x(); diff --git a/src/ipa/libipa/awb_bayes.h b/src/ipa/libipa/awb_bayes.h index 1e3373676bc0..8d4cff076d4a 100644 --- a/src/ipa/libipa/awb_bayes.h +++ b/src/ipa/libipa/awb_bayes.h @@ -20,22 +20,24 @@ namespace libcamera { namespace ipa { -class AwbBayes : public AwbAlgorithm +class AwbBayes : public AwbImplementation { public: AwbBayes() = default; int init(const ValueNode &tuningData) override; - AwbResult calculateAwb(const AwbStats &stats, unsigned int lux) override; + AwbImplementation::Result + calculateAwb(const AwbStats &stats, unsigned int lux, + std::array range) override; std::optional> gainsFromColourTemperature(double temperatureK) override; - void handleControls(const ControlList &controls) override; private: int readPriors(const ValueNode &tuningData); void fineSearch(double &t, double &r, double &b, ipa::Pwl const &prior, const AwbStats &stats) const; - double coarseSearch(const ipa::Pwl &prior, const AwbStats &stats) const; + double coarseSearch(const ipa::Pwl &prior, const AwbStats &stats, + Span range) const; double interpolateQuadratic(ipa::Pwl::Point const &a, ipa::Pwl::Point const &b, ipa::Pwl::Point const &c) const; @@ -50,8 +52,6 @@ private: double transversePos_; double transverseNeg_; - - ModeConfig *currentMode_ = nullptr; }; } /* namespace ipa */ diff --git a/src/ipa/libipa/awb_grey.cpp b/src/ipa/libipa/awb_grey.cpp index b14b096810ae..1e4af717f62d 100644 --- a/src/ipa/libipa/awb_grey.cpp +++ b/src/ipa/libipa/awb_grey.cpp @@ -60,6 +60,7 @@ int AwbGrey::init(const ValueNode &tuningData) * \brief Calculate AWB data from the given statistics * \param[in] stats The statistics to use for the calculation * \param[in] lux The lux value of the scene + * \param[in] range The colour temperature search limits * * The colour temperature is estimated based on the colours::estimateCCT() * function. The gains are calculated purely based on the RGB means provided by @@ -70,20 +71,23 @@ int AwbGrey::init(const ValueNode &tuningData) * * \return The AWB result */ -AwbResult AwbGrey::calculateAwb(const AwbStats &stats, [[maybe_unused]] unsigned int lux) +AwbImplementation::Result +AwbGrey::calculateAwb(const AwbStats &stats, [[maybe_unused]] unsigned int lux, + [[maybe_unused]] std::array range) { - AwbResult result; + AwbImplementation::Result result; auto means = stats.rgbMeans(); - result.colourTemperature = estimateCCT(means); + result.temperatureK = estimateCCT(means); /* - * Estimate the red and blue gains to apply in a grey world. The green - * gain is hardcoded to 1.0. Avoid divisions by zero by clamping the - * divisor to a minimum value of 1.0. + * Calculate the red and blue gains to apply in a grey world by simply + * inverting the red/green and blue/green ratios as reported in + * statistics. The green gain is hardcoded to 1.0. */ result.gains.r() = means.g() / std::max(means.r(), 1.0); result.gains.g() = 1.0; result.gains.b() = means.g() / std::max(means.b(), 1.0); + return result; } @@ -93,15 +97,16 @@ AwbResult AwbGrey::calculateAwb(const AwbStats &stats, [[maybe_unused]] unsigned * * Compute the white balance gains from a \a colourTemperature. This function * does not take any statistics into account. It simply interpolates the colour - * gains configured in the colour temperature curve. + * gains curve (if provided in the tuning file) on a new colour temperature. * - * \return The colour gains if a colour temperature curve is available, - * [1, 1, 1] otherwise. + * \return The RGB gain values adjusted to \a temperatureK or std::nullopt if + * no gain curve is specified in the tuning data */ std::optional> AwbGrey::gainsFromColourTemperature(double colourTemperature) { if (!colourGainCurve_) { - LOG(Awb, Error) << "No gains defined"; + LOG(Awb, Info) << "No gains curve defined, " + << "unable to interpolate gains to colour temperature"; return std::nullopt; } diff --git a/src/ipa/libipa/awb_grey.h b/src/ipa/libipa/awb_grey.h index 154a2af97f15..d1b0088f0470 100644 --- a/src/ipa/libipa/awb_grey.h +++ b/src/ipa/libipa/awb_grey.h @@ -18,13 +18,15 @@ namespace libcamera { namespace ipa { -class AwbGrey : public AwbAlgorithm +class AwbGrey : public AwbImplementation { public: AwbGrey() = default; int init(const ValueNode &tuningData) override; - AwbResult calculateAwb(const AwbStats &stats, unsigned int lux) override; + AwbImplementation::Result + calculateAwb(const AwbStats &stats, unsigned int lux, + std::array range) override; std::optional> gainsFromColourTemperature(double colourTemperature) override; private: diff --git a/src/ipa/rkisp1/algorithms/awb.cpp b/src/ipa/rkisp1/algorithms/awb.cpp index 0eb05c2b6e1a..ae9f6a0c243e 100644 --- a/src/ipa/rkisp1/algorithms/awb.cpp +++ b/src/ipa/rkisp1/algorithms/awb.cpp @@ -8,17 +8,12 @@ #include "awb.h" #include -#include #include -#include - #include -#include "libipa/awb_bayes.h" -#include "libipa/awb_grey.h" -#include "libipa/colours.h" +#include "libcamera/internal/vector.h" /** * \file awb.h @@ -28,26 +23,16 @@ namespace libcamera { namespace ipa::rkisp1::algorithms { -/** - * \class Awb - * \brief Manage the white balance with automatic and manual controls - */ - LOG_DEFINE_CATEGORY(RkISP1Awb) -constexpr int32_t kMinColourTemperature = 2500; -constexpr int32_t kMaxColourTemperature = 10000; -constexpr int32_t kDefaultColourTemperature = 5000; - -/* Minimum mean value below which AWB can't operate. */ -constexpr double kMeanMinThreshold = 2.0; - class RkISP1AwbStats final : public AwbStats { public: - RkISP1AwbStats(const RGB &rgbMeans) - : rgbMeans_(rgbMeans) + RkISP1AwbStats(const IPAFrameContext &frameContext, + const rkisp1_stat_buffer *stats) { + calculateRgbMeans(frameContext, stats); + rg_ = rgbMeans_.r() / rgbMeans_.g(); bg_ = rgbMeans_.b() / rgbMeans_.g(); } @@ -65,17 +50,106 @@ public: return delta2; } - RGB rgbMeans() const override + bool valid() const override { - return rgbMeans_; + /* Minimum mean value below which AWB can't operate. */ + constexpr double minValue = 2.0; + + return rgbMeans_.r() > minValue || rgbMeans_.g() > minValue || + rgbMeans_.b() > minValue; } private: - RGB rgbMeans_; - double rg_; - double bg_; + void calculateRgbMeans(const IPAFrameContext &frameContext, + const rkisp1_stat_buffer *stats); + bool rgbMode_ = false; }; +void RkISP1AwbStats::calculateRgbMeans(const IPAFrameContext &frameContext, + const rkisp1_stat_buffer *stats) +{ + if (!stats || !(stats->meas_type & RKISP1_CIF_ISP_STAT_AWB)) { + LOG(RkISP1Awb, Error) << "AWB data is missing in statistics"; + return; + } + + const rkisp1_cif_isp_awb_stat *awb = &stats->params.awb; + + if (awb->awb_mean[0].cnt == 0) { + LOG(RkISP1Awb, Debug) << "AWB statistics are empty"; + return; + } + + if (rgbMode_) { + rgbMeans_ = {{ + static_cast(awb->awb_mean[0].mean_cr_or_r), + static_cast(awb->awb_mean[0].mean_y_or_g), + static_cast(awb->awb_mean[0].mean_cb_or_b) + }}; + } else { + /* Get the YCbCr mean values */ + Vector yuvMeans({ + static_cast(awb->awb_mean[0].mean_y_or_g), + static_cast(awb->awb_mean[0].mean_cb_or_b), + static_cast(awb->awb_mean[0].mean_cr_or_r) + }); + + /* + * Convert from YCbCr to RGB. The hardware uses the following + * formulas: + * + * Y = 16 + 0.2500 R + 0.5000 G + 0.1094 B + * Cb = 128 - 0.1406 R - 0.2969 G + 0.4375 B + * Cr = 128 + 0.4375 R - 0.3750 G - 0.0625 B + * + * This seems to be based on limited range BT.601 with Q1.6 + * precision. + * + * The inverse matrix is: + * + * [[1,1636, -0,0623, 1,6008] + * [1,1636, -0,4045, -0,7949] + * [1,1636, 1,9912, -0,0250]] + */ + static const Matrix yuv2rgbMatrix({ + 1.1636, -0.0623, 1.6008, + 1.1636, -0.4045, -0.7949, + 1.1636, 1.9912, -0.0250 + }); + static const Vector yuv2rgbOffset({ + 16, 128, 128 + }); + + rgbMeans_ = yuv2rgbMatrix * (yuvMeans - yuv2rgbOffset); + + /* + * Due to hardware rounding errors in the YCbCr means, the + * calculated RGB means may be negative. This would lead to + * negative gains, messing up calculation. Prevent this by + * clamping the means to positive values. + */ + rgbMeans_ = rgbMeans_.max(0.0); + } + + /* + * The ISP computes the AWB means after applying the CCM. Apply the + * inverse as we want to get the raw means before the colour gains. + */ + rgbMeans_ = frameContext.ccm.ccm.inverse() * rgbMeans_; + + /* + * The ISP computes the AWB means after applying the colour gains, + * divide by the gains that were used to get the raw means from the + * sensor. Apply a minimum value to avoid divisions by near-zero. + */ + rgbMeans_ /= frameContext.awb.gains.max(0.01); +} + +/** + * \class Awb + * \brief Manage the white balance with automatic and manual controls + */ + Awb::Awb() : rgbMode_(false) { @@ -86,40 +160,7 @@ Awb::Awb() */ int Awb::init(IPAContext &context, const ValueNode &tuningData) { - auto &cmap = context.ctrlMap; - cmap[&controls::ColourTemperature] = ControlInfo(kMinColourTemperature, - kMaxColourTemperature, - kDefaultColourTemperature); - cmap[&controls::AwbEnable] = ControlInfo(false, true); - - cmap[&controls::ColourGains] = ControlInfo(0.0f, 3.996f, - Span{ { 1.0f, 1.0f } }); - - if (!tuningData.contains("algorithm")) - LOG(RkISP1Awb, Info) << "No AWB algorithm specified." - << " Default to grey world"; - - auto mode = tuningData["algorithm"].get("grey"); - if (mode == "grey") { - awbAlgo_ = std::make_unique(); - } else if (mode == "bayes") { - awbAlgo_ = std::make_unique(); - } else { - LOG(RkISP1Awb, Error) << "Unknown AWB algorithm: " << mode; - return -EINVAL; - } - LOG(RkISP1Awb, Debug) << "Using AWB algorithm: " << mode; - - int ret = awbAlgo_->init(tuningData); - if (ret) { - LOG(RkISP1Awb, Error) << "Failed to init AWB algorithm"; - return ret; - } - - const auto &src = awbAlgo_->controls(); - cmap.insert(src.begin(), src.end()); - - return 0; + return awbAlgo_.init(tuningData, context.ctrlMap); } /** @@ -128,16 +169,7 @@ int Awb::init(IPAContext &context, const ValueNode &tuningData) int Awb::configure(IPAContext &context, const IPACameraSensorInfo &configInfo) { - context.activeState.awb.manual.gains = RGB{ 1.0 }; - auto gains = awbAlgo_->gainsFromColourTemperature(kDefaultColourTemperature); - if (gains) - context.activeState.awb.automatic.gains = *gains; - else - context.activeState.awb.automatic.gains = RGB{ 1.0 }; - - context.activeState.awb.autoEnabled = true; - context.activeState.awb.manual.temperatureK = kDefaultColourTemperature; - context.activeState.awb.automatic.temperatureK = kDefaultColourTemperature; + awbAlgo_.configure(context.activeState.awb); /* * Define the measurement window for AWB as a centered rectangle @@ -156,56 +188,12 @@ int Awb::configure(IPAContext &context, /** * \copydoc libcamera::ipa::Algorithm::queueRequest */ -void Awb::queueRequest(IPAContext &context, - [[maybe_unused]] const uint32_t frame, +void Awb::queueRequest(IPAContext &context, const uint32_t frame, IPAFrameContext &frameContext, const ControlList &controls) { - auto &awb = context.activeState.awb; - - const auto &awbEnable = controls.get(controls::AwbEnable); - if (awbEnable && *awbEnable != awb.autoEnabled) { - awb.autoEnabled = *awbEnable; - - LOG(RkISP1Awb, Debug) - << (*awbEnable ? "Enabling" : "Disabling") << " AWB"; - } - - awbAlgo_->handleControls(controls); - - frameContext.awb.autoEnabled = awb.autoEnabled; - - if (awb.autoEnabled) - return; - - const auto &colourGains = controls.get(controls::ColourGains); - const auto &colourTemperature = controls.get(controls::ColourTemperature); - bool update = false; - if (colourGains) { - awb.manual.gains.r() = (*colourGains)[0]; - awb.manual.gains.b() = (*colourGains)[1]; - /* - * \todo Colour temperature reported in metadata is now - * incorrect, as we can't deduce the temperature from the gains. - * This will be fixed with the bayes AWB algorithm. - */ - update = true; - } else if (colourTemperature) { - awb.manual.temperatureK = *colourTemperature; - const auto &gains = awbAlgo_->gainsFromColourTemperature(*colourTemperature); - if (gains) { - awb.manual.gains.r() = gains->r(); - awb.manual.gains.b() = gains->b(); - update = true; - } - } - - if (update) - LOG(RkISP1Awb, Debug) - << "Set colour gains to " << awb.manual.gains; - - frameContext.awb.gains = awb.manual.gains; - frameContext.awb.temperatureK = awb.manual.temperatureK; + awbAlgo_.queueRequest(context.activeState.awb, frame, frameContext.awb, + controls); } /** @@ -214,15 +202,7 @@ void Awb::queueRequest(IPAContext &context, void Awb::prepare(IPAContext &context, const uint32_t frame, IPAFrameContext &frameContext, RkISP1Params *params) { - /* - * This is the latest time we can read the active state. This is the - * most up-to-date automatic values we can read. - */ - if (frameContext.awb.autoEnabled) { - const auto &awb = context.activeState.awb; - frameContext.awb.gains = awb.automatic.gains; - frameContext.awb.temperatureK = awb.automatic.temperatureK; - } + awbAlgo_.prepare(context.activeState.awb, frameContext.awb); auto gainConfig = params->block(); gainConfig.setEnabled(true); @@ -291,136 +271,12 @@ void Awb::process(IPAContext &context, const rkisp1_stat_buffer *stats, ControlList &metadata) { - IPAActiveState &activeState = context.activeState; - - metadata.set(controls::AwbEnable, frameContext.awb.autoEnabled); - metadata.set(controls::ColourGains, { - static_cast(frameContext.awb.gains.r()), - static_cast(frameContext.awb.gains.b()) - }); - metadata.set(controls::ColourTemperature, frameContext.awb.temperatureK); - - if (!stats || !(stats->meas_type & RKISP1_CIF_ISP_STAT_AWB)) { - LOG(RkISP1Awb, Error) << "AWB data is missing in statistics"; - return; - } - - const rkisp1_cif_isp_stat *params = &stats->params; - const rkisp1_cif_isp_awb_stat *awb = ¶ms->awb; + RkISP1AwbStats awbStats(frameContext, stats); - if (awb->awb_mean[0].cnt == 0) { - LOG(RkISP1Awb, Debug) << "AWB statistics are empty"; - return; - } - - RGB rgbMeans = calculateRgbMeans(frameContext, awb); - - /* - * If the means are too small we don't have enough information to - * meaningfully calculate gains. Freeze the algorithm in that case. - */ - if (rgbMeans.r() < kMeanMinThreshold && rgbMeans.g() < kMeanMinThreshold && - rgbMeans.b() < kMeanMinThreshold) - return; - - RkISP1AwbStats awbStats{ rgbMeans }; - AwbResult awbResult = awbAlgo_->calculateAwb(awbStats, frameContext.lux.lux); - - /* - * Clamp the gain values to the hardware, which expresses gains as Q2.8 - * unsigned integer values. Set the minimum just above zero to avoid - * divisions by zero when computing the raw means in subsequent - * iterations. - */ - awbResult.gains = awbResult.gains.clamp(1.0 / 256, 1023.0 / 256); - - /* Filter the values to avoid oscillations. */ - double speed = 0.2; - double ct = awbResult.colourTemperature; - ct = ct * speed + activeState.awb.automatic.temperatureK * (1 - speed); - awbResult.gains = awbResult.gains * speed + - activeState.awb.automatic.gains * (1 - speed); - - activeState.awb.automatic.temperatureK = static_cast(ct); - activeState.awb.automatic.gains = awbResult.gains; - - LOG(RkISP1Awb, Debug) - << std::showpoint - << "Means " << rgbMeans << ", gains " - << activeState.awb.automatic.gains << ", temp " - << activeState.awb.automatic.temperatureK << "K"; + awbAlgo_.process(context.activeState.awb, frameContext.awb, awbStats, + frameContext.lux.lux, metadata); } -RGB Awb::calculateRgbMeans(const IPAFrameContext &frameContext, const rkisp1_cif_isp_awb_stat *awb) const -{ - Vector rgbMeans; - - if (rgbMode_) { - rgbMeans = {{ - static_cast(awb->awb_mean[0].mean_cr_or_r), - static_cast(awb->awb_mean[0].mean_y_or_g), - static_cast(awb->awb_mean[0].mean_cb_or_b) - }}; - } else { - /* Get the YCbCr mean values */ - Vector yuvMeans({ - static_cast(awb->awb_mean[0].mean_y_or_g), - static_cast(awb->awb_mean[0].mean_cb_or_b), - static_cast(awb->awb_mean[0].mean_cr_or_r) - }); - - /* - * Convert from YCbCr to RGB. The hardware uses the following - * formulas: - * - * Y = 16 + 0.2500 R + 0.5000 G + 0.1094 B - * Cb = 128 - 0.1406 R - 0.2969 G + 0.4375 B - * Cr = 128 + 0.4375 R - 0.3750 G - 0.0625 B - * - * This seems to be based on limited range BT.601 with Q1.6 - * precision. - * - * The inverse matrix is: - * - * [[1,1636, -0,0623, 1,6008] - * [1,1636, -0,4045, -0,7949] - * [1,1636, 1,9912, -0,0250]] - */ - static const Matrix yuv2rgbMatrix({ - 1.1636, -0.0623, 1.6008, - 1.1636, -0.4045, -0.7949, - 1.1636, 1.9912, -0.0250 - }); - static const Vector yuv2rgbOffset({ - 16, 128, 128 - }); - - rgbMeans = yuv2rgbMatrix * (yuvMeans - yuv2rgbOffset); - - /* - * Due to hardware rounding errors in the YCbCr means, the - * calculated RGB means may be negative. This would lead to - * negative gains, messing up calculation. Prevent this by - * clamping the means to positive values. - */ - rgbMeans = rgbMeans.max(0.0); - } - - /* - * The ISP computes the AWB means after applying the CCM. Apply the - * inverse as we want to get the raw means before the colour gains. - */ - rgbMeans = frameContext.ccm.ccm.inverse() * rgbMeans; - - /* - * The ISP computes the AWB means after applying the colour gains, - * divide by the gains that were used to get the raw means from the - * sensor. Apply a minimum value to avoid divisions by near-zero. - */ - rgbMeans /= frameContext.awb.gains.max(0.01); - - return rgbMeans; -} REGISTER_IPA_ALGORITHM(Awb, "Awb") diff --git a/src/ipa/rkisp1/algorithms/awb.h b/src/ipa/rkisp1/algorithms/awb.h index 60d9ef111495..c7cb59a75bde 100644 --- a/src/ipa/rkisp1/algorithms/awb.h +++ b/src/ipa/rkisp1/algorithms/awb.h @@ -7,17 +7,25 @@ #pragma once -#include "libcamera/internal/vector.h" +#include + +#include + +#include "libcamera/internal/value_node.h" #include "libipa/awb.h" -#include "libipa/interpolator.h" +#include "libipa/fixedpoint.h" #include "algorithm.h" +#include "ipa_context.h" +#include "params.h" namespace libcamera { namespace ipa::rkisp1::algorithms { +class RkISP1AwbStats; + class Awb : public Algorithm { public: @@ -38,10 +46,10 @@ public: ControlList &metadata) override; private: - RGB calculateRgbMeans(const IPAFrameContext &frameContext, - const rkisp1_cif_isp_awb_stat *awb) const; + RkISP1AwbStats calculateRgbMeans(const IPAFrameContext &frameContext, + const rkisp1_stat_buffer *stats) const; - std::unique_ptr awbAlgo_; + AwbAlgorithm> awbAlgo_; bool rgbMode_; }; diff --git a/src/ipa/rkisp1/ipa_context.h b/src/ipa/rkisp1/ipa_context.h index e5f70d934d67..c36c1f7e0084 100644 --- a/src/ipa/rkisp1/ipa_context.h +++ b/src/ipa/rkisp1/ipa_context.h @@ -25,6 +25,7 @@ #include "libcamera/internal/vector.h" #include "libipa/agc_mean_luminance.h" +#include "libipa/awb.h" #include "libipa/camera_sensor_helper.h" #include "libipa/fc_queue.h" #include "libipa/fixedpoint.h" @@ -48,15 +49,17 @@ struct IPAHwSettings { bool compand; }; +struct RKISP1AwbSession { + struct rkisp1_cif_isp_window measureWindow; + bool enabled; +}; + struct IPASessionConfiguration { struct { struct rkisp1_cif_isp_window measureWindow; } agc; - struct { - struct rkisp1_cif_isp_window measureWindow; - bool enabled; - } awb; + struct RKISP1AwbSession awb; struct { bool supported; @@ -100,17 +103,7 @@ struct IPAActiveState { utils::Duration maxFrameDuration; } agc; - struct { - struct AwbState { - RGB gains; - unsigned int temperatureK; - }; - - AwbState manual; - AwbState automatic; - - bool autoEnabled; - } awb; + ipa::awb::ActiveState awb; struct { Matrix manual; @@ -176,11 +169,7 @@ struct IPAFrameContext : public FrameContext { bool autoGainModeChange; } agc; - struct { - RGB gains; - bool autoEnabled; - unsigned int temperatureK; - } awb; + ipa::awb::FrameContext awb; struct { float actualBrightness;