diff --git a/src/ipa/ipu3/algorithms/agc_metering.cpp b/src/ipa/ipu3/algorithms/agc_metering.cpp new file mode 100644 index 00000000..1dc05082 --- /dev/null +++ b/src/ipa/ipu3/algorithms/agc_metering.cpp @@ -0,0 +1,427 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Based on the implementation from the Raspberry Pi IPA, + * Copyright (C) 2019-2021, Raspberry Pi (Trading) Ltd. + * Copyright (C) 2021, Google inc. + * + * agc_metering.cpp - AGC/AEC metering-based control algorithm + */ + +#include "agc_metering.h" +#include "awb.h" + +#include +#include +#include +#include + +#include + +#include +#include + +#include "libipa/histogram.h" + +/** + * \file agc_metering.h + */ + +namespace libcamera { + +using namespace std::literals::chrono_literals; + +namespace ipa::ipu3::algorithms { + +/** + * \class AgcMetering + * \brief The class to use the metering-based auto-exposure algorithm + * + * The metering-based algorithm is calculating an exposure and gain value such + * as a given quantity of pixels lie in the top 2% of the histogram. The AWB + * gains are also used here, and all cells in the grid are weighted using a + * specific metering matrix. The default here is Spot metering. + */ + +LOG_DEFINE_CATEGORY(IPU3AgcMetering) + +/* Histogram constants */ +static constexpr uint32_t knumHistogramBins = 256; + +/* seems to be a 10-bit pipeline */ +static constexpr uint8_t kPipelineBits = 10; + +/* width of the AGC stats grid */ +static constexpr uint32_t kAgcStatsSizeX = 7; +/* height of the AGC stats grid */ +static constexpr uint32_t kAgcStatsSizeY = 5; +/* size of the AGC stats grid */ +static constexpr uint32_t kAgcStatsSize = kAgcStatsSizeX * kAgcStatsSizeY; + +/** + * The AGC algorithm uses region-based metering. + * The image is divided up into regions as: + * + * +--+--------------+--+ + * |11| 9 |12| + * +--+--+--------+--+--+ + * | | | 3 | | | + * | | +--+--+--+ | | + * |7 |5 |1 |0 |2 |6 |8 | + * | | +--+--+--+ | | + * | | | 4 | | | + * +--+--+--------+--+--+ + * |13| 10 |14| + * +--+--------------+--+ + * + * The metering-based algorithm is calculating an exposure and gain value such + * as a given quantity of weighted pixels lie in the top 2% of the histogram.The + * AWB gains applied are also used to estimate the total gain to apply. + * + * An average luminance value for the image is calculated according to: + * \f$Y = \frac{\sum_{i=0}^{i=kNumAgcWeightedZones}{kCenteredWeights_{i}Y_{i}}} + * {\sum_{i=0}^{i=kNumAgcWeightedZones}{w_{i}}}\f$ + */ + +/* Weight applied on each region */ +static constexpr double kSpotWeights[kNumAgcWeightedZones] = { 2, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }; + +/* Region number repartition in the image */ +static constexpr uint32_t kAgcStatsRegions[kAgcStatsSize] = { + 11, 9, 9, 9, 9, 9, 12, + 7, 5, 3, 3, 3, 6, 8, + 7, 5, 1, 0, 2, 6, 8, + 7, 5, 4, 4, 4, 6, 8, + 13, 10, 10, 10, 10, 10, 14 +}; + +/* Limit the speed of change between two exposure levels */ +static constexpr double kFastReduceThreshold = 0.3; + +AgcMetering::AgcMetering() + : iqMean_(0.0), prevExposure_(0s), prevExposureNoDg_(0s), + currentExposure_(0s), currentExposureNoDg_(0s), currentShutter_(1.0s), + currentAnalogueGain_(1.0) +{ +} + +/** + * \brief Configure the AGC given a configInfo + * \param[in] context The shared IPA context + * \param[in] configInfo The IPA configuration data, received from the pipeline + * handler + * + * \return 0 + */ +int AgcMetering::configure(IPAContext &context, const IPAConfigInfo &configInfo) +{ + /* Store the line duration in the IPASessionConfiguration */ + context.configuration.agc.lineDuration = configInfo.sensorInfo.lineLength + * (1.0s / configInfo.sensorInfo.pixelRate); + + /* \todo: those values need to be extracted from a configuration file */ + shutterConstraints_.push_back(100us); + shutterConstraints_.push_back(10ms); + shutterConstraints_.push_back(33ms); + gainConstraints_.push_back(1.0); + gainConstraints_.push_back(4.0); + gainConstraints_.push_back(16.0); + + fixedShutter_ = 0s; + fixedAnalogueGain_ = 0.0; + + return 0; +} + +/** + * \brief Translate the IPU3 statistics to AGC regions + * \param[in] stats The statistics buffer coming from the pipeline handler + * \param[in] grid The grid used to store the statistics in the IPU3 + */ +void AgcMetering::generateStats(const ipu3_uapi_stats_3a *stats, + const ipu3_uapi_grid_config &grid) +{ + /* We need to have a AGC grid of kAgcStatsSizeX * kAgcStatsSizeY */ + uint32_t regionWidth = round(grid.width / static_cast(kAgcStatsSizeX)); + uint32_t regionHeight = round(grid.height / static_cast(kAgcStatsSizeY)); + uint32_t hist[knumHistogramBins] = { 0 }; + + /* Clear the statistics of the previous frame */ + for (unsigned int i = 0; i < kNumAgcWeightedZones; i++) { + agcStats_[i].bSum = 0; + agcStats_[i].rSum = 0; + agcStats_[i].gSum = 0; + agcStats_[i].counted = 0; + agcStats_[i].total = 0; + } + + LOG(IPU3AgcMetering, Debug) << "[" << (int)grid.width + << "x" << (int)grid.height << "] cells" + << " scaled to [" << regionWidth + << "x" << regionHeight << "] AGC regions"; + + /* + * Generate a (kAgcStatsSizeX x kAgcStatsSizeY) array from the IPU3 grid + * which is (grid.width x grid.height). + */ + for (unsigned int j = 0; j < kAgcStatsSizeY * regionHeight; j++) { + for (unsigned int i = 0; i < kAgcStatsSizeX * regionWidth; i++) { + uint32_t cellPosition = j * grid.width + i; + uint32_t cellX = (cellPosition / regionWidth) + % kAgcStatsSizeX; + uint32_t cellY = ((cellPosition / grid.width) / regionHeight) + % kAgcStatsSizeY; + + uint32_t agcRegionPosition = kAgcStatsRegions[cellY * kAgcStatsSizeX + cellX]; + weights_[agcRegionPosition] = kSpotWeights[agcRegionPosition]; + cellPosition *= sizeof(Ipu3AwbCell); + + /* Cast the initial IPU3 structure to simplify the reading */ + Ipu3AwbCell *currentCell = reinterpret_cast(const_cast(&stats->awb_raw_buffer.meta_data[cellPosition])); + if (currentCell->satRatio == 0) { + /* The cell is not saturated, use the current cell */ + agcStats_[agcRegionPosition].counted++; + uint32_t greenValue = currentCell->greenRedAvg + currentCell->greenBlueAvg; + hist[greenValue / 2]++; + agcStats_[agcRegionPosition].gSum += greenValue / 2; + agcStats_[agcRegionPosition].rSum += currentCell->redAvg; + agcStats_[agcRegionPosition].bSum += currentCell->blueAvg; + } + } + } + + /* Estimate the quantile mean of the top 2% of the histogram */ + iqMean_ = Histogram(Span(hist)).interQuantileMean(0.98, 1.0); +} + +/** + * \brief Apply a filter on the exposure value to limit the speed of changes + */ +void AgcMetering::filterExposure() +{ + double speed = 0.08; + if (prevExposure_ == 0s) { + /* DG stands for digital gain.*/ + prevExposure_ = currentExposure_; + prevExposureNoDg_ = currentExposureNoDg_; + } else { + /* + * If we are close to the desired result, go faster to avoid + * making multiple micro-adjustments. + * \todo: Make this customisable? + */ + if (prevExposure_ < 1.2 * currentExposure_ && + prevExposure_ > 0.8 * currentExposure_) + speed = sqrt(speed); + + prevExposure_ = speed * currentExposure_ + + prevExposure_ * (1.0 - speed); + prevExposureNoDg_ = speed * currentExposureNoDg_ + + prevExposureNoDg_ * (1.0 - speed); + } + /* + * We can't let the no_dg exposure deviate too far below the + * total exposure, as there might not be enough digital gain available + * in the ISP to hide it (which will cause nasty oscillation). + * \todo: add the support for digital gain + */ + if (prevExposureNoDg_ < + prevExposure_ * kFastReduceThreshold) + prevExposureNoDg_ = prevExposure_ * kFastReduceThreshold; + LOG(IPU3AgcMetering, Debug) << "After filtering, total_exposure " << prevExposure_; +} + +/** + * \brief Estimate the weighted brightness + * \param[in] gain The current gain applied + * \param[in] context The shared IPA context + */ +double AgcMetering::computeInitialY(double gain, IPAContext &context) +{ + /* + * Note how the calculation below means that equal weights_ give you + * "average" metering (i.e. all pixels equally important). + */ + double redSum = 0, greenSum = 0, blueSum = 0, pixelSum = 0; + for (unsigned int i = 0; i < kNumAgcWeightedZones; i++) { + /* We will exclude the saturated pixels from the sum */ + double counted = agcStats_[i].counted; + double rSum = std::min(agcStats_[i].rSum * gain, ((1 << kPipelineBits) - 1) * counted); + double gSum = std::min(agcStats_[i].gSum * gain, ((1 << kPipelineBits) - 1) * counted); + double bSum = std::min(agcStats_[i].bSum * gain, ((1 << kPipelineBits) - 1) * counted); + /* Weight each channel with the selected metering method */ + redSum += rSum * weights_[i]; + greenSum += gSum * weights_[i]; + blueSum += bSum * weights_[i]; + pixelSum += counted * weights_[i]; + } + /* We don't want to have a division by 0.0 :-) */ + if (pixelSum == 0.0) { + LOG(IPU3AgcMetering, Warning) << "computeInitialY: pixel_sum is zero"; + return 0; + } + /* + * Estimate the sum of the brightness values, weighted with the gains + * applied on the channels in AWB. + */ + double Y_sum = redSum * context.frameContext.awb.gains.red * .299 + + greenSum * context.frameContext.awb.gains.green * .587 + + blueSum * context.frameContext.awb.gains.blue * .114; + + /* And return the average brightness */ + return Y_sum / pixelSum / (1 << kPipelineBits); +} + +/** + * \brief Compute the exposure value + * \param[in] gain The current gain applied + */ +void AgcMetering::computeTargetExposure(double gain) +{ + currentExposure_ = currentExposureNoDg_ * gain; + /* \todo: have a list of shutter speeds */ + Duration maxShutterSpeed = shutterConstraints_.back(); + Duration maxTotalExposure = maxShutterSpeed * gainConstraints_.back(); + + currentExposure_ = std::min(currentExposure_, maxTotalExposure); + LOG(IPU3AgcMetering, Debug) << "Target total_exposure " << currentExposure_; +} + +/** + * \brief Split exposure value as shutter time and gain + */ +void AgcMetering::divideUpExposure() +{ + Duration exposureValue = prevExposure_; + Duration shutterTime; + double analogueGain; + shutterTime = shutterConstraints_[0]; + shutterTime = std::min(shutterTime, shutterConstraints_.back()); + analogueGain = gainConstraints_[0]; + + /** + * We have an exposure profile with a list of shutter time and gains + * An example is graphed below: + * + * gain shutter time + * (ms) + * ^ ^ + * | | + * 8x+---------------------------------------------------------xxxxx+30 + * | xxxxx | + * | xxxxx | + * | xxxxx | + * | xxxxx | + * | xxxxx | + * | xxxxx | + * | xxxxx | + * | xxxxx | + * | xxxxx | + * 4x+----------xx--------------------------------------------------+10 + * | xx | + * | xx | + * | xx | + * | xx | + * | xx | + * 1x+--xx----------------------------------------------------------+0.1 + * | x | + * |x | + * +---------------------------------------------------------------> + * total exposure + */ + if (shutterTime * analogueGain < exposureValue) { + for (unsigned int stage = 1; + stage < gainConstraints_.size(); stage++) { + if (fixedShutter_ == 0s) { + Duration stageShutter = + std::min(shutterConstraints_[stage], shutterConstraints_.back()); + if (stageShutter * analogueGain >= + exposureValue) { + shutterTime = + exposureValue / analogueGain; + break; + } + shutterTime = stageShutter; + } + if (fixedAnalogueGain_ == 0.0) { + if (gainConstraints_[stage] * shutterTime >= exposureValue) { + analogueGain = exposureValue / shutterTime; + break; + } + analogueGain = gainConstraints_[stage]; + } + } + } + LOG(IPU3AgcMetering, Debug) << "Divided up shutter and gain are " + << shutterTime << " and " << analogueGain; + + /* \todo: flickering avoidance ? */ + filteredShutter_ = shutterTime; + filteredAnalogueGain_ = analogueGain; +} + +/** + * \brief Calculate the gain for the target value to be in the top 2% of the + * histogram + * \param[in] currentGain The current gain applied + * \param[in] context The shared IPA context + */ +void AgcMetering::computeGain(double ¤tGain, IPAContext &context) +{ + currentGain = 1.0; + /* \todo: the target Y needs to be grabbed from a configuration */ + double targetY = 0.162; + for (int i = 0; i < 8; i++) { + double initialY = computeInitialY(currentGain, context); + double extra_gain = std::min(10.0, targetY / (initialY + .001)); + + currentGain *= extra_gain; + LOG(IPU3AgcMetering, Debug) << "Initial Y " << initialY + << " target " << targetY + << " gives gain " << currentGain; + if (extra_gain < 1.01) + break; + } + + /* + * Require the top 2% of pixels to lie at or below 0.5 in the pixel + * range (for a range from 0 to 255, it is 205). This lowers the + * exposure to stop pixels saturating. + */ + double newGain = (0.5 * knumHistogramBins) / iqMean_; + LOG(IPU3AgcMetering, Debug) << "gain: " << currentGain + << " new gain: " << newGain; + if (newGain > currentGain) + currentGain = newGain; +} + +/** + * \brief Process IPU3 statistics, and run AGC operations + * \param[in] context The shared IPA context + * \param[in] stats The IPU3 statistics and ISP results + */ +void AgcMetering::process(IPAContext &context, const ipu3_uapi_stats_3a *stats) +{ + ASSERT(stats->stats_3a_status.awb_en); + generateStats(stats, context.configuration.grid.bdsGrid); + + currentShutter_ = context.frameContext.agc.exposure + * context.configuration.agc.lineDuration; + currentAnalogueGain_ = context.frameContext.agc.gain; + + /* Estimate the current exposure value */ + currentExposureNoDg_ = currentShutter_ * currentAnalogueGain_; + + double currentGain = 1.0; + computeGain(currentGain, context); + computeTargetExposure(currentGain); + filterExposure(); + divideUpExposure(); + + context.frameContext.agc.exposure = filteredShutter_ + / context.configuration.agc.lineDuration; + context.frameContext.agc.gain = filteredAnalogueGain_; +} + +} /* namespace ipa::ipu3::algorithms */ + +} /* namespace libcamera */ diff --git a/src/ipa/ipu3/algorithms/agc_metering.h b/src/ipa/ipu3/algorithms/agc_metering.h new file mode 100644 index 00000000..4fd603e1 --- /dev/null +++ b/src/ipa/ipu3/algorithms/agc_metering.h @@ -0,0 +1,78 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Based on the implementation from the Raspberry Pi IPA, + * Copyright (C) 2019-2021, Raspberry Pi (Trading) Ltd. + * Copyright (C) 2021, Ideas On Board + * + * agc_metering.h - IPU3 AGC/AEC control algorithm + */ +#ifndef __LIBCAMERA_IPU3_AGC_H__ +#define __LIBCAMERA_IPU3_AGC_H__ + +#include + +#include + +#include + +#include "algorithm.h" +#include "awb.h" + +namespace libcamera { + +struct IPACameraSensorInfo; + +namespace ipa::ipu3::algorithms { + +using utils::Duration; + +/* Number of weighted zones for metering */ +static constexpr uint32_t kNumAgcWeightedZones = 15; + +class AgcMetering : public Algorithm +{ +public: + AgcMetering(); + ~AgcMetering() = default; + + int configure(IPAContext &context, const IPAConfigInfo &configInfo) override; + void process(IPAContext &context, const ipu3_uapi_stats_3a *stats) override; + +private: + void processBrightness(const ipu3_uapi_stats_3a *stats); + void filterExposure(); + void lockExposureGain(uint32_t &exposure, double &gain); + void generateStats(const ipu3_uapi_stats_3a *stats, + const ipu3_uapi_grid_config &grid); + void generateZones(std::vector &zones); + double computeInitialY(double gain, IPAContext &context); + void computeTargetExposure(double currentGain); + void divideUpExposure(); + void computeGain(double ¤tGain, IPAContext &context); + + double weights_[kNumAgcWeightedZones]; + struct Accumulator agcStats_[kNumAgcWeightedZones]; + + double iqMean_; + + Duration prevExposure_; + Duration prevExposureNoDg_; + Duration currentExposure_; + Duration currentExposureNoDg_; + + Duration currentShutter_; + std::vector shutterConstraints_; + Duration fixedShutter_; + Duration filteredShutter_; + + double currentAnalogueGain_; + std::vector gainConstraints_; + double fixedAnalogueGain_; + double filteredAnalogueGain_; +}; + +} /* namespace ipa::ipu3::algorithms */ + +} /* namespace libcamera */ + +#endif /* __LIBCAMERA_IPU3_AGC_H__ */ diff --git a/src/ipa/ipu3/algorithms/meson.build b/src/ipa/ipu3/algorithms/meson.build index 807b53ea..f31b2070 100644 --- a/src/ipa/ipu3/algorithms/meson.build +++ b/src/ipa/ipu3/algorithms/meson.build @@ -2,6 +2,7 @@ ipu3_ipa_algorithms = files([ 'agc_mean.cpp', + 'agc_metering.cpp', 'algorithm.cpp', 'awb.cpp', 'tone_mapping.cpp', diff --git a/src/ipa/ipu3/ipa_context.h b/src/ipa/ipu3/ipa_context.h index 3a292ad7..190a3468 100644 --- a/src/ipa/ipu3/ipa_context.h +++ b/src/ipa/ipu3/ipa_context.h @@ -10,6 +10,8 @@ #include +#include + #include namespace libcamera { @@ -17,6 +19,10 @@ namespace libcamera { namespace ipa::ipu3 { struct IPASessionConfiguration { + struct { + utils::Duration lineDuration; + } agc; + struct { ipu3_uapi_grid_config bdsGrid; Size bdsOutputSize; diff --git a/src/ipa/ipu3/ipu3.cpp b/src/ipa/ipu3/ipu3.cpp index 6332fc06..be4a082a 100644 --- a/src/ipa/ipu3/ipu3.cpp +++ b/src/ipa/ipu3/ipu3.cpp @@ -81,6 +81,14 @@ * are run. This needs to be turned into real per-frame data storage. */ +/** + * \struct IPASessionConfiguration::agc + * \brief AGC configuration of the IPA + * + * \var IPASessionConfiguration::agc::lineDuration + * \brief Duration of one line dependant on the sensor configuration + */ + /** * \struct IPASessionConfiguration::grid * \brief Grid configuration of the IPA