| Message ID | 20240417131536.484129-7-dan.scally@ideasonboard.com |
|---|---|
| State | New |
| Headers | show |
| Series |
|
| Related | show |
Hi Dan On Wed, Apr 17, 2024 at 02:15:34PM +0100, Daniel Scally wrote: > Now that the IPU3's Agc is derived from MeanLuminanceAgc we can > delete all the unecessary bespoke functions. > > Reviewed-by: Stefan Klug <stefan.klug@ideasonboard.com> > Signed-off-by: Daniel Scally <dan.scally@ideasonboard.com> > --- > Changes in v2: > > - Kept the documentation for estimateLuminance() > > src/ipa/ipu3/algorithms/agc.cpp | 227 +------------------------------- > src/ipa/ipu3/algorithms/agc.h | 13 -- > 2 files changed, 3 insertions(+), 237 deletions(-) > > diff --git a/src/ipa/ipu3/algorithms/agc.cpp b/src/ipa/ipu3/algorithms/agc.cpp > index 3b9761bd..46fc3b33 100644 > --- a/src/ipa/ipu3/algorithms/agc.cpp > +++ b/src/ipa/ipu3/algorithms/agc.cpp > @@ -125,8 +125,6 @@ int Agc::configure(IPAContext &context, > activeState.agc.gain = minAnalogueGain_; > activeState.agc.exposure = 10ms / configuration.sensor.lineDuration; > > - frameCount_ = 0; > - > context.activeState.agc.constraintMode = constraintModes().begin()->first; > context.activeState.agc.exposureMode = exposureModeHelpers().begin()->first; > > @@ -139,42 +137,6 @@ int Agc::configure(IPAContext &context, > return 0; > } > > -/** > - * \brief Estimate the mean value of the top 2% of the histogram > - * \param[in] stats The statistics computed by the ImgU > - * \param[in] grid The grid used to store the statistics in the IPU3 > - * \return The mean value of the top 2% of the histogram > - */ > -double Agc::measureBrightness(const ipu3_uapi_stats_3a *stats, > - const ipu3_uapi_grid_config &grid) const > -{ > - /* Initialise the histogram array */ > - uint32_t hist[knumHistogramBins] = { 0 }; > - > - for (unsigned int cellY = 0; cellY < grid.height; cellY++) { > - for (unsigned int cellX = 0; cellX < grid.width; cellX++) { > - uint32_t cellPosition = cellY * stride_ + cellX; > - > - const ipu3_uapi_awb_set_item *cell = > - reinterpret_cast<const ipu3_uapi_awb_set_item *>( > - &stats->awb_raw_buffer.meta_data[cellPosition] > - ); > - > - uint8_t gr = cell->Gr_avg; > - uint8_t gb = cell->Gb_avg; > - /* > - * Store the average green value to estimate the > - * brightness. Even the overexposed pixels are > - * taken into account. > - */ > - hist[(gr + gb) / 2]++; > - } > - } > - > - /* Estimate the quantile mean of the top 2% of the histogram. */ > - return Histogram(Span<uint32_t>(hist)).interQuantileMean(0.98, 1.0); > -} > - > Histogram Agc::parseStatistics(const ipu3_uapi_stats_3a *stats, > const ipu3_uapi_grid_config &grid) > { > @@ -208,123 +170,9 @@ Histogram Agc::parseStatistics(const ipu3_uapi_stats_3a *stats, > return Histogram(Span<uint32_t>(hist)); > } > > -/** > - * \brief Apply a filter on the exposure value to limit the speed of changes > - * \param[in] exposureValue The target exposure from the AGC algorithm > - * > - * The speed of the filter is adaptive, and will produce the target quicker > - * during startup, or when the target exposure is within 20% of the most recent > - * filter output. > - * > - * \return The filtered exposure > - */ > -utils::Duration Agc::filterExposure(utils::Duration exposureValue) > -{ > - double speed = 0.2; > - > - /* Adapt instantly if we are in startup phase. */ > - if (frameCount_ < kNumStartupFrames) > - speed = 1.0; > - > - /* > - * If we are close to the desired result, go faster to avoid making > - * multiple micro-adjustments. > - * \todo Make this customisable? > - */ > - if (filteredExposure_ < 1.2 * exposureValue && > - filteredExposure_ > 0.8 * exposureValue) > - speed = sqrt(speed); > - > - filteredExposure_ = speed * exposureValue + > - filteredExposure_ * (1.0 - speed); > - > - LOG(IPU3Agc, Debug) << "After filtering, exposure " << filteredExposure_; > - > - return filteredExposure_; > -} > - > -/** > - * \brief Estimate the new exposure and gain values > - * \param[inout] frameContext The shared IPA frame Context > - * \param[in] yGain The gain calculated based on the relative luminance target > - * \param[in] iqMeanGain The gain calculated based on the relative luminance target > - */ > -void Agc::computeExposure(IPAContext &context, IPAFrameContext &frameContext, > - double yGain, double iqMeanGain) > -{ > - const IPASessionConfiguration &configuration = context.configuration; > - /* Get the effective exposure and gain applied on the sensor. */ > - uint32_t exposure = frameContext.sensor.exposure; > - double analogueGain = frameContext.sensor.gain; > - > - /* Use the highest of the two gain estimates. */ > - double evGain = std::max(yGain, iqMeanGain); > - > - /* Consider within 1% of the target as correctly exposed */ > - if (utils::abs_diff(evGain, 1.0) < 0.01) > - LOG(IPU3Agc, Debug) << "We are well exposed (evGain = " > - << evGain << ")"; > - > - /* extracted from Rpi::Agc::computeTargetExposure */ > - > - /* Calculate the shutter time in seconds */ > - utils::Duration currentShutter = exposure * configuration.sensor.lineDuration; > - > - /* > - * Update the exposure value for the next computation using the values > - * of exposure and gain really used by the sensor. > - */ > - utils::Duration effectiveExposureValue = currentShutter * analogueGain; > - > - LOG(IPU3Agc, Debug) << "Actual total exposure " << currentShutter * analogueGain > - << " Shutter speed " << currentShutter > - << " Gain " << analogueGain > - << " Needed ev gain " << evGain; > - > - /* > - * Calculate the current exposure value for the scene as the latest > - * exposure value applied multiplied by the new estimated gain. > - */ > - utils::Duration exposureValue = effectiveExposureValue * evGain; > - > - /* Clamp the exposure value to the min and max authorized */ > - utils::Duration maxTotalExposure = maxShutterSpeed_ * maxAnalogueGain_; > - exposureValue = std::min(exposureValue, maxTotalExposure); > - LOG(IPU3Agc, Debug) << "Target total exposure " << exposureValue > - << ", maximum is " << maxTotalExposure; > - > - /* > - * Filter the exposure. > - * \todo estimate if we need to desaturate > - */ > - exposureValue = filterExposure(exposureValue); > - > - /* > - * Divide the exposure value as new exposure and gain values. > - * > - * Push the shutter time up to the maximum first, and only then > - * increase the gain. > - */ > - utils::Duration shutterTime = > - std::clamp<utils::Duration>(exposureValue / minAnalogueGain_, > - minShutterSpeed_, maxShutterSpeed_); > - double stepGain = std::clamp(exposureValue / shutterTime, > - minAnalogueGain_, maxAnalogueGain_); > - LOG(IPU3Agc, Debug) << "Divided up shutter and gain are " > - << shutterTime << " and " > - << stepGain; > -} > - > /** > * \brief Estimate the relative luminance of the frame with a given gain > - * \param[in] frameContext The shared IPA frame context > - * \param[in] grid The grid used to store the statistics in the IPU3 > - * \param[in] stats The IPU3 statistics and ISP results > - * \param[in] gain The gain to apply to the frame > - * \return The relative luminance > - * > - * This function estimates the average relative luminance of the frame that > - * would be output by the sensor if an additional \a gain was applied. > + * \param[in] gain The gain to apply in estimating luminance > * > * The estimation is based on the AWB statistics for the current frame. Red, > * green and blue averages for all cells are first multiplied by the gain, and > @@ -339,42 +187,9 @@ void Agc::computeExposure(IPAContext &context, IPAFrameContext &frameContext, > * > * More detailed information can be found in: > * https://en.wikipedia.org/wiki/Relative_luminance > + * > + * @return The relative luminance of the frame While this is valid Doxygen, it's usually "\return" With this fixed Reviewed-by: Jacopo Mondi <jacopo.mondi@ideasonboard.com> Thanks j > */ > -double Agc::estimateLuminance(IPAActiveState &activeState, > - const ipu3_uapi_grid_config &grid, > - const ipu3_uapi_stats_3a *stats, > - double gain) > -{ > - double redSum = 0, greenSum = 0, blueSum = 0; > - > - /* Sum the per-channel averages, saturated to 255. */ > - for (unsigned int cellY = 0; cellY < grid.height; cellY++) { > - for (unsigned int cellX = 0; cellX < grid.width; cellX++) { > - uint32_t cellPosition = cellY * stride_ + cellX; > - > - const ipu3_uapi_awb_set_item *cell = > - reinterpret_cast<const ipu3_uapi_awb_set_item *>( > - &stats->awb_raw_buffer.meta_data[cellPosition] > - ); > - const uint8_t G_avg = (cell->Gr_avg + cell->Gb_avg) / 2; > - > - redSum += std::min(cell->R_avg * gain, 255.0); > - greenSum += std::min(G_avg * gain, 255.0); > - blueSum += std::min(cell->B_avg * gain, 255.0); > - } > - } > - > - /* > - * Apply the AWB gains to approximate colours correctly, use the Rec. > - * 601 formula to calculate the relative luminance, and normalize it. > - */ > - double ySum = redSum * activeState.awb.gains.red * 0.299 > - + greenSum * activeState.awb.gains.green * 0.587 > - + blueSum * activeState.awb.gains.blue * 0.114; > - > - return ySum / (grid.height * grid.width) / 255; > -} > - > double Agc::estimateLuminance(double gain) > { > double redSum = 0, greenSum = 0, blueSum = 0; > @@ -408,42 +223,6 @@ void Agc::process(IPAContext &context, [[maybe_unused]] const uint32_t frame, > const ipu3_uapi_stats_3a *stats, > ControlList &metadata) > { > - /* > - * Estimate the gain needed to have the proportion of pixels in a given > - * desired range. iqMean is the mean value of the top 2% of the > - * cumulative histogram, and we want it to be as close as possible to a > - * configured target. > - */ > - double iqMean = measureBrightness(stats, context.configuration.grid.bdsGrid); > - double iqMeanGain = kEvGainTarget * knumHistogramBins / iqMean; > - > - /* > - * Estimate the gain needed to achieve a relative luminance target. To > - * account for non-linearity caused by saturation, the value needs to be > - * estimated in an iterative process, as multiplying by a gain will not > - * increase the relative luminance by the same factor if some image > - * regions are saturated. > - */ > - double yGain = 1.0; > - double yTarget = kRelativeLuminanceTarget; > - > - for (unsigned int i = 0; i < 8; i++) { > - double yValue = estimateLuminance(context.activeState, > - context.configuration.grid.bdsGrid, > - stats, yGain); > - double extraGain = std::min(10.0, yTarget / (yValue + .001)); > - > - yGain *= extraGain; > - LOG(IPU3Agc, Debug) << "Y value: " << yValue > - << ", Y target: " << yTarget > - << ", gives gain " << yGain; > - if (extraGain < 1.01) > - break; > - } > - > - computeExposure(context, frameContext, yGain, iqMeanGain); > - frameCount_++; > - > Histogram hist = parseStatistics(stats, context.configuration.grid.bdsGrid); > rGain_ = context.activeState.awb.gains.red; > gGain_ = context.activeState.awb.gains.blue; > diff --git a/src/ipa/ipu3/algorithms/agc.h b/src/ipa/ipu3/algorithms/agc.h > index 40f32188..945d1846 100644 > --- a/src/ipa/ipu3/algorithms/agc.h > +++ b/src/ipa/ipu3/algorithms/agc.h > @@ -38,29 +38,16 @@ public: > ControlList &metadata) override; > > private: > - double measureBrightness(const ipu3_uapi_stats_3a *stats, > - const ipu3_uapi_grid_config &grid) const; > - utils::Duration filterExposure(utils::Duration currentExposure); > - void computeExposure(IPAContext &context, IPAFrameContext &frameContext, > - double yGain, double iqMeanGain); > - double estimateLuminance(IPAActiveState &activeState, > - const ipu3_uapi_grid_config &grid, > - const ipu3_uapi_stats_3a *stats, > - double gain); > double estimateLuminance(double gain) override; > Histogram parseStatistics(const ipu3_uapi_stats_3a *stats, > const ipu3_uapi_grid_config &grid); > > - uint64_t frameCount_; > - > utils::Duration minShutterSpeed_; > utils::Duration maxShutterSpeed_; > > double minAnalogueGain_; > double maxAnalogueGain_; > > - utils::Duration filteredExposure_; > - > uint32_t stride_; > double rGain_; > double gGain_; > -- > 2.34.1 >
diff --git a/src/ipa/ipu3/algorithms/agc.cpp b/src/ipa/ipu3/algorithms/agc.cpp index 3b9761bd..46fc3b33 100644 --- a/src/ipa/ipu3/algorithms/agc.cpp +++ b/src/ipa/ipu3/algorithms/agc.cpp @@ -125,8 +125,6 @@ int Agc::configure(IPAContext &context, activeState.agc.gain = minAnalogueGain_; activeState.agc.exposure = 10ms / configuration.sensor.lineDuration; - frameCount_ = 0; - context.activeState.agc.constraintMode = constraintModes().begin()->first; context.activeState.agc.exposureMode = exposureModeHelpers().begin()->first; @@ -139,42 +137,6 @@ int Agc::configure(IPAContext &context, return 0; } -/** - * \brief Estimate the mean value of the top 2% of the histogram - * \param[in] stats The statistics computed by the ImgU - * \param[in] grid The grid used to store the statistics in the IPU3 - * \return The mean value of the top 2% of the histogram - */ -double Agc::measureBrightness(const ipu3_uapi_stats_3a *stats, - const ipu3_uapi_grid_config &grid) const -{ - /* Initialise the histogram array */ - uint32_t hist[knumHistogramBins] = { 0 }; - - for (unsigned int cellY = 0; cellY < grid.height; cellY++) { - for (unsigned int cellX = 0; cellX < grid.width; cellX++) { - uint32_t cellPosition = cellY * stride_ + cellX; - - const ipu3_uapi_awb_set_item *cell = - reinterpret_cast<const ipu3_uapi_awb_set_item *>( - &stats->awb_raw_buffer.meta_data[cellPosition] - ); - - uint8_t gr = cell->Gr_avg; - uint8_t gb = cell->Gb_avg; - /* - * Store the average green value to estimate the - * brightness. Even the overexposed pixels are - * taken into account. - */ - hist[(gr + gb) / 2]++; - } - } - - /* Estimate the quantile mean of the top 2% of the histogram. */ - return Histogram(Span<uint32_t>(hist)).interQuantileMean(0.98, 1.0); -} - Histogram Agc::parseStatistics(const ipu3_uapi_stats_3a *stats, const ipu3_uapi_grid_config &grid) { @@ -208,123 +170,9 @@ Histogram Agc::parseStatistics(const ipu3_uapi_stats_3a *stats, return Histogram(Span<uint32_t>(hist)); } -/** - * \brief Apply a filter on the exposure value to limit the speed of changes - * \param[in] exposureValue The target exposure from the AGC algorithm - * - * The speed of the filter is adaptive, and will produce the target quicker - * during startup, or when the target exposure is within 20% of the most recent - * filter output. - * - * \return The filtered exposure - */ -utils::Duration Agc::filterExposure(utils::Duration exposureValue) -{ - double speed = 0.2; - - /* Adapt instantly if we are in startup phase. */ - if (frameCount_ < kNumStartupFrames) - speed = 1.0; - - /* - * If we are close to the desired result, go faster to avoid making - * multiple micro-adjustments. - * \todo Make this customisable? - */ - if (filteredExposure_ < 1.2 * exposureValue && - filteredExposure_ > 0.8 * exposureValue) - speed = sqrt(speed); - - filteredExposure_ = speed * exposureValue + - filteredExposure_ * (1.0 - speed); - - LOG(IPU3Agc, Debug) << "After filtering, exposure " << filteredExposure_; - - return filteredExposure_; -} - -/** - * \brief Estimate the new exposure and gain values - * \param[inout] frameContext The shared IPA frame Context - * \param[in] yGain The gain calculated based on the relative luminance target - * \param[in] iqMeanGain The gain calculated based on the relative luminance target - */ -void Agc::computeExposure(IPAContext &context, IPAFrameContext &frameContext, - double yGain, double iqMeanGain) -{ - const IPASessionConfiguration &configuration = context.configuration; - /* Get the effective exposure and gain applied on the sensor. */ - uint32_t exposure = frameContext.sensor.exposure; - double analogueGain = frameContext.sensor.gain; - - /* Use the highest of the two gain estimates. */ - double evGain = std::max(yGain, iqMeanGain); - - /* Consider within 1% of the target as correctly exposed */ - if (utils::abs_diff(evGain, 1.0) < 0.01) - LOG(IPU3Agc, Debug) << "We are well exposed (evGain = " - << evGain << ")"; - - /* extracted from Rpi::Agc::computeTargetExposure */ - - /* Calculate the shutter time in seconds */ - utils::Duration currentShutter = exposure * configuration.sensor.lineDuration; - - /* - * Update the exposure value for the next computation using the values - * of exposure and gain really used by the sensor. - */ - utils::Duration effectiveExposureValue = currentShutter * analogueGain; - - LOG(IPU3Agc, Debug) << "Actual total exposure " << currentShutter * analogueGain - << " Shutter speed " << currentShutter - << " Gain " << analogueGain - << " Needed ev gain " << evGain; - - /* - * Calculate the current exposure value for the scene as the latest - * exposure value applied multiplied by the new estimated gain. - */ - utils::Duration exposureValue = effectiveExposureValue * evGain; - - /* Clamp the exposure value to the min and max authorized */ - utils::Duration maxTotalExposure = maxShutterSpeed_ * maxAnalogueGain_; - exposureValue = std::min(exposureValue, maxTotalExposure); - LOG(IPU3Agc, Debug) << "Target total exposure " << exposureValue - << ", maximum is " << maxTotalExposure; - - /* - * Filter the exposure. - * \todo estimate if we need to desaturate - */ - exposureValue = filterExposure(exposureValue); - - /* - * Divide the exposure value as new exposure and gain values. - * - * Push the shutter time up to the maximum first, and only then - * increase the gain. - */ - utils::Duration shutterTime = - std::clamp<utils::Duration>(exposureValue / minAnalogueGain_, - minShutterSpeed_, maxShutterSpeed_); - double stepGain = std::clamp(exposureValue / shutterTime, - minAnalogueGain_, maxAnalogueGain_); - LOG(IPU3Agc, Debug) << "Divided up shutter and gain are " - << shutterTime << " and " - << stepGain; -} - /** * \brief Estimate the relative luminance of the frame with a given gain - * \param[in] frameContext The shared IPA frame context - * \param[in] grid The grid used to store the statistics in the IPU3 - * \param[in] stats The IPU3 statistics and ISP results - * \param[in] gain The gain to apply to the frame - * \return The relative luminance - * - * This function estimates the average relative luminance of the frame that - * would be output by the sensor if an additional \a gain was applied. + * \param[in] gain The gain to apply in estimating luminance * * The estimation is based on the AWB statistics for the current frame. Red, * green and blue averages for all cells are first multiplied by the gain, and @@ -339,42 +187,9 @@ void Agc::computeExposure(IPAContext &context, IPAFrameContext &frameContext, * * More detailed information can be found in: * https://en.wikipedia.org/wiki/Relative_luminance + * + * @return The relative luminance of the frame */ -double Agc::estimateLuminance(IPAActiveState &activeState, - const ipu3_uapi_grid_config &grid, - const ipu3_uapi_stats_3a *stats, - double gain) -{ - double redSum = 0, greenSum = 0, blueSum = 0; - - /* Sum the per-channel averages, saturated to 255. */ - for (unsigned int cellY = 0; cellY < grid.height; cellY++) { - for (unsigned int cellX = 0; cellX < grid.width; cellX++) { - uint32_t cellPosition = cellY * stride_ + cellX; - - const ipu3_uapi_awb_set_item *cell = - reinterpret_cast<const ipu3_uapi_awb_set_item *>( - &stats->awb_raw_buffer.meta_data[cellPosition] - ); - const uint8_t G_avg = (cell->Gr_avg + cell->Gb_avg) / 2; - - redSum += std::min(cell->R_avg * gain, 255.0); - greenSum += std::min(G_avg * gain, 255.0); - blueSum += std::min(cell->B_avg * gain, 255.0); - } - } - - /* - * Apply the AWB gains to approximate colours correctly, use the Rec. - * 601 formula to calculate the relative luminance, and normalize it. - */ - double ySum = redSum * activeState.awb.gains.red * 0.299 - + greenSum * activeState.awb.gains.green * 0.587 - + blueSum * activeState.awb.gains.blue * 0.114; - - return ySum / (grid.height * grid.width) / 255; -} - double Agc::estimateLuminance(double gain) { double redSum = 0, greenSum = 0, blueSum = 0; @@ -408,42 +223,6 @@ void Agc::process(IPAContext &context, [[maybe_unused]] const uint32_t frame, const ipu3_uapi_stats_3a *stats, ControlList &metadata) { - /* - * Estimate the gain needed to have the proportion of pixels in a given - * desired range. iqMean is the mean value of the top 2% of the - * cumulative histogram, and we want it to be as close as possible to a - * configured target. - */ - double iqMean = measureBrightness(stats, context.configuration.grid.bdsGrid); - double iqMeanGain = kEvGainTarget * knumHistogramBins / iqMean; - - /* - * Estimate the gain needed to achieve a relative luminance target. To - * account for non-linearity caused by saturation, the value needs to be - * estimated in an iterative process, as multiplying by a gain will not - * increase the relative luminance by the same factor if some image - * regions are saturated. - */ - double yGain = 1.0; - double yTarget = kRelativeLuminanceTarget; - - for (unsigned int i = 0; i < 8; i++) { - double yValue = estimateLuminance(context.activeState, - context.configuration.grid.bdsGrid, - stats, yGain); - double extraGain = std::min(10.0, yTarget / (yValue + .001)); - - yGain *= extraGain; - LOG(IPU3Agc, Debug) << "Y value: " << yValue - << ", Y target: " << yTarget - << ", gives gain " << yGain; - if (extraGain < 1.01) - break; - } - - computeExposure(context, frameContext, yGain, iqMeanGain); - frameCount_++; - Histogram hist = parseStatistics(stats, context.configuration.grid.bdsGrid); rGain_ = context.activeState.awb.gains.red; gGain_ = context.activeState.awb.gains.blue; diff --git a/src/ipa/ipu3/algorithms/agc.h b/src/ipa/ipu3/algorithms/agc.h index 40f32188..945d1846 100644 --- a/src/ipa/ipu3/algorithms/agc.h +++ b/src/ipa/ipu3/algorithms/agc.h @@ -38,29 +38,16 @@ public: ControlList &metadata) override; private: - double measureBrightness(const ipu3_uapi_stats_3a *stats, - const ipu3_uapi_grid_config &grid) const; - utils::Duration filterExposure(utils::Duration currentExposure); - void computeExposure(IPAContext &context, IPAFrameContext &frameContext, - double yGain, double iqMeanGain); - double estimateLuminance(IPAActiveState &activeState, - const ipu3_uapi_grid_config &grid, - const ipu3_uapi_stats_3a *stats, - double gain); double estimateLuminance(double gain) override; Histogram parseStatistics(const ipu3_uapi_stats_3a *stats, const ipu3_uapi_grid_config &grid); - uint64_t frameCount_; - utils::Duration minShutterSpeed_; utils::Duration maxShutterSpeed_; double minAnalogueGain_; double maxAnalogueGain_; - utils::Duration filteredExposure_; - uint32_t stride_; double rGain_; double gGain_;