| Message ID | 20240322131451.3092931-11-dan.scally@ideasonboard.com |
|---|---|
| State | Superseded |
| Headers | show |
| Series |
|
| Related | show |
Hi Daniel, thanks for the patch. I didn't test it, but that will get cought by CI. Does every commit build? I just noticed, that the filterExposure missed an overwrite in the previous patch. Otherwise looks good to me. Reviewed-by: Stefan Klug <stefan.klug@ideasonboard.com> Cheers, Stefan On Fri, Mar 22, 2024 at 01:14:51PM +0000, Daniel Scally wrote: > Now that the rkisp1 Agc algorithm is a derivation of MeanLuminanceAgc > we can remove the bespoke functions from the IPA's class. > > Signed-off-by: Daniel Scally <dan.scally@ideasonboard.com> > --- > src/ipa/rkisp1/algorithms/agc.cpp | 222 ------------------------------ > src/ipa/rkisp1/algorithms/agc.h | 9 -- > 2 files changed, 231 deletions(-) > > diff --git a/src/ipa/rkisp1/algorithms/agc.cpp b/src/ipa/rkisp1/algorithms/agc.cpp > index 3389c471..5e6a8ba0 100644 > --- a/src/ipa/rkisp1/algorithms/agc.cpp > +++ b/src/ipa/rkisp1/algorithms/agc.cpp > @@ -42,24 +42,7 @@ static constexpr double kMinAnalogueGain = 1.0; > /* \todo Honour the FrameDurationLimits control instead of hardcoding a limit */ > static constexpr utils::Duration kMaxShutterSpeed = 60ms; > > -/* Number of frames to wait before calculating stats on minimum exposure */ > -static constexpr uint32_t kNumStartupFrames = 10; > - > -/* Target value to reach for the top 2% of the histogram */ > -static constexpr double kEvGainTarget = 0.5; > - > -/* > - * Relative luminance target. > - * > - * It's a number that's chosen so that, when the camera points at a grey > - * target, the resulting image brightness is considered right. > - * > - * \todo Why is the value different between IPU3 and RkISP1 ? > - */ > -static constexpr double kRelativeLuminanceTarget = 0.4; > - > Agc::Agc() > - : frameCount_(0), filteredExposure_(0s) > { > supportsRaw_ = true; > } > @@ -127,12 +110,6 @@ int Agc::configure(IPAContext &context, const IPACameraSensorInfo &configInfo) > context.configuration.agc.measureWindow.h_size = 3 * configInfo.outputSize.width / 4; > context.configuration.agc.measureWindow.v_size = 3 * configInfo.outputSize.height / 4; > > - /* > - * \todo Use the upcoming per-frame context API that will provide a > - * frame index > - */ > - frameCount_ = 0; > - > for (auto &[id, helper] : exposureModeHelpers()) { > /* \todo Run this again when FrameDurationLimits is passed in */ > helper->configure(context.configuration.sensor.minShutterSpeed, > @@ -234,170 +211,6 @@ void Agc::prepare(IPAContext &context, const uint32_t frame, > params->module_en_update |= RKISP1_CIF_ISP_MODULE_HST; > } > > -/** > - * \brief Apply a filter on the exposure value to limit the speed of changes > - * \param[in] exposureValue The target exposure from the AGC algorithm > - * > - * The speed of the filter is adaptive, and will produce the target quicker > - * during startup, or when the target exposure is within 20% of the most recent > - * filter output. > - * > - * \return The filtered exposure > - */ > -utils::Duration Agc::filterExposure(utils::Duration exposureValue) > -{ > - double speed = 0.2; > - > - /* Adapt instantly if we are in startup phase. */ > - if (frameCount_ < kNumStartupFrames) > - speed = 1.0; > - > - /* > - * If we are close to the desired result, go faster to avoid making > - * multiple micro-adjustments. > - * \todo Make this customisable? > - */ > - if (filteredExposure_ < 1.2 * exposureValue && > - filteredExposure_ > 0.8 * exposureValue) > - speed = sqrt(speed); > - > - filteredExposure_ = speed * exposureValue + > - filteredExposure_ * (1.0 - speed); > - > - LOG(RkISP1Agc, Debug) << "After filtering, exposure " << filteredExposure_; > - > - return filteredExposure_; > -} > - > -/** > - * \brief Estimate the new exposure and gain values > - * \param[inout] context The shared IPA Context > - * \param[in] frameContext The FrameContext for this frame > - * \param[in] yGain The gain calculated on the current brightness level > - * \param[in] iqMeanGain The gain calculated based on the relative luminance target > - */ > -void Agc::computeExposure(IPAContext &context, IPAFrameContext &frameContext, > - double yGain, double iqMeanGain) > -{ > - IPASessionConfiguration &configuration = context.configuration; > - > - /* Get the effective exposure and gain applied on the sensor. */ > - uint32_t exposure = frameContext.sensor.exposure; > - double analogueGain = frameContext.sensor.gain; > - > - /* Use the highest of the two gain estimates. */ > - double evGain = std::max(yGain, iqMeanGain); > - > - utils::Duration minShutterSpeed = configuration.sensor.minShutterSpeed; > - utils::Duration maxShutterSpeed = std::min(configuration.sensor.maxShutterSpeed, > - kMaxShutterSpeed); > - > - double minAnalogueGain = std::max(configuration.sensor.minAnalogueGain, > - kMinAnalogueGain); > - double maxAnalogueGain = configuration.sensor.maxAnalogueGain; > - > - /* Consider within 1% of the target as correctly exposed. */ > - if (utils::abs_diff(evGain, 1.0) < 0.01) > - return; > - > - /* extracted from Rpi::Agc::computeTargetExposure. */ > - > - /* Calculate the shutter time in seconds. */ > - utils::Duration currentShutter = exposure * configuration.sensor.lineDuration; > - > - /* > - * Update the exposure value for the next computation using the values > - * of exposure and gain really used by the sensor. > - */ > - utils::Duration effectiveExposureValue = currentShutter * analogueGain; > - > - LOG(RkISP1Agc, Debug) << "Actual total exposure " << currentShutter * analogueGain > - << " Shutter speed " << currentShutter > - << " Gain " << analogueGain > - << " Needed ev gain " << evGain; > - > - /* > - * Calculate the current exposure value for the scene as the latest > - * exposure value applied multiplied by the new estimated gain. > - */ > - utils::Duration exposureValue = effectiveExposureValue * evGain; > - > - /* Clamp the exposure value to the min and max authorized. */ > - utils::Duration maxTotalExposure = maxShutterSpeed * maxAnalogueGain; > - exposureValue = std::min(exposureValue, maxTotalExposure); > - LOG(RkISP1Agc, Debug) << "Target total exposure " << exposureValue > - << ", maximum is " << maxTotalExposure; > - > - /* > - * Divide the exposure value as new exposure and gain values. > - * \todo estimate if we need to desaturate > - */ > - exposureValue = filterExposure(exposureValue); > - > - /* > - * Push the shutter time up to the maximum first, and only then > - * increase the gain. > - */ > - utils::Duration shutterTime = std::clamp<utils::Duration>(exposureValue / minAnalogueGain, > - minShutterSpeed, maxShutterSpeed); > - double stepGain = std::clamp(exposureValue / shutterTime, > - minAnalogueGain, maxAnalogueGain); > - LOG(RkISP1Agc, Debug) << "Divided up shutter and gain are " > - << shutterTime << " and " > - << stepGain; > -} > - > -/** > - * \brief Estimate the relative luminance of the frame with a given gain > - * \param[in] expMeans The mean luminance values, from the RkISP1 statistics > - * \param[in] gain The gain to apply to the frame > - * > - * This function estimates the average relative luminance of the frame that > - * would be output by the sensor if an additional \a gain was applied. > - * > - * The estimation is based on the AE statistics for the current frame. Y > - * averages for all cells are first multiplied by the gain, and then saturated > - * to approximate the sensor behaviour at high brightness values. The > - * approximation is quite rough, as it doesn't take into account non-linearities > - * when approaching saturation. In this case, saturating after the conversion to > - * YUV doesn't take into account the fact that the R, G and B components > - * contribute differently to the relative luminance. > - * > - * \todo Have a dedicated YUV algorithm ? > - * > - * The values are normalized to the [0.0, 1.0] range, where 1.0 corresponds to a > - * theoretical perfect reflector of 100% reference white. > - * > - * More detailed information can be found in: > - * https://en.wikipedia.org/wiki/Relative_luminance > - * > - * \return The relative luminance > - */ > -double Agc::estimateLuminance(Span<const uint8_t> expMeans, double gain) > -{ > - double ySum = 0.0; > - > - /* Sum the averages, saturated to 255. */ > - for (uint8_t expMean : expMeans) > - ySum += std::min(expMean * gain, 255.0); > - > - /* \todo Weight with the AWB gains */ > - > - return ySum / expMeans.size() / 255; > -} > - > -/** > - * \brief Estimate the mean value of the top 2% of the histogram > - * \param[in] hist The histogram statistics computed by the RkISP1 > - * \return The mean value of the top 2% of the histogram > - */ > -double Agc::measureBrightness(Span<const uint32_t> hist) const > -{ > - Histogram histogram{ hist }; > - /* Estimate the quantile mean of the top 2% of the histogram. */ > - return histogram.interQuantileMean(0.98, 1.0); > -} > - > void Agc::fillMetadata(IPAContext &context, IPAFrameContext &frameContext, > ControlList &metadata) > { > @@ -465,43 +278,8 @@ void Agc::process(IPAContext &context, [[maybe_unused]] const uint32_t frame, > * we receive), but is important in manual mode. > */ > > - const rkisp1_cif_isp_stat *params = &stats->params; > ASSERT(stats->meas_type & RKISP1_CIF_ISP_STAT_AUTOEXP); > > - Span<const uint8_t> ae{ params->ae.exp_mean, context.hw->numAeCells }; > - Span<const uint32_t> hist{ > - params->hist.hist_bins, > - context.hw->numHistogramBins > - }; > - > - double iqMean = measureBrightness(hist); > - double iqMeanGain = kEvGainTarget * hist.size() / iqMean; > - > - /* > - * Estimate the gain needed to achieve a relative luminance target. To > - * account for non-linearity caused by saturation, the value needs to be > - * estimated in an iterative process, as multiplying by a gain will not > - * increase the relative luminance by the same factor if some image > - * regions are saturated. > - */ > - double yGain = 1.0; > - double yTarget = kRelativeLuminanceTarget; > - > - for (unsigned int i = 0; i < 8; i++) { > - double yValue = estimateLuminance(ae, yGain); > - double extra_gain = std::min(10.0, yTarget / (yValue + .001)); > - > - yGain *= extra_gain; > - LOG(RkISP1Agc, Debug) << "Y value: " << yValue > - << ", Y target: " << yTarget > - << ", gives gain " << yGain; > - if (extra_gain < 1.01) > - break; > - } > - > - computeExposure(context, frameContext, yGain, iqMeanGain); > - frameCount_++; > - > parseStatistics(stats, context); > > /* > diff --git a/src/ipa/rkisp1/algorithms/agc.h b/src/ipa/rkisp1/algorithms/agc.h > index 1271741e..311d4e94 100644 > --- a/src/ipa/rkisp1/algorithms/agc.h > +++ b/src/ipa/rkisp1/algorithms/agc.h > @@ -44,21 +44,12 @@ public: > ControlList &metadata) override; > > private: > - void computeExposure(IPAContext &Context, IPAFrameContext &frameContext, > - double yGain, double iqMeanGain); > - utils::Duration filterExposure(utils::Duration exposureValue); > - double estimateLuminance(Span<const uint8_t> expMeans, double gain); > - double measureBrightness(Span<const uint32_t> hist) const; > void fillMetadata(IPAContext &context, IPAFrameContext &frameContext, > ControlList &metadata); > void parseStatistics(const rkisp1_stat_buffer *stats, > IPAContext &context); > double estimateLuminance(double gain) override; > > - uint64_t frameCount_; > - > - utils::Duration filteredExposure_; > - > Histogram hist_; > Span<const uint8_t> expMeans_; > }; > -- > 2.34.1 >
Hi Stefan On 27/03/2024 16:22, Stefan Klug wrote: > Hi Daniel, > > thanks for the patch. > > I didn't test it, but that will get cought by CI. Does every commit > build? I just noticed, that the filterExposure missed an overwrite in > the previous patch. Every commit builds yes. You mean the bespoke filterExposure() is still present? That's true but it shouldn't matter for the base class to use the function, should it? > > Otherwise looks good to me. > > Reviewed-by: Stefan Klug <stefan.klug@ideasonboard.com> Thanks! Dan > > Cheers, > Stefan > > On Fri, Mar 22, 2024 at 01:14:51PM +0000, Daniel Scally wrote: >> Now that the rkisp1 Agc algorithm is a derivation of MeanLuminanceAgc >> we can remove the bespoke functions from the IPA's class. >> >> Signed-off-by: Daniel Scally <dan.scally@ideasonboard.com> >> --- >> src/ipa/rkisp1/algorithms/agc.cpp | 222 ------------------------------ >> src/ipa/rkisp1/algorithms/agc.h | 9 -- >> 2 files changed, 231 deletions(-) >> >> diff --git a/src/ipa/rkisp1/algorithms/agc.cpp b/src/ipa/rkisp1/algorithms/agc.cpp >> index 3389c471..5e6a8ba0 100644 >> --- a/src/ipa/rkisp1/algorithms/agc.cpp >> +++ b/src/ipa/rkisp1/algorithms/agc.cpp >> @@ -42,24 +42,7 @@ static constexpr double kMinAnalogueGain = 1.0; >> /* \todo Honour the FrameDurationLimits control instead of hardcoding a limit */ >> static constexpr utils::Duration kMaxShutterSpeed = 60ms; >> >> -/* Number of frames to wait before calculating stats on minimum exposure */ >> -static constexpr uint32_t kNumStartupFrames = 10; >> - >> -/* Target value to reach for the top 2% of the histogram */ >> -static constexpr double kEvGainTarget = 0.5; >> - >> -/* >> - * Relative luminance target. >> - * >> - * It's a number that's chosen so that, when the camera points at a grey >> - * target, the resulting image brightness is considered right. >> - * >> - * \todo Why is the value different between IPU3 and RkISP1 ? >> - */ >> -static constexpr double kRelativeLuminanceTarget = 0.4; >> - >> Agc::Agc() >> - : frameCount_(0), filteredExposure_(0s) >> { >> supportsRaw_ = true; >> } >> @@ -127,12 +110,6 @@ int Agc::configure(IPAContext &context, const IPACameraSensorInfo &configInfo) >> context.configuration.agc.measureWindow.h_size = 3 * configInfo.outputSize.width / 4; >> context.configuration.agc.measureWindow.v_size = 3 * configInfo.outputSize.height / 4; >> >> - /* >> - * \todo Use the upcoming per-frame context API that will provide a >> - * frame index >> - */ >> - frameCount_ = 0; >> - >> for (auto &[id, helper] : exposureModeHelpers()) { >> /* \todo Run this again when FrameDurationLimits is passed in */ >> helper->configure(context.configuration.sensor.minShutterSpeed, >> @@ -234,170 +211,6 @@ void Agc::prepare(IPAContext &context, const uint32_t frame, >> params->module_en_update |= RKISP1_CIF_ISP_MODULE_HST; >> } >> >> -/** >> - * \brief Apply a filter on the exposure value to limit the speed of changes >> - * \param[in] exposureValue The target exposure from the AGC algorithm >> - * >> - * The speed of the filter is adaptive, and will produce the target quicker >> - * during startup, or when the target exposure is within 20% of the most recent >> - * filter output. >> - * >> - * \return The filtered exposure >> - */ >> -utils::Duration Agc::filterExposure(utils::Duration exposureValue) >> -{ >> - double speed = 0.2; >> - >> - /* Adapt instantly if we are in startup phase. */ >> - if (frameCount_ < kNumStartupFrames) >> - speed = 1.0; >> - >> - /* >> - * If we are close to the desired result, go faster to avoid making >> - * multiple micro-adjustments. >> - * \todo Make this customisable? >> - */ >> - if (filteredExposure_ < 1.2 * exposureValue && >> - filteredExposure_ > 0.8 * exposureValue) >> - speed = sqrt(speed); >> - >> - filteredExposure_ = speed * exposureValue + >> - filteredExposure_ * (1.0 - speed); >> - >> - LOG(RkISP1Agc, Debug) << "After filtering, exposure " << filteredExposure_; >> - >> - return filteredExposure_; >> -} >> - >> -/** >> - * \brief Estimate the new exposure and gain values >> - * \param[inout] context The shared IPA Context >> - * \param[in] frameContext The FrameContext for this frame >> - * \param[in] yGain The gain calculated on the current brightness level >> - * \param[in] iqMeanGain The gain calculated based on the relative luminance target >> - */ >> -void Agc::computeExposure(IPAContext &context, IPAFrameContext &frameContext, >> - double yGain, double iqMeanGain) >> -{ >> - IPASessionConfiguration &configuration = context.configuration; >> - >> - /* Get the effective exposure and gain applied on the sensor. */ >> - uint32_t exposure = frameContext.sensor.exposure; >> - double analogueGain = frameContext.sensor.gain; >> - >> - /* Use the highest of the two gain estimates. */ >> - double evGain = std::max(yGain, iqMeanGain); >> - >> - utils::Duration minShutterSpeed = configuration.sensor.minShutterSpeed; >> - utils::Duration maxShutterSpeed = std::min(configuration.sensor.maxShutterSpeed, >> - kMaxShutterSpeed); >> - >> - double minAnalogueGain = std::max(configuration.sensor.minAnalogueGain, >> - kMinAnalogueGain); >> - double maxAnalogueGain = configuration.sensor.maxAnalogueGain; >> - >> - /* Consider within 1% of the target as correctly exposed. */ >> - if (utils::abs_diff(evGain, 1.0) < 0.01) >> - return; >> - >> - /* extracted from Rpi::Agc::computeTargetExposure. */ >> - >> - /* Calculate the shutter time in seconds. */ >> - utils::Duration currentShutter = exposure * configuration.sensor.lineDuration; >> - >> - /* >> - * Update the exposure value for the next computation using the values >> - * of exposure and gain really used by the sensor. >> - */ >> - utils::Duration effectiveExposureValue = currentShutter * analogueGain; >> - >> - LOG(RkISP1Agc, Debug) << "Actual total exposure " << currentShutter * analogueGain >> - << " Shutter speed " << currentShutter >> - << " Gain " << analogueGain >> - << " Needed ev gain " << evGain; >> - >> - /* >> - * Calculate the current exposure value for the scene as the latest >> - * exposure value applied multiplied by the new estimated gain. >> - */ >> - utils::Duration exposureValue = effectiveExposureValue * evGain; >> - >> - /* Clamp the exposure value to the min and max authorized. */ >> - utils::Duration maxTotalExposure = maxShutterSpeed * maxAnalogueGain; >> - exposureValue = std::min(exposureValue, maxTotalExposure); >> - LOG(RkISP1Agc, Debug) << "Target total exposure " << exposureValue >> - << ", maximum is " << maxTotalExposure; >> - >> - /* >> - * Divide the exposure value as new exposure and gain values. >> - * \todo estimate if we need to desaturate >> - */ >> - exposureValue = filterExposure(exposureValue); >> - >> - /* >> - * Push the shutter time up to the maximum first, and only then >> - * increase the gain. >> - */ >> - utils::Duration shutterTime = std::clamp<utils::Duration>(exposureValue / minAnalogueGain, >> - minShutterSpeed, maxShutterSpeed); >> - double stepGain = std::clamp(exposureValue / shutterTime, >> - minAnalogueGain, maxAnalogueGain); >> - LOG(RkISP1Agc, Debug) << "Divided up shutter and gain are " >> - << shutterTime << " and " >> - << stepGain; >> -} >> - >> -/** >> - * \brief Estimate the relative luminance of the frame with a given gain >> - * \param[in] expMeans The mean luminance values, from the RkISP1 statistics >> - * \param[in] gain The gain to apply to the frame >> - * >> - * This function estimates the average relative luminance of the frame that >> - * would be output by the sensor if an additional \a gain was applied. >> - * >> - * The estimation is based on the AE statistics for the current frame. Y >> - * averages for all cells are first multiplied by the gain, and then saturated >> - * to approximate the sensor behaviour at high brightness values. The >> - * approximation is quite rough, as it doesn't take into account non-linearities >> - * when approaching saturation. In this case, saturating after the conversion to >> - * YUV doesn't take into account the fact that the R, G and B components >> - * contribute differently to the relative luminance. >> - * >> - * \todo Have a dedicated YUV algorithm ? >> - * >> - * The values are normalized to the [0.0, 1.0] range, where 1.0 corresponds to a >> - * theoretical perfect reflector of 100% reference white. >> - * >> - * More detailed information can be found in: >> - * https://en.wikipedia.org/wiki/Relative_luminance >> - * >> - * \return The relative luminance >> - */ >> -double Agc::estimateLuminance(Span<const uint8_t> expMeans, double gain) >> -{ >> - double ySum = 0.0; >> - >> - /* Sum the averages, saturated to 255. */ >> - for (uint8_t expMean : expMeans) >> - ySum += std::min(expMean * gain, 255.0); >> - >> - /* \todo Weight with the AWB gains */ >> - >> - return ySum / expMeans.size() / 255; >> -} >> - >> -/** >> - * \brief Estimate the mean value of the top 2% of the histogram >> - * \param[in] hist The histogram statistics computed by the RkISP1 >> - * \return The mean value of the top 2% of the histogram >> - */ >> -double Agc::measureBrightness(Span<const uint32_t> hist) const >> -{ >> - Histogram histogram{ hist }; >> - /* Estimate the quantile mean of the top 2% of the histogram. */ >> - return histogram.interQuantileMean(0.98, 1.0); >> -} >> - >> void Agc::fillMetadata(IPAContext &context, IPAFrameContext &frameContext, >> ControlList &metadata) >> { >> @@ -465,43 +278,8 @@ void Agc::process(IPAContext &context, [[maybe_unused]] const uint32_t frame, >> * we receive), but is important in manual mode. >> */ >> >> - const rkisp1_cif_isp_stat *params = &stats->params; >> ASSERT(stats->meas_type & RKISP1_CIF_ISP_STAT_AUTOEXP); >> >> - Span<const uint8_t> ae{ params->ae.exp_mean, context.hw->numAeCells }; >> - Span<const uint32_t> hist{ >> - params->hist.hist_bins, >> - context.hw->numHistogramBins >> - }; >> - >> - double iqMean = measureBrightness(hist); >> - double iqMeanGain = kEvGainTarget * hist.size() / iqMean; >> - >> - /* >> - * Estimate the gain needed to achieve a relative luminance target. To >> - * account for non-linearity caused by saturation, the value needs to be >> - * estimated in an iterative process, as multiplying by a gain will not >> - * increase the relative luminance by the same factor if some image >> - * regions are saturated. >> - */ >> - double yGain = 1.0; >> - double yTarget = kRelativeLuminanceTarget; >> - >> - for (unsigned int i = 0; i < 8; i++) { >> - double yValue = estimateLuminance(ae, yGain); >> - double extra_gain = std::min(10.0, yTarget / (yValue + .001)); >> - >> - yGain *= extra_gain; >> - LOG(RkISP1Agc, Debug) << "Y value: " << yValue >> - << ", Y target: " << yTarget >> - << ", gives gain " << yGain; >> - if (extra_gain < 1.01) >> - break; >> - } >> - >> - computeExposure(context, frameContext, yGain, iqMeanGain); >> - frameCount_++; >> - >> parseStatistics(stats, context); >> >> /* >> diff --git a/src/ipa/rkisp1/algorithms/agc.h b/src/ipa/rkisp1/algorithms/agc.h >> index 1271741e..311d4e94 100644 >> --- a/src/ipa/rkisp1/algorithms/agc.h >> +++ b/src/ipa/rkisp1/algorithms/agc.h >> @@ -44,21 +44,12 @@ public: >> ControlList &metadata) override; >> >> private: >> - void computeExposure(IPAContext &Context, IPAFrameContext &frameContext, >> - double yGain, double iqMeanGain); >> - utils::Duration filterExposure(utils::Duration exposureValue); >> - double estimateLuminance(Span<const uint8_t> expMeans, double gain); >> - double measureBrightness(Span<const uint32_t> hist) const; >> void fillMetadata(IPAContext &context, IPAFrameContext &frameContext, >> ControlList &metadata); >> void parseStatistics(const rkisp1_stat_buffer *stats, >> IPAContext &context); >> double estimateLuminance(double gain) override; >> >> - uint64_t frameCount_; >> - >> - utils::Duration filteredExposure_; >> - >> Histogram hist_; >> Span<const uint8_t> expMeans_; >> }; >> -- >> 2.34.1 >>
On Fri, Mar 22, 2024 at 01:14:51PM +0000, Daniel Scally wrote: > Now that the rkisp1 Agc algorithm is a derivation of MeanLuminanceAgc > we can remove the bespoke functions from the IPA's class. > > Signed-off-by: Daniel Scally <dan.scally@ideasonboard.com> My comments are in a patch! ("fixup: ipa: rkisp1: Remove bespoke Agc functions" [1]) [1] https://patchwork.libcamera.org/patch/19853/ Thanks, Paul > --- > src/ipa/rkisp1/algorithms/agc.cpp | 222 ------------------------------ > src/ipa/rkisp1/algorithms/agc.h | 9 -- > 2 files changed, 231 deletions(-) > > diff --git a/src/ipa/rkisp1/algorithms/agc.cpp b/src/ipa/rkisp1/algorithms/agc.cpp > index 3389c471..5e6a8ba0 100644 > --- a/src/ipa/rkisp1/algorithms/agc.cpp > +++ b/src/ipa/rkisp1/algorithms/agc.cpp > @@ -42,24 +42,7 @@ static constexpr double kMinAnalogueGain = 1.0; > /* \todo Honour the FrameDurationLimits control instead of hardcoding a limit */ > static constexpr utils::Duration kMaxShutterSpeed = 60ms; > > -/* Number of frames to wait before calculating stats on minimum exposure */ > -static constexpr uint32_t kNumStartupFrames = 10; > - > -/* Target value to reach for the top 2% of the histogram */ > -static constexpr double kEvGainTarget = 0.5; > - > -/* > - * Relative luminance target. > - * > - * It's a number that's chosen so that, when the camera points at a grey > - * target, the resulting image brightness is considered right. > - * > - * \todo Why is the value different between IPU3 and RkISP1 ? > - */ > -static constexpr double kRelativeLuminanceTarget = 0.4; > - > Agc::Agc() > - : frameCount_(0), filteredExposure_(0s) > { > supportsRaw_ = true; > } > @@ -127,12 +110,6 @@ int Agc::configure(IPAContext &context, const IPACameraSensorInfo &configInfo) > context.configuration.agc.measureWindow.h_size = 3 * configInfo.outputSize.width / 4; > context.configuration.agc.measureWindow.v_size = 3 * configInfo.outputSize.height / 4; > > - /* > - * \todo Use the upcoming per-frame context API that will provide a > - * frame index > - */ > - frameCount_ = 0; > - > for (auto &[id, helper] : exposureModeHelpers()) { > /* \todo Run this again when FrameDurationLimits is passed in */ > helper->configure(context.configuration.sensor.minShutterSpeed, > @@ -234,170 +211,6 @@ void Agc::prepare(IPAContext &context, const uint32_t frame, > params->module_en_update |= RKISP1_CIF_ISP_MODULE_HST; > } > > -/** > - * \brief Apply a filter on the exposure value to limit the speed of changes > - * \param[in] exposureValue The target exposure from the AGC algorithm > - * > - * The speed of the filter is adaptive, and will produce the target quicker > - * during startup, or when the target exposure is within 20% of the most recent > - * filter output. > - * > - * \return The filtered exposure > - */ > -utils::Duration Agc::filterExposure(utils::Duration exposureValue) > -{ > - double speed = 0.2; > - > - /* Adapt instantly if we are in startup phase. */ > - if (frameCount_ < kNumStartupFrames) > - speed = 1.0; > - > - /* > - * If we are close to the desired result, go faster to avoid making > - * multiple micro-adjustments. > - * \todo Make this customisable? > - */ > - if (filteredExposure_ < 1.2 * exposureValue && > - filteredExposure_ > 0.8 * exposureValue) > - speed = sqrt(speed); > - > - filteredExposure_ = speed * exposureValue + > - filteredExposure_ * (1.0 - speed); > - > - LOG(RkISP1Agc, Debug) << "After filtering, exposure " << filteredExposure_; > - > - return filteredExposure_; > -} > - > -/** > - * \brief Estimate the new exposure and gain values > - * \param[inout] context The shared IPA Context > - * \param[in] frameContext The FrameContext for this frame > - * \param[in] yGain The gain calculated on the current brightness level > - * \param[in] iqMeanGain The gain calculated based on the relative luminance target > - */ > -void Agc::computeExposure(IPAContext &context, IPAFrameContext &frameContext, > - double yGain, double iqMeanGain) > -{ > - IPASessionConfiguration &configuration = context.configuration; > - > - /* Get the effective exposure and gain applied on the sensor. */ > - uint32_t exposure = frameContext.sensor.exposure; > - double analogueGain = frameContext.sensor.gain; > - > - /* Use the highest of the two gain estimates. */ > - double evGain = std::max(yGain, iqMeanGain); > - > - utils::Duration minShutterSpeed = configuration.sensor.minShutterSpeed; > - utils::Duration maxShutterSpeed = std::min(configuration.sensor.maxShutterSpeed, > - kMaxShutterSpeed); > - > - double minAnalogueGain = std::max(configuration.sensor.minAnalogueGain, > - kMinAnalogueGain); > - double maxAnalogueGain = configuration.sensor.maxAnalogueGain; > - > - /* Consider within 1% of the target as correctly exposed. */ > - if (utils::abs_diff(evGain, 1.0) < 0.01) > - return; > - > - /* extracted from Rpi::Agc::computeTargetExposure. */ > - > - /* Calculate the shutter time in seconds. */ > - utils::Duration currentShutter = exposure * configuration.sensor.lineDuration; > - > - /* > - * Update the exposure value for the next computation using the values > - * of exposure and gain really used by the sensor. > - */ > - utils::Duration effectiveExposureValue = currentShutter * analogueGain; > - > - LOG(RkISP1Agc, Debug) << "Actual total exposure " << currentShutter * analogueGain > - << " Shutter speed " << currentShutter > - << " Gain " << analogueGain > - << " Needed ev gain " << evGain; > - > - /* > - * Calculate the current exposure value for the scene as the latest > - * exposure value applied multiplied by the new estimated gain. > - */ > - utils::Duration exposureValue = effectiveExposureValue * evGain; > - > - /* Clamp the exposure value to the min and max authorized. */ > - utils::Duration maxTotalExposure = maxShutterSpeed * maxAnalogueGain; > - exposureValue = std::min(exposureValue, maxTotalExposure); > - LOG(RkISP1Agc, Debug) << "Target total exposure " << exposureValue > - << ", maximum is " << maxTotalExposure; > - > - /* > - * Divide the exposure value as new exposure and gain values. > - * \todo estimate if we need to desaturate > - */ > - exposureValue = filterExposure(exposureValue); > - > - /* > - * Push the shutter time up to the maximum first, and only then > - * increase the gain. > - */ > - utils::Duration shutterTime = std::clamp<utils::Duration>(exposureValue / minAnalogueGain, > - minShutterSpeed, maxShutterSpeed); > - double stepGain = std::clamp(exposureValue / shutterTime, > - minAnalogueGain, maxAnalogueGain); > - LOG(RkISP1Agc, Debug) << "Divided up shutter and gain are " > - << shutterTime << " and " > - << stepGain; > -} > - > -/** > - * \brief Estimate the relative luminance of the frame with a given gain > - * \param[in] expMeans The mean luminance values, from the RkISP1 statistics > - * \param[in] gain The gain to apply to the frame > - * > - * This function estimates the average relative luminance of the frame that > - * would be output by the sensor if an additional \a gain was applied. > - * > - * The estimation is based on the AE statistics for the current frame. Y > - * averages for all cells are first multiplied by the gain, and then saturated > - * to approximate the sensor behaviour at high brightness values. The > - * approximation is quite rough, as it doesn't take into account non-linearities > - * when approaching saturation. In this case, saturating after the conversion to > - * YUV doesn't take into account the fact that the R, G and B components > - * contribute differently to the relative luminance. > - * > - * \todo Have a dedicated YUV algorithm ? > - * > - * The values are normalized to the [0.0, 1.0] range, where 1.0 corresponds to a > - * theoretical perfect reflector of 100% reference white. > - * > - * More detailed information can be found in: > - * https://en.wikipedia.org/wiki/Relative_luminance > - * > - * \return The relative luminance > - */ > -double Agc::estimateLuminance(Span<const uint8_t> expMeans, double gain) > -{ > - double ySum = 0.0; > - > - /* Sum the averages, saturated to 255. */ > - for (uint8_t expMean : expMeans) > - ySum += std::min(expMean * gain, 255.0); > - > - /* \todo Weight with the AWB gains */ > - > - return ySum / expMeans.size() / 255; > -} > - > -/** > - * \brief Estimate the mean value of the top 2% of the histogram > - * \param[in] hist The histogram statistics computed by the RkISP1 > - * \return The mean value of the top 2% of the histogram > - */ > -double Agc::measureBrightness(Span<const uint32_t> hist) const > -{ > - Histogram histogram{ hist }; > - /* Estimate the quantile mean of the top 2% of the histogram. */ > - return histogram.interQuantileMean(0.98, 1.0); > -} > - > void Agc::fillMetadata(IPAContext &context, IPAFrameContext &frameContext, > ControlList &metadata) > { > @@ -465,43 +278,8 @@ void Agc::process(IPAContext &context, [[maybe_unused]] const uint32_t frame, > * we receive), but is important in manual mode. > */ > > - const rkisp1_cif_isp_stat *params = &stats->params; > ASSERT(stats->meas_type & RKISP1_CIF_ISP_STAT_AUTOEXP); > > - Span<const uint8_t> ae{ params->ae.exp_mean, context.hw->numAeCells }; > - Span<const uint32_t> hist{ > - params->hist.hist_bins, > - context.hw->numHistogramBins > - }; > - > - double iqMean = measureBrightness(hist); > - double iqMeanGain = kEvGainTarget * hist.size() / iqMean; > - > - /* > - * Estimate the gain needed to achieve a relative luminance target. To > - * account for non-linearity caused by saturation, the value needs to be > - * estimated in an iterative process, as multiplying by a gain will not > - * increase the relative luminance by the same factor if some image > - * regions are saturated. > - */ > - double yGain = 1.0; > - double yTarget = kRelativeLuminanceTarget; > - > - for (unsigned int i = 0; i < 8; i++) { > - double yValue = estimateLuminance(ae, yGain); > - double extra_gain = std::min(10.0, yTarget / (yValue + .001)); > - > - yGain *= extra_gain; > - LOG(RkISP1Agc, Debug) << "Y value: " << yValue > - << ", Y target: " << yTarget > - << ", gives gain " << yGain; > - if (extra_gain < 1.01) > - break; > - } > - > - computeExposure(context, frameContext, yGain, iqMeanGain); > - frameCount_++; > - > parseStatistics(stats, context); > > /* > diff --git a/src/ipa/rkisp1/algorithms/agc.h b/src/ipa/rkisp1/algorithms/agc.h > index 1271741e..311d4e94 100644 > --- a/src/ipa/rkisp1/algorithms/agc.h > +++ b/src/ipa/rkisp1/algorithms/agc.h > @@ -44,21 +44,12 @@ public: > ControlList &metadata) override; > > private: > - void computeExposure(IPAContext &Context, IPAFrameContext &frameContext, > - double yGain, double iqMeanGain); > - utils::Duration filterExposure(utils::Duration exposureValue); > - double estimateLuminance(Span<const uint8_t> expMeans, double gain); > - double measureBrightness(Span<const uint32_t> hist) const; > void fillMetadata(IPAContext &context, IPAFrameContext &frameContext, > ControlList &metadata); > void parseStatistics(const rkisp1_stat_buffer *stats, > IPAContext &context); > double estimateLuminance(double gain) override; > > - uint64_t frameCount_; > - > - utils::Duration filteredExposure_; > - > Histogram hist_; > Span<const uint8_t> expMeans_; > }; > -- > 2.34.1 >
diff --git a/src/ipa/rkisp1/algorithms/agc.cpp b/src/ipa/rkisp1/algorithms/agc.cpp index 3389c471..5e6a8ba0 100644 --- a/src/ipa/rkisp1/algorithms/agc.cpp +++ b/src/ipa/rkisp1/algorithms/agc.cpp @@ -42,24 +42,7 @@ static constexpr double kMinAnalogueGain = 1.0; /* \todo Honour the FrameDurationLimits control instead of hardcoding a limit */ static constexpr utils::Duration kMaxShutterSpeed = 60ms; -/* Number of frames to wait before calculating stats on minimum exposure */ -static constexpr uint32_t kNumStartupFrames = 10; - -/* Target value to reach for the top 2% of the histogram */ -static constexpr double kEvGainTarget = 0.5; - -/* - * Relative luminance target. - * - * It's a number that's chosen so that, when the camera points at a grey - * target, the resulting image brightness is considered right. - * - * \todo Why is the value different between IPU3 and RkISP1 ? - */ -static constexpr double kRelativeLuminanceTarget = 0.4; - Agc::Agc() - : frameCount_(0), filteredExposure_(0s) { supportsRaw_ = true; } @@ -127,12 +110,6 @@ int Agc::configure(IPAContext &context, const IPACameraSensorInfo &configInfo) context.configuration.agc.measureWindow.h_size = 3 * configInfo.outputSize.width / 4; context.configuration.agc.measureWindow.v_size = 3 * configInfo.outputSize.height / 4; - /* - * \todo Use the upcoming per-frame context API that will provide a - * frame index - */ - frameCount_ = 0; - for (auto &[id, helper] : exposureModeHelpers()) { /* \todo Run this again when FrameDurationLimits is passed in */ helper->configure(context.configuration.sensor.minShutterSpeed, @@ -234,170 +211,6 @@ void Agc::prepare(IPAContext &context, const uint32_t frame, params->module_en_update |= RKISP1_CIF_ISP_MODULE_HST; } -/** - * \brief Apply a filter on the exposure value to limit the speed of changes - * \param[in] exposureValue The target exposure from the AGC algorithm - * - * The speed of the filter is adaptive, and will produce the target quicker - * during startup, or when the target exposure is within 20% of the most recent - * filter output. - * - * \return The filtered exposure - */ -utils::Duration Agc::filterExposure(utils::Duration exposureValue) -{ - double speed = 0.2; - - /* Adapt instantly if we are in startup phase. */ - if (frameCount_ < kNumStartupFrames) - speed = 1.0; - - /* - * If we are close to the desired result, go faster to avoid making - * multiple micro-adjustments. - * \todo Make this customisable? - */ - if (filteredExposure_ < 1.2 * exposureValue && - filteredExposure_ > 0.8 * exposureValue) - speed = sqrt(speed); - - filteredExposure_ = speed * exposureValue + - filteredExposure_ * (1.0 - speed); - - LOG(RkISP1Agc, Debug) << "After filtering, exposure " << filteredExposure_; - - return filteredExposure_; -} - -/** - * \brief Estimate the new exposure and gain values - * \param[inout] context The shared IPA Context - * \param[in] frameContext The FrameContext for this frame - * \param[in] yGain The gain calculated on the current brightness level - * \param[in] iqMeanGain The gain calculated based on the relative luminance target - */ -void Agc::computeExposure(IPAContext &context, IPAFrameContext &frameContext, - double yGain, double iqMeanGain) -{ - IPASessionConfiguration &configuration = context.configuration; - - /* Get the effective exposure and gain applied on the sensor. */ - uint32_t exposure = frameContext.sensor.exposure; - double analogueGain = frameContext.sensor.gain; - - /* Use the highest of the two gain estimates. */ - double evGain = std::max(yGain, iqMeanGain); - - utils::Duration minShutterSpeed = configuration.sensor.minShutterSpeed; - utils::Duration maxShutterSpeed = std::min(configuration.sensor.maxShutterSpeed, - kMaxShutterSpeed); - - double minAnalogueGain = std::max(configuration.sensor.minAnalogueGain, - kMinAnalogueGain); - double maxAnalogueGain = configuration.sensor.maxAnalogueGain; - - /* Consider within 1% of the target as correctly exposed. */ - if (utils::abs_diff(evGain, 1.0) < 0.01) - return; - - /* extracted from Rpi::Agc::computeTargetExposure. */ - - /* Calculate the shutter time in seconds. */ - utils::Duration currentShutter = exposure * configuration.sensor.lineDuration; - - /* - * Update the exposure value for the next computation using the values - * of exposure and gain really used by the sensor. - */ - utils::Duration effectiveExposureValue = currentShutter * analogueGain; - - LOG(RkISP1Agc, Debug) << "Actual total exposure " << currentShutter * analogueGain - << " Shutter speed " << currentShutter - << " Gain " << analogueGain - << " Needed ev gain " << evGain; - - /* - * Calculate the current exposure value for the scene as the latest - * exposure value applied multiplied by the new estimated gain. - */ - utils::Duration exposureValue = effectiveExposureValue * evGain; - - /* Clamp the exposure value to the min and max authorized. */ - utils::Duration maxTotalExposure = maxShutterSpeed * maxAnalogueGain; - exposureValue = std::min(exposureValue, maxTotalExposure); - LOG(RkISP1Agc, Debug) << "Target total exposure " << exposureValue - << ", maximum is " << maxTotalExposure; - - /* - * Divide the exposure value as new exposure and gain values. - * \todo estimate if we need to desaturate - */ - exposureValue = filterExposure(exposureValue); - - /* - * Push the shutter time up to the maximum first, and only then - * increase the gain. - */ - utils::Duration shutterTime = std::clamp<utils::Duration>(exposureValue / minAnalogueGain, - minShutterSpeed, maxShutterSpeed); - double stepGain = std::clamp(exposureValue / shutterTime, - minAnalogueGain, maxAnalogueGain); - LOG(RkISP1Agc, Debug) << "Divided up shutter and gain are " - << shutterTime << " and " - << stepGain; -} - -/** - * \brief Estimate the relative luminance of the frame with a given gain - * \param[in] expMeans The mean luminance values, from the RkISP1 statistics - * \param[in] gain The gain to apply to the frame - * - * This function estimates the average relative luminance of the frame that - * would be output by the sensor if an additional \a gain was applied. - * - * The estimation is based on the AE statistics for the current frame. Y - * averages for all cells are first multiplied by the gain, and then saturated - * to approximate the sensor behaviour at high brightness values. The - * approximation is quite rough, as it doesn't take into account non-linearities - * when approaching saturation. In this case, saturating after the conversion to - * YUV doesn't take into account the fact that the R, G and B components - * contribute differently to the relative luminance. - * - * \todo Have a dedicated YUV algorithm ? - * - * The values are normalized to the [0.0, 1.0] range, where 1.0 corresponds to a - * theoretical perfect reflector of 100% reference white. - * - * More detailed information can be found in: - * https://en.wikipedia.org/wiki/Relative_luminance - * - * \return The relative luminance - */ -double Agc::estimateLuminance(Span<const uint8_t> expMeans, double gain) -{ - double ySum = 0.0; - - /* Sum the averages, saturated to 255. */ - for (uint8_t expMean : expMeans) - ySum += std::min(expMean * gain, 255.0); - - /* \todo Weight with the AWB gains */ - - return ySum / expMeans.size() / 255; -} - -/** - * \brief Estimate the mean value of the top 2% of the histogram - * \param[in] hist The histogram statistics computed by the RkISP1 - * \return The mean value of the top 2% of the histogram - */ -double Agc::measureBrightness(Span<const uint32_t> hist) const -{ - Histogram histogram{ hist }; - /* Estimate the quantile mean of the top 2% of the histogram. */ - return histogram.interQuantileMean(0.98, 1.0); -} - void Agc::fillMetadata(IPAContext &context, IPAFrameContext &frameContext, ControlList &metadata) { @@ -465,43 +278,8 @@ void Agc::process(IPAContext &context, [[maybe_unused]] const uint32_t frame, * we receive), but is important in manual mode. */ - const rkisp1_cif_isp_stat *params = &stats->params; ASSERT(stats->meas_type & RKISP1_CIF_ISP_STAT_AUTOEXP); - Span<const uint8_t> ae{ params->ae.exp_mean, context.hw->numAeCells }; - Span<const uint32_t> hist{ - params->hist.hist_bins, - context.hw->numHistogramBins - }; - - double iqMean = measureBrightness(hist); - double iqMeanGain = kEvGainTarget * hist.size() / iqMean; - - /* - * Estimate the gain needed to achieve a relative luminance target. To - * account for non-linearity caused by saturation, the value needs to be - * estimated in an iterative process, as multiplying by a gain will not - * increase the relative luminance by the same factor if some image - * regions are saturated. - */ - double yGain = 1.0; - double yTarget = kRelativeLuminanceTarget; - - for (unsigned int i = 0; i < 8; i++) { - double yValue = estimateLuminance(ae, yGain); - double extra_gain = std::min(10.0, yTarget / (yValue + .001)); - - yGain *= extra_gain; - LOG(RkISP1Agc, Debug) << "Y value: " << yValue - << ", Y target: " << yTarget - << ", gives gain " << yGain; - if (extra_gain < 1.01) - break; - } - - computeExposure(context, frameContext, yGain, iqMeanGain); - frameCount_++; - parseStatistics(stats, context); /* diff --git a/src/ipa/rkisp1/algorithms/agc.h b/src/ipa/rkisp1/algorithms/agc.h index 1271741e..311d4e94 100644 --- a/src/ipa/rkisp1/algorithms/agc.h +++ b/src/ipa/rkisp1/algorithms/agc.h @@ -44,21 +44,12 @@ public: ControlList &metadata) override; private: - void computeExposure(IPAContext &Context, IPAFrameContext &frameContext, - double yGain, double iqMeanGain); - utils::Duration filterExposure(utils::Duration exposureValue); - double estimateLuminance(Span<const uint8_t> expMeans, double gain); - double measureBrightness(Span<const uint32_t> hist) const; void fillMetadata(IPAContext &context, IPAFrameContext &frameContext, ControlList &metadata); void parseStatistics(const rkisp1_stat_buffer *stats, IPAContext &context); double estimateLuminance(double gain) override; - uint64_t frameCount_; - - utils::Duration filteredExposure_; - Histogram hist_; Span<const uint8_t> expMeans_; };
Now that the rkisp1 Agc algorithm is a derivation of MeanLuminanceAgc we can remove the bespoke functions from the IPA's class. Signed-off-by: Daniel Scally <dan.scally@ideasonboard.com> --- src/ipa/rkisp1/algorithms/agc.cpp | 222 ------------------------------ src/ipa/rkisp1/algorithms/agc.h | 9 -- 2 files changed, 231 deletions(-)