Message ID | 20211111140928.136111-8-jeanmichel.hautbois@ideasonboard.com |
---|---|
State | Accepted |
Headers | show |
Series |
|
Related | show |
Hi Jean-Michel, On Thu, Nov 11, 2021 at 03:09:21PM +0100, Jean-Michel Hautbois wrote: > When an image is partially saturated, its brightness is not increasing > linearly when the shutter time or gain increases. It is a big issue with > a backlight as the algorithm is fading to darkness right now. > > Introduce a function to estimate the brightness of the frame, based on > the current exposure/gain and loop on it several times to estimate it > again and approach the non linear function. > > Inspired-by: 7de5506c30b3 ("libcamera: src: ipa: raspberrypi: agc: Improve gain update calculation for partly saturated images") > Signed-off-by: Jean-Michel Hautbois <jeanmichel.hautbois@ideasonboard.com> > Reviewed-by: Kieran Bingham <kieran.bingham@ideasonboard.com> > --- > src/ipa/ipu3/algorithms/agc.cpp | 104 +++++++++++++++++++++++++++++++- > src/ipa/ipu3/algorithms/agc.h | 6 +- > 2 files changed, 106 insertions(+), 4 deletions(-) > > diff --git a/src/ipa/ipu3/algorithms/agc.cpp b/src/ipa/ipu3/algorithms/agc.cpp > index 61ca8b3f..a34f070e 100644 > --- a/src/ipa/ipu3/algorithms/agc.cpp > +++ b/src/ipa/ipu3/algorithms/agc.cpp > @@ -67,6 +67,17 @@ static constexpr uint32_t kMinCellsPerZoneRatio = 255 * 20 / 100; > /* Number of frames to wait before calculating stats on minimum exposure */ > static constexpr uint32_t kNumStartupFrames = 10; > > +/* Maximum luminance used for brightness normalization */ > +static constexpr uint32_t kMaxLuminance = 255; > + > +/* > + * Normalized luma value target. > + * > + * It's a number that's chosen so that, when the camera points at a grey > + * target, the resulting image brightness is considered right. > + */ > +static constexpr double kNormalizedLumaTarget = 0.16; > + > Agc::Agc() > : frameCount_(0), iqMean_(0.0), lineDuration_(0s), minExposureLines_(0), > maxExposureLines_(0), filteredExposure_(0s), currentExposure_(0s), > @@ -185,14 +196,20 @@ void Agc::filterExposure() > /** > * \brief Estimate the new exposure and gain values > * \param[inout] frameContext The shared IPA frame Context > + * \param[in] currentYGain The gain calculated on the current brightness level > */ > -void Agc::computeExposure(IPAFrameContext &frameContext) > +void Agc::computeExposure(IPAFrameContext &frameContext, double currentYGain) > { > /* Get the effective exposure and gain applied on the sensor. */ > uint32_t exposure = frameContext.sensor.exposure; > double analogueGain = frameContext.sensor.gain; > > - /* Estimate the gain needed to have the proportion wanted */ > + /* > + * Estimate the gain needed to have the proportion of pixels in a given > + * range wanted. iqMean_ returns the mean value of the top 2% of the > + * cumulative histogram, and we want it to be as close as possible to a > + * configured target. > + */ > double evGain = kEvGainTarget * knumHistogramBins / iqMean_; > > if (std::abs(evGain - 1.0) < 0.01) { > @@ -202,6 +219,7 @@ void Agc::computeExposure(IPAFrameContext &frameContext) > } > > /* extracted from Rpi::Agc::computeTargetExposure */ > + > /* Calculate the shutter time in seconds */ > utils::Duration currentShutter = exposure * lineDuration_; > LOG(IPU3Agc, Debug) << "Actual total exposure " << currentShutter * analogueGain > @@ -209,6 +227,14 @@ void Agc::computeExposure(IPAFrameContext &frameContext) > << " Gain " << analogueGain > << " Needed ev gain " << evGain; > > + if (evGain < currentYGain) > + evGain = currentYGain; > + > + /* Consider within 1% of the target as correctly exposed */ > + if (std::abs(evGain - 1.0) < 0.01) > + LOG(IPU3Agc, Debug) << "We are well exposed (iqMean = " > + << iqMean_ << ")"; I thought we saw this stanza in the last patch, right before this hunk? > + > /* > * Calculate the current exposure value for the scene as the latest > * exposure value applied multiplied by the new estimated gain. > @@ -257,6 +283,56 @@ void Agc::computeExposure(IPAFrameContext &frameContext) > prevExposureValue_ = shutterTime * analogueGain; > } > > +/** > + * \brief Estimate the average brightness of the frame > + * \param[in] frameContext The shared IPA frame context > + * \param[in] grid The grid used to store the statistics in the IPU3 > + * \param[in] stats The IPU3 statistics and ISP results > + * \param[in] currentYGain The gain calculated on the current brightness level > + * \return The normalized luma > + * > + * Luma is the weighted sum of gamma-compressed R′G′B′ components of a color > + * video. The luma values are normalized as 0.0 to 1.0, with 1.0 being a > + * theoretical perfect reflector of 100% reference white. We use the Rec. 601 > + * luma here. > + * > + * More detailed information can be found in: > + * https://en.wikipedia.org/wiki/Luma_(video) > + */ > +double Agc::computeInitialY(IPAFrameContext &frameContext, > + const ipu3_uapi_grid_config &grid, > + const ipu3_uapi_stats_3a *stats, > + double currentYGain) > +{ > + double redSum = 0, greenSum = 0, blueSum = 0; > + > + for (unsigned int cellY = 0; cellY < grid.height; cellY++) { > + for (unsigned int cellX = 0; cellX < grid.width; cellX++) { > + uint32_t cellPosition = cellY * stride_ + cellX; > + > + const ipu3_uapi_awb_set_item *cell = > + reinterpret_cast<const ipu3_uapi_awb_set_item *>( > + &stats->awb_raw_buffer.meta_data[cellPosition] > + ); > + > + redSum += cell->R_avg * currentYGain; > + greenSum += (cell->Gr_avg + cell->Gb_avg) / 2 * currentYGain; > + blueSum += cell->B_avg * currentYGain; > + } > + } > + > + /* > + * Estimate the sum of the brightness values, weighted with the gains > + * applied on the channels in AWB as the Rec. 601 luma. > + */ > + double Y_sum = redSum * frameContext.awb.gains.red * .299 + > + greenSum * frameContext.awb.gains.green * .587 + > + blueSum * frameContext.awb.gains.blue * .114; > + > + /* Return the normalized relative luminance. */ > + return Y_sum / (grid.height * grid.width) / kMaxLuminance; > +} > + > /** > * \brief Process IPU3 statistics, and run AGC operations > * \param[in] context The shared IPA context > @@ -268,7 +344,29 @@ void Agc::computeExposure(IPAFrameContext &frameContext) > void Agc::process(IPAContext &context, const ipu3_uapi_stats_3a *stats) > { > measureBrightness(stats, context.configuration.grid.bdsGrid); > - computeExposure(context.frameContext); > + > + double currentYGain = 1.0; > + double targetY = kNormalizedLumaTarget; > + > + /* > + * Do this calculation a few times as brightness increase can be > + * non-linear when there are saturated regions. > + */ > + for (int i = 0; i < 8; i++) { > + double initialY = computeInitialY(context.frameContext, > + context.configuration.grid.bdsGrid, > + stats, currentYGain); > + double extra_gain = std::min(10.0, targetY / (initialY + .001)); > + > + currentYGain *= extra_gain; > + LOG(IPU3Agc, Debug) << "Initial Y " << initialY > + << " target " << targetY > + << " gives gain " << currentYGain; > + if (extra_gain < 1.01) > + break; > + } > + > + computeExposure(context.frameContext, currentYGain); > frameCount_++; > } > > diff --git a/src/ipa/ipu3/algorithms/agc.h b/src/ipa/ipu3/algorithms/agc.h > index f0db25ee..79736283 100644 > --- a/src/ipa/ipu3/algorithms/agc.h > +++ b/src/ipa/ipu3/algorithms/agc.h > @@ -34,7 +34,11 @@ private: > void measureBrightness(const ipu3_uapi_stats_3a *stats, > const ipu3_uapi_grid_config &grid); > void filterExposure(); > - void computeExposure(IPAFrameContext &frameContext); > + void computeExposure(IPAFrameContext &frameContext, double currentYGain); > + double computeInitialY(IPAFrameContext &frameContext, > + const ipu3_uapi_grid_config &grid, > + const ipu3_uapi_stats_3a *stats, > + double currentYGain); The rest looks good. Reviewed-by: Paul Elder <paul.elder@ideasonboard.com> > > uint64_t frameCount_; > uint64_t lastFrame_; > -- > 2.32.0 >
Hi Paul, On 13/11/2021 00:12, Paul Elder wrote: > Hi Jean-Michel, > > On Thu, Nov 11, 2021 at 03:09:21PM +0100, Jean-Michel Hautbois wrote: >> When an image is partially saturated, its brightness is not increasing >> linearly when the shutter time or gain increases. It is a big issue with >> a backlight as the algorithm is fading to darkness right now. >> >> Introduce a function to estimate the brightness of the frame, based on >> the current exposure/gain and loop on it several times to estimate it >> again and approach the non linear function. >> >> Inspired-by: 7de5506c30b3 ("libcamera: src: ipa: raspberrypi: agc: Improve gain update calculation for partly saturated images") >> Signed-off-by: Jean-Michel Hautbois <jeanmichel.hautbois@ideasonboard.com> >> Reviewed-by: Kieran Bingham <kieran.bingham@ideasonboard.com> >> --- >> src/ipa/ipu3/algorithms/agc.cpp | 104 +++++++++++++++++++++++++++++++- >> src/ipa/ipu3/algorithms/agc.h | 6 +- >> 2 files changed, 106 insertions(+), 4 deletions(-) >> >> diff --git a/src/ipa/ipu3/algorithms/agc.cpp b/src/ipa/ipu3/algorithms/agc.cpp >> index 61ca8b3f..a34f070e 100644 >> --- a/src/ipa/ipu3/algorithms/agc.cpp >> +++ b/src/ipa/ipu3/algorithms/agc.cpp >> @@ -67,6 +67,17 @@ static constexpr uint32_t kMinCellsPerZoneRatio = 255 * 20 / 100; >> /* Number of frames to wait before calculating stats on minimum exposure */ >> static constexpr uint32_t kNumStartupFrames = 10; >> >> +/* Maximum luminance used for brightness normalization */ >> +static constexpr uint32_t kMaxLuminance = 255; >> + >> +/* >> + * Normalized luma value target. >> + * >> + * It's a number that's chosen so that, when the camera points at a grey >> + * target, the resulting image brightness is considered right. >> + */ >> +static constexpr double kNormalizedLumaTarget = 0.16; >> + >> Agc::Agc() >> : frameCount_(0), iqMean_(0.0), lineDuration_(0s), minExposureLines_(0), >> maxExposureLines_(0), filteredExposure_(0s), currentExposure_(0s), >> @@ -185,14 +196,20 @@ void Agc::filterExposure() >> /** >> * \brief Estimate the new exposure and gain values >> * \param[inout] frameContext The shared IPA frame Context >> + * \param[in] currentYGain The gain calculated on the current brightness level >> */ >> -void Agc::computeExposure(IPAFrameContext &frameContext) >> +void Agc::computeExposure(IPAFrameContext &frameContext, double currentYGain) >> { >> /* Get the effective exposure and gain applied on the sensor. */ >> uint32_t exposure = frameContext.sensor.exposure; >> double analogueGain = frameContext.sensor.gain; >> >> - /* Estimate the gain needed to have the proportion wanted */ >> + /* >> + * Estimate the gain needed to have the proportion of pixels in a given >> + * range wanted. iqMean_ returns the mean value of the top 2% of the >> + * cumulative histogram, and we want it to be as close as possible to a >> + * configured target. >> + */ >> double evGain = kEvGainTarget * knumHistogramBins / iqMean_; >> >> if (std::abs(evGain - 1.0) < 0.01) { >> @@ -202,6 +219,7 @@ void Agc::computeExposure(IPAFrameContext &frameContext) >> } >> >> /* extracted from Rpi::Agc::computeTargetExposure */ >> + >> /* Calculate the shutter time in seconds */ >> utils::Duration currentShutter = exposure * lineDuration_; >> LOG(IPU3Agc, Debug) << "Actual total exposure " << currentShutter * analogueGain >> @@ -209,6 +227,14 @@ void Agc::computeExposure(IPAFrameContext &frameContext) >> << " Gain " << analogueGain >> << " Needed ev gain " << evGain; >> >> + if (evGain < currentYGain) >> + evGain = currentYGain; >> + >> + /* Consider within 1% of the target as correctly exposed */ >> + if (std::abs(evGain - 1.0) < 0.01) >> + LOG(IPU3Agc, Debug) << "We are well exposed (iqMean = " >> + << iqMean_ << ")"; > > I thought we saw this stanza in the last patch, right before this hunk? Good catch ! This is indeed a bug :-). I will send v5, with all the reviewed-by and commit messages changed, with this bug fixed. > >> + >> /* >> * Calculate the current exposure value for the scene as the latest >> * exposure value applied multiplied by the new estimated gain. >> @@ -257,6 +283,56 @@ void Agc::computeExposure(IPAFrameContext &frameContext) >> prevExposureValue_ = shutterTime * analogueGain; >> } >> >> +/** >> + * \brief Estimate the average brightness of the frame >> + * \param[in] frameContext The shared IPA frame context >> + * \param[in] grid The grid used to store the statistics in the IPU3 >> + * \param[in] stats The IPU3 statistics and ISP results >> + * \param[in] currentYGain The gain calculated on the current brightness level >> + * \return The normalized luma >> + * >> + * Luma is the weighted sum of gamma-compressed R′G′B′ components of a color >> + * video. The luma values are normalized as 0.0 to 1.0, with 1.0 being a >> + * theoretical perfect reflector of 100% reference white. We use the Rec. 601 >> + * luma here. >> + * >> + * More detailed information can be found in: >> + * https://en.wikipedia.org/wiki/Luma_(video) >> + */ >> +double Agc::computeInitialY(IPAFrameContext &frameContext, >> + const ipu3_uapi_grid_config &grid, >> + const ipu3_uapi_stats_3a *stats, >> + double currentYGain) >> +{ >> + double redSum = 0, greenSum = 0, blueSum = 0; >> + >> + for (unsigned int cellY = 0; cellY < grid.height; cellY++) { >> + for (unsigned int cellX = 0; cellX < grid.width; cellX++) { >> + uint32_t cellPosition = cellY * stride_ + cellX; >> + >> + const ipu3_uapi_awb_set_item *cell = >> + reinterpret_cast<const ipu3_uapi_awb_set_item *>( >> + &stats->awb_raw_buffer.meta_data[cellPosition] >> + ); >> + >> + redSum += cell->R_avg * currentYGain; >> + greenSum += (cell->Gr_avg + cell->Gb_avg) / 2 * currentYGain; >> + blueSum += cell->B_avg * currentYGain; >> + } >> + } >> + >> + /* >> + * Estimate the sum of the brightness values, weighted with the gains >> + * applied on the channels in AWB as the Rec. 601 luma. >> + */ >> + double Y_sum = redSum * frameContext.awb.gains.red * .299 + >> + greenSum * frameContext.awb.gains.green * .587 + >> + blueSum * frameContext.awb.gains.blue * .114; >> + >> + /* Return the normalized relative luminance. */ >> + return Y_sum / (grid.height * grid.width) / kMaxLuminance; >> +} >> + >> /** >> * \brief Process IPU3 statistics, and run AGC operations >> * \param[in] context The shared IPA context >> @@ -268,7 +344,29 @@ void Agc::computeExposure(IPAFrameContext &frameContext) >> void Agc::process(IPAContext &context, const ipu3_uapi_stats_3a *stats) >> { >> measureBrightness(stats, context.configuration.grid.bdsGrid); >> - computeExposure(context.frameContext); >> + >> + double currentYGain = 1.0; >> + double targetY = kNormalizedLumaTarget; >> + >> + /* >> + * Do this calculation a few times as brightness increase can be >> + * non-linear when there are saturated regions. >> + */ >> + for (int i = 0; i < 8; i++) { >> + double initialY = computeInitialY(context.frameContext, >> + context.configuration.grid.bdsGrid, >> + stats, currentYGain); >> + double extra_gain = std::min(10.0, targetY / (initialY + .001)); >> + >> + currentYGain *= extra_gain; >> + LOG(IPU3Agc, Debug) << "Initial Y " << initialY >> + << " target " << targetY >> + << " gives gain " << currentYGain; >> + if (extra_gain < 1.01) >> + break; >> + } >> + >> + computeExposure(context.frameContext, currentYGain); >> frameCount_++; >> } >> >> diff --git a/src/ipa/ipu3/algorithms/agc.h b/src/ipa/ipu3/algorithms/agc.h >> index f0db25ee..79736283 100644 >> --- a/src/ipa/ipu3/algorithms/agc.h >> +++ b/src/ipa/ipu3/algorithms/agc.h >> @@ -34,7 +34,11 @@ private: >> void measureBrightness(const ipu3_uapi_stats_3a *stats, >> const ipu3_uapi_grid_config &grid); >> void filterExposure(); >> - void computeExposure(IPAFrameContext &frameContext); >> + void computeExposure(IPAFrameContext &frameContext, double currentYGain); >> + double computeInitialY(IPAFrameContext &frameContext, >> + const ipu3_uapi_grid_config &grid, >> + const ipu3_uapi_stats_3a *stats, >> + double currentYGain); > > The rest looks good. > > Reviewed-by: Paul Elder <paul.elder@ideasonboard.com> > >> >> uint64_t frameCount_; >> uint64_t lastFrame_; >> -- >> 2.32.0 >>
diff --git a/src/ipa/ipu3/algorithms/agc.cpp b/src/ipa/ipu3/algorithms/agc.cpp index 61ca8b3f..a34f070e 100644 --- a/src/ipa/ipu3/algorithms/agc.cpp +++ b/src/ipa/ipu3/algorithms/agc.cpp @@ -67,6 +67,17 @@ static constexpr uint32_t kMinCellsPerZoneRatio = 255 * 20 / 100; /* Number of frames to wait before calculating stats on minimum exposure */ static constexpr uint32_t kNumStartupFrames = 10; +/* Maximum luminance used for brightness normalization */ +static constexpr uint32_t kMaxLuminance = 255; + +/* + * Normalized luma value target. + * + * It's a number that's chosen so that, when the camera points at a grey + * target, the resulting image brightness is considered right. + */ +static constexpr double kNormalizedLumaTarget = 0.16; + Agc::Agc() : frameCount_(0), iqMean_(0.0), lineDuration_(0s), minExposureLines_(0), maxExposureLines_(0), filteredExposure_(0s), currentExposure_(0s), @@ -185,14 +196,20 @@ void Agc::filterExposure() /** * \brief Estimate the new exposure and gain values * \param[inout] frameContext The shared IPA frame Context + * \param[in] currentYGain The gain calculated on the current brightness level */ -void Agc::computeExposure(IPAFrameContext &frameContext) +void Agc::computeExposure(IPAFrameContext &frameContext, double currentYGain) { /* Get the effective exposure and gain applied on the sensor. */ uint32_t exposure = frameContext.sensor.exposure; double analogueGain = frameContext.sensor.gain; - /* Estimate the gain needed to have the proportion wanted */ + /* + * Estimate the gain needed to have the proportion of pixels in a given + * range wanted. iqMean_ returns the mean value of the top 2% of the + * cumulative histogram, and we want it to be as close as possible to a + * configured target. + */ double evGain = kEvGainTarget * knumHistogramBins / iqMean_; if (std::abs(evGain - 1.0) < 0.01) { @@ -202,6 +219,7 @@ void Agc::computeExposure(IPAFrameContext &frameContext) } /* extracted from Rpi::Agc::computeTargetExposure */ + /* Calculate the shutter time in seconds */ utils::Duration currentShutter = exposure * lineDuration_; LOG(IPU3Agc, Debug) << "Actual total exposure " << currentShutter * analogueGain @@ -209,6 +227,14 @@ void Agc::computeExposure(IPAFrameContext &frameContext) << " Gain " << analogueGain << " Needed ev gain " << evGain; + if (evGain < currentYGain) + evGain = currentYGain; + + /* Consider within 1% of the target as correctly exposed */ + if (std::abs(evGain - 1.0) < 0.01) + LOG(IPU3Agc, Debug) << "We are well exposed (iqMean = " + << iqMean_ << ")"; + /* * Calculate the current exposure value for the scene as the latest * exposure value applied multiplied by the new estimated gain. @@ -257,6 +283,56 @@ void Agc::computeExposure(IPAFrameContext &frameContext) prevExposureValue_ = shutterTime * analogueGain; } +/** + * \brief Estimate the average brightness of the frame + * \param[in] frameContext The shared IPA frame context + * \param[in] grid The grid used to store the statistics in the IPU3 + * \param[in] stats The IPU3 statistics and ISP results + * \param[in] currentYGain The gain calculated on the current brightness level + * \return The normalized luma + * + * Luma is the weighted sum of gamma-compressed R′G′B′ components of a color + * video. The luma values are normalized as 0.0 to 1.0, with 1.0 being a + * theoretical perfect reflector of 100% reference white. We use the Rec. 601 + * luma here. + * + * More detailed information can be found in: + * https://en.wikipedia.org/wiki/Luma_(video) + */ +double Agc::computeInitialY(IPAFrameContext &frameContext, + const ipu3_uapi_grid_config &grid, + const ipu3_uapi_stats_3a *stats, + double currentYGain) +{ + double redSum = 0, greenSum = 0, blueSum = 0; + + for (unsigned int cellY = 0; cellY < grid.height; cellY++) { + for (unsigned int cellX = 0; cellX < grid.width; cellX++) { + uint32_t cellPosition = cellY * stride_ + cellX; + + const ipu3_uapi_awb_set_item *cell = + reinterpret_cast<const ipu3_uapi_awb_set_item *>( + &stats->awb_raw_buffer.meta_data[cellPosition] + ); + + redSum += cell->R_avg * currentYGain; + greenSum += (cell->Gr_avg + cell->Gb_avg) / 2 * currentYGain; + blueSum += cell->B_avg * currentYGain; + } + } + + /* + * Estimate the sum of the brightness values, weighted with the gains + * applied on the channels in AWB as the Rec. 601 luma. + */ + double Y_sum = redSum * frameContext.awb.gains.red * .299 + + greenSum * frameContext.awb.gains.green * .587 + + blueSum * frameContext.awb.gains.blue * .114; + + /* Return the normalized relative luminance. */ + return Y_sum / (grid.height * grid.width) / kMaxLuminance; +} + /** * \brief Process IPU3 statistics, and run AGC operations * \param[in] context The shared IPA context @@ -268,7 +344,29 @@ void Agc::computeExposure(IPAFrameContext &frameContext) void Agc::process(IPAContext &context, const ipu3_uapi_stats_3a *stats) { measureBrightness(stats, context.configuration.grid.bdsGrid); - computeExposure(context.frameContext); + + double currentYGain = 1.0; + double targetY = kNormalizedLumaTarget; + + /* + * Do this calculation a few times as brightness increase can be + * non-linear when there are saturated regions. + */ + for (int i = 0; i < 8; i++) { + double initialY = computeInitialY(context.frameContext, + context.configuration.grid.bdsGrid, + stats, currentYGain); + double extra_gain = std::min(10.0, targetY / (initialY + .001)); + + currentYGain *= extra_gain; + LOG(IPU3Agc, Debug) << "Initial Y " << initialY + << " target " << targetY + << " gives gain " << currentYGain; + if (extra_gain < 1.01) + break; + } + + computeExposure(context.frameContext, currentYGain); frameCount_++; } diff --git a/src/ipa/ipu3/algorithms/agc.h b/src/ipa/ipu3/algorithms/agc.h index f0db25ee..79736283 100644 --- a/src/ipa/ipu3/algorithms/agc.h +++ b/src/ipa/ipu3/algorithms/agc.h @@ -34,7 +34,11 @@ private: void measureBrightness(const ipu3_uapi_stats_3a *stats, const ipu3_uapi_grid_config &grid); void filterExposure(); - void computeExposure(IPAFrameContext &frameContext); + void computeExposure(IPAFrameContext &frameContext, double currentYGain); + double computeInitialY(IPAFrameContext &frameContext, + const ipu3_uapi_grid_config &grid, + const ipu3_uapi_stats_3a *stats, + double currentYGain); uint64_t frameCount_; uint64_t lastFrame_;