| Message ID | 20210628202255.138874-8-jeanmichel.hautbois@ideasonboard.com |
|---|---|
| State | Changes Requested |
| Headers | show |
| Series |
|
| Related | show |
Hi Jean-Michel Thanks for your work - very happy to see our code being useful elsewhere! One minor thing - would it be ok to include a Raspberry Pi copyright alongside the IdeasonBoard one? Thanks! David On Mon, 28 Jun 2021 at 21:23, Jean-Michel Hautbois < jeanmichel.hautbois@ideasonboard.com> wrote: > This one comes from RPi for most if it, except that we are not > exchanging any metadata between algorithms for now. > When process() is called, the current analogue gain and shutter time are > calculated. The AWB stats from IPU3 are then parsed to generate new > statistics dedicated to AGC. This new grid is used to estimate the > luminance and each region is weighted. A default centered metering is > used as is should be the most used one. > > After calculating weighted regions, analogue gain and shutter time are > divided up and the values are then sent back to the IPAIPU3. > > Signed-off-by: Jean-Michel Hautbois <jeanmichel.hautbois@ideasonboard.com> > --- > src/ipa/ipu3/ipu3_agc.cpp | 276 ++++++++++++++++++++++++-------------- > src/ipa/ipu3/ipu3_agc.h | 23 +++- > 2 files changed, 196 insertions(+), 103 deletions(-) > > diff --git a/src/ipa/ipu3/ipu3_agc.cpp b/src/ipa/ipu3/ipu3_agc.cpp > index 042d67fa..0d421404 100644 > --- a/src/ipa/ipu3/ipu3_agc.cpp > +++ b/src/ipa/ipu3/ipu3_agc.cpp > @@ -27,37 +27,19 @@ namespace ipa::ipu3 { > > LOG_DEFINE_CATEGORY(IPU3Agc) > > -/* Number of frames to wait before calculating stats on minimum exposure > */ > -static constexpr uint32_t kInitialFrameMinAECount = 4; > -/* Number of frames to wait between new gain/exposure estimations */ > -static constexpr uint32_t kFrameSkipCount = 6; > - > -/* Maximum ISO value for analogue gain */ > -static constexpr uint32_t kMinISO = 100; > -static constexpr uint32_t kMaxISO = 1500; > - > -/* Maximum analogue gain value > - * \todo grab it from a camera helper */ > -static constexpr uint32_t kMinGain = kMinISO / 100; > -static constexpr uint32_t kMaxGain = kMaxISO / 100; > - > -/* \todo use calculated value based on sensor */ > -static constexpr uint32_t kMinExposure = 1; > -static constexpr uint32_t kMaxExposure = 1976; > - > /* Histogram constants */ > static constexpr uint32_t knumHistogramBins = 256; > -static constexpr double kEvGainTarget = 0.5; > > -/* A cell is 8 bytes and contains averages for RGB values and saturation > ratio */ > -static constexpr uint8_t kCellSize = 8; > +/* seems to be a 8-bit pipeline */ > +static constexpr uint8_t kPipelineBits = 8; > > IPU3Agc::IPU3Agc() > : frameCount_(0), lastFrame_(0), converged_(false), > updateControls_(false), iqMean_(0.0), gamma_(1.0), > lineDuration_(0s), maxExposureTime_(0s), > prevExposure_(0s), prevExposureNoDg_(0s), > - currentExposure_(0s), currentExposureNoDg_(0s) > + currentExposure_(0s), currentExposureNoDg_(0s), > + currentShutter_(1.0s), currentAnalogueGain_(1.0) > { > } > > @@ -83,55 +65,79 @@ void IPU3Agc::initialise(struct ipu3_uapi_grid_config > &bdsGrid, const IPAConfigI > } > minGain_ = std::max(itGain->second.min().get<int32_t>(), 1); > maxGain_ = itGain->second.max().get<int32_t>(); > + > + /* \todo: those values need to be extracted from a configuration > file */ > + shutterConstraints_.push_back(100us); > + shutterConstraints_.push_back(10ms); > + shutterConstraints_.push_back(33ms); > + gainConstraints_.push_back(1.0); > + gainConstraints_.push_back(4.0); > + gainConstraints_.push_back(16.0); > + > + fixedShutter_ = 0s; > + fixedAnalogueGain_ = 0.0; > } > > -void IPU3Agc::processBrightness(const ipu3_uapi_stats_3a *stats) > +/* Translate the IPU3 statistics into the default statistics region array > */ > +void IPU3Agc::generateStats(const ipu3_uapi_stats_3a *stats) > { > - const struct ipu3_uapi_grid_config statsAeGrid = > stats->stats_4a_config.awb_config.grid; > - Rectangle aeRegion = { statsAeGrid.x_start, > - statsAeGrid.y_start, > - static_cast<unsigned int>(statsAeGrid.x_end > - statsAeGrid.x_start) + 1, > - static_cast<unsigned int>(statsAeGrid.y_end > - statsAeGrid.y_start) + 1 }; > - Point topleft = aeRegion.topLeft(); > - int topleftX = topleft.x >> aeGrid_.block_width_log2; > - int topleftY = topleft.y >> aeGrid_.block_height_log2; > - > - /* Align to the grid cell width and height */ > - uint32_t startX = topleftX << aeGrid_.block_width_log2; > - uint32_t startY = topleftY * aeGrid_.width << > aeGrid_.block_width_log2; > - uint32_t endX = (startX + (aeRegion.size().width >> > aeGrid_.block_width_log2)) << aeGrid_.block_width_log2; > - uint32_t i, j; > - uint32_t count = 0; > - > + uint32_t regionWidth = round(aeGrid_.width / > static_cast<double>(kAgcStatsSizeX)); > + uint32_t regionHeight = round(aeGrid_.height / > static_cast<double>(kAgcStatsSizeY)); > uint32_t hist[knumHistogramBins] = { 0 }; > - for (j = topleftY; > - j < topleftY + (aeRegion.size().height >> > aeGrid_.block_height_log2); > - j++) { > - for (i = startX + startY; i < endX + startY; i += > kCellSize) { > - /* > - * The grid width (and maybe height) is not > reliable. > - * We observed a bit shift which makes the value > 160 to be 32 in the stats grid. > - * Use the one passed at init time. > - */ > - if (stats->awb_raw_buffer.meta_data[i + 4 + j * > aeGrid_.width] == 0) { > - uint8_t Gr = > stats->awb_raw_buffer.meta_data[i + 0 + j * aeGrid_.width]; > - uint8_t Gb = > stats->awb_raw_buffer.meta_data[i + 3 + j * aeGrid_.width]; > - hist[(Gr + Gb) / 2]++; > - count++; > + > + LOG(IPU3Agc, Debug) << "[" << (int)aeGrid_.width << "x" << > (int)aeGrid_.height << "] regions" > + << " scaled to [" << regionWidth << "x" << > regionHeight << "] AGC stats"; > + > + /* > + * Generate a (kAgcStatsSizeX x kAgcStatsSizeY) array from the > IPU3 grid which is > + * (aeGrid_.width x aeGrid_.height). > + */ > + for (unsigned int j = 0; j < kAgcStatsSizeY * regionHeight; j++) { > + for (unsigned int i = 0; i < kAgcStatsSizeX * regionWidth; > i++) { > + uint32_t cellPosition = j * aeGrid_.width + i; > + uint32_t cellX = (cellPosition / regionWidth) % > kAgcStatsSizeX; > + uint32_t cellY = ((cellPosition / aeGrid_.width) / > regionHeight) % kAgcStatsSizeY; > + > + uint32_t agcRegionPosition = > kAgcStatsRegions[cellY * kAgcStatsSizeX + cellX]; > + weights_[agcRegionPosition] = > kCenteredWeights[agcRegionPosition]; > + cellPosition *= sizeof(Ipu3AwbCell); > + > + /* Cast the initial IPU3 structure to simplify the > reading */ > + Ipu3AwbCell *currentCell = > reinterpret_cast<Ipu3AwbCell *>(const_cast<uint8_t > *>(&stats->awb_raw_buffer.meta_data[cellPosition])); > + if (currentCell->satRatio == 0) { > + /* The cell is not saturated, use the > current cell */ > + agcStats_[agcRegionPosition].counted++; > + uint32_t greenValue = > currentCell->greenRedAvg + currentCell->greenBlueAvg; > + hist[greenValue / 2]++; > + agcStats_[agcRegionPosition].gSum += > greenValue / 2; > + agcStats_[agcRegionPosition].rSum += > currentCell->redAvg; > + agcStats_[agcRegionPosition].bSum += > currentCell->blueAvg; > } > } > } > > - /* Limit the gamma effect for now */ > - gamma_ = 1.1; > - > /* Estimate the quantile mean of the top 2% of the histogram */ > iqMean_ = Histogram(Span<uint32_t>(hist)).interQuantileMean(0.98, > 1.0); > } > > +void IPU3Agc::clearStats() > +{ > + for (unsigned int i = 0; i < kNumAgcWeightedZones; i++) { > + agcStats_[i].bSum = 0; > + agcStats_[i].rSum = 0; > + agcStats_[i].gSum = 0; > + agcStats_[i].counted = 0; > + agcStats_[i].uncounted = 0; > + } > + > + awb_.blueGain = 1.0; > + awb_.greenGain = 1.0; > + awb_.redGain = 1.0; > +} > + > void IPU3Agc::filterExposure() > { > - double speed = 0.2; > + double speed = 0.08; > if (prevExposure_ == 0s) { > /* DG stands for digital gain.*/ > prevExposure_ = currentExposure_; > @@ -156,65 +162,131 @@ void IPU3Agc::filterExposure() > * total exposure, as there might not be enough digital gain > available > * in the ISP to hide it (which will cause nasty oscillation). > */ > - double fastReduceThreshold = 0.4; > + double fastReduceThreshold = 0.3; > if (prevExposureNoDg_ < > prevExposure_ * fastReduceThreshold) > prevExposureNoDg_ = prevExposure_ * fastReduceThreshold; > LOG(IPU3Agc, Debug) << "After filtering, total_exposure " << > prevExposure_; > } > > -void IPU3Agc::lockExposureGain(uint32_t &exposure, double &gain) > +double IPU3Agc::computeInitialY(IspStatsRegion regions[], AwbStatus const > &awb, > + double weights[], double gain) > { > - updateControls_ = false; > + /* Note how the calculation below means that equal weights give you > + * "average" metering (i.e. all pixels equally important). */ > + double redSum = 0, greenSum = 0, blueSum = 0, pixelSum = 0; > + for (unsigned int i = 0; i < kNumAgcWeightedZones; i++) { > + double counted = regions[i].counted; > + double rSum = std::min(regions[i].rSum * gain, ((1 << > kPipelineBits) - 1) * counted); > + double gSum = std::min(regions[i].gSum * gain, ((1 << > kPipelineBits) - 1) * counted); > + double bSum = std::min(regions[i].bSum * gain, ((1 << > kPipelineBits) - 1) * counted); > + redSum += rSum * weights[i]; > + greenSum += gSum * weights[i]; > + blueSum += bSum * weights[i]; > + pixelSum += counted * weights[i]; > + } > + if (pixelSum == 0.0) { > + LOG(IPU3Agc, Warning) << "computeInitialY: pixel_sum is > zero"; > + return 0; > + } > + double Y_sum = redSum * awb.redGain * .299 + > + greenSum * awb.greenGain * .587 + > + blueSum * awb.blueGain * .114; > > - /* Algorithm initialization should wait for first valid frames */ > - /* \todo - have a number of frames given by DelayedControls ? > - * - implement a function for IIR */ > - if ((frameCount_ < kInitialFrameMinAECount) || (frameCount_ - > lastFrame_ < kFrameSkipCount)) > - return; > + return Y_sum / pixelSum / (1 << kPipelineBits); > +} > > - /* Are we correctly exposed ? */ > - if (std::abs(iqMean_ - kEvGainTarget * knumHistogramBins) <= 1) { > - LOG(IPU3Agc, Debug) << "!!! Good exposure with iqMean = " > << iqMean_; > - converged_ = true; > - } else { > - double newGain = kEvGainTarget * knumHistogramBins / > iqMean_; > - > - /* extracted from Rpi::Agc::computeTargetExposure */ > - libcamera::utils::Duration currentShutter = exposure * > lineDuration_; > - currentExposureNoDg_ = currentShutter * gain; > - LOG(IPU3Agc, Debug) << "Actual total exposure " << > currentExposureNoDg_ > - << " Shutter speed " << currentShutter > - << " Gain " << gain; > - currentExposure_ = currentExposureNoDg_ * newGain; > - libcamera::utils::Duration maxTotalExposure = > maxExposureTime_ * kMaxGain; > - currentExposure_ = std::min(currentExposure_, > maxTotalExposure); > - LOG(IPU3Agc, Debug) << "Target total exposure " << > currentExposure_; > - > - /* \todo: estimate if we need to desaturate */ > - filterExposure(); > - > - libcamera::utils::Duration newExposure = 0.0s; > - if (currentShutter < maxExposureTime_) { > - exposure = > std::clamp(static_cast<uint32_t>(exposure * currentExposure_ / > currentExposureNoDg_), kMinExposure, kMaxExposure); > - newExposure = currentExposure_ / exposure; > - gain = std::clamp(static_cast<uint32_t>(gain * > currentExposure_ / newExposure), kMinGain, kMaxGain); > - updateControls_ = true; > - } else if (currentShutter >= maxExposureTime_) { > - gain = std::clamp(static_cast<uint32_t>(gain * > currentExposure_ / currentExposureNoDg_), kMinGain, kMaxGain); > - newExposure = currentExposure_ / gain; > - exposure = > std::clamp(static_cast<uint32_t>(exposure * currentExposure_ / > newExposure), kMinExposure, kMaxExposure); > - updateControls_ = true; > +void IPU3Agc::computeTargetExposure(double gain) > +{ > + currentExposure_ = currentExposureNoDg_ * gain; > + /* \todo: have a list of shutter speeds */ > + Duration maxShutterSpeed = shutterConstraints_.back(); > + Duration maxTotalExposure = maxShutterSpeed * > gainConstraints_.back(); > + > + currentExposure_ = std::min(currentExposure_, maxTotalExposure); > + LOG(IPU3Agc, Debug) << "Target total_exposure " << > currentExposure_; > +} > + > +void IPU3Agc::divideUpExposure() > +{ > + Duration exposureValue = prevExposure_; > + Duration shutterTime; > + double analogueGain; > + shutterTime = shutterConstraints_[0]; > + shutterTime = std::min(shutterTime, shutterConstraints_.back()); > + analogueGain = gainConstraints_[0]; > + > + if (shutterTime * analogueGain < exposureValue) { > + for (unsigned int stage = 1; > + stage < gainConstraints_.size(); stage++) { > + if (fixedShutter_ == 0s) { > + Duration stageShutter = > + > std::min(shutterConstraints_[stage], shutterConstraints_.back()); > + if (stageShutter * analogueGain >= > + exposureValue) { > + shutterTime = > + exposureValue / > analogueGain; > + break; > + } > + shutterTime = stageShutter; > + } > + if (fixedAnalogueGain_ == 0.0) { > + if (gainConstraints_[stage] * shutterTime > >= exposureValue) { > + analogueGain = exposureValue / > shutterTime; > + break; > + } > + analogueGain = gainConstraints_[stage]; > + } > } > - LOG(IPU3Agc, Debug) << "Adjust exposure " << exposure * > lineDuration_ << " and gain " << gain; > } > - lastFrame_ = frameCount_; > + LOG(IPU3Agc, Debug) << "Divided up shutter and gain are " << > shutterTime << " and " > + << analogueGain; > + > + /* \todo: flickering avoidance ? */ > + filteredShutter_ = shutterTime; > + filteredAnalogueGain_ = analogueGain; > +} > + > +void IPU3Agc::computeGain(double ¤tGain) > +{ > + currentGain = 1.0; > + /* \todo: the target Y needs to be grabbed from a configuration */ > + double targetY = 0.162; > + for (int i = 0; i < 8; i++) { > + double initialY = computeInitialY(agcStats_, awb_, > weights_, currentGain); > + double extra_gain = std::min(10.0, targetY / (initialY + > .001)); > + > + currentGain *= extra_gain; > + LOG(IPU3Agc, Debug) << "Initial Y " << initialY << " > target " << targetY > + << " gives gain " << currentGain; > + if (extra_gain < 1.01) > + break; > + } > + > + double newGain = 128 / iqMean_; > + LOG(IPU3Agc, Debug) << "gain: " << currentGain << " new gain: " << > newGain; > } > > -void IPU3Agc::process(const ipu3_uapi_stats_3a *stats, uint32_t > &exposure, double &gain) > +void IPU3Agc::process(const ipu3_uapi_stats_3a *stats, uint32_t > &exposure, double &analogueGain) > { > - processBrightness(stats); > - lockExposureGain(exposure, gain); > + ASSERT(stats->stats_3a_status.awb_en); > + clearStats(); > + generateStats(stats); > + currentShutter_ = exposure * lineDuration_; > + /* \todo: the gain needs to be calculated based on sensor > informations */ > + currentAnalogueGain_ = analogueGain; > + currentExposureNoDg_ = currentShutter_ * currentAnalogueGain_; > + > + double currentGain = 1; > + computeGain(currentGain); > + computeTargetExposure(currentGain); > + filterExposure(); > + divideUpExposure(); > + > + exposure = filteredShutter_ / lineDuration_; > + analogueGain = filteredAnalogueGain_; > + > + updateControls_ = true; > frameCount_++; > } > > diff --git a/src/ipa/ipu3/ipu3_agc.h b/src/ipa/ipu3/ipu3_agc.h > index ce43c534..f1b1157b 100644 > --- a/src/ipa/ipu3/ipu3_agc.h > +++ b/src/ipa/ipu3/ipu3_agc.h > @@ -35,8 +35,8 @@ public: > IPU3Agc(); > ~IPU3Agc() = default; > > - void process(const ipu3_uapi_stats_3a *stats, uint32_t &exposure, > double &gain); > void initialise(struct ipu3_uapi_grid_config &bdsGrid, const > IPAConfigInfo &configInfo); > + void process(const ipu3_uapi_stats_3a *stats, uint32_t &exposure, > double &analogueGain); > bool converged() { return converged_; } > bool updateControls() { return updateControls_; } > /* \todo Use a metadata exchange between IPAs */ > @@ -46,6 +46,17 @@ private: > void processBrightness(const ipu3_uapi_stats_3a *stats); > void filterExposure(); > void lockExposureGain(uint32_t &exposure, double &gain); > + void generateStats(const ipu3_uapi_stats_3a *stats); > + void clearStats(); > + void generateZones(std::vector<RGB> &zones); > + double computeInitialY(IspStatsRegion regions[], AwbStatus const > &awb, double weights[], double gain); > + void computeTargetExposure(double currentGain); > + void divideUpExposure(); > + void computeGain(double ¤tGain); > + > + AwbStatus awb_; > + double weights_[kNumAgcWeightedZones]; > + IspStatsRegion agcStats_[kNumAgcWeightedZones]; > > struct ipu3_uapi_grid_config aeGrid_; > ControlInfoMap ctrls_; > @@ -72,6 +83,16 @@ private: > Duration prevExposureNoDg_; > Duration currentExposure_; > Duration currentExposureNoDg_; > + > + Duration currentShutter_; > + std::vector<Duration> shutterConstraints_; > + Duration fixedShutter_; > + Duration filteredShutter_; > + > + double currentAnalogueGain_; > + std::vector<double> gainConstraints_; > + double fixedAnalogueGain_; > + double filteredAnalogueGain_; > }; > > } /* namespace ipa::ipu3 */ > -- > 2.30.2 > >
Hi David, On Tue, Jun 29, 2021 at 02:22:25PM +0100, David Plowman wrote: > Hi Jean-Michel > > Thanks for your work - very happy to see our code being useful elsewhere! > > One minor thing - would it be ok to include a Raspberry Pi copyright > alongside the IdeasonBoard one? I haven't looked at the code yet, but if a substantial part of it comes from the RPi implementation, it should certainly mention appropriate authorship and copyright information. In the kernel, we often mention it as follows: * Based on the implementation from the Raspberry Pi IPA, * Copyright (C) 2019-2021, Raspberry Pi (Trading) Ltd. Would that be fine with you ? > On Mon, 28 Jun 2021 at 21:23, Jean-Michel Hautboiswrote: > > > This one comes from RPi for most if it, except that we are not > > exchanging any metadata between algorithms for now. > > When process() is called, the current analogue gain and shutter time are > > calculated. The AWB stats from IPU3 are then parsed to generate new > > statistics dedicated to AGC. This new grid is used to estimate the > > luminance and each region is weighted. A default centered metering is > > used as is should be the most used one. > > > > After calculating weighted regions, analogue gain and shutter time are > > divided up and the values are then sent back to the IPAIPU3. > > > > Signed-off-by: Jean-Michel Hautbois <jeanmichel.hautbois@ideasonboard.com> > > --- > > src/ipa/ipu3/ipu3_agc.cpp | 276 ++++++++++++++++++++++++-------------- > > src/ipa/ipu3/ipu3_agc.h | 23 +++- > > 2 files changed, 196 insertions(+), 103 deletions(-)
HI Laurent Yes, I think that would be fine. Thanks David On Tue, 29 Jun 2021 at 14:55, Laurent Pinchart < laurent.pinchart@ideasonboard.com> wrote: > Hi David, > > On Tue, Jun 29, 2021 at 02:22:25PM +0100, David Plowman wrote: > > Hi Jean-Michel > > > > Thanks for your work - very happy to see our code being useful elsewhere! > > > > One minor thing - would it be ok to include a Raspberry Pi copyright > > alongside the IdeasonBoard one? > > I haven't looked at the code yet, but if a substantial part of it comes > from the RPi implementation, it should certainly mention appropriate > authorship and copyright information. In the kernel, we often mention it > as follows: > > * Based on the implementation from the Raspberry Pi IPA, > * Copyright (C) 2019-2021, Raspberry Pi (Trading) Ltd. > > Would that be fine with you ? > > > On Mon, 28 Jun 2021 at 21:23, Jean-Michel Hautboiswrote: > > > > > This one comes from RPi for most if it, except that we are not > > > exchanging any metadata between algorithms for now. > > > When process() is called, the current analogue gain and shutter time > are > > > calculated. The AWB stats from IPU3 are then parsed to generate new > > > statistics dedicated to AGC. This new grid is used to estimate the > > > luminance and each region is weighted. A default centered metering is > > > used as is should be the most used one. > > > > > > After calculating weighted regions, analogue gain and shutter time are > > > divided up and the values are then sent back to the IPAIPU3. > > > > > > Signed-off-by: Jean-Michel Hautbois < > jeanmichel.hautbois@ideasonboard.com> > > > --- > > > src/ipa/ipu3/ipu3_agc.cpp | 276 ++++++++++++++++++++++++-------------- > > > src/ipa/ipu3/ipu3_agc.h | 23 +++- > > > 2 files changed, 196 insertions(+), 103 deletions(-) > > -- > Regards, > > Laurent Pinchart >
Hi David ! On 29/06/2021 16:03, David Plowman wrote: > HI Laurent > > Yes, I think that would be fine. > > Thanks > David > > On Tue, 29 Jun 2021 at 14:55, Laurent Pinchart > <laurent.pinchart@ideasonboard.com > <mailto:laurent.pinchart@ideasonboard.com>> wrote: > > Hi David, > > On Tue, Jun 29, 2021 at 02:22:25PM +0100, David Plowman wrote: > > Hi Jean-Michel > > > > Thanks for your work - very happy to see our code being useful > elsewhere! > > > > One minor thing - would it be ok to include a Raspberry Pi copyright > > alongside the IdeasonBoard one? > > I haven't looked at the code yet, but if a substantial part of it comes > from the RPi implementation, it should certainly mention appropriate > authorship and copyright information. In the kernel, we often mention it > as follows: > > * Based on the implementation from the Raspberry Pi IPA, > * Copyright (C) 2019-2021, Raspberry Pi (Trading) Ltd. > I missed it, I wanted to do it ! Done on my -v2 ;-) The implementation is a lot from RPi, but with small differences, if you have comments, please shot :-). > Would that be fine with you ? > > > On Mon, 28 Jun 2021 at 21:23, Jean-Michel Hautboiswrote: > > > > > This one comes from RPi for most if it, except that we are not > > > exchanging any metadata between algorithms for now. > > > When process() is called, the current analogue gain and shutter > time are > > > calculated. The AWB stats from IPU3 are then parsed to generate new > > > statistics dedicated to AGC. This new grid is used to estimate the > > > luminance and each region is weighted. A default centered > metering is > > > used as is should be the most used one. > > > > > > After calculating weighted regions, analogue gain and shutter > time are > > > divided up and the values are then sent back to the IPAIPU3. > > > > > > Signed-off-by: Jean-Michel Hautbois > <jeanmichel.hautbois@ideasonboard.com > <mailto:jeanmichel.hautbois@ideasonboard.com>> > > > --- > > > src/ipa/ipu3/ipu3_agc.cpp | 276 > ++++++++++++++++++++++++-------------- > > > src/ipa/ipu3/ipu3_agc.h | 23 +++- > > > 2 files changed, 196 insertions(+), 103 deletions(-) > > -- > Regards, > > Laurent Pinchart >
diff --git a/src/ipa/ipu3/ipu3_agc.cpp b/src/ipa/ipu3/ipu3_agc.cpp index 042d67fa..0d421404 100644 --- a/src/ipa/ipu3/ipu3_agc.cpp +++ b/src/ipa/ipu3/ipu3_agc.cpp @@ -27,37 +27,19 @@ namespace ipa::ipu3 { LOG_DEFINE_CATEGORY(IPU3Agc) -/* Number of frames to wait before calculating stats on minimum exposure */ -static constexpr uint32_t kInitialFrameMinAECount = 4; -/* Number of frames to wait between new gain/exposure estimations */ -static constexpr uint32_t kFrameSkipCount = 6; - -/* Maximum ISO value for analogue gain */ -static constexpr uint32_t kMinISO = 100; -static constexpr uint32_t kMaxISO = 1500; - -/* Maximum analogue gain value - * \todo grab it from a camera helper */ -static constexpr uint32_t kMinGain = kMinISO / 100; -static constexpr uint32_t kMaxGain = kMaxISO / 100; - -/* \todo use calculated value based on sensor */ -static constexpr uint32_t kMinExposure = 1; -static constexpr uint32_t kMaxExposure = 1976; - /* Histogram constants */ static constexpr uint32_t knumHistogramBins = 256; -static constexpr double kEvGainTarget = 0.5; -/* A cell is 8 bytes and contains averages for RGB values and saturation ratio */ -static constexpr uint8_t kCellSize = 8; +/* seems to be a 8-bit pipeline */ +static constexpr uint8_t kPipelineBits = 8; IPU3Agc::IPU3Agc() : frameCount_(0), lastFrame_(0), converged_(false), updateControls_(false), iqMean_(0.0), gamma_(1.0), lineDuration_(0s), maxExposureTime_(0s), prevExposure_(0s), prevExposureNoDg_(0s), - currentExposure_(0s), currentExposureNoDg_(0s) + currentExposure_(0s), currentExposureNoDg_(0s), + currentShutter_(1.0s), currentAnalogueGain_(1.0) { } @@ -83,55 +65,79 @@ void IPU3Agc::initialise(struct ipu3_uapi_grid_config &bdsGrid, const IPAConfigI } minGain_ = std::max(itGain->second.min().get<int32_t>(), 1); maxGain_ = itGain->second.max().get<int32_t>(); + + /* \todo: those values need to be extracted from a configuration file */ + shutterConstraints_.push_back(100us); + shutterConstraints_.push_back(10ms); + shutterConstraints_.push_back(33ms); + gainConstraints_.push_back(1.0); + gainConstraints_.push_back(4.0); + gainConstraints_.push_back(16.0); + + fixedShutter_ = 0s; + fixedAnalogueGain_ = 0.0; } -void IPU3Agc::processBrightness(const ipu3_uapi_stats_3a *stats) +/* Translate the IPU3 statistics into the default statistics region array */ +void IPU3Agc::generateStats(const ipu3_uapi_stats_3a *stats) { - const struct ipu3_uapi_grid_config statsAeGrid = stats->stats_4a_config.awb_config.grid; - Rectangle aeRegion = { statsAeGrid.x_start, - statsAeGrid.y_start, - static_cast<unsigned int>(statsAeGrid.x_end - statsAeGrid.x_start) + 1, - static_cast<unsigned int>(statsAeGrid.y_end - statsAeGrid.y_start) + 1 }; - Point topleft = aeRegion.topLeft(); - int topleftX = topleft.x >> aeGrid_.block_width_log2; - int topleftY = topleft.y >> aeGrid_.block_height_log2; - - /* Align to the grid cell width and height */ - uint32_t startX = topleftX << aeGrid_.block_width_log2; - uint32_t startY = topleftY * aeGrid_.width << aeGrid_.block_width_log2; - uint32_t endX = (startX + (aeRegion.size().width >> aeGrid_.block_width_log2)) << aeGrid_.block_width_log2; - uint32_t i, j; - uint32_t count = 0; - + uint32_t regionWidth = round(aeGrid_.width / static_cast<double>(kAgcStatsSizeX)); + uint32_t regionHeight = round(aeGrid_.height / static_cast<double>(kAgcStatsSizeY)); uint32_t hist[knumHistogramBins] = { 0 }; - for (j = topleftY; - j < topleftY + (aeRegion.size().height >> aeGrid_.block_height_log2); - j++) { - for (i = startX + startY; i < endX + startY; i += kCellSize) { - /* - * The grid width (and maybe height) is not reliable. - * We observed a bit shift which makes the value 160 to be 32 in the stats grid. - * Use the one passed at init time. - */ - if (stats->awb_raw_buffer.meta_data[i + 4 + j * aeGrid_.width] == 0) { - uint8_t Gr = stats->awb_raw_buffer.meta_data[i + 0 + j * aeGrid_.width]; - uint8_t Gb = stats->awb_raw_buffer.meta_data[i + 3 + j * aeGrid_.width]; - hist[(Gr + Gb) / 2]++; - count++; + + LOG(IPU3Agc, Debug) << "[" << (int)aeGrid_.width << "x" << (int)aeGrid_.height << "] regions" + << " scaled to [" << regionWidth << "x" << regionHeight << "] AGC stats"; + + /* + * Generate a (kAgcStatsSizeX x kAgcStatsSizeY) array from the IPU3 grid which is + * (aeGrid_.width x aeGrid_.height). + */ + for (unsigned int j = 0; j < kAgcStatsSizeY * regionHeight; j++) { + for (unsigned int i = 0; i < kAgcStatsSizeX * regionWidth; i++) { + uint32_t cellPosition = j * aeGrid_.width + i; + uint32_t cellX = (cellPosition / regionWidth) % kAgcStatsSizeX; + uint32_t cellY = ((cellPosition / aeGrid_.width) / regionHeight) % kAgcStatsSizeY; + + uint32_t agcRegionPosition = kAgcStatsRegions[cellY * kAgcStatsSizeX + cellX]; + weights_[agcRegionPosition] = kCenteredWeights[agcRegionPosition]; + cellPosition *= sizeof(Ipu3AwbCell); + + /* Cast the initial IPU3 structure to simplify the reading */ + Ipu3AwbCell *currentCell = reinterpret_cast<Ipu3AwbCell *>(const_cast<uint8_t *>(&stats->awb_raw_buffer.meta_data[cellPosition])); + if (currentCell->satRatio == 0) { + /* The cell is not saturated, use the current cell */ + agcStats_[agcRegionPosition].counted++; + uint32_t greenValue = currentCell->greenRedAvg + currentCell->greenBlueAvg; + hist[greenValue / 2]++; + agcStats_[agcRegionPosition].gSum += greenValue / 2; + agcStats_[agcRegionPosition].rSum += currentCell->redAvg; + agcStats_[agcRegionPosition].bSum += currentCell->blueAvg; } } } - /* Limit the gamma effect for now */ - gamma_ = 1.1; - /* Estimate the quantile mean of the top 2% of the histogram */ iqMean_ = Histogram(Span<uint32_t>(hist)).interQuantileMean(0.98, 1.0); } +void IPU3Agc::clearStats() +{ + for (unsigned int i = 0; i < kNumAgcWeightedZones; i++) { + agcStats_[i].bSum = 0; + agcStats_[i].rSum = 0; + agcStats_[i].gSum = 0; + agcStats_[i].counted = 0; + agcStats_[i].uncounted = 0; + } + + awb_.blueGain = 1.0; + awb_.greenGain = 1.0; + awb_.redGain = 1.0; +} + void IPU3Agc::filterExposure() { - double speed = 0.2; + double speed = 0.08; if (prevExposure_ == 0s) { /* DG stands for digital gain.*/ prevExposure_ = currentExposure_; @@ -156,65 +162,131 @@ void IPU3Agc::filterExposure() * total exposure, as there might not be enough digital gain available * in the ISP to hide it (which will cause nasty oscillation). */ - double fastReduceThreshold = 0.4; + double fastReduceThreshold = 0.3; if (prevExposureNoDg_ < prevExposure_ * fastReduceThreshold) prevExposureNoDg_ = prevExposure_ * fastReduceThreshold; LOG(IPU3Agc, Debug) << "After filtering, total_exposure " << prevExposure_; } -void IPU3Agc::lockExposureGain(uint32_t &exposure, double &gain) +double IPU3Agc::computeInitialY(IspStatsRegion regions[], AwbStatus const &awb, + double weights[], double gain) { - updateControls_ = false; + /* Note how the calculation below means that equal weights give you + * "average" metering (i.e. all pixels equally important). */ + double redSum = 0, greenSum = 0, blueSum = 0, pixelSum = 0; + for (unsigned int i = 0; i < kNumAgcWeightedZones; i++) { + double counted = regions[i].counted; + double rSum = std::min(regions[i].rSum * gain, ((1 << kPipelineBits) - 1) * counted); + double gSum = std::min(regions[i].gSum * gain, ((1 << kPipelineBits) - 1) * counted); + double bSum = std::min(regions[i].bSum * gain, ((1 << kPipelineBits) - 1) * counted); + redSum += rSum * weights[i]; + greenSum += gSum * weights[i]; + blueSum += bSum * weights[i]; + pixelSum += counted * weights[i]; + } + if (pixelSum == 0.0) { + LOG(IPU3Agc, Warning) << "computeInitialY: pixel_sum is zero"; + return 0; + } + double Y_sum = redSum * awb.redGain * .299 + + greenSum * awb.greenGain * .587 + + blueSum * awb.blueGain * .114; - /* Algorithm initialization should wait for first valid frames */ - /* \todo - have a number of frames given by DelayedControls ? - * - implement a function for IIR */ - if ((frameCount_ < kInitialFrameMinAECount) || (frameCount_ - lastFrame_ < kFrameSkipCount)) - return; + return Y_sum / pixelSum / (1 << kPipelineBits); +} - /* Are we correctly exposed ? */ - if (std::abs(iqMean_ - kEvGainTarget * knumHistogramBins) <= 1) { - LOG(IPU3Agc, Debug) << "!!! Good exposure with iqMean = " << iqMean_; - converged_ = true; - } else { - double newGain = kEvGainTarget * knumHistogramBins / iqMean_; - - /* extracted from Rpi::Agc::computeTargetExposure */ - libcamera::utils::Duration currentShutter = exposure * lineDuration_; - currentExposureNoDg_ = currentShutter * gain; - LOG(IPU3Agc, Debug) << "Actual total exposure " << currentExposureNoDg_ - << " Shutter speed " << currentShutter - << " Gain " << gain; - currentExposure_ = currentExposureNoDg_ * newGain; - libcamera::utils::Duration maxTotalExposure = maxExposureTime_ * kMaxGain; - currentExposure_ = std::min(currentExposure_, maxTotalExposure); - LOG(IPU3Agc, Debug) << "Target total exposure " << currentExposure_; - - /* \todo: estimate if we need to desaturate */ - filterExposure(); - - libcamera::utils::Duration newExposure = 0.0s; - if (currentShutter < maxExposureTime_) { - exposure = std::clamp(static_cast<uint32_t>(exposure * currentExposure_ / currentExposureNoDg_), kMinExposure, kMaxExposure); - newExposure = currentExposure_ / exposure; - gain = std::clamp(static_cast<uint32_t>(gain * currentExposure_ / newExposure), kMinGain, kMaxGain); - updateControls_ = true; - } else if (currentShutter >= maxExposureTime_) { - gain = std::clamp(static_cast<uint32_t>(gain * currentExposure_ / currentExposureNoDg_), kMinGain, kMaxGain); - newExposure = currentExposure_ / gain; - exposure = std::clamp(static_cast<uint32_t>(exposure * currentExposure_ / newExposure), kMinExposure, kMaxExposure); - updateControls_ = true; +void IPU3Agc::computeTargetExposure(double gain) +{ + currentExposure_ = currentExposureNoDg_ * gain; + /* \todo: have a list of shutter speeds */ + Duration maxShutterSpeed = shutterConstraints_.back(); + Duration maxTotalExposure = maxShutterSpeed * gainConstraints_.back(); + + currentExposure_ = std::min(currentExposure_, maxTotalExposure); + LOG(IPU3Agc, Debug) << "Target total_exposure " << currentExposure_; +} + +void IPU3Agc::divideUpExposure() +{ + Duration exposureValue = prevExposure_; + Duration shutterTime; + double analogueGain; + shutterTime = shutterConstraints_[0]; + shutterTime = std::min(shutterTime, shutterConstraints_.back()); + analogueGain = gainConstraints_[0]; + + if (shutterTime * analogueGain < exposureValue) { + for (unsigned int stage = 1; + stage < gainConstraints_.size(); stage++) { + if (fixedShutter_ == 0s) { + Duration stageShutter = + std::min(shutterConstraints_[stage], shutterConstraints_.back()); + if (stageShutter * analogueGain >= + exposureValue) { + shutterTime = + exposureValue / analogueGain; + break; + } + shutterTime = stageShutter; + } + if (fixedAnalogueGain_ == 0.0) { + if (gainConstraints_[stage] * shutterTime >= exposureValue) { + analogueGain = exposureValue / shutterTime; + break; + } + analogueGain = gainConstraints_[stage]; + } } - LOG(IPU3Agc, Debug) << "Adjust exposure " << exposure * lineDuration_ << " and gain " << gain; } - lastFrame_ = frameCount_; + LOG(IPU3Agc, Debug) << "Divided up shutter and gain are " << shutterTime << " and " + << analogueGain; + + /* \todo: flickering avoidance ? */ + filteredShutter_ = shutterTime; + filteredAnalogueGain_ = analogueGain; +} + +void IPU3Agc::computeGain(double ¤tGain) +{ + currentGain = 1.0; + /* \todo: the target Y needs to be grabbed from a configuration */ + double targetY = 0.162; + for (int i = 0; i < 8; i++) { + double initialY = computeInitialY(agcStats_, awb_, weights_, currentGain); + double extra_gain = std::min(10.0, targetY / (initialY + .001)); + + currentGain *= extra_gain; + LOG(IPU3Agc, Debug) << "Initial Y " << initialY << " target " << targetY + << " gives gain " << currentGain; + if (extra_gain < 1.01) + break; + } + + double newGain = 128 / iqMean_; + LOG(IPU3Agc, Debug) << "gain: " << currentGain << " new gain: " << newGain; } -void IPU3Agc::process(const ipu3_uapi_stats_3a *stats, uint32_t &exposure, double &gain) +void IPU3Agc::process(const ipu3_uapi_stats_3a *stats, uint32_t &exposure, double &analogueGain) { - processBrightness(stats); - lockExposureGain(exposure, gain); + ASSERT(stats->stats_3a_status.awb_en); + clearStats(); + generateStats(stats); + currentShutter_ = exposure * lineDuration_; + /* \todo: the gain needs to be calculated based on sensor informations */ + currentAnalogueGain_ = analogueGain; + currentExposureNoDg_ = currentShutter_ * currentAnalogueGain_; + + double currentGain = 1; + computeGain(currentGain); + computeTargetExposure(currentGain); + filterExposure(); + divideUpExposure(); + + exposure = filteredShutter_ / lineDuration_; + analogueGain = filteredAnalogueGain_; + + updateControls_ = true; frameCount_++; } diff --git a/src/ipa/ipu3/ipu3_agc.h b/src/ipa/ipu3/ipu3_agc.h index ce43c534..f1b1157b 100644 --- a/src/ipa/ipu3/ipu3_agc.h +++ b/src/ipa/ipu3/ipu3_agc.h @@ -35,8 +35,8 @@ public: IPU3Agc(); ~IPU3Agc() = default; - void process(const ipu3_uapi_stats_3a *stats, uint32_t &exposure, double &gain); void initialise(struct ipu3_uapi_grid_config &bdsGrid, const IPAConfigInfo &configInfo); + void process(const ipu3_uapi_stats_3a *stats, uint32_t &exposure, double &analogueGain); bool converged() { return converged_; } bool updateControls() { return updateControls_; } /* \todo Use a metadata exchange between IPAs */ @@ -46,6 +46,17 @@ private: void processBrightness(const ipu3_uapi_stats_3a *stats); void filterExposure(); void lockExposureGain(uint32_t &exposure, double &gain); + void generateStats(const ipu3_uapi_stats_3a *stats); + void clearStats(); + void generateZones(std::vector<RGB> &zones); + double computeInitialY(IspStatsRegion regions[], AwbStatus const &awb, double weights[], double gain); + void computeTargetExposure(double currentGain); + void divideUpExposure(); + void computeGain(double ¤tGain); + + AwbStatus awb_; + double weights_[kNumAgcWeightedZones]; + IspStatsRegion agcStats_[kNumAgcWeightedZones]; struct ipu3_uapi_grid_config aeGrid_; ControlInfoMap ctrls_; @@ -72,6 +83,16 @@ private: Duration prevExposureNoDg_; Duration currentExposure_; Duration currentExposureNoDg_; + + Duration currentShutter_; + std::vector<Duration> shutterConstraints_; + Duration fixedShutter_; + Duration filteredShutter_; + + double currentAnalogueGain_; + std::vector<double> gainConstraints_; + double fixedAnalogueGain_; + double filteredAnalogueGain_; }; } /* namespace ipa::ipu3 */
This one comes from RPi for most if it, except that we are not exchanging any metadata between algorithms for now. When process() is called, the current analogue gain and shutter time are calculated. The AWB stats from IPU3 are then parsed to generate new statistics dedicated to AGC. This new grid is used to estimate the luminance and each region is weighted. A default centered metering is used as is should be the most used one. After calculating weighted regions, analogue gain and shutter time are divided up and the values are then sent back to the IPAIPU3. Signed-off-by: Jean-Michel Hautbois <jeanmichel.hautbois@ideasonboard.com> --- src/ipa/ipu3/ipu3_agc.cpp | 276 ++++++++++++++++++++++++-------------- src/ipa/ipu3/ipu3_agc.h | 23 +++- 2 files changed, 196 insertions(+), 103 deletions(-)