diff --git a/src/ipa/ipu3/ipu3_agc.cpp b/src/ipa/ipu3/ipu3_agc.cpp index 042d67fa..0d421404 100644 --- a/src/ipa/ipu3/ipu3_agc.cpp +++ b/src/ipa/ipu3/ipu3_agc.cpp @@ -27,37 +27,19 @@ namespace ipa::ipu3 { LOG_DEFINE_CATEGORY(IPU3Agc) -/* Number of frames to wait before calculating stats on minimum exposure */ -static constexpr uint32_t kInitialFrameMinAECount = 4; -/* Number of frames to wait between new gain/exposure estimations */ -static constexpr uint32_t kFrameSkipCount = 6; - -/* Maximum ISO value for analogue gain */ -static constexpr uint32_t kMinISO = 100; -static constexpr uint32_t kMaxISO = 1500; - -/* Maximum analogue gain value - * \todo grab it from a camera helper */ -static constexpr uint32_t kMinGain = kMinISO / 100; -static constexpr uint32_t kMaxGain = kMaxISO / 100; - -/* \todo use calculated value based on sensor */ -static constexpr uint32_t kMinExposure = 1; -static constexpr uint32_t kMaxExposure = 1976; - /* Histogram constants */ static constexpr uint32_t knumHistogramBins = 256; -static constexpr double kEvGainTarget = 0.5; -/* A cell is 8 bytes and contains averages for RGB values and saturation ratio */ -static constexpr uint8_t kCellSize = 8; +/* seems to be a 8-bit pipeline */ +static constexpr uint8_t kPipelineBits = 8; IPU3Agc::IPU3Agc() : frameCount_(0), lastFrame_(0), converged_(false), updateControls_(false), iqMean_(0.0), gamma_(1.0), lineDuration_(0s), maxExposureTime_(0s), prevExposure_(0s), prevExposureNoDg_(0s), - currentExposure_(0s), currentExposureNoDg_(0s) + currentExposure_(0s), currentExposureNoDg_(0s), + currentShutter_(1.0s), currentAnalogueGain_(1.0) { } @@ -83,55 +65,79 @@ void IPU3Agc::initialise(struct ipu3_uapi_grid_config &bdsGrid, const IPAConfigI } minGain_ = std::max(itGain->second.min().get(), 1); maxGain_ = itGain->second.max().get(); + + /* \todo: those values need to be extracted from a configuration file */ + shutterConstraints_.push_back(100us); + shutterConstraints_.push_back(10ms); + shutterConstraints_.push_back(33ms); + gainConstraints_.push_back(1.0); + gainConstraints_.push_back(4.0); + gainConstraints_.push_back(16.0); + + fixedShutter_ = 0s; + fixedAnalogueGain_ = 0.0; } -void IPU3Agc::processBrightness(const ipu3_uapi_stats_3a *stats) +/* Translate the IPU3 statistics into the default statistics region array */ +void IPU3Agc::generateStats(const ipu3_uapi_stats_3a *stats) { - const struct ipu3_uapi_grid_config statsAeGrid = stats->stats_4a_config.awb_config.grid; - Rectangle aeRegion = { statsAeGrid.x_start, - statsAeGrid.y_start, - static_cast(statsAeGrid.x_end - statsAeGrid.x_start) + 1, - static_cast(statsAeGrid.y_end - statsAeGrid.y_start) + 1 }; - Point topleft = aeRegion.topLeft(); - int topleftX = topleft.x >> aeGrid_.block_width_log2; - int topleftY = topleft.y >> aeGrid_.block_height_log2; - - /* Align to the grid cell width and height */ - uint32_t startX = topleftX << aeGrid_.block_width_log2; - uint32_t startY = topleftY * aeGrid_.width << aeGrid_.block_width_log2; - uint32_t endX = (startX + (aeRegion.size().width >> aeGrid_.block_width_log2)) << aeGrid_.block_width_log2; - uint32_t i, j; - uint32_t count = 0; - + uint32_t regionWidth = round(aeGrid_.width / static_cast(kAgcStatsSizeX)); + uint32_t regionHeight = round(aeGrid_.height / static_cast(kAgcStatsSizeY)); uint32_t hist[knumHistogramBins] = { 0 }; - for (j = topleftY; - j < topleftY + (aeRegion.size().height >> aeGrid_.block_height_log2); - j++) { - for (i = startX + startY; i < endX + startY; i += kCellSize) { - /* - * The grid width (and maybe height) is not reliable. - * We observed a bit shift which makes the value 160 to be 32 in the stats grid. - * Use the one passed at init time. - */ - if (stats->awb_raw_buffer.meta_data[i + 4 + j * aeGrid_.width] == 0) { - uint8_t Gr = stats->awb_raw_buffer.meta_data[i + 0 + j * aeGrid_.width]; - uint8_t Gb = stats->awb_raw_buffer.meta_data[i + 3 + j * aeGrid_.width]; - hist[(Gr + Gb) / 2]++; - count++; + + LOG(IPU3Agc, Debug) << "[" << (int)aeGrid_.width << "x" << (int)aeGrid_.height << "] regions" + << " scaled to [" << regionWidth << "x" << regionHeight << "] AGC stats"; + + /* + * Generate a (kAgcStatsSizeX x kAgcStatsSizeY) array from the IPU3 grid which is + * (aeGrid_.width x aeGrid_.height). + */ + for (unsigned int j = 0; j < kAgcStatsSizeY * regionHeight; j++) { + for (unsigned int i = 0; i < kAgcStatsSizeX * regionWidth; i++) { + uint32_t cellPosition = j * aeGrid_.width + i; + uint32_t cellX = (cellPosition / regionWidth) % kAgcStatsSizeX; + uint32_t cellY = ((cellPosition / aeGrid_.width) / regionHeight) % kAgcStatsSizeY; + + uint32_t agcRegionPosition = kAgcStatsRegions[cellY * kAgcStatsSizeX + cellX]; + weights_[agcRegionPosition] = kCenteredWeights[agcRegionPosition]; + cellPosition *= sizeof(Ipu3AwbCell); + + /* Cast the initial IPU3 structure to simplify the reading */ + Ipu3AwbCell *currentCell = reinterpret_cast(const_cast(&stats->awb_raw_buffer.meta_data[cellPosition])); + if (currentCell->satRatio == 0) { + /* The cell is not saturated, use the current cell */ + agcStats_[agcRegionPosition].counted++; + uint32_t greenValue = currentCell->greenRedAvg + currentCell->greenBlueAvg; + hist[greenValue / 2]++; + agcStats_[agcRegionPosition].gSum += greenValue / 2; + agcStats_[agcRegionPosition].rSum += currentCell->redAvg; + agcStats_[agcRegionPosition].bSum += currentCell->blueAvg; } } } - /* Limit the gamma effect for now */ - gamma_ = 1.1; - /* Estimate the quantile mean of the top 2% of the histogram */ iqMean_ = Histogram(Span(hist)).interQuantileMean(0.98, 1.0); } +void IPU3Agc::clearStats() +{ + for (unsigned int i = 0; i < kNumAgcWeightedZones; i++) { + agcStats_[i].bSum = 0; + agcStats_[i].rSum = 0; + agcStats_[i].gSum = 0; + agcStats_[i].counted = 0; + agcStats_[i].uncounted = 0; + } + + awb_.blueGain = 1.0; + awb_.greenGain = 1.0; + awb_.redGain = 1.0; +} + void IPU3Agc::filterExposure() { - double speed = 0.2; + double speed = 0.08; if (prevExposure_ == 0s) { /* DG stands for digital gain.*/ prevExposure_ = currentExposure_; @@ -156,65 +162,131 @@ void IPU3Agc::filterExposure() * total exposure, as there might not be enough digital gain available * in the ISP to hide it (which will cause nasty oscillation). */ - double fastReduceThreshold = 0.4; + double fastReduceThreshold = 0.3; if (prevExposureNoDg_ < prevExposure_ * fastReduceThreshold) prevExposureNoDg_ = prevExposure_ * fastReduceThreshold; LOG(IPU3Agc, Debug) << "After filtering, total_exposure " << prevExposure_; } -void IPU3Agc::lockExposureGain(uint32_t &exposure, double &gain) +double IPU3Agc::computeInitialY(IspStatsRegion regions[], AwbStatus const &awb, + double weights[], double gain) { - updateControls_ = false; + /* Note how the calculation below means that equal weights give you + * "average" metering (i.e. all pixels equally important). */ + double redSum = 0, greenSum = 0, blueSum = 0, pixelSum = 0; + for (unsigned int i = 0; i < kNumAgcWeightedZones; i++) { + double counted = regions[i].counted; + double rSum = std::min(regions[i].rSum * gain, ((1 << kPipelineBits) - 1) * counted); + double gSum = std::min(regions[i].gSum * gain, ((1 << kPipelineBits) - 1) * counted); + double bSum = std::min(regions[i].bSum * gain, ((1 << kPipelineBits) - 1) * counted); + redSum += rSum * weights[i]; + greenSum += gSum * weights[i]; + blueSum += bSum * weights[i]; + pixelSum += counted * weights[i]; + } + if (pixelSum == 0.0) { + LOG(IPU3Agc, Warning) << "computeInitialY: pixel_sum is zero"; + return 0; + } + double Y_sum = redSum * awb.redGain * .299 + + greenSum * awb.greenGain * .587 + + blueSum * awb.blueGain * .114; - /* Algorithm initialization should wait for first valid frames */ - /* \todo - have a number of frames given by DelayedControls ? - * - implement a function for IIR */ - if ((frameCount_ < kInitialFrameMinAECount) || (frameCount_ - lastFrame_ < kFrameSkipCount)) - return; + return Y_sum / pixelSum / (1 << kPipelineBits); +} - /* Are we correctly exposed ? */ - if (std::abs(iqMean_ - kEvGainTarget * knumHistogramBins) <= 1) { - LOG(IPU3Agc, Debug) << "!!! Good exposure with iqMean = " << iqMean_; - converged_ = true; - } else { - double newGain = kEvGainTarget * knumHistogramBins / iqMean_; - - /* extracted from Rpi::Agc::computeTargetExposure */ - libcamera::utils::Duration currentShutter = exposure * lineDuration_; - currentExposureNoDg_ = currentShutter * gain; - LOG(IPU3Agc, Debug) << "Actual total exposure " << currentExposureNoDg_ - << " Shutter speed " << currentShutter - << " Gain " << gain; - currentExposure_ = currentExposureNoDg_ * newGain; - libcamera::utils::Duration maxTotalExposure = maxExposureTime_ * kMaxGain; - currentExposure_ = std::min(currentExposure_, maxTotalExposure); - LOG(IPU3Agc, Debug) << "Target total exposure " << currentExposure_; - - /* \todo: estimate if we need to desaturate */ - filterExposure(); - - libcamera::utils::Duration newExposure = 0.0s; - if (currentShutter < maxExposureTime_) { - exposure = std::clamp(static_cast(exposure * currentExposure_ / currentExposureNoDg_), kMinExposure, kMaxExposure); - newExposure = currentExposure_ / exposure; - gain = std::clamp(static_cast(gain * currentExposure_ / newExposure), kMinGain, kMaxGain); - updateControls_ = true; - } else if (currentShutter >= maxExposureTime_) { - gain = std::clamp(static_cast(gain * currentExposure_ / currentExposureNoDg_), kMinGain, kMaxGain); - newExposure = currentExposure_ / gain; - exposure = std::clamp(static_cast(exposure * currentExposure_ / newExposure), kMinExposure, kMaxExposure); - updateControls_ = true; +void IPU3Agc::computeTargetExposure(double gain) +{ + currentExposure_ = currentExposureNoDg_ * gain; + /* \todo: have a list of shutter speeds */ + Duration maxShutterSpeed = shutterConstraints_.back(); + Duration maxTotalExposure = maxShutterSpeed * gainConstraints_.back(); + + currentExposure_ = std::min(currentExposure_, maxTotalExposure); + LOG(IPU3Agc, Debug) << "Target total_exposure " << currentExposure_; +} + +void IPU3Agc::divideUpExposure() +{ + Duration exposureValue = prevExposure_; + Duration shutterTime; + double analogueGain; + shutterTime = shutterConstraints_[0]; + shutterTime = std::min(shutterTime, shutterConstraints_.back()); + analogueGain = gainConstraints_[0]; + + if (shutterTime * analogueGain < exposureValue) { + for (unsigned int stage = 1; + stage < gainConstraints_.size(); stage++) { + if (fixedShutter_ == 0s) { + Duration stageShutter = + std::min(shutterConstraints_[stage], shutterConstraints_.back()); + if (stageShutter * analogueGain >= + exposureValue) { + shutterTime = + exposureValue / analogueGain; + break; + } + shutterTime = stageShutter; + } + if (fixedAnalogueGain_ == 0.0) { + if (gainConstraints_[stage] * shutterTime >= exposureValue) { + analogueGain = exposureValue / shutterTime; + break; + } + analogueGain = gainConstraints_[stage]; + } } - LOG(IPU3Agc, Debug) << "Adjust exposure " << exposure * lineDuration_ << " and gain " << gain; } - lastFrame_ = frameCount_; + LOG(IPU3Agc, Debug) << "Divided up shutter and gain are " << shutterTime << " and " + << analogueGain; + + /* \todo: flickering avoidance ? */ + filteredShutter_ = shutterTime; + filteredAnalogueGain_ = analogueGain; +} + +void IPU3Agc::computeGain(double ¤tGain) +{ + currentGain = 1.0; + /* \todo: the target Y needs to be grabbed from a configuration */ + double targetY = 0.162; + for (int i = 0; i < 8; i++) { + double initialY = computeInitialY(agcStats_, awb_, weights_, currentGain); + double extra_gain = std::min(10.0, targetY / (initialY + .001)); + + currentGain *= extra_gain; + LOG(IPU3Agc, Debug) << "Initial Y " << initialY << " target " << targetY + << " gives gain " << currentGain; + if (extra_gain < 1.01) + break; + } + + double newGain = 128 / iqMean_; + LOG(IPU3Agc, Debug) << "gain: " << currentGain << " new gain: " << newGain; } -void IPU3Agc::process(const ipu3_uapi_stats_3a *stats, uint32_t &exposure, double &gain) +void IPU3Agc::process(const ipu3_uapi_stats_3a *stats, uint32_t &exposure, double &analogueGain) { - processBrightness(stats); - lockExposureGain(exposure, gain); + ASSERT(stats->stats_3a_status.awb_en); + clearStats(); + generateStats(stats); + currentShutter_ = exposure * lineDuration_; + /* \todo: the gain needs to be calculated based on sensor informations */ + currentAnalogueGain_ = analogueGain; + currentExposureNoDg_ = currentShutter_ * currentAnalogueGain_; + + double currentGain = 1; + computeGain(currentGain); + computeTargetExposure(currentGain); + filterExposure(); + divideUpExposure(); + + exposure = filteredShutter_ / lineDuration_; + analogueGain = filteredAnalogueGain_; + + updateControls_ = true; frameCount_++; } diff --git a/src/ipa/ipu3/ipu3_agc.h b/src/ipa/ipu3/ipu3_agc.h index ce43c534..f1b1157b 100644 --- a/src/ipa/ipu3/ipu3_agc.h +++ b/src/ipa/ipu3/ipu3_agc.h @@ -35,8 +35,8 @@ public: IPU3Agc(); ~IPU3Agc() = default; - void process(const ipu3_uapi_stats_3a *stats, uint32_t &exposure, double &gain); void initialise(struct ipu3_uapi_grid_config &bdsGrid, const IPAConfigInfo &configInfo); + void process(const ipu3_uapi_stats_3a *stats, uint32_t &exposure, double &analogueGain); bool converged() { return converged_; } bool updateControls() { return updateControls_; } /* \todo Use a metadata exchange between IPAs */ @@ -46,6 +46,17 @@ private: void processBrightness(const ipu3_uapi_stats_3a *stats); void filterExposure(); void lockExposureGain(uint32_t &exposure, double &gain); + void generateStats(const ipu3_uapi_stats_3a *stats); + void clearStats(); + void generateZones(std::vector &zones); + double computeInitialY(IspStatsRegion regions[], AwbStatus const &awb, double weights[], double gain); + void computeTargetExposure(double currentGain); + void divideUpExposure(); + void computeGain(double ¤tGain); + + AwbStatus awb_; + double weights_[kNumAgcWeightedZones]; + IspStatsRegion agcStats_[kNumAgcWeightedZones]; struct ipu3_uapi_grid_config aeGrid_; ControlInfoMap ctrls_; @@ -72,6 +83,16 @@ private: Duration prevExposureNoDg_; Duration currentExposure_; Duration currentExposureNoDg_; + + Duration currentShutter_; + std::vector shutterConstraints_; + Duration fixedShutter_; + Duration filteredShutter_; + + double currentAnalogueGain_; + std::vector gainConstraints_; + double fixedAnalogueGain_; + double filteredAnalogueGain_; }; } /* namespace ipa::ipu3 */