diff --git a/src/ipa/ipu3/ipu3.cpp b/src/ipa/ipu3/ipu3.cpp index ab052a8a..48dc28c1 100644 --- a/src/ipa/ipu3/ipu3.cpp +++ b/src/ipa/ipu3/ipu3.cpp @@ -21,10 +21,11 @@ #include "libcamera/internal/buffer.h" #include "libcamera/internal/log.h" +#include "ipu3_agc.h" #include "ipu3_awb.h" static constexpr uint32_t kMaxCellWidthPerSet = 160; -static constexpr uint32_t kMaxCellHeightPerSet = 80; +static constexpr uint32_t kMaxCellHeightPerSet = 56; namespace libcamera { @@ -70,6 +71,8 @@ private: /* Interface to the AWB algorithm */ std::unique_ptr awbAlgo_; + /* Interface to the AEC/AGC algorithm */ + std::unique_ptr agcAlgo_; /* Local parameter storage */ struct ipu3_uapi_params params_; @@ -168,6 +171,9 @@ void IPAIPU3::configure(const std::map &entityControls awbAlgo_ = std::make_unique(); awbAlgo_->initialise(params_, bdsOutputSize, bdsGrid_); + + agcAlgo_ = std::make_unique(); + agcAlgo_->initialise(bdsGrid_); } void IPAIPU3::mapBuffers(const std::vector &buffers) @@ -239,8 +245,8 @@ void IPAIPU3::processControls([[maybe_unused]] unsigned int frame, void IPAIPU3::fillParams(unsigned int frame, ipu3_uapi_params *params) { - /* Pass a default gamma of 1.0 (default linear correction) */ - awbAlgo_->updateWbParameters(params_, 1.0); + if (agcAlgo_->updateControls()) + awbAlgo_->updateWbParameters(params_, agcAlgo_->gamma()); *params = params_; @@ -255,8 +261,12 @@ void IPAIPU3::parseStatistics(unsigned int frame, { ControlList ctrls(controls::controls); + agcAlgo_->process(stats, exposure_, gain_); awbAlgo_->calculateWBGains(stats); + if (agcAlgo_->updateControls()) + setControls(frame); + ipa::ipu3::IPU3Action op; op.op = ipa::ipu3::ActionMetadataReady; op.controls = ctrls; diff --git a/src/ipa/ipu3/ipu3_agc.cpp b/src/ipa/ipu3/ipu3_agc.cpp new file mode 100644 index 00000000..04ab55a1 --- /dev/null +++ b/src/ipa/ipu3/ipu3_agc.cpp @@ -0,0 +1,206 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2021, Ideas On Board + * + * ipu3_agc.cpp - AGC/AEC control algorithm + */ + +#include "ipu3_agc.h" + +#include +#include +#include + +#include "libcamera/internal/log.h" + +#include "libipa/histogram.h" + +namespace libcamera { + +namespace ipa { + +LOG_DEFINE_CATEGORY(IPU3Agc) + +/* Number of frames to wait before calculating stats on minimum exposure */ +static constexpr uint32_t kInitialFrameMinAECount = 4; +/* Number of frames to wait between new gain/exposure estimations */ +static constexpr uint32_t kFrameSkipCount = 6; + +/* Maximum ISO value for analogue gain */ +static constexpr uint32_t kMinISO = 100; +static constexpr uint32_t kMaxISO = 1500; + +/* Maximum analogue gain value + * \todo grab it from a camera helper */ +static constexpr uint32_t kMinGain = kMinISO / 100; +static constexpr uint32_t kMaxGain = kMaxISO / 100; + +/* \todo use calculated value based on sensor */ +static constexpr uint32_t kMinExposure = 1; +static constexpr uint32_t kMaxExposure = 1976; + +/* \todo those should be get from pipeline handler ! */ +/* line duration in microseconds */ +static constexpr double kLineDuration = 16.8; +static constexpr double kMaxExposureTime = kMaxExposure * kLineDuration; + +/* Histogram constants */ +static constexpr uint32_t knumHistogramBins = 256; +static constexpr double kEvGainTarget = 0.5; + +IPU3Agc::IPU3Agc() + : frameCount_(0), lastFrame_(0), converged_(false), + updateControls_(false), iqMean_(0.0), gamma_(1.0), + prevExposure_(0.0), prevExposureNoDg_(0.0), + currentExposure_(0.0), currentExposureNoDg_(0.0) +{ +} + +void IPU3Agc::initialise(struct ipu3_uapi_grid_config &bdsGrid) +{ + aeGrid_ = bdsGrid; +} + +void IPU3Agc::processBrightness(const ipu3_uapi_stats_3a *stats) +{ + const struct ipu3_uapi_grid_config statsAeGrid = stats->stats_4a_config.awb_config.grid; + Rectangle aeRegion = { statsAeGrid.x_start, + statsAeGrid.y_start, + static_cast(statsAeGrid.x_end - statsAeGrid.x_start) + 1, + static_cast(statsAeGrid.y_end - statsAeGrid.y_start) + 1 }; + Point topleft = aeRegion.topLeft(); + int topleftX = topleft.x >> aeGrid_.block_width_log2; + int topleftY = topleft.y >> aeGrid_.block_height_log2; + + uint32_t startY = topleftY * aeGrid_.width << aeGrid_.block_width_log2; + uint32_t startX = topleftX << aeGrid_.block_width_log2; + uint32_t endX = (startX + (aeRegion.size().width >> aeGrid_.block_width_log2)) << aeGrid_.block_width_log2; + uint32_t i, j; + uint32_t count = 0; + + uint32_t hist[knumHistogramBins] = { 0 }; + for (j = topleftY; + j < topleftY + (aeRegion.size().height >> aeGrid_.block_height_log2); + j++) { + for (i = startX + startY; i < endX + startY; i += 8) { + /** + * The grid width (and maybe height) is not reliable. + * We observed a bit shift which makes the value 160 to be 32 in the stats grid. + * Use the one passed at init time. + */ + if (stats->awb_raw_buffer.meta_data[i + 4 + j * aeGrid_.width] == 0) { + uint8_t Gr = stats->awb_raw_buffer.meta_data[i + j * aeGrid_.width]; + uint8_t Gb = stats->awb_raw_buffer.meta_data[i + 3 + j * aeGrid_.width]; + hist[(Gr + Gb) / 2]++; + count++; + } + } + } + + /* Limit the gamma effect for now */ + gamma_ = 1.1; + + /* Estimate the quantile mean of the top 2% of the histogram */ + iqMean_ = Histogram(Span(hist)).interQuantileMean(0.98, 1.0); +} + +void IPU3Agc::filterExposure(bool desaturate) +{ + double speed = 0.2; + if (prevExposure_ == 0.0) { + /* DG stands for digital gain.*/ + prevExposure_ = currentExposure_; + prevExposureNoDg_ = currentExposureNoDg_; + } else { + /** + * If we are close to the desired result, go faster to avoid making + * multiple micro-adjustments. + * \ todo: Make this customisable? + */ + if (prevExposure_ < 1.2 * currentExposure_ && + prevExposure_ > 0.8 * currentExposure_) + speed = sqrt(speed); + prevExposure_ = speed * currentExposure_ + + prevExposure_ * (1.0 - speed); + /** + * When desaturing, take a big jump down in exposure_no_dg, + * which we'll hide with digital gain. + */ + if (desaturate) + prevExposureNoDg_ = + currentExposureNoDg_; + else + prevExposureNoDg_ = + speed * currentExposureNoDg_ + + prevExposureNoDg_ * (1.0 - speed); + } + /** + * We can't let the no_dg exposure deviate too far below the + * total exposure, as there might not be enough digital gain available + * in the ISP to hide it (which will cause nasty oscillation). + */ + double fastReduceThreshold = 0.4; + if (prevExposureNoDg_ < + prevExposure_ * fastReduceThreshold) + prevExposureNoDg_ = prevExposure_ * fastReduceThreshold; + LOG(IPU3Agc, Debug) << "After filtering, total_exposure " << prevExposure_; +} + +void IPU3Agc::lockExposureGain(uint32_t &exposure, uint32_t &gain) +{ + updateControls_ = false; + + /* Algorithm initialization should wait for first valid frames */ + /* \todo - have a number of frames given by DelayedControls ? + * - implement a function for IIR */ + if ((frameCount_ < kInitialFrameMinAECount) || (frameCount_ - lastFrame_ < kFrameSkipCount)) + return; + + /* Are we correctly exposed ? */ + if (std::abs(iqMean_ - kEvGainTarget * knumHistogramBins) <= 1) { + LOG(IPU3Agc, Debug) << "!!! Good exposure with iqMean = " << iqMean_; + converged_ = true; + } else { + double newGain = kEvGainTarget * knumHistogramBins / iqMean_; + + /* extracted from Rpi::Agc::computeTargetExposure */ + double currentShutter = exposure * kLineDuration; + currentExposureNoDg_ = currentShutter * gain; + LOG(IPU3Agc, Debug) << "Actual total exposure " << currentExposureNoDg_ + << " Shutter speed " << currentShutter + << " Gain " << gain; + currentExposure_ = currentExposureNoDg_ * newGain; + double maxTotalExposure = kMaxExposureTime * kMaxGain; + currentExposure_ = std::min(currentExposure_, maxTotalExposure); + LOG(IPU3Agc, Debug) << "Target total exposure " << currentExposure_; + + /* \todo: estimate if we need to desaturate */ + filterExposure(false); + + double newExposure = 0.0; + if (currentShutter < kMaxExposureTime) { + exposure = std::clamp(static_cast(exposure * currentExposure_ / currentExposureNoDg_), kMinExposure, kMaxExposure); + newExposure = currentExposure_ / exposure; + gain = std::clamp(static_cast(gain * currentExposure_ / newExposure), kMinGain, kMaxGain); + updateControls_ = true; + } else if (currentShutter >= kMaxExposureTime) { + gain = std::clamp(static_cast(gain * currentExposure_ / currentExposureNoDg_), kMinGain, kMaxGain); + newExposure = currentExposure_ / gain; + exposure = std::clamp(static_cast(exposure * currentExposure_ / newExposure), kMinExposure, kMaxExposure); + updateControls_ = true; + } + LOG(IPU3Agc, Debug) << "Adjust exposure " << exposure * kLineDuration << " and gain " << gain; + } + lastFrame_ = frameCount_; +} + +void IPU3Agc::process(const ipu3_uapi_stats_3a *stats, uint32_t &exposure, uint32_t &gain) +{ + processBrightness(stats); + lockExposureGain(exposure, gain); + frameCount_++; +} + +} /* namespace ipa */ + +} /* namespace libcamera */ diff --git a/src/ipa/ipu3/ipu3_agc.h b/src/ipa/ipu3/ipu3_agc.h new file mode 100644 index 00000000..f59bc420 --- /dev/null +++ b/src/ipa/ipu3/ipu3_agc.h @@ -0,0 +1,62 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2021, Ideas On Board + * + * ipu3_agc.h - IPU3 AGC/AEC control algorithm + */ +#ifndef __LIBCAMERA_IPU3_AGC_H__ +#define __LIBCAMERA_IPU3_AGC_H__ + +#include +#include + +#include + +#include + +#include "libipa/algorithm.h" + +namespace libcamera { + +namespace ipa { + +class IPU3Agc : public Algorithm +{ +public: + IPU3Agc(); + ~IPU3Agc() = default; + + void initialise(struct ipu3_uapi_grid_config &bdsGrid); + void process(const ipu3_uapi_stats_3a *stats, uint32_t &exposure, uint32_t &gain); + bool converged() { return converged_; } + bool updateControls() { return updateControls_; } + /* \todo Use a metadata exchange between IPAs */ + double gamma() { return gamma_; } + +private: + void processBrightness(const ipu3_uapi_stats_3a *stats); + void filterExposure(bool desaturate); + void lockExposureGain(uint32_t &exposure, uint32_t &gain); + + struct ipu3_uapi_grid_config aeGrid_; + + uint64_t frameCount_; + uint64_t lastFrame_; + + bool converged_; + bool updateControls_; + + double iqMean_; + double gamma_; + + double prevExposure_; + double prevExposureNoDg_; + double currentExposure_; + double currentExposureNoDg_; +}; + +} /* namespace ipa */ + +} /* namespace libcamera */ + +#endif /* __LIBCAMERA_IPU3_AGC_H__ */ diff --git a/src/ipa/ipu3/meson.build b/src/ipa/ipu3/meson.build index 1040698e..0d843846 100644 --- a/src/ipa/ipu3/meson.build +++ b/src/ipa/ipu3/meson.build @@ -4,6 +4,7 @@ ipa_name = 'ipa_ipu3' ipu3_ipa_sources = files([ 'ipu3.cpp', + 'ipu3_agc.cpp', 'ipu3_awb.cpp', ])