[libcamera-devel,v1,6/8] ipa: rkisp1: Introduce AGC

14655 diff mbox series

Now that we have IPAContext and Algorithm, we can implement a simple AGC
based on the IPU3 one. It is very similar, except that there is no
histogram used for an inter quantile mean. The RkISP1 is returning a 5x5
array (for V10) of luminance means. Estimating the relative luminance is
thus a simple mean of all the blocks already calculated by the ISP.

Signed-off-by: Jean-Michel Hautbois <jeanmichel.hautbois@ideasonboard.com>
---
 src/ipa/rkisp1/algorithms/agc.cpp     | 249 ++++++++++++++++++++++++++
 src/ipa/rkisp1/algorithms/agc.h       |  55 ++++++
 src/ipa/rkisp1/algorithms/meson.build |   1 +
 src/ipa/rkisp1/ipa_context.cpp        |  44 +++++
 src/ipa/rkisp1/ipa_context.h          |  15 ++
 src/ipa/rkisp1/rkisp1.cpp             |  72 ++++----
 6 files changed, 401 insertions(+), 35 deletions(-)
 create mode 100644 src/ipa/rkisp1/algorithms/agc.cpp
 create mode 100644 src/ipa/rkisp1/algorithms/agc.h

Quoting Jean-Michel Hautbois (2021-11-19 11:16:52)
> Now that we have IPAContext and Algorithm, we can implement a simple AGC
> based on the IPU3 one. It is very similar, except that there is no
> histogram used for an inter quantile mean. The RkISP1 is returning a 5x5
> array (for V10) of luminance means. Estimating the relative luminance is
> thus a simple mean of all the blocks already calculated by the ISP.
> 
> Signed-off-by: Jean-Michel Hautbois <jeanmichel.hautbois@ideasonboard.com>
> ---
>  src/ipa/rkisp1/algorithms/agc.cpp     | 249 ++++++++++++++++++++++++++
>  src/ipa/rkisp1/algorithms/agc.h       |  55 ++++++
>  src/ipa/rkisp1/algorithms/meson.build |   1 +
>  src/ipa/rkisp1/ipa_context.cpp        |  44 +++++
>  src/ipa/rkisp1/ipa_context.h          |  15 ++
>  src/ipa/rkisp1/rkisp1.cpp             |  72 ++++----
>  6 files changed, 401 insertions(+), 35 deletions(-)
>  create mode 100644 src/ipa/rkisp1/algorithms/agc.cpp
>  create mode 100644 src/ipa/rkisp1/algorithms/agc.h
> 
> diff --git a/src/ipa/rkisp1/algorithms/agc.cpp b/src/ipa/rkisp1/algorithms/agc.cpp
> new file mode 100644
> index 00000000..3d4a17ff
> --- /dev/null
> +++ b/src/ipa/rkisp1/algorithms/agc.cpp
> @@ -0,0 +1,249 @@
> +/* SPDX-License-Identifier: LGPL-2.1-or-later */
> +/*
> + * Copyright (C) 2021, Ideas On Board
> + *
> + * agc.cpp - AGC/AEC mean-based control algorithm
> + */
> +
> +#include "agc.h"
> +
> +#include <algorithm>
> +#include <chrono>
> +#include <cmath>
> +
> +#include <libcamera/base/log.h>
> +
> +#include <libcamera/ipa/core_ipa_interface.h>
> +
> +/**
> + * \file agc.h
> + */
> +
> +namespace libcamera {
> +
> +using namespace std::literals::chrono_literals;
> +
> +namespace ipa::rkisp1::algorithms {
> +
> +/**
> + * \class Agc
> + * \brief A mean-based auto-exposure algorithm
> + */
> +
> +LOG_DEFINE_CATEGORY(RkISP1Agc)
> +
> +/* Limits for analogue gain values */
> +static constexpr double kMinAnalogueGain = 1.0;
> +static constexpr double kMaxAnalogueGain = 8.0;
> +
> +/* \todo Honour the FrameDurationLimits control instead of hardcoding a limit */
> +static constexpr utils::Duration kMaxShutterSpeed = 60ms;
> +
> +/* Number of frames to wait before calculating stats on minimum exposure */
> +static constexpr uint32_t kNumStartupFrames = 10;
> +
> +/* Maximum luminance used for brightness normalization */
> +static constexpr double kMaxLuminance = 255.0;
> +
> +/*
> + * Normalized luma value target.
> + *
> + * It's a number that's chosen so that, when the camera points at a grey
> + * target, the resulting image brightness is considered right.
> + */
> +static constexpr double kNormalizedLumaTarget = 0.4;
> +
> +Agc::Agc()
> +       : frameCount_(0), lineDuration_(0s), minShutterSpeed_(0s),
> +         maxShutterSpeed_(0s), filteredExposure_(0s), currentExposure_(0s)
> +{
> +}
> +
> +/**
> + * \brief Configure the AGC given a configInfo
> + * \param[in] context The shared IPA context
> + * \param[in] configInfo The IPA configuration data
> + *
> + * \return 0
> + */
> +int Agc::configure(IPAContext &context, const IPACameraSensorInfo &configInfo)
> +{
> +       /* \todo use the IPAContext to provide the limits */
> +       lineDuration_ = configInfo.lineLength * 1.0s / configInfo.pixelRate;
> +
> +       minShutterSpeed_ = context.configuration.agc.minShutterSpeed;
> +       maxShutterSpeed_ = std::min(context.configuration.agc.maxShutterSpeed,
> +                                   kMaxShutterSpeed);
> +
> +       minAnalogueGain_ = std::max(context.configuration.agc.minAnalogueGain, kMinAnalogueGain);
> +       maxAnalogueGain_ = std::min(context.configuration.agc.maxAnalogueGain, kMaxAnalogueGain);
> +
> +       /* Configure the default exposure and gain. */
> +       context.frameContext.agc.gain = minAnalogueGain_;
> +       context.frameContext.agc.exposure = 10ms / lineDuration_;
> +
> +       return 0;
> +}
> +
> +/**
> + * \brief Apply a filter on the exposure value to limit the speed of changes
> + */
> +void Agc::filterExposure()
> +{
> +       double speed = 0.2;
> +
> +       /* Adapt instantly if we are in startup phase */
> +       if (frameCount_ < kNumStartupFrames)
> +               speed = 1.0;
> +
> +       if (filteredExposure_ == 0s) {
> +               /* DG stands for digital gain.*/
> +               filteredExposure_ = currentExposure_;
> +       } else {
> +               /*
> +                * If we are close to the desired result, go faster to avoid making
> +                * multiple micro-adjustments.
> +                * \todo Make this customisable?
> +                */
> +               if (filteredExposure_ < 1.2 * currentExposure_ &&
> +                   filteredExposure_ > 0.8 * currentExposure_)
> +                       speed = sqrt(speed);
> +
> +               filteredExposure_ = speed * currentExposure_ +
> +                                   filteredExposure_ * (1.0 - speed);
> +       }
> +
> +       LOG(RkISP1Agc, Debug) << "After filtering, total_exposure " << filteredExposure_;
> +}
> +
> +/**
> + * \brief Estimate the new exposure and gain values
> + * \param[inout] frameContext The shared IPA frame Context
> + * \param[in] currentYGain The gain calculated on the current brightness level
> + */
> +void Agc::computeExposure(IPAFrameContext &frameContext, double currentYGain)
> +{
> +       /* Get the effective exposure and gain applied on the sensor. */
> +       uint32_t exposure = frameContext.sensor.exposure;
> +       double analogueGain = frameContext.sensor.gain;
> +
> +       /* Consider within 1% of the target as correctly exposed */
> +       if (std::abs(currentYGain - 1.0) < 0.01)
> +               LOG(RkISP1Agc, Debug) << "We are well exposed (iqMean = "
> +                                     << currentYGain << ")";
> +
> +       /* extracted from Rpi::Agc::computeTargetExposure */
> +
> +       /* Calculate the shutter time in seconds */
> +       utils::Duration currentShutter = exposure * lineDuration_;
> +
> +       /*
> +        * Update the exposure value for the next computation using the values
> +        * of exposure and gain really used by the sensor.
> +        */
> +       utils::Duration effectiveExposureValue = currentShutter * analogueGain;
> +
> +       LOG(RkISP1Agc, Debug) << "Actual total exposure " << currentShutter * analogueGain
> +                             << " Shutter speed " << currentShutter
> +                             << " Gain " << analogueGain
> +                             << " Needed ev gain " << currentYGain;
> +
> +       /*
> +        * Calculate the current exposure value for the scene as the latest
> +        * exposure value applied multiplied by the new estimated gain.
> +        */
> +       currentExposure_ = effectiveExposureValue * currentYGain;
> +
> +       /* Clamp the exposure value to the min and max authorized */
> +       utils::Duration maxTotalExposure = maxShutterSpeed_ * maxAnalogueGain_;
> +       currentExposure_ = std::min(currentExposure_, maxTotalExposure);
> +       LOG(RkISP1Agc, Debug) << "Target total exposure " << currentExposure_
> +                             << ", maximum is " << maxTotalExposure;
> +
> +       /* \todo: estimate if we need to desaturate */
> +       filterExposure();
> +
> +       /* Divide the exposure value as new exposure and gain values */
> +       utils::Duration exposureValue = filteredExposure_;
> +       utils::Duration shutterTime;
> +
> +       /*
> +       * Push the shutter time up to the maximum first, and only then
> +       * increase the gain.
> +       */
> +       shutterTime = std::clamp<utils::Duration>(exposureValue / minAnalogueGain_,
> +                                                 minShutterSpeed_, maxShutterSpeed_);
> +       double stepGain = std::clamp(exposureValue / shutterTime,
> +                                    minAnalogueGain_, maxAnalogueGain_);
> +       LOG(RkISP1Agc, Debug) << "Divided up shutter and gain are "
> +                             << shutterTime << " and "
> +                             << stepGain;
> +
> +       /* Update the estimated exposure and gain. */
> +       frameContext.agc.exposure = shutterTime / lineDuration_;
> +       frameContext.agc.gain = stepGain;
> +}
> +
> +/**
> + * \brief Estimate the average brightness of the frame
> + * \param[in] ae The RkISP1 statistics and ISP results
> + * \param[in] currentYGain The gain calculated on the current brightness level
> + * \return The normalized luma
> + */
> +double Agc::computeInitialY(const rkisp1_cif_isp_ae_stat *ae, double currentYGain)
> +{
> +       double ySum = 0.0;
> +       unsigned int num = 0;
> +
> +       for (unsigned int aeCell = 0; aeCell < RKISP1_CIF_ISP_AE_MEAN_MAX_V10; aeCell++) {
> +               ySum += ae->exp_mean[aeCell] * currentYGain;

Do we need to clamp
	ae->exp_mean[aeCell] * currentYGain

to
	std::min(ae->exp_mean[aeCell] * currentYGain, kMaxLuminance);


Of course there is a pending discussion in Laurent's thread as to
whether kMaxLuminance should be kept or replaced by 255.0...

Personally I like that it's stated as a named variable ... ?

--
Kieran



> +               num++;
> +       }
> +
> +       /* \todo Weight with the AWB gains */
> +
> +       /* Return the normalized relative luminance. */
> +       return ySum / num / kMaxLuminance;
> +}
> +
> +/**
> + * \brief Process RkISP1 statistics, and run AGC operations
> + * \param[in] context The shared IPA context
> + * \param[in] stats The RKIsp1 statistics and ISP results
> + *
> + * Identify the current image brightness, and use that to estimate the optimal
> + * new exposure and gain for the scene.
> + */
> +void Agc::process(IPAContext &context, const rkisp1_stat_buffer *stats)
> +{
> +       const rkisp1_cif_isp_stat *params = &stats->params;
> +       ASSERT(stats->meas_type & RKISP1_CIF_ISP_STAT_AUTOEXP);
> +
> +       const rkisp1_cif_isp_ae_stat *ae = &params->ae;
> +
> +       double currentYGain = 1.0;
> +       double targetY = kNormalizedLumaTarget;
> +
> +       /*
> +        * Do this calculation a few times as brightness increase can be
> +        * non-linear when there are saturated regions.
> +        */
> +       for (int i = 0; i < 8; i++) {
> +               double initialY = computeInitialY(ae, currentYGain);
> +               double extra_gain = std::min(10.0, targetY / (initialY + .001));
> +
> +               currentYGain *= extra_gain;
> +               LOG(RkISP1Agc, Debug) << "Initial Y " << initialY
> +                                     << " target " << targetY
> +                                     << " gives gain " << currentYGain;
> +               if (extra_gain < 1.01)
> +                       break;
> +       }
> +
> +       computeExposure(context.frameContext, currentYGain);
> +       frameCount_++;
> +}
> +
> +} /* namespace ipa::rkisp1::algorithms */
> +
> +} /* namespace libcamera */
> diff --git a/src/ipa/rkisp1/algorithms/agc.h b/src/ipa/rkisp1/algorithms/agc.h
> new file mode 100644
> index 00000000..fbb8ea98
> --- /dev/null
> +++ b/src/ipa/rkisp1/algorithms/agc.h
> @@ -0,0 +1,55 @@
> +/* SPDX-License-Identifier: LGPL-2.1-or-later */
> +/*
> + * Copyright (C) 2021, Ideas On Board
> + *
> + * agc.h - RkISP1 AGC/AEC mean-based control algorithm
> + */
> +#ifndef __LIBCAMERA_RKISP1_ALGORITHMS_AGC_H__
> +#define __LIBCAMERA_RKISP1_ALGORITHMS_AGC_H__
> +
> +#include <linux/rkisp1-config.h>
> +
> +#include <libcamera/base/utils.h>
> +
> +#include <libcamera/geometry.h>
> +
> +#include "algorithm.h"
> +
> +namespace libcamera {
> +
> +struct IPACameraSensorInfo;
> +
> +namespace ipa::rkisp1::algorithms {
> +
> +class Agc : public Algorithm
> +{
> +public:
> +       Agc();
> +       ~Agc() = default;
> +
> +       int configure(IPAContext &context, const IPACameraSensorInfo &configInfo) override;
> +       void process(IPAContext &context, const rkisp1_stat_buffer *stats) override;
> +
> +private:
> +       void filterExposure();
> +       void computeExposure(IPAFrameContext &frameContext, double currentYGain);
> +       double computeInitialY(const rkisp1_cif_isp_ae_stat *ae, double currentYGain);
> +
> +       uint64_t frameCount_;
> +
> +       utils::Duration lineDuration_;
> +       utils::Duration minShutterSpeed_;
> +       utils::Duration maxShutterSpeed_;
> +
> +       double minAnalogueGain_;
> +       double maxAnalogueGain_;
> +
> +       utils::Duration filteredExposure_;
> +       utils::Duration currentExposure_;
> +};
> +
> +} /* namespace ipa::rkisp1::algorithms */
> +
> +} /* namespace libcamera */
> +
> +#endif /* __LIBCAMERA_RKISP1_ALGORITHMS_AGC_H__ */
> diff --git a/src/ipa/rkisp1/algorithms/meson.build b/src/ipa/rkisp1/algorithms/meson.build
> index d98f77e2..05195d99 100644
> --- a/src/ipa/rkisp1/algorithms/meson.build
> +++ b/src/ipa/rkisp1/algorithms/meson.build
> @@ -1,5 +1,6 @@
>  # SPDX-License-Identifier: CC0-1.0
>  
>  rkisp1_ipa_algorithms = files([
> +    'agc.cpp',
>      'algorithm.cpp',
>  ])
> diff --git a/src/ipa/rkisp1/ipa_context.cpp b/src/ipa/rkisp1/ipa_context.cpp
> index 819b2c73..16fc248f 100644
> --- a/src/ipa/rkisp1/ipa_context.cpp
> +++ b/src/ipa/rkisp1/ipa_context.cpp
> @@ -55,4 +55,48 @@ namespace libcamera::ipa::rkisp1 {
>   * are run. This needs to be turned into real per-frame data storage.
>   */
>  
> +/**
> + * \var IPASessionConfiguration::agc
> + * \brief AGC parameters configuration of the IPA
> + *
> + * \var IPASessionConfiguration::agc.minShutterSpeed
> + * \brief Minimum shutter speed supported with the configured sensor
> + *
> + * \var IPASessionConfiguration::agc.maxShutterSpeed
> + * \brief Maximum shutter speed supported with the configured sensor
> + *
> + * \var IPASessionConfiguration::agc.minAnalogueGain
> + * \brief Minimum analogue gain supported with the configured sensor
> + *
> + * \var IPASessionConfiguration::agc.maxAnalogueGain
> + * \brief Maximum analogue gain supported with the configured sensor
> + */
> +
> +/**
> + * \var IPAFrameContext::agc
> + * \brief Context for the Automatic Gain Control algorithm
> + *
> + * The exposure and gain determined are expected to be applied to the sensor
> + * at the earliest opportunity.
> + *
> + * \var IPAFrameContext::agc.exposure
> + * \brief Exposure time expressed as a number of lines
> + *
> + * \var IPAFrameContext::agc.gain
> + * \brief Analogue gain multiplier
> + *
> + * The gain should be adapted to the sensor specific gain code before applying.
> + */
> +
> +/**
> + * \var IPAFrameContext::sensor
> + * \brief Effective sensor values
> + *
> + * \var IPAFrameContext::sensor.exposure
> + * \brief Exposure time expressed as a number of lines
> + *
> + * \var IPAFrameContext::sensor.gain
> + * \brief Analogue gain multiplier
> + */
> +
>  } /* namespace libcamera::ipa::rkisp1 */
> diff --git a/src/ipa/rkisp1/ipa_context.h b/src/ipa/rkisp1/ipa_context.h
> index a1dc39fe..ca48ffc5 100644
> --- a/src/ipa/rkisp1/ipa_context.h
> +++ b/src/ipa/rkisp1/ipa_context.h
> @@ -19,9 +19,24 @@ namespace libcamera {
>  namespace ipa::rkisp1 {
>  
>  struct IPASessionConfiguration {
> +       struct {
> +               utils::Duration minShutterSpeed;
> +               utils::Duration maxShutterSpeed;
> +               double minAnalogueGain;
> +               double maxAnalogueGain;
> +       } agc;
>  };
>  
>  struct IPAFrameContext {
> +       struct {
> +               uint32_t exposure;
> +               double gain;
> +       } agc;
> +
> +       struct {
> +               uint32_t exposure;
> +               double gain;
> +       } sensor;
>  };
>  
>  struct IPAContext {
> diff --git a/src/ipa/rkisp1/rkisp1.cpp b/src/ipa/rkisp1/rkisp1.cpp
> index d2f4380a..1783f91f 100644
> --- a/src/ipa/rkisp1/rkisp1.cpp
> +++ b/src/ipa/rkisp1/rkisp1.cpp
> @@ -25,6 +25,7 @@
>  
>  #include <libcamera/internal/mapped_framebuffer.h>
>  
> +#include "algorithms/agc.h"
>  #include "algorithms/algorithm.h"
>  #include "libipa/camera_sensor_helper.h"
>  
> @@ -32,6 +33,8 @@ namespace libcamera {
>  
>  LOG_DEFINE_CATEGORY(IPARkISP1)
>  
> +using namespace std::literals::chrono_literals;
> +
>  namespace ipa::rkisp1 {
>  
>  class IPARkISP1 : public IPARkISP1Interface
> @@ -77,6 +80,8 @@ private:
>         unsigned int hwGammaOutMaxSamples_;
>         unsigned int hwHistogramWeightGridsSize_;
>  
> +       utils::Duration lineDuration_;
> +
>         /* Interface to the Camera Helper */
>         std::unique_ptr<CameraSensorHelper> camHelper_;
>  
> @@ -121,6 +126,9 @@ int IPARkISP1::init([[maybe_unused]] const IPASettings &settings,
>                 return -ENODEV;
>         }
>  
> +       /* Construct our Algorithms */
> +       algorithms_.push_back(std::make_unique<algorithms::Agc>());
> +
>         return 0;
>  }
>  
> @@ -172,9 +180,29 @@ int IPARkISP1::configure([[maybe_unused]] const IPACameraSensorInfo &info,
>                 << "Exposure: " << minExposure_ << "-" << maxExposure_
>                 << " Gain: " << minGain_ << "-" << maxGain_;
>  
> +       lineDuration_ = info.lineLength * 1.0s / info.pixelRate;
> +
>         /* Clean context at configuration */
>         context_ = {};
>  
> +       /*
> +        * When the AGC computes the new exposure values for a frame, it needs
> +        * to know the limits for shutter speed and analogue gain.
> +        * As it depends on the sensor, update it with the controls.
> +        *
> +        * \todo take VBLANK into account for maximum shutter speed
> +        */
> +       context_.configuration.agc.minShutterSpeed = minExposure_ * lineDuration_;
> +       context_.configuration.agc.maxShutterSpeed = maxExposure_ * lineDuration_;
> +       context_.configuration.agc.minAnalogueGain = camHelper_->gain(minGain_);
> +       context_.configuration.agc.maxAnalogueGain = camHelper_->gain(maxGain_);
> +
> +       for (auto const &algo : algorithms_) {
> +               int ret = algo->configure(context_, info);
> +               if (ret)
> +                       return ret;
> +       }
> +
>         return 0;
>  }
>  
> @@ -219,6 +247,9 @@ void IPARkISP1::processEvent(const RkISP1Event &event)
>                         reinterpret_cast<rkisp1_stat_buffer *>(
>                                 mappedBuffers_.at(bufferId).planes()[0].data());
>  
> +               context_.frameContext.sensor.exposure = event.sensorControls.get(V4L2_CID_EXPOSURE).get<int32_t>();
> +               context_.frameContext.sensor.gain = camHelper_->gain(event.sensorControls.get(V4L2_CID_ANALOGUE_GAIN).get<int32_t>());
> +
>                 updateStatistics(frame, stats);
>                 break;
>         }
> @@ -263,44 +294,12 @@ void IPARkISP1::queueRequest(unsigned int frame, rkisp1_params_cfg *params,
>  void IPARkISP1::updateStatistics(unsigned int frame,
>                                  const rkisp1_stat_buffer *stats)
>  {
> -       const rkisp1_cif_isp_stat *params = &stats->params;
>         unsigned int aeState = 0;
>  
> -       if (stats->meas_type & RKISP1_CIF_ISP_STAT_AUTOEXP) {
> -               const rkisp1_cif_isp_ae_stat *ae = &params->ae;
> -
> -               const unsigned int target = 60;
> +       for (auto const &algo : algorithms_)
> +               algo->process(context_, stats);
>  
> -               unsigned int value = 0;
> -               unsigned int num = 0;
> -               for (unsigned int i = 0; i < hwAeMeanMax_; i++) {
> -                       if (ae->exp_mean[i] <= 15)
> -                               continue;
> -
> -                       value += ae->exp_mean[i];
> -                       num++;
> -               }
> -               value /= num;
> -
> -               double factor = (double)target / value;
> -
> -               if (frame % 3 == 0) {
> -                       double exposure;
> -
> -                       exposure = factor * exposure_ * gain_ / minGain_;
> -                       exposure_ = std::clamp<uint64_t>((uint64_t)exposure,
> -                                                        minExposure_,
> -                                                        maxExposure_);
> -
> -                       exposure = exposure / exposure_ * minGain_;
> -                       gain_ = std::clamp<uint64_t>((uint64_t)exposure,
> -                                                    minGain_, maxGain_);
> -
> -                       setControls(frame + 1);
> -               }
> -
> -               aeState = fabs(factor - 1.0f) < 0.05f ? 2 : 1;
> -       }
> +       setControls(frame + 1);
>  
>         metadataReady(frame, aeState);
>  }
> @@ -310,6 +309,9 @@ void IPARkISP1::setControls(unsigned int frame)
>         RkISP1Action op;
>         op.op = ActionV4L2Set;
>  
> +       exposure_ = context_.frameContext.agc.exposure;
> +       gain_ = camHelper_->gainCode(context_.frameContext.agc.gain);
> +
>         ControlList ctrls(ctrls_);
>         ctrls.set(V4L2_CID_EXPOSURE, static_cast<int32_t>(exposure_));
>         ctrls.set(V4L2_CID_ANALOGUE_GAIN, static_cast<int32_t>(gain_));
> -- 
> 2.32.0
>

Hi Jean-Michel,

(CC'ing David)

Thank you for the patch.

On Fri, Nov 19, 2021 at 12:16:52PM +0100, Jean-Michel Hautbois wrote:
> Now that we have IPAContext and Algorithm, we can implement a simple AGC
> based on the IPU3 one. It is very similar, except that there is no
> histogram used for an inter quantile mean. The RkISP1 is returning a 5x5
> array (for V10) of luminance means. Estimating the relative luminance is
> thus a simple mean of all the blocks already calculated by the ISP.

We now have three AGC implementations, based on two different concepts:

- Interquantile mean (used in the IPU3 and Raspberry Pi IPAs): using a
  histogram (produced by the hardware for Raspberry Pi, recreated from
  the grid-based AWB averages for IPU3 as the ImgU has no histogram
  support), we calculate the interquantile mean of the 2% brightest
  pixels, and compute the gain needed to bring that value to a given
  target.

- Relative luminance estimation (used in all three IPAs): we estimate
  the value that the average relative luminance of the frame would reach
  if a given gain was applied (using the AWB averages that we sum and
  convert to luminance using the Bt.601 formula for Raspberry Pi and
  IPU3, and using the AEC Y averages computed on a 5x5 or 9x9 grid for
  RkISP1), and iterate to find the gain value that would bring the
  average relative luminance to a target.

Those two gain values are then combined in different ways:

- On IPU3, the maximum of the gains is used.

- On Raspberry Pi, a more complex mechanism (that I don't understand
  yet) is used.

- On RkISP1, only the relative luminance-based gain is used at the
  moment, but the histogram-based gain could be calculated and used in
  the future as the hardware supports histogram computation.

Using the maximum of the two gains seems very arbitraty to me. David,
would you be able to explain the way that the Raspberry Pi IPA combines
the two gains ? Is there one of the gains that should take precedence
over the other in some use cases or circumstances ? Are there other ways
to evaluate the needed gain that could be useful to try in the future ?

> Signed-off-by: Jean-Michel Hautbois <jeanmichel.hautbois@ideasonboard.com>
> ---
>  src/ipa/rkisp1/algorithms/agc.cpp     | 249 ++++++++++++++++++++++++++
>  src/ipa/rkisp1/algorithms/agc.h       |  55 ++++++
>  src/ipa/rkisp1/algorithms/meson.build |   1 +
>  src/ipa/rkisp1/ipa_context.cpp        |  44 +++++
>  src/ipa/rkisp1/ipa_context.h          |  15 ++
>  src/ipa/rkisp1/rkisp1.cpp             |  72 ++++----
>  6 files changed, 401 insertions(+), 35 deletions(-)
>  create mode 100644 src/ipa/rkisp1/algorithms/agc.cpp
>  create mode 100644 src/ipa/rkisp1/algorithms/agc.h
> 
> diff --git a/src/ipa/rkisp1/algorithms/agc.cpp b/src/ipa/rkisp1/algorithms/agc.cpp
> new file mode 100644
> index 00000000..3d4a17ff
> --- /dev/null
> +++ b/src/ipa/rkisp1/algorithms/agc.cpp
> @@ -0,0 +1,249 @@
> +/* SPDX-License-Identifier: LGPL-2.1-or-later */
> +/*
> + * Copyright (C) 2021, Ideas On Board
> + *
> + * agc.cpp - AGC/AEC mean-based control algorithm
> + */
> +
> +#include "agc.h"
> +
> +#include <algorithm>
> +#include <chrono>
> +#include <cmath>
> +
> +#include <libcamera/base/log.h>
> +
> +#include <libcamera/ipa/core_ipa_interface.h>
> +
> +/**
> + * \file agc.h
> + */
> +
> +namespace libcamera {
> +
> +using namespace std::literals::chrono_literals;
> +
> +namespace ipa::rkisp1::algorithms {
> +
> +/**
> + * \class Agc
> + * \brief A mean-based auto-exposure algorithm
> + */
> +
> +LOG_DEFINE_CATEGORY(RkISP1Agc)
> +
> +/* Limits for analogue gain values */
> +static constexpr double kMinAnalogueGain = 1.0;
> +static constexpr double kMaxAnalogueGain = 8.0;
> +
> +/* \todo Honour the FrameDurationLimits control instead of hardcoding a limit */
> +static constexpr utils::Duration kMaxShutterSpeed = 60ms;
> +
> +/* Number of frames to wait before calculating stats on minimum exposure */
> +static constexpr uint32_t kNumStartupFrames = 10;
> +
> +/* Maximum luminance used for brightness normalization */
> +static constexpr double kMaxLuminance = 255.0;

Same comment as for the IPU3, I'd drop this constant.

> +
> +/*
> + * Normalized luma value target.
> + *
> + * It's a number that's chosen so that, when the camera points at a grey
> + * target, the resulting image brightness is considered right.
> + */
> +static constexpr double kNormalizedLumaTarget = 0.4;
> +
> +Agc::Agc()
> +	: frameCount_(0), lineDuration_(0s), minShutterSpeed_(0s),
> +	  maxShutterSpeed_(0s), filteredExposure_(0s), currentExposure_(0s)
> +{
> +}
> +
> +/**
> + * \brief Configure the AGC given a configInfo
> + * \param[in] context The shared IPA context
> + * \param[in] configInfo The IPA configuration data
> + *
> + * \return 0
> + */
> +int Agc::configure(IPAContext &context, const IPACameraSensorInfo &configInfo)
> +{
> +	/* \todo use the IPAContext to provide the limits */
> +	lineDuration_ = configInfo.lineLength * 1.0s / configInfo.pixelRate;
> +
> +	minShutterSpeed_ = context.configuration.agc.minShutterSpeed;
> +	maxShutterSpeed_ = std::min(context.configuration.agc.maxShutterSpeed,
> +				    kMaxShutterSpeed);
> +
> +	minAnalogueGain_ = std::max(context.configuration.agc.minAnalogueGain, kMinAnalogueGain);
> +	maxAnalogueGain_ = std::min(context.configuration.agc.maxAnalogueGain, kMaxAnalogueGain);
> +
> +	/* Configure the default exposure and gain. */
> +	context.frameContext.agc.gain = minAnalogueGain_;
> +	context.frameContext.agc.exposure = 10ms / lineDuration_;
> +
> +	return 0;
> +}
> +
> +/**
> + * \brief Apply a filter on the exposure value to limit the speed of changes
> + */
> +void Agc::filterExposure()
> +{
> +	double speed = 0.2;
> +
> +	/* Adapt instantly if we are in startup phase */
> +	if (frameCount_ < kNumStartupFrames)
> +		speed = 1.0;
> +
> +	if (filteredExposure_ == 0s) {
> +		/* DG stands for digital gain.*/
> +		filteredExposure_ = currentExposure_;
> +	} else {
> +		/*
> +		 * If we are close to the desired result, go faster to avoid making
> +		 * multiple micro-adjustments.
> +		 * \todo Make this customisable?
> +		 */
> +		if (filteredExposure_ < 1.2 * currentExposure_ &&
> +		    filteredExposure_ > 0.8 * currentExposure_)
> +			speed = sqrt(speed);
> +
> +		filteredExposure_ = speed * currentExposure_ +
> +				    filteredExposure_ * (1.0 - speed);
> +	}
> +
> +	LOG(RkISP1Agc, Debug) << "After filtering, total_exposure " << filteredExposure_;
> +}
> +
> +/**
> + * \brief Estimate the new exposure and gain values
> + * \param[inout] frameContext The shared IPA frame Context
> + * \param[in] currentYGain The gain calculated on the current brightness level
> + */
> +void Agc::computeExposure(IPAFrameContext &frameContext, double currentYGain)
> +{
> +	/* Get the effective exposure and gain applied on the sensor. */
> +	uint32_t exposure = frameContext.sensor.exposure;
> +	double analogueGain = frameContext.sensor.gain;
> +
> +	/* Consider within 1% of the target as correctly exposed */
> +	if (std::abs(currentYGain - 1.0) < 0.01)
> +		LOG(RkISP1Agc, Debug) << "We are well exposed (iqMean = "
> +				      << currentYGain << ")";

We used to return early when well exposed before commit f8f07f9468c6
("ipa: ipu3: agc: Improve gain calculation"). Why has that been removed
?

> +
> +	/* extracted from Rpi::Agc::computeTargetExposure */
> +
> +	/* Calculate the shutter time in seconds */
> +	utils::Duration currentShutter = exposure * lineDuration_;
> +
> +	/*
> +	 * Update the exposure value for the next computation using the values
> +	 * of exposure and gain really used by the sensor.
> +	 */
> +	utils::Duration effectiveExposureValue = currentShutter * analogueGain;
> +
> +	LOG(RkISP1Agc, Debug) << "Actual total exposure " << currentShutter * analogueGain
> +			      << " Shutter speed " << currentShutter
> +			      << " Gain " << analogueGain
> +			      << " Needed ev gain " << currentYGain;
> +
> +	/*
> +	 * Calculate the current exposure value for the scene as the latest
> +	 * exposure value applied multiplied by the new estimated gain.
> +	 */
> +	currentExposure_ = effectiveExposureValue * currentYGain;
> +
> +	/* Clamp the exposure value to the min and max authorized */
> +	utils::Duration maxTotalExposure = maxShutterSpeed_ * maxAnalogueGain_;
> +	currentExposure_ = std::min(currentExposure_, maxTotalExposure);
> +	LOG(RkISP1Agc, Debug) << "Target total exposure " << currentExposure_
> +			      << ", maximum is " << maxTotalExposure;
> +
> +	/* \todo: estimate if we need to desaturate */
> +	filterExposure();
> +
> +	/* Divide the exposure value as new exposure and gain values */
> +	utils::Duration exposureValue = filteredExposure_;
> +	utils::Duration shutterTime;
> +
> +	/*
> +	* Push the shutter time up to the maximum first, and only then
> +	* increase the gain.
> +	*/
> +	shutterTime = std::clamp<utils::Duration>(exposureValue / minAnalogueGain_,
> +						  minShutterSpeed_, maxShutterSpeed_);
> +	double stepGain = std::clamp(exposureValue / shutterTime,
> +				     minAnalogueGain_, maxAnalogueGain_);
> +	LOG(RkISP1Agc, Debug) << "Divided up shutter and gain are "
> +			      << shutterTime << " and "
> +			      << stepGain;
> +
> +	/* Update the estimated exposure and gain. */
> +	frameContext.agc.exposure = shutterTime / lineDuration_;
> +	frameContext.agc.gain = stepGain;
> +}
> +
> +/**
> + * \brief Estimate the average brightness of the frame
> + * \param[in] ae The RkISP1 statistics and ISP results
> + * \param[in] currentYGain The gain calculated on the current brightness level
> + * \return The normalized luma
> + */
> +double Agc::computeInitialY(const rkisp1_cif_isp_ae_stat *ae, double currentYGain)
> +{
> +	double ySum = 0.0;
> +	unsigned int num = 0;
> +
> +	for (unsigned int aeCell = 0; aeCell < RKISP1_CIF_ISP_AE_MEAN_MAX_V10; aeCell++) {
> +		ySum += ae->exp_mean[aeCell] * currentYGain;

This needs to be saturated, or it will defeat the point of the iterative
algorithm. In general, please take into account the series I've posted
for the IPU3 in this patch.

> +		num++;

Drop this and device by RKISP1_CIF_ISP_AE_MEAN_MAX_V10 below.

> +	}
> +
> +	/* \todo Weight with the AWB gains */
> +
> +	/* Return the normalized relative luminance. */
> +	return ySum / num / kMaxLuminance;
> +}
> +
> +/**
> + * \brief Process RkISP1 statistics, and run AGC operations
> + * \param[in] context The shared IPA context
> + * \param[in] stats The RKIsp1 statistics and ISP results
> + *
> + * Identify the current image brightness, and use that to estimate the optimal
> + * new exposure and gain for the scene.
> + */
> +void Agc::process(IPAContext &context, const rkisp1_stat_buffer *stats)
> +{
> +	const rkisp1_cif_isp_stat *params = &stats->params;
> +	ASSERT(stats->meas_type & RKISP1_CIF_ISP_STAT_AUTOEXP);
> +
> +	const rkisp1_cif_isp_ae_stat *ae = &params->ae;
> +
> +	double currentYGain = 1.0;
> +	double targetY = kNormalizedLumaTarget;
> +
> +	/*
> +	 * Do this calculation a few times as brightness increase can be
> +	 * non-linear when there are saturated regions.
> +	 */
> +	for (int i = 0; i < 8; i++) {
> +		double initialY = computeInitialY(ae, currentYGain);
> +		double extra_gain = std::min(10.0, targetY / (initialY + .001));
> +
> +		currentYGain *= extra_gain;
> +		LOG(RkISP1Agc, Debug) << "Initial Y " << initialY
> +				      << " target " << targetY
> +				      << " gives gain " << currentYGain;
> +		if (extra_gain < 1.01)
> +			break;
> +	}
> +
> +	computeExposure(context.frameContext, currentYGain);
> +	frameCount_++;
> +}
> +
> +} /* namespace ipa::rkisp1::algorithms */
> +
> +} /* namespace libcamera */
> diff --git a/src/ipa/rkisp1/algorithms/agc.h b/src/ipa/rkisp1/algorithms/agc.h
> new file mode 100644
> index 00000000..fbb8ea98
> --- /dev/null
> +++ b/src/ipa/rkisp1/algorithms/agc.h
> @@ -0,0 +1,55 @@
> +/* SPDX-License-Identifier: LGPL-2.1-or-later */
> +/*
> + * Copyright (C) 2021, Ideas On Board
> + *
> + * agc.h - RkISP1 AGC/AEC mean-based control algorithm
> + */
> +#ifndef __LIBCAMERA_RKISP1_ALGORITHMS_AGC_H__
> +#define __LIBCAMERA_RKISP1_ALGORITHMS_AGC_H__
> +
> +#include <linux/rkisp1-config.h>
> +
> +#include <libcamera/base/utils.h>
> +
> +#include <libcamera/geometry.h>
> +
> +#include "algorithm.h"
> +
> +namespace libcamera {
> +
> +struct IPACameraSensorInfo;
> +
> +namespace ipa::rkisp1::algorithms {
> +
> +class Agc : public Algorithm
> +{
> +public:
> +	Agc();
> +	~Agc() = default;
> +
> +	int configure(IPAContext &context, const IPACameraSensorInfo &configInfo) override;
> +	void process(IPAContext &context, const rkisp1_stat_buffer *stats) override;
> +
> +private:
> +	void filterExposure();
> +	void computeExposure(IPAFrameContext &frameContext, double currentYGain);
> +	double computeInitialY(const rkisp1_cif_isp_ae_stat *ae, double currentYGain);
> +
> +	uint64_t frameCount_;
> +
> +	utils::Duration lineDuration_;
> +	utils::Duration minShutterSpeed_;
> +	utils::Duration maxShutterSpeed_;
> +
> +	double minAnalogueGain_;
> +	double maxAnalogueGain_;
> +
> +	utils::Duration filteredExposure_;
> +	utils::Duration currentExposure_;
> +};
> +
> +} /* namespace ipa::rkisp1::algorithms */
> +
> +} /* namespace libcamera */
> +
> +#endif /* __LIBCAMERA_RKISP1_ALGORITHMS_AGC_H__ */
> diff --git a/src/ipa/rkisp1/algorithms/meson.build b/src/ipa/rkisp1/algorithms/meson.build
> index d98f77e2..05195d99 100644
> --- a/src/ipa/rkisp1/algorithms/meson.build
> +++ b/src/ipa/rkisp1/algorithms/meson.build
> @@ -1,5 +1,6 @@
>  # SPDX-License-Identifier: CC0-1.0
>  
>  rkisp1_ipa_algorithms = files([
> +    'agc.cpp',
>      'algorithm.cpp',
>  ])
> diff --git a/src/ipa/rkisp1/ipa_context.cpp b/src/ipa/rkisp1/ipa_context.cpp
> index 819b2c73..16fc248f 100644
> --- a/src/ipa/rkisp1/ipa_context.cpp
> +++ b/src/ipa/rkisp1/ipa_context.cpp
> @@ -55,4 +55,48 @@ namespace libcamera::ipa::rkisp1 {
>   * are run. This needs to be turned into real per-frame data storage.
>   */
>  
> +/**
> + * \var IPASessionConfiguration::agc
> + * \brief AGC parameters configuration of the IPA
> + *
> + * \var IPASessionConfiguration::agc.minShutterSpeed
> + * \brief Minimum shutter speed supported with the configured sensor
> + *
> + * \var IPASessionConfiguration::agc.maxShutterSpeed
> + * \brief Maximum shutter speed supported with the configured sensor
> + *
> + * \var IPASessionConfiguration::agc.minAnalogueGain
> + * \brief Minimum analogue gain supported with the configured sensor
> + *
> + * \var IPASessionConfiguration::agc.maxAnalogueGain
> + * \brief Maximum analogue gain supported with the configured sensor
> + */
> +
> +/**
> + * \var IPAFrameContext::agc
> + * \brief Context for the Automatic Gain Control algorithm
> + *
> + * The exposure and gain determined are expected to be applied to the sensor
> + * at the earliest opportunity.
> + *
> + * \var IPAFrameContext::agc.exposure
> + * \brief Exposure time expressed as a number of lines
> + *
> + * \var IPAFrameContext::agc.gain
> + * \brief Analogue gain multiplier
> + *
> + * The gain should be adapted to the sensor specific gain code before applying.
> + */
> +
> +/**
> + * \var IPAFrameContext::sensor
> + * \brief Effective sensor values
> + *
> + * \var IPAFrameContext::sensor.exposure
> + * \brief Exposure time expressed as a number of lines
> + *
> + * \var IPAFrameContext::sensor.gain
> + * \brief Analogue gain multiplier
> + */
> +
>  } /* namespace libcamera::ipa::rkisp1 */
> diff --git a/src/ipa/rkisp1/ipa_context.h b/src/ipa/rkisp1/ipa_context.h
> index a1dc39fe..ca48ffc5 100644
> --- a/src/ipa/rkisp1/ipa_context.h
> +++ b/src/ipa/rkisp1/ipa_context.h
> @@ -19,9 +19,24 @@ namespace libcamera {
>  namespace ipa::rkisp1 {
>  
>  struct IPASessionConfiguration {
> +	struct {
> +		utils::Duration minShutterSpeed;
> +		utils::Duration maxShutterSpeed;
> +		double minAnalogueGain;
> +		double maxAnalogueGain;
> +	} agc;
>  };
>  
>  struct IPAFrameContext {
> +	struct {
> +		uint32_t exposure;
> +		double gain;
> +	} agc;
> +
> +	struct {
> +		uint32_t exposure;
> +		double gain;
> +	} sensor;
>  };
>  
>  struct IPAContext {
> diff --git a/src/ipa/rkisp1/rkisp1.cpp b/src/ipa/rkisp1/rkisp1.cpp
> index d2f4380a..1783f91f 100644
> --- a/src/ipa/rkisp1/rkisp1.cpp
> +++ b/src/ipa/rkisp1/rkisp1.cpp
> @@ -25,6 +25,7 @@
>  
>  #include <libcamera/internal/mapped_framebuffer.h>
>  
> +#include "algorithms/agc.h"
>  #include "algorithms/algorithm.h"
>  #include "libipa/camera_sensor_helper.h"
>  
> @@ -32,6 +33,8 @@ namespace libcamera {
>  
>  LOG_DEFINE_CATEGORY(IPARkISP1)
>  
> +using namespace std::literals::chrono_literals;
> +
>  namespace ipa::rkisp1 {
>  
>  class IPARkISP1 : public IPARkISP1Interface
> @@ -77,6 +80,8 @@ private:
>  	unsigned int hwGammaOutMaxSamples_;
>  	unsigned int hwHistogramWeightGridsSize_;
>  
> +	utils::Duration lineDuration_;
> +
>  	/* Interface to the Camera Helper */
>  	std::unique_ptr<CameraSensorHelper> camHelper_;
>  
> @@ -121,6 +126,9 @@ int IPARkISP1::init([[maybe_unused]] const IPASettings &settings,
>  		return -ENODEV;
>  	}
>  
> +	/* Construct our Algorithms */
> +	algorithms_.push_back(std::make_unique<algorithms::Agc>());
> +
>  	return 0;
>  }
>  
> @@ -172,9 +180,29 @@ int IPARkISP1::configure([[maybe_unused]] const IPACameraSensorInfo &info,
>  		<< "Exposure: " << minExposure_ << "-" << maxExposure_
>  		<< " Gain: " << minGain_ << "-" << maxGain_;
>  
> +	lineDuration_ = info.lineLength * 1.0s / info.pixelRate;
> +
>  	/* Clean context at configuration */
>  	context_ = {};
>  
> +	/*
> +	 * When the AGC computes the new exposure values for a frame, it needs
> +	 * to know the limits for shutter speed and analogue gain.
> +	 * As it depends on the sensor, update it with the controls.
> +	 *
> +	 * \todo take VBLANK into account for maximum shutter speed
> +	 */
> +	context_.configuration.agc.minShutterSpeed = minExposure_ * lineDuration_;
> +	context_.configuration.agc.maxShutterSpeed = maxExposure_ * lineDuration_;
> +	context_.configuration.agc.minAnalogueGain = camHelper_->gain(minGain_);
> +	context_.configuration.agc.maxAnalogueGain = camHelper_->gain(maxGain_);
> +
> +	for (auto const &algo : algorithms_) {
> +		int ret = algo->configure(context_, info);
> +		if (ret)
> +			return ret;
> +	}
> +
>  	return 0;
>  }
>  
> @@ -219,6 +247,9 @@ void IPARkISP1::processEvent(const RkISP1Event &event)
>  			reinterpret_cast<rkisp1_stat_buffer *>(
>  				mappedBuffers_.at(bufferId).planes()[0].data());
>  
> +		context_.frameContext.sensor.exposure = event.sensorControls.get(V4L2_CID_EXPOSURE).get<int32_t>();
> +		context_.frameContext.sensor.gain = camHelper_->gain(event.sensorControls.get(V4L2_CID_ANALOGUE_GAIN).get<int32_t>());
> +
>  		updateStatistics(frame, stats);
>  		break;
>  	}
> @@ -263,44 +294,12 @@ void IPARkISP1::queueRequest(unsigned int frame, rkisp1_params_cfg *params,
>  void IPARkISP1::updateStatistics(unsigned int frame,
>  				 const rkisp1_stat_buffer *stats)
>  {
> -	const rkisp1_cif_isp_stat *params = &stats->params;
>  	unsigned int aeState = 0;
>  
> -	if (stats->meas_type & RKISP1_CIF_ISP_STAT_AUTOEXP) {
> -		const rkisp1_cif_isp_ae_stat *ae = &params->ae;
> -
> -		const unsigned int target = 60;
> +	for (auto const &algo : algorithms_)
> +		algo->process(context_, stats);
>  
> -		unsigned int value = 0;
> -		unsigned int num = 0;
> -		for (unsigned int i = 0; i < hwAeMeanMax_; i++) {
> -			if (ae->exp_mean[i] <= 15)
> -				continue;
> -
> -			value += ae->exp_mean[i];
> -			num++;
> -		}
> -		value /= num;
> -
> -		double factor = (double)target / value;
> -
> -		if (frame % 3 == 0) {
> -			double exposure;
> -
> -			exposure = factor * exposure_ * gain_ / minGain_;
> -			exposure_ = std::clamp<uint64_t>((uint64_t)exposure,
> -							 minExposure_,
> -							 maxExposure_);
> -
> -			exposure = exposure / exposure_ * minGain_;
> -			gain_ = std::clamp<uint64_t>((uint64_t)exposure,
> -						     minGain_, maxGain_);
> -
> -			setControls(frame + 1);
> -		}
> -
> -		aeState = fabs(factor - 1.0f) < 0.05f ? 2 : 1;
> -	}
> +	setControls(frame + 1);
>  
>  	metadataReady(frame, aeState);
>  }
> @@ -310,6 +309,9 @@ void IPARkISP1::setControls(unsigned int frame)
>  	RkISP1Action op;
>  	op.op = ActionV4L2Set;
>  
> +	exposure_ = context_.frameContext.agc.exposure;
> +	gain_ = camHelper_->gainCode(context_.frameContext.agc.gain);
> +
>  	ControlList ctrls(ctrls_);
>  	ctrls.set(V4L2_CID_EXPOSURE, static_cast<int32_t>(exposure_));
>  	ctrls.set(V4L2_CID_ANALOGUE_GAIN, static_cast<int32_t>(gain_));