From patchwork Fri Nov 19 11:16:52 2021 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Jean-Michel Hautbois X-Patchwork-Id: 14655 Return-Path: X-Original-To: parsemail@patchwork.libcamera.org Delivered-To: parsemail@patchwork.libcamera.org Received: from lancelot.ideasonboard.com (lancelot.ideasonboard.com [92.243.16.209]) by patchwork.libcamera.org (Postfix) with ESMTPS id D0BEEC3251 for ; Fri, 19 Nov 2021 11:17:07 +0000 (UTC) Received: from lancelot.ideasonboard.com (localhost [IPv6:::1]) by lancelot.ideasonboard.com (Postfix) with ESMTP id 8298860371; Fri, 19 Nov 2021 12:17:07 +0100 (CET) Authentication-Results: lancelot.ideasonboard.com; dkim=fail reason="signature verification failed" (1024-bit key; unprotected) header.d=ideasonboard.com header.i=@ideasonboard.com header.b="GBWQXqXx"; dkim-atps=neutral Received: from perceval.ideasonboard.com (perceval.ideasonboard.com [213.167.242.64]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id 628616037A for ; Fri, 19 Nov 2021 12:17:00 +0100 (CET) Received: from tatooine.ideasonboard.com (unknown [IPv6:2a01:e0a:169:7140:db30:8e54:a96:9838]) by perceval.ideasonboard.com (Postfix) with ESMTPSA id 2901C1C19; Fri, 19 Nov 2021 12:17:00 +0100 (CET) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=ideasonboard.com; s=mail; t=1637320620; bh=ZWG9gX474vUUOgKnjwrRR35c6icXmt+2acx/eE8WtAo=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=GBWQXqXxXHIsqw9sCooHnbYVOF/ryr759JPvNkdbo7RIjBQ3Y5IVHVWle2CwYcwZS k6Qbu5KVGvtCJXvQC+DfxabwdeP3eSnGOgRR8HOIMI8NuhNCcHzAHiIqkDkCyO9Cnq 5Z6rYTczfxMxuxfuvA0M5B5osJsohwjQNoDW7Bm4= From: Jean-Michel Hautbois To: libcamera-devel@lists.libcamera.org Date: Fri, 19 Nov 2021 12:16:52 +0100 Message-Id: <20211119111654.68445-7-jeanmichel.hautbois@ideasonboard.com> X-Mailer: git-send-email 2.32.0 In-Reply-To: <20211119111654.68445-1-jeanmichel.hautbois@ideasonboard.com> References: <20211119111654.68445-1-jeanmichel.hautbois@ideasonboard.com> MIME-Version: 1.0 Subject: [libcamera-devel] [PATCH v1 6/8] ipa: rkisp1: Introduce AGC X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: libcamera-devel-bounces@lists.libcamera.org Sender: "libcamera-devel" Now that we have IPAContext and Algorithm, we can implement a simple AGC based on the IPU3 one. It is very similar, except that there is no histogram used for an inter quantile mean. The RkISP1 is returning a 5x5 array (for V10) of luminance means. Estimating the relative luminance is thus a simple mean of all the blocks already calculated by the ISP. Signed-off-by: Jean-Michel Hautbois --- src/ipa/rkisp1/algorithms/agc.cpp | 249 ++++++++++++++++++++++++++ src/ipa/rkisp1/algorithms/agc.h | 55 ++++++ src/ipa/rkisp1/algorithms/meson.build | 1 + src/ipa/rkisp1/ipa_context.cpp | 44 +++++ src/ipa/rkisp1/ipa_context.h | 15 ++ src/ipa/rkisp1/rkisp1.cpp | 72 ++++---- 6 files changed, 401 insertions(+), 35 deletions(-) create mode 100644 src/ipa/rkisp1/algorithms/agc.cpp create mode 100644 src/ipa/rkisp1/algorithms/agc.h diff --git a/src/ipa/rkisp1/algorithms/agc.cpp b/src/ipa/rkisp1/algorithms/agc.cpp new file mode 100644 index 00000000..3d4a17ff --- /dev/null +++ b/src/ipa/rkisp1/algorithms/agc.cpp @@ -0,0 +1,249 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2021, Ideas On Board + * + * agc.cpp - AGC/AEC mean-based control algorithm + */ + +#include "agc.h" + +#include +#include +#include + +#include + +#include + +/** + * \file agc.h + */ + +namespace libcamera { + +using namespace std::literals::chrono_literals; + +namespace ipa::rkisp1::algorithms { + +/** + * \class Agc + * \brief A mean-based auto-exposure algorithm + */ + +LOG_DEFINE_CATEGORY(RkISP1Agc) + +/* Limits for analogue gain values */ +static constexpr double kMinAnalogueGain = 1.0; +static constexpr double kMaxAnalogueGain = 8.0; + +/* \todo Honour the FrameDurationLimits control instead of hardcoding a limit */ +static constexpr utils::Duration kMaxShutterSpeed = 60ms; + +/* Number of frames to wait before calculating stats on minimum exposure */ +static constexpr uint32_t kNumStartupFrames = 10; + +/* Maximum luminance used for brightness normalization */ +static constexpr double kMaxLuminance = 255.0; + +/* + * Normalized luma value target. + * + * It's a number that's chosen so that, when the camera points at a grey + * target, the resulting image brightness is considered right. + */ +static constexpr double kNormalizedLumaTarget = 0.4; + +Agc::Agc() + : frameCount_(0), lineDuration_(0s), minShutterSpeed_(0s), + maxShutterSpeed_(0s), filteredExposure_(0s), currentExposure_(0s) +{ +} + +/** + * \brief Configure the AGC given a configInfo + * \param[in] context The shared IPA context + * \param[in] configInfo The IPA configuration data + * + * \return 0 + */ +int Agc::configure(IPAContext &context, const IPACameraSensorInfo &configInfo) +{ + /* \todo use the IPAContext to provide the limits */ + lineDuration_ = configInfo.lineLength * 1.0s / configInfo.pixelRate; + + minShutterSpeed_ = context.configuration.agc.minShutterSpeed; + maxShutterSpeed_ = std::min(context.configuration.agc.maxShutterSpeed, + kMaxShutterSpeed); + + minAnalogueGain_ = std::max(context.configuration.agc.minAnalogueGain, kMinAnalogueGain); + maxAnalogueGain_ = std::min(context.configuration.agc.maxAnalogueGain, kMaxAnalogueGain); + + /* Configure the default exposure and gain. */ + context.frameContext.agc.gain = minAnalogueGain_; + context.frameContext.agc.exposure = 10ms / lineDuration_; + + return 0; +} + +/** + * \brief Apply a filter on the exposure value to limit the speed of changes + */ +void Agc::filterExposure() +{ + double speed = 0.2; + + /* Adapt instantly if we are in startup phase */ + if (frameCount_ < kNumStartupFrames) + speed = 1.0; + + if (filteredExposure_ == 0s) { + /* DG stands for digital gain.*/ + filteredExposure_ = currentExposure_; + } else { + /* + * If we are close to the desired result, go faster to avoid making + * multiple micro-adjustments. + * \todo Make this customisable? + */ + if (filteredExposure_ < 1.2 * currentExposure_ && + filteredExposure_ > 0.8 * currentExposure_) + speed = sqrt(speed); + + filteredExposure_ = speed * currentExposure_ + + filteredExposure_ * (1.0 - speed); + } + + LOG(RkISP1Agc, Debug) << "After filtering, total_exposure " << filteredExposure_; +} + +/** + * \brief Estimate the new exposure and gain values + * \param[inout] frameContext The shared IPA frame Context + * \param[in] currentYGain The gain calculated on the current brightness level + */ +void Agc::computeExposure(IPAFrameContext &frameContext, double currentYGain) +{ + /* Get the effective exposure and gain applied on the sensor. */ + uint32_t exposure = frameContext.sensor.exposure; + double analogueGain = frameContext.sensor.gain; + + /* Consider within 1% of the target as correctly exposed */ + if (std::abs(currentYGain - 1.0) < 0.01) + LOG(RkISP1Agc, Debug) << "We are well exposed (iqMean = " + << currentYGain << ")"; + + /* extracted from Rpi::Agc::computeTargetExposure */ + + /* Calculate the shutter time in seconds */ + utils::Duration currentShutter = exposure * lineDuration_; + + /* + * Update the exposure value for the next computation using the values + * of exposure and gain really used by the sensor. + */ + utils::Duration effectiveExposureValue = currentShutter * analogueGain; + + LOG(RkISP1Agc, Debug) << "Actual total exposure " << currentShutter * analogueGain + << " Shutter speed " << currentShutter + << " Gain " << analogueGain + << " Needed ev gain " << currentYGain; + + /* + * Calculate the current exposure value for the scene as the latest + * exposure value applied multiplied by the new estimated gain. + */ + currentExposure_ = effectiveExposureValue * currentYGain; + + /* Clamp the exposure value to the min and max authorized */ + utils::Duration maxTotalExposure = maxShutterSpeed_ * maxAnalogueGain_; + currentExposure_ = std::min(currentExposure_, maxTotalExposure); + LOG(RkISP1Agc, Debug) << "Target total exposure " << currentExposure_ + << ", maximum is " << maxTotalExposure; + + /* \todo: estimate if we need to desaturate */ + filterExposure(); + + /* Divide the exposure value as new exposure and gain values */ + utils::Duration exposureValue = filteredExposure_; + utils::Duration shutterTime; + + /* + * Push the shutter time up to the maximum first, and only then + * increase the gain. + */ + shutterTime = std::clamp(exposureValue / minAnalogueGain_, + minShutterSpeed_, maxShutterSpeed_); + double stepGain = std::clamp(exposureValue / shutterTime, + minAnalogueGain_, maxAnalogueGain_); + LOG(RkISP1Agc, Debug) << "Divided up shutter and gain are " + << shutterTime << " and " + << stepGain; + + /* Update the estimated exposure and gain. */ + frameContext.agc.exposure = shutterTime / lineDuration_; + frameContext.agc.gain = stepGain; +} + +/** + * \brief Estimate the average brightness of the frame + * \param[in] ae The RkISP1 statistics and ISP results + * \param[in] currentYGain The gain calculated on the current brightness level + * \return The normalized luma + */ +double Agc::computeInitialY(const rkisp1_cif_isp_ae_stat *ae, double currentYGain) +{ + double ySum = 0.0; + unsigned int num = 0; + + for (unsigned int aeCell = 0; aeCell < RKISP1_CIF_ISP_AE_MEAN_MAX_V10; aeCell++) { + ySum += ae->exp_mean[aeCell] * currentYGain; + num++; + } + + /* \todo Weight with the AWB gains */ + + /* Return the normalized relative luminance. */ + return ySum / num / kMaxLuminance; +} + +/** + * \brief Process RkISP1 statistics, and run AGC operations + * \param[in] context The shared IPA context + * \param[in] stats The RKIsp1 statistics and ISP results + * + * Identify the current image brightness, and use that to estimate the optimal + * new exposure and gain for the scene. + */ +void Agc::process(IPAContext &context, const rkisp1_stat_buffer *stats) +{ + const rkisp1_cif_isp_stat *params = &stats->params; + ASSERT(stats->meas_type & RKISP1_CIF_ISP_STAT_AUTOEXP); + + const rkisp1_cif_isp_ae_stat *ae = ¶ms->ae; + + double currentYGain = 1.0; + double targetY = kNormalizedLumaTarget; + + /* + * Do this calculation a few times as brightness increase can be + * non-linear when there are saturated regions. + */ + for (int i = 0; i < 8; i++) { + double initialY = computeInitialY(ae, currentYGain); + double extra_gain = std::min(10.0, targetY / (initialY + .001)); + + currentYGain *= extra_gain; + LOG(RkISP1Agc, Debug) << "Initial Y " << initialY + << " target " << targetY + << " gives gain " << currentYGain; + if (extra_gain < 1.01) + break; + } + + computeExposure(context.frameContext, currentYGain); + frameCount_++; +} + +} /* namespace ipa::rkisp1::algorithms */ + +} /* namespace libcamera */ diff --git a/src/ipa/rkisp1/algorithms/agc.h b/src/ipa/rkisp1/algorithms/agc.h new file mode 100644 index 00000000..fbb8ea98 --- /dev/null +++ b/src/ipa/rkisp1/algorithms/agc.h @@ -0,0 +1,55 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2021, Ideas On Board + * + * agc.h - RkISP1 AGC/AEC mean-based control algorithm + */ +#ifndef __LIBCAMERA_RKISP1_ALGORITHMS_AGC_H__ +#define __LIBCAMERA_RKISP1_ALGORITHMS_AGC_H__ + +#include + +#include + +#include + +#include "algorithm.h" + +namespace libcamera { + +struct IPACameraSensorInfo; + +namespace ipa::rkisp1::algorithms { + +class Agc : public Algorithm +{ +public: + Agc(); + ~Agc() = default; + + int configure(IPAContext &context, const IPACameraSensorInfo &configInfo) override; + void process(IPAContext &context, const rkisp1_stat_buffer *stats) override; + +private: + void filterExposure(); + void computeExposure(IPAFrameContext &frameContext, double currentYGain); + double computeInitialY(const rkisp1_cif_isp_ae_stat *ae, double currentYGain); + + uint64_t frameCount_; + + utils::Duration lineDuration_; + utils::Duration minShutterSpeed_; + utils::Duration maxShutterSpeed_; + + double minAnalogueGain_; + double maxAnalogueGain_; + + utils::Duration filteredExposure_; + utils::Duration currentExposure_; +}; + +} /* namespace ipa::rkisp1::algorithms */ + +} /* namespace libcamera */ + +#endif /* __LIBCAMERA_RKISP1_ALGORITHMS_AGC_H__ */ diff --git a/src/ipa/rkisp1/algorithms/meson.build b/src/ipa/rkisp1/algorithms/meson.build index d98f77e2..05195d99 100644 --- a/src/ipa/rkisp1/algorithms/meson.build +++ b/src/ipa/rkisp1/algorithms/meson.build @@ -1,5 +1,6 @@ # SPDX-License-Identifier: CC0-1.0 rkisp1_ipa_algorithms = files([ + 'agc.cpp', 'algorithm.cpp', ]) diff --git a/src/ipa/rkisp1/ipa_context.cpp b/src/ipa/rkisp1/ipa_context.cpp index 819b2c73..16fc248f 100644 --- a/src/ipa/rkisp1/ipa_context.cpp +++ b/src/ipa/rkisp1/ipa_context.cpp @@ -55,4 +55,48 @@ namespace libcamera::ipa::rkisp1 { * are run. This needs to be turned into real per-frame data storage. */ +/** + * \var IPASessionConfiguration::agc + * \brief AGC parameters configuration of the IPA + * + * \var IPASessionConfiguration::agc.minShutterSpeed + * \brief Minimum shutter speed supported with the configured sensor + * + * \var IPASessionConfiguration::agc.maxShutterSpeed + * \brief Maximum shutter speed supported with the configured sensor + * + * \var IPASessionConfiguration::agc.minAnalogueGain + * \brief Minimum analogue gain supported with the configured sensor + * + * \var IPASessionConfiguration::agc.maxAnalogueGain + * \brief Maximum analogue gain supported with the configured sensor + */ + +/** + * \var IPAFrameContext::agc + * \brief Context for the Automatic Gain Control algorithm + * + * The exposure and gain determined are expected to be applied to the sensor + * at the earliest opportunity. + * + * \var IPAFrameContext::agc.exposure + * \brief Exposure time expressed as a number of lines + * + * \var IPAFrameContext::agc.gain + * \brief Analogue gain multiplier + * + * The gain should be adapted to the sensor specific gain code before applying. + */ + +/** + * \var IPAFrameContext::sensor + * \brief Effective sensor values + * + * \var IPAFrameContext::sensor.exposure + * \brief Exposure time expressed as a number of lines + * + * \var IPAFrameContext::sensor.gain + * \brief Analogue gain multiplier + */ + } /* namespace libcamera::ipa::rkisp1 */ diff --git a/src/ipa/rkisp1/ipa_context.h b/src/ipa/rkisp1/ipa_context.h index a1dc39fe..ca48ffc5 100644 --- a/src/ipa/rkisp1/ipa_context.h +++ b/src/ipa/rkisp1/ipa_context.h @@ -19,9 +19,24 @@ namespace libcamera { namespace ipa::rkisp1 { struct IPASessionConfiguration { + struct { + utils::Duration minShutterSpeed; + utils::Duration maxShutterSpeed; + double minAnalogueGain; + double maxAnalogueGain; + } agc; }; struct IPAFrameContext { + struct { + uint32_t exposure; + double gain; + } agc; + + struct { + uint32_t exposure; + double gain; + } sensor; }; struct IPAContext { diff --git a/src/ipa/rkisp1/rkisp1.cpp b/src/ipa/rkisp1/rkisp1.cpp index d2f4380a..1783f91f 100644 --- a/src/ipa/rkisp1/rkisp1.cpp +++ b/src/ipa/rkisp1/rkisp1.cpp @@ -25,6 +25,7 @@ #include +#include "algorithms/agc.h" #include "algorithms/algorithm.h" #include "libipa/camera_sensor_helper.h" @@ -32,6 +33,8 @@ namespace libcamera { LOG_DEFINE_CATEGORY(IPARkISP1) +using namespace std::literals::chrono_literals; + namespace ipa::rkisp1 { class IPARkISP1 : public IPARkISP1Interface @@ -77,6 +80,8 @@ private: unsigned int hwGammaOutMaxSamples_; unsigned int hwHistogramWeightGridsSize_; + utils::Duration lineDuration_; + /* Interface to the Camera Helper */ std::unique_ptr camHelper_; @@ -121,6 +126,9 @@ int IPARkISP1::init([[maybe_unused]] const IPASettings &settings, return -ENODEV; } + /* Construct our Algorithms */ + algorithms_.push_back(std::make_unique()); + return 0; } @@ -172,9 +180,29 @@ int IPARkISP1::configure([[maybe_unused]] const IPACameraSensorInfo &info, << "Exposure: " << minExposure_ << "-" << maxExposure_ << " Gain: " << minGain_ << "-" << maxGain_; + lineDuration_ = info.lineLength * 1.0s / info.pixelRate; + /* Clean context at configuration */ context_ = {}; + /* + * When the AGC computes the new exposure values for a frame, it needs + * to know the limits for shutter speed and analogue gain. + * As it depends on the sensor, update it with the controls. + * + * \todo take VBLANK into account for maximum shutter speed + */ + context_.configuration.agc.minShutterSpeed = minExposure_ * lineDuration_; + context_.configuration.agc.maxShutterSpeed = maxExposure_ * lineDuration_; + context_.configuration.agc.minAnalogueGain = camHelper_->gain(minGain_); + context_.configuration.agc.maxAnalogueGain = camHelper_->gain(maxGain_); + + for (auto const &algo : algorithms_) { + int ret = algo->configure(context_, info); + if (ret) + return ret; + } + return 0; } @@ -219,6 +247,9 @@ void IPARkISP1::processEvent(const RkISP1Event &event) reinterpret_cast( mappedBuffers_.at(bufferId).planes()[0].data()); + context_.frameContext.sensor.exposure = event.sensorControls.get(V4L2_CID_EXPOSURE).get(); + context_.frameContext.sensor.gain = camHelper_->gain(event.sensorControls.get(V4L2_CID_ANALOGUE_GAIN).get()); + updateStatistics(frame, stats); break; } @@ -263,44 +294,12 @@ void IPARkISP1::queueRequest(unsigned int frame, rkisp1_params_cfg *params, void IPARkISP1::updateStatistics(unsigned int frame, const rkisp1_stat_buffer *stats) { - const rkisp1_cif_isp_stat *params = &stats->params; unsigned int aeState = 0; - if (stats->meas_type & RKISP1_CIF_ISP_STAT_AUTOEXP) { - const rkisp1_cif_isp_ae_stat *ae = ¶ms->ae; - - const unsigned int target = 60; + for (auto const &algo : algorithms_) + algo->process(context_, stats); - unsigned int value = 0; - unsigned int num = 0; - for (unsigned int i = 0; i < hwAeMeanMax_; i++) { - if (ae->exp_mean[i] <= 15) - continue; - - value += ae->exp_mean[i]; - num++; - } - value /= num; - - double factor = (double)target / value; - - if (frame % 3 == 0) { - double exposure; - - exposure = factor * exposure_ * gain_ / minGain_; - exposure_ = std::clamp((uint64_t)exposure, - minExposure_, - maxExposure_); - - exposure = exposure / exposure_ * minGain_; - gain_ = std::clamp((uint64_t)exposure, - minGain_, maxGain_); - - setControls(frame + 1); - } - - aeState = fabs(factor - 1.0f) < 0.05f ? 2 : 1; - } + setControls(frame + 1); metadataReady(frame, aeState); } @@ -310,6 +309,9 @@ void IPARkISP1::setControls(unsigned int frame) RkISP1Action op; op.op = ActionV4L2Set; + exposure_ = context_.frameContext.agc.exposure; + gain_ = camHelper_->gainCode(context_.frameContext.agc.gain); + ControlList ctrls(ctrls_); ctrls.set(V4L2_CID_EXPOSURE, static_cast(exposure_)); ctrls.set(V4L2_CID_ANALOGUE_GAIN, static_cast(gain_));