From patchwork Fri Aug 27 08:02:24 2021 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Jean-Michel Hautbois X-Patchwork-Id: 13536 Return-Path: X-Original-To: parsemail@patchwork.libcamera.org Delivered-To: parsemail@patchwork.libcamera.org Received: from lancelot.ideasonboard.com (lancelot.ideasonboard.com [92.243.16.209]) by patchwork.libcamera.org (Postfix) with ESMTPS id C5B13C3241 for ; Fri, 27 Aug 2021 08:02:35 +0000 (UTC) Received: from lancelot.ideasonboard.com (localhost [IPv6:::1]) by lancelot.ideasonboard.com (Postfix) with ESMTP id 85DEC68941; Fri, 27 Aug 2021 10:02:35 +0200 (CEST) Authentication-Results: lancelot.ideasonboard.com; dkim=fail reason="signature verification failed" (1024-bit key; unprotected) header.d=ideasonboard.com header.i=@ideasonboard.com header.b="fZMS1ePW"; dkim-atps=neutral Received: from perceval.ideasonboard.com (perceval.ideasonboard.com [213.167.242.64]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id CFBF568891 for ; Fri, 27 Aug 2021 10:02:32 +0200 (CEST) Received: from tatooine.ideasonboard.com (unknown [IPv6:2a01:e0a:169:7140:ccf5:c267:eba8:cbb5]) by perceval.ideasonboard.com (Postfix) with ESMTPSA id 923735A1; Fri, 27 Aug 2021 10:02:32 +0200 (CEST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=ideasonboard.com; s=mail; t=1630051352; bh=roLTNhhQ6mwaUArnd99VQn+pHjsSMQk8+O43g1eD6+k=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=fZMS1ePWTf9q/NJC037NWzSlTI61C3rZXXJesNfKCsIoBWLusfXOcjRRtwjbaZ03o yUoOpZWDPfvfwJAvOJ70ZhRicNBz3j9TmsZ7iH9iNiW1hZSQeXR8L7Bk7nA93BY602 YxvM1rW3K27aOF8iNXXlGhtwKaBbOxyTnmBvGZ9U= From: Jean-Michel Hautbois To: libcamera-devel@lists.libcamera.org Date: Fri, 27 Aug 2021 10:02:24 +0200 Message-Id: <20210827080227.26370-2-jeanmichel.hautbois@ideasonboard.com> X-Mailer: git-send-email 2.30.2 In-Reply-To: <20210827080227.26370-1-jeanmichel.hautbois@ideasonboard.com> References: <20210827080227.26370-1-jeanmichel.hautbois@ideasonboard.com> MIME-Version: 1.0 Subject: [libcamera-devel] [PATCH v2 1/4] ipa: ipu3: Move the AWB stats structures X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: libcamera-devel-bounces@lists.libcamera.org Sender: "libcamera-devel" The structure Ipu3AwbCell describes the AWB stats layout on the kernel side. We will need it to be used by the AGC algorithm to be introduced later, so let's make it visible from ipa::ipu3::algorithms and not only for the AWB class. This structure should probably go into the intel-ipu3.h file, whichs means a kernel patch, let's keep it in mind for the moment. - IspStatsRegion is renamed Accumulator, and is storing the total number of pixels along with the counted pixels now. - The RGB structure is a more generic structure, it should be in a more generic header. - AwbStatus is used for the internal storage of the awb gains to update the context in prepare. Align the IPAFrameContext to have the same structure layout. Signed-off-by: Jean-Michel Hautbois --- src/ipa/ipu3/algorithms/awb.cpp | 47 ++++++++++--------- src/ipa/ipu3/algorithms/awb.h | 81 +++++++++++++++++---------------- src/ipa/ipu3/ipa_context.h | 1 + src/ipa/ipu3/ipu3.cpp | 3 ++ 4 files changed, 71 insertions(+), 61 deletions(-) diff --git a/src/ipa/ipu3/algorithms/awb.cpp b/src/ipa/ipu3/algorithms/awb.cpp index e05647c9..136e79e0 100644 --- a/src/ipa/ipu3/algorithms/awb.cpp +++ b/src/ipa/ipu3/algorithms/awb.cpp @@ -21,25 +21,27 @@ static constexpr uint32_t kMinZonesCounted = 16; static constexpr uint32_t kMinGreenLevelInZone = 32; /** - * \struct IspStatsRegion + * \struct Accumulator * \brief RGB statistics for a given region * - * The IspStatsRegion structure is intended to abstract the ISP specific - * statistics and use an agnostic algorithm to compute AWB. + * The Accumulator structure stores the sum of the pixel values in a region of + * the image, as well as the number of relevant pixels in this same region. A + * relevant pixel is an unsaturated pixel for this algorithm. + * \todo extend the notion of relevant to something else ? * - * \var IspStatsRegion::counted - * \brief Number of pixels used to calculate the sums + * \var Accumulator::total + * \brief Total number of pixels in the region * - * \var IspStatsRegion::uncounted - * \brief Remaining number of pixels in the region + * \var Accumulator::counted + * \brief Number of relevant pixels used to calculate the sums * - * \var IspStatsRegion::rSum + * \var Accumulator::rSum * \brief Sum of the red values in the region * - * \var IspStatsRegion::gSum + * \var Accumulator::gSum * \brief Sum of the green values in the region * - * \var IspStatsRegion::bSum + * \var Accumulator::bSum * \brief Sum of the blue values in the region */ @@ -117,9 +119,9 @@ static const struct ipu3_uapi_ccm_mat_config imguCssCcmDefault = { Awb::Awb() : Algorithm() { - asyncResults_.blueGain = 1.0; - asyncResults_.greenGain = 1.0; - asyncResults_.redGain = 1.0; + asyncResults_.gains.blue = 1.0; + asyncResults_.gains.green = 1.0; + asyncResults_.gains.red = 1.0; asyncResults_.temperatureK = 4500; zones_.reserve(kAwbStatsSizeX * kAwbStatsSizeY); @@ -204,6 +206,7 @@ void Awb::generateAwbStats(const ipu3_uapi_stats_3a *stats, awbStats_[awbRegionPosition].rSum += currentCell->redAvg; awbStats_[awbRegionPosition].bSum += currentCell->blueAvg; } + awbStats_[awbRegionPosition].total++; } } } @@ -215,7 +218,7 @@ void Awb::clearAwbStats() awbStats_[i].rSum = 0; awbStats_[i].gSum = 0; awbStats_[i].counted = 0; - awbStats_[i].uncounted = 0; + awbStats_[i].total = 0; } } @@ -254,9 +257,9 @@ void Awb::awbGreyWorld() /* Color temperature is not relevant in Grey world but still useful to estimate it :-) */ asyncResults_.temperatureK = estimateCCT(sumRed.R, sumRed.G, sumBlue.B); - asyncResults_.redGain = redGain; - asyncResults_.greenGain = 1.0; - asyncResults_.blueGain = blueGain; + asyncResults_.gains.red = redGain; + asyncResults_.gains.green = 1.0; + asyncResults_.gains.blue = blueGain; } void Awb::calculateWBGains(const ipu3_uapi_stats_3a *stats, @@ -270,8 +273,8 @@ void Awb::calculateWBGains(const ipu3_uapi_stats_3a *stats, LOG(IPU3Awb, Debug) << "Valid zones: " << zones_.size(); if (zones_.size() > 10) { awbGreyWorld(); - LOG(IPU3Awb, Debug) << "Gain found for red: " << asyncResults_.redGain - << " and for blue: " << asyncResults_.blueGain; + LOG(IPU3Awb, Debug) << "Gain found for red: " << asyncResults_.gains.red + << " and for blue: " << asyncResults_.gains.blue; } } @@ -284,9 +287,9 @@ void Awb::process(IPAContext &context, const ipu3_uapi_stats_3a *stats) * The results are cached, so if no results were calculated, we set the * cached values from asyncResults_ here. */ - context.frameContext.awb.gains.blue = asyncResults_.blueGain; - context.frameContext.awb.gains.green = asyncResults_.greenGain; - context.frameContext.awb.gains.red = asyncResults_.redGain; + context.frameContext.awb.gains.blue = asyncResults_.gains.blue; + context.frameContext.awb.gains.green = asyncResults_.gains.green; + context.frameContext.awb.gains.red = asyncResults_.gains.red; } void Awb::prepare(IPAContext &context, ipu3_uapi_params *params) diff --git a/src/ipa/ipu3/algorithms/awb.h b/src/ipa/ipu3/algorithms/awb.h index a16dd68d..f7e2f4cd 100644 --- a/src/ipa/ipu3/algorithms/awb.h +++ b/src/ipa/ipu3/algorithms/awb.h @@ -23,6 +23,38 @@ namespace ipa::ipu3::algorithms { static constexpr uint32_t kAwbStatsSizeX = 16; static constexpr uint32_t kAwbStatsSizeY = 12; +/* \todo Move the cell layout into intel-ipu3.h kernel header */ +struct Ipu3AwbCell { + unsigned char greenRedAvg; + unsigned char redAvg; + unsigned char blueAvg; + unsigned char greenBlueAvg; + unsigned char satRatio; + unsigned char padding[3]; +} __attribute__((packed)); + +/* \todo Make these structs available to all the ISPs ? */ +struct RGB { + RGB(double _R = 0, double _G = 0, double _B = 0) + : R(_R), G(_G), B(_B) + { + } + double R, G, B; + RGB &operator+=(RGB const &other) + { + R += other.R, G += other.G, B += other.B; + return *this; + } +}; + +struct Accumulator { + unsigned int total; + unsigned int counted; + unsigned long long rSum; + unsigned long long gSum; + unsigned long long bSum; +}; + class Awb : public Algorithm { public: @@ -32,44 +64,6 @@ public: void prepare(IPAContext &context, ipu3_uapi_params *params) override; void process(IPAContext &context, const ipu3_uapi_stats_3a *stats) override; - struct Ipu3AwbCell { - unsigned char greenRedAvg; - unsigned char redAvg; - unsigned char blueAvg; - unsigned char greenBlueAvg; - unsigned char satRatio; - unsigned char padding[3]; - } __attribute__((packed)); - - /* \todo Make these three structs available to all the ISPs ? */ - struct RGB { - RGB(double _R = 0, double _G = 0, double _B = 0) - : R(_R), G(_G), B(_B) - { - } - double R, G, B; - RGB &operator+=(RGB const &other) - { - R += other.R, G += other.G, B += other.B; - return *this; - } - }; - - struct IspStatsRegion { - unsigned int counted; - unsigned int uncounted; - unsigned long long rSum; - unsigned long long gSum; - unsigned long long bSum; - }; - - struct AwbStatus { - double temperatureK; - double redGain; - double greenGain; - double blueGain; - }; - private: void calculateWBGains(const ipu3_uapi_stats_3a *stats, const ipu3_uapi_grid_config &grid); @@ -80,8 +74,17 @@ private: void awbGreyWorld(); uint32_t estimateCCT(double red, double green, double blue); + struct AwbStatus { + double temperatureK; + struct { + double red; + double green; + double blue; + } gains; + }; + std::vector zones_; - IspStatsRegion awbStats_[kAwbStatsSizeX * kAwbStatsSizeY]; + Accumulator awbStats_[kAwbStatsSizeX * kAwbStatsSizeY]; AwbStatus asyncResults_; }; diff --git a/src/ipa/ipu3/ipa_context.h b/src/ipa/ipu3/ipa_context.h index 9d9444dc..3a292ad7 100644 --- a/src/ipa/ipu3/ipa_context.h +++ b/src/ipa/ipu3/ipa_context.h @@ -30,6 +30,7 @@ struct IPAFrameContext { } agc; struct { + double temperatureK; struct { double red; double green; diff --git a/src/ipa/ipu3/ipu3.cpp b/src/ipa/ipu3/ipu3.cpp index 0ed0a6f1..cbb3f440 100644 --- a/src/ipa/ipu3/ipu3.cpp +++ b/src/ipa/ipu3/ipu3.cpp @@ -112,6 +112,9 @@ * \struct IPAFrameContext::awb * \brief Context for the Automatic White Balance algorithm * + * \var IPAFrameContext::awb::temperatureK + * \brief Estimated color temperature in Kelvin + * * \struct IPAFrameContext::awb::gains * \brief White balance gains * From patchwork Fri Aug 27 08:02:25 2021 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Jean-Michel Hautbois X-Patchwork-Id: 13537 Return-Path: X-Original-To: parsemail@patchwork.libcamera.org Delivered-To: parsemail@patchwork.libcamera.org Received: from lancelot.ideasonboard.com (lancelot.ideasonboard.com [92.243.16.209]) by patchwork.libcamera.org (Postfix) with ESMTPS id 06726C3242 for ; Fri, 27 Aug 2021 08:02:36 +0000 (UTC) Received: from lancelot.ideasonboard.com (localhost [IPv6:::1]) by lancelot.ideasonboard.com (Postfix) with ESMTP id 2196168944; Fri, 27 Aug 2021 10:02:36 +0200 (CEST) Authentication-Results: lancelot.ideasonboard.com; dkim=fail reason="signature verification failed" (1024-bit key; unprotected) header.d=ideasonboard.com header.i=@ideasonboard.com header.b="uNLgUwU6"; dkim-atps=neutral Received: from perceval.ideasonboard.com (perceval.ideasonboard.com [IPv6:2001:4b98:dc2:55:216:3eff:fef7:d647]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id 9620068934 for ; Fri, 27 Aug 2021 10:02:33 +0200 (CEST) Received: from tatooine.ideasonboard.com (unknown [IPv6:2a01:e0a:169:7140:ccf5:c267:eba8:cbb5]) by perceval.ideasonboard.com (Postfix) with ESMTPSA id 3F81D5A1; Fri, 27 Aug 2021 10:02:33 +0200 (CEST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=ideasonboard.com; s=mail; t=1630051353; bh=ylY8JUqFcCTcaOX8cROKMKWFYzNWCUw4JDPb2NVnA7E=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=uNLgUwU6wEYznryuU1hcthpmwmxnzX/72WHn9DIF7u913AeUNvvmIP1NJqxiO+PNu Wec/eCkvX32MXUe4K4tAyAvuu781muC6t2eCOZICuyRxxA/sBpEEcqpbyKBHxUcpNu c70MMbod/jEVBeFuiLRh889p4gi79G++jd+t9V3g= From: Jean-Michel Hautbois To: libcamera-devel@lists.libcamera.org Date: Fri, 27 Aug 2021 10:02:25 +0200 Message-Id: <20210827080227.26370-3-jeanmichel.hautbois@ideasonboard.com> X-Mailer: git-send-email 2.30.2 In-Reply-To: <20210827080227.26370-1-jeanmichel.hautbois@ideasonboard.com> References: <20210827080227.26370-1-jeanmichel.hautbois@ideasonboard.com> MIME-Version: 1.0 Subject: [libcamera-devel] [PATCH v2 2/4] ipa: ipu3: Rename AGC algorithm X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: libcamera-devel-bounces@lists.libcamera.org Sender: "libcamera-devel" The initial AGC algorithm is a simple one, which calculates the exposure and gain values by trying to have a target of 0.5 for the top 2% of the histogram. Rename it as "AgcMean" to make it easy to distinguish. Now that we are modular, we can have multiple algorithms for one functionality (here, AGC). Naming those algorithms makes it easier to chose between them. Signed-off-by: Jean-Michel Hautbois Reviewed-by: Laurent Pinchart --- .../ipu3/algorithms/{agc.cpp => agc_mean.cpp} | 31 +++++++++---------- src/ipa/ipu3/algorithms/{agc.h => agc_mean.h} | 8 ++--- src/ipa/ipu3/algorithms/meson.build | 2 +- src/ipa/ipu3/ipu3.cpp | 4 +-- 4 files changed, 22 insertions(+), 23 deletions(-) rename src/ipa/ipu3/algorithms/{agc.cpp => agc_mean.cpp} (87%) rename src/ipa/ipu3/algorithms/{agc.h => agc_mean.h} (90%) diff --git a/src/ipa/ipu3/algorithms/agc.cpp b/src/ipa/ipu3/algorithms/agc_mean.cpp similarity index 87% rename from src/ipa/ipu3/algorithms/agc.cpp rename to src/ipa/ipu3/algorithms/agc_mean.cpp index 5ff50f4a..193f6e9a 100644 --- a/src/ipa/ipu3/algorithms/agc.cpp +++ b/src/ipa/ipu3/algorithms/agc_mean.cpp @@ -2,10 +2,10 @@ /* * Copyright (C) 2021, Ideas On Board * - * ipu3_agc.cpp - AGC/AEC control algorithm + * agc_mean.cpp - AGC/AEC control algorithm */ -#include "agc.h" +#include "agc_mean.h" #include #include @@ -23,7 +23,7 @@ using namespace std::literals::chrono_literals; namespace ipa::ipu3::algorithms { -LOG_DEFINE_CATEGORY(IPU3Agc) +LOG_DEFINE_CATEGORY(IPU3AgcMean) /* Number of frames to wait before calculating stats on minimum exposure */ static constexpr uint32_t kInitialFrameMinAECount = 4; @@ -50,16 +50,15 @@ static constexpr double kEvGainTarget = 0.5; /* A cell is 8 bytes and contains averages for RGB values and saturation ratio */ static constexpr uint8_t kCellSize = 8; -Agc::Agc() +AgcMean::AgcMean() : frameCount_(0), lastFrame_(0), iqMean_(0.0), lineDuration_(0s), maxExposureTime_(0s), prevExposure_(0s), prevExposureNoDg_(0s), currentExposure_(0s), currentExposureNoDg_(0s) { } -int Agc::configure([[maybe_unused]] IPAContext &context, - const IPAConfigInfo &configInfo) -{ +int AgcMean::configure([[maybe_unused]] IPAContext &context, + const IPAConfigInfo &configInfo){ lineDuration_ = configInfo.sensorInfo.lineLength * 1.0s / configInfo.sensorInfo.pixelRate; maxExposureTime_ = kMaxExposure * lineDuration_; @@ -67,7 +66,7 @@ int Agc::configure([[maybe_unused]] IPAContext &context, return 0; } -void Agc::processBrightness(const ipu3_uapi_stats_3a *stats, +void AgcMean::processBrightness(const ipu3_uapi_stats_3a *stats, const ipu3_uapi_grid_config &grid) { const struct ipu3_uapi_grid_config statsAeGrid = stats->stats_4a_config.awb_config.grid; @@ -109,7 +108,7 @@ void Agc::processBrightness(const ipu3_uapi_stats_3a *stats, iqMean_ = Histogram(Span(hist)).interQuantileMean(0.98, 1.0); } -void Agc::filterExposure() +void AgcMean::filterExposure() { double speed = 0.2; if (prevExposure_ == 0s) { @@ -140,10 +139,10 @@ void Agc::filterExposure() if (prevExposureNoDg_ < prevExposure_ * fastReduceThreshold) prevExposureNoDg_ = prevExposure_ * fastReduceThreshold; - LOG(IPU3Agc, Debug) << "After filtering, total_exposure " << prevExposure_; + LOG(IPU3AgcMean, Debug) << "After filtering, total_exposure " << prevExposure_; } -void Agc::lockExposureGain(uint32_t &exposure, double &gain) +void AgcMean::lockExposureGain(uint32_t &exposure, double &gain) { /* Algorithm initialization should wait for first valid frames */ /* \todo - have a number of frames given by DelayedControls ? @@ -153,20 +152,20 @@ void Agc::lockExposureGain(uint32_t &exposure, double &gain) /* Are we correctly exposed ? */ if (std::abs(iqMean_ - kEvGainTarget * knumHistogramBins) <= 1) { - LOG(IPU3Agc, Debug) << "!!! Good exposure with iqMean = " << iqMean_; + LOG(IPU3AgcMean, Debug) << "!!! Good exposure with iqMean = " << iqMean_; } else { double newGain = kEvGainTarget * knumHistogramBins / iqMean_; /* extracted from Rpi::Agc::computeTargetExposure */ libcamera::utils::Duration currentShutter = exposure * lineDuration_; currentExposureNoDg_ = currentShutter * gain; - LOG(IPU3Agc, Debug) << "Actual total exposure " << currentExposureNoDg_ + LOG(IPU3AgcMean, Debug) << "Actual total exposure " << currentExposureNoDg_ << " Shutter speed " << currentShutter << " Gain " << gain; currentExposure_ = currentExposureNoDg_ * newGain; libcamera::utils::Duration maxTotalExposure = maxExposureTime_ * kMaxGain; currentExposure_ = std::min(currentExposure_, maxTotalExposure); - LOG(IPU3Agc, Debug) << "Target total exposure " << currentExposure_; + LOG(IPU3AgcMean, Debug) << "Target total exposure " << currentExposure_; /* \todo: estimate if we need to desaturate */ filterExposure(); @@ -181,12 +180,12 @@ void Agc::lockExposureGain(uint32_t &exposure, double &gain) newExposure = currentExposure_ / gain; exposure = std::clamp(static_cast(exposure * currentExposure_ / newExposure), kMinExposure, kMaxExposure); } - LOG(IPU3Agc, Debug) << "Adjust exposure " << exposure * lineDuration_ << " and gain " << gain; + LOG(IPU3AgcMean, Debug) << "Adjust exposure " << exposure * lineDuration_ << " and gain " << gain; } lastFrame_ = frameCount_; } -void Agc::process(IPAContext &context, const ipu3_uapi_stats_3a *stats) +void AgcMean::process(IPAContext &context, const ipu3_uapi_stats_3a *stats) { uint32_t &exposure = context.frameContext.agc.exposure; double &gain = context.frameContext.agc.gain; diff --git a/src/ipa/ipu3/algorithms/agc.h b/src/ipa/ipu3/algorithms/agc_mean.h similarity index 90% rename from src/ipa/ipu3/algorithms/agc.h rename to src/ipa/ipu3/algorithms/agc_mean.h index e36797d3..97114121 100644 --- a/src/ipa/ipu3/algorithms/agc.h +++ b/src/ipa/ipu3/algorithms/agc_mean.h @@ -2,7 +2,7 @@ /* * Copyright (C) 2021, Ideas On Board * - * agc.h - IPU3 AGC/AEC control algorithm + * agc_mean.h - IPU3 AGC/AEC control algorithm */ #ifndef __LIBCAMERA_IPU3_ALGORITHMS_AGC_H__ #define __LIBCAMERA_IPU3_ALGORITHMS_AGC_H__ @@ -23,11 +23,11 @@ namespace ipa::ipu3::algorithms { using utils::Duration; -class Agc : public Algorithm +class AgcMean : public Algorithm { public: - Agc(); - ~Agc() = default; + AgcMean(); + ~AgcMean() = default; int configure(IPAContext &context, const IPAConfigInfo &configInfo) override; void process(IPAContext &context, const ipu3_uapi_stats_3a *stats) override; diff --git a/src/ipa/ipu3/algorithms/meson.build b/src/ipa/ipu3/algorithms/meson.build index deae225b..807b53ea 100644 --- a/src/ipa/ipu3/algorithms/meson.build +++ b/src/ipa/ipu3/algorithms/meson.build @@ -1,7 +1,7 @@ # SPDX-License-Identifier: CC0-1.0 ipu3_ipa_algorithms = files([ - 'agc.cpp', + 'agc_mean.cpp', 'algorithm.cpp', 'awb.cpp', 'tone_mapping.cpp', diff --git a/src/ipa/ipu3/ipu3.cpp b/src/ipa/ipu3/ipu3.cpp index cbb3f440..6332fc06 100644 --- a/src/ipa/ipu3/ipu3.cpp +++ b/src/ipa/ipu3/ipu3.cpp @@ -29,7 +29,7 @@ #include "libcamera/internal/mapped_framebuffer.h" -#include "algorithms/agc.h" +#include "algorithms/agc_mean.h" #include "algorithms/algorithm.h" #include "algorithms/awb.h" #include "algorithms/tone_mapping.h" @@ -269,7 +269,7 @@ int IPAIPU3::init(const IPASettings &settings, *ipaControls = ControlInfoMap(std::move(controls), controls::controls); /* Construct our Algorithms */ - algorithms_.push_back(std::make_unique()); + algorithms_.push_back(std::make_unique()); algorithms_.push_back(std::make_unique()); algorithms_.push_back(std::make_unique()); From patchwork Fri Aug 27 08:02:26 2021 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Jean-Michel Hautbois X-Patchwork-Id: 13538 Return-Path: X-Original-To: parsemail@patchwork.libcamera.org Delivered-To: parsemail@patchwork.libcamera.org Received: from lancelot.ideasonboard.com (lancelot.ideasonboard.com [92.243.16.209]) by patchwork.libcamera.org (Postfix) with ESMTPS id 8A11EBD87C for ; Fri, 27 Aug 2021 08:02:38 +0000 (UTC) Received: from lancelot.ideasonboard.com (localhost [IPv6:::1]) by lancelot.ideasonboard.com (Postfix) with ESMTP id ED3A56893E; Fri, 27 Aug 2021 10:02:36 +0200 (CEST) Authentication-Results: lancelot.ideasonboard.com; dkim=fail reason="signature verification failed" (1024-bit key; unprotected) header.d=ideasonboard.com header.i=@ideasonboard.com header.b="UTphcs4Q"; dkim-atps=neutral Received: from perceval.ideasonboard.com (perceval.ideasonboard.com [213.167.242.64]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id 096106893B for ; Fri, 27 Aug 2021 10:02:34 +0200 (CEST) Received: from tatooine.ideasonboard.com (unknown [IPv6:2a01:e0a:169:7140:ccf5:c267:eba8:cbb5]) by perceval.ideasonboard.com (Postfix) with ESMTPSA id C00155A1; Fri, 27 Aug 2021 10:02:33 +0200 (CEST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=ideasonboard.com; s=mail; t=1630051353; bh=jo/UG3hHfzS0A3qGlAUTIz02Qi2QLXk8m8kR5U0LNh4=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=UTphcs4QHxaqctbAVTsJJqrtzP/iRwIjMiibLATqu06VV6bjEN+z21VSkptAH/Xeh vDL12Tihcdl/GGjCv54ObOFqSsnGljP3SYdgxj6wOcEPZWutjciyd2zWmdJ3VTGs0x lGSal18tyYk3RltjQ+NG9Fm15H7Rti31ckvVuRkA= From: Jean-Michel Hautbois To: libcamera-devel@lists.libcamera.org Date: Fri, 27 Aug 2021 10:02:26 +0200 Message-Id: <20210827080227.26370-4-jeanmichel.hautbois@ideasonboard.com> X-Mailer: git-send-email 2.30.2 In-Reply-To: <20210827080227.26370-1-jeanmichel.hautbois@ideasonboard.com> References: <20210827080227.26370-1-jeanmichel.hautbois@ideasonboard.com> MIME-Version: 1.0 Subject: [libcamera-devel] [PATCH v2 3/4] ipa: ipu3: Document AGC mean-based algorithm X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: libcamera-devel-bounces@lists.libcamera.org Sender: "libcamera-devel" The AGC mean-based algorithm is partially documented. Improve the documentation to help understand how it works and mark some \todo for future improvements. Signed-off-by: Jean-Michel Hautbois --- src/ipa/ipu3/algorithms/agc_mean.cpp | 91 +++++++++++++++++++++++++--- src/ipa/ipu3/algorithms/agc_mean.h | 2 +- 2 files changed, 84 insertions(+), 9 deletions(-) diff --git a/src/ipa/ipu3/algorithms/agc_mean.cpp b/src/ipa/ipu3/algorithms/agc_mean.cpp index 193f6e9a..b535e14b 100644 --- a/src/ipa/ipu3/algorithms/agc_mean.cpp +++ b/src/ipa/ipu3/algorithms/agc_mean.cpp @@ -2,7 +2,7 @@ /* * Copyright (C) 2021, Ideas On Board * - * agc_mean.cpp - AGC/AEC control algorithm + * agc_mean.cpp - AGC/AEC mean-based control algorithm */ #include "agc_mean.h" @@ -17,12 +17,26 @@ #include "libipa/histogram.h" +/** + * \file agc_mean.h + */ + namespace libcamera { using namespace std::literals::chrono_literals; namespace ipa::ipu3::algorithms { +/** + * \class AgcMean + * \brief The class to use the mean-based auto-exposure algorithm + * + * The mean-based algorithm is calculating an exposure and gain value such as + * a given quantity of pixels lie in the top 2% of the histogram. The AWB gains + * are not used here, and all cells in the grid have the same weight, like an + * average-metering case. + */ + LOG_DEFINE_CATEGORY(IPU3AgcMean) /* Number of frames to wait before calculating stats on minimum exposure */ @@ -30,13 +44,20 @@ static constexpr uint32_t kInitialFrameMinAECount = 4; /* Number of frames to wait between new gain/exposure estimations */ static constexpr uint32_t kFrameSkipCount = 6; -/* Maximum ISO value for analogue gain */ +/* Minimum ISO value for analogue gain (no digital gain supported) */ static constexpr uint32_t kMinISO = 100; +/* Maximum ISO value for analogue gain (no digital gain supported) */ static constexpr uint32_t kMaxISO = 1500; -/* Maximum analogue gain value - * \todo grab it from a camera helper */ +/* + * Minimum analogue gain value + * \todo grab it from a camera helper + */ static constexpr uint32_t kMinGain = kMinISO / 100; +/* + * Maximum analogue gain value + * \todo grab it from a camera helper + */ static constexpr uint32_t kMaxGain = kMaxISO / 100; /* \todo use calculated value based on sensor */ @@ -45,6 +66,7 @@ static constexpr uint32_t kMaxExposure = 1976; /* Histogram constants */ static constexpr uint32_t knumHistogramBins = 256; +/* Target value to reach for the top 2% of the histogram */ static constexpr double kEvGainTarget = 0.5; /* A cell is 8 bytes and contains averages for RGB values and saturation ratio */ @@ -57,8 +79,18 @@ AgcMean::AgcMean() { } +/** + * \brief Configure the AGC given a configInfo + * \param[in] context The shared IPA context (\todo not used yet) + * \param[in] configInfo The IPA configuration data, received from the pipeline + * handler + * + * \return 0 + */ int AgcMean::configure([[maybe_unused]] IPAContext &context, - const IPAConfigInfo &configInfo){ + const IPAConfigInfo &configInfo) +{ + /* \todo use the configInfo fields and IPAContext to store the limits */ lineDuration_ = configInfo.sensorInfo.lineLength * 1.0s / configInfo.sensorInfo.pixelRate; maxExposureTime_ = kMaxExposure * lineDuration_; @@ -66,9 +98,22 @@ int AgcMean::configure([[maybe_unused]] IPAContext &context, return 0; } +/** + * \brief Estimate the mean quantile of the top 2% of the histogram + * \param[in] stats The statistics buffer coming from the pipeline handler + * \param[in] grid The grid used to store the statistics in the IPU3 + */ void AgcMean::processBrightness(const ipu3_uapi_stats_3a *stats, const ipu3_uapi_grid_config &grid) { + /* + * Get the applied grid from the statistics buffer. When the kernel + * receives a grid from the parameters buffer, it will check and align + * all the values. For instance, it will automatically fill the x_end + * value based on x_start, grid width and log2 width. + * \todo Use the grid calculated in configure as there is a bug in IPU3 + * causing the width (maybe height) to be bit-shifted. + */ const struct ipu3_uapi_grid_config statsAeGrid = stats->stats_4a_config.awb_config.grid; Rectangle aeRegion = { statsAeGrid.x_start, statsAeGrid.y_start, @@ -85,6 +130,7 @@ void AgcMean::processBrightness(const ipu3_uapi_stats_3a *stats, uint32_t i, j; uint32_t count = 0; + /* Initialise the histogram array */ uint32_t hist[knumHistogramBins] = { 0 }; for (j = topleftY; j < topleftY + (aeRegion.size().height >> grid.block_height_log2); @@ -92,12 +138,18 @@ void AgcMean::processBrightness(const ipu3_uapi_stats_3a *stats, for (i = startX + startY; i < endX + startY; i += kCellSize) { /* * The grid width (and maybe height) is not reliable. - * We observed a bit shift which makes the value 160 to be 32 in the stats grid. - * Use the one passed at init time. + * We observed a bit shift which makes the value 160 to + * be 32 in the stats grid. Use the one from configure. */ if (stats->awb_raw_buffer.meta_data[i + 4 + j * grid.width] == 0) { uint8_t Gr = stats->awb_raw_buffer.meta_data[i + 0 + j * grid.width]; uint8_t Gb = stats->awb_raw_buffer.meta_data[i + 3 + j * grid.width]; + /* + * Store the average green value to estimate the + * brightness. Even the over exposed pixels are + * taken into account. + * \todo remove count which is not used. + */ hist[(Gr + Gb) / 2]++; count++; } @@ -108,11 +160,14 @@ void AgcMean::processBrightness(const ipu3_uapi_stats_3a *stats, iqMean_ = Histogram(Span(hist)).interQuantileMean(0.98, 1.0); } +/** + * \brief Apply a filter on the exposure value to limit the speed of changes + */ void AgcMean::filterExposure() { double speed = 0.2; if (prevExposure_ == 0s) { - /* DG stands for digital gain.*/ + /* DG stands for digital gain, which is always 1.0 for now. */ prevExposure_ = currentExposure_; prevExposureNoDg_ = currentExposureNoDg_; } else { @@ -134,6 +189,7 @@ void AgcMean::filterExposure() * We can't let the no_dg exposure deviate too far below the * total exposure, as there might not be enough digital gain available * in the ISP to hide it (which will cause nasty oscillation). + * \todo add the digital gain usage */ double fastReduceThreshold = 0.4; if (prevExposureNoDg_ < @@ -142,6 +198,11 @@ void AgcMean::filterExposure() LOG(IPU3AgcMean, Debug) << "After filtering, total_exposure " << prevExposure_; } +/** + * \brief Estimate the new exposure and gain values + * \param[in] exposure The exposure value reference as a number of lines + * \param[in] gain The gain reference to be updated + */ void AgcMean::lockExposureGain(uint32_t &exposure, double &gain) { /* Algorithm initialization should wait for first valid frames */ @@ -154,15 +215,20 @@ void AgcMean::lockExposureGain(uint32_t &exposure, double &gain) if (std::abs(iqMean_ - kEvGainTarget * knumHistogramBins) <= 1) { LOG(IPU3AgcMean, Debug) << "!!! Good exposure with iqMean = " << iqMean_; } else { + /* Estimate the gain needed to have the proportion wanted */ double newGain = kEvGainTarget * knumHistogramBins / iqMean_; /* extracted from Rpi::Agc::computeTargetExposure */ + /* Calculate the shutter time in seconds */ libcamera::utils::Duration currentShutter = exposure * lineDuration_; + /* Ev = shutter_time * gain */ currentExposureNoDg_ = currentShutter * gain; LOG(IPU3AgcMean, Debug) << "Actual total exposure " << currentExposureNoDg_ << " Shutter speed " << currentShutter << " Gain " << gain; + /* Apply the gain calculated to the current exposure value */ currentExposure_ = currentExposureNoDg_ * newGain; + /* Clamp the exposure value to the min and max authorized */ libcamera::utils::Duration maxTotalExposure = maxExposureTime_ * kMaxGain; currentExposure_ = std::min(currentExposure_, maxTotalExposure); LOG(IPU3AgcMean, Debug) << "Target total exposure " << currentExposure_; @@ -170,6 +236,7 @@ void AgcMean::lockExposureGain(uint32_t &exposure, double &gain) /* \todo: estimate if we need to desaturate */ filterExposure(); + /* Divide the exposure value as new exposure and gain values */ libcamera::utils::Duration newExposure = 0.0s; if (currentShutter < maxExposureTime_) { exposure = std::clamp(static_cast(exposure * currentExposure_ / currentExposureNoDg_), kMinExposure, kMaxExposure); @@ -185,11 +252,19 @@ void AgcMean::lockExposureGain(uint32_t &exposure, double &gain) lastFrame_ = frameCount_; } +/** + * \brief Process IPU3 statistics, and run AGC operations + * \param[in] context The shared IPA context + * \param[in] stats The IPU3 statistics and ISP results + */ void AgcMean::process(IPAContext &context, const ipu3_uapi_stats_3a *stats) { + /* Get the latest exposure and gain applied */ uint32_t &exposure = context.frameContext.agc.exposure; double &gain = context.frameContext.agc.gain; + /* Calculate the current brightness */ processBrightness(stats, context.configuration.grid.bdsGrid); + /* Update the exposure and gain values */ lockExposureGain(exposure, gain); frameCount_++; } diff --git a/src/ipa/ipu3/algorithms/agc_mean.h b/src/ipa/ipu3/algorithms/agc_mean.h index 97114121..6232597d 100644 --- a/src/ipa/ipu3/algorithms/agc_mean.h +++ b/src/ipa/ipu3/algorithms/agc_mean.h @@ -2,7 +2,7 @@ /* * Copyright (C) 2021, Ideas On Board * - * agc_mean.h - IPU3 AGC/AEC control algorithm + * agc_mean.h - AGC/AEC mean-based control algorithm */ #ifndef __LIBCAMERA_IPU3_ALGORITHMS_AGC_H__ #define __LIBCAMERA_IPU3_ALGORITHMS_AGC_H__ From patchwork Fri Aug 27 08:02:27 2021 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Jean-Michel Hautbois X-Patchwork-Id: 13539 Return-Path: X-Original-To: parsemail@patchwork.libcamera.org Delivered-To: parsemail@patchwork.libcamera.org Received: from lancelot.ideasonboard.com (lancelot.ideasonboard.com [92.243.16.209]) by patchwork.libcamera.org (Postfix) with ESMTPS id 5E4F6BD87C for ; Fri, 27 Aug 2021 08:02:40 +0000 (UTC) Received: from lancelot.ideasonboard.com (localhost [IPv6:::1]) by lancelot.ideasonboard.com (Postfix) with ESMTP id 24F466893B; Fri, 27 Aug 2021 10:02:40 +0200 (CEST) Authentication-Results: lancelot.ideasonboard.com; dkim=fail reason="signature verification failed" (1024-bit key; unprotected) header.d=ideasonboard.com header.i=@ideasonboard.com header.b="EkobqDm4"; dkim-atps=neutral Received: from perceval.ideasonboard.com (perceval.ideasonboard.com [IPv6:2001:4b98:dc2:55:216:3eff:fef7:d647]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id 1DB7B68928 for ; Fri, 27 Aug 2021 10:02:35 +0200 (CEST) Received: from tatooine.ideasonboard.com (unknown [IPv6:2a01:e0a:169:7140:ccf5:c267:eba8:cbb5]) by perceval.ideasonboard.com (Postfix) with ESMTPSA id BC2615A1; Fri, 27 Aug 2021 10:02:34 +0200 (CEST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=ideasonboard.com; s=mail; t=1630051354; bh=AfXQ1+rt2Z0+KV0SZ9nHLOlbGSw6IL7JSbdkqFNVOKs=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=EkobqDm4efPEbXX1t5XdqGF0DJcmkY1YosVGU9AanZUqJOGkgeKGD8CrBC+HMolSq 1erevsylSwG3icnAhncS0rBKlTvtyZDvKafC68g7lzGVso44ED9nxw5fwTmtKFCXC1 1s9Rqmev8gaLWz1I5M3CqO6yf9pINyiDhBUDvIoo= From: Jean-Michel Hautbois To: libcamera-devel@lists.libcamera.org Date: Fri, 27 Aug 2021 10:02:27 +0200 Message-Id: <20210827080227.26370-5-jeanmichel.hautbois@ideasonboard.com> X-Mailer: git-send-email 2.30.2 In-Reply-To: <20210827080227.26370-1-jeanmichel.hautbois@ideasonboard.com> References: <20210827080227.26370-1-jeanmichel.hautbois@ideasonboard.com> MIME-Version: 1.0 Subject: [libcamera-devel] [PATCH v2 4/4] ipa: ipu3: Introduce a new AGC algorithm X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: libcamera-devel-bounces@lists.libcamera.org Sender: "libcamera-devel" The algorithm used until then is a simple one, let's introduce a new one, based on the one used by the Raspberry Pi code. We can keep both compiled, and chose to instanciate only one, which demonstrates the modularity and ease to add functionnalities to the IPA. This algorithm uses the IPAFrameContext to get the latest AWB gains applied and use them to estimate the next shutter time and gain values to set. For the moment it is not activated as the default algorithm as it may be a bit unstable. More testing is need ;-). Signed-off-by: Jean-Michel Hautbois --- src/ipa/ipu3/algorithms/agc_metering.cpp | 427 +++++++++++++++++++++++ src/ipa/ipu3/algorithms/agc_metering.h | 78 +++++ src/ipa/ipu3/algorithms/meson.build | 1 + src/ipa/ipu3/ipa_context.h | 6 + src/ipa/ipu3/ipu3.cpp | 8 + 5 files changed, 520 insertions(+) create mode 100644 src/ipa/ipu3/algorithms/agc_metering.cpp create mode 100644 src/ipa/ipu3/algorithms/agc_metering.h diff --git a/src/ipa/ipu3/algorithms/agc_metering.cpp b/src/ipa/ipu3/algorithms/agc_metering.cpp new file mode 100644 index 00000000..1dc05082 --- /dev/null +++ b/src/ipa/ipu3/algorithms/agc_metering.cpp @@ -0,0 +1,427 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Based on the implementation from the Raspberry Pi IPA, + * Copyright (C) 2019-2021, Raspberry Pi (Trading) Ltd. + * Copyright (C) 2021, Google inc. + * + * agc_metering.cpp - AGC/AEC metering-based control algorithm + */ + +#include "agc_metering.h" +#include "awb.h" + +#include +#include +#include +#include + +#include + +#include +#include + +#include "libipa/histogram.h" + +/** + * \file agc_metering.h + */ + +namespace libcamera { + +using namespace std::literals::chrono_literals; + +namespace ipa::ipu3::algorithms { + +/** + * \class AgcMetering + * \brief The class to use the metering-based auto-exposure algorithm + * + * The metering-based algorithm is calculating an exposure and gain value such + * as a given quantity of pixels lie in the top 2% of the histogram. The AWB + * gains are also used here, and all cells in the grid are weighted using a + * specific metering matrix. The default here is Spot metering. + */ + +LOG_DEFINE_CATEGORY(IPU3AgcMetering) + +/* Histogram constants */ +static constexpr uint32_t knumHistogramBins = 256; + +/* seems to be a 10-bit pipeline */ +static constexpr uint8_t kPipelineBits = 10; + +/* width of the AGC stats grid */ +static constexpr uint32_t kAgcStatsSizeX = 7; +/* height of the AGC stats grid */ +static constexpr uint32_t kAgcStatsSizeY = 5; +/* size of the AGC stats grid */ +static constexpr uint32_t kAgcStatsSize = kAgcStatsSizeX * kAgcStatsSizeY; + +/** + * The AGC algorithm uses region-based metering. + * The image is divided up into regions as: + * + * +--+--------------+--+ + * |11| 9 |12| + * +--+--+--------+--+--+ + * | | | 3 | | | + * | | +--+--+--+ | | + * |7 |5 |1 |0 |2 |6 |8 | + * | | +--+--+--+ | | + * | | | 4 | | | + * +--+--+--------+--+--+ + * |13| 10 |14| + * +--+--------------+--+ + * + * The metering-based algorithm is calculating an exposure and gain value such + * as a given quantity of weighted pixels lie in the top 2% of the histogram.The + * AWB gains applied are also used to estimate the total gain to apply. + * + * An average luminance value for the image is calculated according to: + * \f$Y = \frac{\sum_{i=0}^{i=kNumAgcWeightedZones}{kCenteredWeights_{i}Y_{i}}} + * {\sum_{i=0}^{i=kNumAgcWeightedZones}{w_{i}}}\f$ + */ + +/* Weight applied on each region */ +static constexpr double kSpotWeights[kNumAgcWeightedZones] = { 2, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }; + +/* Region number repartition in the image */ +static constexpr uint32_t kAgcStatsRegions[kAgcStatsSize] = { + 11, 9, 9, 9, 9, 9, 12, + 7, 5, 3, 3, 3, 6, 8, + 7, 5, 1, 0, 2, 6, 8, + 7, 5, 4, 4, 4, 6, 8, + 13, 10, 10, 10, 10, 10, 14 +}; + +/* Limit the speed of change between two exposure levels */ +static constexpr double kFastReduceThreshold = 0.3; + +AgcMetering::AgcMetering() + : iqMean_(0.0), prevExposure_(0s), prevExposureNoDg_(0s), + currentExposure_(0s), currentExposureNoDg_(0s), currentShutter_(1.0s), + currentAnalogueGain_(1.0) +{ +} + +/** + * \brief Configure the AGC given a configInfo + * \param[in] context The shared IPA context + * \param[in] configInfo The IPA configuration data, received from the pipeline + * handler + * + * \return 0 + */ +int AgcMetering::configure(IPAContext &context, const IPAConfigInfo &configInfo) +{ + /* Store the line duration in the IPASessionConfiguration */ + context.configuration.agc.lineDuration = configInfo.sensorInfo.lineLength + * (1.0s / configInfo.sensorInfo.pixelRate); + + /* \todo: those values need to be extracted from a configuration file */ + shutterConstraints_.push_back(100us); + shutterConstraints_.push_back(10ms); + shutterConstraints_.push_back(33ms); + gainConstraints_.push_back(1.0); + gainConstraints_.push_back(4.0); + gainConstraints_.push_back(16.0); + + fixedShutter_ = 0s; + fixedAnalogueGain_ = 0.0; + + return 0; +} + +/** + * \brief Translate the IPU3 statistics to AGC regions + * \param[in] stats The statistics buffer coming from the pipeline handler + * \param[in] grid The grid used to store the statistics in the IPU3 + */ +void AgcMetering::generateStats(const ipu3_uapi_stats_3a *stats, + const ipu3_uapi_grid_config &grid) +{ + /* We need to have a AGC grid of kAgcStatsSizeX * kAgcStatsSizeY */ + uint32_t regionWidth = round(grid.width / static_cast(kAgcStatsSizeX)); + uint32_t regionHeight = round(grid.height / static_cast(kAgcStatsSizeY)); + uint32_t hist[knumHistogramBins] = { 0 }; + + /* Clear the statistics of the previous frame */ + for (unsigned int i = 0; i < kNumAgcWeightedZones; i++) { + agcStats_[i].bSum = 0; + agcStats_[i].rSum = 0; + agcStats_[i].gSum = 0; + agcStats_[i].counted = 0; + agcStats_[i].total = 0; + } + + LOG(IPU3AgcMetering, Debug) << "[" << (int)grid.width + << "x" << (int)grid.height << "] cells" + << " scaled to [" << regionWidth + << "x" << regionHeight << "] AGC regions"; + + /* + * Generate a (kAgcStatsSizeX x kAgcStatsSizeY) array from the IPU3 grid + * which is (grid.width x grid.height). + */ + for (unsigned int j = 0; j < kAgcStatsSizeY * regionHeight; j++) { + for (unsigned int i = 0; i < kAgcStatsSizeX * regionWidth; i++) { + uint32_t cellPosition = j * grid.width + i; + uint32_t cellX = (cellPosition / regionWidth) + % kAgcStatsSizeX; + uint32_t cellY = ((cellPosition / grid.width) / regionHeight) + % kAgcStatsSizeY; + + uint32_t agcRegionPosition = kAgcStatsRegions[cellY * kAgcStatsSizeX + cellX]; + weights_[agcRegionPosition] = kSpotWeights[agcRegionPosition]; + cellPosition *= sizeof(Ipu3AwbCell); + + /* Cast the initial IPU3 structure to simplify the reading */ + Ipu3AwbCell *currentCell = reinterpret_cast(const_cast(&stats->awb_raw_buffer.meta_data[cellPosition])); + if (currentCell->satRatio == 0) { + /* The cell is not saturated, use the current cell */ + agcStats_[agcRegionPosition].counted++; + uint32_t greenValue = currentCell->greenRedAvg + currentCell->greenBlueAvg; + hist[greenValue / 2]++; + agcStats_[agcRegionPosition].gSum += greenValue / 2; + agcStats_[agcRegionPosition].rSum += currentCell->redAvg; + agcStats_[agcRegionPosition].bSum += currentCell->blueAvg; + } + } + } + + /* Estimate the quantile mean of the top 2% of the histogram */ + iqMean_ = Histogram(Span(hist)).interQuantileMean(0.98, 1.0); +} + +/** + * \brief Apply a filter on the exposure value to limit the speed of changes + */ +void AgcMetering::filterExposure() +{ + double speed = 0.08; + if (prevExposure_ == 0s) { + /* DG stands for digital gain.*/ + prevExposure_ = currentExposure_; + prevExposureNoDg_ = currentExposureNoDg_; + } else { + /* + * If we are close to the desired result, go faster to avoid + * making multiple micro-adjustments. + * \todo: Make this customisable? + */ + if (prevExposure_ < 1.2 * currentExposure_ && + prevExposure_ > 0.8 * currentExposure_) + speed = sqrt(speed); + + prevExposure_ = speed * currentExposure_ + + prevExposure_ * (1.0 - speed); + prevExposureNoDg_ = speed * currentExposureNoDg_ + + prevExposureNoDg_ * (1.0 - speed); + } + /* + * We can't let the no_dg exposure deviate too far below the + * total exposure, as there might not be enough digital gain available + * in the ISP to hide it (which will cause nasty oscillation). + * \todo: add the support for digital gain + */ + if (prevExposureNoDg_ < + prevExposure_ * kFastReduceThreshold) + prevExposureNoDg_ = prevExposure_ * kFastReduceThreshold; + LOG(IPU3AgcMetering, Debug) << "After filtering, total_exposure " << prevExposure_; +} + +/** + * \brief Estimate the weighted brightness + * \param[in] gain The current gain applied + * \param[in] context The shared IPA context + */ +double AgcMetering::computeInitialY(double gain, IPAContext &context) +{ + /* + * Note how the calculation below means that equal weights_ give you + * "average" metering (i.e. all pixels equally important). + */ + double redSum = 0, greenSum = 0, blueSum = 0, pixelSum = 0; + for (unsigned int i = 0; i < kNumAgcWeightedZones; i++) { + /* We will exclude the saturated pixels from the sum */ + double counted = agcStats_[i].counted; + double rSum = std::min(agcStats_[i].rSum * gain, ((1 << kPipelineBits) - 1) * counted); + double gSum = std::min(agcStats_[i].gSum * gain, ((1 << kPipelineBits) - 1) * counted); + double bSum = std::min(agcStats_[i].bSum * gain, ((1 << kPipelineBits) - 1) * counted); + /* Weight each channel with the selected metering method */ + redSum += rSum * weights_[i]; + greenSum += gSum * weights_[i]; + blueSum += bSum * weights_[i]; + pixelSum += counted * weights_[i]; + } + /* We don't want to have a division by 0.0 :-) */ + if (pixelSum == 0.0) { + LOG(IPU3AgcMetering, Warning) << "computeInitialY: pixel_sum is zero"; + return 0; + } + /* + * Estimate the sum of the brightness values, weighted with the gains + * applied on the channels in AWB. + */ + double Y_sum = redSum * context.frameContext.awb.gains.red * .299 + + greenSum * context.frameContext.awb.gains.green * .587 + + blueSum * context.frameContext.awb.gains.blue * .114; + + /* And return the average brightness */ + return Y_sum / pixelSum / (1 << kPipelineBits); +} + +/** + * \brief Compute the exposure value + * \param[in] gain The current gain applied + */ +void AgcMetering::computeTargetExposure(double gain) +{ + currentExposure_ = currentExposureNoDg_ * gain; + /* \todo: have a list of shutter speeds */ + Duration maxShutterSpeed = shutterConstraints_.back(); + Duration maxTotalExposure = maxShutterSpeed * gainConstraints_.back(); + + currentExposure_ = std::min(currentExposure_, maxTotalExposure); + LOG(IPU3AgcMetering, Debug) << "Target total_exposure " << currentExposure_; +} + +/** + * \brief Split exposure value as shutter time and gain + */ +void AgcMetering::divideUpExposure() +{ + Duration exposureValue = prevExposure_; + Duration shutterTime; + double analogueGain; + shutterTime = shutterConstraints_[0]; + shutterTime = std::min(shutterTime, shutterConstraints_.back()); + analogueGain = gainConstraints_[0]; + + /** + * We have an exposure profile with a list of shutter time and gains + * An example is graphed below: + * + * gain shutter time + * (ms) + * ^ ^ + * | | + * 8x+---------------------------------------------------------xxxxx+30 + * | xxxxx | + * | xxxxx | + * | xxxxx | + * | xxxxx | + * | xxxxx | + * | xxxxx | + * | xxxxx | + * | xxxxx | + * | xxxxx | + * 4x+----------xx--------------------------------------------------+10 + * | xx | + * | xx | + * | xx | + * | xx | + * | xx | + * 1x+--xx----------------------------------------------------------+0.1 + * | x | + * |x | + * +---------------------------------------------------------------> + * total exposure + */ + if (shutterTime * analogueGain < exposureValue) { + for (unsigned int stage = 1; + stage < gainConstraints_.size(); stage++) { + if (fixedShutter_ == 0s) { + Duration stageShutter = + std::min(shutterConstraints_[stage], shutterConstraints_.back()); + if (stageShutter * analogueGain >= + exposureValue) { + shutterTime = + exposureValue / analogueGain; + break; + } + shutterTime = stageShutter; + } + if (fixedAnalogueGain_ == 0.0) { + if (gainConstraints_[stage] * shutterTime >= exposureValue) { + analogueGain = exposureValue / shutterTime; + break; + } + analogueGain = gainConstraints_[stage]; + } + } + } + LOG(IPU3AgcMetering, Debug) << "Divided up shutter and gain are " + << shutterTime << " and " << analogueGain; + + /* \todo: flickering avoidance ? */ + filteredShutter_ = shutterTime; + filteredAnalogueGain_ = analogueGain; +} + +/** + * \brief Calculate the gain for the target value to be in the top 2% of the + * histogram + * \param[in] currentGain The current gain applied + * \param[in] context The shared IPA context + */ +void AgcMetering::computeGain(double ¤tGain, IPAContext &context) +{ + currentGain = 1.0; + /* \todo: the target Y needs to be grabbed from a configuration */ + double targetY = 0.162; + for (int i = 0; i < 8; i++) { + double initialY = computeInitialY(currentGain, context); + double extra_gain = std::min(10.0, targetY / (initialY + .001)); + + currentGain *= extra_gain; + LOG(IPU3AgcMetering, Debug) << "Initial Y " << initialY + << " target " << targetY + << " gives gain " << currentGain; + if (extra_gain < 1.01) + break; + } + + /* + * Require the top 2% of pixels to lie at or below 0.5 in the pixel + * range (for a range from 0 to 255, it is 205). This lowers the + * exposure to stop pixels saturating. + */ + double newGain = (0.5 * knumHistogramBins) / iqMean_; + LOG(IPU3AgcMetering, Debug) << "gain: " << currentGain + << " new gain: " << newGain; + if (newGain > currentGain) + currentGain = newGain; +} + +/** + * \brief Process IPU3 statistics, and run AGC operations + * \param[in] context The shared IPA context + * \param[in] stats The IPU3 statistics and ISP results + */ +void AgcMetering::process(IPAContext &context, const ipu3_uapi_stats_3a *stats) +{ + ASSERT(stats->stats_3a_status.awb_en); + generateStats(stats, context.configuration.grid.bdsGrid); + + currentShutter_ = context.frameContext.agc.exposure + * context.configuration.agc.lineDuration; + currentAnalogueGain_ = context.frameContext.agc.gain; + + /* Estimate the current exposure value */ + currentExposureNoDg_ = currentShutter_ * currentAnalogueGain_; + + double currentGain = 1.0; + computeGain(currentGain, context); + computeTargetExposure(currentGain); + filterExposure(); + divideUpExposure(); + + context.frameContext.agc.exposure = filteredShutter_ + / context.configuration.agc.lineDuration; + context.frameContext.agc.gain = filteredAnalogueGain_; +} + +} /* namespace ipa::ipu3::algorithms */ + +} /* namespace libcamera */ diff --git a/src/ipa/ipu3/algorithms/agc_metering.h b/src/ipa/ipu3/algorithms/agc_metering.h new file mode 100644 index 00000000..4fd603e1 --- /dev/null +++ b/src/ipa/ipu3/algorithms/agc_metering.h @@ -0,0 +1,78 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Based on the implementation from the Raspberry Pi IPA, + * Copyright (C) 2019-2021, Raspberry Pi (Trading) Ltd. + * Copyright (C) 2021, Ideas On Board + * + * agc_metering.h - IPU3 AGC/AEC control algorithm + */ +#ifndef __LIBCAMERA_IPU3_AGC_H__ +#define __LIBCAMERA_IPU3_AGC_H__ + +#include + +#include + +#include + +#include "algorithm.h" +#include "awb.h" + +namespace libcamera { + +struct IPACameraSensorInfo; + +namespace ipa::ipu3::algorithms { + +using utils::Duration; + +/* Number of weighted zones for metering */ +static constexpr uint32_t kNumAgcWeightedZones = 15; + +class AgcMetering : public Algorithm +{ +public: + AgcMetering(); + ~AgcMetering() = default; + + int configure(IPAContext &context, const IPAConfigInfo &configInfo) override; + void process(IPAContext &context, const ipu3_uapi_stats_3a *stats) override; + +private: + void processBrightness(const ipu3_uapi_stats_3a *stats); + void filterExposure(); + void lockExposureGain(uint32_t &exposure, double &gain); + void generateStats(const ipu3_uapi_stats_3a *stats, + const ipu3_uapi_grid_config &grid); + void generateZones(std::vector &zones); + double computeInitialY(double gain, IPAContext &context); + void computeTargetExposure(double currentGain); + void divideUpExposure(); + void computeGain(double ¤tGain, IPAContext &context); + + double weights_[kNumAgcWeightedZones]; + struct Accumulator agcStats_[kNumAgcWeightedZones]; + + double iqMean_; + + Duration prevExposure_; + Duration prevExposureNoDg_; + Duration currentExposure_; + Duration currentExposureNoDg_; + + Duration currentShutter_; + std::vector shutterConstraints_; + Duration fixedShutter_; + Duration filteredShutter_; + + double currentAnalogueGain_; + std::vector gainConstraints_; + double fixedAnalogueGain_; + double filteredAnalogueGain_; +}; + +} /* namespace ipa::ipu3::algorithms */ + +} /* namespace libcamera */ + +#endif /* __LIBCAMERA_IPU3_AGC_H__ */ diff --git a/src/ipa/ipu3/algorithms/meson.build b/src/ipa/ipu3/algorithms/meson.build index 807b53ea..f31b2070 100644 --- a/src/ipa/ipu3/algorithms/meson.build +++ b/src/ipa/ipu3/algorithms/meson.build @@ -2,6 +2,7 @@ ipu3_ipa_algorithms = files([ 'agc_mean.cpp', + 'agc_metering.cpp', 'algorithm.cpp', 'awb.cpp', 'tone_mapping.cpp', diff --git a/src/ipa/ipu3/ipa_context.h b/src/ipa/ipu3/ipa_context.h index 3a292ad7..190a3468 100644 --- a/src/ipa/ipu3/ipa_context.h +++ b/src/ipa/ipu3/ipa_context.h @@ -10,6 +10,8 @@ #include +#include + #include namespace libcamera { @@ -17,6 +19,10 @@ namespace libcamera { namespace ipa::ipu3 { struct IPASessionConfiguration { + struct { + utils::Duration lineDuration; + } agc; + struct { ipu3_uapi_grid_config bdsGrid; Size bdsOutputSize; diff --git a/src/ipa/ipu3/ipu3.cpp b/src/ipa/ipu3/ipu3.cpp index 6332fc06..be4a082a 100644 --- a/src/ipa/ipu3/ipu3.cpp +++ b/src/ipa/ipu3/ipu3.cpp @@ -81,6 +81,14 @@ * are run. This needs to be turned into real per-frame data storage. */ +/** + * \struct IPASessionConfiguration::agc + * \brief AGC configuration of the IPA + * + * \var IPASessionConfiguration::agc::lineDuration + * \brief Duration of one line dependant on the sensor configuration + */ + /** * \struct IPASessionConfiguration::grid * \brief Grid configuration of the IPA