From patchwork Mon Jul 12 13:16:30 2021 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Jean-Michel Hautbois X-Patchwork-Id: 12911 Return-Path: X-Original-To: parsemail@patchwork.libcamera.org Delivered-To: parsemail@patchwork.libcamera.org Received: from lancelot.ideasonboard.com (lancelot.ideasonboard.com [92.243.16.209]) by patchwork.libcamera.org (Postfix) with ESMTPS id 7245CC3226 for ; Mon, 12 Jul 2021 13:16:44 +0000 (UTC) Received: from lancelot.ideasonboard.com (localhost [IPv6:::1]) by lancelot.ideasonboard.com (Postfix) with ESMTP id D00DC6852B; Mon, 12 Jul 2021 15:16:40 +0200 (CEST) Authentication-Results: lancelot.ideasonboard.com; dkim=fail reason="signature verification failed" (1024-bit key; unprotected) header.d=ideasonboard.com header.i=@ideasonboard.com header.b="bxrkm45l"; dkim-atps=neutral Received: from perceval.ideasonboard.com (perceval.ideasonboard.com [213.167.242.64]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id B4CA068523 for ; Mon, 12 Jul 2021 15:16:36 +0200 (CEST) Received: from tatooine.ideasonboard.com (unknown [IPv6:2a01:e0a:169:7140:8515:881:eba9:bd61]) by perceval.ideasonboard.com (Postfix) with ESMTPSA id 6A3BBCC; Mon, 12 Jul 2021 15:16:36 +0200 (CEST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=ideasonboard.com; s=mail; t=1626095796; bh=AvrEUNy22xEBAGpLmTRAT/NPybCzo2iEeAXXV4J8b3k=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=bxrkm45l28K2TESX8wkrejQVVcAWt8w7mikewgetM0QEFY7/67rz2IebDUe1ewENq 7Udc06VbXdNP14PvIy2Y8EaRaXVLVcSUy/L+Nz2vkwFRZvglCxBZEo/sAcqokEX9HC kGqvBWsFRVXMO4fVZ3Y6mD0XZSaJGRcrzeeJoeKs= From: Jean-Michel Hautbois To: libcamera-devel@lists.libcamera.org Date: Mon, 12 Jul 2021 15:16:30 +0200 Message-Id: <20210712131630.73914-3-jeanmichel.hautbois@ideasonboard.com> X-Mailer: git-send-email 2.30.2 In-Reply-To: <20210712131630.73914-1-jeanmichel.hautbois@ideasonboard.com> References: <20210712131630.73914-1-jeanmichel.hautbois@ideasonboard.com> MIME-Version: 1.0 Subject: [libcamera-devel] [PATCH 2/2] ipa: ipu3: Use metadata and improve the doc X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: libcamera-devel-bounces@lists.libcamera.org Sender: "libcamera-devel" Using the metadata to exchange the status of the awb algorithm helps to simplify the interface. The structures used by the AWB statistics are in ipu3_awb.h and documented. A bit of the doc has been improved in this same patch. There is one metadata variable which will be used and set in the AGC algorithm in a near future, which is the agcGamma. Use it as if it exists, the Metadata class won't find it and awb will default to a 1.0 value. Doing it now is convenient to have nice process() and updateParamaters() calls. Signed-off-by: Jean-Michel Hautbois --- src/ipa/ipu3/ipu3.cpp | 8 ++- src/ipa/ipu3/ipu3_awb.cpp | 116 ++++++++++++++++++++++++++++---------- src/ipa/ipu3/ipu3_awb.h | 34 ++++++----- 3 files changed, 111 insertions(+), 47 deletions(-) diff --git a/src/ipa/ipu3/ipu3.cpp b/src/ipa/ipu3/ipu3.cpp index 091856f5..1a3d98e9 100644 --- a/src/ipa/ipu3/ipu3.cpp +++ b/src/ipa/ipu3/ipu3.cpp @@ -80,6 +80,8 @@ private: std::unique_ptr agcAlgo_; /* Interface to the Camera Helper */ std::unique_ptr camHelper_; + /* Metadata storage */ + Metadata metadata_; /* Local parameter storage */ struct ipu3_uapi_params params_; @@ -277,7 +279,7 @@ void IPAIPU3::processControls([[maybe_unused]] unsigned int frame, void IPAIPU3::fillParams(unsigned int frame, ipu3_uapi_params *params) { if (agcAlgo_->updateControls()) - awbAlgo_->updateWbParameters(params_, agcAlgo_->gamma()); + awbAlgo_->updateWbParameters(params_, &metadata_); *params = params_; @@ -297,8 +299,10 @@ void IPAIPU3::parseStatistics(unsigned int frame, agcAlgo_->process(stats, exposure_, gain); gain_ = camHelper_->gainCode(gain); - awbAlgo_->calculateWBGains(stats); + /* Calculate the AWB gains */ + awbAlgo_->process(stats, &metadata_); + /* Update the exposure and gains on sensor side */ if (agcAlgo_->updateControls()) setControls(frame); diff --git a/src/ipa/ipu3/ipu3_awb.cpp b/src/ipa/ipu3/ipu3_awb.cpp index 9b409c8f..d441e835 100644 --- a/src/ipa/ipu3/ipu3_awb.cpp +++ b/src/ipa/ipu3/ipu3_awb.cpp @@ -18,29 +18,54 @@ namespace ipa::ipu3 { LOG_DEFINE_CATEGORY(IPU3Awb) -static constexpr uint32_t kMinZonesCounted = 16; -static constexpr uint32_t kMinGreenLevelInZone = 32; +/** + * The Grey World algorithm assumes that the scene, in average, is neutral grey. + * Reference: Lam, Edmund & Fung, George. (2008). Automatic White Balancing in + * Digital Photography. 10.1201/9781420054538.ch10. + * + * The IPU3 is generating statistics from the Bayer Demosaic Scaler output + * into a grid defined in the ipu3_uapi_awb_config_s structure. + * + * For example, when the BDS output is 2592x1944 then the grid is calculated to be: + * 81*30 with a cell beeing of size 32*64. + * We then have an average of 2048 R, G and B pixels per cell. + * + * The AWB algorithm could use those variable grid sizes as an input, but it would + * make it a bit more complex. In order to have something consistent with what is + * done on RPi, fix a default grid size to kAwbStatsSizeX x kAwbStatsSizeY. + * + * Before calculating the gains, we will convert the statistics to go from the BDS + * grid configuration to this intern grid size in generateAwbStats. + * When the stats are converted, the saturation flag in the initial grid is used to + * decide if the zone is saturated or not, making the zone relevant or not. + * + * The Grey World algorithm will then estimate the red and blue gains to apply, and + * send those back through metadata. + */ /** - * \struct IspStatsRegion + * \struct StatsRegion * \brief RGB statistics for a given region * - * The IspStatsRegion structure is intended to abstract the ISP specific - * statistics and use an agnostic algorithm to compute AWB. + * The StatsRegion structure is intended to abstract the ISP specific + * statistics to compute AWB. The Grey World algorithm uses an average + * for a specific counted pixels. When a specific zone in the scene is + * saturated, we want to exclude it from the calculation, and consider + * it as an outlier. * - * \var IspStatsRegion::counted - * \brief Number of pixels used to calculate the sums + * \var StatsRegion::counted + * \brief Number of unsatured pixels used to calculate the sums * - * \var IspStatsRegion::uncounted - * \brief Remaining number of pixels in the region + * \var StatsRegion::uncounted + * \brief Remaining number of pixels in the region (ie saturated) * - * \var IspStatsRegion::rSum + * \var StatsRegion::rSum * \brief Sum of the red values in the region * - * \var IspStatsRegion::gSum + * \var StatsRegion::gSum * \brief Sum of the green values in the region * - * \var IspStatsRegion::bSum + * \var StatsRegion::bSum * \brief Sum of the blue values in the region */ @@ -48,26 +73,35 @@ static constexpr uint32_t kMinGreenLevelInZone = 32; * \struct AwbStatus * \brief AWB parameters calculated * - * The AwbStatus structure is intended to store the AWB - * parameters calculated by the algorithm + * The AwbStatus structure is intended to store the AWB parameters + * calculated by the algorithm, and shared through the metadata + * object, for other algorithms * * \var AwbStatus::temperatureK - * \brief Color temperature calculated + * \brief Color temperature calculated, in Kelvin * * \var AwbStatus::redGain - * \brief Gain calculated for the red channel + * \brief Gain calculated for the red channel. This is a floating-point + * value used as a multiplier on the ISP side * * \var AwbStatus::greenGain - * \brief Gain calculated for the green channel + * \brief Gain calculated for the green channel. This is a floating-point + * value used as a multiplier on the ISP side * * \var AwbStatus::blueGain - * \brief Gain calculated for the blue channel + * \brief Gain calculated for the blue channel. This is a floating-point + * value used as a multiplier on the ISP side */ /** * \struct Ipu3AwbCell * \brief Memory layout for each cell in AWB metadata * + * This is the internal layout on IPU3 for one cell of AWB statistics. + * There is ipu3_uapi_awb_config_s->grid.width * 2^block_width_log2 per + * ipu3_uapi_awb_config_s->grid.height * 2^block_height_log2 cells for + * one frame statistics. + * * The Ipu3AwbCell structure is used to get individual values * such as red average or saturation ratio in a particular cell. * @@ -84,7 +118,8 @@ static constexpr uint32_t kMinGreenLevelInZone = 32; * \brief Green average for blue lines * * \var Ipu3AwbCell::satRatio - * \brief Saturation ratio in the cell + * \brief Saturation ratio in the cell. It depends on the rgbs_thr_* values, and + * will be set if rgbs_thr_b has IPU3_UAPI_AWB_RGBS_THR_B_INCL_SAT bit set. * * \var Ipu3AwbCell::padding * \brief array of unused bytes for padding @@ -159,6 +194,9 @@ const struct ipu3_uapi_gamma_corr_lut imguCssGammaLut = { { 7807, 7871, 7935, 7999, 8063, 8127, 8191 } }; +/* Minimum level of green (on a 8 bits base) in a given zone */ +static constexpr uint32_t kMinGreenLevelInZone = 16; + IPU3Awb::IPU3Awb() : Algorithm() { @@ -166,6 +204,7 @@ IPU3Awb::IPU3Awb() asyncResults_.greenGain = 1.0; asyncResults_.redGain = 1.0; asyncResults_.temperatureK = 4500; + minZonesCounted_ = 0; } IPU3Awb::~IPU3Awb() @@ -202,7 +241,7 @@ void IPU3Awb::initialise(ipu3_uapi_params ¶ms, const Size &bdsOutputSize, st params.acc_param.gamma.gc_lut = imguCssGammaLut; params.acc_param.gamma.gc_ctrl.enable = 1; - zones_.reserve(kAwbStatsSizeX * kAwbStatsSizeY); + zones_.reserve(kAwbStatsSize); } /** @@ -238,10 +277,10 @@ uint32_t IPU3Awb::estimateCCT(double red, double green, double blue) /* Generate an RGB vector with the average values for each region */ void IPU3Awb::generateZones(std::vector &zones) { - for (unsigned int i = 0; i < kAwbStatsSizeX * kAwbStatsSizeY; i++) { + for (unsigned int i = 0; i < kAwbStatsSize; i++) { RGB zone; double counted = awbStats_[i].counted; - if (counted >= kMinZonesCounted) { + if (counted >= minZonesCounted_) { zone.G = awbStats_[i].gSum / counted; if (zone.G >= kMinGreenLevelInZone) { zone.R = awbStats_[i].rSum / counted; @@ -258,6 +297,7 @@ void IPU3Awb::generateAwbStats(const ipu3_uapi_stats_3a *stats) uint32_t regionWidth = round(awbGrid_.width / static_cast(kAwbStatsSizeX)); uint32_t regionHeight = round(awbGrid_.height / static_cast(kAwbStatsSizeY)); + minZonesCounted_ = ((regionWidth * regionHeight) * 4) / 5; /* * Generate a (kAwbStatsSizeX x kAwbStatsSizeY) array from the IPU3 grid which is * (awbGrid_.width x awbGrid_.height). @@ -269,7 +309,7 @@ void IPU3Awb::generateAwbStats(const ipu3_uapi_stats_3a *stats) uint32_t cellY = ((cellPosition / awbGrid_.width) / regionHeight) % kAwbStatsSizeY; uint32_t awbRegionPosition = cellY * kAwbStatsSizeX + cellX; - cellPosition *= 8; + cellPosition *= sizeof(Ipu3AwbCell); /* Cast the initial IPU3 structure to simplify the reading */ Ipu3AwbCell *currentCell = reinterpret_cast(const_cast(&stats->awb_raw_buffer.meta_data[cellPosition])); @@ -287,7 +327,7 @@ void IPU3Awb::generateAwbStats(const ipu3_uapi_stats_3a *stats) void IPU3Awb::clearAwbStats() { - for (unsigned int i = 0; i < kAwbStatsSizeX * kAwbStatsSizeY; i++) { + for (unsigned int i = 0; i < kAwbStatsSize; i++) { awbStats_[i].bSum = 0; awbStats_[i].rSum = 0; awbStats_[i].gSum = 0; @@ -344,24 +384,38 @@ void IPU3Awb::calculateWBGains(const ipu3_uapi_stats_3a *stats) generateAwbStats(stats); generateZones(zones_); LOG(IPU3Awb, Debug) << "Valid zones: " << zones_.size(); - if (zones_.size() > 10) { + + /* We need at least 5% of valid zones to estimate the gain correction */ + if (zones_.size() > kAwbStatsSize / 20) { awbGreyWorld(); LOG(IPU3Awb, Debug) << "Gain found for red: " << asyncResults_.redGain << " and for blue: " << asyncResults_.blueGain; } } -void IPU3Awb::updateWbParameters(ipu3_uapi_params ¶ms, double agcGamma) +void IPU3Awb::process(const ipu3_uapi_stats_3a *stats, Metadata *imageMetadata) +{ + calculateWBGains(stats); + /* We need to update the AWB status, to give back the gains */ + imageMetadata->set("awb.status", asyncResults_); +} + +void IPU3Awb::updateWbParameters(ipu3_uapi_params ¶ms, Metadata *imageMetadata) { /* * Green gains should not be touched and considered 1. * Default is 16, so do not change it at all. - * 4096 is the value for a gain of 1.0 + * 8192 is the value for a gain of 1.0 */ - params.acc_param.bnr.wb_gains.gr = 16; - params.acc_param.bnr.wb_gains.r = 4096 * asyncResults_.redGain; - params.acc_param.bnr.wb_gains.b = 4096 * asyncResults_.blueGain; - params.acc_param.bnr.wb_gains.gb = 16; + params.acc_param.bnr.wb_gains.gr = 8192; + params.acc_param.bnr.wb_gains.r = 8192 * asyncResults_.redGain; + params.acc_param.bnr.wb_gains.b = 8192 * asyncResults_.blueGain; + params.acc_param.bnr.wb_gains.gb = 8192; + + /* When the AGC algorithm has run, it may have set a new gamma */ + double agcGamma = 1.0; + if (imageMetadata->get("agc.gamma", agcGamma) != 0) + LOG(IPU3Awb, Debug) << "Awb: no gamma found, defaulted to 1.0"; LOG(IPU3Awb, Debug) << "Color temperature estimated: " << asyncResults_.temperatureK << " and gamma calculated: " << agcGamma; diff --git a/src/ipa/ipu3/ipu3_awb.h b/src/ipa/ipu3/ipu3_awb.h index 122cf68c..cf12032d 100644 --- a/src/ipa/ipu3/ipu3_awb.h +++ b/src/ipa/ipu3/ipu3_awb.h @@ -14,14 +14,18 @@ #include #include "libipa/algorithm.h" +#include "libipa/metadata.h" namespace libcamera { namespace ipa::ipu3 { -/* Region size for the statistics generation algorithm */ +/* Width of the AWB regions used for Grey World calculation */ static constexpr uint32_t kAwbStatsSizeX = 16; +/* Height of the AWB regions used for Grey World calculation */ static constexpr uint32_t kAwbStatsSizeY = 12; +/* Total size of the AWB regions used for Grey World calculation */ +static constexpr uint32_t kAwbStatsSize = kAwbStatsSizeX * kAwbStatsSizeY; class IPU3Awb : public Algorithm { @@ -30,17 +34,8 @@ public: ~IPU3Awb(); void initialise(ipu3_uapi_params ¶ms, const Size &bdsOutputSize, struct ipu3_uapi_grid_config &bdsGrid); - void calculateWBGains(const ipu3_uapi_stats_3a *stats); - void updateWbParameters(ipu3_uapi_params ¶ms, double agcGamma); - - struct Ipu3AwbCell { - unsigned char greenRedAvg; - unsigned char redAvg; - unsigned char blueAvg; - unsigned char greenBlueAvg; - unsigned char satRatio; - unsigned char padding[3]; - } __attribute__((packed)); + void process(const ipu3_uapi_stats_3a *stats, Metadata *imageMetadata); + void updateWbParameters(ipu3_uapi_params ¶ms, Metadata *imageMetadata); /* \todo Make these three structs available to all the ISPs ? */ struct RGB { @@ -56,7 +51,7 @@ public: } }; - struct IspStatsRegion { + struct StatsRegion { unsigned int counted; unsigned int uncounted; unsigned long long rSum; @@ -71,18 +66,29 @@ public: double blueGain; }; + struct Ipu3AwbCell { + unsigned char greenRedAvg; + unsigned char redAvg; + unsigned char blueAvg; + unsigned char greenBlueAvg; + unsigned char satRatio; + unsigned char padding[3]; + }; + private: void generateZones(std::vector &zones); void generateAwbStats(const ipu3_uapi_stats_3a *stats); void clearAwbStats(); void awbGreyWorld(); uint32_t estimateCCT(double red, double green, double blue); + void calculateWBGains(const ipu3_uapi_stats_3a *stats); struct ipu3_uapi_grid_config awbGrid_; std::vector zones_; - IspStatsRegion awbStats_[kAwbStatsSizeX * kAwbStatsSizeY]; + StatsRegion awbStats_[kAwbStatsSize]; AwbStatus asyncResults_; + uint32_t minZonesCounted_; }; } /* namespace ipa::ipu3 */