From patchwork Fri Apr 16 07:49:06 2021 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Jean-Michel Hautbois X-Patchwork-Id: 11958 Return-Path: X-Original-To: parsemail@patchwork.libcamera.org Delivered-To: parsemail@patchwork.libcamera.org Received: from lancelot.ideasonboard.com (lancelot.ideasonboard.com [92.243.16.209]) by patchwork.libcamera.org (Postfix) with ESMTPS id 5ECAEBD235 for ; Fri, 16 Apr 2021 07:49:15 +0000 (UTC) Received: from lancelot.ideasonboard.com (localhost [IPv6:::1]) by lancelot.ideasonboard.com (Postfix) with ESMTP id E65D26880E; Fri, 16 Apr 2021 09:49:14 +0200 (CEST) Authentication-Results: lancelot.ideasonboard.com; dkim=fail reason="signature verification failed" (1024-bit key; unprotected) header.d=ideasonboard.com header.i=@ideasonboard.com header.b="hiH9Wf/Z"; dkim-atps=neutral Received: from perceval.ideasonboard.com (perceval.ideasonboard.com [213.167.242.64]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id 0570B68803 for ; Fri, 16 Apr 2021 09:49:13 +0200 (CEST) Received: from localhost.localdomain (unknown [IPv6:2a01:e0a:169:7140:5b63:445c:7960:347a]) by perceval.ideasonboard.com (Postfix) with ESMTPSA id 7752D6F2; Fri, 16 Apr 2021 09:49:12 +0200 (CEST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=ideasonboard.com; s=mail; t=1618559352; bh=VGhDvdOozrLwBj9ocsQecnLamsF1P2b0CqyFd4fsCA8=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=hiH9Wf/ZW4flnicx7IW6cxCxvvG8T58xohVY2RcBSFDjiDOtCZ4EXwJHYhjKmy+Ti scBtfRMOCPC8nOgVCpfYSaFVRz68noyU/GpsubLCZKuPoXYcdedss92UBt/Eo4/d12 2PIxzuvzojQBaBEg3AiJZt9vHCdL+us3cR09QxSY= From: Jean-Michel Hautbois To: libcamera-devel@lists.libcamera.org Date: Fri, 16 Apr 2021 09:49:06 +0200 Message-Id: <20210416074909.24218-2-jeanmichel.hautbois@ideasonboard.com> X-Mailer: git-send-email 2.27.0 In-Reply-To: <20210416074909.24218-1-jeanmichel.hautbois@ideasonboard.com> References: <20210416074909.24218-1-jeanmichel.hautbois@ideasonboard.com> MIME-Version: 1.0 Subject: [libcamera-devel] [PATCH v5 1/4] ipa: Add a common interface for algorithm objects X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: libcamera-devel-bounces@lists.libcamera.org Sender: "libcamera-devel" In order to instantiate and use algorithms (AWB, AGC, etc.) there is a need for a common class to define mandatory methods. Instead of reinventing the wheel, reuse what Raspberry Pi has done and adapt to the minimum requirements expected. Signed-off-by: Jean-Michel Hautbois Reviewed-by: Laurent Pinchart Reviewed-by: Kieran Bingham --- src/ipa/libipa/algorithm.cpp | 39 ++++++++++++++++++++++++++++++++++++ src/ipa/libipa/algorithm.h | 24 ++++++++++++++++++++++ src/ipa/libipa/meson.build | 4 +++- 3 files changed, 66 insertions(+), 1 deletion(-) create mode 100644 src/ipa/libipa/algorithm.cpp create mode 100644 src/ipa/libipa/algorithm.h diff --git a/src/ipa/libipa/algorithm.cpp b/src/ipa/libipa/algorithm.cpp new file mode 100644 index 00000000..930f9353 --- /dev/null +++ b/src/ipa/libipa/algorithm.cpp @@ -0,0 +1,39 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2021, Ideas On Board + * + * algorithm.cpp - ISP control algorithms + */ + +#include "algorithm.h" + +/** + * \file algorithm.h + * \brief Algorithm common interface + */ + +namespace libcamera { + +/** + * \brief The IPA namespace + * + * The IPA namespace groups all types specific to IPA modules. It serves as the + * top-level namespace for the IPA library libipa, and also contains + * module-specific namespaces for IPA modules. + */ +namespace ipa { + +/** + * \class Algorithm + * \brief The base class for all IPA algorithms + * + * The Algorithm class defines a standard interface for IPA algorithms. By + * abstracting algorithms, it makes possible the implementation of generic code + * to manage algorithms regardless of their specific type. + */ + +Algorithm::~Algorithm() = default; + +} /* namespace ipa */ + +} /* namespace libcamera */ diff --git a/src/ipa/libipa/algorithm.h b/src/ipa/libipa/algorithm.h new file mode 100644 index 00000000..89cee4c4 --- /dev/null +++ b/src/ipa/libipa/algorithm.h @@ -0,0 +1,24 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2021, Ideas On Board + * + * algorithm.h - ISP control algorithm interface + */ +#ifndef __LIBCAMERA_IPA_LIBIPA_ALGORITHM_H__ +#define __LIBCAMERA_IPA_LIBIPA_ALGORITHM_H__ + +namespace libcamera { + +namespace ipa { + +class Algorithm +{ +public: + virtual ~Algorithm(); +}; + +} /* namespace ipa */ + +} /* namespace libcamera */ + +#endif /* __LIBCAMERA_IPA_LIBIPA_ALGORITHM_H__ */ diff --git a/src/ipa/libipa/meson.build b/src/ipa/libipa/meson.build index b29ef0f4..1819711d 100644 --- a/src/ipa/libipa/meson.build +++ b/src/ipa/libipa/meson.build @@ -1,13 +1,15 @@ # SPDX-License-Identifier: CC0-1.0 libipa_headers = files([ + 'algorithm.h', ]) libipa_sources = files([ + 'algorithm.cpp', ]) libipa_includes = include_directories('..') -libipa = static_library('ipa', libipa_sources, +libipa = static_library('ipa', [libipa_sources, libipa_headers], include_directories : ipa_includes, dependencies : libcamera_dep) From patchwork Fri Apr 16 07:49:07 2021 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Patchwork-Submitter: Jean-Michel Hautbois X-Patchwork-Id: 11959 Return-Path: X-Original-To: parsemail@patchwork.libcamera.org Delivered-To: parsemail@patchwork.libcamera.org Received: from lancelot.ideasonboard.com (lancelot.ideasonboard.com [92.243.16.209]) by patchwork.libcamera.org (Postfix) with ESMTPS id 33FFDBD235 for ; Fri, 16 Apr 2021 07:49:16 +0000 (UTC) Received: from lancelot.ideasonboard.com (localhost [IPv6:::1]) by lancelot.ideasonboard.com (Postfix) with ESMTP id 55ADC68816; Fri, 16 Apr 2021 09:49:15 +0200 (CEST) Authentication-Results: lancelot.ideasonboard.com; dkim=fail reason="signature verification failed" (1024-bit key; unprotected) header.d=ideasonboard.com header.i=@ideasonboard.com header.b="kqgen6xY"; dkim-atps=neutral Received: from perceval.ideasonboard.com (perceval.ideasonboard.com [213.167.242.64]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id A47576880E for ; Fri, 16 Apr 2021 09:49:13 +0200 (CEST) Received: from localhost.localdomain (unknown [IPv6:2a01:e0a:169:7140:5b63:445c:7960:347a]) by perceval.ideasonboard.com (Postfix) with ESMTPSA id 4C7DB5A5; Fri, 16 Apr 2021 09:49:13 +0200 (CEST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=ideasonboard.com; s=mail; t=1618559353; bh=l1kyhlXZo2JexfDP6i3vOBsI/GZ1w7A+8gwJ8PpQrEE=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=kqgen6xYmxpYG9c9otdszwscR6EPWcpwV0qZDk4Lmdeq5RFTmiHYpv7Rvi6gF3iBf ddc5FnlBskKYsTEgR8GmSENKMJNWkx3Eilo9zQlruRupUSmC6r5LlrJRq+VrqXVmFp YE6AV2zErgZElSJgs7A0iKOMD7epm1bJU90d5Rbg= From: Jean-Michel Hautbois To: libcamera-devel@lists.libcamera.org Date: Fri, 16 Apr 2021 09:49:07 +0200 Message-Id: <20210416074909.24218-3-jeanmichel.hautbois@ideasonboard.com> X-Mailer: git-send-email 2.27.0 In-Reply-To: <20210416074909.24218-1-jeanmichel.hautbois@ideasonboard.com> References: <20210416074909.24218-1-jeanmichel.hautbois@ideasonboard.com> MIME-Version: 1.0 Subject: [libcamera-devel] [PATCH v5 2/4] ipa: ipu3: Add a histogram class X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: libcamera-devel-bounces@lists.libcamera.org Sender: "libcamera-devel" This class will be used at least by AGC algorithm when quantiles are needed for example. It stores a cumulative frequency histogram. Going from cumulative frequency back to per-bin values is a single subtraction, while going the other way is a loop. Signed-off-by: Jean-Michel Hautbois Reviewed-by: Kieran Bingham --- src/ipa/libipa/histogram.cpp | 150 +++++++++++++++++++++++++++++++++++ src/ipa/libipa/histogram.h | 40 ++++++++++ src/ipa/libipa/meson.build | 2 + 3 files changed, 192 insertions(+) create mode 100644 src/ipa/libipa/histogram.cpp create mode 100644 src/ipa/libipa/histogram.h diff --git a/src/ipa/libipa/histogram.cpp b/src/ipa/libipa/histogram.cpp new file mode 100644 index 00000000..b92d8305 --- /dev/null +++ b/src/ipa/libipa/histogram.cpp @@ -0,0 +1,150 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (C) 2019, Raspberry Pi (Trading) Limited + * + * histogram.cpp - histogram calculations + */ +#include "histogram.h" + +#include + +#include "libcamera/internal/log.h" + +/** + * \file histogram.h + * \brief Class to represent Histograms and manipulate them + */ + +namespace libcamera { + +namespace ipa { + +/** + * \class Histogram + * \brief The base class for creating histograms + * + * This class stores a cumulative frequency histogram, which is a mapping that + * counts the cumulative number of observations in all of the bins up to the + * specified bin. It can be used to find quantiles and averages between quantiles. + */ + +/** + * \brief Create a cumulative histogram + * \param[in] data A pre-sorted histogram to be passed + */ +Histogram::Histogram(Span data) +{ + cumulative_.reserve(data.size()); + cumulative_.push_back(0); + for (const uint32_t &value : data) + cumulative_.push_back(cumulative_.back() + value); +} +/** + * \fn Histogram::bins() + * \brief Retrieve the number of bins currently used by the Histogram + * \return Number of bins + */ +/** + * \fn Histogram::total() + * \brief Retrieve the total number of values in the data set + * \return Number of values + */ + +/** + * \brief Cumulative frequency up to a (fractional) point in a bin. + * \param[in] bin The bin up to which to cumulate + * + * With F(p) the cumulative frequency of the histogram, the value is 0 at + * the bottom of the histogram, and the maximum is the number of bins. + * The pixels are spread evenly throughout the “bin” in which they lie, so that + * F(p) is a continuous (monotonically increasing) function. + * + * \return The cumulative frequency from 0 up to the specified bin + */ +uint64_t Histogram::cumulativeFrequency(double bin) const +{ + if (bin <= 0) + return 0; + else if (bin >= bins()) + return total(); + int b = static_cast(bin); + return cumulative_[b] + + (bin - b) * (cumulative_[b + 1] - cumulative_[b]); +} + +/** + * \brief Return the (fractional) bin of the point through the histogram + * \param[in] q the desired point (0 <= q <= 1) + * \param[in] first low limit (default is 0) + * \param[in] last high limit (default is UINT_MAX) + * + * A quantile gives us the point p = Q(q) in the range such that a proportion + * q of the pixels lie below p. A familiar quantile is Q(0.5) which is the median + * of a distribution. + * + * \return The fractional bin of the point + */ +double Histogram::quantile(double q, uint32_t first, uint32_t last) const +{ + if (last == UINT_MAX) + last = cumulative_.size() - 2; + ASSERT(first <= last); + + uint64_t item = q * total(); + /* Binary search to find the right bin */ + while (first < last) { + int middle = (first + last) / 2; + /* Is it between first and middle ? */ + if (cumulative_[middle + 1] > item) + last = middle; + else + first = middle + 1; + } + ASSERT(item >= cumulative_[first] && item <= cumulative_[last + 1]); + + double frac; + if (cumulative_[first + 1] == cumulative_[first]) + frac = 0; + else + frac = (item - cumulative_[first]) / (cumulative_[first + 1] - cumulative_[first]); + return first + frac; +} + +/** + * \brief Calculate the mean between two quantiles + * \param[in] lowQuantile low Quantile + * \param[in] highQuantile high Quantile + * + * Quantiles are not ideal for metering as they suffer several limitations. + * Instead, a concept is introduced here: inter-quantile mean. + * It returns the mean of all pixels between lowQuantile and highQuantile. + * + * \return The mean histogram bin value between the two quantiles + */ +double Histogram::interQuantileMean(double lowQuantile, double highQuantile) const +{ + ASSERT(highQuantile > lowQuantile); + /* Proportion of pixels which lies below lowQuantile */ + double lowPoint = quantile(lowQuantile); + /* Proportion of pixels which lies below highQuantile */ + double highPoint = quantile(highQuantile, static_cast(lowPoint)); + double sumBinFreq = 0, cumulFreq = 0; + + for (double p_next = floor(lowPoint) + 1.0; + p_next <= ceil(highPoint); + lowPoint = p_next, p_next += 1.0) { + int bin = floor(lowPoint); + double freq = (cumulative_[bin + 1] - cumulative_[bin]) * (std::min(p_next, highPoint) - lowPoint); + + /* Accumulate weigthed bin */ + sumBinFreq += bin * freq; + /* Accumulate weights */ + cumulFreq += freq; + } + /* add 0.5 to give an average for bin mid-points */ + return sumBinFreq / cumulFreq + 0.5; +} + +} /* namespace ipa */ + +} /* namespace libcamera */ diff --git a/src/ipa/libipa/histogram.h b/src/ipa/libipa/histogram.h new file mode 100644 index 00000000..e06f1884 --- /dev/null +++ b/src/ipa/libipa/histogram.h @@ -0,0 +1,40 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (C) 2019, Raspberry Pi (Trading) Limited + * + * histogram.h - histogram calculation interface + */ +#ifndef __LIBCAMERA_IPA_LIBIPA_HISTOGRAM_H__ +#define __LIBCAMERA_IPA_LIBIPA_HISTOGRAM_H__ + +#include +#include +#include + +#include + +#include + +namespace libcamera { + +namespace ipa { + +class Histogram +{ +public: + Histogram(Span data); + size_t bins() const { return cumulative_.size() - 1; } + uint64_t total() const { return cumulative_[cumulative_.size() - 1]; } + uint64_t cumulativeFrequency(double bin) const; + double quantile(double q, uint32_t first = 0, uint32_t last = UINT_MAX) const; + double interQuantileMean(double lowQuantile, double hiQuantile) const; + +private: + std::vector cumulative_; +}; + +} /* namespace ipa */ + +} /* namespace libcamera */ + +#endif /* __LIBCAMERA_IPA_LIBIPA_HISTOGRAM_H__ */ diff --git a/src/ipa/libipa/meson.build b/src/ipa/libipa/meson.build index 1819711d..038fc490 100644 --- a/src/ipa/libipa/meson.build +++ b/src/ipa/libipa/meson.build @@ -2,10 +2,12 @@ libipa_headers = files([ 'algorithm.h', + 'histogram.h' ]) libipa_sources = files([ 'algorithm.cpp', + 'histogram.cpp' ]) libipa_includes = include_directories('..') From patchwork Fri Apr 16 07:49:08 2021 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Jean-Michel Hautbois X-Patchwork-Id: 11960 Return-Path: X-Original-To: parsemail@patchwork.libcamera.org Delivered-To: parsemail@patchwork.libcamera.org Received: from lancelot.ideasonboard.com (lancelot.ideasonboard.com [92.243.16.209]) by patchwork.libcamera.org (Postfix) with ESMTPS id 4561BBD235 for ; Fri, 16 Apr 2021 07:49:17 +0000 (UTC) Received: from lancelot.ideasonboard.com (localhost [IPv6:::1]) by lancelot.ideasonboard.com (Postfix) with ESMTP id EB4B568822; Fri, 16 Apr 2021 09:49:16 +0200 (CEST) Authentication-Results: lancelot.ideasonboard.com; dkim=fail reason="signature verification failed" (1024-bit key; unprotected) header.d=ideasonboard.com header.i=@ideasonboard.com header.b="qDVk56uZ"; dkim-atps=neutral Received: from perceval.ideasonboard.com (perceval.ideasonboard.com [213.167.242.64]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id 5AE016880C for ; Fri, 16 Apr 2021 09:49:14 +0200 (CEST) Received: from localhost.localdomain (unknown [IPv6:2a01:e0a:169:7140:5b63:445c:7960:347a]) by perceval.ideasonboard.com (Postfix) with ESMTPSA id E9F906F2; Fri, 16 Apr 2021 09:49:13 +0200 (CEST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=ideasonboard.com; s=mail; t=1618559354; bh=iwgxUULtKQLUMktvlxHjZ+aJ7PEFL76PoHQB3vYrSeI=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=qDVk56uZJpE6T5+U/amHT4XKwYobCULKWH9p03v6qoxeYJzsY/8s+hoEKuDdgdwzM AUxVFpTGEtMpkWGASsJm9m9jWko3JOMCp0IkcqD++3wbaDnH7ryGyfjSDKCk1T96M6 a18xOeM09X5bcW1ZvZ4wPEyahRAcDGaAmKjjibDI= From: Jean-Michel Hautbois To: libcamera-devel@lists.libcamera.org Date: Fri, 16 Apr 2021 09:49:08 +0200 Message-Id: <20210416074909.24218-4-jeanmichel.hautbois@ideasonboard.com> X-Mailer: git-send-email 2.27.0 In-Reply-To: <20210416074909.24218-1-jeanmichel.hautbois@ideasonboard.com> References: <20210416074909.24218-1-jeanmichel.hautbois@ideasonboard.com> MIME-Version: 1.0 Subject: [libcamera-devel] [PATCH v5 3/4] ipa: ipu3: Add support for IPU3 AWB algorithm X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: libcamera-devel-bounces@lists.libcamera.org Sender: "libcamera-devel" The IPA will locally modify the parameters before they are passed down to the ImgU. Use a local parameter object to give a reference to those algorithms. Inherit from the Algorithm class to implement basic AWB functions. The configure() call will set exposure and gain to their minimum value, so while AGC is not there, the frames will be dark. Once AWB is done, a color temperature is estimated and a default CCM matrix will be used (yet to be tuned). Implement a basic "grey-world" AWB algorithm just for demonstration purpose. The BDS output size is passed by the pipeline handler to the IPA. The best grid is then calculated to maximize the number of pixels taken into account in each cells. As commented in the source code, it can be improved, as it has (at least) one limitation: if a cell is big (say 128 pixels wide) and indicated as saturated, it won't be taken into account at all. Maybe is it possible to have a smaller one, at the cost of a few pixels to lose, in which case we can center the grid using the x_start and y_start parameters. Signed-off-by: Jean-Michel Hautbois Reviewed-by: Kieran Bingham --- src/ipa/ipu3/ipu3.cpp | 86 +++++++++- src/ipa/ipu3/ipu3_awb.cpp | 351 ++++++++++++++++++++++++++++++++++++++ src/ipa/ipu3/ipu3_awb.h | 91 ++++++++++ src/ipa/ipu3/meson.build | 7 +- 4 files changed, 526 insertions(+), 9 deletions(-) create mode 100644 src/ipa/ipu3/ipu3_awb.cpp create mode 100644 src/ipa/ipu3/ipu3_awb.h diff --git a/src/ipa/ipu3/ipu3.cpp b/src/ipa/ipu3/ipu3.cpp index 34a907f2..ab052a8a 100644 --- a/src/ipa/ipu3/ipu3.cpp +++ b/src/ipa/ipu3/ipu3.cpp @@ -21,6 +21,11 @@ #include "libcamera/internal/buffer.h" #include "libcamera/internal/log.h" +#include "ipu3_awb.h" + +static constexpr uint32_t kMaxCellWidthPerSet = 160; +static constexpr uint32_t kMaxCellHeightPerSet = 80; + namespace libcamera { LOG_DEFINE_CATEGORY(IPAIPU3) @@ -49,6 +54,7 @@ private: const ipu3_uapi_stats_3a *stats); void setControls(unsigned int frame); + void calculateBdsGrid(const Size &bdsOutputSize); std::map buffers_; @@ -61,6 +67,14 @@ private: uint32_t gain_; uint32_t minGain_; uint32_t maxGain_; + + /* Interface to the AWB algorithm */ + std::unique_ptr awbAlgo_; + + /* Local parameter storage */ + struct ipu3_uapi_params params_; + + struct ipu3_uapi_grid_config bdsGrid_; }; int IPAIPU3::start() @@ -70,8 +84,58 @@ int IPAIPU3::start() return 0; } +/** + * This method calculates a grid for the AWB algorithm in the IPU3 firmware. + * Its input is the BDS output size calculated in the ImgU. + * It is limited for now to the simplest method: find the lesser error + * with the width/height and respective log2 width/height of the cells. + * + * \todo The frame is divided into cells which can be 8x8 => 128x128. + * As a smaller cell improves the algorithm precision, adapting the + * x_start and y_start parameters of the grid would provoke a loss of + * some pixels but would also result in more accurate algorithms. + */ +void IPAIPU3::calculateBdsGrid(const Size &bdsOutputSize) +{ + uint32_t minError = std::numeric_limits::max(); + Size best; + Size bestLog2; + bdsGrid_ = {}; + + for (uint32_t widthShift = 3; widthShift <= 7; ++widthShift) { + uint32_t width = std::min(kMaxCellWidthPerSet, + bdsOutputSize.width >> widthShift); + width = width << widthShift; + for (uint32_t heightShift = 3; heightShift <= 7; ++heightShift) { + int32_t height = std::min(kMaxCellHeightPerSet, + bdsOutputSize.height >> heightShift); + height = height << heightShift; + uint32_t error = std::abs(static_cast(width - bdsOutputSize.width)) + + std::abs(static_cast(height - bdsOutputSize.height)); + + if (error > minError) + continue; + + minError = error; + best.width = width; + best.height = height; + bestLog2.width = widthShift; + bestLog2.height = heightShift; + } + } + + bdsGrid_.width = best.width >> bestLog2.width; + bdsGrid_.block_width_log2 = bestLog2.width; + bdsGrid_.height = best.height >> bestLog2.height; + bdsGrid_.block_height_log2 = bestLog2.height; + + LOG(IPAIPU3, Debug) << "Best grid found is: (" + << (int)bdsGrid_.width << " << " << (int)bdsGrid_.block_width_log2 << ") x (" + << (int)bdsGrid_.height << " <<" << (int)bdsGrid_.block_height_log2 << ")"; +} + void IPAIPU3::configure(const std::map &entityControls, - [[maybe_unused]] const Size &bdsOutputSize) + const Size &bdsOutputSize) { if (entityControls.empty()) return; @@ -92,11 +156,18 @@ void IPAIPU3::configure(const std::map &entityControls minExposure_ = std::max(itExp->second.min().get(), 1); maxExposure_ = itExp->second.max().get(); - exposure_ = maxExposure_; + exposure_ = minExposure_; minGain_ = std::max(itGain->second.min().get(), 1); maxGain_ = itGain->second.max().get(); - gain_ = maxGain_; + gain_ = minGain_; + + params_ = {}; + + calculateBdsGrid(bdsOutputSize); + + awbAlgo_ = std::make_unique(); + awbAlgo_->initialise(params_, bdsOutputSize, bdsGrid_); } void IPAIPU3::mapBuffers(const std::vector &buffers) @@ -168,10 +239,10 @@ void IPAIPU3::processControls([[maybe_unused]] unsigned int frame, void IPAIPU3::fillParams(unsigned int frame, ipu3_uapi_params *params) { - /* Prepare parameters buffer. */ - memset(params, 0, sizeof(*params)); + /* Pass a default gamma of 1.0 (default linear correction) */ + awbAlgo_->updateWbParameters(params_, 1.0); - /* \todo Fill in parameters buffer. */ + *params = params_; ipa::ipu3::IPU3Action op; op.op = ipa::ipu3::ActionParamFilled; @@ -184,8 +255,7 @@ void IPAIPU3::parseStatistics(unsigned int frame, { ControlList ctrls(controls::controls); - /* \todo React to statistics and update internal state machine. */ - /* \todo Add meta-data information to ctrls. */ + awbAlgo_->calculateWBGains(stats); ipa::ipu3::IPU3Action op; op.op = ipa::ipu3::ActionMetadataReady; diff --git a/src/ipa/ipu3/ipu3_awb.cpp b/src/ipa/ipu3/ipu3_awb.cpp new file mode 100644 index 00000000..b4920e4b --- /dev/null +++ b/src/ipa/ipu3/ipu3_awb.cpp @@ -0,0 +1,351 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2021, Ideas On Board + * + * ipu3_awb.cpp - AWB control algorithm + */ +#include "ipu3_awb.h" + +#include +#include +#include + +#include "libcamera/internal/log.h" + +namespace libcamera { + +namespace ipa { + +LOG_DEFINE_CATEGORY(IPU3Awb) + +/** + * \struct IspStatsRegion + * \brief RGB statistics for a given region + * + * The IspStatsRegion structure is intended to abstract the ISP specific + * statistics and use an agnostic algorithm to compute AWB. + * + * \var IspStatsRegion::counted + * \brief Number of pixels used to calculate the sums + * + * \var IspStatsRegion::notcounted + * \brief Remaining number of pixels in the region + * + * \var IspStatsRegion::rSum + * \brief Sum of the red values in the region + * + * \var IspStatsRegion::gSum + * \brief Sum of the green values in the region + * + * \var IspStatsRegion::bSum + * \brief Sum of the blue values in the region + */ + +/** + * \struct AwbStatus + * \brief AWB parameters calculated + * + * The AwbStatus structure is intended to store the AWB + * parameters calculated by the algorithm + * + * \var AwbStatus::temperatureK + * \brief Color temperature calculated + * + * \var AwbStatus::redGain + * \brief Gain calculated for the red channel + * + * \var AwbStatus::greenGain + * \brief Gain calculated for the green channel + * + * \var AwbStatus::blueGain + * \brief Gain calculated for the blue channel + */ + +/** + * \struct Ipu3AwbCell + * \brief Memory layout for each cell in AWB metadata + * + * The Ipu3AwbCell structure is used to get individual values + * such as red average or saturation ratio in a particular cell. + * + * \var Ipu3AwbCell::greenRedAvg + * \brief Green average for red lines in the cell + * + * \var Ipu3AwbCell::redAvg + * \brief Red average in the cell + * + * \var Ipu3AwbCell::blueAvg + * \brief blue average in the cell + * + * \var Ipu3AwbCell::greenBlueAvg + * \brief Green average for blue lines + * + * \var Ipu3AwbCell::satRatio + * \brief Saturation ratio in the cell + * + * \var Ipu3AwbCell::padding + * \brief array of unused bytes for padding + */ + +/* Default settings for Bayer noise reduction replicated from the Kernel */ +static const struct ipu3_uapi_bnr_static_config imguCssBnrDefaults = { + .wb_gains = { 16, 16, 16, 16 }, + .wb_gains_thr = { 255, 255, 255, 255 }, + .thr_coeffs = { 1700, 0, 31, 31, 0, 16 }, + .thr_ctrl_shd = { 26, 26, 26, 26 }, + .opt_center{ -648, 0, -366, 0 }, + .lut = { + { 17, 23, 28, 32, 36, 39, 42, 45, + 48, 51, 53, 55, 58, 60, 62, 64, + 66, 68, 70, 72, 73, 75, 77, 78, + 80, 82, 83, 85, 86, 88, 89, 90 } }, + .bp_ctrl = { 20, 0, 1, 40, 0, 6, 0, 6, 0 }, + .dn_detect_ctrl{ 9, 3, 4, 0, 8, 0, 1, 1, 1, 1, 0 }, + .column_size = 1296, + .opt_center_sqr = { 419904, 133956 }, +}; + +/* Default settings for Auto White Balance replicated from the Kernel*/ +static const struct ipu3_uapi_awb_config_s imguCssAwbDefaults = { + .rgbs_thr_gr = 8191, + .rgbs_thr_r = 8191, + .rgbs_thr_gb = 8191, + .rgbs_thr_b = 8191 | IPU3_UAPI_AWB_RGBS_THR_B_EN | IPU3_UAPI_AWB_RGBS_THR_B_INCL_SAT, + .grid = { + .width = 160, + .height = 36, + .block_width_log2 = 3, + .block_height_log2 = 4, + .height_per_slice = 1, /* Overridden by kernel. */ + .x_start = 0, + .y_start = 0, + .x_end = 0, + .y_end = 0, + }, +}; + +/* Default color correction matrix defined as an identity matrix */ +static const struct ipu3_uapi_ccm_mat_config imguCssCcmDefault = { + 8191, 0, 0, 0, + 0, 8191, 0, 0, + 0, 0, 8191, 0 +}; + +IPU3Awb::IPU3Awb() + : Algorithm() +{ + asyncResults_.blueGain = 1.0; + asyncResults_.greenGain = 1.0; + asyncResults_.redGain = 1.0; + asyncResults_.temperatureK = 4500; +} + +IPU3Awb::~IPU3Awb() +{ +} + +void IPU3Awb::initialise(ipu3_uapi_params ¶ms, const Size &bdsOutputSize, struct ipu3_uapi_grid_config &bdsGrid) +{ + params.use.acc_awb = 1; + params.acc_param.awb.config = imguCssAwbDefaults; + + awbGrid_ = bdsGrid; + params.acc_param.awb.config.grid = awbGrid_; + + params.use.acc_bnr = 1; + params.acc_param.bnr = imguCssBnrDefaults; + /** + * Optical center is colum (resp row) start - X (resp Y) center. + * For the moment use BDS as a first approximation, but it should + * be calculated based on Shading (SHD) parameters. + */ + params.acc_param.bnr.column_size = bdsOutputSize.width; + params.acc_param.bnr.opt_center.x_reset = awbGrid_.x_start - (bdsOutputSize.width / 2); + params.acc_param.bnr.opt_center.y_reset = awbGrid_.y_start - (bdsOutputSize.height / 2); + params.acc_param.bnr.opt_center_sqr.x_sqr_reset = params.acc_param.bnr.opt_center.x_reset + * params.acc_param.bnr.opt_center.x_reset; + params.acc_param.bnr.opt_center_sqr.y_sqr_reset = params.acc_param.bnr.opt_center.y_reset + * params.acc_param.bnr.opt_center.y_reset; + + params.use.acc_ccm = 1; + params.acc_param.ccm = imguCssCcmDefault; + + params.use.acc_gamma = 1; + params.acc_param.gamma.gc_ctrl.enable = 1; + + zones_.reserve(kAwbStatsSizeX * kAwbStatsSizeY); +} + +/** + * The function estimates the correlated color temperature using + * from RGB color space input. + * In physics and color science, the Planckian locus or black body locus is + * the path or locus that the color of an incandescent black body would take + * in a particular chromaticity space as the blackbody temperature changes. + * + * If a narrow range of color temperatures is considered (those encapsulating + * daylight being the most practical case) one can approximate the Planckian + * locus in order to calculate the CCT in terms of chromaticity coordinates. + * + * More detailed information can be found in: + * https://en.wikipedia.org/wiki/Color_temperature#Approximation + */ +uint32_t IPU3Awb::estimateCCT(double red, double green, double blue) +{ + /* Convert the RGB values to CIE tristimulus values (XYZ) */ + double X = (-0.14282) * (red) + (1.54924) * (green) + (-0.95641) * (blue); + double Y = (-0.32466) * (red) + (1.57837) * (green) + (-0.73191) * (blue); + double Z = (-0.68202) * (red) + (0.77073) * (green) + (0.56332) * (blue); + + /* Calculate the normalized chromaticity values */ + double x = X / (X + Y + Z); + double y = Y / (X + Y + Z); + + /* Calculate CCT */ + double n = (x - 0.3320) / (0.1858 - y); + return 449 * n * n * n + 3525 * n * n + 6823.3 * n + 5520.33; +} + +/* Generate a RGB vector with the average values for each region */ +void IPU3Awb::generateZones(std::vector &zones) +{ + for (unsigned int i = 0; i < kAwbStatsSizeX * kAwbStatsSizeY; i++) { + RGB zone; + double counted = awbStats_[i].counted; + if (counted >= 16) { + zone.G = awbStats_[i].gSum / counted; + if (zone.G >= 32) { + zone.R = awbStats_[i].rSum / counted; + zone.B = awbStats_[i].bSum / counted; + zones.push_back(zone); + } + } + } +} + +/* Translate the IPU3 statistics into the default statistics region array */ +void IPU3Awb::generateAwbStats(const ipu3_uapi_stats_3a *stats) +{ + uint32_t regionWidth = round(awbGrid_.width / static_cast(kAwbStatsSizeX)); + uint32_t regionHeight = round(awbGrid_.height / static_cast(kAwbStatsSizeY)); + + /** + * Generate a (kAwbStatsSizeX x kAwbStatsSizeY) array from the IPU3 grid which is + * (awbGrid_.width x awbGrid_.height). + */ + for (unsigned int j = 0; j < kAwbStatsSizeY * regionHeight; j++) { + for (unsigned int i = 0; i < kAwbStatsSizeX * regionWidth; i++) { + uint32_t cellPosition = j * awbGrid_.width + i; + uint32_t cellX = (cellPosition / regionWidth) % kAwbStatsSizeX; + uint32_t cellY = ((cellPosition / awbGrid_.width) / regionHeight) % kAwbStatsSizeY; + + uint32_t awbRegionPosition = cellY * kAwbStatsSizeX + cellX; + cellPosition *= 8; + + /* Cast the initial IPU3 structure to simplify the reading */ + Ipu3AwbCell *currentCell = reinterpret_cast(const_cast(&stats->awb_raw_buffer.meta_data[cellPosition])); + if (currentCell->satRatio == 0) { + /* The cell is not saturated, use the current cell */ + awbStats_[awbRegionPosition].counted++; + uint32_t greenValue = currentCell->greenRedAvg + currentCell->greenBlueAvg; + awbStats_[awbRegionPosition].gSum += greenValue / 2; + awbStats_[awbRegionPosition].rSum += currentCell->redAvg; + awbStats_[awbRegionPosition].bSum += currentCell->blueAvg; + } + } + } +} + +void IPU3Awb::clearAwbStats() +{ + for (unsigned int i = 0; i < kAwbStatsSizeX * kAwbStatsSizeY; i++) { + awbStats_[i].bSum = 0; + awbStats_[i].rSum = 0; + awbStats_[i].gSum = 0; + awbStats_[i].counted = 0; + awbStats_[i].notcounted = 0; + } +} + +void IPU3Awb::awbGrey() +{ + LOG(IPU3Awb, Debug) << "Grey world AWB"; + /** + * Make a separate list of the derivatives for each of red and blue, so + * that we can sort them to exclude the extreme gains. We could + * consider some variations, such as normalising all the zones first, or + * doing an L2 average etc. + */ + std::vector &redDerivative(zones_); + std::vector blueDerivative(redDerivative); + std::sort(redDerivative.begin(), redDerivative.end(), + [](RGB const &a, RGB const &b) { + return a.G * b.R < b.G * a.R; + }); + std::sort(blueDerivative.begin(), blueDerivative.end(), + [](RGB const &a, RGB const &b) { + return a.G * b.B < b.G * a.B; + }); + + /* Average the middle half of the values. */ + int discard = redDerivative.size() / 4; + RGB sumRed(0, 0, 0), sumBlue(0, 0, 0); + for (auto ri = redDerivative.begin() + discard, + bi = blueDerivative.begin() + discard; + ri != redDerivative.end() - discard; ri++, bi++) + sumRed += *ri, sumBlue += *bi; + + double redGain = sumRed.G / (sumRed.R + 1), + blueGain = sumBlue.G / (sumBlue.B + 1); + + /* Color temperature is not relevant in Gray world but still useful to estimate it :-) */ + asyncResults_.temperatureK = estimateCCT(sumRed.R, sumRed.G, sumBlue.B); + asyncResults_.redGain = redGain; + asyncResults_.greenGain = 1.0; + asyncResults_.blueGain = blueGain; +} + +void IPU3Awb::calculateWBGains(const ipu3_uapi_stats_3a *stats) +{ + ASSERT(stats->stats_3a_status.awb_en); + zones_.clear(); + clearAwbStats(); + generateAwbStats(stats); + generateZones(zones_); + LOG(IPU3Awb, Debug) << "Valid zones: " << zones_.size(); + if (zones_.size() > 10) + awbGrey(); + + LOG(IPU3Awb, Debug) << "Gain found for red: " << asyncResults_.redGain + << " and for blue: " << asyncResults_.blueGain; +} + +void IPU3Awb::updateWbParameters(ipu3_uapi_params ¶ms, double agcGamma) +{ + /** + * Green gains should not be touched and considered 1. + * Default is 16, so do not change it at all. + * 4096 is the value for a gain of 1.0 + */ + params.acc_param.bnr.wb_gains.gr = 16; + params.acc_param.bnr.wb_gains.r = 4096 * asyncResults_.redGain; + params.acc_param.bnr.wb_gains.b = 4096 * asyncResults_.blueGain; + params.acc_param.bnr.wb_gains.gb = 16; + + LOG(IPU3Awb, Debug) << "Color temperature estimated: " << asyncResults_.temperatureK + << " and gamma calculated: " << agcGamma; + + /* The CCM matrix may change when color temperature will be used */ + params.acc_param.ccm = imguCssCcmDefault; + + for (uint32_t i = 0; i < 256; i++) { + double j = i / 255.0; + double gamma = std::pow(j, 1.0 / agcGamma); + /* The maximum value 255 is represented on 13 bits in the IPU3 */ + params.acc_param.gamma.gc_lut.lut[i] = gamma * 8191; + } +} + +} /* namespace ipa */ + +} /* namespace libcamera */ diff --git a/src/ipa/ipu3/ipu3_awb.h b/src/ipa/ipu3/ipu3_awb.h new file mode 100644 index 00000000..0994d18d --- /dev/null +++ b/src/ipa/ipu3/ipu3_awb.h @@ -0,0 +1,91 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2021, Ideas On Board + * + * ipu3_awb.h - IPU3 AWB control algorithm + */ +#ifndef __LIBCAMERA_IPU3_AWB_H__ +#define __LIBCAMERA_IPU3_AWB_H__ + +#include + +#include + +#include + +#include "libipa/algorithm.h" + +namespace libcamera { + +namespace ipa { + +/* Region size for the statistics generation algorithm */ +static constexpr uint32_t kAwbStatsSizeX = 16; +static constexpr uint32_t kAwbStatsSizeY = 12; + +class IPU3Awb : public Algorithm +{ +public: + IPU3Awb(); + ~IPU3Awb(); + + void initialise(ipu3_uapi_params ¶ms, const Size &bdsOutputSize, struct ipu3_uapi_grid_config &bdsGrid); + void calculateWBGains(const ipu3_uapi_stats_3a *stats); + void updateWbParameters(ipu3_uapi_params ¶ms, double agcGamma); + + struct Ipu3AwbCell { + unsigned char greenRedAvg; + unsigned char redAvg; + unsigned char blueAvg; + unsigned char greenBlueAvg; + unsigned char satRatio; + unsigned char padding[3]; + } __attribute__((packed)); + + /* \todo Make these three structs available to all the ISPs ? */ + struct RGB { + RGB(double _R = 0, double _G = 0, double _B = 0) + : R(_R), G(_G), B(_B) + { + } + double R, G, B; + RGB &operator+=(RGB const &other) + { + R += other.R, G += other.G, B += other.B; + return *this; + } + }; + + struct IspStatsRegion { + unsigned int counted; + unsigned int notcounted; + unsigned long long rSum; + unsigned long long gSum; + unsigned long long bSum; + }; + + struct AwbStatus { + double temperatureK; + double redGain; + double greenGain; + double blueGain; + }; + +private: + void generateZones(std::vector &zones); + void generateAwbStats(const ipu3_uapi_stats_3a *stats); + void clearAwbStats(); + void awbGrey(); + uint32_t estimateCCT(double red, double green, double blue); + + struct ipu3_uapi_grid_config awbGrid_; + + std::vector zones_; + IspStatsRegion awbStats_[kAwbStatsSizeX * kAwbStatsSizeY]; + AwbStatus asyncResults_; +}; + +} /* namespace ipa */ + +} /* namespace libcamera*/ +#endif /* __LIBCAMERA_IPU3_AWB_H__ */ diff --git a/src/ipa/ipu3/meson.build b/src/ipa/ipu3/meson.build index a241f617..1040698e 100644 --- a/src/ipa/ipu3/meson.build +++ b/src/ipa/ipu3/meson.build @@ -2,8 +2,13 @@ ipa_name = 'ipa_ipu3' +ipu3_ipa_sources = files([ + 'ipu3.cpp', + 'ipu3_awb.cpp', +]) + mod = shared_module(ipa_name, - ['ipu3.cpp', libcamera_generated_ipa_headers], + [ipu3_ipa_sources, libcamera_generated_ipa_headers], name_prefix : '', include_directories : [ipa_includes, libipa_includes], dependencies : libcamera_dep, From patchwork Fri Apr 16 07:49:09 2021 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Jean-Michel Hautbois X-Patchwork-Id: 11961 Return-Path: X-Original-To: parsemail@patchwork.libcamera.org Delivered-To: parsemail@patchwork.libcamera.org Received: from lancelot.ideasonboard.com (lancelot.ideasonboard.com [92.243.16.209]) by patchwork.libcamera.org (Postfix) with ESMTPS id D91ACBD235 for ; Fri, 16 Apr 2021 07:49:17 +0000 (UTC) Received: from lancelot.ideasonboard.com (localhost [IPv6:::1]) by lancelot.ideasonboard.com (Postfix) with ESMTP id 4809E68826; Fri, 16 Apr 2021 09:49:17 +0200 (CEST) Authentication-Results: lancelot.ideasonboard.com; dkim=fail reason="signature verification failed" (1024-bit key; unprotected) header.d=ideasonboard.com header.i=@ideasonboard.com header.b="sCz4FC6n"; dkim-atps=neutral Received: from perceval.ideasonboard.com (perceval.ideasonboard.com [IPv6:2001:4b98:dc2:55:216:3eff:fef7:d647]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id 04D3E68819 for ; Fri, 16 Apr 2021 09:49:15 +0200 (CEST) Received: from localhost.localdomain (unknown [IPv6:2a01:e0a:169:7140:5b63:445c:7960:347a]) by perceval.ideasonboard.com (Postfix) with ESMTPSA id A5FEA8A4; Fri, 16 Apr 2021 09:49:14 +0200 (CEST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=ideasonboard.com; s=mail; t=1618559354; bh=tN07RqwqjZR+1gBYFioom902LBDXcplNMq+SrhrWRng=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=sCz4FC6nE1f1aZgti3DvXdP1xYRrZIjXhuXuJVTF1q/My6K1bp49Qah0pg338mlp7 a7VCxK0/pmky9ydskns0J//UG1D/33VjmBj63GdVCLs3HhKpDPBh/th5tMQD1N1uZZ 3VCZ7mG5u8/U2DguxaCobRmL7+2qbjzkmnQHLi+I= From: Jean-Michel Hautbois To: libcamera-devel@lists.libcamera.org Date: Fri, 16 Apr 2021 09:49:09 +0200 Message-Id: <20210416074909.24218-5-jeanmichel.hautbois@ideasonboard.com> X-Mailer: git-send-email 2.27.0 In-Reply-To: <20210416074909.24218-1-jeanmichel.hautbois@ideasonboard.com> References: <20210416074909.24218-1-jeanmichel.hautbois@ideasonboard.com> MIME-Version: 1.0 Subject: [libcamera-devel] [PATCH v5 4/4] ipa: ipu3: Add support for IPU3 AEC/AGC algorithm X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: libcamera-devel-bounces@lists.libcamera.org Sender: "libcamera-devel" Implement basic auto-exposure (AE) and auto-gain (AG) correction functions. The functions computeTargetExposure() and computeGain() are adapted from the Raspberry Pi AGC implementation to suit the IPU3 structures, and filtering is added to reduce visible stepsize when there are large exposure changes. Signed-off-by: Jean-Michel Hautbois Reviewed-by: Kieran Bingham --- src/ipa/ipu3/ipu3.cpp | 16 ++- src/ipa/ipu3/ipu3_agc.cpp | 206 ++++++++++++++++++++++++++++++++++++++ src/ipa/ipu3/ipu3_agc.h | 62 ++++++++++++ src/ipa/ipu3/meson.build | 1 + 4 files changed, 282 insertions(+), 3 deletions(-) create mode 100644 src/ipa/ipu3/ipu3_agc.cpp create mode 100644 src/ipa/ipu3/ipu3_agc.h diff --git a/src/ipa/ipu3/ipu3.cpp b/src/ipa/ipu3/ipu3.cpp index ab052a8a..48dc28c1 100644 --- a/src/ipa/ipu3/ipu3.cpp +++ b/src/ipa/ipu3/ipu3.cpp @@ -21,10 +21,11 @@ #include "libcamera/internal/buffer.h" #include "libcamera/internal/log.h" +#include "ipu3_agc.h" #include "ipu3_awb.h" static constexpr uint32_t kMaxCellWidthPerSet = 160; -static constexpr uint32_t kMaxCellHeightPerSet = 80; +static constexpr uint32_t kMaxCellHeightPerSet = 56; namespace libcamera { @@ -70,6 +71,8 @@ private: /* Interface to the AWB algorithm */ std::unique_ptr awbAlgo_; + /* Interface to the AEC/AGC algorithm */ + std::unique_ptr agcAlgo_; /* Local parameter storage */ struct ipu3_uapi_params params_; @@ -168,6 +171,9 @@ void IPAIPU3::configure(const std::map &entityControls awbAlgo_ = std::make_unique(); awbAlgo_->initialise(params_, bdsOutputSize, bdsGrid_); + + agcAlgo_ = std::make_unique(); + agcAlgo_->initialise(bdsGrid_); } void IPAIPU3::mapBuffers(const std::vector &buffers) @@ -239,8 +245,8 @@ void IPAIPU3::processControls([[maybe_unused]] unsigned int frame, void IPAIPU3::fillParams(unsigned int frame, ipu3_uapi_params *params) { - /* Pass a default gamma of 1.0 (default linear correction) */ - awbAlgo_->updateWbParameters(params_, 1.0); + if (agcAlgo_->updateControls()) + awbAlgo_->updateWbParameters(params_, agcAlgo_->gamma()); *params = params_; @@ -255,8 +261,12 @@ void IPAIPU3::parseStatistics(unsigned int frame, { ControlList ctrls(controls::controls); + agcAlgo_->process(stats, exposure_, gain_); awbAlgo_->calculateWBGains(stats); + if (agcAlgo_->updateControls()) + setControls(frame); + ipa::ipu3::IPU3Action op; op.op = ipa::ipu3::ActionMetadataReady; op.controls = ctrls; diff --git a/src/ipa/ipu3/ipu3_agc.cpp b/src/ipa/ipu3/ipu3_agc.cpp new file mode 100644 index 00000000..04ab55a1 --- /dev/null +++ b/src/ipa/ipu3/ipu3_agc.cpp @@ -0,0 +1,206 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2021, Ideas On Board + * + * ipu3_agc.cpp - AGC/AEC control algorithm + */ + +#include "ipu3_agc.h" + +#include +#include +#include + +#include "libcamera/internal/log.h" + +#include "libipa/histogram.h" + +namespace libcamera { + +namespace ipa { + +LOG_DEFINE_CATEGORY(IPU3Agc) + +/* Number of frames to wait before calculating stats on minimum exposure */ +static constexpr uint32_t kInitialFrameMinAECount = 4; +/* Number of frames to wait between new gain/exposure estimations */ +static constexpr uint32_t kFrameSkipCount = 6; + +/* Maximum ISO value for analogue gain */ +static constexpr uint32_t kMinISO = 100; +static constexpr uint32_t kMaxISO = 1500; + +/* Maximum analogue gain value + * \todo grab it from a camera helper */ +static constexpr uint32_t kMinGain = kMinISO / 100; +static constexpr uint32_t kMaxGain = kMaxISO / 100; + +/* \todo use calculated value based on sensor */ +static constexpr uint32_t kMinExposure = 1; +static constexpr uint32_t kMaxExposure = 1976; + +/* \todo those should be get from pipeline handler ! */ +/* line duration in microseconds */ +static constexpr double kLineDuration = 16.8; +static constexpr double kMaxExposureTime = kMaxExposure * kLineDuration; + +/* Histogram constants */ +static constexpr uint32_t knumHistogramBins = 256; +static constexpr double kEvGainTarget = 0.5; + +IPU3Agc::IPU3Agc() + : frameCount_(0), lastFrame_(0), converged_(false), + updateControls_(false), iqMean_(0.0), gamma_(1.0), + prevExposure_(0.0), prevExposureNoDg_(0.0), + currentExposure_(0.0), currentExposureNoDg_(0.0) +{ +} + +void IPU3Agc::initialise(struct ipu3_uapi_grid_config &bdsGrid) +{ + aeGrid_ = bdsGrid; +} + +void IPU3Agc::processBrightness(const ipu3_uapi_stats_3a *stats) +{ + const struct ipu3_uapi_grid_config statsAeGrid = stats->stats_4a_config.awb_config.grid; + Rectangle aeRegion = { statsAeGrid.x_start, + statsAeGrid.y_start, + static_cast(statsAeGrid.x_end - statsAeGrid.x_start) + 1, + static_cast(statsAeGrid.y_end - statsAeGrid.y_start) + 1 }; + Point topleft = aeRegion.topLeft(); + int topleftX = topleft.x >> aeGrid_.block_width_log2; + int topleftY = topleft.y >> aeGrid_.block_height_log2; + + uint32_t startY = topleftY * aeGrid_.width << aeGrid_.block_width_log2; + uint32_t startX = topleftX << aeGrid_.block_width_log2; + uint32_t endX = (startX + (aeRegion.size().width >> aeGrid_.block_width_log2)) << aeGrid_.block_width_log2; + uint32_t i, j; + uint32_t count = 0; + + uint32_t hist[knumHistogramBins] = { 0 }; + for (j = topleftY; + j < topleftY + (aeRegion.size().height >> aeGrid_.block_height_log2); + j++) { + for (i = startX + startY; i < endX + startY; i += 8) { + /** + * The grid width (and maybe height) is not reliable. + * We observed a bit shift which makes the value 160 to be 32 in the stats grid. + * Use the one passed at init time. + */ + if (stats->awb_raw_buffer.meta_data[i + 4 + j * aeGrid_.width] == 0) { + uint8_t Gr = stats->awb_raw_buffer.meta_data[i + j * aeGrid_.width]; + uint8_t Gb = stats->awb_raw_buffer.meta_data[i + 3 + j * aeGrid_.width]; + hist[(Gr + Gb) / 2]++; + count++; + } + } + } + + /* Limit the gamma effect for now */ + gamma_ = 1.1; + + /* Estimate the quantile mean of the top 2% of the histogram */ + iqMean_ = Histogram(Span(hist)).interQuantileMean(0.98, 1.0); +} + +void IPU3Agc::filterExposure(bool desaturate) +{ + double speed = 0.2; + if (prevExposure_ == 0.0) { + /* DG stands for digital gain.*/ + prevExposure_ = currentExposure_; + prevExposureNoDg_ = currentExposureNoDg_; + } else { + /** + * If we are close to the desired result, go faster to avoid making + * multiple micro-adjustments. + * \ todo: Make this customisable? + */ + if (prevExposure_ < 1.2 * currentExposure_ && + prevExposure_ > 0.8 * currentExposure_) + speed = sqrt(speed); + prevExposure_ = speed * currentExposure_ + + prevExposure_ * (1.0 - speed); + /** + * When desaturing, take a big jump down in exposure_no_dg, + * which we'll hide with digital gain. + */ + if (desaturate) + prevExposureNoDg_ = + currentExposureNoDg_; + else + prevExposureNoDg_ = + speed * currentExposureNoDg_ + + prevExposureNoDg_ * (1.0 - speed); + } + /** + * We can't let the no_dg exposure deviate too far below the + * total exposure, as there might not be enough digital gain available + * in the ISP to hide it (which will cause nasty oscillation). + */ + double fastReduceThreshold = 0.4; + if (prevExposureNoDg_ < + prevExposure_ * fastReduceThreshold) + prevExposureNoDg_ = prevExposure_ * fastReduceThreshold; + LOG(IPU3Agc, Debug) << "After filtering, total_exposure " << prevExposure_; +} + +void IPU3Agc::lockExposureGain(uint32_t &exposure, uint32_t &gain) +{ + updateControls_ = false; + + /* Algorithm initialization should wait for first valid frames */ + /* \todo - have a number of frames given by DelayedControls ? + * - implement a function for IIR */ + if ((frameCount_ < kInitialFrameMinAECount) || (frameCount_ - lastFrame_ < kFrameSkipCount)) + return; + + /* Are we correctly exposed ? */ + if (std::abs(iqMean_ - kEvGainTarget * knumHistogramBins) <= 1) { + LOG(IPU3Agc, Debug) << "!!! Good exposure with iqMean = " << iqMean_; + converged_ = true; + } else { + double newGain = kEvGainTarget * knumHistogramBins / iqMean_; + + /* extracted from Rpi::Agc::computeTargetExposure */ + double currentShutter = exposure * kLineDuration; + currentExposureNoDg_ = currentShutter * gain; + LOG(IPU3Agc, Debug) << "Actual total exposure " << currentExposureNoDg_ + << " Shutter speed " << currentShutter + << " Gain " << gain; + currentExposure_ = currentExposureNoDg_ * newGain; + double maxTotalExposure = kMaxExposureTime * kMaxGain; + currentExposure_ = std::min(currentExposure_, maxTotalExposure); + LOG(IPU3Agc, Debug) << "Target total exposure " << currentExposure_; + + /* \todo: estimate if we need to desaturate */ + filterExposure(false); + + double newExposure = 0.0; + if (currentShutter < kMaxExposureTime) { + exposure = std::clamp(static_cast(exposure * currentExposure_ / currentExposureNoDg_), kMinExposure, kMaxExposure); + newExposure = currentExposure_ / exposure; + gain = std::clamp(static_cast(gain * currentExposure_ / newExposure), kMinGain, kMaxGain); + updateControls_ = true; + } else if (currentShutter >= kMaxExposureTime) { + gain = std::clamp(static_cast(gain * currentExposure_ / currentExposureNoDg_), kMinGain, kMaxGain); + newExposure = currentExposure_ / gain; + exposure = std::clamp(static_cast(exposure * currentExposure_ / newExposure), kMinExposure, kMaxExposure); + updateControls_ = true; + } + LOG(IPU3Agc, Debug) << "Adjust exposure " << exposure * kLineDuration << " and gain " << gain; + } + lastFrame_ = frameCount_; +} + +void IPU3Agc::process(const ipu3_uapi_stats_3a *stats, uint32_t &exposure, uint32_t &gain) +{ + processBrightness(stats); + lockExposureGain(exposure, gain); + frameCount_++; +} + +} /* namespace ipa */ + +} /* namespace libcamera */ diff --git a/src/ipa/ipu3/ipu3_agc.h b/src/ipa/ipu3/ipu3_agc.h new file mode 100644 index 00000000..f59bc420 --- /dev/null +++ b/src/ipa/ipu3/ipu3_agc.h @@ -0,0 +1,62 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2021, Ideas On Board + * + * ipu3_agc.h - IPU3 AGC/AEC control algorithm + */ +#ifndef __LIBCAMERA_IPU3_AGC_H__ +#define __LIBCAMERA_IPU3_AGC_H__ + +#include +#include + +#include + +#include + +#include "libipa/algorithm.h" + +namespace libcamera { + +namespace ipa { + +class IPU3Agc : public Algorithm +{ +public: + IPU3Agc(); + ~IPU3Agc() = default; + + void initialise(struct ipu3_uapi_grid_config &bdsGrid); + void process(const ipu3_uapi_stats_3a *stats, uint32_t &exposure, uint32_t &gain); + bool converged() { return converged_; } + bool updateControls() { return updateControls_; } + /* \todo Use a metadata exchange between IPAs */ + double gamma() { return gamma_; } + +private: + void processBrightness(const ipu3_uapi_stats_3a *stats); + void filterExposure(bool desaturate); + void lockExposureGain(uint32_t &exposure, uint32_t &gain); + + struct ipu3_uapi_grid_config aeGrid_; + + uint64_t frameCount_; + uint64_t lastFrame_; + + bool converged_; + bool updateControls_; + + double iqMean_; + double gamma_; + + double prevExposure_; + double prevExposureNoDg_; + double currentExposure_; + double currentExposureNoDg_; +}; + +} /* namespace ipa */ + +} /* namespace libcamera */ + +#endif /* __LIBCAMERA_IPU3_AGC_H__ */ diff --git a/src/ipa/ipu3/meson.build b/src/ipa/ipu3/meson.build index 1040698e..0d843846 100644 --- a/src/ipa/ipu3/meson.build +++ b/src/ipa/ipu3/meson.build @@ -4,6 +4,7 @@ ipa_name = 'ipa_ipu3' ipu3_ipa_sources = files([ 'ipu3.cpp', + 'ipu3_agc.cpp', 'ipu3_awb.cpp', ])