From patchwork Tue Feb 23 16:40:41 2021 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Jean-Michel Hautbois X-Patchwork-Id: 11369 Return-Path: X-Original-To: parsemail@patchwork.libcamera.org Delivered-To: parsemail@patchwork.libcamera.org Received: from lancelot.ideasonboard.com (lancelot.ideasonboard.com [92.243.16.209]) by patchwork.libcamera.org (Postfix) with ESMTPS id 65734BD1F1 for ; Tue, 23 Feb 2021 16:40:49 +0000 (UTC) Received: from lancelot.ideasonboard.com (localhost [IPv6:::1]) by lancelot.ideasonboard.com (Postfix) with ESMTP id 715F768A35; Tue, 23 Feb 2021 17:40:48 +0100 (CET) Authentication-Results: lancelot.ideasonboard.com; dkim=fail reason="signature verification failed" (1024-bit key; unprotected) header.d=ideasonboard.com header.i=@ideasonboard.com header.b="ORzSSGu/"; dkim-atps=neutral Received: from perceval.ideasonboard.com (perceval.ideasonboard.com [IPv6:2001:4b98:dc2:55:216:3eff:fef7:d647]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id 2490E68A31 for ; Tue, 23 Feb 2021 17:40:44 +0100 (CET) Received: from localhost.localdomain (unknown [IPv6:2a01:e0a:169:7140:d7c9:eb3:c460:c24a]) by perceval.ideasonboard.com (Postfix) with ESMTPSA id C6F8B9E2; Tue, 23 Feb 2021 17:40:43 +0100 (CET) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=ideasonboard.com; s=mail; t=1614098443; bh=2KXdTw7glBCyYisb7cqKMHtmm379qQ3rGt3x/iJ+ORY=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=ORzSSGu//UST0P04CwlyHc2ti/ORM0zxzIqSIAcwxqzyZKZX9j1CXFvg9iQcYrSaR mE0x3JdqsZ+diPMYjVwyhJ/lsyqWwPDIDR0Orxvb975j4fXlrPd37nbfl0IVW1eoxZ 4BYGotz4b9aJWE6QAbt2OLtIMYWuyYxAiMxT2BSw= From: Jean-Michel Hautbois To: libcamera-devel@lists.libcamera.org Date: Tue, 23 Feb 2021 17:40:41 +0100 Message-Id: <20210223164041.49932-4-jeanmichel.hautbois@ideasonboard.com> X-Mailer: git-send-email 2.27.0 In-Reply-To: <20210223164041.49932-1-jeanmichel.hautbois@ideasonboard.com> References: <20210223164041.49932-1-jeanmichel.hautbois@ideasonboard.com> MIME-Version: 1.0 Subject: [libcamera-devel] [RFC PATCH v2 3/3] WIP: ipa: ipu3: Add support for IPU3 AEC/AGC algorithm X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: libcamera-devel-bounces@lists.libcamera.org Sender: "libcamera-devel" Inherit from the Algorithm class to implement basic auto-exposure and auto-gain functions. While exposure is not locked, AWB is not calculated and corrected. Implement a basic "skewness-based" for demonstration purpose. Signed-off-by: Jean-Michel Hautbois --- src/ipa/ipu3/ipu3.cpp | 10 ++- src/ipa/ipu3/ipu3_agc.cpp | 171 ++++++++++++++++++++++++++++++++++++++ src/ipa/ipu3/ipu3_agc.h | 62 ++++++++++++++ src/ipa/ipu3/meson.build | 1 + 4 files changed, 243 insertions(+), 1 deletion(-) create mode 100644 src/ipa/ipu3/ipu3_agc.cpp create mode 100644 src/ipa/ipu3/ipu3_agc.h diff --git a/src/ipa/ipu3/ipu3.cpp b/src/ipa/ipu3/ipu3.cpp index 6fae5160..cabd0c71 100644 --- a/src/ipa/ipu3/ipu3.cpp +++ b/src/ipa/ipu3/ipu3.cpp @@ -21,6 +21,7 @@ #include "libcamera/internal/buffer.h" #include "libcamera/internal/log.h" +#include "ipu3_agc.h" #include "ipu3_awb.h" namespace libcamera { @@ -65,6 +66,8 @@ private: /* Interface to the AWB algorithm */ ipa::IPU3Awb *awbAlgo_; + /* Interface to the AEC/AGC algorithm */ + ipa::IPU3Agc *agcAlgo_; /* Local parameter storage */ ipu3_uapi_params params_; }; @@ -101,6 +104,8 @@ void IPAIPU3::configure(const std::map &entityControls awbAlgo_ = new ipa::IPU3Awb(); awbAlgo_->initialise(params_); + agcAlgo_ = new ipa::IPU3Agc(); + setControls(0); } @@ -187,7 +192,10 @@ void IPAIPU3::parseStatistics(unsigned int frame, { ControlList ctrls(controls::controls); - awbAlgo_->calculateWBGains(Rectangle(250, 160, 800, 400), stats); + agcAlgo_->process(stats, exposure_, gain_); + if (agcAlgo_->converged()) + awbAlgo_->calculateWBGains(Rectangle(250, 160, 800, 400), stats); + setControls(frame); ipa::ipu3::IPU3Action op; diff --git a/src/ipa/ipu3/ipu3_agc.cpp b/src/ipa/ipu3/ipu3_agc.cpp new file mode 100644 index 00000000..2636e2ea --- /dev/null +++ b/src/ipa/ipu3/ipu3_agc.cpp @@ -0,0 +1,171 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (C) 2019, Raspberry Pi (Trading) Limited + * + * ipu3_agc.cpp - AGC/AEC control algorithm + */ + +#include "ipu3_agc.h" + +#include + +#include "libcamera/internal/log.h" + +namespace libcamera { + +namespace ipa { + +LOG_DEFINE_CATEGORY(IPU3Agc) + +/* Number of frames to wait before calculating stats on minimum exposure */ +static const uint32_t kInitialFrameMinAECount = 4; +/* Number of frames to wait before calculating stats on maximum exposure */ +static const uint32_t kInitialFrameMaxAECount = 8; +/* Number of frames to wait before calculating stats and estimate gain/exposure */ +static const uint32_t kInitialFrameSkipCount = 12; + +/* Number of frames to wait between new gain/exposure estimations */ +static const uint32_t kFrameSkipCount = 4; + +IPU3Agc::IPU3Agc() + : frameCount_(0), converged_(false) +{ +} + +IPU3Agc::~IPU3Agc() +{ +} + +void IPU3Agc::initialise() +{ +} + +void IPU3Agc::process() +{ +} + +/* + * \todo This function is taken from numerical recipes and calculates all + * moments. It needs to be rewritten properly and maybe in a "math" class ? + */ +void IPU3Agc::moments(std::unordered_map &data, int n) +{ + int j; + double ep = 0.0, s, p; + double ave, adev, sdev; + double var, skew, curt; + + s = 0.0; + for (j = 1; j <= n; j++) + s += data[j]; + + ave = s / n; + adev = var = skew = curt = 0.0; + + for (j = 1; j <= n; j++) { + adev += s = data[j] - (ave); + ep += s; + var += (p = s * s); + skew += (p *= s); + curt += (p *= s); + } + + adev /= n; + var = (var - ep * ep / n) / (n - 1); + sdev = std::sqrt(var); + + if (var) { + skew /= n * var * sdev; + curt = curt / (n * var * var) - 3.0; + } + skew_ = skew; +} + +void IPU3Agc::processBrightness(const ipu3_uapi_stats_3a *stats) +{ + cellsBrightness_.clear(); + + /*\todo Replace constant values with real BDS configuration */ + for (uint32_t j = 0; j < 45; j++) { + for (uint32_t i = 0; i < 160 * 45 * 8; i += 8) { + uint8_t Gr = stats->awb_raw_buffer.meta_data[i]; + uint8_t R = stats->awb_raw_buffer.meta_data[i + 1]; + uint8_t B = stats->awb_raw_buffer.meta_data[i + 2]; + uint8_t Gb = stats->awb_raw_buffer.meta_data[i + 3]; + cellsBrightness_.push_back(static_cast(0.299 * R + 0.587 * (Gr + Gb) / 2 + 0.114 * B)); + } + } + std::sort(cellsBrightness_.begin(), cellsBrightness_.end()); + + /* \todo create a class to generate histograms ! */ + std::unordered_map hist; + for (uint32_t const &val : cellsBrightness_) + hist[val]++; + moments(hist, 256); +} + +/* \todo make this function a math one ? */ +uint32_t IPU3Agc::rootApproximation() +{ + return (currentExposure_ * prevSkew_ + prevExposure_ * currentSkew_) / (prevSkew_ + currentSkew_); +} + +void IPU3Agc::lockExposure(uint32_t &exposure, uint32_t &gain) +{ + /* Algorithm initialization wait for first valid frames */ + /* \todo - have a number of frames given by DelayedControls ? + * - implement a function for IIR */ + if (frameCount_ == kInitialFrameMinAECount) { + prevExposure_ = exposure; + + prevSkew_ = skew_; + /* \todo use configured values */ + exposure = 800; + gain = 8; + currentExposure_ = exposure; + } else if (frameCount_ == kInitialFrameMaxAECount) { + currentSkew_ = skew_; + exposure = rootApproximation(); + nextExposure_ = exposure; + lastFrame_ = frameCount_; + } else if ((frameCount_ >= kInitialFrameSkipCount) && (frameCount_ - lastFrame_ >= kFrameSkipCount)) { + currentSkew_ = skew_; + /* \todo properly calculate a gain */ + if (frameCount_ == kInitialFrameSkipCount) + gain = ((8 * prevSkew_) + (1 * currentSkew_)) / (prevSkew_ + currentSkew_); + + if (currentSkew_ - prevSkew_ > 1) { + /* under exposed */ + prevExposure_ = nextExposure_; + exposure = rootApproximation(); + nextExposure_ = exposure; + } else if (currentSkew_ - prevSkew_ < -1) { + /* over exposed */ + currentExposure_ = nextExposure_; + exposure = rootApproximation(); + nextExposure_ = exposure; + } else { + /* we have converged */ + converged_ = true; + } + lastFrame_ = frameCount_; + prevSkew_ = currentSkew_; + } +} + +void IPU3Agc::process(const ipu3_uapi_stats_3a *stats, uint32_t &exposure, uint32_t &gain) +{ + processBrightness(stats); + if (!converged_) + lockExposure(exposure, gain); + else { + /* Are we still well exposed ? */ + if ((skew_ < 2) || (skew_ > 4)) + converged_ = false; + } + frameCount_++; +} + +} /* namespace ipa */ + +} /* namespace libcamera */ diff --git a/src/ipa/ipu3/ipu3_agc.h b/src/ipa/ipu3/ipu3_agc.h new file mode 100644 index 00000000..b14a2a2f --- /dev/null +++ b/src/ipa/ipu3/ipu3_agc.h @@ -0,0 +1,62 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (C) 2019, Raspberry Pi (Trading) Limited + * + * ipu3_agc.h - IPU3 AGC/AEC control algorithm + */ +#ifndef __LIBCAMERA_IPU3_AGC_H__ +#define __LIBCAMERA_IPU3_AGC_H__ + +#include +#include + +#include + +#include + +#include "libipa/algorithm.h" + +namespace libcamera { + +namespace ipa { + +class IPU3Agc : public Algorithm +{ +public: + IPU3Agc(); + ~IPU3Agc(); + + void initialise() override; + void process() override; + + void process(const ipu3_uapi_stats_3a *stats, uint32_t &exposure, uint32_t &gain); + bool converged() { return converged_; } + +private: + void moments(std::unordered_map &data, int n); + void processBrightness(const ipu3_uapi_stats_3a *stats); + uint32_t rootApproximation(); + void lockExposure(uint32_t &exposure, uint32_t &gain); + + uint64_t frameCount_; + uint64_t lastFrame_; + + /* Vector of calculated brightness for each cell */ + std::vector cellsBrightness_; + + /* Values for filtering */ + uint32_t prevExposure_; + uint32_t currentExposure_; + uint32_t nextExposure_; + + double skew_; + double prevSkew_; + double currentSkew_; + bool converged_; +}; + +} /* namespace ipa */ + +} /* namespace libcamera */ + +#endif /* __LIBCAMERA_IPU3_AGC_H__ */ diff --git a/src/ipa/ipu3/meson.build b/src/ipa/ipu3/meson.build index 07a864c8..43ad0e0d 100644 --- a/src/ipa/ipu3/meson.build +++ b/src/ipa/ipu3/meson.build @@ -4,6 +4,7 @@ ipa_name = 'ipa_ipu3' ipu3_ipa_sources = files([ 'ipu3.cpp', + 'ipu3_agc.cpp', 'ipu3_awb.cpp', ])