From patchwork Wed Feb 23 10:48:21 2022 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Jean-Michel Hautbois X-Patchwork-Id: 15371 Return-Path: X-Original-To: parsemail@patchwork.libcamera.org Delivered-To: parsemail@patchwork.libcamera.org Received: from lancelot.ideasonboard.com (lancelot.ideasonboard.com [92.243.16.209]) by patchwork.libcamera.org (Postfix) with ESMTPS id 0819DBE08A for ; Wed, 23 Feb 2022 10:48:31 +0000 (UTC) Received: from lancelot.ideasonboard.com (localhost [IPv6:::1]) by lancelot.ideasonboard.com (Postfix) with ESMTP id 57644610B3; Wed, 23 Feb 2022 11:48:30 +0100 (CET) Authentication-Results: lancelot.ideasonboard.com; dkim=fail reason="signature verification failed" (1024-bit key; unprotected) header.d=ideasonboard.com header.i=@ideasonboard.com header.b="QWE8gAfh"; dkim-atps=neutral Received: from perceval.ideasonboard.com (perceval.ideasonboard.com [213.167.242.64]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id 87CDD610B3 for ; Wed, 23 Feb 2022 11:48:28 +0100 (CET) Received: from tatooine.ideasonboard.com (unknown [IPv6:2a01:e0a:169:7140:7ebc:9e38:c4df:c57]) by perceval.ideasonboard.com (Postfix) with ESMTPSA id 2816D929; Wed, 23 Feb 2022 11:48:28 +0100 (CET) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=ideasonboard.com; s=mail; t=1645613308; bh=69evzyf5yk0B/1jNYt6LwIOVd8DSoIDQs1knjQtkDu0=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=QWE8gAfhxk2v8aW9FgcTKLtpsSE0Sx/IjczoexG82RxUaaJzBfQG7CYoP3Jnz+tkn 5/QNXF2HWh2QBGL+wgp6rO1UZkjyjsB9N5ffeSQwxzxtPiUImKBzzlELVm6n4gOkC3 /Hp6e/vobtQs8jziggOkUwqCyd+m4w68RJgtLTKo= From: Jean-Michel Hautbois To: libcamera-devel@lists.libcamera.org Date: Wed, 23 Feb 2022 11:48:21 +0100 Message-Id: <20220223104824.25723-2-jeanmichel.hautbois@ideasonboard.com> X-Mailer: git-send-email 2.32.0 In-Reply-To: <20220223104824.25723-1-jeanmichel.hautbois@ideasonboard.com> References: <20220223104824.25723-1-jeanmichel.hautbois@ideasonboard.com> MIME-Version: 1.0 Subject: [libcamera-devel] [PATCH v2 1/4] ipa: rkisp1: Use frame index X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: libcamera-devel-bounces@lists.libcamera.org Sender: "libcamera-devel" Instead of incrementing the frameCount manually in a local counter, use the frame index on the EventStatReay event and store it in a new IPAFrameContext variable named frameId. The frameId may be used by other algorithms later. Signed-off-by: Jean-Michel Hautbois Tested-by: Peter Griffin Reviewed-by: Laurent Pinchart --- src/ipa/rkisp1/algorithms/agc.cpp | 5 ++--- src/ipa/rkisp1/ipa_context.cpp | 5 +++++ src/ipa/rkisp1/ipa_context.h | 2 ++ src/ipa/rkisp1/rkisp1.cpp | 11 +++++------ 4 files changed, 14 insertions(+), 9 deletions(-) diff --git a/src/ipa/rkisp1/algorithms/agc.cpp b/src/ipa/rkisp1/algorithms/agc.cpp index dd97afc0..50980835 100644 --- a/src/ipa/rkisp1/algorithms/agc.cpp +++ b/src/ipa/rkisp1/algorithms/agc.cpp @@ -82,8 +82,6 @@ int Agc::configure(IPAContext &context, else numCells_ = RKISP1_CIF_ISP_AE_MEAN_MAX_V12; - /* \todo Use actual frame index by populating it in the frameContext. */ - frameCount_ = 0; return 0; } @@ -255,6 +253,8 @@ void Agc::process(IPAContext &context, const rkisp1_stat_buffer *stats) const rkisp1_cif_isp_ae_stat *ae = ¶ms->ae; + frameCount_ = context.frameContext.frameId; + /* * Estimate the gain needed to achieve a relative luminance target. To * account for non-linearity caused by saturation, the value needs to be @@ -278,7 +278,6 @@ void Agc::process(IPAContext &context, const rkisp1_stat_buffer *stats) } computeExposure(context, yGain); - frameCount_++; } void Agc::prepare([[maybe_unused]] IPAContext &context, diff --git a/src/ipa/rkisp1/ipa_context.cpp b/src/ipa/rkisp1/ipa_context.cpp index 9cb2a9fd..992c9225 100644 --- a/src/ipa/rkisp1/ipa_context.cpp +++ b/src/ipa/rkisp1/ipa_context.cpp @@ -113,4 +113,9 @@ namespace libcamera::ipa::rkisp1 { * \brief Analogue gain multiplier */ +/** + * \var IPAFrameContext::frameId + * \brief Frame number for this frame context + */ + } /* namespace libcamera::ipa::rkisp1 */ diff --git a/src/ipa/rkisp1/ipa_context.h b/src/ipa/rkisp1/ipa_context.h index b94ade0c..c447369f 100644 --- a/src/ipa/rkisp1/ipa_context.h +++ b/src/ipa/rkisp1/ipa_context.h @@ -43,6 +43,8 @@ struct IPAFrameContext { uint32_t exposure; double gain; } sensor; + + unsigned int frameId; }; struct IPAContext { diff --git a/src/ipa/rkisp1/rkisp1.cpp b/src/ipa/rkisp1/rkisp1.cpp index 2d79f15f..732ca2bb 100644 --- a/src/ipa/rkisp1/rkisp1.cpp +++ b/src/ipa/rkisp1/rkisp1.cpp @@ -56,8 +56,7 @@ public: private: void queueRequest(unsigned int frame, rkisp1_params_cfg *params, const ControlList &controls); - void updateStatistics(unsigned int frame, - const rkisp1_stat_buffer *stats); + void updateStatistics(const rkisp1_stat_buffer *stats); void setControls(unsigned int frame); void metadataReady(unsigned int frame, unsigned int aeState); @@ -239,7 +238,6 @@ void IPARkISP1::processEvent(const RkISP1Event &event) { switch (event.op) { case EventSignalStatBuffer: { - unsigned int frame = event.frame; unsigned int bufferId = event.bufferId; const rkisp1_stat_buffer *stats = @@ -250,8 +248,9 @@ void IPARkISP1::processEvent(const RkISP1Event &event) event.sensorControls.get(V4L2_CID_EXPOSURE).get(); context_.frameContext.sensor.gain = camHelper_->gain(event.sensorControls.get(V4L2_CID_ANALOGUE_GAIN).get()); + context_.frameContext.frameId = event.frame; - updateStatistics(frame, stats); + updateStatistics(stats); break; } case EventQueueRequest: { @@ -286,10 +285,10 @@ void IPARkISP1::queueRequest(unsigned int frame, rkisp1_params_cfg *params, queueFrameAction.emit(frame, op); } -void IPARkISP1::updateStatistics(unsigned int frame, - const rkisp1_stat_buffer *stats) +void IPARkISP1::updateStatistics(const rkisp1_stat_buffer *stats) { unsigned int aeState = 0; + unsigned int frame = context_.frameContext.frameId; for (auto const &algo : algorithms_) algo->process(context_, stats); From patchwork Wed Feb 23 10:48:22 2022 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Jean-Michel Hautbois X-Patchwork-Id: 15373 Return-Path: X-Original-To: parsemail@patchwork.libcamera.org Delivered-To: parsemail@patchwork.libcamera.org Received: from lancelot.ideasonboard.com (lancelot.ideasonboard.com [92.243.16.209]) by patchwork.libcamera.org (Postfix) with ESMTPS id A491FBE08A for ; Wed, 23 Feb 2022 10:48:33 +0000 (UTC) Received: from lancelot.ideasonboard.com (localhost [IPv6:::1]) by lancelot.ideasonboard.com (Postfix) with ESMTP id 3F2076116E; Wed, 23 Feb 2022 11:48:31 +0100 (CET) Authentication-Results: lancelot.ideasonboard.com; dkim=fail reason="signature verification failed" (1024-bit key; unprotected) header.d=ideasonboard.com header.i=@ideasonboard.com header.b="ekQXlF3I"; dkim-atps=neutral Received: from perceval.ideasonboard.com (perceval.ideasonboard.com [IPv6:2001:4b98:dc2:55:216:3eff:fef7:d647]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id A5DEC610F9 for ; Wed, 23 Feb 2022 11:48:28 +0100 (CET) Received: from tatooine.ideasonboard.com (unknown [IPv6:2a01:e0a:169:7140:7ebc:9e38:c4df:c57]) by perceval.ideasonboard.com (Postfix) with ESMTPSA id 5605A113C; Wed, 23 Feb 2022 11:48:28 +0100 (CET) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=ideasonboard.com; s=mail; t=1645613308; bh=AuS7HKUAkwJC/4rMgcXb++lzPzJsuEGjdgTWHTEI4BU=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=ekQXlF3IeoTXD/gGolXKZLKAg+rwsxOGAnT9PXwyx8FISXUWL977gHQ9X1m+vCNyS wvRMEM05giGm88bqNGbKN1L3rbaOVv5jZx7XwEYXOqsGUfmVNlLD26+lHFFJRz8upP izDKhHw3t9Q4PdMMgj+Dy+wVOOqmIaYwKDGCa8MA= From: Jean-Michel Hautbois To: libcamera-devel@lists.libcamera.org Date: Wed, 23 Feb 2022 11:48:22 +0100 Message-Id: <20220223104824.25723-3-jeanmichel.hautbois@ideasonboard.com> X-Mailer: git-send-email 2.32.0 In-Reply-To: <20220223104824.25723-1-jeanmichel.hautbois@ideasonboard.com> References: <20220223104824.25723-1-jeanmichel.hautbois@ideasonboard.com> MIME-Version: 1.0 Subject: [libcamera-devel] [PATCH v2 2/4] ipa: rkisp1: Introduce Black Level Correction X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: libcamera-devel-bounces@lists.libcamera.org Sender: "libcamera-devel" In order to have the proper pixel levels, apply a fixed black level correction, based on the imx219 tuning file in RPi. The value is 4096 on 16 bits, and the pipeline for RkISP1 is on 12 bits, scale it. Signed-off-by: Jean-Michel Hautbois Tested-by: Peter Griffin --- src/ipa/rkisp1/algorithms/blc.cpp | 57 +++++++++++++++++++++++++++ src/ipa/rkisp1/algorithms/blc.h | 30 ++++++++++++++ src/ipa/rkisp1/algorithms/meson.build | 1 + src/ipa/rkisp1/rkisp1.cpp | 2 + 4 files changed, 90 insertions(+) create mode 100644 src/ipa/rkisp1/algorithms/blc.cpp create mode 100644 src/ipa/rkisp1/algorithms/blc.h diff --git a/src/ipa/rkisp1/algorithms/blc.cpp b/src/ipa/rkisp1/algorithms/blc.cpp new file mode 100644 index 00000000..788953e3 --- /dev/null +++ b/src/ipa/rkisp1/algorithms/blc.cpp @@ -0,0 +1,57 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2021, Ideas On Board + * + * blc.cpp - RkISP1 Black Level Correction control + */ + +#include "blc.h" + +/** + * \file blc.h + */ + +namespace libcamera { + +namespace ipa::rkisp1::algorithms { + +/** + * \class BlackLevelCorrection + * \brief RkISP1 Black Level Correction control + * + * The pixels output by the camera normally include a black level, because + * sensors do not always report a signal level of '0' for black. Pixels at or + * below this level should be considered black. To achieve that, the RkISP BLC + * algorithm subtracts a configurable offset from all pixels. + * + * The black level can be measured at runtime from an optical dark region of the + * camera sensor, or measured during the camera tuning process. The first option + * isn't currently supported. + */ + +/** + * \copydoc libcamera::ipa::Algorithm::prepare + */ +void BlackLevelCorrection::prepare(IPAContext &context, + rkisp1_params_cfg *params) +{ + /* + * Substract fixed values taken from imx219 tuning file. + * \todo Use a configuration file for it ? + */ + params->others.bls_config.enable_auto = 0; + params->others.bls_config.fixed_val.r = 256; + params->others.bls_config.fixed_val.gr = 256; + params->others.bls_config.fixed_val.gb = 256; + params->others.bls_config.fixed_val.b = 256; + + if (context.frameContext.frameId == 0) { + params->module_en_update |= RKISP1_CIF_ISP_MODULE_BLS; + params->module_ens |= RKISP1_CIF_ISP_MODULE_BLS; + params->module_cfg_update |= RKISP1_CIF_ISP_MODULE_BLS; + } +} + +} /* namespace ipa::rkisp1::algorithms */ + +} /* namespace libcamera */ diff --git a/src/ipa/rkisp1/algorithms/blc.h b/src/ipa/rkisp1/algorithms/blc.h new file mode 100644 index 00000000..331a2209 --- /dev/null +++ b/src/ipa/rkisp1/algorithms/blc.h @@ -0,0 +1,30 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2021, Ideas On Board + * + * blc.h - RkISP1 Black Level Correction control + */ + +#pragma once + +#include + +#include "algorithm.h" + +namespace libcamera { + +struct IPACameraSensorInfo; + +namespace ipa::rkisp1::algorithms { + +class BlackLevelCorrection : public Algorithm +{ +public: + BlackLevelCorrection() = default; + ~BlackLevelCorrection() = default; + + void prepare(IPAContext &context, rkisp1_params_cfg *params) override; +}; + +} /* namespace ipa::rkisp1::algorithms */ +} /* namespace libcamera */ diff --git a/src/ipa/rkisp1/algorithms/meson.build b/src/ipa/rkisp1/algorithms/meson.build index a19c1a4f..27c97731 100644 --- a/src/ipa/rkisp1/algorithms/meson.build +++ b/src/ipa/rkisp1/algorithms/meson.build @@ -2,4 +2,5 @@ rkisp1_ipa_algorithms = files([ 'agc.cpp', + 'blc.cpp', ]) diff --git a/src/ipa/rkisp1/rkisp1.cpp b/src/ipa/rkisp1/rkisp1.cpp index 732ca2bb..f82b7cb3 100644 --- a/src/ipa/rkisp1/rkisp1.cpp +++ b/src/ipa/rkisp1/rkisp1.cpp @@ -27,6 +27,7 @@ #include "algorithms/agc.h" #include "algorithms/algorithm.h" +#include "algorithms/blc.h" #include "libipa/camera_sensor_helper.h" #include "ipa_context.h" @@ -125,6 +126,7 @@ int IPARkISP1::init(const IPASettings &settings, unsigned int hwRevision) /* Construct our Algorithms */ algorithms_.push_back(std::make_unique()); + algorithms_.push_back(std::make_unique()); return 0; } From patchwork Wed Feb 23 10:48:23 2022 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Jean-Michel Hautbois X-Patchwork-Id: 15374 Return-Path: X-Original-To: parsemail@patchwork.libcamera.org Delivered-To: parsemail@patchwork.libcamera.org Received: from lancelot.ideasonboard.com (lancelot.ideasonboard.com [92.243.16.209]) by patchwork.libcamera.org (Postfix) with ESMTPS id 38D6CC3261 for ; Wed, 23 Feb 2022 10:48:34 +0000 (UTC) Received: from lancelot.ideasonboard.com (localhost [IPv6:::1]) by lancelot.ideasonboard.com (Postfix) with ESMTP id 8C69E61183; Wed, 23 Feb 2022 11:48:31 +0100 (CET) Authentication-Results: lancelot.ideasonboard.com; dkim=fail reason="signature verification failed" (1024-bit key; unprotected) header.d=ideasonboard.com header.i=@ideasonboard.com header.b="oF18WPIb"; dkim-atps=neutral Received: from perceval.ideasonboard.com (perceval.ideasonboard.com [213.167.242.64]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id CA03261101 for ; Wed, 23 Feb 2022 11:48:28 +0100 (CET) Received: from tatooine.ideasonboard.com (unknown [IPv6:2a01:e0a:169:7140:7ebc:9e38:c4df:c57]) by perceval.ideasonboard.com (Postfix) with ESMTPSA id 8360A11BF; Wed, 23 Feb 2022 11:48:28 +0100 (CET) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=ideasonboard.com; s=mail; t=1645613308; bh=jxU9hk1cYKxbO9o549ZcQQLvcDjxXbHlDRELGKmSvnc=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=oF18WPIbM+mPX8EmtYXRhHzEPxjfQB63gQT890BiRW4FeOszW44dBmbjT9pN/qI7j JsB2/QW8fmseP12ldYA9X7ktVqAXlFx3AUB6Zr/8fpVB58wn4UcvxwRozKX6KvE/eh 46zA1AWDTOl/16y3AM+nfvBe7aarWg/u/Ovb2/So= From: Jean-Michel Hautbois To: libcamera-devel@lists.libcamera.org Date: Wed, 23 Feb 2022 11:48:23 +0100 Message-Id: <20220223104824.25723-4-jeanmichel.hautbois@ideasonboard.com> X-Mailer: git-send-email 2.32.0 In-Reply-To: <20220223104824.25723-1-jeanmichel.hautbois@ideasonboard.com> References: <20220223104824.25723-1-jeanmichel.hautbois@ideasonboard.com> MIME-Version: 1.0 Subject: [libcamera-devel] [PATCH v2 3/4] ipa: rkisp1: agc: Introduce histogram calculation X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: libcamera-devel-bounces@lists.libcamera.org Sender: "libcamera-devel" As for the IPU3, we can estimate the histogram of the luminance. The RkISP1 can estimate multiple ones, the R, G and B ones, the Y only one and a combination of RGB. The one we are interested by in AGC is the Y histogram. Use the hardware revision to determine the number of bins of the produced histogram, and use it to populate a vector passed down to the libipa::Histogram class. As the sensor deGamma and AWB are applied, we also need to get back to the relative luminance value of 0.16, as for the IPU3. Signed-off-by: Jean-Michel Hautbois Tested-by: Peter Griffin --- src/ipa/rkisp1/algorithms/agc.cpp | 97 ++++++++++++++++++++++++++----- src/ipa/rkisp1/algorithms/agc.h | 4 +- src/ipa/rkisp1/ipa_context.cpp | 3 + src/ipa/rkisp1/ipa_context.h | 1 + 4 files changed, 89 insertions(+), 16 deletions(-) diff --git a/src/ipa/rkisp1/algorithms/agc.cpp b/src/ipa/rkisp1/algorithms/agc.cpp index 50980835..0311ecf4 100644 --- a/src/ipa/rkisp1/algorithms/agc.cpp +++ b/src/ipa/rkisp1/algorithms/agc.cpp @@ -16,6 +16,8 @@ #include +#include "libipa/histogram.h" + /** * \file agc.h */ @@ -43,6 +45,9 @@ static constexpr utils::Duration kMaxShutterSpeed = 60ms; /* Number of frames to wait before calculating stats on minimum exposure */ static constexpr uint32_t kNumStartupFrames = 10; +/* Target value to reach for the top 2% of the histogram */ +static constexpr double kEvGainTarget = 0.5; + /* * Relative luminance target. * @@ -51,10 +56,10 @@ static constexpr uint32_t kNumStartupFrames = 10; * * \todo Why is the value different between IPU3 and RkISP1 ? */ -static constexpr double kRelativeLuminanceTarget = 0.4; +static constexpr double kRelativeLuminanceTarget = 0.16; Agc::Agc() - : frameCount_(0), numCells_(0), filteredExposure_(0s) + : frameCount_(0), numCells_(0), numHistBins_(0), filteredExposure_(0s) { } @@ -65,8 +70,7 @@ Agc::Agc() * * \return 0 */ -int Agc::configure(IPAContext &context, - [[maybe_unused]] const IPACameraSensorInfo &configInfo) +int Agc::configure(IPAContext &context, const IPACameraSensorInfo &configInfo) { /* Configure the default exposure and gain. */ context.frameContext.agc.gain = std::max(context.configuration.agc.minAnalogueGain, kMinAnalogueGain); @@ -77,10 +81,19 @@ int Agc::configure(IPAContext &context, * - versions < V12 have RKISP1_CIF_ISP_AE_MEAN_MAX_V10 entries, * - versions >= V12 have RKISP1_CIF_ISP_AE_MEAN_MAX_V12 entries. */ - if (context.configuration.hw.revision < RKISP1_V12) + if (context.configuration.hw.revision < RKISP1_V12) { numCells_ = RKISP1_CIF_ISP_AE_MEAN_MAX_V10; - else + numHistBins_ = RKISP1_CIF_ISP_HIST_BIN_N_MAX_V10; + } else { numCells_ = RKISP1_CIF_ISP_AE_MEAN_MAX_V12; + numHistBins_ = RKISP1_CIF_ISP_HIST_BIN_N_MAX_V12; + } + + /* Define the measurement window for AGC. */ + context.configuration.agc.measureWindow.h_offs = configInfo.outputSize.width / 8; + context.configuration.agc.measureWindow.v_offs = configInfo.outputSize.height / 8; + context.configuration.agc.measureWindow.h_size = 3 * configInfo.outputSize.width / 4; + context.configuration.agc.measureWindow.v_size = 3 * configInfo.outputSize.height / 4; return 0; } @@ -125,7 +138,7 @@ utils::Duration Agc::filterExposure(utils::Duration exposureValue) * \param[inout] frameContext The shared IPA frame Context * \param[in] yGain The gain calculated on the current brightness level */ -void Agc::computeExposure(IPAContext &context, double yGain) +void Agc::computeExposure(IPAContext &context, double yGain, double iqMeanGain) { IPASessionConfiguration &configuration = context.configuration; IPAFrameContext &frameContext = context.frameContext; @@ -134,6 +147,9 @@ void Agc::computeExposure(IPAContext &context, double yGain) uint32_t exposure = frameContext.sensor.exposure; double analogueGain = frameContext.sensor.gain; + /* Use the highest of the two gain estimates. */ + double evGain = std::max(yGain, iqMeanGain); + utils::Duration minShutterSpeed = configuration.agc.minShutterSpeed; utils::Duration maxShutterSpeed = std::min(configuration.agc.maxShutterSpeed, kMaxShutterSpeed); @@ -144,7 +160,7 @@ void Agc::computeExposure(IPAContext &context, double yGain) kMaxAnalogueGain); /* Consider within 1% of the target as correctly exposed. */ - if (utils::abs_diff(yGain, 1.0) < 0.01) + if (utils::abs_diff(evGain, 1.0) < 0.01) return; /* extracted from Rpi::Agc::computeTargetExposure. */ @@ -161,13 +177,13 @@ void Agc::computeExposure(IPAContext &context, double yGain) LOG(RkISP1Agc, Debug) << "Actual total exposure " << currentShutter * analogueGain << " Shutter speed " << currentShutter << " Gain " << analogueGain - << " Needed ev gain " << yGain; + << " Needed ev gain " << evGain; /* * Calculate the current exposure value for the scene as the latest * exposure value applied multiplied by the new estimated gain. */ - utils::Duration exposureValue = effectiveExposureValue * yGain; + utils::Duration exposureValue = effectiveExposureValue * evGain; /* Clamp the exposure value to the min and max authorized. */ utils::Duration maxTotalExposure = maxShutterSpeed * maxAnalogueGain; @@ -238,6 +254,23 @@ double Agc::estimateLuminance(const rkisp1_cif_isp_ae_stat *ae, return ySum / numCells_ / 255; } +/** + * \brief Estimate the mean value of the top 2% of the histogram + * \param[in] hist The histogram statistics computed by the ImgU + * \return The mean value of the top 2% of the histogram + */ +double Agc::measureBrightness(const rkisp1_cif_isp_hist_stat *hist) const +{ + std::vector measHist(numHistBins_); + + /* Initialise the histogram array using the maximum available size */ + for (unsigned int histBin = 0; histBin < numHistBins_; histBin++) + measHist.push_back(hist->hist_bins[histBin]); + + /* Estimate the quantile mean of the top 2% of the histogram. */ + return Histogram(Span(measHist)).interQuantileMean(0.98, 1.0); +} + /** * \brief Process RkISP1 statistics, and run AGC operations * \param[in] context The shared IPA context @@ -252,9 +285,13 @@ void Agc::process(IPAContext &context, const rkisp1_stat_buffer *stats) ASSERT(stats->meas_type & RKISP1_CIF_ISP_STAT_AUTOEXP); const rkisp1_cif_isp_ae_stat *ae = ¶ms->ae; + const rkisp1_cif_isp_hist_stat *hist = ¶ms->hist; frameCount_ = context.frameContext.frameId; + double iqMean = measureBrightness(hist); + double iqMeanGain = kEvGainTarget * numHistBins_ / iqMean; + /* * Estimate the gain needed to achieve a relative luminance target. To * account for non-linearity caused by saturation, the value needs to be @@ -277,14 +314,44 @@ void Agc::process(IPAContext &context, const rkisp1_stat_buffer *stats) break; } - computeExposure(context, yGain); + computeExposure(context, yGain, iqMeanGain); } -void Agc::prepare([[maybe_unused]] IPAContext &context, - rkisp1_params_cfg *params) +/** + * \copydoc libcamera::ipa::Algorithm::prepare + */ +void Agc::prepare(IPAContext &context, rkisp1_params_cfg *params) { - params->module_ens |= RKISP1_CIF_ISP_MODULE_AEC; - params->module_en_update |= RKISP1_CIF_ISP_MODULE_AEC; + /* Configure the measurement window. */ + params->meas.aec_config.meas_window = context.configuration.agc.measureWindow; + /* Use a continuous methode for measure. */ + params->meas.aec_config.autostop = RKISP1_CIF_ISP_EXP_CTRL_AUTOSTOP_0; + /* Estimate Y as (R + G + B) x (85/256). */ + params->meas.aec_config.mode = RKISP1_CIF_ISP_EXP_MEASURING_MODE_1; + + if (context.frameContext.frameId == 0) { + params->module_cfg_update |= RKISP1_CIF_ISP_MODULE_AEC; + params->module_ens |= RKISP1_CIF_ISP_MODULE_AEC; + params->module_en_update |= RKISP1_CIF_ISP_MODULE_AEC; + } + + /* Configure histogram. */ + params->meas.hst_config.meas_window = context.configuration.agc.measureWindow; + /* Produce the luminance histogram. */ + params->meas.hst_config.mode = RKISP1_CIF_ISP_HISTOGRAM_MODE_Y_HISTOGRAM; + /* Set an average weighted histogram. */ + for (unsigned int histBin = 0; histBin < numHistBins_; histBin++) + params->meas.hst_config.hist_weight[histBin] = 1; + /* Step size can't be less than 3. */ + params->meas.hst_config.histogram_predivider = 4; + + if (context.frameContext.frameId == 0) { + /* Update the configuration for histogram. */ + params->module_cfg_update |= RKISP1_CIF_ISP_MODULE_HST; + /* Enable the histogram measure unit. */ + params->module_ens |= RKISP1_CIF_ISP_MODULE_HST; + params->module_en_update |= RKISP1_CIF_ISP_MODULE_HST; + } } } /* namespace ipa::rkisp1::algorithms */ diff --git a/src/ipa/rkisp1/algorithms/agc.h b/src/ipa/rkisp1/algorithms/agc.h index 942c9d7a..872776d0 100644 --- a/src/ipa/rkisp1/algorithms/agc.h +++ b/src/ipa/rkisp1/algorithms/agc.h @@ -32,13 +32,15 @@ public: void process(IPAContext &context, const rkisp1_stat_buffer *stats) override; private: - void computeExposure(IPAContext &Context, double yGain); + void computeExposure(IPAContext &Context, double yGain, double iqMeanGain); utils::Duration filterExposure(utils::Duration exposureValue); double estimateLuminance(const rkisp1_cif_isp_ae_stat *ae, double gain); + double measureBrightness(const rkisp1_cif_isp_hist_stat *hist) const; uint64_t frameCount_; uint32_t numCells_; + uint32_t numHistBins_; utils::Duration filteredExposure_; }; diff --git a/src/ipa/rkisp1/ipa_context.cpp b/src/ipa/rkisp1/ipa_context.cpp index 992c9225..39acef47 100644 --- a/src/ipa/rkisp1/ipa_context.cpp +++ b/src/ipa/rkisp1/ipa_context.cpp @@ -71,6 +71,9 @@ namespace libcamera::ipa::rkisp1 { * \var IPASessionConfiguration::agc.maxAnalogueGain * \brief Maximum analogue gain supported with the configured sensor * + * \var IPASessionConfiguration::agc.measureWindow + * \brief AGC measure window + * * \var IPASessionConfiguration::hw * \brief RkISP1-specific hardware information * diff --git a/src/ipa/rkisp1/ipa_context.h b/src/ipa/rkisp1/ipa_context.h index c447369f..9ac3b40c 100644 --- a/src/ipa/rkisp1/ipa_context.h +++ b/src/ipa/rkisp1/ipa_context.h @@ -22,6 +22,7 @@ struct IPASessionConfiguration { utils::Duration maxShutterSpeed; double minAnalogueGain; double maxAnalogueGain; + struct rkisp1_cif_isp_window measureWindow; } agc; struct { From patchwork Wed Feb 23 10:48:24 2022 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Jean-Michel Hautbois X-Patchwork-Id: 15375 Return-Path: X-Original-To: parsemail@patchwork.libcamera.org Delivered-To: parsemail@patchwork.libcamera.org Received: from lancelot.ideasonboard.com (lancelot.ideasonboard.com [92.243.16.209]) by patchwork.libcamera.org (Postfix) with ESMTPS id 8EE4EBE08A for ; Wed, 23 Feb 2022 10:48:34 +0000 (UTC) Received: from lancelot.ideasonboard.com (localhost [IPv6:::1]) by lancelot.ideasonboard.com (Postfix) with ESMTP id EC80B6115E; Wed, 23 Feb 2022 11:48:31 +0100 (CET) Authentication-Results: lancelot.ideasonboard.com; dkim=fail reason="signature verification failed" (1024-bit key; unprotected) header.d=ideasonboard.com header.i=@ideasonboard.com header.b="YFDooe7j"; dkim-atps=neutral Received: from perceval.ideasonboard.com (perceval.ideasonboard.com [213.167.242.64]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id F01D961144 for ; Wed, 23 Feb 2022 11:48:28 +0100 (CET) Received: from tatooine.ideasonboard.com (unknown [IPv6:2a01:e0a:169:7140:7ebc:9e38:c4df:c57]) by perceval.ideasonboard.com (Postfix) with ESMTPSA id B18CADD; Wed, 23 Feb 2022 11:48:28 +0100 (CET) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=ideasonboard.com; s=mail; t=1645613308; bh=U7fTjbeaJ4b7A8MX13E3LyauBLmFSa1s9NLsohoJkQU=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=YFDooe7jz+IgNbErrchlUJwdqn5csPl0UXurs7uvTcxDfHjxzAbMlgLa0P++zd+Kr nfcb9j6adj8oDP44VvculetbKrRKTSjLbUf9pe1y7ylHEqE80AiBYaKh4RxfBNHBOH 3HuAcUdX/MYWox4H1lYFcBhMpKkIsjXPWXU7nssA= From: Jean-Michel Hautbois To: libcamera-devel@lists.libcamera.org Date: Wed, 23 Feb 2022 11:48:24 +0100 Message-Id: <20220223104824.25723-5-jeanmichel.hautbois@ideasonboard.com> X-Mailer: git-send-email 2.32.0 In-Reply-To: <20220223104824.25723-1-jeanmichel.hautbois@ideasonboard.com> References: <20220223104824.25723-1-jeanmichel.hautbois@ideasonboard.com> MIME-Version: 1.0 Subject: [libcamera-devel] [PATCH v2 4/4] ipa: rkisp1: Introduce AWB X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: libcamera-devel-bounces@lists.libcamera.org Sender: "libcamera-devel" The RkISP1 ISP calculates a mean value for Y, Cr and Cb at each frame. There is a RGB mode which could theoretically give us the values for R, G and B directly, but it seems to be failing right now. Convert those values into R, G and B and estimate the gain to apply in a grey world. Signed-off-by: Jean-Michel Hautbois Tested-by: Peter Griffin --- src/ipa/rkisp1/algorithms/awb.cpp | 155 ++++++++++++++++++++++++++ src/ipa/rkisp1/algorithms/awb.h | 33 ++++++ src/ipa/rkisp1/algorithms/meson.build | 1 + src/ipa/rkisp1/ipa_context.cpp | 28 +++++ src/ipa/rkisp1/ipa_context.h | 16 +++ src/ipa/rkisp1/rkisp1.cpp | 2 + 6 files changed, 235 insertions(+) create mode 100644 src/ipa/rkisp1/algorithms/awb.cpp create mode 100644 src/ipa/rkisp1/algorithms/awb.h diff --git a/src/ipa/rkisp1/algorithms/awb.cpp b/src/ipa/rkisp1/algorithms/awb.cpp new file mode 100644 index 00000000..8fa8c2f7 --- /dev/null +++ b/src/ipa/rkisp1/algorithms/awb.cpp @@ -0,0 +1,155 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2021, Ideas On Board + * + * awb.cpp - AWB control algorithm + */ + +#include "awb.h" + +#include +#include + +#include + +#include + +/** + * \file awb.h + */ + +namespace libcamera { + +namespace ipa::rkisp1::algorithms { + +/** + * \class Awb + * \brief A Grey world white balance correction algorithm + */ + +LOG_DEFINE_CATEGORY(RkISP1Awb) + +/** + * \copydoc libcamera::ipa::Algorithm::configure + */ +int Awb::configure(IPAContext &context, + const IPACameraSensorInfo &configInfo) +{ + context.frameContext.awb.gains.red = 1.0; + context.frameContext.awb.gains.blue = 1.0; + context.frameContext.awb.gains.green = 1.0; + + /* Define the measurement window for AGC. */ + context.configuration.awb.measureWindow.h_offs = configInfo.outputSize.width / 8; + context.configuration.awb.measureWindow.v_offs = configInfo.outputSize.height / 8; + context.configuration.awb.measureWindow.h_size = 3 * configInfo.outputSize.width / 4; + context.configuration.awb.measureWindow.v_size = 3 * configInfo.outputSize.height / 4; + + return 0; +} + +uint32_t Awb::estimateCCT(double red, double green, double blue) +{ + /* Convert the RGB values to CIE tristimulus values (XYZ) */ + double X = (-0.14282) * (red) + (1.54924) * (green) + (-0.95641) * (blue); + double Y = (-0.32466) * (red) + (1.57837) * (green) + (-0.73191) * (blue); + double Z = (-0.68202) * (red) + (0.77073) * (green) + (0.56332) * (blue); + + /* Calculate the normalized chromaticity values */ + double x = X / (X + Y + Z); + double y = Y / (X + Y + Z); + + /* Calculate CCT */ + double n = (x - 0.3320) / (0.1858 - y); + return 449 * n * n * n + 3525 * n * n + 6823.3 * n + 5520.33; +} + +/** + * \copydoc libcamera::ipa::Algorithm::process + */ +void Awb::process([[maybe_unused]] IPAContext &context, const rkisp1_stat_buffer *stats) +{ + const rkisp1_cif_isp_stat *params = &stats->params; + const rkisp1_cif_isp_awb_stat *awb = ¶ms->awb; + + /* Get the YCbCr mean values */ + double yMean = awb->awb_mean[0].mean_y_or_g; + double crMean = awb->awb_mean[0].mean_cr_or_r; + double cbMean = awb->awb_mean[0].mean_cb_or_b; + + /* Convert from YCbCr to RGB. */ + double redMean = yMean + 1.402 * (crMean - 128); + double blueMean = yMean + 1.772 * (cbMean - 128); + double greenMean = yMean - 0.34414 * (cbMean - 128) - 0.71414 * (crMean - 128); + + /* Estimate the red and blue gains to apply in a grey world. */ + double redGain = greenMean / (redMean + 1); + double blueGain = greenMean / (blueMean + 1); + + /* Filter the values to avoid oscillations. */ + IPAFrameContext &frameContext = context.frameContext; + + frameContext.awb.temperatureK = estimateCCT(redMean, greenMean, blueMean); + frameContext.awb.gains.red = 0.2 * redGain + + 0.8 * frameContext.awb.gains.red; + frameContext.awb.gains.blue = 0.2 * blueGain + + 0.8 * frameContext.awb.gains.blue; + /* Hardcode the green gain to 1.0. */ + frameContext.awb.gains.green = 1.0; + + LOG(RkISP1Awb, Debug) << "Gain found for red: " << context.frameContext.awb.gains.red + << " and for blue: " << context.frameContext.awb.gains.blue; +} + +/** + * \copydoc libcamera::ipa::Algorithm::prepare + */ +void Awb::prepare(IPAContext &context, + rkisp1_params_cfg *params) +{ + params->others.awb_gain_config.gain_green_b = 256 * context.frameContext.awb.gains.green; + params->others.awb_gain_config.gain_blue = 256 * context.frameContext.awb.gains.blue; + params->others.awb_gain_config.gain_red = 256 * context.frameContext.awb.gains.red; + params->others.awb_gain_config.gain_green_r = 256 * context.frameContext.awb.gains.green; + + if (context.frameContext.frameId == 0) { + /* Configure the gains to apply. */ + params->module_en_update |= RKISP1_CIF_ISP_MODULE_AWB_GAIN; + /* Update the ISP to apply the gains configured. */ + params->module_ens |= RKISP1_CIF_ISP_MODULE_AWB_GAIN; + } + /* Update the gains. */ + params->module_cfg_update |= RKISP1_CIF_ISP_MODULE_AWB_GAIN; + + /* Configure the measure window for AWB. */ + params->meas.awb_meas_config.awb_wnd = context.configuration.awb.measureWindow; + /* + * Measure Y, Cr and Cb means. + * \todo RGB is not working, the kernel seems to not configure it ? + */ + params->meas.awb_meas_config.awb_mode = RKISP1_CIF_ISP_AWB_MODE_YCBCR; + /* Reference Cr and Cb. */ + params->meas.awb_meas_config.awb_ref_cb = 128; + params->meas.awb_meas_config.awb_ref_cr = 128; + /* Y values to include are between min_y and max_y only. */ + params->meas.awb_meas_config.min_y = 16; + params->meas.awb_meas_config.max_y = 250; + /* Maximum Cr+Cb value to take into account for awb. */ + params->meas.awb_meas_config.max_csum = 250; + /* Minimum Cr and Cb values to take into account. */ + params->meas.awb_meas_config.min_c = 16; + /* Number of frames to use to estimate the mean (0 means 1 frame). */ + params->meas.awb_meas_config.frames = 0; + + if (context.frameContext.frameId == 0) { + /* Update AWB measurement unit configuration. */ + params->module_cfg_update |= RKISP1_CIF_ISP_MODULE_AWB; + /* Make sure the ISP is measuring the means for the next frame. */ + params->module_en_update |= RKISP1_CIF_ISP_MODULE_AWB; + params->module_ens |= RKISP1_CIF_ISP_MODULE_AWB; + } +} + +} /* namespace ipa::rkisp1::algorithms */ + +} /* namespace libcamera */ diff --git a/src/ipa/rkisp1/algorithms/awb.h b/src/ipa/rkisp1/algorithms/awb.h new file mode 100644 index 00000000..0a9fb82c --- /dev/null +++ b/src/ipa/rkisp1/algorithms/awb.h @@ -0,0 +1,33 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2021, Ideas On Board + * + * awb.h - AWB control algorithm + */ + +#pragma once + +#include + +#include "algorithm.h" + +namespace libcamera { + +namespace ipa::rkisp1::algorithms { + +class Awb : public Algorithm +{ +public: + Awb() = default; + ~Awb() = default; + + int configure(IPAContext &context, const IPACameraSensorInfo &configInfo) override; + void prepare(IPAContext &context, rkisp1_params_cfg *params) override; + void process(IPAContext &context, const rkisp1_stat_buffer *stats) override; + +private: + uint32_t estimateCCT(double red, double green, double blue); +}; + +} /* namespace ipa::rkisp1::algorithms */ +} /* namespace libcamera */ diff --git a/src/ipa/rkisp1/algorithms/meson.build b/src/ipa/rkisp1/algorithms/meson.build index 27c97731..7ec53d89 100644 --- a/src/ipa/rkisp1/algorithms/meson.build +++ b/src/ipa/rkisp1/algorithms/meson.build @@ -2,5 +2,6 @@ rkisp1_ipa_algorithms = files([ 'agc.cpp', + 'awb.cpp', 'blc.cpp', ]) diff --git a/src/ipa/rkisp1/ipa_context.cpp b/src/ipa/rkisp1/ipa_context.cpp index 39acef47..c25f44ec 100644 --- a/src/ipa/rkisp1/ipa_context.cpp +++ b/src/ipa/rkisp1/ipa_context.cpp @@ -81,6 +81,14 @@ namespace libcamera::ipa::rkisp1 { * \brief Hardware revision of the ISP */ +/** + * \var IPASessionConfiguration::awb + * \brief AWB parameters configuration of the IPA + * + * \var IPASessionConfiguration::awb.measureWindow + * \brief AWB measure window + */ + /** * \var IPASessionConfiguration::sensor * \brief Sensor-specific configuration of the IPA @@ -105,6 +113,26 @@ namespace libcamera::ipa::rkisp1 { * The gain should be adapted to the sensor specific gain code before applying. */ +/** + * \var IPAFrameContext::awb + * \brief Context for the Automatic White Balance algorithm + * + * \struct IPAFrameContext::awb.gains + * \brief White balance gains + * + * \var IPAFrameContext::awb.gains.red + * \brief White balance gain for R channel + * + * \var IPAFrameContext::awb.gains.green + * \brief White balance gain for G channel + * + * \var IPAFrameContext::awb.gains.blue + * \brief White balance gain for B channel + * + * \var IPAFrameContext::awb.temperatureK + * \brief Estimated color temperature + */ + /** * \var IPAFrameContext::sensor * \brief Effective sensor values diff --git a/src/ipa/rkisp1/ipa_context.h b/src/ipa/rkisp1/ipa_context.h index 9ac3b40c..51eae8b6 100644 --- a/src/ipa/rkisp1/ipa_context.h +++ b/src/ipa/rkisp1/ipa_context.h @@ -12,6 +12,8 @@ #include +#include + namespace libcamera { namespace ipa::rkisp1 { @@ -25,6 +27,10 @@ struct IPASessionConfiguration { struct rkisp1_cif_isp_window measureWindow; } agc; + struct { + struct rkisp1_cif_isp_window measureWindow; + } awb; + struct { utils::Duration lineDuration; } sensor; @@ -40,6 +46,16 @@ struct IPAFrameContext { double gain; } agc; + struct { + struct { + double red; + double green; + double blue; + } gains; + + double temperatureK; + } awb; + struct { uint32_t exposure; double gain; diff --git a/src/ipa/rkisp1/rkisp1.cpp b/src/ipa/rkisp1/rkisp1.cpp index f82b7cb3..381fc0e0 100644 --- a/src/ipa/rkisp1/rkisp1.cpp +++ b/src/ipa/rkisp1/rkisp1.cpp @@ -27,6 +27,7 @@ #include "algorithms/agc.h" #include "algorithms/algorithm.h" +#include "algorithms/awb.h" #include "algorithms/blc.h" #include "libipa/camera_sensor_helper.h" @@ -126,6 +127,7 @@ int IPARkISP1::init(const IPASettings &settings, unsigned int hwRevision) /* Construct our Algorithms */ algorithms_.push_back(std::make_unique()); + algorithms_.push_back(std::make_unique()); algorithms_.push_back(std::make_unique()); return 0;