From patchwork Tue Mar 16 10:18:41 2021 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Jean-Michel Hautbois X-Patchwork-Id: 11584 Return-Path: X-Original-To: parsemail@patchwork.libcamera.org Delivered-To: parsemail@patchwork.libcamera.org Received: from lancelot.ideasonboard.com (lancelot.ideasonboard.com [92.243.16.209]) by patchwork.libcamera.org (Postfix) with ESMTPS id 14EA1BD80C for ; Tue, 16 Mar 2021 10:18:57 +0000 (UTC) Received: from lancelot.ideasonboard.com (localhost [IPv6:::1]) by lancelot.ideasonboard.com (Postfix) with ESMTP id C57DA68D46; Tue, 16 Mar 2021 11:18:56 +0100 (CET) Authentication-Results: lancelot.ideasonboard.com; dkim=fail reason="signature verification failed" (1024-bit key; unprotected) header.d=ideasonboard.com header.i=@ideasonboard.com header.b="n3rb0U0J"; dkim-atps=neutral Received: from perceval.ideasonboard.com (perceval.ideasonboard.com [213.167.242.64]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id 2AC0F602E8 for ; Tue, 16 Mar 2021 11:18:55 +0100 (CET) Received: from localhost.localdomain (unknown [IPv6:2a01:e0a:169:7140:f840:1816:ba68:a009]) by perceval.ideasonboard.com (Postfix) with ESMTPSA id D005C13CC; Tue, 16 Mar 2021 11:18:54 +0100 (CET) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=ideasonboard.com; s=mail; t=1615889934; bh=nzw4QLjEJdYoJP5wfoaBoNkRsiLj5ozaSOFSmE0EwLs=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=n3rb0U0J1Vqf1XOMAx7rnC3Xp2bPNHq8/QnyoWC3ysWMZvf23hIMNanIfqd5qQwLY W1A3hPWPD0WF1L15e5stTnGuYz+cDumMCf1n9x+pUxT+Ys8Kb5MmTBsFFdHdF36w5U 4q/gndh2GScNTpMA6b5Vrt/NLJwcMvvX7/gE/Lew= From: Jean-Michel Hautbois To: libcamera-devel@lists.libcamera.org Date: Tue, 16 Mar 2021 11:18:41 +0100 Message-Id: <20210316101842.18674-5-jeanmichel.hautbois@ideasonboard.com> X-Mailer: git-send-email 2.27.0 In-Reply-To: <20210316101842.18674-1-jeanmichel.hautbois@ideasonboard.com> References: <20210316101842.18674-1-jeanmichel.hautbois@ideasonboard.com> MIME-Version: 1.0 Subject: [libcamera-devel] [RFC PATCH v2 2/3] WIP: ipa: ipu3: Add support for IPU3 AWB algorithm X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: libcamera-devel-bounces@lists.libcamera.org Sender: "libcamera-devel" Inherit from the Algorithm class to implement basic AWB functions. Once AWB is done, a color temperature is estimated and default CCM matrices are used (yet to be tuned). Implement a basic "grey-world" AWB algorithm just for demonstration purpose. Signed-off-by: Jean-Michel Hautbois --- src/ipa/ipu3/ipu3.cpp | 26 +++-- src/ipa/ipu3/ipu3_awb.cpp | 199 ++++++++++++++++++++++++++++++++++++++ src/ipa/ipu3/ipu3_awb.h | 47 +++++++++ src/ipa/ipu3/meson.build | 7 +- 4 files changed, 270 insertions(+), 9 deletions(-) create mode 100644 src/ipa/ipu3/ipu3_awb.cpp create mode 100644 src/ipa/ipu3/ipu3_awb.h diff --git a/src/ipa/ipu3/ipu3.cpp b/src/ipa/ipu3/ipu3.cpp index b63e58be..6fae5160 100644 --- a/src/ipa/ipu3/ipu3.cpp +++ b/src/ipa/ipu3/ipu3.cpp @@ -21,6 +21,8 @@ #include "libcamera/internal/buffer.h" #include "libcamera/internal/log.h" +#include "ipu3_awb.h" + namespace libcamera { LOG_DEFINE_CATEGORY(IPAIPU3) @@ -60,6 +62,11 @@ private: uint32_t gain_; uint32_t minGain_; uint32_t maxGain_; + + /* Interface to the AWB algorithm */ + ipa::IPU3Awb *awbAlgo_; + /* Local parameter storage */ + ipu3_uapi_params params_; }; void IPAIPU3::configure(const std::map &entityControls) @@ -83,11 +90,16 @@ void IPAIPU3::configure(const std::map &entityControls minExposure_ = std::max(itExp->second.min().get(), 1); maxExposure_ = itExp->second.max().get(); - exposure_ = maxExposure_; + exposure_ = minExposure_; minGain_ = std::max(itGain->second.min().get(), 1); maxGain_ = itGain->second.max().get(); - gain_ = maxGain_; + gain_ = minGain_; + + params_ = {}; + + awbAlgo_ = new ipa::IPU3Awb(); + awbAlgo_->initialise(params_); setControls(0); } @@ -161,10 +173,8 @@ void IPAIPU3::processControls([[maybe_unused]] unsigned int frame, void IPAIPU3::fillParams(unsigned int frame, ipu3_uapi_params *params) { - /* Prepare parameters buffer. */ - memset(params, 0, sizeof(*params)); - - /* \todo Fill in parameters buffer. */ + awbAlgo_->updateWbParameters(params_); + *params = params_; ipa::ipu3::IPU3Action op; op.op = ipa::ipu3::ActionParamFilled; @@ -177,8 +187,8 @@ void IPAIPU3::parseStatistics(unsigned int frame, { ControlList ctrls(controls::controls); - /* \todo React to statistics and update internal state machine. */ - /* \todo Add meta-data information to ctrls. */ + awbAlgo_->calculateWBGains(Rectangle(250, 160, 800, 400), stats); + setControls(frame); ipa::ipu3::IPU3Action op; op.op = ipa::ipu3::ActionMetadataReady; diff --git a/src/ipa/ipu3/ipu3_awb.cpp b/src/ipa/ipu3/ipu3_awb.cpp new file mode 100644 index 00000000..3ff239c0 --- /dev/null +++ b/src/ipa/ipu3/ipu3_awb.cpp @@ -0,0 +1,199 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (C) 2019, Raspberry Pi (Trading) Limited + * + * ipu3_awb.cpp - AWB control algorithm + */ +#include +#include + +#include "libcamera/internal/log.h" + +#include "ipu3_awb.h" + +namespace libcamera { + +namespace ipa { + +LOG_DEFINE_CATEGORY(IPU3Awb) + +static const struct ipu3_uapi_bnr_static_config imgu_css_bnr_defaults = { + { 16, 16, 16, 16 }, /* wb_gains */ + { 255, 255, 255, 255 }, /* wb_gains_thr */ + { 0, 0, 8, 6, 0, 14 }, /* thr_coeffs */ + { 0, 0, 0, 0 }, /* thr_ctrl_shd */ + { -648, 0, -366, 0 }, /* opt_center */ + { /* lut */ + { 17, 23, 28, 32, 36, 39, 42, 45, + 48, 51, 53, 55, 58, 60, 62, 64, + 66, 68, 70, 72, 73, 75, 77, 78, + 80, 82, 83, 85, 86, 88, 89, 90 } }, + { 4, 0, 1, 8, 0, 8, 0, 8, 0 }, /* bp_ctrl */ + { 8, 4, 4, 0, 8, 0, 1, 1, 1, 1, 0 }, /* dn_detect_ctrl */ + 1296, + { 419904, 133956 }, +}; + +/* settings for Auto White Balance */ +static const struct ipu3_uapi_awb_config_s imgu_css_awb_defaults = { + 8191, + 8191, + 8191, + 8191 | /* rgbs_thr_gr/r/gb/b */ + IPU3_UAPI_AWB_RGBS_THR_B_EN | IPU3_UAPI_AWB_RGBS_THR_B_INCL_SAT, + .grid = { + .width = 160, + .height = 45, + .block_width_log2 = 3, + .block_height_log2 = 4, + .x_start = 0, + .y_start = 0, + }, +}; + +static const struct ipu3_uapi_ccm_mat_config imgu_css_ccm_6000k = { + 7239, -750, -37, 0, + -215, 8196, -200, 0, + -70, -589, 6810, 0 +}; + +static const struct ipu3_uapi_ccm_mat_config imgu_css_ccm_3800k = { + 7379, -526, -296, 0, + -411, 7397, -415, 0, + -224, -564, 7244, 0 +}; + +IPU3Awb::IPU3Awb() + : Algorithm() +{ +} + +IPU3Awb::~IPU3Awb() +{ +} + +void IPU3Awb::initialise() +{ +} + +void IPU3Awb::initialise(ipu3_uapi_params ¶ms) +{ + params.use.acc_awb = 1; + /*\todo fill the grid calculated based on BDS configuration */ + params.acc_param.awb.config = imgu_css_awb_defaults; + + params.use.acc_bnr = 1; + params.acc_param.bnr = imgu_css_bnr_defaults; + + params.use.acc_ccm = 1; + params.acc_param.ccm = imgu_css_ccm_3800k; + + params.use.acc_gamma = 1; + params.acc_param.gamma.gc_ctrl.enable = 1; + + uint32_t a = (32 * 245) / (245 - 9); + + for (uint32_t i = 0; i < 10; i++) + params.acc_param.gamma.gc_lut.lut[i] = 0; + for (uint32_t i = 10; i < 245; i++) + params.acc_param.gamma.gc_lut.lut[i] = a * i + (0 - a * 9); + for (uint32_t i = 245; i < 255; i++) + params.acc_param.gamma.gc_lut.lut[i] = 32 * 245; + + wbGains_[0] = 8192 * 0.8; + wbGains_[1] = 8192; + wbGains_[2] = 8192; + wbGains_[3] = 8192 * 0.8; + + frame_count_ = 0; +} + +uint32_t IPU3Awb::estimateCCT(uint8_t red, uint8_t green, uint8_t blue) +{ + double X = (-0.14282) * (red) + (1.54924) * (green) + (-0.95641) * (blue); + double Y = (-0.32466) * (red) + (1.57837) * (green) + (-0.73191) * (blue); + double Z = (-0.68202) * (red) + (0.77073) * (green) + (0.56332) * (blue); + + double x = X / (X + Y + Z); + double y = Y / (X + Y + Z); + + double n = (x - 0.3320) / (0.1858 - y); + return static_cast(449 * n * n * n + 3525 * n * n + 6823.3 * n + 5520.33); +} + +void IPU3Awb::calculateWBGains([[maybe_unused]] Rectangle roi, + const ipu3_uapi_stats_3a *stats) +{ + std::vector redValues, greenRedValues, greenBlueValues, blueValues; + Point topleft = roi.topLeft(); + uint32_t startY = (topleft.y / 16) * 160 * 8; + uint32_t startX = (topleft.x / 8) * 8; + uint32_t endX = (startX + (roi.size().width / 8)) * 8; + + for (uint32_t j = (topleft.y / 16); j < (topleft.y / 16) + (roi.size().height / 16); j++) { + for (uint32_t i = startX + startY; i < endX + startY; i += 8) { + greenRedValues.push_back(stats->awb_raw_buffer.meta_data[i]); + redValues.push_back(stats->awb_raw_buffer.meta_data[i + 1]); + blueValues.push_back(stats->awb_raw_buffer.meta_data[i + 2]); + greenBlueValues.push_back(stats->awb_raw_buffer.meta_data[i + 3]); + } + } + + std::sort(redValues.begin(), redValues.end()); + std::sort(greenRedValues.begin(), greenRedValues.end()); + std::sort(blueValues.begin(), blueValues.end()); + std::sort(greenBlueValues.begin(), greenBlueValues.end()); + + double Grmed = greenRedValues[greenRedValues.size() / 2]; + double Rmed = redValues[redValues.size() / 2]; + double Bmed = blueValues[blueValues.size() / 2]; + double Gbmed = greenBlueValues[greenBlueValues.size() / 2]; + + double Rgain = Grmed / Rmed; + double Bgain = Gbmed / Bmed; + LOG(IPU3Awb, Debug) << "max R, Gr, B, Gb: " + << redValues.back() << "," + << greenRedValues.back() << "," + << blueValues.back() << "," + << greenBlueValues.back(); + tint_ = ((Rmed / Grmed) + (Bmed / Gbmed)) / 2; + + /* \todo Those are corrections when light is really low + * it should be taken into account by AGC somehow */ + if ((Rgain >= 2) && (Bgain < 2)) { + wbGains_[0] = 4096 * tint_; + wbGains_[1] = 8192 * Rgain; + wbGains_[2] = 4096 * Bgain; + wbGains_[3] = 4096 * tint_; + } else if ((Bgain >= 2) && (Rgain < 2)) { + wbGains_[0] = 4096 * tint_; + wbGains_[1] = 4096 * Rgain; + wbGains_[2] = 8192 * Bgain; + wbGains_[3] = 4096 * tint_; + } else { + wbGains_[0] = 8192 * tint_; + wbGains_[1] = 8192 * Rgain; + wbGains_[2] = 8192 * Bgain; + wbGains_[3] = 8192 * tint_; + } + + frame_count_++; + + cct_ = estimateCCT(Rmed, (Grmed + Gbmed) / 2, Bmed); +} + +void IPU3Awb::updateWbParameters(ipu3_uapi_params ¶ms) +{ + params.acc_param.bnr.wb_gains.gr = wbGains_[0]; + params.acc_param.bnr.wb_gains.r = wbGains_[1]; + params.acc_param.bnr.wb_gains.b = wbGains_[2]; + params.acc_param.bnr.wb_gains.gb = wbGains_[3]; + if (cct_ < 5500) + params.acc_param.ccm = imgu_css_ccm_3800k; + else + params.acc_param.ccm = imgu_css_ccm_6000k; +} + +} /* namespace ipa */ + +} /* namespace libcamera */ \ No newline at end of file diff --git a/src/ipa/ipu3/ipu3_awb.h b/src/ipa/ipu3/ipu3_awb.h new file mode 100644 index 00000000..ff6906f2 --- /dev/null +++ b/src/ipa/ipu3/ipu3_awb.h @@ -0,0 +1,47 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (C) 2019, Raspberry Pi (Trading) Limited + * + * ipu3_awb.h - IPU3 AWB control algorithm + */ +#ifndef __LIBCAMERA_IPU3_AWB_H__ +#define __LIBCAMERA_IPU3_AWB_H__ + +#include + +#include + +#include "libipa/algorithm.h" + +namespace libcamera { + +namespace ipa { + +class IPU3Awb : public Algorithm +{ +public: + IPU3Awb(); + ~IPU3Awb(); + + void initialise() override; + + void initialise(ipu3_uapi_params ¶ms); + void calculateWBGains(Rectangle roi, + const ipu3_uapi_stats_3a *stats); + void updateWbParameters(ipu3_uapi_params ¶ms); + +private: + uint32_t estimateCCT(uint8_t red, uint8_t green, uint8_t blue); + + /* WB calculated gains */ + uint16_t wbGains_[4]; + double tint_; + uint32_t cct_; + + uint32_t frame_count_; +}; + +} /* namespace ipa */ + +} /* namespace libcamera*/ +#endif /* __LIBCAMERA_IPU3_AWB_H__ */ diff --git a/src/ipa/ipu3/meson.build b/src/ipa/ipu3/meson.build index a241f617..07a864c8 100644 --- a/src/ipa/ipu3/meson.build +++ b/src/ipa/ipu3/meson.build @@ -2,8 +2,13 @@ ipa_name = 'ipa_ipu3' +ipu3_ipa_sources = files([ + 'ipu3.cpp', + 'ipu3_awb.cpp', +]) + mod = shared_module(ipa_name, - ['ipu3.cpp', libcamera_generated_ipa_headers], + [ipu3_ipa_sources, libcamera_generated_ipa_headers], name_prefix : '', include_directories : [ipa_includes, libipa_includes], dependencies : libcamera_dep,