From patchwork Thu Mar 10 20:51:28 2022 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Umang Jain X-Patchwork-Id: 15439 X-Patchwork-Delegate: umang.jain@ideasonboard.com Return-Path: X-Original-To: parsemail@patchwork.libcamera.org Delivered-To: parsemail@patchwork.libcamera.org Received: from lancelot.ideasonboard.com (lancelot.ideasonboard.com [92.243.16.209]) by patchwork.libcamera.org (Postfix) with ESMTPS id 401FBBF415 for ; Thu, 10 Mar 2022 20:51:43 +0000 (UTC) Received: from lancelot.ideasonboard.com (localhost [IPv6:::1]) by lancelot.ideasonboard.com (Postfix) with ESMTP id ED0BC632E2; Thu, 10 Mar 2022 21:51:42 +0100 (CET) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=libcamera.org; s=mail; t=1646945503; bh=6bRgsnx2Uz4G6N66Zb6iPnZZRza1TY0By0XaUzbovNM=; h=To:Date:In-Reply-To:References:Subject:List-Id:List-Unsubscribe: List-Archive:List-Post:List-Help:List-Subscribe:From:Reply-To: From; b=13rqmajSCDWAhoZ244LNLJ8l98x1noP9jM8Dlrzo1sX14+IYFhiRV+w9Bw7db3p/O 7hg2V8CiQ912xylzaHx8uDy01vmkarrZBIc6bVUIQyqjxDRMQvjq6NCQAV+BxMAoEr UgkFxiCSDqRr+buig9wrph8Pnh2yoNCA7U5VfBbDky5QMfVOwEqtf/5kQYhCOFCth6 swYxAqj1bdYnilRC022rxQykyYmFVq/zBabLeSlizKde8QS03xF7sNwb/Ver3mVDE3 NOIKVCtyeOK7lhLqdoIYNKBMn8CUkl3ZVztlyIynEAAfy2+c3+Bi+RjRH6V1lzpHoC S5c9enxvdbLTQ== Received: from perceval.ideasonboard.com (perceval.ideasonboard.com [213.167.242.64]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id 35398601F8 for ; Thu, 10 Mar 2022 21:51:40 +0100 (CET) Authentication-Results: lancelot.ideasonboard.com; dkim=pass (1024-bit key; unprotected) header.d=ideasonboard.com header.i=@ideasonboard.com header.b="d6kytAAy"; dkim-atps=neutral Received: from perceval.ideasonboard.com (unknown [103.251.226.203]) by perceval.ideasonboard.com (Postfix) with ESMTPSA id 32F90491; Thu, 10 Mar 2022 21:51:38 +0100 (CET) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=ideasonboard.com; s=mail; t=1646945500; bh=6bRgsnx2Uz4G6N66Zb6iPnZZRza1TY0By0XaUzbovNM=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=d6kytAAyTtUHU3yEc9TjdDPJ4vTo1i0uonZqmXc7RQCYnGq8/XNJnMzLfvtLUz8zY baC2dTuuH1t1OrfKYVaLQwOIZFe0WFem26hR5b0eFsZfWMheqGtmGHwg1NPstKV9Gw xXxccpPmxxkdA52Ewqpe2ff4PF2BHTBUDfTpH3zI= To: libcamera-devel@lists.libcamera.org Date: Fri, 11 Mar 2022 02:21:28 +0530 Message-Id: <20220310205130.336361-2-umang.jain@ideasonboard.com> X-Mailer: git-send-email 2.31.1 In-Reply-To: <20220310205130.336361-1-umang.jain@ideasonboard.com> References: <20220310205130.336361-1-umang.jain@ideasonboard.com> MIME-Version: 1.0 Subject: [libcamera-devel] [PATCH 1/3] libipa: algorithm: Pass frame number to prepare() and process() X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-Patchwork-Original-From: Umang Jain via libcamera-devel From: Umang Jain Reply-To: Umang Jain Errors-To: libcamera-devel-bounces@lists.libcamera.org Sender: "libcamera-devel" The frame number will be used to retrieve their respective IPAFrameContext from a container in subsequent commits. Signed-off-by: Umang Jain --- src/ipa/ipu3/algorithms/agc.cpp | 3 ++- src/ipa/ipu3/algorithms/agc.h | 2 +- src/ipa/ipu3/algorithms/awb.cpp | 4 ++-- src/ipa/ipu3/algorithms/awb.h | 4 ++-- src/ipa/ipu3/algorithms/blc.cpp | 6 ++++-- src/ipa/ipu3/algorithms/blc.h | 2 +- src/ipa/ipu3/algorithms/tone_mapping.cpp | 7 +++++-- src/ipa/ipu3/algorithms/tone_mapping.h | 4 ++-- src/ipa/ipu3/ipu3.cpp | 4 ++-- src/ipa/libipa/algorithm.cpp | 2 ++ src/ipa/libipa/algorithm.h | 6 ++++-- 11 files changed, 27 insertions(+), 17 deletions(-) diff --git a/src/ipa/ipu3/algorithms/agc.cpp b/src/ipa/ipu3/algorithms/agc.cpp index 1eb1bcef..cdc1327a 100644 --- a/src/ipa/ipu3/algorithms/agc.cpp +++ b/src/ipa/ipu3/algorithms/agc.cpp @@ -315,13 +315,14 @@ double Agc::estimateLuminance(IPAFrameContext &frameContext, /** * \brief Process IPU3 statistics, and run AGC operations + * \param frame The frame number * \param[in] context The shared IPA context * \param[in] stats The IPU3 statistics and ISP results * * Identify the current image brightness, and use that to estimate the optimal * new exposure and gain for the scene. */ -void Agc::process(IPAContext &context, const ipu3_uapi_stats_3a *stats) +void Agc::process(const uint32_t frame, IPAContext &context, const ipu3_uapi_stats_3a *stats) { /* * Estimate the gain needed to have the proportion of pixels in a given diff --git a/src/ipa/ipu3/algorithms/agc.h b/src/ipa/ipu3/algorithms/agc.h index ad705605..e48506e5 100644 --- a/src/ipa/ipu3/algorithms/agc.h +++ b/src/ipa/ipu3/algorithms/agc.h @@ -28,7 +28,7 @@ public: ~Agc() = default; int configure(IPAContext &context, const IPAConfigInfo &configInfo) override; - void process(IPAContext &context, const ipu3_uapi_stats_3a *stats) override; + void process(const uint32_t frame, IPAContext &context, const ipu3_uapi_stats_3a *stats) override; private: double measureBrightness(const ipu3_uapi_stats_3a *stats, diff --git a/src/ipa/ipu3/algorithms/awb.cpp b/src/ipa/ipu3/algorithms/awb.cpp index dc25be81..f1e753e6 100644 --- a/src/ipa/ipu3/algorithms/awb.cpp +++ b/src/ipa/ipu3/algorithms/awb.cpp @@ -387,7 +387,7 @@ void Awb::calculateWBGains(const ipu3_uapi_stats_3a *stats) /** * \copydoc libcamera::ipa::Algorithm::process */ -void Awb::process(IPAContext &context, const ipu3_uapi_stats_3a *stats) +void Awb::process(const uint32_t frame, IPAContext &context, const ipu3_uapi_stats_3a *stats) { calculateWBGains(stats); @@ -411,7 +411,7 @@ constexpr uint16_t Awb::threshold(float value) /** * \copydoc libcamera::ipa::Algorithm::prepare */ -void Awb::prepare(IPAContext &context, ipu3_uapi_params *params) +void Awb::prepare([[maybe_unused]] const uint32_t frame, IPAContext &context, ipu3_uapi_params *params) { /* * Green saturation thresholds are reduced because we are using the diff --git a/src/ipa/ipu3/algorithms/awb.h b/src/ipa/ipu3/algorithms/awb.h index ab4b0a33..ef177a4a 100644 --- a/src/ipa/ipu3/algorithms/awb.h +++ b/src/ipa/ipu3/algorithms/awb.h @@ -39,8 +39,8 @@ public: ~Awb(); int configure(IPAContext &context, const IPAConfigInfo &configInfo) override; - void prepare(IPAContext &context, ipu3_uapi_params *params) override; - void process(IPAContext &context, const ipu3_uapi_stats_3a *stats) override; + void prepare(const uint32_t frame, IPAContext &context, ipu3_uapi_params *params) override; + void process(const uint32_t frame, IPAContext &context, const ipu3_uapi_stats_3a *stats) override; private: /* \todo Make these structs available to all the ISPs ? */ diff --git a/src/ipa/ipu3/algorithms/blc.cpp b/src/ipa/ipu3/algorithms/blc.cpp index 78ab7bff..e9b4bed8 100644 --- a/src/ipa/ipu3/algorithms/blc.cpp +++ b/src/ipa/ipu3/algorithms/blc.cpp @@ -38,14 +38,16 @@ BlackLevelCorrection::BlackLevelCorrection() /** * \brief Fill in the parameter structure, and enable black level correction + * \param frame The frame number * \param context The shared IPA context * \param params The IPU3 parameters * * Populate the IPU3 parameter structure with the correction values for each * channel and enable the corresponding ImgU block processing. */ -void BlackLevelCorrection::prepare([[maybe_unused]] IPAContext &context, - ipu3_uapi_params *params) +void BlackLevelCorrection::prepare([[maybe_unused]] const uint32_t frame, + [[maybe_unused]] IPAContext &context, + ipu3_uapi_params *params) { /* * The Optical Black Level correction values diff --git a/src/ipa/ipu3/algorithms/blc.h b/src/ipa/ipu3/algorithms/blc.h index d8da1748..e4c22070 100644 --- a/src/ipa/ipu3/algorithms/blc.h +++ b/src/ipa/ipu3/algorithms/blc.h @@ -18,7 +18,7 @@ class BlackLevelCorrection : public Algorithm public: BlackLevelCorrection(); - void prepare(IPAContext &context, ipu3_uapi_params *params) override; + void prepare(const uint32_t frame, IPAContext &context, ipu3_uapi_params *params) override; }; } /* namespace ipa::ipu3::algorithms */ diff --git a/src/ipa/ipu3/algorithms/tone_mapping.cpp b/src/ipa/ipu3/algorithms/tone_mapping.cpp index 2040eda5..bba5bc9a 100644 --- a/src/ipa/ipu3/algorithms/tone_mapping.cpp +++ b/src/ipa/ipu3/algorithms/tone_mapping.cpp @@ -49,13 +49,15 @@ int ToneMapping::configure(IPAContext &context, /** * \brief Fill in the parameter structure, and enable gamma control + * \param frame The frame number * \param context The shared IPA context * \param params The IPU3 parameters * * Populate the IPU3 parameter structure with our tone mapping look up table and * enable the gamma control module in the processing blocks. */ -void ToneMapping::prepare([[maybe_unused]] IPAContext &context, +void ToneMapping::prepare([[maybe_unused]] const uint32_t frame, + [[maybe_unused]] IPAContext &context, ipu3_uapi_params *params) { /* Copy the calculated LUT into the parameters buffer. */ @@ -71,13 +73,14 @@ void ToneMapping::prepare([[maybe_unused]] IPAContext &context, /** * \brief Calculate the tone mapping look up table + * \param frame The frame number * \param context The shared IPA context * \param stats The IPU3 statistics and ISP results * * The tone mapping look up table is generated as an inverse power curve from * our gamma setting. */ -void ToneMapping::process(IPAContext &context, +void ToneMapping::process(const uint32_t frame, IPAContext &context, [[maybe_unused]] const ipu3_uapi_stats_3a *stats) { /* diff --git a/src/ipa/ipu3/algorithms/tone_mapping.h b/src/ipa/ipu3/algorithms/tone_mapping.h index b727ab1e..041c9c0d 100644 --- a/src/ipa/ipu3/algorithms/tone_mapping.h +++ b/src/ipa/ipu3/algorithms/tone_mapping.h @@ -19,8 +19,8 @@ public: ToneMapping(); int configure(IPAContext &context, const IPAConfigInfo &configInfo) override; - void prepare(IPAContext &context, ipu3_uapi_params *params) override; - void process(IPAContext &context, const ipu3_uapi_stats_3a *stats) override; + void prepare(const uint32_t frame, IPAContext &context, ipu3_uapi_params *params) override; + void process(const uint32_t frame, IPAContext &context, const ipu3_uapi_stats_3a *stats) override; private: double gamma_; diff --git a/src/ipa/ipu3/ipu3.cpp b/src/ipa/ipu3/ipu3.cpp index 8f079588..81788b9a 100644 --- a/src/ipa/ipu3/ipu3.cpp +++ b/src/ipa/ipu3/ipu3.cpp @@ -573,7 +573,7 @@ void IPAIPU3::fillParams(unsigned int frame, ipu3_uapi_params *params) params->use = {}; for (auto const &algo : algorithms_) - algo->prepare(context_, params); + algo->prepare(frame, context_, params); paramsBufferReady.emit(frame); } @@ -597,7 +597,7 @@ void IPAIPU3::parseStatistics(unsigned int frame, ControlList ctrls(controls::controls); for (auto const &algo : algorithms_) - algo->process(context_, stats); + algo->process(frame, context_, stats); setControls(frame); diff --git a/src/ipa/libipa/algorithm.cpp b/src/ipa/libipa/algorithm.cpp index 398d5372..01967f07 100644 --- a/src/ipa/libipa/algorithm.cpp +++ b/src/ipa/libipa/algorithm.cpp @@ -48,6 +48,7 @@ namespace ipa { /** * \fn Algorithm::prepare() * \brief Fill the \a params buffer with ISP processing parameters for a frame + * \param[in] frame The frame number * \param[in] context The shared IPA context * \param[out] params The ISP specific parameters. * @@ -63,6 +64,7 @@ namespace ipa { /** * \fn Algorithm::process() * \brief Process ISP statistics, and run algorithm operations + * \param[in] frame The frame number * \param[in] context The shared IPA context * \param[in] stats The IPA statistics and ISP results * diff --git a/src/ipa/libipa/algorithm.h b/src/ipa/libipa/algorithm.h index 766aee5d..1df93625 100644 --- a/src/ipa/libipa/algorithm.h +++ b/src/ipa/libipa/algorithm.h @@ -22,12 +22,14 @@ public: return 0; } - virtual void prepare([[maybe_unused]] Context &context, + virtual void prepare([[maybe_unused]] const uint32_t frame, + [[maybe_unused]] Context &context, [[maybe_unused]] Params *params) { } - virtual void process([[maybe_unused]] Context &context, + virtual void process([[maybe_unused]] const uint32_t frame, + [[maybe_unused]] Context &context, [[maybe_unused]] const Stats *stats) { } From patchwork Thu Mar 10 20:51:29 2022 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Umang Jain X-Patchwork-Id: 15440 X-Patchwork-Delegate: umang.jain@ideasonboard.com Return-Path: X-Original-To: parsemail@patchwork.libcamera.org Delivered-To: parsemail@patchwork.libcamera.org Received: from lancelot.ideasonboard.com (lancelot.ideasonboard.com [92.243.16.209]) by patchwork.libcamera.org (Postfix) with ESMTPS id 6B2C4BF415 for ; Thu, 10 Mar 2022 20:51:44 +0000 (UTC) Received: from lancelot.ideasonboard.com (localhost [IPv6:::1]) by lancelot.ideasonboard.com (Postfix) with ESMTP id AABEF632ED; Thu, 10 Mar 2022 21:51:43 +0100 (CET) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=libcamera.org; s=mail; t=1646945503; bh=VO8/u67fpPf51y20xXn2IZR/lGfY0XBRVR13z/rHl3Y=; h=To:Date:In-Reply-To:References:Subject:List-Id:List-Unsubscribe: List-Archive:List-Post:List-Help:List-Subscribe:From:Reply-To: From; b=K2MYaQDW5no1y9o3kSi8tuHntAcKHN+H0IlaTUR7lWoHs11kOJTr3N1nWesiJZFCG e3Cmr+efKbH3Q7/UpcsD7AYxoBpeNNdTzBSH+IZdSAEppTCUjnGqDQyA56lpiH6wc9 855w/RC8MMW1bpNVG9PvHpr71LrGoqC1H3FVUZhDRpxmGOXz6t2xx0u3NtjJqJnpkA kKbJ5JzjTbXzsIzdlt7d4QiB7x55DJbvHzGb15acwRl+muetzHLnXcaIVpeVPywk8l eLBD4DcbxyEr5hPN+y+f1nld6aK0mZyKtVFvz49RTzJMSptqj4eD6Jawh84b6lpadV onA1BEZp0C4kw== Received: from perceval.ideasonboard.com (perceval.ideasonboard.com [IPv6:2001:4b98:dc2:55:216:3eff:fef7:d647]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id 964F5604E9 for ; Thu, 10 Mar 2022 21:51:42 +0100 (CET) Authentication-Results: lancelot.ideasonboard.com; dkim=pass (1024-bit key; unprotected) header.d=ideasonboard.com header.i=@ideasonboard.com header.b="R6gv7Pjy"; dkim-atps=neutral Received: from perceval.ideasonboard.com (unknown [103.251.226.203]) by perceval.ideasonboard.com (Postfix) with ESMTPSA id 25109491; Thu, 10 Mar 2022 21:51:40 +0100 (CET) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=ideasonboard.com; s=mail; t=1646945502; bh=VO8/u67fpPf51y20xXn2IZR/lGfY0XBRVR13z/rHl3Y=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=R6gv7PjyoO2s+hlvODfxfPzo1z6qmhZbVxBJTn6Jljkxmxjo0kmzfQYYXFgVd5Yok mb1NfyXO1uZdBlzspAXykK1lJwpOyK9C4+7UB0GO5E6MrYmCoUvivPtd//se5KjZqX OidPG4WCWU5Wc9M1CJtwKm4INPnChlvqkBm/ZjLQ= To: libcamera-devel@lists.libcamera.org Date: Fri, 11 Mar 2022 02:21:29 +0530 Message-Id: <20220310205130.336361-3-umang.jain@ideasonboard.com> X-Mailer: git-send-email 2.31.1 In-Reply-To: <20220310205130.336361-1-umang.jain@ideasonboard.com> References: <20220310205130.336361-1-umang.jain@ideasonboard.com> MIME-Version: 1.0 Subject: [libcamera-devel] [PATCH 2/3] ipa: ipu3: Mark the beginning and end of a frame X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-Patchwork-Original-From: Umang Jain via libcamera-devel From: Umang Jain Reply-To: Umang Jain Errors-To: libcamera-devel-bounces@lists.libcamera.org Sender: "libcamera-devel" From: Jean-Michel Hautbois Introduce the skeleton for two functions which will be used to instantiate a frame context, and do everything needed when a frame is received. Do the same for the other end, once the algorithms have run and updated the frame context to later deallocate the corresponding frame context. Signed-off-by: Jean-Michel Hautbois Reviewed-by: Kieran Bingham Signed-off-by: Umang Jain --- src/ipa/ipu3/ipu3.cpp | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/src/ipa/ipu3/ipu3.cpp b/src/ipa/ipu3/ipu3.cpp index 81788b9a..3d5c5706 100644 --- a/src/ipa/ipu3/ipu3.cpp +++ b/src/ipa/ipu3/ipu3.cpp @@ -163,6 +163,14 @@ private: void setControls(unsigned int frame); void calculateBdsGrid(const Size &bdsOutputSize); + /* + * Internal events that mark the beginning of processing a new frame + * to the point that it has successfully completed processing its + * statistics. + */ + void frameStarted(const uint32_t frame); + void frameCompleted(const uint32_t frame); + std::map buffers_; ControlInfoMap ctrls_; @@ -491,6 +499,14 @@ void IPAIPU3::unmapBuffers(const std::vector &ids) } } +void IPAIPU3::frameStarted([[maybe_unused]] const uint32_t frame) +{ +} + +void IPAIPU3::frameCompleted([[maybe_unused]] const uint32_t frame) +{ +} + /** * \brief Prepare the ISP to process the Request * \param[in] frame The frame number @@ -549,6 +565,8 @@ void IPAIPU3::processControls(const uint32_t frame, [[maybe_unused]] const ControlList &controls) { /* \todo Start processing for 'frame' based on 'controls'. */ + + frameStarted(frame); } /** @@ -620,6 +638,8 @@ void IPAIPU3::parseStatistics(unsigned int frame, */ statsBufferReady.emit(frame, ctrls); + + frameCompleted(frame); } /** From patchwork Thu Mar 10 20:51:30 2022 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Umang Jain X-Patchwork-Id: 15441 X-Patchwork-Delegate: umang.jain@ideasonboard.com Return-Path: X-Original-To: parsemail@patchwork.libcamera.org Delivered-To: parsemail@patchwork.libcamera.org Received: from lancelot.ideasonboard.com (lancelot.ideasonboard.com [92.243.16.209]) by patchwork.libcamera.org (Postfix) with ESMTPS id 86320BF415 for ; Thu, 10 Mar 2022 20:51:46 +0000 (UTC) Received: from lancelot.ideasonboard.com (localhost [IPv6:::1]) by lancelot.ideasonboard.com (Postfix) with ESMTP id 37C35601F8; Thu, 10 Mar 2022 21:51:46 +0100 (CET) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=libcamera.org; s=mail; t=1646945506; bh=IVaFY/hnHs02d0E2RY5oc/+SqGASU0V6X5B/MkD0QQE=; h=To:Date:In-Reply-To:References:Subject:List-Id:List-Unsubscribe: List-Archive:List-Post:List-Help:List-Subscribe:From:Reply-To: From; b=gfq4zUAI3GbjIu0TVbZpGrhBonyqfxiZ8wV1U2fYofk2lQMhMx4EVDyTqbGc8870I yM6Pyh61+u+kL0irrJ0KrypdWuwD7qai2Ha/HbJEDdwgob2bmj43aBKO1//kYBUSQ3 dxkSlkhGLIJX0+8E/Uk7TVSW3ZbmeXzw3QkwTo6c2cZeOOPq61e/mL3OCwxOwnhT8H uNbPJLAgD6om3Y3fEq7qVCyEOQ95aFnwdaQLY6Htu9VzKwrvyLKuTABdRII8Dgm87p +lTv6zhsfhP2Cl9IknR7kdCpzuQImw6KC0BIe2sEMtPUbOyHgFqsaWHUTtZtSPGcR5 Mk6zg8dm6kqdg== Received: from perceval.ideasonboard.com (perceval.ideasonboard.com [213.167.242.64]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id C4F60601F8 for ; Thu, 10 Mar 2022 21:51:44 +0100 (CET) Authentication-Results: lancelot.ideasonboard.com; dkim=pass (1024-bit key; unprotected) header.d=ideasonboard.com header.i=@ideasonboard.com header.b="dLMRG4hb"; dkim-atps=neutral Received: from perceval.ideasonboard.com (unknown [103.251.226.203]) by perceval.ideasonboard.com (Postfix) with ESMTPSA id 807CB49C; Thu, 10 Mar 2022 21:51:43 +0100 (CET) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=ideasonboard.com; s=mail; t=1646945504; bh=IVaFY/hnHs02d0E2RY5oc/+SqGASU0V6X5B/MkD0QQE=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=dLMRG4hbmI5BYGCmSlzpjjb/5aIedZGZhiprg44+BQAYgAW28sgzRFexwzhe1lhGM 7So8CR31ZVsoyCEuknotqOrcONpOUGhyYR4P5GEIhGmCT+42hsoc79rPSeH4rEU1rn RlLHZdaALPguDf/Lo9AnuSB6Ukl2/I6F/tGXXfCE= To: libcamera-devel@lists.libcamera.org Date: Fri, 11 Mar 2022 02:21:30 +0530 Message-Id: <20220310205130.336361-4-umang.jain@ideasonboard.com> X-Mailer: git-send-email 2.31.1 In-Reply-To: <20220310205130.336361-1-umang.jain@ideasonboard.com> References: <20220310205130.336361-1-umang.jain@ideasonboard.com> MIME-Version: 1.0 Subject: [libcamera-devel] [PATCH 3/3] ipa: ipu3: Add a IPAFrameContext queue X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-Patchwork-Original-From: Umang Jain via libcamera-devel From: Umang Jain Reply-To: Umang Jain Errors-To: libcamera-devel-bounces@lists.libcamera.org Sender: "libcamera-devel" Having a single IPAFrameContext queue is limiting especially when we need to preserve per-frame controls coming in through libcamera::Request. Right now, we are not processing any controls on the IPA side (processControls()) but sooner or later we need to preserve the controls setting for the frames in the context in a retrievable fashion. Hence a std::deque is introduced to preserve the frame context of the incoming request's settings as soon as it is queued. Since IPAIPU3::processControls() is executed on IPU3CameraData::queuePendingRequests() code path, we need to store the incoming control setting in a separate IPAFrameContext and push that into the queue. The IPAFrameContext is then dropped when processing for that frame has been finished. Signed-off-by: Umang Jain --- src/ipa/ipu3/algorithms/agc.cpp | 16 ++++----- src/ipa/ipu3/algorithms/agc.h | 4 +-- src/ipa/ipu3/algorithms/awb.cpp | 17 +++++----- src/ipa/ipu3/algorithms/tone_mapping.cpp | 14 ++++---- src/ipa/ipu3/ipa_context.cpp | 42 ++++++++++++++++++++++++ src/ipa/ipu3/ipa_context.h | 12 +++++++ src/ipa/ipu3/ipu3.cpp | 41 +++++++++++++++++++---- 7 files changed, 115 insertions(+), 31 deletions(-) diff --git a/src/ipa/ipu3/algorithms/agc.cpp b/src/ipa/ipu3/algorithms/agc.cpp index cdc1327a..b24744f2 100644 --- a/src/ipa/ipu3/algorithms/agc.cpp +++ b/src/ipa/ipu3/algorithms/agc.cpp @@ -87,7 +87,7 @@ int Agc::configure(IPAContext &context, [[maybe_unused]] const IPAConfigInfo &configInfo) { IPASessionConfiguration &configuration = context.configuration; - IPAFrameContext &frameContext = context.frameContext; + IPAFrameContext &frameContext = context.getFrameContext(0); stride_ = configuration.grid.stride; @@ -182,14 +182,14 @@ utils::Duration Agc::filterExposure(utils::Duration exposureValue) * \param[in] yGain The gain calculated based on the relative luminance target * \param[in] iqMeanGain The gain calculated based on the relative luminance target */ -void Agc::computeExposure(IPAContext &context, double yGain, - double iqMeanGain) +void Agc::computeExposure(const uint32_t frame, IPAContext &context, + double yGain, double iqMeanGain) { const IPASessionConfiguration &configuration = context.configuration; - IPAFrameContext &frameContext = context.frameContext; /* Get the effective exposure and gain applied on the sensor. */ - uint32_t exposure = frameContext.sensor.exposure; - double analogueGain = frameContext.sensor.gain; + uint32_t exposure = context.prevFrameContext.sensor.exposure; + double analogueGain = context.prevFrameContext.sensor.gain; + IPAFrameContext &frameContext = context.getFrameContext(frame); /* Use the highest of the two gain estimates. */ double evGain = std::max(yGain, iqMeanGain); @@ -344,7 +344,7 @@ void Agc::process(const uint32_t frame, IPAContext &context, const ipu3_uapi_sta double yTarget = kRelativeLuminanceTarget; for (unsigned int i = 0; i < 8; i++) { - double yValue = estimateLuminance(context.frameContext, + double yValue = estimateLuminance(context.prevFrameContext, context.configuration.grid.bdsGrid, stats, yGain); double extraGain = std::min(10.0, yTarget / (yValue + .001)); @@ -357,7 +357,7 @@ void Agc::process(const uint32_t frame, IPAContext &context, const ipu3_uapi_sta break; } - computeExposure(context, yGain, iqMeanGain); + computeExposure(frame, context, yGain, iqMeanGain); frameCount_++; } diff --git a/src/ipa/ipu3/algorithms/agc.h b/src/ipa/ipu3/algorithms/agc.h index e48506e5..f35de05f 100644 --- a/src/ipa/ipu3/algorithms/agc.h +++ b/src/ipa/ipu3/algorithms/agc.h @@ -34,8 +34,8 @@ private: double measureBrightness(const ipu3_uapi_stats_3a *stats, const ipu3_uapi_grid_config &grid) const; utils::Duration filterExposure(utils::Duration currentExposure); - void computeExposure(IPAContext &context, double yGain, - double iqMeanGain); + void computeExposure(const uint32_t frame, IPAContext &context, + double yGain, double iqMeanGain); double estimateLuminance(IPAFrameContext &frameContext, const ipu3_uapi_grid_config &grid, const ipu3_uapi_stats_3a *stats, diff --git a/src/ipa/ipu3/algorithms/awb.cpp b/src/ipa/ipu3/algorithms/awb.cpp index f1e753e6..00ac2984 100644 --- a/src/ipa/ipu3/algorithms/awb.cpp +++ b/src/ipa/ipu3/algorithms/awb.cpp @@ -390,16 +390,17 @@ void Awb::calculateWBGains(const ipu3_uapi_stats_3a *stats) void Awb::process(const uint32_t frame, IPAContext &context, const ipu3_uapi_stats_3a *stats) { calculateWBGains(stats); + IPAFrameContext &frameContext = context.getFrameContext(frame); /* * Gains are only recalculated if enough zones were detected. * The results are cached, so if no results were calculated, we set the * cached values from asyncResults_ here. */ - context.frameContext.awb.gains.blue = asyncResults_.blueGain; - context.frameContext.awb.gains.green = asyncResults_.greenGain; - context.frameContext.awb.gains.red = asyncResults_.redGain; - context.frameContext.awb.temperatureK = asyncResults_.temperatureK; + frameContext.awb.gains.blue = asyncResults_.blueGain; + frameContext.awb.gains.green = asyncResults_.greenGain; + frameContext.awb.gains.red = asyncResults_.redGain; + frameContext.awb.temperatureK = asyncResults_.temperatureK; } constexpr uint16_t Awb::threshold(float value) @@ -450,10 +451,10 @@ void Awb::prepare([[maybe_unused]] const uint32_t frame, IPAContext &context, ip params->acc_param.bnr.opt_center_sqr.y_sqr_reset = params->acc_param.bnr.opt_center.y_reset * params->acc_param.bnr.opt_center.y_reset; /* Convert to u3.13 fixed point values */ - params->acc_param.bnr.wb_gains.gr = 8192 * context.frameContext.awb.gains.green; - params->acc_param.bnr.wb_gains.r = 8192 * context.frameContext.awb.gains.red; - params->acc_param.bnr.wb_gains.b = 8192 * context.frameContext.awb.gains.blue; - params->acc_param.bnr.wb_gains.gb = 8192 * context.frameContext.awb.gains.green; + params->acc_param.bnr.wb_gains.gr = 8192 * context.prevFrameContext.awb.gains.green; + params->acc_param.bnr.wb_gains.r = 8192 * context.prevFrameContext.awb.gains.red; + params->acc_param.bnr.wb_gains.b = 8192 * context.prevFrameContext.awb.gains.blue; + params->acc_param.bnr.wb_gains.gb = 8192 * context.prevFrameContext.awb.gains.green; LOG(IPU3Awb, Debug) << "Color temperature estimated: " << asyncResults_.temperatureK; diff --git a/src/ipa/ipu3/algorithms/tone_mapping.cpp b/src/ipa/ipu3/algorithms/tone_mapping.cpp index bba5bc9a..f0fbaaf7 100644 --- a/src/ipa/ipu3/algorithms/tone_mapping.cpp +++ b/src/ipa/ipu3/algorithms/tone_mapping.cpp @@ -42,7 +42,7 @@ int ToneMapping::configure(IPAContext &context, [[maybe_unused]] const IPAConfigInfo &configInfo) { /* Initialise tone mapping gamma value. */ - context.frameContext.toneMapping.gamma = 0.0; + context.frameContextQueue.front().toneMapping.gamma = 0.0; return 0; } @@ -62,7 +62,7 @@ void ToneMapping::prepare([[maybe_unused]] const uint32_t frame, { /* Copy the calculated LUT into the parameters buffer. */ memcpy(params->acc_param.gamma.gc_lut.lut, - context.frameContext.toneMapping.gammaCorrection.lut, + context.prevFrameContext.toneMapping.gammaCorrection.lut, IPU3_UAPI_GAMMA_CORR_LUT_ENTRIES * sizeof(params->acc_param.gamma.gc_lut.lut[0])); @@ -73,7 +73,7 @@ void ToneMapping::prepare([[maybe_unused]] const uint32_t frame, /** * \brief Calculate the tone mapping look up table - * \param frame The frame number + te* \param frame The frame number * \param context The shared IPA context * \param stats The IPU3 statistics and ISP results * @@ -83,6 +83,8 @@ void ToneMapping::prepare([[maybe_unused]] const uint32_t frame, void ToneMapping::process(const uint32_t frame, IPAContext &context, [[maybe_unused]] const ipu3_uapi_stats_3a *stats) { + IPAFrameContext &frameContext = context.getFrameContext(frame); + /* * Hardcode gamma to 1.1 as a default for now. * @@ -90,11 +92,11 @@ void ToneMapping::process(const uint32_t frame, IPAContext &context, */ gamma_ = 1.1; - if (context.frameContext.toneMapping.gamma == gamma_) + if (frameContext.toneMapping.gamma == gamma_) return; struct ipu3_uapi_gamma_corr_lut &lut = - context.frameContext.toneMapping.gammaCorrection; + frameContext.toneMapping.gammaCorrection; for (uint32_t i = 0; i < std::size(lut.lut); i++) { double j = static_cast(i) / (std::size(lut.lut) - 1); @@ -104,7 +106,7 @@ void ToneMapping::process(const uint32_t frame, IPAContext &context, lut.lut[i] = gamma * 8191; } - context.frameContext.toneMapping.gamma = gamma_; + frameContext.toneMapping.gamma = gamma_; } } /* namespace ipa::ipu3::algorithms */ diff --git a/src/ipa/ipu3/ipa_context.cpp b/src/ipa/ipu3/ipa_context.cpp index 9c4ec936..41cbf717 100644 --- a/src/ipa/ipu3/ipa_context.cpp +++ b/src/ipa/ipu3/ipa_context.cpp @@ -24,6 +24,48 @@ namespace libcamera::ipa::ipu3 { * may also be updated in the start() operation. */ +/** + * \brief Retrieve the context of a particular frame + * \param[in] frame Frame number + * + * Retrieve the frame context of the \a frame. + * + * \return The frame context of the given frame number or nullptr, if not found + */ +IPAFrameContext &IPAContext::getFrameContext(const uint32_t frame) +{ + auto iter = frameContextQueue.begin(); + while (iter != frameContextQueue.end()) { + if (iter->frame == frame) + return *iter; + + iter++; + } + + /* + * \todo Handle the case where frame-context is not found here. + * Should we be FATAL ? + */ + return *iter; /* returns frameContextQueue.end() */ +} + +/** + * \brief Construct a IPAFrameContext instance + */ +IPAFrameContext::IPAFrameContext() = default; + +/** + * \brief Move constructor for IPAFrameContext + * \param[in] other The other IPAFrameContext + */ +IPAFrameContext::IPAFrameContext(IPAFrameContext &&other) = default; + +/** + * \brief Move assignment operator for IPAFrameContext + * \param[in] other The other IPAFrameContext + */ +IPAFrameContext &IPAFrameContext::operator=(IPAFrameContext &&other) = default; + /** * \struct IPAFrameContext * \brief Per-frame context for algorithms diff --git a/src/ipa/ipu3/ipa_context.h b/src/ipa/ipu3/ipa_context.h index d993359a..eca346d6 100644 --- a/src/ipa/ipu3/ipa_context.h +++ b/src/ipa/ipu3/ipa_context.h @@ -8,6 +8,8 @@ #pragma once +#include + #include #include @@ -39,6 +41,12 @@ struct IPASessionConfiguration { }; struct IPAFrameContext { + uint32_t frame; + + IPAFrameContext(); + IPAFrameContext(IPAFrameContext &&other); + IPAFrameContext &operator=(IPAFrameContext &&other); + struct { uint32_t exposure; double gain; @@ -66,8 +74,12 @@ struct IPAFrameContext { }; struct IPAContext { + IPAFrameContext &getFrameContext(const uint32_t frame); IPASessionConfiguration configuration; IPAFrameContext frameContext; + + std::deque frameContextQueue; + IPAFrameContext prevFrameContext; }; } /* namespace ipa::ipu3 */ diff --git a/src/ipa/ipu3/ipu3.cpp b/src/ipa/ipu3/ipu3.cpp index 3d5c5706..f3128b0a 100644 --- a/src/ipa/ipu3/ipu3.cpp +++ b/src/ipa/ipu3/ipu3.cpp @@ -349,6 +349,8 @@ int IPAIPU3::start() */ void IPAIPU3::stop() { + while (!context_.frameContextQueue.empty()) + context_.frameContextQueue.pop_front(); } /** @@ -451,6 +453,14 @@ int IPAIPU3::configure(const IPAConfigInfo &configInfo, */ ctrls_ = configInfo.sensorControls; + /* + * Insert a initial context into the queue to faciliate + * algo->configure() below. + */ + IPAFrameContext initContext; + initContext.frame = 0; + context_.frameContextQueue.push_back(std::move(initContext)); + calculateBdsGrid(configInfo.bdsOutputSize); /* Clean frameContext at each reconfiguration. */ @@ -501,10 +511,25 @@ void IPAIPU3::unmapBuffers(const std::vector &ids) void IPAIPU3::frameStarted([[maybe_unused]] const uint32_t frame) { + IPAFrameContext newContext; + newContext.frame = frame; + + context_.frameContextQueue.push_back(std::move(newContext)); } void IPAIPU3::frameCompleted([[maybe_unused]] const uint32_t frame) { + while (!context_.frameContextQueue.empty()) { + auto &fc = context_.frameContextQueue.front(); + if (fc.frame < frame) + context_.frameContextQueue.pop_front(); + + /* Keep newer frames */ + if (fc.frame >= frame) { + context_.prevFrameContext = std::move(fc); + break; + } + } } /** @@ -547,8 +572,9 @@ void IPAIPU3::processStatsBuffer(const uint32_t frame, const int64_t frameTimest const ipu3_uapi_stats_3a *stats = reinterpret_cast(mem.data()); - context_.frameContext.sensor.exposure = sensorControls.get(V4L2_CID_EXPOSURE).get(); - context_.frameContext.sensor.gain = camHelper_->gain(sensorControls.get(V4L2_CID_ANALOGUE_GAIN).get()); + IPAFrameContext &curFrameContext = context_.getFrameContext(frame); + curFrameContext.sensor.exposure = sensorControls.get(V4L2_CID_EXPOSURE).get(); + curFrameContext.sensor.gain = camHelper_->gain(sensorControls.get(V4L2_CID_ANALOGUE_GAIN).get()); parseStatistics(frame, frameTimestamp, stats); } @@ -623,11 +649,11 @@ void IPAIPU3::parseStatistics(unsigned int frame, int64_t frameDuration = (vBlank + sensorInfo_.outputSize.height) * lineDuration; ctrls.set(controls::FrameDuration, frameDuration); - ctrls.set(controls::AnalogueGain, context_.frameContext.sensor.gain); + ctrls.set(controls::AnalogueGain, context_.prevFrameContext.sensor.gain); - ctrls.set(controls::ColourTemperature, context_.frameContext.awb.temperatureK); + ctrls.set(controls::ColourTemperature, context_.prevFrameContext.awb.temperatureK); - ctrls.set(controls::ExposureTime, context_.frameContext.sensor.exposure * lineDuration); + ctrls.set(controls::ExposureTime, context_.prevFrameContext.sensor.exposure * lineDuration); /* * \todo The Metadata provides a path to getting extended data @@ -651,8 +677,9 @@ void IPAIPU3::parseStatistics(unsigned int frame, */ void IPAIPU3::setControls(unsigned int frame) { - int32_t exposure = context_.frameContext.agc.exposure; - int32_t gain = camHelper_->gainCode(context_.frameContext.agc.gain); + IPAFrameContext &context = context_.getFrameContext(frame); + int32_t exposure = context.agc.exposure; + int32_t gain = camHelper_->gainCode(context.agc.gain); ControlList ctrls(ctrls_); ctrls.set(V4L2_CID_EXPOSURE, exposure);