From patchwork Mon Jan 3 17:09:53 2022 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Umang Jain X-Patchwork-Id: 15235 X-Patchwork-Delegate: umang.jain@ideasonboard.com Return-Path: X-Original-To: parsemail@patchwork.libcamera.org Delivered-To: parsemail@patchwork.libcamera.org Received: from lancelot.ideasonboard.com (lancelot.ideasonboard.com [92.243.16.209]) by patchwork.libcamera.org (Postfix) with ESMTPS id C3157C3258 for ; Mon, 3 Jan 2022 17:10:11 +0000 (UTC) Received: from lancelot.ideasonboard.com (localhost [IPv6:::1]) by lancelot.ideasonboard.com (Postfix) with ESMTP id 3C95F60913; Mon, 3 Jan 2022 18:10:11 +0100 (CET) Authentication-Results: lancelot.ideasonboard.com; dkim=fail reason="signature verification failed" (1024-bit key; unprotected) header.d=ideasonboard.com header.i=@ideasonboard.com header.b="THRqp/Yx"; dkim-atps=neutral Received: from perceval.ideasonboard.com (perceval.ideasonboard.com [213.167.242.64]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id 702F2604F4 for ; Mon, 3 Jan 2022 18:10:09 +0100 (CET) Received: from perceval.ideasonboard.com (unknown [IPv6:2401:4900:1f3e:193e:9a73:f356:8c6a:a1aa]) by perceval.ideasonboard.com (Postfix) with ESMTPSA id 0C725CC; Mon, 3 Jan 2022 18:10:07 +0100 (CET) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=ideasonboard.com; s=mail; t=1641229809; bh=o768ONRMiGpPQx6mMtiV4b5RR9Md3sAfMrkAH+/FKUA=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=THRqp/Yxo4WY3/YdpgHGsfSqluj+sUkeHj8WT/1iGFQfXhctR+y14hgt2y3eQaKyA dqxcw/fLMs8+uuZTH0/XWTycDa/gPshM/Q7Stcd9bxUuD2FuLtzchQvaiXYanStW4e 3rTJvBzsaGM8xK9Usqh7+93pY2ZHdNhjfLdNKT0w= From: Umang Jain To: libcamera-devel@lists.libcamera.org Date: Mon, 3 Jan 2022 22:39:53 +0530 Message-Id: <20220103170956.323025-2-umang.jain@ideasonboard.com> X-Mailer: git-send-email 2.31.1 In-Reply-To: <20220103170956.323025-1-umang.jain@ideasonboard.com> References: <20220103170956.323025-1-umang.jain@ideasonboard.com> MIME-Version: 1.0 Subject: [libcamera-devel] [PATCH 1/4] libipa: algorithm: Pass frame number to prepare() and process() X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: libcamera-devel-bounces@lists.libcamera.org Sender: "libcamera-devel" The frame number will be used to retrieve their respective IPAFrameContext from a container in subsequent commits. Signed-off-by: Umang Jain --- src/ipa/ipu3/algorithms/agc.cpp | 3 ++- src/ipa/ipu3/algorithms/agc.h | 2 +- src/ipa/ipu3/algorithms/awb.cpp | 4 ++-- src/ipa/ipu3/algorithms/awb.h | 4 ++-- src/ipa/ipu3/algorithms/blc.cpp | 6 ++++-- src/ipa/ipu3/algorithms/blc.h | 2 +- src/ipa/ipu3/algorithms/tone_mapping.cpp | 7 +++++-- src/ipa/ipu3/algorithms/tone_mapping.h | 4 ++-- src/ipa/ipu3/ipu3.cpp | 4 ++-- src/ipa/libipa/algorithm.cpp | 2 ++ src/ipa/libipa/algorithm.h | 6 ++++-- 11 files changed, 27 insertions(+), 17 deletions(-) diff --git a/src/ipa/ipu3/algorithms/agc.cpp b/src/ipa/ipu3/algorithms/agc.cpp index 8d6f18f6..1d0778d8 100644 --- a/src/ipa/ipu3/algorithms/agc.cpp +++ b/src/ipa/ipu3/algorithms/agc.cpp @@ -307,13 +307,14 @@ double Agc::estimateLuminance(IPAFrameContext &frameContext, /** * \brief Process IPU3 statistics, and run AGC operations + * \param frame The frame number * \param[in] context The shared IPA context * \param[in] stats The IPU3 statistics and ISP results * * Identify the current image brightness, and use that to estimate the optimal * new exposure and gain for the scene. */ -void Agc::process(IPAContext &context, const ipu3_uapi_stats_3a *stats) +void Agc::process(const uint32_t frame, IPAContext &context, const ipu3_uapi_stats_3a *stats) { /* * Estimate the gain needed to have the proportion of pixels in a given diff --git a/src/ipa/ipu3/algorithms/agc.h b/src/ipa/ipu3/algorithms/agc.h index 96ec7005..c6ab8e91 100644 --- a/src/ipa/ipu3/algorithms/agc.h +++ b/src/ipa/ipu3/algorithms/agc.h @@ -28,7 +28,7 @@ public: ~Agc() = default; int configure(IPAContext &context, const IPAConfigInfo &configInfo) override; - void process(IPAContext &context, const ipu3_uapi_stats_3a *stats) override; + void process(const uint32_t frame, IPAContext &context, const ipu3_uapi_stats_3a *stats) override; private: double measureBrightness(const ipu3_uapi_stats_3a *stats, diff --git a/src/ipa/ipu3/algorithms/awb.cpp b/src/ipa/ipu3/algorithms/awb.cpp index 1dc27fc9..99fb5305 100644 --- a/src/ipa/ipu3/algorithms/awb.cpp +++ b/src/ipa/ipu3/algorithms/awb.cpp @@ -379,7 +379,7 @@ void Awb::calculateWBGains(const ipu3_uapi_stats_3a *stats) /** * \copydoc libcamera::ipa::Algorithm::process */ -void Awb::process(IPAContext &context, const ipu3_uapi_stats_3a *stats) +void Awb::process(const uint32_t frame, IPAContext &context, const ipu3_uapi_stats_3a *stats) { calculateWBGains(stats); @@ -403,7 +403,7 @@ constexpr uint16_t Awb::threshold(float value) /** * \copydoc libcamera::ipa::Algorithm::prepare */ -void Awb::prepare(IPAContext &context, ipu3_uapi_params *params) +void Awb::prepare([[maybe_unused]] const uint32_t frame, IPAContext &context, ipu3_uapi_params *params) { /* * Green saturation thresholds are reduced because we are using the diff --git a/src/ipa/ipu3/algorithms/awb.h b/src/ipa/ipu3/algorithms/awb.h index ab4b0a33..ef177a4a 100644 --- a/src/ipa/ipu3/algorithms/awb.h +++ b/src/ipa/ipu3/algorithms/awb.h @@ -39,8 +39,8 @@ public: ~Awb(); int configure(IPAContext &context, const IPAConfigInfo &configInfo) override; - void prepare(IPAContext &context, ipu3_uapi_params *params) override; - void process(IPAContext &context, const ipu3_uapi_stats_3a *stats) override; + void prepare(const uint32_t frame, IPAContext &context, ipu3_uapi_params *params) override; + void process(const uint32_t frame, IPAContext &context, const ipu3_uapi_stats_3a *stats) override; private: /* \todo Make these structs available to all the ISPs ? */ diff --git a/src/ipa/ipu3/algorithms/blc.cpp b/src/ipa/ipu3/algorithms/blc.cpp index 78ab7bff..e9b4bed8 100644 --- a/src/ipa/ipu3/algorithms/blc.cpp +++ b/src/ipa/ipu3/algorithms/blc.cpp @@ -38,14 +38,16 @@ BlackLevelCorrection::BlackLevelCorrection() /** * \brief Fill in the parameter structure, and enable black level correction + * \param frame The frame number * \param context The shared IPA context * \param params The IPU3 parameters * * Populate the IPU3 parameter structure with the correction values for each * channel and enable the corresponding ImgU block processing. */ -void BlackLevelCorrection::prepare([[maybe_unused]] IPAContext &context, - ipu3_uapi_params *params) +void BlackLevelCorrection::prepare([[maybe_unused]] const uint32_t frame, + [[maybe_unused]] IPAContext &context, + ipu3_uapi_params *params) { /* * The Optical Black Level correction values diff --git a/src/ipa/ipu3/algorithms/blc.h b/src/ipa/ipu3/algorithms/blc.h index d8da1748..e4c22070 100644 --- a/src/ipa/ipu3/algorithms/blc.h +++ b/src/ipa/ipu3/algorithms/blc.h @@ -18,7 +18,7 @@ class BlackLevelCorrection : public Algorithm public: BlackLevelCorrection(); - void prepare(IPAContext &context, ipu3_uapi_params *params) override; + void prepare(const uint32_t frame, IPAContext &context, ipu3_uapi_params *params) override; }; } /* namespace ipa::ipu3::algorithms */ diff --git a/src/ipa/ipu3/algorithms/tone_mapping.cpp b/src/ipa/ipu3/algorithms/tone_mapping.cpp index 2040eda5..bba5bc9a 100644 --- a/src/ipa/ipu3/algorithms/tone_mapping.cpp +++ b/src/ipa/ipu3/algorithms/tone_mapping.cpp @@ -49,13 +49,15 @@ int ToneMapping::configure(IPAContext &context, /** * \brief Fill in the parameter structure, and enable gamma control + * \param frame The frame number * \param context The shared IPA context * \param params The IPU3 parameters * * Populate the IPU3 parameter structure with our tone mapping look up table and * enable the gamma control module in the processing blocks. */ -void ToneMapping::prepare([[maybe_unused]] IPAContext &context, +void ToneMapping::prepare([[maybe_unused]] const uint32_t frame, + [[maybe_unused]] IPAContext &context, ipu3_uapi_params *params) { /* Copy the calculated LUT into the parameters buffer. */ @@ -71,13 +73,14 @@ void ToneMapping::prepare([[maybe_unused]] IPAContext &context, /** * \brief Calculate the tone mapping look up table + * \param frame The frame number * \param context The shared IPA context * \param stats The IPU3 statistics and ISP results * * The tone mapping look up table is generated as an inverse power curve from * our gamma setting. */ -void ToneMapping::process(IPAContext &context, +void ToneMapping::process(const uint32_t frame, IPAContext &context, [[maybe_unused]] const ipu3_uapi_stats_3a *stats) { /* diff --git a/src/ipa/ipu3/algorithms/tone_mapping.h b/src/ipa/ipu3/algorithms/tone_mapping.h index b727ab1e..041c9c0d 100644 --- a/src/ipa/ipu3/algorithms/tone_mapping.h +++ b/src/ipa/ipu3/algorithms/tone_mapping.h @@ -19,8 +19,8 @@ public: ToneMapping(); int configure(IPAContext &context, const IPAConfigInfo &configInfo) override; - void prepare(IPAContext &context, ipu3_uapi_params *params) override; - void process(IPAContext &context, const ipu3_uapi_stats_3a *stats) override; + void prepare(const uint32_t frame, IPAContext &context, ipu3_uapi_params *params) override; + void process(const uint32_t frame, IPAContext &context, const ipu3_uapi_stats_3a *stats) override; private: double gamma_; diff --git a/src/ipa/ipu3/ipu3.cpp b/src/ipa/ipu3/ipu3.cpp index 3d307708..aef232c3 100644 --- a/src/ipa/ipu3/ipu3.cpp +++ b/src/ipa/ipu3/ipu3.cpp @@ -595,7 +595,7 @@ void IPAIPU3::fillParams(unsigned int frame, ipu3_uapi_params *params) params->use = {}; for (auto const &algo : algorithms_) - algo->prepare(context_, params); + algo->prepare(frame, context_, params); IPU3Action op; op.op = ActionParamFilled; @@ -620,7 +620,7 @@ void IPAIPU3::parseStatistics(unsigned int frame, ControlList ctrls(controls::controls); for (auto const &algo : algorithms_) - algo->process(context_, stats); + algo->process(frame, context_, stats); setControls(frame); diff --git a/src/ipa/libipa/algorithm.cpp b/src/ipa/libipa/algorithm.cpp index 398d5372..01967f07 100644 --- a/src/ipa/libipa/algorithm.cpp +++ b/src/ipa/libipa/algorithm.cpp @@ -48,6 +48,7 @@ namespace ipa { /** * \fn Algorithm::prepare() * \brief Fill the \a params buffer with ISP processing parameters for a frame + * \param[in] frame The frame number * \param[in] context The shared IPA context * \param[out] params The ISP specific parameters. * @@ -63,6 +64,7 @@ namespace ipa { /** * \fn Algorithm::process() * \brief Process ISP statistics, and run algorithm operations + * \param[in] frame The frame number * \param[in] context The shared IPA context * \param[in] stats The IPA statistics and ISP results * diff --git a/src/ipa/libipa/algorithm.h b/src/ipa/libipa/algorithm.h index 766aee5d..1df93625 100644 --- a/src/ipa/libipa/algorithm.h +++ b/src/ipa/libipa/algorithm.h @@ -22,12 +22,14 @@ public: return 0; } - virtual void prepare([[maybe_unused]] Context &context, + virtual void prepare([[maybe_unused]] const uint32_t frame, + [[maybe_unused]] Context &context, [[maybe_unused]] Params *params) { } - virtual void process([[maybe_unused]] Context &context, + virtual void process([[maybe_unused]] const uint32_t frame, + [[maybe_unused]] Context &context, [[maybe_unused]] const Stats *stats) { } From patchwork Mon Jan 3 17:09:54 2022 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Patchwork-Submitter: Umang Jain X-Patchwork-Id: 15236 X-Patchwork-Delegate: umang.jain@ideasonboard.com Return-Path: X-Original-To: parsemail@patchwork.libcamera.org Delivered-To: parsemail@patchwork.libcamera.org Received: from lancelot.ideasonboard.com (lancelot.ideasonboard.com [92.243.16.209]) by patchwork.libcamera.org (Postfix) with ESMTPS id 200ECBE080 for ; Mon, 3 Jan 2022 17:10:14 +0000 (UTC) Received: from lancelot.ideasonboard.com (localhost [IPv6:::1]) by lancelot.ideasonboard.com (Postfix) with ESMTP id A13876090D; Mon, 3 Jan 2022 18:10:13 +0100 (CET) Authentication-Results: lancelot.ideasonboard.com; dkim=fail reason="signature verification failed" (1024-bit key; unprotected) header.d=ideasonboard.com header.i=@ideasonboard.com header.b="Uu/ED6iR"; dkim-atps=neutral Received: from perceval.ideasonboard.com (perceval.ideasonboard.com [213.167.242.64]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id 1E493604F4 for ; Mon, 3 Jan 2022 18:10:12 +0100 (CET) Received: from perceval.ideasonboard.com (unknown [IPv6:2401:4900:1f3e:193e:9a73:f356:8c6a:a1aa]) by perceval.ideasonboard.com (Postfix) with ESMTPSA id 9064CCC; Mon, 3 Jan 2022 18:10:10 +0100 (CET) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=ideasonboard.com; s=mail; t=1641229811; bh=w0/DQSKgsAUOI2FVwi3uM0hVA/KSCmZjrOLTOJfhdYM=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=Uu/ED6iR6yMbFRQj9y65SDrEdLFVAYiCKfjSqHgYWySDNRpbDdzPqPFs3qotxlaGK qYs9Uhqsd//EYSL4BAACdlre7F+OXagbpFVmdUrYa+b9/tPd2BTfQjBJJId/LDcvoh Qib+/ZOmfT6cMunS5EuTogYs8Iv3dRyeKw24rnnM= From: Umang Jain To: libcamera-devel@lists.libcamera.org Date: Mon, 3 Jan 2022 22:39:54 +0530 Message-Id: <20220103170956.323025-3-umang.jain@ideasonboard.com> X-Mailer: git-send-email 2.31.1 In-Reply-To: <20220103170956.323025-1-umang.jain@ideasonboard.com> References: <20220103170956.323025-1-umang.jain@ideasonboard.com> MIME-Version: 1.0 Subject: [libcamera-devel] [PATCH 2/4] ipa: ipu3: Replace event-based operations with dedicated functions X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: libcamera-devel-bounces@lists.libcamera.org Sender: "libcamera-devel" The IPAIPU3 interface currently uses event-type based structures in order to communicate with the pipeline-handler (and vice-versa). Replace the event based structures with dedicated functions associated to each operation. Signed-off-by: Umang Jain --- include/libcamera/ipa/ipu3.mojom | 36 ++------ src/ipa/ipu3/ipu3-ipa-design-guide.rst | 19 ++-- src/ipa/ipu3/ipu3.cpp | 117 ++++++++++------------- src/libcamera/pipeline/ipu3/ipu3.cpp | 123 +++++++++++-------------- 4 files changed, 122 insertions(+), 173 deletions(-) diff --git a/include/libcamera/ipa/ipu3.mojom b/include/libcamera/ipa/ipu3.mojom index cc0d822f..12b8bb2f 100644 --- a/include/libcamera/ipa/ipu3.mojom +++ b/include/libcamera/ipa/ipu3.mojom @@ -8,32 +8,6 @@ module ipa.ipu3; import "include/libcamera/ipa/core.mojom"; -enum IPU3Operations { - ActionSetSensorControls = 1, - ActionParamFilled = 2, - ActionMetadataReady = 3, - EventProcessControls = 4, - EventStatReady = 5, - EventFillParams = 6, -}; - -struct IPU3Event { - IPU3Operations op; - uint32 frame; - int64 frameTimestamp; - uint32 bufferId; - libcamera.ControlList controls; - libcamera.ControlList sensorControls; - libcamera.ControlList lensControls; -}; - -struct IPU3Action { - IPU3Operations op; - libcamera.ControlList controls; - libcamera.ControlList sensorControls; - libcamera.ControlList lensControls; -}; - struct IPAConfigInfo { libcamera.IPACameraSensorInfo sensorInfo; libcamera.ControlInfoMap sensorControls; @@ -55,9 +29,15 @@ interface IPAIPU3Interface { mapBuffers(array buffers); unmapBuffers(array ids); - [async] processEvent(IPU3Event ev); + [async] fillParameterBuffer(uint32 frame, uint32 bufferId); + [async] processControls(uint32 frame, libcamera.ControlList controls); + [async] statsReady(uint32 frame, int64 frameTimestamp, uint32 bufferId, + libcamera.ControlList sensorControls); }; interface IPAIPU3EventInterface { - queueFrameAction(uint32 frame, IPU3Action action); + setSensorControls(uint32 frame, libcamera.ControlList sensorControls, + libcamera.ControlList lensControls); + paramFilled(uint32 frame); + metadataReady(uint32 frame, libcamera.ControlList metadata); }; diff --git a/src/ipa/ipu3/ipu3-ipa-design-guide.rst b/src/ipa/ipu3/ipu3-ipa-design-guide.rst index 89c71108..83f8634d 100644 --- a/src/ipa/ipu3/ipu3-ipa-design-guide.rst +++ b/src/ipa/ipu3/ipu3-ipa-design-guide.rst @@ -25,7 +25,8 @@ from applications, and managing events from the pipeline handler. └─┬───┬───┬──────┬────┬────┬────┬─┴────▼─┬──┘ 1: init() │ │ │ │ ▲ │ ▲ │ ▲ │ ▲ │ 2: configure() │1 │2 │3 │4│ │4│ │4│ │4│ │5 3: mapBuffers(), start() - ▼ ▼ ▼ ▼ │ ▼ │ ▼ │ ▼ │ ▼ 4: processEvent() + │ │ │ │ │ │ │ │ │ │ │ │ 4: (▼) processControls(), fillParametersBuffer(), statsReady() + ▼ ▼ ▼ ▼ │ ▼ │ ▼ │ ▼ │ ▼ (▲) setSensorControls, paramFilled, metadataReady Signals ┌──────────────────┴────┴────┴────┴─────────┐ 5: stop(), unmapBuffers() │ IPU3 IPA │ │ ┌───────────────────────┐ │ @@ -100,8 +101,9 @@ There are three main interactions with the algorithms for the IPU3 IPA to operate when running: - configure() -- processEvent(``EventFillParams``) -- processEvent(``EventStatReady``) +- fillParameterBuffer() +- processControls() +- statsReady() The configuration phase allows the pipeline-handler to inform the IPA of the current stream configurations, which is then passed into each @@ -115,7 +117,7 @@ When configured, the IPA is notified by the pipeline handler of the Camera ``start()`` event, after which incoming requests will be queued for processing, requiring a parameter buffer (``ipu3_uapi_params``) to be populated for the ImgU. This is given to the IPA through the -``EventFillParams`` event, and then passed directly to each algorithm +``fillParameterBuffer()``, and then passed directly to each algorithm through the ``prepare()`` call allowing the ISP configuration to be updated for the needs of each component that the algorithm is responsible for. @@ -125,7 +127,7 @@ structure that it modifies, and it should take care to ensure that any structure set by a use flag is fully initialised to suitable values. The parameter buffer is returned to the pipeline handler through the -``ActionParamFilled`` event, and from there queued to the ImgU along +``paramFilled`` signal, and from there queued to the ImgU along with a raw frame captured with the CIO2. Post-frame completion @@ -134,7 +136,7 @@ Post-frame completion When the capture of an image is completed, and successfully processed through the ImgU, the generated statistics buffer (``ipu3_uapi_stats_3a``) is given to the IPA through the -``EventStatReady`` event. This provides the IPA with an opportunity to +``statsReady()``. This provides the IPA with an opportunity to examine the results of the ISP and run the calculations required by each algorithm on the new data. The algorithms may require context from the operations of other algorithms, for example, the AWB might choose to use @@ -145,11 +147,14 @@ before they are needed. The ordering of the algorithm processing is determined by their placement in the ``IPU3::algorithms_`` ordered list. +Finally, the IPA metadata for the frame completed is returned back via +``metadataReady`` Signal. + Sensor Controls ~~~~~~~~~~~~~~~ The AutoExposure and AutoGain (AGC) algorithm differs slightly from the others as it requires operating directly on the sensor, as opposed to through the ImgU ISP. To support this, there is a dedicated action -`ActionSetSensorControls` to allow the IPA to request controls to be set +``setSensorControls`` signal to allow the IPA to request controls to be set on the camera sensor through the pipeline handler. diff --git a/src/ipa/ipu3/ipu3.cpp b/src/ipa/ipu3/ipu3.cpp index aef232c3..4f03d7e9 100644 --- a/src/ipa/ipu3/ipu3.cpp +++ b/src/ipa/ipu3/ipu3.cpp @@ -142,14 +142,18 @@ public: void mapBuffers(const std::vector &buffers) override; void unmapBuffers(const std::vector &ids) override; - void processEvent(const IPU3Event &event) override; + + void fillParameterBuffer(const uint32_t frame, const uint32_t bufferId) override; + void processControls(const uint32_t frame, const ControlList &controls) override; + void statsReady(const uint32_t frame, const int64_t frameTimestamp, + const uint32_t bufferId, const ControlList &sensorControls) override; private: void updateControls(const IPACameraSensorInfo &sensorInfo, const ControlInfoMap &sensorControls, ControlInfoMap *ipaControls); void updateSessionConfiguration(const ControlInfoMap &sensorControls); - void processControls(unsigned int frame, const ControlList &controls); + void fillParams(unsigned int frame, ipu3_uapi_params *params); void parseStatistics(unsigned int frame, int64_t frameTimestamp, @@ -505,58 +509,49 @@ void IPAIPU3::unmapBuffers(const std::vector &ids) } /** - * \brief Process an event generated by the pipeline handler - * \param[in] event The event sent from pipeline handler - * - * The expected event handling over the lifetime of a Request has - * the following sequence: - * - * - EventProcessControls : Handle controls from a new Request - * - EventFillParams : Prepare the ISP to process the Request - * - EventStatReady : Process statistics after ISP completion + * \brief Prepare the ISP to process the Request + * \param[in] frame The frame number + * \param[in] bufferId Buffer ID */ -void IPAIPU3::processEvent(const IPU3Event &event) +void IPAIPU3::fillParameterBuffer(const uint32_t frame, const uint32_t bufferId) { - switch (event.op) { - case EventProcessControls: { - processControls(event.frame, event.controls); - break; - } - case EventFillParams: { - auto it = buffers_.find(event.bufferId); - if (it == buffers_.end()) { - LOG(IPAIPU3, Error) << "Could not find param buffer!"; - return; - } - - Span mem = it->second.planes()[0]; - ipu3_uapi_params *params = - reinterpret_cast(mem.data()); - - fillParams(event.frame, params); - break; + auto it = buffers_.find(bufferId); + if (it == buffers_.end()) { + LOG(IPAIPU3, Error) << "Could not find param buffer!"; + return; } - case EventStatReady: { - auto it = buffers_.find(event.bufferId); - if (it == buffers_.end()) { - LOG(IPAIPU3, Error) << "Could not find stats buffer!"; - return; - } - - Span mem = it->second.planes()[0]; - const ipu3_uapi_stats_3a *stats = - reinterpret_cast(mem.data()); - - context_.frameContext.sensor.exposure = event.sensorControls.get(V4L2_CID_EXPOSURE).get(); - context_.frameContext.sensor.gain = camHelper_->gain(event.sensorControls.get(V4L2_CID_ANALOGUE_GAIN).get()); - - parseStatistics(event.frame, event.frameTimestamp, stats); - break; - } - default: - LOG(IPAIPU3, Error) << "Unknown event " << event.op; - break; + + Span mem = it->second.planes()[0]; + ipu3_uapi_params *params = + reinterpret_cast(mem.data()); + + fillParams(frame, params); +} + +/** + * \brief Process statistics after ISP completion + * \param[in] frame The frame number + * \param[in] frameTimestamp Timestamp of the frame + * \param[in] bufferId Buffer ID + * \param[in] sensorControls Sensor controls + */ +void IPAIPU3::statsReady(const uint32_t frame, const int64_t frameTimestamp, + const uint32_t bufferId, const ControlList &sensorControls) +{ + auto it = buffers_.find(bufferId); + if (it == buffers_.end()) { + LOG(IPAIPU3, Error) << "Could not find stats buffer!"; + return; } + + Span mem = it->second.planes()[0]; + const ipu3_uapi_stats_3a *stats = + reinterpret_cast(mem.data()); + + context_.frameContext.sensor.exposure = sensorControls.get(V4L2_CID_EXPOSURE).get(); + context_.frameContext.sensor.gain = camHelper_->gain(sensorControls.get(V4L2_CID_ANALOGUE_GAIN).get()); + + parseStatistics(frame, frameTimestamp, stats); } /** @@ -567,7 +562,7 @@ void IPAIPU3::processEvent(const IPU3Event &event) * Parse the request to handle any IPA-managed controls that were set from the * application such as manual sensor settings. */ -void IPAIPU3::processControls([[maybe_unused]] unsigned int frame, +void IPAIPU3::processControls(const uint32_t frame, [[maybe_unused]] const ControlList &controls) { /* \todo Start processing for 'frame' based on 'controls'. */ @@ -597,10 +592,7 @@ void IPAIPU3::fillParams(unsigned int frame, ipu3_uapi_params *params) for (auto const &algo : algorithms_) algo->prepare(frame, context_, params); - IPU3Action op; - op.op = ActionParamFilled; - - queueFrameAction.emit(frame, op); + paramFilled.emit(frame); } /** @@ -642,11 +634,7 @@ void IPAIPU3::parseStatistics(unsigned int frame, * likely want to avoid putting platform specific metadata in. */ - IPU3Action op; - op.op = ActionMetadataReady; - op.controls = ctrls; - - queueFrameAction.emit(frame, op); + metadataReady.emit(frame, ctrls); } /** @@ -658,18 +646,13 @@ void IPAIPU3::parseStatistics(unsigned int frame, */ void IPAIPU3::setControls(unsigned int frame) { - IPU3Action op; - op.op = ActionSetSensorControls; - exposure_ = context_.frameContext.agc.exposure; gain_ = camHelper_->gainCode(context_.frameContext.agc.gain); ControlList ctrls(ctrls_); - ctrls.set(V4L2_CID_EXPOSURE, static_cast(exposure_)); - ctrls.set(V4L2_CID_ANALOGUE_GAIN, static_cast(gain_)); - op.sensorControls = ctrls; + ControlList lensCtrls; - queueFrameAction.emit(frame, op); + setSensorControls.emit(frame, ctrls, lensCtrls); } } /* namespace ipa::ipu3 */ diff --git a/src/libcamera/pipeline/ipu3/ipu3.cpp b/src/libcamera/pipeline/ipu3/ipu3.cpp index 6c5617cd..dd1780ef 100644 --- a/src/libcamera/pipeline/ipu3/ipu3.cpp +++ b/src/libcamera/pipeline/ipu3/ipu3.cpp @@ -86,8 +86,10 @@ public: ControlInfoMap ipaControls_; private: - void queueFrameAction(unsigned int id, - const ipa::ipu3::IPU3Action &action); + void metadataReady(unsigned int id, const ControlList &metadata); + void paramFilled(unsigned int id); + void setSensorControls(unsigned int id, const ControlList &sensorControls, + const ControlList &lensControls); }; class IPU3CameraConfiguration : public CameraConfiguration @@ -866,11 +868,7 @@ void IPU3CameraData::queuePendingRequests() info->rawBuffer = rawBuffer; - ipa::ipu3::IPU3Event ev; - ev.op = ipa::ipu3::EventProcessControls; - ev.frame = info->id; - ev.controls = request->controls(); - ipa_->processEvent(ev); + ipa_->processControls(info->id, request->controls()); pendingRequests_.pop(); processingRequests_.push(request); @@ -1213,7 +1211,9 @@ int IPU3CameraData::loadIPA() if (!ipa_) return -ENOENT; - ipa_->queueFrameAction.connect(this, &IPU3CameraData::queueFrameAction); + ipa_->setSensorControls.connect(this, &IPU3CameraData::setSensorControls); + ipa_->paramFilled.connect(this, &IPU3CameraData::paramFilled); + ipa_->metadataReady.connect(this, &IPU3CameraData::metadataReady); /* * Pass the sensor info to the IPA to initialize controls. @@ -1248,69 +1248,59 @@ int IPU3CameraData::loadIPA() return 0; } -void IPU3CameraData::queueFrameAction(unsigned int id, - const ipa::ipu3::IPU3Action &action) +void IPU3CameraData::setSensorControls([[maybe_unused]] unsigned int id, + const ControlList &sensorControls, + const ControlList &lensControls) { - switch (action.op) { - case ipa::ipu3::ActionSetSensorControls: { - const ControlList &sensorControls = action.sensorControls; - delayedCtrls_->push(sensorControls); + delayedCtrls_->push(sensorControls); - CameraLens *focusLens = cio2_.sensor()->focusLens(); - if (!focusLens) - break; - - const ControlList lensControls = action.lensControls; - if (!lensControls.contains(V4L2_CID_FOCUS_ABSOLUTE)) - break; + CameraLens *focusLens = cio2_.sensor()->focusLens(); + if (!focusLens) + return; - const ControlValue &focusValue = - lensControls.get(V4L2_CID_FOCUS_ABSOLUTE); + if (!lensControls.contains(V4L2_CID_FOCUS_ABSOLUTE)) + return; - focusLens->setFocusPostion(focusValue.get()); + const ControlValue &focusValue = + lensControls.get(V4L2_CID_FOCUS_ABSOLUTE); - break; - } - case ipa::ipu3::ActionParamFilled: { - IPU3Frames::Info *info = frameInfos_.find(id); - if (!info) - break; - - /* Queue all buffers from the request aimed for the ImgU. */ - for (auto it : info->request->buffers()) { - const Stream *stream = it.first; - FrameBuffer *outbuffer = it.second; + focusLens->setFocusPostion(focusValue.get()); +} - if (stream == &outStream_) - imgu_->output_->queueBuffer(outbuffer); - else if (stream == &vfStream_) - imgu_->viewfinder_->queueBuffer(outbuffer); - } +void IPU3CameraData::paramFilled(unsigned int id) +{ + IPU3Frames::Info *info = frameInfos_.find(id); + if (!info) + return; - imgu_->param_->queueBuffer(info->paramBuffer); - imgu_->stat_->queueBuffer(info->statBuffer); - imgu_->input_->queueBuffer(info->rawBuffer); + /* Queue all buffers from the request aimed for the ImgU. */ + for (auto it : info->request->buffers()) { + const Stream *stream = it.first; + FrameBuffer *outbuffer = it.second; - break; + if (stream == &outStream_) + imgu_->output_->queueBuffer(outbuffer); + else if (stream == &vfStream_) + imgu_->viewfinder_->queueBuffer(outbuffer); } - case ipa::ipu3::ActionMetadataReady: { - IPU3Frames::Info *info = frameInfos_.find(id); - if (!info) - break; - Request *request = info->request; - request->metadata().merge(action.controls); + imgu_->param_->queueBuffer(info->paramBuffer); + imgu_->stat_->queueBuffer(info->statBuffer); + imgu_->input_->queueBuffer(info->rawBuffer); +} - info->metadataProcessed = true; - if (frameInfos_.tryComplete(info)) - pipe()->completeRequest(request); +void IPU3CameraData::metadataReady(unsigned int id, const ControlList &metadata) +{ + IPU3Frames::Info *info = frameInfos_.find(id); + if (!info) + return; - break; - } - default: - LOG(IPU3, Error) << "Unknown action " << action.op; - break; - } + Request *request = info->request; + request->metadata().merge(metadata); + + info->metadataProcessed = true; + if (frameInfos_.tryComplete(info)) + pipe()->completeRequest(request); } /* ----------------------------------------------------------------------------- @@ -1385,11 +1375,7 @@ void IPU3CameraData::cio2BufferReady(FrameBuffer *buffer) if (request->findBuffer(&rawStream_)) pipe()->completeBuffer(request, buffer); - ipa::ipu3::IPU3Event ev; - ev.op = ipa::ipu3::EventFillParams; - ev.frame = info->id; - ev.bufferId = info->paramBuffer->cookie(); - ipa_->processEvent(ev); + ipa_->fillParameterBuffer(info->id, info->paramBuffer->cookie()); } void IPU3CameraData::paramBufferReady(FrameBuffer *buffer) @@ -1433,13 +1419,8 @@ void IPU3CameraData::statBufferReady(FrameBuffer *buffer) return; } - ipa::ipu3::IPU3Event ev; - ev.op = ipa::ipu3::EventStatReady; - ev.frame = info->id; - ev.bufferId = info->statBuffer->cookie(); - ev.frameTimestamp = request->metadata().get(controls::SensorTimestamp); - ev.sensorControls = info->effectiveSensorControls; - ipa_->processEvent(ev); + ipa_->statsReady(info->id, request->metadata().get(controls::SensorTimestamp), + info->statBuffer->cookie(), info->effectiveSensorControls); } /* From patchwork Mon Jan 3 17:09:55 2022 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Umang Jain X-Patchwork-Id: 15237 X-Patchwork-Delegate: umang.jain@ideasonboard.com Return-Path: X-Original-To: parsemail@patchwork.libcamera.org Delivered-To: parsemail@patchwork.libcamera.org Received: from lancelot.ideasonboard.com (lancelot.ideasonboard.com [92.243.16.209]) by patchwork.libcamera.org (Postfix) with ESMTPS id 03A6FC3258 for ; Mon, 3 Jan 2022 17:10:17 +0000 (UTC) Received: from lancelot.ideasonboard.com (localhost [IPv6:::1]) by lancelot.ideasonboard.com (Postfix) with ESMTP id 6ABD3608E9; Mon, 3 Jan 2022 18:10:17 +0100 (CET) Authentication-Results: lancelot.ideasonboard.com; dkim=fail reason="signature verification failed" (1024-bit key; unprotected) header.d=ideasonboard.com header.i=@ideasonboard.com header.b="Bwt+vSLD"; dkim-atps=neutral Received: from perceval.ideasonboard.com (perceval.ideasonboard.com [213.167.242.64]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id CBCBC604F4 for ; Mon, 3 Jan 2022 18:10:15 +0100 (CET) Received: from perceval.ideasonboard.com (unknown [IPv6:2401:4900:1f3e:193e:9a73:f356:8c6a:a1aa]) by perceval.ideasonboard.com (Postfix) with ESMTPSA id D8E4DE57; Mon, 3 Jan 2022 18:10:13 +0100 (CET) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=ideasonboard.com; s=mail; t=1641229815; bh=7Um9WZFfBnDVnoqeIk6skONFCu4zOkLwAMXZZWy0xVU=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=Bwt+vSLD07i54b/arA0Snw122dAVYH+GqBuw+jhrcCLUV6yMHp+oyU280DgRE4e79 3mSPFEOimhLX+wt5WqW6ATqRDXQEcB5fuDOsN/niBu3+7skj0HsdeUiVVT924+JmNV mrj3Kzt7ye3vlF+w1yKh2Uah97yeOSJCSzdYDmk4= From: Umang Jain To: libcamera-devel@lists.libcamera.org Date: Mon, 3 Jan 2022 22:39:55 +0530 Message-Id: <20220103170956.323025-4-umang.jain@ideasonboard.com> X-Mailer: git-send-email 2.31.1 In-Reply-To: <20220103170956.323025-1-umang.jain@ideasonboard.com> References: <20220103170956.323025-1-umang.jain@ideasonboard.com> MIME-Version: 1.0 Subject: [libcamera-devel] [PATCH 3/4] ipa: ipu3: Mark the beginning and end of a frame X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: libcamera-devel-bounces@lists.libcamera.org Sender: "libcamera-devel" From: Jean-Michel Hautbois Introduce the skeleton for two functions which will be used to instantiate a frame context, and do everything needed when a frame is received. Do the same for the other end, once the algorithms have run and updated the frame context to later deallocate the corresponding frame context. Signed-off-by: Jean-Michel Hautbois Reviewed-by: Kieran Bingham Signed-off-by: Umang Jain --- src/ipa/ipu3/ipu3.cpp | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/src/ipa/ipu3/ipu3.cpp b/src/ipa/ipu3/ipu3.cpp index 4f03d7e9..fa40c41f 100644 --- a/src/ipa/ipu3/ipu3.cpp +++ b/src/ipa/ipu3/ipu3.cpp @@ -162,6 +162,14 @@ private: void setControls(unsigned int frame); void calculateBdsGrid(const Size &bdsOutputSize); + /* + * Internal events that mark the beginning of processing a new frame + * to the point that it has successfully completed processing its + * statistics. + */ + void frameStarted(const uint32_t frame); + void frameCompleted(const uint32_t frame); + std::map buffers_; ControlInfoMap ctrls_; @@ -508,6 +516,14 @@ void IPAIPU3::unmapBuffers(const std::vector &ids) } } +void IPAIPU3::frameStarted([[maybe_unused]] const uint32_t frame) +{ +} + +void IPAIPU3::frameCompleted([[maybe_unused]] const uint32_t frame) +{ +} + /** * \brief Prepare the ISP to process the Request * \param[in] frame The frame number @@ -566,6 +582,15 @@ void IPAIPU3::processControls(const uint32_t frame, [[maybe_unused]] const ControlList &controls) { /* \todo Start processing for 'frame' based on 'controls'. */ + + /* + * To save incurring extra IPC calls, we do not send explicit events + * when a new request is started or completed. + * ProcessControls is the first event handled upon receipt of a new + * request, so we can handle all actions required to start processing + * a new frame. + */ + frameStarted(frame); } /** @@ -635,6 +660,14 @@ void IPAIPU3::parseStatistics(unsigned int frame, */ metadataReady.emit(frame, ctrls); + + /* + * To save incurring extra IPC calls, we do not send explicit events + * when we have completed all handling of a request. + * Once the statistics are fully processed, we will no longer handle this + * frame. + */ + frameCompleted(frame); } /** From patchwork Mon Jan 3 17:09:56 2022 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Umang Jain X-Patchwork-Id: 15238 X-Patchwork-Delegate: umang.jain@ideasonboard.com Return-Path: X-Original-To: parsemail@patchwork.libcamera.org Delivered-To: parsemail@patchwork.libcamera.org Received: from lancelot.ideasonboard.com (lancelot.ideasonboard.com [92.243.16.209]) by patchwork.libcamera.org (Postfix) with ESMTPS id 90EC2C3259 for ; Mon, 3 Jan 2022 17:10:19 +0000 (UTC) Received: from lancelot.ideasonboard.com (localhost [IPv6:::1]) by lancelot.ideasonboard.com (Postfix) with ESMTP id 08BED60915; Mon, 3 Jan 2022 18:10:19 +0100 (CET) Authentication-Results: lancelot.ideasonboard.com; dkim=fail reason="signature verification failed" (1024-bit key; unprotected) header.d=ideasonboard.com header.i=@ideasonboard.com header.b="SZZA1CFt"; dkim-atps=neutral Received: from perceval.ideasonboard.com (perceval.ideasonboard.com [213.167.242.64]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id 71452604F4 for ; Mon, 3 Jan 2022 18:10:18 +0100 (CET) Received: from perceval.ideasonboard.com (unknown [IPv6:2401:4900:1f3e:193e:9a73:f356:8c6a:a1aa]) by perceval.ideasonboard.com (Postfix) with ESMTPSA id 28A84CC; Mon, 3 Jan 2022 18:10:16 +0100 (CET) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=ideasonboard.com; s=mail; t=1641229818; bh=GbL0mvguMBDWLn/Hc3KT+sigc0IjfR0x/sQ+mE8HHxc=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=SZZA1CFtacT35zMURgi1KC/KxhFpEP9glGVnX5Z56S/snAEAC2Viy1cxiDNke1eTs Jxt3uR0GyuUsnz+O52SoTJC1uUjWsz1tMzAaNlr/oSd+BurUZi3SzCKbIITyjZoBna njcsaWAG2NypfnH4RosH+tr6lxxK70yU8y4OkCi8= From: Umang Jain To: libcamera-devel@lists.libcamera.org Date: Mon, 3 Jan 2022 22:39:56 +0530 Message-Id: <20220103170956.323025-5-umang.jain@ideasonboard.com> X-Mailer: git-send-email 2.31.1 In-Reply-To: <20220103170956.323025-1-umang.jain@ideasonboard.com> References: <20220103170956.323025-1-umang.jain@ideasonboard.com> MIME-Version: 1.0 Subject: [libcamera-devel] [PATCH 4/4] ipa: ipu3: Add a IPAFrameContext queue X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: libcamera-devel-bounces@lists.libcamera.org Sender: "libcamera-devel" Having a single IPAFrameContext queue is limiting especially when we need to preserve per-frame controls. Right now, we are not processing any controls on the IPA side (processControls()) but sooner or later we need to preserve the controls setting for the frames in the context in a retrievable fashion. Hence a std::deque is introduced to preserve the frame context of the incoming request's settings as soon as it is queued. Since IPAIPU3::processControls() is executed on IPU3CameraData::queuePendingRequests() code path, we need to store the incoming control setting in a separate IPAFrameContext and push that into the queue. The IPAFrameContext is then dropped when processing for that frame has been finished. Signed-off-by: Umang Jain --- src/ipa/ipu3/algorithms/agc.cpp | 18 ++++---- src/ipa/ipu3/algorithms/agc.h | 2 +- src/ipa/ipu3/algorithms/awb.cpp | 18 ++++---- src/ipa/ipu3/algorithms/tone_mapping.cpp | 11 ++--- src/ipa/ipu3/ipa_context.cpp | 57 +++++++++++++++++++++--- src/ipa/ipu3/ipa_context.h | 13 +++++- src/ipa/ipu3/ipu3.cpp | 41 ++++++++++++++--- 7 files changed, 124 insertions(+), 36 deletions(-) diff --git a/src/ipa/ipu3/algorithms/agc.cpp b/src/ipa/ipu3/algorithms/agc.cpp index 1d0778d8..f8e1fef7 100644 --- a/src/ipa/ipu3/algorithms/agc.cpp +++ b/src/ipa/ipu3/algorithms/agc.cpp @@ -99,8 +99,9 @@ int Agc::configure(IPAContext &context, const IPAConfigInfo &configInfo) maxAnalogueGain_ = std::min(context.configuration.agc.maxAnalogueGain, kMaxAnalogueGain); /* Configure the default exposure and gain. */ - context.frameContext.agc.gain = minAnalogueGain_; - context.frameContext.agc.exposure = minShutterSpeed_ / lineDuration_; + IPAFrameContext &frameContext = context.frameContextQueue.front(); + frameContext.agc.gain = minAnalogueGain_; + frameContext.agc.exposure = minShutterSpeed_ / lineDuration_; return 0; } @@ -174,16 +175,17 @@ void Agc::filterExposure() /** * \brief Estimate the new exposure and gain values + * \param[in] frame The frame number * \param[inout] frameContext The shared IPA frame Context * \param[in] yGain The gain calculated based on the relative luminance target * \param[in] iqMeanGain The gain calculated based on the relative luminance target */ -void Agc::computeExposure(IPAFrameContext &frameContext, double yGain, - double iqMeanGain) +void Agc::computeExposure(const uint32_t frame, IPAContext &context, double yGain, double iqMeanGain) { /* Get the effective exposure and gain applied on the sensor. */ - uint32_t exposure = frameContext.sensor.exposure; - double analogueGain = frameContext.sensor.gain; + uint32_t exposure = context.prevFrameContext.sensor.exposure; + double analogueGain = context.prevFrameContext.sensor.gain; + IPAFrameContext &frameContext = context.getFrameContext(frame); /* Use the highest of the two gain estimates. */ double evGain = std::max(yGain, iqMeanGain); @@ -336,7 +338,7 @@ void Agc::process(const uint32_t frame, IPAContext &context, const ipu3_uapi_sta double yTarget = kRelativeLuminanceTarget; for (unsigned int i = 0; i < 8; i++) { - double yValue = estimateLuminance(context.frameContext, + double yValue = estimateLuminance(context.prevFrameContext, context.configuration.grid.bdsGrid, stats, yGain); double extraGain = std::min(10.0, yTarget / (yValue + .001)); @@ -349,7 +351,7 @@ void Agc::process(const uint32_t frame, IPAContext &context, const ipu3_uapi_sta break; } - computeExposure(context.frameContext, yGain, iqMeanGain); + computeExposure(frame, context, yGain, iqMeanGain); frameCount_++; } diff --git a/src/ipa/ipu3/algorithms/agc.h b/src/ipa/ipu3/algorithms/agc.h index c6ab8e91..a3c52fc7 100644 --- a/src/ipa/ipu3/algorithms/agc.h +++ b/src/ipa/ipu3/algorithms/agc.h @@ -34,7 +34,7 @@ private: double measureBrightness(const ipu3_uapi_stats_3a *stats, const ipu3_uapi_grid_config &grid) const; void filterExposure(); - void computeExposure(IPAFrameContext &frameContext, double yGain, + void computeExposure(const uint32_t frame, IPAContext &context, double yGain, double iqMeanGain); double estimateLuminance(IPAFrameContext &frameContext, const ipu3_uapi_grid_config &grid, diff --git a/src/ipa/ipu3/algorithms/awb.cpp b/src/ipa/ipu3/algorithms/awb.cpp index 99fb5305..a8347d0f 100644 --- a/src/ipa/ipu3/algorithms/awb.cpp +++ b/src/ipa/ipu3/algorithms/awb.cpp @@ -382,16 +382,17 @@ void Awb::calculateWBGains(const ipu3_uapi_stats_3a *stats) void Awb::process(const uint32_t frame, IPAContext &context, const ipu3_uapi_stats_3a *stats) { calculateWBGains(stats); + IPAFrameContext &frameContext = context.getFrameContext(frame); /* * Gains are only recalculated if enough zones were detected. * The results are cached, so if no results were calculated, we set the * cached values from asyncResults_ here. */ - context.frameContext.awb.gains.blue = asyncResults_.blueGain; - context.frameContext.awb.gains.green = asyncResults_.greenGain; - context.frameContext.awb.gains.red = asyncResults_.redGain; - context.frameContext.awb.temperatureK = asyncResults_.temperatureK; + frameContext.awb.gains.blue = asyncResults_.blueGain; + frameContext.awb.gains.green = asyncResults_.greenGain; + frameContext.awb.gains.red = asyncResults_.redGain; + frameContext.awb.temperatureK = asyncResults_.temperatureK; } constexpr uint16_t Awb::threshold(float value) @@ -434,6 +435,7 @@ void Awb::prepare([[maybe_unused]] const uint32_t frame, IPAContext &context, ip */ params->acc_param.bnr = imguCssBnrDefaults; Size &bdsOutputSize = context.configuration.grid.bdsOutputSize; + IPAFrameContext &frameContext = context.frameContextQueue.front(); params->acc_param.bnr.column_size = bdsOutputSize.width; params->acc_param.bnr.opt_center.x_reset = grid.x_start - (bdsOutputSize.width / 2); params->acc_param.bnr.opt_center.y_reset = grid.y_start - (bdsOutputSize.height / 2); @@ -442,10 +444,10 @@ void Awb::prepare([[maybe_unused]] const uint32_t frame, IPAContext &context, ip params->acc_param.bnr.opt_center_sqr.y_sqr_reset = params->acc_param.bnr.opt_center.y_reset * params->acc_param.bnr.opt_center.y_reset; /* Convert to u3.13 fixed point values */ - params->acc_param.bnr.wb_gains.gr = 8192 * context.frameContext.awb.gains.green; - params->acc_param.bnr.wb_gains.r = 8192 * context.frameContext.awb.gains.red; - params->acc_param.bnr.wb_gains.b = 8192 * context.frameContext.awb.gains.blue; - params->acc_param.bnr.wb_gains.gb = 8192 * context.frameContext.awb.gains.green; + params->acc_param.bnr.wb_gains.gr = 8192 * frameContext.awb.gains.green; + params->acc_param.bnr.wb_gains.r = 8192 * frameContext.awb.gains.red; + params->acc_param.bnr.wb_gains.b = 8192 * frameContext.awb.gains.blue; + params->acc_param.bnr.wb_gains.gb = 8192 * frameContext.awb.gains.green; LOG(IPU3Awb, Debug) << "Color temperature estimated: " << asyncResults_.temperatureK; diff --git a/src/ipa/ipu3/algorithms/tone_mapping.cpp b/src/ipa/ipu3/algorithms/tone_mapping.cpp index bba5bc9a..ce6c330d 100644 --- a/src/ipa/ipu3/algorithms/tone_mapping.cpp +++ b/src/ipa/ipu3/algorithms/tone_mapping.cpp @@ -42,7 +42,7 @@ int ToneMapping::configure(IPAContext &context, [[maybe_unused]] const IPAConfigInfo &configInfo) { /* Initialise tone mapping gamma value. */ - context.frameContext.toneMapping.gamma = 0.0; + context.frameContextQueue.front().toneMapping.gamma = 0.0; return 0; } @@ -62,7 +62,7 @@ void ToneMapping::prepare([[maybe_unused]] const uint32_t frame, { /* Copy the calculated LUT into the parameters buffer. */ memcpy(params->acc_param.gamma.gc_lut.lut, - context.frameContext.toneMapping.gammaCorrection.lut, + context.frameContextQueue.front().toneMapping.gammaCorrection.lut, IPU3_UAPI_GAMMA_CORR_LUT_ENTRIES * sizeof(params->acc_param.gamma.gc_lut.lut[0])); @@ -83,6 +83,7 @@ void ToneMapping::prepare([[maybe_unused]] const uint32_t frame, void ToneMapping::process(const uint32_t frame, IPAContext &context, [[maybe_unused]] const ipu3_uapi_stats_3a *stats) { + IPAFrameContext &frameContext = context.getFrameContext(frame); /* * Hardcode gamma to 1.1 as a default for now. * @@ -90,11 +91,11 @@ void ToneMapping::process(const uint32_t frame, IPAContext &context, */ gamma_ = 1.1; - if (context.frameContext.toneMapping.gamma == gamma_) + if (frameContext.toneMapping.gamma == gamma_) return; struct ipu3_uapi_gamma_corr_lut &lut = - context.frameContext.toneMapping.gammaCorrection; + frameContext.toneMapping.gammaCorrection; for (uint32_t i = 0; i < std::size(lut.lut); i++) { double j = static_cast(i) / (std::size(lut.lut) - 1); @@ -104,7 +105,7 @@ void ToneMapping::process(const uint32_t frame, IPAContext &context, lut.lut[i] = gamma * 8191; } - context.frameContext.toneMapping.gamma = gamma_; + frameContext.toneMapping.gamma = gamma_; } } /* namespace ipa::ipu3::algorithms */ diff --git a/src/ipa/ipu3/ipa_context.cpp b/src/ipa/ipu3/ipa_context.cpp index 86794ac1..95a08547 100644 --- a/src/ipa/ipu3/ipa_context.cpp +++ b/src/ipa/ipu3/ipa_context.cpp @@ -39,6 +39,48 @@ namespace libcamera::ipa::ipu3 { * algorithm, but should only be written by its owner. */ +/** + * \brief Retrieve the context of a particular frame + * \param[in] frame Frame number + * + * Retrieve the frame context of the \a frame. + * + * \return The frame context of the given frame number or nullptr, if not found + */ +IPAFrameContext &IPAContext::getFrameContext(const uint32_t frame) +{ + auto iter = frameContextQueue.begin(); + while (iter != frameContextQueue.end()) { + if (iter->frame == frame) + return *iter; + + iter++; + } + + /* + * \todo Handle the case where frame-context is not found here. + * Should we be FATAL ? + */ + return *iter; /* returns frameContextQueue.end() */ +} + +/** + * \brief Construct a IPAFrameContext instance + */ +IPAFrameContext::IPAFrameContext() = default; + +/** + * \brief Move constructor for IPAFrameContext + * \param[in] other The other IPAFrameContext + */ +IPAFrameContext::IPAFrameContext(IPAFrameContext &&other) = default; + +/** + * \brief Move assignment operator for IPAFrameContext + * \param[in] other The other IPAFrameContext + */ +IPAFrameContext &IPAFrameContext::operator=(IPAFrameContext &&other) = default; + /** * \struct IPAContext * \brief Global IPA context data shared between all algorithms @@ -46,13 +88,11 @@ namespace libcamera::ipa::ipu3 { * \var IPAContext::configuration * \brief The IPA session configuration, immutable during the session * - * \var IPAContext::frameContext - * \brief The frame context for the frame being processed + * \var IPAContext::frameContextQueue + * \brief A queue of frame contexts to be processed by the IPA * - * \todo While the frame context is supposed to be per-frame, this - * single frame context stores data related to both the current frame - * and the previous frames, with fields being updated as the algorithms - * are run. This needs to be turned into real per-frame data storage. + * \var IPAContext::prevFrameContext + * \brief The latest frame context which the IPA has finished processing */ /** @@ -86,6 +126,11 @@ namespace libcamera::ipa::ipu3 { * \brief Maximum analogue gain supported with the configured sensor */ +/** + * \var IPAFrameContext::frame + * \brief Frame number of the corresponding frame context + */ + /** * \var IPAFrameContext::agc * \brief Context for the Automatic Gain Control algorithm diff --git a/src/ipa/ipu3/ipa_context.h b/src/ipa/ipu3/ipa_context.h index c6dc0814..df2a9779 100644 --- a/src/ipa/ipu3/ipa_context.h +++ b/src/ipa/ipu3/ipa_context.h @@ -8,6 +8,8 @@ #pragma once +#include + #include #include @@ -34,6 +36,12 @@ struct IPASessionConfiguration { }; struct IPAFrameContext { + uint32_t frame; + + IPAFrameContext(); + IPAFrameContext(IPAFrameContext &&other); + IPAFrameContext &operator=(IPAFrameContext &&other); + struct { uint32_t exposure; double gain; @@ -61,8 +69,11 @@ struct IPAFrameContext { }; struct IPAContext { + IPAFrameContext &getFrameContext(const uint32_t frame); + IPASessionConfiguration configuration; - IPAFrameContext frameContext; + std::deque frameContextQueue; + IPAFrameContext prevFrameContext; }; } /* namespace ipa::ipu3 */ diff --git a/src/ipa/ipu3/ipu3.cpp b/src/ipa/ipu3/ipu3.cpp index fa40c41f..9c3d5ff4 100644 --- a/src/ipa/ipu3/ipu3.cpp +++ b/src/ipa/ipu3/ipu3.cpp @@ -336,6 +336,8 @@ int IPAIPU3::start() */ void IPAIPU3::stop() { + while (!context_.frameContextQueue.empty()) + context_.frameContextQueue.pop_front(); } /** @@ -469,6 +471,14 @@ int IPAIPU3::configure(const IPAConfigInfo &configInfo, /* Clean context at configuration */ context_ = {}; + /* + * Insert a initial context into the queue to faciliate + * algo->configure() below. + */ + IPAFrameContext initContext; + initContext.frame = 0; + context_.frameContextQueue.push_back(std::move(initContext)); + calculateBdsGrid(configInfo.bdsOutputSize); lineDuration_ = sensorInfo_.lineLength * 1.0s / sensorInfo_.pixelRate; @@ -518,10 +528,25 @@ void IPAIPU3::unmapBuffers(const std::vector &ids) void IPAIPU3::frameStarted([[maybe_unused]] const uint32_t frame) { + IPAFrameContext newContext; + newContext.frame = frame; + + context_.frameContextQueue.push_back(std::move(newContext)); } void IPAIPU3::frameCompleted([[maybe_unused]] const uint32_t frame) { + while (!context_.frameContextQueue.empty()) { + auto &fc = context_.frameContextQueue.front(); + if (fc.frame < frame) + context_.frameContextQueue.pop_front(); + + /* Keep newer frames */ + if (fc.frame >= frame) { + context_.prevFrameContext = std::move(fc); + break; + } + } } /** @@ -564,8 +589,9 @@ void IPAIPU3::statsReady(const uint32_t frame, const int64_t frameTimestamp, const ipu3_uapi_stats_3a *stats = reinterpret_cast(mem.data()); - context_.frameContext.sensor.exposure = sensorControls.get(V4L2_CID_EXPOSURE).get(); - context_.frameContext.sensor.gain = camHelper_->gain(sensorControls.get(V4L2_CID_ANALOGUE_GAIN).get()); + IPAFrameContext &curFrameContext = context_.frameContextQueue.front(); + curFrameContext.sensor.exposure = sensorControls.get(V4L2_CID_EXPOSURE).get(); + curFrameContext.sensor.gain = camHelper_->gain(sensorControls.get(V4L2_CID_ANALOGUE_GAIN).get()); parseStatistics(frame, frameTimestamp, stats); } @@ -645,11 +671,11 @@ void IPAIPU3::parseStatistics(unsigned int frame, int64_t frameDuration = (defVBlank_ + sensorInfo_.outputSize.height) * lineDuration_.get(); ctrls.set(controls::FrameDuration, frameDuration); - ctrls.set(controls::AnalogueGain, context_.frameContext.sensor.gain); + ctrls.set(controls::AnalogueGain, context_.prevFrameContext.sensor.gain); - ctrls.set(controls::ColourTemperature, context_.frameContext.awb.temperatureK); + ctrls.set(controls::ColourTemperature, context_.prevFrameContext.awb.temperatureK); - ctrls.set(controls::ExposureTime, context_.frameContext.sensor.exposure * lineDuration_.get()); + ctrls.set(controls::ExposureTime, context_.prevFrameContext.sensor.exposure * lineDuration_.get()); /* * \todo The Metadata provides a path to getting extended data @@ -679,8 +705,9 @@ void IPAIPU3::parseStatistics(unsigned int frame, */ void IPAIPU3::setControls(unsigned int frame) { - exposure_ = context_.frameContext.agc.exposure; - gain_ = camHelper_->gainCode(context_.frameContext.agc.gain); + IPAFrameContext &context = context_.frameContextQueue.front(); + exposure_ = context.agc.exposure; + gain_ = camHelper_->gainCode(context.agc.gain); ControlList ctrls(ctrls_); ControlList lensCtrls;