diff --git a/src/ipa/ipu3/ipa_context.cpp b/src/ipa/ipu3/ipa_context.cpp index 13cdb835ef7f..dd139cb4c868 100644 --- a/src/ipa/ipu3/ipa_context.cpp +++ b/src/ipa/ipu3/ipa_context.cpp @@ -180,27 +180,10 @@ namespace libcamera::ipa::ipu3 { * struct ipu3_uapi_gamma_corr_lut for further details. */ -/** - * \brief Default constructor for IPAFrameContext - */ -IPAFrameContext::IPAFrameContext() = default; - -/** - * \brief Construct a IPAFrameContext instance - */ -IPAFrameContext::IPAFrameContext(uint32_t id, const ControlList &reqControls) - : frame(id), frameControls(reqControls) -{ - sensor = {}; -} - /** * \var IPAFrameContext::frame * \brief The frame number * - * \var IPAFrameContext::frameControls - * \brief Controls sent in by the application while queuing the request - * * \var IPAFrameContext::sensor * \brief Effective sensor values that were applied for the frame * diff --git a/src/ipa/ipu3/ipa_context.h b/src/ipa/ipu3/ipa_context.h index 42e11141d3a1..193fc44a821a 100644 --- a/src/ipa/ipu3/ipa_context.h +++ b/src/ipa/ipu3/ipa_context.h @@ -8,8 +8,6 @@ #pragma once -#include - #include #include @@ -17,13 +15,12 @@ #include #include +#include + namespace libcamera { namespace ipa::ipu3 { -/* Maximum number of frame contexts to be held */ -static constexpr uint32_t kMaxFrameContexts = 16; - struct IPASessionConfiguration { struct { ipu3_uapi_grid_config bdsGrid; @@ -77,23 +74,19 @@ struct IPAActiveState { }; struct IPAFrameContext { - IPAFrameContext(); - IPAFrameContext(uint32_t id, const ControlList &reqControls); + uint32_t frame; struct { uint32_t exposure; double gain; } sensor; - - uint32_t frame; - ControlList frameControls; }; struct IPAContext { IPASessionConfiguration configuration; IPAActiveState activeState; - std::array frameContexts; + FCQueue frameContexts; }; } /* namespace ipa::ipu3 */ diff --git a/src/ipa/ipu3/ipu3.cpp b/src/ipa/ipu3/ipu3.cpp index e37b2fa0f30e..5396b0704950 100644 --- a/src/ipa/ipu3/ipu3.cpp +++ b/src/ipa/ipu3/ipu3.cpp @@ -40,6 +40,8 @@ #include "algorithms/tone_mapping.h" #include "libipa/camera_sensor_helper.h" +#include "ipa_context.h" + /* Minimum grid width, expressed as a number of cells */ static constexpr uint32_t kMinGridWidth = 16; /* Maximum grid width, expressed as a number of cells */ @@ -205,6 +207,10 @@ void IPAIPU3::updateSessionConfiguration(const ControlInfoMap &sensorControls) int32_t minGain = v4l2Gain.min().get(); int32_t maxGain = v4l2Gain.max().get(); + /* Clear the IPA context before the streaming session. */ + context_.frameContexts.clear(); + context_ = {}; + /* * When the AGC computes the new exposure values for a frame, it needs * to know the limits for shutter speed and analogue gain. @@ -382,6 +388,7 @@ int IPAIPU3::start() */ void IPAIPU3::stop() { + context_.frameContexts.clear(); } /** @@ -488,11 +495,6 @@ int IPAIPU3::configure(const IPAConfigInfo &configInfo, calculateBdsGrid(configInfo.bdsOutputSize); - /* Clean IPAActiveState at each reconfiguration. */ - context_.activeState = {}; - IPAFrameContext initFrameContext; - context_.frameContexts.fill(initFrameContext); - if (!validateSensorControls()) { LOG(IPAIPU3, Error) << "Sensor control validation failed."; return -EINVAL; @@ -603,7 +605,7 @@ void IPAIPU3::processStatsBuffer(const uint32_t frame, const ipu3_uapi_stats_3a *stats = reinterpret_cast(mem.data()); - IPAFrameContext &frameContext = context_.frameContexts[frame % kMaxFrameContexts]; + IPAFrameContext &frameContext = context_.frameContexts.get(frame); if (frameContext.frame != frame) LOG(IPAIPU3, Warning) << "Frame " << frame << " does not match its frame context"; @@ -652,7 +654,11 @@ void IPAIPU3::processStatsBuffer(const uint32_t frame, void IPAIPU3::queueRequest(const uint32_t frame, const ControlList &controls) { /* \todo Start processing for 'frame' based on 'controls'. */ - context_.frameContexts[frame % kMaxFrameContexts] = { frame, controls }; + IPAFrameContext &frameContext = context_.frameContexts.initialise(frame); + + /* \todo Implement queueRequest to each algorithm. */ + (void)frameContext; + (void)controls; } /** diff --git a/src/ipa/libipa/fc_queue.cpp b/src/ipa/libipa/fc_queue.cpp new file mode 100644 index 000000000000..a69579290043 --- /dev/null +++ b/src/ipa/libipa/fc_queue.cpp @@ -0,0 +1,111 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2022, Google Inc. + * + * fc_queue.cpp - IPA Frame context queue + */ + +#include "fc_queue.h" + +#include + +namespace libcamera { + +LOG_DEFINE_CATEGORY(FCQueue) + +namespace ipa { + +/** + * \file fc_queue.h + * \brief Queue to access per-frame Context + */ + +/** + * \class FCQueue + * \brief A support class for queueing FrameContext instances through the IPA + * \tparam FrameContext The IPA specific FrameContext derived class type + * + * The frame context queue provides a simple circular buffer implementation to + * store IPA specific context for each frame through its lifetime within the + * IPA. + * + * FrameContext instances are expected to be referenced by a monotonically + * increasing sequence count corresponding to a frame sequence number. + * + * A FrameContext is initialised for a given frame when the corresponding + * Request is first queued into the IPA. From that point on the FrameContext can + * be obtained by the IPA and its algorithms by referencing it from the frame + * sequence number. + * + * A frame sequence number from the image source must correspond to the request + * sequence number for this implementation to be supported in an IPA. It is + * expected that the same sequence number will be used to reference the context + * of the frame from the point of queueing the request, specifying controls for + * a given frame, and processing of any ISP related controls and statistics for + * the same corresponding image. + * + * IPA specific frame context implementations shall inherit from the + * IPAFrameContext base class to support the minimum required features for a + * FrameContext. + * + * FrameContexts are overwritten when they are recycled and re-initialised by + * the first access made on them by either initialise(frame) or get(frame). It + * is required that the number of available slots in the frame context queue is + * larger or equal to the maximum number of in-flight requests a pipeline can + * handle to avoid overwriting frame context instances that still have to be + * processed. + * + * In the case an application does not queue requests to the camera fast + * enough, frames can be produced and processed by the IPA without a + * corresponding Request being queued. In this case the IPA algorithm + * will try to access the FrameContext with a call to get() before it + * had been initialized at queueRequest() time. In this case, there + * is no way the controls associated with the late Request could be + * applied in time, as the frame as already been processed by the IPA. + * + * \todo Set an error flag for per-frame control errors. + */ + +/** + * \fn FCQueue::clear() + * \brief Clear the context queue + * + * Reset the queue and ensure all FrameContext slots are re-initialised. + * IPA modules are expected to clear the frame context queue at the beginning of + * a new streaming session, in IPAModule::start(). + */ + +/** + * \fn FCQueue::initialise(uint32_t frame) + * \brief Initialize and return the Frame Context at slot specified by \a frame + * \param[in] frame The frame context sequence number + * + * The first call to obtain a FrameContext from the FCQueue should be handled + * through this call. The FrameContext will be initialised for the frame and + * returned to the caller if it was not already initialised. + * + * If the FrameContext was already initialized for this sequence, a warning will + * be reported and the previously initialized FrameContext is returned. + * + * Frame contexts are expected to be initialised when a Request is first passed + * to the IPA module in IPAModule::queueRequest(). + * + * \return A reference to the FrameContext for sequence \a frame + */ + +/** + * \fn FCQueue::get() + * \brief Obtain the Frame Context at slot specified by \a frame + * \param[in] frame The frame context sequence number + * + * Obtains an existing FrameContext from the queue and returns it to the caller. + * + * If the FrameContext is not correctly initialised for the \a frame, it will be + * initialised. + * + * \return A reference to the FrameContext for sequence \a frame + */ + +} /* namespace ipa */ + +} /* namespace libcamera */ diff --git a/src/ipa/libipa/fc_queue.h b/src/ipa/libipa/fc_queue.h new file mode 100644 index 000000000000..5966817ce673 --- /dev/null +++ b/src/ipa/libipa/fc_queue.h @@ -0,0 +1,107 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2022, Google Inc. + * + * fc_queue.h - IPA Frame context queue + */ + +#pragma once + +#include + +#include + +namespace libcamera { + +LOG_DECLARE_CATEGORY(FCQueue) + +namespace ipa { + +/* + * Maximum number of frame contexts to be held onto + * + * \todo Should be platform-specific and match the pipeline depth + */ +static constexpr uint32_t kMaxFrameContexts = 16; + +template +class FCQueue : private std::array +{ +private: + void initialise(FrameContext &frameContext, const uint32_t frame) + { + frameContext = {}; + frameContext.frame = frame; + } + +public: + void clear() + { + this->fill({}); + } + + FrameContext &initialise(const uint32_t frame) + { + FrameContext &frameContext = this->at(frame % kMaxFrameContexts); + + /* + * Do not re-initialise if a get() call has already fetched this + * frame context to preseve the context. + * + * \todo If the the sequence number of the context to initialise + * is smaller than the sequence number of the queue slot to use, + * it means that we had a serious request underrun and more + * frames than the queue size has been produced since the last + * time the application has queued a request. Does this deserve + * an error condition ? + */ + if (frame != 0 && frame <= frameContext.frame) + LOG(FCQueue, Warning) + << "Frame " << frame << " already initialised"; + else + initialise(frameContext, frame); + + return frameContext; + } + + FrameContext &get(uint32_t frame) + { + FrameContext &frameContext = this->at(frame % kMaxFrameContexts); + + /* + * If the IPA algorithms try to access a frame context slot which + * has been already overwritten by a newer context, it means the + * frame context queue has overflowed and the desired context + * has been forever lost. The pipeline handler shall avoid + * queueing more requests to the IPA than the frame context + * queue size. + */ + if (frame < frameContext.frame) + LOG(FCQueue, Fatal) << "Frame Context for " << frame + << " has been overwritten by " + << frameContext.frame; + + if (frame == frameContext.frame) + return frameContext; + + /* + * The frame context has been retrieved before it was + * initialised through the initialise() call. This indicates an + * algorithm attempted to access a Frame context before it was + * queued to the IPA. Controls applied for this request may be + * left unhandled. + * + * \todo Set an error flag for per-frame control errors. + */ + LOG(FCQueue, Warning) + << "Obtained an uninitialised FrameContext for " << frame; + + initialise(frameContext, frame); + + return frameContext; + } +}; + +} /* namespace ipa */ + +} /* namespace libcamera */ diff --git a/src/ipa/libipa/meson.build b/src/ipa/libipa/meson.build index fb894bc614af..016b8e0ec9be 100644 --- a/src/ipa/libipa/meson.build +++ b/src/ipa/libipa/meson.build @@ -3,6 +3,7 @@ libipa_headers = files([ 'algorithm.h', 'camera_sensor_helper.h', + 'fc_queue.h', 'histogram.h', 'module.h', ]) @@ -10,6 +11,7 @@ libipa_headers = files([ libipa_sources = files([ 'algorithm.cpp', 'camera_sensor_helper.cpp', + 'fc_queue.cpp', 'histogram.cpp', 'module.cpp', ])