| Message ID | 20260327144726.7983-4-david.plowman@raspberrypi.com |
|---|---|
| State | Superseded |
| Headers | show |
| Series |
|
| Related | show |
Hi David, On Fri, Mar 27, 2026 at 02:42:36PM +0000, David Plowman wrote: > When a request is about to be processed, this commit separates > "delayed controls" (camera-related ones that take "a few frames" to > apply) from "immediate controls" (ISP-related controls) that will > happen instantly. > > The immediate controls are held back until the delayed controls that > they were submitted with, have happened. This means all the controls > submitted in a request happen atomically. > > We therefore already have the sequence number of the request whose > controls have just been applied, so we additionally attach this to the > current request as "ControlListSequence" metadata. > > Signed-off-by: David Plowman <david.plowman@raspberrypi.com> > Reviewed-by: Naushir Patuck <naush@raspberrypi.com> > --- > src/ipa/rpi/common/ipa_base.cpp | 24 ++++++---- > .../pipeline/rpi/common/pipeline_base.cpp | 46 +++++++++++++++++++ > .../pipeline/rpi/common/pipeline_base.h | 8 ++++ > src/libcamera/pipeline/rpi/pisp/pisp.cpp | 3 ++ > src/libcamera/pipeline/rpi/vc4/vc4.cpp | 3 ++ > 5 files changed, 76 insertions(+), 8 deletions(-) > > diff --git a/src/ipa/rpi/common/ipa_base.cpp b/src/ipa/rpi/common/ipa_base.cpp > index faa77719..dacafa57 100644 > --- a/src/ipa/rpi/common/ipa_base.cpp > +++ b/src/ipa/rpi/common/ipa_base.cpp > @@ -431,6 +431,15 @@ void IpaBase::prepareIsp(const PrepareParams ¶ms) > rpiMetadata.clear(); > fillDeviceStatus(params.sensorControls, ipaContext); > > + /* > + * When there are controls, it's important that we don't skip running the > + * IPAs, as that can mess with synchronisation. Crucially though, we need > + * to know whether there were controls when this comes back as the > + * _delayed_ metadata, hence why we flag this in the metadata itself. > + */ > + if (!params.requestControls.empty()) > + rpiMetadata.set("ipa.request_controls", true); > + > if (params.buffers.embedded) { > /* > * Pipeline handler has supplied us with an embedded data buffer, > @@ -451,7 +460,7 @@ void IpaBase::prepareIsp(const PrepareParams ¶ms) > */ > AgcStatus agcStatus; > bool hdrChange = false; > - RPiController::Metadata &delayedMetadata = rpiMetadata_[params.delayContext]; > + RPiController::Metadata &delayedMetadata = rpiMetadata_[params.delayContext % rpiMetadata_.size()]; > if (!delayedMetadata.get<AgcStatus>("agc.status", agcStatus)) { > rpiMetadata.set("agc.delayed_status", agcStatus); > hdrChange = agcStatus.hdr.mode != hdrStatus_.mode; > @@ -464,9 +473,13 @@ void IpaBase::prepareIsp(const PrepareParams ¶ms) > */ > helper_->prepare(embeddedBuffer, rpiMetadata); > > + bool delayedRequestControls = false; > + delayedMetadata.get<bool>("ipa.request_controls", delayedRequestControls); > + > /* Allow a 10% margin on the comparison below. */ > Duration delta = (frameTimestamp - lastRunTimestamp_) * 1.0ns; > - if (lastRunTimestamp_ && frameCount_ > invalidCount_ && > + if (!delayedRequestControls && params.requestControls.empty() && > + lastRunTimestamp_ && frameCount_ > invalidCount_ && > delta < controllerMinFrameDuration_ * 0.9 && !hdrChange) { > /* > * Ensure we merge the previous frame's metadata with the current > @@ -535,7 +548,7 @@ void IpaBase::processStats(const ProcessParams ¶ms) > ControlList ctrls(sensorCtrls_); > applyAGC(&agcStatus, ctrls); > rpiMetadata.set("agc.status", agcStatus); > - setDelayedControls.emit(ctrls, ipaContext); > + setDelayedControls.emit(ctrls, params.ipaContext); So this used to cycle DelayedControls over rpiMetadata_.size() while now we're using an unbound counter (which the Request counter) ? How did this used to work if the index used for DelayedControls::push() was alywas cycling within rpiMetadata_.size() ? > setCameraTimeoutValue(); > } > > @@ -951,8 +964,6 @@ void IpaBase::applyControls(const ControlList &controls) > > /* The control provides units of microseconds. */ > agc->setFixedExposureTime(0, ctrl.second.get<int32_t>() * 1.0us); > - > - libcameraMetadata_.set(controls::ExposureTime, ctrl.second.get<int32_t>()); > break; > } > > @@ -976,9 +987,6 @@ void IpaBase::applyControls(const ControlList &controls) > break; > > agc->setFixedGain(0, ctrl.second.get<float>()); > - > - libcameraMetadata_.set(controls::AnalogueGain, > - ctrl.second.get<float>()); I might have missed where these two are now reported.. > break; > } > > diff --git a/src/libcamera/pipeline/rpi/common/pipeline_base.cpp b/src/libcamera/pipeline/rpi/common/pipeline_base.cpp > index 867ecf1b..c6df0934 100644 > --- a/src/libcamera/pipeline/rpi/common/pipeline_base.cpp > +++ b/src/libcamera/pipeline/rpi/common/pipeline_base.cpp > @@ -1528,4 +1528,50 @@ void CameraData::fillRequestMetadata(const ControlList &bufferControls, Request > } > } > > +static bool isControlDelayed(unsigned int id) > +{ > + return id == controls::ExposureTime || > + id == controls::AnalogueGain || > + id == controls::FrameDurationLimits || > + id == controls::AeEnable || > + id == controls::ExposureTimeMode || > + id == controls::AnalogueGainMode; > +} > + > +void CameraData::handleControlLists(uint32_t delayContext) > +{ > + /* > + * THe delayContext is the sequence number after it's gone through the various The > + * pipeline delays, so that's what gets reported as the "ControlListSequence" > + * in the metadata, being the sequence number of the request whose ControlList > + * has just been applied. > + */ > + Request *request = requestQueue_.front(); > + request->_d()->metadata().set(controls::rpi::ControlListSequence, delayContext); > + > + /* > + * Controls that take effect immediately (typically ISP controls) have to be > + * delayed so as to synchronise with those controls that do get delayed. So we > + * must remove them from the current request, and push them onto a queue so > + * that they can be used later. > + */ > + ControlList controls = std::move(request->controls()); > + request->controls().clear(); > + immediateControls_.push({ request->sequence(), {} }); > + for (const auto &ctrl : controls) { > + if (isControlDelayed(ctrl.first)) > + request->controls().set(ctrl.first, ctrl.second); > + else > + immediateControls_.back().controls.set(ctrl.first, ctrl.second); > + } > + > + /* "Immediate" controls that have become due are now merged back into this request. */ > + while (!immediateControls_.empty() && > + immediateControls_.front().controlListId <= delayContext) { > + request->controls().merge(immediateControls_.front().controls, > + ControlList::MergePolicy::OverwriteExisting); > + immediateControls_.pop(); > + } So this now returns a Request with a different list of controls than the one the application had submitted. Particularly, the flow in my understanding is as follow (I'm using pisp and cfe here) - A CfeBuffer has completed: a new frame is available - We use the Cfe buffer sequence to know from delayed controls which sensor controls have been applied for that frame and the id of the Request those controls where submitted with. The id of that Request becomes 'delayedContext' here - This function populates a queue of ISP controls indexed by the Request id they belong to - We here collect all ISP controls whose Request id is smaller than 'delayedContext' (aka the Request id whose controls have completed) Now, this ControlList goes into the IPA then at some point will get back to the pipeline which populates a parameters buffer with the result of the IPA processing. The parameters buffer gets queued, and the ISP consumes buffer one after the other. When the parameters buffer completes, it will be re-associated with the Request's ControlList created here and signalled as complete. Did I get it right ? I will save never ending discussions about the overall per-frame control issues and design, this change if I got it right makes sure a Request is returned with a list of Controls applied at the time the Request was processed, and that list might be different than the one it was created with, right ? > +} > + > } /* namespace libcamera */ > diff --git a/src/libcamera/pipeline/rpi/common/pipeline_base.h b/src/libcamera/pipeline/rpi/common/pipeline_base.h > index 597eb587..65d8efdc 100644 > --- a/src/libcamera/pipeline/rpi/common/pipeline_base.h > +++ b/src/libcamera/pipeline/rpi/common/pipeline_base.h > @@ -180,10 +180,18 @@ public: > > ClockRecovery wallClockRecovery_; > > + struct ImmediateControlsEntry { > + uint64_t controlListId; > + ControlList controls; > + }; > + std::queue<ImmediateControlsEntry> immediateControls_; > + > protected: > void fillRequestMetadata(const ControlList &bufferControls, > Request *request); > > + void handleControlLists(uint32_t delayContext); > + > virtual void tryRunPipeline() = 0; > > private: > diff --git a/src/libcamera/pipeline/rpi/pisp/pisp.cpp b/src/libcamera/pipeline/rpi/pisp/pisp.cpp > index dff73a79..cc8aa4d4 100644 > --- a/src/libcamera/pipeline/rpi/pisp/pisp.cpp > +++ b/src/libcamera/pipeline/rpi/pisp/pisp.cpp > @@ -2322,6 +2322,9 @@ void PiSPCameraData::tryRunPipeline() > > fillRequestMetadata(job.sensorControls, request); > > + /* This sorts out synchronisation with ControlLists in earlier requests. */ > + handleControlLists(job.delayContext); > + > /* Set our state to say the pipeline is active. */ > state_ = State::Busy; > > diff --git a/src/libcamera/pipeline/rpi/vc4/vc4.cpp b/src/libcamera/pipeline/rpi/vc4/vc4.cpp > index b734889d..f743c8a7 100644 > --- a/src/libcamera/pipeline/rpi/vc4/vc4.cpp > +++ b/src/libcamera/pipeline/rpi/vc4/vc4.cpp > @@ -939,6 +939,9 @@ void Vc4CameraData::tryRunPipeline() > > fillRequestMetadata(bayerFrame.controls, request); > > + /* This sorts out synchronisation with ControlLists in earlier requests. */ > + handleControlLists(bayerFrame.delayContext); > + > /* Set our state to say the pipeline is active. */ > state_ = State::Busy; > > -- > 2.47.3 >
Hi Jacopo Thanks for taking a look at this! On Mon, 27 Apr 2026 at 08:53, Jacopo Mondi <jacopo.mondi@ideasonboard.com> wrote: > > Hi David, > > On Fri, Mar 27, 2026 at 02:42:36PM +0000, David Plowman wrote: > > When a request is about to be processed, this commit separates > > "delayed controls" (camera-related ones that take "a few frames" to > > apply) from "immediate controls" (ISP-related controls) that will > > happen instantly. > > > > The immediate controls are held back until the delayed controls that > > they were submitted with, have happened. This means all the controls > > submitted in a request happen atomically. > > > > We therefore already have the sequence number of the request whose > > controls have just been applied, so we additionally attach this to the > > current request as "ControlListSequence" metadata. > > > > Signed-off-by: David Plowman <david.plowman@raspberrypi.com> > > Reviewed-by: Naushir Patuck <naush@raspberrypi.com> > > --- > > src/ipa/rpi/common/ipa_base.cpp | 24 ++++++---- > > .../pipeline/rpi/common/pipeline_base.cpp | 46 +++++++++++++++++++ > > .../pipeline/rpi/common/pipeline_base.h | 8 ++++ > > src/libcamera/pipeline/rpi/pisp/pisp.cpp | 3 ++ > > src/libcamera/pipeline/rpi/vc4/vc4.cpp | 3 ++ > > 5 files changed, 76 insertions(+), 8 deletions(-) > > > > diff --git a/src/ipa/rpi/common/ipa_base.cpp b/src/ipa/rpi/common/ipa_base.cpp > > index faa77719..dacafa57 100644 > > --- a/src/ipa/rpi/common/ipa_base.cpp > > +++ b/src/ipa/rpi/common/ipa_base.cpp > > @@ -431,6 +431,15 @@ void IpaBase::prepareIsp(const PrepareParams ¶ms) > > rpiMetadata.clear(); > > fillDeviceStatus(params.sensorControls, ipaContext); > > > > + /* > > + * When there are controls, it's important that we don't skip running the > > + * IPAs, as that can mess with synchronisation. Crucially though, we need > > + * to know whether there were controls when this comes back as the > > + * _delayed_ metadata, hence why we flag this in the metadata itself. > > + */ > > + if (!params.requestControls.empty()) > > + rpiMetadata.set("ipa.request_controls", true); > > + > > if (params.buffers.embedded) { > > /* > > * Pipeline handler has supplied us with an embedded data buffer, > > @@ -451,7 +460,7 @@ void IpaBase::prepareIsp(const PrepareParams ¶ms) > > */ > > AgcStatus agcStatus; > > bool hdrChange = false; > > - RPiController::Metadata &delayedMetadata = rpiMetadata_[params.delayContext]; > > + RPiController::Metadata &delayedMetadata = rpiMetadata_[params.delayContext % rpiMetadata_.size()]; > > if (!delayedMetadata.get<AgcStatus>("agc.status", agcStatus)) { > > rpiMetadata.set("agc.delayed_status", agcStatus); > > hdrChange = agcStatus.hdr.mode != hdrStatus_.mode; > > @@ -464,9 +473,13 @@ void IpaBase::prepareIsp(const PrepareParams ¶ms) > > */ > > helper_->prepare(embeddedBuffer, rpiMetadata); > > > > + bool delayedRequestControls = false; > > + delayedMetadata.get<bool>("ipa.request_controls", delayedRequestControls); > > + > > /* Allow a 10% margin on the comparison below. */ > > Duration delta = (frameTimestamp - lastRunTimestamp_) * 1.0ns; > > - if (lastRunTimestamp_ && frameCount_ > invalidCount_ && > > + if (!delayedRequestControls && params.requestControls.empty() && > > + lastRunTimestamp_ && frameCount_ > invalidCount_ && > > delta < controllerMinFrameDuration_ * 0.9 && !hdrChange) { > > /* > > * Ensure we merge the previous frame's metadata with the current > > @@ -535,7 +548,7 @@ void IpaBase::processStats(const ProcessParams ¶ms) > > ControlList ctrls(sensorCtrls_); > > applyAGC(&agcStatus, ctrls); > > rpiMetadata.set("agc.status", agcStatus); > > - setDelayedControls.emit(ctrls, ipaContext); > > + setDelayedControls.emit(ctrls, params.ipaContext); > > So this used to cycle DelayedControls over rpiMetadata_.size() while > now we're using an unbound counter (which the Request counter) ? How > did this used to work if the index used for DelayedControls::push() > was alywas cycling within rpiMetadata_.size() ? params.ipaContext is actually given to DelayedControls::push as a cookie. It then comes back to us about 20 lines higher as params.delayContext. Previously, we were doing the % here, and not above. But now, we're doing % above, and not here, so it is actually the same. But the difference is that we carry the actual request sequence number around instead, as that's now more useful to us. > > > > setCameraTimeoutValue(); > > } > > > > @@ -951,8 +964,6 @@ void IpaBase::applyControls(const ControlList &controls) > > > > /* The control provides units of microseconds. */ > > agc->setFixedExposureTime(0, ctrl.second.get<int32_t>() * 1.0us); > > - > > - libcameraMetadata_.set(controls::ExposureTime, ctrl.second.get<int32_t>()); > > break; > > } > > > > @@ -976,9 +987,6 @@ void IpaBase::applyControls(const ControlList &controls) > > break; > > > > agc->setFixedGain(0, ctrl.second.get<float>()); > > - > > - libcameraMetadata_.set(controls::AnalogueGain, > > - ctrl.second.get<float>()); > > I might have missed where these two are now reported.. This is already happening in IpaBase::reportMetadata a little further down the file (line 1577). It's just a bug that it's here as well, as it will start writing these new values into the metadata when the IPA sees them, several frames before they actually happen. > > > break; > > } > > > > diff --git a/src/libcamera/pipeline/rpi/common/pipeline_base.cpp b/src/libcamera/pipeline/rpi/common/pipeline_base.cpp > > index 867ecf1b..c6df0934 100644 > > --- a/src/libcamera/pipeline/rpi/common/pipeline_base.cpp > > +++ b/src/libcamera/pipeline/rpi/common/pipeline_base.cpp > > @@ -1528,4 +1528,50 @@ void CameraData::fillRequestMetadata(const ControlList &bufferControls, Request > > } > > } > > > > +static bool isControlDelayed(unsigned int id) > > +{ > > + return id == controls::ExposureTime || > > + id == controls::AnalogueGain || > > + id == controls::FrameDurationLimits || > > + id == controls::AeEnable || > > + id == controls::ExposureTimeMode || > > + id == controls::AnalogueGainMode; > > +} > > + > > +void CameraData::handleControlLists(uint32_t delayContext) > > +{ > > + /* > > + * THe delayContext is the sequence number after it's gone through the various > > The Thanks! > > > + * pipeline delays, so that's what gets reported as the "ControlListSequence" > > + * in the metadata, being the sequence number of the request whose ControlList > > + * has just been applied. > > + */ > > + Request *request = requestQueue_.front(); > > + request->_d()->metadata().set(controls::rpi::ControlListSequence, delayContext); > > + > > + /* > > + * Controls that take effect immediately (typically ISP controls) have to be > > + * delayed so as to synchronise with those controls that do get delayed. So we > > + * must remove them from the current request, and push them onto a queue so > > + * that they can be used later. > > + */ > > + ControlList controls = std::move(request->controls()); > > + request->controls().clear(); > > + immediateControls_.push({ request->sequence(), {} }); > > + for (const auto &ctrl : controls) { > > + if (isControlDelayed(ctrl.first)) > > + request->controls().set(ctrl.first, ctrl.second); > > + else > > + immediateControls_.back().controls.set(ctrl.first, ctrl.second); > > + } > > + > > + /* "Immediate" controls that have become due are now merged back into this request. */ > > + while (!immediateControls_.empty() && > > + immediateControls_.front().controlListId <= delayContext) { > > + request->controls().merge(immediateControls_.front().controls, > > + ControlList::MergePolicy::OverwriteExisting); > > + immediateControls_.pop(); > > + } > > So this now returns a Request with a different list of controls than > the one the application had submitted. > > Particularly, the flow in my understanding is as follow (I'm using > pisp and cfe here) > > - A CfeBuffer has completed: a new frame is available > - We use the Cfe buffer sequence to know from delayed controls which > sensor controls have been applied for that frame and the id of the > Request those controls where submitted with. The id of that > Request becomes 'delayedContext' here > > - This function populates a queue of ISP controls indexed by the > Request id they belong to > > - We here collect all ISP controls whose Request id is smaller than > 'delayedContext' (aka the Request id whose controls have completed) > > Now, this ControlList goes into the IPA then at some point will get > back to the pipeline which populates a parameters buffer with the > result of the IPA processing. > > The parameters buffer gets queued, and the ISP consumes buffer one > after the other. When the parameters buffer completes, it will be > re-associated with the Request's ControlList created here and > signalled as complete. > > Did I get it right ? Yes, that sounds right, I think you probably deserve a medal! When we send stuff off to the backend, it needs both the sensor controls that need to be dispatched asap, plus the immediate (ISP) controls that are now due because the sensor controls they were originally submitted with, have happened. > > I will save never ending discussions about the overall per-frame > control issues and design, this change if I got it right makes sure a > Request is returned with a list of Controls applied at the time the > Request was processed, and that list might be different than the one > it was created with, right ? Yes. To be honest, I've never actually looked at the controls in a request when it completes because it contains nothing of interest. Previously there would have been some controls that may have been applied, and some that may not have been applied, and the situation here is no different. The metadata is clearly where you would go to find out. But that is a discussion one could have. If you wanted to preserve the original control list, you could save it somewhere else or restore it upon completion, but I can't really see any point. Alternatively, I'd be quite happy to clear it, at least that's easy, and there's no danger you'd think it means something when it doesn't! Actually I don't really regard this commit as having very much to do with per-frame controls, it's about making control lists atomic, but that's kind of a prerequisite whatever we do next. Thanks! David > > > > +} > > + > > } /* namespace libcamera */ > > diff --git a/src/libcamera/pipeline/rpi/common/pipeline_base.h b/src/libcamera/pipeline/rpi/common/pipeline_base.h > > index 597eb587..65d8efdc 100644 > > --- a/src/libcamera/pipeline/rpi/common/pipeline_base.h > > +++ b/src/libcamera/pipeline/rpi/common/pipeline_base.h > > @@ -180,10 +180,18 @@ public: > > > > ClockRecovery wallClockRecovery_; > > > > + struct ImmediateControlsEntry { > > + uint64_t controlListId; > > + ControlList controls; > > + }; > > + std::queue<ImmediateControlsEntry> immediateControls_; > > + > > protected: > > void fillRequestMetadata(const ControlList &bufferControls, > > Request *request); > > > > + void handleControlLists(uint32_t delayContext); > > + > > virtual void tryRunPipeline() = 0; > > > > private: > > diff --git a/src/libcamera/pipeline/rpi/pisp/pisp.cpp b/src/libcamera/pipeline/rpi/pisp/pisp.cpp > > index dff73a79..cc8aa4d4 100644 > > --- a/src/libcamera/pipeline/rpi/pisp/pisp.cpp > > +++ b/src/libcamera/pipeline/rpi/pisp/pisp.cpp > > @@ -2322,6 +2322,9 @@ void PiSPCameraData::tryRunPipeline() > > > > fillRequestMetadata(job.sensorControls, request); > > > > + /* This sorts out synchronisation with ControlLists in earlier requests. */ > > + handleControlLists(job.delayContext); > > + > > /* Set our state to say the pipeline is active. */ > > state_ = State::Busy; > > > > diff --git a/src/libcamera/pipeline/rpi/vc4/vc4.cpp b/src/libcamera/pipeline/rpi/vc4/vc4.cpp > > index b734889d..f743c8a7 100644 > > --- a/src/libcamera/pipeline/rpi/vc4/vc4.cpp > > +++ b/src/libcamera/pipeline/rpi/vc4/vc4.cpp > > @@ -939,6 +939,9 @@ void Vc4CameraData::tryRunPipeline() > > > > fillRequestMetadata(bayerFrame.controls, request); > > > > + /* This sorts out synchronisation with ControlLists in earlier requests. */ > > + handleControlLists(bayerFrame.delayContext); > > + > > /* Set our state to say the pipeline is active. */ > > state_ = State::Busy; > > > > -- > > 2.47.3 > >
diff --git a/src/ipa/rpi/common/ipa_base.cpp b/src/ipa/rpi/common/ipa_base.cpp index faa77719..dacafa57 100644 --- a/src/ipa/rpi/common/ipa_base.cpp +++ b/src/ipa/rpi/common/ipa_base.cpp @@ -431,6 +431,15 @@ void IpaBase::prepareIsp(const PrepareParams ¶ms) rpiMetadata.clear(); fillDeviceStatus(params.sensorControls, ipaContext); + /* + * When there are controls, it's important that we don't skip running the + * IPAs, as that can mess with synchronisation. Crucially though, we need + * to know whether there were controls when this comes back as the + * _delayed_ metadata, hence why we flag this in the metadata itself. + */ + if (!params.requestControls.empty()) + rpiMetadata.set("ipa.request_controls", true); + if (params.buffers.embedded) { /* * Pipeline handler has supplied us with an embedded data buffer, @@ -451,7 +460,7 @@ void IpaBase::prepareIsp(const PrepareParams ¶ms) */ AgcStatus agcStatus; bool hdrChange = false; - RPiController::Metadata &delayedMetadata = rpiMetadata_[params.delayContext]; + RPiController::Metadata &delayedMetadata = rpiMetadata_[params.delayContext % rpiMetadata_.size()]; if (!delayedMetadata.get<AgcStatus>("agc.status", agcStatus)) { rpiMetadata.set("agc.delayed_status", agcStatus); hdrChange = agcStatus.hdr.mode != hdrStatus_.mode; @@ -464,9 +473,13 @@ void IpaBase::prepareIsp(const PrepareParams ¶ms) */ helper_->prepare(embeddedBuffer, rpiMetadata); + bool delayedRequestControls = false; + delayedMetadata.get<bool>("ipa.request_controls", delayedRequestControls); + /* Allow a 10% margin on the comparison below. */ Duration delta = (frameTimestamp - lastRunTimestamp_) * 1.0ns; - if (lastRunTimestamp_ && frameCount_ > invalidCount_ && + if (!delayedRequestControls && params.requestControls.empty() && + lastRunTimestamp_ && frameCount_ > invalidCount_ && delta < controllerMinFrameDuration_ * 0.9 && !hdrChange) { /* * Ensure we merge the previous frame's metadata with the current @@ -535,7 +548,7 @@ void IpaBase::processStats(const ProcessParams ¶ms) ControlList ctrls(sensorCtrls_); applyAGC(&agcStatus, ctrls); rpiMetadata.set("agc.status", agcStatus); - setDelayedControls.emit(ctrls, ipaContext); + setDelayedControls.emit(ctrls, params.ipaContext); setCameraTimeoutValue(); } @@ -951,8 +964,6 @@ void IpaBase::applyControls(const ControlList &controls) /* The control provides units of microseconds. */ agc->setFixedExposureTime(0, ctrl.second.get<int32_t>() * 1.0us); - - libcameraMetadata_.set(controls::ExposureTime, ctrl.second.get<int32_t>()); break; } @@ -976,9 +987,6 @@ void IpaBase::applyControls(const ControlList &controls) break; agc->setFixedGain(0, ctrl.second.get<float>()); - - libcameraMetadata_.set(controls::AnalogueGain, - ctrl.second.get<float>()); break; } diff --git a/src/libcamera/pipeline/rpi/common/pipeline_base.cpp b/src/libcamera/pipeline/rpi/common/pipeline_base.cpp index 867ecf1b..c6df0934 100644 --- a/src/libcamera/pipeline/rpi/common/pipeline_base.cpp +++ b/src/libcamera/pipeline/rpi/common/pipeline_base.cpp @@ -1528,4 +1528,50 @@ void CameraData::fillRequestMetadata(const ControlList &bufferControls, Request } } +static bool isControlDelayed(unsigned int id) +{ + return id == controls::ExposureTime || + id == controls::AnalogueGain || + id == controls::FrameDurationLimits || + id == controls::AeEnable || + id == controls::ExposureTimeMode || + id == controls::AnalogueGainMode; +} + +void CameraData::handleControlLists(uint32_t delayContext) +{ + /* + * THe delayContext is the sequence number after it's gone through the various + * pipeline delays, so that's what gets reported as the "ControlListSequence" + * in the metadata, being the sequence number of the request whose ControlList + * has just been applied. + */ + Request *request = requestQueue_.front(); + request->_d()->metadata().set(controls::rpi::ControlListSequence, delayContext); + + /* + * Controls that take effect immediately (typically ISP controls) have to be + * delayed so as to synchronise with those controls that do get delayed. So we + * must remove them from the current request, and push them onto a queue so + * that they can be used later. + */ + ControlList controls = std::move(request->controls()); + request->controls().clear(); + immediateControls_.push({ request->sequence(), {} }); + for (const auto &ctrl : controls) { + if (isControlDelayed(ctrl.first)) + request->controls().set(ctrl.first, ctrl.second); + else + immediateControls_.back().controls.set(ctrl.first, ctrl.second); + } + + /* "Immediate" controls that have become due are now merged back into this request. */ + while (!immediateControls_.empty() && + immediateControls_.front().controlListId <= delayContext) { + request->controls().merge(immediateControls_.front().controls, + ControlList::MergePolicy::OverwriteExisting); + immediateControls_.pop(); + } +} + } /* namespace libcamera */ diff --git a/src/libcamera/pipeline/rpi/common/pipeline_base.h b/src/libcamera/pipeline/rpi/common/pipeline_base.h index 597eb587..65d8efdc 100644 --- a/src/libcamera/pipeline/rpi/common/pipeline_base.h +++ b/src/libcamera/pipeline/rpi/common/pipeline_base.h @@ -180,10 +180,18 @@ public: ClockRecovery wallClockRecovery_; + struct ImmediateControlsEntry { + uint64_t controlListId; + ControlList controls; + }; + std::queue<ImmediateControlsEntry> immediateControls_; + protected: void fillRequestMetadata(const ControlList &bufferControls, Request *request); + void handleControlLists(uint32_t delayContext); + virtual void tryRunPipeline() = 0; private: diff --git a/src/libcamera/pipeline/rpi/pisp/pisp.cpp b/src/libcamera/pipeline/rpi/pisp/pisp.cpp index dff73a79..cc8aa4d4 100644 --- a/src/libcamera/pipeline/rpi/pisp/pisp.cpp +++ b/src/libcamera/pipeline/rpi/pisp/pisp.cpp @@ -2322,6 +2322,9 @@ void PiSPCameraData::tryRunPipeline() fillRequestMetadata(job.sensorControls, request); + /* This sorts out synchronisation with ControlLists in earlier requests. */ + handleControlLists(job.delayContext); + /* Set our state to say the pipeline is active. */ state_ = State::Busy; diff --git a/src/libcamera/pipeline/rpi/vc4/vc4.cpp b/src/libcamera/pipeline/rpi/vc4/vc4.cpp index b734889d..f743c8a7 100644 --- a/src/libcamera/pipeline/rpi/vc4/vc4.cpp +++ b/src/libcamera/pipeline/rpi/vc4/vc4.cpp @@ -939,6 +939,9 @@ void Vc4CameraData::tryRunPipeline() fillRequestMetadata(bayerFrame.controls, request); + /* This sorts out synchronisation with ControlLists in earlier requests. */ + handleControlLists(bayerFrame.delayContext); + /* Set our state to say the pipeline is active. */ state_ = State::Busy;