diff --git a/include/libcamera/ipa/raspberrypi.mojom b/include/libcamera/ipa/raspberrypi.mojom index 5a27b1e4fc2d..f733a2cd2a40 100644 --- a/include/libcamera/ipa/raspberrypi.mojom +++ b/include/libcamera/ipa/raspberrypi.mojom @@ -29,6 +29,8 @@ struct SensorConfig { struct ISPConfig { uint32 embeddedBufferId; uint32 bayerBufferId; + bool embeddedBufferPresent; + ControlList controls; }; struct ConfigInput { diff --git a/src/ipa/raspberrypi/raspberrypi.cpp b/src/ipa/raspberrypi/raspberrypi.cpp index 1226ea514521..f9ab6417866e 100644 --- a/src/ipa/raspberrypi/raspberrypi.cpp +++ b/src/ipa/raspberrypi/raspberrypi.cpp @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: BSD-2-Clause */ /* - * Copyright (C) 2019-2020, Raspberry Pi (Trading) Ltd. + * Copyright (C) 2019-2021, Raspberry Pi (Trading) Ltd. * * rpi.cpp - Raspberry Pi Image Processing Algorithms */ @@ -101,9 +101,11 @@ private: bool validateIspControls(); void queueRequest(const ControlList &controls); void returnEmbeddedBuffer(unsigned int bufferId); - void prepareISP(unsigned int bufferId); + void prepareISP(const ipa::RPi::ISPConfig &data); void reportMetadata(); bool parseEmbeddedData(unsigned int bufferId, struct DeviceStatus &deviceStatus); + void fillDeviceStatus(uint32_t exposureLines, uint32_t gainCode, + struct DeviceStatus &deviceStatus); void processStats(unsigned int bufferId); void applyFrameDurations(double minFrameDuration, double maxFrameDuration); void applyAGC(const struct AgcStatus *agcStatus, ControlList &ctrls); @@ -447,7 +449,7 @@ void IPARPi::signalIspPrepare(const ipa::RPi::ISPConfig &data) * avoid running the control algos for a few frames in case * they are "unreliable". */ - prepareISP(data.embeddedBufferId); + prepareISP(data); frameCount_++; /* Ready to push the input buffer into the ISP. */ @@ -909,67 +911,84 @@ void IPARPi::returnEmbeddedBuffer(unsigned int bufferId) embeddedComplete.emit(bufferId & ipa::RPi::MaskID); } -void IPARPi::prepareISP(unsigned int bufferId) +void IPARPi::prepareISP(const ipa::RPi::ISPConfig &data) { struct DeviceStatus deviceStatus = {}; - bool success = parseEmbeddedData(bufferId, deviceStatus); + bool success = false; - /* Done with embedded data now, return to pipeline handler asap. */ - returnEmbeddedBuffer(bufferId); + if (data.embeddedBufferPresent) { + /* + * Pipeline handler has supplied us with an embedded data buffer, + * so parse it and extract the exposure and gain. + */ + success = parseEmbeddedData(data.embeddedBufferId, deviceStatus); - if (success) { - ControlList ctrls(ispCtrls_); + /* Done with embedded data now, return to pipeline handler asap. */ + returnEmbeddedBuffer(data.embeddedBufferId); + } - rpiMetadata_.Clear(); - rpiMetadata_.Set("device.status", deviceStatus); - controller_.Prepare(&rpiMetadata_); + if (!success) { + /* + * Pipeline handler has not supplied an embedded data buffer, + * or embedded data buffer parsing has failed for some reason, + * so pull the exposure and gain values from the control list. + */ + int32_t exposureLines = data.controls.get(V4L2_CID_EXPOSURE).get(); + int32_t gainCode = data.controls.get(V4L2_CID_ANALOGUE_GAIN).get(); + fillDeviceStatus(exposureLines, gainCode, deviceStatus); + } - /* Lock the metadata buffer to avoid constant locks/unlocks. */ - std::unique_lock lock(rpiMetadata_); + ControlList ctrls(ispCtrls_); - AwbStatus *awbStatus = rpiMetadata_.GetLocked("awb.status"); - if (awbStatus) - applyAWB(awbStatus, ctrls); + rpiMetadata_.Clear(); + rpiMetadata_.Set("device.status", deviceStatus); + controller_.Prepare(&rpiMetadata_); - CcmStatus *ccmStatus = rpiMetadata_.GetLocked("ccm.status"); - if (ccmStatus) - applyCCM(ccmStatus, ctrls); + /* Lock the metadata buffer to avoid constant locks/unlocks. */ + std::unique_lock lock(rpiMetadata_); - AgcStatus *dgStatus = rpiMetadata_.GetLocked("agc.status"); - if (dgStatus) - applyDG(dgStatus, ctrls); + AwbStatus *awbStatus = rpiMetadata_.GetLocked("awb.status"); + if (awbStatus) + applyAWB(awbStatus, ctrls); - AlscStatus *lsStatus = rpiMetadata_.GetLocked("alsc.status"); - if (lsStatus) - applyLS(lsStatus, ctrls); + CcmStatus *ccmStatus = rpiMetadata_.GetLocked("ccm.status"); + if (ccmStatus) + applyCCM(ccmStatus, ctrls); - ContrastStatus *contrastStatus = rpiMetadata_.GetLocked("contrast.status"); - if (contrastStatus) - applyGamma(contrastStatus, ctrls); + AgcStatus *dgStatus = rpiMetadata_.GetLocked("agc.status"); + if (dgStatus) + applyDG(dgStatus, ctrls); - BlackLevelStatus *blackLevelStatus = rpiMetadata_.GetLocked("black_level.status"); - if (blackLevelStatus) - applyBlackLevel(blackLevelStatus, ctrls); + AlscStatus *lsStatus = rpiMetadata_.GetLocked("alsc.status"); + if (lsStatus) + applyLS(lsStatus, ctrls); - GeqStatus *geqStatus = rpiMetadata_.GetLocked("geq.status"); - if (geqStatus) - applyGEQ(geqStatus, ctrls); + ContrastStatus *contrastStatus = rpiMetadata_.GetLocked("contrast.status"); + if (contrastStatus) + applyGamma(contrastStatus, ctrls); - DenoiseStatus *denoiseStatus = rpiMetadata_.GetLocked("denoise.status"); - if (denoiseStatus) - applyDenoise(denoiseStatus, ctrls); + BlackLevelStatus *blackLevelStatus = rpiMetadata_.GetLocked("black_level.status"); + if (blackLevelStatus) + applyBlackLevel(blackLevelStatus, ctrls); - SharpenStatus *sharpenStatus = rpiMetadata_.GetLocked("sharpen.status"); - if (sharpenStatus) - applySharpen(sharpenStatus, ctrls); + GeqStatus *geqStatus = rpiMetadata_.GetLocked("geq.status"); + if (geqStatus) + applyGEQ(geqStatus, ctrls); - DpcStatus *dpcStatus = rpiMetadata_.GetLocked("dpc.status"); - if (dpcStatus) - applyDPC(dpcStatus, ctrls); + DenoiseStatus *denoiseStatus = rpiMetadata_.GetLocked("denoise.status"); + if (denoiseStatus) + applyDenoise(denoiseStatus, ctrls); - if (!ctrls.empty()) - setIspControls.emit(ctrls); - } + SharpenStatus *sharpenStatus = rpiMetadata_.GetLocked("sharpen.status"); + if (sharpenStatus) + applySharpen(sharpenStatus, ctrls); + + DpcStatus *dpcStatus = rpiMetadata_.GetLocked("dpc.status"); + if (dpcStatus) + applyDPC(dpcStatus, ctrls); + + if (!ctrls.empty()) + setIspControls.emit(ctrls); } bool IPARPi::parseEmbeddedData(unsigned int bufferId, struct DeviceStatus &deviceStatus) @@ -985,6 +1004,7 @@ bool IPARPi::parseEmbeddedData(unsigned int bufferId, struct DeviceStatus &devic RPiController::MdParser::Status status = helper_->Parser().Parse(mem.data()); if (status != RPiController::MdParser::Status::OK) { LOG(IPARPI, Error) << "Embedded Buffer parsing failed, error " << status; + return false; } else { uint32_t exposureLines, gainCode; if (helper_->Parser().GetExposureLines(exposureLines) != RPiController::MdParser::Status::OK) { @@ -992,21 +1012,29 @@ bool IPARPi::parseEmbeddedData(unsigned int bufferId, struct DeviceStatus &devic return false; } - deviceStatus.shutter_speed = helper_->Exposure(exposureLines); if (helper_->Parser().GetGainCode(gainCode) != RPiController::MdParser::Status::OK) { LOG(IPARPI, Error) << "Gain failed"; return false; } - deviceStatus.analogue_gain = helper_->Gain(gainCode); - LOG(IPARPI, Debug) << "Metadata - Exposure : " - << deviceStatus.shutter_speed << " Gain : " - << deviceStatus.analogue_gain; + fillDeviceStatus(exposureLines, gainCode, deviceStatus); } return true; } +void IPARPi::fillDeviceStatus(uint32_t exposureLines, uint32_t gainCode, + struct DeviceStatus &deviceStatus) +{ + deviceStatus.shutter_speed = helper_->Exposure(exposureLines); + deviceStatus.analogue_gain = helper_->Gain(gainCode); + + LOG(IPARPI, Debug) << "Metadata - Exposure : " + << deviceStatus.shutter_speed + << " Gain : " + << deviceStatus.analogue_gain; +} + void IPARPi::processStats(unsigned int bufferId) { auto it = buffers_.find(bufferId); diff --git a/src/libcamera/pipeline/raspberrypi/raspberrypi.cpp b/src/libcamera/pipeline/raspberrypi/raspberrypi.cpp index ba74ace183db..d057241b9c76 100644 --- a/src/libcamera/pipeline/raspberrypi/raspberrypi.cpp +++ b/src/libcamera/pipeline/raspberrypi/raspberrypi.cpp @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: LGPL-2.1-or-later */ /* - * Copyright (C) 2019-2020, Raspberry Pi (Trading) Ltd. + * Copyright (C) 2019-2021, Raspberry Pi (Trading) Ltd. * * raspberrypi.cpp - Pipeline handler for Raspberry Pi devices */ @@ -197,7 +197,13 @@ public: */ enum class State { Stopped, Idle, Busy, IpaComplete }; State state_; - std::queue bayerQueue_; + + struct BayerFrame { + FrameBuffer *buffer; + ControlList controls; + }; + + std::queue bayerQueue_; std::queue embeddedQueue_; std::deque requestQueue_; @@ -222,7 +228,7 @@ public: private: void checkRequestCompleted(); void tryRunPipeline(); - bool findMatchingBuffers(FrameBuffer *&bayerBuffer, FrameBuffer *&embeddedBuffer); + bool findMatchingBuffers(BayerFrame &bayerFrame, FrameBuffer *&embeddedBuffer); unsigned int ispOutputCount_; }; @@ -1355,7 +1361,7 @@ void RPiCameraData::setIspControls(const ControlList &controls) void RPiCameraData::setDelayedControls(const ControlList &controls) { if (!delayedCtrls_->push(controls)) - LOG(RPI, Error) << "V4L2 staggered set failed"; + LOG(RPI, Error) << "V4L2 DelayedControl set failed"; handleState(); } @@ -1383,29 +1389,14 @@ void RPiCameraData::unicamBufferDequeue(FrameBuffer *buffer) << ", timestamp: " << buffer->metadata().timestamp; if (stream == &unicam_[Unicam::Image]) { - bayerQueue_.push(buffer); - } else { - embeddedQueue_.push(buffer); - - ControlList ctrl = delayedCtrls_->get(buffer->metadata().sequence); - /* - * Sensor metadata is unavailable, so put the expected ctrl - * values (accounting for the staggered delays) into the empty - * metadata buffer. + * Lookup the sensor controls used for this frame sequence from + * DelayedControl and queue them along with the frame buffer. */ - if (!sensorMetadata_) { - unsigned int bufferId = unicam_[Unicam::Embedded].getBufferId(buffer); - auto it = mappedEmbeddedBuffers_.find(bufferId); - if (it != mappedEmbeddedBuffers_.end()) { - uint32_t *mem = reinterpret_cast(it->second.maps()[0].data()); - mem[0] = ctrl.get(V4L2_CID_EXPOSURE).get(); - mem[1] = ctrl.get(V4L2_CID_ANALOGUE_GAIN).get(); - } else { - LOG(RPI, Warning) << "Failed to find embedded buffer " - << bufferId; - } - } + ControlList ctrl = delayedCtrls_->get(buffer->metadata().sequence); + bayerQueue_.push({ buffer, std::move(ctrl) }); + } else { + embeddedQueue_.push(buffer); } handleState(); @@ -1656,14 +1647,15 @@ void RPiCameraData::applyScalerCrop(const ControlList &controls) void RPiCameraData::tryRunPipeline() { - FrameBuffer *bayerBuffer, *embeddedBuffer; + FrameBuffer *embeddedBuffer; + BayerFrame bayerFrame; /* If any of our request or buffer queues are empty, we cannot proceed. */ if (state_ != State::Idle || requestQueue_.empty() || bayerQueue_.empty() || embeddedQueue_.empty()) return; - if (!findMatchingBuffers(bayerBuffer, embeddedBuffer)) + if (!findMatchingBuffers(bayerFrame, embeddedBuffer)) return; /* Take the first request from the queue and action the IPA. */ @@ -1682,7 +1674,7 @@ void RPiCameraData::tryRunPipeline() /* Set our state to say the pipeline is active. */ state_ = State::Busy; - unsigned int bayerId = unicam_[Unicam::Image].getBufferId(bayerBuffer); + unsigned int bayerId = unicam_[Unicam::Image].getBufferId(bayerFrame.buffer); unsigned int embeddedId = unicam_[Unicam::Embedded].getBufferId(embeddedBuffer); LOG(RPI, Debug) << "Signalling signalIspPrepare:" @@ -1692,17 +1684,19 @@ void RPiCameraData::tryRunPipeline() ipa::RPi::ISPConfig ispPrepare; ispPrepare.embeddedBufferId = ipa::RPi::MaskEmbeddedData | embeddedId; ispPrepare.bayerBufferId = ipa::RPi::MaskBayerData | bayerId; + ispPrepare.embeddedBufferPresent = sensorMetadata_; + ispPrepare.controls = std::move(bayerFrame.controls); ipa_->signalIspPrepare(ispPrepare); } -bool RPiCameraData::findMatchingBuffers(FrameBuffer *&bayerBuffer, FrameBuffer *&embeddedBuffer) +bool RPiCameraData::findMatchingBuffers(BayerFrame &bayerFrame, FrameBuffer *&embeddedBuffer) { unsigned int embeddedRequeueCount = 0, bayerRequeueCount = 0; /* Loop until we find a matching bayer and embedded data buffer. */ while (!bayerQueue_.empty()) { /* Start with the front of the bayer queue. */ - bayerBuffer = bayerQueue_.front(); + FrameBuffer *bayerBuffer = bayerQueue_.front().buffer; /* * Find the embedded data buffer with a matching timestamp to pass to @@ -1739,7 +1733,7 @@ bool RPiCameraData::findMatchingBuffers(FrameBuffer *&bayerBuffer, FrameBuffer * * the front of the queue. This buffer is now orphaned, so requeue * it back to the device. */ - unicam_[Unicam::Image].queueBuffer(bayerQueue_.front()); + unicam_[Unicam::Image].queueBuffer(bayerQueue_.front().buffer); bayerQueue_.pop(); bayerRequeueCount++; LOG(RPI, Warning) << "Dropping unmatched input frame in stream " @@ -1757,7 +1751,7 @@ bool RPiCameraData::findMatchingBuffers(FrameBuffer *&bayerBuffer, FrameBuffer * LOG(RPI, Warning) << "Flushing bayer stream!"; while (!bayerQueue_.empty()) { - unicam_[Unicam::Image].queueBuffer(bayerQueue_.front()); + unicam_[Unicam::Image].queueBuffer(bayerQueue_.front().buffer); bayerQueue_.pop(); } flushedBuffers = true; @@ -1790,8 +1784,10 @@ bool RPiCameraData::findMatchingBuffers(FrameBuffer *&bayerBuffer, FrameBuffer * } else { /* * We have found a matching bayer and embedded data buffer, so - * nothing more to do apart from popping the buffers from the queue. + * nothing more to do apart from assigning the bayer frame and + * popping the buffers from the queue. */ + bayerFrame = std::move(bayerQueue_.front()); bayerQueue_.pop(); embeddedQueue_.pop(); return true;