[v7,7/8] libcamera: mali-c55: Implement capture for memory-to-memory

26397 diff mbox series

Plumb in the MaliC55 pipeline handler support for capturing frames
from memory using the CRU.

Introduce a data flow which uses the CRU to feed the ISP through
the IVC.

In detail:

- push incoming request to a pending queue until a buffer from the CRU
  is available
- delay the call to ipa_->fillParams() to the CRU buffer ready event
- once the IPA has computed parameters feed the ISP through the IVC
  with buffers from the CRU, params and statistics
- Handle completion of in-flight requests: in m2m mode buffers are
  queued earlier to the ISP capture devices. If the camera is stopped
  these buffers are returned earlier in error state: complete the
  Request bypassing the IPA operations
- As cancelled Requests might now be completed earlier then IPA events,
  modify the IPA event handler to ignore Requests that have been
  completed already

Signed-off-by: Daniel Scally <dan.scally@ideasonboard.com>
Signed-off-by: Jacopo Mondi <jacopo.mondi@ideasonboard.com>
---
 src/libcamera/pipeline/mali-c55/mali-c55.cpp | 211 ++++++++++++++++++++++-----
 1 file changed, 174 insertions(+), 37 deletions(-)

Hi me

On Tue, Mar 31, 2026 at 06:36:35PM +0200, Jacopo Mondi wrote:
> Plumb in the MaliC55 pipeline handler support for capturing frames
> from memory using the CRU.
>
> Introduce a data flow which uses the CRU to feed the ISP through
> the IVC.
>
> In detail:
>
> - push incoming request to a pending queue until a buffer from the CRU
>   is available
> - delay the call to ipa_->fillParams() to the CRU buffer ready event
> - once the IPA has computed parameters feed the ISP through the IVC
>   with buffers from the CRU, params and statistics
> - Handle completion of in-flight requests: in m2m mode buffers are
>   queued earlier to the ISP capture devices. If the camera is stopped
>   these buffers are returned earlier in error state: complete the
>   Request bypassing the IPA operations
> - As cancelled Requests might now be completed earlier then IPA events,
>   modify the IPA event handler to ignore Requests that have been
>   completed already
>
> Signed-off-by: Daniel Scally <dan.scally@ideasonboard.com>
> Signed-off-by: Jacopo Mondi <jacopo.mondi@ideasonboard.com>
> ---
>  src/libcamera/pipeline/mali-c55/mali-c55.cpp | 211 ++++++++++++++++++++++-----
>  1 file changed, 174 insertions(+), 37 deletions(-)
>
> diff --git a/src/libcamera/pipeline/mali-c55/mali-c55.cpp b/src/libcamera/pipeline/mali-c55/mali-c55.cpp
> index ce1f0b65cd6e..c3856f7c0a11 100644
> --- a/src/libcamera/pipeline/mali-c55/mali-c55.cpp
> +++ b/src/libcamera/pipeline/mali-c55/mali-c55.cpp
> @@ -21,6 +21,7 @@
>  #include <libcamera/base/utils.h>
>
>  #include <libcamera/camera.h>
> +#include <libcamera/controls.h>
>  #include <libcamera/formats.h>
>  #include <libcamera/geometry.h>
>  #include <libcamera/property_ids.h>
> @@ -90,6 +91,7 @@ struct MaliC55FrameInfo {
>
>  	FrameBuffer *paramBuffer;
>  	FrameBuffer *statBuffer;
> +	FrameBuffer *rawBuffer;
>
>  	bool paramsDone;
>  	bool statsDone;
> @@ -693,6 +695,7 @@ public:
>  	void imageBufferReady(FrameBuffer *buffer);
>  	void paramsBufferReady(FrameBuffer *buffer);
>  	void statsBufferReady(FrameBuffer *buffer);
> +	void cruBufferReady(FrameBuffer *buffer);
>  	void paramsComputed(unsigned int requestId, uint32_t bytesused);
>  	void statsProcessed(unsigned int requestId, const ControlList &metadata);
>
> @@ -741,7 +744,7 @@ private:
>
>  	MaliC55FrameInfo *findFrameInfo(FrameBuffer *buffer);
>  	MaliC55FrameInfo *findFrameInfo(Request *request);
> -	void tryComplete(MaliC55FrameInfo *info);
> +	void tryComplete(MaliC55FrameInfo *info, bool cancelled = false);
>
>  	int configureRawStream(MaliC55CameraData *data,
>  			       const StreamConfiguration &config,
> @@ -750,6 +753,9 @@ private:
>  				     const StreamConfiguration &config,
>  				     V4L2SubdeviceFormat &subdevFormat);
>
> +	MaliC55FrameInfo *prepareFrameInfo(Request *request);
> +	void queueRequestToCru(MaliC55CameraData *data, Request *request);
> +
>  	void applyScalerCrop(Camera *camera, const ControlList &controls);
>
>  	bool registerMaliCamera(std::unique_ptr<MaliC55CameraData> data,
> @@ -1222,6 +1228,13 @@ void PipelineHandlerMaliC55::freeBuffers(Camera *camera)
>  	if (params_->releaseBuffers())
>  		LOG(MaliC55, Error) << "Failed to release params buffers";
>
> +	if (auto *mem = std::get_if<MaliC55CameraData::Memory>(&data->input_)) {
> +		if (ivc_->releaseBuffers())
> +			LOG(MaliC55, Error) << "Failed to release input buffers";
> +		if (mem->cru_->freeBuffers())
> +			LOG(MaliC55, Error) << "Failed to release CRU buffers";
> +	}
> +
>  	return;
>  }
>
> @@ -1245,6 +1258,12 @@ int PipelineHandlerMaliC55::allocateBuffers(Camera *camera)
>  		}
>  	};
>
> +	if (std::holds_alternative<MaliC55CameraData::Memory>(data->input_)) {
> +		ret = ivc_->importBuffers(kMaliC55BufferCount);
> +		if (ret < 0)
> +			return ret;
> +	}
> +
>  	ret = stats_->allocateBuffers(kMaliC55BufferCount, &statsBuffers_);
>  	if (ret < 0)
>  		return ret;
> @@ -1276,6 +1295,24 @@ int PipelineHandlerMaliC55::start(Camera *camera, [[maybe_unused]] const Control
>  	if (ret)
>  		return ret;
>
> +	if (auto *mem = std::get_if<MaliC55CameraData::Memory>(&data->input_)) {
> +		ret = mem->cru_->start(kMaliC55BufferCount);
> +		if (ret) {
> +			LOG(MaliC55, Error)
> +				<< "Failed to start CRU " << camera->id();
> +			freeBuffers(camera);
> +			return ret;
> +		}
> +
> +		ret = ivc_->streamOn();
> +		if (ret) {
> +			LOG(MaliC55, Error)
> +				<< "Failed to start IVC" << camera->id();
> +			freeBuffers(camera);
> +			return ret;
> +		}
> +	}
> +
>  	if (data->ipa_) {
>  		ret = data->ipa_->start();
>  		if (ret) {
> @@ -1357,6 +1394,11 @@ void PipelineHandlerMaliC55::stopDevice(Camera *camera)
>
>  	isp_->setFrameStartEnabled(false);
>
> +	if (auto *mem = std::get_if<MaliC55CameraData::Memory>(&data->input_)) {
> +		ivc_->streamOff();
> +		mem->cru_->stop();
> +	}
> +
>  	for (MaliC55Pipe &pipe : pipes_) {
>  		if (!pipe.stream)
>  			continue;
> @@ -1468,10 +1510,67 @@ void PipelineHandlerMaliC55::applyScalerCrop(Camera *camera,
>  	}
>  }
>
> +MaliC55FrameInfo *PipelineHandlerMaliC55::prepareFrameInfo(Request *request)
> +{
> +	if (availableStatsBuffers_.empty()) {
> +		LOG(MaliC55, Error) << "Stats buffer underrun";
> +		return nullptr;
> +	}
> +
> +	if (availableParamsBuffers_.empty()) {
> +		LOG(MaliC55, Error) << "Params buffer underrun";
> +		return nullptr;
> +	}
> +
> +	MaliC55FrameInfo &frameInfo = frameInfoMap_[request->sequence()];
> +	frameInfo.request = request;
> +	frameInfo.statBuffer = availableStatsBuffers_.front();
> +	availableStatsBuffers_.pop();
> +	frameInfo.paramBuffer = availableParamsBuffers_.front();
> +	availableParamsBuffers_.pop();
> +
> +	frameInfo.paramsDone = false;
> +	frameInfo.statsDone = false;
> +
> +	return &frameInfo;
> +}
> +
> +void PipelineHandlerMaliC55::queueRequestToCru(MaliC55CameraData *data,
> +					       Request *request)
> +{
> +	auto *mem = std::get_if<MaliC55CameraData::Memory>(&data->input_);
> +	ASSERT(mem);
> +
> +	FrameBuffer *cruBuffer = mem->cru_->queueBuffer(request);
> +	ASSERT(cruBuffer);
> +
> +	auto frameInfo = prepareFrameInfo(request);
> +	ASSERT(frameInfo);
> +	frameInfo->rawBuffer = cruBuffer;
> +
> +	for (auto &[stream, buffer] : request->buffers()) {
> +		MaliC55Pipe *pipe = pipeFromStream(data, stream);
> +
> +		pipe->cap->queueBuffer(buffer);
> +	}
> +
> +	data->ipa_->queueRequest(request->sequence(), request->controls());
> +}
> +
>  int PipelineHandlerMaliC55::queueRequestDevice(Camera *camera, Request *request)
>  {
>  	MaliC55CameraData *data = cameraData(camera);
>
> +	/*
> +	 * If we're in memory input mode, we need to queue the Request to the
> +	 * CRU, otherwise we can just do everything immediately.
> +	 */
> +	if (std::holds_alternative<MaliC55CameraData::Memory>(data->input_)) {
> +		queueRequestToCru(data, request);
> +
> +		return 0;
> +	}
> +
>  	/* Do not run the IPA if the TPG is in use. */
>  	if (!data->ipa_) {
>  		MaliC55FrameInfo frameInfo;
> @@ -1492,32 +1591,13 @@ int PipelineHandlerMaliC55::queueRequestDevice(Camera *camera, Request *request)
>  		return 0;
>  	}
>
> -	if (availableStatsBuffers_.empty()) {
> -		LOG(MaliC55, Error) << "Stats buffer underrun";
> +	auto frameInfo = prepareFrameInfo(request);
> +	if (!frameInfo)
>  		return -ENOENT;
> -	}

You could very well assert here as well now

> -
> -	if (availableParamsBuffers_.empty()) {
> -		LOG(MaliC55, Error) << "Params buffer underrun";
> -		return -ENOENT;
> -	}
> -
> -	MaliC55FrameInfo frameInfo;
> -	frameInfo.request = request;
> -
> -	frameInfo.statBuffer = availableStatsBuffers_.front();
> -	availableStatsBuffers_.pop();
> -	frameInfo.paramBuffer = availableParamsBuffers_.front();
> -	availableParamsBuffers_.pop();
> -
> -	frameInfo.paramsDone = false;
> -	frameInfo.statsDone = false;
> -
> -	frameInfoMap_[request->sequence()] = frameInfo;
>
>  	data->ipa_->queueRequest(request->sequence(), request->controls());
>  	data->ipa_->fillParams(request->sequence(),
> -			       frameInfo.paramBuffer->cookie());
> +			       frameInfo->paramBuffer->cookie());
>
>  	return 0;
>  }
> @@ -1536,18 +1616,21 @@ MaliC55FrameInfo *PipelineHandlerMaliC55::findFrameInfo(FrameBuffer *buffer)
>  {
>  	for (auto &[sequence, info] : frameInfoMap_) {
>  		if (info.paramBuffer == buffer ||
> -		    info.statBuffer == buffer)
> +		    info.statBuffer == buffer ||
> +		    info.rawBuffer == buffer)
>  			return &info;
>  	}
>
>  	return nullptr;
>  }
>
> -void PipelineHandlerMaliC55::tryComplete(MaliC55FrameInfo *info)
> +void PipelineHandlerMaliC55::tryComplete(MaliC55FrameInfo *info, bool cancelled)
>  {
> -	if (!info->paramsDone)
> -		return;
> -	if (!info->statsDone)
> +	/*
> +	 * If the buffer has been cancelled, we complete the request without
> +	 * waiting for the IPA.
> +	 */
> +	if (!cancelled && (!info->paramsDone || !info->statsDone))
>  		return;
>
>  	Request *request = info->request;
> @@ -1571,13 +1654,15 @@ void PipelineHandlerMaliC55::imageBufferReady(FrameBuffer *buffer)
>  	ASSERT(info);
>
>  	if (completeBuffer(request, buffer))
> -		tryComplete(info);
> +		tryComplete(info,
> +			    buffer->metadata().status == FrameMetadata::FrameCancelled);
>  }
>
>  void PipelineHandlerMaliC55::paramsBufferReady(FrameBuffer *buffer)
>  {
>  	MaliC55FrameInfo *info = findFrameInfo(buffer);
> -	ASSERT(info);
> +	if (!info)
> +		return;
>
>  	info->paramsDone = true;
>
> @@ -1587,7 +1672,8 @@ void PipelineHandlerMaliC55::paramsBufferReady(FrameBuffer *buffer)
>  void PipelineHandlerMaliC55::statsBufferReady(FrameBuffer *buffer)
>  {
>  	MaliC55FrameInfo *info = findFrameInfo(buffer);
> -	ASSERT(info);
> +	if (!info)
> +		return;
>
>  	Request *request = info->request;
>  	MaliC55CameraData *data = cameraData(request->_d()->camera());
> @@ -1598,32 +1684,80 @@ void PipelineHandlerMaliC55::statsBufferReady(FrameBuffer *buffer)
>  				 sensorControls);
>  }
>
> +void PipelineHandlerMaliC55::cruBufferReady(FrameBuffer *buffer)
> +{
> +	/*
> +	 * If the buffer has been cancelled, do not ask the IPA to prepare
> +	 * parameters.
> +	 *
> +	 * The Request this cancelled buffer belongs to will be handled by
> +	 * imageBufferReady() as we have queued buffers to the ISP capture
> +	 * devices at the same time we have queued this buffer to the CRU
> +	 * when running in m2m mode.
> +	 */
> +	if (buffer->metadata().status == FrameMetadata::FrameCancelled)
> +		return;
> +
> +	MaliC55FrameInfo *info = findFrameInfo(buffer);
> +	ASSERT(info);
> +
> +	Request *request = info->request;
> +	request->_d()->metadata().set(controls::SensorTimestamp,
> +				      buffer->metadata().timestamp);
> +
> +	/* Ought we do something with the sensor's controls here...? */

and remove this leftover comment

I'll send a slightly re-freshed series


> +	MaliC55CameraData *data = cameraData(request->_d()->camera());
> +	data->ipa_->fillParams(request->sequence(), info->paramBuffer->cookie());
> +}
> +
>  void PipelineHandlerMaliC55::paramsComputed(unsigned int requestId, uint32_t bytesused)
>  {
> -	MaliC55FrameInfo &frameInfo = frameInfoMap_[requestId];
> +	auto it = frameInfoMap_.find(requestId);
> +	if (it == frameInfoMap_.end())
> +		return;
> +
> +	MaliC55FrameInfo &frameInfo = it->second;
>  	Request *request = frameInfo.request;
> +	if (!request)
> +		return;
> +
>  	MaliC55CameraData *data = cameraData(request->_d()->camera());
>
>  	/*
>  	 * Queue buffers for stats and params, then queue buffers to the capture
> -	 * video devices.
> +	 * video devices if we're running in Inline mode or with the TPG.
> +	 *
> +	 * If we're running in M2M buffers have been queued to the capture
> +	 * devices at queueRequestToCru() time and here we only have to queue
> +	 * buffers to the IVC input to start a transfer.
>  	 */
>
>  	frameInfo.paramBuffer->_d()->metadata().planes()[0].bytesused = bytesused;
>  	params_->queueBuffer(frameInfo.paramBuffer);
>  	stats_->queueBuffer(frameInfo.statBuffer);
>
> -	for (auto &[stream, buffer] : request->buffers()) {
> -		MaliC55Pipe *pipe = pipeFromStream(data, stream);
> +	if (!std::holds_alternative<MaliC55CameraData::Memory>(data->input_)) {
> +		for (auto &[stream, buffer] : request->buffers()) {
> +			MaliC55Pipe *pipe = pipeFromStream(data, stream);
>
> -		pipe->cap->queueBuffer(buffer);
> +			pipe->cap->queueBuffer(buffer);
> +		}
> +	} else {
> +		ivc_->queueBuffer(frameInfo.rawBuffer);
> +		frameInfo.rawBuffer = nullptr;
>  	}
>  }
>
>  void PipelineHandlerMaliC55::statsProcessed(unsigned int requestId,
>  					    const ControlList &metadata)
>  {
> -	MaliC55FrameInfo &frameInfo = frameInfoMap_[requestId];
> +	auto it = frameInfoMap_.find(requestId);
> +	if (it == frameInfoMap_.end())
> +		return;
> +
> +	MaliC55FrameInfo &frameInfo = it->second;
> +	if (!frameInfo.request)
> +		return;
>
>  	frameInfo.statsDone = true;
>  	frameInfo.request->_d()->metadata().merge(metadata);
> @@ -1785,6 +1919,9 @@ bool PipelineHandlerMaliC55::registerMemoryInputCamera(MediaLink *link)
>
>  	ivc_->bufferReady.connect(mem->cru_.get(), &RZG2LCRU::returnBuffer);
>
> +	V4L2VideoDevice *cruOutput = mem->cru_->output();
> +	cruOutput->bufferReady.connect(this, &PipelineHandlerMaliC55::cruBufferReady);
> +
>  	return registerMaliCamera(std::move(data), sensor->device()->entity()->name());
>  }
>
>
> --
> 2.53.0
>