[{"id":38487,"web_url":"https://patchwork.libcamera.org/comment/38487/","msgid":"<ac1FE7JpUMURSby4@zed>","date":"2026-04-01T16:18:38","subject":"Re: [PATCH v7 7/8] libcamera: mali-c55: Implement capture for\n\tmemory-to-memory","submitter":{"id":143,"url":"https://patchwork.libcamera.org/api/people/143/","name":"Jacopo Mondi","email":"jacopo.mondi@ideasonboard.com"},"content":"Hi me\n\nOn Tue, Mar 31, 2026 at 06:36:35PM +0200, Jacopo Mondi wrote:\n> Plumb in the MaliC55 pipeline handler support for capturing frames\n> from memory using the CRU.\n>\n> Introduce a data flow which uses the CRU to feed the ISP through\n> the IVC.\n>\n> In detail:\n>\n> - push incoming request to a pending queue until a buffer from the CRU\n>   is available\n> - delay the call to ipa_->fillParams() to the CRU buffer ready event\n> - once the IPA has computed parameters feed the ISP through the IVC\n>   with buffers from the CRU, params and statistics\n> - Handle completion of in-flight requests: in m2m mode buffers are\n>   queued earlier to the ISP capture devices. If the camera is stopped\n>   these buffers are returned earlier in error state: complete the\n>   Request bypassing the IPA operations\n> - As cancelled Requests might now be completed earlier then IPA events,\n>   modify the IPA event handler to ignore Requests that have been\n>   completed already\n>\n> Signed-off-by: Daniel Scally <dan.scally@ideasonboard.com>\n> Signed-off-by: Jacopo Mondi <jacopo.mondi@ideasonboard.com>\n> ---\n>  src/libcamera/pipeline/mali-c55/mali-c55.cpp | 211 ++++++++++++++++++++++-----\n>  1 file changed, 174 insertions(+), 37 deletions(-)\n>\n> diff --git a/src/libcamera/pipeline/mali-c55/mali-c55.cpp b/src/libcamera/pipeline/mali-c55/mali-c55.cpp\n> index ce1f0b65cd6e..c3856f7c0a11 100644\n> --- a/src/libcamera/pipeline/mali-c55/mali-c55.cpp\n> +++ b/src/libcamera/pipeline/mali-c55/mali-c55.cpp\n> @@ -21,6 +21,7 @@\n>  #include <libcamera/base/utils.h>\n>\n>  #include <libcamera/camera.h>\n> +#include <libcamera/controls.h>\n>  #include <libcamera/formats.h>\n>  #include <libcamera/geometry.h>\n>  #include <libcamera/property_ids.h>\n> @@ -90,6 +91,7 @@ struct MaliC55FrameInfo {\n>\n>  \tFrameBuffer *paramBuffer;\n>  \tFrameBuffer *statBuffer;\n> +\tFrameBuffer *rawBuffer;\n>\n>  \tbool paramsDone;\n>  \tbool statsDone;\n> @@ -693,6 +695,7 @@ public:\n>  \tvoid imageBufferReady(FrameBuffer *buffer);\n>  \tvoid paramsBufferReady(FrameBuffer *buffer);\n>  \tvoid statsBufferReady(FrameBuffer *buffer);\n> +\tvoid cruBufferReady(FrameBuffer *buffer);\n>  \tvoid paramsComputed(unsigned int requestId, uint32_t bytesused);\n>  \tvoid statsProcessed(unsigned int requestId, const ControlList &metadata);\n>\n> @@ -741,7 +744,7 @@ private:\n>\n>  \tMaliC55FrameInfo *findFrameInfo(FrameBuffer *buffer);\n>  \tMaliC55FrameInfo *findFrameInfo(Request *request);\n> -\tvoid tryComplete(MaliC55FrameInfo *info);\n> +\tvoid tryComplete(MaliC55FrameInfo *info, bool cancelled = false);\n>\n>  \tint configureRawStream(MaliC55CameraData *data,\n>  \t\t\t       const StreamConfiguration &config,\n> @@ -750,6 +753,9 @@ private:\n>  \t\t\t\t     const StreamConfiguration &config,\n>  \t\t\t\t     V4L2SubdeviceFormat &subdevFormat);\n>\n> +\tMaliC55FrameInfo *prepareFrameInfo(Request *request);\n> +\tvoid queueRequestToCru(MaliC55CameraData *data, Request *request);\n> +\n>  \tvoid applyScalerCrop(Camera *camera, const ControlList &controls);\n>\n>  \tbool registerMaliCamera(std::unique_ptr<MaliC55CameraData> data,\n> @@ -1222,6 +1228,13 @@ void PipelineHandlerMaliC55::freeBuffers(Camera *camera)\n>  \tif (params_->releaseBuffers())\n>  \t\tLOG(MaliC55, Error) << \"Failed to release params buffers\";\n>\n> +\tif (auto *mem = std::get_if<MaliC55CameraData::Memory>(&data->input_)) {\n> +\t\tif (ivc_->releaseBuffers())\n> +\t\t\tLOG(MaliC55, Error) << \"Failed to release input buffers\";\n> +\t\tif (mem->cru_->freeBuffers())\n> +\t\t\tLOG(MaliC55, Error) << \"Failed to release CRU buffers\";\n> +\t}\n> +\n>  \treturn;\n>  }\n>\n> @@ -1245,6 +1258,12 @@ int PipelineHandlerMaliC55::allocateBuffers(Camera *camera)\n>  \t\t}\n>  \t};\n>\n> +\tif (std::holds_alternative<MaliC55CameraData::Memory>(data->input_)) {\n> +\t\tret = ivc_->importBuffers(kMaliC55BufferCount);\n> +\t\tif (ret < 0)\n> +\t\t\treturn ret;\n> +\t}\n> +\n>  \tret = stats_->allocateBuffers(kMaliC55BufferCount, &statsBuffers_);\n>  \tif (ret < 0)\n>  \t\treturn ret;\n> @@ -1276,6 +1295,24 @@ int PipelineHandlerMaliC55::start(Camera *camera, [[maybe_unused]] const Control\n>  \tif (ret)\n>  \t\treturn ret;\n>\n> +\tif (auto *mem = std::get_if<MaliC55CameraData::Memory>(&data->input_)) {\n> +\t\tret = mem->cru_->start(kMaliC55BufferCount);\n> +\t\tif (ret) {\n> +\t\t\tLOG(MaliC55, Error)\n> +\t\t\t\t<< \"Failed to start CRU \" << camera->id();\n> +\t\t\tfreeBuffers(camera);\n> +\t\t\treturn ret;\n> +\t\t}\n> +\n> +\t\tret = ivc_->streamOn();\n> +\t\tif (ret) {\n> +\t\t\tLOG(MaliC55, Error)\n> +\t\t\t\t<< \"Failed to start IVC\" << camera->id();\n> +\t\t\tfreeBuffers(camera);\n> +\t\t\treturn ret;\n> +\t\t}\n> +\t}\n> +\n>  \tif (data->ipa_) {\n>  \t\tret = data->ipa_->start();\n>  \t\tif (ret) {\n> @@ -1357,6 +1394,11 @@ void PipelineHandlerMaliC55::stopDevice(Camera *camera)\n>\n>  \tisp_->setFrameStartEnabled(false);\n>\n> +\tif (auto *mem = std::get_if<MaliC55CameraData::Memory>(&data->input_)) {\n> +\t\tivc_->streamOff();\n> +\t\tmem->cru_->stop();\n> +\t}\n> +\n>  \tfor (MaliC55Pipe &pipe : pipes_) {\n>  \t\tif (!pipe.stream)\n>  \t\t\tcontinue;\n> @@ -1468,10 +1510,67 @@ void PipelineHandlerMaliC55::applyScalerCrop(Camera *camera,\n>  \t}\n>  }\n>\n> +MaliC55FrameInfo *PipelineHandlerMaliC55::prepareFrameInfo(Request *request)\n> +{\n> +\tif (availableStatsBuffers_.empty()) {\n> +\t\tLOG(MaliC55, Error) << \"Stats buffer underrun\";\n> +\t\treturn nullptr;\n> +\t}\n> +\n> +\tif (availableParamsBuffers_.empty()) {\n> +\t\tLOG(MaliC55, Error) << \"Params buffer underrun\";\n> +\t\treturn nullptr;\n> +\t}\n> +\n> +\tMaliC55FrameInfo &frameInfo = frameInfoMap_[request->sequence()];\n> +\tframeInfo.request = request;\n> +\tframeInfo.statBuffer = availableStatsBuffers_.front();\n> +\tavailableStatsBuffers_.pop();\n> +\tframeInfo.paramBuffer = availableParamsBuffers_.front();\n> +\tavailableParamsBuffers_.pop();\n> +\n> +\tframeInfo.paramsDone = false;\n> +\tframeInfo.statsDone = false;\n> +\n> +\treturn &frameInfo;\n> +}\n> +\n> +void PipelineHandlerMaliC55::queueRequestToCru(MaliC55CameraData *data,\n> +\t\t\t\t\t       Request *request)\n> +{\n> +\tauto *mem = std::get_if<MaliC55CameraData::Memory>(&data->input_);\n> +\tASSERT(mem);\n> +\n> +\tFrameBuffer *cruBuffer = mem->cru_->queueBuffer(request);\n> +\tASSERT(cruBuffer);\n> +\n> +\tauto frameInfo = prepareFrameInfo(request);\n> +\tASSERT(frameInfo);\n> +\tframeInfo->rawBuffer = cruBuffer;\n> +\n> +\tfor (auto &[stream, buffer] : request->buffers()) {\n> +\t\tMaliC55Pipe *pipe = pipeFromStream(data, stream);\n> +\n> +\t\tpipe->cap->queueBuffer(buffer);\n> +\t}\n> +\n> +\tdata->ipa_->queueRequest(request->sequence(), request->controls());\n> +}\n> +\n>  int PipelineHandlerMaliC55::queueRequestDevice(Camera *camera, Request *request)\n>  {\n>  \tMaliC55CameraData *data = cameraData(camera);\n>\n> +\t/*\n> +\t * If we're in memory input mode, we need to queue the Request to the\n> +\t * CRU, otherwise we can just do everything immediately.\n> +\t */\n> +\tif (std::holds_alternative<MaliC55CameraData::Memory>(data->input_)) {\n> +\t\tqueueRequestToCru(data, request);\n> +\n> +\t\treturn 0;\n> +\t}\n> +\n>  \t/* Do not run the IPA if the TPG is in use. */\n>  \tif (!data->ipa_) {\n>  \t\tMaliC55FrameInfo frameInfo;\n> @@ -1492,32 +1591,13 @@ int PipelineHandlerMaliC55::queueRequestDevice(Camera *camera, Request *request)\n>  \t\treturn 0;\n>  \t}\n>\n> -\tif (availableStatsBuffers_.empty()) {\n> -\t\tLOG(MaliC55, Error) << \"Stats buffer underrun\";\n> +\tauto frameInfo = prepareFrameInfo(request);\n> +\tif (!frameInfo)\n>  \t\treturn -ENOENT;\n> -\t}\n\nYou could very well assert here as well now\n\n> -\n> -\tif (availableParamsBuffers_.empty()) {\n> -\t\tLOG(MaliC55, Error) << \"Params buffer underrun\";\n> -\t\treturn -ENOENT;\n> -\t}\n> -\n> -\tMaliC55FrameInfo frameInfo;\n> -\tframeInfo.request = request;\n> -\n> -\tframeInfo.statBuffer = availableStatsBuffers_.front();\n> -\tavailableStatsBuffers_.pop();\n> -\tframeInfo.paramBuffer = availableParamsBuffers_.front();\n> -\tavailableParamsBuffers_.pop();\n> -\n> -\tframeInfo.paramsDone = false;\n> -\tframeInfo.statsDone = false;\n> -\n> -\tframeInfoMap_[request->sequence()] = frameInfo;\n>\n>  \tdata->ipa_->queueRequest(request->sequence(), request->controls());\n>  \tdata->ipa_->fillParams(request->sequence(),\n> -\t\t\t       frameInfo.paramBuffer->cookie());\n> +\t\t\t       frameInfo->paramBuffer->cookie());\n>\n>  \treturn 0;\n>  }\n> @@ -1536,18 +1616,21 @@ MaliC55FrameInfo *PipelineHandlerMaliC55::findFrameInfo(FrameBuffer *buffer)\n>  {\n>  \tfor (auto &[sequence, info] : frameInfoMap_) {\n>  \t\tif (info.paramBuffer == buffer ||\n> -\t\t    info.statBuffer == buffer)\n> +\t\t    info.statBuffer == buffer ||\n> +\t\t    info.rawBuffer == buffer)\n>  \t\t\treturn &info;\n>  \t}\n>\n>  \treturn nullptr;\n>  }\n>\n> -void PipelineHandlerMaliC55::tryComplete(MaliC55FrameInfo *info)\n> +void PipelineHandlerMaliC55::tryComplete(MaliC55FrameInfo *info, bool cancelled)\n>  {\n> -\tif (!info->paramsDone)\n> -\t\treturn;\n> -\tif (!info->statsDone)\n> +\t/*\n> +\t * If the buffer has been cancelled, we complete the request without\n> +\t * waiting for the IPA.\n> +\t */\n> +\tif (!cancelled && (!info->paramsDone || !info->statsDone))\n>  \t\treturn;\n>\n>  \tRequest *request = info->request;\n> @@ -1571,13 +1654,15 @@ void PipelineHandlerMaliC55::imageBufferReady(FrameBuffer *buffer)\n>  \tASSERT(info);\n>\n>  \tif (completeBuffer(request, buffer))\n> -\t\ttryComplete(info);\n> +\t\ttryComplete(info,\n> +\t\t\t    buffer->metadata().status == FrameMetadata::FrameCancelled);\n>  }\n>\n>  void PipelineHandlerMaliC55::paramsBufferReady(FrameBuffer *buffer)\n>  {\n>  \tMaliC55FrameInfo *info = findFrameInfo(buffer);\n> -\tASSERT(info);\n> +\tif (!info)\n> +\t\treturn;\n>\n>  \tinfo->paramsDone = true;\n>\n> @@ -1587,7 +1672,8 @@ void PipelineHandlerMaliC55::paramsBufferReady(FrameBuffer *buffer)\n>  void PipelineHandlerMaliC55::statsBufferReady(FrameBuffer *buffer)\n>  {\n>  \tMaliC55FrameInfo *info = findFrameInfo(buffer);\n> -\tASSERT(info);\n> +\tif (!info)\n> +\t\treturn;\n>\n>  \tRequest *request = info->request;\n>  \tMaliC55CameraData *data = cameraData(request->_d()->camera());\n> @@ -1598,32 +1684,80 @@ void PipelineHandlerMaliC55::statsBufferReady(FrameBuffer *buffer)\n>  \t\t\t\t sensorControls);\n>  }\n>\n> +void PipelineHandlerMaliC55::cruBufferReady(FrameBuffer *buffer)\n> +{\n> +\t/*\n> +\t * If the buffer has been cancelled, do not ask the IPA to prepare\n> +\t * parameters.\n> +\t *\n> +\t * The Request this cancelled buffer belongs to will be handled by\n> +\t * imageBufferReady() as we have queued buffers to the ISP capture\n> +\t * devices at the same time we have queued this buffer to the CRU\n> +\t * when running in m2m mode.\n> +\t */\n> +\tif (buffer->metadata().status == FrameMetadata::FrameCancelled)\n> +\t\treturn;\n> +\n> +\tMaliC55FrameInfo *info = findFrameInfo(buffer);\n> +\tASSERT(info);\n> +\n> +\tRequest *request = info->request;\n> +\trequest->_d()->metadata().set(controls::SensorTimestamp,\n> +\t\t\t\t      buffer->metadata().timestamp);\n> +\n> +\t/* Ought we do something with the sensor's controls here...? */\n\nand remove this leftover comment\n\nI'll send a slightly re-freshed series\n\n\n> +\tMaliC55CameraData *data = cameraData(request->_d()->camera());\n> +\tdata->ipa_->fillParams(request->sequence(), info->paramBuffer->cookie());\n> +}\n> +\n>  void PipelineHandlerMaliC55::paramsComputed(unsigned int requestId, uint32_t bytesused)\n>  {\n> -\tMaliC55FrameInfo &frameInfo = frameInfoMap_[requestId];\n> +\tauto it = frameInfoMap_.find(requestId);\n> +\tif (it == frameInfoMap_.end())\n> +\t\treturn;\n> +\n> +\tMaliC55FrameInfo &frameInfo = it->second;\n>  \tRequest *request = frameInfo.request;\n> +\tif (!request)\n> +\t\treturn;\n> +\n>  \tMaliC55CameraData *data = cameraData(request->_d()->camera());\n>\n>  \t/*\n>  \t * Queue buffers for stats and params, then queue buffers to the capture\n> -\t * video devices.\n> +\t * video devices if we're running in Inline mode or with the TPG.\n> +\t *\n> +\t * If we're running in M2M buffers have been queued to the capture\n> +\t * devices at queueRequestToCru() time and here we only have to queue\n> +\t * buffers to the IVC input to start a transfer.\n>  \t */\n>\n>  \tframeInfo.paramBuffer->_d()->metadata().planes()[0].bytesused = bytesused;\n>  \tparams_->queueBuffer(frameInfo.paramBuffer);\n>  \tstats_->queueBuffer(frameInfo.statBuffer);\n>\n> -\tfor (auto &[stream, buffer] : request->buffers()) {\n> -\t\tMaliC55Pipe *pipe = pipeFromStream(data, stream);\n> +\tif (!std::holds_alternative<MaliC55CameraData::Memory>(data->input_)) {\n> +\t\tfor (auto &[stream, buffer] : request->buffers()) {\n> +\t\t\tMaliC55Pipe *pipe = pipeFromStream(data, stream);\n>\n> -\t\tpipe->cap->queueBuffer(buffer);\n> +\t\t\tpipe->cap->queueBuffer(buffer);\n> +\t\t}\n> +\t} else {\n> +\t\tivc_->queueBuffer(frameInfo.rawBuffer);\n> +\t\tframeInfo.rawBuffer = nullptr;\n>  \t}\n>  }\n>\n>  void PipelineHandlerMaliC55::statsProcessed(unsigned int requestId,\n>  \t\t\t\t\t    const ControlList &metadata)\n>  {\n> -\tMaliC55FrameInfo &frameInfo = frameInfoMap_[requestId];\n> +\tauto it = frameInfoMap_.find(requestId);\n> +\tif (it == frameInfoMap_.end())\n> +\t\treturn;\n> +\n> +\tMaliC55FrameInfo &frameInfo = it->second;\n> +\tif (!frameInfo.request)\n> +\t\treturn;\n>\n>  \tframeInfo.statsDone = true;\n>  \tframeInfo.request->_d()->metadata().merge(metadata);\n> @@ -1785,6 +1919,9 @@ bool PipelineHandlerMaliC55::registerMemoryInputCamera(MediaLink *link)\n>\n>  \tivc_->bufferReady.connect(mem->cru_.get(), &RZG2LCRU::returnBuffer);\n>\n> +\tV4L2VideoDevice *cruOutput = mem->cru_->output();\n> +\tcruOutput->bufferReady.connect(this, &PipelineHandlerMaliC55::cruBufferReady);\n> +\n>  \treturn registerMaliCamera(std::move(data), sensor->device()->entity()->name());\n>  }\n>\n>\n> --\n> 2.53.0\n>","headers":{"Return-Path":"<libcamera-devel-bounces@lists.libcamera.org>","X-Original-To":"parsemail@patchwork.libcamera.org","Delivered-To":"parsemail@patchwork.libcamera.org","Received":["from lancelot.ideasonboard.com (lancelot.ideasonboard.com\n\t[92.243.16.209])\n\tby patchwork.libcamera.org (Postfix) with ESMTPS id C7795BEFBE\n\tfor <parsemail@patchwork.libcamera.org>;\n\tWed,  1 Apr 2026 16:18:43 +0000 (UTC)","from lancelot.ideasonboard.com (localhost [IPv6:::1])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTP id 16E2B62D59;\n\tWed,  1 Apr 2026 18:18:43 +0200 (CEST)","from perceval.ideasonboard.com (perceval.ideasonboard.com\n\t[IPv6:2001:4b98:dc2:55:216:3eff:fef7:d647])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id 58C5E62CCA\n\tfor <libcamera-devel@lists.libcamera.org>;\n\tWed,  1 Apr 2026 18:18:41 +0200 (CEST)","from ideasonboard.com (net-93-65-100-155.cust.vodafonedsl.it\n\t[93.65.100.155])\n\tby perceval.ideasonboard.com (Postfix) with ESMTPSA id 119E7CE7;\n\tWed,  1 Apr 2026 18:17:17 +0200 (CEST)"],"Authentication-Results":"lancelot.ideasonboard.com; dkim=pass (1024-bit key;\n\tunprotected) header.d=ideasonboard.com header.i=@ideasonboard.com\n\theader.b=\"NzfmR4QM\"; dkim-atps=neutral","DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/simple; d=ideasonboard.com;\n\ts=mail; t=1775060238;\n\tbh=qh3GLumXoIq+ZGElTI7T079KPbWmG1RwaKWnxsyMCss=;\n\th=Date:From:To:Cc:Subject:References:In-Reply-To:From;\n\tb=NzfmR4QMfSZZJ5T+uRNx91BJWueps09HEkzGnI/zB3sD8ZeJ+S08ygmo2Uf/uSbUq\n\tm3GsAeNkl/2p5GhHncIGwocnsQQ8E89ouScHaSiFo4ZtD1mzq0Q74rbLHd0YvGmom3\n\tRMFcBkg2JG+Pqb9vNUK11+7DHe8JNQJBsApbp0V8=","Date":"Wed, 1 Apr 2026 18:18:38 +0200","From":"Jacopo Mondi <jacopo.mondi@ideasonboard.com>","To":"Jacopo Mondi <jacopo.mondi@ideasonboard.com>","Cc":"Daniel Scally <dan.scally@ideasonboard.com>, \n\tlibcamera-devel@lists.libcamera.org, =?utf-8?b?QmFybmFiw6FzIFDFkWN6?=\n\t=?utf-8?q?e?= <barnabas.pocze@ideasonboard.com>","Subject":"Re: [PATCH v7 7/8] libcamera: mali-c55: Implement capture for\n\tmemory-to-memory","Message-ID":"<ac1FE7JpUMURSby4@zed>","References":"<20260331-mali-cru-v7-0-4caedc898a0e@ideasonboard.com>\n\t<20260331-mali-cru-v7-7-4caedc898a0e@ideasonboard.com>","MIME-Version":"1.0","Content-Type":"text/plain; charset=utf-8","Content-Disposition":"inline","In-Reply-To":"<20260331-mali-cru-v7-7-4caedc898a0e@ideasonboard.com>","X-BeenThere":"libcamera-devel@lists.libcamera.org","X-Mailman-Version":"2.1.29","Precedence":"list","List-Id":"<libcamera-devel.lists.libcamera.org>","List-Unsubscribe":"<https://lists.libcamera.org/options/libcamera-devel>,\n\t<mailto:libcamera-devel-request@lists.libcamera.org?subject=unsubscribe>","List-Archive":"<https://lists.libcamera.org/pipermail/libcamera-devel/>","List-Post":"<mailto:libcamera-devel@lists.libcamera.org>","List-Help":"<mailto:libcamera-devel-request@lists.libcamera.org?subject=help>","List-Subscribe":"<https://lists.libcamera.org/listinfo/libcamera-devel>,\n\t<mailto:libcamera-devel-request@lists.libcamera.org?subject=subscribe>","Errors-To":"libcamera-devel-bounces@lists.libcamera.org","Sender":"\"libcamera-devel\" <libcamera-devel-bounces@lists.libcamera.org>"}}]