[{"id":38646,"web_url":"https://patchwork.libcamera.org/comment/38646/","msgid":"<9e3a29de-2e7b-4923-a506-72340bd6ba6d@ideasonboard.com>","date":"2026-04-24T11:38:09","subject":"Re: [PATCH v8 7/8] libcamera: mali-c55: Implement capture for\n\tmemory-to-memory","submitter":{"id":156,"url":"https://patchwork.libcamera.org/api/people/156/","name":"Dan Scally","email":"dan.scally@ideasonboard.com"},"content":"Hi Jacopo\n\nOn 01/04/2026 17:25, Jacopo Mondi wrote:\n> Plumb in the MaliC55 pipeline handler support for capturing frames\n> from memory using the CRU.\n> \n> Introduce a data flow which uses the CRU to feed the ISP through\n> the IVC.\n> \n> In detail:\n> \n> - push incoming request to a pending queue until a buffer from the CRU\n>    is available\n> - delay the call to ipa_->fillParams() to the CRU buffer ready event\n> - once the IPA has computed parameters feed the ISP through the IVC\n>    with buffers from the CRU, params and statistics\n> - Handle completion of in-flight requests: in m2m mode buffers are\n>    queued earlier to the ISP capture devices. If the camera is stopped\n>    these buffers are returned earlier in error state: complete the\n>    Request bypassing the IPA operations\n> - As cancelled Requests might now be completed earlier then IPA events,\n>    modify the IPA event handler to ignore Requests that have been\n>    completed already\n> \n> Signed-off-by: Daniel Scally <dan.scally@ideasonboard.com>\n> Signed-off-by: Jacopo Mondi <jacopo.mondi@ideasonboard.com>\n> ---\n\nReviewed-by: Daniel Scally <dan.scally@ideasonboard.com>\n\n>   src/libcamera/pipeline/mali-c55/mali-c55.cpp | 212 ++++++++++++++++++++++-----\n>   1 file changed, 174 insertions(+), 38 deletions(-)\n> \n> diff --git a/src/libcamera/pipeline/mali-c55/mali-c55.cpp b/src/libcamera/pipeline/mali-c55/mali-c55.cpp\n> index 9df35585fef0..a78ffc21c808 100644\n> --- a/src/libcamera/pipeline/mali-c55/mali-c55.cpp\n> +++ b/src/libcamera/pipeline/mali-c55/mali-c55.cpp\n> @@ -21,6 +21,7 @@\n>   #include <libcamera/base/utils.h>\n>   \n>   #include <libcamera/camera.h>\n> +#include <libcamera/controls.h>\n>   #include <libcamera/formats.h>\n>   #include <libcamera/geometry.h>\n>   #include <libcamera/property_ids.h>\n> @@ -90,6 +91,7 @@ struct MaliC55FrameInfo {\n>   \n>   \tFrameBuffer *paramBuffer;\n>   \tFrameBuffer *statBuffer;\n> +\tFrameBuffer *rawBuffer;\n>   \n>   \tbool paramsDone;\n>   \tbool statsDone;\n> @@ -693,6 +695,7 @@ public:\n>   \tvoid imageBufferReady(FrameBuffer *buffer);\n>   \tvoid paramsBufferReady(FrameBuffer *buffer);\n>   \tvoid statsBufferReady(FrameBuffer *buffer);\n> +\tvoid cruBufferReady(FrameBuffer *buffer);\n>   \tvoid paramsComputed(unsigned int requestId, uint32_t bytesused);\n>   \tvoid statsProcessed(unsigned int requestId, const ControlList &metadata);\n>   \n> @@ -741,7 +744,7 @@ private:\n>   \n>   \tMaliC55FrameInfo *findFrameInfo(FrameBuffer *buffer);\n>   \tMaliC55FrameInfo *findFrameInfo(Request *request);\n> -\tvoid tryComplete(MaliC55FrameInfo *info);\n> +\tvoid tryComplete(MaliC55FrameInfo *info, bool cancelled = false);\n>   \n>   \tint configureRawStream(MaliC55CameraData *data,\n>   \t\t\t       const StreamConfiguration &config,\n> @@ -750,6 +753,9 @@ private:\n>   \t\t\t\t     const StreamConfiguration &config,\n>   \t\t\t\t     V4L2SubdeviceFormat &subdevFormat);\n>   \n> +\tMaliC55FrameInfo *prepareFrameInfo(Request *request);\n> +\tvoid queueRequestToCru(MaliC55CameraData *data, Request *request);\n> +\n>   \tvoid applyScalerCrop(Camera *camera, const ControlList &controls);\n>   \n>   \tbool registerMaliCamera(std::unique_ptr<MaliC55CameraData> data,\n> @@ -1222,6 +1228,13 @@ void PipelineHandlerMaliC55::freeBuffers(Camera *camera)\n>   \tif (params_->releaseBuffers())\n>   \t\tLOG(MaliC55, Error) << \"Failed to release params buffers\";\n>   \n> +\tif (auto *mem = std::get_if<MaliC55CameraData::Memory>(&data->input_)) {\n> +\t\tif (ivc_->releaseBuffers())\n> +\t\t\tLOG(MaliC55, Error) << \"Failed to release input buffers\";\n> +\t\tif (mem->cru_->freeBuffers())\n> +\t\t\tLOG(MaliC55, Error) << \"Failed to release CRU buffers\";\n> +\t}\n> +\n>   \treturn;\n>   }\n>   \n> @@ -1245,6 +1258,12 @@ int PipelineHandlerMaliC55::allocateBuffers(Camera *camera)\n>   \t\t}\n>   \t};\n>   \n> +\tif (std::holds_alternative<MaliC55CameraData::Memory>(data->input_)) {\n> +\t\tret = ivc_->importBuffers(kMaliC55BufferCount);\n> +\t\tif (ret < 0)\n> +\t\t\treturn ret;\n> +\t}\n> +\n>   \tret = stats_->allocateBuffers(kMaliC55BufferCount, &statsBuffers_);\n>   \tif (ret < 0)\n>   \t\treturn ret;\n> @@ -1276,6 +1295,24 @@ int PipelineHandlerMaliC55::start(Camera *camera, [[maybe_unused]] const Control\n>   \tif (ret)\n>   \t\treturn ret;\n>   \n> +\tif (auto *mem = std::get_if<MaliC55CameraData::Memory>(&data->input_)) {\n> +\t\tret = mem->cru_->start(kMaliC55BufferCount);\n> +\t\tif (ret) {\n> +\t\t\tLOG(MaliC55, Error)\n> +\t\t\t\t<< \"Failed to start CRU \" << camera->id();\n> +\t\t\tfreeBuffers(camera);\n> +\t\t\treturn ret;\n> +\t\t}\n> +\n> +\t\tret = ivc_->streamOn();\n> +\t\tif (ret) {\n> +\t\t\tLOG(MaliC55, Error)\n> +\t\t\t\t<< \"Failed to start IVC\" << camera->id();\n> +\t\t\tfreeBuffers(camera);\n> +\t\t\treturn ret;\n> +\t\t}\n> +\t}\n> +\n>   \tif (data->ipa_) {\n>   \t\tret = data->ipa_->start();\n>   \t\tif (ret) {\n> @@ -1357,6 +1394,11 @@ void PipelineHandlerMaliC55::stopDevice(Camera *camera)\n>   \n>   \tisp_->setFrameStartEnabled(false);\n>   \n> +\tif (auto *mem = std::get_if<MaliC55CameraData::Memory>(&data->input_)) {\n> +\t\tivc_->streamOff();\n> +\t\tmem->cru_->stop();\n> +\t}\n> +\n>   \tfor (MaliC55Pipe &pipe : pipes_) {\n>   \t\tif (!pipe.stream)\n>   \t\t\tcontinue;\n> @@ -1468,10 +1510,68 @@ void PipelineHandlerMaliC55::applyScalerCrop(Camera *camera,\n>   \t}\n>   }\n>   \n> +MaliC55FrameInfo *PipelineHandlerMaliC55::prepareFrameInfo(Request *request)\n> +{\n> +\tif (availableStatsBuffers_.empty()) {\n> +\t\tLOG(MaliC55, Error) << \"Stats buffer underrun\";\n> +\t\treturn nullptr;\n> +\t}\n> +\n> +\tif (availableParamsBuffers_.empty()) {\n> +\t\tLOG(MaliC55, Error) << \"Params buffer underrun\";\n> +\t\treturn nullptr;\n> +\t}\n> +\n> +\tMaliC55FrameInfo &frameInfo = frameInfoMap_[request->sequence()];\n> +\tframeInfo.request = request;\n> +\tframeInfo.statBuffer = availableStatsBuffers_.front();\n> +\tavailableStatsBuffers_.pop();\n> +\tframeInfo.paramBuffer = availableParamsBuffers_.front();\n> +\tavailableParamsBuffers_.pop();\n> +\n> +\tframeInfo.paramsDone = false;\n> +\tframeInfo.statsDone = false;\n> +\n> +\treturn &frameInfo;\n> +}\n> +\n> +void PipelineHandlerMaliC55::queueRequestToCru(MaliC55CameraData *data,\n> +\t\t\t\t\t       Request *request)\n> +{\n> +\tauto *mem = std::get_if<MaliC55CameraData::Memory>(&data->input_);\n> +\tASSERT(mem);\n> +\n> +\tFrameBuffer *cruBuffer = mem->cru_->queueBuffer(request);\n> +\tASSERT(cruBuffer);\n> +\n> +\tauto frameInfo = prepareFrameInfo(request);\n> +\tASSERT(frameInfo);\n> +\n> +\tframeInfo->rawBuffer = cruBuffer;\n> +\n> +\tfor (auto &[stream, buffer] : request->buffers()) {\n> +\t\tMaliC55Pipe *pipe = pipeFromStream(data, stream);\n> +\n> +\t\tpipe->cap->queueBuffer(buffer);\n> +\t}\n> +\n> +\tdata->ipa_->queueRequest(request->sequence(), request->controls());\n> +}\n> +\n>   int PipelineHandlerMaliC55::queueRequestDevice(Camera *camera, Request *request)\n>   {\n>   \tMaliC55CameraData *data = cameraData(camera);\n>   \n> +\t/*\n> +\t * If we're in memory input mode, we need to queue the Request to the\n> +\t * CRU, otherwise we can just do everything immediately.\n> +\t */\n> +\tif (std::holds_alternative<MaliC55CameraData::Memory>(data->input_)) {\n> +\t\tqueueRequestToCru(data, request);\n> +\n> +\t\treturn 0;\n> +\t}\n> +\n>   \t/* Do not run the IPA if the TPG is in use. */\n>   \tif (!data->ipa_) {\n>   \t\tMaliC55FrameInfo frameInfo;\n> @@ -1492,32 +1592,12 @@ int PipelineHandlerMaliC55::queueRequestDevice(Camera *camera, Request *request)\n>   \t\treturn 0;\n>   \t}\n>   \n> -\tif (availableStatsBuffers_.empty()) {\n> -\t\tLOG(MaliC55, Error) << \"Stats buffer underrun\";\n> -\t\treturn -ENOENT;\n> -\t}\n> -\n> -\tif (availableParamsBuffers_.empty()) {\n> -\t\tLOG(MaliC55, Error) << \"Params buffer underrun\";\n> -\t\treturn -ENOENT;\n> -\t}\n> -\n> -\tMaliC55FrameInfo frameInfo;\n> -\tframeInfo.request = request;\n> -\n> -\tframeInfo.statBuffer = availableStatsBuffers_.front();\n> -\tavailableStatsBuffers_.pop();\n> -\tframeInfo.paramBuffer = availableParamsBuffers_.front();\n> -\tavailableParamsBuffers_.pop();\n> -\n> -\tframeInfo.paramsDone = false;\n> -\tframeInfo.statsDone = false;\n> -\n> -\tframeInfoMap_[request->sequence()] = frameInfo;\n> +\tauto frameInfo = prepareFrameInfo(request);\n> +\tASSERT(frameInfo);\n>   \n>   \tdata->ipa_->queueRequest(request->sequence(), request->controls());\n>   \tdata->ipa_->fillParams(request->sequence(),\n> -\t\t\t       frameInfo.paramBuffer->cookie());\n> +\t\t\t       frameInfo->paramBuffer->cookie());\n>   \n>   \treturn 0;\n>   }\n> @@ -1536,18 +1616,21 @@ MaliC55FrameInfo *PipelineHandlerMaliC55::findFrameInfo(FrameBuffer *buffer)\n>   {\n>   \tfor (auto &[sequence, info] : frameInfoMap_) {\n>   \t\tif (info.paramBuffer == buffer ||\n> -\t\t    info.statBuffer == buffer)\n> +\t\t    info.statBuffer == buffer ||\n> +\t\t    info.rawBuffer == buffer)\n>   \t\t\treturn &info;\n>   \t}\n>   \n>   \treturn nullptr;\n>   }\n>   \n> -void PipelineHandlerMaliC55::tryComplete(MaliC55FrameInfo *info)\n> +void PipelineHandlerMaliC55::tryComplete(MaliC55FrameInfo *info, bool cancelled)\n>   {\n> -\tif (!info->paramsDone)\n> -\t\treturn;\n> -\tif (!info->statsDone)\n> +\t/*\n> +\t * If the buffer has been cancelled, we complete the request without\n> +\t * waiting for the IPA.\n> +\t */\n> +\tif (!cancelled && (!info->paramsDone || !info->statsDone))\n>   \t\treturn;\n>   \n>   \tRequest *request = info->request;\n> @@ -1571,13 +1654,15 @@ void PipelineHandlerMaliC55::imageBufferReady(FrameBuffer *buffer)\n>   \tASSERT(info);\n>   \n>   \tif (completeBuffer(request, buffer))\n> -\t\ttryComplete(info);\n> +\t\ttryComplete(info,\n> +\t\t\t    buffer->metadata().status == FrameMetadata::FrameCancelled);\n>   }\n>   \n>   void PipelineHandlerMaliC55::paramsBufferReady(FrameBuffer *buffer)\n>   {\n>   \tMaliC55FrameInfo *info = findFrameInfo(buffer);\n> -\tASSERT(info);\n> +\tif (!info)\n> +\t\treturn;\n>   \n>   \tinfo->paramsDone = true;\n>   \n> @@ -1587,7 +1672,8 @@ void PipelineHandlerMaliC55::paramsBufferReady(FrameBuffer *buffer)\n>   void PipelineHandlerMaliC55::statsBufferReady(FrameBuffer *buffer)\n>   {\n>   \tMaliC55FrameInfo *info = findFrameInfo(buffer);\n> -\tASSERT(info);\n> +\tif (!info)\n> +\t\treturn;\n>   \n>   \tRequest *request = info->request;\n>   \tMaliC55CameraData *data = cameraData(request->_d()->camera());\n> @@ -1598,32 +1684,79 @@ void PipelineHandlerMaliC55::statsBufferReady(FrameBuffer *buffer)\n>   \t\t\t\t sensorControls);\n>   }\n>   \n> +void PipelineHandlerMaliC55::cruBufferReady(FrameBuffer *buffer)\n> +{\n> +\t/*\n> +\t * If the buffer has been cancelled, do not ask the IPA to prepare\n> +\t * parameters.\n> +\t *\n> +\t * The Request this cancelled buffer belongs to will be handled by\n> +\t * imageBufferReady() as we have queued buffers to the ISP capture\n> +\t * devices at the same time we have queued this buffer to the CRU\n> +\t * when running in m2m mode.\n> +\t */\n> +\tif (buffer->metadata().status == FrameMetadata::FrameCancelled)\n> +\t\treturn;\n> +\n> +\tMaliC55FrameInfo *info = findFrameInfo(buffer);\n> +\tASSERT(info);\n> +\n> +\tRequest *request = info->request;\n> +\trequest->_d()->metadata().set(controls::SensorTimestamp,\n> +\t\t\t\t      buffer->metadata().timestamp);\n> +\n> +\tMaliC55CameraData *data = cameraData(request->_d()->camera());\n> +\tdata->ipa_->fillParams(request->sequence(), info->paramBuffer->cookie());\n> +}\n> +\n>   void PipelineHandlerMaliC55::paramsComputed(unsigned int requestId, uint32_t bytesused)\n>   {\n> -\tMaliC55FrameInfo &frameInfo = frameInfoMap_[requestId];\n> +\tauto it = frameInfoMap_.find(requestId);\n> +\tif (it == frameInfoMap_.end())\n> +\t\treturn;\n> +\n> +\tMaliC55FrameInfo &frameInfo = it->second;\n>   \tRequest *request = frameInfo.request;\n> +\tif (!request)\n> +\t\treturn;\n> +\n>   \tMaliC55CameraData *data = cameraData(request->_d()->camera());\n>   \n>   \t/*\n>   \t * Queue buffers for stats and params, then queue buffers to the capture\n> -\t * video devices.\n> +\t * video devices if we're running in Inline mode or with the TPG.\n> +\t *\n> +\t * If we're running in M2M buffers have been queued to the capture\n> +\t * devices at queueRequestToCru() time and here we only have to queue\n> +\t * buffers to the IVC input to start a transfer.\n>   \t */\n>   \n>   \tframeInfo.paramBuffer->_d()->metadata().planes()[0].bytesused = bytesused;\n>   \tparams_->queueBuffer(frameInfo.paramBuffer);\n>   \tstats_->queueBuffer(frameInfo.statBuffer);\n>   \n> -\tfor (auto &[stream, buffer] : request->buffers()) {\n> -\t\tMaliC55Pipe *pipe = pipeFromStream(data, stream);\n> +\tif (!std::holds_alternative<MaliC55CameraData::Memory>(data->input_)) {\n> +\t\tfor (auto &[stream, buffer] : request->buffers()) {\n> +\t\t\tMaliC55Pipe *pipe = pipeFromStream(data, stream);\n>   \n> -\t\tpipe->cap->queueBuffer(buffer);\n> +\t\t\tpipe->cap->queueBuffer(buffer);\n> +\t\t}\n> +\t} else {\n> +\t\tivc_->queueBuffer(frameInfo.rawBuffer);\n> +\t\tframeInfo.rawBuffer = nullptr;\n>   \t}\n>   }\n>   \n>   void PipelineHandlerMaliC55::statsProcessed(unsigned int requestId,\n>   \t\t\t\t\t    const ControlList &metadata)\n>   {\n> -\tMaliC55FrameInfo &frameInfo = frameInfoMap_[requestId];\n> +\tauto it = frameInfoMap_.find(requestId);\n> +\tif (it == frameInfoMap_.end())\n> +\t\treturn;\n> +\n> +\tMaliC55FrameInfo &frameInfo = it->second;\n> +\tif (!frameInfo.request)\n> +\t\treturn;\n>   \n>   \tframeInfo.statsDone = true;\n>   \tframeInfo.request->_d()->metadata().merge(metadata);\n> @@ -1785,6 +1918,9 @@ bool PipelineHandlerMaliC55::registerMemoryInputCamera(MediaLink *link)\n>   \n>   \tivc_->bufferReady.connect(mem->cru_.get(), &RZG2LCRU::returnBuffer);\n>   \n> +\tV4L2VideoDevice *cruOutput = mem->cru_->output();\n> +\tcruOutput->bufferReady.connect(this, &PipelineHandlerMaliC55::cruBufferReady);\n> +\n>   \treturn registerMaliCamera(std::move(data), sensor->device()->entity()->name());\n>   }\n>   \n>","headers":{"Return-Path":"<libcamera-devel-bounces@lists.libcamera.org>","X-Original-To":"parsemail@patchwork.libcamera.org","Delivered-To":"parsemail@patchwork.libcamera.org","Received":["from lancelot.ideasonboard.com (lancelot.ideasonboard.com\n\t[92.243.16.209])\n\tby patchwork.libcamera.org (Postfix) with ESMTPS id 4D428BE173\n\tfor <parsemail@patchwork.libcamera.org>;\n\tFri, 24 Apr 2026 11:38:16 +0000 (UTC)","from lancelot.ideasonboard.com (localhost [IPv6:::1])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTP id 5E5E362F7A;\n\tFri, 24 Apr 2026 13:38:15 +0200 (CEST)","from perceval.ideasonboard.com (perceval.ideasonboard.com\n\t[213.167.242.64])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id 6A9B662E6A\n\tfor <libcamera-devel@lists.libcamera.org>;\n\tFri, 24 Apr 2026 13:38:13 +0200 (CEST)","from [192.168.0.43]\n\t(chfd-03-b2-v4wan-176392-cust229.vm15.cable.virginm.net\n\t[82.19.20.230])\n\tby perceval.ideasonboard.com (Postfix) with ESMTPSA id 3E9CC103F;\n\tFri, 24 Apr 2026 13:36:33 +0200 (CEST)"],"Authentication-Results":"lancelot.ideasonboard.com; dkim=pass (1024-bit key;\n\tunprotected) header.d=ideasonboard.com header.i=@ideasonboard.com\n\theader.b=\"IpV1V7U5\"; dkim-atps=neutral","DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/simple; d=ideasonboard.com;\n\ts=mail; t=1777030593;\n\tbh=JcRbLj95UPI6r8liwB780F7LtuvPH1sOzU/hWdQatKA=;\n\th=Date:Subject:To:Cc:References:From:In-Reply-To:From;\n\tb=IpV1V7U5/2NOPsc784q6rd5n4XT5m8C+//6/I28Q7is3KaP99rHmXWwyWrOim8xW/\n\tYaeJNeA9bnLEwvetzCzOG9DhmUbvJZiaBqeVu4htODjO64bhwzbWm+HMsflhrV9I9z\n\tqyrKoettVVjgpx077eD95Lzb+ifkZjC4m4Q3qUPc=","Message-ID":"<9e3a29de-2e7b-4923-a506-72340bd6ba6d@ideasonboard.com>","Date":"Fri, 24 Apr 2026 12:38:09 +0100","MIME-Version":"1.0","User-Agent":"Mozilla Thunderbird","Subject":"Re: [PATCH v8 7/8] libcamera: mali-c55: Implement capture for\n\tmemory-to-memory","To":"Jacopo Mondi <jacopo.mondi@ideasonboard.com>,\n\tlibcamera-devel@lists.libcamera.org","Cc":"=?utf-8?q?Barnab=C3=A1s_P=C5=91cze?= <barnabas.pocze@ideasonboard.com>","References":"<20260401-mali-cru-v8-0-44c48f990e28@ideasonboard.com>\n\t<20260401-mali-cru-v8-7-44c48f990e28@ideasonboard.com>","Content-Language":"en-US","From":"Dan Scally <dan.scally@ideasonboard.com>","In-Reply-To":"<20260401-mali-cru-v8-7-44c48f990e28@ideasonboard.com>","Content-Type":"text/plain; charset=UTF-8; format=flowed","Content-Transfer-Encoding":"7bit","X-BeenThere":"libcamera-devel@lists.libcamera.org","X-Mailman-Version":"2.1.29","Precedence":"list","List-Id":"<libcamera-devel.lists.libcamera.org>","List-Unsubscribe":"<https://lists.libcamera.org/options/libcamera-devel>,\n\t<mailto:libcamera-devel-request@lists.libcamera.org?subject=unsubscribe>","List-Archive":"<https://lists.libcamera.org/pipermail/libcamera-devel/>","List-Post":"<mailto:libcamera-devel@lists.libcamera.org>","List-Help":"<mailto:libcamera-devel-request@lists.libcamera.org?subject=help>","List-Subscribe":"<https://lists.libcamera.org/listinfo/libcamera-devel>,\n\t<mailto:libcamera-devel-request@lists.libcamera.org?subject=subscribe>","Errors-To":"libcamera-devel-bounces@lists.libcamera.org","Sender":"\"libcamera-devel\" <libcamera-devel-bounces@lists.libcamera.org>"}}]