From patchwork Thu Jul 24 06:52:55 2025 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Dan Scally X-Patchwork-Id: 23926 Return-Path: X-Original-To: parsemail@patchwork.libcamera.org Delivered-To: parsemail@patchwork.libcamera.org Received: from lancelot.ideasonboard.com (lancelot.ideasonboard.com [92.243.16.209]) by patchwork.libcamera.org (Postfix) with ESMTPS id 5E613C3323 for ; Thu, 24 Jul 2025 06:53:32 +0000 (UTC) Received: from lancelot.ideasonboard.com (localhost [IPv6:::1]) by lancelot.ideasonboard.com (Postfix) with ESMTP id 32B056909F; Thu, 24 Jul 2025 08:53:29 +0200 (CEST) Authentication-Results: lancelot.ideasonboard.com; dkim=pass (1024-bit key; unprotected) header.d=ideasonboard.com header.i=@ideasonboard.com header.b="GdueKGGM"; dkim-atps=neutral Received: from perceval.ideasonboard.com (perceval.ideasonboard.com [213.167.242.64]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id 929B269086 for ; Thu, 24 Jul 2025 08:53:16 +0200 (CEST) Received: from mail.ideasonboard.com (cpc141996-chfd3-2-0-cust928.12-3.cable.virginm.net [86.13.91.161]) by perceval.ideasonboard.com (Postfix) with ESMTPSA id B86ED7F0; Thu, 24 Jul 2025 08:52:37 +0200 (CEST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=ideasonboard.com; s=mail; t=1753339957; bh=Py0ZdPv1jQAnOc+V9n/Y+D6Y4uSbbR6I3rIxip3/9EY=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=GdueKGGMYpnrQ/kj18TpZNM9Hotb+3wqu10YX/31JQj3WanSS2r0WOTOFvgkR6tl0 WsiwkKn1Iv4QK6sXMbq50BFeVLAPlYpVEyeogllYrNMKF5+Mo3i/KjzkeM12JjQUTD qiz8lE9cvllR02RCLCqxIJMHC/NcVNfcXN17/t/U= From: Daniel Scally To: libcamera-devel@lists.libcamera.org Cc: Daniel Scally Subject: [PATCH 09/10] libcamera: mali-c55: Use the RZG2LCRU class Date: Thu, 24 Jul 2025 07:52:55 +0100 Message-Id: <20250724065256.75175-10-dan.scally@ideasonboard.com> X-Mailer: git-send-email 2.34.1 In-Reply-To: <20250724065256.75175-1-dan.scally@ideasonboard.com> References: <20250724065256.75175-1-dan.scally@ideasonboard.com> MIME-Version: 1.0 X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: libcamera-devel-bounces@lists.libcamera.org Sender: "libcamera-devel" Use the RZG2LCRU class to add the ability to the mali-c55 pipeline handler to read frames from memory and process them. The data flow is different to an inline configuration; instead of filling the params buffer immediately on queueRequestDevice(), queued requests are parked in a pending queue until a buffer from the CRU device is available to be queued to that device. Once that buffer is filled with image data by the CRU the parameter buffer is filled and the image buffer queued to the ISP. Signed-off-by: Daniel Scally --- src/libcamera/pipeline/mali-c55/mali-c55.cpp | 208 ++++++++++++++++++- 1 file changed, 201 insertions(+), 7 deletions(-) diff --git a/src/libcamera/pipeline/mali-c55/mali-c55.cpp b/src/libcamera/pipeline/mali-c55/mali-c55.cpp index 5944823d..2a396950 100644 --- a/src/libcamera/pipeline/mali-c55/mali-c55.cpp +++ b/src/libcamera/pipeline/mali-c55/mali-c55.cpp @@ -19,6 +19,7 @@ #include #include +#include #include #include #include @@ -42,6 +43,8 @@ #include "libcamera/internal/v4l2_subdevice.h" #include "libcamera/internal/v4l2_videodevice.h" +#include "rzg2l-cru.h" + namespace { bool isFormatRaw(const libcamera::PixelFormat &pixFmt) @@ -84,6 +87,7 @@ struct MaliC55FrameInfo { FrameBuffer *paramBuffer; FrameBuffer *statBuffer; + FrameBuffer *rawBuffer; bool paramsDone; bool statsDone; @@ -112,13 +116,14 @@ public: PixelFormat adjustRawFormat(const PixelFormat &pixFmt) const; Size adjustRawSizes(const PixelFormat &pixFmt, const Size &rawSize) const; - std::unique_ptr sensor_; + std::shared_ptr sensor_; MediaEntity *entity_; - std::unique_ptr csi_; + std::shared_ptr csi_; std::unique_ptr sd_; Stream frStream_; Stream dsStream_; + std::unique_ptr cru_; std::unique_ptr ipa_; std::vector ipaStatBuffers_; @@ -199,6 +204,9 @@ void MaliC55CameraData::setSensorControls(const ControlList &sensorControls) const std::vector MaliC55CameraData::sizes(unsigned int mbusCode) const { + if (cru_) + return cru_->sizes(mbusCode); + if (sensor_) return sensor_->sizes(mbusCode); @@ -222,6 +230,9 @@ const std::vector MaliC55CameraData::sizes(unsigned int mbusCode) const const Size MaliC55CameraData::resolution() const { + if (cru_) + return cru_->resolution(); + if (sensor_) return sensor_->resolution(); @@ -615,11 +626,14 @@ public: int start(Camera *camera, const ControlList *controls) override; void stopDevice(Camera *camera) override; + void queuePendingRequests(); + void cancelPendingRequests(); int queueRequestDevice(Camera *camera, Request *request) override; void imageBufferReady(FrameBuffer *buffer); void paramsBufferReady(FrameBuffer *buffer); void statsBufferReady(FrameBuffer *buffer); + void cruBufferReady(FrameBuffer *buffer); void paramsComputed(unsigned int requestId, uint32_t bytesused); void statsProcessed(unsigned int requestId, const ControlList &metadata); @@ -683,11 +697,15 @@ private: const std::string &name); bool registerTPGCamera(MediaLink *link); bool registerSensorCamera(MediaLink *link); + bool registerMemoryInputCamera(); MediaDevice *media_; + MediaDevice *cruMedia_; std::unique_ptr isp_; std::unique_ptr stats_; std::unique_ptr params_; + std::unique_ptr ivc_; + std::unique_ptr input_; std::vector> statsBuffers_; std::queue availableStatsBuffers_; @@ -697,6 +715,11 @@ private: std::map frameInfoMap_; + /* Requests for which no buffer has been queued to the CRU device yet. */ + std::queue pendingRequests_; + /* Requests queued to the CRU device but not yet processed by the ISP. */ + std::queue processingRequests_; + std::array pipes_; bool dsFitted_; @@ -929,9 +952,16 @@ int PipelineHandlerMaliC55::configure(Camera *camera, /* Link the graph depending if we are operating the TPG or a sensor. */ MaliC55CameraData *data = cameraData(camera); - if (data->csi_) { + if (data->cru_) { + const MediaEntity *ivcEntity = ivc_->entity(); + ret = ivcEntity->getPadByIndex(1)->links()[0]->setEnabled(true); + if (ret) + return ret; + } else if (data->csi_) { const MediaEntity *csiEntity = data->csi_->entity(); ret = csiEntity->getPadByIndex(1)->links()[0]->setEnabled(true); + if (ret) + return ret; } else { ret = data->entity_->getPadByIndex(0)->links()[0]->setEnabled(true); } @@ -952,14 +982,43 @@ int PipelineHandlerMaliC55::configure(Camera *camera, return ret; } + /* + * This could be a CSI receiver directly connected to the ISP, or else + * one in the CRU's graph. + */ if (data->csi_) { ret = data->csi_->setFormat(0, &subdevFormat); if (ret) return ret; - ret = data->csi_->getFormat(1, &subdevFormat); - if (ret) - return ret; + if (data->cru_) { + V4L2DeviceFormat inputFormat; + + ret = data->cru_->configure(&subdevFormat, &inputFormat); + if (ret) + return ret; + + /* + * The IVC video device needs to be configured with the same + * format as the CRU. + */ + + ret = input_->setFormat(&inputFormat); + if (ret) + return ret; + + ret = ivc_->setFormat(0, &subdevFormat); + if (ret) + return ret; + + ret = ivc_->getFormat(1, &subdevFormat); + if (ret) + return ret; + } else { + ret = data->csi_->getFormat(1, &subdevFormat); + if (ret) + return ret; + } } V4L2DeviceFormat statsFormat; @@ -1120,6 +1179,11 @@ void PipelineHandlerMaliC55::freeBuffers(Camera *camera) if (params_->releaseBuffers()) LOG(MaliC55, Error) << "Failed to release params buffers"; + if (data->cru_) { + if (input_->releaseBuffers()) + LOG(MaliC55, Error) << "Failed to release input buffers"; + } + return; } @@ -1135,6 +1199,12 @@ int PipelineHandlerMaliC55::allocateBuffers(Camera *camera) data->dsStream_.configuration().bufferCount, }); + if (input_) { + ret = input_->importBuffers(RZG2LCRU::kBufferCount); + if (ret < 0) + return ret; + } + ret = stats_->allocateBuffers(bufferCount, &statsBuffers_); if (ret < 0) return ret; @@ -1174,6 +1244,24 @@ int PipelineHandlerMaliC55::start(Camera *camera, [[maybe_unused]] const Control if (ret) return ret; + if (data->cru_) { + ret = data->cru_->start(); + if (ret) { + LOG(MaliC55, Error) + << "Failed to start CRU " << camera->id(); + freeBuffers(camera); + return ret; + } + + ret = input_->streamOn(); + if (ret) { + LOG(MaliC55, Error) + << "Failed to start IVC" << camera->id(); + freeBuffers(camera); + return ret; + } + } + if (data->ipa_) { ret = data->ipa_->start(); if (ret) { @@ -1263,6 +1351,12 @@ void PipelineHandlerMaliC55::stopDevice(Camera *camera) pipe.cap->releaseBuffers(); } + if (data->cru_) { + cancelPendingRequests(); + input_->streamOff(); + data->cru_->stop(); + } + stats_->streamOff(); params_->streamOff(); if (data->ipa_) @@ -1366,10 +1460,80 @@ void PipelineHandlerMaliC55::applyScalerCrop(Camera *camera, } } +void PipelineHandlerMaliC55::cancelPendingRequests() +{ + processingRequests_ = {}; + + while (!pendingRequests_.empty()) { + Request *request = pendingRequests_.front(); + + for (auto it : request->buffers()) { + FrameBuffer *buffer = it.second; + buffer->_d()->cancel(); + completeBuffer(request, buffer); + } + + completeRequest(request); + pendingRequests_.pop(); + } +} + +void PipelineHandlerMaliC55::queuePendingRequests() +{ + while (!pendingRequests_.empty()) { + Request *request = pendingRequests_.front(); + + if (availableStatsBuffers_.empty()) { + LOG(MaliC55, Error) << "Stats buffer underrun"; + return; + } + + if (availableParamsBuffers_.empty()) { + LOG(MaliC55, Error) << "Params buffer underrun"; + return; + } + + MaliC55FrameInfo frameInfo; + frameInfo.request = request; + + MaliC55CameraData *data = cameraData(request->_d()->camera()); + frameInfo.rawBuffer = data->cru_->queueBuffer(request); + if (!frameInfo.rawBuffer) + return; + + frameInfo.statBuffer = availableStatsBuffers_.front(); + availableStatsBuffers_.pop(); + frameInfo.paramBuffer = availableParamsBuffers_.front(); + availableParamsBuffers_.pop(); + + frameInfo.paramsDone = false; + frameInfo.statsDone = false; + + frameInfoMap_[request->sequence()] = frameInfo; + + data->ipa_->queueRequest(request->sequence(), request->controls()); + + pendingRequests_.pop(); + processingRequests_.push(request); + } +} + int PipelineHandlerMaliC55::queueRequestDevice(Camera *camera, Request *request) { MaliC55CameraData *data = cameraData(camera); + /* + * If we're in memory input mode, we need to pop the requests onto the + * pending list until a CRU buffer is ready...otherwise we can just do + * everything immediately. + */ + if (data->cru_) { + pendingRequests_.push(request); + queuePendingRequests(); + + return 0; + } + /* Do not run the IPA if the TPG is in use. */ if (!data->ipa_) { MaliC55FrameInfo frameInfo; @@ -1434,7 +1598,8 @@ MaliC55FrameInfo *PipelineHandlerMaliC55::findFrameInfo(FrameBuffer *buffer) { for (auto &[sequence, info] : frameInfoMap_) { if (info.paramBuffer == buffer || - info.statBuffer == buffer) + info.statBuffer == buffer || + info.rawBuffer == buffer) return &info; } @@ -1496,6 +1661,32 @@ void PipelineHandlerMaliC55::statsBufferReady(FrameBuffer *buffer) sensorControls); } +void PipelineHandlerMaliC55::cruBufferReady(FrameBuffer *buffer) +{ + MaliC55FrameInfo *info = findFrameInfo(buffer); + Request *request = info->request; + ASSERT(info); + + if (buffer->metadata().status == FrameMetadata::FrameCancelled) { + for (auto it : request->buffers()) { + FrameBuffer *b = it.second; + b->_d()->cancel(); + completeBuffer(request, b); + } + + frameInfoMap_.erase(request->sequence()); + completeRequest(request); + return; + } + + request->metadata().set(controls::SensorTimestamp, + buffer->metadata().timestamp); + + /* Ought we do something with the sensor's controls here...? */ + MaliC55CameraData *data = cameraData(request->_d()->camera()); + data->ipa_->fillParams(request->sequence(), info->paramBuffer->cookie()); +} + void PipelineHandlerMaliC55::paramsComputed(unsigned int requestId, uint32_t bytesused) { MaliC55FrameInfo &frameInfo = frameInfoMap_[requestId]; @@ -1516,6 +1707,9 @@ void PipelineHandlerMaliC55::paramsComputed(unsigned int requestId, uint32_t byt pipe->cap->queueBuffer(buffer); } + + if (data->cru_) + input_->queueBuffer(frameInfo.rawBuffer); } void PipelineHandlerMaliC55::statsProcessed(unsigned int requestId,