From patchwork Thu Jun 13 13:25:57 2024 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Dan Scally X-Patchwork-Id: 20293 Return-Path: X-Original-To: parsemail@patchwork.libcamera.org Delivered-To: parsemail@patchwork.libcamera.org Received: from lancelot.ideasonboard.com (lancelot.ideasonboard.com [92.243.16.209]) by patchwork.libcamera.org (Postfix) with ESMTPS id 21CAAC3237 for ; Thu, 13 Jun 2024 13:26:47 +0000 (UTC) Received: from lancelot.ideasonboard.com (localhost [IPv6:::1]) by lancelot.ideasonboard.com (Postfix) with ESMTP id 3E6A3654B1; Thu, 13 Jun 2024 15:26:41 +0200 (CEST) Authentication-Results: lancelot.ideasonboard.com; dkim=pass (1024-bit key; unprotected) header.d=ideasonboard.com header.i=@ideasonboard.com header.b="f3d4YCRK"; dkim-atps=neutral Received: from perceval.ideasonboard.com (perceval.ideasonboard.com [IPv6:2001:4b98:dc2:55:216:3eff:fef7:d647]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id 25A1565498 for ; Thu, 13 Jun 2024 15:26:30 +0200 (CEST) Received: from mail.ideasonboard.com (cpc141996-chfd3-2-0-cust928.12-3.cable.virginm.net [86.13.91.161]) by perceval.ideasonboard.com (Postfix) with ESMTPSA id 0B8901B37; Thu, 13 Jun 2024 15:26:16 +0200 (CEST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=ideasonboard.com; s=mail; t=1718285176; bh=Qpwi7XM4UqxNYe30zFHw7zy39FdUZepnTI9mwgHTGec=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=f3d4YCRKWVbxlf3dNVopFnQJXUnt5V/pm+T3uerXp3IBf2Y7tICcqhcdyZ/itR50x H6VlsntR/bchSmiy0/aF7MZZItHuaoUv4IrsNp+raR1sLAAwB10Z6Jce3rEpSHwk6O 0UcZ8UOeR8MVBoIFoGRE8Il2HaTeP7oUSC7Jyyh4= From: Daniel Scally To: libcamera-devel@lists.libcamera.org Cc: dan.scally@ideasonboard.com, nayden.kanchev@arm.com, jacopo.mondi@ideasonboard.com Subject: [PATCH 05/10] mali-c55: Plumb the IPA module in Date: Thu, 13 Jun 2024 14:25:57 +0100 Message-Id: <20240613132602.1021721-6-dan.scally@ideasonboard.com> X-Mailer: git-send-email 2.34.1 In-Reply-To: <20240613132602.1021721-1-dan.scally@ideasonboard.com> References: <20240613132602.1021721-1-dan.scally@ideasonboard.com> MIME-Version: 1.0 X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: libcamera-devel-bounces@lists.libcamera.org Sender: "libcamera-devel" From: Jacopo Mondi Plumb the Pipeline-IPA loop in. Load the IPA module at camera creation time and create the loop between the pipeline and the IPA. When a new Request is queued the IPA is asked to prepare the parameters buffer, once ready it notifies the pipeline which queues the parameters to the ISP along with a buffer for statistics and frames, Once statistics are ready they get passed to the IPA which upates its settings for the next frame. Acked-by: Nayden Kanchev Signed-off-by: Jacopo Mondi --- src/libcamera/pipeline/mali-c55/mali-c55.cpp | 353 +++++++++++++++++-- 1 file changed, 314 insertions(+), 39 deletions(-) diff --git a/src/libcamera/pipeline/mali-c55/mali-c55.cpp b/src/libcamera/pipeline/mali-c55/mali-c55.cpp index b9be1207..871674cb 100644 --- a/src/libcamera/pipeline/mali-c55/mali-c55.cpp +++ b/src/libcamera/pipeline/mali-c55/mali-c55.cpp @@ -23,13 +23,19 @@ #include #include +#include +#include + #include "libcamera/internal/bayer_format.h" #include "libcamera/internal/camera.h" #include "libcamera/internal/camera_sensor.h" +#include "libcamera/internal/delayed_controls.h" #include "libcamera/internal/device_enumerator.h" #include "libcamera/internal/framebuffer.h" +#include "libcamera/internal/ipa_manager.h" #include "libcamera/internal/media_device.h" #include "libcamera/internal/pipeline_handler.h" +#include "libcamera/internal/request.h" #include "libcamera/internal/v4l2_subdevice.h" #include "libcamera/internal/v4l2_videodevice.h" @@ -85,6 +91,16 @@ constexpr Size kMaliC55MinSize = { 128, 128 }; constexpr Size kMaliC55MaxSize = { 8192, 8192 }; constexpr unsigned int kMaliC55ISPInternalFormat = MEDIA_BUS_FMT_RGB121212_1X36; +struct MaliC55FrameInfo { + Request *request; + + FrameBuffer *paramBuffer; + FrameBuffer *statBuffer; + + bool paramsDone; + bool statsDone; +}; + class MaliC55CameraData : public Camera::Private { public: @@ -94,6 +110,7 @@ public: } int init(); + int loadIPA(); /* Deflect these functionalities to either TPG or CameraSensor. */ const std::vector mbusCodes() const; @@ -102,7 +119,7 @@ public: PixelFormat bestRawFormat() const; - void updateControls(); + void updateControls(const ControlInfoMap &ipaControls); PixelFormat adjustRawFormat(const PixelFormat &pixFmt) const; Size adjustRawSizes(const PixelFormat &pixFmt, const Size &rawSize) const; @@ -115,8 +132,15 @@ public: Stream frStream_; Stream dsStream_; + std::unique_ptr ipa_; + std::vector ipaStatBuffers_; + std::vector ipaParamBuffers_; + + std::unique_ptr delayedCtrls_; + private: void initTPGData(); + void setSensorControls(const ControlList &sensorControls); std::string id_; std::vector tpgCodes_; @@ -181,6 +205,11 @@ void MaliC55CameraData::initTPGData() tpgResolution_ = tpgSizes_.back(); } +void MaliC55CameraData::setSensorControls(const ControlList &sensorControls) +{ + delayedCtrls_->push(sensorControls); +} + const std::vector MaliC55CameraData::mbusCodes() const { if (sensor_) @@ -249,7 +278,7 @@ PixelFormat MaliC55CameraData::bestRawFormat() const return rawFormat; } -void MaliC55CameraData::updateControls() +void MaliC55CameraData::updateControls(const ControlInfoMap &ipaControls) { if (!sensor_) return; @@ -267,6 +296,9 @@ void MaliC55CameraData::updateControls() ControlInfo(ispMinCrop, sensorInfo.analogCrop, sensorInfo.analogCrop); + for (auto const &c : ipaControls) + controls.emplace(c.first, c.second); + controlInfo_ = ControlInfoMap(std::move(controls), controls::controls); } @@ -321,6 +353,46 @@ Size MaliC55CameraData::adjustRawSizes(const PixelFormat &rawFmt, const Size &ra return bestSize; } +int MaliC55CameraData::loadIPA() +{ + int ret; + + ipa_ = IPAManager::createIPA(pipe(), 1, 1); + if (!ipa_) + return -ENOENT; + + ipa_->setSensorControls.connect(this, &MaliC55CameraData::setSensorControls); + + /* Do not initialize IPA for TPG. */ + if (!sensor_) + return 0; + + std::string ipaTuningFile = ipa_->configurationFile(sensor_->model() + ".yaml"); + if (ipaTuningFile.empty()) + ipaTuningFile = ipa_->configurationFile("uncalibrated.yaml"); + + /* We need to inform the IPA of the sensor configuration */ + ipa::mali_c55::IPAConfigInfo ipaConfig{}; + + ret = sensor_->sensorInfo(&ipaConfig.sensorInfo); + if (ret) + return ret; + + ipaConfig.sensorControls = sensor_->controls(); + + ControlInfoMap ipaControls; + ret = ipa_->init({ ipaTuningFile, sensor_->model() }, ipaConfig, + &ipaControls); + if (ret) { + LOG(MaliC55, Error) << "Failed to initialise the Mali-C55 IPA"; + return ret; + } + + updateControls(ipaControls); + + return 0; +} + class MaliC55CameraConfiguration : public CameraConfiguration { public: @@ -330,6 +402,7 @@ public: } Status validate() override; + const Transform &combinedTransform() { return combinedTransform_; } V4L2SubdeviceFormat sensorFormat_; @@ -337,6 +410,7 @@ private: static constexpr unsigned int kMaxStreams = 2; const MaliC55CameraData *data_; + Transform combinedTransform_; }; CameraConfiguration::Status MaliC55CameraConfiguration::validate() @@ -346,6 +420,11 @@ CameraConfiguration::Status MaliC55CameraConfiguration::validate() if (config_.empty()) return Invalid; + Orientation requestedOrientation = orientation; + combinedTransform_ = data_->sensor_->computeTransform(&orientation); + if (orientation != requestedOrientation) + status = Adjusted; + /* Only 2 streams available. */ if (config_.size() > kMaxStreams) { config_.resize(kMaxStreams); @@ -492,7 +571,10 @@ public: int queueRequestDevice(Camera *camera, Request *request) override; void bufferReady(FrameBuffer *buffer); + void paramsBufferReady(FrameBuffer *buffer); void statsBufferReady(FrameBuffer *buffer); + void paramsComputed(unsigned int requestId); + void statsProcessed(unsigned int requestId, const ControlList &metadata); bool match(DeviceEnumerator *enumerator) override; @@ -536,6 +618,10 @@ private: pipe.stream = nullptr; } + MaliC55FrameInfo *findFrameInfo(FrameBuffer *buffer); + MaliC55FrameInfo *findFrameInfo(Request *request); + void tryComplete(MaliC55FrameInfo *info); + int configureRawStream(MaliC55CameraData *data, const StreamConfiguration &config, V4L2SubdeviceFormat &subdevFormat); @@ -545,7 +631,7 @@ private: void applyScalerCrop(Camera *camera, const ControlList &controls); - void registerMaliCamera(std::unique_ptr data, + bool registerMaliCamera(std::unique_ptr data, const std::string &name); bool registerTPGCamera(MediaLink *link); bool registerSensorCamera(MediaLink *link); @@ -561,6 +647,8 @@ private: std::vector> paramsBuffers_; std::queue availableParamsBuffers_; + std::map frameInfoMap_; + std::array pipes_; bool dsFitted_; @@ -806,6 +894,11 @@ int PipelineHandlerMaliC55::configure(Camera *camera, if (ret) return ret; + ret = data->sensor_->setFormat(&subdevFormat, + maliConfig->combinedTransform()); + if (ret) + return ret; + if (data->csi_) { ret = data->csi_->setFormat(0, &subdevFormat); if (ret) @@ -876,7 +969,31 @@ int PipelineHandlerMaliC55::configure(Camera *camera, pipe->stream = stream; } - data->updateControls(); + /* We need to inform the IPA of the sensor configuration */ + ipa::mali_c55::IPAConfigInfo ipaConfig{}; + + ret = data->sensor_->sensorInfo(&ipaConfig.sensorInfo); + if (ret) + return ret; + + ipaConfig.sensorControls = data->sensor_->controls(); + + /* + * And we also need to tell the IPA the bayerOrder of the data (as + * affected by any flips that we've configured) + */ + const Transform &combinedTransform = maliConfig->combinedTransform(); + BayerFormat::Order bayerOrder = data->sensor_->bayerOrder(combinedTransform); + + ControlInfoMap ipaControls; + ret = data->ipa_->configure(ipaConfig, utils::to_underlying(bayerOrder), + &ipaControls); + if (ret) { + LOG(MaliC55, Error) << "Failed to configure IPA"; + return ret; + } + + data->updateControls(ipaControls); return 0; } @@ -890,8 +1007,10 @@ int PipelineHandlerMaliC55::exportFrameBuffers(Camera *camera, Stream *stream, return pipe->cap->exportBuffers(count, buffers); } -void PipelineHandlerMaliC55::freeBuffers([[maybe_unused]] Camera *camera) +void PipelineHandlerMaliC55::freeBuffers(Camera *camera) { + MaliC55CameraData *data = cameraData(camera); + while (!availableStatsBuffers_.empty()) availableStatsBuffers_.pop(); while (!availableParamsBuffers_.empty()) @@ -900,11 +1019,16 @@ void PipelineHandlerMaliC55::freeBuffers([[maybe_unused]] Camera *camera) statsBuffers_.clear(); paramsBuffers_.clear(); + data->ipa_->unmapBuffers(data->ipaStatBuffers_); + data->ipaStatBuffers_.clear(); + data->ipa_->unmapBuffers(data->ipaParamBuffers_); + data->ipaParamBuffers_.clear(); + if (stats_->releaseBuffers()) LOG(MaliC55, Error) << "Failed to release stats buffers"; if (params_->releaseBuffers()) - LOG(MaliC55, Error) << "Failed to release stats buffers"; + LOG(MaliC55, Error) << "Failed to release params buffers"; return; } @@ -912,6 +1036,7 @@ void PipelineHandlerMaliC55::freeBuffers([[maybe_unused]] Camera *camera) int PipelineHandlerMaliC55::allocateBuffers(Camera *camera) { MaliC55CameraData *data = cameraData(camera); + unsigned int ipaBufferId = 1; unsigned int bufferCount; int ret; @@ -924,27 +1049,47 @@ int PipelineHandlerMaliC55::allocateBuffers(Camera *camera) if (ret < 0) return ret; - for (std::unique_ptr &buffer : statsBuffers_) + for (std::unique_ptr &buffer : statsBuffers_) { + buffer->setCookie(ipaBufferId++); + data->ipaStatBuffers_.emplace_back(buffer->cookie(), + buffer->planes()); availableStatsBuffers_.push(buffer.get()); + } + + data->ipa_->mapBuffers(data->ipaStatBuffers_, true); ret = params_->allocateBuffers(bufferCount, ¶msBuffers_); if (ret < 0) return ret; - for (std::unique_ptr &buffer : paramsBuffers_) + for (std::unique_ptr &buffer : paramsBuffers_) { + buffer->setCookie(ipaBufferId++); + data->ipaParamBuffers_.emplace_back(buffer->cookie(), + buffer->planes()); availableParamsBuffers_.push(buffer.get()); + } + + data->ipa_->mapBuffers(data->ipaParamBuffers_, false); return 0; } int PipelineHandlerMaliC55::start(Camera *camera, [[maybe_unused]] const ControlList *controls) { + MaliC55CameraData *data = cameraData(camera); int ret; ret = allocateBuffers(camera); if (ret) return ret; + ret = data->ipa_->start(); + if (ret) { + LOG(MaliC55, Error) << "Failed to start IPA" << camera->id(); + freeBuffers(camera); + return ret; + } + for (MaliC55Pipe &pipe : pipes_) { if (!pipe.stream) continue; @@ -954,6 +1099,7 @@ int PipelineHandlerMaliC55::start(Camera *camera, [[maybe_unused]] const Control ret = pipe.cap->importBuffers(stream->configuration().bufferCount); if (ret) { LOG(MaliC55, Error) << "Failed to import buffers"; + data->ipa_->stop(); freeBuffers(camera); return ret; } @@ -961,6 +1107,7 @@ int PipelineHandlerMaliC55::start(Camera *camera, [[maybe_unused]] const Control ret = pipe.cap->streamOn(); if (ret) { LOG(MaliC55, Error) << "Failed to start stream"; + data->ipa_->stop(); freeBuffers(camera); return ret; } @@ -970,6 +1117,8 @@ int PipelineHandlerMaliC55::start(Camera *camera, [[maybe_unused]] const Control if (ret) { LOG(MaliC55, Error) << "Failed to start stats stream"; + data->ipa_->stop(); + for (MaliC55Pipe &pipe : pipes_) { if (pipe.stream) pipe.cap->streamOff(); @@ -984,6 +1133,7 @@ int PipelineHandlerMaliC55::start(Camera *camera, [[maybe_unused]] const Control LOG(MaliC55, Error) << "Failed to start params stream"; stats_->streamOff(); + data->ipa_->stop(); for (MaliC55Pipe &pipe : pipes_) { if (pipe.stream) @@ -994,11 +1144,19 @@ int PipelineHandlerMaliC55::start(Camera *camera, [[maybe_unused]] const Control return ret; } + ret = isp_->setFrameStartEnabled(true); + if (ret) + LOG(MaliC55, Error) << "Failed to enable frame start events"; + return 0; } -void PipelineHandlerMaliC55::stopDevice([[maybe_unused]] Camera *camera) +void PipelineHandlerMaliC55::stopDevice(Camera *camera) { + MaliC55CameraData *data = cameraData(camera); + + isp_->setFrameStartEnabled(false); + for (MaliC55Pipe &pipe : pipes_) { if (!pipe.stream) continue; @@ -1009,6 +1167,7 @@ void PipelineHandlerMaliC55::stopDevice([[maybe_unused]] Camera *camera) stats_->streamOff(); params_->streamOff(); + data->ipa_->stop(); freeBuffers(camera); } @@ -1112,30 +1271,16 @@ void PipelineHandlerMaliC55::applyScalerCrop(Camera *camera, int PipelineHandlerMaliC55::queueRequestDevice(Camera *camera, Request *request) { - FrameBuffer *statsBuffer; - int ret; + MaliC55CameraData *data = cameraData(camera); if (availableStatsBuffers_.empty()) { LOG(MaliC55, Error) << "Stats buffer underrun"; return -ENOENT; } - statsBuffer = availableStatsBuffers_.front(); - availableStatsBuffers_.pop(); - - /* - * We need to associate the Request to this buffer even though it's a - * purely internal one because we will need to use request->sequence() - * later. - */ - statsBuffer->_d()->setRequest(request); - - for (auto &[stream, buffer] : request->buffers()) { - MaliC55Pipe *pipe = pipeFromStream(cameraData(camera), stream); - - ret = pipe->cap->queueBuffer(buffer); - if (ret) - return ret; + if (availableParamsBuffers_.empty()) { + LOG(MaliC55, Error) << "Params buffer underrun"; + return -ENOENT; } /* @@ -1147,29 +1292,143 @@ int PipelineHandlerMaliC55::queueRequestDevice(Camera *camera, Request *request) */ applyScalerCrop(camera, request->controls()); - ret = stats_->queueBuffer(statsBuffer); - if (ret) - return ret; + MaliC55FrameInfo frameInfo; + frameInfo.request = request; + + frameInfo.statBuffer = availableStatsBuffers_.front(); + availableStatsBuffers_.pop(); + frameInfo.paramBuffer = availableParamsBuffers_.front(); + availableParamsBuffers_.pop(); + + frameInfo.paramsDone = false; + frameInfo.statsDone = false; + + frameInfoMap_[request->sequence()] = frameInfo; + + data->ipa_->queueRequest(request->sequence(), request->controls()); + data->ipa_->fillParams(request->sequence(), + frameInfo.paramBuffer->cookie()); return 0; } +MaliC55FrameInfo *PipelineHandlerMaliC55::findFrameInfo(Request *request) +{ + for (auto &[sequence, info] : frameInfoMap_) { + if (info.request == request) + return &info; + } + + return nullptr; +} + +MaliC55FrameInfo *PipelineHandlerMaliC55::findFrameInfo(FrameBuffer *buffer) +{ + for (auto &[sequence, info] : frameInfoMap_) { + if (info.paramBuffer == buffer || + info.statBuffer == buffer) + return &info; + } + + return nullptr; +} + +void PipelineHandlerMaliC55::tryComplete(MaliC55FrameInfo *info) +{ + if (!info->paramsDone) + return; + if (!info->statsDone) + return; + + Request *request = info->request; + if (request->hasPendingBuffers()) + return; + + availableStatsBuffers_.push(info->statBuffer); + availableParamsBuffers_.push(info->paramBuffer); + + frameInfoMap_.erase(request->sequence()); + + completeRequest(request); +} + void PipelineHandlerMaliC55::bufferReady(FrameBuffer *buffer) { Request *request = buffer->request(); + MaliC55FrameInfo *info = findFrameInfo(request); + ASSERT(info); if (completeBuffer(request, buffer)) - completeRequest(request); + tryComplete(info); +} + +void PipelineHandlerMaliC55::paramsBufferReady(FrameBuffer *buffer) +{ + MaliC55FrameInfo *info = findFrameInfo(buffer); + ASSERT(info); + + info->paramsDone = true; + + tryComplete(info); } void PipelineHandlerMaliC55::statsBufferReady(FrameBuffer *buffer) { - availableStatsBuffers_.push(buffer); + MaliC55FrameInfo *info = findFrameInfo(buffer); + ASSERT(info); + + Request *request = info->request; + MaliC55CameraData *data = cameraData(request->_d()->camera()); + + ControlList sensorControls = data->delayedCtrls_->get(buffer->metadata().sequence); + + data->ipa_->processStats(request->sequence(), buffer->cookie(), + sensorControls); +} + +void PipelineHandlerMaliC55::paramsComputed(unsigned int requestId) +{ + MaliC55FrameInfo &frameInfo = frameInfoMap_[requestId]; + Request *request = frameInfo.request; + MaliC55CameraData *data = cameraData(request->_d()->camera()); + + /* + * Queue buffers for stats and params, then queue buffers to the capture + * video devices. + */ + + frameInfo.paramBuffer->_d()->metadata().planes()[0].bytesused = + sizeof(struct mali_c55_params_buffer); + params_->queueBuffer(frameInfo.paramBuffer); + stats_->queueBuffer(frameInfo.statBuffer); + + for (auto &[stream, buffer] : request->buffers()) { + MaliC55Pipe *pipe = pipeFromStream(data, stream); + + pipe->cap->queueBuffer(buffer); + } } -void PipelineHandlerMaliC55::registerMaliCamera(std::unique_ptr data, +void PipelineHandlerMaliC55::statsProcessed(unsigned int requestId, + const ControlList &metadata) +{ + MaliC55FrameInfo &frameInfo = frameInfoMap_[requestId]; + + frameInfo.statsDone = true; + frameInfo.request->metadata().merge(metadata); + + tryComplete(&frameInfo); +} + +bool PipelineHandlerMaliC55::registerMaliCamera(std::unique_ptr data, const std::string &name) { + if (data->loadIPA()) + return false; + + data->ipa_->statsProcessed.connect(this, &PipelineHandlerMaliC55::statsProcessed); + data->ipa_->paramsComputed.connect(this, &PipelineHandlerMaliC55::paramsComputed); + std::set streams{ &data->frStream_ }; if (dsFitted_) streams.insert(&data->dsStream_); @@ -1177,6 +1436,8 @@ void PipelineHandlerMaliC55::registerMaliCamera(std::unique_ptr camera = Camera::create(std::move(data), name, streams); registerCamera(std::move(camera)); + + return true; } /* @@ -1202,9 +1463,7 @@ bool PipelineHandlerMaliC55::registerTPGCamera(MediaLink *link) if (data->init()) return false; - registerMaliCamera(std::move(data), name); - - return true; + return registerMaliCamera(std::move(data), name); } /* @@ -1230,11 +1489,26 @@ bool PipelineHandlerMaliC55::registerSensorCamera(MediaLink *ispLink) if (data->init()) return false; - /* \todo: Init properties. */ + /* + * \todo Read delay values from the sensor itself or from a + * a sensor database. For now use generic values taken from + * the Raspberry Pi and listed as 'generic values'. + */ + std::unordered_map params = { + { V4L2_CID_ANALOGUE_GAIN, { 1, false } }, + { V4L2_CID_EXPOSURE, { 2, false } }, + }; + + data->delayedCtrls_ = + std::make_unique(data->sensor_->device(), + params); + isp_->frameStart.connect(data->delayedCtrls_.get(), + &DelayedControls::applyControls); - data->updateControls(); + /* \todo: Init properties. */ - registerMaliCamera(std::move(data), sensor->name()); + if (!registerMaliCamera(std::move(data), sensor->name())) + return false; } return true; @@ -1301,6 +1575,7 @@ bool PipelineHandlerMaliC55::match(DeviceEnumerator *enumerator) } stats_->bufferReady.connect(this, &PipelineHandlerMaliC55::statsBufferReady); + params_->bufferReady.connect(this, &PipelineHandlerMaliC55::paramsBufferReady); ispSink = isp_->entity()->getPadByIndex(0); if (!ispSink || ispSink->links().empty()) {