Patch Detail
Show a patch.
GET /api/1.1/patches/21172/?format=api
{ "id": 21172, "url": "https://patchwork.libcamera.org/api/1.1/patches/21172/?format=api", "web_url": "https://patchwork.libcamera.org/patch/21172/", "project": { "id": 1, "url": "https://patchwork.libcamera.org/api/1.1/projects/1/?format=api", "name": "libcamera", "link_name": "libcamera", "list_id": "libcamera_core", "list_email": "libcamera-devel@lists.libcamera.org", "web_url": "", "scm_url": "", "webscm_url": "" }, "msgid": "<20240906063444.32772-5-umang.jain@ideasonboard.com>", "date": "2024-09-06T06:34:44", "name": "[v7,4/4] libcamera: rkisp1: Plumb the dw100 dewarper as V4L2M2M converter", "commit_ref": null, "pull_url": null, "state": "superseded", "archived": false, "hash": "51d37121651d9ba8e45cc6bfcbd68e15d7c6f11e", "submitter": { "id": 86, "url": "https://patchwork.libcamera.org/api/1.1/people/86/?format=api", "name": "Umang Jain", "email": "umang.jain@ideasonboard.com" }, "delegate": null, "mbox": "https://patchwork.libcamera.org/patch/21172/mbox/", "series": [ { "id": 4565, "url": "https://patchwork.libcamera.org/api/1.1/series/4565/?format=api", "web_url": "https://patchwork.libcamera.org/project/libcamera/list/?series=4565", "date": "2024-09-06T06:34:40", "name": "libcamera: rkisp1: Plumb the ConverterDW100 converter", "version": 7, "mbox": "https://patchwork.libcamera.org/series/4565/mbox/" } ], "comments": "https://patchwork.libcamera.org/api/patches/21172/comments/", "check": "pending", "checks": "https://patchwork.libcamera.org/api/patches/21172/checks/", "tags": {}, "headers": { "Return-Path": "<libcamera-devel-bounces@lists.libcamera.org>", "X-Original-To": "parsemail@patchwork.libcamera.org", "Delivered-To": "parsemail@patchwork.libcamera.org", "Received": [ "from lancelot.ideasonboard.com (lancelot.ideasonboard.com\n\t[92.243.16.209])\n\tby patchwork.libcamera.org (Postfix) with ESMTPS id 6F847C324C\n\tfor <parsemail@patchwork.libcamera.org>;\n\tFri, 6 Sep 2024 06:35:08 +0000 (UTC)", "from lancelot.ideasonboard.com (localhost [IPv6:::1])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTP id 02D72634F1;\n\tFri, 6 Sep 2024 08:35:08 +0200 (CEST)", "from perceval.ideasonboard.com (perceval.ideasonboard.com\n\t[IPv6:2001:4b98:dc2:55:216:3eff:fef7:d647])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id 307F1634E3\n\tfor <libcamera-devel@lists.libcamera.org>;\n\tFri, 6 Sep 2024 08:34:59 +0200 (CEST)", "from localhost.localdomain (unknown\n\t[IPv6:2405:201:2015:f873:55f8:639e:8e9f:12ec])\n\tby perceval.ideasonboard.com (Postfix) with ESMTPSA id ED22945B;\n\tFri, 6 Sep 2024 08:33:44 +0200 (CEST)" ], "Authentication-Results": "lancelot.ideasonboard.com; dkim=pass (1024-bit key;\n\tunprotected) header.d=ideasonboard.com header.i=@ideasonboard.com\n\theader.b=\"WxHI3LDh\"; dkim-atps=neutral", "DKIM-Signature": "v=1; a=rsa-sha256; c=relaxed/simple; d=ideasonboard.com;\n\ts=mail; t=1725604425;\n\tbh=5oqWCZjI3h4cDkzjcq0wg1NjKsnBcb4fzpNtJe94xOI=;\n\th=From:To:Cc:Subject:Date:In-Reply-To:References:From;\n\tb=WxHI3LDh5ufrpIHzqJDQkoE80/Rn3a9S/u3RQCsx/T/BIko/jZrJQZ8xjls2DfuVr\n\tasIc+CUuh61vzAP4KG5wzSSzPVh54xf653HkuGj/0JBLFaH1JVvhxaAfIRnu7fpQqI\n\tzU7iZPf+KgUb9ltgwQuyRoWIH+kDcQOHdzG7A/vU=", "From": "Umang Jain <umang.jain@ideasonboard.com>", "To": "libcamera-devel@lists.libcamera.org", "Cc": "Umang Jain <umang.jain@ideasonboard.com>", "Subject": "[PATCH v7 4/4] libcamera: rkisp1: Plumb the dw100 dewarper as\n\tV4L2M2M converter", "Date": "Fri, 6 Sep 2024 12:04:44 +0530", "Message-ID": "<20240906063444.32772-5-umang.jain@ideasonboard.com>", "X-Mailer": "git-send-email 2.45.0", "In-Reply-To": "<20240906063444.32772-1-umang.jain@ideasonboard.com>", "References": "<20240906063444.32772-1-umang.jain@ideasonboard.com>", "MIME-Version": "1.0", "Content-Transfer-Encoding": "8bit", "X-BeenThere": "libcamera-devel@lists.libcamera.org", "X-Mailman-Version": "2.1.29", "Precedence": "list", "List-Id": "<libcamera-devel.lists.libcamera.org>", "List-Unsubscribe": "<https://lists.libcamera.org/options/libcamera-devel>,\n\t<mailto:libcamera-devel-request@lists.libcamera.org?subject=unsubscribe>", "List-Archive": "<https://lists.libcamera.org/pipermail/libcamera-devel/>", "List-Post": "<mailto:libcamera-devel@lists.libcamera.org>", "List-Help": "<mailto:libcamera-devel-request@lists.libcamera.org?subject=help>", "List-Subscribe": "<https://lists.libcamera.org/listinfo/libcamera-devel>,\n\t<mailto:libcamera-devel-request@lists.libcamera.org?subject=subscribe>", "Errors-To": "libcamera-devel-bounces@lists.libcamera.org", "Sender": "\"libcamera-devel\" <libcamera-devel-bounces@lists.libcamera.org>" }, "content": "Plumb the dw100 dewarper as a V4L2M2M converter in the rkisp1 pipeline\nhandler. If the dewarper is found, it is instantiated and buffers are\nexported from it, instead of RkISP1Path. Internal buffers are allocated\nfor the RkISP1Path in case where dewarper is going to be used.\n\nThe RKISP1 pipeline handler now supports scaler crop control through\nthe converter. Register the ScalerCrop control for the cameras created\nin the RKISP1 pipeline handler.\n\nSigned-off-by: Umang Jain <umang.jain@ideasonboard.com>\n---\n src/libcamera/pipeline/rkisp1/rkisp1.cpp | 183 ++++++++++++++++++++++-\n 1 file changed, 176 insertions(+), 7 deletions(-)", "diff": "diff --git a/src/libcamera/pipeline/rkisp1/rkisp1.cpp b/src/libcamera/pipeline/rkisp1/rkisp1.cpp\nindex 651258e3..0a794d63 100644\n--- a/src/libcamera/pipeline/rkisp1/rkisp1.cpp\n+++ b/src/libcamera/pipeline/rkisp1/rkisp1.cpp\n@@ -6,10 +6,12 @@\n */\n \n #include <algorithm>\n+#include <map>\n #include <memory>\n #include <numeric>\n #include <optional>\n #include <queue>\n+#include <vector>\n \n #include <linux/media-bus-format.h>\n #include <linux/rkisp1-config.h>\n@@ -32,6 +34,7 @@\n \n #include \"libcamera/internal/camera.h\"\n #include \"libcamera/internal/camera_sensor.h\"\n+#include \"libcamera/internal/converter/converter_v4l2_m2m.h\"\n #include \"libcamera/internal/delayed_controls.h\"\n #include \"libcamera/internal/device_enumerator.h\"\n #include \"libcamera/internal/framebuffer.h\"\n@@ -180,6 +183,7 @@ private:\n \tvoid bufferReady(FrameBuffer *buffer);\n \tvoid paramReady(FrameBuffer *buffer);\n \tvoid statReady(FrameBuffer *buffer);\n+\tvoid dewarpBufferReady(FrameBuffer *buffer);\n \tvoid frameStart(uint32_t sequence);\n \n \tint allocateBuffers(Camera *camera);\n@@ -199,6 +203,15 @@ private:\n \tRkISP1MainPath mainPath_;\n \tRkISP1SelfPath selfPath_;\n \n+\tstd::unique_ptr<V4L2M2MConverter> dewarper_;\n+\tbool useDewarper_;\n+\n+\tstd::optional<Rectangle> activeCrop_;\n+\n+\t/* Internal buffers used when dewarper is being used */\n+\tstd::vector<std::unique_ptr<FrameBuffer>> mainPathBuffers_;\n+\tstd::queue<FrameBuffer *> availableMainPathBuffers_;\n+\n \tstd::vector<std::unique_ptr<FrameBuffer>> paramBuffers_;\n \tstd::vector<std::unique_ptr<FrameBuffer>> statBuffers_;\n \tstd::queue<FrameBuffer *> availableParamBuffers_;\n@@ -221,6 +234,8 @@ RkISP1FrameInfo *RkISP1Frames::create(const RkISP1CameraData *data, Request *req\n \n \tFrameBuffer *paramBuffer = nullptr;\n \tFrameBuffer *statBuffer = nullptr;\n+\tFrameBuffer *mainPathBuffer = nullptr;\n+\tFrameBuffer *selfPathBuffer = nullptr;\n \n \tif (!isRaw) {\n \t\tif (pipe_->availableParamBuffers_.empty()) {\n@@ -238,10 +253,16 @@ RkISP1FrameInfo *RkISP1Frames::create(const RkISP1CameraData *data, Request *req\n \n \t\tstatBuffer = pipe_->availableStatBuffers_.front();\n \t\tpipe_->availableStatBuffers_.pop();\n+\n+\t\tif (pipe_->useDewarper_) {\n+\t\t\tmainPathBuffer = pipe_->availableMainPathBuffers_.front();\n+\t\t\tpipe_->availableMainPathBuffers_.pop();\n+\t\t}\n \t}\n \n-\tFrameBuffer *mainPathBuffer = request->findBuffer(&data->mainPathStream_);\n-\tFrameBuffer *selfPathBuffer = request->findBuffer(&data->selfPathStream_);\n+\tif (!mainPathBuffer)\n+\t\tmainPathBuffer = request->findBuffer(&data->mainPathStream_);\n+\tselfPathBuffer = request->findBuffer(&data->selfPathStream_);\n \n \tRkISP1FrameInfo *info = new RkISP1FrameInfo;\n \n@@ -267,6 +288,7 @@ int RkISP1Frames::destroy(unsigned int frame)\n \n \tpipe_->availableParamBuffers_.push(info->paramBuffer);\n \tpipe_->availableStatBuffers_.push(info->statBuffer);\n+\tpipe_->availableMainPathBuffers_.push(info->mainPathBuffer);\n \n \tframeInfo_.erase(info->frame);\n \n@@ -282,6 +304,7 @@ void RkISP1Frames::clear()\n \n \t\tpipe_->availableParamBuffers_.push(info->paramBuffer);\n \t\tpipe_->availableStatBuffers_.push(info->statBuffer);\n+\t\tpipe_->availableMainPathBuffers_.push(info->mainPathBuffer);\n \n \t\tdelete info;\n \t}\n@@ -600,7 +623,7 @@ CameraConfiguration::Status RkISP1CameraConfiguration::validate()\n */\n \n PipelineHandlerRkISP1::PipelineHandlerRkISP1(CameraManager *manager)\n-\t: PipelineHandler(manager), hasSelfPath_(true)\n+\t: PipelineHandler(manager), hasSelfPath_(true), useDewarper_(false)\n {\n }\n \n@@ -778,12 +801,19 @@ int PipelineHandlerRkISP1::configure(Camera *camera, CameraConfiguration *c)\n \t\t<< \" crop \" << rect;\n \n \tstd::map<unsigned int, IPAStream> streamConfig;\n+\tstd::vector<std::reference_wrapper<StreamConfiguration>> outputCfgs;\n \n \tfor (const StreamConfiguration &cfg : *config) {\n \t\tif (cfg.stream() == &data->mainPathStream_) {\n \t\t\tret = mainPath_.configure(cfg, format);\n \t\t\tstreamConfig[0] = IPAStream(cfg.pixelFormat,\n \t\t\t\t\t\t cfg.size);\n+\t\t\t/* Configure dewarp */\n+\t\t\tif (dewarper_ && !isRaw_) {\n+\t\t\t\toutputCfgs.push_back(const_cast<StreamConfiguration &>(cfg));\n+\t\t\t\tret = dewarper_->configure(cfg, outputCfgs);\n+\t\t\t\tuseDewarper_ = ret ? false : true;\n+\t\t\t}\n \t\t} else if (hasSelfPath_) {\n \t\t\tret = selfPath_.configure(cfg, format);\n \t\t\tstreamConfig[1] = IPAStream(cfg.pixelFormat,\n@@ -833,10 +863,19 @@ int PipelineHandlerRkISP1::exportFrameBuffers([[maybe_unused]] Camera *camera, S\n \tRkISP1CameraData *data = cameraData(camera);\n \tunsigned int count = stream->configuration().bufferCount;\n \n-\tif (stream == &data->mainPathStream_)\n-\t\treturn mainPath_.exportBuffers(count, buffers);\n-\telse if (hasSelfPath_ && stream == &data->selfPathStream_)\n+\tif (stream == &data->mainPathStream_) {\n+\t\t/*\n+\t\t * Currently, i.MX8MP is the only platform with DW100 dewarper.\n+\t\t * It has mainpath and no self path. Hence, export buffers from\n+\t\t * dewarper just for the main path stream, for now.\n+\t\t */\n+\t\tif (useDewarper_)\n+\t\t\treturn dewarper_->exportBuffers(&data->mainPathStream_, count, buffers);\n+\t\telse\n+\t\t\treturn mainPath_.exportBuffers(count, buffers);\n+\t} else if (hasSelfPath_ && stream == &data->selfPathStream_) {\n \t\treturn selfPath_.exportBuffers(count, buffers);\n+\t}\n \n \treturn -EINVAL;\n }\n@@ -860,6 +899,16 @@ int PipelineHandlerRkISP1::allocateBuffers(Camera *camera)\n \t\tret = stat_->allocateBuffers(maxCount, &statBuffers_);\n \t\tif (ret < 0)\n \t\t\tgoto error;\n+\n+\t\t/* If the dewarper is being used, allocate internal buffers for ISP. */\n+\t\tif (useDewarper_) {\n+\t\t\tret = mainPath_.exportBuffers(maxCount, &mainPathBuffers_);\n+\t\t\tif (ret < 0)\n+\t\t\t\tgoto error;\n+\n+\t\t\tfor (std::unique_ptr<FrameBuffer> &buffer : mainPathBuffers_)\n+\t\t\t\tavailableMainPathBuffers_.push(buffer.get());\n+\t\t}\n \t}\n \n \tfor (std::unique_ptr<FrameBuffer> &buffer : paramBuffers_) {\n@@ -883,6 +932,7 @@ int PipelineHandlerRkISP1::allocateBuffers(Camera *camera)\n error:\n \tparamBuffers_.clear();\n \tstatBuffers_.clear();\n+\tmainPathBuffers_.clear();\n \n \treturn ret;\n }\n@@ -897,8 +947,12 @@ int PipelineHandlerRkISP1::freeBuffers(Camera *camera)\n \twhile (!availableParamBuffers_.empty())\n \t\tavailableParamBuffers_.pop();\n \n+\twhile (!availableMainPathBuffers_.empty())\n+\t\tavailableMainPathBuffers_.pop();\n+\n \tparamBuffers_.clear();\n \tstatBuffers_.clear();\n+\tmainPathBuffers_.clear();\n \n \tstd::vector<unsigned int> ids;\n \tfor (IPABuffer &ipabuf : data->ipaBuffers_)\n@@ -954,6 +1008,15 @@ int PipelineHandlerRkISP1::start(Camera *camera, [[maybe_unused]] const ControlL\n \t\t\treturn ret;\n \t\t}\n \t\tactions += [&]() { stat_->streamOff(); };\n+\n+\t\tif (useDewarper_) {\n+\t\t\tret = dewarper_->start();\n+\t\t\tif (ret) {\n+\t\t\t\tLOG(RkISP1, Error) << \"Failed to start dewarper\";\n+\t\t\t\treturn ret;\n+\t\t\t}\n+\t\t}\n+\t\tactions += [&]() { dewarper_->stop(); };\n \t}\n \n \tif (data->mainPath_->isEnabled()) {\n@@ -1000,6 +1063,9 @@ void PipelineHandlerRkISP1::stopDevice(Camera *camera)\n \t\tif (ret)\n \t\t\tLOG(RkISP1, Warning)\n \t\t\t\t<< \"Failed to stop parameters for \" << camera->id();\n+\n+\t\tif (useDewarper_)\n+\t\t\tdewarper_->stop();\n \t}\n \n \tASSERT(data->queuedRequests_.empty());\n@@ -1110,6 +1176,16 @@ int PipelineHandlerRkISP1::updateControls(RkISP1CameraData *data)\n {\n \tControlInfoMap::Map controls;\n \n+\tif (dewarper_) {\n+\t\tstd::pair<Rectangle, Rectangle> cropLimits =\n+\t\t\tdewarper_->inputCropBounds(&data->mainPathStream_);\n+\n+\t\tcontrols[&controls::ScalerCrop] = ControlInfo(cropLimits.first,\n+\t\t\t\t\t\t\t cropLimits.second,\n+\t\t\t\t\t\t\t cropLimits.second);\n+\t\tactiveCrop_ = cropLimits.second;\n+\t}\n+\n \t/* Add the IPA registered controls to list of camera controls. */\n \tfor (const auto &ipaControl : data->ipaControls_)\n \t\tcontrols[ipaControl.first] = ipaControl.second;\n@@ -1236,6 +1312,26 @@ bool PipelineHandlerRkISP1::match(DeviceEnumerator *enumerator)\n \tstat_->bufferReady.connect(this, &PipelineHandlerRkISP1::statReady);\n \tparam_->bufferReady.connect(this, &PipelineHandlerRkISP1::paramReady);\n \n+\t/* If dewarper is present, create its instance. */\n+\tDeviceMatch dwp(\"dw100\");\n+\tdwp.add(\"dw100-source\");\n+\tdwp.add(\"dw100-sink\");\n+\n+\tstd::shared_ptr<MediaDevice> dwpMediaDevice = enumerator->search(dwp);\n+\tif (dwpMediaDevice) {\n+\t\tdewarper_ = std::make_unique<V4L2M2MConverter>(dwpMediaDevice.get());\n+\t\tif (dewarper_->isValid()) {\n+\t\t\tdewarper_->outputBufferReady.connect(\n+\t\t\t\tthis, &PipelineHandlerRkISP1::dewarpBufferReady);\n+\n+\t\t\tLOG(RkISP1, Info) << \"Using DW100 dewarper \" << dewarper_->deviceNode();\n+\t\t} else {\n+\t\t\tLOG(RkISP1, Warning) << \"Found DW100 dewarper \" << dewarper_->deviceNode()\n+\t\t\t\t\t << \" but invalid\";\n+\t\t\tdewarper_.reset();\n+\t\t}\n+\t}\n+\n \t/*\n \t * Enumerate all sensors connected to the ISP and create one\n \t * camera instance for each of them.\n@@ -1282,7 +1378,7 @@ void PipelineHandlerRkISP1::bufferReady(FrameBuffer *buffer)\n \t\treturn;\n \n \tconst FrameMetadata &metadata = buffer->metadata();\n-\tRequest *request = buffer->request();\n+\tRequest *request = info->request;\n \n \tif (metadata.status != FrameMetadata::FrameCancelled) {\n \t\t/*\n@@ -1304,6 +1400,79 @@ void PipelineHandlerRkISP1::bufferReady(FrameBuffer *buffer)\n \t\t\tinfo->metadataProcessed = true;\n \t}\n \n+\tif (!useDewarper_) {\n+\t\tcompleteBuffer(request, buffer);\n+\t\ttryCompleteRequest(info);\n+\n+\t\treturn;\n+\t}\n+\n+\t/* Do not queue cancelled frames to dewarper. */\n+\tif (metadata.status == FrameMetadata::FrameCancelled) {\n+\t\t/*\n+\t\t * i.MX8MP is the only known platform with dewarper. It has\n+\t\t * no self path. Hence, only main path buffer completion is\n+\t\t * required.\n+\t\t *\n+\t\t * Also, we cannot completeBuffer(request, buffer) as buffer\n+\t\t * here, is an internal buffer (between ISP and dewarper) and\n+\t\t * is not associated to the any specific request. The request\n+\t\t * buffer associated with main path stream is the one that\n+\t\t * is required to be completed (not the internal buffer).\n+\t\t */\n+\t\tfor (auto it : request->buffers()) {\n+\t\t\tif (it.first == &data->mainPathStream_)\n+\t\t\t\tcompleteBuffer(request, it.second);\n+\t\t}\n+\n+\t\ttryCompleteRequest(info);\n+\t\treturn;\n+\t}\n+\n+\t/* Handle scaler crop control. */\n+\tconst auto &crop = request->controls().get(controls::ScalerCrop);\n+\tif (crop) {\n+\t\tRectangle appliedRect = crop.value();\n+\n+\t\tint ret = dewarper_->setInputCrop(&data->mainPathStream_,\n+\t\t\t\t\t\t &appliedRect);\n+\t\tif (!ret && appliedRect != crop.value()) {\n+\t\t\t/*\n+\t\t\t * If the rectangle is changed by setInputCrop on the\n+\t\t\t * dewarper, log a debug message and cache the actual\n+\t\t\t * applied rectangle for metadata reporting.\n+\t\t\t */\n+\t\t\tLOG(RkISP1, Debug)\n+\t\t\t\t<< \"Applied rectangle \" << appliedRect.toString()\n+\t\t\t\t<< \" differs from requested \" << crop.value().toString();\n+\t\t}\n+\n+\t\tactiveCrop_ = appliedRect;\n+\t}\n+\n+\t/*\n+\t * Queue input and output buffers to the dewarper. The output\n+\t * buffers for the dewarper are the buffers of the request, supplied\n+\t * by the application.\n+\t */\n+\tint ret = dewarper_->queueBuffers(buffer, request->buffers());\n+\tif (ret < 0)\n+\t\tLOG(RkISP1, Error) << \"Cannot queue buffers to dewarper: \"\n+\t\t\t\t << strerror(-ret);\n+}\n+\n+void PipelineHandlerRkISP1::dewarpBufferReady(FrameBuffer *buffer)\n+{\n+\tASSERT(activeCamera_);\n+\tRkISP1CameraData *data = cameraData(activeCamera_);\n+\tRequest *request = buffer->request();\n+\n+\tRkISP1FrameInfo *info = data->frameInfo_.find(buffer->request());\n+\tif (!info)\n+\t\treturn;\n+\n+\trequest->metadata().set(controls::ScalerCrop, activeCrop_.value());\n+\n \tcompleteBuffer(request, buffer);\n \ttryCompleteRequest(info);\n }\n", "prefixes": [ "v7", "4/4" ] }