[{"id":31761,"web_url":"https://patchwork.libcamera.org/comment/31761/","msgid":"","date":"2024-10-16T12:49:12","subject":"Re: [PATCH v8 4/4] libcamera: rkisp1: Plumb the dw100 dewarper as\n\tV4L2M2M converter","submitter":{"id":184,"url":"https://patchwork.libcamera.org/api/people/184/","name":"Stefan Klug","email":"stefan.klug@ideasonboard.com"},"content":"Hi Umang,\n\nThank you for the patch. \n\nOn Thu, Sep 26, 2024 at 03:06:23PM +0530, Umang Jain wrote:\n> Plumb the dw100 dewarper as a V4L2M2M converter in the rkisp1 pipeline\n> handler. If the dewarper is found, it is instantiated and buffers are\n> exported from it, instead of RkISP1Path. Internal buffers are allocated\n> for the RkISP1Path in case where dewarper is going to be used.\n> \n> The RKISP1 pipeline handler now supports scaler crop control through\n> the converter. Register the ScalerCrop control for the cameras created\n> in the RKISP1 pipeline handler.\n> \n> Signed-off-by: Umang Jain \n\nThe only thing where I'm uncertain is the implementation of the\nScalerCrop. As we know, the scaler crop implementation of the dw100\nkernel driver is a bit limited and results in unexpected crops beeing\napplied. But I think that can/should be fixed in later patches where the\nvertex map can be used for that. So I think we should merge that in.\n\nReviewed-by: Stefan Klug \n\nCheers,\nStefan\n\n> ---\n> src/libcamera/pipeline/rkisp1/rkisp1.cpp | 186 ++++++++++++++++++++++-\n> 1 file changed, 179 insertions(+), 7 deletions(-)\n> \n> diff --git a/src/libcamera/pipeline/rkisp1/rkisp1.cpp b/src/libcamera/pipeline/rkisp1/rkisp1.cpp\n> index 651258e3..84c2796e 100644\n> --- a/src/libcamera/pipeline/rkisp1/rkisp1.cpp\n> +++ b/src/libcamera/pipeline/rkisp1/rkisp1.cpp\n> @@ -6,10 +6,12 @@\n> */\n> \n> #include \n> +#include \n> #include \n> #include \n> #include \n> #include \n> +#include \n> \n> #include \n> #include \n> @@ -32,6 +34,7 @@\n> \n> #include \"libcamera/internal/camera.h\"\n> #include \"libcamera/internal/camera_sensor.h\"\n> +#include \"libcamera/internal/converter/converter_v4l2_m2m.h\"\n> #include \"libcamera/internal/delayed_controls.h\"\n> #include \"libcamera/internal/device_enumerator.h\"\n> #include \"libcamera/internal/framebuffer.h\"\n> @@ -180,6 +183,7 @@ private:\n> \tvoid bufferReady(FrameBuffer *buffer);\n> \tvoid paramReady(FrameBuffer *buffer);\n> \tvoid statReady(FrameBuffer *buffer);\n> +\tvoid dewarpBufferReady(FrameBuffer *buffer);\n> \tvoid frameStart(uint32_t sequence);\n> \n> \tint allocateBuffers(Camera *camera);\n> @@ -199,6 +203,15 @@ private:\n> \tRkISP1MainPath mainPath_;\n> \tRkISP1SelfPath selfPath_;\n> \n> +\tstd::unique_ptr dewarper_;\n> +\tbool useDewarper_;\n> +\n> +\tstd::optional activeCrop_;\n> +\n> +\t/* Internal buffers used when dewarper is being used */\n> +\tstd::vector> mainPathBuffers_;\n> +\tstd::queue availableMainPathBuffers_;\n> +\n> \tstd::vector> paramBuffers_;\n> \tstd::vector> statBuffers_;\n> \tstd::queue availableParamBuffers_;\n> @@ -221,6 +234,8 @@ RkISP1FrameInfo *RkISP1Frames::create(const RkISP1CameraData *data, Request *req\n> \n> \tFrameBuffer *paramBuffer = nullptr;\n> \tFrameBuffer *statBuffer = nullptr;\n> +\tFrameBuffer *mainPathBuffer = nullptr;\n> +\tFrameBuffer *selfPathBuffer = nullptr;\n> \n> \tif (!isRaw) {\n> \t\tif (pipe_->availableParamBuffers_.empty()) {\n> @@ -238,10 +253,16 @@ RkISP1FrameInfo *RkISP1Frames::create(const RkISP1CameraData *data, Request *req\n> \n> \t\tstatBuffer = pipe_->availableStatBuffers_.front();\n> \t\tpipe_->availableStatBuffers_.pop();\n> +\n> +\t\tif (pipe_->useDewarper_) {\n> +\t\t\tmainPathBuffer = pipe_->availableMainPathBuffers_.front();\n> +\t\t\tpipe_->availableMainPathBuffers_.pop();\n> +\t\t}\n> \t}\n> \n> -\tFrameBuffer *mainPathBuffer = request->findBuffer(&data->mainPathStream_);\n> -\tFrameBuffer *selfPathBuffer = request->findBuffer(&data->selfPathStream_);\n> +\tif (!mainPathBuffer)\n> +\t\tmainPathBuffer = request->findBuffer(&data->mainPathStream_);\n> +\tselfPathBuffer = request->findBuffer(&data->selfPathStream_);\n> \n> \tRkISP1FrameInfo *info = new RkISP1FrameInfo;\n> \n> @@ -267,6 +288,7 @@ int RkISP1Frames::destroy(unsigned int frame)\n> \n> \tpipe_->availableParamBuffers_.push(info->paramBuffer);\n> \tpipe_->availableStatBuffers_.push(info->statBuffer);\n> +\tpipe_->availableMainPathBuffers_.push(info->mainPathBuffer);\n> \n> \tframeInfo_.erase(info->frame);\n> \n> @@ -282,6 +304,7 @@ void RkISP1Frames::clear()\n> \n> \t\tpipe_->availableParamBuffers_.push(info->paramBuffer);\n> \t\tpipe_->availableStatBuffers_.push(info->statBuffer);\n> +\t\tpipe_->availableMainPathBuffers_.push(info->mainPathBuffer);\n> \n> \t\tdelete info;\n> \t}\n> @@ -600,7 +623,7 @@ CameraConfiguration::Status RkISP1CameraConfiguration::validate()\n> */\n> \n> PipelineHandlerRkISP1::PipelineHandlerRkISP1(CameraManager *manager)\n> -\t: PipelineHandler(manager), hasSelfPath_(true)\n> +\t: PipelineHandler(manager), hasSelfPath_(true), useDewarper_(false)\n> {\n> }\n> \n> @@ -778,12 +801,19 @@ int PipelineHandlerRkISP1::configure(Camera *camera, CameraConfiguration *c)\n> \t\t<< \" crop \" << rect;\n> \n> \tstd::map streamConfig;\n> +\tstd::vector> outputCfgs;\n> \n> \tfor (const StreamConfiguration &cfg : *config) {\n> \t\tif (cfg.stream() == &data->mainPathStream_) {\n> \t\t\tret = mainPath_.configure(cfg, format);\n> \t\t\tstreamConfig[0] = IPAStream(cfg.pixelFormat,\n> \t\t\t\t\t\t cfg.size);\n> +\t\t\t/* Configure dewarp */\n> +\t\t\tif (dewarper_ && !isRaw_) {\n> +\t\t\t\toutputCfgs.push_back(const_cast(cfg));\n> +\t\t\t\tret = dewarper_->configure(cfg, outputCfgs);\n> +\t\t\t\tuseDewarper_ = ret ? false : true;\n> +\t\t\t}\n> \t\t} else if (hasSelfPath_) {\n> \t\t\tret = selfPath_.configure(cfg, format);\n> \t\t\tstreamConfig[1] = IPAStream(cfg.pixelFormat,\n> @@ -833,10 +863,19 @@ int PipelineHandlerRkISP1::exportFrameBuffers([[maybe_unused]] Camera *camera, S\n> \tRkISP1CameraData *data = cameraData(camera);\n> \tunsigned int count = stream->configuration().bufferCount;\n> \n> -\tif (stream == &data->mainPathStream_)\n> -\t\treturn mainPath_.exportBuffers(count, buffers);\n> -\telse if (hasSelfPath_ && stream == &data->selfPathStream_)\n> +\tif (stream == &data->mainPathStream_) {\n> +\t\t/*\n> +\t\t * Currently, i.MX8MP is the only platform with DW100 dewarper.\n> +\t\t * It has mainpath and no self path. Hence, export buffers from\n> +\t\t * dewarper just for the main path stream, for now.\n> +\t\t */\n> +\t\tif (useDewarper_)\n> +\t\t\treturn dewarper_->exportBuffers(&data->mainPathStream_, count, buffers);\n> +\t\telse\n> +\t\t\treturn mainPath_.exportBuffers(count, buffers);\n> +\t} else if (hasSelfPath_ && stream == &data->selfPathStream_) {\n> \t\treturn selfPath_.exportBuffers(count, buffers);\n> +\t}\n> \n> \treturn -EINVAL;\n> }\n> @@ -860,6 +899,16 @@ int PipelineHandlerRkISP1::allocateBuffers(Camera *camera)\n> \t\tret = stat_->allocateBuffers(maxCount, &statBuffers_);\n> \t\tif (ret < 0)\n> \t\t\tgoto error;\n> +\n> +\t\t/* If the dewarper is being used, allocate internal buffers for ISP. */\n> +\t\tif (useDewarper_) {\n> +\t\t\tret = mainPath_.exportBuffers(maxCount, &mainPathBuffers_);\n> +\t\t\tif (ret < 0)\n> +\t\t\t\tgoto error;\n> +\n> +\t\t\tfor (std::unique_ptr &buffer : mainPathBuffers_)\n> +\t\t\t\tavailableMainPathBuffers_.push(buffer.get());\n> +\t\t}\n> \t}\n> \n> \tfor (std::unique_ptr &buffer : paramBuffers_) {\n> @@ -883,6 +932,7 @@ int PipelineHandlerRkISP1::allocateBuffers(Camera *camera)\n> error:\n> \tparamBuffers_.clear();\n> \tstatBuffers_.clear();\n> +\tmainPathBuffers_.clear();\n> \n> \treturn ret;\n> }\n> @@ -897,8 +947,12 @@ int PipelineHandlerRkISP1::freeBuffers(Camera *camera)\n> \twhile (!availableParamBuffers_.empty())\n> \t\tavailableParamBuffers_.pop();\n> \n> +\twhile (!availableMainPathBuffers_.empty())\n> +\t\tavailableMainPathBuffers_.pop();\n> +\n> \tparamBuffers_.clear();\n> \tstatBuffers_.clear();\n> +\tmainPathBuffers_.clear();\n> \n> \tstd::vector ids;\n> \tfor (IPABuffer &ipabuf : data->ipaBuffers_)\n> @@ -954,6 +1008,15 @@ int PipelineHandlerRkISP1::start(Camera *camera, [[maybe_unused]] const ControlL\n> \t\t\treturn ret;\n> \t\t}\n> \t\tactions += [&]() { stat_->streamOff(); };\n> +\n> +\t\tif (useDewarper_) {\n> +\t\t\tret = dewarper_->start();\n> +\t\t\tif (ret) {\n> +\t\t\t\tLOG(RkISP1, Error) << \"Failed to start dewarper\";\n> +\t\t\t\treturn ret;\n> +\t\t\t}\n> +\t\t}\n> +\t\tactions += [&]() { dewarper_->stop(); };\n> \t}\n> \n> \tif (data->mainPath_->isEnabled()) {\n> @@ -1000,6 +1063,9 @@ void PipelineHandlerRkISP1::stopDevice(Camera *camera)\n> \t\tif (ret)\n> \t\t\tLOG(RkISP1, Warning)\n> \t\t\t\t<< \"Failed to stop parameters for \" << camera->id();\n> +\n> +\t\tif (useDewarper_)\n> +\t\t\tdewarper_->stop();\n> \t}\n> \n> \tASSERT(data->queuedRequests_.empty());\n> @@ -1110,6 +1176,16 @@ int PipelineHandlerRkISP1::updateControls(RkISP1CameraData *data)\n> {\n> \tControlInfoMap::Map controls;\n> \n> +\tif (dewarper_) {\n> +\t\tstd::pair cropLimits =\n> +\t\t\tdewarper_->inputCropBounds(&data->mainPathStream_);\n> +\n> +\t\tcontrols[&controls::ScalerCrop] = ControlInfo(cropLimits.first,\n> +\t\t\t\t\t\t\t cropLimits.second,\n> +\t\t\t\t\t\t\t cropLimits.second);\n> +\t\tactiveCrop_ = cropLimits.second;\n> +\t}\n> +\n> \t/* Add the IPA registered controls to list of camera controls. */\n> \tfor (const auto &ipaControl : data->ipaControls_)\n> \t\tcontrols[ipaControl.first] = ipaControl.second;\n> @@ -1236,6 +1312,29 @@ bool PipelineHandlerRkISP1::match(DeviceEnumerator *enumerator)\n> \tstat_->bufferReady.connect(this, &PipelineHandlerRkISP1::statReady);\n> \tparam_->bufferReady.connect(this, &PipelineHandlerRkISP1::paramReady);\n> \n> +\t/* If dewarper is present, create its instance. */\n> +\tDeviceMatch dwp(\"dw100\");\n> +\tdwp.add(\"dw100-source\");\n> +\tdwp.add(\"dw100-sink\");\n> +\n> +\tstd::shared_ptr dwpMediaDevice = enumerator->search(dwp);\n> +\tif (dwpMediaDevice) {\n> +\t\tdewarper_ = std::make_unique(dwpMediaDevice.get());\n> +\t\tif (dewarper_->isValid()) {\n> +\t\t\tdewarper_->outputBufferReady.connect(\n> +\t\t\t\tthis, &PipelineHandlerRkISP1::dewarpBufferReady);\n> +\n> +\t\t\tLOG(RkISP1, Info)\n> +\t\t\t\t<< \"Using DW100 dewarper \" << dewarper_->deviceNode();\n> +\t\t} else {\n> +\t\t\tLOG(RkISP1, Warning)\n> +\t\t\t\t<< \"Found DW100 dewarper \" << dewarper_->deviceNode()\n> +\t\t\t\t<< \" but invalid\";\n> +\n> +\t\t\tdewarper_.reset();\n> +\t\t}\n> +\t}\n> +\n> \t/*\n> \t * Enumerate all sensors connected to the ISP and create one\n> \t * camera instance for each of them.\n> @@ -1282,7 +1381,7 @@ void PipelineHandlerRkISP1::bufferReady(FrameBuffer *buffer)\n> \t\treturn;\n> \n> \tconst FrameMetadata &metadata = buffer->metadata();\n> -\tRequest *request = buffer->request();\n> +\tRequest *request = info->request;\n> \n> \tif (metadata.status != FrameMetadata::FrameCancelled) {\n> \t\t/*\n> @@ -1304,6 +1403,79 @@ void PipelineHandlerRkISP1::bufferReady(FrameBuffer *buffer)\n> \t\t\tinfo->metadataProcessed = true;\n> \t}\n> \n> +\tif (!useDewarper_) {\n> +\t\tcompleteBuffer(request, buffer);\n> +\t\ttryCompleteRequest(info);\n> +\n> +\t\treturn;\n> +\t}\n> +\n> +\t/* Do not queue cancelled frames to dewarper. */\n> +\tif (metadata.status == FrameMetadata::FrameCancelled) {\n> +\t\t/*\n> +\t\t * i.MX8MP is the only known platform with dewarper. It has\n> +\t\t * no self path. Hence, only main path buffer completion is\n> +\t\t * required.\n> +\t\t *\n> +\t\t * Also, we cannot completeBuffer(request, buffer) as buffer\n> +\t\t * here, is an internal buffer (between ISP and dewarper) and\n> +\t\t * is not associated to the any specific request. The request\n> +\t\t * buffer associated with main path stream is the one that\n> +\t\t * is required to be completed (not the internal buffer).\n> +\t\t */\n> +\t\tfor (auto it : request->buffers()) {\n> +\t\t\tif (it.first == &data->mainPathStream_)\n> +\t\t\t\tcompleteBuffer(request, it.second);\n> +\t\t}\n> +\n> +\t\ttryCompleteRequest(info);\n> +\t\treturn;\n> +\t}\n> +\n> +\t/* Handle scaler crop control. */\n> +\tconst auto &crop = request->controls().get(controls::ScalerCrop);\n> +\tif (crop) {\n> +\t\tRectangle appliedRect = crop.value();\n> +\n> +\t\tint ret = dewarper_->setInputCrop(&data->mainPathStream_,\n> +\t\t\t\t\t\t &appliedRect);\n> +\t\tif (!ret && appliedRect != crop.value()) {\n> +\t\t\t/*\n> +\t\t\t * If the rectangle is changed by setInputCrop on the\n> +\t\t\t * dewarper, log a debug message and cache the actual\n> +\t\t\t * applied rectangle for metadata reporting.\n> +\t\t\t */\n> +\t\t\tLOG(RkISP1, Debug)\n> +\t\t\t\t<< \"Applied rectangle \" << appliedRect.toString()\n> +\t\t\t\t<< \" differs from requested \" << crop.value().toString();\n> +\t\t}\n> +\n> +\t\tactiveCrop_ = appliedRect;\n> +\t}\n> +\n> +\t/*\n> +\t * Queue input and output buffers to the dewarper. The output\n> +\t * buffers for the dewarper are the buffers of the request, supplied\n> +\t * by the application.\n> +\t */\n> +\tint ret = dewarper_->queueBuffers(buffer, request->buffers());\n> +\tif (ret < 0)\n> +\t\tLOG(RkISP1, Error) << \"Cannot queue buffers to dewarper: \"\n> +\t\t\t\t << strerror(-ret);\n> +\n> +\trequest->metadata().set(controls::ScalerCrop, activeCrop_.value());\n> +}\n> +\n> +void PipelineHandlerRkISP1::dewarpBufferReady(FrameBuffer *buffer)\n> +{\n> +\tASSERT(activeCamera_);\n> +\tRkISP1CameraData *data = cameraData(activeCamera_);\n> +\tRequest *request = buffer->request();\n> +\n> +\tRkISP1FrameInfo *info = data->frameInfo_.find(buffer->request());\n> +\tif (!info)\n> +\t\treturn;\n> +\n> \tcompleteBuffer(request, buffer);\n> \ttryCompleteRequest(info);\n> }\n> -- \n> 2.45.0\n>","headers":{"Return-Path":"","X-Original-To":"parsemail@patchwork.libcamera.org","Delivered-To":"parsemail@patchwork.libcamera.org","Received":["from lancelot.ideasonboard.com (lancelot.ideasonboard.com\n\t[92.243.16.209])\n\tby patchwork.libcamera.org (Postfix) with ESMTPS id 3A46CC32FA\n\tfor ;\n\tWed, 16 Oct 2024 12:49:18 +0000 (UTC)","from lancelot.ideasonboard.com (localhost [IPv6:::1])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTP id 4822B6537F;\n\tWed, 16 Oct 2024 14:49:17 +0200 (CEST)","from perceval.ideasonboard.com (perceval.ideasonboard.com\n\t[213.167.242.64])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id BD68A6537E\n\tfor ;\n\tWed, 16 Oct 2024 14:49:15 +0200 (CEST)","from ideasonboard.com (unknown\n\t[IPv6:2a00:6020:448c:6c00:dcec:e732:9245:1a01])\n\tby perceval.ideasonboard.com (Postfix) with ESMTPSA id 55BD1A2F;\n\tWed, 16 Oct 2024 14:47:33 +0200 (CEST)"],"Authentication-Results":"lancelot.ideasonboard.com; dkim=pass (1024-bit key;\n\tunprotected) header.d=ideasonboard.com header.i=@ideasonboard.com\n\theader.b=\"LwF8NsAb\"; dkim-atps=neutral","DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/simple; d=ideasonboard.com;\n\ts=mail; t=1729082853;\n\tbh=hxhT8BNdB7Tj6PzR1hq8e77/F7S9OkMLlNOJmAo02DE=;\n\th=Date:From:To:Cc:Subject:References:In-Reply-To:From;\n\tb=LwF8NsAbGEOMqAozXvGbTHJvAfyqd7geLSX4pWmHgt0XCxbmMDyshTg3PBfhGpGBR\n\tQKZdafW9QnYN7gl+6uFCcBNQtlYhK090wriqzSQ6n6Mudj4cX3gd2PuKc9JCSJqldq\n\tHGlisrSwqu416Rckh0VUgyCbolMRKYM9ZGj+nq1k=","Date":"Wed, 16 Oct 2024 14:49:12 +0200","From":"Stefan Klug ","To":"Umang Jain ","Cc":"libcamera-devel@lists.libcamera.org, \n\tLaurent Pinchart ","Subject":"Re: [PATCH v8 4/4] libcamera: rkisp1: Plumb the dw100 dewarper as\n\tV4L2M2M converter","Message-ID":"","References":"<20240926093623.94136-1-umang.jain@ideasonboard.com>\n\t<20240926093623.94136-5-umang.jain@ideasonboard.com>","MIME-Version":"1.0","Content-Type":"text/plain; charset=utf-8","Content-Disposition":"inline","In-Reply-To":"<20240926093623.94136-5-umang.jain@ideasonboard.com>","X-BeenThere":"libcamera-devel@lists.libcamera.org","X-Mailman-Version":"2.1.29","Precedence":"list","List-Id":"","List-Unsubscribe":",\n\t","List-Archive":"","List-Post":"","List-Help":"","List-Subscribe":",\n\t","Errors-To":"libcamera-devel-bounces@lists.libcamera.org","Sender":"\"libcamera-devel\" "}},{"id":31887,"web_url":"https://patchwork.libcamera.org/comment/31887/","msgid":"","date":"2024-10-23T08:59:04","subject":"Re: [PATCH v8 4/4] libcamera: rkisp1: Plumb the dw100 dewarper as\n\tV4L2M2M converter","submitter":{"id":17,"url":"https://patchwork.libcamera.org/api/people/17/","name":"Paul Elder","email":"paul.elder@ideasonboard.com"},"content":"On Thu, Sep 26, 2024 at 03:06:23PM +0530, Umang Jain wrote:\n> Plumb the dw100 dewarper as a V4L2M2M converter in the rkisp1 pipeline\n> handler. If the dewarper is found, it is instantiated and buffers are\n> exported from it, instead of RkISP1Path. Internal buffers are allocated\n> for the RkISP1Path in case where dewarper is going to be used.\n> \n> The RKISP1 pipeline handler now supports scaler crop control through\n> the converter. Register the ScalerCrop control for the cameras created\n> in the RKISP1 pipeline handler.\n> \n> Signed-off-by: Umang Jain \n\nLooks good to me.\n\nReviewed-by: Paul Elder \n\n> ---\n> src/libcamera/pipeline/rkisp1/rkisp1.cpp | 186 ++++++++++++++++++++++-\n> 1 file changed, 179 insertions(+), 7 deletions(-)\n> \n> diff --git a/src/libcamera/pipeline/rkisp1/rkisp1.cpp b/src/libcamera/pipeline/rkisp1/rkisp1.cpp\n> index 651258e3..84c2796e 100644\n> --- a/src/libcamera/pipeline/rkisp1/rkisp1.cpp\n> +++ b/src/libcamera/pipeline/rkisp1/rkisp1.cpp\n> @@ -6,10 +6,12 @@\n> */\n> \n> #include \n> +#include \n> #include \n> #include \n> #include \n> #include \n> +#include \n> \n> #include \n> #include \n> @@ -32,6 +34,7 @@\n> \n> #include \"libcamera/internal/camera.h\"\n> #include \"libcamera/internal/camera_sensor.h\"\n> +#include \"libcamera/internal/converter/converter_v4l2_m2m.h\"\n> #include \"libcamera/internal/delayed_controls.h\"\n> #include \"libcamera/internal/device_enumerator.h\"\n> #include \"libcamera/internal/framebuffer.h\"\n> @@ -180,6 +183,7 @@ private:\n> \tvoid bufferReady(FrameBuffer *buffer);\n> \tvoid paramReady(FrameBuffer *buffer);\n> \tvoid statReady(FrameBuffer *buffer);\n> +\tvoid dewarpBufferReady(FrameBuffer *buffer);\n> \tvoid frameStart(uint32_t sequence);\n> \n> \tint allocateBuffers(Camera *camera);\n> @@ -199,6 +203,15 @@ private:\n> \tRkISP1MainPath mainPath_;\n> \tRkISP1SelfPath selfPath_;\n> \n> +\tstd::unique_ptr dewarper_;\n> +\tbool useDewarper_;\n> +\n> +\tstd::optional activeCrop_;\n> +\n> +\t/* Internal buffers used when dewarper is being used */\n> +\tstd::vector> mainPathBuffers_;\n> +\tstd::queue availableMainPathBuffers_;\n> +\n> \tstd::vector> paramBuffers_;\n> \tstd::vector> statBuffers_;\n> \tstd::queue availableParamBuffers_;\n> @@ -221,6 +234,8 @@ RkISP1FrameInfo *RkISP1Frames::create(const RkISP1CameraData *data, Request *req\n> \n> \tFrameBuffer *paramBuffer = nullptr;\n> \tFrameBuffer *statBuffer = nullptr;\n> +\tFrameBuffer *mainPathBuffer = nullptr;\n> +\tFrameBuffer *selfPathBuffer = nullptr;\n> \n> \tif (!isRaw) {\n> \t\tif (pipe_->availableParamBuffers_.empty()) {\n> @@ -238,10 +253,16 @@ RkISP1FrameInfo *RkISP1Frames::create(const RkISP1CameraData *data, Request *req\n> \n> \t\tstatBuffer = pipe_->availableStatBuffers_.front();\n> \t\tpipe_->availableStatBuffers_.pop();\n> +\n> +\t\tif (pipe_->useDewarper_) {\n> +\t\t\tmainPathBuffer = pipe_->availableMainPathBuffers_.front();\n> +\t\t\tpipe_->availableMainPathBuffers_.pop();\n> +\t\t}\n> \t}\n> \n> -\tFrameBuffer *mainPathBuffer = request->findBuffer(&data->mainPathStream_);\n> -\tFrameBuffer *selfPathBuffer = request->findBuffer(&data->selfPathStream_);\n> +\tif (!mainPathBuffer)\n> +\t\tmainPathBuffer = request->findBuffer(&data->mainPathStream_);\n> +\tselfPathBuffer = request->findBuffer(&data->selfPathStream_);\n> \n> \tRkISP1FrameInfo *info = new RkISP1FrameInfo;\n> \n> @@ -267,6 +288,7 @@ int RkISP1Frames::destroy(unsigned int frame)\n> \n> \tpipe_->availableParamBuffers_.push(info->paramBuffer);\n> \tpipe_->availableStatBuffers_.push(info->statBuffer);\n> +\tpipe_->availableMainPathBuffers_.push(info->mainPathBuffer);\n> \n> \tframeInfo_.erase(info->frame);\n> \n> @@ -282,6 +304,7 @@ void RkISP1Frames::clear()\n> \n> \t\tpipe_->availableParamBuffers_.push(info->paramBuffer);\n> \t\tpipe_->availableStatBuffers_.push(info->statBuffer);\n> +\t\tpipe_->availableMainPathBuffers_.push(info->mainPathBuffer);\n> \n> \t\tdelete info;\n> \t}\n> @@ -600,7 +623,7 @@ CameraConfiguration::Status RkISP1CameraConfiguration::validate()\n> */\n> \n> PipelineHandlerRkISP1::PipelineHandlerRkISP1(CameraManager *manager)\n> -\t: PipelineHandler(manager), hasSelfPath_(true)\n> +\t: PipelineHandler(manager), hasSelfPath_(true), useDewarper_(false)\n> {\n> }\n> \n> @@ -778,12 +801,19 @@ int PipelineHandlerRkISP1::configure(Camera *camera, CameraConfiguration *c)\n> \t\t<< \" crop \" << rect;\n> \n> \tstd::map streamConfig;\n> +\tstd::vector> outputCfgs;\n> \n> \tfor (const StreamConfiguration &cfg : *config) {\n> \t\tif (cfg.stream() == &data->mainPathStream_) {\n> \t\t\tret = mainPath_.configure(cfg, format);\n> \t\t\tstreamConfig[0] = IPAStream(cfg.pixelFormat,\n> \t\t\t\t\t\t cfg.size);\n> +\t\t\t/* Configure dewarp */\n> +\t\t\tif (dewarper_ && !isRaw_) {\n> +\t\t\t\toutputCfgs.push_back(const_cast(cfg));\n> +\t\t\t\tret = dewarper_->configure(cfg, outputCfgs);\n> +\t\t\t\tuseDewarper_ = ret ? false : true;\n> +\t\t\t}\n> \t\t} else if (hasSelfPath_) {\n> \t\t\tret = selfPath_.configure(cfg, format);\n> \t\t\tstreamConfig[1] = IPAStream(cfg.pixelFormat,\n> @@ -833,10 +863,19 @@ int PipelineHandlerRkISP1::exportFrameBuffers([[maybe_unused]] Camera *camera, S\n> \tRkISP1CameraData *data = cameraData(camera);\n> \tunsigned int count = stream->configuration().bufferCount;\n> \n> -\tif (stream == &data->mainPathStream_)\n> -\t\treturn mainPath_.exportBuffers(count, buffers);\n> -\telse if (hasSelfPath_ && stream == &data->selfPathStream_)\n> +\tif (stream == &data->mainPathStream_) {\n> +\t\t/*\n> +\t\t * Currently, i.MX8MP is the only platform with DW100 dewarper.\n> +\t\t * It has mainpath and no self path. Hence, export buffers from\n> +\t\t * dewarper just for the main path stream, for now.\n> +\t\t */\n> +\t\tif (useDewarper_)\n> +\t\t\treturn dewarper_->exportBuffers(&data->mainPathStream_, count, buffers);\n> +\t\telse\n> +\t\t\treturn mainPath_.exportBuffers(count, buffers);\n> +\t} else if (hasSelfPath_ && stream == &data->selfPathStream_) {\n> \t\treturn selfPath_.exportBuffers(count, buffers);\n> +\t}\n> \n> \treturn -EINVAL;\n> }\n> @@ -860,6 +899,16 @@ int PipelineHandlerRkISP1::allocateBuffers(Camera *camera)\n> \t\tret = stat_->allocateBuffers(maxCount, &statBuffers_);\n> \t\tif (ret < 0)\n> \t\t\tgoto error;\n> +\n> +\t\t/* If the dewarper is being used, allocate internal buffers for ISP. */\n> +\t\tif (useDewarper_) {\n> +\t\t\tret = mainPath_.exportBuffers(maxCount, &mainPathBuffers_);\n> +\t\t\tif (ret < 0)\n> +\t\t\t\tgoto error;\n> +\n> +\t\t\tfor (std::unique_ptr &buffer : mainPathBuffers_)\n> +\t\t\t\tavailableMainPathBuffers_.push(buffer.get());\n> +\t\t}\n> \t}\n> \n> \tfor (std::unique_ptr &buffer : paramBuffers_) {\n> @@ -883,6 +932,7 @@ int PipelineHandlerRkISP1::allocateBuffers(Camera *camera)\n> error:\n> \tparamBuffers_.clear();\n> \tstatBuffers_.clear();\n> +\tmainPathBuffers_.clear();\n> \n> \treturn ret;\n> }\n> @@ -897,8 +947,12 @@ int PipelineHandlerRkISP1::freeBuffers(Camera *camera)\n> \twhile (!availableParamBuffers_.empty())\n> \t\tavailableParamBuffers_.pop();\n> \n> +\twhile (!availableMainPathBuffers_.empty())\n> +\t\tavailableMainPathBuffers_.pop();\n> +\n> \tparamBuffers_.clear();\n> \tstatBuffers_.clear();\n> +\tmainPathBuffers_.clear();\n> \n> \tstd::vector ids;\n> \tfor (IPABuffer &ipabuf : data->ipaBuffers_)\n> @@ -954,6 +1008,15 @@ int PipelineHandlerRkISP1::start(Camera *camera, [[maybe_unused]] const ControlL\n> \t\t\treturn ret;\n> \t\t}\n> \t\tactions += [&]() { stat_->streamOff(); };\n> +\n> +\t\tif (useDewarper_) {\n> +\t\t\tret = dewarper_->start();\n> +\t\t\tif (ret) {\n> +\t\t\t\tLOG(RkISP1, Error) << \"Failed to start dewarper\";\n> +\t\t\t\treturn ret;\n> +\t\t\t}\n> +\t\t}\n> +\t\tactions += [&]() { dewarper_->stop(); };\n> \t}\n> \n> \tif (data->mainPath_->isEnabled()) {\n> @@ -1000,6 +1063,9 @@ void PipelineHandlerRkISP1::stopDevice(Camera *camera)\n> \t\tif (ret)\n> \t\t\tLOG(RkISP1, Warning)\n> \t\t\t\t<< \"Failed to stop parameters for \" << camera->id();\n> +\n> +\t\tif (useDewarper_)\n> +\t\t\tdewarper_->stop();\n> \t}\n> \n> \tASSERT(data->queuedRequests_.empty());\n> @@ -1110,6 +1176,16 @@ int PipelineHandlerRkISP1::updateControls(RkISP1CameraData *data)\n> {\n> \tControlInfoMap::Map controls;\n> \n> +\tif (dewarper_) {\n> +\t\tstd::pair cropLimits =\n> +\t\t\tdewarper_->inputCropBounds(&data->mainPathStream_);\n> +\n> +\t\tcontrols[&controls::ScalerCrop] = ControlInfo(cropLimits.first,\n> +\t\t\t\t\t\t\t cropLimits.second,\n> +\t\t\t\t\t\t\t cropLimits.second);\n> +\t\tactiveCrop_ = cropLimits.second;\n> +\t}\n> +\n> \t/* Add the IPA registered controls to list of camera controls. */\n> \tfor (const auto &ipaControl : data->ipaControls_)\n> \t\tcontrols[ipaControl.first] = ipaControl.second;\n> @@ -1236,6 +1312,29 @@ bool PipelineHandlerRkISP1::match(DeviceEnumerator *enumerator)\n> \tstat_->bufferReady.connect(this, &PipelineHandlerRkISP1::statReady);\n> \tparam_->bufferReady.connect(this, &PipelineHandlerRkISP1::paramReady);\n> \n> +\t/* If dewarper is present, create its instance. */\n> +\tDeviceMatch dwp(\"dw100\");\n> +\tdwp.add(\"dw100-source\");\n> +\tdwp.add(\"dw100-sink\");\n> +\n> +\tstd::shared_ptr dwpMediaDevice = enumerator->search(dwp);\n> +\tif (dwpMediaDevice) {\n> +\t\tdewarper_ = std::make_unique(dwpMediaDevice.get());\n> +\t\tif (dewarper_->isValid()) {\n> +\t\t\tdewarper_->outputBufferReady.connect(\n> +\t\t\t\tthis, &PipelineHandlerRkISP1::dewarpBufferReady);\n> +\n> +\t\t\tLOG(RkISP1, Info)\n> +\t\t\t\t<< \"Using DW100 dewarper \" << dewarper_->deviceNode();\n> +\t\t} else {\n> +\t\t\tLOG(RkISP1, Warning)\n> +\t\t\t\t<< \"Found DW100 dewarper \" << dewarper_->deviceNode()\n> +\t\t\t\t<< \" but invalid\";\n> +\n> +\t\t\tdewarper_.reset();\n> +\t\t}\n> +\t}\n> +\n> \t/*\n> \t * Enumerate all sensors connected to the ISP and create one\n> \t * camera instance for each of them.\n> @@ -1282,7 +1381,7 @@ void PipelineHandlerRkISP1::bufferReady(FrameBuffer *buffer)\n> \t\treturn;\n> \n> \tconst FrameMetadata &metadata = buffer->metadata();\n> -\tRequest *request = buffer->request();\n> +\tRequest *request = info->request;\n> \n> \tif (metadata.status != FrameMetadata::FrameCancelled) {\n> \t\t/*\n> @@ -1304,6 +1403,79 @@ void PipelineHandlerRkISP1::bufferReady(FrameBuffer *buffer)\n> \t\t\tinfo->metadataProcessed = true;\n> \t}\n> \n> +\tif (!useDewarper_) {\n> +\t\tcompleteBuffer(request, buffer);\n> +\t\ttryCompleteRequest(info);\n> +\n> +\t\treturn;\n> +\t}\n> +\n> +\t/* Do not queue cancelled frames to dewarper. */\n> +\tif (metadata.status == FrameMetadata::FrameCancelled) {\n> +\t\t/*\n> +\t\t * i.MX8MP is the only known platform with dewarper. It has\n> +\t\t * no self path. Hence, only main path buffer completion is\n> +\t\t * required.\n> +\t\t *\n> +\t\t * Also, we cannot completeBuffer(request, buffer) as buffer\n> +\t\t * here, is an internal buffer (between ISP and dewarper) and\n> +\t\t * is not associated to the any specific request. The request\n> +\t\t * buffer associated with main path stream is the one that\n> +\t\t * is required to be completed (not the internal buffer).\n> +\t\t */\n> +\t\tfor (auto it : request->buffers()) {\n> +\t\t\tif (it.first == &data->mainPathStream_)\n> +\t\t\t\tcompleteBuffer(request, it.second);\n> +\t\t}\n> +\n> +\t\ttryCompleteRequest(info);\n> +\t\treturn;\n> +\t}\n> +\n> +\t/* Handle scaler crop control. */\n> +\tconst auto &crop = request->controls().get(controls::ScalerCrop);\n> +\tif (crop) {\n> +\t\tRectangle appliedRect = crop.value();\n> +\n> +\t\tint ret = dewarper_->setInputCrop(&data->mainPathStream_,\n> +\t\t\t\t\t\t &appliedRect);\n> +\t\tif (!ret && appliedRect != crop.value()) {\n> +\t\t\t/*\n> +\t\t\t * If the rectangle is changed by setInputCrop on the\n> +\t\t\t * dewarper, log a debug message and cache the actual\n> +\t\t\t * applied rectangle for metadata reporting.\n> +\t\t\t */\n> +\t\t\tLOG(RkISP1, Debug)\n> +\t\t\t\t<< \"Applied rectangle \" << appliedRect.toString()\n> +\t\t\t\t<< \" differs from requested \" << crop.value().toString();\n> +\t\t}\n> +\n> +\t\tactiveCrop_ = appliedRect;\n> +\t}\n> +\n> +\t/*\n> +\t * Queue input and output buffers to the dewarper. The output\n> +\t * buffers for the dewarper are the buffers of the request, supplied\n> +\t * by the application.\n> +\t */\n> +\tint ret = dewarper_->queueBuffers(buffer, request->buffers());\n> +\tif (ret < 0)\n> +\t\tLOG(RkISP1, Error) << \"Cannot queue buffers to dewarper: \"\n> +\t\t\t\t << strerror(-ret);\n> +\n> +\trequest->metadata().set(controls::ScalerCrop, activeCrop_.value());\n> +}\n> +\n> +void PipelineHandlerRkISP1::dewarpBufferReady(FrameBuffer *buffer)\n> +{\n> +\tASSERT(activeCamera_);\n> +\tRkISP1CameraData *data = cameraData(activeCamera_);\n> +\tRequest *request = buffer->request();\n> +\n> +\tRkISP1FrameInfo *info = data->frameInfo_.find(buffer->request());\n> +\tif (!info)\n> +\t\treturn;\n> +\n> \tcompleteBuffer(request, buffer);\n> \ttryCompleteRequest(info);\n> }\n> -- \n> 2.45.0\n>","headers":{"Return-Path":"","X-Original-To":"parsemail@patchwork.libcamera.org","Delivered-To":"parsemail@patchwork.libcamera.org","Received":["from lancelot.ideasonboard.com (lancelot.ideasonboard.com\n\t[92.243.16.209])\n\tby patchwork.libcamera.org (Postfix) with ESMTPS id 432DEC0DA4\n\tfor ;\n\tWed, 23 Oct 2024 08:59:17 +0000 (UTC)","from lancelot.ideasonboard.com (localhost [IPv6:::1])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTP id 74FA265391;\n\tWed, 23 Oct 2024 10:59:16 +0200 (CEST)","from perceval.ideasonboard.com (perceval.ideasonboard.com\n\t[213.167.242.64])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id D28E76042C\n\tfor ;\n\tWed, 23 Oct 2024 10:59:14 +0200 (CEST)","from pyrite.rasen.tech (unknown\n\t[IPv6:2404:7a81:160:2100:84f2:dbaf:ccb4:d22c])\n\tby perceval.ideasonboard.com (Postfix) with ESMTPSA id 027832EE;\n\tWed, 23 Oct 2024 10:57:25 +0200 (CEST)"],"Authentication-Results":"lancelot.ideasonboard.com; dkim=pass (1024-bit key;\n\tunprotected) header.d=ideasonboard.com header.i=@ideasonboard.com\n\theader.b=\"tI+s4YZg\"; dkim-atps=neutral","DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/simple; d=ideasonboard.com;\n\ts=mail; t=1729673847;\n\tbh=rMSCjabhD4hFC2aiPs9ffSPgL0MpLncgbuqrQamxgj0=;\n\th=Date:From:To:Cc:Subject:References:In-Reply-To:From;\n\tb=tI+s4YZgluPJ7fKKCLG4mBWxRytka+en3fCRh2c9FyEVrx2ESRYpU4x66i6skm0gL\n\t0sbUxTHcOXbbXpay+0DVIys8LpBwfxDwjydMaGMEo4Zz8rteXbZMYJOkoyWNF8empM\n\tAHR2QwQ/vmiAmucSUnUf4Onna62Qa7QAB/0GWPhc=","Date":"Wed, 23 Oct 2024 17:59:04 +0900","From":"Paul Elder ","To":"Umang Jain ","Cc":"libcamera-devel@lists.libcamera.org,\n\tLaurent Pinchart ,\n\tStefan Klug ","Subject":"Re: [PATCH v8 4/4] libcamera: rkisp1: Plumb the dw100 dewarper as\n\tV4L2M2M converter","Message-ID":"","References":"<20240926093623.94136-1-umang.jain@ideasonboard.com>\n\t<20240926093623.94136-5-umang.jain@ideasonboard.com>","MIME-Version":"1.0","Content-Type":"text/plain; charset=us-ascii","Content-Disposition":"inline","In-Reply-To":"<20240926093623.94136-5-umang.jain@ideasonboard.com>","X-BeenThere":"libcamera-devel@lists.libcamera.org","X-Mailman-Version":"2.1.29","Precedence":"list","List-Id":"","List-Unsubscribe":",\n\t","List-Archive":"","List-Post":"","List-Help":"","List-Subscribe":",\n\t","Errors-To":"libcamera-devel-bounces@lists.libcamera.org","Sender":"\"libcamera-devel\" "}}]