[{"id":32781,"web_url":"https://patchwork.libcamera.org/comment/32781/","msgid":"","date":"2024-12-16T18:29:56","subject":"Re: [PATCH v4 11/20] pipeline: rkisp1: Fix ScalerCrop to be in\n\tsensor coordinates","submitter":{"id":143,"url":"https://patchwork.libcamera.org/api/people/143/","name":"Jacopo Mondi","email":"jacopo.mondi@ideasonboard.com"},"content":"Hi Stefan\n\nOn Mon, Dec 16, 2024 at 04:40:51PM +0100, Stefan Klug wrote:\n> ScalerCrop is specified as being in sensor coordinates. The current\n> dewarper implementation on the imx8mp handles ScalerCrop in dewarper\n> coordinates. This leads to unexpected results and an unusable ScalerCrop\n> control in camshark. Fix that by transforming back and forth between\n> sensor coordinates and dewarper coordinates.\n>\n> Signed-off-by: Stefan Klug \n> Reviewed-by: Paul Elder \n>\n> ---\n> Changes in v3:\n> - Rename dewarpSensorCrop_ to scalerMaxCrop_\n\nWhy not scalerCropMax_ ?\n:)\n\nThe rest, according to my memories of your clarifications seems good\nto me\nReviewed-by: Jacopo Mondi \n\nThanks\n j\n\n> - Remove unnecessary ScalerCrop max calculation\n>\n> Changes in v2:\n> - Initialize dewarperSensorCrop_ to sane defaults\n> - Collect tags\n> ---\n> src/libcamera/pipeline/rkisp1/rkisp1.cpp | 50 +++++++++++++++++++-----\n> 1 file changed, 41 insertions(+), 9 deletions(-)\n>\n> diff --git a/src/libcamera/pipeline/rkisp1/rkisp1.cpp b/src/libcamera/pipeline/rkisp1/rkisp1.cpp\n> index 56192451eb3c..ef4aa38478f5 100644\n> --- a/src/libcamera/pipeline/rkisp1/rkisp1.cpp\n> +++ b/src/libcamera/pipeline/rkisp1/rkisp1.cpp\n> @@ -205,6 +205,7 @@ private:\n> \tRkISP1SelfPath selfPath_;\n>\n> \tstd::unique_ptr dewarper_;\n> +\tRectangle scalerMaxCrop_;\n> \tbool useDewarper_;\n>\n> \tstd::optional activeCrop_;\n> @@ -861,6 +862,15 @@ int PipelineHandlerRkISP1::configure(Camera *camera, CameraConfiguration *c)\n> \t\t\t\toutputCfgs.push_back(const_cast(cfg));\n> \t\t\t\tret = dewarper_->configure(cfg, outputCfgs);\n> \t\t\t\tuseDewarper_ = ret ? false : true;\n> +\n> +\t\t\t\t/*\n> +\t\t\t\t * Calculate the crop rectangle of the data\n> +\t\t\t\t * flowing into the dewarper in sensor\n> +\t\t\t\t * coordinates.\n> +\t\t\t\t */\n> +\t\t\t\tscalerMaxCrop_ =\n> +\t\t\t\t\toutputCrop.transformedBetween(inputCrop,\n> +\t\t\t\t\t\t\t\t sensorInfo.analogCrop);\n> \t\t\t}\n> \t\t} else if (hasSelfPath_) {\n> \t\t\tret = selfPath_.configure(cfg, format);\n> @@ -1226,10 +1236,19 @@ int PipelineHandlerRkISP1::updateControls(RkISP1CameraData *data)\n> \t\telse\n> \t\t\tcropLimits = dewarper_->inputCropBounds();\n>\n> -\t\tcontrols[&controls::ScalerCrop] = ControlInfo(cropLimits.first,\n> -\t\t\t\t\t\t\t cropLimits.second,\n> -\t\t\t\t\t\t\t cropLimits.second);\n> -\t\tactiveCrop_ = cropLimits.second;\n> +\t\t/*\n> +\t\t * ScalerCrop is specified to be in Sensor coordinates.\n> +\t\t * So we need to transform the limits to sensor coordinates.\n> +\t\t * We can safely assume that the maximum crop limit contains the\n> +\t\t * full fov of the dewarper.\n> +\t\t */\n> +\t\tRectangle min = cropLimits.first.transformedBetween(cropLimits.second,\n> +\t\t\t\t\t\t\t\t scalerMaxCrop_);\n> +\n> +\t\tcontrols[&controls::ScalerCrop] = ControlInfo(min,\n> +\t\t\t\t\t\t\t scalerMaxCrop_,\n> +\t\t\t\t\t\t\t scalerMaxCrop_);\n> +\t\tactiveCrop_ = scalerMaxCrop_;\n> \t}\n>\n> \t/* Add the IPA registered controls to list of camera controls. */\n> @@ -1257,6 +1276,8 @@ int PipelineHandlerRkISP1::createCamera(MediaEntity *sensor)\n> \t/* Initialize the camera properties. */\n> \tdata->properties_ = data->sensor_->properties();\n>\n> +\tscalerMaxCrop_ = Rectangle(data->sensor_->resolution());\n> +\n> \tconst CameraSensorProperties::SensorDelays &delays = data->sensor_->sensorDelays();\n> \tstd::unordered_map params = {\n> \t\t{ V4L2_CID_ANALOGUE_GAIN, { delays.gainDelay, false } },\n> @@ -1476,22 +1497,33 @@ void PipelineHandlerRkISP1::imageBufferReady(FrameBuffer *buffer)\n> \t/* Handle scaler crop control. */\n> \tconst auto &crop = request->controls().get(controls::ScalerCrop);\n> \tif (crop) {\n> -\t\tRectangle appliedRect = crop.value();\n> +\t\tRectangle rect = crop.value();\n> +\n> +\t\t/*\n> +\t\t * ScalerCrop is specified to be in Sensor coordinates.\n> +\t\t * So we need to transform it into dewarper coordinates.\n> +\t\t * We can safely assume that the maximum crop limit contains the\n> +\t\t * full fov of the dewarper.\n> +\t\t */\n> +\t\tstd::pair cropLimits =\n> +\t\t\tdewarper_->inputCropBounds(&data->mainPathStream_);\n>\n> +\t\trect = rect.transformedBetween(scalerMaxCrop_, cropLimits.second);\n> \t\tint ret = dewarper_->setInputCrop(&data->mainPathStream_,\n> -\t\t\t\t\t\t &appliedRect);\n> -\t\tif (!ret && appliedRect != crop.value()) {\n> +\t\t\t\t\t\t &rect);\n> +\t\trect = rect.transformedBetween(cropLimits.second, scalerMaxCrop_);\n> +\t\tif (!ret && rect != crop.value()) {\n> \t\t\t/*\n> \t\t\t * If the rectangle is changed by setInputCrop on the\n> \t\t\t * dewarper, log a debug message and cache the actual\n> \t\t\t * applied rectangle for metadata reporting.\n> \t\t\t */\n> \t\t\tLOG(RkISP1, Debug)\n> -\t\t\t\t<< \"Applied rectangle \" << appliedRect.toString()\n> +\t\t\t\t<< \"Applied rectangle \" << rect.toString()\n> \t\t\t\t<< \" differs from requested \" << crop.value().toString();\n> \t\t}\n>\n> -\t\tactiveCrop_ = appliedRect;\n> +\t\tactiveCrop_ = rect;\n> \t}\n>\n> \t/*\n> --\n> 2.43.0\n>","headers":{"Return-Path":"","X-Original-To":"parsemail@patchwork.libcamera.org","Delivered-To":"parsemail@patchwork.libcamera.org","Received":["from lancelot.ideasonboard.com (lancelot.ideasonboard.com\n\t[92.243.16.209])\n\tby patchwork.libcamera.org (Postfix) with ESMTPS id 0BD2FC32F6\n\tfor ;\n\tMon, 16 Dec 2024 18:30:02 +0000 (UTC)","from lancelot.ideasonboard.com (localhost [IPv6:::1])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTP id 0E0CB67F97;\n\tMon, 16 Dec 2024 19:30:01 +0100 (CET)","from perceval.ideasonboard.com (perceval.ideasonboard.com\n\t[213.167.242.64])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id 63ED167F7F\n\tfor ;\n\tMon, 16 Dec 2024 19:29:59 +0100 (CET)","from ideasonboard.com (93-61-96-190.ip145.fastwebnet.it\n\t[93.61.96.190])\n\tby perceval.ideasonboard.com (Postfix) with ESMTPSA id 8EEE8160;\n\tMon, 16 Dec 2024 19:29:22 +0100 (CET)"],"Authentication-Results":"lancelot.ideasonboard.com; dkim=pass (1024-bit key;\n\tunprotected) header.d=ideasonboard.com header.i=@ideasonboard.com\n\theader.b=\"WdKWUkT4\"; dkim-atps=neutral","DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/simple; d=ideasonboard.com;\n\ts=mail; t=1734373762;\n\tbh=PuLKQnjVAMzDFaxjEFYcBAws44SS8wuDHgMNSq5DTZE=;\n\th=Date:From:To:Cc:Subject:References:In-Reply-To:From;\n\tb=WdKWUkT4JKKKiya/GJM9cstsbOIYwKFnC61bHQBFIgUS0Ll5R2fA/KonT1XGuPaAv\n\tHXM5GsOahFs7d1kwx5Lj7FIhCNLnisu0XxspZHA1wNFRgcCfZea0nOM7AM3LSPhK4m\n\t0nsNibZMJy7VBVwDv/qJ9qfoTsBmX5WnEKwxMytM=","Date":"Mon, 16 Dec 2024 19:29:56 +0100","From":"Jacopo Mondi ","To":"Stefan Klug ","Cc":"libcamera-devel@lists.libcamera.org, \n\tPaul Elder ","Subject":"Re: [PATCH v4 11/20] pipeline: rkisp1: Fix ScalerCrop to be in\n\tsensor coordinates","Message-ID":"","References":"<20241216154124.203650-1-stefan.klug@ideasonboard.com>\n\t<20241216154124.203650-12-stefan.klug@ideasonboard.com>","MIME-Version":"1.0","Content-Type":"text/plain; charset=utf-8","Content-Disposition":"inline","In-Reply-To":"<20241216154124.203650-12-stefan.klug@ideasonboard.com>","X-BeenThere":"libcamera-devel@lists.libcamera.org","X-Mailman-Version":"2.1.29","Precedence":"list","List-Id":"","List-Unsubscribe":",\n\t","List-Archive":"","List-Post":"","List-Help":"","List-Subscribe":",\n\t","Errors-To":"libcamera-devel-bounces@lists.libcamera.org","Sender":"\"libcamera-devel\" "}},{"id":32787,"web_url":"https://patchwork.libcamera.org/comment/32787/","msgid":"","date":"2024-12-16T20:11:09","subject":"Re: [PATCH v4 11/20] pipeline: rkisp1: Fix ScalerCrop to be in\n\tsensor coordinates","submitter":{"id":184,"url":"https://patchwork.libcamera.org/api/people/184/","name":"Stefan Klug","email":"stefan.klug@ideasonboard.com"},"content":"Hi Jacopo,\n\nThank you for the review.\n\nOn Mon, Dec 16, 2024 at 07:29:56PM +0100, Jacopo Mondi wrote:\n> Hi Stefan\n> \n> On Mon, Dec 16, 2024 at 04:40:51PM +0100, Stefan Klug wrote:\n> > ScalerCrop is specified as being in sensor coordinates. The current\n> > dewarper implementation on the imx8mp handles ScalerCrop in dewarper\n> > coordinates. This leads to unexpected results and an unusable ScalerCrop\n> > control in camshark. Fix that by transforming back and forth between\n> > sensor coordinates and dewarper coordinates.\n> >\n> > Signed-off-by: Stefan Klug \n> > Reviewed-by: Paul Elder \n> >\n> > ---\n> > Changes in v3:\n> > - Rename dewarpSensorCrop_ to scalerMaxCrop_\n> \n> Why not scalerCropMax_ ?\n> :)\n\nHaha... I actually thought about renaming it to scalerCropMax_ because\nI also like it better. But in Message-ID:\n<3i57xvgmtl7ey26isg5t7xubghj26pitm37szvyq6q5ingi7wz@d2udte44eqsq> you\nproposed scalerMaxCrop_ and I confirmed it afterwards and then didn't\nwant to bring that up again :-). Good to know that we actually preferred\nthe same. Let's leave it like that for now. In the upcoming dewarper\nseries the variable will be removed anyways.\n\n> \n> The rest, according to my memories of your clarifications seems good\n> to me\n> Reviewed-by: Jacopo Mondi \n\nThanks,\nStefan\n\n> \n> Thanks\n> j\n> \n> > - Remove unnecessary ScalerCrop max calculation\n> >\n> > Changes in v2:\n> > - Initialize dewarperSensorCrop_ to sane defaults\n> > - Collect tags\n> > ---\n> > src/libcamera/pipeline/rkisp1/rkisp1.cpp | 50 +++++++++++++++++++-----\n> > 1 file changed, 41 insertions(+), 9 deletions(-)\n> >\n> > diff --git a/src/libcamera/pipeline/rkisp1/rkisp1.cpp b/src/libcamera/pipeline/rkisp1/rkisp1.cpp\n> > index 56192451eb3c..ef4aa38478f5 100644\n> > --- a/src/libcamera/pipeline/rkisp1/rkisp1.cpp\n> > +++ b/src/libcamera/pipeline/rkisp1/rkisp1.cpp\n> > @@ -205,6 +205,7 @@ private:\n> > \tRkISP1SelfPath selfPath_;\n> >\n> > \tstd::unique_ptr dewarper_;\n> > +\tRectangle scalerMaxCrop_;\n> > \tbool useDewarper_;\n> >\n> > \tstd::optional activeCrop_;\n> > @@ -861,6 +862,15 @@ int PipelineHandlerRkISP1::configure(Camera *camera, CameraConfiguration *c)\n> > \t\t\t\toutputCfgs.push_back(const_cast(cfg));\n> > \t\t\t\tret = dewarper_->configure(cfg, outputCfgs);\n> > \t\t\t\tuseDewarper_ = ret ? false : true;\n> > +\n> > +\t\t\t\t/*\n> > +\t\t\t\t * Calculate the crop rectangle of the data\n> > +\t\t\t\t * flowing into the dewarper in sensor\n> > +\t\t\t\t * coordinates.\n> > +\t\t\t\t */\n> > +\t\t\t\tscalerMaxCrop_ =\n> > +\t\t\t\t\toutputCrop.transformedBetween(inputCrop,\n> > +\t\t\t\t\t\t\t\t sensorInfo.analogCrop);\n> > \t\t\t}\n> > \t\t} else if (hasSelfPath_) {\n> > \t\t\tret = selfPath_.configure(cfg, format);\n> > @@ -1226,10 +1236,19 @@ int PipelineHandlerRkISP1::updateControls(RkISP1CameraData *data)\n> > \t\telse\n> > \t\t\tcropLimits = dewarper_->inputCropBounds();\n> >\n> > -\t\tcontrols[&controls::ScalerCrop] = ControlInfo(cropLimits.first,\n> > -\t\t\t\t\t\t\t cropLimits.second,\n> > -\t\t\t\t\t\t\t cropLimits.second);\n> > -\t\tactiveCrop_ = cropLimits.second;\n> > +\t\t/*\n> > +\t\t * ScalerCrop is specified to be in Sensor coordinates.\n> > +\t\t * So we need to transform the limits to sensor coordinates.\n> > +\t\t * We can safely assume that the maximum crop limit contains the\n> > +\t\t * full fov of the dewarper.\n> > +\t\t */\n> > +\t\tRectangle min = cropLimits.first.transformedBetween(cropLimits.second,\n> > +\t\t\t\t\t\t\t\t scalerMaxCrop_);\n> > +\n> > +\t\tcontrols[&controls::ScalerCrop] = ControlInfo(min,\n> > +\t\t\t\t\t\t\t scalerMaxCrop_,\n> > +\t\t\t\t\t\t\t scalerMaxCrop_);\n> > +\t\tactiveCrop_ = scalerMaxCrop_;\n> > \t}\n> >\n> > \t/* Add the IPA registered controls to list of camera controls. */\n> > @@ -1257,6 +1276,8 @@ int PipelineHandlerRkISP1::createCamera(MediaEntity *sensor)\n> > \t/* Initialize the camera properties. */\n> > \tdata->properties_ = data->sensor_->properties();\n> >\n> > +\tscalerMaxCrop_ = Rectangle(data->sensor_->resolution());\n> > +\n> > \tconst CameraSensorProperties::SensorDelays &delays = data->sensor_->sensorDelays();\n> > \tstd::unordered_map params = {\n> > \t\t{ V4L2_CID_ANALOGUE_GAIN, { delays.gainDelay, false } },\n> > @@ -1476,22 +1497,33 @@ void PipelineHandlerRkISP1::imageBufferReady(FrameBuffer *buffer)\n> > \t/* Handle scaler crop control. */\n> > \tconst auto &crop = request->controls().get(controls::ScalerCrop);\n> > \tif (crop) {\n> > -\t\tRectangle appliedRect = crop.value();\n> > +\t\tRectangle rect = crop.value();\n> > +\n> > +\t\t/*\n> > +\t\t * ScalerCrop is specified to be in Sensor coordinates.\n> > +\t\t * So we need to transform it into dewarper coordinates.\n> > +\t\t * We can safely assume that the maximum crop limit contains the\n> > +\t\t * full fov of the dewarper.\n> > +\t\t */\n> > +\t\tstd::pair cropLimits =\n> > +\t\t\tdewarper_->inputCropBounds(&data->mainPathStream_);\n> >\n> > +\t\trect = rect.transformedBetween(scalerMaxCrop_, cropLimits.second);\n> > \t\tint ret = dewarper_->setInputCrop(&data->mainPathStream_,\n> > -\t\t\t\t\t\t &appliedRect);\n> > -\t\tif (!ret && appliedRect != crop.value()) {\n> > +\t\t\t\t\t\t &rect);\n> > +\t\trect = rect.transformedBetween(cropLimits.second, scalerMaxCrop_);\n> > +\t\tif (!ret && rect != crop.value()) {\n> > \t\t\t/*\n> > \t\t\t * If the rectangle is changed by setInputCrop on the\n> > \t\t\t * dewarper, log a debug message and cache the actual\n> > \t\t\t * applied rectangle for metadata reporting.\n> > \t\t\t */\n> > \t\t\tLOG(RkISP1, Debug)\n> > -\t\t\t\t<< \"Applied rectangle \" << appliedRect.toString()\n> > +\t\t\t\t<< \"Applied rectangle \" << rect.toString()\n> > \t\t\t\t<< \" differs from requested \" << crop.value().toString();\n> > \t\t}\n> >\n> > -\t\tactiveCrop_ = appliedRect;\n> > +\t\tactiveCrop_ = rect;\n> > \t}\n> >\n> > \t/*\n> > --\n> > 2.43.0\n> >","headers":{"Return-Path":"","X-Original-To":"parsemail@patchwork.libcamera.org","Delivered-To":"parsemail@patchwork.libcamera.org","Received":["from lancelot.ideasonboard.com (lancelot.ideasonboard.com\n\t[92.243.16.209])\n\tby patchwork.libcamera.org (Postfix) with ESMTPS id 76BB5C326C\n\tfor ;\n\tMon, 16 Dec 2024 20:11:14 +0000 (UTC)","from lancelot.ideasonboard.com (localhost [IPv6:::1])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTP id B96BD67FAC;\n\tMon, 16 Dec 2024 21:11:13 +0100 (CET)","from perceval.ideasonboard.com (perceval.ideasonboard.com\n\t[213.167.242.64])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id B3CB762C8B\n\tfor ;\n\tMon, 16 Dec 2024 21:11:12 +0100 (CET)","from ideasonboard.com (unknown\n\t[IPv6:2a00:6020:448c:6c00:15e9:bfc7:2fd9:3f8a])\n\tby perceval.ideasonboard.com (Postfix) with ESMTPSA id D76BF675;\n\tMon, 16 Dec 2024 21:10:35 +0100 (CET)"],"Authentication-Results":"lancelot.ideasonboard.com; dkim=pass (1024-bit key;\n\tunprotected) header.d=ideasonboard.com header.i=@ideasonboard.com\n\theader.b=\"eg/OvCcw\"; dkim-atps=neutral","DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/simple; d=ideasonboard.com;\n\ts=mail; t=1734379835;\n\tbh=Dy7L3p9wF3I5mbtK5/eIQ8atpgKYIeQumoyFBMUDlXg=;\n\th=Date:From:To:Cc:Subject:References:In-Reply-To:From;\n\tb=eg/OvCcwmlAgY0T0wUi163/40WSJUfwwHRK+nUVOgfARAmGdH6KPULn/bhyXEa8Og\n\tX6l3SJ18Xv8Iuu85g72HVNzLePO06xfmrbd9nU7N1P+hzTFOmnpz0vY5elRi5Oc3UX\n\toSN2GW8pDAiI3N8+YnceoLgZ6H73HnP3qFp6tXIw=","Date":"Mon, 16 Dec 2024 21:11:09 +0100","From":"Stefan Klug ","To":"Jacopo Mondi ","Cc":"libcamera-devel@lists.libcamera.org, \n\tPaul Elder ","Subject":"Re: [PATCH v4 11/20] pipeline: rkisp1: Fix ScalerCrop to be in\n\tsensor coordinates","Message-ID":"","References":"<20241216154124.203650-1-stefan.klug@ideasonboard.com>\n\t<20241216154124.203650-12-stefan.klug@ideasonboard.com>\n\t","MIME-Version":"1.0","Content-Type":"text/plain; charset=utf-8","Content-Disposition":"inline","In-Reply-To":"","X-BeenThere":"libcamera-devel@lists.libcamera.org","X-Mailman-Version":"2.1.29","Precedence":"list","List-Id":"","List-Unsubscribe":",\n\t","List-Archive":"","List-Post":"","List-Help":"","List-Subscribe":",\n\t","Errors-To":"libcamera-devel-bounces@lists.libcamera.org","Sender":"\"libcamera-devel\" "}}]