[{"id":37061,"web_url":"https://patchwork.libcamera.org/comment/37061/","msgid":"<176409267333.567526.4979173454280875275@ping.linuxembedded.co.uk>","date":"2025-11-25T17:44:33","subject":"Re: [PATCH v3 23/29] libcamera: rkisp1: Handle requested orientation\n\tusing dewarper","submitter":{"id":4,"url":"https://patchwork.libcamera.org/api/people/4/","name":"Kieran Bingham","email":"kieran.bingham@ideasonboard.com"},"content":"Quoting Stefan Klug (2025-11-25 16:28:35)\n> When the dewarper is present it can handle arbitrary orientations\n> specified in the requested camera configuration. In that case handle all\n> transformations inside the dewarper (even if the sensor supports some of\n> them) because that makes it easier to handle coordinates for lens\n> dewarping inside the dewarper.\n> \n> This complicates the path selection a bit, as for transformations that\n> include a transpose, the format before the dewarper has swapped\n> width/height.\n> \n> Signed-off-by: Stefan Klug \n> Reviewed-by: Paul Elder \n> \n> ---\n> \n> Changes in v3:\n> - Collected tag\n> \n> Changes in v2:\n> - Fix path validation for cases where a transpose is involved\n\nLooks good...\n\n\nReviewed-by: Kieran Bingham \n\n> ---\n> src/libcamera/pipeline/rkisp1/rkisp1.cpp | 89 ++++++++++++++++++------\n> 1 file changed, 67 insertions(+), 22 deletions(-)\n> \n> diff --git a/src/libcamera/pipeline/rkisp1/rkisp1.cpp b/src/libcamera/pipeline/rkisp1/rkisp1.cpp\n> index e3dca8b837e8..746eeffab207 100644\n> --- a/src/libcamera/pipeline/rkisp1/rkisp1.cpp\n> +++ b/src/libcamera/pipeline/rkisp1/rkisp1.cpp\n> @@ -593,11 +593,6 @@ CameraConfiguration::Status RkISP1CameraConfiguration::validate()\n> status = Adjusted;\n> }\n> \n> - Orientation requestedOrientation = orientation;\n> - combinedTransform_ = data_->sensor_->computeTransform(&orientation);\n> - if (orientation != requestedOrientation)\n> - status = Adjusted;\n> -\n> /*\n> * Simultaneous capture of raw and processed streams isn't possible.\n> * Let the first stream decide on the type.\n> @@ -622,7 +617,23 @@ CameraConfiguration::Status RkISP1CameraConfiguration::validate()\n> }\n> }\n> \n> + /*\n> + * If the dewarper supports orientation adjustments, apply that completely\n> + * there. Even if the sensor supports flips, it is beneficial to do that\n> + * in the dewarper so that lens dewarping happens on the unflipped image\n> + */\n> + bool transposeAfterIsp = false;\n> bool useDewarper = (data_->canUseDewarper_ && !isRaw);\n> + if (useDewarper) {\n> + combinedTransform_ = orientation / data_->sensor_->mountingOrientation();\n> + if (!!(combinedTransform_ & Transform::Transpose))\n> + transposeAfterIsp = true;\n> + } else {\n> + Orientation requestedOrientation = orientation;\n> + combinedTransform_ = data_->sensor_->computeTransform(&orientation);\n> + if (orientation != requestedOrientation)\n> + status = Adjusted;\n> + }\n> \n> /*\n> * If there are more than one stream in the configuration figure out the\n> @@ -638,12 +649,18 @@ CameraConfiguration::Status RkISP1CameraConfiguration::validate()\n> \n> /*\n> * Validate the configuration against the desired path and, if the\n> - * platform supports it, the dewarper.\n> + * platform supports it, the dewarper. While iterating over the\n> + * configurations collect the smallest common sensor format.\n> */\n> + Size accumulatedSensorSize;\n> auto validateConfig = [&](StreamConfiguration &cfg, RkISP1Path *path,\n> Stream *stream, Status expectedStatus) {\n> StreamConfiguration tryCfg = cfg;\n> \n> + /* Need to validate the path before the transpose */\n> + if (transposeAfterIsp)\n> + tryCfg.size.transpose();\n> +\n> Status ret = path->validate(sensor, sensorConfig, &tryCfg);\n> if (ret == Invalid)\n> return false;\n> @@ -652,6 +669,8 @@ CameraConfiguration::Status RkISP1CameraConfiguration::validate()\n> (expectedStatus == Valid && ret == Adjusted))\n> return false;\n> \n> + Size sensorSize = tryCfg.size;\n> +\n> if (useDewarper) {\n> /*\n> * The dewarper output is independent of the ISP path.\n> @@ -674,6 +693,8 @@ CameraConfiguration::Status RkISP1CameraConfiguration::validate()\n> \n> cfg = tryCfg;\n> cfg.setStream(stream);\n> +\n> + accumulatedSensorSize = std::max(accumulatedSensorSize, sensorSize);\n> return true;\n> };\n> \n> @@ -724,13 +745,6 @@ CameraConfiguration::Status RkISP1CameraConfiguration::validate()\n> return Invalid;\n> }\n> \n> - /* Select the sensor format. */\n> - Size maxSize;\n> -\n> - for (const StreamConfiguration &cfg : config_) {\n> - maxSize = std::max(maxSize, cfg.size);\n> - }\n> -\n> std::vector mbusCodes;\n> \n> if (isRaw) {\n> @@ -741,7 +755,7 @@ CameraConfiguration::Status RkISP1CameraConfiguration::validate()\n> [](const auto &value) { return value.second; });\n> }\n> \n> - sensorFormat_ = sensor->getFormat(mbusCodes, maxSize,\n> + sensorFormat_ = sensor->getFormat(mbusCodes, accumulatedSensorSize,\n> mainPath->maxResolution());\n> \n> if (sensorFormat_.size.isNull())\n> @@ -776,6 +790,22 @@ PipelineHandlerRkISP1::generateConfiguration(Camera *camera,\n> if (roles.empty())\n> return config;\n> \n> + Transform transform = Transform::Identity;\n> + Size previewSize = kRkISP1PreviewSize;\n> + bool transposeAfterIsp = false;\n> + if (data->canUseDewarper_) {\n> + transform = Orientation::Rotate0 / data->sensor_->mountingOrientation();\n> + if (!!(transform & Transform::Transpose))\n> + transposeAfterIsp = true;\n> + }\n> +\n> + /*\n> + * In case of a transpose transform we need to create a path for the\n> + * transposed size.\n> + */\n> + if (transposeAfterIsp)\n> + previewSize.transpose();\n> +\n> /*\n> * As the ISP can't output different color spaces for the main and self\n> * path, pick a sensible default color space based on the role of the\n> @@ -804,7 +834,7 @@ PipelineHandlerRkISP1::generateConfiguration(Camera *camera,\n> if (!colorSpace)\n> colorSpace = ColorSpace::Sycc;\n> \n> - size = kRkISP1PreviewSize;\n> + size = previewSize;\n> break;\n> \n> case StreamRole::VideoRecording:\n> @@ -812,7 +842,7 @@ PipelineHandlerRkISP1::generateConfiguration(Camera *camera,\n> if (!colorSpace)\n> colorSpace = ColorSpace::Rec709;\n> \n> - size = kRkISP1PreviewSize;\n> + size = previewSize;\n> break;\n> \n> case StreamRole::Raw:\n> @@ -854,6 +884,9 @@ PipelineHandlerRkISP1::generateConfiguration(Camera *camera,\n> if (!cfg.pixelFormat.isValid())\n> return nullptr;\n> \n> + if (transposeAfterIsp && role != StreamRole::Raw)\n> + cfg.size.transpose();\n> +\n> cfg.colorSpace = colorSpace;\n> cfg.bufferCount = kRkISP1MinBufferCount;\n> config->addConfiguration(cfg);\n> @@ -876,6 +909,19 @@ int PipelineHandlerRkISP1::configure(Camera *camera, CameraConfiguration *c)\n> if (ret)\n> return ret;\n> \n> + const PixelFormat &streamFormat = config->at(0).pixelFormat;\n> + const PixelFormatInfo &info = PixelFormatInfo::info(streamFormat);\n> + isRaw_ = info.colourEncoding == PixelFormatInfo::ColourEncodingRAW;\n> + data->usesDewarper_ = data->canUseDewarper_ && !isRaw_;\n> +\n> + Transform transform = config->combinedTransform();\n> + bool transposeAfterIsp = false;\n> + if (data->usesDewarper_) {\n> + if (!!(transform & Transform::Transpose))\n> + transposeAfterIsp = true;\n> + transform = Transform::Identity;\n> + }\n> +\n> /*\n> * Configure the format on the sensor output and propagate it through\n> * the pipeline.\n> @@ -885,10 +931,10 @@ int PipelineHandlerRkISP1::configure(Camera *camera, CameraConfiguration *c)\n> \n> if (config->sensorConfig)\n> ret = sensor->applyConfiguration(*config->sensorConfig,\n> - config->combinedTransform(),\n> + transform,\n> &format);\n> else\n> - ret = sensor->setFormat(&format, config->combinedTransform());\n> + ret = sensor->setFormat(&format, transform);\n> \n> if (ret < 0)\n> return ret;\n> @@ -915,10 +961,6 @@ int PipelineHandlerRkISP1::configure(Camera *camera, CameraConfiguration *c)\n> << \" crop \" << inputCrop;\n> \n> Rectangle outputCrop = inputCrop;\n> - const PixelFormat &streamFormat = config->at(0).pixelFormat;\n> - const PixelFormatInfo &info = PixelFormatInfo::info(streamFormat);\n> - isRaw_ = info.colourEncoding == PixelFormatInfo::ColourEncodingRAW;\n> - data->usesDewarper_ = dewarper_ && !isRaw_;\n> \n> /* YUYV8_2X8 is required on the ISP source path pad for YUV output. */\n> if (!isRaw_)\n> @@ -1013,11 +1055,14 @@ int PipelineHandlerRkISP1::configure(Camera *camera, CameraConfiguration *c)\n> if (ret)\n> return ret;\n> \n> + dewarper_->setTransform(cfg.stream(), config->combinedTransform());\n> /*\n> * Apply a default scaler crop that keeps the\n> * aspect ratio.\n> */\n> Size size = cfg.size;\n> + if (transposeAfterIsp)\n> + size.transpose();\n> size = sensorCrop.size().boundedToAspectRatio(size);\n> \n> ControlList ctrls;\n> -- \n> 2.51.0\n>","headers":{"Return-Path":"","X-Original-To":"parsemail@patchwork.libcamera.org","Delivered-To":"parsemail@patchwork.libcamera.org","Received":["from lancelot.ideasonboard.com (lancelot.ideasonboard.com\n\t[92.243.16.209])\n\tby patchwork.libcamera.org (Postfix) with ESMTPS id 2B8ABC32F1\n\tfor ;\n\tTue, 25 Nov 2025 17:44:39 +0000 (UTC)","from lancelot.ideasonboard.com (localhost [IPv6:::1])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTP id 6D8B260A9E;\n\tTue, 25 Nov 2025 18:44:38 +0100 (CET)","from perceval.ideasonboard.com (perceval.ideasonboard.com\n\t[IPv6:2001:4b98:dc2:55:216:3eff:fef7:d647])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id 267A5609D8\n\tfor ;\n\tTue, 25 Nov 2025 18:44:37 +0100 (CET)","from pendragon.ideasonboard.com\n\t(cpc89244-aztw30-2-0-cust6594.18-1.cable.virginm.net [86.31.185.195])\n\tby perceval.ideasonboard.com (Postfix) with ESMTPSA id C848343;\n\tTue, 25 Nov 2025 18:42:27 +0100 (CET)"],"Authentication-Results":"lancelot.ideasonboard.com; dkim=pass (1024-bit key;\n\tunprotected) header.d=ideasonboard.com header.i=@ideasonboard.com\n\theader.b=\"oSlPz3oS\"; dkim-atps=neutral","DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/simple; d=ideasonboard.com;\n\ts=mail; t=1764092547;\n\tbh=u4e/BWe8AwMeRtaBhABP9p6ZNhZo0XU/f0VpLpypQSc=;\n\th=In-Reply-To:References:Subject:From:Cc:To:Date:From;\n\tb=oSlPz3oSW8B2SRPQZozKdna+f2w4H0fBIlJpQCW+GI3jCS8KAOf0x61feMfFRS/sd\n\tIKfFZ4I3wzIguiq2wz9sYiERWiH5V0htBBHcGmSWEwLeZpeWhuoIIKpKQkR02NbOzC\n\tQyzBjUzRm9DuJ0Cn6CHu4QxsyrAYhhzOLyH7u2No=","Content-Type":"text/plain; charset=\"utf-8\"","MIME-Version":"1.0","Content-Transfer-Encoding":"quoted-printable","In-Reply-To":"<20251125162851.2301793-24-stefan.klug@ideasonboard.com>","References":"<20251125162851.2301793-1-stefan.klug@ideasonboard.com>\n\t<20251125162851.2301793-24-stefan.klug@ideasonboard.com>","Subject":"Re: [PATCH v3 23/29] libcamera: rkisp1: Handle requested orientation\n\tusing dewarper","From":"Kieran Bingham ","Cc":"Stefan Klug ,\n\tPaul Elder ","To":"Stefan Klug ,\n\tlibcamera-devel@lists.libcamera.org","Date":"Tue, 25 Nov 2025 17:44:33 +0000","Message-ID":"<176409267333.567526.4979173454280875275@ping.linuxembedded.co.uk>","User-Agent":"alot/0.9.1","X-BeenThere":"libcamera-devel@lists.libcamera.org","X-Mailman-Version":"2.1.29","Precedence":"list","List-Id":"","List-Unsubscribe":",\n\t","List-Archive":"","List-Post":"","List-Help":"","List-Subscribe":",\n\t","Errors-To":"libcamera-devel-bounces@lists.libcamera.org","Sender":"\"libcamera-devel\" "}}]