[{"id":31648,"web_url":"https://patchwork.libcamera.org/comment/31648/","msgid":"<172848567239.532453.15915815094731257379@ping.linuxembedded.co.uk>","date":"2024-10-09T14:54:32","subject":"Re: [PATCH v4 10/13] libcamera: mali-c55: Correct input/output\n\tformat representation","submitter":{"id":4,"url":"https://patchwork.libcamera.org/api/people/4/","name":"Kieran Bingham","email":"kieran.bingham@ideasonboard.com"},"content":"Quoting Daniel Scally (2024-07-09 15:39:10)\n> At present we configure raw streams by looping through the pixel\n> formats we support and finding one with an associated media bus\n> format code that the sensor can produce. In the new representation\n> of raw data from the kernel driver this will not work - the sensor\n> could produce 8, 10, 12, 14 or 16 bit data and the ISP will force\n> it to RAW16, which is the only actually supported output.\n> \n> To fix the issue move to simply finding a pixel format with a bayer\n> order that matches that of the media bus format produced by the\n> sensor. If the sensor can produce multiple formats with the same\n> bayer order use the one with the largest bitdepth.\n> \n> Finally, remove the claim to support RAW formats of less than 16 bits.\n> \n> Signed-off-by: Daniel Scally \n> ---\n> src/libcamera/pipeline/mali-c55/mali-c55.cpp | 121 +++++++++++--------\n> 1 file changed, 73 insertions(+), 48 deletions(-)\n> \n> diff --git a/src/libcamera/pipeline/mali-c55/mali-c55.cpp b/src/libcamera/pipeline/mali-c55/mali-c55.cpp\n> index 156560c1..c456a798 100644\n> --- a/src/libcamera/pipeline/mali-c55/mali-c55.cpp\n> +++ b/src/libcamera/pipeline/mali-c55/mali-c55.cpp\n> @@ -57,22 +57,6 @@ const std::map maliC55FmtToCode = {\n> { formats::NV21, MEDIA_BUS_FMT_YUV10_1X30 },\n> \n> /* RAW formats, FR pipe only. */\n> - { formats::SGBRG8, MEDIA_BUS_FMT_SGBRG8_1X8 },\n> - { formats::SRGGB8, MEDIA_BUS_FMT_SRGGB8_1X8 },\n> - { formats::SBGGR8, MEDIA_BUS_FMT_SBGGR8_1X8 },\n> - { formats::SGRBG8, MEDIA_BUS_FMT_SGRBG8_1X8 },\n> - { formats::SGBRG10, MEDIA_BUS_FMT_SGBRG10_1X10 },\n> - { formats::SRGGB10, MEDIA_BUS_FMT_SRGGB10_1X10 },\n> - { formats::SBGGR10, MEDIA_BUS_FMT_SBGGR10_1X10 },\n> - { formats::SGRBG10, MEDIA_BUS_FMT_SGRBG10_1X10 },\n> - { formats::SGBRG12, MEDIA_BUS_FMT_SGBRG12_1X12 },\n> - { formats::SRGGB12, MEDIA_BUS_FMT_SRGGB12_1X12 },\n> - { formats::SBGGR12, MEDIA_BUS_FMT_SBGGR12_1X12 },\n> - { formats::SGRBG12, MEDIA_BUS_FMT_SGRBG12_1X12 },\n> - { formats::SGBRG14, MEDIA_BUS_FMT_SGBRG14_1X14 },\n> - { formats::SRGGB14, MEDIA_BUS_FMT_SRGGB14_1X14 },\n> - { formats::SBGGR14, MEDIA_BUS_FMT_SBGGR14_1X14 },\n> - { formats::SGRBG14, MEDIA_BUS_FMT_SGRBG14_1X14 },\n> { formats::SGBRG16, MEDIA_BUS_FMT_SGBRG16_1X16 },\n> { formats::SRGGB16, MEDIA_BUS_FMT_SRGGB16_1X16 },\n> { formats::SBGGR16, MEDIA_BUS_FMT_SBGGR16_1X16 },\n> @@ -98,7 +82,8 @@ public:\n> const std::vector sizes(unsigned int mbusCode) const;\n> const Size resolution() const;\n> \n> - PixelFormat bestRawFormat() const;\n> + int pixfmtToMbusCode(const PixelFormat &pixFmt) const;\n> + const PixelFormat &bestRawFormat() const;\n> \n> PixelFormat adjustRawFormat(const PixelFormat &pixFmt) const;\n> Size adjustRawSizes(const PixelFormat &pixFmt, const Size &rawSize) const;\n> @@ -208,33 +193,75 @@ const Size MaliC55CameraData::resolution() const\n> return tpgResolution_;\n> }\n> \n> -PixelFormat MaliC55CameraData::bestRawFormat() const\n> +/*\n> + * The Mali C55 ISP can only produce 16-bit RAW output in bypass modes, but the\n> + * sensors connected to it might produce 8/10/12/16 bits. We simply search the\n> + * sensor's supported formats for the one with a matching bayer order and the\n> + * greatest bitdepth.\n> + */\n> +int MaliC55CameraData::pixfmtToMbusCode(const PixelFormat &pixFmt) const\n> {\n> + auto it = maliC55FmtToCode.find(pixFmt);\n> + if (it == maliC55FmtToCode.end())\n> + return -EINVAL;\n> +\n> + BayerFormat bayerFormat = BayerFormat::fromMbusCode(it->second);\n> + if (!bayerFormat.isValid())\n> + return -EINVAL;\n> +\n> + V4L2Subdevice::Formats formats = sd_->formats(0);\n> + unsigned int sensorMbusCode = 0;\n> unsigned int bitDepth = 0;\n> - PixelFormat rawFormat;\n> \n> - /*\n> - * Iterate over all the supported PixelFormat and find the one\n> - * supported by the camera with the largest bitdepth.\n> - */\n> - for (const auto &maliFormat : maliC55FmtToCode) {\n> - PixelFormat pixFmt = maliFormat.first;\n> - if (!isFormatRaw(pixFmt))\n> + for (const auto &[code, sizes] : formats) {\n> + BayerFormat sdBayerFormat = BayerFormat::fromMbusCode(code);\n> + if (!sdBayerFormat.isValid())\n> continue;\n> \n> - unsigned int rawCode = maliFormat.second;\n> - const auto rawSizes = sizes(rawCode);\n> - if (rawSizes.empty())\n> + if (sdBayerFormat.order != bayerFormat.order)\n> continue;\n> \n> - BayerFormat bayer = BayerFormat::fromMbusCode(rawCode);\n> - if (bayer.bitDepth > bitDepth) {\n> - bitDepth = bayer.bitDepth;\n> - rawFormat = pixFmt;\n> + if (sdBayerFormat.bitDepth > bitDepth) {\n> + bitDepth = sdBayerFormat.bitDepth;\n> + sensorMbusCode = code;\n> }\n> }\n> \n> - return rawFormat;\n> + if (!sensorMbusCode)\n> + return -EINVAL;\n> +\n> + return sensorMbusCode;\n> +}\n> +\n> +/*\n> + * Find a RAW PixelFormat supported by both the ISP and the sensor.\n> + *\n> + * The situation is mildly complicated by the fact that we expect the sensor to\n> + * output something like RAW8/10/12/16, but the ISP can only accept as input\n> + * RAW20 and can only produce as output RAW16. The one constant in that is the\n> + * bayer order of the data, so we'll simply check that the sensor produces a\n> + * format with a bayer order that matches that of one of the formats we support,\n> + * and select that.\n> + */\n> +const PixelFormat &MaliC55CameraData::bestRawFormat() const\n> +{\n> + static const PixelFormat invalidPixFmt = {};\n> +\n> + for (const auto &fmt : sd_->formats(0)) {\n> + BayerFormat sensorBayer = BayerFormat::fromMbusCode(fmt.first);\n> +\n> + for (const auto &[pixFmt, rawCode] : maliC55FmtToCode) {\n> + if (!isFormatRaw(pixFmt))\n> + continue;\n> +\n> + BayerFormat bayer = BayerFormat::fromMbusCode(rawCode);\n> + if (bayer.order == sensorBayer.order)\n> + return pixFmt;\n> + }\n> + }\n> +\n> + LOG(MaliC55, Fatal) << \"Sensor doesn't provide a compatible format\";\n\nPlease don't make this Fatal. That would 'crash' libcamera applications\n(such as pipewire) in the event it was run on a sensor driver that\ndidn't have a format compatible with Mali-C55, even if perhaps there was\nanother camera that was compatible...\n\nI think this can just be Error.\n\nOther than that,\n\n\nReviewed-by: Kieran Bingham \n\n> + return invalidPixFmt;\n> }\n> \n> /*\n> @@ -243,13 +270,11 @@ PixelFormat MaliC55CameraData::bestRawFormat() const\n> */\n> PixelFormat MaliC55CameraData::adjustRawFormat(const PixelFormat &rawFmt) const\n> {\n> - /* Make sure the provided raw format is supported by the pipeline. */\n> - auto it = maliC55FmtToCode.find(rawFmt);\n> - if (it == maliC55FmtToCode.end())\n> + /* Make sure the RAW mbus code is supported by the image source. */\n> + int rawCode = pixfmtToMbusCode(rawFmt);\n> + if (rawCode < 0)\n> return bestRawFormat();\n> \n> - /* Now make sure the RAW mbus code is supported by the image source. */\n> - unsigned int rawCode = it->second;\n> const auto rawSizes = sizes(rawCode);\n> if (rawSizes.empty())\n> return bestRawFormat();\n> @@ -259,16 +284,14 @@ PixelFormat MaliC55CameraData::adjustRawFormat(const PixelFormat &rawFmt) const\n> \n> Size MaliC55CameraData::adjustRawSizes(const PixelFormat &rawFmt, const Size &size) const\n> {\n> - /* Just make sure the format is supported. */\n> - auto it = maliC55FmtToCode.find(rawFmt);\n> - if (it == maliC55FmtToCode.end())\n> - return {};\n> -\n> /* Expand the RAW size to the minimum ISP input size. */\n> Size rawSize = size.expandedTo(kMaliC55MinInputSize);\n> \n> /* Check if the size is natively supported. */\n> - unsigned int rawCode = it->second;\n> + int rawCode = pixfmtToMbusCode(rawFmt);\n> + if (rawCode < 0)\n> + return {};\n> +\n> const auto rawSizes = sizes(rawCode);\n> auto sizeIt = std::find(rawSizes.begin(), rawSizes.end(), rawSize);\n> if (sizeIt != rawSizes.end())\n> @@ -418,8 +441,7 @@ CameraConfiguration::Status MaliC55CameraConfiguration::validate()\n> \n> /* If there's a RAW config, sensor configuration follows it. */\n> if (rawConfig) {\n> - const auto it = maliC55FmtToCode.find(rawConfig->pixelFormat);\n> - sensorFormat_.code = it->second;\n> + sensorFormat_.code = data_->pixfmtToMbusCode(rawConfig->pixelFormat);\n> sensorFormat_.size = rawConfig->size.expandedTo(minSensorSize);\n> \n> return status;\n> @@ -614,7 +636,10 @@ PipelineHandlerMaliC55::generateConfiguration(Camera *camera,\n> \n> if (isRaw) {\n> /* Make sure the mbus code is supported. */\n> - unsigned int rawCode = maliFormat.second;\n> + int rawCode = data->pixfmtToMbusCode(pixFmt);\n> + if (rawCode < 0)\n> + continue;\n> +\n> const auto sizes = data->sizes(rawCode);\n> if (sizes.empty())\n> continue;\n> -- \n> 2.34.1\n>","headers":{"Return-Path":"","X-Original-To":"parsemail@patchwork.libcamera.org","Delivered-To":"parsemail@patchwork.libcamera.org","Received":["from lancelot.ideasonboard.com (lancelot.ideasonboard.com\n\t[92.243.16.209])\n\tby patchwork.libcamera.org (Postfix) with ESMTPS id 49F39C32DE\n\tfor ;\n\tWed, 9 Oct 2024 14:54:39 +0000 (UTC)","from lancelot.ideasonboard.com (localhost [IPv6:::1])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTP id 6A882618C9;\n\tWed, 9 Oct 2024 16:54:38 +0200 (CEST)","from perceval.ideasonboard.com (perceval.ideasonboard.com\n\t[IPv6:2001:4b98:dc2:55:216:3eff:fef7:d647])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id EA931618C5\n\tfor ;\n\tWed, 9 Oct 2024 16:54:35 +0200 (CEST)","from pendragon.ideasonboard.com\n\t(cpc89244-aztw30-2-0-cust6594.18-1.cable.virginm.net [86.31.185.195])\n\tby perceval.ideasonboard.com (Postfix) with ESMTPSA id 7445B2EC;\n\tWed, 9 Oct 2024 16:52:58 +0200 (CEST)"],"Authentication-Results":"lancelot.ideasonboard.com; dkim=pass (1024-bit key;\n\tunprotected) header.d=ideasonboard.com header.i=@ideasonboard.com\n\theader.b=\"i+UMt0/O\"; dkim-atps=neutral","DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/simple; d=ideasonboard.com;\n\ts=mail; t=1728485578;\n\tbh=vk9EbKRRodsa64tB49JbOwy2BdaXIilQ/84Zpl+F/6I=;\n\th=In-Reply-To:References:Subject:From:Cc:To:Date:From;\n\tb=i+UMt0/O7Bgd50JyckhAo2/CRdZVMmOCSbE8LmGMy86pvI85FSI3HBwUQqorEcGH1\n\t6hQ3bGChbqsGjvblPf+K7l93TA++TUfxQCVeWMKHwhR/aXQYtAlxqjQJXpxU2Ir2bJ\n\tirwaX09r6Mui4rwEh0w4l/HZ6yQEs6JuTUaWI9oA=","Content-Type":"text/plain; charset=\"utf-8\"","MIME-Version":"1.0","Content-Transfer-Encoding":"quoted-printable","In-Reply-To":"<20240709143913.3276983-11-dan.scally@ideasonboard.com>","References":"<20240709143913.3276983-1-dan.scally@ideasonboard.com>\n\t<20240709143913.3276983-11-dan.scally@ideasonboard.com>","Subject":"Re: [PATCH v4 10/13] libcamera: mali-c55: Correct input/output\n\tformat representation","From":"Kieran Bingham ","Cc":"Daniel Scally ","To":"Daniel Scally ,\n\tlibcamera-devel@lists.libcamera.org","Date":"Wed, 09 Oct 2024 15:54:32 +0100","Message-ID":"<172848567239.532453.15915815094731257379@ping.linuxembedded.co.uk>","User-Agent":"alot/0.10","X-BeenThere":"libcamera-devel@lists.libcamera.org","X-Mailman-Version":"2.1.29","Precedence":"list","List-Id":"","List-Unsubscribe":",\n\t","List-Archive":"","List-Post":"","List-Help":"","List-Subscribe":",\n\t","Errors-To":"libcamera-devel-bounces@lists.libcamera.org","Sender":"\"libcamera-devel\" "}}]