Patch Detail
Show a patch.
GET /api/1.1/patches/15778/?format=api
{ "id": 15778, "url": "https://patchwork.libcamera.org/api/1.1/patches/15778/?format=api", "web_url": "https://patchwork.libcamera.org/patch/15778/", "project": { "id": 1, "url": "https://patchwork.libcamera.org/api/1.1/projects/1/?format=api", "name": "libcamera", "link_name": "libcamera", "list_id": "libcamera_core", "list_email": "libcamera-devel@lists.libcamera.org", "web_url": "", "scm_url": "", "webscm_url": "" }, "msgid": "<20220504132154.9681-6-laurent.pinchart@ideasonboard.com>", "date": "2022-05-04T13:21:54", "name": "[libcamera-devel,5/5] libcamera: pipeline: simple: Support scaling on the sensor", "commit_ref": null, "pull_url": null, "state": "accepted", "archived": false, "hash": "0bf2770db7873cd4273776ad307fe583bad19c98", "submitter": { "id": 2, "url": "https://patchwork.libcamera.org/api/1.1/people/2/?format=api", "name": "Laurent Pinchart", "email": "laurent.pinchart@ideasonboard.com" }, "delegate": null, "mbox": "https://patchwork.libcamera.org/patch/15778/mbox/", "series": [ { "id": 3090, "url": "https://patchwork.libcamera.org/api/1.1/series/3090/?format=api", "web_url": "https://patchwork.libcamera.org/project/libcamera/list/?series=3090", "date": "2022-05-04T13:21:49", "name": "libcamera: pipeline: simple: Support scaling on the camea sensor", "version": 1, "mbox": "https://patchwork.libcamera.org/series/3090/mbox/" } ], "comments": "https://patchwork.libcamera.org/api/patches/15778/comments/", "check": "pending", "checks": "https://patchwork.libcamera.org/api/patches/15778/checks/", "tags": {}, "headers": { "Return-Path": "<libcamera-devel-bounces@lists.libcamera.org>", "X-Original-To": "parsemail@patchwork.libcamera.org", "Delivered-To": "parsemail@patchwork.libcamera.org", "Received": [ "from lancelot.ideasonboard.com (lancelot.ideasonboard.com\n\t[92.243.16.209])\n\tby patchwork.libcamera.org (Postfix) with ESMTPS id A1761C326E\n\tfor <parsemail@patchwork.libcamera.org>;\n\tWed, 4 May 2022 13:22:07 +0000 (UTC)", "from lancelot.ideasonboard.com (localhost [IPv6:::1])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTP id F109D6564F;\n\tWed, 4 May 2022 15:22:06 +0200 (CEST)", "from perceval.ideasonboard.com (perceval.ideasonboard.com\n\t[213.167.242.64])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id A47CF6564A\n\tfor <libcamera-devel@lists.libcamera.org>;\n\tWed, 4 May 2022 15:22:01 +0200 (CEST)", "from pendragon.lan (62-78-145-57.bb.dnainternet.fi [62.78.145.57])\n\tby perceval.ideasonboard.com (Postfix) with ESMTPSA id 1AB37835\n\tfor <libcamera-devel@lists.libcamera.org>;\n\tWed, 4 May 2022 15:22:01 +0200 (CEST)" ], "DKIM-Signature": [ "v=1; a=rsa-sha256; c=relaxed/simple; d=libcamera.org;\n\ts=mail; t=1651670527;\n\tbh=6S82CbC3QGg3iKprxwbo6/cXnfsOIuf+cMBGA3qp2R0=;\n\th=To:Date:In-Reply-To:References:Subject:List-Id:List-Unsubscribe:\n\tList-Archive:List-Post:List-Help:List-Subscribe:From:Reply-To:\n\tFrom;\n\tb=RXWt17ZzBXBqlyit8/ipkHHGeDNfzwSFjb/WvYbdCTqp19v0qAPDf/SUKs4WjB61e\n\t8+LngsUClxs8SUlL1UZEOtDA6rv8N7b3Zj+fzHpYEdQOZKJrRZzcEbdM8MQSRCR+/q\n\tkejK0hJrKQqdxNb2pYz4QcjqYGAdZ3zFCtP4i33CMQfWaESpVJswxBf18hJvyYEcVy\n\trLypCvAIalx3HBvBIV2W+C6RCIfLoDHUmVmIAF01KS2kcrzjzfsd4y/cv+twLRARxK\n\tHAfLDCV7NPuCHAyjdg0FwhKvXXoJc1MTFnwBz7FVq3uCefEVPYPWlRBDBgF5IPExGS\n\t48u9z4lUkipxg==", "v=1; a=rsa-sha256; c=relaxed/simple; d=ideasonboard.com;\n\ts=mail; t=1651670521;\n\tbh=6S82CbC3QGg3iKprxwbo6/cXnfsOIuf+cMBGA3qp2R0=;\n\th=From:To:Subject:Date:In-Reply-To:References:From;\n\tb=Ds7vp8KjlTfwFbael8MzNIB8RvVpTMOayrlA4yaxnbrzCH+ObwL94eytel3v0jzil\n\tOuQcZ9VNiMv6tAC8xJgzgOHUYw0787n0z0YiIzNcztLgHo0JpHx1yadqedbCX3qCpK\n\tXdBtJD9btIM7UcXVtF/Ep+VIM/NSVOBVRlYeNoIg=" ], "Authentication-Results": "lancelot.ideasonboard.com; dkim=pass (1024-bit key; \n\tunprotected) header.d=ideasonboard.com\n\theader.i=@ideasonboard.com\n\theader.b=\"Ds7vp8Kj\"; dkim-atps=neutral", "To": "libcamera-devel@lists.libcamera.org", "Date": "Wed, 4 May 2022 16:21:54 +0300", "Message-Id": "<20220504132154.9681-6-laurent.pinchart@ideasonboard.com>", "X-Mailer": "git-send-email 2.35.1", "In-Reply-To": "<20220504132154.9681-1-laurent.pinchart@ideasonboard.com>", "References": "<20220504132154.9681-1-laurent.pinchart@ideasonboard.com>", "MIME-Version": "1.0", "Content-Transfer-Encoding": "8bit", "Subject": "[libcamera-devel] [PATCH 5/5] libcamera: pipeline: simple: Support\n\tscaling on the sensor", "X-BeenThere": "libcamera-devel@lists.libcamera.org", "X-Mailman-Version": "2.1.29", "Precedence": "list", "List-Id": "<libcamera-devel.lists.libcamera.org>", "List-Unsubscribe": "<https://lists.libcamera.org/options/libcamera-devel>,\n\t<mailto:libcamera-devel-request@lists.libcamera.org?subject=unsubscribe>", "List-Archive": "<https://lists.libcamera.org/pipermail/libcamera-devel/>", "List-Post": "<mailto:libcamera-devel@lists.libcamera.org>", "List-Help": "<mailto:libcamera-devel-request@lists.libcamera.org?subject=help>", "List-Subscribe": "<https://lists.libcamera.org/listinfo/libcamera-devel>,\n\t<mailto:libcamera-devel-request@lists.libcamera.org?subject=subscribe>", "From": "Laurent Pinchart via libcamera-devel\n\t<libcamera-devel@lists.libcamera.org>", "Reply-To": "Laurent Pinchart <laurent.pinchart@ideasonboard.com>", "Errors-To": "libcamera-devel-bounces@lists.libcamera.org", "Sender": "\"libcamera-devel\" <libcamera-devel-bounces@lists.libcamera.org>" }, "content": "As the simple pipeline handler targets simple pipelines on the SoC side,\nit often gets used with platforms that have a YUV sensor capable of\noutputting different sizes. Extend the heuristics used for pipeline\ndiscovery and configuration to scale as much as possible on the sensor\nside, in order to minimize the required bandwidth.\n\nSigned-off-by: Laurent Pinchart <laurent.pinchart@ideasonboard.com>\n---\n src/libcamera/pipeline/simple/simple.cpp | 142 +++++++++++++++++------\n 1 file changed, 109 insertions(+), 33 deletions(-)", "diff": "diff --git a/src/libcamera/pipeline/simple/simple.cpp b/src/libcamera/pipeline/simple/simple.cpp\nindex e7fda239d6b5..5858deb01ee4 100644\n--- a/src/libcamera/pipeline/simple/simple.cpp\n+++ b/src/libcamera/pipeline/simple/simple.cpp\n@@ -115,16 +115,25 @@ LOG_DEFINE_CATEGORY(SimplePipeline)\n *\n * The simple pipeline handler configures the pipeline by propagating V4L2\n * subdev formats from the camera sensor to the video node. The format is first\n- * set on the camera sensor's output, using the native camera sensor\n- * resolution. Then, on every link in the pipeline, the format is retrieved on\n- * the link source and set unmodified on the link sink.\n+ * set on the camera sensor's output, picking a resolution supported by the\n+ * sensor that best matches the needs of the requested streams. Then, on every\n+ * link in the pipeline, the format is retrieved on the link source and set\n+ * unmodified on the link sink.\n *\n- * When initializating the camera data, this above procedure is repeated for\n- * every media bus format supported by the camera sensor. Upon reaching the\n- * video node, the pixel formats compatible with the media bus format are\n- * enumerated. Each of those pixel formats corresponds to one possible pipeline\n- * configuration, stored as an instance of SimpleCameraData::Configuration in\n- * the SimpleCameraData::formats_ map.\n+ * The best sensor resolution is selected using a heuristic that tries to\n+ * minimize the required bus and memory bandwidth, as the simple pipeline\n+ * handler is typically used on smaller, less powerful systems. To avoid the\n+ * need to upscale, the pipeline handler picks the smallest sensor resolution\n+ * large enough to accommodate the need of all streams. Resolutions that\n+ * significantly restrict the field of view are ignored.\n+ *\n+ * When initializating the camera data, the above format propagation procedure\n+ * is repeated for every media bus format and size supported by the camera\n+ * sensor. Upon reaching the video node, the pixel formats compatible with the\n+ * media bus format are enumerated. Each combination of the input media bus\n+ * format, output pixel format and output size are recorded in an instance of\n+ * the SimpleCameraData::Configuration structure, stored in the\n+ * SimpleCameraData::configs_ vector.\n *\n * Format Conversion and Scaling\n * -----------------------------\n@@ -243,7 +252,7 @@ public:\n \tV4L2VideoDevice *video_;\n \n \tstd::vector<Configuration> configs_;\n-\tstd::map<PixelFormat, const Configuration *> formats_;\n+\tstd::map<PixelFormat, std::vector<const Configuration *>> formats_;\n \n \tstd::unique_ptr<SimpleConverter> converter_;\n \tstd::vector<std::unique_ptr<FrameBuffer>> converterBuffers_;\n@@ -471,26 +480,24 @@ int SimpleCameraData::init()\n \n \t/*\n \t * Generate the list of possible pipeline configurations by trying each\n-\t * media bus format supported by the sensor.\n+\t * media bus format and size supported by the sensor.\n \t */\n-\tfor (unsigned int code : sensor_->mbusCodes())\n-\t\ttryPipeline(code, sensor_->resolution());\n+\tfor (unsigned int code : sensor_->mbusCodes()) {\n+\t\tfor (const Size &size : sensor_->sizes(code))\n+\t\t\ttryPipeline(code, size);\n+\t}\n \n \tif (configs_.empty()) {\n \t\tLOG(SimplePipeline, Error) << \"No valid configuration found\";\n \t\treturn -EINVAL;\n \t}\n \n-\t/*\n-\t * Map the pixel formats to configurations. Any previously stored value\n-\t * is overwritten, as the pipeline handler currently doesn't care about\n-\t * how a particular PixelFormat is achieved.\n-\t */\n+\t/* Map the pixel formats to configurations. */\n \tfor (const Configuration &config : configs_) {\n-\t\tformats_[config.captureFormat] = &config;\n+\t\tformats_[config.captureFormat].push_back(&config);\n \n \t\tfor (PixelFormat fmt : config.outputFormats)\n-\t\t\tformats_[fmt] = &config;\n+\t\t\tformats_[fmt].push_back(&config);\n \t}\n \n \tproperties_ = sensor_->properties();\n@@ -511,8 +518,10 @@ void SimpleCameraData::tryPipeline(unsigned int code, const Size &size)\n \tint ret = setupFormats(&format, V4L2Subdevice::TryFormat);\n \tif (ret < 0) {\n \t\t/* Pipeline configuration failed, skip this configuration. */\n+\t\tformat.mbus_code = code;\n+\t\tformat.size = size;\n \t\tLOG(SimplePipeline, Debug)\n-\t\t\t<< \"Media bus code \" << utils::hex(code, 4)\n+\t\t\t<< \"Sensor format \" << format\n \t\t\t<< \" not supported for this pipeline\";\n \t\treturn;\n \t}\n@@ -783,22 +792,70 @@ CameraConfiguration::Status SimpleCameraConfiguration::validate()\n \t\tstatus = Adjusted;\n \t}\n \n+\t/* Find the largest stream size. */\n+\tSize maxStreamSize;\n+\tfor (const StreamConfiguration &cfg : config_)\n+\t\tmaxStreamSize.expandTo(cfg.size);\n+\n+\tLOG(SimplePipeline, Debug)\n+\t\t<< \"Largest stream size is \" << maxStreamSize;\n+\n \t/*\n-\t * Pick a configuration for the pipeline based on the pixel format for\n-\t * the streams (ordered from highest to lowest priority). Default to\n-\t * the first pipeline configuration if no streams requests a supported\n+\t * Find the best configuration for the pipeline using a heuristic.\n+\t * First select the pixel format based on the streams (which are\n+\t * considered ordered from highest to lowest priority). Default to the\n+\t * first pipeline configuration if no streams requests a supported\n \t * pixel format.\n \t */\n-\tpipeConfig_ = data_->formats_.begin()->second;\n+\tconst std::vector<const SimpleCameraData::Configuration *> *configs =\n+\t\t&data_->formats_.begin()->second;\n \n \tfor (const StreamConfiguration &cfg : config_) {\n \t\tauto it = data_->formats_.find(cfg.pixelFormat);\n \t\tif (it != data_->formats_.end()) {\n-\t\t\tpipeConfig_ = it->second;\n+\t\t\tconfigs = &it->second;\n \t\t\tbreak;\n \t\t}\n \t}\n \n+\t/*\n+\t * \\todo Pick the best sensor output media bus format when the\n+\t * requested pixel format can be produced from multiple sensor media\n+\t * bus formats.\n+\t */\n+\n+\t/*\n+\t * Then pick, among the possible configuration for the pixel format,\n+\t * the smallest sensor resolution that can accommodate all streams\n+\t * without upscaling.\n+\t */\n+\tconst SimpleCameraData::Configuration *maxPipeConfig = nullptr;\n+\tpipeConfig_ = nullptr;\n+\n+\tfor (const SimpleCameraData::Configuration *pipeConfig : *configs) {\n+\t\tconst Size &size = pipeConfig->captureSize;\n+\n+\t\tif (size.width >= maxStreamSize.width &&\n+\t\t size.height >= maxStreamSize.height) {\n+\t\t\tif (!pipeConfig_ || size < pipeConfig_->captureSize)\n+\t\t\t\tpipeConfig_ = pipeConfig;\n+\t\t}\n+\n+\t\tif (!maxPipeConfig || maxPipeConfig->captureSize < size)\n+\t\t\tmaxPipeConfig = pipeConfig;\n+\t}\n+\n+\t/* If no configuration was large enough, select the largest one. */\n+\tif (!pipeConfig_)\n+\t\tpipeConfig_ = maxPipeConfig;\n+\n+\tLOG(SimplePipeline, Debug)\n+\t\t<< \"Picked \"\n+\t\t<< V4L2SubdeviceFormat{ pipeConfig_->code, pipeConfig_->sensorSize, {} }\n+\t\t<< \" -> \" << pipeConfig_->captureSize\n+\t\t<< \"-\" << pipeConfig_->captureFormat\n+\t\t<< \" for max stream size \" << maxStreamSize;\n+\n \t/*\n \t * Adjust the requested streams.\n \t *\n@@ -891,13 +948,32 @@ CameraConfiguration *SimplePipelineHandler::generateConfiguration(Camera *camera\n \n \t/* Create the formats map. */\n \tstd::map<PixelFormat, std::vector<SizeRange>> formats;\n-\tstd::transform(data->formats_.begin(), data->formats_.end(),\n-\t\t std::inserter(formats, formats.end()),\n-\t\t [](const auto &format) -> decltype(formats)::value_type {\n-\t\t\t const PixelFormat &pixelFormat = format.first;\n-\t\t\t const Size &size = format.second->captureSize;\n-\t\t\t return { pixelFormat, { size } };\n-\t\t });\n+\n+\tfor (const SimpleCameraData::Configuration &cfg : data->configs_) {\n+\t\tfor (PixelFormat format : cfg.outputFormats)\n+\t\t\tformats[format].push_back(cfg.outputSizes);\n+\t}\n+\n+\t/* Sort the sizes and merge any consecutive overlapping ranges. */\n+\tfor (auto &[format, sizes] : formats) {\n+\t\tstd::sort(sizes.begin(), sizes.end(),\n+\t\t\t [](SizeRange &a, SizeRange &b) {\n+\t\t\t\t return a.min < b.min;\n+\t\t\t });\n+\n+\t\tauto cur = sizes.begin();\n+\t\tauto next = cur;\n+\n+\t\twhile (++next != sizes.end()) {\n+\t\t\tif (cur->max.width >= next->min.width &&\n+\t\t\t cur->max.height >= next->min.height)\n+\t\t\t\tcur->max = next->max;\n+\t\t\telse if (++cur != next)\n+\t\t\t\t*cur = *next;\n+\t\t}\n+\n+\t\tsizes.erase(++cur, sizes.end());\n+\t}\n \n \t/*\n \t * Create the stream configurations. Take the first entry in the formats\n", "prefixes": [ "libcamera-devel", "5/5" ] }