[{"id":37098,"web_url":"https://patchwork.libcamera.org/comment/37098/","msgid":"<176426129907.890597.4848170234631360658@ping.linuxembedded.co.uk>","date":"2025-11-27T16:34:59","subject":"Re: [PATCH v3 2/2] pipeline: imx8-isi: Integrating MediaPipeline\n\tclass","submitter":{"id":4,"url":"https://patchwork.libcamera.org/api/people/4/","name":"Kieran Bingham","email":"kieran.bingham@ideasonboard.com"},"content":"Quoting Antoine Bouyer (2025-11-27 15:45:18)\n> From: Andrei Gansari \n> \n> This change integrates the MediaPipeline class into the imx8-isi\n> pipeline handler. Purpose is to allow a dynamic discovery and\n> configuration of the actual subdevices graph between the sensor and\n> the ISI crossbar. This brings support for more complex topologies and\n> simplifies the implementation.\n> \n> Signed-off-by: Andrei Gansari \n> Signed-off-by: Antoine Bouyer \n> Reviewed-by: Jacopo Mondi \n> ---\n> src/libcamera/pipeline/imx8-isi/imx8-isi.cpp | 159 ++++++++++++-------\n> 1 file changed, 98 insertions(+), 61 deletions(-)\n> \n> diff --git a/src/libcamera/pipeline/imx8-isi/imx8-isi.cpp b/src/libcamera/pipeline/imx8-isi/imx8-isi.cpp\n> index 049d9b1c44cb..008155ff21a7 100644\n> --- a/src/libcamera/pipeline/imx8-isi/imx8-isi.cpp\n> +++ b/src/libcamera/pipeline/imx8-isi/imx8-isi.cpp\n> @@ -25,6 +25,7 @@\n> #include \"libcamera/internal/camera_sensor.h\"\n> #include \"libcamera/internal/device_enumerator.h\"\n> #include \"libcamera/internal/media_device.h\"\n> +#include \"libcamera/internal/media_pipeline.h\"\n> #include \"libcamera/internal/pipeline_handler.h\"\n> #include \"libcamera/internal/v4l2_subdevice.h\"\n> #include \"libcamera/internal/v4l2_videodevice.h\"\n> @@ -62,14 +63,15 @@ public:\n> unsigned int getYuvMediaBusFormat(const PixelFormat &pixelFormat) const;\n> unsigned int getMediaBusFormat(PixelFormat *pixelFormat) const;\n> \n> + /* All entities, from the sensor to the ISI. */\n> + MediaPipeline mediaPipeline_;\n> +\n> std::unique_ptr sensor_;\n> - std::unique_ptr csis_;\n> \n> std::vector streams_;\n> \n> std::vector enabledStreams_;\n> \n> - unsigned int xbarSink_ = 0;\n> unsigned int xbarSourceOffset_ = 0;\n> };\n> \n> @@ -141,6 +143,8 @@ private:\n> \n> void bufferReady(FrameBuffer *buffer);\n> \n> + std::vector locateSensors(MediaDevice *media);\n> +\n> std::shared_ptr isiDev_;\n> \n> std::unique_ptr crossbar_;\n> @@ -164,10 +168,6 @@ int ISICameraData::init()\n> if (!sensor_)\n> return -ENODEV;\n> \n> - int ret = csis_->open();\n> - if (ret)\n> - return ret;\n> -\n> properties_ = sensor_->properties();\n> \n> return 0;\n> @@ -811,18 +811,29 @@ int PipelineHandlerISI::configure(Camera *camera, CameraConfiguration *c)\n> {\n> ISICameraConfiguration *camConfig = static_cast(c);\n> ISICameraData *data = cameraData(camera);\n> + CameraSensor *sensor = data->sensor_.get();\n> + int ret;\n> \n> - /* Apply format to the sensor, CSIS receiver and crossbar sink pad. */\n> - V4L2SubdeviceFormat format = camConfig->sensorFormat_;\n> - int ret = data->sensor_->setFormat(&format);\n> - if (ret)\n> + /*\n> + * Enable the links all the way up to the ISI, through any connected CSI\n> + * receiver and optional formatter.\n> + */\n> + ret = data->mediaPipeline_.initLinks();\n> + if (ret) {\n> + LOG(ISI, Error) << \"Failed to set up pipe links\";\n> return ret;\n> + }\n> \n> - ret = data->csis_->setFormat(0, &format);\n> + /*\n> + * Configure the format on the sensor output and propagate it through\n> + * the pipeline.\n> + */\n> + V4L2SubdeviceFormat format = camConfig->sensorFormat_;\n> + ret = sensor->setFormat(&format);\n> if (ret)\n> return ret;\n> \n> - ret = crossbar_->setFormat(data->xbarSink_, &format);\n> + ret = data->mediaPipeline_.configure(sensor, &format);\n> if (ret)\n> return ret;\n> \n> @@ -979,13 +990,8 @@ bool PipelineHandlerISI::match(DeviceEnumerator *enumerator)\n> return false;\n> \n> /* Count the number of sensors, to create one camera per sensor. */\n> - unsigned cameraCount = 0;\n> - for (MediaEntity *entity : isiDev_->entities()) {\n> - if (entity->function() != MEDIA_ENT_F_CAM_SENSOR)\n> - continue;\n> -\n> - cameraCount++;\n> - }\n> + std::vector sensorEntities = locateSensors(isiDev_.get());\n> + unsigned int cameraCount = sensorEntities.size();\n> \n> if (!cameraCount) {\n> LOG(ISI, Error) << \"No camera sensor found\";\n> @@ -1048,60 +1054,28 @@ bool PipelineHandlerISI::match(DeviceEnumerator *enumerator)\n> * sensors to get at least one dedicated pipe.\n> */\n> unsigned int numCameras = 0;\n> - unsigned int numSinks = 0;\n> const unsigned int xbarFirstSource = crossbar_->entity()->pads().size() - pipes_.size();\n> const unsigned int maxStreams = pipes_.size() / cameraCount;\n> \n> - for (MediaPad *pad : crossbar_->entity()->pads()) {\n> - unsigned int sink = numSinks;\n> -\n> - if (!(pad->flags() & MEDIA_PAD_FL_SINK))\n> - continue;\n> -\n> - /*\n> - * Count each crossbar sink pad to correctly configure\n> - * routing and format for this camera.\n> - */\n> - numSinks++;\n> -\n> - if (pad->links().empty())\n> - continue;\n> -\n> - MediaEntity *csi = pad->links()[0]->source()->entity();\n> - if (csi->pads().size() != 2) {\n> - LOG(ISI, Debug) << \"Skip unsupported CSI-2 receiver \"\n> - << csi->name();\n> - continue;\n> - }\n> -\n> - pad = csi->pads()[0];\n> - if (!(pad->flags() & MEDIA_PAD_FL_SINK) || pad->links().empty())\n> - continue;\n> -\n> - MediaEntity *sensor = pad->links()[0]->source()->entity();\n> - if (sensor->function() != MEDIA_ENT_F_CAM_SENSOR) {\n> - LOG(ISI, Debug) << \"Skip unsupported subdevice \"\n> - << sensor->name();\n> - continue;\n> - }\n> -\n> - /* All links are immutable except the sensor -> csis link. */\n> - const MediaPad *sensorSrc = sensor->getPadByIndex(0);\n> - sensorSrc->links()[0]->setEnabled(true);\n> -\n> + for (MediaEntity *sensor : sensorEntities) {\n> /* Create the camera data. */\n> std::unique_ptr data =\n> std::make_unique(this, maxStreams);\n> \n> + ret = data->mediaPipeline_.init(sensor, \"crossbar\");\n> + if (ret)\n> + continue;\n> +\n> + const MediaPipeline::Entity *xbarEntity = &data->mediaPipeline_.entities().back();\n> + unsigned int xbarSinkIndex = xbarEntity->sink->index();\n> +\n> data->sensor_ = CameraSensorFactoryBase::create(sensor);\n> - data->csis_ = std::make_unique(csi);\n> - data->xbarSink_ = sink;\n> data->xbarSourceOffset_ = numCameras * data->streams_.size();\n> \n> LOG(ISI, Debug)\n> << \"cam\" << numCameras\n> << \" streams \" << data->streams_.size()\n> - << \" sink \" << data->xbarSink_\n> + << \" sink \" << xbarSinkIndex\n> << \" offset \" << data->xbarSourceOffset_;\n> \n> ret = data->init();\n> @@ -1120,7 +1094,7 @@ bool PipelineHandlerISI::match(DeviceEnumerator *enumerator)\n> /* Add routes to the crossbar switch routing table. */\n> for (unsigned i = 0; i < data->streams_.size(); i++) {\n> unsigned int sourcePad = xbarFirstSource + data->xbarSourceOffset_ + i;\n> - routing_.emplace_back(V4L2Subdevice::Stream{ data->xbarSink_, 0 },\n> + routing_.emplace_back(V4L2Subdevice::Stream{ xbarSinkIndex, 0 },\n> V4L2Subdevice::Stream{ sourcePad, 0 },\n> V4L2_SUBDEV_ROUTE_FL_ACTIVE);\n> }\n> @@ -1163,6 +1137,69 @@ void PipelineHandlerISI::bufferReady(FrameBuffer *buffer)\n> completeRequest(request);\n> }\n> \n> +/* Original function taken from simple.cpp */\n\nPerhaps this should be part of MediaPipeline object so that we don't\nduplicate it ?\n\n\nBut CI is green,\nhttps://gitlab.freedesktop.org/camera/libcamera/-/pipelines/1555963\n\nAnd factoring this out can be on top when we know who to make it usable\nfor more users.\n\nReviewed-by: Kieran Bingham \n\nMerging.\n\n\n> +std::vector\n> +PipelineHandlerISI::locateSensors(MediaDevice *media)\n> +{\n> + std::vector entities;\n> +\n> + /*\n> + * Gather all the camera sensor entities based on the function they\n> + * expose.\n> + */\n> + for (MediaEntity *entity : media->entities()) {\n> + if (entity->function() == MEDIA_ENT_F_CAM_SENSOR)\n> + entities.push_back(entity);\n> + }\n> +\n> + if (entities.empty())\n> + return {};\n> +\n> + /*\n> + * Sensors can be made of multiple entities. For instance, a raw sensor\n> + * can be connected to an ISP, and the combination of both should be\n> + * treated as one sensor. To support this, as a crude heuristic, check\n> + * the downstream entity from the camera sensor, and if it is an ISP,\n> + * use it instead of the sensor.\n> + */\n> + std::vector sensors;\n> +\n> + for (MediaEntity *entity : entities) {\n> + /*\n> + * Locate the downstream entity by following the first link\n> + * from a source pad.\n> + */\n> + const MediaLink *link = nullptr;\n> +\n> + for (const MediaPad *pad : entity->pads()) {\n> + if ((pad->flags() & MEDIA_PAD_FL_SOURCE) &&\n> + !pad->links().empty()) {\n> + link = pad->links()[0];\n> + break;\n> + }\n> + }\n> +\n> + if (!link)\n> + continue;\n> +\n> + MediaEntity *remote = link->sink()->entity();\n> + if (remote->function() == MEDIA_ENT_F_PROC_VIDEO_ISP)\n> + sensors.push_back(remote);\n> + else\n> + sensors.push_back(entity);\n> + }\n> +\n> + /*\n> + * Remove duplicates, in case multiple sensors are connected to the\n> + * same ISP.\n> + */\n> + std::sort(sensors.begin(), sensors.end());\n> + auto last = std::unique(sensors.begin(), sensors.end());\n> + sensors.erase(last, sensors.end());\n> +\n> + return sensors;\n> +}\n> +\n> REGISTER_PIPELINE_HANDLER(PipelineHandlerISI, \"imx8-isi\")\n> \n> } /* namespace libcamera */\n> -- \n> 2.51.2\n>","headers":{"Return-Path":"","X-Original-To":"parsemail@patchwork.libcamera.org","Delivered-To":"parsemail@patchwork.libcamera.org","Received":["from lancelot.ideasonboard.com (lancelot.ideasonboard.com\n\t[92.243.16.209])\n\tby patchwork.libcamera.org (Postfix) with ESMTPS id 59474C3257\n\tfor ;\n\tThu, 27 Nov 2025 16:35:04 +0000 (UTC)","from lancelot.ideasonboard.com (localhost [IPv6:::1])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTP id 8B34460A9D;\n\tThu, 27 Nov 2025 17:35:03 +0100 (CET)","from perceval.ideasonboard.com (perceval.ideasonboard.com\n\t[IPv6:2001:4b98:dc2:55:216:3eff:fef7:d647])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id 662BA606A0\n\tfor ;\n\tThu, 27 Nov 2025 17:35:02 +0100 (CET)","from pendragon.ideasonboard.com\n\t(cpc89244-aztw30-2-0-cust6594.18-1.cable.virginm.net [86.31.185.195])\n\tby perceval.ideasonboard.com (Postfix) with ESMTPSA id 8A2E0A8F;\n\tThu, 27 Nov 2025 17:32:51 +0100 (CET)"],"Authentication-Results":"lancelot.ideasonboard.com; dkim=pass (1024-bit key;\n\tunprotected) header.d=ideasonboard.com header.i=@ideasonboard.com\n\theader.b=\"RXtLe3b9\"; dkim-atps=neutral","DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/simple; d=ideasonboard.com;\n\ts=mail; t=1764261171;\n\tbh=XunnOYOcFo/rwwv9RGavcnfCnvsh/UWnIrpUnA0SSuw=;\n\th=In-Reply-To:References:Subject:From:Cc:To:Date:From;\n\tb=RXtLe3b9gXdG/LWMCXWWSneLFLfPi0FWp6icjJwLzXSZEgVp/+e8hIrdDO327emTx\n\t1jZFgxrA5y+8/bhKnD3Ai+G58kZVX8EiM5s4o6iN/TBvX/9iq5kosU26JMnKMN2yaD\n\tUGclAZQxy+HWK3/45f54pgLVTf8HhSn+q86iMUPE=","Content-Type":"text/plain; charset=\"utf-8\"","MIME-Version":"1.0","Content-Transfer-Encoding":"quoted-printable","In-Reply-To":"<20251127154519.2038844-3-antoine.bouyer@nxp.com>","References":"<20251127154519.2038844-1-antoine.bouyer@nxp.com>\n\t<20251127154519.2038844-3-antoine.bouyer@nxp.com>","Subject":"Re: [PATCH v3 2/2] pipeline: imx8-isi: Integrating MediaPipeline\n\tclass","From":"Kieran Bingham ","Cc":"julien.vuillaumier@nxp.com, jacopo.mondi@ideasonboard.com,\n\tbarnabas.pocze@ideasonboard.com, isaac.scott@ideasonboard.com,\n\tAndrei Gansari ,\n\tAntoine Bouyer ","To":"Antoine Bouyer ,\n\tlibcamera-devel@lists.libcamera.org","Date":"Thu, 27 Nov 2025 16:34:59 +0000","Message-ID":"<176426129907.890597.4848170234631360658@ping.linuxembedded.co.uk>","User-Agent":"alot/0.9.1","X-BeenThere":"libcamera-devel@lists.libcamera.org","X-Mailman-Version":"2.1.29","Precedence":"list","List-Id":"","List-Unsubscribe":",\n\t","List-Archive":"","List-Post":"","List-Help":"","List-Subscribe":",\n\t","Errors-To":"libcamera-devel-bounces@lists.libcamera.org","Sender":"\"libcamera-devel\" "}}]