From patchwork Sun May 10 11:58:05 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Patchwork-Submitter: Laurent Pinchart X-Patchwork-Id: 3740 Return-Path: Received: from perceval.ideasonboard.com (perceval.ideasonboard.com [IPv6:2001:4b98:dc2:55:216:3eff:fef7:d647]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id B38D860BDC for ; Sun, 10 May 2020 13:58:20 +0200 (CEST) Authentication-Results: lancelot.ideasonboard.com; dkim=pass (1024-bit key; unprotected) header.d=ideasonboard.com header.i=@ideasonboard.com header.b="MiOsq26V"; dkim-atps=neutral Received: from pendragon.bb.dnainternet.fi (81-175-216-236.bb.dnainternet.fi [81.175.216.236]) by perceval.ideasonboard.com (Postfix) with ESMTPSA id E96C673E; Sun, 10 May 2020 13:58:19 +0200 (CEST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=ideasonboard.com; s=mail; t=1589111900; bh=AqHTyIgG4uJULpInqC95U3DCKIG3ZeY0cyXDPh3EdnE=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=MiOsq26VELgiOZS/ddX0hWLnuyC3emIZKQSjPtQGmhq7VQEG9hJlrOQ7n6V6dVO/Y C3euzoz32ysbthmfjAV4eLs/r5SoTHcFZQ6j/n3b/5xPDq5cfrf1ZrT3ROpx5YtMWL LvAXRuvRbGS2it5AiCIUQkj8K/E8XPQSRtVtrhiA= From: Laurent Pinchart To: libcamera-devel@lists.libcamera.org Cc: Martijn Braam Date: Sun, 10 May 2020 14:58:05 +0300 Message-Id: <20200510115810.21938-2-laurent.pinchart@ideasonboard.com> X-Mailer: git-send-email 2.26.2 In-Reply-To: <20200510115810.21938-1-laurent.pinchart@ideasonboard.com> References: <20200510115810.21938-1-laurent.pinchart@ideasonboard.com> MIME-Version: 1.0 Subject: [libcamera-devel] [PATCH v5 1/6] include: linux: Extend VIDIOC_ENUM_FMT to support MC-centric devices X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Sun, 10 May 2020 11:58:20 -0000 This integrates the Linux kernel upstream commits e5b6b07a1b45 ("media: v4l2: Extend VIDIOC_ENUM_FMT to support MC-centric devices") and f645e6256bd1 ("media: v4l2-dev/ioctl: Add V4L2_CAP_IO_MC") that are scheduled for v5.8. Signed-off-by: Laurent Pinchart Reviewed-by: Niklas Söderlund --- include/linux/videodev2.h | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/include/linux/videodev2.h b/include/linux/videodev2.h index ab40b3272ed2..66f34fc05e3e 100644 --- a/include/linux/videodev2.h +++ b/include/linux/videodev2.h @@ -476,6 +476,8 @@ struct v4l2_capability { #define V4L2_CAP_TOUCH 0x10000000 /* Is a touch device */ +#define V4L2_CAP_IO_MC 0x20000000 /* Is input/output controlled by the media controller */ + #define V4L2_CAP_DEVICE_CAPS 0x80000000 /* sets device capabilities field */ /* @@ -769,7 +771,8 @@ struct v4l2_fmtdesc { __u32 flags; __u8 description[32]; /* Description string */ __u32 pixelformat; /* Format fourcc */ - __u32 reserved[4]; + __u32 mbus_code; /* Media bus code */ + __u32 reserved[3]; }; #define V4L2_FMT_FLAG_COMPRESSED 0x0001 From patchwork Sun May 10 11:58:06 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Patchwork-Submitter: Laurent Pinchart X-Patchwork-Id: 3741 Return-Path: Received: from perceval.ideasonboard.com (perceval.ideasonboard.com [IPv6:2001:4b98:dc2:55:216:3eff:fef7:d647]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id 1805360BDC for ; Sun, 10 May 2020 13:58:21 +0200 (CEST) Authentication-Results: lancelot.ideasonboard.com; dkim=pass (1024-bit key; unprotected) header.d=ideasonboard.com header.i=@ideasonboard.com header.b="pXo7CTsN"; dkim-atps=neutral Received: from pendragon.bb.dnainternet.fi (81-175-216-236.bb.dnainternet.fi [81.175.216.236]) by perceval.ideasonboard.com (Postfix) with ESMTPSA id AB56CE1E; Sun, 10 May 2020 13:58:20 +0200 (CEST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=ideasonboard.com; s=mail; t=1589111900; bh=sVnTQMDv4xkPyP0pRkuopt2ayTAwvBYZyttY3EM1SqY=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=pXo7CTsNeBDz0rygbVx5QEg35sfDfU7pv3rBpKAX/UdyrBa7NsW5clb2mznYYgqc/ Y+1hESXPw1VQfL29H67X02rLcEqN3CsiJ1IWu7vbIofe5XLvko/STg5BhXOVbWUcRF PVjtu9PYK/l9ELfk3uBYtAPrzY2pGRhY/dT+nOTg= From: Laurent Pinchart To: libcamera-devel@lists.libcamera.org Cc: Martijn Braam Date: Sun, 10 May 2020 14:58:06 +0300 Message-Id: <20200510115810.21938-3-laurent.pinchart@ideasonboard.com> X-Mailer: git-send-email 2.26.2 In-Reply-To: <20200510115810.21938-1-laurent.pinchart@ideasonboard.com> References: <20200510115810.21938-1-laurent.pinchart@ideasonboard.com> MIME-Version: 1.0 Subject: [libcamera-devel] [PATCH v5 2/6] libcamera: v4l2_videodevice: Support filtering formats by media bus code X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Sun, 10 May 2020 11:58:21 -0000 Add support for the recent V4L2 extension to VIDIOC_ENUM_FMT that allows filtering pixel formats by media bus codes. Signed-off-by: Laurent Pinchart Reviewed-by: Niklas Söderlund --- src/libcamera/include/v4l2_videodevice.h | 4 ++-- src/libcamera/v4l2_videodevice.cpp | 16 +++++++++++++--- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/src/libcamera/include/v4l2_videodevice.h b/src/libcamera/include/v4l2_videodevice.h index 982fee6bf740..94565b97b6b1 100644 --- a/src/libcamera/include/v4l2_videodevice.h +++ b/src/libcamera/include/v4l2_videodevice.h @@ -187,7 +187,7 @@ public: int getFormat(V4L2DeviceFormat *format); int setFormat(V4L2DeviceFormat *format); - std::map> formats(); + std::map> formats(uint32_t code = 0); int setSelection(unsigned int target, Rectangle *rect); @@ -225,7 +225,7 @@ private: int getFormatSingleplane(V4L2DeviceFormat *format); int setFormatSingleplane(V4L2DeviceFormat *format); - std::vector enumPixelformats(); + std::vector enumPixelformats(uint32_t code); std::vector enumSizes(V4L2PixelFormat pixelFormat); int requestBuffers(unsigned int count, enum v4l2_memory memoryType); diff --git a/src/libcamera/v4l2_videodevice.cpp b/src/libcamera/v4l2_videodevice.cpp index e95b0c01cc85..4b9f8b5c0413 100644 --- a/src/libcamera/v4l2_videodevice.cpp +++ b/src/libcamera/v4l2_videodevice.cpp @@ -917,16 +917,19 @@ int V4L2VideoDevice::setFormatSingleplane(V4L2DeviceFormat *format) /** * \brief Enumerate all pixel formats and frame sizes + * \param[in] code Restrict formats to this media bus code. * * Enumerate all pixel formats and frame sizes supported by the video device. + * If the \a code argument is not zero, only formats compatible with that media + * bus code will be enumerated. * * \return A list of the supported video device formats */ -std::map> V4L2VideoDevice::formats() +std::map> V4L2VideoDevice::formats(uint32_t code) { std::map> formats; - for (V4L2PixelFormat pixelFormat : enumPixelformats()) { + for (V4L2PixelFormat pixelFormat : enumPixelformats(code)) { std::vector sizes = enumSizes(pixelFormat); if (sizes.empty()) return {}; @@ -944,15 +947,22 @@ std::map> V4L2VideoDevice::formats() return formats; } -std::vector V4L2VideoDevice::enumPixelformats() +std::vector V4L2VideoDevice::enumPixelformats(uint32_t code) { std::vector formats; int ret; + if (code && !(caps_.device_caps() & V4L2_CAP_IO_MC)) { + LOG(V4L2, Error) + << "Media bus code filtering not supported by the device"; + return {}; + } + for (unsigned int index = 0; ; index++) { struct v4l2_fmtdesc pixelformatEnum = {}; pixelformatEnum.index = index; pixelformatEnum.type = bufferType_; + pixelformatEnum.mbus_code = code; ret = ioctl(VIDIOC_ENUM_FMT, &pixelformatEnum); if (ret) From patchwork Sun May 10 11:58:07 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Patchwork-Submitter: Laurent Pinchart X-Patchwork-Id: 3742 Return-Path: Received: from perceval.ideasonboard.com (perceval.ideasonboard.com [IPv6:2001:4b98:dc2:55:216:3eff:fef7:d647]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id 8096260BDC for ; Sun, 10 May 2020 13:58:21 +0200 (CEST) Authentication-Results: lancelot.ideasonboard.com; dkim=pass (1024-bit key; unprotected) header.d=ideasonboard.com header.i=@ideasonboard.com header.b="oTZ31S3M"; dkim-atps=neutral Received: from pendragon.bb.dnainternet.fi (81-175-216-236.bb.dnainternet.fi [81.175.216.236]) by perceval.ideasonboard.com (Postfix) with ESMTPSA id 0E15FE3D; Sun, 10 May 2020 13:58:21 +0200 (CEST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=ideasonboard.com; s=mail; t=1589111901; bh=I3nuds+nBrvBLVbAEdY4FjBIF1LoAFvAEkbYaXt1E40=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=oTZ31S3Mc2DFtLC5wXxjbpKFHqxBkX9rl35Roqe2JC4c1XXukTW9+Ml67eDmz9kHu i8OrcbDiV32+4mdiwF6t5ijBF2nGWhGP5wlfA/EbR7Y0TjGZ86i2AZZFQ4GVbaITjy EvwjK8rL6blfDMRsbikNR8r0tOl0GAdQcei+01Ck= From: Laurent Pinchart To: libcamera-devel@lists.libcamera.org Cc: Martijn Braam Date: Sun, 10 May 2020 14:58:07 +0300 Message-Id: <20200510115810.21938-4-laurent.pinchart@ideasonboard.com> X-Mailer: git-send-email 2.26.2 In-Reply-To: <20200510115810.21938-1-laurent.pinchart@ideasonboard.com> References: <20200510115810.21938-1-laurent.pinchart@ideasonboard.com> MIME-Version: 1.0 Subject: [libcamera-devel] [PATCH v5 3/6] libcamera: pipeline: Add a simple pipeline handler X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Sun, 10 May 2020 11:58:21 -0000 From: Martijn Braam This new pipeline handler aims at supporting any simple device without requiring any device-specific code. Simple devices are currently defined as a graph made of one or multiple camera sensors and a single video node, with each sensor connected to the video node through a linear pipeline. The simple pipeline handler will automatically parse the media graph, enumerate sensors, build supported stream configurations, and configure the pipeline, without any device-specific knowledge. It doesn't support configuration of any processing in the pipeline at the moment, but may be extended to support simple processing such as format conversion or scaling in the future. The only device-specific information in the pipeline handler is the list of supported drivers, required for device matching. We may be able to remove this in the future by matching with the simple pipeline handler as a last resort option, after all other pipeline handlers have been tried. Signed-off-by: Martijn Braam Signed-off-by: Laurent Pinchart Reviewed-by: Kieran Bingham Reviewed-by: Niklas Söderlund Reviewed-by: Andrey Konovalov --- Changes since v4: - Improved link selection comment - Rebase on top of V4L2PixelFormat changes Changes since v3: - Fix typos - Improve comments - Add local variable to increase readability - Move list of supported drivers to the top of the file Changes since v2: - Log an error when setupFormats() fail - Propagate getFormat() and setFormat() errors to the caller of setupFormats() - Reorder variable declarations in validate() - Add \todo comment related to the selection of the default format - Use log Error instead of Info if pipeline isn't valid - Rebase on top of V4L2PixelFormat Changes since v1: - Rebase on top of buffer API rework - Expose stream formats - Rework camera data config --- meson_options.txt | 2 +- src/libcamera/pipeline/simple/meson.build | 3 + src/libcamera/pipeline/simple/simple.cpp | 720 ++++++++++++++++++++++ 3 files changed, 724 insertions(+), 1 deletion(-) create mode 100644 src/libcamera/pipeline/simple/meson.build create mode 100644 src/libcamera/pipeline/simple/simple.cpp diff --git a/meson_options.txt b/meson_options.txt index 6464df837cc3..166429f8583e 100644 --- a/meson_options.txt +++ b/meson_options.txt @@ -14,7 +14,7 @@ option('gstreamer', option('pipelines', type : 'array', - choices : ['ipu3', 'rkisp1', 'uvcvideo', 'vimc'], + choices : ['ipu3', 'rkisp1', 'simple', 'uvcvideo', 'vimc'], description : 'Select which pipeline handlers to include') option('test', diff --git a/src/libcamera/pipeline/simple/meson.build b/src/libcamera/pipeline/simple/meson.build new file mode 100644 index 000000000000..4945a3e173cf --- /dev/null +++ b/src/libcamera/pipeline/simple/meson.build @@ -0,0 +1,3 @@ +libcamera_sources += files([ + 'simple.cpp', +]) diff --git a/src/libcamera/pipeline/simple/simple.cpp b/src/libcamera/pipeline/simple/simple.cpp new file mode 100644 index 000000000000..95417500b6b5 --- /dev/null +++ b/src/libcamera/pipeline/simple/simple.cpp @@ -0,0 +1,720 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2020, Laurent Pinchart + * Copyright (C) 2019, Martijn Braam + * + * simple.cpp - Pipeline handler for simple pipelines + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include +#include +#include + +#include "camera_sensor.h" +#include "device_enumerator.h" +#include "log.h" +#include "media_device.h" +#include "pipeline_handler.h" +#include "v4l2_subdevice.h" +#include "v4l2_videodevice.h" + +namespace libcamera { + +LOG_DEFINE_CATEGORY(SimplePipeline) + +class SimplePipelineHandler; + +namespace { + +static const char * const drivers[] = { + "imx7-csi", + "sun6i-csi", +}; + +} /* namespace */ + +class SimpleCameraData : public CameraData +{ +public: + SimpleCameraData(PipelineHandler *pipe, MediaEntity *sensor, + MediaEntity *video); + + bool isValid() const { return sensor_ != nullptr; } + std::set streams() { return { &stream_ }; } + + int init(); + int setupLinks(); + int setupFormats(V4L2SubdeviceFormat *format, + V4L2Subdevice::Whence whence); + + struct Entity { + MediaEntity *entity; + MediaLink *link; + }; + + struct Configuration { + uint32_t code; + PixelFormat pixelFormat; + Size size; + }; + + Stream stream_; + std::unique_ptr sensor_; + std::list entities_; + + std::vector configs_; + std::map formats_; +}; + +class SimpleCameraConfiguration : public CameraConfiguration +{ +public: + SimpleCameraConfiguration(Camera *camera, SimpleCameraData *data); + + Status validate() override; + + const V4L2SubdeviceFormat &sensorFormat() { return sensorFormat_; } + +private: + /* + * The SimpleCameraData instance is guaranteed to be valid as long as + * the corresponding Camera instance is valid. In order to borrow a + * reference to the camera data, store a new reference to the camera. + */ + std::shared_ptr camera_; + const SimpleCameraData *data_; + + V4L2SubdeviceFormat sensorFormat_; +}; + +class SimplePipelineHandler : public PipelineHandler +{ +public: + SimplePipelineHandler(CameraManager *manager); + ~SimplePipelineHandler(); + + CameraConfiguration *generateConfiguration(Camera *camera, + const StreamRoles &roles) override; + int configure(Camera *camera, CameraConfiguration *config) override; + + int exportFrameBuffers(Camera *camera, Stream *stream, + std::vector> *buffers) override; + + int start(Camera *camera) override; + void stop(Camera *camera) override; + + bool match(DeviceEnumerator *enumerator) override; + + V4L2VideoDevice *video() { return video_; } + V4L2Subdevice *subdev(const MediaEntity *entity); + +protected: + int queueRequestDevice(Camera *camera, Request *request) override; + +private: + SimpleCameraData *cameraData(const Camera *camera) + { + return static_cast( + PipelineHandler::cameraData(camera)); + } + + int initLinks(); + + int createCamera(MediaEntity *sensor); + + void bufferReady(FrameBuffer *buffer); + + MediaDevice *media_; + V4L2VideoDevice *video_; + std::map subdevs_; + + Camera *activeCamera_; +}; + +/* ----------------------------------------------------------------------------- + * Camera Data + */ + +SimpleCameraData::SimpleCameraData(PipelineHandler *pipe, MediaEntity *sensor, + MediaEntity *video) + : CameraData(pipe) +{ + int ret; + + /* + * Walk the pipeline towards the video node and store all entities + * along the way. + */ + MediaEntity *source = sensor; + + while (source) { + /* If we have reached the video node, we're done. */ + if (source == video) + break; + + /* Use the first output pad that has links. */ + MediaPad *sourcePad = nullptr; + for (MediaPad *pad : source->pads()) { + if ((pad->flags() & MEDIA_PAD_FL_SOURCE) && + !pad->links().empty()) { + sourcePad = pad; + break; + } + } + + if (!sourcePad) + return; + + /* + * Use the first link that is enabled or can be enabled (not + * immutable). + */ + MediaLink *sourceLink = nullptr; + for (MediaLink *link : sourcePad->links()) { + if ((link->flags() & MEDIA_LNK_FL_ENABLED) || + !(link->flags() & MEDIA_LNK_FL_IMMUTABLE)) { + sourceLink = link; + break; + } + } + + if (!sourceLink) + return; + + entities_.push_back({ source, sourceLink }); + + source = sourceLink->sink()->entity(); + + /* Avoid infinite loops. */ + auto iter = std::find_if(entities_.begin(), entities_.end(), + [&](const Entity &entity) { + return entity.entity == source; + }); + if (iter != entities_.end()) { + LOG(SimplePipeline, Info) << "Loop detected in pipeline"; + return; + } + } + + /* We have a valid pipeline, create the camera sensor. */ + sensor_ = std::make_unique(sensor); + ret = sensor_->init(); + if (ret) { + sensor_.reset(); + return; + } +} + +int SimpleCameraData::init() +{ + SimplePipelineHandler *pipe = static_cast(pipe_); + V4L2VideoDevice *video = pipe->video(); + int ret; + + /* + * Enumerate the possible pipeline configurations. For each media bus + * format supported by the sensor, propagate the formats through the + * pipeline, and enumerate the corresponding possible V4L2 pixel + * formats on the video node. + */ + for (unsigned int code : sensor_->mbusCodes()) { + V4L2SubdeviceFormat format{ code, sensor_->resolution() }; + + /* + * Setup links first as some subdev drivers take active links + * into account to propagate TRY formats. Such is life :-( + */ + ret = setupLinks(); + if (ret < 0) + return ret; + + ret = setupFormats(&format, V4L2Subdevice::TryFormat); + if (ret < 0) { + LOG(SimplePipeline, Error) + << "Failed to setup pipeline for media bus code " + << utils::hex(code, 4); + return ret; + } + + std::map> videoFormats = + video->formats(format.mbus_code); + + LOG(SimplePipeline, Debug) + << "Adding configuration for " << format.size.toString() + << " in pixel formats [ " + << utils::join(videoFormats, ", ", + [](const auto &f) { + return f.first.toString(); + }) + << " ]"; + + /* + * Store the configuration in the formats_ map, mapping the + * PixelFormat to the corresponding configuration. Any + * previously stored value is overwritten, as the pipeline + * handler currently doesn't care about how a particular + * PixelFormat is achieved. + */ + for (const auto &videoFormat : videoFormats) { + PixelFormat pixelFormat = videoFormat.first.toPixelFormat(); + if (!pixelFormat) + continue; + + Configuration config; + config.code = code; + config.pixelFormat = pixelFormat; + config.size = format.size; + + formats_[pixelFormat] = config; + } + } + + if (formats_.empty()) { + LOG(SimplePipeline, Error) << "No valid configuration found"; + return -EINVAL; + } + + return 0; +} + +int SimpleCameraData::setupLinks() +{ + int ret; + + /* + * Configure all links along the pipeline. Some entities may not allow + * multiple sink links to be enabled together, even on different sink + * pads. We must thus start by disabling all sink links (but the one we + * want to enable) before enabling the pipeline link. + */ + for (SimpleCameraData::Entity &e : entities_) { + MediaEntity *remote = e.link->sink()->entity(); + for (MediaPad *pad : remote->pads()) { + for (MediaLink *link : pad->links()) { + if (link == e.link) + continue; + + if ((link->flags() & MEDIA_LNK_FL_ENABLED) && + !(link->flags() & MEDIA_LNK_FL_IMMUTABLE)) { + ret = link->setEnabled(false); + if (ret < 0) + return ret; + } + } + } + + if (!(e.link->flags() & MEDIA_LNK_FL_ENABLED)) { + ret = e.link->setEnabled(true); + if (ret < 0) + return ret; + } + } + + return 0; +} + +int SimpleCameraData::setupFormats(V4L2SubdeviceFormat *format, + V4L2Subdevice::Whence whence) +{ + SimplePipelineHandler *pipe = static_cast(pipe_); + int ret; + + /* + * Configure the format on the sensor output and propagate it through + * the pipeline. + */ + ret = sensor_->setFormat(format); + if (ret < 0) + return ret; + + for (const Entity &e : entities_) { + MediaLink *link = e.link; + MediaPad *source = link->source(); + MediaPad *sink = link->sink(); + + if (source->entity() != sensor_->entity()) { + V4L2Subdevice *subdev = pipe->subdev(source->entity()); + ret = subdev->getFormat(source->index(), format, whence); + if (ret < 0) + return ret; + } + + if (sink->entity()->function() != MEDIA_ENT_F_IO_V4L) { + V4L2Subdevice *subdev = pipe->subdev(sink->entity()); + ret = subdev->setFormat(sink->index(), format, whence); + if (ret < 0) + return ret; + } + + LOG(SimplePipeline, Debug) + << "Link '" << source->entity()->name() + << "':" << source->index() + << " -> '" << sink->entity()->name() + << "':" << sink->index() + << " configured with format " << format->toString(); + } + + return 0; +} + +/* ----------------------------------------------------------------------------- + * Camera Configuration + */ + +SimpleCameraConfiguration::SimpleCameraConfiguration(Camera *camera, + SimpleCameraData *data) + : CameraConfiguration(), camera_(camera->shared_from_this()), + data_(data) +{ +} + +CameraConfiguration::Status SimpleCameraConfiguration::validate() +{ + Status status = Valid; + + if (config_.empty()) + return Invalid; + + /* Cap the number of entries to the available streams. */ + if (config_.size() > 1) { + config_.resize(1); + status = Adjusted; + } + + StreamConfiguration &cfg = config_[0]; + + /* Adjust the pixel format. */ + auto it = data_->formats_.find(cfg.pixelFormat); + if (it == data_->formats_.end()) + it = data_->formats_.begin(); + + PixelFormat pixelFormat = it->first; + if (cfg.pixelFormat != pixelFormat) { + LOG(SimplePipeline, Debug) << "Adjusting pixel format"; + cfg.pixelFormat = pixelFormat; + status = Adjusted; + } + + const SimpleCameraData::Configuration &pipeConfig = it->second; + if (cfg.size != pipeConfig.size) { + LOG(SimplePipeline, Debug) + << "Adjusting size from " << cfg.size.toString() + << " to " << pipeConfig.size.toString(); + cfg.size = pipeConfig.size; + status = Adjusted; + } + + cfg.bufferCount = 3; + + return status; +} + +/* ----------------------------------------------------------------------------- + * Pipeline Handler + */ + +SimplePipelineHandler::SimplePipelineHandler(CameraManager *manager) + : PipelineHandler(manager), video_(nullptr) +{ +} + +SimplePipelineHandler::~SimplePipelineHandler() +{ + delete video_; +} + +CameraConfiguration *SimplePipelineHandler::generateConfiguration(Camera *camera, + const StreamRoles &roles) +{ + SimpleCameraData *data = cameraData(camera); + CameraConfiguration *config = + new SimpleCameraConfiguration(camera, data); + + if (roles.empty()) + return config; + + /* Create the formats map. */ + std::map> formats; + std::transform(data->formats_.begin(), data->formats_.end(), + std::inserter(formats, formats.end()), + [](const auto &format) -> decltype(formats)::value_type { + const PixelFormat &pixelFormat = format.first; + const Size &size = format.second.size; + return { pixelFormat, { size } }; + }); + + /* + * Create the stream configuration. Take the first entry in the formats + * map as the default, for lack of a better option. + * + * \todo Implement a better way to pick the default format + */ + StreamConfiguration cfg{ StreamFormats{ formats } }; + cfg.pixelFormat = formats.begin()->first; + cfg.size = formats.begin()->second[0].max; + + config->addConfiguration(cfg); + + config->validate(); + + return config; +} + +int SimplePipelineHandler::configure(Camera *camera, CameraConfiguration *c) +{ + SimpleCameraConfiguration *config = + static_cast(c); + SimpleCameraData *data = cameraData(camera); + StreamConfiguration &cfg = config->at(0); + int ret; + + /* + * Configure links on the pipeline and propagate formats from the + * sensor to the video node. + */ + ret = data->setupLinks(); + if (ret < 0) + return ret; + + const SimpleCameraData::Configuration &pipeConfig = + data->formats_[cfg.pixelFormat]; + + V4L2SubdeviceFormat format{ pipeConfig.code, data->sensor_->resolution() }; + + ret = data->setupFormats(&format, V4L2Subdevice::ActiveFormat); + if (ret < 0) + return ret; + + /* Configure the video node. */ + V4L2PixelFormat videoFormat = video_->toV4L2PixelFormat(cfg.pixelFormat); + + V4L2DeviceFormat outputFormat = {}; + outputFormat.fourcc = videoFormat; + outputFormat.size = cfg.size; + + ret = video_->setFormat(&outputFormat); + if (ret) + return ret; + + if (outputFormat.size != cfg.size || outputFormat.fourcc != videoFormat) { + LOG(SimplePipeline, Error) + << "Unable to configure capture in " << cfg.toString(); + return -EINVAL; + } + + cfg.setStream(&data->stream_); + + return 0; +} + +int SimplePipelineHandler::exportFrameBuffers(Camera *camera, Stream *stream, + std::vector> *buffers) +{ + unsigned int count = stream->configuration().bufferCount; + + return video_->exportBuffers(count, buffers); +} + +int SimplePipelineHandler::start(Camera *camera) +{ + SimpleCameraData *data = cameraData(camera); + unsigned int count = data->stream_.configuration().bufferCount; + + int ret = video_->importBuffers(count); + if (ret < 0) + return ret; + + ret = video_->streamOn(); + if (ret < 0) { + video_->releaseBuffers(); + return ret; + } + + activeCamera_ = camera; + + return 0; +} + +void SimplePipelineHandler::stop(Camera *camera) +{ + video_->streamOff(); + video_->releaseBuffers(); + activeCamera_ = nullptr; +} + +int SimplePipelineHandler::queueRequestDevice(Camera *camera, Request *request) +{ + SimpleCameraData *data = cameraData(camera); + Stream *stream = &data->stream_; + + FrameBuffer *buffer = request->findBuffer(stream); + if (!buffer) { + LOG(SimplePipeline, Error) + << "Attempt to queue request with invalid stream"; + return -ENOENT; + } + + return video_->queueBuffer(buffer); +} + +/* ----------------------------------------------------------------------------- + * Match and Setup + */ + +bool SimplePipelineHandler::match(DeviceEnumerator *enumerator) +{ + for (const char *driver : drivers) { + DeviceMatch dm(driver); + media_ = acquireMediaDevice(enumerator, dm); + if (media_) + break; + } + + if (!media_) + return false; + + /* + * Locate sensors and video nodes. We only support pipelines with at + * least one sensor and exactly one video capture node. + */ + std::vector sensors; + std::vector videos; + + for (MediaEntity *entity : media_->entities()) { + switch (entity->function()) { + case MEDIA_ENT_F_CAM_SENSOR: + sensors.push_back(entity); + break; + + case MEDIA_ENT_F_IO_V4L: + if (entity->pads().size() == 1 && + (entity->pads()[0]->flags() & MEDIA_PAD_FL_SINK)) + videos.push_back(entity); + break; + + default: + break; + } + } + + if (sensors.empty()) { + LOG(SimplePipeline, Error) << "No sensor found"; + return false; + } + + if (videos.size() != 1) { + LOG(SimplePipeline, Error) + << "Pipeline with " << videos.size() + << " video capture nodes is not supported"; + return false; + } + + /* Locate and open the capture video node. */ + video_ = new V4L2VideoDevice(videos[0]); + if (video_->open() < 0) + return false; + + if (video_->caps().isMultiplanar()) { + LOG(SimplePipeline, Error) + << "V4L2 multiplanar devices are not supported"; + return false; + } + + video_->bufferReady.connect(this, &SimplePipelineHandler::bufferReady); + + /* + * Create one camera data instance for each sensor and gather all + * entities in all pipelines. + */ + std::vector> pipelines; + std::set entities; + + pipelines.reserve(sensors.size()); + + for (MediaEntity *sensor : sensors) { + std::unique_ptr data = + std::make_unique(this, sensor, + videos[0]); + if (!data->isValid()) { + LOG(SimplePipeline, Error) + << "No valid pipeline for sensor '" + << sensor->name() << "', skipping"; + continue; + } + + for (SimpleCameraData::Entity &entity : data->entities_) + entities.insert(entity.entity); + + pipelines.push_back(std::move(data)); + } + + if (entities.empty()) + return false; + + /* Create and open V4L2Subdev instances for all the entities. */ + for (MediaEntity *entity : entities) { + auto elem = subdevs_.emplace(std::piecewise_construct, + std::forward_as_tuple(entity), + std::forward_as_tuple(entity)); + V4L2Subdevice *subdev = &elem.first->second; + int ret = subdev->open(); + if (ret < 0) { + LOG(SimplePipeline, Error) + << "Failed to open " << subdev->deviceNode() + << ": " << strerror(-ret); + return false; + } + } + + /* Initialize each pipeline and register a corresponding camera. */ + for (std::unique_ptr &data : pipelines) { + int ret = data->init(); + if (ret < 0) + continue; + + std::shared_ptr camera = + Camera::create(this, data->sensor_->entity()->name(), + data->streams()); + registerCamera(std::move(camera), std::move(data)); + } + + return true; +} + +V4L2Subdevice *SimplePipelineHandler::subdev(const MediaEntity *entity) +{ + auto iter = subdevs_.find(entity); + if (iter == subdevs_.end()) + return nullptr; + + return &iter->second; +} + +/* ----------------------------------------------------------------------------- + * Buffer Handling + */ + +void SimplePipelineHandler::bufferReady(FrameBuffer *buffer) +{ + ASSERT(activeCamera_); + Request *request = buffer->request(); + completeBuffer(activeCamera_, request, buffer); + completeRequest(activeCamera_, request); +} + +REGISTER_PIPELINE_HANDLER(SimplePipelineHandler); + +} /* namespace libcamera */ From patchwork Sun May 10 11:58:08 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Patchwork-Submitter: Laurent Pinchart X-Patchwork-Id: 3743 Return-Path: Received: from perceval.ideasonboard.com (perceval.ideasonboard.com [IPv6:2001:4b98:dc2:55:216:3eff:fef7:d647]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id EF09560BDC for ; Sun, 10 May 2020 13:58:21 +0200 (CEST) Authentication-Results: lancelot.ideasonboard.com; dkim=pass (1024-bit key; unprotected) header.d=ideasonboard.com header.i=@ideasonboard.com header.b="veGBnF4c"; dkim-atps=neutral Received: from pendragon.bb.dnainternet.fi (81-175-216-236.bb.dnainternet.fi [81.175.216.236]) by perceval.ideasonboard.com (Postfix) with ESMTPSA id 7715A304; Sun, 10 May 2020 13:58:21 +0200 (CEST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=ideasonboard.com; s=mail; t=1589111901; bh=N7CZkeb1u2lLdgBSpVXcglB+P3bYrGReq8lFxS8NaCY=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=veGBnF4cIx5v3YooTn2e7MbRGOBf57yh1RH/rWosPMNC/4GCCNHw/1OqZfnqyRSSu RsbCWGL2NDPVrweDU0/l4H3ehUT8WyyOJXuElwbGUfARgh8W4pa/YKw1WkCZKmk887 U5MJkwuMG9HehiSaOzWsl/v5MGRLR8Un5ZAoAbQ4= From: Laurent Pinchart To: libcamera-devel@lists.libcamera.org Cc: Martijn Braam Date: Sun, 10 May 2020 14:58:08 +0300 Message-Id: <20200510115810.21938-5-laurent.pinchart@ideasonboard.com> X-Mailer: git-send-email 2.26.2 In-Reply-To: <20200510115810.21938-1-laurent.pinchart@ideasonboard.com> References: <20200510115810.21938-1-laurent.pinchart@ideasonboard.com> MIME-Version: 1.0 Subject: [libcamera-devel] [PATCH v5 4/6] libcamera: pipeline: simple: Add simple format converter X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Sun, 10 May 2020 11:58:22 -0000 The simple format converter supports V4L2 M2M devices that convert pixel formats. Signed-off-by: Laurent Pinchart Reviewed-by: Kieran Bingham Reviewed-by: Niklas Söderlund --- Changes since v4: - Add comment - Rebase on top of V4L2PixelFormat changes Changes since v3: - Add a comment to explain format enumeration Changes since v2: - Rebase on top of V4L2PixelFormat --- src/libcamera/pipeline/simple/converter.cpp | 217 ++++++++++++++++++++ src/libcamera/pipeline/simple/converter.h | 60 ++++++ src/libcamera/pipeline/simple/meson.build | 1 + 3 files changed, 278 insertions(+) create mode 100644 src/libcamera/pipeline/simple/converter.cpp create mode 100644 src/libcamera/pipeline/simple/converter.h diff --git a/src/libcamera/pipeline/simple/converter.cpp b/src/libcamera/pipeline/simple/converter.cpp new file mode 100644 index 000000000000..6d88776dc2f0 --- /dev/null +++ b/src/libcamera/pipeline/simple/converter.cpp @@ -0,0 +1,217 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2020, Laurent Pinchart + * + * converter.cpp - Format converter for simple pipeline handler + */ + +#include "converter.h" + +#include + +#include +#include +#include + +#include "log.h" +#include "media_device.h" +#include "v4l2_videodevice.h" + +namespace libcamera { + +LOG_DECLARE_CATEGORY(SimplePipeline); + +SimpleConverter::SimpleConverter(MediaDevice *media) + : m2m_(nullptr) +{ + /* + * Locate the video node. There's no need to validate the pipeline + * further, the caller guarantees that this is a V4L2 mem2mem device. + */ + const std::vector &entities = media->entities(); + auto it = std::find_if(entities.begin(), entities.end(), + [](MediaEntity *entity) { + return entity->function() == MEDIA_ENT_F_IO_V4L; + }); + if (it == entities.end()) + return; + + m2m_ = new V4L2M2MDevice((*it)->deviceNode()); + + m2m_->output()->bufferReady.connect(this, &SimpleConverter::outputBufferReady); + m2m_->capture()->bufferReady.connect(this, &SimpleConverter::captureBufferReady); +} + +SimpleConverter::~SimpleConverter() +{ + delete m2m_; +} + +int SimpleConverter::open() +{ + if (!m2m_) + return -ENODEV; + + return m2m_->open(); +} + +void SimpleConverter::close() +{ + if (m2m_) + m2m_->close(); +} + +std::vector SimpleConverter::formats(PixelFormat input) +{ + if (!m2m_) + return {}; + + /* + * Set the format on the input side (V4L2 output) of the converter to + * enumerate the conversion capabilities on its output (V4L2 capture). + */ + V4L2DeviceFormat format; + format.fourcc = m2m_->output()->toV4L2PixelFormat(input); + format.size = { 1, 1 }; + + int ret = m2m_->output()->setFormat(&format); + if (ret < 0) { + LOG(SimplePipeline, Error) + << "Failed to set format: " << strerror(-ret); + return {}; + } + + std::vector pixelFormats; + + for (const auto &format : m2m_->capture()->formats()) { + PixelFormat pixelFormat = format.first.toPixelFormat(); + if (pixelFormat) + pixelFormats.push_back(pixelFormat); + } + + return pixelFormats; +} + +int SimpleConverter::configure(PixelFormat inputFormat, + PixelFormat outputFormat, const Size &size) +{ + V4L2DeviceFormat format; + int ret; + + V4L2PixelFormat videoFormat = m2m_->output()->toV4L2PixelFormat(inputFormat); + format.fourcc = videoFormat; + format.size = size; + + ret = m2m_->output()->setFormat(&format); + if (ret < 0) { + LOG(SimplePipeline, Error) + << "Failed to set input format: " << strerror(-ret); + return ret; + } + + if (format.fourcc != videoFormat || format.size != size) { + LOG(SimplePipeline, Error) + << "Input format not supported"; + return -EINVAL; + } + + /* + * Set the pixel format on the output, the size is identical to the + * input as we don't support scaling. + */ + videoFormat = m2m_->capture()->toV4L2PixelFormat(outputFormat); + format.fourcc = videoFormat; + + ret = m2m_->capture()->setFormat(&format); + if (ret < 0) { + LOG(SimplePipeline, Error) + << "Failed to set output format: " << strerror(-ret); + return ret; + } + + if (format.fourcc != videoFormat || format.size != size) { + LOG(SimplePipeline, Error) + << "Output format not supported"; + return -EINVAL; + } + + return 0; +} + +int SimpleConverter::exportBuffers(unsigned int count, + std::vector> *buffers) +{ + return m2m_->capture()->exportBuffers(count, buffers); +} + +int SimpleConverter::start(unsigned int count) +{ + int ret = m2m_->output()->importBuffers(count); + if (ret < 0) + return ret; + + ret = m2m_->capture()->importBuffers(count); + if (ret < 0) { + stop(); + return ret; + } + + ret = m2m_->output()->streamOn(); + if (ret < 0) { + stop(); + return ret; + } + + ret = m2m_->capture()->streamOn(); + if (ret < 0) { + stop(); + return ret; + } + + return 0; +} + +void SimpleConverter::stop() +{ + m2m_->capture()->streamOff(); + m2m_->output()->streamOff(); + m2m_->capture()->releaseBuffers(); + m2m_->output()->releaseBuffers(); +} + +int SimpleConverter::queueBuffers(FrameBuffer *input, FrameBuffer *output) +{ + int ret = m2m_->output()->queueBuffer(input); + if (ret < 0) + return ret; + + ret = m2m_->capture()->queueBuffer(output); + if (ret < 0) + return ret; + + return 0; +} + +void SimpleConverter::captureBufferReady(FrameBuffer *buffer) +{ + if (!outputDoneQueue_.empty()) { + FrameBuffer *other = outputDoneQueue_.front(); + outputDoneQueue_.pop(); + bufferReady.emit(other, buffer); + } else { + captureDoneQueue_.push(buffer); + } +} + +void SimpleConverter::outputBufferReady(FrameBuffer *buffer) +{ + if (!captureDoneQueue_.empty()) { + FrameBuffer *other = captureDoneQueue_.front(); + captureDoneQueue_.pop(); + bufferReady.emit(buffer, other); + } else { + outputDoneQueue_.push(buffer); + } +} + +} /* namespace libcamera */ diff --git a/src/libcamera/pipeline/simple/converter.h b/src/libcamera/pipeline/simple/converter.h new file mode 100644 index 000000000000..a33071fa8578 --- /dev/null +++ b/src/libcamera/pipeline/simple/converter.h @@ -0,0 +1,60 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2020, Laurent Pinchart + * + * converter.h - Format converter for simple pipeline handler + */ + +#ifndef __LIBCAMERA_PIPELINE_SIMPLE_CONVERTER_H__ +#define __LIBCAMERA_PIPELINE_SIMPLE_CONVERTER_H__ + +#include +#include +#include + +#include +#include + +namespace libcamera { + +class FrameBuffer; +class MediaDevice; +struct Size; +class V4L2M2MDevice; + +class SimpleConverter +{ +public: + SimpleConverter(MediaDevice *media); + ~SimpleConverter(); + + int open(); + void close(); + + std::vector formats(PixelFormat input); + + int configure(PixelFormat inputFormat, PixelFormat outputFormat, + const Size &size); + int exportBuffers(unsigned int count, + std::vector> *buffers); + + int start(unsigned int count); + void stop(); + + int queueBuffers(FrameBuffer *input, FrameBuffer *output); + + Signal bufferReady; + +private: + void captureBufferReady(FrameBuffer *buffer); + void outputBufferReady(FrameBuffer *buffer); + + V4L2M2MDevice *m2m_; + + std::queue captureDoneQueue_; + std::queue outputDoneQueue_; +}; + +} /* namespace libcamera */ + +#endif /* __LIBCAMERA_PIPELINE_SIMPLE_CONVERTER_H__ */ diff --git a/src/libcamera/pipeline/simple/meson.build b/src/libcamera/pipeline/simple/meson.build index 4945a3e173cf..8372f24e3788 100644 --- a/src/libcamera/pipeline/simple/meson.build +++ b/src/libcamera/pipeline/simple/meson.build @@ -1,3 +1,4 @@ libcamera_sources += files([ + 'converter.cpp', 'simple.cpp', ]) From patchwork Sun May 10 11:58:09 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Patchwork-Submitter: Laurent Pinchart X-Patchwork-Id: 3744 Return-Path: Received: from perceval.ideasonboard.com (perceval.ideasonboard.com [213.167.242.64]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id B9D8D60BEC for ; Sun, 10 May 2020 13:58:22 +0200 (CEST) Authentication-Results: lancelot.ideasonboard.com; dkim=pass (1024-bit key; unprotected) header.d=ideasonboard.com header.i=@ideasonboard.com header.b="EidLgGt9"; dkim-atps=neutral Received: from pendragon.bb.dnainternet.fi (81-175-216-236.bb.dnainternet.fi [81.175.216.236]) by perceval.ideasonboard.com (Postfix) with ESMTPSA id 08C4C73E; Sun, 10 May 2020 13:58:21 +0200 (CEST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=ideasonboard.com; s=mail; t=1589111902; bh=+wZqfUbabNXDp4NXGLaBy/11cFyxT8yJfj+mt8/YUjc=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=EidLgGt9Ne05EG53K2TBqlPCqflNwJmso0WnSoO1eQ5piZpkOCSCj1n8Wly+VGMt8 Wg95qXRjSPWo2XqX4X1kql/GxVUVjpnpaB2aDJJA8PV0XSVF6WG7qin4e9FJ/91M8F 6PXJGBq/kf0oZJntthRs4umdRtLUYgLIuDljSFVE= From: Laurent Pinchart To: libcamera-devel@lists.libcamera.org Cc: Martijn Braam Date: Sun, 10 May 2020 14:58:09 +0300 Message-Id: <20200510115810.21938-6-laurent.pinchart@ideasonboard.com> X-Mailer: git-send-email 2.26.2 In-Reply-To: <20200510115810.21938-1-laurent.pinchart@ideasonboard.com> References: <20200510115810.21938-1-laurent.pinchart@ideasonboard.com> MIME-Version: 1.0 Subject: [libcamera-devel] [PATCH v5 5/6] libcamera: pipeline: simple: Integrate converter support X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Sun, 10 May 2020 11:58:24 -0000 Add support for an optional format converter, supported by the SimpleConverter class. If a converter is available for the pipeline, it will be used to expose additional pixel formats. Signed-off-by: Laurent Pinchart Reviewed-by: Niklas Söderlund --- Changes since v2: - Rebase on top of V4L2PixelFormat - Don't crash if the device has no converter --- src/libcamera/pipeline/simple/simple.cpp | 196 ++++++++++++++++++++--- 1 file changed, 178 insertions(+), 18 deletions(-) diff --git a/src/libcamera/pipeline/simple/simple.cpp b/src/libcamera/pipeline/simple/simple.cpp index 95417500b6b5..d5a3bf61252b 100644 --- a/src/libcamera/pipeline/simple/simple.cpp +++ b/src/libcamera/pipeline/simple/simple.cpp @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include @@ -31,17 +32,24 @@ #include "v4l2_subdevice.h" #include "v4l2_videodevice.h" +#include "converter.h" + namespace libcamera { LOG_DEFINE_CATEGORY(SimplePipeline) class SimplePipelineHandler; +struct SimplePipelineInfo { + const char *driver; + const char *converter; +}; + namespace { -static const char * const drivers[] = { - "imx7-csi", - "sun6i-csi", +static const SimplePipelineInfo supportedDevices[] = { + { "imx7-csi", "pxp" }, + { "sun6i-csi", nullptr }, }; } /* namespace */ @@ -88,6 +96,8 @@ public: const V4L2SubdeviceFormat &sensorFormat() { return sensorFormat_; } + bool needConversion() const { return needConversion_; } + private: /* * The SimpleCameraData instance is guaranteed to be valid as long as @@ -98,6 +108,7 @@ private: const SimpleCameraData *data_; V4L2SubdeviceFormat sensorFormat_; + bool needConversion_; }; class SimplePipelineHandler : public PipelineHandler @@ -120,6 +131,7 @@ public: V4L2VideoDevice *video() { return video_; } V4L2Subdevice *subdev(const MediaEntity *entity); + SimpleConverter *converter() { return converter_; } protected: int queueRequestDevice(Camera *camera, Request *request) override; @@ -136,11 +148,17 @@ private: int createCamera(MediaEntity *sensor); void bufferReady(FrameBuffer *buffer); + void converterDone(FrameBuffer *input, FrameBuffer *output); MediaDevice *media_; V4L2VideoDevice *video_; std::map subdevs_; + SimpleConverter *converter_; + bool useConverter_; + std::vector> converterBuffers_; + std::queue converterQueue_; + Camera *activeCamera_; }; @@ -222,6 +240,7 @@ int SimpleCameraData::init() { SimplePipelineHandler *pipe = static_cast(pipe_); V4L2VideoDevice *video = pipe->video(); + SimpleConverter *converter = pipe->converter(); int ret; /* @@ -278,7 +297,13 @@ int SimpleCameraData::init() config.pixelFormat = pixelFormat; config.size = format.size; - formats_[pixelFormat] = config; + if (!converter) { + formats_[pixelFormat] = config; + continue; + } + + for (PixelFormat format : converter->formats(pixelFormat)) + formats_[format] = config; } } @@ -417,6 +442,8 @@ CameraConfiguration::Status SimpleCameraConfiguration::validate() status = Adjusted; } + needConversion_ = cfg.pixelFormat != pipeConfig.pixelFormat; + cfg.bufferCount = 3; return status; @@ -427,13 +454,14 @@ CameraConfiguration::Status SimpleCameraConfiguration::validate() */ SimplePipelineHandler::SimplePipelineHandler(CameraManager *manager) - : PipelineHandler(manager), video_(nullptr) + : PipelineHandler(manager), video_(nullptr), converter_(nullptr) { } SimplePipelineHandler::~SimplePipelineHandler() { delete video_; + delete converter_; } CameraConfiguration *SimplePipelineHandler::generateConfiguration(Camera *camera, @@ -499,22 +527,37 @@ int SimplePipelineHandler::configure(Camera *camera, CameraConfiguration *c) return ret; /* Configure the video node. */ - V4L2PixelFormat videoFormat = video_->toV4L2PixelFormat(cfg.pixelFormat); + V4L2PixelFormat videoFormat = video_->toV4L2PixelFormat(pipeConfig.pixelFormat); - V4L2DeviceFormat outputFormat = {}; - outputFormat.fourcc = videoFormat; - outputFormat.size = cfg.size; + V4L2DeviceFormat captureFormat = {}; + captureFormat.fourcc = videoFormat; + captureFormat.size = cfg.size; - ret = video_->setFormat(&outputFormat); + ret = video_->setFormat(&captureFormat); if (ret) return ret; - if (outputFormat.size != cfg.size || outputFormat.fourcc != videoFormat) { + if (captureFormat.fourcc != videoFormat || captureFormat.size != cfg.size) { LOG(SimplePipeline, Error) << "Unable to configure capture in " << cfg.toString(); return -EINVAL; } + /* Configure the converter if required. */ + useConverter_ = config->needConversion(); + + if (useConverter_) { + int ret = converter_->configure(pipeConfig.pixelFormat, + cfg.pixelFormat, cfg.size); + if (ret < 0) { + LOG(SimplePipeline, Error) + << "Unable to configure converter"; + return ret; + } + + LOG(SimplePipeline, Debug) << "Using format converter"; + } + cfg.setStream(&data->stream_); return 0; @@ -525,24 +568,47 @@ int SimplePipelineHandler::exportFrameBuffers(Camera *camera, Stream *stream, { unsigned int count = stream->configuration().bufferCount; - return video_->exportBuffers(count, buffers); + /* + * Export buffers on the converter or capture video node, depending on + * whether the converter is used or not. + */ + if (useConverter_) + return converter_->exportBuffers(count, buffers); + else + return video_->exportBuffers(count, buffers); } int SimplePipelineHandler::start(Camera *camera) { SimpleCameraData *data = cameraData(camera); unsigned int count = data->stream_.configuration().bufferCount; + int ret; - int ret = video_->importBuffers(count); + if (useConverter_) + ret = video_->allocateBuffers(count, &converterBuffers_); + else + ret = video_->importBuffers(count); if (ret < 0) return ret; ret = video_->streamOn(); if (ret < 0) { - video_->releaseBuffers(); + stop(camera); return ret; } + if (useConverter_) { + ret = converter_->start(count); + if (ret < 0) { + stop(camera); + return ret; + } + + /* Queue all internal buffers for capture. */ + for (std::unique_ptr &buffer : converterBuffers_) + video_->queueBuffer(buffer.get()); + } + activeCamera_ = camera; return 0; @@ -550,8 +616,13 @@ int SimplePipelineHandler::start(Camera *camera) void SimplePipelineHandler::stop(Camera *camera) { + if (useConverter_) + converter_->stop(); + video_->streamOff(); video_->releaseBuffers(); + + converterBuffers_.clear(); activeCamera_ = nullptr; } @@ -567,6 +638,15 @@ int SimplePipelineHandler::queueRequestDevice(Camera *camera, Request *request) return -ENOENT; } + /* + * If conversion is needed, push the buffer to the converter queue, it + * will be handed to the converter in the capture completion handler. + */ + if (useConverter_) { + converterQueue_.push(buffer); + return 0; + } + return video_->queueBuffer(buffer); } @@ -576,11 +656,20 @@ int SimplePipelineHandler::queueRequestDevice(Camera *camera, Request *request) bool SimplePipelineHandler::match(DeviceEnumerator *enumerator) { - for (const char *driver : drivers) { - DeviceMatch dm(driver); + MediaDevice *converter = nullptr; + + for (const SimplePipelineInfo &info : supportedDevices) { + DeviceMatch dm(info.driver); media_ = acquireMediaDevice(enumerator, dm); - if (media_) + if (!media_) + continue; + + if (!info.converter) break; + + DeviceMatch converterMatch(info.converter); + converter = acquireMediaDevice(enumerator, converterMatch); + break; } if (!media_) @@ -635,6 +724,19 @@ bool SimplePipelineHandler::match(DeviceEnumerator *enumerator) video_->bufferReady.connect(this, &SimplePipelineHandler::bufferReady); + /* Open the converter, if any. */ + if (converter) { + converter_ = new SimpleConverter(converter); + if (converter_->open() < 0) { + LOG(SimplePipeline, Warning) + << "Failed to open converter, disabling format conversion"; + delete converter_; + converter_ = nullptr; + } + + converter_->bufferReady.connect(this, &SimplePipelineHandler::converterDone); + } + /* * Create one camera data instance for each sensor and gather all * entities in all pipelines. @@ -709,12 +811,70 @@ V4L2Subdevice *SimplePipelineHandler::subdev(const MediaEntity *entity) void SimplePipelineHandler::bufferReady(FrameBuffer *buffer) { - ASSERT(activeCamera_); + /* + * If an error occurred during capture, or if the buffer was cancelled, + * complete the request, even if the converter is in use as there's no + * point converting an erroneous buffer. + */ + if (buffer->metadata().status != FrameMetadata::FrameSuccess) { + if (useConverter_) { + /* Requeue the buffer for capture. */ + video_->queueBuffer(buffer); + + /* + * Get the next user-facing buffer to complete the + * request. + */ + if (converterQueue_.empty()) + return; + + buffer = converterQueue_.front(); + converterQueue_.pop(); + } + + Request *request = buffer->request(); + completeBuffer(activeCamera_, request, buffer); + completeRequest(activeCamera_, request); + return; + } + + /* + * Queue the captured and the request buffer to the converter if format + * conversion is needed. If there's no queued request, just requeue the + * captured buffer for capture. + */ + if (useConverter_) { + if (converterQueue_.empty()) { + video_->queueBuffer(buffer); + return; + } + + FrameBuffer *output = converterQueue_.front(); + converterQueue_.pop(); + + converter_->queueBuffers(buffer, output); + return; + } + + /* Otherwise simply complete the request. */ Request *request = buffer->request(); completeBuffer(activeCamera_, request, buffer); completeRequest(activeCamera_, request); } +void SimplePipelineHandler::converterDone(FrameBuffer *input, + FrameBuffer *output) +{ + /* Complete the request. */ + ASSERT(activeCamera_); + Request *request = output->request(); + completeBuffer(activeCamera_, request, output); + completeRequest(activeCamera_, request); + + /* Queue the input buffer back for capture. */ + video_->queueBuffer(input); +} + REGISTER_PIPELINE_HANDLER(SimplePipelineHandler); } /* namespace libcamera */ From patchwork Sun May 10 11:58:10 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Patchwork-Submitter: Laurent Pinchart X-Patchwork-Id: 3745 Return-Path: Received: from perceval.ideasonboard.com (perceval.ideasonboard.com [213.167.242.64]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id F20D960C62 for ; Sun, 10 May 2020 13:58:22 +0200 (CEST) Authentication-Results: lancelot.ideasonboard.com; dkim=pass (1024-bit key; unprotected) header.d=ideasonboard.com header.i=@ideasonboard.com header.b="ikH1tj4s"; dkim-atps=neutral Received: from pendragon.bb.dnainternet.fi (81-175-216-236.bb.dnainternet.fi [81.175.216.236]) by perceval.ideasonboard.com (Postfix) with ESMTPSA id 85588304; Sun, 10 May 2020 13:58:22 +0200 (CEST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=ideasonboard.com; s=mail; t=1589111902; bh=YphSCwjeed+BnzWqF24Iyexqyr5e/GYBLrCQIZldMLY=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=ikH1tj4s978ng0C9rDV6dfwq+zeDeQCDbcBPMvIaZA0mSOOtbmO4QGUTBDZyYiua3 wqJXuD031lDJgRo+mF/ik/WESvjkeoOWACo2xj15QJJqyG830M1tGVpB/fC/1hsv3k VbOuxnK7jYSXpfaSIOFlsckv3NDmGUf+WO/3YADw= From: Laurent Pinchart To: libcamera-devel@lists.libcamera.org Cc: Martijn Braam Date: Sun, 10 May 2020 14:58:10 +0300 Message-Id: <20200510115810.21938-7-laurent.pinchart@ideasonboard.com> X-Mailer: git-send-email 2.26.2 In-Reply-To: <20200510115810.21938-1-laurent.pinchart@ideasonboard.com> References: <20200510115810.21938-1-laurent.pinchart@ideasonboard.com> MIME-Version: 1.0 Subject: [libcamera-devel] [PATCH v5 6/6] libcamera: pipeline: simple: Support multiple capture video nodes X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Sun, 10 May 2020 11:58:24 -0000 The simple pipeline handler rejects devices that have multiple capture video nodes. There's no real reason to do so, a more dynamic approach is possible as the pipeline handler already locates the video device by walking the media graph. Rework the match sequence by skipping any check on the video nodes, and create the V4L2VideoDevice for the media entity at the end of the pipeline when initializing the camera data. The V4L2VideoDevice instances are managed by the pipeline handler itself, to avoid creating separate instances in the camera data if multiple sensors are routed to the same video device. Signed-off-by: Laurent Pinchart Reviewed-by: Andrey Konovalov Reviewed-by: Niklas Söderlund --- src/libcamera/pipeline/simple/simple.cpp | 134 +++++++++++++---------- 1 file changed, 74 insertions(+), 60 deletions(-) diff --git a/src/libcamera/pipeline/simple/simple.cpp b/src/libcamera/pipeline/simple/simple.cpp index d5a3bf61252b..6673606153b8 100644 --- a/src/libcamera/pipeline/simple/simple.cpp +++ b/src/libcamera/pipeline/simple/simple.cpp @@ -57,8 +57,7 @@ static const SimplePipelineInfo supportedDevices[] = { class SimpleCameraData : public CameraData { public: - SimpleCameraData(PipelineHandler *pipe, MediaEntity *sensor, - MediaEntity *video); + SimpleCameraData(SimplePipelineHandler *pipe, MediaEntity *sensor); bool isValid() const { return sensor_ != nullptr; } std::set streams() { return { &stream_ }; } @@ -82,6 +81,7 @@ public: Stream stream_; std::unique_ptr sensor_; std::list entities_; + V4L2VideoDevice *video_; std::vector configs_; std::map formats_; @@ -129,7 +129,7 @@ public: bool match(DeviceEnumerator *enumerator) override; - V4L2VideoDevice *video() { return video_; } + V4L2VideoDevice *video(const MediaEntity *entity); V4L2Subdevice *subdev(const MediaEntity *entity); SimpleConverter *converter() { return converter_; } @@ -151,7 +151,7 @@ private: void converterDone(FrameBuffer *input, FrameBuffer *output); MediaDevice *media_; - V4L2VideoDevice *video_; + std::map> videos_; std::map subdevs_; SimpleConverter *converter_; @@ -166,8 +166,8 @@ private: * Camera Data */ -SimpleCameraData::SimpleCameraData(PipelineHandler *pipe, MediaEntity *sensor, - MediaEntity *video) +SimpleCameraData::SimpleCameraData(SimplePipelineHandler *pipe, + MediaEntity *sensor) : CameraData(pipe) { int ret; @@ -179,8 +179,8 @@ SimpleCameraData::SimpleCameraData(PipelineHandler *pipe, MediaEntity *sensor, MediaEntity *source = sensor; while (source) { - /* If we have reached the video node, we're done. */ - if (source == video) + /* If we have reached a video node, we're done. */ + if (source->function() == MEDIA_ENT_F_IO_V4L) break; /* Use the first output pad that has links. */ @@ -227,7 +227,14 @@ SimpleCameraData::SimpleCameraData(PipelineHandler *pipe, MediaEntity *sensor, } } - /* We have a valid pipeline, create the camera sensor. */ + /* + * We have a valid pipeline, get the video device and create the camera + * sensor. + */ + video_ = pipe->video(source); + if (!video_) + return; + sensor_ = std::make_unique(sensor); ret = sensor_->init(); if (ret) { @@ -239,7 +246,6 @@ SimpleCameraData::SimpleCameraData(PipelineHandler *pipe, MediaEntity *sensor, int SimpleCameraData::init() { SimplePipelineHandler *pipe = static_cast(pipe_); - V4L2VideoDevice *video = pipe->video(); SimpleConverter *converter = pipe->converter(); int ret; @@ -269,7 +275,7 @@ int SimpleCameraData::init() } std::map> videoFormats = - video->formats(format.mbus_code); + video_->formats(format.mbus_code); LOG(SimplePipeline, Debug) << "Adding configuration for " << format.size.toString() @@ -454,13 +460,12 @@ CameraConfiguration::Status SimpleCameraConfiguration::validate() */ SimplePipelineHandler::SimplePipelineHandler(CameraManager *manager) - : PipelineHandler(manager), video_(nullptr), converter_(nullptr) + : PipelineHandler(manager), converter_(nullptr) { } SimplePipelineHandler::~SimplePipelineHandler() { - delete video_; delete converter_; } @@ -506,6 +511,7 @@ int SimplePipelineHandler::configure(Camera *camera, CameraConfiguration *c) SimpleCameraConfiguration *config = static_cast(c); SimpleCameraData *data = cameraData(camera); + V4L2VideoDevice *video = data->video_; StreamConfiguration &cfg = config->at(0); int ret; @@ -527,13 +533,13 @@ int SimplePipelineHandler::configure(Camera *camera, CameraConfiguration *c) return ret; /* Configure the video node. */ - V4L2PixelFormat videoFormat = video_->toV4L2PixelFormat(pipeConfig.pixelFormat); + V4L2PixelFormat videoFormat = video->toV4L2PixelFormat(pipeConfig.pixelFormat); V4L2DeviceFormat captureFormat = {}; captureFormat.fourcc = videoFormat; captureFormat.size = cfg.size; - ret = video_->setFormat(&captureFormat); + ret = video->setFormat(&captureFormat); if (ret) return ret; @@ -566,6 +572,7 @@ int SimplePipelineHandler::configure(Camera *camera, CameraConfiguration *c) int SimplePipelineHandler::exportFrameBuffers(Camera *camera, Stream *stream, std::vector> *buffers) { + SimpleCameraData *data = cameraData(camera); unsigned int count = stream->configuration().bufferCount; /* @@ -575,23 +582,24 @@ int SimplePipelineHandler::exportFrameBuffers(Camera *camera, Stream *stream, if (useConverter_) return converter_->exportBuffers(count, buffers); else - return video_->exportBuffers(count, buffers); + return data->video_->exportBuffers(count, buffers); } int SimplePipelineHandler::start(Camera *camera) { SimpleCameraData *data = cameraData(camera); + V4L2VideoDevice *video = data->video_; unsigned int count = data->stream_.configuration().bufferCount; int ret; if (useConverter_) - ret = video_->allocateBuffers(count, &converterBuffers_); + ret = video->allocateBuffers(count, &converterBuffers_); else - ret = video_->importBuffers(count); + ret = video->importBuffers(count); if (ret < 0) return ret; - ret = video_->streamOn(); + ret = video->streamOn(); if (ret < 0) { stop(camera); return ret; @@ -606,7 +614,7 @@ int SimplePipelineHandler::start(Camera *camera) /* Queue all internal buffers for capture. */ for (std::unique_ptr &buffer : converterBuffers_) - video_->queueBuffer(buffer.get()); + video->queueBuffer(buffer.get()); } activeCamera_ = camera; @@ -616,11 +624,14 @@ int SimplePipelineHandler::start(Camera *camera) void SimplePipelineHandler::stop(Camera *camera) { + SimpleCameraData *data = cameraData(camera); + V4L2VideoDevice *video = data->video_; + if (useConverter_) converter_->stop(); - video_->streamOff(); - video_->releaseBuffers(); + video->streamOff(); + video->releaseBuffers(); converterBuffers_.clear(); activeCamera_ = nullptr; @@ -647,7 +658,7 @@ int SimplePipelineHandler::queueRequestDevice(Camera *camera, Request *request) return 0; } - return video_->queueBuffer(buffer); + return data->video_->queueBuffer(buffer); } /* ----------------------------------------------------------------------------- @@ -675,12 +686,8 @@ bool SimplePipelineHandler::match(DeviceEnumerator *enumerator) if (!media_) return false; - /* - * Locate sensors and video nodes. We only support pipelines with at - * least one sensor and exactly one video capture node. - */ + /* Locate the sensors. */ std::vector sensors; - std::vector videos; for (MediaEntity *entity : media_->entities()) { switch (entity->function()) { @@ -688,12 +695,6 @@ bool SimplePipelineHandler::match(DeviceEnumerator *enumerator) sensors.push_back(entity); break; - case MEDIA_ENT_F_IO_V4L: - if (entity->pads().size() == 1 && - (entity->pads()[0]->flags() & MEDIA_PAD_FL_SINK)) - videos.push_back(entity); - break; - default: break; } @@ -704,26 +705,6 @@ bool SimplePipelineHandler::match(DeviceEnumerator *enumerator) return false; } - if (videos.size() != 1) { - LOG(SimplePipeline, Error) - << "Pipeline with " << videos.size() - << " video capture nodes is not supported"; - return false; - } - - /* Locate and open the capture video node. */ - video_ = new V4L2VideoDevice(videos[0]); - if (video_->open() < 0) - return false; - - if (video_->caps().isMultiplanar()) { - LOG(SimplePipeline, Error) - << "V4L2 multiplanar devices are not supported"; - return false; - } - - video_->bufferReady.connect(this, &SimplePipelineHandler::bufferReady); - /* Open the converter, if any. */ if (converter) { converter_ = new SimpleConverter(converter); @@ -748,8 +729,7 @@ bool SimplePipelineHandler::match(DeviceEnumerator *enumerator) for (MediaEntity *sensor : sensors) { std::unique_ptr data = - std::make_unique(this, sensor, - videos[0]); + std::make_unique(this, sensor); if (!data->isValid()) { LOG(SimplePipeline, Error) << "No valid pipeline for sensor '" @@ -796,6 +776,35 @@ bool SimplePipelineHandler::match(DeviceEnumerator *enumerator) return true; } +V4L2VideoDevice *SimplePipelineHandler::video(const MediaEntity *entity) +{ + /* + * Return the V4L2VideoDevice corresponding to the media entity, either + * as a previously constructed device if available from the cache, or + * by constructing a new one. + */ + + auto iter = videos_.find(entity); + if (iter != videos_.end()) + return iter->second.get(); + + std::unique_ptr video = + std::make_unique(entity); + if (video->open() < 0) + return nullptr; + + if (video->caps().isMultiplanar()) { + LOG(SimplePipeline, Error) + << "V4L2 multiplanar devices are not supported"; + return nullptr; + } + + video->bufferReady.connect(this, &SimplePipelineHandler::bufferReady); + + auto element = videos_.emplace(entity, std::move(video)); + return element.first->second.get(); +} + V4L2Subdevice *SimplePipelineHandler::subdev(const MediaEntity *entity) { auto iter = subdevs_.find(entity); @@ -811,6 +820,9 @@ V4L2Subdevice *SimplePipelineHandler::subdev(const MediaEntity *entity) void SimplePipelineHandler::bufferReady(FrameBuffer *buffer) { + ASSERT(activeCamera_); + SimpleCameraData *data = cameraData(activeCamera_); + /* * If an error occurred during capture, or if the buffer was cancelled, * complete the request, even if the converter is in use as there's no @@ -819,7 +831,7 @@ void SimplePipelineHandler::bufferReady(FrameBuffer *buffer) if (buffer->metadata().status != FrameMetadata::FrameSuccess) { if (useConverter_) { /* Requeue the buffer for capture. */ - video_->queueBuffer(buffer); + data->video_->queueBuffer(buffer); /* * Get the next user-facing buffer to complete the @@ -845,7 +857,7 @@ void SimplePipelineHandler::bufferReady(FrameBuffer *buffer) */ if (useConverter_) { if (converterQueue_.empty()) { - video_->queueBuffer(buffer); + data->video_->queueBuffer(buffer); return; } @@ -865,14 +877,16 @@ void SimplePipelineHandler::bufferReady(FrameBuffer *buffer) void SimplePipelineHandler::converterDone(FrameBuffer *input, FrameBuffer *output) { + ASSERT(activeCamera_); + SimpleCameraData *data = cameraData(activeCamera_); + /* Complete the request. */ - ASSERT(activeCamera_); Request *request = output->request(); completeBuffer(activeCamera_, request, output); completeRequest(activeCamera_, request); /* Queue the input buffer back for capture. */ - video_->queueBuffer(input); + data->video_->queueBuffer(input); } REGISTER_PIPELINE_HANDLER(SimplePipelineHandler);