[libcamera-devel,06/14] libcamera: converter: Add v4l2 m2m converter implementation

17348 diff mbox series

Introduce a converter implementation relying on a v4l2 m2m device, mostly
based on the current simple pipeline converter implementation.

The main change is the introduction of Mapping object which can be
loaded through a configuration file which define vertices remapping
coordinates. Those latters can be applied by any classes derived from
this base class which define the apply_mapping() method.

Signed-off-by: Xavier Roumegue <xavier.roumegue@oss.nxp.com>
---
 .../libcamera/internal/converter_v4l2_m2m.h   | 120 +++++
 include/libcamera/internal/meson.build        |   1 +
 src/libcamera/converter_v4l2_m2m.cpp          | 504 ++++++++++++++++++
 src/libcamera/meson.build                     |   1 +
 4 files changed, 626 insertions(+)
 create mode 100644 include/libcamera/internal/converter_v4l2_m2m.h
 create mode 100644 src/libcamera/converter_v4l2_m2m.cpp

Hi Xavier

On Thu, Sep 08, 2022 at 08:48:42PM +0200, Xavier Roumegue via libcamera-devel wrote:
> Introduce a converter implementation relying on a v4l2 m2m device, mostly
> based on the current simple pipeline converter implementation.
>
> The main change is the introduction of Mapping object which can be
> loaded through a configuration file which define vertices remapping
> coordinates. Those latters can be applied by any classes derived from
> this base class which define the apply_mapping() method.
>

I'll skip questions on things not clear to me about the converter
implementation as I understand this basically comes from the simple
pipeline converter implementation [*]

I would then focus more on two general questions:
1) Do we want to make a dir for converter/post-processors instead of
placing them in src/libcamera

2) Is the format expected in the configuration file documented
anywhere ?

[*] Not true, I will ask a question anyway, see below :)

> Signed-off-by: Xavier Roumegue <xavier.roumegue@oss.nxp.com>
> ---
>  .../libcamera/internal/converter_v4l2_m2m.h   | 120 +++++
>  include/libcamera/internal/meson.build        |   1 +
>  src/libcamera/converter_v4l2_m2m.cpp          | 504 ++++++++++++++++++
>  src/libcamera/meson.build                     |   1 +
>  4 files changed, 626 insertions(+)
>  create mode 100644 include/libcamera/internal/converter_v4l2_m2m.h
>  create mode 100644 src/libcamera/converter_v4l2_m2m.cpp
>
> diff --git a/include/libcamera/internal/converter_v4l2_m2m.h b/include/libcamera/internal/converter_v4l2_m2m.h
> new file mode 100644
> index 00000000..3667b128
> --- /dev/null
> +++ b/include/libcamera/internal/converter_v4l2_m2m.h
> @@ -0,0 +1,120 @@
> +/* SPDX-License-Identifier: LGPL-2.1-or-later */
> +/*
> + * Copyright 2022 NXP
> + *
> + * converter_v4l2_m2m.h - V4l2 M2M Format converter interface
> + */
> +
> +#pragma once
> +
> +#include <functional>
> +#include <map>
> +#include <memory>
> +#include <string>
> +#include <tuple>
> +#include <vector>
> +
> +#include <libcamera/base/log.h>
> +#include <libcamera/base/signal.h>
> +
> +#include <libcamera/geometry.h>
> +#include <libcamera/pixel_format.h>
> +
> +#include "libcamera/internal/converter.h"
> +
> +namespace libcamera {
> +
> +class FrameBuffer;
> +class MediaDevice;
> +class Size;
> +class SizeRange;
> +struct StreamConfiguration;
> +class V4L2M2MDevice;
> +class V4L2M2MConverter;
> +class Converter;
> +
> +class V4L2M2MConverter : public Converter
> +{
> +protected:
> +	class Mapping
> +	{
> +	public:
> +		Mapping(const Size &input, const Size &output, const std::vector<uint32_t> &map)
> +			: input_(input), output_(output), map_(map) {}
> +		Size getInputSize() const { return input_; }
> +		Size getOutputSize() const { return output_; }
> +		std::size_t getLength() const { return map_.size(); }
> +		const uint32_t *getMapping() const { return map_.data(); }
> +
> +	private:
> +		Size input_;
> +		Size output_;
> +		std::vector<uint32_t> map_;
> +	};
> +
> +	class Stream : protected Loggable
> +	{
> +	public:
> +		Stream(V4L2M2MConverter *converter, unsigned int index);
> +
> +		bool isValid() const { return m2m_ != nullptr; }
> +
> +		int configure(const StreamConfiguration &inputCfg,
> +			      const StreamConfiguration &outputCfg);
> +		int exportBuffers(unsigned int count,
> +				  std::vector<std::unique_ptr<FrameBuffer>> *buffers);
> +
> +		int start();
> +		void stop();
> +
> +		int queueBuffers(FrameBuffer *input, FrameBuffer *output);
> +		std::unique_ptr<V4L2M2MDevice> m2m_;
> +
> +	protected:
> +		std::string logPrefix() const override;
> +
> +	private:
> +		void captureBufferReady(FrameBuffer *buffer);
> +		void outputBufferReady(FrameBuffer *buffer);
> +
> +		V4L2M2MConverter *converter_;
> +		unsigned int index_;
> +
> +		unsigned int inputBufferCount_;
> +		unsigned int outputBufferCount_;
> +	};
> +
> +	std::unique_ptr<V4L2M2MDevice> m2m_;
> +
> +	std::vector<Stream> streams_;
> +	std::vector<Mapping> mappings_;
> +	std::map<FrameBuffer *, unsigned int> queue_;
> +
> +public:
> +	V4L2M2MConverter(MediaDevice *media);
> +
> +	int loadConfiguration(const std::string &filename) override;
> +
> +	bool isValid() const { return m2m_ != nullptr; }
> +
> +	std::vector<PixelFormat> formats(PixelFormat input);
> +	SizeRange sizes(const Size &input);
> +
> +	std::tuple<unsigned int, unsigned int>
> +	strideAndFrameSize(const PixelFormat &pixelFormat, const Size &size);
> +
> +	int configure(const StreamConfiguration &inputCfg,
> +		      const std::vector<std::reference_wrapper<StreamConfiguration>> &outputCfg);
> +	int exportBuffers(unsigned int ouput, unsigned int count,
> +			  std::vector<std::unique_ptr<FrameBuffer>> *buffers);
> +
> +	int start();
> +	void stop();
> +
> +	int queueBuffers(FrameBuffer *input,
> +			 const std::map<unsigned int, FrameBuffer *> &outputs);
> +
> +	virtual int applyMapping([[maybe_unused]] Stream *stream, [[maybe_unused]] Mapping &mapping) { return 0; };
> +};
> +
> +} /* namespace libcamera */
> diff --git a/include/libcamera/internal/meson.build b/include/libcamera/internal/meson.build
> index 8f50d755..132de5ef 100644
> --- a/include/libcamera/internal/meson.build
> +++ b/include/libcamera/internal/meson.build
> @@ -20,6 +20,7 @@ libcamera_internal_headers = files([
>      'control_serializer.h',
>      'control_validator.h',
>      'converter.h',
> +    'converter_v4l2_m2m.h',
>      'delayed_controls.h',
>      'device_enumerator.h',
>      'device_enumerator_sysfs.h',
> diff --git a/src/libcamera/converter_v4l2_m2m.cpp b/src/libcamera/converter_v4l2_m2m.cpp
> new file mode 100644
> index 00000000..942e6e6f
> --- /dev/null
> +++ b/src/libcamera/converter_v4l2_m2m.cpp
> @@ -0,0 +1,504 @@
> +/* SPDX-License-Identifier: LGPL-2.1-or-later */
> +/*
> + * Copyright (C) 2020, Laurent Pinchart
> + * Copyright 2022 NXP
> + *
> + * converter_v4l2_m2m.cpp - V4L2 M2M Format converter
> + */
> +
> +#include <algorithm>
> +#include <limits.h>
> +
> +#include <libcamera/base/file.h>
> +#include <libcamera/base/log.h>
> +#include <libcamera/base/signal.h>
> +#include <libcamera/base/utils.h>
> +
> +#include <libcamera/framebuffer.h>
> +#include <libcamera/geometry.h>
> +#include <libcamera/stream.h>
> +
> +#include "libcamera/internal/converter_v4l2_m2m.h"
> +#include "libcamera/internal/media_device.h"
> +#include "libcamera/internal/v4l2_videodevice.h"
> +#include "libcamera/internal/yaml_parser.h"
> +
> +namespace libcamera {
> +
> +LOG_DECLARE_CATEGORY(Converter)
> +
> +/* -----------------------------------------------------------------------------
> + * V4L2M2MConverter::Stream
> + */
> +
> +V4L2M2MConverter::Stream::Stream(V4L2M2MConverter *converter, unsigned int index)
> +	: converter_(converter), index_(index)
> +{
> +	m2m_ = std::make_unique<V4L2M2MDevice>(converter->deviceNode_);
> +
> +	m2m_->output()->bufferReady.connect(this, &Stream::outputBufferReady);
> +	m2m_->capture()->bufferReady.connect(this, &Stream::captureBufferReady);
> +
> +	int ret = m2m_->open();
> +	if (ret < 0)
> +		m2m_.reset();

Why are we re-creating and opening the m2m_ device everytime we create
a stream ? Isn't it the same created and open when the
V4L2M2MConverter is created ?

Thanks
  j

> +}
> +
> +int V4L2M2MConverter::Stream::configure(const StreamConfiguration &inputCfg,
> +					const StreamConfiguration &outputCfg)
> +{
> +	V4L2PixelFormat videoFormat =
> +		m2m_->output()->toV4L2PixelFormat(inputCfg.pixelFormat);
> +
> +	V4L2DeviceFormat format;
> +	format.fourcc = videoFormat;
> +	format.size = inputCfg.size;
> +	format.planesCount = 1;
> +	format.planes[0].bpl = inputCfg.stride;
> +
> +	int ret = m2m_->output()->setFormat(&format);
> +	if (ret < 0) {
> +		LOG(Converter, Error)
> +			<< "Failed to set input format: " << strerror(-ret);
> +		return ret;
> +	}
> +
> +	if (format.fourcc != videoFormat || format.size != inputCfg.size ||
> +	    format.planes[0].bpl != inputCfg.stride) {
> +		LOG(Converter, Error)
> +			<< "Input format not supported (requested "
> +			<< inputCfg.size << "-" << videoFormat
> +			<< ", got " << format << ")";
> +		return -EINVAL;
> +	}
> +
> +	/* Set the pixel format and size on the output. */
> +	videoFormat = m2m_->capture()->toV4L2PixelFormat(outputCfg.pixelFormat);
> +	format = {};
> +	format.fourcc = videoFormat;
> +	format.size = outputCfg.size;
> +
> +	ret = m2m_->capture()->setFormat(&format);
> +	if (ret < 0) {
> +		LOG(Converter, Error)
> +			<< "Failed to set output format: " << strerror(-ret);
> +		return ret;
> +	}
> +
> +	if (format.fourcc != videoFormat || format.size != outputCfg.size) {
> +		LOG(Converter, Error)
> +			<< "Output format not supported";
> +		return -EINVAL;
> +	}
> +
> +	inputBufferCount_ = inputCfg.bufferCount;
> +	outputBufferCount_ = outputCfg.bufferCount;
> +
> +	for (Mapping &mapping : converter_->mappings_) {
> +		ControlList ctrls;
> +		if (mapping.getInputSize() == inputCfg.size && mapping.getOutputSize() == outputCfg.size) {
> +			LOG(Converter, Debug)
> +				<< "Got a configuration match "
> +				<< inputCfg.size << " --> " << outputCfg.size;
> +			converter_->applyMapping(this, mapping);
> +		}
> +	}
> +
> +	return 0;
> +}
> +
> +int V4L2M2MConverter::Stream::exportBuffers(unsigned int count,
> +					    std::vector<std::unique_ptr<FrameBuffer>> *buffers)
> +{
> +	return m2m_->capture()->exportBuffers(count, buffers);
> +}
> +
> +int V4L2M2MConverter::Stream::start()
> +{
> +	int ret = m2m_->output()->importBuffers(inputBufferCount_);
> +	if (ret < 0)
> +		return ret;
> +
> +	ret = m2m_->capture()->importBuffers(outputBufferCount_);
> +	if (ret < 0) {
> +		stop();
> +		return ret;
> +	}
> +
> +	ret = m2m_->output()->streamOn();
> +	if (ret < 0) {
> +		stop();
> +		return ret;
> +	}
> +
> +	ret = m2m_->capture()->streamOn();
> +	if (ret < 0) {
> +		stop();
> +		return ret;
> +	}
> +
> +	return 0;
> +}
> +
> +void V4L2M2MConverter::Stream::stop()
> +{
> +	m2m_->capture()->streamOff();
> +	m2m_->output()->streamOff();
> +	m2m_->capture()->releaseBuffers();
> +	m2m_->output()->releaseBuffers();
> +}
> +
> +int V4L2M2MConverter::Stream::queueBuffers(FrameBuffer *input, FrameBuffer *output)
> +{
> +	int ret = m2m_->output()->queueBuffer(input);
> +	if (ret < 0)
> +		return ret;
> +
> +	ret = m2m_->capture()->queueBuffer(output);
> +	if (ret < 0)
> +		return ret;
> +
> +	return 0;
> +}
> +
> +std::string V4L2M2MConverter::Stream::logPrefix() const
> +{
> +	return "stream" + std::to_string(index_);
> +}
> +
> +void V4L2M2MConverter::Stream::outputBufferReady(FrameBuffer *buffer)
> +{
> +	auto it = converter_->queue_.find(buffer);
> +	if (it == converter_->queue_.end())
> +		return;
> +
> +	if (!--it->second) {
> +		converter_->inputBufferReady.emit(buffer);
> +		converter_->queue_.erase(it);
> +	}
> +}
> +
> +void V4L2M2MConverter::Stream::captureBufferReady(FrameBuffer *buffer)
> +{
> +	converter_->outputBufferReady.emit(buffer);
> +}
> +
> +/* -----------------------------------------------------------------------------
> + * V4L2M2MConverter
> + */
> +
> +V4L2M2MConverter::V4L2M2MConverter(MediaDevice *media)
> +	: Converter(media)
> +{
> +	if (deviceNode_.empty())
> +		return;
> +
> +	m2m_ = std::make_unique<V4L2M2MDevice>(deviceNode_);
> +	int ret = m2m_->open();
> +	if (ret < 0) {
> +		m2m_.reset();
> +		return;
> +	}
> +}
> +
> +int V4L2M2MConverter::loadConfiguration(const std::string &filename)
> +{
> +	LOG(Converter, Debug)
> +		<< "Parsing configuration file " << filename;
> +
> +	File file(filename);
> +
> +	if (!file.open(File::OpenModeFlag::ReadOnly)) {
> +		int ret = file.error();
> +		LOG(Converter, Error)
> +			<< "Failed to open configuration file "
> +			<< filename << ": " << strerror(-ret);
> +		return ret;
> +	}
> +
> +	std::unique_ptr<libcamera::YamlObject> data = YamlParser::parse(file);
> +	if (!data)
> +		return -EINVAL;
> +
> +	if (!data->contains("mappings")) {
> +		LOG(Converter, Error)
> +			<< "Vertex mapping key missing";
> +		return -EINVAL;
> +	}
> +
> +	const YamlObject &mappings = (*data)["mappings"];
> +	if (!mappings.isList() || mappings.size() == 0) {
> +		LOG(Converter, Error)
> +			<< "Invalid mappings entry";
> +		return -EINVAL;
> +	}
> +
> +	LOG(Converter, Debug)
> +		<< "Parsing " << mappings.size() << " mappings";
> +	mappings_.clear();
> +	mappings_.reserve(mappings.size());
> +
> +	for (std::size_t i = 0; i < mappings.size(); i++) {
> +		const YamlObject &mapping = mappings[i];
> +		if (!mapping.isDictionary()) {
> +			LOG(Converter, Error)
> +				<< "Mapping is not a dictionnary";
> +			return -EINVAL;
> +		}
> +
> +		if (!mapping.contains("input-resolution")) {
> +			LOG(Converter, Error)
> +				<< "Input resolution missing";
> +			return -EINVAL;
> +		}
> +
> +		if (!mapping.contains("output-resolution")) {
> +			LOG(Converter, Error)
> +				<< "Output resolution missing";
> +			return -EINVAL;
> +		}
> +
> +		if (!mapping.contains("mapping")) {
> +			LOG(Converter, Error)
> +				<< "Mapping table missing";
> +			return -EINVAL;
> +		}
> +
> +		const YamlObject &input_res = mapping["input-resolution"];
> +		if (!input_res.isList() || input_res.size() != 2) {
> +			LOG(Converter, Error)
> +				<< "Incorrect input resolution";
> +			return -EINVAL;
> +		}
> +
> +		const YamlObject &output_res = mapping["output-resolution"];
> +		if (!output_res.isList() || output_res.size() != 2) {
> +			LOG(Converter, Error)
> +				<< "Incorrect output resolution";
> +			return -EINVAL;
> +		}
> +
> +		const YamlObject &map = mapping["mapping"];
> +		if (!map.isList() || map.size() == 0) {
> +			LOG(Converter, Error)
> +				<< "Incorrect mapping entries";
> +			return -EINVAL;
> +		}
> +
> +		Size input(input_res[0].get<uint32_t>(0), input_res[1].get<uint32_t>(0));
> +		Size output(output_res[0].get<uint32_t>(0), output_res[1].get<uint32_t>(0));
> +		const auto &mapVector = map.getList<uint32_t>().value_or(utils::defopt);
> +
> +		LOG(Converter, Debug)
> +			<< "Input/Output mapping resolution " << input << " ---> " << output;
> +		mappings_.emplace_back(Mapping(input, output, mapVector));
> +	}
> +
> +	return mappings.size();
> +}
> +
> +std::vector<PixelFormat> V4L2M2MConverter::formats(PixelFormat input)
> +{
> +	if (!m2m_)
> +		return {};
> +
> +	/*
> +	 * Set the format on the input side (V4L2 output) of the converter to
> +	 * enumerate the conversion capabilities on its output (V4L2 capture).
> +	 */
> +	V4L2DeviceFormat v4l2Format;
> +	v4l2Format.fourcc = m2m_->output()->toV4L2PixelFormat(input);
> +	v4l2Format.size = { 1, 1 };
> +
> +	int ret = m2m_->output()->setFormat(&v4l2Format);
> +	if (ret < 0) {
> +		LOG(Converter, Error)
> +			<< "Failed to set format: " << strerror(-ret);
> +		return {};
> +	}
> +
> +	if (v4l2Format.fourcc != m2m_->output()->toV4L2PixelFormat(input)) {
> +		LOG(Converter, Debug)
> +			<< "Input format " << input << " not supported.";
> +		return {};
> +	}
> +
> +	std::vector<PixelFormat> pixelFormats;
> +
> +	for (const auto &format : m2m_->capture()->formats()) {
> +		PixelFormat pixelFormat = format.first.toPixelFormat();
> +		if (pixelFormat)
> +			pixelFormats.push_back(pixelFormat);
> +	}
> +
> +	return pixelFormats;
> +}
> +
> +SizeRange V4L2M2MConverter::sizes(const Size &input)
> +{
> +	if (!m2m_)
> +		return {};
> +
> +	/*
> +	 * Set the size on the input side (V4L2 output) of the converter to
> +	 * enumerate the scaling capabilities on its output (V4L2 capture).
> +	 */
> +	V4L2DeviceFormat format;
> +	format.fourcc = V4L2PixelFormat();
> +	format.size = input;
> +
> +	int ret = m2m_->output()->setFormat(&format);
> +	if (ret < 0) {
> +		LOG(Converter, Error)
> +			<< "Failed to set format: " << strerror(-ret);
> +		return {};
> +	}
> +
> +	SizeRange sizes;
> +
> +	format.size = { 1, 1 };
> +	ret = m2m_->capture()->setFormat(&format);
> +	if (ret < 0) {
> +		LOG(Converter, Error)
> +			<< "Failed to set format: " << strerror(-ret);
> +		return {};
> +	}
> +
> +	sizes.min = format.size;
> +
> +	format.size = { UINT_MAX, UINT_MAX };
> +	ret = m2m_->capture()->setFormat(&format);
> +	if (ret < 0) {
> +		LOG(Converter, Error)
> +			<< "Failed to set format: " << strerror(-ret);
> +		return {};
> +	}
> +
> +	sizes.max = format.size;
> +
> +	return sizes;
> +}
> +
> +std::tuple<unsigned int, unsigned int>
> +V4L2M2MConverter::strideAndFrameSize(const PixelFormat &pixelFormat,
> +				     const Size &size)
> +{
> +	V4L2DeviceFormat format;
> +	format.fourcc = m2m_->capture()->toV4L2PixelFormat(pixelFormat);
> +	format.size = size;
> +
> +	int ret = m2m_->capture()->tryFormat(&format);
> +	if (ret < 0)
> +		return std::make_tuple(0, 0);
> +
> +	return std::make_tuple(format.planes[0].bpl, format.planes[0].size);
> +}
> +
> +int V4L2M2MConverter::configure(const StreamConfiguration &inputCfg,
> +				const std::vector<std::reference_wrapper<StreamConfiguration>> &outputCfgs)
> +{
> +	int ret = 0;
> +
> +	streams_.clear();
> +	streams_.reserve(outputCfgs.size());
> +
> +	for (unsigned int i = 0; i < outputCfgs.size(); ++i) {
> +		Stream &stream = streams_.emplace_back(this, i);
> +
> +		if (!stream.isValid()) {
> +			LOG(Converter, Error)
> +				<< "Failed to create stream " << i;
> +			ret = -EINVAL;
> +			break;
> +		}
> +
> +		ret = stream.configure(inputCfg, outputCfgs[i]);
> +		if (ret < 0)
> +			break;
> +	}
> +
> +	if (ret < 0) {
> +		streams_.clear();
> +		return ret;
> +	}
> +
> +	return 0;
> +}
> +
> +int V4L2M2MConverter::exportBuffers(unsigned int output, unsigned int count,
> +				    std::vector<std::unique_ptr<FrameBuffer>> *buffers)
> +{
> +	if (output >= streams_.size())
> +		return -EINVAL;
> +
> +	return streams_[output].exportBuffers(count, buffers);
> +}
> +
> +int V4L2M2MConverter::start()
> +{
> +	int ret;
> +
> +	for (Stream &stream : streams_) {
> +		ret = stream.start();
> +		if (ret < 0) {
> +			stop();
> +			return ret;
> +		}
> +	}
> +
> +	return 0;
> +}
> +
> +void V4L2M2MConverter::stop()
> +{
> +	for (Stream &stream : utils::reverse(streams_))
> +		stream.stop();
> +}
> +
> +int V4L2M2MConverter::queueBuffers(FrameBuffer *input,
> +				   const std::map<unsigned int, FrameBuffer *> &outputs)
> +{
> +	unsigned int mask = 0;
> +	int ret;
> +
> +	/*
> +	 * Validate the outputs as a sanity check: at least one output is
> +	 * required, all outputs must reference a valid stream and no two
> +	 * outputs can reference the same stream.
> +	 */
> +	if (outputs.empty())
> +		return -EINVAL;
> +
> +	for (auto [index, buffer] : outputs) {
> +		if (!buffer)
> +			return -EINVAL;
> +		if (index >= streams_.size())
> +			return -EINVAL;
> +		if (mask & (1 << index))
> +			return -EINVAL;
> +
> +		mask |= 1 << index;
> +	}
> +
> +	/* Queue the input and output buffers to all the streams. */
> +	for (auto [index, buffer] : outputs) {
> +		ret = streams_[index].queueBuffers(input, buffer);
> +		if (ret < 0)
> +			return ret;
> +	}
> +
> +	/*
> +	 * Add the input buffer to the queue, with the number of streams as a
> +	 * reference count. Completion of the input buffer will be signalled by
> +	 * the stream that releases the last reference.
> +	 */
> +	queue_.emplace(std::piecewise_construct,
> +		       std::forward_as_tuple(input),
> +		       std::forward_as_tuple(outputs.size()));
> +
> +	return 0;
> +}
> +
> +REGISTER_CONVERTER("v4l2_m2m", V4L2M2MConverter, "pxp")
> +
> +} /* namespace libcamera */
> diff --git a/src/libcamera/meson.build b/src/libcamera/meson.build
> index a261d4b4..b12c8401 100644
> --- a/src/libcamera/meson.build
> +++ b/src/libcamera/meson.build
> @@ -14,6 +14,7 @@ libcamera_sources = files([
>      'control_serializer.cpp',
>      'control_validator.cpp',
>      'converter.cpp',
> +    'converter_v4l2_m2m.cpp',
>      'delayed_controls.cpp',
>      'device_enumerator.cpp',
>      'device_enumerator_sysfs.cpp',
> --
> 2.37.3
>

Hi Jacopo,

On 9/15/22 11:46, Jacopo Mondi wrote:
> Hi Xavier
> 
> On Thu, Sep 08, 2022 at 08:48:42PM +0200, Xavier Roumegue via libcamera-devel wrote:
>> Introduce a converter implementation relying on a v4l2 m2m device, mostly
>> based on the current simple pipeline converter implementation.
>>
>> The main change is the introduction of Mapping object which can be
>> loaded through a configuration file which define vertices remapping
>> coordinates. Those latters can be applied by any classes derived from
>> this base class which define the apply_mapping() method.
>>
> 
> I'll skip questions on things not clear to me about the converter
> implementation as I understand this basically comes from the simple
> pipeline converter implementation [*]
> 
> I would then focus more on two general questions:
> 1) Do we want to make a dir for converter/post-processors instead of
> placing them in src/libcamera
Might make sense, or could be done later - that's not my call.
I can move them to a dedicated folder on the next patch set version if there is 
a willing to do so.
> 
> 2) Is the format expected in the configuration file documented
> anywhere ?
Nope. I will do it on the next version, hopefully on an agreed configuration 
file format.
> 
> [*] Not true, I will ask a question anyway, see below :)
> 
>> Signed-off-by: Xavier Roumegue <xavier.roumegue@oss.nxp.com>
>> ---
>>   .../libcamera/internal/converter_v4l2_m2m.h   | 120 +++++
>>   include/libcamera/internal/meson.build        |   1 +
>>   src/libcamera/converter_v4l2_m2m.cpp          | 504 ++++++++++++++++++
>>   src/libcamera/meson.build                     |   1 +
>>   4 files changed, 626 insertions(+)
>>   create mode 100644 include/libcamera/internal/converter_v4l2_m2m.h
>>   create mode 100644 src/libcamera/converter_v4l2_m2m.cpp
>>
>> diff --git a/include/libcamera/internal/converter_v4l2_m2m.h b/include/libcamera/internal/converter_v4l2_m2m.h
>> new file mode 100644
>> index 00000000..3667b128
>> --- /dev/null
>> +++ b/include/libcamera/internal/converter_v4l2_m2m.h
>> @@ -0,0 +1,120 @@
>> +/* SPDX-License-Identifier: LGPL-2.1-or-later */
>> +/*
>> + * Copyright 2022 NXP
>> + *
>> + * converter_v4l2_m2m.h - V4l2 M2M Format converter interface
>> + */
>> +
>> +#pragma once
>> +
>> +#include <functional>
>> +#include <map>
>> +#include <memory>
>> +#include <string>
>> +#include <tuple>
>> +#include <vector>
>> +
>> +#include <libcamera/base/log.h>
>> +#include <libcamera/base/signal.h>
>> +
>> +#include <libcamera/geometry.h>
>> +#include <libcamera/pixel_format.h>
>> +
>> +#include "libcamera/internal/converter.h"
>> +
>> +namespace libcamera {
>> +
>> +class FrameBuffer;
>> +class MediaDevice;
>> +class Size;
>> +class SizeRange;
>> +struct StreamConfiguration;
>> +class V4L2M2MDevice;
>> +class V4L2M2MConverter;
>> +class Converter;
>> +
>> +class V4L2M2MConverter : public Converter
>> +{
>> +protected:
>> +	class Mapping
>> +	{
>> +	public:
>> +		Mapping(const Size &input, const Size &output, const std::vector<uint32_t> &map)
>> +			: input_(input), output_(output), map_(map) {}
>> +		Size getInputSize() const { return input_; }
>> +		Size getOutputSize() const { return output_; }
>> +		std::size_t getLength() const { return map_.size(); }
>> +		const uint32_t *getMapping() const { return map_.data(); }
>> +
>> +	private:
>> +		Size input_;
>> +		Size output_;
>> +		std::vector<uint32_t> map_;
>> +	};
>> +
>> +	class Stream : protected Loggable
>> +	{
>> +	public:
>> +		Stream(V4L2M2MConverter *converter, unsigned int index);
>> +
>> +		bool isValid() const { return m2m_ != nullptr; }
>> +
>> +		int configure(const StreamConfiguration &inputCfg,
>> +			      const StreamConfiguration &outputCfg);
>> +		int exportBuffers(unsigned int count,
>> +				  std::vector<std::unique_ptr<FrameBuffer>> *buffers);
>> +
>> +		int start();
>> +		void stop();
>> +
>> +		int queueBuffers(FrameBuffer *input, FrameBuffer *output);
>> +		std::unique_ptr<V4L2M2MDevice> m2m_;
>> +
>> +	protected:
>> +		std::string logPrefix() const override;
>> +
>> +	private:
>> +		void captureBufferReady(FrameBuffer *buffer);
>> +		void outputBufferReady(FrameBuffer *buffer);
>> +
>> +		V4L2M2MConverter *converter_;
>> +		unsigned int index_;
>> +
>> +		unsigned int inputBufferCount_;
>> +		unsigned int outputBufferCount_;
>> +	};
>> +
>> +	std::unique_ptr<V4L2M2MDevice> m2m_;
>> +
>> +	std::vector<Stream> streams_;
>> +	std::vector<Mapping> mappings_;
>> +	std::map<FrameBuffer *, unsigned int> queue_;
>> +
>> +public:
>> +	V4L2M2MConverter(MediaDevice *media);
>> +
>> +	int loadConfiguration(const std::string &filename) override;
>> +
>> +	bool isValid() const { return m2m_ != nullptr; }
>> +
>> +	std::vector<PixelFormat> formats(PixelFormat input);
>> +	SizeRange sizes(const Size &input);
>> +
>> +	std::tuple<unsigned int, unsigned int>
>> +	strideAndFrameSize(const PixelFormat &pixelFormat, const Size &size);
>> +
>> +	int configure(const StreamConfiguration &inputCfg,
>> +		      const std::vector<std::reference_wrapper<StreamConfiguration>> &outputCfg);
>> +	int exportBuffers(unsigned int ouput, unsigned int count,
>> +			  std::vector<std::unique_ptr<FrameBuffer>> *buffers);
>> +
>> +	int start();
>> +	void stop();
>> +
>> +	int queueBuffers(FrameBuffer *input,
>> +			 const std::map<unsigned int, FrameBuffer *> &outputs);
>> +
>> +	virtual int applyMapping([[maybe_unused]] Stream *stream, [[maybe_unused]] Mapping &mapping) { return 0; };
>> +};
>> +
>> +} /* namespace libcamera */
>> diff --git a/include/libcamera/internal/meson.build b/include/libcamera/internal/meson.build
>> index 8f50d755..132de5ef 100644
>> --- a/include/libcamera/internal/meson.build
>> +++ b/include/libcamera/internal/meson.build
>> @@ -20,6 +20,7 @@ libcamera_internal_headers = files([
>>       'control_serializer.h',
>>       'control_validator.h',
>>       'converter.h',
>> +    'converter_v4l2_m2m.h',
>>       'delayed_controls.h',
>>       'device_enumerator.h',
>>       'device_enumerator_sysfs.h',
>> diff --git a/src/libcamera/converter_v4l2_m2m.cpp b/src/libcamera/converter_v4l2_m2m.cpp
>> new file mode 100644
>> index 00000000..942e6e6f
>> --- /dev/null
>> +++ b/src/libcamera/converter_v4l2_m2m.cpp
>> @@ -0,0 +1,504 @@
>> +/* SPDX-License-Identifier: LGPL-2.1-or-later */
>> +/*
>> + * Copyright (C) 2020, Laurent Pinchart
>> + * Copyright 2022 NXP
>> + *
>> + * converter_v4l2_m2m.cpp - V4L2 M2M Format converter
>> + */
>> +
>> +#include <algorithm>
>> +#include <limits.h>
>> +
>> +#include <libcamera/base/file.h>
>> +#include <libcamera/base/log.h>
>> +#include <libcamera/base/signal.h>
>> +#include <libcamera/base/utils.h>
>> +
>> +#include <libcamera/framebuffer.h>
>> +#include <libcamera/geometry.h>
>> +#include <libcamera/stream.h>
>> +
>> +#include "libcamera/internal/converter_v4l2_m2m.h"
>> +#include "libcamera/internal/media_device.h"
>> +#include "libcamera/internal/v4l2_videodevice.h"
>> +#include "libcamera/internal/yaml_parser.h"
>> +
>> +namespace libcamera {
>> +
>> +LOG_DECLARE_CATEGORY(Converter)
>> +
>> +/* -----------------------------------------------------------------------------
>> + * V4L2M2MConverter::Stream
>> + */
>> +
>> +V4L2M2MConverter::Stream::Stream(V4L2M2MConverter *converter, unsigned int index)
>> +	: converter_(converter), index_(index)
>> +{
>> +	m2m_ = std::make_unique<V4L2M2MDevice>(converter->deviceNode_);
>> +
>> +	m2m_->output()->bufferReady.connect(this, &Stream::outputBufferReady);
>> +	m2m_->capture()->bufferReady.connect(this, &Stream::captureBufferReady);
>> +
>> +	int ret = m2m_->open();
>> +	if (ret < 0)
>> +		m2m_.reset();
> 
> Why are we re-creating and opening the m2m_ device everytime we create
> a stream ? Isn't it the same created and open when the
> V4L2M2MConverter is created ?Would it be for proper reference counting ?

> 
> Thanks
>    j
> 
>> +}
>> +
>> +int V4L2M2MConverter::Stream::configure(const StreamConfiguration &inputCfg,
>> +					const StreamConfiguration &outputCfg)
>> +{
>> +	V4L2PixelFormat videoFormat =
>> +		m2m_->output()->toV4L2PixelFormat(inputCfg.pixelFormat);
>> +
>> +	V4L2DeviceFormat format;
>> +	format.fourcc = videoFormat;
>> +	format.size = inputCfg.size;
>> +	format.planesCount = 1;
>> +	format.planes[0].bpl = inputCfg.stride;
>> +
>> +	int ret = m2m_->output()->setFormat(&format);
>> +	if (ret < 0) {
>> +		LOG(Converter, Error)
>> +			<< "Failed to set input format: " << strerror(-ret);
>> +		return ret;
>> +	}
>> +
>> +	if (format.fourcc != videoFormat || format.size != inputCfg.size ||
>> +	    format.planes[0].bpl != inputCfg.stride) {
>> +		LOG(Converter, Error)
>> +			<< "Input format not supported (requested "
>> +			<< inputCfg.size << "-" << videoFormat
>> +			<< ", got " << format << ")";
>> +		return -EINVAL;
>> +	}
>> +
>> +	/* Set the pixel format and size on the output. */
>> +	videoFormat = m2m_->capture()->toV4L2PixelFormat(outputCfg.pixelFormat);
>> +	format = {};
>> +	format.fourcc = videoFormat;
>> +	format.size = outputCfg.size;
>> +
>> +	ret = m2m_->capture()->setFormat(&format);
>> +	if (ret < 0) {
>> +		LOG(Converter, Error)
>> +			<< "Failed to set output format: " << strerror(-ret);
>> +		return ret;
>> +	}
>> +
>> +	if (format.fourcc != videoFormat || format.size != outputCfg.size) {
>> +		LOG(Converter, Error)
>> +			<< "Output format not supported";
>> +		return -EINVAL;
>> +	}
>> +
>> +	inputBufferCount_ = inputCfg.bufferCount;
>> +	outputBufferCount_ = outputCfg.bufferCount;
>> +
>> +	for (Mapping &mapping : converter_->mappings_) {
>> +		ControlList ctrls;
>> +		if (mapping.getInputSize() == inputCfg.size && mapping.getOutputSize() == outputCfg.size) {
>> +			LOG(Converter, Debug)
>> +				<< "Got a configuration match "
>> +				<< inputCfg.size << " --> " << outputCfg.size;
>> +			converter_->applyMapping(this, mapping);
>> +		}
>> +	}
>> +
>> +	return 0;
>> +}
>> +
>> +int V4L2M2MConverter::Stream::exportBuffers(unsigned int count,
>> +					    std::vector<std::unique_ptr<FrameBuffer>> *buffers)
>> +{
>> +	return m2m_->capture()->exportBuffers(count, buffers);
>> +}
>> +
>> +int V4L2M2MConverter::Stream::start()
>> +{
>> +	int ret = m2m_->output()->importBuffers(inputBufferCount_);
>> +	if (ret < 0)
>> +		return ret;
>> +
>> +	ret = m2m_->capture()->importBuffers(outputBufferCount_);
>> +	if (ret < 0) {
>> +		stop();
>> +		return ret;
>> +	}
>> +
>> +	ret = m2m_->output()->streamOn();
>> +	if (ret < 0) {
>> +		stop();
>> +		return ret;
>> +	}
>> +
>> +	ret = m2m_->capture()->streamOn();
>> +	if (ret < 0) {
>> +		stop();
>> +		return ret;
>> +	}
>> +
>> +	return 0;
>> +}
>> +
>> +void V4L2M2MConverter::Stream::stop()
>> +{
>> +	m2m_->capture()->streamOff();
>> +	m2m_->output()->streamOff();
>> +	m2m_->capture()->releaseBuffers();
>> +	m2m_->output()->releaseBuffers();
>> +}
>> +
>> +int V4L2M2MConverter::Stream::queueBuffers(FrameBuffer *input, FrameBuffer *output)
>> +{
>> +	int ret = m2m_->output()->queueBuffer(input);
>> +	if (ret < 0)
>> +		return ret;
>> +
>> +	ret = m2m_->capture()->queueBuffer(output);
>> +	if (ret < 0)
>> +		return ret;
>> +
>> +	return 0;
>> +}
>> +
>> +std::string V4L2M2MConverter::Stream::logPrefix() const
>> +{
>> +	return "stream" + std::to_string(index_);
>> +}
>> +
>> +void V4L2M2MConverter::Stream::outputBufferReady(FrameBuffer *buffer)
>> +{
>> +	auto it = converter_->queue_.find(buffer);
>> +	if (it == converter_->queue_.end())
>> +		return;
>> +
>> +	if (!--it->second) {
>> +		converter_->inputBufferReady.emit(buffer);
>> +		converter_->queue_.erase(it);
>> +	}
>> +}
>> +
>> +void V4L2M2MConverter::Stream::captureBufferReady(FrameBuffer *buffer)
>> +{
>> +	converter_->outputBufferReady.emit(buffer);
>> +}
>> +
>> +/* -----------------------------------------------------------------------------
>> + * V4L2M2MConverter
>> + */
>> +
>> +V4L2M2MConverter::V4L2M2MConverter(MediaDevice *media)
>> +	: Converter(media)
>> +{
>> +	if (deviceNode_.empty())
>> +		return;
>> +
>> +	m2m_ = std::make_unique<V4L2M2MDevice>(deviceNode_);
>> +	int ret = m2m_->open();
>> +	if (ret < 0) {
>> +		m2m_.reset();
>> +		return;
>> +	}
>> +}
>> +
>> +int V4L2M2MConverter::loadConfiguration(const std::string &filename)
>> +{
>> +	LOG(Converter, Debug)
>> +		<< "Parsing configuration file " << filename;
>> +
>> +	File file(filename);
>> +
>> +	if (!file.open(File::OpenModeFlag::ReadOnly)) {
>> +		int ret = file.error();
>> +		LOG(Converter, Error)
>> +			<< "Failed to open configuration file "
>> +			<< filename << ": " << strerror(-ret);
>> +		return ret;
>> +	}
>> +
>> +	std::unique_ptr<libcamera::YamlObject> data = YamlParser::parse(file);
>> +	if (!data)
>> +		return -EINVAL;
>> +
>> +	if (!data->contains("mappings")) {
>> +		LOG(Converter, Error)
>> +			<< "Vertex mapping key missing";
>> +		return -EINVAL;
>> +	}
>> +
>> +	const YamlObject &mappings = (*data)["mappings"];
>> +	if (!mappings.isList() || mappings.size() == 0) {
>> +		LOG(Converter, Error)
>> +			<< "Invalid mappings entry";
>> +		return -EINVAL;
>> +	}
>> +
>> +	LOG(Converter, Debug)
>> +		<< "Parsing " << mappings.size() << " mappings";
>> +	mappings_.clear();
>> +	mappings_.reserve(mappings.size());
>> +
>> +	for (std::size_t i = 0; i < mappings.size(); i++) {
>> +		const YamlObject &mapping = mappings[i];
>> +		if (!mapping.isDictionary()) {
>> +			LOG(Converter, Error)
>> +				<< "Mapping is not a dictionnary";
>> +			return -EINVAL;
>> +		}
>> +
>> +		if (!mapping.contains("input-resolution")) {
>> +			LOG(Converter, Error)
>> +				<< "Input resolution missing";
>> +			return -EINVAL;
>> +		}
>> +
>> +		if (!mapping.contains("output-resolution")) {
>> +			LOG(Converter, Error)
>> +				<< "Output resolution missing";
>> +			return -EINVAL;
>> +		}
>> +
>> +		if (!mapping.contains("mapping")) {
>> +			LOG(Converter, Error)
>> +				<< "Mapping table missing";
>> +			return -EINVAL;
>> +		}
>> +
>> +		const YamlObject &input_res = mapping["input-resolution"];
>> +		if (!input_res.isList() || input_res.size() != 2) {
>> +			LOG(Converter, Error)
>> +				<< "Incorrect input resolution";
>> +			return -EINVAL;
>> +		}
>> +
>> +		const YamlObject &output_res = mapping["output-resolution"];
>> +		if (!output_res.isList() || output_res.size() != 2) {
>> +			LOG(Converter, Error)
>> +				<< "Incorrect output resolution";
>> +			return -EINVAL;
>> +		}
>> +
>> +		const YamlObject &map = mapping["mapping"];
>> +		if (!map.isList() || map.size() == 0) {
>> +			LOG(Converter, Error)
>> +				<< "Incorrect mapping entries";
>> +			return -EINVAL;
>> +		}
>> +
>> +		Size input(input_res[0].get<uint32_t>(0), input_res[1].get<uint32_t>(0));
>> +		Size output(output_res[0].get<uint32_t>(0), output_res[1].get<uint32_t>(0));
>> +		const auto &mapVector = map.getList<uint32_t>().value_or(utils::defopt);
>> +
>> +		LOG(Converter, Debug)
>> +			<< "Input/Output mapping resolution " << input << " ---> " << output;
>> +		mappings_.emplace_back(Mapping(input, output, mapVector));
>> +	}
>> +
>> +	return mappings.size();
>> +}
>> +
>> +std::vector<PixelFormat> V4L2M2MConverter::formats(PixelFormat input)
>> +{
>> +	if (!m2m_)
>> +		return {};
>> +
>> +	/*
>> +	 * Set the format on the input side (V4L2 output) of the converter to
>> +	 * enumerate the conversion capabilities on its output (V4L2 capture).
>> +	 */
>> +	V4L2DeviceFormat v4l2Format;
>> +	v4l2Format.fourcc = m2m_->output()->toV4L2PixelFormat(input);
>> +	v4l2Format.size = { 1, 1 };
>> +
>> +	int ret = m2m_->output()->setFormat(&v4l2Format);
>> +	if (ret < 0) {
>> +		LOG(Converter, Error)
>> +			<< "Failed to set format: " << strerror(-ret);
>> +		return {};
>> +	}
>> +
>> +	if (v4l2Format.fourcc != m2m_->output()->toV4L2PixelFormat(input)) {
>> +		LOG(Converter, Debug)
>> +			<< "Input format " << input << " not supported.";
>> +		return {};
>> +	}
>> +
>> +	std::vector<PixelFormat> pixelFormats;
>> +
>> +	for (const auto &format : m2m_->capture()->formats()) {
>> +		PixelFormat pixelFormat = format.first.toPixelFormat();
>> +		if (pixelFormat)
>> +			pixelFormats.push_back(pixelFormat);
>> +	}
>> +
>> +	return pixelFormats;
>> +}
>> +
>> +SizeRange V4L2M2MConverter::sizes(const Size &input)
>> +{
>> +	if (!m2m_)
>> +		return {};
>> +
>> +	/*
>> +	 * Set the size on the input side (V4L2 output) of the converter to
>> +	 * enumerate the scaling capabilities on its output (V4L2 capture).
>> +	 */
>> +	V4L2DeviceFormat format;
>> +	format.fourcc = V4L2PixelFormat();
>> +	format.size = input;
>> +
>> +	int ret = m2m_->output()->setFormat(&format);
>> +	if (ret < 0) {
>> +		LOG(Converter, Error)
>> +			<< "Failed to set format: " << strerror(-ret);
>> +		return {};
>> +	}
>> +
>> +	SizeRange sizes;
>> +
>> +	format.size = { 1, 1 };
>> +	ret = m2m_->capture()->setFormat(&format);
>> +	if (ret < 0) {
>> +		LOG(Converter, Error)
>> +			<< "Failed to set format: " << strerror(-ret);
>> +		return {};
>> +	}
>> +
>> +	sizes.min = format.size;
>> +
>> +	format.size = { UINT_MAX, UINT_MAX };
>> +	ret = m2m_->capture()->setFormat(&format);
>> +	if (ret < 0) {
>> +		LOG(Converter, Error)
>> +			<< "Failed to set format: " << strerror(-ret);
>> +		return {};
>> +	}
>> +
>> +	sizes.max = format.size;
>> +
>> +	return sizes;
>> +}
>> +
>> +std::tuple<unsigned int, unsigned int>
>> +V4L2M2MConverter::strideAndFrameSize(const PixelFormat &pixelFormat,
>> +				     const Size &size)
>> +{
>> +	V4L2DeviceFormat format;
>> +	format.fourcc = m2m_->capture()->toV4L2PixelFormat(pixelFormat);
>> +	format.size = size;
>> +
>> +	int ret = m2m_->capture()->tryFormat(&format);
>> +	if (ret < 0)
>> +		return std::make_tuple(0, 0);
>> +
>> +	return std::make_tuple(format.planes[0].bpl, format.planes[0].size);
>> +}
>> +
>> +int V4L2M2MConverter::configure(const StreamConfiguration &inputCfg,
>> +				const std::vector<std::reference_wrapper<StreamConfiguration>> &outputCfgs)
>> +{
>> +	int ret = 0;
>> +
>> +	streams_.clear();
>> +	streams_.reserve(outputCfgs.size());
>> +
>> +	for (unsigned int i = 0; i < outputCfgs.size(); ++i) {
>> +		Stream &stream = streams_.emplace_back(this, i);
>> +
>> +		if (!stream.isValid()) {
>> +			LOG(Converter, Error)
>> +				<< "Failed to create stream " << i;
>> +			ret = -EINVAL;
>> +			break;
>> +		}
>> +
>> +		ret = stream.configure(inputCfg, outputCfgs[i]);
>> +		if (ret < 0)
>> +			break;
>> +	}
>> +
>> +	if (ret < 0) {
>> +		streams_.clear();
>> +		return ret;
>> +	}
>> +
>> +	return 0;
>> +}
>> +
>> +int V4L2M2MConverter::exportBuffers(unsigned int output, unsigned int count,
>> +				    std::vector<std::unique_ptr<FrameBuffer>> *buffers)
>> +{
>> +	if (output >= streams_.size())
>> +		return -EINVAL;
>> +
>> +	return streams_[output].exportBuffers(count, buffers);
>> +}
>> +
>> +int V4L2M2MConverter::start()
>> +{
>> +	int ret;
>> +
>> +	for (Stream &stream : streams_) {
>> +		ret = stream.start();
>> +		if (ret < 0) {
>> +			stop();
>> +			return ret;
>> +		}
>> +	}
>> +
>> +	return 0;
>> +}
>> +
>> +void V4L2M2MConverter::stop()
>> +{
>> +	for (Stream &stream : utils::reverse(streams_))
>> +		stream.stop();
>> +}
>> +
>> +int V4L2M2MConverter::queueBuffers(FrameBuffer *input,
>> +				   const std::map<unsigned int, FrameBuffer *> &outputs)
>> +{
>> +	unsigned int mask = 0;
>> +	int ret;
>> +
>> +	/*
>> +	 * Validate the outputs as a sanity check: at least one output is
>> +	 * required, all outputs must reference a valid stream and no two
>> +	 * outputs can reference the same stream.
>> +	 */
>> +	if (outputs.empty())
>> +		return -EINVAL;
>> +
>> +	for (auto [index, buffer] : outputs) {
>> +		if (!buffer)
>> +			return -EINVAL;
>> +		if (index >= streams_.size())
>> +			return -EINVAL;
>> +		if (mask & (1 << index))
>> +			return -EINVAL;
>> +
>> +		mask |= 1 << index;
>> +	}
>> +
>> +	/* Queue the input and output buffers to all the streams. */
>> +	for (auto [index, buffer] : outputs) {
>> +		ret = streams_[index].queueBuffers(input, buffer);
>> +		if (ret < 0)
>> +			return ret;
>> +	}
>> +
>> +	/*
>> +	 * Add the input buffer to the queue, with the number of streams as a
>> +	 * reference count. Completion of the input buffer will be signalled by
>> +	 * the stream that releases the last reference.
>> +	 */
>> +	queue_.emplace(std::piecewise_construct,
>> +		       std::forward_as_tuple(input),
>> +		       std::forward_as_tuple(outputs.size()));
>> +
>> +	return 0;
>> +}
>> +
>> +REGISTER_CONVERTER("v4l2_m2m", V4L2M2MConverter, "pxp")
>> +
>> +} /* namespace libcamera */
>> diff --git a/src/libcamera/meson.build b/src/libcamera/meson.build
>> index a261d4b4..b12c8401 100644
>> --- a/src/libcamera/meson.build
>> +++ b/src/libcamera/meson.build
>> @@ -14,6 +14,7 @@ libcamera_sources = files([
>>       'control_serializer.cpp',
>>       'control_validator.cpp',
>>       'converter.cpp',
>> +    'converter_v4l2_m2m.cpp',
>>       'delayed_controls.cpp',
>>       'device_enumerator.cpp',
>>       'device_enumerator_sysfs.cpp',
>> --
>> 2.37.3
>>

Hello,

s/v4l2 m2m/V4L2 M2M/ in the subject line.

On Mon, Sep 19, 2022 at 10:45:59AM +0200, Xavier Roumegue (OSS) via libcamera-devel wrote:
> On 9/15/22 11:46, Jacopo Mondi wrote:
> > On Thu, Sep 08, 2022 at 08:48:42PM +0200, Xavier Roumegue via libcamera-devel wrote:
> >> Introduce a converter implementation relying on a v4l2 m2m device, mostly

Here too.

> >> based on the current simple pipeline converter implementation.
> >>
> >> The main change is the introduction of Mapping object which can be
> >> loaded through a configuration file which define vertices remapping
> >> coordinates. Those latters can be applied by any classes derived from
> >> this base class which define the apply_mapping() method.

s/apply_mapping() method/applyMapping() function/

Could you please split this patch in two, with a first patch that copies
the simple pipeline handler converter with minimal changes (ideally no
changes beside the class rename, if there are any other changes, please
document them in the commit message), and a second patch to add support
for the mappings ? That will be easier to review. I would actually
squash patch 10/14 into this one, to show that this is really just a
move.

> > I'll skip questions on things not clear to me about the converter
> > implementation as I understand this basically comes from the simple
> > pipeline converter implementation [*]
> > 
> > I would then focus more on two general questions:
> > 1) Do we want to make a dir for converter/post-processors instead of
> > placing them in src/libcamera
> 
> Might make sense, or could be done later - that's not my call.
> I can move them to a dedicated folder on the next patch set version if there is 
> a willing to do so.

src/libcamera/converter/ sounds good to me.

> > 2) Is the format expected in the configuration file documented
> > anywhere ?
> 
> Nope. I will do it on the next version, hopefully on an agreed configuration 
> file format.
> 
> > [*] Not true, I will ask a question anyway, see below :)
> > 
> >> Signed-off-by: Xavier Roumegue <xavier.roumegue@oss.nxp.com>
> >> ---
> >>   .../libcamera/internal/converter_v4l2_m2m.h   | 120 +++++
> >>   include/libcamera/internal/meson.build        |   1 +
> >>   src/libcamera/converter_v4l2_m2m.cpp          | 504 ++++++++++++++++++
> >>   src/libcamera/meson.build                     |   1 +
> >>   4 files changed, 626 insertions(+)
> >>   create mode 100644 include/libcamera/internal/converter_v4l2_m2m.h
> >>   create mode 100644 src/libcamera/converter_v4l2_m2m.cpp
> >>
> >> diff --git a/include/libcamera/internal/converter_v4l2_m2m.h b/include/libcamera/internal/converter_v4l2_m2m.h
> >> new file mode 100644
> >> index 00000000..3667b128
> >> --- /dev/null
> >> +++ b/include/libcamera/internal/converter_v4l2_m2m.h
> >> @@ -0,0 +1,120 @@
> >> +/* SPDX-License-Identifier: LGPL-2.1-or-later */
> >> +/*
> >> + * Copyright 2022 NXP
> >> + *
> >> + * converter_v4l2_m2m.h - V4l2 M2M Format converter interface
> >> + */
> >> +
> >> +#pragma once
> >> +
> >> +#include <functional>
> >> +#include <map>
> >> +#include <memory>
> >> +#include <string>
> >> +#include <tuple>
> >> +#include <vector>
> >> +
> >> +#include <libcamera/base/log.h>
> >> +#include <libcamera/base/signal.h>
> >> +
> >> +#include <libcamera/geometry.h>
> >> +#include <libcamera/pixel_format.h>
> >> +
> >> +#include "libcamera/internal/converter.h"
> >> +
> >> +namespace libcamera {
> >> +
> >> +class FrameBuffer;
> >> +class MediaDevice;
> >> +class Size;
> >> +class SizeRange;
> >> +struct StreamConfiguration;
> >> +class V4L2M2MDevice;
> >> +class V4L2M2MConverter;
> >> +class Converter;
> >> +
> >> +class V4L2M2MConverter : public Converter
> >> +{
> >> +protected:
> >> +	class Mapping
> >> +	{
> >> +	public:
> >> +		Mapping(const Size &input, const Size &output, const std::vector<uint32_t> &map)
> >> +			: input_(input), output_(output), map_(map) {}
> >> +		Size getInputSize() const { return input_; }
> >> +		Size getOutputSize() const { return output_; }
> >> +		std::size_t getLength() const { return map_.size(); }
> >> +		const uint32_t *getMapping() const { return map_.data(); }
> >> +
> >> +	private:
> >> +		Size input_;
> >> +		Size output_;
> >> +		std::vector<uint32_t> map_;
> >> +	};
> >> +
> >> +	class Stream : protected Loggable
> >> +	{
> >> +	public:
> >> +		Stream(V4L2M2MConverter *converter, unsigned int index);
> >> +
> >> +		bool isValid() const { return m2m_ != nullptr; }
> >> +
> >> +		int configure(const StreamConfiguration &inputCfg,
> >> +			      const StreamConfiguration &outputCfg);
> >> +		int exportBuffers(unsigned int count,
> >> +				  std::vector<std::unique_ptr<FrameBuffer>> *buffers);
> >> +
> >> +		int start();
> >> +		void stop();
> >> +
> >> +		int queueBuffers(FrameBuffer *input, FrameBuffer *output);
> >> +		std::unique_ptr<V4L2M2MDevice> m2m_;
> >> +
> >> +	protected:
> >> +		std::string logPrefix() const override;
> >> +
> >> +	private:
> >> +		void captureBufferReady(FrameBuffer *buffer);
> >> +		void outputBufferReady(FrameBuffer *buffer);
> >> +
> >> +		V4L2M2MConverter *converter_;
> >> +		unsigned int index_;
> >> +
> >> +		unsigned int inputBufferCount_;
> >> +		unsigned int outputBufferCount_;
> >> +	};
> >> +
> >> +	std::unique_ptr<V4L2M2MDevice> m2m_;
> >> +
> >> +	std::vector<Stream> streams_;
> >> +	std::vector<Mapping> mappings_;
> >> +	std::map<FrameBuffer *, unsigned int> queue_;
> >> +
> >> +public:
> >> +	V4L2M2MConverter(MediaDevice *media);
> >> +
> >> +	int loadConfiguration(const std::string &filename) override;
> >> +
> >> +	bool isValid() const { return m2m_ != nullptr; }
> >> +
> >> +	std::vector<PixelFormat> formats(PixelFormat input);
> >> +	SizeRange sizes(const Size &input);
> >> +
> >> +	std::tuple<unsigned int, unsigned int>
> >> +	strideAndFrameSize(const PixelFormat &pixelFormat, const Size &size);
> >> +
> >> +	int configure(const StreamConfiguration &inputCfg,
> >> +		      const std::vector<std::reference_wrapper<StreamConfiguration>> &outputCfg);
> >> +	int exportBuffers(unsigned int ouput, unsigned int count,
> >> +			  std::vector<std::unique_ptr<FrameBuffer>> *buffers);
> >> +
> >> +	int start();
> >> +	void stop();
> >> +
> >> +	int queueBuffers(FrameBuffer *input,
> >> +			 const std::map<unsigned int, FrameBuffer *> &outputs);
> >> +
> >> +	virtual int applyMapping([[maybe_unused]] Stream *stream, [[maybe_unused]] Mapping &mapping) { return 0; };

This should be protected.

> >> +};
> >> +
> >> +} /* namespace libcamera */
> >> diff --git a/include/libcamera/internal/meson.build b/include/libcamera/internal/meson.build
> >> index 8f50d755..132de5ef 100644
> >> --- a/include/libcamera/internal/meson.build
> >> +++ b/include/libcamera/internal/meson.build
> >> @@ -20,6 +20,7 @@ libcamera_internal_headers = files([
> >>       'control_serializer.h',
> >>       'control_validator.h',
> >>       'converter.h',
> >> +    'converter_v4l2_m2m.h',
> >>       'delayed_controls.h',
> >>       'device_enumerator.h',
> >>       'device_enumerator_sysfs.h',
> >> diff --git a/src/libcamera/converter_v4l2_m2m.cpp b/src/libcamera/converter_v4l2_m2m.cpp
> >> new file mode 100644
> >> index 00000000..942e6e6f
> >> --- /dev/null
> >> +++ b/src/libcamera/converter_v4l2_m2m.cpp
> >> @@ -0,0 +1,504 @@
> >> +/* SPDX-License-Identifier: LGPL-2.1-or-later */
> >> +/*
> >> + * Copyright (C) 2020, Laurent Pinchart
> >> + * Copyright 2022 NXP
> >> + *
> >> + * converter_v4l2_m2m.cpp - V4L2 M2M Format converter
> >> + */
> >> +
> >> +#include <algorithm>
> >> +#include <limits.h>
> >> +
> >> +#include <libcamera/base/file.h>
> >> +#include <libcamera/base/log.h>
> >> +#include <libcamera/base/signal.h>
> >> +#include <libcamera/base/utils.h>
> >> +
> >> +#include <libcamera/framebuffer.h>
> >> +#include <libcamera/geometry.h>
> >> +#include <libcamera/stream.h>
> >> +
> >> +#include "libcamera/internal/converter_v4l2_m2m.h"
> >> +#include "libcamera/internal/media_device.h"
> >> +#include "libcamera/internal/v4l2_videodevice.h"
> >> +#include "libcamera/internal/yaml_parser.h"
> >> +
> >> +namespace libcamera {
> >> +
> >> +LOG_DECLARE_CATEGORY(Converter)
> >> +
> >> +/* -----------------------------------------------------------------------------
> >> + * V4L2M2MConverter::Stream
> >> + */
> >> +
> >> +V4L2M2MConverter::Stream::Stream(V4L2M2MConverter *converter, unsigned int index)
> >> +	: converter_(converter), index_(index)
> >> +{
> >> +	m2m_ = std::make_unique<V4L2M2MDevice>(converter->deviceNode_);
> >> +
> >> +	m2m_->output()->bufferReady.connect(this, &Stream::outputBufferReady);
> >> +	m2m_->capture()->bufferReady.connect(this, &Stream::captureBufferReady);
> >> +
> >> +	int ret = m2m_->open();
> >> +	if (ret < 0)
> >> +		m2m_.reset();
> > 
> > Why are we re-creating and opening the m2m_ device everytime we create
> > a stream ? Isn't it the same created and open when the
> > V4L2M2MConverter is created ? Would it be for proper reference counting ?

That's how the V4L2 M2M API works, every open creates a separate
processing context.

> >> +}
> >> +
> >> +int V4L2M2MConverter::Stream::configure(const StreamConfiguration &inputCfg,
> >> +					const StreamConfiguration &outputCfg)
> >> +{
> >> +	V4L2PixelFormat videoFormat =
> >> +		m2m_->output()->toV4L2PixelFormat(inputCfg.pixelFormat);
> >> +
> >> +	V4L2DeviceFormat format;
> >> +	format.fourcc = videoFormat;
> >> +	format.size = inputCfg.size;
> >> +	format.planesCount = 1;
> >> +	format.planes[0].bpl = inputCfg.stride;
> >> +
> >> +	int ret = m2m_->output()->setFormat(&format);
> >> +	if (ret < 0) {
> >> +		LOG(Converter, Error)
> >> +			<< "Failed to set input format: " << strerror(-ret);
> >> +		return ret;
> >> +	}
> >> +
> >> +	if (format.fourcc != videoFormat || format.size != inputCfg.size ||
> >> +	    format.planes[0].bpl != inputCfg.stride) {
> >> +		LOG(Converter, Error)
> >> +			<< "Input format not supported (requested "
> >> +			<< inputCfg.size << "-" << videoFormat
> >> +			<< ", got " << format << ")";
> >> +		return -EINVAL;
> >> +	}
> >> +
> >> +	/* Set the pixel format and size on the output. */
> >> +	videoFormat = m2m_->capture()->toV4L2PixelFormat(outputCfg.pixelFormat);
> >> +	format = {};
> >> +	format.fourcc = videoFormat;
> >> +	format.size = outputCfg.size;
> >> +
> >> +	ret = m2m_->capture()->setFormat(&format);
> >> +	if (ret < 0) {
> >> +		LOG(Converter, Error)
> >> +			<< "Failed to set output format: " << strerror(-ret);
> >> +		return ret;
> >> +	}
> >> +
> >> +	if (format.fourcc != videoFormat || format.size != outputCfg.size) {
> >> +		LOG(Converter, Error)
> >> +			<< "Output format not supported";
> >> +		return -EINVAL;
> >> +	}
> >> +
> >> +	inputBufferCount_ = inputCfg.bufferCount;
> >> +	outputBufferCount_ = outputCfg.bufferCount;
> >> +
> >> +	for (Mapping &mapping : converter_->mappings_) {
> >> +		ControlList ctrls;
> >> +		if (mapping.getInputSize() == inputCfg.size && mapping.getOutputSize() == outputCfg.size) {
> >> +			LOG(Converter, Debug)
> >> +				<< "Got a configuration match "
> >> +				<< inputCfg.size << " --> " << outputCfg.size;
> >> +			converter_->applyMapping(this, mapping);
> >> +		}
> >> +	}

This won't work very well, as the ISP can scale to arbitrary
resolutions. I think you should instead express the mapping in the
configuration file using floating-point values, and resample it when
configuring the stream. The crop rectangle, if any, will also need to be
taken into account, but that could be done later (a \todo comment should
be added then).

> >> +
> >> +	return 0;
> >> +}
> >> +
> >> +int V4L2M2MConverter::Stream::exportBuffers(unsigned int count,
> >> +					    std::vector<std::unique_ptr<FrameBuffer>> *buffers)
> >> +{
> >> +	return m2m_->capture()->exportBuffers(count, buffers);
> >> +}
> >> +
> >> +int V4L2M2MConverter::Stream::start()
> >> +{
> >> +	int ret = m2m_->output()->importBuffers(inputBufferCount_);
> >> +	if (ret < 0)
> >> +		return ret;
> >> +
> >> +	ret = m2m_->capture()->importBuffers(outputBufferCount_);
> >> +	if (ret < 0) {
> >> +		stop();
> >> +		return ret;
> >> +	}
> >> +
> >> +	ret = m2m_->output()->streamOn();
> >> +	if (ret < 0) {
> >> +		stop();
> >> +		return ret;
> >> +	}
> >> +
> >> +	ret = m2m_->capture()->streamOn();
> >> +	if (ret < 0) {
> >> +		stop();
> >> +		return ret;
> >> +	}
> >> +
> >> +	return 0;
> >> +}
> >> +
> >> +void V4L2M2MConverter::Stream::stop()
> >> +{
> >> +	m2m_->capture()->streamOff();
> >> +	m2m_->output()->streamOff();
> >> +	m2m_->capture()->releaseBuffers();
> >> +	m2m_->output()->releaseBuffers();
> >> +}
> >> +
> >> +int V4L2M2MConverter::Stream::queueBuffers(FrameBuffer *input, FrameBuffer *output)
> >> +{
> >> +	int ret = m2m_->output()->queueBuffer(input);
> >> +	if (ret < 0)
> >> +		return ret;
> >> +
> >> +	ret = m2m_->capture()->queueBuffer(output);
> >> +	if (ret < 0)
> >> +		return ret;
> >> +
> >> +	return 0;
> >> +}
> >> +
> >> +std::string V4L2M2MConverter::Stream::logPrefix() const
> >> +{
> >> +	return "stream" + std::to_string(index_);
> >> +}
> >> +
> >> +void V4L2M2MConverter::Stream::outputBufferReady(FrameBuffer *buffer)
> >> +{
> >> +	auto it = converter_->queue_.find(buffer);
> >> +	if (it == converter_->queue_.end())
> >> +		return;
> >> +
> >> +	if (!--it->second) {
> >> +		converter_->inputBufferReady.emit(buffer);
> >> +		converter_->queue_.erase(it);
> >> +	}
> >> +}
> >> +
> >> +void V4L2M2MConverter::Stream::captureBufferReady(FrameBuffer *buffer)
> >> +{
> >> +	converter_->outputBufferReady.emit(buffer);
> >> +}
> >> +
> >> +/* -----------------------------------------------------------------------------
> >> + * V4L2M2MConverter
> >> + */
> >> +
> >> +V4L2M2MConverter::V4L2M2MConverter(MediaDevice *media)
> >> +	: Converter(media)
> >> +{
> >> +	if (deviceNode_.empty())
> >> +		return;
> >> +
> >> +	m2m_ = std::make_unique<V4L2M2MDevice>(deviceNode_);
> >> +	int ret = m2m_->open();
> >> +	if (ret < 0) {
> >> +		m2m_.reset();
> >> +		return;
> >> +	}
> >> +}
> >> +
> >> +int V4L2M2MConverter::loadConfiguration(const std::string &filename)
> >> +{
> >> +	LOG(Converter, Debug)
> >> +		<< "Parsing configuration file " << filename;
> >> +
> >> +	File file(filename);
> >> +
> >> +	if (!file.open(File::OpenModeFlag::ReadOnly)) {
> >> +		int ret = file.error();
> >> +		LOG(Converter, Error)
> >> +			<< "Failed to open configuration file "
> >> +			<< filename << ": " << strerror(-ret);
> >> +		return ret;
> >> +	}
> >> +
> >> +	std::unique_ptr<libcamera::YamlObject> data = YamlParser::parse(file);
> >> +	if (!data)
> >> +		return -EINVAL;
> >> +
> >> +	if (!data->contains("mappings")) {
> >> +		LOG(Converter, Error)
> >> +			<< "Vertex mapping key missing";
> >> +		return -EINVAL;
> >> +	}
> >> +
> >> +	const YamlObject &mappings = (*data)["mappings"];
> >> +	if (!mappings.isList() || mappings.size() == 0) {
> >> +		LOG(Converter, Error)
> >> +			<< "Invalid mappings entry";
> >> +		return -EINVAL;
> >> +	}
> >> +
> >> +	LOG(Converter, Debug)
> >> +		<< "Parsing " << mappings.size() << " mappings";
> >> +	mappings_.clear();
> >> +	mappings_.reserve(mappings.size());
> >> +
> >> +	for (std::size_t i = 0; i < mappings.size(); i++) {
> >> +		const YamlObject &mapping = mappings[i];
> >> +		if (!mapping.isDictionary()) {
> >> +			LOG(Converter, Error)
> >> +				<< "Mapping is not a dictionnary";
> >> +			return -EINVAL;
> >> +		}
> >> +
> >> +		if (!mapping.contains("input-resolution")) {
> >> +			LOG(Converter, Error)
> >> +				<< "Input resolution missing";
> >> +			return -EINVAL;
> >> +		}
> >> +
> >> +		if (!mapping.contains("output-resolution")) {
> >> +			LOG(Converter, Error)
> >> +				<< "Output resolution missing";
> >> +			return -EINVAL;
> >> +		}
> >> +
> >> +		if (!mapping.contains("mapping")) {
> >> +			LOG(Converter, Error)
> >> +				<< "Mapping table missing";
> >> +			return -EINVAL;
> >> +		}
> >> +
> >> +		const YamlObject &input_res = mapping["input-resolution"];
> >> +		if (!input_res.isList() || input_res.size() != 2) {
> >> +			LOG(Converter, Error)
> >> +				<< "Incorrect input resolution";
> >> +			return -EINVAL;
> >> +		}
> >> +
> >> +		const YamlObject &output_res = mapping["output-resolution"];
> >> +		if (!output_res.isList() || output_res.size() != 2) {
> >> +			LOG(Converter, Error)
> >> +				<< "Incorrect output resolution";
> >> +			return -EINVAL;
> >> +		}
> >> +
> >> +		const YamlObject &map = mapping["mapping"];
> >> +		if (!map.isList() || map.size() == 0) {
> >> +			LOG(Converter, Error)
> >> +				<< "Incorrect mapping entries";
> >> +			return -EINVAL;
> >> +		}
> >> +
> >> +		Size input(input_res[0].get<uint32_t>(0), input_res[1].get<uint32_t>(0));

YamlObject::get<Size>() to the rescue ?

> >> +		Size output(output_res[0].get<uint32_t>(0), output_res[1].get<uint32_t>(0));
> >> +		const auto &mapVector = map.getList<uint32_t>().value_or(utils::defopt);
> >> +
> >> +		LOG(Converter, Debug)
> >> +			<< "Input/Output mapping resolution " << input << " ---> " << output;
> >> +		mappings_.emplace_back(Mapping(input, output, mapVector));
> >> +	}
> >> +
> >> +	return mappings.size();
> >> +}
> >> +
> >> +std::vector<PixelFormat> V4L2M2MConverter::formats(PixelFormat input)
> >> +{
> >> +	if (!m2m_)
> >> +		return {};
> >> +
> >> +	/*
> >> +	 * Set the format on the input side (V4L2 output) of the converter to
> >> +	 * enumerate the conversion capabilities on its output (V4L2 capture).
> >> +	 */
> >> +	V4L2DeviceFormat v4l2Format;
> >> +	v4l2Format.fourcc = m2m_->output()->toV4L2PixelFormat(input);
> >> +	v4l2Format.size = { 1, 1 };
> >> +
> >> +	int ret = m2m_->output()->setFormat(&v4l2Format);
> >> +	if (ret < 0) {
> >> +		LOG(Converter, Error)
> >> +			<< "Failed to set format: " << strerror(-ret);
> >> +		return {};
> >> +	}
> >> +
> >> +	if (v4l2Format.fourcc != m2m_->output()->toV4L2PixelFormat(input)) {
> >> +		LOG(Converter, Debug)
> >> +			<< "Input format " << input << " not supported.";
> >> +		return {};
> >> +	}
> >> +
> >> +	std::vector<PixelFormat> pixelFormats;
> >> +
> >> +	for (const auto &format : m2m_->capture()->formats()) {
> >> +		PixelFormat pixelFormat = format.first.toPixelFormat();
> >> +		if (pixelFormat)
> >> +			pixelFormats.push_back(pixelFormat);
> >> +	}
> >> +
> >> +	return pixelFormats;
> >> +}
> >> +
> >> +SizeRange V4L2M2MConverter::sizes(const Size &input)
> >> +{
> >> +	if (!m2m_)
> >> +		return {};
> >> +
> >> +	/*
> >> +	 * Set the size on the input side (V4L2 output) of the converter to
> >> +	 * enumerate the scaling capabilities on its output (V4L2 capture).
> >> +	 */
> >> +	V4L2DeviceFormat format;
> >> +	format.fourcc = V4L2PixelFormat();
> >> +	format.size = input;
> >> +
> >> +	int ret = m2m_->output()->setFormat(&format);
> >> +	if (ret < 0) {
> >> +		LOG(Converter, Error)
> >> +			<< "Failed to set format: " << strerror(-ret);
> >> +		return {};
> >> +	}
> >> +
> >> +	SizeRange sizes;
> >> +
> >> +	format.size = { 1, 1 };
> >> +	ret = m2m_->capture()->setFormat(&format);
> >> +	if (ret < 0) {
> >> +		LOG(Converter, Error)
> >> +			<< "Failed to set format: " << strerror(-ret);
> >> +		return {};
> >> +	}
> >> +
> >> +	sizes.min = format.size;
> >> +
> >> +	format.size = { UINT_MAX, UINT_MAX };
> >> +	ret = m2m_->capture()->setFormat(&format);
> >> +	if (ret < 0) {
> >> +		LOG(Converter, Error)
> >> +			<< "Failed to set format: " << strerror(-ret);
> >> +		return {};
> >> +	}
> >> +
> >> +	sizes.max = format.size;
> >> +
> >> +	return sizes;
> >> +}
> >> +
> >> +std::tuple<unsigned int, unsigned int>
> >> +V4L2M2MConverter::strideAndFrameSize(const PixelFormat &pixelFormat,
> >> +				     const Size &size)
> >> +{
> >> +	V4L2DeviceFormat format;
> >> +	format.fourcc = m2m_->capture()->toV4L2PixelFormat(pixelFormat);
> >> +	format.size = size;
> >> +
> >> +	int ret = m2m_->capture()->tryFormat(&format);
> >> +	if (ret < 0)
> >> +		return std::make_tuple(0, 0);
> >> +
> >> +	return std::make_tuple(format.planes[0].bpl, format.planes[0].size);
> >> +}
> >> +
> >> +int V4L2M2MConverter::configure(const StreamConfiguration &inputCfg,
> >> +				const std::vector<std::reference_wrapper<StreamConfiguration>> &outputCfgs)
> >> +{
> >> +	int ret = 0;
> >> +
> >> +	streams_.clear();
> >> +	streams_.reserve(outputCfgs.size());
> >> +
> >> +	for (unsigned int i = 0; i < outputCfgs.size(); ++i) {
> >> +		Stream &stream = streams_.emplace_back(this, i);
> >> +
> >> +		if (!stream.isValid()) {
> >> +			LOG(Converter, Error)
> >> +				<< "Failed to create stream " << i;
> >> +			ret = -EINVAL;
> >> +			break;
> >> +		}
> >> +
> >> +		ret = stream.configure(inputCfg, outputCfgs[i]);
> >> +		if (ret < 0)
> >> +			break;
> >> +	}
> >> +
> >> +	if (ret < 0) {
> >> +		streams_.clear();
> >> +		return ret;
> >> +	}
> >> +
> >> +	return 0;
> >> +}
> >> +
> >> +int V4L2M2MConverter::exportBuffers(unsigned int output, unsigned int count,
> >> +				    std::vector<std::unique_ptr<FrameBuffer>> *buffers)
> >> +{
> >> +	if (output >= streams_.size())
> >> +		return -EINVAL;
> >> +
> >> +	return streams_[output].exportBuffers(count, buffers);
> >> +}
> >> +
> >> +int V4L2M2MConverter::start()
> >> +{
> >> +	int ret;
> >> +
> >> +	for (Stream &stream : streams_) {
> >> +		ret = stream.start();
> >> +		if (ret < 0) {
> >> +			stop();
> >> +			return ret;
> >> +		}
> >> +	}
> >> +
> >> +	return 0;
> >> +}
> >> +
> >> +void V4L2M2MConverter::stop()
> >> +{
> >> +	for (Stream &stream : utils::reverse(streams_))
> >> +		stream.stop();
> >> +}
> >> +
> >> +int V4L2M2MConverter::queueBuffers(FrameBuffer *input,
> >> +				   const std::map<unsigned int, FrameBuffer *> &outputs)
> >> +{
> >> +	unsigned int mask = 0;
> >> +	int ret;
> >> +
> >> +	/*
> >> +	 * Validate the outputs as a sanity check: at least one output is
> >> +	 * required, all outputs must reference a valid stream and no two
> >> +	 * outputs can reference the same stream.
> >> +	 */
> >> +	if (outputs.empty())
> >> +		return -EINVAL;
> >> +
> >> +	for (auto [index, buffer] : outputs) {
> >> +		if (!buffer)
> >> +			return -EINVAL;
> >> +		if (index >= streams_.size())
> >> +			return -EINVAL;
> >> +		if (mask & (1 << index))
> >> +			return -EINVAL;
> >> +
> >> +		mask |= 1 << index;
> >> +	}
> >> +
> >> +	/* Queue the input and output buffers to all the streams. */
> >> +	for (auto [index, buffer] : outputs) {
> >> +		ret = streams_[index].queueBuffers(input, buffer);
> >> +		if (ret < 0)
> >> +			return ret;
> >> +	}
> >> +
> >> +	/*
> >> +	 * Add the input buffer to the queue, with the number of streams as a
> >> +	 * reference count. Completion of the input buffer will be signalled by
> >> +	 * the stream that releases the last reference.
> >> +	 */
> >> +	queue_.emplace(std::piecewise_construct,
> >> +		       std::forward_as_tuple(input),
> >> +		       std::forward_as_tuple(outputs.size()));
> >> +
> >> +	return 0;
> >> +}
> >> +
> >> +REGISTER_CONVERTER("v4l2_m2m", V4L2M2MConverter, "pxp")
> >> +
> >> +} /* namespace libcamera */
> >> diff --git a/src/libcamera/meson.build b/src/libcamera/meson.build
> >> index a261d4b4..b12c8401 100644
> >> --- a/src/libcamera/meson.build
> >> +++ b/src/libcamera/meson.build
> >> @@ -14,6 +14,7 @@ libcamera_sources = files([
> >>       'control_serializer.cpp',
> >>       'control_validator.cpp',
> >>       'converter.cpp',
> >> +    'converter_v4l2_m2m.cpp',
> >>       'delayed_controls.cpp',
> >>       'device_enumerator.cpp',
> >>       'device_enumerator_sysfs.cpp',