[v3,16/29] libcamera: converter: Add dw100 converter module

25195 diff mbox series

The DW100 Dewarp engine is present on i.MX8MP SoC and possibly others.
This patch provides a dedicated converter module that allows easy
integration of such a dewarper into a pipeline handler.

In this patch only the ScalerCrop control is implemented. Support for
additional functionality will be added in later patches.

Signed-off-by: Stefan Klug <stefan.klug@ideasonboard.com>

---

Chnages in v3:
- Major rewrite to implement the libcamera controls in the converter class
  instead of leaving that to the pipeline handler.
- Moved V4L2 requests handling into the class instead of leaving that to
  the pipeline handler

Changes in v2:
- Drop validateSize() as it is not used anywhere

Changes in v0.9
- Use shared_ptr in constructor
---
 .../internal/converter/converter_dw100.h      |  85 ++++
 .../libcamera/internal/converter/meson.build  |   1 +
 src/libcamera/converter/converter_dw100.cpp   | 413 ++++++++++++++++++
 src/libcamera/converter/meson.build           |   1 +
 4 files changed, 500 insertions(+)
 create mode 100644 include/libcamera/internal/converter/converter_dw100.h
 create mode 100644 src/libcamera/converter/converter_dw100.cpp

Quoting Stefan Klug (2025-11-25 16:28:28)
> The DW100 Dewarp engine is present on i.MX8MP SoC and possibly others.
> This patch provides a dedicated converter module that allows easy
> integration of such a dewarper into a pipeline handler.
> 
> In this patch only the ScalerCrop control is implemented. Support for
> additional functionality will be added in later patches.
> 
> Signed-off-by: Stefan Klug <stefan.klug@ideasonboard.com>
> 
> ---
> 
> Chnages in v3:
> - Major rewrite to implement the libcamera controls in the converter class
>   instead of leaving that to the pipeline handler.


 \o/  \o/  \o/  \o/

thank you :-)

> - Moved V4L2 requests handling into the class instead of leaving that to
>   the pipeline handler
> 

And this!

> Changes in v2:
> - Drop validateSize() as it is not used anywhere
> 
> Changes in v0.9
> - Use shared_ptr in constructor
> ---
>  .../internal/converter/converter_dw100.h      |  85 ++++
>  .../libcamera/internal/converter/meson.build  |   1 +
>  src/libcamera/converter/converter_dw100.cpp   | 413 ++++++++++++++++++
>  src/libcamera/converter/meson.build           |   1 +
>  4 files changed, 500 insertions(+)
>  create mode 100644 include/libcamera/internal/converter/converter_dw100.h
>  create mode 100644 src/libcamera/converter/converter_dw100.cpp
> 
> diff --git a/include/libcamera/internal/converter/converter_dw100.h b/include/libcamera/internal/converter/converter_dw100.h
> new file mode 100644
> index 000000000000..8dd21a6228f1
> --- /dev/null
> +++ b/include/libcamera/internal/converter/converter_dw100.h
> @@ -0,0 +1,85 @@
> +/* SPDX-License-Identifier: GPL-2.0-or-later */
> +/*
> + * Copyright (C) 2024, Ideas On Board Oy
> + *
> + * i.MX8MP Dewarp Engine integration
> + */
> +
> +#pragma once
> +
> +#include <memory>
> +#include <queue>
> +
> +#include <libcamera/control_ids.h>
> +#include <libcamera/controls.h>
> +#include <libcamera/framebuffer.h>
> +
> +#include "libcamera/internal/converter/converter_dw100_vertexmap.h"
> +#include "libcamera/internal/converter/converter_v4l2_m2m.h"
> +#include "libcamera/internal/device_enumerator.h"
> +
> +namespace libcamera {
> +
> +class MediaDevice;
> +class Rectangle;
> +class Stream;
> +
> +class ConverterDW100Module
> +{
> +public:
> +       virtual ~ConverterDW100Module() = default;
> +
> +       static std::unique_ptr<ConverterDW100Module> createModule(DeviceEnumerator *enumerator);
> +
> +       int configure(const StreamConfiguration &inputCfg,
> +                     const std::vector<std::reference_wrapper<StreamConfiguration>>
> +                             &outputCfg);
> +       bool isConfigured(const Stream *stream) const;
> +
> +       Size adjustInputSize(const PixelFormat &pixFmt, const Size &size,
> +                            Converter::Alignment align = Converter::Alignment::Down);
> +       Size adjustOutputSize(const PixelFormat &pixFmt, const Size &size,
> +                             Converter::Alignment align = Converter::Alignment::Down);
> +
> +       int exportBuffers(const Stream *stream, unsigned int count,
> +                         std::vector<std::unique_ptr<FrameBuffer>> *buffers);
> +       int validateOutput(StreamConfiguration *cfg, bool *adjusted,
> +                          Converter::Alignment align = Converter::Alignment::Down);
> +       int queueBuffers(FrameBuffer *input,
> +                        const std::map<const Stream *, FrameBuffer *> &outputs);
> +
> +       int start();
> +       void stop();
> +
> +       void updateControlInfos(const Stream *stream, ControlInfoMap::Map &infos);
> +       void setControls(const Stream *stream, const ControlList &controls);
> +       void populateMetadata(const Stream *stream, ControlList &meta);
> +
> +       void setSensorCrop(const Rectangle &rect);
> +       void setTransform(const Stream *stream, const Transform &transform);
> +
> +       Signal<FrameBuffer *> inputBufferReady;
> +       Signal<FrameBuffer *> outputBufferReady;
> +
> +private:
> +       ConverterDW100Module(std::shared_ptr<MediaDevice> media);
> +
> +       int applyControls(const Stream *stream, const V4L2Request *request);
> +       void reinitRequest(V4L2Request *request);
> +
> +       struct VertexMapInfo {
> +               Dw100VertexMap map;
> +               bool update;
> +       };
> +
> +       std::map<const Stream *, VertexMapInfo> vertexMaps_;
> +       unsigned int inputBufferCount_;
> +       V4L2M2MConverter converter_;
> +       Rectangle sensorCrop_;
> +       bool running_;
> +
> +       std::vector<std::unique_ptr<V4L2Request>> requests_;
> +       std::queue<V4L2Request *> availableRequests_;
> +};

Hard to know how 'modular' this is until we add more modules - but I
very much appreciate pulling it out to be something we can start
working towards.

And that this will be easier to integrate in say the ISI pipeline
handler if desired later.


> +
> +} /* namespace libcamera */
> diff --git a/include/libcamera/internal/converter/meson.build b/include/libcamera/internal/converter/meson.build
> index 9d586293f63a..128c644cb73f 100644
> --- a/include/libcamera/internal/converter/meson.build
> +++ b/include/libcamera/internal/converter/meson.build
> @@ -1,6 +1,7 @@
>  # SPDX-License-Identifier: CC0-1.0
>  
>  libcamera_internal_headers += files([
> +    'converter_dw100.h',
>      'converter_dw100_vertexmap.h',
>      'converter_v4l2_m2m.h',
>  ])
> diff --git a/src/libcamera/converter/converter_dw100.cpp b/src/libcamera/converter/converter_dw100.cpp
> new file mode 100644
> index 000000000000..cba7cc9f709b
> --- /dev/null
> +++ b/src/libcamera/converter/converter_dw100.cpp
> @@ -0,0 +1,413 @@
> +/* SPDX-License-Identifier: GPL-2.0-or-later */
> +/*
> + * Copyright (C) 2024, Ideas On Board Oy
> + *
> + * i.MX8MP Dewarp Engine integration
> + */
> +
> +#include "libcamera/internal/converter/converter_dw100.h"
> +
> +#include <linux/dw100.h>
> +
> +#include <libcamera/base/log.h>
> +
> +#include <libcamera/geometry.h>
> +#include <libcamera/stream.h>
> +
> +#include "libcamera/internal/converter.h"
> +#include "libcamera/internal/converter/converter_v4l2_m2m.h"
> +#include "libcamera/internal/media_device.h"
> +#include "libcamera/internal/v4l2_videodevice.h"
> +
> +namespace libcamera {
> +
> +LOG_DECLARE_CATEGORY(Converter)
> +
> +/**
> + * \class libcamera::ConverterDW100Module
> + * \brief A converter module for the dw100 dewarper
> + *
> + * This class implements a converter module with direct support for libcamera
> + * controls. Functionality wise it closely resembles the libcamera::Converter
> + * interface. The main difference is that V4L2 requests are handled internally
> + * and it has direct support for libcamera controls.
> + */
> +
> +ConverterDW100Module::ConverterDW100Module(std::shared_ptr<MediaDevice> media)
> +       : converter_(media), running_(false)
> +{
> +       converter_.outputBufferReady.connect(&this->outputBufferReady, &Signal<FrameBuffer *>::emit);
> +       converter_.inputBufferReady.connect(&this->inputBufferReady, &Signal<FrameBuffer *>::emit);
> +}
> +
> +/**
> + * \brief Create a ConverterDW100Module
> + * \param[in] enumerator The enumerator
> + *
> + * Static factory function that searches for the dw100 device using the provided
> + * \a enumerator. If found, a ConverterDW100Module is instantiated and returned.
> + *
> + * \return A ConverterDW100Module or null if no converter was found
> + */
> +std::unique_ptr<ConverterDW100Module>
> +ConverterDW100Module::createModule(DeviceEnumerator *enumerator)
> +{
> +       DeviceMatch dwp("dw100");
> +       dwp.add("dw100-source");
> +       dwp.add("dw100-sink");
> +
> +       std::shared_ptr<MediaDevice> dwpMediaDevice = enumerator->search(dwp);
> +       if (!dwpMediaDevice)
> +               return {};
> +
> +       std::unique_ptr<ConverterDW100Module> dwpModule{ new ConverterDW100Module(dwpMediaDevice) };
> +       if (dwpModule->converter_.isValid())
> +               return dwpModule;
> +
> +       LOG(Converter, Warning)
> +               << "Found DW100 dewarper " << dwpMediaDevice->deviceNode()
> +               << " but invalid";
> +       return {};
> +}

Nice.


> +
> +/**
> + * \copydoc libcamera::V4L2M2MConverter::configure
> + */
> +int ConverterDW100Module::configure(const StreamConfiguration &inputCfg,
> +                                   const std::vector<std::reference_wrapper<StreamConfiguration>>
> +                                           &outputCfgs)
> +{
> +       int ret;
> +
> +       vertexMaps_.clear();
> +       ret = converter_.configure(inputCfg, outputCfgs);
> +       if (ret)
> +               return ret;
> +
> +       inputBufferCount_ = inputCfg.bufferCount;
> +
> +       for (auto &ref : outputCfgs) {
> +               const auto &outputCfg = ref.get();
> +               auto &info = vertexMaps_[outputCfg.stream()];
> +               auto &vertexMap = info.map;
> +               vertexMap.setInputSize(inputCfg.size);
> +               vertexMap.setOutputSize(outputCfg.size);
> +               vertexMap.setSensorCrop(sensorCrop_);
> +               info.update = true;
> +       }

Looking forward to seeing multi-stream outputs extended for the i.MX8MP
... but that's definitely later.

> +
> +       return 0;
> +}
> +
> +/**
> + * \copydoc libcamera::V4L2M2MConverter::isConfigured
> + */
> +bool ConverterDW100Module::isConfigured(const Stream *stream) const
> +{
> +       return vertexMaps_.find(stream) != vertexMaps_.end();
> +}
> +
> +/**
> + * \copydoc libcamera::V4L2M2MConverter::adjustInputSize
> + */
> +Size ConverterDW100Module::adjustInputSize(const PixelFormat &pixFmt,
> +                                          const Size &size,
> +                                          Converter::Alignment align)
> +{
> +       return converter_.adjustInputSize(pixFmt, size, align);
> +}
> +
> +/**
> + * \copydoc libcamera::V4L2M2MConverter::adjustOutputSize
> + */
> +Size ConverterDW100Module::adjustOutputSize(const PixelFormat &pixFmt,
> +                                           const Size &size,
> +                                           Converter::Alignment align)
> +{
> +       return converter_.adjustOutputSize(pixFmt, size, align);
> +}
> +
> +/**
> + * \copydoc libcamera::V4L2M2MConverter::exportBuffers
> + */
> +int ConverterDW100Module::exportBuffers(const Stream *stream, unsigned int count,
> +                                       std::vector<std::unique_ptr<FrameBuffer>> *buffers)
> +{
> +       return converter_.exportBuffers(stream, count, buffers);
> +}
> +
> +/**
> + * \copydoc libcamera::V4L2M2MConverter::validateOutput
> + */
> +int ConverterDW100Module::validateOutput(StreamConfiguration *cfg,
> +                                        bool *adjusted,
> +                                        Converter::Alignment align)
> +{
> +       return converter_.validateOutput(cfg, adjusted, align);
> +}
> +
> +/**
> + * \brief Queue buffers to converter device
> + * \param[in] input The frame buffer to apply the conversion
> + * \param[out] outputs The container holding the output stream pointers and
> + * their respective frame buffer outputs.
> + *
> + * This function queues the \a input frame buffer and the output frame buffers
> + * contained in \a outputs to the device for processing.
> + *
> + * Controls are automatically applied to the device before queuing buffers. V4L2
> + * requests are used to atomically apply the controls if the kernel supports it.
> + *
> + * \return 0 on success or a negative error code otherwise
> + */
> +int ConverterDW100Module::queueBuffers(FrameBuffer *input,
> +                                      const std::map<const Stream *, FrameBuffer *> &outputs)
> +{
> +       int ret;
> +
> +       V4L2Request *request = nullptr;
> +       if (!requests_.empty()) {
> +               /* If we have requests support, there must be one available */
> +               ASSERT(!availableRequests_.empty());
> +               request = availableRequests_.front();
> +               availableRequests_.pop();
> +       }
> +
> +       for (auto &[stream, buffer] : outputs) {
> +               ret = applyControls(stream, request);
> +               if (ret) {
> +                       reinitRequest(request);
> +                       return ret;
> +               }
> +       }
> +
> +       ret = converter_.queueBuffers(input, outputs, request);
> +       if (ret) {
> +               reinitRequest(request);
> +               return ret;
> +       }
> +
> +       if (!request)
> +               return 0;
> +
> +       ret = request->queue();
> +       if (ret < 0) {
> +               LOG(Converter, Error) << "Failed to queue dewarp request: -"
> +                                     << strerror(-ret);
> +               /* Push it back into the queue. */
> +               reinitRequest(request);
> +       }
> +
> +       return ret;
> +}
> +
> +/**
> + * \copydoc libcamera::V4L2M2MConverter::start
> + */
> +int ConverterDW100Module::start()
> +{
> +       int ret;
> +
> +       if (converter_.supportsRequests()) {
> +               ret = converter_.allocateRequests(inputBufferCount_,
> +                                                 &requests_);
> +               if (ret < 0) {
> +                       LOG(Converter, Error) << "Failed to allocate requests.";
> +                       return ret;
> +               }
> +       }
> +
> +       for (std::unique_ptr<V4L2Request> &request : requests_) {
> +               request->requestDone.connect(this, &ConverterDW100Module::reinitRequest);
> +               availableRequests_.push(request.get());
> +       }
> +
> +       /*
> +        * Apply controls on all streams, to support older kernels without
> +        * request and dynamic vertex map support.
> +        */
> +       for (auto &[stream, info] : vertexMaps_)
> +               applyControls(stream, nullptr);

What controls are we applying here? Should we be passing controls
through ::start(ControlList ...) or such?


> +
> +       ret = converter_.start();
> +       if (!ret) {
> +               running_ = true;
> +               return 0;
> +       }
> +
> +       availableRequests_ = {};
> +       requests_.clear();
> +       return ret;
> +}
> +
> +/**
> + * \copydoc libcamera::V4L2M2MConverter::stop
> + */
> +void ConverterDW100Module::stop()
> +{
> +       running_ = false;
> +       converter_.stop();
> +       availableRequests_ = {};
> +       requests_.clear();
> +}
> +
> +/**
> + * \brief Update the controls
> + * \param[in] stream The stream
> + * \param[inout] controls The controls info map to update
> + *
> + * Updated the \a controls map with all the controls and limits provided by this
> + * class.
> + */
> +void ConverterDW100Module::updateControlInfos(const Stream *stream, ControlInfoMap::Map &controls)
> +{
> +       ControlValue scalerCropDefault = sensorCrop_;
> +
> +       if (isConfigured(stream)) {
> +               auto &info = vertexMaps_[stream];
> +               info.map.applyLimits();
> +               scalerCropDefault = info.map.effectiveScalerCrop();
> +       }
> +
> +       controls[&controls::ScalerCrop] = ControlInfo(Rectangle(sensorCrop_.x, sensorCrop_.y, 1, 1),
> +                                                     sensorCrop_, sensorCrop_);
> +
> +       if (!converter_.supportsRequests())
> +               LOG(Converter, Warning)
> +                       << "dw100 kernel driver has no requests support."
> +                          " Dynamic configuration is not possible.";
> +}
> +
> +/**
> + * \brief Set libcamera controls
> + * \param[in] stream The stream to update
> + * \param[in] controls The controls
> + *
> + * Looks up all supported controls in \a controls and sets them on stream \a
> + * stream. The controls will be applied to the device on the next call to
> + * queueBuffers().
> + */
> +void ConverterDW100Module::setControls(const Stream *stream, const ControlList &controls)
> +{
> +       if (!isConfigured(stream))
> +               return;
> +
> +       auto &info = vertexMaps_[stream];
> +       auto &vertexMap = info.map;
> +
> +       const auto &crop = controls.get(controls::ScalerCrop);
> +       if (crop) {
> +               vertexMap.setScalerCrop(*crop);
> +               info.update = true;
> +       }
> +
> +       if (info.update && running_ && !converter_.supportsRequests())
> +               LOG(Converter, Error)
> +                       << "Dynamically setting dw100 specific controls requires"
> +                          " a dw100 kernel driver with requests support";
> +}
> +
> +/**
> + * \brief Retrieve updated metadata
> + * \param[in] stream The stream
> + * \param[in] meta The metadata list
> + *
> + * This function retrieves the metadata for the provided \a stream and writes it
> + * to \a list. It shall be called after queueBuffers().
> + */
> +void ConverterDW100Module::populateMetadata(const Stream *stream, ControlList &meta)
> +{
> +       if (!isConfigured(stream))
> +               return;
> +
> +       auto &vertexMap = vertexMaps_[stream].map;
> +
> +       meta.set(controls::ScalerCrop, vertexMap.effectiveScalerCrop());
> +}
> +
> +/**
> + * \var ConverterDW100Module::inputBufferReady
> + * \brief A signal emitted when the input frame buffer completes
> + */
> +
> +/**
> + * \var ConverterDW100Module::outputBufferReady
> + * \brief A signal emitted on each frame buffer completion of the output queue
> + */
> +
> +/**
> + * \brief Set sensor crop rectangle
> + * \param[in] rect The crop rectangle
> + *
> + * Set the sensor crop rectangle to \a rect. This rectangle describes the area
> + * covered by the input buffers in sensor coordinates. It is used internally
> + * to handle the ScalerCrop control and related metadata.
> + */
> +void ConverterDW100Module::setSensorCrop(const Rectangle &rect)
> +{
> +       sensorCrop_ = rect;
> +       for (auto &[stream, vertexMap] : vertexMaps_) {
> +               vertexMap.map.setSensorCrop(rect);
> +               vertexMap.update = true;
> +       }
> +}
> +
> +/**
> + * \brief Set transform
> + * \param[in] stream The stream
> + * \param[in] transform The transform
> + *
> + * Set the transform that shall be applied by the dewarper on the given stream.
> + * As orientation is a property of libcamera::CameraConfiguration, the transform
> + * needs to be set at configure time.
> + */
> +void ConverterDW100Module::setTransform(const Stream *stream, const Transform &transform)
> +{
> +       if (!isConfigured(stream))
> +               return;
> +
> +       vertexMaps_[stream].map.setTransform(transform);
> +}
> +
> +void ConverterDW100Module::reinitRequest(V4L2Request *request)
> +{
> +       if (!request)
> +               return;
> +
> +       request->reinit();
> +       availableRequests_.push(request);
> +}
> +
> +/**
> + * \brief Apply the vertex map for a given stream
> + * \param[in] stream The stream to update
> + * \param[in] request An optional request
> + *
> + * This function updates the vertex map on the stream \a stream. If \a request
> + * is provided, the updated happens in that request.

s/updated/update/

Reviewed-by: Kieran Bingham <kieran.bingham@ideasonboard.com>

> + *
> + * \return 0 on success or a negative error code otherwise
> + */
> +int ConverterDW100Module::applyControls(const Stream *stream, const V4L2Request *request)
> +{
> +       if (!isConfigured(stream))
> +               return -EINVAL;
> +
> +       auto &info = vertexMaps_[stream];
> +       if (!info.update)
> +               return 0;
> +
> +       std::vector<uint32_t> map = info.map.getVertexMap();
> +       auto value = Span<const int32_t>(reinterpret_cast<const int32_t *>(&map[0]), map.size());
> +
> +       ControlList ctrls;
> +       ctrls.set(V4L2_CID_DW100_DEWARPING_16x16_VERTEX_MAP, value);
> +
> +       int ret = converter_.applyControls(stream, ctrls, request);
> +       if (!ret)
> +               info.update = false;
> +
> +       return ret;
> +}
> +
> +} /* namespace libcamera */
> diff --git a/src/libcamera/converter/meson.build b/src/libcamera/converter/meson.build
> index 558d63a1bdd4..48f27cad5997 100644
> --- a/src/libcamera/converter/meson.build
> +++ b/src/libcamera/converter/meson.build
> @@ -1,6 +1,7 @@
>  # SPDX-License-Identifier: CC0-1.0
>  
>  libcamera_internal_sources += files([
> +        'converter_dw100.cpp',
>          'converter_dw100_vertexmap.cpp',
>          'converter_v4l2_m2m.cpp'
>  ])
> -- 
> 2.51.0
>