[03/11] libcamera: sensor: Add CameraSensorMemory class

25468 diff mbox series

Representation of a "camera" that actually takes its input from
a memory buffer.

libcamera is, unsurprisingly, very dependent on having a camera
connected to the system. But sometimes we may want to process raw
camera images from elsewhere, and we may not have a camera connected
to the system at all.

In such cases, the path of a "memory buffer" through the code is eased
considerably by introducing the CameraSensorMemory class, which allows
the memory buffer to behave, to an extent at least, like a real
camera.

Signed-off-by: David Plowman <david.plowman@raspberrypi.com>
---
 include/libcamera/internal/camera_sensor.h    |   2 +
 .../libcamera/internal/camera_sensor_memory.h | 110 ++++++++
 include/libcamera/internal/meson.build        |   1 +
 src/libcamera/sensor/camera_sensor_memory.cpp | 241 ++++++++++++++++++
 src/libcamera/sensor/meson.build              |   1 +
 5 files changed, 355 insertions(+)
 create mode 100644 include/libcamera/internal/camera_sensor_memory.h
 create mode 100644 src/libcamera/sensor/camera_sensor_memory.cpp

Hi David,

Thank you for the patch!

Quoting David Plowman (2025-12-10 16:15:18)
> Representation of a "camera" that actually takes its input from
> a memory buffer.
> 
> libcamera is, unsurprisingly, very dependent on having a camera
> connected to the system. But sometimes we may want to process raw
> camera images from elsewhere, and we may not have a camera connected
> to the system at all.
> 
> In such cases, the path of a "memory buffer" through the code is eased
> considerably by introducing the CameraSensorMemory class, which allows
> the memory buffer to behave, to an extent at least, like a real
> camera.
> 
> Signed-off-by: David Plowman <david.plowman@raspberrypi.com>
> ---
>  include/libcamera/internal/camera_sensor.h    |   2 +
>  .../libcamera/internal/camera_sensor_memory.h | 110 ++++++++
>  include/libcamera/internal/meson.build        |   1 +
>  src/libcamera/sensor/camera_sensor_memory.cpp | 241 ++++++++++++++++++
>  src/libcamera/sensor/meson.build              |   1 +
>  5 files changed, 355 insertions(+)
>  create mode 100644 include/libcamera/internal/camera_sensor_memory.h
>  create mode 100644 src/libcamera/sensor/camera_sensor_memory.cpp
> 
> diff --git a/include/libcamera/internal/camera_sensor.h b/include/libcamera/internal/camera_sensor.h
> index e6b72d22..5580d6ec 100644
> --- a/include/libcamera/internal/camera_sensor.h
> +++ b/include/libcamera/internal/camera_sensor.h
> @@ -49,6 +49,8 @@ public:
>  
>         virtual CameraLens *focusLens() = 0;
>  
> +       virtual bool isMemory() const { return false; }
> +
>         virtual const std::vector<unsigned int> &mbusCodes() const = 0;
>         virtual std::vector<Size> sizes(unsigned int mbusCode) const = 0;
>         virtual Size resolution() const = 0;
> diff --git a/include/libcamera/internal/camera_sensor_memory.h b/include/libcamera/internal/camera_sensor_memory.h
> new file mode 100644
> index 00000000..944d4c96
> --- /dev/null
> +++ b/include/libcamera/internal/camera_sensor_memory.h
> @@ -0,0 +1,110 @@
> +/* SPDX-License-Identifier: LGPL-2.1-or-later */
> +/*
> + * Copyright (C) 2025, Raspberry Pi plc
> + *
> + * camera_sensor_memory.h - A fake camera sensor for reading raw data from memory
> + */
> +
> +#pragma once
> +
> +#include <optional>
> +#include <string>
> +#include <vector>
> +
> +#include <libcamera/camera.h>
> +
> +#include "libcamera/internal/camera_sensor.h"
> +
> +namespace libcamera {
> +
> +class BayerFormat;
> +class Camera;
> +class CameraLens;
> +class MediaEntity;
> +class SensorConfiguration;
> +
> +struct CameraSensorProperties;
> +
> +enum class Orientation;
> +
> +LOG_DECLARE_CATEGORY(CameraSensor)
> +
> +class CameraSensorMemory : public CameraSensor, protected Loggable
> +{
> +public:
> +       CameraSensorMemory(const StreamConfiguration &rawInput, unsigned int mbusCode);
> +       ~CameraSensorMemory();
> +
> +       static std::variant<std::unique_ptr<CameraSensor>, int>
> +       match(MediaEntity *entity);
> +
> +       const std::string &model() const override { return model_; }
> +       const std::string &id() const override { return id_; }
> +
> +       const MediaEntity *entity() const override { return nullptr; }
> +       V4L2Subdevice *device() override { return nullptr; }
> +
> +       CameraLens *focusLens() override { return nullptr; }
> +
> +       virtual bool isMemory() const override { return true; }
> +
> +       const std::vector<unsigned int> &mbusCodes() const override;
> +       std::vector<Size> sizes(unsigned int mbusCode) const override;
> +       Size resolution() const override;
> +
> +       V4L2SubdeviceFormat getFormat(Span<const unsigned int> mbusCodes,
> +                                     const Size &size,
> +                                     const Size maxSize) const override;
> +       int setFormat(V4L2SubdeviceFormat *format,
> +                     Transform transform = Transform::Identity) override;
> +       int tryFormat(V4L2SubdeviceFormat *format) const override;
> +
> +       int applyConfiguration(const SensorConfiguration &config,
> +                              Transform transform = Transform::Identity,
> +                              V4L2SubdeviceFormat *sensorFormat = nullptr) override;
> +
> +       V4L2Subdevice::Stream imageStream() const override;
> +       std::optional<V4L2Subdevice::Stream> embeddedDataStream() const override;
> +       V4L2SubdeviceFormat embeddedDataFormat() const override;
> +       int setEmbeddedDataEnabled(bool enable) override;
> +
> +       const ControlList &properties() const override;
> +       int sensorInfo(IPACameraSensorInfo *info) const override;
> +       Transform computeTransform(Orientation *orientation) const override;
> +       BayerFormat::Order bayerOrder(Transform t) const override;
> +       Orientation mountingOrientation() const override;
> +
> +       const ControlInfoMap &controls() const override;
> +       ControlList getControls(Span<const uint32_t> ids) override;
> +       int setControls(ControlList *ctrls) override;
> +
> +       const std::vector<controls::draft::TestPatternModeEnum> &
> +       testPatternModes() const override { return testPatternModes_; }
> +       int setTestPatternMode(controls::draft::TestPatternModeEnum mode) override;
> +       const CameraSensorProperties::SensorDelays &sensorDelays() override;
> +
> +protected:
> +       std::string logPrefix() const override;
> +
> +private:
> +       LIBCAMERA_DISABLE_COPY(CameraSensorMemory)
> +
> +       StreamConfiguration rawInput_;
> +
> +       std::string model_;
> +       std::string id_;
> +
> +       BayerFormat bayerFormat_;
> +       std::vector<unsigned int> mbusCodes_;
> +
> +       V4L2SubdeviceFormat v4l2SubdeviceFormat_;
> +
> +       ControlInfoMap propertiesInfoMap_;
> +       ControlInfoMap controlsInfoMap_;
> +       ControlList properties_;
> +       ControlList controls_;
> +
> +       std::vector<controls::draft::TestPatternModeEnum> testPatternModes_;
> +};
> +
> +} /* namespace libcamera */
> diff --git a/include/libcamera/internal/meson.build b/include/libcamera/internal/meson.build
> index e9540a2f..9994baae 100644
> --- a/include/libcamera/internal/meson.build
> +++ b/include/libcamera/internal/meson.build
> @@ -10,6 +10,7 @@ libcamera_internal_headers = files([
>      'camera_lens.h',
>      'camera_manager.h',
>      'camera_sensor.h',
> +    'camera_sensor_memory.h',
>      'camera_sensor_properties.h',
>      'clock_recovery.h',
>      'control_serializer.h',
> diff --git a/src/libcamera/sensor/camera_sensor_memory.cpp b/src/libcamera/sensor/camera_sensor_memory.cpp
> new file mode 100644
> index 00000000..8e344016
> --- /dev/null
> +++ b/src/libcamera/sensor/camera_sensor_memory.cpp
> @@ -0,0 +1,241 @@
> +/* SPDX-License-Identifier: LGPL-2.1-or-later */
> +/*
> + * Copyright (C) 2025, Raspberry Pi plc
> + *
> + * camera_sensor_memory.cpp - A fake camera sensor for reading raw data from memory
> + */
> +
> +#include "libcamera/internal/camera_sensor_memory.h"
> +
> +#include <algorithm>
> +#include <map>
> +#include <sstream>
> +
> +#include <libcamera/base/log.h>
> +#include <libcamera/base/utils.h>
> +
> +#include <libcamera/control_ids.h>
> +#include <libcamera/controls.h>
> +#include <libcamera/geometry.h>
> +#include <libcamera/orientation.h>
> +#include <libcamera/property_ids.h>
> +#include <libcamera/transform.h>
> +
> +#include <libcamera/ipa/core_ipa_interface.h>
> +
> +#include "libcamera/internal/bayer_format.h"
> +#include "libcamera/internal/formats.h"
> +#include "libcamera/internal/v4l2_subdevice.h"
> +
> +namespace libcamera {
> +
> +LOG_DECLARE_CATEGORY(CameraSensor)
> +
> +static bool v4l2SubdeviceFormatEqual(const V4L2SubdeviceFormat &lhs, const V4L2SubdeviceFormat &rhs)
> +{
> +       return lhs.code == rhs.code && lhs.size == rhs.size && lhs.colorSpace == rhs.colorSpace;
> +}
> +
> +CameraSensorMemory::CameraSensorMemory(const StreamConfiguration &rawInput, unsigned int mbusCode)
> +       : rawInput_(rawInput), properties_(propertiesInfoMap_), controls_(controlsInfoMap_)
> +{
> +       model_ = "memory";
> +
> +       std::ostringstream oss;
> +       oss << &rawInput;
> +       id_ = oss.str();
> +
> +       /* The "camera" must appear to return the format the raw input wants. */
> +       bayerFormat_ = BayerFormat::fromPixelFormat(rawInput.pixelFormat);
> +       mbusCodes_ = { mbusCode };
> +
> +       v4l2SubdeviceFormat_ = V4L2SubdeviceFormat{
> +               .code = mbusCode,
> +               .size = rawInput.size,
> +               .colorSpace = ColorSpace::Raw,
> +       };
> +}
> +
> +CameraSensorMemory::~CameraSensorMemory() = default;
> +
> +std::variant<std::unique_ptr<CameraSensor>, int>
> +CameraSensorMemory::match([[maybe_unused]] MediaEntity *entity)
> +{
> +       return {};
> +}
> +
> +const std::vector<unsigned int> &CameraSensorMemory::mbusCodes() const
> +{
> +       return mbusCodes_;
> +}
> +
> +std::vector<Size> CameraSensorMemory::sizes(unsigned int mbusCode) const
> +{
> +       if (mbusCode == mbusCodes_[0])
> +               return { rawInput_.size };
> +       else
> +               return {};
> +}
> +
> +Size CameraSensorMemory::resolution() const
> +{
> +       return rawInput_.size;
> +}
> +
> +V4L2SubdeviceFormat CameraSensorMemory::getFormat(Span<const unsigned int> mbusCodes,
> +                                                 [[maybe_unused]] const Size &size,
> +                                                 const Size maxSize) const
> +{
> +       if (std::find(mbusCodes.begin(), mbusCodes.end(), mbusCodes_[0]) == mbusCodes.end())
> +               return {};
> +
> +       if (maxSize.width < rawInput_.size.width || maxSize.height < rawInput_.size.height)
> +               return {};
> +
> +       return v4l2SubdeviceFormat_;
> +}
> +
> +int CameraSensorMemory::setFormat(V4L2SubdeviceFormat *format,
> +                                 Transform transform)
> +{
> +       if (v4l2SubdeviceFormatEqual(*format, v4l2SubdeviceFormat_) &&
> +           transform == Transform::Identity)
> +               return 0;
> +
> +       return -EPERM;
> +}
> +
> +int CameraSensorMemory::tryFormat(V4L2SubdeviceFormat *format) const
> +{
> +       if (v4l2SubdeviceFormatEqual(*format, v4l2SubdeviceFormat_))
> +               return 0;
> +
> +       return -EPERM;
> +}
> +
> +int CameraSensorMemory::applyConfiguration(const SensorConfiguration &config,
> +                                          Transform transform,
> +                                          V4L2SubdeviceFormat *sensorFormat)
> +{
> +       if (config.bitDepth != bayerFormat_.bitDepth ||
> +           config.outputSize != rawInput_.size ||
> +           config.binning.binX != 1 || config.binning.binY != 1 ||
> +           config.skipping.xOddInc != 1 || config.skipping.xEvenInc != 1 ||
> +           config.skipping.yOddInc != 1 || config.skipping.yEvenInc != 1 ||
> +           transform != Transform::Identity)
> +               return -EPERM;
> +
> +       if (sensorFormat)
> +               *sensorFormat = v4l2SubdeviceFormat_;
> +
> +       return 0;
> +}
> +
> +V4L2Subdevice::Stream CameraSensorMemory::imageStream() const
> +{
> +       return V4L2Subdevice::Stream();
> +}
> +
> +std::optional<V4L2Subdevice::Stream> CameraSensorMemory::embeddedDataStream() const
> +{
> +       return {};
> +}
> +
> +V4L2SubdeviceFormat CameraSensorMemory::embeddedDataFormat() const
> +{
> +       return {};
> +}
> +
> +int CameraSensorMemory::setEmbeddedDataEnabled(bool enable)
> +{
> +       return enable ? -ENOSTR : 0;
> +}
> +
> +const ControlList &CameraSensorMemory::properties() const
> +{
> +       return properties_;
> +}
> +
> +int CameraSensorMemory::sensorInfo([[maybe_unused]] IPACameraSensorInfo *info) const
> +{
> +       info->model = model();
> +
> +       info->bitsPerPixel = bayerFormat_.bitDepth;
> +       info->cfaPattern = properties::draft::RGB;
> +
> +       info->activeAreaSize = rawInput_.size;
> +       info->analogCrop = Rectangle(rawInput_.size);
> +       info->outputSize = rawInput_.size;
> +
> +       /*
> +        * These are meaningless for us, fill with ones rather than zeros because the
> +        * code will divide by some of these numbers.
> +        */
> +       info->pixelRate = 1;
> +       info->minLineLength = 1;
> +       info->maxLineLength = 1;
> +       info->minFrameLength = 1;
> +       info->maxFrameLength = 1;
> +
> +       return 0;
> +}
> +
> +Transform CameraSensorMemory::computeTransform(Orientation *orientation) const
> +{
> +       *orientation = Orientation::Rotate0;
> +       return Transform::Identity;
> +}
> +
> +BayerFormat::Order CameraSensorMemory::bayerOrder([[maybe_unused]] Transform t) const
> +{
> +       return bayerFormat_.order;
> +}
> +
> +Orientation CameraSensorMemory::mountingOrientation() const
> +{
> +       return Orientation::Rotate0;
> +}
> +
> +const ControlInfoMap &CameraSensorMemory::controls() const
> +{
> +       return *controls_.infoMap();
> +}
> +
> +ControlList CameraSensorMemory::getControls([[maybe_unused]] Span<const uint32_t> ids)
> +{
> +       return ControlList();
> +}
> +
> +int CameraSensorMemory::setControls([[maybe_unused]] ControlList *ctrls)
> +{
> +       return -EPERM;
> +}
> +
> +int CameraSensorMemory::setTestPatternMode([[maybe_unused]] controls::draft::TestPatternModeEnum mode)
> +{
> +       return -EPERM;
> +}
> +
> +const CameraSensorProperties::SensorDelays &CameraSensorMemory::sensorDelays()
> +{
> +       static constexpr CameraSensorProperties::SensorDelays defaultSensorDelays = {
> +               .exposureDelay = 2,
> +               .gainDelay = 1,
> +               .vblankDelay = 2,
> +               .hblankDelay = 2,
> +       };
> +
> +       return defaultSensorDelays; /* but doesn't mean anything */
> +}
> +
> +std::string CameraSensorMemory::logPrefix() const
> +{
> +       return "'memory'";
> +}
> +
> +/*
> + * We're not going to register this camera sensor as it doesn't match media entities
> + * like other sensors. Pipeline handlers will have to call it explicitly.
> + */

Very interesting!

Reviewed-by: Isaac Scott <isaac.scott@ideasonboard.com>

> +
> +} /* namespace libcamera */
> diff --git a/src/libcamera/sensor/meson.build b/src/libcamera/sensor/meson.build
> index dce74ed6..b9b87612 100644
> --- a/src/libcamera/sensor/meson.build
> +++ b/src/libcamera/sensor/meson.build
> @@ -3,6 +3,7 @@
>  libcamera_internal_sources += files([
>      'camera_sensor.cpp',
>      'camera_sensor_legacy.cpp',
> +    'camera_sensor_memory.cpp',
>      'camera_sensor_properties.cpp',
>      'camera_sensor_raw.cpp',
>  ])
> -- 
> 2.47.3
>