[RFC,v1,2/2] libcamera: camera_sensor: Add support for auxiliary image
diff mbox series

Message ID 20251024151058.1523308-3-julien.vuillaumier@nxp.com
State New
Headers show
Series
  • libcamera: camera_sensor: Add support for an additional image stream
Related show

Commit Message

Julien Vuillaumier Oct. 24, 2025, 3:10 p.m. UTC 
Some sensors can produce and transmit an auxiliary image over a
dedicated stream, separate from the image stream. Examples for such
sensor are the ones supporting HDR or A/B mode captures.

Add support for that type of stream in the CameraSensor interface,
and implement it for the CameraSensorRaw class.
The CameraSensorLegacy uses the default stub implementation, as
the corresponding kernel drivers don't support an auxiliary image.

Signed-off-by: Julien Vuillaumier <julien.vuillaumier@nxp.com>
---
 include/libcamera/internal/camera_sensor.h |   3 +
 src/libcamera/sensor/camera_sensor.cpp     |  52 ++++++++++
 src/libcamera/sensor/camera_sensor_raw.cpp | 112 ++++++++++++++++++---
 3 files changed, 151 insertions(+), 16 deletions(-)

Patch
diff mbox series 

diff --git a/include/libcamera/internal/camera_sensor.h b/include/libcamera/internal/camera_sensor.h
index f6ef4df1..6df928f5 100644
--- a/include/libcamera/internal/camera_sensor.h
+++ b/include/libcamera/internal/camera_sensor.h
@@ -68,6 +68,9 @@  public:
 	virtual std::optional<V4L2Subdevice::Stream> embeddedDataStream() const;
 	virtual V4L2SubdeviceFormat embeddedDataFormat() const;
 	virtual int setEmbeddedDataEnabled(bool enable);
+	virtual std::optional<V4L2Subdevice::Stream> auxiliaryStream() const;
+	virtual V4L2SubdeviceFormat auxiliaryFormat() const;
+	virtual int setAuxiliaryEnabled(bool enable);
 
 	virtual const ControlList &properties() const = 0;
 	virtual int sensorInfo(IPACameraSensorInfo *info) const = 0;
diff --git a/src/libcamera/sensor/camera_sensor.cpp b/src/libcamera/sensor/camera_sensor.cpp
index 4f2fd269..3bf05f11 100644
--- a/src/libcamera/sensor/camera_sensor.cpp
+++ b/src/libcamera/sensor/camera_sensor.cpp
@@ -266,6 +266,58 @@  int CameraSensor::setEmbeddedDataEnabled(bool enable)
 	return enable ? -ENOSTR : 0;
 }
 
+/**
+ * \brief Retrieve the auxiliary image source stream
+ *
+ * Some sensors produce an auxiliary image stream separate from the image
+ * stream. This function indicates if the sensor supports this feature by
+ * returning the auxialiary stream on the sensor's source pad if available,
+ * or an std::optional<> without a value otheriwse.
+ *
+ * \return The auxiliary source stream
+ */
+std::optional<V4L2Subdevice::Stream> CameraSensor::auxiliaryStream() const
+{
+	return {};
+}
+
+/**
+ * \brief Retrieve the format on the auxiliary stream
+ *
+ * When an auxiliary image stream is available, this function returns the
+ * corresponding format on the sensor's source pad. The format may vary with
+ * the image stream format, and should therefore be retrieved after configuring
+ * the image stream.
+ *
+ * If the sensor doesn't support auxiliary stream, this function returns a
+ * default-constructed format.
+ *
+ * \return The format on the embedded data stream
+ */
+V4L2SubdeviceFormat CameraSensor::auxiliaryFormat() const
+{
+	return {};
+}
+
+/**
+ * \brief Enable or disable the auxiliary image stream
+ * \param[in] enable True to enable the auxiliary image stream, false to disable it
+ *
+ * For sensors that support it function enables or disables generation of
+ * auxiliary image stream. Some of such sensors always produce an auxiliary
+ * stream, in which case this function return -EISCONN if the caller attempts to
+ * disable it.
+ *
+ * If the sensor doesn't support an auxiliary image stream, this function
+ * returns 0 when \a enable is false, and -ENOSTR otherwise.
+ *
+ * \return 0 on success, or a negative error code otherwise
+ */
+int CameraSensor::setAuxiliaryEnabled(bool enable)
+{
+	return enable ? -ENOSTR : 0;
+}
+
 /**
  * \fn CameraSensor::properties()
  * \brief Retrieve the camera sensor properties
diff --git a/src/libcamera/sensor/camera_sensor_raw.cpp b/src/libcamera/sensor/camera_sensor_raw.cpp
index ad4a94ad..ecc64b44 100644
--- a/src/libcamera/sensor/camera_sensor_raw.cpp
+++ b/src/libcamera/sensor/camera_sensor_raw.cpp
@@ -89,6 +89,9 @@  public:
 	std::optional<V4L2Subdevice::Stream> embeddedDataStream() const override;
 	V4L2SubdeviceFormat embeddedDataFormat() const override;
 	int setEmbeddedDataEnabled(bool enable) override;
+	virtual std::optional<V4L2Subdevice::Stream> auxiliaryStream() const override;
+	virtual V4L2SubdeviceFormat auxiliaryFormat() const override;
+	virtual int setAuxiliaryEnabled(bool enable) override;
 
 	const ControlList &properties() const override { return properties_; }
 	int sensorInfo(IPACameraSensorInfo *info) const override;
@@ -127,6 +130,7 @@  private:
 	struct {
 		Streams image;
 		std::optional<Streams> edata;
+		std::optional<Streams> auxiliary;
 	} streams_;
 
 	const CameraSensorProperties *staticProps_;
@@ -206,7 +210,7 @@  CameraSensorRaw::match(MediaEntity *entity)
 		}
 	}
 
-	if (numSinks < 1 || numSinks > 2 || numSources != 1) {
+	if (numSinks < 1 || numSinks > 3 || numSources != 1) {
 		libcamera::LOG(CameraSensor, Debug)
 			<< entity->name() << ": unsupported number of sinks ("
 			<< numSinks << ") or sources (" << numSources << ")";
@@ -277,6 +281,7 @@  std::optional<int> CameraSensorRaw::init()
 	}
 
 	bool imageStreamFound = false;
+	bool auxiliaryStreamFound = false;
 
 	for (const V4L2Subdevice::Route &route : routing) {
 		if (route.source.pad != sourcePad) {
@@ -306,17 +311,21 @@  std::optional<int> CameraSensorRaw::init()
 
 		switch (*type) {
 		case MediaBusFormatInfo::Type::Image:
-			if (imageStreamFound) {
+			/* Assume that primary image is on the stream 0/0 */
+			if (!imageStreamFound &&
+			    route.source.pad == 0 && route.source.stream == 0) {
+				imageStreamFound = true;
+				streams_.image = { route.sink, route.source };
+			} else if (!auxiliaryStreamFound) {
+				auxiliaryStreamFound = true;
+				streams_.auxiliary = { route.sink, route.source };
+			} else {
 				LOG(CameraSensor, Error)
 					<< "Multiple internal image streams ("
 					<< streams_.image.sink << " and "
 					<< route.sink << ")";
 				return { -EINVAL };
 			}
-
-			imageStreamFound = true;
-			streams_.image.sink = route.sink;
-			streams_.image.source = route.source;
 			break;
 
 		case MediaBusFormatInfo::Type::Metadata:
@@ -363,6 +372,11 @@  std::optional<int> CameraSensorRaw::init()
 			<< "Found embedded data stream " << streams_.edata->sink
 			<< " -> " << streams_.edata->source;
 
+	if (streams_.auxiliary)
+		LOG(CameraSensor, Debug)
+			<< "Found auxiliary stream " << streams_.auxiliary->sink
+			<< " -> " << streams_.auxiliary->source;
+
 	/* Restore the routes to their initial state */
 	ret = subdev_->setRouting(&routing);
 	if (ret) {
@@ -963,20 +977,22 @@  V4L2SubdeviceFormat CameraSensorRaw::embeddedDataFormat() const
 
 int CameraSensorRaw::setEmbeddedDataEnabled(bool enable)
 {
+	int ret;
+
 	if (!streams_.edata)
 		return enable ? -ENOSTR : 0;
 
-	V4L2Subdevice::Routing routing{ 2 };
-
-	routing[0].sink = streams_.image.sink;
-	routing[0].source = streams_.image.source;
-	routing[0].flags = V4L2_SUBDEV_ROUTE_FL_ACTIVE;
-
-	routing[1].sink = streams_.edata->sink;
-	routing[1].source = streams_.edata->source;
-	routing[1].flags = enable ? V4L2_SUBDEV_ROUTE_FL_ACTIVE : 0;
+	V4L2Subdevice::Routing routing;
+	ret = subdev_->getRouting(&routing);
+	if (ret)
+		return ret;
 
-	int ret = subdev_->setRouting(&routing);
+	for (V4L2Subdevice::Route &route : routing) {
+		if (route.source != streams_.edata->source)
+			continue;
+		route.flags = enable ? V4L2_SUBDEV_ROUTE_FL_ACTIVE : 0;
+	}
+	ret = subdev_->setRouting(&routing);
 	if (ret)
 		return ret;
 
@@ -1002,6 +1018,70 @@  int CameraSensorRaw::setEmbeddedDataEnabled(bool enable)
 	return 0;
 }
 
+std::optional<V4L2Subdevice::Stream> CameraSensorRaw::auxiliaryStream() const
+{
+	if (!streams_.auxiliary)
+		return {};
+
+	return { streams_.auxiliary->source };
+}
+
+V4L2SubdeviceFormat CameraSensorRaw::auxiliaryFormat() const
+{
+	if (!streams_.auxiliary)
+		return {};
+
+	V4L2SubdeviceFormat format;
+	int ret = subdev_->getFormat(streams_.auxiliary->source, &format);
+	if (ret)
+		return {};
+
+	return format;
+}
+
+int CameraSensorRaw::setAuxiliaryEnabled(bool enable)
+{
+	int ret;
+
+	if (!streams_.auxiliary)
+		return enable ? -ENOSTR : 0;
+
+	V4L2Subdevice::Routing routing;
+	ret = subdev_->getRouting(&routing);
+	if (ret)
+		return ret;
+
+	for (V4L2Subdevice::Route &route : routing) {
+		if (route.source != streams_.auxiliary->source)
+			continue;
+		route.flags = enable ? V4L2_SUBDEV_ROUTE_FL_ACTIVE : 0;
+	}
+	ret = subdev_->setRouting(&routing);
+	if (ret)
+		return ret;
+
+	/*
+	 * Check if the auxiliary stream has been enabled or disabled
+	 * correctly. Assume at least one route will match the auxiliary
+	 * source stream, as there would be something seriously wrong
+	 * otherwise.
+	 */
+	bool enabled = false;
+
+	for (const V4L2Subdevice::Route &route : routing) {
+		if (route.source != streams_.auxiliary->source)
+			continue;
+
+		enabled = route.flags & V4L2_SUBDEV_ROUTE_FL_ACTIVE;
+		break;
+	}
+
+	if (enabled != enable)
+		return enabled ? -EISCONN : -ENOSTR;
+
+	return 0;
+}
+
 int CameraSensorRaw::sensorInfo(IPACameraSensorInfo *info) const
 {
 	info->model = model();