[libcamera-devel,v1,1/3] gstreamer: Add sensor mode selection

18455 diff mbox series

From: Nicolas Dufresne <nicolas.dufresne@collabora.com>

This add support for selecting the sensor mode. A new read-only
property called sensor-modes is filled when the element reaches
READY state. It contains the list of all available sensor modes,
including the supported framerate range. This is exposed as GstCaps
in the form of sensor/mode,width=X,height=Y,format=Y,framerate=[...].
The format string matches the libcamera format string representation.

The application can then select a mode using the read/write sensor-mode
control. The selected mode is also a caps, it will be intersected with
the supported mode and the "best" match will be picked. This allows
application to use simple filter when they want to pick a mode for lets
say a specific framerate (e.g. sensor/mode,framerate=60/1).

Signed-off-by: Nicolas Dufresne <nicolas.dufresne@collabora.com>
---
 src/gstreamer/gstlibcamera-utils.cpp |   4 +
 src/gstreamer/gstlibcamerasrc.cpp    | 123 ++++++++++++++++++++++++++-
 2 files changed, 126 insertions(+), 1 deletion(-)

Le vendredi 24 mars 2023 à 13:40 -0400, Nicolas Dufresne a écrit :
> From: Nicolas Dufresne <nicolas.dufresne@collabora.com>
> 
> This add support for selecting the sensor mode. A new read-only
> property called sensor-modes is filled when the element reaches
> READY state. It contains the list of all available sensor modes,
> including the supported framerate range. This is exposed as GstCaps
> in the form of sensor/mode,width=X,height=Y,format=Y,framerate=[...].
> The format string matches the libcamera format string representation.
> 
> The application can then select a mode using the read/write sensor-mode
> control. The selected mode is also a caps, it will be intersected with
> the supported mode and the "best" match will be picked. This allows
> application to use simple filter when they want to pick a mode for lets
> say a specific framerate (e.g. sensor/mode,framerate=60/1).
> 
> Signed-off-by: Nicolas Dufresne <nicolas.dufresne@collabora.com>
> ---
>  src/gstreamer/gstlibcamera-utils.cpp |   4 +
>  src/gstreamer/gstlibcamerasrc.cpp    | 123 ++++++++++++++++++++++++++-
>  2 files changed, 126 insertions(+), 1 deletion(-)
> 
> diff --git a/src/gstreamer/gstlibcamera-utils.cpp b/src/gstreamer/gstlibcamera-utils.cpp
> index 750ec351..c8a8df09 100644
> --- a/src/gstreamer/gstlibcamera-utils.cpp
> +++ b/src/gstreamer/gstlibcamera-utils.cpp
> @@ -416,6 +416,10 @@ gst_libcamera_configure_stream_from_caps(StreamConfiguration &stream_cfg,
>  		stream_cfg.pixelFormat = gst_format_to_pixel_format(gst_format);
>  	} else if (gst_structure_has_name(s, "image/jpeg")) {
>  		stream_cfg.pixelFormat = formats::MJPEG;
> +	} else if (gst_structure_has_name(s, "sensor/mode")) {
> +		gst_structure_fixate_field(s, "format");
> +		const gchar *format = gst_structure_get_string(s, "format");
> +		stream_cfg.pixelFormat = PixelFormat::fromString(format);
>  	} else {
>  		g_critical("Unsupported media type: %s", gst_structure_get_name(s));
>  	}
> diff --git a/src/gstreamer/gstlibcamerasrc.cpp b/src/gstreamer/gstlibcamerasrc.cpp
> index a10cbd4f..c448a9fe 100644
> --- a/src/gstreamer/gstlibcamerasrc.cpp
> +++ b/src/gstreamer/gstlibcamerasrc.cpp
> @@ -147,6 +147,15 @@ struct _GstLibcameraSrc {
>  
>  	gchar *camera_name;
>  
> +<<<<<<< HEAD
> +=======

My apology, scratch this please.

> +	gboolean hflip;
> +	gboolean vflip;
> +
> +	GstCaps *sensor_modes;
> +	GstCaps *sensor_mode;
> +
> +>>>>>>> 210f0a67 (gstreamer: Add sensor mode selection)
>  	GstLibcameraSrcState *state;
>  	GstLibcameraAllocator *allocator;
>  	GstFlowCombiner *flow_combiner;
> @@ -154,7 +163,16 @@ struct _GstLibcameraSrc {
>  
>  enum {
>  	PROP_0,
> +<<<<<<< HEAD
>  	PROP_CAMERA_NAME
> +=======
> +	PROP_CAMERA_NAME,
> +	PROP_HFLIP,
> +	PROP_VFLIP,
> +	PROP_SENSOR_MODES,
> +	PROP_SENSOR_MODE,
> +	PROP_LAST
> +>>>>>>> 210f0a67 (gstreamer: Add sensor mode selection)
>  };
>  
>  G_DEFINE_TYPE_WITH_CODE(GstLibcameraSrc, gst_libcamera_src, GST_TYPE_ELEMENT,
> @@ -318,6 +336,61 @@ int GstLibcameraSrcState::processRequest()
>  	return err;
>  }
>  
> +static GstCaps *
> +gst_libcamera_src_enumerate_sensor_modes(GstLibcameraSrc *self)
> +{
> +	GstCaps *modes = gst_caps_new_empty();
> +	GstLibcameraSrcState *state = self->state;
> +	auto config = state->cam_->generateConfiguration({ libcamera::StreamRole::Raw,
> +							   libcamera::StreamRole::VideoRecording });
> +	if (config == nullptr) {
> +		GST_DEBUG_OBJECT(self, "Sensor mode selection is not supported, skipping enumeration.");
> +		return modes;
> +	}
> +
> +	const libcamera::StreamFormats &formats = config->at(0).formats();
> +
> +	for (const auto &pixfmt : formats.pixelformats()) {
> +		for (const auto &size : formats.sizes(pixfmt)) {
> +			config->at(0).size = size;
> +			config->at(0).pixelFormat = pixfmt;
> +
> +			if (config->validate() == CameraConfiguration::Invalid)
> +				continue;
> +
> +			if (state->cam_->configure(config.get()))
> +				continue;
> +
> +			auto fd_ctrl = state->cam_->controls().find(&controls::FrameDurationLimits);
> +			if (fd_ctrl == state->cam_->controls().end())
> +				continue;
> +
> +			int minrate_num, minrate_denom;
> +			int maxrate_num, maxrate_denom;
> +			double min_framerate = gst_util_guint64_to_gdouble(1.0e6) /
> +					       gst_util_guint64_to_gdouble(fd_ctrl->second.max().get<int64_t>());
> +			double max_framerate = gst_util_guint64_to_gdouble(1.0e6) /
> +					       gst_util_guint64_to_gdouble(fd_ctrl->second.min().get<int64_t>());
> +			gst_util_double_to_fraction(min_framerate, &minrate_num, &minrate_denom);
> +			gst_util_double_to_fraction(max_framerate, &maxrate_num, &maxrate_denom);
> +
> +			GstStructure *s = gst_structure_new("sensor/mode",
> +							    "format", G_TYPE_STRING, pixfmt.toString().c_str(),
> +							    "width", G_TYPE_INT, size.width,
> +							    "height", G_TYPE_INT, size.height,
> +							    "framerate", GST_TYPE_FRACTION_RANGE,
> +							    minrate_num, minrate_denom,
> +							    maxrate_num, maxrate_denom,
> +							    nullptr);
> +			gst_caps_append_structure(modes, s);
> +		}
> +	}
> +
> +	GST_DEBUG_OBJECT(self, "Camera sensor modes: %" GST_PTR_FORMAT, modes);
> +
> +	return modes;
> +}
> +
>  static bool
>  gst_libcamera_src_open(GstLibcameraSrc *self)
>  {
> @@ -375,6 +448,7 @@ gst_libcamera_src_open(GstLibcameraSrc *self)
>  	/* No need to lock here, we didn't start our threads yet. */
>  	self->state->cm_ = cm;
>  	self->state->cam_ = cam;
> +	self->sensor_modes = gst_libcamera_src_enumerate_sensor_modes(self);
>  
>  	return true;
>  }
> @@ -462,6 +536,7 @@ gst_libcamera_src_task_enter(GstTask *task, [[maybe_unused]] GThread *thread,
>  	GstLibcameraSrcState *state = self->state;
>  	GstFlowReturn flow_ret = GST_FLOW_OK;
>  	gint ret;
> +	g_autoptr(GstCaps) sensor_mode = nullptr;
>  
>  	g_autoptr(GstStructure) element_caps = gst_structure_new_empty("caps");
>  
> @@ -481,6 +556,16 @@ gst_libcamera_src_task_enter(GstTask *task, [[maybe_unused]] GThread *thread,
>  		roles.push_back(gst_libcamera_pad_get_role(srcpad));
>  	}
>  
> +	if (!gst_caps_is_any(self->sensor_mode)) {
> +		sensor_mode = gst_caps_intersect(self->sensor_mode, self->sensor_modes);
> +		if (!gst_caps_is_empty(sensor_mode)) {
> +			roles.push_back(libcamera::StreamRole::Raw);
> +		} else {
> +			GST_WARNING_OBJECT(self, "No sensor mode matching the selection, ignoring.");
> +			gst_clear_caps(&sensor_mode);
> +		}
> +	}
> +
>  	/* Generate the stream configurations, there should be one per pad. */
>  	state->config_ = state->cam_->generateConfiguration(roles);
>  	if (state->config_ == nullptr) {
> @@ -490,7 +575,7 @@ gst_libcamera_src_task_enter(GstTask *task, [[maybe_unused]] GThread *thread,
>  		gst_task_stop(task);
>  		return;
>  	}
> -	g_assert(state->config_->size() == state->srcpads_.size());
> +	g_assert(state->config_->size() == state->srcpads_.size() + (!!sensor_mode));
>  
>  	for (gsize i = 0; i < state->srcpads_.size(); i++) {
>  		GstPad *srcpad = state->srcpads_[i];
> @@ -510,6 +595,12 @@ gst_libcamera_src_task_enter(GstTask *task, [[maybe_unused]] GThread *thread,
>  		gst_libcamera_get_framerate_from_caps(caps, element_caps);
>  	}
>  
> +	if (sensor_mode) {
> +		StreamConfiguration &stream_cfg = state->config_->at(state->srcpads_.size());
> +		g_assert(gst_caps_is_writable(sensor_mode));
> +		gst_libcamera_configure_stream_from_caps(stream_cfg, sensor_mode);
> +	}
> +
>  	if (flow_ret != GST_FLOW_OK)
>  		goto done;
>  
> @@ -624,6 +715,7 @@ gst_libcamera_src_task_leave([[maybe_unused]] GstTask *task,
>  	g_clear_object(&self->allocator);
>  	g_clear_pointer(&self->flow_combiner,
>  			(GDestroyNotify)gst_flow_combiner_free);
> +	gst_clear_caps(&self->sensor_modes);
>  }
>  
>  static void
> @@ -659,6 +751,10 @@ gst_libcamera_src_set_property(GObject *object, guint prop_id,
>  		g_free(self->camera_name);
>  		self->camera_name = g_value_dup_string(value);
>  		break;
> +	case PROP_SENSOR_MODE:
> +		gst_clear_caps(&self->sensor_mode);
> +		self->sensor_mode = GST_CAPS(g_value_dup_boxed(value));
> +		break;
>  	default:
>  		G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec);
>  		break;
> @@ -676,6 +772,12 @@ gst_libcamera_src_get_property(GObject *object, guint prop_id, GValue *value,
>  	case PROP_CAMERA_NAME:
>  		g_value_set_string(value, self->camera_name);
>  		break;
> +	case PROP_SENSOR_MODES:
> +		g_value_set_boxed(value, self->sensor_modes);
> +		break;
> +	case PROP_SENSOR_MODE:
> +		g_value_set_boxed(value, self->sensor_mode);
> +		break;
>  	default:
>  		G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec);
>  		break;
> @@ -763,6 +865,7 @@ gst_libcamera_src_init(GstLibcameraSrc *self)
>  	/* C-style friend. */
>  	state->src_ = self;
>  	self->state = state;
> +	self->sensor_mode = gst_caps_new_any();
>  }
>  
>  static GstPad *
> @@ -844,4 +947,22 @@ gst_libcamera_src_class_init(GstLibcameraSrcClass *klass)
>  							     | G_PARAM_READWRITE
>  							     | G_PARAM_STATIC_STRINGS));
>  	g_object_class_install_property(object_class, PROP_CAMERA_NAME, spec);
> +
> +	spec = g_param_spec_boxed("sensor-modes", "Sensor Modes",
> +				  "GstCaps representing available sensor modes.",
> +				  GST_TYPE_CAPS,
> +				  (GParamFlags)(G_PARAM_READABLE
> +					        | G_PARAM_STATIC_STRINGS));
> +	g_object_class_install_property(object_class, PROP_SENSOR_MODES, spec);
> +
> +	spec = g_param_spec_boxed("sensor-mode", "Sensor Mode",
> +				  "GstCaps representing selected sensor mode.",
> +				  GST_TYPE_CAPS,
> +				  (GParamFlags)(GST_PARAM_MUTABLE_READY
> +					        | G_PARAM_READWRITE
> +					        | G_PARAM_STATIC_STRINGS));
> +	g_object_class_install_property(object_class, PROP_SENSOR_MODE, spec);
> +
> +
> +	GstCameraControls::installProperties(object_class, PROP_LAST);
>  }