[v4] gstreamer: Add GstVideoMeta support

22030 diff mbox series

GStreamer video-info calculated stride and offset may differ from
those used by the camera.

This patch enhances downstream plugin's support for videometa by
adding videometa support for libcamerasrc. It ensures that when
downstream plugin supports videometa by allocation query, libcamerasrc
also attaches videometa to buffer, preventing discrepancies
between the stride and offset calculated by video-info and camera.

Signed-off-by: Hou Qi <qi.hou@nxp.com>
---
 src/gstreamer/gstlibcamera-utils.cpp | 33 ++++++++++++++++
 src/gstreamer/gstlibcamera-utils.h   |  4 ++
 src/gstreamer/gstlibcamerapool.cpp   | 57 +++++++++++++++++++++++++++-
 src/gstreamer/gstlibcamerapool.h     |  5 ++-
 src/gstreamer/gstlibcamerasrc.cpp    | 47 +++++++++++++++++++++--
 5 files changed, 141 insertions(+), 5 deletions(-)

Hi,

Le mercredi 20 novembre 2024 à 19:04 +0900, Hou Qi a écrit :
> GStreamer video-info calculated stride and offset may differ from
> those used by the camera.
> 
> This patch enhances downstream plugin's support for videometa by
> adding videometa support for libcamerasrc. It ensures that when
> downstream plugin supports videometa by allocation query, libcamerasrc
> also attaches videometa to buffer, preventing discrepancies
> between the stride and offset calculated by video-info and camera.
> 
> Signed-off-by: Hou Qi <qi.hou@nxp.com>
> ---
>  src/gstreamer/gstlibcamera-utils.cpp | 33 ++++++++++++++++
>  src/gstreamer/gstlibcamera-utils.h   |  4 ++
>  src/gstreamer/gstlibcamerapool.cpp   | 57 +++++++++++++++++++++++++++-
>  src/gstreamer/gstlibcamerapool.h     |  5 ++-
>  src/gstreamer/gstlibcamerasrc.cpp    | 47 +++++++++++++++++++++--
>  5 files changed, 141 insertions(+), 5 deletions(-)
> 
> diff --git a/src/gstreamer/gstlibcamera-utils.cpp b/src/gstreamer/gstlibcamera-utils.cpp
> index 732987ef..09af9204 100644
> --- a/src/gstreamer/gstlibcamera-utils.cpp
> +++ b/src/gstreamer/gstlibcamera-utils.cpp
> @@ -591,6 +591,39 @@ gst_task_resume(GstTask *task)
>  }
>  #endif
>  
> +#if !GST_CHECK_VERSION(1, 22, 0)
> +/*
> + * Copyright (C) <1999> Erik Walthinsen <omega@cse.ogi.edu>
> + * Library       <2002> Ronald Bultje <rbultje@ronald.bitfreak.net>
> + * Copyright (C) <2007> David A. Schleef <ds@schleef.org>
> + */
> +/* This function has been imported directly from the gstreamer project to
> + * support backwards compatibility and should be removed when the older
> + * version is no longer supported. */
> +gint gst_video_format_info_extrapolate_stride(const GstVideoFormatInfo *finfo, gint plane, gint stride)
> +{
> +	gint estride;
> +	gint comp[GST_VIDEO_MAX_COMPONENTS];
> +	gint i;
> +
> +	/* there is nothing to extrapolate on first plane */
> +	if (plane == 0)
> +		return stride;
> +
> +	gst_video_format_info_component(finfo, plane, comp);
> +
> +	/* For now, all planar formats have a single component on first plane, but
> +	* if there was a planar format with more, we'd have to make a ratio of the
> +	* number of component on the first plane against the number of component on
> +	* the current plane. */
> +	estride = 0;
> +	for (i = 0; i < GST_VIDEO_MAX_COMPONENTS && comp[i] >= 0; i++)
> +		estride += GST_VIDEO_FORMAT_INFO_SCALE_WIDTH(finfo, comp[i], stride);
> +
> +	return estride;
> +}
> +#endif
> +
>  G_LOCK_DEFINE_STATIC(cm_singleton_lock);
>  static std::weak_ptr<CameraManager> cm_singleton_ptr;
>  
> diff --git a/src/gstreamer/gstlibcamera-utils.h b/src/gstreamer/gstlibcamera-utils.h
> index cab1c814..81149280 100644
> --- a/src/gstreamer/gstlibcamera-utils.h
> +++ b/src/gstreamer/gstlibcamera-utils.h
> @@ -35,6 +35,10 @@ static inline void gst_clear_event(GstEvent **event_ptr)
>  #if !GST_CHECK_VERSION(1, 17, 1)
>  gboolean gst_task_resume(GstTask *task);
>  #endif
> +
> +#if !GST_CHECK_VERSION(1, 22, 0)
> +gint gst_video_format_info_extrapolate_stride(const GstVideoFormatInfo *finfo, gint plane, gint stride);
> +#endif
>  std::shared_ptr<libcamera::CameraManager> gst_libcamera_get_camera_manager(int &ret);
>  
>  /**
> diff --git a/src/gstreamer/gstlibcamerapool.cpp b/src/gstreamer/gstlibcamerapool.cpp
> index 9cd7eccb..bd1cc473 100644
> --- a/src/gstreamer/gstlibcamerapool.cpp
> +++ b/src/gstreamer/gstlibcamerapool.cpp
> @@ -29,6 +29,8 @@ struct _GstLibcameraPool {
>  	std::deque<GstBuffer *> *queue;
>  	GstLibcameraAllocator *allocator;
>  	Stream *stream;
> +	gboolean frame_copy;
> +	GstVideoInfo info;
>  };
>  
>  G_DEFINE_TYPE(GstLibcameraPool, gst_libcamera_pool, GST_TYPE_BUFFER_POOL)
> @@ -135,16 +137,29 @@ gst_libcamera_pool_class_init(GstLibcameraPoolClass *klass)
>  }
>  
>  GstLibcameraPool *
> -gst_libcamera_pool_new(GstLibcameraAllocator *allocator, Stream *stream)
> +gst_libcamera_pool_new(GstLibcameraAllocator *allocator, Stream *stream,
> +		       GstVideoInfo *info, gboolean stride_mismatch, gboolean has_video_meta)
>  {
>  	auto *pool = GST_LIBCAMERA_POOL(g_object_new(GST_TYPE_LIBCAMERA_POOL, nullptr));
>  
>  	pool->allocator = GST_LIBCAMERA_ALLOCATOR(g_object_ref(allocator));
>  	pool->stream = stream;
> +	pool->info = *info;
> +
> +	if (stride_mismatch && !has_video_meta)
> +		pool->frame_copy = true;
> +	else
> +		pool->frame_copy = false;
>  
>  	gsize pool_size = gst_libcamera_allocator_get_pool_size(allocator, stream);
>  	for (gsize i = 0; i < pool_size; i++) {
>  		GstBuffer *buffer = gst_buffer_new();
> +		if (stride_mismatch && has_video_meta) {
> +			gst_buffer_add_video_meta_full(buffer, GST_VIDEO_FRAME_FLAG_NONE,
> +						       GST_VIDEO_INFO_FORMAT(info), GST_VIDEO_INFO_WIDTH(info),
> +						       GST_VIDEO_INFO_HEIGHT(info), GST_VIDEO_INFO_N_PLANES(info),
> +						       info->offset, info->stride);
> +		}
>  		pool->queue->push_back(buffer);
>  	}
>  
> @@ -163,3 +178,43 @@ gst_libcamera_buffer_get_frame_buffer(GstBuffer *buffer)
>  	GstMemory *mem = gst_buffer_peek_memory(buffer, 0);
>  	return gst_libcamera_memory_get_frame_buffer(mem);
>  }
> +
> +GstBuffer *
> +gst_libcamera_copy_buffer(GstLibcameraPool *self, GstBuffer *src, FrameBuffer *fb, guint32 stride)

A function that says "copy_buffer" should always copy the buffers. The naming
should be reconsidered, that being said see comment below.

> +{
> +	if (self->frame_copy) {
> +		GstVideoInfo src_info = self->info;
> +		gsize size = GST_VIDEO_INFO_SIZE(&self->info);
> +		GstBuffer *dest = gst_buffer_new_allocate(NULL, size, NULL);

The copy condition should be moved into the libcamerasrc element code. This way,
you can run an allocation query, and keep copy into downstream provided buffer
instead of doing dynamic allocation. You can also use a generic GstVideoPool to
avoid dynamic allocations even if downstream did not provide a pool.

> +		GstVideoFrame src_frame, dest_frame;
> +		int i = 0;
> +
> +		for (const FrameBuffer::Plane &plane : fb->planes()) {
> +			src_info.stride[i] = stride;
> +			src_info.offset[i] = plane.offset;
> +			i++;
> +		}
> +		src_info.size = gst_buffer_get_size(src);
> +
> +		if (!gst_video_frame_map(&src_frame, &src_info, src, GST_MAP_READ)) {
> +			GST_WARNING("fail to map src_frame");
> +			return src;
> +		}
> +
> +		if (!gst_video_frame_map(&dest_frame, &self->info, dest, GST_MAP_WRITE)) {
> +			gst_video_frame_unmap(&src_frame);
> +			GST_WARNING("fail to map dest_frame");
> +			return src;
> +		}

A map failure would indicate serious issues, we should not ignore these and
continue with the original.

> +
> +		gst_video_frame_copy(&dest_frame, &src_frame);
> +
> +		gst_video_frame_unmap(&src_frame);
> +		gst_video_frame_unmap(&dest_frame);
> +
> +		gst_buffer_unref(src);
> +		return dest;
> +	}
> +
> +	return src;
> +}
> diff --git a/src/gstreamer/gstlibcamerapool.h b/src/gstreamer/gstlibcamerapool.h
> index 2a7a9c77..ffbd3173 100644
> --- a/src/gstreamer/gstlibcamerapool.h
> +++ b/src/gstreamer/gstlibcamerapool.h
> @@ -14,6 +14,7 @@
>  #include "gstlibcameraallocator.h"
>  
>  #include <gst/gst.h>
> +#include <gst/video/video.h>
>  
>  #include <libcamera/stream.h>
>  
> @@ -21,8 +22,10 @@
>  G_DECLARE_FINAL_TYPE(GstLibcameraPool, gst_libcamera_pool, GST_LIBCAMERA, POOL, GstBufferPool)
>  
>  GstLibcameraPool *gst_libcamera_pool_new(GstLibcameraAllocator *allocator,
> -					 libcamera::Stream *stream);
> +					 libcamera::Stream *stream, GstVideoInfo *info, gboolean stride_mismatch, gboolean has_video_meta);
>  
>  libcamera::Stream *gst_libcamera_pool_get_stream(GstLibcameraPool *self);
>  
>  libcamera::FrameBuffer *gst_libcamera_buffer_get_frame_buffer(GstBuffer *buffer);
> +
> +GstBuffer *gst_libcamera_copy_buffer(GstLibcameraPool *self, GstBuffer *buffer, libcamera::FrameBuffer *fb, guint32 stride);
> diff --git a/src/gstreamer/gstlibcamerasrc.cpp b/src/gstreamer/gstlibcamerasrc.cpp
> index 8efa25f4..1d46536a 100644
> --- a/src/gstreamer/gstlibcamerasrc.cpp
> +++ b/src/gstreamer/gstlibcamerasrc.cpp
> @@ -292,12 +292,17 @@ int GstLibcameraSrcState::processRequest()
>  	GstFlowReturn ret = GST_FLOW_OK;
>  	gst_flow_combiner_reset(src_->flow_combiner);
>  
> -	for (GstPad *srcpad : srcpads_) {
> +	for (gsize i = 0; i < src_->state->srcpads_.size(); i++) {
> +		GstPad *srcpad = src_->state->srcpads_[i];
>  		Stream *stream = gst_libcamera_pad_get_stream(srcpad);
>  		GstBuffer *buffer = wrap->detachBuffer(stream);
> +		const StreamConfiguration &stream_cfg = src_->state->config_->at(i);
> +		GstLibcameraPool *pool = gst_libcamera_pad_get_pool(srcpad);
>  
>  		FrameBuffer *fb = gst_libcamera_buffer_get_frame_buffer(buffer);
>  
> +		buffer = gst_libcamera_copy_buffer(pool, buffer, fb, stream_cfg.stride);
> +
>  		if (GST_CLOCK_TIME_IS_VALID(wrap->pts_)) {
>  			GST_BUFFER_PTS(buffer) = wrap->pts_;
>  			gst_libcamera_pad_set_latency(srcpad, wrap->latency_);
> @@ -497,9 +502,45 @@ gst_libcamera_src_negotiate(GstLibcameraSrc *self)
>  	for (gsize i = 0; i < state->srcpads_.size(); i++) {
>  		GstPad *srcpad = state->srcpads_[i];
>  		const StreamConfiguration &stream_cfg = state->config_->at(i);
> +		gboolean stride_mismatch = false, has_video_meta = false;
> +		GstVideoInfo info;
> +
> +		g_autoptr(GstCaps) caps = gst_libcamera_stream_configuration_to_caps(stream_cfg);
> +		gst_libcamera_framerate_to_caps(caps, element_caps);
> +
> +		gst_video_info_init(&info);
> +		gst_video_info_from_caps(&info, caps);
> +
> +		/* stride mismatch between camera stride and that calculated by video-info */
> +		if (static_cast<unsigned int>(info.stride[0]) != stream_cfg.stride &&
> +		    GST_VIDEO_INFO_FORMAT(&info) != GST_VIDEO_FORMAT_ENCODED) {
> +			GstQuery *query = NULL;
> +			gboolean need_pool = false;
> +			stride_mismatch = true;
> +
> +			query = gst_query_new_allocation(caps, need_pool);
> +			if (!gst_pad_peer_query(srcpad, query))
> +				GST_DEBUG_OBJECT(self, "didn't get downstream ALLOCATION hints");
> +			else
> +				has_video_meta = gst_query_find_allocation_meta(query, GST_VIDEO_META_API_TYPE, NULL);
> +			gst_query_unref(query);
> +
> +			if (has_video_meta) {
> +				guint k, stride;
> +				gsize offset = 0;
> +
> +				/* this should be updated if tiled formats get added in the future. */
> +				for (k = 0; k < GST_VIDEO_INFO_N_PLANES(&info); k++) {
> +					stride = gst_video_format_info_extrapolate_stride(info.finfo, k, stream_cfg.stride);
> +					info.stride[k] = stride;
> +					info.offset[k] = offset;
> +					offset += stride * GST_VIDEO_FORMAT_INFO_SCALE_HEIGHT(info.finfo, k, GST_VIDEO_INFO_HEIGHT(&info));
> +				}
> +			}
> +		}
>  
> -		GstLibcameraPool *pool = gst_libcamera_pool_new(self->allocator,
> -								stream_cfg.stream());
> +		GstLibcameraPool *pool = gst_libcamera_pool_new(self->allocator, stream_cfg.stream(),
> +								&info, stride_mismatch, has_video_meta);
>  		g_signal_connect_swapped(pool, "buffer-notify",
>  					 G_CALLBACK(gst_task_resume), self->task);
>