[v4] gstreamer: Add GstVideoMeta support
diff mbox series

Message ID 20241120100414.219130-1-qi.hou@nxp.com
State Superseded
Headers show
Series
  • [v4] gstreamer: Add GstVideoMeta support
Related show

Commit Message

Hou Qi Nov. 20, 2024, 10:04 a.m. UTC 
GStreamer video-info calculated stride and offset may differ from
those used by the camera.

This patch enhances downstream plugin's support for videometa by
adding videometa support for libcamerasrc. It ensures that when
downstream plugin supports videometa by allocation query, libcamerasrc
also attaches videometa to buffer, preventing discrepancies
between the stride and offset calculated by video-info and camera.

Signed-off-by: Hou Qi <qi.hou@nxp.com>
---
 src/gstreamer/gstlibcamera-utils.cpp | 33 ++++++++++++++++
 src/gstreamer/gstlibcamera-utils.h   |  4 ++
 src/gstreamer/gstlibcamerapool.cpp   | 57 +++++++++++++++++++++++++++-
 src/gstreamer/gstlibcamerapool.h     |  5 ++-
 src/gstreamer/gstlibcamerasrc.cpp    | 47 +++++++++++++++++++++--
 5 files changed, 141 insertions(+), 5 deletions(-)

Comments

Nicolas Dufresne Nov. 20, 2024, 7:43 p.m. UTC | #1 
Hi,

Le mercredi 20 novembre 2024 à 19:04 +0900, Hou Qi a écrit :
> GStreamer video-info calculated stride and offset may differ from
> those used by the camera.
> 
> This patch enhances downstream plugin's support for videometa by
> adding videometa support for libcamerasrc. It ensures that when
> downstream plugin supports videometa by allocation query, libcamerasrc
> also attaches videometa to buffer, preventing discrepancies
> between the stride and offset calculated by video-info and camera.
> 
> Signed-off-by: Hou Qi <qi.hou@nxp.com>
> ---
>  src/gstreamer/gstlibcamera-utils.cpp | 33 ++++++++++++++++
>  src/gstreamer/gstlibcamera-utils.h   |  4 ++
>  src/gstreamer/gstlibcamerapool.cpp   | 57 +++++++++++++++++++++++++++-
>  src/gstreamer/gstlibcamerapool.h     |  5 ++-
>  src/gstreamer/gstlibcamerasrc.cpp    | 47 +++++++++++++++++++++--
>  5 files changed, 141 insertions(+), 5 deletions(-)
> 
> diff --git a/src/gstreamer/gstlibcamera-utils.cpp b/src/gstreamer/gstlibcamera-utils.cpp
> index 732987ef..09af9204 100644
> --- a/src/gstreamer/gstlibcamera-utils.cpp
> +++ b/src/gstreamer/gstlibcamera-utils.cpp
> @@ -591,6 +591,39 @@ gst_task_resume(GstTask *task)
>  }
>  #endif
>  
> +#if !GST_CHECK_VERSION(1, 22, 0)
> +/*
> + * Copyright (C) <1999> Erik Walthinsen <omega@cse.ogi.edu>
> + * Library       <2002> Ronald Bultje <rbultje@ronald.bitfreak.net>
> + * Copyright (C) <2007> David A. Schleef <ds@schleef.org>
> + */
> +/* This function has been imported directly from the gstreamer project to
> + * support backwards compatibility and should be removed when the older
> + * version is no longer supported. */
> +gint gst_video_format_info_extrapolate_stride(const GstVideoFormatInfo *finfo, gint plane, gint stride)
> +{
> +	gint estride;
> +	gint comp[GST_VIDEO_MAX_COMPONENTS];
> +	gint i;
> +
> +	/* there is nothing to extrapolate on first plane */
> +	if (plane == 0)
> +		return stride;
> +
> +	gst_video_format_info_component(finfo, plane, comp);
> +
> +	/* For now, all planar formats have a single component on first plane, but
> +	* if there was a planar format with more, we'd have to make a ratio of the
> +	* number of component on the first plane against the number of component on
> +	* the current plane. */
> +	estride = 0;
> +	for (i = 0; i < GST_VIDEO_MAX_COMPONENTS && comp[i] >= 0; i++)
> +		estride += GST_VIDEO_FORMAT_INFO_SCALE_WIDTH(finfo, comp[i], stride);
> +
> +	return estride;
> +}
> +#endif
> +
>  G_LOCK_DEFINE_STATIC(cm_singleton_lock);
>  static std::weak_ptr<CameraManager> cm_singleton_ptr;
>  
> diff --git a/src/gstreamer/gstlibcamera-utils.h b/src/gstreamer/gstlibcamera-utils.h
> index cab1c814..81149280 100644
> --- a/src/gstreamer/gstlibcamera-utils.h
> +++ b/src/gstreamer/gstlibcamera-utils.h
> @@ -35,6 +35,10 @@ static inline void gst_clear_event(GstEvent **event_ptr)
>  #if !GST_CHECK_VERSION(1, 17, 1)
>  gboolean gst_task_resume(GstTask *task);
>  #endif
> +
> +#if !GST_CHECK_VERSION(1, 22, 0)
> +gint gst_video_format_info_extrapolate_stride(const GstVideoFormatInfo *finfo, gint plane, gint stride);
> +#endif
>  std::shared_ptr<libcamera::CameraManager> gst_libcamera_get_camera_manager(int &ret);
>  
>  /**
> diff --git a/src/gstreamer/gstlibcamerapool.cpp b/src/gstreamer/gstlibcamerapool.cpp
> index 9cd7eccb..bd1cc473 100644
> --- a/src/gstreamer/gstlibcamerapool.cpp
> +++ b/src/gstreamer/gstlibcamerapool.cpp
> @@ -29,6 +29,8 @@ struct _GstLibcameraPool {
>  	std::deque<GstBuffer *> *queue;
>  	GstLibcameraAllocator *allocator;
>  	Stream *stream;
> +	gboolean frame_copy;
> +	GstVideoInfo info;
>  };
>  
>  G_DEFINE_TYPE(GstLibcameraPool, gst_libcamera_pool, GST_TYPE_BUFFER_POOL)
> @@ -135,16 +137,29 @@ gst_libcamera_pool_class_init(GstLibcameraPoolClass *klass)
>  }
>  
>  GstLibcameraPool *
> -gst_libcamera_pool_new(GstLibcameraAllocator *allocator, Stream *stream)
> +gst_libcamera_pool_new(GstLibcameraAllocator *allocator, Stream *stream,
> +		       GstVideoInfo *info, gboolean stride_mismatch, gboolean has_video_meta)
>  {
>  	auto *pool = GST_LIBCAMERA_POOL(g_object_new(GST_TYPE_LIBCAMERA_POOL, nullptr));
>  
>  	pool->allocator = GST_LIBCAMERA_ALLOCATOR(g_object_ref(allocator));
>  	pool->stream = stream;
> +	pool->info = *info;
> +
> +	if (stride_mismatch && !has_video_meta)
> +		pool->frame_copy = true;
> +	else
> +		pool->frame_copy = false;
>  
>  	gsize pool_size = gst_libcamera_allocator_get_pool_size(allocator, stream);
>  	for (gsize i = 0; i < pool_size; i++) {
>  		GstBuffer *buffer = gst_buffer_new();
> +		if (stride_mismatch && has_video_meta) {
> +			gst_buffer_add_video_meta_full(buffer, GST_VIDEO_FRAME_FLAG_NONE,
> +						       GST_VIDEO_INFO_FORMAT(info), GST_VIDEO_INFO_WIDTH(info),
> +						       GST_VIDEO_INFO_HEIGHT(info), GST_VIDEO_INFO_N_PLANES(info),
> +						       info->offset, info->stride);
> +		}
>  		pool->queue->push_back(buffer);
>  	}
>  
> @@ -163,3 +178,43 @@ gst_libcamera_buffer_get_frame_buffer(GstBuffer *buffer)
>  	GstMemory *mem = gst_buffer_peek_memory(buffer, 0);
>  	return gst_libcamera_memory_get_frame_buffer(mem);
>  }
> +
> +GstBuffer *
> +gst_libcamera_copy_buffer(GstLibcameraPool *self, GstBuffer *src, FrameBuffer *fb, guint32 stride)

A function that says "copy_buffer" should always copy the buffers. The naming
should be reconsidered, that being said see comment below.

> +{
> +	if (self->frame_copy) {
> +		GstVideoInfo src_info = self->info;
> +		gsize size = GST_VIDEO_INFO_SIZE(&self->info);
> +		GstBuffer *dest = gst_buffer_new_allocate(NULL, size, NULL);

The copy condition should be moved into the libcamerasrc element code. This way,
you can run an allocation query, and keep copy into downstream provided buffer
instead of doing dynamic allocation. You can also use a generic GstVideoPool to
avoid dynamic allocations even if downstream did not provide a pool.

> +		GstVideoFrame src_frame, dest_frame;
> +		int i = 0;
> +
> +		for (const FrameBuffer::Plane &plane : fb->planes()) {
> +			src_info.stride[i] = stride;
> +			src_info.offset[i] = plane.offset;
> +			i++;
> +		}
> +		src_info.size = gst_buffer_get_size(src);
> +
> +		if (!gst_video_frame_map(&src_frame, &src_info, src, GST_MAP_READ)) {
> +			GST_WARNING("fail to map src_frame");
> +			return src;
> +		}
> +
> +		if (!gst_video_frame_map(&dest_frame, &self->info, dest, GST_MAP_WRITE)) {
> +			gst_video_frame_unmap(&src_frame);
> +			GST_WARNING("fail to map dest_frame");
> +			return src;
> +		}

A map failure would indicate serious issues, we should not ignore these and
continue with the original.

> +
> +		gst_video_frame_copy(&dest_frame, &src_frame);
> +
> +		gst_video_frame_unmap(&src_frame);
> +		gst_video_frame_unmap(&dest_frame);
> +
> +		gst_buffer_unref(src);
> +		return dest;
> +	}
> +
> +	return src;
> +}
> diff --git a/src/gstreamer/gstlibcamerapool.h b/src/gstreamer/gstlibcamerapool.h
> index 2a7a9c77..ffbd3173 100644
> --- a/src/gstreamer/gstlibcamerapool.h
> +++ b/src/gstreamer/gstlibcamerapool.h
> @@ -14,6 +14,7 @@
>  #include "gstlibcameraallocator.h"
>  
>  #include <gst/gst.h>
> +#include <gst/video/video.h>
>  
>  #include <libcamera/stream.h>
>  
> @@ -21,8 +22,10 @@
>  G_DECLARE_FINAL_TYPE(GstLibcameraPool, gst_libcamera_pool, GST_LIBCAMERA, POOL, GstBufferPool)
>  
>  GstLibcameraPool *gst_libcamera_pool_new(GstLibcameraAllocator *allocator,
> -					 libcamera::Stream *stream);
> +					 libcamera::Stream *stream, GstVideoInfo *info, gboolean stride_mismatch, gboolean has_video_meta);
>  
>  libcamera::Stream *gst_libcamera_pool_get_stream(GstLibcameraPool *self);
>  
>  libcamera::FrameBuffer *gst_libcamera_buffer_get_frame_buffer(GstBuffer *buffer);
> +
> +GstBuffer *gst_libcamera_copy_buffer(GstLibcameraPool *self, GstBuffer *buffer, libcamera::FrameBuffer *fb, guint32 stride);
> diff --git a/src/gstreamer/gstlibcamerasrc.cpp b/src/gstreamer/gstlibcamerasrc.cpp
> index 8efa25f4..1d46536a 100644
> --- a/src/gstreamer/gstlibcamerasrc.cpp
> +++ b/src/gstreamer/gstlibcamerasrc.cpp
> @@ -292,12 +292,17 @@ int GstLibcameraSrcState::processRequest()
>  	GstFlowReturn ret = GST_FLOW_OK;
>  	gst_flow_combiner_reset(src_->flow_combiner);
>  
> -	for (GstPad *srcpad : srcpads_) {
> +	for (gsize i = 0; i < src_->state->srcpads_.size(); i++) {
> +		GstPad *srcpad = src_->state->srcpads_[i];
>  		Stream *stream = gst_libcamera_pad_get_stream(srcpad);
>  		GstBuffer *buffer = wrap->detachBuffer(stream);
> +		const StreamConfiguration &stream_cfg = src_->state->config_->at(i);
> +		GstLibcameraPool *pool = gst_libcamera_pad_get_pool(srcpad);
>  
>  		FrameBuffer *fb = gst_libcamera_buffer_get_frame_buffer(buffer);
>  
> +		buffer = gst_libcamera_copy_buffer(pool, buffer, fb, stream_cfg.stride);
> +
>  		if (GST_CLOCK_TIME_IS_VALID(wrap->pts_)) {
>  			GST_BUFFER_PTS(buffer) = wrap->pts_;
>  			gst_libcamera_pad_set_latency(srcpad, wrap->latency_);
> @@ -497,9 +502,45 @@ gst_libcamera_src_negotiate(GstLibcameraSrc *self)
>  	for (gsize i = 0; i < state->srcpads_.size(); i++) {
>  		GstPad *srcpad = state->srcpads_[i];
>  		const StreamConfiguration &stream_cfg = state->config_->at(i);
> +		gboolean stride_mismatch = false, has_video_meta = false;
> +		GstVideoInfo info;
> +
> +		g_autoptr(GstCaps) caps = gst_libcamera_stream_configuration_to_caps(stream_cfg);
> +		gst_libcamera_framerate_to_caps(caps, element_caps);
> +
> +		gst_video_info_init(&info);
> +		gst_video_info_from_caps(&info, caps);
> +
> +		/* stride mismatch between camera stride and that calculated by video-info */
> +		if (static_cast<unsigned int>(info.stride[0]) != stream_cfg.stride &&
> +		    GST_VIDEO_INFO_FORMAT(&info) != GST_VIDEO_FORMAT_ENCODED) {
> +			GstQuery *query = NULL;
> +			gboolean need_pool = false;
> +			stride_mismatch = true;
> +
> +			query = gst_query_new_allocation(caps, need_pool);
> +			if (!gst_pad_peer_query(srcpad, query))
> +				GST_DEBUG_OBJECT(self, "didn't get downstream ALLOCATION hints");
> +			else
> +				has_video_meta = gst_query_find_allocation_meta(query, GST_VIDEO_META_API_TYPE, NULL);
> +			gst_query_unref(query);
> +
> +			if (has_video_meta) {
> +				guint k, stride;
> +				gsize offset = 0;
> +
> +				/* this should be updated if tiled formats get added in the future. */
> +				for (k = 0; k < GST_VIDEO_INFO_N_PLANES(&info); k++) {
> +					stride = gst_video_format_info_extrapolate_stride(info.finfo, k, stream_cfg.stride);
> +					info.stride[k] = stride;
> +					info.offset[k] = offset;
> +					offset += stride * GST_VIDEO_FORMAT_INFO_SCALE_HEIGHT(info.finfo, k, GST_VIDEO_INFO_HEIGHT(&info));
> +				}
> +			}
> +		}
>  
> -		GstLibcameraPool *pool = gst_libcamera_pool_new(self->allocator,
> -								stream_cfg.stream());
> +		GstLibcameraPool *pool = gst_libcamera_pool_new(self->allocator, stream_cfg.stream(),
> +								&info, stride_mismatch, has_video_meta);
>  		g_signal_connect_swapped(pool, "buffer-notify",
>  					 G_CALLBACK(gst_task_resume), self->task);
>
Hou Qi Nov. 22, 2024, 4:25 a.m. UTC | #2 
Hi Nicolas,

I have sent out v5 according to your comment. Please help review. Thanks very much.

Regards,
Qi Hou

-----Original Message-----
From: Nicolas Dufresne <nicolas@ndufresne.ca> 
Sent: 2024年11月21日 3:43
To: Qi Hou <qi.hou@nxp.com>; libcamera-devel@lists.libcamera.org
Cc: Jared Hu <jared.hu@nxp.com>; Julien Vuillaumier <julien.vuillaumier@nxp.com>
Subject: [EXT] Re: [PATCH v4] gstreamer: Add GstVideoMeta support

Caution: This is an external email. Please take care when clicking links or opening attachments. When in doubt, report the message using the 'Report this email' button


Hi,

Le mercredi 20 novembre 2024 à 19:04 +0900, Hou Qi a écrit :
> GStreamer video-info calculated stride and offset may differ from 
> those used by the camera.
>
> This patch enhances downstream plugin's support for videometa by 
> adding videometa support for libcamerasrc. It ensures that when 
> downstream plugin supports videometa by allocation query, libcamerasrc 
> also attaches videometa to buffer, preventing discrepancies between 
> the stride and offset calculated by video-info and camera.
>
> Signed-off-by: Hou Qi <qi.hou@nxp.com>
> ---
>  src/gstreamer/gstlibcamera-utils.cpp | 33 ++++++++++++++++
>  src/gstreamer/gstlibcamera-utils.h   |  4 ++
>  src/gstreamer/gstlibcamerapool.cpp   | 57 +++++++++++++++++++++++++++-
>  src/gstreamer/gstlibcamerapool.h     |  5 ++-
>  src/gstreamer/gstlibcamerasrc.cpp    | 47 +++++++++++++++++++++--
>  5 files changed, 141 insertions(+), 5 deletions(-)
>
> diff --git a/src/gstreamer/gstlibcamera-utils.cpp 
> b/src/gstreamer/gstlibcamera-utils.cpp
> index 732987ef..09af9204 100644
> --- a/src/gstreamer/gstlibcamera-utils.cpp
> +++ b/src/gstreamer/gstlibcamera-utils.cpp
> @@ -591,6 +591,39 @@ gst_task_resume(GstTask *task)  }  #endif
>
> +#if !GST_CHECK_VERSION(1, 22, 0)
> +/*
> + * Copyright (C) <1999> Erik Walthinsen <omega@cse.ogi.edu>
> + * Library       <2002> Ronald Bultje <rbultje@ronald.bitfreak.net>
> + * Copyright (C) <2007> David A. Schleef <ds@schleef.org>  */
> +/* This function has been imported directly from the gstreamer 
> +project to
> + * support backwards compatibility and should be removed when the 
> +older
> + * version is no longer supported. */ gint 
> +gst_video_format_info_extrapolate_stride(const GstVideoFormatInfo 
> +*finfo, gint plane, gint stride) {
> +     gint estride;
> +     gint comp[GST_VIDEO_MAX_COMPONENTS];
> +     gint i;
> +
> +     /* there is nothing to extrapolate on first plane */
> +     if (plane == 0)
> +             return stride;
> +
> +     gst_video_format_info_component(finfo, plane, comp);
> +
> +     /* For now, all planar formats have a single component on first plane, but
> +     * if there was a planar format with more, we'd have to make a ratio of the
> +     * number of component on the first plane against the number of component on
> +     * the current plane. */
> +     estride = 0;
> +     for (i = 0; i < GST_VIDEO_MAX_COMPONENTS && comp[i] >= 0; i++)
> +             estride += GST_VIDEO_FORMAT_INFO_SCALE_WIDTH(finfo, 
> + comp[i], stride);
> +
> +     return estride;
> +}
> +#endif
> +
>  G_LOCK_DEFINE_STATIC(cm_singleton_lock);
>  static std::weak_ptr<CameraManager> cm_singleton_ptr;
>
> diff --git a/src/gstreamer/gstlibcamera-utils.h 
> b/src/gstreamer/gstlibcamera-utils.h
> index cab1c814..81149280 100644
> --- a/src/gstreamer/gstlibcamera-utils.h
> +++ b/src/gstreamer/gstlibcamera-utils.h
> @@ -35,6 +35,10 @@ static inline void gst_clear_event(GstEvent 
> **event_ptr)  #if !GST_CHECK_VERSION(1, 17, 1)  gboolean 
> gst_task_resume(GstTask *task);  #endif
> +
> +#if !GST_CHECK_VERSION(1, 22, 0)
> +gint gst_video_format_info_extrapolate_stride(const 
> +GstVideoFormatInfo *finfo, gint plane, gint stride); #endif
>  std::shared_ptr<libcamera::CameraManager> 
> gst_libcamera_get_camera_manager(int &ret);
>
>  /**
> diff --git a/src/gstreamer/gstlibcamerapool.cpp 
> b/src/gstreamer/gstlibcamerapool.cpp
> index 9cd7eccb..bd1cc473 100644
> --- a/src/gstreamer/gstlibcamerapool.cpp
> +++ b/src/gstreamer/gstlibcamerapool.cpp
> @@ -29,6 +29,8 @@ struct _GstLibcameraPool {
>       std::deque<GstBuffer *> *queue;
>       GstLibcameraAllocator *allocator;
>       Stream *stream;
> +     gboolean frame_copy;
> +     GstVideoInfo info;
>  };
>
>  G_DEFINE_TYPE(GstLibcameraPool, gst_libcamera_pool, 
> GST_TYPE_BUFFER_POOL) @@ -135,16 +137,29 @@ 
> gst_libcamera_pool_class_init(GstLibcameraPoolClass *klass)  }
>
>  GstLibcameraPool *
> -gst_libcamera_pool_new(GstLibcameraAllocator *allocator, Stream 
> *stream)
> +gst_libcamera_pool_new(GstLibcameraAllocator *allocator, Stream *stream,
> +                    GstVideoInfo *info, gboolean stride_mismatch, 
> +gboolean has_video_meta)
>  {
>       auto *pool = 
> GST_LIBCAMERA_POOL(g_object_new(GST_TYPE_LIBCAMERA_POOL, nullptr));
>
>       pool->allocator = GST_LIBCAMERA_ALLOCATOR(g_object_ref(allocator));
>       pool->stream = stream;
> +     pool->info = *info;
> +
> +     if (stride_mismatch && !has_video_meta)
> +             pool->frame_copy = true;
> +     else
> +             pool->frame_copy = false;
>
>       gsize pool_size = gst_libcamera_allocator_get_pool_size(allocator, stream);
>       for (gsize i = 0; i < pool_size; i++) {
>               GstBuffer *buffer = gst_buffer_new();
> +             if (stride_mismatch && has_video_meta) {
> +                     gst_buffer_add_video_meta_full(buffer, GST_VIDEO_FRAME_FLAG_NONE,
> +                                                    GST_VIDEO_INFO_FORMAT(info), GST_VIDEO_INFO_WIDTH(info),
> +                                                    GST_VIDEO_INFO_HEIGHT(info), GST_VIDEO_INFO_N_PLANES(info),
> +                                                    info->offset, info->stride);
> +             }
>               pool->queue->push_back(buffer);
>       }
>
> @@ -163,3 +178,43 @@ gst_libcamera_buffer_get_frame_buffer(GstBuffer *buffer)
>       GstMemory *mem = gst_buffer_peek_memory(buffer, 0);
>       return gst_libcamera_memory_get_frame_buffer(mem);
>  }
> +
> +GstBuffer *
> +gst_libcamera_copy_buffer(GstLibcameraPool *self, GstBuffer *src, 
> +FrameBuffer *fb, guint32 stride)

A function that says "copy_buffer" should always copy the buffers. The naming should be reconsidered, that being said see comment below.

> +{
> +     if (self->frame_copy) {
> +             GstVideoInfo src_info = self->info;
> +             gsize size = GST_VIDEO_INFO_SIZE(&self->info);
> +             GstBuffer *dest = gst_buffer_new_allocate(NULL, size, 
> +NULL);

The copy condition should be moved into the libcamerasrc element code. This way, you can run an allocation query, and keep copy into downstream provided buffer instead of doing dynamic allocation. You can also use a generic GstVideoPool to avoid dynamic allocations even if downstream did not provide a pool.

> +             GstVideoFrame src_frame, dest_frame;
> +             int i = 0;
> +
> +             for (const FrameBuffer::Plane &plane : fb->planes()) {
> +                     src_info.stride[i] = stride;
> +                     src_info.offset[i] = plane.offset;
> +                     i++;
> +             }
> +             src_info.size = gst_buffer_get_size(src);
> +
> +             if (!gst_video_frame_map(&src_frame, &src_info, src, GST_MAP_READ)) {
> +                     GST_WARNING("fail to map src_frame");
> +                     return src;
> +             }
> +
> +             if (!gst_video_frame_map(&dest_frame, &self->info, dest, GST_MAP_WRITE)) {
> +                     gst_video_frame_unmap(&src_frame);
> +                     GST_WARNING("fail to map dest_frame");
> +                     return src;
> +             }

A map failure would indicate serious issues, we should not ignore these and continue with the original.

> +
> +             gst_video_frame_copy(&dest_frame, &src_frame);
> +
> +             gst_video_frame_unmap(&src_frame);
> +             gst_video_frame_unmap(&dest_frame);
> +
> +             gst_buffer_unref(src);
> +             return dest;
> +     }
> +
> +     return src;
> +}
> diff --git a/src/gstreamer/gstlibcamerapool.h 
> b/src/gstreamer/gstlibcamerapool.h
> index 2a7a9c77..ffbd3173 100644
> --- a/src/gstreamer/gstlibcamerapool.h
> +++ b/src/gstreamer/gstlibcamerapool.h
> @@ -14,6 +14,7 @@
>  #include "gstlibcameraallocator.h"
>
>  #include <gst/gst.h>
> +#include <gst/video/video.h>
>
>  #include <libcamera/stream.h>
>
> @@ -21,8 +22,10 @@
>  G_DECLARE_FINAL_TYPE(GstLibcameraPool, gst_libcamera_pool, 
> GST_LIBCAMERA, POOL, GstBufferPool)
>
>  GstLibcameraPool *gst_libcamera_pool_new(GstLibcameraAllocator *allocator,
> -                                      libcamera::Stream *stream);
> +                                      libcamera::Stream *stream, 
> + GstVideoInfo *info, gboolean stride_mismatch, gboolean 
> + has_video_meta);
>
>  libcamera::Stream *gst_libcamera_pool_get_stream(GstLibcameraPool 
> *self);
>
>  libcamera::FrameBuffer 
> *gst_libcamera_buffer_get_frame_buffer(GstBuffer *buffer);
> +
> +GstBuffer *gst_libcamera_copy_buffer(GstLibcameraPool *self, 
> +GstBuffer *buffer, libcamera::FrameBuffer *fb, guint32 stride);
> diff --git a/src/gstreamer/gstlibcamerasrc.cpp 
> b/src/gstreamer/gstlibcamerasrc.cpp
> index 8efa25f4..1d46536a 100644
> --- a/src/gstreamer/gstlibcamerasrc.cpp
> +++ b/src/gstreamer/gstlibcamerasrc.cpp
> @@ -292,12 +292,17 @@ int GstLibcameraSrcState::processRequest()
>       GstFlowReturn ret = GST_FLOW_OK;
>       gst_flow_combiner_reset(src_->flow_combiner);
>
> -     for (GstPad *srcpad : srcpads_) {
> +     for (gsize i = 0; i < src_->state->srcpads_.size(); i++) {
> +             GstPad *srcpad = src_->state->srcpads_[i];
>               Stream *stream = gst_libcamera_pad_get_stream(srcpad);
>               GstBuffer *buffer = wrap->detachBuffer(stream);
> +             const StreamConfiguration &stream_cfg = src_->state->config_->at(i);
> +             GstLibcameraPool *pool = 
> + gst_libcamera_pad_get_pool(srcpad);
>
>               FrameBuffer *fb = 
> gst_libcamera_buffer_get_frame_buffer(buffer);
>
> +             buffer = gst_libcamera_copy_buffer(pool, buffer, fb, 
> + stream_cfg.stride);
> +
>               if (GST_CLOCK_TIME_IS_VALID(wrap->pts_)) {
>                       GST_BUFFER_PTS(buffer) = wrap->pts_;
>                       gst_libcamera_pad_set_latency(srcpad, 
> wrap->latency_); @@ -497,9 +502,45 @@ gst_libcamera_src_negotiate(GstLibcameraSrc *self)
>       for (gsize i = 0; i < state->srcpads_.size(); i++) {
>               GstPad *srcpad = state->srcpads_[i];
>               const StreamConfiguration &stream_cfg = 
> state->config_->at(i);
> +             gboolean stride_mismatch = false, has_video_meta = false;
> +             GstVideoInfo info;
> +
> +             g_autoptr(GstCaps) caps = gst_libcamera_stream_configuration_to_caps(stream_cfg);
> +             gst_libcamera_framerate_to_caps(caps, element_caps);
> +
> +             gst_video_info_init(&info);
> +             gst_video_info_from_caps(&info, caps);
> +
> +             /* stride mismatch between camera stride and that calculated by video-info */
> +             if (static_cast<unsigned int>(info.stride[0]) != stream_cfg.stride &&
> +                 GST_VIDEO_INFO_FORMAT(&info) != GST_VIDEO_FORMAT_ENCODED) {
> +                     GstQuery *query = NULL;
> +                     gboolean need_pool = false;
> +                     stride_mismatch = true;
> +
> +                     query = gst_query_new_allocation(caps, need_pool);
> +                     if (!gst_pad_peer_query(srcpad, query))
> +                             GST_DEBUG_OBJECT(self, "didn't get downstream ALLOCATION hints");
> +                     else
> +                             has_video_meta = gst_query_find_allocation_meta(query, GST_VIDEO_META_API_TYPE, NULL);
> +                     gst_query_unref(query);
> +
> +                     if (has_video_meta) {
> +                             guint k, stride;
> +                             gsize offset = 0;
> +
> +                             /* this should be updated if tiled formats get added in the future. */
> +                             for (k = 0; k < GST_VIDEO_INFO_N_PLANES(&info); k++) {
> +                                     stride = gst_video_format_info_extrapolate_stride(info.finfo, k, stream_cfg.stride);
> +                                     info.stride[k] = stride;
> +                                     info.offset[k] = offset;
> +                                     offset += stride * GST_VIDEO_FORMAT_INFO_SCALE_HEIGHT(info.finfo, k, GST_VIDEO_INFO_HEIGHT(&info));
> +                             }
> +                     }
> +             }
>
> -             GstLibcameraPool *pool = gst_libcamera_pool_new(self->allocator,
> -                                                             stream_cfg.stream());
> +             GstLibcameraPool *pool = gst_libcamera_pool_new(self->allocator, stream_cfg.stream(),
> +                                                             &info, 
> + stride_mismatch, has_video_meta);
>               g_signal_connect_swapped(pool, "buffer-notify",
>                                        G_CALLBACK(gst_task_resume), 
> self->task);
>

Patch
diff mbox series 

diff --git a/src/gstreamer/gstlibcamera-utils.cpp b/src/gstreamer/gstlibcamera-utils.cpp
index 732987ef..09af9204 100644
--- a/src/gstreamer/gstlibcamera-utils.cpp
+++ b/src/gstreamer/gstlibcamera-utils.cpp
@@ -591,6 +591,39 @@  gst_task_resume(GstTask *task)
 }
 #endif
 
+#if !GST_CHECK_VERSION(1, 22, 0)
+/*
+ * Copyright (C) <1999> Erik Walthinsen <omega@cse.ogi.edu>
+ * Library       <2002> Ronald Bultje <rbultje@ronald.bitfreak.net>
+ * Copyright (C) <2007> David A. Schleef <ds@schleef.org>
+ */
+/* This function has been imported directly from the gstreamer project to
+ * support backwards compatibility and should be removed when the older
+ * version is no longer supported. */
+gint gst_video_format_info_extrapolate_stride(const GstVideoFormatInfo *finfo, gint plane, gint stride)
+{
+	gint estride;
+	gint comp[GST_VIDEO_MAX_COMPONENTS];
+	gint i;
+
+	/* there is nothing to extrapolate on first plane */
+	if (plane == 0)
+		return stride;
+
+	gst_video_format_info_component(finfo, plane, comp);
+
+	/* For now, all planar formats have a single component on first plane, but
+	* if there was a planar format with more, we'd have to make a ratio of the
+	* number of component on the first plane against the number of component on
+	* the current plane. */
+	estride = 0;
+	for (i = 0; i < GST_VIDEO_MAX_COMPONENTS && comp[i] >= 0; i++)
+		estride += GST_VIDEO_FORMAT_INFO_SCALE_WIDTH(finfo, comp[i], stride);
+
+	return estride;
+}
+#endif
+
 G_LOCK_DEFINE_STATIC(cm_singleton_lock);
 static std::weak_ptr<CameraManager> cm_singleton_ptr;
 
diff --git a/src/gstreamer/gstlibcamera-utils.h b/src/gstreamer/gstlibcamera-utils.h
index cab1c814..81149280 100644
--- a/src/gstreamer/gstlibcamera-utils.h
+++ b/src/gstreamer/gstlibcamera-utils.h
@@ -35,6 +35,10 @@  static inline void gst_clear_event(GstEvent **event_ptr)
 #if !GST_CHECK_VERSION(1, 17, 1)
 gboolean gst_task_resume(GstTask *task);
 #endif
+
+#if !GST_CHECK_VERSION(1, 22, 0)
+gint gst_video_format_info_extrapolate_stride(const GstVideoFormatInfo *finfo, gint plane, gint stride);
+#endif
 std::shared_ptr<libcamera::CameraManager> gst_libcamera_get_camera_manager(int &ret);
 
 /**
diff --git a/src/gstreamer/gstlibcamerapool.cpp b/src/gstreamer/gstlibcamerapool.cpp
index 9cd7eccb..bd1cc473 100644
--- a/src/gstreamer/gstlibcamerapool.cpp
+++ b/src/gstreamer/gstlibcamerapool.cpp
@@ -29,6 +29,8 @@  struct _GstLibcameraPool {
 	std::deque<GstBuffer *> *queue;
 	GstLibcameraAllocator *allocator;
 	Stream *stream;
+	gboolean frame_copy;
+	GstVideoInfo info;
 };
 
 G_DEFINE_TYPE(GstLibcameraPool, gst_libcamera_pool, GST_TYPE_BUFFER_POOL)
@@ -135,16 +137,29 @@  gst_libcamera_pool_class_init(GstLibcameraPoolClass *klass)
 }
 
 GstLibcameraPool *
-gst_libcamera_pool_new(GstLibcameraAllocator *allocator, Stream *stream)
+gst_libcamera_pool_new(GstLibcameraAllocator *allocator, Stream *stream,
+		       GstVideoInfo *info, gboolean stride_mismatch, gboolean has_video_meta)
 {
 	auto *pool = GST_LIBCAMERA_POOL(g_object_new(GST_TYPE_LIBCAMERA_POOL, nullptr));
 
 	pool->allocator = GST_LIBCAMERA_ALLOCATOR(g_object_ref(allocator));
 	pool->stream = stream;
+	pool->info = *info;
+
+	if (stride_mismatch && !has_video_meta)
+		pool->frame_copy = true;
+	else
+		pool->frame_copy = false;
 
 	gsize pool_size = gst_libcamera_allocator_get_pool_size(allocator, stream);
 	for (gsize i = 0; i < pool_size; i++) {
 		GstBuffer *buffer = gst_buffer_new();
+		if (stride_mismatch && has_video_meta) {
+			gst_buffer_add_video_meta_full(buffer, GST_VIDEO_FRAME_FLAG_NONE,
+						       GST_VIDEO_INFO_FORMAT(info), GST_VIDEO_INFO_WIDTH(info),
+						       GST_VIDEO_INFO_HEIGHT(info), GST_VIDEO_INFO_N_PLANES(info),
+						       info->offset, info->stride);
+		}
 		pool->queue->push_back(buffer);
 	}
 
@@ -163,3 +178,43 @@  gst_libcamera_buffer_get_frame_buffer(GstBuffer *buffer)
 	GstMemory *mem = gst_buffer_peek_memory(buffer, 0);
 	return gst_libcamera_memory_get_frame_buffer(mem);
 }
+
+GstBuffer *
+gst_libcamera_copy_buffer(GstLibcameraPool *self, GstBuffer *src, FrameBuffer *fb, guint32 stride)
+{
+	if (self->frame_copy) {
+		GstVideoInfo src_info = self->info;
+		gsize size = GST_VIDEO_INFO_SIZE(&self->info);
+		GstBuffer *dest = gst_buffer_new_allocate(NULL, size, NULL);
+		GstVideoFrame src_frame, dest_frame;
+		int i = 0;
+
+		for (const FrameBuffer::Plane &plane : fb->planes()) {
+			src_info.stride[i] = stride;
+			src_info.offset[i] = plane.offset;
+			i++;
+		}
+		src_info.size = gst_buffer_get_size(src);
+
+		if (!gst_video_frame_map(&src_frame, &src_info, src, GST_MAP_READ)) {
+			GST_WARNING("fail to map src_frame");
+			return src;
+		}
+
+		if (!gst_video_frame_map(&dest_frame, &self->info, dest, GST_MAP_WRITE)) {
+			gst_video_frame_unmap(&src_frame);
+			GST_WARNING("fail to map dest_frame");
+			return src;
+		}
+
+		gst_video_frame_copy(&dest_frame, &src_frame);
+
+		gst_video_frame_unmap(&src_frame);
+		gst_video_frame_unmap(&dest_frame);
+
+		gst_buffer_unref(src);
+		return dest;
+	}
+
+	return src;
+}
diff --git a/src/gstreamer/gstlibcamerapool.h b/src/gstreamer/gstlibcamerapool.h
index 2a7a9c77..ffbd3173 100644
--- a/src/gstreamer/gstlibcamerapool.h
+++ b/src/gstreamer/gstlibcamerapool.h
@@ -14,6 +14,7 @@ 
 #include "gstlibcameraallocator.h"
 
 #include <gst/gst.h>
+#include <gst/video/video.h>
 
 #include <libcamera/stream.h>
 
@@ -21,8 +22,10 @@ 
 G_DECLARE_FINAL_TYPE(GstLibcameraPool, gst_libcamera_pool, GST_LIBCAMERA, POOL, GstBufferPool)
 
 GstLibcameraPool *gst_libcamera_pool_new(GstLibcameraAllocator *allocator,
-					 libcamera::Stream *stream);
+					 libcamera::Stream *stream, GstVideoInfo *info, gboolean stride_mismatch, gboolean has_video_meta);
 
 libcamera::Stream *gst_libcamera_pool_get_stream(GstLibcameraPool *self);
 
 libcamera::FrameBuffer *gst_libcamera_buffer_get_frame_buffer(GstBuffer *buffer);
+
+GstBuffer *gst_libcamera_copy_buffer(GstLibcameraPool *self, GstBuffer *buffer, libcamera::FrameBuffer *fb, guint32 stride);
diff --git a/src/gstreamer/gstlibcamerasrc.cpp b/src/gstreamer/gstlibcamerasrc.cpp
index 8efa25f4..1d46536a 100644
--- a/src/gstreamer/gstlibcamerasrc.cpp
+++ b/src/gstreamer/gstlibcamerasrc.cpp
@@ -292,12 +292,17 @@  int GstLibcameraSrcState::processRequest()
 	GstFlowReturn ret = GST_FLOW_OK;
 	gst_flow_combiner_reset(src_->flow_combiner);
 
-	for (GstPad *srcpad : srcpads_) {
+	for (gsize i = 0; i < src_->state->srcpads_.size(); i++) {
+		GstPad *srcpad = src_->state->srcpads_[i];
 		Stream *stream = gst_libcamera_pad_get_stream(srcpad);
 		GstBuffer *buffer = wrap->detachBuffer(stream);
+		const StreamConfiguration &stream_cfg = src_->state->config_->at(i);
+		GstLibcameraPool *pool = gst_libcamera_pad_get_pool(srcpad);
 
 		FrameBuffer *fb = gst_libcamera_buffer_get_frame_buffer(buffer);
 
+		buffer = gst_libcamera_copy_buffer(pool, buffer, fb, stream_cfg.stride);
+
 		if (GST_CLOCK_TIME_IS_VALID(wrap->pts_)) {
 			GST_BUFFER_PTS(buffer) = wrap->pts_;
 			gst_libcamera_pad_set_latency(srcpad, wrap->latency_);
@@ -497,9 +502,45 @@  gst_libcamera_src_negotiate(GstLibcameraSrc *self)
 	for (gsize i = 0; i < state->srcpads_.size(); i++) {
 		GstPad *srcpad = state->srcpads_[i];
 		const StreamConfiguration &stream_cfg = state->config_->at(i);
+		gboolean stride_mismatch = false, has_video_meta = false;
+		GstVideoInfo info;
+
+		g_autoptr(GstCaps) caps = gst_libcamera_stream_configuration_to_caps(stream_cfg);
+		gst_libcamera_framerate_to_caps(caps, element_caps);
+
+		gst_video_info_init(&info);
+		gst_video_info_from_caps(&info, caps);
+
+		/* stride mismatch between camera stride and that calculated by video-info */
+		if (static_cast<unsigned int>(info.stride[0]) != stream_cfg.stride &&
+		    GST_VIDEO_INFO_FORMAT(&info) != GST_VIDEO_FORMAT_ENCODED) {
+			GstQuery *query = NULL;
+			gboolean need_pool = false;
+			stride_mismatch = true;
+
+			query = gst_query_new_allocation(caps, need_pool);
+			if (!gst_pad_peer_query(srcpad, query))
+				GST_DEBUG_OBJECT(self, "didn't get downstream ALLOCATION hints");
+			else
+				has_video_meta = gst_query_find_allocation_meta(query, GST_VIDEO_META_API_TYPE, NULL);
+			gst_query_unref(query);
+
+			if (has_video_meta) {
+				guint k, stride;
+				gsize offset = 0;
+
+				/* this should be updated if tiled formats get added in the future. */
+				for (k = 0; k < GST_VIDEO_INFO_N_PLANES(&info); k++) {
+					stride = gst_video_format_info_extrapolate_stride(info.finfo, k, stream_cfg.stride);
+					info.stride[k] = stride;
+					info.offset[k] = offset;
+					offset += stride * GST_VIDEO_FORMAT_INFO_SCALE_HEIGHT(info.finfo, k, GST_VIDEO_INFO_HEIGHT(&info));
+				}
+			}
+		}
 
-		GstLibcameraPool *pool = gst_libcamera_pool_new(self->allocator,
-								stream_cfg.stream());
+		GstLibcameraPool *pool = gst_libcamera_pool_new(self->allocator, stream_cfg.stream(),
+								&info, stride_mismatch, has_video_meta);
 		g_signal_connect_swapped(pool, "buffer-notify",
 					 G_CALLBACK(gst_task_resume), self->task);