[{"id":34271,"web_url":"https://patchwork.libcamera.org/comment/34271/","msgid":"<174764685801.40401.111016603272465895@ping.linuxembedded.co.uk>","date":"2025-05-19T09:27:38","subject":"Re: [PATCH v9] gstreamer: Add GstVideoMeta support","submitter":{"id":4,"url":"https://patchwork.libcamera.org/api/people/4/","name":"Kieran Bingham","email":"kieran.bingham@ideasonboard.com"},"content":"Quoting Hou Qi (2025-05-19 08:52:10)\n> GStreamer video-info calculated stride and offset may differ from\n> those used by the camera.\n> \n> For stride and offset mismatch, this patch adds video meta to buffer\n> if downstream supports VideoMeta through allocation query. Otherwise,\n> create a internal VideoPool using the caps, and copy video frame to\n> this system memory.\n> \n> Signed-off-by: Hou Qi \n> Reviewed-by: Kieran Bingham \n> Tested-by: Nicolas Dufresne \n> Reviewed-by: Nicolas Dufresne \n\nThanks, CI is now passed at https://gitlab.freedesktop.org/camera/libcamera/-/pipelines/1427421\n\nMerging.\n\n\n> ---\n> src/gstreamer/gstlibcamera-utils.cpp | 37 +++++++\n> src/gstreamer/gstlibcamera-utils.h | 5 +\n> src/gstreamer/gstlibcamerapad.cpp | 31 ++++++\n> src/gstreamer/gstlibcamerapad.h | 8 ++\n> src/gstreamer/gstlibcamerapool.cpp | 15 ++-\n> src/gstreamer/gstlibcamerapool.h | 3 +-\n> src/gstreamer/gstlibcamerasrc.cpp | 146 ++++++++++++++++++++++++++-\n> 7 files changed, 241 insertions(+), 4 deletions(-)\n> \n> diff --git a/src/gstreamer/gstlibcamera-utils.cpp b/src/gstreamer/gstlibcamera-utils.cpp\n> index 2edebba0..a548b0c1 100644\n> --- a/src/gstreamer/gstlibcamera-utils.cpp\n> +++ b/src/gstreamer/gstlibcamera-utils.cpp\n> @@ -599,6 +599,43 @@ gst_task_resume(GstTask *task)\n> }\n> #endif\n> \n> +#if !GST_CHECK_VERSION(1, 22, 0)\n> +/*\n> + * Copyright (C) <1999> Erik Walthinsen \n> + * Library <2002> Ronald Bultje \n> + * Copyright (C) <2007> David A. Schleef \n> + */\n> +/*\n> + * This function has been imported directly from the gstreamer project to\n> + * support backwards compatibility and should be removed when the older version\n> + * is no longer supported.\n> + */\n> +gint gst_video_format_info_extrapolate_stride(const GstVideoFormatInfo *finfo, gint plane, gint stride)\n> +{\n> + gint estride;\n> + gint comp[GST_VIDEO_MAX_COMPONENTS];\n> + gint i;\n> +\n> + /* There is nothing to extrapolate on first plane. */\n> + if (plane == 0)\n> + return stride;\n> +\n> + gst_video_format_info_component(finfo, plane, comp);\n> +\n> + /*\n> + * For now, all planar formats have a single component on first plane, but\n> + * if there was a planar format with more, we'd have to make a ratio of the\n> + * number of component on the first plane against the number of component on\n> + * the current plane.\n> + */\n> + estride = 0;\n> + for (i = 0; i < GST_VIDEO_MAX_COMPONENTS && comp[i] >= 0; i++)\n> + estride += GST_VIDEO_FORMAT_INFO_SCALE_WIDTH(finfo, comp[i], stride);\n> +\n> + return estride;\n> +}\n> +#endif\n> +\n> G_LOCK_DEFINE_STATIC(cm_singleton_lock);\n> static std::weak_ptr cm_singleton_ptr;\n> \n> diff --git a/src/gstreamer/gstlibcamera-utils.h b/src/gstreamer/gstlibcamera-utils.h\n> index 4978987c..5f4e8a0f 100644\n> --- a/src/gstreamer/gstlibcamera-utils.h\n> +++ b/src/gstreamer/gstlibcamera-utils.h\n> @@ -36,6 +36,11 @@ static inline void gst_clear_event(GstEvent **event_ptr)\n> #if !GST_CHECK_VERSION(1, 17, 1)\n> gboolean gst_task_resume(GstTask *task);\n> #endif\n> +\n> +#if !GST_CHECK_VERSION(1, 22, 0)\n> +gint gst_video_format_info_extrapolate_stride(const GstVideoFormatInfo *finfo, gint plane, gint stride);\n> +#endif\n> +\n> std::shared_ptr gst_libcamera_get_camera_manager(int &ret);\n> \n> /**\n> diff --git a/src/gstreamer/gstlibcamerapad.cpp b/src/gstreamer/gstlibcamerapad.cpp\n> index 7b22aebe..3bc2bc87 100644\n> --- a/src/gstreamer/gstlibcamerapad.cpp\n> +++ b/src/gstreamer/gstlibcamerapad.cpp\n> @@ -18,6 +18,8 @@ struct _GstLibcameraPad {\n> GstPad parent;\n> StreamRole role;\n> GstLibcameraPool *pool;\n> + GstBufferPool *video_pool;\n> + GstVideoInfo info;\n> GstClockTime latency;\n> };\n> \n> @@ -153,6 +155,35 @@ gst_libcamera_pad_set_pool(GstPad *pad, GstLibcameraPool *pool)\n> self->pool = pool;\n> }\n> \n> +GstBufferPool *\n> +gst_libcamera_pad_get_video_pool(GstPad *pad)\n> +{\n> + auto *self = GST_LIBCAMERA_PAD(pad);\n> + return self->video_pool;\n> +}\n> +\n> +void gst_libcamera_pad_set_video_pool(GstPad *pad, GstBufferPool *video_pool)\n> +{\n> + auto *self = GST_LIBCAMERA_PAD(pad);\n> +\n> + if (self->video_pool)\n> + g_object_unref(self->video_pool);\n> + self->video_pool = video_pool;\n> +}\n> +\n> +GstVideoInfo gst_libcamera_pad_get_video_info(GstPad *pad)\n> +{\n> + auto *self = GST_LIBCAMERA_PAD(pad);\n> + return self->info;\n> +}\n> +\n> +void gst_libcamera_pad_set_video_info(GstPad *pad, const GstVideoInfo *info)\n> +{\n> + auto *self = GST_LIBCAMERA_PAD(pad);\n> +\n> + self->info = *info;\n> +}\n> +\n> Stream *\n> gst_libcamera_pad_get_stream(GstPad *pad)\n> {\n> diff --git a/src/gstreamer/gstlibcamerapad.h b/src/gstreamer/gstlibcamerapad.h\n> index 630c168a..f98b8a7f 100644\n> --- a/src/gstreamer/gstlibcamerapad.h\n> +++ b/src/gstreamer/gstlibcamerapad.h\n> @@ -23,6 +23,14 @@ GstLibcameraPool *gst_libcamera_pad_get_pool(GstPad *pad);\n> \n> void gst_libcamera_pad_set_pool(GstPad *pad, GstLibcameraPool *pool);\n> \n> +GstBufferPool *gst_libcamera_pad_get_video_pool(GstPad *pad);\n> +\n> +void gst_libcamera_pad_set_video_pool(GstPad *pad, GstBufferPool *video_pool);\n> +\n> +GstVideoInfo gst_libcamera_pad_get_video_info(GstPad *pad);\n> +\n> +void gst_libcamera_pad_set_video_info(GstPad *pad, const GstVideoInfo *info);\n> +\n> libcamera::Stream *gst_libcamera_pad_get_stream(GstPad *pad);\n> \n> void gst_libcamera_pad_set_latency(GstPad *pad, GstClockTime latency);\n> diff --git a/src/gstreamer/gstlibcamerapool.cpp b/src/gstreamer/gstlibcamerapool.cpp\n> index 9cd7eccb..8278144f 100644\n> --- a/src/gstreamer/gstlibcamerapool.cpp\n> +++ b/src/gstreamer/gstlibcamerapool.cpp\n> @@ -134,8 +134,20 @@ gst_libcamera_pool_class_init(GstLibcameraPoolClass *klass)\n> G_TYPE_NONE, 0);\n> }\n> \n> +static void\n> +gst_libcamera_buffer_add_video_meta(GstBuffer *buffer, GstVideoInfo *info)\n> +{\n> + GstVideoMeta *vmeta;\n> + vmeta = gst_buffer_add_video_meta_full(buffer, GST_VIDEO_FRAME_FLAG_NONE,\n> + GST_VIDEO_INFO_FORMAT(info), GST_VIDEO_INFO_WIDTH(info),\n> + GST_VIDEO_INFO_HEIGHT(info), GST_VIDEO_INFO_N_PLANES(info),\n> + info->offset, info->stride);\n> + GST_META_FLAGS(vmeta) = (GstMetaFlags)(GST_META_FLAGS(vmeta) | GST_META_FLAG_POOLED);\n> +}\n> +\n> GstLibcameraPool *\n> -gst_libcamera_pool_new(GstLibcameraAllocator *allocator, Stream *stream)\n> +gst_libcamera_pool_new(GstLibcameraAllocator *allocator, Stream *stream,\n> + GstVideoInfo *info)\n> {\n> auto *pool = GST_LIBCAMERA_POOL(g_object_new(GST_TYPE_LIBCAMERA_POOL, nullptr));\n> \n> @@ -145,6 +157,7 @@ gst_libcamera_pool_new(GstLibcameraAllocator *allocator, Stream *stream)\n> gsize pool_size = gst_libcamera_allocator_get_pool_size(allocator, stream);\n> for (gsize i = 0; i < pool_size; i++) {\n> GstBuffer *buffer = gst_buffer_new();\n> + gst_libcamera_buffer_add_video_meta(buffer, info);\n> pool->queue->push_back(buffer);\n> }\n> \n> diff --git a/src/gstreamer/gstlibcamerapool.h b/src/gstreamer/gstlibcamerapool.h\n> index 2a7a9c77..02ee4dd4 100644\n> --- a/src/gstreamer/gstlibcamerapool.h\n> +++ b/src/gstreamer/gstlibcamerapool.h\n> @@ -14,6 +14,7 @@\n> #include \"gstlibcameraallocator.h\"\n> \n> #include \n> +#include \n> \n> #include \n> \n> @@ -21,7 +22,7 @@\n> G_DECLARE_FINAL_TYPE(GstLibcameraPool, gst_libcamera_pool, GST_LIBCAMERA, POOL, GstBufferPool)\n> \n> GstLibcameraPool *gst_libcamera_pool_new(GstLibcameraAllocator *allocator,\n> - libcamera::Stream *stream);\n> + libcamera::Stream *stream, GstVideoInfo *info);\n> \n> libcamera::Stream *gst_libcamera_pool_get_stream(GstLibcameraPool *self);\n> \n> diff --git a/src/gstreamer/gstlibcamerasrc.cpp b/src/gstreamer/gstlibcamerasrc.cpp\n> index 5e9e843d..b34f0897 100644\n> --- a/src/gstreamer/gstlibcamerasrc.cpp\n> +++ b/src/gstreamer/gstlibcamerasrc.cpp\n> @@ -268,6 +268,55 @@ GstLibcameraSrcState::requestCompleted(Request *request)\n> gst_task_resume(src_->task);\n> }\n> \n> +static void\n> +gst_libcamera_extrapolate_info(GstVideoInfo *info, guint32 stride)\n> +{\n> + guint i, estride;\n> + gsize offset = 0;\n> +\n> + /* This should be updated if tiled formats get added in the future. */\n> + for (i = 0; i < GST_VIDEO_INFO_N_PLANES(info); i++) {\n> + estride = gst_video_format_info_extrapolate_stride(info->finfo, i, stride);\n> + info->stride[i] = estride;\n> + info->offset[i] = offset;\n> + offset += estride * GST_VIDEO_FORMAT_INFO_SCALE_HEIGHT(info->finfo, i,\n> + GST_VIDEO_INFO_HEIGHT(info));\n> + }\n> +}\n> +\n> +static GstFlowReturn\n> +gst_libcamera_video_frame_copy(GstBuffer *src, GstBuffer *dest, const GstVideoInfo *dest_info, guint32 stride)\n> +{\n> + GstVideoInfo src_info = *dest_info;\n> + GstVideoFrame src_frame, dest_frame;\n> +\n> + gst_libcamera_extrapolate_info(&src_info, stride);\n> + src_info.size = gst_buffer_get_size(src);\n> +\n> + if (!gst_video_frame_map(&src_frame, &src_info, src, GST_MAP_READ)) {\n> + GST_ERROR(\"Could not map src buffer\");\n> + return GST_FLOW_ERROR;\n> + }\n> +\n> + if (!gst_video_frame_map(&dest_frame, const_cast(dest_info), dest, GST_MAP_WRITE)) {\n> + GST_ERROR(\"Could not map dest buffer\");\n> + gst_video_frame_unmap(&src_frame);\n> + return GST_FLOW_ERROR;\n> + }\n> +\n> + if (!gst_video_frame_copy(&dest_frame, &src_frame)) {\n> + GST_ERROR(\"Could not copy frame\");\n> + gst_video_frame_unmap(&src_frame);\n> + gst_video_frame_unmap(&dest_frame);\n> + return GST_FLOW_ERROR;\n> + }\n> +\n> + gst_video_frame_unmap(&src_frame);\n> + gst_video_frame_unmap(&dest_frame);\n> +\n> + return GST_FLOW_OK;\n> +}\n> +\n> /* Must be called with stream_lock held. */\n> int GstLibcameraSrcState::processRequest()\n> {\n> @@ -292,11 +341,41 @@ int GstLibcameraSrcState::processRequest()\n> GstFlowReturn ret = GST_FLOW_OK;\n> gst_flow_combiner_reset(src_->flow_combiner);\n> \n> - for (GstPad *srcpad : srcpads_) {\n> + for (gsize i = 0; i < srcpads_.size(); i++) {\n> + GstPad *srcpad = srcpads_[i];\n> Stream *stream = gst_libcamera_pad_get_stream(srcpad);\n> GstBuffer *buffer = wrap->detachBuffer(stream);\n> \n> FrameBuffer *fb = gst_libcamera_buffer_get_frame_buffer(buffer);\n> + const StreamConfiguration &stream_cfg = config_->at(i);\n> + GstBufferPool *video_pool = gst_libcamera_pad_get_video_pool(srcpad);\n> +\n> + if (video_pool) {\n> + /* Only set video pool when a copy is needed. */\n> + GstBuffer *copy = NULL;\n> + const GstVideoInfo info = gst_libcamera_pad_get_video_info(srcpad);\n> +\n> + ret = gst_buffer_pool_acquire_buffer(video_pool, ©, NULL);\n> + if (ret != GST_FLOW_OK) {\n> + gst_buffer_unref(buffer);\n> + GST_ELEMENT_ERROR(src_, RESOURCE, SETTINGS,\n> + (\"Failed to acquire buffer\"),\n> + (\"GstLibcameraSrcState::processRequest() failed: %s\", g_strerror(-ret)));\n> + return -EPIPE;\n> + }\n> +\n> + ret = gst_libcamera_video_frame_copy(buffer, copy, &info, stream_cfg.stride);\n> + gst_buffer_unref(buffer);\n> + if (ret != GST_FLOW_OK) {\n> + gst_buffer_unref(copy);\n> + GST_ELEMENT_ERROR(src_, RESOURCE, SETTINGS,\n> + (\"Failed to copy buffer\"),\n> + (\"GstLibcameraSrcState::processRequest() failed: %s\", g_strerror(-ret)));\n> + return -EPIPE;\n> + }\n> +\n> + buffer = copy;\n> + }\n> \n> if (GST_CLOCK_TIME_IS_VALID(wrap->pts_)) {\n> GST_BUFFER_PTS(buffer) = wrap->pts_;\n> @@ -499,13 +578,70 @@ gst_libcamera_src_negotiate(GstLibcameraSrc *self)\n> for (gsize i = 0; i < state->srcpads_.size(); i++) {\n> GstPad *srcpad = state->srcpads_[i];\n> const StreamConfiguration &stream_cfg = state->config_->at(i);\n> + GstBufferPool *video_pool = NULL;\n> + GstVideoInfo info;\n> +\n> + g_autoptr(GstCaps) caps = gst_libcamera_stream_configuration_to_caps(stream_cfg, transfer[i]);\n> +\n> + gst_video_info_from_caps(&info, caps);\n> + gst_libcamera_pad_set_video_info(srcpad, &info);\n> +\n> + /* Stride mismatch between camera stride and that calculated by video-info. */\n> + if (static_cast(info.stride[0]) != stream_cfg.stride &&\n> + GST_VIDEO_INFO_FORMAT(&info) != GST_VIDEO_FORMAT_ENCODED) {\n> + GstQuery *query = NULL;\n> + const gboolean need_pool = true;\n> + gboolean has_video_meta = false;\n> +\n> + gst_libcamera_extrapolate_info(&info, stream_cfg.stride);\n> +\n> + query = gst_query_new_allocation(caps, need_pool);\n> + if (!gst_pad_peer_query(srcpad, query))\n> + GST_DEBUG_OBJECT(self, \"Didn't get downstream ALLOCATION hints\");\n> + else\n> + has_video_meta = gst_query_find_allocation_meta(query, GST_VIDEO_META_API_TYPE, NULL);\n> +\n> + if (!has_video_meta) {\n> + GstBufferPool *pool = NULL;\n> +\n> + if (gst_query_get_n_allocation_pools(query) > 0)\n> + gst_query_parse_nth_allocation_pool(query, 0, &pool, NULL, NULL, NULL);\n> +\n> + if (pool)\n> + video_pool = pool;\n> + else {\n> + GstStructure *config;\n> + guint min_buffers = 3;\n> + video_pool = gst_video_buffer_pool_new();\n> +\n> + config = gst_buffer_pool_get_config(video_pool);\n> + gst_buffer_pool_config_set_params(config, caps, info.size, min_buffers, 0);\n> +\n> + GST_DEBUG_OBJECT(self, \"Own pool config is %\" GST_PTR_FORMAT, config);\n> +\n> + gst_buffer_pool_set_config(GST_BUFFER_POOL_CAST(video_pool), config);\n> + }\n> +\n> + GST_WARNING_OBJECT(self, \"Downstream doesn't support video meta, need to copy frame.\");\n> +\n> + if (!gst_buffer_pool_set_active(video_pool, true)) {\n> + gst_caps_unref(caps);\n> + GST_ELEMENT_ERROR(self, RESOURCE, SETTINGS,\n> + (\"Failed to active buffer pool\"),\n> + (\"gst_libcamera_src_negotiate() failed.\"));\n> + return false;\n> + }\n> + }\n> + gst_query_unref(query);\n> + }\n> \n> GstLibcameraPool *pool = gst_libcamera_pool_new(self->allocator,\n> - stream_cfg.stream());\n> + stream_cfg.stream(), &info);\n> g_signal_connect_swapped(pool, \"buffer-notify\",\n> G_CALLBACK(gst_task_resume), self->task);\n> \n> gst_libcamera_pad_set_pool(srcpad, pool);\n> + gst_libcamera_pad_set_video_pool(srcpad, video_pool);\n> \n> /* Clear all reconfigure flags. */\n> gst_pad_check_reconfigure(srcpad);\n> @@ -922,6 +1058,12 @@ gst_libcamera_src_release_pad(GstElement *element, GstPad *pad)\n> auto end_iterator = pads.end();\n> auto pad_iterator = std::find(begin_iterator, end_iterator, pad);\n> \n> + GstBufferPool *video_pool = gst_libcamera_pad_get_video_pool(pad);\n> + if (video_pool) {\n> + gst_buffer_pool_set_active(video_pool, false);\n> + gst_object_unref(video_pool);\n> + }\n> +\n> if (pad_iterator != end_iterator) {\n> g_object_unref(*pad_iterator);\n> pads.erase(pad_iterator);\n> -- \n> 2.34.1\n>","headers":{"Return-Path":"","X-Original-To":"parsemail@patchwork.libcamera.org","Delivered-To":"parsemail@patchwork.libcamera.org","Received":["from lancelot.ideasonboard.com (lancelot.ideasonboard.com\n\t[92.243.16.209])\n\tby patchwork.libcamera.org (Postfix) with ESMTPS id 56898C31E9\n\tfor ;\n\tMon, 19 May 2025 09:27:44 +0000 (UTC)","from lancelot.ideasonboard.com (localhost [IPv6:::1])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTP id 42302616A3;\n\tMon, 19 May 2025 11:27:43 +0200 (CEST)","from perceval.ideasonboard.com (perceval.ideasonboard.com\n\t[IPv6:2001:4b98:dc2:55:216:3eff:fef7:d647])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id 537F6616A3\n\tfor ;\n\tMon, 19 May 2025 11:27:41 +0200 (CEST)","from pendragon.ideasonboard.com\n\t(cpc89244-aztw30-2-0-cust6594.18-1.cable.virginm.net [86.31.185.195])\n\tby perceval.ideasonboard.com (Postfix) with ESMTPSA id 5B7C6E9B;\n\tMon, 19 May 2025 11:27:21 +0200 (CEST)"],"Authentication-Results":"lancelot.ideasonboard.com; dkim=pass (1024-bit key;\n\tunprotected) header.d=ideasonboard.com header.i=@ideasonboard.com\n\theader.b=\"vFc5SagV\"; dkim-atps=neutral","DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/simple; d=ideasonboard.com;\n\ts=mail; t=1747646841;\n\tbh=o/IhhyEzyyRtF8lFvy5jzCCHDissjgOCHbmLL+XVzlY=;\n\th=In-Reply-To:References:Subject:From:Cc:To:Date:From;\n\tb=vFc5SagVimWq7NmopLokk+LM63/iA/r7omGEXTObfo+YDiji6NVeKJbc+28FLEo31\n\ta0SD2agtJGhJBvX6SAnOEF2B3zR/zPLrytN6rV2zmNH4X38oQd7A+wnpjZjw7S4ry7\n\t1X+m+uyYYFCdR0eT9feutyXr8m0idQ5CzL+ovx5o=","Content-Type":"text/plain; charset=\"utf-8\"","MIME-Version":"1.0","Content-Transfer-Encoding":"quoted-printable","In-Reply-To":"<20250519075210.3765048-1-qi.hou@nxp.com>","References":"<20250519075210.3765048-1-qi.hou@nxp.com>","Subject":"Re: [PATCH v9] gstreamer: Add GstVideoMeta support","From":"Kieran Bingham ","Cc":"nicolas@ndufresne.ca, laurent.pinchart@ideasonboard.com, jared.hu@nxp.com,\n\tqi.hou@nxp.com, julien.vuillaumier@nxp.com","To":"Hou Qi , libcamera-devel@lists.libcamera.org","Date":"Mon, 19 May 2025 10:27:38 +0100","Message-ID":"<174764685801.40401.111016603272465895@ping.linuxembedded.co.uk>","User-Agent":"alot/0.10","X-BeenThere":"libcamera-devel@lists.libcamera.org","X-Mailman-Version":"2.1.29","Precedence":"list","List-Id":"","List-Unsubscribe":",\n\t","List-Archive":"","List-Post":"","List-Help":"","List-Subscribe":",\n\t","Errors-To":"libcamera-devel-bounces@lists.libcamera.org","Sender":"\"libcamera-devel\" "}}]