[{"id":3975,"web_url":"https://patchwork.libcamera.org/comment/3975/","msgid":"<20200306205912.GX4878@pendragon.ideasonboard.com>","date":"2020-03-06T20:59:12","subject":"Re: [libcamera-devel] [PATCH v3 22/27] gst: libcamerasrc: Implement\n\tinitial streaming","submitter":{"id":2,"url":"https://patchwork.libcamera.org/api/people/2/","name":"Laurent Pinchart","email":"laurent.pinchart@ideasonboard.com"},"content":"Hi Nicolas,\n\nThank you for the patch.\n\nOn Fri, Mar 06, 2020 at 03:26:32PM -0500, Nicolas Dufresne wrote:\n> From: Nicolas Dufresne <nicolas.dufresne@collabora.com>\n> \n> With this patch, the element is now able to push buffers to the next\n> element in the graph. The buffers are currently missing any metadata\n> like timestamp, sequence number. This will be added in the next commit.\n> \n> Signed-off-by: Nicolas Dufresne <nicolas.dufresne@collabora.com>\n\nReviewed-by: Laurent Pinchart <laurent.pinchart@ideasonboard.com>\n\n> ---\n>  src/gstreamer/gstlibcamerapad.cpp |   9 ++\n>  src/gstreamer/gstlibcamerapad.h   |   2 +\n>  src/gstreamer/gstlibcamerasrc.cpp | 192 +++++++++++++++++++++++++++++-\n>  3 files changed, 202 insertions(+), 1 deletion(-)\n> \n> diff --git a/src/gstreamer/gstlibcamerapad.cpp b/src/gstreamer/gstlibcamerapad.cpp\n> index 49dd35b..840f391 100644\n> --- a/src/gstreamer/gstlibcamerapad.cpp\n> +++ b/src/gstreamer/gstlibcamerapad.cpp\n> @@ -19,6 +19,7 @@ struct _GstLibcameraPad {\n>  \tStreamRole role;\n>  \tGstLibcameraPool *pool;\n>  \tGQueue pending_buffers;\n> +\tGstClockTime latency;\n>  };\n>  \n>  enum {\n> @@ -164,3 +165,11 @@ gst_libcamera_pad_push_pending(GstPad *pad)\n>  \n>  \treturn gst_pad_push(pad, buffer);\n>  }\n> +\n> +bool\n> +gst_libcamera_pad_has_pending(GstPad *pad)\n> +{\n> +\tauto *self = GST_LIBCAMERA_PAD(pad);\n> +\tGLibLocker lock(GST_OBJECT(self));\n> +\treturn self->pending_buffers.length > 0;\n> +}\n> diff --git a/src/gstreamer/gstlibcamerapad.h b/src/gstreamer/gstlibcamerapad.h\n> index 2e9ec20..9d43129 100644\n> --- a/src/gstreamer/gstlibcamerapad.h\n> +++ b/src/gstreamer/gstlibcamerapad.h\n> @@ -30,4 +30,6 @@ void gst_libcamera_pad_queue_buffer(GstPad *pad, GstBuffer *buffer);\n>  \n>  GstFlowReturn gst_libcamera_pad_push_pending(GstPad *pad);\n>  \n> +bool gst_libcamera_pad_has_pending(GstPad *pad);\n> +\n>  #endif /* __GST_LIBCAMERA_PAD_H__ */\n> diff --git a/src/gstreamer/gstlibcamerasrc.cpp b/src/gstreamer/gstlibcamerasrc.cpp\n> index 5ffc004..e3718db 100644\n> --- a/src/gstreamer/gstlibcamerasrc.cpp\n> +++ b/src/gstreamer/gstlibcamerasrc.cpp\n> @@ -14,8 +14,11 @@\n>  \n>  #include \"gstlibcamerasrc.h\"\n>  \n> +#include <queue>\n>  #include <vector>\n>  \n> +#include <gst/base/base.h>\n> +\n>  #include <libcamera/camera.h>\n>  #include <libcamera/camera_manager.h>\n>  \n> @@ -29,12 +32,71 @@ using namespace libcamera;\n>  GST_DEBUG_CATEGORY_STATIC(source_debug);\n>  #define GST_CAT_DEFAULT source_debug\n>  \n> +struct RequestWrap {\n> +\tRequestWrap(Request *request);\n> +\t~RequestWrap();\n> +\n> +\tvoid attachBuffer(GstBuffer *buffer);\n> +\tGstBuffer *detachBuffer(Stream *stream);\n> +\n> +\t/* For ptr comparison only. */\n> +\tRequest *request_;\n> +\tstd::map<Stream *, GstBuffer *> buffers_;\n> +};\n> +\n> +RequestWrap::RequestWrap(Request *request)\n> +\t: request_(request)\n> +{\n> +}\n> +\n> +RequestWrap::~RequestWrap()\n> +{\n> +\tfor (std::pair<Stream *const, GstBuffer *> &item : buffers_) {\n> +\t\tif (item.second)\n> +\t\t\tgst_buffer_unref(item.second);\n> +\t}\n> +}\n> +\n> +void RequestWrap::attachBuffer(GstBuffer *buffer)\n> +{\n> +\tFrameBuffer *fb = gst_libcamera_buffer_get_frame_buffer(buffer);\n> +\tStream *stream = gst_libcamera_buffer_get_stream(buffer);\n> +\n> +\trequest_->addBuffer(stream, fb);\n> +\n> +\tauto item = buffers_.find(stream);\n> +\tif (item != buffers_.end()) {\n> +\t\tgst_buffer_unref(item->second);\n> +\t\titem->second = buffer;\n> +\t} else {\n> +\t\tbuffers_[stream] = buffer;\n> +\t}\n> +}\n> +\n> +GstBuffer *RequestWrap::detachBuffer(Stream *stream)\n> +{\n> +\tGstBuffer *buffer = nullptr;\n> +\n> +\tauto item = buffers_.find(stream);\n> +\tif (item != buffers_.end()) {\n> +\t\tbuffer = item->second;\n> +\t\titem->second = nullptr;\n> +\t}\n> +\n> +\treturn buffer;\n> +}\n> +\n>  /* Used for C++ object with destructors. */\n>  struct GstLibcameraSrcState {\n> +\tGstLibcameraSrc *src_;\n> +\n>  \tstd::unique_ptr<CameraManager> cm_;\n>  \tstd::shared_ptr<Camera> cam_;\n>  \tstd::unique_ptr<CameraConfiguration> config_;\n>  \tstd::vector<GstPad *> srcpads_;\n> +\tstd::queue<std::unique_ptr<RequestWrap>> requests_;\n> +\n> +\tvoid requestCompleted(Request *request);\n>  };\n>  \n>  struct _GstLibcameraSrc {\n> @@ -47,6 +109,7 @@ struct _GstLibcameraSrc {\n>  \n>  \tGstLibcameraSrcState *state;\n>  \tGstLibcameraAllocator *allocator;\n> +\tGstFlowCombiner *flow_combiner;\n>  };\n>  \n>  enum {\n> @@ -70,6 +133,41 @@ GstStaticPadTemplate request_src_template = {\n>  \t\"src_%s\", GST_PAD_SRC, GST_PAD_REQUEST, TEMPLATE_CAPS\n>  };\n>  \n> +void\n> +GstLibcameraSrcState::requestCompleted(Request *request)\n> +{\n> +\tGLibLocker lock(GST_OBJECT(src_));\n> +\n> +\tGST_DEBUG_OBJECT(src_, \"buffers are ready\");\n> +\n> +\tstd::unique_ptr<RequestWrap> wrap = std::move(requests_.front());\n> +\trequests_.pop();\n> +\n> +\tg_return_if_fail(wrap->request_ == request);\n> +\n> +\tif ((request->status() == Request::RequestCancelled)) {\n> +\t\tGST_DEBUG_OBJECT(src_, \"Request was cancelled\");\n> +\t\treturn;\n> +\t}\n> +\n> +\tGstBuffer *buffer;\n> +\tfor (GstPad *srcpad : srcpads_) {\n> +\t\tStream *stream = gst_libcamera_pad_get_stream(srcpad);\n> +\t\tbuffer = wrap->detachBuffer(stream);\n> +\t\tgst_libcamera_pad_queue_buffer(srcpad, buffer);\n> +\t}\n> +\n> +\t{\n> +\t\t/* We only want to resume the task if it's paused. */\n> +\t\tGstTask *task = src_->task;\n> +\t\tGLibLocker lock(GST_OBJECT(task));\n> +\t\tif (GST_TASK_STATE(task) == GST_TASK_PAUSED) {\n> +\t\t\tGST_TASK_STATE(task) = GST_TASK_STARTED;\n> +\t\t\tGST_TASK_SIGNAL(task);\n> +\t\t}\n> +\t}\n> +}\n> +\n>  static bool\n>  gst_libcamera_src_open(GstLibcameraSrc *self)\n>  {\n> @@ -122,6 +220,8 @@ gst_libcamera_src_open(GstLibcameraSrc *self)\n>  \t\treturn false;\n>  \t}\n>  \n> +\tcam->requestCompleted.connect(self->state, &GstLibcameraSrcState::requestCompleted);\n> +\n>  \t/* No need to lock here, we didn't start our threads yet. */\n>  \tself->state->cm_ = std::move(cm);\n>  \tself->state->cam_ = cam;\n> @@ -133,8 +233,77 @@ static void\n>  gst_libcamera_src_task_run(gpointer user_data)\n>  {\n>  \tGstLibcameraSrc *self = GST_LIBCAMERA_SRC(user_data);\n> +\tGstLibcameraSrcState *state = self->state;\n> +\n> +\tRequest *request = state->cam_->createRequest();\n> +\tauto wrap = std::make_unique<RequestWrap>(request);\n> +\tfor (GstPad *srcpad : state->srcpads_) {\n> +\t\tGstLibcameraPool *pool = gst_libcamera_pad_get_pool(srcpad);\n> +\t\tGstBuffer *buffer;\n> +\t\tGstFlowReturn ret;\n> +\n> +\t\tret = gst_buffer_pool_acquire_buffer(GST_BUFFER_POOL(pool),\n> +\t\t\t\t\t\t     &buffer, nullptr);\n> +\t\tif (ret != GST_FLOW_OK) {\n> +\t\t\t/*\n> +\t\t\t * RequestWrap does not take ownership, and we won't be\n> +\t\t\t * queueing this one due to lack of buffers.\n> +\t\t\t */\n> +\t\t\tdelete request;\n> +\t\t\trequest = nullptr;\n> +\t\t\tbreak;\n> +\t\t}\n> +\n> +\t\twrap->attachBuffer(buffer);\n> +\t}\n> +\n> +\tif (request) {\n> +\t\tGLibLocker lock(GST_OBJECT(self));\n> +\t\tGST_TRACE_OBJECT(self, \"Requesting buffers\");\n> +\t\tstate->cam_->queueRequest(request);\n> +\t\tstate->requests_.push(std::move(wrap));\n> +\t}\n> +\n> +\tGstFlowReturn ret = GST_FLOW_OK;\n> +\tgst_flow_combiner_reset(self->flow_combiner);\n> +\tfor (GstPad *srcpad : state->srcpads_) {\n> +\t\tret = gst_libcamera_pad_push_pending(srcpad);\n> +\t\tret = gst_flow_combiner_update_pad_flow(self->flow_combiner,\n> +\t\t\t\t\t\t\tsrcpad, ret);\n> +\t}\n>  \n> -\tGST_DEBUG_OBJECT(self, \"Streaming thread is now capturing\");\n> +\t{\n> +\t\t/*\n> +\t\t * Here we need to decide if we want to pause or stop the task. This\n> +\t\t * needs to happen in lock step with the callback thread which may want\n> +\t\t * to resume the task.\n> +\t\t */\n> +\t\tGLibLocker lock(GST_OBJECT(self));\n> +\t\tif (ret != GST_FLOW_OK) {\n> +\t\t\tif (ret == GST_FLOW_EOS) {\n> +\t\t\t\tg_autoptr(GstEvent) eos = gst_event_new_eos();\n> +\t\t\t\tguint32 seqnum = gst_util_seqnum_next();\n> +\t\t\t\tgst_event_set_seqnum(eos, seqnum);\n> +\t\t\t\tfor (GstPad *srcpad : state->srcpads_)\n> +\t\t\t\t\tgst_pad_push_event(srcpad, gst_event_ref(eos));\n> +\t\t\t} else if (ret != GST_FLOW_FLUSHING) {\n> +\t\t\t\tGST_ELEMENT_FLOW_ERROR(self, ret);\n> +\t\t\t}\n> +\t\t\tgst_task_stop(self->task);\n> +\t\t\treturn;\n> +\t\t}\n> +\n> +\t\tbool do_pause = true;\n> +\t\tfor (GstPad *srcpad : state->srcpads_) {\n> +\t\t\tif (gst_libcamera_pad_has_pending(srcpad)) {\n> +\t\t\t\tdo_pause = false;\n> +\t\t\t\tbreak;\n> +\t\t\t}\n> +\t\t}\n> +\n> +\t\tif (do_pause)\n> +\t\t\tgst_task_pause(self->task);\n> +\t}\n>  }\n>  \n>  static void\n> @@ -233,12 +402,23 @@ gst_libcamera_src_task_enter(GstTask *task, GThread *thread, gpointer user_data)\n>  \t\treturn;\n>  \t}\n>  \n> +\tself->flow_combiner = gst_flow_combiner_new();\n>  \tfor (gsize i = 0; i < state->srcpads_.size(); i++) {\n>  \t\tGstPad *srcpad = state->srcpads_[i];\n>  \t\tconst StreamConfiguration &stream_cfg = state->config_->at(i);\n>  \t\tGstLibcameraPool *pool = gst_libcamera_pool_new(self->allocator,\n>  \t\t\t\t\t\t\t\tstream_cfg.stream());\n>  \t\tgst_libcamera_pad_set_pool(srcpad, pool);\n> +\t\tgst_flow_combiner_add_pad(self->flow_combiner, srcpad);\n> +\t}\n> +\n> +\tret = state->cam_->start();\n> +\tif (ret) {\n> +\t\tGST_ELEMENT_ERROR(self, RESOURCE, SETTINGS,\n> +\t\t\t\t  (\"Failed to start the camera: %s\", g_strerror(-ret)),\n> +\t\t\t\t  (\"Camera.start() failed with error code %i\", ret));\n> +\t\tgst_task_stop(task);\n> +\t\treturn;\n>  \t}\n>  \n>  done:\n> @@ -260,10 +440,14 @@ gst_libcamera_src_task_leave(GstTask *task, GThread *thread, gpointer user_data)\n>  \n>  \tGST_DEBUG_OBJECT(self, \"Streaming thread is about to stop\");\n>  \n> +\tstate->cam_->stop();\n> +\n>  \tfor (GstPad *srcpad : state->srcpads_)\n>  \t\tgst_libcamera_pad_set_pool(srcpad, NULL);\n>  \n>  \tg_clear_object(&self->allocator);\n> +\tg_clear_pointer(&self->flow_combiner,\n> +\t\t\t(GDestroyNotify)gst_flow_combiner_free);\n>  }\n>  \n>  static void\n> @@ -343,6 +527,9 @@ gst_libcamera_src_change_state(GstElement *element, GstStateChange transition)\n>  \t\t\treturn GST_STATE_CHANGE_FAILURE;\n>  \t\tret = GST_STATE_CHANGE_NO_PREROLL;\n>  \t\tbreak;\n> +\tcase GST_STATE_CHANGE_PAUSED_TO_PLAYING:\n> +\t\tgst_task_start(self->task);\n> +\t\tbreak;\n>  \tcase GST_STATE_CHANGE_PLAYING_TO_PAUSED:\n>  \t\tret = GST_STATE_CHANGE_NO_PREROLL;\n>  \t\tbreak;\n> @@ -394,6 +581,9 @@ gst_libcamera_src_init(GstLibcameraSrc *self)\n>  \n>  \tstate->srcpads_.push_back(gst_pad_new_from_template(templ, \"src\"));\n>  \tgst_element_add_pad(GST_ELEMENT(self), state->srcpads_[0]);\n> +\n> +\t/* C-style friend. */\n> +\tstate->src_ = self;\n>  \tself->state = state;\n>  }\n>","headers":{"Return-Path":"<laurent.pinchart@ideasonboard.com>","Received":["from perceval.ideasonboard.com (perceval.ideasonboard.com\n\t[213.167.242.64])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id 635E460424\n\tfor <libcamera-devel@lists.libcamera.org>;\n\tFri,  6 Mar 2020 21:59:16 +0100 (CET)","from pendragon.ideasonboard.com (81-175-216-236.bb.dnainternet.fi\n\t[81.175.216.236])\n\tby perceval.ideasonboard.com (Postfix) with ESMTPSA id C852624B;\n\tFri,  6 Mar 2020 21:59:15 +0100 (CET)"],"DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/simple; d=ideasonboard.com;\n\ts=mail; t=1583528356;\n\tbh=TmqDCmSOL6fwXcsbl4qbd8yVffcnCDWDHZf50jz9bLg=;\n\th=Date:From:To:Cc:Subject:References:In-Reply-To:From;\n\tb=EXNWaw4ATVrjUb9DEK4iYN2AnqLM/CsSw8SrfA48Xm02Ig2U43tORTZdYMsHpu86T\n\tN8cd399F9P3hOjcgjxa9S5eafLTh7UjcG5wJ7+N318ag01btnoPknYO8Vyrn1PeC1p\n\t/RVuke2t4epEphLb0Wa/yyP5bFODt6+GZE5UAVRE=","Date":"Fri, 6 Mar 2020 22:59:12 +0200","From":"Laurent Pinchart <laurent.pinchart@ideasonboard.com>","To":"Nicolas Dufresne <nicolas@ndufresne.ca>","Cc":"libcamera-devel@lists.libcamera.org,\n\tNicolas Dufresne <nicolas.dufresne@collabora.com>","Message-ID":"<20200306205912.GX4878@pendragon.ideasonboard.com>","References":"<20200306202637.525587-1-nicolas@ndufresne.ca>\n\t<20200306202637.525587-23-nicolas@ndufresne.ca>","MIME-Version":"1.0","Content-Type":"text/plain; charset=utf-8","Content-Disposition":"inline","In-Reply-To":"<20200306202637.525587-23-nicolas@ndufresne.ca>","User-Agent":"Mutt/1.10.1 (2018-07-13)","Subject":"Re: [libcamera-devel] [PATCH v3 22/27] gst: libcamerasrc: Implement\n\tinitial streaming","X-BeenThere":"libcamera-devel@lists.libcamera.org","X-Mailman-Version":"2.1.29","Precedence":"list","List-Id":"<libcamera-devel.lists.libcamera.org>","List-Unsubscribe":"<https://lists.libcamera.org/options/libcamera-devel>,\n\t<mailto:libcamera-devel-request@lists.libcamera.org?subject=unsubscribe>","List-Archive":"<https://lists.libcamera.org/pipermail/libcamera-devel/>","List-Post":"<mailto:libcamera-devel@lists.libcamera.org>","List-Help":"<mailto:libcamera-devel-request@lists.libcamera.org?subject=help>","List-Subscribe":"<https://lists.libcamera.org/listinfo/libcamera-devel>,\n\t<mailto:libcamera-devel-request@lists.libcamera.org?subject=subscribe>","X-List-Received-Date":"Fri, 06 Mar 2020 20:59:16 -0000"}}]