[{"id":26742,"web_url":"https://patchwork.libcamera.org/comment/26742/","msgid":"<425de849757b8cdd66b5e40da82a2d437c02d790.camel@collabora.com>","date":"2023-03-24T18:03:35","subject":"Re: [libcamera-devel] [PATCH v1 1/3] gstreamer: Add sensor mode\n\tselection","submitter":{"id":31,"url":"https://patchwork.libcamera.org/api/people/31/","name":"Nicolas Dufresne","email":"nicolas.dufresne@collabora.com"},"content":"Le vendredi 24 mars 2023 à 13:40 -0400, Nicolas Dufresne a écrit :\n> From: Nicolas Dufresne <nicolas.dufresne@collabora.com>\n> \n> This add support for selecting the sensor mode. A new read-only\n> property called sensor-modes is filled when the element reaches\n> READY state. It contains the list of all available sensor modes,\n> including the supported framerate range. This is exposed as GstCaps\n> in the form of sensor/mode,width=X,height=Y,format=Y,framerate=[...].\n> The format string matches the libcamera format string representation.\n> \n> The application can then select a mode using the read/write sensor-mode\n> control. The selected mode is also a caps, it will be intersected with\n> the supported mode and the \"best\" match will be picked. This allows\n> application to use simple filter when they want to pick a mode for lets\n> say a specific framerate (e.g. sensor/mode,framerate=60/1).\n> \n> Signed-off-by: Nicolas Dufresne <nicolas.dufresne@collabora.com>\n> ---\n>  src/gstreamer/gstlibcamera-utils.cpp |   4 +\n>  src/gstreamer/gstlibcamerasrc.cpp    | 123 ++++++++++++++++++++++++++-\n>  2 files changed, 126 insertions(+), 1 deletion(-)\n> \n> diff --git a/src/gstreamer/gstlibcamera-utils.cpp b/src/gstreamer/gstlibcamera-utils.cpp\n> index 750ec351..c8a8df09 100644\n> --- a/src/gstreamer/gstlibcamera-utils.cpp\n> +++ b/src/gstreamer/gstlibcamera-utils.cpp\n> @@ -416,6 +416,10 @@ gst_libcamera_configure_stream_from_caps(StreamConfiguration &stream_cfg,\n>  \t\tstream_cfg.pixelFormat = gst_format_to_pixel_format(gst_format);\n>  \t} else if (gst_structure_has_name(s, \"image/jpeg\")) {\n>  \t\tstream_cfg.pixelFormat = formats::MJPEG;\n> +\t} else if (gst_structure_has_name(s, \"sensor/mode\")) {\n> +\t\tgst_structure_fixate_field(s, \"format\");\n> +\t\tconst gchar *format = gst_structure_get_string(s, \"format\");\n> +\t\tstream_cfg.pixelFormat = PixelFormat::fromString(format);\n>  \t} else {\n>  \t\tg_critical(\"Unsupported media type: %s\", gst_structure_get_name(s));\n>  \t}\n> diff --git a/src/gstreamer/gstlibcamerasrc.cpp b/src/gstreamer/gstlibcamerasrc.cpp\n> index a10cbd4f..c448a9fe 100644\n> --- a/src/gstreamer/gstlibcamerasrc.cpp\n> +++ b/src/gstreamer/gstlibcamerasrc.cpp\n> @@ -147,6 +147,15 @@ struct _GstLibcameraSrc {\n>  \n>  \tgchar *camera_name;\n>  \n> +<<<<<<< HEAD\n> +=======\n\nMy apology, scratch this please.\n\n> +\tgboolean hflip;\n> +\tgboolean vflip;\n> +\n> +\tGstCaps *sensor_modes;\n> +\tGstCaps *sensor_mode;\n> +\n> +>>>>>>> 210f0a67 (gstreamer: Add sensor mode selection)\n>  \tGstLibcameraSrcState *state;\n>  \tGstLibcameraAllocator *allocator;\n>  \tGstFlowCombiner *flow_combiner;\n> @@ -154,7 +163,16 @@ struct _GstLibcameraSrc {\n>  \n>  enum {\n>  \tPROP_0,\n> +<<<<<<< HEAD\n>  \tPROP_CAMERA_NAME\n> +=======\n> +\tPROP_CAMERA_NAME,\n> +\tPROP_HFLIP,\n> +\tPROP_VFLIP,\n> +\tPROP_SENSOR_MODES,\n> +\tPROP_SENSOR_MODE,\n> +\tPROP_LAST\n> +>>>>>>> 210f0a67 (gstreamer: Add sensor mode selection)\n>  };\n>  \n>  G_DEFINE_TYPE_WITH_CODE(GstLibcameraSrc, gst_libcamera_src, GST_TYPE_ELEMENT,\n> @@ -318,6 +336,61 @@ int GstLibcameraSrcState::processRequest()\n>  \treturn err;\n>  }\n>  \n> +static GstCaps *\n> +gst_libcamera_src_enumerate_sensor_modes(GstLibcameraSrc *self)\n> +{\n> +\tGstCaps *modes = gst_caps_new_empty();\n> +\tGstLibcameraSrcState *state = self->state;\n> +\tauto config = state->cam_->generateConfiguration({ libcamera::StreamRole::Raw,\n> +\t\t\t\t\t\t\t   libcamera::StreamRole::VideoRecording });\n> +\tif (config == nullptr) {\n> +\t\tGST_DEBUG_OBJECT(self, \"Sensor mode selection is not supported, skipping enumeration.\");\n> +\t\treturn modes;\n> +\t}\n> +\n> +\tconst libcamera::StreamFormats &formats = config->at(0).formats();\n> +\n> +\tfor (const auto &pixfmt : formats.pixelformats()) {\n> +\t\tfor (const auto &size : formats.sizes(pixfmt)) {\n> +\t\t\tconfig->at(0).size = size;\n> +\t\t\tconfig->at(0).pixelFormat = pixfmt;\n> +\n> +\t\t\tif (config->validate() == CameraConfiguration::Invalid)\n> +\t\t\t\tcontinue;\n> +\n> +\t\t\tif (state->cam_->configure(config.get()))\n> +\t\t\t\tcontinue;\n> +\n> +\t\t\tauto fd_ctrl = state->cam_->controls().find(&controls::FrameDurationLimits);\n> +\t\t\tif (fd_ctrl == state->cam_->controls().end())\n> +\t\t\t\tcontinue;\n> +\n> +\t\t\tint minrate_num, minrate_denom;\n> +\t\t\tint maxrate_num, maxrate_denom;\n> +\t\t\tdouble min_framerate = gst_util_guint64_to_gdouble(1.0e6) /\n> +\t\t\t\t\t       gst_util_guint64_to_gdouble(fd_ctrl->second.max().get<int64_t>());\n> +\t\t\tdouble max_framerate = gst_util_guint64_to_gdouble(1.0e6) /\n> +\t\t\t\t\t       gst_util_guint64_to_gdouble(fd_ctrl->second.min().get<int64_t>());\n> +\t\t\tgst_util_double_to_fraction(min_framerate, &minrate_num, &minrate_denom);\n> +\t\t\tgst_util_double_to_fraction(max_framerate, &maxrate_num, &maxrate_denom);\n> +\n> +\t\t\tGstStructure *s = gst_structure_new(\"sensor/mode\",\n> +\t\t\t\t\t\t\t    \"format\", G_TYPE_STRING, pixfmt.toString().c_str(),\n> +\t\t\t\t\t\t\t    \"width\", G_TYPE_INT, size.width,\n> +\t\t\t\t\t\t\t    \"height\", G_TYPE_INT, size.height,\n> +\t\t\t\t\t\t\t    \"framerate\", GST_TYPE_FRACTION_RANGE,\n> +\t\t\t\t\t\t\t    minrate_num, minrate_denom,\n> +\t\t\t\t\t\t\t    maxrate_num, maxrate_denom,\n> +\t\t\t\t\t\t\t    nullptr);\n> +\t\t\tgst_caps_append_structure(modes, s);\n> +\t\t}\n> +\t}\n> +\n> +\tGST_DEBUG_OBJECT(self, \"Camera sensor modes: %\" GST_PTR_FORMAT, modes);\n> +\n> +\treturn modes;\n> +}\n> +\n>  static bool\n>  gst_libcamera_src_open(GstLibcameraSrc *self)\n>  {\n> @@ -375,6 +448,7 @@ gst_libcamera_src_open(GstLibcameraSrc *self)\n>  \t/* No need to lock here, we didn't start our threads yet. */\n>  \tself->state->cm_ = cm;\n>  \tself->state->cam_ = cam;\n> +\tself->sensor_modes = gst_libcamera_src_enumerate_sensor_modes(self);\n>  \n>  \treturn true;\n>  }\n> @@ -462,6 +536,7 @@ gst_libcamera_src_task_enter(GstTask *task, [[maybe_unused]] GThread *thread,\n>  \tGstLibcameraSrcState *state = self->state;\n>  \tGstFlowReturn flow_ret = GST_FLOW_OK;\n>  \tgint ret;\n> +\tg_autoptr(GstCaps) sensor_mode = nullptr;\n>  \n>  \tg_autoptr(GstStructure) element_caps = gst_structure_new_empty(\"caps\");\n>  \n> @@ -481,6 +556,16 @@ gst_libcamera_src_task_enter(GstTask *task, [[maybe_unused]] GThread *thread,\n>  \t\troles.push_back(gst_libcamera_pad_get_role(srcpad));\n>  \t}\n>  \n> +\tif (!gst_caps_is_any(self->sensor_mode)) {\n> +\t\tsensor_mode = gst_caps_intersect(self->sensor_mode, self->sensor_modes);\n> +\t\tif (!gst_caps_is_empty(sensor_mode)) {\n> +\t\t\troles.push_back(libcamera::StreamRole::Raw);\n> +\t\t} else {\n> +\t\t\tGST_WARNING_OBJECT(self, \"No sensor mode matching the selection, ignoring.\");\n> +\t\t\tgst_clear_caps(&sensor_mode);\n> +\t\t}\n> +\t}\n> +\n>  \t/* Generate the stream configurations, there should be one per pad. */\n>  \tstate->config_ = state->cam_->generateConfiguration(roles);\n>  \tif (state->config_ == nullptr) {\n> @@ -490,7 +575,7 @@ gst_libcamera_src_task_enter(GstTask *task, [[maybe_unused]] GThread *thread,\n>  \t\tgst_task_stop(task);\n>  \t\treturn;\n>  \t}\n> -\tg_assert(state->config_->size() == state->srcpads_.size());\n> +\tg_assert(state->config_->size() == state->srcpads_.size() + (!!sensor_mode));\n>  \n>  \tfor (gsize i = 0; i < state->srcpads_.size(); i++) {\n>  \t\tGstPad *srcpad = state->srcpads_[i];\n> @@ -510,6 +595,12 @@ gst_libcamera_src_task_enter(GstTask *task, [[maybe_unused]] GThread *thread,\n>  \t\tgst_libcamera_get_framerate_from_caps(caps, element_caps);\n>  \t}\n>  \n> +\tif (sensor_mode) {\n> +\t\tStreamConfiguration &stream_cfg = state->config_->at(state->srcpads_.size());\n> +\t\tg_assert(gst_caps_is_writable(sensor_mode));\n> +\t\tgst_libcamera_configure_stream_from_caps(stream_cfg, sensor_mode);\n> +\t}\n> +\n>  \tif (flow_ret != GST_FLOW_OK)\n>  \t\tgoto done;\n>  \n> @@ -624,6 +715,7 @@ gst_libcamera_src_task_leave([[maybe_unused]] GstTask *task,\n>  \tg_clear_object(&self->allocator);\n>  \tg_clear_pointer(&self->flow_combiner,\n>  \t\t\t(GDestroyNotify)gst_flow_combiner_free);\n> +\tgst_clear_caps(&self->sensor_modes);\n>  }\n>  \n>  static void\n> @@ -659,6 +751,10 @@ gst_libcamera_src_set_property(GObject *object, guint prop_id,\n>  \t\tg_free(self->camera_name);\n>  \t\tself->camera_name = g_value_dup_string(value);\n>  \t\tbreak;\n> +\tcase PROP_SENSOR_MODE:\n> +\t\tgst_clear_caps(&self->sensor_mode);\n> +\t\tself->sensor_mode = GST_CAPS(g_value_dup_boxed(value));\n> +\t\tbreak;\n>  \tdefault:\n>  \t\tG_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec);\n>  \t\tbreak;\n> @@ -676,6 +772,12 @@ gst_libcamera_src_get_property(GObject *object, guint prop_id, GValue *value,\n>  \tcase PROP_CAMERA_NAME:\n>  \t\tg_value_set_string(value, self->camera_name);\n>  \t\tbreak;\n> +\tcase PROP_SENSOR_MODES:\n> +\t\tg_value_set_boxed(value, self->sensor_modes);\n> +\t\tbreak;\n> +\tcase PROP_SENSOR_MODE:\n> +\t\tg_value_set_boxed(value, self->sensor_mode);\n> +\t\tbreak;\n>  \tdefault:\n>  \t\tG_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec);\n>  \t\tbreak;\n> @@ -763,6 +865,7 @@ gst_libcamera_src_init(GstLibcameraSrc *self)\n>  \t/* C-style friend. */\n>  \tstate->src_ = self;\n>  \tself->state = state;\n> +\tself->sensor_mode = gst_caps_new_any();\n>  }\n>  \n>  static GstPad *\n> @@ -844,4 +947,22 @@ gst_libcamera_src_class_init(GstLibcameraSrcClass *klass)\n>  \t\t\t\t\t\t\t     | G_PARAM_READWRITE\n>  \t\t\t\t\t\t\t     | G_PARAM_STATIC_STRINGS));\n>  \tg_object_class_install_property(object_class, PROP_CAMERA_NAME, spec);\n> +\n> +\tspec = g_param_spec_boxed(\"sensor-modes\", \"Sensor Modes\",\n> +\t\t\t\t  \"GstCaps representing available sensor modes.\",\n> +\t\t\t\t  GST_TYPE_CAPS,\n> +\t\t\t\t  (GParamFlags)(G_PARAM_READABLE\n> +\t\t\t\t\t        | G_PARAM_STATIC_STRINGS));\n> +\tg_object_class_install_property(object_class, PROP_SENSOR_MODES, spec);\n> +\n> +\tspec = g_param_spec_boxed(\"sensor-mode\", \"Sensor Mode\",\n> +\t\t\t\t  \"GstCaps representing selected sensor mode.\",\n> +\t\t\t\t  GST_TYPE_CAPS,\n> +\t\t\t\t  (GParamFlags)(GST_PARAM_MUTABLE_READY\n> +\t\t\t\t\t        | G_PARAM_READWRITE\n> +\t\t\t\t\t        | G_PARAM_STATIC_STRINGS));\n> +\tg_object_class_install_property(object_class, PROP_SENSOR_MODE, spec);\n> +\n> +\n> +\tGstCameraControls::installProperties(object_class, PROP_LAST);\n>  }","headers":{"Return-Path":"<libcamera-devel-bounces@lists.libcamera.org>","X-Original-To":"parsemail@patchwork.libcamera.org","Delivered-To":"parsemail@patchwork.libcamera.org","Received":["from lancelot.ideasonboard.com (lancelot.ideasonboard.com\n\t[92.243.16.209])\n\tby patchwork.libcamera.org (Postfix) with ESMTPS id 9EFF5C0F2A\n\tfor <parsemail@patchwork.libcamera.org>;\n\tFri, 24 Mar 2023 18:03:46 +0000 (UTC)","from lancelot.ideasonboard.com (localhost [IPv6:::1])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTP id F13F062718;\n\tFri, 24 Mar 2023 19:03:45 +0100 (CET)","from madras.collabora.co.uk (madras.collabora.co.uk\n\t[IPv6:2a00:1098:0:82:1000:25:2eeb:e5ab])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id 4ABCF603AA\n\tfor <libcamera-devel@lists.libcamera.org>;\n\tFri, 24 Mar 2023 19:03:44 +0100 (CET)","from nicolas-tpx395.localdomain (192-222-136-102.qc.cable.ebox.net\n\t[192.222.136.102])\n\t(using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits)\n\tkey-exchange X25519 server-signature RSA-PSS (4096 bits)\n\tserver-digest SHA256)\n\t(No client certificate requested) (Authenticated sender: nicolas)\n\tby madras.collabora.co.uk (Postfix) with ESMTPSA id BB41E660312C\n\tfor <libcamera-devel@lists.libcamera.org>;\n\tFri, 24 Mar 2023 18:03:43 +0000 (GMT)"],"DKIM-Signature":["v=1; a=rsa-sha256; c=relaxed/simple; d=libcamera.org;\n\ts=mail; t=1679681026;\n\tbh=XwBy/zy8wfB24bXIoQ//jbDzO2kiSPZr7mAC570zz5w=;\n\th=To:Date:In-Reply-To:References:Subject:List-Id:List-Unsubscribe:\n\tList-Archive:List-Post:List-Help:List-Subscribe:From:Reply-To:\n\tFrom;\n\tb=VHF9TNjRFZic6/5w32dPkvS9qgZKoWi0Hdbxif6vcLcIOFa/J7UaxxCo78R76aGzt\n\trTPoPUwtJ9yAAnOQiyphpmU7m6TK5ODz/6JWYZ/glK5GcBDDkTeu+CwR/ZwupA3WUp\n\tUM55SFG05THLsw5mzr5G9AXslqzxGDhW8T6Md2W/FUk2K18stVG1yNx93hRA6NbHfk\n\t6Odev70187XCFXY+yIfi7q/hadAvyN+zgm2moAJSdAHTVCUHHWnpKWhb+NqybCWG0A\n\tuW0HaFB02ymeBU1usHh+PSaPetvOtejTPO9YDXkrKWEgc+Mq1z4sX1VC4c/7frLVk9\n\tIjGvB51C4Wv9w==","v=1; a=rsa-sha256; c=relaxed/simple; d=collabora.com;\n\ts=mail; t=1679681024;\n\tbh=XwBy/zy8wfB24bXIoQ//jbDzO2kiSPZr7mAC570zz5w=;\n\th=Subject:From:To:Date:In-Reply-To:References:From;\n\tb=VOYxEkHmx3+QlQbLgoVsBJI+4Ti0y2z4b7nvTgt+e8DfAy06hciAWgHEMmYcRvI40\n\tcZrVgwOiOBmGvQOpWzsaRisDhPe7Yp7fBus4sNRCV3GENFWNuE9pQITaEIg4wqsK6K\n\tU7aQhQUJZADRoxVVlvAglEf9U0CqmGn5gIquZDk/519QJRdLcXTt32W2uBB9yvG2+L\n\t6pMG5TcHN+yuUhmUfCr4/6XbDj9FRV4Czlqhzcxg0GBuC41NWYyMwOaBIQG6SrY5GH\n\tMiBN6He9uMqtbA9HgSow3AsczxJ6CSs6KCz6K3wm4a034vQhKSI7pNdJ8CnzkIvuzk\n\t/8wA5tybh0W2Q=="],"Authentication-Results":"lancelot.ideasonboard.com; dkim=pass (2048-bit key; \n\tunprotected) header.d=collabora.com\n\theader.i=@collabora.com\n\theader.b=\"VOYxEkHm\"; dkim-atps=neutral","Message-ID":"<425de849757b8cdd66b5e40da82a2d437c02d790.camel@collabora.com>","To":"libcamera-devel@lists.libcamera.org","Date":"Fri, 24 Mar 2023 14:03:35 -0400","In-Reply-To":"<20230324174009.300123-2-nicolas@ndufresne.ca>","References":"<20230324174009.300123-1-nicolas@ndufresne.ca>\n\t<20230324174009.300123-2-nicolas@ndufresne.ca>","Content-Type":"text/plain; charset=\"UTF-8\"","Content-Transfer-Encoding":"quoted-printable","User-Agent":"Evolution 3.46.4 (3.46.4-1.fc37) ","MIME-Version":"1.0","Subject":"Re: [libcamera-devel] [PATCH v1 1/3] gstreamer: Add sensor mode\n\tselection","X-BeenThere":"libcamera-devel@lists.libcamera.org","X-Mailman-Version":"2.1.29","Precedence":"list","List-Id":"<libcamera-devel.lists.libcamera.org>","List-Unsubscribe":"<https://lists.libcamera.org/options/libcamera-devel>,\n\t<mailto:libcamera-devel-request@lists.libcamera.org?subject=unsubscribe>","List-Archive":"<https://lists.libcamera.org/pipermail/libcamera-devel/>","List-Post":"<mailto:libcamera-devel@lists.libcamera.org>","List-Help":"<mailto:libcamera-devel-request@lists.libcamera.org?subject=help>","List-Subscribe":"<https://lists.libcamera.org/listinfo/libcamera-devel>,\n\t<mailto:libcamera-devel-request@lists.libcamera.org?subject=subscribe>","From":"Nicolas Dufresne via libcamera-devel\n\t<libcamera-devel@lists.libcamera.org>","Reply-To":"Nicolas Dufresne <nicolas.dufresne@collabora.com>","Errors-To":"libcamera-devel-bounces@lists.libcamera.org","Sender":"\"libcamera-devel\" <libcamera-devel-bounces@lists.libcamera.org>"}}]