Patch Detail
Show a patch.
GET /api/1.1/patches/17272/?format=api
{ "id": 17272, "url": "https://patchwork.libcamera.org/api/1.1/patches/17272/?format=api", "web_url": "https://patchwork.libcamera.org/patch/17272/", "project": { "id": 1, "url": "https://patchwork.libcamera.org/api/1.1/projects/1/?format=api", "name": "libcamera", "link_name": "libcamera", "list_id": "libcamera_core", "list_email": "libcamera-devel@lists.libcamera.org", "web_url": "", "scm_url": "", "webscm_url": "" }, "msgid": "<20220902055049.15183-1-rishikeshdonadkar@gmail.com>", "date": "2022-09-02T05:50:49", "name": "[libcamera-devel,v2] gstreamer: Provide framerate support for libcamerasrc.", "commit_ref": null, "pull_url": null, "state": "superseded", "archived": false, "hash": "078981fd79866b1815b119a7a9db9675fad97671", "submitter": { "id": 118, "url": "https://patchwork.libcamera.org/api/1.1/people/118/?format=api", "name": "Rishikesh Donadkar", "email": "rishikeshdonadkar@gmail.com" }, "delegate": null, "mbox": "https://patchwork.libcamera.org/patch/17272/mbox/", "series": [ { "id": 3458, "url": "https://patchwork.libcamera.org/api/1.1/series/3458/?format=api", "web_url": "https://patchwork.libcamera.org/project/libcamera/list/?series=3458", "date": "2022-09-02T05:50:49", "name": "[libcamera-devel,v2] gstreamer: Provide framerate support for libcamerasrc.", "version": 2, "mbox": "https://patchwork.libcamera.org/series/3458/mbox/" } ], "comments": "https://patchwork.libcamera.org/api/patches/17272/comments/", "check": "pending", "checks": "https://patchwork.libcamera.org/api/patches/17272/checks/", "tags": {}, "headers": { "Return-Path": "<libcamera-devel-bounces@lists.libcamera.org>", "X-Original-To": "parsemail@patchwork.libcamera.org", "Delivered-To": "parsemail@patchwork.libcamera.org", "Received": [ "from lancelot.ideasonboard.com (lancelot.ideasonboard.com\n\t[92.243.16.209])\n\tby patchwork.libcamera.org (Postfix) with ESMTPS id D35E7C3272\n\tfor <parsemail@patchwork.libcamera.org>;\n\tFri, 2 Sep 2022 05:51:11 +0000 (UTC)", "from lancelot.ideasonboard.com (localhost [IPv6:::1])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTP id 2EAE561FDA;\n\tFri, 2 Sep 2022 07:51:11 +0200 (CEST)", "from mail-pj1-x102c.google.com (mail-pj1-x102c.google.com\n\t[IPv6:2607:f8b0:4864:20::102c])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id 7E0DF61F99\n\tfor <libcamera-devel@lists.libcamera.org>;\n\tFri, 2 Sep 2022 07:51:10 +0200 (CEST)", "by mail-pj1-x102c.google.com with SMTP id\n\tz3-20020a17090abd8300b001fd803e34f1so4530759pjr.1\n\tfor <libcamera-devel@lists.libcamera.org>;\n\tThu, 01 Sep 2022 22:51:10 -0700 (PDT)", "from localhost.localdomain ([49.36.101.190])\n\tby smtp.googlemail.com with ESMTPSA id\n\tfy10-20020a17090b020a00b001fb18855440sm610496pjb.31.2022.09.01.22.51.06\n\t(version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256);\n\tThu, 01 Sep 2022 22:51:08 -0700 (PDT)" ], "DKIM-Signature": [ "v=1; a=rsa-sha256; c=relaxed/simple; d=libcamera.org;\n\ts=mail; t=1662097871;\n\tbh=YZhWPtrCBSyJjRr7SKpLTzIlaSEdbc0Mfi1qrq4vQGY=;\n\th=To:Date:Subject:List-Id:List-Unsubscribe:List-Archive:List-Post:\n\tList-Help:List-Subscribe:From:Reply-To:Cc:From;\n\tb=3lVi1ViY4FUxIMEg24VPfRTQur/UXNqEoNKYCdKfgAqk+HUnLhMr4lZGn5ks1wYkW\n\tjWzurvCF7frdyhjNfNLfum9g/D0/NsifAp2D6SSOgxG0E2kGY+2igJiy7y2b69L0Y+\n\tKidkqFX+/c1TjCtUNakqR6vnkyI91yxvPbTzy+CZEbSSCSDsYkXo/IcZoHuDUuYY4F\n\tB3z2UerKMxkBXXKvl0I3uehuy85uzbMzDVJvIrh2eJW/g+I1+LWPqQPf44nXh6yWz4\n\t8kC8C3zvTBar5VEBcjpk3yQlf1GSq9UzSjxxoJbZQCueu6o3v4dhx0pYNDNtTZgmQe\n\tXOAx0Fxsk4s7A==", "v=1; a=rsa-sha256; c=relaxed/relaxed; d=gmail.com; s=20210112;\n\th=content-transfer-encoding:mime-version:message-id:date:subject:cc\n\t:to:from:from:to:cc:subject:date;\n\tbh=bP9RXE5UQsl8VPOeVrdR9c/RC6UhFv9CGnys6DQju7A=;\n\tb=qzssOw0AukM2LoK6vpq/p1THTLKpnbdfHGzbp5iX8noVSRZUB0CVPDigd3z9hhvQp+\n\tqKusRJ80fA0kWdTFJNt2x5nPzrVftcROvkyVHU86hopha/etx6Od4uFrsXZRyCqHGJsP\n\t1PS1SwMhGsgNoVuOAbKRiIOdH/ExTDY3BGAhYNneb1zUqmhbOcmSyArW8M5D0c3iHipa\n\tN0BLOCesVHwzxDKvZ8YRNeRO9RJSfK+wTkfWVh+oDDgY8vobs9RL3+Gi2qOrcYS6mk1C\n\twzR9JACyWRrfyzMyQY291TkH8LRi8+vgRuvtBkvU9t8Q2floPYWo3pnrczPAYDZ2WQuZ\n\tB5cA==" ], "Authentication-Results": "lancelot.ideasonboard.com; dkim=pass (2048-bit key; \n\tunprotected) header.d=gmail.com header.i=@gmail.com\n\theader.b=\"qzssOw0A\"; dkim-atps=neutral", "X-Google-DKIM-Signature": "v=1; a=rsa-sha256; c=relaxed/relaxed;\n\td=1e100.net; s=20210112;\n\th=content-transfer-encoding:mime-version:message-id:date:subject:cc\n\t:to:from:x-gm-message-state:from:to:cc:subject:date;\n\tbh=bP9RXE5UQsl8VPOeVrdR9c/RC6UhFv9CGnys6DQju7A=;\n\tb=TLx3k86NiNWKaPNiTHpA1FcAGaSFLanMrUnCxquoXme9WGo9gpyxXY5YtWJIux3b53\n\tWE7ezBWbPXz0g2nqaZr/BgbE24vMIwY3dHk8dMM6WWPykQQRb25E5PqTSbDSmckP4BB8\n\tbXhD82Lmk6RAVA7NFl+QNysXGFTRPUZ2Hnj8ngh2Z1f+3ex1WYJu9uEooKrKjS9ULQNv\n\t1Po/q30PmxDcDEekmSqjKpK+jlXSLXDDBlsqiJUhJRoV3wWdtgKkXwk7ktBBGGSRg+FR\n\tGJ2EcxHRAissCHjvFQDpVtPR7O7IkFYCw8QdBmHDvPtvOAwYnpycuIV/Cq+z+FlqE8vE\n\tjakg==", "X-Gm-Message-State": "ACgBeo3/Sdk3wiM32xFab2Q9sEc8HxRC21UkNUMzv0EQqrnaFIC5hx76\n\tAr28kh8Vcze6Y7vjFZKdMpJn9VrBXA8=", "X-Google-Smtp-Source": "AA6agR7+6bupM4y8Kf7VlZqRYHCiT/jzZIlQqsIC2c52gyGeq8vax8zCQMJmHgwcmg3drH2O3ZGlGg==", "X-Received": "by 2002:a17:902:ab55:b0:173:4116:8d3d with SMTP id\n\tij21-20020a170902ab5500b0017341168d3dmr33451088plb.157.1662097868433; \n\tThu, 01 Sep 2022 22:51:08 -0700 (PDT)", "To": "libcamera-devel@lists.libcamera.org", "Date": "Fri, 2 Sep 2022 11:20:49 +0530", "Message-Id": "<20220902055049.15183-1-rishikeshdonadkar@gmail.com>", "X-Mailer": "git-send-email 2.25.1", "MIME-Version": "1.0", "Content-Transfer-Encoding": "8bit", "Subject": "[libcamera-devel] [PATCH v2] gstreamer: Provide framerate support\n\tfor libcamerasrc.", "X-BeenThere": "libcamera-devel@lists.libcamera.org", "X-Mailman-Version": "2.1.29", "Precedence": "list", "List-Id": "<libcamera-devel.lists.libcamera.org>", "List-Unsubscribe": "<https://lists.libcamera.org/options/libcamera-devel>,\n\t<mailto:libcamera-devel-request@lists.libcamera.org?subject=unsubscribe>", "List-Archive": "<https://lists.libcamera.org/pipermail/libcamera-devel/>", "List-Post": "<mailto:libcamera-devel@lists.libcamera.org>", "List-Help": "<mailto:libcamera-devel-request@lists.libcamera.org?subject=help>", "List-Subscribe": "<https://lists.libcamera.org/listinfo/libcamera-devel>,\n\t<mailto:libcamera-devel-request@lists.libcamera.org?subject=subscribe>", "From": "Rishikesh Donadkar via libcamera-devel\n\t<libcamera-devel@lists.libcamera.org>", "Reply-To": "Rishikesh Donadkar <rishikeshdonadkar@gmail.com>", "Cc": "Rishikesh Donadkar <rishikeshdonadkar@gmail.com>,\n\tnicolas.dufresne@collabora.com, vedantparanjape160201@gmail.com", "Errors-To": "libcamera-devel-bounces@lists.libcamera.org", "Sender": "\"libcamera-devel\" <libcamera-devel-bounces@lists.libcamera.org>" }, "content": "Add a field of the type ControlList to GstLibcameraSrcState.\n\nGet the framerate from the caps and convert it to FrameDuration.\nSet the FrameDurationLimits control in the initControls_\n\nPassing initCtrls_ at camera::start() doesn't tell us if the controls\nhave been applied to the camera successfully or not (or they have adjusted in\nsome fashion). Until that is introduced in camera::start() API,\nwe need to carry out such handling on the application side.\nCheck the frame_duration whether it is in-bound to the max / min\nframe-duration supported by the camera.\nIf the framer_duartion is out of bounds, bind the frame_duration\nbetween the max and min FrameDuration that is supported by the camera.\n\nPass the initControls_ at the time of starting camera.\n\nChange in the existing code structure[RFC]: Shift the call to\ncamera::cofigure() above the for loop which does not touch\nanything form libcamera just reads from it. This is done because\nthe bounds checking for frame duration cannot be done without the\ncall to camera::configure(). This will ease exposing the framerate to\nGStreamer. The framerate can be exposed with the same new CAPS event\nwhich is exposing the width, height and the colorimetry.\n\nSolve the complications in exposing the correct framerate due to loss in\nprecision as a result of casting the frameduration to int64_t(which is\nrequired in libcamera to set the FrameDurationLimits control).\n\nExample -\n\n* Suppose the framerate requested is 35/1. The framerate is read form\n the caps in the from of fraction that has a numerator and\n denominator.\n\n* Converting to FrameDuration (in microseconds)\n (1 * 10^6) / 35 = 28571.4286\n int64_t frame_duration = 28571\n (the precision here is lost.)\n* To expose the framerate in caps, Inverting the frame_duration to get\n back the framerate and converting to Seconds.\n double framerate = 10^6 / 28571\n and\n 10^6/28571 which is close but not equal to 35/1 will fail the negotiation.\n\nTo solve the above problem, introduce a global std::pair<int, int> variable that\nstores the numerator and denominator of the framerate in the negotiatied caps.\nTry to apply the framerate and check if the floor value of framerate that gets applied\nclosely matches with the floor value framerate requested in the negotiatied caps. If\nthe value matches set the framerate that is stored in the global container, else set the\nframerate to 0/1.\n\n---\nTesting using fpsdisplaysink element on RPi 4 + OV5647.\nhttps://paste.debian.net/1252473/\nhttps://paste.debian.net/1252474/\nframerate out of max/min bounds supported by the camera:\nhttps://paste.debian.net/1252475/\n---\n\nSigned-off-by: Umang Jain <umang.jain@ideasonboard.com>\nSigned-off-by: Rishikesh Donadkar <rishikeshdonadkar@gmail.com>\n---\n src/gstreamer/gstlibcamera-utils.cpp | 63 ++++++++++++++++++++++++++++\n src/gstreamer/gstlibcamera-utils.h | 5 +++\n src/gstreamer/gstlibcamerasrc.cpp | 27 +++++++-----\n 3 files changed, 85 insertions(+), 10 deletions(-)", "diff": "diff --git a/src/gstreamer/gstlibcamera-utils.cpp b/src/gstreamer/gstlibcamera-utils.cpp\nindex 4df5dd6c..d657be70 100644\n--- a/src/gstreamer/gstlibcamera-utils.cpp\n+++ b/src/gstreamer/gstlibcamera-utils.cpp\n@@ -8,10 +8,13 @@\n \n #include \"gstlibcamera-utils.h\"\n \n+#include <libcamera/control_ids.h>\n #include <libcamera/formats.h>\n \n using namespace libcamera;\n \n+static std::pair<int, int> framerate_container = { -1, -1 };\n+\n static struct {\n \tGstVideoFormat gst_format;\n \tPixelFormat format;\n@@ -405,6 +408,66 @@ gst_libcamera_configure_stream_from_caps(StreamConfiguration &stream_cfg,\n \t}\n }\n \n+void\n+gst_libcamera_get_init_controls_from_caps(ControlList &controls, [[maybe_unused]] GstCaps *caps)\n+{\n+\t/* read framerate from caps - convert to integer and set to frame_time. */\n+\tGstStructure *s = gst_caps_get_structure(caps, 0);\n+\tgint fps_n = -1, fps_d = -1;\n+\n+\tif (gst_structure_has_field_typed(s, \"framerate\", GST_TYPE_FRACTION)) {\n+\t\tif (!gst_structure_get_fraction(s, \"framerate\", &fps_n, &fps_d)) {\n+\t\t\tGST_WARNING(\"invalid framerate in the caps.\");\n+\t\t\treturn;\n+\t\t}\n+\t}\n+\n+\tif (fps_n < 0 || fps_d < 0)\n+\t\treturn;\n+\n+\tframerate_container = { fps_n, fps_d };\n+\tint64_t frame_duration = (fps_d * 1000000.0) / fps_n;\n+\tcontrols.set(controls::FrameDurationLimits,\n+\t\t Span<const int64_t, 2>({ frame_duration, frame_duration }));\n+}\n+\n+void\n+gst_libcamera_set_frame_duration_bounds(const ControlInfoMap &defaults, ControlList ¤t)\n+{\n+\tint64_t min_frame_duration, max_frame_duration;\n+\n+\tauto iterFrameDuration = defaults.find(controls::FrameDurationLimits.id());\n+\tif (!(iterFrameDuration != defaults.end() &&\n+\t current.contains(controls::FRAME_DURATION_LIMITS)))\n+\t\treturn;\n+\n+\tmin_frame_duration = std::max(iterFrameDuration->second.min().get<int64_t>(),\n+\t\t\t\t current.get(controls::FrameDurationLimits).value()[0]);\n+\tmax_frame_duration = std::min(iterFrameDuration->second.max().get<int64_t>(),\n+\t\t\t\t current.get(controls::FrameDurationLimits).value()[1]);\n+\n+\tcurrent.set(controls::FrameDurationLimits,\n+\t\t Span<const int64_t, 2>({ min_frame_duration, max_frame_duration }));\n+}\n+\n+void\n+gst_libcamera_controllist_to_caps(const ControlList &controls, GstCaps *caps)\n+{\n+\tgint numerator, denominator;\n+\tif (!controls.contains(controls::FRAME_DURATION_LIMITS))\n+\t\treturn;\n+\n+\tGstStructure *s = gst_caps_get_structure(caps, 0);\n+\tdouble framerate = 1000000 / static_cast<double>(controls.get(controls::FrameDurationLimits).value()[0]);\n+\tgst_util_double_to_fraction(framerate, &numerator, &denominator);\n+\n+\tif (framerate_container.first / framerate_container.second == numerator / denominator)\n+\t\tgst_structure_set(s, \"framerate\", GST_TYPE_FRACTION,\n+\t\t\t\t framerate_container.first, framerate_container.second, nullptr);\n+\telse\n+\t\tgst_structure_set(s, \"framerate\", GST_TYPE_FRACTION, 0, 1, nullptr);\n+}\n+\n #if !GST_CHECK_VERSION(1, 17, 1)\n gboolean\n gst_task_resume(GstTask *task)\ndiff --git a/src/gstreamer/gstlibcamera-utils.h b/src/gstreamer/gstlibcamera-utils.h\nindex 164189a2..a3594942 100644\n--- a/src/gstreamer/gstlibcamera-utils.h\n+++ b/src/gstreamer/gstlibcamera-utils.h\n@@ -9,6 +9,7 @@\n #pragma once\n \n #include <libcamera/camera_manager.h>\n+#include <libcamera/controls.h>\n #include <libcamera/stream.h>\n \n #include <gst/gst.h>\n@@ -18,6 +19,10 @@ GstCaps *gst_libcamera_stream_formats_to_caps(const libcamera::StreamFormats &fo\n GstCaps *gst_libcamera_stream_configuration_to_caps(const libcamera::StreamConfiguration &stream_cfg);\n void gst_libcamera_configure_stream_from_caps(libcamera::StreamConfiguration &stream_cfg,\n \t\t\t\t\t GstCaps *caps);\n+void gst_libcamera_get_init_controls_from_caps(libcamera::ControlList &controls, GstCaps *caps);\n+void gst_libcamera_set_frame_duration_bounds(const libcamera::ControlInfoMap &defaults, libcamera::ControlList ¤t);\n+void gst_libcamera_controllist_to_caps(const libcamera::ControlList &controls, GstCaps *caps);\n+\n #if !GST_CHECK_VERSION(1, 17, 1)\n gboolean gst_task_resume(GstTask *task);\n #endif\ndiff --git a/src/gstreamer/gstlibcamerasrc.cpp b/src/gstreamer/gstlibcamerasrc.cpp\nindex 16d70fea..ee9fa5e9 100644\n--- a/src/gstreamer/gstlibcamerasrc.cpp\n+++ b/src/gstreamer/gstlibcamerasrc.cpp\n@@ -131,6 +131,7 @@ struct GstLibcameraSrcState {\n \tstd::queue<std::unique_ptr<RequestWrap>> completedRequests_\n \t\tLIBCAMERA_TSA_GUARDED_BY(lock_);\n \n+\tControlList initControls_;\n \tguint group_id_;\n \n \tint queueRequest();\n@@ -496,6 +497,7 @@ gst_libcamera_src_task_enter(GstTask *task, [[maybe_unused]] GThread *thread,\n \t\t/* Retrieve the supported caps. */\n \t\tg_autoptr(GstCaps) filter = gst_libcamera_stream_formats_to_caps(stream_cfg.formats());\n \t\tg_autoptr(GstCaps) caps = gst_pad_peer_query_caps(srcpad, filter);\n+\n \t\tif (gst_caps_is_empty(caps)) {\n \t\t\tflow_ret = GST_FLOW_NOT_NEGOTIATED;\n \t\t\tbreak;\n@@ -504,6 +506,7 @@ gst_libcamera_src_task_enter(GstTask *task, [[maybe_unused]] GThread *thread,\n \t\t/* Fixate caps and configure the stream. */\n \t\tcaps = gst_caps_make_writable(caps);\n \t\tgst_libcamera_configure_stream_from_caps(stream_cfg, caps);\n+\t\tgst_libcamera_get_init_controls_from_caps(state->initControls_, caps);\n \t}\n \n \tif (flow_ret != GST_FLOW_OK)\n@@ -515,6 +518,16 @@ gst_libcamera_src_task_enter(GstTask *task, [[maybe_unused]] GThread *thread,\n \t\tgoto done;\n \t}\n \n+\tret = state->cam_->configure(state->config_.get());\n+\tif (ret) {\n+\t\tGST_ELEMENT_ERROR(self, RESOURCE, SETTINGS,\n+\t\t\t\t (\"Failed to configure camera: %s\", g_strerror(-ret)),\n+\t\t\t\t (\"Camera::configure() failed with error code %i\", ret));\n+\t\tgst_task_stop(task);\n+\t\treturn;\n+\t}\n+\tgst_libcamera_set_frame_duration_bounds(state->cam_->controls(), state->initControls_);\n+\n \t/*\n \t * Regardless if it has been modified, create clean caps and push the\n \t * caps event. Downstream will decide if the caps are acceptable.\n@@ -524,6 +537,8 @@ gst_libcamera_src_task_enter(GstTask *task, [[maybe_unused]] GThread *thread,\n \t\tconst StreamConfiguration &stream_cfg = state->config_->at(i);\n \n \t\tg_autoptr(GstCaps) caps = gst_libcamera_stream_configuration_to_caps(stream_cfg);\n+\t\tgst_libcamera_controllist_to_caps(state->initControls_, caps);\n+\n \t\tif (!gst_pad_push_event(srcpad, gst_event_new_caps(caps))) {\n \t\t\tflow_ret = GST_FLOW_NOT_NEGOTIATED;\n \t\t\tbreak;\n@@ -535,15 +550,6 @@ gst_libcamera_src_task_enter(GstTask *task, [[maybe_unused]] GThread *thread,\n \t\tgst_pad_push_event(srcpad, gst_event_new_segment(&segment));\n \t}\n \n-\tret = state->cam_->configure(state->config_.get());\n-\tif (ret) {\n-\t\tGST_ELEMENT_ERROR(self, RESOURCE, SETTINGS,\n-\t\t\t\t (\"Failed to configure camera: %s\", g_strerror(-ret)),\n-\t\t\t\t (\"Camera::configure() failed with error code %i\", ret));\n-\t\tgst_task_stop(task);\n-\t\treturn;\n-\t}\n-\n \tself->allocator = gst_libcamera_allocator_new(state->cam_, state->config_.get());\n \tif (!self->allocator) {\n \t\tGST_ELEMENT_ERROR(self, RESOURCE, NO_SPACE_LEFT,\n@@ -566,7 +572,7 @@ gst_libcamera_src_task_enter(GstTask *task, [[maybe_unused]] GThread *thread,\n \t\tgst_flow_combiner_add_pad(self->flow_combiner, srcpad);\n \t}\n \n-\tret = state->cam_->start();\n+\tret = state->cam_->start(&state->initControls_);\n \tif (ret) {\n \t\tGST_ELEMENT_ERROR(self, RESOURCE, SETTINGS,\n \t\t\t\t (\"Failed to start the camera: %s\", g_strerror(-ret)),\n@@ -576,6 +582,7 @@ gst_libcamera_src_task_enter(GstTask *task, [[maybe_unused]] GThread *thread,\n \t}\n \n done:\n+\tstate->initControls_.clear();\n \tswitch (flow_ret) {\n \tcase GST_FLOW_NOT_NEGOTIATED:\n \t\tGST_ELEMENT_FLOW_ERROR(self, flow_ret);\n", "prefixes": [ "libcamera-devel", "v2" ] }