Patch Detail
Show a patch.
GET /api/patches/22099/?format=api
{ "id": 22099, "url": "https://patchwork.libcamera.org/api/patches/22099/?format=api", "web_url": "https://patchwork.libcamera.org/patch/22099/", "project": { "id": 1, "url": "https://patchwork.libcamera.org/api/projects/1/?format=api", "name": "libcamera", "link_name": "libcamera", "list_id": "libcamera_core", "list_email": "libcamera-devel@lists.libcamera.org", "web_url": "", "scm_url": "", "webscm_url": "" }, "msgid": "<20241126121706.4350-4-david.plowman@raspberrypi.com>", "date": "2024-11-26T12:17:06", "name": "[RFC,3/3] libcamera: v4l2: Add wallclock metadata to video devices", "commit_ref": null, "pull_url": null, "state": "superseded", "archived": false, "hash": "256d8e42e4f556c2ecec93b3a7b33641cd663dfa", "submitter": { "id": 42, "url": "https://patchwork.libcamera.org/api/people/42/?format=api", "name": "David Plowman", "email": "david.plowman@raspberrypi.com" }, "delegate": null, "mbox": "https://patchwork.libcamera.org/patch/22099/mbox/", "series": [ { "id": 4822, "url": "https://patchwork.libcamera.org/api/series/4822/?format=api", "web_url": "https://patchwork.libcamera.org/project/libcamera/list/?series=4822", "date": "2024-11-26T12:17:03", "name": "Frame wallclock timestamps and metadata", "version": 1, "mbox": "https://patchwork.libcamera.org/series/4822/mbox/" } ], "comments": "https://patchwork.libcamera.org/api/patches/22099/comments/", "check": "pending", "checks": "https://patchwork.libcamera.org/api/patches/22099/checks/", "tags": {}, "headers": { "Return-Path": "<libcamera-devel-bounces@lists.libcamera.org>", "X-Original-To": "parsemail@patchwork.libcamera.org", "Delivered-To": "parsemail@patchwork.libcamera.org", "Received": [ "from lancelot.ideasonboard.com (lancelot.ideasonboard.com\n\t[92.243.16.209])\n\tby patchwork.libcamera.org (Postfix) with ESMTPS id D69B1C0DA4\n\tfor <parsemail@patchwork.libcamera.org>;\n\tTue, 26 Nov 2024 12:17:19 +0000 (UTC)", "from lancelot.ideasonboard.com (localhost [IPv6:::1])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTP id 9D10466070;\n\tTue, 26 Nov 2024 13:17:16 +0100 (CET)", "from mail-wr1-x435.google.com (mail-wr1-x435.google.com\n\t[IPv6:2a00:1450:4864:20::435])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id D337B66069\n\tfor <libcamera-devel@lists.libcamera.org>;\n\tTue, 26 Nov 2024 13:17:12 +0100 (CET)", "by mail-wr1-x435.google.com with SMTP id\n\tffacd0b85a97d-3825c05cc90so3607333f8f.1\n\tfor <libcamera-devel@lists.libcamera.org>;\n\tTue, 26 Nov 2024 04:17:12 -0800 (PST)", "from raspberrypi.pitowers.org\n\t([2a00:1098:3142:1f:c68a:6be1:5ba3:eddd])\n\tby smtp.gmail.com with ESMTPSA id\n\t5b1f17b1804b1-434a15d86a4sm51070325e9.36.2024.11.26.04.17.10\n\t(version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256);\n\tTue, 26 Nov 2024 04:17:10 -0800 (PST)" ], "Authentication-Results": "lancelot.ideasonboard.com; dkim=pass (2048-bit key;\n\tunprotected) header.d=raspberrypi.com header.i=@raspberrypi.com\n\theader.b=\"R6p3T/dr\"; dkim-atps=neutral", "DKIM-Signature": "v=1; a=rsa-sha256; c=relaxed/relaxed;\n\td=raspberrypi.com; s=google; t=1732623432; x=1733228232;\n\tdarn=lists.libcamera.org; \n\th=content-transfer-encoding:mime-version:references:in-reply-to\n\t:message-id:date:subject:cc:to:from:from:to:cc:subject:date\n\t:message-id:reply-to;\n\tbh=E3I3DGbsQGhBGHn93NSpiB8CG8LBqlqFxrF4Mwh1Yc8=;\n\tb=R6p3T/druiQLii0WSZfSN9owqcYdtlvTzYQwMwGwZhue+ZGnM81WMwIPwrFnsYRuN3\n\tkOcY19N6FZTPLb0XaGs9EaP7HOaqOhO6BQDIjF98gI7EpztTB3BJ60/6FVHZ+UyxEOrN\n\tr5pRxkwifUH+cqBL8jOzU+EPEh4G7BJZvaRb98E69nm8y+WhsSNLSB+abnmzIuTLqAKY\n\tVwavd9qt+s+VshCvUj86OGNt90WJHNSsCSGQI3GuaJk1lyXL+30pWSaqPOd/rYfxo842\n\tLdQpocX3T/Yx5xRoGxDB1iLWbGhp0qq2Oa6C0ls/I3r5Pu6kTKPDjKcrvBPf+7Vf8BvZ\n\tpKxg==", "X-Google-DKIM-Signature": "v=1; a=rsa-sha256; c=relaxed/relaxed;\n\td=1e100.net; s=20230601; t=1732623432; x=1733228232;\n\th=content-transfer-encoding:mime-version:references:in-reply-to\n\t:message-id:date:subject:cc:to:from:x-gm-message-state:from:to:cc\n\t:subject:date:message-id:reply-to;\n\tbh=E3I3DGbsQGhBGHn93NSpiB8CG8LBqlqFxrF4Mwh1Yc8=;\n\tb=UpfjG0M8F9wHvpbQr5ujGakgGB1GqSrMcV3+Po7qysEnSkjiKux4Dkf1CJZOAsHjoQ\n\thpc/C38xxawQAyUFCG+WYu+6c9uGgp1jf47S7/fhbz5VCCQB+6UODW8u+vv8LWLjFUVs\n\tpA8PSg4z533ypYatngAHOaRTfqBbZNJjxjRFNM5Z9IatbMUwodw+tcwE1z5zq3S+EedD\n\tZE7JdzB8NY65E1xfPQ/HECK6sbBEi5P9PE2KXh9MN8IY7yMcdH6XK1I7YOdWb6qw1601\n\twlphrQvAkglf+n55HQGA2ANkssdb7WPz1QxgIrltRovvNUwAKh1glTox/qNOw7cX6WXl\n\tRuuQ==", "X-Gm-Message-State": "AOJu0Yz3Dkmov1j+ohE69opIYIy44/OJrAcorVnMG+TfUujez7ZuEY+G\n\tUYcUl+b7qbbmvG+IHGcw1T6CW32+37M8Ia9mkA0OzfOwDV7c6Er/dxrf8lbsnSPOgyK7BccEC0B\n\tn", "X-Gm-Gg": "ASbGnctcVo/cJu+KPkuvyZ9btq30SuqdwBrFhxSUOMhRcp+CAPUVa4qlZhehWS9Luff\n\tRUwUNoG1z66Pa+DWaO8yNuQ235JwkNnvI+r4ekTFSjxekUFSlNdxB9AcuZxfoE309gdSHIs1dzx\n\tzWZ2HwX65s7ANBma9VKGTh0F4uqYnzNYvAOQkMTvInPti2oI7cVI+xO3SoXsU/Ok2GupNlJCObk\n\tDdmMa/4aLgGNkvUhQ3+b8+kwyuonxJDjOzjiveGGi+iw1iogZBTybU3D5R881apIhUgJ07GlSdc\n\tScezKA==", "X-Google-Smtp-Source": "AGHT+IF52tpJMal+t5FhVB6TTekZ3zvTRlSt2Lz523dIcLU961Hvjyta3OOiGBUagtca1/Fek5Sadg==", "X-Received": "by 2002:a5d:6d82:0:b0:382:4b6f:24f5 with SMTP id\n\tffacd0b85a97d-38260bce4femr14359745f8f.41.1732623431742; \n\tTue, 26 Nov 2024 04:17:11 -0800 (PST)", "From": "David Plowman <david.plowman@raspberrypi.com>", "To": "libcamera-devel@lists.libcamera.org", "Cc": "David Plowman <david.plowman@raspberrypi.com>", "Subject": "[RFC PATCH 3/3] libcamera: v4l2: Add wallclock metadata to video\n\tdevices", "Date": "Tue, 26 Nov 2024 12:17:06 +0000", "Message-Id": "<20241126121706.4350-4-david.plowman@raspberrypi.com>", "X-Mailer": "git-send-email 2.39.5", "In-Reply-To": "<20241126121706.4350-1-david.plowman@raspberrypi.com>", "References": "<20241126121706.4350-1-david.plowman@raspberrypi.com>", "MIME-Version": "1.0", "Content-Transfer-Encoding": "8bit", "X-BeenThere": "libcamera-devel@lists.libcamera.org", "X-Mailman-Version": "2.1.29", "Precedence": "list", "List-Id": "<libcamera-devel.lists.libcamera.org>", "List-Unsubscribe": "<https://lists.libcamera.org/options/libcamera-devel>,\n\t<mailto:libcamera-devel-request@lists.libcamera.org?subject=unsubscribe>", "List-Archive": "<https://lists.libcamera.org/pipermail/libcamera-devel/>", "List-Post": "<mailto:libcamera-devel@lists.libcamera.org>", "List-Help": "<mailto:libcamera-devel-request@lists.libcamera.org?subject=help>", "List-Subscribe": "<https://lists.libcamera.org/listinfo/libcamera-devel>,\n\t<mailto:libcamera-devel-request@lists.libcamera.org?subject=subscribe>", "Errors-To": "libcamera-devel-bounces@lists.libcamera.org", "Sender": "\"libcamera-devel\" <libcamera-devel-bounces@lists.libcamera.org>" }, "content": "FrameMetadata is extended to include wallclock timestamps, both\n\"raw\" (unsmoothed) and de-jittered versions.\n\nThe \"raw\" wallclock timestamps are recorded when frames start, and at\nframe end we generate a de-jittered version of this wallclock, handing\nboth out to pipeline handlers.\n\nSigned-off-by: David Plowman <david.plowman@raspberrypi.com>\n---\n include/libcamera/framebuffer.h | 2 ++\n include/libcamera/internal/v4l2_device.h | 4 +++\n include/libcamera/internal/v4l2_videodevice.h | 3 ++\n src/libcamera/v4l2_device.cpp | 22 ++++++++++++++\n src/libcamera/v4l2_videodevice.cpp | 29 +++++++++++++++++++\n 5 files changed, 60 insertions(+)", "diff": "diff --git a/include/libcamera/framebuffer.h b/include/libcamera/framebuffer.h\nindex ff839243..a181d97a 100644\n--- a/include/libcamera/framebuffer.h\n+++ b/include/libcamera/framebuffer.h\n@@ -35,6 +35,8 @@ struct FrameMetadata {\n \tStatus status;\n \tunsigned int sequence;\n \tuint64_t timestamp;\n+\tuint64_t wallClockRaw;\n+\tuint64_t wallClock;\n \n \tSpan<Plane> planes() { return planes_; }\n \tSpan<const Plane> planes() const { return planes_; }\ndiff --git a/include/libcamera/internal/v4l2_device.h b/include/libcamera/internal/v4l2_device.h\nindex f5aa5024..90dbc4a8 100644\n--- a/include/libcamera/internal/v4l2_device.h\n+++ b/include/libcamera/internal/v4l2_device.h\n@@ -10,6 +10,7 @@\n #include <map>\n #include <memory>\n #include <optional>\n+#include <queue>\n #include <vector>\n \n #include <linux/videodev2.h>\n@@ -67,6 +68,9 @@ protected:\n \ttemplate<typename T>\n \tstatic int fromColorSpace(const std::optional<ColorSpace> &colorSpace, T &v4l2Format);\n \n+\tstd::queue<std::pair<uint64_t, uint64_t>> wallClockQueue_;\n+\tbool frameStartEnabled() const { return frameStartEnabled_; }\n+\n private:\n \tstatic ControlType v4l2CtrlType(uint32_t ctrlType);\n \tstatic std::unique_ptr<ControlId> v4l2ControlId(const v4l2_query_ext_ctrl &ctrl);\ndiff --git a/include/libcamera/internal/v4l2_videodevice.h b/include/libcamera/internal/v4l2_videodevice.h\nindex f021c2a0..d6127928 100644\n--- a/include/libcamera/internal/v4l2_videodevice.h\n+++ b/include/libcamera/internal/v4l2_videodevice.h\n@@ -31,6 +31,7 @@\n #include <libcamera/geometry.h>\n #include <libcamera/pixel_format.h>\n \n+#include \"libcamera/internal/clock_recovery.h\"\n #include \"libcamera/internal/formats.h\"\n #include \"libcamera/internal/v4l2_device.h\"\n #include \"libcamera/internal/v4l2_pixelformat.h\"\n@@ -290,6 +291,8 @@ private:\n \n \tTimer watchdog_;\n \tutils::Duration watchdogDuration_;\n+\n+\tClockRecovery wallClockRecovery_;\n };\n \n class V4L2M2MDevice\ndiff --git a/src/libcamera/v4l2_device.cpp b/src/libcamera/v4l2_device.cpp\nindex 7d21cf15..285527b4 100644\n--- a/src/libcamera/v4l2_device.cpp\n+++ b/src/libcamera/v4l2_device.cpp\n@@ -474,6 +474,11 @@ int V4L2Device::ioctl(unsigned long request, void *argp)\n * \\return The V4L2 device file descriptor, -1 if the device node is not open\n */\n \n+/**\n+ * \\fn V4L2Device::frameStartEnabled()\n+ * \\return True if frame start notifications are enabled, otherwise false\n+ */\n+\n /**\n * \\brief Retrieve the libcamera control type associated with the V4L2 control\n * \\param[in] ctrlType The V4L2 control type\n@@ -761,6 +766,23 @@ void V4L2Device::eventAvailable()\n \t\treturn;\n \t}\n \n+\t/*\n+\t * Record this frame (by its sequence number) and its corresponding wallclock value.\n+\t * Use a queue as these two events may not interleave perfectly.\n+\t */\n+\tauto now = std::chrono::system_clock::now();\n+\tuint64_t wallClock = std::chrono::duration_cast<std::chrono::microseconds>(now.time_since_epoch()).count();\n+\n+\twallClockQueue_.emplace(event.u.frame_sync.frame_sequence, wallClock);\n+\n+\t/*\n+\t * Also avoid growing the queue indefiniteily. It seems highly unlikely that you could\n+\t * get more than a few \"frame starts\" being processed without a \"frame end\", so the value\n+\t * 5, while arbitrary, appears to be more than enough in practice.\n+\t */\n+\twhile (wallClockQueue_.size() > 5)\n+\t\twallClockQueue_.pop();\n+\n \tframeStart.emit(event.u.frame_sync.frame_sequence);\n }\n \ndiff --git a/src/libcamera/v4l2_videodevice.cpp b/src/libcamera/v4l2_videodevice.cpp\nindex 14eba056..5c5dfc79 100644\n--- a/src/libcamera/v4l2_videodevice.cpp\n+++ b/src/libcamera/v4l2_videodevice.cpp\n@@ -1865,6 +1865,33 @@ FrameBuffer *V4L2VideoDevice::dequeueBuffer()\n \tmetadata.timestamp = buf.timestamp.tv_sec * 1000000000ULL\n \t\t\t + buf.timestamp.tv_usec * 1000ULL;\n \n+\tif (frameStartEnabled()) {\n+\t\t/*\n+\t\t * Find the wallclock that should have been recorded for this frame, discarding any\n+\t\t * stale frames on the way.\n+\t\t */\n+\t\twhile (!wallClockQueue_.empty() && wallClockQueue_.front().first < buf.sequence)\n+\t\t\twallClockQueue_.pop();\n+\n+\t\tif (!wallClockQueue_.empty() && wallClockQueue_.front().first == buf.sequence) {\n+\t\t\tmetadata.wallClockRaw = wallClockQueue_.front().second;\n+\t\t\twallClockQueue_.pop();\n+\t\t} else {\n+\t\t\t/*\n+\t\t\t * At higher framerates it can happen that this gets handled before the frame\n+\t\t\t * start event, meaning there's no wallclock time in the queue. So the best we\n+\t\t\t * can do is sample the wallclock now. (The frame start will subsequently add\n+\t\t\t * another wallclock timestamp, but this will get safely discarded.)\n+\t\t\t */\n+\t\t\tauto now = std::chrono::system_clock::now();\n+\t\t\tmetadata.wallClockRaw = std::chrono::duration_cast<std::chrono::microseconds>(now.time_since_epoch()).count();\n+\t\t}\n+\n+\t\t/* Now recover a de-jittered wallclock value. Everything must be in microseconds. */\n+\t\twallClockRecovery_.addSample(metadata.timestamp / 1000, metadata.wallClockRaw);\n+\t\tmetadata.wallClock = wallClockRecovery_.getOutput(metadata.timestamp / 1000);\n+\t}\n+\n \tif (V4L2_TYPE_IS_OUTPUT(buf.type))\n \t\treturn buffer;\n \n@@ -1958,6 +1985,8 @@ int V4L2VideoDevice::streamOn()\n \n \tfirstFrame_.reset();\n \n+\twallClockQueue_ = {};\n+\n \tret = ioctl(VIDIOC_STREAMON, &bufferType_);\n \tif (ret < 0) {\n \t\tLOG(V4L2, Error)\n", "prefixes": [ "RFC", "3/3" ] }