Patch Detail
Show a patch.
GET /api/patches/23603/?format=api
{ "id": 23603, "url": "https://patchwork.libcamera.org/api/patches/23603/?format=api", "web_url": "https://patchwork.libcamera.org/patch/23603/", "project": { "id": 1, "url": "https://patchwork.libcamera.org/api/projects/1/?format=api", "name": "libcamera", "link_name": "libcamera", "list_id": "libcamera_core", "list_email": "libcamera-devel@lists.libcamera.org", "web_url": "", "scm_url": "", "webscm_url": "" }, "msgid": "<20250619100857.124809-3-naush@raspberrypi.com>", "date": "2025-06-19T10:05:54", "name": "[2/4] libcamera: Add ClockRecovery class to generate wallclock timestamps", "commit_ref": "2a4e347dfea352901cb2d609489f47157e4c2c2c", "pull_url": null, "state": "accepted", "archived": false, "hash": "9ae4352e6eb5d14c4de921ecf032200b3969c1c8", "submitter": { "id": 34, "url": "https://patchwork.libcamera.org/api/people/34/?format=api", "name": "Naushir Patuck", "email": "naush@raspberrypi.com" }, "delegate": null, "mbox": "https://patchwork.libcamera.org/patch/23603/mbox/", "series": [ { "id": 5232, "url": "https://patchwork.libcamera.org/api/series/5232/?format=api", "web_url": "https://patchwork.libcamera.org/project/libcamera/list/?series=5232", "date": "2025-06-19T10:05:52", "name": "Wallclock support", "version": 1, "mbox": "https://patchwork.libcamera.org/series/5232/mbox/" } ], "comments": "https://patchwork.libcamera.org/api/patches/23603/comments/", "check": "pending", "checks": "https://patchwork.libcamera.org/api/patches/23603/checks/", "tags": {}, "headers": { "Return-Path": "<libcamera-devel-bounces@lists.libcamera.org>", "X-Original-To": "parsemail@patchwork.libcamera.org", "Delivered-To": "parsemail@patchwork.libcamera.org", "Received": [ "from lancelot.ideasonboard.com (lancelot.ideasonboard.com\n\t[92.243.16.209])\n\tby patchwork.libcamera.org (Postfix) with ESMTPS id 50C4BC3237\n\tfor <parsemail@patchwork.libcamera.org>;\n\tThu, 19 Jun 2025 10:09:10 +0000 (UTC)", "from lancelot.ideasonboard.com (localhost [IPv6:::1])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTP id C0BE268DC6;\n\tThu, 19 Jun 2025 12:09:06 +0200 (CEST)", "from mail-wm1-x32e.google.com (mail-wm1-x32e.google.com\n\t[IPv6:2a00:1450:4864:20::32e])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id EDB5968DC6\n\tfor <libcamera-devel@lists.libcamera.org>;\n\tThu, 19 Jun 2025 12:09:02 +0200 (CEST)", "by mail-wm1-x32e.google.com with SMTP id\n\t5b1f17b1804b1-450ddb35583so534915e9.1\n\tfor <libcamera-devel@lists.libcamera.org>;\n\tThu, 19 Jun 2025 03:09:02 -0700 (PDT)", "from NAUSH-P-DELL.pitowers.org ([93.93.133.154])\n\tby smtp.gmail.com with ESMTPSA id\n\t5b1f17b1804b1-4535ebcecb5sm23926555e9.37.2025.06.19.03.09.01\n\t(version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256);\n\tThu, 19 Jun 2025 03:09:01 -0700 (PDT)" ], "Authentication-Results": "lancelot.ideasonboard.com; dkim=pass (2048-bit key;\n\tunprotected) header.d=raspberrypi.com header.i=@raspberrypi.com\n\theader.b=\"LK4yHjBY\"; dkim-atps=neutral", "DKIM-Signature": "v=1; a=rsa-sha256; c=relaxed/relaxed;\n\td=raspberrypi.com; s=google; t=1750327742; x=1750932542;\n\tdarn=lists.libcamera.org; \n\th=content-transfer-encoding:mime-version:references:in-reply-to\n\t:message-id:date:subject:cc:to:from:from:to:cc:subject:date\n\t:message-id:reply-to;\n\tbh=LpSvAHfLJ3s+U22kjYDMqV0WiShYD8LM29GMSrXDETg=;\n\tb=LK4yHjBYtNdg/2i6fv2SpyBcjtb0j7HXculovw3BAYYwqG6EL81YK/EPHdAwRgfRmT\n\tsxQWExxde7O1iVGaBJbuqwhrkJoWLK5+MjiiWUfcjlD7iOZdsJQjV2JvY8rCX4wlfmJd\n\tC9FS2aGFiIH5l/DMBd42/IfBxeHUeat4bUbx35PRkbu1JEjnzz2f3+dNbcrSLBnhnYBE\n\t4SFzct4B7iAMo/4rbeR/Vlyq5TNTu+dsuJPn9LnoDN/q0fyvGSIaRs/c/SuKka1wq86W\n\tClEpWzI05tjKQBEu3/ioKNnB8XbIN5p1pwbMoeyZX8t5bfBQFQxRkmXPCGh+wZ/Qjr1c\n\t/R1A==", "X-Google-DKIM-Signature": "v=1; a=rsa-sha256; c=relaxed/relaxed;\n\td=1e100.net; s=20230601; t=1750327742; x=1750932542;\n\th=content-transfer-encoding:mime-version:references:in-reply-to\n\t:message-id:date:subject:cc:to:from:x-gm-message-state:from:to:cc\n\t:subject:date:message-id:reply-to;\n\tbh=LpSvAHfLJ3s+U22kjYDMqV0WiShYD8LM29GMSrXDETg=;\n\tb=CSl4cODw4p9V8plpobQLMfER1Gqw0o9k2h5Ly9GqQquNNasTUGFpvaeYppSFv5zYvZ\n\tcZ/mwbQu9F3X/svrWeNOCP8BT3D1REy8vFJQRMijQ+Rn64QGA+yvThQEyTZ54NA/zQFE\n\tUpXBZTDpD5j6ZWK1G/CTZaIvnzfpuVPnQhUCIDHkMH3VkWn9MvLxQVPmXNOo7NJ05tMs\n\tVxc+N6RE3en5h22vUm1j9+6C5pKb76rkCvkLZJd8KFc/vHaRa3jSe5nSnsT82IiEjNfx\n\tI/SaDvhV9QnUkQjleIXHKPFr/NPH7iofJOKnA8TnhweniUZx1nggbSpwId5wt/EJiuRV\n\tmX0g==", "X-Gm-Message-State": "AOJu0YxWl8GU95Q73k0B1Z0nDVT2FUjvVE8m2CyLFP6Skxv+0OM/3czT\n\t6GGrxLxdGR/5xrf5AgXjnmxwGgxzpjMAQ8FKsjSH0vTPfg073+dLwqQnZ82Nyq40nmxSnnPUqlV\n\tbASDE", "X-Gm-Gg": "ASbGnctuxPIvnF4pmj+i92cq3duovRLd03Nu+1GtPtPVj1LZcvCQAhsqgTezDvQKC5q\n\tmtaKwaPiw+fqXUIOFgs6zGFGjCg0KBHCiqNgBu08YWxuBwozBwBpzZ4YrxgB2RjJtM8AR86OL6c\n\tO78wVAo1VKY4QVZ209Nc2rD/itCFthjBV1Jp39ukQxuOKJ4g2L24uaj/lhe0FeLeGGLE4WFswSE\n\t/0mPM5JbYjZv0nkvyGV3z5mfhoC6PeAUw/3nKHOe99Cj5kH+gCOaO7+AFhvs2dBLvi+6K3cD/s9\n\tqzh4sIRsKAI4Ytk9rqK06hp5KE4q5sFlBnv/yYCbBcBh8V3GJYhdMmfuxy2GwzhTrORYkosduWX\n\tQxyek/Q==", "X-Google-Smtp-Source": "AGHT+IHFlnDKgmvT9vjneCyg/TA3DOrvI/cwCPwmA+rGb91gd97oMReSGqXRRcxCre5RqlilBEWiRA==", "X-Received": "by 2002:a05:600c:a402:b0:451:dee4:cd07 with SMTP id\n\t5b1f17b1804b1-4535eee3c98mr7918565e9.0.1750327742109; \n\tThu, 19 Jun 2025 03:09:02 -0700 (PDT)", "From": "Naushir Patuck <naush@raspberrypi.com>", "To": "libcamera-devel@lists.libcamera.org", "Cc": "kieran.bingham@ideasonboard.com,\n\tDavid Plowman <david.plowman@raspberrypi.com>,\n\tNaushir Patuck <naush@raspberrypi.com>", "Subject": "[PATCH 2/4] libcamera: Add ClockRecovery class to generate wallclock\n\ttimestamps", "Date": "Thu, 19 Jun 2025 11:05:54 +0100", "Message-ID": "<20250619100857.124809-3-naush@raspberrypi.com>", "X-Mailer": "git-send-email 2.43.0", "In-Reply-To": "<20250619100857.124809-1-naush@raspberrypi.com>", "References": "<20250619100857.124809-1-naush@raspberrypi.com>", "MIME-Version": "1.0", "Content-Transfer-Encoding": "8bit", "X-BeenThere": "libcamera-devel@lists.libcamera.org", "X-Mailman-Version": "2.1.29", "Precedence": "list", "List-Id": "<libcamera-devel.lists.libcamera.org>", "List-Unsubscribe": "<https://lists.libcamera.org/options/libcamera-devel>,\n\t<mailto:libcamera-devel-request@lists.libcamera.org?subject=unsubscribe>", "List-Archive": "<https://lists.libcamera.org/pipermail/libcamera-devel/>", "List-Post": "<mailto:libcamera-devel@lists.libcamera.org>", "List-Help": "<mailto:libcamera-devel-request@lists.libcamera.org?subject=help>", "List-Subscribe": "<https://lists.libcamera.org/listinfo/libcamera-devel>,\n\t<mailto:libcamera-devel-request@lists.libcamera.org?subject=subscribe>", "Errors-To": "libcamera-devel-bounces@lists.libcamera.org", "Sender": "\"libcamera-devel\" <libcamera-devel-bounces@lists.libcamera.org>" }, "content": "From: David Plowman <david.plowman@raspberrypi.com>\n\nThe ClockRecovery class takes pairs of timestamps from two different\nclocks, and models the second (\"output\") clock from the first (\"input\")\nclock.\n\nWe can use it, in particular, to get a good wallclock estimate for a\nframe's SensorTimestamp.\n\nSigned-off-by: David Plowman <david.plowman@raspberrypi.com>\nReviewed-by: Kieran Bingham <kieran.bingham@ideasonboard.com>\nReviewed-by: Naushir Patuck <naush@raspberrypi.com>\nSigned-off-by: Naushir Patuck <naush@raspberrypi.com>\n---\n include/libcamera/internal/clock_recovery.h | 68 ++++++\n include/libcamera/internal/meson.build | 1 +\n src/libcamera/clock_recovery.cpp | 230 ++++++++++++++++++++\n src/libcamera/meson.build | 1 +\n 4 files changed, 300 insertions(+)\n create mode 100644 include/libcamera/internal/clock_recovery.h\n create mode 100644 src/libcamera/clock_recovery.cpp", "diff": "diff --git a/include/libcamera/internal/clock_recovery.h b/include/libcamera/internal/clock_recovery.h\nnew file mode 100644\nindex 000000000000..43e46b7dcf9f\n--- /dev/null\n+++ b/include/libcamera/internal/clock_recovery.h\n@@ -0,0 +1,68 @@\n+/* SPDX-License-Identifier: LGPL-2.1-or-later */\n+/*\n+ * Copyright (C) 2024, Raspberry Pi Ltd\n+ *\n+ * Camera recovery algorithm\n+ */\n+#pragma once\n+\n+#include <stdint.h>\n+\n+namespace libcamera {\n+\n+class ClockRecovery\n+{\n+public:\n+\tClockRecovery();\n+\n+\tvoid configure(unsigned int numSamples = 100, unsigned int maxJitter = 2000,\n+\t\t unsigned int minSamples = 10, unsigned int errorThreshold = 50000);\n+\tvoid reset();\n+\n+\tvoid addSample();\n+\tvoid addSample(uint64_t input, uint64_t output);\n+\n+\tuint64_t getOutput(uint64_t input);\n+\n+private:\n+\t/* Approximate number of samples over which the model state persists. */\n+\tunsigned int numSamples_;\n+\t/* Remove any output jitter larger than this immediately. */\n+\tunsigned int maxJitter_;\n+\t/* Number of samples required before we start to use model estimates. */\n+\tunsigned int minSamples_;\n+\t/* Threshold above which we assume the wallclock has been reset. */\n+\tunsigned int errorThreshold_;\n+\n+\t/* How many samples seen (up to numSamples_). */\n+\tunsigned int count_;\n+\t/* This gets subtracted from all input values, just to make the numbers easier. */\n+\tuint64_t inputBase_;\n+\t/* As above, for the output. */\n+\tuint64_t outputBase_;\n+\t/* The previous input sample. */\n+\tuint64_t lastInput_;\n+\t/* The previous output sample. */\n+\tuint64_t lastOutput_;\n+\n+\t/* Average x value seen so far. */\n+\tdouble xAve_;\n+\t/* Average y value seen so far */\n+\tdouble yAve_;\n+\t/* Average x^2 value seen so far. */\n+\tdouble x2Ave_;\n+\t/* Average x*y value seen so far. */\n+\tdouble xyAve_;\n+\n+\t/*\n+\t * The latest estimate of linear parameters to derive the output clock\n+\t * from the input.\n+\t */\n+\tdouble slope_;\n+\tdouble offset_;\n+\n+\t/* Use this cumulative error to monitor for spontaneous clock updates. */\n+\tdouble error_;\n+};\n+\n+} /* namespace libcamera */\ndiff --git a/include/libcamera/internal/meson.build b/include/libcamera/internal/meson.build\nindex 33f318b2b602..5c80a28c4cbe 100644\n--- a/include/libcamera/internal/meson.build\n+++ b/include/libcamera/internal/meson.build\n@@ -11,6 +11,7 @@ libcamera_internal_headers = files([\n 'camera_manager.h',\n 'camera_sensor.h',\n 'camera_sensor_properties.h',\n+ 'clock_recovery.h',\n 'control_serializer.h',\n 'control_validator.h',\n 'converter.h',\ndiff --git a/src/libcamera/clock_recovery.cpp b/src/libcamera/clock_recovery.cpp\nnew file mode 100644\nindex 000000000000..abacf444fbf8\n--- /dev/null\n+++ b/src/libcamera/clock_recovery.cpp\n@@ -0,0 +1,230 @@\n+/* SPDX-License-Identifier: LGPL-2.1-or-later */\n+/*\n+ * Copyright (C) 2024, Raspberry Pi Ltd\n+ *\n+ * Clock recovery algorithm\n+ */\n+\n+#include \"libcamera/internal/clock_recovery.h\"\n+\n+#include <time.h>\n+\n+#include <libcamera/base/log.h>\n+\n+/**\n+ * \\file clock_recovery.h\n+ * \\brief Clock recovery - deriving one clock from another independent clock\n+ */\n+\n+namespace libcamera {\n+\n+LOG_DEFINE_CATEGORY(ClockRec)\n+\n+/**\n+ * \\class ClockRecovery\n+ * \\brief Recover an output clock from an input clock\n+ *\n+ * The ClockRecovery class derives an output clock from an input clock,\n+ * modelling the output clock as being linearly related to the input clock.\n+ * For example, we may use it to derive wall clock timestamps from timestamps\n+ * measured by the internal system clock which counts local time since boot.\n+ *\n+ * When pairs of corresponding input and output timestamps are available,\n+ * they should be submitted to the model with addSample(). The model will\n+ * update, and output clock values for known input clock values can be\n+ * obtained using getOutput().\n+ *\n+ * As a convenience, if the input clock is indeed the time since boot, and the\n+ * output clock represents a real wallclock time, then addSample() can be\n+ * called with no arguments, and a pair of timestamps will be captured at\n+ * that moment.\n+ *\n+ * The configure() function accepts some configuration parameters to control\n+ * the linear fitting process.\n+ */\n+\n+/**\n+ * \\brief Construct a ClockRecovery\n+ */\n+ClockRecovery::ClockRecovery()\n+{\n+\tconfigure();\n+\treset();\n+}\n+\n+/**\n+ * \\brief Set configuration parameters\n+ * \\param[in] numSamples The approximate duration for which the state of the model\n+ * is persistent\n+ * \\param[in] maxJitter New output samples are clamped to no more than this\n+ * amount of jitter, to prevent sudden swings from having a large effect\n+ * \\param[in] minSamples The fitted clock model is not used to generate outputs\n+ * until this many samples have been received\n+ * \\param[in] errorThreshold If the accumulated differences between input and\n+ * output clocks reaches this amount over a few frames, the model is reset\n+ */\n+void ClockRecovery::configure(unsigned int numSamples, unsigned int maxJitter,\n+\t\t\t unsigned int minSamples, unsigned int errorThreshold)\n+{\n+\tLOG(ClockRec, Debug)\n+\t\t<< \"configure \" << numSamples << \" \" << maxJitter << \" \" << minSamples << \" \" << errorThreshold;\n+\n+\tnumSamples_ = numSamples;\n+\tmaxJitter_ = maxJitter;\n+\tminSamples_ = minSamples;\n+\terrorThreshold_ = errorThreshold;\n+}\n+\n+/**\n+ * \\brief Reset the clock recovery model and start again from scratch\n+ */\n+void ClockRecovery::reset()\n+{\n+\tLOG(ClockRec, Debug) << \"reset\";\n+\n+\tlastInput_ = 0;\n+\tlastOutput_ = 0;\n+\txAve_ = 0;\n+\tyAve_ = 0;\n+\tx2Ave_ = 0;\n+\txyAve_ = 0;\n+\tcount_ = 0;\n+\terror_ = 0.0;\n+\t/*\n+\t * Setting slope_ and offset_ to zero initially means that the clocks\n+\t * advance at exactly the same rate.\n+\t */\n+\tslope_ = 0.0;\n+\toffset_ = 0.0;\n+}\n+\n+/**\n+ * \\brief Add a sample point to the clock recovery model, for recovering a wall\n+ * clock value from the internal system time since boot\n+ *\n+ * This is a convenience function to make it easy to derive a wall clock value\n+ * (using the Linux CLOCK_REALTIME) from the time since the system started\n+ * (measured by CLOCK_BOOTTIME).\n+ */\n+void ClockRecovery::addSample()\n+{\n+\tLOG(ClockRec, Debug) << \"addSample\";\n+\n+\tstruct timespec bootTime1;\n+\tstruct timespec bootTime2;\n+\tstruct timespec wallTime;\n+\n+\t/* Get boot and wall clocks in microseconds. */\n+\tclock_gettime(CLOCK_BOOTTIME, &bootTime1);\n+\tclock_gettime(CLOCK_REALTIME, &wallTime);\n+\tclock_gettime(CLOCK_BOOTTIME, &bootTime2);\n+\tuint64_t boot1 = bootTime1.tv_sec * 1000000ULL + bootTime1.tv_nsec / 1000;\n+\tuint64_t boot2 = bootTime2.tv_sec * 1000000ULL + bootTime2.tv_nsec / 1000;\n+\tuint64_t boot = (boot1 + boot2) / 2;\n+\tuint64_t wall = wallTime.tv_sec * 1000000ULL + wallTime.tv_nsec / 1000;\n+\n+\taddSample(boot, wall);\n+}\n+\n+/**\n+ * \\brief Add a sample point to the clock recovery model, specifying the exact\n+ * input and output clock values\n+ * \\param[in] input The input clock value\n+ * \\param[in] output The value of the output clock at the same moment, as far\n+ * as possible, that the input clock was sampled\n+ *\n+ * This function should be used for corresponding clocks other than the Linux\n+ * BOOTTIME and REALTIME clocks.\n+ */\n+void ClockRecovery::addSample(uint64_t input, uint64_t output)\n+{\n+\tLOG(ClockRec, Debug) << \"addSample \" << input << \" \" << output;\n+\n+\tif (count_ == 0) {\n+\t\tinputBase_ = input;\n+\t\toutputBase_ = output;\n+\t}\n+\n+\t/*\n+\t * We keep an eye on cumulative drift over the last several frames. If this exceeds a\n+\t * threshold, then probably the system clock has been updated and we're going to have to\n+\t * reset everything and start over.\n+\t */\n+\tif (lastOutput_) {\n+\t\tint64_t inputDiff = getOutput(input) - getOutput(lastInput_);\n+\t\tint64_t outputDiff = output - lastOutput_;\n+\t\terror_ = error_ * 0.95 + (outputDiff - inputDiff);\n+\t\tif (std::abs(error_) > errorThreshold_) {\n+\t\t\treset();\n+\t\t\tinputBase_ = input;\n+\t\t\toutputBase_ = output;\n+\t\t}\n+\t}\n+\tlastInput_ = input;\n+\tlastOutput_ = output;\n+\n+\t/*\n+\t * Never let the new output value be more than maxJitter_ away from what\n+\t * we would have expected. This is just to reduce the effect of sudden\n+\t * large delays in the measured output.\n+\t */\n+\tuint64_t expectedOutput = getOutput(input);\n+\toutput = std::clamp(output, expectedOutput - maxJitter_, expectedOutput + maxJitter_);\n+\n+\t/*\n+\t * We use x, y, x^2 and x*y sums to calculate the best fit line. Here we\n+\t * update them by pretending we have count_ samples at the previous fit,\n+\t * and now one new one. Gradually the effect of the older values gets\n+\t * lost. This is a very simple way of updating the fit (there are much\n+\t * more complicated ones!), but it works well enough. Using averages\n+\t * instead of sums makes the relative effect of old values and the new\n+\t * sample clearer.\n+\t */\n+\tdouble x = static_cast<int64_t>(input - inputBase_);\n+\tdouble y = static_cast<int64_t>(output - outputBase_) - x;\n+\tunsigned int count1 = count_ + 1;\n+\txAve_ = (count_ * xAve_ + x) / count1;\n+\tyAve_ = (count_ * yAve_ + y) / count1;\n+\tx2Ave_ = (count_ * x2Ave_ + x * x) / count1;\n+\txyAve_ = (count_ * xyAve_ + x * y) / count1;\n+\n+\t/*\n+\t * Don't update slope and offset until we've seen \"enough\" sample\n+\t * points. Note that the initial settings for slope_ and offset_\n+\t * ensures that the wallclock advances at the same rate as the realtime\n+\t * clock (but with their respective initial offsets).\n+\t */\n+\tif (count_ > minSamples_) {\n+\t\t/* These are the standard equations for least squares linear regression. */\n+\t\tslope_ = (count1 * count1 * xyAve_ - count1 * xAve_ * count1 * yAve_) /\n+\t\t\t (count1 * count1 * x2Ave_ - count1 * xAve_ * count1 * xAve_);\n+\t\toffset_ = yAve_ - slope_ * xAve_;\n+\t}\n+\n+\t/*\n+\t * Don't increase count_ above numSamples_, as this controls the long-term\n+\t * amount of the residual fit.\n+\t */\n+\tif (count1 < numSamples_)\n+\t\tcount_++;\n+}\n+\n+/**\n+ * \\brief Calculate the output clock value according to the model from an input\n+ * clock value\n+ * \\param[in] input The input clock value\n+ *\n+ * \\return Output clock value\n+ */\n+uint64_t ClockRecovery::getOutput(uint64_t input)\n+{\n+\tdouble x = static_cast<int64_t>(input - inputBase_);\n+\tdouble y = slope_ * x + offset_;\n+\tuint64_t output = y + x + outputBase_;\n+\n+\tLOG(ClockRec, Debug) << \"getOutput \" << input << \" \" << output;\n+\n+\treturn output;\n+}\n+\n+} /* namespace libcamera */\ndiff --git a/src/libcamera/meson.build b/src/libcamera/meson.build\nindex 202db1efe844..28a3b0f36d53 100644\n--- a/src/libcamera/meson.build\n+++ b/src/libcamera/meson.build\n@@ -21,6 +21,7 @@ libcamera_internal_sources = files([\n 'byte_stream_buffer.cpp',\n 'camera_controls.cpp',\n 'camera_lens.cpp',\n+ 'clock_recovery.cpp',\n 'control_serializer.cpp',\n 'control_validator.cpp',\n 'converter.cpp',\n", "prefixes": [ "2/4" ] }