[{"id":32977,"web_url":"https://patchwork.libcamera.org/comment/32977/","msgid":"<173643990289.2992722.17589369404896871209@ping.linuxembedded.co.uk>","date":"2025-01-09T16:25:02","subject":"Re: [PATCH v3 2/7] libcamera: Add ClockRecovery class to generate\n\twallclock timestamps","submitter":{"id":4,"url":"https://patchwork.libcamera.org/api/people/4/","name":"Kieran Bingham","email":"kieran.bingham@ideasonboard.com"},"content":"Quoting David Plowman (2025-01-09 14:32:06)\n> The ClockRecovery class takes pairs of timestamps from two different\n> clocks, and models the second (\"output\") clock from the first (\"input\")\n> clock.\n> \n> We can use it, in particular, to get a good wallclock estimate for a\n> frame's SensorTimestamp.\n> \n> Signed-off-by: David Plowman \n> ---\n> include/libcamera/internal/clock_recovery.h | 68 ++++++\n> include/libcamera/internal/meson.build | 1 +\n> src/libcamera/clock_recovery.cpp | 230 ++++++++++++++++++++\n> src/libcamera/meson.build | 1 +\n> 4 files changed, 300 insertions(+)\n> create mode 100644 include/libcamera/internal/clock_recovery.h\n> create mode 100644 src/libcamera/clock_recovery.cpp\n> \n> diff --git a/include/libcamera/internal/clock_recovery.h b/include/libcamera/internal/clock_recovery.h\n> new file mode 100644\n> index 00000000..43e46b7d\n> --- /dev/null\n> +++ b/include/libcamera/internal/clock_recovery.h\n> @@ -0,0 +1,68 @@\n> +/* SPDX-License-Identifier: LGPL-2.1-or-later */\n> +/*\n> + * Copyright (C) 2024, Raspberry Pi Ltd\n> + *\n> + * Camera recovery algorithm\n\nClock ?\n\nAside from that, as a standalone component, I would suspect this is a\ngood candidate for some unit tests in the future - but as we're going\nto see this hooked into the pipeline already that's going to provide the\nreal use case testing for the moment, so I won't insist on unit-tests.\n\nBonus points to anyone who wants to add something though.\n\n\nReviewed-by: Kieran Bingham \n\n\n> + */\n> +#pragma once\n> +\n> +#include \n> +\n> +namespace libcamera {\n> +\n> +class ClockRecovery\n> +{\n> +public:\n> + ClockRecovery();\n> +\n> + void configure(unsigned int numSamples = 100, unsigned int maxJitter = 2000,\n> + unsigned int minSamples = 10, unsigned int errorThreshold = 50000);\n> + void reset();\n> +\n> + void addSample();\n> + void addSample(uint64_t input, uint64_t output);\n> +\n> + uint64_t getOutput(uint64_t input);\n> +\n> +private:\n> + /* Approximate number of samples over which the model state persists. */\n> + unsigned int numSamples_;\n> + /* Remove any output jitter larger than this immediately. */\n> + unsigned int maxJitter_;\n> + /* Number of samples required before we start to use model estimates. */\n> + unsigned int minSamples_;\n> + /* Threshold above which we assume the wallclock has been reset. */\n> + unsigned int errorThreshold_;\n> +\n> + /* How many samples seen (up to numSamples_). */\n> + unsigned int count_;\n> + /* This gets subtracted from all input values, just to make the numbers easier. */\n> + uint64_t inputBase_;\n> + /* As above, for the output. */\n> + uint64_t outputBase_;\n> + /* The previous input sample. */\n> + uint64_t lastInput_;\n> + /* The previous output sample. */\n> + uint64_t lastOutput_;\n> +\n> + /* Average x value seen so far. */\n> + double xAve_;\n> + /* Average y value seen so far */\n> + double yAve_;\n> + /* Average x^2 value seen so far. */\n> + double x2Ave_;\n> + /* Average x*y value seen so far. */\n> + double xyAve_;\n> +\n> + /*\n> + * The latest estimate of linear parameters to derive the output clock\n> + * from the input.\n> + */\n> + double slope_;\n> + double offset_;\n> +\n> + /* Use this cumulative error to monitor for spontaneous clock updates. */\n> + double error_;\n> +};\n> +\n> +} /* namespace libcamera */\n> diff --git a/include/libcamera/internal/meson.build b/include/libcamera/internal/meson.build\n> index 7d6aa8b7..41500636 100644\n> --- a/include/libcamera/internal/meson.build\n> +++ b/include/libcamera/internal/meson.build\n> @@ -11,6 +11,7 @@ libcamera_internal_headers = files([\n> 'camera_manager.h',\n> 'camera_sensor.h',\n> 'camera_sensor_properties.h',\n> + 'clock_recovery.h',\n> 'control_serializer.h',\n> 'control_validator.h',\n> 'converter.h',\n> diff --git a/src/libcamera/clock_recovery.cpp b/src/libcamera/clock_recovery.cpp\n> new file mode 100644\n> index 00000000..abacf444\n> --- /dev/null\n> +++ b/src/libcamera/clock_recovery.cpp\n> @@ -0,0 +1,230 @@\n> +/* SPDX-License-Identifier: LGPL-2.1-or-later */\n> +/*\n> + * Copyright (C) 2024, Raspberry Pi Ltd\n> + *\n> + * Clock recovery algorithm\n> + */\n> +\n> +#include \"libcamera/internal/clock_recovery.h\"\n> +\n> +#include \n> +\n> +#include \n> +\n> +/**\n> + * \\file clock_recovery.h\n> + * \\brief Clock recovery - deriving one clock from another independent clock\n> + */\n> +\n> +namespace libcamera {\n> +\n> +LOG_DEFINE_CATEGORY(ClockRec)\n> +\n> +/**\n> + * \\class ClockRecovery\n> + * \\brief Recover an output clock from an input clock\n> + *\n> + * The ClockRecovery class derives an output clock from an input clock,\n> + * modelling the output clock as being linearly related to the input clock.\n> + * For example, we may use it to derive wall clock timestamps from timestamps\n> + * measured by the internal system clock which counts local time since boot.\n> + *\n> + * When pairs of corresponding input and output timestamps are available,\n> + * they should be submitted to the model with addSample(). The model will\n> + * update, and output clock values for known input clock values can be\n> + * obtained using getOutput().\n> + *\n> + * As a convenience, if the input clock is indeed the time since boot, and the\n> + * output clock represents a real wallclock time, then addSample() can be\n> + * called with no arguments, and a pair of timestamps will be captured at\n> + * that moment.\n> + *\n> + * The configure() function accepts some configuration parameters to control\n> + * the linear fitting process.\n> + */\n> +\n> +/**\n> + * \\brief Construct a ClockRecovery\n> + */\n> +ClockRecovery::ClockRecovery()\n> +{\n> + configure();\n> + reset();\n> +}\n> +\n> +/**\n> + * \\brief Set configuration parameters\n> + * \\param[in] numSamples The approximate duration for which the state of the model\n> + * is persistent\n> + * \\param[in] maxJitter New output samples are clamped to no more than this\n> + * amount of jitter, to prevent sudden swings from having a large effect\n> + * \\param[in] minSamples The fitted clock model is not used to generate outputs\n> + * until this many samples have been received\n> + * \\param[in] errorThreshold If the accumulated differences between input and\n> + * output clocks reaches this amount over a few frames, the model is reset\n> + */\n> +void ClockRecovery::configure(unsigned int numSamples, unsigned int maxJitter,\n> + unsigned int minSamples, unsigned int errorThreshold)\n> +{\n> + LOG(ClockRec, Debug)\n> + << \"configure \" << numSamples << \" \" << maxJitter << \" \" << minSamples << \" \" << errorThreshold;\n> +\n> + numSamples_ = numSamples;\n> + maxJitter_ = maxJitter;\n> + minSamples_ = minSamples;\n> + errorThreshold_ = errorThreshold;\n> +}\n> +\n> +/**\n> + * \\brief Reset the clock recovery model and start again from scratch\n> + */\n> +void ClockRecovery::reset()\n> +{\n> + LOG(ClockRec, Debug) << \"reset\";\n> +\n> + lastInput_ = 0;\n> + lastOutput_ = 0;\n> + xAve_ = 0;\n> + yAve_ = 0;\n> + x2Ave_ = 0;\n> + xyAve_ = 0;\n> + count_ = 0;\n> + error_ = 0.0;\n> + /*\n> + * Setting slope_ and offset_ to zero initially means that the clocks\n> + * advance at exactly the same rate.\n> + */\n> + slope_ = 0.0;\n> + offset_ = 0.0;\n> +}\n> +\n> +/**\n> + * \\brief Add a sample point to the clock recovery model, for recovering a wall\n> + * clock value from the internal system time since boot\n> + *\n> + * This is a convenience function to make it easy to derive a wall clock value\n> + * (using the Linux CLOCK_REALTIME) from the time since the system started\n> + * (measured by CLOCK_BOOTTIME).\n> + */\n> +void ClockRecovery::addSample()\n> +{\n> + LOG(ClockRec, Debug) << \"addSample\";\n> +\n> + struct timespec bootTime1;\n> + struct timespec bootTime2;\n> + struct timespec wallTime;\n> +\n> + /* Get boot and wall clocks in microseconds. */\n> + clock_gettime(CLOCK_BOOTTIME, &bootTime1);\n> + clock_gettime(CLOCK_REALTIME, &wallTime);\n> + clock_gettime(CLOCK_BOOTTIME, &bootTime2);\n> + uint64_t boot1 = bootTime1.tv_sec * 1000000ULL + bootTime1.tv_nsec / 1000;\n> + uint64_t boot2 = bootTime2.tv_sec * 1000000ULL + bootTime2.tv_nsec / 1000;\n> + uint64_t boot = (boot1 + boot2) / 2;\n> + uint64_t wall = wallTime.tv_sec * 1000000ULL + wallTime.tv_nsec / 1000;\n> +\n> + addSample(boot, wall);\n> +}\n> +\n> +/**\n> + * \\brief Add a sample point to the clock recovery model, specifying the exact\n> + * input and output clock values\n> + * \\param[in] input The input clock value\n> + * \\param[in] output The value of the output clock at the same moment, as far\n> + * as possible, that the input clock was sampled\n> + *\n> + * This function should be used for corresponding clocks other than the Linux\n> + * BOOTTIME and REALTIME clocks.\n> + */\n> +void ClockRecovery::addSample(uint64_t input, uint64_t output)\n> +{\n> + LOG(ClockRec, Debug) << \"addSample \" << input << \" \" << output;\n> +\n> + if (count_ == 0) {\n> + inputBase_ = input;\n> + outputBase_ = output;\n> + }\n> +\n> + /*\n> + * We keep an eye on cumulative drift over the last several frames. If this exceeds a\n> + * threshold, then probably the system clock has been updated and we're going to have to\n> + * reset everything and start over.\n> + */\n> + if (lastOutput_) {\n> + int64_t inputDiff = getOutput(input) - getOutput(lastInput_);\n> + int64_t outputDiff = output - lastOutput_;\n> + error_ = error_ * 0.95 + (outputDiff - inputDiff);\n> + if (std::abs(error_) > errorThreshold_) {\n> + reset();\n> + inputBase_ = input;\n> + outputBase_ = output;\n> + }\n> + }\n> + lastInput_ = input;\n> + lastOutput_ = output;\n> +\n> + /*\n> + * Never let the new output value be more than maxJitter_ away from what\n> + * we would have expected. This is just to reduce the effect of sudden\n> + * large delays in the measured output.\n> + */\n> + uint64_t expectedOutput = getOutput(input);\n> + output = std::clamp(output, expectedOutput - maxJitter_, expectedOutput + maxJitter_);\n> +\n> + /*\n> + * We use x, y, x^2 and x*y sums to calculate the best fit line. Here we\n> + * update them by pretending we have count_ samples at the previous fit,\n> + * and now one new one. Gradually the effect of the older values gets\n> + * lost. This is a very simple way of updating the fit (there are much\n> + * more complicated ones!), but it works well enough. Using averages\n> + * instead of sums makes the relative effect of old values and the new\n> + * sample clearer.\n> + */\n> + double x = static_cast(input - inputBase_);\n> + double y = static_cast(output - outputBase_) - x;\n> + unsigned int count1 = count_ + 1;\n> + xAve_ = (count_ * xAve_ + x) / count1;\n> + yAve_ = (count_ * yAve_ + y) / count1;\n> + x2Ave_ = (count_ * x2Ave_ + x * x) / count1;\n> + xyAve_ = (count_ * xyAve_ + x * y) / count1;\n> +\n> + /*\n> + * Don't update slope and offset until we've seen \"enough\" sample\n> + * points. Note that the initial settings for slope_ and offset_\n> + * ensures that the wallclock advances at the same rate as the realtime\n> + * clock (but with their respective initial offsets).\n> + */\n> + if (count_ > minSamples_) {\n> + /* These are the standard equations for least squares linear regression. */\n> + slope_ = (count1 * count1 * xyAve_ - count1 * xAve_ * count1 * yAve_) /\n> + (count1 * count1 * x2Ave_ - count1 * xAve_ * count1 * xAve_);\n> + offset_ = yAve_ - slope_ * xAve_;\n> + }\n> +\n> + /*\n> + * Don't increase count_ above numSamples_, as this controls the long-term\n> + * amount of the residual fit.\n> + */\n> + if (count1 < numSamples_)\n> + count_++;\n> +}\n> +\n> +/**\n> + * \\brief Calculate the output clock value according to the model from an input\n> + * clock value\n> + * \\param[in] input The input clock value\n> + *\n> + * \\return Output clock value\n> + */\n> +uint64_t ClockRecovery::getOutput(uint64_t input)\n> +{\n> + double x = static_cast(input - inputBase_);\n> + double y = slope_ * x + offset_;\n> + uint64_t output = y + x + outputBase_;\n> +\n> + LOG(ClockRec, Debug) << \"getOutput \" << input << \" \" << output;\n> +\n> + return output;\n> +}\n> +\n> +} /* namespace libcamera */\n> diff --git a/src/libcamera/meson.build b/src/libcamera/meson.build\n> index 57fde8a8..4eaa1c8e 100644\n> --- a/src/libcamera/meson.build\n> +++ b/src/libcamera/meson.build\n> @@ -21,6 +21,7 @@ libcamera_internal_sources = files([\n> 'byte_stream_buffer.cpp',\n> 'camera_controls.cpp',\n> 'camera_lens.cpp',\n> + 'clock_recovery.cpp',\n> 'control_serializer.cpp',\n> 'control_validator.cpp',\n> 'converter.cpp',\n> -- \n> 2.39.5\n>","headers":{"Return-Path":"","X-Original-To":"parsemail@patchwork.libcamera.org","Delivered-To":"parsemail@patchwork.libcamera.org","Received":["from lancelot.ideasonboard.com (lancelot.ideasonboard.com\n\t[92.243.16.209])\n\tby patchwork.libcamera.org (Postfix) with ESMTPS id 69F9EC32EF\n\tfor ;\n\tThu, 9 Jan 2025 16:25:09 +0000 (UTC)","from lancelot.ideasonboard.com (localhost [IPv6:::1])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTP id 7203961884;\n\tThu, 9 Jan 2025 17:25:08 +0100 (CET)","from perceval.ideasonboard.com (perceval.ideasonboard.com\n\t[IPv6:2001:4b98:dc2:55:216:3eff:fef7:d647])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id 51B6161880\n\tfor ;\n\tThu, 9 Jan 2025 17:25:06 +0100 (CET)","from pendragon.ideasonboard.com\n\t(cpc89244-aztw30-2-0-cust6594.18-1.cable.virginm.net [86.31.185.195])\n\tby perceval.ideasonboard.com (Postfix) with ESMTPSA id 82EC7110F;\n\tThu, 9 Jan 2025 17:24:12 +0100 (CET)"],"Authentication-Results":"lancelot.ideasonboard.com; dkim=pass (1024-bit key;\n\tunprotected) header.d=ideasonboard.com header.i=@ideasonboard.com\n\theader.b=\"umpIuzUZ\"; dkim-atps=neutral","DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/simple; d=ideasonboard.com;\n\ts=mail; t=1736439852;\n\tbh=fDvHbr4NrkqM18ULI9oO0r0XqJdTneRG5kZ3yWj2npM=;\n\th=In-Reply-To:References:Subject:From:Cc:To:Date:From;\n\tb=umpIuzUZ19HwXeXA7YviMuDOrK/ch1bvMDeviWIJ0Wg4mKxe//Ng5hz4CuWCWlsSF\n\t+18XliCIIGGfoMJwZx7G38NHeUrzhSqeUbtX/+9IICDq7P9rUFyIL0kxfNtnNORkPJ\n\tk/fNlfeFCM8r9Te5eEFiaJo9nV6qluP00uBcR4y0=","Content-Type":"text/plain; charset=\"utf-8\"","MIME-Version":"1.0","Content-Transfer-Encoding":"quoted-printable","In-Reply-To":"<20250109143211.11939-3-david.plowman@raspberrypi.com>","References":"<20250109143211.11939-1-david.plowman@raspberrypi.com>\n\t<20250109143211.11939-3-david.plowman@raspberrypi.com>","Subject":"Re: [PATCH v3 2/7] libcamera: Add ClockRecovery class to generate\n\twallclock timestamps","From":"Kieran Bingham ","Cc":"David Plowman ","To":"David Plowman ,\n\tlibcamera-devel@lists.libcamera.org","Date":"Thu, 09 Jan 2025 16:25:02 +0000","Message-ID":"<173643990289.2992722.17589369404896871209@ping.linuxembedded.co.uk>","User-Agent":"alot/0.10","X-BeenThere":"libcamera-devel@lists.libcamera.org","X-Mailman-Version":"2.1.29","Precedence":"list","List-Id":"","List-Unsubscribe":",\n\t","List-Archive":"","List-Post":"","List-Help":"","List-Subscribe":",\n\t","Errors-To":"libcamera-devel-bounces@lists.libcamera.org","Sender":"\"libcamera-devel\" "}},{"id":33087,"web_url":"https://patchwork.libcamera.org/comment/33087/","msgid":"","date":"2025-01-17T09:03:47","subject":"Re: [PATCH v3 2/7] libcamera: Add ClockRecovery class to generate\n\twallclock timestamps","submitter":{"id":34,"url":"https://patchwork.libcamera.org/api/people/34/","name":"Naushir Patuck","email":"naush@raspberrypi.com"},"content":"Hi David,\n\nOn Thu, 9 Jan 2025 at 14:32, David Plowman\n wrote:\n>\n> The ClockRecovery class takes pairs of timestamps from two different\n> clocks, and models the second (\"output\") clock from the first (\"input\")\n> clock.\n>\n> We can use it, in particular, to get a good wallclock estimate for a\n> frame's SensorTimestamp.\n>\n> Signed-off-by: David Plowman \n\nReviewed-by: Naushir Patuck \n\n> ---\n> include/libcamera/internal/clock_recovery.h | 68 ++++++\n> include/libcamera/internal/meson.build | 1 +\n> src/libcamera/clock_recovery.cpp | 230 ++++++++++++++++++++\n> src/libcamera/meson.build | 1 +\n> 4 files changed, 300 insertions(+)\n> create mode 100644 include/libcamera/internal/clock_recovery.h\n> create mode 100644 src/libcamera/clock_recovery.cpp\n>\n> diff --git a/include/libcamera/internal/clock_recovery.h b/include/libcamera/internal/clock_recovery.h\n> new file mode 100644\n> index 00000000..43e46b7d\n> --- /dev/null\n> +++ b/include/libcamera/internal/clock_recovery.h\n> @@ -0,0 +1,68 @@\n> +/* SPDX-License-Identifier: LGPL-2.1-or-later */\n> +/*\n> + * Copyright (C) 2024, Raspberry Pi Ltd\n> + *\n> + * Camera recovery algorithm\n> + */\n> +#pragma once\n> +\n> +#include \n> +\n> +namespace libcamera {\n> +\n> +class ClockRecovery\n> +{\n> +public:\n> + ClockRecovery();\n> +\n> + void configure(unsigned int numSamples = 100, unsigned int maxJitter = 2000,\n> + unsigned int minSamples = 10, unsigned int errorThreshold = 50000);\n> + void reset();\n> +\n> + void addSample();\n> + void addSample(uint64_t input, uint64_t output);\n> +\n> + uint64_t getOutput(uint64_t input);\n> +\n> +private:\n> + /* Approximate number of samples over which the model state persists. */\n> + unsigned int numSamples_;\n> + /* Remove any output jitter larger than this immediately. */\n> + unsigned int maxJitter_;\n> + /* Number of samples required before we start to use model estimates. */\n> + unsigned int minSamples_;\n> + /* Threshold above which we assume the wallclock has been reset. */\n> + unsigned int errorThreshold_;\n> +\n> + /* How many samples seen (up to numSamples_). */\n> + unsigned int count_;\n> + /* This gets subtracted from all input values, just to make the numbers easier. */\n> + uint64_t inputBase_;\n> + /* As above, for the output. */\n> + uint64_t outputBase_;\n> + /* The previous input sample. */\n> + uint64_t lastInput_;\n> + /* The previous output sample. */\n> + uint64_t lastOutput_;\n> +\n> + /* Average x value seen so far. */\n> + double xAve_;\n> + /* Average y value seen so far */\n> + double yAve_;\n> + /* Average x^2 value seen so far. */\n> + double x2Ave_;\n> + /* Average x*y value seen so far. */\n> + double xyAve_;\n> +\n> + /*\n> + * The latest estimate of linear parameters to derive the output clock\n> + * from the input.\n> + */\n> + double slope_;\n> + double offset_;\n> +\n> + /* Use this cumulative error to monitor for spontaneous clock updates. */\n> + double error_;\n> +};\n> +\n> +} /* namespace libcamera */\n> diff --git a/include/libcamera/internal/meson.build b/include/libcamera/internal/meson.build\n> index 7d6aa8b7..41500636 100644\n> --- a/include/libcamera/internal/meson.build\n> +++ b/include/libcamera/internal/meson.build\n> @@ -11,6 +11,7 @@ libcamera_internal_headers = files([\n> 'camera_manager.h',\n> 'camera_sensor.h',\n> 'camera_sensor_properties.h',\n> + 'clock_recovery.h',\n> 'control_serializer.h',\n> 'control_validator.h',\n> 'converter.h',\n> diff --git a/src/libcamera/clock_recovery.cpp b/src/libcamera/clock_recovery.cpp\n> new file mode 100644\n> index 00000000..abacf444\n> --- /dev/null\n> +++ b/src/libcamera/clock_recovery.cpp\n> @@ -0,0 +1,230 @@\n> +/* SPDX-License-Identifier: LGPL-2.1-or-later */\n> +/*\n> + * Copyright (C) 2024, Raspberry Pi Ltd\n> + *\n> + * Clock recovery algorithm\n> + */\n> +\n> +#include \"libcamera/internal/clock_recovery.h\"\n> +\n> +#include \n> +\n> +#include \n> +\n> +/**\n> + * \\file clock_recovery.h\n> + * \\brief Clock recovery - deriving one clock from another independent clock\n> + */\n> +\n> +namespace libcamera {\n> +\n> +LOG_DEFINE_CATEGORY(ClockRec)\n> +\n> +/**\n> + * \\class ClockRecovery\n> + * \\brief Recover an output clock from an input clock\n> + *\n> + * The ClockRecovery class derives an output clock from an input clock,\n> + * modelling the output clock as being linearly related to the input clock.\n> + * For example, we may use it to derive wall clock timestamps from timestamps\n> + * measured by the internal system clock which counts local time since boot.\n> + *\n> + * When pairs of corresponding input and output timestamps are available,\n> + * they should be submitted to the model with addSample(). The model will\n> + * update, and output clock values for known input clock values can be\n> + * obtained using getOutput().\n> + *\n> + * As a convenience, if the input clock is indeed the time since boot, and the\n> + * output clock represents a real wallclock time, then addSample() can be\n> + * called with no arguments, and a pair of timestamps will be captured at\n> + * that moment.\n> + *\n> + * The configure() function accepts some configuration parameters to control\n> + * the linear fitting process.\n> + */\n> +\n> +/**\n> + * \\brief Construct a ClockRecovery\n> + */\n> +ClockRecovery::ClockRecovery()\n> +{\n> + configure();\n> + reset();\n> +}\n> +\n> +/**\n> + * \\brief Set configuration parameters\n> + * \\param[in] numSamples The approximate duration for which the state of the model\n> + * is persistent\n> + * \\param[in] maxJitter New output samples are clamped to no more than this\n> + * amount of jitter, to prevent sudden swings from having a large effect\n> + * \\param[in] minSamples The fitted clock model is not used to generate outputs\n> + * until this many samples have been received\n> + * \\param[in] errorThreshold If the accumulated differences between input and\n> + * output clocks reaches this amount over a few frames, the model is reset\n> + */\n> +void ClockRecovery::configure(unsigned int numSamples, unsigned int maxJitter,\n> + unsigned int minSamples, unsigned int errorThreshold)\n> +{\n> + LOG(ClockRec, Debug)\n> + << \"configure \" << numSamples << \" \" << maxJitter << \" \" << minSamples << \" \" << errorThreshold;\n> +\n> + numSamples_ = numSamples;\n> + maxJitter_ = maxJitter;\n> + minSamples_ = minSamples;\n> + errorThreshold_ = errorThreshold;\n> +}\n> +\n> +/**\n> + * \\brief Reset the clock recovery model and start again from scratch\n> + */\n> +void ClockRecovery::reset()\n> +{\n> + LOG(ClockRec, Debug) << \"reset\";\n> +\n> + lastInput_ = 0;\n> + lastOutput_ = 0;\n> + xAve_ = 0;\n> + yAve_ = 0;\n> + x2Ave_ = 0;\n> + xyAve_ = 0;\n> + count_ = 0;\n> + error_ = 0.0;\n> + /*\n> + * Setting slope_ and offset_ to zero initially means that the clocks\n> + * advance at exactly the same rate.\n> + */\n> + slope_ = 0.0;\n> + offset_ = 0.0;\n> +}\n> +\n> +/**\n> + * \\brief Add a sample point to the clock recovery model, for recovering a wall\n> + * clock value from the internal system time since boot\n> + *\n> + * This is a convenience function to make it easy to derive a wall clock value\n> + * (using the Linux CLOCK_REALTIME) from the time since the system started\n> + * (measured by CLOCK_BOOTTIME).\n> + */\n> +void ClockRecovery::addSample()\n> +{\n> + LOG(ClockRec, Debug) << \"addSample\";\n> +\n> + struct timespec bootTime1;\n> + struct timespec bootTime2;\n> + struct timespec wallTime;\n> +\n> + /* Get boot and wall clocks in microseconds. */\n> + clock_gettime(CLOCK_BOOTTIME, &bootTime1);\n> + clock_gettime(CLOCK_REALTIME, &wallTime);\n> + clock_gettime(CLOCK_BOOTTIME, &bootTime2);\n> + uint64_t boot1 = bootTime1.tv_sec * 1000000ULL + bootTime1.tv_nsec / 1000;\n> + uint64_t boot2 = bootTime2.tv_sec * 1000000ULL + bootTime2.tv_nsec / 1000;\n> + uint64_t boot = (boot1 + boot2) / 2;\n> + uint64_t wall = wallTime.tv_sec * 1000000ULL + wallTime.tv_nsec / 1000;\n> +\n> + addSample(boot, wall);\n> +}\n> +\n> +/**\n> + * \\brief Add a sample point to the clock recovery model, specifying the exact\n> + * input and output clock values\n> + * \\param[in] input The input clock value\n> + * \\param[in] output The value of the output clock at the same moment, as far\n> + * as possible, that the input clock was sampled\n> + *\n> + * This function should be used for corresponding clocks other than the Linux\n> + * BOOTTIME and REALTIME clocks.\n> + */\n> +void ClockRecovery::addSample(uint64_t input, uint64_t output)\n> +{\n> + LOG(ClockRec, Debug) << \"addSample \" << input << \" \" << output;\n> +\n> + if (count_ == 0) {\n> + inputBase_ = input;\n> + outputBase_ = output;\n> + }\n> +\n> + /*\n> + * We keep an eye on cumulative drift over the last several frames. If this exceeds a\n> + * threshold, then probably the system clock has been updated and we're going to have to\n> + * reset everything and start over.\n> + */\n> + if (lastOutput_) {\n> + int64_t inputDiff = getOutput(input) - getOutput(lastInput_);\n> + int64_t outputDiff = output - lastOutput_;\n> + error_ = error_ * 0.95 + (outputDiff - inputDiff);\n> + if (std::abs(error_) > errorThreshold_) {\n> + reset();\n> + inputBase_ = input;\n> + outputBase_ = output;\n> + }\n> + }\n> + lastInput_ = input;\n> + lastOutput_ = output;\n> +\n> + /*\n> + * Never let the new output value be more than maxJitter_ away from what\n> + * we would have expected. This is just to reduce the effect of sudden\n> + * large delays in the measured output.\n> + */\n> + uint64_t expectedOutput = getOutput(input);\n> + output = std::clamp(output, expectedOutput - maxJitter_, expectedOutput + maxJitter_);\n> +\n> + /*\n> + * We use x, y, x^2 and x*y sums to calculate the best fit line. Here we\n> + * update them by pretending we have count_ samples at the previous fit,\n> + * and now one new one. Gradually the effect of the older values gets\n> + * lost. This is a very simple way of updating the fit (there are much\n> + * more complicated ones!), but it works well enough. Using averages\n> + * instead of sums makes the relative effect of old values and the new\n> + * sample clearer.\n> + */\n> + double x = static_cast(input - inputBase_);\n> + double y = static_cast(output - outputBase_) - x;\n> + unsigned int count1 = count_ + 1;\n> + xAve_ = (count_ * xAve_ + x) / count1;\n> + yAve_ = (count_ * yAve_ + y) / count1;\n> + x2Ave_ = (count_ * x2Ave_ + x * x) / count1;\n> + xyAve_ = (count_ * xyAve_ + x * y) / count1;\n> +\n> + /*\n> + * Don't update slope and offset until we've seen \"enough\" sample\n> + * points. Note that the initial settings for slope_ and offset_\n> + * ensures that the wallclock advances at the same rate as the realtime\n> + * clock (but with their respective initial offsets).\n> + */\n> + if (count_ > minSamples_) {\n> + /* These are the standard equations for least squares linear regression. */\n> + slope_ = (count1 * count1 * xyAve_ - count1 * xAve_ * count1 * yAve_) /\n> + (count1 * count1 * x2Ave_ - count1 * xAve_ * count1 * xAve_);\n> + offset_ = yAve_ - slope_ * xAve_;\n> + }\n> +\n> + /*\n> + * Don't increase count_ above numSamples_, as this controls the long-term\n> + * amount of the residual fit.\n> + */\n> + if (count1 < numSamples_)\n> + count_++;\n> +}\n> +\n> +/**\n> + * \\brief Calculate the output clock value according to the model from an input\n> + * clock value\n> + * \\param[in] input The input clock value\n> + *\n> + * \\return Output clock value\n> + */\n> +uint64_t ClockRecovery::getOutput(uint64_t input)\n> +{\n> + double x = static_cast(input - inputBase_);\n> + double y = slope_ * x + offset_;\n> + uint64_t output = y + x + outputBase_;\n> +\n> + LOG(ClockRec, Debug) << \"getOutput \" << input << \" \" << output;\n> +\n> + return output;\n> +}\n> +\n> +} /* namespace libcamera */\n> diff --git a/src/libcamera/meson.build b/src/libcamera/meson.build\n> index 57fde8a8..4eaa1c8e 100644\n> --- a/src/libcamera/meson.build\n> +++ b/src/libcamera/meson.build\n> @@ -21,6 +21,7 @@ libcamera_internal_sources = files([\n> 'byte_stream_buffer.cpp',\n> 'camera_controls.cpp',\n> 'camera_lens.cpp',\n> + 'clock_recovery.cpp',\n> 'control_serializer.cpp',\n> 'control_validator.cpp',\n> 'converter.cpp',\n> --\n> 2.39.5\n>","headers":{"Return-Path":"","X-Original-To":"parsemail@patchwork.libcamera.org","Delivered-To":"parsemail@patchwork.libcamera.org","Received":["from lancelot.ideasonboard.com (lancelot.ideasonboard.com\n\t[92.243.16.209])\n\tby patchwork.libcamera.org (Postfix) with ESMTPS id 1C12ABD7D8\n\tfor ;\n\tFri, 17 Jan 2025 09:04:27 +0000 (UTC)","from lancelot.ideasonboard.com (localhost [IPv6:::1])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTP id B9FE568543;\n\tFri, 17 Jan 2025 10:04:25 +0100 (CET)","from mail-yb1-xb2c.google.com (mail-yb1-xb2c.google.com\n\t[IPv6:2607:f8b0:4864:20::b2c])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id 584BF68503\n\tfor ;\n\tFri, 17 Jan 2025 10:04:23 +0100 (CET)","by mail-yb1-xb2c.google.com with SMTP id\n\t3f1490d57ef6-e3c88163a00so320606276.2\n\tfor ;\n\tFri, 17 Jan 2025 01:04:23 -0800 (PST)"],"Authentication-Results":"lancelot.ideasonboard.com; dkim=pass (2048-bit key;\n\tunprotected) header.d=raspberrypi.com header.i=@raspberrypi.com\n\theader.b=\"j8g1pyx5\"; dkim-atps=neutral","DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/relaxed;\n\td=raspberrypi.com; s=google; t=1737104662; x=1737709462;\n\tdarn=lists.libcamera.org; \n\th=cc:to:subject:message-id:date:from:in-reply-to:references\n\t:mime-version:from:to:cc:subject:date:message-id:reply-to;\n\tbh=q7dvcSvuVPPK2YlNhe9DC2gjGdp09OLt7MJ//tIg5s4=;\n\tb=j8g1pyx5vIov6wZLGv0bZbgmHOvmXD6as872ZiOKxHJ10rdelDH/eq1uUSXaJsSoZH\n\t2R+JX8qRgAypEj8YQJgVMufJVPtnVqHMp9eFrsAEcZmBsW3ChB4jYdt8BqcY043UBJHV\n\tjROVOPKp5WkueRVpeM6Q37Ng9zSV00pbcNASGUK5GcJ9rB2DcB9Ck4pHli8teki4XGFr\n\teNwy+t3RXnjlfpi3hurVR/kVRXFIAWR9gyXd/Psk2K7mepZ89LXPIYQOJfj5Rjnacy3x\n\thwHuri7OTTgBTosqx3wHPRRRYmRdvziZtVWIh1+XS08cIeMB5/0Solx3CdomszCoZLB1\n\txcWw==","X-Google-DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/relaxed;\n\td=1e100.net; s=20230601; t=1737104662; x=1737709462;\n\th=cc:to:subject:message-id:date:from:in-reply-to:references\n\t:mime-version:x-gm-message-state:from:to:cc:subject:date:message-id\n\t:reply-to;\n\tbh=q7dvcSvuVPPK2YlNhe9DC2gjGdp09OLt7MJ//tIg5s4=;\n\tb=ZZUtggxIut3wCLyrpzm1yufWEP2gVKQalhERBhRzpDow0SKrnNQ25/pcd8k/nB5wGD\n\tu09N8df1ePzQ1KL584ZjrMx0niYYSmBoby/BrlgUlETOhnfdm7OKqWcM5soZi9RvKqYC\n\tER/AjqXoGRDJVdbE3yLoRanZrm70CHHBnsHLrFF0ndTlO9lCrYbW85Wc0p6LqZI8fZj1\n\tsCpHNVrEVihqJ/zZCUHhny5iGRGQ+MN6AgraMsCdQy6WBeM2I9xOpbjIsPvJngOSzsWu\n\tGprOwirR43AWfoRwN0HlOZQEK3Qq2VFxEzFfl9GqxHtL/wScJrqfvP7vj61NfICghce0\n\tKJNg==","X-Gm-Message-State":"AOJu0YyBiNhMkb4O+inmFU/mn7LIgTyOVrxr7WHDreL2zgoE3h2cbOIP\n\trF01eKbmhOdX2VsJz7BYQHsBQdlr+3q3vgByA6weoyvycKrIiGKrZGeMIZBV94BmD4gt98hFo5I\n\ty1nregaoDCVux9+2DsUSX1h7shKJnh9iaW2xxzkbKc7pNSb05LMI=","X-Gm-Gg":"ASbGncstBvRcezP6/seZfqpdVRbN4E1QuM63sVK1I/EZVFeW8h8xjpXHst3BQtySmoI\n\t6BLWvlUQW/7GxRDabstUTC8iyhLKKcR49scSj5wg0MI/iOEaG4FF/0k/GeLPKqgBTHGVnRw==","X-Google-Smtp-Source":"AGHT+IE3EtC/1DoP9onL39SXs/FOIHImtvxkgd42i5EVCInHzbQpjKwlDf9QIhv4UFsdOHf7H3B1PU6PqlLQtOaK50s=","X-Received":"by 2002:a05:6902:a87:b0:e53:64a6:2de5 with SMTP id\n\t3f1490d57ef6-e57b1364789mr469426276.8.1737104662017; Fri, 17 Jan 2025\n\t01:04:22 -0800 (PST)","MIME-Version":"1.0","References":"<20250109143211.11939-1-david.plowman@raspberrypi.com>\n\t<20250109143211.11939-3-david.plowman@raspberrypi.com>","In-Reply-To":"<20250109143211.11939-3-david.plowman@raspberrypi.com>","From":"Naushir Patuck ","Date":"Fri, 17 Jan 2025 09:03:47 +0000","X-Gm-Features":"AbW1kvbPNjcZuIkX_A1EaTIHQoE1mrKpY61t20UOFPUsNd5EArrZRb9Visa_OYI","Message-ID":"","Subject":"Re: [PATCH v3 2/7] libcamera: Add ClockRecovery class to generate\n\twallclock timestamps","To":"David Plowman ","Cc":"libcamera-devel@lists.libcamera.org","Content-Type":"text/plain; charset=\"UTF-8\"","X-BeenThere":"libcamera-devel@lists.libcamera.org","X-Mailman-Version":"2.1.29","Precedence":"list","List-Id":"","List-Unsubscribe":",\n\t","List-Archive":"","List-Post":"","List-Help":"","List-Subscribe":",\n\t","Errors-To":"libcamera-devel-bounces@lists.libcamera.org","Sender":"\"libcamera-devel\" "}}]