Patch Detail
Show a patch.
GET /api/patches/22613/?format=api
{ "id": 22613, "url": "https://patchwork.libcamera.org/api/patches/22613/?format=api", "web_url": "https://patchwork.libcamera.org/patch/22613/", "project": { "id": 1, "url": "https://patchwork.libcamera.org/api/projects/1/?format=api", "name": "libcamera", "link_name": "libcamera", "list_id": "libcamera_core", "list_email": "libcamera-devel@lists.libcamera.org", "web_url": "", "scm_url": "", "webscm_url": "" }, "msgid": "<20250122145349.7220-6-david.plowman@raspberrypi.com>", "date": "2025-01-22T14:53:47", "name": "[v4,5/7] ipa: rpi: Add base classes and plumbing for sync algorithm", "commit_ref": null, "pull_url": null, "state": "new", "archived": false, "hash": "437a80b181cc8cd747e6178ccc6bb461cb009c67", "submitter": { "id": 42, "url": "https://patchwork.libcamera.org/api/people/42/?format=api", "name": "David Plowman", "email": "david.plowman@raspberrypi.com" }, "delegate": null, "mbox": "https://patchwork.libcamera.org/patch/22613/mbox/", "series": [ { "id": 4964, "url": "https://patchwork.libcamera.org/api/series/4964/?format=api", "web_url": "https://patchwork.libcamera.org/project/libcamera/list/?series=4964", "date": "2025-01-22T14:53:47", "name": "Camera synchronisation", "version": 4, "mbox": "https://patchwork.libcamera.org/series/4964/mbox/" } ], "comments": "https://patchwork.libcamera.org/api/patches/22613/comments/", "check": "pending", "checks": "https://patchwork.libcamera.org/api/patches/22613/checks/", "tags": {}, "headers": { "Return-Path": "<libcamera-devel-bounces@lists.libcamera.org>", "X-Original-To": "parsemail@patchwork.libcamera.org", "Delivered-To": "parsemail@patchwork.libcamera.org", "Received": [ "from lancelot.ideasonboard.com (lancelot.ideasonboard.com\n\t[92.243.16.209])\n\tby patchwork.libcamera.org (Postfix) with ESMTPS id 3E9C5C3319\n\tfor <parsemail@patchwork.libcamera.org>;\n\tWed, 22 Jan 2025 14:54:09 +0000 (UTC)", "from lancelot.ideasonboard.com (localhost [IPv6:::1])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTP id E982C68567;\n\tWed, 22 Jan 2025 15:54:07 +0100 (CET)", "from mail-wr1-x42c.google.com (mail-wr1-x42c.google.com\n\t[IPv6:2a00:1450:4864:20::42c])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id E09FA68559\n\tfor <libcamera-devel@lists.libcamera.org>;\n\tWed, 22 Jan 2025 15:53:56 +0100 (CET)", "by mail-wr1-x42c.google.com with SMTP id\n\tffacd0b85a97d-38634c35129so5941096f8f.3\n\tfor <libcamera-devel@lists.libcamera.org>;\n\tWed, 22 Jan 2025 06:53:56 -0800 (PST)", "from raspberrypi.pitowers.org\n\t([2a00:1098:3142:1f:ffc9:aff6:7f7f:893b])\n\tby smtp.gmail.com with ESMTPSA id\n\t5b1f17b1804b1-438b66dc08bsm11551395e9.37.2025.01.22.06.53.55\n\t(version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256);\n\tWed, 22 Jan 2025 06:53:55 -0800 (PST)" ], "Authentication-Results": "lancelot.ideasonboard.com; dkim=pass (2048-bit key;\n\tunprotected) header.d=raspberrypi.com header.i=@raspberrypi.com\n\theader.b=\"lC/b6pRD\"; dkim-atps=neutral", "DKIM-Signature": "v=1; a=rsa-sha256; c=relaxed/relaxed;\n\td=raspberrypi.com; s=google; t=1737557636; x=1738162436;\n\tdarn=lists.libcamera.org; \n\th=content-transfer-encoding:mime-version:references:in-reply-to\n\t:message-id:date:subject:cc:to:from:from:to:cc:subject:date\n\t:message-id:reply-to;\n\tbh=dZRsfHRIhVt9X4suCupUrPLxrY9XKlNXZm04oJrTHek=;\n\tb=lC/b6pRDQc5GBYRQxFMdA3LuenWBkRtCRnnn9klPHISXSJbFCJDI9hBysmFRJqqiag\n\t0XXdcfhX8zJm8Q53sfaJX5mwtkErcCyqnDT7ah2tiV05O0xi5EsfMIPFobzsl63G1l1w\n\t9VKPRKBOPPZc/G7D76H4GTOpX6uPD5+KPlts04bjWcg875C1jEHS9uEkDahvJpHE7FRs\n\tHTNRYFQBZw5gHM35UzNK9yzrI3VTDt+d8j+OjpN1Sf4p1rgM6Z3w79dzqQFMFhpafHGe\n\tB3fd4yUe73mzyUizLTuslFwCSIdR3bBTibdcf0B3y1AvghQYr931cuEP15L4sNWHz6we\n\tusVQ==", "X-Google-DKIM-Signature": "v=1; a=rsa-sha256; c=relaxed/relaxed;\n\td=1e100.net; s=20230601; t=1737557636; x=1738162436;\n\th=content-transfer-encoding:mime-version:references:in-reply-to\n\t:message-id:date:subject:cc:to:from:x-gm-message-state:from:to:cc\n\t:subject:date:message-id:reply-to;\n\tbh=dZRsfHRIhVt9X4suCupUrPLxrY9XKlNXZm04oJrTHek=;\n\tb=AvBQe6sw/aIlVuNOqcT24WGKuG3blgVR59JgkPwJNI+MxWocBmmvQCGJ+6fOHtdZy5\n\tEQbgoaXUNXhKgC0naFR9SLpdrhO41MzboRuDb2ucgYGqkJzMIreUfTq2YXcIzv7y75aV\n\tZiCI8CkFfnSW9W+QcwHCkd485j46tzLFdxvzH8hAqxXyFX7PyL0M9SNK7o+2pm0ZKndb\n\tgcUEX+KLNy4jyQchylA96543RYvnaV+6bxYNOIR2tiM6QRy7oBVbgzOtN13+Z+36SYSl\n\tosjuRskYIIzAaUhoW5bCqkfMJfQEUlTJNiwsDOAP9q55k1HFl1JoA6upznW36zrKetZA\n\taBSQ==", "X-Gm-Message-State": "AOJu0YxsO3WbpFKTdOoy6fbkT7yNwQYLfaqlo+e5o3ma1lZl5IYOUymj\n\tR/vCwacT4daEGNg436927h9xUpARUlVVmxA3M01Wo2IPlL8EpsuHuLaYvAYoL6E7Ms8SOzZeX6B\n\ts", "X-Gm-Gg": "ASbGncskZg21wIMvl46r3joKM/W+ii7UnVPk9AICgFlv05aC4T44MihR/QsGx+7WZnm\n\t7CaTOeA59LbufdOFauYhJlcipr8L+ST5b0uk3IRhTVMzFVYkjy/KV3gjuGNnHvgpW7oPBZDHJ3y\n\t7WTzj5iurGo7YfV1Gd4dSz9VBw0r1Hgyet47lLpCeiz9VGIp/OPfv9cdDSZViIxJr65lerwbVxj\n\tWvHGtIQUJ/FI9jSh32BhHRECfMtIIPE+5E4OT7duNzdhhyvBk25HZXIfSLp940g6lP9ySsnZt4D\n\tqgW3afrssMXiIXVFwinaOTYvcw==", "X-Google-Smtp-Source": "AGHT+IEm1yvPzu9FMPWb+e6Pqx1T79DY1cxylGpCoBXikxCqCCijptf6vT9/iW6s60rPQjVFjIsudg==", "X-Received": "by 2002:adf:f141:0:b0:38a:418e:1171 with SMTP id\n\tffacd0b85a97d-38bf58e88admr14678658f8f.37.1737557636073; \n\tWed, 22 Jan 2025 06:53:56 -0800 (PST)", "From": "David Plowman <david.plowman@raspberrypi.com>", "To": "libcamera-devel@lists.libcamera.org", "Cc": "Naushir Patuck <naush@raspberrypi.com>,\n\tDavid Plowman <david.plowman@raspberrypi.com>", "Subject": "[PATCH v4 5/7] ipa: rpi: Add base classes and plumbing for sync\n\talgorithm", "Date": "Wed, 22 Jan 2025 14:53:47 +0000", "Message-Id": "<20250122145349.7220-6-david.plowman@raspberrypi.com>", "X-Mailer": "git-send-email 2.39.5", "In-Reply-To": "<20250122145349.7220-1-david.plowman@raspberrypi.com>", "References": "<20250122145349.7220-1-david.plowman@raspberrypi.com>", "MIME-Version": "1.0", "Content-Transfer-Encoding": "8bit", "X-BeenThere": "libcamera-devel@lists.libcamera.org", "X-Mailman-Version": "2.1.29", "Precedence": "list", "List-Id": "<libcamera-devel.lists.libcamera.org>", "List-Unsubscribe": "<https://lists.libcamera.org/options/libcamera-devel>,\n\t<mailto:libcamera-devel-request@lists.libcamera.org?subject=unsubscribe>", "List-Archive": "<https://lists.libcamera.org/pipermail/libcamera-devel/>", "List-Post": "<mailto:libcamera-devel@lists.libcamera.org>", "List-Help": "<mailto:libcamera-devel-request@lists.libcamera.org?subject=help>", "List-Subscribe": "<https://lists.libcamera.org/listinfo/libcamera-devel>,\n\t<mailto:libcamera-devel-request@lists.libcamera.org?subject=subscribe>", "Errors-To": "libcamera-devel-bounces@lists.libcamera.org", "Sender": "\"libcamera-devel\" <libcamera-devel-bounces@lists.libcamera.org>" }, "content": "From: Naushir Patuck <naush@raspberrypi.com>\n\nWe add a base class for a \"sync algorithm\", and define its inputs and\noutputs in the SyncStatus class.\n\nWe add the necessary plumbing to the base IPA code so as to arrange\nfor the necessary parameters to be made available to such an\nalgorithm, and also to handle the return values, passing them back as\nnecessary to the pipeline handler.\n\nSigned-off-by: Naushir Patuck <naush@raspberrypi.com>\nSigned-off-by: David Plowman <david.plowman@raspberrypi.com>\n---\n src/ipa/rpi/common/ipa_base.cpp | 61 +++++++++++++++++++++++--\n src/ipa/rpi/common/ipa_base.h | 4 +-\n src/ipa/rpi/controller/sync_algorithm.h | 31 +++++++++++++\n src/ipa/rpi/controller/sync_status.h | 27 +++++++++++\n 4 files changed, 118 insertions(+), 5 deletions(-)\n create mode 100644 src/ipa/rpi/controller/sync_algorithm.h\n create mode 100644 src/ipa/rpi/controller/sync_status.h", "diff": "diff --git a/src/ipa/rpi/common/ipa_base.cpp b/src/ipa/rpi/common/ipa_base.cpp\nindex bd3c2200..528e163c 100644\n--- a/src/ipa/rpi/common/ipa_base.cpp\n+++ b/src/ipa/rpi/common/ipa_base.cpp\n@@ -28,6 +28,8 @@\n #include \"controller/lux_status.h\"\n #include \"controller/sharpen_algorithm.h\"\n #include \"controller/statistics.h\"\n+#include \"controller/sync_algorithm.h\"\n+#include \"controller/sync_status.h\"\n \n namespace libcamera {\n \n@@ -81,6 +83,8 @@ const ControlInfoMap::Map ipaControls{\n \t{ &controls::Sharpness, ControlInfo(0.0f, 16.0f, 1.0f) },\n \t{ &controls::ScalerCrop, ControlInfo(Rectangle{}, Rectangle(65535, 65535, 65535, 65535), Rectangle{}) },\n \t{ &controls::FrameDurationLimits, ControlInfo(INT64_C(33333), INT64_C(120000)) },\n+\t{ &controls::rpi::SyncMode, ControlInfo(controls::rpi::SyncModeValues) },\n+\t{ &controls::rpi::SyncFrames, ControlInfo(1, 1000000, 100) },\n \t{ &controls::draft::NoiseReductionMode, ControlInfo(controls::draft::NoiseReductionModeValues) },\n \t{ &controls::rpi::StatsOutputEnable, ControlInfo(false, true, false) },\n };\n@@ -399,6 +403,7 @@ void IpaBase::prepareIsp(const PrepareParams ¶ms)\n \n \trpiMetadata.clear();\n \tfillDeviceStatus(params.sensorControls, ipaContext);\n+\tfillSyncParams(params, ipaContext);\n \n \tif (params.buffers.embedded) {\n \t\t/*\n@@ -497,10 +502,24 @@ void IpaBase::processStats(const ProcessParams ¶ms)\n \t\thelper_->process(statistics, rpiMetadata);\n \t\tcontroller_.process(statistics, &rpiMetadata);\n \n+\t\t/* Send any sync algorithm outputs back to the pipeline handler */\n+\t\tDuration offset(0s);\n+\t\tstruct SyncStatus syncStatus;\n+\t\tif (rpiMetadata.get(\"sync.status\", syncStatus) == 0) {\n+\t\t\tif (minFrameDuration_ != maxFrameDuration_)\n+\t\t\t\tLOG(IPARPI, Error) << \"Sync algorithm enabled with variable framerate. \"\n+\t\t\t\t\t\t << minFrameDuration_ << \" \" << maxFrameDuration_;\n+\t\t\toffset = syncStatus.frameDurationOffset;\n+\n+\t\t\tlibcameraMetadata_.set(controls::rpi::SyncReady, syncStatus.ready);\n+\t\t\tif (syncStatus.timerKnown)\n+\t\t\t\tlibcameraMetadata_.set(controls::rpi::SyncTimer, syncStatus.timerValue);\n+\t\t}\n+\n \t\tstruct AgcStatus agcStatus;\n \t\tif (rpiMetadata.get(\"agc.status\", agcStatus) == 0) {\n \t\t\tControlList ctrls(sensorCtrls_);\n-\t\t\tapplyAGC(&agcStatus, ctrls);\n+\t\t\tapplyAGC(&agcStatus, ctrls, offset);\n \t\t\tsetDelayedControls.emit(ctrls, ipaContext);\n \t\t\tsetCameraTimeoutValue();\n \t\t}\n@@ -737,6 +756,7 @@ void IpaBase::applyControls(const ControlList &controls)\n \tusing RPiController::ContrastAlgorithm;\n \tusing RPiController::DenoiseAlgorithm;\n \tusing RPiController::HdrAlgorithm;\n+\tusing RPiController::SyncAlgorithm;\n \n \t/* Clear the return metadata buffer. */\n \tlibcameraMetadata_.clear();\n@@ -1328,6 +1348,17 @@ void IpaBase::applyControls(const ControlList &controls)\n \t\t\tstatsMetadataOutput_ = ctrl.second.get<bool>();\n \t\t\tbreak;\n \n+\t\tcase controls::rpi::SYNC_FRAMES: {\n+\t\t\tSyncAlgorithm *sync = dynamic_cast<SyncAlgorithm *>(controller_.getAlgorithm(\"sync\"));\n+\n+\t\t\tif (sync) {\n+\t\t\t\tint frames = ctrl.second.get<int32_t>();\n+\t\t\t\tif (frames > 0)\n+\t\t\t\t\tsync->setReadyFrame(frames);\n+\t\t\t}\n+\t\t\tbreak;\n+\t\t}\n+\n \t\tdefault:\n \t\t\tLOG(IPARPI, Warning)\n \t\t\t\t<< \"Ctrl \" << controls::controls.at(ctrl.first)->name()\n@@ -1364,6 +1395,19 @@ void IpaBase::fillDeviceStatus(const ControlList &sensorControls, unsigned int i\n \trpiMetadata_[ipaContext].set(\"device.status\", deviceStatus);\n }\n \n+void IpaBase::fillSyncParams(const PrepareParams ¶ms, unsigned int ipaContext)\n+{\n+\tRPiController::SyncAlgorithm *sync = dynamic_cast<RPiController::SyncAlgorithm *>(\n+\t\tcontroller_.getAlgorithm(\"sync\"));\n+\tif (!sync)\n+\t\treturn;\n+\n+\tSyncParams syncParams;\n+\tsyncParams.wallClock = *params.sensorControls.get(controls::FrameWallClock);\n+\tsyncParams.sensorTimestamp = *params.sensorControls.get(controls::SensorTimestamp);\n+\trpiMetadata_[ipaContext].set(\"sync.params\", syncParams);\n+}\n+\n void IpaBase::reportMetadata(unsigned int ipaContext)\n {\n \tRPiController::Metadata &rpiMetadata = rpiMetadata_[ipaContext];\n@@ -1542,14 +1586,22 @@ void IpaBase::applyFrameDurations(Duration minFrameDuration, Duration maxFrameDu\n \t * value possible.\n \t */\n \tDuration maxExposureTime = Duration::max();\n-\thelper_->getBlanking(maxExposureTime, minFrameDuration_, maxFrameDuration_);\n+\tauto [vblank, hblank] = helper_->getBlanking(maxExposureTime, minFrameDuration_, maxFrameDuration_);\n \n \tRPiController::AgcAlgorithm *agc = dynamic_cast<RPiController::AgcAlgorithm *>(\n \t\tcontroller_.getAlgorithm(\"agc\"));\n \tagc->setMaxExposureTime(maxExposureTime);\n+\n+\tRPiController::SyncAlgorithm *sync = dynamic_cast<RPiController::SyncAlgorithm *>(\n+\t\tcontroller_.getAlgorithm(\"sync\"));\n+\tif (sync) {\n+\t\tDuration duration = (mode_.height + vblank) * ((mode_.width + hblank) * 1.0s / mode_.pixelRate);\n+\t\tLOG(IPARPI, Debug) << \"setting sync frame duration to \" << duration;\n+\t\tsync->setFrameDuration(duration);\n+\t}\n }\n \n-void IpaBase::applyAGC(const struct AgcStatus *agcStatus, ControlList &ctrls)\n+void IpaBase::applyAGC(const struct AgcStatus *agcStatus, ControlList &ctrls, Duration frameDurationOffset)\n {\n \tconst int32_t minGainCode = helper_->gainCode(mode_.minAnalogueGain);\n \tconst int32_t maxGainCode = helper_->gainCode(mode_.maxAnalogueGain);\n@@ -1564,7 +1616,8 @@ void IpaBase::applyAGC(const struct AgcStatus *agcStatus, ControlList &ctrls)\n \n \t/* getBlanking might clip exposure time to the fps limits. */\n \tDuration exposure = agcStatus->exposureTime;\n-\tauto [vblank, hblank] = helper_->getBlanking(exposure, minFrameDuration_, maxFrameDuration_);\n+\tauto [vblank, hblank] = helper_->getBlanking(exposure, minFrameDuration_ - frameDurationOffset,\n+\t\t\t\t\t\t maxFrameDuration_ - frameDurationOffset);\n \tint32_t exposureLines = helper_->exposureLines(exposure,\n \t\t\t\t\t\t helper_->hblankToLineLength(hblank));\n \ndiff --git a/src/ipa/rpi/common/ipa_base.h b/src/ipa/rpi/common/ipa_base.h\nindex 1a811beb..b53d0bfb 100644\n--- a/src/ipa/rpi/common/ipa_base.h\n+++ b/src/ipa/rpi/common/ipa_base.h\n@@ -95,9 +95,11 @@ private:\n \tvoid applyControls(const ControlList &controls);\n \tvirtual void handleControls(const ControlList &controls) = 0;\n \tvoid fillDeviceStatus(const ControlList &sensorControls, unsigned int ipaContext);\n+\tvoid fillSyncParams(const PrepareParams ¶ms, unsigned int ipaContext);\n \tvoid reportMetadata(unsigned int ipaContext);\n \tvoid applyFrameDurations(utils::Duration minFrameDuration, utils::Duration maxFrameDuration);\n-\tvoid applyAGC(const struct AgcStatus *agcStatus, ControlList &ctrls);\n+\tvoid applyAGC(const struct AgcStatus *agcStatus, ControlList &ctrls,\n+\t\t utils::Duration frameDurationOffset = utils::Duration(0));\n \n \tstd::map<unsigned int, MappedFrameBuffer> buffers_;\n \ndiff --git a/src/ipa/rpi/controller/sync_algorithm.h b/src/ipa/rpi/controller/sync_algorithm.h\nnew file mode 100644\nindex 00000000..c242def6\n--- /dev/null\n+++ b/src/ipa/rpi/controller/sync_algorithm.h\n@@ -0,0 +1,31 @@\n+/* SPDX-License-Identifier: BSD-2-Clause */\n+/*\n+ * Copyright (C) 2024, Raspberry Pi Ltd\n+ *\n+ * sync_algorithm.h - Camera sync algorithm interface\n+ */\n+#pragma once\n+\n+#include <libcamera/base/utils.h>\n+\n+#include \"algorithm.h\"\n+\n+namespace RPiController {\n+\n+class SyncAlgorithm : public Algorithm\n+{\n+public:\n+\tenum class Mode {\n+\t\tOff,\n+\t\tServer,\n+\t\tClient,\n+\t};\n+\n+\tSyncAlgorithm(Controller *controller)\n+\t\t: Algorithm(controller) {}\n+\tvirtual void setFrameDuration(libcamera::utils::Duration frameDuration) = 0;\n+\tvirtual void setReadyFrame(unsigned int frame) = 0;\n+\tvirtual void setMode(Mode mode) = 0;\n+};\n+\n+} /* namespace RPiController */\ndiff --git a/src/ipa/rpi/controller/sync_status.h b/src/ipa/rpi/controller/sync_status.h\nnew file mode 100644\nindex 00000000..10f97502\n--- /dev/null\n+++ b/src/ipa/rpi/controller/sync_status.h\n@@ -0,0 +1,27 @@\n+/* SPDX-License-Identifier: BSD-2-Clause */\n+/*\n+ * Copyright (C) 2024, Raspberry Pi Ltd\n+ *\n+ * sync_status.h - Sync algorithm params and status structures\n+ */\n+#pragma once\n+\n+#include <libcamera/base/utils.h>\n+\n+struct SyncParams {\n+\t/* Wall clock time for this frame */\n+\tuint64_t wallClock;\n+\t/* Kernel timestamp for this frame */\n+\tuint64_t sensorTimestamp;\n+};\n+\n+struct SyncStatus {\n+\t/* Frame length correction to apply */\n+\tlibcamera::utils::Duration frameDurationOffset;\n+\t/* Whether the \"ready time\" has been reached */\n+\tbool ready;\n+\t/* Time until the \"ready time\" */\n+\tint64_t timerValue;\n+\t/* Whether timerValue is known (client has to wait for a server message) */\n+\tbool timerKnown;\n+};\n", "prefixes": [ "v4", "5/7" ] }