Patch Detail
Show a patch.
GET /api/patches/22505/?format=api
{ "id": 22505, "url": "https://patchwork.libcamera.org/api/patches/22505/?format=api", "web_url": "https://patchwork.libcamera.org/patch/22505/", "project": { "id": 1, "url": "https://patchwork.libcamera.org/api/projects/1/?format=api", "name": "libcamera", "link_name": "libcamera", "list_id": "libcamera_core", "list_email": "libcamera-devel@lists.libcamera.org", "web_url": "", "scm_url": "", "webscm_url": "" }, "msgid": "<20250109143211.11939-6-david.plowman@raspberrypi.com>", "date": "2025-01-09T14:32:09", "name": "[v3,5/7] ipa: rpi: Add base classes and plumbing for sync algorithm", "commit_ref": null, "pull_url": null, "state": "superseded", "archived": false, "hash": "af970c00f6e44fe7e10349966937f0053094d753", "submitter": { "id": 42, "url": "https://patchwork.libcamera.org/api/people/42/?format=api", "name": "David Plowman", "email": "david.plowman@raspberrypi.com" }, "delegate": null, "mbox": "https://patchwork.libcamera.org/patch/22505/mbox/", "series": [ { "id": 4939, "url": "https://patchwork.libcamera.org/api/series/4939/?format=api", "web_url": "https://patchwork.libcamera.org/project/libcamera/list/?series=4939", "date": "2025-01-09T14:32:04", "name": "Camera synchronisation", "version": 3, "mbox": "https://patchwork.libcamera.org/series/4939/mbox/" } ], "comments": "https://patchwork.libcamera.org/api/patches/22505/comments/", "check": "pending", "checks": "https://patchwork.libcamera.org/api/patches/22505/checks/", "tags": {}, "headers": { "Return-Path": "<libcamera-devel-bounces@lists.libcamera.org>", "X-Original-To": "parsemail@patchwork.libcamera.org", "Delivered-To": "parsemail@patchwork.libcamera.org", "Received": [ "from lancelot.ideasonboard.com (lancelot.ideasonboard.com\n\t[92.243.16.209])\n\tby patchwork.libcamera.org (Postfix) with ESMTPS id E33E6C32EA\n\tfor <parsemail@patchwork.libcamera.org>;\n\tThu, 9 Jan 2025 14:32:33 +0000 (UTC)", "from lancelot.ideasonboard.com (localhost [IPv6:::1])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTP id 6B11968503;\n\tThu, 9 Jan 2025 15:32:33 +0100 (CET)", "from mail-wr1-x433.google.com (mail-wr1-x433.google.com\n\t[IPv6:2a00:1450:4864:20::433])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id 0F01F68503\n\tfor <libcamera-devel@lists.libcamera.org>;\n\tThu, 9 Jan 2025 15:32:22 +0100 (CET)", "by mail-wr1-x433.google.com with SMTP id\n\tffacd0b85a97d-385e87b25f0so1402834f8f.0\n\tfor <libcamera-devel@lists.libcamera.org>;\n\tThu, 09 Jan 2025 06:32:22 -0800 (PST)", "from raspberrypi.pitowers.org\n\t([2a00:1098:3142:1f:ffc9:aff6:7f7f:893b])\n\tby smtp.gmail.com with ESMTPSA id\n\tffacd0b85a97d-38a8e4b80b2sm1952569f8f.80.2025.01.09.06.32.19\n\t(version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256);\n\tThu, 09 Jan 2025 06:32:19 -0800 (PST)" ], "Authentication-Results": "lancelot.ideasonboard.com; dkim=pass (2048-bit key;\n\tunprotected) header.d=raspberrypi.com header.i=@raspberrypi.com\n\theader.b=\"TTT8QR2K\"; dkim-atps=neutral", "DKIM-Signature": "v=1; a=rsa-sha256; c=relaxed/relaxed;\n\td=raspberrypi.com; s=google; t=1736433141; x=1737037941;\n\tdarn=lists.libcamera.org; \n\th=content-transfer-encoding:mime-version:references:in-reply-to\n\t:message-id:date:subject:cc:to:from:from:to:cc:subject:date\n\t:message-id:reply-to;\n\tbh=yKDZcqefLc17FC1rrbX68afdo5XuZS6xj1yrJzIz0E8=;\n\tb=TTT8QR2KNPGsiR3lUhfdGNeXtV8TUe+btohZoqywftlnZZGsMy9FpsOFmyDlWD1kNF\n\tqFPnwrRP7ELH06fpIJu73/5EROJ/547XVNUUJU5z99R1ZIpE480FJ1yoq5UGJJ6cQ2Dn\n\t10+VFRmJuwruLoLMx6XQm2PCoU56kyfIrIDiQhfF5POzQZ/4T1xoeyjeFfhVer1ZcfjP\n\t7fqTkRVXjO3/Ftie6dVBH8CPHBZbn23n4TlYmQYPsM4Aazo6vVr1KmzAX1/k1DEaJx6m\n\tnR1h66SdxjrV3vqrNHikF253pwStHyG04ERyqlyHrh4LBbxyydJgrNK0vmTi9ZqUlQiE\n\tlaag==", "X-Google-DKIM-Signature": "v=1; a=rsa-sha256; c=relaxed/relaxed;\n\td=1e100.net; s=20230601; t=1736433141; x=1737037941;\n\th=content-transfer-encoding:mime-version:references:in-reply-to\n\t:message-id:date:subject:cc:to:from:x-gm-message-state:from:to:cc\n\t:subject:date:message-id:reply-to;\n\tbh=yKDZcqefLc17FC1rrbX68afdo5XuZS6xj1yrJzIz0E8=;\n\tb=vmFcpuxvP/plzEfHDjg0sKzQiFSEaelPGMYu27LVLQFyLxrlRw27Erz+GcGck2h4BX\n\t7y5b1c+7kzKfUQyYmL5KAJ8ooLyKPdclFRpazxhZegD0DzHNEQxmt2WQJzD93KrogFFc\n\tjENCslNbdwq/7GRk66dfyKqQQzglJ+FdjckaWhxb7G5I3Kr+ug0OH9pP7NlwJjNh/Y3m\n\tbQ7HExFqcGSveVUcpQUP/uCwGW7Csx0qbBy2O8u9slaDwZjOfEznkkOIlvaWbrJGI3AD\n\tkjQH64/Hg+iskC14NcQXxLQ87Fk34spl7EyxiX14o+gFfllcwUUfvxxG3/Il9cTfj1ei\n\tmG2A==", "X-Gm-Message-State": "AOJu0Yy79YLY+Npp3ufd3H6mTfGlHa6MMXfR97L4t+QeK+yE1itZQCvV\n\tCh1q+o9DfIbzrPqDOAJlXflFrLuvH4JtAeq25BlwP6Hd/k7OZmmbh6MKbyTFfqs7BclNwmKqkEr\n\tF", "X-Gm-Gg": "ASbGncs4V4pcXS+tyd7x2Q0znZFH9PxQ8aJ37EKcCbe2+EBfY4Od0lz77cTcP8eUTG0\n\tTGQoDFwhrzjp/cXZt4uqk8nP7QhqqFT2IMGqMkkWS3xrxEjQvtTCW5DIFrDWnxKZ4oesanXdHK7\n\tuLYNO50iMiVa5K38gQxgG4EdjCzmb8iS3qWq2VctSUOZs+raxz2G1znPIgBhNBY7qjTjJ4TeLJv\n\tcPRT2CfOnFQ8z6ah5q9QF3y6g2TJ0igo11YzhQQnroVneqDdI9cMmohJ6e9H8fOPfKC76aMaeao\n\t7PcYs8Ey8LSV", "X-Google-Smtp-Source": "AGHT+IHq0xP6QSB6NgJEmFnmTsPGaBBNH1bK4P6K3KOrhv/7jSLPXI3Ao01cKK+ef4v6MdJmF++pJA==", "X-Received": "by 2002:a05:6000:1f8a:b0:385:faec:d945 with SMTP id\n\tffacd0b85a97d-38a8b0d3202mr3391183f8f.9.1736433139842; \n\tThu, 09 Jan 2025 06:32:19 -0800 (PST)", "From": "David Plowman <david.plowman@raspberrypi.com>", "To": "libcamera-devel@lists.libcamera.org", "Cc": "Naushir Patuck <naush@raspberrypi.com>", "Subject": "[PATCH v3 5/7] ipa: rpi: Add base classes and plumbing for sync\n\talgorithm", "Date": "Thu, 9 Jan 2025 14:32:09 +0000", "Message-Id": "<20250109143211.11939-6-david.plowman@raspberrypi.com>", "X-Mailer": "git-send-email 2.39.5", "In-Reply-To": "<20250109143211.11939-1-david.plowman@raspberrypi.com>", "References": "<20250109143211.11939-1-david.plowman@raspberrypi.com>", "MIME-Version": "1.0", "Content-Transfer-Encoding": "8bit", "X-BeenThere": "libcamera-devel@lists.libcamera.org", "X-Mailman-Version": "2.1.29", "Precedence": "list", "List-Id": "<libcamera-devel.lists.libcamera.org>", "List-Unsubscribe": "<https://lists.libcamera.org/options/libcamera-devel>,\n\t<mailto:libcamera-devel-request@lists.libcamera.org?subject=unsubscribe>", "List-Archive": "<https://lists.libcamera.org/pipermail/libcamera-devel/>", "List-Post": "<mailto:libcamera-devel@lists.libcamera.org>", "List-Help": "<mailto:libcamera-devel-request@lists.libcamera.org?subject=help>", "List-Subscribe": "<https://lists.libcamera.org/listinfo/libcamera-devel>,\n\t<mailto:libcamera-devel-request@lists.libcamera.org?subject=subscribe>", "Errors-To": "libcamera-devel-bounces@lists.libcamera.org", "Sender": "\"libcamera-devel\" <libcamera-devel-bounces@lists.libcamera.org>" }, "content": "From: Naushir Patuck <naush@raspberrypi.com>\n\nWe add a base class for a \"sync algorithm\", and define its inputs and\noutputs in the SyncStatus class.\n\nWe add the necessary plumbing to the base IPA code so as to arrange\nfor the necessary parameters to be made available to such an\nalgorithm, and also to handle the return values, passing them back as\nnecessary to the pipeline handler.\n\nSigned-off-by: Naushir Patuck <naush@raspberrypi.com>\n---\n src/ipa/rpi/common/ipa_base.cpp | 78 +++++++++++++++++++++++--\n src/ipa/rpi/common/ipa_base.h | 4 +-\n src/ipa/rpi/controller/sync_algorithm.h | 31 ++++++++++\n src/ipa/rpi/controller/sync_status.h | 27 +++++++++\n 4 files changed, 135 insertions(+), 5 deletions(-)\n create mode 100644 src/ipa/rpi/controller/sync_algorithm.h\n create mode 100644 src/ipa/rpi/controller/sync_status.h", "diff": "diff --git a/src/ipa/rpi/common/ipa_base.cpp b/src/ipa/rpi/common/ipa_base.cpp\nindex 6ff1e22b..f1e4a161 100644\n--- a/src/ipa/rpi/common/ipa_base.cpp\n+++ b/src/ipa/rpi/common/ipa_base.cpp\n@@ -28,6 +28,8 @@\n #include \"controller/lux_status.h\"\n #include \"controller/sharpen_algorithm.h\"\n #include \"controller/statistics.h\"\n+#include \"controller/sync_algorithm.h\"\n+#include \"controller/sync_status.h\"\n \n namespace libcamera {\n \n@@ -72,6 +74,8 @@ const ControlInfoMap::Map ipaControls{\n \t{ &controls::Sharpness, ControlInfo(0.0f, 16.0f, 1.0f) },\n \t{ &controls::ScalerCrop, ControlInfo(Rectangle{}, Rectangle(65535, 65535, 65535, 65535), Rectangle{}) },\n \t{ &controls::FrameDurationLimits, ControlInfo(INT64_C(33333), INT64_C(120000)) },\n+\t{ &controls::rpi::SyncMode, ControlInfo(controls::rpi::SyncModeValues) },\n+\t{ &controls::rpi::SyncFrames, ControlInfo(1, 1000000, 100) },\n \t{ &controls::draft::NoiseReductionMode, ControlInfo(controls::draft::NoiseReductionModeValues) },\n \t{ &controls::rpi::StatsOutputEnable, ControlInfo(false, true, false) },\n };\n@@ -390,6 +394,7 @@ void IpaBase::prepareIsp(const PrepareParams ¶ms)\n \n \trpiMetadata.clear();\n \tfillDeviceStatus(params.sensorControls, ipaContext);\n+\tfillSyncParams(params, ipaContext);\n \n \tif (params.buffers.embedded) {\n \t\t/*\n@@ -488,10 +493,23 @@ void IpaBase::processStats(const ProcessParams ¶ms)\n \t\thelper_->process(statistics, rpiMetadata);\n \t\tcontroller_.process(statistics, &rpiMetadata);\n \n+\t\t/* Send any sync algorithm outputs back to the pipeline handler */\n+\t\tDuration offset(0s);\n+\t\tstruct SyncStatus syncStatus;\n+\t\tif (rpiMetadata.get(\"sync.status\", syncStatus) == 0) {\n+\t\t\tif (minFrameDuration_ != maxFrameDuration_)\n+\t\t\t\tLOG(IPARPI, Error) << \"Sync algorithm enabled with variable framerate. \" << minFrameDuration_ << \" \" << maxFrameDuration_;\n+\t\t\toffset = syncStatus.frameDurationOffset;\n+\n+\t\t\tlibcameraMetadata_.set(controls::rpi::SyncReady, syncStatus.ready);\n+\t\t\tif (syncStatus.timerKnown)\n+\t\t\t\tlibcameraMetadata_.set(controls::rpi::SyncTimer, syncStatus.timerValue);\n+\t\t}\n+\n \t\tstruct AgcStatus agcStatus;\n \t\tif (rpiMetadata.get(\"agc.status\", agcStatus) == 0) {\n \t\t\tControlList ctrls(sensorCtrls_);\n-\t\t\tapplyAGC(&agcStatus, ctrls);\n+\t\t\tapplyAGC(&agcStatus, ctrls, offset);\n \t\t\tsetDelayedControls.emit(ctrls, ipaContext);\n \t\t\tsetCameraTimeoutValue();\n \t\t}\n@@ -728,6 +746,7 @@ void IpaBase::applyControls(const ControlList &controls)\n \tusing RPiController::ContrastAlgorithm;\n \tusing RPiController::DenoiseAlgorithm;\n \tusing RPiController::HdrAlgorithm;\n+\tusing RPiController::SyncAlgorithm;\n \n \t/* Clear the return metadata buffer. */\n \tlibcameraMetadata_.clear();\n@@ -1274,6 +1293,35 @@ void IpaBase::applyControls(const ControlList &controls)\n \t\t\tstatsMetadataOutput_ = ctrl.second.get<bool>();\n \t\t\tbreak;\n \n+\t\tcase controls::rpi::SYNC_MODE: {\n+\t\t\tSyncAlgorithm *sync = dynamic_cast<SyncAlgorithm *>(controller_.getAlgorithm(\"sync\"));\n+\n+\t\t\tif (sync) {\n+\t\t\t\tint mode = ctrl.second.get<int32_t>();\n+\t\t\t\tSyncAlgorithm::Mode m = SyncAlgorithm::Mode::Off;\n+\t\t\t\tif (mode == controls::rpi::SyncModeServer) {\n+\t\t\t\t\tm = SyncAlgorithm::Mode::Server;\n+\t\t\t\t\tLOG(IPARPI, Info) << \"Sync mode set to server\";\n+\t\t\t\t} else if (mode == controls::rpi::SyncModeClient) {\n+\t\t\t\t\tm = SyncAlgorithm::Mode::Client;\n+\t\t\t\t\tLOG(IPARPI, Info) << \"Sync mode set to client\";\n+\t\t\t\t}\n+\t\t\t\tsync->setMode(m);\n+\t\t\t}\n+\t\t\tbreak;\n+\t\t}\n+\n+\t\tcase controls::rpi::SYNC_FRAMES: {\n+\t\t\tSyncAlgorithm *sync = dynamic_cast<SyncAlgorithm *>(controller_.getAlgorithm(\"sync\"));\n+\n+\t\t\tif (sync) {\n+\t\t\t\tint frames = ctrl.second.get<int32_t>();\n+\t\t\t\tif (frames > 0)\n+\t\t\t\t\tsync->setReadyFrame(frames);\n+\t\t\t}\n+\t\t\tbreak;\n+\t\t}\n+\n \t\tdefault:\n \t\t\tLOG(IPARPI, Warning)\n \t\t\t\t<< \"Ctrl \" << controls::controls.at(ctrl.first)->name()\n@@ -1310,6 +1358,19 @@ void IpaBase::fillDeviceStatus(const ControlList &sensorControls, unsigned int i\n \trpiMetadata_[ipaContext].set(\"device.status\", deviceStatus);\n }\n \n+void IpaBase::fillSyncParams(const PrepareParams ¶ms, unsigned int ipaContext)\n+{\n+\tRPiController::SyncAlgorithm *sync = dynamic_cast<RPiController::SyncAlgorithm *>(\n+\t\tcontroller_.getAlgorithm(\"sync\"));\n+\tif (!sync)\n+\t\treturn;\n+\n+\tSyncParams syncParams;\n+\tsyncParams.wallClock = *params.sensorControls.get(controls::FrameWallClock);\n+\tsyncParams.sensorTimestamp = *params.sensorControls.get(controls::SensorTimestamp);\n+\trpiMetadata_[ipaContext].set(\"sync.params\", syncParams);\n+}\n+\n void IpaBase::reportMetadata(unsigned int ipaContext)\n {\n \tRPiController::Metadata &rpiMetadata = rpiMetadata_[ipaContext];\n@@ -1478,14 +1539,22 @@ void IpaBase::applyFrameDurations(Duration minFrameDuration, Duration maxFrameDu\n \t * value possible.\n \t */\n \tDuration maxExposureTime = Duration::max();\n-\thelper_->getBlanking(maxExposureTime, minFrameDuration_, maxFrameDuration_);\n+\tauto [vblank, hblank] = helper_->getBlanking(maxExposureTime, minFrameDuration_, maxFrameDuration_);\n \n \tRPiController::AgcAlgorithm *agc = dynamic_cast<RPiController::AgcAlgorithm *>(\n \t\tcontroller_.getAlgorithm(\"agc\"));\n \tagc->setMaxExposureTime(maxExposureTime);\n+\n+\tRPiController::SyncAlgorithm *sync = dynamic_cast<RPiController::SyncAlgorithm *>(\n+\t\tcontroller_.getAlgorithm(\"sync\"));\n+\tif (sync) {\n+\t\tDuration duration = (mode_.height + vblank) * ((mode_.width + hblank) * 1.0s / mode_.pixelRate);\n+\t\tLOG(IPARPI, Debug) << \"setting sync frame duration to \" << duration;\n+\t\tsync->setFrameDuration(duration);\n+\t}\n }\n \n-void IpaBase::applyAGC(const struct AgcStatus *agcStatus, ControlList &ctrls)\n+void IpaBase::applyAGC(const struct AgcStatus *agcStatus, ControlList &ctrls, Duration frameDurationOffset)\n {\n \tconst int32_t minGainCode = helper_->gainCode(mode_.minAnalogueGain);\n \tconst int32_t maxGainCode = helper_->gainCode(mode_.maxAnalogueGain);\n@@ -1500,7 +1569,8 @@ void IpaBase::applyAGC(const struct AgcStatus *agcStatus, ControlList &ctrls)\n \n \t/* getBlanking might clip exposure time to the fps limits. */\n \tDuration exposure = agcStatus->exposureTime;\n-\tauto [vblank, hblank] = helper_->getBlanking(exposure, minFrameDuration_, maxFrameDuration_);\n+\tauto [vblank, hblank] = helper_->getBlanking(exposure, minFrameDuration_ - frameDurationOffset,\n+\t\t\t\t\t\t maxFrameDuration_ - frameDurationOffset);\n \tint32_t exposureLines = helper_->exposureLines(exposure,\n \t\t\t\t\t\t helper_->hblankToLineLength(hblank));\n \ndiff --git a/src/ipa/rpi/common/ipa_base.h b/src/ipa/rpi/common/ipa_base.h\nindex 1a811beb..b53d0bfb 100644\n--- a/src/ipa/rpi/common/ipa_base.h\n+++ b/src/ipa/rpi/common/ipa_base.h\n@@ -95,9 +95,11 @@ private:\n \tvoid applyControls(const ControlList &controls);\n \tvirtual void handleControls(const ControlList &controls) = 0;\n \tvoid fillDeviceStatus(const ControlList &sensorControls, unsigned int ipaContext);\n+\tvoid fillSyncParams(const PrepareParams ¶ms, unsigned int ipaContext);\n \tvoid reportMetadata(unsigned int ipaContext);\n \tvoid applyFrameDurations(utils::Duration minFrameDuration, utils::Duration maxFrameDuration);\n-\tvoid applyAGC(const struct AgcStatus *agcStatus, ControlList &ctrls);\n+\tvoid applyAGC(const struct AgcStatus *agcStatus, ControlList &ctrls,\n+\t\t utils::Duration frameDurationOffset = utils::Duration(0));\n \n \tstd::map<unsigned int, MappedFrameBuffer> buffers_;\n \ndiff --git a/src/ipa/rpi/controller/sync_algorithm.h b/src/ipa/rpi/controller/sync_algorithm.h\nnew file mode 100644\nindex 00000000..c242def6\n--- /dev/null\n+++ b/src/ipa/rpi/controller/sync_algorithm.h\n@@ -0,0 +1,31 @@\n+/* SPDX-License-Identifier: BSD-2-Clause */\n+/*\n+ * Copyright (C) 2024, Raspberry Pi Ltd\n+ *\n+ * sync_algorithm.h - Camera sync algorithm interface\n+ */\n+#pragma once\n+\n+#include <libcamera/base/utils.h>\n+\n+#include \"algorithm.h\"\n+\n+namespace RPiController {\n+\n+class SyncAlgorithm : public Algorithm\n+{\n+public:\n+\tenum class Mode {\n+\t\tOff,\n+\t\tServer,\n+\t\tClient,\n+\t};\n+\n+\tSyncAlgorithm(Controller *controller)\n+\t\t: Algorithm(controller) {}\n+\tvirtual void setFrameDuration(libcamera::utils::Duration frameDuration) = 0;\n+\tvirtual void setReadyFrame(unsigned int frame) = 0;\n+\tvirtual void setMode(Mode mode) = 0;\n+};\n+\n+} /* namespace RPiController */\ndiff --git a/src/ipa/rpi/controller/sync_status.h b/src/ipa/rpi/controller/sync_status.h\nnew file mode 100644\nindex 00000000..10f97502\n--- /dev/null\n+++ b/src/ipa/rpi/controller/sync_status.h\n@@ -0,0 +1,27 @@\n+/* SPDX-License-Identifier: BSD-2-Clause */\n+/*\n+ * Copyright (C) 2024, Raspberry Pi Ltd\n+ *\n+ * sync_status.h - Sync algorithm params and status structures\n+ */\n+#pragma once\n+\n+#include <libcamera/base/utils.h>\n+\n+struct SyncParams {\n+\t/* Wall clock time for this frame */\n+\tuint64_t wallClock;\n+\t/* Kernel timestamp for this frame */\n+\tuint64_t sensorTimestamp;\n+};\n+\n+struct SyncStatus {\n+\t/* Frame length correction to apply */\n+\tlibcamera::utils::Duration frameDurationOffset;\n+\t/* Whether the \"ready time\" has been reached */\n+\tbool ready;\n+\t/* Time until the \"ready time\" */\n+\tint64_t timerValue;\n+\t/* Whether timerValue is known (client has to wait for a server message) */\n+\tbool timerKnown;\n+};\n", "prefixes": [ "v3", "5/7" ] }