Patch Detail
Show a patch.
GET /api/1.1/patches/21515/?format=api
{ "id": 21515, "url": "https://patchwork.libcamera.org/api/1.1/patches/21515/?format=api", "web_url": "https://patchwork.libcamera.org/patch/21515/", "project": { "id": 1, "url": "https://patchwork.libcamera.org/api/1.1/projects/1/?format=api", "name": "libcamera", "link_name": "libcamera", "list_id": "libcamera_core", "list_email": "libcamera-devel@lists.libcamera.org", "web_url": "", "scm_url": "", "webscm_url": "" }, "msgid": "<20241004115558.9166-5-david.plowman@raspberrypi.com>", "date": "2024-10-04T11:55:56", "name": "[4/6] ipa: rpi: Add base classes and plumbing for sync algorithm", "commit_ref": null, "pull_url": null, "state": "superseded", "archived": false, "hash": "5bf28c718f1132f4803cf305eef1d4a9ba9f91f5", "submitter": { "id": 42, "url": "https://patchwork.libcamera.org/api/1.1/people/42/?format=api", "name": "David Plowman", "email": "david.plowman@raspberrypi.com" }, "delegate": null, "mbox": "https://patchwork.libcamera.org/patch/21515/mbox/", "series": [ { "id": 4658, "url": "https://patchwork.libcamera.org/api/1.1/series/4658/?format=api", "web_url": "https://patchwork.libcamera.org/project/libcamera/list/?series=4658", "date": "2024-10-04T11:55:52", "name": "Raspberry Pi software camera sync algorithm", "version": 1, "mbox": "https://patchwork.libcamera.org/series/4658/mbox/" } ], "comments": "https://patchwork.libcamera.org/api/patches/21515/comments/", "check": "pending", "checks": "https://patchwork.libcamera.org/api/patches/21515/checks/", "tags": {}, "headers": { "Return-Path": "<libcamera-devel-bounces@lists.libcamera.org>", "X-Original-To": "parsemail@patchwork.libcamera.org", "Delivered-To": "parsemail@patchwork.libcamera.org", "Received": [ "from lancelot.ideasonboard.com (lancelot.ideasonboard.com\n\t[92.243.16.209])\n\tby patchwork.libcamera.org (Postfix) with ESMTPS id 7AC8AC3263\n\tfor <parsemail@patchwork.libcamera.org>;\n\tFri, 4 Oct 2024 11:56:19 +0000 (UTC)", "from lancelot.ideasonboard.com (localhost [IPv6:::1])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTP id A82FB6352E;\n\tFri, 4 Oct 2024 13:56:18 +0200 (CEST)", "from mail-wr1-x432.google.com (mail-wr1-x432.google.com\n\t[IPv6:2a00:1450:4864:20::432])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id A90FC62C8F\n\tfor <libcamera-devel@lists.libcamera.org>;\n\tFri, 4 Oct 2024 13:56:05 +0200 (CEST)", "by mail-wr1-x432.google.com with SMTP id\n\tffacd0b85a97d-37cc60c9838so1208580f8f.1\n\tfor <libcamera-devel@lists.libcamera.org>;\n\tFri, 04 Oct 2024 04:56:05 -0700 (PDT)", "from raspberrypi.pitowers.org\n\t([2a00:1098:3142:1f:daa2:371b:a97:3e3e])\n\tby smtp.gmail.com with ESMTPSA id\n\tffacd0b85a97d-37d081f743esm3107147f8f.21.2024.10.04.04.56.04\n\t(version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256);\n\tFri, 04 Oct 2024 04:56:04 -0700 (PDT)" ], "Authentication-Results": "lancelot.ideasonboard.com; dkim=pass (2048-bit key;\n\tunprotected) header.d=raspberrypi.com header.i=@raspberrypi.com\n\theader.b=\"LmtZAUU+\"; dkim-atps=neutral", "DKIM-Signature": "v=1; a=rsa-sha256; c=relaxed/relaxed;\n\td=raspberrypi.com; s=google; t=1728042965; x=1728647765;\n\tdarn=lists.libcamera.org; \n\th=content-transfer-encoding:mime-version:references:in-reply-to\n\t:message-id:date:subject:cc:to:from:from:to:cc:subject:date\n\t:message-id:reply-to;\n\tbh=vTL6DimDDA9r/zGhGutNdHJ4z5oVSpHpu6UwaAbb2SM=;\n\tb=LmtZAUU+E8MQmocUmAklarujj0MEbG0P0pc8r7ULItYQ/0JC5R5TtYJhH2EsmiJ1xu\n\tneeN2iXnMeDIbQFGM1urv1OKYoO+oGUmPbpk3zdiCYMsoBTEQ2uOLzdWQNkrnrRrjT42\n\tgJiYe7tty8x/m+twbH88lOwLD1NUNU7PM8UQLq2qol9057AnNJxZ8FqTIRkiQYv7qGyh\n\tYQGqUVXc2t0Q97cBMYPVpkF84n+ZsmZH2Fv6UTwy/gpXJCZeeynI/Zt8tRH+WHqytfb8\n\tqFvJPLcD+add6aKwFq0nVCGcRHx1B0esgsPGPQBl15RLzH2tVqgNqS1HySwih2CKznoh\n\tHx0w==", "X-Google-DKIM-Signature": "v=1; a=rsa-sha256; c=relaxed/relaxed;\n\td=1e100.net; s=20230601; t=1728042965; x=1728647765;\n\th=content-transfer-encoding:mime-version:references:in-reply-to\n\t:message-id:date:subject:cc:to:from:x-gm-message-state:from:to:cc\n\t:subject:date:message-id:reply-to;\n\tbh=vTL6DimDDA9r/zGhGutNdHJ4z5oVSpHpu6UwaAbb2SM=;\n\tb=nln00oqug7Cjp+MsNuf9jB0ptJmCrpAVnH6nKEB8BaOuKXPR4A0bSx34FH6Mknxmuv\n\tatgv3ET/ihOaHnZnZ/w1t2fqZaX8oPioWCoJkMrksV+pO2EwrpfzMHsZi3Cbaysgwj+K\n\t+6drmJ5E+7K6g3l4hITDMsD2Lbk2Q58mNVW8hlONlIZFfZbK/id/D6CCQcZDv9yH8mFl\n\tqJNhya3VZTLw494jUzqTVT7hBmApyz70pfSClzdZ2fRiiyPvNx6rMniOjEnZV+p//tWI\n\tvTLYTZ8/UtixWVf3+co/ngbQEgoa8QH+lkLt3bm4PZ0C4+AQed+Ro+WvmEb3TQmLii9L\n\tn1tg==", "X-Gm-Message-State": "AOJu0Yxrw0cbTbwkm7DWkz3XAibCW0VDN3vpoXN7f2pwmYByYJPj9zuQ\n\tjSCSnlKz7a1JnQS891k/KT90AS5tYILd5NbG1k7qFtokLEN34Cga7JToD5rKzmgu952eoZVPKx0\n\tB", "X-Google-Smtp-Source": "AGHT+IGWtR7pK77SJNmhk/qhH4mOhADTbEAGgq3yA8ofYsOo/aumQKRwPNN3g1WWUZDhCDe3t9ivaA==", "X-Received": "by 2002:a05:6000:1569:b0:374:c6b6:c656 with SMTP id\n\tffacd0b85a97d-37d0f70ca1emr1793382f8f.21.1728042964666; \n\tFri, 04 Oct 2024 04:56:04 -0700 (PDT)", "From": "David Plowman <david.plowman@raspberrypi.com>", "To": "libcamera-devel@lists.libcamera.org", "Cc": "Naushir Patuck <naush@raspberrypi.com>,\n\tDavid Plowman <david.plowman@raspberrypi.com>", "Subject": "[PATCH 4/6] ipa: rpi: Add base classes and plumbing for sync\n\talgorithm", "Date": "Fri, 4 Oct 2024 12:55:56 +0100", "Message-Id": "<20241004115558.9166-5-david.plowman@raspberrypi.com>", "X-Mailer": "git-send-email 2.39.5", "In-Reply-To": "<20241004115558.9166-1-david.plowman@raspberrypi.com>", "References": "<20241004115558.9166-1-david.plowman@raspberrypi.com>", "MIME-Version": "1.0", "Content-Transfer-Encoding": "8bit", "X-BeenThere": "libcamera-devel@lists.libcamera.org", "X-Mailman-Version": "2.1.29", "Precedence": "list", "List-Id": "<libcamera-devel.lists.libcamera.org>", "List-Unsubscribe": "<https://lists.libcamera.org/options/libcamera-devel>,\n\t<mailto:libcamera-devel-request@lists.libcamera.org?subject=unsubscribe>", "List-Archive": "<https://lists.libcamera.org/pipermail/libcamera-devel/>", "List-Post": "<mailto:libcamera-devel@lists.libcamera.org>", "List-Help": "<mailto:libcamera-devel-request@lists.libcamera.org?subject=help>", "List-Subscribe": "<https://lists.libcamera.org/listinfo/libcamera-devel>,\n\t<mailto:libcamera-devel-request@lists.libcamera.org?subject=subscribe>", "Errors-To": "libcamera-devel-bounces@lists.libcamera.org", "Sender": "\"libcamera-devel\" <libcamera-devel-bounces@lists.libcamera.org>" }, "content": "From: Naushir Patuck <naush@raspberrypi.com>\n\nWe add a base class for a \"sync algorithm\", and define its inputs and\noutputs in the SyncStatus class.\n\nWe add the necessary plumbing to the base IPA code so as to arrange\nfor the necessary parameters to be made available to such an\nalgorithm, and also to handle the return values, passing them back as\nnecessary to the pipeline handler.\n\nSigned-off-by: Naushir Patuck <naush@raspberrypi.com>\nReviewed-by: David Plowman <david.plowman@raspberrypi.com>\n---\n src/ipa/rpi/common/ipa_base.cpp | 66 +++++++++++++++++++++++--\n src/ipa/rpi/common/ipa_base.h | 4 +-\n src/ipa/rpi/controller/sync_algorithm.h | 30 +++++++++++\n src/ipa/rpi/controller/sync_status.h | 27 ++++++++++\n 4 files changed, 122 insertions(+), 5 deletions(-)\n create mode 100644 src/ipa/rpi/controller/sync_algorithm.h\n create mode 100644 src/ipa/rpi/controller/sync_status.h", "diff": "diff --git a/src/ipa/rpi/common/ipa_base.cpp b/src/ipa/rpi/common/ipa_base.cpp\nindex ee3848b5..ac228791 100644\n--- a/src/ipa/rpi/common/ipa_base.cpp\n+++ b/src/ipa/rpi/common/ipa_base.cpp\n@@ -28,6 +28,8 @@\n #include \"controller/lux_status.h\"\n #include \"controller/sharpen_algorithm.h\"\n #include \"controller/statistics.h\"\n+#include \"controller/sync_algorithm.h\"\n+#include \"controller/sync_status.h\"\n \n namespace libcamera {\n \n@@ -74,6 +76,7 @@ const ControlInfoMap::Map ipaControls{\n \t{ &controls::FrameDurationLimits, ControlInfo(INT64_C(33333), INT64_C(120000)) },\n \t{ &controls::draft::NoiseReductionMode, ControlInfo(controls::draft::NoiseReductionModeValues) },\n \t{ &controls::rpi::StatsOutputEnable, ControlInfo(false, true, false) },\n+\t{ &controls::rpi::SyncMode, ControlInfo(controls::rpi::SyncModeValues) },\n };\n \n /* IPA controls handled conditionally, if the sensor is not mono */\n@@ -387,6 +390,7 @@ void IpaBase::prepareIsp(const PrepareParams ¶ms)\n \n \trpiMetadata.clear();\n \tfillDeviceStatus(params.sensorControls, ipaContext);\n+\tfillSyncParams(params, ipaContext);\n \n \tif (params.buffers.embedded) {\n \t\t/*\n@@ -485,10 +489,23 @@ void IpaBase::processStats(const ProcessParams ¶ms)\n \t\thelper_->process(statistics, rpiMetadata);\n \t\tcontroller_.process(statistics, &rpiMetadata);\n \n+\t\t/* Send any sync algorithm outputs back to the pipeline handler */\n+\t\tDuration offset(0s);\n+\t\tstruct SyncStatus syncStatus;\n+\t\tif (rpiMetadata.get(\"sync.status\", syncStatus) == 0) {\n+\t\t\tif (minFrameDuration_ != maxFrameDuration_)\n+\t\t\t\tLOG(IPARPI, Error) << \"Sync algorithm enabled with variable framerate. \" << minFrameDuration_ << \" \" << maxFrameDuration_;\n+\t\t\toffset = syncStatus.frameDurationOffset;\n+\n+\t\t\tlibcameraMetadata_.set(controls::rpi::SyncWait, !syncStatus.ready);\n+\t\t\tif (syncStatus.lagKnown)\n+\t\t\t\tlibcameraMetadata_.set(controls::rpi::SyncLag, syncStatus.lag);\n+\t\t}\n+\n \t\tstruct AgcStatus agcStatus;\n \t\tif (rpiMetadata.get(\"agc.status\", agcStatus) == 0) {\n \t\t\tControlList ctrls(sensorCtrls_);\n-\t\t\tapplyAGC(&agcStatus, ctrls);\n+\t\t\tapplyAGC(&agcStatus, ctrls, offset);\n \t\t\tsetDelayedControls.emit(ctrls, ipaContext);\n \t\t\tsetCameraTimeoutValue();\n \t\t}\n@@ -721,6 +738,7 @@ void IpaBase::applyControls(const ControlList &controls)\n \tusing RPiController::ContrastAlgorithm;\n \tusing RPiController::DenoiseAlgorithm;\n \tusing RPiController::HdrAlgorithm;\n+\tusing RPiController::SyncAlgorithm;\n \n \t/* Clear the return metadata buffer. */\n \tlibcameraMetadata_.clear();\n@@ -1247,6 +1265,24 @@ void IpaBase::applyControls(const ControlList &controls)\n \t\t\tstatsMetadataOutput_ = ctrl.second.get<bool>();\n \t\t\tbreak;\n \n+\t\tcase controls::rpi::SYNC_MODE: {\n+\t\t\tSyncAlgorithm *sync = dynamic_cast<SyncAlgorithm *>(controller_.getAlgorithm(\"sync\"));\n+\n+\t\t\tif (sync) {\n+\t\t\t\tint mode = ctrl.second.get<int32_t>();\n+\t\t\t\tSyncAlgorithm::Mode m = SyncAlgorithm::Mode::Off;\n+\t\t\t\tif (mode == controls::rpi::SyncModeServer) {\n+\t\t\t\t\tm = SyncAlgorithm::Mode::Server;\n+\t\t\t\t\tLOG(IPARPI, Info) << \"Sync mode set to server\";\n+\t\t\t\t} else if (mode == controls::rpi::SyncModeClient) {\n+\t\t\t\t\tm = SyncAlgorithm::Mode::Client;\n+\t\t\t\t\tLOG(IPARPI, Info) << \"Sync mode set to client\";\n+\t\t\t\t}\n+\t\t\t\tsync->setMode(m);\n+\t\t\t}\n+\t\t\tbreak;\n+\t\t}\n+\n \t\tdefault:\n \t\t\tLOG(IPARPI, Warning)\n \t\t\t\t<< \"Ctrl \" << controls::controls.at(ctrl.first)->name()\n@@ -1283,6 +1319,19 @@ void IpaBase::fillDeviceStatus(const ControlList &sensorControls, unsigned int i\n \trpiMetadata_[ipaContext].set(\"device.status\", deviceStatus);\n }\n \n+void IpaBase::fillSyncParams(const PrepareParams ¶ms, unsigned int ipaContext)\n+{\n+\tRPiController::SyncAlgorithm *sync = dynamic_cast<RPiController::SyncAlgorithm *>(\n+\t\tcontroller_.getAlgorithm(\"sync\"));\n+\tif (!sync)\n+\t\treturn;\n+\n+\tSyncParams syncParams;\n+\tsyncParams.wallClock = *params.sensorControls.get(controls::rpi::FrameWallClock);\n+\tsyncParams.sensorTimestamp = *params.sensorControls.get(controls::SensorTimestamp);\n+\trpiMetadata_[ipaContext].set(\"sync.params\", syncParams);\n+}\n+\n void IpaBase::reportMetadata(unsigned int ipaContext)\n {\n \tRPiController::Metadata &rpiMetadata = rpiMetadata_[ipaContext];\n@@ -1451,14 +1500,22 @@ void IpaBase::applyFrameDurations(Duration minFrameDuration, Duration maxFrameDu\n \t * value possible.\n \t */\n \tDuration maxShutter = Duration::max();\n-\thelper_->getBlanking(maxShutter, minFrameDuration_, maxFrameDuration_);\n+\tauto [vblank, hblank] = helper_->getBlanking(maxShutter, minFrameDuration_, maxFrameDuration_);\n \n \tRPiController::AgcAlgorithm *agc = dynamic_cast<RPiController::AgcAlgorithm *>(\n \t\tcontroller_.getAlgorithm(\"agc\"));\n \tagc->setMaxShutter(maxShutter);\n+\n+\tRPiController::SyncAlgorithm *sync = dynamic_cast<RPiController::SyncAlgorithm *>(\n+\t\tcontroller_.getAlgorithm(\"sync\"));\n+\tif (sync) {\n+\t\tDuration duration = (mode_.height + vblank) * ((mode_.width + hblank) * 1.0s / mode_.pixelRate);\n+\t\tLOG(IPARPI, Debug) << \"setting sync frame duration to \" << duration;\n+\t\tsync->setFrameDuration(duration);\n+\t}\n }\n \n-void IpaBase::applyAGC(const struct AgcStatus *agcStatus, ControlList &ctrls)\n+void IpaBase::applyAGC(const struct AgcStatus *agcStatus, ControlList &ctrls, Duration frameDurationOffset)\n {\n \tconst int32_t minGainCode = helper_->gainCode(mode_.minAnalogueGain);\n \tconst int32_t maxGainCode = helper_->gainCode(mode_.maxAnalogueGain);\n@@ -1473,7 +1530,8 @@ void IpaBase::applyAGC(const struct AgcStatus *agcStatus, ControlList &ctrls)\n \n \t/* getBlanking might clip exposure time to the fps limits. */\n \tDuration exposure = agcStatus->shutterTime;\n-\tauto [vblank, hblank] = helper_->getBlanking(exposure, minFrameDuration_, maxFrameDuration_);\n+\tauto [vblank, hblank] = helper_->getBlanking(exposure, minFrameDuration_ - frameDurationOffset,\n+\t\t\t\t\t\t maxFrameDuration_ - frameDurationOffset);\n \tint32_t exposureLines = helper_->exposureLines(exposure,\n \t\t\t\t\t\t helper_->hblankToLineLength(hblank));\n \ndiff --git a/src/ipa/rpi/common/ipa_base.h b/src/ipa/rpi/common/ipa_base.h\nindex 1a811beb..b53d0bfb 100644\n--- a/src/ipa/rpi/common/ipa_base.h\n+++ b/src/ipa/rpi/common/ipa_base.h\n@@ -95,9 +95,11 @@ private:\n \tvoid applyControls(const ControlList &controls);\n \tvirtual void handleControls(const ControlList &controls) = 0;\n \tvoid fillDeviceStatus(const ControlList &sensorControls, unsigned int ipaContext);\n+\tvoid fillSyncParams(const PrepareParams ¶ms, unsigned int ipaContext);\n \tvoid reportMetadata(unsigned int ipaContext);\n \tvoid applyFrameDurations(utils::Duration minFrameDuration, utils::Duration maxFrameDuration);\n-\tvoid applyAGC(const struct AgcStatus *agcStatus, ControlList &ctrls);\n+\tvoid applyAGC(const struct AgcStatus *agcStatus, ControlList &ctrls,\n+\t\t utils::Duration frameDurationOffset = utils::Duration(0));\n \n \tstd::map<unsigned int, MappedFrameBuffer> buffers_;\n \ndiff --git a/src/ipa/rpi/controller/sync_algorithm.h b/src/ipa/rpi/controller/sync_algorithm.h\nnew file mode 100644\nindex 00000000..298b5951\n--- /dev/null\n+++ b/src/ipa/rpi/controller/sync_algorithm.h\n@@ -0,0 +1,30 @@\n+/* SPDX-License-Identifier: BSD-2-Clause */\n+/*\n+ * Copyright (C) 2024, Raspberry Pi Ltd\n+ *\n+ * sync_algorithm.h - Camera sync algorithm interface\n+ */\n+#pragma once\n+\n+#include <libcamera/base/utils.h>\n+\n+#include \"algorithm.h\"\n+\n+namespace RPiController {\n+\n+class SyncAlgorithm : public Algorithm\n+{\n+public:\n+\tenum class Mode {\n+\t\tOff,\n+\t\tServer,\n+\t\tClient,\n+\t};\n+\n+\tSyncAlgorithm(Controller *controller)\n+\t\t: Algorithm(controller) {}\n+\tvirtual void setFrameDuration(libcamera::utils::Duration frameDuration) = 0;\n+\tvirtual void setMode(Mode mode) = 0;\n+};\n+\n+} /* namespace RPiController */\ndiff --git a/src/ipa/rpi/controller/sync_status.h b/src/ipa/rpi/controller/sync_status.h\nnew file mode 100644\nindex 00000000..289f0182\n--- /dev/null\n+++ b/src/ipa/rpi/controller/sync_status.h\n@@ -0,0 +1,27 @@\n+/* SPDX-License-Identifier: BSD-2-Clause */\n+/*\n+ * Copyright (C) 2024, Raspberry Pi Ltd\n+ *\n+ * sync_status.h - Sync algorithm params and status structures\n+ */\n+#pragma once\n+\n+#include <libcamera/base/utils.h>\n+\n+struct SyncParams {\n+\t/* Wall clock time for this frame */\n+\tuint64_t wallClock;\n+\t/* Kernel timestamp for this frame */\n+\tuint64_t sensorTimestamp;\n+};\n+\n+struct SyncStatus {\n+\t/* Frame length correction to apply */\n+\tlibcamera::utils::Duration frameDurationOffset;\n+\t/* Whether the \"ready time\" has been reached */\n+\tbool ready;\n+\t/* Lag between camera frame and the \"ready time\" */\n+\tint64_t lag;\n+\t/* Whether lag is known (client has to wait for a server message) */\n+\tbool lagKnown;\n+};\n", "prefixes": [ "4/6" ] }