[{"id":33089,"web_url":"https://patchwork.libcamera.org/comment/33089/","msgid":"","date":"2025-01-17T09:15:13","subject":"Re: [PATCH v3 5/7] ipa: rpi: Add base classes and plumbing for sync\n\talgorithm","submitter":{"id":34,"url":"https://patchwork.libcamera.org/api/people/34/","name":"Naushir Patuck","email":"naush@raspberrypi.com"},"content":"Hi David,\n\nOn Thu, 9 Jan 2025 at 14:32, David Plowman\n wrote:\n>\n> From: Naushir Patuck \n>\n> We add a base class for a \"sync algorithm\", and define its inputs and\n> outputs in the SyncStatus class.\n>\n> We add the necessary plumbing to the base IPA code so as to arrange\n> for the necessary parameters to be made available to such an\n> algorithm, and also to handle the return values, passing them back as\n> necessary to the pipeline handler.\n>\n> Signed-off-by: Naushir Patuck \n\nYou probably want your SOB tag as well :)\n\nReviewed-by: Naushir Patuck \n\n> ---\n> src/ipa/rpi/common/ipa_base.cpp | 78 +++++++++++++++++++++++--\n> src/ipa/rpi/common/ipa_base.h | 4 +-\n> src/ipa/rpi/controller/sync_algorithm.h | 31 ++++++++++\n> src/ipa/rpi/controller/sync_status.h | 27 +++++++++\n> 4 files changed, 135 insertions(+), 5 deletions(-)\n> create mode 100644 src/ipa/rpi/controller/sync_algorithm.h\n> create mode 100644 src/ipa/rpi/controller/sync_status.h\n>\n> diff --git a/src/ipa/rpi/common/ipa_base.cpp b/src/ipa/rpi/common/ipa_base.cpp\n> index 6ff1e22b..f1e4a161 100644\n> --- a/src/ipa/rpi/common/ipa_base.cpp\n> +++ b/src/ipa/rpi/common/ipa_base.cpp\n> @@ -28,6 +28,8 @@\n> #include \"controller/lux_status.h\"\n> #include \"controller/sharpen_algorithm.h\"\n> #include \"controller/statistics.h\"\n> +#include \"controller/sync_algorithm.h\"\n> +#include \"controller/sync_status.h\"\n>\n> namespace libcamera {\n>\n> @@ -72,6 +74,8 @@ const ControlInfoMap::Map ipaControls{\n> { &controls::Sharpness, ControlInfo(0.0f, 16.0f, 1.0f) },\n> { &controls::ScalerCrop, ControlInfo(Rectangle{}, Rectangle(65535, 65535, 65535, 65535), Rectangle{}) },\n> { &controls::FrameDurationLimits, ControlInfo(INT64_C(33333), INT64_C(120000)) },\n> + { &controls::rpi::SyncMode, ControlInfo(controls::rpi::SyncModeValues) },\n> + { &controls::rpi::SyncFrames, ControlInfo(1, 1000000, 100) },\n> { &controls::draft::NoiseReductionMode, ControlInfo(controls::draft::NoiseReductionModeValues) },\n> { &controls::rpi::StatsOutputEnable, ControlInfo(false, true, false) },\n> };\n> @@ -390,6 +394,7 @@ void IpaBase::prepareIsp(const PrepareParams ¶ms)\n>\n> rpiMetadata.clear();\n> fillDeviceStatus(params.sensorControls, ipaContext);\n> + fillSyncParams(params, ipaContext);\n>\n> if (params.buffers.embedded) {\n> /*\n> @@ -488,10 +493,23 @@ void IpaBase::processStats(const ProcessParams ¶ms)\n> helper_->process(statistics, rpiMetadata);\n> controller_.process(statistics, &rpiMetadata);\n>\n> + /* Send any sync algorithm outputs back to the pipeline handler */\n> + Duration offset(0s);\n> + struct SyncStatus syncStatus;\n> + if (rpiMetadata.get(\"sync.status\", syncStatus) == 0) {\n> + if (minFrameDuration_ != maxFrameDuration_)\n> + LOG(IPARPI, Error) << \"Sync algorithm enabled with variable framerate. \" << minFrameDuration_ << \" \" << maxFrameDuration_;\n\nMight need to wrap this line?\n\n> + offset = syncStatus.frameDurationOffset;\n> +\n> + libcameraMetadata_.set(controls::rpi::SyncReady, syncStatus.ready);\n> + if (syncStatus.timerKnown)\n> + libcameraMetadata_.set(controls::rpi::SyncTimer, syncStatus.timerValue);\n> + }\n> +\n> struct AgcStatus agcStatus;\n> if (rpiMetadata.get(\"agc.status\", agcStatus) == 0) {\n> ControlList ctrls(sensorCtrls_);\n> - applyAGC(&agcStatus, ctrls);\n> + applyAGC(&agcStatus, ctrls, offset);\n> setDelayedControls.emit(ctrls, ipaContext);\n> setCameraTimeoutValue();\n> }\n> @@ -728,6 +746,7 @@ void IpaBase::applyControls(const ControlList &controls)\n> using RPiController::ContrastAlgorithm;\n> using RPiController::DenoiseAlgorithm;\n> using RPiController::HdrAlgorithm;\n> + using RPiController::SyncAlgorithm;\n>\n> /* Clear the return metadata buffer. */\n> libcameraMetadata_.clear();\n> @@ -1274,6 +1293,35 @@ void IpaBase::applyControls(const ControlList &controls)\n> statsMetadataOutput_ = ctrl.second.get();\n> break;\n>\n> + case controls::rpi::SYNC_MODE: {\n> + SyncAlgorithm *sync = dynamic_cast(controller_.getAlgorithm(\"sync\"));\n> +\n> + if (sync) {\n> + int mode = ctrl.second.get();\n> + SyncAlgorithm::Mode m = SyncAlgorithm::Mode::Off;\n> + if (mode == controls::rpi::SyncModeServer) {\n> + m = SyncAlgorithm::Mode::Server;\n> + LOG(IPARPI, Info) << \"Sync mode set to server\";\n> + } else if (mode == controls::rpi::SyncModeClient) {\n> + m = SyncAlgorithm::Mode::Client;\n> + LOG(IPARPI, Info) << \"Sync mode set to client\";\n> + }\n> + sync->setMode(m);\n> + }\n> + break;\n> + }\n> +\n> + case controls::rpi::SYNC_FRAMES: {\n> + SyncAlgorithm *sync = dynamic_cast(controller_.getAlgorithm(\"sync\"));\n> +\n> + if (sync) {\n> + int frames = ctrl.second.get();\n> + if (frames > 0)\n> + sync->setReadyFrame(frames);\n> + }\n> + break;\n> + }\n> +\n> default:\n> LOG(IPARPI, Warning)\n> << \"Ctrl \" << controls::controls.at(ctrl.first)->name()\n> @@ -1310,6 +1358,19 @@ void IpaBase::fillDeviceStatus(const ControlList &sensorControls, unsigned int i\n> rpiMetadata_[ipaContext].set(\"device.status\", deviceStatus);\n> }\n>\n> +void IpaBase::fillSyncParams(const PrepareParams ¶ms, unsigned int ipaContext)\n> +{\n> + RPiController::SyncAlgorithm *sync = dynamic_cast(\n> + controller_.getAlgorithm(\"sync\"));\n> + if (!sync)\n> + return;\n> +\n> + SyncParams syncParams;\n> + syncParams.wallClock = *params.sensorControls.get(controls::FrameWallClock);\n> + syncParams.sensorTimestamp = *params.sensorControls.get(controls::SensorTimestamp);\n> + rpiMetadata_[ipaContext].set(\"sync.params\", syncParams);\n> +}\n> +\n> void IpaBase::reportMetadata(unsigned int ipaContext)\n> {\n> RPiController::Metadata &rpiMetadata = rpiMetadata_[ipaContext];\n> @@ -1478,14 +1539,22 @@ void IpaBase::applyFrameDurations(Duration minFrameDuration, Duration maxFrameDu\n> * value possible.\n> */\n> Duration maxExposureTime = Duration::max();\n> - helper_->getBlanking(maxExposureTime, minFrameDuration_, maxFrameDuration_);\n> + auto [vblank, hblank] = helper_->getBlanking(maxExposureTime, minFrameDuration_, maxFrameDuration_);\n>\n> RPiController::AgcAlgorithm *agc = dynamic_cast(\n> controller_.getAlgorithm(\"agc\"));\n> agc->setMaxExposureTime(maxExposureTime);\n> +\n> + RPiController::SyncAlgorithm *sync = dynamic_cast(\n> + controller_.getAlgorithm(\"sync\"));\n> + if (sync) {\n> + Duration duration = (mode_.height + vblank) * ((mode_.width + hblank) * 1.0s / mode_.pixelRate);\n> + LOG(IPARPI, Debug) << \"setting sync frame duration to \" << duration;\n> + sync->setFrameDuration(duration);\n> + }\n> }\n>\n> -void IpaBase::applyAGC(const struct AgcStatus *agcStatus, ControlList &ctrls)\n> +void IpaBase::applyAGC(const struct AgcStatus *agcStatus, ControlList &ctrls, Duration frameDurationOffset)\n> {\n> const int32_t minGainCode = helper_->gainCode(mode_.minAnalogueGain);\n> const int32_t maxGainCode = helper_->gainCode(mode_.maxAnalogueGain);\n> @@ -1500,7 +1569,8 @@ void IpaBase::applyAGC(const struct AgcStatus *agcStatus, ControlList &ctrls)\n>\n> /* getBlanking might clip exposure time to the fps limits. */\n> Duration exposure = agcStatus->exposureTime;\n> - auto [vblank, hblank] = helper_->getBlanking(exposure, minFrameDuration_, maxFrameDuration_);\n> + auto [vblank, hblank] = helper_->getBlanking(exposure, minFrameDuration_ - frameDurationOffset,\n> + maxFrameDuration_ - frameDurationOffset);\n> int32_t exposureLines = helper_->exposureLines(exposure,\n> helper_->hblankToLineLength(hblank));\n>\n> diff --git a/src/ipa/rpi/common/ipa_base.h b/src/ipa/rpi/common/ipa_base.h\n> index 1a811beb..b53d0bfb 100644\n> --- a/src/ipa/rpi/common/ipa_base.h\n> +++ b/src/ipa/rpi/common/ipa_base.h\n> @@ -95,9 +95,11 @@ private:\n> void applyControls(const ControlList &controls);\n> virtual void handleControls(const ControlList &controls) = 0;\n> void fillDeviceStatus(const ControlList &sensorControls, unsigned int ipaContext);\n> + void fillSyncParams(const PrepareParams ¶ms, unsigned int ipaContext);\n> void reportMetadata(unsigned int ipaContext);\n> void applyFrameDurations(utils::Duration minFrameDuration, utils::Duration maxFrameDuration);\n> - void applyAGC(const struct AgcStatus *agcStatus, ControlList &ctrls);\n> + void applyAGC(const struct AgcStatus *agcStatus, ControlList &ctrls,\n> + utils::Duration frameDurationOffset = utils::Duration(0));\n>\n> std::map buffers_;\n>\n> diff --git a/src/ipa/rpi/controller/sync_algorithm.h b/src/ipa/rpi/controller/sync_algorithm.h\n> new file mode 100644\n> index 00000000..c242def6\n> --- /dev/null\n> +++ b/src/ipa/rpi/controller/sync_algorithm.h\n> @@ -0,0 +1,31 @@\n> +/* SPDX-License-Identifier: BSD-2-Clause */\n> +/*\n> + * Copyright (C) 2024, Raspberry Pi Ltd\n> + *\n> + * sync_algorithm.h - Camera sync algorithm interface\n> + */\n> +#pragma once\n> +\n> +#include \n> +\n> +#include \"algorithm.h\"\n> +\n> +namespace RPiController {\n> +\n> +class SyncAlgorithm : public Algorithm\n> +{\n> +public:\n> + enum class Mode {\n> + Off,\n> + Server,\n> + Client,\n> + };\n> +\n> + SyncAlgorithm(Controller *controller)\n> + : Algorithm(controller) {}\n> + virtual void setFrameDuration(libcamera::utils::Duration frameDuration) = 0;\n> + virtual void setReadyFrame(unsigned int frame) = 0;\n> + virtual void setMode(Mode mode) = 0;\n> +};\n> +\n> +} /* namespace RPiController */\n> diff --git a/src/ipa/rpi/controller/sync_status.h b/src/ipa/rpi/controller/sync_status.h\n> new file mode 100644\n> index 00000000..10f97502\n> --- /dev/null\n> +++ b/src/ipa/rpi/controller/sync_status.h\n> @@ -0,0 +1,27 @@\n> +/* SPDX-License-Identifier: BSD-2-Clause */\n> +/*\n> + * Copyright (C) 2024, Raspberry Pi Ltd\n> + *\n> + * sync_status.h - Sync algorithm params and status structures\n> + */\n> +#pragma once\n> +\n> +#include \n> +\n> +struct SyncParams {\n> + /* Wall clock time for this frame */\n> + uint64_t wallClock;\n> + /* Kernel timestamp for this frame */\n> + uint64_t sensorTimestamp;\n> +};\n> +\n> +struct SyncStatus {\n> + /* Frame length correction to apply */\n> + libcamera::utils::Duration frameDurationOffset;\n> + /* Whether the \"ready time\" has been reached */\n> + bool ready;\n> + /* Time until the \"ready time\" */\n> + int64_t timerValue;\n> + /* Whether timerValue is known (client has to wait for a server message) */\n> + bool timerKnown;\n> +};\n> --\n> 2.39.5\n>","headers":{"Return-Path":"","X-Original-To":"parsemail@patchwork.libcamera.org","Delivered-To":"parsemail@patchwork.libcamera.org","Received":["from lancelot.ideasonboard.com (lancelot.ideasonboard.com\n\t[92.243.16.209])\n\tby patchwork.libcamera.org (Postfix) with ESMTPS id A14ABC3273\n\tfor ;\n\tFri, 17 Jan 2025 09:15:51 +0000 (UTC)","from lancelot.ideasonboard.com (localhost [IPv6:::1])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTP id C803168543;\n\tFri, 17 Jan 2025 10:15:50 +0100 (CET)","from mail-yb1-xb33.google.com (mail-yb1-xb33.google.com\n\t[IPv6:2607:f8b0:4864:20::b33])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id C24E468503\n\tfor ;\n\tFri, 17 Jan 2025 10:15:49 +0100 (CET)","by mail-yb1-xb33.google.com with SMTP id\n\t3f1490d57ef6-e545f6750abso365842276.2\n\tfor ;\n\tFri, 17 Jan 2025 01:15:49 -0800 (PST)"],"Authentication-Results":"lancelot.ideasonboard.com; dkim=pass (2048-bit key;\n\tunprotected) header.d=raspberrypi.com header.i=@raspberrypi.com\n\theader.b=\"eOTW1UCT\"; dkim-atps=neutral","DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/relaxed;\n\td=raspberrypi.com; s=google; t=1737105349; x=1737710149;\n\tdarn=lists.libcamera.org; \n\th=cc:to:subject:message-id:date:from:in-reply-to:references\n\t:mime-version:from:to:cc:subject:date:message-id:reply-to;\n\tbh=ckinl5e4NpocRz0JPVA7eNyi3FVhNKUP6HNjwLFe3MI=;\n\tb=eOTW1UCTWcBm4ruRaM6k2uJ1UsL7GX1VfcGiGBo5JkMPHbnU+fLMkE0GBpCUH9aEHN\n\tPcx1oijpN6sRVaMflQRfJ0hfIwQgiCOv3f0tRBa0wAGPM7aleqbfxjyDrcAaBrX/cGdM\n\tym0q2NQIKZp34QofZ+3+l2QfHxb2MJ3gLVSvFS8VPgAZ/2BJKWLlSifcX3gORkd4qF1u\n\t7XxkpbES1uL9l9bNVKYHGoWONsFToa+QsuM5Ak6MNTsUrOJZeuWBJ6XvzXbvvQF/I+4f\n\t2B6+9qAfthxc8iMmgsxnZgUbOYMYzoErS2zOzSRjOMhvIxzllhUyEiuTXQ+g1B2OMFBR\n\toQnw==","X-Google-DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/relaxed;\n\td=1e100.net; s=20230601; t=1737105349; x=1737710149;\n\th=cc:to:subject:message-id:date:from:in-reply-to:references\n\t:mime-version:x-gm-message-state:from:to:cc:subject:date:message-id\n\t:reply-to;\n\tbh=ckinl5e4NpocRz0JPVA7eNyi3FVhNKUP6HNjwLFe3MI=;\n\tb=cZ3ALJOup8u26EpMLpOplqmkUalhmWkY937lA6Ugj7b4OuYsS4RdlWjNhwD22zhUA0\n\tt6JE9gozwrGp+9/luhvfTHNWMAdV/0YqD+MmJkDc5AtsJlAPRO1ARori529KI2KZfhxE\n\tIWLkK2JCedFtY984U5OGi6wCYpOYyqlI0Rta20w/QzOR8NybpJTjw2//VvBZEfxMq6wZ\n\tmXWQS19Yv8l0sCEbUutwKe72nVz2e/GI8uH5jaH/wEgGEiQdCTstAoKSlVHXRYE3u9LI\n\tFQc98FsvmX/bP3tykmEbqAfP/0kd4llzC8vBSB6UsXxEW8iIiXk0yiyvrvtsiS/AL68E\n\t8fUw==","X-Gm-Message-State":"AOJu0YyCKneTikfQqyB0C9IijidTokAESTpI7oc/c8bOW2Z3Ra6KT+++\n\tOKl7ab6ivG5oJ+4hpnSp7xxCFdM1xiBnyvZx+eGXfAFBLG/+eNRFTpjVFWzFV2to3lkaNT81yK4\n\tG5CqSYX0VUCu7tuK9QxaZgXiz9oD6tX5/M7Ys+J04J/+QkfdY4LT89g==","X-Gm-Gg":"ASbGncu+Y6q2EgdCbPWWwNsvZnMbX8fiPzs0mBsVUBto3xWXu/dUOZ47dTRn3QJULLn\n\t+aDiE4rxyBxlmNYdfTAsw7eDGOl1cugrDeImaSg6LSHwSbhE7z0DVEOvQLBS4oBXRS/D/QA==","X-Google-Smtp-Source":"AGHT+IGaCklOIfNVIHVXShpRaBr/8EIfkxlnFm9g+mdPRnovc5vGXXZdkcW5ZZ/KgwByZq+fQxVcCPY8yCePfJEWfik=","X-Received":"by 2002:a05:6902:220b:b0:e4c:66ab:ed41 with SMTP id\n\t3f1490d57ef6-e57b102f563mr561870276.2.1737105348619; Fri, 17 Jan 2025\n\t01:15:48 -0800 (PST)","MIME-Version":"1.0","References":"<20250109143211.11939-1-david.plowman@raspberrypi.com>\n\t<20250109143211.11939-6-david.plowman@raspberrypi.com>","In-Reply-To":"<20250109143211.11939-6-david.plowman@raspberrypi.com>","From":"Naushir Patuck ","Date":"Fri, 17 Jan 2025 09:15:13 +0000","X-Gm-Features":"AbW1kvap9KP76K3Nci7dZsbvtHKvysS4NJfJkMaTQUjhpzd8AICbwUh3_xiGoao","Message-ID":"","Subject":"Re: [PATCH v3 5/7] ipa: rpi: Add base classes and plumbing for sync\n\talgorithm","To":"David Plowman ","Cc":"libcamera-devel@lists.libcamera.org","Content-Type":"text/plain; charset=\"UTF-8\"","X-BeenThere":"libcamera-devel@lists.libcamera.org","X-Mailman-Version":"2.1.29","Precedence":"list","List-Id":"","List-Unsubscribe":",\n\t","List-Archive":"","List-Post":"","List-Help":"","List-Subscribe":",\n\t","Errors-To":"libcamera-devel-bounces@lists.libcamera.org","Sender":"\"libcamera-devel\" "}}]