From patchwork Fri Oct 4 11:55:56 2024 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: David Plowman X-Patchwork-Id: 21515 Return-Path: X-Original-To: parsemail@patchwork.libcamera.org Delivered-To: parsemail@patchwork.libcamera.org Received: from lancelot.ideasonboard.com (lancelot.ideasonboard.com [92.243.16.209]) by patchwork.libcamera.org (Postfix) with ESMTPS id 7AC8AC3263 for ; Fri, 4 Oct 2024 11:56:19 +0000 (UTC) Received: from lancelot.ideasonboard.com (localhost [IPv6:::1]) by lancelot.ideasonboard.com (Postfix) with ESMTP id A82FB6352E; Fri, 4 Oct 2024 13:56:18 +0200 (CEST) Authentication-Results: lancelot.ideasonboard.com; dkim=pass (2048-bit key; unprotected) header.d=raspberrypi.com header.i=@raspberrypi.com header.b="LmtZAUU+"; dkim-atps=neutral Received: from mail-wr1-x432.google.com (mail-wr1-x432.google.com [IPv6:2a00:1450:4864:20::432]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id A90FC62C8F for ; Fri, 4 Oct 2024 13:56:05 +0200 (CEST) Received: by mail-wr1-x432.google.com with SMTP id ffacd0b85a97d-37cc60c9838so1208580f8f.1 for ; Fri, 04 Oct 2024 04:56:05 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=raspberrypi.com; s=google; t=1728042965; x=1728647765; darn=lists.libcamera.org; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:from:to:cc:subject:date :message-id:reply-to; bh=vTL6DimDDA9r/zGhGutNdHJ4z5oVSpHpu6UwaAbb2SM=; b=LmtZAUU+E8MQmocUmAklarujj0MEbG0P0pc8r7ULItYQ/0JC5R5TtYJhH2EsmiJ1xu neeN2iXnMeDIbQFGM1urv1OKYoO+oGUmPbpk3zdiCYMsoBTEQ2uOLzdWQNkrnrRrjT42 gJiYe7tty8x/m+twbH88lOwLD1NUNU7PM8UQLq2qol9057AnNJxZ8FqTIRkiQYv7qGyh YQGqUVXc2t0Q97cBMYPVpkF84n+ZsmZH2Fv6UTwy/gpXJCZeeynI/Zt8tRH+WHqytfb8 qFvJPLcD+add6aKwFq0nVCGcRHx1B0esgsPGPQBl15RLzH2tVqgNqS1HySwih2CKznoh Hx0w== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20230601; t=1728042965; x=1728647765; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:x-gm-message-state:from:to:cc :subject:date:message-id:reply-to; bh=vTL6DimDDA9r/zGhGutNdHJ4z5oVSpHpu6UwaAbb2SM=; b=nln00oqug7Cjp+MsNuf9jB0ptJmCrpAVnH6nKEB8BaOuKXPR4A0bSx34FH6Mknxmuv atgv3ET/ihOaHnZnZ/w1t2fqZaX8oPioWCoJkMrksV+pO2EwrpfzMHsZi3Cbaysgwj+K +6drmJ5E+7K6g3l4hITDMsD2Lbk2Q58mNVW8hlONlIZFfZbK/id/D6CCQcZDv9yH8mFl qJNhya3VZTLw494jUzqTVT7hBmApyz70pfSClzdZ2fRiiyPvNx6rMniOjEnZV+p//tWI vTLYTZ8/UtixWVf3+co/ngbQEgoa8QH+lkLt3bm4PZ0C4+AQed+Ro+WvmEb3TQmLii9L n1tg== X-Gm-Message-State: AOJu0Yxrw0cbTbwkm7DWkz3XAibCW0VDN3vpoXN7f2pwmYByYJPj9zuQ jSCSnlKz7a1JnQS891k/KT90AS5tYILd5NbG1k7qFtokLEN34Cga7JToD5rKzmgu952eoZVPKx0 B X-Google-Smtp-Source: AGHT+IGWtR7pK77SJNmhk/qhH4mOhADTbEAGgq3yA8ofYsOo/aumQKRwPNN3g1WWUZDhCDe3t9ivaA== X-Received: by 2002:a05:6000:1569:b0:374:c6b6:c656 with SMTP id ffacd0b85a97d-37d0f70ca1emr1793382f8f.21.1728042964666; Fri, 04 Oct 2024 04:56:04 -0700 (PDT) Received: from raspberrypi.pitowers.org ([2a00:1098:3142:1f:daa2:371b:a97:3e3e]) by smtp.gmail.com with ESMTPSA id ffacd0b85a97d-37d081f743esm3107147f8f.21.2024.10.04.04.56.04 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Fri, 04 Oct 2024 04:56:04 -0700 (PDT) From: David Plowman To: libcamera-devel@lists.libcamera.org Cc: Naushir Patuck , David Plowman Subject: [PATCH 4/6] ipa: rpi: Add base classes and plumbing for sync algorithm Date: Fri, 4 Oct 2024 12:55:56 +0100 Message-Id: <20241004115558.9166-5-david.plowman@raspberrypi.com> X-Mailer: git-send-email 2.39.5 In-Reply-To: <20241004115558.9166-1-david.plowman@raspberrypi.com> References: <20241004115558.9166-1-david.plowman@raspberrypi.com> MIME-Version: 1.0 X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: libcamera-devel-bounces@lists.libcamera.org Sender: "libcamera-devel" From: Naushir Patuck We add a base class for a "sync algorithm", and define its inputs and outputs in the SyncStatus class. We add the necessary plumbing to the base IPA code so as to arrange for the necessary parameters to be made available to such an algorithm, and also to handle the return values, passing them back as necessary to the pipeline handler. Signed-off-by: Naushir Patuck Reviewed-by: David Plowman --- src/ipa/rpi/common/ipa_base.cpp | 66 +++++++++++++++++++++++-- src/ipa/rpi/common/ipa_base.h | 4 +- src/ipa/rpi/controller/sync_algorithm.h | 30 +++++++++++ src/ipa/rpi/controller/sync_status.h | 27 ++++++++++ 4 files changed, 122 insertions(+), 5 deletions(-) create mode 100644 src/ipa/rpi/controller/sync_algorithm.h create mode 100644 src/ipa/rpi/controller/sync_status.h diff --git a/src/ipa/rpi/common/ipa_base.cpp b/src/ipa/rpi/common/ipa_base.cpp index ee3848b5..ac228791 100644 --- a/src/ipa/rpi/common/ipa_base.cpp +++ b/src/ipa/rpi/common/ipa_base.cpp @@ -28,6 +28,8 @@ #include "controller/lux_status.h" #include "controller/sharpen_algorithm.h" #include "controller/statistics.h" +#include "controller/sync_algorithm.h" +#include "controller/sync_status.h" namespace libcamera { @@ -74,6 +76,7 @@ const ControlInfoMap::Map ipaControls{ { &controls::FrameDurationLimits, ControlInfo(INT64_C(33333), INT64_C(120000)) }, { &controls::draft::NoiseReductionMode, ControlInfo(controls::draft::NoiseReductionModeValues) }, { &controls::rpi::StatsOutputEnable, ControlInfo(false, true, false) }, + { &controls::rpi::SyncMode, ControlInfo(controls::rpi::SyncModeValues) }, }; /* IPA controls handled conditionally, if the sensor is not mono */ @@ -387,6 +390,7 @@ void IpaBase::prepareIsp(const PrepareParams ¶ms) rpiMetadata.clear(); fillDeviceStatus(params.sensorControls, ipaContext); + fillSyncParams(params, ipaContext); if (params.buffers.embedded) { /* @@ -485,10 +489,23 @@ void IpaBase::processStats(const ProcessParams ¶ms) helper_->process(statistics, rpiMetadata); controller_.process(statistics, &rpiMetadata); + /* Send any sync algorithm outputs back to the pipeline handler */ + Duration offset(0s); + struct SyncStatus syncStatus; + if (rpiMetadata.get("sync.status", syncStatus) == 0) { + if (minFrameDuration_ != maxFrameDuration_) + LOG(IPARPI, Error) << "Sync algorithm enabled with variable framerate. " << minFrameDuration_ << " " << maxFrameDuration_; + offset = syncStatus.frameDurationOffset; + + libcameraMetadata_.set(controls::rpi::SyncWait, !syncStatus.ready); + if (syncStatus.lagKnown) + libcameraMetadata_.set(controls::rpi::SyncLag, syncStatus.lag); + } + struct AgcStatus agcStatus; if (rpiMetadata.get("agc.status", agcStatus) == 0) { ControlList ctrls(sensorCtrls_); - applyAGC(&agcStatus, ctrls); + applyAGC(&agcStatus, ctrls, offset); setDelayedControls.emit(ctrls, ipaContext); setCameraTimeoutValue(); } @@ -721,6 +738,7 @@ void IpaBase::applyControls(const ControlList &controls) using RPiController::ContrastAlgorithm; using RPiController::DenoiseAlgorithm; using RPiController::HdrAlgorithm; + using RPiController::SyncAlgorithm; /* Clear the return metadata buffer. */ libcameraMetadata_.clear(); @@ -1247,6 +1265,24 @@ void IpaBase::applyControls(const ControlList &controls) statsMetadataOutput_ = ctrl.second.get(); break; + case controls::rpi::SYNC_MODE: { + SyncAlgorithm *sync = dynamic_cast(controller_.getAlgorithm("sync")); + + if (sync) { + int mode = ctrl.second.get(); + SyncAlgorithm::Mode m = SyncAlgorithm::Mode::Off; + if (mode == controls::rpi::SyncModeServer) { + m = SyncAlgorithm::Mode::Server; + LOG(IPARPI, Info) << "Sync mode set to server"; + } else if (mode == controls::rpi::SyncModeClient) { + m = SyncAlgorithm::Mode::Client; + LOG(IPARPI, Info) << "Sync mode set to client"; + } + sync->setMode(m); + } + break; + } + default: LOG(IPARPI, Warning) << "Ctrl " << controls::controls.at(ctrl.first)->name() @@ -1283,6 +1319,19 @@ void IpaBase::fillDeviceStatus(const ControlList &sensorControls, unsigned int i rpiMetadata_[ipaContext].set("device.status", deviceStatus); } +void IpaBase::fillSyncParams(const PrepareParams ¶ms, unsigned int ipaContext) +{ + RPiController::SyncAlgorithm *sync = dynamic_cast( + controller_.getAlgorithm("sync")); + if (!sync) + return; + + SyncParams syncParams; + syncParams.wallClock = *params.sensorControls.get(controls::rpi::FrameWallClock); + syncParams.sensorTimestamp = *params.sensorControls.get(controls::SensorTimestamp); + rpiMetadata_[ipaContext].set("sync.params", syncParams); +} + void IpaBase::reportMetadata(unsigned int ipaContext) { RPiController::Metadata &rpiMetadata = rpiMetadata_[ipaContext]; @@ -1451,14 +1500,22 @@ void IpaBase::applyFrameDurations(Duration minFrameDuration, Duration maxFrameDu * value possible. */ Duration maxShutter = Duration::max(); - helper_->getBlanking(maxShutter, minFrameDuration_, maxFrameDuration_); + auto [vblank, hblank] = helper_->getBlanking(maxShutter, minFrameDuration_, maxFrameDuration_); RPiController::AgcAlgorithm *agc = dynamic_cast( controller_.getAlgorithm("agc")); agc->setMaxShutter(maxShutter); + + RPiController::SyncAlgorithm *sync = dynamic_cast( + controller_.getAlgorithm("sync")); + if (sync) { + Duration duration = (mode_.height + vblank) * ((mode_.width + hblank) * 1.0s / mode_.pixelRate); + LOG(IPARPI, Debug) << "setting sync frame duration to " << duration; + sync->setFrameDuration(duration); + } } -void IpaBase::applyAGC(const struct AgcStatus *agcStatus, ControlList &ctrls) +void IpaBase::applyAGC(const struct AgcStatus *agcStatus, ControlList &ctrls, Duration frameDurationOffset) { const int32_t minGainCode = helper_->gainCode(mode_.minAnalogueGain); const int32_t maxGainCode = helper_->gainCode(mode_.maxAnalogueGain); @@ -1473,7 +1530,8 @@ void IpaBase::applyAGC(const struct AgcStatus *agcStatus, ControlList &ctrls) /* getBlanking might clip exposure time to the fps limits. */ Duration exposure = agcStatus->shutterTime; - auto [vblank, hblank] = helper_->getBlanking(exposure, minFrameDuration_, maxFrameDuration_); + auto [vblank, hblank] = helper_->getBlanking(exposure, minFrameDuration_ - frameDurationOffset, + maxFrameDuration_ - frameDurationOffset); int32_t exposureLines = helper_->exposureLines(exposure, helper_->hblankToLineLength(hblank)); diff --git a/src/ipa/rpi/common/ipa_base.h b/src/ipa/rpi/common/ipa_base.h index 1a811beb..b53d0bfb 100644 --- a/src/ipa/rpi/common/ipa_base.h +++ b/src/ipa/rpi/common/ipa_base.h @@ -95,9 +95,11 @@ private: void applyControls(const ControlList &controls); virtual void handleControls(const ControlList &controls) = 0; void fillDeviceStatus(const ControlList &sensorControls, unsigned int ipaContext); + void fillSyncParams(const PrepareParams ¶ms, unsigned int ipaContext); void reportMetadata(unsigned int ipaContext); void applyFrameDurations(utils::Duration minFrameDuration, utils::Duration maxFrameDuration); - void applyAGC(const struct AgcStatus *agcStatus, ControlList &ctrls); + void applyAGC(const struct AgcStatus *agcStatus, ControlList &ctrls, + utils::Duration frameDurationOffset = utils::Duration(0)); std::map buffers_; diff --git a/src/ipa/rpi/controller/sync_algorithm.h b/src/ipa/rpi/controller/sync_algorithm.h new file mode 100644 index 00000000..298b5951 --- /dev/null +++ b/src/ipa/rpi/controller/sync_algorithm.h @@ -0,0 +1,30 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (C) 2024, Raspberry Pi Ltd + * + * sync_algorithm.h - Camera sync algorithm interface + */ +#pragma once + +#include + +#include "algorithm.h" + +namespace RPiController { + +class SyncAlgorithm : public Algorithm +{ +public: + enum class Mode { + Off, + Server, + Client, + }; + + SyncAlgorithm(Controller *controller) + : Algorithm(controller) {} + virtual void setFrameDuration(libcamera::utils::Duration frameDuration) = 0; + virtual void setMode(Mode mode) = 0; +}; + +} /* namespace RPiController */ diff --git a/src/ipa/rpi/controller/sync_status.h b/src/ipa/rpi/controller/sync_status.h new file mode 100644 index 00000000..289f0182 --- /dev/null +++ b/src/ipa/rpi/controller/sync_status.h @@ -0,0 +1,27 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (C) 2024, Raspberry Pi Ltd + * + * sync_status.h - Sync algorithm params and status structures + */ +#pragma once + +#include + +struct SyncParams { + /* Wall clock time for this frame */ + uint64_t wallClock; + /* Kernel timestamp for this frame */ + uint64_t sensorTimestamp; +}; + +struct SyncStatus { + /* Frame length correction to apply */ + libcamera::utils::Duration frameDurationOffset; + /* Whether the "ready time" has been reached */ + bool ready; + /* Lag between camera frame and the "ready time" */ + int64_t lag; + /* Whether lag is known (client has to wait for a server message) */ + bool lagKnown; +};