From patchwork Fri Dec 13 09:38:28 2024 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Naushir Patuck X-Patchwork-Id: 22310 Return-Path: X-Original-To: parsemail@patchwork.libcamera.org Delivered-To: parsemail@patchwork.libcamera.org Received: from lancelot.ideasonboard.com (lancelot.ideasonboard.com [92.243.16.209]) by patchwork.libcamera.org (Postfix) with ESMTPS id 35B99C32F2 for ; Fri, 13 Dec 2024 09:46:20 +0000 (UTC) Received: from lancelot.ideasonboard.com (localhost [IPv6:::1]) by lancelot.ideasonboard.com (Postfix) with ESMTP id C40B367EFB; Fri, 13 Dec 2024 10:46:18 +0100 (CET) Authentication-Results: lancelot.ideasonboard.com; dkim=pass (2048-bit key; unprotected) header.d=raspberrypi.com header.i=@raspberrypi.com header.b="dN5/vvac"; dkim-atps=neutral Received: from mail-wr1-x42d.google.com (mail-wr1-x42d.google.com [IPv6:2a00:1450:4864:20::42d]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id CCDFE67EE8 for ; Fri, 13 Dec 2024 10:46:11 +0100 (CET) Received: by mail-wr1-x42d.google.com with SMTP id ffacd0b85a97d-3862f11a13dso139015f8f.2 for ; Fri, 13 Dec 2024 01:46:11 -0800 (PST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=raspberrypi.com; s=google; t=1734083171; x=1734687971; darn=lists.libcamera.org; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:from:to:cc:subject:date :message-id:reply-to; bh=ABhzmtZRvpmXxSShgjDOKB8Yq6zpZHSD4zqVCCPH2wI=; b=dN5/vvac5Vqv5/uLoA9LpQfz2AnieWcuVSvI9Q3hMUpiQ8KWlfn4+eHQ8y1Si79UR9 v2Traq3DHDklur9CQCk21ey6IvctE//srmdJBcSL7k7NifsZSt60zaq2+xW949pLQEkx fwvIQRIdC5zeH4ebHxX6Q2YZ1fXYUcXRe1OCL60YNZAwRP30sVN/dkNfzXyjXIEhvq4t 2rh0xO6j0gUP0Az8Z+e5skizhtG9Uc9WjNSPNMwePGh6WdK53O2Mox5zbyqNxqA0WVr5 Jss9VMrLGx6HalZhU92VrRUGxQq2denrmNc9KMnmy1UEP910cqQCclgJRZLEg5Qh6OCA DJCw== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20230601; t=1734083171; x=1734687971; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:x-gm-message-state:from:to:cc :subject:date:message-id:reply-to; bh=ABhzmtZRvpmXxSShgjDOKB8Yq6zpZHSD4zqVCCPH2wI=; b=sqU/TjTVOZ0lvANeRLbHFdf7hswjlcVNABI06WJmnnezUtN9VCQEqxZGnR3WoNcvt/ 6H9i8zXntJzq9y4zn/8Yd6HVssjmnO3oNLGWf9KgaNqdOjWET5+4657YaVRiSZb48l0D 6/6wWozbCI+y2krnuSyFzMTlNS+b0W6CJqhRafpRoXrJOx8XQRiRFwGc7hmwO1XMMJF6 8cqObh53kAKK7QpFcm5S5deYFJzU6Ed1EhyjhE7d0m50NFChk6KuH9lxYAANMGK8o6j6 ZqyYdUAPpJMObkbrxTPFQ/qX0EVKUErHO85kvpJ8Pd2na59EEwxx07lRQSKu2eye9Wzh dFLQ== X-Gm-Message-State: AOJu0YwaxMdrfgvF7pRo4s4ACSu+/my8wScSfFL6VuvOdeyvl3G+O4qr zCCsDLMScDFggN6M2nM6GzVftXnJcEYuRCSTNfvW6/hjbrDPCuBZjLvMc6t3nUQph9Ez608Q1u0 g X-Gm-Gg: ASbGncvl7xawp3YbjtZ3/X+A1deYjyZ/8CCtyPCTBO9p1Eb/FdRqeLW20+3/Vi3gp3d Pv9YRA9Qu2PS8Vx7Sl6qRs92Clqkow8XpxCuce0f7i2i6BNkg79nBUPNGGbB7XTWftWf4qrH1OP DRYwJTqzbi0NncZcGWVqpNYQa6w8HI/eCoMBMHry2gct9XVBXpQsJewxrcNTVIQ3tfaS4KATZfp /U2/fdpNBEh7BzCFfMBc3g4WqGsKMpDNw0I/X1GsDmNV5FzTkHZun+MsvZB26TkIP90n5PV1XXP X-Google-Smtp-Source: AGHT+IEdjHRI0bMg3gh4pBka3rxvuJympF2JjHNdnTMcxGnEqJxvqbG0pp1zk4qL0pNU6cy2+/V2ow== X-Received: by 2002:a05:600c:3b02:b0:434:fecf:cb2f with SMTP id 5b1f17b1804b1-4362aa8faa0mr5396425e9.5.1734083171063; Fri, 13 Dec 2024 01:46:11 -0800 (PST) Received: from NAUSH-P-DELL.pitowers.org ([93.93.133.154]) by smtp.gmail.com with ESMTPSA id 5b1f17b1804b1-4362557c502sm43989105e9.11.2024.12.13.01.46.10 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Fri, 13 Dec 2024 01:46:10 -0800 (PST) From: Naushir Patuck To: libcamera-devel@lists.libcamera.org Cc: Naushir Patuck Subject: [PATCH 5/6] ipa: rpi: Handle the new CNN controls in the IPA Date: Fri, 13 Dec 2024 09:38:28 +0000 Message-ID: <20241213094602.2083174-6-naush@raspberrypi.com> X-Mailer: git-send-email 2.43.0 In-Reply-To: <20241213094602.2083174-1-naush@raspberrypi.com> References: <20241213094602.2083174-1-naush@raspberrypi.com> MIME-Version: 1.0 X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: libcamera-devel-bounces@lists.libcamera.org Sender: "libcamera-devel" Add code to handle the new CNN vendor controls in the Raspberry Pi IPA. The value of CnnInputTensorInfo is cached as it is the only stateful input control. All other controls are output controls, and the values are copied into directly from the rpiMetadata object if present. The camera helpers populate the rpiMetadata object if the sensor supports on-board CNN processing, such as the IMX500. Signed-off-by: Naushir Patuck Reviewed-by: David Plowman --- src/ipa/rpi/common/ipa_base.cpp | 52 ++++++++++++++++++++++++++++++++- src/ipa/rpi/common/ipa_base.h | 2 ++ 2 files changed, 53 insertions(+), 1 deletion(-) diff --git a/src/ipa/rpi/common/ipa_base.cpp b/src/ipa/rpi/common/ipa_base.cpp index 5fce17e67bd6..b3656cbc730b 100644 --- a/src/ipa/rpi/common/ipa_base.cpp +++ b/src/ipa/rpi/common/ipa_base.cpp @@ -74,6 +74,7 @@ const ControlInfoMap::Map ipaControls{ { &controls::FrameDurationLimits, ControlInfo(INT64_C(33333), INT64_C(120000)) }, { &controls::draft::NoiseReductionMode, ControlInfo(controls::draft::NoiseReductionModeValues) }, { &controls::rpi::StatsOutputEnable, ControlInfo(false, true, false) }, + { &controls::rpi::CnnEnableInputTensor, ControlInfo(false, true, false) }, }; /* IPA controls handled conditionally, if the sensor is not mono */ @@ -112,7 +113,7 @@ namespace ipa::RPi { IpaBase::IpaBase() : controller_(), frameLengths_(FrameLengthsQueueSize, 0s), statsMetadataOutput_(false), stitchSwapBuffers_(false), frameCount_(0), mistrustCount_(0), lastRunTimestamp_(0), - firstStart_(true), flickerState_({ 0, 0s }) + firstStart_(true), flickerState_({ 0, 0s }), cnnEnableInputTensor_(false) { } @@ -1263,6 +1264,10 @@ void IpaBase::applyControls(const ControlList &controls) statsMetadataOutput_ = ctrl.second.get(); break; + case controls::rpi::CNN_ENABLE_INPUT_TENSOR: + cnnEnableInputTensor_ = ctrl.second.get(); + break; + default: LOG(IPARPI, Warning) << "Ctrl " << controls::controls.at(ctrl.first)->name() @@ -1439,6 +1444,51 @@ void IpaBase::reportMetadata(unsigned int ipaContext) libcameraMetadata_.set(controls::HdrChannel, controls::HdrChannelNone); } + const std::shared_ptr *inputTensor = + rpiMetadata.getLocked>("cnn.input_tensor"); + if (cnnEnableInputTensor_ && inputTensor) { + unsigned int size = *rpiMetadata.getLocked("cnn.input_tensor_size"); + Span tensor{ inputTensor->get(), size }; + libcameraMetadata_.set(controls::rpi::CnnInputTensor, tensor); + /* No need to keep these big buffers any more. */ + rpiMetadata.eraseLocked("cnn.input_tensor"); + } + + const RPiController::CnnInputTensorInfo *inputTensorInfo = + rpiMetadata.getLocked("cnn.input_tensor_info"); + if (inputTensorInfo) { + Span tensorInfo{ reinterpret_cast(inputTensorInfo), + sizeof(*inputTensorInfo) }; + libcameraMetadata_.set(controls::rpi::CnnInputTensorInfo, tensorInfo); + } + + const std::shared_ptr *outputTensor = + rpiMetadata.getLocked>("cnn.output_tensor"); + if (outputTensor) { + unsigned int size = *rpiMetadata.getLocked("cnn.output_tensor_size"); + Span tensor{ reinterpret_cast(outputTensor->get()), + size }; + libcameraMetadata_.set(controls::rpi::CnnOutputTensor, tensor); + /* No need to keep these big buffers any more. */ + rpiMetadata.eraseLocked("cnn.output_tensor"); + } + + const RPiController::CnnOutputTensorInfo *outputTensorInfo = + rpiMetadata.getLocked("cnn.output_tensor_info"); + if (outputTensorInfo) { + Span tensorInfo{ reinterpret_cast(outputTensorInfo), + sizeof(*outputTensorInfo) }; + libcameraMetadata_.set(controls::rpi::CnnOutputTensorInfo, tensorInfo); + } + + const RPiController::CnnKpiInfo *kpiInfo = + rpiMetadata.getLocked("cnn.kpi_info"); + if (kpiInfo) { + libcameraMetadata_.set(controls::rpi::CnnKpiInfo, + { static_cast(kpiInfo->dnnRuntime), + static_cast(kpiInfo->dspRuntime) }); + } + metadataReady.emit(libcameraMetadata_); } diff --git a/src/ipa/rpi/common/ipa_base.h b/src/ipa/rpi/common/ipa_base.h index 1a811beb31f2..a55ce7ca9fa3 100644 --- a/src/ipa/rpi/common/ipa_base.h +++ b/src/ipa/rpi/common/ipa_base.h @@ -136,6 +136,8 @@ private: int32_t mode; utils::Duration manualPeriod; } flickerState_; + + bool cnnEnableInputTensor_; }; } /* namespace ipa::RPi */