Patch Detail
Show a patch.
GET /api/patches/22310/?format=api
{ "id": 22310, "url": "https://patchwork.libcamera.org/api/patches/22310/?format=api", "web_url": "https://patchwork.libcamera.org/patch/22310/", "project": { "id": 1, "url": "https://patchwork.libcamera.org/api/projects/1/?format=api", "name": "libcamera", "link_name": "libcamera", "list_id": "libcamera_core", "list_email": "libcamera-devel@lists.libcamera.org", "web_url": "", "scm_url": "", "webscm_url": "" }, "msgid": "<20241213094602.2083174-6-naush@raspberrypi.com>", "date": "2024-12-13T09:38:28", "name": "[5/6] ipa: rpi: Handle the new CNN controls in the IPA", "commit_ref": null, "pull_url": null, "state": "superseded", "archived": false, "hash": "1ad7d8089ba250fdbc459d7b2808ce7921732a25", "submitter": { "id": 34, "url": "https://patchwork.libcamera.org/api/people/34/?format=api", "name": "Naushir Patuck", "email": "naush@raspberrypi.com" }, "delegate": null, "mbox": "https://patchwork.libcamera.org/patch/22310/mbox/", "series": [ { "id": 4881, "url": "https://patchwork.libcamera.org/api/series/4881/?format=api", "web_url": "https://patchwork.libcamera.org/project/libcamera/list/?series=4881", "date": "2024-12-13T09:38:23", "name": "Raspberry Pi: Various changes", "version": 1, "mbox": "https://patchwork.libcamera.org/series/4881/mbox/" } ], "comments": "https://patchwork.libcamera.org/api/patches/22310/comments/", "check": "pending", "checks": "https://patchwork.libcamera.org/api/patches/22310/checks/", "tags": {}, "headers": { "Return-Path": "<libcamera-devel-bounces@lists.libcamera.org>", "X-Original-To": "parsemail@patchwork.libcamera.org", "Delivered-To": "parsemail@patchwork.libcamera.org", "Received": [ "from lancelot.ideasonboard.com (lancelot.ideasonboard.com\n\t[92.243.16.209])\n\tby patchwork.libcamera.org (Postfix) with ESMTPS id 35B99C32F2\n\tfor <parsemail@patchwork.libcamera.org>;\n\tFri, 13 Dec 2024 09:46:20 +0000 (UTC)", "from lancelot.ideasonboard.com (localhost [IPv6:::1])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTP id C40B367EFB;\n\tFri, 13 Dec 2024 10:46:18 +0100 (CET)", "from mail-wr1-x42d.google.com (mail-wr1-x42d.google.com\n\t[IPv6:2a00:1450:4864:20::42d])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id CCDFE67EE8\n\tfor <libcamera-devel@lists.libcamera.org>;\n\tFri, 13 Dec 2024 10:46:11 +0100 (CET)", "by mail-wr1-x42d.google.com with SMTP id\n\tffacd0b85a97d-3862f11a13dso139015f8f.2\n\tfor <libcamera-devel@lists.libcamera.org>;\n\tFri, 13 Dec 2024 01:46:11 -0800 (PST)", "from NAUSH-P-DELL.pitowers.org ([93.93.133.154])\n\tby smtp.gmail.com with ESMTPSA id\n\t5b1f17b1804b1-4362557c502sm43989105e9.11.2024.12.13.01.46.10\n\t(version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256);\n\tFri, 13 Dec 2024 01:46:10 -0800 (PST)" ], "Authentication-Results": "lancelot.ideasonboard.com; dkim=pass (2048-bit key;\n\tunprotected) header.d=raspberrypi.com header.i=@raspberrypi.com\n\theader.b=\"dN5/vvac\"; dkim-atps=neutral", "DKIM-Signature": "v=1; a=rsa-sha256; c=relaxed/relaxed;\n\td=raspberrypi.com; s=google; t=1734083171; x=1734687971;\n\tdarn=lists.libcamera.org; \n\th=content-transfer-encoding:mime-version:references:in-reply-to\n\t:message-id:date:subject:cc:to:from:from:to:cc:subject:date\n\t:message-id:reply-to;\n\tbh=ABhzmtZRvpmXxSShgjDOKB8Yq6zpZHSD4zqVCCPH2wI=;\n\tb=dN5/vvac5Vqv5/uLoA9LpQfz2AnieWcuVSvI9Q3hMUpiQ8KWlfn4+eHQ8y1Si79UR9\n\tv2Traq3DHDklur9CQCk21ey6IvctE//srmdJBcSL7k7NifsZSt60zaq2+xW949pLQEkx\n\tfwvIQRIdC5zeH4ebHxX6Q2YZ1fXYUcXRe1OCL60YNZAwRP30sVN/dkNfzXyjXIEhvq4t\n\t2rh0xO6j0gUP0Az8Z+e5skizhtG9Uc9WjNSPNMwePGh6WdK53O2Mox5zbyqNxqA0WVr5\n\tJss9VMrLGx6HalZhU92VrRUGxQq2denrmNc9KMnmy1UEP910cqQCclgJRZLEg5Qh6OCA\n\tDJCw==", "X-Google-DKIM-Signature": "v=1; a=rsa-sha256; c=relaxed/relaxed;\n\td=1e100.net; s=20230601; t=1734083171; x=1734687971;\n\th=content-transfer-encoding:mime-version:references:in-reply-to\n\t:message-id:date:subject:cc:to:from:x-gm-message-state:from:to:cc\n\t:subject:date:message-id:reply-to;\n\tbh=ABhzmtZRvpmXxSShgjDOKB8Yq6zpZHSD4zqVCCPH2wI=;\n\tb=sqU/TjTVOZ0lvANeRLbHFdf7hswjlcVNABI06WJmnnezUtN9VCQEqxZGnR3WoNcvt/\n\t6H9i8zXntJzq9y4zn/8Yd6HVssjmnO3oNLGWf9KgaNqdOjWET5+4657YaVRiSZb48l0D\n\t6/6wWozbCI+y2krnuSyFzMTlNS+b0W6CJqhRafpRoXrJOx8XQRiRFwGc7hmwO1XMMJF6\n\t8cqObh53kAKK7QpFcm5S5deYFJzU6Ed1EhyjhE7d0m50NFChk6KuH9lxYAANMGK8o6j6\n\tZqyYdUAPpJMObkbrxTPFQ/qX0EVKUErHO85kvpJ8Pd2na59EEwxx07lRQSKu2eye9Wzh\n\tdFLQ==", "X-Gm-Message-State": "AOJu0YwaxMdrfgvF7pRo4s4ACSu+/my8wScSfFL6VuvOdeyvl3G+O4qr\n\tzCCsDLMScDFggN6M2nM6GzVftXnJcEYuRCSTNfvW6/hjbrDPCuBZjLvMc6t3nUQph9Ez608Q1u0\n\tg", "X-Gm-Gg": "ASbGncvl7xawp3YbjtZ3/X+A1deYjyZ/8CCtyPCTBO9p1Eb/FdRqeLW20+3/Vi3gp3d\n\tPv9YRA9Qu2PS8Vx7Sl6qRs92Clqkow8XpxCuce0f7i2i6BNkg79nBUPNGGbB7XTWftWf4qrH1OP\n\tDRYwJTqzbi0NncZcGWVqpNYQa6w8HI/eCoMBMHry2gct9XVBXpQsJewxrcNTVIQ3tfaS4KATZfp\n\t/U2/fdpNBEh7BzCFfMBc3g4WqGsKMpDNw0I/X1GsDmNV5FzTkHZun+MsvZB26TkIP90n5PV1XXP", "X-Google-Smtp-Source": "AGHT+IEdjHRI0bMg3gh4pBka3rxvuJympF2JjHNdnTMcxGnEqJxvqbG0pp1zk4qL0pNU6cy2+/V2ow==", "X-Received": "by 2002:a05:600c:3b02:b0:434:fecf:cb2f with SMTP id\n\t5b1f17b1804b1-4362aa8faa0mr5396425e9.5.1734083171063; \n\tFri, 13 Dec 2024 01:46:11 -0800 (PST)", "From": "Naushir Patuck <naush@raspberrypi.com>", "To": "libcamera-devel@lists.libcamera.org", "Cc": "Naushir Patuck <naush@raspberrypi.com>", "Subject": "[PATCH 5/6] ipa: rpi: Handle the new CNN controls in the IPA", "Date": "Fri, 13 Dec 2024 09:38:28 +0000", "Message-ID": "<20241213094602.2083174-6-naush@raspberrypi.com>", "X-Mailer": "git-send-email 2.43.0", "In-Reply-To": "<20241213094602.2083174-1-naush@raspberrypi.com>", "References": "<20241213094602.2083174-1-naush@raspberrypi.com>", "MIME-Version": "1.0", "Content-Transfer-Encoding": "8bit", "X-BeenThere": "libcamera-devel@lists.libcamera.org", "X-Mailman-Version": "2.1.29", "Precedence": "list", "List-Id": "<libcamera-devel.lists.libcamera.org>", "List-Unsubscribe": "<https://lists.libcamera.org/options/libcamera-devel>,\n\t<mailto:libcamera-devel-request@lists.libcamera.org?subject=unsubscribe>", "List-Archive": "<https://lists.libcamera.org/pipermail/libcamera-devel/>", "List-Post": "<mailto:libcamera-devel@lists.libcamera.org>", "List-Help": "<mailto:libcamera-devel-request@lists.libcamera.org?subject=help>", "List-Subscribe": "<https://lists.libcamera.org/listinfo/libcamera-devel>,\n\t<mailto:libcamera-devel-request@lists.libcamera.org?subject=subscribe>", "Errors-To": "libcamera-devel-bounces@lists.libcamera.org", "Sender": "\"libcamera-devel\" <libcamera-devel-bounces@lists.libcamera.org>" }, "content": "Add code to handle the new CNN vendor controls in the Raspberry Pi IPA.\n\nThe value of CnnInputTensorInfo is cached as it is the only stateful\ninput control.\n\nAll other controls are output controls, and the values are copied into\ndirectly from the rpiMetadata object if present. The camera helpers\npopulate the rpiMetadata object if the sensor supports on-board CNN\nprocessing, such as the IMX500.\n\nSigned-off-by: Naushir Patuck <naush@raspberrypi.com>\n---\n src/ipa/rpi/common/ipa_base.cpp | 52 ++++++++++++++++++++++++++++++++-\n src/ipa/rpi/common/ipa_base.h | 2 ++\n 2 files changed, 53 insertions(+), 1 deletion(-)", "diff": "diff --git a/src/ipa/rpi/common/ipa_base.cpp b/src/ipa/rpi/common/ipa_base.cpp\nindex 5fce17e67bd6..b3656cbc730b 100644\n--- a/src/ipa/rpi/common/ipa_base.cpp\n+++ b/src/ipa/rpi/common/ipa_base.cpp\n@@ -74,6 +74,7 @@ const ControlInfoMap::Map ipaControls{\n \t{ &controls::FrameDurationLimits, ControlInfo(INT64_C(33333), INT64_C(120000)) },\n \t{ &controls::draft::NoiseReductionMode, ControlInfo(controls::draft::NoiseReductionModeValues) },\n \t{ &controls::rpi::StatsOutputEnable, ControlInfo(false, true, false) },\n+\t{ &controls::rpi::CnnEnableInputTensor, ControlInfo(false, true, false) },\n };\n \n /* IPA controls handled conditionally, if the sensor is not mono */\n@@ -112,7 +113,7 @@ namespace ipa::RPi {\n IpaBase::IpaBase()\n \t: controller_(), frameLengths_(FrameLengthsQueueSize, 0s), statsMetadataOutput_(false),\n \t stitchSwapBuffers_(false), frameCount_(0), mistrustCount_(0), lastRunTimestamp_(0),\n-\t firstStart_(true), flickerState_({ 0, 0s })\n+\t firstStart_(true), flickerState_({ 0, 0s }), cnnEnableInputTensor_(false)\n {\n }\n \n@@ -1263,6 +1264,10 @@ void IpaBase::applyControls(const ControlList &controls)\n \t\t\tstatsMetadataOutput_ = ctrl.second.get<bool>();\n \t\t\tbreak;\n \n+\t\tcase controls::rpi::CNN_ENABLE_INPUT_TENSOR:\n+\t\t\tcnnEnableInputTensor_ = ctrl.second.get<bool>();\n+\t\t\tbreak;\n+\n \t\tdefault:\n \t\t\tLOG(IPARPI, Warning)\n \t\t\t\t<< \"Ctrl \" << controls::controls.at(ctrl.first)->name()\n@@ -1439,6 +1444,51 @@ void IpaBase::reportMetadata(unsigned int ipaContext)\n \t\t\tlibcameraMetadata_.set(controls::HdrChannel, controls::HdrChannelNone);\n \t}\n \n+\tconst std::shared_ptr<uint8_t[]> *inputTensor =\n+\t\trpiMetadata.getLocked<std::shared_ptr<uint8_t[]>>(\"cnn.input_tensor\");\n+\tif (cnnEnableInputTensor_ && inputTensor) {\n+\t\tunsigned int size = *rpiMetadata.getLocked<unsigned int>(\"cnn.input_tensor_size\");\n+\t\tSpan<const uint8_t> tensor{ inputTensor->get(), size };\n+\t\tlibcameraMetadata_.set(controls::rpi::CnnInputTensor, tensor);\n+\t\t/* No need to keep these big buffers any more. */\n+\t\trpiMetadata.eraseLocked(\"cnn.input_tensor\");\n+\t}\n+\n+\tconst RPiController::CnnInputTensorInfo *inputTensorInfo =\n+\t\trpiMetadata.getLocked<RPiController::CnnInputTensorInfo>(\"cnn.input_tensor_info\");\n+\tif (inputTensorInfo) {\n+\t\tSpan<const uint8_t> tensorInfo{ reinterpret_cast<const uint8_t *>(inputTensorInfo),\n+\t\t\t\t\t\tsizeof(*inputTensorInfo) };\n+\t\tlibcameraMetadata_.set(controls::rpi::CnnInputTensorInfo, tensorInfo);\n+\t}\n+\n+\tconst std::shared_ptr<float[]> *outputTensor =\n+\t\trpiMetadata.getLocked<std::shared_ptr<float[]>>(\"cnn.output_tensor\");\n+\tif (outputTensor) {\n+\t\tunsigned int size = *rpiMetadata.getLocked<unsigned int>(\"cnn.output_tensor_size\");\n+\t\tSpan<const float> tensor{ reinterpret_cast<const float *>(outputTensor->get()),\n+\t\t\t\t\t size };\n+\t\tlibcameraMetadata_.set(controls::rpi::CnnOutputTensor, tensor);\n+\t\t/* No need to keep these big buffers any more. */\n+\t\trpiMetadata.eraseLocked(\"cnn.output_tensor\");\n+\t}\n+\n+\tconst RPiController::CnnOutputTensorInfo *outputTensorInfo =\n+\t\trpiMetadata.getLocked<RPiController::CnnOutputTensorInfo>(\"cnn.output_tensor_info\");\n+\tif (outputTensorInfo) {\n+\t\tSpan<const uint8_t> tensorInfo{ reinterpret_cast<const uint8_t *>(outputTensorInfo),\n+\t\t\t\t\t\tsizeof(*outputTensorInfo) };\n+\t\tlibcameraMetadata_.set(controls::rpi::CnnOutputTensorInfo, tensorInfo);\n+\t}\n+\n+\tconst RPiController::CnnKpiInfo *kpiInfo =\n+\t\trpiMetadata.getLocked<RPiController::CnnKpiInfo>(\"cnn.kpi_info\");\n+\tif (kpiInfo) {\n+\t\tlibcameraMetadata_.set(controls::rpi::CnnKpiInfo,\n+\t\t\t\t { static_cast<int32_t>(kpiInfo->dnnRuntime),\n+\t\t\t\t\t static_cast<int32_t>(kpiInfo->dspRuntime) });\n+\t}\n+\n \tmetadataReady.emit(libcameraMetadata_);\n }\n \ndiff --git a/src/ipa/rpi/common/ipa_base.h b/src/ipa/rpi/common/ipa_base.h\nindex 1a811beb31f2..a55ce7ca9fa3 100644\n--- a/src/ipa/rpi/common/ipa_base.h\n+++ b/src/ipa/rpi/common/ipa_base.h\n@@ -136,6 +136,8 @@ private:\n \t\tint32_t mode;\n \t\tutils::Duration manualPeriod;\n \t} flickerState_;\n+\n+\tbool cnnEnableInputTensor_;\n };\n \n } /* namespace ipa::RPi */\n", "prefixes": [ "5/6" ] }