[{"id":32718,"web_url":"https://patchwork.libcamera.org/comment/32718/","msgid":"","date":"2024-12-13T10:32:53","subject":"Re: [PATCH 5/6] ipa: rpi: Handle the new CNN controls in the IPA","submitter":{"id":42,"url":"https://patchwork.libcamera.org/api/people/42/","name":"David Plowman","email":"david.plowman@raspberrypi.com"},"content":"Hi Naush\n\nThanks for the patch!\n\nOn Fri, 13 Dec 2024 at 09:46, Naushir Patuck wrote:\n>\n> Add code to handle the new CNN vendor controls in the Raspberry Pi IPA.\n>\n> The value of CnnInputTensorInfo is cached as it is the only stateful\n> input control.\n\nJust wanted to check that you didn't mean CnnEnableInputTensor, rather\nthan CnnInputTensorInfo?\n\nOther than that, it all looked fine to me, so apart from the above question:\n\nReviewed-by: David Plowman \n\nThanks\nDavid\n\n>\n> All other controls are output controls, and the values are copied into\n> directly from the rpiMetadata object if present. The camera helpers\n> populate the rpiMetadata object if the sensor supports on-board CNN\n> processing, such as the IMX500.\n>\n> Signed-off-by: Naushir Patuck \n> ---\n> src/ipa/rpi/common/ipa_base.cpp | 52 ++++++++++++++++++++++++++++++++-\n> src/ipa/rpi/common/ipa_base.h | 2 ++\n> 2 files changed, 53 insertions(+), 1 deletion(-)\n>\n> diff --git a/src/ipa/rpi/common/ipa_base.cpp b/src/ipa/rpi/common/ipa_base.cpp\n> index 5fce17e67bd6..b3656cbc730b 100644\n> --- a/src/ipa/rpi/common/ipa_base.cpp\n> +++ b/src/ipa/rpi/common/ipa_base.cpp\n> @@ -74,6 +74,7 @@ const ControlInfoMap::Map ipaControls{\n> { &controls::FrameDurationLimits, ControlInfo(INT64_C(33333), INT64_C(120000)) },\n> { &controls::draft::NoiseReductionMode, ControlInfo(controls::draft::NoiseReductionModeValues) },\n> { &controls::rpi::StatsOutputEnable, ControlInfo(false, true, false) },\n> + { &controls::rpi::CnnEnableInputTensor, ControlInfo(false, true, false) },\n> };\n>\n> /* IPA controls handled conditionally, if the sensor is not mono */\n> @@ -112,7 +113,7 @@ namespace ipa::RPi {\n> IpaBase::IpaBase()\n> : controller_(), frameLengths_(FrameLengthsQueueSize, 0s), statsMetadataOutput_(false),\n> stitchSwapBuffers_(false), frameCount_(0), mistrustCount_(0), lastRunTimestamp_(0),\n> - firstStart_(true), flickerState_({ 0, 0s })\n> + firstStart_(true), flickerState_({ 0, 0s }), cnnEnableInputTensor_(false)\n> {\n> }\n>\n> @@ -1263,6 +1264,10 @@ void IpaBase::applyControls(const ControlList &controls)\n> statsMetadataOutput_ = ctrl.second.get();\n> break;\n>\n> + case controls::rpi::CNN_ENABLE_INPUT_TENSOR:\n> + cnnEnableInputTensor_ = ctrl.second.get();\n> + break;\n> +\n> default:\n> LOG(IPARPI, Warning)\n> << \"Ctrl \" << controls::controls.at(ctrl.first)->name()\n> @@ -1439,6 +1444,51 @@ void IpaBase::reportMetadata(unsigned int ipaContext)\n> libcameraMetadata_.set(controls::HdrChannel, controls::HdrChannelNone);\n> }\n>\n> + const std::shared_ptr *inputTensor =\n> + rpiMetadata.getLocked>(\"cnn.input_tensor\");\n> + if (cnnEnableInputTensor_ && inputTensor) {\n> + unsigned int size = *rpiMetadata.getLocked(\"cnn.input_tensor_size\");\n> + Span tensor{ inputTensor->get(), size };\n> + libcameraMetadata_.set(controls::rpi::CnnInputTensor, tensor);\n> + /* No need to keep these big buffers any more. */\n> + rpiMetadata.eraseLocked(\"cnn.input_tensor\");\n> + }\n> +\n> + const RPiController::CnnInputTensorInfo *inputTensorInfo =\n> + rpiMetadata.getLocked(\"cnn.input_tensor_info\");\n> + if (inputTensorInfo) {\n> + Span tensorInfo{ reinterpret_cast(inputTensorInfo),\n> + sizeof(*inputTensorInfo) };\n> + libcameraMetadata_.set(controls::rpi::CnnInputTensorInfo, tensorInfo);\n> + }\n> +\n> + const std::shared_ptr *outputTensor =\n> + rpiMetadata.getLocked>(\"cnn.output_tensor\");\n> + if (outputTensor) {\n> + unsigned int size = *rpiMetadata.getLocked(\"cnn.output_tensor_size\");\n> + Span tensor{ reinterpret_cast(outputTensor->get()),\n> + size };\n> + libcameraMetadata_.set(controls::rpi::CnnOutputTensor, tensor);\n> + /* No need to keep these big buffers any more. */\n> + rpiMetadata.eraseLocked(\"cnn.output_tensor\");\n> + }\n> +\n> + const RPiController::CnnOutputTensorInfo *outputTensorInfo =\n> + rpiMetadata.getLocked(\"cnn.output_tensor_info\");\n> + if (outputTensorInfo) {\n> + Span tensorInfo{ reinterpret_cast(outputTensorInfo),\n> + sizeof(*outputTensorInfo) };\n> + libcameraMetadata_.set(controls::rpi::CnnOutputTensorInfo, tensorInfo);\n> + }\n> +\n> + const RPiController::CnnKpiInfo *kpiInfo =\n> + rpiMetadata.getLocked(\"cnn.kpi_info\");\n> + if (kpiInfo) {\n> + libcameraMetadata_.set(controls::rpi::CnnKpiInfo,\n> + { static_cast(kpiInfo->dnnRuntime),\n> + static_cast(kpiInfo->dspRuntime) });\n> + }\n> +\n> metadataReady.emit(libcameraMetadata_);\n> }\n>\n> diff --git a/src/ipa/rpi/common/ipa_base.h b/src/ipa/rpi/common/ipa_base.h\n> index 1a811beb31f2..a55ce7ca9fa3 100644\n> --- a/src/ipa/rpi/common/ipa_base.h\n> +++ b/src/ipa/rpi/common/ipa_base.h\n> @@ -136,6 +136,8 @@ private:\n> int32_t mode;\n> utils::Duration manualPeriod;\n> } flickerState_;\n> +\n> + bool cnnEnableInputTensor_;\n> };\n>\n> } /* namespace ipa::RPi */\n> --\n> 2.43.0\n>","headers":{"Return-Path":"","X-Original-To":"parsemail@patchwork.libcamera.org","Delivered-To":"parsemail@patchwork.libcamera.org","Received":["from lancelot.ideasonboard.com (lancelot.ideasonboard.com\n\t[92.243.16.209])\n\tby patchwork.libcamera.org (Postfix) with ESMTPS id 18E68C32EA\n\tfor ;\n\tFri, 13 Dec 2024 10:33:08 +0000 (UTC)","from lancelot.ideasonboard.com (localhost [IPv6:::1])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTP id 3007467EEE;\n\tFri, 13 Dec 2024 11:33:07 +0100 (CET)","from mail-qt1-x82c.google.com (mail-qt1-x82c.google.com\n\t[IPv6:2607:f8b0:4864:20::82c])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id 399E2618AD\n\tfor ;\n\tFri, 13 Dec 2024 11:33:05 +0100 (CET)","by mail-qt1-x82c.google.com with SMTP id\n\td75a77b69052e-46788c32a69so18920631cf.2\n\tfor ;\n\tFri, 13 Dec 2024 02:33:05 -0800 (PST)"],"Authentication-Results":"lancelot.ideasonboard.com; dkim=pass (2048-bit key;\n\tunprotected) header.d=raspberrypi.com header.i=@raspberrypi.com\n\theader.b=\"HtKCkP0g\"; dkim-atps=neutral","DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/relaxed;\n\td=raspberrypi.com; s=google; t=1734085984; x=1734690784;\n\tdarn=lists.libcamera.org; \n\th=cc:to:subject:message-id:date:from:in-reply-to:references\n\t:mime-version:from:to:cc:subject:date:message-id:reply-to;\n\tbh=LXa0fNo5XwDcOSq4Ru+23bhEj8nnEjk+yGZxnvKWcgA=;\n\tb=HtKCkP0gIfozHM3ZkdCLlr89Apr6n0iotNjBYtobTbcyTXGGD9gdfDZlHINrkcO6WO\n\tQtw2wz/UuLEV513TkPlOIYsdxs6feHguI86MHXmsk6f2WJadnUYLKtVVIDL5WBjcEwee\n\tS0KPa4JiIQH+i8lpsJH/WY4digU2Yhim20Ohx5/vWZ9dVRc63gXZ5i/7YTiVenQEoVrE\n\t/aOreH+2mrHognIAtWqwYTai3MgIAPHwJpXjOFHaaQP9rz2sql/zRrf211MwQJCziUyX\n\tOpcFn3RZtvFJTZL7vJRdepYyeQsFVGR06T9BPVj0Mjg23NVuBkPfqaYk07Omkwqglxgb\n\t5MJQ==","X-Google-DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/relaxed;\n\td=1e100.net; s=20230601; t=1734085984; x=1734690784;\n\th=cc:to:subject:message-id:date:from:in-reply-to:references\n\t:mime-version:x-gm-message-state:from:to:cc:subject:date:message-id\n\t:reply-to;\n\tbh=LXa0fNo5XwDcOSq4Ru+23bhEj8nnEjk+yGZxnvKWcgA=;\n\tb=h+ntG+LauQTSysFoG8WtdjVMd04Qh0V86Yi/+lEPP+v+yNY1MsZdRmRqhXQS5F1QKQ\n\tRuc4Vd4g2qIvKfD9MCQkh5QgOHn2EwmYqTLjb9nN9deUvz024fp46/NAfTfNt4Mqi5vj\n\tEsa0fza/b5U3UA0vWOIqOY735Z3lpdBBxgLtUTj91qmSRgoYmqBFumTaWlmb7UfhFvLG\n\tJpJc+eSCmydQiWTO1IXe3vRGNp1I9UxAEInlwmThKZ5sgD8ttcmU4ZUKyKDG2+e3sH8R\n\ttuNAWBEjxqgoKtb/Lm8Hd4V7cnKXu89hCl4ms8MUN4K1CGCwEFj+qvb8qSAM0ovDOp/+\n\tA0JA==","X-Gm-Message-State":"AOJu0Yy1/iFPQ7IAd4QJ5VY//hqIUlKEtC5n/AwcqznuSIYfeaICjVfW\n\trguCK+WE+7Vuwctgm/FMcAJ/qFLC+zt8fcvrXc63+HYttSeBxgljty1yKm/lWS1Xm1iqo8S8CBI\n\tG6nhBL9V+dxlwAOyPWPSVAwxtbRtljLek5jlXsg==","X-Gm-Gg":"ASbGncvRjPL4YebLrIx1+JvEkB4ghs5Pe3yoTxq0tbcua1gv1fj5pDEiRf4tJPnZzfT\n\tKS92oIvx9U1Hao8VECZpIw3GdG9KXSvqudJu6/g==","X-Google-Smtp-Source":"AGHT+IEOgMcLqLHZAZLA38ShM/OIcqmdQdHD1bEYBT7HqwSVKrIZAqaQ5yRb8qdWvISuFcmCt9kbgHFQMVE3jvuz3mw=","X-Received":"by 2002:ac8:590f:0:b0:467:664f:3d4 with SMTP id\n\td75a77b69052e-467a585a7e6mr31236411cf.53.1734085984098;\n\tFri, 13 Dec 2024 02:33:04 -0800 (PST)","MIME-Version":"1.0","References":"<20241213094602.2083174-1-naush@raspberrypi.com>\n\t<20241213094602.2083174-6-naush@raspberrypi.com>","In-Reply-To":"<20241213094602.2083174-6-naush@raspberrypi.com>","From":"David Plowman ","Date":"Fri, 13 Dec 2024 10:32:53 +0000","Message-ID":"","Subject":"Re: [PATCH 5/6] ipa: rpi: Handle the new CNN controls in the IPA","To":"Naushir Patuck ","Cc":"libcamera-devel@lists.libcamera.org","Content-Type":"text/plain; charset=\"UTF-8\"","X-BeenThere":"libcamera-devel@lists.libcamera.org","X-Mailman-Version":"2.1.29","Precedence":"list","List-Id":"","List-Unsubscribe":",\n\t","List-Archive":"","List-Post":"","List-Help":"","List-Subscribe":",\n\t","Errors-To":"libcamera-devel-bounces@lists.libcamera.org","Sender":"\"libcamera-devel\" "}}]