[5/6] ipa: rpi: Handle the new CNN controls in the IPA

22310 diff mbox series

Add code to handle the new CNN vendor controls in the Raspberry Pi IPA.

The value of CnnInputTensorInfo is cached as it is the only stateful
input control.

All other controls are output controls, and the values are copied into
directly from the rpiMetadata object if present. The camera helpers
populate the rpiMetadata object if the sensor supports on-board CNN
processing, such as the IMX500.

Signed-off-by: Naushir Patuck <naush@raspberrypi.com>
---
 src/ipa/rpi/common/ipa_base.cpp | 52 ++++++++++++++++++++++++++++++++-
 src/ipa/rpi/common/ipa_base.h   |  2 ++
 2 files changed, 53 insertions(+), 1 deletion(-)

Hi Naush

Thanks for the patch!

On Fri, 13 Dec 2024 at 09:46, Naushir Patuck <naush@raspberrypi.com> wrote:
>
> Add code to handle the new CNN vendor controls in the Raspberry Pi IPA.
>
> The value of CnnInputTensorInfo is cached as it is the only stateful
> input control.

Just wanted to check that you didn't mean CnnEnableInputTensor, rather
than CnnInputTensorInfo?

Other than that, it all looked fine to me, so apart from the above question:

Reviewed-by: David Plowman <david.plowman@raspberrypi.com>

Thanks
David

>
> All other controls are output controls, and the values are copied into
> directly from the rpiMetadata object if present. The camera helpers
> populate the rpiMetadata object if the sensor supports on-board CNN
> processing, such as the IMX500.
>
> Signed-off-by: Naushir Patuck <naush@raspberrypi.com>
> ---
>  src/ipa/rpi/common/ipa_base.cpp | 52 ++++++++++++++++++++++++++++++++-
>  src/ipa/rpi/common/ipa_base.h   |  2 ++
>  2 files changed, 53 insertions(+), 1 deletion(-)
>
> diff --git a/src/ipa/rpi/common/ipa_base.cpp b/src/ipa/rpi/common/ipa_base.cpp
> index 5fce17e67bd6..b3656cbc730b 100644
> --- a/src/ipa/rpi/common/ipa_base.cpp
> +++ b/src/ipa/rpi/common/ipa_base.cpp
> @@ -74,6 +74,7 @@ const ControlInfoMap::Map ipaControls{
>         { &controls::FrameDurationLimits, ControlInfo(INT64_C(33333), INT64_C(120000)) },
>         { &controls::draft::NoiseReductionMode, ControlInfo(controls::draft::NoiseReductionModeValues) },
>         { &controls::rpi::StatsOutputEnable, ControlInfo(false, true, false) },
> +       { &controls::rpi::CnnEnableInputTensor, ControlInfo(false, true, false) },
>  };
>
>  /* IPA controls handled conditionally, if the sensor is not mono */
> @@ -112,7 +113,7 @@ namespace ipa::RPi {
>  IpaBase::IpaBase()
>         : controller_(), frameLengths_(FrameLengthsQueueSize, 0s), statsMetadataOutput_(false),
>           stitchSwapBuffers_(false), frameCount_(0), mistrustCount_(0), lastRunTimestamp_(0),
> -         firstStart_(true), flickerState_({ 0, 0s })
> +         firstStart_(true), flickerState_({ 0, 0s }), cnnEnableInputTensor_(false)
>  {
>  }
>
> @@ -1263,6 +1264,10 @@ void IpaBase::applyControls(const ControlList &controls)
>                         statsMetadataOutput_ = ctrl.second.get<bool>();
>                         break;
>
> +               case controls::rpi::CNN_ENABLE_INPUT_TENSOR:
> +                       cnnEnableInputTensor_ = ctrl.second.get<bool>();
> +                       break;
> +
>                 default:
>                         LOG(IPARPI, Warning)
>                                 << "Ctrl " << controls::controls.at(ctrl.first)->name()
> @@ -1439,6 +1444,51 @@ void IpaBase::reportMetadata(unsigned int ipaContext)
>                         libcameraMetadata_.set(controls::HdrChannel, controls::HdrChannelNone);
>         }
>
> +       const std::shared_ptr<uint8_t[]> *inputTensor =
> +               rpiMetadata.getLocked<std::shared_ptr<uint8_t[]>>("cnn.input_tensor");
> +       if (cnnEnableInputTensor_ && inputTensor) {
> +               unsigned int size = *rpiMetadata.getLocked<unsigned int>("cnn.input_tensor_size");
> +               Span<const uint8_t> tensor{ inputTensor->get(), size };
> +               libcameraMetadata_.set(controls::rpi::CnnInputTensor, tensor);
> +               /* No need to keep these big buffers any more. */
> +               rpiMetadata.eraseLocked("cnn.input_tensor");
> +       }
> +
> +       const RPiController::CnnInputTensorInfo *inputTensorInfo =
> +               rpiMetadata.getLocked<RPiController::CnnInputTensorInfo>("cnn.input_tensor_info");
> +       if (inputTensorInfo) {
> +               Span<const uint8_t> tensorInfo{ reinterpret_cast<const uint8_t *>(inputTensorInfo),
> +                                               sizeof(*inputTensorInfo) };
> +               libcameraMetadata_.set(controls::rpi::CnnInputTensorInfo, tensorInfo);
> +       }
> +
> +       const std::shared_ptr<float[]> *outputTensor =
> +               rpiMetadata.getLocked<std::shared_ptr<float[]>>("cnn.output_tensor");
> +       if (outputTensor) {
> +               unsigned int size = *rpiMetadata.getLocked<unsigned int>("cnn.output_tensor_size");
> +               Span<const float> tensor{ reinterpret_cast<const float *>(outputTensor->get()),
> +                                         size };
> +               libcameraMetadata_.set(controls::rpi::CnnOutputTensor, tensor);
> +               /* No need to keep these big buffers any more. */
> +               rpiMetadata.eraseLocked("cnn.output_tensor");
> +       }
> +
> +       const RPiController::CnnOutputTensorInfo *outputTensorInfo =
> +               rpiMetadata.getLocked<RPiController::CnnOutputTensorInfo>("cnn.output_tensor_info");
> +       if (outputTensorInfo) {
> +               Span<const uint8_t> tensorInfo{ reinterpret_cast<const uint8_t *>(outputTensorInfo),
> +                                               sizeof(*outputTensorInfo) };
> +               libcameraMetadata_.set(controls::rpi::CnnOutputTensorInfo, tensorInfo);
> +       }
> +
> +       const RPiController::CnnKpiInfo *kpiInfo =
> +               rpiMetadata.getLocked<RPiController::CnnKpiInfo>("cnn.kpi_info");
> +       if (kpiInfo) {
> +               libcameraMetadata_.set(controls::rpi::CnnKpiInfo,
> +                                      { static_cast<int32_t>(kpiInfo->dnnRuntime),
> +                                        static_cast<int32_t>(kpiInfo->dspRuntime) });
> +       }
> +
>         metadataReady.emit(libcameraMetadata_);
>  }
>
> diff --git a/src/ipa/rpi/common/ipa_base.h b/src/ipa/rpi/common/ipa_base.h
> index 1a811beb31f2..a55ce7ca9fa3 100644
> --- a/src/ipa/rpi/common/ipa_base.h
> +++ b/src/ipa/rpi/common/ipa_base.h
> @@ -136,6 +136,8 @@ private:
>                 int32_t mode;
>                 utils::Duration manualPeriod;
>         } flickerState_;
> +
> +       bool cnnEnableInputTensor_;
>  };
>
>  } /* namespace ipa::RPi */
> --
> 2.43.0
>