[{"id":37247,"web_url":"https://patchwork.libcamera.org/comment/37247/","msgid":"","date":"2025-12-10T14:08:53","subject":"Re: [PATCH 2/4] ipa: rpi: controller: awb: Add Neural Network Awb","submitter":{"id":34,"url":"https://patchwork.libcamera.org/api/people/34/","name":"Naushir Patuck","email":"naush@raspberrypi.com"},"content":"Hi David and Peter,\n\nOnly minor styling comments below:\n\nOn Fri, 24 Oct 2025 at 15:41, David Plowman\n wrote:\n>\n> From: Peter Bailey \n>\n> Add an Awb algorithm which uses neural networks.\n>\n> Signed-off-by: Peter Bailey \n> ---\n> src/ipa/rpi/controller/meson.build | 9 +\n> src/ipa/rpi/controller/rpi/awb_nn.cpp | 442 ++++++++++++++++++++++++++\n> 2 files changed, 451 insertions(+)\n> create mode 100644 src/ipa/rpi/controller/rpi/awb_nn.cpp\n>\n> diff --git a/src/ipa/rpi/controller/meson.build b/src/ipa/rpi/controller/meson.build\n> index 73c93dca..2541d073 100644\n> --- a/src/ipa/rpi/controller/meson.build\n> +++ b/src/ipa/rpi/controller/meson.build\n> @@ -32,6 +32,15 @@ rpi_ipa_controller_deps = [\n> libcamera_private,\n> ]\n>\n> +tflite_dep = dependency('tensorflow-lite', required : false)\n> +\n> +if tflite_dep.found()\n> + rpi_ipa_controller_sources += files([\n> + 'rpi/awb_nn.cpp',\n> + ])\n> + rpi_ipa_controller_deps += tflite_dep\n> +endif\n> +\n> rpi_ipa_controller_lib = static_library('rpi_ipa_controller', rpi_ipa_controller_sources,\n> include_directories : libipa_includes,\n> dependencies : rpi_ipa_controller_deps)\n> diff --git a/src/ipa/rpi/controller/rpi/awb_nn.cpp b/src/ipa/rpi/controller/rpi/awb_nn.cpp\n> new file mode 100644\n> index 00000000..c309ca3f\n> --- /dev/null\n> +++ b/src/ipa/rpi/controller/rpi/awb_nn.cpp\n> @@ -0,0 +1,442 @@\n> +/* SPDX-License-Identifier: BSD-2-Clause */\n> +/*\n> + * Copyright (C) 2025, Raspberry Pi Ltd\n> + *\n> + * AWB control algorithm using neural network\n> + */\n> +\n> +#include \n> +#include \n> +#include \n> +\n> +#include \n> +#include \n> +\n> +#include \n> +#include \n> +#include \n> +\n> +#include \"../awb_algorithm.h\"\n> +#include \"../awb_status.h\"\n> +#include \"../lux_status.h\"\n> +#include \"libipa/pwl.h\"\n> +\n> +#include \"alsc_status.h\"\n> +#include \"awb.h\"\n> +\n> +using namespace libcamera;\n> +\n> +LOG_DECLARE_CATEGORY(RPiAwb)\n> +\n> +constexpr double kDefaultCT = 4500.0;\n> +\n> +#define NAME \"rpi.nn.awb\"\n> +\n> +namespace RPiController {\n> +\n> +struct AwbNNConfig {\n> + AwbNNConfig() {}\n> + int read(const libcamera::YamlObject ¶ms, AwbConfig &config);\n> +\n> + /* An empty model will check default locations for model.tflite */\n> + std::string model;\n> + float minTemp;\n> + float maxTemp;\n> +\n> + bool enableNn;\n> +\n> + /* CCM matrix for 5000K temperature */\n> + double ccm[9];\n> +};\n> +\n> +class AwbNN : public Awb\n> +{\n> +public:\n> + AwbNN(Controller *controller = NULL);\n> + ~AwbNN();\n> + char const *name() const override;\n> + void initialise() override;\n> + int read(const libcamera::YamlObject ¶ms) override;\n> +\n> +protected:\n> + void doAwb() override;\n> + void prepareStats() override;\n> +\n> +private:\n> + bool isAutoEnabled() const;\n> + AwbNNConfig nnConfig_;\n> + void transverseSearch(double t, double &r, double &b);\n> + RGB processZone(RGB zone, float red_gain, float blue_gain);\n> + void awbNN();\n> + void loadModel();\n> +\n> + libcamera::Size zoneSize_;\n> + std::unique_ptr model_;\n> + std::unique_ptr interpreter_;\n> +};\n> +\n> +int AwbNNConfig::read(const libcamera::YamlObject ¶ms, AwbConfig &config)\n> +{\n> + model = params[\"model\"].get(\"\");\n> + minTemp = params[\"min_temp\"].get(2800.0);\n> + maxTemp = params[\"max_temp\"].get(7600.0);\n> +\n> + for (int i = 0; i < 9; i++)\n> + ccm[i] = params[\"ccm\"][i].get(0.0);\n> +\n> + enableNn = params[\"enable_nn\"].get(1);\n> +\n> + if (enableNn) {\n> + if (!config.hasCtCurve()) {\n> + LOG(RPiAwb, Error) << \"CT curve not specified\";\n> + enableNn = false;\n> + }\n> +\n> + if (!model.empty() && model.find(\".tflite\") == std::string::npos) {\n> + LOG(RPiAwb, Error) << \"Model must be a .tflite file\";\n> + enableNn = false;\n> + }\n> +\n> + bool validCcm = true;\n> + for (int i = 0; i < 9; i++)\n> + if (ccm[i] == 0.0)\n> + validCcm = false;\n> +\n> + if (!validCcm) {\n> + LOG(RPiAwb, Error) << \"CCM not specified or invalid\";\n> + enableNn = false;\n> + }\n> +\n> + if (!enableNn) {\n> + LOG(RPiAwb, Warning) << \"Neural Network AWB mis-configured - switch to Grey method\";\n> + }\n> + }\n> +\n> + if (!enableNn) {\n> + config.sensitivityR = config.sensitivityB = 1.0;\n> + config.greyWorld = true;\n> + }\n> +\n> + return 0;\n> +}\n> +\n> +AwbNN::AwbNN(Controller *controller)\n> + : Awb(controller)\n> +{\n> + zoneSize_ = getHardwareConfig().awbRegions;\n> +}\n> +\n> +AwbNN::~AwbNN()\n> +{\n> +}\n> +\n> +char const *AwbNN::name() const\n> +{\n> + return NAME;\n> +}\n> +\n> +int AwbNN::read(const libcamera::YamlObject ¶ms)\n> +{\n> + int ret;\n> +\n> + ret = config_.read(params);\n> + if (ret)\n> + return ret;\n> +\n> + ret = nnConfig_.read(params, config_);\n> + if (ret)\n> + return ret;\n> +\n> + return 0;\n> +}\n> +\n> +static bool checkTensorShape(TfLiteTensor *tensor, const int *expectedDims, const int expectedDimsSize)\n> +{\n> + if (tensor->dims->size != expectedDimsSize) {\n> + return false;\n> + }\n> +\n> + for (int i = 0; i < tensor->dims->size; i++) {\n> + if (tensor->dims->data[i] != expectedDims[i]) {\n> + return false;\n> + }\n\nno need for brackets here and above.\n\n> + }\n> + return true;\n> +}\n> +\n> +static std::string buildDimString(const int *dims, const int dimsSize)\n> +{\n> + std::string s = \"[\";\n> + for (int i = 0; i < dimsSize; i++) {\n> + s += std::to_string(dims[i]);\n> + if (i < dimsSize - 1)\n> + s += \",\";\n> + else\n> + s += \"]\";\n> + }\n> + return s;\n> +}\n> +\n> +void AwbNN::loadModel()\n> +{\n> + std::string modelPath;\n> + if (getTarget() == \"bcm2835\") {\n> + modelPath = \"/ipa/rpi/vc4/awb_model.tflite\";\n> + } else {\n> + modelPath = \"/ipa/rpi/pisp/awb_model.tflite\";\n> + }\n> +\n> + if (nnConfig_.model.empty()) {\n> + std::string root = utils::libcameraSourcePath();\n> + if (!root.empty()) {\n> + modelPath = root + modelPath;\n> + } else {\n> + modelPath = LIBCAMERA_DATA_DIR + modelPath;\n> + }\n> +\n> + if (!File::exists(modelPath)) {\n> + LOG(RPiAwb, Error) << \"No model file found in standard locations\";\n> + nnConfig_.enableNn = false;\n> + return;\n> + }\n> + } else {\n> + modelPath = nnConfig_.model;\n> + }\n> +\n> + LOG(RPiAwb, Debug) << \"Attempting to load model from: \" << modelPath;\n> +\n> + model_ = tflite::FlatBufferModel::BuildFromFile(modelPath.c_str());\n> +\n> + if (!model_) {\n> + LOG(RPiAwb, Error) << \"Failed to load model from \" << modelPath;\n> + nnConfig_.enableNn = false;\n> + return;\n> + }\n> +\n> + tflite::MutableOpResolver resolver;\n> + tflite::ops::builtin::BuiltinOpResolver builtin_resolver;\n> + resolver.AddAll(builtin_resolver);\n> + tflite::InterpreterBuilder(*model_, resolver)(&interpreter_);\n> + if (!interpreter_) {\n> + LOG(RPiAwb, Error) << \"Failed to build interpreter for model \" << nnConfig_.model;\n> + nnConfig_.enableNn = false;\n> + return;\n> + }\n> +\n> + interpreter_->AllocateTensors();\n> + TfLiteTensor *inputTensor = interpreter_->input_tensor(0);\n> + TfLiteTensor *inputLuxTensor = interpreter_->input_tensor(1);\n> + TfLiteTensor *outputTensor = interpreter_->output_tensor(0);\n> + if (!inputTensor || !inputLuxTensor || !outputTensor) {\n> + LOG(RPiAwb, Error) << \"Model missing input or output tensor\";\n> + nnConfig_.enableNn = false;\n> + return;\n> + }\n> +\n> + const int expectedInputDims[] = { 1, (int)zoneSize_.height, (int)zoneSize_.width, 3 };\n> + const int expectedInputLuxDims[] = { 1 };\n> + const int expectedOutputDims[] = { 1 };\n> +\n> + if (!checkTensorShape(inputTensor, expectedInputDims, 4)) {\n> + LOG(RPiAwb, Error) << \"Model input tensor dimension mismatch. Expected: \" << buildDimString(expectedInputDims, 4)\n> + << \", Got: \" << buildDimString(inputTensor->dims->data, inputTensor->dims->size);\n> + nnConfig_.enableNn = false;\n> + return;\n> + }\n> +\n> + if (!checkTensorShape(inputLuxTensor, expectedInputLuxDims, 1)) {\n> + LOG(RPiAwb, Error) << \"Model input lux tensor dimension mismatch. Expected: \" << buildDimString(expectedInputLuxDims, 1)\n> + << \", Got: \" << buildDimString(inputLuxTensor->dims->data, inputLuxTensor->dims->size);\n> + nnConfig_.enableNn = false;\n> + return;\n> + }\n> +\n> + if (!checkTensorShape(outputTensor, expectedOutputDims, 1)) {\n> + LOG(RPiAwb, Error) << \"Model output tensor dimension mismatch. Expected: \" << buildDimString(expectedOutputDims, 1)\n> + << \", Got: \" << buildDimString(outputTensor->dims->data, outputTensor->dims->size);\n> + nnConfig_.enableNn = false;\n> + return;\n> + }\n> +\n> + if (inputTensor->type != kTfLiteFloat32 || inputLuxTensor->type != kTfLiteFloat32 || outputTensor->type != kTfLiteFloat32) {\n> + LOG(RPiAwb, Error) << \"Model input and output tensors must be float32\";\n> + nnConfig_.enableNn = false;\n> + return;\n> + }\n> +\n> + LOG(RPiAwb, Info) << \"Model loaded successfully from \" << modelPath;\n> + LOG(RPiAwb, Debug) << \"Model validation successful - Input Image: \"\n> + << buildDimString(expectedInputDims, 4)\n> + << \", Input Lux: \" << buildDimString(expectedInputLuxDims, 1)\n> + << \", Output: \" << buildDimString(expectedOutputDims, 1) << \" floats\";\n> +}\n> +\n> +void AwbNN::initialise()\n> +{\n> + Awb::initialise();\n> +\n> + if (nnConfig_.enableNn) {\n> + loadModel();\n> + if (!nnConfig_.enableNn) {\n> + LOG(RPiAwb, Warning) << \"Neural Network AWB failed to load - switch to Grey method\";\n> + config_.greyWorld = true;\n> + config_.sensitivityR = config_.sensitivityB = 1.0;\n> + }\n> + }\n> +}\n> +\n> +void AwbNN::prepareStats()\n> +{\n> + zones_.clear();\n> + /*\n> + * LSC has already been applied to the stats in this pipeline, so stop\n> + * any LSC compensation. We also ignore config_.fast in this version.\n> + */\n> + generateStats(zones_, statistics_, 0.0, 0.0, getGlobalMetadata(), 0.0, 0.0, 0.0);\n> + /*\n> + * apply sensitivities, so values appear to come from our \"canonical\"\n> + * sensor.\n> + */\n> + for (auto &zone : zones_) {\n> + zone.R *= config_.sensitivityR;\n> + zone.B *= config_.sensitivityB;\n> + }\n> +}\n> +\n> +void AwbNN::transverseSearch(double t, double &r, double &b)\n> +{\n> + int spanR = -1, spanB = -1;\n> + config_.ctR.eval(t, &spanR);\n> + config_.ctB.eval(t, &spanB);\n> +\n> + const int diff = 10;\n> + double rDiff = config_.ctR.eval(t + diff, &spanR) -\n> + config_.ctR.eval(t - diff, &spanR);\n> + double bDiff = config_.ctB.eval(t + diff, &spanB) -\n> + config_.ctB.eval(t - diff, &spanB);\n> +\n> + ipa::Pwl::Point transverse({ bDiff, -rDiff });\n> + if (transverse.length2() < 1e-6)\n> + return;\n> +\n> + transverse = transverse / transverse.length();\n> + double transverseRange = config_.transverseNeg + config_.transversePos;\n> + const int maxNumDeltas = 12;\n> + int numDeltas = floor(transverseRange * 100 + 0.5) + 1;\n> + numDeltas = numDeltas < 3 ? 3 : (numDeltas > maxNumDeltas ? maxNumDeltas : numDeltas);\n> +\n> + ipa::Pwl::Point points[maxNumDeltas];\n> + int bestPoint = 0;\n> +\n> + for (int i = 0; i < numDeltas; i++) {\n> + points[i][0] = -config_.transverseNeg +\n> + (transverseRange * i) / (numDeltas - 1);\n> + ipa::Pwl::Point rbTest = ipa::Pwl::Point({ r, b }) +\n> + transverse * points[i].x();\n> + double rTest = rbTest.x(), bTest = rbTest.y();\n> + double gainR = 1 / rTest, gainB = 1 / bTest;\n> + double delta2Sum = computeDelta2Sum(gainR, gainB, 0.0, 0.0);\n> + points[i][1] = delta2Sum;\n> + if (points[i].y() < points[bestPoint].y())\n> + bestPoint = i;\n> + }\n> +\n> + bestPoint = std::clamp(bestPoint, 1, numDeltas - 2);\n> + ipa::Pwl::Point rbBest = ipa::Pwl::Point({ r, b }) +\n> + transverse * interpolateQuadatric(points[bestPoint - 1],\n> + points[bestPoint],\n> + points[bestPoint + 1]);\n> + double rBest = rbBest.x(), bBest = rbBest.y();\n> +\n> + r = rBest, b = bBest;\n> +}\n> +\n> +AwbNN::RGB AwbNN::processZone(AwbNN::RGB zone, float redGain, float blueGain)\n> +{\n> + /*\n> + * Renders the pixel at 5000K temperature\n> + */\n> + RGB zoneGains = zone;\n> +\n> + zoneGains.R *= redGain;\n> + zoneGains.G *= 1.0;\n> + zoneGains.B *= blueGain;\n> +\n> + RGB zoneCcm;\n> +\n> + zoneCcm.R = nnConfig_.ccm[0] * zoneGains.R + nnConfig_.ccm[1] * zoneGains.G + nnConfig_.ccm[2] * zoneGains.B;\n> + zoneCcm.G = nnConfig_.ccm[3] * zoneGains.R + nnConfig_.ccm[4] * zoneGains.G + nnConfig_.ccm[5] * zoneGains.B;\n> + zoneCcm.B = nnConfig_.ccm[6] * zoneGains.R + nnConfig_.ccm[7] * zoneGains.G + nnConfig_.ccm[8] * zoneGains.B;\n> +\n> + return zoneCcm;\n> +}\n> +\n> +void AwbNN::awbNN()\n> +{\n> + float *inputData = interpreter_->typed_input_tensor(0);\n> + float *inputLux = interpreter_->typed_input_tensor(1);\n> +\n> + float redGain = 1.0 / config_.ctR.eval(5000);\n> + float blueGain = 1.0 / config_.ctB.eval(5000);\n\nMaybe the 5000k here and elsewhere needs to be a constant\nkCanonicalTemp or something?\n\n> +\n> + for (uint i = 0; i < zoneSize_.height; i++) {\n> + for (uint j = 0; j < zoneSize_.width; j++) {\n> + uint zoneIdx = i * zoneSize_.width + j;\n> +\n> + RGB processedZone = processZone(zones_[zoneIdx] * (1.0 / 65535), redGain, blueGain);\n> + uint baseIdx = zoneIdx * 3;\n> +\n> + inputData[baseIdx + 0] = static_cast(processedZone.R);\n> + inputData[baseIdx + 1] = static_cast(processedZone.G);\n> + inputData[baseIdx + 2] = static_cast(processedZone.B);\n> + }\n> + }\n> +\n> + inputLux[0] = static_cast(lux_);\n> +\n> + TfLiteStatus status = interpreter_->Invoke();\n> + if (status != kTfLiteOk) {\n> + LOG(RPiAwb, Error) << \"Model inference failed with status: \" << status;\n> + return;\n> + }\n> +\n> + float *outputData = interpreter_->typed_output_tensor(0);\n> +\n> + double t = outputData[0];\n> +\n> + LOG(RPiAwb, Debug) << \"Model output temperature: \" << t;\n> +\n> + t = std::clamp(t, mode_->ctLo, mode_->ctHi);\n> +\n> + double r = config_.ctR.eval(t);\n> + double b = config_.ctB.eval(t);\n> +\n> + transverseSearch(t, r, b);\n> +\n> + LOG(RPiAwb, Debug) << \"After transverse search: Temperature: \" << t << \" Red gain: \" << 1.0 / r << \" Blue gain: \" << 1.0 / b;\n> +\n> + asyncResults_.temperatureK = t;\n> + asyncResults_.gainR = 1.0 / r * config_.sensitivityR;\n> + asyncResults_.gainG = 1.0;\n> + asyncResults_.gainB = 1.0 / b * config_.sensitivityB;\n> +}\n> +\n> +void AwbNN::doAwb()\n> +{\n> + prepareStats();\n> + if (zones_.size() == (zoneSize_.width * zoneSize_.height) && nnConfig_.enableNn) {\n> + awbNN();\n> + } else {\n> + awbGrey();\n> + }\n\nNo need for the brackets here.\n\nReviewed-by: Naushir Patuck \n\n> + statistics_.reset();\n> +}\n> +\n> +/* Register algorithm with the system. */\n> +static Algorithm *create(Controller *controller)\n> +{\n> + return (Algorithm *)new AwbNN(controller);\n> +}\n> +static RegisterAlgorithm reg(NAME, &create);\n> +\n> +} /* namespace RPiController */\n> --\n> 2.47.3\n>","headers":{"Return-Path":"","X-Original-To":"parsemail@patchwork.libcamera.org","Delivered-To":"parsemail@patchwork.libcamera.org","Received":["from lancelot.ideasonboard.com (lancelot.ideasonboard.com\n\t[92.243.16.209])\n\tby patchwork.libcamera.org (Postfix) with ESMTPS id DC921C3257\n\tfor ;\n\tWed, 10 Dec 2025 14:09:32 +0000 (UTC)","from lancelot.ideasonboard.com (localhost [IPv6:::1])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTP id 47B676146D;\n\tWed, 10 Dec 2025 15:09:32 +0100 (CET)","from mail-vs1-xe2e.google.com (mail-vs1-xe2e.google.com\n\t[IPv6:2607:f8b0:4864:20::e2e])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id 12C8C613CB\n\tfor ;\n\tWed, 10 Dec 2025 15:09:31 +0100 (CET)","by mail-vs1-xe2e.google.com with SMTP id\n\tada2fe7eead31-5dfb3297151so368151137.0\n\tfor ;\n\tWed, 10 Dec 2025 06:09:30 -0800 (PST)"],"Authentication-Results":"lancelot.ideasonboard.com; dkim=pass (2048-bit key;\n\tunprotected) header.d=raspberrypi.com header.i=@raspberrypi.com\n\theader.b=\"VGI7gCmD\"; dkim-atps=neutral","DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/relaxed;\n\td=raspberrypi.com; s=google; t=1765375770; x=1765980570;\n\tdarn=lists.libcamera.org; \n\th=cc:to:subject:message-id:date:from:in-reply-to:references\n\t:mime-version:from:to:cc:subject:date:message-id:reply-to;\n\tbh=NO1TkJrWu15sCFeRt7KugDdT+ZJ15SYgKNomcIFwdKM=;\n\tb=VGI7gCmDLmLkprotEFUkxHEOB61NWZ6ym1VuefaU9erqAqgYVCSCSvAL+LdPETLzeM\n\tDniITMkuqMkP1/AfiUexOfgZCt15iH8oQZJnQM3M+DpFWnnruGt5h8miV2OrfLj5Ba8c\n\taclCtJX/AUA8EgfWhBPbHYGhg/J+e10ZIgR4dXskOYtcP6etg84ciAPQGxU1cGFbxZfL\n\td9LRIn7pBbVjVcMiGSXTAz5vv1grmP+/Ouc7OMAgrJlE39FkCsSIKK2gqsNibN0eh//G\n\tlPG9Wvu32EPkCq6j3V40Ttuy+qU7DkujXBnZ41l8VVQfrf6YFc6oxLINkK6Oj5IFXE1f\n\t4JFQ==","X-Google-DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/relaxed;\n\td=1e100.net; s=20230601; t=1765375770; x=1765980570;\n\th=cc:to:subject:message-id:date:from:in-reply-to:references\n\t:mime-version:x-gm-gg:x-gm-message-state:from:to:cc:subject:date\n\t:message-id:reply-to;\n\tbh=NO1TkJrWu15sCFeRt7KugDdT+ZJ15SYgKNomcIFwdKM=;\n\tb=SF+VCNQtdGhIV//Eii5dYkGqMxOtpTENna9UfU1YYvFo337h4UsIuXkcoL6f/zkpRQ\n\thbA0SZ0hnb2FbuQJ8/rUJ0jp2q1Ge00yIVGmM734hog1MKYl5LBLMVuhL/8PeP/bOv7N\n\tMZcomre7AuQxm6CLitK+iJYBTFURM0+nJtYIPntGPJuPzWiNDdYyx0g8A+W1tQ3KmfDl\n\t1qfzX2XyQAj5BavdjNHoSJV8vo5EkUCeY0fstMr3cgvfHsoxoedou8+yEW+PMFRb++wn\n\tmy8F4ZLI+c0tWqhfa+WKEbiKhnGanbEK9NFF3PbgWtWigW9WEYeL6PBKAkoxMfwhqpaW\n\ts3uQ==","X-Gm-Message-State":"AOJu0YzgukES4V+a1Wev7hzAbCl3xab0lo/b/VWnqihriSW1yIukV/10\n\tbr2+lkzivSWjTi6KUN0Jz8oN6O6pp5oV3t6M8ZMALM3gWvNbU4KTsWNaJe0TwsfkqR2dHdBVwzV\n\tanlVtUy58foUWYLuHSYijUyi9LbGTys70vfIlhelbY27kAgxVOPL1YQI=","X-Gm-Gg":"AY/fxX7A2a6vlES1JWpGhQNQVgJRp6LITjhryyXykqbcYdK21lOHa8YUK/irqEKsy9/\n\ttpyBM5pGlp+a/RN/qzsiimGwSJ90eqjjG8EQxnVvS0mDH9XN5G7xHY1/gvnwj1Fa60gTtYGxsAC\n\tkN2Vd/nyDfoNwxF0FEk2f/1M5LJZPO1rO9/d2gT8/3pLFassOoYN1v8Wrc2T71qFNAfcQIfQxP+\n\tXvGxVezOsbV3r0AR7gYT/tOUi4vzMtrE1dpJL85A4Jg9HNJFLyf3nBwyqhFC8PXO8+WC1QCdfiC\n\tveRLrSDNio2mt33khkzaO2L1Sc8=","X-Google-Smtp-Source":"AGHT+IEYT9apy9yMCADa40+i15gisZ+nAxayZZplbbT1Wx6omCHhFmjnrBoza+gmnbe0ENgG4bIDvra4FCyD7CHWMVI=","X-Received":"by 2002:a05:6102:5124:b0:5db:25d3:28b4 with SMTP id\n\tada2fe7eead31-5e571f192cbmr466565137.5.1765375769798; Wed, 10 Dec 2025\n\t06:09:29 -0800 (PST)","MIME-Version":"1.0","References":"<20251024144049.3311-1-david.plowman@raspberrypi.com>\n\t<20251024144049.3311-3-david.plowman@raspberrypi.com>","In-Reply-To":"<20251024144049.3311-3-david.plowman@raspberrypi.com>","From":"Naushir Patuck ","Date":"Wed, 10 Dec 2025 14:08:53 +0000","X-Gm-Features":"AQt7F2o_sDp373KLSE3m1CnS42KSvdrfQdkt7lA5-T_D7pau1YfXKX8b4x62AW8","Message-ID":"","Subject":"Re: [PATCH 2/4] ipa: rpi: controller: awb: Add Neural Network Awb","To":"David Plowman ","Cc":"libcamera-devel@lists.libcamera.org, \n\tPeter Bailey ","Content-Type":"text/plain; charset=\"UTF-8\"","X-BeenThere":"libcamera-devel@lists.libcamera.org","X-Mailman-Version":"2.1.29","Precedence":"list","List-Id":"","List-Unsubscribe":",\n\t","List-Archive":"","List-Post":"","List-Help":"","List-Subscribe":",\n\t","Errors-To":"libcamera-devel-bounces@lists.libcamera.org","Sender":"\"libcamera-devel\" "}},{"id":37287,"web_url":"https://patchwork.libcamera.org/comment/37287/","msgid":"<20251211082111.GB28411@pendragon.ideasonboard.com>","date":"2025-12-11T08:21:11","subject":"Re: [PATCH 2/4] ipa: rpi: controller: awb: Add Neural Network Awb","submitter":{"id":2,"url":"https://patchwork.libcamera.org/api/people/2/","name":"Laurent Pinchart","email":"laurent.pinchart@ideasonboard.com"},"content":"Hi David,\n\nThank you for the patch.\n\nOn Fri, Oct 24, 2025 at 03:16:01PM +0100, David Plowman wrote:\n> From: Peter Bailey \n> \n> Add an Awb algorithm which uses neural networks.\n> \n> Signed-off-by: Peter Bailey \n> ---\n> src/ipa/rpi/controller/meson.build | 9 +\n> src/ipa/rpi/controller/rpi/awb_nn.cpp | 442 ++++++++++++++++++++++++++\n> 2 files changed, 451 insertions(+)\n> create mode 100644 src/ipa/rpi/controller/rpi/awb_nn.cpp\n> \n> diff --git a/src/ipa/rpi/controller/meson.build b/src/ipa/rpi/controller/meson.build\n> index 73c93dca..2541d073 100644\n> --- a/src/ipa/rpi/controller/meson.build\n> +++ b/src/ipa/rpi/controller/meson.build\n> @@ -32,6 +32,15 @@ rpi_ipa_controller_deps = [\n> libcamera_private,\n> ]\n> \n> +tflite_dep = dependency('tensorflow-lite', required : false)\n\nThis needs a configuration option, otherwise libcamera may accidentally\ndepend on tensorflow-lite when not desired. Package maintainers for\ndistributions hate those accidental dependencies that make builds\nnon-reproducible.\n\n> +\n> +if tflite_dep.found()\n> + rpi_ipa_controller_sources += files([\n> + 'rpi/awb_nn.cpp',\n> + ])\n> + rpi_ipa_controller_deps += tflite_dep\n> +endif\n> +\n> rpi_ipa_controller_lib = static_library('rpi_ipa_controller', rpi_ipa_controller_sources,\n> include_directories : libipa_includes,\n> dependencies : rpi_ipa_controller_deps)\n> diff --git a/src/ipa/rpi/controller/rpi/awb_nn.cpp b/src/ipa/rpi/controller/rpi/awb_nn.cpp\n> new file mode 100644\n> index 00000000..c309ca3f\n> --- /dev/null\n> +++ b/src/ipa/rpi/controller/rpi/awb_nn.cpp\n> @@ -0,0 +1,442 @@\n> +/* SPDX-License-Identifier: BSD-2-Clause */\n> +/*\n> + * Copyright (C) 2025, Raspberry Pi Ltd\n> + *\n> + * AWB control algorithm using neural network\n> + */\n> +\n> +#include \n> +#include \n> +#include \n> +\n> +#include \n> +#include \n> +\n> +#include \n> +#include \n> +#include \n> +\n> +#include \"../awb_algorithm.h\"\n> +#include \"../awb_status.h\"\n> +#include \"../lux_status.h\"\n> +#include \"libipa/pwl.h\"\n> +\n> +#include \"alsc_status.h\"\n> +#include \"awb.h\"\n> +\n> +using namespace libcamera;\n> +\n> +LOG_DECLARE_CATEGORY(RPiAwb)\n> +\n> +constexpr double kDefaultCT = 4500.0;\n> +\n> +#define NAME \"rpi.nn.awb\"\n> +\n> +namespace RPiController {\n> +\n> +struct AwbNNConfig {\n> +\tAwbNNConfig() {}\n> +\tint read(const libcamera::YamlObject ¶ms, AwbConfig &config);\n> +\n> +\t/* An empty model will check default locations for model.tflite */\n> +\tstd::string model;\n> +\tfloat minTemp;\n> +\tfloat maxTemp;\n> +\n> +\tbool enableNn;\n> +\n> +\t/* CCM matrix for 5000K temperature */\n> +\tdouble ccm[9];\n> +};\n> +\n> +class AwbNN : public Awb\n> +{\n> +public:\n> +\tAwbNN(Controller *controller = NULL);\n> +\t~AwbNN();\n> +\tchar const *name() const override;\n> +\tvoid initialise() override;\n> +\tint read(const libcamera::YamlObject ¶ms) override;\n> +\n> +protected:\n> +\tvoid doAwb() override;\n> +\tvoid prepareStats() override;\n> +\n> +private:\n> +\tbool isAutoEnabled() const;\n> +\tAwbNNConfig nnConfig_;\n> +\tvoid transverseSearch(double t, double &r, double &b);\n> +\tRGB processZone(RGB zone, float red_gain, float blue_gain);\n> +\tvoid awbNN();\n> +\tvoid loadModel();\n> +\n> +\tlibcamera::Size zoneSize_;\n> +\tstd::unique_ptr model_;\n> +\tstd::unique_ptr interpreter_;\n> +};\n> +\n> +int AwbNNConfig::read(const libcamera::YamlObject ¶ms, AwbConfig &config)\n> +{\n> +\tmodel = params[\"model\"].get(\"\");\n> +\tminTemp = params[\"min_temp\"].get(2800.0);\n> +\tmaxTemp = params[\"max_temp\"].get(7600.0);\n> +\n> +\tfor (int i = 0; i < 9; i++)\n> +\t\tccm[i] = params[\"ccm\"][i].get(0.0);\n> +\n> +\tenableNn = params[\"enable_nn\"].get(1);\n> +\n> +\tif (enableNn) {\n> +\t\tif (!config.hasCtCurve()) {\n> +\t\t\tLOG(RPiAwb, Error) << \"CT curve not specified\";\n> +\t\t\tenableNn = false;\n> +\t\t}\n> +\n> +\t\tif (!model.empty() && model.find(\".tflite\") == std::string::npos) {\n> +\t\t\tLOG(RPiAwb, Error) << \"Model must be a .tflite file\";\n> +\t\t\tenableNn = false;\n> +\t\t}\n> +\n> +\t\tbool validCcm = true;\n> +\t\tfor (int i = 0; i < 9; i++)\n> +\t\t\tif (ccm[i] == 0.0)\n> +\t\t\t\tvalidCcm = false;\n> +\n> +\t\tif (!validCcm) {\n> +\t\t\tLOG(RPiAwb, Error) << \"CCM not specified or invalid\";\n> +\t\t\tenableNn = false;\n> +\t\t}\n> +\n> +\t\tif (!enableNn) {\n> +\t\t\tLOG(RPiAwb, Warning) << \"Neural Network AWB mis-configured - switch to Grey method\";\n> +\t\t}\n> +\t}\n> +\n> +\tif (!enableNn) {\n> +\t\tconfig.sensitivityR = config.sensitivityB = 1.0;\n> +\t\tconfig.greyWorld = true;\n> +\t}\n> +\n> +\treturn 0;\n> +}\n> +\n> +AwbNN::AwbNN(Controller *controller)\n> +\t: Awb(controller)\n> +{\n> +\tzoneSize_ = getHardwareConfig().awbRegions;\n> +}\n> +\n> +AwbNN::~AwbNN()\n> +{\n> +}\n> +\n> +char const *AwbNN::name() const\n> +{\n> +\treturn NAME;\n> +}\n> +\n> +int AwbNN::read(const libcamera::YamlObject ¶ms)\n> +{\n> +\tint ret;\n> +\n> +\tret = config_.read(params);\n> +\tif (ret)\n> +\t\treturn ret;\n> +\n> +\tret = nnConfig_.read(params, config_);\n> +\tif (ret)\n> +\t\treturn ret;\n> +\n> +\treturn 0;\n> +}\n> +\n> +static bool checkTensorShape(TfLiteTensor *tensor, const int *expectedDims, const int expectedDimsSize)\n> +{\n> +\tif (tensor->dims->size != expectedDimsSize) {\n> +\t\treturn false;\n> +\t}\n> +\n> +\tfor (int i = 0; i < tensor->dims->size; i++) {\n> +\t\tif (tensor->dims->data[i] != expectedDims[i]) {\n> +\t\t\treturn false;\n> +\t\t}\n> +\t}\n> +\treturn true;\n> +}\n> +\n> +static std::string buildDimString(const int *dims, const int dimsSize)\n> +{\n> +\tstd::string s = \"[\";\n> +\tfor (int i = 0; i < dimsSize; i++) {\n> +\t\ts += std::to_string(dims[i]);\n> +\t\tif (i < dimsSize - 1)\n> +\t\t\ts += \",\";\n> +\t\telse\n> +\t\t\ts += \"]\";\n> +\t}\n> +\treturn s;\n> +}\n> +\n> +void AwbNN::loadModel()\n> +{\n> +\tstd::string modelPath;\n> +\tif (getTarget() == \"bcm2835\") {\n> +\t\tmodelPath = \"/ipa/rpi/vc4/awb_model.tflite\";\n> +\t} else {\n> +\t\tmodelPath = \"/ipa/rpi/pisp/awb_model.tflite\";\n> +\t}\n> +\n> +\tif (nnConfig_.model.empty()) {\n> +\t\tstd::string root = utils::libcameraSourcePath();\n> +\t\tif (!root.empty()) {\n> +\t\t\tmodelPath = root + modelPath;\n> +\t\t} else {\n> +\t\t\tmodelPath = LIBCAMERA_DATA_DIR + modelPath;\n> +\t\t}\n> +\n> +\t\tif (!File::exists(modelPath)) {\n> +\t\t\tLOG(RPiAwb, Error) << \"No model file found in standard locations\";\n> +\t\t\tnnConfig_.enableNn = false;\n> +\t\t\treturn;\n> +\t\t}\n> +\t} else {\n> +\t\tmodelPath = nnConfig_.model;\n> +\t}\n> +\n> +\tLOG(RPiAwb, Debug) << \"Attempting to load model from: \" << modelPath;\n> +\n> +\tmodel_ = tflite::FlatBufferModel::BuildFromFile(modelPath.c_str());\n> +\n> +\tif (!model_) {\n> +\t\tLOG(RPiAwb, Error) << \"Failed to load model from \" << modelPath;\n> +\t\tnnConfig_.enableNn = false;\n> +\t\treturn;\n> +\t}\n> +\n> +\ttflite::MutableOpResolver resolver;\n> +\ttflite::ops::builtin::BuiltinOpResolver builtin_resolver;\n> +\tresolver.AddAll(builtin_resolver);\n> +\ttflite::InterpreterBuilder(*model_, resolver)(&interpreter_);\n> +\tif (!interpreter_) {\n> +\t\tLOG(RPiAwb, Error) << \"Failed to build interpreter for model \" << nnConfig_.model;\n> +\t\tnnConfig_.enableNn = false;\n> +\t\treturn;\n> +\t}\n> +\n> +\tinterpreter_->AllocateTensors();\n> +\tTfLiteTensor *inputTensor = interpreter_->input_tensor(0);\n> +\tTfLiteTensor *inputLuxTensor = interpreter_->input_tensor(1);\n> +\tTfLiteTensor *outputTensor = interpreter_->output_tensor(0);\n> +\tif (!inputTensor || !inputLuxTensor || !outputTensor) {\n> +\t\tLOG(RPiAwb, Error) << \"Model missing input or output tensor\";\n> +\t\tnnConfig_.enableNn = false;\n> +\t\treturn;\n> +\t}\n> +\n> +\tconst int expectedInputDims[] = { 1, (int)zoneSize_.height, (int)zoneSize_.width, 3 };\n> +\tconst int expectedInputLuxDims[] = { 1 };\n> +\tconst int expectedOutputDims[] = { 1 };\n> +\n> +\tif (!checkTensorShape(inputTensor, expectedInputDims, 4)) {\n> +\t\tLOG(RPiAwb, Error) << \"Model input tensor dimension mismatch. Expected: \" << buildDimString(expectedInputDims, 4)\n> +\t\t\t\t << \", Got: \" << buildDimString(inputTensor->dims->data, inputTensor->dims->size);\n> +\t\tnnConfig_.enableNn = false;\n> +\t\treturn;\n> +\t}\n> +\n> +\tif (!checkTensorShape(inputLuxTensor, expectedInputLuxDims, 1)) {\n> +\t\tLOG(RPiAwb, Error) << \"Model input lux tensor dimension mismatch. Expected: \" << buildDimString(expectedInputLuxDims, 1)\n> +\t\t\t\t << \", Got: \" << buildDimString(inputLuxTensor->dims->data, inputLuxTensor->dims->size);\n> +\t\tnnConfig_.enableNn = false;\n> +\t\treturn;\n> +\t}\n> +\n> +\tif (!checkTensorShape(outputTensor, expectedOutputDims, 1)) {\n> +\t\tLOG(RPiAwb, Error) << \"Model output tensor dimension mismatch. Expected: \" << buildDimString(expectedOutputDims, 1)\n> +\t\t\t\t << \", Got: \" << buildDimString(outputTensor->dims->data, outputTensor->dims->size);\n> +\t\tnnConfig_.enableNn = false;\n> +\t\treturn;\n> +\t}\n> +\n> +\tif (inputTensor->type != kTfLiteFloat32 || inputLuxTensor->type != kTfLiteFloat32 || outputTensor->type != kTfLiteFloat32) {\n> +\t\tLOG(RPiAwb, Error) << \"Model input and output tensors must be float32\";\n> +\t\tnnConfig_.enableNn = false;\n> +\t\treturn;\n> +\t}\n> +\n> +\tLOG(RPiAwb, Info) << \"Model loaded successfully from \" << modelPath;\n> +\tLOG(RPiAwb, Debug) << \"Model validation successful - Input Image: \"\n> +\t\t\t << buildDimString(expectedInputDims, 4)\n> +\t\t\t << \", Input Lux: \" << buildDimString(expectedInputLuxDims, 1)\n> +\t\t\t << \", Output: \" << buildDimString(expectedOutputDims, 1) << \" floats\";\n> +}\n> +\n> +void AwbNN::initialise()\n> +{\n> +\tAwb::initialise();\n> +\n> +\tif (nnConfig_.enableNn) {\n> +\t\tloadModel();\n> +\t\tif (!nnConfig_.enableNn) {\n> +\t\t\tLOG(RPiAwb, Warning) << \"Neural Network AWB failed to load - switch to Grey method\";\n> +\t\t\tconfig_.greyWorld = true;\n> +\t\t\tconfig_.sensitivityR = config_.sensitivityB = 1.0;\n> +\t\t}\n> +\t}\n> +}\n> +\n> +void AwbNN::prepareStats()\n> +{\n> +\tzones_.clear();\n> +\t/*\n> +\t * LSC has already been applied to the stats in this pipeline, so stop\n> +\t * any LSC compensation. We also ignore config_.fast in this version.\n> +\t */\n> +\tgenerateStats(zones_, statistics_, 0.0, 0.0, getGlobalMetadata(), 0.0, 0.0, 0.0);\n> +\t/*\n> +\t * apply sensitivities, so values appear to come from our \"canonical\"\n> +\t * sensor.\n> +\t */\n> +\tfor (auto &zone : zones_) {\n> +\t\tzone.R *= config_.sensitivityR;\n> +\t\tzone.B *= config_.sensitivityB;\n> +\t}\n> +}\n> +\n> +void AwbNN::transverseSearch(double t, double &r, double &b)\n> +{\n> +\tint spanR = -1, spanB = -1;\n> +\tconfig_.ctR.eval(t, &spanR);\n> +\tconfig_.ctB.eval(t, &spanB);\n> +\n> +\tconst int diff = 10;\n> +\tdouble rDiff = config_.ctR.eval(t + diff, &spanR) -\n> +\t\t config_.ctR.eval(t - diff, &spanR);\n> +\tdouble bDiff = config_.ctB.eval(t + diff, &spanB) -\n> +\t\t config_.ctB.eval(t - diff, &spanB);\n> +\n> +\tipa::Pwl::Point transverse({ bDiff, -rDiff });\n> +\tif (transverse.length2() < 1e-6)\n> +\t\treturn;\n> +\n> +\ttransverse = transverse / transverse.length();\n> +\tdouble transverseRange = config_.transverseNeg + config_.transversePos;\n> +\tconst int maxNumDeltas = 12;\n> +\tint numDeltas = floor(transverseRange * 100 + 0.5) + 1;\n> +\tnumDeltas = numDeltas < 3 ? 3 : (numDeltas > maxNumDeltas ? maxNumDeltas : numDeltas);\n> +\n> +\tipa::Pwl::Point points[maxNumDeltas];\n> +\tint bestPoint = 0;\n> +\n> +\tfor (int i = 0; i < numDeltas; i++) {\n> +\t\tpoints[i][0] = -config_.transverseNeg +\n> +\t\t\t (transverseRange * i) / (numDeltas - 1);\n> +\t\tipa::Pwl::Point rbTest = ipa::Pwl::Point({ r, b }) +\n> +\t\t\t\t\t transverse * points[i].x();\n> +\t\tdouble rTest = rbTest.x(), bTest = rbTest.y();\n> +\t\tdouble gainR = 1 / rTest, gainB = 1 / bTest;\n> +\t\tdouble delta2Sum = computeDelta2Sum(gainR, gainB, 0.0, 0.0);\n> +\t\tpoints[i][1] = delta2Sum;\n> +\t\tif (points[i].y() < points[bestPoint].y())\n> +\t\t\tbestPoint = i;\n> +\t}\n> +\n> +\tbestPoint = std::clamp(bestPoint, 1, numDeltas - 2);\n> +\tipa::Pwl::Point rbBest = ipa::Pwl::Point({ r, b }) +\n> +\t\t\t\t transverse * interpolateQuadatric(points[bestPoint - 1],\n> +\t\t\t\t\t\t\t\t points[bestPoint],\n> +\t\t\t\t\t\t\t\t points[bestPoint + 1]);\n> +\tdouble rBest = rbBest.x(), bBest = rbBest.y();\n> +\n> +\tr = rBest, b = bBest;\n> +}\n> +\n> +AwbNN::RGB AwbNN::processZone(AwbNN::RGB zone, float redGain, float blueGain)\n> +{\n> +\t/*\n> +\t * Renders the pixel at 5000K temperature\n> +\t */\n> +\tRGB zoneGains = zone;\n> +\n> +\tzoneGains.R *= redGain;\n> +\tzoneGains.G *= 1.0;\n> +\tzoneGains.B *= blueGain;\n> +\n> +\tRGB zoneCcm;\n> +\n> +\tzoneCcm.R = nnConfig_.ccm[0] * zoneGains.R + nnConfig_.ccm[1] * zoneGains.G + nnConfig_.ccm[2] * zoneGains.B;\n> +\tzoneCcm.G = nnConfig_.ccm[3] * zoneGains.R + nnConfig_.ccm[4] * zoneGains.G + nnConfig_.ccm[5] * zoneGains.B;\n> +\tzoneCcm.B = nnConfig_.ccm[6] * zoneGains.R + nnConfig_.ccm[7] * zoneGains.G + nnConfig_.ccm[8] * zoneGains.B;\n> +\n> +\treturn zoneCcm;\n> +}\n> +\n> +void AwbNN::awbNN()\n> +{\n> +\tfloat *inputData = interpreter_->typed_input_tensor(0);\n> +\tfloat *inputLux = interpreter_->typed_input_tensor(1);\n> +\n> +\tfloat redGain = 1.0 / config_.ctR.eval(5000);\n> +\tfloat blueGain = 1.0 / config_.ctB.eval(5000);\n> +\n> +\tfor (uint i = 0; i < zoneSize_.height; i++) {\n> +\t\tfor (uint j = 0; j < zoneSize_.width; j++) {\n> +\t\t\tuint zoneIdx = i * zoneSize_.width + j;\n> +\n> +\t\t\tRGB processedZone = processZone(zones_[zoneIdx] * (1.0 / 65535), redGain, blueGain);\n> +\t\t\tuint baseIdx = zoneIdx * 3;\n> +\n> +\t\t\tinputData[baseIdx + 0] = static_cast(processedZone.R);\n> +\t\t\tinputData[baseIdx + 1] = static_cast(processedZone.G);\n> +\t\t\tinputData[baseIdx + 2] = static_cast(processedZone.B);\n> +\t\t}\n> +\t}\n> +\n> +\tinputLux[0] = static_cast(lux_);\n> +\n> +\tTfLiteStatus status = interpreter_->Invoke();\n\nHow long does this typically take ?\n\n> +\tif (status != kTfLiteOk) {\n> +\t\tLOG(RPiAwb, Error) << \"Model inference failed with status: \" << status;\n> +\t\treturn;\n> +\t}\n> +\n> +\tfloat *outputData = interpreter_->typed_output_tensor(0);\n> +\n> +\tdouble t = outputData[0];\n> +\n> +\tLOG(RPiAwb, Debug) << \"Model output temperature: \" << t;\n> +\n> +\tt = std::clamp(t, mode_->ctLo, mode_->ctHi);\n> +\n> +\tdouble r = config_.ctR.eval(t);\n> +\tdouble b = config_.ctB.eval(t);\n> +\n> +\ttransverseSearch(t, r, b);\n> +\n> +\tLOG(RPiAwb, Debug) << \"After transverse search: Temperature: \" << t << \" Red gain: \" << 1.0 / r << \" Blue gain: \" << 1.0 / b;\n> +\n> +\tasyncResults_.temperatureK = t;\n> +\tasyncResults_.gainR = 1.0 / r * config_.sensitivityR;\n> +\tasyncResults_.gainG = 1.0;\n> +\tasyncResults_.gainB = 1.0 / b * config_.sensitivityB;\n> +}\n> +\n> +void AwbNN::doAwb()\n> +{\n> +\tprepareStats();\n> +\tif (zones_.size() == (zoneSize_.width * zoneSize_.height) && nnConfig_.enableNn) {\n> +\t\tawbNN();\n> +\t} else {\n> +\t\tawbGrey();\n> +\t}\n> +\tstatistics_.reset();\n> +}\n> +\n> +/* Register algorithm with the system. */\n> +static Algorithm *create(Controller *controller)\n> +{\n> +\treturn (Algorithm *)new AwbNN(controller);\n> +}\n> +static RegisterAlgorithm reg(NAME, &create);\n> +\n> +} /* namespace RPiController */","headers":{"Return-Path":"","X-Original-To":"parsemail@patchwork.libcamera.org","Delivered-To":"parsemail@patchwork.libcamera.org","Received":["from lancelot.ideasonboard.com (lancelot.ideasonboard.com\n\t[92.243.16.209])\n\tby patchwork.libcamera.org (Postfix) with ESMTPS id 36039C3257\n\tfor ;\n\tThu, 11 Dec 2025 08:21:33 +0000 (UTC)","from lancelot.ideasonboard.com (localhost [IPv6:::1])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTP id 1F11861592;\n\tThu, 11 Dec 2025 09:21:32 +0100 (CET)","from perceval.ideasonboard.com (perceval.ideasonboard.com\n\t[213.167.242.64])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id 48273610A6\n\tfor ;\n\tThu, 11 Dec 2025 09:21:30 +0100 (CET)","from pendragon.ideasonboard.com (fs96f9c361.tkyc007.ap.nuro.jp\n\t[150.249.195.97])\n\tby perceval.ideasonboard.com (Postfix) with UTF8SMTPSA id 32E311352; \n\tThu, 11 Dec 2025 09:21:26 +0100 (CET)"],"Authentication-Results":"lancelot.ideasonboard.com; dkim=pass (1024-bit key;\n\tunprotected) header.d=ideasonboard.com header.i=@ideasonboard.com\n\theader.b=\"Mqpcw8Y/\"; dkim-atps=neutral","DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/simple; d=ideasonboard.com;\n\ts=mail; t=1765441288;\n\tbh=0O/5oQ+i5D8AzAu2nGjXb4aqjQgf6/wYFiBa25onbcg=;\n\th=Date:From:To:Cc:Subject:References:In-Reply-To:From;\n\tb=Mqpcw8Y/qmZ7Oq7Hgn9R4XS3/ktmqgw201kCSR69TuS2fkfS024eQgi0wyVZLDViN\n\tdcsWQijVuvgn+B66lIbTSkRlXDWL7muKk8YFd1CCJSxTepdnM/IrbAeDul4gLqcB2o\n\tKVvBQYtjF49fO0ijwE768vZJjGC3lr/BICA4rqh0=","Date":"Thu, 11 Dec 2025 17:21:11 +0900","From":"Laurent Pinchart ","To":"David Plowman ","Cc":"libcamera-devel@lists.libcamera.org,\n\tPeter Bailey ","Subject":"Re: [PATCH 2/4] ipa: rpi: controller: awb: Add Neural Network Awb","Message-ID":"<20251211082111.GB28411@pendragon.ideasonboard.com>","References":"<20251024144049.3311-1-david.plowman@raspberrypi.com>\n\t<20251024144049.3311-3-david.plowman@raspberrypi.com>","MIME-Version":"1.0","Content-Type":"text/plain; charset=utf-8","Content-Disposition":"inline","In-Reply-To":"<20251024144049.3311-3-david.plowman@raspberrypi.com>","X-BeenThere":"libcamera-devel@lists.libcamera.org","X-Mailman-Version":"2.1.29","Precedence":"list","List-Id":"","List-Unsubscribe":",\n\t","List-Archive":"","List-Post":"","List-Help":"","List-Subscribe":",\n\t","Errors-To":"libcamera-devel-bounces@lists.libcamera.org","Sender":"\"libcamera-devel\" "}},{"id":37316,"web_url":"https://patchwork.libcamera.org/comment/37316/","msgid":"","date":"2025-12-11T15:56:27","subject":"Re: [PATCH 2/4] ipa: rpi: controller: awb: Add Neural Network Awb","submitter":{"id":42,"url":"https://patchwork.libcamera.org/api/people/42/","name":"David Plowman","email":"david.plowman@raspberrypi.com"},"content":"Hi Laruent\n\nThanks for the comments.\n\nOn Thu, 11 Dec 2025 at 08:21, Laurent Pinchart\n wrote:\n>\n> Hi David,\n>\n> Thank you for the patch.\n>\n> On Fri, Oct 24, 2025 at 03:16:01PM +0100, David Plowman wrote:\n> > From: Peter Bailey \n> >\n> > Add an Awb algorithm which uses neural networks.\n> >\n> > Signed-off-by: Peter Bailey \n> > ---\n> > src/ipa/rpi/controller/meson.build | 9 +\n> > src/ipa/rpi/controller/rpi/awb_nn.cpp | 442 ++++++++++++++++++++++++++\n> > 2 files changed, 451 insertions(+)\n> > create mode 100644 src/ipa/rpi/controller/rpi/awb_nn.cpp\n> >\n> > diff --git a/src/ipa/rpi/controller/meson.build b/src/ipa/rpi/controller/meson.build\n> > index 73c93dca..2541d073 100644\n> > --- a/src/ipa/rpi/controller/meson.build\n> > +++ b/src/ipa/rpi/controller/meson.build\n> > @@ -32,6 +32,15 @@ rpi_ipa_controller_deps = [\n> > libcamera_private,\n> > ]\n> >\n> > +tflite_dep = dependency('tensorflow-lite', required : false)\n>\n> This needs a configuration option, otherwise libcamera may accidentally\n> depend on tensorflow-lite when not desired. Package maintainers for\n> distributions hate those accidental dependencies that make builds\n> non-reproducible.\n\nWill do. Thought I'd done it for v2 but failed to save the file! Will be in v3.\n\n>\n> > +\n> > +if tflite_dep.found()\n> > + rpi_ipa_controller_sources += files([\n> > + 'rpi/awb_nn.cpp',\n> > + ])\n> > + rpi_ipa_controller_deps += tflite_dep\n> > +endif\n> > +\n> > rpi_ipa_controller_lib = static_library('rpi_ipa_controller', rpi_ipa_controller_sources,\n> > include_directories : libipa_includes,\n> > dependencies : rpi_ipa_controller_deps)\n> > diff --git a/src/ipa/rpi/controller/rpi/awb_nn.cpp b/src/ipa/rpi/controller/rpi/awb_nn.cpp\n> > new file mode 100644\n> > index 00000000..c309ca3f\n> > --- /dev/null\n> > +++ b/src/ipa/rpi/controller/rpi/awb_nn.cpp\n> > @@ -0,0 +1,442 @@\n> > +/* SPDX-License-Identifier: BSD-2-Clause */\n> > +/*\n> > + * Copyright (C) 2025, Raspberry Pi Ltd\n> > + *\n> > + * AWB control algorithm using neural network\n> > + */\n> > +\n> > +#include \n> > +#include \n> > +#include \n> > +\n> > +#include \n> > +#include \n> > +\n> > +#include \n> > +#include \n> > +#include \n> > +\n> > +#include \"../awb_algorithm.h\"\n> > +#include \"../awb_status.h\"\n> > +#include \"../lux_status.h\"\n> > +#include \"libipa/pwl.h\"\n> > +\n> > +#include \"alsc_status.h\"\n> > +#include \"awb.h\"\n> > +\n> > +using namespace libcamera;\n> > +\n> > +LOG_DECLARE_CATEGORY(RPiAwb)\n> > +\n> > +constexpr double kDefaultCT = 4500.0;\n> > +\n> > +#define NAME \"rpi.nn.awb\"\n> > +\n> > +namespace RPiController {\n> > +\n> > +struct AwbNNConfig {\n> > + AwbNNConfig() {}\n> > + int read(const libcamera::YamlObject ¶ms, AwbConfig &config);\n> > +\n> > + /* An empty model will check default locations for model.tflite */\n> > + std::string model;\n> > + float minTemp;\n> > + float maxTemp;\n> > +\n> > + bool enableNn;\n> > +\n> > + /* CCM matrix for 5000K temperature */\n> > + double ccm[9];\n> > +};\n> > +\n> > +class AwbNN : public Awb\n> > +{\n> > +public:\n> > + AwbNN(Controller *controller = NULL);\n> > + ~AwbNN();\n> > + char const *name() const override;\n> > + void initialise() override;\n> > + int read(const libcamera::YamlObject ¶ms) override;\n> > +\n> > +protected:\n> > + void doAwb() override;\n> > + void prepareStats() override;\n> > +\n> > +private:\n> > + bool isAutoEnabled() const;\n> > + AwbNNConfig nnConfig_;\n> > + void transverseSearch(double t, double &r, double &b);\n> > + RGB processZone(RGB zone, float red_gain, float blue_gain);\n> > + void awbNN();\n> > + void loadModel();\n> > +\n> > + libcamera::Size zoneSize_;\n> > + std::unique_ptr model_;\n> > + std::unique_ptr interpreter_;\n> > +};\n> > +\n> > +int AwbNNConfig::read(const libcamera::YamlObject ¶ms, AwbConfig &config)\n> > +{\n> > + model = params[\"model\"].get(\"\");\n> > + minTemp = params[\"min_temp\"].get(2800.0);\n> > + maxTemp = params[\"max_temp\"].get(7600.0);\n> > +\n> > + for (int i = 0; i < 9; i++)\n> > + ccm[i] = params[\"ccm\"][i].get(0.0);\n> > +\n> > + enableNn = params[\"enable_nn\"].get(1);\n> > +\n> > + if (enableNn) {\n> > + if (!config.hasCtCurve()) {\n> > + LOG(RPiAwb, Error) << \"CT curve not specified\";\n> > + enableNn = false;\n> > + }\n> > +\n> > + if (!model.empty() && model.find(\".tflite\") == std::string::npos) {\n> > + LOG(RPiAwb, Error) << \"Model must be a .tflite file\";\n> > + enableNn = false;\n> > + }\n> > +\n> > + bool validCcm = true;\n> > + for (int i = 0; i < 9; i++)\n> > + if (ccm[i] == 0.0)\n> > + validCcm = false;\n> > +\n> > + if (!validCcm) {\n> > + LOG(RPiAwb, Error) << \"CCM not specified or invalid\";\n> > + enableNn = false;\n> > + }\n> > +\n> > + if (!enableNn) {\n> > + LOG(RPiAwb, Warning) << \"Neural Network AWB mis-configured - switch to Grey method\";\n> > + }\n> > + }\n> > +\n> > + if (!enableNn) {\n> > + config.sensitivityR = config.sensitivityB = 1.0;\n> > + config.greyWorld = true;\n> > + }\n> > +\n> > + return 0;\n> > +}\n> > +\n> > +AwbNN::AwbNN(Controller *controller)\n> > + : Awb(controller)\n> > +{\n> > + zoneSize_ = getHardwareConfig().awbRegions;\n> > +}\n> > +\n> > +AwbNN::~AwbNN()\n> > +{\n> > +}\n> > +\n> > +char const *AwbNN::name() const\n> > +{\n> > + return NAME;\n> > +}\n> > +\n> > +int AwbNN::read(const libcamera::YamlObject ¶ms)\n> > +{\n> > + int ret;\n> > +\n> > + ret = config_.read(params);\n> > + if (ret)\n> > + return ret;\n> > +\n> > + ret = nnConfig_.read(params, config_);\n> > + if (ret)\n> > + return ret;\n> > +\n> > + return 0;\n> > +}\n> > +\n> > +static bool checkTensorShape(TfLiteTensor *tensor, const int *expectedDims, const int expectedDimsSize)\n> > +{\n> > + if (tensor->dims->size != expectedDimsSize) {\n> > + return false;\n> > + }\n> > +\n> > + for (int i = 0; i < tensor->dims->size; i++) {\n> > + if (tensor->dims->data[i] != expectedDims[i]) {\n> > + return false;\n> > + }\n> > + }\n> > + return true;\n> > +}\n> > +\n> > +static std::string buildDimString(const int *dims, const int dimsSize)\n> > +{\n> > + std::string s = \"[\";\n> > + for (int i = 0; i < dimsSize; i++) {\n> > + s += std::to_string(dims[i]);\n> > + if (i < dimsSize - 1)\n> > + s += \",\";\n> > + else\n> > + s += \"]\";\n> > + }\n> > + return s;\n> > +}\n> > +\n> > +void AwbNN::loadModel()\n> > +{\n> > + std::string modelPath;\n> > + if (getTarget() == \"bcm2835\") {\n> > + modelPath = \"/ipa/rpi/vc4/awb_model.tflite\";\n> > + } else {\n> > + modelPath = \"/ipa/rpi/pisp/awb_model.tflite\";\n> > + }\n> > +\n> > + if (nnConfig_.model.empty()) {\n> > + std::string root = utils::libcameraSourcePath();\n> > + if (!root.empty()) {\n> > + modelPath = root + modelPath;\n> > + } else {\n> > + modelPath = LIBCAMERA_DATA_DIR + modelPath;\n> > + }\n> > +\n> > + if (!File::exists(modelPath)) {\n> > + LOG(RPiAwb, Error) << \"No model file found in standard locations\";\n> > + nnConfig_.enableNn = false;\n> > + return;\n> > + }\n> > + } else {\n> > + modelPath = nnConfig_.model;\n> > + }\n> > +\n> > + LOG(RPiAwb, Debug) << \"Attempting to load model from: \" << modelPath;\n> > +\n> > + model_ = tflite::FlatBufferModel::BuildFromFile(modelPath.c_str());\n> > +\n> > + if (!model_) {\n> > + LOG(RPiAwb, Error) << \"Failed to load model from \" << modelPath;\n> > + nnConfig_.enableNn = false;\n> > + return;\n> > + }\n> > +\n> > + tflite::MutableOpResolver resolver;\n> > + tflite::ops::builtin::BuiltinOpResolver builtin_resolver;\n> > + resolver.AddAll(builtin_resolver);\n> > + tflite::InterpreterBuilder(*model_, resolver)(&interpreter_);\n> > + if (!interpreter_) {\n> > + LOG(RPiAwb, Error) << \"Failed to build interpreter for model \" << nnConfig_.model;\n> > + nnConfig_.enableNn = false;\n> > + return;\n> > + }\n> > +\n> > + interpreter_->AllocateTensors();\n> > + TfLiteTensor *inputTensor = interpreter_->input_tensor(0);\n> > + TfLiteTensor *inputLuxTensor = interpreter_->input_tensor(1);\n> > + TfLiteTensor *outputTensor = interpreter_->output_tensor(0);\n> > + if (!inputTensor || !inputLuxTensor || !outputTensor) {\n> > + LOG(RPiAwb, Error) << \"Model missing input or output tensor\";\n> > + nnConfig_.enableNn = false;\n> > + return;\n> > + }\n> > +\n> > + const int expectedInputDims[] = { 1, (int)zoneSize_.height, (int)zoneSize_.width, 3 };\n> > + const int expectedInputLuxDims[] = { 1 };\n> > + const int expectedOutputDims[] = { 1 };\n> > +\n> > + if (!checkTensorShape(inputTensor, expectedInputDims, 4)) {\n> > + LOG(RPiAwb, Error) << \"Model input tensor dimension mismatch. Expected: \" << buildDimString(expectedInputDims, 4)\n> > + << \", Got: \" << buildDimString(inputTensor->dims->data, inputTensor->dims->size);\n> > + nnConfig_.enableNn = false;\n> > + return;\n> > + }\n> > +\n> > + if (!checkTensorShape(inputLuxTensor, expectedInputLuxDims, 1)) {\n> > + LOG(RPiAwb, Error) << \"Model input lux tensor dimension mismatch. Expected: \" << buildDimString(expectedInputLuxDims, 1)\n> > + << \", Got: \" << buildDimString(inputLuxTensor->dims->data, inputLuxTensor->dims->size);\n> > + nnConfig_.enableNn = false;\n> > + return;\n> > + }\n> > +\n> > + if (!checkTensorShape(outputTensor, expectedOutputDims, 1)) {\n> > + LOG(RPiAwb, Error) << \"Model output tensor dimension mismatch. Expected: \" << buildDimString(expectedOutputDims, 1)\n> > + << \", Got: \" << buildDimString(outputTensor->dims->data, outputTensor->dims->size);\n> > + nnConfig_.enableNn = false;\n> > + return;\n> > + }\n> > +\n> > + if (inputTensor->type != kTfLiteFloat32 || inputLuxTensor->type != kTfLiteFloat32 || outputTensor->type != kTfLiteFloat32) {\n> > + LOG(RPiAwb, Error) << \"Model input and output tensors must be float32\";\n> > + nnConfig_.enableNn = false;\n> > + return;\n> > + }\n> > +\n> > + LOG(RPiAwb, Info) << \"Model loaded successfully from \" << modelPath;\n> > + LOG(RPiAwb, Debug) << \"Model validation successful - Input Image: \"\n> > + << buildDimString(expectedInputDims, 4)\n> > + << \", Input Lux: \" << buildDimString(expectedInputLuxDims, 1)\n> > + << \", Output: \" << buildDimString(expectedOutputDims, 1) << \" floats\";\n> > +}\n> > +\n> > +void AwbNN::initialise()\n> > +{\n> > + Awb::initialise();\n> > +\n> > + if (nnConfig_.enableNn) {\n> > + loadModel();\n> > + if (!nnConfig_.enableNn) {\n> > + LOG(RPiAwb, Warning) << \"Neural Network AWB failed to load - switch to Grey method\";\n> > + config_.greyWorld = true;\n> > + config_.sensitivityR = config_.sensitivityB = 1.0;\n> > + }\n> > + }\n> > +}\n> > +\n> > +void AwbNN::prepareStats()\n> > +{\n> > + zones_.clear();\n> > + /*\n> > + * LSC has already been applied to the stats in this pipeline, so stop\n> > + * any LSC compensation. We also ignore config_.fast in this version.\n> > + */\n> > + generateStats(zones_, statistics_, 0.0, 0.0, getGlobalMetadata(), 0.0, 0.0, 0.0);\n> > + /*\n> > + * apply sensitivities, so values appear to come from our \"canonical\"\n> > + * sensor.\n> > + */\n> > + for (auto &zone : zones_) {\n> > + zone.R *= config_.sensitivityR;\n> > + zone.B *= config_.sensitivityB;\n> > + }\n> > +}\n> > +\n> > +void AwbNN::transverseSearch(double t, double &r, double &b)\n> > +{\n> > + int spanR = -1, spanB = -1;\n> > + config_.ctR.eval(t, &spanR);\n> > + config_.ctB.eval(t, &spanB);\n> > +\n> > + const int diff = 10;\n> > + double rDiff = config_.ctR.eval(t + diff, &spanR) -\n> > + config_.ctR.eval(t - diff, &spanR);\n> > + double bDiff = config_.ctB.eval(t + diff, &spanB) -\n> > + config_.ctB.eval(t - diff, &spanB);\n> > +\n> > + ipa::Pwl::Point transverse({ bDiff, -rDiff });\n> > + if (transverse.length2() < 1e-6)\n> > + return;\n> > +\n> > + transverse = transverse / transverse.length();\n> > + double transverseRange = config_.transverseNeg + config_.transversePos;\n> > + const int maxNumDeltas = 12;\n> > + int numDeltas = floor(transverseRange * 100 + 0.5) + 1;\n> > + numDeltas = numDeltas < 3 ? 3 : (numDeltas > maxNumDeltas ? maxNumDeltas : numDeltas);\n> > +\n> > + ipa::Pwl::Point points[maxNumDeltas];\n> > + int bestPoint = 0;\n> > +\n> > + for (int i = 0; i < numDeltas; i++) {\n> > + points[i][0] = -config_.transverseNeg +\n> > + (transverseRange * i) / (numDeltas - 1);\n> > + ipa::Pwl::Point rbTest = ipa::Pwl::Point({ r, b }) +\n> > + transverse * points[i].x();\n> > + double rTest = rbTest.x(), bTest = rbTest.y();\n> > + double gainR = 1 / rTest, gainB = 1 / bTest;\n> > + double delta2Sum = computeDelta2Sum(gainR, gainB, 0.0, 0.0);\n> > + points[i][1] = delta2Sum;\n> > + if (points[i].y() < points[bestPoint].y())\n> > + bestPoint = i;\n> > + }\n> > +\n> > + bestPoint = std::clamp(bestPoint, 1, numDeltas - 2);\n> > + ipa::Pwl::Point rbBest = ipa::Pwl::Point({ r, b }) +\n> > + transverse * interpolateQuadatric(points[bestPoint - 1],\n> > + points[bestPoint],\n> > + points[bestPoint + 1]);\n> > + double rBest = rbBest.x(), bBest = rbBest.y();\n> > +\n> > + r = rBest, b = bBest;\n> > +}\n> > +\n> > +AwbNN::RGB AwbNN::processZone(AwbNN::RGB zone, float redGain, float blueGain)\n> > +{\n> > + /*\n> > + * Renders the pixel at 5000K temperature\n> > + */\n> > + RGB zoneGains = zone;\n> > +\n> > + zoneGains.R *= redGain;\n> > + zoneGains.G *= 1.0;\n> > + zoneGains.B *= blueGain;\n> > +\n> > + RGB zoneCcm;\n> > +\n> > + zoneCcm.R = nnConfig_.ccm[0] * zoneGains.R + nnConfig_.ccm[1] * zoneGains.G + nnConfig_.ccm[2] * zoneGains.B;\n> > + zoneCcm.G = nnConfig_.ccm[3] * zoneGains.R + nnConfig_.ccm[4] * zoneGains.G + nnConfig_.ccm[5] * zoneGains.B;\n> > + zoneCcm.B = nnConfig_.ccm[6] * zoneGains.R + nnConfig_.ccm[7] * zoneGains.G + nnConfig_.ccm[8] * zoneGains.B;\n> > +\n> > + return zoneCcm;\n> > +}\n> > +\n> > +void AwbNN::awbNN()\n> > +{\n> > + float *inputData = interpreter_->typed_input_tensor(0);\n> > + float *inputLux = interpreter_->typed_input_tensor(1);\n> > +\n> > + float redGain = 1.0 / config_.ctR.eval(5000);\n> > + float blueGain = 1.0 / config_.ctB.eval(5000);\n> > +\n> > + for (uint i = 0; i < zoneSize_.height; i++) {\n> > + for (uint j = 0; j < zoneSize_.width; j++) {\n> > + uint zoneIdx = i * zoneSize_.width + j;\n> > +\n> > + RGB processedZone = processZone(zones_[zoneIdx] * (1.0 / 65535), redGain, blueGain);\n> > + uint baseIdx = zoneIdx * 3;\n> > +\n> > + inputData[baseIdx + 0] = static_cast(processedZone.R);\n> > + inputData[baseIdx + 1] = static_cast(processedZone.G);\n> > + inputData[baseIdx + 2] = static_cast(processedZone.B);\n> > + }\n> > + }\n> > +\n> > + inputLux[0] = static_cast(lux_);\n> > +\n> > + TfLiteStatus status = interpreter_->Invoke();\n>\n> How long does this typically take ?\n\nBecause they run on the image statistics they're actually tiny models\n(compared to most image processing/analysis models). I forget exactly,\nbut it's down at the millisecond or two level on a Pi 5, so \"a few\ntimes slower\" on other Pis.\n\nThanks\nDavid\n\n>\n> > + if (status != kTfLiteOk) {\n> > + LOG(RPiAwb, Error) << \"Model inference failed with status: \" << status;\n> > + return;\n> > + }\n> > +\n> > + float *outputData = interpreter_->typed_output_tensor(0);\n> > +\n> > + double t = outputData[0];\n> > +\n> > + LOG(RPiAwb, Debug) << \"Model output temperature: \" << t;\n> > +\n> > + t = std::clamp(t, mode_->ctLo, mode_->ctHi);\n> > +\n> > + double r = config_.ctR.eval(t);\n> > + double b = config_.ctB.eval(t);\n> > +\n> > + transverseSearch(t, r, b);\n> > +\n> > + LOG(RPiAwb, Debug) << \"After transverse search: Temperature: \" << t << \" Red gain: \" << 1.0 / r << \" Blue gain: \" << 1.0 / b;\n> > +\n> > + asyncResults_.temperatureK = t;\n> > + asyncResults_.gainR = 1.0 / r * config_.sensitivityR;\n> > + asyncResults_.gainG = 1.0;\n> > + asyncResults_.gainB = 1.0 / b * config_.sensitivityB;\n> > +}\n> > +\n> > +void AwbNN::doAwb()\n> > +{\n> > + prepareStats();\n> > + if (zones_.size() == (zoneSize_.width * zoneSize_.height) && nnConfig_.enableNn) {\n> > + awbNN();\n> > + } else {\n> > + awbGrey();\n> > + }\n> > + statistics_.reset();\n> > +}\n> > +\n> > +/* Register algorithm with the system. */\n> > +static Algorithm *create(Controller *controller)\n> > +{\n> > + return (Algorithm *)new AwbNN(controller);\n> > +}\n> > +static RegisterAlgorithm reg(NAME, &create);\n> > +\n> > +} /* namespace RPiController */\n>\n> --\n> Regards,\n>\n> Laurent Pinchart","headers":{"Return-Path":"","X-Original-To":"parsemail@patchwork.libcamera.org","Delivered-To":"parsemail@patchwork.libcamera.org","Received":["from lancelot.ideasonboard.com (lancelot.ideasonboard.com\n\t[92.243.16.209])\n\tby patchwork.libcamera.org (Postfix) with ESMTPS id 00E1AC3257\n\tfor ;\n\tThu, 11 Dec 2025 15:56:42 +0000 (UTC)","from lancelot.ideasonboard.com (localhost [IPv6:::1])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTP id A9B1661612;\n\tThu, 11 Dec 2025 16:56:42 +0100 (CET)","from mail-qt1-x82d.google.com (mail-qt1-x82d.google.com\n\t[IPv6:2607:f8b0:4864:20::82d])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id E7E4061603\n\tfor ;\n\tThu, 11 Dec 2025 16:56:40 +0100 (CET)","by mail-qt1-x82d.google.com with SMTP id\n\td75a77b69052e-4eda6a8cc12so2340891cf.0\n\tfor ;\n\tThu, 11 Dec 2025 07:56:40 -0800 (PST)"],"Authentication-Results":"lancelot.ideasonboard.com; dkim=pass (2048-bit key;\n\tunprotected) header.d=raspberrypi.com header.i=@raspberrypi.com\n\theader.b=\"kEYaRBYc\"; dkim-atps=neutral","DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/relaxed;\n\td=raspberrypi.com; s=google; t=1765468600; x=1766073400;\n\tdarn=lists.libcamera.org; \n\th=cc:to:subject:message-id:date:from:in-reply-to:references\n\t:mime-version:from:to:cc:subject:date:message-id:reply-to;\n\tbh=vrg4PnRQzKQfJomoPQUT76ApQ7x+6PcCsAZrdnre+9I=;\n\tb=kEYaRBYc2yw8vuoHVKiUwqpFnacIET1TVNLoXjE+3G5Cio02l7V/C6LfjOffp5zfe7\n\ttJlE2/3R49ME4Et8lUhApt4oMVW7fs3z3IoUgipL/0svgKaVcllyNVXjTsCt6ZMcEpZU\n\taxZA/C1gUpgpEMh9otyScxnTvGqebZd71nnfUCY4ucKQz8MXN3xBvG1fCddnvQzDSFjL\n\ttePCvmm4wDnD/KViO18zazfFbDf2nBhTcBaZLWPOfGVTx1MB3PspCCyfgbHzycneBQPy\n\tloF5JzC/EDNtdhgMo3ROm53Z3pQCi1VtgRj55HZqtJRSBaCzc4j8gOhM4T2QZws12EQo\n\tdHPw==","X-Google-DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/relaxed;\n\td=1e100.net; s=20230601; t=1765468600; x=1766073400;\n\th=cc:to:subject:message-id:date:from:in-reply-to:references\n\t:mime-version:x-gm-gg:x-gm-message-state:from:to:cc:subject:date\n\t:message-id:reply-to;\n\tbh=vrg4PnRQzKQfJomoPQUT76ApQ7x+6PcCsAZrdnre+9I=;\n\tb=IJdLQPXBAKeEoIwdQV/wDdBHoCeMP5QxUSYGnQA+OKIquxv7RS7TbI+8EKlN/TYQBJ\n\t1ve+Kjjiq6mecOA817BZfw17C0LM/MqzaULCHE9T1sS/Wd8oGZsJ6hCM93h9xgYR3O6P\n\t6kYtTFMuOy0Bf/j4tWI+W/WwM0t24JaK+ypOql7awHSyU10ZqawYcKicfA39vh+vph+x\n\tCnjo7LYef5buTo62SKGSwYvoeyE/90k8+RoFg3Y90Ws+S8Cbw56gVGDxM6AcxdCsj45G\n\tbLgnH+TpMoCyEo1AGKItvgjP2Ej+EN4GUav9nP0zgg6go0f2GxVuVoMt7pcdlf+5k5XU\n\tFlYA==","X-Gm-Message-State":"AOJu0YxSxEOu1FGcmfm3y7rkfV6LqdnnAFpBbQA3UZ9+mwvIe/ydj833\n\ti4bfR/66jdDle8nNRqtGO9DftQR4APq3MbHtJjJ3IHGk9L44E18leWICzjxSol2D6dJHBYTGl/d\n\tkVJWlwXH9O0fsi0lVatlEWf38Fc3d5Pva/1Rf7p/gmw==","X-Gm-Gg":"AY/fxX4A32XV4mBbvtcYbvXawq9XnFM8VBVgnsdjbO2mevyDv9E9Gd5WHen6Chvu8aR\n\tJ/5EjVLGp1zMbJF1O/lT9c7dObf/BzB+RFPmOwKkVKh7c+mpp3b9YkZpA2hrjXvmpy+SkFS/Kz6\n\t/43BgaOQ7eba7tkOww1XOzB/L+MBuNR8vUKrrq9O74osgyuWoRNNukv+ECHwPG4qwm2kD3eHgs4\n\t9/m3JcpbDw9LxKLluxFyqd0XBUFOUzqhHl1H3zIB2kreHEJObVV4jCXfqgqGTEJSbKTPheZT855\n\tn+dH4UfUho86GKaTRTTNC4S0IeQMQYq+7gO041MQJB9kL6/JuMHw/deAHxt6ai5Ojh13zTClrz2\n\tt8k3BjAFMYpGoUSOGuBPUnIQ=","X-Google-Smtp-Source":"AGHT+IHcn1xVdi3juAwkn1xnRPTcJb5HkI2vlJRB0XmeczPtzeezUdwX0D44tg6g4UulLBdoBe3+Z+8759qgngulfaw=","X-Received":"by 2002:a05:622a:1a98:b0:4ed:6782:12c4 with SMTP id\n\td75a77b69052e-4f1b1a7eb82mr86464311cf.33.1765468599679;\n\tThu, 11 Dec 2025 07:56:39 -0800 (PST)","MIME-Version":"1.0","References":"<20251024144049.3311-1-david.plowman@raspberrypi.com>\n\t<20251024144049.3311-3-david.plowman@raspberrypi.com>\n\t<20251211082111.GB28411@pendragon.ideasonboard.com>","In-Reply-To":"<20251211082111.GB28411@pendragon.ideasonboard.com>","From":"David Plowman ","Date":"Thu, 11 Dec 2025 15:56:27 +0000","X-Gm-Features":"AQt7F2qKBBgEt9qw5bfYEP5z937vPj5_Tei6M4867pBqlow2qeAzcRjN3df9LIk","Message-ID":"","Subject":"Re: [PATCH 2/4] ipa: rpi: controller: awb: Add Neural Network Awb","To":"Laurent Pinchart ","Cc":"libcamera-devel@lists.libcamera.org, \n\tPeter Bailey ","Content-Type":"text/plain; charset=\"UTF-8\"","X-BeenThere":"libcamera-devel@lists.libcamera.org","X-Mailman-Version":"2.1.29","Precedence":"list","List-Id":"","List-Unsubscribe":",\n\t","List-Archive":"","List-Post":"","List-Help":"","List-Subscribe":",\n\t","Errors-To":"libcamera-devel-bounces@lists.libcamera.org","Sender":"\"libcamera-devel\" "}},{"id":37318,"web_url":"https://patchwork.libcamera.org/comment/37318/","msgid":"","date":"2025-12-11T15:59:26","subject":"Re: [PATCH 2/4] ipa: rpi: controller: awb: Add Neural Network Awb","submitter":{"id":34,"url":"https://patchwork.libcamera.org/api/people/34/","name":"Naushir Patuck","email":"naush@raspberrypi.com"},"content":"On Thu, 11 Dec 2025 at 15:56, David Plowman\n wrote:\n>\n> Hi Laruent\n>\n> Thanks for the comments.\n>\n> On Thu, 11 Dec 2025 at 08:21, Laurent Pinchart\n> wrote:\n> >\n> > Hi David,\n> >\n> > Thank you for the patch.\n> >\n> > On Fri, Oct 24, 2025 at 03:16:01PM +0100, David Plowman wrote:\n> > > From: Peter Bailey \n> > >\n> > > Add an Awb algorithm which uses neural networks.\n> > >\n> > > Signed-off-by: Peter Bailey \n> > > ---\n> > > src/ipa/rpi/controller/meson.build | 9 +\n> > > src/ipa/rpi/controller/rpi/awb_nn.cpp | 442 ++++++++++++++++++++++++++\n> > > 2 files changed, 451 insertions(+)\n> > > create mode 100644 src/ipa/rpi/controller/rpi/awb_nn.cpp\n> > >\n> > > diff --git a/src/ipa/rpi/controller/meson.build b/src/ipa/rpi/controller/meson.build\n> > > index 73c93dca..2541d073 100644\n> > > --- a/src/ipa/rpi/controller/meson.build\n> > > +++ b/src/ipa/rpi/controller/meson.build\n> > > @@ -32,6 +32,15 @@ rpi_ipa_controller_deps = [\n> > > libcamera_private,\n> > > ]\n> > >\n> > > +tflite_dep = dependency('tensorflow-lite', required : false)\n> >\n> > This needs a configuration option, otherwise libcamera may accidentally\n> > depend on tensorflow-lite when not desired. Package maintainers for\n> > distributions hate those accidental dependencies that make builds\n> > non-reproducible.\n>\n> Will do. Thought I'd done it for v2 but failed to save the file! Will be in v3.\n>\n> >\n> > > +\n> > > +if tflite_dep.found()\n> > > + rpi_ipa_controller_sources += files([\n> > > + 'rpi/awb_nn.cpp',\n> > > + ])\n> > > + rpi_ipa_controller_deps += tflite_dep\n> > > +endif\n> > > +\n> > > rpi_ipa_controller_lib = static_library('rpi_ipa_controller', rpi_ipa_controller_sources,\n> > > include_directories : libipa_includes,\n> > > dependencies : rpi_ipa_controller_deps)\n> > > diff --git a/src/ipa/rpi/controller/rpi/awb_nn.cpp b/src/ipa/rpi/controller/rpi/awb_nn.cpp\n> > > new file mode 100644\n> > > index 00000000..c309ca3f\n> > > --- /dev/null\n> > > +++ b/src/ipa/rpi/controller/rpi/awb_nn.cpp\n> > > @@ -0,0 +1,442 @@\n> > > +/* SPDX-License-Identifier: BSD-2-Clause */\n> > > +/*\n> > > + * Copyright (C) 2025, Raspberry Pi Ltd\n> > > + *\n> > > + * AWB control algorithm using neural network\n> > > + */\n> > > +\n> > > +#include \n> > > +#include \n> > > +#include \n> > > +\n> > > +#include \n> > > +#include \n> > > +\n> > > +#include \n> > > +#include \n> > > +#include \n> > > +\n> > > +#include \"../awb_algorithm.h\"\n> > > +#include \"../awb_status.h\"\n> > > +#include \"../lux_status.h\"\n> > > +#include \"libipa/pwl.h\"\n> > > +\n> > > +#include \"alsc_status.h\"\n> > > +#include \"awb.h\"\n> > > +\n> > > +using namespace libcamera;\n> > > +\n> > > +LOG_DECLARE_CATEGORY(RPiAwb)\n> > > +\n> > > +constexpr double kDefaultCT = 4500.0;\n> > > +\n> > > +#define NAME \"rpi.nn.awb\"\n> > > +\n> > > +namespace RPiController {\n> > > +\n> > > +struct AwbNNConfig {\n> > > + AwbNNConfig() {}\n> > > + int read(const libcamera::YamlObject ¶ms, AwbConfig &config);\n> > > +\n> > > + /* An empty model will check default locations for model.tflite */\n> > > + std::string model;\n> > > + float minTemp;\n> > > + float maxTemp;\n> > > +\n> > > + bool enableNn;\n> > > +\n> > > + /* CCM matrix for 5000K temperature */\n> > > + double ccm[9];\n> > > +};\n> > > +\n> > > +class AwbNN : public Awb\n> > > +{\n> > > +public:\n> > > + AwbNN(Controller *controller = NULL);\n> > > + ~AwbNN();\n> > > + char const *name() const override;\n> > > + void initialise() override;\n> > > + int read(const libcamera::YamlObject ¶ms) override;\n> > > +\n> > > +protected:\n> > > + void doAwb() override;\n> > > + void prepareStats() override;\n> > > +\n> > > +private:\n> > > + bool isAutoEnabled() const;\n> > > + AwbNNConfig nnConfig_;\n> > > + void transverseSearch(double t, double &r, double &b);\n> > > + RGB processZone(RGB zone, float red_gain, float blue_gain);\n> > > + void awbNN();\n> > > + void loadModel();\n> > > +\n> > > + libcamera::Size zoneSize_;\n> > > + std::unique_ptr model_;\n> > > + std::unique_ptr interpreter_;\n> > > +};\n> > > +\n> > > +int AwbNNConfig::read(const libcamera::YamlObject ¶ms, AwbConfig &config)\n> > > +{\n> > > + model = params[\"model\"].get(\"\");\n> > > + minTemp = params[\"min_temp\"].get(2800.0);\n> > > + maxTemp = params[\"max_temp\"].get(7600.0);\n> > > +\n> > > + for (int i = 0; i < 9; i++)\n> > > + ccm[i] = params[\"ccm\"][i].get(0.0);\n> > > +\n> > > + enableNn = params[\"enable_nn\"].get(1);\n> > > +\n> > > + if (enableNn) {\n> > > + if (!config.hasCtCurve()) {\n> > > + LOG(RPiAwb, Error) << \"CT curve not specified\";\n> > > + enableNn = false;\n> > > + }\n> > > +\n> > > + if (!model.empty() && model.find(\".tflite\") == std::string::npos) {\n> > > + LOG(RPiAwb, Error) << \"Model must be a .tflite file\";\n> > > + enableNn = false;\n> > > + }\n> > > +\n> > > + bool validCcm = true;\n> > > + for (int i = 0; i < 9; i++)\n> > > + if (ccm[i] == 0.0)\n> > > + validCcm = false;\n> > > +\n> > > + if (!validCcm) {\n> > > + LOG(RPiAwb, Error) << \"CCM not specified or invalid\";\n> > > + enableNn = false;\n> > > + }\n> > > +\n> > > + if (!enableNn) {\n> > > + LOG(RPiAwb, Warning) << \"Neural Network AWB mis-configured - switch to Grey method\";\n> > > + }\n> > > + }\n> > > +\n> > > + if (!enableNn) {\n> > > + config.sensitivityR = config.sensitivityB = 1.0;\n> > > + config.greyWorld = true;\n> > > + }\n> > > +\n> > > + return 0;\n> > > +}\n> > > +\n> > > +AwbNN::AwbNN(Controller *controller)\n> > > + : Awb(controller)\n> > > +{\n> > > + zoneSize_ = getHardwareConfig().awbRegions;\n> > > +}\n> > > +\n> > > +AwbNN::~AwbNN()\n> > > +{\n> > > +}\n> > > +\n> > > +char const *AwbNN::name() const\n> > > +{\n> > > + return NAME;\n> > > +}\n> > > +\n> > > +int AwbNN::read(const libcamera::YamlObject ¶ms)\n> > > +{\n> > > + int ret;\n> > > +\n> > > + ret = config_.read(params);\n> > > + if (ret)\n> > > + return ret;\n> > > +\n> > > + ret = nnConfig_.read(params, config_);\n> > > + if (ret)\n> > > + return ret;\n> > > +\n> > > + return 0;\n> > > +}\n> > > +\n> > > +static bool checkTensorShape(TfLiteTensor *tensor, const int *expectedDims, const int expectedDimsSize)\n> > > +{\n> > > + if (tensor->dims->size != expectedDimsSize) {\n> > > + return false;\n> > > + }\n> > > +\n> > > + for (int i = 0; i < tensor->dims->size; i++) {\n> > > + if (tensor->dims->data[i] != expectedDims[i]) {\n> > > + return false;\n> > > + }\n> > > + }\n> > > + return true;\n> > > +}\n> > > +\n> > > +static std::string buildDimString(const int *dims, const int dimsSize)\n> > > +{\n> > > + std::string s = \"[\";\n> > > + for (int i = 0; i < dimsSize; i++) {\n> > > + s += std::to_string(dims[i]);\n> > > + if (i < dimsSize - 1)\n> > > + s += \",\";\n> > > + else\n> > > + s += \"]\";\n> > > + }\n> > > + return s;\n> > > +}\n> > > +\n> > > +void AwbNN::loadModel()\n> > > +{\n> > > + std::string modelPath;\n> > > + if (getTarget() == \"bcm2835\") {\n> > > + modelPath = \"/ipa/rpi/vc4/awb_model.tflite\";\n> > > + } else {\n> > > + modelPath = \"/ipa/rpi/pisp/awb_model.tflite\";\n> > > + }\n> > > +\n> > > + if (nnConfig_.model.empty()) {\n> > > + std::string root = utils::libcameraSourcePath();\n> > > + if (!root.empty()) {\n> > > + modelPath = root + modelPath;\n> > > + } else {\n> > > + modelPath = LIBCAMERA_DATA_DIR + modelPath;\n> > > + }\n> > > +\n> > > + if (!File::exists(modelPath)) {\n> > > + LOG(RPiAwb, Error) << \"No model file found in standard locations\";\n> > > + nnConfig_.enableNn = false;\n> > > + return;\n> > > + }\n> > > + } else {\n> > > + modelPath = nnConfig_.model;\n> > > + }\n> > > +\n> > > + LOG(RPiAwb, Debug) << \"Attempting to load model from: \" << modelPath;\n> > > +\n> > > + model_ = tflite::FlatBufferModel::BuildFromFile(modelPath.c_str());\n> > > +\n> > > + if (!model_) {\n> > > + LOG(RPiAwb, Error) << \"Failed to load model from \" << modelPath;\n> > > + nnConfig_.enableNn = false;\n> > > + return;\n> > > + }\n> > > +\n> > > + tflite::MutableOpResolver resolver;\n> > > + tflite::ops::builtin::BuiltinOpResolver builtin_resolver;\n> > > + resolver.AddAll(builtin_resolver);\n> > > + tflite::InterpreterBuilder(*model_, resolver)(&interpreter_);\n> > > + if (!interpreter_) {\n> > > + LOG(RPiAwb, Error) << \"Failed to build interpreter for model \" << nnConfig_.model;\n> > > + nnConfig_.enableNn = false;\n> > > + return;\n> > > + }\n> > > +\n> > > + interpreter_->AllocateTensors();\n> > > + TfLiteTensor *inputTensor = interpreter_->input_tensor(0);\n> > > + TfLiteTensor *inputLuxTensor = interpreter_->input_tensor(1);\n> > > + TfLiteTensor *outputTensor = interpreter_->output_tensor(0);\n> > > + if (!inputTensor || !inputLuxTensor || !outputTensor) {\n> > > + LOG(RPiAwb, Error) << \"Model missing input or output tensor\";\n> > > + nnConfig_.enableNn = false;\n> > > + return;\n> > > + }\n> > > +\n> > > + const int expectedInputDims[] = { 1, (int)zoneSize_.height, (int)zoneSize_.width, 3 };\n> > > + const int expectedInputLuxDims[] = { 1 };\n> > > + const int expectedOutputDims[] = { 1 };\n> > > +\n> > > + if (!checkTensorShape(inputTensor, expectedInputDims, 4)) {\n> > > + LOG(RPiAwb, Error) << \"Model input tensor dimension mismatch. Expected: \" << buildDimString(expectedInputDims, 4)\n> > > + << \", Got: \" << buildDimString(inputTensor->dims->data, inputTensor->dims->size);\n> > > + nnConfig_.enableNn = false;\n> > > + return;\n> > > + }\n> > > +\n> > > + if (!checkTensorShape(inputLuxTensor, expectedInputLuxDims, 1)) {\n> > > + LOG(RPiAwb, Error) << \"Model input lux tensor dimension mismatch. Expected: \" << buildDimString(expectedInputLuxDims, 1)\n> > > + << \", Got: \" << buildDimString(inputLuxTensor->dims->data, inputLuxTensor->dims->size);\n> > > + nnConfig_.enableNn = false;\n> > > + return;\n> > > + }\n> > > +\n> > > + if (!checkTensorShape(outputTensor, expectedOutputDims, 1)) {\n> > > + LOG(RPiAwb, Error) << \"Model output tensor dimension mismatch. Expected: \" << buildDimString(expectedOutputDims, 1)\n> > > + << \", Got: \" << buildDimString(outputTensor->dims->data, outputTensor->dims->size);\n> > > + nnConfig_.enableNn = false;\n> > > + return;\n> > > + }\n> > > +\n> > > + if (inputTensor->type != kTfLiteFloat32 || inputLuxTensor->type != kTfLiteFloat32 || outputTensor->type != kTfLiteFloat32) {\n> > > + LOG(RPiAwb, Error) << \"Model input and output tensors must be float32\";\n> > > + nnConfig_.enableNn = false;\n> > > + return;\n> > > + }\n> > > +\n> > > + LOG(RPiAwb, Info) << \"Model loaded successfully from \" << modelPath;\n> > > + LOG(RPiAwb, Debug) << \"Model validation successful - Input Image: \"\n> > > + << buildDimString(expectedInputDims, 4)\n> > > + << \", Input Lux: \" << buildDimString(expectedInputLuxDims, 1)\n> > > + << \", Output: \" << buildDimString(expectedOutputDims, 1) << \" floats\";\n> > > +}\n> > > +\n> > > +void AwbNN::initialise()\n> > > +{\n> > > + Awb::initialise();\n> > > +\n> > > + if (nnConfig_.enableNn) {\n> > > + loadModel();\n> > > + if (!nnConfig_.enableNn) {\n> > > + LOG(RPiAwb, Warning) << \"Neural Network AWB failed to load - switch to Grey method\";\n> > > + config_.greyWorld = true;\n> > > + config_.sensitivityR = config_.sensitivityB = 1.0;\n> > > + }\n> > > + }\n> > > +}\n> > > +\n> > > +void AwbNN::prepareStats()\n> > > +{\n> > > + zones_.clear();\n> > > + /*\n> > > + * LSC has already been applied to the stats in this pipeline, so stop\n> > > + * any LSC compensation. We also ignore config_.fast in this version.\n> > > + */\n> > > + generateStats(zones_, statistics_, 0.0, 0.0, getGlobalMetadata(), 0.0, 0.0, 0.0);\n> > > + /*\n> > > + * apply sensitivities, so values appear to come from our \"canonical\"\n> > > + * sensor.\n> > > + */\n> > > + for (auto &zone : zones_) {\n> > > + zone.R *= config_.sensitivityR;\n> > > + zone.B *= config_.sensitivityB;\n> > > + }\n> > > +}\n> > > +\n> > > +void AwbNN::transverseSearch(double t, double &r, double &b)\n> > > +{\n> > > + int spanR = -1, spanB = -1;\n> > > + config_.ctR.eval(t, &spanR);\n> > > + config_.ctB.eval(t, &spanB);\n> > > +\n> > > + const int diff = 10;\n> > > + double rDiff = config_.ctR.eval(t + diff, &spanR) -\n> > > + config_.ctR.eval(t - diff, &spanR);\n> > > + double bDiff = config_.ctB.eval(t + diff, &spanB) -\n> > > + config_.ctB.eval(t - diff, &spanB);\n> > > +\n> > > + ipa::Pwl::Point transverse({ bDiff, -rDiff });\n> > > + if (transverse.length2() < 1e-6)\n> > > + return;\n> > > +\n> > > + transverse = transverse / transverse.length();\n> > > + double transverseRange = config_.transverseNeg + config_.transversePos;\n> > > + const int maxNumDeltas = 12;\n> > > + int numDeltas = floor(transverseRange * 100 + 0.5) + 1;\n> > > + numDeltas = numDeltas < 3 ? 3 : (numDeltas > maxNumDeltas ? maxNumDeltas : numDeltas);\n> > > +\n> > > + ipa::Pwl::Point points[maxNumDeltas];\n> > > + int bestPoint = 0;\n> > > +\n> > > + for (int i = 0; i < numDeltas; i++) {\n> > > + points[i][0] = -config_.transverseNeg +\n> > > + (transverseRange * i) / (numDeltas - 1);\n> > > + ipa::Pwl::Point rbTest = ipa::Pwl::Point({ r, b }) +\n> > > + transverse * points[i].x();\n> > > + double rTest = rbTest.x(), bTest = rbTest.y();\n> > > + double gainR = 1 / rTest, gainB = 1 / bTest;\n> > > + double delta2Sum = computeDelta2Sum(gainR, gainB, 0.0, 0.0);\n> > > + points[i][1] = delta2Sum;\n> > > + if (points[i].y() < points[bestPoint].y())\n> > > + bestPoint = i;\n> > > + }\n> > > +\n> > > + bestPoint = std::clamp(bestPoint, 1, numDeltas - 2);\n> > > + ipa::Pwl::Point rbBest = ipa::Pwl::Point({ r, b }) +\n> > > + transverse * interpolateQuadatric(points[bestPoint - 1],\n> > > + points[bestPoint],\n> > > + points[bestPoint + 1]);\n> > > + double rBest = rbBest.x(), bBest = rbBest.y();\n> > > +\n> > > + r = rBest, b = bBest;\n> > > +}\n> > > +\n> > > +AwbNN::RGB AwbNN::processZone(AwbNN::RGB zone, float redGain, float blueGain)\n> > > +{\n> > > + /*\n> > > + * Renders the pixel at 5000K temperature\n> > > + */\n> > > + RGB zoneGains = zone;\n> > > +\n> > > + zoneGains.R *= redGain;\n> > > + zoneGains.G *= 1.0;\n> > > + zoneGains.B *= blueGain;\n> > > +\n> > > + RGB zoneCcm;\n> > > +\n> > > + zoneCcm.R = nnConfig_.ccm[0] * zoneGains.R + nnConfig_.ccm[1] * zoneGains.G + nnConfig_.ccm[2] * zoneGains.B;\n> > > + zoneCcm.G = nnConfig_.ccm[3] * zoneGains.R + nnConfig_.ccm[4] * zoneGains.G + nnConfig_.ccm[5] * zoneGains.B;\n> > > + zoneCcm.B = nnConfig_.ccm[6] * zoneGains.R + nnConfig_.ccm[7] * zoneGains.G + nnConfig_.ccm[8] * zoneGains.B;\n> > > +\n> > > + return zoneCcm;\n> > > +}\n> > > +\n> > > +void AwbNN::awbNN()\n> > > +{\n> > > + float *inputData = interpreter_->typed_input_tensor(0);\n> > > + float *inputLux = interpreter_->typed_input_tensor(1);\n> > > +\n> > > + float redGain = 1.0 / config_.ctR.eval(5000);\n> > > + float blueGain = 1.0 / config_.ctB.eval(5000);\n> > > +\n> > > + for (uint i = 0; i < zoneSize_.height; i++) {\n> > > + for (uint j = 0; j < zoneSize_.width; j++) {\n> > > + uint zoneIdx = i * zoneSize_.width + j;\n> > > +\n> > > + RGB processedZone = processZone(zones_[zoneIdx] * (1.0 / 65535), redGain, blueGain);\n> > > + uint baseIdx = zoneIdx * 3;\n> > > +\n> > > + inputData[baseIdx + 0] = static_cast(processedZone.R);\n> > > + inputData[baseIdx + 1] = static_cast(processedZone.G);\n> > > + inputData[baseIdx + 2] = static_cast(processedZone.B);\n> > > + }\n> > > + }\n> > > +\n> > > + inputLux[0] = static_cast(lux_);\n> > > +\n> > > + TfLiteStatus status = interpreter_->Invoke();\n> >\n> > How long does this typically take ?\n>\n> Because they run on the image statistics they're actually tiny models\n> (compared to most image processing/analysis models). I forget exactly,\n> but it's down at the millisecond or two level on a Pi 5, so \"a few\n> times slower\" on other Pis.\n\nIf I recall, the runtime was around 100-200us per frame on a Pi 5.\n\n>\n> Thanks\n> David\n>\n> >\n> > > + if (status != kTfLiteOk) {\n> > > + LOG(RPiAwb, Error) << \"Model inference failed with status: \" << status;\n> > > + return;\n> > > + }\n> > > +\n> > > + float *outputData = interpreter_->typed_output_tensor(0);\n> > > +\n> > > + double t = outputData[0];\n> > > +\n> > > + LOG(RPiAwb, Debug) << \"Model output temperature: \" << t;\n> > > +\n> > > + t = std::clamp(t, mode_->ctLo, mode_->ctHi);\n> > > +\n> > > + double r = config_.ctR.eval(t);\n> > > + double b = config_.ctB.eval(t);\n> > > +\n> > > + transverseSearch(t, r, b);\n> > > +\n> > > + LOG(RPiAwb, Debug) << \"After transverse search: Temperature: \" << t << \" Red gain: \" << 1.0 / r << \" Blue gain: \" << 1.0 / b;\n> > > +\n> > > + asyncResults_.temperatureK = t;\n> > > + asyncResults_.gainR = 1.0 / r * config_.sensitivityR;\n> > > + asyncResults_.gainG = 1.0;\n> > > + asyncResults_.gainB = 1.0 / b * config_.sensitivityB;\n> > > +}\n> > > +\n> > > +void AwbNN::doAwb()\n> > > +{\n> > > + prepareStats();\n> > > + if (zones_.size() == (zoneSize_.width * zoneSize_.height) && nnConfig_.enableNn) {\n> > > + awbNN();\n> > > + } else {\n> > > + awbGrey();\n> > > + }\n> > > + statistics_.reset();\n> > > +}\n> > > +\n> > > +/* Register algorithm with the system. */\n> > > +static Algorithm *create(Controller *controller)\n> > > +{\n> > > + return (Algorithm *)new AwbNN(controller);\n> > > +}\n> > > +static RegisterAlgorithm reg(NAME, &create);\n> > > +\n> > > +} /* namespace RPiController */\n> >\n> > --\n> > Regards,\n> >\n> > Laurent Pinchart","headers":{"Return-Path":"","X-Original-To":"parsemail@patchwork.libcamera.org","Delivered-To":"parsemail@patchwork.libcamera.org","Received":["from lancelot.ideasonboard.com (lancelot.ideasonboard.com\n\t[92.243.16.209])\n\tby patchwork.libcamera.org (Postfix) with ESMTPS id 9A78CBD1F1\n\tfor ;\n\tThu, 11 Dec 2025 16:00:05 +0000 (UTC)","from lancelot.ideasonboard.com (localhost [IPv6:::1])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTP id 3CA7D61628;\n\tThu, 11 Dec 2025 17:00:05 +0100 (CET)","from mail-vs1-xe33.google.com (mail-vs1-xe33.google.com\n\t[IPv6:2607:f8b0:4864:20::e33])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id 3697261603\n\tfor ;\n\tThu, 11 Dec 2025 17:00:03 +0100 (CET)","by mail-vs1-xe33.google.com with SMTP id\n\tada2fe7eead31-5dfac1bac03so19128137.2\n\tfor ;\n\tThu, 11 Dec 2025 08:00:03 -0800 (PST)"],"Authentication-Results":"lancelot.ideasonboard.com; dkim=pass (2048-bit key;\n\tunprotected) header.d=raspberrypi.com header.i=@raspberrypi.com\n\theader.b=\"bwWZ4mmk\"; dkim-atps=neutral","DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/relaxed;\n\td=raspberrypi.com; s=google; t=1765468802; x=1766073602;\n\tdarn=lists.libcamera.org; \n\th=cc:to:subject:message-id:date:from:in-reply-to:references\n\t:mime-version:from:to:cc:subject:date:message-id:reply-to;\n\tbh=NL5Yrb5+Ax1zMMBiR/NfVhwNvTFM+uZOccC2vPFT4gw=;\n\tb=bwWZ4mmk44BanN2QTQ6GuRjs8PEhCSuOZmTWeMi05K2LFB7PkA5EZ3u+EgVM38FGQM\n\ticy4CyHqjhiV/xj9bSApNs1xzJyJL7MQmYSPfu6qFcJP3lL9AW1PBQRdzfFZEMNw8n6p\n\tM4XXVGkYbRawewk4T8yqW4uqBjSZplb+b/AJM9RWSAmID46TGQYqQoSNOV6Q6uhHdavP\n\tpzIJM5zsbxcCOrZ8gbXNHhpfChYKsxkrWymDwNn3VJC3hHl2KAb/lWun/M+6lWVwVcz5\n\tXGtX52HaOBstBsPNQ1vsdPv+QpZdoSA/Jxrko1bAMTOnZhdHSSMyyTpjTlMuSMdvNI0N\n\tM+4w==","X-Google-DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/relaxed;\n\td=1e100.net; s=20230601; t=1765468802; x=1766073602;\n\th=cc:to:subject:message-id:date:from:in-reply-to:references\n\t:mime-version:x-gm-gg:x-gm-message-state:from:to:cc:subject:date\n\t:message-id:reply-to;\n\tbh=NL5Yrb5+Ax1zMMBiR/NfVhwNvTFM+uZOccC2vPFT4gw=;\n\tb=fJxo87L5YBeU7YpgdSdGD0OpbdWe9hU/9fr4A/aBe+29G+U7uIt8ILzsA7jO0pQcF7\n\tgxWtiAORUJUixX2Pw8IDen+5NMt00YUpj8t8MQ44fa9mZW8AFMM67v67sY1rrnRT1tNO\n\tLdHecra30uRsv54ct+xtsTiX2eZyFFw+JwzlRvChvOZ0r1lrW/JRQ66sNpbOvVYtZwJG\n\tI9k6Z9v6gE9hXe0+l61IAJqvXuOav0+LCM/I1Uovkgq0WM5ZZxLMtI+QVzln+itHuwVd\n\tcuQ65BLcPOWXnO6YLz4wSYc20lY27HmmsusZ+dQVAzT+QKWdcuF6yHsIFRBghhqrQKyz\n\t1YoQ==","X-Forwarded-Encrypted":"i=1;\n\tAJvYcCWhDjIqFGGgB4K4fomCuFAqDjxvqUvPRBjFciLWGcTGMM9YBzXXOTHM5N8Ae5lP0dp9mVA/X6qxLZ1Bmme0GrE=@lists.libcamera.org","X-Gm-Message-State":"AOJu0YzbhEq5cP+8615+EajeRgKJCO+ZqqE+sAap2O4Xf//S9fxNJK4I\n\t9EX/Bw6rTv63H1E3UM0H4qtGevuIt8XTrbsllBHb4CjcM8rBmEx4iNVyEUhmNFiwjFz1Oy/UEzK\n\tbpYQji9QPh8lm9lcz0ig95o0b1xb0zS2/f1q9DImsuQ==","X-Gm-Gg":"AY/fxX67sVGqsnBUJXrGqA0DeR5g2ZLm5odjATrbo5wvfXCZwgWKa10se7u/JCXLbBb\n\tW0q+La+WnsmCIqjP/b5FQyc+fUVVYtuERkWipH0G1nrEGoNAS0+KnyZli3AcXFb42aQasy1funn\n\tZarybj5lMyySWRnaiTAPvJHN+lIsZyXjVZSHOIC87KPhrqXbHQ7N+liIjtvAQ0kKCDSPaRdAmKz\n\tzjGUyCTKYptoADJASsw4JTXxhRA30SEVbVnILNh/iS931pgetuATdAwWPlUcsGJyXIk/G3Sin70\n\tyzsY1OB9z1zg9wE7wGmwYnD3OOGtCrjw5WWG","X-Google-Smtp-Source":"AGHT+IGK2l/QXjMzTnMuVC7FdY3o0/oBI7oKfahGmSg2pdcgPIiK/1ACikXzmPAkbv9+v5iH//lpDsJ1/YzMmBTnoO4=","X-Received":"by 2002:a05:6102:e0a:b0:5df:b2cd:12ad with SMTP id\n\tada2fe7eead31-5e7d0dd3c7dmr614406137.4.1765468801840; Thu, 11 Dec 2025\n\t08:00:01 -0800 (PST)","MIME-Version":"1.0","References":"<20251024144049.3311-1-david.plowman@raspberrypi.com>\n\t<20251024144049.3311-3-david.plowman@raspberrypi.com>\n\t<20251211082111.GB28411@pendragon.ideasonboard.com>\n\t","In-Reply-To":"","From":"Naushir Patuck ","Date":"Thu, 11 Dec 2025 15:59:26 +0000","X-Gm-Features":"AQt7F2rPmdgC_EZ4hQmmW9wQ5nGRQ_auWdTLD4obbx9CPFsfTpomjLf1w-olUao","Message-ID":"","Subject":"Re: [PATCH 2/4] ipa: rpi: controller: awb: Add Neural Network Awb","To":"David Plowman ","Cc":"Laurent Pinchart ,\n\tlibcamera-devel@lists.libcamera.org, \n\tPeter Bailey ","Content-Type":"text/plain; charset=\"UTF-8\"","X-BeenThere":"libcamera-devel@lists.libcamera.org","X-Mailman-Version":"2.1.29","Precedence":"list","List-Id":"","List-Unsubscribe":",\n\t","List-Archive":"","List-Post":"","List-Help":"","List-Subscribe":",\n\t","Errors-To":"libcamera-devel-bounces@lists.libcamera.org","Sender":"\"libcamera-devel\" "}}]