[{"id":37968,"web_url":"https://patchwork.libcamera.org/comment/37968/","msgid":"<1de0b8bf-c56d-45cc-85c5-b39334b80bf2@ideasonboard.com>","date":"2026-01-27T14:28:51","subject":"Re: [PATCH v5 2/4] ipa: rpi: controller: awb: Add Neural Network AWB","submitter":{"id":216,"url":"https://patchwork.libcamera.org/api/people/216/","name":"Barnabás Pőcze","email":"barnabas.pocze@ideasonboard.com"},"content":"Hi\n\nJust a couple quick comments.\n\n\n2026. 01. 27. 12:59 keltezéssel, David Plowman írta:\n> From: Peter Bailey \n> \n> Add an AWB algorithm which uses neural networks.\n> \n> Signed-off-by: Peter Bailey \n> Reviewed-by: David Plowman \n> Reviewed-by: Naushir Patuck \n> ---\n> meson_options.txt | 5 +\n> src/ipa/rpi/controller/meson.build | 9 +\n> src/ipa/rpi/controller/rpi/awb_nn.cpp | 456 ++++++++++++++++++++++++++\n> 3 files changed, 470 insertions(+)\n> create mode 100644 src/ipa/rpi/controller/rpi/awb_nn.cpp\n> \n> diff --git a/meson_options.txt b/meson_options.txt\n> index c052e85a..07847294 100644\n> --- a/meson_options.txt\n> +++ b/meson_options.txt\n> @@ -76,6 +76,11 @@ option('qcam',\n> value : 'auto',\n> description : 'Compile the qcam test application')\n> \n> +option('rpi-awb-nn',\n\nIf dots work, then I think `rpi.awb-nn` is better name.\n\n\n> + type : 'feature',\n> + value : 'auto',\n> + description : 'Enable the Raspberry Pi Neural Network AWB algorithm')\n> +\n> option('test',\n> type : 'boolean',\n> value : false,\n> diff --git a/src/ipa/rpi/controller/meson.build b/src/ipa/rpi/controller/meson.build\n> index c8637906..03ee7c20 100644\n> --- a/src/ipa/rpi/controller/meson.build\n> +++ b/src/ipa/rpi/controller/meson.build\n> @@ -32,6 +32,15 @@ rpi_ipa_controller_deps = [\n> libcamera_private,\n> ]\n> \n> +tflite_dep = dependency('tensorflow-lite', required : get_option('rpi-awb-nn'))\n> +\n> +if tflite_dep.found()\n> + rpi_ipa_controller_sources += files([\n> + 'rpi/awb_nn.cpp',\n> + ])\n> + rpi_ipa_controller_deps += tflite_dep\n> +endif\n> +\n> rpi_ipa_controller_lib = static_library('rpi_ipa_controller', rpi_ipa_controller_sources,\n> include_directories : libipa_includes,\n> dependencies : rpi_ipa_controller_deps)\n> diff --git a/src/ipa/rpi/controller/rpi/awb_nn.cpp b/src/ipa/rpi/controller/rpi/awb_nn.cpp\n> new file mode 100644\n> index 00000000..395add85\n> --- /dev/null\n> +++ b/src/ipa/rpi/controller/rpi/awb_nn.cpp\n> @@ -0,0 +1,456 @@\n> +/* SPDX-License-Identifier: BSD-2-Clause */\n> +/*\n> + * Copyright (C) 2025, Raspberry Pi Ltd\n> + *\n> + * AWB control algorithm using neural network\n> + *\n> + * The AWB Neural Network algorithm can be run entirely with the code here\n> + * and the suppllied TFLite models. Those interested in the full model\n> + * definitions, or who may want to re-train the models should visit\n> + *\n> + * https://github.com/raspberrypi/awb_nn\n> + *\n> + * where you will find full source code for the models, the full datasets\n> + * used for training our supplied models, and full instructions for capturing\n> + * your own images and re-training the models for your own use cases.\n> + */\n> +\n> +#include \n> +#include \n> +#include \n\nThe ones above don't seem to be used.\n\n\n> +\n> +#include \n> +#include \n> +\n> +#include \n> +#include \n> +#include \n> +\n> +#include \"../awb_algorithm.h\"\n> +#include \"../awb_status.h\"\n> +#include \"../lux_status.h\"\n> +#include \"libipa/pwl.h\"\n> +\n> +#include \"alsc_status.h\"\n\nThis also does not look used.\n\n\n> +#include \"awb.h\"\n> +\n> +using namespace libcamera;\n> +\n> +LOG_DECLARE_CATEGORY(RPiAwb)\n> +\n> +constexpr double kDefaultCT = 4500.0;\n> +\n> +/*\n> + * The neural networks are trained to work on images rendered at a canonical\n> + * colour temperature. That value is 5000K, which must be reproduced here.\n> + */\n> +constexpr double kNetworkCanonicalCT = 5000.0;\n> +\n> +#define NAME \"rpi.nn.awb\"\n> +\n> +namespace RPiController {\n> +\n> +struct AwbNNConfig {\n> +\tAwbNNConfig() {}\n\nIs this empty constructor needed?\n\n\n> +\tint read(const libcamera::YamlObject ¶ms, AwbConfig &config);\n> +\n> +\t/* An empty model will check default locations for model.tflite */\n> +\tstd::string model;\n> +\tfloat minTemp;\n> +\tfloat maxTemp;\n> +\n> +\tbool enableNn;\n> +\n> +\t/* CCM matrix for canonical network CT */\n> +\tdouble ccm[9];\n> +};\n> +\n> +class AwbNN : public Awb\n> +{\n> +public:\n> +\tAwbNN(Controller *controller = NULL);\n\n nullptr\n\n\n> +\t~AwbNN();\n> +\tchar const *name() const override;\n> +\tvoid initialise() override;\n> +\tint read(const libcamera::YamlObject ¶ms) override;\n> +\n> +protected:\n> +\tvoid doAwb() override;\n> +\tvoid prepareStats() override;\n> +\n> +private:\n> +\tbool isAutoEnabled() const;\n> +\tAwbNNConfig nnConfig_;\n> +\tvoid transverseSearch(double t, double &r, double &b);\n> +\tRGB processZone(RGB zone, float red_gain, float blue_gain);\n> +\tvoid awbNN();\n> +\tvoid loadModel();\n> +\n> +\tlibcamera::Size zoneSize_;\n> +\tstd::unique_ptr model_;\n> +\tstd::unique_ptr interpreter_;\n> +};\n> +\n> +int AwbNNConfig::read(const libcamera::YamlObject ¶ms, AwbConfig &config)\n> +{\n> +\tmodel = params[\"model\"].get(\"\");\n> +\tminTemp = params[\"min_temp\"].get(2800.0);\n> +\tmaxTemp = params[\"max_temp\"].get(7600.0);\n> +\n> +\tfor (int i = 0; i < 9; i++)\n> +\t\tccm[i] = params[\"ccm\"][i].get(0.0);\n> +\n> +\tenableNn = params[\"enable_nn\"].get(1);\n> +\n> +\tif (enableNn) {\n> +\t\tif (!config.hasCtCurve()) {\n> +\t\t\tLOG(RPiAwb, Error) << \"CT curve not specified\";\n> +\t\t\tenableNn = false;\n> +\t\t}\n> +\n> +\t\tif (!model.empty() && model.find(\".tflite\") == std::string::npos) {\n> +\t\t\tLOG(RPiAwb, Error) << \"Model must be a .tflite file\";\n> +\t\t\tenableNn = false;\n\nIs it useful to force the extension?\n\n\n> +\t\t}\n> +\n> +\t\tbool validCcm = true;\n> +\t\tfor (int i = 0; i < 9; i++)\n> +\t\t\tif (ccm[i] == 0.0)\n> +\t\t\t\tvalidCcm = false;\n> +\n> +\t\tif (!validCcm) {\n> +\t\t\tLOG(RPiAwb, Error) << \"CCM not specified or invalid\";\n> +\t\t\tenableNn = false;\n> +\t\t}\n> +\n> +\t\tif (!enableNn) {\n> +\t\t\tLOG(RPiAwb, Warning) << \"Neural Network AWB mis-configured - switch to Grey method\";\n\n \"misconfigured\" ?\n\n> +\t\t}\n> +\t}\n> +\n> +\tif (!enableNn) {\n> +\t\tconfig.sensitivityR = config.sensitivityB = 1.0;\n> +\t\tconfig.greyWorld = true;\n> +\t}\n> +\n> +\treturn 0;\n> +}\n> +\n> +AwbNN::AwbNN(Controller *controller)\n> +\t: Awb(controller)\n> +{\n> +\tzoneSize_ = getHardwareConfig().awbRegions;\n> +}\n> +\n> +AwbNN::~AwbNN()\n> +{\n> +}\n> +\n> +char const *AwbNN::name() const\n> +{\n> +\treturn NAME;\n> +}\n> +\n> +int AwbNN::read(const libcamera::YamlObject ¶ms)\n> +{\n> +\tint ret;\n> +\n> +\tret = config_.read(params);\n> +\tif (ret)\n> +\t\treturn ret;\n> +\n> +\tret = nnConfig_.read(params, config_);\n> +\tif (ret)\n> +\t\treturn ret;\n> +\n> +\treturn 0;\n> +}\n> +\n> +static bool checkTensorShape(TfLiteTensor *tensor, const int *expectedDims, const int expectedDimsSize)\n> +{\n> +\tif (tensor->dims->size != expectedDimsSize)\n> +\t\treturn false;\n> +\n> +\tfor (int i = 0; i < tensor->dims->size; i++) {\n> +\t\tif (tensor->dims->data[i] != expectedDims[i]) {\n> +\t\t\treturn false;\n> +\t\t}\n> +\t}\n> +\treturn true;\n\nfrom \n\n return std::equal(expectedDims, expectedDims + expectedDimsSize,\n tensor->dims->data, tensor->dims->data + tensor->dims->size);\n\n?\n\n\n> +}\n> +\n> +static std::string buildDimString(const int *dims, const int dimsSize)\n> +{\n> +\tstd::string s = \"[\";\n> +\tfor (int i = 0; i < dimsSize; i++) {\n> +\t\ts += std::to_string(dims[i]);\n> +\t\tif (i < dimsSize - 1)\n> +\t\t\ts += \",\";\n> +\t\telse\n> +\t\t\ts += \"]\";\n> +\t}\n> +\treturn s;\n\n return '[' + utils::join(Span{ dims, dimsSize }, \",\") + ']';\n\n?\n\n\n> +}\n> +\n> +void AwbNN::loadModel()\n> +{\n> +\tstd::string modelPath;\n> +\tif (getTarget() == \"bcm2835\") {\n> +\t\tmodelPath = \"/ipa/rpi/vc4/awb_model.tflite\";\n> +\t} else {\n> +\t\tmodelPath = \"/ipa/rpi/pisp/awb_model.tflite\";\n> +\t}\n> +\n> +\tif (nnConfig_.model.empty()) {\n> +\t\tstd::string root = utils::libcameraSourcePath();\n> +\t\tif (!root.empty()) {\n> +\t\t\tmodelPath = root + modelPath;\n> +\t\t} else {\n> +\t\t\tmodelPath = LIBCAMERA_DATA_DIR + modelPath;\n> +\t\t}\n> +\n> +\t\tif (!File::exists(modelPath)) {\n> +\t\t\tLOG(RPiAwb, Error) << \"No model file found in standard locations\";\n> +\t\t\tnnConfig_.enableNn = false;\n> +\t\t\treturn;\n> +\t\t}\n> +\t} else {\n> +\t\tmodelPath = nnConfig_.model;\n> +\t}\n> +\n> +\tLOG(RPiAwb, Debug) << \"Attempting to load model from: \" << modelPath;\n> +\n> +\tmodel_ = tflite::FlatBufferModel::BuildFromFile(modelPath.c_str());\n\nAs far as I can see, `BuildFromFile` takes an `ErrorReporter` parameter. Would it\nmake sense to create a static instance of one and use it to route messages into\nlibcamera log? If not specified, does it report anything to stderr or similar?\n(And the errors from tflite are logged, then I would probably also remove the\n `File::exists()` check as well.)\n\n\n> +\n> +\tif (!model_) {\n> +\t\tLOG(RPiAwb, Error) << \"Failed to load model from \" << modelPath;\n> +\t\tnnConfig_.enableNn = false;\n> +\t\treturn;\n> +\t}\n> +\n> +\ttflite::MutableOpResolver resolver;\n> +\ttflite::ops::builtin::BuiltinOpResolver builtin_resolver;\n> +\tresolver.AddAll(builtin_resolver);\n> +\ttflite::InterpreterBuilder(*model_, resolver)(&interpreter_);\n> +\tif (!interpreter_) {\n> +\t\tLOG(RPiAwb, Error) << \"Failed to build interpreter for model \" << nnConfig_.model;\n> +\t\tnnConfig_.enableNn = false;\n> +\t\treturn;\n> +\t}\n> +\n> +\tinterpreter_->AllocateTensors();\n> +\tTfLiteTensor *inputTensor = interpreter_->input_tensor(0);\n> +\tTfLiteTensor *inputLuxTensor = interpreter_->input_tensor(1);\n> +\tTfLiteTensor *outputTensor = interpreter_->output_tensor(0);\n> +\tif (!inputTensor || !inputLuxTensor || !outputTensor) {\n> +\t\tLOG(RPiAwb, Error) << \"Model missing input or output tensor\";\n> +\t\tnnConfig_.enableNn = false;\n> +\t\treturn;\n> +\t}\n> +\n> +\tconst int expectedInputDims[] = { 1, (int)zoneSize_.height, (int)zoneSize_.width, 3 };\n> +\tconst int expectedInputLuxDims[] = { 1 };\n> +\tconst int expectedOutputDims[] = { 1 };\n> +\n> +\tif (!checkTensorShape(inputTensor, expectedInputDims, 4)) {\n> +\t\tLOG(RPiAwb, Error) << \"Model input tensor dimension mismatch. Expected: \" << buildDimString(expectedInputDims, 4)\n> +\t\t\t\t << \", Got: \" << buildDimString(inputTensor->dims->data, inputTensor->dims->size);\n> +\t\tnnConfig_.enableNn = false;\n> +\t\treturn;\n> +\t}\n> +\n> +\tif (!checkTensorShape(inputLuxTensor, expectedInputLuxDims, 1)) {\n> +\t\tLOG(RPiAwb, Error) << \"Model input lux tensor dimension mismatch. Expected: \" << buildDimString(expectedInputLuxDims, 1)\n> +\t\t\t\t << \", Got: \" << buildDimString(inputLuxTensor->dims->data, inputLuxTensor->dims->size);\n> +\t\tnnConfig_.enableNn = false;\n> +\t\treturn;\n> +\t}\n> +\n> +\tif (!checkTensorShape(outputTensor, expectedOutputDims, 1)) {\n> +\t\tLOG(RPiAwb, Error) << \"Model output tensor dimension mismatch. Expected: \" << buildDimString(expectedOutputDims, 1)\n> +\t\t\t\t << \", Got: \" << buildDimString(outputTensor->dims->data, outputTensor->dims->size);\n> +\t\tnnConfig_.enableNn = false;\n> +\t\treturn;\n> +\t}\n> +\n> +\tif (inputTensor->type != kTfLiteFloat32 || inputLuxTensor->type != kTfLiteFloat32 || outputTensor->type != kTfLiteFloat32) {\n> +\t\tLOG(RPiAwb, Error) << \"Model input and output tensors must be float32\";\n> +\t\tnnConfig_.enableNn = false;\n> +\t\treturn;\n> +\t}\n> +\n> +\tLOG(RPiAwb, Info) << \"Model loaded successfully from \" << modelPath;\n> +\tLOG(RPiAwb, Debug) << \"Model validation successful - Input Image: \"\n> +\t\t\t << buildDimString(expectedInputDims, 4)\n> +\t\t\t << \", Input Lux: \" << buildDimString(expectedInputLuxDims, 1)\n> +\t\t\t << \", Output: \" << buildDimString(expectedOutputDims, 1) << \" floats\";\n> +}\n> +\n> +void AwbNN::initialise()\n> +{\n> +\tAwb::initialise();\n> +\n> +\tif (nnConfig_.enableNn) {\n> +\t\tloadModel();\n> +\t\tif (!nnConfig_.enableNn) {\n> +\t\t\tLOG(RPiAwb, Warning) << \"Neural Network AWB failed to load - switch to Grey method\";\n> +\t\t\tconfig_.greyWorld = true;\n> +\t\t\tconfig_.sensitivityR = config_.sensitivityB = 1.0;\n> +\t\t}\n> +\t}\n> +}\n> +\n> +void AwbNN::prepareStats()\n> +{\n> +\tzones_.clear();\n> +\t/*\n> +\t * LSC has already been applied to the stats in this pipeline, so stop\n> +\t * any LSC compensation. We also ignore config_.fast in this version.\n> +\t */\n> +\tgenerateStats(zones_, statistics_, 0.0, 0.0, getGlobalMetadata(), 0.0, 0.0, 0.0);\n> +\t/*\n> +\t * apply sensitivities, so values appear to come from our \"canonical\"\n> +\t * sensor.\n> +\t */\n> +\tfor (auto &zone : zones_) {\n> +\t\tzone.R *= config_.sensitivityR;\n> +\t\tzone.B *= config_.sensitivityB;\n> +\t}\n> +}\n> +\n> +void AwbNN::transverseSearch(double t, double &r, double &b)\n> +{\n> +\tint spanR = -1, spanB = -1;\n> +\tconfig_.ctR.eval(t, &spanR);\n> +\tconfig_.ctB.eval(t, &spanB);\n> +\n> +\tconst int diff = 10;\n> +\tdouble rDiff = config_.ctR.eval(t + diff, &spanR) -\n> +\t\t config_.ctR.eval(t - diff, &spanR);\n> +\tdouble bDiff = config_.ctB.eval(t + diff, &spanB) -\n> +\t\t config_.ctB.eval(t - diff, &spanB);\n> +\n> +\tipa::Pwl::Point transverse({ bDiff, -rDiff });\n> +\tif (transverse.length2() < 1e-6)\n> +\t\treturn;\n> +\n> +\ttransverse = transverse / transverse.length();\n> +\tdouble transverseRange = config_.transverseNeg + config_.transversePos;\n> +\tconst int maxNumDeltas = 12;\n> +\tint numDeltas = floor(transverseRange * 100 + 0.5) + 1;\n> +\tnumDeltas = numDeltas < 3 ? 3 : (numDeltas > maxNumDeltas ? maxNumDeltas : numDeltas);\n\n int numDeltas = std::clamp(floor(transverseRange * 100 + 0.5) + 1, 3, maxNumDeltas);\n\n?\n\n\n> +\n> +\tipa::Pwl::Point points[maxNumDeltas];\n> +\tint bestPoint = 0;\n> +\n> +\tfor (int i = 0; i < numDeltas; i++) {\n> +\t\tpoints[i][0] = -config_.transverseNeg +\n> +\t\t\t (transverseRange * i) / (numDeltas - 1);\n> +\t\tipa::Pwl::Point rbTest = ipa::Pwl::Point({ r, b }) +\n> +\t\t\t\t\t transverse * points[i].x();\n> +\t\tdouble rTest = rbTest.x(), bTest = rbTest.y();\n> +\t\tdouble gainR = 1 / rTest, gainB = 1 / bTest;\n> +\t\tdouble delta2Sum = computeDelta2Sum(gainR, gainB, 0.0, 0.0);\n> +\t\tpoints[i][1] = delta2Sum;\n> +\t\tif (points[i].y() < points[bestPoint].y())\n> +\t\t\tbestPoint = i;\n> +\t}\n> +\n> +\tbestPoint = std::clamp(bestPoint, 1, numDeltas - 2);\n> +\tipa::Pwl::Point rbBest = ipa::Pwl::Point({ r, b }) +\n> +\t\t\t\t transverse * interpolateQuadatric(points[bestPoint - 1],\n> +\t\t\t\t\t\t\t\t points[bestPoint],\n> +\t\t\t\t\t\t\t\t points[bestPoint + 1]);\n> +\tdouble rBest = rbBest.x(), bBest = rbBest.y();\n> +\n> +\tr = rBest, b = bBest;\n\n r = rbBest.x();\n b = rbBest.y();\n\n?\n\n\n> +}\n> +\n> +AwbNN::RGB AwbNN::processZone(AwbNN::RGB zone, float redGain, float blueGain)\n> +{\n> +\t/*\n> +\t * Renders the pixel at canonical network colour temperature\n> +\t */\n> +\tRGB zoneGains = zone;\n> +\n> +\tzoneGains.R *= redGain;\n> +\tzoneGains.G *= 1.0;\n> +\tzoneGains.B *= blueGain;\n> +\n> +\tRGB zoneCcm;\n> +\n> +\tzoneCcm.R = nnConfig_.ccm[0] * zoneGains.R + nnConfig_.ccm[1] * zoneGains.G + nnConfig_.ccm[2] * zoneGains.B;\n> +\tzoneCcm.G = nnConfig_.ccm[3] * zoneGains.R + nnConfig_.ccm[4] * zoneGains.G + nnConfig_.ccm[5] * zoneGains.B;\n> +\tzoneCcm.B = nnConfig_.ccm[6] * zoneGains.R + nnConfig_.ccm[7] * zoneGains.G + nnConfig_.ccm[8] * zoneGains.B;\n> +\n> +\treturn zoneCcm;\n> +}\n> +\n> +void AwbNN::awbNN()\n> +{\n> +\tfloat *inputData = interpreter_->typed_input_tensor(0);\n> +\tfloat *inputLux = interpreter_->typed_input_tensor(1);\n> +\n> +\tfloat redGain = 1.0 / config_.ctR.eval(kNetworkCanonicalCT);\n> +\tfloat blueGain = 1.0 / config_.ctB.eval(kNetworkCanonicalCT);\n> +\n> +\tfor (uint i = 0; i < zoneSize_.height; i++) {\n> +\t\tfor (uint j = 0; j < zoneSize_.width; j++) {\n> +\t\t\tuint zoneIdx = i * zoneSize_.width + j;\n> +\n> +\t\t\tRGB processedZone = processZone(zones_[zoneIdx] * (1.0 / 65535), redGain, blueGain);\n> +\t\t\tuint baseIdx = zoneIdx * 3;\n\nWhere is this `uint` type coming from? tflite? Given that `zoneSize_` is `libcamera::Size`,\nwhich uses `unsigned int`, is this `uint` type necessary here?\n\n\n> +\n> +\t\t\tinputData[baseIdx + 0] = static_cast(processedZone.R);\n> +\t\t\tinputData[baseIdx + 1] = static_cast(processedZone.G);\n> +\t\t\tinputData[baseIdx + 2] = static_cast(processedZone.B);\n> +\t\t}\n> +\t}\n> +\n> +\tinputLux[0] = static_cast(lux_);\n> +\n> +\tTfLiteStatus status = interpreter_->Invoke();\n> +\tif (status != kTfLiteOk) {\n> +\t\tLOG(RPiAwb, Error) << \"Model inference failed with status: \" << status;\n> +\t\treturn;\n> +\t}\n> +\n> +\tfloat *outputData = interpreter_->typed_output_tensor(0);\n> +\n> +\tdouble t = outputData[0];\n> +\n> +\tLOG(RPiAwb, Debug) << \"Model output temperature: \" << t;\n> +\n> +\tt = std::clamp(t, mode_->ctLo, mode_->ctHi);\n> +\n> +\tdouble r = config_.ctR.eval(t);\n> +\tdouble b = config_.ctB.eval(t);\n> +\n> +\ttransverseSearch(t, r, b);\n> +\n> +\tLOG(RPiAwb, Debug) << \"After transverse search: Temperature: \" << t << \" Red gain: \" << 1.0 / r << \" Blue gain: \" << 1.0 / b;\n> +\n> +\tasyncResults_.temperatureK = t;\n> +\tasyncResults_.gainR = 1.0 / r * config_.sensitivityR;\n> +\tasyncResults_.gainG = 1.0;\n> +\tasyncResults_.gainB = 1.0 / b * config_.sensitivityB;\n> +}\n> +\n> +void AwbNN::doAwb()\n> +{\n> +\tprepareStats();\n> +\tif (zones_.size() == (zoneSize_.width * zoneSize_.height) && nnConfig_.enableNn)\n> +\t\tawbNN();\n> +\telse\n> +\t\tawbGrey();\n> +\tstatistics_.reset();\n> +}\n> +\n> +/* Register algorithm with the system. */\n> +static Algorithm *create(Controller *controller)\n> +{\n> +\treturn (Algorithm *)new AwbNN(controller);\n\nPlease omit this cast.\n\n\nRegards,\nBarnabás Pőcze\n\n> +}\n> +static RegisterAlgorithm reg(NAME, &create);\n> +\n> +} /* namespace RPiController */","headers":{"Return-Path":"","X-Original-To":"parsemail@patchwork.libcamera.org","Delivered-To":"parsemail@patchwork.libcamera.org","Received":["from lancelot.ideasonboard.com (lancelot.ideasonboard.com\n\t[92.243.16.209])\n\tby patchwork.libcamera.org (Postfix) with ESMTPS id D7F9AC3220\n\tfor ;\n\tTue, 27 Jan 2026 14:28:58 +0000 (UTC)","from lancelot.ideasonboard.com (localhost [IPv6:::1])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTP id A866E61FCE;\n\tTue, 27 Jan 2026 15:28:57 +0100 (CET)","from perceval.ideasonboard.com (perceval.ideasonboard.com\n\t[IPv6:2001:4b98:dc2:55:216:3eff:fef7:d647])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id 6145061FBF\n\tfor ;\n\tTue, 27 Jan 2026 15:28:55 +0100 (CET)","from [192.168.33.37] (185.221.142.123.nat.pool.zt.hu\n\t[185.221.142.123])\n\tby perceval.ideasonboard.com (Postfix) with ESMTPSA id 08A4D186F;\n\tTue, 27 Jan 2026 15:28:19 +0100 (CET)"],"Authentication-Results":"lancelot.ideasonboard.com; dkim=pass (1024-bit key;\n\tunprotected) header.d=ideasonboard.com header.i=@ideasonboard.com\n\theader.b=\"NDGz+JZp\"; dkim-atps=neutral","DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/simple; d=ideasonboard.com;\n\ts=mail; t=1769524099;\n\tbh=aBy0DLe3Ta0oDf2eM75xtxG1U3BE6mMWCfpcRz6PwTE=;\n\th=Date:Subject:To:Cc:References:From:In-Reply-To:From;\n\tb=NDGz+JZpTZfQXCevo+St08v1rRawQoBOVcB8EG237oXP1J4E8EKGClXUhIpdIWeNZ\n\td6++s3gDOvTXSgRz65KUWPJdvvIcOXE3BLyhfxpwVURayEOe2RN1D6ms6VUpOMmfRV\n\tr2pu9nmfoNCPMLq55lDwbUZaeO4YqGmb4DBJXQRY=","Message-ID":"<1de0b8bf-c56d-45cc-85c5-b39334b80bf2@ideasonboard.com>","Date":"Tue, 27 Jan 2026 15:28:51 +0100","MIME-Version":"1.0","User-Agent":"Mozilla Thunderbird","Subject":"Re: [PATCH v5 2/4] ipa: rpi: controller: awb: Add Neural Network AWB","To":"David Plowman ,\n\tlibcamera-devel@lists.libcamera.org","Cc":"Peter Bailey ,\n\tNaushir Patuck ","References":"<20260127120604.6560-1-david.plowman@raspberrypi.com>\n\t<20260127120604.6560-3-david.plowman@raspberrypi.com>","From":"=?utf-8?q?Barnab=C3=A1s_P=C5=91cze?= ","Content-Language":"en-US, hu-HU","In-Reply-To":"<20260127120604.6560-3-david.plowman@raspberrypi.com>","Content-Type":"text/plain; charset=UTF-8; format=flowed","Content-Transfer-Encoding":"8bit","X-BeenThere":"libcamera-devel@lists.libcamera.org","X-Mailman-Version":"2.1.29","Precedence":"list","List-Id":"","List-Unsubscribe":",\n\t","List-Archive":"","List-Post":"","List-Help":"","List-Subscribe":",\n\t","Errors-To":"libcamera-devel-bounces@lists.libcamera.org","Sender":"\"libcamera-devel\" "}},{"id":37978,"web_url":"https://patchwork.libcamera.org/comment/37978/","msgid":"","date":"2026-01-27T17:10:01","subject":"Re: [PATCH v5 2/4] ipa: rpi: controller: awb: Add Neural Network AWB","submitter":{"id":42,"url":"https://patchwork.libcamera.org/api/people/42/","name":"David Plowman","email":"david.plowman@raspberrypi.com"},"content":"Hi Barnabas\n\nThank you for the comments.\n\nOn Tue, 27 Jan 2026 at 14:28, Barnabás Pőcze\n wrote:\n>\n> Hi\n>\n> Just a couple quick comments.\n>\n>\n> 2026. 01. 27. 12:59 keltezéssel, David Plowman írta:\n> > From: Peter Bailey \n> >\n> > Add an AWB algorithm which uses neural networks.\n> >\n> > Signed-off-by: Peter Bailey \n> > Reviewed-by: David Plowman \n> > Reviewed-by: Naushir Patuck \n> > ---\n> > meson_options.txt | 5 +\n> > src/ipa/rpi/controller/meson.build | 9 +\n> > src/ipa/rpi/controller/rpi/awb_nn.cpp | 456 ++++++++++++++++++++++++++\n> > 3 files changed, 470 insertions(+)\n> > create mode 100644 src/ipa/rpi/controller/rpi/awb_nn.cpp\n> >\n> > diff --git a/meson_options.txt b/meson_options.txt\n> > index c052e85a..07847294 100644\n> > --- a/meson_options.txt\n> > +++ b/meson_options.txt\n> > @@ -76,6 +76,11 @@ option('qcam',\n> > value : 'auto',\n> > description : 'Compile the qcam test application')\n> >\n> > +option('rpi-awb-nn',\n>\n> If dots work, then I think `rpi.awb-nn` is better name.\n\nI don't believe this works, unfortunately. Someone please correct me\nif I'm wrong!\n\n>\n>\n> > + type : 'feature',\n> > + value : 'auto',\n> > + description : 'Enable the Raspberry Pi Neural Network AWB algorithm')\n> > +\n> > option('test',\n> > type : 'boolean',\n> > value : false,\n> > diff --git a/src/ipa/rpi/controller/meson.build b/src/ipa/rpi/controller/meson.build\n> > index c8637906..03ee7c20 100644\n> > --- a/src/ipa/rpi/controller/meson.build\n> > +++ b/src/ipa/rpi/controller/meson.build\n> > @@ -32,6 +32,15 @@ rpi_ipa_controller_deps = [\n> > libcamera_private,\n> > ]\n> >\n> > +tflite_dep = dependency('tensorflow-lite', required : get_option('rpi-awb-nn'))\n> > +\n> > +if tflite_dep.found()\n> > + rpi_ipa_controller_sources += files([\n> > + 'rpi/awb_nn.cpp',\n> > + ])\n> > + rpi_ipa_controller_deps += tflite_dep\n> > +endif\n> > +\n> > rpi_ipa_controller_lib = static_library('rpi_ipa_controller', rpi_ipa_controller_sources,\n> > include_directories : libipa_includes,\n> > dependencies : rpi_ipa_controller_deps)\n> > diff --git a/src/ipa/rpi/controller/rpi/awb_nn.cpp b/src/ipa/rpi/controller/rpi/awb_nn.cpp\n> > new file mode 100644\n> > index 00000000..395add85\n> > --- /dev/null\n> > +++ b/src/ipa/rpi/controller/rpi/awb_nn.cpp\n> > @@ -0,0 +1,456 @@\n> > +/* SPDX-License-Identifier: BSD-2-Clause */\n> > +/*\n> > + * Copyright (C) 2025, Raspberry Pi Ltd\n> > + *\n> > + * AWB control algorithm using neural network\n> > + *\n> > + * The AWB Neural Network algorithm can be run entirely with the code here\n> > + * and the suppllied TFLite models. Those interested in the full model\n> > + * definitions, or who may want to re-train the models should visit\n> > + *\n> > + * https://github.com/raspberrypi/awb_nn\n> > + *\n> > + * where you will find full source code for the models, the full datasets\n> > + * used for training our supplied models, and full instructions for capturing\n> > + * your own images and re-training the models for your own use cases.\n> > + */\n> > +\n> > +#include \n> > +#include \n> > +#include \n>\n> The ones above don't seem to be used.\n\nYes, will remove.\n\n>\n>\n> > +\n> > +#include \n> > +#include \n> > +\n> > +#include \n> > +#include \n> > +#include \n> > +\n> > +#include \"../awb_algorithm.h\"\n> > +#include \"../awb_status.h\"\n> > +#include \"../lux_status.h\"\n> > +#include \"libipa/pwl.h\"\n> > +\n> > +#include \"alsc_status.h\"\n>\n> This also does not look used.\n\nYes, this too.\n\n>\n>\n> > +#include \"awb.h\"\n> > +\n> > +using namespace libcamera;\n> > +\n> > +LOG_DECLARE_CATEGORY(RPiAwb)\n> > +\n> > +constexpr double kDefaultCT = 4500.0;\n> > +\n> > +/*\n> > + * The neural networks are trained to work on images rendered at a canonical\n> > + * colour temperature. That value is 5000K, which must be reproduced here.\n> > + */\n> > +constexpr double kNetworkCanonicalCT = 5000.0;\n> > +\n> > +#define NAME \"rpi.nn.awb\"\n> > +\n> > +namespace RPiController {\n> > +\n> > +struct AwbNNConfig {\n> > + AwbNNConfig() {}\n>\n> Is this empty constructor needed?\n\nTrue, it probably isn't. Though I quite like making it explicit if\nit's actually being used somewhere. But \"AwbNNConfig() = default;\"\nwould certainly be better, so maybe I'll go with that?\n\n>\n>\n> > + int read(const libcamera::YamlObject ¶ms, AwbConfig &config);\n> > +\n> > + /* An empty model will check default locations for model.tflite */\n> > + std::string model;\n> > + float minTemp;\n> > + float maxTemp;\n> > +\n> > + bool enableNn;\n> > +\n> > + /* CCM matrix for canonical network CT */\n> > + double ccm[9];\n> > +};\n> > +\n> > +class AwbNN : public Awb\n> > +{\n> > +public:\n> > + AwbNN(Controller *controller = NULL);\n>\n> nullptr\n\nI'll change that. Actually there's one in awb.h too so I'll also\nchange that as well.\n\n>\n>\n> > + ~AwbNN();\n> > + char const *name() const override;\n> > + void initialise() override;\n> > + int read(const libcamera::YamlObject ¶ms) override;\n> > +\n> > +protected:\n> > + void doAwb() override;\n> > + void prepareStats() override;\n> > +\n> > +private:\n> > + bool isAutoEnabled() const;\n> > + AwbNNConfig nnConfig_;\n> > + void transverseSearch(double t, double &r, double &b);\n> > + RGB processZone(RGB zone, float red_gain, float blue_gain);\n> > + void awbNN();\n> > + void loadModel();\n> > +\n> > + libcamera::Size zoneSize_;\n> > + std::unique_ptr model_;\n> > + std::unique_ptr interpreter_;\n> > +};\n> > +\n> > +int AwbNNConfig::read(const libcamera::YamlObject ¶ms, AwbConfig &config)\n> > +{\n> > + model = params[\"model\"].get(\"\");\n> > + minTemp = params[\"min_temp\"].get(2800.0);\n> > + maxTemp = params[\"max_temp\"].get(7600.0);\n> > +\n> > + for (int i = 0; i < 9; i++)\n> > + ccm[i] = params[\"ccm\"][i].get(0.0);\n> > +\n> > + enableNn = params[\"enable_nn\"].get(1);\n> > +\n> > + if (enableNn) {\n> > + if (!config.hasCtCurve()) {\n> > + LOG(RPiAwb, Error) << \"CT curve not specified\";\n> > + enableNn = false;\n> > + }\n> > +\n> > + if (!model.empty() && model.find(\".tflite\") == std::string::npos) {\n> > + LOG(RPiAwb, Error) << \"Model must be a .tflite file\";\n> > + enableNn = false;\n>\n> Is it useful to force the extension?\n\nI quite like forcing the extension, it makes it very clear that we\nwant the .tflite file from our repo, not the .keras or anything else.\n\n>\n>\n> > + }\n> > +\n> > + bool validCcm = true;\n> > + for (int i = 0; i < 9; i++)\n> > + if (ccm[i] == 0.0)\n> > + validCcm = false;\n> > +\n> > + if (!validCcm) {\n> > + LOG(RPiAwb, Error) << \"CCM not specified or invalid\";\n> > + enableNn = false;\n> > + }\n> > +\n> > + if (!enableNn) {\n> > + LOG(RPiAwb, Warning) << \"Neural Network AWB mis-configured - switch to Grey method\";\n>\n> \"misconfigured\" ?\n\nYes, I'm inclined to agree.\n\n>\n> > + }\n> > + }\n> > +\n> > + if (!enableNn) {\n> > + config.sensitivityR = config.sensitivityB = 1.0;\n> > + config.greyWorld = true;\n> > + }\n> > +\n> > + return 0;\n> > +}\n> > +\n> > +AwbNN::AwbNN(Controller *controller)\n> > + : Awb(controller)\n> > +{\n> > + zoneSize_ = getHardwareConfig().awbRegions;\n> > +}\n> > +\n> > +AwbNN::~AwbNN()\n> > +{\n> > +}\n> > +\n> > +char const *AwbNN::name() const\n> > +{\n> > + return NAME;\n> > +}\n> > +\n> > +int AwbNN::read(const libcamera::YamlObject ¶ms)\n> > +{\n> > + int ret;\n> > +\n> > + ret = config_.read(params);\n> > + if (ret)\n> > + return ret;\n> > +\n> > + ret = nnConfig_.read(params, config_);\n> > + if (ret)\n> > + return ret;\n> > +\n> > + return 0;\n> > +}\n> > +\n> > +static bool checkTensorShape(TfLiteTensor *tensor, const int *expectedDims, const int expectedDimsSize)\n> > +{\n> > + if (tensor->dims->size != expectedDimsSize)\n> > + return false;\n> > +\n> > + for (int i = 0; i < tensor->dims->size; i++) {\n> > + if (tensor->dims->data[i] != expectedDims[i]) {\n> > + return false;\n> > + }\n> > + }\n> > + return true;\n>\n> from \n>\n> return std::equal(expectedDims, expectedDims + expectedDimsSize,\n> tensor->dims->data, tensor->dims->data + tensor->dims->size);\n>\n> ?\n\nAgree.\n\n>\n>\n> > +}\n> > +\n> > +static std::string buildDimString(const int *dims, const int dimsSize)\n> > +{\n> > + std::string s = \"[\";\n> > + for (int i = 0; i < dimsSize; i++) {\n> > + s += std::to_string(dims[i]);\n> > + if (i < dimsSize - 1)\n> > + s += \",\";\n> > + else\n> > + s += \"]\";\n> > + }\n> > + return s;\n>\n> return '[' + utils::join(Span{ dims, dimsSize }, \",\") + ']';\n>\n> ?\n\nYes, that's nicer.\n\n>\n>\n> > +}\n> > +\n> > +void AwbNN::loadModel()\n> > +{\n> > + std::string modelPath;\n> > + if (getTarget() == \"bcm2835\") {\n> > + modelPath = \"/ipa/rpi/vc4/awb_model.tflite\";\n> > + } else {\n> > + modelPath = \"/ipa/rpi/pisp/awb_model.tflite\";\n> > + }\n> > +\n> > + if (nnConfig_.model.empty()) {\n> > + std::string root = utils::libcameraSourcePath();\n> > + if (!root.empty()) {\n> > + modelPath = root + modelPath;\n> > + } else {\n> > + modelPath = LIBCAMERA_DATA_DIR + modelPath;\n> > + }\n> > +\n> > + if (!File::exists(modelPath)) {\n> > + LOG(RPiAwb, Error) << \"No model file found in standard locations\";\n> > + nnConfig_.enableNn = false;\n> > + return;\n> > + }\n> > + } else {\n> > + modelPath = nnConfig_.model;\n> > + }\n> > +\n> > + LOG(RPiAwb, Debug) << \"Attempting to load model from: \" << modelPath;\n> > +\n> > + model_ = tflite::FlatBufferModel::BuildFromFile(modelPath.c_str());\n>\n> As far as I can see, `BuildFromFile` takes an `ErrorReporter` parameter. Would it\n> make sense to create a static instance of one and use it to route messages into\n> libcamera log? If not specified, does it report anything to stderr or similar?\n> (And the errors from tflite are logged, then I would probably also remove the\n> `File::exists()` check as well.)\n\nSo it's my belief that if you don't say anything, they go to stderr.\n\nI'm happy to remove the File::exists() check, the error message\ndoesn't even tell you where it looked, whereas the TFLite message\nwill.\n\nAgain, I'm slightly inclined not to bother with an ErrorReporter. It\nworks like printf which is slightly irritating, and then I'll just\nstart to worry about parsing the messages for errors, warnings, info\netc. and I'm not really seeing a benefit. If anyone disagrees, perhaps\nwe can leave that as a subsequent patch?\n\n>\n>\n> > +\n> > + if (!model_) {\n> > + LOG(RPiAwb, Error) << \"Failed to load model from \" << modelPath;\n> > + nnConfig_.enableNn = false;\n> > + return;\n> > + }\n> > +\n> > + tflite::MutableOpResolver resolver;\n> > + tflite::ops::builtin::BuiltinOpResolver builtin_resolver;\n> > + resolver.AddAll(builtin_resolver);\n> > + tflite::InterpreterBuilder(*model_, resolver)(&interpreter_);\n> > + if (!interpreter_) {\n> > + LOG(RPiAwb, Error) << \"Failed to build interpreter for model \" << nnConfig_.model;\n> > + nnConfig_.enableNn = false;\n> > + return;\n> > + }\n> > +\n> > + interpreter_->AllocateTensors();\n> > + TfLiteTensor *inputTensor = interpreter_->input_tensor(0);\n> > + TfLiteTensor *inputLuxTensor = interpreter_->input_tensor(1);\n> > + TfLiteTensor *outputTensor = interpreter_->output_tensor(0);\n> > + if (!inputTensor || !inputLuxTensor || !outputTensor) {\n> > + LOG(RPiAwb, Error) << \"Model missing input or output tensor\";\n> > + nnConfig_.enableNn = false;\n> > + return;\n> > + }\n> > +\n> > + const int expectedInputDims[] = { 1, (int)zoneSize_.height, (int)zoneSize_.width, 3 };\n> > + const int expectedInputLuxDims[] = { 1 };\n> > + const int expectedOutputDims[] = { 1 };\n> > +\n> > + if (!checkTensorShape(inputTensor, expectedInputDims, 4)) {\n> > + LOG(RPiAwb, Error) << \"Model input tensor dimension mismatch. Expected: \" << buildDimString(expectedInputDims, 4)\n> > + << \", Got: \" << buildDimString(inputTensor->dims->data, inputTensor->dims->size);\n> > + nnConfig_.enableNn = false;\n> > + return;\n> > + }\n> > +\n> > + if (!checkTensorShape(inputLuxTensor, expectedInputLuxDims, 1)) {\n> > + LOG(RPiAwb, Error) << \"Model input lux tensor dimension mismatch. Expected: \" << buildDimString(expectedInputLuxDims, 1)\n> > + << \", Got: \" << buildDimString(inputLuxTensor->dims->data, inputLuxTensor->dims->size);\n> > + nnConfig_.enableNn = false;\n> > + return;\n> > + }\n> > +\n> > + if (!checkTensorShape(outputTensor, expectedOutputDims, 1)) {\n> > + LOG(RPiAwb, Error) << \"Model output tensor dimension mismatch. Expected: \" << buildDimString(expectedOutputDims, 1)\n> > + << \", Got: \" << buildDimString(outputTensor->dims->data, outputTensor->dims->size);\n> > + nnConfig_.enableNn = false;\n> > + return;\n> > + }\n> > +\n> > + if (inputTensor->type != kTfLiteFloat32 || inputLuxTensor->type != kTfLiteFloat32 || outputTensor->type != kTfLiteFloat32) {\n> > + LOG(RPiAwb, Error) << \"Model input and output tensors must be float32\";\n> > + nnConfig_.enableNn = false;\n> > + return;\n> > + }\n> > +\n> > + LOG(RPiAwb, Info) << \"Model loaded successfully from \" << modelPath;\n> > + LOG(RPiAwb, Debug) << \"Model validation successful - Input Image: \"\n> > + << buildDimString(expectedInputDims, 4)\n> > + << \", Input Lux: \" << buildDimString(expectedInputLuxDims, 1)\n> > + << \", Output: \" << buildDimString(expectedOutputDims, 1) << \" floats\";\n> > +}\n> > +\n> > +void AwbNN::initialise()\n> > +{\n> > + Awb::initialise();\n> > +\n> > + if (nnConfig_.enableNn) {\n> > + loadModel();\n> > + if (!nnConfig_.enableNn) {\n> > + LOG(RPiAwb, Warning) << \"Neural Network AWB failed to load - switch to Grey method\";\n> > + config_.greyWorld = true;\n> > + config_.sensitivityR = config_.sensitivityB = 1.0;\n> > + }\n> > + }\n> > +}\n> > +\n> > +void AwbNN::prepareStats()\n> > +{\n> > + zones_.clear();\n> > + /*\n> > + * LSC has already been applied to the stats in this pipeline, so stop\n> > + * any LSC compensation. We also ignore config_.fast in this version.\n> > + */\n> > + generateStats(zones_, statistics_, 0.0, 0.0, getGlobalMetadata(), 0.0, 0.0, 0.0);\n> > + /*\n> > + * apply sensitivities, so values appear to come from our \"canonical\"\n> > + * sensor.\n> > + */\n> > + for (auto &zone : zones_) {\n> > + zone.R *= config_.sensitivityR;\n> > + zone.B *= config_.sensitivityB;\n> > + }\n> > +}\n> > +\n> > +void AwbNN::transverseSearch(double t, double &r, double &b)\n> > +{\n> > + int spanR = -1, spanB = -1;\n> > + config_.ctR.eval(t, &spanR);\n> > + config_.ctB.eval(t, &spanB);\n> > +\n> > + const int diff = 10;\n> > + double rDiff = config_.ctR.eval(t + diff, &spanR) -\n> > + config_.ctR.eval(t - diff, &spanR);\n> > + double bDiff = config_.ctB.eval(t + diff, &spanB) -\n> > + config_.ctB.eval(t - diff, &spanB);\n> > +\n> > + ipa::Pwl::Point transverse({ bDiff, -rDiff });\n> > + if (transverse.length2() < 1e-6)\n> > + return;\n> > +\n> > + transverse = transverse / transverse.length();\n> > + double transverseRange = config_.transverseNeg + config_.transversePos;\n> > + const int maxNumDeltas = 12;\n> > + int numDeltas = floor(transverseRange * 100 + 0.5) + 1;\n> > + numDeltas = numDeltas < 3 ? 3 : (numDeltas > maxNumDeltas ? maxNumDeltas : numDeltas);\n>\n> int numDeltas = std::clamp(floor(transverseRange * 100 + 0.5) + 1, 3, maxNumDeltas);\n>\n> ?\n\nYes, will tidy that.\n\n>\n>\n> > +\n> > + ipa::Pwl::Point points[maxNumDeltas];\n> > + int bestPoint = 0;\n> > +\n> > + for (int i = 0; i < numDeltas; i++) {\n> > + points[i][0] = -config_.transverseNeg +\n> > + (transverseRange * i) / (numDeltas - 1);\n> > + ipa::Pwl::Point rbTest = ipa::Pwl::Point({ r, b }) +\n> > + transverse * points[i].x();\n> > + double rTest = rbTest.x(), bTest = rbTest.y();\n> > + double gainR = 1 / rTest, gainB = 1 / bTest;\n> > + double delta2Sum = computeDelta2Sum(gainR, gainB, 0.0, 0.0);\n> > + points[i][1] = delta2Sum;\n> > + if (points[i].y() < points[bestPoint].y())\n> > + bestPoint = i;\n> > + }\n> > +\n> > + bestPoint = std::clamp(bestPoint, 1, numDeltas - 2);\n> > + ipa::Pwl::Point rbBest = ipa::Pwl::Point({ r, b }) +\n> > + transverse * interpolateQuadatric(points[bestPoint - 1],\n> > + points[bestPoint],\n> > + points[bestPoint + 1]);\n> > + double rBest = rbBest.x(), bBest = rbBest.y();\n> > +\n> > + r = rBest, b = bBest;\n>\n> r = rbBest.x();\n> b = rbBest.y();\n>\n> ?\n\nYes, this too.\n\n>\n>\n> > +}\n> > +\n> > +AwbNN::RGB AwbNN::processZone(AwbNN::RGB zone, float redGain, float blueGain)\n> > +{\n> > + /*\n> > + * Renders the pixel at canonical network colour temperature\n> > + */\n> > + RGB zoneGains = zone;\n> > +\n> > + zoneGains.R *= redGain;\n> > + zoneGains.G *= 1.0;\n> > + zoneGains.B *= blueGain;\n> > +\n> > + RGB zoneCcm;\n> > +\n> > + zoneCcm.R = nnConfig_.ccm[0] * zoneGains.R + nnConfig_.ccm[1] * zoneGains.G + nnConfig_.ccm[2] * zoneGains.B;\n> > + zoneCcm.G = nnConfig_.ccm[3] * zoneGains.R + nnConfig_.ccm[4] * zoneGains.G + nnConfig_.ccm[5] * zoneGains.B;\n> > + zoneCcm.B = nnConfig_.ccm[6] * zoneGains.R + nnConfig_.ccm[7] * zoneGains.G + nnConfig_.ccm[8] * zoneGains.B;\n> > +\n> > + return zoneCcm;\n> > +}\n> > +\n> > +void AwbNN::awbNN()\n> > +{\n> > + float *inputData = interpreter_->typed_input_tensor(0);\n> > + float *inputLux = interpreter_->typed_input_tensor(1);\n> > +\n> > + float redGain = 1.0 / config_.ctR.eval(kNetworkCanonicalCT);\n> > + float blueGain = 1.0 / config_.ctB.eval(kNetworkCanonicalCT);\n> > +\n> > + for (uint i = 0; i < zoneSize_.height; i++) {\n> > + for (uint j = 0; j < zoneSize_.width; j++) {\n> > + uint zoneIdx = i * zoneSize_.width + j;\n> > +\n> > + RGB processedZone = processZone(zones_[zoneIdx] * (1.0 / 65535), redGain, blueGain);\n> > + uint baseIdx = zoneIdx * 3;\n>\n> Where is this `uint` type coming from? tflite? Given that `zoneSize_` is `libcamera::Size`,\n> which uses `unsigned int`, is this `uint` type necessary here?\n\nIndeed, uint isn't a standard thing, though it often gets defined.\nunsigned int is clearly more portable and better.\n\n>\n>\n> > +\n> > + inputData[baseIdx + 0] = static_cast(processedZone.R);\n> > + inputData[baseIdx + 1] = static_cast(processedZone.G);\n> > + inputData[baseIdx + 2] = static_cast(processedZone.B);\n> > + }\n> > + }\n> > +\n> > + inputLux[0] = static_cast(lux_);\n> > +\n> > + TfLiteStatus status = interpreter_->Invoke();\n> > + if (status != kTfLiteOk) {\n> > + LOG(RPiAwb, Error) << \"Model inference failed with status: \" << status;\n> > + return;\n> > + }\n> > +\n> > + float *outputData = interpreter_->typed_output_tensor(0);\n> > +\n> > + double t = outputData[0];\n> > +\n> > + LOG(RPiAwb, Debug) << \"Model output temperature: \" << t;\n> > +\n> > + t = std::clamp(t, mode_->ctLo, mode_->ctHi);\n> > +\n> > + double r = config_.ctR.eval(t);\n> > + double b = config_.ctB.eval(t);\n> > +\n> > + transverseSearch(t, r, b);\n> > +\n> > + LOG(RPiAwb, Debug) << \"After transverse search: Temperature: \" << t << \" Red gain: \" << 1.0 / r << \" Blue gain: \" << 1.0 / b;\n> > +\n> > + asyncResults_.temperatureK = t;\n> > + asyncResults_.gainR = 1.0 / r * config_.sensitivityR;\n> > + asyncResults_.gainG = 1.0;\n> > + asyncResults_.gainB = 1.0 / b * config_.sensitivityB;\n> > +}\n> > +\n> > +void AwbNN::doAwb()\n> > +{\n> > + prepareStats();\n> > + if (zones_.size() == (zoneSize_.width * zoneSize_.height) && nnConfig_.enableNn)\n> > + awbNN();\n> > + else\n> > + awbGrey();\n> > + statistics_.reset();\n> > +}\n> > +\n> > +/* Register algorithm with the system. */\n> > +static Algorithm *create(Controller *controller)\n> > +{\n> > + return (Algorithm *)new AwbNN(controller);\n>\n> Please omit this cast.\n\nActually I think there are quite a few of these. I'll fix it in the\nfiles touched by this patch set, but the others will have to wait!\n\nWill post a v6 shortly.\n\nBest regards\nDavid\n\n>\n>\n> Regards,\n> Barnabás Pőcze\n>\n> > +}\n> > +static RegisterAlgorithm reg(NAME, &create);\n> > +\n> > +} /* namespace RPiController */\n>","headers":{"Return-Path":"","X-Original-To":"parsemail@patchwork.libcamera.org","Delivered-To":"parsemail@patchwork.libcamera.org","Received":["from lancelot.ideasonboard.com (lancelot.ideasonboard.com\n\t[92.243.16.209])\n\tby patchwork.libcamera.org (Postfix) with ESMTPS id F3F5DC3200\n\tfor ;\n\tTue, 27 Jan 2026 17:10:16 +0000 (UTC)","from lancelot.ideasonboard.com (localhost [IPv6:::1])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTP id AAF0461FD1;\n\tTue, 27 Jan 2026 18:10:16 +0100 (CET)","from mail-qv1-xf2a.google.com (mail-qv1-xf2a.google.com\n\t[IPv6:2607:f8b0:4864:20::f2a])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id C3C6561FC4\n\tfor ;\n\tTue, 27 Jan 2026 18:10:14 +0100 (CET)","by mail-qv1-xf2a.google.com with SMTP id\n\t6a1803df08f44-88a3d2f3299so63556026d6.2\n\tfor ;\n\tTue, 27 Jan 2026 09:10:14 -0800 (PST)"],"Authentication-Results":"lancelot.ideasonboard.com; dkim=pass (2048-bit key;\n\tunprotected) header.d=raspberrypi.com header.i=@raspberrypi.com\n\theader.b=\"ozr7Y2En\"; dkim-atps=neutral","ARC-Seal":"i=1; a=rsa-sha256; t=1769533814; cv=none;\n\td=google.com; s=arc-20240605;\n\tb=E++NGfZjW5MZQgzPlOF+hcAzQ1vWfPWtphwAXSi/8ZJyMqFCBogjVfz7pJhEOuNVN9\n\tfH9lXLcnZwi4MZosmnH9nOBzBG3GtCfeT5Wnq8DQqr2LJkOJUfC283bX/T//5TN0JGJO\n\tshMiiE8YpwuKqAvFGoF8ukpZkOklonk/6W4NLVRCag68NlTihVug9yL/fjFyNCib5EZ3\n\tfK6a4ssnrf45CFwMUUfH/MmKbjkn5gGYDwJr/euhw+0eGOvcRtb275q5ZMgCXs18noNV\n\t+shaxFomSDWDt1Ub9PnzzHBKDJ+0BNXVbad3XMuOXAFhyJ2GO0i/zXSVLT4u6CFS5HF7\n\totQA==","ARC-Message-Signature":"i=1; a=rsa-sha256; c=relaxed/relaxed; d=google.com;\n\ts=arc-20240605; \n\th=content-transfer-encoding:cc:to:subject:message-id:date:from\n\t:in-reply-to:references:mime-version:dkim-signature;\n\tbh=Y3jPGU4cptMDjg6AQmvxWzRMZZzZNsuqoK+nU+cW6lg=;\n\tfh=BgHaaLnEzGn36KDRwWemjp6pEHNEfWuRaD9lws6BCZE=;\n\tb=OMZGUCPJOtM5ojN68PulegCHr0ebW6Mq+ZSKwtba6ufV8O5eW7I0ebfOFYAVOokZkf\n\tTcgW/ZexXN3TFXViweteioy8yIe4NNjrYb7WlQeElrS2qET+SKyfdf7MYN+WWuAW+Cr5\n\tg4D68QtpVm6avFaszqH4gAXLfhS6Cc1euM6SyFk1MrcIthlW7HkVdVqUuqh6kqxiM0ZU\n\tp1+uSg1ZyhFHGC7xT/p9GTN6POYotkt2k8L667OA7c2uTRgCWPs0UbZSWQx1Yk5BvHxF\n\txHXgYa+5XBTnirXcAJgyIvbtndK5Cp9DwMvzlXXvik94d68JJ8T05q/4Nz0pJIc+AqGD\n\tTAGg==; darn=lists.libcamera.org","ARC-Authentication-Results":"i=1; mx.google.com; arc=none","DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/relaxed;\n\td=raspberrypi.com; s=google; t=1769533814; x=1770138614;\n\tdarn=lists.libcamera.org; \n\th=content-transfer-encoding:cc:to:subject:message-id:date:from\n\t:in-reply-to:references:mime-version:from:to:cc:subject:date\n\t:message-id:reply-to;\n\tbh=Y3jPGU4cptMDjg6AQmvxWzRMZZzZNsuqoK+nU+cW6lg=;\n\tb=ozr7Y2Enn50iuwHBAQXBv5U34NXnY6gz9/Cic2DCPBAniEONffjNLyBBqKEGmzKIxO\n\tPVu3xLImLFEBuUsWDlzu+yOygTCyI0CEUb40VHDPllyEsXW0HC+b4ym5MLsq5oh6d2s0\n\tucfrTtouPrjv2StkPLmGiFP4dQ6eXBTaqhenncBidcav1TRaq/CEa3k6PVn0LJUpk1oM\n\tYL1aw3hWsMTRGbsMy02Z0raDhk7J5C8XtGMgl346CAlPKOPK2tiub3nikkQQ/lbCKvI7\n\tktFsdJlKiD+AReEIcV5BpAk++FCeN3A8ZEu9jiWPQhlcv4hayVnTIiqLwkUj7fVooNIF\n\tvOXw==","X-Google-DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/relaxed;\n\td=1e100.net; s=20230601; t=1769533814; x=1770138614;\n\th=content-transfer-encoding:cc:to:subject:message-id:date:from\n\t:in-reply-to:references:mime-version:x-gm-gg:x-gm-message-state:from\n\t:to:cc:subject:date:message-id:reply-to;\n\tbh=Y3jPGU4cptMDjg6AQmvxWzRMZZzZNsuqoK+nU+cW6lg=;\n\tb=S+T/6SssvgFnWILTSEh7M7wZdSUnkS85SHgdFxu7nigjVxn4ktNI/km4aj06jn5elb\n\tcslQaVI9W+xr/RZOaJ5eDzhROwAMkZS3gOOB1HgpWcde3/0B6uC1SvDezJOhFv+1p/Hv\n\tBMGAZBWb/hZemOAVJZl1d2ISipsBylcU/QGs2ztW/9T8Yykw++yDfZn0xSaoiCD+cChc\n\t1mPHUFiB3bU1epYmcyZTx5D/2Wvg9QnhyLq+JONEa1IVrg8ThRFx6RkDVsZpJDXwvEur\n\taAmuHT2GPVokxF3geM+dY8obe87JwMKnwmqVPAae4R1f5Mndfa88V/g+9H50xPHdI503\n\tuwQA==","X-Gm-Message-State":"AOJu0YxHNByIMJ0DuCOXMCYaOwBqCFtbsuJGOff2hsZKiT2h1MUj66jP\n\tZXXY1ASlShwd3XZ4KKdnkpS2Y1SP6BS5s94eDTYnAeHypPtKGTO9wOvn9b3/Ec32KoTbO47AiXL\n\t81JXVFmt2CBZZnSAup3u9Da1452eBSLJIayxzqP0xESXlLbUtKrPaclA=","X-Gm-Gg":"AZuq6aIA+7fxCRr9GdxNUC5TrVooE87dNRqXoIosO3Vy9XAkCxqhOViRSqsI8v5EJdx\n\tcF7naMsqFc6Qx74jUwHGYdhX5X5eQYF+Rd08iWYLg+wAH4d1Iel4G3KKzmt42FH+Dut2C/0+MFZ\n\tbzAmIQWqmpDMrVoW8LJRJl8ah2nyRVe/bhSGmjTvKL915d3zyfPE0Q61N3aVdJq+VMyxmbET0D4\n\tfuW12WkMX3Lq78gfZKWuM7pwmgX+VXOaosFPvnyAn/e8QO9AZy4NVvJQcSdCDZvDbqqL+3Omqet\n\tthMzsZFgJRZJYUKSJNkh9wOctIegljr8EF2Dupw7hpnnIcTUutYn0OL+Q0xyZj02SjoyX5LbPGw\n\t09aggOGfEyrV2","X-Received":"by 2002:a05:6214:f62:b0:882:3759:9158 with SMTP id\n\t6a1803df08f44-894cc989dffmr35448026d6.61.1769533813423;\n\tTue, 27 Jan 2026 09:10:13 -0800 (PST)","MIME-Version":"1.0","References":"<20260127120604.6560-1-david.plowman@raspberrypi.com>\n\t<20260127120604.6560-3-david.plowman@raspberrypi.com>\n\t<1de0b8bf-c56d-45cc-85c5-b39334b80bf2@ideasonboard.com>","In-Reply-To":"<1de0b8bf-c56d-45cc-85c5-b39334b80bf2@ideasonboard.com>","From":"David Plowman ","Date":"Tue, 27 Jan 2026 17:10:01 +0000","X-Gm-Features":"AZwV_QjKrvKCZ8_hU6c7qeQm7kdjIR7wVse0GiBER2IHHv4yC3Fcf1sQpSSsF6g","Message-ID":"","Subject":"Re: [PATCH v5 2/4] ipa: rpi: controller: awb: Add Neural Network AWB","To":"=?utf-8?q?Barnab=C3=A1s_P=C5=91cze?= ","Cc":"libcamera-devel@lists.libcamera.org, \n\tPeter Bailey ,\n\tNaushir Patuck ","Content-Type":"text/plain; charset=\"UTF-8\"","Content-Transfer-Encoding":"quoted-printable","X-BeenThere":"libcamera-devel@lists.libcamera.org","X-Mailman-Version":"2.1.29","Precedence":"list","List-Id":"","List-Unsubscribe":",\n\t","List-Archive":"","List-Post":"","List-Help":"","List-Subscribe":",\n\t","Errors-To":"libcamera-devel-bounces@lists.libcamera.org","Sender":"\"libcamera-devel\" "}}]