Patch Detail
Show a patch.
GET /api/1.1/patches/14998/?format=api
{ "id": 14998, "url": "https://patchwork.libcamera.org/api/1.1/patches/14998/?format=api", "web_url": "https://patchwork.libcamera.org/patch/14998/", "project": { "id": 1, "url": "https://patchwork.libcamera.org/api/1.1/projects/1/?format=api", "name": "libcamera", "link_name": "libcamera", "list_id": "libcamera_core", "list_email": "libcamera-devel@lists.libcamera.org", "web_url": "", "scm_url": "", "webscm_url": "" }, "msgid": "<20211202180410.518232-5-jeanmichel.hautbois@ideasonboard.com>", "date": "2021-12-02T18:04:08", "name": "[libcamera-devel,v1,4/6] ipa: rkisp1: Introduce AWB", "commit_ref": null, "pull_url": null, "state": "changes-requested", "archived": false, "hash": "3e7870c416daf130c9e6bdce7ed3d3f60cb4e1b5", "submitter": { "id": 75, "url": "https://patchwork.libcamera.org/api/1.1/people/75/?format=api", "name": "Jean-Michel Hautbois", "email": "jeanmichel.hautbois@ideasonboard.com" }, "delegate": null, "mbox": "https://patchwork.libcamera.org/patch/14998/mbox/", "series": [ { "id": 2805, "url": "https://patchwork.libcamera.org/api/1.1/series/2805/?format=api", "web_url": "https://patchwork.libcamera.org/project/libcamera/list/?series=2805", "date": "2021-12-02T18:04:04", "name": "IPA RkISP1 awb and misc improvements", "version": 1, "mbox": "https://patchwork.libcamera.org/series/2805/mbox/" } ], "comments": "https://patchwork.libcamera.org/api/patches/14998/comments/", "check": "pending", "checks": "https://patchwork.libcamera.org/api/patches/14998/checks/", "tags": {}, "headers": { "Return-Path": "<libcamera-devel-bounces@lists.libcamera.org>", "X-Original-To": "parsemail@patchwork.libcamera.org", "Delivered-To": "parsemail@patchwork.libcamera.org", "Received": [ "from lancelot.ideasonboard.com (lancelot.ideasonboard.com\n\t[92.243.16.209])\n\tby patchwork.libcamera.org (Postfix) with ESMTPS id 81664BDB13\n\tfor <parsemail@patchwork.libcamera.org>;\n\tThu, 2 Dec 2021 18:04:22 +0000 (UTC)", "from lancelot.ideasonboard.com (localhost [IPv6:::1])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTP id C97F86087A;\n\tThu, 2 Dec 2021 19:04:21 +0100 (CET)", "from perceval.ideasonboard.com (perceval.ideasonboard.com\n\t[213.167.242.64])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id 4737660832\n\tfor <libcamera-devel@lists.libcamera.org>;\n\tThu, 2 Dec 2021 19:04:16 +0100 (CET)", "from tatooine.ideasonboard.com (unknown\n\t[IPv6:2a01:e0a:169:7140:7fc3:78ca:aeee:c4f2])\n\tby perceval.ideasonboard.com (Postfix) with ESMTPSA id 0C0D7B7E;\n\tThu, 2 Dec 2021 19:04:16 +0100 (CET)" ], "Authentication-Results": "lancelot.ideasonboard.com;\n\tdkim=fail reason=\"signature verification failed\" (1024-bit key;\n\tunprotected) header.d=ideasonboard.com header.i=@ideasonboard.com\n\theader.b=\"Cp9afH0H\"; dkim-atps=neutral", "DKIM-Signature": "v=1; a=rsa-sha256; c=relaxed/simple; d=ideasonboard.com;\n\ts=mail; t=1638468256;\n\tbh=+P1PRbbKluiWqN0D4e88UsmtAEl5YMGxAHgOWtfnoo4=;\n\th=From:To:Cc:Subject:Date:In-Reply-To:References:From;\n\tb=Cp9afH0HpoGU/+uL4DNPltMBTkyA2Jp1HAdzZGcpkjs1NDSKiuPewcjO5TIsMRSTK\n\trncP3fjz6Q/wo6lGG/QjTyFvFJ3CBSrmXJ0zFyRf+nGCIMiCOH7gfUvwLfixjtkIvC\n\tSNAwhHkHWvgyZ1PHEK3tqE7cn9bgbBBpUlzBUgf8=", "From": "Jean-Michel Hautbois <jeanmichel.hautbois@ideasonboard.com>", "To": "libcamera-devel@lists.libcamera.org", "Date": "Thu, 2 Dec 2021 19:04:08 +0100", "Message-Id": "<20211202180410.518232-5-jeanmichel.hautbois@ideasonboard.com>", "X-Mailer": "git-send-email 2.32.0", "In-Reply-To": "<20211202180410.518232-1-jeanmichel.hautbois@ideasonboard.com>", "References": "<20211202180410.518232-1-jeanmichel.hautbois@ideasonboard.com>", "MIME-Version": "1.0", "Content-Transfer-Encoding": "8bit", "Subject": "[libcamera-devel] [PATCH v1 4/6] ipa: rkisp1: Introduce AWB", "X-BeenThere": "libcamera-devel@lists.libcamera.org", "X-Mailman-Version": "2.1.29", "Precedence": "list", "List-Id": "<libcamera-devel.lists.libcamera.org>", "List-Unsubscribe": "<https://lists.libcamera.org/options/libcamera-devel>,\n\t<mailto:libcamera-devel-request@lists.libcamera.org?subject=unsubscribe>", "List-Archive": "<https://lists.libcamera.org/pipermail/libcamera-devel/>", "List-Post": "<mailto:libcamera-devel@lists.libcamera.org>", "List-Help": "<mailto:libcamera-devel-request@lists.libcamera.org?subject=help>", "List-Subscribe": "<https://lists.libcamera.org/listinfo/libcamera-devel>,\n\t<mailto:libcamera-devel-request@lists.libcamera.org?subject=subscribe>", "Errors-To": "libcamera-devel-bounces@lists.libcamera.org", "Sender": "\"libcamera-devel\" <libcamera-devel-bounces@lists.libcamera.org>" }, "content": "The RkISP1 ISP calculates a mean value for Y, Cr and Cb at each frame.\nThere is a RGB mode which could theoretically give us the values for R,\nG and B directly, but it seems to be failing right now.\n\nConvert those values into R, G and B and estimate the gain to apply in a\ngrey world.\n\nSigned-off-by: Jean-Michel Hautbois <jeanmichel.hautbois@ideasonboard.com>\n---\n src/ipa/rkisp1/algorithms/awb.cpp | 149 ++++++++++++++++++++++++++\n src/ipa/rkisp1/algorithms/awb.h | 33 ++++++\n src/ipa/rkisp1/algorithms/meson.build | 1 +\n src/ipa/rkisp1/ipa_context.cpp | 28 +++++\n src/ipa/rkisp1/ipa_context.h | 17 +++\n src/ipa/rkisp1/rkisp1.cpp | 2 +\n 6 files changed, 230 insertions(+)\n create mode 100644 src/ipa/rkisp1/algorithms/awb.cpp\n create mode 100644 src/ipa/rkisp1/algorithms/awb.h", "diff": "diff --git a/src/ipa/rkisp1/algorithms/awb.cpp b/src/ipa/rkisp1/algorithms/awb.cpp\nnew file mode 100644\nindex 000000000..0ec94764a\n--- /dev/null\n+++ b/src/ipa/rkisp1/algorithms/awb.cpp\n@@ -0,0 +1,149 @@\n+/* SPDX-License-Identifier: LGPL-2.1-or-later */\n+/*\n+ * Copyright (C) 2021, Ideas On Board\n+ *\n+ * awb.cpp - AWB control algorithm\n+ */\n+\n+#include \"awb.h\"\n+\n+#include <algorithm>\n+#include <cmath>\n+\n+#include <libcamera/base/log.h>\n+\n+#include <libcamera/ipa/core_ipa_interface.h>\n+\n+/**\n+ * \\file awb.h\n+ */\n+\n+namespace libcamera {\n+\n+namespace ipa::rkisp1::algorithms {\n+\n+/**\n+ * \\class Awb\n+ * \\brief A Grey world white balance correction algorithm\n+ */\n+\n+LOG_DEFINE_CATEGORY(RkISP1Awb)\n+\n+/**\n+ * \\copydoc libcamera::ipa::Algorithm::configure\n+ */\n+int Awb::configure(IPAContext &context,\n+\t\t const IPACameraSensorInfo &configInfo)\n+{\n+\tcontext.frameContext.awb.gains.red = 1.0;\n+\tcontext.frameContext.awb.gains.blue = 1.0;\n+\tcontext.frameContext.awb.gains.green = 1.0;\n+\n+\t/* Define the measurement window for AGC. */\n+\tcontext.configuration.awb.measureWindow.h_offs = configInfo.outputSize.width / 8;\n+\tcontext.configuration.awb.measureWindow.v_offs = configInfo.outputSize.height / 8;\n+\tcontext.configuration.awb.measureWindow.h_size = 3 * configInfo.outputSize.width / 4;\n+\tcontext.configuration.awb.measureWindow.v_size = 3 * configInfo.outputSize.height / 4;\n+\n+\treturn 0;\n+}\n+\n+uint32_t Awb::estimateCCT(double red, double green, double blue)\n+{\n+\t/* Convert the RGB values to CIE tristimulus values (XYZ) */\n+\tdouble X = (-0.14282) * (red) + (1.54924) * (green) + (-0.95641) * (blue);\n+\tdouble Y = (-0.32466) * (red) + (1.57837) * (green) + (-0.73191) * (blue);\n+\tdouble Z = (-0.68202) * (red) + (0.77073) * (green) + (0.56332) * (blue);\n+\n+\t/* Calculate the normalized chromaticity values */\n+\tdouble x = X / (X + Y + Z);\n+\tdouble y = Y / (X + Y + Z);\n+\n+\t/* Calculate CCT */\n+\tdouble n = (x - 0.3320) / (0.1858 - y);\n+\treturn 449 * n * n * n + 3525 * n * n + 6823.3 * n + 5520.33;\n+}\n+\n+/**\n+ * \\copydoc libcamera::ipa::Algorithm::process\n+ */\n+void Awb::process([[maybe_unused]] IPAContext &context, const rkisp1_stat_buffer *stats)\n+{\n+\tconst rkisp1_cif_isp_stat *params = &stats->params;\n+\tconst rkisp1_cif_isp_awb_stat *awb = ¶ms->awb;\n+\n+\t/* Get the YCbCr mean values */\n+\tdouble yMean = awb->awb_mean[0].mean_y_or_g;\n+\tdouble crMean = awb->awb_mean[0].mean_cr_or_r;\n+\tdouble cbMean = awb->awb_mean[0].mean_cb_or_b;\n+\n+\t/* Convert from YCbCr to RGB. */\n+\tdouble redMean = yMean + 1.402 * (crMean - 128);\n+\tdouble blueMean = yMean + 1.772 * (cbMean - 128);\n+\tdouble greenMean = yMean - 0.34414 * (cbMean - 128) - 0.71414 * (crMean - 128);\n+\n+\t/* Estimate the red and blue gains to apply in a grey world. */\n+\tdouble redGain = greenMean / (redMean + 1);\n+\tdouble blueGain = greenMean / (blueMean + 1);\n+\n+\t/* Filter the values to avoid oscillations. */\n+\tIPAFrameContext &frameContext = context.frameContext;\n+\n+\tframeContext.awb.temperatureK = estimateCCT(redMean, greenMean, blueMean);\n+\tframeContext.awb.gains.red = 0.2 * redGain +\n+\t\t\t\t 0.8 * frameContext.awb.gains.red;\n+\tframeContext.awb.gains.blue = 0.2 * blueGain +\n+\t\t\t\t 0.8 * frameContext.awb.gains.blue;\n+\t/* Hardcode the green gain to 1.0. */\n+\tframeContext.awb.gains.green = 1.0;\n+\n+\tLOG(RkISP1Awb, Debug) << \"Gain found for red: \" << context.frameContext.awb.gains.red\n+\t\t\t << \" and for blue: \" << context.frameContext.awb.gains.blue;\n+}\n+\n+/**\n+ * \\copydoc libcamera::ipa::Algorithm::prepare\n+ */\n+void Awb::prepare([[maybe_unused]] IPAContext &context,\n+\t\t rkisp1_params_cfg *params)\n+{\n+\tparams->others.awb_gain_config.gain_green_b = 256 * context.frameContext.awb.gains.green;\n+\tparams->others.awb_gain_config.gain_blue = 256 * context.frameContext.awb.gains.blue;\n+\tparams->others.awb_gain_config.gain_red = 256 * context.frameContext.awb.gains.red;\n+\tparams->others.awb_gain_config.gain_green_r = 256 * context.frameContext.awb.gains.green;\n+\n+\t/* Configure the gains to apply. */\n+\tparams->module_en_update |= RKISP1_CIF_ISP_MODULE_AWB_GAIN;\n+\t/* Update the ISP to apply the gains configured. */\n+\tparams->module_ens |= RKISP1_CIF_ISP_MODULE_AWB_GAIN;\n+\tparams->module_cfg_update |= RKISP1_CIF_ISP_MODULE_AWB_GAIN;\n+\n+\t/* Configure the measure window for AWB. */\n+\tparams->meas.awb_meas_config.awb_wnd = context.configuration.awb.measureWindow;\n+\t/*\n+\t * Measure Y, Cr and Cb means.\n+\t * \\todo RGB is not working, the kernel seems to not configure it ?\n+\t */\n+\tparams->meas.awb_meas_config.awb_mode = RKISP1_CIF_ISP_AWB_MODE_YCBCR;\n+\t/* Reference Cr and Cb. */\n+\tparams->meas.awb_meas_config.awb_ref_cb = 128;\n+\tparams->meas.awb_meas_config.awb_ref_cr = 128;\n+\t/* Y values to include are between min_y and max_y only. */\n+\tparams->meas.awb_meas_config.min_y = 16;\n+\tparams->meas.awb_meas_config.max_y = 250;\n+\t/* Maximum Cr+Cb value to take into account for awb. */\n+\tparams->meas.awb_meas_config.max_csum = 250;\n+\t/* Minimum Cr and Cb values to take into account. */\n+\tparams->meas.awb_meas_config.min_c = 16;\n+\t/* Number of frames to use to estimate the mean (0 means 1 frame). */\n+\tparams->meas.awb_meas_config.frames = 0;\n+\t/* Update AWB measurement unit configuration. */\n+\tparams->module_cfg_update |= RKISP1_CIF_ISP_MODULE_AWB;\n+\t/* Make sure the ISP is measuring the means for the next frame. */\n+\tparams->module_en_update |= RKISP1_CIF_ISP_MODULE_AWB;\n+\tparams->module_ens |= RKISP1_CIF_ISP_MODULE_AWB;\n+}\n+\n+} /* namespace ipa::rkisp1::algorithms */\n+\n+} /* namespace libcamera */\ndiff --git a/src/ipa/rkisp1/algorithms/awb.h b/src/ipa/rkisp1/algorithms/awb.h\nnew file mode 100644\nindex 000000000..0a9fb82c1\n--- /dev/null\n+++ b/src/ipa/rkisp1/algorithms/awb.h\n@@ -0,0 +1,33 @@\n+/* SPDX-License-Identifier: LGPL-2.1-or-later */\n+/*\n+ * Copyright (C) 2021, Ideas On Board\n+ *\n+ * awb.h - AWB control algorithm\n+ */\n+\n+#pragma once\n+\n+#include <linux/rkisp1-config.h>\n+\n+#include \"algorithm.h\"\n+\n+namespace libcamera {\n+\n+namespace ipa::rkisp1::algorithms {\n+\n+class Awb : public Algorithm\n+{\n+public:\n+\tAwb() = default;\n+\t~Awb() = default;\n+\n+\tint configure(IPAContext &context, const IPACameraSensorInfo &configInfo) override;\n+\tvoid prepare(IPAContext &context, rkisp1_params_cfg *params) override;\n+\tvoid process(IPAContext &context, const rkisp1_stat_buffer *stats) override;\n+\n+private:\n+\tuint32_t estimateCCT(double red, double green, double blue);\n+};\n+\n+} /* namespace ipa::rkisp1::algorithms */\n+} /* namespace libcamera */\ndiff --git a/src/ipa/rkisp1/algorithms/meson.build b/src/ipa/rkisp1/algorithms/meson.build\nindex e6767aa5f..faec0eb3e 100644\n--- a/src/ipa/rkisp1/algorithms/meson.build\n+++ b/src/ipa/rkisp1/algorithms/meson.build\n@@ -2,6 +2,7 @@\n \n rkisp1_ipa_algorithms = files([\n 'agc.cpp',\n+ 'awb.cpp',\n 'blc.cpp',\n 'sdg.cpp',\n ])\ndiff --git a/src/ipa/rkisp1/ipa_context.cpp b/src/ipa/rkisp1/ipa_context.cpp\nindex 992c92256..2f53eb2f6 100644\n--- a/src/ipa/rkisp1/ipa_context.cpp\n+++ b/src/ipa/rkisp1/ipa_context.cpp\n@@ -78,6 +78,14 @@ namespace libcamera::ipa::rkisp1 {\n * \\brief Hardware revision of the ISP\n */\n \n+/**\n+ * \\var IPASessionConfiguration::awb\n+ * \\brief AWB parameters configuration of the IPA\n+ *\n+ * \\var IPASessionConfiguration::awb.measureWindow\n+ * \\brief AWB measure window\n+ */\n+\n /**\n * \\var IPASessionConfiguration::sensor\n * \\brief Sensor-specific configuration of the IPA\n@@ -102,6 +110,26 @@ namespace libcamera::ipa::rkisp1 {\n * The gain should be adapted to the sensor specific gain code before applying.\n */\n \n+/**\n+ * \\var IPAFrameContext::awb\n+ * \\brief Context for the Automatic White Balance algorithm\n+ *\n+ * \\struct IPAFrameContext::awb.gains\n+ * \\brief White balance gains\n+ *\n+ * \\var IPAFrameContext::awb.gains.red\n+ * \\brief White balance gain for R channel\n+ *\n+ * \\var IPAFrameContext::awb.gains.green\n+ * \\brief White balance gain for G channel\n+ *\n+ * \\var IPAFrameContext::awb.gains.blue\n+ * \\brief White balance gain for B channel\n+ *\n+ * \\var IPAFrameContext::awb.temperatureK\n+ * \\brief Estimated color temperature\n+ */\n+\n /**\n * \\var IPAFrameContext::sensor\n * \\brief Effective sensor values\ndiff --git a/src/ipa/rkisp1/ipa_context.h b/src/ipa/rkisp1/ipa_context.h\nindex c447369f3..51eae8b61 100644\n--- a/src/ipa/rkisp1/ipa_context.h\n+++ b/src/ipa/rkisp1/ipa_context.h\n@@ -12,6 +12,8 @@\n \n #include <libcamera/base/utils.h>\n \n+#include <libcamera/geometry.h>\n+\n namespace libcamera {\n \n namespace ipa::rkisp1 {\n@@ -22,8 +24,13 @@ struct IPASessionConfiguration {\n \t\tutils::Duration maxShutterSpeed;\n \t\tdouble minAnalogueGain;\n \t\tdouble maxAnalogueGain;\n+\t\tstruct rkisp1_cif_isp_window measureWindow;\n \t} agc;\n \n+\tstruct {\n+\t\tstruct rkisp1_cif_isp_window measureWindow;\n+\t} awb;\n+\n \tstruct {\n \t\tutils::Duration lineDuration;\n \t} sensor;\n@@ -39,6 +46,16 @@ struct IPAFrameContext {\n \t\tdouble gain;\n \t} agc;\n \n+\tstruct {\n+\t\tstruct {\n+\t\t\tdouble red;\n+\t\t\tdouble green;\n+\t\t\tdouble blue;\n+\t\t} gains;\n+\n+\t\tdouble temperatureK;\n+\t} awb;\n+\n \tstruct {\n \t\tuint32_t exposure;\n \t\tdouble gain;\ndiff --git a/src/ipa/rkisp1/rkisp1.cpp b/src/ipa/rkisp1/rkisp1.cpp\nindex ec879ba37..979f28420 100644\n--- a/src/ipa/rkisp1/rkisp1.cpp\n+++ b/src/ipa/rkisp1/rkisp1.cpp\n@@ -27,6 +27,7 @@\n \n #include \"algorithms/agc.h\"\n #include \"algorithms/algorithm.h\"\n+#include \"algorithms/awb.h\"\n #include \"algorithms/blc.h\"\n #include \"algorithms/sdg.h\"\n #include \"libipa/camera_sensor_helper.h\"\n@@ -127,6 +128,7 @@ int IPARkISP1::init(const IPASettings &settings, unsigned int hwRevision)\n \n \t/* Construct our Algorithms */\n \talgorithms_.push_back(std::make_unique<algorithms::Agc>());\n+\talgorithms_.push_back(std::make_unique<algorithms::Awb>());\n \talgorithms_.push_back(std::make_unique<algorithms::BlackLevelCorrection>());\n \talgorithms_.push_back(std::make_unique<algorithms::SensorDeGamma>());\n \n", "prefixes": [ "libcamera-devel", "v1", "4/6" ] }