Patch Detail
Show a patch.
GET /api/1.1/patches/15381/?format=api
{ "id": 15381, "url": "https://patchwork.libcamera.org/api/1.1/patches/15381/?format=api", "web_url": "https://patchwork.libcamera.org/patch/15381/", "project": { "id": 1, "url": "https://patchwork.libcamera.org/api/1.1/projects/1/?format=api", "name": "libcamera", "link_name": "libcamera", "list_id": "libcamera_core", "list_email": "libcamera-devel@lists.libcamera.org", "web_url": "", "scm_url": "", "webscm_url": "" }, "msgid": "<20220224113347.80001-6-jeanmichel.hautbois@ideasonboard.com>", "date": "2022-02-24T11:33:47", "name": "[libcamera-devel,v3,5/5] ipa: rkisp1: Introduce AWB", "commit_ref": null, "pull_url": null, "state": "superseded", "archived": false, "hash": "f5e336c3e1db54cc3d6e716d313f2e18744e872b", "submitter": { "id": 75, "url": "https://patchwork.libcamera.org/api/1.1/people/75/?format=api", "name": "Jean-Michel Hautbois", "email": "jeanmichel.hautbois@ideasonboard.com" }, "delegate": null, "mbox": "https://patchwork.libcamera.org/patch/15381/mbox/", "series": [ { "id": 2937, "url": "https://patchwork.libcamera.org/api/1.1/series/2937/?format=api", "web_url": "https://patchwork.libcamera.org/project/libcamera/list/?series=2937", "date": "2022-02-24T11:33:42", "name": "IPA RkISP1 awb and misc improvements", "version": 3, "mbox": "https://patchwork.libcamera.org/series/2937/mbox/" } ], "comments": "https://patchwork.libcamera.org/api/patches/15381/comments/", "check": "pending", "checks": "https://patchwork.libcamera.org/api/patches/15381/checks/", "tags": {}, "headers": { "Return-Path": "<libcamera-devel-bounces@lists.libcamera.org>", "X-Original-To": "parsemail@patchwork.libcamera.org", "Delivered-To": "parsemail@patchwork.libcamera.org", "Received": [ "from lancelot.ideasonboard.com (lancelot.ideasonboard.com\n\t[92.243.16.209])\n\tby patchwork.libcamera.org (Postfix) with ESMTPS id E4AA9BF415\n\tfor <parsemail@patchwork.libcamera.org>;\n\tThu, 24 Feb 2022 11:34:00 +0000 (UTC)", "from lancelot.ideasonboard.com (localhost [IPv6:::1])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTP id AD6A261174;\n\tThu, 24 Feb 2022 12:33:59 +0100 (CET)", "from perceval.ideasonboard.com (perceval.ideasonboard.com\n\t[IPv6:2001:4b98:dc2:55:216:3eff:fef7:d647])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id BB6DC61159\n\tfor <libcamera-devel@lists.libcamera.org>;\n\tThu, 24 Feb 2022 12:33:53 +0100 (CET)", "from tatooine.ideasonboard.com (unknown\n\t[IPv6:2a01:e0a:169:7140:ce74:6df2:4b76:b230])\n\tby perceval.ideasonboard.com (Postfix) with ESMTPSA id 74FFB484;\n\tThu, 24 Feb 2022 12:33:53 +0100 (CET)" ], "Authentication-Results": "lancelot.ideasonboard.com;\n\tdkim=fail reason=\"signature verification failed\" (1024-bit key;\n\tunprotected) header.d=ideasonboard.com header.i=@ideasonboard.com\n\theader.b=\"nSTfUB3o\"; dkim-atps=neutral", "DKIM-Signature": "v=1; a=rsa-sha256; c=relaxed/simple; d=ideasonboard.com;\n\ts=mail; t=1645702433;\n\tbh=HpEZjeyGDMf5kOyoPnvUJwMC9zVpxp37TiT3OTwMwgk=;\n\th=From:To:Cc:Subject:Date:In-Reply-To:References:From;\n\tb=nSTfUB3odsPAdsVEcgqRnfgNAwcC2mix+oDMfWG7K4SwM24O7ylMH0uV23t7otlRp\n\tmnw/KvuaougZpoUpPyqmVDqhWXL1z7Sdh1v5NvQbQ5KV61ovZZlMeW+Ulwz2NuvOeI\n\t36XbSWVZ3c6LsHbTqXM5JIjVDkzRfTdwrsbUPXEU=", "From": "Jean-Michel Hautbois <jeanmichel.hautbois@ideasonboard.com>", "To": "libcamera-devel@lists.libcamera.org", "Date": "Thu, 24 Feb 2022 12:33:47 +0100", "Message-Id": "<20220224113347.80001-6-jeanmichel.hautbois@ideasonboard.com>", "X-Mailer": "git-send-email 2.32.0", "In-Reply-To": "<20220224113347.80001-1-jeanmichel.hautbois@ideasonboard.com>", "References": "<20220224113347.80001-1-jeanmichel.hautbois@ideasonboard.com>", "MIME-Version": "1.0", "Content-Transfer-Encoding": "8bit", "Subject": "[libcamera-devel] [PATCH v3 5/5] ipa: rkisp1: Introduce AWB", "X-BeenThere": "libcamera-devel@lists.libcamera.org", "X-Mailman-Version": "2.1.29", "Precedence": "list", "List-Id": "<libcamera-devel.lists.libcamera.org>", "List-Unsubscribe": "<https://lists.libcamera.org/options/libcamera-devel>,\n\t<mailto:libcamera-devel-request@lists.libcamera.org?subject=unsubscribe>", "List-Archive": "<https://lists.libcamera.org/pipermail/libcamera-devel/>", "List-Post": "<mailto:libcamera-devel@lists.libcamera.org>", "List-Help": "<mailto:libcamera-devel-request@lists.libcamera.org?subject=help>", "List-Subscribe": "<https://lists.libcamera.org/listinfo/libcamera-devel>,\n\t<mailto:libcamera-devel-request@lists.libcamera.org?subject=subscribe>", "Errors-To": "libcamera-devel-bounces@lists.libcamera.org", "Sender": "\"libcamera-devel\" <libcamera-devel-bounces@lists.libcamera.org>" }, "content": "The RkISP1 ISP calculates a mean value for Y, Cr and Cb at each frame.\nThere is a RGB mode which could theoretically give us the values for R,\nG and B directly, but it seems to be failing right now.\n\nConvert those values into R, G and B and estimate the gain to apply in a\ngrey world.\n\nSigned-off-by: Jean-Michel Hautbois <jeanmichel.hautbois@ideasonboard.com>\nTested-by: Peter Griffin <peter.griffin@linaro.org>\n\n---\nv3: - Change the date of the copyright\n - Fix the YCbCr to RGB formula\n---\n src/ipa/rkisp1/algorithms/awb.cpp | 180 ++++++++++++++++++++++++++\n src/ipa/rkisp1/algorithms/awb.h | 33 +++++\n src/ipa/rkisp1/algorithms/meson.build | 1 +\n src/ipa/rkisp1/ipa_context.cpp | 28 ++++\n src/ipa/rkisp1/ipa_context.h | 16 +++\n src/ipa/rkisp1/rkisp1.cpp | 2 +\n 6 files changed, 260 insertions(+)\n create mode 100644 src/ipa/rkisp1/algorithms/awb.cpp\n create mode 100644 src/ipa/rkisp1/algorithms/awb.h", "diff": "diff --git a/src/ipa/rkisp1/algorithms/awb.cpp b/src/ipa/rkisp1/algorithms/awb.cpp\nnew file mode 100644\nindex 00000000..fce4c498\n--- /dev/null\n+++ b/src/ipa/rkisp1/algorithms/awb.cpp\n@@ -0,0 +1,180 @@\n+/* SPDX-License-Identifier: LGPL-2.1-or-later */\n+/*\n+ * Copyright (C) 2021-2022, Ideas On Board\n+ *\n+ * awb.cpp - AWB control algorithm\n+ */\n+\n+#include \"awb.h\"\n+\n+#include <algorithm>\n+#include <cmath>\n+\n+#include <libcamera/base/log.h>\n+\n+#include <libcamera/ipa/core_ipa_interface.h>\n+\n+/**\n+ * \\file awb.h\n+ */\n+\n+namespace libcamera {\n+\n+namespace ipa::rkisp1::algorithms {\n+\n+/**\n+ * \\class Awb\n+ * \\brief A Grey world white balance correction algorithm\n+ */\n+\n+LOG_DEFINE_CATEGORY(RkISP1Awb)\n+\n+/**\n+ * \\copydoc libcamera::ipa::Algorithm::configure\n+ */\n+int Awb::configure(IPAContext &context,\n+\t\t const IPACameraSensorInfo &configInfo)\n+{\n+\tcontext.frameContext.awb.gains.red = 1.0;\n+\tcontext.frameContext.awb.gains.blue = 1.0;\n+\tcontext.frameContext.awb.gains.green = 1.0;\n+\n+\t/*\n+\t * Define the measurement window for AWB as a centered rectangle\n+\t * covering 3/4 of the image width and height.\n+\t */\n+\tcontext.configuration.awb.measureWindow.h_offs = configInfo.outputSize.width / 8;\n+\tcontext.configuration.awb.measureWindow.v_offs = configInfo.outputSize.height / 8;\n+\tcontext.configuration.awb.measureWindow.h_size = 3 * configInfo.outputSize.width / 4;\n+\tcontext.configuration.awb.measureWindow.v_size = 3 * configInfo.outputSize.height / 4;\n+\n+\treturn 0;\n+}\n+\n+uint32_t Awb::estimateCCT(double red, double green, double blue)\n+{\n+\t/* Convert the RGB values to CIE tristimulus values (XYZ) */\n+\tdouble X = (-0.14282) * (red) + (1.54924) * (green) + (-0.95641) * (blue);\n+\tdouble Y = (-0.32466) * (red) + (1.57837) * (green) + (-0.73191) * (blue);\n+\tdouble Z = (-0.68202) * (red) + (0.77073) * (green) + (0.56332) * (blue);\n+\n+\t/* Calculate the normalized chromaticity values */\n+\tdouble x = X / (X + Y + Z);\n+\tdouble y = Y / (X + Y + Z);\n+\n+\t/* Calculate CCT */\n+\tdouble n = (x - 0.3320) / (0.1858 - y);\n+\treturn 449 * n * n * n + 3525 * n * n + 6823.3 * n + 5520.33;\n+}\n+\n+/**\n+ * \\copydoc libcamera::ipa::Algorithm::prepare\n+ */\n+void Awb::prepare(IPAContext &context, rkisp1_params_cfg *params)\n+{\n+\tparams->others.awb_gain_config.gain_green_b = 256 * context.frameContext.awb.gains.green;\n+\tparams->others.awb_gain_config.gain_blue = 256 * context.frameContext.awb.gains.blue;\n+\tparams->others.awb_gain_config.gain_red = 256 * context.frameContext.awb.gains.red;\n+\tparams->others.awb_gain_config.gain_green_r = 256 * context.frameContext.awb.gains.green;\n+\n+\t/* Update the gains. */\n+\tparams->module_cfg_update |= RKISP1_CIF_ISP_MODULE_AWB_GAIN;\n+\n+\t/* If we already have configured the gains and window, return. */\n+\tif (context.frameContext.frameCount > 0)\n+\t\treturn;\n+\n+\t/* Configure the gains to apply. */\n+\tparams->module_en_update |= RKISP1_CIF_ISP_MODULE_AWB_GAIN;\n+\t/* Update the ISP to apply the gains configured. */\n+\tparams->module_ens |= RKISP1_CIF_ISP_MODULE_AWB_GAIN;\n+\n+\t/* Configure the measure window for AWB. */\n+\tparams->meas.awb_meas_config.awb_wnd = context.configuration.awb.measureWindow;\n+\t/*\n+\t * Measure Y, Cr and Cb means.\n+\t * \\todo RGB is not working, the kernel seems to not configure it ?\n+\t */\n+\tparams->meas.awb_meas_config.awb_mode = RKISP1_CIF_ISP_AWB_MODE_YCBCR;\n+\t/* Reference Cr and Cb. */\n+\tparams->meas.awb_meas_config.awb_ref_cb = 128;\n+\tparams->meas.awb_meas_config.awb_ref_cr = 128;\n+\t/* Y values to include are between min_y and max_y only. */\n+\tparams->meas.awb_meas_config.min_y = 16;\n+\tparams->meas.awb_meas_config.max_y = 250;\n+\t/* Maximum Cr+Cb value to take into account for awb. */\n+\tparams->meas.awb_meas_config.max_csum = 250;\n+\t/* Minimum Cr and Cb values to take into account. */\n+\tparams->meas.awb_meas_config.min_c = 16;\n+\t/* Number of frames to use to estimate the mean (0 means 1 frame). */\n+\tparams->meas.awb_meas_config.frames = 0;\n+\n+\t/* Update AWB measurement unit configuration. */\n+\tparams->module_cfg_update |= RKISP1_CIF_ISP_MODULE_AWB;\n+\t/* Make sure the ISP is measuring the means for the next frame. */\n+\tparams->module_en_update |= RKISP1_CIF_ISP_MODULE_AWB;\n+\tparams->module_ens |= RKISP1_CIF_ISP_MODULE_AWB;\n+}\n+\n+/**\n+ * \\copydoc libcamera::ipa::Algorithm::process\n+ */\n+void Awb::process([[maybe_unused]] IPAContext &context, const rkisp1_stat_buffer *stats)\n+{\n+\tconst rkisp1_cif_isp_stat *params = &stats->params;\n+\tconst rkisp1_cif_isp_awb_stat *awb = ¶ms->awb;\n+\n+\t/* Get the YCbCr mean values */\n+\tdouble yMean = awb->awb_mean[0].mean_y_or_g;\n+\tdouble crMean = awb->awb_mean[0].mean_cr_or_r;\n+\tdouble cbMean = awb->awb_mean[0].mean_cb_or_b;\n+\n+\t/*\n+\t * Convert from YCbCr to RGB.\n+\t * The hardware uses the following formulas:\n+\t * Y = 16 + 0.2500 R + 0.5000 G + 0.1094 B\n+\t * Cb = 128 - 0.1406 R - 0.2969 G + 0.4375 B\n+\t * Cr = 128 + 0.4375 R - 0.3750 G - 0.0625 B\n+\t *\n+\t * The inverse matrix is thus:\n+\t * [[1,1636, -0,0623, 1,6008]\n+\t * [1,1636, -0,4045, -0,7949]\n+\t * [1,1636, 1,9912, -0,0250]]\n+\t */\n+\tyMean -= 16;\n+\tcbMean -= 128;\n+\tcrMean -= 128;\n+\tdouble redMean = 1.1636 * yMean - 0.0623 * cbMean + 1.6008 * crMean;\n+\tdouble blueMean = 1.1636 * yMean - 0.4045 * cbMean - 0.7949 * crMean;\n+\tdouble greenMean = 1.1636 * yMean + 1.9912 * cbMean - 0.0250 * crMean;\n+\n+\t/*\n+\t * Gain values are unsigned integer value, range 0 to 4 with 8 bit\n+\t * fractional part.\n+\t */\n+\tredMean = std::clamp(redMean, 0.0, 1023.0 / 256);\n+\tblueMean = std::clamp(redMean, 0.0, 1023.0 / 256);\n+\tgreenMean = std::clamp(redMean, 0.0, 1023.0 / 256);\n+\n+\t/* Estimate the red and blue gains to apply in a grey world. */\n+\tdouble redGain = greenMean / (redMean + 1);\n+\tdouble blueGain = greenMean / (blueMean + 1);\n+\n+\t/* Filter the values to avoid oscillations. */\n+\tIPAFrameContext &frameContext = context.frameContext;\n+\n+\tframeContext.awb.temperatureK = estimateCCT(redMean, greenMean, blueMean);\n+\tframeContext.awb.gains.red = 0.2 * redGain +\n+\t\t\t\t 0.8 * frameContext.awb.gains.red;\n+\tframeContext.awb.gains.blue = 0.2 * blueGain +\n+\t\t\t\t 0.8 * frameContext.awb.gains.blue;\n+\t/* Hardcode the green gain to 1.0. */\n+\tframeContext.awb.gains.green = 1.0;\n+\n+\tLOG(RkISP1Awb, Debug) << \"Gain found for red: \" << context.frameContext.awb.gains.red\n+\t\t\t << \" and for blue: \" << context.frameContext.awb.gains.blue;\n+}\n+\n+} /* namespace ipa::rkisp1::algorithms */\n+\n+} /* namespace libcamera */\ndiff --git a/src/ipa/rkisp1/algorithms/awb.h b/src/ipa/rkisp1/algorithms/awb.h\nnew file mode 100644\nindex 00000000..11946643\n--- /dev/null\n+++ b/src/ipa/rkisp1/algorithms/awb.h\n@@ -0,0 +1,33 @@\n+/* SPDX-License-Identifier: LGPL-2.1-or-later */\n+/*\n+ * Copyright (C) 2021-2022, Ideas On Board\n+ *\n+ * awb.h - AWB control algorithm\n+ */\n+\n+#pragma once\n+\n+#include <linux/rkisp1-config.h>\n+\n+#include \"algorithm.h\"\n+\n+namespace libcamera {\n+\n+namespace ipa::rkisp1::algorithms {\n+\n+class Awb : public Algorithm\n+{\n+public:\n+\tAwb() = default;\n+\t~Awb() = default;\n+\n+\tint configure(IPAContext &context, const IPACameraSensorInfo &configInfo) override;\n+\tvoid prepare(IPAContext &context, rkisp1_params_cfg *params) override;\n+\tvoid process(IPAContext &context, const rkisp1_stat_buffer *stats) override;\n+\n+private:\n+\tuint32_t estimateCCT(double red, double green, double blue);\n+};\n+\n+} /* namespace ipa::rkisp1::algorithms */\n+} /* namespace libcamera */\ndiff --git a/src/ipa/rkisp1/algorithms/meson.build b/src/ipa/rkisp1/algorithms/meson.build\nindex 27c97731..7ec53d89 100644\n--- a/src/ipa/rkisp1/algorithms/meson.build\n+++ b/src/ipa/rkisp1/algorithms/meson.build\n@@ -2,5 +2,6 @@\n \n rkisp1_ipa_algorithms = files([\n 'agc.cpp',\n+ 'awb.cpp',\n 'blc.cpp',\n ])\ndiff --git a/src/ipa/rkisp1/ipa_context.cpp b/src/ipa/rkisp1/ipa_context.cpp\nindex 790e159b..d6d8456c 100644\n--- a/src/ipa/rkisp1/ipa_context.cpp\n+++ b/src/ipa/rkisp1/ipa_context.cpp\n@@ -81,6 +81,14 @@ namespace libcamera::ipa::rkisp1 {\n * \\brief Hardware revision of the ISP\n */\n \n+/**\n+ * \\var IPASessionConfiguration::awb\n+ * \\brief AWB parameters configuration of the IPA\n+ *\n+ * \\var IPASessionConfiguration::awb.measureWindow\n+ * \\brief AWB measure window\n+ */\n+\n /**\n * \\var IPASessionConfiguration::sensor\n * \\brief Sensor-specific configuration of the IPA\n@@ -105,6 +113,26 @@ namespace libcamera::ipa::rkisp1 {\n * The gain should be adapted to the sensor specific gain code before applying.\n */\n \n+/**\n+ * \\var IPAFrameContext::awb\n+ * \\brief Context for the Automatic White Balance algorithm\n+ *\n+ * \\struct IPAFrameContext::awb.gains\n+ * \\brief White balance gains\n+ *\n+ * \\var IPAFrameContext::awb.gains.red\n+ * \\brief White balance gain for R channel\n+ *\n+ * \\var IPAFrameContext::awb.gains.green\n+ * \\brief White balance gain for G channel\n+ *\n+ * \\var IPAFrameContext::awb.gains.blue\n+ * \\brief White balance gain for B channel\n+ *\n+ * \\var IPAFrameContext::awb.temperatureK\n+ * \\brief Estimated color temperature\n+ */\n+\n /**\n * \\var IPAFrameContext::sensor\n * \\brief Effective sensor values\ndiff --git a/src/ipa/rkisp1/ipa_context.h b/src/ipa/rkisp1/ipa_context.h\nindex 35e9b8e5..f387cace 100644\n--- a/src/ipa/rkisp1/ipa_context.h\n+++ b/src/ipa/rkisp1/ipa_context.h\n@@ -12,6 +12,8 @@\n \n #include <libcamera/base/utils.h>\n \n+#include <libcamera/geometry.h>\n+\n namespace libcamera {\n \n namespace ipa::rkisp1 {\n@@ -25,6 +27,10 @@ struct IPASessionConfiguration {\n \t\tstruct rkisp1_cif_isp_window measureWindow;\n \t} agc;\n \n+\tstruct {\n+\t\tstruct rkisp1_cif_isp_window measureWindow;\n+\t} awb;\n+\n \tstruct {\n \t\tutils::Duration lineDuration;\n \t} sensor;\n@@ -40,6 +46,16 @@ struct IPAFrameContext {\n \t\tdouble gain;\n \t} agc;\n \n+\tstruct {\n+\t\tstruct {\n+\t\t\tdouble red;\n+\t\t\tdouble green;\n+\t\t\tdouble blue;\n+\t\t} gains;\n+\n+\t\tdouble temperatureK;\n+\t} awb;\n+\n \tstruct {\n \t\tuint32_t exposure;\n \t\tdouble gain;\ndiff --git a/src/ipa/rkisp1/rkisp1.cpp b/src/ipa/rkisp1/rkisp1.cpp\nindex bb3deb93..f4db3a21 100644\n--- a/src/ipa/rkisp1/rkisp1.cpp\n+++ b/src/ipa/rkisp1/rkisp1.cpp\n@@ -27,6 +27,7 @@\n \n #include \"algorithms/agc.h\"\n #include \"algorithms/algorithm.h\"\n+#include \"algorithms/awb.h\"\n #include \"algorithms/blc.h\"\n #include \"libipa/camera_sensor_helper.h\"\n \n@@ -127,6 +128,7 @@ int IPARkISP1::init(const IPASettings &settings, unsigned int hwRevision)\n \n \t/* Construct our Algorithms */\n \talgorithms_.push_back(std::make_unique<algorithms::Agc>());\n+\talgorithms_.push_back(std::make_unique<algorithms::Awb>());\n \talgorithms_.push_back(std::make_unique<algorithms::BlackLevelCorrection>());\n \n \treturn 0;\n", "prefixes": [ "libcamera-devel", "v3", "5/5" ] }