[{"id":24642,"web_url":"https://patchwork.libcamera.org/comment/24642/","msgid":"<20220818133442.GL2412817@pyrite.rasen.tech>","date":"2022-08-18T13:34:42","subject":"Re: [libcamera-devel] [PATCH v4 9/9] ipa: rkisp1: Add support of\n\tDenoise Pre-Filter control","submitter":{"id":97,"url":"https://patchwork.libcamera.org/api/people/97/","name":"Nicolas Dufresne via libcamera-devel","email":"libcamera-devel@lists.libcamera.org"},"content":"On Tue, Aug 16, 2022 at 04:54:14AM +0300, Laurent Pinchart via libcamera-devel wrote:\n> From: Florian Sylvestre \n> \n> The denoise pre-filter algorithm is a bilateral filter which combines a range\n> filter and a domain filter. The denoise pre-filter is applied before\n> demosaicing.\n> \n> Signed-off-by: Florian Sylvestre \n> Reviewed-by: Laurent Pinchart \n> Signed-off-by: Laurent Pinchart \n> Reviewed-by: Umang Jain \n\nReviewed-by: Paul Elder \n\n> ---\n> Changes since v3:\n> \n> - Use YamlObject::getList for domain filter coefficients\n> - Add comment to explain gain handling\n> \n> Changes since v2:\n> \n> - Change the scope of some variables in Dpf::prepare()\n> - Line-wrap coefficients in tuning file\n> ---\n> src/ipa/rkisp1/algorithms/dpf.cpp | 258 ++++++++++++++++++++++++++\n> src/ipa/rkisp1/algorithms/dpf.h | 36 ++++\n> src/ipa/rkisp1/algorithms/meson.build | 1 +\n> src/ipa/rkisp1/data/ov5640.yaml | 15 ++\n> src/ipa/rkisp1/ipa_context.cpp | 11 ++\n> src/ipa/rkisp1/ipa_context.h | 5 +\n> 6 files changed, 326 insertions(+)\n> create mode 100644 src/ipa/rkisp1/algorithms/dpf.cpp\n> create mode 100644 src/ipa/rkisp1/algorithms/dpf.h\n> \n> diff --git a/src/ipa/rkisp1/algorithms/dpf.cpp b/src/ipa/rkisp1/algorithms/dpf.cpp\n> new file mode 100644\n> index 000000000000..c93c8361b5da\n> --- /dev/null\n> +++ b/src/ipa/rkisp1/algorithms/dpf.cpp\n> @@ -0,0 +1,258 @@\n> +/* SPDX-License-Identifier: LGPL-2.1-or-later */\n> +/*\n> + * Copyright (C) 2021-2022, Ideas On Board\n> + *\n> + * dpf.cpp - RkISP1 Denoise Pre-Filter control\n> + */\n> +\n> +#include \"dpf.h\"\n> +\n> +#include \n> +\n> +#include \n> +\n> +#include \n> +\n> +#include \"linux/rkisp1-config.h\"\n> +\n> +/**\n> + * \\file dpf.h\n> + */\n> +\n> +namespace libcamera {\n> +\n> +namespace ipa::rkisp1::algorithms {\n> +\n> +/**\n> + * \\class Dpf\n> + * \\brief RkISP1 Denoise Pre-Filter control\n> + *\n> + * The denoise pre-filter algorithm is a bilateral filter which combines a\n> + * range filter and a domain filter. The denoise pre-filter is applied before\n> + * demosaicing.\n> + */\n> +\n> +LOG_DEFINE_CATEGORY(RkISP1Dpf)\n> +\n> +Dpf::Dpf()\n> +\t: initialized_(false), config_({}), strengthConfig_({})\n> +{\n> +}\n> +\n> +/**\n> + * \\copydoc libcamera::ipa::Algorithm::init\n> + */\n> +int Dpf::init([[maybe_unused]] IPAContext &context,\n> +\t const YamlObject &tuningData)\n> +{\n> +\tstd::vector values;\n> +\n> +\t/*\n> +\t * The domain kernel is configured with a 9x9 kernel for the green\n> +\t * pixels, and a 13x9 or 9x9 kernel for red and blue pixels.\n> +\t */\n> +\tconst YamlObject &dFObject = tuningData[\"DomainFilter\"];\n> +\n> +\t/*\n> +\t * For the green component, we have the 9x9 kernel specified\n> +\t * as 6 coefficients:\n> +\t * Y\n> +\t * ^\n> +\t * 4 | 6 5 4 5 6\n> +\t * 3 | 5 3 3 5\n> +\t * 2 | 5 3 2 3 5\n> +\t * 1 | 3 1 1 3\n> +\t * 0 - 4 2 0 2 4\n> +\t * -1 | 3 1 1 3\n> +\t * -2 | 5 3 2 3 5\n> +\t * -3 | 5 3 3 5\n> +\t * -4 | 6 5 4 5 6\n> +\t * +---------|--------> X\n> +\t * -4....-1 0 1 2 3 4\n> +\t */\n> +\tvalues = dFObject[\"g\"].getList().value_or(utils::defopt);\n> +\tif (values.size() != RKISP1_CIF_ISP_DPF_MAX_SPATIAL_COEFFS) {\n> +\t\tLOG(RkISP1Dpf, Error)\n> +\t\t\t<< \"Invalid 'DomainFilter:g': expected \"\n> +\t\t\t<< RKISP1_CIF_ISP_DPF_MAX_SPATIAL_COEFFS\n> +\t\t\t<< \" elements, got \" << values.size();\n> +\t\treturn -EINVAL;\n> +\t}\n> +\n> +\tstd::copy_n(values.begin(), values.size(),\n> +\t\t std::begin(config_.g_flt.spatial_coeff));\n> +\n> +\tconfig_.g_flt.gr_enable = true;\n> +\tconfig_.g_flt.gb_enable = true;\n> +\n> +\t/*\n> +\t * For the red and blue components, we have the 13x9 kernel specified\n> +\t * as 6 coefficients:\n> +\t *\n> +\t * Y\n> +\t * ^\n> +\t * 4 | 6 5 4 3 4 5 6\n> +\t * |\n> +\t * 2 | 5 4 2 1 2 4 5\n> +\t * |\n> +\t * 0 - 5 3 1 0 1 3 5\n> +\t * |\n> +\t * -2 | 5 4 2 1 2 4 5\n> +\t * |\n> +\t * -4 | 6 5 4 3 4 5 6\n> +\t * +-------------|------------> X\n> +\t * -6 -4 -2 0 2 4 6\n> +\t *\n> +\t * For a 9x9 kernel, columns -6 and 6 are dropped, so coefficient\n> +\t * number 6 is not used.\n> +\t */\n> +\tvalues = dFObject[\"rb\"].getList().value_or(utils::defopt);\n> +\tif (values.size() != RKISP1_CIF_ISP_DPF_MAX_SPATIAL_COEFFS &&\n> +\t values.size() != RKISP1_CIF_ISP_DPF_MAX_SPATIAL_COEFFS - 1) {\n> +\t\tLOG(RkISP1Dpf, Error)\n> +\t\t\t<< \"Invalid 'DomainFilter:rb': expected \"\n> +\t\t\t<< RKISP1_CIF_ISP_DPF_MAX_SPATIAL_COEFFS - 1\n> +\t\t\t<< \" or \" << RKISP1_CIF_ISP_DPF_MAX_SPATIAL_COEFFS\n> +\t\t\t<< \" elements, got \" << values.size();\n> +\t\treturn -EINVAL;\n> +\t}\n> +\n> +\tconfig_.rb_flt.fltsize = values.size() == RKISP1_CIF_ISP_DPF_MAX_SPATIAL_COEFFS\n> +\t\t\t ? RKISP1_CIF_ISP_DPF_RB_FILTERSIZE_13x9\n> +\t\t\t : RKISP1_CIF_ISP_DPF_RB_FILTERSIZE_9x9;\n> +\n> +\tstd::copy_n(values.begin(), values.size(),\n> +\t\t std::begin(config_.rb_flt.spatial_coeff));\n> +\n> +\tconfig_.rb_flt.r_enable = true;\n> +\tconfig_.rb_flt.b_enable = true;\n> +\n> +\t/*\n> +\t * The range kernel is configured with a noise level lookup table (NLL)\n> +\t * which stores a piecewise linear function that characterizes the\n> +\t * sensor noise profile as a noise level function curve (NLF).\n> +\t */\n> +\tconst YamlObject &rFObject = tuningData[\"NoiseLevelFunction\"];\n> +\n> +\tstd::vector nllValues;\n> +\tnllValues = rFObject[\"coeff\"].getList().value_or(utils::defopt);\n> +\tif (nllValues.size() != RKISP1_CIF_ISP_DPF_MAX_NLF_COEFFS) {\n> +\t\tLOG(RkISP1Dpf, Error)\n> +\t\t\t<< \"Invalid 'RangeFilter:coeff': expected \"\n> +\t\t\t<< RKISP1_CIF_ISP_DPF_MAX_NLF_COEFFS\n> +\t\t\t<< \" elements, got \" << nllValues.size();\n> +\t\treturn -EINVAL;\n> +\t}\n> +\n> +\tstd::copy_n(nllValues.begin(), nllValues.size(),\n> +\t\t std::begin(config_.nll.coeff));\n> +\n> +\tstd::string scaleMode = rFObject[\"scale-mode\"].get(\"\");\n> +\tif (scaleMode == \"linear\") {\n> +\t\tconfig_.nll.scale_mode = RKISP1_CIF_ISP_NLL_SCALE_LINEAR;\n> +\t} else if (scaleMode == \"logarithmic\") {\n> +\t\tconfig_.nll.scale_mode = RKISP1_CIF_ISP_NLL_SCALE_LOGARITHMIC;\n> +\t} else {\n> +\t\tLOG(RkISP1Dpf, Error)\n> +\t\t\t<< \"Invalid 'RangeFilter:scale-mode': expected \"\n> +\t\t\t<< \"'linear' or 'logarithmic' value, got \"\n> +\t\t\t<< scaleMode;\n> +\t\treturn -EINVAL;\n> +\t}\n> +\n> +\tconst YamlObject &fSObject = tuningData[\"FilterStrength\"];\n> +\n> +\tstrengthConfig_.r = fSObject[\"r\"].get(64);\n> +\tstrengthConfig_.g = fSObject[\"g\"].get(64);\n> +\tstrengthConfig_.b = fSObject[\"b\"].get(64);\n> +\n> +\tinitialized_ = true;\n> +\n> +\treturn 0;\n> +}\n> +\n> +/**\n> + * \\copydoc libcamera::ipa::Algorithm::queueRequest\n> + */\n> +void Dpf::queueRequest(IPAContext &context,\n> +\t\t [[maybe_unused]] const uint32_t frame,\n> +\t\t const ControlList &controls)\n> +{\n> +\tauto &dpf = context.frameContext.dpf;\n> +\n> +\tconst auto &denoise = controls.get(controls::draft::NoiseReductionMode);\n> +\tif (denoise) {\n> +\t\tLOG(RkISP1Dpf, Debug) << \"Set denoise to \" << *denoise;\n> +\n> +\t\tswitch (*denoise) {\n> +\t\tcase controls::draft::NoiseReductionModeOff:\n> +\t\t\tdpf.denoise = false;\n> +\t\t\tdpf.updateParams = true;\n> +\t\t\tbreak;\n> +\t\tcase controls::draft::NoiseReductionModeMinimal:\n> +\t\tcase controls::draft::NoiseReductionModeHighQuality:\n> +\t\tcase controls::draft::NoiseReductionModeFast:\n> +\t\t\tdpf.denoise = true;\n> +\t\t\tdpf.updateParams = true;\n> +\t\t\tbreak;\n> +\t\tdefault:\n> +\t\t\tLOG(RkISP1Dpf, Error)\n> +\t\t\t\t<< \"Unsupported denoise value \"\n> +\t\t\t\t<< *denoise;\n> +\t\t}\n> +\t}\n> +}\n> +\n> +/**\n> + * \\copydoc libcamera::ipa::Algorithm::prepare\n> + */\n> +void Dpf::prepare(IPAContext &context, rkisp1_params_cfg *params)\n> +{\n> +\tif (!initialized_)\n> +\t\treturn;\n> +\n> +\tauto &dpf = context.frameContext.dpf;\n> +\n> +\tif (context.frameContext.frameCount == 0) {\n> +\t\tparams->others.dpf_config = config_;\n> +\t\tparams->others.dpf_strength_config = strengthConfig_;\n> +\n> +\t\tconst auto &awb = context.configuration.awb;\n> +\t\tconst auto &lsc = context.configuration.lsc;\n> +\t\tauto &mode = params->others.dpf_config.gain.mode;\n> +\n> +\t\t/*\n> +\t\t * The DPF needs to take into account the total amount of\n> +\t\t * digital gain, which comes from the AWB and LSC modules. The\n> +\t\t * DPF hardware can be programmed with a digital gain value\n> +\t\t * manually, but can also use the gains supplied by the AWB and\n> +\t\t * LSC modules automatically when they are enabled. Use that\n> +\t\t * mode of operation as it simplifies control of the DPF.\n> +\t\t */\n> +\t\tif (awb.enabled && lsc.enabled)\n> +\t\t\tmode = RKISP1_CIF_ISP_DPF_GAIN_USAGE_AWB_LSC_GAINS;\n> +\t\telse if (awb.enabled)\n> +\t\t\tmode = RKISP1_CIF_ISP_DPF_GAIN_USAGE_AWB_GAINS;\n> +\t\telse if (lsc.enabled)\n> +\t\t\tmode = RKISP1_CIF_ISP_DPF_GAIN_USAGE_LSC_GAINS;\n> +\t\telse\n> +\t\t\tmode = RKISP1_CIF_ISP_DPF_GAIN_USAGE_DISABLED;\n> +\n> +\t\tparams->module_cfg_update |= RKISP1_CIF_ISP_MODULE_DPF |\n> +\t\t\t\t\t RKISP1_CIF_ISP_MODULE_DPF_STRENGTH;\n> +\t}\n> +\n> +\tif (dpf.updateParams) {\n> +\t\tparams->module_en_update |= RKISP1_CIF_ISP_MODULE_DPF;\n> +\t\tif (dpf.denoise)\n> +\t\t\tparams->module_ens |= RKISP1_CIF_ISP_MODULE_DPF;\n> +\n> +\t\tdpf.updateParams = false;\n> +\t}\n> +}\n> +\n> +REGISTER_IPA_ALGORITHM(Dpf, \"Dpf\")\n> +\n> +} /* namespace ipa::rkisp1::algorithms */\n> +\n> +} /* namespace libcamera */\n> diff --git a/src/ipa/rkisp1/algorithms/dpf.h b/src/ipa/rkisp1/algorithms/dpf.h\n> new file mode 100644\n> index 000000000000..128ebd5e02e3\n> --- /dev/null\n> +++ b/src/ipa/rkisp1/algorithms/dpf.h\n> @@ -0,0 +1,36 @@\n> +/* SPDX-License-Identifier: LGPL-2.1-or-later */\n> +/*\n> + * Copyright (C) 2021-2022, Ideas On Board\n> + *\n> + * dpf.h - RkISP1 Denoise Pre-Filter control\n> + */\n> +\n> +#pragma once\n> +\n> +#include \n> +\n> +#include \"algorithm.h\"\n> +\n> +namespace libcamera {\n> +\n> +namespace ipa::rkisp1::algorithms {\n> +\n> +class Dpf : public Algorithm\n> +{\n> +public:\n> +\tDpf();\n> +\t~Dpf() = default;\n> +\n> +\tint init(IPAContext &context, const YamlObject &tuningData) override;\n> +\tvoid queueRequest(IPAContext &context, const uint32_t frame,\n> +\t\t\t const ControlList &controls) override;\n> +\tvoid prepare(IPAContext &context, rkisp1_params_cfg *params) override;\n> +\n> +private:\n> +\tbool initialized_;\n> +\tstruct rkisp1_cif_isp_dpf_config config_;\n> +\tstruct rkisp1_cif_isp_dpf_strength_config strengthConfig_;\n> +};\n> +\n> +} /* namespace ipa::rkisp1::algorithms */\n> +} /* namespace libcamera */\n> diff --git a/src/ipa/rkisp1/algorithms/meson.build b/src/ipa/rkisp1/algorithms/meson.build\n> index e48974b454b5..93a483292753 100644\n> --- a/src/ipa/rkisp1/algorithms/meson.build\n> +++ b/src/ipa/rkisp1/algorithms/meson.build\n> @@ -6,6 +6,7 @@ rkisp1_ipa_algorithms = files([\n> 'blc.cpp',\n> 'cproc.cpp',\n> 'dpcc.cpp',\n> + 'dpf.cpp',\n> 'filter.cpp',\n> 'gsl.cpp',\n> 'lsc.cpp',\n> diff --git a/src/ipa/rkisp1/data/ov5640.yaml b/src/ipa/rkisp1/data/ov5640.yaml\n> index 45d4bb77f8ca..3dc369ac3681 100644\n> --- a/src/ipa/rkisp1/data/ov5640.yaml\n> +++ b/src/ipa/rkisp1/data/ov5640.yaml\n> @@ -156,5 +156,20 @@ algorithms:\n> rnd-offsets:\n> green: 2\n> red-blue: 2\n> + - Dpf:\n> + DomainFilter:\n> + g: [ 16, 16, 16, 16, 16, 16 ]\n> + rb: [ 16, 16, 16, 16, 16, 16 ]\n> + NoiseLevelFunction:\n> + coeff: [\n> + 1023, 1023, 1023, 1023, 1023, 1023, 1023, 1023,\n> + 1023, 1023, 1023, 1023, 1023, 1023, 1023, 1023,\n> + 1023\n> + ]\n> + scale-mode: \"linear\"\n> + FilterStrength:\n> + r: 64\n> + g: 64\n> + b: 64\n> - Filter:\n> ...\n> diff --git a/src/ipa/rkisp1/ipa_context.cpp b/src/ipa/rkisp1/ipa_context.cpp\n> index 1a549c092d73..17f15fd4e6b5 100644\n> --- a/src/ipa/rkisp1/ipa_context.cpp\n> +++ b/src/ipa/rkisp1/ipa_context.cpp\n> @@ -164,6 +164,17 @@ namespace libcamera::ipa::rkisp1 {\n> * \\brief Indicates if ISP parameters need to be updated\n> */\n> \n> +/**\n> + * \\var IPAFrameContext::dpf\n> + * \\brief Context for the Denoise Pre-Filter algorithm\n> + *\n> + * \\var IPAFrameContext::dpf.denoise\n> + * \\brief Indicates if denoise is activated\n> + *\n> + * \\var IPAFrameContext::dpf.updateParams\n> + * \\brief Indicates if ISP parameters need to be updated\n> + */\n> +\n> /**\n> * \\var IPAFrameContext::filter\n> * \\brief Context for the Filter algorithm\n> diff --git a/src/ipa/rkisp1/ipa_context.h b/src/ipa/rkisp1/ipa_context.h\n> index 0cd6aadb83ed..3a743ac325da 100644\n> --- a/src/ipa/rkisp1/ipa_context.h\n> +++ b/src/ipa/rkisp1/ipa_context.h\n> @@ -69,6 +69,11 @@ struct IPAFrameContext {\n> \t\tbool updateParams;\n> \t} cproc;\n> \n> +\tstruct {\n> +\t\tbool denoise;\n> +\t\tbool updateParams;\n> +\t} dpf;\n> +\n> \tstruct {\n> \t\tuint8_t denoise;\n> \t\tuint8_t sharpness;","headers":{"Return-Path":"","X-Original-To":"parsemail@patchwork.libcamera.org","Delivered-To":"parsemail@patchwork.libcamera.org","Received":["from lancelot.ideasonboard.com (lancelot.ideasonboard.com\n\t[92.243.16.209])\n\tby patchwork.libcamera.org (Postfix) with ESMTPS id 2CFA5BE173\n\tfor ;\n\tThu, 18 Aug 2022 13:34:52 +0000 (UTC)","from lancelot.ideasonboard.com (localhost [IPv6:::1])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTP id 529C861FC0;\n\tThu, 18 Aug 2022 15:34:51 +0200 (CEST)","from perceval.ideasonboard.com (perceval.ideasonboard.com\n\t[IPv6:2001:4b98:dc2:55:216:3eff:fef7:d647])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id B6F5061FA7\n\tfor ;\n\tThu, 18 Aug 2022 15:34:49 +0200 (CEST)","from pyrite.rasen.tech (KD027085204050.au-net.ne.jp [27.85.204.50])\n\tby perceval.ideasonboard.com (Postfix) with ESMTPSA id 104318B;\n\tThu, 18 Aug 2022 15:34:47 +0200 (CEST)"],"DKIM-Signature":["v=1; a=rsa-sha256; c=relaxed/simple; d=libcamera.org;\n\ts=mail; t=1660829691;\n\tbh=FqLyLm7Sv0B9PkZ+l58b4BdrAQRLwGdWDmPI+xO1r0I=;\n\th=Date:To:References:In-Reply-To:Subject:List-Id:List-Unsubscribe:\n\tList-Archive:List-Post:List-Help:List-Subscribe:From:Reply-To:Cc:\n\tFrom;\n\tb=VY7CJTfuzh5Z8ekTrhYqFS/JOIXa+xhp1ZRlPXlnV146ybmCCNrqsMB2AQ694nJxM\n\tb5G5bbBBfQuMtVxbf4MFIS1F68f22sidZPsJDSbPcDnmT1mppDUAYMarIwX/yOLw+x\n\tOZRbyiDrl5THudpL4RM4lJiHQkamcaR600mNeZSOThrHJpmUbWR72GB/2l0lbbtsCW\n\tAiQEU//+tMTBTZpBX0gSTpWsKiaa4DqhsLyQt375zfsXZ1dIoXxaN8L5S9DenetP5B\n\tFpYpydVb3tvRL56Baodv+3CLp5ReRXLQNeHEtJSWfTUz/NlY7DxtOLbGV6LkQ44uTQ\n\txv0aFPaGGwN2A==","v=1; a=rsa-sha256; c=relaxed/simple; d=ideasonboard.com;\n\ts=mail; t=1660829689;\n\tbh=FqLyLm7Sv0B9PkZ+l58b4BdrAQRLwGdWDmPI+xO1r0I=;\n\th=Date:From:To:Cc:Subject:References:In-Reply-To:From;\n\tb=rbbUaiYMhRE17yQkt+vCVL21DzbCb2QclznVg5DKLkhQ/O4eXWgy4PHyJgcvZgIsg\n\tZEjEbHYe4s2zHw9ChVTUFkMIEtSzmCRf9g/O2ECBd57/95W+xD0IJ8AKe3v2efxrYV\n\tcrsu09R8ufMijUVdTj/unk2PJwWmFyi9vCDgp+JE="],"Authentication-Results":"lancelot.ideasonboard.com; dkim=pass (1024-bit key; \n\tunprotected) header.d=ideasonboard.com\n\theader.i=@ideasonboard.com\n\theader.b=\"rbbUaiYM\"; dkim-atps=neutral","Date":"Thu, 18 Aug 2022 22:34:42 +0900","To":"Laurent Pinchart ","Message-ID":"<20220818133442.GL2412817@pyrite.rasen.tech>","References":"<20220816015414.7462-1-laurent.pinchart@ideasonboard.com>\n\t<20220816015414.7462-10-laurent.pinchart@ideasonboard.com>","MIME-Version":"1.0","Content-Type":"text/plain; charset=us-ascii","Content-Disposition":"inline","In-Reply-To":"<20220816015414.7462-10-laurent.pinchart@ideasonboard.com>","Subject":"Re: [libcamera-devel] [PATCH v4 9/9] ipa: rkisp1: Add support of\n\tDenoise Pre-Filter control","X-BeenThere":"libcamera-devel@lists.libcamera.org","X-Mailman-Version":"2.1.29","Precedence":"list","List-Id":"","List-Unsubscribe":",\n\t","List-Archive":"","List-Post":"","List-Help":"","List-Subscribe":",\n\t","From":"Paul Elder via libcamera-devel ","Reply-To":"paul.elder@ideasonboard.com","Cc":"libcamera-devel@lists.libcamera.org","Errors-To":"libcamera-devel-bounces@lists.libcamera.org","Sender":"\"libcamera-devel\" "}}]