[{"id":25854,"web_url":"https://patchwork.libcamera.org/comment/25854/","msgid":"","date":"2022-11-23T00:16:24","subject":"Re: [libcamera-devel] [PATCH v3 05/12] utils: libtuning: modules:\n\talsc: Add raspberrypi ALSC module","submitter":{"id":2,"url":"https://patchwork.libcamera.org/api/people/2/","name":"Laurent Pinchart","email":"laurent.pinchart@ideasonboard.com"},"content":"Hi Paul,\n\nThank you for the patch.\n\nOn Fri, Nov 11, 2022 at 02:31:47AM +0900, Paul Elder wrote:\n> Add an ALSC module for Raspberry Pi.\n> \n> Signed-off-by: Paul Elder \n> Reviewed-by: Laurent Pinchart \n> \n> ---\n> Changes in v3:\n> - add file description\n> - add comment about type name change\n> - override the type name to alsc now that the base type name is lsc\n> - use Param's getter\n> \n> New in v2\n> ---\n> .../tuning/libtuning/modules/lsc/__init__.py | 1 +\n> .../libtuning/modules/lsc/raspberrypi.py | 250 ++++++++++++++++++\n> 2 files changed, 251 insertions(+)\n> create mode 100644 utils/tuning/libtuning/modules/lsc/raspberrypi.py\n> \n> diff --git a/utils/tuning/libtuning/modules/lsc/__init__.py b/utils/tuning/libtuning/modules/lsc/__init__.py\n> index 47d9b846..7cdecdb8 100644\n> --- a/utils/tuning/libtuning/modules/lsc/__init__.py\n> +++ b/utils/tuning/libtuning/modules/lsc/__init__.py\n> @@ -3,3 +3,4 @@\n> # Copyright (C) 2022, Paul Elder \n> \n> from libtuning.modules.lsc.lsc import LSC\n> +from libtuning.modules.lsc.raspberrypi import ALSCRaspberryPi\n> diff --git a/utils/tuning/libtuning/modules/lsc/raspberrypi.py b/utils/tuning/libtuning/modules/lsc/raspberrypi.py\n> new file mode 100644\n> index 00000000..7fd346cc\n> --- /dev/null\n> +++ b/utils/tuning/libtuning/modules/lsc/raspberrypi.py\n> @@ -0,0 +1,250 @@\n> +# SPDX-License-Identifier: BSD-2-Clause\n> +#\n> +# Copyright (C) 2019, Raspberry Pi Ltd\n> +# Copyright (C) 2022, Paul Elder \n> +#\n> +# raspberrypi.py - ALSC module for tuning Raspberry Pi\n> +\n> +from .lsc import LSC\n> +\n> +import libtuning as lt\n> +import libtuning.utils as utils\n> +\n> +from numbers import Number\n> +import numpy as np\n> +\n> +\n> +class ALSCRaspberryPi(LSC):\n> + # Override the type name so that the parser can match the entry in the\n> + # config file.\n> + type = 'alsc'\n> + hr_name = 'ALSC (Raspberry Pi)'\n> + out_name = 'rpi.alsc'\n> + compatible = ['raspberrypi']\n> +\n> + def __init__(self, *,\n> + do_color: lt.Param,\n> + luminance_strength: lt.Param,\n> + **kwargs):\n> + super().__init__(**kwargs)\n> +\n> + self.do_color = do_color\n> + self.luminance_strength = luminance_strength\n> +\n> + self.output_range = (0, 3.999)\n> +\n> + def validate_config(self, config: dict) -> bool:\n> + if self not in config:\n> + utils.eprint(f'{self.type} not in config')\n> + return False\n> +\n> + valid = True\n> +\n> + conf = config[self]\n> +\n> + lum_key = self.luminance_strength.name\n> + color_key = self.do_color.name\n> +\n> + if lum_key not in conf and self.luminance_strength.required:\n> + utils.eprint(f'{lum_key} is not in config')\n> + valid = False\n> +\n> + if lum_key in conf and (conf[lum_key] < 0 or conf[lum_key] > 1):\n> + utils.eprint(f'Warning: {lum_key} is not in range [0, 1]; defaulting to 0.5')\n> +\n> + if color_key not in conf and self.do_color.required:\n> + utils.eprint(f'{color_key} is not in config')\n> + valid = False\n> +\n> + return valid\n> +\n> + # @return Image color temperature, flattened array of red calibration table\n> + # (containing {sector size} elements), flattened array of blue\n> + # calibration table, flattened array of green calibration\n> + # table\n> +\n> + def _do_single_alsc(self, image: lt.Image, do_alsc_colour: bool):\n> + average_green = np.mean((image.channels[lt.Color.GR:lt.Color.GB + 1]), axis=0)\n> +\n> + cg, g = self._lsc_single_channel(average_green, image)\n> +\n> + if not do_alsc_colour:\n> + return image.color, None, None, cg.flatten()\n> +\n> + cr, _ = self._lsc_single_channel(image.channels[lt.Color.R], image, g)\n> + cb, _ = self._lsc_single_channel(image.channels[lt.Color.B], image, g)\n> +\n> + # \\todo implement debug\n> +\n> + return image.color, cr.flatten(), cb.flatten(), cg.flatten()\n> +\n> + # @return Red shading table, Blue shading table, Green shading table,\n> + # number of images processed\n> +\n> + def _do_all_alsc(self, images: list, do_alsc_colour: bool, general_conf: dict) -> (list, list, list, Number, int):\n> + # List of colour temperatures\n> + list_col = []\n> + # Associated calibration tables\n> + list_cr = []\n> + list_cb = []\n> + list_cg = []\n> + count = 0\n> + for image in self._enumerate_lsc_images(images):\n> + col, cr, cb, cg = self._do_single_alsc(image, do_alsc_colour)\n> + list_col.append(col)\n> + list_cr.append(cr)\n> + list_cb.append(cb)\n> + list_cg.append(cg)\n> + count += 1\n> +\n> + # Convert to numpy array for data manipulation\n> + list_col = np.array(list_col)\n> + list_cr = np.array(list_cr)\n> + list_cb = np.array(list_cb)\n> + list_cg = np.array(list_cg)\n> +\n> + cal_cr_list = []\n> + cal_cb_list = []\n> +\n> + # Note: Calculation of average corners and center of the shading tables\n> + # has been removed (which ctt had, as it was being unused)\n> +\n> + # \\todo Handle the second green table\n> +\n> + # Average all values for luminance shading and return one table for all temperatures\n> + lum_lut = list(utils.round_with_sigfigs(np.mean(list_cg, axis=0), self.output_range))\n> +\n> + if not do_alsc_colour:\n> + return None, None, lum_lut, count\n> +\n> + for ct in sorted(set(list_col)):\n> + # Average tables for the same colour temperature\n> + indices = np.where(list_col == ct)\n> + ct = int(ct)\n> + t_r = utils.round_with_sigfigs(np.mean(list_cr[indices], axis=0),\n> + self.output_range)\n> + t_b = utils.round_with_sigfigs(np.mean(list_cb[indices], axis=0),\n> + self.output_range)\n> +\n> + cr_dict = {\n> + 'ct': ct,\n> + 'table': list(t_r)\n> + }\n> + cb_dict = {\n> + 'ct': ct,\n> + 'table': list(t_b)\n> + }\n> + cal_cr_list.append(cr_dict)\n> + cal_cb_list.append(cb_dict)\n> +\n> + return cal_cr_list, cal_cb_list, lum_lut, count\n> +\n> + # @brief Calculate sigma from two adjacent gain tables\n> + def _calcSigma(self, g1, g2):\n> + g1 = np.reshape(g1, self.sector_shape[::-1])\n> + g2 = np.reshape(g2, self.sector_shape[::-1])\n> +\n> + # Apply gains to gain table\n> + gg = g1 / g2\n> + if np.mean(gg) < 1:\n> + gg = 1 / gg\n> +\n> + # For each internal patch, compute average difference between it and\n> + # its 4 neighbours, then append to list\n> + diffs = []\n> + for i in range(self.sector_shape[1] - 2):\n> + for j in range(self.sector_shape[0] - 2):\n> + # Indexing is incremented by 1 since all patches on borders are\n> + # not counted\n> + diff = np.abs(gg[i + 1][j + 1] - gg[i][j + 1])\n> + diff += np.abs(gg[i + 1][j + 1] - gg[i + 2][j + 1])\n> + diff += np.abs(gg[i + 1][j + 1] - gg[i + 1][j])\n> + diff += np.abs(gg[i + 1][j + 1] - gg[i + 1][j + 2])\n> + diffs.append(diff / 4)\n> +\n> + mean_diff = np.mean(diffs)\n> + return (np.round(mean_diff, 5))\n\nNo need for the outer parentheses.\n\n> +\n> + # @brief Obtains sigmas for red and blue, effectively a measure of the\n> + # 'error'\n> + def _get_sigma(self, cal_cr_list, cal_cb_list):\n> + # Provided colour alsc tables were generated for two different colour\n> + # temperatures sigma is calculated by comparing two calibration temperatures\n> + # adjacent in colour space\n> +\n> + color_temps = [cal['ct'] for cal in cal_cr_list]\n> +\n> + # Calculate sigmas for each adjacent color_temps and return worst one\n> + sigma_rs = []\n> + sigma_bs = []\n> + for i in range(len(color_temps) - 1):\n> + sigma_rs.append(self._calcSigma(cal_cr_list[i]['table'], cal_cr_list[i + 1]['table']))\n> + sigma_bs.append(self._calcSigma(cal_cb_list[i]['table'], cal_cb_list[i + 1]['table']))\n> +\n> + # Return maximum sigmas, not necessarily from the same colour\n> + # temperature interval\n> + sigma_r = max(sigma_rs) if sigma_rs else 0.005\n> + sigma_b = max(sigma_bs) if sigma_bs else 0.005\n> +\n> + return sigma_r, sigma_b\n> +\n> + def process(self, args, config: dict, images: list, outputs: dict) -> dict:\n> + output = {\n> + 'omega': 1.3,\n> + 'n_iter': 100,\n> + 'luminance_strength': 0.7\n> + }\n> +\n> + conf = config[self]\n> + general_conf = config['general']\n> +\n> + do_alsc_colour = self.do_color.get_value(conf)\n> +\n> + # \\todo I have no idea where this input parameter is used\n> + luminance_strength = self.luminance_strength.get_value(conf)\n> + if luminance_strength < 0 or luminance_strength > 1:\n> + luminance_strength = 0.5\n> +\n> + output['luminance_strength'] = luminance_strength\n> +\n> + # \\todo Validate images from greyscale camera and force grescale mode\n> + # \\todo Debug functionality\n> +\n> + alsc_out = self._do_all_alsc(images, do_alsc_colour, general_conf)\n> + # \\todo Handle the second green lut\n> + cal_cr_list, cal_cb_list, luminance_lut, count = alsc_out\n> +\n> + if not do_alsc_colour:\n> + output['luminance_lut'] = luminance_lut\n> + output['n_iter'] = 0\n> + return output\n> +\n> + output['calibrations_Cr'] = cal_cr_list\n> + output['calibrations_Cb'] = cal_cb_list\n> + output['luminance_lut'] = luminance_lut\n> +\n> + # The sigmas determine the strength of the adaptive algorithm, that\n> + # cleans up any lens shading that has slipped through the alsc. These\n> + # are determined by measuring a 'worst-case' difference between two\n> + # alsc tables that are adjacent in colour space. If, however, only one\n> + # colour temperature has been provided, then this difference can not be\n> + # computed as only one table is available.\n> + # To determine the sigmas you would have to estimate the error of an\n> + # alsc table with only the image it was taken on as a check. To avoid\n> + # circularity, dfault exaggerated sigmas are used, which can result in\n> + # too much alsc and is therefore not advised.\n> + # In general, just take another alsc picture at another colour\n> + # temperature!\n> +\n> + if count == 1:\n> + output['sigma'] = 0.005\n> + output['sigma_Cb'] = 0.005\n> + utils.eprint('Warning: Only one alsc calibration found; standard sigmas used for adaptive algorithm.')\n> + return output\n> +\n> + # Obtain worst-case scenario residual sigmas\n> + sigma_r, sigma_b = self._get_sigma(cal_cr_list, cal_cb_list)\n> + output['sigma'] = np.round(sigma_r, 5)\n> + output['sigma_Cb'] = np.round(sigma_b, 5)\n> +\n> + return output","headers":{"Return-Path":"","X-Original-To":"parsemail@patchwork.libcamera.org","Delivered-To":"parsemail@patchwork.libcamera.org","Received":["from lancelot.ideasonboard.com (lancelot.ideasonboard.com\n\t[92.243.16.209])\n\tby patchwork.libcamera.org (Postfix) with ESMTPS id D71C1BE08B\n\tfor ;\n\tWed, 23 Nov 2022 00:16:42 +0000 (UTC)","from lancelot.ideasonboard.com (localhost [IPv6:::1])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTP id 05BC763314;\n\tWed, 23 Nov 2022 01:16:42 +0100 (CET)","from perceval.ideasonboard.com (perceval.ideasonboard.com\n\t[213.167.242.64])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id B676961F2B\n\tfor ;\n\tWed, 23 Nov 2022 01:16:40 +0100 (CET)","from pendragon.ideasonboard.com (62-78-145-57.bb.dnainternet.fi\n\t[62.78.145.57])\n\tby perceval.ideasonboard.com (Postfix) with ESMTPSA id 22C97890;\n\tWed, 23 Nov 2022 01:16:40 +0100 (CET)"],"DKIM-Signature":["v=1; a=rsa-sha256; c=relaxed/simple; d=libcamera.org;\n\ts=mail; t=1669162602;\n\tbh=VE/JtKr7LwYN+ONqSvXdK7Y9T3GzYztXuWmLqOKZskc=;\n\th=Date:To:References:In-Reply-To:Subject:List-Id:List-Unsubscribe:\n\tList-Archive:List-Post:List-Help:List-Subscribe:From:Reply-To:Cc:\n\tFrom;\n\tb=G1T9H58Z2qaFl44jVHZXfJ4byCPHLWBoHG+pqF4wjTlShgogct/MgSaicfKO/HVc8\n\tGNh/Ih1joovDt9AwgBWChMSIyXAIhhZKbfuhwuiw5ABetW3cbjTSTwUqNQuoDJ45dd\n\tmStiqypfxTfaBEnMuPLdNHSdlsIey0pENgN2POAV5aufuIlv7rvFibQOT1Lm08kMfC\n\tyqBVd1otYr5aV3VeeAVFncWFFHtqDsabPvpraf8qW1VLKlU+MexRjKCPdPmhxQgBCv\n\tFeKNs94LJNhYGhQQki79pj/sTO7zcH1cKddNw3KOsaJOKXs1t7mFixtep4YxSlHSvc\n\thnSQNQ98tDlpw==","v=1; a=rsa-sha256; c=relaxed/simple; d=ideasonboard.com;\n\ts=mail; t=1669162600;\n\tbh=VE/JtKr7LwYN+ONqSvXdK7Y9T3GzYztXuWmLqOKZskc=;\n\th=Date:From:To:Cc:Subject:References:In-Reply-To:From;\n\tb=D6PaaiiRfUo413VF/2v2H4Fec3Ivu/5D7JmA/jKcAbNmkVdhwy9ehdsbZt2d7+hju\n\tKj0OSi6bojo6K//ud0OX7gMCUQo3KDfwp5KgZSMciQec05y+1u1KsGD21C1YXkqZDz\n\trEV/fK0tdFieaiKOPisUPlQ6y6YYuSltktrl3UXM="],"Authentication-Results":"lancelot.ideasonboard.com; dkim=pass (1024-bit key; \n\tunprotected) header.d=ideasonboard.com\n\theader.i=@ideasonboard.com\n\theader.b=\"D6PaaiiR\"; dkim-atps=neutral","Date":"Wed, 23 Nov 2022 02:16:24 +0200","To":"Paul Elder ","Message-ID":"","References":"<20221110173154.488445-1-paul.elder@ideasonboard.com>\n\t<20221110173154.488445-6-paul.elder@ideasonboard.com>","MIME-Version":"1.0","Content-Type":"text/plain; charset=utf-8","Content-Disposition":"inline","In-Reply-To":"<20221110173154.488445-6-paul.elder@ideasonboard.com>","Subject":"Re: [libcamera-devel] [PATCH v3 05/12] utils: libtuning: modules:\n\talsc: Add raspberrypi ALSC module","X-BeenThere":"libcamera-devel@lists.libcamera.org","X-Mailman-Version":"2.1.29","Precedence":"list","List-Id":"","List-Unsubscribe":",\n\t","List-Archive":"","List-Post":"","List-Help":"","List-Subscribe":",\n\t","From":"Laurent Pinchart via libcamera-devel\n\t","Reply-To":"Laurent Pinchart ","Cc":"libcamera-devel@lists.libcamera.org","Errors-To":"libcamera-devel-bounces@lists.libcamera.org","Sender":"\"libcamera-devel\" "}}]