{"id":17535,"url":"https://patchwork.libcamera.org/api/patches/17535/?format=json","web_url":"https://patchwork.libcamera.org/patch/17535/","project":{"id":1,"url":"https://patchwork.libcamera.org/api/projects/1/?format=json","name":"libcamera","link_name":"libcamera","list_id":"libcamera_core","list_email":"libcamera-devel@lists.libcamera.org","web_url":"","scm_url":"","webscm_url":""},"msgid":"<20221006120105.3861831-3-paul.elder@ideasonboard.com>","date":"2022-10-06T12:01:00","name":"[libcamera-devel,2/7] utils: libtuning: modules: Add ALSC module","commit_ref":null,"pull_url":null,"state":"superseded","archived":false,"hash":"fb28aae3b4768afc1f5cee5eb7143c39dceca626","submitter":{"id":17,"url":"https://patchwork.libcamera.org/api/people/17/?format=json","name":"Paul Elder","email":"paul.elder@ideasonboard.com"},"delegate":null,"mbox":"https://patchwork.libcamera.org/patch/17535/mbox/","series":[{"id":3536,"url":"https://patchwork.libcamera.org/api/series/3536/?format=json","web_url":"https://patchwork.libcamera.org/project/libcamera/list/?series=3536","date":"2022-10-06T12:00:58","name":"utils: tuning: Add a new tuning infrastructure","version":1,"mbox":"https://patchwork.libcamera.org/series/3536/mbox/"}],"comments":"https://patchwork.libcamera.org/api/patches/17535/comments/","check":"pending","checks":"https://patchwork.libcamera.org/api/patches/17535/checks/","tags":{},"headers":{"Return-Path":"","X-Original-To":"parsemail@patchwork.libcamera.org","Delivered-To":"parsemail@patchwork.libcamera.org","Received":["from lancelot.ideasonboard.com (lancelot.ideasonboard.com\n\t[92.243.16.209])\n\tby patchwork.libcamera.org (Postfix) with ESMTPS id D0709C0DA4\n\tfor ;\n\tThu, 6 Oct 2022 12:01:23 +0000 (UTC)","from lancelot.ideasonboard.com (localhost [IPv6:::1])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTP id 9219362CF8;\n\tThu, 6 Oct 2022 14:01:23 +0200 (CEST)","from perceval.ideasonboard.com (perceval.ideasonboard.com\n\t[213.167.242.64])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id 7EAC962CF2\n\tfor ;\n\tThu, 6 Oct 2022 14:01:22 +0200 (CEST)","from pyrite.rasen.tech (h175-177-042-159.catv02.itscom.jp\n\t[175.177.42.159])\n\tby perceval.ideasonboard.com (Postfix) with ESMTPSA id DC7B94DD;\n\tThu, 6 Oct 2022 14:01:20 +0200 (CEST)"],"DKIM-Signature":["v=1; a=rsa-sha256; c=relaxed/simple; d=libcamera.org;\n\ts=mail; t=1665057683;\n\tbh=c+qS29ldGV7Aa1nD7UelnDk7nzDuEuxLLD/x4uniRP4=;\n\th=To:Date:In-Reply-To:References:Subject:List-Id:List-Unsubscribe:\n\tList-Archive:List-Post:List-Help:List-Subscribe:From:Reply-To:\n\tFrom;\n\tb=3cQrxvgClIDiSFcd6qdkzdlA6aBDNbJ03G0qZtE7BkmtRAO2BFWtlsEOCrmuBRXfm\n\tPO17VcWBUGSiFGEvmyX+15U4z2WSzGVEO3V+PhmG7V/VeyVnyInC2wfMToJplhwsGv\n\tUcVaxgsD/GiomyEJSo5DtXnXPdwZOJb641azS8ohpLtVzhMIkZ49NQO2xk2otDmrho\n\tSAi3U7A9EtR/n3vg3m1qWEeVwitlhU9zu1JJ/hWtuagPH0aqofjbdkTMbH1HrvFQpk\n\tT9MTf5i7K37K/oMTnbbkSbcmGU31B9gteo3A4Bu4W91vLRyg7mZQNLy3Ewmjk7RS94\n\tELkYTTwbHw3iA==","v=1; a=rsa-sha256; c=relaxed/simple; d=ideasonboard.com;\n\ts=mail; t=1665057682;\n\tbh=c+qS29ldGV7Aa1nD7UelnDk7nzDuEuxLLD/x4uniRP4=;\n\th=From:To:Cc:Subject:Date:In-Reply-To:References:From;\n\tb=B17kxgAKGzgMufbZ0+zJZUhbQBgLdaD3uLSEW1PE25ijcecbsbhaaZARQtiGO4odB\n\t7zRURrGK98COOC3ap8+1dNJfvHrdddSv4dU4dL43Kn0JQg1CGH6WWvNeLi7ILe0H8E\n\t/x0H5fRTUH0+EH6VhjxnVPX1ZGXs2j7jlo/FTdNg="],"Authentication-Results":"lancelot.ideasonboard.com; dkim=pass (1024-bit key; \n\tunprotected) header.d=ideasonboard.com\n\theader.i=@ideasonboard.com\n\theader.b=\"B17kxgAK\"; dkim-atps=neutral","To":"libcamera-devel@lists.libcamera.org","Date":"Thu, 6 Oct 2022 21:01:00 +0900","Message-Id":"<20221006120105.3861831-3-paul.elder@ideasonboard.com>","X-Mailer":"git-send-email 2.30.2","In-Reply-To":"<20221006120105.3861831-1-paul.elder@ideasonboard.com>","References":"<20221006120105.3861831-1-paul.elder@ideasonboard.com>","MIME-Version":"1.0","Content-Transfer-Encoding":"8bit","Subject":"[libcamera-devel] [PATCH 2/7] utils: libtuning: modules: Add ALSC\n\tmodule","X-BeenThere":"libcamera-devel@lists.libcamera.org","X-Mailman-Version":"2.1.29","Precedence":"list","List-Id":"","List-Unsubscribe":",\n\t","List-Archive":"","List-Post":"","List-Help":"","List-Subscribe":",\n\t","From":"Paul Elder via libcamera-devel ","Reply-To":"Paul Elder ","Errors-To":"libcamera-devel-bounces@lists.libcamera.org","Sender":"\"libcamera-devel\" "},"content":"Add an ALSC module to libtuning's collection of modules. It is based on\nraspberrypi's ctt's ALSC, but customizable for different lens shading\ntable sizes, among other things.\n\nSigned-off-by: Paul Elder \n---\n utils/tuning/libtuning/modules/__init__.py | 1 +\n utils/tuning/libtuning/modules/alsc.py | 321 +++++++++++++++++++++\n 2 files changed, 322 insertions(+)\n create mode 100644 utils/tuning/libtuning/modules/alsc.py","diff":"diff --git a/utils/tuning/libtuning/modules/__init__.py b/utils/tuning/libtuning/modules/__init__.py\nindex e69de29b..f41ef550 100644\n--- a/utils/tuning/libtuning/modules/__init__.py\n+++ b/utils/tuning/libtuning/modules/__init__.py\n@@ -0,0 +1 @@\n+from libtuning.modules.alsc import ALSC\ndiff --git a/utils/tuning/libtuning/modules/alsc.py b/utils/tuning/libtuning/modules/alsc.py\nnew file mode 100644\nindex 00000000..788cf2a9\n--- /dev/null\n+++ b/utils/tuning/libtuning/modules/alsc.py\n@@ -0,0 +1,321 @@\n+from .module import Module\n+\n+import libtuning as lt\n+import libtuning.utils as utils\n+\n+import decimal\n+import numbers\n+import numpy as np\n+from typing import Callable\n+\n+\n+class ALSC(Module):\n+ def __init__(self, *,\n+ do_color: lt.param,\n+ debug: list,\n+ luminance_strength: lt.param,\n+ sector_shape: tuple,\n+ sector_x_gradient: lt.gradient.Gradient,\n+ sector_y_gradient: lt.gradient.Gradient,\n+ sector_x_remainder: lt.remainder = None,\n+ sector_y_remainder: lt.remainder = None,\n+ sector_average_function: Callable,\n+ sector_average_function_parameters: list = [],\n+ smoothing_function: Callable,\n+ smoothing_parameters: list = [],\n+ output_type: type,\n+ output_color_channels: list,\n+ output_range: tuple):\n+ super().__init__()\n+ self.hr_name = \"ALSC\"\n+ self.name = \"alsc\"\n+ self.options = {}\n+\n+ if (sector_x_gradient is lt.gradient.Linear and sector_x_remainder is None):\n+ raise ValueError('sector_x_remainder must be specified if sector_x_gradient is Linear')\n+\n+ if (sector_y_gradient is lt.gradient.Linear and sector_y_remainder is None):\n+ raise ValueError('sector_y_remainder must be specified if sector_y_gradient is Linear')\n+\n+ self.do_color = do_color\n+ self.luminance_strength = luminance_strength\n+\n+ self.debug = debug\n+\n+ self.sector_shape = sector_shape\n+ self.sector_x_gradient = sector_x_gradient(sector_x_remainder)\n+ self.sector_y_gradient = sector_y_gradient(sector_y_remainder)\n+ self.sector_average_function = sector_average_function(sector_average_function_parameters)\n+\n+ self.smoothing_function = smoothing_function(smoothing_parameters)\n+\n+ # todo This isn't actually used. Can we assume/force float? Is there a\n+ # use case for non-float? Or maybe we should use numpy types like\n+ # float64? I don't think it makes a difference in the output tuning\n+ # anyway though.\n+ self.output_type = output_type\n+\n+ # This isn't used for now. Once we add support for two green channels\n+ # we can.\n+ self.output_color_channels = output_color_channels\n+\n+ self.output_range = output_range\n+\n+ def enumerate_alsc_images(images):\n+ for image in images:\n+ if image.alsc_only:\n+ yield image\n+\n+ def __validateConfig__(self, config: dict) -> bool:\n+ if not super().__validateConfig__(config):\n+ return False\n+\n+ conf = config[self]\n+ valid = True\n+\n+ lum_key = self.luminance_strength.name\n+ color_key = self.do_color.name\n+\n+ if lum_key not in conf and self.luminance_strength.isRequired():\n+ utils.eprint(f'{lum_key} is not in config')\n+ valid = False\n+\n+ if lum_key in conf and (conf[lum_key] < 0 or conf[lum_key] > 1):\n+ utils.eprint(f'Warning: {lum_key} is not in range [0, 1]; defaulting to 0.5')\n+\n+ if color_key not in conf and self.do_color.isRequired():\n+ utils.eprint(f'{color_key} is not in config')\n+ valid = False\n+\n+ return valid\n+\n+ def _getGrid(self, channel, image):\n+ # List of number of pixels in each sector\n+ sectors_x = self.sector_x_gradient.distribute(image.w, self.sector_shape[0])\n+ sectors_y = self.sector_y_gradient.distribute(image.h, self.sector_shape[1])\n+\n+ grid = []\n+\n+ r = 0\n+ for y in sectors_y:\n+ c = 0\n+ for x in sectors_x:\n+ grid.append(self.sector_average_function(channel[r:r + y, c:c + x]))\n+ c += x\n+ r += y\n+\n+ return np.array(grid)\n+\n+ # @return Image color temperature, flattened array of red calibration table\n+ # (containing {sector size} elements), flattened array of blue\n+ # calibration table, flattened array of (red's) green calibration\n+ # table, flattened array of (blue's) green calibration table\n+\n+ def _doSingleAlsc(self, image: lt.Image, do_alsc_colour: bool):\n+ # Get channel in correct order: [R, GR, GB, B]\n+ channels = [image.channels[i] for i in image.order]\n+\n+ channels.append(np.mean((channels[color.GR:color.GB + 1]), axis=0))\n+\n+ g = self._getGrid(av_ch_g, image) - image.blacklevel_16\n+ cg = np.reshape(1 / g, self.sector_shape[::-1])\n+ cg = self.smoothing_function.smoothing(cg, 3)\n+ cg = cg / np.min(cg)\n+\n+ if not do_alsc_colour:\n+ return image.color, None, None, cg.flatten(), cg.flatten()\n+\n+ r = self._getGrid(channels[0], image) - image.blacklevel_16\n+ cr = np.reshape(g / r, self.sector_shape[::-1])\n+ cr = self.smoothing_function.smoothing(cr, 3)\n+\n+ b = self._getGrid(channels[3], image) - image.blacklevel_16\n+ cb = np.reshape(g / b, self.sector_shape[::-1])\n+ cb = self.smoothing_function.smoothing(cb, 3)\n+\n+ # todo implement debug\n+\n+ return image.color, cr.flatten(), cb.flatten(), cg.flatten(), cg.flatten()\n+\n+ # todo Figure out if we can do something cooler if we can keep the greens\n+ # separate. For now, just average them and use the average, but duplicate\n+ # them so that it's easier to handle them in the future, especially since\n+ # some platforms have separate tables for GR and GB.\n+ #\n+ # todo Should we return a third shading table for the average of greens?\n+ # For platforms that only want one green, as opposed to just using red's\n+ # green shading table\n+ #\n+ # @return Red shading table, Blue shading table, (red's) Green shading\n+ # table, (blue's) Green shading table, number of images processed\n+\n+ def _doAllAlsc(self, images: list, do_alsc_colour: bool, general_conf: dict) -> (list, list, list, numbers.Number, int):\n+ # List of colour temperatures\n+ list_col = []\n+ # Associated calibration tables\n+ list_cr = []\n+ list_cb = []\n+ list_cg1 = []\n+ list_cg2 = []\n+ count = 0\n+ for image in enumerate_alsc_images(images):\n+ col, cr, cb, cg1, cg2 = self._doSingleAlsc(image, do_alsc_colour)\n+ list_col.append(col)\n+ list_cr.append(cr)\n+ list_cb.append(cb)\n+ list_cg1.append(cg1)\n+ list_cg2.append(cg2)\n+ count += 1\n+\n+ # Convert to numpy array for data manipulation\n+ list_col = np.array(list_col)\n+ list_cr = np.array(list_cr)\n+ list_cb = np.array(list_cb)\n+ list_cg1 = np.array(list_cg1)\n+ list_cg2 = np.array(list_cg2)\n+\n+ cal_cr_list = []\n+ cal_cb_list = []\n+\n+ # Note: Calculation of average corners and center of the shading tables\n+ # has been removed (which ctt had, as it was being unused)\n+\n+ # todo Handle the second green table\n+\n+ # Average all values for luminance shading and return one table for all temperatures\n+ lum_lut = roundWithSigfigs(np.mean(list_cg1, axis=0), self.output_range)\n+\n+ if not do_alsc_colour:\n+ return None, None, lum_lut, lum_lut, count\n+\n+ for ct in sorted(set(list_col)):\n+ # Average tables for the same colour temperature\n+ indices = np.where(list_col == ct)\n+ ct = int(ct)\n+ t_r = utils.roundWithSigfigs(np.mean(list_cr[indices], axis=0),\n+ self.output_range)\n+ t_b = utils.roundWithSigfigs(np.mean(list_cb[indices], axis=0),\n+ self.output_range)\n+\n+ cr_dict = {\n+ 'ct': ct,\n+ 'table': list(t_r)\n+ }\n+ cb_dict = {\n+ 'ct': ct,\n+ 'table': list(t_b)\n+ }\n+ cal_cr_list.append(cr_dict)\n+ cal_cb_list.append(cb_dict)\n+\n+ return cal_cr_list, cal_cb_list, lum_lut, lum_lut, count\n+\n+ # @brief Calculate sigma from two adjacent gain tables\n+ def _calcSigma(self, g1, g2):\n+ g1 = np.reshape(g1, self.sector_shape[::-1])\n+ g2 = np.reshape(g2, self.sector_shape[::-1])\n+\n+ # Apply gains to gain table\n+ gg = g1 / g2\n+ if np.mean(gg) < 1:\n+ gg = 1 / gg\n+\n+ # For each internal patch, compute average difference between it and\n+ # its 4 neighbours, then append to list\n+ diffs = []\n+ for i in range(self.sector_shape[1] - 2):\n+ for j in range(self.sector_shape[0] - 2):\n+ # Indexing is incremented by 1 since all patches on borders are\n+ # not counted\n+ diff = np.abs(gg[i + 1][j + 1] - gg[i][j + 1])\n+ diff += np.abs(gg[i + 1][j + 1] - gg[i + 2][j + 1])\n+ diff += np.abs(gg[i + 1][j + 1] - gg[i + 1][j])\n+ diff += np.abs(gg[i + 1][j + 1] - gg[i + 1][j + 2])\n+ diffs.append(diff / 4)\n+\n+ mean_diff = np.mean(diffs)\n+ return(np.round(mean_diff, 5))\n+\n+ # @brief Obtains sigmas for red and blue, effectively a measure of the\n+ # 'error'\n+ def _getSigma(self, cal_cr_list, cal_cb_list):\n+ # Provided colour alsc tables were generated for two different colour\n+ # temperatures sigma is calculated by comparing two calibration temperatures\n+ # adjacent in colour space\n+\n+ color_temps = [cal['ct'] for cal in cal_cr_list]\n+\n+ # Calculate sigmas for each adjacent color_temps and return worst one\n+ sigma_rs = []\n+ sigma_bs = []\n+ for i in range(len(color_temps) - 1):\n+ sigma_rs.append(self._calcSigma(cal_cr_list[i]['table'], cal_cr_list[i + 1]['table']))\n+ sigma_bs.append(self._calcSigma(cal_cb_list[i]['table'], cal_cb_list[i + 1]['table']))\n+\n+ # Return maximum sigmas, not necessarily from the same colour\n+ # temperature interval\n+ sigma_r = max(sigma_rs) if sigma_rs else 0.005\n+ sigma_b = max(sigma_bs) if sigma_bs else 0.005\n+\n+ return sigma_r, sigma_b\n+\n+ def __process__(self, args, config: dict, images: list, outputs: dict) -> dict:\n+ output = {\n+ 'omega': 1.3,\n+ 'n_iter': 100,\n+ 'luminance_strength': 0.7\n+ }\n+\n+ conf = config[self]\n+ general_conf = config['general']\n+\n+ do_alsc_colour = self.do_color.getValue(conf)\n+\n+ # todo I have no idea where this input parameter is used\n+ luminance_strength = self.luminance_strength.getValue(conf)\n+ if luminance_strength < 0 or luminance_strength > 1:\n+ luminance_strength = 0.5\n+\n+ output['luminance_strength'] = luminance_strength\n+\n+ # todo Validate images from greyscale camera and force grescale mode\n+ # todo Debug functionality\n+\n+ alsc_out = self._doAllAlsc(images, do_alsc_colour, general_conf)\n+ # todo Handle the second green lut\n+ cal_cr_list, cal_cb_list, luminance_lut, _, count = alsc_out\n+\n+ output['luminance_lut'] = luminance_lut\n+\n+ if not do_alsc_colour:\n+ output['n_iter'] = 0\n+ return output\n+\n+ output['calibrations_Cr'] = cal_cr_list\n+ output['calibrations_Cb'] = cal_cb_list\n+\n+ # The sigmas determine the strength of the adaptive algorithm, that\n+ # cleans up any lens shading that has slipped through the alsc. These\n+ # are determined by measuring a 'worst-case' difference between two\n+ # alsc tables that are adjacent in colour space. If, however, only one\n+ # colour temperature has been provided, then this difference can not be\n+ # computed as only one table is available.\n+ # To determine the sigmas you would have to estimate the error of an\n+ # alsc table with only the image it was taken on as a check. To avoid\n+ # circularity, dfault exaggerated sigmas are used, which can result in\n+ # too much alsc and is therefore not advised.\n+ # In general, just take another alsc picture at another colour\n+ # temperature!\n+\n+ if count == 1:\n+ output['sigma'] = 0.005\n+ output['sigma_Cb'] = 0.005\n+ utils.eprint('Warning: Only one alsc calibration found; standard sigmas used for adaptive algorithm.')\n+ return output\n+\n+ # Obtain worst-case scenario residual sigmas\n+ sigma_r, sigma_b = self._getSigma(self, cal_cr_list, cal_cb_list)\n+ output['sigma'] = np.round(sigma_r, 5)\n+ output['sigma_Cb'] = np.round(sigma_b, 5)\n+\n+ return output\n","prefixes":["libcamera-devel","2/7"]}