From patchwork Thu Nov 24 11:35:43 2022 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Paul Elder X-Patchwork-Id: 17871 Return-Path: X-Original-To: parsemail@patchwork.libcamera.org Delivered-To: parsemail@patchwork.libcamera.org Received: from lancelot.ideasonboard.com (lancelot.ideasonboard.com [92.243.16.209]) by patchwork.libcamera.org (Postfix) with ESMTPS id B3553BDE6B for ; Thu, 24 Nov 2022 11:36:12 +0000 (UTC) Received: from lancelot.ideasonboard.com (localhost [IPv6:::1]) by lancelot.ideasonboard.com (Postfix) with ESMTP id 70F1963321; Thu, 24 Nov 2022 12:36:12 +0100 (CET) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=libcamera.org; s=mail; t=1669289772; bh=cDCq7JJNR2UWkIvSoj8eSeAuK3jO7tRfJNCRiiTdAog=; h=To:Date:In-Reply-To:References:Subject:List-Id:List-Unsubscribe: List-Archive:List-Post:List-Help:List-Subscribe:From:Reply-To: From; b=Ldpd+DPEsQeOYNJzfsflTdbdG9jcIKfd4GK0iOvQNRvLwmXhruOcTlHdB//gwXa34 BOUpDKU+JLi/dynja53kZGMdJVxlr3B6+wB7tGb2aheDHh3H5tSLLZu07v79/9NsRn kgckAVMzABt9WwzEAzBmewsV3sGeioW+oI4B/lxdW7irZmRLrrBJftfMY8HrFLy8KY 8LHSTgBK8QOMYemwuriBuwqBJTftFlooye9jzgG18ZwVyF99Chz91s3OiT8PgzrphY Alt6UtTcBqD6EMrxYU/lZpT72cjWU9VV2+wGU4/KxKMOOGQEzMPffECd9+aFHtMW+m OwDdrhr0FCAzQ== Received: from perceval.ideasonboard.com (perceval.ideasonboard.com [IPv6:2001:4b98:dc2:55:216:3eff:fef7:d647]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id 28BD563321 for ; Thu, 24 Nov 2022 12:36:11 +0100 (CET) Authentication-Results: lancelot.ideasonboard.com; dkim=pass (1024-bit key; unprotected) header.d=ideasonboard.com header.i=@ideasonboard.com header.b="qyVO3hxn"; dkim-atps=neutral Received: from pyrite.tail37cf.ts.net (h175-177-042-159.catv02.itscom.jp [175.177.42.159]) by perceval.ideasonboard.com (Postfix) with ESMTPSA id 962107FA; Thu, 24 Nov 2022 12:36:09 +0100 (CET) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=ideasonboard.com; s=mail; t=1669289770; bh=cDCq7JJNR2UWkIvSoj8eSeAuK3jO7tRfJNCRiiTdAog=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=qyVO3hxnGpO09hHszj6r+4m6TT4ouhqRAYR1htyFPfCcOQbkmoghHY8gSj6cl4lZd SGkvSmjnGr6Rvviqf5NCCIvoHAlxkdFOXEF817gvm5m7fLxMVpxawsOb2XWHxF9u3J IuJ2TO3/4kaw7qGR+lqI1JQ6+Efr59zOuNyxczKg= To: libcamera-devel@lists.libcamera.org Date: Thu, 24 Nov 2022 20:35:43 +0900 Message-Id: <20221124113550.2182342-6-paul.elder@ideasonboard.com> X-Mailer: git-send-email 2.35.1 In-Reply-To: <20221124113550.2182342-1-paul.elder@ideasonboard.com> References: <20221124113550.2182342-1-paul.elder@ideasonboard.com> MIME-Version: 1.0 Subject: [libcamera-devel] [PATCH v4 05/12] utils: libtuning: modules: alsc: Add raspberrypi ALSC module X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-Patchwork-Original-From: Paul Elder via libcamera-devel From: Paul Elder Reply-To: Paul Elder Errors-To: libcamera-devel-bounces@lists.libcamera.org Sender: "libcamera-devel" Add an ALSC module for Raspberry Pi. Signed-off-by: Paul Elder Reviewed-by: Laurent Pinchart --- Changes in v4: - minor style fix - remove cli args from module.process parameters - replace special rounding with standard rounding as it has higher accuracy Changes in v3: - add file description - add comment about type name change - override the type name to alsc now that the base type name is lsc - use Param's getter New in v2 --- .../tuning/libtuning/modules/lsc/__init__.py | 1 + .../libtuning/modules/lsc/raspberrypi.py | 246 ++++++++++++++++++ 2 files changed, 247 insertions(+) create mode 100644 utils/tuning/libtuning/modules/lsc/raspberrypi.py diff --git a/utils/tuning/libtuning/modules/lsc/__init__.py b/utils/tuning/libtuning/modules/lsc/__init__.py index 47d9b846..7cdecdb8 100644 --- a/utils/tuning/libtuning/modules/lsc/__init__.py +++ b/utils/tuning/libtuning/modules/lsc/__init__.py @@ -3,3 +3,4 @@ # Copyright (C) 2022, Paul Elder from libtuning.modules.lsc.lsc import LSC +from libtuning.modules.lsc.raspberrypi import ALSCRaspberryPi diff --git a/utils/tuning/libtuning/modules/lsc/raspberrypi.py b/utils/tuning/libtuning/modules/lsc/raspberrypi.py new file mode 100644 index 00000000..58f5000d --- /dev/null +++ b/utils/tuning/libtuning/modules/lsc/raspberrypi.py @@ -0,0 +1,246 @@ +# SPDX-License-Identifier: BSD-2-Clause +# +# Copyright (C) 2019, Raspberry Pi Ltd +# Copyright (C) 2022, Paul Elder +# +# raspberrypi.py - ALSC module for tuning Raspberry Pi + +from .lsc import LSC + +import libtuning as lt +import libtuning.utils as utils + +from numbers import Number +import numpy as np + + +class ALSCRaspberryPi(LSC): + # Override the type name so that the parser can match the entry in the + # config file. + type = 'alsc' + hr_name = 'ALSC (Raspberry Pi)' + out_name = 'rpi.alsc' + compatible = ['raspberrypi'] + + def __init__(self, *, + do_color: lt.Param, + luminance_strength: lt.Param, + **kwargs): + super().__init__(**kwargs) + + self.do_color = do_color + self.luminance_strength = luminance_strength + + self.output_range = (0, 3.999) + + def validate_config(self, config: dict) -> bool: + if self not in config: + utils.eprint(f'{self.type} not in config') + return False + + valid = True + + conf = config[self] + + lum_key = self.luminance_strength.name + color_key = self.do_color.name + + if lum_key not in conf and self.luminance_strength.required: + utils.eprint(f'{lum_key} is not in config') + valid = False + + if lum_key in conf and (conf[lum_key] < 0 or conf[lum_key] > 1): + utils.eprint(f'Warning: {lum_key} is not in range [0, 1]; defaulting to 0.5') + + if color_key not in conf and self.do_color.required: + utils.eprint(f'{color_key} is not in config') + valid = False + + return valid + + # @return Image color temperature, flattened array of red calibration table + # (containing {sector size} elements), flattened array of blue + # calibration table, flattened array of green calibration + # table + + def _do_single_alsc(self, image: lt.Image, do_alsc_colour: bool): + average_green = np.mean((image.channels[lt.Color.GR:lt.Color.GB + 1]), axis=0) + + cg, g = self._lsc_single_channel(average_green, image) + + if not do_alsc_colour: + return image.color, None, None, cg.flatten() + + cr, _ = self._lsc_single_channel(image.channels[lt.Color.R], image, g) + cb, _ = self._lsc_single_channel(image.channels[lt.Color.B], image, g) + + # \todo implement debug + + return image.color, cr.flatten(), cb.flatten(), cg.flatten() + + # @return Red shading table, Blue shading table, Green shading table, + # number of images processed + + def _do_all_alsc(self, images: list, do_alsc_colour: bool, general_conf: dict) -> (list, list, list, Number, int): + # List of colour temperatures + list_col = [] + # Associated calibration tables + list_cr = [] + list_cb = [] + list_cg = [] + count = 0 + for image in self._enumerate_lsc_images(images): + col, cr, cb, cg = self._do_single_alsc(image, do_alsc_colour) + list_col.append(col) + list_cr.append(cr) + list_cb.append(cb) + list_cg.append(cg) + count += 1 + + # Convert to numpy array for data manipulation + list_col = np.array(list_col) + list_cr = np.array(list_cr) + list_cb = np.array(list_cb) + list_cg = np.array(list_cg) + + cal_cr_list = [] + cal_cb_list = [] + + # Note: Calculation of average corners and center of the shading tables + # has been removed (which ctt had, as it was unused) + + # Average all values for luminance shading and return one table for all temperatures + lum_lut = list(np.round(np.mean(list_cg, axis=0), 3)) + + if not do_alsc_colour: + return None, None, lum_lut, count + + for ct in sorted(set(list_col)): + # Average tables for the same colour temperature + indices = np.where(list_col == ct) + ct = int(ct) + t_r = np.round(np.mean(list_cr[indices], axis=0), 3) + t_b = np.round(np.mean(list_cb[indices], axis=0), 3) + + cr_dict = { + 'ct': ct, + 'table': list(t_r) + } + cb_dict = { + 'ct': ct, + 'table': list(t_b) + } + cal_cr_list.append(cr_dict) + cal_cb_list.append(cb_dict) + + return cal_cr_list, cal_cb_list, lum_lut, count + + # @brief Calculate sigma from two adjacent gain tables + def _calcSigma(self, g1, g2): + g1 = np.reshape(g1, self.sector_shape[::-1]) + g2 = np.reshape(g2, self.sector_shape[::-1]) + + # Apply gains to gain table + gg = g1 / g2 + if np.mean(gg) < 1: + gg = 1 / gg + + # For each internal patch, compute average difference between it and + # its 4 neighbours, then append to list + diffs = [] + for i in range(self.sector_shape[1] - 2): + for j in range(self.sector_shape[0] - 2): + # Indexing is incremented by 1 since all patches on borders are + # not counted + diff = np.abs(gg[i + 1][j + 1] - gg[i][j + 1]) + diff += np.abs(gg[i + 1][j + 1] - gg[i + 2][j + 1]) + diff += np.abs(gg[i + 1][j + 1] - gg[i + 1][j]) + diff += np.abs(gg[i + 1][j + 1] - gg[i + 1][j + 2]) + diffs.append(diff / 4) + + mean_diff = np.mean(diffs) + return np.round(mean_diff, 5) + + # @brief Obtains sigmas for red and blue, effectively a measure of the + # 'error' + def _get_sigma(self, cal_cr_list, cal_cb_list): + # Provided colour alsc tables were generated for two different colour + # temperatures sigma is calculated by comparing two calibration temperatures + # adjacent in colour space + + color_temps = [cal['ct'] for cal in cal_cr_list] + + # Calculate sigmas for each adjacent color_temps and return worst one + sigma_rs = [] + sigma_bs = [] + for i in range(len(color_temps) - 1): + sigma_rs.append(self._calcSigma(cal_cr_list[i]['table'], cal_cr_list[i + 1]['table'])) + sigma_bs.append(self._calcSigma(cal_cb_list[i]['table'], cal_cb_list[i + 1]['table'])) + + # Return maximum sigmas, not necessarily from the same colour + # temperature interval + sigma_r = max(sigma_rs) if sigma_rs else 0.005 + sigma_b = max(sigma_bs) if sigma_bs else 0.005 + + return sigma_r, sigma_b + + def process(self, config: dict, images: list, outputs: dict) -> dict: + output = { + 'omega': 1.3, + 'n_iter': 100, + 'luminance_strength': 0.7 + } + + conf = config[self] + general_conf = config['general'] + + do_alsc_colour = self.do_color.get_value(conf) + + # \todo I have no idea where this input parameter is used + luminance_strength = self.luminance_strength.get_value(conf) + if luminance_strength < 0 or luminance_strength > 1: + luminance_strength = 0.5 + + output['luminance_strength'] = luminance_strength + + # \todo Validate images from greyscale camera and force grescale mode + # \todo Debug functionality + + alsc_out = self._do_all_alsc(images, do_alsc_colour, general_conf) + # \todo Handle the second green lut + cal_cr_list, cal_cb_list, luminance_lut, count = alsc_out + + if not do_alsc_colour: + output['luminance_lut'] = luminance_lut + output['n_iter'] = 0 + return output + + output['calibrations_Cr'] = cal_cr_list + output['calibrations_Cb'] = cal_cb_list + output['luminance_lut'] = luminance_lut + + # The sigmas determine the strength of the adaptive algorithm, that + # cleans up any lens shading that has slipped through the alsc. These + # are determined by measuring a 'worst-case' difference between two + # alsc tables that are adjacent in colour space. If, however, only one + # colour temperature has been provided, then this difference can not be + # computed as only one table is available. + # To determine the sigmas you would have to estimate the error of an + # alsc table with only the image it was taken on as a check. To avoid + # circularity, dfault exaggerated sigmas are used, which can result in + # too much alsc and is therefore not advised. + # In general, just take another alsc picture at another colour + # temperature! + + if count == 1: + output['sigma'] = 0.005 + output['sigma_Cb'] = 0.005 + utils.eprint('Warning: Only one alsc calibration found; standard sigmas used for adaptive algorithm.') + return output + + # Obtain worst-case scenario residual sigmas + sigma_r, sigma_b = self._get_sigma(cal_cr_list, cal_cb_list) + output['sigma'] = np.round(sigma_r, 5) + output['sigma_Cb'] = np.round(sigma_b, 5) + + return output