From patchwork Thu Oct 6 12:01:00 2022 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Paul Elder X-Patchwork-Id: 17535 Return-Path: X-Original-To: parsemail@patchwork.libcamera.org Delivered-To: parsemail@patchwork.libcamera.org Received: from lancelot.ideasonboard.com (lancelot.ideasonboard.com [92.243.16.209]) by patchwork.libcamera.org (Postfix) with ESMTPS id D0709C0DA4 for ; Thu, 6 Oct 2022 12:01:23 +0000 (UTC) Received: from lancelot.ideasonboard.com (localhost [IPv6:::1]) by lancelot.ideasonboard.com (Postfix) with ESMTP id 9219362CF8; Thu, 6 Oct 2022 14:01:23 +0200 (CEST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=libcamera.org; s=mail; t=1665057683; bh=c+qS29ldGV7Aa1nD7UelnDk7nzDuEuxLLD/x4uniRP4=; h=To:Date:In-Reply-To:References:Subject:List-Id:List-Unsubscribe: List-Archive:List-Post:List-Help:List-Subscribe:From:Reply-To: From; b=3cQrxvgClIDiSFcd6qdkzdlA6aBDNbJ03G0qZtE7BkmtRAO2BFWtlsEOCrmuBRXfm PO17VcWBUGSiFGEvmyX+15U4z2WSzGVEO3V+PhmG7V/VeyVnyInC2wfMToJplhwsGv UcVaxgsD/GiomyEJSo5DtXnXPdwZOJb641azS8ohpLtVzhMIkZ49NQO2xk2otDmrho SAi3U7A9EtR/n3vg3m1qWEeVwitlhU9zu1JJ/hWtuagPH0aqofjbdkTMbH1HrvFQpk T9MTf5i7K37K/oMTnbbkSbcmGU31B9gteo3A4Bu4W91vLRyg7mZQNLy3Ewmjk7RS94 ELkYTTwbHw3iA== Received: from perceval.ideasonboard.com (perceval.ideasonboard.com [213.167.242.64]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id 7EAC962CF2 for ; Thu, 6 Oct 2022 14:01:22 +0200 (CEST) Authentication-Results: lancelot.ideasonboard.com; dkim=pass (1024-bit key; unprotected) header.d=ideasonboard.com header.i=@ideasonboard.com header.b="B17kxgAK"; dkim-atps=neutral Received: from pyrite.rasen.tech (h175-177-042-159.catv02.itscom.jp [175.177.42.159]) by perceval.ideasonboard.com (Postfix) with ESMTPSA id DC7B94DD; Thu, 6 Oct 2022 14:01:20 +0200 (CEST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=ideasonboard.com; s=mail; t=1665057682; bh=c+qS29ldGV7Aa1nD7UelnDk7nzDuEuxLLD/x4uniRP4=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=B17kxgAKGzgMufbZ0+zJZUhbQBgLdaD3uLSEW1PE25ijcecbsbhaaZARQtiGO4odB 7zRURrGK98COOC3ap8+1dNJfvHrdddSv4dU4dL43Kn0JQg1CGH6WWvNeLi7ILe0H8E /x0H5fRTUH0+EH6VhjxnVPX1ZGXs2j7jlo/FTdNg= To: libcamera-devel@lists.libcamera.org Date: Thu, 6 Oct 2022 21:01:00 +0900 Message-Id: <20221006120105.3861831-3-paul.elder@ideasonboard.com> X-Mailer: git-send-email 2.30.2 In-Reply-To: <20221006120105.3861831-1-paul.elder@ideasonboard.com> References: <20221006120105.3861831-1-paul.elder@ideasonboard.com> MIME-Version: 1.0 Subject: [libcamera-devel] [PATCH 2/7] utils: libtuning: modules: Add ALSC module X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-Patchwork-Original-From: Paul Elder via libcamera-devel From: Paul Elder Reply-To: Paul Elder Errors-To: libcamera-devel-bounces@lists.libcamera.org Sender: "libcamera-devel" Add an ALSC module to libtuning's collection of modules. It is based on raspberrypi's ctt's ALSC, but customizable for different lens shading table sizes, among other things. Signed-off-by: Paul Elder --- utils/tuning/libtuning/modules/__init__.py | 1 + utils/tuning/libtuning/modules/alsc.py | 321 +++++++++++++++++++++ 2 files changed, 322 insertions(+) create mode 100644 utils/tuning/libtuning/modules/alsc.py diff --git a/utils/tuning/libtuning/modules/__init__.py b/utils/tuning/libtuning/modules/__init__.py index e69de29b..f41ef550 100644 --- a/utils/tuning/libtuning/modules/__init__.py +++ b/utils/tuning/libtuning/modules/__init__.py @@ -0,0 +1 @@ +from libtuning.modules.alsc import ALSC diff --git a/utils/tuning/libtuning/modules/alsc.py b/utils/tuning/libtuning/modules/alsc.py new file mode 100644 index 00000000..788cf2a9 --- /dev/null +++ b/utils/tuning/libtuning/modules/alsc.py @@ -0,0 +1,321 @@ +from .module import Module + +import libtuning as lt +import libtuning.utils as utils + +import decimal +import numbers +import numpy as np +from typing import Callable + + +class ALSC(Module): + def __init__(self, *, + do_color: lt.param, + debug: list, + luminance_strength: lt.param, + sector_shape: tuple, + sector_x_gradient: lt.gradient.Gradient, + sector_y_gradient: lt.gradient.Gradient, + sector_x_remainder: lt.remainder = None, + sector_y_remainder: lt.remainder = None, + sector_average_function: Callable, + sector_average_function_parameters: list = [], + smoothing_function: Callable, + smoothing_parameters: list = [], + output_type: type, + output_color_channels: list, + output_range: tuple): + super().__init__() + self.hr_name = "ALSC" + self.name = "alsc" + self.options = {} + + if (sector_x_gradient is lt.gradient.Linear and sector_x_remainder is None): + raise ValueError('sector_x_remainder must be specified if sector_x_gradient is Linear') + + if (sector_y_gradient is lt.gradient.Linear and sector_y_remainder is None): + raise ValueError('sector_y_remainder must be specified if sector_y_gradient is Linear') + + self.do_color = do_color + self.luminance_strength = luminance_strength + + self.debug = debug + + self.sector_shape = sector_shape + self.sector_x_gradient = sector_x_gradient(sector_x_remainder) + self.sector_y_gradient = sector_y_gradient(sector_y_remainder) + self.sector_average_function = sector_average_function(sector_average_function_parameters) + + self.smoothing_function = smoothing_function(smoothing_parameters) + + # todo This isn't actually used. Can we assume/force float? Is there a + # use case for non-float? Or maybe we should use numpy types like + # float64? I don't think it makes a difference in the output tuning + # anyway though. + self.output_type = output_type + + # This isn't used for now. Once we add support for two green channels + # we can. + self.output_color_channels = output_color_channels + + self.output_range = output_range + + def enumerate_alsc_images(images): + for image in images: + if image.alsc_only: + yield image + + def __validateConfig__(self, config: dict) -> bool: + if not super().__validateConfig__(config): + return False + + conf = config[self] + valid = True + + lum_key = self.luminance_strength.name + color_key = self.do_color.name + + if lum_key not in conf and self.luminance_strength.isRequired(): + utils.eprint(f'{lum_key} is not in config') + valid = False + + if lum_key in conf and (conf[lum_key] < 0 or conf[lum_key] > 1): + utils.eprint(f'Warning: {lum_key} is not in range [0, 1]; defaulting to 0.5') + + if color_key not in conf and self.do_color.isRequired(): + utils.eprint(f'{color_key} is not in config') + valid = False + + return valid + + def _getGrid(self, channel, image): + # List of number of pixels in each sector + sectors_x = self.sector_x_gradient.distribute(image.w, self.sector_shape[0]) + sectors_y = self.sector_y_gradient.distribute(image.h, self.sector_shape[1]) + + grid = [] + + r = 0 + for y in sectors_y: + c = 0 + for x in sectors_x: + grid.append(self.sector_average_function(channel[r:r + y, c:c + x])) + c += x + r += y + + return np.array(grid) + + # @return Image color temperature, flattened array of red calibration table + # (containing {sector size} elements), flattened array of blue + # calibration table, flattened array of (red's) green calibration + # table, flattened array of (blue's) green calibration table + + def _doSingleAlsc(self, image: lt.Image, do_alsc_colour: bool): + # Get channel in correct order: [R, GR, GB, B] + channels = [image.channels[i] for i in image.order] + + channels.append(np.mean((channels[color.GR:color.GB + 1]), axis=0)) + + g = self._getGrid(av_ch_g, image) - image.blacklevel_16 + cg = np.reshape(1 / g, self.sector_shape[::-1]) + cg = self.smoothing_function.smoothing(cg, 3) + cg = cg / np.min(cg) + + if not do_alsc_colour: + return image.color, None, None, cg.flatten(), cg.flatten() + + r = self._getGrid(channels[0], image) - image.blacklevel_16 + cr = np.reshape(g / r, self.sector_shape[::-1]) + cr = self.smoothing_function.smoothing(cr, 3) + + b = self._getGrid(channels[3], image) - image.blacklevel_16 + cb = np.reshape(g / b, self.sector_shape[::-1]) + cb = self.smoothing_function.smoothing(cb, 3) + + # todo implement debug + + return image.color, cr.flatten(), cb.flatten(), cg.flatten(), cg.flatten() + + # todo Figure out if we can do something cooler if we can keep the greens + # separate. For now, just average them and use the average, but duplicate + # them so that it's easier to handle them in the future, especially since + # some platforms have separate tables for GR and GB. + # + # todo Should we return a third shading table for the average of greens? + # For platforms that only want one green, as opposed to just using red's + # green shading table + # + # @return Red shading table, Blue shading table, (red's) Green shading + # table, (blue's) Green shading table, number of images processed + + def _doAllAlsc(self, images: list, do_alsc_colour: bool, general_conf: dict) -> (list, list, list, numbers.Number, int): + # List of colour temperatures + list_col = [] + # Associated calibration tables + list_cr = [] + list_cb = [] + list_cg1 = [] + list_cg2 = [] + count = 0 + for image in enumerate_alsc_images(images): + col, cr, cb, cg1, cg2 = self._doSingleAlsc(image, do_alsc_colour) + list_col.append(col) + list_cr.append(cr) + list_cb.append(cb) + list_cg1.append(cg1) + list_cg2.append(cg2) + count += 1 + + # Convert to numpy array for data manipulation + list_col = np.array(list_col) + list_cr = np.array(list_cr) + list_cb = np.array(list_cb) + list_cg1 = np.array(list_cg1) + list_cg2 = np.array(list_cg2) + + cal_cr_list = [] + cal_cb_list = [] + + # Note: Calculation of average corners and center of the shading tables + # has been removed (which ctt had, as it was being unused) + + # todo Handle the second green table + + # Average all values for luminance shading and return one table for all temperatures + lum_lut = roundWithSigfigs(np.mean(list_cg1, axis=0), self.output_range) + + if not do_alsc_colour: + return None, None, lum_lut, lum_lut, count + + for ct in sorted(set(list_col)): + # Average tables for the same colour temperature + indices = np.where(list_col == ct) + ct = int(ct) + t_r = utils.roundWithSigfigs(np.mean(list_cr[indices], axis=0), + self.output_range) + t_b = utils.roundWithSigfigs(np.mean(list_cb[indices], axis=0), + self.output_range) + + cr_dict = { + 'ct': ct, + 'table': list(t_r) + } + cb_dict = { + 'ct': ct, + 'table': list(t_b) + } + cal_cr_list.append(cr_dict) + cal_cb_list.append(cb_dict) + + return cal_cr_list, cal_cb_list, lum_lut, lum_lut, count + + # @brief Calculate sigma from two adjacent gain tables + def _calcSigma(self, g1, g2): + g1 = np.reshape(g1, self.sector_shape[::-1]) + g2 = np.reshape(g2, self.sector_shape[::-1]) + + # Apply gains to gain table + gg = g1 / g2 + if np.mean(gg) < 1: + gg = 1 / gg + + # For each internal patch, compute average difference between it and + # its 4 neighbours, then append to list + diffs = [] + for i in range(self.sector_shape[1] - 2): + for j in range(self.sector_shape[0] - 2): + # Indexing is incremented by 1 since all patches on borders are + # not counted + diff = np.abs(gg[i + 1][j + 1] - gg[i][j + 1]) + diff += np.abs(gg[i + 1][j + 1] - gg[i + 2][j + 1]) + diff += np.abs(gg[i + 1][j + 1] - gg[i + 1][j]) + diff += np.abs(gg[i + 1][j + 1] - gg[i + 1][j + 2]) + diffs.append(diff / 4) + + mean_diff = np.mean(diffs) + return(np.round(mean_diff, 5)) + + # @brief Obtains sigmas for red and blue, effectively a measure of the + # 'error' + def _getSigma(self, cal_cr_list, cal_cb_list): + # Provided colour alsc tables were generated for two different colour + # temperatures sigma is calculated by comparing two calibration temperatures + # adjacent in colour space + + color_temps = [cal['ct'] for cal in cal_cr_list] + + # Calculate sigmas for each adjacent color_temps and return worst one + sigma_rs = [] + sigma_bs = [] + for i in range(len(color_temps) - 1): + sigma_rs.append(self._calcSigma(cal_cr_list[i]['table'], cal_cr_list[i + 1]['table'])) + sigma_bs.append(self._calcSigma(cal_cb_list[i]['table'], cal_cb_list[i + 1]['table'])) + + # Return maximum sigmas, not necessarily from the same colour + # temperature interval + sigma_r = max(sigma_rs) if sigma_rs else 0.005 + sigma_b = max(sigma_bs) if sigma_bs else 0.005 + + return sigma_r, sigma_b + + def __process__(self, args, config: dict, images: list, outputs: dict) -> dict: + output = { + 'omega': 1.3, + 'n_iter': 100, + 'luminance_strength': 0.7 + } + + conf = config[self] + general_conf = config['general'] + + do_alsc_colour = self.do_color.getValue(conf) + + # todo I have no idea where this input parameter is used + luminance_strength = self.luminance_strength.getValue(conf) + if luminance_strength < 0 or luminance_strength > 1: + luminance_strength = 0.5 + + output['luminance_strength'] = luminance_strength + + # todo Validate images from greyscale camera and force grescale mode + # todo Debug functionality + + alsc_out = self._doAllAlsc(images, do_alsc_colour, general_conf) + # todo Handle the second green lut + cal_cr_list, cal_cb_list, luminance_lut, _, count = alsc_out + + output['luminance_lut'] = luminance_lut + + if not do_alsc_colour: + output['n_iter'] = 0 + return output + + output['calibrations_Cr'] = cal_cr_list + output['calibrations_Cb'] = cal_cb_list + + # The sigmas determine the strength of the adaptive algorithm, that + # cleans up any lens shading that has slipped through the alsc. These + # are determined by measuring a 'worst-case' difference between two + # alsc tables that are adjacent in colour space. If, however, only one + # colour temperature has been provided, then this difference can not be + # computed as only one table is available. + # To determine the sigmas you would have to estimate the error of an + # alsc table with only the image it was taken on as a check. To avoid + # circularity, dfault exaggerated sigmas are used, which can result in + # too much alsc and is therefore not advised. + # In general, just take another alsc picture at another colour + # temperature! + + if count == 1: + output['sigma'] = 0.005 + output['sigma_Cb'] = 0.005 + utils.eprint('Warning: Only one alsc calibration found; standard sigmas used for adaptive algorithm.') + return output + + # Obtain worst-case scenario residual sigmas + sigma_r, sigma_b = self._getSigma(self, cal_cr_list, cal_cb_list) + output['sigma'] = np.round(sigma_r, 5) + output['sigma_Cb'] = np.round(sigma_b, 5) + + return output