From patchwork Sat Oct 22 06:23:02 2022 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Paul Elder X-Patchwork-Id: 17654 Return-Path: X-Original-To: parsemail@patchwork.libcamera.org Delivered-To: parsemail@patchwork.libcamera.org Received: from lancelot.ideasonboard.com (lancelot.ideasonboard.com [92.243.16.209]) by patchwork.libcamera.org (Postfix) with ESMTPS id 3D2F4C327C for ; Sat, 22 Oct 2022 06:23:30 +0000 (UTC) Received: from lancelot.ideasonboard.com (localhost [IPv6:::1]) by lancelot.ideasonboard.com (Postfix) with ESMTP id A5D4F62ED1; Sat, 22 Oct 2022 08:23:29 +0200 (CEST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=libcamera.org; s=mail; t=1666419809; bh=QwIIKLKs60P8RlVm7Uxw8cwAWj1D3bY4a7SjBVzL3yQ=; h=To:Date:In-Reply-To:References:Subject:List-Id:List-Unsubscribe: List-Archive:List-Post:List-Help:List-Subscribe:From:Reply-To: From; b=Xa/QJBXl4sJdSKP87xcYyJOQ+D7kBcBFzCKJgknfAdLM0kZlBrIVb+CulPZ8RfFiD Joj7eEalGp6joNmQXIbsNuKnkdAMslh5wloYMl2SOkdpLelxMC7rrHzYzkvoTgYGGv SchE+KKQ4+eB8LLgECBs+wJ77npZz4b1x0HEBmZo+Af7stfdCOqNvNwcPVZeVfXrBA bBwRGcepQMYKgfghvXHqL4/wtDXqj7qBVjS+wds3IQI6cGSDZiikxnYln43xtcUGNX RHMcR8YC20KX+6Z+3jL+T20qHQ2jG4k07sGl3YZrmvByAyqCU0qWK0nBl9ib7vtgpk FBvOhArn7VikA== Received: from perceval.ideasonboard.com (perceval.ideasonboard.com [213.167.242.64]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id 6532B62EC5 for ; Sat, 22 Oct 2022 08:23:26 +0200 (CEST) Authentication-Results: lancelot.ideasonboard.com; dkim=pass (1024-bit key; unprotected) header.d=ideasonboard.com header.i=@ideasonboard.com header.b="k10sPJsa"; dkim-atps=neutral Received: from pyrite.rasen.tech (h175-177-042-159.catv02.itscom.jp [175.177.42.159]) by perceval.ideasonboard.com (Postfix) with ESMTPSA id 286FE891; Sat, 22 Oct 2022 08:23:24 +0200 (CEST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=ideasonboard.com; s=mail; t=1666419806; bh=QwIIKLKs60P8RlVm7Uxw8cwAWj1D3bY4a7SjBVzL3yQ=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=k10sPJsaQgZUMH1gj/Pg9jQ0NlbxQdgV58wk39K0024fhpV9Vm8cFyuGDbbI5mMiK 8YAGbPFkG9GjEHhsX4lNVIFy4QyF/iVVOwJvkXqr/EnIqCfhJpVOtNmoXUl55YREuO X/qPiWM5QORJw+95nSYrmX/fh3+pAuApCFQCjaXY= To: libcamera-devel@lists.libcamera.org Date: Sat, 22 Oct 2022 15:23:02 +0900 Message-Id: <20221022062310.2545463-4-paul.elder@ideasonboard.com> X-Mailer: git-send-email 2.30.2 In-Reply-To: <20221022062310.2545463-1-paul.elder@ideasonboard.com> References: <20221022062310.2545463-1-paul.elder@ideasonboard.com> MIME-Version: 1.0 Subject: [libcamera-devel] [PATCH v2 03/11] utils: libtuning: modules: alsc: Add raspberrypi ALSC module X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-Patchwork-Original-From: Paul Elder via libcamera-devel From: Paul Elder Reply-To: Paul Elder Errors-To: libcamera-devel-bounces@lists.libcamera.org Sender: "libcamera-devel" Add an ALSC module for Raspberry Pi. Signed-off-by: Paul Elder Reviewed-by: Laurent Pinchart --- New in v2 --- .../tuning/libtuning/modules/alsc/__init__.py | 1 + .../libtuning/modules/alsc/raspberrypi.py | 246 ++++++++++++++++++ 2 files changed, 247 insertions(+) create mode 100644 utils/tuning/libtuning/modules/alsc/raspberrypi.py diff --git a/utils/tuning/libtuning/modules/alsc/__init__.py b/utils/tuning/libtuning/modules/alsc/__init__.py index a8f28923..ef6300c2 100644 --- a/utils/tuning/libtuning/modules/alsc/__init__.py +++ b/utils/tuning/libtuning/modules/alsc/__init__.py @@ -3,3 +3,4 @@ # Copyright (C) 2022, Paul Elder from libtuning.modules.alsc.alsc import ALSC +from libtuning.modules.alsc.raspberrypi import ALSCRaspberryPi diff --git a/utils/tuning/libtuning/modules/alsc/raspberrypi.py b/utils/tuning/libtuning/modules/alsc/raspberrypi.py new file mode 100644 index 00000000..bea1a9a9 --- /dev/null +++ b/utils/tuning/libtuning/modules/alsc/raspberrypi.py @@ -0,0 +1,246 @@ +# SPDX-License-Identifier: BSD-2-Clause +# +# Copyright (C) 2019, Raspberry Pi Ltd +# Copyright (C) 2022, Paul Elder + +from .alsc import ALSC + +import libtuning as lt +import libtuning.utils as utils + +from numbers import Number +import numpy as np + + +class ALSCRaspberryPi(ALSC): + hr_name = 'ALSC (Raspberry Pi)' + out_name = 'rpi.alsc' + # todo Do something with the compatible list + compatible = ['raspberrypi'] + + def __init__(self, *, + do_color: lt.Param, + luminance_strength: lt.Param, + **kwargs): + super().__init__(**kwargs) + + self.do_color = do_color + self.luminance_strength = luminance_strength + + self.output_range = (0, 3.999) + + def _validate_config(self, config: dict) -> bool: + if self not in config: + utils.eprint(f'{self.type} not in config') + return False + + valid = True + + conf = config[self] + + lum_key = self.luminance_strength.name + color_key = self.do_color.name + + if lum_key not in conf and self.luminance_strength.is_required(): + utils.eprint(f'{lum_key} is not in config') + valid = False + + if lum_key in conf and (conf[lum_key] < 0 or conf[lum_key] > 1): + utils.eprint(f'Warning: {lum_key} is not in range [0, 1]; defaulting to 0.5') + + if color_key not in conf and self.do_color.is_required(): + utils.eprint(f'{color_key} is not in config') + valid = False + + return valid + + # @return Image color temperature, flattened array of red calibration table + # (containing {sector size} elements), flattened array of blue + # calibration table, flattened array of green calibration + # table + + def _do_single_alsc(self, image: lt.Image, do_alsc_colour: bool): + average_green = np.mean((image.channels[lt.Color.GR:lt.Color.GB + 1]), axis=0) + + cg, g = self._lsc_single_channel(average_green, image) + + if not do_alsc_colour: + return image.color, None, None, cg.flatten() + + cr, _ = self._lsc_single_channel(image.channels[lt.Color.R], image, g) + cb, _ = self._lsc_single_channel(image.channels[lt.Color.B], image, g) + + # todo implement debug + + return image.color, cr.flatten(), cb.flatten(), cg.flatten() + + # @return Red shading table, Blue shading table, Green shading table, + # number of images processed + + def _do_all_alsc(self, images: list, do_alsc_colour: bool, general_conf: dict) -> (list, list, list, Number, int): + # List of colour temperatures + list_col = [] + # Associated calibration tables + list_cr = [] + list_cb = [] + list_cg = [] + count = 0 + for image in self._enumerate_alsc_images(images): + col, cr, cb, cg = self._do_single_alsc(image, do_alsc_colour) + list_col.append(col) + list_cr.append(cr) + list_cb.append(cb) + list_cg.append(cg) + count += 1 + + # Convert to numpy array for data manipulation + list_col = np.array(list_col) + list_cr = np.array(list_cr) + list_cb = np.array(list_cb) + list_cg = np.array(list_cg) + + cal_cr_list = [] + cal_cb_list = [] + + # Note: Calculation of average corners and center of the shading tables + # has been removed (which ctt had, as it was being unused) + + # todo Handle the second green table + + # Average all values for luminance shading and return one table for all temperatures + lum_lut = list(utils.round_with_sigfigs(np.mean(list_cg, axis=0), self.output_range)) + + if not do_alsc_colour: + return None, None, lum_lut, count + + for ct in sorted(set(list_col)): + # Average tables for the same colour temperature + indices = np.where(list_col == ct) + ct = int(ct) + t_r = utils.round_with_sigfigs(np.mean(list_cr[indices], axis=0), + self.output_range) + t_b = utils.round_with_sigfigs(np.mean(list_cb[indices], axis=0), + self.output_range) + + cr_dict = { + 'ct': ct, + 'table': list(t_r) + } + cb_dict = { + 'ct': ct, + 'table': list(t_b) + } + cal_cr_list.append(cr_dict) + cal_cb_list.append(cb_dict) + + return cal_cr_list, cal_cb_list, lum_lut, count + + # @brief Calculate sigma from two adjacent gain tables + def _calcSigma(self, g1, g2): + g1 = np.reshape(g1, self.sector_shape[::-1]) + g2 = np.reshape(g2, self.sector_shape[::-1]) + + # Apply gains to gain table + gg = g1 / g2 + if np.mean(gg) < 1: + gg = 1 / gg + + # For each internal patch, compute average difference between it and + # its 4 neighbours, then append to list + diffs = [] + for i in range(self.sector_shape[1] - 2): + for j in range(self.sector_shape[0] - 2): + # Indexing is incremented by 1 since all patches on borders are + # not counted + diff = np.abs(gg[i + 1][j + 1] - gg[i][j + 1]) + diff += np.abs(gg[i + 1][j + 1] - gg[i + 2][j + 1]) + diff += np.abs(gg[i + 1][j + 1] - gg[i + 1][j]) + diff += np.abs(gg[i + 1][j + 1] - gg[i + 1][j + 2]) + diffs.append(diff / 4) + + mean_diff = np.mean(diffs) + return(np.round(mean_diff, 5)) + + # @brief Obtains sigmas for red and blue, effectively a measure of the + # 'error' + def _get_sigma(self, cal_cr_list, cal_cb_list): + # Provided colour alsc tables were generated for two different colour + # temperatures sigma is calculated by comparing two calibration temperatures + # adjacent in colour space + + color_temps = [cal['ct'] for cal in cal_cr_list] + + # Calculate sigmas for each adjacent color_temps and return worst one + sigma_rs = [] + sigma_bs = [] + for i in range(len(color_temps) - 1): + sigma_rs.append(self._calcSigma(cal_cr_list[i]['table'], cal_cr_list[i + 1]['table'])) + sigma_bs.append(self._calcSigma(cal_cb_list[i]['table'], cal_cb_list[i + 1]['table'])) + + # Return maximum sigmas, not necessarily from the same colour + # temperature interval + sigma_r = max(sigma_rs) if sigma_rs else 0.005 + sigma_b = max(sigma_bs) if sigma_bs else 0.005 + + return sigma_r, sigma_b + + def _process(self, args, config: dict, images: list, outputs: dict) -> dict: + output = { + 'omega': 1.3, + 'n_iter': 100, + 'luminance_strength': 0.7 + } + + conf = config[self] + general_conf = config['general'] + + do_alsc_colour = self.do_color.get_value(conf) + + # todo I have no idea where this input parameter is used + luminance_strength = self.luminance_strength.get_value(conf) + if luminance_strength < 0 or luminance_strength > 1: + luminance_strength = 0.5 + + output['luminance_strength'] = luminance_strength + + # todo Validate images from greyscale camera and force grescale mode + # todo Debug functionality + + alsc_out = self._do_all_alsc(images, do_alsc_colour, general_conf) + # todo Handle the second green lut + cal_cr_list, cal_cb_list, luminance_lut, count = alsc_out + + if not do_alsc_colour: + output['luminance_lut'] = luminance_lut + output['n_iter'] = 0 + return output + + output['calibrations_Cr'] = cal_cr_list + output['calibrations_Cb'] = cal_cb_list + output['luminance_lut'] = luminance_lut + + # The sigmas determine the strength of the adaptive algorithm, that + # cleans up any lens shading that has slipped through the alsc. These + # are determined by measuring a 'worst-case' difference between two + # alsc tables that are adjacent in colour space. If, however, only one + # colour temperature has been provided, then this difference can not be + # computed as only one table is available. + # To determine the sigmas you would have to estimate the error of an + # alsc table with only the image it was taken on as a check. To avoid + # circularity, dfault exaggerated sigmas are used, which can result in + # too much alsc and is therefore not advised. + # In general, just take another alsc picture at another colour + # temperature! + + if count == 1: + output['sigma'] = 0.005 + output['sigma_Cb'] = 0.005 + utils.eprint('Warning: Only one alsc calibration found; standard sigmas used for adaptive algorithm.') + return output + + # Obtain worst-case scenario residual sigmas + sigma_r, sigma_b = self._get_sigma(cal_cr_list, cal_cb_list) + output['sigma'] = np.round(sigma_r, 5) + output['sigma_Cb'] = np.round(sigma_b, 5) + + return output