From patchwork Wed May 29 13:15:57 2024 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Stefan Klug X-Patchwork-Id: 20117 Return-Path: X-Original-To: parsemail@patchwork.libcamera.org Delivered-To: parsemail@patchwork.libcamera.org Received: from lancelot.ideasonboard.com (lancelot.ideasonboard.com [92.243.16.209]) by patchwork.libcamera.org (Postfix) with ESMTPS id D986CBD87C for ; Wed, 29 May 2024 13:16:06 +0000 (UTC) Received: from lancelot.ideasonboard.com (localhost [IPv6:::1]) by lancelot.ideasonboard.com (Postfix) with ESMTP id C8FE9634B6; Wed, 29 May 2024 15:16:05 +0200 (CEST) Authentication-Results: lancelot.ideasonboard.com; dkim=pass (1024-bit key; unprotected) header.d=ideasonboard.com header.i=@ideasonboard.com header.b="uC2ORedJ"; dkim-atps=neutral Received: from perceval.ideasonboard.com (perceval.ideasonboard.com [IPv6:2001:4b98:dc2:55:216:3eff:fef7:d647]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id E27036347E for ; Wed, 29 May 2024 15:16:03 +0200 (CEST) Received: from ideasonboard.com (unknown [IPv6:2a00:6020:448c:6c00:e209:3057:8cae:acff]) by perceval.ideasonboard.com (Postfix) with ESMTPSA id 173A34D1; Wed, 29 May 2024 15:16:00 +0200 (CEST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=ideasonboard.com; s=mail; t=1716988560; bh=ZZaLnDfMkEGTWwAW9n/x+MJQnyiYyFD3alomF8LeR/w=; h=From:To:Cc:Subject:Date:From; b=uC2ORedJntaewte6wdIjZnEvDIsBK63+wjb/OfX1wrVb7/x3aI1YCQKFh+e8TS4l5 VHT93krVKnEf1tlO6taOc6NLdVjPg5KnlnwyNrogzeNbn6aE9wlrxgWwh1UWPCKsJD fybS/2a58vqi8eKs0YZaK2BVQFjsGDUv6roX+mwY= From: Stefan Klug To: libcamera-devel@lists.libcamera.org Cc: Stefan Klug Subject: [PATCH v1] utils: Add script and schema to validate tuning files Date: Wed, 29 May 2024 15:15:57 +0200 Message-ID: <20240529131559.95179-1-stefan.klug@ideasonboard.com> X-Mailer: git-send-email 2.43.0 MIME-Version: 1.0 X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: libcamera-devel-bounces@lists.libcamera.org Sender: "libcamera-devel" Add a schema file to validate the rkisp1 tuning files. This allows us to easily check if tuning files got outdated and can be used as implicit documentation for the file format. For now only rkisp1 is supported. To validate all tuning files, run utils/validate-tuning-files.py --all Signed-off-by: Stefan Klug --- src/ipa/rkisp1/data/tuning-rkisp1.schema.yaml | 376 ++++++++++++++++++ utils/validate-tuning-files.py | 112 ++++++ 2 files changed, 488 insertions(+) create mode 100644 src/ipa/rkisp1/data/tuning-rkisp1.schema.yaml create mode 100755 utils/validate-tuning-files.py diff --git a/src/ipa/rkisp1/data/tuning-rkisp1.schema.yaml b/src/ipa/rkisp1/data/tuning-rkisp1.schema.yaml new file mode 100644 index 00000000..4f72f53e --- /dev/null +++ b/src/ipa/rkisp1/data/tuning-rkisp1.schema.yaml @@ -0,0 +1,376 @@ +# SPDX-License-Identifier: CC0-1.0 +%YAML 1.1 +--- +$id: https://libcamera.org/tuning-rkisp1.schema.yaml +$schema: https://json-schema.org/draft/2020-12/schema +description: Tuning file schema for the RKISP1 IPA +type: object +required: ['version', 'algorithms'] +properties: + version: + const: 1 + algorithms: + type: array + items: + oneOf: + - $ref: '#/$defs/Agc' + - $ref: '#/$defs/Awb' + - $ref: '#/$defs/BlackLevelCorrection' + - $ref: '#/$defs/Ccm' + - $ref: '#/$defs/ColorProcessing' + - $ref: '#/$defs/Dpf' + - $ref: '#/$defs/Filter' + - $ref: '#/$defs/GammaSensorLinearization' + - $ref: '#/$defs/DefectPixelClusterCorrection' + - $ref: '#/$defs/LensShadingCorrection' +$defs: + Agc: + type: object + additionalProperties: false + required: ['Agc'] + properties: + Agc: + type: ['object', 'null'] + additionalProperties: false + required: ['AeMeteringMode'] + properties: + AeMeteringMode: + type: object + additionalProperties: false + required: ['MeteringCentreWeighted', 'MeteringSpot', 'MeteringMatrix'] + properties: + MeteringCentreWeighted: { $ref: '#/$defs/_AgcMeteringMode' } + MeteringSpot: { $ref: '#/$defs/_AgcMeteringMode' } + MeteringMatrix: { $ref: '#/$defs/_AgcMeteringMode' } + MeteringCustom: { $ref: '#/$defs/_AgcMeteringMode' } + AeExposureMode: + type: object + additionalProperties: false + properties: + ExposureNormal: { $ref: '#/$defs/_AgcExposureMode' } + ExposureShort: { $ref: '#/$defs/_AgcExposureMode' } + ExposureLong: { $ref: '#/$defs/_AgcExposureMode' } + ExposureCustom: { $ref: '#/$defs/_AgcExposureMode' } + AeConstraintMode: + type: object + additionalProperties: false + properties: + ConstraintNormal: { $ref: '#/$defs/_AgcConstraintMode' } + ConstraintHighlight: { $ref: '#/$defs/_AgcConstraintMode' } + ConstraintShadows: { $ref: '#/$defs/_AgcConstraintMode' } + ConstraintCustom: { $ref: '#/$defs/_AgcConstraintMode' } + relativeLuminanceTarget: + type: array + items: + type: number + + # Ideally these should be defined inside Agc and referenced using relative paths + # but apparently this is not supported by the spec. + # See https://github.com/python-jsonschema/jsonschema/issues/1265 + _AgcMeteringMode: + type: array + items: + type: integer + minItems: 25 + maxItems: 25 + _AgcConstraintMode: + type: object + additionalProperties: false + properties: + lower: + type: object + additionalProperties: false + properties: + qLo: + type: number + qHi: + type: number + yTarget: + type: array + items: + type: number + upper: + type: object + additionalProperties: false + properties: + qLo: + type: number + qHi: + type: number + yTarget: + type: array + items: + type: number + _AgcExposureMode: + type: object + additionalProperties: false + required: ['shutter', 'gain'] + properties: + shutter: + type: array + items: + type: integer + gain: + type: array + items: + type: number + + Awb: + type: object + additionalProperties: false + required: ['Awb'] + properties: + Awb: { $ref: '#/$defs/_Empty' } + BlackLevelCorrection: + type: object + additionalProperties: false + required: ['BlackLevelCorrection'] + properties: + BlackLevelCorrection: + type: object + additionalProperties: false + required: ['B', 'Gb', 'Gr', 'R'] + properties: + B: + type: integer + Gb: + type: integer + Gr: + type: integer + R: + type: integer + Ccm: + type: object + additionalProperties: false + required: ['Ccm'] + properties: + Ccm: + type: object + additionalProperties: false + required: ['ccms'] + properties: + ccms: + type: array + items: + type: object + additionalProperties: false + required: ['ct', 'ccm'] + properties: + ct: + type: integer + ccm: + type: array + items: + type: number + minItems: 9 + maxItems: 9 + offsets: + type: array + items: + type: integer + minItems: 3 + maxItems: 3 + ColorProcessing: + type: object + additionalProperties: false + required: ['ColorProcessing'] + properties: + ColorProcessing: { $ref: '#/$defs/_Empty' } + DefectPixelClusterCorrection: + type: object + additionalProperties: false + required: ['DefectPixelClusterCorrection'] + properties: + DefectPixelClusterCorrection: + type: object + additionalProperties: false + required: ['sets'] + properties: + fixed-set: + type: boolean + sets: + type: array + maxItems: 3 + items: + type: object + additionalProperties: false + # Todo clarify what is really required here ov5640 misses + # 'rg-factor', 'rnd-offsets', 'rnd-threshold' + required: ['line-mad-factor', 'line-threshold', 'pg-factor', 'ro-limits'] + properties: + line-mad-factor: { $ref: '#/$defs/_DefectPixelClusterCorrectionFactor' } + line-threshold: { $ref: '#/$defs/_DefectPixelClusterCorrectionFactor' } + pg-factor: { $ref: '#/$defs/_DefectPixelClusterCorrectionFactor' } + rg-factor: { $ref: '#/$defs/_DefectPixelClusterCorrectionFactor' } + rnd-offsets: { $ref: '#/$defs/_DefectPixelClusterCorrectionFactor' } + rnd-threshold: { $ref: '#/$defs/_DefectPixelClusterCorrectionFactor' } + ro-limits: { $ref: '#/$defs/_DefectPixelClusterCorrectionFactor' } + _DefectPixelClusterCorrectionFactor: + type: object + additionalProperties: false + properties: + green: + type: integer + red-blue: + type: integer + Dpf: + type: object + additionalProperties: false + required: ['Dpf'] + properties: + Dpf: + type: object + additionalProperties: false + required: ['DomainFilter', 'FilterStrength', 'NoiseLevelFunction'] + properties: + DomainFilter: + type: object + additionalProperties: false + required: ['g', 'rb'] + properties: + g: + type: array + items: + type: integer + minItems: 6 + maxItems: 6 + rb: + type: array + items: + type: integer + minItems: 6 + maxItems: 6 + FilterStrength: + type: object + additionalProperties: false + required: ['b', 'g', 'r'] + properties: + b: + type: integer + g: + type: integer + r: + type: integer + NoiseLevelFunction: + type: object + additionalProperties: false + required: ['coeff', 'scale-mode'] + properties: + coeff: + type: array + items: + type: integer + minItems: 17 + maxItems: 17 + scale-mode: + type: string + enum: ['linear', 'logarithmic'] + Filter: + type: object + additionalProperties: false + required: ['Filter'] + properties: + Filter: { $ref: '#/$defs/_Empty' } + GammaSensorLinearization: + type: object + additionalProperties: false + required: ['GammaSensorLinearization'] + properties: + GammaSensorLinearization: + type: object + additionalProperties: false + required: ['x-intervals', 'y'] + properties: + x-intervals: + type: array + items: + type: number + minItems: 16 + maxItems: 16 + y: + type: object + additionalProperties: false + required: ['red', 'green', 'blue'] + properties: + red: + type: array + items: + type: integer + minItems: 17 + maxItems: 17 + green: + type: array + items: + type: integer + minItems: 17 + maxItems: 17 + blue: + type: array + items: + type: integer + minItems: 17 + maxItems: 17 + LensShadingCorrection: + type: object + additionalProperties: false + required: ['LensShadingCorrection'] + properties: + LensShadingCorrection: + type: object + additionalProperties: false + required: [x-size, y-size, sets] + properties: + x-size: + type: array + items: + type: number + minItems: 8 + maxItems: 8 + y-size: + type: array + items: + type: number + minItems: 8 + maxItems: 8 + sets: + type: array + items: + type: object + additionalProperties: false + required: ['ct', 'r', 'gr', 'gb', 'b'] + properties: + ct: + type: integer + r: + type: array + items: + type: integer + minItems: 289 + maxItems: 289 + gr: + type: array + items: + type: integer + minItems: 289 + maxItems: 289 + gb: + type: array + items: + type: integer + minItems: 289 + maxItems: 289 + b: + type: array + items: + type: integer + minItems: 289 + maxItems: 289 + # ToDo: We have some tuning files with that property. Is this needed? + resolution: + type: string + # special definition to denote algorithms that do not have any parameters + _Empty: + type: [ 'object', 'null' ] + additionalProperties: false + properties: {} +... diff --git a/utils/validate-tuning-files.py b/utils/validate-tuning-files.py new file mode 100755 index 00000000..45f1d863 --- /dev/null +++ b/utils/validate-tuning-files.py @@ -0,0 +1,112 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-or-later +# Copyright (C) 2024, Ideas on Board Oy +# +# Author: Stefan Klug +# +# Validate tuning files against schema + +import argparse +import logging +import jsonschema +import yaml +import sys +import json +import os +from pathlib import Path + +try: + import coloredlogs + coloredlogs.install(level=logging.INFO, fmt='%(levelname)s: %(message)s') +except ImportError: + logging.basicConfig(level=logging.INFO, + format="%(levelname)s: %(message)s") + +logger = logging.getLogger() + + +def do_schema_check(schema_file, files): + res = True + with open(schema_file, 'r') as file: + schema = yaml.safe_load(file) + + for file in files: + with open(file, 'r') as f: + logger.info(f"Validating file {file} against {schema_file}") + data = yaml.safe_load(f) + try: + jsonschema.validate(instance=data, schema=schema) + except jsonschema.exceptions.ValidationError as e: + logging.error(f"Validation error in file {file}: {e}") + res = False + return res + +# Checks for the given ipa. If files is None, all files for that ipa get checked + + +def do_schema_check_ipa(ipa, files=None): + res = True + + if ipa != 'rkisp1': + raise ValueError(f"Ipa '{ipa}' is not supported") + + logger.info(f"Checking ipa {ipa}") + + top_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + # Todo implement for other pipelines + data_dir = 'src/ipa/rkisp1/data' + + full_dir = Path(os.path.join(top_dir, data_dir)) + + schema_path = os.path.join(full_dir, f'tuning-{ipa}.schema.yaml') + if not os.path.isfile(schema_path): + raise ValueError(f"Schema file {schema_path} doesn't exist") + + if files is None: + files = [f for f in full_dir.glob( + '*.yaml') if f.is_file() and not f.name.endswith('.schema.yaml')] + + if not do_schema_check(schema_path, files): + res = False + + return res + + +def do_schema_check_all(): + # We only support rkisp1 for now + return do_schema_check_ipa('rkisp1') + + +def main(): + + parser = argparse.ArgumentParser() + group = parser.add_mutually_exclusive_group(required=True) + group.add_argument('--all', '-a', action='store_true', + help='automatically find and check all tuning files.') + group.add_argument('--schema', '-s', + help='schema file to check against') + group.add_argument('--pipeline', '-p', + help='pipeline to check against (Currently only rkisp1 is supported)') + parser.add_argument('files', metavar='FILE', nargs='*', + help='tuning files to check') + + args = parser.parse_args() + + if args.all: + if args.files != []: + parser.error("No files should be given when using --all") + + if not do_schema_check_all(): + return 1 + return 0 + + if not args.files: + argparse.error("No files to check") + return 1 + + if not do_schema_check(args.schema, args.files): + return 1 + + +if __name__ == '__main__': + sys.exit(main())