{"id":20117,"url":"https://patchwork.libcamera.org/api/1.1/patches/20117/?format=json","web_url":"https://patchwork.libcamera.org/patch/20117/","project":{"id":1,"url":"https://patchwork.libcamera.org/api/1.1/projects/1/?format=json","name":"libcamera","link_name":"libcamera","list_id":"libcamera_core","list_email":"libcamera-devel@lists.libcamera.org","web_url":"","scm_url":"","webscm_url":""},"msgid":"<20240529131559.95179-1-stefan.klug@ideasonboard.com>","date":"2024-05-29T13:15:57","name":"[v1] utils: Add script and schema to validate tuning files","commit_ref":null,"pull_url":null,"state":"new","archived":false,"hash":"b7c873106a64a3180251f7efe24ccc815e1aa62d","submitter":{"id":184,"url":"https://patchwork.libcamera.org/api/1.1/people/184/?format=json","name":"Stefan Klug","email":"stefan.klug@ideasonboard.com"},"delegate":null,"mbox":"https://patchwork.libcamera.org/patch/20117/mbox/","series":[{"id":4332,"url":"https://patchwork.libcamera.org/api/1.1/series/4332/?format=json","web_url":"https://patchwork.libcamera.org/project/libcamera/list/?series=4332","date":"2024-05-29T13:15:57","name":"[v1] utils: Add script and schema to validate tuning files","version":1,"mbox":"https://patchwork.libcamera.org/series/4332/mbox/"}],"comments":"https://patchwork.libcamera.org/api/patches/20117/comments/","check":"pending","checks":"https://patchwork.libcamera.org/api/patches/20117/checks/","tags":{},"headers":{"Return-Path":"","X-Original-To":"parsemail@patchwork.libcamera.org","Delivered-To":"parsemail@patchwork.libcamera.org","Received":["from lancelot.ideasonboard.com (lancelot.ideasonboard.com\n\t[92.243.16.209])\n\tby patchwork.libcamera.org (Postfix) with ESMTPS id D986CBD87C\n\tfor ;\n\tWed, 29 May 2024 13:16:06 +0000 (UTC)","from lancelot.ideasonboard.com (localhost [IPv6:::1])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTP id C8FE9634B6;\n\tWed, 29 May 2024 15:16:05 +0200 (CEST)","from perceval.ideasonboard.com (perceval.ideasonboard.com\n\t[IPv6:2001:4b98:dc2:55:216:3eff:fef7:d647])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id E27036347E\n\tfor ;\n\tWed, 29 May 2024 15:16:03 +0200 (CEST)","from ideasonboard.com (unknown\n\t[IPv6:2a00:6020:448c:6c00:e209:3057:8cae:acff])\n\tby perceval.ideasonboard.com (Postfix) with ESMTPSA id 173A34D1;\n\tWed, 29 May 2024 15:16:00 +0200 (CEST)"],"Authentication-Results":"lancelot.ideasonboard.com; dkim=pass (1024-bit key;\n\tunprotected) header.d=ideasonboard.com header.i=@ideasonboard.com\n\theader.b=\"uC2ORedJ\"; dkim-atps=neutral","DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/simple; d=ideasonboard.com;\n\ts=mail; t=1716988560;\n\tbh=ZZaLnDfMkEGTWwAW9n/x+MJQnyiYyFD3alomF8LeR/w=;\n\th=From:To:Cc:Subject:Date:From;\n\tb=uC2ORedJntaewte6wdIjZnEvDIsBK63+wjb/OfX1wrVb7/x3aI1YCQKFh+e8TS4l5\n\tVHT93krVKnEf1tlO6taOc6NLdVjPg5KnlnwyNrogzeNbn6aE9wlrxgWwh1UWPCKsJD\n\tfybS/2a58vqi8eKs0YZaK2BVQFjsGDUv6roX+mwY=","From":"Stefan Klug ","To":"libcamera-devel@lists.libcamera.org","Cc":"Stefan Klug ","Subject":"[PATCH v1] utils: Add script and schema to validate tuning files","Date":"Wed, 29 May 2024 15:15:57 +0200","Message-ID":"<20240529131559.95179-1-stefan.klug@ideasonboard.com>","X-Mailer":"git-send-email 2.43.0","MIME-Version":"1.0","Content-Transfer-Encoding":"8bit","X-BeenThere":"libcamera-devel@lists.libcamera.org","X-Mailman-Version":"2.1.29","Precedence":"list","List-Id":"","List-Unsubscribe":",\n\t","List-Archive":"","List-Post":"","List-Help":"","List-Subscribe":",\n\t","Errors-To":"libcamera-devel-bounces@lists.libcamera.org","Sender":"\"libcamera-devel\" "},"content":"Add a schema file to validate the rkisp1 tuning files. This allows us to\neasily check if tuning files got outdated and can be used as implicit\ndocumentation for the file format. For now only rkisp1 is supported.\n\nTo validate all tuning files, run\nutils/validate-tuning-files.py --all\n\nSigned-off-by: Stefan Klug \n---\n src/ipa/rkisp1/data/tuning-rkisp1.schema.yaml | 376 ++++++++++++++++++\n utils/validate-tuning-files.py | 112 ++++++\n 2 files changed, 488 insertions(+)\n create mode 100644 src/ipa/rkisp1/data/tuning-rkisp1.schema.yaml\n create mode 100755 utils/validate-tuning-files.py","diff":"diff --git a/src/ipa/rkisp1/data/tuning-rkisp1.schema.yaml b/src/ipa/rkisp1/data/tuning-rkisp1.schema.yaml\nnew file mode 100644\nindex 00000000..4f72f53e\n--- /dev/null\n+++ b/src/ipa/rkisp1/data/tuning-rkisp1.schema.yaml\n@@ -0,0 +1,376 @@\n+# SPDX-License-Identifier: CC0-1.0\n+%YAML 1.1\n+---\n+$id: https://libcamera.org/tuning-rkisp1.schema.yaml\n+$schema: https://json-schema.org/draft/2020-12/schema\n+description: Tuning file schema for the RKISP1 IPA\n+type: object\n+required: ['version', 'algorithms']\n+properties:\n+ version:\n+ const: 1\n+ algorithms:\n+ type: array\n+ items:\n+ oneOf:\n+ - $ref: '#/$defs/Agc'\n+ - $ref: '#/$defs/Awb'\n+ - $ref: '#/$defs/BlackLevelCorrection'\n+ - $ref: '#/$defs/Ccm'\n+ - $ref: '#/$defs/ColorProcessing'\n+ - $ref: '#/$defs/Dpf'\n+ - $ref: '#/$defs/Filter'\n+ - $ref: '#/$defs/GammaSensorLinearization'\n+ - $ref: '#/$defs/DefectPixelClusterCorrection'\n+ - $ref: '#/$defs/LensShadingCorrection'\n+$defs:\n+ Agc:\n+ type: object\n+ additionalProperties: false\n+ required: ['Agc']\n+ properties:\n+ Agc:\n+ type: ['object', 'null']\n+ additionalProperties: false\n+ required: ['AeMeteringMode']\n+ properties:\n+ AeMeteringMode:\n+ type: object\n+ additionalProperties: false\n+ required: ['MeteringCentreWeighted', 'MeteringSpot', 'MeteringMatrix']\n+ properties:\n+ MeteringCentreWeighted: { $ref: '#/$defs/_AgcMeteringMode' }\n+ MeteringSpot: { $ref: '#/$defs/_AgcMeteringMode' }\n+ MeteringMatrix: { $ref: '#/$defs/_AgcMeteringMode' }\n+ MeteringCustom: { $ref: '#/$defs/_AgcMeteringMode' }\n+ AeExposureMode:\n+ type: object\n+ additionalProperties: false\n+ properties:\n+ ExposureNormal: { $ref: '#/$defs/_AgcExposureMode' }\n+ ExposureShort: { $ref: '#/$defs/_AgcExposureMode' }\n+ ExposureLong: { $ref: '#/$defs/_AgcExposureMode' }\n+ ExposureCustom: { $ref: '#/$defs/_AgcExposureMode' }\n+ AeConstraintMode:\n+ type: object\n+ additionalProperties: false\n+ properties:\n+ ConstraintNormal: { $ref: '#/$defs/_AgcConstraintMode' }\n+ ConstraintHighlight: { $ref: '#/$defs/_AgcConstraintMode' }\n+ ConstraintShadows: { $ref: '#/$defs/_AgcConstraintMode' }\n+ ConstraintCustom: { $ref: '#/$defs/_AgcConstraintMode' }\n+ relativeLuminanceTarget:\n+ type: array\n+ items:\n+ type: number\n+\n+ # Ideally these should be defined inside Agc and referenced using relative paths\n+ # but apparently this is not supported by the spec.\n+ # See https://github.com/python-jsonschema/jsonschema/issues/1265\n+ _AgcMeteringMode:\n+ type: array\n+ items:\n+ type: integer\n+ minItems: 25\n+ maxItems: 25\n+ _AgcConstraintMode:\n+ type: object\n+ additionalProperties: false\n+ properties:\n+ lower:\n+ type: object\n+ additionalProperties: false\n+ properties:\n+ qLo:\n+ type: number\n+ qHi:\n+ type: number\n+ yTarget:\n+ type: array\n+ items:\n+ type: number\n+ upper:\n+ type: object\n+ additionalProperties: false\n+ properties:\n+ qLo:\n+ type: number\n+ qHi:\n+ type: number\n+ yTarget:\n+ type: array\n+ items:\n+ type: number\n+ _AgcExposureMode:\n+ type: object\n+ additionalProperties: false\n+ required: ['shutter', 'gain']\n+ properties:\n+ shutter:\n+ type: array\n+ items:\n+ type: integer\n+ gain:\n+ type: array\n+ items:\n+ type: number\n+\n+ Awb:\n+ type: object\n+ additionalProperties: false\n+ required: ['Awb']\n+ properties:\n+ Awb: { $ref: '#/$defs/_Empty' }\n+ BlackLevelCorrection:\n+ type: object\n+ additionalProperties: false\n+ required: ['BlackLevelCorrection']\n+ properties:\n+ BlackLevelCorrection:\n+ type: object\n+ additionalProperties: false\n+ required: ['B', 'Gb', 'Gr', 'R']\n+ properties:\n+ B:\n+ type: integer\n+ Gb:\n+ type: integer\n+ Gr:\n+ type: integer\n+ R:\n+ type: integer\n+ Ccm:\n+ type: object\n+ additionalProperties: false\n+ required: ['Ccm']\n+ properties:\n+ Ccm:\n+ type: object\n+ additionalProperties: false\n+ required: ['ccms']\n+ properties:\n+ ccms:\n+ type: array\n+ items:\n+ type: object\n+ additionalProperties: false\n+ required: ['ct', 'ccm']\n+ properties:\n+ ct:\n+ type: integer\n+ ccm:\n+ type: array\n+ items:\n+ type: number\n+ minItems: 9\n+ maxItems: 9\n+ offsets:\n+ type: array\n+ items:\n+ type: integer\n+ minItems: 3\n+ maxItems: 3\n+ ColorProcessing:\n+ type: object\n+ additionalProperties: false\n+ required: ['ColorProcessing']\n+ properties:\n+ ColorProcessing: { $ref: '#/$defs/_Empty' }\n+ DefectPixelClusterCorrection:\n+ type: object\n+ additionalProperties: false\n+ required: ['DefectPixelClusterCorrection']\n+ properties:\n+ DefectPixelClusterCorrection:\n+ type: object\n+ additionalProperties: false\n+ required: ['sets']\n+ properties:\n+ fixed-set:\n+ type: boolean\n+ sets:\n+ type: array\n+ maxItems: 3\n+ items:\n+ type: object\n+ additionalProperties: false\n+ # Todo clarify what is really required here ov5640 misses\n+ # 'rg-factor', 'rnd-offsets', 'rnd-threshold'\n+ required: ['line-mad-factor', 'line-threshold', 'pg-factor', 'ro-limits']\n+ properties:\n+ line-mad-factor: { $ref: '#/$defs/_DefectPixelClusterCorrectionFactor' }\n+ line-threshold: { $ref: '#/$defs/_DefectPixelClusterCorrectionFactor' }\n+ pg-factor: { $ref: '#/$defs/_DefectPixelClusterCorrectionFactor' }\n+ rg-factor: { $ref: '#/$defs/_DefectPixelClusterCorrectionFactor' }\n+ rnd-offsets: { $ref: '#/$defs/_DefectPixelClusterCorrectionFactor' }\n+ rnd-threshold: { $ref: '#/$defs/_DefectPixelClusterCorrectionFactor' }\n+ ro-limits: { $ref: '#/$defs/_DefectPixelClusterCorrectionFactor' }\n+ _DefectPixelClusterCorrectionFactor:\n+ type: object\n+ additionalProperties: false\n+ properties:\n+ green:\n+ type: integer\n+ red-blue:\n+ type: integer\n+ Dpf:\n+ type: object\n+ additionalProperties: false\n+ required: ['Dpf']\n+ properties:\n+ Dpf:\n+ type: object\n+ additionalProperties: false\n+ required: ['DomainFilter', 'FilterStrength', 'NoiseLevelFunction']\n+ properties:\n+ DomainFilter:\n+ type: object\n+ additionalProperties: false\n+ required: ['g', 'rb']\n+ properties:\n+ g:\n+ type: array\n+ items:\n+ type: integer\n+ minItems: 6\n+ maxItems: 6\n+ rb:\n+ type: array\n+ items:\n+ type: integer\n+ minItems: 6\n+ maxItems: 6\n+ FilterStrength:\n+ type: object\n+ additionalProperties: false\n+ required: ['b', 'g', 'r']\n+ properties:\n+ b:\n+ type: integer\n+ g:\n+ type: integer\n+ r:\n+ type: integer\n+ NoiseLevelFunction:\n+ type: object\n+ additionalProperties: false\n+ required: ['coeff', 'scale-mode']\n+ properties:\n+ coeff:\n+ type: array\n+ items:\n+ type: integer\n+ minItems: 17\n+ maxItems: 17\n+ scale-mode:\n+ type: string\n+ enum: ['linear', 'logarithmic']\n+ Filter:\n+ type: object\n+ additionalProperties: false\n+ required: ['Filter']\n+ properties:\n+ Filter: { $ref: '#/$defs/_Empty' }\n+ GammaSensorLinearization:\n+ type: object\n+ additionalProperties: false\n+ required: ['GammaSensorLinearization']\n+ properties:\n+ GammaSensorLinearization:\n+ type: object\n+ additionalProperties: false\n+ required: ['x-intervals', 'y']\n+ properties:\n+ x-intervals:\n+ type: array\n+ items:\n+ type: number\n+ minItems: 16\n+ maxItems: 16\n+ y:\n+ type: object\n+ additionalProperties: false\n+ required: ['red', 'green', 'blue']\n+ properties:\n+ red:\n+ type: array\n+ items:\n+ type: integer\n+ minItems: 17\n+ maxItems: 17\n+ green:\n+ type: array\n+ items:\n+ type: integer\n+ minItems: 17\n+ maxItems: 17\n+ blue:\n+ type: array\n+ items:\n+ type: integer\n+ minItems: 17\n+ maxItems: 17\n+ LensShadingCorrection:\n+ type: object\n+ additionalProperties: false\n+ required: ['LensShadingCorrection']\n+ properties:\n+ LensShadingCorrection:\n+ type: object\n+ additionalProperties: false\n+ required: [x-size, y-size, sets]\n+ properties:\n+ x-size:\n+ type: array\n+ items:\n+ type: number\n+ minItems: 8\n+ maxItems: 8\n+ y-size:\n+ type: array\n+ items:\n+ type: number\n+ minItems: 8\n+ maxItems: 8\n+ sets:\n+ type: array\n+ items:\n+ type: object\n+ additionalProperties: false\n+ required: ['ct', 'r', 'gr', 'gb', 'b']\n+ properties:\n+ ct:\n+ type: integer\n+ r:\n+ type: array\n+ items:\n+ type: integer\n+ minItems: 289\n+ maxItems: 289\n+ gr:\n+ type: array\n+ items:\n+ type: integer\n+ minItems: 289\n+ maxItems: 289\n+ gb:\n+ type: array\n+ items:\n+ type: integer\n+ minItems: 289\n+ maxItems: 289\n+ b:\n+ type: array\n+ items:\n+ type: integer\n+ minItems: 289\n+ maxItems: 289\n+ # ToDo: We have some tuning files with that property. Is this needed?\n+ resolution:\n+ type: string\n+ # special definition to denote algorithms that do not have any parameters\n+ _Empty:\n+ type: [ 'object', 'null' ]\n+ additionalProperties: false\n+ properties: {}\n+...\ndiff --git a/utils/validate-tuning-files.py b/utils/validate-tuning-files.py\nnew file mode 100755\nindex 00000000..45f1d863\n--- /dev/null\n+++ b/utils/validate-tuning-files.py\n@@ -0,0 +1,112 @@\n+#!/usr/bin/env python3\n+# SPDX-License-Identifier: GPL-2.0-or-later\n+# Copyright (C) 2024, Ideas on Board Oy\n+#\n+# Author: Stefan Klug \n+#\n+# Validate tuning files against schema\n+\n+import argparse\n+import logging\n+import jsonschema\n+import yaml\n+import sys\n+import json\n+import os\n+from pathlib import Path\n+\n+try:\n+ import coloredlogs\n+ coloredlogs.install(level=logging.INFO, fmt='%(levelname)s: %(message)s')\n+except ImportError:\n+ logging.basicConfig(level=logging.INFO,\n+ format=\"%(levelname)s: %(message)s\")\n+\n+logger = logging.getLogger()\n+\n+\n+def do_schema_check(schema_file, files):\n+ res = True\n+ with open(schema_file, 'r') as file:\n+ schema = yaml.safe_load(file)\n+\n+ for file in files:\n+ with open(file, 'r') as f:\n+ logger.info(f\"Validating file {file} against {schema_file}\")\n+ data = yaml.safe_load(f)\n+ try:\n+ jsonschema.validate(instance=data, schema=schema)\n+ except jsonschema.exceptions.ValidationError as e:\n+ logging.error(f\"Validation error in file {file}: {e}\")\n+ res = False\n+ return res\n+\n+# Checks for the given ipa. If files is None, all files for that ipa get checked\n+\n+\n+def do_schema_check_ipa(ipa, files=None):\n+ res = True\n+\n+ if ipa != 'rkisp1':\n+ raise ValueError(f\"Ipa '{ipa}' is not supported\")\n+\n+ logger.info(f\"Checking ipa {ipa}\")\n+\n+ top_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))\n+ # Todo implement for other pipelines\n+ data_dir = 'src/ipa/rkisp1/data'\n+\n+ full_dir = Path(os.path.join(top_dir, data_dir))\n+\n+ schema_path = os.path.join(full_dir, f'tuning-{ipa}.schema.yaml')\n+ if not os.path.isfile(schema_path):\n+ raise ValueError(f\"Schema file {schema_path} doesn't exist\")\n+\n+ if files is None:\n+ files = [f for f in full_dir.glob(\n+ '*.yaml') if f.is_file() and not f.name.endswith('.schema.yaml')]\n+\n+ if not do_schema_check(schema_path, files):\n+ res = False\n+\n+ return res\n+\n+\n+def do_schema_check_all():\n+ # We only support rkisp1 for now\n+ return do_schema_check_ipa('rkisp1')\n+\n+\n+def main():\n+\n+ parser = argparse.ArgumentParser()\n+ group = parser.add_mutually_exclusive_group(required=True)\n+ group.add_argument('--all', '-a', action='store_true',\n+ help='automatically find and check all tuning files.')\n+ group.add_argument('--schema', '-s',\n+ help='schema file to check against')\n+ group.add_argument('--pipeline', '-p',\n+ help='pipeline to check against (Currently only rkisp1 is supported)')\n+ parser.add_argument('files', metavar='FILE', nargs='*',\n+ help='tuning files to check')\n+\n+ args = parser.parse_args()\n+\n+ if args.all:\n+ if args.files != []:\n+ parser.error(\"No files should be given when using --all\")\n+\n+ if not do_schema_check_all():\n+ return 1\n+ return 0\n+\n+ if not args.files:\n+ argparse.error(\"No files to check\")\n+ return 1\n+\n+ if not do_schema_check(args.schema, args.files):\n+ return 1\n+\n+\n+if __name__ == '__main__':\n+ sys.exit(main())\n","prefixes":["v1"]}