[{"id":24070,"web_url":"https://patchwork.libcamera.org/comment/24070/","msgid":"","date":"2022-07-23T16:43:29","subject":"Re: [libcamera-devel] [PATCH v6 6/8] utils: raspberrypi: ctt:\n\tOutput version 2.0 format tuning files","submitter":{"id":2,"url":"https://patchwork.libcamera.org/api/people/2/","name":"Laurent Pinchart","email":"laurent.pinchart@ideasonboard.com"},"content":"Hi Naush,\n\nThank you for the patch.\n\nOn Mon, Jul 18, 2022 at 09:16:00AM +0100, Naushir Patuck via libcamera-devel wrote:\n> Update the ctt_pretty_print_json.py script to generate the new version 2.0\n> format camera tuning file. This script can be called through the command line\n> to prettify an existing JSON file, or programatically by the CTT to format a\n> new JSON config dictionary.\n> \n> Update the CTT to produce a version 2.0 format json structure and use\n> ctt_pretty_print_json.pretty_print to prettify the output.\n> \n> Signed-off-by: Naushir Patuck \n\nReviewed-by: Laurent Pinchart \n\n> ---\n> utils/raspberrypi/ctt/ctt.py | 18 +-\n> .../raspberrypi/ctt/ctt_pretty_print_json.py | 194 +++++++++---------\n> 2 files changed, 113 insertions(+), 99 deletions(-)\n> mode change 100644 => 100755 utils/raspberrypi/ctt/ctt_pretty_print_json.py\n> \n> diff --git a/utils/raspberrypi/ctt/ctt.py b/utils/raspberrypi/ctt/ctt.py\n> index 15064634c67f..0ec5b49016df 100755\n> --- a/utils/raspberrypi/ctt/ctt.py\n> +++ b/utils/raspberrypi/ctt/ctt.py\n> @@ -15,7 +15,7 @@ from ctt_alsc import *\n> from ctt_lux import *\n> from ctt_noise import *\n> from ctt_geq import *\n> -from ctt_pretty_print_json import *\n> +from ctt_pretty_print_json import pretty_print\n> import random\n> import json\n> import re\n> @@ -511,13 +511,17 @@ class Camera:\n> \"\"\"\n> def write_json(self):\n> \"\"\"\n> - Write json dictionary to file\n> + Write json dictionary to file using our version 2 format\n> \"\"\"\n> - jstring = json.dumps(self.json, sort_keys=False)\n> - \"\"\"\n> - make it pretty :)\n> - \"\"\"\n> - pretty_print_json(jstring, self.jf)\n> +\n> + out_json = {\n> + \"version\": 2.0,\n> + 'target': 'bcm2835',\n> + \"algorithms\": [{name: data} for name, data in self.json.items()],\n> + }\n> +\n> + with open(self.jf, 'w') as f:\n> + f.write(pretty_print(out_json))\n> \n> \"\"\"\n> add a new section to the log file\n> diff --git a/utils/raspberrypi/ctt/ctt_pretty_print_json.py b/utils/raspberrypi/ctt/ctt_pretty_print_json.py\n> old mode 100644\n> new mode 100755\n> index d38ae6178524..ee5e8e10e437\n> --- a/utils/raspberrypi/ctt/ctt_pretty_print_json.py\n> +++ b/utils/raspberrypi/ctt/ctt_pretty_print_json.py\n> @@ -1,106 +1,116 @@\n> +#!/usr/bin/env python3\n> +#\n> # SPDX-License-Identifier: BSD-2-Clause\n> #\n> -# Copyright (C) 2019, Raspberry Pi (Trading) Limited\n> +# Script to pretty print a Raspberry Pi tuning config JSON structure in\n> +# version 2.0 and later formats.\n> #\n> -# ctt_pretty_print_json.py - camera tuning tool JSON formatter\n> -\n> -import sys\n> -\n> -\n> -class JSONPrettyPrinter(object):\n> - \"\"\"\n> - Take a collapsed JSON file and make it more readable\n> - \"\"\"\n> - def __init__(self, fout):\n> - self.state = {\n> - \"indent\": 0,\n> - \"inarray\": [False],\n> - \"arraycount\": [],\n> - \"skipnewline\": True,\n> - \"need_indent\": False,\n> - \"need_space\": False,\n> +# Copyright 2022 Raspberry Pi Ltd.\n> +\n> +import argparse\n> +import json\n> +import textwrap\n> +\n> +\n> +class Encoder(json.JSONEncoder):\n> +\n> + def __init__(self, *args, **kwargs):\n> + super().__init__(*args, **kwargs)\n> + self.indentation_level = 0\n> + self.hard_break = 120\n> + self.custom_elems = {\n> + 'table': 16,\n> + 'luminance_lut': 16,\n> + 'ct_curve': 3,\n> + 'ccm': 3,\n> + 'gamma_curve': 2,\n> + 'y_target': 2,\n> + 'prior': 2\n> }\n> \n> - self.fout = fout\n> -\n> - def newline(self):\n> - if not self.state[\"skipnewline\"]:\n> - self.fout.write('\\n')\n> - self.state[\"need_indent\"] = True\n> - self.state[\"need_space\"] = False\n> - self.state[\"skipnewline\"] = True\n> -\n> - def write(self, c):\n> - if self.state[\"need_indent\"]:\n> - self.fout.write(' ' * self.state[\"indent\"] * 4)\n> - self.state[\"need_indent\"] = False\n> - if self.state[\"need_space\"]:\n> - self.fout.write(' ')\n> - self.state[\"need_space\"] = False\n> - self.fout.write(c)\n> - self.state[\"skipnewline\"] = False\n> -\n> - def process_char(self, c):\n> - if c == '{':\n> - self.newline()\n> - self.write(c)\n> - self.state[\"indent\"] += 1\n> - self.newline()\n> - elif c == '}':\n> - self.state[\"indent\"] -= 1\n> - self.newline()\n> - self.write(c)\n> - elif c == '[':\n> - self.newline()\n> - self.write(c)\n> - self.state[\"indent\"] += 1\n> - self.newline()\n> - self.state[\"inarray\"] = [True] + self.state[\"inarray\"]\n> - self.state[\"arraycount\"] = [0] + self.state[\"arraycount\"]\n> - elif c == ']':\n> - self.state[\"indent\"] -= 1\n> - self.newline()\n> - self.state[\"inarray\"].pop(0)\n> - self.state[\"arraycount\"].pop(0)\n> - self.write(c)\n> - elif c == ':':\n> - self.write(c)\n> - self.state[\"need_space\"] = True\n> - elif c == ',':\n> - if not self.state[\"inarray\"][0]:\n> - self.write(c)\n> - self.newline()\n> + def encode(self, o, node_key=None):\n> + if isinstance(o, (list, tuple)):\n> + # Check if we are a flat list of numbers.\n> + if not any(isinstance(el, (list, tuple, dict)) for el in o):\n> + s = ', '.join(json.dumps(el) for el in o)\n> + if node_key in self.custom_elems.keys():\n> + # Special case handling to specify number of elements in a row for tables, ccm, etc.\n> + self.indentation_level += 1\n> + sl = s.split(', ')\n> + num = self.custom_elems[node_key]\n> + chunk = [self.indent_str + ', '.join(sl[x:x + num]) for x in range(0, len(sl), num)]\n> + t = ',\\n'.join(chunk)\n> + self.indentation_level -= 1\n> + output = f'\\n{self.indent_str}[\\n{t}\\n{self.indent_str}]'\n> + elif len(s) > self.hard_break - len(self.indent_str):\n> + # Break a long list with wraps.\n> + self.indentation_level += 1\n> + t = textwrap.fill(s, self.hard_break, break_long_words=False,\n> + initial_indent=self.indent_str, subsequent_indent=self.indent_str)\n> + self.indentation_level -= 1\n> + output = f'\\n{self.indent_str}[\\n{t}\\n{self.indent_str}]'\n> + else:\n> + # Smaller lists can remain on a single line.\n> + output = f' [ {s} ]'\n> + return output\n> else:\n> - self.write(c)\n> - self.state[\"arraycount\"][0] += 1\n> - if self.state[\"arraycount\"][0] == 16:\n> - self.state[\"arraycount\"][0] = 0\n> - self.newline()\n> + # Sub-structures in the list case.\n> + self.indentation_level += 1\n> + output = [self.indent_str + self.encode(el) for el in o]\n> + self.indentation_level -= 1\n> + output = ',\\n'.join(output)\n> + return f' [\\n{output}\\n{self.indent_str}]'\n> +\n> + elif isinstance(o, dict):\n> + self.indentation_level += 1\n> + output = []\n> + for k, v in o.items():\n> + if isinstance(v, dict) and len(v) == 0:\n> + # Empty config block special case.\n> + output.append(self.indent_str + f'{json.dumps(k)}: {{ }}')\n> else:\n> - self.state[\"need_space\"] = True\n> - elif c.isspace():\n> - pass\n> + # Only linebreak if the next node is a config block.\n> + sep = f'\\n{self.indent_str}' if isinstance(v, dict) else ''\n> + output.append(self.indent_str + f'{json.dumps(k)}:{sep}{self.encode(v, k)}')\n> + output = ',\\n'.join(output)\n> + self.indentation_level -= 1\n> + return f'{{\\n{output}\\n{self.indent_str}}}'\n> +\n> else:\n> - self.write(c)\n> + return ' ' + json.dumps(o)\n> +\n> + @property\n> + def indent_str(self) -> str:\n> + return ' ' * self.indentation_level * self.indent\n> +\n> + def iterencode(self, o, **kwargs):\n> + return self.encode(o)\n> +\n> +\n> +def pretty_print(in_json: dict) -> str:\n> +\n> + if 'version' not in in_json or \\\n> + 'target' not in in_json or \\\n> + 'algorithms' not in in_json or \\\n> + in_json['version'] < 2.0:\n> + raise RuntimeError('Incompatible JSON dictionary has been provided')\n> \n> - def print(self, string):\n> - for c in string:\n> - self.process_char(c)\n> - self.newline()\n> + return json.dumps(in_json, cls=Encoder, indent=4, sort_keys=False)\n> \n> \n> -def pretty_print_json(str_in, output_filename):\n> - with open(output_filename, \"w\") as fout:\n> - printer = JSONPrettyPrinter(fout)\n> - printer.print(str_in)\n> +if __name__ == \"__main__\":\n> + parser = argparse.ArgumentParser(formatter_class=argparse.RawTextHelpFormatter, description=\n> + 'Prettify a version 2.0 camera tuning config JSON file.')\n> + parser.add_argument('input', type=str, help='Input tuning file.')\n> + parser.add_argument('output', type=str, nargs='?',\n> + help='Output converted tuning file. If not provided, the input file will be updated in-place.',\n> + default=None)\n> + args = parser.parse_args()\n> \n> + with open(args.input, 'r') as f:\n> + in_json = json.load(f)\n> \n> -if __name__ == '__main__':\n> - if len(sys.argv) != 2:\n> - print(\"Usage: %s filename\" % sys.argv[0])\n> - sys.exit(1)\n> + out_json = pretty_print(in_json)\n> \n> - input_filename = sys.argv[1]\n> - with open(input_filename, \"r\") as fin:\n> - printer = JSONPrettyPrinter(sys.stdout)\n> - printer.print(fin.read())\n> + with open(args.output if args.output is not None else args.input, 'w') as f:\n> + f.write(out_json)","headers":{"Return-Path":"","X-Original-To":"parsemail@patchwork.libcamera.org","Delivered-To":"parsemail@patchwork.libcamera.org","Received":["from lancelot.ideasonboard.com (lancelot.ideasonboard.com\n\t[92.243.16.209])\n\tby patchwork.libcamera.org (Postfix) with ESMTPS id 10114BE173\n\tfor ;\n\tSat, 23 Jul 2022 16:43:35 +0000 (UTC)","from lancelot.ideasonboard.com (localhost [IPv6:::1])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTP id 850DC63312;\n\tSat, 23 Jul 2022 18:43:34 +0200 (CEST)","from perceval.ideasonboard.com (perceval.ideasonboard.com\n\t[213.167.242.64])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id 2BB01603F8\n\tfor ;\n\tSat, 23 Jul 2022 18:43:33 +0200 (CEST)","from pendragon.ideasonboard.com (62-78-145-57.bb.dnainternet.fi\n\t[62.78.145.57])\n\tby perceval.ideasonboard.com (Postfix) with ESMTPSA id 983B59F7;\n\tSat, 23 Jul 2022 18:43:32 +0200 (CEST)"],"DKIM-Signature":["v=1; a=rsa-sha256; c=relaxed/simple; d=libcamera.org;\n\ts=mail; t=1658594614;\n\tbh=Y8ucy8q5VAOFaiJRJkAJRJ2d9FsFBIvQs73A65B28t0=;\n\th=Date:To:References:In-Reply-To:Subject:List-Id:List-Unsubscribe:\n\tList-Archive:List-Post:List-Help:List-Subscribe:From:Reply-To:Cc:\n\tFrom;\n\tb=qsefb420oCelbdmyVL6BWDF7FtovXpt6W01bhQIoAs7M6pl7wgPkpWRFAHG7qxJxX\n\twWkjr6/eDsqyt/xGSlXwusUq/25J3witxf8ZQWdGtq/R4ZTK+n7egHaEq+9FN2Jsi3\n\tBDkik5bGluAhBrZhSUTn6BBPJLg2mv6CJbY8H4wK3jax6ba+0g/oApn383ESXcxWZI\n\toTmMytfB05HAhVqcmoNK1U93Ruj2fVjC30oYnBAxt9LJ8FmgyeXNAvK6VXObrouC/L\n\tyY7+ERp+0Lqu6zH3N6kmVxUCx0tzuhTqxl/u9s6DGB1vB8FCVTPQbIVdk7pcIrIrdL\n\tJNyRP0KkmB/BQ==","v=1; a=rsa-sha256; c=relaxed/simple; d=ideasonboard.com;\n\ts=mail; t=1658594612;\n\tbh=Y8ucy8q5VAOFaiJRJkAJRJ2d9FsFBIvQs73A65B28t0=;\n\th=Date:From:To:Cc:Subject:References:In-Reply-To:From;\n\tb=Rpd8B0UsXDyi3xx6zS+7OOl1haEWK3qa2VNkdGgR/sM5d1ugAwikFvA47tu/x9vhO\n\tLZXxNhXUQ2ghOondZ9bTT/ukuhoDM1stQSpv9ppdBwu7RCE2+6I0y8J7bgpuZulqI8\n\tA2xrK3xDkamoV2RnL2yEmIUXFbhQI0xyCa3pWVeM="],"Authentication-Results":"lancelot.ideasonboard.com; dkim=pass (1024-bit key; \n\tunprotected) header.d=ideasonboard.com\n\theader.i=@ideasonboard.com\n\theader.b=\"Rpd8B0Us\"; dkim-atps=neutral","Date":"Sat, 23 Jul 2022 19:43:29 +0300","To":"Naushir Patuck ","Message-ID":"","References":"<20220718081602.32535-1-naush@raspberrypi.com>\n\t<20220718081602.32535-7-naush@raspberrypi.com>","MIME-Version":"1.0","Content-Type":"text/plain; charset=utf-8","Content-Disposition":"inline","In-Reply-To":"<20220718081602.32535-7-naush@raspberrypi.com>","Subject":"Re: [libcamera-devel] [PATCH v6 6/8] utils: raspberrypi: ctt:\n\tOutput version 2.0 format tuning files","X-BeenThere":"libcamera-devel@lists.libcamera.org","X-Mailman-Version":"2.1.29","Precedence":"list","List-Id":"","List-Unsubscribe":",\n\t","List-Archive":"","List-Post":"","List-Help":"","List-Subscribe":",\n\t","From":"Laurent Pinchart via libcamera-devel\n\t","Reply-To":"Laurent Pinchart ","Cc":"libcamera-devel@lists.libcamera.org","Errors-To":"libcamera-devel-bounces@lists.libcamera.org","Sender":"\"libcamera-devel\" "}}]