[{"id":25855,"web_url":"https://patchwork.libcamera.org/comment/25855/","msgid":"","date":"2022-11-23T00:32:50","subject":"Re: [libcamera-devel] [PATCH v3 10/12] utils: libtuning:\n\tgenerators: Add yaml output","submitter":{"id":2,"url":"https://patchwork.libcamera.org/api/people/2/","name":"Laurent Pinchart","email":"laurent.pinchart@ideasonboard.com"},"content":"Hi Paul,\n\nThank you for the patch.\n\nOn Fri, Nov 11, 2022 at 02:31:52AM +0900, Paul Elder via libcamera-devel wrote:\n> Add a generator to libtuning for writing tuning output to a yaml file.\n> \n> Signed-off-by: Paul Elder \n> \n> ---\n> Changes in v3:\n> - add file description\n> - remove indirection from fake polymorphism\n> \n> New in 2\n> ---\n> utils/tuning/libtuning/generators/__init__.py | 1 +\n> .../libtuning/generators/yaml_output.py | 123 ++++++++++++++++++\n> 2 files changed, 124 insertions(+)\n> create mode 100644 utils/tuning/libtuning/generators/yaml_output.py\n> \n> diff --git a/utils/tuning/libtuning/generators/__init__.py b/utils/tuning/libtuning/generators/__init__.py\n> index 937aff30..f28b6149 100644\n> --- a/utils/tuning/libtuning/generators/__init__.py\n> +++ b/utils/tuning/libtuning/generators/__init__.py\n> @@ -3,3 +3,4 @@\n> # Copyright (C) 2022, Paul Elder \n> \n> from libtuning.generators.raspberrypi_output import RaspberryPiOutput\n> +from libtuning.generators.yaml_output import YamlOutput\n> diff --git a/utils/tuning/libtuning/generators/yaml_output.py b/utils/tuning/libtuning/generators/yaml_output.py\n> new file mode 100644\n> index 00000000..6398fc59\n> --- /dev/null\n> +++ b/utils/tuning/libtuning/generators/yaml_output.py\n> @@ -0,0 +1,123 @@\n> +# SPDX-License-Identifier: GPL-2.0-or-later\n> +#\n> +# Copyright 2022 Paul Elder \n> +#\n> +# yaml_output.py - Generate tuning file in YAML format\n> +\n> +from .generator import Generator\n> +\n> +from numbers import Number\n> +from pathlib import Path\n> +\n> +import libtuning.utils as utils\n> +\n> +\n> +class YamlOutput(Generator):\n> + def __init__(self):\n> + super().__init__()\n> +\n> + def _stringify_number_list(self, listt: list):\n> + line_wrap = 80\n> +\n> + line = '[ ' + ', '.join([str(x) for x in listt]) + ' ]'\n> + if len(line) <= line_wrap:\n> + return [line]\n> +\n> + out_lines = ['[']\n> + line = ' '\n> + for x in listt:\n> + x_str = str(x)\n> + # If the first number is longer than line_wrap, it'll add an extra line\n> + if len(line) + len(x_str) > line_wrap:\n> + out_lines.append(line)\n> + line = f' {x_str},'\n> + continue\n> + line += f' {x_str},'\n> + out_lines.append(line)\n> + out_lines.append(']')\n> +\n> + return out_lines\n> +\n> + # @return Array of lines, and boolean of if all elements were numbers\n> + def _stringify_list(self, listt: list):\n> + out_lines = []\n> +\n> + all_numbers = set([isinstance(x, Number) for x in listt]).issubset({True})\n> +\n> + if all_numbers:\n> + return self._stringify_number_list(listt), True\n> +\n> + for value in listt:\n> + if isinstance(value, Number):\n> + out_lines.append(f'- {str(value)}')\n> + elif isinstance(value, str):\n> + out_lines.append(f'- \"{value}\"')\n> + elif isinstance(value, list):\n> + lines, all_numbers = self._stringify_list(value)\n> +\n> + if all_numbers:\n> + out_lines.append( f'- {lines[0]}')\n\nExtra space after (. Same below.\n\n> + out_lines += [f' {line}' for line in lines[1:]]\n\nToo many spaces ? Or did you indent this on purpose ? Same below.\n\n> + else:\n> + out_lines.append( f'-')\n> + out_lines += [f' {line}' for line in lines]\n> + elif isinstance(value, dict):\n> + lines = self._stringify_dict(value)\n> + out_lines.append( f'- {lines[0]}')\n> + out_lines += [f' {line}' for line in lines[1:]]\n> +\n> + return out_lines, False\n> +\n> + def _stringify_dict(self, dictt: dict):\n> + out_lines = []\n> +\n> + for key in dictt:\n> + value = dictt[key]\n> +\n> + if isinstance(value, Number):\n> + out_lines.append(f'{key}: {str(value)}')\n> + elif isinstance(value, str):\n> + out_lines.append(f'{key}: \"{value}\"')\n> + elif isinstance(value, list):\n> + lines, all_numbers = self._stringify_list(value)\n> +\n> + if all_numbers:\n> + out_lines.append( f'{key}: {lines[0]}')\n> + out_lines += [f'{\" \" * (len(key) + 2)}{line}' for line in lines[1:]]\n> + else:\n> + out_lines.append( f'{key}:')\n> + out_lines += [f' {line}' for line in lines]\n> + elif isinstance(value, dict):\n> + lines = self._stringify_dict(value)\n> + out_lines.append( f'{key}:')\n> + out_lines += [f' {line}' for line in lines]\n> +\n> + return out_lines\n> +\n> + def write(self, output_file: Path, output_dict: dict, output_order: list):\n> + out_lines = [\n> + '%YAML 1.1',\n> + '---',\n> + 'version: 1',\n> + # No need to condition this, as libtuning already guarantees that\n> + # we have at least one module. Even if the module has no output,\n> + # its prescence is sufficient.\n> + 'algorithms:'\n> + ]\n> +\n> + for module in output_order:\n> + out_lines.append(f' - {module.out_name}:')\n> +\n> + if len(output_dict[module]) == 0:\n> + continue\n> +\n> + if not isinstance(output_dict[module], dict):\n> + utils.eprint(f'Error: Output of {module.type} is not a dictionary')\n> + continue\n> +\n> + lines = self._stringify_dict(output_dict[module])\n> + out_lines += [f' {line}' for line in lines]\n> +\n> + with open(output_file, 'w') as f:\n\nShould we set the encoding to UTF-8 explicitly ?\n\nReviewed-by: Laurent Pinchart \n\n> + for line in out_lines:\n> + f.write(f'{line}\\n')","headers":{"Return-Path":"","X-Original-To":"parsemail@patchwork.libcamera.org","Delivered-To":"parsemail@patchwork.libcamera.org","Received":["from lancelot.ideasonboard.com (lancelot.ideasonboard.com\n\t[92.243.16.209])\n\tby patchwork.libcamera.org (Postfix) with ESMTPS id 61A24BDE6B\n\tfor ;\n\tWed, 23 Nov 2022 00:33:09 +0000 (UTC)","from lancelot.ideasonboard.com (localhost [IPv6:::1])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTP id B27C463314;\n\tWed, 23 Nov 2022 01:33:08 +0100 (CET)","from perceval.ideasonboard.com (perceval.ideasonboard.com\n\t[IPv6:2001:4b98:dc2:55:216:3eff:fef7:d647])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id 1D93261F2B\n\tfor ;\n\tWed, 23 Nov 2022 01:33:07 +0100 (CET)","from pendragon.ideasonboard.com (62-78-145-57.bb.dnainternet.fi\n\t[62.78.145.57])\n\tby perceval.ideasonboard.com (Postfix) with ESMTPSA id 6E324890;\n\tWed, 23 Nov 2022 01:33:06 +0100 (CET)"],"DKIM-Signature":["v=1; a=rsa-sha256; c=relaxed/simple; d=libcamera.org;\n\ts=mail; t=1669163588;\n\tbh=uyKEwEbECTbfkpzYIJcsUUR7ICGrd/c+lWFWpQBKFV4=;\n\th=Date:To:References:In-Reply-To:Subject:List-Id:List-Unsubscribe:\n\tList-Archive:List-Post:List-Help:List-Subscribe:From:Reply-To:Cc:\n\tFrom;\n\tb=MXkF0Za4tjBkGJyiSkMQBBGGieXQiQJqBc8SYVJKjfHxAEpM8jTsmScxnG4Hl33T2\n\tPJMRFG3F4szbgwi+Eo1tfSqIaTbbKxiK8b/RLphgyJnKrHquSMNFJg+P1YW1/UKF+9\n\txtIg/3+vkBcJiyhXCqpobunpoHO4O7zc0j8N/tN3N4rEUq9owDqwb6pCsyf7c62IMG\n\tovjixBS5opaxLo6A9yGKWXGTZcBCNKa4zs3Jt3xjmvfC9t0Z5k5CdOpExkTUUIKDKu\n\t6TEH0hOrE+yftKX1cAwApX3MV1dI/ODGgD3LPKgHIcFaZCoOk0xFK7cFG4ecWai5kZ\n\tK1DWFSpwz9EzA==","v=1; a=rsa-sha256; c=relaxed/simple; d=ideasonboard.com;\n\ts=mail; t=1669163586;\n\tbh=uyKEwEbECTbfkpzYIJcsUUR7ICGrd/c+lWFWpQBKFV4=;\n\th=Date:From:To:Cc:Subject:References:In-Reply-To:From;\n\tb=iMFpkg5hmYBp8cKooD5GvjKY6aFYSM4w86r59mK0rphGixeThsaUuWuCjFYVqwPQl\n\tfdJyTyUnJE/xkL1ayyEKZxd+KxHRT5R4HQip6boSx3uF4Vim47WxAuTF1atYTEZE+G\n\ti3zgBa09AajLBImtq1pPvyyoEg/Y8XlCXsrCBimQ="],"Authentication-Results":"lancelot.ideasonboard.com; dkim=pass (1024-bit key; \n\tunprotected) header.d=ideasonboard.com\n\theader.i=@ideasonboard.com\n\theader.b=\"iMFpkg5h\"; dkim-atps=neutral","Date":"Wed, 23 Nov 2022 02:32:50 +0200","To":"Paul Elder ","Message-ID":"","References":"<20221110173154.488445-1-paul.elder@ideasonboard.com>\n\t<20221110173154.488445-11-paul.elder@ideasonboard.com>","MIME-Version":"1.0","Content-Type":"text/plain; charset=utf-8","Content-Disposition":"inline","In-Reply-To":"<20221110173154.488445-11-paul.elder@ideasonboard.com>","Subject":"Re: [libcamera-devel] [PATCH v3 10/12] utils: libtuning:\n\tgenerators: Add yaml output","X-BeenThere":"libcamera-devel@lists.libcamera.org","X-Mailman-Version":"2.1.29","Precedence":"list","List-Id":"","List-Unsubscribe":",\n\t","List-Archive":"","List-Post":"","List-Help":"","List-Subscribe":",\n\t","From":"Laurent Pinchart via libcamera-devel\n\t","Reply-To":"Laurent Pinchart ","Cc":"libcamera-devel@lists.libcamera.org","Errors-To":"libcamera-devel-bounces@lists.libcamera.org","Sender":"\"libcamera-devel\" "}},{"id":25865,"web_url":"https://patchwork.libcamera.org/comment/25865/","msgid":"","date":"2022-11-23T10:29:16","subject":"Re: [libcamera-devel] [PATCH v3 10/12] utils: libtuning:\n\tgenerators: Add yaml output","submitter":{"id":17,"url":"https://patchwork.libcamera.org/api/people/17/","name":"Paul Elder","email":"paul.elder@ideasonboard.com"},"content":"On Wed, Nov 23, 2022 at 02:32:50AM +0200, Laurent Pinchart wrote:\n> Hi Paul,\n> \n> Thank you for the patch.\n> \n> On Fri, Nov 11, 2022 at 02:31:52AM +0900, Paul Elder via libcamera-devel wrote:\n> > Add a generator to libtuning for writing tuning output to a yaml file.\n> > \n> > Signed-off-by: Paul Elder \n> > \n> > ---\n> > Changes in v3:\n> > - add file description\n> > - remove indirection from fake polymorphism\n> > \n> > New in 2\n> > ---\n> > utils/tuning/libtuning/generators/__init__.py | 1 +\n> > .../libtuning/generators/yaml_output.py | 123 ++++++++++++++++++\n> > 2 files changed, 124 insertions(+)\n> > create mode 100644 utils/tuning/libtuning/generators/yaml_output.py\n> > \n> > diff --git a/utils/tuning/libtuning/generators/__init__.py b/utils/tuning/libtuning/generators/__init__.py\n> > index 937aff30..f28b6149 100644\n> > --- a/utils/tuning/libtuning/generators/__init__.py\n> > +++ b/utils/tuning/libtuning/generators/__init__.py\n> > @@ -3,3 +3,4 @@\n> > # Copyright (C) 2022, Paul Elder \n> > \n> > from libtuning.generators.raspberrypi_output import RaspberryPiOutput\n> > +from libtuning.generators.yaml_output import YamlOutput\n> > diff --git a/utils/tuning/libtuning/generators/yaml_output.py b/utils/tuning/libtuning/generators/yaml_output.py\n> > new file mode 100644\n> > index 00000000..6398fc59\n> > --- /dev/null\n> > +++ b/utils/tuning/libtuning/generators/yaml_output.py\n> > @@ -0,0 +1,123 @@\n> > +# SPDX-License-Identifier: GPL-2.0-or-later\n> > +#\n> > +# Copyright 2022 Paul Elder \n> > +#\n> > +# yaml_output.py - Generate tuning file in YAML format\n> > +\n> > +from .generator import Generator\n> > +\n> > +from numbers import Number\n> > +from pathlib import Path\n> > +\n> > +import libtuning.utils as utils\n> > +\n> > +\n> > +class YamlOutput(Generator):\n> > + def __init__(self):\n> > + super().__init__()\n> > +\n> > + def _stringify_number_list(self, listt: list):\n> > + line_wrap = 80\n> > +\n> > + line = '[ ' + ', '.join([str(x) for x in listt]) + ' ]'\n> > + if len(line) <= line_wrap:\n> > + return [line]\n> > +\n> > + out_lines = ['[']\n> > + line = ' '\n> > + for x in listt:\n> > + x_str = str(x)\n> > + # If the first number is longer than line_wrap, it'll add an extra line\n> > + if len(line) + len(x_str) > line_wrap:\n> > + out_lines.append(line)\n> > + line = f' {x_str},'\n> > + continue\n> > + line += f' {x_str},'\n> > + out_lines.append(line)\n> > + out_lines.append(']')\n> > +\n> > + return out_lines\n> > +\n> > + # @return Array of lines, and boolean of if all elements were numbers\n> > + def _stringify_list(self, listt: list):\n> > + out_lines = []\n> > +\n> > + all_numbers = set([isinstance(x, Number) for x in listt]).issubset({True})\n> > +\n> > + if all_numbers:\n> > + return self._stringify_number_list(listt), True\n> > +\n> > + for value in listt:\n> > + if isinstance(value, Number):\n> > + out_lines.append(f'- {str(value)}')\n> > + elif isinstance(value, str):\n> > + out_lines.append(f'- \"{value}\"')\n> > + elif isinstance(value, list):\n> > + lines, all_numbers = self._stringify_list(value)\n> > +\n> > + if all_numbers:\n> > + out_lines.append( f'- {lines[0]}')\n> \n> Extra space after (. Same below.\n\nI wanted the extra space so that the output text lines up so it's easier\nto see how it'll look.\n\nIf it's not worth the PEP warning I could remove it.\n\n> \n> > + out_lines += [f' {line}' for line in lines[1:]]\n> \n> Too many spaces ? Or did you indent this on purpose ? Same below.\n> \n> > + else:\n> > + out_lines.append( f'-')\n> > + out_lines += [f' {line}' for line in lines]\n> > + elif isinstance(value, dict):\n> > + lines = self._stringify_dict(value)\n> > + out_lines.append( f'- {lines[0]}')\n> > + out_lines += [f' {line}' for line in lines[1:]]\n> > +\n> > + return out_lines, False\n> > +\n> > + def _stringify_dict(self, dictt: dict):\n> > + out_lines = []\n> > +\n> > + for key in dictt:\n> > + value = dictt[key]\n> > +\n> > + if isinstance(value, Number):\n> > + out_lines.append(f'{key}: {str(value)}')\n> > + elif isinstance(value, str):\n> > + out_lines.append(f'{key}: \"{value}\"')\n> > + elif isinstance(value, list):\n> > + lines, all_numbers = self._stringify_list(value)\n> > +\n> > + if all_numbers:\n> > + out_lines.append( f'{key}: {lines[0]}')\n> > + out_lines += [f'{\" \" * (len(key) + 2)}{line}' for line in lines[1:]]\n> > + else:\n> > + out_lines.append( f'{key}:')\n> > + out_lines += [f' {line}' for line in lines]\n> > + elif isinstance(value, dict):\n> > + lines = self._stringify_dict(value)\n> > + out_lines.append( f'{key}:')\n> > + out_lines += [f' {line}' for line in lines]\n> > +\n> > + return out_lines\n> > +\n> > + def write(self, output_file: Path, output_dict: dict, output_order: list):\n> > + out_lines = [\n> > + '%YAML 1.1',\n> > + '---',\n> > + 'version: 1',\n> > + # No need to condition this, as libtuning already guarantees that\n> > + # we have at least one module. Even if the module has no output,\n> > + # its prescence is sufficient.\n> > + 'algorithms:'\n> > + ]\n> > +\n> > + for module in output_order:\n> > + out_lines.append(f' - {module.out_name}:')\n> > +\n> > + if len(output_dict[module]) == 0:\n> > + continue\n> > +\n> > + if not isinstance(output_dict[module], dict):\n> > + utils.eprint(f'Error: Output of {module.type} is not a dictionary')\n> > + continue\n> > +\n> > + lines = self._stringify_dict(output_dict[module])\n> > + out_lines += [f' {line}' for line in lines]\n> > +\n> > + with open(output_file, 'w') as f:\n> \n> Should we set the encoding to UTF-8 explicitly ?\n\nYeah we probably should.\n\n> \n> Reviewed-by: Laurent Pinchart \n\n\nThanks,\n\nPaul\n\n> \n> > + for line in out_lines:\n> > + f.write(f'{line}\\n')","headers":{"Return-Path":"","X-Original-To":"parsemail@patchwork.libcamera.org","Delivered-To":"parsemail@patchwork.libcamera.org","Received":["from lancelot.ideasonboard.com (lancelot.ideasonboard.com\n\t[92.243.16.209])\n\tby patchwork.libcamera.org (Postfix) with ESMTPS id 8755DBDE6B\n\tfor ;\n\tWed, 23 Nov 2022 10:29:25 +0000 (UTC)","from lancelot.ideasonboard.com (localhost [IPv6:::1])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTP id C4F9363319;\n\tWed, 23 Nov 2022 11:29:24 +0100 (CET)","from perceval.ideasonboard.com (perceval.ideasonboard.com\n\t[213.167.242.64])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id 2C4606330D\n\tfor ;\n\tWed, 23 Nov 2022 11:29:23 +0100 (CET)","from pyrite.rasen.tech (h175-177-042-159.catv02.itscom.jp\n\t[175.177.42.159])\n\tby perceval.ideasonboard.com (Postfix) with ESMTPSA id BA49B88F;\n\tWed, 23 Nov 2022 11:29:21 +0100 (CET)"],"DKIM-Signature":["v=1; a=rsa-sha256; c=relaxed/simple; d=libcamera.org;\n\ts=mail; t=1669199364;\n\tbh=DE+//PtOlpIYwxtO5XTcUyiTduNtbaqCCO5WNw0umxk=;\n\th=Date:To:References:In-Reply-To:Subject:List-Id:List-Unsubscribe:\n\tList-Archive:List-Post:List-Help:List-Subscribe:From:Reply-To:Cc:\n\tFrom;\n\tb=0HUsY6LmuWgfQI/tNueBLe23brDqxKBPVX1rDNgCZsWjSF15tKNsrhJDj22NAO3FL\n\t+/mTvV5/ctpJw2hrBDJIl56Nhs0MyA+hreTPgYk/IWX8fz6kuYSb0VpZ48k8D1w61r\n\t5IJ401/Gw2rXOA2aIbRUVbiwHmwabvv9Ny0Y6NjmXBY6wJm1+uyokIMRiIDJEqvEws\n\ttFzyWXAQ1SJRHkiye+tmDOurU3N0BWP5EUNu4zgWvFJDrnMe6Wp5iIBB4dqXuOHG58\n\ttwvTuWbqaAzgNJ1CTG5H/zWb6UueA7pr4u2qIARSxmAo8L1y8Yry7xi6mU3g+iBFQD\n\tUZcxCsQRYi6Iw==","v=1; a=rsa-sha256; c=relaxed/simple; d=ideasonboard.com;\n\ts=mail; t=1669199362;\n\tbh=DE+//PtOlpIYwxtO5XTcUyiTduNtbaqCCO5WNw0umxk=;\n\th=Date:From:To:Cc:Subject:References:In-Reply-To:From;\n\tb=jEZbZ7WJEtNk7ZzhWEeSlY7vBRvFsYbWa2rTDQw7DYLW33cCW0P+tJIdrxpkRwvnt\n\t4veLwZfK2VSSiW6RTOZ//fQyJlg8OlXdsMPs5mvF9RcavIh5CzQ1O+BjDaoX+IqC+3\n\tFjdPxveCE3tHcdlOZzPNAH8Q4+9lnI8D/GXxG5i8="],"Authentication-Results":"lancelot.ideasonboard.com; dkim=pass (1024-bit key; \n\tunprotected) header.d=ideasonboard.com\n\theader.i=@ideasonboard.com\n\theader.b=\"jEZbZ7WJ\"; dkim-atps=neutral","Date":"Wed, 23 Nov 2022 19:29:16 +0900","To":"Laurent Pinchart ","Message-ID":"","References":"<20221110173154.488445-1-paul.elder@ideasonboard.com>\n\t<20221110173154.488445-11-paul.elder@ideasonboard.com>\n\t","MIME-Version":"1.0","Content-Type":"text/plain; charset=us-ascii","Content-Disposition":"inline","In-Reply-To":"","Subject":"Re: [libcamera-devel] [PATCH v3 10/12] utils: libtuning:\n\tgenerators: Add yaml output","X-BeenThere":"libcamera-devel@lists.libcamera.org","X-Mailman-Version":"2.1.29","Precedence":"list","List-Id":"","List-Unsubscribe":",\n\t","List-Archive":"","List-Post":"","List-Help":"","List-Subscribe":",\n\t","From":"Paul Elder via libcamera-devel ","Reply-To":"Paul Elder ","Cc":"libcamera-devel@lists.libcamera.org","Errors-To":"libcamera-devel-bounces@lists.libcamera.org","Sender":"\"libcamera-devel\" "}}]