[07/10] utils: codegen: gen-controls.py: Convert to jinja2 templates
diff mbox series

Message ID 20240809005914.20662-8-laurent.pinchart@ideasonboard.com
State Accepted
Headers show
Series
  • libcamera: Improve code generation for controls
Related show

Commit Message

Laurent Pinchart Aug. 9, 2024, 12:59 a.m. UTC
Jinja2 templates help separating the logic related to the template from
the generation of the data. The python code gets much clearer as a
result.

As an added bonus, we can use a single template file for both controls
and properties.

Signed-off-by: Laurent Pinchart <laurent.pinchart@ideasonboard.com>
---
 include/libcamera/control_ids.h.in  |  40 +++-
 include/libcamera/meson.build       |   2 +-
 include/libcamera/property_ids.h.in |  34 ----
 src/libcamera/control_ids.cpp.in    | 101 ++++++++--
 src/libcamera/meson.build           |   5 +-
 src/libcamera/property_ids.cpp.in   |  48 -----
 utils/codegen/gen-controls.py       | 285 ++++++----------------------
 7 files changed, 176 insertions(+), 339 deletions(-)
 delete mode 100644 include/libcamera/property_ids.h.in
 delete mode 100644 src/libcamera/property_ids.cpp.in

Comments

Paul Elder Aug. 15, 2024, 4:16 a.m. UTC | #1
On Fri, Aug 09, 2024 at 03:59:11AM +0300, Laurent Pinchart wrote:
> Jinja2 templates help separating the logic related to the template from
> the generation of the data. The python code gets much clearer as a
> result.
> 
> As an added bonus, we can use a single template file for both controls
> and properties.
> 
> Signed-off-by: Laurent Pinchart <laurent.pinchart@ideasonboard.com>
> ---
>  include/libcamera/control_ids.h.in  |  40 +++-
>  include/libcamera/meson.build       |   2 +-
>  include/libcamera/property_ids.h.in |  34 ----
>  src/libcamera/control_ids.cpp.in    | 101 ++++++++--
>  src/libcamera/meson.build           |   5 +-
>  src/libcamera/property_ids.cpp.in   |  48 -----
>  utils/codegen/gen-controls.py       | 285 ++++++----------------------
>  7 files changed, 176 insertions(+), 339 deletions(-)
>  delete mode 100644 include/libcamera/property_ids.h.in
>  delete mode 100644 src/libcamera/property_ids.cpp.in
> 
> diff --git a/include/libcamera/control_ids.h.in b/include/libcamera/control_ids.h.in
> index 293ba966fbc4..858ef872e9ee 100644
> --- a/include/libcamera/control_ids.h.in
> +++ b/include/libcamera/control_ids.h.in
> @@ -2,7 +2,7 @@
>  /*
>   * Copyright (C) 2019, Google Inc.
>   *
> - * Control ID list
> + * {{mode|capitalize}} ID list
>   *
>   * This file is auto-generated. Do not edit.
>   */
> @@ -18,18 +18,42 @@
>  
>  namespace libcamera {
>  
> -namespace controls {
> +namespace {{mode}} {
> +
> +extern const ControlIdMap {{mode}};
> +
> +{%- for vendor, ctrls in controls -%}
> +
> +{% if vendor != 'libcamera' %}
> +namespace {{vendor}} {
> +
> +#define LIBCAMERA_HAS_{{vendor|upper}}_VENDOR_{{mode|upper}}
> +{%- endif %}
>  
>  enum {
> -${ids}
> +{%- for ctrl in ctrls %}
> +	{{ctrl.name|snake_case|upper}} = {{ctrl.id}},
> +{%- endfor %}
>  };
>  
> -${controls}
> +{% for ctrl in ctrls -%}
> +{% if ctrl.is_enum -%}
> +enum {{ctrl.name}}Enum {
> +{%- for enum in ctrl.enum_values %}
> +	{{enum.name}} = {{enum.value}},
> +{%- endfor %}
> +};
> +extern const std::array<const ControlValue, {{ctrl.enum_values_count}}> {{ctrl.name}}Values;
> +extern const std::map<std::string, {{ctrl.type}}> {{ctrl.name}}NameValueMap;
> +{% endif -%}
> +extern const Control<{{ctrl.type}}> {{ctrl.name}};
> +{% endfor -%}
>  
> -extern const ControlIdMap controls;
> +{% if vendor != 'libcamera' %}
> +} /* namespace {{vendor}} */
> +{% endif -%}
>  
> -${vendor_controls}
> -
> -} /* namespace controls */
> +{% endfor %}
> +} /* namespace {{mode}} */
>  
>  } /* namespace libcamera */
> diff --git a/include/libcamera/meson.build b/include/libcamera/meson.build
> index 87b9a9412fe7..d90a8615e52d 100644
> --- a/include/libcamera/meson.build
> +++ b/include/libcamera/meson.build
> @@ -80,7 +80,7 @@ foreach mode, entry : controls_map
>          properties_files_names += files_list
>      endif
>  
> -    template_file = files(outfile + '.in')
> +    template_file = files('control_ids.h.in')
>      ranges_file = files('../../src/libcamera/control_ranges.yaml')
>      control_headers += custom_target(header + '_h',
>                                       input : input_files,
> diff --git a/include/libcamera/property_ids.h.in b/include/libcamera/property_ids.h.in
> deleted file mode 100644
> index e6edabca771f..000000000000
> --- a/include/libcamera/property_ids.h.in
> +++ /dev/null
> @@ -1,34 +0,0 @@
> -/* SPDX-License-Identifier: LGPL-2.1-or-later */
> -/*
> - * Copyright (C) 2019, Google Inc.
> - *
> - * Property ID list
> - *
> - * This file is auto-generated. Do not edit.
> - */
> -
> -#pragma once
> -
> -#include <map>
> -#include <stdint.h>
> -#include <string>
> -
> -#include <libcamera/controls.h>
> -
> -namespace libcamera {
> -
> -namespace properties {
> -
> -enum {
> -${ids}
> -};
> -
> -${controls}
> -
> -extern const ControlIdMap properties;
> -
> -${vendor_controls}
> -
> -} /* namespace properties */
> -
> -} /* namespace libcamera */
> diff --git a/src/libcamera/control_ids.cpp.in b/src/libcamera/control_ids.cpp.in
> index 0b028c92d852..05c8fb385d20 100644
> --- a/src/libcamera/control_ids.cpp.in
> +++ b/src/libcamera/control_ids.cpp.in
> @@ -2,51 +2,120 @@
>  /*
>   * Copyright (C) 2019, Google Inc.
>   *
> - * Control ID list
> + * {{mode}} ID list
>   *
>   * This file is auto-generated. Do not edit.
>   */
>  
> -#include <libcamera/control_ids.h>
> +#include <libcamera/{{filename}}.h>
>  #include <libcamera/controls.h>
>  
>  /**
> - * \file control_ids.h
> - * \brief Camera control identifiers
> + * \file {{filename}}.h
> + * \brief Camera {{mode}} identifiers
>   */
>  
>  namespace libcamera {
>  
>  /**
> - * \brief Namespace for libcamera controls
> + * \brief Namespace for libcamera {{mode}}
>   */
> -namespace controls {
> +namespace {{mode}} {
>  
> -${controls_doc}
> +{%- for vendor, ctrls in controls -%}
>  
> -${vendor_controls_doc}
> +{%- if vendor != 'libcamera' %}
> +/**
> + * \brief Namespace for {{vendor}} {{mode}}
> + */
> +namespace {{vendor}} {
> +{%- endif -%}
> +
> +{% for ctrl in ctrls %}
> +
> +{% if ctrl.is_enum -%}
> +/**
> + * \enum {{ctrl.name}}Enum
> + * \brief Supported {{ctrl.name}} values
> +{%- for enum in ctrl.enum_values %}
> + *
> + * \var {{enum.name}}
> + * \brief {{enum.description|format_description}}
> +{%- endfor %}
> + */
> +
> +/**
> + * \var {{ctrl.name}}Values
> + * \brief List of all {{ctrl.name}} supported values
> + */
> +
> +/**
> + * \var {{ctrl.name}}NameValueMap
> + * \brief Map of all {{ctrl.name}} supported value names (in std::string format) to value
> + */
> +
> +{% endif -%}
> +/**
> + * \var {{ctrl.name}}
> + * \brief {{ctrl.description|format_description}}
> + */
> +{%- endfor %}
> +{% if vendor != 'libcamera' %}
> +} /* namespace {{vendor}} */
> +{% endif -%}
> +
> +{%- endfor %}
>  
>  #ifndef __DOXYGEN__
>  /*
> - * Keep the controls definitions hidden from doxygen as it incorrectly parses
> + * Keep the {{mode}} definitions hidden from doxygen as it incorrectly parses
>   * them as functions.
>   */
> -${controls_def}
> +{% for vendor, ctrls in controls -%}
>  
> -${vendor_controls_def}
> +{% if vendor != 'libcamera' %}
> +namespace {{vendor}} {
> +{% endif %}
>  
> -#endif
> +{%- for ctrl in ctrls %}
> +{% if ctrl.is_enum -%}
> +extern const std::array<const ControlValue, {{ctrl.enum_values_count}}> {{ctrl.name}}Values = {
> +{%- for enum in ctrl.enum_values %}
> +	static_cast<{{ctrl.type}}>({{enum.name}}),
> +{%- endfor %}
> +};
> +extern const std::map<std::string, {{ctrl.type}}> {{ctrl.name}}NameValueMap = {
> +{%- for enum in ctrl.enum_values %}
> +	{ "{{enum.name}}", {{enum.name}} },
> +{%- endfor %}
> +};
> +{% endif -%}
> +extern const Control<{{ctrl.type}}> {{ctrl.name}}({{ctrl.name|snake_case|upper}}, "{{ctrl.name}}");
> +{%- endfor %}
> +
> +{% if vendor != 'libcamera' %}
> +} /* namespace {{vendor}} */
> +{% endif -%}
> +
> +{%- endfor %}
> +#endif /* __DOXYGEN__ */
>  
>  /**
> - * \brief List of all supported libcamera controls
> + * \brief List of all supported libcamera {{mode}}
> +{%- if mode == 'controls' %}
>   *
>   * Unless otherwise stated, all controls are bi-directional, i.e. they can be
>   * set through Request::controls() and returned out through Request::metadata().
> +{%- endif %}
>   */
> -extern const ControlIdMap controls {
> -${controls_map}
> +extern const ControlIdMap {{mode}} {
> +{%- for vendor, ctrls in controls -%}
> +{%- for ctrl in ctrls %}
> +	{ {{ctrl.namespace}}{{ctrl.name|snake_case|upper}}, &{{ctrl.namespace}}{{ctrl.name}} },
> +{%- endfor -%}
> +{%- endfor %}
>  };
>  
> -} /* namespace controls */
> +} /* namespace {{mode}} */
>  
>  } /* namespace libcamera */
> diff --git a/src/libcamera/meson.build b/src/libcamera/meson.build
> index e5e959d9c7bd..3fd3a87e9f95 100644
> --- a/src/libcamera/meson.build
> +++ b/src/libcamera/meson.build
> @@ -143,9 +143,10 @@ foreach mode, inout_files : controls_mode_files
>      input_files = inout_files[0]
>      output_file = inout_files[1]
>  
> -    template_file = files(output_file + '.in')
> +    template_file = files('control_ids.cpp.in')
>      ranges_file = files('control_ranges.yaml')
> -    control_sources += custom_target(mode + '_cpp',
> +
> +    control_sources += custom_target(mode + '_ids_cpp',
>                                       input : input_files,
>                                       output : output_file,
>                                       command : [gen_controls, '-o', '@OUTPUT@',
> diff --git a/src/libcamera/property_ids.cpp.in b/src/libcamera/property_ids.cpp.in
> deleted file mode 100644
> index 2d3f192eb6ef..000000000000
> --- a/src/libcamera/property_ids.cpp.in
> +++ /dev/null
> @@ -1,48 +0,0 @@
> -/* SPDX-License-Identifier: LGPL-2.1-or-later */
> -/*
> - * Copyright (C) 2019, Google Inc.
> - *
> - * Property ID list
> - *
> - * This file is auto-generated. Do not edit.
> - */
> -
> -#include <libcamera/property_ids.h>
> -
> -/**
> - * \file property_ids.h
> - * \brief Camera property identifiers
> - */
> -
> -namespace libcamera {
> -
> -/**
> - * \brief Namespace for libcamera properties
> - */
> -namespace properties {
> -
> -${controls_doc}
> -
> -${vendor_controls_doc}
> -
> -#ifndef __DOXYGEN__
> -/*
> - * Keep the properties definitions hidden from doxygen as it incorrectly parses
> - * them as functions.
> - */
> -${controls_def}
> -
> -${vendor_controls_def}
> -
> -#endif
> -
> -/**
> - * \brief List of all supported libcamera properties
> - */
> -extern const ControlIdMap properties {
> -${controls_map}
> -};
> -
> -} /* namespace properties */
> -
> -} /* namespace libcamera */
> diff --git a/utils/codegen/gen-controls.py b/utils/codegen/gen-controls.py
> index 56315f5089b4..685ef7a00d5f 100755
> --- a/utils/codegen/gen-controls.py
> +++ b/utils/codegen/gen-controls.py
> @@ -7,12 +7,10 @@
>  # Generate control definitions from YAML
>  
>  import argparse
> -from functools import reduce
> -import operator
> -import string
> +import jinja2
> +import os
>  import sys
>  import yaml
> -import os
>  
>  
>  class ControlEnum(object):
> @@ -81,6 +79,13 @@ class Control(object):
>          for enum in self.__enum_values:
>              yield enum
>  
> +    @property
> +    def enum_values_count(self):
> +        """The number of enum values, if the control is an enumeration"""
> +        if self.__enum_values is None:
> +            return 0
> +        return len(self.__enum_values)
> +
>      @property
>      def is_enum(self):
>          """Is the control an enumeration"""
> @@ -119,221 +124,23 @@ def snake_case(s):
>  
>  def format_description(description):
>      description = description.strip('\n').split('\n')
> -    description[0] = '\\brief ' + description[0]
> -    return '\n'.join([(line and ' * ' or ' *') + line for line in description])
> +    for i in range(1, len(description)):
> +        line = description[i]
> +        description[i] = (line and ' * ' or ' *') + line
> +    return '\n'.join(description)
>  
>  
> -def generate_cpp(controls):
> -    enum_doc_start_template = string.Template('''/**
> - * \\enum ${name}Enum
> - * \\brief Supported ${name} values''')
> -    enum_doc_value_template = string.Template(''' * \\var ${value}
> -${description}''')
> -    doc_template = string.Template('''/**
> - * \\var ${name}
> -${description}
> - */''')
> -    def_template = string.Template('extern const Control<${type}> ${name}(${id_name}, "${name}");')
> -    enum_values_doc = string.Template('''/**
> - * \\var ${name}Values
> - * \\brief List of all $name supported values
> - */''')
> -    enum_values_start = string.Template('''extern const std::array<const ControlValue, ${size}> ${name}Values = {''')
> -    enum_values_values = string.Template('''\tstatic_cast<int32_t>(${name}),''')
> -    name_value_map_doc = string.Template('''/**
> - * \\var ${name}NameValueMap
> - * \\brief Map of all $name supported value names (in std::string format) to value
> - */''')
> -    name_value_map_start = string.Template('''extern const std::map<std::string, ${type}> ${name}NameValueMap = {''')
> -    name_value_values = string.Template('''\t{ "${name}", ${name} },''')
> +def extend_control(ctrl, id, ranges):
> +    ctrl.id = ranges[ctrl.vendor] + id + 1
>  
> -    ctrls_doc = {}
> -    ctrls_def = {}
> -    ctrls_map = []
> +    if ctrl.vendor != 'libcamera':
> +        ctrl.namespace = f'{ctrl.vendor}::'
> +    else:
> +        ctrl.namespace = ''
>  
> -    for ctrl in controls:
> -        id_name = snake_case(ctrl.name).upper()
> +    ctrl.documentation = format_description(ctrl.description)

This is unused.

Other than that, looks good to me. I'm impressed at how easy it was to
read (it still took a while though).

Reviewed-by: Paul Elder <paul.elder@ideasonboard.com>

>  
> -        vendor = ctrl.vendor
> -        if vendor not in ctrls_doc:
> -            ctrls_doc[vendor] = []
> -            ctrls_def[vendor] = []
> -
> -        info = {
> -            'name': ctrl.name,
> -            'type': ctrl.type,
> -            'description': format_description(ctrl.description),
> -            'id_name': id_name,
> -        }
> -
> -        target_doc = ctrls_doc[vendor]
> -        target_def = ctrls_def[vendor]
> -
> -        if ctrl.is_enum:
> -            enum_doc = []
> -            enum_doc.append(enum_doc_start_template.substitute(info))
> -
> -            num_entries = 0
> -            for enum in ctrl.enum_values:
> -                value_info = {
> -                    'name': ctrl.name,
> -                    'value': enum.name,
> -                    'description': format_description(enum.description),
> -                }
> -                enum_doc.append(enum_doc_value_template.substitute(value_info))
> -                num_entries += 1
> -
> -            enum_doc = '\n *\n'.join(enum_doc)
> -            enum_doc += '\n */'
> -            target_doc.append(enum_doc)
> -
> -            values_info = {
> -                'name': info['name'],
> -                'type': ctrl.type,
> -                'size': num_entries,
> -            }
> -            target_doc.append(enum_values_doc.substitute(values_info))
> -            target_def.append(enum_values_start.substitute(values_info))
> -            for enum in ctrl.enum_values:
> -                value_info = {
> -                    'name': enum.name
> -                }
> -                target_def.append(enum_values_values.substitute(value_info))
> -            target_def.append("};")
> -
> -            target_doc.append(name_value_map_doc.substitute(values_info))
> -            target_def.append(name_value_map_start.substitute(values_info))
> -            for enum in ctrl.enum_values:
> -                value_info = {
> -                    'name': enum.name
> -                }
> -                target_def.append(name_value_values.substitute(value_info))
> -            target_def.append("};")
> -
> -        target_doc.append(doc_template.substitute(info))
> -        target_def.append(def_template.substitute(info))
> -
> -        vendor_ns = vendor + '::' if vendor != "libcamera" else ''
> -        ctrls_map.append('\t{ ' + vendor_ns + id_name + ', &' + vendor_ns + ctrl.name + ' },')
> -
> -    vendor_ctrl_doc_sub = []
> -    vendor_ctrl_template = string.Template('''
> -/**
> - * \\brief Namespace for ${vendor} controls
> - */
> -namespace ${vendor} {
> -
> -${vendor_controls_str}
> -
> -} /* namespace ${vendor} */''')
> -
> -    for vendor in [v for v in ctrls_doc.keys() if v not in ['libcamera']]:
> -        vendor_ctrl_doc_sub.append(vendor_ctrl_template.substitute({'vendor': vendor, 'vendor_controls_str': '\n\n'.join(ctrls_doc[vendor])}))
> -
> -    vendor_ctrl_def_sub = []
> -    for vendor in [v for v in ctrls_def.keys() if v not in ['libcamera']]:
> -        vendor_ctrl_def_sub.append(vendor_ctrl_template.substitute({'vendor': vendor, 'vendor_controls_str': '\n'.join(ctrls_def[vendor])}))
> -
> -    return {
> -        'controls_doc': '\n\n'.join(ctrls_doc['libcamera']),
> -        'controls_def': '\n'.join(ctrls_def['libcamera']),
> -        'controls_map': '\n'.join(ctrls_map),
> -        'vendor_controls_doc': '\n'.join(vendor_ctrl_doc_sub),
> -        'vendor_controls_def': '\n'.join(vendor_ctrl_def_sub),
> -    }
> -
> -
> -def generate_h(controls, mode, ranges):
> -    enum_template_start = string.Template('''enum ${name}Enum {''')
> -    enum_value_template = string.Template('''\t${name} = ${value},''')
> -    enum_values_template = string.Template('''extern const std::array<const ControlValue, ${size}> ${name}Values;''')
> -    name_value_map_template = string.Template('''extern const std::map<std::string, ${type}> ${name}NameValueMap;''')
> -    template = string.Template('''extern const Control<${type}> ${name};''')
> -
> -    ctrls = {}
> -    ids = {}
> -    id_value = {}
> -
> -    for ctrl in controls:
> -        id_name = snake_case(ctrl.name).upper()
> -
> -        vendor = ctrl.vendor
> -        if vendor not in ctrls:
> -            if vendor not in ranges.keys():
> -                raise RuntimeError(f'Control id range is not defined for vendor {vendor}')
> -            id_value[vendor] = ranges[vendor] + 1
> -            ids[vendor] = []
> -            ctrls[vendor] = []
> -
> -        target_ids = ids[vendor]
> -        target_ids.append('\t' + id_name + ' = ' + str(id_value[vendor]) + ',')
> -
> -        info = {
> -            'name': ctrl.name,
> -            'type': ctrl.type,
> -        }
> -
> -        target_ctrls = ctrls[vendor]
> -
> -        if ctrl.is_enum:
> -            target_ctrls.append(enum_template_start.substitute(info))
> -
> -            num_entries = 0
> -            for enum in ctrl.enum_values:
> -                value_info = {
> -                    'name': enum.name,
> -                    'value': enum.value,
> -                }
> -                target_ctrls.append(enum_value_template.substitute(value_info))
> -                num_entries += 1
> -            target_ctrls.append("};")
> -
> -            values_info = {
> -                'name': info['name'],
> -                'type': ctrl.type,
> -                'size': num_entries,
> -            }
> -            target_ctrls.append(enum_values_template.substitute(values_info))
> -            target_ctrls.append(name_value_map_template.substitute(values_info))
> -
> -        target_ctrls.append(template.substitute(info))
> -        id_value[vendor] += 1
> -
> -    vendor_template = string.Template('''
> -namespace ${vendor} {
> -
> -#define LIBCAMERA_HAS_${vendor_def}_VENDOR_${mode}
> -
> -enum {
> -${vendor_enums}
> -};
> -
> -${vendor_controls}
> -
> -} /* namespace ${vendor} */
> -''')
> -
> -    vendor_sub = []
> -    for vendor in [v for v in ctrls.keys() if v != 'libcamera']:
> -        vendor_sub.append(vendor_template.substitute({'mode': mode.upper(),
> -                                                      'vendor': vendor,
> -                                                      'vendor_def': vendor.upper(),
> -                                                      'vendor_enums': '\n'.join(ids[vendor]),
> -                                                      'vendor_controls': '\n'.join(ctrls[vendor])}))
> -
> -    return {
> -        'ids': '\n'.join(ids['libcamera']),
> -        'controls': '\n'.join(ctrls['libcamera']),
> -        'vendor_controls': '\n'.join(vendor_sub)
> -    }
> -
> -
> -def fill_template(template, data):
> -
> -    template = open(template, 'rb').read()
> -    template = template.decode('utf-8')
> -    template = string.Template(template)
> -    return template.substitute(data)
> +    return ctrl
>  
>  
>  def main(argv):
> @@ -358,29 +165,47 @@ def main(argv):
>          data = open(args.ranges, 'rb').read()
>          ranges = yaml.safe_load(data)['ranges']
>  
> -    controls = []
> +    controls = {}
>      for input in args.input:
> -        with open(input, 'rb') as f:
> -            data = f.read()
> -            vendor = yaml.safe_load(data)['vendor']
> -            ctrls = yaml.safe_load(data)['controls']
> -            controls = controls + [Control(*ctrl.popitem(), vendor) for ctrl in ctrls]
> +        data = yaml.safe_load(open(input, 'rb').read())
>  
> -    if args.template.endswith('.cpp.in'):
> -        data = generate_cpp(controls)
> -    elif args.template.endswith('.h.in'):
> -        data = generate_h(controls, args.mode, ranges)
> -    else:
> -        raise RuntimeError('Unknown template type')
> +        vendor = data['vendor']
> +        if vendor not in ranges.keys():
> +            raise RuntimeError(f'Control id range is not defined for vendor {vendor}')
>  
> -    data = fill_template(args.template, data)
> +        ctrls = controls.setdefault(vendor, [])
> +
> +        for i, ctrl in enumerate(data['controls']):
> +            ctrl = Control(*ctrl.popitem(), vendor)
> +            ctrls.append(extend_control(ctrl, i, ranges))
> +
> +    # Sort the vendors by range numerical value
> +    controls = [[vendor, ctrls] for vendor, ctrls in controls.items()]
> +    controls.sort(key=lambda item: ranges[item[0]])
> +
> +    filename = {
> +        'controls': 'control_ids',
> +        'properties': 'property_ids',
> +    }[args.mode]
> +
> +    data = {
> +        'filename': filename,
> +        'mode': args.mode,
> +        'controls': controls,
> +    }
> +
> +    env = jinja2.Environment()
> +    env.filters['format_description'] = format_description
> +    env.filters['snake_case'] = snake_case
> +    template = env.from_string(open(args.template, 'r', encoding='utf-8').read())
> +    string = template.render(data)
>  
>      if args.output:
> -        output = open(args.output, 'wb')
> -        output.write(data.encode('utf-8'))
> +        output = open(args.output, 'w', encoding='utf-8')
> +        output.write(string)
>          output.close()
>      else:
> -        sys.stdout.write(data)
> +        sys.stdout.write(string)
>  
>      return 0
>
Paul Elder Aug. 15, 2024, 5:01 a.m. UTC | #2
On Fri, Aug 09, 2024 at 03:59:11AM +0300, Laurent Pinchart wrote:
> Jinja2 templates help separating the logic related to the template from

Oh I just remembered,

s/separating/separate/

> the generation of the data. The python code gets much clearer as a
> result.

s/gets/becomes/

Same for 10/10

Paul

<snip>

Patch
diff mbox series

diff --git a/include/libcamera/control_ids.h.in b/include/libcamera/control_ids.h.in
index 293ba966fbc4..858ef872e9ee 100644
--- a/include/libcamera/control_ids.h.in
+++ b/include/libcamera/control_ids.h.in
@@ -2,7 +2,7 @@ 
 /*
  * Copyright (C) 2019, Google Inc.
  *
- * Control ID list
+ * {{mode|capitalize}} ID list
  *
  * This file is auto-generated. Do not edit.
  */
@@ -18,18 +18,42 @@ 
 
 namespace libcamera {
 
-namespace controls {
+namespace {{mode}} {
+
+extern const ControlIdMap {{mode}};
+
+{%- for vendor, ctrls in controls -%}
+
+{% if vendor != 'libcamera' %}
+namespace {{vendor}} {
+
+#define LIBCAMERA_HAS_{{vendor|upper}}_VENDOR_{{mode|upper}}
+{%- endif %}
 
 enum {
-${ids}
+{%- for ctrl in ctrls %}
+	{{ctrl.name|snake_case|upper}} = {{ctrl.id}},
+{%- endfor %}
 };
 
-${controls}
+{% for ctrl in ctrls -%}
+{% if ctrl.is_enum -%}
+enum {{ctrl.name}}Enum {
+{%- for enum in ctrl.enum_values %}
+	{{enum.name}} = {{enum.value}},
+{%- endfor %}
+};
+extern const std::array<const ControlValue, {{ctrl.enum_values_count}}> {{ctrl.name}}Values;
+extern const std::map<std::string, {{ctrl.type}}> {{ctrl.name}}NameValueMap;
+{% endif -%}
+extern const Control<{{ctrl.type}}> {{ctrl.name}};
+{% endfor -%}
 
-extern const ControlIdMap controls;
+{% if vendor != 'libcamera' %}
+} /* namespace {{vendor}} */
+{% endif -%}
 
-${vendor_controls}
-
-} /* namespace controls */
+{% endfor %}
+} /* namespace {{mode}} */
 
 } /* namespace libcamera */
diff --git a/include/libcamera/meson.build b/include/libcamera/meson.build
index 87b9a9412fe7..d90a8615e52d 100644
--- a/include/libcamera/meson.build
+++ b/include/libcamera/meson.build
@@ -80,7 +80,7 @@  foreach mode, entry : controls_map
         properties_files_names += files_list
     endif
 
-    template_file = files(outfile + '.in')
+    template_file = files('control_ids.h.in')
     ranges_file = files('../../src/libcamera/control_ranges.yaml')
     control_headers += custom_target(header + '_h',
                                      input : input_files,
diff --git a/include/libcamera/property_ids.h.in b/include/libcamera/property_ids.h.in
deleted file mode 100644
index e6edabca771f..000000000000
--- a/include/libcamera/property_ids.h.in
+++ /dev/null
@@ -1,34 +0,0 @@ 
-/* SPDX-License-Identifier: LGPL-2.1-or-later */
-/*
- * Copyright (C) 2019, Google Inc.
- *
- * Property ID list
- *
- * This file is auto-generated. Do not edit.
- */
-
-#pragma once
-
-#include <map>
-#include <stdint.h>
-#include <string>
-
-#include <libcamera/controls.h>
-
-namespace libcamera {
-
-namespace properties {
-
-enum {
-${ids}
-};
-
-${controls}
-
-extern const ControlIdMap properties;
-
-${vendor_controls}
-
-} /* namespace properties */
-
-} /* namespace libcamera */
diff --git a/src/libcamera/control_ids.cpp.in b/src/libcamera/control_ids.cpp.in
index 0b028c92d852..05c8fb385d20 100644
--- a/src/libcamera/control_ids.cpp.in
+++ b/src/libcamera/control_ids.cpp.in
@@ -2,51 +2,120 @@ 
 /*
  * Copyright (C) 2019, Google Inc.
  *
- * Control ID list
+ * {{mode}} ID list
  *
  * This file is auto-generated. Do not edit.
  */
 
-#include <libcamera/control_ids.h>
+#include <libcamera/{{filename}}.h>
 #include <libcamera/controls.h>
 
 /**
- * \file control_ids.h
- * \brief Camera control identifiers
+ * \file {{filename}}.h
+ * \brief Camera {{mode}} identifiers
  */
 
 namespace libcamera {
 
 /**
- * \brief Namespace for libcamera controls
+ * \brief Namespace for libcamera {{mode}}
  */
-namespace controls {
+namespace {{mode}} {
 
-${controls_doc}
+{%- for vendor, ctrls in controls -%}
 
-${vendor_controls_doc}
+{%- if vendor != 'libcamera' %}
+/**
+ * \brief Namespace for {{vendor}} {{mode}}
+ */
+namespace {{vendor}} {
+{%- endif -%}
+
+{% for ctrl in ctrls %}
+
+{% if ctrl.is_enum -%}
+/**
+ * \enum {{ctrl.name}}Enum
+ * \brief Supported {{ctrl.name}} values
+{%- for enum in ctrl.enum_values %}
+ *
+ * \var {{enum.name}}
+ * \brief {{enum.description|format_description}}
+{%- endfor %}
+ */
+
+/**
+ * \var {{ctrl.name}}Values
+ * \brief List of all {{ctrl.name}} supported values
+ */
+
+/**
+ * \var {{ctrl.name}}NameValueMap
+ * \brief Map of all {{ctrl.name}} supported value names (in std::string format) to value
+ */
+
+{% endif -%}
+/**
+ * \var {{ctrl.name}}
+ * \brief {{ctrl.description|format_description}}
+ */
+{%- endfor %}
+{% if vendor != 'libcamera' %}
+} /* namespace {{vendor}} */
+{% endif -%}
+
+{%- endfor %}
 
 #ifndef __DOXYGEN__
 /*
- * Keep the controls definitions hidden from doxygen as it incorrectly parses
+ * Keep the {{mode}} definitions hidden from doxygen as it incorrectly parses
  * them as functions.
  */
-${controls_def}
+{% for vendor, ctrls in controls -%}
 
-${vendor_controls_def}
+{% if vendor != 'libcamera' %}
+namespace {{vendor}} {
+{% endif %}
 
-#endif
+{%- for ctrl in ctrls %}
+{% if ctrl.is_enum -%}
+extern const std::array<const ControlValue, {{ctrl.enum_values_count}}> {{ctrl.name}}Values = {
+{%- for enum in ctrl.enum_values %}
+	static_cast<{{ctrl.type}}>({{enum.name}}),
+{%- endfor %}
+};
+extern const std::map<std::string, {{ctrl.type}}> {{ctrl.name}}NameValueMap = {
+{%- for enum in ctrl.enum_values %}
+	{ "{{enum.name}}", {{enum.name}} },
+{%- endfor %}
+};
+{% endif -%}
+extern const Control<{{ctrl.type}}> {{ctrl.name}}({{ctrl.name|snake_case|upper}}, "{{ctrl.name}}");
+{%- endfor %}
+
+{% if vendor != 'libcamera' %}
+} /* namespace {{vendor}} */
+{% endif -%}
+
+{%- endfor %}
+#endif /* __DOXYGEN__ */
 
 /**
- * \brief List of all supported libcamera controls
+ * \brief List of all supported libcamera {{mode}}
+{%- if mode == 'controls' %}
  *
  * Unless otherwise stated, all controls are bi-directional, i.e. they can be
  * set through Request::controls() and returned out through Request::metadata().
+{%- endif %}
  */
-extern const ControlIdMap controls {
-${controls_map}
+extern const ControlIdMap {{mode}} {
+{%- for vendor, ctrls in controls -%}
+{%- for ctrl in ctrls %}
+	{ {{ctrl.namespace}}{{ctrl.name|snake_case|upper}}, &{{ctrl.namespace}}{{ctrl.name}} },
+{%- endfor -%}
+{%- endfor %}
 };
 
-} /* namespace controls */
+} /* namespace {{mode}} */
 
 } /* namespace libcamera */
diff --git a/src/libcamera/meson.build b/src/libcamera/meson.build
index e5e959d9c7bd..3fd3a87e9f95 100644
--- a/src/libcamera/meson.build
+++ b/src/libcamera/meson.build
@@ -143,9 +143,10 @@  foreach mode, inout_files : controls_mode_files
     input_files = inout_files[0]
     output_file = inout_files[1]
 
-    template_file = files(output_file + '.in')
+    template_file = files('control_ids.cpp.in')
     ranges_file = files('control_ranges.yaml')
-    control_sources += custom_target(mode + '_cpp',
+
+    control_sources += custom_target(mode + '_ids_cpp',
                                      input : input_files,
                                      output : output_file,
                                      command : [gen_controls, '-o', '@OUTPUT@',
diff --git a/src/libcamera/property_ids.cpp.in b/src/libcamera/property_ids.cpp.in
deleted file mode 100644
index 2d3f192eb6ef..000000000000
--- a/src/libcamera/property_ids.cpp.in
+++ /dev/null
@@ -1,48 +0,0 @@ 
-/* SPDX-License-Identifier: LGPL-2.1-or-later */
-/*
- * Copyright (C) 2019, Google Inc.
- *
- * Property ID list
- *
- * This file is auto-generated. Do not edit.
- */
-
-#include <libcamera/property_ids.h>
-
-/**
- * \file property_ids.h
- * \brief Camera property identifiers
- */
-
-namespace libcamera {
-
-/**
- * \brief Namespace for libcamera properties
- */
-namespace properties {
-
-${controls_doc}
-
-${vendor_controls_doc}
-
-#ifndef __DOXYGEN__
-/*
- * Keep the properties definitions hidden from doxygen as it incorrectly parses
- * them as functions.
- */
-${controls_def}
-
-${vendor_controls_def}
-
-#endif
-
-/**
- * \brief List of all supported libcamera properties
- */
-extern const ControlIdMap properties {
-${controls_map}
-};
-
-} /* namespace properties */
-
-} /* namespace libcamera */
diff --git a/utils/codegen/gen-controls.py b/utils/codegen/gen-controls.py
index 56315f5089b4..685ef7a00d5f 100755
--- a/utils/codegen/gen-controls.py
+++ b/utils/codegen/gen-controls.py
@@ -7,12 +7,10 @@ 
 # Generate control definitions from YAML
 
 import argparse
-from functools import reduce
-import operator
-import string
+import jinja2
+import os
 import sys
 import yaml
-import os
 
 
 class ControlEnum(object):
@@ -81,6 +79,13 @@  class Control(object):
         for enum in self.__enum_values:
             yield enum
 
+    @property
+    def enum_values_count(self):
+        """The number of enum values, if the control is an enumeration"""
+        if self.__enum_values is None:
+            return 0
+        return len(self.__enum_values)
+
     @property
     def is_enum(self):
         """Is the control an enumeration"""
@@ -119,221 +124,23 @@  def snake_case(s):
 
 def format_description(description):
     description = description.strip('\n').split('\n')
-    description[0] = '\\brief ' + description[0]
-    return '\n'.join([(line and ' * ' or ' *') + line for line in description])
+    for i in range(1, len(description)):
+        line = description[i]
+        description[i] = (line and ' * ' or ' *') + line
+    return '\n'.join(description)
 
 
-def generate_cpp(controls):
-    enum_doc_start_template = string.Template('''/**
- * \\enum ${name}Enum
- * \\brief Supported ${name} values''')
-    enum_doc_value_template = string.Template(''' * \\var ${value}
-${description}''')
-    doc_template = string.Template('''/**
- * \\var ${name}
-${description}
- */''')
-    def_template = string.Template('extern const Control<${type}> ${name}(${id_name}, "${name}");')
-    enum_values_doc = string.Template('''/**
- * \\var ${name}Values
- * \\brief List of all $name supported values
- */''')
-    enum_values_start = string.Template('''extern const std::array<const ControlValue, ${size}> ${name}Values = {''')
-    enum_values_values = string.Template('''\tstatic_cast<int32_t>(${name}),''')
-    name_value_map_doc = string.Template('''/**
- * \\var ${name}NameValueMap
- * \\brief Map of all $name supported value names (in std::string format) to value
- */''')
-    name_value_map_start = string.Template('''extern const std::map<std::string, ${type}> ${name}NameValueMap = {''')
-    name_value_values = string.Template('''\t{ "${name}", ${name} },''')
+def extend_control(ctrl, id, ranges):
+    ctrl.id = ranges[ctrl.vendor] + id + 1
 
-    ctrls_doc = {}
-    ctrls_def = {}
-    ctrls_map = []
+    if ctrl.vendor != 'libcamera':
+        ctrl.namespace = f'{ctrl.vendor}::'
+    else:
+        ctrl.namespace = ''
 
-    for ctrl in controls:
-        id_name = snake_case(ctrl.name).upper()
+    ctrl.documentation = format_description(ctrl.description)
 
-        vendor = ctrl.vendor
-        if vendor not in ctrls_doc:
-            ctrls_doc[vendor] = []
-            ctrls_def[vendor] = []
-
-        info = {
-            'name': ctrl.name,
-            'type': ctrl.type,
-            'description': format_description(ctrl.description),
-            'id_name': id_name,
-        }
-
-        target_doc = ctrls_doc[vendor]
-        target_def = ctrls_def[vendor]
-
-        if ctrl.is_enum:
-            enum_doc = []
-            enum_doc.append(enum_doc_start_template.substitute(info))
-
-            num_entries = 0
-            for enum in ctrl.enum_values:
-                value_info = {
-                    'name': ctrl.name,
-                    'value': enum.name,
-                    'description': format_description(enum.description),
-                }
-                enum_doc.append(enum_doc_value_template.substitute(value_info))
-                num_entries += 1
-
-            enum_doc = '\n *\n'.join(enum_doc)
-            enum_doc += '\n */'
-            target_doc.append(enum_doc)
-
-            values_info = {
-                'name': info['name'],
-                'type': ctrl.type,
-                'size': num_entries,
-            }
-            target_doc.append(enum_values_doc.substitute(values_info))
-            target_def.append(enum_values_start.substitute(values_info))
-            for enum in ctrl.enum_values:
-                value_info = {
-                    'name': enum.name
-                }
-                target_def.append(enum_values_values.substitute(value_info))
-            target_def.append("};")
-
-            target_doc.append(name_value_map_doc.substitute(values_info))
-            target_def.append(name_value_map_start.substitute(values_info))
-            for enum in ctrl.enum_values:
-                value_info = {
-                    'name': enum.name
-                }
-                target_def.append(name_value_values.substitute(value_info))
-            target_def.append("};")
-
-        target_doc.append(doc_template.substitute(info))
-        target_def.append(def_template.substitute(info))
-
-        vendor_ns = vendor + '::' if vendor != "libcamera" else ''
-        ctrls_map.append('\t{ ' + vendor_ns + id_name + ', &' + vendor_ns + ctrl.name + ' },')
-
-    vendor_ctrl_doc_sub = []
-    vendor_ctrl_template = string.Template('''
-/**
- * \\brief Namespace for ${vendor} controls
- */
-namespace ${vendor} {
-
-${vendor_controls_str}
-
-} /* namespace ${vendor} */''')
-
-    for vendor in [v for v in ctrls_doc.keys() if v not in ['libcamera']]:
-        vendor_ctrl_doc_sub.append(vendor_ctrl_template.substitute({'vendor': vendor, 'vendor_controls_str': '\n\n'.join(ctrls_doc[vendor])}))
-
-    vendor_ctrl_def_sub = []
-    for vendor in [v for v in ctrls_def.keys() if v not in ['libcamera']]:
-        vendor_ctrl_def_sub.append(vendor_ctrl_template.substitute({'vendor': vendor, 'vendor_controls_str': '\n'.join(ctrls_def[vendor])}))
-
-    return {
-        'controls_doc': '\n\n'.join(ctrls_doc['libcamera']),
-        'controls_def': '\n'.join(ctrls_def['libcamera']),
-        'controls_map': '\n'.join(ctrls_map),
-        'vendor_controls_doc': '\n'.join(vendor_ctrl_doc_sub),
-        'vendor_controls_def': '\n'.join(vendor_ctrl_def_sub),
-    }
-
-
-def generate_h(controls, mode, ranges):
-    enum_template_start = string.Template('''enum ${name}Enum {''')
-    enum_value_template = string.Template('''\t${name} = ${value},''')
-    enum_values_template = string.Template('''extern const std::array<const ControlValue, ${size}> ${name}Values;''')
-    name_value_map_template = string.Template('''extern const std::map<std::string, ${type}> ${name}NameValueMap;''')
-    template = string.Template('''extern const Control<${type}> ${name};''')
-
-    ctrls = {}
-    ids = {}
-    id_value = {}
-
-    for ctrl in controls:
-        id_name = snake_case(ctrl.name).upper()
-
-        vendor = ctrl.vendor
-        if vendor not in ctrls:
-            if vendor not in ranges.keys():
-                raise RuntimeError(f'Control id range is not defined for vendor {vendor}')
-            id_value[vendor] = ranges[vendor] + 1
-            ids[vendor] = []
-            ctrls[vendor] = []
-
-        target_ids = ids[vendor]
-        target_ids.append('\t' + id_name + ' = ' + str(id_value[vendor]) + ',')
-
-        info = {
-            'name': ctrl.name,
-            'type': ctrl.type,
-        }
-
-        target_ctrls = ctrls[vendor]
-
-        if ctrl.is_enum:
-            target_ctrls.append(enum_template_start.substitute(info))
-
-            num_entries = 0
-            for enum in ctrl.enum_values:
-                value_info = {
-                    'name': enum.name,
-                    'value': enum.value,
-                }
-                target_ctrls.append(enum_value_template.substitute(value_info))
-                num_entries += 1
-            target_ctrls.append("};")
-
-            values_info = {
-                'name': info['name'],
-                'type': ctrl.type,
-                'size': num_entries,
-            }
-            target_ctrls.append(enum_values_template.substitute(values_info))
-            target_ctrls.append(name_value_map_template.substitute(values_info))
-
-        target_ctrls.append(template.substitute(info))
-        id_value[vendor] += 1
-
-    vendor_template = string.Template('''
-namespace ${vendor} {
-
-#define LIBCAMERA_HAS_${vendor_def}_VENDOR_${mode}
-
-enum {
-${vendor_enums}
-};
-
-${vendor_controls}
-
-} /* namespace ${vendor} */
-''')
-
-    vendor_sub = []
-    for vendor in [v for v in ctrls.keys() if v != 'libcamera']:
-        vendor_sub.append(vendor_template.substitute({'mode': mode.upper(),
-                                                      'vendor': vendor,
-                                                      'vendor_def': vendor.upper(),
-                                                      'vendor_enums': '\n'.join(ids[vendor]),
-                                                      'vendor_controls': '\n'.join(ctrls[vendor])}))
-
-    return {
-        'ids': '\n'.join(ids['libcamera']),
-        'controls': '\n'.join(ctrls['libcamera']),
-        'vendor_controls': '\n'.join(vendor_sub)
-    }
-
-
-def fill_template(template, data):
-
-    template = open(template, 'rb').read()
-    template = template.decode('utf-8')
-    template = string.Template(template)
-    return template.substitute(data)
+    return ctrl
 
 
 def main(argv):
@@ -358,29 +165,47 @@  def main(argv):
         data = open(args.ranges, 'rb').read()
         ranges = yaml.safe_load(data)['ranges']
 
-    controls = []
+    controls = {}
     for input in args.input:
-        with open(input, 'rb') as f:
-            data = f.read()
-            vendor = yaml.safe_load(data)['vendor']
-            ctrls = yaml.safe_load(data)['controls']
-            controls = controls + [Control(*ctrl.popitem(), vendor) for ctrl in ctrls]
+        data = yaml.safe_load(open(input, 'rb').read())
 
-    if args.template.endswith('.cpp.in'):
-        data = generate_cpp(controls)
-    elif args.template.endswith('.h.in'):
-        data = generate_h(controls, args.mode, ranges)
-    else:
-        raise RuntimeError('Unknown template type')
+        vendor = data['vendor']
+        if vendor not in ranges.keys():
+            raise RuntimeError(f'Control id range is not defined for vendor {vendor}')
 
-    data = fill_template(args.template, data)
+        ctrls = controls.setdefault(vendor, [])
+
+        for i, ctrl in enumerate(data['controls']):
+            ctrl = Control(*ctrl.popitem(), vendor)
+            ctrls.append(extend_control(ctrl, i, ranges))
+
+    # Sort the vendors by range numerical value
+    controls = [[vendor, ctrls] for vendor, ctrls in controls.items()]
+    controls.sort(key=lambda item: ranges[item[0]])
+
+    filename = {
+        'controls': 'control_ids',
+        'properties': 'property_ids',
+    }[args.mode]
+
+    data = {
+        'filename': filename,
+        'mode': args.mode,
+        'controls': controls,
+    }
+
+    env = jinja2.Environment()
+    env.filters['format_description'] = format_description
+    env.filters['snake_case'] = snake_case
+    template = env.from_string(open(args.template, 'r', encoding='utf-8').read())
+    string = template.render(data)
 
     if args.output:
-        output = open(args.output, 'wb')
-        output.write(data.encode('utf-8'))
+        output = open(args.output, 'w', encoding='utf-8')
+        output.write(string)
         output.close()
     else:
-        sys.stdout.write(data)
+        sys.stdout.write(string)
 
     return 0