[v3,3/7] utils: Add script to generate control_ids_debug.yaml

21541 diff mbox series

For flexible debugging it is helpful to minimize the roundtrip time.
This script parses the source tree and looks for usages of

set<type>(controls::debug::Something,

and adds (or removes) the controls as necessary from the yaml
description. It is meant to be used during development to ease the
creation of the correct yaml entries.

Signed-off-by: Stefan Klug <stefan.klug@ideasonboard.com>

---
Changes in v3:
- Remove superfluous comma and args variable
- Default to ruamel.yaml instead of ruyaml
- Small fixes on layout/comments

Changes in v2:
- Search only until the first comma of the set() call, to allow
  linebreaks there.
- Support ruamel.yaml as fallback
- Rename output to ctrl_file
- Add "generated by" comment in yaml file
---
 utils/gen-debug-controls.py | 162 ++++++++++++++++++++++++++++++++++++
 1 file changed, 162 insertions(+)
 create mode 100755 utils/gen-debug-controls.py

Quoting Stefan Klug (2024-10-08 16:29:41)
> For flexible debugging it is helpful to minimize the roundtrip time.
> This script parses the source tree and looks for usages of
> 
> set<type>(controls::debug::Something,
> 
> and adds (or removes) the controls as necessary from the yaml
> description. It is meant to be used during development to ease the
> creation of the correct yaml entries.
 

Should we include this in the build steps for debug builds? or would
automating this be problematic? I guess that's where it would then
introduce the ruyaml as a build time dependency which perhaps we don't
want yet.

As this is a new script to assist development, it can't introduce
regressions, and can be worked on on top. So the only thing I care
specifically about is that the formatter is clean :-)


Reviewed-by: Kieran Bingham <kieran.bingham@ideasonboard.com>


> Signed-off-by: Stefan Klug <stefan.klug@ideasonboard.com>
> 
> ---
> Changes in v3:
> - Remove superfluous comma and args variable
> - Default to ruamel.yaml instead of ruyaml
> - Small fixes on layout/comments
> 
> Changes in v2:
> - Search only until the first comma of the set() call, to allow
>   linebreaks there.
> - Support ruamel.yaml as fallback
> - Rename output to ctrl_file
> - Add "generated by" comment in yaml file
> ---
>  utils/gen-debug-controls.py | 162 ++++++++++++++++++++++++++++++++++++
>  1 file changed, 162 insertions(+)
>  create mode 100755 utils/gen-debug-controls.py
> 
> diff --git a/utils/gen-debug-controls.py b/utils/gen-debug-controls.py
> new file mode 100755
> index 000000000000..025850731c0b
> --- /dev/null
> +++ b/utils/gen-debug-controls.py
> @@ -0,0 +1,162 @@
> +#!/usr/bin/env python3
> +# SPDX-License-Identifier: GPL-2.0-or-later
> +# Copyright (C) 2024, Google Inc.
> +#
> +# Author: Stefan Klug <stefan.klug@ideasonboard.com>
> +#
> +# This script looks for occurrences of the debug metadata controls in the source
> +# tree and updates src/libcamera/control_ids_debug.yaml accordingly. It is meant
> +# to be used during development to ease updating of the yaml file while
> +# debugging.
> +
> +import argparse
> +import logging
> +import os
> +import re
> +import sys
> +from dataclasses import dataclass
> +from pathlib import Path
> +
> +logger = logging.getLogger(__name__)
> +logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s')
> +
> +try:
> +    import ruamel.yaml as ruyaml
> +except:
> +    logger.error(
> +        f'Failed to import ruamel.yaml. Please install the ruamel.yaml package.')
> +    sys.exit(1)
> +
> +@dataclass
> +class FoundMatch:
> +    file: os.PathLike
> +    whole_match: str
> +    line: int
> +    type: str
> +    name: str
> +    size: str = None
> +
> +
> +def get_control_name(control):
> +    k = list(control.keys())
> +    if len(k) != 1:
> +        raise Exception(f"Can't handle control entry with {len(k)} keys")
> +    return k[0]
> +
> +
> +def find_debug_controls(dir):
> +    extensions = ['.cpp', '.h']
> +    files = [p for p in dir.rglob('*') if p.suffix in extensions]
> +
> +    # The following regex was tested on
> +    # set<Span<type>>( controls::debug::something , static_cast<type>(var) )
> +    # set<>( controls::debug::something , static_cast<type>(var) )
> +    # set( controls::debug::something , static_cast<type> (var) )
> +    exp = re.compile(r'set'  # set function
> +                     r'(?:\<((?:[^)(])*)\>)?'  # followed by a optional template param
> +                     r'\(\s*controls::debug::(\w+)\s*,'  # referencing a debug control
> +                     )
> +    matches = []
> +    for p in files:
> +        with p.open('r') as f:
> +            for idx, line in enumerate(f):
> +                match = exp.search(line)
> +                if match:
> +                    m = FoundMatch(file=p, line=idx, type=match.group(1),
> +                                   name=match.group(2), whole_match=match.group(0))
> +                    if m.type is not None and m.type.startswith('Span'):
> +                        # Simple span type detection treating the last word
> +                        # inside <> as type.
> +                        r = re.match(r'Span<(?:.*\s+)(.*)>', m.type)
> +                        m.type = r.group(1)
> +                        m.size = '[n]'
> +                    matches.append(m)
> +    return matches
> +
> +
> +def main(argv):
> +    parser = argparse.ArgumentParser(
> +        description='Automatically updates control_ids_debug.yaml')
> +    parser.parse_args(argv[1:])
> +
> +    yaml = ruyaml.YAML()
> +    root_dir = Path(__file__).resolve().parent.parent
> +    ctrl_file = root_dir.joinpath('src/libcamera/control_ids_debug.yaml')
> +
> +    matches = find_debug_controls(root_dir.joinpath('src'))
> +
> +    doc = yaml.load(ctrl_file)
> +
> +    controls = doc['controls']
> +
> +    # Create a map of names in the existing yaml for easier updating.
> +    controls_map = {}
> +    for control in controls:
> +        for k, v in control.items():
> +            controls_map[k] = v
> +
> +    obsolete_names = list(controls_map.keys())
> +
> +    for m in matches:
> +        if not m.type:
> +            p = m.file.relative_to(Path.cwd(), walk_up=True)
> +            logger.warning(
> +                f'{p}:{m.line + 1}: Failed to deduce type from {m.whole_match} ... skipping')
> +            continue
> +
> +        p = m.file.relative_to(root_dir)
> +        desc = {'type': m.type,
> +                'description': f'Debug control {m.name} found in {p}:{m.line}'}
> +        if m.size is not None:
> +            desc['size'] = m.size
> +
> +        if m.name in controls_map:
> +            # Can't use == for modified check because of the special yaml dicts.
> +            update_needed = False
> +            if list(controls_map[m.name].keys()) != list(desc.keys()):
> +                update_needed = True
> +            else:
> +                for k, v in controls_map[m.name].items():
> +                    if v != desc[k]:
> +                        update_needed = True
> +                        break
> +
> +            if update_needed:
> +                logger.info(f"Update control '{m.name}'")
> +                controls_map[m.name].clear()
> +                controls_map[m.name].update(desc)
> +
> +            obsolete_names.remove(m.name)
> +        else:
> +            logger.info(f"Add control '{m.name}'")
> +            insert_before = len(controls)
> +            for idx, control in enumerate(controls):
> +                if get_control_name(control).lower() > m.name.lower():
> +                    insert_before = idx
> +                    break
> +            controls.insert(insert_before, {m.name: desc})
> +
> +    # Remove elements from controls without recreating the list (to keep
> +    # comments etc.).
> +    idx = 0
> +    while idx < len(controls):
> +        name = get_control_name(controls[idx])
> +        if name in obsolete_names:
> +            logger.info(f"Remove control '{name}'")
> +            controls.pop(idx)
> +        else:
> +            idx += 1
> +
> +    with ctrl_file.open('w') as f:
> +        # Ruyaml looses the header.
> +        f.write(("# SPDX-License-Identifier: LGPL-2.1-or-later\n"
> +                 "#\n"
> +                 "# This file was generated by utils/gen-debug-controls.py\n"
> +                 "#\n"))
> +        yaml.dump(doc, f)
> +
> +    return 0
> +
> +
> +if __name__ == '__main__':
> +    sys.exit(main(sys.argv))
> -- 
> 2.43.0
>

On Thu, Oct 10, 2024 at 09:27:43AM +0100, Kieran Bingham wrote:
> Quoting Stefan Klug (2024-10-08 16:29:41)
> > For flexible debugging it is helpful to minimize the roundtrip time.
> > This script parses the source tree and looks for usages of
> > 
> > set<type>(controls::debug::Something,
> > 
> > and adds (or removes) the controls as necessary from the yaml
> > description. It is meant to be used during development to ease the
> > creation of the correct yaml entries.
> 
> Should we include this in the build steps for debug builds? or would
> automating this be problematic? I guess that's where it would then
> introduce the ruyaml as a build time dependency which perhaps we don't
> want yet.

It would create a chicken-and-egg issue. The yaml file is needed to
compile libcamera, the script generates it by parsing IPA source files,
and IPA modules need to link against libcamera. That's why we decide to
make it a developer helper script for the time being.

> As this is a new script to assist development, it can't introduce
> regressions, and can be worked on on top. So the only thing I care
> specifically about is that the formatter is clean :-)
> 
> Reviewed-by: Kieran Bingham <kieran.bingham@ideasonboard.com>
> 
> > Signed-off-by: Stefan Klug <stefan.klug@ideasonboard.com>

Reviewed-by: Laurent Pinchart <laurent.pinchart@ideasonboard.com>

> > 
> > ---
> > Changes in v3:
> > - Remove superfluous comma and args variable
> > - Default to ruamel.yaml instead of ruyaml
> > - Small fixes on layout/comments
> > 
> > Changes in v2:
> > - Search only until the first comma of the set() call, to allow
> >   linebreaks there.
> > - Support ruamel.yaml as fallback
> > - Rename output to ctrl_file
> > - Add "generated by" comment in yaml file
> > ---
> >  utils/gen-debug-controls.py | 162 ++++++++++++++++++++++++++++++++++++
> >  1 file changed, 162 insertions(+)
> >  create mode 100755 utils/gen-debug-controls.py
> > 
> > diff --git a/utils/gen-debug-controls.py b/utils/gen-debug-controls.py
> > new file mode 100755
> > index 000000000000..025850731c0b
> > --- /dev/null
> > +++ b/utils/gen-debug-controls.py
> > @@ -0,0 +1,162 @@
> > +#!/usr/bin/env python3
> > +# SPDX-License-Identifier: GPL-2.0-or-later
> > +# Copyright (C) 2024, Google Inc.
> > +#
> > +# Author: Stefan Klug <stefan.klug@ideasonboard.com>
> > +#
> > +# This script looks for occurrences of the debug metadata controls in the source
> > +# tree and updates src/libcamera/control_ids_debug.yaml accordingly. It is meant
> > +# to be used during development to ease updating of the yaml file while
> > +# debugging.
> > +
> > +import argparse
> > +import logging
> > +import os
> > +import re
> > +import sys
> > +from dataclasses import dataclass
> > +from pathlib import Path
> > +
> > +logger = logging.getLogger(__name__)
> > +logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s')
> > +
> > +try:
> > +    import ruamel.yaml as ruyaml
> > +except:
> > +    logger.error(
> > +        f'Failed to import ruamel.yaml. Please install the ruamel.yaml package.')
> > +    sys.exit(1)
> > +
> > +@dataclass
> > +class FoundMatch:
> > +    file: os.PathLike
> > +    whole_match: str
> > +    line: int
> > +    type: str
> > +    name: str
> > +    size: str = None
> > +
> > +
> > +def get_control_name(control):
> > +    k = list(control.keys())
> > +    if len(k) != 1:
> > +        raise Exception(f"Can't handle control entry with {len(k)} keys")
> > +    return k[0]
> > +
> > +
> > +def find_debug_controls(dir):
> > +    extensions = ['.cpp', '.h']
> > +    files = [p for p in dir.rglob('*') if p.suffix in extensions]
> > +
> > +    # The following regex was tested on
> > +    # set<Span<type>>( controls::debug::something , static_cast<type>(var) )
> > +    # set<>( controls::debug::something , static_cast<type>(var) )
> > +    # set( controls::debug::something , static_cast<type> (var) )
> > +    exp = re.compile(r'set'  # set function
> > +                     r'(?:\<((?:[^)(])*)\>)?'  # followed by a optional template param
> > +                     r'\(\s*controls::debug::(\w+)\s*,'  # referencing a debug control
> > +                     )
> > +    matches = []
> > +    for p in files:
> > +        with p.open('r') as f:
> > +            for idx, line in enumerate(f):
> > +                match = exp.search(line)
> > +                if match:
> > +                    m = FoundMatch(file=p, line=idx, type=match.group(1),
> > +                                   name=match.group(2), whole_match=match.group(0))
> > +                    if m.type is not None and m.type.startswith('Span'):
> > +                        # Simple span type detection treating the last word
> > +                        # inside <> as type.
> > +                        r = re.match(r'Span<(?:.*\s+)(.*)>', m.type)
> > +                        m.type = r.group(1)
> > +                        m.size = '[n]'
> > +                    matches.append(m)
> > +    return matches
> > +
> > +
> > +def main(argv):
> > +    parser = argparse.ArgumentParser(
> > +        description='Automatically updates control_ids_debug.yaml')
> > +    parser.parse_args(argv[1:])
> > +
> > +    yaml = ruyaml.YAML()
> > +    root_dir = Path(__file__).resolve().parent.parent
> > +    ctrl_file = root_dir.joinpath('src/libcamera/control_ids_debug.yaml')
> > +
> > +    matches = find_debug_controls(root_dir.joinpath('src'))
> > +
> > +    doc = yaml.load(ctrl_file)
> > +
> > +    controls = doc['controls']
> > +
> > +    # Create a map of names in the existing yaml for easier updating.
> > +    controls_map = {}
> > +    for control in controls:
> > +        for k, v in control.items():
> > +            controls_map[k] = v
> > +
> > +    obsolete_names = list(controls_map.keys())
> > +
> > +    for m in matches:
> > +        if not m.type:
> > +            p = m.file.relative_to(Path.cwd(), walk_up=True)
> > +            logger.warning(
> > +                f'{p}:{m.line + 1}: Failed to deduce type from {m.whole_match} ... skipping')
> > +            continue
> > +
> > +        p = m.file.relative_to(root_dir)
> > +        desc = {'type': m.type,
> > +                'description': f'Debug control {m.name} found in {p}:{m.line}'}
> > +        if m.size is not None:
> > +            desc['size'] = m.size
> > +
> > +        if m.name in controls_map:
> > +            # Can't use == for modified check because of the special yaml dicts.
> > +            update_needed = False
> > +            if list(controls_map[m.name].keys()) != list(desc.keys()):
> > +                update_needed = True
> > +            else:
> > +                for k, v in controls_map[m.name].items():
> > +                    if v != desc[k]:
> > +                        update_needed = True
> > +                        break
> > +
> > +            if update_needed:
> > +                logger.info(f"Update control '{m.name}'")
> > +                controls_map[m.name].clear()
> > +                controls_map[m.name].update(desc)
> > +
> > +            obsolete_names.remove(m.name)
> > +        else:
> > +            logger.info(f"Add control '{m.name}'")
> > +            insert_before = len(controls)
> > +            for idx, control in enumerate(controls):
> > +                if get_control_name(control).lower() > m.name.lower():
> > +                    insert_before = idx
> > +                    break
> > +            controls.insert(insert_before, {m.name: desc})
> > +
> > +    # Remove elements from controls without recreating the list (to keep
> > +    # comments etc.).
> > +    idx = 0
> > +    while idx < len(controls):
> > +        name = get_control_name(controls[idx])
> > +        if name in obsolete_names:
> > +            logger.info(f"Remove control '{name}'")
> > +            controls.pop(idx)
> > +        else:
> > +            idx += 1
> > +
> > +    with ctrl_file.open('w') as f:
> > +        # Ruyaml looses the header.
> > +        f.write(("# SPDX-License-Identifier: LGPL-2.1-or-later\n"
> > +                 "#\n"
> > +                 "# This file was generated by utils/gen-debug-controls.py\n"
> > +                 "#\n"))
> > +        yaml.dump(doc, f)
> > +
> > +    return 0
> > +
> > +
> > +if __name__ == '__main__':
> > +    sys.exit(main(sys.argv))