[v2,3/7] utils: Add script to generate control_ids_debug.yaml

21526 diff mbox series

For flexible debugging it is helpful to minimize the roundtrip time.
This script parses the sourcetree and looks for usages of

set<type>(controls::debug::Something,

and adds (or removes) the controls as necessary from the yaml
description. It is meant to be used during development to ease the
creation of the correct yaml entries.

Signed-off-by: Stefan Klug <stefan.klug@ideasonboard.com>

---
Changes in v2:
- Search only until the first comma of the set() call, to allow
  linebreaks there.
- Support ruamel.yaml as fallback
- Rename output to ctrl_file
- Add "generated by" comment in yaml file
---
 utils/gen-debug-controls.py | 165 ++++++++++++++++++++++++++++++++++++
 1 file changed, 165 insertions(+)
 create mode 100755 utils/gen-debug-controls.py

Hi Stefan,

Thank you for the patch.

On Mon, Oct 07, 2024 at 11:54:07AM +0200, Stefan Klug wrote:
> For flexible debugging it is helpful to minimize the roundtrip time.
> This script parses the sourcetree and looks for usages of

s/sourcetree/source tree/

> 
> set<type>(controls::debug::Something,
> 
> and adds (or removes) the controls as necessary from the yaml
> description. It is meant to be used during development to ease the
> creation of the correct yaml entries.
> 
> Signed-off-by: Stefan Klug <stefan.klug@ideasonboard.com>
> 
> ---
> Changes in v2:
> - Search only until the first comma of the set() call, to allow
>   linebreaks there.
> - Support ruamel.yaml as fallback
> - Rename output to ctrl_file
> - Add "generated by" comment in yaml file
> ---
>  utils/gen-debug-controls.py | 165 ++++++++++++++++++++++++++++++++++++
>  1 file changed, 165 insertions(+)
>  create mode 100755 utils/gen-debug-controls.py
> 
> diff --git a/utils/gen-debug-controls.py b/utils/gen-debug-controls.py
> new file mode 100755
> index 000000000000..c5c4570ffd00
> --- /dev/null
> +++ b/utils/gen-debug-controls.py
> @@ -0,0 +1,165 @@
> +#!/usr/bin/env python3
> +# SPDX-License-Identifier: GPL-2.0-or-later
> +# Copyright (C) 2024, Ideas on Board Inc.
> +#
> +# Author: Stefan Klug <stefan.klug@ideasonboard.com>
> +#
> +# This script looks for occurrences of the debug metadata controls in the source
> +# tree and updates src/libcamera/control_ids_debug.yaml accordingly. It is meant
> +# to be used during development to ease updating of the yaml file while
> +# debugging.
> +
> +import argparse
> +import logging
> +import os
> +import re
> +import sys
> +from dataclasses import dataclass
> +from pathlib import Path
> +
> +logger = logging.getLogger(__name__)
> +logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s')
> +
> +try:
> +    import ruyaml
> +except:
> +    try:
> +        import ruamel.yaml as ruyaml

Do we want to try both ruyaml and ruamel.yaml ? If their behaviour is
identical then I'd use the latter only as it's packaged by Debian. If
their behaviour is different, then this is calling for trouble as one of
the two will se less testing. I'd thus just try ruamel.yaml.

> +    except:
> +        logger.error(
> +            f'Failed to import ruyaml. Please install the ruyaml or the ruamel.yaml package.')
> +        sys.exit(1)
> +
> +@dataclass
> +class FoundMatch:
> +    file: os.PathLike
> +    whole_match: str
> +    line: int
> +    type: str
> +    name: str
> +    size: str = None
> +
> +
> +def get_control_name(control):
> +    k = list(control.keys())
> +    if len(k) != 1:
> +        raise Exception(f"Can't handle control entry with {len(k)} keys")
> +    return k[0]
> +
> +
> +def find_debug_controls(dir):
> +    extensions = ['.cpp', '.h']
> +    files = [p for p in dir.rglob('*') if p.suffix in extensions]
> +
> +    # The following regex was tested on
> +    # set<Span<type>>( controls::debug::something , static_cast<type>(var) )
> +    # set<>( controls::debug::something , static_cast<type>(var) )
> +    # set( controls::debug::something , static_cast<type> (var) )
> +    exp = re.compile(r'set'  # set function
> +                     # possibly followed by template param
> +                     r'(?:\<((?:[^)(])*)\>)?'
> +                     # referencing a debug control
> +                     r'\(\s*controls::debug::(\w+)\s*,'
> +                     )
> +    matches = []
> +    for p in files:
> +        with p.open('r') as f:
> +            for idx, line in enumerate(f):
> +                match = exp.search(line)
> +                if match:
> +                    m = FoundMatch(file=p, line=idx, type=match.group(1),
> +                                   name=match.group(2), whole_match=match.group(0))
> +                    if m.type is not None and m.type.startswith('Span'):
> +                        # simple span type detection treating the last word inside <> as type
> +                        r = re.match(r'Span<(?:.*\s+)(.*)>', m.type)
> +                        m.type = r.group(1)
> +                        m.size = '[n]'
> +                    matches.append(m)
> +    return matches
> +
> +
> +def main(argv):
> +    parser = argparse.ArgumentParser(
> +        description='Automatically updates control_ids_debug.yaml',)

I think you can drop the trailing comma.

> +    args = parser.parse_args(argv[1:])
> +
> +    yaml = ruyaml.YAML()
> +    root_dir = Path(__file__).resolve().parent.parent
> +    ctrl_file = root_dir.joinpath('src/libcamera/control_ids_debug.yaml')
> +
> +    matches = find_debug_controls(root_dir.joinpath('src'))
> +
> +    doc = yaml.load(ctrl_file)
> +
> +    controls = doc['controls']
> +
> +    # create a map of names in the existing yaml for easier updating

    # Create a map of names in the existing yaml for easier updating.

> +    controls_map = {}
> +    for control in controls:
> +        for k, v in control.items():
> +            controls_map[k] = v
> +
> +    obsolete_names = list(controls_map.keys())
> +
> +    for m in matches:
> +        if not m.type:
> +            p = m.file.relative_to(Path.cwd(), walk_up=True)
> +            logger.warning(
> +                f'{p}:{m.line + 1}: Failed to deduce type from {m.whole_match} ... skipping')
> +            continue
> +
> +        p = m.file.relative_to(root_dir)
> +        desc = {'type': m.type,
> +                'description': f'Debug control {m.name} found in {p}:{m.line}'}
> +        if m.size is not None:
> +            desc['size'] = m.size
> +
> +        if m.name in controls_map:
> +            # Can't use == for modified check because of the special yaml dicts.
> +            update_needed = False
> +            if list(controls_map[m.name].keys()) != list(desc.keys()):
> +                update_needed = True
> +            else:
> +                for k, v in controls_map[m.name].items():
> +                    if v != desc[k]:
> +                        update_needed = True
> +                        break
> +
> +            if update_needed:
> +                logger.info(f"Update control '{m.name}'")
> +                controls_map[m.name].clear()
> +                controls_map[m.name].update(desc)
> +
> +            obsolete_names.remove(m.name)
> +        else:
> +            logger.info(f"Add control '{m.name}'")
> +            insert_before = len(controls)
> +            for idx, control in enumerate(controls):
> +                if get_control_name(control).lower() > m.name.lower():
> +                    insert_before = idx
> +                    break
> +            controls.insert(insert_before, {m.name: desc})
> +
> +    # Remove elements from controls without recreating the list (to keep comments etc.)

    # Remove elements from controls without recreating the list (to keep
    # comments etc.).

> +    idx = 0
> +    while idx < len(controls):
> +        name = get_control_name(controls[idx])
> +        if name in obsolete_names:
> +            logger.info(f"Remove control '{name}'")
> +            controls.pop(idx)
> +        else:
> +            idx += 1
> +
> +    with ctrl_file.open('w') as f:
> +        # ruyaml looses the copyright header
> +        f.write(("# SPDX-License-Identifier: LGPL-2.1-or-later\n"
> +                 "#\n"
> +                 "# This file is generated by utils/gen-debug-controls.py\n"
> +                 "#\n"))
> +        yaml.dump(doc, f)
> +
> +    return 0
> +
> +
> +if __name__ == '__main__':
> +    sys.exit(main(sys.argv))

On Tue, Oct 08, 2024 at 01:12:29AM +0300, Laurent Pinchart wrote:
> Hi Stefan,
> 
> Thank you for the patch.
> 
> On Mon, Oct 07, 2024 at 11:54:07AM +0200, Stefan Klug wrote:
> > For flexible debugging it is helpful to minimize the roundtrip time.
> > This script parses the sourcetree and looks for usages of
> 
> s/sourcetree/source tree/
> 
> > 
> > set<type>(controls::debug::Something,
> > 
> > and adds (or removes) the controls as necessary from the yaml
> > description. It is meant to be used during development to ease the
> > creation of the correct yaml entries.
> > 
> > Signed-off-by: Stefan Klug <stefan.klug@ideasonboard.com>
> > 
> > ---
> > Changes in v2:
> > - Search only until the first comma of the set() call, to allow
> >   linebreaks there.
> > - Support ruamel.yaml as fallback
> > - Rename output to ctrl_file
> > - Add "generated by" comment in yaml file
> > ---
> >  utils/gen-debug-controls.py | 165 ++++++++++++++++++++++++++++++++++++
> >  1 file changed, 165 insertions(+)
> >  create mode 100755 utils/gen-debug-controls.py
> > 
> > diff --git a/utils/gen-debug-controls.py b/utils/gen-debug-controls.py
> > new file mode 100755
> > index 000000000000..c5c4570ffd00
> > --- /dev/null
> > +++ b/utils/gen-debug-controls.py
> > @@ -0,0 +1,165 @@
> > +#!/usr/bin/env python3
> > +# SPDX-License-Identifier: GPL-2.0-or-later
> > +# Copyright (C) 2024, Ideas on Board Inc.
> > +#
> > +# Author: Stefan Klug <stefan.klug@ideasonboard.com>
> > +#
> > +# This script looks for occurrences of the debug metadata controls in the source
> > +# tree and updates src/libcamera/control_ids_debug.yaml accordingly. It is meant
> > +# to be used during development to ease updating of the yaml file while
> > +# debugging.
> > +
> > +import argparse
> > +import logging
> > +import os
> > +import re
> > +import sys
> > +from dataclasses import dataclass
> > +from pathlib import Path
> > +
> > +logger = logging.getLogger(__name__)
> > +logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s')
> > +
> > +try:
> > +    import ruyaml
> > +except:
> > +    try:
> > +        import ruamel.yaml as ruyaml
> 
> Do we want to try both ruyaml and ruamel.yaml ? If their behaviour is
> identical then I'd use the latter only as it's packaged by Debian. If
> their behaviour is different, then this is calling for trouble as one of
> the two will se less testing. I'd thus just try ruamel.yaml.

It is a bit of an awkward situation.  ruamel.yaml is still listed as
beta, ruyaml as stable. As ruyaml is the fork that seems to have more
traction I would consider ruamel.yaml to be a dead end. debian sid and
trixie package ruyaml. As followup on my comment in the series v1
thread: I tried to switch to pyyaml. It is really bad, as it reorders
the yaml file and "vendor" ends at the bottom.

I can go back to the ruamel if you like, but it seems the wrong
direction to me. In the end it doesn't matter and the benefit of ruamel
is the broad availability.

> 
> > +    except:
> > +        logger.error(
> > +            f'Failed to import ruyaml. Please install the ruyaml or the ruamel.yaml package.')
> > +        sys.exit(1)
> > +
> > +@dataclass
> > +class FoundMatch:
> > +    file: os.PathLike
> > +    whole_match: str
> > +    line: int
> > +    type: str
> > +    name: str
> > +    size: str = None
> > +
> > +
> > +def get_control_name(control):
> > +    k = list(control.keys())
> > +    if len(k) != 1:
> > +        raise Exception(f"Can't handle control entry with {len(k)} keys")
> > +    return k[0]
> > +
> > +
> > +def find_debug_controls(dir):
> > +    extensions = ['.cpp', '.h']
> > +    files = [p for p in dir.rglob('*') if p.suffix in extensions]
> > +
> > +    # The following regex was tested on
> > +    # set<Span<type>>( controls::debug::something , static_cast<type>(var) )
> > +    # set<>( controls::debug::something , static_cast<type>(var) )
> > +    # set( controls::debug::something , static_cast<type> (var) )
> > +    exp = re.compile(r'set'  # set function
> > +                     # possibly followed by template param
> > +                     r'(?:\<((?:[^)(])*)\>)?'
> > +                     # referencing a debug control
> > +                     r'\(\s*controls::debug::(\w+)\s*,'
> > +                     )
> > +    matches = []
> > +    for p in files:
> > +        with p.open('r') as f:
> > +            for idx, line in enumerate(f):
> > +                match = exp.search(line)
> > +                if match:
> > +                    m = FoundMatch(file=p, line=idx, type=match.group(1),
> > +                                   name=match.group(2), whole_match=match.group(0))
> > +                    if m.type is not None and m.type.startswith('Span'):
> > +                        # simple span type detection treating the last word inside <> as type
> > +                        r = re.match(r'Span<(?:.*\s+)(.*)>', m.type)
> > +                        m.type = r.group(1)
> > +                        m.size = '[n]'
> > +                    matches.append(m)
> > +    return matches
> > +
> > +
> > +def main(argv):
> > +    parser = argparse.ArgumentParser(
> > +        description='Automatically updates control_ids_debug.yaml',)
> 
> I think you can drop the trailing comma.

I'm sure I fixed that and the args below... now for sure.

> 
> > +    args = parser.parse_args(argv[1:])
> > +
> > +    yaml = ruyaml.YAML()
> > +    root_dir = Path(__file__).resolve().parent.parent
> > +    ctrl_file = root_dir.joinpath('src/libcamera/control_ids_debug.yaml')
> > +
> > +    matches = find_debug_controls(root_dir.joinpath('src'))
> > +
> > +    doc = yaml.load(ctrl_file)
> > +
> > +    controls = doc['controls']
> > +
> > +    # create a map of names in the existing yaml for easier updating
> 
>     # Create a map of names in the existing yaml for easier updating.

ack

> 
> > +    controls_map = {}
> > +    for control in controls:
> > +        for k, v in control.items():
> > +            controls_map[k] = v
> > +
> > +    obsolete_names = list(controls_map.keys())
> > +
> > +    for m in matches:
> > +        if not m.type:
> > +            p = m.file.relative_to(Path.cwd(), walk_up=True)
> > +            logger.warning(
> > +                f'{p}:{m.line + 1}: Failed to deduce type from {m.whole_match} ... skipping')
> > +            continue
> > +
> > +        p = m.file.relative_to(root_dir)
> > +        desc = {'type': m.type,
> > +                'description': f'Debug control {m.name} found in {p}:{m.line}'}
> > +        if m.size is not None:
> > +            desc['size'] = m.size
> > +
> > +        if m.name in controls_map:
> > +            # Can't use == for modified check because of the special yaml dicts.
> > +            update_needed = False
> > +            if list(controls_map[m.name].keys()) != list(desc.keys()):
> > +                update_needed = True
> > +            else:
> > +                for k, v in controls_map[m.name].items():
> > +                    if v != desc[k]:
> > +                        update_needed = True
> > +                        break
> > +
> > +            if update_needed:
> > +                logger.info(f"Update control '{m.name}'")
> > +                controls_map[m.name].clear()
> > +                controls_map[m.name].update(desc)
> > +
> > +            obsolete_names.remove(m.name)
> > +        else:
> > +            logger.info(f"Add control '{m.name}'")
> > +            insert_before = len(controls)
> > +            for idx, control in enumerate(controls):
> > +                if get_control_name(control).lower() > m.name.lower():
> > +                    insert_before = idx
> > +                    break
> > +            controls.insert(insert_before, {m.name: desc})
> > +
> > +    # Remove elements from controls without recreating the list (to keep comments etc.)
> 
>     # Remove elements from controls without recreating the list (to keep
>     # comments etc.).

ack

> 
> > +    idx = 0
> > +    while idx < len(controls):
> > +        name = get_control_name(controls[idx])
> > +        if name in obsolete_names:
> > +            logger.info(f"Remove control '{name}'")
> > +            controls.pop(idx)
> > +        else:
> > +            idx += 1
> > +
> > +    with ctrl_file.open('w') as f:
> > +        # ruyaml looses the copyright header
> > +        f.write(("# SPDX-License-Identifier: LGPL-2.1-or-later\n"
> > +                 "#\n"
> > +                 "# This file is generated by utils/gen-debug-controls.py\n"
> > +                 "#\n"))
> > +        yaml.dump(doc, f)
> > +
> > +    return 0
> > +
> > +
> > +if __name__ == '__main__':
> > +    sys.exit(main(sys.argv))
> 
> -- 
> Regards,
> 
> Laurent Pinchart