tools/extract_configs.py - third_party/github/raspberrypi/pico-sdk - Git at Google


            
            #!/usr/bin/env python3

            
            #

            
            # Copyright (c) 2021 Raspberry Pi (Trading) Ltd.

            
            #

            
            # SPDX-License-Identifier: BSD-3-Clause

            
            #

            
            #

            
            # Script to scan the Raspberry Pi Pico SDK tree searching for configuration items

            
            # Outputs a tab separated file of the configuration item:

            
            # name	location	platform	chip	description	type	advanced	default	depends	enumvalues	group	max	min

            
            #

            
            # Usage:

            
            #

            
            # tools/extract_configs.py <root of repo> [output file]

            
            #

            
            # If not specified, output file will be `pico_configs.tsv`


            import os

            
            import sys

            
            import re

            
            import csv

            
            import logging


            from collections import defaultdict


            if sys.version_info < (3, 11):

            
                # Python <3.11 doesn't have ExceptionGroup, so define a simple one

            
                class ExceptionGroup(Exception):

            
                    def __init__(self, message, errors):

            
                        message += "\n" + "\n".join(str(e) for e in errors)

            
                        super().__init__(message)


            logger = logging.getLogger(__name__)

            
            logging.basicConfig(level=logging.INFO)


            scandir = sys.argv[1]

            
            outfile = sys.argv[2] if len(sys.argv) > 2 else 'pico_configs.tsv'


            BASE_CONFIG_NAME = 'PICO_CONFIG'

            
            BASE_CONFIG_RE = re.compile(r'\b{}\b'.format(BASE_CONFIG_NAME))

            
            BASE_CMAKE_CONFIG_NAME = 'PICO_CMAKE_CONFIG'

            
            BASE_CMAKE_CONFIG_RE = re.compile(r'\b{}\b'.format(BASE_CMAKE_CONFIG_NAME))

            
            BASE_BUILD_DEFINE_NAME = 'PICO_BUILD_DEFINE'

            
            BASE_BUILD_DEFINE_RE = re.compile(r'\b{}\b'.format(BASE_BUILD_DEFINE_NAME))


            CONFIG_RE = re.compile(r'//\s+{}:\s+(\w+),\s+([^,]+)(?:,\s+(.*))?$'.format(BASE_CONFIG_NAME))

            
            DEFINE_RE = re.compile(r'#define\s+(\w+)\s+(.+?)(\s*///.*)?$')


            PROPERTY_TYPE = 'type'

            
            PROPERTY_DEFAULT = 'default'

            
            PROPERTY_MIN = 'min'

            
            PROPERTY_MAX = 'max'

            
            PROPERTY_ENUMVALUES = 'enumvalues'

            
            PROPERTY_GROUP = 'group'

            
            PROPERTY_ADVANCED = 'advanced'

            
            PROPERTY_DEPENDS = 'depends'

            
            ALLOWED_CONFIG_PROPERTIES = set([PROPERTY_TYPE, PROPERTY_DEFAULT, PROPERTY_MIN, PROPERTY_MAX, PROPERTY_ENUMVALUES, PROPERTY_GROUP, PROPERTY_ADVANCED, PROPERTY_DEPENDS])


            PROPERTY_TYPE_INT = 'int'

            
            PROPERTY_TYPE_BOOL = 'bool'

            
            PROPERTY_TYPE_ENUM = 'enum'

            
            PROPERTY_TYPE_LIST = 'list'


            CHIP_NAMES = ["rp2040", "rp2350"]


            chips_all_configs = defaultdict(dict)

            
            all_attrs = set()

            
            chips_all_descriptions = defaultdict(dict)

            
            chips_all_defines = defaultdict(dict)


            def get_first_dict_key(some_dict):

            
                return list(some_dict.keys())[0]


            def look_for_integer_define(config_name, attr_name, attr_str, file_path, linenum, applicable):

            
                defined_str = None

            
                if re.match(r'^\w+$', attr_str):

            
                    # See if we have a matching define

            
                    all_defines = chips_all_defines[applicable]

            
                    if attr_str in all_defines:

            
                        # There _may_ be multiple matching defines, but arbitrarily choose just one

            
                        defined_str = get_first_dict_key(all_defines[attr_str])

            
                    else:

            
                        if applicable == 'all':

            
                            # Look for it in the chip-specific defines

            
                            found_chips = []

            
                            missing_chips = []

            
                            for chip in CHIP_NAMES:

            
                                all_defines = chips_all_defines[chip]

            
                                if attr_str in all_defines:

            
                                    found_chips.append(chip)

            
                                    # There _may_ be multiple matching defines, but arbitrarily choose just one

            
                                    defined_str = get_first_dict_key(all_defines[attr_str])

            
                                else:

            
                                    missing_chips.append(chip)

            
                            # Report if it's found for some chips but not others (but ignore if it's missing for all)

            
                            if found_chips and missing_chips:

            
                                logger.info('{} at {}:{} has {} value "{}" which has a matching define for {} but not for {}'.format(config_name, file_path, linenum, attr_name, attr_str, found_chips, missing_chips))

            
                return defined_str


            def ValidateAttrs(config_name, config_attrs, file_path, linenum, applicable):

            
                type_str = config_attrs.get(PROPERTY_TYPE, PROPERTY_TYPE_INT)

            
                errors = []


                # Validate attrs

            
                for key in config_attrs.keys():

            
                    if key not in ALLOWED_CONFIG_PROPERTIES:

            
                        errors.append(Exception('{} at {}:{} has unexpected property "{}"'.format(config_name, file_path, linenum, key)))


                str_values = dict()

            
                parsed_values = dict()

            
                if type_str == PROPERTY_TYPE_INT:

            
                    assert PROPERTY_ENUMVALUES not in config_attrs


                    for attr_name in (PROPERTY_MIN, PROPERTY_MAX, PROPERTY_DEFAULT):

            
                        str_values[attr_name] = config_attrs.get(attr_name, None)

            
                        if str_values[attr_name] is not None:

            
                            try:

            
                                m = re.match(r'^(\d+)e(\d+)$', str_values[attr_name].lower())

            
                                if m:

            
                                    parsed_values[attr_name] = int(m.group(1)) * 10**int(m.group(2))

            
                                else:

            
                                    parsed_values[attr_name] = int(str_values[attr_name], 0)

            
                            except ValueError:

            
                                defined_str = look_for_integer_define(config_name, attr_name, str_values[attr_name], file_path, linenum, applicable)

            
                                if defined_str is not None:

            
                                    try:

            
                                        m = re.match(r'^(\d+)e(\d+)$', str_values[attr_name].lower())

            
                                        if m:

            
                                            parsed_values[attr_name] = int(m.group(1)) * 10**int(m.group(2))

            
                                        else:

            
                                            parsed_values[attr_name] = int(defined_str, 0)

            
                                    except ValueError:

            
                                        logger.info('{} at {}:{} has {} value "{}" which resolves to the non-integer value "{}"'.format(config_name, file_path, linenum, attr_name, str_values[attr_name], defined_str))

            
                                else:

            
                                    logger.info('{} at {}:{} has non-integer {} value "{}"'.format(config_name, file_path, linenum, attr_name, str_values[attr_name]))

            
                    for (small_attr, large_attr) in (

            
                        (PROPERTY_MIN, PROPERTY_MAX),

            
                        (PROPERTY_MIN, PROPERTY_DEFAULT),

            
                        (PROPERTY_DEFAULT, PROPERTY_MAX),

            
                    ):

            
                        if small_attr in parsed_values and large_attr in parsed_values and parsed_values[small_attr] > parsed_values[large_attr]:

            
                            errors.append(Exception('{} at {}:{} has {} {} > {} {}'.format(config_name, file_path, linenum, small_attr, str_values[small_attr], large_attr, str_values[large_attr])))


                elif type_str == PROPERTY_TYPE_BOOL:

            
                    assert PROPERTY_MIN not in config_attrs

            
                    assert PROPERTY_MAX not in config_attrs

            
                    assert PROPERTY_ENUMVALUES not in config_attrs


                    attr_name = PROPERTY_DEFAULT

            
                    str_values[attr_name] = config_attrs.get(attr_name, None)

            
                    if str_values[attr_name] is not None:

            
                        if (str_values[attr_name] not in ('0', '1')) and (str_values[attr_name] not in all_config_names):

            
                            logger.info('{} at {}:{} has non-integer {} value "{}"'.format(config_name, file_path, linenum, attr_name, str_values[attr_name]))


                elif type_str == PROPERTY_TYPE_ENUM:

            
                    assert PROPERTY_ENUMVALUES in config_attrs

            
                    assert PROPERTY_MIN not in config_attrs

            
                    assert PROPERTY_MAX not in config_attrs


                    attr_name = PROPERTY_ENUMVALUES

            
                    str_values[attr_name] = config_attrs[attr_name]

            
                    parsed_values[attr_name] = tuple(str_values[attr_name].split('|'))


                    attr_name = PROPERTY_DEFAULT

            
                    str_values[attr_name] = config_attrs.get(attr_name, None)

            
                    if str_values[attr_name] is not None:

            
                        if str_values[attr_name] not in _enumvalues:

            
                            errors.append(Exception('{} at {}:{} has {} value {} which isn\'t in list of {} {}'.format(config_name, file_path, linenum, attr_name, str_values[attr_name], PROPERTY_ENUMVALUES, str_values[PROPERTY_ENUMVALUES])))


                elif type_str == PROPERTY_TYPE_LIST:

            
                    assert PROPERTY_MIN not in config_attrs

            
                    assert PROPERTY_MAX not in config_attrs

            
                    assert PROPERTY_ENUMVALUES not in config_attrs


                    attr_name = PROPERTY_DEFAULT

            
                    str_values[attr_name] = config_attrs.get(attr_name, None)


                else:

            
                    errors.append(Exception("Found unknown {} type {} at {}:{}".format(BASE_CONFIG_NAME, type_str, file_path, linenum)))


                return errors


            errors = []


            # Scan all .c and .h and .S files in the specific path, recursively.


            for dirpath, dirnames, filenames in os.walk(scandir):

            
                # Don't descend into CMake build trees; they hold generated copies of the files we are

            
                # scanning for, and a developer may have several of them inside the source tree.

            
                if 'CMakeCache.txt' in filenames:

            
                    dirnames[:] = []

            
                    continue

            
                # lib/ holds third-party submodules, which don't use these markers.

            
                if os.path.relpath(dirpath, scandir) == 'lib':

            
                    dirnames[:] = []

            
                    continue

            
                for filename in filenames:

            
                    file_ext = os.path.splitext(filename)[1]

            
                    if file_ext in ('.c', '.h', '.S'):

            
                        file_path = os.path.join(dirpath, filename)

            
                        applicable = "all"

            
                        for chip in (*CHIP_NAMES, "host"):

            
                            if "/{}/".format(chip) in dirpath:

            
                                applicable = chip

            
                                break


                        with open(file_path, encoding="ISO-8859-1") as fh:

            
                            linenum = 0

            
                            for line in fh.readlines():

            
                                linenum += 1

            
                                line = line.strip()

            
                                if BASE_CMAKE_CONFIG_RE.search(line):

            
                                    errors.append(Exception("Found {} at {}:{} ({}) which isn't expected in {} files".format(BASE_CMAKE_CONFIG_NAME, file_path, linenum, line, file_ext)))

            
                                elif BASE_BUILD_DEFINE_RE.search(line):

            
                                    errors.append(Exception("Found {} at {}:{} ({}) which isn't expected in {} files".format(BASE_BUILD_DEFINE_NAME, file_path, linenum, line, file_ext)))

            
                                elif BASE_CONFIG_RE.search(line):

            
                                    m = CONFIG_RE.match(line)

            
                                    if not m:

            
                                        if re.match(r"^\s*//\s*// ", line):

            
                                            logger.info("Possible misformatted {} at {}:{} ({})".format(BASE_CONFIG_NAME, file_path, linenum, line))

            
                                        else:

            
                                            errors.append(Exception("Found misformatted {} at {}:{} ({})".format(BASE_CONFIG_NAME, file_path, linenum, line)))

            
                                    else:

            
                                        config_name = m.group(1)

            
                                        config_description = m.group(2)

            
                                        _attrs = m.group(3)

            
                                        # allow commas to appear inside brackets by converting them to and from NULL chars

            
                                        _attrs = re.sub(r'(\(.+\))', lambda m: m.group(1).replace(',', '\0'), _attrs)


                                        if '=' in config_description and not '==' in config_description:

            
                                            errors.append(Exception("For {} at {}:{} the description was set to '{}' - has the description field been omitted?".format(config_name, file_path, linenum, config_description)))

            
                                        all_descriptions = chips_all_descriptions[applicable]

            
                                        if config_description in all_descriptions:

            
                                            errors.append(Exception("Found description {} at {}:{} but it was already used at {}:{}".format(config_description, file_path, linenum, os.path.join(scandir, all_descriptions[config_description]['filename']), all_descriptions[config_description]['line_number'])))

            
                                        else:

            
                                            all_descriptions[config_description] = {'config_name': config_name, 'filename': os.path.relpath(file_path, scandir), 'line_number': linenum}


                                        config_attrs = {}

            
                                        prev = None

            
                                        # Handle case where attr value contains a comma

            
                                        for item in _attrs.split(','):

            
                                            if "=" not in item:

            
                                                assert(prev)

            
                                                item = prev + "," + item

            
                                            try:

            
                                                k, v = (i.strip() for i in item.split('='))

            
                                            except ValueError:

            
                                                errors.append(Exception('{} at {}:{} has malformed value {}'.format(config_name, file_path, linenum, item)))

            
                                            config_attrs[k] = v.replace('\0', ',')

            
                                            all_attrs.add(k)

            
                                            prev = item

            
                                        #print(file_path, config_name, config_attrs)


                                        if 'group' not in config_attrs:

            
                                            errors.append(Exception('{} at {}:{} has no group attribute'.format(config_name, file_path, linenum)))


                                        #print(file_path, config_name, config_attrs)

            
                                        all_configs = chips_all_configs[applicable]

            
                                        if config_name in all_configs:

            
                                            errors.append(Exception("Found {} at {}:{} but it was already declared at {}:{}".format(config_name, file_path, linenum, os.path.join(scandir, all_configs[config_name]['filename']), all_configs[config_name]['line_number'])))

            
                                        else:

            
                                            all_configs[config_name] = {'attrs': config_attrs, 'filename': os.path.relpath(file_path, scandir), 'line_number': linenum, 'description': config_description}

            
                                else:

            
                                    m = DEFINE_RE.match(line)

            
                                    if m:

            
                                        name = m.group(1)

            
                                        value = m.group(2)

            
                                        # discard any 'u' qualifier

            
                                        m = re.match(r'^((0x)?\d+)u$', value.lower())

            
                                        if m:

            
                                            value = m.group(1)

            
                                        else:

            
                                            # discard any '_u(X)' macro

            
                                            m = re.match(r'^_u\(((0x)?\d+)\)$', value.lower())

            
                                            if m:

            
                                                value = m.group(1)

            
                                        all_defines = chips_all_defines[applicable]

            
                                        if name not in all_defines:

            
                                            all_defines[name] = dict()

            
                                        if value not in all_defines[name]:

            
                                            all_defines[name][value] = set()

            
                                        all_defines[name][value] = (file_path, linenum)


            all_config_names = set()

            
            for all_configs in chips_all_configs.values():

            
                all_config_names.update(all_configs.keys())


            # Check for defines with missing PICO_CONFIG entries

            
            chips_resolved_defines = defaultdict(dict)

            
            for applicable, all_defines in chips_all_defines.items():

            
                for d in all_defines:

            
                    for define_prefix in ('PICO', 'PARAM', 'CYW43'):

            
                        if d not in all_config_names and d.startswith(define_prefix+'_'):

            
                            logger.warning("#define {} is potentially missing a {}: entry".format(d, BASE_CONFIG_NAME))

            
                    resolved_defines = chips_resolved_defines[applicable]

            
                    # resolve "nested defines" - this allows e.g. USB_DPRAM_MAX to resolve to USB_DPRAM_SIZE which is set to 4096 (which then matches the relevant PICO_CONFIG entry)

            
                    for val in all_defines[d]:

            
                        if val in all_defines:

            
                            resolved_defines[d] = all_defines[val]


            for applicable, all_configs in chips_all_configs.items():

            
                all_defines = chips_all_defines[applicable]

            
                resolved_defines = chips_resolved_defines[applicable]

            
                for config_name, config_obj in all_configs.items():

            
                    file_path = os.path.join(scandir, config_obj['filename'])

            
                    linenum = config_obj['line_number']


                    errors.extend(ValidateAttrs(config_name, config_obj['attrs'], file_path, linenum, applicable))


                    # Check that default values match up

            
                    if 'default' in config_obj['attrs']:

            
                        config_default = config_obj['attrs']['default']

            
                        if config_name in all_defines:

            
                            defines_obj = all_defines[config_name]

            
                            if config_default not in defines_obj and (config_name not in resolved_defines or config_default not in resolved_defines[config_name]):

            
                                if '/' in config_default or ' ' in config_default:

            
                                    continue

            
                                # There _may_ be multiple matching defines, but arbitrarily display just one in the error message

            
                                first_define_value = get_first_dict_key(defines_obj)

            
                                first_define_file_path, first_define_linenum = defines_obj[first_define_value]

            
                                errors.append(Exception('Found {} at {}:{} with a default of {}, but #define says {} (at {}:{})'.format(config_name, file_path, linenum, config_default, first_define_value, first_define_file_path, first_define_linenum)))

            
                        elif config_obj['attrs']['type'] == "bool" and config_default == "0":

            
                            # a bool with a missing #define defaults to 0 (false) anyway

            
                            logger.info('Found {} (bool) at {}:{} with a default of {}, but no matching #define found'.format(config_name, file_path, linenum, config_default))

            
                        else:

            
                            errors.append(Exception('Found {} at {}:{} with a default of {}, but no matching #define found'.format(config_name, file_path, linenum, config_default)))


            # All settings in "host" should also be in "all"

            
            for config_name, config_obj in chips_all_configs["host"].items():

            
                if config_name not in chips_all_configs["all"]:

            
                    file_path = os.path.join(scandir, config_obj['filename'])

            
                    linenum = config_obj['line_number']

            
                    errors.append(Exception("Found 'host' config {} at {}:{}, but no matching non-host config found".format(config_name, file_path, linenum)))


            # Any chip-specific settings should not be in "all"

            
            for chip in CHIP_NAMES:

            
                for config_name, chip_config_obj in chips_all_configs[chip].items():

            
                    if config_name in chips_all_configs["all"]:

            
                        all_config_obj = chips_all_configs["all"][config_name]

            
                        chip_file_path = os.path.join(scandir, chip_config_obj['filename'])

            
                        chip_linenum = chip_config_obj['line_number']

            
                        all_file_path = os.path.join(scandir, all_config_obj['filename'])

            
                        all_linenum = all_config_obj['line_number']

            
                        errors.append(Exception("'{}' config {} at {}:{} also found at {}:{}".format(chip, config_name, chip_file_path, chip_linenum, all_file_path, all_linenum)))


            def build_mismatch_exception_message(name, thing, config_obj1, value1, config_obj2, value2):

            
                obj1_filepath = os.path.join(scandir, config_obj1['filename'])

            
                obj2_filepath = os.path.join(scandir, config_obj2['filename'])

            
                return "'{}' {} mismatch at {}:{} ({}) and {}:{} ({})".format(name, thing, obj1_filepath, config_obj1['line_number'], value1, obj2_filepath, config_obj2['line_number'], value2)


            # Check that any identically-named settings have appropriate matching attributes

            
            for applicable in chips_all_configs:

            
                for other in chips_all_configs:

            
                    if other == applicable:

            
                        continue

            
                    for config_name, applicable_config_obj in chips_all_configs[applicable].items():

            
                        if config_name in chips_all_configs[other]:

            
                            other_config_obj = chips_all_configs[other][config_name]

            
                            # Check that fields match

            
                            for field in ['description']:

            
                                applicable_value = applicable_config_obj[field]

            
                                other_value = other_config_obj[field]

            
                                if applicable_value != other_value:

            
                                    errors.append(Exception(build_mismatch_exception_message(config_name, field, applicable_config_obj, applicable_value, other_config_obj, other_value)))

            
                            # Check that attributes match

            
                            for attr in applicable_config_obj['attrs']:

            
                                if attr != 'default': # totally fine for defaults to vary per-platform

            
                                    applicable_value = applicable_config_obj['attrs'][attr]

            
                                    other_value = other_config_obj['attrs'][attr]

            
                                    if applicable_value != other_value:

            
                                        errors.append(Exception(build_mismatch_exception_message(config_name, "attribute '{}'".format(attr), applicable_config_obj, applicable_value, other_config_obj, other_value)))


            # Raise errors if any were found

            
            if errors:

            
                raise ExceptionGroup("Errors in {}".format(outfile), errors)


            # Sort the output alphabetically by name and then by chip

            
            output_rows = set()

            
            for chip in (*CHIP_NAMES, "host", "all"):

            
                if chip in chips_all_configs:

            
                    all_configs = chips_all_configs[chip]

            
                    for config_name in all_configs:

            
                        output_rows.add((config_name, chip))


            with open(outfile, 'w', newline='') as csvfile:

            
                fieldnames = ('name', 'location', 'platform', 'chip', 'description', 'type') + tuple(sorted(all_attrs - set(['type'])))

            
                writer = csv.DictWriter(csvfile, fieldnames=fieldnames, extrasaction='ignore', dialect='excel-tab')


                writer.writeheader()

            
                for config_name, chip in sorted(output_rows):

            
                    config_obj = chips_all_configs[chip][config_name]

            
                    writer.writerow({'name': config_name, 'location': '/{}:{}'.format(config_obj['filename'], config_obj['line_number']), 'platform': "host" if chip == "host" else "rp2", 'chip': chip if chip in CHIP_NAMES else "all", 'description': config_obj['description'], **config_obj['attrs']})
	#!/usr/bin/env python3
	#
	# Copyright (c) 2021 Raspberry Pi (Trading) Ltd.
	#
	# SPDX-License-Identifier: BSD-3-Clause
	#
	#
	# Script to scan the Raspberry Pi Pico SDK tree searching for configuration items
	# Outputs a tab separated file of the configuration item:
	# name location platform chip description type advanced default depends enumvalues group max min
	#
	# Usage:
	#
	# tools/extract_configs.py <root of repo> [output file]
	#
	# If not specified, output file will be `pico_configs.tsv`


	import os
	import sys
	import re
	import csv
	import logging

	from collections import defaultdict

	if sys.version_info < (3, 11):
	# Python <3.11 doesn't have ExceptionGroup, so define a simple one
	class ExceptionGroup(Exception):
	def __init__(self, message, errors):
	message += "\n" + "\n".join(str(e) for e in errors)
	super().__init__(message)

	logger = logging.getLogger(__name__)
	logging.basicConfig(level=logging.INFO)

	scandir = sys.argv[1]
	outfile = sys.argv[2] if len(sys.argv) > 2 else 'pico_configs.tsv'

	BASE_CONFIG_NAME = 'PICO_CONFIG'
	BASE_CONFIG_RE = re.compile(r'\b{}\b'.format(BASE_CONFIG_NAME))
	BASE_CMAKE_CONFIG_NAME = 'PICO_CMAKE_CONFIG'
	BASE_CMAKE_CONFIG_RE = re.compile(r'\b{}\b'.format(BASE_CMAKE_CONFIG_NAME))
	BASE_BUILD_DEFINE_NAME = 'PICO_BUILD_DEFINE'
	BASE_BUILD_DEFINE_RE = re.compile(r'\b{}\b'.format(BASE_BUILD_DEFINE_NAME))

	CONFIG_RE = re.compile(r'//\s+{}:\s+(\w+),\s+([^,]+)(?:,\s+(.*))?$'.format(BASE_CONFIG_NAME))
	DEFINE_RE = re.compile(r'#define\s+(\w+)\s+(.+?)(\s///.)?$')

	PROPERTY_TYPE = 'type'
	PROPERTY_DEFAULT = 'default'
	PROPERTY_MIN = 'min'
	PROPERTY_MAX = 'max'
	PROPERTY_ENUMVALUES = 'enumvalues'
	PROPERTY_GROUP = 'group'
	PROPERTY_ADVANCED = 'advanced'
	PROPERTY_DEPENDS = 'depends'
	ALLOWED_CONFIG_PROPERTIES = set([PROPERTY_TYPE, PROPERTY_DEFAULT, PROPERTY_MIN, PROPERTY_MAX, PROPERTY_ENUMVALUES, PROPERTY_GROUP, PROPERTY_ADVANCED, PROPERTY_DEPENDS])

	PROPERTY_TYPE_INT = 'int'
	PROPERTY_TYPE_BOOL = 'bool'
	PROPERTY_TYPE_ENUM = 'enum'
	PROPERTY_TYPE_LIST = 'list'

	CHIP_NAMES = ["rp2040", "rp2350"]

	chips_all_configs = defaultdict(dict)
	all_attrs = set()
	chips_all_descriptions = defaultdict(dict)
	chips_all_defines = defaultdict(dict)


	def get_first_dict_key(some_dict):
	return list(some_dict.keys())[0]

	def look_for_integer_define(config_name, attr_name, attr_str, file_path, linenum, applicable):
	defined_str = None
	if re.match(r'^\w+$', attr_str):
	# See if we have a matching define
	all_defines = chips_all_defines[applicable]
	if attr_str in all_defines:
	# There _may_ be multiple matching defines, but arbitrarily choose just one
	defined_str = get_first_dict_key(all_defines[attr_str])
	else:
	if applicable == 'all':
	# Look for it in the chip-specific defines
	found_chips = []
	missing_chips = []
	for chip in CHIP_NAMES:
	all_defines = chips_all_defines[chip]
	if attr_str in all_defines:
	found_chips.append(chip)
	# There _may_ be multiple matching defines, but arbitrarily choose just one
	defined_str = get_first_dict_key(all_defines[attr_str])
	else:
	missing_chips.append(chip)
	# Report if it's found for some chips but not others (but ignore if it's missing for all)
	if found_chips and missing_chips:
	logger.info('{} at {}:{} has {} value "{}" which has a matching define for {} but not for {}'.format(config_name, file_path, linenum, attr_name, attr_str, found_chips, missing_chips))
	return defined_str

	def ValidateAttrs(config_name, config_attrs, file_path, linenum, applicable):
	type_str = config_attrs.get(PROPERTY_TYPE, PROPERTY_TYPE_INT)
	errors = []

	# Validate attrs
	for key in config_attrs.keys():
	if key not in ALLOWED_CONFIG_PROPERTIES:
	errors.append(Exception('{} at {}:{} has unexpected property "{}"'.format(config_name, file_path, linenum, key)))

	str_values = dict()
	parsed_values = dict()
	if type_str == PROPERTY_TYPE_INT:
	assert PROPERTY_ENUMVALUES not in config_attrs

	for attr_name in (PROPERTY_MIN, PROPERTY_MAX, PROPERTY_DEFAULT):
	str_values[attr_name] = config_attrs.get(attr_name, None)
	if str_values[attr_name] is not None:
	try:
	m = re.match(r'^(\d+)e(\d+)$', str_values[attr_name].lower())
	if m:
	parsed_values[attr_name] = int(m.group(1)) * 10**int(m.group(2))
	else:
	parsed_values[attr_name] = int(str_values[attr_name], 0)
	except ValueError:
	defined_str = look_for_integer_define(config_name, attr_name, str_values[attr_name], file_path, linenum, applicable)
	if defined_str is not None:
	try:
	m = re.match(r'^(\d+)e(\d+)$', str_values[attr_name].lower())
	if m:
	parsed_values[attr_name] = int(m.group(1)) * 10**int(m.group(2))
	else:
	parsed_values[attr_name] = int(defined_str, 0)
	except ValueError:
	logger.info('{} at {}:{} has {} value "{}" which resolves to the non-integer value "{}"'.format(config_name, file_path, linenum, attr_name, str_values[attr_name], defined_str))
	else:
	logger.info('{} at {}:{} has non-integer {} value "{}"'.format(config_name, file_path, linenum, attr_name, str_values[attr_name]))
	for (small_attr, large_attr) in (
	(PROPERTY_MIN, PROPERTY_MAX),
	(PROPERTY_MIN, PROPERTY_DEFAULT),
	(PROPERTY_DEFAULT, PROPERTY_MAX),
	):
	if small_attr in parsed_values and large_attr in parsed_values and parsed_values[small_attr] > parsed_values[large_attr]:
	errors.append(Exception('{} at {}:{} has {} {} > {} {}'.format(config_name, file_path, linenum, small_attr, str_values[small_attr], large_attr, str_values[large_attr])))

	elif type_str == PROPERTY_TYPE_BOOL:
	assert PROPERTY_MIN not in config_attrs
	assert PROPERTY_MAX not in config_attrs
	assert PROPERTY_ENUMVALUES not in config_attrs

	attr_name = PROPERTY_DEFAULT
	str_values[attr_name] = config_attrs.get(attr_name, None)
	if str_values[attr_name] is not None:
	if (str_values[attr_name] not in ('0', '1')) and (str_values[attr_name] not in all_config_names):
	logger.info('{} at {}:{} has non-integer {} value "{}"'.format(config_name, file_path, linenum, attr_name, str_values[attr_name]))

	elif type_str == PROPERTY_TYPE_ENUM:
	assert PROPERTY_ENUMVALUES in config_attrs
	assert PROPERTY_MIN not in config_attrs
	assert PROPERTY_MAX not in config_attrs

	attr_name = PROPERTY_ENUMVALUES
	str_values[attr_name] = config_attrs[attr_name]
	parsed_values[attr_name] = tuple(str_values[attr_name].split('\|'))

	attr_name = PROPERTY_DEFAULT
	str_values[attr_name] = config_attrs.get(attr_name, None)
	if str_values[attr_name] is not None:
	if str_values[attr_name] not in _enumvalues:
	errors.append(Exception('{} at {}:{} has {} value {} which isn\'t in list of {} {}'.format(config_name, file_path, linenum, attr_name, str_values[attr_name], PROPERTY_ENUMVALUES, str_values[PROPERTY_ENUMVALUES])))

	elif type_str == PROPERTY_TYPE_LIST:
	assert PROPERTY_MIN not in config_attrs
	assert PROPERTY_MAX not in config_attrs
	assert PROPERTY_ENUMVALUES not in config_attrs

	attr_name = PROPERTY_DEFAULT
	str_values[attr_name] = config_attrs.get(attr_name, None)

	else:
	errors.append(Exception("Found unknown {} type {} at {}:{}".format(BASE_CONFIG_NAME, type_str, file_path, linenum)))

	return errors

	errors = []

	# Scan all .c and .h and .S files in the specific path, recursively.

	for dirpath, dirnames, filenames in os.walk(scandir):
	# Don't descend into CMake build trees; they hold generated copies of the files we are
	# scanning for, and a developer may have several of them inside the source tree.
	if 'CMakeCache.txt' in filenames:
	dirnames[:] = []
	continue
	# lib/ holds third-party submodules, which don't use these markers.
	if os.path.relpath(dirpath, scandir) == 'lib':
	dirnames[:] = []
	continue
	for filename in filenames:
	file_ext = os.path.splitext(filename)[1]
	if file_ext in ('.c', '.h', '.S'):
	file_path = os.path.join(dirpath, filename)
	applicable = "all"
	for chip in (*CHIP_NAMES, "host"):
	if "/{}/".format(chip) in dirpath:
	applicable = chip
	break

	with open(file_path, encoding="ISO-8859-1") as fh:
	linenum = 0
	for line in fh.readlines():
	linenum += 1
	line = line.strip()
	if BASE_CMAKE_CONFIG_RE.search(line):
	errors.append(Exception("Found {} at {}:{} ({}) which isn't expected in {} files".format(BASE_CMAKE_CONFIG_NAME, file_path, linenum, line, file_ext)))
	elif BASE_BUILD_DEFINE_RE.search(line):
	errors.append(Exception("Found {} at {}:{} ({}) which isn't expected in {} files".format(BASE_BUILD_DEFINE_NAME, file_path, linenum, line, file_ext)))
	elif BASE_CONFIG_RE.search(line):
	m = CONFIG_RE.match(line)
	if not m:
	if re.match(r"^\s//\s// ", line):
	logger.info("Possible misformatted {} at {}:{} ({})".format(BASE_CONFIG_NAME, file_path, linenum, line))
	else:
	errors.append(Exception("Found misformatted {} at {}:{} ({})".format(BASE_CONFIG_NAME, file_path, linenum, line)))
	else:
	config_name = m.group(1)
	config_description = m.group(2)
	_attrs = m.group(3)
	# allow commas to appear inside brackets by converting them to and from NULL chars
	_attrs = re.sub(r'(\(.+\))', lambda m: m.group(1).replace(',', '\0'), _attrs)

	if '=' in config_description and not '==' in config_description:
	errors.append(Exception("For {} at {}:{} the description was set to '{}' - has the description field been omitted?".format(config_name, file_path, linenum, config_description)))
	all_descriptions = chips_all_descriptions[applicable]
	if config_description in all_descriptions:
	errors.append(Exception("Found description {} at {}:{} but it was already used at {}:{}".format(config_description, file_path, linenum, os.path.join(scandir, all_descriptions[config_description]['filename']), all_descriptions[config_description]['line_number'])))
	else:
	all_descriptions[config_description] = {'config_name': config_name, 'filename': os.path.relpath(file_path, scandir), 'line_number': linenum}

	config_attrs = {}
	prev = None
	# Handle case where attr value contains a comma
	for item in _attrs.split(','):
	if "=" not in item:
	assert(prev)
	item = prev + "," + item
	try:
	k, v = (i.strip() for i in item.split('='))
	except ValueError:
	errors.append(Exception('{} at {}:{} has malformed value {}'.format(config_name, file_path, linenum, item)))
	config_attrs[k] = v.replace('\0', ',')
	all_attrs.add(k)
	prev = item
	#print(file_path, config_name, config_attrs)

	if 'group' not in config_attrs:
	errors.append(Exception('{} at {}:{} has no group attribute'.format(config_name, file_path, linenum)))

	#print(file_path, config_name, config_attrs)
	all_configs = chips_all_configs[applicable]
	if config_name in all_configs:
	errors.append(Exception("Found {} at {}:{} but it was already declared at {}:{}".format(config_name, file_path, linenum, os.path.join(scandir, all_configs[config_name]['filename']), all_configs[config_name]['line_number'])))
	else:
	all_configs[config_name] = {'attrs': config_attrs, 'filename': os.path.relpath(file_path, scandir), 'line_number': linenum, 'description': config_description}
	else:
	m = DEFINE_RE.match(line)
	if m:
	name = m.group(1)
	value = m.group(2)
	# discard any 'u' qualifier
	m = re.match(r'^((0x)?\d+)u$', value.lower())
	if m:
	value = m.group(1)
	else:
	# discard any '_u(X)' macro
	m = re.match(r'^_u\(((0x)?\d+)\)$', value.lower())
	if m:
	value = m.group(1)
	all_defines = chips_all_defines[applicable]
	if name not in all_defines:
	all_defines[name] = dict()
	if value not in all_defines[name]:
	all_defines[name][value] = set()
	all_defines[name][value] = (file_path, linenum)

	all_config_names = set()
	for all_configs in chips_all_configs.values():
	all_config_names.update(all_configs.keys())

	# Check for defines with missing PICO_CONFIG entries
	chips_resolved_defines = defaultdict(dict)
	for applicable, all_defines in chips_all_defines.items():
	for d in all_defines:
	for define_prefix in ('PICO', 'PARAM', 'CYW43'):
	if d not in all_config_names and d.startswith(define_prefix+'_'):
	logger.warning("#define {} is potentially missing a {}: entry".format(d, BASE_CONFIG_NAME))
	resolved_defines = chips_resolved_defines[applicable]
	# resolve "nested defines" - this allows e.g. USB_DPRAM_MAX to resolve to USB_DPRAM_SIZE which is set to 4096 (which then matches the relevant PICO_CONFIG entry)
	for val in all_defines[d]:
	if val in all_defines:
	resolved_defines[d] = all_defines[val]

	for applicable, all_configs in chips_all_configs.items():
	all_defines = chips_all_defines[applicable]
	resolved_defines = chips_resolved_defines[applicable]
	for config_name, config_obj in all_configs.items():
	file_path = os.path.join(scandir, config_obj['filename'])
	linenum = config_obj['line_number']

	errors.extend(ValidateAttrs(config_name, config_obj['attrs'], file_path, linenum, applicable))

	# Check that default values match up
	if 'default' in config_obj['attrs']:
	config_default = config_obj['attrs']['default']
	if config_name in all_defines:
	defines_obj = all_defines[config_name]
	if config_default not in defines_obj and (config_name not in resolved_defines or config_default not in resolved_defines[config_name]):
	if '/' in config_default or ' ' in config_default:
	continue
	# There _may_ be multiple matching defines, but arbitrarily display just one in the error message
	first_define_value = get_first_dict_key(defines_obj)
	first_define_file_path, first_define_linenum = defines_obj[first_define_value]
	errors.append(Exception('Found {} at {}:{} with a default of {}, but #define says {} (at {}:{})'.format(config_name, file_path, linenum, config_default, first_define_value, first_define_file_path, first_define_linenum)))
	elif config_obj['attrs']['type'] == "bool" and config_default == "0":
	# a bool with a missing #define defaults to 0 (false) anyway
	logger.info('Found {} (bool) at {}:{} with a default of {}, but no matching #define found'.format(config_name, file_path, linenum, config_default))
	else:
	errors.append(Exception('Found {} at {}:{} with a default of {}, but no matching #define found'.format(config_name, file_path, linenum, config_default)))

	# All settings in "host" should also be in "all"
	for config_name, config_obj in chips_all_configs["host"].items():
	if config_name not in chips_all_configs["all"]:
	file_path = os.path.join(scandir, config_obj['filename'])
	linenum = config_obj['line_number']
	errors.append(Exception("Found 'host' config {} at {}:{}, but no matching non-host config found".format(config_name, file_path, linenum)))

	# Any chip-specific settings should not be in "all"
	for chip in CHIP_NAMES:
	for config_name, chip_config_obj in chips_all_configs[chip].items():
	if config_name in chips_all_configs["all"]:
	all_config_obj = chips_all_configs["all"][config_name]
	chip_file_path = os.path.join(scandir, chip_config_obj['filename'])
	chip_linenum = chip_config_obj['line_number']
	all_file_path = os.path.join(scandir, all_config_obj['filename'])
	all_linenum = all_config_obj['line_number']
	errors.append(Exception("'{}' config {} at {}:{} also found at {}:{}".format(chip, config_name, chip_file_path, chip_linenum, all_file_path, all_linenum)))

	def build_mismatch_exception_message(name, thing, config_obj1, value1, config_obj2, value2):
	obj1_filepath = os.path.join(scandir, config_obj1['filename'])
	obj2_filepath = os.path.join(scandir, config_obj2['filename'])
	return "'{}' {} mismatch at {}:{} ({}) and {}:{} ({})".format(name, thing, obj1_filepath, config_obj1['line_number'], value1, obj2_filepath, config_obj2['line_number'], value2)

	# Check that any identically-named settings have appropriate matching attributes
	for applicable in chips_all_configs:
	for other in chips_all_configs:
	if other == applicable:
	continue
	for config_name, applicable_config_obj in chips_all_configs[applicable].items():
	if config_name in chips_all_configs[other]:
	other_config_obj = chips_all_configs[other][config_name]
	# Check that fields match
	for field in ['description']:
	applicable_value = applicable_config_obj[field]
	other_value = other_config_obj[field]
	if applicable_value != other_value:
	errors.append(Exception(build_mismatch_exception_message(config_name, field, applicable_config_obj, applicable_value, other_config_obj, other_value)))
	# Check that attributes match
	for attr in applicable_config_obj['attrs']:
	if attr != 'default': # totally fine for defaults to vary per-platform
	applicable_value = applicable_config_obj['attrs'][attr]
	other_value = other_config_obj['attrs'][attr]
	if applicable_value != other_value:
	errors.append(Exception(build_mismatch_exception_message(config_name, "attribute '{}'".format(attr), applicable_config_obj, applicable_value, other_config_obj, other_value)))

	# Raise errors if any were found
	if errors:
	raise ExceptionGroup("Errors in {}".format(outfile), errors)

	# Sort the output alphabetically by name and then by chip
	output_rows = set()
	for chip in (*CHIP_NAMES, "host", "all"):
	if chip in chips_all_configs:
	all_configs = chips_all_configs[chip]
	for config_name in all_configs:
	output_rows.add((config_name, chip))

	with open(outfile, 'w', newline='') as csvfile:
	fieldnames = ('name', 'location', 'platform', 'chip', 'description', 'type') + tuple(sorted(all_attrs - set(['type'])))
	writer = csv.DictWriter(csvfile, fieldnames=fieldnames, extrasaction='ignore', dialect='excel-tab')

	writer.writeheader()
	for config_name, chip in sorted(output_rows):
	config_obj = chips_all_configs[chip][config_name]
	writer.writerow({'name': config_name, 'location': '/{}:{}'.format(config_obj['filename'], config_obj['line_number']), 'platform': "host" if chip == "host" else "rp2", 'chip': chip if chip in CHIP_NAMES else "all", 'description': config_obj['description'], **config_obj['attrs']})