| #!/usr/bin/env python3 |
| # |
| # Copyright (c) 2021 Raspberry Pi (Trading) Ltd. |
| # |
| # SPDX-License-Identifier: BSD-3-Clause |
| # |
| # |
| # Script to scan the Raspberry Pi Pico SDK tree searching for configuration items |
| # Outputs a tab separated file of the configuration item: |
| # name location platform chip description type advanced default depends enumvalues group max min |
| # |
| # Usage: |
| # |
| # tools/extract_configs.py <root of repo> [output file] |
| # |
| # If not specified, output file will be `pico_configs.tsv` |
| |
| |
| import os |
| import sys |
| import re |
| import csv |
| import logging |
| |
| from collections import defaultdict |
| |
| if sys.version_info < (3, 11): |
| # Python <3.11 doesn't have ExceptionGroup, so define a simple one |
| class ExceptionGroup(Exception): |
| def __init__(self, message, errors): |
| message += "\n" + "\n".join(str(e) for e in errors) |
| super().__init__(message) |
| |
| logger = logging.getLogger(__name__) |
| logging.basicConfig(level=logging.INFO) |
| |
| scandir = sys.argv[1] |
| outfile = sys.argv[2] if len(sys.argv) > 2 else 'pico_configs.tsv' |
| |
| BASE_CONFIG_NAME = 'PICO_CONFIG' |
| BASE_CONFIG_RE = re.compile(r'\b{}\b'.format(BASE_CONFIG_NAME)) |
| BASE_CMAKE_CONFIG_NAME = 'PICO_CMAKE_CONFIG' |
| BASE_CMAKE_CONFIG_RE = re.compile(r'\b{}\b'.format(BASE_CMAKE_CONFIG_NAME)) |
| BASE_BUILD_DEFINE_NAME = 'PICO_BUILD_DEFINE' |
| BASE_BUILD_DEFINE_RE = re.compile(r'\b{}\b'.format(BASE_BUILD_DEFINE_NAME)) |
| |
| CONFIG_RE = re.compile(r'//\s+{}:\s+(\w+),\s+([^,]+)(?:,\s+(.*))?$'.format(BASE_CONFIG_NAME)) |
| DEFINE_RE = re.compile(r'#define\s+(\w+)\s+(.+?)(\s*///.*)?$') |
| |
| PROPERTY_TYPE = 'type' |
| PROPERTY_DEFAULT = 'default' |
| PROPERTY_MIN = 'min' |
| PROPERTY_MAX = 'max' |
| PROPERTY_ENUMVALUES = 'enumvalues' |
| PROPERTY_GROUP = 'group' |
| PROPERTY_ADVANCED = 'advanced' |
| PROPERTY_DEPENDS = 'depends' |
| ALLOWED_CONFIG_PROPERTIES = set([PROPERTY_TYPE, PROPERTY_DEFAULT, PROPERTY_MIN, PROPERTY_MAX, PROPERTY_ENUMVALUES, PROPERTY_GROUP, PROPERTY_ADVANCED, PROPERTY_DEPENDS]) |
| |
| PROPERTY_TYPE_INT = 'int' |
| PROPERTY_TYPE_BOOL = 'bool' |
| PROPERTY_TYPE_ENUM = 'enum' |
| |
| CHIP_NAMES = ["rp2040", "rp2350"] |
| |
| chips_all_configs = defaultdict(dict) |
| all_attrs = set() |
| chips_all_descriptions = defaultdict(dict) |
| chips_all_defines = defaultdict(dict) |
| |
| |
| def get_first_dict_key(some_dict): |
| return list(some_dict.keys())[0] |
| |
| def look_for_integer_define(config_name, attr_name, attr_str, file_path, linenum, applicable): |
| defined_str = None |
| if re.match('^\w+$', attr_str): |
| # See if we have a matching define |
| all_defines = chips_all_defines[applicable] |
| if attr_str in all_defines: |
| # There _may_ be multiple matching defines, but arbitrarily choose just one |
| defined_str = get_first_dict_key(all_defines[attr_str]) |
| else: |
| if applicable == 'all': |
| # Look for it in the chip-specific defines |
| found_chips = [] |
| missing_chips = [] |
| for chip in CHIP_NAMES: |
| all_defines = chips_all_defines[chip] |
| if attr_str in all_defines: |
| found_chips.append(chip) |
| # There _may_ be multiple matching defines, but arbitrarily choose just one |
| defined_str = get_first_dict_key(all_defines[attr_str]) |
| else: |
| missing_chips.append(chip) |
| # Report if it's found for some chips but not others (but ignore if it's missing for all) |
| if found_chips and missing_chips: |
| logger.info('{} at {}:{} has {} value "{}" which has a matching define for {} but not for {}'.format(config_name, file_path, linenum, attr_name, attr_str, found_chips, missing_chips)) |
| return defined_str |
| |
| def ValidateAttrs(config_name, config_attrs, file_path, linenum, applicable): |
| type_str = config_attrs.get(PROPERTY_TYPE, PROPERTY_TYPE_INT) |
| errors = [] |
| |
| # Validate attrs |
| for key in config_attrs.keys(): |
| if key not in ALLOWED_CONFIG_PROPERTIES: |
| errors.append(Exception('{} at {}:{} has unexpected property "{}"'.format(config_name, file_path, linenum, key))) |
| |
| str_values = dict() |
| parsed_values = dict() |
| if type_str == PROPERTY_TYPE_INT: |
| assert PROPERTY_ENUMVALUES not in config_attrs |
| |
| for attr_name in (PROPERTY_MIN, PROPERTY_MAX, PROPERTY_DEFAULT): |
| str_values[attr_name] = config_attrs.get(attr_name, None) |
| if str_values[attr_name] is not None: |
| try: |
| m = re.match(r'^(\d+)e(\d+)$', str_values[attr_name].lower()) |
| if m: |
| parsed_values[attr_name] = int(m.group(1)) * 10**int(m.group(2)) |
| else: |
| parsed_values[attr_name] = int(str_values[attr_name], 0) |
| except ValueError: |
| defined_str = look_for_integer_define(config_name, attr_name, str_values[attr_name], file_path, linenum, applicable) |
| if defined_str is not None: |
| try: |
| m = re.match(r'^(\d+)e(\d+)$', str_values[attr_name].lower()) |
| if m: |
| parsed_values[attr_name] = int(m.group(1)) * 10**int(m.group(2)) |
| else: |
| parsed_values[attr_name] = int(defined_str, 0) |
| except ValueError: |
| logger.info('{} at {}:{} has {} value "{}" which resolves to the non-integer value "{}"'.format(config_name, file_path, linenum, attr_name, str_values[attr_name], defined_str)) |
| else: |
| logger.info('{} at {}:{} has non-integer {} value "{}"'.format(config_name, file_path, linenum, attr_name, str_values[attr_name])) |
| for (small_attr, large_attr) in ( |
| (PROPERTY_MIN, PROPERTY_MAX), |
| (PROPERTY_MIN, PROPERTY_DEFAULT), |
| (PROPERTY_DEFAULT, PROPERTY_MAX), |
| ): |
| if small_attr in parsed_values and large_attr in parsed_values and parsed_values[small_attr] > parsed_values[large_attr]: |
| errors.append(Exception('{} at {}:{} has {} {} > {} {}'.format(config_name, file_path, linenum, small_attr, str_values[small_attr], large_attr, str_values[large_attr]))) |
| |
| elif type_str == PROPERTY_TYPE_BOOL: |
| assert PROPERTY_MIN not in config_attrs |
| assert PROPERTY_MAX not in config_attrs |
| assert PROPERTY_ENUMVALUES not in config_attrs |
| |
| attr_name = PROPERTY_DEFAULT |
| str_values[attr_name] = config_attrs.get(attr_name, None) |
| if str_values[attr_name] is not None: |
| if (str_values[attr_name] not in ('0', '1')) and (str_values[attr_name] not in all_config_names): |
| logger.info('{} at {}:{} has non-integer {} value "{}"'.format(config_name, file_path, linenum, attr_name, str_values[attr_name])) |
| |
| elif type_str == PROPERTY_TYPE_ENUM: |
| assert PROPERTY_ENUMVALUES in config_attrs |
| assert PROPERTY_MIN not in config_attrs |
| assert PROPERTY_MAX not in config_attrs |
| |
| attr_name = PROPERTY_ENUMVALUES |
| str_values[attr_name] = config_attrs[attr_name] |
| parsed_values[attr_name] = tuple(str_values[attr_name].split('|')) |
| |
| attr_name = PROPERTY_DEFAULT |
| str_values[attr_name] = config_attrs.get(attr_name, None) |
| if str_values[attr_name] is not None: |
| if str_values[attr_name] not in _enumvalues: |
| errors.append(Exception('{} at {}:{} has {} value {} which isn\'t in list of {} {}'.format(config_name, file_path, linenum, attr_name, str_values[attr_name], PROPERTY_ENUMVALUES, str_values[PROPERTY_ENUMVALUES]))) |
| |
| else: |
| errors.append(Exception("Found unknown {} type {} at {}:{}".format(BASE_CONFIG_NAME, type_str, file_path, linenum))) |
| |
| return errors |
| |
| errors = [] |
| |
| # Scan all .c and .h and .S files in the specific path, recursively. |
| |
| for dirpath, dirnames, filenames in os.walk(scandir): |
| for filename in filenames: |
| file_ext = os.path.splitext(filename)[1] |
| if file_ext in ('.c', '.h', '.S'): |
| file_path = os.path.join(dirpath, filename) |
| applicable = "all" |
| for chip in (*CHIP_NAMES, "host"): |
| if "/{}/".format(chip) in dirpath: |
| applicable = chip |
| break |
| |
| with open(file_path, encoding="ISO-8859-1") as fh: |
| linenum = 0 |
| for line in fh.readlines(): |
| linenum += 1 |
| line = line.strip() |
| if BASE_CMAKE_CONFIG_RE.search(line): |
| errors.append(Exception("Found {} at {}:{} ({}) which isn't expected in {} files".format(BASE_CMAKE_CONFIG_NAME, file_path, linenum, line, file_ext))) |
| elif BASE_BUILD_DEFINE_RE.search(line): |
| errors.append(Exception("Found {} at {}:{} ({}) which isn't expected in {} files".format(BASE_BUILD_DEFINE_NAME, file_path, linenum, line, file_ext))) |
| elif BASE_CONFIG_RE.search(line): |
| m = CONFIG_RE.match(line) |
| if not m: |
| if re.match(r"^\s*//\s*// ", line): |
| logger.info("Possible misformatted {} at {}:{} ({})".format(BASE_CONFIG_NAME, file_path, linenum, line)) |
| else: |
| errors.append(Exception("Found misformatted {} at {}:{} ({})".format(BASE_CONFIG_NAME, file_path, linenum, line))) |
| else: |
| config_name = m.group(1) |
| config_description = m.group(2) |
| _attrs = m.group(3) |
| # allow commas to appear inside brackets by converting them to and from NULL chars |
| _attrs = re.sub(r'(\(.+\))', lambda m: m.group(1).replace(',', '\0'), _attrs) |
| |
| if '=' in config_description and not '==' in config_description: |
| errors.append(Exception("For {} at {}:{} the description was set to '{}' - has the description field been omitted?".format(config_name, file_path, linenum, config_description))) |
| all_descriptions = chips_all_descriptions[applicable] |
| if config_description in all_descriptions: |
| errors.append(Exception("Found description {} at {}:{} but it was already used at {}:{}".format(config_description, file_path, linenum, os.path.join(scandir, all_descriptions[config_description]['filename']), all_descriptions[config_description]['line_number']))) |
| else: |
| all_descriptions[config_description] = {'config_name': config_name, 'filename': os.path.relpath(file_path, scandir), 'line_number': linenum} |
| |
| config_attrs = {} |
| prev = None |
| # Handle case where attr value contains a comma |
| for item in _attrs.split(','): |
| if "=" not in item: |
| assert(prev) |
| item = prev + "," + item |
| try: |
| k, v = (i.strip() for i in item.split('=')) |
| except ValueError: |
| errors.append(Exception('{} at {}:{} has malformed value {}'.format(config_name, file_path, linenum, item))) |
| config_attrs[k] = v.replace('\0', ',') |
| all_attrs.add(k) |
| prev = item |
| #print(file_path, config_name, config_attrs) |
| |
| if 'group' not in config_attrs: |
| errors.append(Exception('{} at {}:{} has no group attribute'.format(config_name, file_path, linenum))) |
| |
| #print(file_path, config_name, config_attrs) |
| all_configs = chips_all_configs[applicable] |
| if config_name in all_configs: |
| errors.append(Exception("Found {} at {}:{} but it was already declared at {}:{}".format(config_name, file_path, linenum, os.path.join(scandir, all_configs[config_name]['filename']), all_configs[config_name]['line_number']))) |
| else: |
| all_configs[config_name] = {'attrs': config_attrs, 'filename': os.path.relpath(file_path, scandir), 'line_number': linenum, 'description': config_description} |
| else: |
| m = DEFINE_RE.match(line) |
| if m: |
| name = m.group(1) |
| value = m.group(2) |
| # discard any 'u' qualifier |
| m = re.match(r'^((0x)?\d+)u$', value.lower()) |
| if m: |
| value = m.group(1) |
| else: |
| # discard any '_u(X)' macro |
| m = re.match(r'^_u\(((0x)?\d+)\)$', value.lower()) |
| if m: |
| value = m.group(1) |
| all_defines = chips_all_defines[applicable] |
| if name not in all_defines: |
| all_defines[name] = dict() |
| if value not in all_defines[name]: |
| all_defines[name][value] = set() |
| all_defines[name][value] = (file_path, linenum) |
| |
| all_config_names = set() |
| for all_configs in chips_all_configs.values(): |
| all_config_names.update(all_configs.keys()) |
| |
| # Check for defines with missing PICO_CONFIG entries |
| chips_resolved_defines = defaultdict(dict) |
| for applicable, all_defines in chips_all_defines.items(): |
| for d in all_defines: |
| for define_prefix in ('PICO', 'PARAM', 'CYW43'): |
| if d not in all_config_names and d.startswith(define_prefix+'_'): |
| logger.warning("#define {} is potentially missing a {}: entry".format(d, BASE_CONFIG_NAME)) |
| resolved_defines = chips_resolved_defines[applicable] |
| # resolve "nested defines" - this allows e.g. USB_DPRAM_MAX to resolve to USB_DPRAM_SIZE which is set to 4096 (which then matches the relevant PICO_CONFIG entry) |
| for val in all_defines[d]: |
| if val in all_defines: |
| resolved_defines[d] = all_defines[val] |
| |
| for applicable, all_configs in chips_all_configs.items(): |
| all_defines = chips_all_defines[applicable] |
| resolved_defines = chips_resolved_defines[applicable] |
| for config_name, config_obj in all_configs.items(): |
| file_path = os.path.join(scandir, config_obj['filename']) |
| linenum = config_obj['line_number'] |
| |
| errors.extend(ValidateAttrs(config_name, config_obj['attrs'], file_path, linenum, applicable)) |
| |
| # Check that default values match up |
| if 'default' in config_obj['attrs']: |
| config_default = config_obj['attrs']['default'] |
| if config_name in all_defines: |
| defines_obj = all_defines[config_name] |
| if config_default not in defines_obj and (config_name not in resolved_defines or config_default not in resolved_defines[config_name]): |
| if '/' in config_default or ' ' in config_default: |
| continue |
| # There _may_ be multiple matching defines, but arbitrarily display just one in the error message |
| first_define_value = get_first_dict_key(defines_obj) |
| first_define_file_path, first_define_linenum = defines_obj[first_define_value] |
| errors.append(Exception('Found {} at {}:{} with a default of {}, but #define says {} (at {}:{})'.format(config_name, file_path, linenum, config_default, first_define_value, first_define_file_path, first_define_linenum))) |
| else: |
| errors.append(Exception('Found {} at {}:{} with a default of {}, but no matching #define found'.format(config_name, file_path, linenum, config_default))) |
| |
| # All settings in "host" should also be in "all" |
| for config_name, config_obj in chips_all_configs["host"].items(): |
| if config_name not in chips_all_configs["all"]: |
| file_path = os.path.join(scandir, config_obj['filename']) |
| linenum = config_obj['line_number'] |
| errors.append(Exception("Found 'host' config {} at {}:{}, but no matching non-host config found".format(config_name, file_path, linenum))) |
| |
| # Any chip-specific settings should not be in "all" |
| for chip in CHIP_NAMES: |
| for config_name, chip_config_obj in chips_all_configs[chip].items(): |
| if config_name in chips_all_configs["all"]: |
| all_config_obj = chips_all_configs["all"][config_name] |
| chip_file_path = os.path.join(scandir, chip_config_obj['filename']) |
| chip_linenum = chip_config_obj['line_number'] |
| all_file_path = os.path.join(scandir, all_config_obj['filename']) |
| all_linenum = all_config_obj['line_number'] |
| errors.append(Exception("'{}' config {} at {}:{} also found at {}:{}".format(chip, config_name, chip_file_path, chip_linenum, all_file_path, all_linenum))) |
| |
| def build_mismatch_exception_message(name, thing, config_obj1, value1, config_obj2, value2): |
| obj1_filepath = os.path.join(scandir, config_obj1['filename']) |
| obj2_filepath = os.path.join(scandir, config_obj2['filename']) |
| return "'{}' {} mismatch at {}:{} ({}) and {}:{} ({})".format(name, thing, obj1_filepath, config_obj1['line_number'], value1, obj2_filepath, config_obj2['line_number'], value2) |
| |
| # Check that any identically-named setttings have appropriate matching attributes |
| for applicable in chips_all_configs: |
| for other in chips_all_configs: |
| if other == applicable: |
| continue |
| for config_name, applicable_config_obj in chips_all_configs[applicable].items(): |
| if config_name in chips_all_configs[other]: |
| other_config_obj = chips_all_configs[other][config_name] |
| # Check that fields match |
| for field in ['description']: |
| applicable_value = applicable_config_obj[field] |
| other_value = other_config_obj[field] |
| if applicable_value != other_value: |
| errors.append(Exception(build_mismatch_exception_message(config_name, field, applicable_config_obj, applicable_value, other_config_obj, other_value))) |
| # Check that attributes match |
| for attr in applicable_config_obj['attrs']: |
| if attr != 'default': # totally fine for defaults to vary per-platform |
| applicable_value = applicable_config_obj['attrs'][attr] |
| other_value = other_config_obj['attrs'][attr] |
| if applicable_value != other_value: |
| errors.append(Exception(build_mismatch_exception_message(config_name, "attribute '{}'".format(attr), applicable_config_obj, applicable_value, other_config_obj, other_value))) |
| |
| # Raise errors if any were found |
| if errors: |
| raise ExceptionGroup("Errors in {}".format(outfile), errors) |
| |
| # Sort the output alphabetically by name and then by chip |
| output_rows = set() |
| for chip in (*CHIP_NAMES, "host", "all"): |
| if chip in chips_all_configs: |
| all_configs = chips_all_configs[chip] |
| for config_name in all_configs: |
| output_rows.add((config_name, chip)) |
| |
| with open(outfile, 'w', newline='') as csvfile: |
| fieldnames = ('name', 'location', 'platform', 'chip', 'description', 'type') + tuple(sorted(all_attrs - set(['type']))) |
| writer = csv.DictWriter(csvfile, fieldnames=fieldnames, extrasaction='ignore', dialect='excel-tab') |
| |
| writer.writeheader() |
| for config_name, chip in sorted(output_rows): |
| config_obj = chips_all_configs[chip][config_name] |
| writer.writerow({'name': config_name, 'location': '/{}:{}'.format(config_obj['filename'], config_obj['line_number']), 'platform': "host" if chip == "host" else "rp2", 'chip': chip if chip in CHIP_NAMES else "all", 'description': config_obj['description'], **config_obj['attrs']}) |
