| #!/usr/bin/env python3 | |
| # SPDX-License-Identifier: MIT | |
| """Change impact: which builds and which HIL tests a set of changed files can affect | |
| (build = board families and their examples, HIL = rig boards and their tests). | |
| Stdlib-only (runs on bare CI runners; imports hil_util for the example rosters, never | |
| hil_test/pyserial — test_hil_util.BottomLayer enforces the stdlib closure). Fail-open: | |
| any file no rule classifies forces the full matrix. | |
| JSON: full, boards (name -> 'all' | [tests]), families (bsp families the diff | |
| touches, including ones with no rig board - build-only consumers | |
| sample from these), args (hil_test.py args per config) and args_flasher (the same | |
| args split by each board's flasher, for CI legs that split one rig by flasher). | |
| --manifest prints the selection manifest v1 instead (manifest(); its consumers go | |
| through `check_build.py --select-only`, the build skill's SKILL.md documents it). | |
| THE RULE TABLE. First match wins; answers union per family (build) and per board | |
| (HIL). Every row id needs a `# rule N` marker on its branch (TestRuleTableMatchesTheCode). | |
| `FAM` = the families whose family.cmake references the changed path (CMake only; make | |
| follows it). `DEV`/`HOST`/`DUAL`/`TYPEC`/`ALL` are the example role sets. The Build | |
| families column is PRE-PRUNE: _prune_buildable then intersects each family with what it | |
| can actually build. | |
| | # | Changed path | Build families | Build examples | HIL boards → tests | | |
| | 1 | `docs/`, `.claude/`, `*.md`, `*.rst`, `src/**/*.py`, `LICENSE` | — | — | — | | |
| | 1b | `.gitignore`, `.clang-format`, `.agents`, `.codex/**`, `.idea/**`, `test/{fuzz,unit-test}/**`, `test/hil/test/**`, non-build `.github/**`, packaging manifests | — | — | — | | |
| | 2 | `test/hil/**` (not `test/hil/test/**`) | — | — | all boards → all tests | | |
| | 2b | `tools/drivers_coverage_check.py` | — (local-only tooling, no CI build runs it) | — | — (nothing on the rig runs it) | | |
| | 2c | `.github/ci-pinned-boards.json` | `ALL` | `ALL` | — (CI board data; no rig board's behaviour depends on it) | | |
| | 2d | `tools/membrowse_cli.py`, `tools/code_size.py` | `ALL` | `ALL` | — (size scripts the pinned build legs run; no rig board runs them) | | |
| | 2e | `.claude/skills/build/scripts/check_build.py` (ahead of rule 1) | `ALL` | `ALL` | all boards → all tests (the selection contract CI and agents call) | | |
| | 3 | `src/portable/<port>/dcd_*`, `*_device.[ch]` | `FAM` | `DEV`+`DUAL` | `FAM`'s device-role boards → device+dual tests | | |
| | 4 | `src/portable/<port>/hcd_*`, `*_host.[ch]` | `FAM` | `HOST`+`DUAL` | `FAM`'s host-role boards → host+dual tests | | |
| | 5 | `src/portable/<port>/**` (anything else) | `FAM` | `ALL` | `FAM`'s boards → all their tests | | |
| | 5b | `src/portable/<port>/**` where `FAM` is empty | — | — | — (empty resolves to nothing on BOTH axes) | | |
| | 6 | `hw/bsp/<family>/**` | that family | `ALL` | that family's boards → all tests (a `boards/<board>/` path narrows to that board) | | |
| | 7 | `hw/mcu/<vendor>/**` | `FAM` — empty resolves to nothing (maintainer ruling) | `ALL` | `FAM`'s boards → all tests; empty resolves to nothing (maintainer ruling) | | |
| | 8 | `src/class/<cls>/*_device.[ch]` | `ALL` | examples enabling `CFG_TUD_<CLS>` | device-role boards → HIL tests enabling `CFG_TUD_<CLS>` | | |
| | 9 | `src/class/<cls>/*_host.[ch]` | `ALL` | examples enabling `CFG_TUH_<CLS>` | host-role boards → HIL tests enabling `CFG_TUH_<CLS>` | | |
| | 10 | `src/class/<cls>/**` (shared header) | `ALL` | either, **plus include-edge classes and direct includers** | both roles → same, plus include-edge classes and direct includers | | |
| | 11 | `src/device/**` | `ALL` | `DEV`+`DUAL` | device-role boards → device+dual tests | | |
| | 12 | `src/host/**` | `ALL` | `HOST`+`DUAL` | host-role boards → host+dual tests | | |
| | 12b | `src/typec/**` | `ALL` | examples enabling `CFG_TUC_ENABLED` | — (no rig board runs a typec test) | | |
| | 13 | `examples/<role>/<name>/**` | `ALL` | just `<name>` | if `<name>` is a HIL test: all boards → that test; else nothing | | |
| | 14 | `examples/device/board_test/**` | `ALL` | just `board_test` | all boards → all tests (HIL parking firmware) | | |
| | 15 | `examples/build_system/**`, `examples/CMakeLists.txt`, `examples/<role>/CMakeLists.txt` | `ALL` | `ALL` | all boards → all tests | | |
| | 16 | `src/common/`, `src/osal/`, `src/tusb.[ch]`, `src/tusb_option.h`, `tools/{build,build_utils,change_impact,family_json}.py`, `src/CMakeLists.txt`, `src/tinyusb.mk`, `hw/bsp/{family_support.{cmake,mk},family_rules.mk,zephyr_board_aliases.cmake,board.c,board_api.h,ansi_escape.h}`, `.github/**`, `.circleci/**` | `ALL` | `ALL` | all boards → all tests | | |
| | 16a | `lib/<name>/**` | `ALL` | examples whose own `CMakeLists.txt`/`Makefile` names `lib/<name>` | those examples that are HIL tests, on all boards; empty resolves to nothing | | |
| | 16b | `tools/get_deps.py` | families whose `deps_mandatory`/`deps_optional` entries changed | `ALL` | those families' boards → all tests; a logic change, an `'all'` entry, no base content or a changed token naming no family → full | | |
| | 17 | anything unclassified (no tracked file reaches this — TestNoTrackedFileIsUnclassified) | `ALL` | `ALL` | all boards → all tests (fail-open) | | |
| """ | |
| import argparse | |
| import ast | |
| import contextlib | |
| import functools | |
| import glob | |
| import io | |
| import json | |
| import os | |
| import re | |
| import subprocess | |
| import sys | |
| # tools/ -> repo root is ONE level up. Guarded by TestModuleMove.test_repo_root_guard: | |
| # a wrong parent count here silently re-points every repo-relative glob (it happened | |
| # at the helper/ move). | |
| _REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) | |
| sys.path.insert(0, os.path.join(_REPO_ROOT, 'test', 'hil')) # for `from helper...` | |
| from helper.hil_util import device_tests, dual_tests, host_test | |
| sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) # tools/, for build helpers | |
| import build_utils | |
| import build as build_py | |
| ALL_TESTS = {'device': device_tests, 'dual': dual_tests, 'host': host_test} | |
| # class dir -> config macro suffix exceptions (rule 3); dfu is per-file, handled inline | |
| NET_MACROS = ('ECM_RNDIS', 'NCM') | |
| def _read(path: str) -> str: | |
| """Read a source file with a fixed encoding. The locale's is not it: several tracked | |
| sources carry non-ASCII bytes, and under LC_ALL=C the decode raises UnicodeDecodeError | |
| - a ValueError, which every `except OSError` fail-open below would let through as a | |
| traceback instead of a full matrix.""" | |
| with open(path, encoding='utf-8', errors='replace') as f: | |
| return f.read() | |
| _NONCODE_RE = re.compile( | |
| # LICENSE is anchored and LICENSES/ named separately: a bare `LICENSE` alternative | |
| # also swallowed anything merely STARTING with it (a future LICENSE_extra.c), | |
| # which is the silent-under-selection direction. src/ builds from C sources only; its | |
| # Python is hand-run helpers (dwc2_info.py) | |
| r'^(docs/|\.claude/|.*\.(md|rst)$|src/.*\.py$|LICENSE$|LICENSES/)') | |
| # Repo metadata and tooling that no CI build reads. Enumerated rather than left to | |
| # rule 17, which widens BOTH axes: a PR touching only .gitignore and a README was | |
| # creating 74 cmake legs (each a runner doing checkout + toolchain + get_deps before | |
| # skipping the build) and booking the whole 30-board rig. | |
| # | |
| # Deliberately NOT here, and still full: .circleci/**, .github/workflows/build*.yml, | |
| # .github/actions/**, .github/scripts/** - those decide what gets built. The line is | |
| # "does any Build step read this file", not "is it source": code_size_ci.py is read by | |
| # pr_comment.yml only, so it is metadata. | |
| # | |
| # test/{fuzz,unit-test} have their own jobs (cifuzz.yml, the unit-test pre-commit hook | |
| # and workflow); the Build matrix never compiles them, and test/hil is rule 2. | |
| _META_RE = re.compile( | |
| r'^(' | |
| r'\.(gitignore|gitattributes|clang-format|codespellrc|readthedocs\.yaml)$|' | |
| r'\.pre-commit-config\.yaml$|\.PVS-Studio/|\.oss-scanner/|\.agents$|\.codex/|\.idea/|\.vscode/|' | |
| r'sonar-project\.properties$|library\.json$|pkg\.yml$|repository\.yml$|' | |
| r'version\.yml$|SConscript$|' | |
| r'.*CMakePresets\.json$|hw/bsp/BoardPresets\.json$|examples/west\.yml$|' | |
| # what each board's default configure compiles: the build skill reads it and | |
| # tools/build.py rewrites a row, no firmware build depends on it | |
| r'hw/bsp/family\.json$|' | |
| r'.*/[0-9]+-tinyusb[^/]*\.rules$|tools/usb_drivers/|tools/codespell/|' | |
| # test/hil/test/ holds the harness's own unit tests, not the harness: nothing on | |
| # the rig runs them (pre-commit does, and build.yml runs test_change_impact.py as the | |
| # gate before trusting a selection), so they cannot change what the rig does. | |
| # The harness itself stays under _FULL_RE's test/hil/ prefix. | |
| r'test/(fuzz|unit-test)/|test/hil/test/|' | |
| # .github, minus the build machinery named in _FULL_RE and _CI_BOARDS_RE | |
| r'\.github/(FUNDING\.yml$|labeler\.yml$|' | |
| r'ISSUE_TEMPLATE/|scripts/code_size_ci\.py$|' | |
| r'workflows/(cifuzz|claude|claude-code-review|labeler|' | |
| r'pr_comment|pre-commit|static_analysis|trigger)\.yml$)|' | |
| # tools/ scripts no build invokes (tools/build*.py and local tooling are handled separately) | |
| r'tools/(check_example_pids|file2carray|iar_gen|mksunxi|pcapng_to_corpus)\.py$|' | |
| r'tools/iar_template\.ipcf$' | |
| r')') | |
| # Local-only coverage tooling: no CI build and no rig board runs it (the | |
| # drivers-coverage checker is pre-commit only), so no contribution on either axis. | |
| _LOCAL_TOOLING_RE = re.compile(r'^tools/drivers_coverage_check\.py$') | |
| # CI board data (tools/build.py --ci-pinned-boards): it decides which boards a family's | |
| # build legs compile, so a bad edit can silently drop a family - full build matrix. No | |
| # rig board depends on it. | |
| _CI_BOARDS_RE = re.compile(r'^\.github/ci-pinned-boards\.json$') | |
| # Size scripts every pinned build leg runs (membrowse_cli.py from family_add_membrowse(), | |
| # code_size.py's CI snapshot): which family they break is data, not code, so full build | |
| # matrix as rule 2c. No rig board runs them. | |
| _MEMBROWSE_SCRIPT_RE = re.compile(r'^tools/membrowse_cli\.py$') | |
| _SIZE_SCRIPT_RE = re.compile(r'^tools/code_size\.py$') | |
| MEMBROWSE_TARGET = 'examples-membrowse-upload' | |
| # The selection contract's entry point: CI and agents select through | |
| # `check_build.py --select-only`, so a change to it can change any selection. Ahead | |
| # of the .claude/ non-code rule, which would otherwise swallow it. | |
| _SELECTOR_RE = re.compile(r'^\.claude/skills/build/scripts/check_build\.py$') | |
| _FULL_RE = re.compile( | |
| r'^(src/common/|src/osal/|src/tusb\.c$|src/tusb\.h$|src/tusb_option\.h$|' | |
| # tools/rtt.py is part of the harness, not a standalone tool: hil_util imports it | |
| # at module load, so a break in it breaks every rig run the same way a test/hil/ | |
| # edit can (the pre-commit hil-test hook runs its unit tests for the same reason) | |
| r'test/hil/|tools/rtt\.py$|' | |
| r'\.github/workflows/build.*\.yml$|\.github/actions/|\.github/scripts/|' | |
| # generates the whole CircleCI matrix, same authority as .github/** | |
| r'\.circleci/|' | |
| # rule 16 says `tools/build*.py`; name the siblings the glob implies. build_utils | |
| # and change_impact decide what gets built, so neither can be trusted to narrow its own | |
| # change; family_json runs inside build.py after every default configure, so a | |
| # break in it fails the build the same way | |
| r'tools/(build|build_utils|change_impact|family_json)\.py$|' | |
| # the make twins of family_support.cmake are the same authority for the make legs | |
| r'hw/bsp/(family_support\.(cmake|mk)|family_rules\.mk|zephyr_board_aliases\.cmake|' | |
| r'board_api\.h|board\.c|ansi_escape\.h)$|' | |
| # rule 15 lists examples/<role>/CMakeLists.txt - it registers every target in that | |
| # role, so it was only ever reaching `full` through rule 17's fall-through | |
| r'examples/build_system/|examples/CMakeLists\.txt$|' | |
| r'examples/[^/]+/CMakeLists\.txt$|' | |
| # every firmware compiles these unconditionally (src/CMakeLists.txt, src/tinyusb.mk) | |
| r'src/CMakeLists\.txt$|src/tinyusb\.mk$|' | |
| # board_test is HIL infrastructure, not a test: hil_test.py flashes it to park | |
| # every board (variant boundary + end-of-board teardown), so every board depends on it | |
| r'examples/device/board_test/)') | |
| # --no-renames: with rename detection git reports only a rename's destination, so code | |
| # moved out of an HIL-relevant path would be classified by its new path alone | |
| GIT_DIFF_ARGV = ['git', 'diff', '--no-renames', '--name-only'] | |
| def test_role(test: str) -> str: | |
| return test.split('/', 1)[0] # 'device' | 'dual' | 'host' | |
| def board_roles(board: dict) -> set: | |
| t = board.get('tests', {}) | |
| roles = set() | |
| if t.get('device'): | |
| roles.add('device') | |
| if t.get('host'): | |
| roles.add('host') | |
| if t.get('dual'): | |
| roles.update(('device', 'host')) | |
| for only in t.get('only', []): | |
| r = test_role(only) | |
| roles.update(('device', 'host') if r == 'dual' else (r,)) | |
| return roles | |
| def board_tests(board: dict) -> list: | |
| """Every test this board would run today (mirrors hil_test.test_board's default).""" | |
| t = board.get('tests', {}) | |
| if 'only' in t: | |
| run = list(t['only']) | |
| else: | |
| run = [] | |
| if t.get('device'): | |
| run += device_tests | |
| if t.get('dual'): | |
| run += dual_tests | |
| if t.get('host'): | |
| run += host_test | |
| return [x for x in run if x not in t.get('skip', [])] | |
| def _rg(repo_root: str, *parts: str) -> str: | |
| """A glob pattern rooted at repo_root, with the ROOT escaped and the parts left as | |
| patterns. The root is a filesystem path, not a pattern: a checkout at | |
| /w/pr[1]/tinyusb (a worktree named after a PR, a CI workspace with brackets) makes | |
| an unescaped '[1]' a character class that matches nothing, and every lookup below | |
| then resolves to zero - families=0 instead of 30, i.e. the selector fails CLOSED | |
| and the whole matrix compiles nothing while reporting green.""" | |
| return os.path.join(glob.escape(repo_root), *parts) | |
| # cached: called per changed file x roster board, and the tree doesn't change mid-run | |
| @functools.lru_cache(maxsize=None) | |
| def board_family(board_name: str, repo_root: str): | |
| hits = glob.glob(_rg(repo_root, 'hw/bsp/*/boards', board_name)) | |
| return os.path.basename(os.path.dirname(os.path.dirname(hits[0]))) if hits else None | |
| # `if (OPTION STREQUAL "1")` guards in family_support.cmake, and the option tokens | |
| # a roster entry passes to the build (NAME=VALUE / -DNAME=VALUE) | |
| _CM_IF_RE = re.compile(r'if\s*\(') | |
| _CM_ELSE_RE = re.compile(r'else(if)?\s*\(') | |
| _CM_ENDIF_RE = re.compile(r'endif\s*\(') | |
| _CM_OPT_RE = re.compile(r'if\s*\(\s*\$?\{?([A-Za-z_]\w*)\}?\s+STREQUAL\s+"?1"?\s*\)') | |
| _CM_PORT_RE = re.compile(r'src/portable/((?:[^/\s]+/)?[^/\s]+)/') | |
| _FALSY = ('', '0', 'off', 'false', 'no') | |
| @functools.lru_cache(maxsize=None) | |
| def port_option_gates(repo_root: str) -> dict: | |
| """port dir -> build options that compile it regardless of the board's family | |
| file, e.g. {'analog/max3421': {'MAX3421_HOST'}} from family_support.cmake.""" | |
| gates = {} | |
| try: | |
| text = _read(os.path.join(repo_root, 'hw/bsp/family_support.cmake')) | |
| except OSError: | |
| return gates | |
| stack = [] # one entry per open if(): its option, or None | |
| for line in text.splitlines(): | |
| line = line.strip() | |
| if _CM_IF_RE.match(line): | |
| m = _CM_OPT_RE.match(line) | |
| stack.append(m.group(1) if m else None) | |
| elif _CM_ELSE_RE.match(line): | |
| if stack: | |
| stack[-1] = None # the guard doesn't hold in this branch | |
| elif _CM_ENDIF_RE.match(line): | |
| if stack: | |
| stack.pop() | |
| opts = {o for o in stack if o} | |
| m = _CM_PORT_RE.search(line) | |
| if opts and m: | |
| gates.setdefault(m.group(1), set()).update(opts) | |
| return gates | |
| _CM_SET_RE = re.compile(r'set\s*\(\s*([A-Za-z_]\w*)\s+([^)\s]+)\s*\)') | |
| # cached: called per changed portable file x roster board | |
| @functools.lru_cache(maxsize=None) | |
| def bsp_board_options(board_name: str, repo_root: str) -> frozenset: | |
| """Build options a board turns on in its own BSP: `set(<OPT> <value>)` in | |
| hw/bsp/<family>/boards/<board>/board.cmake, e.g. MAX3421_HOST on the espressif | |
| and rp2040 max3421 boards. CMake only - HIL CI builds nothing with Make, so a | |
| board.mk-only option (e.g. nrf5340dk's MAX3421_HOST) compiles no port here.""" | |
| fam = board_family(board_name, repo_root) | |
| if not fam: | |
| return frozenset() | |
| path = os.path.join(repo_root, 'hw/bsp', fam, 'boards', board_name, 'board.cmake') | |
| try: | |
| text = _read(path) | |
| except OSError: | |
| return frozenset() | |
| out = set() | |
| for line in text.splitlines(): | |
| line = line.strip() | |
| if line.startswith('#'): | |
| continue | |
| m = _CM_SET_RE.match(line) | |
| if m and m.group(2).strip('"').lower() not in _FALSY: | |
| out.add(m.group(1)) | |
| return frozenset(out) | |
| def board_options(board: dict, repo_root: str) -> set: | |
| """Build options a board has truthy: each variant's defines (NAME=VALUE) and raw | |
| CFLAGS (-DNAME=VALUE), plus whatever its own board.cmake sets (a board can enable a | |
| gated port without the roster saying so). A board whose option is always on carries | |
| a single variant named after itself - metro_m4_express and MAX3421_HOST=1, which is | |
| what makes it the one rig board that compiles hcd_max3421.c.""" | |
| toks = [] | |
| for v in board.get('variant', []): | |
| toks += list(v.get('defines', [])) | |
| toks += v.get('flags', '').split() | |
| out = set(bsp_board_options(board['name'], repo_root)) | |
| for t in toks: | |
| name, _, val = (t[2:] if t.startswith('-D') else t).partition('=') | |
| if name and val.strip().strip('"').lower() not in _FALSY: | |
| out.add(name.strip()) | |
| return out | |
| @functools.lru_cache(maxsize=None) | |
| def path_families(rel_dir: str, repo_root: str) -> set: | |
| """Board families whose family.cmake (or espressif component CMakeLists) | |
| references rel_dir at a directory boundary. CMake only, on every axis: CMake | |
| is the first-class build system and Make follows it, so family.mk is never | |
| read - a port wired up in family.mk alone (microchip/pic32mz) is built by no | |
| CI job and resolves to nothing. Boundary = '/', whitespace, quote, paren, | |
| brace or end: `${TOP}/hw/mcu/nordic/nrfx` has no trailing slash, while bare | |
| 'microchip/pic' must not match '.../microchip/pic32mz/...'.""" | |
| pat = re.compile(re.escape(rel_dir) + r'(?=[/\s"\')}]|$)', re.M) | |
| return {fam for fam, text in _family_file_texts(repo_root) if pat.search(text)} | |
| @functools.lru_cache(maxsize=None) | |
| def _family_file_texts(repo_root: str) -> tuple: | |
| """((family, text), ...) for every family.cmake and espressif component | |
| CMakeLists.txt, read once. path_families is called per distinct directory in the | |
| diff and its own cache only helps repeats: a 6,000-file hw/mcu dep bump re-read | |
| these 84 files 99,892 times (2.2 s) before this.""" | |
| bsp_root = os.path.join(repo_root, 'hw/bsp') # escaped by _rg below | |
| out = [] | |
| for f in sorted(glob.glob(_rg(bsp_root, '*/family.cmake')) + | |
| glob.glob(_rg(bsp_root, '*/components/*/CMakeLists.txt'))): | |
| try: | |
| out.append((os.path.relpath(f, bsp_root).split(os.sep, 1)[0], _read(f))) | |
| except OSError: | |
| pass | |
| return tuple(out) | |
| def port_families(port_dir: str, repo_root: str) -> set: | |
| # 'portable/', not 'src/portable/': family.cmake always spells the full literal | |
| # path ('${TOP}/src/portable/...'), but espressif's component CMakeLists.txt | |
| # assigns 'src' into a ${tusb_src} variable first (`${tusb_src}/portable/...`), | |
| # so a leading 'src/' in the needle would never match there and silently drop | |
| # espressif boards (see TestRealRosterPortFamilies). | |
| return path_families('portable/' + port_dir, repo_root) | |
| def mcu_families(path: str, repo_root: str) -> set: | |
| """Families referencing a changed hw/mcu path: longest resolving dir prefix, | |
| hw/mcu/<vendor>/<sub>/... down to hw/mcu/<vendor>. The same scan as port_families on | |
| a second tree: hw/mcu is mostly gitignored deps, so only its tracked dirs reach a diff.""" | |
| parts = path.split('/') | |
| for n in range(len(parts) - 1, 2, -1): | |
| fams = path_families('/'.join(parts[:n]), repo_root) | |
| if fams: | |
| return fams | |
| return set() | |
| GET_DEPS_PATH = 'tools/get_deps.py' | |
| _DEPS_DICTS = ('deps_mandatory', 'deps_optional') | |
| def _deps_split(text: str): | |
| """(module dump with the two dep-dict assigns removed, {dict name: entries}). | |
| Parsed with ast, never exec'd: this runs on PR content.""" | |
| mod = ast.parse(text) | |
| dicts, rest = {}, [] | |
| for node in mod.body: | |
| if (isinstance(node, ast.Assign) and len(node.targets) == 1 and | |
| isinstance(node.targets[0], ast.Name) and | |
| node.targets[0].id in _DEPS_DICTS and isinstance(node.value, ast.Dict)): | |
| dicts[node.targets[0].id] = ast.literal_eval(node.value) | |
| else: | |
| rest.append(node) | |
| mod.body = rest | |
| # annotate_fields=False keeps the dump readable-length; line numbers are not | |
| # included unless asked for, so reformatting alone never reads as a logic change | |
| return ast.dump(mod, annotate_fields=False), dicts | |
| # Family tokens in tools/get_deps.py that name no hw/bsp directory. get_deps matches a | |
| # token against a requested family name verbatim (`f in deps_optional[d][2].split()`), | |
| # so a token like these matches nothing - a stale spelling in get_deps.py, not a | |
| # selector bug, and out of scope to change here. Pinned so that any OTHER unresolvable | |
| # token (real drift) falls open to the full matrix instead of silently selecting | |
| # nothing, and so TestOrphanInvariant fails the day one is fixed or a new one appears. | |
| # sam3x, samd21, samd51, same5x -> pre-rename spellings, listed alongside the current | |
| # samd2x_l2x / samd5x_e5x / same7x in the same entry | |
| # stm32l1, stm32l5 -> no hw/bsp family in the tree at all | |
| _DEPS_ALIAS_TOKENS = frozenset({'sam3x', 'samd21', 'samd51', 'same5x', | |
| 'stm32l1', 'stm32l5'}) | |
| def get_deps_changed_families(base_text: str, head_text: str, repo_root: str): | |
| """Families whose tools/get_deps.py dep entries changed between two versions of | |
| the file, or None meaning 'cannot tell - use the full matrix'. | |
| None on: anything outside deps_mandatory/deps_optional differing (a logic change | |
| to get_deps affects every family), a mandatory `'all'` entry changing, a token | |
| that resolves to no family and is not a known alias, or text that will not parse. | |
| Callers with no base content at all - `--diff-file` mode has no git and therefore | |
| no merge-base blob - pass None themselves. | |
| An entry that is added, removed or edited contributes the family tokens of BOTH | |
| sides (a removed entry has only a base side). The two dicts are diffed SEPARATELY: | |
| merging them first would hide a move between deps_mandatory and deps_optional, | |
| which changes which families fetch the dep even though the value is untouched.""" | |
| try: | |
| base_rest, base_d = _deps_split(base_text) | |
| head_rest, head_d = _deps_split(head_text) | |
| except (SyntaxError, ValueError, TypeError): | |
| return None | |
| if base_rest != head_rest: | |
| return None | |
| toks = set() | |
| for name in _DEPS_DICTS: | |
| base_x, head_x = base_d.get(name, {}), head_d.get(name, {}) | |
| for key in set(base_x) | set(head_x): | |
| if base_x.get(key) == head_x.get(key): | |
| continue | |
| for entry in (base_x.get(key), head_x.get(key)): | |
| if entry and len(entry) > 2: | |
| toks.update(str(entry[2]).split()) | |
| if 'all' in toks: | |
| return None | |
| fams = set(all_bsp_families(repo_root)) | |
| if toks - fams - _DEPS_ALIAS_TOKENS: | |
| # a changed entry we cannot map to a family. "changed but unmappable" is NOT | |
| # "nothing changed": reading it as the latter empties the entire build matrix | |
| # for a dep bump, so fall open instead | |
| return None | |
| return toks & fams | |
| _CLS_INC_RE = re.compile(r'#\s*include\s*[<"]class/([^/"<>]+)/([^"<>]+)[">]') | |
| @functools.lru_cache(maxsize=None) | |
| def class_include_edges(repo_root: str) -> dict: | |
| """'<class>/<header>' -> {(class dir, file)} of the other classes' files that include | |
| it. A class header pulled in by a second class ships in every firmware enabling that | |
| second class: src/class/midi/midi{,2}_{device,host}.h include class/audio/audio.h, and | |
| net_device.h includes class/cdc/cdc.h. The class rule derives macros from the | |
| directory name alone, so without this edge a change to the included header | |
| selects only its own class's examples - and on a board that skips those (e.g. | |
| metro_m4_express skips audio_test_freertos), nothing at all. The including file | |
| is kept because its own macro can differ from the directory's: midi2_device.h is | |
| compiled under CFG_TUD_MIDI2, not CFG_TUD_MIDI (class_macros). | |
| Derived from the actual #include lines rather than a hand-written table so it | |
| cannot rot when a class picks up or drops a cross-class include.""" | |
| edges = {} | |
| for f in sorted(glob.glob(_rg(repo_root, 'src/class/*/*.[ch]'))): | |
| cls = os.path.basename(os.path.dirname(f)) | |
| try: | |
| text = _read(f) | |
| except OSError: | |
| continue | |
| for inc_cls, inc_hdr in _CLS_INC_RE.findall(text): | |
| if inc_cls != cls: | |
| edges.setdefault(f'{inc_cls}/{inc_hdr}', set()).add((cls, os.path.basename(f))) | |
| return edges | |
| def edge_macros(via, prefix: str) -> list: | |
| """Config macros of the including files class_include_edges() returned.""" | |
| return [m for c, b in via for m in class_macros(c, b, prefix)] | |
| @functools.lru_cache(maxsize=None) | |
| def example_class_includes(repo_root: str) -> dict: | |
| """'<class>/<header>' -> examples whose own sources #include it. That header ships in | |
| the example whatever its config enables: host/bare_api includes class/hid/hid.h and | |
| device/midi2_device's descriptors include class/audio/audio.h, enabling neither class.""" | |
| out = {} | |
| for ex in all_examples(repo_root): | |
| for f in sorted(glob.glob(_rg(repo_root, 'examples', ex, '**', '*.[ch]'), recursive=True)): | |
| try: | |
| text = _read(f) | |
| except OSError: | |
| continue | |
| for inc_cls, inc_hdr in _CLS_INC_RE.findall(text): | |
| out.setdefault(f'{inc_cls}/{inc_hdr}', set()).add(ex) | |
| return out | |
| _CLS_STEM_RE = re.compile(r'(.*?)(?:_(?:device|host))?\.[ch]$') | |
| def class_macros(cls: str, base: str, prefix: str) -> list: | |
| """Config macros that compile a class dir's code, for role prefix TUD/TUH. | |
| `base` refines dfu (it splits DFU from DFU_RUNTIME per file) and adds the file's | |
| own macro where that differs from the directory's.""" | |
| if cls == 'net': | |
| return [f'CFG_{prefix}_{m}' for m in NET_MACROS] | |
| if cls == 'dfu': | |
| if base.startswith('dfu_rt'): | |
| return [f'CFG_{prefix}_DFU_RUNTIME'] | |
| if base.startswith('dfu_device') or base.startswith('dfu_host'): | |
| return [f'CFG_{prefix}_DFU'] | |
| return [f'CFG_{prefix}_DFU', f'CFG_{prefix}_DFU_RUNTIME'] | |
| out = [f'CFG_{prefix}_{cls.upper()}'] | |
| # A class directory can hold more than one class. src/class/midi ships MIDI 1.0 | |
| # AND MIDI 2.0: midi2_device.c is `#if CFG_TUD_ENABLED && CFG_TUD_MIDI2`, and | |
| # examples/device/midi2_device is the only example that enables it - so the | |
| # directory macro alone selected the midi_test examples, which do not compile the | |
| # changed file, and none of the ones that do. Union, never replace: the file may | |
| # still be pulled in by the directory's own macro, and over-selecting costs a build | |
| # while under-selecting merges a break. | |
| m = _CLS_STEM_RE.match(base) | |
| if m and m.group(1) and m.group(1) != cls: | |
| out.append(f'CFG_{prefix}_{m.group(1).upper()}') | |
| return out | |
| # A define is OFF only when its value is a literal zero (0, 00, (0)), optionally | |
| # followed by a comment. Anything else counts as ON - including a value this cannot | |
| # evaluate, e.g. `#define CFG_TUH_MIDI CFG_TUH_DEVICE_MAX` (examples/host/midi_rx). | |
| # Fail-open: reading such a define as OFF made midi_host.c select zero families and | |
| # let a compile break merge green. | |
| # | |
| # A macro defined more than once is ON if ANY of its defines is non-zero, because | |
| # the preprocessor branches are not evaluated here: uac2_speaker_fb defines | |
| # CFG_TUD_HID 1 under `#if CFG_AUDIO_DEBUG` and 0 in the #else, and the default | |
| # build (CFG_AUDIO_DEBUG defaults to 1) compiles the HID class in. Deciding on the | |
| # LAST/only match found made that example invisible to CFG_TUD_HID changes. | |
| _DEF_VALUE = r'^[ \t]*#[ \t]*define[ \t]+{}[ \t]+(\S[^\n]*?)[ \t]*$' | |
| _DEF_ZERO_VALUE = re.compile(r'\(?\s*0+\s*\)?\s*(?://.*|/\*.*)?') | |
| # Shared rule-recognition primitives. The two classifiers walk the same diff with | |
| # different answers, but they must RECOGNISE the same things: one copy each, so a | |
| # new naming convention cannot land in one walk and be missed by the other. | |
| _PORT_PATH_RE = re.compile(r'src/portable/((?:[^/]+/)?[^/]+)/') | |
| def _port_roles(base: str) -> set: | |
| """Which USB role a src/portable file serves, from its name: dcd_*/ *_device is | |
| the device-controller side, hcd_*/ *_host the host side, anything else (shared | |
| headers, glue) both.""" | |
| if re.match(r'(dcd_|.*_device)', base): | |
| return {'device'} | |
| if re.match(r'(hcd_|.*_host)', base): | |
| return {'host'} | |
| return {'device', 'host'} | |
| def _class_roles(base: str) -> set: | |
| """Same question for a src/class file: <cls>_device.[ch] / <cls>_host.[ch], | |
| else both - the class's shared header ships in either role.""" | |
| if re.search(r'_device\.[ch]$', base): | |
| return {'device'} | |
| if re.search(r'_host\.[ch]$', base): | |
| return {'host'} | |
| return {'device', 'host'} | |
| @functools.lru_cache(maxsize=None) | |
| def _config_text(cfg_path: str) -> str: | |
| """An example's tusb_config.h, read once. Every class path re-asks the same 46 | |
| configs on both axes, so the reads go up with the diff: 4,240 of the same 46 files | |
| for a diff touching all of src/class (0.48s -> 0.13s), and they cannot change | |
| mid-run. Cached here rather than on _config_enables so the macros argument stays an | |
| ordinary list at every call site.""" | |
| try: | |
| with open(cfg_path, encoding='utf-8', errors='replace') as f: | |
| return f.read() | |
| except OSError: | |
| return '' | |
| def _config_enables(cfg_path: str, macros) -> bool: | |
| text = _config_text(cfg_path) | |
| if not text: | |
| return False | |
| for m in macros: | |
| for value in re.findall(_DEF_VALUE.format(m), text, re.M): | |
| if not _DEF_ZERO_VALUE.fullmatch(value): | |
| return True | |
| return False | |
| def examples_enabling(pool, macros, repo_root: str) -> set: | |
| """The 'role/name' entries of `pool` whose src/tusb_config.h turns any of | |
| `macros` on. The pool differs per classifier (HIL test lists vs every example), | |
| the question does not.""" | |
| return {ex for ex in pool | |
| if _config_enables(os.path.join(repo_root, 'examples', ex, 'src', | |
| 'tusb_config.h'), macros)} | |
| # cached: called per changed lib file, and the tree doesn't change mid-run | |
| @functools.lru_cache(maxsize=None) | |
| def lib_examples(lib_name: str, repo_root: str) -> set: | |
| """Examples whose OWN examples/<role>/<name>/{CMakeLists.txt,Makefile} references | |
| lib/<lib_name> at a directory boundary (same boundary rule as path_families, so | |
| 'lib/net' cannot inherit lib/networking's example). | |
| Per-example on purpose: lib/SEGGER_RTT is named by family_support.cmake's | |
| LOGGER=rtt plumbing, which no CI example build turns on (all three references - | |
| family_support.cmake, family_support.mk, rp2040/family.cmake - sit inside a | |
| LOGGER=rtt guard), so a family-file scan would wrongly narrow it to three families | |
| instead of answering 'nobody'. | |
| The whole example TREE is scanned, not just its top-level files: examples/host/ | |
| msc_file_explorer_freertos/src/CMakeLists.txt names lib/embedded-cli, and that | |
| example survived only because its top-level file happens to name it too.""" | |
| pat = re.compile(re.escape('lib/' + lib_name) + r'(?=[/\s"\')}]|$)', re.M) | |
| out = set() | |
| for ex in all_examples(repo_root): | |
| # the two filenames directly: '**/*' enumerated 489 entries per lib against a | |
| # clean tree to use 107, and grows without bound once `make BOARD=... all` has | |
| # written examples/<role>/<name>/_build/ - which is where a local build runs | |
| for f in sorted(glob.glob(_rg(repo_root, 'examples', ex, '**', 'CMakeLists.txt'), | |
| recursive=True) + | |
| glob.glob(_rg(repo_root, 'examples', ex, '**', 'Makefile'), | |
| recursive=True)): | |
| try: | |
| text = _read(f) | |
| except OSError: | |
| continue | |
| if pat.search(text): | |
| out.add(ex) | |
| break | |
| return out | |
| def roster_only_tests(all_boards) -> set: | |
| """Test paths that only appear in a roster board's tests.only list (e.g. | |
| espressif boards), not in the shared device/dual/host_test lists.""" | |
| out = set() | |
| for b in all_boards: | |
| out.update(b.get('tests', {}).get('only', [])) | |
| return out | |
| def class_examples(macros, role: str, repo_root: str, extra_tests: set) -> set: | |
| """Tests (from role's + dual lists, plus roster-only-list tests of that role) | |
| whose example config enables any macro.""" | |
| return examples_enabling(role_tests({role}, extra_tests), macros, repo_root) | |
| def role_tests(roles: set, extras: set) -> set: | |
| """Every test for the given role(s): each role's own list + dual tests, | |
| plus roster-only-list tests (extras) matching those roles or 'dual'.""" | |
| pool = set(dual_tests) | |
| for r in roles: | |
| pool |= set(ALL_TESTS[r]) | |
| pool |= {t for t in extras if test_role(t) in roles or test_role(t) == 'dual'} | |
| return pool | |
| class _Sel: | |
| """Accumulates contributions. board->set(tests) plus 'all-board' markers.""" | |
| def __init__(self): | |
| self.full = False | |
| self.by_board = {} # name -> set of tests, or 'all' | |
| self.roles = set() # roles touched by any contribution | |
| self.families = set() # bsp families touched (incl. off-rig ones: build-only consumers) | |
| self.reasons = [] | |
| def add(self, boards, tests, reason): | |
| """tests: 'all' or iterable of test paths.""" | |
| self.reasons.append(reason) | |
| for b in boards: | |
| cur = self.by_board.get(b) | |
| if tests == 'all' or cur == 'all': | |
| self.by_board[b] = 'all' | |
| else: | |
| self.by_board[b] = (cur or set()) | set(tests) | |
| def force_full(self, reason): | |
| self.full = True | |
| self.reasons.append(reason) | |
| def _classify_one(path, repo_root, roster_boards, extras: set, s: _Sel, | |
| get_deps_families=None): | |
| base = os.path.basename(path) | |
| if _SELECTOR_RE.match(path): # rule 2e | |
| s.force_full(f'{path}: the selection contract -> full matrix') | |
| return | |
| if _NONCODE_RE.match(path) or _META_RE.match(path): | |
| s.reasons.append(f'{path}: non-code, no contribution') | |
| return | |
| if _LOCAL_TOOLING_RE.match(path): # rule 2b | |
| s.reasons.append(f'{path}: local coverage tooling, no HIL contribution') | |
| return | |
| if _CI_BOARDS_RE.match(path): # rule 2c | |
| s.reasons.append(f'{path}: CI board data, no HIL contribution') | |
| return | |
| if _MEMBROWSE_SCRIPT_RE.match(path) or _SIZE_SCRIPT_RE.match(path): # rule 2d | |
| s.reasons.append(f'{path}: CI size script, no HIL contribution') | |
| return | |
| if _FULL_RE.match(path): | |
| s.force_full(f'{path}: core/infra -> full matrix') | |
| return | |
| if path == GET_DEPS_PATH: | |
| if get_deps_families is None: | |
| s.force_full(f'{path}: dep changes not resolvable -> full matrix') | |
| return | |
| if not get_deps_families: | |
| s.reasons.append(f'{path}: no build-family dependency changed, no contribution') | |
| return | |
| fams = sorted(get_deps_families) | |
| s.families.update(fams) | |
| boards = [b['name'] for b in roster_boards | |
| if board_family(b['name'], repo_root) in get_deps_families] | |
| s.roles.update(('device', 'host')) | |
| s.add(boards, 'all', f'{path}: dep entries changed -> families {fams} -> ' | |
| f'boards {boards}') | |
| return | |
| m = _PORT_PATH_RE.match(path) | |
| if m: | |
| port = m.group(1) | |
| roles = _port_roles(base) | |
| fams = port_families(port, repo_root) | |
| if not fams: | |
| # empty means empty (maintainer ruling), same reading as hw/mcu and as the | |
| # build walk: no family's build references this port, so nothing compiles it | |
| # and there is nothing to run. Forcing the full 30-board rig here bought no | |
| # coverage at all - the build side selected zero families for the same path. | |
| # Live for src/portable/template and the two microchip pic ports; | |
| # TestPortFamiliesCoverage is the drift guard for a port that stops resolving. | |
| s.reasons.append(f'{path}: port {port} maps to no board family, no contribution') | |
| return | |
| s.families.update(fams) | |
| # a board can also pull the port in through a build option (e.g. MAX3421_HOST=1 | |
| # from the roster on metro_m4_express, or from its own board.cmake), which its | |
| # family file never names | |
| gates = port_option_gates(repo_root).get(port, set()) | |
| boards = [b['name'] for b in roster_boards | |
| if (board_family(b['name'], repo_root) in fams or | |
| (gates and board_options(b, repo_root) & gates)) and (board_roles(b) & roles)] | |
| tests = role_tests(roles, extras) | |
| s.roles.update(roles) | |
| why = f'{path}: port {port} -> families {sorted(fams)}' | |
| if gates: | |
| why += f' + option {sorted(gates)}' | |
| s.add(boards, tests, f'{why} -> boards {boards} ({"/".join(sorted(roles))})') | |
| return | |
| m = re.match(r'src/class/([^/]+)/', path) | |
| if m: | |
| cls = m.group(1) | |
| roles = _class_roles(base) | |
| # this file's own class, plus any class whose headers include it | |
| via = sorted(class_include_edges(repo_root).get(f'{cls}/{base}', ())) | |
| def macros(prefix): | |
| return class_macros(cls, base, prefix) + edge_macros(via, prefix) | |
| tests = set() | |
| if 'device' in roles: | |
| tests |= class_examples(macros('TUD'), 'device', repo_root, extras) | |
| if 'host' in roles: | |
| tests |= class_examples(macros('TUH'), 'host', repo_root, extras) | |
| # an example including the header directly ships it, whatever its config enables | |
| direct = set(example_class_includes(repo_root).get(f'{cls}/{base}', ())) & role_tests(roles, extras) | |
| tests |= direct | |
| boards = [b['name'] for b in roster_boards if board_roles(b) & roles] | |
| s.roles.update(roles) | |
| why = f'{path}: class {cls}' + (f' (+ included by {sorted({c for c, _ in via})})' if via else '') + \ | |
| (f' (+ included directly by {sorted(direct)})' if direct else '') | |
| s.add(boards, tests, f'{why} -> {sorted(tests)} ({"/".join(sorted(roles))})') | |
| return | |
| m = re.match(r'src/(device|host)/', path) | |
| if m: | |
| role = m.group(1) | |
| boards = [b['name'] for b in roster_boards if role in board_roles(b)] | |
| s.roles.add(role) | |
| s.add(boards, role_tests({role}, extras), f'{path}: core {role} stack -> all {role} tests') | |
| return | |
| m = re.match(r'hw/bsp/([^/]+)/(?:boards/([^/]+)/)?', path) | |
| if m: | |
| fam, brd = m.group(1), m.group(2) | |
| s.families.add(fam) | |
| if brd: | |
| boards = [b['name'] for b in roster_boards if b['name'] == brd] | |
| why = f'{path}: bsp board {brd}' | |
| else: | |
| boards = [b['name'] for b in roster_boards | |
| if board_family(b['name'], repo_root) == fam] | |
| why = f'{path}: bsp family {fam}' | |
| s.roles.update(('device', 'host')) | |
| s.add(boards, 'all', f'{why} -> boards {boards}') | |
| return | |
| if re.match(r'hw/mcu/', path): | |
| fams = mcu_families(path, repo_root) | |
| if not fams: | |
| # empty means empty (maintainer ruling): if no family's build references | |
| # the path, no build consumes the change - there is nothing to compile, | |
| # so there is nothing to run either. TestOrphanInvariant's | |
| # test_tracked_mcu_vendors_resolve is the drift guard: a real vendor dir | |
| # that stops resolving fails pre-commit instead of silently vanishing | |
| s.reasons.append(f'{path}: hw/mcu path resolves to no family, no contribution') | |
| return | |
| s.families.update(fams) | |
| boards = [b['name'] for b in roster_boards | |
| if board_family(b['name'], repo_root) in fams] | |
| s.roles.update(('device', 'host')) | |
| s.add(boards, 'all', f'{path}: mcu dir -> families {sorted(fams)} -> boards {boards}') | |
| return | |
| m = re.match(r'lib/([^/]+)/', path) | |
| if m: | |
| lib = m.group(1) | |
| # only the tests whose example builds the lib, and only those the rig runs | |
| tests = {e for e in lib_examples(lib, repo_root) | |
| if any(e in pool for pool in ALL_TESTS.values()) or e in extras} | |
| if lib == 'SEGGER_RTT': | |
| # no example names this lib, but a board whose roster entry says | |
| # "logger": "rtt" (variant defines LOGGER=rtt) reads EVERY test's console | |
| # through it -- a break here silently breaks all of that board's rows | |
| rtt_boards = [b['name'] for b in roster_boards if b.get('logger') == 'rtt'] | |
| if rtt_boards: | |
| s.roles.update(('device', 'host')) | |
| s.add(rtt_boards, 'all', | |
| f'{path}: SEGGER_RTT is the rtt console on {rtt_boards} -> all tests') | |
| return | |
| if not tests: | |
| s.reasons.append(f'{path}: lib {lib} used by no HIL test, no contribution') | |
| return | |
| roles = set() | |
| for test in tests: | |
| r = test_role(test) | |
| roles.update(('device', 'host') if r == 'dual' else (r,)) | |
| boards = [b['name'] for b in roster_boards] | |
| s.roles.update(roles) | |
| s.add(boards, sorted(tests), f'{path}: lib {lib} -> {sorted(tests)} on all boards') | |
| return | |
| if re.match(r'src/typec/', path): | |
| # only examples/typec enables CFG_TUC_ENABLED, and no rig board runs a typec | |
| # test (see _HIL_EX_ROLES) - so the build axis covers it and the rig cannot | |
| s.reasons.append(f'{path}: typec, no HIL contribution') | |
| return | |
| m = _BUILD_EX_RE.match(path) | |
| if m: | |
| if m.group(1) not in _HIL_EX_ROLES: | |
| # examples/typec: the build matrix compiles it, nothing on the rig runs it | |
| s.reasons.append(f'{path}: {m.group(1)} example, no HIL contribution') | |
| return | |
| test = f'{m.group(1)}/{m.group(2)}' | |
| known = any(test in pool for pool in ALL_TESTS.values()) or test in extras | |
| if known: | |
| boards = [b['name'] for b in roster_boards] | |
| role = test_role(test) | |
| s.roles.update(('device', 'host') if role == 'dual' else (role,)) | |
| s.add(boards, [test], f'{path}: example -> {test} on all boards') | |
| else: | |
| s.reasons.append(f'{path}: example not in HIL lists, no contribution') | |
| return | |
| s.force_full(f'{path}: unclassified -> full matrix') | |
| def classify(changed_files, repo_root, rosters, get_deps_families=None): | |
| all_boards = [] | |
| seen = set() | |
| for _, boards in rosters: | |
| for b in boards: | |
| if b['name'] not in seen: | |
| seen.add(b['name']) | |
| all_boards.append(b) | |
| extras = roster_only_tests(all_boards) | |
| s = _Sel() | |
| # no early exit once full: keep classifying so `families` still reports every | |
| # family the diff touches (build-only consumers need it). Nothing after the first | |
| # force_full can change full/boards/args - the full branch below ignores by_board. | |
| for path in changed_files: | |
| _classify_one(path, repo_root, all_boards, extras, s, get_deps_families) | |
| if s.full: | |
| return {'full': True, 'boards': {b['name']: 'all' for b in all_boards}, | |
| 'families': sorted(s.families), 'reasons': s.reasons} | |
| # role pruning: single-role selections drop the other role's tests and boards | |
| by_name = {b['name']: b for b in all_boards} | |
| out = {} | |
| for name, tests in s.by_board.items(): | |
| allowed = board_tests(by_name[name]) | |
| if tests == 'all': | |
| kept = list(allowed) | |
| else: | |
| kept = [t for t in allowed if t in tests] | |
| if s.roles and s.roles != {'device', 'host'}: | |
| role = next(iter(s.roles)) | |
| kept = [t for t in kept if test_role(t) in (role, 'dual')] | |
| if kept: | |
| out[name] = 'all' if set(kept) == set(allowed) else sorted(kept) | |
| return {'full': False, 'boards': out, 'families': sorted(s.families), | |
| 'reasons': s.reasons} | |
| def _board_args(name, chosen) -> list: | |
| parts = [f'-b {name}'] | |
| if chosen != 'all': | |
| parts.append(f'-bt {name}:{",".join(chosen)}') | |
| return parts | |
| def hil_examples(sel, rosters): | |
| """{board: examples hil-build must produce}: the board's selected tests plus | |
| device/board_test, which hil_test.py flashes to park at every variant | |
| boundary and at end-of-board teardown. Emitted for full selections too - the | |
| HIL example universe is a fraction of the tree regardless of the diff.""" | |
| by_name = {} | |
| for _, boards in rosters: | |
| for b in boards: | |
| by_name.setdefault(b['name'], []).append(b) | |
| if sel['full']: | |
| chosen = {n: 'all' for n in by_name} | |
| else: | |
| chosen = sel['boards'] | |
| out = {} | |
| for name, tests in chosen.items(): | |
| if tests == 'all': | |
| # a board named by two rosters (rig migration, or shared between rigs) | |
| # may run different tests on each: union them. Superset firmware costs a | |
| # build; a missing image fails the run on whichever rig lost the toss. | |
| run = set().union(*(board_tests(b) for b in by_name[name])) | |
| else: | |
| run = set(tests) | |
| out[name] = sorted(run | {'device/board_test'}) | |
| return out | |
| def selection_args(sel, rosters): | |
| """hil_test.py args per config. Empty means either 'full matrix' or 'nothing | |
| selected' - callers must read sel['full'] to tell them apart.""" | |
| args = {} | |
| for cfg_path, boards in rosters: | |
| parts = [] | |
| if not sel['full']: | |
| for b in boards: | |
| chosen = sel['boards'].get(b['name']) | |
| if chosen is not None: | |
| parts += _board_args(b['name'], chosen) | |
| args[os.path.basename(cfg_path)] = ' '.join(parts) | |
| return args | |
| def selection_args_by_flasher(sel, rosters): | |
| """{config: {flasher name: args}}. CI runs one rig as several jobs split by | |
| flasher (esptool vs the rest); each must gate on its own subset, otherwise the | |
| other leg runs a filter matching zero boards and reports a vacuous green.""" | |
| out = {} | |
| for cfg_path, boards in rosters: | |
| per = {} | |
| if not sel['full']: | |
| for b in boards: | |
| chosen = sel['boards'].get(b['name']) | |
| if chosen is None: | |
| continue | |
| per.setdefault(b.get('flasher', {}).get('name', ''), []).extend( | |
| _board_args(b['name'], chosen)) | |
| out[os.path.basename(cfg_path)] = {f: ' '.join(p) for f, p in per.items()} | |
| return out | |
| class SelectError(Exception): | |
| """An input the selector refuses rather than answer: exit 2, never a fallback guess.""" | |
| def _git(argv, repo_root): | |
| return subprocess.run(['git', *argv], cwd=repo_root, capture_output=True, text=True, | |
| check=True).stdout | |
| def rev(ref, repo_root): | |
| try: | |
| return _git(['rev-parse', '--verify', '--quiet', f'{ref}^{{commit}}'], repo_root).strip() | |
| except subprocess.CalledProcessError: | |
| raise SelectError(f'{ref!r} does not name a commit here') from None | |
| def git_show(spec, repo_root): | |
| return _git(['show', spec], repo_root) | |
| def _deps_families(base_text_fn, head_text_fn, repo_root): | |
| """The changed dep entries' families, or None (-> full matrix) if git cannot produce | |
| both sides of tools/get_deps.py.""" | |
| try: | |
| return get_deps_changed_families(base_text_fn(), head_text_fn(), repo_root) | |
| except (subprocess.CalledProcessError, OSError) as e: | |
| print(f'change_impact: {GET_DEPS_PATH}: base content unreadable ({e})', file=sys.stderr) | |
| return None | |
| def _diff_files(spec, repo_root): | |
| return [f for f in _git(GIT_DIFF_ARGV[1:] + [spec], repo_root).splitlines() if f.strip()] | |
| def _worktree_files(base_sha, repo_root): | |
| untracked = _git(['ls-files', '--others', '--exclude-standard'], repo_root).splitlines() | |
| return [f for f in dict.fromkeys(_diff_files(base_sha, repo_root) + untracked) if f.strip()] | |
| def select_input(repo_root, base=None, endpoints=None, worktree=False, diff_file=None, deps_base=None): | |
| """(changed files, get_deps families or None, the manifest's `input`) for one mode. | |
| Committed modes classify the checked-out tree, so their head must be HEAD.""" | |
| inp = {'mode': None, 'base': base, 'base_sha': None, 'merge_base': None, 'head': None, | |
| 'deps_base': None} | |
| gd = None | |
| if diff_file is not None: | |
| inp['mode'] = 'paths' | |
| files = [f for f in _read(diff_file).splitlines() if f.strip()] | |
| if deps_base is not None and GET_DEPS_PATH in files: | |
| sha = rev(deps_base, repo_root) | |
| inp['deps_base'] = sha | |
| base_text = git_show(f'{sha}:{GET_DEPS_PATH}', repo_root) | |
| work = _read(os.path.join(repo_root, GET_DEPS_PATH)) | |
| if work == base_text: | |
| raise SelectError( | |
| f'{GET_DEPS_PATH} is in the scope with no working-tree change against {deps_base} to ' | |
| f'read its dep entries off, so the families the edit affects cannot be resolved: ' | |
| f'select with --base <ref> for a dep bump that is already committed') | |
| gd = get_deps_changed_families(base_text, work, repo_root) | |
| return files, gd, inp | |
| head = rev('HEAD', repo_root) | |
| if endpoints is not None: | |
| inp['mode'] = 'endpoints' | |
| a, sep, b = endpoints.partition('..') | |
| if not sep or not a or not b or b.startswith('.'): | |
| raise SelectError(f'--endpoints {endpoints!r} is not A..B') | |
| if set(a) == {'0'}: | |
| raise SelectError(f'--endpoints {endpoints!r}: A is the zero commit (a new branch has no before)') | |
| inp['base'], a_sha, b_sha = a, rev(a, repo_root), rev(b, repo_root) | |
| if b_sha != head: | |
| raise SelectError(f'--endpoints {endpoints!r}: B is not the checked-out HEAD {head}') | |
| inp.update(base_sha=a_sha, head=head) | |
| files = _diff_files(f'{a_sha}..{b_sha}', repo_root) | |
| if GET_DEPS_PATH in files: | |
| gd = _deps_families(lambda: git_show(f'{a_sha}:{GET_DEPS_PATH}', repo_root), | |
| lambda: git_show(f'{b_sha}:{GET_DEPS_PATH}', repo_root), repo_root) | |
| return files, gd, inp | |
| base_sha = rev(base, repo_root) | |
| mb = _git(['merge-base', head, base_sha], repo_root).strip() | |
| inp.update(base_sha=base_sha, merge_base=mb, head=head) | |
| if worktree: | |
| inp['mode'] = 'worktree' | |
| files = _worktree_files(mb, repo_root) | |
| head_text = lambda: _read(os.path.join(repo_root, GET_DEPS_PATH)) # noqa: E731 | |
| else: | |
| inp['mode'] = 'base' | |
| files = _diff_files(f'{mb}..{head}', repo_root) | |
| head_text = lambda: git_show(f'{head}:{GET_DEPS_PATH}', repo_root) # noqa: E731 | |
| if GET_DEPS_PATH in files: | |
| gd = _deps_families(lambda: git_show(f'{mb}:{GET_DEPS_PATH}', repo_root), head_text, repo_root) | |
| return files, gd, inp | |
| _BOARD_DIR_RE = re.compile(r'hw/bsp/([^/]+)/boards/([^/]+)/') | |
| def manifest(files, repo_root, rosters, gd, inp): | |
| """Selection manifest v1: the one structured answer every consumer reads.""" | |
| s = classify(files, repo_root, rosters, gd) | |
| b = classify_build(files, repo_root, gd) | |
| fam_ex = b['family_examples'] | |
| build = {'full': b['full'], | |
| 'needed': bool(b['full'] or b['families']), | |
| 'families': {} if b['full'] else | |
| {f: {'examples': fam_ex.get(f, 'all')} for f in b['families']}, | |
| 'required_boards': sorted({m.group(2) for m in map(_BOARD_DIR_RE.match, files) | |
| if m and os.path.isdir(os.path.join(repo_root, m.group(0)))}), | |
| 'required_targets': sorted({t for r in b['paths'] for t in r['targets']}), | |
| 'dropped': b['dropped'], | |
| 'paths': b['paths']} | |
| hil = {'full': s['full'], 'needed': bool(s['full'] or s['boards']), 'boards': s['boards'], | |
| 'args': selection_args(s, rosters), 'args_flasher': selection_args_by_flasher(s, rosters), | |
| 'hil_examples': hil_examples(s, rosters) if rosters else {}} | |
| return {'version': 1, 'input': inp, 'build': build, 'hil': hil}, s['reasons'], b['reasons'] | |
| EFFECTS = ('none', 'gap', 'select', 'full') | |
| def check_manifest(m): | |
| """Raise ValueError unless `m` is a usable manifest v1: what every CI reader checks | |
| before it scopes or skips anything, so a malformed, contradictory or legacy | |
| selection falls open instead.""" | |
| def need(ok, what): | |
| if not ok: | |
| raise ValueError(f'manifest: {what}') | |
| def strs(x): | |
| return isinstance(x, list) and all(isinstance(s, str) for s in x) | |
| need(isinstance(m, dict) and type(m.get('version')) is int and m['version'] == 1, 'not version 1') | |
| b, h = m.get('build'), m.get('hil') | |
| need(isinstance(b, dict) and isinstance(h, dict), 'build/hil not objects') | |
| for name, axis in (('build', b), ('hil', h)): | |
| need(isinstance(axis.get('full'), bool) and isinstance(axis.get('needed'), bool), | |
| f'{name}.full/needed not booleans') | |
| fams = b.get('families') | |
| need(isinstance(fams, dict) and all( | |
| isinstance(f, str) and isinstance(v, dict) and | |
| (v.get('examples') == 'all' or (strs(v.get('examples')) and v['examples'])) | |
| for f, v in fams.items()), 'build.families malformed') | |
| need(not (b['full'] and fams), 'build.families not empty under build.full') | |
| need(b['needed'] == bool(b['full'] or fams), 'build.needed contradicts full/families') | |
| for key in ('required_boards', 'required_targets'): | |
| need(strs(b.get(key)), f'build.{key} malformed') | |
| need(isinstance(b.get('paths'), list) and all( | |
| isinstance(r, dict) and r.get('effect') in EFFECTS and isinstance(r.get('path'), str) | |
| for r in b['paths']), 'build.paths malformed') | |
| boards = h.get('boards') | |
| need(isinstance(boards, dict) and all(v == 'all' or strs(v) for v in boards.values()), | |
| 'hil.boards malformed') | |
| need(h['needed'] == bool(h['full'] or boards), 'hil.needed contradicts full/boards') | |
| need(isinstance(h.get('args'), dict) and all(isinstance(v, str) for v in h['args'].values()), | |
| 'hil.args malformed') | |
| need(isinstance(h.get('args_flasher'), dict) and all( | |
| isinstance(v, dict) and all(isinstance(a, str) for a in v.values()) for v in h['args_flasher'].values()), | |
| 'hil.args_flasher malformed') | |
| need(isinstance(h.get('hil_examples'), dict) and all(strs(v) for v in h['hil_examples'].values()), | |
| 'hil.hil_examples malformed') | |
| return m | |
| def build_gate(m): | |
| """Whether a workflow has any work: something to build or test, or a firmware path | |
| no build compiles (effect gap), which keeps the full builds' syntax-check net.""" | |
| b = check_manifest(m)['build'] | |
| return b['needed'] or m['hil']['needed'] or any(r['effect'] == 'gap' for r in b['paths']) | |
| def main(): | |
| ap = argparse.ArgumentParser(description=__doc__) | |
| g = ap.add_mutually_exclusive_group(required=True) | |
| g.add_argument('--base', help='git ref to diff against (merge-base(HEAD, ref)..HEAD)') | |
| g.add_argument('--endpoints', metavar='A..B', | |
| help='diff A..B directly (a push: before..after); B must be the checked-out HEAD') | |
| g.add_argument('--diff-file', help='newline-separated changed-file list') | |
| ap.add_argument('--worktree', action='store_true', | |
| help='with --base: diff the merge-base against the working tree, untracked files included') | |
| ap.add_argument('--deps-base', metavar='REF', | |
| help='with --diff-file: resolve a tools/get_deps.py edit as REF vs the working file') | |
| ap.add_argument('--manifest', action='store_true', help='print the selection manifest v1') | |
| ap.add_argument('configs', nargs='*', help='rig roster JSON file(s); omit for the build view alone') | |
| a = ap.parse_args() | |
| if a.worktree and not a.base: | |
| ap.error('--worktree needs --base') | |
| if a.deps_base and not a.diff_file: | |
| ap.error('--deps-base needs --diff-file') | |
| repo_root = _REPO_ROOT | |
| rosters = [] | |
| for c in a.configs: | |
| with open(c, encoding='utf-8', errors='replace') as f: | |
| rosters.append((c, json.load(f)['boards'])) | |
| try: | |
| files, gd, inp = select_input(repo_root, a.base, a.endpoints, a.worktree, a.diff_file, a.deps_base) | |
| except SelectError as e: | |
| print(f'change_impact: {e}', file=sys.stderr) | |
| sys.exit(2) | |
| if a.manifest: | |
| m, hil_reasons, build_reasons = manifest(files, repo_root, rosters, gd, inp) | |
| for r in build_reasons: | |
| print(f'change_impact[build]: {r}', file=sys.stderr) | |
| for r in hil_reasons: | |
| print(f'change_impact: {r}', file=sys.stderr) | |
| print(json.dumps(m)) | |
| return | |
| s = classify(files, repo_root, rosters, gd) | |
| s['args'] = selection_args(s, rosters) | |
| s['args_flasher'] = selection_args_by_flasher(s, rosters) | |
| if rosters: | |
| s['hil_examples'] = hil_examples(s, rosters) | |
| s['build'] = classify_build(files, repo_root, gd) | |
| for r in s['build']['reasons']: | |
| print(f'change_impact[build]: {r}', file=sys.stderr) | |
| for r in s['reasons']: | |
| print(f'change_impact: {r}', file=sys.stderr) | |
| # reasons go to stderr ONLY - they are a human diagnostic and no consumer reads them | |
| # back. They are also ~97% of the payload (a whole-tree diff: 453 KB -> 12 KB), which | |
| # build.yml re-parses with ci_set_matrix, hil_ci_set_matrix, an inline python and | |
| # three jq calls. The in-process dicts still carry them, for the log and the tests. | |
| # The legacy shape carries no per-path records: the manifest is their interface. | |
| out = {k: v for k, v in s.items() if k != 'reasons'} | |
| out['build'] = {k: v for k, v in s['build'].items() if k not in ('reasons', 'paths', 'dropped')} | |
| print(json.dumps(out)) | |
| # ------------------------------------------------------------- | |
| # Build-axis classifier (the rule table in the module docstring). Independent of the HIL | |
| # classifier: same diff, second walk, its own fail-open. | |
| # ------------------------------------------------------------- | |
| # Both walks recognise an example path with the SAME regex, so a role can never be | |
| # known to one walk and unclassified (-> full matrix) to the other. What differs is the | |
| # answer: the rig runs device/host/dual tests, while the build matrix also compiles | |
| # examples/typec, which nothing on the rig runs. | |
| _EX_ROLES = ('device', 'dual', 'host', 'typec') | |
| _HIL_EX_ROLES = ('device', 'host', 'dual') | |
| _BUILD_EX_RE = re.compile(r'examples/(%s)/([^/]+)/' % '|'.join(_EX_ROLES)) | |
| @functools.lru_cache(maxsize=None) | |
| def all_examples(repo_root: str) -> tuple: | |
| """Every examples/<role>/<name> with a CMakeLists.txt, as 'role/name'.""" | |
| out = [] | |
| for role in _EX_ROLES: | |
| for d in sorted(glob.glob(_rg(repo_root, 'examples', role, '*/'))): | |
| if os.path.isfile(os.path.join(d, 'CMakeLists.txt')): | |
| out.append(f'{role}/{os.path.basename(d.rstrip(os.sep))}') | |
| return tuple(out) | |
| def role_examples(repo_root: str, roles) -> set: | |
| want = set(roles) | |
| return {e for e in all_examples(repo_root) if e.split('/', 1)[0] in want} | |
| @functools.lru_cache(maxsize=None) | |
| def all_bsp_families(repo_root: str) -> tuple: | |
| return tuple(sorted(d for d in os.listdir(os.path.join(repo_root, 'hw/bsp')) | |
| if os.path.isdir(os.path.join(repo_root, 'hw/bsp', d)))) | |
| def _build_class_examples(cls: str, base: str, roles: set, repo_root: str) -> set: | |
| """Examples (all 46, not the HIL lists) whose tusb_config.h enables the class's | |
| macros for the given roles, plus classes that #include the changed header, plus | |
| examples that #include it themselves.""" | |
| via = sorted(class_include_edges(repo_root).get(f'{cls}/{base}', ())) | |
| out = set(example_class_includes(repo_root).get(f'{cls}/{base}', ())) | |
| for prefix, role in (('TUD', 'device'), ('TUH', 'host')): | |
| if role not in roles: | |
| continue | |
| macros = class_macros(cls, base, prefix) + edge_macros(via, prefix) | |
| out |= examples_enabling(all_examples(repo_root), macros, repo_root) | |
| return out | |
| class _BSel: | |
| """family -> set(examples) | 'all', unioned per family, plus one record per path | |
| (build_record()): its rule, what it selects and why.""" | |
| def __init__(self): | |
| self.full = False | |
| self.fam_ex = {} | |
| self.reasons = [] | |
| self.paths = [] | |
| def note(self, path, rule, effect, reason, families=None, examples=None, port=None, targets=()): | |
| self.reasons.append(reason) | |
| self.paths.append(build_record(path, rule, effect, reason, families, examples, port, targets)) | |
| def add(self, path, rule, fams, examples, reason, named=None, port=None): | |
| """Select `examples` on `fams`; `named` is the families the rule itself names | |
| (None: the rule applies to every family).""" | |
| self.note(path, rule, 'select', reason, named, | |
| None if examples == 'all' or rule == 'port' else examples, port) | |
| for f in fams: | |
| cur = self.fam_ex.get(f) | |
| if examples == 'all' or cur == 'all': | |
| self.fam_ex[f] = 'all' | |
| else: | |
| self.fam_ex[f] = (cur or set()) | set(examples) | |
| def force_full(self, path, rule, reason, targets=()): | |
| self.full = True | |
| self.note(path, rule, 'full', reason, targets=targets) | |
| def build_record(path, rule, effect, reason, families=None, examples=None, port=None, targets=()): | |
| """One path's build-axis classification. `effect` is its contribution (none: nothing | |
| to verify; gap: firmware no build compiles; select; full), not a coverage verdict. | |
| `families`/`examples` are None when the rule names none, i.e. every one.""" | |
| return {'path': path, 'rule': rule, 'effect': effect, | |
| 'families': None if families is None else sorted(families), | |
| 'examples': None if examples is None else sorted(examples), | |
| 'port': port, 'targets': sorted(targets), 'reason': reason} | |
| def _classify_build_one(path, repo_root, s: _BSel, get_deps_families=None): | |
| base = os.path.basename(path) | |
| if _SELECTOR_RE.match(path): # rule 2e | |
| s.force_full(path, 'selector', f'{path}: the selection contract -> full build matrix') | |
| return | |
| if _NONCODE_RE.match(path) or _META_RE.match(path): # rules 1, 1b | |
| s.note(path, 'noncode', 'none', f'{path}: non-code, no build contribution') | |
| return | |
| if re.match(r'test/hil/', path): # rule 2 | |
| s.note(path, 'hil-harness', 'none', f'{path}: HIL harness, no build contribution') | |
| return | |
| if path == GET_DEPS_PATH: # rule 16b | |
| if get_deps_families is None: | |
| s.force_full(path, 'get-deps', f'{path}: dep changes not resolvable -> full build matrix') | |
| return | |
| if not get_deps_families: | |
| s.note(path, 'get-deps', 'none', f'{path}: no build-family dependency changed, no contribution') | |
| return | |
| fams = sorted(get_deps_families) | |
| s.add(path, 'get-deps', fams, 'all', f'{path}: dep entries changed -> families {fams}', fams) | |
| return | |
| m = _PORT_PATH_RE.match(path) | |
| if m: # rules 3-5 | |
| port = m.group(1) | |
| fams = port_families(port, repo_root) | |
| roles = _port_roles(base) | |
| exs = 'all' if roles == {'device', 'host'} else \ | |
| role_examples(repo_root, tuple(roles) + ('dual',)) | |
| reason = f'{path}: port {port} -> families {sorted(fams)}' | |
| if not fams: # rule 5b | |
| s.note(path, 'port', 'gap', reason, port=port) | |
| return | |
| s.add(path, 'port', fams, exs, reason, fams, port) | |
| return | |
| if re.match(r'hw/bsp/[^/]+/', path): # rule 6 | |
| fam = path.split('/')[2] | |
| s.add(path, 'bsp', {fam}, 'all', f'{path}: bsp family {fam}', {fam}) | |
| return | |
| if re.match(r'hw/mcu/', path): # rule 7 | |
| fams = mcu_families(path, repo_root) | |
| if not fams: | |
| # empty means empty, same reading as the HIL walk: no family's build | |
| # references the path, so no build compiles it | |
| s.note(path, 'mcu', 'gap', f'{path}: hw/mcu path resolves to no family, no contribution') | |
| return | |
| s.add(path, 'mcu', fams, 'all', f'{path}: mcu -> families {sorted(fams)}', fams) | |
| return | |
| m = re.match(r'src/class/([^/]+)/', path) | |
| if m: # rules 8-10 | |
| cls = m.group(1) | |
| roles = _class_roles(base) | |
| exs = _build_class_examples(cls, base, roles, repo_root) | |
| if not exs: | |
| # Empty means empty - maintainer decision. No example config enables this | |
| # class, so no build exercises it and | |
| # nothing is selected. The file IS still parsed by every full build | |
| # (src/CMakeLists.txt, src/tinyusb.mk list class sources unconditionally, | |
| # the CFG_ guard sits inside), so a break outside the guard surfaces on the | |
| # next master push - the accepted safety net. | |
| s.note(path, 'class', 'gap', f'{path}: class {cls} enabled by no example config, ' | |
| f'no contribution') | |
| return | |
| s.add(path, 'class', all_bsp_families(repo_root), exs, | |
| f'{path}: class {cls} -> {sorted(exs)}') | |
| return | |
| m = re.match(r'src/(device|host)/', path) | |
| if m: # rules 11-12 | |
| role = m.group(1) | |
| s.add(path, 'core', all_bsp_families(repo_root), role_examples(repo_root, (role, 'dual')), | |
| f'{path}: core {role} stack') | |
| return | |
| m = _BUILD_EX_RE.match(path) | |
| if m: # rules 13-14 | |
| ex = f'{m.group(1)}/{m.group(2)}' | |
| if ex in all_examples(repo_root): | |
| s.add(path, 'example', all_bsp_families(repo_root), {ex}, f'{path}: example {ex}') | |
| else: | |
| # a deleted example builds nothing; removing it from the role | |
| # CMakeLists (rule 15) is what forces the full matrix | |
| s.note(path, 'example', 'none', f'{path}: not an example dir, no build contribution') | |
| return | |
| if re.match(r'src/typec/', path): # rule 12b | |
| # listed unconditionally by src/CMakeLists.txt and src/tinyusb.mk, but the whole | |
| # body is `#if CFG_TUC_ENABLED` - so it is PARSED by every build and COMPILED | |
| # only for examples that enable it. Same shape as the class rule, same answer: | |
| # the examples whose tusb_config.h turns it on, and empty means empty. | |
| exs = examples_enabling(role_examples(repo_root, ('typec',)), | |
| ('CFG_TUC_ENABLED',), repo_root) | |
| if not exs: | |
| s.note(path, 'typec', 'gap', f'{path}: typec enabled by no example config, no contribution') | |
| return | |
| s.add(path, 'typec', all_bsp_families(repo_root), exs, f'{path}: typec -> {sorted(exs)}') | |
| return | |
| m = re.match(r'lib/([^/]+)/', path) | |
| if m: # rule 16a | |
| lib = m.group(1) | |
| exs = lib_examples(lib, repo_root) | |
| if not exs: | |
| # empty means empty: no example's build pulls this lib in, so no MAIN- | |
| # matrix build compiles it. (lib/SEGGER_RTT is reached through LOGGER=rtt, | |
| # which the main matrix never sets; the hil-build legs set it only for | |
| # roster boards whose variant defines carry it, via the HIL SEGGER_RTT rule. | |
| # No committed CI roster has such a board yet, so a SEGGER_RTT edit is | |
| # currently neither built nor HIL-tested by CI -- verify vendor bumps | |
| # manually until a rig board adopts "logger": "rtt".) | |
| s.note(path, 'lib', 'gap', f'{path}: lib {lib} built by no example, no contribution') | |
| return | |
| s.add(path, 'lib', all_bsp_families(repo_root), exs, f'{path}: lib {lib} -> {sorted(exs)}') | |
| return | |
| if _LOCAL_TOOLING_RE.match(path): # rule 2b | |
| s.note(path, 'local-tooling', 'none', f'{path}: local coverage tooling, no build contribution') | |
| return | |
| if _CI_BOARDS_RE.match(path): # rule 2c | |
| s.force_full(path, 'ci-boards', f'{path}: CI board data changes which boards build -> full build matrix') | |
| return | |
| if _MEMBROWSE_SCRIPT_RE.match(path): # rule 2d | |
| # verified by the upload target family_add_membrowse() declares, which `all` never runs | |
| s.force_full(path, 'membrowse-script', f'{path}: CI size script -> full build matrix', | |
| (MEMBROWSE_TARGET,)) | |
| return | |
| if _SIZE_SCRIPT_RE.match(path): # rule 2d | |
| # no build target runs it: CI's code-size step does (build_util.yml snapshot) | |
| s.force_full(path, 'size-script', f'{path}: CI size script -> full build matrix') | |
| return | |
| if _FULL_RE.match(path): # rules 15-16 | |
| # attribution, not behaviour: these already reached `full` through the | |
| # fall-through below. Naming them means a future narrowing of rule 17 cannot | |
| # silently change what they do. Deliberately last, so every earlier rule keeps | |
| # priority - examples/device/board_test is rule 14 (just board_test), not ALL. | |
| s.force_full(path, 'core-infra', f'{path}: core/infra -> full build matrix') | |
| return | |
| s.force_full(path, 'unclassified', f'{path}: unclassified -> full build matrix') # rule 17 | |
| @contextlib.contextmanager | |
| def _in_repo(repo_root): | |
| """build_utils/build.py use repo-relative paths; scope a chdir around them. | |
| get_family_boards also prints on an empty family - swallow stdout so the | |
| selector's machine-read JSON stays clean (diagnostics belong on stderr).""" | |
| old = os.getcwd() | |
| os.chdir(repo_root) | |
| try: | |
| with contextlib.redirect_stdout(io.StringIO()): | |
| yield | |
| finally: | |
| os.chdir(old) | |
| def _prune_buildable(fams, fam_ex, repo_root): | |
| """Intersect each family's selection with what the family can build at all | |
| (build_utils.skip_example - the same skip.txt/only.txt data CMake's | |
| family_filter reads). | |
| ANY board of the family counts, not just the one GHA's --one-first picks: | |
| CircleCI's cmake legs build every board of a family, so an example gated to a | |
| single board (only.txt board:mimxrt1060_evk) would otherwise lose ALL compile | |
| coverage exactly when a PR touches it. get_family_boards(.., False, False) is | |
| that full list. | |
| EITHER build system counts too. This one list gates CircleCI's make legs as well | |
| as its cmake ones, and the two answer different questions (build_utils.skip_example): | |
| examples/device/dfu carries `mcu:BCM2835` in skip.txt, which the cmake FAMILY_MCUS | |
| union applies to every broadcom_64bit board while the make scrape applies it to | |
| none - asking cmake alone drops the only aarch64-gcc family in the matrix and | |
| `build-make-aarch64-gcc` stops compiling dfu at all.""" | |
| out_fams, out_ex, reasons, dropped = [], {}, [], {} | |
| allex = list(all_examples(repo_root)) | |
| with _in_repo(repo_root): | |
| for fam in fams: | |
| if not os.path.isdir(os.path.join(repo_root, 'hw/bsp', fam, 'boards')): | |
| # a PR that deletes or renames hw/bsp/<fam> still names it in the | |
| # diff (rule 6); the family builds nothing now, and get_family_boards | |
| # would raise FileNotFoundError out of the whole selector | |
| dropped[fam] = f'{fam}: family dir gone from tree, dropped' | |
| reasons.append(dropped[fam]) | |
| continue | |
| try: | |
| boards = build_py.get_family_boards(fam, False, False) | |
| except OSError as e: # belt and braces: never traceback here | |
| dropped[fam] = f'{fam}: boards unreadable ({e}), dropped' | |
| reasons.append(dropped[fam]) | |
| continue | |
| if not boards: | |
| out_fams.append(fam) # unknown layout: keep unfiltered | |
| continue | |
| # what this family's build path can even see, asked the same way for | |
| # every family. build.py's espressif branch builds get_examples('espressif') | |
| # only (the examples with an ESP-IDF component); keeping the family | |
| # for anything else spins up CI's most expensive leg to skip every example | |
| # it was given. Identical to the unfiltered list on all 81 other families. | |
| pool = set(build_py.get_examples(fam)) | |
| # asked per example instead of materialising the family's whole buildable | |
| # list: skip_example is by far the hottest call in the selector, and every | |
| # question below short-circuits (one cdc_device.c diff: 6,883 calls -> 1,889) | |
| def can_build(ex): | |
| # EITHER build system: this one list gates CircleCI's make legs too, and | |
| # the two answer differently (build_utils.skip_example) | |
| return ex in pool and any( | |
| not build_utils.skip_example(ex, b) or | |
| not build_utils.skip_example(ex, b, (), 'make') for b in boards) | |
| want = fam_ex.get(fam) | |
| try: | |
| if want is None: | |
| kept = None if any(can_build(e) for e in allex) else [] | |
| else: | |
| kept = [e for e in want if can_build(e)] | |
| if kept and not any(can_build(e) for e in allex if e not in want): | |
| kept = None # already everything the family can build | |
| except OSError as e: | |
| # a family mid-bring-up (boards/ but no family.cmake/family.mk yet) | |
| # reads as unbuildable to the scrape; keep it rather than tracebacking | |
| # out of the selector and losing the scoping for the whole PR | |
| reasons.append(f'{fam}: mcu scrape unreadable ({e}), kept unfiltered') | |
| out_fams.append(fam) | |
| continue | |
| if kept == []: | |
| continue # this diff builds nothing for this family | |
| out_fams.append(fam) | |
| if kept is not None: | |
| out_ex[fam] = kept | |
| return out_fams, out_ex, reasons, dropped | |
| def classify_build(changed_files, repo_root, get_deps_families=None): | |
| s = _BSel() | |
| for p in changed_files: | |
| _classify_build_one(p, repo_root, s, get_deps_families) | |
| if s.full: | |
| return {'full': True, 'families': list(all_bsp_families(repo_root)), | |
| 'family_examples': {}, 'reasons': s.reasons, 'paths': s.paths, 'dropped': {}} | |
| fams = sorted(s.fam_ex) | |
| fam_ex = {f: sorted(e) for f, e in s.fam_ex.items() if e != 'all'} | |
| fams, fam_ex, pruned, dropped = _prune_buildable(fams, fam_ex, repo_root) | |
| s.reasons += pruned | |
| return {'full': False, 'families': fams, 'family_examples': fam_ex, | |
| 'reasons': s.reasons, 'paths': s.paths, 'dropped': dropped} | |
| if __name__ == '__main__': | |
| main() |