| #!/usr/bin/env python3 | |
| import argparse | |
| import json | |
| import os | |
| import shlex | |
| import subprocess | |
| import sys | |
| # toolchain, url | |
| toolchain_list = [ | |
| "aarch64-gcc", | |
| "arm-clang", | |
| "arm-iar", | |
| "arm-gcc", | |
| "esp-idf", | |
| "ft9xx-gcc", | |
| "msp430-gcc", | |
| "riscv-gcc", | |
| "rx-gcc" | |
| ] | |
| # family: [cmake toolchains that build it]. An empty list means CI builds the family | |
| # by board name instead of as a matrix leg (espressif: hil-build-esp, in an IDF | |
| # container), so it belongs here - "not in family_list" means CI never builds it. | |
| family_list = { | |
| "apm32f0xx": ["arm-gcc"], | |
| "at32f402_405": ["arm-gcc"], | |
| "at32f403a_407": ["arm-gcc"], | |
| "at32f413": ["arm-gcc"], | |
| "at32f415": ["arm-gcc"], | |
| "at32f423": ["arm-gcc"], | |
| "at32f425": ["arm-gcc"], | |
| "at32f435_437": ["arm-gcc"], | |
| "at32f45x": ["arm-gcc"], | |
| "broadcom_32bit": ["arm-gcc"], | |
| "broadcom_64bit": ["aarch64-gcc"], | |
| "ch32f20x": ["arm-gcc"], | |
| "ch32v10x": ["riscv-gcc"], | |
| "ch32v20x": ["riscv-gcc"], | |
| "ch32v30x": ["riscv-gcc"], | |
| "ch32x035": ["riscv-gcc"], | |
| "ch583": ["riscv-gcc"], | |
| "da1469x": ["arm-gcc"], | |
| "efm32": ["arm-gcc"], | |
| "f1c100s": ["arm-gcc"], | |
| "fomu": ["riscv-gcc"], | |
| "espressif": [], | |
| "ft9xx": ["ft9xx-gcc"], | |
| "gd32vf103": ["riscv-gcc"], | |
| "hpmicro": ["riscv-gcc"], | |
| "imxrt": ["arm-gcc", "arm-clang"], | |
| "kinetis_k": ["arm-gcc"], | |
| "kinetis_k32l": ["arm-gcc"], | |
| "kinetis_kl": ["arm-gcc"], | |
| "lpc11": ["arm-gcc", "arm-clang"], | |
| "lpc13": ["arm-gcc", "arm-clang"], | |
| "lpc15": ["arm-gcc", "arm-clang"], | |
| "lpc17": ["arm-gcc", "arm-clang"], | |
| "lpc18": ["arm-gcc", "arm-clang"], | |
| "lpc40": ["arm-gcc", "arm-clang"], | |
| "lpc43": ["arm-gcc", "arm-clang"], | |
| "lpc51": ["arm-gcc", "arm-clang"], | |
| "lpc54": ["arm-gcc", "arm-clang"], | |
| "lpc55": ["arm-gcc", "arm-clang"], | |
| "maxim": ["arm-gcc"], | |
| "mcx": ["arm-gcc"], | |
| "mm32": ["arm-gcc"], | |
| "msp430": ["msp430-gcc"], | |
| "msp432e4": ["arm-gcc"], | |
| "nrf": ["arm-gcc", "arm-clang"], | |
| "nuc100_120": ["arm-gcc"], | |
| "nuc121_125": ["arm-gcc"], | |
| "nuc126": ["arm-gcc"], | |
| "nuc505": ["arm-gcc"], | |
| "ra": ["arm-gcc"], | |
| "rp2040": ["arm-gcc"], | |
| "rw61x": ["arm-gcc"], | |
| "rx": ["rx-gcc"], | |
| "samd11": ["arm-gcc", "arm-clang"], | |
| "samd2x_l2x": ["arm-gcc", "arm-clang"], | |
| "samd5x_e5x": ["arm-gcc", "arm-clang"], | |
| "same7x": ["arm-gcc"], | |
| "samg": ["arm-gcc", "arm-clang"], | |
| "stm32c0": ["arm-gcc", "arm-clang", "arm-iar"], | |
| "stm32c5": ["arm-gcc", "arm-clang", "arm-iar"], | |
| "stm32f0": ["arm-gcc", "arm-clang", "arm-iar"], | |
| "stm32f1": ["arm-gcc", "arm-clang", "arm-iar"], | |
| "stm32f2": ["arm-gcc", "arm-clang", "arm-iar"], | |
| "stm32f3": ["arm-gcc", "arm-clang", "arm-iar"], | |
| "stm32f4": ["arm-gcc", "arm-clang", "arm-iar"], | |
| "stm32f7": ["arm-gcc", "arm-clang", "arm-iar"], | |
| "stm32g0": ["arm-gcc", "arm-clang", "arm-iar"], | |
| "stm32g4": ["arm-gcc", "arm-clang", "arm-iar"], | |
| "stm32h5": ["arm-gcc", "arm-clang", "arm-iar"], | |
| "stm32h7": ["arm-gcc", "arm-clang", "arm-iar"], | |
| "stm32h7rs": ["arm-gcc", "arm-clang", "arm-iar"], | |
| "stm32l0": ["arm-gcc", "arm-clang", "arm-iar"], | |
| "stm32l4": ["arm-gcc", "arm-clang", "arm-iar"], | |
| "stm32n6": ["arm-gcc"], | |
| "stm32u0": ["arm-gcc", "arm-clang", "arm-iar"], | |
| "stm32u5": ["arm-gcc", "arm-clang", "arm-iar"], | |
| "stm32wb": ["arm-gcc", "arm-clang", "arm-iar"], | |
| "stm32wba": ["arm-gcc", "arm-clang", "arm-iar"], | |
| "tm4c": ["arm-gcc"], | |
| "xmc4000": ["arm-gcc"], | |
| } | |
| REPO = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) | |
| # the toolchains cmake-required builds a changed board with: every one a GitHub runner | |
| # can set up (.github/actions/setup_toolchain), espressif's through its IDF container | |
| REQUIRED_TOOLCHAINS = ['aarch64-gcc', 'arm-gcc', 'esp-idf', 'ft9xx-gcc', 'msp430-gcc', 'riscv-gcc', 'rx-gcc'] | |
| # build.yml's `cmake` job matrix (its cmake_toolchains output): the pinned boards built there | |
| CMAKE_JOB_TOOLCHAINS = ['aarch64-gcc', 'arm-gcc', 'ft9xx-gcc', 'msp430-gcc', 'riscv-gcc'] | |
| def pinned_boards(repo_root=REPO): | |
| with open(os.path.join(repo_root, '.github', 'ci-pinned-boards.json')) as f: | |
| return {e['board'] for e in json.load(f)['boards']} | |
| def board_family(board, repo_root=REPO): | |
| bsp = os.path.join(repo_root, 'hw', 'bsp') | |
| return next((fam for fam in sorted(os.listdir(bsp)) | |
| if os.path.isdir(os.path.join(bsp, fam, 'boards', board))), None) | |
| def pinned_families(repo_root=REPO): | |
| """Families with a board in .github/ci-pinned-boards.json. Board names are | |
| unique across hw/bsp/*/boards, so the board alone gives the family.""" | |
| return {f for f in map(lambda b: board_family(b, repo_root), pinned_boards(repo_root)) if f} | |
| def usable(select): | |
| """The manifest, or None after saying UNSCOPED: the marker build.yml and | |
| .circleci/config.yml grep for to drop the build extras along with the scoping.""" | |
| if select is None: | |
| return None | |
| try: | |
| # imported here, under the fall-open: a broken selector must still leave the | |
| # unscoped matrix | |
| sys.path.insert(0, os.path.join(REPO, 'tools')) | |
| import change_impact | |
| return change_impact.check_manifest(select) | |
| except Exception as e: | |
| print(f'ci_set_matrix: UNSCOPED - selection unusable ({e}), emitting the full matrix', file=sys.stderr) | |
| return None | |
| def required_json(select): | |
| """{toolchain: ['-b <board>', ...]} for the boards the change edits that the pinned | |
| matrix does not build (D3), from any usable manifest, full or scoped. A board of a | |
| family no CI toolchain builds is listed under 'unbuildable'.""" | |
| select = usable(select) | |
| out = {tc: [] for tc in REQUIRED_TOOLCHAINS} | |
| out['unbuildable'] = [] | |
| if select is None: | |
| print(json.dumps(out)) | |
| return | |
| b = select['build'] | |
| built = {f for f, tcs in family_list.items() if set(tcs) & set(CMAKE_JOB_TOOLCHAINS)} | |
| covered_fams = pinned_families() & built & (built if b['full'] else set(b['families'])) | |
| pinned = pinned_boards() | |
| for board in b['required_boards']: | |
| fam = board_family(board) | |
| if board in pinned and fam in covered_fams: | |
| continue | |
| tcs = ['esp-idf'] if fam == 'espressif' else family_list.get(fam, []) | |
| tc = next((t for t in tcs if t in REQUIRED_TOOLCHAINS), None) | |
| if tc is None: | |
| out['unbuildable'].append(board) | |
| print(f'ci_set_matrix: required board {board} ({fam}) is built by no CI toolchain', file=sys.stderr) | |
| else: | |
| out[tc].append(f'-b {board}') | |
| print(json.dumps(out)) | |
| def example_map_json(select): | |
| """{family: [examples]} for build_util.yml/config2.yml's -e filter: a family that | |
| builds every example has no entry, and an unusable selection filters nothing.""" | |
| select = usable(select) | |
| fams = select['build']['families'] if select else {} | |
| print(json.dumps({f: v['examples'] for f, v in fams.items() if v['examples'] != 'all'})) | |
| def leg_key(arg): | |
| """(board, upload name) of a `-b <board> [--build-name <name>] ...` leg.""" | |
| words = shlex.split(arg) | |
| board = words[words.index('-b') + 1] | |
| return board, words[words.index('--build-name') + 1] if '--build-name' in words else board | |
| def membrowse_json(pinned, hil_esp, hil_esp_all, with_esp): | |
| """{'all': legs, 'identical': legs} for the membrowse-identical job. Every leg of the | |
| universe - the pinned boards, plus the tinyusb roster's espressif legs where | |
| hil-build-esp runs (with_esp) - is uploaded once per commit: by the leg that owns it, | |
| else --identical. A cmake leg uploads every pinned board of its family (build.py | |
| --ci-pinned-boards-only), hil-build-esp its scoped espressif legs. 'all' is for a run | |
| whose gate builds nothing.""" | |
| universe = {(b, b): f'-b {b}' for b in sorted(pinned_boards()) if board_family(b) != 'espressif'} | |
| if with_esp: | |
| universe.update((leg_key(a), a) for a in hil_esp_all) | |
| measured = {leg_key(a) for a in hil_esp} if with_esp else set() | |
| fams = {f for tc in CMAKE_JOB_TOOLCHAINS for f in pinned.get(tc, [])} | |
| measured.update((b, b) for b in pinned_boards() if board_family(b) in fams) | |
| print(json.dumps({'all': list(universe.values()), | |
| 'identical': [a for k, a in universe.items() if k not in measured]})) | |
| def set_matrix_json(select=None, pinned=False): | |
| sel_fams = None | |
| select = usable(select) | |
| if select is not None and not select['build']['full']: | |
| # an explicit empty families map is a legitimate nothing-selected | |
| sel_fams = set(select['build']['families']) | |
| matrix = {} | |
| for toolchain in toolchain_list: | |
| fams = [family for family, tc in family_list.items() if toolchain in tc] | |
| if sel_fams is not None: | |
| fams = [f for f in fams if f in sel_fams] | |
| matrix[toolchain] = fams | |
| if sel_fams: | |
| # a family this file does not list builds on no toolchain, so it contributes no | |
| # leg. hw/bsp holds a few CI has never built (pic32mz, py32f0, ...) - a | |
| # missing family.cmake or no CI toolchain support, not a gap in this file. | |
| # espressif is in family_list with no toolchain: hil-build-esp builds its boards | |
| # BY NAME in an IDF container (an esp-idf leg here would double-build them, and | |
| # CircleCI would build every espressif board), so falling open to the full matrix | |
| # for an espressif-only selection would add 74 legs, none of which can compile it. | |
| unbuilt = sorted(f for f in sel_fams if f not in family_list) | |
| if unbuilt and not any(matrix.values()): | |
| # NONE of the selected families is buildable here, so every leg would skip | |
| # and the PR would go green from a build job that ran no compiler. That is | |
| # an unusable selection, not "nothing selected": say UNSCOPED - which | |
| # build.yml and .circleci/config.yml both grep for - and emit the full | |
| # matrix. An explicit families: [] is still a legitimate nothing-selected, | |
| # and a PARTIAL miss still scopes to the families that do build. | |
| print(f'ci_set_matrix: UNSCOPED - no selected family is built by any ' | |
| f'toolchain here ({", ".join(unbuilt)}), emitting the full matrix', | |
| file=sys.stderr) | |
| return set_matrix_json(None, pinned) | |
| if unbuilt: | |
| print(f'ci_set_matrix: selected families built by no toolchain here: ' | |
| f'{", ".join(unbuilt)}', file=sys.stderr) | |
| if pinned: | |
| # last, after every UNSCOPED decision above: those rules ask whether the | |
| # SELECTION is usable, and an empty pinned matrix is a legitimate "no pinned | |
| # family selected", not a selection to widen back to the full matrix. | |
| keep = pinned_families() | |
| matrix = {tc: [f for f in fams if f in keep] for tc, fams in matrix.items()} | |
| print(json.dumps(matrix)) | |
| def main(): | |
| parser = argparse.ArgumentParser() | |
| group = parser.add_mutually_exclusive_group() | |
| group.add_argument('--select', help='selection manifest v1 (check_build.py --select-only); ' | |
| 'scopes families when build.full is false') | |
| # a whole selection as one argv/env value can exceed the exec limits on a big | |
| # diff, which fails the calling step BEFORE it can fall open; callers that | |
| # already have the selection on disk pass the path instead | |
| group.add_argument('--select-file', help='file holding the same JSON as --select') | |
| group.add_argument('--base', help='git ref: select with check_build.py --select-only --base REF') | |
| parser.add_argument('--pinned', action='store_true', | |
| help='keep only families with a board in .github/ci-pinned-boards.json ' | |
| '(the GHA cmake leg; CircleCI builds every board unfiltered)') | |
| parser.add_argument('--required', action='store_true', | |
| help='print the cmake-required legs instead: changed boards the pinned matrix does not build') | |
| parser.add_argument('--example-map', action='store_true', | |
| help="print the selection's per-family example filter instead") | |
| mb = parser.add_argument_group('membrowse', 'print the membrowse-identical legs instead (JSON values)') | |
| mb.add_argument('--membrowse', action='store_true') | |
| mb.add_argument('--pinned-json', default='{}', help="the cmake job's matrix") | |
| mb.add_argument('--hil-json', default='{}', help="hil-build's matrix (its esp-idf legs)") | |
| mb.add_argument('--hil-full-json', default='{}', help='the unscoped tinyusb.json matrix') | |
| mb.add_argument('--with-esp', action='store_true', help='hil-build-esp runs here (repository owner)') | |
| args = parser.parse_args() | |
| if args.membrowse: | |
| membrowse_json(json.loads(args.pinned_json), json.loads(args.hil_json).get('esp-idf', []), | |
| json.loads(args.hil_full_json).get('esp-idf', []), args.with_esp) | |
| return | |
| select = None | |
| try: | |
| if args.select: | |
| select = json.loads(args.select) | |
| elif args.select_file: | |
| with open(args.select_file) as f: | |
| select = json.load(f) | |
| elif args.base: | |
| r = subprocess.run([sys.executable, os.path.join(REPO, '.claude', 'skills', 'build', 'scripts', | |
| 'check_build.py'), '--select-only', '--base', args.base], | |
| capture_output=True, text=True, cwd=REPO, check=True) | |
| select = json.loads(r.stdout.splitlines()[-1]) | |
| except Exception as e: # fail-open: an unusable selection must never turn into a red job | |
| # UNSCOPED is the marker build.yml greps for: it must then drop the build extras | |
| # (example map, family regex) too, or a full build gets labelled and filtered as | |
| # a scoped one. Keep the token on every fall-open path. | |
| print(f'ci_set_matrix: UNSCOPED - selection unusable ({e}), emitting the full ' | |
| f'matrix', file=sys.stderr) | |
| select = None | |
| if args.example_map: | |
| example_map_json(select) | |
| elif args.required: | |
| required_json(select) | |
| else: | |
| set_matrix_json(select, args.pinned) | |
| if __name__ == '__main__': | |
| main() |