| #!/usr/bin/env python3 |
| """TinyUSB front end to the membrowse CLI. |
| |
| MEMBROWSE_API_KEY is read from the environment at run time (only with --upload), |
| never at CMake configure time, and redacted from the logged command line. |
| """ |
| import argparse |
| import os |
| import re |
| import shlex |
| import subprocess |
| import sys |
| |
| # `--defsym=SYM=VAL` or `--defsym,SYM=VAL` from a link line |
| DEFSYM_RE = re.compile(r'--defsym[=,](\S+)') |
| # ESP-IDF's generated scripts, relative to the build dir (family_support.cmake's MEMBROWSE_LD_OVERRIDE) |
| IDF_LD_SCRIPTS = ('esp-idf/esp_system/ld/memory.ld', 'esp-idf/esp_system/ld/sections.ld') |
| NINJA_UTF8_REMEDY = ('CMake writes build.ninja in the ANSI code page when ninja is older than 1.11 or ' |
| '`ninja -t wincodepage` is not UTF-8; use ninja >= 1.11 whose `ninja -t wincodepage` ' |
| 'reports UTF-8, or keep the checkout, CMake and toolchain paths ASCII-only') |
| |
| |
| def link_command(ninja, build_dir, elf): |
| """The command that links `elf` (a path valid from the cwd), from |
| `<ninja> -C <build_dir> -t commands -s`: only the final command, not the helper |
| executables' links (pico-sdk's boot_stage2) a target also depends on. |
| |
| Raises RuntimeError on failure or no command: an empty result would silently |
| report against membrowse's default regions instead of the real linker scripts.""" |
| rel = os.path.relpath(elf, build_dir) |
| cmd = [ninja, '-C', build_dir, '-t', 'commands', '-s', rel] |
| r = subprocess.run(cmd, capture_output=True) |
| # decoded here: text=True decodes in a reader thread on Windows, which drops a bad stdout to None |
| try: |
| stdout, stderr = r.stdout.decode('utf-8'), r.stderr.decode('utf-8') |
| except UnicodeDecodeError as e: |
| raise RuntimeError(f"'{' '.join(cmd)}' output is not UTF-8: {NINJA_UTF8_REMEDY}") from e |
| if r.returncode != 0: |
| raise RuntimeError(f"'{' '.join(cmd)}' failed (exit {r.returncode}):\n{stderr}") |
| if not stdout.strip(): |
| raise RuntimeError(f'no command builds {rel!r} in the ninja build graph of ' |
| f'{build_dir} - cannot determine its linker scripts') |
| return stdout |
| |
| |
| def extract_ld_scripts(commands_text): |
| """Deduped linker scripts referenced by a link command.""" |
| scripts = [] |
| for line in commands_text.splitlines(): |
| try: |
| args = shlex.split(line) |
| # Windows ninja links through `cmd.exe /C "cd . && <link command>"` |
| if len(args) >= 3 and args[0].lower().endswith('cmd.exe') and args[1].lower() == '/c': |
| args = shlex.split(args[2]) |
| except ValueError: |
| continue |
| for i, arg in enumerate(args): |
| if arg == '-T' and i + 1 < len(args): |
| scripts.append(args[i + 1]) |
| elif arg.startswith('-T') and len(arg) > 2: |
| scripts.append(arg[2:]) |
| elif arg.startswith('-Wl,--script='): |
| scripts.append(arg.split('=', 1)[1]) |
| return list(dict.fromkeys(scripts)) |
| |
| |
| def extract_defsyms(commands_text): |
| """`VAR=VALUE` --defsym values referenced in a link command, deduped, in the |
| order first encountered.""" |
| return list(dict.fromkeys(DEFSYM_RE.findall(commands_text))) |
| |
| |
| def report_inputs(elf, ld_scripts, defsyms): |
| """The membrowse report argv naming one elf's inputs: elf, linker scripts, |
| --def per defsym, and its .map when present.""" |
| inputs = [elf, ' '.join(ld_scripts)] |
| for sym in defsyms: |
| inputs += ['--def', sym] |
| map_path = elf + '.map' |
| if os.path.isfile(map_path): |
| inputs += ['--map-file', map_path] |
| return inputs |
| |
| |
| def build_membrowse_cmd(args, commands_text): |
| """Compose the membrowse argv (list form, no shell) for this report.""" |
| if args.ld is not None: |
| ld_scripts = list(args.ld) |
| else: |
| ld_scripts = extract_ld_scripts(commands_text) |
| |
| cmd = ['membrowse', 'report'] + shlex.split(args.option) |
| if args.elf and os.path.isfile(args.elf): |
| if not ld_scripts: |
| # same silent default-regions report as a failed ninja query |
| sys.exit(f'error: no linker script found in the ninja build graph for ' |
| f'{args.elf!r}; pass --ld to supply linker scripts explicitly') |
| cmd += report_inputs(args.elf, ld_scripts, extract_defsyms(commands_text)) |
| else: |
| cmd += ['--identical'] |
| |
| if args.upload: |
| cmd += ['--upload', '--github', '--target-name', args.target_name] |
| # no key (fork PRs: GHA withholds secrets) -> membrowse's tokenless GHA auth |
| key = os.environ.get('MEMBROWSE_API_KEY') |
| if key: |
| cmd += ['--api-key', key] |
| |
| return cmd |
| |
| |
| def redacted(cmd, secret): |
| """argv with `secret` masked, for logging. By value, not by the flag before it: |
| membrowse onboard takes the key as a positional (compose()).""" |
| return [('***' if secret and part == secret else part) for part in cmd] |
| |
| |
| def expected_built(target_name): |
| """Whether the leg's Build step was meant to produce this target: TUSB_MEMBROWSE_EXPECT_BUILT |
| (build.py --expect-built) holds 'all' or the example names it built.""" |
| expect = os.environ.get('TUSB_MEMBROWSE_EXPECT_BUILT', '').split() |
| return 'all' in expect or target_name.rsplit('/', 1)[-1] in expect |
| |
| |
| def report(args): |
| built = not args.identical_only and os.path.isfile(args.elf) |
| if not built and expected_built(args.target_name): |
| # a failed build is a failure, never an --identical upload that hides it |
| raise RuntimeError(f'{args.target_name} was expected built, but there is no elf' |
| f'{" at " + args.elf if args.elf else ""}') |
| # an --identical report (no elf) must work against a never-configured build dir |
| link = link_command(args.ninja, args.build_dir, args.elf) if built else '' |
| cmd = build_membrowse_cmd(args, link) |
| |
| # flush: piped stdout is block-buffered and the child's output would land first |
| logged = redacted(cmd, os.environ.get('MEMBROWSE_API_KEY')) |
| print(' '.join(shlex.quote(p) for p in logged), flush=True) |
| |
| # from the link's own working dir, so a `-L .` INCLUDE resolves (pico-sdk's |
| # generated pico_flash_region.ld) |
| return subprocess.run(cmd, cwd=args.build_dir if link else None).returncode |
| |
| |
| def compose(board, example, num_commits, upload, api_key, extra, family=None): |
| """Return the membrowse onboard argv for one board/example backfill.""" |
| basename = os.path.basename(example.rstrip('/')) |
| build_dir = f'examples/cmake-build-{board}' |
| # `membrowse onboard` runs `git clean -fdx` before every historical build, |
| # which deletes this ignored build_dir - reconfigure it fresh each time. |
| python = shlex.quote(sys.executable) |
| quoted_build_dir = shlex.quote(build_dir) |
| if family == 'espressif': |
| build = (f'idf.py -C {shlex.quote(f"examples/{example}")} -B {quoted_build_dir} ' |
| f'-GNinja {shlex.quote(f"-DBOARD={board}")} build') |
| elf_path = f'{build_dir}/{basename}.elf' |
| shim_scripts = IDF_LD_SCRIPTS |
| else: |
| configure = (f'cmake -S examples -B {quoted_build_dir} ' |
| f'{shlex.quote(f"-DBOARD={board}")} -G Ninja ' |
| f'-DCMAKE_BUILD_TYPE=MinSizeRel') |
| build = f'{configure} && cmake --build {quoted_build_dir} --target {shlex.quote(basename)}' |
| elf_path = f'{build_dir}/{example}/{basename}.elf' |
| shim_scripts = () |
| build_script = f'{python} tools/get_deps.py -b {shlex.quote(board)} && {build}' |
| shim_path = f'{build_dir}/.membrowse-onboard.ld' |
| # absolute path of this script: the historical tree may predate linker-shim |
| write_shim = (f'{python} {shlex.quote(os.path.abspath(__file__))} ' |
| f'linker-shim ninja ' |
| f'{quoted_build_dir} {shlex.quote(elf_path)} {shlex.quote(shim_path)}') |
| if shim_scripts: |
| write_shim += ' ' + ' '.join(map(shlex.quote, shim_scripts)) |
| build_script = f'{build_script} && {write_shim}' |
| cmd = ['membrowse', 'onboard'] |
| if '--commits' not in extra: |
| cmd.append(str(num_commits)) |
| cmd += [ |
| build_script, |
| elf_path, |
| f'{board}/{basename}', |
| api_key, |
| ] |
| if '--binary-search' not in extra: |
| role = example.split('/', 1)[0] |
| cmd += ['--build-dirs', 'src/', 'hw/', 'lib/', 'examples/build_system/', |
| 'examples/CMakeLists.txt', f'examples/{role}/CMakeLists.txt', |
| f'examples/{example.rstrip("/")}/', 'tools/get_deps.py'] |
| cmd += ['--ld-scripts', shim_path] |
| if not upload: |
| cmd.append('--dry-run') |
| cmd += extra |
| return cmd |
| |
| |
| def write_linker_shim(ninja, build_dir, elf, output, *scripts): |
| """Write the linker scripts and defsyms of `elf`'s link to a stable path.""" |
| commands = link_command(ninja, build_dir, elf) |
| scripts = scripts or extract_ld_scripts(commands) |
| if not scripts: |
| sys.exit(f'no linker script found in the ninja build graph for {elf!r}') |
| |
| lines = [] |
| for sym in extract_defsyms(commands): |
| name, separator, value = sym.partition('=') |
| if not separator: |
| sys.exit(f'invalid --defsym value in ninja build graph: {sym!r}') |
| lines.append(f'{name} = {value};') |
| for script in scripts: |
| path = script if os.path.isabs(script) else os.path.abspath(os.path.join(build_dir, script)) |
| if '"' in path or not os.path.isfile(path): |
| sys.exit(f'linker script not found or unsupported: {path!r}') |
| lines.append(f'INCLUDE "{path}"') |
| with open(output, 'w') as f: |
| f.write('\n'.join(lines) + '\n') |
| return 0 |
| |
| |
| def onboard(args, extra): |
| """Backfill membrowse history for one CI board/example. |
| |
| Thin wrapper over `membrowse onboard` that derives every convention-bound |
| argument from the board and example name, so a backfill cannot land under a |
| target name that differs from what CI uploads (family_support.cmake uses |
| `<board>/<cmake-target>`, i.e. the example BASENAME - not the role/name path). |
| Each historical build writes a small linker shim from that commit's ninja graph, |
| so linker scripts and --defsym values match the ELF being reported rather than |
| the caller's current checkout. |
| |
| `membrowse onboard` checks out and `git clean -fdx`s every historical commit in |
| place. This wrapper confines that to a disposable worktree; each build then runs |
| that commit's `tools/get_deps.py` so dependency revisions match the ELF. |
| |
| `membrowse onboard` reports from the repo root, dropping an INCLUDE found only via -L: |
| rp2040/rp2350 backfills infer FLASH from the ELF (pico-sdk's pico_flash_region.ld). |
| """ |
| api_key = os.environ.get('MEMBROWSE_API_KEY') |
| if args.upload and not api_key: |
| sys.exit('MEMBROWSE_API_KEY must be set in the environment for --upload') |
| |
| repo_root = os.getcwd() |
| worktree_dir = os.path.join(repo_root, 'cmake-metrics', '_onboard_worktree') |
| |
| family = 'espressif' if os.path.isdir( |
| os.path.join(repo_root, 'hw', 'bsp', 'espressif', 'boards', args.board)) else None |
| cmd = compose(args.board, args.example, args.num_commits, args.upload, |
| api_key or 'dry-run-placeholder', extra, family) |
| |
| print('+ ' + ' '.join(shlex.quote(p) for p in redacted(cmd, api_key)), flush=True) |
| |
| if os.path.isdir(worktree_dir): |
| subprocess.run(['git', 'worktree', 'remove', '--force', worktree_dir], |
| capture_output=True) |
| os.makedirs(os.path.dirname(worktree_dir), exist_ok=True) |
| ret = subprocess.run(['git', 'worktree', 'add', '--detach', worktree_dir, 'HEAD'], |
| capture_output=True, text=True) |
| if ret.returncode != 0: |
| sys.exit(f'failed to create disposable worktree at {worktree_dir}:\n{ret.stderr}') |
| try: |
| return subprocess.run(cmd, cwd=worktree_dir).returncode |
| finally: |
| subprocess.run(['git', 'worktree', 'remove', '--force', worktree_dir], |
| capture_output=True) |
| |
| |
| def main(argv=None): |
| parser = argparse.ArgumentParser(description=__doc__, |
| formatter_class=argparse.RawDescriptionHelpFormatter) |
| # linker-shim is compose()'s internal callback: no help= and off the metavar hides it |
| sub = parser.add_subparsers(dest='command', required=True, metavar='{report,onboard}') |
| |
| rep = sub.add_parser('report', help='size report for one build target') |
| rep.add_argument('--build-dir', help='CMake binary dir (ninja -C)') |
| rep.add_argument('--ninja', help='ninja executable') |
| # absolute: membrowse runs from the build dir |
| rep.add_argument('--elf', type=os.path.abspath, help='path to the built ELF') |
| rep.add_argument('--identical-only', action='store_true', |
| help='report the target unchanged (--identical), with no build to read; ' |
| 'takes none of --build-dir/--ninja/--elf') |
| rep.add_argument('--option', default='', |
| help='extra membrowse report options, space-separated ' |
| '(cmake -DMEMBROWSE_OPTION=...)') |
| rep.add_argument('--target-name', required=True, help='<board>/<example>, used with --upload') |
| rep.add_argument('--upload', action='store_true', help='upload the report to membrowse') |
| rep.add_argument('--ld', nargs='+', default=None, |
| help='override linker scripts verbatim, skipping ninja extraction ' |
| '(espressif: its final link references generated scripts by bare ' |
| 'filename, which ninja-graph extraction cannot resolve)') |
| |
| onb = sub.add_parser('onboard', help='backfill membrowse history for one CI board/example', |
| description='Backfill membrowse history for one CI board/example ' |
| '(dry-run unless --upload). Extra args after -- go to ' |
| '`membrowse onboard` verbatim (see its --help). ' |
| '--upload reads MEMBROWSE_API_KEY from the environment.') |
| onb.add_argument('board', help='Board name, e.g. stm32f407disco') |
| onb.add_argument('example', help='Example as role/name, e.g. device/cdc_msc') |
| onb.add_argument('-n', '--num-commits', type=int, default=30, |
| help='Most recent N commits to backfill (default: 30)') |
| onb.add_argument('--upload', action='store_true', default=False, |
| help='Really upload (default is --dry-run)') |
| |
| shim = sub.add_parser('linker-shim') |
| shim.add_argument('ninja') |
| shim.add_argument('build_dir') |
| shim.add_argument('elf') |
| shim.add_argument('output') |
| shim.add_argument('scripts', nargs='*', help='linker scripts, instead of the ninja graph\'s') |
| |
| args, extra = parser.parse_known_args(argv) |
| if extra and args.command != 'onboard': |
| sub.choices[args.command].error(f'unrecognized arguments: {" ".join(extra)}') |
| if args.command == 'report': |
| given = [f for f in ('build_dir', 'ninja', 'elf') if getattr(args, f) is not None] |
| if args.identical_only and given: |
| rep.error('--identical-only takes none of --build-dir/--ninja/--elf') |
| if not args.identical_only and len(given) < 3: |
| rep.error('--build-dir, --ninja and --elf are required unless --identical-only') |
| try: |
| if args.command == 'onboard': |
| return onboard(args, extra) |
| if args.command == 'report': |
| return report(args) |
| return write_linker_shim(args.ninja, args.build_dir, args.elf, args.output, *args.scripts) |
| except RuntimeError as e: |
| sys.exit(f'error: {e}') |
| |
| |
| if __name__ == '__main__': |
| sys.exit(main()) |
