tools/build.py

#!/usr/bin/env python3
import argparse
import random
import os
import sys
import time
import subprocess
from pathlib import Path
from multiprocessing import Pool

import build_utils

STATUS_OK = "\033[32mOK\033[0m"
STATUS_FAILED = "\033[31mFailed\033[0m"
STATUS_SKIPPED = "\033[33mSkipped\033[0m"

RET_OK = 0
RET_FAILED = 1
RET_SKIPPED = 2

build_format = '| {:30} | {:40} | {:16} | {:5} |'
build_separator = '-' * 95
build_status = [STATUS_OK, STATUS_FAILED, STATUS_SKIPPED]

verbose = False
parallel_jobs = os.cpu_count()

# -----------------------------
# Helper
# -----------------------------
def run_cmd(cmd):
    #print(cmd)
    r = subprocess.run(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
    title = f'Command Error: {cmd}'
    if r.returncode != 0:
        # print build output if failed
        if os.getenv('GITHUB_ACTIONS'):
            print(f"::group::{title}")
            print(r.stdout.decode("utf-8"))
            print(f"::endgroup::")
        else:
            print(title)
            print(r.stdout.decode("utf-8"))
    elif verbose:
        print(cmd)
        print(r.stdout.decode("utf-8"))
    return r


def find_family(board):
    bsp_dir = Path("hw/bsp")
    for family_dir in bsp_dir.iterdir():
        if family_dir.is_dir():
            board_dir = family_dir / 'boards' / board
            if board_dir.exists():
                return family_dir.name
    return None


def get_examples(family):
    all_examples = []
    for d in os.scandir("examples"):
        if d.is_dir() and 'cmake' not in d.name and 'build_system' not in d.name:
            for entry in os.scandir(d.path):
                if entry.is_dir() and 'cmake' not in entry.name:
                    if family != 'espressif' or 'freertos' in entry.name:
                        all_examples.append(d.name + '/' + entry.name)

    if family == 'espressif':
        all_examples.append('device/board_test')
        all_examples.append('device/video_capture')
        all_examples.append('host/device_info')
    all_examples.sort()
    return all_examples


def print_build_result(board, example, status, duration):
    if isinstance(duration, (int, float)):
        duration = "{:.2f}s".format(duration)
    print(build_format.format(board, example, build_status[status], duration))

# -----------------------------
# CMake
# -----------------------------
def cmake_board(board, build_args, build_flags_on):
    ret = [0, 0, 0]
    start_time = time.monotonic()

    build_dir = f'cmake-build/cmake-build-{board}'
    build_flags = ''
    if len(build_flags_on) > 0:
        build_flags =  ' '.join(f'-D{flag}=1' for flag in build_flags_on)
        build_flags = f'-DCFLAGS_CLI="{build_flags}"'
        build_dir += '-f1_' + '_'.join(build_flags_on)

    family = find_family(board)
    if family == 'espressif':
        # for espressif, we have to build example individually
        all_examples = get_examples(family)
        for example in all_examples:
            if build_utils.skip_example(example, board):
                ret[2] += 1
            else:
                rcmd = run_cmd(f'idf.py -C examples/{example} -B {build_dir}/{example} -G Ninja '
                               f'-DBOARD={board} {build_flags} build')
                ret[0 if rcmd.returncode == 0 else 1] += 1
    else:
        rcmd = run_cmd(f'cmake examples -B {build_dir} -G Ninja -DBOARD={board} -DCMAKE_BUILD_TYPE=MinSizeRel '
                       f'{build_args} {build_flags}')
        if rcmd.returncode == 0:
            cmd = f"cmake --build {build_dir}"
            njobs = parallel_jobs

            # circleci docker return $nproc as 36 core, limit parallel according to resource class.
            # Required for IAR, also prevent crashed/killed by docker
            if os.getenv('CIRCLECI'):
                resource_class = { 'small': 1, 'medium': 2, 'medium+': 3, 'large': 4 }
                for rc in resource_class:
                    if rc in os.getenv('CIRCLE_JOB'):
                        njobs = resource_class[rc]
                        break
            cmd += f' --parallel {njobs}'
            rcmd = run_cmd(cmd)
        ret[0 if rcmd.returncode == 0 else 1] += 1

    example = 'all'
    print_build_result(board, example, 0 if ret[1] == 0 else 1, time.monotonic() - start_time)
    return ret


# -----------------------------
# Make
# -----------------------------
def make_one_example(example, board, make_option):
    # Check if board is skipped
    if build_utils.skip_example(example, board):
        print_build_result(board, example, 2, '-')
        r = 2
    else:
        start_time = time.monotonic()
        # skip -j for circleci
        if not os.getenv('CIRCLECI'):
            make_option += ' -j'
        make_cmd = f"make -C examples/{example} BOARD={board} {make_option}"
        # run_cmd(f"{make_cmd} clean")
        build_result = run_cmd(f"{make_cmd} all")
        r = 0 if build_result.returncode == 0 else 1
        print_build_result(board, example, r, time.monotonic() - start_time)

    ret = [0, 0, 0]
    ret[r] = 1
    return ret


def make_board(board, build_args):
    print(build_separator)
    family = find_family(board);
    all_examples = get_examples(family)
    start_time = time.monotonic()
    ret = [0, 0, 0]
    if family == 'espressif' or family == 'rp2040':
        # espressif and rp2040 do not support make, use cmake instead
        final_status = 2
    else:
        with Pool(processes=os.cpu_count()) as pool:
            pool_args = list((map(lambda e, b=board, o=f"{build_args}": [e, b, o], all_examples)))
            r = pool.starmap(make_one_example, pool_args)
            # sum all element of same index (column sum)
            ret = list(map(sum, list(zip(*r))))
        final_status = 0 if ret[1] == 0 else 1
    print_build_result(board, 'all', final_status, time.monotonic() - start_time)
    return ret


# -----------------------------
# Build Family
# -----------------------------
def build_boards_list(boards, build_defines, build_system, build_flags_on):
    ret = [0, 0, 0]
    for b in boards:
        r = [0, 0, 0]
        if build_system == 'cmake':
            build_args = ' '.join(f'-D{d}' for d in build_defines)
            r = cmake_board(b, build_args, build_flags_on)
        elif build_system == 'make':
            build_args = ' '.join(f'{d}' for d in build_defines)
            r = make_board(b, build_args)
        ret[0] += r[0]
        ret[1] += r[1]
        ret[2] += r[2]
    return ret


def build_family(family, build_defines, build_system, build_flags_on, one_per_family, boards):
    skip_ci = ['pico_sdk']
    if os.getenv('GITHUB_ACTIONS') or os.getenv('CIRCLECI'):
        skip_ci_file = Path(f"hw/bsp/{family}/skip_ci.txt")
        if skip_ci_file.exists():
            skip_ci = skip_ci_file.read_text().split()
    all_boards = []
    for entry in os.scandir(f"hw/bsp/{family}/boards"):
        if entry.is_dir() and not entry.name in skip_ci:
            all_boards.append(entry.name)
    all_boards.sort()

    ret = [0, 0, 0]
    # If only-one flag is set, select one random board
    if one_per_family:
        for b in boards:
            # skip if -b already specify one in this family
            if find_family(b) == family:
                return ret
        all_boards = [random.choice(all_boards)]

    ret = build_boards_list(all_boards, build_defines, build_system, build_flags_on)
    return ret


# -----------------------------
# Main
# -----------------------------
def main():
    global verbose
    global parallel_jobs

    parser = argparse.ArgumentParser()
    parser.add_argument('families', nargs='*', default=[], help='Families to build')
    parser.add_argument('-b', '--board', action='append', default=[], help='Boards to build')
    parser.add_argument('-t', '--toolchain', default='gcc', help='Toolchain to use, default is gcc')
    parser.add_argument('-s', '--build-system', default='cmake', help='Build system to use, default is cmake')
    parser.add_argument('-D', '--define-symbol', action='append', default=[], help='Define to pass to build system')
    parser.add_argument('-f1', '--build-flags-on', action='append', default=[], help='Build flag to pass to build system')
    parser.add_argument('-1', '--one-per-family', action='store_true', default=False, help='Build only one random board inside a family')
    parser.add_argument('-j', '--jobs', type=int, default=os.cpu_count(), help='Number of jobs to run in parallel')
    parser.add_argument('-v', '--verbose', action='store_true', help='Verbose output')
    args = parser.parse_args()

    families = args.families
    boards = args.board
    toolchain = args.toolchain
    build_system = args.build_system
    build_defines = args.define_symbol
    build_flags_on = args.build_flags_on
    one_per_family = args.one_per_family
    verbose = args.verbose
    parallel_jobs = args.jobs

    build_defines.append(f'TOOLCHAIN={toolchain}')

    if len(families) == 0 and len(boards) == 0:
        print("Please specify families or board to build")
        return 1

    print(build_separator)
    print(build_format.format('Board', 'Example', '\033[39mResult\033[0m', 'Time'))
    total_time = time.monotonic()
    result = [0, 0, 0]

    # build families
    all_families = []
    if 'all' in families:
        for entry in os.scandir("hw/bsp"):
            if entry.is_dir() and entry.name != 'espressif' and os.path.isfile(entry.path + "/family.cmake"):
                all_families.append(entry.name)
    else:
        all_families = list(families)
    all_families.sort()

    # succeeded, failed, skipped
    for f in all_families:
        r = build_family(f, build_defines, build_system, build_flags_on, one_per_family, boards)
        result[0] += r[0]
        result[1] += r[1]
        result[2] += r[2]

    # build boards
    r = build_boards_list(boards, build_defines, build_system, build_flags_on)
    result[0] += r[0]
    result[1] += r[1]
    result[2] += r[2]

    total_time = time.monotonic() - total_time
    print(build_separator)
    print(f"Build Summary: {result[0]} {STATUS_OK}, {result[1]} {STATUS_FAILED} and took {total_time:.2f}s")
    print(build_separator)
    return result[1]


if __name__ == '__main__':
    sys.exit(main())