| #!/usr/bin/env python3 |
| """Zephyr Sanity Tests |
| |
| This script scans for the set of unit test applications in the git |
| repository and attempts to execute them. By default, it tries to |
| build each test case on one platform per architecture, using a precedence |
| list defined in an archtecture configuration file, and if possible |
| run the tests in the QEMU emulator. |
| |
| Test cases are detected by the presence of a 'testcase.ini' file in |
| the application's project directory. This file may contain one or |
| more blocks, each identifying a test scenario. The title of the block |
| is a name for the test case, which only needs to be unique for the |
| test cases specified in that testcase.ini file. The full canonical |
| name for each test case is <path to test case under samples/>/<block>. |
| |
| Each testcase.ini block can define the following key/value pairs: |
| |
| tags = <list of tags> (required) |
| A set of string tags for the testcase. Usually pertains to |
| functional domains but can be anything. Command line invocations |
| of this script can filter the set of tests to run based on tag. |
| |
| skip = <True|False> (default False) |
| skip testcase unconditionally. This can be used for broken tests. |
| |
| slow = <True|False> (default False) |
| Don't run this test case unless --enable-slow was passed in on the |
| command line. Intended for time-consuming test cases that are only |
| run under certain circumstances, like daily builds. These test cases |
| are still compiled. |
| |
| kernel = <nano|micro> |
| Specify the kernel being tested instead of relying on parsing |
| Makefile for KERNEL_TYPE |
| |
| extra_args = <list of extra arguments> |
| Extra arguments to pass to Make when building or running the |
| test case. |
| |
| build_only = <True|False> (default False) |
| If true, don't try to run the test under QEMU even if the |
| selected platform supports it. |
| |
| timeout = <number of seconds> |
| Length of time to run test in QEMU before automatically killing it. |
| Default to 60 seconds. |
| |
| arch_whitelist = <list of arches, such as x86, arm, arc> |
| Set of architectures that this test case should only be run for. |
| |
| arch_exclude = <list of arches, such as x86, arm, arc> |
| Set of architectures that this test case should not run on. |
| |
| platform_whitelist = <list of platforms> |
| Set of platforms that this test case should only be run for. |
| |
| platform_exclude = <list of platforms> |
| Set of platforms that this test case should not run on. |
| |
| config_whitelist = <list of config options> |
| Config options can either be config names like CONFIG_FOO which |
| match if the configuration is defined to any value, or key/value |
| pairs like CONFIG_FOO=bar which match if it is set to a specific |
| value. May prepend a '!' to invert the match. |
| |
| Architectures and platforms are defined in an archtecture configuration |
| file which are stored by default in scripts/sanity_chk/arches/. These |
| each define an [arch] block with the following key/value pairs: |
| |
| name = <arch name> |
| The name of the arch. Example: x86 |
| |
| platforms = <list of supported platforms in order of precedence> |
| List of supported platforms for this arch. The ordering here |
| is used to select a default platform to build for that arch. |
| |
| For every platform defined, there must be a corresponding block for it |
| in the arch configuration file. This block can be empty if there are |
| no special definitions for that arch. Options are: |
| |
| qemu_support = <True|False> (default False) |
| Indicates whether binaries for this platform can run under QEMU |
| |
| microkernel_support = <True|False> (default True) |
| Indicates whether this platform supports microkernel or just nanokernel |
| |
| The set of test cases that actually run depends on directives in the |
| testcase and archtecture .ini file and options passed in on the command |
| line. If there is every any confusion, running with -v or --discard-report |
| can help show why particular test cases were skipped. |
| |
| Metrics (such as pass/fail state and binary size) for the last code |
| release are stored in scripts/sanity_chk/sanity_last_release.csv. |
| To update this, pass the --all --release options. |
| |
| Most everyday users will run with no arguments. |
| """ |
| |
| import argparse |
| import os |
| import sys |
| import configparser |
| import re |
| import tempfile |
| import subprocess |
| import multiprocessing |
| import select |
| import shutil |
| import signal |
| import threading |
| import time |
| import csv |
| import glob |
| import concurrent |
| import concurrent.futures |
| |
| os.environ["DISABLE_TRYRUN"] = "1" |
| if "ZEPHYR_BASE" not in os.environ: |
| sys.stderr.write("$ZEPHYR_BASE environment variable undefined.\n") |
| exit(1) |
| ZEPHYR_BASE = os.environ["ZEPHYR_BASE"] |
| VERBOSE = 0 |
| LAST_SANITY = os.path.join(ZEPHYR_BASE, "scripts", "sanity_chk", |
| "last_sanity.csv") |
| RELEASE_DATA = os.path.join(ZEPHYR_BASE, "scripts", "sanity_chk", |
| "sanity_last_release.csv") |
| CPU_COUNTS = multiprocessing.cpu_count() |
| |
| if os.isatty(sys.stdout.fileno()): |
| TERMINAL = True |
| COLOR_NORMAL = '\033[0m' |
| COLOR_RED = '\033[91m' |
| COLOR_GREEN = '\033[92m' |
| COLOR_YELLOW = '\033[93m' |
| else: |
| TERMINAL = False |
| COLOR_NORMAL = "" |
| COLOR_RED = "" |
| COLOR_GREEN = "" |
| COLOR_YELLOW = "" |
| |
| class SanityCheckException(Exception): |
| pass |
| |
| class SanityRuntimeError(SanityCheckException): |
| pass |
| |
| class ConfigurationError(SanityCheckException): |
| def __init__(self, cfile, message): |
| self.cfile = cfile |
| self.message = message |
| |
| def __str__(self): |
| return repr(self.cfile + ": " + self.message) |
| |
| class MakeError(SanityCheckException): |
| pass |
| |
| class BuildError(MakeError): |
| pass |
| |
| class ExecutionError(MakeError): |
| pass |
| |
| # Debug Functions |
| def info(what): |
| sys.stdout.write(what + "\n") |
| |
| def error(what): |
| sys.stderr.write(COLOR_RED + what + COLOR_NORMAL + "\n") |
| |
| def debug(what): |
| if VERBOSE >= 1: |
| info(what) |
| |
| def verbose(what): |
| if VERBOSE >= 2: |
| info(what) |
| |
| # Utility functions |
| class QEMUHandler: |
| """Spawns a thread to monitor QEMU output from pipes |
| |
| We pass QEMU_PIPE to 'make qemu' and monitor the pipes for output. |
| We need to do this as once qemu starts, it runs forever until killed. |
| Test cases emit special messages to the console as they run, we check |
| for these to collect whether the test passed or failed. |
| """ |
| RUN_PASSED = "PROJECT EXECUTION SUCCESSFUL" |
| RUN_FAILED = "PROJECT EXECUTION FAILED" |
| |
| @staticmethod |
| def _thread(handler, timeout, outdir, logfile, fifo_fn, pid_fn, results): |
| fifo_in = fifo_fn + ".in" |
| fifo_out = fifo_fn + ".out" |
| |
| # These in/out nodes are named from QEMU's perspective, not ours |
| if os.path.exists(fifo_in): |
| os.unlink(fifo_in) |
| os.mkfifo(fifo_in) |
| if os.path.exists(fifo_out): |
| os.unlink(fifo_out) |
| os.mkfifo(fifo_out) |
| |
| # We don't do anything with out_fp but we need to open it for |
| # writing so that QEMU doesn't block, due to the way pipes work |
| out_fp = open(fifo_in, "wb") |
| # Disable internal buffering, we don't |
| # want read() or poll() to ever block if there is data in there |
| in_fp = open(fifo_out, "rb", buffering=0) |
| log_out_fp = open(logfile, "wt") |
| |
| start_time = time.time() |
| timeout_time = start_time + timeout |
| p = select.poll() |
| p.register(in_fp, select.POLLIN) |
| |
| metrics = {} |
| line = "" |
| while True: |
| this_timeout = int((timeout_time - time.time()) * 1000) |
| if this_timeout < 0 or not p.poll(this_timeout): |
| out_state = "timeout" |
| break |
| |
| c = in_fp.read(1).decode("utf-8") |
| if c == "": |
| # EOF, this shouldn't happen unless QEMU crashes |
| out_state = "unexpected eof" |
| break |
| line = line + c |
| if c != "\n": |
| continue |
| |
| # If we get here, line contains a full line of data output from QEMU |
| log_out_fp.write(line) |
| log_out_fp.flush() |
| line = line.strip() |
| verbose("QEMU: %s" % line) |
| |
| if line == QEMUHandler.RUN_PASSED: |
| out_state = "passed" |
| break |
| |
| if line == QEMUHandler.RUN_FAILED: |
| out_state = "failed" |
| break |
| |
| # TODO: Add support for getting numerical performance data |
| # from test cases. Will involve extending test case reporting |
| # APIs. Add whatever gets reported to the metrics dictionary |
| line = "" |
| |
| metrics["qemu_time"] = time.time() - start_time |
| verbose("QEMU complete (%s) after %f seconds" % |
| (out_state, metrics["qemu_time"])) |
| handler.set_state(out_state, metrics) |
| |
| log_out_fp.close() |
| out_fp.close() |
| in_fp.close() |
| |
| pid = int(open(pid_fn).read()) |
| os.unlink(pid_fn) |
| try: |
| os.kill(pid, signal.SIGTERM) |
| except ProcessLookupError: |
| # Oh well, as long as it's dead! User probably sent Ctrl-C |
| pass |
| |
| os.unlink(fifo_in) |
| os.unlink(fifo_out) |
| |
| |
| def __init__(self, name, outdir, log_fn, timeout): |
| """Constructor |
| |
| @param name Arbitrary name of the created thread |
| @param outdir Working directory, shoudl be where qemu.pid gets created |
| by kbuild |
| @param log_fn Absolute path to write out QEMU's log data |
| @param timeout Kill the QEMU process if it doesn't finish up within |
| the given number of seconds |
| """ |
| # Create pipe to get QEMU's serial output |
| self.results = {} |
| self.state = "waiting" |
| self.lock = threading.Lock() |
| |
| # We pass this to QEMU which looks for fifos with .in and .out |
| # suffixes. |
| self.fifo_fn = os.path.join(outdir, "qemu-fifo") |
| |
| self.pid_fn = os.path.join(outdir, "qemu.pid") |
| if os.path.exists(self.pid_fn): |
| os.unlink(self.pid_fn) |
| |
| self.log_fn = log_fn |
| self.thread = threading.Thread(name=name, target=QEMUHandler._thread, |
| args=(self, timeout, outdir, self.log_fn, |
| self.fifo_fn, self.pid_fn, |
| self.results)) |
| self.thread.daemon = True |
| verbose("Spawning QEMU process for %s" % name) |
| self.thread.start() |
| |
| def set_state(self, state, metrics): |
| self.lock.acquire() |
| self.state = state |
| self.metrics = metrics |
| self.lock.release() |
| |
| def get_state(self): |
| self.lock.acquire() |
| ret = (self.state, self.metrics) |
| self.lock.release() |
| return ret |
| |
| def get_fifo(self): |
| return self.fifo_fn |
| |
| |
| class SizeCalculator: |
| |
| alloc_sections = ["bss", "noinit"] |
| rw_sections = ["datas", "initlevel", "_k_mem_map_ptr", "_k_pipe_ptr", |
| "_k_task_ptr", "_k_task_list", "_k_event_list"] |
| # These get copied into RAM only on non-XIP |
| ro_sections = ["text", "ctors", "init_array", |
| "rodata", "devconfig", "gpio_compat"] |
| |
| def __init__(self, filename): |
| """Constructor |
| |
| @param filename Path to the output binary |
| The <filename> is parsed by objdump to determine section sizes |
| """ |
| # Make sure this is an ELF binary |
| with open(filename, "rb") as f: |
| magic = f.read(4) |
| |
| if (magic != b'\x7fELF'): |
| raise SanityRuntimeError("%s is not an ELF binary" % filename) |
| |
| # Search for CONFIG_XIP in the ELF's list of symbols using NM and AWK. |
| # GREP can not be used as it returns an error if the symbol is not found. |
| is_xip_command = "nm " + filename + " | awk '/CONFIG_XIP/ { print $3 }'" |
| is_xip_output = subprocess.check_output(is_xip_command, shell=True, |
| stderr=subprocess.STDOUT).decode("utf-8").strip() |
| if is_xip_output.endswith("no symbols"): |
| raise SanityRuntimeError("%s has no symbol information" % filename) |
| self.is_xip = (len(is_xip_output) != 0) |
| |
| self.filename = filename |
| self.sections = [] |
| self.rom_size = 0 |
| self.ram_size = 0 |
| self.mismatches = [] |
| |
| self._calculate_sizes() |
| |
| def get_ram_size(self): |
| """Get the amount of RAM the application will use up on the device |
| |
| @return amount of RAM, in bytes |
| """ |
| return self.ram_size |
| |
| def get_rom_size(self): |
| """Get the size of the data that this application uses on device's flash |
| |
| @return amount of ROM, in bytes |
| """ |
| return self.rom_size |
| |
| def unrecognized_sections(self): |
| """Get a list of sections inside the binary that weren't recognized |
| |
| @return list of unrecogized section names |
| """ |
| slist = [] |
| for v in self.sections: |
| if not v["recognized"]: |
| slist.append(v["name"]) |
| return slist |
| |
| def mismatched_sections(self): |
| """Get a list of sections in the binary whose LMA and VMA offsets |
| from the previous section aren't proportional. This leads to issues |
| on XIP systems as they aren't correctly copied in to RAM |
| """ |
| slist = [] |
| for v in self.sections: |
| if v["lma_off"] != v["vma_off"]: |
| slist.append((v["name"], v["lma_off"], v["vma_off"])) |
| return slist |
| |
| def _calculate_sizes(self): |
| """ Calculate RAM and ROM usage by section """ |
| objdump_command = "objdump -h " + self.filename |
| objdump_output = subprocess.check_output(objdump_command, |
| shell=True).decode("utf-8").splitlines() |
| |
| for line in objdump_output: |
| words = line.split() |
| |
| if (len(words) == 0): # Skip lines that are too short |
| continue |
| |
| index = words[0] |
| if (not index[0].isdigit()): # Skip lines that do not start |
| continue # with a digit |
| |
| name = words[1] # Skip lines with section names |
| if (name[0] == '.'): # starting with '.' |
| continue |
| |
| # TODO this doesn't actually reflect the size in flash or RAM as |
| # it doesn't include linker-imposed padding between sections. |
| # It is close though. |
| size = int(words[2], 16) |
| if size == 0: |
| continue |
| |
| load_addr = int(words[4], 16) |
| virt_addr = int(words[3], 16) |
| |
| # Add section to memory use totals (for both non-XIP and XIP scenarios) |
| # Unrecognized section names are not included in the calculations. |
| recognized = True |
| if name in SizeCalculator.alloc_sections: |
| self.ram_size += size |
| stype = "alloc" |
| elif name in SizeCalculator.rw_sections: |
| self.ram_size += size |
| self.rom_size += size |
| stype = "rw" |
| elif name in SizeCalculator.ro_sections: |
| self.rom_size += size |
| if not self.is_xip: |
| self.ram_size += size |
| stype = "ro" |
| else: |
| stype = "unknown" |
| recognized = False |
| |
| lma_off = 0 |
| vma_off = 0 |
| |
| # Look for different section padding for LMA and VMA, if present |
| # this really messes up XIP systems as __csSet() copies all of |
| # them off flash into RAM as a single large block of memory |
| if self.is_xip and len(self.sections) > 0: |
| p = self.sections[-1] |
| |
| if stype == "rw" and p["type"] == "rw": |
| lma_off = load_addr - p["load_addr"] |
| vma_off = virt_addr - p["virt_addr"] |
| |
| self.sections.append({"name" : name, "load_addr" : load_addr, |
| "size" : size, "virt_addr" : virt_addr, |
| "type" : stype, "recognized" : recognized, |
| "lma_off" : lma_off, "vma_off" : vma_off}) |
| |
| |
| class MakeGoal: |
| """Metadata class representing one of the sub-makes called by MakeGenerator |
| |
| MakeGenerator returns a dictionary of these which can then be associdated |
| with TestInstances to get a complete picture of what happened during a test. |
| MakeGenerator is used for tasks outside of building tests (such as |
| defconfigs) which is why MakeGoal is a separate class from TestInstance. |
| """ |
| def __init__(self, name, text, qemu, make_log, build_log, run_log, |
| qemu_log): |
| self.name = name |
| self.text = text |
| self.qemu = qemu |
| self.make_log = make_log |
| self.build_log = build_log |
| self.run_log = run_log |
| self.qemu_log = qemu_log |
| self.make_state = "waiting" |
| self.failed = False |
| self.finished = False |
| self.reason = None |
| self.metrics = {} |
| |
| def get_error_log(self): |
| if self.make_state == "waiting": |
| # Shouldn't ever see this; breakage in the main Makefile itself. |
| return self.make_log |
| elif self.make_state == "building": |
| # Failure when calling the sub-make to build the code |
| return self.build_log |
| elif self.make_state == "running": |
| # Failure in sub-make for "make qemu", qemu probably failed to start |
| return self.run_log |
| elif self.make_state == "finished": |
| # QEMU finished, but timed out or otherwise wasn't successful |
| return self.qemu_log |
| |
| def fail(self, reason): |
| self.failed = True |
| self.finished = True |
| self.reason = reason |
| |
| def success(self): |
| self.finished = True |
| |
| def __str__(self): |
| if self.finished: |
| if self.failed: |
| return "[%s] failed (%s: see %s)" % (self.name, self.reason, |
| self.get_error_log()) |
| else: |
| return "[%s] passed" % self.name |
| else: |
| return "[%s] in progress (%s)" % (self.name, self.make_state) |
| |
| |
| class MakeGenerator: |
| """Generates a Makefile which just calls a bunch of sub-make sessions |
| |
| In any given test suite we may need to build dozens if not hundreds of |
| test cases. The cleanest way to parallelize this is to just let Make |
| do the parallelization, sharing the jobserver among all the different |
| sub-make targets. |
| """ |
| |
| GOAL_HEADER_TMPL = """.PHONY: {goal} |
| {goal}: |
| """ |
| |
| MAKE_RULE_TMPL = """\t@echo sanity_test_{phase} {goal} >&2 |
| \t$(MAKE) -C {directory} O={outdir} V={verb} EXTRA_CFLAGS=-Werror EXTRA_ASMFLAGS=-Wa,--fatal-warnings EXTRA_LDFLAGS=--fatal-warnings {args} >{logfile} 2>&1 |
| """ |
| |
| GOAL_FOOTER_TMPL = "\t@echo sanity_test_finished {goal} >&2\n\n" |
| |
| re_make = re.compile("sanity_test_([A-Za-z0-9]+) (.+)|$|make[:] \*\*\* [[](.+)[]] Error.+$") |
| |
| def __init__(self, base_outdir): |
| """MakeGenerator constructor |
| |
| @param base_outdir Intended to be the base out directory. A make.log |
| file will be created here which contains the output of the |
| top-level Make session, as well as the dynamic control Makefile |
| @param verbose If true, pass V=1 to all the sub-makes which greatly |
| increases their verbosity |
| """ |
| self.goals = {} |
| if not os.path.exists(base_outdir): |
| os.makedirs(base_outdir) |
| self.logfile = os.path.join(base_outdir, "make.log") |
| self.makefile = os.path.join(base_outdir, "Makefile") |
| |
| def _get_rule_header(self, name): |
| return MakeGenerator.GOAL_HEADER_TMPL.format(goal=name) |
| |
| def _get_sub_make(self, name, phase, workdir, outdir, logfile, args): |
| verb = "1" if VERBOSE else "0" |
| args = " ".join(args) |
| return MakeGenerator.MAKE_RULE_TMPL.format(phase=phase, goal=name, |
| outdir=outdir, |
| directory=workdir, verb=verb, |
| args=args, logfile=logfile) |
| |
| def _get_rule_footer(self, name): |
| return MakeGenerator.GOAL_FOOTER_TMPL.format(goal=name) |
| |
| def _add_goal(self, outdir): |
| if not os.path.exists(outdir): |
| os.makedirs(outdir) |
| |
| def add_build_goal(self, name, directory, outdir, args): |
| """Add a goal to invoke a Kbuild session |
| |
| @param name A unique string name for this build goal. The results |
| dictionary returned by execute() will be keyed by this name. |
| @param directory Absolute path to working directory, will be passed |
| to make -C |
| @param outdir Absolute path to output directory, will be passed to |
| Kbuild via -O=<path> |
| @param args Extra command line arguments to pass to 'make', typically |
| environment variables or specific Make goals |
| """ |
| self._add_goal(outdir) |
| build_logfile = os.path.join(outdir, "build.log") |
| text = (self._get_rule_header(name) + |
| self._get_sub_make(name, "building", directory, |
| outdir, build_logfile, args) + |
| self._get_rule_footer(name)) |
| self.goals[name] = MakeGoal(name, text, None, self.logfile, build_logfile, |
| None, None) |
| |
| def add_qemu_goal(self, name, directory, outdir, args, timeout=30): |
| """Add a goal to build a Zephyr project and then run it under QEMU |
| |
| The generated make goal invokes Make twice, the first time it will |
| build the default goal, and the second will invoke the 'qemu' goal. |
| The output of the QEMU session will be monitored, and terminated |
| either upon pass/fail result of the test program, or the timeout |
| is reached. |
| |
| @param name A unique string name for this build goal. The results |
| dictionary returned by execute() will be keyed by this name. |
| @param directory Absolute path to working directory, will be passed |
| to make -C |
| @param outdir Absolute path to output directory, will be passed to |
| Kbuild via -O=<path> |
| @param args Extra command line arguments to pass to 'make', typically |
| environment variables. Do not pass specific Make goals here. |
| @param timeout Maximum length of time QEMU session should be allowed |
| to run before automatically killing it. Default is 30 seconds. |
| """ |
| |
| self._add_goal(outdir) |
| build_logfile = os.path.join(outdir, "build.log") |
| run_logfile = os.path.join(outdir, "run.log") |
| qemu_logfile = os.path.join(outdir, "qemu.log") |
| |
| q = QEMUHandler(name, outdir, qemu_logfile, timeout) |
| args.append("QEMU_PIPE=%s" % q.get_fifo()) |
| text = (self._get_rule_header(name) + |
| self._get_sub_make(name, "building", directory, |
| outdir, build_logfile, args) + |
| self._get_sub_make(name, "running", directory, |
| outdir, run_logfile, |
| args + ["qemu"]) + |
| self._get_rule_footer(name)) |
| self.goals[name] = MakeGoal(name, text, q, self.logfile, build_logfile, |
| run_logfile, qemu_logfile) |
| |
| |
| def add_test_instance(self, ti, build_only=False, enable_slow=False): |
| """Add a goal to build/test a TestInstance object |
| |
| @param ti TestInstance object to build. The status dictionary returned |
| by execute() will be keyed by its .name field. |
| """ |
| args = ti.test.extra_args[:] |
| args.extend(["ARCH=%s" % ti.platform.arch.name, |
| "BOARD=%s" % ti.platform.name]) |
| if (ti.platform.qemu_support and (not ti.build_only) and |
| (not build_only) and (enable_slow or not ti.test.slow)): |
| self.add_qemu_goal(ti.name, ti.test.code_location, ti.outdir, |
| args, ti.test.timeout) |
| else: |
| self.add_build_goal(ti.name, ti.test.code_location, ti.outdir, args) |
| |
| def execute(self, callback_fn=None, context=None): |
| """Execute all the registered build goals |
| |
| @param callback_fn If not None, a callback function will be called |
| as individual goals transition between states. This function |
| should accept two parameters: a string state and an arbitrary |
| context object, supplied here |
| @param context Context object to pass to the callback function. |
| Type and semantics are specific to that callback function. |
| @return A dictionary mapping goal names to final status. |
| """ |
| |
| with open(self.makefile, "wt") as tf, \ |
| open(os.devnull, "wb") as devnull, \ |
| open(self.logfile, "wt") as make_log: |
| # Create our dynamic Makefile and execute it. |
| # Watch stderr output which is where we will keep |
| # track of build state |
| for name, goal in self.goals.items(): |
| tf.write(goal.text) |
| tf.write("all: %s\n" % (" ".join(self.goals.keys()))) |
| tf.flush() |
| |
| # os.environ["CC"] = "ccache gcc" FIXME doesn't work |
| |
| cmd = ["make", "-k", "-j", str(CPU_COUNTS * 2), "-f", tf.name, "all"] |
| p = subprocess.Popen(cmd, stderr=subprocess.PIPE, |
| stdout=devnull) |
| |
| for line in iter(p.stderr.readline, b''): |
| line = line.decode("utf-8") |
| make_log.write(line) |
| verbose("MAKE: " + repr(line.strip())) |
| m = MakeGenerator.re_make.match(line) |
| if not m: |
| continue |
| |
| state, name, error = m.groups() |
| if error: |
| goal = self.goals[error] |
| else: |
| goal = self.goals[name] |
| goal.make_state = state |
| |
| |
| if error: |
| goal.fail("build_error") |
| else: |
| if state == "finished": |
| if goal.qemu: |
| thread_status, metrics = goal.qemu.get_state() |
| goal.metrics.update(metrics) |
| if thread_status == "passed": |
| goal.success() |
| else: |
| goal.fail(thread_status) |
| else: |
| goal.success() |
| |
| if callback_fn: |
| callback_fn(context, self.goals, goal) |
| |
| p.wait() |
| return self.goals |
| |
| |
| # "list" - List of strings |
| # "list:<type>" - List of <type> |
| # "set" - Set of unordered, unique strings |
| # "set:<type>" - Set of <type> |
| # "float" - Floating point |
| # "int" - Integer |
| # "bool" - Boolean |
| # "str" - String |
| |
| # XXX Be sure to update __doc__ if you change any of this!! |
| |
| arch_valid_keys = {"name" : {"type" : "str", "required" : True}, |
| "platforms" : {"type" : "list", "required" : True}} |
| |
| platform_valid_keys = {"qemu_support" : {"type" : "bool", "default" : False}, |
| "microkernel_support" : {"type" : "bool", |
| "default" : True}} |
| |
| testcase_valid_keys = {"tags" : {"type" : "set", "required" : True}, |
| "extra_args" : {"type" : "list"}, |
| "build_only" : {"type" : "bool", "default" : False}, |
| "skip" : {"type" : "bool", "default" : False}, |
| "slow" : {"type" : "bool", "default" : False}, |
| "timeout" : {"type" : "int", "default" : 60}, |
| "kernel" : {"type": "str", "required" : False}, |
| "arch_whitelist" : {"type" : "set"}, |
| "arch_exclude" : {"type" : "set"}, |
| "platform_exclude" : {"type" : "set"}, |
| "platform_whitelist" : {"type" : "set"}, |
| "config_whitelist" : {"type" : "set"}} |
| |
| |
| class SanityConfigParser: |
| """Class to read architecture and test case .ini files with semantic checking |
| """ |
| def __init__(self, filename): |
| """Instantiate a new SanityConfigParser object |
| |
| @param filename Source .ini file to read |
| """ |
| cp = configparser.SafeConfigParser() |
| cp.readfp(open(filename)) |
| self.filename = filename |
| self.cp = cp |
| |
| def _cast_value(self, value, typestr): |
| v = value.strip() |
| if typestr == "str": |
| return v |
| |
| elif typestr == "float": |
| return float(v) |
| |
| elif typestr == "int": |
| return int(v) |
| |
| elif typestr == "bool": |
| v = v.lower() |
| if v == "true" or v == "1": |
| return True |
| elif v == "" or v == "false" or v == "0": |
| return False |
| raise ConfigurationError(self.filename, |
| "bad value for boolean: '%s'" % value) |
| |
| elif typestr.startswith("list"): |
| vs = v.split() |
| if len(typestr) > 4 and typestr[4] == ":": |
| return [self._cast_value(vsi, typestr[5:]) for vsi in vs] |
| else: |
| return vs |
| |
| elif typestr.startswith("set"): |
| vs = v.split() |
| if len(typestr) > 3 and typestr[3] == ":": |
| return set([self._cast_value(vsi, typestr[4:]) for vsi in vs]) |
| else: |
| return set(vs) |
| |
| else: |
| raise ConfigurationError(self.filename, "unknown type '%s'" % value) |
| |
| |
| def sections(self): |
| """Get the set of sections within the .ini file |
| |
| @return a list of string section names""" |
| return self.cp.sections() |
| |
| def get_section(self, section, valid_keys): |
| """Get a dictionary representing the keys/values within a section |
| |
| @param section The section in the .ini file to retrieve data from |
| @param valid_keys A dictionary representing the intended semantics |
| for this section. Each key in this dictionary is a key that could |
| be specified, if a key is given in the .ini file which isn't in |
| here, it will generate an error. Each value in this dictionary |
| is another dictionary containing metadata: |
| |
| "default" - Default value if not given |
| "type" - Data type to convert the text value to. Simple types |
| supported are "str", "float", "int", "bool" which will get |
| converted to respective Python data types. "set" and "list" |
| may also be specified which will split the value by |
| whitespace (but keep the elements as strings). finally, |
| "list:<type>" and "set:<type>" may be given which will |
| perform a type conversion after splitting the value up. |
| "required" - If true, raise an error if not defined. If false |
| and "default" isn't specified, a type conversion will be |
| done on an empty string |
| @return A dictionary containing the section key-value pairs with |
| type conversion and default values filled in per valid_keys |
| """ |
| |
| d = {} |
| cp = self.cp |
| |
| if not cp.has_section(section): |
| # Just fill it with defaults |
| cp.add_section(section) |
| |
| for k, v in cp.items(section): |
| if k not in valid_keys: |
| raise ConfigurationError(self.filename, |
| "Unknown config key '%s' in defintiion for '%s'" |
| % (k, section)) |
| d[k] = v |
| |
| for k, kinfo in valid_keys.items(): |
| if k not in d: |
| if "required" in kinfo: |
| required = kinfo["required"] |
| else: |
| required = False |
| |
| if required: |
| raise ConfigurationError(self.filename, |
| "missing required value for '%s' in section '%s'" |
| % (k, section)) |
| else: |
| if "default" in kinfo: |
| default = kinfo["default"] |
| else: |
| default = self._cast_value("", kinfo["type"]) |
| d[k] = default |
| else: |
| try: |
| d[k] = self._cast_value(d[k], kinfo["type"]) |
| except ValueError as ve: |
| raise ConfigurationError(self.filename, |
| "bad %s value '%s' for key '%s' in section '%s'" |
| % (kinfo["type"], d[k], k, section)) |
| |
| return d |
| |
| |
| class Platform: |
| """Class representing metadata for a particular platform |
| |
| Maps directly to BOARD when building""" |
| def __init__(self, arch, name, plat_dict): |
| """Constructor. |
| |
| @param arch Architecture object for this platform |
| @param name String name for this platform, same as BOARD |
| @param plat_dict SanityConfigParser output on the relevant section |
| in the architecture configuration file which has lots of metadata. |
| See the Architecture class. |
| """ |
| self.name = name |
| self.qemu_support = plat_dict["qemu_support"] |
| self.microkernel_support = plat_dict["microkernel_support"] |
| self.arch = arch |
| # Gets populated in a separate step |
| self.defconfig = {"micro" : None, "nano" : None} |
| pass |
| |
| def set_defconfig(self, ktype, defconfig): |
| """Set defconfig information for a particular kernel type. |
| |
| We do this in another step because all the defconfigs are generated |
| at once from a sub-make, see TestSuite constructor |
| |
| @param ktype Kernel type, either "micro" or "nano" |
| @param defconfig Dictionary containing defconfig information |
| """ |
| self.defconfig[ktype] = defconfig |
| |
| def get_defconfig(self, ktype): |
| """Return a dictionary representing the key/value pairs expressed |
| in the kernel defconfig used for this arch/platform. Used to identify |
| platform features. |
| |
| @param ktype Kernel type, either "micro" or "nano" |
| @return dictionary corresponding to the defconfig contents. unset |
| values will not be defined |
| """ |
| |
| if ktype == "micro" and not self.microkernel_support: |
| raise SanityRuntimeError("Invalid kernel type queried") |
| |
| return self.defconfig[ktype] |
| |
| def __repr__(self): |
| return "<%s on %s>" % (self.name, self.arch.name) |
| |
| |
| class Architecture: |
| """Class representing metadata for a particular architecture |
| """ |
| def __init__(self, cfile): |
| """Architecture constructor |
| |
| @param cfile Path to Architecture configuration file, which gives |
| info about the arch and all the platforms for it |
| """ |
| cp = SanityConfigParser(cfile) |
| self.platforms = [] |
| |
| arch = cp.get_section("arch", arch_valid_keys) |
| |
| self.name = arch["name"] |
| |
| for plat_name in arch["platforms"]: |
| verbose("Platform: %s" % plat_name) |
| plat_dict = cp.get_section(plat_name, platform_valid_keys) |
| self.platforms.append(Platform(self, plat_name, plat_dict)) |
| |
| def __repr__(self): |
| return "<arch %s>" % self.name |
| |
| |
| class TestCase: |
| """Class representing a test application |
| """ |
| makefile_re = re.compile("\s*KERNEL_TYPE\s*[?=]+\s*(micro|nano)\s*") |
| |
| def __init__(self, testcase_root, workdir, name, tc_dict): |
| """TestCase constructor. |
| |
| This gets called by TestSuite as it finds and reads testcase.ini files. |
| Multiple TestCase instances may be generated from a single testcase.ini, |
| each one corresponds to a section within that file. |
| |
| Reads the Makefile inside the testcase directory to figure out the |
| kernel type for purposes of configuration filtering |
| |
| We need to have a unique name for every single test case. Since |
| a testcase.ini can define multiple tests, the canonical name for |
| the test case is <workdir>/<name>. |
| |
| @param testcase_root Absolute path to the root directory where |
| all the test cases live |
| @param workdir Relative path to the project directory for this |
| test application from the test_case root. |
| @param name Name of this test case, corresponding to the section name |
| in the test case configuration file. For many test cases that just |
| define one test, can be anything and is usually "test". This is |
| really only used to distinguish between different cases when |
| the testcase.ini defines multiple tests |
| @param tc_dict Dictionary with section values for this test case |
| from the testcase.ini file |
| """ |
| self.code_location = os.path.join(testcase_root, workdir) |
| self.tags = tc_dict["tags"] |
| self.extra_args = tc_dict["extra_args"] |
| self.arch_whitelist = tc_dict["arch_whitelist"] |
| self.arch_exclude = tc_dict["arch_exclude"] |
| self.skip = tc_dict["skip"] |
| self.kernel = tc_dict["kernel"] |
| self.platform_exclude = tc_dict["platform_exclude"] |
| self.platform_whitelist = tc_dict["platform_whitelist"] |
| self.config_whitelist = tc_dict["config_whitelist"] |
| self.timeout = tc_dict["timeout"] |
| self.build_only = tc_dict["build_only"] |
| self.slow = tc_dict["slow"] |
| self.path = os.path.join(os.path.basename(os.path.abspath(testcase_root)), |
| workdir, name) |
| self.name = self.path # for now |
| self.defconfig = {} |
| self.ktype = None |
| |
| if self.kernel: |
| self.ktype = self.kernel |
| else: |
| with open(os.path.join(testcase_root, workdir, "Makefile")) as makefile: |
| for line in makefile.readlines(): |
| m = TestCase.makefile_re.match(line) |
| if m: |
| self.ktype = m.group(1) |
| break |
| if not self.ktype: |
| raise ConfigurationError(os.path.join(workdir, "Makefile"), |
| "KERNEL_TYPE not found") |
| |
| def __repr__(self): |
| return self.name |
| |
| |
| |
| class TestInstance: |
| """Class representing the execution of a particular TestCase on a platform |
| |
| @param test The TestCase object we want to build/execute |
| @param platform Platform object that we want to build and run against |
| @param base_outdir Base directory for all test results. The actual |
| out directory used is <outdir>/<platform>/<test case name> |
| """ |
| def __init__(self, test, platform, base_outdir, build_only=False, |
| slow=False): |
| self.test = test |
| self.platform = platform |
| self.name = os.path.join(platform.name, test.path) |
| self.outdir = os.path.join(base_outdir, platform.name, test.path) |
| self.build_only = build_only or test.build_only |
| |
| def calculate_sizes(self): |
| """Get the RAM/ROM sizes of a test case. |
| |
| This can only be run after the instance has been executed by |
| MakeGenerator, otherwise there won't be any binaries to measure. |
| |
| @return A SizeCalculator object |
| """ |
| fns = glob.glob(os.path.join(self.outdir, "*.elf")) |
| if (len(fns) != 1): |
| raise BuildError("Missing/multiple output ELF binary") |
| |
| return SizeCalculator(fns[0]) |
| |
| def __repr__(self): |
| return "<TestCase %s on %s>" % (self.test.name, self.platform.name) |
| |
| |
| def defconfig_cb(context, goals, goal): |
| if not goal.failed: |
| return |
| |
| info("%sCould not build defconfig for %s%s" % |
| (COLOR_RED, goal.name, COLOR_NORMAL)); |
| if INLINE_LOGS: |
| with open(goal.get_error_log()) as fp: |
| sys.stdout.write(fp.read()) |
| else: |
| info("\tsee: " + COLOR_YELLOW + goal.get_error_log() + COLOR_NORMAL) |
| |
| |
| class TestSuite: |
| config_re = re.compile('(CONFIG_[A-Z0-9_]+)[=](.+)$') |
| |
| def __init__(self, arch_root, testcase_roots, outdir): |
| # Keep track of which test cases we've filtered out and why |
| discards = {} |
| self.arches = {} |
| self.testcases = {} |
| self.platforms = [] |
| self.outdir = os.path.abspath(outdir) |
| self.instances = {} |
| self.goals = None |
| self.discards = None |
| |
| arch_root = os.path.abspath(arch_root) |
| |
| for testcase_root in testcase_roots: |
| testcase_root = os.path.abspath(testcase_root) |
| |
| debug("Reading test case configuration files under %s..." % |
| testcase_root) |
| for dirpath, dirnames, filenames in os.walk(testcase_root, |
| topdown=True): |
| verbose("scanning %s" % dirpath) |
| if "testcase.ini" in filenames: |
| verbose("Found test case in " + dirpath) |
| dirnames[:] = [] |
| cp = SanityConfigParser(os.path.join(dirpath, "testcase.ini")) |
| workdir = os.path.relpath(dirpath, testcase_root) |
| |
| for section in cp.sections(): |
| tc_dict = cp.get_section(section, testcase_valid_keys) |
| tc = TestCase(testcase_root, workdir, section, tc_dict) |
| self.testcases[tc.name] = tc |
| |
| debug("Reading architecture configuration files under %s..." % arch_root) |
| for dirpath, dirnames, filenames in os.walk(arch_root): |
| for filename in filenames: |
| if filename.endswith(".ini"): |
| fn = os.path.join(dirpath, filename) |
| verbose("Found arch configuration " + fn) |
| arch = Architecture(fn) |
| self.arches[arch.name] = arch |
| self.platforms.extend(arch.platforms) |
| |
| |
| self.instances = {} |
| |
| def get_last_failed(self): |
| if not os.path.exists(LAST_SANITY): |
| return [] |
| result = [] |
| with open(LAST_SANITY, "r") as fp: |
| cr = csv.DictReader(fp) |
| for row in cr: |
| if row["passed"] == "True": |
| continue |
| test = row["test"] |
| platform = row["platform"] |
| result.append((test, platform)) |
| return result |
| |
| def apply_filters(self, platform_filter, arch_filter, tag_filter, |
| config_filter, testcase_filter, last_failed, all_plats, |
| platform_limit): |
| instances = [] |
| discards = {} |
| verbose("platform filter: " + str(platform_filter)) |
| verbose(" arch_filter: " + str(arch_filter)) |
| verbose(" tag_filter: " + str(tag_filter)) |
| verbose(" config_filter: " + str(config_filter)) |
| |
| if last_failed: |
| failed_tests = self.get_last_failed() |
| |
| default_platforms = False |
| |
| if all_plats: |
| info("Selecting all possible platforms per test case") |
| # When --all used, any --platform arguments ignored |
| platform_filter = [] |
| elif not platform_filter: |
| info("Selecting default platforms per test case") |
| default_platforms = True |
| |
| mg = MakeGenerator(self.outdir) |
| dlist = {} |
| for tc_name, tc in self.testcases.items(): |
| for arch_name, arch in self.arches.items(): |
| instance_list = [] |
| for plat in arch.platforms: |
| instance = TestInstance(tc, plat, self.outdir) |
| |
| if tc.skip: |
| continue |
| |
| if tag_filter and not tc.tags.intersection(tag_filter): |
| continue |
| |
| if testcase_filter and tc_name not in testcase_filter: |
| continue |
| |
| if last_failed and (tc.name, plat.name) not in failed_tests: |
| continue |
| |
| if arch_filter and arch_name not in arch_filter: |
| continue |
| |
| if tc.arch_whitelist and arch.name not in tc.arch_whitelist: |
| continue |
| |
| if tc.arch_exclude and arch.name in tc.arch_exclude: |
| continue |
| |
| if tc.platform_exclude and plat.name in tc.platform_exclude: |
| continue |
| |
| if platform_filter and plat.name not in platform_filter: |
| continue |
| |
| if tc.platform_whitelist and plat.name not in tc.platform_whitelist: |
| continue |
| |
| if not plat.microkernel_support and tc.ktype == "micro": |
| continue |
| |
| for cw in tc.config_whitelist: |
| args = tc.extra_args[:] |
| args.extend(["ARCH=" + plat.arch.name, |
| "BOARD=" + plat.name, "initconfig"]) |
| # FIXME would be nice to use a common outdir for this so that |
| # conf, gen_idt, etc aren't rebuilt for every plat/ktype combo, |
| # need a way to avoid different Make processe from clobbering |
| # each other since they all try to build them simultaneously |
| |
| o = os.path.join(self.outdir, plat.name, tc.path) |
| dlist[tc, plat, tc.ktype, tc.name.split("/")[-1]] = os.path.join(o,".config") |
| goal = "_".join([plat.name, tc.ktype, "_".join(tc.name.split("/")), "initconfig"]) |
| mg.add_build_goal(goal, os.path.join(ZEPHYR_BASE, tc.code_location), o, args) |
| |
| info("Building testcase defconfigs...") |
| results = mg.execute(defconfig_cb) |
| |
| for name, goal in results.items(): |
| if goal.failed: |
| raise SanityRuntimeError("Couldn't build some defconfigs") |
| |
| for k, out_config in dlist.items(): |
| test, plat, ktype, name = k |
| defconfig = {} |
| with open(out_config, "r") as fp: |
| for line in fp.readlines(): |
| m = TestSuite.config_re.match(line) |
| if not m: |
| continue |
| defconfig[m.group(1)] = m.group(2).strip() |
| test.defconfig[plat,ktype] = defconfig |
| |
| for tc_name, tc in self.testcases.items(): |
| for arch_name, arch in self.arches.items(): |
| instance_list = [] |
| for plat in arch.platforms: |
| instance = TestInstance(tc, plat, self.outdir) |
| |
| if tc.skip: |
| discards[instance] = "Skip filter" |
| continue |
| |
| if tag_filter and not tc.tags.intersection(tag_filter): |
| discards[instance] = "Command line testcase tag filter" |
| continue |
| |
| if testcase_filter and tc_name not in testcase_filter: |
| discards[instance] = "Testcase name filter" |
| continue |
| |
| if last_failed and (tc.name, plat.name) not in failed_tests: |
| discards[instance] = "Passed or skipped during last run" |
| continue |
| |
| if arch_filter and arch_name not in arch_filter: |
| discards[instance] = "Command line testcase arch filter" |
| continue |
| |
| if tc.arch_whitelist and arch.name not in tc.arch_whitelist: |
| discards[instance] = "Not in test case arch whitelist" |
| continue |
| |
| if tc.arch_exclude and arch.name in tc.arch_exclude: |
| discards[instance] = "In test case arch exclude" |
| continue |
| |
| if tc.platform_exclude and plat.name in tc.platform_exclude: |
| discards[instance] = "In test case platform exclude" |
| continue |
| |
| if platform_filter and plat.name not in platform_filter: |
| discards[instance] = "Command line platform filter" |
| continue |
| |
| if tc.platform_whitelist and plat.name not in tc.platform_whitelist: |
| discards[instance] = "Not in testcase platform whitelist" |
| continue |
| |
| if not plat.microkernel_support and tc.ktype == "micro": |
| discards[instance] = "No microkernel support for platform" |
| continue |
| |
| defconfig = {} |
| for tcase, tdefconfig in tc.defconfig.items(): |
| p, k = tcase |
| if k == tc.ktype and p == plat: |
| defconfig = tdefconfig |
| break |
| |
| config_pass = True |
| |
| # FIXME this is kind of gross clean it up |
| for cw in tc.config_whitelist: |
| invert = (cw[0] == "!") |
| if invert: |
| cw = cw[1:] |
| |
| if "=" in cw: |
| k, v = cw.split("=") |
| testval = k not in defconfig or defconfig[k] != v |
| if invert: |
| testval = not testval |
| if testval: |
| discards[instance] = "%s%s in platform defconfig" % ( |
| cw, " not" if not invert else "") |
| config_pass = False |
| break |
| else: |
| testval = cw not in defconfig |
| if invert: |
| testval = not testval |
| if testval: |
| discards[instance] = "%s%s set in platform defconfig" % ( |
| cw, " not" if not invert else "") |
| config_pass = False |
| break |
| |
| if not config_pass: |
| continue |
| |
| |
| instance_list.append(instance) |
| |
| if not instance_list: |
| # Every platform in this arch was rejected already |
| continue |
| |
| if default_platforms: |
| self.add_instances(instance_list[:platform_limit]) |
| for instance in instance_list[platform_limit:]: |
| discards[instance] = "Not in first %d platform(s) for arch" % platform_limit |
| else: |
| self.add_instances(instance_list) |
| self.discards = discards |
| return discards |
| |
| def add_instances(self, ti_list): |
| for ti in ti_list: |
| self.instances[ti.name] = ti |
| |
| def execute(self, cb, cb_context, build_only, enable_slow): |
| |
| def calc_one_elf_size(name, goal): |
| if not goal.failed: |
| i = self.instances[name] |
| sc = i.calculate_sizes() |
| goal.metrics["ram_size"] = sc.get_ram_size() |
| goal.metrics["rom_size"] = sc.get_rom_size() |
| goal.metrics["unrecognized"] = sc.unrecognized_sections() |
| goal.metrics["mismatched"] = sc.mismatched_sections() |
| |
| mg = MakeGenerator(self.outdir) |
| for i in self.instances.values(): |
| mg.add_test_instance(i, build_only, enable_slow) |
| self.goals = mg.execute(cb, cb_context) |
| |
| # Parallelize size calculation |
| executor = concurrent.futures.ThreadPoolExecutor(CPU_COUNTS) |
| futures = [executor.submit(calc_one_elf_size, name, goal) \ |
| for name, goal in self.goals.items()] |
| concurrent.futures.wait(futures) |
| |
| return self.goals |
| |
| def discard_report(self, filename): |
| if self.discards == None: |
| raise SanityRuntimeException("apply_filters() hasn't been run!") |
| |
| with open(filename, "wb") as csvfile: |
| fieldnames = ["test", "arch", "platform", "reason"] |
| cw = csv.DictWriter(csvfile, fieldnames, lineterminator=os.linesep) |
| cw.writeheader() |
| for instance, reason in self.discards.items(): |
| rowdict = {"test" : i.test.name, |
| "arch" : i.platform.arch.name, |
| "platform" : i.platform.name, |
| "reason" : reason} |
| cw.writerow(rowdict) |
| |
| def compare_metrics(self, filename): |
| # name, datatype, lower results better |
| interesting_metrics = [("ram_size", int, True), |
| ("rom_size", int, True)] |
| |
| if self.goals == None: |
| raise SanityRuntimeException("execute() hasn't been run!") |
| |
| if not os.path.exists(filename): |
| info("Cannot compare metrics, %s not found" % filename) |
| return [] |
| |
| results = [] |
| saved_metrics = {} |
| with open(filename) as fp: |
| cr = csv.DictReader(fp) |
| for row in cr: |
| d = {} |
| for m, _, _ in interesting_metrics: |
| d[m] = row[m] |
| saved_metrics[(row["test"], row["platform"])] = d |
| |
| for name, goal in self.goals.items(): |
| i = self.instances[name] |
| mkey = (i.test.name, i.platform.name) |
| if mkey not in saved_metrics: |
| continue |
| sm = saved_metrics[mkey] |
| for metric, mtype, lower_better in interesting_metrics: |
| if metric not in goal.metrics: |
| continue |
| if sm[metric] == "": |
| continue |
| delta = goal.metrics[metric] - mtype(sm[metric]) |
| if delta == 0: |
| continue |
| results.append((i, metric, goal.metrics[metric], delta, |
| lower_better)) |
| return results |
| |
| def testcase_report(self, filename): |
| if self.goals == None: |
| raise SanityRuntimeException("execute() hasn't been run!") |
| |
| with open(filename, "wt") as csvfile: |
| fieldnames = ["test", "arch", "platform", "passed", "status", |
| "extra_args", "qemu", "qemu_time", "ram_size", |
| "rom_size"] |
| cw = csv.DictWriter(csvfile, fieldnames, lineterminator=os.linesep) |
| cw.writeheader() |
| for name, goal in self.goals.items(): |
| i = self.instances[name] |
| rowdict = {"test" : i.test.name, |
| "arch" : i.platform.arch.name, |
| "platform" : i.platform.name, |
| "extra_args" : " ".join(i.test.extra_args), |
| "qemu" : i.platform.qemu_support} |
| if goal.failed: |
| rowdict["passed"] = False |
| rowdict["status"] = goal.reason |
| else: |
| rowdict["passed"] = True |
| if goal.qemu: |
| rowdict["qemu_time"] = goal.metrics["qemu_time"] |
| rowdict["ram_size"] = goal.metrics["ram_size"] |
| rowdict["rom_size"] = goal.metrics["rom_size"] |
| cw.writerow(rowdict) |
| |
| |
| def parse_arguments(): |
| |
| parser = argparse.ArgumentParser(description = __doc__, |
| formatter_class = argparse.RawDescriptionHelpFormatter) |
| |
| parser.add_argument("-p", "--platform", action="append", |
| help="Platform filter for testing. This option may be used multiple " |
| "times. Testcases will only be built/run on the platforms " |
| "specified. If this option is not used, then N platforms will " |
| "automatically be chosen from each arch to build and test, " |
| "where N is provided by the --platform-limit option.") |
| parser.add_argument("-L", "--platform-limit", action="store", type=int, |
| metavar="N", default=1, |
| help="Controls what platforms are tested if --platform or " |
| "--all are not used. For each architecture specified by " |
| "--arch (defaults to all of them), choose the first " |
| "N platforms to test in the arch-specific .ini file " |
| "'platforms' list. Defaults to 1.") |
| parser.add_argument("-a", "--arch", action="append", |
| help="Arch filter for testing. Takes precedence over --platform. " |
| "If unspecified, test all arches. Multiple invocations " |
| "are treated as a logical 'or' relationship") |
| parser.add_argument("-t", "--tag", action="append", |
| help="Specify tags to restrict which tests to run by tag value. " |
| "Default is to not do any tag filtering. Multiple invocations " |
| "are treated as a logical 'or' relationship") |
| parser.add_argument("-f", "--only-failed", action="store_true", |
| help="Run only those tests that failed the previous sanity check " |
| "invocation.") |
| parser.add_argument("-c", "--config", action="append", |
| help="Specify platform configuration values filtering. This can be " |
| "specified two ways: <config>=<value> or just <config>. The " |
| "defconfig for all platforms, for all kernel types will be " |
| "checked. For the <config>=<value> case, only match defconfig " |
| "that have that value defined. For the <config> case, match " |
| "defconfig that have that value assigned to any value. " |
| "Prepend a '!' to invert the match.") |
| parser.add_argument("-s", "--test", action="append", |
| help="Run only the specified test cases. These are named by " |
| "<path to test project relative to " |
| "--testcase-root>/<testcase.ini section name>") |
| parser.add_argument("-l", "--all", action="store_true", |
| help="Build/test on all platforms. Any --platform arguments " |
| "ignored.") |
| |
| parser.add_argument("-o", "--testcase-report", |
| help="Output a CSV spreadsheet containing results of the test run") |
| parser.add_argument("-d", "--discard-report", |
| help="Output a CSV spreadhseet showing tests that were skipped " |
| "and why") |
| parser.add_argument("--compare-report", |
| help="Use this report file for size comparision") |
| |
| parser.add_argument("-y", "--dry-run", action="store_true", |
| help="Create the filtered list of test cases, but don't actually " |
| "run them. Useful if you're just interested in " |
| "--discard-report") |
| |
| parser.add_argument("-r", "--release", action="store_true", |
| help="Update the benchmark database with the results of this test " |
| "run. Intended to be run by CI when tagging an official " |
| "release. This database is used as a basis for comparison " |
| "when looking for deltas in metrics such as footprint") |
| parser.add_argument("-w", "--warnings-as-errors", action="store_true", |
| help="Treat warning conditions as errors") |
| parser.add_argument("-v", "--verbose", action="count", default=0, |
| help="Emit debugging information, call multiple times to increase " |
| "verbosity") |
| parser.add_argument("-i", "--inline-logs", action="store_true", |
| help="Upon test failure, print relevant log data to stdout " |
| "instead of just a path to it") |
| parser.add_argument("-m", "--last-metrics", action="store_true", |
| help="Instead of comparing metrics from the last --release, " |
| "compare with the results of the previous sanity check " |
| "invocation") |
| parser.add_argument("-u", "--no-update", action="store_true", |
| help="do not update the results of the last run of the sanity " |
| "checks") |
| parser.add_argument("-b", "--build-only", action="store_true", |
| help="Only build the code, do not execute any of it in QEMU") |
| parser.add_argument("-j", "--jobs", type=int, |
| help="Number of cores to use when building, defaults to " |
| "number of CPUs * 2") |
| parser.add_argument("-H", "--footprint-threshold", type=float, default=5, |
| help="When checking test case footprint sizes, warn the user if " |
| "the new app size is greater then the specified percentage " |
| "from the last release. Default is 5. 0 to warn on any " |
| "increase on app size") |
| parser.add_argument("-D", "--all-deltas", action="store_true", |
| help="Show all footprint deltas, positive or negative. Implies " |
| "--footprint-threshold=0") |
| parser.add_argument("-O", "--outdir", |
| default="%s/sanity-out" % ZEPHYR_BASE, |
| help="Output directory for logs and binaries.") |
| parser.add_argument("-n", "--no-clean", action="store_true", |
| help="Do not delete the outdir before building. Will result in " |
| "faster compilation since builds will be incremental") |
| parser.add_argument("-T", "--testcase-root", action="append", default=[], |
| help="Base directory to recursively search for test cases. All " |
| "testcase.ini files under here will be processed. May be " |
| "called multiple times. Defaults to the 'samples' and " |
| "'tests' directories in the Zephyr tree.") |
| parser.add_argument("-A", "--arch-root", |
| default="%s/scripts/sanity_chk/arches" % ZEPHYR_BASE, |
| help="Directory to search for arch configuration files. All .ini " |
| "files in the directory will be processed.") |
| parser.add_argument("-z", "--size", action="append", |
| help="Don't run sanity checks. Instead, produce a report to " |
| "stdout detailing RAM/ROM sizes on the specified filenames. " |
| "All other command line arguments ignored.") |
| parser.add_argument("-S", "--enable-slow", action="store_true", |
| help="Execute time-consuming test cases that have been marked " |
| "as 'slow' in testcase.ini. Normally these are only built.") |
| |
| return parser.parse_args() |
| |
| def log_info(filename): |
| filename = os.path.relpath(filename) |
| if INLINE_LOGS: |
| info("{:-^100}".format(filename)) |
| with open(filename) as fp: |
| sys.stdout.write(fp.read()) |
| info("{:-^100}".format(filename)) |
| else: |
| info("\tsee: " + COLOR_YELLOW + filename + COLOR_NORMAL) |
| |
| def terse_test_cb(instances, goals, goal): |
| total_tests = len(goals) |
| total_done = 0 |
| total_failed = 0 |
| |
| for k, g in goals.items(): |
| if g.finished: |
| total_done += 1 |
| if g.failed: |
| total_failed += 1 |
| |
| if goal.failed: |
| i = instances[goal.name] |
| info("\n\n{:<25} {:<50} {}FAILED{}: {}".format(i.platform.name, |
| i.test.name, COLOR_RED, COLOR_NORMAL, goal.reason)) |
| log_info(goal.get_error_log()) |
| info("") |
| |
| sys.stdout.write("\rtotal complete: %s%3d/%3d%s failed: %s%3d%s" % ( |
| COLOR_GREEN, total_done, total_tests, COLOR_NORMAL, |
| COLOR_RED if total_failed > 0 else COLOR_NORMAL, |
| total_failed, COLOR_NORMAL)) |
| sys.stdout.flush() |
| |
| def chatty_test_cb(instances, goals, goal): |
| i = instances[goal.name] |
| |
| if VERBOSE < 2 and not goal.finished: |
| return |
| |
| if goal.failed: |
| status = COLOR_RED + "FAILED" + COLOR_NORMAL + ": " + goal.reason |
| elif goal.finished: |
| status = COLOR_GREEN + "PASSED" + COLOR_NORMAL |
| else: |
| status = goal.make_state |
| |
| info("{:<25} {:<50} {}".format(i.platform.name, i.test.name, status)) |
| if goal.failed: |
| log_info(goal.get_error_log()) |
| |
| |
| def size_report(sc): |
| info(sc.filename) |
| info("SECTION NAME VMA LMA SIZE HEX SZ TYPE") |
| for i in range(len(sc.sections)): |
| v = sc.sections[i] |
| |
| info("%-17s 0x%08x 0x%08x %8d 0x%05x %-7s" % |
| (v["name"], v["virt_addr"], v["load_addr"], v["size"], v["size"], |
| v["type"])) |
| if v["lma_off"] != v["vma_off"]: |
| info(" WARNING: LMA and VMA offsets between %s and %s differ: %d vs %d" % |
| (sc.sections[i-1]["name"], v["name"], v["lma_off"], |
| v["vma_off"])) |
| |
| info("Totals: %d bytes (ROM), %d bytes (RAM)" % |
| (sc.rom_size, sc.ram_size)) |
| info("") |
| |
| |
| def main(): |
| start_time = time.time() |
| global VERBOSE, INLINE_LOGS, CPU_COUNTS |
| args = parse_arguments() |
| |
| if args.size: |
| for fn in args.size: |
| size_report(SizeCalculator(fn)) |
| sys.exit(0) |
| |
| VERBOSE += args.verbose |
| INLINE_LOGS = args.inline_logs |
| if args.jobs: |
| CPU_COUNTS = args.jobs |
| |
| if os.path.exists(args.outdir) and not args.no_clean: |
| info("Cleaning output directory " + args.outdir) |
| shutil.rmtree(args.outdir) |
| |
| if not args.testcase_root: |
| args.testcase_root = [os.path.join(ZEPHYR_BASE, "tests"), |
| os.path.join(ZEPHYR_BASE, "samples")] |
| |
| ts = TestSuite(args.arch_root, args.testcase_root, args.outdir) |
| discards = ts.apply_filters(args.platform, args.arch, args.tag, args.config, |
| args.test, args.only_failed, args.all, |
| args.platform_limit) |
| |
| if args.discard_report: |
| ts.discard_report(args.discard_report) |
| |
| if VERBOSE: |
| for i, reason in discards.items(): |
| debug("{:<25} {:<50} {}SKIPPED{}: {}".format(i.platform.name, |
| i.test.name, COLOR_YELLOW, COLOR_NORMAL, reason)) |
| |
| info("%d tests selected, %d tests discarded due to filters" % |
| (len(ts.instances), len(discards))) |
| |
| if args.dry_run: |
| return |
| |
| if VERBOSE or not TERMINAL: |
| goals = ts.execute(chatty_test_cb, ts.instances, args.build_only, |
| args.enable_slow) |
| else: |
| goals = ts.execute(terse_test_cb, ts.instances, args.build_only, |
| args.enable_slow) |
| info("") |
| |
| # figure out which report to use for size comparison |
| if args.compare_report: |
| report_to_use = args.compare_report |
| elif args.last_metrics: |
| report_to_use = LAST_SANITY |
| else: |
| report_to_use = RELEASE_DATA |
| |
| deltas = ts.compare_metrics(report_to_use) |
| warnings = 0 |
| if deltas: |
| for i, metric, value, delta, lower_better in deltas: |
| if not args.all_deltas and ((delta < 0 and lower_better) or |
| (delta > 0 and not lower_better)): |
| continue |
| |
| percentage = (float(delta) / float(value - delta)) |
| if not args.all_deltas and (percentage < |
| (args.footprint_threshold / 100.0)): |
| continue |
| |
| info("{:<25} {:<60} {}{}{}: {} {:<+4}, is now {:6} {:+.2%}".format( |
| i.platform.name, i.test.name, COLOR_YELLOW, |
| "INFO" if args.all_deltas else "WARNING", COLOR_NORMAL, |
| metric, delta, value, percentage)) |
| warnings += 1 |
| |
| if warnings: |
| info("Deltas based on metrics from last %s" % |
| ("release" if not args.last_metrics else "run")) |
| |
| failed = 0 |
| for name, goal in goals.items(): |
| if goal.failed: |
| failed += 1 |
| elif goal.metrics["unrecognized"]: |
| info("%sFAILED%s: %s has unrecognized binary sections: %s" % |
| (COLOR_RED, COLOR_NORMAL, goal.name, |
| str(goal.metrics["unrecognized"]))) |
| failed += 1 |
| elif goal.metrics["mismatched"]: |
| info("%sFAILED%s: %s has mismatched section offsets for: %s" % |
| (COLOR_RED, COLOR_NORMAL, goal.name, |
| str(goal.metrics["mismatched"]))) |
| failed += 1 |
| |
| info("%s%d of %d%s tests passed with %s%d%s warnings in %d seconds" % |
| (COLOR_RED if failed else COLOR_GREEN, len(goals) - failed, |
| len(goals), COLOR_NORMAL, COLOR_YELLOW if warnings else COLOR_NORMAL, |
| warnings, COLOR_NORMAL, time.time() - start_time)) |
| |
| if args.testcase_report: |
| ts.testcase_report(args.testcase_report) |
| if not args.no_update: |
| ts.testcase_report(LAST_SANITY) |
| if args.release: |
| ts.testcase_report(RELEASE_DATA) |
| |
| if failed or (warnings and args.warnings_as_errors): |
| sys.exit(1) |
| |
| if __name__ == "__main__": |
| main() |
